From 17df50a58d5a9bb4f74c08ec63674fae2b6e3bcc Mon Sep 17 00:00:00 2001 From: Ximin Luo Date: Tue, 5 Oct 2021 23:23:53 +0100 Subject: [PATCH] New upstream version 1.54.0+dfsg1 --- Cargo.lock | 287 +- README.md | 24 +- RELEASES.md | 149 +- compiler/rustc_apfloat/src/lib.rs | 1 - compiler/rustc_ast/src/ast.rs | 30 +- compiler/rustc_ast/src/ast_like.rs | 3 +- compiler/rustc_ast/src/lib.rs | 6 +- compiler/rustc_ast/src/mut_visit.rs | 58 +- compiler/rustc_ast/src/tokenstream.rs | 5 +- compiler/rustc_ast/src/visit.rs | 71 +- compiler/rustc_ast_lowering/src/asm.rs | 329 + compiler/rustc_ast_lowering/src/expr.rs | 405 +- compiler/rustc_ast_lowering/src/item.rs | 13 +- compiler/rustc_ast_lowering/src/lib.rs | 101 +- compiler/rustc_ast_lowering/src/pat.rs | 8 +- compiler/rustc_ast_lowering/src/path.rs | 72 +- .../rustc_ast_passes/src/ast_validation.rs | 263 +- compiler/rustc_ast_passes/src/feature_gate.rs | 46 +- compiler/rustc_ast_passes/src/lib.rs | 1 - compiler/rustc_ast_pretty/src/lib.rs | 1 - compiler/rustc_ast_pretty/src/pprust/state.rs | 327 +- compiler/rustc_attr/src/lib.rs | 2 - compiler/rustc_builtin_macros/src/asm.rs | 102 +- .../rustc_builtin_macros/src/global_asm.rs | 68 - compiler/rustc_builtin_macros/src/lib.rs | 10 +- .../rustc_builtin_macros/src/source_util.rs | 4 +- compiler/rustc_builtin_macros/src/test.rs | 4 + .../.github/workflows/main.yml | 7 + .../.vscode/settings.json | 2 +- compiler/rustc_codegen_cranelift/Cargo.lock | 145 +- compiler/rustc_codegen_cranelift/Cargo.toml | 28 +- compiler/rustc_codegen_cranelift/Readme.md | 5 +- .../build_sysroot/Cargo.lock | 27 +- .../build_sysroot/prepare_sysroot_src.sh | 2 +- ...ns-Disable-128bit-atomic-operations.patch} | 0 ...builtins-Remove-rotate_left-from-Int.patch | 35 - .../rustc_codegen_cranelift/docs/env_vars.md | 15 - .../example/mini_core_hello_world.rs | 43 +- .../example/std_example.rs | 19 +- ...022-core-Disable-not-compiling-tests.patch | 40 - .../rustc_codegen_cranelift/rust-toolchain | 2 +- .../scripts/ext_config.sh | 2 +- .../scripts/setup_rust_fork.sh | 17 +- .../scripts/test_rustc_tests.sh | 7 +- .../rustc_codegen_cranelift/scripts/tests.sh | 1 + .../rustc_codegen_cranelift/src/abi/mod.rs | 15 +- .../rustc_codegen_cranelift/src/allocator.rs | 6 +- .../rustc_codegen_cranelift/src/archive.rs | 15 +- .../rustc_codegen_cranelift/src/backend.rs | 20 +- compiler/rustc_codegen_cranelift/src/base.rs | 157 +- .../rustc_codegen_cranelift/src/common.rs | 37 +- .../rustc_codegen_cranelift/src/config.rs | 116 + .../rustc_codegen_cranelift/src/constant.rs | 169 +- .../src/debuginfo/line_info.rs | 4 +- .../src/debuginfo/mod.rs | 2 +- .../src/debuginfo/unwind.rs | 71 +- .../rustc_codegen_cranelift/src/driver/aot.rs | 126 +- .../rustc_codegen_cranelift/src/driver/jit.rs | 215 +- .../rustc_codegen_cranelift/src/driver/mod.rs | 58 +- .../rustc_codegen_cranelift/src/inline_asm.rs | 76 +- .../src/intrinsics/cpuid.rs | 9 +- .../src/intrinsics/llvm.rs | 10 +- .../src/intrinsics/mod.rs | 144 +- .../src/intrinsics/simd.rs | 10 +- compiler/rustc_codegen_cranelift/src/lib.rs | 159 +- .../rustc_codegen_cranelift/src/linkage.rs | 1 + .../rustc_codegen_cranelift/src/main_shim.rs | 77 +- .../rustc_codegen_cranelift/src/metadata.rs | 99 +- compiler/rustc_codegen_cranelift/src/num.rs | 17 +- .../src/optimize/code_layout.rs | 34 - .../src/optimize/mod.rs | 17 +- .../src/optimize/stack2reg.rs | 486 - .../src/pretty_clif.rs | 51 +- .../rustc_codegen_cranelift/src/toolchain.rs | 89 +- compiler/rustc_codegen_cranelift/src/trap.rs | 8 +- .../src/value_and_place.rs | 3 +- .../rustc_codegen_cranelift/src/vtable.rs | 36 +- compiler/rustc_codegen_llvm/Cargo.toml | 1 + compiler/rustc_codegen_llvm/src/abi.rs | 13 +- compiler/rustc_codegen_llvm/src/asm.rs | 59 +- compiler/rustc_codegen_llvm/src/attributes.rs | 31 - .../rustc_codegen_llvm/src/back/archive.rs | 88 +- compiler/rustc_codegen_llvm/src/back/lto.rs | 39 +- compiler/rustc_codegen_llvm/src/back/write.rs | 57 +- compiler/rustc_codegen_llvm/src/base.rs | 20 +- compiler/rustc_codegen_llvm/src/builder.rs | 53 +- compiler/rustc_codegen_llvm/src/callee.rs | 3 +- compiler/rustc_codegen_llvm/src/consts.rs | 10 + compiler/rustc_codegen_llvm/src/context.rs | 3 + .../src/coverageinfo/mapgen.rs | 16 +- .../src/coverageinfo/mod.rs | 3 +- .../src/debuginfo/metadata.rs | 333 +- .../rustc_codegen_llvm/src/debuginfo/mod.rs | 4 +- compiler/rustc_codegen_llvm/src/intrinsic.rs | 3 +- compiler/rustc_codegen_llvm/src/lib.rs | 32 +- compiler/rustc_codegen_llvm/src/llvm/ffi.rs | 87 +- compiler/rustc_codegen_llvm/src/llvm/mod.rs | 44 - compiler/rustc_codegen_llvm/src/llvm_util.rs | 44 +- compiler/rustc_codegen_llvm/src/metadata.rs | 112 - compiler/rustc_codegen_llvm/src/mono_item.rs | 42 +- compiler/rustc_codegen_ssa/Cargo.toml | 9 +- .../rustc_codegen_ssa/src/back/archive.rs | 25 +- compiler/rustc_codegen_ssa/src/back/link.rs | 758 +- compiler/rustc_codegen_ssa/src/back/linker.rs | 331 +- compiler/rustc_codegen_ssa/src/back/lto.rs | 2 +- .../rustc_codegen_ssa/src/back/metadata.rs | 85 + compiler/rustc_codegen_ssa/src/back/mod.rs | 1 + compiler/rustc_codegen_ssa/src/back/rpath.rs | 27 +- .../rustc_codegen_ssa/src/back/rpath/tests.rs | 2 - .../src/back/symbol_export.rs | 60 +- compiler/rustc_codegen_ssa/src/back/write.rs | 61 +- compiler/rustc_codegen_ssa/src/base.rs | 58 +- compiler/rustc_codegen_ssa/src/common.rs | 30 +- .../rustc_codegen_ssa/src/coverageinfo/map.rs | 13 +- .../src/debuginfo/type_names.rs | 59 +- compiler/rustc_codegen_ssa/src/lib.rs | 18 +- compiler/rustc_codegen_ssa/src/mir/analyze.rs | 82 +- compiler/rustc_codegen_ssa/src/mir/block.rs | 206 +- .../rustc_codegen_ssa/src/mir/coverageinfo.rs | 2 +- compiler/rustc_codegen_ssa/src/mir/mod.rs | 143 +- compiler/rustc_codegen_ssa/src/mir/rvalue.rs | 4 - compiler/rustc_codegen_ssa/src/mono_item.rs | 36 +- .../rustc_codegen_ssa/src/target_features.rs | 85 +- compiler/rustc_codegen_ssa/src/traits/asm.rs | 15 +- .../rustc_codegen_ssa/src/traits/backend.rs | 13 +- .../rustc_codegen_ssa/src/traits/builder.rs | 16 +- compiler/rustc_codegen_ssa/src/traits/mod.rs | 2 +- .../rustc_codegen_ssa/src/traits/write.rs | 2 +- compiler/rustc_data_structures/Cargo.toml | 2 +- .../rustc_data_structures/src/box_region.rs | 169 - compiler/rustc_data_structures/src/flock.rs | 13 + compiler/rustc_data_structures/src/lib.rs | 6 +- .../src/obligation_forest/mod.rs | 27 +- .../src/stable_hasher.rs | 32 - compiler/rustc_data_structures/src/sync.rs | 62 +- .../rustc_data_structures/src/tagged_ptr.rs | 10 + compiler/rustc_data_structures/src/vec_map.rs | 155 + .../src/vec_map/tests.rs | 48 + compiler/rustc_driver/src/lib.rs | 233 +- compiler/rustc_driver/src/pretty.rs | 21 +- compiler/rustc_error_codes/src/error_codes.rs | 34 +- .../src/error_codes/E0277.md | 10 +- .../src/error_codes/E0316.md | 32 + .../src/error_codes/E0759.md | 6 +- .../src/error_codes/E0782.md | 26 + .../src/error_codes/E0783.md | 22 + .../src/annotate_snippet_emitter_writer.rs | 3 +- compiler/rustc_errors/src/diagnostic.rs | 18 +- compiler/rustc_errors/src/emitter.rs | 38 +- compiler/rustc_errors/src/json.rs | 6 +- compiler/rustc_errors/src/lib.rs | 41 +- compiler/rustc_expand/src/base.rs | 33 +- compiler/rustc_expand/src/build.rs | 11 +- compiler/rustc_expand/src/config.rs | 54 +- compiler/rustc_expand/src/expand.rs | 152 +- compiler/rustc_expand/src/lib.rs | 3 +- compiler/rustc_expand/src/mbe/macro_parser.rs | 9 +- compiler/rustc_expand/src/mbe/macro_rules.rs | 8 +- compiler/rustc_expand/src/mbe/quoted.rs | 27 +- compiler/rustc_expand/src/module.rs | 23 +- compiler/rustc_expand/src/proc_macro.rs | 10 +- .../rustc_expand/src/proc_macro_server.rs | 112 +- compiler/rustc_feature/src/accepted.rs | 6 + compiler/rustc_feature/src/active.rs | 41 +- compiler/rustc_feature/src/builtin_attrs.rs | 7 +- compiler/rustc_feature/src/removed.rs | 7 + compiler/rustc_hir/src/arena.rs | 1 - compiler/rustc_hir/src/definitions.rs | 23 +- compiler/rustc_hir/src/hir.rs | 56 +- compiler/rustc_hir/src/intravisit.rs | 47 +- compiler/rustc_hir/src/lang_items.rs | 14 +- compiler/rustc_hir/src/lib.rs | 5 +- compiler/rustc_hir_pretty/src/lib.rs | 215 +- compiler/rustc_incremental/Cargo.toml | 1 + .../src/assert_module_sources.rs | 2 +- .../src/persist/dirty_clean.rs | 128 +- compiler/rustc_incremental/src/persist/fs.rs | 66 +- .../rustc_incremental/src/persist/load.rs | 18 +- .../rustc_incremental/src/persist/save.rs | 5 +- compiler/rustc_index/src/bit_set.rs | 10 +- compiler/rustc_index/src/lib.rs | 1 - compiler/rustc_index/src/vec.rs | 7 +- .../src/infer/canonical/query_response.rs | 7 +- compiler/rustc_infer/src/infer/combine.rs | 25 +- compiler/rustc_infer/src/infer/equate.rs | 1 + .../src/infer/error_reporting/mod.rs | 50 +- .../nice_region_error/named_anon_conflict.rs | 16 +- compiler/rustc_infer/src/infer/glb.rs | 3 +- compiler/rustc_infer/src/infer/lub.rs | 3 +- compiler/rustc_infer/src/infer/mod.rs | 3 - .../rustc_infer/src/infer/nll_relate/mod.rs | 34 +- .../src/infer/region_constraints/mod.rs | 58 +- compiler/rustc_infer/src/infer/sub.rs | 1 + compiler/rustc_infer/src/infer/undo_log.rs | 5 +- compiler/rustc_infer/src/lib.rs | 3 +- compiler/rustc_interface/src/lib.rs | 3 - compiler/rustc_interface/src/passes.rs | 185 +- .../rustc_interface/src/proc_macro_decls.rs | 12 +- compiler/rustc_interface/src/queries.rs | 35 +- compiler/rustc_interface/src/tests.rs | 187 +- compiler/rustc_interface/src/util.rs | 60 +- compiler/rustc_lint/src/builtin.rs | 87 +- compiler/rustc_lint/src/context.rs | 39 +- compiler/rustc_lint/src/early.rs | 6 +- compiler/rustc_lint/src/internal.rs | 19 +- compiler/rustc_lint/src/late.rs | 10 +- compiler/rustc_lint/src/levels.rs | 24 +- compiler/rustc_lint/src/lib.rs | 3 - compiler/rustc_lint/src/non_fmt_panic.rs | 18 +- compiler/rustc_lint/src/nonstandard_style.rs | 1 + compiler/rustc_lint/src/types.rs | 65 +- compiler/rustc_lint/src/unused.rs | 4 +- compiler/rustc_lint_defs/src/builtin.rs | 49 +- compiler/rustc_lint_defs/src/lib.rs | 6 +- compiler/rustc_llvm/Cargo.toml | 2 +- compiler/rustc_llvm/build.rs | 1 + .../rustc_llvm/llvm-wrapper/PassWrapper.cpp | 53 +- .../rustc_llvm/llvm-wrapper/RustWrapper.cpp | 87 +- compiler/rustc_llvm/src/lib.rs | 8 + compiler/rustc_macros/src/serialize.rs | 18 +- compiler/rustc_metadata/src/creader.rs | 6 +- compiler/rustc_metadata/src/dynamic_lib.rs | 6 +- compiler/rustc_metadata/src/lib.rs | 5 +- compiler/rustc_metadata/src/native_libs.rs | 194 +- compiler/rustc_metadata/src/rmeta/decoder.rs | 102 +- .../src/rmeta/decoder/cstore_impl.rs | 92 +- compiler/rustc_metadata/src/rmeta/encoder.rs | 74 +- compiler/rustc_metadata/src/rmeta/mod.rs | 10 +- compiler/rustc_middle/src/arena.rs | 3 +- .../rustc_middle/src/dep_graph/dep_node.rs | 2 +- compiler/rustc_middle/src/dep_graph/mod.rs | 1 - .../rustc_middle/src/hir/map/collector.rs | 202 +- compiler/rustc_middle/src/hir/map/mod.rs | 336 +- compiler/rustc_middle/src/hir/mod.rs | 55 +- compiler/rustc_middle/src/ich/impls_syntax.rs | 10 +- compiler/rustc_middle/src/infer/canonical.rs | 1 + compiler/rustc_middle/src/infer/unify_key.rs | 55 +- compiler/rustc_middle/src/lib.rs | 5 +- compiler/rustc_middle/src/lint.rs | 44 +- compiler/rustc_middle/src/middle/cstore.rs | 12 +- .../src/middle/exported_symbols.rs | 2 +- .../src/middle/resolve_lifetime.rs | 14 + compiler/rustc_middle/src/middle/stability.rs | 22 +- .../src/mir/interpret/allocation.rs | 393 +- .../rustc_middle/src/mir/interpret/error.rs | 43 +- .../rustc_middle/src/mir/interpret/mod.rs | 3 +- .../rustc_middle/src/mir/interpret/value.rs | 6 +- compiler/rustc_middle/src/mir/mod.rs | 30 +- compiler/rustc_middle/src/mir/mono.rs | 25 +- compiler/rustc_middle/src/mir/query.rs | 13 +- compiler/rustc_middle/src/query/mod.rs | 161 +- compiler/rustc_middle/src/thir.rs | 747 + compiler/rustc_middle/src/ty/_match.rs | 1 + compiler/rustc_middle/src/ty/adjustment.rs | 10 +- compiler/rustc_middle/src/ty/closure.rs | 26 +- compiler/rustc_middle/src/ty/codec.rs | 4 +- compiler/rustc_middle/src/ty/context.rs | 116 +- compiler/rustc_middle/src/ty/diagnostics.rs | 2 +- compiler/rustc_middle/src/ty/error.rs | 17 +- compiler/rustc_middle/src/ty/flags.rs | 6 +- compiler/rustc_middle/src/ty/generics.rs | 1 + compiler/rustc_middle/src/ty/layout.rs | 84 +- compiler/rustc_middle/src/ty/list.rs | 2 + compiler/rustc_middle/src/ty/mod.rs | 39 +- compiler/rustc_middle/src/ty/print/pretty.rs | 55 +- compiler/rustc_middle/src/ty/query/mod.rs | 8 +- .../src/ty/query/on_disk_cache.rs | 158 +- compiler/rustc_middle/src/ty/relate.rs | 85 +- compiler/rustc_middle/src/ty/sty.rs | 69 +- compiler/rustc_middle/src/ty/subst.rs | 2 +- compiler/rustc_middle/src/ty/trait_def.rs | 11 +- compiler/rustc_middle/src/ty/util.rs | 33 + .../src/borrow_check/constraints/graph.rs | 68 +- .../src/borrow_check/constraints/mod.rs | 31 +- .../diagnostics/conflict_errors.rs | 55 +- .../diagnostics/explain_borrow.rs | 85 +- .../src/borrow_check/diagnostics/mod.rs | 115 +- .../borrow_check/diagnostics/move_errors.rs | 11 +- .../diagnostics/mutability_errors.rs | 118 +- .../borrow_check/diagnostics/region_errors.rs | 21 +- compiler/rustc_mir/src/borrow_check/mod.rs | 18 +- compiler/rustc_mir/src/borrow_check/nll.rs | 9 +- .../src/borrow_check/places_conflict.rs | 19 +- .../src/borrow_check/region_infer/dump_mir.rs | 6 +- .../src/borrow_check/region_infer/graphviz.rs | 12 +- .../src/borrow_check/region_infer/mod.rs | 109 +- .../borrow_check/region_infer/opaque_types.rs | 22 +- .../type_check/constraint_conversion.rs | 1 + .../borrow_check/type_check/liveness/mod.rs | 2 +- .../src/borrow_check/type_check/mod.rs | 56 +- .../src/borrow_check/type_check/relate_tys.rs | 8 +- compiler/rustc_mir/src/const_eval/error.rs | 108 +- .../rustc_mir/src/const_eval/eval_queries.rs | 23 +- compiler/rustc_mir/src/const_eval/machine.rs | 6 +- compiler/rustc_mir/src/const_eval/mod.rs | 2 +- .../src/dataflow/move_paths/builder.rs | 11 +- compiler/rustc_mir/src/interpret/cast.rs | 8 - .../rustc_mir/src/interpret/eval_context.rs | 67 +- compiler/rustc_mir/src/interpret/intern.rs | 1 - .../rustc_mir/src/interpret/intrinsics.rs | 47 +- .../interpret/intrinsics/caller_location.rs | 6 +- .../src/interpret/intrinsics/type_name.rs | 11 +- compiler/rustc_mir/src/interpret/machine.rs | 57 +- compiler/rustc_mir/src/interpret/memory.rs | 456 +- compiler/rustc_mir/src/interpret/mod.rs | 6 +- compiler/rustc_mir/src/interpret/operand.rs | 26 +- compiler/rustc_mir/src/interpret/place.rs | 104 +- compiler/rustc_mir/src/interpret/step.rs | 76 +- .../rustc_mir/src/interpret/terminator.rs | 88 +- compiler/rustc_mir/src/interpret/traits.rs | 75 +- compiler/rustc_mir/src/interpret/validity.rs | 71 +- compiler/rustc_mir/src/lib.rs | 6 +- .../rustc_mir/src/monomorphize/collector.rs | 65 +- .../src/monomorphize/partitioning/default.rs | 12 +- .../src/monomorphize/partitioning/mod.rs | 20 +- .../src/transform/check_consts/ops.rs | 36 +- .../src/transform/check_consts/validation.rs | 45 +- .../rustc_mir/src/transform/check_unsafety.rs | 8 +- .../rustc_mir/src/transform/const_goto.rs | 2 +- .../rustc_mir/src/transform/const_prop.rs | 5 +- .../rustc_mir/src/transform/coverage/debug.rs | 5 +- .../rustc_mir/src/transform/coverage/graph.rs | 4 +- .../rustc_mir/src/transform/coverage/mod.rs | 64 +- .../rustc_mir/src/transform/coverage/query.rs | 4 +- .../rustc_mir/src/transform/coverage/spans.rs | 253 +- .../rustc_mir/src/transform/coverage/tests.rs | 15 +- .../src/transform/deduplicate_blocks.rs | 2 +- compiler/rustc_mir/src/transform/dest_prop.rs | 15 +- .../src/transform/early_otherwise_branch.rs | 7 +- compiler/rustc_mir/src/transform/generator.rs | 4 +- compiler/rustc_mir/src/transform/inline.rs | 2 +- .../rustc_mir/src/transform/instcombine.rs | 21 +- .../src/transform/lower_intrinsics.rs | 16 +- .../rustc_mir/src/transform/match_branches.rs | 2 +- compiler/rustc_mir/src/transform/mod.rs | 9 +- .../transform/multiple_return_terminators.rs | 2 +- .../rustc_mir/src/transform/promote_consts.rs | 199 +- .../src/transform/remove_unneeded_drops.rs | 2 +- .../rustc_mir/src/transform/remove_zsts.rs | 58 +- compiler/rustc_mir/src/transform/simplify.rs | 85 +- .../rustc_mir/src/transform/simplify_try.rs | 7 +- .../src/transform/unreachable_prop.rs | 2 +- compiler/rustc_mir/src/transform/validate.rs | 22 +- compiler/rustc_mir/src/util/generic_graph.rs | 1 - compiler/rustc_mir/src/util/pretty.rs | 27 +- compiler/rustc_mir/src/util/spanview.rs | 35 +- compiler/rustc_mir_build/src/build/block.rs | 35 +- .../src/build/expr/as_constant.rs | 8 +- .../src/build/expr/as_operand.rs | 21 +- .../src/build/expr/as_place.rs | 46 +- .../src/build/expr/as_rvalue.rs | 82 +- .../rustc_mir_build/src/build/expr/as_temp.rs | 8 +- .../src/build/expr/category.rs | 4 +- .../rustc_mir_build/src/build/expr/into.rs | 89 +- .../rustc_mir_build/src/build/expr/stmt.rs | 36 +- .../rustc_mir_build/src/build/matches/mod.rs | 41 +- .../src/build/matches/simplify.rs | 4 +- .../rustc_mir_build/src/build/matches/test.rs | 2 +- .../rustc_mir_build/src/build/matches/util.rs | 2 +- compiler/rustc_mir_build/src/build/mod.rs | 104 +- compiler/rustc_mir_build/src/build/scope.rs | 7 +- .../rustc_mir_build/src/check_unsafety.rs | 425 + compiler/rustc_mir_build/src/lib.rs | 8 +- compiler/rustc_mir_build/src/thir/arena.rs | 98 - compiler/rustc_mir_build/src/thir/constant.rs | 4 +- compiler/rustc_mir_build/src/thir/cx/block.rs | 127 +- compiler/rustc_mir_build/src/thir/cx/expr.rs | 452 +- compiler/rustc_mir_build/src/thir/cx/mod.rs | 39 +- compiler/rustc_mir_build/src/thir/mod.rs | 374 +- .../src/thir/pattern/check_match.rs | 20 +- .../src/thir/pattern/const_to_pat.rs | 3 +- .../src/thir/pattern/deconstruct_pat.rs | 25 +- .../rustc_mir_build/src/thir/pattern/mod.rs | 351 +- .../src/thir/pattern/usefulness.rs | 54 +- compiler/rustc_mir_build/src/thir/visit.rs | 179 + compiler/rustc_parse/src/lexer/mod.rs | 70 +- compiler/rustc_parse/src/lib.rs | 8 +- compiler/rustc_parse/src/parser/attr.rs | 1 - .../rustc_parse/src/parser/attr_wrapper.rs | 10 +- .../rustc_parse/src/parser/diagnostics.rs | 8 +- compiler/rustc_parse/src/parser/expr.rs | 51 +- compiler/rustc_parse/src/parser/item.rs | 109 +- compiler/rustc_parse/src/parser/mod.rs | 53 +- .../rustc_parse/src/parser/nonterminal.rs | 17 +- compiler/rustc_parse/src/parser/pat.rs | 23 +- compiler/rustc_parse/src/parser/path.rs | 84 +- compiler/rustc_parse/src/parser/stmt.rs | 27 +- compiler/rustc_parse/src/parser/ty.rs | 53 +- compiler/rustc_parse_format/src/lib.rs | 1 - compiler/rustc_passes/src/check_attr.rs | 273 +- compiler/rustc_passes/src/dead.rs | 72 +- compiler/rustc_passes/src/diagnostic_items.rs | 18 +- compiler/rustc_passes/src/entry.rs | 10 +- compiler/rustc_passes/src/intrinsicck.rs | 16 +- compiler/rustc_passes/src/lang_items.rs | 136 +- compiler/rustc_passes/src/lib.rs | 3 +- compiler/rustc_passes/src/lib_features.rs | 8 +- compiler/rustc_passes/src/liveness.rs | 6 +- compiler/rustc_passes/src/reachable.rs | 12 +- compiler/rustc_passes/src/region.rs | 2 +- compiler/rustc_passes/src/stability.rs | 16 +- compiler/rustc_plugin_impl/src/build.rs | 38 +- compiler/rustc_privacy/src/lib.rs | 13 +- compiler/rustc_query_impl/src/keys.rs | 10 + compiler/rustc_query_impl/src/lib.rs | 6 +- compiler/rustc_query_impl/src/plumbing.rs | 67 +- .../rustc_query_impl/src/profiling_support.rs | 2 +- .../rustc_query_system/src/dep_graph/graph.rs | 326 +- .../rustc_query_system/src/dep_graph/mod.rs | 2 - .../rustc_query_system/src/dep_graph/prev.rs | 56 - .../src/dep_graph/serialized.rs | 52 +- compiler/rustc_query_system/src/lib.rs | 3 - .../rustc_query_system/src/query/caches.rs | 4 +- .../rustc_query_system/src/query/config.rs | 4 - compiler/rustc_query_system/src/query/job.rs | 21 +- compiler/rustc_query_system/src/query/mod.rs | 4 - .../rustc_query_system/src/query/plumbing.rs | 275 +- .../rustc_resolve/src/build_reduced_graph.rs | 21 +- compiler/rustc_resolve/src/diagnostics.rs | 25 +- compiler/rustc_resolve/src/late.rs | 52 +- .../rustc_resolve/src/late/diagnostics.rs | 418 +- compiler/rustc_resolve/src/late/lifetimes.rs | 160 +- compiler/rustc_resolve/src/lib.rs | 32 +- compiler/rustc_resolve/src/macros.rs | 12 +- .../rustc_save_analysis/src/dump_visitor.rs | 4 +- compiler/rustc_save_analysis/src/lib.rs | 25 +- .../rustc_save_analysis/src/span_utils.rs | 14 +- .../rustc_serialize/src/collection_impls.rs | 48 +- compiler/rustc_serialize/src/json.rs | 171 +- compiler/rustc_serialize/src/lib.rs | 3 - compiler/rustc_serialize/src/serialize.rs | 133 +- .../rustc_session/src/cgu_reuse_tracker.rs | 2 +- compiler/rustc_session/src/config.rs | 334 +- compiler/rustc_session/src/filesearch.rs | 68 +- compiler/rustc_session/src/lib.rs | 1 - compiler/rustc_session/src/options.rs | 1030 +- compiler/rustc_session/src/output.rs | 5 + compiler/rustc_session/src/parse.rs | 14 + compiler/rustc_session/src/session.rs | 53 +- compiler/rustc_session/src/utils.rs | 33 +- compiler/rustc_span/src/def_id.rs | 78 +- compiler/rustc_span/src/hygiene.rs | 17 +- compiler/rustc_span/src/lib.rs | 243 +- compiler/rustc_span/src/source_map.rs | 151 +- compiler/rustc_span/src/source_map/tests.rs | 4 +- compiler/rustc_span/src/span_encoding.rs | 7 +- compiler/rustc_span/src/symbol.rs | 32 +- compiler/rustc_symbol_mangling/src/legacy.rs | 6 +- compiler/rustc_symbol_mangling/src/lib.rs | 5 +- compiler/rustc_symbol_mangling/src/v0.rs | 45 +- compiler/rustc_target/src/abi/call/bpf.rs | 31 + compiler/rustc_target/src/abi/call/mod.rs | 2 + compiler/rustc_target/src/abi/mod.rs | 20 +- compiler/rustc_target/src/asm/aarch64.rs | 10 +- compiler/rustc_target/src/asm/arm.rs | 3 +- compiler/rustc_target/src/asm/bpf.rs | 129 + compiler/rustc_target/src/asm/hexagon.rs | 3 +- compiler/rustc_target/src/asm/mod.rs | 57 +- compiler/rustc_target/src/asm/powerpc.rs | 152 + compiler/rustc_target/src/asm/riscv.rs | 3 +- compiler/rustc_target/src/asm/x86.rs | 32 +- compiler/rustc_target/src/lib.rs | 53 +- .../src/spec/aarch64_apple_ios.rs | 1 - .../src/spec/aarch64_apple_ios_macabi.rs | 1 - .../src/spec/aarch64_apple_ios_sim.rs | 1 - .../src/spec/aarch64_apple_tvos.rs | 1 - .../src/spec/aarch64_unknown_none.rs | 1 - .../spec/aarch64_unknown_none_softfloat.rs | 1 - compiler/rustc_target/src/spec/apple_base.rs | 4 +- .../rustc_target/src/spec/apple_sdk_base.rs | 1 - .../rustc_target/src/spec/avr_gnu_base.rs | 1 - compiler/rustc_target/src/spec/bpf_base.rs | 42 + .../src/spec/bpfeb_unknown_none.rs | 12 + .../src/spec/bpfel_unknown_none.rs | 12 + .../rustc_target/src/spec/dragonfly_base.rs | 3 +- .../rustc_target/src/spec/freebsd_base.rs | 3 +- .../rustc_target/src/spec/fuchsia_base.rs | 3 +- compiler/rustc_target/src/spec/haiku_base.rs | 3 +- compiler/rustc_target/src/spec/hermit_base.rs | 1 - .../src/spec/hermit_kernel_base.rs | 1 - .../rustc_target/src/spec/illumos_base.rs | 14 +- compiler/rustc_target/src/spec/l4re_base.rs | 3 +- compiler/rustc_target/src/spec/linux_base.rs | 3 +- .../src/spec/linux_kernel_base.rs | 1 - .../rustc_target/src/spec/mipsel_sony_psp.rs | 1 - compiler/rustc_target/src/spec/mod.rs | 75 +- .../rustc_target/src/spec/msp430_none_elf.rs | 1 + compiler/rustc_target/src/spec/msvc_base.rs | 1 + compiler/rustc_target/src/spec/netbsd_base.rs | 3 +- .../src/spec/nvptx64_nvidia_cuda.rs | 1 + .../rustc_target/src/spec/openbsd_base.rs | 3 +- compiler/rustc_target/src/spec/redox_base.rs | 3 +- .../rustc_target/src/spec/solaris_base.rs | 3 +- .../src/spec/thumbv4t_none_eabi.rs | 1 - .../rustc_target/src/spec/vxworks_base.rs | 3 +- .../src/spec/wasm32_unknown_emscripten.rs | 3 +- compiler/rustc_target/src/spec/wasm_base.rs | 8 +- .../rustc_target/src/spec/windows_gnu_base.rs | 2 +- .../src/spec/windows_msvc_base.rs | 2 +- .../src/spec/x86_64_fortanix_unknown_sgx.rs | 1 - .../spec/x86_64_unknown_none_hermitkernel.rs | 2 +- .../rustc_trait_selection/src/autoderef.rs | 4 +- compiler/rustc_trait_selection/src/lib.rs | 2 +- .../rustc_trait_selection/src/opaque_types.rs | 152 +- .../src/traits/error_reporting/mod.rs | 8 +- .../error_reporting/on_unimplemented.rs | 9 + .../src/traits/error_reporting/suggestions.rs | 85 +- .../src/traits/fulfill.rs | 13 +- .../src/traits/project.rs | 10 +- .../src/traits/select/mod.rs | 134 +- compiler/rustc_ty_utils/src/needs_drop.rs | 53 +- .../rustc_ty_utils/src/representability.rs | 219 +- compiler/rustc_ty_utils/src/ty.rs | 13 - compiler/rustc_type_ir/src/lib.rs | 17 + compiler/rustc_typeck/src/astconv/generics.rs | 138 +- compiler/rustc_typeck/src/astconv/mod.rs | 22 +- compiler/rustc_typeck/src/check/callee.rs | 40 +- compiler/rustc_typeck/src/check/cast.rs | 62 +- compiler/rustc_typeck/src/check/check.rs | 21 +- compiler/rustc_typeck/src/check/demand.rs | 52 +- compiler/rustc_typeck/src/check/dropck.rs | 1 + compiler/rustc_typeck/src/check/expr.rs | 4 +- .../rustc_typeck/src/check/fn_ctxt/_impl.rs | 75 +- .../rustc_typeck/src/check/fn_ctxt/checks.rs | 32 +- .../src/check/generator_interior.rs | 30 +- compiler/rustc_typeck/src/check/inherited.rs | 5 +- compiler/rustc_typeck/src/check/method/mod.rs | 10 +- .../rustc_typeck/src/check/method/probe.rs | 2 +- .../rustc_typeck/src/check/method/suggest.rs | 108 +- compiler/rustc_typeck/src/check/mod.rs | 27 +- compiler/rustc_typeck/src/check/op.rs | 19 +- compiler/rustc_typeck/src/check/pat.rs | 9 +- compiler/rustc_typeck/src/check/place_op.rs | 34 +- compiler/rustc_typeck/src/check/upvar.rs | 356 +- compiler/rustc_typeck/src/check/wfcheck.rs | 113 +- compiler/rustc_typeck/src/check/writeback.rs | 34 +- compiler/rustc_typeck/src/check_unused.rs | 4 +- .../src/coherence/inherent_impls.rs | 12 +- .../src/coherence/inherent_impls_overlap.rs | 5 +- compiler/rustc_typeck/src/coherence/mod.rs | 6 +- compiler/rustc_typeck/src/collect.rs | 21 +- compiler/rustc_typeck/src/collect/type_of.rs | 33 +- compiler/rustc_typeck/src/expr_use_visitor.rs | 4 +- compiler/rustc_typeck/src/lib.rs | 7 +- compiler/rustc_typeck/src/outlives/mod.rs | 8 +- .../wrong_number_of_generic_args.rs | 795 +- compiler/rustc_typeck/src/variance/mod.rs | 7 +- config.toml.example | 12 +- git-commit-hash | 2 +- library/alloc/Cargo.toml | 2 +- library/alloc/benches/vec.rs | 10 +- library/alloc/src/alloc.rs | 9 +- library/alloc/src/borrow.rs | 9 +- library/alloc/src/boxed.rs | 39 +- library/alloc/src/collections/binary_heap.rs | 2 + library/alloc/src/collections/btree/map.rs | 51 +- .../alloc/src/collections/btree/navigate.rs | 80 +- library/alloc/src/collections/btree/node.rs | 39 +- library/alloc/src/collections/linked_list.rs | 48 +- library/alloc/src/collections/mod.rs | 11 + .../alloc/src/collections/vec_deque/iter.rs | 4 +- .../alloc/src/collections/vec_deque/mod.rs | 13 +- .../src/collections/vec_deque/pair_slices.rs | 3 +- library/alloc/src/fmt.rs | 2 + library/alloc/src/lib.rs | 13 +- library/alloc/src/raw_vec.rs | 16 +- library/alloc/src/rc.rs | 65 +- library/alloc/src/slice.rs | 32 +- library/alloc/src/str.rs | 13 + library/alloc/src/string.rs | 271 +- library/alloc/src/sync.rs | 52 +- library/alloc/src/vec/cow.rs | 18 + library/alloc/src/vec/into_iter.rs | 4 + library/alloc/src/vec/is_zero.rs | 33 + library/alloc/src/vec/mod.rs | 69 +- library/alloc/src/vec/partial_eq.rs | 4 + library/alloc/src/vec/spec_from_iter.rs | 35 +- library/alloc/src/vec/splice.rs | 2 +- library/alloc/tests/arc.rs | 15 + library/alloc/tests/lib.rs | 1 - library/alloc/tests/rc.rs | 15 + library/alloc/tests/vec.rs | 58 +- library/core/src/array/mod.rs | 9 +- library/core/src/cell.rs | 9 + library/core/src/char/methods.rs | 28 +- library/core/src/cmp.rs | 17 +- library/core/src/fmt/mod.rs | 43 +- library/core/src/future/future.rs | 3 +- library/core/src/hint.rs | 2 +- library/core/src/intrinsics.rs | 26 +- library/core/src/iter/adapters/chain.rs | 4 +- library/core/src/iter/adapters/cloned.rs | 4 +- library/core/src/iter/adapters/copied.rs | 4 +- library/core/src/iter/adapters/cycle.rs | 2 +- library/core/src/iter/adapters/enumerate.rs | 4 +- library/core/src/iter/adapters/filter.rs | 6 +- library/core/src/iter/adapters/filter_map.rs | 6 +- library/core/src/iter/adapters/flatten.rs | 16 +- library/core/src/iter/adapters/fuse.rs | 16 +- library/core/src/iter/adapters/inspect.rs | 4 +- library/core/src/iter/adapters/map.rs | 4 +- library/core/src/iter/adapters/map_while.rs | 2 +- library/core/src/iter/adapters/mod.rs | 3 +- library/core/src/iter/adapters/peekable.rs | 31 +- library/core/src/iter/adapters/rev.rs | 4 +- library/core/src/iter/adapters/scan.rs | 4 +- library/core/src/iter/adapters/skip.rs | 6 +- library/core/src/iter/adapters/skip_while.rs | 2 +- library/core/src/iter/adapters/step_by.rs | 4 +- library/core/src/iter/adapters/take.rs | 6 +- library/core/src/iter/adapters/take_while.rs | 4 +- library/core/src/iter/adapters/zip.rs | 2 +- library/core/src/iter/mod.rs | 2 + library/core/src/iter/range.rs | 472 +- library/core/src/iter/sources/repeat.rs | 35 + library/core/src/iter/traits/collect.rs | 2 +- library/core/src/iter/traits/double_ended.rs | 2 +- library/core/src/iter/traits/iterator.rs | 117 +- library/core/src/iter/traits/marker.rs | 18 + library/core/src/iter/traits/mod.rs | 2 + library/core/src/lib.rs | 35 +- library/core/src/macros/mod.rs | 104 +- library/core/src/marker.rs | 1 + library/core/src/mem/maybe_uninit.rs | 4 +- library/core/src/num/bignum.rs | 4 +- library/core/src/num/f32.rs | 33 +- library/core/src/num/f64.rs | 33 +- library/core/src/num/int_macros.rs | 128 +- library/core/src/num/uint_macros.rs | 128 +- library/core/src/num/wrapping.rs | 15 + library/core/src/ops/control_flow.rs | 53 +- library/core/src/ops/mod.rs | 12 +- library/core/src/ops/range.rs | 38 +- library/core/src/ops/try.rs | 14 +- library/core/src/ops/try_trait.rs | 130 +- library/core/src/option.rs | 10 +- library/core/src/panicking.rs | 69 +- library/core/src/prelude/mod.rs | 20 +- library/core/src/ptr/const_ptr.rs | 6 +- library/core/src/ptr/mod.rs | 93 +- library/core/src/ptr/mut_ptr.rs | 62 +- library/core/src/ptr/non_null.rs | 2 +- library/core/src/result.rs | 5 +- library/core/src/slice/ascii.rs | 2 +- library/core/src/slice/mod.rs | 35 +- library/core/src/str/iter.rs | 2 +- library/core/src/str/mod.rs | 18 +- library/core/src/task/poll.rs | 20 +- library/core/src/time.rs | 10 +- library/core/tests/any.rs | 13 + library/core/tests/array.rs | 46 +- library/core/tests/char.rs | 12 +- library/core/tests/iter/adapters/zip.rs | 4 +- library/core/tests/lib.rs | 3 +- library/core/tests/option.rs | 12 - library/core/tests/result.rs | 30 +- library/panic_abort/src/lib.rs | 5 +- library/panic_unwind/src/lib.rs | 5 +- library/panic_unwind/src/seh.rs | 2 +- library/proc_macro/src/bridge/buffer.rs | 54 +- library/proc_macro/src/bridge/mod.rs | 17 + library/proc_macro/src/bridge/rpc.rs | 4 +- library/proc_macro/src/lib.rs | 60 +- library/proc_macro/src/quote.rs | 10 +- library/profiler_builtins/Cargo.toml | 2 +- library/std/Cargo.toml | 2 +- library/std/src/alloc.rs | 4 +- library/std/src/collections/hash/map.rs | 30 +- library/std/src/env.rs | 21 +- library/std/src/ffi/c_str.rs | 6 +- library/std/src/ffi/os_str.rs | 2 +- library/std/src/fs.rs | 26 + library/std/src/io/buffered/bufwriter.rs | 217 +- library/std/src/io/buffered/linewritershim.rs | 2 +- library/std/src/io/cursor.rs | 19 +- library/std/src/io/error.rs | 6 + library/std/src/io/impls.rs | 10 + library/std/src/io/mod.rs | 53 +- library/std/src/keyword_docs.rs | 24 +- library/std/src/lib.rs | 16 +- library/std/src/macros.rs | 7 +- library/std/src/net/ip.rs | 189 +- library/std/src/net/ip/tests.rs | 18 +- library/std/src/net/udp.rs | 4 +- library/std/src/os/emscripten/raw.rs | 1 - .../{sys/sgx/ext => os/fortanix_sgx}/arch.rs | 10 +- .../{sys/sgx/ext => os/fortanix_sgx}/ffi.rs | 0 .../{sys/sgx/ext => os/fortanix_sgx}/io.rs | 0 library/std/src/os/fortanix_sgx/mod.rs | 6 +- .../src/{sys/hermit/ext => os/hermit}/ffi.rs | 0 .../src/{sys/hermit/ext => os/hermit}/mod.rs | 1 - library/std/src/os/linux/mod.rs | 1 + library/std/src/os/linux/raw.rs | 1 - library/std/src/os/mod.rs | 169 +- library/std/src/os/redox/raw.rs | 1 - .../std/src/{sys/unix/ext => os/unix}/ffi.rs | 0 .../std/src/{sys/unix/ext => os/unix}/fs.rs | 2 +- .../std/src/{sys/unix/ext => os/unix}/io.rs | 0 .../std/src/{sys/unix/ext => os/unix}/mod.rs | 73 +- .../src/{sys/unix/ext => os/unix}/net/addr.rs | 0 .../unix/ext => os/unix}/net/ancillary.rs | 4 +- .../{sys/unix/ext => os/unix}/net/datagram.rs | 0 .../{sys/unix/ext => os/unix}/net/listener.rs | 0 .../src/{sys/unix/ext => os/unix}/net/mod.rs | 0 .../{sys/unix/ext => os/unix}/net/raw_fd.rs | 0 .../{sys/unix/ext => os/unix}/net/stream.rs | 0 .../{sys/unix/ext => os/unix}/net/tests.rs | 0 .../src/{sys/unix/ext => os/unix}/process.rs | 59 +- .../std/src/{sys/unix/ext => os/unix}/raw.rs | 0 .../src/{sys/unix/ext => os/unix}/thread.rs | 0 .../src/{sys/unix/ext => os/unix}/ucred.rs | 0 .../{sys/unix/ext => os/unix}/ucred/tests.rs | 0 library/std/src/os/wasi.rs | 6 - .../std/src/{sys/wasi/ext => os/wasi}/ffi.rs | 0 .../std/src/{sys/wasi/ext => os/wasi}/fs.rs | 0 .../std/src/{sys/wasi/ext => os/wasi}/io.rs | 0 .../std/src/{sys/wasi/ext => os/wasi}/mod.rs | 3 +- .../{sys/windows/ext => os/windows}/ffi.rs | 0 .../src/{sys/windows/ext => os/windows}/fs.rs | 0 .../src/{sys/windows/ext => os/windows}/io.rs | 0 .../{sys/windows/ext => os/windows}/mod.rs | 1 - .../windows/ext => os/windows}/process.rs | 0 .../{sys/windows/ext => os/windows}/raw.rs | 0 .../{sys/windows/ext => os/windows}/thread.rs | 0 library/std/src/panic.rs | 39 + library/std/src/panicking.rs | 86 +- library/std/src/path.rs | 50 +- library/std/src/prelude/mod.rs | 14 +- library/std/src/primitive_docs.rs | 34 +- library/std/src/process.rs | 144 +- library/std/src/sync/mutex.rs | 10 +- library/std/src/sync/rwlock.rs | 61 +- library/std/src/sys/hermit/fs.rs | 2 +- library/std/src/sys/hermit/mod.rs | 1 - library/std/src/sys/hermit/os.rs | 2 +- library/std/src/sys/hermit/rwlock.rs | 2 + library/std/src/sys/mod.rs | 81 +- library/std/src/sys/sgx/abi/mod.rs | 2 +- .../src/sys/sgx/abi/{tls.rs => tls/mod.rs} | 0 library/std/src/sys/sgx/ext/mod.rs | 5 - library/std/src/sys/sgx/mod.rs | 1 - library/std/src/sys/sgx/mutex.rs | 2 +- library/std/src/sys/sgx/rwlock.rs | 2 + .../sgx/{waitqueue.rs => waitqueue/mod.rs} | 7 +- .../std/src/sys/sgx/waitqueue/spin_mutex.rs | 3 + .../std/src/sys/sgx/waitqueue/unsafe_list.rs | 3 + library/std/src/sys/unix/fs.rs | 8 +- library/std/src/sys/unix/mod.rs | 3 +- library/std/src/sys/unix/net.rs | 6 +- library/std/src/sys/unix/os.rs | 21 +- library/std/src/sys/unix/process/mod.rs | 2 +- .../src/sys/unix/process/process_common.rs | 2 +- .../src/sys/unix/process/process_fuchsia.rs | 26 +- .../std/src/sys/unix/process/process_unix.rs | 32 +- .../sys/unix/process/process_unix/tests.rs | 27 + .../src/sys/unix/process/process_vxworks.rs | 30 +- library/std/src/sys/unix/rwlock.rs | 54 +- library/std/src/sys/unix/stack_overflow.rs | 8 +- library/std/src/sys/unsupported/args.rs | 1 + library/std/src/sys/unsupported/fs.rs | 4 + library/std/src/sys/unsupported/process.rs | 18 +- library/std/src/sys/unsupported/rwlock.rs | 2 + library/std/src/sys/wasi/fs.rs | 2 +- library/std/src/sys/wasi/mod.rs | 2 +- library/std/src/sys/wasm/args.rs | 42 - .../condvar.rs} | 0 .../{futex_atomics.rs => atomics/futex.rs} | 0 .../{mutex_atomics.rs => atomics/mutex.rs} | 0 .../{rwlock_atomics.rs => atomics/rwlock.rs} | 2 + .../std/src/sys/wasm/{ => atomics}/thread.rs | 22 +- library/std/src/sys/wasm/mod.rs | 14 +- library/std/src/sys/windows/c.rs | 325 +- library/std/src/sys/windows/fs.rs | 29 + library/std/src/sys/windows/mod.rs | 16 +- library/std/src/sys/windows/process.rs | 25 +- library/std/src/sys/windows/rwlock.rs | 2 + library/std/src/sys/windows/stack_overflow.rs | 7 +- library/std/src/sys_common/fs.rs | 8 + library/std/src/{ => sys_common}/memchr.rs | 6 +- .../std/src/{ => sys_common}/memchr/tests.rs | 0 library/std/src/sys_common/mod.rs | 2 +- library/std/src/sys_common/rt.rs | 21 +- library/std/src/sys_common/rwlock.rs | 120 +- library/std/src/sys_common/util.rs | 28 - library/std/src/thread/local.rs | 26 +- library/std/src/time.rs | 4 +- library/std/tests/run-time-detect.rs | 1 - library/stdarch/.cirrus.yml | 6 +- .../stdarch/ci/docker/wasm32-wasi/Dockerfile | 6 +- library/stdarch/ci/run.sh | 6 - .../crates/assert-instr-macro/Cargo.toml | 1 + .../crates/assert-instr-macro/src/lib.rs | 20 +- library/stdarch/crates/core_arch/build.rs | 14 - .../crates/core_arch/src/aarch64/armclang.rs | 23 + .../crates/core_arch/src/aarch64/mod.rs | 6 +- .../core_arch/src/aarch64/neon/generated.rs | 17327 ++++++++++- .../crates/core_arch/src/aarch64/neon/mod.rs | 1611 +- .../crates/core_arch/src/aarch64/prefetch.rs | 50 +- .../core_arch/src/aarch64/test_support.rs | 2 +- .../crates/core_arch/src/aarch64/tme.rs | 18 +- .../crates/core_arch/src/arm/armclang.rs | 53 +- .../crates/core_arch/src/{acle => arm}/dsp.rs | 0 .../crates/core_arch/src/{acle => arm}/ex.rs | 24 +- .../stdarch/crates/core_arch/src/arm/mod.rs | 105 +- .../stdarch/crates/core_arch/src/arm/neon.rs | 1286 + .../core_arch/src/arm/neon/generated.rs | 5237 ---- .../crates/core_arch/src/{acle => arm}/sat.rs | 0 .../core_arch/src/{acle => arm}/simd32.rs | 2 +- .../{acle => arm_shared}/barrier/common.rs | 0 .../src/{acle => arm_shared}/barrier/cp15.rs | 15 +- .../src/{acle => arm_shared}/barrier/mod.rs | 0 .../barrier/not_mclass.rs | 0 .../src/{acle => arm_shared}/barrier/v8.rs | 0 .../core_arch/src/{arm => arm_shared}/crc.rs | 2 +- .../src/{arm => arm_shared}/crypto.rs | 146 +- .../src/{acle => arm_shared}/hints.rs | 52 +- .../core_arch/src/{acle => arm_shared}/mod.rs | 80 +- .../src/arm_shared/neon/generated.rs | 24866 ++++++++++++++++ .../{arm => arm_shared}/neon/load_tests.rs | 0 .../src/{arm => arm_shared}/neon/mod.rs | 8493 +++--- .../neon/shift_and_insert_tests.rs | 4 +- .../src/arm_shared/neon/store_tests.rs | 366 + .../neon/table_lookup_tests.rs | 0 .../{acle => arm_shared}/registers/aarch32.rs | 0 .../src/{acle => arm_shared}/registers/mod.rs | 8 +- .../src/{acle => arm_shared}/registers/v6m.rs | 0 .../src/{acle => arm_shared}/registers/v7m.rs | 0 .../src/{arm => arm_shared}/test_support.rs | 8 +- library/stdarch/crates/core_arch/src/lib.rs | 10 +- .../stdarch/crates/core_arch/src/macros.rs | 535 +- .../stdarch/crates/core_arch/src/mips/msa.rs | 1686 +- .../crates/core_arch/src/mips/msa/macros.rs | 4352 +-- library/stdarch/crates/core_arch/src/mod.rs | 116 +- .../crates/core_arch/src/powerpc/vsx.rs | 23 +- .../stdarch/crates/core_arch/src/simd_llvm.rs | 9 +- .../crates/core_arch/src/wasm32/atomic.rs | 49 +- .../crates/core_arch/src/wasm32/memory.rs | 36 +- .../crates/core_arch/src/wasm32/mod.rs | 2 - .../crates/core_arch/src/wasm32/simd128.rs | 4616 ++- .../stdarch/crates/core_arch/src/x86/aes.rs | 18 +- .../stdarch/crates/core_arch/src/x86/avx.rs | 1014 +- .../stdarch/crates/core_arch/src/x86/avx2.rs | 1847 +- .../crates/core_arch/src/x86/avx512bw.rs | 2153 +- .../crates/core_arch/src/x86/avx512f.rs | 14209 ++++----- .../crates/core_arch/src/x86/avx512gfni.rs | 469 +- .../crates/core_arch/src/x86/avx512vaes.rs | 36 +- .../crates/core_arch/src/x86/avx512vbmi2.rs | 865 +- .../core_arch/src/x86/avx512vpclmulqdq.rs | 114 +- .../stdarch/crates/core_arch/src/x86/bt.rs | 64 +- .../stdarch/crates/core_arch/src/x86/cpuid.rs | 62 +- .../crates/core_arch/src/x86/eflags.rs | 8 +- .../stdarch/crates/core_arch/src/x86/f16c.rs | 56 +- .../crates/core_arch/src/x86/macros.rs | 1600 +- .../crates/core_arch/src/x86/pclmulqdq.rs | 32 +- .../stdarch/crates/core_arch/src/x86/rtm.rs | 18 +- .../stdarch/crates/core_arch/src/x86/sha.rs | 27 +- .../stdarch/crates/core_arch/src/x86/sse.rs | 140 +- .../stdarch/crates/core_arch/src/x86/sse2.rs | 680 +- .../stdarch/crates/core_arch/src/x86/sse3.rs | 6 +- .../stdarch/crates/core_arch/src/x86/sse41.rs | 372 +- .../stdarch/crates/core_arch/src/x86/sse42.rs | 312 +- .../stdarch/crates/core_arch/src/x86/ssse3.rs | 104 +- .../stdarch/crates/core_arch/src/x86/test.rs | 20 +- .../crates/core_arch/src/x86_64/avx.rs | 15 +- .../crates/core_arch/src/x86_64/avx2.rs | 17 +- .../crates/core_arch/src/x86_64/avx512f.rs | 1455 +- .../stdarch/crates/core_arch/src/x86_64/bt.rs | 64 +- .../crates/core_arch/src/x86_64/macros.rs | 50 +- .../crates/core_arch/src/x86_64/sse41.rs | 41 +- .../stdarch/crates/simd-test-macro/Cargo.toml | 1 + .../stdarch/crates/simd-test-macro/src/lib.rs | 2 - .../std_detect/src/detect/arch/aarch64.rs | 138 +- .../crates/std_detect/src/detect/arch/arm.rs | 6 +- .../crates/std_detect/src/detect/bit.rs | 2 +- .../crates/std_detect/src/detect/macros.rs | 8 +- .../std_detect/src/detect/os/aarch64.rs | 30 +- .../src/detect/os/freebsd/aarch64.rs | 3 +- .../src/detect/os/freebsd/auxvec.rs | 9 +- .../std_detect/src/detect/os/linux/aarch64.rs | 204 +- .../std_detect/src/detect/os/linux/arm.rs | 13 + .../src/detect/os/windows/aarch64.rs | 6 +- library/stdarch/crates/std_detect/src/lib.rs | 4 +- .../crates/std_detect/tests/cpu-detection.rs | 40 +- library/stdarch/crates/stdarch-gen/neon.spec | 4944 ++- .../stdarch/crates/stdarch-gen/src/main.rs | 2173 +- .../stdarch/crates/stdarch-test/Cargo.toml | 3 +- .../stdarch/crates/stdarch-test/src/lib.rs | 20 +- .../stdarch/crates/stdarch-verify/src/lib.rs | 40 +- .../crates/stdarch-verify/tests/arm.rs | 132 + .../crates/stdarch-verify/tests/mips.rs | 71 +- .../crates/stdarch-verify/tests/x86-intel.rs | 31 +- library/stdarch/examples/hex.rs | 23 +- library/term/src/lib.rs | 6 +- library/test/src/cli.rs | 14 +- library/test/src/console.rs | 3 +- library/test/src/formatters/junit.rs | 174 + library/test/src/formatters/mod.rs | 2 + library/test/src/formatters/pretty.rs | 6 +- library/test/src/formatters/terse.rs | 6 +- library/test/src/lib.rs | 4 +- library/test/src/options.rs | 2 + library/test/src/tests.rs | 76 + library/test/src/types.rs | 34 + library/unwind/Cargo.toml | 9 +- library/unwind/build.rs | 160 +- library/unwind/src/lib.rs | 21 +- src/bootstrap/Cargo.toml | 2 +- src/bootstrap/bin/rustc.rs | 9 +- src/bootstrap/bin/rustdoc.rs | 7 +- src/bootstrap/bootstrap.py | 22 +- src/bootstrap/builder.rs | 36 +- src/bootstrap/builder/tests.rs | 3 + src/bootstrap/check.rs | 16 +- src/bootstrap/compile.rs | 20 +- src/bootstrap/dist.rs | 16 + src/bootstrap/doc.rs | 36 +- src/bootstrap/flags.rs | 17 +- src/bootstrap/format.rs | 17 +- src/bootstrap/install.rs | 25 +- src/bootstrap/lib.rs | 30 +- src/bootstrap/mk/Makefile.in | 4 - src/bootstrap/native.rs | 4 +- src/bootstrap/sanity.rs | 5 +- src/bootstrap/test.rs | 159 +- src/bootstrap/tool.rs | 29 +- src/bootstrap/toolstate.rs | 8 +- src/bootstrap/util.rs | 4 +- src/ci/channel | 1 + .../riscv64-unknown-linux-gnu.config | 2 +- .../docker/host-x86_64/mingw-check/Dockerfile | 6 +- .../host-x86_64/x86_64-gnu-aux/Dockerfile | 24 +- .../host-x86_64/x86_64-gnu-tools/Dockerfile | 58 +- .../x86_64-gnu-tools/checktools.sh | 2 +- src/ci/docker/run.sh | 1 + src/ci/github-actions/ci.yml | 15 + src/ci/init_repo.sh | 2 +- src/ci/pgo.sh | 2 +- src/ci/run.sh | 9 +- src/ci/scripts/install-clang.sh | 6 +- src/ci/scripts/should-skip-this.sh | 24 +- src/ci/scripts/verify-channel.sh | 38 + src/ci/shared.sh | 20 + src/doc/book/.github/workflows/main.yml | 4 +- .../no-listing-02-without-expect/output.txt | 2 +- .../output.txt | 1 + .../output.txt | 4 +- .../listing-11-01/output.txt | 2 +- .../listing-11-03/output.txt | 2 +- .../listing-11-06/output.txt | 2 +- .../listing-11-07/output.txt | 2 +- .../listing-11-08/output.txt | 2 +- .../listing-11-10/output.txt | 2 +- .../listing-11-11/output.txt | 2 +- .../listing-11-13/output.txt | 4 +- .../output.txt | 2 +- .../output.txt | 2 +- .../output.txt | 2 +- .../no-listing-04-bug-in-add-two/output.txt | 2 +- .../no-listing-06-greeter-with-bug/output.txt | 2 +- .../output.txt | 2 +- .../no-listing-08-guess-with-bug/output.txt | 2 +- .../output.txt | 2 +- .../no-listing-11-ignore-a-test/output.txt | 2 +- .../output.txt | 6 +- .../output-only-01-show-output/output.txt | 2 +- .../output-only-02-single-test/output.txt | 2 +- .../output-only-03-multiple-tests/output.txt | 2 +- .../output-only-04-running-ignored/output.txt | 2 +- .../output.txt | 2 +- .../listing-12-12/output.txt | 2 +- .../listing-15-15/output.txt | 8 +- .../listing-18-10/output.txt | 4 +- src/doc/book/rust-toolchain | 2 +- src/doc/book/src/ch11-01-writing-tests.md | 2 +- src/doc/book/src/title-page.md | 2 +- src/doc/book/tools/update-rustc.sh | 6 +- .../edition-guide/.github/workflows/main.yml | 2 +- src/doc/edition-guide/book.toml | 88 + src/doc/edition-guide/src/SUMMARY.md | 106 +- src/doc/edition-guide/src/editions/index.md | 100 +- ...ng-an-existing-project-to-a-new-edition.md | 2 +- src/doc/edition-guide/src/introduction.md | 13 +- ...go-can-use-a-local-registry-replacement.md | 38 - .../cargo-check-for-faster-checking.md | 54 - ...-install-for-easy-installation-of-tools.md | 32 - .../cargo-new-defaults-to-a-binary-project.md | 18 - ...tc-for-passing-arbitrary-flags-to-rustc.md | 21 - ...o-workspaces-for-multi-package-projects.md | 30 - ...ates-io-disallows-wildcard-dependencies.md | 25 - .../rust-2018/cargo-and-crates-io/index.md | 7 - .../multi-file-examples.md | 22 - .../replacing-dependencies-with-patch.md | 35 - src/doc/edition-guide/src/rust-2018/cargo.md | 10 + .../async-await-for-easier-concurrency.md | 7 - .../src/rust-2018/control-flow/index.md | 6 - .../loops-can-break-with-a-value.md | 26 - .../rust-2018/data-types/128-bit-integers.md | 17 - ...osing-alignment-with-the-repr-attribute.md | 55 - .../data-types/field-init-shorthand.md | 49 - .../rust-2018/data-types/inclusive-ranges.md | 72 - .../src/rust-2018/data-types/index.md | 6 - .../operator-equals-are-now-implementable.md | 33 - .../union-for-an-unsafe-form-of-enum.md | 60 - .../src/rust-2018/documentation/index.md | 6 - .../documentation/new-editions-of-the-book.md | 31 - ...-os-has-documentation-for-all-platforms.md | 11 - .../documentation/the-rust-bookshelf.md | 33 - .../documentation/the-rustonomicon.md | 10 - .../src/rust-2018/edition-changes.md | 36 - .../aborting-on-panic.md | 18 - .../controlling-panics-with-std-panic.md | 80 - .../error-handling-and-panics/index.md | 6 - .../question-mark-in-main-and-tests.md | 131 - ...mark-operator-for-easier-error-handling.md | 120 - src/doc/edition-guide/src/rust-2018/index.md | 12 +- .../src/rust-2018/macros/at-most-once.md | 42 - .../src/rust-2018/macros/custom-derive.md | 49 - .../src/rust-2018/macros/index.md | 6 - .../src/rust-2018/macros/macro-changes.md | 271 - .../src/rust-2018/module-system/index.md | 6 - .../more-visibility-modifiers.md | 16 - .../module-system/nested-imports-with-use.md | 35 - .../module-system/raw-identifiers.md | 68 - ...t-for-trait-objects.md => new-keywords.md} | 32 +- .../default-match-bindings.md | 61 - .../ownership-and-lifetimes/index.md | 6 - .../inference-in-structs.md | 72 - .../lifetime-elision-in-impl.md | 75 - .../non-lexical-lifetimes.md | 85 - .../simpler-lifetimes-in-static-and-const.md | 41 - .../the-anonymous-lifetime.md | 96 - .../path-clarity.md => path-changes.md} | 46 +- .../cdylib-crates-for-c-interoperability.md | 18 - .../global-allocators.md | 35 - .../platform-and-target-support/index.md | 6 - .../libcore-for-low-level-rust.md | 31 - .../msvc-toolchain-support.md | 18 - .../musl-support-for-fully-static-binaries.md | 45 - .../webassembly-support.md | 28 - ...ocumentation-tests-can-now-compile-fail.md | 19 - .../src/rust-2018/rustdoc/index.md | 6 - .../rustdoc/rustdoc-uses-commonmark.md | 16 - .../rustup-for-managing-rust-versions.md | 224 - .../rust-2018/simd-for-faster-computing.md | 108 - .../src/rust-2018/slice-patterns.md | 91 - .../an-attribute-for-deprecation.md | 35 - .../the-compiler/improved-error-messages.md | 50 - ...emental-compilation-for-faster-compiles.md | 23 - .../src/rust-2018/the-compiler/index.md | 6 - ...-anon-params.md => trait-fn-parameters.md} | 11 +- .../trait-system/associated-constants.md | 117 - ...t-for-returning-complex-types-with-ease.md | 168 - .../src/rust-2018/trait-system/index.md | 6 - ...e-container-types-support-trait-objects.md | 29 - .../src/rust-2018/tyvar-behind-raw-pointer.md | 11 + .../src/rust-2021/IntoIterator-for-arrays.md | 49 + .../src/rust-2021/default-cargo-resolver.md | 18 + .../rust-2021/disjoint-capture-in-closures.md | 37 + src/doc/edition-guide/src/rust-2021/index.md | 22 + .../src/rust-2021/or-patterns-macro-rules.md | 28 + .../src/rust-2021/panic-macro-consistency.md | 44 + .../edition-guide/src/rust-2021/prelude.md | 32 + .../src/rust-2021/reserving-syntax.md | 40 + .../rust-2021/warnings-promoted-to-error.md | 17 + src/doc/edition-guide/src/rust-next/alloc.md | 28 - .../rust-next/alternative-cargo-registries.md | 26 - .../src/rust-next/cargo-vendor.md | 9 - .../edition-guide/src/rust-next/const-fn.md | 241 - .../edition-guide/src/rust-next/dbg-macro.md | 104 - .../src/rust-next/edition-changes.md | 3 - src/doc/edition-guide/src/rust-next/future.md | 10 - src/doc/edition-guide/src/rust-next/index.md | 7 - .../src/rust-next/literal-macro-matcher.md | 18 - .../src/rust-next/maybe-uninit.md | 28 - .../src/rust-next/no-jemalloc.md | 43 - .../src/rust-next/no-more-fnbox.md | 38 - src/doc/edition-guide/src/rust-next/pin.md | 21 - .../qustion-mark-operator-in-macros.md | 14 - .../src/rust-next/tryfrom-and-tryinto.md | 23 - .../src/rust-next/uniform-paths.md | 23 - src/doc/embedded-book/src/intro/install.md | 4 +- src/doc/nomicon/src/atomics.md | 4 +- src/doc/nomicon/src/exception-safety.md | 4 +- src/doc/nomicon/src/safe-unsafe-meaning.md | 23 +- src/doc/nomicon/src/vec-alloc.md | 109 +- src/doc/nomicon/src/vec-dealloc.md | 9 +- src/doc/nomicon/src/vec-drain.md | 2 +- src/doc/nomicon/src/vec-final.md | 204 +- src/doc/nomicon/src/vec-insert-remove.md | 18 +- src/doc/nomicon/src/vec-into-iter.md | 12 +- src/doc/nomicon/src/vec-layout.md | 61 +- src/doc/nomicon/src/vec-push-pop.md | 4 +- src/doc/nomicon/src/vec-raw.md | 95 +- src/doc/nomicon/src/vec-zsts.md | 78 +- src/doc/nomicon/src/vec.md | 14 +- src/doc/nomicon/src/working-with-unsafe.md | 2 +- src/doc/reference/src/attributes.md | 38 +- src/doc/reference/src/attributes/limits.md | 13 +- .../reference/src/conditional-compilation.md | 5 +- .../src/expressions/operator-expr.md | 100 +- src/doc/reference/src/glossary.md | 22 + src/doc/reference/src/items/generics.md | 6 +- .../reference/src/items/implementations.md | 2 +- src/doc/reference/src/items/traits.md | 4 +- src/doc/reference/src/macros-by-example.md | 7 +- src/doc/reference/src/trait-bounds.md | 4 + .../reference/src/types/function-pointer.md | 2 +- src/doc/reference/src/types/impl-trait.md | 119 +- src/doc/rust-by-example/src/cargo/test.md | 92 +- .../src/custom_types/structs.md | 2 +- src/doc/rust-by-example/src/hello/print.md | 2 +- src/doc/rustc-dev-guide/ci/check-in.sh | 5 +- .../src/appendix/bibliography.md | 2 +- .../src/building/prerequisites.md | 2 +- src/doc/rustc-dev-guide/src/compiler-team.md | 4 +- src/doc/rustc-dev-guide/src/contributing.md | 73 +- src/doc/rustc-dev-guide/src/diagnostics.md | 6 +- src/doc/rustc-dev-guide/src/git.md | 5 +- src/doc/rustc-dev-guide/src/identifiers.md | 2 +- .../src/llvm-coverage-instrumentation.md | 142 +- src/doc/rustc-dev-guide/src/miri.md | 7 +- .../incremental-compilation-in-detail.md | 20 + src/doc/rustc-dev-guide/src/query.md | 4 +- .../rustc-dev-guide/src/rustdoc-internals.md | 2 +- src/doc/rustc-dev-guide/src/rustdoc.md | 2 +- src/doc/rustc-dev-guide/src/thir.md | 9 +- src/doc/rustc/src/SUMMARY.md | 1 + src/doc/rustc/src/codegen-options/index.md | 8 +- src/doc/rustc/src/platform-support.md | 311 +- .../platform-support/aarch64-apple-ios-sim.md | 56 + src/doc/rustc/src/target-tier-policy.md | 21 +- src/doc/rustdoc/README.md | 5 + src/doc/rustdoc/src/documentation-tests.md | 4 +- .../rustdoc/src/how-to-write-documentation.md | 10 +- src/doc/rustdoc/src/lints.md | 44 + src/doc/rustdoc/src/the-doc-attribute.md | 7 + src/doc/rustdoc/src/unstable-features.md | 16 - .../src/compiler-flags/force-warns.md | 21 + .../src/compiler-flags/instrument-coverage.md | 346 + .../source-based-code-coverage.md | 326 +- .../src/language-features/const-fn.md | 10 - .../src/language-features/external-doc.md | 40 - .../language-features/member-constraints.md | 29 - .../language-features/more-qualified-paths.md | 29 + .../native-link-modifiers-as-needed.md | 18 + .../native-link-modifiers-bundle.md | 19 + .../native-link-modifiers-verbatim.md | 20 + .../native-link-modifiers-whole-archive.md | 18 + .../native-link-modifiers.md | 11 + .../src/language-features/no-coverage.md | 30 + .../unstable-book/src/library-features/asm.md | 44 +- .../src/library-features/global-asm.md | 69 +- src/etc/htmldocck.py | 4 + src/etc/natvis/intrinsic.natvis | 53 + src/etc/natvis/libcore.natvis | 17 - src/librustdoc/Cargo.toml | 2 +- src/librustdoc/clean/auto_trait.rs | 10 +- src/librustdoc/clean/blanket_impl.rs | 14 +- src/librustdoc/clean/inline.rs | 42 +- src/librustdoc/clean/mod.rs | 63 +- src/librustdoc/clean/types.rs | 396 +- src/librustdoc/clean/utils.rs | 91 +- src/librustdoc/clean/utils/tests.rs | 41 + src/librustdoc/config.rs | 42 +- src/librustdoc/core.rs | 130 +- src/librustdoc/doctest.rs | 32 +- src/librustdoc/doctree.rs | 13 +- src/librustdoc/fold.rs | 15 +- src/librustdoc/formats/cache.rs | 92 +- src/librustdoc/formats/mod.rs | 4 +- src/librustdoc/html/format.rs | 62 +- src/librustdoc/html/highlight.rs | 54 +- src/librustdoc/html/highlight/tests.rs | 27 +- src/librustdoc/html/layout.rs | 19 +- src/librustdoc/html/markdown.rs | 2 + src/librustdoc/html/render/cache.rs | 49 +- src/librustdoc/html/render/context.rs | 44 +- src/librustdoc/html/render/mod.rs | 522 +- src/librustdoc/html/render/print_item.rs | 203 +- src/librustdoc/html/render/write_shared.rs | 8 +- src/librustdoc/html/sources.rs | 25 +- src/librustdoc/html/static/clipboard.svg | 1 + src/librustdoc/html/static/main.js | 1131 +- src/librustdoc/html/static/noscript.css | 25 - ...oto-sans-kr-v13-korean-regular-LICENSE.txt | 93 + .../noto-sans-kr-v13-korean-regular.woff | Bin 0 -> 287068 bytes src/librustdoc/html/static/rustdoc.css | 694 +- src/librustdoc/html/static/search.js | 365 +- src/librustdoc/html/static/sidebar-items.js | 1 - src/librustdoc/html/static/source-script.js | 109 +- src/librustdoc/html/static/storage.js | 16 +- src/librustdoc/html/static/themes/ayu.css | 92 +- src/librustdoc/html/static/themes/dark.css | 90 +- src/librustdoc/html/static/themes/light.css | 86 +- src/librustdoc/html/static_files.rs | 17 +- src/librustdoc/json/conversions.rs | 60 +- src/librustdoc/json/mod.rs | 29 +- src/librustdoc/lib.rs | 27 +- src/librustdoc/lint.rs | 13 + .../passes/calculate_doc_coverage.rs | 14 +- .../passes/check_code_block_syntax.rs | 120 +- .../passes/collect_intra_doc_links.rs | 76 +- .../passes/collect_intra_doc_links/early.rs | 63 - src/librustdoc/passes/collect_trait_impls.rs | 23 +- src/librustdoc/passes/doc_test_lints.rs | 9 +- src/librustdoc/passes/mod.rs | 2 +- src/librustdoc/passes/strip_hidden.rs | 11 +- src/librustdoc/passes/strip_private.rs | 6 +- src/librustdoc/passes/stripper.rs | 23 +- src/librustdoc/theme.rs | 2 +- src/librustdoc/visit_ast.rs | 18 +- src/rustdoc-json-types/lib.rs | 2 +- src/stage0.txt | 6 +- src/test/assembly/asm/bpf-types.rs | 154 + src/test/assembly/asm/global_asm.rs | 14 + src/test/assembly/asm/powerpc-types.rs | 195 + src/test/assembly/static-relocation-model.rs | 50 +- src/test/codegen/asm-multiple-options.rs | 2 +- src/test/codegen/asm-options.rs | 2 +- src/test/codegen/async-fn-debug-msvc.rs | 38 +- src/test/codegen/async-fn-debug.rs | 29 +- .../codegen/auxiliary/thread_local_aux.rs | 6 + src/test/codegen/bpf-alu32.rs | 11 + src/test/codegen/drop.rs | 9 +- .../codegen/external-no-mangle-statics.rs | 4 - src/test/codegen/function-arguments.rs | 6 +- src/test/codegen/generator-debug-msvc.rs | 38 +- src/test/codegen/generator-debug.rs | 29 +- src/test/codegen/match.rs | 6 +- .../issue-73167-remap-std.rs | 15 + src/test/codegen/thread-local.rs | 50 + src/test/codegen/try_identity.rs | 21 +- src/test/debuginfo/generator-objects.rs | 20 +- src/test/debuginfo/issue-57822.rs | 4 +- src/test/debuginfo/msvc-pretty-enums.rs | 97 + src/test/debuginfo/pretty-std.rs | 7 +- src/test/debuginfo/should-fail.rs | 1 + .../struct_point.rs | 10 +- .../callee_caller_cross_crate/b.rs | 4 +- .../change_add_field/struct_point.rs | 14 +- .../change_private_fn/struct_point.rs | 10 +- .../change_private_fn_cc/struct_point.rs | 10 +- .../struct_point.rs | 10 +- .../struct_point.rs | 10 +- .../struct_point.rs | 10 +- .../struct_point.rs | 10 +- src/test/incremental/crate_hash_reorder.rs | 4 +- src/test/incremental/dirty_clean.rs | 15 +- .../incremental/hashes/call_expressions.rs | 8 +- src/test/incremental/hashes/enum_defs.rs | 20 +- src/test/incremental/hashes/extern_mods.rs | 26 +- .../hashes/indexing_expressions.rs | 42 +- src/test/incremental/hashes/inherent_impls.rs | 15 +- src/test/incremental/hashes/struct_defs.rs | 252 +- src/test/incremental/hashes/trait_defs.rs | 426 +- src/test/incremental/hashes/trait_impls.rs | 161 +- src/test/incremental/hashes/type_defs.rs | 14 +- src/test/incremental/hello_world.rs | 4 +- .../hygiene/auxiliary/cached_hygiene.rs | 2 +- .../ich_method_call_trait_scope.rs | 8 +- src/test/incremental/ich_nested_items.rs | 9 +- src/test/incremental/ich_resolve_results.rs | 12 +- src/test/incremental/issue-61530.rs | 5 +- .../link_order/auxiliary/my_lib.rs | 3 + src/test/incremental/link_order/main.rs | 12 + src/test/incremental/rlib_cross_crate/b.rs | 8 +- src/test/incremental/source_loc_macros.rs | 15 +- .../span_hash_stable/auxiliary/sub1.rs | 2 +- .../span_hash_stable/auxiliary/sub2.rs | 2 +- .../spans_significant_w_debuginfo.rs | 3 +- .../incremental/spans_significant_w_panic.rs | 2 +- src/test/incremental/string_constant.rs | 9 +- src/test/incremental/struct_add_field.rs | 6 +- .../incremental/struct_change_field_name.rs | 6 +- .../incremental/struct_change_field_type.rs | 6 +- .../struct_change_field_type_cross_crate/b.rs | 6 +- src/test/incremental/struct_change_nothing.rs | 6 +- src/test/incremental/struct_remove_field.rs | 6 +- .../incremental/type_alias_cross_crate/b.rs | 8 +- src/test/incremental/unchecked_dirty_clean.rs | 18 +- .../bool_compare.opt1.InstCombine.diff | 35 + .../bool_compare.opt2.InstCombine.diff | 35 + .../bool_compare.opt3.InstCombine.diff | 35 + .../bool_compare.opt4.InstCombine.diff | 35 + src/test/mir-opt/bool_compare.rs | 26 + .../const_debuginfo.main.ConstDebugInfo.diff | 4 +- .../checked_add.main.ConstProp.diff | 2 +- ...l_flow_simplification.hello.ConstProp.diff | 4 +- .../const_prop/indirect.main.ConstProp.diff | 2 +- .../issue_67019.main.ConstProp.diff | 2 +- ...ble_variable_aggregate.main.ConstProp.diff | 2 +- ...es_into_variable.main.ConstProp.32bit.diff | 2 +- ...es_into_variable.main.ConstProp.64bit.diff | 2 +- .../return_place.add.ConstProp.diff | 2 +- ...le_literal_propagation.main.ConstProp.diff | 2 +- ...main-{closure#0}.StateTransform.before.mir | 4 +- .../inline/inline_diverging.g.Inline.diff | 4 +- ...line_into_box_place.main.Inline.32bit.diff | 4 +- ...line_into_box_place.main.Inline.64bit.diff | 4 +- ...issue_62289.test.ElaborateDrops.before.mir | 70 +- ...76432.test.SimplifyComparisonIntegral.diff | 4 +- ...ue_59352.num_to_digit.PreCodegen.after.mir | 4 +- ....main.SimplifyCfg-promote-consts.after.mir | 2 +- ...fg-initial.after-ElaborateDrops.after.diff | 8 +- ...wrap.SimplifyCfg-elaborate-drops.after.mir | 4 +- ..._after_call.main.ElaborateDrops.before.mir | 4 +- ...age_markers.main.RemoveStorageMarkers.diff | 25 +- ...asts.SimplifyCfg-elaborate-drops.after.mir | 199 + ...main.SimplifyCfg-elaborate-drops.after.mir | 46 +- src/test/mir-opt/retag.rs | 14 + src/test/mir-opt/simplify-arm.rs | 17 +- ...mplify_arm.id_try.SimplifyArmIdentity.diff | 140 +- ...implify_arm.id_try.SimplifyBranchSame.diff | 91 +- src/test/mir-opt/simplify_try.rs | 18 +- ...y.try_identity.DestinationPropagation.diff | 93 +- ..._try.try_identity.SimplifyArmIdentity.diff | 136 +- ....try_identity.SimplifyBranchSame.after.mir | 75 +- ..._try.try_identity.SimplifyLocals.after.mir | 34 +- ...age_live_dead_in_statics.XXX.mir_map.0.mir | 4 +- ...after-uninhabited-enum-branching.after.mir | 12 +- ...anching.main.UninhabitedEnumBranching.diff | 20 +- ...le_storage.while_loop.PreCodegen.after.mir | 2 +- src/test/pretty/anonymous-types.rs | 24 + src/test/pretty/asm.pp | 2 +- src/test/pretty/asm.rs | 2 +- src/test/pretty/ast-stmt-expr-attr.rs | 28 +- src/test/pretty/delimited-token-groups.rs | 2 +- src/test/pretty/stmt_expr_attributes.rs | 42 +- .../alloc-extern-crates/Makefile | 5 - .../alloc-extern-crates/fakealloc.rs | 25 - .../alloc-no-oom-handling/Makefile | 4 + .../expected_show_coverage.async2.txt | 13 +- .../expected_show_coverage.closure.txt | 106 +- .../expected_show_coverage.closure_macro.txt | 42 + ...cted_show_coverage.closure_macro_async.txt | 83 + .../expected_show_coverage.conditions.txt | 8 +- .../expected_show_coverage.doctest.txt | 4 +- .../expected_show_coverage.drop_trait.txt | 10 +- .../expected_show_coverage.generator.txt | 32 + .../expected_show_coverage.generics.txt | 28 +- .../expected_show_coverage.inner_items.txt | 6 +- .../expected_show_coverage.issue-83601.txt | 4 +- .../expected_show_coverage.issue-84561.txt | 195 + .../expected_show_coverage.loops_branches.txt | 38 +- .../expected_show_coverage.no_cov_crate.txt | 27 +- .../expected_show_coverage.panic_unwind.txt | 18 - .../expected_show_coverage.tight_inf_loop.txt | 2 +- .../expected_show_coverage.uses_crate.txt | 10 +- ...pected_show_coverage.uses_inline_crate.txt | 12 +- src/test/run-make-fulldeps/coverage/async2.rs | 12 +- .../run-make-fulldeps/coverage/closure.rs | 76 +- .../coverage/closure_macro.rs | 40 + .../coverage/closure_macro_async.rs | 81 + .../run-make-fulldeps/coverage/conditions.rs | 4 +- .../run-make-fulldeps/coverage/generator.rs | 30 + .../run-make-fulldeps/coverage/generics.rs | 10 +- .../run-make-fulldeps/coverage/issue-84561.rs | 182 + .../coverage/loops_branches.rs | 8 +- .../coverage/no_cov_crate.rs | 19 + .../coverage/panic_unwind.rs | 18 - .../hotplug_codegen_backend/the_backend.rs | 35 +- .../include_bytes_deps/main.rs | 4 +- .../run-make-fulldeps/link-dedup/Makefile | 12 + src/test/run-make-fulldeps/link-dedup/depa.rs | 7 + src/test/run-make-fulldeps/link-dedup/depb.rs | 8 + src/test/run-make-fulldeps/link-dedup/depc.rs | 4 + .../run-make-fulldeps/link-dedup/empty.rs | 5 + .../pgo-branch-weights/Makefile | 1 + .../print-unversioned-files/Makefile | 4 - .../unversioned-files.txt | 16 - .../rustdoc-target-spec-json-path/Makefile | 9 + .../dummy_core.rs | 2 + .../rustdoc-target-spec-json-path/my_crate.rs | 3 + .../rustdoc-target-spec-json-path/target.json | 39 + .../run-make-fulldeps/save-analysis/foo.rs | 7 +- src/test/run-make/emit-named-files/Makefile | 33 + src/test/run-make/emit-named-files/foo.rs | 1 + .../incremental-session-fail/Makefile | 14 + .../run-make/incremental-session-fail/foo.rs | 1 + src/test/run-make/issue-71519/Makefile | 6 + src/test/run-make/issue-71519/main.rs | 4 + src/test/run-make/raw-dylib/Makefile | 21 + src/test/run-make/raw-dylib/driver.rs | 5 + src/test/run-make/raw-dylib/extern_1.c | 16 + src/test/run-make/raw-dylib/extern_2.c | 6 + src/test/run-make/raw-dylib/lib.rs | 22 + src/test/run-make/raw-dylib/output.txt | 3 + .../run-make/unstable-flag-required/Makefile | 1 + .../unstable-flag-required/force-warns.stderr | 2 + .../enclave/src/main.rs | 2 +- src/test/rustdoc-gui/basic-code.goml | 3 - src/test/rustdoc-gui/basic.goml | 4 - .../rustdoc-gui/check_info_sign_position.goml | 9 - src/test/rustdoc-gui/code-sidebar-toggle.goml | 6 - src/test/rustdoc-gui/hash-item-expansion.goml | 18 - .../rustdoc-gui/impl-default-expansion.goml | 3 - src/test/rustdoc-gui/list_code_block.goml | 3 - src/test/rustdoc-gui/search-input-mobile.goml | 11 - ...rch-tab-selection-if-current-is-empty.goml | 21 - src/test/rustdoc-gui/shortcuts.goml | 26 - src/test/rustdoc-gui/{ => src}/lib.rs | 8 + src/test/rustdoc-gui/src/lib2.rs | 11 + .../rustdoc-gui/src/settings/.package-cache | 0 src/test/rustdoc-gui/theme-change.goml | 10 - src/test/rustdoc-gui/toggle-docs.goml | 7 - .../rustdoc-gui/trait-sidebar-item-order.goml | 7 - src/test/rustdoc-gui/type-weight.rs | 2 + src/test/rustdoc-ui/auxiliary/panic-item.rs | 17 - src/test/rustdoc-ui/check.rs | 1 + src/test/rustdoc-ui/check.stderr | 16 +- src/test/rustdoc-ui/doc-attr2.rs | 11 - src/test/rustdoc-ui/doc-attr2.stderr | 26 - src/test/rustdoc-ui/doc-include-suggestion.rs | 10 + .../rustdoc-ui/doc-include-suggestion.stderr | 12 + src/test/rustdoc-ui/doc-spotlight.fixed | 5 +- src/test/rustdoc-ui/doc-spotlight.rs | 5 +- src/test/rustdoc-ui/doc-spotlight.stderr | 13 +- .../failed-doctest-compile-fail.stdout | 2 +- .../failed-doctest-missing-codes.stdout | 2 +- src/test/rustdoc-ui/ignore-block-help.rs | 5 +- src/test/rustdoc-ui/ignore-block-help.stderr | 10 +- .../intra-doc/email-address-localhost.rs | 1 + .../intra-doc/email-address-localhost.stderr | 5 +- .../intra-doc/unknown-disambiguator.rs | 1 + .../intra-doc/unknown-disambiguator.stderr | 25 +- src/test/rustdoc-ui/intra-doc/warning.stderr | 2 +- src/test/rustdoc-ui/invalid-cfg.rs | 4 + src/test/rustdoc-ui/invalid-cfg.stderr | 14 + src/test/rustdoc-ui/invalid-doc-attr.rs | 32 + src/test/rustdoc-ui/invalid-doc-attr.stderr | 78 + src/test/rustdoc-ui/invalid-syntax.rs | 2 +- src/test/rustdoc-ui/invalid-syntax.stderr | 3 +- src/test/rustdoc-ui/issue-80992.stdout | 2 +- .../rustdoc-ui/no-crate-level-doc-lint.rs | 1 + .../rustdoc-ui/no-crate-level-doc-lint.stderr | 4 +- src/test/rustdoc-ui/no-run-flag-error.rs | 6 + src/test/rustdoc-ui/no-run-flag-error.stderr | 2 + src/test/rustdoc-ui/no-run-flag.rs | 38 + src/test/rustdoc-ui/no-run-flag.stdout | 12 + src/test/rustdoc-ui/run-directory.rs | 4 +- src/test/rustdoc-ui/test-type.rs | 26 + src/test/rustdoc-ui/test-type.stdout | 10 + src/test/rustdoc-ui/unused-extern-crate.rs | 3 - src/test/rustdoc-ui/wasm-safe.rs | 5 + src/test/rustdoc/assoc-consts.rs | 8 +- src/test/rustdoc/assoc-types.rs | 6 +- src/test/rustdoc/async-fn.rs | 12 +- src/test/rustdoc/auto_aliases.rs | 2 +- .../rustdoc/auxiliary/external-cross-doc.md | 2 +- src/test/rustdoc/auxiliary/external-cross.rs | 6 +- src/test/rustdoc/auxiliary/external-doc.md | 2 +- src/test/rustdoc/auxiliary/issue-66159-1.rs | 2 + .../rustdoc/auxiliary/trait-alias-mention.rs | 3 + .../rustdoc/auxiliary/trait-visibility.rs | 3 + src/test/rustdoc/blanket-reexport-item.rs | 2 +- src/test/rustdoc/const-display.rs | 8 +- src/test/rustdoc/const-fn.rs | 2 +- src/test/rustdoc/const-generics/add-impl.rs | 2 +- .../const-generics/const-generic-defaults.rs | 6 + .../const-generics/const-generic-slice.rs | 2 +- .../const-generics/const-generics-docs.rs | 12 +- src/test/rustdoc/const-generics/const-impl.rs | 10 +- .../const-equate-pred.rs | 2 +- src/test/rustdoc/const.rs | 2 +- src/test/rustdoc/deref-typedef.rs | 2 +- src/test/rustdoc/doc-assoc-item.rs | 2 +- src/test/rustdoc/doc-cfg.rs | 8 + .../rustdoc/duplicate_impls/issue-33054.rs | 10 +- src/test/rustdoc/empty-impls.rs | 6 +- src/test/rustdoc/ensure-src-link.rs | 2 +- src/test/rustdoc/extern-impl-trait.rs | 4 +- src/test/rustdoc/extern-impl.rs | 16 +- src/test/rustdoc/extern-method.rs | 8 +- src/test/rustdoc/external-doc.rs | 10 - src/test/rustdoc/generic-impl.rs | 4 +- src/test/rustdoc/impl-disambiguation.rs | 10 +- src/test/rustdoc/impl-parts.rs | 4 +- src/test/rustdoc/inline_cross/assoc-items.rs | 6 +- .../inline_cross/impl-inline-without-trait.rs | 2 +- src/test/rustdoc/inline_cross/impl_trait.rs | 4 +- .../rustdoc/inline_cross/issue-31948-1.rs | 20 +- .../rustdoc/inline_cross/issue-31948-2.rs | 12 +- src/test/rustdoc/inline_cross/issue-31948.rs | 20 +- src/test/rustdoc/inline_cross/issue-32881.rs | 4 +- src/test/rustdoc/inline_cross/issue-33113.rs | 4 +- src/test/rustdoc/inline_cross/trait-vis.rs | 2 +- src/test/rustdoc/inline_local/trait-vis.rs | 4 +- .../rustdoc/intra-doc/associated-items.rs | 6 +- src/test/rustdoc/intra-doc/builtin-macros.rs | 2 +- src/test/rustdoc/intra-doc/field.rs | 4 + src/test/rustdoc/intra-doc/generic-params.rs | 28 +- .../rustdoc/intra-doc/non-path-primitives.rs | 34 +- src/test/rustdoc/intra-doc/prim-assoc.rs | 2 +- .../intra-doc/prim-methods-external-core.rs | 4 +- .../rustdoc/intra-doc/prim-methods-local.rs | 4 +- src/test/rustdoc/intra-doc/prim-methods.rs | 4 +- src/test/rustdoc/intra-doc/prim-precedence.rs | 4 +- .../intra-doc/primitive-disambiguators.rs | 2 +- .../intra-doc/primitive-non-default-impl.rs | 20 +- src/test/rustdoc/intra-doc/pub-use.rs | 4 +- src/test/rustdoc/intra-doc/trait-item.rs | 2 +- src/test/rustdoc/intra-doc/true-false.rs | 4 +- src/test/rustdoc/intra-link-prim-self.rs | 7 +- src/test/rustdoc/issue-19190.rs | 4 +- src/test/rustdoc/issue-21474.rs | 2 +- src/test/rustdoc/issue-25001.rs | 18 +- src/test/rustdoc/issue-27362.rs | 2 +- src/test/rustdoc/issue-29503.rs | 2 +- src/test/rustdoc/issue-33302.rs | 6 +- src/test/rustdoc/issue-33592.rs | 4 +- src/test/rustdoc/issue-35169-2.rs | 14 +- src/test/rustdoc/issue-35169.rs | 14 +- src/test/rustdoc/issue-45584.rs | 8 +- src/test/rustdoc/issue-46727.rs | 2 +- src/test/rustdoc/issue-50159.rs | 6 +- src/test/rustdoc/issue-51236.rs | 4 +- src/test/rustdoc/issue-53689.rs | 2 +- src/test/rustdoc/issue-53812.rs | 11 +- src/test/rustdoc/issue-54705.rs | 8 +- src/test/rustdoc/issue-55321.rs | 14 +- src/test/rustdoc/issue-55364.rs | 20 +- src/test/rustdoc/issue-56822.rs | 4 +- src/test/rustdoc/issue-60726.rs | 8 +- src/test/rustdoc/issue-66159.rs | 10 + src/test/rustdoc/issue-75588.rs | 4 +- src/test/rustdoc/issue-76501.rs | 3 +- src/test/rustdoc/issue-78673.rs | 8 +- .../issue-80233-normalize-auto-trait.rs | 2 +- .../issue-82465-asref-for-and-of-local.rs | 4 +- src/test/rustdoc/issue-85454.rs | 17 + src/test/rustdoc/keyword.rs | 3 +- src/test/rustdoc/manual_impl.rs | 16 +- src/test/rustdoc/mut-params.rs | 2 +- src/test/rustdoc/negative-impl-sidebar.rs | 2 +- src/test/rustdoc/negative-impl.rs | 6 +- src/test/rustdoc/primitive-generic-impl.rs | 2 +- src/test/rustdoc/primitive-link.rs | 10 +- src/test/rustdoc/primitive-reexport.rs | 12 +- src/test/rustdoc/pub-method.rs | 4 +- src/test/rustdoc/sidebar-items.rs | 14 +- .../rustdoc/sidebar-links-to-foreign-impl.rs | 6 +- src/test/rustdoc/sized_trait.rs | 6 +- src/test/rustdoc/spotlight-from-dependency.rs | 2 +- src/test/rustdoc/src-links-auto-impls.rs | 12 +- src/test/rustdoc/synthetic_auto/basic.rs | 8 +- src/test/rustdoc/synthetic_auto/complex.rs | 4 +- .../rustdoc/synthetic_auto/crate-local.rs | 6 +- src/test/rustdoc/synthetic_auto/lifetimes.rs | 8 +- src/test/rustdoc/synthetic_auto/manual.rs | 10 +- src/test/rustdoc/synthetic_auto/negative.rs | 8 +- src/test/rustdoc/synthetic_auto/nested.rs | 6 +- .../rustdoc/synthetic_auto/no-redundancy.rs | 4 +- src/test/rustdoc/synthetic_auto/overflow.rs | 2 +- src/test/rustdoc/synthetic_auto/project.rs | 9 +- .../synthetic_auto/self-referential.rs | 4 +- .../rustdoc/synthetic_auto/static-region.rs | 4 +- ...e-threshold.rs => toggle-item-contents.rs} | 22 +- src/test/rustdoc/toggle-method.rs | 18 + src/test/rustdoc/toggle-trait-fn.rs | 23 + src/test/rustdoc/trait-alias-mention.rs | 10 + src/test/rustdoc/trait-attributes.rs | 6 +- .../trait-impl-items-links-and-anchors.rs | 68 +- src/test/rustdoc/trait-impl.rs | 31 +- src/test/rustdoc/trait-visibility.rs | 8 + src/test/rustdoc/trait_alias.rs | 2 + src/test/rustdoc/traits-in-bodies.rs | 6 +- src/test/rustdoc/type-layout-flag-required.rs | 4 + src/test/rustdoc/type-layout.rs | 54 + src/test/rustdoc/typedef.rs | 6 +- src/test/rustdoc/unindent.rs | 6 +- src/test/rustdoc/visibility.rs | 12 +- src/test/rustdoc/where.rs | 10 +- .../hash-stable-is-unstable.stderr | 2 +- .../lint_pass_impl_without_macro.stderr | 2 +- .../internal-lints/pass_ty_by_ref_self.rs | 1 + .../internal-lints/pass_ty_by_ref_self.stderr | 6 +- src/test/ui-fulldeps/pprust-expr-roundtrip.rs | 127 +- src/test/ui-fulldeps/session-derive-errors.rs | 6 + .../ui-fulldeps/session-derive-errors.stderr | 87 +- src/test/ui/allocator/not-an-allocator.stderr | 8 +- src/test/ui/allocator/two-allocators.stderr | 2 +- src/test/ui/annotate-snippet/missing-type.rs | 2 +- src/test/ui/annotate-snippet/multispan.rs | 2 +- src/test/ui/asm/bad-arch.mirunsafeck.stderr | 16 + src/test/ui/asm/bad-arch.rs | 11 +- src/test/ui/asm/bad-arch.stderr | 8 - src/test/ui/asm/bad-arch.thirunsafeck.stderr | 16 + src/test/ui/asm/bad-options.rs | 15 +- src/test/ui/asm/bad-options.stderr | 38 +- .../ui/asm/bad-template.mirunsafeck.stderr | 187 + src/test/ui/asm/bad-template.rs | 23 +- src/test/ui/asm/bad-template.stderr | 102 - .../ui/asm/bad-template.thirunsafeck.stderr | 187 + src/test/ui/asm/const.rs | 8 +- src/test/ui/asm/duplicate-options.fixed | 5 +- src/test/ui/asm/duplicate-options.rs | 5 +- src/test/ui/asm/duplicate-options.stderr | 8 +- src/test/ui/asm/inline-syntax.arm.stderr | 28 +- src/test/ui/asm/inline-syntax.rs | 8 + src/test/ui/asm/inline-syntax.x86_64.stderr | 24 +- src/test/ui/asm/interpolated-idents.stderr | 6 +- src/test/ui/asm/naked-functions.stderr | 2 +- src/test/ui/asm/parse-error.rs | 36 +- src/test/ui/asm/parse-error.stderr | 112 +- src/test/ui/asm/type-check-1.rs | 22 +- src/test/ui/asm/type-check-1.stderr | 34 +- src/test/ui/asm/type-check-2.rs | 8 - src/test/ui/asm/type-check-2.stderr | 30 +- src/test/ui/asm/type-check-3.rs | 20 +- src/test/ui/asm/type-check-3.stderr | 27 +- .../defaults-not-assumed-fail.stderr | 2 +- ...ssociated-type-destructuring-assignment.rs | 11 + .../associated-types/associated-type-macro.rs | 4 + .../associated-type-macro.stderr | 8 + .../associated-type-struct-construction.rs | 24 + ...sociated-type-tuple-struct-construction.rs | 24 + ...ated-type-tuple-struct-construction.stderr | 19 + src/test/ui/async-await/async-await.rs | 3 +- ...async-borrowck-escaping-block-error.stderr | 6 +- ...> async-unsafe-fn-call-in-safe.mir.stderr} | 8 +- .../async-unsafe-fn-call-in-safe.rs | 6 +- .../async-unsafe-fn-call-in-safe.thir.stderr | 19 + src/test/ui/async-await/issue-61076.rs | 4 +- src/test/ui/async-await/issue-61076.stderr | 4 +- src/test/ui/async-await/issue-73541-2.stderr | 2 +- .../ui/async-await/issue-73741-type-err.rs | 14 + .../async-await/issue-73741-type-err.stderr | 11 + src/test/ui/async-await/issues/issue-65159.rs | 2 +- .../ui/async-await/issues/issue-65159.stderr | 10 +- .../multiple-lifetimes/ret-impl-trait-fg.rs | 7 +- .../ret-impl-trait-no-fg.rs | 20 - .../ret-impl-trait-no-fg.stderr | 43 - .../ret-impl-trait-one.nll.stderr | 2 +- .../multiple-lifetimes/ret-impl-trait-one.rs | 2 - .../ret-impl-trait-one.stderr | 2 +- .../async-await/try-on-option-in-async.stderr | 30 +- src/test/ui/attributes/doc-attr2.rs | 11 - src/test/ui/attributes/doc-attr2.stderr | 26 - src/test/ui/attributes/invalid-doc-attr.rs | 32 + .../ui/attributes/invalid-doc-attr.stderr | 78 + .../attributes/key-value-expansion-on-mac.rs | 1 - .../key-value-expansion-on-mac.stderr | 2 +- .../ui/attributes/key-value-expansion.stderr | 4 +- .../attributes/nonterminal-expansion.stderr | 4 +- src/test/ui/backtrace-debuginfo.rs | 7 +- src/test/ui/binop/issue-77910-1.stderr | 4 +- .../borrowck-borrowed-uniq-rvalue-2.stderr | 2 +- .../borrowck-closures-mut-and-imm.stderr | 8 +- .../borrowck-closures-mut-of-imm.stderr | 4 +- .../borrowck-closures-mut-of-mut.stderr | 4 +- .../borrowck-closures-slice-patterns.stderr | 8 +- .../borrowck-closures-two-mut-fail.stderr | 4 +- .../borrowck/borrowck-closures-two-mut.stderr | 4 +- .../borrowck/borrowck-closures-unique.stderr | 2 +- .../borrowck-closures-use-after-free.stderr | 2 +- .../borrowck-insert-during-each.stderr | 4 +- .../borrowck-loan-blocks-move-cc.stderr | 4 +- .../ui/borrowck/borrowck-loan-rcvr.stderr | 2 +- .../borrowck/borrowck-move-by-capture.stderr | 8 +- ...rowck-move-moved-value-into-closure.stderr | 4 +- src/test/ui/borrowck/issue-25793.stderr | 2 +- ...27282-mutate-before-diverging-arm-2.stderr | 2 +- ...sue-27282-reborrow-ref-mut-in-guard.stderr | 2 +- src/test/ui/borrowck/issue-64453.stderr | 4 +- .../issue-82126-mismatched-subst-and-hir.rs | 2 +- ...ssue-82126-mismatched-subst-and-hir.stderr | 8 +- src/test/ui/borrowck/issue-85765.rs | 8 + src/test/ui/borrowck/issue-85765.stderr | 12 + .../ui/borrowck/move-error-snippets.stderr | 2 +- src/test/ui/bound-suggestions.stderr | 12 +- src/test/ui/c-variadic/variadic-ffi-4.stderr | 8 + ...tderr => cast-ptr-to-int-const.mir.stderr} | 4 +- src/test/ui/cast/cast-ptr-to-int-const.rs | 20 +- .../ui/cast/cast-ptr-to-int-const.thir.stderr | 19 + ...eature-gate-const_raw_ptr_to_usize_cast.rs | 13 + ...e-gate-const_raw_ptr_to_usize_cast.stderr} | 17 +- src/test/ui/cast/issue-84213.fixed | 1 + src/test/ui/cast/issue-84213.rs | 1 + src/test/ui/cast/issue-84213.stderr | 8 +- src/test/ui/cfg/cfg-family.rs | 4 +- src/test/ui/cfg/cfg-panic.rs | 1 + src/test/ui/cfg/cfg-target-family.rs | 7 +- .../diagnostics/borrowck/borrowck-1.rs | 20 + .../diagnostics/borrowck/borrowck-1.stderr | 28 + .../diagnostics/borrowck/borrowck-2.rs | 20 + .../diagnostics/borrowck/borrowck-2.stderr | 28 + .../diagnostics/borrowck/borrowck-3.rs | 19 + .../diagnostics/borrowck/borrowck-3.stderr | 27 + .../diagnostics/borrowck/borrowck-4.rs | 21 + .../diagnostics/borrowck/borrowck-4.stderr | 31 + .../borrowck/borrowck-closures-mut-and-imm.rs | 26 + .../borrowck-closures-mut-and-imm.stderr | 30 + .../diagnostics/box.stderr | 6 +- .../diagnostics/cant-mutate-imm-borrow.stderr | 2 +- .../diagnostics/cant-mutate-imm.rs | 4 +- .../diagnostics/cant-mutate-imm.stderr | 4 +- .../diagnostics/multilevel-path.stderr | 2 +- .../diagnostics/mut_ref.stderr | 4 +- .../simple-struct-min-capture.stderr | 4 +- .../migrations/auto_traits.fixed | 67 + .../migrations/auto_traits.rs | 67 + .../migrations/auto_traits.stderr | 69 + .../migrations/insignificant_drop.fixed | 2 +- .../migrations/insignificant_drop.rs | 2 +- .../migrations/insignificant_drop.stderr | 4 +- .../insignificant_drop_attr_migrations.fixed | 67 + .../insignificant_drop_attr_migrations.rs | 67 + .../insignificant_drop_attr_migrations.stderr | 47 + .../insignificant_drop_attr_no_migrations.rs | 45 + .../migrations/issue-78720.rs | 2 +- .../migrations/migrations_rustfix.fixed | 2 +- .../migrations/migrations_rustfix.rs | 2 +- .../migrations/migrations_rustfix.stderr | 4 +- .../migrations/mir_calls_to_shims.fixed | 39 + .../migrations/mir_calls_to_shims.rs | 39 + .../migrations/mir_calls_to_shims.stderr | 27 + .../migrations/no_migrations.rs | 2 +- .../migrations/precise.fixed | 2 +- .../migrations/precise.rs | 2 +- .../migrations/precise.stderr | 4 +- .../migrations/precise_no_migrations.rs | 2 +- .../migrations/significant_drop.fixed | 2 +- .../migrations/significant_drop.rs | 2 +- .../migrations/significant_drop.stderr | 4 +- ...ds-static-cant-capture-borrowed.nll.stderr | 2 - ...bounds-static-cant-capture-borrowed.stderr | 2 - ...re_no_cap_coerce_many_unsafe_0.mir.stderr} | 4 +- .../closure_no_cap_coerce_many_unsafe_0.rs | 3 + ...re_no_cap_coerce_many_unsafe_0.thir.stderr | 19 + ...nsafe-closure-to-unsafe-fn-ptr.mir.stderr} | 2 +- .../coerce-unsafe-closure-to-unsafe-fn-ptr.rs | 3 + ...nsafe-closure-to-unsafe-fn-ptr.thir.stderr | 11 + .../ui/codemap_tests/bad-format-args.stderr | 2 +- src/test/ui/codemap_tests/issue-28308.stderr | 2 +- src/test/ui/command/command-pre-exec.rs | 2 + .../cfg-attr-syntax-validation.stderr | 4 +- ...r-unknown-attribute-macro-expansion.stderr | 2 +- .../issue-18343.stderr | 2 +- .../issue-2392.stderr | 12 +- .../array-impls/into-iter-impls-length-32.rs | 14 +- .../array-impls/into-iter-impls-length-33.rs | 14 +- ...ram-type-depends-on-type-param.full.stderr | 1 + ...aram-type-depends-on-type-param.min.stderr | 1 + .../const_evaluatable_checked/different-fn.rs | 1 + .../different-fn.stderr | 12 +- .../evaluated-to-ambig.rs | 22 + .../from-sig-fail.rs | 2 +- .../from-sig-fail.stderr | 2 +- .../simple_fail.full.stderr | 6 +- .../simple_fail.min.stderr | 2 +- .../const_evaluatable_checked/simple_fail.rs | 5 +- .../defaults/const-param-as-default-value.rs | 1 - .../defaults/const-param-in-ty-defaults.rs | 1 - .../defaults/default-param-wf-concrete.rs | 1 - .../defaults/default-param-wf-concrete.stderr | 2 +- .../defaults/forward-declared.rs | 15 + .../defaults/forward-declared.stderr | 27 + .../defaults/intermixed-lifetime.min.stderr | 4 +- .../defaults/mismatch.full.stderr | 14 +- .../defaults/mismatch.min.stderr | 14 +- .../ui/const-generics/defaults/mismatch.rs | 7 +- .../param-order-err-pretty-prints-default.rs | 5 + ...ram-order-err-pretty-prints-default.stderr | 8 + .../defaults/pretty-printing-ast.rs | 1 - .../defaults/pretty-printing-ast.stdout | 1 - .../defaults/repr-c-issue-82792.rs | 1 - src/test/ui/const-generics/diagnostics.stderr | 22 +- ...incorrect-number-of-const-args.full.stderr | 18 +- .../incorrect-number-of-const-args.min.stderr | 18 +- .../incorrect-number-of-const-args.rs | 4 +- .../invalid-const-arg-for-type-param.rs | 4 +- .../invalid-const-arg-for-type-param.stderr | 12 +- .../invalid-constant-in-args.stderr | 2 +- .../ui/const-generics/issue-67375.full.stderr | 1 + .../ui/const-generics/issue-67375.min.stderr | 1 + .../const-generics/issue-67945-1.full.stderr | 1 + .../const-generics/issue-67945-1.min.stderr | 1 + .../const-generics/issue-67945-2.full.stderr | 1 + .../const-generics/issue-67945-2.min.stderr | 1 + .../issues/issue-62504.full.stderr | 12 +- .../ui/const-generics/issues/issue-62504.rs | 2 +- .../ui/const-generics/issues/issue-76595.rs | 2 +- .../const-generics/issues/issue-76595.stderr | 2 +- .../ui/const-generics/issues/issue-83466.rs | 17 + .../const-generics/issues/issue-83466.stderr | 22 + .../macro_rules-braces.full.stderr | 8 +- ...const-expression-suggest-missing-braces.rs | 1 + ...t-expression-suggest-missing-braces.stderr | 22 +- .../default_function_param.rs | 1 - .../default_function_param.stderr | 2 +- .../min_const_generics/macro-fail.stderr | 6 +- .../transmute-const-param-static-reference.rs | 10 - ...e-const-param-static-reference.min.stderr} | 2 +- .../transmute-const-param-static-reference.rs | 16 + .../unused-type-param-suggestion.rs | 4 + .../unused-type-param-suggestion.stderr | 12 + src/test/ui/constructor-lifetime-args.rs | 8 +- src/test/ui/constructor-lifetime-args.stderr | 8 +- src/test/ui/consts/async-block.rs | 17 +- src/test/ui/consts/async-block.stderr | 8 - ...stderr => async-block.with_feature.stderr} | 2 +- .../consts/async-block.without_feature.stderr | 21 + .../auxiliary/post_monomorphization_error.rs | 20 + src/test/ui/consts/const-eval/const_panic.rs | 10 - .../ui/consts/const-eval/const_panic.stderr | 98 +- .../const-eval/const_panic_libcore_bin.rs | 3 - .../const-eval/const_panic_libcore_bin.stderr | 28 +- src/test/ui/consts/const-eval/dangling.rs | 6 +- src/test/ui/consts/const-eval/dangling.stderr | 1 - .../ui/consts/const-eval/double_promotion.rs | 17 - .../ui/consts/const-eval/erroneous-const.rs | 2 + .../consts/const-eval/erroneous-const.stderr | 6 +- .../ui/consts/const-eval/erroneous-const2.rs | 21 + .../consts/const-eval/erroneous-const2.stderr | 37 + .../feature-gate-const_panic.stderr | 6 +- .../ui/consts/const-eval/issue-50814-2.stderr | 2 +- .../ui/consts/const-eval/issue-50814.stderr | 2 +- src/test/ui/consts/const-eval/issue-85155.rs | 21 + .../ui/consts/const-eval/issue-85155.stderr | 15 + .../const-eval/panic-assoc-never-type.rs | 3 +- .../const-eval/panic-assoc-never-type.stderr | 15 +- .../ui/consts/const-eval/panic-never-type.rs | 6 +- .../consts/const-eval/panic-never-type.stderr | 23 +- .../const-eval/promoted_errors.noopt.stderr | 10 +- .../const-eval/promoted_errors.opt.stderr | 10 +- ...ted_errors.opt_with_overflow_checks.stderr | 10 +- .../ui/consts/const-eval/promoted_errors.rs | 24 +- .../consts/const-eval/ub-incorrect-vtable.rs | 21 + .../const-eval/ub-incorrect-vtable.stderr | 27 + .../consts/const-eval/ub-nonnull.32bit.stderr | 2 +- .../consts/const-eval/ub-nonnull.64bit.stderr | 2 +- src/test/ui/consts/const-eval/ub-nonnull.rs | 2 +- .../consts/const-eval/ub-ref-ptr.32bit.stderr | 4 +- .../consts/const-eval/ub-ref-ptr.64bit.stderr | 4 +- .../consts/const-eval/ub-upvars.32bit.stderr | 2 +- .../consts/const-eval/ub-upvars.64bit.stderr | 2 +- .../const-eval/ub-wide-ptr.32bit.stderr | 2 +- .../const-eval/ub-wide-ptr.64bit.stderr | 2 +- src/test/ui/consts/const-eval/unwind-abort.rs | 3 +- .../ui/consts/const-eval/unwind-abort.stderr | 10 +- ...onst-extern-fn-requires-unsafe.mir.stderr} | 4 +- .../const-extern-fn-requires-unsafe.rs | 5 +- ...onst-extern-fn-requires-unsafe.thir.stderr | 11 + .../const-external-macro-const-err.stderr | 2 +- .../ui/consts/const-needs_drop-monomorphic.rs | 17 + .../const-needs_drop-monomorphic.stderr | 20 + src/test/ui/consts/const-points-to-static.rs | 4 +- src/test/ui/consts/const-unwrap.stderr | 8 +- src/test/ui/consts/const_arg_local.rs | 13 - src/test/ui/consts/const_arg_local.stderr | 8 - src/test/ui/consts/const_arg_promotable.rs | 12 - .../ui/consts/const_arg_promotable.stderr | 8 - src/test/ui/consts/const_arg_promotable2.rs | 18 - .../ui/consts/const_arg_promotable2.stderr | 8 - src/test/ui/consts/const_arg_wrapper.rs | 12 - src/test/ui/consts/const_arg_wrapper.stderr | 8 - src/test/ui/consts/const_fn_unsize.rs | 23 +- .../ui/consts/const_fn_unsize.stock.stderr | 12 - .../consts/const_unsafe_unreachable_ub.stderr | 2 +- .../control-flow/assert.const_panic.stderr | 8 +- src/test/ui/consts/control-flow/assert.rs | 1 - .../consts/control-flow/assert.stock.stderr | 4 +- .../ui/consts/control-flow/issue-50577.stderr | 2 +- src/test/ui/consts/enum-discr-type-err.stderr | 4 +- src/test/ui/consts/inline_asm.stderr | 2 +- src/test/ui/consts/issue-32829.stderr | 2 +- .../issue-66693-panic-in-array-len.stderr | 4 +- src/test/ui/consts/issue-66693.stderr | 6 +- src/test/ui/consts/issue-76064.stderr | 2 +- src/test/ui/consts/issue-83182.32bit.stderr | 14 + src/test/ui/consts/issue-83182.64bit.stderr | 14 + src/test/ui/consts/issue-83182.rs | 8 + .../min_const_fn/bad_const_fn_body_ice.stderr | 4 +- .../ui/consts/min_const_fn/min_const_fn.rs | 6 +- .../consts/min_const_fn/min_const_fn.stderr | 34 +- .../consts/min_const_fn/min_const_fn_dyn.rs | 2 +- .../min_const_fn/min_const_fn_dyn.stderr | 6 +- .../const_refers_to_static2.32bit.stderr | 12 +- .../const_refers_to_static2.64bit.stderr | 12 +- .../miri_unleashed/const_refers_to_static2.rs | 8 +- ..._refers_to_static_cross_crate.32bit.stderr | 56 +- ..._refers_to_static_cross_crate.64bit.stderr | 56 +- .../const_refers_to_static_cross_crate.rs | 23 +- .../consts/miri_unleashed/inline_asm.stderr | 4 +- src/test/ui/consts/offset_from_ub.stderr | 2 +- src/test/ui/consts/offset_ub.stderr | 8 +- src/test/ui/consts/ptr_comparisons.rs | 12 +- src/test/ui/consts/ptr_comparisons.stderr | 9 +- .../ui/consts/rustc-args-required-const.rs | 27 - .../consts/rustc-args-required-const.stderr | 14 - src/test/ui/consts/unsizing-cast-non-null.rs | 10 - .../ui/consts/unsizing-cast-non-null.stderr | 12 - .../cross-crate/auxiliary/static_init_aux.rs | 10 + src/test/ui/cross-crate/static-init.rs | 15 + .../cross-crate-macro-backtrace/main.stderr | 2 +- .../ui/cross/cross-file-errors/main.stderr | 4 +- .../ui/custom_test_frameworks/mismatch.stderr | 2 +- .../deduplicate-diagnostics.duplicate.stderr | 8 +- src/test/ui/deduplicate-diagnostics.rs | 1 - src/test/ui/dep-graph/dep-graph-check-attr.rs | 2 +- .../ui/dep-graph/dep-graph-check-attr.stderr | 2 +- .../ui/dep-graph/dep-graph-variance-alias.rs | 2 +- .../ui/deprecation/deprecation-lint-2.stderr | 2 +- .../ui/deprecation/deprecation-lint-3.stderr | 2 +- src/test/ui/deprecation/deprecation-lint.rs | 18 + .../ui/deprecation/deprecation-lint.stderr | 64 +- ...84637-deprecated-associated-function.fixed | 9 + ...ue-84637-deprecated-associated-function.rs | 9 + ...4637-deprecated-associated-function.stderr | 20 + src/test/ui/deref-suggestion.stderr | 4 +- .../ui/derives/derive-macro-const-default.rs | 1 - ...ives-span-Clone-enum-struct-variant.stderr | 2 +- .../ui/derives/derives-span-Clone-enum.stderr | 2 +- .../derives/derives-span-Clone-struct.stderr | 2 +- .../derives-span-Clone-tuple-struct.stderr | 2 +- ...ives-span-Debug-enum-struct-variant.stderr | 2 +- .../ui/derives/derives-span-Debug-enum.stderr | 2 +- .../derives/derives-span-Debug-struct.stderr | 2 +- .../derives-span-Debug-tuple-struct.stderr | 2 +- .../derives-span-Default-struct.stderr | 2 +- .../derives-span-Default-tuple-struct.stderr | 2 +- ...derives-span-Eq-enum-struct-variant.stderr | 2 +- .../ui/derives/derives-span-Eq-enum.stderr | 2 +- .../ui/derives/derives-span-Eq-struct.stderr | 2 +- .../derives-span-Eq-tuple-struct.stderr | 2 +- ...rives-span-Hash-enum-struct-variant.stderr | 2 +- .../ui/derives/derives-span-Hash-enum.stderr | 2 +- .../derives/derives-span-Hash-struct.stderr | 2 +- .../derives-span-Hash-tuple-struct.stderr | 2 +- ...erives-span-Ord-enum-struct-variant.stderr | 2 +- .../ui/derives/derives-span-Ord-enum.stderr | 2 +- .../ui/derives/derives-span-Ord-struct.stderr | 2 +- .../derives-span-Ord-tuple-struct.stderr | 2 +- ...-span-PartialEq-enum-struct-variant.stderr | 4 +- .../derives-span-PartialEq-enum.stderr | 4 +- .../derives-span-PartialEq-struct.stderr | 4 +- ...derives-span-PartialEq-tuple-struct.stderr | 4 +- ...span-PartialOrd-enum-struct-variant.stderr | 2 +- .../derives-span-PartialOrd-enum.stderr | 2 +- .../derives-span-PartialOrd-struct.stderr | 2 +- ...erives-span-PartialOrd-tuple-struct.stderr | 2 +- ...eriving-no-inner-impl-error-message.stderr | 6 +- .../derives/deriving-with-repr-packed.stderr | 8 +- .../ui/did_you_mean/bad-assoc-expr.stderr | 2 +- src/test/ui/did_you_mean/bad-assoc-pat.stderr | 4 +- src/test/ui/did_you_mean/bad-assoc-ty.stderr | 4 +- .../did_you_mean/recursion_limit_macro.stderr | 2 +- .../macro-expanded-mod.stderr | 2 +- .../dollar-crate-is-keyword-2.stderr | 6 +- .../dollar-crate-is-keyword.stderr | 8 +- .../ui/dyn-keyword/dyn-2018-edition-lint.rs | 16 + .../dyn-keyword/dyn-2018-edition-lint.stderr | 34 + .../ui/dyn-keyword/dyn-2021-edition-error.rs | 12 + .../dyn-keyword/dyn-2021-edition-error.stderr | 36 + .../ui/editions/edition-imports-2015.stderr | 2 +- .../ui/editions/edition-imports-2018.stderr | 2 +- .../edition-imports-virtual-2015-gated.stderr | 2 +- ...dition-keywords-2015-2018-expansion.stderr | 2 +- ...dition-keywords-2018-2018-expansion.stderr | 2 +- .../ui/empty/empty-struct-unit-pat.stderr | 102 +- .../imported_main_from_extern_crate.rs | 6 +- .../imported_main_from_extern_crate.stderr | 10 - .../issue-70453-generics-in-discr-ice.stderr | 1 + src/test/ui/enum/issue-67945-1.stderr | 1 + src/test/ui/enum/issue-67945-2.stderr | 1 + src/test/ui/error-codes/E0107.rs | 32 +- src/test/ui/error-codes/E0107.stderr | 100 +- src/test/ui/error-codes/E0121.stderr | 2 +- .../{E0133.stderr => E0133.mir.stderr} | 2 +- src/test/ui/error-codes/E0133.rs | 3 + .../E0133.thir.stderr} | 2 +- src/test/ui/error-codes/E0184.stderr | 2 +- src/test/ui/error-codes/E0392.stderr | 1 + src/test/ui/error-codes/E0452.rs | 2 - src/test/ui/error-codes/E0452.stderr | 14 +- src/test/ui/error-codes/E0453.rs | 1 - src/test/ui/error-codes/E0453.stderr | 11 +- src/test/ui/error-codes/E0504.stderr | 2 +- src/test/ui/error-codes/E0583.stderr | 2 +- src/test/ui/error-codes/E0602.stderr | 6 +- src/test/ui/error-codes/E0605.stderr | 7 +- src/test/ui/error-codes/E0665.stderr | 2 +- src/test/ui/exclusive-drop-and-copy.stderr | 4 +- src/test/ui/expr/if/if-let.stderr | 4 +- .../extenv/extenv-not-defined-custom.stderr | 2 +- .../extenv/extenv-not-defined-default.stderr | 2 +- src/test/ui/extenv/issue-55897.stderr | 2 +- src/test/ui/extern/external-doc-error.rs | 31 - src/test/ui/extern/external-doc-error.stderr | 38 - ...-allow-internal-unsafe-nested-macro.stderr | 2 +- ...llow-internal-unstable-nested-macro.stderr | 2 +- .../feature-gate-concat_idents2.stderr | 2 +- .../ui/feature-gates/feature-gate-const_fn.rs | 35 - .../feature-gate-const_fn.stderr | 21 - ...eature-gate-const_fn_transmute.mir.stderr} | 24 +- .../feature-gate-const_fn_transmute.rs | 3 + ...eature-gate-const_fn_transmute.thir.stderr | 118 + ...ture-gate-extended_key_value_attributes.rs | 8 - ...-gate-extended_key_value_attributes.stderr | 39 - .../feature-gate-external_doc.rs | 3 - .../feature-gate-external_doc.stderr | 21 - .../feature-gate-lint-reasons.rs | 1 - .../feature-gate-lint-reasons.stderr | 11 +- .../feature-gate-member-constraints.rs | 10 - .../feature-gate-member-constraints.stderr | 18 - .../feature-gate-more-qualified-paths.rs | 27 + .../feature-gate-more-qualified-paths.stderr | 30 + .../feature-gate-native_link_modifiers.rs | 5 + .../feature-gate-native_link_modifiers.stderr | 12 + ...re-gate-native_link_modifiers_as_needed.rs | 8 + ...ate-native_link_modifiers_as_needed.stderr | 12 + ...ature-gate-native_link_modifiers_bundle.rs | 8 + ...e-gate-native_link_modifiers_bundle.stderr | 12 + ...ure-gate-native_link_modifiers_verbatim.rs | 8 + ...gate-native_link_modifiers_verbatim.stderr | 12 + ...ate-native_link_modifiers_whole_archive.rs | 8 + ...native_link_modifiers_whole_archive.stderr | 12 + .../feature-gate-raw-dylib-2.rs | 0 .../feature-gate-raw-dylib-2.stderr | 0 .../feature-gate-raw-dylib-windows-gnu.rs | 8 + .../feature-gate-raw-dylib-windows-gnu.stderr | 18 + .../feature-gate-raw-dylib-windows-msvc.rs} | 2 + ...eature-gate-raw-dylib-windows-msvc.stderr} | 2 +- .../feature-gate-rustc_const_unstable.rs | 2 - .../feature-gate-rustc_const_unstable.stderr | 2 +- .../feature-gate-static-nobundle-2.stderr | 2 + .../feature-gate-static-nobundle.rs | 3 +- .../feature-gate-static-nobundle.stderr | 8 +- .../feature-gate-unnamed_fields.rs | 27 + .../feature-gate-unnamed_fields.stderr | 111 + .../thread-local-const-init.stderr | 2 +- .../ui/feature-gates/trace_macros-gate.stderr | 2 +- .../format-args-capture-macro-hygiene.stderr | 4 +- src/test/ui/fmt/ifmt-bad-arg.stderr | 4 +- src/test/ui/fmt/ifmt-bad-format-args.stderr | 2 +- src/test/ui/fmt/ifmt-unimpl.stderr | 2 +- ...rr => foreign-unsafe-fn-called.mir.stderr} | 2 +- src/test/ui/foreign-unsafe-fn-called.rs | 3 + .../ui/foreign-unsafe-fn-called.thir.stderr | 11 + .../generator-region-requirements.stderr | 3 - ...ssue-45729-unsafe-in-generator.mir.stderr} | 4 +- .../issue-45729-unsafe-in-generator.rs | 3 + ...ssue-45729-unsafe-in-generator.thir.stderr | 11 + .../static-mut-reference-across-yield.rs | 3 + .../yield-while-ref-reborrowed.stderr | 2 +- .../gat-trait-path-missing-lifetime.rs | 4 +- .../gat-trait-path-missing-lifetime.stderr | 24 +- .../gat-trait-path-parenthesised-args.rs | 5 +- .../gat-trait-path-parenthesised-args.stderr | 38 +- .../generic-associated-types/issue-71176.rs | 2 +- .../issue-71176.stderr | 12 +- .../generic-associated-types/issue-76535.rs | 6 +- .../issue-76535.stderr | 16 +- .../generic-associated-types/issue-78671.rs | 4 +- .../issue-78671.stderr | 16 +- .../generic-associated-types/issue-79422.rs | 6 +- .../issue-79422.stderr | 16 +- .../generic-associated-types/issue-79636-1.rs | 2 +- .../issue-79636-1.stderr | 14 +- .../generic-associated-types/issue-79636-2.rs | 2 +- .../issue-79636-2.stderr | 14 +- .../generic-associated-types/issue-80433.rs | 2 +- .../issue-80433.stderr | 12 +- .../issue-81712-cyclic-traits.rs | 2 +- .../issue-81712-cyclic-traits.stderr | 14 +- .../generic-associated-types/issue-81862.rs | 13 + .../issue-81862.stderr | 19 + .../missing_lifetime_args.rs | 23 + .../missing_lifetime_args.stderr | 64 + .../missing_lifetime_const.rs | 13 + .../missing_lifetime_const.stderr | 28 + .../parameter_number_and_kind.rs | 4 +- .../parameter_number_and_kind.stderr | 18 +- .../trait-path-type-error-once-implemented.rs | 4 +- ...it-path-type-error-once-implemented.stderr | 33 +- .../projection-type-lifetime-mismatch.stderr | 6 - .../ui/generics/bad-mid-path-type-params.rs | 6 +- .../generics/bad-mid-path-type-params.stderr | 28 +- .../generics/generic-arg-mismatch-recover.rs | 2 +- .../generic-arg-mismatch-recover.stderr | 12 +- .../generic-impl-less-params-with-defaults.rs | 2 +- ...eric-impl-less-params-with-defaults.stderr | 10 +- .../generic-impl-more-params-with-defaults.rs | 2 +- ...eric-impl-more-params-with-defaults.stderr | 8 +- ...eric-type-less-params-with-defaults.stderr | 8 +- .../generic-type-more-params-with-defaults.rs | 2 +- ...eric-type-more-params-with-defaults.stderr | 8 +- src/test/ui/generics/wrong-number-of-args.rs | 262 +- .../ui/generics/wrong-number-of-args.stderr | 829 +- ...pats-inclusive-dotdotdot-bad-syntax.stderr | 2 +- ...lf-open-range-pats-inclusive-no-end.stderr | 4 +- ...ound_a_b_ret_a_vs_bound_a_ret_a.nll.stderr | 2 +- ...pe.bound_a_b_ret_a_vs_bound_a_ret_a.stderr | 2 +- .../hr-subtype.bound_a_vs_free_x.nll.stderr | 2 +- .../hr-subtype.bound_a_vs_free_x.stderr | 2 +- ...pe.bound_inv_a_b_vs_bound_inv_a.nll.stderr | 4 +- ...ubtype.bound_inv_a_b_vs_bound_inv_a.stderr | 2 +- ...ubtype.free_inv_x_vs_free_inv_y.nll.stderr | 4 +- ...hr-subtype.free_inv_x_vs_free_inv_y.stderr | 4 +- .../hr-subtype.free_x_vs_free_y.nll.stderr | 2 +- .../hr-subtype.free_x_vs_free_y.stderr | 2 +- src/test/ui/hygiene/assoc_item_ctxt.stderr | 4 +- .../ui/hygiene/duplicate_lifetimes.stderr | 4 +- .../extern-prelude-from-opaque-fail.stderr | 4 +- src/test/ui/hygiene/fields-definition.stderr | 2 +- src/test/ui/hygiene/fields-move.stderr | 2 +- src/test/ui/hygiene/fields.stderr | 8 +- src/test/ui/hygiene/generate-mod.stderr | 8 +- src/test/ui/hygiene/globs.stderr | 6 +- src/test/ui/hygiene/hygienic-label-1.stderr | 2 +- src/test/ui/hygiene/hygienic-label-3.stderr | 2 +- .../ui/hygiene/hygienic-labels-in-let.stderr | 32 +- src/test/ui/hygiene/hygienic-labels.stderr | 32 +- src/test/ui/hygiene/impl_items.stderr | 2 +- src/test/ui/hygiene/intercrate.stderr | 2 +- src/test/ui/hygiene/missing-self-diag.stderr | 2 +- .../ui/hygiene/no_implicit_prelude.stderr | 4 +- src/test/ui/hygiene/privacy-early.stderr | 2 +- src/test/ui/hygiene/trait_items.stderr | 2 +- src/test/ui/hygiene/unpretty-debug.stdout | 2 +- src/test/ui/impl-trait/example-calendar.rs | 3 +- .../ui/impl-trait/issues/issue-78721.stderr | 8 +- .../issues/issue-78722.full_tait.stderr | 8 +- .../issues/issue-78722.min_tait.stderr | 8 +- .../error-handling.full_tait.stderr | 4 +- .../error-handling.min_tait.stderr | 2 +- .../multiple-lifetimes/error-handling.rs | 1 - .../multiple-lifetimes/inverse-bounds.rs | 2 - .../ordinary-bounds-pick-original-elided.rs | 8 +- ...nds-pick-original-type-alias-impl-trait.rs | 7 +- .../ordinary-bounds-pick-original.rs | 8 +- .../ordinary-bounds-pick-other.rs | 2 - .../ordinary-bounds-unrelated.nll.stderr | 2 +- .../ordinary-bounds-unrelated.rs | 2 - .../ordinary-bounds-unrelated.stderr | 4 +- .../ordinary-bounds-unsuited.nll.stderr | 2 +- .../ordinary-bounds-unsuited.rs | 4 +- .../ordinary-bounds-unsuited.stderr | 4 +- .../impl-trait/needs_least_region_or_bound.rs | 2 - .../extern-prelude-extern-crate-fail.stderr | 2 +- ...e-extern-crate-restricted-shadowing.stderr | 4 +- src/test/ui/imports/import-crate-var.stderr | 2 +- .../ui/imports/import-prefix-macro-1.stderr | 2 +- .../ui/imports/import-prefix-macro-2.stderr | 2 +- .../local-modularized-tricky-fail-1.stderr | 8 +- .../local-modularized-tricky-fail-2.stderr | 4 +- .../ui/imports/shadow_builtin_macros.stderr | 2 +- .../in-band-lifetimes/elided-lifetimes.stderr | 4 +- .../auxiliary/same-file-in-two-crates-aux.rs | 4 + .../ui/include-macros/mismatched-types.stderr | 4 +- .../include-macros/same-file-in-two-crates.rs | 21 + ...infer-async-enabled-impl-trait-bindings.rs | 4 +- ...r-async-enabled-impl-trait-bindings.stderr | 11 +- src/test/ui/inference/cannot-infer-async.rs | 4 +- .../ui/inference/cannot-infer-async.stderr | 9 +- .../cannot-infer-closure-circular.stderr | 2 +- src/test/ui/inference/cannot-infer-closure.rs | 4 +- .../ui/inference/cannot-infer-closure.stderr | 7 +- .../cannot-infer-partial-try-return.stderr | 3 +- .../infinite/infinite-macro-expansion.stderr | 2 +- .../ui/inner-static-type-parameter.stderr | 1 + .../ui/internal/internal-unstable-const.rs | 2 +- .../internal/internal-unstable-noallow.stderr | 8 +- src/test/ui/internal/internal-unstable.stderr | 2 +- src/test/ui/intrinsics/intrinsic-alignment.rs | 1 + ...ue-28575.stderr => issue-28575.mir.stderr} | 2 +- src/test/ui/intrinsics/issue-28575.rs | 3 + .../ui/intrinsics/issue-28575.thir.stderr | 11 + .../intrinsics/panic-uninitialized-zeroed.rs | 2 + ...tderr => unchecked_math_unsafe.mir.stderr} | 6 +- .../ui/intrinsics/unchecked_math_unsafe.rs | 3 + .../unchecked_math_unsafe.thir.stderr | 27 + .../invalid-module-declaration.stderr | 2 +- src/test/ui/invalid/invalid-llvm-passes.rs | 4 + .../ui/invalid/invalid-llvm-passes.stderr | 4 + ...lid-rustc_args_required_const-arguments.rs | 32 - ...rustc_args_required_const-arguments.stderr | 60 - src/test/ui/issue-85713-align-with-no-arg.rs | 6 - .../ui/issue-85713-align-with-no-arg.stderr | 15 - src/test/ui/issues/auxiliary/issue-75907.rs | 12 + src/test/ui/issues/issue-11192.stderr | 2 +- src/test/ui/issues/issue-11740.rs | 2 + src/test/ui/issues/issue-12997-2.stderr | 2 +- src/test/ui/issues/issue-13446.stderr | 2 +- src/test/ui/issues/issue-14091-2.stderr | 2 +- src/test/ui/issues/issue-14091.stderr | 2 +- src/test/ui/issues/issue-14092.stderr | 8 +- ...ue-14227.stderr => issue-14227.mir.stderr} | 2 +- src/test/ui/issues/issue-14227.rs | 3 + src/test/ui/issues/issue-14227.thir.stderr | 11 + src/test/ui/issues/issue-15167.stderr | 8 +- src/test/ui/issues/issue-16098.stderr | 2 +- ...ue-16538.stderr => issue-16538.mir.stderr} | 6 +- src/test/ui/issues/issue-16538.rs | 3 + src/test/ui/issues/issue-16538.thir.stderr | 27 + src/test/ui/issues/issue-16966.stderr | 2 +- src/test/ui/issues/issue-17904-2.stderr | 1 + src/test/ui/issues/issue-18423.stderr | 2 +- src/test/ui/issues/issue-20413.stderr | 1 + src/test/ui/issues/issue-21160.stderr | 2 +- src/test/ui/issues/issue-22289.stderr | 2 +- src/test/ui/issues/issue-22312.stderr | 2 +- src/test/ui/issues/issue-23024.stderr | 8 +- src/test/ui/issues/issue-25385.stderr | 2 +- src/test/ui/issues/issue-25386.stderr | 2 +- src/test/ui/issues/issue-26093.stderr | 4 +- ...27282-mutate-before-diverging-arm-1.stderr | 2 +- ...27282-mutate-before-diverging-arm-3.stderr | 2 +- src/test/ui/issues/issue-27340.stderr | 2 +- src/test/ui/issues/issue-27592.stderr | 4 +- ...ue-28324.stderr => issue-28324.mir.stderr} | 2 +- src/test/ui/issues/issue-28324.rs | 3 + src/test/ui/issues/issue-28324.thir.stderr | 11 + ...ue-28776.stderr => issue-28776.mir.stderr} | 2 +- src/test/ui/issues/issue-28776.rs | 3 + src/test/ui/issues/issue-28776.thir.stderr | 11 + src/test/ui/issues/issue-29084.stderr | 2 +- src/test/ui/issues/issue-2995.stderr | 7 +- src/test/ui/issues/issue-30123.stderr | 3 + ...ssue-3080.stderr => issue-3080.mir.stderr} | 2 +- src/test/ui/issues/issue-3080.rs | 3 + src/test/ui/issues/issue-3080.thir.stderr | 11 + src/test/ui/issues/issue-31011.stderr | 2 +- src/test/ui/issues/issue-32004.stderr | 5 +- src/test/ui/issues/issue-3214.rs | 2 +- src/test/ui/issues/issue-3214.stderr | 6 +- src/test/ui/issues/issue-32655.stderr | 2 +- src/test/ui/issues/issue-32709.stderr | 3 +- src/test/ui/issues/issue-32782.stderr | 2 +- src/test/ui/issues/issue-32950.stderr | 2 +- src/test/ui/issues/issue-34229.stderr | 2 +- src/test/ui/issues/issue-36299.stderr | 1 + src/test/ui/issues/issue-36638.stderr | 1 + src/test/ui/issues/issue-37534.stderr | 1 + src/test/ui/issues/issue-38074.rs | 20 - src/test/ui/issues/issue-38821.stderr | 2 +- src/test/ui/issues/issue-39367.rs | 3 + src/test/ui/issues/issue-39848.stderr | 2 +- src/test/ui/issues/issue-41880.stderr | 2 +- src/test/ui/issues/issue-42954.stderr | 2 +- ...-unnecessary-unsafe-in-closure.mir.stderr} | 8 +- ...sue-45107-unnecessary-unsafe-in-closure.rs | 3 + ...-unnecessary-unsafe-in-closure.thir.stderr | 35 + src/test/ui/issues/issue-46983.stderr | 2 - ...ue-47412.stderr => issue-47412.mir.stderr} | 6 +- src/test/ui/issues/issue-47412.rs | 6 +- src/test/ui/issues/issue-47412.thir.stderr | 11 + ...ue-48131.stderr => issue-48131.mir.stderr} | 6 +- src/test/ui/issues/issue-48131.rs | 3 + src/test/ui/issues/issue-48131.thir.stderr | 20 + src/test/ui/issues/issue-48364.stderr | 2 +- src/test/ui/issues/issue-48728.stderr | 2 +- src/test/ui/issues/issue-50480.stderr | 2 +- src/test/ui/issues/issue-51848.stderr | 2 +- src/test/ui/issues/issue-53251.rs | 4 +- src/test/ui/issues/issue-53251.stderr | 16 +- src/test/ui/issues/issue-54954.rs | 2 - src/test/ui/issues/issue-54954.stderr | 4 +- ...ssue-5844.stderr => issue-5844.mir.stderr} | 2 +- src/test/ui/issues/issue-5844.rs | 2 + src/test/ui/issues/issue-5844.thir.stderr | 11 + src/test/ui/issues/issue-59488.stderr | 4 +- src/test/ui/issues/issue-60622.rs | 2 +- src/test/ui/issues/issue-60622.stderr | 8 +- src/test/ui/issues/issue-61623.stderr | 2 +- src/test/ui/issues/issue-6596-1.stderr | 2 +- src/test/ui/issues/issue-6596-2.stderr | 2 +- src/test/ui/issues/issue-6801.stderr | 2 +- .../issue-68091-unicode-ident-after-if.stderr | 2 +- .../issue-69396-const-no-type-in-macro.stderr | 8 +- src/test/ui/issues/issue-70093.rs | 1 + ...70724-add_type_neq_err_label-unwrap.stderr | 6 +- src/test/ui/issues/issue-75907_b.rs | 5 +- src/test/ui/issues/issue-75907_b.stderr | 26 +- src/test/ui/issues/issue-pr29383.stderr | 10 +- .../lang-item-missing-generator.rs | 0 .../lang-item-missing-generator.stderr | 0 .../ui/{ => lang-items}/lang-item-missing.rs | 0 .../{ => lang-items}/lang-item-missing.stderr | 0 .../wrong-number-generic-args-add.rs | 20 + .../wrong-number-generic-args-add.stderr | 20 + .../wrong-number-generic-args-index.rs | 19 + .../wrong-number-generic-args-index.stderr | 18 + src/test/ui/layout/issue-84108.rs | 14 + src/test/ui/layout/issue-84108.stderr | 44 + .../lifetimes/borrowck-let-suggestion.stderr | 2 +- src/test/ui/lint/bare-trait-objects-path.rs | 12 +- .../ui/lint/bare-trait-objects-path.stderr | 14 +- src/test/ui/lint/crate_level_only_lint.rs | 3 - src/test/ui/lint/crate_level_only_lint.stderr | 28 +- src/test/ui/lint/dead-code/issue-85255.rs | 51 + src/test/ui/lint/dead-code/issue-85255.stderr | 80 + src/test/ui/lint/expr_attr_paren_order.rs | 6 +- src/test/ui/lint/expr_attr_paren_order.stderr | 8 +- src/test/ui/lint/forbid-group-group-2.rs | 6 - src/test/ui/lint/forbid-group-group-2.stderr | 38 +- src/test/ui/lint/forbid-group-member.rs | 2 - src/test/ui/lint/forbid-group-member.stderr | 14 +- src/test/ui/lint/forbid-member-group.rs | 1 - src/test/ui/lint/forbid-member-group.stderr | 11 +- .../force-allowed-by-default-lint.rs | 11 + .../force-allowed-by-default-lint.stderr | 10 + .../force-allowed-deny-by-default-lint.rs | 9 + .../force-allowed-deny-by-default-lint.stderr | 14 + .../lint/force-warn/force-allowed-warning.rs | 9 + .../force-warn/force-allowed-warning.stderr | 10 + .../force-warn/force-deny-by-default-lint.rs | 8 + .../force-deny-by-default-lint.stderr | 14 + .../force-lint-allow-all-warnings.rs | 9 + .../force-lint-allow-all-warnings.stderr | 10 + .../force-lint-group-allow-all-warnings.rs | 9 + ...force-lint-group-allow-all-warnings.stderr | 10 + .../force-warn/force-lint-in-allowed-group.rs | 12 + .../force-lint-in-allowed-group.stderr | 12 + .../force-warn-group-allow-warning.rs | 12 + .../force-warn-group-allow-warning.stderr | 12 + .../ui/lint/force-warn/force-warn-group.rs | 12 + .../lint/force-warn/force-warn-group.stderr | 12 + .../lint/inclusive-range-pattern-syntax.fixed | 2 + .../ui/lint/inclusive-range-pattern-syntax.rs | 2 + .../inclusive-range-pattern-syntax.stderr | 7 +- src/test/ui/lint/issue-80988.rs | 2 - src/test/ui/lint/issue-80988.stderr | 14 +- src/test/ui/lint/issue-83477.rs | 1 + src/test/ui/lint/issue-83477.stderr | 8 +- src/test/ui/lint/lint-ctypes-fn.rs | 11 + src/test/ui/lint/lint-ctypes-fn.stderr | 64 +- src/test/ui/lint/lint-forbid-attr.rs | 1 - src/test/ui/lint/lint-forbid-attr.stderr | 11 +- src/test/ui/lint/lint-forbid-cmdline.rs | 1 - src/test/ui/lint/lint-forbid-cmdline.stderr | 10 +- src/test/ui/lint/lint-malformed.rs | 2 - src/test/ui/lint/lint-malformed.stderr | 14 +- src/test/ui/lint/lint-removed-cmdline.stderr | 6 +- src/test/ui/lint/lint-renamed-cmdline.stderr | 6 +- .../ui/lint/lint-stability-deprecated.stderr | 96 +- src/test/ui/lint/lint-stability2.stderr | 2 +- src/test/ui/lint/lint-stability3.stderr | 2 +- .../ui/lint/lint-unexported-no-mangle.stderr | 10 +- .../ui/lint/lint-unknown-lint-cmdline.stderr | 11 +- src/test/ui/lint/lint-unsafe-code.stderr | 10 +- .../ui/lint/lints-in-foreign-macros.stderr | 2 +- src/test/ui/lint/reasons-erroneous.rs | 20 - src/test/ui/lint/reasons-erroneous.stderr | 100 +- src/test/ui/lint/register-tool-lint.rs | 2 - src/test/ui/lint/register-tool-lint.stderr | 10 +- ...emicolon-in-expressions-from-macros.stderr | 4 +- src/test/ui/lint/test-inner-fn.stderr | 4 +- ...unaligned_references_external_macro.stderr | 2 +- .../ui/lint/unreachable_pub-pub_crate.stderr | 2 +- src/test/ui/lint/unreachable_pub.stderr | 2 +- src/test/ui/lint/use_suggestion_json.stderr | 421 +- .../liveness-return-last-stmt-semi.stderr | 2 +- .../llvm-asm/inline-asm-bad-constraint.stderr | 6 +- src/test/ui/llvm-asm/issue-62046.stderr | 2 +- src/test/ui/lto-still-runs-thread-dtors.rs | 2 + .../ui/macro_backtrace/main.default.stderr | 6 +- .../ui/macros/assert-matches-macro-msg.rs | 13 + src/test/ui/macros/assert.stderr | 2 +- .../auxiliary/foreign-crate-macro-pat.rs | 11 + .../macros/auxiliary/proc_macro_sequence.rs | 19 - src/test/ui/macros/cfg.stderr | 2 +- src/test/ui/macros/cross-crate-pat-span.rs | 12 + .../derive-in-eager-expansion-hang.stderr | 2 +- src/test/ui/macros/format-parse-errors.stderr | 2 +- src/test/ui/macros/global-asm.rs | 4 +- src/test/ui/macros/global-asm.stderr | 6 +- ...421-mac-expr-bad-stmt-good-add-semi.stderr | 4 +- ...issue-78325-inconsistent-resolution.stderr | 2 +- .../macros/issue-84195-lint-anon-const.stderr | 2 +- .../ui/macros/issue-84429-matches-edition.rs | 9 + .../macros/issue-85432-ungated-attr-macro.rs | 30 - ...-ambiguity-multiple-parsing-options.stderr | 4 +- .../macro-backtrace-invalid-internals.stderr | 16 +- .../ui/macros/macro-backtrace-nested.stderr | 4 +- .../ui/macros/macro-backtrace-println.stderr | 2 +- src/test/ui/macros/macro-context.stderr | 6 +- .../macro-lifetime-used-with-labels.stderr | 2 +- .../macros/macro-local-data-key-priv.stderr | 2 +- src/test/ui/macros/macro-shadowing.stderr | 4 +- .../ui/macros/macros-nonfatal-errors.stderr | 8 +- .../ui/macros/must-use-in-macro-55516.stderr | 2 +- .../ui/macros/nonterminal-matching.stderr | 2 +- src/test/ui/macros/not-utf8.stderr | 2 +- .../macros/restricted-shadowing-legacy.stderr | 16 +- .../macros/restricted-shadowing-modern.stderr | 12 +- src/test/ui/macros/same-sequence-span.stderr | 16 +- .../ui/macros/span-covering-argument-1.stderr | 2 +- src/test/ui/macros/trace-macro.stderr | 2 +- src/test/ui/macros/trace_faulty_macros.stderr | 18 +- .../ui/macros/vec-macro-in-pattern.stderr | 2 +- .../malformed/malformed-derive-entry.stderr | 4 +- .../ui/manual/manual-link-unsupported-kind.rs | 5 + .../manual-link-unsupported-kind.stderr | 2 + .../auxiliary/match_non_exhaustive_lib.rs | 5 + .../match-pattern-field-mismatch-2.stderr | 5 +- .../match/match-ref-mut-invariance.nll.stderr | 3 + .../match-ref-mut-let-invariance.nll.stderr | 3 + src/test/ui/match/match_non_exhaustive.rs | 32 + src/test/ui/match/match_non_exhaustive.stderr | 36 + src/test/ui/matches2021.rs | 14 + src/test/ui/meta/revision-bad.rs | 1 + .../methods/method-call-lifetime-args-fail.rs | 4 +- .../method-call-lifetime-args-fail.stderr | 8 +- .../method-not-found-generic-arg-elision.rs | 106 + ...ethod-not-found-generic-arg-elision.stderr | 97 + src/test/ui/mir/issue-80742.stderr | 4 +- .../ui/mismatched_types/cast-rfc0401.stderr | 7 +- .../ui/mismatched_types/const-fn-in-trait.rs | 2 - .../mismatched_types/const-fn-in-trait.stderr | 4 +- .../ui/mismatched_types/issue-26480.stderr | 4 +- src/test/ui/mismatched_types/issue-84976.rs | 25 + .../ui/mismatched_types/issue-84976.stderr | 27 + .../missing-items/missing-type-parameter2.rs | 22 + .../missing-type-parameter2.stderr | 121 + .../missing_non_modrs_mod.stderr | 2 +- .../missing_non_modrs_mod_inline.stderr | 2 +- src/test/ui/modules/issue-56411.stderr | 4 +- .../mod_file_disambig.stderr | 2 +- src/test/ui/multiple-plugin-registrars.stderr | 8 +- .../native-library-link-flags/empty-kind-1.rs | 6 + .../empty-kind-1.stderr | 2 + .../native-library-link-flags/empty-kind-2.rs | 6 + .../empty-kind-2.stderr | 2 + src/test/ui/nll/closure-access-spans.stderr | 4 +- src/test/ui/nll/closure-borrow-spans.stderr | 12 +- src/test/ui/nll/closure-captures.stderr | 12 +- ...on-lbr-anon-does-not-outlive-static.stderr | 2 - src/test/ui/nll/closure-use-spans.stderr | 4 +- src/test/ui/nll/closures-in-loops.stderr | 2 +- src/test/ui/nll/guarantor-issue-46974.stderr | 3 - src/test/ui/nll/issue-51268.stderr | 2 +- .../nll/type-check-pointer-coercions.stderr | 6 + .../nll/type-check-pointer-comparisons.stderr | 18 + src/test/ui/no-stdio.rs | 2 + src/test/ui/non-fmt-panic.rs | 2 + src/test/ui/non-fmt-panic.stderr | 32 +- src/test/ui/on-unimplemented/no-debug.stderr | 8 +- src/test/ui/option-to-result.stderr | 35 - .../exhaustiveness-unreachable-pattern.stderr | 2 +- src/test/ui/or-patterns/macro-pat.rs | 3 +- .../or-patterns-syntactic-pass-2021.rs | 2 - src/test/ui/out-of-order-shadowing.stderr | 2 +- src/test/ui/panic-handler/weak-lang-item.rs | 1 + .../ui/panic-handler/weak-lang-item.stderr | 2 +- .../abort-link-to-unwind-dylib.rs | 1 + src/test/ui/panic-runtime/lto-unwind.rs | 1 + .../panic-runtime/transitive-link-a-bunch.rs | 1 + .../ui/panic-runtime/want-unwind-got-abort.rs | 1 + .../panic-runtime/want-unwind-got-abort2.rs | 1 + src/test/ui/panics/abort-on-panic.rs | 85 +- src/test/ui/panics/panic-macro-any-wrapped.rs | 2 +- src/test/ui/panics/panic-macro-any.rs | 2 +- .../ui/parser/bad-interpolated-block.stderr | 6 +- .../brace-after-qualified-path-in-match.rs | 7 - ...brace-after-qualified-path-in-match.stderr | 10 - .../ui/parser/float-field-interpolated.stderr | 8 +- .../ui/parser/fn-field-parse-error-ice.rs | 10 + .../ui/parser/fn-field-parse-error-ice.stderr | 24 + src/test/ui/parser/fn-header-semantic-fail.rs | 1 - .../ui/parser/fn-header-semantic-fail.stderr | 40 +- ...interpolate-impl-items-bad-variants.stderr | 6 +- ...sue-65122-mac-invoc-in-mut-patterns.stderr | 6 +- src/test/ui/parser/issue-68890-2.rs | 1 + src/test/ui/parser/issue-68890-2.stderr | 2 + .../parser/issue-73568-lifetime-after-mut.rs | 2 + .../issue-73568-lifetime-after-mut.stderr | 11 +- src/test/ui/parser/issue-84104.rs | 3 + src/test/ui/parser/issue-84104.stderr | 16 + src/test/ui/parser/issue-84148-1.rs | 4 + src/test/ui/parser/issue-84148-1.stderr | 23 + src/test/ui/parser/issue-84148-2.rs | 4 + src/test/ui/parser/issue-84148-2.stderr | 31 + .../ui/parser/labeled-no-colon-expr.stderr | 2 +- src/test/ui/parser/macro/issue-37113.stderr | 2 +- src/test/ui/parser/macro/issue-37234.stderr | 2 +- .../macro/macro-incomplete-parse.stderr | 2 +- .../ui/parser/macro/pub-item-macro.stderr | 4 +- .../macro/trait-object-macro-matcher.rs | 1 + .../macro/trait-object-macro-matcher.stderr | 2 + ...closing-angle-bracket-eq-constraint.stderr | 4 +- src/test/ui/parser/missing-semicolon.stderr | 2 +- src/test/ui/parser/mod_file_not_exist.stderr | 2 +- .../parser/mod_file_not_exist_windows.stderr | 2 +- src/test/ui/parser/mut-patterns.stderr | 2 +- .../paren-after-qualified-path-in-match.rs | 7 - ...paren-after-qualified-path-in-match.stderr | 10 - src/test/ui/parser/recover-range-pats.rs | 21 +- src/test/ui/parser/recover-range-pats.stderr | 127 +- .../ui/parser/stmt_expr_attrs_placement.rs | 16 + .../parser/stmt_expr_attrs_placement.stderr | 50 +- ...t-values-and-missing-field-separator.fixed | 35 + ...ault-values-and-missing-field-separator.rs | 35 + ...-values-and-missing-field-separator.stderr | 92 + src/test/ui/parser/trait-object-delimiters.rs | 17 + .../ui/parser/trait-object-delimiters.stderr | 77 + .../ui/parser/trait-object-trait-parens.rs | 7 +- .../parser/trait-object-trait-parens.stderr | 52 +- src/test/ui/parser/unmatched-langle-1.rs | 9 + src/test/ui/parser/unmatched-langle-1.stderr | 22 + src/test/ui/parser/unmatched-langle-2.rs | 15 + src/test/ui/parser/unmatched-langle-2.stderr | 8 + src/test/ui/parser/unsafe-mod.stderr | 2 +- .../ui/pattern/pattern-error-continue.stderr | 16 +- .../rest-pat-semantic-disallowed.stderr | 2 +- src/test/ui/prelude2021.rs | 7 + .../associated-item-privacy-inherent.stderr | 42 +- .../associated-item-privacy-trait.stderr | 60 +- ...ssociated-item-privacy-type-binding.stderr | 32 +- .../ui/privacy/private-inferred-type-3.stderr | 14 +- .../ui/privacy/private-inferred-type.stderr | 12 +- .../proc-macro/allowed-attr-stmt-expr.stdout | 4 +- src/test/ui/proc-macro/attr-complex-fn.stdout | 2 +- src/test/ui/proc-macro/attr-stmt-expr.stdout | 4 +- .../test/ui/proc-macro/auxiliary/api/cmp.rs | 17 +- src/test/ui/proc-macro/auxiliary/api/mod.rs | 24 + src/test/ui/proc-macro/auxiliary/api/parse.rs | 23 + .../auxiliary/attr-stmt-expr-rpass.rs | 4 +- .../ui/proc-macro/auxiliary/attr-stmt-expr.rs | 4 +- .../ui/proc-macro/auxiliary/custom-quote.rs | 31 + .../auxiliary/span-from-proc-macro.rs | 49 + src/test/ui/proc-macro/cfg-eval-inner.stdout | 6 +- src/test/ui/proc-macro/derive-bad.stderr | 2 +- .../proc-macro/derive-helper-shadowing.stderr | 4 +- .../dollar-crate-issue-62325.stdout | 4 +- .../ui/proc-macro/expand-to-unstable-2.stderr | 2 +- .../ui/proc-macro/expand-to-unstable.stderr | 2 +- .../expr-stmt-nonterminal-tokens.rs | 37 + .../expr-stmt-nonterminal-tokens.stdout | 540 + .../proc-macro/gen-macro-rules-hygiene.stderr | 4 +- src/test/ui/proc-macro/generate-mod.stderr | 16 +- .../group-compat-hack.stderr | 20 +- .../inner-attr-non-inline-mod.stdout | 2 +- src/test/ui/proc-macro/inner-attrs.rs | 32 +- src/test/ui/proc-macro/inner-attrs.stderr | 40 +- src/test/ui/proc-macro/inner-attrs.stdout | 573 +- .../proc-macro/invalid-punct-ident-4.stderr | 2 +- src/test/ui/proc-macro/issue-38586.stderr | 2 +- .../issue-59191-replace-root-with-fn.stderr | 2 +- .../ui/proc-macro/issue-75734-pp-paren.stdout | 2 +- .../proc-macro/issue-75930-derive-cfg.stdout | 9 +- .../issue-78675-captured-inner-attrs.stdout | 4 +- src/test/ui/proc-macro/issue-83510.stderr | 8 +- src/test/ui/proc-macro/lifetimes.stderr | 2 +- .../ui/proc-macro/lints_in_proc_macros.stderr | 2 +- .../proc-macro/macro-rules-derive-cfg.stdout | 2 +- .../ui/proc-macro/macro-rules-derive.stderr | 2 +- .../ui/proc-macro/meta-macro-hygiene.stdout | 8 +- src/test/ui/proc-macro/mixed-site-span.stderr | 6 +- src/test/ui/proc-macro/multispan.stderr | 14 +- src/test/ui/proc-macro/nodelim-groups.stdout | 1 - .../proc-macro/nonterminal-expansion.stdout | 2 +- .../nonterminal-token-hygiene.stdout | 10 +- .../ui/proc-macro/parent-source-spans.stderr | 18 +- src/test/ui/proc-macro/pretty-print-tts.rs | 20 + .../ui/proc-macro/pretty-print-tts.stdout | 102 + src/test/ui/proc-macro/quote-debug.rs | 18 + src/test/ui/proc-macro/quote-debug.stdout | 52 + src/test/ui/proc-macro/raw-ident.stderr | 2 +- .../ui/proc-macro/resolved-located-at.stderr | 4 +- src/test/ui/proc-macro/simple-tuple.rs | 19 - src/test/ui/proc-macro/simple-tuple.stdout | 79 - .../ui/proc-macro/span-from-proc-macro.rs | 17 + .../ui/proc-macro/span-from-proc-macro.stderr | 62 + src/test/ui/proc-macro/subspan.stderr | 16 +- src/test/ui/proc-macro/test.rs | 12 + src/test/ui/proc-macro/three-equals.stderr | 2 +- src/test/ui/proc-macro/weird-braces.stdout | 6 +- src/test/ui/proc-macro/weird-hygiene.stderr | 4 +- .../ui/process/process-panic-after-fork.rs | 151 + src/test/ui/question-mark-type-infer.stderr | 7 +- .../ui/range/exclusive-range-patterns-2021.rs | 14 + .../exclusive-range-patterns-2021.stderr | 27 + .../range-inclusive-pattern-precedence.fixed | 3 +- .../range-inclusive-pattern-precedence.rs | 3 +- .../range-inclusive-pattern-precedence.stderr | 4 +- .../range-inclusive-pattern-precedence2.rs | 1 + ...range-inclusive-pattern-precedence2.stderr | 4 +- src/test/ui/range/range_traits-1.stderr | 24 +- src/test/ui/range/range_traits-2.stderr | 2 +- src/test/ui/range/range_traits-3.stderr | 2 +- src/test/ui/range/range_traits-6.stderr | 2 +- src/test/ui/reachable/expr_again.stderr | 2 +- src/test/ui/reachable/expr_block.stderr | 2 +- src/test/ui/reachable/expr_if.stderr | 2 +- src/test/ui/reachable/expr_loop.stderr | 6 +- src/test/ui/reachable/expr_match.stderr | 4 +- .../ui/reachable/unreachable-code-ret.stderr | 2 +- ...time-bounds-on-fns-where-clause.nll.stderr | 3 + ...time-bounds-on-fns-where-clause.nll.stderr | 3 + .../regions-addr-of-upvar-self.nll.stderr | 2 +- .../regions-lifetime-bounds-on-fns.nll.stderr | 3 + .../regions-static-bound.migrate.nll.stderr | 5 - .../regions-static-bound.migrate.stderr | 5 - .../regions/regions-static-bound.nll.stderr | 5 - .../regions-trait-object-subtyping.nll.stderr | 6 + src/test/ui/resolve/issue-82865.stderr | 2 +- src/test/ui/resolve/issue-85348.rs | 12 + src/test/ui/resolve/issue-85348.stderr | 25 + src/test/ui/resolve/resolve-hint-macro.rs | 4 +- src/test/ui/resolve/resolve-hint-macro.stderr | 10 +- src/test/ui/resolve/shadow-const-param.rs | 20 + src/test/ui/resolve/shadow-const-param.stderr | 20 + .../termination-trait-test-wrong-type.stderr | 2 +- .../ui/rfc-2008-non-exhaustive/enum.stderr | 6 +- .../ui/rfc-2008-non-exhaustive/struct.stderr | 2 +- .../uninhabited/match.stderr | 2 +- .../match_with_exhaustive_patterns.stderr | 2 +- .../caller-location-fnptr-rt-ctfe-equiv.rs | 2 +- .../const-caller-location.rs | 2 +- .../dbg-macro-requires-debug.stderr | 2 +- .../mod_file_nonascii_forbidden.stderr | 2 +- .../disallowed-positions.stderr | 42 +- .../rfc-2627-raw-dylib/raw-dylib-msvc-only.rs | 8 + .../raw-dylib-msvc-only.stderr | 17 + .../raw-dylib-windows-only.rs | 7 + .../raw-dylib-windows-only.stderr | 17 + .../rfc-2396-target_feature-11/check-pass.rs | 2 + .../closures-inherit-target_feature.rs | 2 + .../{fn-ptr.stderr => fn-ptr.mir.stderr} | 2 +- .../rfcs/rfc-2396-target_feature-11/fn-ptr.rs | 2 + .../fn-ptr.thir.stderr | 18 + ...afe-calls.stderr => safe-calls.mir.stderr} | 20 +- .../rfc-2396-target_feature-11/safe-calls.rs | 2 + .../safe-calls.thir.stderr | 83 + src/test/ui/running-with-no-runtime.rs | 2 + .../suggestions-not-always-applicable.fixed | 8 +- .../suggestions-not-always-applicable.rs | 8 +- .../suggestions-not-always-applicable.stderr | 28 - .../ambiguity-macros-nested.stderr | 2 +- .../uniform-paths/ambiguity-macros.stderr | 2 +- src/test/ui/rustc-args-required-const2.rs | 10 - src/test/ui/rustc-args-required-const2.stderr | 8 - src/test/ui/rustdoc/deny-invalid-doc-attrs.rs | 7 + .../ui/rustdoc/deny-invalid-doc-attrs.stderr | 16 + .../ui/rustdoc/doc-inline-extern-crate.rs | 9 + .../ui/rustdoc/doc-inline-extern-crate.stderr | 13 + ...err => safe-extern-statics-mut.mir.stderr} | 8 +- src/test/ui/safe-extern-statics-mut.rs | 2 + .../ui/safe-extern-statics-mut.thir.stderr | 35 + ....stderr => safe-extern-statics.mir.stderr} | 8 +- src/test/ui/safe-extern-statics.rs | 2 + src/test/ui/safe-extern-statics.thir.stderr | 35 + src/test/ui/sanitize/crt-static.rs | 5 + src/test/ui/sanitize/crt-static.stderr | 4 + src/test/ui/seq-args.rs | 4 +- src/test/ui/seq-args.stderr | 12 +- .../simd-intrinsic-generic-elements.rs | 21 +- .../simd-intrinsic-generic-elements.stderr | 54 +- .../simd-intrinsic-inlining-issue67557-ice.rs | 3 +- .../simd-intrinsic-inlining-issue67557.rs | 6 +- src/test/ui/simd/issue-85915-simd-ptrs.rs | 67 + .../ui/simd/shuffle-not-out-of-bounds.stderr | 12 +- .../simd/simd-intrinsic-generic-comparison.rs | 2 + .../simd/simd-intrinsic-generic-elements.rs | 20 +- ...ric-monomorphisation-extern-nonnull-ptr.rs | 23 + ...-type-generic-monomorphisation-wide-ptr.rs | 12 + ...e-generic-monomorphisation-wide-ptr.stderr | 4 + src/test/ui/simd/simd-type-wide-ptr.rs | 12 + src/test/ui/simd/simd-type-wide-ptr.stderr | 4 + src/test/ui/simd/wasm-simd-indirect.rs | 31 + src/test/ui/span/E0204.stderr | 4 +- src/test/ui/span/coerce-suggestions.stderr | 2 +- src/test/ui/span/issue-33884.stderr | 2 +- ...e.stderr => lint-unused-unsafe.mir.stderr} | 18 +- src/test/ui/span/lint-unused-unsafe.rs | 3 + .../ui/span/lint-unused-unsafe.thir.stderr | 66 + .../ui/span/macro-span-replacement.stderr | 2 +- src/test/ui/span/slice-borrow.stderr | 2 +- .../stability-attribute-sanity.rs | 2 +- src/test/ui/static/static-drop-scope.rs | 2 - src/test/ui/static/static-drop-scope.stderr | 20 +- ...ic-mut-foreign-requires-unsafe.mir.stderr} | 6 +- .../static-mut-foreign-requires-unsafe.rs | 3 + ...ic-mut-foreign-requires-unsafe.thir.stderr | 27 + ... => static-mut-requires-unsafe.mir.stderr} | 6 +- .../ui/static/static-mut-requires-unsafe.rs | 3 + .../static-mut-requires-unsafe.thir.stderr | 27 + src/test/ui/structs-enums/rec-align-u64.rs | 1 + .../structs-enums/struct-rec/issue-74224.rs | 11 + .../struct-rec/issue-74224.stderr | 17 + .../structs-enums/struct-rec/issue-84611.rs | 11 + .../struct-rec/issue-84611.stderr | 17 + .../struct-rec/mutual-struct-recursion.rs | 23 + .../struct-rec/mutual-struct-recursion.stderr | 59 + .../ui/structs/struct-fn-in-definition.rs | 33 + .../ui/structs/struct-fn-in-definition.stderr | 29 + .../structure-constructor-type-mismatch.rs | 4 +- ...structure-constructor-type-mismatch.stderr | 12 +- src/test/ui/suffixed-literal-meta.rs | 2 +- .../auxiliary/proc-macro-type-error.rs | 18 + ...gest-deref-inside-macro-issue-58298.stderr | 2 +- .../dont-suggest-try_into-in-macros.stderr | 2 +- src/test/ui/suggestions/format-borrow.rs | 4 + src/test/ui/suggestions/format-borrow.stderr | 22 +- .../imm-ref-trait-object-literal.stderr | 8 +- src/test/ui/suggestions/issue-61963.rs | 4 +- src/test/ui/suggestions/issue-61963.stderr | 7 +- src/test/ui/suggestions/issue-68049-1.rs | 16 + src/test/ui/suggestions/issue-68049-1.stderr | 9 + src/test/ui/suggestions/issue-68049-2.rs | 21 + src/test/ui/suggestions/issue-68049-2.stderr | 21 + src/test/ui/suggestions/issue-72766.stderr | 2 +- src/test/ui/suggestions/issue-84592.rs | 17 + src/test/ui/suggestions/issue-84592.stderr | 17 + src/test/ui/suggestions/issue-84700.rs | 26 + src/test/ui/suggestions/issue-84700.stderr | 21 + src/test/ui/suggestions/issue-84973-2.rs | 13 + src/test/ui/suggestions/issue-84973-2.stderr | 15 + .../ui/suggestions/issue-84973-blacklist.rs | 29 + .../suggestions/issue-84973-blacklist.stderr | 64 + .../ui/suggestions/issue-84973-negative.rs | 12 + .../suggestions/issue-84973-negative.stderr | 24 + src/test/ui/suggestions/issue-84973.rs | 33 + src/test/ui/suggestions/issue-84973.stderr | 15 + src/test/ui/suggestions/issue-85347.rs | 10 + src/test/ui/suggestions/issue-85347.stderr | 19 + ...est-unsized-indirection-in-where-clause.rs | 9 + ...unsized-indirection-in-where-clause.stderr | 20 + .../issue-86100-tuple-paren-comma.rs | 25 + .../issue-86100-tuple-paren-comma.stderr | 55 + .../suggestions/missing-lifetime-specifier.rs | 44 +- .../missing-lifetime-specifier.stderr | 24 +- .../ui/suggestions/missing-lt-for-hrtb.stderr | 4 + .../suggestions/mut-borrow-needed-by-trait.rs | 1 - .../mut-borrow-needed-by-trait.stderr | 19 +- src/test/ui/suggestions/path-display.stderr | 2 +- .../ui/suggestions/return-elided-lifetime.rs | 37 + .../suggestions/return-elided-lifetime.stderr | 198 + src/test/ui/suggestions/suggest-ref-macro.rs | 29 + .../ui/suggestions/suggest-ref-macro.stderr | 34 + .../unsized-function-parameter.fixed | 23 + .../suggestions/unsized-function-parameter.rs | 23 + .../unsized-function-parameter.stderr | 42 + ...use-type-argument-instead-of-assoc-type.rs | 2 +- ...type-argument-instead-of-assoc-type.stderr | 8 +- src/test/ui/symbol-names/trait-objects.rs | 48 + .../ui/symbol-names/trait-objects.v0.stderr | 56 + src/test/ui/tag-type-args.stderr | 8 +- src/test/ui/target-feature/gate.rs | 1 + src/test/ui/target-feature/gate.stderr | 2 +- .../rust-specific-name-no-warnings.rs | 5 + src/test/ui/target-feature/wasm-safe.rs | 44 + src/test/ui/test-attrs/test-type.rs | 28 + src/test/ui/test-attrs/test-type.run.stdout | 8 + src/test/ui/test-panic-abort-disabled.rs | 2 +- .../ui/test-panic-abort-nocapture.run.stdout | 2 +- src/test/ui/test-panic-abort.run.stdout | 2 +- src/test/ui/thread-local-in-ctfe.rs | 2 +- src/test/ui/thread-local-static.rs | 17 + src/test/ui/thread-local-static.stderr | 50 + .../auxiliary/thread-local-extern-static.rs | 2 +- ...ue-43733.stderr => issue-43733.mir.stderr} | 8 +- src/test/ui/threads-sendsync/issue-43733.rs | 20 +- .../threads-sendsync/issue-43733.thir.stderr | 19 + src/test/ui/tool_lints.rs | 1 - src/test/ui/tool_lints.stderr | 10 +- src/test/ui/traits/alias/style_lint.rs | 8 + src/test/ui/traits/alias/style_lint.stderr | 10 + src/test/ui/traits/bound/not-on-bare-trait.rs | 5 +- .../ui/traits/bound/not-on-bare-trait.stderr | 2 + src/test/ui/traits/cache-reached-depth-ice.rs | 45 + .../ui/traits/cache-reached-depth-ice.stderr | 11 + .../ui/traits/inductive-overflow/lifetime.rs | 2 + .../traits/inductive-overflow/lifetime.stderr | 4 +- src/test/ui/traits/issue-71136.stderr | 2 +- src/test/ui/traits/issue-75627.stderr | 4 +- src/test/ui/traits/issue-78372.stderr | 11 +- src/test/ui/traits/issue-79458.stderr | 2 +- .../issue-83538-tainted-cache-after-cycle.rs | 66 + ...sue-83538-tainted-cache-after-cycle.stderr | 38 + .../traits/issue-84399-bad-fresh-caching.rs | 55 + src/test/ui/traits/object/vs-lifetime.rs | 2 +- src/test/ui/traits/object/vs-lifetime.stderr | 10 +- ...-body.stderr => safety-fn-body.mir.stderr} | 4 +- src/test/ui/traits/safety-fn-body.rs | 3 + src/test/ui/traits/safety-fn-body.thir.stderr | 11 + src/test/ui/traits/test-2.rs | 4 +- src/test/ui/traits/test-2.stderr | 14 +- src/test/ui/try-block/try-block-bad-type.rs | 5 +- .../ui/try-block/try-block-bad-type.stderr | 31 +- src/test/ui/try-block/try-block-in-while.rs | 2 +- .../ui/try-block/try-block-in-while.stderr | 7 +- .../ui/try-block/try-block-opt-init.stderr | 2 +- .../ui/try-block/try-block-type-error.stderr | 4 +- src/test/ui/try-on-option.stderr | 33 - src/test/ui/try-operator-custom.rs | 63 - src/test/ui/try-trait/bad-interconversion.rs | 48 + .../ui/try-trait/bad-interconversion.stderr | 113 + .../ui/{ => try-trait}/option-to-result.rs | 4 +- src/test/ui/try-trait/option-to-result.stderr | 31 + src/test/ui/try-trait/try-as-monad.rs | 24 + .../try-on-option-diagnostics.rs | 0 .../try-on-option-diagnostics.stderr | 40 +- src/test/ui/{ => try-trait}/try-on-option.rs | 2 +- src/test/ui/try-trait/try-on-option.stderr | 31 + src/test/ui/try-trait/try-operator-custom.rs | 91 + .../{ => try-trait}/try-operator-on-main.rs | 7 +- .../try-operator-on-main.stderr | 38 +- src/test/ui/{ => try-trait}/try-poll.rs | 0 .../ui/tuple/tuple-struct-fields/test2.stderr | 2 +- .../ui/tuple/tuple-struct-fields/test3.stderr | 2 +- .../enum-variant-generic-args.rs | 12 +- .../enum-variant-generic-args.stderr | 36 +- .../issue-74761.full_tait.stderr | 6 +- .../issue-74761.min_tait.stderr | 4 +- .../ui/type-alias-impl-trait/issue-74761.rs | 1 - .../multiple-def-uses-in-one-fn-pass.rs | 12 + .../multiple-def-uses-in-one-fn.rs | 16 + .../multiple-def-uses-in-one-fn.stderr | 15 + .../multiple-def-uses-in-one-fn2.rs | 16 + .../multiple-def-uses-in-one-fn2.stderr | 8 + .../multiple-def-uses-in-one-fn3.rs | 18 + .../multiple-def-uses-in-one-fn3.stderr | 18 + .../ui/type/ascription/issue-34255-1.stderr | 8 +- .../ui/type/ascription/issue-47666.stderr | 2 +- .../cannot_infer_local_or_vec.stderr | 2 +- ...cannot_infer_local_or_vec_in_tuples.stderr | 2 +- src/test/ui/typeck/issue-75883.rs | 4 +- src/test/ui/typeck/issue-75883.stderr | 20 +- src/test/ui/typeck/issue-81943.stderr | 4 +- .../typeck-builtin-bound-type-parameters.rs | 8 +- ...ypeck-builtin-bound-type-parameters.stderr | 28 +- ...eck_type_placeholder_item.full_tait.stderr | 16 +- ...peck_type_placeholder_item.min_tait.stderr | 16 +- .../typeck_type_placeholder_item_help.stderr | 8 +- .../typeck_type_placeholder_lifetime_1.rs | 2 +- .../typeck_type_placeholder_lifetime_1.stderr | 8 +- .../typeck_type_placeholder_lifetime_2.rs | 2 +- .../typeck_type_placeholder_lifetime_2.stderr | 8 +- src/test/ui/ufcs/ufcs-qpath-missing-params.rs | 6 +- .../ui/ufcs/ufcs-qpath-missing-params.stderr | 22 +- .../unboxed-closure-sugar-used-on-struct-1.rs | 2 +- ...oxed-closure-sugar-used-on-struct-1.stderr | 12 +- .../unboxed-closure-sugar-used-on-struct.rs | 2 +- ...nboxed-closure-sugar-used-on-struct.stderr | 12 +- ...r-wrong-number-number-type-parameters-3.rs | 2 +- ...ong-number-number-type-parameters-3.stderr | 8 +- ...gar-wrong-number-number-type-parameters.rs | 10 +- ...wrong-number-number-type-parameters.stderr | 30 +- .../unboxed-closure-sugar-wrong-trait.rs | 2 +- .../unboxed-closure-sugar-wrong-trait.stderr | 6 +- src/test/ui/union/union-const-eval-field.rs | 2 - src/test/ui/union/union-derive-clone.stderr | 2 +- src/test/ui/union/union-derive-eq.stderr | 2 +- src/test/ui/unknown-lint-tool-name.rs | 2 - src/test/ui/unknown-lint-tool-name.stderr | 22 +- .../ui/unnamed_fields/restrict_anonymous.rs | 52 + .../unnamed_fields/restrict_anonymous.stderr | 175 + src/test/ui/unsafe/inline_asm.mir.stderr | 20 + src/test/ui/unsafe/inline_asm.rs | 10 + src/test/ui/unsafe/inline_asm.thir.stderr | 20 + ...issue-45087-unreachable-unsafe.mir.stderr} | 4 +- .../unsafe/issue-45087-unreachable-unsafe.rs | 3 + ...issue-45087-unreachable-unsafe.thir.stderr | 11 + ...fe-op-in-let-under-unsafe-under-closure.rs | 2 + ...ged_ints.stderr => ranged_ints.mir.stderr} | 2 +- src/test/ui/unsafe/ranged_ints.rs | 3 + src/test/ui/unsafe/ranged_ints.thir.stderr | 11 + ...st.stderr => ranged_ints_const.mir.stderr} | 2 +- src/test/ui/unsafe/ranged_ints_const.rs | 3 + .../ui/unsafe/ranged_ints_const.thir.stderr | 11 + src/test/ui/unsafe/ranged_ints_macro.rs | 3 + ...fc-2585-unsafe_op_in_unsafe_fn.mir.stderr} | 36 +- .../unsafe/rfc-2585-unsafe_op_in_unsafe_fn.rs | 3 + ...fc-2585-unsafe_op_in_unsafe_fn.thir.stderr | 122 + ...ound-compiler-generated-unsafe.mir.stderr} | 4 +- ...unsafe-around-compiler-generated-unsafe.rs | 3 + ...ound-compiler-generated-unsafe.thir.stderr | 14 + ...t-fn.stderr => unsafe-const-fn.mir.stderr} | 2 +- src/test/ui/unsafe/unsafe-const-fn.rs | 3 + .../ui/unsafe/unsafe-const-fn.thir.stderr | 11 + ... => unsafe-fn-assign-deref-ptr.mir.stderr} | 4 +- .../ui/unsafe/unsafe-fn-assign-deref-ptr.rs | 3 + .../unsafe-fn-assign-deref-ptr.thir.stderr | 11 + .../unsafe-fn-called-from-safe.mir.stderr | 11 + .../ui/unsafe/unsafe-fn-called-from-safe.rs | 3 + .../unsafe-fn-called-from-safe.thir.stderr | 11 + ....stderr => unsafe-fn-deref-ptr.mir.stderr} | 4 +- src/test/ui/unsafe/unsafe-fn-deref-ptr.rs | 3 + .../ui/unsafe/unsafe-fn-deref-ptr.thir.stderr | 11 + ...err => unsafe-fn-used-as-value.mir.stderr} | 2 +- src/test/ui/unsafe/unsafe-fn-used-as-value.rs | 3 + .../unsafe-fn-used-as-value.thir.stderr | 11 + ...rr => unsafe-unstable-const-fn.mir.stderr} | 4 +- .../ui/unsafe/unsafe-unstable-const-fn.rs | 4 +- .../unsafe-unstable-const-fn.thir.stderr | 11 + .../extern-loc-bad-loctype.rs | 2 +- .../extern-loc-json-bad-json.rs | 2 +- .../unused-crate-deps/extern-loc-json-json.rs | 2 +- .../extern-loc-json-json.stderr | 2 +- .../ui/unused-crate-deps/extern-loc-json.rs | 2 +- .../extern-loc-missing-loctype.rs | 2 +- .../unused-crate-deps/extern-loc-raw-json.rs | 2 +- .../extern-loc-raw-json.stderr | 2 +- .../extern-loc-raw-missing-loc.rs | 2 +- .../ui/unused-crate-deps/extern-loc-raw.rs | 2 +- src/test/ui/unused/unused-attr-macro-rules.rs | 34 + .../ui/unused/unused-attr-macro-rules.stderr | 32 + src/test/ui/unused/unused-macro-rules.stderr | 2 +- src/test/ui/unwind-no-uwtable.rs | 1 + .../variance-unused-type-param.stderr | 3 + src/test/ui/wf/wf-static-method.nll.stderr | 2 +- src/test/ui/wf/wf-static-method.rs | 3 +- src/test/ui/wf/wf-static-method.stderr | 29 +- .../ui/where-clauses/where-for-self.stderr | 1 + src/test/ui/while-let.stderr | 4 +- src/test/ui/x86stdcall.rs | 1 + src/tools/build-manifest/src/main.rs | 2 + src/tools/cargotest/main.rs | 4 +- src/tools/clippy/.cargo/config | 1 + .../clippy/.github/workflows/clippy_bors.yml | 5 + src/tools/clippy/CHANGELOG.md | 201 +- src/tools/clippy/CONTRIBUTING.md | 2 +- src/tools/clippy/COPYRIGHT | 2 +- src/tools/clippy/Cargo.toml | 8 +- src/tools/clippy/LICENSE-APACHE | 2 +- src/tools/clippy/LICENSE-MIT | 2 +- src/tools/clippy/README.md | 3 +- src/tools/clippy/build.rs | 2 +- src/tools/clippy/clippy.toml | 1 + src/tools/clippy/clippy_dev/src/lib.rs | 4 +- src/tools/clippy/clippy_dev/src/main.rs | 2 + src/tools/clippy/clippy_lints/Cargo.toml | 6 +- .../src/absurd_extreme_comparisons.rs | 3 +- .../clippy/clippy_lints/src/arithmetic.rs | 2 +- .../src/assertions_on_constants.rs | 4 +- src/tools/clippy/clippy_lints/src/attrs.rs | 6 +- src/tools/clippy/clippy_lints/src/bit_mask.rs | 8 +- src/tools/clippy/clippy_lints/src/booleans.rs | 6 +- .../clippy_lints/src/casts/cast_sign_loss.rs | 2 +- .../clippy/clippy_lints/src/collapsible_if.rs | 2 +- .../clippy_lints/src/comparison_chain.rs | 8 +- src/tools/clippy/clippy_lints/src/consts.rs | 1 - src/tools/clippy/clippy_lints/src/copies.rs | 2 +- src/tools/clippy/clippy_lints/src/default.rs | 3 +- .../src/default_numeric_fallback.rs | 4 +- .../clippy_lints/src/deprecated_lints.rs | 30 +- src/tools/clippy/clippy_lints/src/derive.rs | 16 +- src/tools/clippy/clippy_lints/src/doc.rs | 2 +- .../clippy_lints/src/double_comparison.rs | 8 +- .../clippy_lints/src/duration_subsec.rs | 2 +- src/tools/clippy/clippy_lints/src/entry.rs | 2 +- .../clippy/clippy_lints/src/enum_clike.rs | 2 +- .../clippy/clippy_lints/src/enum_variants.rs | 80 +- src/tools/clippy/clippy_lints/src/eq_op.rs | 10 +- .../clippy/clippy_lints/src/erasing_op.rs | 3 +- .../clippy/clippy_lints/src/eta_reduction.rs | 31 +- .../clippy_lints/src/eval_order_dependence.rs | 32 +- .../src/floating_point_arithmetic.rs | 14 +- .../clippy_lints/src/functions/must_use.rs | 2 +- .../src/functions/not_unsafe_ptr_arg_deref.rs | 2 +- .../src/functions/too_many_lines.rs | 24 +- .../clippy_lints/src/future_not_send.rs | 2 +- .../clippy/clippy_lints/src/identity_op.rs | 2 +- .../clippy_lints/src/implicit_return.rs | 255 +- .../src/inconsistent_struct_constructor.rs | 2 +- .../clippy_lints/src/indexing_slicing.rs | 2 +- .../clippy/clippy_lints/src/infinite_iter.rs | 2 +- .../clippy/clippy_lints/src/inherent_impl.rs | 134 +- .../src/invalid_upcast_comparisons.rs | 7 +- src/tools/clippy/clippy_lints/src/len_zero.rs | 4 +- .../clippy/clippy_lints/src/let_underscore.rs | 8 +- src/tools/clippy/clippy_lints/src/lib.rs | 1099 +- .../clippy/clippy_lints/src/lifetimes.rs | 4 +- .../src/literal_representation.rs | 8 +- .../src/loops/explicit_iter_loop.rs | 2 +- .../clippy_lints/src/loops/mut_range_bound.rs | 8 +- .../src/loops/needless_collect.rs | 86 +- .../clippy_lints/src/loops/same_item_push.rs | 2 +- .../clippy/clippy_lints/src/loops/utils.rs | 8 +- .../src/loops/while_immutable_condition.rs | 15 +- .../src/loops/while_let_on_iterator.rs | 445 +- .../clippy/clippy_lints/src/macro_use.rs | 13 +- .../clippy/clippy_lints/src/manual_strip.rs | 4 +- .../clippy_lints/src/manual_unwrap_or.rs | 26 +- .../clippy/clippy_lints/src/map_clone.rs | 6 +- src/tools/clippy/clippy_lints/src/matches.rs | 47 +- .../clippy_lints/src/mem_discriminant.rs | 3 +- .../src/methods/bind_instead_of_map.rs | 2 +- .../clippy_lints/src/methods/clone_on_copy.rs | 5 +- .../src/methods/cloned_instead_of_copied.rs | 2 +- .../src/methods/flat_map_option.rs | 2 +- .../methods/from_iter_instead_of_collect.rs | 47 +- .../clippy_lints/src/methods/iter_nth_zero.rs | 2 +- .../src/methods/iterator_step_by_zero.rs | 2 +- .../src/methods/manual_str_repeat.rs | 99 + .../clippy/clippy_lints/src/methods/mod.rs | 156 +- .../src/methods/single_char_pattern.rs | 24 +- .../src/methods/suspicious_splitn.rs | 56 + .../src/methods/unnecessary_filter_map.rs | 40 +- .../src/methods/unnecessary_fold.rs | 2 +- .../src/methods/wrong_self_convention.rs | 15 +- src/tools/clippy/clippy_lints/src/minmax.rs | 2 +- src/tools/clippy/clippy_lints/src/misc.rs | 15 +- .../clippy/clippy_lints/src/misc_early.rs | 569 - .../src/misc_early/builtin_type_shadow.rs | 19 + .../clippy_lints/src/misc_early/double_neg.rs | 23 + .../src/misc_early/mixed_case_hex_literals.rs | 34 + .../clippy/clippy_lints/src/misc_early/mod.rs | 348 + .../src/misc_early/redundant_pattern.rs | 31 + .../src/misc_early/unneeded_field_pattern.rs | 72 + .../misc_early/unneeded_wildcard_pattern.rs | 52 + .../misc_early/unseparated_literal_suffix.rs | 26 + .../src/misc_early/zero_prefixed_literal.rs | 29 + .../clippy/clippy_lints/src/missing_doc.rs | 23 +- .../clippy_lints/src/modulo_arithmetic.rs | 2 +- .../clippy/clippy_lints/src/mut_reference.rs | 2 +- .../src/mutable_debug_assertion.rs | 2 +- .../src/needless_arbitrary_self_type.rs | 4 +- .../clippy_lints/src/needless_bitwise_bool.rs | 86 + .../clippy/clippy_lints/src/needless_bool.rs | 18 +- .../clippy_lints/src/needless_borrow.rs | 219 +- .../clippy_lints/src/needless_for_each.rs | 4 +- .../src/needless_question_mark.rs | 112 +- .../src/neg_cmp_op_on_partial_ord.rs | 2 +- .../clippy/clippy_lints/src/neg_multiply.rs | 3 +- .../clippy/clippy_lints/src/no_effect.rs | 13 +- .../clippy_lints/src/non_expressive_names.rs | 3 +- .../clippy/clippy_lints/src/open_options.rs | 10 +- .../clippy_lints/src/option_if_let_else.rs | 47 +- .../clippy_lints/src/pass_by_ref_or_value.rs | 13 +- src/tools/clippy/clippy_lints/src/ptr.rs | 2 +- .../clippy/clippy_lints/src/question_mark.rs | 4 +- src/tools/clippy/clippy_lints/src/ranges.rs | 2 +- .../src/redundant_closure_call.rs | 2 +- .../clippy_lints/src/redundant_pub_crate.rs | 2 +- src/tools/clippy/clippy_lints/src/regex.rs | 2 +- .../clippy/clippy_lints/src/repeat_once.rs | 2 +- src/tools/clippy/clippy_lints/src/returns.rs | 4 +- .../src/semicolon_if_nothing_returned.rs | 10 +- src/tools/clippy/clippy_lints/src/shadow.rs | 10 +- .../src/single_component_path_imports.rs | 4 +- .../src/slow_vector_initialization.rs | 24 +- .../src/suspicious_operation_groupings.rs | 8 +- .../clippy/clippy_lints/src/trait_bounds.rs | 3 +- .../clippy_lints/src/transmuting_null.rs | 2 +- src/tools/clippy/clippy_lints/src/try_err.rs | 2 +- .../clippy/clippy_lints/src/types/mod.rs | 4 +- src/tools/clippy/clippy_lints/src/unicode.rs | 4 +- .../clippy_lints/src/unit_types/unit_cmp.rs | 7 +- .../clippy_lints/src/unnecessary_wraps.rs | 21 +- .../clippy_lints/src/unnested_or_patterns.rs | 16 +- .../clippy/clippy_lints/src/unused_async.rs | 92 + .../clippy_lints/src/unused_io_amount.rs | 4 +- .../clippy/clippy_lints/src/unused_unit.rs | 4 +- .../clippy_lints/src/upper_case_acronyms.rs | 34 +- src/tools/clippy/clippy_lints/src/use_self.rs | 14 +- .../clippy_lints/src/useless_conversion.rs | 15 +- .../clippy/clippy_lints/src/utils/author.rs | 2 +- .../clippy/clippy_lints/src/utils/conf.rs | 298 +- .../clippy_lints/src/utils/internal_lints.rs | 9 +- .../internal_lints/metadata_collector.rs | 845 + .../clippy/clippy_lints/src/utils/mod.rs | 2 +- src/tools/clippy/clippy_lints/src/vec.rs | 2 +- .../clippy_lints/src/verbose_file_reads.rs | 2 +- src/tools/clippy/clippy_lints/src/write.rs | 25 +- .../clippy/clippy_lints/src/zero_div_zero.rs | 2 +- src/tools/clippy/clippy_utils/Cargo.toml | 4 +- .../clippy/clippy_utils/src/ast_utils.rs | 17 +- src/tools/clippy/clippy_utils/src/attrs.rs | 2 +- src/tools/clippy/clippy_utils/src/consts.rs | 22 +- .../clippy/clippy_utils/src/diagnostics.rs | 12 +- src/tools/clippy/clippy_utils/src/higher.rs | 36 +- .../clippy/clippy_utils/src/hir_utils.rs | 348 +- src/tools/clippy/clippy_utils/src/lib.rs | 176 +- src/tools/clippy/clippy_utils/src/msrvs.rs | 1 + .../clippy_utils/src/numeric_literal.rs | 2 +- src/tools/clippy/clippy_utils/src/paths.rs | 15 +- src/tools/clippy/clippy_utils/src/ptr.rs | 4 +- .../clippy_utils/src/qualify_min_const_fn.rs | 2 +- src/tools/clippy/clippy_utils/src/source.rs | 53 +- src/tools/clippy/clippy_utils/src/sugg.rs | 31 +- src/tools/clippy/clippy_utils/src/ty.rs | 48 +- src/tools/clippy/clippy_utils/src/usage.rs | 6 +- src/tools/clippy/clippy_utils/src/visitors.rs | 78 +- src/tools/clippy/doc/adding_lints.md | 4 +- src/tools/clippy/doc/basics.md | 2 + src/tools/clippy/doc/release.md | 2 +- src/tools/clippy/mini-macro/Cargo.toml | 14 - src/tools/clippy/mini-macro/src/lib.rs | 29 - src/tools/clippy/rust-toolchain | 2 +- src/tools/clippy/rustc_tools_util/src/lib.rs | 12 +- src/tools/clippy/src/driver.rs | 2 +- src/tools/clippy/tests/clippy.toml | 1 + src/tools/clippy/tests/compile-test.rs | 42 +- src/tools/clippy/tests/dogfood.rs | 100 +- .../tests/ui-internal/default_lint.stderr | 2 +- .../tests/ui-internal/if_chain_style.stderr | 2 +- .../ui-internal/lint_without_lint_pass.stderr | 2 +- .../bad_toml_type/conf_bad_type.stderr | 2 +- .../conf_deprecated_key.stderr | 2 +- .../toml_unknown_key/conf_unknown_key.stderr | 2 +- .../tests/ui/assertions_on_constants.stderr | 18 +- .../tests/ui/auxiliary/proc_macro_derive.rs | 19 + ...-type-shadow.rs => builtin_type_shadow.rs} | 0 ...adow.stderr => builtin_type_shadow.stderr} | 4 +- .../checked_unwrap/simple_conditionals.stderr | 2 +- .../clippy/tests/ui/collapsible_match2.stderr | 2 +- src/tools/clippy/tests/ui/comparison_chain.rs | 28 + src/tools/clippy/tests/ui/copy_iterator.rs | 6 +- .../ui/crashes/auxiliary/ice-7272-aux.rs | 14 + .../clippy/tests/ui/crashes/ice-3969.stderr | 8 + .../clippy/tests/ui/crashes/ice-6255.stderr | 2 +- src/tools/clippy/tests/ui/crashes/ice-7231.rs | 10 + src/tools/clippy/tests/ui/crashes/ice-7272.rs | 12 + .../tests/ui/crashes/procedural_macro.rs | 11 - .../others.stderr | 2 +- .../traits.stderr | 2 +- src/tools/clippy/tests/ui/def_id_nocore.rs | 2 +- src/tools/clippy/tests/ui/deprecated.rs | 2 + src/tools/clippy/tests/ui/deprecated.stderr | 16 +- .../clippy/tests/ui/deprecated_old.stderr | 8 +- src/tools/clippy/tests/ui/deref_addrof.stderr | 4 +- .../clippy/tests/ui/derive_hash_xor_eq.stderr | 8 +- .../ui/derive_ord_xor_partial_ord.stderr | 8 +- src/tools/clippy/tests/ui/doc_unsafe.stderr | 2 +- src/tools/clippy/tests/ui/enum_variants.rs | 7 +- .../clippy/tests/ui/enum_variants.stderr | 37 +- src/tools/clippy/tests/ui/eq_op_macros.stderr | 8 +- src/tools/clippy/tests/ui/eta.fixed | 3 + src/tools/clippy/tests/ui/eta.rs | 3 + src/tools/clippy/tests/ui/eta.stderr | 16 +- .../clippy/tests/ui/eval_order_dependence.rs | 6 + .../tests/ui/eval_order_dependence.stderr | 24 +- .../clippy/tests/ui/fallible_impl_from.stderr | 6 +- .../clippy/tests/ui/floating_point_powi.fixed | 5 +- .../clippy/tests/ui/floating_point_powi.rs | 5 +- .../tests/ui/floating_point_powi.stderr | 34 +- .../ui/from_iter_instead_of_collect.fixed | 14 + .../tests/ui/from_iter_instead_of_collect.rs | 14 + .../ui/from_iter_instead_of_collect.stderr | 40 +- src/tools/clippy/tests/ui/functions.stderr | 18 +- src/tools/clippy/tests/ui/impl.rs | 31 + src/tools/clippy/tests/ui/impl.stderr | 30 +- .../clippy/tests/ui/implicit_hasher.stderr | 6 +- .../clippy/tests/ui/implicit_return.fixed | 68 +- src/tools/clippy/tests/ui/implicit_return.rs | 68 +- .../clippy/tests/ui/implicit_return.stderr | 55 +- src/tools/clippy/tests/ui/infinite_loop.rs | 12 + .../tests/ui/item_after_statement.stderr | 2 +- .../clippy/tests/ui/macro_use_imports.fixed | 3 +- .../clippy/tests/ui/macro_use_imports.rs | 3 +- .../clippy/tests/ui/macro_use_imports.stderr | 14 +- .../clippy/tests/ui/manual_str_repeat.fixed | 66 + .../clippy/tests/ui/manual_str_repeat.rs | 66 + .../clippy/tests/ui/manual_str_repeat.stderr | 64 + .../clippy/tests/ui/manual_unwrap_or.fixed | 15 + src/tools/clippy/tests/ui/manual_unwrap_or.rs | 15 + .../clippy/tests/ui/match_same_arms2.stderr | 4 +- .../tests/ui/match_single_binding.fixed | 5 +- .../clippy/tests/ui/match_single_binding.rs | 10 +- .../tests/ui/match_single_binding.stderr | 13 +- .../tests/ui/match_single_binding2.fixed | 16 + .../clippy/tests/ui/match_single_binding2.rs | 18 + .../tests/ui/match_single_binding2.stderr | 36 +- .../clippy/tests/ui/mem_discriminant.stderr | 2 +- .../clippy/tests/ui/mem_replace_macro.stderr | 2 +- .../clippy/tests/ui/missing-doc-crate.rs | 3 +- src/tools/clippy/tests/ui/missing-doc-impl.rs | 5 +- .../clippy/tests/ui/missing-doc-impl.stderr | 8 +- .../clippy/tests/ui/missing_panics_doc.stderr | 12 +- .../tests/ui/module_name_repetitions.rs | 8 - src/tools/clippy/tests/ui/mut_mut.stderr | 2 +- .../tests/ui/needless_bitwise_bool.fixed | 40 + .../clippy/tests/ui/needless_bitwise_bool.rs | 40 + .../tests/ui/needless_bitwise_bool.stderr | 10 + .../clippy/tests/ui/needless_borrow.fixed | 17 - src/tools/clippy/tests/ui/needless_borrow.rs | 17 - .../clippy/tests/ui/needless_borrow.stderr | 16 +- .../clippy/tests/ui/needless_borrow_pat.rs | 151 + .../tests/ui/needless_borrow_pat.stderr | 112 + .../clippy/tests/ui/needless_collect.fixed | 19 +- src/tools/clippy/tests/ui/needless_collect.rs | 17 +- .../clippy/tests/ui/needless_collect.stderr | 50 +- .../tests/ui/needless_collect_indirect.rs | 40 +- .../tests/ui/needless_collect_indirect.stderr | 58 +- .../tests/ui/needless_question_mark.fixed | 5 + .../clippy/tests/ui/needless_question_mark.rs | 5 + .../tests/ui/needless_question_mark.stderr | 4 +- .../clippy/tests/ui/new_without_default.rs | 1 - .../tests/ui/new_without_default.stderr | 12 +- src/tools/clippy/tests/ui/no_effect.rs | 3 + .../clippy/tests/ui/option_env_unwrap.stderr | 8 +- .../clippy/tests/ui/option_if_let_else.fixed | 6 +- .../clippy/tests/ui/option_if_let_else.stderr | 13 +- .../clippy/tests/ui/panic_in_result_fn.stderr | 12 +- .../ui/panic_in_result_fn_assertions.stderr | 6 +- .../clippy/tests/ui/panicking_macros.stderr | 24 +- .../ui/pattern_type_mismatch/syntax.stderr | 2 +- .../tests/ui/ref_binding_to_reference.rs | 76 + .../tests/ui/ref_binding_to_reference.stderr | 88 + src/tools/clippy/tests/ui/rename.stderr | 8 +- src/tools/clippy/tests/ui/similar_names.rs | 4 + .../clippy/tests/ui/similar_names.stderr | 4 +- .../clippy/tests/ui/single_char_pattern.fixed | 6 +- .../clippy/tests/ui/single_char_pattern.rs | 6 +- .../tests/ui/single_char_pattern.stderr | 38 +- .../clippy/tests/ui/suspicious_splitn.rs | 20 + .../clippy/tests/ui/suspicious_splitn.stderr | 75 + .../clippy/tests/ui/toplevel_ref_arg.stderr | 2 +- .../ui/toplevel_ref_arg_non_rustfix.stderr | 2 +- .../ui/trivially_copy_pass_by_ref.stderr | 8 +- src/tools/clippy/tests/ui/try_err.stderr | 4 +- src/tools/clippy/tests/ui/unit_cmp.stderr | 8 +- .../tests/ui/unknown_clippy_lints.stderr | 8 +- .../clippy/tests/ui/unnecessary_filter_map.rs | 4 + .../clippy/tests/ui/unnecessary_wraps.rs | 2 +- .../tests/ui/unsafe_derive_deserialize.stderr | 8 +- .../ui/unseparated_prefix_literals.fixed | 3 +- .../tests/ui/unseparated_prefix_literals.rs | 3 +- .../ui/unseparated_prefix_literals.stderr | 20 +- src/tools/clippy/tests/ui/unused_async.rs | 15 + src/tools/clippy/tests/ui/unused_async.stderr | 13 + src/tools/clippy/tests/ui/unused_unit.fixed | 7 + src/tools/clippy/tests/ui/unused_unit.rs | 7 + src/tools/clippy/tests/ui/use_self.fixed | 30 + src/tools/clippy/tests/ui/use_self.rs | 30 + src/tools/clippy/tests/ui/use_self.stderr | 8 +- .../clippy/tests/ui/useless_conversion.fixed | 19 + .../clippy/tests/ui/useless_conversion.rs | 19 + .../clippy/tests/ui/useless_conversion.stderr | 20 +- .../tests/ui/while_let_on_iterator.fixed | 175 +- .../clippy/tests/ui/while_let_on_iterator.rs | 175 +- .../tests/ui/while_let_on_iterator.stderr | 78 +- .../tests/ui/wildcard_enum_match_arm.fixed | 3 +- .../tests/ui/wildcard_enum_match_arm.rs | 1 + .../tests/ui/wildcard_enum_match_arm.stderr | 2 +- .../clippy/tests/ui/write_with_newline.stderr | 4 +- .../clippy/tests/ui/wrong_self_convention.rs | 1 - .../tests/ui/wrong_self_convention.stderr | 48 +- .../clippy/tests/ui/wrong_self_convention2.rs | 30 +- .../tests/ui/wrong_self_convention2.stderr | 18 +- src/tools/clippy/util/cov.sh | 37 - src/tools/clippy/util/lintlib.py | 2 +- src/tools/compiletest/src/common.rs | 25 + src/tools/compiletest/src/header.rs | 29 +- src/tools/compiletest/src/header/tests.rs | 15 + src/tools/compiletest/src/main.rs | 33 +- src/tools/compiletest/src/runtest.rs | 135 +- src/tools/compiletest/src/util.rs | 2 + src/tools/jsondocck/Cargo.toml | 5 +- src/tools/jsondocck/src/main.rs | 12 +- src/tools/linkchecker/linkcheck.sh | 10 +- src/tools/linkchecker/main.rs | 547 +- .../linkchecker/tests/basic_broken/foo.html | 5 + .../tests/broken_fragment_local/foo.html | 5 + .../tests/broken_fragment_remote/bar.html | 4 + .../broken_fragment_remote/inner/foo.html | 5 + .../linkchecker/tests/broken_redir/foo.html | 5 + .../tests/broken_redir/redir-bad.html | 10 + src/tools/linkchecker/tests/checks.rs | 77 + .../linkchecker/tests/directory_link/foo.html | 5 + .../tests/directory_link/somedir/index.html | 4 + .../linkchecker/tests/redirect_loop/foo.html | 5 + .../tests/redirect_loop/redir-bad.html | 10 + .../linkchecker/tests/valid/inner/bar.html | 7 + .../linkchecker/tests/valid/inner/foo.html | 14 + .../tests/valid/inner/redir-bad.html | 11 + .../tests/valid/inner/redir-target.html | 5 + .../linkchecker/tests/valid/inner/redir.html | 10 + src/tools/linkchecker/tests/valid/outer.html | 5 + src/tools/lint-docs/src/groups.rs | 1 + src/tools/rustdoc-gui/tester.js | 61 +- src/tools/rustdoc-js/tester.js | 3 +- src/tools/rustfmt/Cargo.toml | 2 +- src/tools/rustfmt/rust-toolchain | 2 +- src/tools/rustfmt/src/config/file_lines.rs | 2 +- src/tools/rustfmt/src/expr.rs | 4 +- src/tools/rustfmt/src/items.rs | 11 +- src/tools/rustfmt/src/lib.rs | 7 +- src/tools/rustfmt/src/modules.rs | 2 +- src/tools/rustfmt/src/patterns.rs | 6 +- src/tools/rustfmt/src/source_file.rs | 2 +- src/tools/rustfmt/src/syntux/session.rs | 19 +- src/tools/rustfmt/src/test/mod.rs | 1 + src/tools/rustfmt/src/test/mod_resolver.rs | 25 + src/tools/rustfmt/src/types.rs | 57 +- .../tests/mod-resolver/issue-4874/bar/baz.rs | 5 + .../tests/mod-resolver/issue-4874/foo.rs | 1 + .../tests/mod-resolver/issue-4874/foo/qux.rs | 5 + .../tests/mod-resolver/issue-4874/main.rs | 8 + src/tools/tidy/src/deps.rs | 174 +- src/tools/tidy/src/error_codes_check.rs | 27 +- src/tools/tidy/src/features.rs | 14 +- src/tools/tidy/src/pal.rs | 76 +- src/tools/tidy/src/ui_tests.rs | 4 +- src/tools/tier-check/src/main.rs | 2 +- src/version | 2 +- .../.cargo-checksum.json | 2 +- .../CHANGELOG.md | 14 - .../Cargo.lock | 6 +- .../Cargo.toml | 6 +- .../LICENSE-APACHE | 0 .../LICENSE-MIT | 0 .../{addr2line => addr2line-0.14.0}/README.md | 0 .../bench.plot.r | 0 .../benchmark.sh | 0 .../{addr2line => addr2line-0.14.0}/coverage | 0 .../examples/addr2line.rs | 2 - .../rustfmt.toml | 0 .../src/lazy.rs | 0 .../src/lib.rs | 324 +- .../tests/correctness.rs | 0 .../tests/output_equivalence.rs | 0 .../tests/parse.rs | 0 vendor/cc/.cargo-checksum.json | 2 +- vendor/cc/Cargo.lock | 2 +- vendor/cc/Cargo.toml | 2 +- vendor/cc/src/lib.rs | 10 +- vendor/cc/src/vs_instances.rs | 199 + vendor/cc/src/windows_registry.rs | 180 +- vendor/compiler_builtins/.cargo-checksum.json | 2 +- vendor/compiler_builtins/Cargo.lock | 4 +- vendor/compiler_builtins/Cargo.toml | 3 +- vendor/compiler_builtins/build.rs | 66 +- .../compiler_builtins/examples/intrinsics.rs | 9 +- vendor/compiler_builtins/src/arm.rs | 193 +- vendor/compiler_builtins/src/float/add.rs | 5 +- vendor/compiler_builtins/src/float/cmp.rs | 21 +- vendor/compiler_builtins/src/float/conv.rs | 299 +- vendor/compiler_builtins/src/float/div.rs | 4 + vendor/compiler_builtins/src/float/mod.rs | 45 +- vendor/compiler_builtins/src/float/mul.rs | 2 +- vendor/compiler_builtins/src/float/pow.rs | 46 +- .../src/int/leading_zeros.rs | 10 +- vendor/compiler_builtins/src/int/mod.rs | 112 +- vendor/compiler_builtins/src/int/shift.rs | 16 - .../src/int/specialized_div_rem/asymmetric.rs | 3 +- .../int/specialized_div_rem/binary_long.rs | 3 +- .../src/int/specialized_div_rem/delegate.rs | 10 +- .../src/int/specialized_div_rem/mod.rs | 14 + .../src/int/specialized_div_rem/norm_shift.rs | 3 +- .../src/int/specialized_div_rem/trifecta.rs | 3 +- vendor/compiler_builtins/src/int/udiv.rs | 7 +- vendor/compiler_builtins/src/lib.rs | 14 +- vendor/compiler_builtins/src/macros.rs | 16 + vendor/compiler_builtins/src/mem/mod.rs | 11 + vendor/compiler_builtins/src/probestack.rs | 6 + vendor/compiler_builtins/src/x86.rs | 96 +- vendor/compiler_builtins/src/x86_64.rs | 99 +- vendor/crossbeam-epoch/.cargo-checksum.json | 2 +- vendor/crossbeam-epoch/CHANGELOG.md | 11 + vendor/crossbeam-epoch/Cargo.lock | 39 +- vendor/crossbeam-epoch/Cargo.toml | 6 +- vendor/crossbeam-epoch/README.md | 2 +- vendor/crossbeam-epoch/build.rs | 32 + vendor/crossbeam-epoch/no_atomic.rs | 59 + vendor/crossbeam-epoch/src/atomic.rs | 30 +- vendor/crossbeam-epoch/src/lib.rs | 11 +- vendor/crossbeam-utils/.cargo-checksum.json | 2 +- vendor/crossbeam-utils/CHANGELOG.md | 9 + vendor/crossbeam-utils/Cargo.toml | 6 +- vendor/crossbeam-utils/README.md | 2 +- vendor/crossbeam-utils/build.rs | 35 +- vendor/crossbeam-utils/no_atomic.rs | 59 + .../crossbeam-utils/src/atomic/atomic_cell.rs | 79 +- vendor/crossbeam-utils/src/atomic/consume.rs | 24 +- vendor/crossbeam-utils/src/atomic/mod.rs | 8 +- vendor/crossbeam-utils/src/lib.rs | 27 +- vendor/crossbeam-utils/tests/atomic_cell.rs | 54 +- vendor/crossbeam-utils/tests/cache_padded.rs | 1 + vendor/crossbeam-utils/tests/sharded_lock.rs | 2 +- .../.cargo-checksum.json | 0 vendor/{gimli => gimli-0.23.0}/CHANGELOG.md | 0 .../{gimli => gimli-0.23.0}/CONTRIBUTING.md | 0 vendor/{gimli => gimli-0.23.0}/Cargo.lock | 0 vendor/{gimli => gimli-0.23.0}/Cargo.toml | 0 vendor/{gimli => gimli-0.23.0}/LICENSE-APACHE | 0 vendor/{gimli => gimli-0.23.0}/LICENSE-MIT | 0 vendor/{gimli => gimli-0.23.0}/README.md | 0 .../{gimli => gimli-0.23.0}/benches/bench.rs | 0 .../examples/dwarf-validate.rs | 0 .../examples/dwarfdump.rs | 0 .../examples/simple.rs | 0 .../examples/simple_line.rs | 0 .../fixtures/self/README.md | 0 .../fixtures/self/debug_abbrev | Bin .../fixtures/self/debug_aranges | Bin .../fixtures/self/debug_info | Bin .../fixtures/self/debug_inlined | Bin .../fixtures/self/debug_line | Bin .../fixtures/self/debug_loc | Bin .../fixtures/self/debug_pubnames | Bin .../fixtures/self/debug_pubtypes | Bin .../fixtures/self/debug_ranges | Bin .../fixtures/self/debug_str | Bin .../fixtures/self/eh_frame | Bin .../fixtures/self/eh_frame_hdr | Bin vendor/gimli-0.23.0/rustfmt.toml | 0 vendor/{gimli => gimli-0.23.0}/src/arch.rs | 0 vendor/{gimli => gimli-0.23.0}/src/common.rs | 0 .../{gimli => gimli-0.23.0}/src/constants.rs | 0 .../{gimli => gimli-0.23.0}/src/endianity.rs | 0 vendor/{gimli => gimli-0.23.0}/src/leb128.rs | 0 vendor/{gimli => gimli-0.23.0}/src/lib.rs | 0 .../src/read/abbrev.rs | 0 .../{gimli => gimli-0.23.0}/src/read/addr.rs | 0 .../src/read/aranges.rs | 0 .../{gimli => gimli-0.23.0}/src/read/cfi.rs | 0 .../{gimli => gimli-0.23.0}/src/read/dwarf.rs | 0 .../src/read/endian_reader.rs | 0 .../src/read/endian_slice.rs | 0 .../{gimli => gimli-0.23.0}/src/read/line.rs | 0 .../{gimli => gimli-0.23.0}/src/read/lists.rs | 0 .../src/read/loclists.rs | 0 .../src/read/lookup.rs | 0 .../{gimli => gimli-0.23.0}/src/read/mod.rs | 0 vendor/{gimli => gimli-0.23.0}/src/read/op.rs | 0 .../src/read/pubnames.rs | 0 .../src/read/pubtypes.rs | 0 .../src/read/reader.rs | 0 .../src/read/rnglists.rs | 0 .../{gimli => gimli-0.23.0}/src/read/str.rs | 0 .../{gimli => gimli-0.23.0}/src/read/unit.rs | 0 .../{gimli => gimli-0.23.0}/src/read/value.rs | 0 .../{gimli => gimli-0.23.0}/src/test_util.rs | 0 .../src/write/abbrev.rs | 0 .../{gimli => gimli-0.23.0}/src/write/cfi.rs | 0 .../src/write/dwarf.rs | 0 .../src/write/endian_vec.rs | 0 .../{gimli => gimli-0.23.0}/src/write/line.rs | 0 .../{gimli => gimli-0.23.0}/src/write/loc.rs | 0 .../{gimli => gimli-0.23.0}/src/write/mod.rs | 0 .../{gimli => gimli-0.23.0}/src/write/op.rs | 0 .../src/write/range.rs | 0 .../src/write/section.rs | 0 .../{gimli => gimli-0.23.0}/src/write/str.rs | 0 .../{gimli => gimli-0.23.0}/src/write/unit.rs | 0 .../src/write/writer.rs | 0 .../tests/convert_self.rs | 0 .../tests/parse_self.rs | 0 vendor/libc/.cargo-checksum.json | 2 +- vendor/libc/Cargo.toml | 2 +- vendor/libc/build.rs | 7 - vendor/libc/src/fuchsia/mod.rs | 1 + vendor/libc/src/unix/bsd/apple/mod.rs | 243 + .../src/unix/bsd/freebsdlike/dragonfly/mod.rs | 33 + .../bsd/freebsdlike/freebsd/freebsd12/mod.rs | 1 + .../bsd/freebsdlike/freebsd/freebsd13/mod.rs | 2 + .../src/unix/bsd/freebsdlike/freebsd/mod.rs | 87 + vendor/libc/src/unix/bsd/freebsdlike/mod.rs | 6 + vendor/libc/src/unix/bsd/netbsdlike/mod.rs | 6 + .../src/unix/bsd/netbsdlike/netbsd/mod.rs | 23 +- .../src/unix/bsd/netbsdlike/netbsd/x86_64.rs | 17 + .../src/unix/bsd/netbsdlike/openbsd/mod.rs | 23 + vendor/libc/src/unix/haiku/native.rs | 83 +- .../src/unix/linux_like/android/b32/mod.rs | 4 +- .../linux_like/android/b64/aarch64/mod.rs | 4 +- .../libc/src/unix/linux_like/android/mod.rs | 86 +- .../src/unix/linux_like/emscripten/mod.rs | 13 +- .../libc/src/unix/linux_like/linux/gnu/mod.rs | 209 +- vendor/libc/src/unix/linux_like/linux/mod.rs | 196 +- .../unix/linux_like/linux/musl/b32/arm/mod.rs | 1 + .../linux_like/linux/musl/b32/mips/mod.rs | 1 + .../unix/linux_like/linux/musl/b32/powerpc.rs | 1 + .../unix/linux_like/linux/musl/b32/x86/mod.rs | 1 + .../linux_like/linux/musl/b64/aarch64/mod.rs | 35 + .../linux_like/linux/musl/b64/powerpc64.rs | 1 + .../unix/linux_like/linux/musl/b64/s390x.rs | 4 + .../linux_like/linux/musl/b64/x86_64/mod.rs | 1 + .../src/unix/linux_like/linux/musl/mod.rs | 15 +- .../unix/linux_like/linux/uclibc/mips/mod.rs | 1 + vendor/libc/src/unix/linux_like/mod.rs | 38 + vendor/libc/src/unix/redox/mod.rs | 29 + vendor/libc/src/windows/mod.rs | 31 +- vendor/lock_api/.cargo-checksum.json | 2 +- vendor/lock_api/Cargo.toml | 2 +- vendor/lock_api/src/lib.rs | 2 +- vendor/measureme/.cargo-checksum.json | 2 +- vendor/measureme/Cargo.toml | 6 +- vendor/measureme/src/counters.rs | 2024 +- vendor/measureme/src/lib.rs | 116 +- vendor/measureme/src/profiler.rs | 398 +- vendor/memchr/.cargo-checksum.json | 2 +- vendor/memchr/Cargo.toml | 14 +- vendor/memchr/README.md | 66 +- vendor/memchr/src/cow.rs | 97 + vendor/memchr/src/lib.rs | 588 +- vendor/memchr/src/{ => memchr}/c.rs | 8 +- vendor/memchr/src/{ => memchr}/fallback.rs | 21 +- vendor/memchr/src/{ => memchr}/iter.rs | 8 +- vendor/memchr/src/memchr/mod.rs | 410 + vendor/memchr/src/{ => memchr}/naive.rs | 0 vendor/memchr/src/{ => memchr}/x86/avx.rs | 164 +- vendor/memchr/src/memchr/x86/mod.rs | 148 + vendor/memchr/src/{ => memchr}/x86/sse2.rs | 16 +- vendor/memchr/src/{ => memchr}/x86/sse42.rs | 17 +- vendor/memchr/src/memmem/byte_frequencies.rs | 258 + vendor/memchr/src/memmem/genericsimd.rs | 264 + vendor/memchr/src/memmem/mod.rs | 1296 + .../memchr/src/memmem/prefilter/fallback.rs | 122 + .../src/memmem/prefilter/genericsimd.rs | 207 + vendor/memchr/src/memmem/prefilter/mod.rs | 562 + vendor/memchr/src/memmem/prefilter/x86/avx.rs | 46 + vendor/memchr/src/memmem/prefilter/x86/mod.rs | 5 + vendor/memchr/src/memmem/prefilter/x86/sse.rs | 55 + vendor/memchr/src/memmem/rabinkarp.rs | 233 + vendor/memchr/src/memmem/rarebytes.rs | 136 + vendor/memchr/src/memmem/twoway.rs | 878 + vendor/memchr/src/memmem/util.rs | 88 + vendor/memchr/src/memmem/vector.rs | 98 + vendor/memchr/src/memmem/x86/avx.rs | 139 + vendor/memchr/src/memmem/x86/mod.rs | 2 + vendor/memchr/src/memmem/x86/sse.rs | 89 + vendor/memchr/src/tests/{ => memchr}/iter.rs | 5 +- .../memchr/src/tests/{ => memchr}/memchr.rs | 13 +- vendor/memchr/src/tests/memchr/mod.rs | 7 + .../src/tests/{miri.rs => memchr/simple.rs} | 8 +- vendor/memchr/src/tests/memchr/testdata.rs | 351 + vendor/memchr/src/tests/mod.rs | 355 +- vendor/memchr/src/x86/mod.rs | 119 - vendor/memmap/.cargo-checksum.json | 1 - vendor/memmap/Cargo.toml | 33 - vendor/memmap/README.md | 53 - vendor/memmap/ci/install.sh | 47 - vendor/memmap/ci/script.sh | 20 - vendor/memmap/examples/cat.rs | 23 - vendor/memmap/src/lib.rs | 1008 - vendor/memmap/src/unix.rs | 214 - vendor/memmap/src/windows.rs | 300 - vendor/memmap2/.cargo-checksum.json | 2 +- vendor/memmap2/CHANGELOG.md | 16 +- vendor/memmap2/Cargo.lock | 14 +- vendor/memmap2/Cargo.toml | 2 +- vendor/memmap2/src/lib.rs | 19 +- vendor/memmap2/src/stub.rs | 76 + vendor/memoffset/.cargo-checksum.json | 2 +- vendor/memoffset/Cargo.toml | 2 +- vendor/memoffset/README.md | 4 +- vendor/memoffset/src/lib.rs | 3 +- vendor/memoffset/src/offset_of.rs | 12 + vendor/minifier/.cargo-checksum.json | 2 +- vendor/minifier/Cargo.lock | 2 +- vendor/minifier/Cargo.toml | 2 +- vendor/minifier/src/css/tests.rs | 26 +- vendor/minifier/src/css/token.rs | 31 +- vendor/object/.cargo-checksum.json | 1 + vendor/object/CHANGELOG.md | 79 + vendor/object/Cargo.lock | 126 + vendor/object/Cargo.toml | 102 + vendor/{regex => object}/LICENSE-APACHE | 0 vendor/{memmap => object}/LICENSE-MIT | 2 +- vendor/object/README.md | 29 + vendor/object/clippy.toml | 1 + vendor/object/examples/ar.rs | 46 + vendor/object/examples/dyldcachedump.rs | 50 + vendor/object/examples/nm.rs | 95 + vendor/object/examples/objcopy.rs | 174 + vendor/object/examples/objdump.rs | 230 + vendor/object/examples/objectmap.rs | 67 + vendor/object/examples/readobj.rs | 5091 ++++ vendor/object/src/archive.rs | 39 + vendor/object/src/common.rs | 412 + vendor/object/src/elf.rs | 6073 ++++ vendor/object/src/endian.rs | 771 + vendor/object/src/lib.rs | 83 + vendor/object/src/macho.rs | 3254 ++ vendor/object/src/pe.rs | 2948 ++ vendor/object/src/pod.rs | 660 + vendor/object/src/read/any.rs | 1195 + vendor/object/src/read/archive.rs | 437 + vendor/object/src/read/coff/comdat.rs | 162 + vendor/object/src/read/coff/file.rs | 238 + vendor/object/src/read/coff/mod.rs | 18 + vendor/object/src/read/coff/relocation.rs | 85 + vendor/object/src/read/coff/section.rs | 417 + vendor/object/src/read/coff/symbol.rs | 487 + vendor/object/src/read/elf/comdat.rs | 161 + vendor/object/src/read/elf/compression.rs | 56 + vendor/object/src/read/elf/dynamic.rs | 45 + vendor/object/src/read/elf/file.rs | 854 + vendor/object/src/read/elf/mod.rs | 33 + vendor/object/src/read/elf/note.rs | 185 + vendor/object/src/read/elf/relocation.rs | 530 + vendor/object/src/read/elf/section.rs | 713 + vendor/object/src/read/elf/segment.rs | 322 + vendor/object/src/read/elf/symbol.rs | 515 + vendor/object/src/read/macho/dyld_cache.rs | 216 + vendor/object/src/read/macho/fat.rs | 120 + vendor/object/src/read/macho/file.rs | 648 + vendor/object/src/read/macho/load_command.rs | 350 + vendor/object/src/read/macho/mod.rs | 30 + vendor/object/src/read/macho/relocation.rs | 126 + vendor/object/src/read/macho/section.rs | 366 + vendor/object/src/read/macho/segment.rs | 284 + vendor/object/src/read/macho/symbol.rs | 483 + vendor/object/src/read/mod.rs | 689 + vendor/object/src/read/pe/file.rs | 1046 + vendor/object/src/read/pe/mod.rs | 16 + vendor/object/src/read/pe/section.rs | 343 + vendor/object/src/read/read_cache.rs | 179 + vendor/object/src/read/read_ref.rs | 135 + vendor/object/src/read/traits.rs | 429 + vendor/object/src/read/util.rs | 41 + vendor/object/src/read/wasm.rs | 859 + vendor/object/src/write/coff.rs | 627 + vendor/object/src/write/elf.rs | 1308 + vendor/object/src/write/macho.rs | 845 + vendor/object/src/write/mod.rs | 858 + vendor/object/src/write/string.rs | 139 + vendor/object/src/write/util.rs | 15 + vendor/object/tests/integration.rs | 1 + vendor/object/tests/parse_self.rs | 25 + vendor/object/tests/round_trip/bss.rs | 255 + vendor/object/tests/round_trip/comdat.rs | 225 + vendor/object/tests/round_trip/common.rs | 245 + vendor/object/tests/round_trip/elf.rs | 218 + vendor/object/tests/round_trip/macho.rs | 24 + vendor/object/tests/round_trip/mod.rs | 445 + vendor/object/tests/round_trip/tls.rs | 316 + vendor/proc-macro2/.cargo-checksum.json | 2 +- vendor/proc-macro2/Cargo.toml | 2 +- vendor/proc-macro2/src/fallback.rs | 20 + vendor/proc-macro2/src/lib.rs | 13 +- vendor/proc-macro2/src/parse.rs | 4 +- vendor/proc-macro2/src/wrapper.rs | 38 +- vendor/proc-macro2/tests/test.rs | 14 + vendor/rayon-core/.cargo-checksum.json | 2 +- vendor/rayon-core/Cargo.toml | 6 +- vendor/rayon-core/src/latch.rs | 42 +- vendor/rayon-core/src/lib.rs | 32 +- vendor/rayon-core/src/registry.rs | 18 +- vendor/rayon-core/src/scope/mod.rs | 255 +- vendor/rayon-core/src/scope/test.rs | 4 +- vendor/rayon-core/src/thread_pool/mod.rs | 25 + vendor/rayon-core/src/thread_pool/test.rs | 30 + vendor/rayon-core/src/util.rs | 10 - vendor/rayon/.cargo-checksum.json | 2 +- vendor/rayon/Cargo.lock | 224 +- vendor/rayon/Cargo.toml | 8 +- vendor/rayon/FAQ.md | 2 +- vendor/rayon/README.md | 4 +- vendor/rayon/RELEASES.md | 33 +- vendor/rayon/build.rs | 3 + vendor/rayon/src/array.rs | 90 + vendor/rayon/src/iter/collect/consumer.rs | 72 +- vendor/rayon/src/iter/collect/mod.rs | 78 +- vendor/rayon/src/iter/collect/test.rs | 2 +- vendor/rayon/src/iter/mod.rs | 81 +- vendor/rayon/src/iter/par_bridge.rs | 38 +- vendor/rayon/src/iter/unzip.rs | 61 + vendor/rayon/src/lib.rs | 5 +- vendor/rayon/src/range.rs | 156 +- vendor/rayon/src/range_inclusive.rs | 142 +- vendor/rayon/src/slice/mergesort.rs | 10 +- vendor/rayon/src/slice/mod.rs | 18 - vendor/rayon/src/slice/quicksort.rs | 4 +- vendor/rayon/src/slice/test.rs | 4 +- vendor/rayon/src/vec.rs | 39 +- vendor/rayon/tests/clones.rs | 6 + vendor/rayon/tests/debug.rs | 6 + vendor/rayon/tests/producer_split_at.rs | 6 + vendor/rayon/tests/sort-panic-safe.rs | 2 +- .../redox_syscall-0.1.57/.cargo-checksum.json | 1 - vendor/redox_syscall-0.1.57/Cargo.toml | 23 - vendor/redox_syscall-0.1.57/LICENSE | 22 - vendor/redox_syscall-0.1.57/README.md | 6 - .../redox_syscall-0.1.57/src/arch/aarch64.rs | 77 - vendor/redox_syscall-0.1.57/src/arch/arm.rs | 73 - .../redox_syscall-0.1.57/src/arch/nonredox.rs | 31 - vendor/redox_syscall-0.1.57/src/arch/x86.rs | 73 - .../redox_syscall-0.1.57/src/arch/x86_64.rs | 74 - vendor/redox_syscall-0.1.57/src/call.rs | 393 - vendor/redox_syscall-0.1.57/src/data.rs | 309 - vendor/redox_syscall-0.1.57/src/error.rs | 311 - vendor/redox_syscall-0.1.57/src/flag.rs | 169 - vendor/redox_syscall-0.1.57/src/io/dma.rs | 76 - vendor/redox_syscall-0.1.57/src/io/io.rs | 67 - vendor/redox_syscall-0.1.57/src/io/mmio.rs | 31 - vendor/redox_syscall-0.1.57/src/io/mod.rs | 11 - vendor/redox_syscall-0.1.57/src/io/pio.rs | 89 - vendor/redox_syscall-0.1.57/src/lib.rs | 59 - vendor/redox_syscall-0.1.57/src/number.rs | 76 - .../src/scheme/generate.sh | 20 - vendor/redox_syscall-0.1.57/src/scheme/mod.rs | 9 - .../redox_syscall-0.1.57/src/scheme/scheme.rs | 167 - .../src/scheme/scheme_block.rs | 167 - .../src/scheme/scheme_block_mut.rs | 167 - .../src/scheme/scheme_mut.rs | 167 - vendor/redox_syscall-0.1.57/src/tests.rs | 129 - vendor/redox_syscall/.cargo-checksum.json | 2 +- vendor/redox_syscall/Cargo.toml | 2 +- vendor/redox_syscall/src/arch/aarch64.rs | 11 +- vendor/redox_syscall/src/arch/riscv64.rs | 93 + vendor/redox_syscall/src/lib.rs | 4 + vendor/regex-1.4.3/.cargo-checksum.json | 1 + vendor/{regex => regex-1.4.3}/CHANGELOG.md | 41 - vendor/regex-1.4.3/Cargo.lock | 230 + vendor/{regex => regex-1.4.3}/Cargo.toml | 14 +- vendor/{regex => regex-1.4.3}/HACKING.md | 0 vendor/{memmap => regex-1.4.3}/LICENSE-APACHE | 2 +- vendor/{regex => regex-1.4.3}/LICENSE-MIT | 0 vendor/{regex => regex-1.4.3}/PERFORMANCE.md | 0 vendor/{regex => regex-1.4.3}/README.md | 6 +- vendor/{regex => regex-1.4.3}/UNICODE.md | 34 +- .../examples/regexdna-input.txt | 0 .../examples/regexdna-output.txt | 0 .../examples/shootout-regex-dna-bytes.rs | 0 .../examples/shootout-regex-dna-cheat.rs | 0 .../examples/shootout-regex-dna-replace.rs | 0 .../shootout-regex-dna-single-cheat.rs | 0 .../examples/shootout-regex-dna-single.rs | 0 .../examples/shootout-regex-dna.rs | 0 vendor/{regex => regex-1.4.3}/rustfmt.toml | 0 .../{regex => regex-1.4.3}/src/backtrack.rs | 4 +- vendor/regex-1.4.3/src/cache.rs | 100 + vendor/{regex => regex-1.4.3}/src/compile.rs | 17 +- vendor/{regex => regex-1.4.3}/src/dfa.rs | 20 +- vendor/{regex => regex-1.4.3}/src/error.rs | 0 vendor/{regex => regex-1.4.3}/src/exec.rs | 41 +- vendor/{regex => regex-1.4.3}/src/expand.rs | 3 +- .../{regex => regex-1.4.3}/src/find_byte.rs | 0 vendor/{regex => regex-1.4.3}/src/freqs.rs | 0 vendor/{regex => regex-1.4.3}/src/input.rs | 0 vendor/{regex => regex-1.4.3}/src/lib.rs | 18 +- .../{regex => regex-1.4.3}/src/literal/imp.rs | 0 .../{regex => regex-1.4.3}/src/literal/mod.rs | 0 vendor/{regex => regex-1.4.3}/src/pattern.rs | 0 vendor/{regex => regex-1.4.3}/src/pikevm.rs | 0 vendor/{regex => regex-1.4.3}/src/prog.rs | 19 +- .../{regex => regex-1.4.3}/src/re_builder.rs | 0 vendor/{regex => regex-1.4.3}/src/re_bytes.rs | 59 +- vendor/{regex => regex-1.4.3}/src/re_set.rs | 2 +- vendor/{regex => regex-1.4.3}/src/re_trait.rs | 0 .../{regex => regex-1.4.3}/src/re_unicode.rs | 59 +- vendor/{regex => regex-1.4.3}/src/sparse.rs | 2 +- .../src/testdata/LICENSE | 0 .../src/testdata/README | 0 .../src/testdata/basic.dat | 0 .../src/testdata/nullsubexpr.dat | 0 .../src/testdata/repetition.dat | 0 vendor/{regex => regex-1.4.3}/src/utf8.rs | 0 vendor/{regex => regex-1.4.3}/test | 0 vendor/{regex => regex-1.4.3}/tests/api.rs | 0 .../{regex => regex-1.4.3}/tests/api_str.rs | 0 vendor/{regex => regex-1.4.3}/tests/bytes.rs | 0 .../tests/consistent.rs | 5 +- .../tests/crates_regex.rs | 0 vendor/{regex => regex-1.4.3}/tests/crazy.rs | 5 +- vendor/{regex => regex-1.4.3}/tests/flags.rs | 0 vendor/{regex => regex-1.4.3}/tests/fowler.rs | 0 vendor/{regex => regex-1.4.3}/tests/macros.rs | 0 .../tests/macros_bytes.rs | 1 + .../tests/macros_str.rs | 1 - vendor/{regex => regex-1.4.3}/tests/misc.rs | 0 .../{regex => regex-1.4.3}/tests/multiline.rs | 0 .../{regex => regex-1.4.3}/tests/noparse.rs | 0 .../tests/regression.rs | 0 .../tests/regression_fuzz.rs | 12 - .../{regex => regex-1.4.3}/tests/replace.rs | 98 - .../{regex => regex-1.4.3}/tests/searcher.rs | 0 vendor/{regex => regex-1.4.3}/tests/set.rs | 0 .../tests/shortest_match.rs | 0 .../tests/suffix_reverse.rs | 0 .../tests/test_backtrack.rs | 0 .../tests/test_backtrack_bytes.rs | 0 .../tests/test_backtrack_utf8bytes.rs | 0 .../tests/test_crates_regex.rs | 0 .../tests/test_default.rs | 42 +- .../tests/test_default_bytes.rs | 0 .../{regex => regex-1.4.3}/tests/test_nfa.rs | 0 .../tests/test_nfa_bytes.rs | 0 .../tests/test_nfa_utf8bytes.rs | 0 .../{regex => regex-1.4.3}/tests/unicode.rs | 0 .../tests/word_boundary.rs | 0 .../tests/word_boundary_ascii.rs | 0 .../tests/word_boundary_unicode.rs | 0 vendor/regex-syntax/.cargo-checksum.json | 2 +- vendor/regex-syntax/Cargo.toml | 3 +- vendor/regex-syntax/benches/bench.rs | 1 - vendor/regex-syntax/src/ast/mod.rs | 16 +- vendor/regex-syntax/src/ast/parse.rs | 27 +- vendor/regex-syntax/src/ast/print.rs | 32 +- vendor/regex-syntax/src/ast/visitor.rs | 6 +- vendor/regex-syntax/src/error.rs | 12 +- vendor/regex-syntax/src/hir/interval.rs | 6 +- vendor/regex-syntax/src/hir/literal/mod.rs | 12 +- vendor/regex-syntax/src/hir/mod.rs | 28 +- vendor/regex-syntax/src/hir/print.rs | 8 +- vendor/regex-syntax/src/hir/translate.rs | 30 +- vendor/regex-syntax/src/hir/visitor.rs | 2 +- vendor/regex-syntax/src/lib.rs | 6 +- vendor/regex-syntax/src/parser.rs | 6 +- vendor/regex-syntax/src/unicode.rs | 46 +- vendor/regex-syntax/src/utf8.rs | 12 +- vendor/regex-syntax/test | 2 + vendor/regex/.cargo-checksum.json | 1 - vendor/regex/Cargo.lock | 98 - vendor/regex/src/pool.rs | 333 - vendor/rustc-demangle/.cargo-checksum.json | 2 +- vendor/rustc-demangle/Cargo.toml | 2 +- vendor/rustc-demangle/src/lib.rs | 19 + vendor/rustc-demangle/src/v0.rs | 83 +- .../.cargo-checksum.json | 0 vendor/{semver => semver-0.11.0}/Cargo.toml | 0 .../{semver => semver-0.11.0}/LICENSE-APACHE | 0 vendor/{semver => semver-0.11.0}/LICENSE-MIT | 0 vendor/{semver => semver-0.11.0}/README.md | 0 .../src/diesel_impls.rs | 0 vendor/{semver => semver-0.11.0}/src/lib.rs | 0 .../{semver => semver-0.11.0}/src/version.rs | 0 .../src/version_req.rs | 0 .../tests/deprecation.rs | 0 .../{semver => semver-0.11.0}/tests/diesel.rs | 0 .../{semver => semver-0.11.0}/tests/serde.rs | 0 vendor/serde/.cargo-checksum.json | 2 +- vendor/serde/Cargo.toml | 4 +- vendor/serde/src/lib.rs | 2 +- vendor/serde_derive/.cargo-checksum.json | 2 +- vendor/serde_derive/Cargo.toml | 2 +- vendor/serde_derive/src/bound.rs | 4 +- vendor/serde_derive/src/dummy.rs | 2 +- vendor/serde_derive/src/lib.rs | 7 +- vendor/snap/.cargo-checksum.json | 2 +- vendor/snap/Cargo.lock | 6 +- vendor/snap/Cargo.toml | 2 +- vendor/snap/src/read.rs | 22 +- vendor/syn/.cargo-checksum.json | 2 +- vendor/syn/Cargo.toml | 2 +- vendor/syn/src/generics.rs | 15 +- vendor/syn/src/item.rs | 29 +- vendor/syn/src/lib.rs | 4 +- vendor/syn/src/path.rs | 12 + vendor/syn/src/ty.rs | 11 +- vendor/syn/tests/repo/mod.rs | 6 +- vendor/syn/tests/test_item.rs | 33 + vendor/tar/.cargo-checksum.json | 2 +- vendor/tar/Cargo.lock | 135 +- vendor/tar/Cargo.toml | 4 +- vendor/tar/src/archive.rs | 49 +- vendor/tar/src/builder.rs | 23 +- vendor/tar/src/entry.rs | 53 +- vendor/tar/src/header.rs | 18 +- vendor/tar/src/pax.rs | 23 + vendor/tar/tests/all.rs | 166 +- vendor/thiserror-impl/.cargo-checksum.json | 2 +- vendor/thiserror-impl/Cargo.toml | 4 +- vendor/thiserror-impl/src/expand.rs | 14 +- vendor/thiserror-impl/src/fmt.rs | 19 +- vendor/thiserror-impl/src/lib.rs | 11 +- vendor/thiserror-impl/src/valid.rs | 45 +- vendor/thiserror/.cargo-checksum.json | 2 +- vendor/thiserror/Cargo.toml | 4 +- vendor/thiserror/README.md | 2 +- vendor/thiserror/src/lib.rs | 5 +- vendor/thiserror/tests/test_backtrace.rs | 213 - vendor/thiserror/tests/test_display.rs | 42 - vendor/thiserror/tests/test_expr.rs | 1 - vendor/thiserror/tests/test_source.rs | 13 - vendor/thiserror/tests/ui/concat-display.rs | 15 - .../thiserror/tests/ui/concat-display.stderr | 10 - vendor/thiserror/tests/ui/lifetime.rs | 11 - vendor/thiserror/tests/ui/lifetime.stderr | 14 +- vendor/thiserror/tests/ui/missing-display.rs | 9 + .../thiserror/tests/ui/missing-display.stderr | 8 + vendor/thiserror/tests/ui/no-display.rs | 6 +- vendor/thiserror/tests/ui/no-display.stderr | 21 +- .../tests/ui/source-enum-not-error.rs | 5 +- .../tests/ui/source-enum-not-error.stderr | 32 +- .../tests/ui/source-struct-not-error.rs | 5 +- .../tests/ui/source-struct-not-error.stderr | 29 +- vendor/tracing-core/.cargo-checksum.json | 2 +- vendor/tracing-core/CHANGELOG.md | 19 + vendor/tracing-core/Cargo.toml | 2 +- vendor/tracing-core/README.md | 30 +- vendor/tracing-core/src/callsite.rs | 5 +- vendor/tracing-core/src/dispatcher.rs | 15 +- vendor/tracing-core/src/event.rs | 10 +- vendor/tracing-core/src/field.rs | 2 +- vendor/tracing-core/src/lib.rs | 8 +- vendor/tracing-core/src/metadata.rs | 18 +- vendor/tracing-core/src/span.rs | 64 +- vendor/tracing-core/src/subscriber.rs | 175 +- .../tracing-subscriber/.cargo-checksum.json | 2 +- vendor/tracing-subscriber/CHANGELOG.md | 32 +- vendor/tracing-subscriber/Cargo.toml | 2 +- vendor/tracing-subscriber/README.md | 2 +- .../src/filter/env/directive.rs | 32 +- .../tracing-subscriber/src/fmt/fmt_layer.rs | 2 +- .../tracing-subscriber/src/fmt/format/json.rs | 244 +- .../tracing-subscriber/src/fmt/format/mod.rs | 85 +- .../src/fmt/format/pretty.rs | 3 +- vendor/tracing-subscriber/src/fmt/mod.rs | 11 +- vendor/tracing-subscriber/src/fmt/time/mod.rs | 30 - vendor/tracing-subscriber/src/lib.rs | 5 +- .../tracing-subscriber/tests/field_filter.rs | 2 + vendor/tracing/.cargo-checksum.json | 2 +- vendor/tracing/CHANGELOG.md | 32 + vendor/tracing/Cargo.toml | 6 +- vendor/tracing/README.md | 10 +- vendor/tracing/src/lib.rs | 23 +- vendor/tracing/src/macros.rs | 8 - vendor/tracing/src/span.rs | 36 +- vendor/tracing/tests/subscriber.rs | 92 +- vendor/tracing/tests/support/field.rs | 6 +- vendor/tracing/tests/support/span.rs | 8 +- .../.cargo-checksum.json | 2 +- vendor/unicode-normalization/Cargo.toml | 2 +- vendor/unicode-normalization/README.md | 2 +- .../unicode-normalization/scripts/unicode.py | 40 + vendor/unicode-normalization/src/lib.rs | 7 +- .../src/no_std_prelude.rs | 0 vendor/unicode-normalization/src/tables.rs | 695 + vendor/unicode-xid/.cargo-checksum.json | 2 +- vendor/unicode-xid/Cargo.toml | 10 +- vendor/unicode-xid/README.md | 9 + vendor/unicode-xid/benches/xid.rs | 60 + vendor/unicode-xid/src/lib.rs | 12 +- vendor/url/.cargo-checksum.json | 2 +- vendor/url/Cargo.toml | 2 +- vendor/url/src/lib.rs | 166 +- vendor/url/src/quirks.rs | 12 +- vendor/url/tests/unit.rs | 439 +- version | 2 +- 3915 files changed, 178795 insertions(+), 76468 deletions(-) create mode 100644 compiler/rustc_ast_lowering/src/asm.rs delete mode 100644 compiler/rustc_builtin_macros/src/global_asm.rs rename compiler/rustc_codegen_cranelift/crate_patches/{0002-compiler-builtins-Disable-128bit-atomic-operations.patch => 0001-compiler-builtins-Disable-128bit-atomic-operations.patch} (100%) delete mode 100644 compiler/rustc_codegen_cranelift/crate_patches/0001-compiler-builtins-Remove-rotate_left-from-Int.patch delete mode 100644 compiler/rustc_codegen_cranelift/docs/env_vars.md create mode 100644 compiler/rustc_codegen_cranelift/src/config.rs delete mode 100644 compiler/rustc_codegen_cranelift/src/optimize/code_layout.rs delete mode 100644 compiler/rustc_codegen_cranelift/src/optimize/stack2reg.rs delete mode 100644 compiler/rustc_codegen_llvm/src/metadata.rs create mode 100644 compiler/rustc_codegen_ssa/src/back/metadata.rs delete mode 100644 compiler/rustc_data_structures/src/box_region.rs create mode 100644 compiler/rustc_data_structures/src/vec_map.rs create mode 100644 compiler/rustc_data_structures/src/vec_map/tests.rs create mode 100644 compiler/rustc_error_codes/src/error_codes/E0316.md create mode 100644 compiler/rustc_error_codes/src/error_codes/E0782.md create mode 100644 compiler/rustc_error_codes/src/error_codes/E0783.md create mode 100644 compiler/rustc_middle/src/thir.rs create mode 100644 compiler/rustc_mir_build/src/check_unsafety.rs delete mode 100644 compiler/rustc_mir_build/src/thir/arena.rs create mode 100644 compiler/rustc_mir_build/src/thir/visit.rs delete mode 100644 compiler/rustc_query_system/src/dep_graph/prev.rs create mode 100644 compiler/rustc_target/src/abi/call/bpf.rs create mode 100644 compiler/rustc_target/src/asm/bpf.rs create mode 100644 compiler/rustc_target/src/asm/powerpc.rs create mode 100644 compiler/rustc_target/src/spec/bpf_base.rs create mode 100644 compiler/rustc_target/src/spec/bpfeb_unknown_none.rs create mode 100644 compiler/rustc_target/src/spec/bpfel_unknown_none.rs rename library/std/src/{sys/sgx/ext => os/fortanix_sgx}/arch.rs (88%) rename library/std/src/{sys/sgx/ext => os/fortanix_sgx}/ffi.rs (100%) rename library/std/src/{sys/sgx/ext => os/fortanix_sgx}/io.rs (100%) rename library/std/src/{sys/hermit/ext => os/hermit}/ffi.rs (100%) rename library/std/src/{sys/hermit/ext => os/hermit}/mod.rs (94%) rename library/std/src/{sys/unix/ext => os/unix}/ffi.rs (100%) rename library/std/src/{sys/unix/ext => os/unix}/fs.rs (99%) rename library/std/src/{sys/unix/ext => os/unix}/io.rs (100%) rename library/std/src/{sys/unix/ext => os/unix}/mod.rs (61%) rename library/std/src/{sys/unix/ext => os/unix}/net/addr.rs (100%) rename library/std/src/{sys/unix/ext => os/unix}/net/ancillary.rs (99%) rename library/std/src/{sys/unix/ext => os/unix}/net/datagram.rs (100%) rename library/std/src/{sys/unix/ext => os/unix}/net/listener.rs (100%) rename library/std/src/{sys/unix/ext => os/unix}/net/mod.rs (100%) rename library/std/src/{sys/unix/ext => os/unix}/net/raw_fd.rs (100%) rename library/std/src/{sys/unix/ext => os/unix}/net/stream.rs (100%) rename library/std/src/{sys/unix/ext => os/unix}/net/tests.rs (100%) rename library/std/src/{sys/unix/ext => os/unix}/process.rs (86%) rename library/std/src/{sys/unix/ext => os/unix}/raw.rs (100%) rename library/std/src/{sys/unix/ext => os/unix}/thread.rs (100%) rename library/std/src/{sys/unix/ext => os/unix}/ucred.rs (100%) rename library/std/src/{sys/unix/ext => os/unix}/ucred/tests.rs (100%) delete mode 100644 library/std/src/os/wasi.rs rename library/std/src/{sys/wasi/ext => os/wasi}/ffi.rs (100%) rename library/std/src/{sys/wasi/ext => os/wasi}/fs.rs (100%) rename library/std/src/{sys/wasi/ext => os/wasi}/io.rs (100%) rename library/std/src/{sys/wasi/ext => os/wasi}/mod.rs (92%) rename library/std/src/{sys/windows/ext => os/windows}/ffi.rs (100%) rename library/std/src/{sys/windows/ext => os/windows}/fs.rs (100%) rename library/std/src/{sys/windows/ext => os/windows}/io.rs (100%) rename library/std/src/{sys/windows/ext => os/windows}/mod.rs (98%) rename library/std/src/{sys/windows/ext => os/windows}/process.rs (100%) rename library/std/src/{sys/windows/ext => os/windows}/raw.rs (100%) rename library/std/src/{sys/windows/ext => os/windows}/thread.rs (100%) rename library/std/src/sys/sgx/abi/{tls.rs => tls/mod.rs} (100%) delete mode 100644 library/std/src/sys/sgx/ext/mod.rs rename library/std/src/sys/sgx/{waitqueue.rs => waitqueue/mod.rs} (97%) delete mode 100644 library/std/src/sys/wasm/args.rs rename library/std/src/sys/wasm/{condvar_atomics.rs => atomics/condvar.rs} (100%) rename library/std/src/sys/wasm/{futex_atomics.rs => atomics/futex.rs} (100%) rename library/std/src/sys/wasm/{mutex_atomics.rs => atomics/mutex.rs} (100%) rename library/std/src/sys/wasm/{rwlock_atomics.rs => atomics/rwlock.rs} (99%) rename library/std/src/sys/wasm/{ => atomics}/thread.rs (83%) rename library/std/src/{ => sys_common}/memchr.rs (92%) rename library/std/src/{ => sys_common}/memchr/tests.rs (100%) delete mode 100644 library/std/src/sys_common/util.rs create mode 100644 library/stdarch/crates/core_arch/src/aarch64/armclang.rs rename library/stdarch/crates/core_arch/src/{acle => arm}/dsp.rs (100%) rename library/stdarch/crates/core_arch/src/{acle => arm}/ex.rs (95%) create mode 100644 library/stdarch/crates/core_arch/src/arm/neon.rs delete mode 100644 library/stdarch/crates/core_arch/src/arm/neon/generated.rs rename library/stdarch/crates/core_arch/src/{acle => arm}/sat.rs (100%) rename library/stdarch/crates/core_arch/src/{acle => arm}/simd32.rs (99%) rename library/stdarch/crates/core_arch/src/{acle => arm_shared}/barrier/common.rs (100%) rename library/stdarch/crates/core_arch/src/{acle => arm_shared}/barrier/cp15.rs (61%) rename library/stdarch/crates/core_arch/src/{acle => arm_shared}/barrier/mod.rs (100%) rename library/stdarch/crates/core_arch/src/{acle => arm_shared}/barrier/not_mclass.rs (100%) rename library/stdarch/crates/core_arch/src/{acle => arm_shared}/barrier/v8.rs (100%) rename library/stdarch/crates/core_arch/src/{arm => arm_shared}/crc.rs (98%) rename library/stdarch/crates/core_arch/src/{arm => arm_shared}/crypto.rs (70%) rename library/stdarch/crates/core_arch/src/{acle => arm_shared}/hints.rs (63%) rename library/stdarch/crates/core_arch/src/{acle => arm_shared}/mod.rs (65%) create mode 100644 library/stdarch/crates/core_arch/src/arm_shared/neon/generated.rs rename library/stdarch/crates/core_arch/src/{arm => arm_shared}/neon/load_tests.rs (100%) rename library/stdarch/crates/core_arch/src/{arm => arm_shared}/neon/mod.rs (62%) rename library/stdarch/crates/core_arch/src/{arm => arm_shared}/neon/shift_and_insert_tests.rs (98%) create mode 100644 library/stdarch/crates/core_arch/src/arm_shared/neon/store_tests.rs rename library/stdarch/crates/core_arch/src/{arm => arm_shared}/neon/table_lookup_tests.rs (100%) rename library/stdarch/crates/core_arch/src/{acle => arm_shared}/registers/aarch32.rs (100%) rename library/stdarch/crates/core_arch/src/{acle => arm_shared}/registers/mod.rs (85%) rename library/stdarch/crates/core_arch/src/{acle => arm_shared}/registers/v6m.rs (100%) rename library/stdarch/crates/core_arch/src/{acle => arm_shared}/registers/v7m.rs (100%) rename library/stdarch/crates/core_arch/src/{arm => arm_shared}/test_support.rs (98%) create mode 100644 library/test/src/formatters/junit.rs create mode 100644 src/ci/channel create mode 100755 src/ci/scripts/verify-channel.sh delete mode 100644 src/doc/edition-guide/src/rust-2018/cargo-and-crates-io/cargo-can-use-a-local-registry-replacement.md delete mode 100644 src/doc/edition-guide/src/rust-2018/cargo-and-crates-io/cargo-check-for-faster-checking.md delete mode 100644 src/doc/edition-guide/src/rust-2018/cargo-and-crates-io/cargo-install-for-easy-installation-of-tools.md delete mode 100644 src/doc/edition-guide/src/rust-2018/cargo-and-crates-io/cargo-new-defaults-to-a-binary-project.md delete mode 100644 src/doc/edition-guide/src/rust-2018/cargo-and-crates-io/cargo-rustc-for-passing-arbitrary-flags-to-rustc.md delete mode 100644 src/doc/edition-guide/src/rust-2018/cargo-and-crates-io/cargo-workspaces-for-multi-package-projects.md delete mode 100644 src/doc/edition-guide/src/rust-2018/cargo-and-crates-io/crates-io-disallows-wildcard-dependencies.md delete mode 100644 src/doc/edition-guide/src/rust-2018/cargo-and-crates-io/index.md delete mode 100644 src/doc/edition-guide/src/rust-2018/cargo-and-crates-io/multi-file-examples.md delete mode 100644 src/doc/edition-guide/src/rust-2018/cargo-and-crates-io/replacing-dependencies-with-patch.md create mode 100644 src/doc/edition-guide/src/rust-2018/cargo.md delete mode 100644 src/doc/edition-guide/src/rust-2018/control-flow/async-await-for-easier-concurrency.md delete mode 100644 src/doc/edition-guide/src/rust-2018/control-flow/index.md delete mode 100644 src/doc/edition-guide/src/rust-2018/control-flow/loops-can-break-with-a-value.md delete mode 100644 src/doc/edition-guide/src/rust-2018/data-types/128-bit-integers.md delete mode 100644 src/doc/edition-guide/src/rust-2018/data-types/choosing-alignment-with-the-repr-attribute.md delete mode 100644 src/doc/edition-guide/src/rust-2018/data-types/field-init-shorthand.md delete mode 100644 src/doc/edition-guide/src/rust-2018/data-types/inclusive-ranges.md delete mode 100644 src/doc/edition-guide/src/rust-2018/data-types/index.md delete mode 100644 src/doc/edition-guide/src/rust-2018/data-types/operator-equals-are-now-implementable.md delete mode 100644 src/doc/edition-guide/src/rust-2018/data-types/union-for-an-unsafe-form-of-enum.md delete mode 100644 src/doc/edition-guide/src/rust-2018/documentation/index.md delete mode 100644 src/doc/edition-guide/src/rust-2018/documentation/new-editions-of-the-book.md delete mode 100644 src/doc/edition-guide/src/rust-2018/documentation/std-os-has-documentation-for-all-platforms.md delete mode 100644 src/doc/edition-guide/src/rust-2018/documentation/the-rust-bookshelf.md delete mode 100644 src/doc/edition-guide/src/rust-2018/documentation/the-rustonomicon.md delete mode 100644 src/doc/edition-guide/src/rust-2018/edition-changes.md delete mode 100644 src/doc/edition-guide/src/rust-2018/error-handling-and-panics/aborting-on-panic.md delete mode 100644 src/doc/edition-guide/src/rust-2018/error-handling-and-panics/controlling-panics-with-std-panic.md delete mode 100644 src/doc/edition-guide/src/rust-2018/error-handling-and-panics/index.md delete mode 100644 src/doc/edition-guide/src/rust-2018/error-handling-and-panics/question-mark-in-main-and-tests.md delete mode 100644 src/doc/edition-guide/src/rust-2018/error-handling-and-panics/the-question-mark-operator-for-easier-error-handling.md delete mode 100644 src/doc/edition-guide/src/rust-2018/macros/at-most-once.md delete mode 100644 src/doc/edition-guide/src/rust-2018/macros/custom-derive.md delete mode 100644 src/doc/edition-guide/src/rust-2018/macros/index.md delete mode 100644 src/doc/edition-guide/src/rust-2018/macros/macro-changes.md delete mode 100644 src/doc/edition-guide/src/rust-2018/module-system/index.md delete mode 100644 src/doc/edition-guide/src/rust-2018/module-system/more-visibility-modifiers.md delete mode 100644 src/doc/edition-guide/src/rust-2018/module-system/nested-imports-with-use.md delete mode 100644 src/doc/edition-guide/src/rust-2018/module-system/raw-identifiers.md rename src/doc/edition-guide/src/rust-2018/{trait-system/dyn-trait-for-trait-objects.md => new-keywords.md} (50%) delete mode 100644 src/doc/edition-guide/src/rust-2018/ownership-and-lifetimes/default-match-bindings.md delete mode 100644 src/doc/edition-guide/src/rust-2018/ownership-and-lifetimes/index.md delete mode 100644 src/doc/edition-guide/src/rust-2018/ownership-and-lifetimes/inference-in-structs.md delete mode 100644 src/doc/edition-guide/src/rust-2018/ownership-and-lifetimes/lifetime-elision-in-impl.md delete mode 100644 src/doc/edition-guide/src/rust-2018/ownership-and-lifetimes/non-lexical-lifetimes.md delete mode 100644 src/doc/edition-guide/src/rust-2018/ownership-and-lifetimes/simpler-lifetimes-in-static-and-const.md delete mode 100644 src/doc/edition-guide/src/rust-2018/ownership-and-lifetimes/the-anonymous-lifetime.md rename src/doc/edition-guide/src/rust-2018/{module-system/path-clarity.md => path-changes.md} (91%) delete mode 100644 src/doc/edition-guide/src/rust-2018/platform-and-target-support/cdylib-crates-for-c-interoperability.md delete mode 100644 src/doc/edition-guide/src/rust-2018/platform-and-target-support/global-allocators.md delete mode 100644 src/doc/edition-guide/src/rust-2018/platform-and-target-support/index.md delete mode 100644 src/doc/edition-guide/src/rust-2018/platform-and-target-support/libcore-for-low-level-rust.md delete mode 100644 src/doc/edition-guide/src/rust-2018/platform-and-target-support/msvc-toolchain-support.md delete mode 100644 src/doc/edition-guide/src/rust-2018/platform-and-target-support/musl-support-for-fully-static-binaries.md delete mode 100644 src/doc/edition-guide/src/rust-2018/platform-and-target-support/webassembly-support.md delete mode 100644 src/doc/edition-guide/src/rust-2018/rustdoc/documentation-tests-can-now-compile-fail.md delete mode 100644 src/doc/edition-guide/src/rust-2018/rustdoc/index.md delete mode 100644 src/doc/edition-guide/src/rust-2018/rustdoc/rustdoc-uses-commonmark.md delete mode 100644 src/doc/edition-guide/src/rust-2018/rustup-for-managing-rust-versions.md delete mode 100644 src/doc/edition-guide/src/rust-2018/simd-for-faster-computing.md delete mode 100644 src/doc/edition-guide/src/rust-2018/slice-patterns.md delete mode 100644 src/doc/edition-guide/src/rust-2018/the-compiler/an-attribute-for-deprecation.md delete mode 100644 src/doc/edition-guide/src/rust-2018/the-compiler/improved-error-messages.md delete mode 100644 src/doc/edition-guide/src/rust-2018/the-compiler/incremental-compilation-for-faster-compiles.md delete mode 100644 src/doc/edition-guide/src/rust-2018/the-compiler/index.md rename src/doc/edition-guide/src/rust-2018/{trait-system/no-anon-params.md => trait-fn-parameters.md} (65%) delete mode 100644 src/doc/edition-guide/src/rust-2018/trait-system/associated-constants.md delete mode 100644 src/doc/edition-guide/src/rust-2018/trait-system/impl-trait-for-returning-complex-types-with-ease.md delete mode 100644 src/doc/edition-guide/src/rust-2018/trait-system/index.md delete mode 100644 src/doc/edition-guide/src/rust-2018/trait-system/more-container-types-support-trait-objects.md create mode 100644 src/doc/edition-guide/src/rust-2018/tyvar-behind-raw-pointer.md create mode 100644 src/doc/edition-guide/src/rust-2021/IntoIterator-for-arrays.md create mode 100644 src/doc/edition-guide/src/rust-2021/default-cargo-resolver.md create mode 100644 src/doc/edition-guide/src/rust-2021/disjoint-capture-in-closures.md create mode 100644 src/doc/edition-guide/src/rust-2021/index.md create mode 100644 src/doc/edition-guide/src/rust-2021/or-patterns-macro-rules.md create mode 100644 src/doc/edition-guide/src/rust-2021/panic-macro-consistency.md create mode 100644 src/doc/edition-guide/src/rust-2021/prelude.md create mode 100644 src/doc/edition-guide/src/rust-2021/reserving-syntax.md create mode 100644 src/doc/edition-guide/src/rust-2021/warnings-promoted-to-error.md delete mode 100644 src/doc/edition-guide/src/rust-next/alloc.md delete mode 100644 src/doc/edition-guide/src/rust-next/alternative-cargo-registries.md delete mode 100644 src/doc/edition-guide/src/rust-next/cargo-vendor.md delete mode 100644 src/doc/edition-guide/src/rust-next/const-fn.md delete mode 100644 src/doc/edition-guide/src/rust-next/dbg-macro.md delete mode 100644 src/doc/edition-guide/src/rust-next/edition-changes.md delete mode 100644 src/doc/edition-guide/src/rust-next/future.md delete mode 100644 src/doc/edition-guide/src/rust-next/index.md delete mode 100644 src/doc/edition-guide/src/rust-next/literal-macro-matcher.md delete mode 100644 src/doc/edition-guide/src/rust-next/maybe-uninit.md delete mode 100644 src/doc/edition-guide/src/rust-next/no-jemalloc.md delete mode 100644 src/doc/edition-guide/src/rust-next/no-more-fnbox.md delete mode 100644 src/doc/edition-guide/src/rust-next/pin.md delete mode 100644 src/doc/edition-guide/src/rust-next/qustion-mark-operator-in-macros.md delete mode 100644 src/doc/edition-guide/src/rust-next/tryfrom-and-tryinto.md delete mode 100644 src/doc/edition-guide/src/rust-next/uniform-paths.md create mode 100644 src/doc/rustc/src/platform-support/aarch64-apple-ios-sim.md create mode 100644 src/doc/rustdoc/README.md create mode 100644 src/doc/unstable-book/src/compiler-flags/force-warns.md create mode 100644 src/doc/unstable-book/src/compiler-flags/instrument-coverage.md delete mode 100644 src/doc/unstable-book/src/language-features/const-fn.md delete mode 100644 src/doc/unstable-book/src/language-features/external-doc.md delete mode 100644 src/doc/unstable-book/src/language-features/member-constraints.md create mode 100644 src/doc/unstable-book/src/language-features/more-qualified-paths.md create mode 100644 src/doc/unstable-book/src/language-features/native-link-modifiers-as-needed.md create mode 100644 src/doc/unstable-book/src/language-features/native-link-modifiers-bundle.md create mode 100644 src/doc/unstable-book/src/language-features/native-link-modifiers-verbatim.md create mode 100644 src/doc/unstable-book/src/language-features/native-link-modifiers-whole-archive.md create mode 100644 src/doc/unstable-book/src/language-features/native-link-modifiers.md create mode 100644 src/doc/unstable-book/src/language-features/no-coverage.md create mode 100644 src/librustdoc/clean/utils/tests.rs create mode 100644 src/librustdoc/html/static/clipboard.svg create mode 100644 src/librustdoc/html/static/noto-sans-kr-v13-korean-regular-LICENSE.txt create mode 100644 src/librustdoc/html/static/noto-sans-kr-v13-korean-regular.woff delete mode 100644 src/librustdoc/html/static/sidebar-items.js delete mode 100644 src/librustdoc/passes/collect_intra_doc_links/early.rs create mode 100644 src/test/assembly/asm/bpf-types.rs create mode 100644 src/test/assembly/asm/global_asm.rs create mode 100644 src/test/assembly/asm/powerpc-types.rs create mode 100644 src/test/codegen/auxiliary/thread_local_aux.rs create mode 100644 src/test/codegen/bpf-alu32.rs create mode 100644 src/test/codegen/remap_path_prefix/issue-73167-remap-std.rs create mode 100644 src/test/codegen/thread-local.rs create mode 100644 src/test/debuginfo/msvc-pretty-enums.rs create mode 100644 src/test/incremental/link_order/auxiliary/my_lib.rs create mode 100644 src/test/incremental/link_order/main.rs create mode 100644 src/test/mir-opt/bool_compare.opt1.InstCombine.diff create mode 100644 src/test/mir-opt/bool_compare.opt2.InstCombine.diff create mode 100644 src/test/mir-opt/bool_compare.opt3.InstCombine.diff create mode 100644 src/test/mir-opt/bool_compare.opt4.InstCombine.diff create mode 100644 src/test/mir-opt/bool_compare.rs create mode 100644 src/test/mir-opt/retag.array_casts.SimplifyCfg-elaborate-drops.after.mir create mode 100644 src/test/pretty/anonymous-types.rs delete mode 100644 src/test/run-make-fulldeps/alloc-extern-crates/Makefile delete mode 100644 src/test/run-make-fulldeps/alloc-extern-crates/fakealloc.rs create mode 100644 src/test/run-make-fulldeps/alloc-no-oom-handling/Makefile create mode 100644 src/test/run-make-fulldeps/coverage-reports/expected_show_coverage.closure_macro.txt create mode 100644 src/test/run-make-fulldeps/coverage-reports/expected_show_coverage.closure_macro_async.txt create mode 100644 src/test/run-make-fulldeps/coverage-reports/expected_show_coverage.generator.txt create mode 100644 src/test/run-make-fulldeps/coverage-reports/expected_show_coverage.issue-84561.txt create mode 100644 src/test/run-make-fulldeps/coverage/closure_macro.rs create mode 100644 src/test/run-make-fulldeps/coverage/closure_macro_async.rs create mode 100644 src/test/run-make-fulldeps/coverage/generator.rs create mode 100644 src/test/run-make-fulldeps/coverage/issue-84561.rs create mode 100644 src/test/run-make-fulldeps/link-dedup/Makefile create mode 100644 src/test/run-make-fulldeps/link-dedup/depa.rs create mode 100644 src/test/run-make-fulldeps/link-dedup/depb.rs create mode 100644 src/test/run-make-fulldeps/link-dedup/depc.rs create mode 100644 src/test/run-make-fulldeps/link-dedup/empty.rs delete mode 100644 src/test/run-make-fulldeps/print-unversioned-files/Makefile delete mode 100644 src/test/run-make-fulldeps/print-unversioned-files/unversioned-files.txt create mode 100644 src/test/run-make-fulldeps/rustdoc-target-spec-json-path/Makefile create mode 100644 src/test/run-make-fulldeps/rustdoc-target-spec-json-path/dummy_core.rs create mode 100644 src/test/run-make-fulldeps/rustdoc-target-spec-json-path/my_crate.rs create mode 100644 src/test/run-make-fulldeps/rustdoc-target-spec-json-path/target.json create mode 100644 src/test/run-make/emit-named-files/Makefile create mode 100644 src/test/run-make/emit-named-files/foo.rs create mode 100644 src/test/run-make/incremental-session-fail/Makefile create mode 100644 src/test/run-make/incremental-session-fail/foo.rs create mode 100644 src/test/run-make/issue-71519/Makefile create mode 100644 src/test/run-make/issue-71519/main.rs create mode 100644 src/test/run-make/raw-dylib/Makefile create mode 100644 src/test/run-make/raw-dylib/driver.rs create mode 100644 src/test/run-make/raw-dylib/extern_1.c create mode 100644 src/test/run-make/raw-dylib/extern_2.c create mode 100644 src/test/run-make/raw-dylib/lib.rs create mode 100644 src/test/run-make/raw-dylib/output.txt create mode 100644 src/test/run-make/unstable-flag-required/force-warns.stderr delete mode 100644 src/test/rustdoc-gui/basic-code.goml delete mode 100644 src/test/rustdoc-gui/basic.goml delete mode 100644 src/test/rustdoc-gui/check_info_sign_position.goml delete mode 100644 src/test/rustdoc-gui/code-sidebar-toggle.goml delete mode 100644 src/test/rustdoc-gui/hash-item-expansion.goml delete mode 100644 src/test/rustdoc-gui/impl-default-expansion.goml delete mode 100644 src/test/rustdoc-gui/list_code_block.goml delete mode 100644 src/test/rustdoc-gui/search-input-mobile.goml delete mode 100644 src/test/rustdoc-gui/search-tab-selection-if-current-is-empty.goml delete mode 100644 src/test/rustdoc-gui/shortcuts.goml rename src/test/rustdoc-gui/{ => src}/lib.rs (90%) create mode 100644 src/test/rustdoc-gui/src/lib2.rs rename vendor/gimli/rustfmt.toml => src/test/rustdoc-gui/src/settings/.package-cache (100%) delete mode 100644 src/test/rustdoc-gui/theme-change.goml delete mode 100644 src/test/rustdoc-gui/toggle-docs.goml delete mode 100644 src/test/rustdoc-gui/trait-sidebar-item-order.goml create mode 100644 src/test/rustdoc-gui/type-weight.rs delete mode 100644 src/test/rustdoc-ui/auxiliary/panic-item.rs delete mode 100644 src/test/rustdoc-ui/doc-attr2.rs delete mode 100644 src/test/rustdoc-ui/doc-attr2.stderr create mode 100644 src/test/rustdoc-ui/doc-include-suggestion.rs create mode 100644 src/test/rustdoc-ui/doc-include-suggestion.stderr create mode 100644 src/test/rustdoc-ui/invalid-cfg.rs create mode 100644 src/test/rustdoc-ui/invalid-cfg.stderr create mode 100644 src/test/rustdoc-ui/invalid-doc-attr.rs create mode 100644 src/test/rustdoc-ui/invalid-doc-attr.stderr create mode 100644 src/test/rustdoc-ui/no-run-flag-error.rs create mode 100644 src/test/rustdoc-ui/no-run-flag-error.stderr create mode 100644 src/test/rustdoc-ui/no-run-flag.rs create mode 100644 src/test/rustdoc-ui/no-run-flag.stdout create mode 100644 src/test/rustdoc-ui/test-type.rs create mode 100644 src/test/rustdoc-ui/test-type.stdout delete mode 100644 src/test/rustdoc-ui/unused-extern-crate.rs create mode 100644 src/test/rustdoc-ui/wasm-safe.rs create mode 100644 src/test/rustdoc/auxiliary/issue-66159-1.rs create mode 100644 src/test/rustdoc/auxiliary/trait-alias-mention.rs create mode 100644 src/test/rustdoc/auxiliary/trait-visibility.rs create mode 100644 src/test/rustdoc/const-generics/const-generic-defaults.rs create mode 100644 src/test/rustdoc/intra-doc/field.rs create mode 100644 src/test/rustdoc/issue-66159.rs create mode 100644 src/test/rustdoc/issue-85454.rs rename src/test/rustdoc/{item-hide-threshold.rs => toggle-item-contents.rs} (85%) create mode 100644 src/test/rustdoc/toggle-method.rs create mode 100644 src/test/rustdoc/toggle-trait-fn.rs create mode 100644 src/test/rustdoc/trait-alias-mention.rs create mode 100644 src/test/rustdoc/trait-visibility.rs create mode 100644 src/test/rustdoc/type-layout-flag-required.rs create mode 100644 src/test/rustdoc/type-layout.rs create mode 100644 src/test/ui/asm/bad-arch.mirunsafeck.stderr delete mode 100644 src/test/ui/asm/bad-arch.stderr create mode 100644 src/test/ui/asm/bad-arch.thirunsafeck.stderr create mode 100644 src/test/ui/asm/bad-template.mirunsafeck.stderr delete mode 100644 src/test/ui/asm/bad-template.stderr create mode 100644 src/test/ui/asm/bad-template.thirunsafeck.stderr create mode 100644 src/test/ui/associated-types/associated-type-destructuring-assignment.rs create mode 100644 src/test/ui/associated-types/associated-type-macro.rs create mode 100644 src/test/ui/associated-types/associated-type-macro.stderr create mode 100644 src/test/ui/associated-types/associated-type-struct-construction.rs create mode 100644 src/test/ui/associated-types/associated-type-tuple-struct-construction.rs create mode 100644 src/test/ui/associated-types/associated-type-tuple-struct-construction.stderr rename src/test/ui/async-await/{async-unsafe-fn-call-in-safe.stderr => async-unsafe-fn-call-in-safe.mir.stderr} (85%) create mode 100644 src/test/ui/async-await/async-unsafe-fn-call-in-safe.thir.stderr create mode 100644 src/test/ui/async-await/issue-73741-type-err.rs create mode 100644 src/test/ui/async-await/issue-73741-type-err.stderr delete mode 100644 src/test/ui/async-await/multiple-lifetimes/ret-impl-trait-no-fg.rs delete mode 100644 src/test/ui/async-await/multiple-lifetimes/ret-impl-trait-no-fg.stderr delete mode 100644 src/test/ui/attributes/doc-attr2.rs delete mode 100644 src/test/ui/attributes/doc-attr2.stderr create mode 100644 src/test/ui/attributes/invalid-doc-attr.rs create mode 100644 src/test/ui/attributes/invalid-doc-attr.stderr create mode 100644 src/test/ui/borrowck/issue-85765.rs create mode 100644 src/test/ui/borrowck/issue-85765.stderr rename src/test/ui/cast/{cast-ptr-to-int-const.with_feature.stderr => cast-ptr-to-int-const.mir.stderr} (88%) create mode 100644 src/test/ui/cast/cast-ptr-to-int-const.thir.stderr create mode 100644 src/test/ui/cast/feature-gate-const_raw_ptr_to_usize_cast.rs rename src/test/ui/cast/{cast-ptr-to-int-const.without_feature.stderr => feature-gate-const_raw_ptr_to_usize_cast.stderr} (66%) create mode 100644 src/test/ui/closures/2229_closure_analysis/diagnostics/borrowck/borrowck-1.rs create mode 100644 src/test/ui/closures/2229_closure_analysis/diagnostics/borrowck/borrowck-1.stderr create mode 100644 src/test/ui/closures/2229_closure_analysis/diagnostics/borrowck/borrowck-2.rs create mode 100644 src/test/ui/closures/2229_closure_analysis/diagnostics/borrowck/borrowck-2.stderr create mode 100644 src/test/ui/closures/2229_closure_analysis/diagnostics/borrowck/borrowck-3.rs create mode 100644 src/test/ui/closures/2229_closure_analysis/diagnostics/borrowck/borrowck-3.stderr create mode 100644 src/test/ui/closures/2229_closure_analysis/diagnostics/borrowck/borrowck-4.rs create mode 100644 src/test/ui/closures/2229_closure_analysis/diagnostics/borrowck/borrowck-4.stderr create mode 100644 src/test/ui/closures/2229_closure_analysis/diagnostics/borrowck/borrowck-closures-mut-and-imm.rs create mode 100644 src/test/ui/closures/2229_closure_analysis/diagnostics/borrowck/borrowck-closures-mut-and-imm.stderr create mode 100644 src/test/ui/closures/2229_closure_analysis/migrations/auto_traits.fixed create mode 100644 src/test/ui/closures/2229_closure_analysis/migrations/auto_traits.rs create mode 100644 src/test/ui/closures/2229_closure_analysis/migrations/auto_traits.stderr create mode 100644 src/test/ui/closures/2229_closure_analysis/migrations/insignificant_drop_attr_migrations.fixed create mode 100644 src/test/ui/closures/2229_closure_analysis/migrations/insignificant_drop_attr_migrations.rs create mode 100644 src/test/ui/closures/2229_closure_analysis/migrations/insignificant_drop_attr_migrations.stderr create mode 100644 src/test/ui/closures/2229_closure_analysis/migrations/insignificant_drop_attr_no_migrations.rs create mode 100644 src/test/ui/closures/2229_closure_analysis/migrations/mir_calls_to_shims.fixed create mode 100644 src/test/ui/closures/2229_closure_analysis/migrations/mir_calls_to_shims.rs create mode 100644 src/test/ui/closures/2229_closure_analysis/migrations/mir_calls_to_shims.stderr rename src/test/ui/closures/{closure_no_cap_coerce_many_unsafe_0.stderr => closure_no_cap_coerce_many_unsafe_0.mir.stderr} (86%) create mode 100644 src/test/ui/closures/closure_no_cap_coerce_many_unsafe_0.thir.stderr rename src/test/ui/closures/{coerce-unsafe-closure-to-unsafe-fn-ptr.stderr => coerce-unsafe-closure-to-unsafe-fn-ptr.mir.stderr} (89%) create mode 100644 src/test/ui/closures/coerce-unsafe-closure-to-unsafe-fn-ptr.thir.stderr create mode 100644 src/test/ui/const-generics/const_evaluatable_checked/evaluated-to-ambig.rs create mode 100644 src/test/ui/const-generics/defaults/forward-declared.rs create mode 100644 src/test/ui/const-generics/defaults/forward-declared.stderr create mode 100644 src/test/ui/const-generics/defaults/param-order-err-pretty-prints-default.rs create mode 100644 src/test/ui/const-generics/defaults/param-order-err-pretty-prints-default.stderr create mode 100644 src/test/ui/const-generics/issues/issue-83466.rs create mode 100644 src/test/ui/const-generics/issues/issue-83466.stderr delete mode 100644 src/test/ui/const-generics/min_const_generics/transmute-const-param-static-reference.rs rename src/test/ui/const-generics/{min_const_generics/transmute-const-param-static-reference.stderr => transmute-const-param-static-reference.min.stderr} (85%) create mode 100644 src/test/ui/const-generics/transmute-const-param-static-reference.rs create mode 100644 src/test/ui/const-generics/unused-type-param-suggestion.rs create mode 100644 src/test/ui/const-generics/unused-type-param-suggestion.stderr delete mode 100644 src/test/ui/consts/async-block.stderr rename src/test/ui/consts/{const_fn_unsize.gated.stderr => async-block.with_feature.stderr} (78%) create mode 100644 src/test/ui/consts/async-block.without_feature.stderr create mode 100644 src/test/ui/consts/const-eval/auxiliary/post_monomorphization_error.rs delete mode 100644 src/test/ui/consts/const-eval/double_promotion.rs create mode 100644 src/test/ui/consts/const-eval/erroneous-const2.rs create mode 100644 src/test/ui/consts/const-eval/erroneous-const2.stderr create mode 100644 src/test/ui/consts/const-eval/issue-85155.rs create mode 100644 src/test/ui/consts/const-eval/issue-85155.stderr create mode 100644 src/test/ui/consts/const-eval/ub-incorrect-vtable.rs create mode 100644 src/test/ui/consts/const-eval/ub-incorrect-vtable.stderr rename src/test/ui/consts/const-extern-fn/{const-extern-fn-requires-unsafe.stderr => const-extern-fn-requires-unsafe.mir.stderr} (86%) create mode 100644 src/test/ui/consts/const-extern-fn/const-extern-fn-requires-unsafe.thir.stderr create mode 100644 src/test/ui/consts/const-needs_drop-monomorphic.rs create mode 100644 src/test/ui/consts/const-needs_drop-monomorphic.stderr delete mode 100644 src/test/ui/consts/const_arg_local.rs delete mode 100644 src/test/ui/consts/const_arg_local.stderr delete mode 100644 src/test/ui/consts/const_arg_promotable.rs delete mode 100644 src/test/ui/consts/const_arg_promotable.stderr delete mode 100644 src/test/ui/consts/const_arg_promotable2.rs delete mode 100644 src/test/ui/consts/const_arg_promotable2.stderr delete mode 100644 src/test/ui/consts/const_arg_wrapper.rs delete mode 100644 src/test/ui/consts/const_arg_wrapper.stderr delete mode 100644 src/test/ui/consts/const_fn_unsize.stock.stderr create mode 100644 src/test/ui/consts/issue-83182.32bit.stderr create mode 100644 src/test/ui/consts/issue-83182.64bit.stderr create mode 100644 src/test/ui/consts/issue-83182.rs delete mode 100644 src/test/ui/consts/rustc-args-required-const.rs delete mode 100644 src/test/ui/consts/rustc-args-required-const.stderr delete mode 100644 src/test/ui/consts/unsizing-cast-non-null.rs delete mode 100644 src/test/ui/consts/unsizing-cast-non-null.stderr create mode 100644 src/test/ui/cross-crate/auxiliary/static_init_aux.rs create mode 100644 src/test/ui/cross-crate/static-init.rs create mode 100644 src/test/ui/deprecation/issue-84637-deprecated-associated-function.fixed create mode 100644 src/test/ui/deprecation/issue-84637-deprecated-associated-function.rs create mode 100644 src/test/ui/deprecation/issue-84637-deprecated-associated-function.stderr create mode 100644 src/test/ui/dyn-keyword/dyn-2018-edition-lint.rs create mode 100644 src/test/ui/dyn-keyword/dyn-2018-edition-lint.stderr create mode 100644 src/test/ui/dyn-keyword/dyn-2021-edition-error.rs create mode 100644 src/test/ui/dyn-keyword/dyn-2021-edition-error.stderr delete mode 100644 src/test/ui/entry-point/imported_main_from_extern_crate.stderr rename src/test/ui/error-codes/{E0133.stderr => E0133.mir.stderr} (93%) rename src/test/ui/{unsafe/unsafe-fn-called-from-safe.stderr => error-codes/E0133.thir.stderr} (88%) delete mode 100644 src/test/ui/extern/external-doc-error.rs delete mode 100644 src/test/ui/extern/external-doc-error.stderr delete mode 100644 src/test/ui/feature-gates/feature-gate-const_fn.rs delete mode 100644 src/test/ui/feature-gates/feature-gate-const_fn.stderr rename src/test/ui/feature-gates/{feature-gate-const_fn_transmute.stderr => feature-gate-const_fn_transmute.mir.stderr} (90%) create mode 100644 src/test/ui/feature-gates/feature-gate-const_fn_transmute.thir.stderr delete mode 100644 src/test/ui/feature-gates/feature-gate-extended_key_value_attributes.rs delete mode 100644 src/test/ui/feature-gates/feature-gate-extended_key_value_attributes.stderr delete mode 100644 src/test/ui/feature-gates/feature-gate-external_doc.rs delete mode 100644 src/test/ui/feature-gates/feature-gate-external_doc.stderr delete mode 100644 src/test/ui/feature-gates/feature-gate-member-constraints.rs delete mode 100644 src/test/ui/feature-gates/feature-gate-member-constraints.stderr create mode 100644 src/test/ui/feature-gates/feature-gate-more-qualified-paths.rs create mode 100644 src/test/ui/feature-gates/feature-gate-more-qualified-paths.stderr create mode 100644 src/test/ui/feature-gates/feature-gate-native_link_modifiers.rs create mode 100644 src/test/ui/feature-gates/feature-gate-native_link_modifiers.stderr create mode 100644 src/test/ui/feature-gates/feature-gate-native_link_modifiers_as_needed.rs create mode 100644 src/test/ui/feature-gates/feature-gate-native_link_modifiers_as_needed.stderr create mode 100644 src/test/ui/feature-gates/feature-gate-native_link_modifiers_bundle.rs create mode 100644 src/test/ui/feature-gates/feature-gate-native_link_modifiers_bundle.stderr create mode 100644 src/test/ui/feature-gates/feature-gate-native_link_modifiers_verbatim.rs create mode 100644 src/test/ui/feature-gates/feature-gate-native_link_modifiers_verbatim.stderr create mode 100644 src/test/ui/feature-gates/feature-gate-native_link_modifiers_whole_archive.rs create mode 100644 src/test/ui/feature-gates/feature-gate-native_link_modifiers_whole_archive.stderr rename src/test/ui/{rfc-2627-raw-dylib => feature-gates}/feature-gate-raw-dylib-2.rs (100%) rename src/test/ui/{rfc-2627-raw-dylib => feature-gates}/feature-gate-raw-dylib-2.stderr (100%) create mode 100644 src/test/ui/feature-gates/feature-gate-raw-dylib-windows-gnu.rs create mode 100644 src/test/ui/feature-gates/feature-gate-raw-dylib-windows-gnu.stderr rename src/test/ui/{rfc-2627-raw-dylib/feature-gate-raw-dylib.rs => feature-gates/feature-gate-raw-dylib-windows-msvc.rs} (71%) rename src/test/ui/{rfc-2627-raw-dylib/feature-gate-raw-dylib.stderr => feature-gates/feature-gate-raw-dylib-windows-msvc.stderr} (88%) create mode 100644 src/test/ui/feature-gates/feature-gate-unnamed_fields.rs create mode 100644 src/test/ui/feature-gates/feature-gate-unnamed_fields.stderr rename src/test/ui/{foreign-unsafe-fn-called.stderr => foreign-unsafe-fn-called.mir.stderr} (89%) create mode 100644 src/test/ui/foreign-unsafe-fn-called.thir.stderr rename src/test/ui/generator/{issue-45729-unsafe-in-generator.stderr => issue-45729-unsafe-in-generator.mir.stderr} (77%) create mode 100644 src/test/ui/generator/issue-45729-unsafe-in-generator.thir.stderr create mode 100644 src/test/ui/generic-associated-types/issue-81862.rs create mode 100644 src/test/ui/generic-associated-types/issue-81862.stderr create mode 100644 src/test/ui/generic-associated-types/missing_lifetime_args.rs create mode 100644 src/test/ui/generic-associated-types/missing_lifetime_args.stderr create mode 100644 src/test/ui/generic-associated-types/missing_lifetime_const.rs create mode 100644 src/test/ui/generic-associated-types/missing_lifetime_const.stderr create mode 100644 src/test/ui/include-macros/auxiliary/same-file-in-two-crates-aux.rs create mode 100644 src/test/ui/include-macros/same-file-in-two-crates.rs rename src/test/ui/intrinsics/{issue-28575.stderr => issue-28575.mir.stderr} (92%) create mode 100644 src/test/ui/intrinsics/issue-28575.thir.stderr rename src/test/ui/intrinsics/{unchecked_math_unsafe.stderr => unchecked_math_unsafe.mir.stderr} (89%) create mode 100644 src/test/ui/intrinsics/unchecked_math_unsafe.thir.stderr create mode 100644 src/test/ui/invalid/invalid-llvm-passes.rs create mode 100644 src/test/ui/invalid/invalid-llvm-passes.stderr delete mode 100644 src/test/ui/invalid/invalid-rustc_args_required_const-arguments.rs delete mode 100644 src/test/ui/invalid/invalid-rustc_args_required_const-arguments.stderr delete mode 100644 src/test/ui/issue-85713-align-with-no-arg.rs delete mode 100644 src/test/ui/issue-85713-align-with-no-arg.stderr rename src/test/ui/issues/{issue-14227.stderr => issue-14227.mir.stderr} (93%) create mode 100644 src/test/ui/issues/issue-14227.thir.stderr rename src/test/ui/issues/{issue-16538.stderr => issue-16538.mir.stderr} (92%) create mode 100644 src/test/ui/issues/issue-16538.thir.stderr rename src/test/ui/issues/{issue-28324.stderr => issue-28324.mir.stderr} (93%) create mode 100644 src/test/ui/issues/issue-28324.thir.stderr rename src/test/ui/issues/{issue-28776.stderr => issue-28776.mir.stderr} (93%) create mode 100644 src/test/ui/issues/issue-28776.thir.stderr rename src/test/ui/issues/{issue-3080.stderr => issue-3080.mir.stderr} (92%) create mode 100644 src/test/ui/issues/issue-3080.thir.stderr delete mode 100644 src/test/ui/issues/issue-38074.rs rename src/test/ui/issues/{issue-45107-unnecessary-unsafe-in-closure.stderr => issue-45107-unnecessary-unsafe-in-closure.mir.stderr} (78%) create mode 100644 src/test/ui/issues/issue-45107-unnecessary-unsafe-in-closure.thir.stderr rename src/test/ui/issues/{issue-47412.stderr => issue-47412.mir.stderr} (83%) create mode 100644 src/test/ui/issues/issue-47412.thir.stderr rename src/test/ui/issues/{issue-48131.stderr => issue-48131.mir.stderr} (81%) create mode 100644 src/test/ui/issues/issue-48131.thir.stderr rename src/test/ui/issues/{issue-5844.stderr => issue-5844.mir.stderr} (93%) create mode 100644 src/test/ui/issues/issue-5844.thir.stderr rename src/test/ui/{ => lang-items}/lang-item-missing-generator.rs (100%) rename src/test/ui/{ => lang-items}/lang-item-missing-generator.stderr (100%) rename src/test/ui/{ => lang-items}/lang-item-missing.rs (100%) rename src/test/ui/{ => lang-items}/lang-item-missing.stderr (100%) create mode 100644 src/test/ui/lang-items/wrong-number-generic-args-add.rs create mode 100644 src/test/ui/lang-items/wrong-number-generic-args-add.stderr create mode 100644 src/test/ui/lang-items/wrong-number-generic-args-index.rs create mode 100644 src/test/ui/lang-items/wrong-number-generic-args-index.stderr create mode 100644 src/test/ui/layout/issue-84108.rs create mode 100644 src/test/ui/layout/issue-84108.stderr create mode 100644 src/test/ui/lint/dead-code/issue-85255.rs create mode 100644 src/test/ui/lint/dead-code/issue-85255.stderr create mode 100644 src/test/ui/lint/force-warn/force-allowed-by-default-lint.rs create mode 100644 src/test/ui/lint/force-warn/force-allowed-by-default-lint.stderr create mode 100644 src/test/ui/lint/force-warn/force-allowed-deny-by-default-lint.rs create mode 100644 src/test/ui/lint/force-warn/force-allowed-deny-by-default-lint.stderr create mode 100644 src/test/ui/lint/force-warn/force-allowed-warning.rs create mode 100644 src/test/ui/lint/force-warn/force-allowed-warning.stderr create mode 100644 src/test/ui/lint/force-warn/force-deny-by-default-lint.rs create mode 100644 src/test/ui/lint/force-warn/force-deny-by-default-lint.stderr create mode 100644 src/test/ui/lint/force-warn/force-lint-allow-all-warnings.rs create mode 100644 src/test/ui/lint/force-warn/force-lint-allow-all-warnings.stderr create mode 100644 src/test/ui/lint/force-warn/force-lint-group-allow-all-warnings.rs create mode 100644 src/test/ui/lint/force-warn/force-lint-group-allow-all-warnings.stderr create mode 100644 src/test/ui/lint/force-warn/force-lint-in-allowed-group.rs create mode 100644 src/test/ui/lint/force-warn/force-lint-in-allowed-group.stderr create mode 100644 src/test/ui/lint/force-warn/force-warn-group-allow-warning.rs create mode 100644 src/test/ui/lint/force-warn/force-warn-group-allow-warning.stderr create mode 100644 src/test/ui/lint/force-warn/force-warn-group.rs create mode 100644 src/test/ui/lint/force-warn/force-warn-group.stderr create mode 100644 src/test/ui/macros/assert-matches-macro-msg.rs create mode 100644 src/test/ui/macros/auxiliary/foreign-crate-macro-pat.rs create mode 100644 src/test/ui/macros/cross-crate-pat-span.rs create mode 100644 src/test/ui/macros/issue-84429-matches-edition.rs delete mode 100644 src/test/ui/macros/issue-85432-ungated-attr-macro.rs create mode 100644 src/test/ui/manual/manual-link-unsupported-kind.rs create mode 100644 src/test/ui/manual/manual-link-unsupported-kind.stderr create mode 100644 src/test/ui/match/auxiliary/match_non_exhaustive_lib.rs create mode 100644 src/test/ui/match/match_non_exhaustive.rs create mode 100644 src/test/ui/match/match_non_exhaustive.stderr create mode 100644 src/test/ui/matches2021.rs create mode 100644 src/test/ui/methods/method-not-found-generic-arg-elision.rs create mode 100644 src/test/ui/methods/method-not-found-generic-arg-elision.stderr create mode 100644 src/test/ui/mismatched_types/issue-84976.rs create mode 100644 src/test/ui/mismatched_types/issue-84976.stderr create mode 100644 src/test/ui/missing/missing-items/missing-type-parameter2.rs create mode 100644 src/test/ui/missing/missing-items/missing-type-parameter2.stderr create mode 100644 src/test/ui/native-library-link-flags/empty-kind-1.rs create mode 100644 src/test/ui/native-library-link-flags/empty-kind-1.stderr create mode 100644 src/test/ui/native-library-link-flags/empty-kind-2.rs create mode 100644 src/test/ui/native-library-link-flags/empty-kind-2.stderr delete mode 100644 src/test/ui/option-to-result.stderr delete mode 100644 src/test/ui/parser/brace-after-qualified-path-in-match.rs delete mode 100644 src/test/ui/parser/brace-after-qualified-path-in-match.stderr create mode 100644 src/test/ui/parser/fn-field-parse-error-ice.rs create mode 100644 src/test/ui/parser/fn-field-parse-error-ice.stderr create mode 100644 src/test/ui/parser/issue-84104.rs create mode 100644 src/test/ui/parser/issue-84104.stderr create mode 100644 src/test/ui/parser/issue-84148-1.rs create mode 100644 src/test/ui/parser/issue-84148-1.stderr create mode 100644 src/test/ui/parser/issue-84148-2.rs create mode 100644 src/test/ui/parser/issue-84148-2.stderr delete mode 100644 src/test/ui/parser/paren-after-qualified-path-in-match.rs delete mode 100644 src/test/ui/parser/paren-after-qualified-path-in-match.stderr create mode 100644 src/test/ui/parser/struct-default-values-and-missing-field-separator.fixed create mode 100644 src/test/ui/parser/struct-default-values-and-missing-field-separator.rs create mode 100644 src/test/ui/parser/struct-default-values-and-missing-field-separator.stderr create mode 100644 src/test/ui/parser/trait-object-delimiters.rs create mode 100644 src/test/ui/parser/trait-object-delimiters.stderr create mode 100644 src/test/ui/parser/unmatched-langle-1.rs create mode 100644 src/test/ui/parser/unmatched-langle-1.stderr create mode 100644 src/test/ui/parser/unmatched-langle-2.rs create mode 100644 src/test/ui/parser/unmatched-langle-2.stderr create mode 100644 src/test/ui/prelude2021.rs rename library/proc_macro/tests/test.rs => src/test/ui/proc-macro/auxiliary/api/cmp.rs (52%) create mode 100644 src/test/ui/proc-macro/auxiliary/api/mod.rs create mode 100644 src/test/ui/proc-macro/auxiliary/api/parse.rs create mode 100644 src/test/ui/proc-macro/auxiliary/custom-quote.rs create mode 100644 src/test/ui/proc-macro/auxiliary/span-from-proc-macro.rs create mode 100644 src/test/ui/proc-macro/expr-stmt-nonterminal-tokens.rs create mode 100644 src/test/ui/proc-macro/expr-stmt-nonterminal-tokens.stdout create mode 100644 src/test/ui/proc-macro/pretty-print-tts.rs create mode 100644 src/test/ui/proc-macro/pretty-print-tts.stdout create mode 100644 src/test/ui/proc-macro/quote-debug.rs create mode 100644 src/test/ui/proc-macro/quote-debug.stdout delete mode 100644 src/test/ui/proc-macro/simple-tuple.rs delete mode 100644 src/test/ui/proc-macro/simple-tuple.stdout create mode 100644 src/test/ui/proc-macro/span-from-proc-macro.rs create mode 100644 src/test/ui/proc-macro/span-from-proc-macro.stderr create mode 100644 src/test/ui/proc-macro/test.rs create mode 100644 src/test/ui/process/process-panic-after-fork.rs create mode 100644 src/test/ui/range/exclusive-range-patterns-2021.rs create mode 100644 src/test/ui/range/exclusive-range-patterns-2021.stderr create mode 100644 src/test/ui/resolve/issue-85348.rs create mode 100644 src/test/ui/resolve/issue-85348.stderr create mode 100644 src/test/ui/resolve/shadow-const-param.rs create mode 100644 src/test/ui/resolve/shadow-const-param.stderr create mode 100644 src/test/ui/rfc-2627-raw-dylib/raw-dylib-msvc-only.rs create mode 100644 src/test/ui/rfc-2627-raw-dylib/raw-dylib-msvc-only.stderr create mode 100644 src/test/ui/rfc-2627-raw-dylib/raw-dylib-windows-only.rs create mode 100644 src/test/ui/rfc-2627-raw-dylib/raw-dylib-windows-only.stderr rename src/test/ui/rfcs/rfc-2396-target_feature-11/{fn-ptr.stderr => fn-ptr.mir.stderr} (95%) create mode 100644 src/test/ui/rfcs/rfc-2396-target_feature-11/fn-ptr.thir.stderr rename src/test/ui/rfcs/rfc-2396-target_feature-11/{safe-calls.stderr => safe-calls.mir.stderr} (92%) create mode 100644 src/test/ui/rfcs/rfc-2396-target_feature-11/safe-calls.thir.stderr delete mode 100644 src/test/ui/rust-2018/suggestions-not-always-applicable.stderr delete mode 100644 src/test/ui/rustc-args-required-const2.rs delete mode 100644 src/test/ui/rustc-args-required-const2.stderr create mode 100644 src/test/ui/rustdoc/deny-invalid-doc-attrs.rs create mode 100644 src/test/ui/rustdoc/deny-invalid-doc-attrs.stderr create mode 100644 src/test/ui/rustdoc/doc-inline-extern-crate.rs create mode 100644 src/test/ui/rustdoc/doc-inline-extern-crate.stderr rename src/test/ui/{safe-extern-statics-mut.stderr => safe-extern-statics-mut.mir.stderr} (87%) create mode 100644 src/test/ui/safe-extern-statics-mut.thir.stderr rename src/test/ui/{safe-extern-statics.stderr => safe-extern-statics.mir.stderr} (89%) create mode 100644 src/test/ui/safe-extern-statics.thir.stderr create mode 100644 src/test/ui/sanitize/crt-static.rs create mode 100644 src/test/ui/sanitize/crt-static.stderr create mode 100644 src/test/ui/simd/issue-85915-simd-ptrs.rs create mode 100644 src/test/ui/simd/simd-type-generic-monomorphisation-extern-nonnull-ptr.rs create mode 100644 src/test/ui/simd/simd-type-generic-monomorphisation-wide-ptr.rs create mode 100644 src/test/ui/simd/simd-type-generic-monomorphisation-wide-ptr.stderr create mode 100644 src/test/ui/simd/simd-type-wide-ptr.rs create mode 100644 src/test/ui/simd/simd-type-wide-ptr.stderr create mode 100644 src/test/ui/simd/wasm-simd-indirect.rs rename src/test/ui/span/{lint-unused-unsafe.stderr => lint-unused-unsafe.mir.stderr} (83%) create mode 100644 src/test/ui/span/lint-unused-unsafe.thir.stderr rename src/test/ui/static/{static-mut-foreign-requires-unsafe.stderr => static-mut-foreign-requires-unsafe.mir.stderr} (85%) create mode 100644 src/test/ui/static/static-mut-foreign-requires-unsafe.thir.stderr rename src/test/ui/static/{static-mut-requires-unsafe.stderr => static-mut-requires-unsafe.mir.stderr} (87%) create mode 100644 src/test/ui/static/static-mut-requires-unsafe.thir.stderr create mode 100644 src/test/ui/structs-enums/struct-rec/issue-74224.rs create mode 100644 src/test/ui/structs-enums/struct-rec/issue-74224.stderr create mode 100644 src/test/ui/structs-enums/struct-rec/issue-84611.rs create mode 100644 src/test/ui/structs-enums/struct-rec/issue-84611.stderr create mode 100644 src/test/ui/structs-enums/struct-rec/mutual-struct-recursion.rs create mode 100644 src/test/ui/structs-enums/struct-rec/mutual-struct-recursion.stderr create mode 100644 src/test/ui/structs/struct-fn-in-definition.rs create mode 100644 src/test/ui/structs/struct-fn-in-definition.stderr create mode 100644 src/test/ui/suggestions/auxiliary/proc-macro-type-error.rs create mode 100644 src/test/ui/suggestions/issue-68049-1.rs create mode 100644 src/test/ui/suggestions/issue-68049-1.stderr create mode 100644 src/test/ui/suggestions/issue-68049-2.rs create mode 100644 src/test/ui/suggestions/issue-68049-2.stderr create mode 100644 src/test/ui/suggestions/issue-84592.rs create mode 100644 src/test/ui/suggestions/issue-84592.stderr create mode 100644 src/test/ui/suggestions/issue-84700.rs create mode 100644 src/test/ui/suggestions/issue-84700.stderr create mode 100644 src/test/ui/suggestions/issue-84973-2.rs create mode 100644 src/test/ui/suggestions/issue-84973-2.stderr create mode 100644 src/test/ui/suggestions/issue-84973-blacklist.rs create mode 100644 src/test/ui/suggestions/issue-84973-blacklist.stderr create mode 100644 src/test/ui/suggestions/issue-84973-negative.rs create mode 100644 src/test/ui/suggestions/issue-84973-negative.stderr create mode 100644 src/test/ui/suggestions/issue-84973.rs create mode 100644 src/test/ui/suggestions/issue-84973.stderr create mode 100644 src/test/ui/suggestions/issue-85347.rs create mode 100644 src/test/ui/suggestions/issue-85347.stderr create mode 100644 src/test/ui/suggestions/issue-85943-no-suggest-unsized-indirection-in-where-clause.rs create mode 100644 src/test/ui/suggestions/issue-85943-no-suggest-unsized-indirection-in-where-clause.stderr create mode 100644 src/test/ui/suggestions/issue-86100-tuple-paren-comma.rs create mode 100644 src/test/ui/suggestions/issue-86100-tuple-paren-comma.stderr create mode 100644 src/test/ui/suggestions/return-elided-lifetime.rs create mode 100644 src/test/ui/suggestions/return-elided-lifetime.stderr create mode 100644 src/test/ui/suggestions/suggest-ref-macro.rs create mode 100644 src/test/ui/suggestions/suggest-ref-macro.stderr create mode 100644 src/test/ui/suggestions/unsized-function-parameter.fixed create mode 100644 src/test/ui/suggestions/unsized-function-parameter.rs create mode 100644 src/test/ui/suggestions/unsized-function-parameter.stderr create mode 100644 src/test/ui/symbol-names/trait-objects.rs create mode 100644 src/test/ui/symbol-names/trait-objects.v0.stderr create mode 100644 src/test/ui/target-feature/rust-specific-name-no-warnings.rs create mode 100644 src/test/ui/target-feature/wasm-safe.rs create mode 100644 src/test/ui/test-attrs/test-type.rs create mode 100644 src/test/ui/test-attrs/test-type.run.stdout create mode 100644 src/test/ui/thread-local-static.rs create mode 100644 src/test/ui/thread-local-static.stderr rename src/test/ui/threads-sendsync/{issue-43733.stderr => issue-43733.mir.stderr} (70%) create mode 100644 src/test/ui/threads-sendsync/issue-43733.thir.stderr create mode 100644 src/test/ui/traits/alias/style_lint.rs create mode 100644 src/test/ui/traits/alias/style_lint.stderr create mode 100644 src/test/ui/traits/cache-reached-depth-ice.rs create mode 100644 src/test/ui/traits/cache-reached-depth-ice.stderr create mode 100644 src/test/ui/traits/issue-83538-tainted-cache-after-cycle.rs create mode 100644 src/test/ui/traits/issue-83538-tainted-cache-after-cycle.stderr create mode 100644 src/test/ui/traits/issue-84399-bad-fresh-caching.rs rename src/test/ui/traits/{safety-fn-body.stderr => safety-fn-body.mir.stderr} (79%) create mode 100644 src/test/ui/traits/safety-fn-body.thir.stderr delete mode 100644 src/test/ui/try-on-option.stderr delete mode 100644 src/test/ui/try-operator-custom.rs create mode 100644 src/test/ui/try-trait/bad-interconversion.rs create mode 100644 src/test/ui/try-trait/bad-interconversion.stderr rename src/test/ui/{ => try-trait}/option-to-result.rs (63%) create mode 100644 src/test/ui/try-trait/option-to-result.stderr create mode 100644 src/test/ui/try-trait/try-as-monad.rs rename src/test/ui/{ => try-trait}/try-on-option-diagnostics.rs (100%) rename src/test/ui/{ => try-trait}/try-on-option-diagnostics.stderr (60%) rename src/test/ui/{ => try-trait}/try-on-option.rs (80%) create mode 100644 src/test/ui/try-trait/try-on-option.stderr create mode 100644 src/test/ui/try-trait/try-operator-custom.rs rename src/test/ui/{ => try-trait}/try-operator-on-main.rs (75%) rename src/test/ui/{ => try-trait}/try-operator-on-main.stderr (52%) rename src/test/ui/{ => try-trait}/try-poll.rs (100%) create mode 100644 src/test/ui/type-alias-impl-trait/multiple-def-uses-in-one-fn-pass.rs create mode 100644 src/test/ui/type-alias-impl-trait/multiple-def-uses-in-one-fn.rs create mode 100644 src/test/ui/type-alias-impl-trait/multiple-def-uses-in-one-fn.stderr create mode 100644 src/test/ui/type-alias-impl-trait/multiple-def-uses-in-one-fn2.rs create mode 100644 src/test/ui/type-alias-impl-trait/multiple-def-uses-in-one-fn2.stderr create mode 100644 src/test/ui/type-alias-impl-trait/multiple-def-uses-in-one-fn3.rs create mode 100644 src/test/ui/type-alias-impl-trait/multiple-def-uses-in-one-fn3.stderr create mode 100644 src/test/ui/unnamed_fields/restrict_anonymous.rs create mode 100644 src/test/ui/unnamed_fields/restrict_anonymous.stderr create mode 100644 src/test/ui/unsafe/inline_asm.mir.stderr create mode 100644 src/test/ui/unsafe/inline_asm.rs create mode 100644 src/test/ui/unsafe/inline_asm.thir.stderr rename src/test/ui/unsafe/{issue-45087-unreachable-unsafe.stderr => issue-45087-unreachable-unsafe.mir.stderr} (77%) create mode 100644 src/test/ui/unsafe/issue-45087-unreachable-unsafe.thir.stderr rename src/test/ui/unsafe/{ranged_ints.stderr => ranged_ints.mir.stderr} (93%) create mode 100644 src/test/ui/unsafe/ranged_ints.thir.stderr rename src/test/ui/unsafe/{ranged_ints_const.stderr => ranged_ints_const.mir.stderr} (93%) create mode 100644 src/test/ui/unsafe/ranged_ints_const.thir.stderr rename src/test/ui/unsafe/{rfc-2585-unsafe_op_in_unsafe_fn.stderr => rfc-2585-unsafe_op_in_unsafe_fn.mir.stderr} (79%) create mode 100644 src/test/ui/unsafe/rfc-2585-unsafe_op_in_unsafe_fn.thir.stderr rename src/test/ui/unsafe/{unsafe-around-compiler-generated-unsafe.stderr => unsafe-around-compiler-generated-unsafe.mir.stderr} (69%) create mode 100644 src/test/ui/unsafe/unsafe-around-compiler-generated-unsafe.thir.stderr rename src/test/ui/unsafe/{unsafe-const-fn.stderr => unsafe-const-fn.mir.stderr} (91%) create mode 100644 src/test/ui/unsafe/unsafe-const-fn.thir.stderr rename src/test/ui/unsafe/{unsafe-fn-assign-deref-ptr.stderr => unsafe-fn-assign-deref-ptr.mir.stderr} (76%) create mode 100644 src/test/ui/unsafe/unsafe-fn-assign-deref-ptr.thir.stderr create mode 100644 src/test/ui/unsafe/unsafe-fn-called-from-safe.mir.stderr create mode 100644 src/test/ui/unsafe/unsafe-fn-called-from-safe.thir.stderr rename src/test/ui/unsafe/{unsafe-fn-deref-ptr.stderr => unsafe-fn-deref-ptr.mir.stderr} (77%) create mode 100644 src/test/ui/unsafe/unsafe-fn-deref-ptr.thir.stderr rename src/test/ui/unsafe/{unsafe-fn-used-as-value.stderr => unsafe-fn-used-as-value.mir.stderr} (89%) create mode 100644 src/test/ui/unsafe/unsafe-fn-used-as-value.thir.stderr rename src/test/ui/unsafe/{unsafe-unstable-const-fn.stderr => unsafe-unstable-const-fn.mir.stderr} (76%) create mode 100644 src/test/ui/unsafe/unsafe-unstable-const-fn.thir.stderr create mode 100644 src/test/ui/unused/unused-attr-macro-rules.rs create mode 100644 src/test/ui/unused/unused-attr-macro-rules.stderr create mode 100644 src/tools/clippy/clippy.toml delete mode 100644 src/tools/clippy/clippy_lints/src/consts.rs create mode 100644 src/tools/clippy/clippy_lints/src/methods/manual_str_repeat.rs create mode 100644 src/tools/clippy/clippy_lints/src/methods/suspicious_splitn.rs delete mode 100644 src/tools/clippy/clippy_lints/src/misc_early.rs create mode 100644 src/tools/clippy/clippy_lints/src/misc_early/builtin_type_shadow.rs create mode 100644 src/tools/clippy/clippy_lints/src/misc_early/double_neg.rs create mode 100644 src/tools/clippy/clippy_lints/src/misc_early/mixed_case_hex_literals.rs create mode 100644 src/tools/clippy/clippy_lints/src/misc_early/mod.rs create mode 100644 src/tools/clippy/clippy_lints/src/misc_early/redundant_pattern.rs create mode 100644 src/tools/clippy/clippy_lints/src/misc_early/unneeded_field_pattern.rs create mode 100644 src/tools/clippy/clippy_lints/src/misc_early/unneeded_wildcard_pattern.rs create mode 100644 src/tools/clippy/clippy_lints/src/misc_early/unseparated_literal_suffix.rs create mode 100644 src/tools/clippy/clippy_lints/src/misc_early/zero_prefixed_literal.rs create mode 100644 src/tools/clippy/clippy_lints/src/needless_bitwise_bool.rs create mode 100644 src/tools/clippy/clippy_lints/src/unused_async.rs create mode 100644 src/tools/clippy/clippy_lints/src/utils/internal_lints/metadata_collector.rs delete mode 100644 src/tools/clippy/mini-macro/Cargo.toml delete mode 100644 src/tools/clippy/mini-macro/src/lib.rs create mode 100644 src/tools/clippy/tests/clippy.toml rename src/tools/clippy/tests/ui/{builtin-type-shadow.rs => builtin_type_shadow.rs} (100%) rename src/tools/clippy/tests/ui/{builtin-type-shadow.stderr => builtin_type_shadow.stderr} (89%) create mode 100644 src/tools/clippy/tests/ui/crashes/auxiliary/ice-7272-aux.rs create mode 100644 src/tools/clippy/tests/ui/crashes/ice-7231.rs create mode 100644 src/tools/clippy/tests/ui/crashes/ice-7272.rs delete mode 100644 src/tools/clippy/tests/ui/crashes/procedural_macro.rs create mode 100644 src/tools/clippy/tests/ui/manual_str_repeat.fixed create mode 100644 src/tools/clippy/tests/ui/manual_str_repeat.rs create mode 100644 src/tools/clippy/tests/ui/manual_str_repeat.stderr create mode 100644 src/tools/clippy/tests/ui/needless_bitwise_bool.fixed create mode 100644 src/tools/clippy/tests/ui/needless_bitwise_bool.rs create mode 100644 src/tools/clippy/tests/ui/needless_bitwise_bool.stderr create mode 100644 src/tools/clippy/tests/ui/needless_borrow_pat.rs create mode 100644 src/tools/clippy/tests/ui/needless_borrow_pat.stderr create mode 100644 src/tools/clippy/tests/ui/ref_binding_to_reference.rs create mode 100644 src/tools/clippy/tests/ui/ref_binding_to_reference.stderr create mode 100644 src/tools/clippy/tests/ui/suspicious_splitn.rs create mode 100644 src/tools/clippy/tests/ui/suspicious_splitn.stderr create mode 100644 src/tools/clippy/tests/ui/unused_async.rs create mode 100644 src/tools/clippy/tests/ui/unused_async.stderr delete mode 100755 src/tools/clippy/util/cov.sh create mode 100644 src/tools/linkchecker/tests/basic_broken/foo.html create mode 100644 src/tools/linkchecker/tests/broken_fragment_local/foo.html create mode 100644 src/tools/linkchecker/tests/broken_fragment_remote/bar.html create mode 100644 src/tools/linkchecker/tests/broken_fragment_remote/inner/foo.html create mode 100644 src/tools/linkchecker/tests/broken_redir/foo.html create mode 100644 src/tools/linkchecker/tests/broken_redir/redir-bad.html create mode 100644 src/tools/linkchecker/tests/checks.rs create mode 100644 src/tools/linkchecker/tests/directory_link/foo.html create mode 100644 src/tools/linkchecker/tests/directory_link/somedir/index.html create mode 100644 src/tools/linkchecker/tests/redirect_loop/foo.html create mode 100644 src/tools/linkchecker/tests/redirect_loop/redir-bad.html create mode 100644 src/tools/linkchecker/tests/valid/inner/bar.html create mode 100644 src/tools/linkchecker/tests/valid/inner/foo.html create mode 100644 src/tools/linkchecker/tests/valid/inner/redir-bad.html create mode 100644 src/tools/linkchecker/tests/valid/inner/redir-target.html create mode 100644 src/tools/linkchecker/tests/valid/inner/redir.html create mode 100644 src/tools/linkchecker/tests/valid/outer.html create mode 100644 src/tools/rustfmt/src/test/mod_resolver.rs create mode 100644 src/tools/rustfmt/tests/mod-resolver/issue-4874/bar/baz.rs create mode 100644 src/tools/rustfmt/tests/mod-resolver/issue-4874/foo.rs create mode 100644 src/tools/rustfmt/tests/mod-resolver/issue-4874/foo/qux.rs create mode 100644 src/tools/rustfmt/tests/mod-resolver/issue-4874/main.rs rename vendor/{addr2line => addr2line-0.14.0}/.cargo-checksum.json (58%) rename vendor/{addr2line => addr2line-0.14.0}/CHANGELOG.md (93%) rename vendor/{addr2line => addr2line-0.14.0}/Cargo.lock (98%) rename vendor/{addr2line => addr2line-0.14.0}/Cargo.toml (96%) rename vendor/{addr2line => addr2line-0.14.0}/LICENSE-APACHE (100%) rename vendor/{addr2line => addr2line-0.14.0}/LICENSE-MIT (100%) rename vendor/{addr2line => addr2line-0.14.0}/README.md (100%) rename vendor/{addr2line => addr2line-0.14.0}/bench.plot.r (100%) rename vendor/{addr2line => addr2line-0.14.0}/benchmark.sh (100%) rename vendor/{addr2line => addr2line-0.14.0}/coverage (100%) rename vendor/{addr2line => addr2line-0.14.0}/examples/addr2line.rs (98%) rename vendor/{addr2line => addr2line-0.14.0}/rustfmt.toml (100%) rename vendor/{addr2line => addr2line-0.14.0}/src/lazy.rs (100%) rename vendor/{addr2line => addr2line-0.14.0}/src/lib.rs (85%) rename vendor/{addr2line => addr2line-0.14.0}/tests/correctness.rs (100%) rename vendor/{addr2line => addr2line-0.14.0}/tests/output_equivalence.rs (100%) rename vendor/{addr2line => addr2line-0.14.0}/tests/parse.rs (100%) create mode 100644 vendor/cc/src/vs_instances.rs create mode 100644 vendor/crossbeam-epoch/build.rs create mode 100644 vendor/crossbeam-epoch/no_atomic.rs create mode 100644 vendor/crossbeam-utils/no_atomic.rs rename vendor/{gimli => gimli-0.23.0}/.cargo-checksum.json (100%) rename vendor/{gimli => gimli-0.23.0}/CHANGELOG.md (100%) rename vendor/{gimli => gimli-0.23.0}/CONTRIBUTING.md (100%) rename vendor/{gimli => gimli-0.23.0}/Cargo.lock (100%) rename vendor/{gimli => gimli-0.23.0}/Cargo.toml (100%) rename vendor/{gimli => gimli-0.23.0}/LICENSE-APACHE (100%) rename vendor/{gimli => gimli-0.23.0}/LICENSE-MIT (100%) rename vendor/{gimli => gimli-0.23.0}/README.md (100%) rename vendor/{gimli => gimli-0.23.0}/benches/bench.rs (100%) rename vendor/{gimli => gimli-0.23.0}/examples/dwarf-validate.rs (100%) rename vendor/{gimli => gimli-0.23.0}/examples/dwarfdump.rs (100%) rename vendor/{gimli => gimli-0.23.0}/examples/simple.rs (100%) rename vendor/{gimli => gimli-0.23.0}/examples/simple_line.rs (100%) rename vendor/{gimli => gimli-0.23.0}/fixtures/self/README.md (100%) rename vendor/{gimli => gimli-0.23.0}/fixtures/self/debug_abbrev (100%) rename vendor/{gimli => gimli-0.23.0}/fixtures/self/debug_aranges (100%) rename vendor/{gimli => gimli-0.23.0}/fixtures/self/debug_info (100%) rename vendor/{gimli => gimli-0.23.0}/fixtures/self/debug_inlined (100%) rename vendor/{gimli => gimli-0.23.0}/fixtures/self/debug_line (100%) rename vendor/{gimli => gimli-0.23.0}/fixtures/self/debug_loc (100%) rename vendor/{gimli => gimli-0.23.0}/fixtures/self/debug_pubnames (100%) rename vendor/{gimli => gimli-0.23.0}/fixtures/self/debug_pubtypes (100%) rename vendor/{gimli => gimli-0.23.0}/fixtures/self/debug_ranges (100%) rename vendor/{gimli => gimli-0.23.0}/fixtures/self/debug_str (100%) rename vendor/{gimli => gimli-0.23.0}/fixtures/self/eh_frame (100%) rename vendor/{gimli => gimli-0.23.0}/fixtures/self/eh_frame_hdr (100%) create mode 100644 vendor/gimli-0.23.0/rustfmt.toml rename vendor/{gimli => gimli-0.23.0}/src/arch.rs (100%) rename vendor/{gimli => gimli-0.23.0}/src/common.rs (100%) rename vendor/{gimli => gimli-0.23.0}/src/constants.rs (100%) rename vendor/{gimli => gimli-0.23.0}/src/endianity.rs (100%) rename vendor/{gimli => gimli-0.23.0}/src/leb128.rs (100%) rename vendor/{gimli => gimli-0.23.0}/src/lib.rs (100%) rename vendor/{gimli => gimli-0.23.0}/src/read/abbrev.rs (100%) rename vendor/{gimli => gimli-0.23.0}/src/read/addr.rs (100%) rename vendor/{gimli => gimli-0.23.0}/src/read/aranges.rs (100%) rename vendor/{gimli => gimli-0.23.0}/src/read/cfi.rs (100%) rename vendor/{gimli => gimli-0.23.0}/src/read/dwarf.rs (100%) rename vendor/{gimli => gimli-0.23.0}/src/read/endian_reader.rs (100%) rename vendor/{gimli => gimli-0.23.0}/src/read/endian_slice.rs (100%) rename vendor/{gimli => gimli-0.23.0}/src/read/line.rs (100%) rename vendor/{gimli => gimli-0.23.0}/src/read/lists.rs (100%) rename vendor/{gimli => gimli-0.23.0}/src/read/loclists.rs (100%) rename vendor/{gimli => gimli-0.23.0}/src/read/lookup.rs (100%) rename vendor/{gimli => gimli-0.23.0}/src/read/mod.rs (100%) rename vendor/{gimli => gimli-0.23.0}/src/read/op.rs (100%) rename vendor/{gimli => gimli-0.23.0}/src/read/pubnames.rs (100%) rename vendor/{gimli => gimli-0.23.0}/src/read/pubtypes.rs (100%) rename vendor/{gimli => gimli-0.23.0}/src/read/reader.rs (100%) rename vendor/{gimli => gimli-0.23.0}/src/read/rnglists.rs (100%) rename vendor/{gimli => gimli-0.23.0}/src/read/str.rs (100%) rename vendor/{gimli => gimli-0.23.0}/src/read/unit.rs (100%) rename vendor/{gimli => gimli-0.23.0}/src/read/value.rs (100%) rename vendor/{gimli => gimli-0.23.0}/src/test_util.rs (100%) rename vendor/{gimli => gimli-0.23.0}/src/write/abbrev.rs (100%) rename vendor/{gimli => gimli-0.23.0}/src/write/cfi.rs (100%) rename vendor/{gimli => gimli-0.23.0}/src/write/dwarf.rs (100%) rename vendor/{gimli => gimli-0.23.0}/src/write/endian_vec.rs (100%) rename vendor/{gimli => gimli-0.23.0}/src/write/line.rs (100%) rename vendor/{gimli => gimli-0.23.0}/src/write/loc.rs (100%) rename vendor/{gimli => gimli-0.23.0}/src/write/mod.rs (100%) rename vendor/{gimli => gimli-0.23.0}/src/write/op.rs (100%) rename vendor/{gimli => gimli-0.23.0}/src/write/range.rs (100%) rename vendor/{gimli => gimli-0.23.0}/src/write/section.rs (100%) rename vendor/{gimli => gimli-0.23.0}/src/write/str.rs (100%) rename vendor/{gimli => gimli-0.23.0}/src/write/unit.rs (100%) rename vendor/{gimli => gimli-0.23.0}/src/write/writer.rs (100%) rename vendor/{gimli => gimli-0.23.0}/tests/convert_self.rs (100%) rename vendor/{gimli => gimli-0.23.0}/tests/parse_self.rs (100%) create mode 100644 vendor/memchr/src/cow.rs rename vendor/memchr/src/{ => memchr}/c.rs (80%) rename vendor/memchr/src/{ => memchr}/fallback.rs (96%) rename vendor/memchr/src/{ => memchr}/iter.rs (96%) create mode 100644 vendor/memchr/src/memchr/mod.rs rename vendor/memchr/src/{ => memchr}/naive.rs (100%) rename vendor/memchr/src/{ => memchr}/x86/avx.rs (86%) create mode 100644 vendor/memchr/src/memchr/x86/mod.rs rename vendor/memchr/src/{ => memchr}/x86/sse2.rs (98%) rename vendor/memchr/src/{ => memchr}/x86/sse42.rs (90%) create mode 100644 vendor/memchr/src/memmem/byte_frequencies.rs create mode 100644 vendor/memchr/src/memmem/genericsimd.rs create mode 100644 vendor/memchr/src/memmem/mod.rs create mode 100644 vendor/memchr/src/memmem/prefilter/fallback.rs create mode 100644 vendor/memchr/src/memmem/prefilter/genericsimd.rs create mode 100644 vendor/memchr/src/memmem/prefilter/mod.rs create mode 100644 vendor/memchr/src/memmem/prefilter/x86/avx.rs create mode 100644 vendor/memchr/src/memmem/prefilter/x86/mod.rs create mode 100644 vendor/memchr/src/memmem/prefilter/x86/sse.rs create mode 100644 vendor/memchr/src/memmem/rabinkarp.rs create mode 100644 vendor/memchr/src/memmem/rarebytes.rs create mode 100644 vendor/memchr/src/memmem/twoway.rs create mode 100644 vendor/memchr/src/memmem/util.rs create mode 100644 vendor/memchr/src/memmem/vector.rs create mode 100644 vendor/memchr/src/memmem/x86/avx.rs create mode 100644 vendor/memchr/src/memmem/x86/mod.rs create mode 100644 vendor/memchr/src/memmem/x86/sse.rs rename vendor/memchr/src/tests/{ => memchr}/iter.rs (98%) rename vendor/memchr/src/tests/{ => memchr}/memchr.rs (93%) create mode 100644 vendor/memchr/src/tests/memchr/mod.rs rename vendor/memchr/src/tests/{miri.rs => memchr/simple.rs} (69%) create mode 100644 vendor/memchr/src/tests/memchr/testdata.rs delete mode 100644 vendor/memchr/src/x86/mod.rs delete mode 100644 vendor/memmap/.cargo-checksum.json delete mode 100644 vendor/memmap/Cargo.toml delete mode 100644 vendor/memmap/README.md delete mode 100644 vendor/memmap/ci/install.sh delete mode 100644 vendor/memmap/ci/script.sh delete mode 100644 vendor/memmap/examples/cat.rs delete mode 100644 vendor/memmap/src/lib.rs delete mode 100644 vendor/memmap/src/unix.rs delete mode 100644 vendor/memmap/src/windows.rs create mode 100644 vendor/memmap2/src/stub.rs create mode 100644 vendor/object/.cargo-checksum.json create mode 100644 vendor/object/CHANGELOG.md create mode 100644 vendor/object/Cargo.lock create mode 100644 vendor/object/Cargo.toml rename vendor/{regex => object}/LICENSE-APACHE (100%) rename vendor/{memmap => object}/LICENSE-MIT (96%) create mode 100644 vendor/object/README.md create mode 100644 vendor/object/clippy.toml create mode 100644 vendor/object/examples/ar.rs create mode 100644 vendor/object/examples/dyldcachedump.rs create mode 100644 vendor/object/examples/nm.rs create mode 100644 vendor/object/examples/objcopy.rs create mode 100644 vendor/object/examples/objdump.rs create mode 100644 vendor/object/examples/objectmap.rs create mode 100644 vendor/object/examples/readobj.rs create mode 100644 vendor/object/src/archive.rs create mode 100644 vendor/object/src/common.rs create mode 100644 vendor/object/src/elf.rs create mode 100644 vendor/object/src/endian.rs create mode 100644 vendor/object/src/lib.rs create mode 100644 vendor/object/src/macho.rs create mode 100644 vendor/object/src/pe.rs create mode 100644 vendor/object/src/pod.rs create mode 100644 vendor/object/src/read/any.rs create mode 100644 vendor/object/src/read/archive.rs create mode 100644 vendor/object/src/read/coff/comdat.rs create mode 100644 vendor/object/src/read/coff/file.rs create mode 100644 vendor/object/src/read/coff/mod.rs create mode 100644 vendor/object/src/read/coff/relocation.rs create mode 100644 vendor/object/src/read/coff/section.rs create mode 100644 vendor/object/src/read/coff/symbol.rs create mode 100644 vendor/object/src/read/elf/comdat.rs create mode 100644 vendor/object/src/read/elf/compression.rs create mode 100644 vendor/object/src/read/elf/dynamic.rs create mode 100644 vendor/object/src/read/elf/file.rs create mode 100644 vendor/object/src/read/elf/mod.rs create mode 100644 vendor/object/src/read/elf/note.rs create mode 100644 vendor/object/src/read/elf/relocation.rs create mode 100644 vendor/object/src/read/elf/section.rs create mode 100644 vendor/object/src/read/elf/segment.rs create mode 100644 vendor/object/src/read/elf/symbol.rs create mode 100644 vendor/object/src/read/macho/dyld_cache.rs create mode 100644 vendor/object/src/read/macho/fat.rs create mode 100644 vendor/object/src/read/macho/file.rs create mode 100644 vendor/object/src/read/macho/load_command.rs create mode 100644 vendor/object/src/read/macho/mod.rs create mode 100644 vendor/object/src/read/macho/relocation.rs create mode 100644 vendor/object/src/read/macho/section.rs create mode 100644 vendor/object/src/read/macho/segment.rs create mode 100644 vendor/object/src/read/macho/symbol.rs create mode 100644 vendor/object/src/read/mod.rs create mode 100644 vendor/object/src/read/pe/file.rs create mode 100644 vendor/object/src/read/pe/mod.rs create mode 100644 vendor/object/src/read/pe/section.rs create mode 100644 vendor/object/src/read/read_cache.rs create mode 100644 vendor/object/src/read/read_ref.rs create mode 100644 vendor/object/src/read/traits.rs create mode 100644 vendor/object/src/read/util.rs create mode 100644 vendor/object/src/read/wasm.rs create mode 100644 vendor/object/src/write/coff.rs create mode 100644 vendor/object/src/write/elf.rs create mode 100644 vendor/object/src/write/macho.rs create mode 100644 vendor/object/src/write/mod.rs create mode 100644 vendor/object/src/write/string.rs create mode 100644 vendor/object/src/write/util.rs create mode 100644 vendor/object/tests/integration.rs create mode 100644 vendor/object/tests/parse_self.rs create mode 100644 vendor/object/tests/round_trip/bss.rs create mode 100644 vendor/object/tests/round_trip/comdat.rs create mode 100644 vendor/object/tests/round_trip/common.rs create mode 100644 vendor/object/tests/round_trip/elf.rs create mode 100644 vendor/object/tests/round_trip/macho.rs create mode 100644 vendor/object/tests/round_trip/mod.rs create mode 100644 vendor/object/tests/round_trip/tls.rs delete mode 100644 vendor/rayon-core/src/util.rs create mode 100644 vendor/rayon/src/array.rs delete mode 100644 vendor/redox_syscall-0.1.57/.cargo-checksum.json delete mode 100644 vendor/redox_syscall-0.1.57/Cargo.toml delete mode 100644 vendor/redox_syscall-0.1.57/LICENSE delete mode 100644 vendor/redox_syscall-0.1.57/README.md delete mode 100644 vendor/redox_syscall-0.1.57/src/arch/aarch64.rs delete mode 100644 vendor/redox_syscall-0.1.57/src/arch/arm.rs delete mode 100644 vendor/redox_syscall-0.1.57/src/arch/nonredox.rs delete mode 100644 vendor/redox_syscall-0.1.57/src/arch/x86.rs delete mode 100644 vendor/redox_syscall-0.1.57/src/arch/x86_64.rs delete mode 100644 vendor/redox_syscall-0.1.57/src/call.rs delete mode 100644 vendor/redox_syscall-0.1.57/src/data.rs delete mode 100644 vendor/redox_syscall-0.1.57/src/error.rs delete mode 100644 vendor/redox_syscall-0.1.57/src/flag.rs delete mode 100644 vendor/redox_syscall-0.1.57/src/io/dma.rs delete mode 100644 vendor/redox_syscall-0.1.57/src/io/io.rs delete mode 100644 vendor/redox_syscall-0.1.57/src/io/mmio.rs delete mode 100644 vendor/redox_syscall-0.1.57/src/io/mod.rs delete mode 100644 vendor/redox_syscall-0.1.57/src/io/pio.rs delete mode 100644 vendor/redox_syscall-0.1.57/src/lib.rs delete mode 100644 vendor/redox_syscall-0.1.57/src/number.rs delete mode 100755 vendor/redox_syscall-0.1.57/src/scheme/generate.sh delete mode 100644 vendor/redox_syscall-0.1.57/src/scheme/mod.rs delete mode 100644 vendor/redox_syscall-0.1.57/src/scheme/scheme.rs delete mode 100644 vendor/redox_syscall-0.1.57/src/scheme/scheme_block.rs delete mode 100644 vendor/redox_syscall-0.1.57/src/scheme/scheme_block_mut.rs delete mode 100644 vendor/redox_syscall-0.1.57/src/scheme/scheme_mut.rs delete mode 100644 vendor/redox_syscall-0.1.57/src/tests.rs create mode 100644 vendor/redox_syscall/src/arch/riscv64.rs create mode 100644 vendor/regex-1.4.3/.cargo-checksum.json rename vendor/{regex => regex-1.4.3}/CHANGELOG.md (95%) create mode 100644 vendor/regex-1.4.3/Cargo.lock rename vendor/{regex => regex-1.4.3}/Cargo.toml (95%) rename vendor/{regex => regex-1.4.3}/HACKING.md (100%) rename vendor/{memmap => regex-1.4.3}/LICENSE-APACHE (99%) rename vendor/{regex => regex-1.4.3}/LICENSE-MIT (100%) rename vendor/{regex => regex-1.4.3}/PERFORMANCE.md (100%) rename vendor/{regex => regex-1.4.3}/README.md (98%) rename vendor/{regex => regex-1.4.3}/UNICODE.md (87%) rename vendor/{regex => regex-1.4.3}/examples/regexdna-input.txt (100%) rename vendor/{regex => regex-1.4.3}/examples/regexdna-output.txt (100%) rename vendor/{regex => regex-1.4.3}/examples/shootout-regex-dna-bytes.rs (100%) rename vendor/{regex => regex-1.4.3}/examples/shootout-regex-dna-cheat.rs (100%) rename vendor/{regex => regex-1.4.3}/examples/shootout-regex-dna-replace.rs (100%) rename vendor/{regex => regex-1.4.3}/examples/shootout-regex-dna-single-cheat.rs (100%) rename vendor/{regex => regex-1.4.3}/examples/shootout-regex-dna-single.rs (100%) rename vendor/{regex => regex-1.4.3}/examples/shootout-regex-dna.rs (100%) rename vendor/{regex => regex-1.4.3}/rustfmt.toml (100%) rename vendor/{regex => regex-1.4.3}/src/backtrack.rs (98%) create mode 100644 vendor/regex-1.4.3/src/cache.rs rename vendor/{regex => regex-1.4.3}/src/compile.rs (98%) rename vendor/{regex => regex-1.4.3}/src/dfa.rs (99%) rename vendor/{regex => regex-1.4.3}/src/error.rs (100%) rename vendor/{regex => regex-1.4.3}/src/exec.rs (97%) rename vendor/{regex => regex-1.4.3}/src/expand.rs (98%) rename vendor/{regex => regex-1.4.3}/src/find_byte.rs (100%) rename vendor/{regex => regex-1.4.3}/src/freqs.rs (100%) rename vendor/{regex => regex-1.4.3}/src/input.rs (100%) rename vendor/{regex => regex-1.4.3}/src/lib.rs (98%) rename vendor/{regex => regex-1.4.3}/src/literal/imp.rs (100%) rename vendor/{regex => regex-1.4.3}/src/literal/mod.rs (100%) rename vendor/{regex => regex-1.4.3}/src/pattern.rs (100%) rename vendor/{regex => regex-1.4.3}/src/pikevm.rs (100%) rename vendor/{regex => regex-1.4.3}/src/prog.rs (97%) rename vendor/{regex => regex-1.4.3}/src/re_builder.rs (100%) rename vendor/{regex => regex-1.4.3}/src/re_bytes.rs (96%) rename vendor/{regex => regex-1.4.3}/src/re_set.rs (99%) rename vendor/{regex => regex-1.4.3}/src/re_trait.rs (100%) rename vendor/{regex => regex-1.4.3}/src/re_unicode.rs (96%) rename vendor/{regex => regex-1.4.3}/src/sparse.rs (96%) rename vendor/{regex => regex-1.4.3}/src/testdata/LICENSE (100%) rename vendor/{regex => regex-1.4.3}/src/testdata/README (100%) rename vendor/{regex => regex-1.4.3}/src/testdata/basic.dat (100%) rename vendor/{regex => regex-1.4.3}/src/testdata/nullsubexpr.dat (100%) rename vendor/{regex => regex-1.4.3}/src/testdata/repetition.dat (100%) rename vendor/{regex => regex-1.4.3}/src/utf8.rs (100%) rename vendor/{regex => regex-1.4.3}/test (100%) rename vendor/{regex => regex-1.4.3}/tests/api.rs (100%) rename vendor/{regex => regex-1.4.3}/tests/api_str.rs (100%) rename vendor/{regex => regex-1.4.3}/tests/bytes.rs (100%) rename vendor/{regex => regex-1.4.3}/tests/consistent.rs (98%) rename vendor/{regex => regex-1.4.3}/tests/crates_regex.rs (100%) rename vendor/{regex => regex-1.4.3}/tests/crazy.rs (98%) rename vendor/{regex => regex-1.4.3}/tests/flags.rs (100%) rename vendor/{regex => regex-1.4.3}/tests/fowler.rs (100%) rename vendor/{regex => regex-1.4.3}/tests/macros.rs (100%) rename vendor/{regex => regex-1.4.3}/tests/macros_bytes.rs (99%) rename vendor/{regex => regex-1.4.3}/tests/macros_str.rs (93%) rename vendor/{regex => regex-1.4.3}/tests/misc.rs (100%) rename vendor/{regex => regex-1.4.3}/tests/multiline.rs (100%) rename vendor/{regex => regex-1.4.3}/tests/noparse.rs (100%) rename vendor/{regex => regex-1.4.3}/tests/regression.rs (100%) rename vendor/{regex => regex-1.4.3}/tests/regression_fuzz.rs (58%) rename vendor/{regex => regex-1.4.3}/tests/replace.rs (60%) rename vendor/{regex => regex-1.4.3}/tests/searcher.rs (100%) rename vendor/{regex => regex-1.4.3}/tests/set.rs (100%) rename vendor/{regex => regex-1.4.3}/tests/shortest_match.rs (100%) rename vendor/{regex => regex-1.4.3}/tests/suffix_reverse.rs (100%) rename vendor/{regex => regex-1.4.3}/tests/test_backtrack.rs (100%) rename vendor/{regex => regex-1.4.3}/tests/test_backtrack_bytes.rs (100%) rename vendor/{regex => regex-1.4.3}/tests/test_backtrack_utf8bytes.rs (100%) rename vendor/{regex => regex-1.4.3}/tests/test_crates_regex.rs (100%) rename vendor/{regex => regex-1.4.3}/tests/test_default.rs (64%) rename vendor/{regex => regex-1.4.3}/tests/test_default_bytes.rs (100%) rename vendor/{regex => regex-1.4.3}/tests/test_nfa.rs (100%) rename vendor/{regex => regex-1.4.3}/tests/test_nfa_bytes.rs (100%) rename vendor/{regex => regex-1.4.3}/tests/test_nfa_utf8bytes.rs (100%) rename vendor/{regex => regex-1.4.3}/tests/unicode.rs (100%) rename vendor/{regex => regex-1.4.3}/tests/word_boundary.rs (100%) rename vendor/{regex => regex-1.4.3}/tests/word_boundary_ascii.rs (100%) rename vendor/{regex => regex-1.4.3}/tests/word_boundary_unicode.rs (100%) delete mode 100644 vendor/regex/.cargo-checksum.json delete mode 100644 vendor/regex/Cargo.lock delete mode 100644 vendor/regex/src/pool.rs rename vendor/{semver => semver-0.11.0}/.cargo-checksum.json (100%) rename vendor/{semver => semver-0.11.0}/Cargo.toml (100%) rename vendor/{semver => semver-0.11.0}/LICENSE-APACHE (100%) rename vendor/{semver => semver-0.11.0}/LICENSE-MIT (100%) rename vendor/{semver => semver-0.11.0}/README.md (100%) rename vendor/{semver => semver-0.11.0}/src/diesel_impls.rs (100%) rename vendor/{semver => semver-0.11.0}/src/lib.rs (100%) rename vendor/{semver => semver-0.11.0}/src/version.rs (100%) rename vendor/{semver => semver-0.11.0}/src/version_req.rs (100%) rename vendor/{semver => semver-0.11.0}/tests/deprecation.rs (100%) rename vendor/{semver => semver-0.11.0}/tests/diesel.rs (100%) rename vendor/{semver => semver-0.11.0}/tests/serde.rs (100%) delete mode 100644 vendor/thiserror/tests/test_backtrace.rs delete mode 100644 vendor/thiserror/tests/ui/concat-display.rs delete mode 100644 vendor/thiserror/tests/ui/concat-display.stderr create mode 100644 vendor/thiserror/tests/ui/missing-display.rs create mode 100644 vendor/thiserror/tests/ui/missing-display.stderr mode change 100755 => 100644 vendor/unicode-normalization/src/no_std_prelude.rs create mode 100644 vendor/unicode-xid/benches/xid.rs diff --git a/Cargo.lock b/Cargo.lock index 0939f19cdf..de110c55a4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -55,7 +55,7 @@ dependencies = [ "markup5ever_rcdom", "matches", "tendril", - "url 2.1.1", + "url 2.2.2", ] [[package]] @@ -252,7 +252,7 @@ checksum = "81a18687293a1546b67c246452202bbbf143d239cb43494cc163da14979082da" [[package]] name = "cargo" -version = "0.54.0" +version = "0.55.0" dependencies = [ "anyhow", "atty", @@ -278,6 +278,7 @@ dependencies = [ "humantime 2.0.1", "ignore", "im-rc", + "itertools 0.10.0", "jobserver", "lazy_static", "lazycell", @@ -293,7 +294,7 @@ dependencies = [ "rand 0.8.3", "rustc-workspace-hack", "rustfix", - "semver 0.10.0", + "semver 1.0.3", "serde", "serde_ignored", "serde_json", @@ -305,7 +306,7 @@ dependencies = [ "toml", "unicode-width", "unicode-xid", - "url 2.1.1", + "url 2.2.2", "walkdir", "winapi 0.3.9", ] @@ -378,7 +379,7 @@ dependencies = [ "serde_json", "tar", "toml", - "url 2.1.1", + "url 2.2.2", ] [[package]] @@ -441,9 +442,9 @@ version = "0.1.0" [[package]] name = "cc" -version = "1.0.60" +version = "1.0.68" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef611cc68ff783f18535d77ddd080185275713d852c4f5cbb6122c462a7a825c" +checksum = "4a72c244c1ff497a746a7e1fb3d14bd08420ecda70c8f25c7112f2781652d787" dependencies = [ "jobserver", ] @@ -548,13 +549,13 @@ dependencies = [ [[package]] name = "clippy" -version = "0.1.53" +version = "0.1.54" dependencies = [ "cargo_metadata 0.12.0", - "clippy-mini-macro-test", "clippy_lints", "compiletest_rs", "derive-new", + "filetime", "quote", "regex", "rustc-workspace-hack", @@ -566,10 +567,6 @@ dependencies = [ "tester", ] -[[package]] -name = "clippy-mini-macro-test" -version = "0.2.0" - [[package]] name = "clippy_dev" version = "0.0.1" @@ -585,7 +582,7 @@ dependencies = [ [[package]] name = "clippy_lints" -version = "0.1.53" +version = "0.1.54" dependencies = [ "cargo_metadata 0.12.0", "clippy_utils", @@ -597,14 +594,15 @@ dependencies = [ "rustc-semver", "semver 0.11.0", "serde", + "serde_json", "toml", "unicode-normalization", - "url 2.1.1", + "url 2.2.2", ] [[package]] name = "clippy_utils" -version = "0.1.53" +version = "0.1.54" dependencies = [ "if_chain", "itertools 0.9.0", @@ -654,9 +652,9 @@ dependencies = [ [[package]] name = "compiler_builtins" -version = "0.1.39" +version = "0.1.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3748f82c7d366a0b4950257d19db685d4958d2fa27c6d164a3f069fec42b748b" +checksum = "787187ae221adfcda34b03006f1617099e4ae26b50e5a4db282496014ab75837" dependencies = [ "cc", "rustc-std-workspace-core", @@ -749,7 +747,7 @@ dependencies = [ "percent-encoding 2.1.0", "serde", "serde_json", - "url 2.1.1", + "url 2.2.2", ] [[package]] @@ -864,24 +862,24 @@ dependencies = [ [[package]] name = "curl" -version = "0.4.34" +version = "0.4.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e268162af1a5fe89917ae25ba3b0a77c8da752bdc58e7dbb4f15b91fbd33756e" +checksum = "d0bac9f84ca0977c4d9b8db998689de55b9e976656a6bc87fada2ca710d504c7" dependencies = [ "curl-sys", "libc", "openssl-probe", "openssl-sys", "schannel", - "socket2", + "socket2 0.4.0", "winapi 0.3.9", ] [[package]] name = "curl-sys" -version = "0.4.39+curl-7.74.0" +version = "0.4.42+curl-7.76.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07a8ce861e7b68a0b394e814d7ee9f1b2750ff8bd10372c6ad3bacc10e86f874" +checksum = "4636d8d6109c842707018a104051436bffb8991ea20b2d1293db70b6e0ee4c7c" dependencies = [ "cc", "libc", @@ -1136,13 +1134,13 @@ checksum = "e88a8acf291dafb59c2d96e8f59828f3838bb1a70398823ade51a84de6a6deed" [[package]] name = "filetime" -version = "0.2.12" +version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ed85775dcc68644b5c950ac06a2b23768d3bc9390464151aaf27136998dcf9e" +checksum = "1d34cfa13a63ae058bfa601fe9e313bbdb3746427c1459185464ce0fcf62e1e8" dependencies = [ - "cfg-if 0.1.10", + "cfg-if 1.0.0", "libc", - "redox_syscall 0.1.57", + "redox_syscall", "winapi 0.3.9", ] @@ -1186,6 +1184,16 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" +[[package]] +name = "form_urlencoded" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fc25a87fa4fd2094bffb06925852034d90a17f0d1e05197d4956d3555752191" +dependencies = [ + "matches", + "percent-encoding 2.1.0", +] + [[package]] name = "fortanix-sgx-abi" version = "0.3.3" @@ -1440,7 +1448,7 @@ dependencies = [ "log", "openssl-probe", "openssl-sys", - "url 2.1.1", + "url 2.2.2", ] [[package]] @@ -1452,7 +1460,7 @@ dependencies = [ "curl", "git2", "log", - "url 2.1.1", + "url 2.2.2", ] [[package]] @@ -1704,6 +1712,15 @@ dependencies = [ "either", ] +[[package]] +name = "itertools" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37d572918e350e82412fe766d24b15e6682fb2ed2bbe018280caa810397cb319" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "0.4.6" @@ -1732,11 +1749,10 @@ dependencies = [ "fs-err", "getopts", "jsonpath_lib", - "lazy_static", + "once_cell", "regex", - "serde", "serde_json", - "shlex 0.1.1", + "shlex", ] [[package]] @@ -1974,11 +1990,11 @@ dependencies = [ [[package]] name = "log" -version = "0.4.11" +version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fabed175da42fed1fa0746b0ea71f412aa9d35e76e95e59b192c64b9dc2bf8b" +checksum = "51b9bbe6c47d51fc3e1a9b945965946b4c44142ab8792c50835a980d362c2710" dependencies = [ - "cfg-if 0.1.10", + "cfg-if 1.0.0", ] [[package]] @@ -2002,7 +2018,7 @@ dependencies = [ "serde", "serde_json", "serde_repr", - "url 2.1.1", + "url 2.2.2", ] [[package]] @@ -2117,19 +2133,19 @@ dependencies = [ "serde", "serde_derive", "serde_json", - "shlex 1.0.0", + "shlex", "tempfile", "toml", ] [[package]] name = "measureme" -version = "9.1.0" +version = "9.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a98e07fe802486895addb2b5467f33f205e82c426bfaf350f5d8109b137767c" +checksum = "78f7a41bc6f856a2cf0e95094ad5121f82500e2d9a0f3c0171d98f6566d8117d" dependencies = [ "log", - "memmap", + "memmap2", "parking_lot", "perf-event-open-sys", "rustc-hash", @@ -2138,19 +2154,9 @@ dependencies = [ [[package]] name = "memchr" -version = "2.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3728d817d99e5ac407411fa471ff9800a778d88a24685968b36824eaf4bee400" - -[[package]] -name = "memmap" -version = "0.7.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6585fd95e7bb50d6cc31e20d4cf9afb4e2ba16c5846fc76793f11218da9c475b" -dependencies = [ - "libc", - "winapi 0.3.9", -] +checksum = "b16bd47d9e329435e309c58469fe0791c2d0d1ba96ec0954152a5ae2b04387dc" [[package]] name = "memmap2" @@ -2194,9 +2200,9 @@ dependencies = [ [[package]] name = "minifier" -version = "0.0.39" +version = "0.0.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6cdf618de5c9c98d4a7b2e0d1f1e44f82a19196cfd94040bb203621c25d28d98" +checksum = "5594542d20834f2b974f5e5fb8e0cf1c67a2119dcadc29ef5d93a081fb30cc08" dependencies = [ "macro-utils", ] @@ -2273,7 +2279,7 @@ version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a33c1b55807fbed163481b5ba66db4b2fa6cde694a5027be10fb724206c5897" dependencies = [ - "socket2", + "socket2 0.3.19", "winapi 0.3.9", ] @@ -2288,6 +2294,7 @@ dependencies = [ "hex 0.4.2", "libc", "log", + "measureme", "rand 0.8.3", "rustc-workspace-hack", "rustc_version", @@ -2352,6 +2359,17 @@ dependencies = [ "rustc-std-workspace-core", ] +[[package]] +name = "object" +version = "0.25.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8bc1d42047cf336f0f939c99e97183cf31551bf0f2865a2ec9c8d91fd4ffb5e" +dependencies = [ + "crc32fast", + "indexmap", + "memchr", +] + [[package]] name = "once_cell" version = "1.7.2" @@ -2390,15 +2408,15 @@ dependencies = [ [[package]] name = "openssl" -version = "0.10.30" +version = "0.10.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d575eff3665419f9b83678ff2815858ad9d11567e082f5ac1814baba4e2bcb4" +checksum = "a61075b62a23fef5a29815de7536d940aa35ce96d18ce0cc5076272db678a577" dependencies = [ "bitflags", - "cfg-if 0.1.10", + "cfg-if 1.0.0", "foreign-types", - "lazy_static", "libc", + "once_cell", "openssl-sys", ] @@ -2410,18 +2428,18 @@ checksum = "77af24da69f9d9341038eba93a073b1fdaaa1b788221b00a69bce9e762cb32de" [[package]] name = "openssl-src" -version = "111.12.0+1.1.1h" +version = "111.15.0+1.1.1k" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "858a4132194f8570a7ee9eb8629e85b23cbc4565f2d4a162e87556e5956abf61" +checksum = "b1a5f6ae2ac04393b217ea9f700cd04fa9bf3d93fae2872069f3d15d908af70a" dependencies = [ "cc", ] [[package]] name = "openssl-sys" -version = "0.9.58" +version = "0.9.61" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a842db4709b604f0fe5d1170ae3565899be2ad3d9cbc72dedc789ac0511f78de" +checksum = "313752393519e876837e09e1fa183ddef0be7735868dced3196f4472d536277f" dependencies = [ "autocfg", "cc", @@ -2515,7 +2533,7 @@ dependencies = [ "cfg-if 1.0.0", "instant", "libc", - "redox_syscall 0.2.5", + "redox_syscall", "smallvec", "winapi 0.3.9", ] @@ -2844,9 +2862,9 @@ dependencies = [ [[package]] name = "racer" -version = "2.1.46" +version = "2.1.48" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7cbda48a9124ed2e83766d2c15e3725710d344abca35fad8cf52341a55883b1" +checksum = "7fec2e85e7a30f8fd31b7cf288ad363b5e51fd2cb6f53b416b0cfaabd84e1ccb" dependencies = [ "bitflags", "clap", @@ -2999,12 +3017,6 @@ dependencies = [ "num_cpus", ] -[[package]] -name = "redox_syscall" -version = "0.1.57" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41cc0f7e4d5d4544e8861606a285bb08d3e70712ccc7d2b84d7c0ccfaf4b05ce" - [[package]] name = "redox_syscall" version = "0.2.5" @@ -3021,7 +3033,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "528532f3d801c87aec9def2add9ca802fe569e44a544afe633765267840abe64" dependencies = [ "getrandom 0.2.0", - "redox_syscall 0.2.5", + "redox_syscall", ] [[package]] @@ -3076,7 +3088,7 @@ dependencies = [ "anyhow", "cargo", "cargo-util", - "cargo_metadata 0.8.2", + "cargo_metadata 0.12.0", "clippy_lints", "crossbeam-channel", "difference", @@ -3113,7 +3125,7 @@ dependencies = [ "tokio", "tokio-util", "toml", - "url 2.1.1", + "url 2.2.2", "walkdir", ] @@ -3209,9 +3221,9 @@ dependencies = [ [[package]] name = "rustc-ap-rustc_arena" -version = "718.0.0" +version = "722.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "526610f47139efa440178239553b59ea805ff57a532b4e295c71d2a9b18fd676" +checksum = "550ca1a0925d31a0af089b18c89f5adf3b286e319e3e1f1a5204c21bd2f17371" dependencies = [ "rustc-ap-rustc_data_structures", "smallvec", @@ -3219,9 +3231,9 @@ dependencies = [ [[package]] name = "rustc-ap-rustc_ast" -version = "718.0.0" +version = "722.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf6a9dda0804a7243b0282e3b75a8cf4654c7a61f033e587751941e1fe39391b" +checksum = "4aa53b68080df17994a54747f7c37b0686288a670efb9ba3b382ce62e744aed2" dependencies = [ "bitflags", "rustc-ap-rustc_data_structures", @@ -3236,9 +3248,9 @@ dependencies = [ [[package]] name = "rustc-ap-rustc_ast_pretty" -version = "718.0.0" +version = "722.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82f5019be8b41a58664169fd2f4b1a37fe82705681db394b76419e4e87d40ab1" +checksum = "0ae71e68fada466a4b2c39c79ca6aee3226587abe6787170d2f6c92237569565" dependencies = [ "rustc-ap-rustc_ast", "rustc-ap-rustc_span", @@ -3247,9 +3259,9 @@ dependencies = [ [[package]] name = "rustc-ap-rustc_data_structures" -version = "718.0.0" +version = "722.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a701717fb14549331085756b9741ae3b4bf35808489f1887d72c1d0e0fe52b77" +checksum = "faa484d6e0ca32d1d82303647275c696f745599b3d97e686f396ceef5b99d7ae" dependencies = [ "arrayvec", "bitflags", @@ -3279,9 +3291,9 @@ dependencies = [ [[package]] name = "rustc-ap-rustc_errors" -version = "718.0.0" +version = "722.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3182ce85e8bfc96443475547f2f5aa2b5e67655d9b88721795f36f0ba9e265a" +checksum = "5f85ba19cca320ad797e3a29c35cab9bddfff0e7adbde336a436249e54cee7b1" dependencies = [ "annotate-snippets", "atty", @@ -3299,9 +3311,9 @@ dependencies = [ [[package]] name = "rustc-ap-rustc_feature" -version = "718.0.0" +version = "722.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eed033b93270126ef60963c3ebbd0e026bf53b985172b6366c7b0e7881c9d507" +checksum = "97d538adab96b8b2b1ca9fcd4c8c47d4e23e862a23d1a38b6c15cd8fd52b34b1" dependencies = [ "rustc-ap-rustc_data_structures", "rustc-ap-rustc_span", @@ -3309,21 +3321,21 @@ dependencies = [ [[package]] name = "rustc-ap-rustc_fs_util" -version = "718.0.0" +version = "722.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28ee6531986a205101e09fd143d7bf31897388f33b1814d4bcc45fd62211dca6" +checksum = "8ad6f13d240944fa8f360d2f3b849a7cadaec75e477829e7dde61e838deda83d" [[package]] name = "rustc-ap-rustc_graphviz" -version = "718.0.0" +version = "722.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3398fddc0e23d2db89c036f8952ddf78cadc597f7059752116e69483e164a5b6" +checksum = "08b3451153cc5828c02cc4f1a0df146d25ac4b3382a112e25fd9d3f5bff15cdc" [[package]] name = "rustc-ap-rustc_index" -version = "718.0.0" +version = "722.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dca4e27eb5b701f6bbd47d8fc9d242378fca3e4107a519a28415c2989c4a3bd3" +checksum = "cd39a9f01b442c629bdff5778cb3dd29b7c2ea4afe62d5ab61d216bd1b556692" dependencies = [ "arrayvec", "rustc-ap-rustc_macros", @@ -3332,18 +3344,18 @@ dependencies = [ [[package]] name = "rustc-ap-rustc_lexer" -version = "718.0.0" +version = "722.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "786bbfe9d4d5264294c1819dbf1497a2480b583d5eda1ca9ae22e12d6661f5df" +checksum = "a5de290c44a90e671d2cd730062b9ef73d11155da7e44e7741d633e1e51e616e" dependencies = [ "unicode-xid", ] [[package]] name = "rustc-ap-rustc_lint_defs" -version = "718.0.0" +version = "722.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be2f045e2b999c154ec505d5fea69c994b742f3ebd2f552d11a6c81723921e47" +checksum = "69570b4beb61088926b131579865bbe70d124d30778c46307a62ec8b310ae462" dependencies = [ "rustc-ap-rustc_ast", "rustc-ap-rustc_data_structures", @@ -3356,9 +3368,9 @@ dependencies = [ [[package]] name = "rustc-ap-rustc_macros" -version = "718.0.0" +version = "722.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27789cd26d6b9e2fdfa68a262a20664d79ca67d31a3886d40fb88ebf6935869c" +checksum = "86bd877df37f15c5a44d9679d1b5207ebc95f3943fbc336eeac670195ac58610" dependencies = [ "proc-macro2", "quote", @@ -3368,9 +3380,9 @@ dependencies = [ [[package]] name = "rustc-ap-rustc_parse" -version = "718.0.0" +version = "722.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1dc331f4958350679679e619d63a891e8d5d34ef99087068c89aa9e657d52caa" +checksum = "02502d8522ba31d0bcad28a78822b68c1b6ba947a2b4aa6a2341b30594379b80" dependencies = [ "bitflags", "rustc-ap-rustc_ast", @@ -3388,9 +3400,9 @@ dependencies = [ [[package]] name = "rustc-ap-rustc_serialize" -version = "718.0.0" +version = "722.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9a6824a462c4c1a379e911b0faf86d303a54bcf8673d4cc445195085966a4a4" +checksum = "5f741f8e9aee6323fbe127329490608a5a250cc0072ac91e684ef62518cdb1ff" dependencies = [ "indexmap", "smallvec", @@ -3398,9 +3410,9 @@ dependencies = [ [[package]] name = "rustc-ap-rustc_session" -version = "718.0.0" +version = "722.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a782a5f6ada0dbe089c6416ad0104f0b8a8bdb4bd26ea95e5fefaec67aed5e8a" +checksum = "dba61eca749f4fced4427ad1cc7f23342cfc6527c3bcc624e3aa56abc1f81298" dependencies = [ "bitflags", "getopts", @@ -3420,9 +3432,9 @@ dependencies = [ [[package]] name = "rustc-ap-rustc_span" -version = "718.0.0" +version = "722.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a257546cb264b250c7abdb81239bb02f18a274a966211755a3ea89411b122214" +checksum = "a642e8d6fc883f34e0778e079f8242ac40c6614a6b7a0ef61681333e847f5e62" dependencies = [ "cfg-if 0.1.10", "md-5", @@ -3440,9 +3452,9 @@ dependencies = [ [[package]] name = "rustc-ap-rustc_target" -version = "718.0.0" +version = "722.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5a72dd689421bcb5750f3ed0dedf367076e714ef0ba56c02ed391b9a8582862" +checksum = "80feebd8c323b80dd73a395fa7fabba9e2098b6277670ff89c473f618ffa07de" dependencies = [ "bitflags", "rustc-ap-rustc_data_structures", @@ -3545,7 +3557,7 @@ dependencies = [ "serde_json", "smallvec", "syn", - "url 2.1.1", + "url 2.2.2", "winapi 0.3.9", ] @@ -3681,6 +3693,7 @@ dependencies = [ "rustc_incremental", "rustc_index", "rustc_llvm", + "rustc_metadata", "rustc_middle", "rustc_serialize", "rustc_session", @@ -3700,6 +3713,7 @@ dependencies = [ "itertools 0.9.0", "jobserver", "libc", + "object 0.25.2", "pathdiff", "rustc_apfloat", "rustc_ast", @@ -3880,6 +3894,7 @@ dependencies = [ "rand 0.7.3", "rustc_ast", "rustc_data_structures", + "rustc_errors", "rustc_fs_util", "rustc_graphviz", "rustc_hir", @@ -4676,6 +4691,15 @@ dependencies = [ "serde", ] +[[package]] +name = "semver" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f3aac57ee7f3272d8395c6e4f502f434f0e289fcd62876f70daa008c20dcabe" +dependencies = [ + "serde", +] + [[package]] name = "semver-parser" version = "0.7.0" @@ -4796,12 +4820,6 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "45bb67a18fa91266cc7807181f62f9178a6873bfad7dc788c42e6430db40184f" -[[package]] -name = "shlex" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fdf1b9db47230893d76faad238fd6097fd6d6a9245cd7a4d90dbd639536bbd2" - [[package]] name = "shlex" version = "1.0.0" @@ -4853,13 +4871,22 @@ checksum = "da73c8f77aebc0e40c300b93f0a5f1bece7a248a36eee287d4e095f35c7b7d6e" [[package]] name = "socket2" -version = "0.3.16" +version = "0.3.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fd8b795c389288baa5f355489c65e71fd48a02104600d15c4cfbc561e9e429d" +checksum = "122e570113d28d773067fab24266b66753f6ea915758651696b6e35e49f88d6e" +dependencies = [ + "cfg-if 1.0.0", + "libc", + "winapi 0.3.9", +] + +[[package]] +name = "socket2" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e3dfc207c526015c632472a77be09cf1b6e46866581aecae5cc38fb4235dea2" dependencies = [ - "cfg-if 0.1.10", "libc", - "redox_syscall 0.1.57", "winapi 0.3.9", ] @@ -4897,7 +4924,7 @@ dependencies = [ "hermit-abi", "libc", "miniz_oxide", - "object", + "object 0.22.0", "panic_abort", "panic_unwind", "profiler_builtins", @@ -5026,13 +5053,12 @@ dependencies = [ [[package]] name = "tar" -version = "0.4.29" +version = "0.4.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8a4c1d0bee3230179544336c15eefb563cf0302955d962e456542323e8c2e8a" +checksum = "7d779dc6aeff029314570f666ec83f19df7280bb36ef338442cfa8c604021b80" dependencies = [ "filetime", "libc", - "redox_syscall 0.1.57", "xattr", ] @@ -5045,7 +5071,7 @@ dependencies = [ "cfg-if 1.0.0", "libc", "rand 0.8.3", - "redox_syscall 0.2.5", + "redox_syscall", "remove_dir_all", "winapi 0.3.9", ] @@ -5522,10 +5548,11 @@ dependencies = [ [[package]] name = "url" -version = "2.1.1" +version = "2.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "829d4a8476c35c9bf0bbce5a3b23f4106f79728039b726d292bb93bc106787cb" +checksum = "a507c383b2d33b5fc35d1861e77e6b383d158b2da5e14fe51b83dfedf6fd578c" dependencies = [ + "form_urlencoded", "idna 0.2.0", "matches", "percent-encoding 2.1.0", @@ -5573,9 +5600,9 @@ dependencies = [ [[package]] name = "version_check" -version = "0.9.1" +version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "078775d0255232fb988e6fccf26ddc9d1ac274299aaedcedce21c6f72cc533ce" +checksum = "5fecdca9a5291cc2b8dcf7dc02453fee791a280f3743cb0905f8822ae463b3fe" [[package]] name = "vte" diff --git a/README.md b/README.md index 5ec94e189f..af6a4090a2 100644 --- a/README.md +++ b/README.md @@ -19,8 +19,28 @@ Read ["Installation"] from [The Book]. ## Installing from Source The Rust build system uses a Python script called `x.py` to build the compiler, -which manages the bootstrapping process. More information about it can be found -by running `./x.py --help` or reading the [rustc dev guide][rustcguidebuild]. +which manages the bootstrapping process. It lives in the root of the project. + +The `x.py` command can be run directly on most systems in the following format: + +```sh +./x.py [flags] +``` + +This is how the documentation and examples assume you are running `x.py`. + +Systems such as Ubuntu 20.04 LTS do not create the necessary `python` command by default when Python is installed that allows `x.py` to be run directly. In that case you can either create a symlink for `python` (Ubuntu provides the `python-is-python3` package for this), or run `x.py` using Python itself: + +```sh +# Python 3 +python3 x.py [flags] + +# Python 2.7 +python2.7 x.py [flags] +``` + +More information about `x.py` can be found +by running it with the `--help` flag or reading the [rustc dev guide][rustcguidebuild]. [gettingstarted]: https://rustc-dev-guide.rust-lang.org/getting-started.html [rustcguidebuild]: https://rustc-dev-guide.rust-lang.org/building/how-to-build-and-run.html diff --git a/RELEASES.md b/RELEASES.md index 28b0c5c458..36d6a5276f 100644 --- a/RELEASES.md +++ b/RELEASES.md @@ -1,3 +1,127 @@ +Version 1.54.0 (2021-07-29) +============================ + +Language +----------------------- + +- [You can now use macros for values in built-in attribute macros.][83366] + While a seemingly minor addition on its own, this enables a lot of + powerful functionality when combined correctly. Most notably you can + now include external documentation in your crate by writing the following. + ```rust + #![doc = include_str!("README.md")] + ``` + You can also use this to include auto-generated modules: + ```rust + #[path = concat!(env!("OUT_DIR"), "/generated.rs")] + mod generated; + ``` + +- [You can now cast between unsized slice types (and types which contain + unsized slices) in `const fn`.][85078] +- [You can now use multiple generic lifetimes with `impl Trait` where the + lifetimes don't explicitly outlive another.][84701] In code this means + that you can now have `impl Trait<'a, 'b>` where as before you could + only have `impl Trait<'a, 'b> where 'b: 'a`. + +Compiler +----------------------- + +- [Rustc will now search for custom JSON targets in + `/lib/rustlib//target.json` where `/` is the "sysroot" + directory.][83800] You can find your sysroot directory by running + `rustc --print sysroot`. +- [Added `wasm` as a `target_family` for WebAssembly platforms.][84072] +- [You can now use `#[target_feature]` on safe functions when targeting + WebAssembly platforms.][84988] +- [Improved debugger output for enums on Windows MSVC platforms.][85292] +- [Added tier 3\* support for `bpfel-unknown-none` + and `bpfeb-unknown-none`.][79608] + +\* Refer to Rust's [platform support page][platform-support-doc] for more + information on Rust's tiered platform support. + +Libraries +----------------------- + +- [`panic::panic_any` will now `#[track_caller]`.][85745] +- [Added `OutOfMemory` as a variant of `io::ErrorKind`.][84744] +- [ `proc_macro::Literal` now implements `FromStr`.][84717] +- [The implementations of vendor intrinsics in core::arch have been + significantly refactored.][83278] The main user-visible changes are + a 50% reduction in the size of libcore.rlib and stricter validation + of constant operands passed to intrinsics. The latter is technically + a breaking change, but allows Rust to more closely match the C vendor + intrinsics API. + +Stabilized APIs +--------------- + +- [`BTreeMap::into_keys`] +- [`BTreeMap::into_values`] +- [`HashMap::into_keys`] +- [`HashMap::into_values`] +- [`arch::wasm32`] +- [`VecDeque::binary_search`] +- [`VecDeque::binary_search_by`] +- [`VecDeque::binary_search_by_key`] +- [`VecDeque::partition_point`] + +Cargo +----- + +- [Added the `--prune ` option to `cargo-tree` to remove a package from + the dependency graph.][cargo/9520] +- [Added the `--depth` option to `cargo-tree` to print only to a certain depth + in the tree ][cargo/9499] +- [Added the `no-proc-macro` value to `cargo-tree --edges` to hide procedural + macro dependencies.][cargo/9488] +- [A new environment variable named `CARGO_TARGET_TMPDIR` is available.][cargo/9375] + This variable points to a directory that integration tests and benches + can use as a "scratchpad" for testing filesystem operations. + +Compatibility Notes +------------------- +- [Mixing Option and Result via `?` is no longer permitted in closures for inferred types.][86831] +- [Previously unsound code is no longer permitted where different constructors in branches + could require different lifetimes.][85574] +- As previously mentioned the [`std::arch` instrinsics now uses stricter const checking][83278] + than before and may reject some previously accepted code. +- [`i128` multiplication on Cortex M0+ platforms currently unconditionally causes overflow + when compiled with `codegen-units = 1`.][86063] + +[85574]: https://github.com/rust-lang/rust/issues/85574 +[86831]: https://github.com/rust-lang/rust/issues/86831 +[86063]: https://github.com/rust-lang/rust/issues/86063 +[86831]: https://github.com/rust-lang/rust/issues/86831 +[79608]: https://github.com/rust-lang/rust/pull/79608 +[84988]: https://github.com/rust-lang/rust/pull/84988 +[84701]: https://github.com/rust-lang/rust/pull/84701 +[84072]: https://github.com/rust-lang/rust/pull/84072 +[85745]: https://github.com/rust-lang/rust/pull/85745 +[84744]: https://github.com/rust-lang/rust/pull/84744 +[85078]: https://github.com/rust-lang/rust/pull/85078 +[84717]: https://github.com/rust-lang/rust/pull/84717 +[83800]: https://github.com/rust-lang/rust/pull/83800 +[83366]: https://github.com/rust-lang/rust/pull/83366 +[83278]: https://github.com/rust-lang/rust/pull/83278 +[85292]: https://github.com/rust-lang/rust/pull/85292 +[cargo/9520]: https://github.com/rust-lang/cargo/pull/9520 +[cargo/9499]: https://github.com/rust-lang/cargo/pull/9499 +[cargo/9488]: https://github.com/rust-lang/cargo/pull/9488 +[cargo/9375]: https://github.com/rust-lang/cargo/pull/9375 +[`BTreeMap::into_keys`]: https://doc.rust-lang.org/std/collections/struct.BTreeMap.html#method.into_keys +[`BTreeMap::into_values`]: https://doc.rust-lang.org/std/collections/struct.BTreeMap.html#method.into_values +[`HashMap::into_keys`]: https://doc.rust-lang.org/std/collections/struct.HashMap.html#method.into_keys +[`HashMap::into_values`]: https://doc.rust-lang.org/std/collections/struct.HashMap.html#method.into_values +[`arch::wasm32`]: https://doc.rust-lang.org/core/arch/wasm32/index.html +[`VecDeque::binary_search`]: https://doc.rust-lang.org/std/collections/struct.VecDeque.html#method.binary_search +[`VecDeque::binary_search_by`]: https://doc.rust-lang.org/std/collections/struct.VecDeque.html#method.binary_search_by + +[`VecDeque::binary_search_by_key`]: https://doc.rust-lang.org/std/collections/struct.VecDeque.html#method.binary_search_by_key + +[`VecDeque::partition_point`]: https://doc.rust-lang.org/std/collections/struct.VecDeque.html#method.partition_point + Version 1.53.0 (2021-06-17) ============================ @@ -81,13 +205,6 @@ Stabilised APIs - [`Vec::extend_from_within`] - [`array::from_mut`] - [`array::from_ref`] -- [`char::MAX`] -- [`char::REPLACEMENT_CHARACTER`] -- [`char::UNICODE_VERSION`] -- [`char::decode_utf16`] -- [`char::from_digit`] -- [`char::from_u32_unchecked`] -- [`char::from_u32`] - [`cmp::max_by_key`] - [`cmp::max_by`] - [`cmp::min_by_key`] @@ -116,7 +233,11 @@ Compatibility Notes to rejecting hexadecimal IP addresses.][83652] The octal format can lead to confusion and potential security vulnerabilities and [is no longer recommended][ietf6943]. - +- [The added `BITS` constant may conflict with external definitions.][85667] + In particular, this was known to be a problem in the `lexical-core` crate, + but they have published fixes for semantic versions 0.4 through 0.7. To + update this dependency alone, use `cargo update -p lexical-core`. +- Incremental compilation remains off by default, unless one uses the `RUSTC_FORCE_INCREMENTAL=1` environment variable added in 1.52.1. Internal Only ------------- @@ -129,6 +250,7 @@ related tools. - [rustdoc: Only look at blanket impls in `get_blanket_impls`][83681] - [Rework rustdoc const type][82873] +[85667]: https://github.com/rust-lang/rust/pull/85667 [83386]: https://github.com/rust-lang/rust/pull/83386 [82771]: https://github.com/rust-lang/rust/pull/82771 [84147]: https://github.com/rust-lang/rust/pull/84147 @@ -152,13 +274,6 @@ related tools. [cargo/9298]: https://github.com/rust-lang/cargo/pull/9298 [cargo/9282]: https://github.com/rust-lang/cargo/pull/9282 [cargo/9392]: https://github.com/rust-lang/cargo/pull/9392 -[`char::MAX`]: https://doc.rust-lang.org/std/primitive.char.html#associatedconstant.MAX -[`char::REPLACEMENT_CHARACTER`]: https://doc.rust-lang.org/std/primitive.char.html#associatedconstant.REPLACEMENT_CHARACTER -[`char::UNICODE_VERSION`]: https://doc.rust-lang.org/std/primitive.char.html#associatedconstant.UNICODE_VERSION -[`char::decode_utf16`]: https://doc.rust-lang.org/std/primitive.char.html#method.decode_utf16 -[`char::from_u32`]: https://doc.rust-lang.org/std/primitive.char.html#method.from_u32 -[`char::from_u32_unchecked`]: https://doc.rust-lang.org/std/primitive.char.html#method.from_u32_unchecked -[`char::from_digit`]: https://doc.rust-lang.org/std/primitive.char.html#method.from_digit [`AtomicBool::fetch_update`]: https://doc.rust-lang.org/std/sync/atomic/struct.AtomicBool.html#method.fetch_update [`AtomicPtr::fetch_update`]: https://doc.rust-lang.org/std/sync/atomic/struct.AtomicPtr.html#method.fetch_update [`BTreeMap::retain`]: https://doc.rust-lang.org/std/collections/struct.BTreeMap.html#method.retain @@ -320,6 +435,7 @@ Compatibility Notes - [Rustc now catches more cases of `pub_use_of_private_extern_crate`][80763] - [Changes in how proc macros handle whitespace may lead to panics when used with older `proc-macro-hack` versions. A `cargo update` should be sufficient to fix this in all cases.][84136] +- [Turn `#[derive]` into a regular macro attribute][79078] [84136]: https://github.com/rust-lang/rust/issues/84136 [80763]: https://github.com/rust-lang/rust/pull/80763 @@ -346,6 +462,7 @@ Compatibility Notes [78429]: https://github.com/rust-lang/rust/pull/78429 [82733]: https://github.com/rust-lang/rust/pull/82733 [82594]: https://github.com/rust-lang/rust/pull/82594 +[79078]: https://github.com/rust-lang/rust/pull/79078 [cargo/9181]: https://github.com/rust-lang/cargo/pull/9181 [`char::MAX`]: https://doc.rust-lang.org/std/primitive.char.html#associatedconstant.MAX [`char::REPLACEMENT_CHARACTER`]: https://doc.rust-lang.org/std/primitive.char.html#associatedconstant.REPLACEMENT_CHARACTER @@ -1756,6 +1873,7 @@ Language - [You can now use `#[repr(transparent)]` on univariant `enum`s.][68122] Meaning that you can create an enum that has the exact layout and ABI of the type it contains. +- [You can now use outer attribute procedural macros on inline modules.][64273] - [There are some *syntax-only* changes:][67131] - `default` is syntactically allowed before items in `trait` definitions. - Items in `impl`s (i.e. `const`s, `type`s, and `fn`s) may syntactically @@ -1817,6 +1935,7 @@ Compatibility Notes [67935]: https://github.com/rust-lang/rust/pull/67935/ [68339]: https://github.com/rust-lang/rust/pull/68339/ [68122]: https://github.com/rust-lang/rust/pull/68122/ +[64273]: https://github.com/rust-lang/rust/pull/64273/ [67712]: https://github.com/rust-lang/rust/pull/67712/ [67887]: https://github.com/rust-lang/rust/pull/67887/ [67131]: https://github.com/rust-lang/rust/pull/67131/ diff --git a/compiler/rustc_apfloat/src/lib.rs b/compiler/rustc_apfloat/src/lib.rs index c648147d10..7eeec4aa86 100644 --- a/compiler/rustc_apfloat/src/lib.rs +++ b/compiler/rustc_apfloat/src/lib.rs @@ -35,7 +35,6 @@ #![forbid(unsafe_code)] #![feature(iter_zip)] #![feature(nll)] -#![cfg_attr(bootstrap, feature(or_patterns))] #[macro_use] extern crate alloc; diff --git a/compiler/rustc_ast/src/ast.rs b/compiler/rustc_ast/src/ast.rs index e7f19f06eb..93d7a59768 100644 --- a/compiler/rustc_ast/src/ast.rs +++ b/compiler/rustc_ast/src/ast.rs @@ -278,7 +278,7 @@ impl ParenthesizedArgs { .cloned() .map(|input| AngleBracketedArg::Arg(GenericArg::Type(input))) .collect(); - AngleBracketedArgs { span: self.span, args } + AngleBracketedArgs { span: self.inputs_span, args } } } @@ -623,12 +623,13 @@ impl Pat { PatKind::Ident(_, _, Some(p)) => p.walk(it), // Walk into each field of struct. - PatKind::Struct(_, fields, _) => fields.iter().for_each(|field| field.pat.walk(it)), + PatKind::Struct(_, _, fields, _) => fields.iter().for_each(|field| field.pat.walk(it)), // Sequence of patterns. - PatKind::TupleStruct(_, s) | PatKind::Tuple(s) | PatKind::Slice(s) | PatKind::Or(s) => { - s.iter().for_each(|p| p.walk(it)) - } + PatKind::TupleStruct(_, _, s) + | PatKind::Tuple(s) + | PatKind::Slice(s) + | PatKind::Or(s) => s.iter().for_each(|p| p.walk(it)), // Trivial wrappers over inner patterns. PatKind::Box(s) | PatKind::Ref(s, _) | PatKind::Paren(s) => s.walk(it), @@ -701,10 +702,10 @@ pub enum PatKind { /// A struct or struct variant pattern (e.g., `Variant {x, y, ..}`). /// The `bool` is `true` in the presence of a `..`. - Struct(Path, Vec, /* recovered */ bool), + Struct(Option, Path, Vec, /* recovered */ bool), /// A tuple struct/variant pattern (`Variant(x, y, .., z)`). - TupleStruct(Path, Vec>), + TupleStruct(Option, Path, Vec>), /// An or-pattern `A | B | C`. /// Invariant: `pats.len() >= 2`. @@ -1247,6 +1248,7 @@ pub enum StructRest { #[derive(Clone, Encodable, Decodable, Debug)] pub struct StructExpr { + pub qself: Option, pub path: Path, pub fields: Vec, pub rest: StructRest, @@ -1861,6 +1863,10 @@ pub enum TyKind { Never, /// A tuple (`(A, B, C, D,...)`). Tup(Vec>), + /// An anonymous struct type i.e. `struct { foo: Type }` + AnonymousStruct(Vec, bool), + /// An anonymous union type i.e. `union { bar: Type }` + AnonymousUnion(Vec, bool), /// A path (`module::module::...::Type`), optionally /// "qualified", e.g., ` as SomeTrait>::SomeType`. /// @@ -2279,14 +2285,6 @@ pub struct ForeignMod { pub items: Vec>, } -/// Global inline assembly. -/// -/// Also known as "module-level assembly" or "file-scoped assembly". -#[derive(Clone, Encodable, Decodable, Debug, Copy)] -pub struct GlobalAsm { - pub asm: Symbol, -} - #[derive(Clone, Encodable, Decodable, Debug)] pub struct EnumDef { pub variants: Vec, @@ -2669,7 +2667,7 @@ pub enum ItemKind { /// E.g., `extern {}` or `extern "C" {}`. ForeignMod(ForeignMod), /// Module-level inline assembly (from `global_asm!()`). - GlobalAsm(GlobalAsm), + GlobalAsm(InlineAsm), /// A type alias (`type`). /// /// E.g., `type Foo = Bar;`. diff --git a/compiler/rustc_ast/src/ast_like.rs b/compiler/rustc_ast/src/ast_like.rs index 945a44ab66..d586426d70 100644 --- a/compiler/rustc_ast/src/ast_like.rs +++ b/compiler/rustc_ast/src/ast_like.rs @@ -82,7 +82,8 @@ impl AstLike for crate::token::Nonterminal { Nonterminal::NtMeta(attr_item) => attr_item.tokens_mut(), Nonterminal::NtPath(path) => path.tokens_mut(), Nonterminal::NtVis(vis) => vis.tokens_mut(), - _ => panic!("Called tokens_mut on {:?}", self), + Nonterminal::NtBlock(block) => block.tokens_mut(), + Nonterminal::NtIdent(..) | Nonterminal::NtLifetime(..) | Nonterminal::NtTT(..) => None, } } } diff --git a/compiler/rustc_ast/src/lib.rs b/compiler/rustc_ast/src/lib.rs index ffec28a395..7c79b4aab3 100644 --- a/compiler/rustc_ast/src/lib.rs +++ b/compiler/rustc_ast/src/lib.rs @@ -10,15 +10,13 @@ )] #![feature(box_syntax)] #![feature(box_patterns)] -#![cfg_attr(bootstrap, feature(const_fn))] // For the `transmute` in `P::new` -#![cfg_attr(not(bootstrap), feature(const_fn_unsize))] // For the `transmute` in `P::new` +#![cfg_attr(bootstrap, feature(const_fn_unsize))] #![feature(const_fn_transmute)] -#![feature(const_panic)] #![feature(crate_visibility_modifier)] #![feature(iter_zip)] #![feature(label_break_value)] #![feature(nll)] -#![cfg_attr(bootstrap, feature(or_patterns))] +#![feature(min_specialization)] #![recursion_limit = "256"] #[macro_use] diff --git a/compiler/rustc_ast/src/mut_visit.rs b/compiler/rustc_ast/src/mut_visit.rs index 05f57f978c..0b6099fd33 100644 --- a/compiler/rustc_ast/src/mut_visit.rs +++ b/compiler/rustc_ast/src/mut_visit.rs @@ -484,6 +484,9 @@ pub fn noop_visit_ty(ty: &mut P, vis: &mut T) { visit_vec(bounds, |bound| vis.visit_param_bound(bound)); } TyKind::MacCall(mac) => vis.visit_mac_call(mac), + TyKind::AnonymousStruct(fields, ..) | TyKind::AnonymousUnion(fields, ..) => { + fields.flat_map_in_place(|field| vis.flat_map_field_def(field)); + } } vis.visit_span(span); visit_lazy_tts(tokens, vis); @@ -965,7 +968,7 @@ pub fn noop_visit_item_kind(kind: &mut ItemKind, vis: &mut T) { ModKind::Unloaded => {} }, ItemKind::ForeignMod(nm) => vis.visit_foreign_mod(nm), - ItemKind::GlobalAsm(_ga) => {} + ItemKind::GlobalAsm(asm) => noop_visit_inline_asm(asm, vis), ItemKind::TyAlias(box TyAliasKind(_, generics, bounds, ty)) => { vis.visit_generics(generics); visit_bounds(bounds, vis); @@ -1136,7 +1139,8 @@ pub fn noop_visit_pat(pat: &mut P, vis: &mut T) { visit_opt(sub, |sub| vis.visit_pat(sub)); } PatKind::Lit(e) => vis.visit_expr(e), - PatKind::TupleStruct(path, elems) => { + PatKind::TupleStruct(qself, path, elems) => { + vis.visit_qself(qself); vis.visit_path(path); visit_vec(elems, |elem| vis.visit_pat(elem)); } @@ -1144,7 +1148,8 @@ pub fn noop_visit_pat(pat: &mut P, vis: &mut T) { vis.visit_qself(qself); vis.visit_path(path); } - PatKind::Struct(path, fields, _etc) => { + PatKind::Struct(qself, path, fields, _etc) => { + vis.visit_qself(qself); vis.visit_path(path); fields.flat_map_in_place(|field| vis.flat_map_pat_field(field)); } @@ -1170,6 +1175,28 @@ pub fn noop_visit_anon_const(AnonConst { id, value }: &mut AnonCo vis.visit_expr(value); } +fn noop_visit_inline_asm(asm: &mut InlineAsm, vis: &mut T) { + for (op, _) in &mut asm.operands { + match op { + InlineAsmOperand::In { expr, .. } + | InlineAsmOperand::InOut { expr, .. } + | InlineAsmOperand::Sym { expr, .. } => vis.visit_expr(expr), + InlineAsmOperand::Out { expr, .. } => { + if let Some(expr) = expr { + vis.visit_expr(expr); + } + } + InlineAsmOperand::SplitInOut { in_expr, out_expr, .. } => { + vis.visit_expr(in_expr); + if let Some(out_expr) = out_expr { + vis.visit_expr(out_expr); + } + } + InlineAsmOperand::Const { anon_const, .. } => vis.visit_anon_const(anon_const), + } + } +} + pub fn noop_visit_expr( Expr { kind, id, span, attrs, tokens }: &mut Expr, vis: &mut T, @@ -1288,27 +1315,7 @@ pub fn noop_visit_expr( ExprKind::Ret(expr) => { visit_opt(expr, |expr| vis.visit_expr(expr)); } - ExprKind::InlineAsm(asm) => { - for (op, _) in &mut asm.operands { - match op { - InlineAsmOperand::In { expr, .. } - | InlineAsmOperand::InOut { expr, .. } - | InlineAsmOperand::Sym { expr, .. } => vis.visit_expr(expr), - InlineAsmOperand::Out { expr, .. } => { - if let Some(expr) = expr { - vis.visit_expr(expr); - } - } - InlineAsmOperand::SplitInOut { in_expr, out_expr, .. } => { - vis.visit_expr(in_expr); - if let Some(out_expr) = out_expr { - vis.visit_expr(out_expr); - } - } - InlineAsmOperand::Const { anon_const, .. } => vis.visit_anon_const(anon_const), - } - } - } + ExprKind::InlineAsm(asm) => noop_visit_inline_asm(asm, vis), ExprKind::LlvmInlineAsm(asm) => { let LlvmInlineAsm { asm: _, @@ -1328,7 +1335,8 @@ pub fn noop_visit_expr( } ExprKind::MacCall(mac) => vis.visit_mac_call(mac), ExprKind::Struct(se) => { - let StructExpr { path, fields, rest } = se.deref_mut(); + let StructExpr { qself, path, fields, rest } = se.deref_mut(); + vis.visit_qself(qself); vis.visit_path(path); fields.flat_map_in_place(|field| vis.flat_map_expr_field(field)); match rest { diff --git a/compiler/rustc_ast/src/tokenstream.rs b/compiler/rustc_ast/src/tokenstream.rs index 2d463a4588..5d994dbad4 100644 --- a/compiler/rustc_ast/src/tokenstream.rs +++ b/compiler/rustc_ast/src/tokenstream.rs @@ -218,8 +218,7 @@ impl AttrAnnotatedTokenStream { AttrAnnotatedTokenTree::Attributes(data) => { let mut outer_attrs = Vec::new(); let mut inner_attrs = Vec::new(); - let attrs: Vec<_> = data.attrs.clone().into(); - for attr in attrs { + for attr in &data.attrs { match attr.style { crate::AttrStyle::Outer => { assert!( @@ -264,7 +263,7 @@ impl AttrAnnotatedTokenStream { // so we never reach this code. let mut builder = TokenStreamBuilder::new(); - for inner_attr in &inner_attrs { + for inner_attr in inner_attrs { builder.push(inner_attr.tokens().to_tokenstream()); } builder.push(delim_tokens.clone()); diff --git a/compiler/rustc_ast/src/visit.rs b/compiler/rustc_ast/src/visit.rs index 3f35919ae6..1ebfcf3671 100644 --- a/compiler/rustc_ast/src/visit.rs +++ b/compiler/rustc_ast/src/visit.rs @@ -90,9 +90,6 @@ pub trait Visitor<'ast>: Sized { fn visit_foreign_item(&mut self, i: &'ast ForeignItem) { walk_foreign_item(self, i) } - fn visit_global_asm(&mut self, ga: &'ast GlobalAsm) { - walk_global_asm(self, ga) - } fn visit_item(&mut self, i: &'ast Item) { walk_item(self, i) } @@ -299,7 +296,7 @@ pub fn walk_item<'a, V: Visitor<'a>>(visitor: &mut V, item: &'a Item) { ItemKind::ForeignMod(ref foreign_module) => { walk_list!(visitor, visit_foreign_item, &foreign_module.items); } - ItemKind::GlobalAsm(ref ga) => visitor.visit_global_asm(ga), + ItemKind::GlobalAsm(ref asm) => walk_inline_asm(visitor, asm), ItemKind::TyAlias(box TyAliasKind(_, ref generics, ref bounds, ref ty)) => { visitor.visit_generics(generics); walk_list!(visitor, visit_param_bound, bounds); @@ -407,6 +404,9 @@ pub fn walk_ty<'a, V: Visitor<'a>>(visitor: &mut V, typ: &'a Ty) { TyKind::Typeof(ref expression) => visitor.visit_anon_const(expression), TyKind::Infer | TyKind::ImplicitSelf | TyKind::Err => {} TyKind::MacCall(ref mac) => visitor.visit_mac_call(mac), + TyKind::AnonymousStruct(ref fields, ..) | TyKind::AnonymousUnion(ref fields, ..) => { + walk_list!(visitor, visit_field_def, fields) + } TyKind::Never | TyKind::CVarArgs => {} } } @@ -497,7 +497,10 @@ pub fn walk_assoc_ty_constraint<'a, V: Visitor<'a>>( pub fn walk_pat<'a, V: Visitor<'a>>(visitor: &mut V, pattern: &'a Pat) { match pattern.kind { - PatKind::TupleStruct(ref path, ref elems) => { + PatKind::TupleStruct(ref opt_qself, ref path, ref elems) => { + if let Some(ref qself) = *opt_qself { + visitor.visit_ty(&qself.ty); + } visitor.visit_path(path, pattern.id); walk_list!(visitor, visit_pat, elems); } @@ -507,7 +510,10 @@ pub fn walk_pat<'a, V: Visitor<'a>>(visitor: &mut V, pattern: &'a Pat) { } visitor.visit_path(path, pattern.id) } - PatKind::Struct(ref path, ref fields, _) => { + PatKind::Struct(ref opt_qself, ref path, ref fields, _) => { + if let Some(ref qself) = *opt_qself { + visitor.visit_ty(&qself.ty); + } visitor.visit_path(path, pattern.id); walk_list!(visitor, visit_pat_field, fields); } @@ -557,10 +563,6 @@ pub fn walk_foreign_item<'a, V: Visitor<'a>>(visitor: &mut V, item: &'a ForeignI } } -pub fn walk_global_asm<'a, V: Visitor<'a>>(_: &mut V, _: &'a GlobalAsm) { - // Empty! -} - pub fn walk_param_bound<'a, V: Visitor<'a>>(visitor: &mut V, bound: &'a GenericBound) { match *bound { GenericBound::Trait(ref typ, ref modifier) => visitor.visit_poly_trait_ref(typ, modifier), @@ -708,6 +710,28 @@ pub fn walk_anon_const<'a, V: Visitor<'a>>(visitor: &mut V, constant: &'a AnonCo visitor.visit_expr(&constant.value); } +fn walk_inline_asm<'a, V: Visitor<'a>>(visitor: &mut V, asm: &'a InlineAsm) { + for (op, _) in &asm.operands { + match op { + InlineAsmOperand::In { expr, .. } + | InlineAsmOperand::InOut { expr, .. } + | InlineAsmOperand::Sym { expr, .. } => visitor.visit_expr(expr), + InlineAsmOperand::Out { expr, .. } => { + if let Some(expr) = expr { + visitor.visit_expr(expr); + } + } + InlineAsmOperand::SplitInOut { in_expr, out_expr, .. } => { + visitor.visit_expr(in_expr); + if let Some(out_expr) = out_expr { + visitor.visit_expr(out_expr); + } + } + InlineAsmOperand::Const { anon_const, .. } => visitor.visit_anon_const(anon_const), + } + } +} + pub fn walk_expr<'a, V: Visitor<'a>>(visitor: &mut V, expression: &'a Expr) { walk_list!(visitor, visit_attribute, expression.attrs.iter()); @@ -722,6 +746,9 @@ pub fn walk_expr<'a, V: Visitor<'a>>(visitor: &mut V, expression: &'a Expr) { visitor.visit_anon_const(count) } ExprKind::Struct(ref se) => { + if let Some(ref qself) = se.qself { + visitor.visit_ty(&qself.ty); + } visitor.visit_path(&se.path, expression.id); walk_list!(visitor, visit_expr_field, &se.fields); match &se.rest { @@ -830,29 +857,7 @@ pub fn walk_expr<'a, V: Visitor<'a>>(visitor: &mut V, expression: &'a Expr) { } ExprKind::MacCall(ref mac) => visitor.visit_mac_call(mac), ExprKind::Paren(ref subexpression) => visitor.visit_expr(subexpression), - ExprKind::InlineAsm(ref ia) => { - for (op, _) in &ia.operands { - match op { - InlineAsmOperand::In { expr, .. } - | InlineAsmOperand::InOut { expr, .. } - | InlineAsmOperand::Sym { expr, .. } => visitor.visit_expr(expr), - InlineAsmOperand::Out { expr, .. } => { - if let Some(expr) = expr { - visitor.visit_expr(expr); - } - } - InlineAsmOperand::SplitInOut { in_expr, out_expr, .. } => { - visitor.visit_expr(in_expr); - if let Some(out_expr) = out_expr { - visitor.visit_expr(out_expr); - } - } - InlineAsmOperand::Const { anon_const, .. } => { - visitor.visit_anon_const(anon_const) - } - } - } - } + ExprKind::InlineAsm(ref asm) => walk_inline_asm(visitor, asm), ExprKind::LlvmInlineAsm(ref ia) => { for &(_, ref input) in &ia.inputs { visitor.visit_expr(input) diff --git a/compiler/rustc_ast_lowering/src/asm.rs b/compiler/rustc_ast_lowering/src/asm.rs new file mode 100644 index 0000000000..1c3fae2afe --- /dev/null +++ b/compiler/rustc_ast_lowering/src/asm.rs @@ -0,0 +1,329 @@ +use super::LoweringContext; + +use rustc_ast::*; +use rustc_data_structures::fx::FxHashMap; +use rustc_errors::struct_span_err; +use rustc_hir as hir; +use rustc_span::{Span, Symbol}; +use rustc_target::asm; +use std::collections::hash_map::Entry; +use std::fmt::Write; + +impl<'a, 'hir> LoweringContext<'a, 'hir> { + crate fn lower_inline_asm(&mut self, sp: Span, asm: &InlineAsm) -> &'hir hir::InlineAsm<'hir> { + // Rustdoc needs to support asm! from foriegn architectures: don't try + // lowering the register contraints in this case. + let asm_arch = if self.sess.opts.actually_rustdoc { None } else { self.sess.asm_arch }; + if asm_arch.is_none() && !self.sess.opts.actually_rustdoc { + struct_span_err!(self.sess, sp, E0472, "inline assembly is unsupported on this target") + .emit(); + } + if asm.options.contains(InlineAsmOptions::ATT_SYNTAX) + && !matches!(asm_arch, Some(asm::InlineAsmArch::X86 | asm::InlineAsmArch::X86_64)) + && !self.sess.opts.actually_rustdoc + { + self.sess + .struct_span_err(sp, "the `att_syntax` option is only supported on x86") + .emit(); + } + + // Lower operands to HIR. We use dummy register classes if an error + // occurs during lowering because we still need to be able to produce a + // valid HIR. + let sess = self.sess; + let operands: Vec<_> = asm + .operands + .iter() + .map(|(op, op_sp)| { + let lower_reg = |reg| match reg { + InlineAsmRegOrRegClass::Reg(s) => { + asm::InlineAsmRegOrRegClass::Reg(if let Some(asm_arch) = asm_arch { + asm::InlineAsmReg::parse( + asm_arch, + |feature| sess.target_features.contains(&Symbol::intern(feature)), + &sess.target, + s, + ) + .unwrap_or_else(|e| { + let msg = format!("invalid register `{}`: {}", s.as_str(), e); + sess.struct_span_err(*op_sp, &msg).emit(); + asm::InlineAsmReg::Err + }) + } else { + asm::InlineAsmReg::Err + }) + } + InlineAsmRegOrRegClass::RegClass(s) => { + asm::InlineAsmRegOrRegClass::RegClass(if let Some(asm_arch) = asm_arch { + asm::InlineAsmRegClass::parse(asm_arch, s).unwrap_or_else(|e| { + let msg = format!("invalid register class `{}`: {}", s.as_str(), e); + sess.struct_span_err(*op_sp, &msg).emit(); + asm::InlineAsmRegClass::Err + }) + } else { + asm::InlineAsmRegClass::Err + }) + } + }; + + let op = match *op { + InlineAsmOperand::In { reg, ref expr } => hir::InlineAsmOperand::In { + reg: lower_reg(reg), + expr: self.lower_expr_mut(expr), + }, + InlineAsmOperand::Out { reg, late, ref expr } => hir::InlineAsmOperand::Out { + reg: lower_reg(reg), + late, + expr: expr.as_ref().map(|expr| self.lower_expr_mut(expr)), + }, + InlineAsmOperand::InOut { reg, late, ref expr } => { + hir::InlineAsmOperand::InOut { + reg: lower_reg(reg), + late, + expr: self.lower_expr_mut(expr), + } + } + InlineAsmOperand::SplitInOut { reg, late, ref in_expr, ref out_expr } => { + hir::InlineAsmOperand::SplitInOut { + reg: lower_reg(reg), + late, + in_expr: self.lower_expr_mut(in_expr), + out_expr: out_expr.as_ref().map(|expr| self.lower_expr_mut(expr)), + } + } + InlineAsmOperand::Const { ref anon_const } => hir::InlineAsmOperand::Const { + anon_const: self.lower_anon_const(anon_const), + }, + InlineAsmOperand::Sym { ref expr } => { + hir::InlineAsmOperand::Sym { expr: self.lower_expr_mut(expr) } + } + }; + (op, *op_sp) + }) + .collect(); + + // Validate template modifiers against the register classes for the operands + for p in &asm.template { + if let InlineAsmTemplatePiece::Placeholder { + operand_idx, + modifier: Some(modifier), + span: placeholder_span, + } = *p + { + let op_sp = asm.operands[operand_idx].1; + match &operands[operand_idx].0 { + hir::InlineAsmOperand::In { reg, .. } + | hir::InlineAsmOperand::Out { reg, .. } + | hir::InlineAsmOperand::InOut { reg, .. } + | hir::InlineAsmOperand::SplitInOut { reg, .. } => { + let class = reg.reg_class(); + if class == asm::InlineAsmRegClass::Err { + continue; + } + let valid_modifiers = class.valid_modifiers(asm_arch.unwrap()); + if !valid_modifiers.contains(&modifier) { + let mut err = sess.struct_span_err( + placeholder_span, + "invalid asm template modifier for this register class", + ); + err.span_label(placeholder_span, "template modifier"); + err.span_label(op_sp, "argument"); + if !valid_modifiers.is_empty() { + let mut mods = format!("`{}`", valid_modifiers[0]); + for m in &valid_modifiers[1..] { + let _ = write!(mods, ", `{}`", m); + } + err.note(&format!( + "the `{}` register class supports \ + the following template modifiers: {}", + class.name(), + mods + )); + } else { + err.note(&format!( + "the `{}` register class does not support template modifiers", + class.name() + )); + } + err.emit(); + } + } + hir::InlineAsmOperand::Const { .. } => { + let mut err = sess.struct_span_err( + placeholder_span, + "asm template modifiers are not allowed for `const` arguments", + ); + err.span_label(placeholder_span, "template modifier"); + err.span_label(op_sp, "argument"); + err.emit(); + } + hir::InlineAsmOperand::Sym { .. } => { + let mut err = sess.struct_span_err( + placeholder_span, + "asm template modifiers are not allowed for `sym` arguments", + ); + err.span_label(placeholder_span, "template modifier"); + err.span_label(op_sp, "argument"); + err.emit(); + } + } + } + } + + let mut used_input_regs = FxHashMap::default(); + let mut used_output_regs = FxHashMap::default(); + let mut required_features: Vec<&str> = vec![]; + for (idx, &(ref op, op_sp)) in operands.iter().enumerate() { + if let Some(reg) = op.reg() { + // Make sure we don't accidentally carry features from the + // previous iteration. + required_features.clear(); + + let reg_class = reg.reg_class(); + if reg_class == asm::InlineAsmRegClass::Err { + continue; + } + + // We ignore target feature requirements for clobbers: if the + // feature is disabled then the compiler doesn't care what we + // do with the registers. + // + // Note that this is only possible for explicit register + // operands, which cannot be used in the asm string. + let is_clobber = matches!( + op, + hir::InlineAsmOperand::Out { + reg: asm::InlineAsmRegOrRegClass::Reg(_), + late: _, + expr: None + } + ); + + if !is_clobber { + // Validate register classes against currently enabled target + // features. We check that at least one type is available for + // the current target. + for &(_, feature) in reg_class.supported_types(asm_arch.unwrap()) { + if let Some(feature) = feature { + if self.sess.target_features.contains(&Symbol::intern(feature)) { + required_features.clear(); + break; + } else { + required_features.push(feature); + } + } else { + required_features.clear(); + break; + } + } + // We are sorting primitive strs here and can use unstable sort here + required_features.sort_unstable(); + required_features.dedup(); + match &required_features[..] { + [] => {} + [feature] => { + let msg = format!( + "register class `{}` requires the `{}` target feature", + reg_class.name(), + feature + ); + sess.struct_span_err(op_sp, &msg).emit(); + } + features => { + let msg = format!( + "register class `{}` requires at least one target feature: {}", + reg_class.name(), + features.join(", ") + ); + sess.struct_span_err(op_sp, &msg).emit(); + } + } + } + + // Check for conflicts between explicit register operands. + if let asm::InlineAsmRegOrRegClass::Reg(reg) = reg { + let (input, output) = match op { + hir::InlineAsmOperand::In { .. } => (true, false), + + // Late output do not conflict with inputs, but normal outputs do + hir::InlineAsmOperand::Out { late, .. } => (!late, true), + + hir::InlineAsmOperand::InOut { .. } + | hir::InlineAsmOperand::SplitInOut { .. } => (true, true), + + hir::InlineAsmOperand::Const { .. } | hir::InlineAsmOperand::Sym { .. } => { + unreachable!() + } + }; + + // Flag to output the error only once per operand + let mut skip = false; + reg.overlapping_regs(|r| { + let mut check = |used_regs: &mut FxHashMap, + input| { + match used_regs.entry(r) { + Entry::Occupied(o) => { + if skip { + return; + } + skip = true; + + let idx2 = *o.get(); + let &(ref op2, op_sp2) = &operands[idx2]; + let reg2 = match op2.reg() { + Some(asm::InlineAsmRegOrRegClass::Reg(r)) => r, + _ => unreachable!(), + }; + + let msg = format!( + "register `{}` conflicts with register `{}`", + reg.name(), + reg2.name() + ); + let mut err = sess.struct_span_err(op_sp, &msg); + err.span_label(op_sp, &format!("register `{}`", reg.name())); + err.span_label(op_sp2, &format!("register `{}`", reg2.name())); + + match (op, op2) { + ( + hir::InlineAsmOperand::In { .. }, + hir::InlineAsmOperand::Out { late, .. }, + ) + | ( + hir::InlineAsmOperand::Out { late, .. }, + hir::InlineAsmOperand::In { .. }, + ) => { + assert!(!*late); + let out_op_sp = if input { op_sp2 } else { op_sp }; + let msg = "use `lateout` instead of \ + `out` to avoid conflict"; + err.span_help(out_op_sp, msg); + } + _ => {} + } + + err.emit(); + } + Entry::Vacant(v) => { + v.insert(idx); + } + } + }; + if input { + check(&mut used_input_regs, true); + } + if output { + check(&mut used_output_regs, false); + } + }); + } + } + } + + let operands = self.arena.alloc_from_iter(operands); + let template = self.arena.alloc_from_iter(asm.template.iter().cloned()); + let line_spans = self.arena.alloc_slice(&asm.line_spans[..]); + let hir_asm = hir::InlineAsm { template, operands, options: asm.options, line_spans }; + self.arena.alloc(hir_asm) + } +} diff --git a/compiler/rustc_ast_lowering/src/expr.rs b/compiler/rustc_ast_lowering/src/expr.rs index 75dfe951c9..b9dcd083c0 100644 --- a/compiler/rustc_ast_lowering/src/expr.rs +++ b/compiler/rustc_ast_lowering/src/expr.rs @@ -3,7 +3,6 @@ use super::{ImplTraitContext, LoweringContext, ParamMode, ParenthesizedGenericAr use rustc_ast::attr; use rustc_ast::ptr::P as AstP; use rustc_ast::*; -use rustc_data_structures::fx::FxHashMap; use rustc_data_structures::stack::ensure_sufficient_stack; use rustc_data_structures::thin_vec::ThinVec; use rustc_errors::struct_span_err; @@ -15,9 +14,6 @@ use rustc_span::hygiene::ExpnId; use rustc_span::source_map::{respan, DesugaringKind, Span, Spanned}; use rustc_span::symbol::{sym, Ident, Symbol}; use rustc_span::{hygiene::ForLoopLoc, DUMMY_SP}; -use rustc_target::asm; -use std::collections::hash_map::Entry; -use std::fmt::Write; impl<'hir> LoweringContext<'_, 'hir> { fn lower_exprs(&mut self, exprs: &[AstP]) -> &'hir [hir::Expr<'hir>] { @@ -222,7 +218,9 @@ impl<'hir> LoweringContext<'_, 'hir> { let e = e.as_ref().map(|x| self.lower_expr(x)); hir::ExprKind::Ret(e) } - ExprKind::InlineAsm(ref asm) => self.lower_expr_asm(e.span, asm), + ExprKind::InlineAsm(ref asm) => { + hir::ExprKind::InlineAsm(self.lower_inline_asm(e.span, asm)) + } ExprKind::LlvmInlineAsm(ref asm) => self.lower_expr_llvm_asm(asm), ExprKind::Struct(ref se) => { let rest = match &se.rest { @@ -239,7 +237,7 @@ impl<'hir> LoweringContext<'_, 'hir> { hir::ExprKind::Struct( self.arena.alloc(self.lower_qpath( e.id, - &None, + &se.qself, &se.path, ParamMode::Optional, ImplTraitContext::disallowed(), @@ -338,7 +336,7 @@ impl<'hir> LoweringContext<'_, 'hir> { let mut generic_args = vec![]; for (idx, arg) in args.into_iter().enumerate() { if legacy_args_idx.contains(&idx) { - let parent_def_id = self.current_hir_id_owner.last().unwrap().0; + let parent_def_id = self.current_hir_id_owner.0; let node_id = self.resolver.next_node_id(); // Add a definition for the in-band const def. @@ -562,8 +560,8 @@ impl<'hir> LoweringContext<'_, 'hir> { ) } - /// Desugar `try { ; }` into `{ ; ::std::ops::Try::from_ok() }`, - /// `try { ; }` into `{ ; ::std::ops::Try::from_ok(()) }` + /// Desugar `try { ; }` into `{ ; ::std::ops::Try::from_output() }`, + /// `try { ; }` into `{ ; ::std::ops::Try::from_output(()) }` /// and save the block id to use it as a break target for desugaring of the `?` operator. fn lower_expr_try_block(&mut self, body: &Block) -> hir::ExprKind<'hir> { self.with_catch_scope(body.id, |this| { @@ -592,9 +590,9 @@ impl<'hir> LoweringContext<'_, 'hir> { let ok_wrapped_span = this.mark_span_with_reason(DesugaringKind::TryBlock, tail_expr.span, None); - // `::std::ops::Try::from_ok($tail_expr)` + // `::std::ops::Try::from_output($tail_expr)` block.expr = Some(this.wrap_in_try_constructor( - hir::LangItem::TryFromOk, + hir::LangItem::TryTraitFromOutput, try_span, tail_expr, ok_wrapped_span, @@ -1043,10 +1041,12 @@ impl<'hir> LoweringContext<'_, 'hir> { /// It is not a complete check, but just tries to reject most paths early /// if they are not tuple structs. /// Type checking will take care of the full validation later. - fn extract_tuple_struct_path<'a>(&mut self, expr: &'a Expr) -> Option<&'a Path> { - // For tuple struct destructuring, it must be a non-qualified path (like in patterns). - if let ExprKind::Path(None, path) = &expr.kind { - // Does the path resolves to something disallowed in a tuple struct/variant pattern? + fn extract_tuple_struct_path<'a>( + &mut self, + expr: &'a Expr, + ) -> Option<(&'a Option, &'a Path)> { + if let ExprKind::Path(qself, path) = &expr.kind { + // Does the path resolve to something disallowed in a tuple struct/variant pattern? if let Some(partial_res) = self.resolver.get_partial_res(expr.id) { if partial_res.unresolved_segments() == 0 && !partial_res.base_res().expected_in_tuple_struct_pat() @@ -1054,7 +1054,7 @@ impl<'hir> LoweringContext<'_, 'hir> { return None; } } - return Some(path); + return Some((qself, path)); } None } @@ -1090,7 +1090,7 @@ impl<'hir> LoweringContext<'_, 'hir> { } // Tuple structs. ExprKind::Call(callee, args) => { - if let Some(path) = self.extract_tuple_struct_path(callee) { + if let Some((qself, path)) = self.extract_tuple_struct_path(callee) { let (pats, rest) = self.destructure_sequence( args, "tuple struct or variant", @@ -1099,7 +1099,7 @@ impl<'hir> LoweringContext<'_, 'hir> { ); let qpath = self.lower_qpath( callee.id, - &None, + qself, path, ParamMode::Optional, ImplTraitContext::disallowed(), @@ -1124,7 +1124,7 @@ impl<'hir> LoweringContext<'_, 'hir> { })); let qpath = self.lower_qpath( lhs.id, - &None, + &se.qself, &se.path, ParamMode::Optional, ImplTraitContext::disallowed(), @@ -1236,9 +1236,7 @@ impl<'hir> LoweringContext<'_, 'hir> { (Some(..), Some(..), HalfOpen) => hir::LangItem::Range, (None, Some(..), Closed) => hir::LangItem::RangeToInclusive, (Some(..), Some(..), Closed) => unreachable!(), - (_, None, Closed) => { - self.diagnostic().span_fatal(span, "inclusive range with no end").raise() - } + (_, None, Closed) => self.diagnostic().span_fatal(span, "inclusive range with no end"), }; let fields = self.arena.alloc_from_iter( @@ -1331,319 +1329,6 @@ impl<'hir> LoweringContext<'_, 'hir> { result } - fn lower_expr_asm(&mut self, sp: Span, asm: &InlineAsm) -> hir::ExprKind<'hir> { - // Rustdoc needs to support asm! from foriegn architectures: don't try - // lowering the register contraints in this case. - let asm_arch = if self.sess.opts.actually_rustdoc { None } else { self.sess.asm_arch }; - if asm_arch.is_none() && !self.sess.opts.actually_rustdoc { - struct_span_err!(self.sess, sp, E0472, "asm! is unsupported on this target").emit(); - } - if asm.options.contains(InlineAsmOptions::ATT_SYNTAX) - && !matches!(asm_arch, Some(asm::InlineAsmArch::X86 | asm::InlineAsmArch::X86_64)) - && !self.sess.opts.actually_rustdoc - { - self.sess - .struct_span_err(sp, "the `att_syntax` option is only supported on x86") - .emit(); - } - - // Lower operands to HIR. We use dummy register classes if an error - // occurs during lowering because we still need to be able to produce a - // valid HIR. - let sess = self.sess; - let operands: Vec<_> = asm - .operands - .iter() - .map(|(op, op_sp)| { - let lower_reg = |reg| match reg { - InlineAsmRegOrRegClass::Reg(s) => { - asm::InlineAsmRegOrRegClass::Reg(if let Some(asm_arch) = asm_arch { - asm::InlineAsmReg::parse( - asm_arch, - |feature| sess.target_features.contains(&Symbol::intern(feature)), - &sess.target, - s, - ) - .unwrap_or_else(|e| { - let msg = format!("invalid register `{}`: {}", s.as_str(), e); - sess.struct_span_err(*op_sp, &msg).emit(); - asm::InlineAsmReg::Err - }) - } else { - asm::InlineAsmReg::Err - }) - } - InlineAsmRegOrRegClass::RegClass(s) => { - asm::InlineAsmRegOrRegClass::RegClass(if let Some(asm_arch) = asm_arch { - asm::InlineAsmRegClass::parse(asm_arch, s).unwrap_or_else(|e| { - let msg = format!("invalid register class `{}`: {}", s.as_str(), e); - sess.struct_span_err(*op_sp, &msg).emit(); - asm::InlineAsmRegClass::Err - }) - } else { - asm::InlineAsmRegClass::Err - }) - } - }; - - let op = match *op { - InlineAsmOperand::In { reg, ref expr } => hir::InlineAsmOperand::In { - reg: lower_reg(reg), - expr: self.lower_expr_mut(expr), - }, - InlineAsmOperand::Out { reg, late, ref expr } => hir::InlineAsmOperand::Out { - reg: lower_reg(reg), - late, - expr: expr.as_ref().map(|expr| self.lower_expr_mut(expr)), - }, - InlineAsmOperand::InOut { reg, late, ref expr } => { - hir::InlineAsmOperand::InOut { - reg: lower_reg(reg), - late, - expr: self.lower_expr_mut(expr), - } - } - InlineAsmOperand::SplitInOut { reg, late, ref in_expr, ref out_expr } => { - hir::InlineAsmOperand::SplitInOut { - reg: lower_reg(reg), - late, - in_expr: self.lower_expr_mut(in_expr), - out_expr: out_expr.as_ref().map(|expr| self.lower_expr_mut(expr)), - } - } - InlineAsmOperand::Const { ref anon_const } => hir::InlineAsmOperand::Const { - anon_const: self.lower_anon_const(anon_const), - }, - InlineAsmOperand::Sym { ref expr } => { - hir::InlineAsmOperand::Sym { expr: self.lower_expr_mut(expr) } - } - }; - (op, *op_sp) - }) - .collect(); - - // Validate template modifiers against the register classes for the operands - for p in &asm.template { - if let InlineAsmTemplatePiece::Placeholder { - operand_idx, - modifier: Some(modifier), - span: placeholder_span, - } = *p - { - let op_sp = asm.operands[operand_idx].1; - match &operands[operand_idx].0 { - hir::InlineAsmOperand::In { reg, .. } - | hir::InlineAsmOperand::Out { reg, .. } - | hir::InlineAsmOperand::InOut { reg, .. } - | hir::InlineAsmOperand::SplitInOut { reg, .. } => { - let class = reg.reg_class(); - if class == asm::InlineAsmRegClass::Err { - continue; - } - let valid_modifiers = class.valid_modifiers(asm_arch.unwrap()); - if !valid_modifiers.contains(&modifier) { - let mut err = sess.struct_span_err( - placeholder_span, - "invalid asm template modifier for this register class", - ); - err.span_label(placeholder_span, "template modifier"); - err.span_label(op_sp, "argument"); - if !valid_modifiers.is_empty() { - let mut mods = format!("`{}`", valid_modifiers[0]); - for m in &valid_modifiers[1..] { - let _ = write!(mods, ", `{}`", m); - } - err.note(&format!( - "the `{}` register class supports \ - the following template modifiers: {}", - class.name(), - mods - )); - } else { - err.note(&format!( - "the `{}` register class does not support template modifiers", - class.name() - )); - } - err.emit(); - } - } - hir::InlineAsmOperand::Const { .. } => { - let mut err = sess.struct_span_err( - placeholder_span, - "asm template modifiers are not allowed for `const` arguments", - ); - err.span_label(placeholder_span, "template modifier"); - err.span_label(op_sp, "argument"); - err.emit(); - } - hir::InlineAsmOperand::Sym { .. } => { - let mut err = sess.struct_span_err( - placeholder_span, - "asm template modifiers are not allowed for `sym` arguments", - ); - err.span_label(placeholder_span, "template modifier"); - err.span_label(op_sp, "argument"); - err.emit(); - } - } - } - } - - let mut used_input_regs = FxHashMap::default(); - let mut used_output_regs = FxHashMap::default(); - let mut required_features: Vec<&str> = vec![]; - for (idx, &(ref op, op_sp)) in operands.iter().enumerate() { - if let Some(reg) = op.reg() { - // Make sure we don't accidentally carry features from the - // previous iteration. - required_features.clear(); - - let reg_class = reg.reg_class(); - if reg_class == asm::InlineAsmRegClass::Err { - continue; - } - - // We ignore target feature requirements for clobbers: if the - // feature is disabled then the compiler doesn't care what we - // do with the registers. - // - // Note that this is only possible for explicit register - // operands, which cannot be used in the asm string. - let is_clobber = matches!( - op, - hir::InlineAsmOperand::Out { - reg: asm::InlineAsmRegOrRegClass::Reg(_), - late: _, - expr: None - } - ); - - if !is_clobber { - // Validate register classes against currently enabled target - // features. We check that at least one type is available for - // the current target. - for &(_, feature) in reg_class.supported_types(asm_arch.unwrap()) { - if let Some(feature) = feature { - if self.sess.target_features.contains(&Symbol::intern(feature)) { - required_features.clear(); - break; - } else { - required_features.push(feature); - } - } else { - required_features.clear(); - break; - } - } - // We are sorting primitive strs here and can use unstable sort here - required_features.sort_unstable(); - required_features.dedup(); - match &required_features[..] { - [] => {} - [feature] => { - let msg = format!( - "register class `{}` requires the `{}` target feature", - reg_class.name(), - feature - ); - sess.struct_span_err(op_sp, &msg).emit(); - } - features => { - let msg = format!( - "register class `{}` requires at least one target feature: {}", - reg_class.name(), - features.join(", ") - ); - sess.struct_span_err(op_sp, &msg).emit(); - } - } - } - - // Check for conflicts between explicit register operands. - if let asm::InlineAsmRegOrRegClass::Reg(reg) = reg { - let (input, output) = match op { - hir::InlineAsmOperand::In { .. } => (true, false), - // Late output do not conflict with inputs, but normal outputs do - hir::InlineAsmOperand::Out { late, .. } => (!late, true), - hir::InlineAsmOperand::InOut { .. } - | hir::InlineAsmOperand::SplitInOut { .. } => (true, true), - hir::InlineAsmOperand::Const { .. } | hir::InlineAsmOperand::Sym { .. } => { - unreachable!() - } - }; - - // Flag to output the error only once per operand - let mut skip = false; - reg.overlapping_regs(|r| { - let mut check = |used_regs: &mut FxHashMap, - input| { - match used_regs.entry(r) { - Entry::Occupied(o) => { - if skip { - return; - } - skip = true; - - let idx2 = *o.get(); - let &(ref op2, op_sp2) = &operands[idx2]; - let reg2 = match op2.reg() { - Some(asm::InlineAsmRegOrRegClass::Reg(r)) => r, - _ => unreachable!(), - }; - - let msg = format!( - "register `{}` conflicts with register `{}`", - reg.name(), - reg2.name() - ); - let mut err = sess.struct_span_err(op_sp, &msg); - err.span_label(op_sp, &format!("register `{}`", reg.name())); - err.span_label(op_sp2, &format!("register `{}`", reg2.name())); - - match (op, op2) { - ( - hir::InlineAsmOperand::In { .. }, - hir::InlineAsmOperand::Out { late, .. }, - ) - | ( - hir::InlineAsmOperand::Out { late, .. }, - hir::InlineAsmOperand::In { .. }, - ) => { - assert!(!*late); - let out_op_sp = if input { op_sp2 } else { op_sp }; - let msg = "use `lateout` instead of \ - `out` to avoid conflict"; - err.span_help(out_op_sp, msg); - } - _ => {} - } - - err.emit(); - } - Entry::Vacant(v) => { - v.insert(idx); - } - } - }; - if input { - check(&mut used_input_regs, true); - } - if output { - check(&mut used_output_regs, false); - } - }); - } - } - } - - let operands = self.arena.alloc_from_iter(operands); - let template = self.arena.alloc_from_iter(asm.template.iter().cloned()); - let line_spans = self.arena.alloc_slice(&asm.line_spans[..]); - let hir_asm = hir::InlineAsm { template, operands, options: asm.options, line_spans }; - hir::ExprKind::InlineAsm(self.arena.alloc(hir_asm)) - } - fn lower_expr_llvm_asm(&mut self, asm: &LlvmInlineAsm) -> hir::ExprKind<'hir> { let inner = hir::LlvmInlineAsmInner { inputs: asm.inputs.iter().map(|&(c, _)| c).collect(), @@ -1896,14 +1581,14 @@ impl<'hir> LoweringContext<'_, 'hir> { self.allow_try_trait.clone(), ); - // `Try::into_result()` + // `Try::branch()` let scrutinee = { // expand let sub_expr = self.lower_expr_mut(sub_expr); self.expr_call_lang_item_fn( unstable_span, - hir::LangItem::TryIntoResult, + hir::LangItem::TryTraitBranch, arena_vec![self; sub_expr], ) }; @@ -1921,8 +1606,8 @@ impl<'hir> LoweringContext<'_, 'hir> { }; let attrs = vec![attr]; - // `Ok(val) => #[allow(unreachable_code)] val,` - let ok_arm = { + // `ControlFlow::Continue(val) => #[allow(unreachable_code)] val,` + let continue_arm = { let val_ident = Ident::with_dummy_span(sym::val); let (val_pat, val_pat_nid) = self.pat_ident(span, val_ident); let val_expr = self.arena.alloc(self.expr_ident_with_attrs( @@ -1931,27 +1616,21 @@ impl<'hir> LoweringContext<'_, 'hir> { val_pat_nid, ThinVec::from(attrs.clone()), )); - let ok_pat = self.pat_ok(span, val_pat); - self.arm(ok_pat, val_expr) + let continue_pat = self.pat_cf_continue(unstable_span, val_pat); + self.arm(continue_pat, val_expr) }; - // `Err(err) => #[allow(unreachable_code)] - // return Try::from_error(From::from(err)),` - let err_arm = { - let err_ident = Ident::with_dummy_span(sym::err); - let (err_local, err_local_nid) = self.pat_ident(try_span, err_ident); - let from_expr = { - let err_expr = self.expr_ident_mut(try_span, err_ident, err_local_nid); - self.expr_call_lang_item_fn( - try_span, - hir::LangItem::FromFrom, - arena_vec![self; err_expr], - ) - }; - let from_err_expr = self.wrap_in_try_constructor( - hir::LangItem::TryFromError, - unstable_span, - from_expr, + // `ControlFlow::Break(residual) => + // #[allow(unreachable_code)] + // return Try::from_residual(residual),` + let break_arm = { + let residual_ident = Ident::with_dummy_span(sym::residual); + let (residual_local, residual_local_nid) = self.pat_ident(try_span, residual_ident); + let residual_expr = self.expr_ident_mut(try_span, residual_ident, residual_local_nid); + let from_residual_expr = self.wrap_in_try_constructor( + hir::LangItem::TryTraitFromResidual, + try_span, + self.arena.alloc(residual_expr), unstable_span, ); let thin_attrs = ThinVec::from(attrs); @@ -1962,25 +1641,25 @@ impl<'hir> LoweringContext<'_, 'hir> { try_span, hir::ExprKind::Break( hir::Destination { label: None, target_id }, - Some(from_err_expr), + Some(from_residual_expr), ), thin_attrs, )) } else { self.arena.alloc(self.expr( try_span, - hir::ExprKind::Ret(Some(from_err_expr)), + hir::ExprKind::Ret(Some(from_residual_expr)), thin_attrs, )) }; - let err_pat = self.pat_err(try_span, err_local); - self.arm(err_pat, ret_expr) + let break_pat = self.pat_cf_break(try_span, residual_local); + self.arm(break_pat, ret_expr) }; hir::ExprKind::Match( scrutinee, - arena_vec![self; err_arm, ok_arm], + arena_vec![self; break_arm, continue_arm], hir::MatchSource::TryDesugar, ) } diff --git a/compiler/rustc_ast_lowering/src/item.rs b/compiler/rustc_ast_lowering/src/item.rs index 5fd8f7eb33..292643d6d7 100644 --- a/compiler/rustc_ast_lowering/src/item.rs +++ b/compiler/rustc_ast_lowering/src/item.rs @@ -329,7 +329,9 @@ impl<'hir> LoweringContext<'_, 'hir> { .alloc_from_iter(fm.items.iter().map(|x| self.lower_foreign_item_ref(x))), } } - ItemKind::GlobalAsm(ref ga) => hir::ItemKind::GlobalAsm(self.lower_global_asm(ga)), + ItemKind::GlobalAsm(ref asm) => { + hir::ItemKind::GlobalAsm(self.lower_inline_asm(span, asm)) + } ItemKind::TyAlias(box TyAliasKind(_, ref gen, _, Some(ref ty))) => { // We lower // @@ -746,10 +748,6 @@ impl<'hir> LoweringContext<'_, 'hir> { } } - fn lower_global_asm(&mut self, ga: &GlobalAsm) -> &'hir hir::GlobalAsm { - self.arena.alloc(hir::GlobalAsm { asm: ga.asm }) - } - fn lower_variant(&mut self, v: &Variant) -> hir::Variant<'hir> { let id = self.lower_node_id(v.id); self.lower_attrs(id, &v.attrs); @@ -791,7 +789,10 @@ impl<'hir> LoweringContext<'_, 'hir> { } } - fn lower_field_def(&mut self, (index, f): (usize, &FieldDef)) -> hir::FieldDef<'hir> { + pub(super) fn lower_field_def( + &mut self, + (index, f): (usize, &FieldDef), + ) -> hir::FieldDef<'hir> { let ty = if let TyKind::Path(ref qself, ref path) = f.ty.kind { let t = self.lower_path_ty( &f.ty, diff --git a/compiler/rustc_ast_lowering/src/lib.rs b/compiler/rustc_ast_lowering/src/lib.rs index 44056df4ab..0ff1efd816 100644 --- a/compiler/rustc_ast_lowering/src/lib.rs +++ b/compiler/rustc_ast_lowering/src/lib.rs @@ -31,7 +31,6 @@ //! in the HIR, especially for multiple identifiers. #![feature(crate_visibility_modifier)] -#![cfg_attr(bootstrap, feature(or_patterns))] #![feature(box_patterns)] #![feature(iter_zip)] #![recursion_limit = "256"] @@ -44,9 +43,9 @@ use rustc_ast::walk_list; use rustc_ast::{self as ast, *}; use rustc_ast_pretty::pprust; use rustc_data_structures::captures::Captures; -use rustc_data_structures::fx::FxHashSet; +use rustc_data_structures::fx::{FxHashMap, FxHashSet}; use rustc_data_structures::sync::Lrc; -use rustc_errors::struct_span_err; +use rustc_errors::{struct_span_err, Applicability}; use rustc_hir as hir; use rustc_hir::def::{DefKind, Namespace, PartialRes, PerNS, Res}; use rustc_hir::def_id::{DefId, DefIdMap, LocalDefId, CRATE_DEF_ID}; @@ -58,10 +57,11 @@ use rustc_session::lint::builtin::{BARE_TRAIT_OBJECTS, MISSING_ABI}; use rustc_session::lint::{BuiltinLintDiagnostics, LintBuffer}; use rustc_session::utils::{FlattenNonterminals, NtToTokenstream}; use rustc_session::Session; +use rustc_span::edition::Edition; use rustc_span::hygiene::ExpnId; use rustc_span::source_map::{respan, DesugaringKind}; use rustc_span::symbol::{kw, sym, Ident, Symbol}; -use rustc_span::Span; +use rustc_span::{Span, DUMMY_SP}; use rustc_target::spec::abi::Abi; use smallvec::{smallvec, SmallVec}; @@ -76,6 +76,7 @@ macro_rules! arena_vec { }); } +mod asm; mod expr; mod item; mod pat; @@ -165,7 +166,7 @@ struct LoweringContext<'a, 'hir: 'a> { type_def_lifetime_params: DefIdMap, - current_hir_id_owner: Vec<(LocalDefId, u32)>, + current_hir_id_owner: (LocalDefId, u32), item_local_id_counters: NodeMap, node_id_to_hir_id: IndexVec>, @@ -197,7 +198,7 @@ pub trait ResolverAstLowering { fn next_node_id(&mut self) -> NodeId; - fn trait_map(&self) -> &NodeMap>; + fn take_trait_map(&mut self) -> NodeMap>; fn opt_local_def_id(&self, node: NodeId) -> Option; @@ -321,7 +322,7 @@ pub fn lower_crate<'a, 'hir>( anonymous_lifetime_mode: AnonymousLifetimeMode::PassThrough, type_def_lifetime_params: Default::default(), current_module: CRATE_DEF_ID, - current_hir_id_owner: vec![(CRATE_DEF_ID, 0)], + current_hir_id_owner: (CRATE_DEF_ID, 0), item_local_id_counters: Default::default(), node_id_to_hir_id: IndexVec::new(), generator_kind: None, @@ -330,7 +331,7 @@ pub fn lower_crate<'a, 'hir>( lifetimes_to_define: Vec::new(), is_collecting_in_band_lifetimes: false, in_scope_lifetimes: Vec::new(), - allow_try_trait: Some([sym::try_trait][..].into()), + allow_try_trait: Some([sym::control_flow_enum, sym::try_trait_v2][..].into()), allow_gen_future: Some([sym::gen_future][..].into()), } .lower_crate(krate) @@ -500,14 +501,13 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> { let proc_macros = c.proc_macros.iter().map(|id| self.node_id_to_hir_id[*id].unwrap()).collect(); - let trait_map = self - .resolver - .trait_map() - .iter() - .filter_map(|(&k, v)| { - self.node_id_to_hir_id.get(k).and_then(|id| id.as_ref()).map(|id| (*id, v.clone())) - }) - .collect(); + let mut trait_map: FxHashMap<_, FxHashMap<_, _>> = FxHashMap::default(); + for (k, v) in self.resolver.take_trait_map().into_iter() { + if let Some(Some(hir_id)) = self.node_id_to_hir_id.get(k) { + let map = trait_map.entry(hir_id.owner).or_default(); + map.insert(hir_id.local_id, v.into_boxed_slice()); + } + } let mut def_id_to_hir_id = IndexVec::default(); @@ -594,9 +594,10 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> { .insert(owner, HIR_ID_COUNTER_LOCKED) .unwrap_or_else(|| panic!("no `item_local_id_counters` entry for {:?}", owner)); let def_id = self.resolver.local_def_id(owner); - self.current_hir_id_owner.push((def_id, counter)); + let old_owner = std::mem::replace(&mut self.current_hir_id_owner, (def_id, counter)); let ret = f(self); - let (new_def_id, new_counter) = self.current_hir_id_owner.pop().unwrap(); + let (new_def_id, new_counter) = + std::mem::replace(&mut self.current_hir_id_owner, old_owner); debug_assert!(def_id == new_def_id); debug_assert!(new_counter >= counter); @@ -614,8 +615,7 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> { /// properly. Calling the method twice with the same `NodeId` is fine though. fn lower_node_id(&mut self, ast_node_id: NodeId) -> hir::HirId { self.lower_node_id_generic(ast_node_id, |this| { - let &mut (owner, ref mut local_id_counter) = - this.current_hir_id_owner.last_mut().unwrap(); + let &mut (owner, ref mut local_id_counter) = &mut this.current_hir_id_owner; let local_id = *local_id_counter; *local_id_counter += 1; hir::HirId { owner, local_id: hir::ItemLocalId::from_u32(local_id) } @@ -868,10 +868,7 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> { // wouldn't have been added yet. let generics = this.lower_generics_mut( generics, - ImplTraitContext::Universal( - &mut params, - this.current_hir_id_owner.last().unwrap().0, - ), + ImplTraitContext::Universal(&mut params, this.current_hir_id_owner.0), ); let res = f(this, &mut params); (params, (generics, res)) @@ -1077,7 +1074,7 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> { } AssocTyConstraintKind::Bound { ref bounds } => { let mut capturable_lifetimes; - let mut parent_def_id = self.current_hir_id_owner.last().unwrap().0; + let mut parent_def_id = self.current_hir_id_owner.0; // Piggy-back on the `impl Trait` context to figure out the correct behavior. let (desugar_to_impl_trait, itctx) = match itctx { // We are in the return position: @@ -1198,7 +1195,7 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> { // Construct a AnonConst where the expr is the "ty"'s path. - let parent_def_id = self.current_hir_id_owner.last().unwrap().0; + let parent_def_id = self.current_hir_id_owner.0; let node_id = self.resolver.next_node_id(); // Add a definition for the in-band const def. @@ -1268,6 +1265,15 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> { let kind = match t.kind { TyKind::Infer => hir::TyKind::Infer, TyKind::Err => hir::TyKind::Err, + // FIXME(unnamed_fields): IMPLEMENTATION IN PROGRESS + TyKind::AnonymousStruct(ref _fields, _recovered) => { + self.sess.struct_span_err(t.span, "anonymous structs are unimplemented").emit(); + hir::TyKind::Err + } + TyKind::AnonymousUnion(ref _fields, _recovered) => { + self.sess.struct_span_err(t.span, "anonymous unions are unimplemented").emit(); + hir::TyKind::Err + } TyKind::Slice(ref ty) => hir::TyKind::Slice(self.lower_ty(ty, itctx)), TyKind::Ptr(ref mt) => hir::TyKind::Ptr(self.lower_mt(mt, itctx)), TyKind::Rptr(ref region, ref mt) => { @@ -1814,10 +1820,7 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> { if let Some((_, ibty)) = &mut in_band_ty_params { this.lower_ty_direct( ¶m.ty, - ImplTraitContext::Universal( - ibty, - this.current_hir_id_owner.last().unwrap().0, - ), + ImplTraitContext::Universal(ibty, this.current_hir_id_owner.0), ) } else { this.lower_ty_direct(¶m.ty, ImplTraitContext::disallowed()) @@ -2089,6 +2092,7 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> { args: &[], bindings: arena_vec![self; self.output_ty_binding(span, output_ty)], parenthesized: false, + span_ext: DUMMY_SP, }); hir::GenericBound::LangItemTrait( @@ -2484,14 +2488,14 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> { self.pat(span, hir::PatKind::Lit(expr)) } - fn pat_ok(&mut self, span: Span, pat: &'hir hir::Pat<'hir>) -> &'hir hir::Pat<'hir> { + fn pat_cf_continue(&mut self, span: Span, pat: &'hir hir::Pat<'hir>) -> &'hir hir::Pat<'hir> { let field = self.single_pat_field(span, pat); - self.pat_lang_item_variant(span, hir::LangItem::ResultOk, field) + self.pat_lang_item_variant(span, hir::LangItem::ControlFlowContinue, field) } - fn pat_err(&mut self, span: Span, pat: &'hir hir::Pat<'hir>) -> &'hir hir::Pat<'hir> { + fn pat_cf_break(&mut self, span: Span, pat: &'hir hir::Pat<'hir>) -> &'hir hir::Pat<'hir> { let field = self.single_pat_field(span, pat); - self.pat_lang_item_variant(span, hir::LangItem::ResultErr, field) + self.pat_lang_item_variant(span, hir::LangItem::ControlFlowBreak, field) } fn pat_some(&mut self, span: Span, pat: &'hir hir::Pat<'hir>) -> &'hir hir::Pat<'hir> { @@ -2736,13 +2740,26 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> { .map(|snippet| snippet.starts_with("#[")) .unwrap_or(true); if !is_macro_callsite { - self.resolver.lint_buffer().buffer_lint_with_diagnostic( - BARE_TRAIT_OBJECTS, - id, - span, - "trait objects without an explicit `dyn` are deprecated", - BuiltinLintDiagnostics::BareTraitObject(span, is_global), - ) + if span.edition() < Edition::Edition2021 { + self.resolver.lint_buffer().buffer_lint_with_diagnostic( + BARE_TRAIT_OBJECTS, + id, + span, + "trait objects without an explicit `dyn` are deprecated", + BuiltinLintDiagnostics::BareTraitObject(span, is_global), + ) + } else { + let msg = "trait objects must include the `dyn` keyword"; + let label = "add `dyn` keyword before this trait"; + let mut err = struct_span_err!(self.sess, span, E0782, "{}", msg,); + err.span_suggestion_verbose( + span.shrink_to_lo(), + label, + String::from("dyn "), + Applicability::MachineApplicable, + ); + err.emit(); + } } } @@ -2780,6 +2797,7 @@ struct GenericArgsCtor<'hir> { args: SmallVec<[hir::GenericArg<'hir>; 4]>, bindings: &'hir [hir::TypeBinding<'hir>], parenthesized: bool, + span: Span, } impl<'hir> GenericArgsCtor<'hir> { @@ -2792,6 +2810,7 @@ impl<'hir> GenericArgsCtor<'hir> { args: arena.alloc_from_iter(self.args), bindings: self.bindings, parenthesized: self.parenthesized, + span_ext: self.span, } } } diff --git a/compiler/rustc_ast_lowering/src/pat.rs b/compiler/rustc_ast_lowering/src/pat.rs index 2451409aac..66e623528f 100644 --- a/compiler/rustc_ast_lowering/src/pat.rs +++ b/compiler/rustc_ast_lowering/src/pat.rs @@ -21,10 +21,10 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> { break self.lower_pat_ident(pattern, binding_mode, ident, lower_sub); } PatKind::Lit(ref e) => break hir::PatKind::Lit(self.lower_expr(e)), - PatKind::TupleStruct(ref path, ref pats) => { + PatKind::TupleStruct(ref qself, ref path, ref pats) => { let qpath = self.lower_qpath( pattern.id, - &None, + qself, path, ParamMode::Optional, ImplTraitContext::disallowed(), @@ -47,10 +47,10 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> { ); break hir::PatKind::Path(qpath); } - PatKind::Struct(ref path, ref fields, etc) => { + PatKind::Struct(ref qself, ref path, ref fields, etc) => { let qpath = self.lower_qpath( pattern.id, - &None, + qself, path, ParamMode::Optional, ImplTraitContext::disallowed(), diff --git a/compiler/rustc_ast_lowering/src/path.rs b/compiler/rustc_ast_lowering/src/path.rs index 46dac2f1af..fe9f1fb20f 100644 --- a/compiler/rustc_ast_lowering/src/path.rs +++ b/compiler/rustc_ast_lowering/src/path.rs @@ -10,7 +10,7 @@ use rustc_hir::GenericArg; use rustc_session::lint::builtin::ELIDED_LIFETIMES_IN_PATHS; use rustc_session::lint::BuiltinLintDiagnostics; use rustc_span::symbol::Ident; -use rustc_span::Span; +use rustc_span::{BytePos, Span, DUMMY_SP}; use smallvec::smallvec; use tracing::debug; @@ -24,6 +24,7 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> { param_mode: ParamMode, mut itctx: ImplTraitContext<'_, 'hir>, ) -> hir::QPath<'hir> { + debug!("lower_qpath(id: {:?}, qself: {:?}, p: {:?})", id, qself, p); let qself_position = qself.as_ref().map(|q| q.position); let qself = qself.as_ref().map(|q| self.lower_ty(&q.ty, itctx.reborrow())); @@ -222,6 +223,10 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> { itctx: ImplTraitContext<'_, 'hir>, explicit_owner: Option, ) -> hir::PathSegment<'hir> { + debug!( + "path_span: {:?}, lower_path_segment(segment: {:?}, expected_lifetimes: {:?})", + path_span, segment, expected_lifetimes + ); let (mut generic_args, infer_args) = if let Some(ref generic_args) = segment.args { let msg = "parenthesized type parameters may only be used with a `Fn` trait"; match **generic_args { @@ -262,23 +267,34 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> { }, } } else { - self.lower_angle_bracketed_parameter_data(&Default::default(), param_mode, itctx) + ( + GenericArgsCtor { + args: Default::default(), + bindings: &[], + parenthesized: false, + span: path_span.shrink_to_hi(), + }, + param_mode == ParamMode::Optional, + ) }; let has_lifetimes = generic_args.args.iter().any(|arg| matches!(arg, GenericArg::Lifetime(_))); - let first_generic_span = generic_args - .args - .iter() - .map(|a| a.span()) - .chain(generic_args.bindings.iter().map(|b| b.span)) - .next(); if !generic_args.parenthesized && !has_lifetimes { + // Note: these spans are used for diagnostics when they can't be inferred. + // See rustc_resolve::late::lifetimes::LifetimeContext::add_missing_lifetime_specifiers_label + let elided_lifetime_span = if generic_args.span.is_empty() { + // If there are no brackets, use the identifier span. + segment.ident.span + } else if generic_args.is_empty() { + // If there are brackets, but not generic arguments, then use the opening bracket + generic_args.span.with_hi(generic_args.span.lo() + BytePos(1)) + } else { + // Else use an empty span right after the opening bracket. + generic_args.span.with_lo(generic_args.span.lo() + BytePos(1)).shrink_to_lo() + }; generic_args.args = self - .elided_path_lifetimes( - first_generic_span.map_or(segment.ident.span, |s| s.shrink_to_lo()), - expected_lifetimes, - ) + .elided_path_lifetimes(elided_lifetime_span, expected_lifetimes) .map(GenericArg::Lifetime) .chain(generic_args.args.into_iter()) .collect(); @@ -287,15 +303,15 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> { let no_non_lt_args = generic_args.args.len() == expected_lifetimes; let no_bindings = generic_args.bindings.is_empty(); let (incl_angl_brckt, insertion_sp, suggestion) = if no_non_lt_args && no_bindings { - // If there are no (non-implicit) generic args or associated type - // bindings, our suggestion includes the angle brackets. + // If there are no generic args, our suggestion can include the angle brackets. (true, path_span.shrink_to_hi(), format!("<{}>", anon_lt_suggestion)) } else { - // Otherwise (sorry, this is kind of gross) we need to infer the - // place to splice in the `'_, ` from the generics that do exist. - let first_generic_span = first_generic_span - .expect("already checked that non-lifetime args or bindings exist"); - (false, first_generic_span.shrink_to_lo(), format!("{}, ", anon_lt_suggestion)) + // Otherwise we'll insert a `'_, ` right after the opening bracket. + let span = generic_args + .span + .with_lo(generic_args.span.lo() + BytePos(1)) + .shrink_to_lo(); + (false, span, format!("{}, ", anon_lt_suggestion)) }; match self.anonymous_lifetime_mode { // In create-parameter mode we error here because we don't want to support @@ -357,7 +373,7 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> { hir_id: Some(id), res: Some(self.lower_res(res)), infer_args, - args: if generic_args.is_empty() { + args: if generic_args.is_empty() && generic_args.span.is_empty() { None } else { Some(self.arena.alloc(generic_args.into_generic_args(self.arena))) @@ -390,7 +406,7 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> { } AngleBracketedArg::Arg(_) => None, })); - let ctor = GenericArgsCtor { args, bindings, parenthesized: false }; + let ctor = GenericArgsCtor { args, bindings, parenthesized: false, span: data.span }; (ctor, !has_non_lt_args && param_mode == ParamMode::Optional) } @@ -415,7 +431,12 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> { let args = smallvec![GenericArg::Type(this.ty_tup(*inputs_span, inputs))]; let binding = this.output_ty_binding(output_ty.span, output_ty); ( - GenericArgsCtor { args, bindings: arena_vec![this; binding], parenthesized: true }, + GenericArgsCtor { + args, + bindings: arena_vec![this; binding], + parenthesized: true, + span: data.inputs_span, + }, false, ) }) @@ -431,7 +452,12 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> { let kind = hir::TypeBindingKind::Equality { ty }; let args = arena_vec![self;]; let bindings = arena_vec![self;]; - let gen_args = self.arena.alloc(hir::GenericArgs { args, bindings, parenthesized: false }); + let gen_args = self.arena.alloc(hir::GenericArgs { + args, + bindings, + parenthesized: false, + span_ext: DUMMY_SP, + }); hir::TypeBinding { hir_id: self.next_id(), gen_args, span, ident, kind } } } diff --git a/compiler/rustc_ast_passes/src/ast_validation.rs b/compiler/rustc_ast_passes/src/ast_validation.rs index 6d6438920c..30aa51a121 100644 --- a/compiler/rustc_ast_passes/src/ast_validation.rs +++ b/compiler/rustc_ast_passes/src/ast_validation.rs @@ -175,10 +175,30 @@ impl<'a> AstValidator<'a> { } } } + TyKind::AnonymousStruct(ref fields, ..) | TyKind::AnonymousUnion(ref fields, ..) => { + self.with_banned_assoc_ty_bound(|this| { + walk_list!(this, visit_struct_field_def, fields) + }); + } _ => visit::walk_ty(self, t), } } + fn visit_struct_field_def(&mut self, field: &'a FieldDef) { + if let Some(ident) = field.ident { + if ident.name == kw::Underscore { + self.check_anonymous_field(field); + self.visit_vis(&field.vis); + self.visit_ident(ident); + self.visit_ty_common(&field.ty); + self.walk_ty(&field.ty); + walk_list!(self, visit_attribute, &field.attrs); + return; + } + } + self.visit_field_def(field); + } + fn err_handler(&self) -> &rustc_errors::Handler { &self.session.diagnostic() } @@ -213,6 +233,66 @@ impl<'a> AstValidator<'a> { err.emit(); } + fn check_anonymous_field(&self, field: &FieldDef) { + let FieldDef { ty, .. } = field; + match &ty.kind { + TyKind::AnonymousStruct(..) | TyKind::AnonymousUnion(..) => { + // We already checked for `kw::Underscore` before calling this function, + // so skip the check + } + TyKind::Path(..) => { + // If the anonymous field contains a Path as type, we can't determine + // if the path is a valid struct or union, so skip the check + } + _ => { + let msg = "unnamed fields can only have struct or union types"; + let label = "not a struct or union"; + self.err_handler() + .struct_span_err(field.span, msg) + .span_label(ty.span, label) + .emit(); + } + } + } + + fn deny_anonymous_struct(&self, ty: &Ty) { + match &ty.kind { + TyKind::AnonymousStruct(..) => { + self.err_handler() + .struct_span_err( + ty.span, + "anonymous structs are not allowed outside of unnamed struct or union fields", + ) + .span_label(ty.span, "anonymous struct declared here") + .emit(); + } + TyKind::AnonymousUnion(..) => { + self.err_handler() + .struct_span_err( + ty.span, + "anonymous unions are not allowed outside of unnamed struct or union fields", + ) + .span_label(ty.span, "anonymous union declared here") + .emit(); + } + _ => {} + } + } + + fn deny_anonymous_field(&self, field: &FieldDef) { + if let Some(ident) = field.ident { + if ident.name == kw::Underscore { + self.err_handler() + .struct_span_err( + field.span, + "anonymous fields are not allowed outside of structs or unions", + ) + .span_label(ident.span, "anonymous field declared here") + .emit() + } + } + } + fn check_decl_no_pat(decl: &FnDecl, mut report_err: impl FnMut(Span, Option, bool)) { for Param { pat, .. } in &decl.inputs { match pat.kind { @@ -732,6 +812,71 @@ impl<'a> AstValidator<'a> { ) .emit(); } + + fn visit_ty_common(&mut self, ty: &'a Ty) { + match ty.kind { + TyKind::BareFn(ref bfty) => { + self.check_fn_decl(&bfty.decl, SelfSemantic::No); + Self::check_decl_no_pat(&bfty.decl, |span, _, _| { + struct_span_err!( + self.session, + span, + E0561, + "patterns aren't allowed in function pointer types" + ) + .emit(); + }); + self.check_late_bound_lifetime_defs(&bfty.generic_params); + } + TyKind::TraitObject(ref bounds, ..) => { + let mut any_lifetime_bounds = false; + for bound in bounds { + if let GenericBound::Outlives(ref lifetime) = *bound { + if any_lifetime_bounds { + struct_span_err!( + self.session, + lifetime.ident.span, + E0226, + "only a single explicit lifetime bound is permitted" + ) + .emit(); + break; + } + any_lifetime_bounds = true; + } + } + self.no_questions_in_bounds(bounds, "trait object types", false); + } + TyKind::ImplTrait(_, ref bounds) => { + if self.is_impl_trait_banned { + struct_span_err!( + self.session, + ty.span, + E0667, + "`impl Trait` is not allowed in path parameters" + ) + .emit(); + } + + if let Some(outer_impl_trait_sp) = self.outer_impl_trait { + struct_span_err!( + self.session, + ty.span, + E0666, + "nested `impl Trait` is not allowed" + ) + .span_label(outer_impl_trait_sp, "outer `impl Trait`") + .span_label(ty.span, "nested `impl Trait` here") + .emit(); + } + + if !bounds.iter().any(|b| matches!(b, GenericBound::Trait(..))) { + self.err_handler().span_err(ty.span, "at least one trait must be specified"); + } + } + _ => {} + } + } } /// Checks that generic parameters are in the correct order, @@ -793,8 +938,11 @@ fn validate_generic_param_order( } GenericParamKind::Type { default: None } => (), GenericParamKind::Lifetime => (), - // FIXME(const_generics_defaults) - GenericParamKind::Const { ty: _, kw_span: _, default: _ } => (), + GenericParamKind::Const { ty: _, kw_span: _, default: Some(default) } => { + ordered_params += " = "; + ordered_params += &pprust::expr_to_string(&*default.value); + } + GenericParamKind::Const { ty: _, kw_span: _, default: None } => (), } first = false; } @@ -814,7 +962,7 @@ fn validate_generic_param_order( span, &format!( "reorder the parameters: lifetimes, {}", - if sess.features_untracked().const_generics { + if sess.features_untracked().unordered_const_ty_params() { "then consts and types" } else { "then types, then consts" @@ -850,72 +998,8 @@ impl<'a> Visitor<'a> for AstValidator<'a> { } fn visit_ty(&mut self, ty: &'a Ty) { - match ty.kind { - TyKind::BareFn(ref bfty) => { - self.check_fn_decl(&bfty.decl, SelfSemantic::No); - Self::check_decl_no_pat(&bfty.decl, |span, _, _| { - struct_span_err!( - self.session, - span, - E0561, - "patterns aren't allowed in function pointer types" - ) - .emit(); - }); - self.check_late_bound_lifetime_defs(&bfty.generic_params); - } - TyKind::TraitObject(ref bounds, ..) => { - let mut any_lifetime_bounds = false; - for bound in bounds { - if let GenericBound::Outlives(ref lifetime) = *bound { - if any_lifetime_bounds { - struct_span_err!( - self.session, - lifetime.ident.span, - E0226, - "only a single explicit lifetime bound is permitted" - ) - .emit(); - break; - } - any_lifetime_bounds = true; - } - } - self.no_questions_in_bounds(bounds, "trait object types", false); - } - TyKind::ImplTrait(_, ref bounds) => { - if self.is_impl_trait_banned { - struct_span_err!( - self.session, - ty.span, - E0667, - "`impl Trait` is not allowed in path parameters" - ) - .emit(); - } - - if let Some(outer_impl_trait_sp) = self.outer_impl_trait { - struct_span_err!( - self.session, - ty.span, - E0666, - "nested `impl Trait` is not allowed" - ) - .span_label(outer_impl_trait_sp, "outer `impl Trait`") - .span_label(ty.span, "nested `impl Trait` here") - .emit(); - } - - if !bounds.iter().any(|b| matches!(b, GenericBound::Trait(..))) { - self.err_handler().span_err(ty.span, "at least one trait must be specified"); - } - - self.walk_ty(ty); - return; - } - _ => {} - } - + self.visit_ty_common(ty); + self.deny_anonymous_struct(ty); self.walk_ty(ty) } @@ -929,6 +1013,11 @@ impl<'a> Visitor<'a> for AstValidator<'a> { visit::walk_lifetime(self, lifetime); } + fn visit_field_def(&mut self, s: &'a FieldDef) { + self.deny_anonymous_field(s); + visit::walk_field_def(self, s) + } + fn visit_item(&mut self, item: &'a Item) { if item.attrs.iter().any(|attr| self.session.is_proc_macro_attr(attr)) { self.has_proc_macro_decls = true; @@ -1084,14 +1173,42 @@ impl<'a> Visitor<'a> for AstValidator<'a> { self.check_mod_file_item_asciionly(item.ident); } } - ItemKind::Union(ref vdata, _) => { - if let VariantData::Tuple(..) | VariantData::Unit(..) = vdata { - self.err_handler() - .span_err(item.span, "tuple and unit unions are not permitted"); + ItemKind::Struct(ref vdata, ref generics) => match vdata { + // Duplicating the `Visitor` logic allows catching all cases + // of `Anonymous(Struct, Union)` outside of a field struct or union. + // + // Inside `visit_ty` the validator catches every `Anonymous(Struct, Union)` it + // encounters, and only on `ItemKind::Struct` and `ItemKind::Union` + // it uses `visit_ty_common`, which doesn't contain that specific check. + VariantData::Struct(ref fields, ..) => { + self.visit_vis(&item.vis); + self.visit_ident(item.ident); + self.visit_generics(generics); + self.with_banned_assoc_ty_bound(|this| { + walk_list!(this, visit_struct_field_def, fields); + }); + walk_list!(self, visit_attribute, &item.attrs); + return; } + _ => {} + }, + ItemKind::Union(ref vdata, ref generics) => { if vdata.fields().is_empty() { self.err_handler().span_err(item.span, "unions cannot have zero fields"); } + match vdata { + VariantData::Struct(ref fields, ..) => { + self.visit_vis(&item.vis); + self.visit_ident(item.ident); + self.visit_generics(generics); + self.with_banned_assoc_ty_bound(|this| { + walk_list!(this, visit_struct_field_def, fields); + }); + walk_list!(self, visit_attribute, &item.attrs); + return; + } + _ => {} + } } ItemKind::Const(def, .., None) => { self.check_defaultness(item.span, def); diff --git a/compiler/rustc_ast_passes/src/feature_gate.rs b/compiler/rustc_ast_passes/src/feature_gate.rs index da516f5cb4..3f98944d85 100644 --- a/compiler/rustc_ast_passes/src/feature_gate.rs +++ b/compiler/rustc_ast_passes/src/feature_gate.rs @@ -318,7 +318,6 @@ impl<'a> Visitor<'a> for PostExpansionVisitor<'a> { }} gate_doc!( - include => external_doc cfg => doc_cfg masked => doc_masked notable_trait => doc_notable_trait @@ -326,6 +325,45 @@ impl<'a> Visitor<'a> for PostExpansionVisitor<'a> { ); } } + + // Check for unstable modifiers on `#[link(..)]` attribute + if self.sess.check_name(attr, sym::link) { + for nested_meta in attr.meta_item_list().unwrap_or_default() { + if nested_meta.has_name(sym::modifiers) { + gate_feature_post!( + self, + native_link_modifiers, + nested_meta.span(), + "native link modifiers are experimental" + ); + + if let Some(modifiers) = nested_meta.value_str() { + for modifier in modifiers.as_str().split(',') { + if let Some(modifier) = modifier.strip_prefix(&['+', '-'][..]) { + macro_rules! gate_modifier { ($($name:literal => $feature:ident)*) => { + $(if modifier == $name { + let msg = concat!("`#[link(modifiers=\"", $name, "\")]` is unstable"); + gate_feature_post!( + self, + $feature, + nested_meta.name_value_literal_span().unwrap(), + msg + ); + })* + }} + + gate_modifier!( + "bundle" => native_link_modifiers_bundle + "verbatim" => native_link_modifiers_verbatim + "whole-archive" => native_link_modifiers_whole_archive + "as-needed" => native_link_modifiers_as_needed + ); + } + } + } + } + } + } } fn visit_item(&mut self, i: &'a ast::Item) { @@ -667,16 +705,13 @@ pub fn check_crate(krate: &ast::Crate, sess: &Session) { "async closures are unstable", "to use an async block, remove the `||`: `async {`" ); + gate_all!(more_qualified_paths, "usage of qualified paths in this context is experimental"); gate_all!(generators, "yield syntax is experimental"); gate_all!(raw_ref_op, "raw address of syntax is experimental"); gate_all!(const_trait_bound_opt_out, "`?const` on trait bounds is experimental"); gate_all!(const_trait_impl, "const trait impls are experimental"); gate_all!(half_open_range_patterns, "half-open range patterns are unstable"); gate_all!(inline_const, "inline-const is experimental"); - gate_all!( - extended_key_value_attributes, - "arbitrary expressions in key-value attributes are unstable" - ); gate_all!( const_generics_defaults, "default values for const generic parameters are experimental" @@ -686,6 +721,7 @@ pub fn check_crate(krate: &ast::Crate, sess: &Session) { // involved, so we only emit errors where there are no other parsing errors. gate_all!(destructuring_assignment, "destructuring assignments are unstable"); } + gate_all!(unnamed_fields, "unnamed fields are not yet fully implemented"); // All uses of `gate_all!` below this point were added in #65742, // and subsequently disabled (with the non-early gating readded). diff --git a/compiler/rustc_ast_passes/src/lib.rs b/compiler/rustc_ast_passes/src/lib.rs index c9e2d202da..26da18b571 100644 --- a/compiler/rustc_ast_passes/src/lib.rs +++ b/compiler/rustc_ast_passes/src/lib.rs @@ -6,7 +6,6 @@ #![feature(bindings_after_at)] #![feature(iter_is_partitioned)] -#![feature(box_syntax)] #![feature(box_patterns)] #![recursion_limit = "256"] diff --git a/compiler/rustc_ast_pretty/src/lib.rs b/compiler/rustc_ast_pretty/src/lib.rs index 67b66284f6..6bd543ff15 100644 --- a/compiler/rustc_ast_pretty/src/lib.rs +++ b/compiler/rustc_ast_pretty/src/lib.rs @@ -1,6 +1,5 @@ #![feature(bool_to_option)] #![feature(crate_visibility_modifier)] -#![cfg_attr(bootstrap, feature(or_patterns))] #![feature(box_patterns)] #![recursion_limit = "256"] diff --git a/compiler/rustc_ast_pretty/src/pprust/state.rs b/compiler/rustc_ast_pretty/src/pprust/state.rs index 789d2c296e..3d2785faad 100644 --- a/compiler/rustc_ast_pretty/src/pprust/state.rs +++ b/compiler/rustc_ast_pretty/src/pprust/state.rs @@ -140,12 +140,15 @@ pub fn print_crate<'a>( // and also addresses some specific regressions described in #63896 and #73345. fn tt_prepend_space(tt: &TokenTree, prev: &TokenTree) -> bool { if let TokenTree::Token(token) = prev { + if matches!(token.kind, token::Dot) { + return false; + } if let token::DocComment(comment_kind, ..) = token.kind { return comment_kind != CommentKind::Line; } } match tt { - TokenTree::Token(token) => token.kind != token::Comma, + TokenTree::Token(token) => !matches!(token.kind, token::Comma | token::Not | token::Dot), TokenTree::Delimited(_, DelimToken::Paren, _) => { !matches!(prev, TokenTree::Token(Token { kind: token::Ident(..), .. })) } @@ -955,6 +958,14 @@ impl<'a> State<'a> { } self.pclose(); } + ast::TyKind::AnonymousStruct(ref fields, ..) => { + self.head("struct"); + self.print_record_struct_body(&fields, ty.span); + } + ast::TyKind::AnonymousUnion(ref fields, ..) => { + self.head("union"); + self.print_record_struct_body(&fields, ty.span); + } ast::TyKind::Paren(ref typ) => { self.popen(); self.print_type(typ); @@ -1168,9 +1179,9 @@ impl<'a> State<'a> { self.print_foreign_mod(nmod, &item.attrs); self.bclose(item.span); } - ast::ItemKind::GlobalAsm(ref ga) => { + ast::ItemKind::GlobalAsm(ref asm) => { self.head(visibility_qualified(&item.vis, "global_asm!")); - self.s.word(ga.asm.to_string()); + self.print_inline_asm(asm); self.end(); } ast::ItemKind::TyAlias(box ast::TyAliasKind(def, ref generics, ref bounds, ref ty)) => { @@ -1390,6 +1401,24 @@ impl<'a> State<'a> { } } + crate fn print_record_struct_body(&mut self, fields: &[ast::FieldDef], span: rustc_span::Span) { + self.bopen(); + self.hardbreak_if_not_bol(); + + for field in fields { + self.hardbreak_if_not_bol(); + self.maybe_print_comment(field.span.lo()); + self.print_outer_attributes(&field.attrs); + self.print_visibility(&field.vis); + self.print_ident(field.ident.unwrap()); + self.word_nbsp(":"); + self.print_type(&field.ty); + self.s.word(","); + } + + self.bclose(span) + } + crate fn print_struct( &mut self, struct_def: &ast::VariantData, @@ -1419,24 +1448,10 @@ impl<'a> State<'a> { self.end(); self.end(); // Close the outer-box. } - ast::VariantData::Struct(..) => { + ast::VariantData::Struct(ref fields, ..) => { self.print_where_clause(&generics.where_clause); self.nbsp(); - self.bopen(); - self.hardbreak_if_not_bol(); - - for field in struct_def.fields() { - self.hardbreak_if_not_bol(); - self.maybe_print_comment(field.span.lo()); - self.print_outer_attributes(&field.attrs); - self.print_visibility(&field.vis); - self.print_ident(field.ident.unwrap()); - self.word_nbsp(":"); - self.print_type(&field.ty); - self.s.word(","); - } - - self.bclose(span) + self.print_record_struct_body(fields, span); } } } @@ -1675,32 +1690,24 @@ impl<'a> State<'a> { } } - fn print_expr_vec(&mut self, exprs: &[P], attrs: &[ast::Attribute]) { + fn print_expr_vec(&mut self, exprs: &[P]) { self.ibox(INDENT_UNIT); self.s.word("["); - self.print_inner_attributes_inline(attrs); self.commasep_exprs(Inconsistent, exprs); self.s.word("]"); self.end(); } - fn print_expr_anon_const(&mut self, expr: &ast::AnonConst, attrs: &[ast::Attribute]) { + fn print_expr_anon_const(&mut self, expr: &ast::AnonConst) { self.ibox(INDENT_UNIT); self.s.word("const"); - self.print_inner_attributes_inline(attrs); self.print_expr(&expr.value); self.end(); } - fn print_expr_repeat( - &mut self, - element: &ast::Expr, - count: &ast::AnonConst, - attrs: &[ast::Attribute], - ) { + fn print_expr_repeat(&mut self, element: &ast::Expr, count: &ast::AnonConst) { self.ibox(INDENT_UNIT); self.s.word("["); - self.print_inner_attributes_inline(attrs); self.print_expr(element); self.word_space(";"); self.print_expr(&count.value); @@ -1710,14 +1717,17 @@ impl<'a> State<'a> { fn print_expr_struct( &mut self, + qself: &Option, path: &ast::Path, fields: &[ast::ExprField], rest: &ast::StructRest, - attrs: &[ast::Attribute], ) { - self.print_path(path, true, 0); + if let Some(qself) = qself { + self.print_qpath(path, qself, true); + } else { + self.print_path(path, true, 0); + } self.s.word("{"); - self.print_inner_attributes_inline(attrs); self.commasep_cmnt( Consistent, fields, @@ -1752,9 +1762,8 @@ impl<'a> State<'a> { self.s.word("}"); } - fn print_expr_tup(&mut self, exprs: &[P], attrs: &[ast::Attribute]) { + fn print_expr_tup(&mut self, exprs: &[P]) { self.popen(); - self.print_inner_attributes_inline(attrs); self.commasep_exprs(Inconsistent, exprs); if exprs.len() == 1 { self.s.word(","); @@ -1865,19 +1874,19 @@ impl<'a> State<'a> { self.print_expr_maybe_paren(expr, parser::PREC_PREFIX); } ast::ExprKind::Array(ref exprs) => { - self.print_expr_vec(&exprs[..], attrs); + self.print_expr_vec(exprs); } ast::ExprKind::ConstBlock(ref anon_const) => { - self.print_expr_anon_const(anon_const, attrs); + self.print_expr_anon_const(anon_const); } ast::ExprKind::Repeat(ref element, ref count) => { - self.print_expr_repeat(element, count, attrs); + self.print_expr_repeat(element, count); } ast::ExprKind::Struct(ref se) => { - self.print_expr_struct(&se.path, &se.fields, &se.rest, attrs); + self.print_expr_struct(&se.qself, &se.path, &se.fields, &se.rest); } ast::ExprKind::Tup(ref exprs) => { - self.print_expr_tup(&exprs[..], attrs); + self.print_expr_tup(exprs); } ast::ExprKind::Call(ref func, ref args) => { self.print_expr_call(func, &args[..]); @@ -2082,117 +2091,8 @@ impl<'a> State<'a> { } } ast::ExprKind::InlineAsm(ref a) => { - enum AsmArg<'a> { - Template(String), - Operand(&'a InlineAsmOperand), - Options(InlineAsmOptions), - } - - let mut args = vec![]; - args.push(AsmArg::Template(InlineAsmTemplatePiece::to_string(&a.template))); - args.extend(a.operands.iter().map(|(o, _)| AsmArg::Operand(o))); - if !a.options.is_empty() { - args.push(AsmArg::Options(a.options)); - } - self.word("asm!"); - self.popen(); - self.commasep(Consistent, &args, |s, arg| match arg { - AsmArg::Template(template) => s.print_string(&template, ast::StrStyle::Cooked), - AsmArg::Operand(op) => { - let print_reg_or_class = |s: &mut Self, r: &InlineAsmRegOrRegClass| match r - { - InlineAsmRegOrRegClass::Reg(r) => { - s.print_symbol(*r, ast::StrStyle::Cooked) - } - InlineAsmRegOrRegClass::RegClass(r) => s.word(r.to_string()), - }; - match op { - InlineAsmOperand::In { reg, expr } => { - s.word("in"); - s.popen(); - print_reg_or_class(s, reg); - s.pclose(); - s.space(); - s.print_expr(expr); - } - InlineAsmOperand::Out { reg, late, expr } => { - s.word(if *late { "lateout" } else { "out" }); - s.popen(); - print_reg_or_class(s, reg); - s.pclose(); - s.space(); - match expr { - Some(expr) => s.print_expr(expr), - None => s.word("_"), - } - } - InlineAsmOperand::InOut { reg, late, expr } => { - s.word(if *late { "inlateout" } else { "inout" }); - s.popen(); - print_reg_or_class(s, reg); - s.pclose(); - s.space(); - s.print_expr(expr); - } - InlineAsmOperand::SplitInOut { reg, late, in_expr, out_expr } => { - s.word(if *late { "inlateout" } else { "inout" }); - s.popen(); - print_reg_or_class(s, reg); - s.pclose(); - s.space(); - s.print_expr(in_expr); - s.space(); - s.word_space("=>"); - match out_expr { - Some(out_expr) => s.print_expr(out_expr), - None => s.word("_"), - } - } - InlineAsmOperand::Const { anon_const } => { - s.word("const"); - s.space(); - s.print_expr(&anon_const.value); - } - InlineAsmOperand::Sym { expr } => { - s.word("sym"); - s.space(); - s.print_expr(expr); - } - } - } - AsmArg::Options(opts) => { - s.word("options"); - s.popen(); - let mut options = vec![]; - if opts.contains(InlineAsmOptions::PURE) { - options.push("pure"); - } - if opts.contains(InlineAsmOptions::NOMEM) { - options.push("nomem"); - } - if opts.contains(InlineAsmOptions::READONLY) { - options.push("readonly"); - } - if opts.contains(InlineAsmOptions::PRESERVES_FLAGS) { - options.push("preserves_flags"); - } - if opts.contains(InlineAsmOptions::NORETURN) { - options.push("noreturn"); - } - if opts.contains(InlineAsmOptions::NOSTACK) { - options.push("nostack"); - } - if opts.contains(InlineAsmOptions::ATT_SYNTAX) { - options.push("att_syntax"); - } - s.commasep(Inconsistent, &options, |s, &opt| { - s.word(opt); - }); - s.pclose(); - } - }); - self.pclose(); + self.print_inline_asm(a); } ast::ExprKind::LlvmInlineAsm(ref a) => { self.s.word("llvm_asm!"); @@ -2253,7 +2153,6 @@ impl<'a> State<'a> { ast::ExprKind::MacCall(ref m) => self.print_mac(m), ast::ExprKind::Paren(ref e) => { self.popen(); - self.print_inner_attributes_inline(attrs); self.print_expr(e); self.pclose(); } @@ -2284,6 +2183,116 @@ impl<'a> State<'a> { self.end(); } + fn print_inline_asm(&mut self, asm: &ast::InlineAsm) { + enum AsmArg<'a> { + Template(String), + Operand(&'a InlineAsmOperand), + Options(InlineAsmOptions), + } + + let mut args = vec![]; + args.push(AsmArg::Template(InlineAsmTemplatePiece::to_string(&asm.template))); + args.extend(asm.operands.iter().map(|(o, _)| AsmArg::Operand(o))); + if !asm.options.is_empty() { + args.push(AsmArg::Options(asm.options)); + } + + self.popen(); + self.commasep(Consistent, &args, |s, arg| match arg { + AsmArg::Template(template) => s.print_string(&template, ast::StrStyle::Cooked), + AsmArg::Operand(op) => { + let print_reg_or_class = |s: &mut Self, r: &InlineAsmRegOrRegClass| match r { + InlineAsmRegOrRegClass::Reg(r) => s.print_symbol(*r, ast::StrStyle::Cooked), + InlineAsmRegOrRegClass::RegClass(r) => s.word(r.to_string()), + }; + match op { + InlineAsmOperand::In { reg, expr } => { + s.word("in"); + s.popen(); + print_reg_or_class(s, reg); + s.pclose(); + s.space(); + s.print_expr(expr); + } + InlineAsmOperand::Out { reg, late, expr } => { + s.word(if *late { "lateout" } else { "out" }); + s.popen(); + print_reg_or_class(s, reg); + s.pclose(); + s.space(); + match expr { + Some(expr) => s.print_expr(expr), + None => s.word("_"), + } + } + InlineAsmOperand::InOut { reg, late, expr } => { + s.word(if *late { "inlateout" } else { "inout" }); + s.popen(); + print_reg_or_class(s, reg); + s.pclose(); + s.space(); + s.print_expr(expr); + } + InlineAsmOperand::SplitInOut { reg, late, in_expr, out_expr } => { + s.word(if *late { "inlateout" } else { "inout" }); + s.popen(); + print_reg_or_class(s, reg); + s.pclose(); + s.space(); + s.print_expr(in_expr); + s.space(); + s.word_space("=>"); + match out_expr { + Some(out_expr) => s.print_expr(out_expr), + None => s.word("_"), + } + } + InlineAsmOperand::Const { anon_const } => { + s.word("const"); + s.space(); + s.print_expr(&anon_const.value); + } + InlineAsmOperand::Sym { expr } => { + s.word("sym"); + s.space(); + s.print_expr(expr); + } + } + } + AsmArg::Options(opts) => { + s.word("options"); + s.popen(); + let mut options = vec![]; + if opts.contains(InlineAsmOptions::PURE) { + options.push("pure"); + } + if opts.contains(InlineAsmOptions::NOMEM) { + options.push("nomem"); + } + if opts.contains(InlineAsmOptions::READONLY) { + options.push("readonly"); + } + if opts.contains(InlineAsmOptions::PRESERVES_FLAGS) { + options.push("preserves_flags"); + } + if opts.contains(InlineAsmOptions::NORETURN) { + options.push("noreturn"); + } + if opts.contains(InlineAsmOptions::NOSTACK) { + options.push("nostack"); + } + if opts.contains(InlineAsmOptions::ATT_SYNTAX) { + options.push("att_syntax"); + } + s.commasep(Inconsistent, &options, |s, &opt| { + s.word(opt); + }); + s.pclose(); + } + }); + self.pclose(); + } + crate fn print_local_decl(&mut self, loc: &ast::Local) { self.print_pat(&loc.pat); if let Some(ref ty) = loc.ty { @@ -2341,8 +2350,12 @@ impl<'a> State<'a> { self.print_pat(p); } } - PatKind::TupleStruct(ref path, ref elts) => { - self.print_path(path, true, 0); + PatKind::TupleStruct(ref qself, ref path, ref elts) => { + if let Some(qself) = qself { + self.print_qpath(path, qself, true); + } else { + self.print_path(path, true, 0); + } self.popen(); self.commasep(Inconsistent, &elts[..], |s, p| s.print_pat(p)); self.pclose(); @@ -2356,8 +2369,12 @@ impl<'a> State<'a> { PatKind::Path(Some(ref qself), ref path) => { self.print_qpath(path, qself, false); } - PatKind::Struct(ref path, ref fields, etc) => { - self.print_path(path, true, 0); + PatKind::Struct(ref qself, ref path, ref fields, etc) => { + if let Some(qself) = qself { + self.print_qpath(path, qself, true); + } else { + self.print_path(path, true, 0); + } self.nbsp(); self.word_space("{"); self.commasep_cmnt( diff --git a/compiler/rustc_attr/src/lib.rs b/compiler/rustc_attr/src/lib.rs index ab68d24e4b..3fb11f7787 100644 --- a/compiler/rustc_attr/src/lib.rs +++ b/compiler/rustc_attr/src/lib.rs @@ -4,8 +4,6 @@ //! The goal is to move the definition of `MetaItem` and things that don't need to be in `syntax` //! to this crate. -#![cfg_attr(bootstrap, feature(or_patterns))] - #[macro_use] extern crate rustc_macros; diff --git a/compiler/rustc_builtin_macros/src/asm.rs b/compiler/rustc_builtin_macros/src/asm.rs index fd976b119b..b28c6f0d99 100644 --- a/compiler/rustc_builtin_macros/src/asm.rs +++ b/compiler/rustc_builtin_macros/src/asm.rs @@ -8,9 +8,11 @@ use rustc_expand::base::{self, *}; use rustc_parse::parser::Parser; use rustc_parse_format as parse; use rustc_session::lint; +use rustc_span::symbol::Ident; use rustc_span::symbol::{kw, sym, Symbol}; use rustc_span::{InnerSpan, Span}; use rustc_target::asm::InlineAsmArch; +use smallvec::smallvec; struct AsmArgs { templates: Vec>, @@ -25,6 +27,7 @@ fn parse_args<'a>( ecx: &mut ExtCtxt<'a>, sp: Span, tts: TokenStream, + is_global_asm: bool, ) -> Result> { let mut p = ecx.new_parser_from_tts(tts); @@ -33,7 +36,7 @@ fn parse_args<'a>( } // Detect use of the legacy llvm_asm! syntax (which used to be called asm!) - if p.look_ahead(1, |t| *t == token::Colon || *t == token::ModSep) { + if !is_global_asm && p.look_ahead(1, |t| *t == token::Colon || *t == token::ModSep) { let mut err = ecx.struct_span_err(sp, "the legacy LLVM-style asm! syntax is no longer supported"); err.note("consider migrating to the new asm! syntax specified in RFC 2873"); @@ -84,7 +87,7 @@ fn parse_args<'a>( // Parse options if p.eat_keyword(sym::options) { - parse_options(&mut p, &mut args)?; + parse_options(&mut p, &mut args, is_global_asm)?; allow_templates = false; continue; } @@ -103,19 +106,19 @@ fn parse_args<'a>( }; let mut explicit_reg = false; - let op = if p.eat_keyword(kw::In) { + let op = if !is_global_asm && p.eat_keyword(kw::In) { let reg = parse_reg(&mut p, &mut explicit_reg)?; let expr = p.parse_expr()?; ast::InlineAsmOperand::In { reg, expr } - } else if p.eat_keyword(sym::out) { + } else if !is_global_asm && p.eat_keyword(sym::out) { let reg = parse_reg(&mut p, &mut explicit_reg)?; let expr = if p.eat_keyword(kw::Underscore) { None } else { Some(p.parse_expr()?) }; ast::InlineAsmOperand::Out { reg, expr, late: false } - } else if p.eat_keyword(sym::lateout) { + } else if !is_global_asm && p.eat_keyword(sym::lateout) { let reg = parse_reg(&mut p, &mut explicit_reg)?; let expr = if p.eat_keyword(kw::Underscore) { None } else { Some(p.parse_expr()?) }; ast::InlineAsmOperand::Out { reg, expr, late: true } - } else if p.eat_keyword(sym::inout) { + } else if !is_global_asm && p.eat_keyword(sym::inout) { let reg = parse_reg(&mut p, &mut explicit_reg)?; let expr = p.parse_expr()?; if p.eat(&token::FatArrow) { @@ -125,7 +128,7 @@ fn parse_args<'a>( } else { ast::InlineAsmOperand::InOut { reg, expr, late: false } } - } else if p.eat_keyword(sym::inlateout) { + } else if !is_global_asm && p.eat_keyword(sym::inlateout) { let reg = parse_reg(&mut p, &mut explicit_reg)?; let expr = p.parse_expr()?; if p.eat(&token::FatArrow) { @@ -138,7 +141,7 @@ fn parse_args<'a>( } else if p.eat_keyword(kw::Const) { let anon_const = p.parse_anon_const_expr()?; ast::InlineAsmOperand::Const { anon_const } - } else if p.eat_keyword(sym::sym) { + } else if !is_global_asm && p.eat_keyword(sym::sym) { let expr = p.parse_expr()?; match expr.kind { ast::ExprKind::Path(..) => {} @@ -329,23 +332,27 @@ fn try_set_option<'a>( } } -fn parse_options<'a>(p: &mut Parser<'a>, args: &mut AsmArgs) -> Result<(), DiagnosticBuilder<'a>> { +fn parse_options<'a>( + p: &mut Parser<'a>, + args: &mut AsmArgs, + is_global_asm: bool, +) -> Result<(), DiagnosticBuilder<'a>> { let span_start = p.prev_token.span; p.expect(&token::OpenDelim(token::DelimToken::Paren))?; while !p.eat(&token::CloseDelim(token::DelimToken::Paren)) { - if p.eat_keyword(sym::pure) { + if !is_global_asm && p.eat_keyword(sym::pure) { try_set_option(p, args, sym::pure, ast::InlineAsmOptions::PURE); - } else if p.eat_keyword(sym::nomem) { + } else if !is_global_asm && p.eat_keyword(sym::nomem) { try_set_option(p, args, sym::nomem, ast::InlineAsmOptions::NOMEM); - } else if p.eat_keyword(sym::readonly) { + } else if !is_global_asm && p.eat_keyword(sym::readonly) { try_set_option(p, args, sym::readonly, ast::InlineAsmOptions::READONLY); - } else if p.eat_keyword(sym::preserves_flags) { + } else if !is_global_asm && p.eat_keyword(sym::preserves_flags) { try_set_option(p, args, sym::preserves_flags, ast::InlineAsmOptions::PRESERVES_FLAGS); - } else if p.eat_keyword(sym::noreturn) { + } else if !is_global_asm && p.eat_keyword(sym::noreturn) { try_set_option(p, args, sym::noreturn, ast::InlineAsmOptions::NORETURN); - } else if p.eat_keyword(sym::nostack) { + } else if !is_global_asm && p.eat_keyword(sym::nostack) { try_set_option(p, args, sym::nostack, ast::InlineAsmOptions::NOSTACK); } else if p.eat_keyword(sym::att_syntax) { try_set_option(p, args, sym::att_syntax, ast::InlineAsmOptions::ATT_SYNTAX); @@ -388,7 +395,7 @@ fn parse_reg<'a>( Ok(result) } -fn expand_preparsed_asm(ecx: &mut ExtCtxt<'_>, sp: Span, args: AsmArgs) -> P { +fn expand_preparsed_asm(ecx: &mut ExtCtxt<'_>, args: AsmArgs) -> Option { let mut template = vec![]; // Register operands are implicitly used since they are not allowed to be // referenced in the template string. @@ -415,7 +422,7 @@ fn expand_preparsed_asm(ecx: &mut ExtCtxt<'_>, sp: Span, args: AsmArgs) -> P, sp: Span, args: AsmArgs) -> P, sp: Span, args: AsmArgs) -> P( @@ -659,8 +658,53 @@ pub fn expand_asm<'cx>( sp: Span, tts: TokenStream, ) -> Box { - match parse_args(ecx, sp, tts) { - Ok(args) => MacEager::expr(expand_preparsed_asm(ecx, sp, args)), + match parse_args(ecx, sp, tts, false) { + Ok(args) => { + let expr = if let Some(inline_asm) = expand_preparsed_asm(ecx, args) { + P(ast::Expr { + id: ast::DUMMY_NODE_ID, + kind: ast::ExprKind::InlineAsm(P(inline_asm)), + span: sp, + attrs: ast::AttrVec::new(), + tokens: None, + }) + } else { + DummyResult::raw_expr(sp, true) + }; + MacEager::expr(expr) + } + Err(mut err) => { + err.emit(); + DummyResult::any(sp) + } + } +} + +pub fn expand_global_asm<'cx>( + ecx: &'cx mut ExtCtxt<'_>, + sp: Span, + tts: TokenStream, +) -> Box { + match parse_args(ecx, sp, tts, true) { + Ok(args) => { + if let Some(inline_asm) = expand_preparsed_asm(ecx, args) { + MacEager::items(smallvec![P(ast::Item { + ident: Ident::invalid(), + attrs: Vec::new(), + id: ast::DUMMY_NODE_ID, + kind: ast::ItemKind::GlobalAsm(inline_asm), + vis: ast::Visibility { + span: sp.shrink_to_lo(), + kind: ast::VisibilityKind::Inherited, + tokens: None, + }, + span: ecx.with_def_site_ctxt(sp), + tokens: None, + })]) + } else { + DummyResult::any(sp) + } + } Err(mut err) => { err.emit(); DummyResult::any(sp) diff --git a/compiler/rustc_builtin_macros/src/global_asm.rs b/compiler/rustc_builtin_macros/src/global_asm.rs deleted file mode 100644 index 76d8745292..0000000000 --- a/compiler/rustc_builtin_macros/src/global_asm.rs +++ /dev/null @@ -1,68 +0,0 @@ -//! Module-level assembly support. -//! -//! The macro defined here allows you to specify "top-level", -//! "file-scoped", or "module-level" assembly. These synonyms -//! all correspond to LLVM's module-level inline assembly instruction. -//! -//! For example, `global_asm!("some assembly here")` codegens to -//! LLVM's `module asm "some assembly here"`. All of LLVM's caveats -//! therefore apply. - -use rustc_ast as ast; -use rustc_ast::ptr::P; -use rustc_ast::token; -use rustc_ast::tokenstream::TokenStream; -use rustc_errors::DiagnosticBuilder; -use rustc_expand::base::{self, *}; -use rustc_span::symbol::Ident; -use rustc_span::Span; -use smallvec::smallvec; - -pub fn expand_global_asm<'cx>( - cx: &'cx mut ExtCtxt<'_>, - sp: Span, - tts: TokenStream, -) -> Box { - match parse_global_asm(cx, sp, tts) { - Ok(Some(global_asm)) => MacEager::items(smallvec![P(ast::Item { - ident: Ident::invalid(), - attrs: Vec::new(), - id: ast::DUMMY_NODE_ID, - kind: ast::ItemKind::GlobalAsm(global_asm), - vis: ast::Visibility { - span: sp.shrink_to_lo(), - kind: ast::VisibilityKind::Inherited, - tokens: None, - }, - span: cx.with_def_site_ctxt(sp), - tokens: None, - })]), - Ok(None) => DummyResult::any(sp), - Err(mut err) => { - err.emit(); - DummyResult::any(sp) - } - } -} - -fn parse_global_asm<'a>( - cx: &mut ExtCtxt<'a>, - sp: Span, - tts: TokenStream, -) -> Result, DiagnosticBuilder<'a>> { - let mut p = cx.new_parser_from_tts(tts); - - if p.token == token::Eof { - let mut err = cx.struct_span_err(sp, "macro requires a string literal as an argument"); - err.span_label(sp, "string literal required"); - return Err(err); - } - - let expr = p.parse_expr()?; - let (asm, _) = match expr_to_string(cx, expr, "inline assembly must be a string literal") { - Some((s, st)) => (s, st), - None => return Ok(None), - }; - - Ok(Some(ast::GlobalAsm { asm })) -} diff --git a/compiler/rustc_builtin_macros/src/lib.rs b/compiler/rustc_builtin_macros/src/lib.rs index d7926ed0e0..2393e0b9eb 100644 --- a/compiler/rustc_builtin_macros/src/lib.rs +++ b/compiler/rustc_builtin_macros/src/lib.rs @@ -9,7 +9,6 @@ #![feature(decl_macro)] #![feature(iter_zip)] #![feature(nll)] -#![cfg_attr(bootstrap, feature(or_patterns))] #![feature(proc_macro_internals)] #![feature(proc_macro_quote)] #![recursion_limit = "256"] @@ -20,6 +19,7 @@ use crate::deriving::*; use rustc_expand::base::{MacroExpanderFn, ResolverExpand, SyntaxExtensionKind}; use rustc_expand::proc_macro::BangProcMacro; +use rustc_span::def_id::LOCAL_CRATE; use rustc_span::symbol::sym; mod asm; @@ -36,7 +36,6 @@ mod env; mod format; mod format_foreign; mod global_allocator; -mod global_asm; mod llvm_asm; mod log_syntax; mod panic; @@ -74,7 +73,7 @@ pub fn register_builtin_macros(resolver: &mut dyn ResolverExpand) { file: source_util::expand_file, format_args_nl: format::expand_format_args_nl, format_args: format::expand_format_args, - global_asm: global_asm::expand_global_asm, + global_asm: asm::expand_global_asm, include_bytes: source_util::expand_include_bytes, include_str: source_util::expand_include_str, include: source_util::expand_include, @@ -114,5 +113,8 @@ pub fn register_builtin_macros(resolver: &mut dyn ResolverExpand) { } let client = proc_macro::bridge::client::Client::expand1(proc_macro::quote); - register(sym::quote, SyntaxExtensionKind::Bang(Box::new(BangProcMacro { client }))); + register( + sym::quote, + SyntaxExtensionKind::Bang(Box::new(BangProcMacro { client, krate: LOCAL_CRATE })), + ); } diff --git a/compiler/rustc_builtin_macros/src/source_util.rs b/compiler/rustc_builtin_macros/src/source_util.rs index 4aafcb2fb6..ccb9f15b0f 100644 --- a/compiler/rustc_builtin_macros/src/source_util.rs +++ b/compiler/rustc_builtin_macros/src/source_util.rs @@ -61,7 +61,9 @@ pub fn expand_file( let topmost = cx.expansion_cause().unwrap_or(sp); let loc = cx.source_map().lookup_char_pos(topmost.lo()); - base::MacEager::expr(cx.expr_str(topmost, Symbol::intern(&loc.file.name.to_string()))) + base::MacEager::expr( + cx.expr_str(topmost, Symbol::intern(&loc.file.name.prefer_remapped().to_string_lossy())), + ) } pub fn expand_stringify( diff --git a/compiler/rustc_builtin_macros/src/test.rs b/compiler/rustc_builtin_macros/src/test.rs index e845f9ec55..99544ddb66 100644 --- a/compiler/rustc_builtin_macros/src/test.rs +++ b/compiler/rustc_builtin_macros/src/test.rs @@ -254,6 +254,10 @@ pub fn expand_test_or_bench( "allow_fail", cx.expr_bool(sp, should_fail(&cx.sess, &item)), ), + // compile_fail: true | false + field("compile_fail", cx.expr_bool(sp, false)), + // no_run: true | false + field("no_run", cx.expr_bool(sp, false)), // should_panic: ... field( "should_panic", diff --git a/compiler/rustc_codegen_cranelift/.github/workflows/main.yml b/compiler/rustc_codegen_cranelift/.github/workflows/main.yml index 2ac516381c..4d45e36c95 100644 --- a/compiler/rustc_codegen_cranelift/.github/workflows/main.yml +++ b/compiler/rustc_codegen_cranelift/.github/workflows/main.yml @@ -80,3 +80,10 @@ jobs: with: name: cg_clif-${{ runner.os }} path: cg_clif.tar.xz + + - name: Upload prebuilt cg_clif (cross compile) + if: matrix.env.TARGET_TRIPLE == 'x86_64-pc-windows-gnu' + uses: actions/upload-artifact@v2 + with: + name: cg_clif-${{ runner.os }}-cross-x86_64-mingw + path: cg_clif.tar.xz diff --git a/compiler/rustc_codegen_cranelift/.vscode/settings.json b/compiler/rustc_codegen_cranelift/.vscode/settings.json index 0cd576e160..9009a532c5 100644 --- a/compiler/rustc_codegen_cranelift/.vscode/settings.json +++ b/compiler/rustc_codegen_cranelift/.vscode/settings.json @@ -1,6 +1,6 @@ { // source for rustc_* is not included in the rust-src component; disable the errors about this - "rust-analyzer.diagnostics.disabled": ["unresolved-extern-crate", "macro-error"], + "rust-analyzer.diagnostics.disabled": ["unresolved-extern-crate", "unresolved-macro-call"], "rust-analyzer.assist.importMergeBehavior": "last", "rust-analyzer.cargo.runBuildScripts": true, "rust-analyzer.linkedProjects": [ diff --git a/compiler/rustc_codegen_cranelift/Cargo.lock b/compiler/rustc_codegen_cranelift/Cargo.lock index dc1cd336e1..a6f5925149 100644 --- a/compiler/rustc_codegen_cranelift/Cargo.lock +++ b/compiler/rustc_codegen_cranelift/Cargo.lock @@ -25,12 +25,6 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693" -[[package]] -name = "byteorder" -version = "1.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae44d1a3d5a19df61dd0c8beb138458ac2a53a7ac09eba97d55592540004306b" - [[package]] name = "cfg-if" version = "1.0.0" @@ -39,18 +33,17 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "cranelift-bforest" -version = "0.72.0" -source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8e43e96410a14143d368273cf1e708f8094bb8e0" +version = "0.74.0" +source = "git+https://github.com/bytecodealliance/wasmtime.git?branch=main#76c6b83f6a21a12a11d4f890490f8acb6329a600" dependencies = [ "cranelift-entity", ] [[package]] name = "cranelift-codegen" -version = "0.72.0" -source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8e43e96410a14143d368273cf1e708f8094bb8e0" +version = "0.74.0" +source = "git+https://github.com/bytecodealliance/wasmtime.git?branch=main#76c6b83f6a21a12a11d4f890490f8acb6329a600" dependencies = [ - "byteorder", "cranelift-bforest", "cranelift-codegen-meta", "cranelift-codegen-shared", @@ -60,13 +53,12 @@ dependencies = [ "regalloc", "smallvec", "target-lexicon", - "thiserror", ] [[package]] name = "cranelift-codegen-meta" -version = "0.72.0" -source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8e43e96410a14143d368273cf1e708f8094bb8e0" +version = "0.74.0" +source = "git+https://github.com/bytecodealliance/wasmtime.git?branch=main#76c6b83f6a21a12a11d4f890490f8acb6329a600" dependencies = [ "cranelift-codegen-shared", "cranelift-entity", @@ -74,18 +66,18 @@ dependencies = [ [[package]] name = "cranelift-codegen-shared" -version = "0.72.0" -source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8e43e96410a14143d368273cf1e708f8094bb8e0" +version = "0.74.0" +source = "git+https://github.com/bytecodealliance/wasmtime.git?branch=main#76c6b83f6a21a12a11d4f890490f8acb6329a600" [[package]] name = "cranelift-entity" -version = "0.72.0" -source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8e43e96410a14143d368273cf1e708f8094bb8e0" +version = "0.74.0" +source = "git+https://github.com/bytecodealliance/wasmtime.git?branch=main#76c6b83f6a21a12a11d4f890490f8acb6329a600" [[package]] name = "cranelift-frontend" -version = "0.72.0" -source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8e43e96410a14143d368273cf1e708f8094bb8e0" +version = "0.74.0" +source = "git+https://github.com/bytecodealliance/wasmtime.git?branch=main#76c6b83f6a21a12a11d4f890490f8acb6329a600" dependencies = [ "cranelift-codegen", "log", @@ -95,15 +87,14 @@ dependencies = [ [[package]] name = "cranelift-jit" -version = "0.72.0" -source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8e43e96410a14143d368273cf1e708f8094bb8e0" +version = "0.74.0" +source = "git+https://github.com/bytecodealliance/wasmtime.git?branch=main#76c6b83f6a21a12a11d4f890490f8acb6329a600" dependencies = [ "anyhow", "cranelift-codegen", "cranelift-entity", "cranelift-module", "cranelift-native", - "errno", "libc", "log", "region", @@ -113,20 +104,19 @@ dependencies = [ [[package]] name = "cranelift-module" -version = "0.72.0" -source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8e43e96410a14143d368273cf1e708f8094bb8e0" +version = "0.74.0" +source = "git+https://github.com/bytecodealliance/wasmtime.git?branch=main#76c6b83f6a21a12a11d4f890490f8acb6329a600" dependencies = [ "anyhow", "cranelift-codegen", "cranelift-entity", "log", - "thiserror", ] [[package]] name = "cranelift-native" -version = "0.72.0" -source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8e43e96410a14143d368273cf1e708f8094bb8e0" +version = "0.74.0" +source = "git+https://github.com/bytecodealliance/wasmtime.git?branch=main#76c6b83f6a21a12a11d4f890490f8acb6329a600" dependencies = [ "cranelift-codegen", "target-lexicon", @@ -134,8 +124,8 @@ dependencies = [ [[package]] name = "cranelift-object" -version = "0.72.0" -source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8e43e96410a14143d368273cf1e708f8094bb8e0" +version = "0.74.0" +source = "git+https://github.com/bytecodealliance/wasmtime.git?branch=main#76c6b83f6a21a12a11d4f890490f8acb6329a600" dependencies = [ "anyhow", "cranelift-codegen", @@ -154,38 +144,11 @@ dependencies = [ "cfg-if", ] -[[package]] -name = "errno" -version = "0.2.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa68f2fb9cae9d37c9b2b3584aba698a2e97f72d7aef7b9f7aa71d8b54ce46fe" -dependencies = [ - "errno-dragonfly", - "libc", - "winapi", -] - -[[package]] -name = "errno-dragonfly" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14ca354e36190500e1e1fb267c647932382b54053c50b14970856c0b00a35067" -dependencies = [ - "gcc", - "libc", -] - -[[package]] -name = "gcc" -version = "0.3.55" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f5f3913fa0bfe7ee1fd8248b6b9f42a5af4b9d65ec2dd2c3c26132b950ecfc2" - [[package]] name = "gimli" -version = "0.23.0" +version = "0.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6503fe142514ca4799d4c26297c4248239fe8838d827db6bd6065c6ed29a6ce" +checksum = "0e4075386626662786ddb0ec9081e7c7eeb1ba31951f447ca780ef9f5d568189" dependencies = [ "indexmap", ] @@ -242,32 +205,14 @@ dependencies = [ [[package]] name = "object" -version = "0.23.0" +version = "0.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9a7ab5d64814df0fe4a4b5ead45ed6c5f181ee3ff04ba344313a6c80446c5d4" +checksum = "1a5b3dd1c072ee7963717671d1ca129f1048fda25edea6b752bfc71ac8854170" dependencies = [ "crc32fast", "indexmap", ] -[[package]] -name = "proc-macro2" -version = "1.0.24" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e0704ee1a7e00d7bb417d0770ea303c1bccbabf0ef1667dae92b5967f5f8a71" -dependencies = [ - "unicode-xid", -] - -[[package]] -name = "quote" -version = "1.0.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3d0b9745dc2debf507c8422de05d7226cc1f0644216dfdfead988f9b1ab32a7" -dependencies = [ - "proc-macro2", -] - [[package]] name = "regalloc" version = "0.0.31" @@ -306,6 +251,7 @@ dependencies = [ "cranelift-frontend", "cranelift-jit", "cranelift-module", + "cranelift-native", "cranelift-object", "gimli", "indexmap", @@ -321,48 +267,11 @@ version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fe0f37c9e8f3c5a4a66ad655a93c74daac4ad00c441533bf5c6e7990bb42604e" -[[package]] -name = "syn" -version = "1.0.60" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c700597eca8a5a762beb35753ef6b94df201c81cca676604f547495a0d7f0081" -dependencies = [ - "proc-macro2", - "quote", - "unicode-xid", -] - [[package]] name = "target-lexicon" -version = "0.11.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "422045212ea98508ae3d28025bc5aaa2bd4a9cdaecd442a08da2ee620ee9ea95" - -[[package]] -name = "thiserror" -version = "1.0.24" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0f4a65597094d4483ddaed134f409b2cb7c1beccf25201a9f73c719254fa98e" -dependencies = [ - "thiserror-impl", -] - -[[package]] -name = "thiserror-impl" -version = "1.0.24" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7765189610d8241a44529806d6fd1f2e0a08734313a35d5b3a556f92b381f3c0" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "unicode-xid" -version = "0.2.1" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7fe0bb3479651439c9112f72b6c505038574c9fbb575ed1bf3b797fa39dd564" +checksum = "64ae3b39281e4b14b8123bdbaddd472b7dfe215e444181f2f9d2443c2444f834" [[package]] name = "winapi" diff --git a/compiler/rustc_codegen_cranelift/Cargo.toml b/compiler/rustc_codegen_cranelift/Cargo.toml index 60946ab280..fd149af454 100644 --- a/compiler/rustc_codegen_cranelift/Cargo.toml +++ b/compiler/rustc_codegen_cranelift/Cargo.toml @@ -9,14 +9,15 @@ crate-type = ["dylib"] [dependencies] # These have to be in sync with each other -cranelift-codegen = { git = "https://github.com/bytecodealliance/wasmtime/", branch = "main", features = ["unwind", "x64"] } -cranelift-frontend = { git = "https://github.com/bytecodealliance/wasmtime/", branch = "main" } -cranelift-module = { git = "https://github.com/bytecodealliance/wasmtime/", branch = "main" } -cranelift-jit = { git = "https://github.com/bytecodealliance/wasmtime/", branch = "main", optional = true } -cranelift-object = { git = "https://github.com/bytecodealliance/wasmtime/", branch = "main" } -target-lexicon = "0.11.0" -gimli = { version = "0.23.0", default-features = false, features = ["write"]} -object = { version = "0.23.0", default-features = false, features = ["std", "read_core", "write", "archive", "coff", "elf", "macho", "pe"] } +cranelift-codegen = { git = "https://github.com/bytecodealliance/wasmtime.git", branch = "main", features = ["unwind"] } +cranelift-frontend = { git = "https://github.com/bytecodealliance/wasmtime.git", branch = "main" } +cranelift-module = { git = "https://github.com/bytecodealliance/wasmtime.git", branch = "main" } +cranelift-native = { git = "https://github.com/bytecodealliance/wasmtime.git", branch = "main" } +cranelift-jit = { git = "https://github.com/bytecodealliance/wasmtime.git", branch = "main", optional = true } +cranelift-object = { git = "https://github.com/bytecodealliance/wasmtime.git", branch = "main" } +target-lexicon = "0.12.0" +gimli = { version = "0.24.0", default-features = false, features = ["write"]} +object = { version = "0.24.0", default-features = false, features = ["std", "read_core", "write", "archive", "coff", "elf", "macho", "pe"] } ar = { git = "https://github.com/bjorn3/rust-ar.git", branch = "do_not_remove_cg_clif_ranlib" } indexmap = "1.0.2" @@ -24,10 +25,11 @@ libloading = { version = "0.6.0", optional = true } smallvec = "1.6.1" # Uncomment to use local checkout of cranelift -#[patch."https://github.com/bytecodealliance/wasmtime/"] +#[patch."https://github.com/bytecodealliance/wasmtime.git"] #cranelift-codegen = { path = "../wasmtime/cranelift/codegen" } #cranelift-frontend = { path = "../wasmtime/cranelift/frontend" } #cranelift-module = { path = "../wasmtime/cranelift/module" } +#cranelift-native = { path = "../wasmtime/cranelift/native" } #cranelift-jit = { path = "../wasmtime/cranelift/jit" } #cranelift-object = { path = "../wasmtime/cranelift/object" } @@ -68,13 +70,5 @@ debug = false opt-level = 0 debug = false -[profile.dev.package.syn] -opt-level = 0 -debug = false - -[profile.release.package.syn] -opt-level = 0 -debug = false - [package.metadata.rust-analyzer] rustc_private = true diff --git a/compiler/rustc_codegen_cranelift/Readme.md b/compiler/rustc_codegen_cranelift/Readme.md index ffe1d9a1e6..08f9373be6 100644 --- a/compiler/rustc_codegen_cranelift/Readme.md +++ b/compiler/rustc_codegen_cranelift/Readme.md @@ -44,9 +44,10 @@ This will build your project with rustc_codegen_cranelift instead of the usual L For additional ways to use rustc_codegen_cranelift like the JIT mode see [usage.md](docs/usage.md). -## Env vars +## Configuration -See [env_vars.md](docs/env_vars.md) for all env vars used by rustc_codegen_cranelift. +See the documentation on the `BackendConfig` struct in [config.rs](src/config.rs) for all +configuration options. ## Not yet supported diff --git a/compiler/rustc_codegen_cranelift/build_sysroot/Cargo.lock b/compiler/rustc_codegen_cranelift/build_sysroot/Cargo.lock index 09c5d7590a..923deb9aec 100644 --- a/compiler/rustc_codegen_cranelift/build_sysroot/Cargo.lock +++ b/compiler/rustc_codegen_cranelift/build_sysroot/Cargo.lock @@ -40,9 +40,9 @@ checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a" [[package]] name = "cc" -version = "1.0.67" +version = "1.0.68" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3c69b077ad434294d3ce9f1f6143a2a4b89a8a2d54ef813d85003a4fd1137fd" +checksum = "4a72c244c1ff497a746a7e1fb3d14bd08420ecda70c8f25c7112f2781652d787" [[package]] name = "cfg-if" @@ -56,7 +56,7 @@ dependencies = [ [[package]] name = "compiler_builtins" -version = "0.1.39" +version = "0.1.43" dependencies = [ "rustc-std-workspace-core", ] @@ -132,9 +132,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.91" +version = "0.2.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8916b1f6ca17130ec6568feccee27c156ad12037880833a3b842a823236502e7" +checksum = "789da6d93f1b866ffe175afc5322a4d76c038605a1c3319bb57b06967ca98a36" dependencies = [ "rustc-std-workspace-core", ] @@ -167,6 +167,7 @@ dependencies = [ name = "panic_abort" version = "0.0.0" dependencies = [ + "alloc", "cfg-if", "compiler_builtins", "core", @@ -194,9 +195,9 @@ dependencies = [ [[package]] name = "rustc-demangle" -version = "0.1.18" +version = "0.1.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e3bad0ee36814ca07d7968269dd4b7ec89ec2da10c4bb613928d3077083c232" +checksum = "410f7acf3cb3a44527c5d9546bad4bf4e6c460915d5f9f2fc524498bfe8f70ce" dependencies = [ "compiler_builtins", "rustc-std-workspace-core", @@ -242,10 +243,22 @@ dependencies = [ "panic_abort", "panic_unwind", "rustc-demangle", + "std_detect", "unwind", "wasi", ] +[[package]] +name = "std_detect" +version = "0.1.5" +dependencies = [ + "cfg-if", + "compiler_builtins", + "libc", + "rustc-std-workspace-alloc", + "rustc-std-workspace-core", +] + [[package]] name = "sysroot" version = "0.0.0" diff --git a/compiler/rustc_codegen_cranelift/build_sysroot/prepare_sysroot_src.sh b/compiler/rustc_codegen_cranelift/build_sysroot/prepare_sysroot_src.sh index c90205db0f..54b7a94750 100755 --- a/compiler/rustc_codegen_cranelift/build_sysroot/prepare_sysroot_src.sh +++ b/compiler/rustc_codegen_cranelift/build_sysroot/prepare_sysroot_src.sh @@ -32,7 +32,7 @@ popd git clone https://github.com/rust-lang/compiler-builtins.git || echo "rust-lang/compiler-builtins has already been cloned" pushd compiler-builtins git checkout -- . -git checkout 0.1.39 +git checkout 0.1.43 git apply ../../crate_patches/000*-compiler-builtins-*.patch popd diff --git a/compiler/rustc_codegen_cranelift/crate_patches/0002-compiler-builtins-Disable-128bit-atomic-operations.patch b/compiler/rustc_codegen_cranelift/crate_patches/0001-compiler-builtins-Disable-128bit-atomic-operations.patch similarity index 100% rename from compiler/rustc_codegen_cranelift/crate_patches/0002-compiler-builtins-Disable-128bit-atomic-operations.patch rename to compiler/rustc_codegen_cranelift/crate_patches/0001-compiler-builtins-Disable-128bit-atomic-operations.patch diff --git a/compiler/rustc_codegen_cranelift/crate_patches/0001-compiler-builtins-Remove-rotate_left-from-Int.patch b/compiler/rustc_codegen_cranelift/crate_patches/0001-compiler-builtins-Remove-rotate_left-from-Int.patch deleted file mode 100644 index e14768910a..0000000000 --- a/compiler/rustc_codegen_cranelift/crate_patches/0001-compiler-builtins-Remove-rotate_left-from-Int.patch +++ /dev/null @@ -1,35 +0,0 @@ -From 7078cca3cb614e1e82da428380b4e16fc3afef46 Mon Sep 17 00:00:00 2001 -From: bjorn3 -Date: Thu, 21 Jan 2021 14:46:36 +0100 -Subject: [PATCH] Remove rotate_left from Int - ---- - src/int/mod.rs | 5 ----- - 1 file changed, 5 deletions(-) - -diff --git a/src/int/mod.rs b/src/int/mod.rs -index 06054c8..3bea17b 100644 ---- a/src/int/mod.rs -+++ b/src/int/mod.rs -@@ -85,7 +85,6 @@ pub trait Int: - fn wrapping_sub(self, other: Self) -> Self; - fn wrapping_shl(self, other: u32) -> Self; - fn wrapping_shr(self, other: u32) -> Self; -- fn rotate_left(self, other: u32) -> Self; - fn overflowing_add(self, other: Self) -> (Self, bool); - fn aborting_div(self, other: Self) -> Self; - fn aborting_rem(self, other: Self) -> Self; -@@ -209,10 +208,6 @@ macro_rules! int_impl_common { - ::wrapping_shr(self, other) - } - -- fn rotate_left(self, other: u32) -> Self { -- ::rotate_left(self, other) -- } -- - fn overflowing_add(self, other: Self) -> (Self, bool) { - ::overflowing_add(self, other) - } --- -2.26.2.7.g19db9cfb68 - diff --git a/compiler/rustc_codegen_cranelift/docs/env_vars.md b/compiler/rustc_codegen_cranelift/docs/env_vars.md deleted file mode 100644 index f7fde1b4f3..0000000000 --- a/compiler/rustc_codegen_cranelift/docs/env_vars.md +++ /dev/null @@ -1,15 +0,0 @@ -# List of env vars recognized by cg_clif - -
-
CG_CLIF_JIT_ARGS
-
When JIT mode is enable pass these arguments to the program.
-
CG_CLIF_INCR_CACHE_DISABLED
-
Don't cache object files in the incremental cache. Useful during development of cg_clif - to make it possible to use incremental mode for all analyses performed by rustc without caching - object files when their content should have been changed by a change to cg_clif.
-
CG_CLIF_DISPLAY_CG_TIME
-
If "1", display the time it took to perform codegen for a crate.
-
CG_CLIF_ENABLE_VERIFIER
-
Enable the Cranelift ir verifier for all compilation passes. If not set it will only run once - before passing the clif ir to Cranelift for compilation. -
diff --git a/compiler/rustc_codegen_cranelift/example/mini_core_hello_world.rs b/compiler/rustc_codegen_cranelift/example/mini_core_hello_world.rs index ea37ca98b5..6570f2bf9f 100644 --- a/compiler/rustc_codegen_cranelift/example/mini_core_hello_world.rs +++ b/compiler/rustc_codegen_cranelift/example/mini_core_hello_world.rs @@ -11,6 +11,22 @@ unsafe extern "C" fn my_puts(s: *const i8) { puts(s); } +macro_rules! assert { + ($e:expr) => { + if !$e { + panic(stringify!(! $e)); + } + }; +} + +macro_rules! assert_eq { + ($l:expr, $r: expr) => { + if $l != $r { + panic(stringify!($l != $r)); + } + } +} + #[lang = "termination"] trait Termination { fn report(self) -> i32; @@ -20,8 +36,9 @@ impl Termination for () { fn report(self) -> i32 { unsafe { NUM = 6 * 7 + 1 + (1u8 == 1u8) as u8; // 44 - *NUM_REF as i32 + assert_eq!(*NUM_REF as i32, 44); } + 0 } } @@ -82,29 +99,12 @@ fn start( unsafe { puts(*((argv as usize + 2 * intrinsics::size_of::<*const u8>()) as *const *const i8)); } } - main().report(); - 0 + main().report() as isize } static mut NUM: u8 = 6 * 7; static NUM_REF: &'static u8 = unsafe { &NUM }; -macro_rules! assert { - ($e:expr) => { - if !$e { - panic(stringify!(! $e)); - } - }; -} - -macro_rules! assert_eq { - ($l:expr, $r: expr) => { - if $l != $r { - panic(stringify!($l != $r)); - } - } -} - struct Unique { pointer: *const T, _marker: PhantomData, @@ -296,6 +296,11 @@ fn main() { unsafe { global_asm_test(); } + + // Both statics have a reference that points to the same anonymous allocation. + static REF1: &u8 = &42; + static REF2: &u8 = REF1; + assert_eq!(*REF1, *REF2); } #[cfg(all(not(jit), target_os = "linux"))] diff --git a/compiler/rustc_codegen_cranelift/example/std_example.rs b/compiler/rustc_codegen_cranelift/example/std_example.rs index 015bbdfed4..7d608df925 100644 --- a/compiler/rustc_codegen_cranelift/example/std_example.rs +++ b/compiler/rustc_codegen_cranelift/example/std_example.rs @@ -48,6 +48,8 @@ fn main() { assert_eq!(2.3f32.copysign(-1.0), -2.3f32); println!("{}", 2.3f32.powf(2.0)); + assert_eq!(i64::MAX.checked_mul(2), None); + assert_eq!(-128i8, (-128i8).saturating_sub(1)); assert_eq!(127i8, 127i8.saturating_sub(-128)); assert_eq!(-128i8, (-128i8).saturating_add(-128)); @@ -84,6 +86,7 @@ fn main() { assert_eq!(houndred_i128 as f64, 100.0); assert_eq!(houndred_f32 as i128, 100); assert_eq!(houndred_f64 as i128, 100); + assert_eq!(1u128.rotate_left(2), 4); // Test signed 128bit comparing let max = usize::MAX as i128; @@ -184,20 +187,6 @@ unsafe fn test_mm_slli_si128() { ); let r = _mm_slli_si128(a, 16); assert_eq_m128i(r, _mm_set1_epi8(0)); - - #[rustfmt::skip] - let a = _mm_setr_epi8( - 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, - ); - let r = _mm_slli_si128(a, -1); - assert_eq_m128i(_mm_set1_epi8(0), r); - - #[rustfmt::skip] - let a = _mm_setr_epi8( - 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, - ); - let r = _mm_slli_si128(a, -0x80000000); - assert_eq_m128i(r, _mm_set1_epi8(0)); } #[cfg(target_arch = "x86_64")] @@ -292,7 +281,7 @@ unsafe fn test_mm_extract_epi8() { 8, 9, 10, 11, 12, 13, 14, 15 ); let r1 = _mm_extract_epi8(a, 0); - let r2 = _mm_extract_epi8(a, 19); + let r2 = _mm_extract_epi8(a, 3); assert_eq!(r1, 0xFF); assert_eq!(r2, 3); } diff --git a/compiler/rustc_codegen_cranelift/patches/0022-core-Disable-not-compiling-tests.patch b/compiler/rustc_codegen_cranelift/patches/0022-core-Disable-not-compiling-tests.patch index 8cfffe580a..ba0eaacd82 100644 --- a/compiler/rustc_codegen_cranelift/patches/0022-core-Disable-not-compiling-tests.patch +++ b/compiler/rustc_codegen_cranelift/patches/0022-core-Disable-not-compiling-tests.patch @@ -39,46 +39,6 @@ index a35897e..f0bf645 100644 pub fn decode_finite(v: T) -> Decoded { match decode(v).1 { -diff --git a/library/core/tests/num/int_macros.rs b/library/core/tests/num/int_macros.rs -index 0475aeb..9558198 100644 ---- a/library/core/tests/num/int_macros.rs -+++ b/library/core/tests/num/int_macros.rs -@@ -88,6 +88,7 @@ mod tests { - assert_eq!(x.trailing_ones(), 0); - } - -+ /* - #[test] - fn test_rotate() { - assert_eq!(A.rotate_left(6).rotate_right(2).rotate_right(4), A); -@@ -112,6 +113,7 @@ mod tests { - assert_eq!(B.rotate_left(128), B); - assert_eq!(C.rotate_left(128), C); - } -+ */ - - #[test] - fn test_swap_bytes() { -diff --git a/library/core/tests/num/uint_macros.rs b/library/core/tests/num/uint_macros.rs -index 04ed14f..a6e372e 100644 ---- a/library/core/tests/num/uint_macros.rs -+++ b/library/core/tests/num/uint_macros.rs -@@ -52,6 +52,7 @@ mod tests { - assert_eq!(x.trailing_ones(), 0); - } - -+ /* - #[test] - fn test_rotate() { - assert_eq!(A.rotate_left(6).rotate_right(2).rotate_right(4), A); -@@ -76,6 +77,7 @@ mod tests { - assert_eq!(B.rotate_left(128), B); - assert_eq!(C.rotate_left(128), C); - } -+ */ - - #[test] - fn test_swap_bytes() { diff --git a/library/core/tests/ptr.rs b/library/core/tests/ptr.rs index 1a6be3a..42dbd59 100644 --- a/library/core/tests/ptr.rs diff --git a/compiler/rustc_codegen_cranelift/rust-toolchain b/compiler/rustc_codegen_cranelift/rust-toolchain index 2917fc7ee3..9fe6e093a7 100644 --- a/compiler/rustc_codegen_cranelift/rust-toolchain +++ b/compiler/rustc_codegen_cranelift/rust-toolchain @@ -1,3 +1,3 @@ [toolchain] -channel = "nightly-2021-03-29" +channel = "nightly-2021-05-26" components = ["rust-src", "rustc-dev", "llvm-tools-preview"] diff --git a/compiler/rustc_codegen_cranelift/scripts/ext_config.sh b/compiler/rustc_codegen_cranelift/scripts/ext_config.sh index 7971f620df..3f98d77d76 100644 --- a/compiler/rustc_codegen_cranelift/scripts/ext_config.sh +++ b/compiler/rustc_codegen_cranelift/scripts/ext_config.sh @@ -5,7 +5,7 @@ set -e export CG_CLIF_DISPLAY_CG_TIME=1 -export CG_CLIF_INCR_CACHE_DISABLED=1 +export CG_CLIF_DISABLE_INCR_CACHE=1 export HOST_TRIPLE=$(rustc -vV | grep host | cut -d: -f2 | tr -d " ") export TARGET_TRIPLE=${TARGET_TRIPLE:-$HOST_TRIPLE} diff --git a/compiler/rustc_codegen_cranelift/scripts/setup_rust_fork.sh b/compiler/rustc_codegen_cranelift/scripts/setup_rust_fork.sh index e8bedf625f..43c4887669 100644 --- a/compiler/rustc_codegen_cranelift/scripts/setup_rust_fork.sh +++ b/compiler/rustc_codegen_cranelift/scripts/setup_rust_fork.sh @@ -24,18 +24,6 @@ index 5bd1147cad5..10d68a2ff14 100644 + [patch."https://github.com/rust-lang/rust-clippy"] clippy_lints = { path = "src/tools/clippy/clippy_lints" } -diff --git a/compiler/rustc_data_structures/Cargo.toml b/compiler/rustc_data_structures/Cargo.toml -index 23e689fcae7..5f077b765b6 100644 ---- a/compiler/rustc_data_structures/Cargo.toml -+++ b/compiler/rustc_data_structures/Cargo.toml -@@ -32,7 +32,6 @@ tempfile = "3.0.5" - - [dependencies.parking_lot] - version = "0.11" --features = ["nightly"] - - [target.'cfg(windows)'.dependencies] - winapi = { version = "0.3", features = ["fileapi", "psapi"] } diff --git a/library/alloc/Cargo.toml b/library/alloc/Cargo.toml index d95b5b7f17f..00b6f0e3635 100644 --- a/library/alloc/Cargo.toml @@ -44,11 +32,12 @@ index d95b5b7f17f..00b6f0e3635 100644 [dependencies] core = { path = "../core" } --compiler_builtins = { version = "0.1.39", features = ['rustc-dep-of-std'] } -+compiler_builtins = { version = "0.1.39", features = ['rustc-dep-of-std', 'no-asm'] } +-compiler_builtins = { version = "0.1.40", features = ['rustc-dep-of-std'] } ++compiler_builtins = { version = "0.1.43", features = ['rustc-dep-of-std', 'no-asm'] } [dev-dependencies] rand = "0.7" + rand_xorshift = "0.2" EOF cat > config.toml <( module: &mut dyn Module, inst: Instance<'tcx>, ) -> FuncId { - let name = tcx.symbol_name(inst).name.to_string(); + let name = tcx.symbol_name(inst).name; let sig = get_function_sig(tcx, module.isa().triple(), inst); - module.declare_function(&name, Linkage::Import, &sig).unwrap() + module.declare_function(name, Linkage::Import, &sig).unwrap() } impl<'tcx> FunctionCx<'_, '_, 'tcx> { /// Instance must be monomorphized pub(crate) fn get_function_ref(&mut self, inst: Instance<'tcx>) -> FuncRef { - let func_id = import_function(self.tcx, self.cx.module, inst); - let func_ref = self.cx.module.declare_func_in_func(func_id, &mut self.bcx.func); + let func_id = import_function(self.tcx, self.module, inst); + let func_ref = self.module.declare_func_in_func(func_id, &mut self.bcx.func); if self.clif_comments.enabled() { self.add_comment(func_ref, format!("{:?}", inst)); @@ -89,8 +89,8 @@ impl<'tcx> FunctionCx<'_, '_, 'tcx> { args: &[Value], ) -> &[Value] { let sig = Signature { params, returns, call_conv: CallConv::triple_default(self.triple()) }; - let func_id = self.cx.module.declare_function(&name, Linkage::Import, &sig).unwrap(); - let func_ref = self.cx.module.declare_func_in_func(func_id, &mut self.bcx.func); + let func_id = self.module.declare_function(name, Linkage::Import, &sig).unwrap(); + let func_ref = self.module.declare_func_in_func(func_id, &mut self.bcx.func); let call_inst = self.bcx.ins().call(func_ref, args); if self.clif_comments.enabled() { self.add_comment(call_inst, format!("easy_call {}", name)); @@ -295,7 +295,6 @@ pub(crate) fn codegen_fn_prelude<'tcx>(fx: &mut FunctionCx<'_, '_, 'tcx>, start_ pub(crate) fn codegen_terminator_call<'tcx>( fx: &mut FunctionCx<'_, '_, 'tcx>, span: Span, - current_block: Block, func: &Operand<'tcx>, args: &[Operand<'tcx>], destination: Option<(Place<'tcx>, BasicBlock)>, @@ -357,7 +356,7 @@ pub(crate) fn codegen_terminator_call<'tcx>( .map(|inst| fx.tcx.codegen_fn_attrs(inst.def_id()).flags.contains(CodegenFnAttrFlags::COLD)) .unwrap_or(false); if is_cold { - fx.cold_blocks.insert(current_block); + // FIXME Mark current_block block as cold once Cranelift supports it } // Unpack arguments tuple for closures diff --git a/compiler/rustc_codegen_cranelift/src/allocator.rs b/compiler/rustc_codegen_cranelift/src/allocator.rs index f60645a9f9..357a9f2daf 100644 --- a/compiler/rustc_codegen_cranelift/src/allocator.rs +++ b/compiler/rustc_codegen_cranelift/src/allocator.rs @@ -11,9 +11,9 @@ use rustc_span::symbol::sym; pub(crate) fn codegen( tcx: TyCtxt<'_>, module: &mut impl Module, - unwind_context: &mut UnwindContext<'_>, + unwind_context: &mut UnwindContext, ) -> bool { - let any_dynamic_crate = tcx.dependency_formats(LOCAL_CRATE).iter().any(|(_, list)| { + let any_dynamic_crate = tcx.dependency_formats(()).iter().any(|(_, list)| { use rustc_middle::middle::dependency_format::Linkage; list.iter().any(|&linkage| linkage == Linkage::Dynamic) }); @@ -29,7 +29,7 @@ pub(crate) fn codegen( fn codegen_inner( module: &mut impl Module, - unwind_context: &mut UnwindContext<'_>, + unwind_context: &mut UnwindContext, kind: AllocatorKind, ) { let usize_ty = module.target_config().pointer_type(); diff --git a/compiler/rustc_codegen_cranelift/src/archive.rs b/compiler/rustc_codegen_cranelift/src/archive.rs index 7583fc4240..22897c43e7 100644 --- a/compiler/rustc_codegen_cranelift/src/archive.rs +++ b/compiler/rustc_codegen_cranelift/src/archive.rs @@ -85,8 +85,8 @@ impl<'a> ArchiveBuilder<'a> for ArArchiveBuilder<'a> { )); } - fn add_native_library(&mut self, name: rustc_span::symbol::Symbol) { - let location = find_library(name, &self.lib_search_paths, self.sess); + fn add_native_library(&mut self, name: rustc_span::symbol::Symbol, verbatim: bool) { + let location = find_library(name, verbatim, &self.lib_search_paths, self.sess); self.add_archive(location.clone(), |_| false).unwrap_or_else(|e| { panic!("failed to add native library {}: {}", location.to_string_lossy(), e); }); @@ -160,7 +160,7 @@ impl<'a> ArchiveBuilder<'a> for ArArchiveBuilder<'a> { }; if !self.no_builtin_ranlib { - match object::File::parse(&data) { + match object::File::parse(&*data) { Ok(object) => { symbol_table.insert( entry_name.as_bytes().to_vec(), @@ -254,6 +254,15 @@ impl<'a> ArchiveBuilder<'a> for ArArchiveBuilder<'a> { } } } + + fn inject_dll_import_lib( + &mut self, + _lib_name: &str, + _dll_imports: &[rustc_middle::middle::cstore::DllImport], + _tmpdir: &rustc_data_structures::temp_dir::MaybeTempDir, + ) { + bug!("injecting dll imports is not supported"); + } } impl<'a> ArArchiveBuilder<'a> { diff --git a/compiler/rustc_codegen_cranelift/src/backend.rs b/compiler/rustc_codegen_cranelift/src/backend.rs index eb7927fc4a..05c06bac27 100644 --- a/compiler/rustc_codegen_cranelift/src/backend.rs +++ b/compiler/rustc_codegen_cranelift/src/backend.rs @@ -5,23 +5,23 @@ use std::convert::{TryFrom, TryInto}; use rustc_data_structures::fx::FxHashMap; use rustc_session::Session; +use cranelift_codegen::isa::TargetIsa; use cranelift_module::FuncId; +use cranelift_object::{ObjectBuilder, ObjectModule, ObjectProduct}; use object::write::*; use object::{RelocationEncoding, SectionKind, SymbolFlags}; -use cranelift_object::{ObjectBuilder, ObjectModule, ObjectProduct}; - use gimli::SectionId; use crate::debuginfo::{DebugReloc, DebugRelocName}; pub(crate) trait WriteMetadata { - fn add_rustc_section(&mut self, symbol_name: String, data: Vec, is_like_osx: bool); + fn add_rustc_section(&mut self, symbol_name: String, data: Vec); } impl WriteMetadata for object::write::Object { - fn add_rustc_section(&mut self, symbol_name: String, data: Vec, _is_like_osx: bool) { + fn add_rustc_section(&mut self, symbol_name: String, data: Vec) { let segment = self.segment_name(object::write::StandardSegment::Data).to_vec(); let section_id = self.add_section(segment, b".rustc".to_vec(), object::SectionKind::Data); let offset = self.append_section_data(section_id, &data, 1); @@ -113,7 +113,7 @@ impl WriteDebugInfo for ObjectProduct { } pub(crate) fn with_object(sess: &Session, name: &str, f: impl FnOnce(&mut Object)) -> Vec { - let triple = crate::build_isa(sess).triple().clone(); + let triple = crate::target_triple(sess); let binary_format = match triple.binary_format { target_lexicon::BinaryFormat::Elf => object::BinaryFormat::Elf, @@ -141,13 +141,9 @@ pub(crate) fn with_object(sess: &Session, name: &str, f: impl FnOnce(&mut Object metadata_object.write().unwrap() } -pub(crate) fn make_module(sess: &Session, name: String) -> ObjectModule { - let mut builder = ObjectBuilder::new( - crate::build_isa(sess), - name + ".o", - cranelift_module::default_libcall_names(), - ) - .unwrap(); +pub(crate) fn make_module(sess: &Session, isa: Box, name: String) -> ObjectModule { + let mut builder = + ObjectBuilder::new(isa, name + ".o", cranelift_module::default_libcall_names()).unwrap(); // Unlike cg_llvm, cg_clif defaults to disabling -Zfunction-sections. For cg_llvm binary size // is important, while cg_clif cares more about compilation times. Enabling -Zfunction-sections // can easily double the amount of time necessary to perform linking. diff --git a/compiler/rustc_codegen_cranelift/src/base.rs b/compiler/rustc_codegen_cranelift/src/base.rs index b34a29c25b..ec3e17e5b7 100644 --- a/compiler/rustc_codegen_cranelift/src/base.rs +++ b/compiler/rustc_codegen_cranelift/src/base.rs @@ -6,9 +6,14 @@ use rustc_middle::ty::adjustment::PointerCast; use rustc_middle::ty::layout::FnAbiExt; use rustc_target::abi::call::FnAbi; +use crate::constant::ConstantCx; use crate::prelude::*; -pub(crate) fn codegen_fn<'tcx>(cx: &mut crate::CodegenCx<'_, 'tcx>, instance: Instance<'tcx>) { +pub(crate) fn codegen_fn<'tcx>( + cx: &mut crate::CodegenCx<'tcx>, + module: &mut dyn Module, + instance: Instance<'tcx>, +) { let tcx = cx.tcx; let _inst_guard = @@ -18,9 +23,9 @@ pub(crate) fn codegen_fn<'tcx>(cx: &mut crate::CodegenCx<'_, 'tcx>, instance: In let mir = tcx.instance_mir(instance.def); // Declare function - let name = tcx.symbol_name(instance).name.to_string(); - let sig = get_function_sig(tcx, cx.module.isa().triple(), instance); - let func_id = cx.module.declare_function(&name, Linkage::Local, &sig).unwrap(); + let symbol_name = tcx.symbol_name(instance); + let sig = get_function_sig(tcx, module.isa().triple(), instance); + let func_id = module.declare_function(symbol_name.name, Linkage::Local, &sig).unwrap(); cx.cached_context.clear(); @@ -39,15 +44,19 @@ pub(crate) fn codegen_fn<'tcx>(cx: &mut crate::CodegenCx<'_, 'tcx>, instance: In (0..mir.basic_blocks().len()).map(|_| bcx.create_block()).collect(); // Make FunctionCx - let pointer_type = cx.module.target_config().pointer_type(); + let pointer_type = module.target_config().pointer_type(); let clif_comments = crate::pretty_clif::CommentWriter::new(tcx, instance); let mut fx = FunctionCx { cx, + module, tcx, pointer_type, + vtables: FxHashMap::default(), + constants_cx: ConstantCx::new(), instance, + symbol_name, mir, fn_abi: Some(FnAbi::of_instance(&RevealAllLayoutCx(tcx), instance, &[])), @@ -55,7 +64,6 @@ pub(crate) fn codegen_fn<'tcx>(cx: &mut crate::CodegenCx<'_, 'tcx>, instance: In block_map, local_map: IndexVec::with_capacity(mir.local_decls.len()), caller_location: None, // set by `codegen_fn_prelude` - cold_blocks: EntitySet::new(), clif_comments, source_info_set: indexmap::IndexSet::new(), @@ -90,7 +98,8 @@ pub(crate) fn codegen_fn<'tcx>(cx: &mut crate::CodegenCx<'_, 'tcx>, instance: In let mut clif_comments = fx.clif_comments; let source_info_set = fx.source_info_set; let local_map = fx.local_map; - let cold_blocks = fx.cold_blocks; + + fx.constants_cx.finalize(fx.tcx, &mut *fx.module); // Store function in context let context = &mut cx.cached_context; @@ -101,32 +110,24 @@ pub(crate) fn codegen_fn<'tcx>(cx: &mut crate::CodegenCx<'_, 'tcx>, instance: In // Verify function verify_func(tcx, &clif_comments, &context.func); - // Perform rust specific optimizations - tcx.sess.time("optimize clif ir", || { - crate::optimize::optimize_function( - tcx, - instance, - context, - &cold_blocks, - &mut clif_comments, - ); - }); - // If the return block is not reachable, then the SSA builder may have inserted an `iconst.i128` // instruction, which doesn't have an encoding. context.compute_cfg(); context.compute_domtree(); - context.eliminate_unreachable_code(cx.module.isa()).unwrap(); - context.dce(cx.module.isa()).unwrap(); + context.eliminate_unreachable_code(module.isa()).unwrap(); + context.dce(module.isa()).unwrap(); // Some Cranelift optimizations expect the domtree to not yet be computed and as such don't // invalidate it when it would change. context.domtree.clear(); - context.want_disasm = crate::pretty_clif::should_write_ir(tcx); + // Perform rust specific optimizations + tcx.sess.time("optimize clif ir", || { + crate::optimize::optimize_function(tcx, instance, context, &mut clif_comments); + }); // Define function - let module = &mut cx.module; tcx.sess.time("define function", || { + context.want_disasm = crate::pretty_clif::should_write_ir(tcx); module .define_function(func_id, context, &mut NullTrapSink {}, &mut NullStackMapSink {}) .unwrap() @@ -136,7 +137,7 @@ pub(crate) fn codegen_fn<'tcx>(cx: &mut crate::CodegenCx<'_, 'tcx>, instance: In crate::pretty_clif::write_clif_file( tcx, "opt", - Some(cx.module.isa()), + Some(module.isa()), instance, &context, &clif_comments, @@ -145,13 +146,13 @@ pub(crate) fn codegen_fn<'tcx>(cx: &mut crate::CodegenCx<'_, 'tcx>, instance: In if let Some(disasm) = &context.mach_compile_result.as_ref().unwrap().disasm { crate::pretty_clif::write_ir_file( tcx, - &format!("{}.vcode", tcx.symbol_name(instance).name), + || format!("{}.vcode", tcx.symbol_name(instance).name), |file| file.write_all(disasm.as_bytes()), ) } // Define debuginfo for function - let isa = cx.module.isa(); + let isa = module.isa(); let debug_context = &mut cx.debug_context; let unwind_context = &mut cx.unwind_context; tcx.sess.time("generate debug info", || { @@ -159,7 +160,7 @@ pub(crate) fn codegen_fn<'tcx>(cx: &mut crate::CodegenCx<'_, 'tcx>, instance: In debug_context.define_function( instance, func_id, - &name, + symbol_name.name, isa, context, &source_info_set, @@ -205,9 +206,8 @@ fn codegen_fn_content(fx: &mut FunctionCx<'_, '_, '_>) { // Unwinding after panicking is not supported continue; - // FIXME once unwinding is supported uncomment next lines - // // Unwinding is unlikely to happen, so mark cleanup block's as cold. - // fx.cold_blocks.insert(block); + // FIXME Once unwinding is supported and Cranelift supports marking blocks as cold, do + // so for cleanup blocks. } fx.bcx.ins().nop(); @@ -262,7 +262,7 @@ fn codegen_fn_content(fx: &mut FunctionCx<'_, '_, '_>) { let target = fx.get_block(*target); let failure = fx.bcx.create_block(); - fx.cold_blocks.insert(failure); + // FIXME Mark failure block as cold once Cranelift supports it if *expected { fx.bcx.ins().brz(cond, failure, &[]); @@ -355,14 +355,7 @@ fn codegen_fn_content(fx: &mut FunctionCx<'_, '_, '_>) { from_hir_call: _, } => { fx.tcx.sess.time("codegen call", || { - crate::abi::codegen_terminator_call( - fx, - *fn_span, - block, - func, - args, - *destination, - ) + crate::abi::codegen_terminator_call(fx, *fn_span, func, args, *destination) }); } TerminatorKind::InlineAsm { @@ -664,7 +657,7 @@ fn codegen_stmt<'tcx>( // FIXME use emit_small_memset where possible let addr = lval.to_ptr().get_addr(fx); let val = operand.load_scalar(fx); - fx.bcx.call_memset(fx.cx.module.target_config(), addr, val, times); + fx.bcx.call_memset(fx.module.target_config(), addr, val, times); } else { let loop_block = fx.bcx.create_block(); let loop_block2 = fx.bcx.create_block(); @@ -750,85 +743,15 @@ fn codegen_stmt<'tcx>( | StatementKind::AscribeUserType(..) => {} StatementKind::LlvmInlineAsm(asm) => { - use rustc_span::symbol::Symbol; - let LlvmInlineAsm { asm, outputs, inputs } = &**asm; - let rustc_hir::LlvmInlineAsmInner { - asm: asm_code, // Name - outputs: output_names, // Vec - inputs: input_names, // Vec - clobbers, // Vec - volatile, // bool - alignstack, // bool - dialect: _, - asm_str_style: _, - } = asm; - match asm_code.as_str().trim() { + match asm.asm.asm.as_str().trim() { "" => { // Black box } - "mov %rbx, %rsi\n cpuid\n xchg %rbx, %rsi" => { - assert_eq!(input_names, &[Symbol::intern("{eax}"), Symbol::intern("{ecx}")]); - assert_eq!(output_names.len(), 4); - for (i, c) in (&["={eax}", "={esi}", "={ecx}", "={edx}"]).iter().enumerate() { - assert_eq!(&output_names[i].constraint.as_str(), c); - assert!(!output_names[i].is_rw); - assert!(!output_names[i].is_indirect); - } - - assert_eq!(clobbers, &[]); - - assert!(!volatile); - assert!(!alignstack); - - assert_eq!(inputs.len(), 2); - let leaf = codegen_operand(fx, &inputs[0].1).load_scalar(fx); // %eax - let subleaf = codegen_operand(fx, &inputs[1].1).load_scalar(fx); // %ecx - - let (eax, ebx, ecx, edx) = - crate::intrinsics::codegen_cpuid_call(fx, leaf, subleaf); - - assert_eq!(outputs.len(), 4); - codegen_place(fx, outputs[0]) - .write_cvalue(fx, CValue::by_val(eax, fx.layout_of(fx.tcx.types.u32))); - codegen_place(fx, outputs[1]) - .write_cvalue(fx, CValue::by_val(ebx, fx.layout_of(fx.tcx.types.u32))); - codegen_place(fx, outputs[2]) - .write_cvalue(fx, CValue::by_val(ecx, fx.layout_of(fx.tcx.types.u32))); - codegen_place(fx, outputs[3]) - .write_cvalue(fx, CValue::by_val(edx, fx.layout_of(fx.tcx.types.u32))); - } - "xgetbv" => { - assert_eq!(input_names, &[Symbol::intern("{ecx}")]); - - assert_eq!(output_names.len(), 2); - for (i, c) in (&["={eax}", "={edx}"]).iter().enumerate() { - assert_eq!(&output_names[i].constraint.as_str(), c); - assert!(!output_names[i].is_rw); - assert!(!output_names[i].is_indirect); - } - - assert_eq!(clobbers, &[]); - - assert!(!volatile); - assert!(!alignstack); - - crate::trap::trap_unimplemented(fx, "_xgetbv arch intrinsic is not supported"); - } - // ___chkstk, ___chkstk_ms and __alloca are only used on Windows - _ if fx.tcx.symbol_name(fx.instance).name.starts_with("___chkstk") => { - crate::trap::trap_unimplemented(fx, "Stack probes are not supported"); - } - _ if fx.tcx.symbol_name(fx.instance).name == "__alloca" => { - crate::trap::trap_unimplemented(fx, "Alloca is not supported"); - } - // Used in sys::windows::abort_internal - "int $$0x29" => { - crate::trap::trap_unimplemented(fx, "Windows abort"); - } - _ => fx - .tcx - .sess - .span_fatal(stmt.source_info.span, "Inline assembly is not supported"), + _ => fx.tcx.sess.span_fatal( + stmt.source_info.span, + "Legacy `llvm_asm!` inline assembly is not supported. \ + Try using the new `asm!` instead.", + ), } } StatementKind::Coverage { .. } => fx.tcx.sess.fatal("-Zcoverage is unimplemented"), @@ -844,7 +767,7 @@ fn codegen_stmt<'tcx>( let elem_size: u64 = pointee.size.bytes(); let bytes = if elem_size != 1 { fx.bcx.ins().imul_imm(count, elem_size as i64) } else { count }; - fx.bcx.call_memcpy(fx.cx.module.target_config(), dst, src, bytes); + fx.bcx.call_memcpy(fx.module.target_config(), dst, src, bytes); } } } @@ -946,7 +869,7 @@ pub(crate) fn codegen_operand<'tcx>( pub(crate) fn codegen_panic<'tcx>(fx: &mut FunctionCx<'_, '_, 'tcx>, msg_str: &str, span: Span) { let location = fx.get_caller_location(span).load_scalar(fx); - let msg_ptr = fx.anonymous_str("assert", msg_str); + let msg_ptr = fx.anonymous_str(msg_str); let msg_len = fx.bcx.ins().iconst(fx.pointer_type, i64::try_from(msg_str.len()).unwrap()); let args = [msg_ptr, msg_len, location]; diff --git a/compiler/rustc_codegen_cranelift/src/common.rs b/compiler/rustc_codegen_cranelift/src/common.rs index b5874f6253..488ff6e134 100644 --- a/compiler/rustc_codegen_cranelift/src/common.rs +++ b/compiler/rustc_codegen_cranelift/src/common.rs @@ -1,8 +1,10 @@ use rustc_index::vec::IndexVec; +use rustc_middle::ty::SymbolName; use rustc_target::abi::call::FnAbi; use rustc_target::abi::{Integer, Primitive}; use rustc_target::spec::{HasTargetSpec, Target}; +use crate::constant::ConstantCx; use crate::prelude::*; pub(crate) fn pointer_ty(tcx: TyCtxt<'_>) -> types::Type { @@ -226,12 +228,16 @@ pub(crate) fn type_sign(ty: Ty<'_>) -> bool { } } -pub(crate) struct FunctionCx<'m, 'clif, 'tcx> { - pub(crate) cx: &'clif mut crate::CodegenCx<'m, 'tcx>, +pub(crate) struct FunctionCx<'m, 'clif, 'tcx: 'm> { + pub(crate) cx: &'clif mut crate::CodegenCx<'tcx>, + pub(crate) module: &'m mut dyn Module, pub(crate) tcx: TyCtxt<'tcx>, pub(crate) pointer_type: Type, // Cached from module + pub(crate) vtables: FxHashMap<(Ty<'tcx>, Option>), DataId>, + pub(crate) constants_cx: ConstantCx, pub(crate) instance: Instance<'tcx>, + pub(crate) symbol_name: SymbolName<'tcx>, pub(crate) mir: &'tcx Body<'tcx>, pub(crate) fn_abi: Option>>, @@ -242,9 +248,6 @@ pub(crate) struct FunctionCx<'m, 'clif, 'tcx> { /// When `#[track_caller]` is used, the implicit caller location is stored in this variable. pub(crate) caller_location: Option>, - /// See [`crate::optimize::code_layout`] for more information. - pub(crate) cold_blocks: EntitySet, - pub(crate) clif_comments: crate::pretty_clif::CommentWriter, pub(crate) source_info_set: indexmap::IndexSet, @@ -331,7 +334,9 @@ impl<'tcx> FunctionCx<'_, '_, 'tcx> { let topmost = span.ctxt().outer_expn().expansion_cause().unwrap_or(span); let caller = self.tcx.sess.source_map().lookup_char_pos(topmost.lo()); let const_loc = self.tcx.const_caller_location(( - rustc_span::symbol::Symbol::intern(&caller.file.name.to_string()), + rustc_span::symbol::Symbol::intern( + &caller.file.name.prefer_remapped().to_string_lossy(), + ), caller.line as u32, caller.col_display as u32 + 1, )); @@ -339,28 +344,18 @@ impl<'tcx> FunctionCx<'_, '_, 'tcx> { } pub(crate) fn triple(&self) -> &target_lexicon::Triple { - self.cx.module.isa().triple() + self.module.isa().triple() } - pub(crate) fn anonymous_str(&mut self, prefix: &str, msg: &str) -> Value { - use std::collections::hash_map::DefaultHasher; - use std::hash::{Hash, Hasher}; - - let mut hasher = DefaultHasher::new(); - msg.hash(&mut hasher); - let msg_hash = hasher.finish(); + pub(crate) fn anonymous_str(&mut self, msg: &str) -> Value { let mut data_ctx = DataContext::new(); data_ctx.define(msg.as_bytes().to_vec().into_boxed_slice()); - let msg_id = self - .cx - .module - .declare_data(&format!("__{}_{:08x}", prefix, msg_hash), Linkage::Local, false, false) - .unwrap(); + let msg_id = self.module.declare_anonymous_data(false, false).unwrap(); // Ignore DuplicateDefinition error, as the data will be the same - let _ = self.cx.module.define_data(msg_id, &data_ctx); + let _ = self.module.define_data(msg_id, &data_ctx); - let local_msg_id = self.cx.module.declare_data_in_func(msg_id, self.bcx.func); + let local_msg_id = self.module.declare_data_in_func(msg_id, self.bcx.func); if self.clif_comments.enabled() { self.add_comment(local_msg_id, msg); } diff --git a/compiler/rustc_codegen_cranelift/src/config.rs b/compiler/rustc_codegen_cranelift/src/config.rs new file mode 100644 index 0000000000..eef3c8c8d6 --- /dev/null +++ b/compiler/rustc_codegen_cranelift/src/config.rs @@ -0,0 +1,116 @@ +use std::env; +use std::str::FromStr; + +fn bool_env_var(key: &str) -> bool { + env::var(key).as_ref().map(|val| &**val) == Ok("1") +} + +/// The mode to use for compilation. +#[derive(Copy, Clone, Debug)] +pub enum CodegenMode { + /// AOT compile the crate. This is the default. + Aot, + /// JIT compile and execute the crate. + Jit, + /// JIT compile and execute the crate, but only compile functions the first time they are used. + JitLazy, +} + +impl FromStr for CodegenMode { + type Err = String; + + fn from_str(s: &str) -> Result { + match s { + "aot" => Ok(CodegenMode::Aot), + "jit" => Ok(CodegenMode::Jit), + "jit-lazy" => Ok(CodegenMode::JitLazy), + _ => Err(format!("Unknown codegen mode `{}`", s)), + } + } +} + +/// Configuration of cg_clif as passed in through `-Cllvm-args` and various env vars. +#[derive(Clone, Debug)] +pub struct BackendConfig { + /// Should the crate be AOT compiled or JIT executed. + /// + /// Defaults to AOT compilation. Can be set using `-Cllvm-args=mode=...`. + pub codegen_mode: CodegenMode, + + /// When JIT mode is enable pass these arguments to the program. + /// + /// Defaults to the value of `CG_CLIF_JIT_ARGS`. + pub jit_args: Vec, + + /// Display the time it took to perform codegen for a crate. + /// + /// Defaults to true when the `CG_CLIF_DISPLAY_CG_TIME` env var is set to 1 or false otherwise. + /// Can be set using `-Cllvm-args=display_cg_time=...`. + pub display_cg_time: bool, + + /// The register allocator to use. + /// + /// Defaults to the value of `CG_CLIF_REGALLOC` or `backtracking` otherwise. Can be set using + /// `-Cllvm-args=regalloc=...`. + pub regalloc: String, + + /// Enable the Cranelift ir verifier for all compilation passes. If not set it will only run + /// once before passing the clif ir to Cranelift for compilation. + /// + /// Defaults to true when the `CG_CLIF_ENABLE_VERIFIER` env var is set to 1 or when cg_clif is + /// compiled with debug assertions enabled or false otherwise. Can be set using + /// `-Cllvm-args=enable_verifier=...`. + pub enable_verifier: bool, + + /// Don't cache object files in the incremental cache. Useful during development of cg_clif + /// to make it possible to use incremental mode for all analyses performed by rustc without + /// caching object files when their content should have been changed by a change to cg_clif. + /// + /// Defaults to true when the `CG_CLIF_DISABLE_INCR_CACHE` env var is set to 1 or false + /// otherwise. Can be set using `-Cllvm-args=disable_incr_cache=...`. + pub disable_incr_cache: bool, +} + +impl Default for BackendConfig { + fn default() -> Self { + BackendConfig { + codegen_mode: CodegenMode::Aot, + jit_args: { + let args = std::env::var("CG_CLIF_JIT_ARGS").unwrap_or_else(|_| String::new()); + args.split(' ').map(|arg| arg.to_string()).collect() + }, + display_cg_time: bool_env_var("CG_CLIF_DISPLAY_CG_TIME"), + regalloc: std::env::var("CG_CLIF_REGALLOC") + .unwrap_or_else(|_| "backtracking".to_string()), + enable_verifier: cfg!(debug_assertions) || bool_env_var("CG_CLIF_ENABLE_VERIFIER"), + disable_incr_cache: bool_env_var("CG_CLIF_DISABLE_INCR_CACHE"), + } + } +} + +impl BackendConfig { + /// Parse the configuration passed in using `-Cllvm-args`. + pub fn from_opts(opts: &[String]) -> Result { + fn parse_bool(name: &str, value: &str) -> Result { + value.parse().map_err(|_| format!("failed to parse value `{}` for {}", value, name)) + } + + let mut config = BackendConfig::default(); + for opt in opts { + if let Some((name, value)) = opt.split_once('=') { + match name { + "mode" => config.codegen_mode = value.parse()?, + "display_cg_time" => config.display_cg_time = parse_bool(name, value)?, + "regalloc" => config.regalloc = value.to_string(), + "enable_verifier" => config.enable_verifier = parse_bool(name, value)?, + "disable_incr_cache" => config.disable_incr_cache = parse_bool(name, value)?, + _ => return Err(format!("Unknown option `{}`", name)), + } + } else { + return Err(format!("Invalid option `{}`", opt)); + } + } + + Ok(config) + } +} diff --git a/compiler/rustc_codegen_cranelift/src/constant.rs b/compiler/rustc_codegen_cranelift/src/constant.rs index fcd41c8446..3ba12c4e96 100644 --- a/compiler/rustc_codegen_cranelift/src/constant.rs +++ b/compiler/rustc_codegen_cranelift/src/constant.rs @@ -2,11 +2,13 @@ use rustc_span::DUMMY_SP; -use rustc_data_structures::fx::FxHashSet; +use rustc_ast::Mutability; +use rustc_data_structures::fx::{FxHashMap, FxHashSet}; use rustc_errors::ErrorReported; use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags; use rustc_middle::mir::interpret::{ - read_target_uint, AllocId, Allocation, ConstValue, ErrorHandled, GlobalAlloc, Pointer, Scalar, + alloc_range, read_target_uint, AllocId, Allocation, ConstValue, ErrorHandled, GlobalAlloc, + Scalar, }; use rustc_middle::ty::ConstKind; @@ -15,10 +17,10 @@ use cranelift_module::*; use crate::prelude::*; -#[derive(Default)] pub(crate) struct ConstantCx { todo: Vec, done: FxHashSet, + anon_allocs: FxHashMap, } #[derive(Copy, Clone, Debug)] @@ -28,6 +30,10 @@ enum TodoItem { } impl ConstantCx { + pub(crate) fn new() -> Self { + ConstantCx { todo: vec![], done: FxHashSet::default(), anon_allocs: FxHashMap::default() } + } + pub(crate) fn finalize(mut self, tcx: TyCtxt<'_>, module: &mut dyn Module) { //println!("todo {:?}", self.todo); define_all_allocs(tcx, module, &mut self); @@ -74,8 +80,10 @@ pub(crate) fn check_constants(fx: &mut FunctionCx<'_, '_, '_>) -> bool { all_constants_ok } -pub(crate) fn codegen_static(constants_cx: &mut ConstantCx, def_id: DefId) { +pub(crate) fn codegen_static(tcx: TyCtxt<'_>, module: &mut dyn Module, def_id: DefId) { + let mut constants_cx = ConstantCx::new(); constants_cx.todo.push(TodoItem::Static(def_id)); + constants_cx.finalize(tcx, module); } pub(crate) fn codegen_tls_ref<'tcx>( @@ -83,8 +91,8 @@ pub(crate) fn codegen_tls_ref<'tcx>( def_id: DefId, layout: TyAndLayout<'tcx>, ) -> CValue<'tcx> { - let data_id = data_id_for_static(fx.tcx, fx.cx.module, def_id, false); - let local_data_id = fx.cx.module.declare_data_in_func(data_id, &mut fx.bcx.func); + let data_id = data_id_for_static(fx.tcx, fx.module, def_id, false); + let local_data_id = fx.module.declare_data_in_func(data_id, &mut fx.bcx.func); if fx.clif_comments.enabled() { fx.add_comment(local_data_id, format!("tls {:?}", def_id)); } @@ -97,8 +105,8 @@ fn codegen_static_ref<'tcx>( def_id: DefId, layout: TyAndLayout<'tcx>, ) -> CPlace<'tcx> { - let data_id = data_id_for_static(fx.tcx, fx.cx.module, def_id, false); - let local_data_id = fx.cx.module.declare_data_in_func(data_id, &mut fx.bcx.func); + let data_id = data_id_for_static(fx.tcx, fx.module, def_id, false); + let local_data_id = fx.module.declare_data_in_func(data_id, &mut fx.bcx.func); if fx.clif_comments.enabled() { fx.add_comment(local_data_id, format!("{:?}", def_id)); } @@ -169,9 +177,9 @@ pub(crate) fn codegen_const_value<'tcx>( let mut alloc = Allocation::from_bytes( std::iter::repeat(0).take(size.bytes_usize()).collect::>(), align, + Mutability::Not, ); - let ptr = Pointer::new(AllocId(!0), Size::ZERO); // The alloc id is never used - alloc.write_scalar(fx, ptr, x.into(), size).unwrap(); + alloc.write_scalar(fx, alloc_range(Size::ZERO, size), x.into()).unwrap(); let alloc = fx.tcx.intern_const_alloc(alloc); return CValue::by_ref(pointer_for_allocation(fx, alloc), layout); } @@ -182,28 +190,31 @@ pub(crate) fn codegen_const_value<'tcx>( let alloc_kind = fx.tcx.get_global_alloc(ptr.alloc_id); let base_addr = match alloc_kind { Some(GlobalAlloc::Memory(alloc)) => { - fx.cx.constants_cx.todo.push(TodoItem::Alloc(ptr.alloc_id)); - let data_id = - data_id_for_alloc_id(fx.cx.module, ptr.alloc_id, alloc.mutability); + fx.constants_cx.todo.push(TodoItem::Alloc(ptr.alloc_id)); + let data_id = data_id_for_alloc_id( + &mut fx.constants_cx, + fx.module, + ptr.alloc_id, + alloc.mutability, + ); let local_data_id = - fx.cx.module.declare_data_in_func(data_id, &mut fx.bcx.func); + fx.module.declare_data_in_func(data_id, &mut fx.bcx.func); if fx.clif_comments.enabled() { fx.add_comment(local_data_id, format!("{:?}", ptr.alloc_id)); } fx.bcx.ins().global_value(fx.pointer_type, local_data_id) } Some(GlobalAlloc::Function(instance)) => { - let func_id = - crate::abi::import_function(fx.tcx, fx.cx.module, instance); + let func_id = crate::abi::import_function(fx.tcx, fx.module, instance); let local_func_id = - fx.cx.module.declare_func_in_func(func_id, &mut fx.bcx.func); + fx.module.declare_func_in_func(func_id, &mut fx.bcx.func); fx.bcx.ins().func_addr(fx.pointer_type, local_func_id) } Some(GlobalAlloc::Static(def_id)) => { assert!(fx.tcx.is_static(def_id)); - let data_id = data_id_for_static(fx.tcx, fx.cx.module, def_id, false); + let data_id = data_id_for_static(fx.tcx, fx.module, def_id, false); let local_data_id = - fx.cx.module.declare_data_in_func(data_id, &mut fx.bcx.func); + fx.module.declare_data_in_func(data_id, &mut fx.bcx.func); if fx.clif_comments.enabled() { fx.add_comment(local_data_id, format!("{:?}", def_id)); } @@ -243,10 +254,11 @@ fn pointer_for_allocation<'tcx>( alloc: &'tcx Allocation, ) -> crate::pointer::Pointer { let alloc_id = fx.tcx.create_memory_alloc(alloc); - fx.cx.constants_cx.todo.push(TodoItem::Alloc(alloc_id)); - let data_id = data_id_for_alloc_id(fx.cx.module, alloc_id, alloc.mutability); + fx.constants_cx.todo.push(TodoItem::Alloc(alloc_id)); + let data_id = + data_id_for_alloc_id(&mut fx.constants_cx, &mut *fx.module, alloc_id, alloc.mutability); - let local_data_id = fx.cx.module.declare_data_in_func(data_id, &mut fx.bcx.func); + let local_data_id = fx.module.declare_data_in_func(data_id, &mut fx.bcx.func); if fx.clif_comments.enabled() { fx.add_comment(local_data_id, format!("{:?}", alloc_id)); } @@ -255,18 +267,14 @@ fn pointer_for_allocation<'tcx>( } fn data_id_for_alloc_id( + cx: &mut ConstantCx, module: &mut dyn Module, alloc_id: AllocId, mutability: rustc_hir::Mutability, ) -> DataId { - module - .declare_data( - &format!(".L__alloc_{:x}", alloc_id.0), - Linkage::Local, - mutability == rustc_hir::Mutability::Mut, - false, - ) - .unwrap() + *cx.anon_allocs.entry(alloc_id).or_insert_with(|| { + module.declare_anonymous_data(mutability == rustc_hir::Mutability::Mut, false).unwrap() + }) } fn data_id_for_static( @@ -344,7 +352,7 @@ fn define_all_allocs(tcx: TyCtxt<'_>, module: &mut dyn Module, cx: &mut Constant GlobalAlloc::Memory(alloc) => alloc, GlobalAlloc::Function(_) | GlobalAlloc::Static(_) => unreachable!(), }; - let data_id = data_id_for_alloc_id(module, alloc_id, alloc.mutability); + let data_id = data_id_for_alloc_id(cx, module, alloc_id, alloc.mutability); (data_id, alloc, None) } TodoItem::Static(def_id) => { @@ -368,8 +376,19 @@ fn define_all_allocs(tcx: TyCtxt<'_>, module: &mut dyn Module, cx: &mut Constant data_ctx.set_align(alloc.align.bytes()); if let Some(section_name) = section_name { - // FIXME set correct segment for Mach-O files - data_ctx.set_segment_section("", &*section_name); + let (segment_name, section_name) = if tcx.sess.target.is_like_osx { + if let Some(names) = section_name.split_once(',') { + names + } else { + tcx.sess.fatal(&format!( + "#[link_section = \"{}\"] is not valid for macos target: must be segment and section separated by comma", + section_name + )); + } + } else { + ("", &*section_name) + }; + data_ctx.set_segment_section(segment_name, section_name); } let bytes = alloc.inspect_with_uninit_and_ptr_outside_interpreter(0..alloc.len()).to_vec(); @@ -397,7 +416,7 @@ fn define_all_allocs(tcx: TyCtxt<'_>, module: &mut dyn Module, cx: &mut Constant } GlobalAlloc::Memory(target_alloc) => { cx.todo.push(TodoItem::Alloc(reloc)); - data_id_for_alloc_id(module, reloc, target_alloc.mutability) + data_id_for_alloc_id(cx, module, reloc, target_alloc.mutability) } GlobalAlloc::Static(def_id) => { if tcx.codegen_fn_attrs(def_id).flags.contains(CodegenFnAttrFlags::THREAD_LOCAL) @@ -419,8 +438,7 @@ fn define_all_allocs(tcx: TyCtxt<'_>, module: &mut dyn Module, cx: &mut Constant data_ctx.write_data_addr(offset.bytes() as u32, global_value, addend as i64); } - // FIXME don't duplicate definitions in lazy jit mode - let _ = module.define_data(data_id, &data_ctx); + module.define_data(data_id, &data_ctx).unwrap(); cx.done.insert(data_id); } @@ -432,12 +450,89 @@ pub(crate) fn mir_operand_get_const_val<'tcx>( operand: &Operand<'tcx>, ) -> Option> { match operand { - Operand::Copy(_) | Operand::Move(_) => None, Operand::Constant(const_) => match const_.literal { ConstantKind::Ty(const_) => { fx.monomorphize(const_).eval(fx.tcx, ParamEnv::reveal_all()).val.try_to_value() } ConstantKind::Val(val, _) => Some(val), }, + // FIXME(rust-lang/rust#85105): Casts like `IMM8 as u32` result in the const being stored + // inside a temporary before being passed to the intrinsic requiring the const argument. + // This code tries to find a single constant defining definition of the referenced local. + Operand::Copy(place) | Operand::Move(place) => { + if !place.projection.is_empty() { + return None; + } + let mut computed_const_val = None; + for bb_data in fx.mir.basic_blocks() { + for stmt in &bb_data.statements { + match &stmt.kind { + StatementKind::Assign(local_and_rvalue) if &local_and_rvalue.0 == place => { + match &local_and_rvalue.1 { + Rvalue::Cast(CastKind::Misc, operand, ty) => { + if computed_const_val.is_some() { + return None; // local assigned twice + } + if !matches!(ty.kind(), ty::Uint(_) | ty::Int(_)) { + return None; + } + let const_val = mir_operand_get_const_val(fx, operand)?; + if fx.layout_of(ty).size + != const_val.try_to_scalar_int()?.size() + { + return None; + } + computed_const_val = Some(const_val); + } + Rvalue::Use(operand) => { + computed_const_val = mir_operand_get_const_val(fx, operand) + } + _ => return None, + } + } + StatementKind::SetDiscriminant { place: stmt_place, variant_index: _ } + if &**stmt_place == place => + { + return None; + } + StatementKind::LlvmInlineAsm(_) | StatementKind::CopyNonOverlapping(_) => { + return None; + } // conservative handling + StatementKind::Assign(_) + | StatementKind::FakeRead(_) + | StatementKind::SetDiscriminant { .. } + | StatementKind::StorageLive(_) + | StatementKind::StorageDead(_) + | StatementKind::Retag(_, _) + | StatementKind::AscribeUserType(_, _) + | StatementKind::Coverage(_) + | StatementKind::Nop => {} + } + } + match &bb_data.terminator().kind { + TerminatorKind::Goto { .. } + | TerminatorKind::SwitchInt { .. } + | TerminatorKind::Resume + | TerminatorKind::Abort + | TerminatorKind::Return + | TerminatorKind::Unreachable + | TerminatorKind::Drop { .. } + | TerminatorKind::Assert { .. } => {} + TerminatorKind::DropAndReplace { .. } + | TerminatorKind::Yield { .. } + | TerminatorKind::GeneratorDrop + | TerminatorKind::FalseEdge { .. } + | TerminatorKind::FalseUnwind { .. } => unreachable!(), + TerminatorKind::InlineAsm { .. } => return None, + TerminatorKind::Call { destination: Some((call_place, _)), .. } + if call_place == place => + { + return None; + } + TerminatorKind::Call { .. } => {} + } + } + computed_const_val + } } } diff --git a/compiler/rustc_codegen_cranelift/src/debuginfo/line_info.rs b/compiler/rustc_codegen_cranelift/src/debuginfo/line_info.rs index 8578ab33ce..9eb0677063 100644 --- a/compiler/rustc_codegen_cranelift/src/debuginfo/line_info.rs +++ b/compiler/rustc_codegen_cranelift/src/debuginfo/line_info.rs @@ -66,7 +66,7 @@ fn line_program_add_file( ) -> FileId { match &file.name { FileName::Real(path) => { - let (dir_path, file_name) = split_path_dir_and_file(path.stable_name()); + let (dir_path, file_name) = split_path_dir_and_file(path.remapped_path_if_available()); let dir_name = osstr_as_utf8_bytes(dir_path.as_os_str()); let file_name = osstr_as_utf8_bytes(file_name); @@ -87,7 +87,7 @@ fn line_program_add_file( filename => { let dir_id = line_program.default_directory(); let dummy_file_name = LineString::new( - filename.to_string().into_bytes(), + filename.prefer_remapped().to_string().into_bytes(), line_program.encoding(), line_strings, ); diff --git a/compiler/rustc_codegen_cranelift/src/debuginfo/mod.rs b/compiler/rustc_codegen_cranelift/src/debuginfo/mod.rs index dc8bc8d9cb..61e54a76f2 100644 --- a/compiler/rustc_codegen_cranelift/src/debuginfo/mod.rs +++ b/compiler/rustc_codegen_cranelift/src/debuginfo/mod.rs @@ -64,7 +64,7 @@ impl<'tcx> DebugContext<'tcx> { // FIXME: how to get version when building out of tree? // Normally this would use option_env!("CFG_VERSION"). let producer = format!("cg_clif (rustc {})", "unknown version"); - let comp_dir = tcx.sess.working_dir.0.to_string_lossy().into_owned(); + let comp_dir = tcx.sess.working_dir.to_string_lossy(false).into_owned(); let (name, file_info) = match tcx.sess.local_crate_source_file.clone() { Some(path) => { let name = path.to_string_lossy().into_owned(); diff --git a/compiler/rustc_codegen_cranelift/src/debuginfo/unwind.rs b/compiler/rustc_codegen_cranelift/src/debuginfo/unwind.rs index 357c9fe6ed..d1251e749f 100644 --- a/compiler/rustc_codegen_cranelift/src/debuginfo/unwind.rs +++ b/compiler/rustc_codegen_cranelift/src/debuginfo/unwind.rs @@ -5,17 +5,19 @@ use crate::prelude::*; use cranelift_codegen::isa::{unwind::UnwindInfo, TargetIsa}; use gimli::write::{Address, CieId, EhFrame, FrameTable, Section}; +use gimli::RunTimeEndian; use crate::backend::WriteDebugInfo; -pub(crate) struct UnwindContext<'tcx> { - tcx: TyCtxt<'tcx>, +pub(crate) struct UnwindContext { + endian: RunTimeEndian, frame_table: FrameTable, cie_id: Option, } -impl<'tcx> UnwindContext<'tcx> { - pub(crate) fn new(tcx: TyCtxt<'tcx>, isa: &dyn TargetIsa, pic_eh_frame: bool) -> Self { +impl UnwindContext { + pub(crate) fn new(tcx: TyCtxt<'_>, isa: &dyn TargetIsa, pic_eh_frame: bool) -> Self { + let endian = super::target_endian(tcx); let mut frame_table = FrameTable::default(); let cie_id = if let Some(mut cie) = isa.create_systemv_cie() { @@ -28,7 +30,7 @@ impl<'tcx> UnwindContext<'tcx> { None }; - UnwindContext { tcx, frame_table, cie_id } + UnwindContext { endian, frame_table, cie_id } } pub(crate) fn add_function(&mut self, func_id: FuncId, context: &Context, isa: &dyn TargetIsa) { @@ -54,8 +56,7 @@ impl<'tcx> UnwindContext<'tcx> { } pub(crate) fn emit(self, product: &mut P) { - let mut eh_frame = - EhFrame::from(super::emit::WriterRelocate::new(super::target_endian(self.tcx))); + let mut eh_frame = EhFrame::from(super::emit::WriterRelocate::new(self.endian)); self.frame_table.write_eh_frame(&mut eh_frame).unwrap(); if !eh_frame.0.writer.slice().is_empty() { @@ -70,17 +71,16 @@ impl<'tcx> UnwindContext<'tcx> { } } - #[cfg(feature = "jit")] - pub(crate) unsafe fn register_jit( - self, - jit_module: &cranelift_jit::JITModule, - ) -> Option { - let mut eh_frame = - EhFrame::from(super::emit::WriterRelocate::new(super::target_endian(self.tcx))); + #[cfg(all(feature = "jit", windows))] + pub(crate) unsafe fn register_jit(self, _jit_module: &cranelift_jit::JITModule) {} + + #[cfg(all(feature = "jit", not(windows)))] + pub(crate) unsafe fn register_jit(self, jit_module: &cranelift_jit::JITModule) { + let mut eh_frame = EhFrame::from(super::emit::WriterRelocate::new(self.endian)); self.frame_table.write_eh_frame(&mut eh_frame).unwrap(); if eh_frame.0.writer.slice().is_empty() { - return None; + return; } let mut eh_frame = eh_frame.0.relocate_for_jit(jit_module); @@ -88,7 +88,10 @@ impl<'tcx> UnwindContext<'tcx> { // GCC expects a terminating "empty" length, so write a 0 length at the end of the table. eh_frame.extend(&[0, 0, 0, 0]); - let mut registrations = Vec::new(); + // FIXME support unregistering unwind tables once cranelift-jit supports deallocating + // individual functions + #[allow(unused_variables)] + let (eh_frame, eh_frame_len, _) = Vec::into_raw_parts(eh_frame); // ======================================================================= // Everything after this line up to the end of the file is loosly based on @@ -96,8 +99,8 @@ impl<'tcx> UnwindContext<'tcx> { #[cfg(target_os = "macos")] { // On macOS, `__register_frame` takes a pointer to a single FDE - let start = eh_frame.as_ptr(); - let end = start.add(eh_frame.len()); + let start = eh_frame; + let end = start.add(eh_frame_len); let mut current = start; // Walk all of the entries in the frame table and register them @@ -107,7 +110,6 @@ impl<'tcx> UnwindContext<'tcx> { // Skip over the CIE if current != start { __register_frame(current); - registrations.push(current as usize); } // Move to the next table entry (+4 because the length itself is not inclusive) @@ -117,41 +119,12 @@ impl<'tcx> UnwindContext<'tcx> { #[cfg(not(target_os = "macos"))] { // On other platforms, `__register_frame` will walk the FDEs until an entry of length 0 - let ptr = eh_frame.as_ptr(); - __register_frame(ptr); - registrations.push(ptr as usize); + __register_frame(eh_frame); } - - Some(UnwindRegistry { _frame_table: eh_frame, registrations }) } } -/// Represents a registry of function unwind information for System V ABI. -pub(crate) struct UnwindRegistry { - _frame_table: Vec, - registrations: Vec, -} - extern "C" { // libunwind import fn __register_frame(fde: *const u8); - fn __deregister_frame(fde: *const u8); -} - -impl Drop for UnwindRegistry { - fn drop(&mut self) { - unsafe { - // libgcc stores the frame entries as a linked list in decreasing sort order - // based on the PC value of the registered entry. - // - // As we store the registrations in increasing order, it would be O(N^2) to - // deregister in that order. - // - // To ensure that we just pop off the first element in the list upon every - // deregistration, walk our list of registrations backwards. - for fde in self.registrations.iter().rev() { - __deregister_frame(*fde as *const _); - } - } - } } diff --git a/compiler/rustc_codegen_cranelift/src/driver/aot.rs b/compiler/rustc_codegen_cranelift/src/driver/aot.rs index ed3bdedddc..6676d88602 100644 --- a/compiler/rustc_codegen_cranelift/src/driver/aot.rs +++ b/compiler/rustc_codegen_cranelift/src/driver/aot.rs @@ -3,6 +3,7 @@ use std::path::PathBuf; +use rustc_ast::{InlineAsmOptions, InlineAsmTemplatePiece}; use rustc_codegen_ssa::back::linker::LinkerInfo; use rustc_codegen_ssa::{CodegenResults, CompiledModule, CrateInfo, ModuleKind}; use rustc_data_structures::stable_hasher::{HashStable, StableHasher}; @@ -16,12 +17,6 @@ use cranelift_object::ObjectModule; use crate::{prelude::*, BackendConfig}; -fn new_module(tcx: TyCtxt<'_>, name: String) -> ObjectModule { - let module = crate::backend::make_module(tcx.sess, name); - assert_eq!(pointer_ty(tcx), module.target_config().pointer_type()); - module -} - struct ModuleCodegenResult(CompiledModule, Option<(WorkProductId, WorkProduct)>); impl HashStable for ModuleCodegenResult { @@ -32,11 +27,12 @@ impl HashStable for ModuleCodegenResult { fn emit_module( tcx: TyCtxt<'_>, + backend_config: &BackendConfig, name: String, kind: ModuleKind, module: ObjectModule, debug: Option>, - unwind_context: UnwindContext<'_>, + unwind_context: UnwindContext, ) -> ModuleCodegenResult { let mut product = module.finish(); @@ -46,13 +42,13 @@ fn emit_module( unwind_context.emit(&mut product); - let tmp_file = tcx.output_filenames(LOCAL_CRATE).temp_path(OutputType::Object, Some(&name)); + let tmp_file = tcx.output_filenames(()).temp_path(OutputType::Object, Some(&name)); let obj = product.object.write().unwrap(); if let Err(err) = std::fs::write(&tmp_file, obj) { tcx.sess.fatal(&format!("error writing object file: {}", err)); } - let work_product = if std::env::var("CG_CLIF_INCR_CACHE_DISABLED").is_ok() { + let work_product = if backend_config.disable_incr_cache { None } else { rustc_incremental::copy_cgu_workproduct_to_incr_comp_cache_dir( @@ -77,9 +73,8 @@ fn reuse_workproduct_for_cgu( let mut object = None; let work_product = cgu.work_product(tcx); if let Some(saved_file) = &work_product.saved_file { - let obj_out = tcx - .output_filenames(LOCAL_CRATE) - .temp_path(OutputType::Object, Some(&cgu.name().as_str())); + let obj_out = + tcx.output_filenames(()).temp_path(OutputType::Object, Some(&cgu.name().as_str())); object = Some(obj_out.clone()); let source_file = rustc_incremental::in_incr_comp_dir(&incr_comp_session_dir, &saved_file); if let Err(err) = rustc_fs_util::link_or_copy(&source_file, &obj_out) { @@ -110,77 +105,82 @@ fn module_codegen( let cgu = tcx.codegen_unit(cgu_name); let mono_items = cgu.items_in_deterministic_order(tcx); - let mut module = new_module(tcx, cgu_name.as_str().to_string()); + let isa = crate::build_isa(tcx.sess, &backend_config); + let mut module = crate::backend::make_module(tcx.sess, isa, cgu_name.as_str().to_string()); let mut cx = crate::CodegenCx::new( tcx, - backend_config, - &mut module, + backend_config.clone(), + module.isa(), tcx.sess.opts.debuginfo != DebugInfo::None, ); - super::predefine_mono_items(&mut cx, &mono_items); + super::predefine_mono_items(tcx, &mut module, &mono_items); for (mono_item, _) in mono_items { match mono_item { MonoItem::Fn(inst) => { - cx.tcx.sess.time("codegen fn", || crate::base::codegen_fn(&mut cx, inst)); - } - MonoItem::Static(def_id) => { - crate::constant::codegen_static(&mut cx.constants_cx, def_id) + cx.tcx + .sess + .time("codegen fn", || crate::base::codegen_fn(&mut cx, &mut module, inst)); } + MonoItem::Static(def_id) => crate::constant::codegen_static(tcx, &mut module, def_id), MonoItem::GlobalAsm(item_id) => { let item = cx.tcx.hir().item(item_id); - if let rustc_hir::ItemKind::GlobalAsm(rustc_hir::GlobalAsm { asm }) = item.kind { - cx.global_asm.push_str(&*asm.as_str()); - cx.global_asm.push_str("\n\n"); + if let rustc_hir::ItemKind::GlobalAsm(asm) = item.kind { + if !asm.options.contains(InlineAsmOptions::ATT_SYNTAX) { + cx.global_asm.push_str("\n.intel_syntax noprefix\n"); + } else { + cx.global_asm.push_str("\n.att_syntax\n"); + } + for piece in asm.template { + match *piece { + InlineAsmTemplatePiece::String(ref s) => cx.global_asm.push_str(s), + InlineAsmTemplatePiece::Placeholder { .. } => todo!(), + } + } + cx.global_asm.push_str("\n.att_syntax\n\n"); } else { bug!("Expected GlobalAsm found {:?}", item); } } } } - let (global_asm, debug, mut unwind_context) = - tcx.sess.time("finalize CodegenCx", || cx.finalize()); - crate::main_shim::maybe_create_entry_wrapper(tcx, &mut module, &mut unwind_context); - - let codegen_result = emit_module( + crate::main_shim::maybe_create_entry_wrapper( tcx, - cgu.name().as_str().to_string(), - ModuleKind::Regular, - module, - debug, - unwind_context, + &mut module, + &mut cx.unwind_context, + false, + cgu.is_primary(), ); - codegen_global_asm(tcx, &cgu.name().as_str(), &global_asm); + let debug_context = cx.debug_context; + let unwind_context = cx.unwind_context; + let codegen_result = tcx.sess.time("write object file", || { + emit_module( + tcx, + &backend_config, + cgu.name().as_str().to_string(), + ModuleKind::Regular, + module, + debug_context, + unwind_context, + ) + }); + + codegen_global_asm(tcx, &cgu.name().as_str(), &cx.global_asm); codegen_result } -pub(super) fn run_aot( +pub(crate) fn run_aot( tcx: TyCtxt<'_>, backend_config: BackendConfig, metadata: EncodedMetadata, need_metadata_module: bool, ) -> Box<(CodegenResults, FxHashMap)> { - use rustc_span::symbol::sym; - - let crate_attrs = tcx.hir().attrs(rustc_hir::CRATE_HIR_ID); - let subsystem = tcx.sess.first_attr_value_str_by_name(crate_attrs, sym::windows_subsystem); - let windows_subsystem = subsystem.map(|subsystem| { - if subsystem != sym::windows && subsystem != sym::console { - tcx.sess.fatal(&format!( - "invalid windows subsystem `{}`, only \ - `windows` and `console` are allowed", - subsystem - )); - } - subsystem.to_string() - }); - let mut work_products = FxHashMap::default(); let cgus = if tcx.sess.opts.output_types.should_codegen() { - tcx.collect_and_partition_mono_items(LOCAL_CRATE).1 + tcx.collect_and_partition_mono_items(()).1 } else { // If only `--emit metadata` is used, we shouldn't perform any codegen. // Also `tcx.collect_and_partition_mono_items` may panic in that case. @@ -193,14 +193,14 @@ pub(super) fn run_aot( } } - let modules = super::time(tcx, "codegen mono items", || { + let modules = super::time(tcx, backend_config.display_cg_time, "codegen mono items", || { cgus.iter() .map(|cgu| { let cgu_reuse = determine_cgu_reuse(tcx, cgu); tcx.sess.cgu_reuse_tracker.set_actual_reuse(&cgu.name().as_str(), cgu_reuse); match cgu_reuse { - _ if std::env::var("CG_CLIF_INCR_CACHE_DISABLED").is_ok() => {} + _ if backend_config.disable_incr_cache => {} CguReuse::No => {} CguReuse::PreLto => { return reuse_workproduct_for_cgu(tcx, &*cgu, &mut work_products); @@ -212,7 +212,7 @@ pub(super) fn run_aot( let (ModuleCodegenResult(module, work_product), _) = tcx.dep_graph.with_task( dep_node, tcx, - (backend_config, cgu.name()), + (backend_config.clone(), cgu.name()), module_codegen, rustc_middle::dep_graph::hash_result, ); @@ -228,7 +228,10 @@ pub(super) fn run_aot( tcx.sess.abort_if_errors(); - let mut allocator_module = new_module(tcx, "allocator_shim".to_string()); + let isa = crate::build_isa(tcx.sess, &backend_config); + let mut allocator_module = + crate::backend::make_module(tcx.sess, isa, "allocator_shim".to_string()); + assert_eq!(pointer_ty(tcx), allocator_module.target_config().pointer_type()); let mut allocator_unwind_context = UnwindContext::new(tcx, allocator_module.isa(), true); let created_alloc_shim = crate::allocator::codegen(tcx, &mut allocator_module, &mut allocator_unwind_context); @@ -236,6 +239,7 @@ pub(super) fn run_aot( let allocator_module = if created_alloc_shim { let ModuleCodegenResult(module, work_product) = emit_module( tcx, + &backend_config, "allocator_shim".to_string(), ModuleKind::Allocator, allocator_module, @@ -261,9 +265,8 @@ pub(super) fn run_aot( .as_str() .to_string(); - let tmp_file = tcx - .output_filenames(LOCAL_CRATE) - .temp_path(OutputType::Metadata, Some(&metadata_cgu_name)); + let tmp_file = + tcx.output_filenames(()).temp_path(OutputType::Metadata, Some(&metadata_cgu_name)); let obj = crate::backend::with_object(tcx.sess, &metadata_cgu_name, |object| { crate::metadata::write_metadata(tcx, object); @@ -289,13 +292,11 @@ pub(super) fn run_aot( Box::new(( CodegenResults { - crate_name: tcx.crate_name(LOCAL_CRATE), modules, allocator_module, metadata_module, metadata, - windows_subsystem, - linker_info: LinkerInfo::new(tcx), + linker_info: LinkerInfo::new(tcx, crate::target_triple(tcx.sess).to_string()), crate_info: CrateInfo::new(tcx), }, work_products, @@ -338,8 +339,7 @@ fn codegen_global_asm(tcx: TyCtxt<'_>, cgu_name: &str, global_asm: &str) { .collect::>() .join("\n"); - let output_object_file = - tcx.output_filenames(LOCAL_CRATE).temp_path(OutputType::Object, Some(cgu_name)); + let output_object_file = tcx.output_filenames(()).temp_path(OutputType::Object, Some(cgu_name)); // Assemble `global_asm` let global_asm_object_file = add_file_stem_postfix(output_object_file.clone(), ".asm"); diff --git a/compiler/rustc_codegen_cranelift/src/driver/jit.rs b/compiler/rustc_codegen_cranelift/src/driver/jit.rs index f585ebca9d..4a99cb727c 100644 --- a/compiler/rustc_codegen_cranelift/src/driver/jit.rs +++ b/compiler/rustc_codegen_cranelift/src/driver/jit.rs @@ -1,4 +1,4 @@ -//! The JIT driver uses [`cranelift_simplejit`] to JIT execute programs without writing any object +//! The JIT driver uses [`cranelift_jit`] to JIT execute programs without writing any object //! files. use std::cell::RefCell; @@ -8,34 +8,65 @@ use std::os::raw::{c_char, c_int}; use cranelift_codegen::binemit::{NullStackMapSink, NullTrapSink}; use rustc_codegen_ssa::CrateInfo; use rustc_middle::mir::mono::MonoItem; -use rustc_session::config::EntryFnType; use cranelift_jit::{JITBuilder, JITModule}; use crate::{prelude::*, BackendConfig}; use crate::{CodegenCx, CodegenMode}; -thread_local! { - pub static BACKEND_CONFIG: RefCell> = RefCell::new(None); - pub static CURRENT_MODULE: RefCell> = RefCell::new(None); +struct JitState { + backend_config: BackendConfig, + jit_module: JITModule, } -pub(super) fn run_jit(tcx: TyCtxt<'_>, backend_config: BackendConfig) -> ! { - if !tcx.sess.opts.output_types.should_codegen() { - tcx.sess.fatal("JIT mode doesn't work with `cargo check`."); - } +thread_local! { + static LAZY_JIT_STATE: RefCell> = RefCell::new(None); +} +fn create_jit_module<'tcx>( + tcx: TyCtxt<'tcx>, + backend_config: &BackendConfig, + hotswap: bool, +) -> (JITModule, CodegenCx<'tcx>) { let imported_symbols = load_imported_symbols_for_jit(tcx); - let mut jit_builder = - JITBuilder::with_isa(crate::build_isa(tcx.sess), cranelift_module::default_libcall_names()); - jit_builder.hotswap(matches!(backend_config.codegen_mode, CodegenMode::JitLazy)); + let isa = crate::build_isa(tcx.sess, backend_config); + let mut jit_builder = JITBuilder::with_isa(isa, cranelift_module::default_libcall_names()); + jit_builder.hotswap(hotswap); crate::compiler_builtins::register_functions_for_jit(&mut jit_builder); jit_builder.symbols(imported_symbols); let mut jit_module = JITModule::new(jit_builder); - assert_eq!(pointer_ty(tcx), jit_module.target_config().pointer_type()); - let (_, cgus) = tcx.collect_and_partition_mono_items(LOCAL_CRATE); + let mut cx = crate::CodegenCx::new(tcx, backend_config.clone(), jit_module.isa(), false); + + crate::allocator::codegen(tcx, &mut jit_module, &mut cx.unwind_context); + crate::main_shim::maybe_create_entry_wrapper( + tcx, + &mut jit_module, + &mut cx.unwind_context, + true, + true, + ); + + (jit_module, cx) +} + +pub(crate) fn run_jit(tcx: TyCtxt<'_>, backend_config: BackendConfig) -> ! { + if !tcx.sess.opts.output_types.should_codegen() { + tcx.sess.fatal("JIT mode doesn't work with `cargo check`"); + } + + if !tcx.sess.crate_types().contains(&rustc_session::config::CrateType::Executable) { + tcx.sess.fatal("can't jit non-executable crate"); + } + + let (mut jit_module, mut cx) = create_jit_module( + tcx, + &backend_config, + matches!(backend_config.codegen_mode, CodegenMode::JitLazy), + ); + + let (_, cgus) = tcx.collect_and_partition_mono_items(()); let mono_items = cgus .iter() .map(|cgu| cgu.items_in_deterministic_order(tcx).into_iter()) @@ -44,52 +75,45 @@ pub(super) fn run_jit(tcx: TyCtxt<'_>, backend_config: BackendConfig) -> ! { .into_iter() .collect::>(); - let mut cx = crate::CodegenCx::new(tcx, backend_config, &mut jit_module, false); - - super::time(tcx, "codegen mono items", || { - super::predefine_mono_items(&mut cx, &mono_items); + super::time(tcx, backend_config.display_cg_time, "codegen mono items", || { + super::predefine_mono_items(tcx, &mut jit_module, &mono_items); for (mono_item, _) in mono_items { match mono_item { MonoItem::Fn(inst) => match backend_config.codegen_mode { CodegenMode::Aot => unreachable!(), CodegenMode::Jit => { - cx.tcx.sess.time("codegen fn", || crate::base::codegen_fn(&mut cx, inst)); + cx.tcx.sess.time("codegen fn", || { + crate::base::codegen_fn(&mut cx, &mut jit_module, inst) + }); } - CodegenMode::JitLazy => codegen_shim(&mut cx, inst), + CodegenMode::JitLazy => codegen_shim(&mut cx, &mut jit_module, inst), }, MonoItem::Static(def_id) => { - crate::constant::codegen_static(&mut cx.constants_cx, def_id); + crate::constant::codegen_static(tcx, &mut jit_module, def_id); } MonoItem::GlobalAsm(item_id) => { - let item = cx.tcx.hir().item(item_id); + let item = tcx.hir().item(item_id); tcx.sess.span_fatal(item.span, "Global asm is not supported in JIT mode"); } } } }); - let (global_asm, _debug, mut unwind_context) = - tcx.sess.time("finalize CodegenCx", || cx.finalize()); - jit_module.finalize_definitions(); - - if !global_asm.is_empty() { + if !cx.global_asm.is_empty() { tcx.sess.fatal("Inline asm is not supported in JIT mode"); } - crate::allocator::codegen(tcx, &mut jit_module, &mut unwind_context); - tcx.sess.abort_if_errors(); jit_module.finalize_definitions(); - let _unwind_register_guard = unsafe { unwind_context.register_jit(&jit_module) }; + unsafe { cx.unwind_context.register_jit(&jit_module) }; println!( "Rustc codegen cranelift will JIT run the executable, because -Cllvm-args=mode=jit was passed" ); - let args = ::std::env::var("CG_CLIF_JIT_ARGS").unwrap_or_else(|_| String::new()); let args = std::iter::once(&*tcx.crate_name(LOCAL_CRATE).as_str().to_string()) - .chain(args.split(' ')) + .chain(backend_config.jit_args.iter().map(|arg| &**arg)) .map(|arg| CString::new(arg).unwrap()) .collect::>(); let mut argv = args.iter().map(|arg| arg.as_ptr()).collect::>(); @@ -98,61 +122,27 @@ pub(super) fn run_jit(tcx: TyCtxt<'_>, backend_config: BackendConfig) -> ! { // useful as some dynamic linkers use it as a marker to jump over. argv.push(std::ptr::null()); - BACKEND_CONFIG.with(|tls_backend_config| { - assert!(tls_backend_config.borrow_mut().replace(backend_config).is_none()) + let start_sig = Signature { + params: vec![ + AbiParam::new(jit_module.target_config().pointer_type()), + AbiParam::new(jit_module.target_config().pointer_type()), + ], + returns: vec![AbiParam::new(jit_module.target_config().pointer_type() /*isize*/)], + call_conv: CallConv::triple_default(&crate::target_triple(tcx.sess)), + }; + let start_func_id = jit_module.declare_function("main", Linkage::Import, &start_sig).unwrap(); + let finalized_start: *const u8 = jit_module.get_finalized_function(start_func_id); + + LAZY_JIT_STATE.with(|lazy_jit_state| { + let mut lazy_jit_state = lazy_jit_state.borrow_mut(); + assert!(lazy_jit_state.is_none()); + *lazy_jit_state = Some(JitState { backend_config, jit_module }); }); - let (main_def_id, entry_ty) = tcx.entry_fn(LOCAL_CRATE).unwrap(); - let instance = Instance::mono(tcx, main_def_id).polymorphize(tcx); - - match entry_ty { - EntryFnType::Main => { - // FIXME set program arguments somehow - - let main_sig = Signature { - params: vec![], - returns: vec![], - call_conv: CallConv::triple_default(&crate::target_triple(tcx.sess)), - }; - let main_func_id = jit_module - .declare_function(tcx.symbol_name(instance).name, Linkage::Import, &main_sig) - .unwrap(); - let finalized_main: *const u8 = jit_module.get_finalized_function(main_func_id); - - CURRENT_MODULE.with(|current_module| { - assert!(current_module.borrow_mut().replace(jit_module).is_none()) - }); - - let f: extern "C" fn() = unsafe { ::std::mem::transmute(finalized_main) }; - f(); - std::process::exit(0); - } - EntryFnType::Start => { - let start_sig = Signature { - params: vec![ - AbiParam::new(jit_module.target_config().pointer_type()), - AbiParam::new(jit_module.target_config().pointer_type()), - ], - returns: vec![AbiParam::new( - jit_module.target_config().pointer_type(), /*isize*/ - )], - call_conv: CallConv::triple_default(&crate::target_triple(tcx.sess)), - }; - let start_func_id = jit_module - .declare_function(tcx.symbol_name(instance).name, Linkage::Import, &start_sig) - .unwrap(); - let finalized_start: *const u8 = jit_module.get_finalized_function(start_func_id); - - CURRENT_MODULE.with(|current_module| { - assert!(current_module.borrow_mut().replace(jit_module).is_none()) - }); - - let f: extern "C" fn(c_int, *const *const c_char) -> c_int = - unsafe { ::std::mem::transmute(finalized_start) }; - let ret = f(args.len() as c_int, argv.as_ptr()); - std::process::exit(ret); - } - } + let f: extern "C" fn(c_int, *const *const c_char) -> c_int = + unsafe { ::std::mem::transmute(finalized_start) }; + let ret = f(args.len() as c_int, argv.as_ptr()); + std::process::exit(ret); } #[no_mangle] @@ -161,24 +151,23 @@ extern "C" fn __clif_jit_fn(instance_ptr: *const Instance<'static>) -> *const u8 // lift is used to ensure the correct lifetime for instance. let instance = tcx.lift(unsafe { *instance_ptr }).unwrap(); - CURRENT_MODULE.with(|jit_module| { - let mut jit_module = jit_module.borrow_mut(); - let jit_module = jit_module.as_mut().unwrap(); - let backend_config = - BACKEND_CONFIG.with(|backend_config| backend_config.borrow().clone().unwrap()); + LAZY_JIT_STATE.with(|lazy_jit_state| { + let mut lazy_jit_state = lazy_jit_state.borrow_mut(); + let lazy_jit_state = lazy_jit_state.as_mut().unwrap(); + let jit_module = &mut lazy_jit_state.jit_module; + let backend_config = lazy_jit_state.backend_config.clone(); - let name = tcx.symbol_name(instance).name.to_string(); + let name = tcx.symbol_name(instance).name; let sig = crate::abi::get_function_sig(tcx, jit_module.isa().triple(), instance); - let func_id = jit_module.declare_function(&name, Linkage::Export, &sig).unwrap(); + let func_id = jit_module.declare_function(name, Linkage::Export, &sig).unwrap(); jit_module.prepare_for_function_redefine(func_id).unwrap(); - let mut cx = crate::CodegenCx::new(tcx, backend_config, jit_module, false); - tcx.sess.time("codegen fn", || crate::base::codegen_fn(&mut cx, instance)); + let mut cx = crate::CodegenCx::new(tcx, backend_config, jit_module.isa(), false); + tcx.sess.time("codegen fn", || crate::base::codegen_fn(&mut cx, jit_module, instance)); - let (global_asm, _debug_context, unwind_context) = cx.finalize(); - assert!(global_asm.is_empty()); + assert!(cx.global_asm.is_empty()); jit_module.finalize_definitions(); - std::mem::forget(unsafe { unwind_context.register_jit(&jit_module) }); + unsafe { cx.unwind_context.register_jit(&jit_module) }; jit_module.get_finalized_function(func_id) }) }) @@ -190,7 +179,7 @@ fn load_imported_symbols_for_jit(tcx: TyCtxt<'_>) -> Vec<(String, *const u8)> { let mut dylib_paths = Vec::new(); let crate_info = CrateInfo::new(tcx); - let formats = tcx.dependency_formats(LOCAL_CRATE); + let formats = tcx.dependency_formats(()); let data = &formats .iter() .find(|(crate_type, _data)| *crate_type == rustc_session::config::CrateType::Executable) @@ -218,7 +207,7 @@ fn load_imported_symbols_for_jit(tcx: TyCtxt<'_>) -> Vec<(String, *const u8)> { use object::{Object, ObjectSymbol}; let lib = libloading::Library::new(&path).unwrap(); let obj = std::fs::read(path).unwrap(); - let obj = object::File::parse(&obj).unwrap(); + let obj = object::File::parse(&*obj).unwrap(); imported_symbols.extend(obj.dynamic_symbols().filter_map(|symbol| { let name = symbol.name().unwrap().to_string(); if name.is_empty() || !symbol.is_global() || symbol.is_undefined() { @@ -248,35 +237,37 @@ fn load_imported_symbols_for_jit(tcx: TyCtxt<'_>) -> Vec<(String, *const u8)> { imported_symbols } -fn codegen_shim<'tcx>(cx: &mut CodegenCx<'_, 'tcx>, inst: Instance<'tcx>) { +fn codegen_shim<'tcx>(cx: &mut CodegenCx<'tcx>, module: &mut JITModule, inst: Instance<'tcx>) { let tcx = cx.tcx; - let pointer_type = cx.module.target_config().pointer_type(); + let pointer_type = module.target_config().pointer_type(); - let name = tcx.symbol_name(inst).name.to_string(); - let sig = crate::abi::get_function_sig(tcx, cx.module.isa().triple(), inst); - let func_id = cx.module.declare_function(&name, Linkage::Export, &sig).unwrap(); + let name = tcx.symbol_name(inst).name; + let sig = crate::abi::get_function_sig(tcx, module.isa().triple(), inst); + let func_id = module.declare_function(name, Linkage::Export, &sig).unwrap(); let instance_ptr = Box::into_raw(Box::new(inst)); - let jit_fn = cx - .module + let jit_fn = module .declare_function( "__clif_jit_fn", Linkage::Import, &Signature { - call_conv: cx.module.target_config().default_call_conv, + call_conv: module.target_config().default_call_conv, params: vec![AbiParam::new(pointer_type)], returns: vec![AbiParam::new(pointer_type)], }, ) .unwrap(); - let mut trampoline = Function::with_name_signature(ExternalName::default(), sig.clone()); + cx.cached_context.clear(); + let trampoline = &mut cx.cached_context.func; + trampoline.signature = sig.clone(); + let mut builder_ctx = FunctionBuilderContext::new(); - let mut trampoline_builder = FunctionBuilder::new(&mut trampoline, &mut builder_ctx); + let mut trampoline_builder = FunctionBuilder::new(trampoline, &mut builder_ctx); - let jit_fn = cx.module.declare_func_in_func(jit_fn, trampoline_builder.func); + let jit_fn = module.declare_func_in_func(jit_fn, trampoline_builder.func); let sig_ref = trampoline_builder.func.import_signature(sig); let entry_block = trampoline_builder.create_block(); @@ -291,10 +282,10 @@ fn codegen_shim<'tcx>(cx: &mut CodegenCx<'_, 'tcx>, inst: Instance<'tcx>) { let ret_vals = trampoline_builder.func.dfg.inst_results(call_inst).to_vec(); trampoline_builder.ins().return_(&ret_vals); - cx.module + module .define_function( func_id, - &mut Context::for_function(trampoline), + &mut cx.cached_context, &mut NullTrapSink {}, &mut NullStackMapSink {}, ) diff --git a/compiler/rustc_codegen_cranelift/src/driver/mod.rs b/compiler/rustc_codegen_cranelift/src/driver/mod.rs index d49182a07b..8f5714ecb4 100644 --- a/compiler/rustc_codegen_cranelift/src/driver/mod.rs +++ b/compiler/rustc_codegen_cranelift/src/driver/mod.rs @@ -1,63 +1,37 @@ -//! Drivers are responsible for calling [`codegen_mono_item`] and performing any further actions -//! like JIT executing or writing object files. +//! Drivers are responsible for calling [`codegen_fn`] or [`codegen_static`] for each mono item and +//! performing any further actions like JIT executing or writing object files. +//! +//! [`codegen_fn`]: crate::base::codegen_fn +//! [`codegen_static`]: crate::constant::codegen_static -use std::any::Any; - -use rustc_middle::middle::cstore::EncodedMetadata; use rustc_middle::mir::mono::{Linkage as RLinkage, MonoItem, Visibility}; use crate::prelude::*; -use crate::CodegenMode; -mod aot; +pub(crate) mod aot; #[cfg(feature = "jit")] -mod jit; - -pub(crate) fn codegen_crate( - tcx: TyCtxt<'_>, - metadata: EncodedMetadata, - need_metadata_module: bool, - backend_config: crate::BackendConfig, -) -> Box { - tcx.sess.abort_if_errors(); - - match backend_config.codegen_mode { - CodegenMode::Aot => aot::run_aot(tcx, backend_config, metadata, need_metadata_module), - CodegenMode::Jit | CodegenMode::JitLazy => { - let is_executable = - tcx.sess.crate_types().contains(&rustc_session::config::CrateType::Executable); - if !is_executable { - tcx.sess.fatal("can't jit non-executable crate"); - } - - #[cfg(feature = "jit")] - let _: ! = jit::run_jit(tcx, backend_config); - - #[cfg(not(feature = "jit"))] - tcx.sess.fatal("jit support was disabled when compiling rustc_codegen_cranelift"); - } - } -} +pub(crate) mod jit; fn predefine_mono_items<'tcx>( - cx: &mut crate::CodegenCx<'_, 'tcx>, + tcx: TyCtxt<'tcx>, + module: &mut dyn Module, mono_items: &[(MonoItem<'tcx>, (RLinkage, Visibility))], ) { - cx.tcx.sess.time("predefine functions", || { - let is_compiler_builtins = cx.tcx.is_compiler_builtins(LOCAL_CRATE); + tcx.sess.time("predefine functions", || { + let is_compiler_builtins = tcx.is_compiler_builtins(LOCAL_CRATE); for &(mono_item, (linkage, visibility)) in mono_items { match mono_item { MonoItem::Fn(instance) => { - let name = cx.tcx.symbol_name(instance).name.to_string(); + let name = tcx.symbol_name(instance).name; let _inst_guard = crate::PrintOnPanic(|| format!("{:?} {}", instance, name)); - let sig = get_function_sig(cx.tcx, cx.module.isa().triple(), instance); + let sig = get_function_sig(tcx, module.isa().triple(), instance); let linkage = crate::linkage::get_clif_linkage( mono_item, linkage, visibility, is_compiler_builtins, ); - cx.module.declare_function(&name, linkage, &sig).unwrap(); + module.declare_function(name, linkage, &sig).unwrap(); } MonoItem::Static(_) | MonoItem::GlobalAsm(_) => {} } @@ -65,8 +39,8 @@ fn predefine_mono_items<'tcx>( }); } -fn time(tcx: TyCtxt<'_>, name: &'static str, f: impl FnOnce() -> R) -> R { - if std::env::var("CG_CLIF_DISPLAY_CG_TIME").as_ref().map(|val| &**val) == Ok("1") { +fn time(tcx: TyCtxt<'_>, display: bool, name: &'static str, f: impl FnOnce() -> R) -> R { + if display { println!("[{:<30}: {}] start", tcx.crate_name(LOCAL_CRATE), name); let before = std::time::Instant::now(); let res = tcx.sess.time(name, f); diff --git a/compiler/rustc_codegen_cranelift/src/inline_asm.rs b/compiler/rustc_codegen_cranelift/src/inline_asm.rs index 1fb5e86aed..09c5e6031c 100644 --- a/compiler/rustc_codegen_cranelift/src/inline_asm.rs +++ b/compiler/rustc_codegen_cranelift/src/inline_asm.rs @@ -24,6 +24,76 @@ pub(crate) fn codegen_inline_asm<'tcx>( let true_ = fx.bcx.ins().iconst(types::I32, 1); fx.bcx.ins().trapnz(true_, TrapCode::User(1)); return; + } else if template[0] == InlineAsmTemplatePiece::String("movq %rbx, ".to_string()) + && matches!( + template[1], + InlineAsmTemplatePiece::Placeholder { operand_idx: 0, modifier: Some('r'), span: _ } + ) + && template[2] == InlineAsmTemplatePiece::String("\n".to_string()) + && template[3] == InlineAsmTemplatePiece::String("cpuid".to_string()) + && template[4] == InlineAsmTemplatePiece::String("\n".to_string()) + && template[5] == InlineAsmTemplatePiece::String("xchgq %rbx, ".to_string()) + && matches!( + template[6], + InlineAsmTemplatePiece::Placeholder { operand_idx: 0, modifier: Some('r'), span: _ } + ) + { + assert_eq!(operands.len(), 4); + let (leaf, eax_place) = match operands[1] { + InlineAsmOperand::InOut { reg, late: true, ref in_value, out_place } => { + let reg = expect_reg(reg); + assert_eq!(reg, InlineAsmReg::X86(X86InlineAsmReg::ax)); + ( + crate::base::codegen_operand(fx, in_value).load_scalar(fx), + crate::base::codegen_place(fx, out_place.unwrap()), + ) + } + _ => unreachable!(), + }; + let ebx_place = match operands[0] { + InlineAsmOperand::Out { reg, late: true, place } => { + assert_eq!( + reg, + InlineAsmRegOrRegClass::RegClass(InlineAsmRegClass::X86( + X86InlineAsmRegClass::reg + )) + ); + crate::base::codegen_place(fx, place.unwrap()) + } + _ => unreachable!(), + }; + let (sub_leaf, ecx_place) = match operands[2] { + InlineAsmOperand::InOut { reg, late: true, ref in_value, out_place } => { + let reg = expect_reg(reg); + assert_eq!(reg, InlineAsmReg::X86(X86InlineAsmReg::cx)); + ( + crate::base::codegen_operand(fx, in_value).load_scalar(fx), + crate::base::codegen_place(fx, out_place.unwrap()), + ) + } + _ => unreachable!(), + }; + let edx_place = match operands[3] { + InlineAsmOperand::Out { reg, late: true, place } => { + let reg = expect_reg(reg); + assert_eq!(reg, InlineAsmReg::X86(X86InlineAsmReg::dx)); + crate::base::codegen_place(fx, place.unwrap()) + } + _ => unreachable!(), + }; + + let (eax, ebx, ecx, edx) = crate::intrinsics::codegen_cpuid_call(fx, leaf, sub_leaf); + + eax_place.write_cvalue(fx, CValue::by_val(eax, fx.layout_of(fx.tcx.types.u32))); + ebx_place.write_cvalue(fx, CValue::by_val(ebx, fx.layout_of(fx.tcx.types.u32))); + ecx_place.write_cvalue(fx, CValue::by_val(ecx, fx.layout_of(fx.tcx.types.u32))); + edx_place.write_cvalue(fx, CValue::by_val(edx, fx.layout_of(fx.tcx.types.u32))); + return; + } else if fx.tcx.symbol_name(fx.instance).name.starts_with("___chkstk") { + // ___chkstk, ___chkstk_ms and __alloca are only used on Windows + crate::trap::trap_unimplemented(fx, "Stack probes are not supported"); + } else if fx.tcx.symbol_name(fx.instance).name == "__alloca" { + crate::trap::trap_unimplemented(fx, "Alloca is not supported"); } let mut slot_size = Size::from_bytes(0); @@ -92,8 +162,7 @@ pub(crate) fn codegen_inline_asm<'tcx>( let inline_asm_index = fx.inline_asm_index; fx.inline_asm_index += 1; - let asm_name = - format!("{}__inline_asm_{}", fx.tcx.symbol_name(fx.instance).name, inline_asm_index); + let asm_name = format!("{}__inline_asm_{}", fx.symbol_name, inline_asm_index); let generated_asm = generate_asm_wrapper( &asm_name, @@ -202,7 +271,6 @@ fn call_inline_asm<'tcx>( } let inline_asm_func = fx - .cx .module .declare_function( asm_name, @@ -214,7 +282,7 @@ fn call_inline_asm<'tcx>( }, ) .unwrap(); - let inline_asm_func = fx.cx.module.declare_func_in_func(inline_asm_func, &mut fx.bcx.func); + let inline_asm_func = fx.module.declare_func_in_func(inline_asm_func, &mut fx.bcx.func); if fx.clif_comments.enabled() { fx.add_comment(inline_asm_func, asm_name); } diff --git a/compiler/rustc_codegen_cranelift/src/intrinsics/cpuid.rs b/compiler/rustc_codegen_cranelift/src/intrinsics/cpuid.rs index b27b0eddfb..d02dfd93c3 100644 --- a/compiler/rustc_codegen_cranelift/src/intrinsics/cpuid.rs +++ b/compiler/rustc_codegen_cranelift/src/intrinsics/cpuid.rs @@ -8,10 +8,11 @@ use crate::prelude::*; pub(crate) fn codegen_cpuid_call<'tcx>( fx: &mut FunctionCx<'_, '_, 'tcx>, leaf: Value, - _subleaf: Value, + _sub_leaf: Value, ) -> (Value, Value, Value, Value) { let leaf_0 = fx.bcx.create_block(); let leaf_1 = fx.bcx.create_block(); + let leaf_7 = fx.bcx.create_block(); let leaf_8000_0000 = fx.bcx.create_block(); let leaf_8000_0001 = fx.bcx.create_block(); let unsupported_leaf = fx.bcx.create_block(); @@ -25,6 +26,7 @@ pub(crate) fn codegen_cpuid_call<'tcx>( let mut switch = cranelift_frontend::Switch::new(); switch.set_entry(0, leaf_0); switch.set_entry(1, leaf_1); + switch.set_entry(7, leaf_7); switch.set_entry(0x8000_0000, leaf_8000_0000); switch.set_entry(0x8000_0001, leaf_8000_0001); switch.emit(&mut fx.bcx, leaf, unsupported_leaf); @@ -43,6 +45,11 @@ pub(crate) fn codegen_cpuid_call<'tcx>( let edx_features = fx.bcx.ins().iconst(types::I32, 1 << 25 /* sse */ | 1 << 26 /* sse2 */); fx.bcx.ins().jump(dest, &[cpu_signature, additional_information, ecx_features, edx_features]); + fx.bcx.switch_to_block(leaf_7); + // This leaf technically has subleaves, but we just return zero for all subleaves. + let zero = fx.bcx.ins().iconst(types::I32, 0); + fx.bcx.ins().jump(dest, &[zero, zero, zero, zero]); + fx.bcx.switch_to_block(leaf_8000_0000); let extended_max_basic_leaf = fx.bcx.ins().iconst(types::I32, 0); let zero = fx.bcx.ins().iconst(types::I32, 0); diff --git a/compiler/rustc_codegen_cranelift/src/intrinsics/llvm.rs b/compiler/rustc_codegen_cranelift/src/intrinsics/llvm.rs index 83c91f789c..ba4ed2162c 100644 --- a/compiler/rustc_codegen_cranelift/src/intrinsics/llvm.rs +++ b/compiler/rustc_codegen_cranelift/src/intrinsics/llvm.rs @@ -22,7 +22,7 @@ pub(crate) fn codegen_llvm_intrinsic_call<'tcx>( }; // Used by `_mm_movemask_epi8` and `_mm256_movemask_epi8` - llvm.x86.sse2.pmovmskb.128 | llvm.x86.avx2.pmovmskb | llvm.x86.sse2.movmsk.pd, (c a) { + "llvm.x86.sse2.pmovmskb.128" | "llvm.x86.avx2.pmovmskb" | "llvm.x86.sse2.movmsk.pd", (c a) { let (lane_count, lane_ty) = a.layout().ty.simd_size_and_type(fx.tcx); let lane_ty = fx.clif_type(lane_ty).unwrap(); assert!(lane_count <= 32); @@ -51,7 +51,7 @@ pub(crate) fn codegen_llvm_intrinsic_call<'tcx>( let res = CValue::by_val(res, fx.layout_of(fx.tcx.types.i32)); ret.write_cvalue(fx, res); }; - llvm.x86.sse2.cmp.ps | llvm.x86.sse2.cmp.pd, (c x, c y, o kind) { + "llvm.x86.sse2.cmp.ps" | "llvm.x86.sse2.cmp.pd", (c x, c y, o kind) { let kind_const = crate::constant::mir_operand_get_const_val(fx, kind).expect("llvm.x86.sse2.cmp.* kind not const"); let flt_cc = match kind_const.try_to_bits(Size::from_bytes(1)).unwrap_or_else(|| panic!("kind not scalar: {:?}", kind_const)) { 0 => FloatCC::Equal, @@ -81,7 +81,7 @@ pub(crate) fn codegen_llvm_intrinsic_call<'tcx>( bool_to_zero_or_max_uint(fx, res_lane_layout, res_lane) }); }; - llvm.x86.sse2.psrli.d, (c a, o imm8) { + "llvm.x86.sse2.psrli.d", (c a, o imm8) { let imm8 = crate::constant::mir_operand_get_const_val(fx, imm8).expect("llvm.x86.sse2.psrli.d imm8 not const"); simd_for_each_lane(fx, a, ret, |fx, _lane_layout, res_lane_layout, lane| { let res_lane = match imm8.try_to_bits(Size::from_bytes(4)).unwrap_or_else(|| panic!("imm8 not scalar: {:?}", imm8)) { @@ -91,7 +91,7 @@ pub(crate) fn codegen_llvm_intrinsic_call<'tcx>( CValue::by_val(res_lane, res_lane_layout) }); }; - llvm.x86.sse2.pslli.d, (c a, o imm8) { + "llvm.x86.sse2.pslli.d", (c a, o imm8) { let imm8 = crate::constant::mir_operand_get_const_val(fx, imm8).expect("llvm.x86.sse2.psrli.d imm8 not const"); simd_for_each_lane(fx, a, ret, |fx, _lane_layout, res_lane_layout, lane| { let res_lane = match imm8.try_to_bits(Size::from_bytes(4)).unwrap_or_else(|| panic!("imm8 not scalar: {:?}", imm8)) { @@ -101,7 +101,7 @@ pub(crate) fn codegen_llvm_intrinsic_call<'tcx>( CValue::by_val(res_lane, res_lane_layout) }); }; - llvm.x86.sse2.storeu.dq, (v mem_addr, c a) { + "llvm.x86.sse2.storeu.dq", (v mem_addr, c a) { // FIXME correctly handle the unalignment let dest = CPlace::for_ptr(Pointer::new(mem_addr), a.layout()); dest.write_cvalue(fx, a); diff --git a/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs b/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs index 39e047a98f..52896fc712 100644 --- a/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs +++ b/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs @@ -8,23 +8,25 @@ mod simd; pub(crate) use cpuid::codegen_cpuid_call; pub(crate) use llvm::codegen_llvm_intrinsic_call; +use rustc_middle::ty::print::with_no_trimmed_paths; +use rustc_span::symbol::{kw, sym}; + use crate::prelude::*; use cranelift_codegen::ir::AtomicRmwOp; -use rustc_middle::ty::print::with_no_trimmed_paths; macro intrinsic_pat { (_) => { _ }, ($name:ident) => { - stringify!($name) + sym::$name + }, + (kw.$name:ident) => { + kw::$name }, ($name:literal) => { - stringify!($name) + $name }, - ($x:ident . $($xs:tt).*) => { - concat!(stringify!($x), ".", intrinsic_pat!($($xs).*)) - } } macro intrinsic_arg { @@ -87,7 +89,7 @@ macro call_intrinsic_match { )*) => { match $intrinsic { $( - stringify!($name) => { + sym::$name => { assert!($substs.is_noop()); if let [$(ref $arg),*] = *$args { let ($($arg,)*) = ( @@ -400,18 +402,17 @@ pub(crate) fn codegen_intrinsic_call<'tcx>( let def_id = instance.def_id(); let substs = instance.substs; - let intrinsic = fx.tcx.item_name(def_id).as_str(); - let intrinsic = &intrinsic[..]; + let intrinsic = fx.tcx.item_name(def_id); let ret = match destination { Some((place, _)) => place, None => { // Insert non returning intrinsics here match intrinsic { - "abort" => { + sym::abort => { trap_abort(fx, "Called intrinsic::abort."); } - "transmute" => { + sym::transmute => { crate::base::codegen_panic(fx, "Transmuting to uninhabited type.", span); } _ => unimplemented!("unsupported instrinsic {}", intrinsic), @@ -420,7 +421,7 @@ pub(crate) fn codegen_intrinsic_call<'tcx>( } }; - if intrinsic.starts_with("simd_") { + if intrinsic.as_str().starts_with("simd_") { self::simd::codegen_simd_intrinsic_call(fx, instance, args, ret, span); let ret_block = fx.get_block(destination.expect("SIMD intrinsics don't diverge").1); fx.bcx.ins().jump(ret_block, &[]); @@ -470,8 +471,6 @@ pub(crate) fn codegen_intrinsic_call<'tcx>( sinf64(flt) -> f64 => sin, cosf32(flt) -> f32 => cosf, cosf64(flt) -> f64 => cos, - tanf32(flt) -> f32 => tanf, - tanf64(flt) -> f64 => tan, } intrinsic_match! { @@ -496,12 +495,12 @@ pub(crate) fn codegen_intrinsic_call<'tcx>( count }; - if intrinsic.contains("nonoverlapping") { + if intrinsic == sym::copy_nonoverlapping { // FIXME emit_small_memcpy - fx.bcx.call_memcpy(fx.cx.module.target_config(), dst, src, byte_amount); + fx.bcx.call_memcpy(fx.module.target_config(), dst, src, byte_amount); } else { // FIXME emit_small_memmove - fx.bcx.call_memmove(fx.cx.module.target_config(), dst, src, byte_amount); + fx.bcx.call_memmove(fx.module.target_config(), dst, src, byte_amount); } }; // NOTE: the volatile variants have src and dst swapped @@ -515,12 +514,12 @@ pub(crate) fn codegen_intrinsic_call<'tcx>( }; // FIXME make the copy actually volatile when using emit_small_mem{cpy,move} - if intrinsic.contains("nonoverlapping") { + if intrinsic == sym::volatile_copy_nonoverlapping_memory { // FIXME emit_small_memcpy - fx.bcx.call_memcpy(fx.cx.module.target_config(), dst, src, byte_amount); + fx.bcx.call_memcpy(fx.module.target_config(), dst, src, byte_amount); } else { // FIXME emit_small_memmove - fx.bcx.call_memmove(fx.cx.module.target_config(), dst, src, byte_amount); + fx.bcx.call_memmove(fx.module.target_config(), dst, src, byte_amount); } }; size_of_val, (c ptr) { @@ -552,27 +551,28 @@ pub(crate) fn codegen_intrinsic_call<'tcx>( ret.write_cvalue(fx, CValue::by_val(align, usize_layout)); }; - _ if intrinsic.starts_with("unchecked_") || intrinsic == "exact_div", (c x, c y) { + unchecked_add | unchecked_sub | unchecked_div | exact_div | unchecked_rem + | unchecked_shl | unchecked_shr, (c x, c y) { // FIXME trap on overflow let bin_op = match intrinsic { - "unchecked_add" => BinOp::Add, - "unchecked_sub" => BinOp::Sub, - "unchecked_div" | "exact_div" => BinOp::Div, - "unchecked_rem" => BinOp::Rem, - "unchecked_shl" => BinOp::Shl, - "unchecked_shr" => BinOp::Shr, - _ => unreachable!("intrinsic {}", intrinsic), + sym::unchecked_add => BinOp::Add, + sym::unchecked_sub => BinOp::Sub, + sym::unchecked_div | sym::exact_div => BinOp::Div, + sym::unchecked_rem => BinOp::Rem, + sym::unchecked_shl => BinOp::Shl, + sym::unchecked_shr => BinOp::Shr, + _ => unreachable!(), }; let res = crate::num::codegen_int_binop(fx, bin_op, x, y); ret.write_cvalue(fx, res); }; - _ if intrinsic.ends_with("_with_overflow"), (c x, c y) { + add_with_overflow | sub_with_overflow | mul_with_overflow, (c x, c y) { assert_eq!(x.layout().ty, y.layout().ty); let bin_op = match intrinsic { - "add_with_overflow" => BinOp::Add, - "sub_with_overflow" => BinOp::Sub, - "mul_with_overflow" => BinOp::Mul, - _ => unreachable!("intrinsic {}", intrinsic), + sym::add_with_overflow => BinOp::Add, + sym::sub_with_overflow => BinOp::Sub, + sym::mul_with_overflow => BinOp::Mul, + _ => unreachable!(), }; let res = crate::num::codegen_checked_int_binop( @@ -583,12 +583,12 @@ pub(crate) fn codegen_intrinsic_call<'tcx>( ); ret.write_cvalue(fx, res); }; - _ if intrinsic.starts_with("saturating_"), (c lhs, c rhs) { + saturating_add | saturating_sub, (c lhs, c rhs) { assert_eq!(lhs.layout().ty, rhs.layout().ty); let bin_op = match intrinsic { - "saturating_add" => BinOp::Add, - "saturating_sub" => BinOp::Sub, - _ => unreachable!("intrinsic {}", intrinsic), + sym::saturating_add => BinOp::Add, + sym::saturating_sub => BinOp::Sub, + _ => unreachable!(), }; let signed = type_sign(T); @@ -609,15 +609,15 @@ pub(crate) fn codegen_intrinsic_call<'tcx>( let (min, max) = type_min_max_value(&mut fx.bcx, clif_ty, signed); let val = match (intrinsic, signed) { - ("saturating_add", false) => fx.bcx.ins().select(has_overflow, max, val), - ("saturating_sub", false) => fx.bcx.ins().select(has_overflow, min, val), - ("saturating_add", true) => { + (sym::saturating_add, false) => fx.bcx.ins().select(has_overflow, max, val), + (sym::saturating_sub, false) => fx.bcx.ins().select(has_overflow, min, val), + (sym::saturating_add, true) => { let rhs = rhs.load_scalar(fx); let rhs_ge_zero = fx.bcx.ins().icmp_imm(IntCC::SignedGreaterThanOrEqual, rhs, 0); let sat_val = fx.bcx.ins().select(rhs_ge_zero, max, min); fx.bcx.ins().select(has_overflow, sat_val, val) } - ("saturating_sub", true) => { + (sym::saturating_sub, true) => { let rhs = rhs.load_scalar(fx); let rhs_ge_zero = fx.bcx.ins().icmp_imm(IntCC::SignedGreaterThanOrEqual, rhs, 0); let sat_val = fx.bcx.ins().select(rhs_ge_zero, min, max); @@ -632,11 +632,21 @@ pub(crate) fn codegen_intrinsic_call<'tcx>( }; rotate_left, (v x, v y) { let layout = fx.layout_of(T); + let y = if fx.bcx.func.dfg.value_type(y) == types::I128 { + fx.bcx.ins().ireduce(types::I64, y) + } else { + y + }; let res = fx.bcx.ins().rotl(x, y); ret.write_cvalue(fx, CValue::by_val(res, layout)); }; rotate_right, (v x, v y) { let layout = fx.layout_of(T); + let y = if fx.bcx.func.dfg.value_type(y) == types::I128 { + fx.bcx.ins().ireduce(types::I64, y) + } else { + y + }; let res = fx.bcx.ins().rotr(x, y); ret.write_cvalue(fx, CValue::by_val(res, layout)); }; @@ -670,7 +680,7 @@ pub(crate) fn codegen_intrinsic_call<'tcx>( let dst_ptr = dst.load_scalar(fx); // FIXME make the memset actually volatile when switching to emit_small_memset // FIXME use emit_small_memset - fx.bcx.call_memset(fx.cx.module.target_config(), dst_ptr, val, count); + fx.bcx.call_memset(fx.module.target_config(), dst_ptr, val, count); }; ctlz | ctlz_nonzero, (v arg) { // FIXME trap on `ctlz_nonzero` with zero arg. @@ -806,7 +816,7 @@ pub(crate) fn codegen_intrinsic_call<'tcx>( return; } - if intrinsic == "assert_zero_valid" && !layout.might_permit_raw_init(fx, /*zero:*/ true).unwrap() { + if intrinsic == sym::assert_zero_valid && !layout.might_permit_raw_init(fx, /*zero:*/ true).unwrap() { with_no_trimmed_paths(|| crate::base::codegen_panic( fx, &format!("attempted to zero-initialize type `{}`, which is invalid", T), @@ -815,7 +825,7 @@ pub(crate) fn codegen_intrinsic_call<'tcx>( return; } - if intrinsic == "assert_uninit_valid" && !layout.might_permit_raw_init(fx, /*zero:*/ false).unwrap() { + if intrinsic == sym::assert_uninit_valid && !layout.might_permit_raw_init(fx, /*zero:*/ false).unwrap() { with_no_trimmed_paths(|| crate::base::codegen_panic( fx, &format!("attempted to leave type `{}` uninitialized, which is invalid", T), @@ -827,7 +837,6 @@ pub(crate) fn codegen_intrinsic_call<'tcx>( volatile_load | unaligned_volatile_load, (c ptr) { // Cranelift treats loads as volatile by default - // FIXME ignore during stack2reg optimization // FIXME correctly handle unaligned_volatile_load let inner_layout = fx.layout_of(ptr.layout().ty.builtin_deref(true).unwrap().ty); @@ -836,7 +845,6 @@ pub(crate) fn codegen_intrinsic_call<'tcx>( }; volatile_store | unaligned_volatile_store, (v ptr, c val) { // Cranelift treats stores as volatile by default - // FIXME ignore during stack2reg optimization // FIXME correctly handle unaligned_volatile_store let dest = CPlace::for_ptr(Pointer::new(ptr), val.layout()); dest.write_cvalue(fx, val); @@ -878,14 +886,14 @@ pub(crate) fn codegen_intrinsic_call<'tcx>( ret.write_cvalue(fx, caller_location); }; - _ if intrinsic.starts_with("atomic_fence"), () { + _ if intrinsic.as_str().starts_with("atomic_fence"), () { fx.bcx.ins().fence(); }; - _ if intrinsic.starts_with("atomic_singlethreadfence"), () { + _ if intrinsic.as_str().starts_with("atomic_singlethreadfence"), () { // FIXME use a compiler fence once Cranelift supports it fx.bcx.ins().fence(); }; - _ if intrinsic.starts_with("atomic_load"), (v ptr) { + _ if intrinsic.as_str().starts_with("atomic_load"), (v ptr) { validate_atomic_type!(fx, intrinsic, span, T); let ty = fx.clif_type(T).unwrap(); @@ -894,14 +902,14 @@ pub(crate) fn codegen_intrinsic_call<'tcx>( let val = CValue::by_val(val, fx.layout_of(T)); ret.write_cvalue(fx, val); }; - _ if intrinsic.starts_with("atomic_store"), (v ptr, c val) { + _ if intrinsic.as_str().starts_with("atomic_store"), (v ptr, c val) { validate_atomic_type!(fx, intrinsic, span, val.layout().ty); let val = val.load_scalar(fx); fx.bcx.ins().atomic_store(MemFlags::trusted(), val, ptr); }; - _ if intrinsic.starts_with("atomic_xchg"), (v ptr, c new) { + _ if intrinsic.as_str().starts_with("atomic_xchg"), (v ptr, c new) { let layout = new.layout(); validate_atomic_type!(fx, intrinsic, span, layout.ty); let ty = fx.clif_type(layout.ty).unwrap(); @@ -913,7 +921,7 @@ pub(crate) fn codegen_intrinsic_call<'tcx>( let old = CValue::by_val(old, layout); ret.write_cvalue(fx, old); }; - _ if intrinsic.starts_with("atomic_cxchg"), (v ptr, c test_old, c new) { // both atomic_cxchg_* and atomic_cxchgweak_* + _ if intrinsic.as_str().starts_with("atomic_cxchg"), (v ptr, c test_old, c new) { // both atomic_cxchg_* and atomic_cxchgweak_* let layout = new.layout(); validate_atomic_type!(fx, intrinsic, span, layout.ty); @@ -927,7 +935,7 @@ pub(crate) fn codegen_intrinsic_call<'tcx>( ret.write_cvalue(fx, ret_val) }; - _ if intrinsic.starts_with("atomic_xadd"), (v ptr, c amount) { + _ if intrinsic.as_str().starts_with("atomic_xadd"), (v ptr, c amount) { let layout = amount.layout(); validate_atomic_type!(fx, intrinsic, span, layout.ty); let ty = fx.clif_type(layout.ty).unwrap(); @@ -939,7 +947,7 @@ pub(crate) fn codegen_intrinsic_call<'tcx>( let old = CValue::by_val(old, layout); ret.write_cvalue(fx, old); }; - _ if intrinsic.starts_with("atomic_xsub"), (v ptr, c amount) { + _ if intrinsic.as_str().starts_with("atomic_xsub"), (v ptr, c amount) { let layout = amount.layout(); validate_atomic_type!(fx, intrinsic, span, layout.ty); let ty = fx.clif_type(layout.ty).unwrap(); @@ -951,7 +959,7 @@ pub(crate) fn codegen_intrinsic_call<'tcx>( let old = CValue::by_val(old, layout); ret.write_cvalue(fx, old); }; - _ if intrinsic.starts_with("atomic_and"), (v ptr, c src) { + _ if intrinsic.as_str().starts_with("atomic_and"), (v ptr, c src) { let layout = src.layout(); validate_atomic_type!(fx, intrinsic, span, layout.ty); let ty = fx.clif_type(layout.ty).unwrap(); @@ -963,7 +971,7 @@ pub(crate) fn codegen_intrinsic_call<'tcx>( let old = CValue::by_val(old, layout); ret.write_cvalue(fx, old); }; - _ if intrinsic.starts_with("atomic_or"), (v ptr, c src) { + _ if intrinsic.as_str().starts_with("atomic_or"), (v ptr, c src) { let layout = src.layout(); validate_atomic_type!(fx, intrinsic, span, layout.ty); let ty = fx.clif_type(layout.ty).unwrap(); @@ -975,7 +983,7 @@ pub(crate) fn codegen_intrinsic_call<'tcx>( let old = CValue::by_val(old, layout); ret.write_cvalue(fx, old); }; - _ if intrinsic.starts_with("atomic_xor"), (v ptr, c src) { + _ if intrinsic.as_str().starts_with("atomic_xor"), (v ptr, c src) { let layout = src.layout(); validate_atomic_type!(fx, intrinsic, span, layout.ty); let ty = fx.clif_type(layout.ty).unwrap(); @@ -989,7 +997,7 @@ pub(crate) fn codegen_intrinsic_call<'tcx>( }; // FIXME https://github.com/bytecodealliance/wasmtime/issues/2647 - _ if intrinsic.starts_with("atomic_nand"), (v ptr, c src) { + _ if intrinsic.as_str().starts_with("atomic_nand"), (v ptr, c src) { let layout = src.layout(); validate_atomic_type!(fx, intrinsic, span, layout.ty); let ty = fx.clif_type(layout.ty).unwrap(); @@ -1001,7 +1009,7 @@ pub(crate) fn codegen_intrinsic_call<'tcx>( let old = CValue::by_val(old, layout); ret.write_cvalue(fx, old); }; - _ if intrinsic.starts_with("atomic_max"), (v ptr, c src) { + _ if intrinsic.as_str().starts_with("atomic_max"), (v ptr, c src) { let layout = src.layout(); validate_atomic_type!(fx, intrinsic, span, layout.ty); let ty = fx.clif_type(layout.ty).unwrap(); @@ -1013,7 +1021,7 @@ pub(crate) fn codegen_intrinsic_call<'tcx>( let old = CValue::by_val(old, layout); ret.write_cvalue(fx, old); }; - _ if intrinsic.starts_with("atomic_umax"), (v ptr, c src) { + _ if intrinsic.as_str().starts_with("atomic_umax"), (v ptr, c src) { let layout = src.layout(); validate_atomic_type!(fx, intrinsic, span, layout.ty); let ty = fx.clif_type(layout.ty).unwrap(); @@ -1025,7 +1033,7 @@ pub(crate) fn codegen_intrinsic_call<'tcx>( let old = CValue::by_val(old, layout); ret.write_cvalue(fx, old); }; - _ if intrinsic.starts_with("atomic_min"), (v ptr, c src) { + _ if intrinsic.as_str().starts_with("atomic_min"), (v ptr, c src) { let layout = src.layout(); validate_atomic_type!(fx, intrinsic, span, layout.ty); let ty = fx.clif_type(layout.ty).unwrap(); @@ -1037,7 +1045,7 @@ pub(crate) fn codegen_intrinsic_call<'tcx>( let old = CValue::by_val(old, layout); ret.write_cvalue(fx, old); }; - _ if intrinsic.starts_with("atomic_umin"), (v ptr, c src) { + _ if intrinsic.as_str().starts_with("atomic_umin"), (v ptr, c src) { let layout = src.layout(); validate_atomic_type!(fx, intrinsic, span, layout.ty); let ty = fx.clif_type(layout.ty).unwrap(); @@ -1071,7 +1079,7 @@ pub(crate) fn codegen_intrinsic_call<'tcx>( ret.write_cvalue(fx, val); }; - try, (v f, v data, v _catch_fn) { + kw.Try, (v f, v data, v _catch_fn) { // FIXME once unwinding is supported, change this to actually catch panics let f_sig = fx.bcx.func.import_signature(Signature { call_conv: CallConv::triple_default(fx.triple()), @@ -1088,11 +1096,11 @@ pub(crate) fn codegen_intrinsic_call<'tcx>( fadd_fast | fsub_fast | fmul_fast | fdiv_fast | frem_fast, (c x, c y) { let res = crate::num::codegen_float_binop(fx, match intrinsic { - "fadd_fast" => BinOp::Add, - "fsub_fast" => BinOp::Sub, - "fmul_fast" => BinOp::Mul, - "fdiv_fast" => BinOp::Div, - "frem_fast" => BinOp::Rem, + sym::fadd_fast => BinOp::Add, + sym::fsub_fast => BinOp::Sub, + sym::fmul_fast => BinOp::Mul, + sym::fdiv_fast => BinOp::Div, + sym::frem_fast => BinOp::Rem, _ => unreachable!(), }, x, y); ret.write_cvalue(fx, res); diff --git a/compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs b/compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs index 27fc2abedc..c2f469fa02 100644 --- a/compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs +++ b/compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs @@ -13,8 +13,7 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>( let def_id = instance.def_id(); let substs = instance.substs; - let intrinsic = fx.tcx.item_name(def_id).as_str(); - let intrinsic = &intrinsic[..]; + let intrinsic = fx.tcx.item_name(def_id); intrinsic_match! { fx, intrinsic, substs, args, @@ -65,10 +64,10 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>( }; // simd_shuffle32(x: T, y: T, idx: [u32; 32]) -> U - _ if intrinsic.starts_with("simd_shuffle"), (c x, c y, o idx) { + _ if intrinsic.as_str().starts_with("simd_shuffle"), (c x, c y, o idx) { validate_simd_type!(fx, intrinsic, span, x.layout().ty); - let n: u16 = intrinsic["simd_shuffle".len()..].parse().unwrap(); + let n: u16 = intrinsic.as_str()["simd_shuffle".len()..].parse().unwrap(); assert_eq!(x.layout(), y.layout()); let layout = x.layout(); @@ -87,9 +86,8 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>( let idx_bytes = match idx_const { ConstValue::ByRef { alloc, offset } => { - let ptr = Pointer::new(AllocId(0 /* dummy */), offset); let size = Size::from_bytes(4 * ret_lane_count /* size_of([u32; ret_lane_count]) */); - alloc.get_bytes(fx, ptr, size).unwrap() + alloc.get_bytes(fx, alloc_range(offset, size)).unwrap() } _ => unreachable!("{:?}", idx_const), }; diff --git a/compiler/rustc_codegen_cranelift/src/lib.rs b/compiler/rustc_codegen_cranelift/src/lib.rs index 720d2a1253..6aadaf8a7c 100644 --- a/compiler/rustc_codegen_cranelift/src/lib.rs +++ b/compiler/rustc_codegen_cranelift/src/lib.rs @@ -1,4 +1,4 @@ -#![feature(rustc_private, decl_macro, never_type, hash_drain_filter)] +#![feature(rustc_private, decl_macro, never_type, hash_drain_filter, vec_into_raw_parts)] #![warn(rust_2018_idioms)] #![warn(unused_lifetimes)] #![warn(unreachable_pub)] @@ -14,6 +14,7 @@ extern crate rustc_fs_util; extern crate rustc_hir; extern crate rustc_incremental; extern crate rustc_index; +extern crate rustc_metadata; extern crate rustc_session; extern crate rustc_span; extern crate rustc_target; @@ -23,20 +24,19 @@ extern crate rustc_target; extern crate rustc_driver; use std::any::Any; -use std::str::FromStr; use rustc_codegen_ssa::traits::CodegenBackend; use rustc_codegen_ssa::CodegenResults; use rustc_errors::ErrorReported; use rustc_middle::dep_graph::{WorkProduct, WorkProductId}; -use rustc_middle::middle::cstore::{EncodedMetadata, MetadataLoader}; -use rustc_middle::ty::query::Providers; +use rustc_middle::middle::cstore::EncodedMetadata; use rustc_session::config::OutputFilenames; use rustc_session::Session; +use cranelift_codegen::isa::TargetIsa; use cranelift_codegen::settings::{self, Configurable}; -use crate::constant::ConstantCx; +pub use crate::config::*; use crate::prelude::*; mod abi; @@ -49,6 +49,7 @@ mod cast; mod codegen_i128; mod common; mod compiler_builtins; +mod config; mod constant; mod debuginfo; mod discriminant; @@ -87,7 +88,6 @@ mod prelude { pub(crate) use rustc_index::vec::Idx; - pub(crate) use cranelift_codegen::entity::EntitySet; pub(crate) use cranelift_codegen::ir::condcodes::{FloatCC, IntCC}; pub(crate) use cranelift_codegen::ir::function::Function; pub(crate) use cranelift_codegen::ir::types; @@ -119,95 +119,36 @@ impl String> Drop for PrintOnPanic { } } -struct CodegenCx<'m, 'tcx: 'm> { +/// The codegen context holds any information shared between the codegen of individual functions +/// inside a single codegen unit with the exception of the Cranelift [`Module`](cranelift_module::Module). +struct CodegenCx<'tcx> { tcx: TyCtxt<'tcx>, - module: &'m mut dyn Module, global_asm: String, - constants_cx: ConstantCx, cached_context: Context, - vtables: FxHashMap<(Ty<'tcx>, Option>), DataId>, debug_context: Option>, - unwind_context: UnwindContext<'tcx>, + unwind_context: UnwindContext, } -impl<'m, 'tcx> CodegenCx<'m, 'tcx> { +impl<'tcx> CodegenCx<'tcx> { fn new( tcx: TyCtxt<'tcx>, backend_config: BackendConfig, - module: &'m mut dyn Module, + isa: &dyn TargetIsa, debug_info: bool, ) -> Self { - let unwind_context = UnwindContext::new( - tcx, - module.isa(), - matches!(backend_config.codegen_mode, CodegenMode::Aot), - ); - let debug_context = - if debug_info { Some(DebugContext::new(tcx, module.isa())) } else { None }; + assert_eq!(pointer_ty(tcx), isa.pointer_type()); + + let unwind_context = + UnwindContext::new(tcx, isa, matches!(backend_config.codegen_mode, CodegenMode::Aot)); + let debug_context = if debug_info { Some(DebugContext::new(tcx, isa)) } else { None }; CodegenCx { tcx, - module, global_asm: String::new(), - constants_cx: ConstantCx::default(), cached_context: Context::new(), - vtables: FxHashMap::default(), debug_context, unwind_context, } } - - fn finalize(self) -> (String, Option>, UnwindContext<'tcx>) { - self.constants_cx.finalize(self.tcx, self.module); - (self.global_asm, self.debug_context, self.unwind_context) - } -} - -#[derive(Copy, Clone, Debug)] -pub enum CodegenMode { - Aot, - Jit, - JitLazy, -} - -impl Default for CodegenMode { - fn default() -> Self { - CodegenMode::Aot - } -} - -impl FromStr for CodegenMode { - type Err = String; - - fn from_str(s: &str) -> Result { - match s { - "aot" => Ok(CodegenMode::Aot), - "jit" => Ok(CodegenMode::Jit), - "jit-lazy" => Ok(CodegenMode::JitLazy), - _ => Err(format!("Unknown codegen mode `{}`", s)), - } - } -} - -#[derive(Copy, Clone, Debug, Default)] -pub struct BackendConfig { - pub codegen_mode: CodegenMode, -} - -impl BackendConfig { - fn from_opts(opts: &[String]) -> Result { - let mut config = BackendConfig::default(); - for opt in opts { - if let Some((name, value)) = opt.split_once('=') { - match name { - "mode" => config.codegen_mode = value.parse()?, - _ => return Err(format!("Unknown option `{}`", name)), - } - } else { - return Err(format!("Invalid option `{}`", opt)); - } - } - Ok(config) - } } pub struct CraneliftCodegenBackend { @@ -223,30 +164,37 @@ impl CodegenBackend for CraneliftCodegenBackend { } } - fn metadata_loader(&self) -> Box { - Box::new(crate::metadata::CraneliftMetadataLoader) - } - - fn provide(&self, _providers: &mut Providers) {} - fn provide_extern(&self, _providers: &mut Providers) {} - fn target_features(&self, _sess: &Session) -> Vec { vec![] } + fn print_version(&self) { + println!("Cranelift version: {}", cranelift_codegen::VERSION); + } + fn codegen_crate( &self, tcx: TyCtxt<'_>, metadata: EncodedMetadata, need_metadata_module: bool, ) -> Box { - let config = if let Some(config) = self.config { + tcx.sess.abort_if_errors(); + let config = if let Some(config) = self.config.clone() { config } else { BackendConfig::from_opts(&tcx.sess.opts.cg.llvm_args) .unwrap_or_else(|err| tcx.sess.fatal(&err)) }; - driver::codegen_crate(tcx, metadata, need_metadata_module, config) + match config.codegen_mode { + CodegenMode::Aot => driver::aot::run_aot(tcx, config, metadata, need_metadata_module), + CodegenMode::Jit | CodegenMode::JitLazy => { + #[cfg(feature = "jit")] + let _: ! = driver::jit::run_jit(tcx, config); + + #[cfg(not(feature = "jit"))] + tcx.sess.fatal("jit support was disabled when compiling rustc_codegen_cranelift"); + } + } } fn join_codegen( @@ -267,13 +215,11 @@ impl CodegenBackend for CraneliftCodegenBackend { ) -> Result<(), ErrorReported> { use rustc_codegen_ssa::back::link::link_binary; - let target_cpu = crate::target_triple(sess).to_string(); link_binary::>( sess, &codegen_results, outputs, - &codegen_results.crate_name.as_str(), - &target_cpu, + &codegen_results.crate_info.local_crate_name.as_str(), ); Ok(()) @@ -284,7 +230,7 @@ fn target_triple(sess: &Session) -> target_lexicon::Triple { sess.target.llvm_target.parse().unwrap() } -fn build_isa(sess: &Session) -> Box { +fn build_isa(sess: &Session, backend_config: &BackendConfig) -> Box { use target_lexicon::BinaryFormat; let target_triple = crate::target_triple(sess); @@ -292,9 +238,8 @@ fn build_isa(sess: &Session) -> Box { let mut flags_builder = settings::builder(); flags_builder.enable("is_pic").unwrap(); flags_builder.set("enable_probestack", "false").unwrap(); // __cranelift_probestack is not provided - let enable_verifier = - cfg!(debug_assertions) || std::env::var("CG_CLIF_ENABLE_VERIFIER").is_ok(); - flags_builder.set("enable_verifier", if enable_verifier { "true" } else { "false" }).unwrap(); + let enable_verifier = if backend_config.enable_verifier { "true" } else { "false" }; + flags_builder.set("enable_verifier", enable_verifier).unwrap(); let tls_model = match target_triple.binary_format { BinaryFormat::Elf => "elf_gd", @@ -308,6 +253,8 @@ fn build_isa(sess: &Session) -> Box { flags_builder.set("enable_llvm_abi_extensions", "true").unwrap(); + flags_builder.set("regalloc", &backend_config.regalloc).unwrap(); + use rustc_session::config::OptLevel; match sess.opts.optimize { OptLevel::No => { @@ -322,10 +269,30 @@ fn build_isa(sess: &Session) -> Box { let flags = settings::Flags::new(flags_builder); let variant = cranelift_codegen::isa::BackendVariant::MachInst; - let mut isa_builder = cranelift_codegen::isa::lookup_variant(target_triple, variant).unwrap(); - // Don't use "haswell", as it implies `has_lzcnt`.macOS CI is still at Ivy Bridge EP, so `lzcnt` - // is interpreted as `bsr`. - isa_builder.enable("nehalem").unwrap(); + + let isa_builder = match sess.opts.cg.target_cpu.as_deref() { + Some("native") => { + let builder = cranelift_native::builder_with_options(variant, true).unwrap(); + builder + } + Some(value) => { + let mut builder = + cranelift_codegen::isa::lookup_variant(target_triple, variant).unwrap(); + if let Err(_) = builder.enable(value) { + sess.fatal("The specified target cpu isn't currently supported by Cranelift."); + } + builder + } + None => { + let mut builder = + cranelift_codegen::isa::lookup_variant(target_triple, variant).unwrap(); + // Don't use "haswell" as the default, as it implies `has_lzcnt`. + // macOS CI is still at Ivy Bridge EP, so `lzcnt` is interpreted as `bsr`. + builder.enable("nehalem").unwrap(); + builder + } + }; + isa_builder.finish(flags) } diff --git a/compiler/rustc_codegen_cranelift/src/linkage.rs b/compiler/rustc_codegen_cranelift/src/linkage.rs index a564a59f72..ca853aac15 100644 --- a/compiler/rustc_codegen_cranelift/src/linkage.rs +++ b/compiler/rustc_codegen_cranelift/src/linkage.rs @@ -13,6 +13,7 @@ pub(crate) fn get_clif_linkage( (RLinkage::External, Visibility::Default) => Linkage::Export, (RLinkage::Internal, Visibility::Default) => Linkage::Local, (RLinkage::External, Visibility::Hidden) => Linkage::Hidden, + (RLinkage::WeakAny, Visibility::Default) => Linkage::Preemptible, _ => panic!("{:?} = {:?} {:?}", mono_item, linkage, visibility), } } diff --git a/compiler/rustc_codegen_cranelift/src/main_shim.rs b/compiler/rustc_codegen_cranelift/src/main_shim.rs index dc86290b3f..8fd1e4f581 100644 --- a/compiler/rustc_codegen_cranelift/src/main_shim.rs +++ b/compiler/rustc_codegen_cranelift/src/main_shim.rs @@ -1,6 +1,9 @@ use cranelift_codegen::binemit::{NullStackMapSink, NullTrapSink}; use rustc_hir::LangItem; +use rustc_middle::ty::subst::GenericArg; +use rustc_middle::ty::AssocKind; use rustc_session::config::EntryFnType; +use rustc_span::symbol::Ident; use crate::prelude::*; @@ -9,9 +12,11 @@ use crate::prelude::*; pub(crate) fn maybe_create_entry_wrapper( tcx: TyCtxt<'_>, module: &mut impl Module, - unwind_context: &mut UnwindContext<'_>, + unwind_context: &mut UnwindContext, + is_jit: bool, + is_primary_cgu: bool, ) { - let (main_def_id, use_start_lang_item) = match tcx.entry_fn(LOCAL_CRATE) { + let (main_def_id, is_main_fn) = match tcx.entry_fn(()) { Some((def_id, entry_ty)) => ( def_id, match entry_ty { @@ -22,19 +27,24 @@ pub(crate) fn maybe_create_entry_wrapper( None => return, }; - let instance = Instance::mono(tcx, main_def_id).polymorphize(tcx); - if module.get_name(&*tcx.symbol_name(instance).name).is_none() { + if main_def_id.is_local() { + let instance = Instance::mono(tcx, main_def_id).polymorphize(tcx); + if !is_jit && module.get_name(&*tcx.symbol_name(instance).name).is_none() { + return; + } + } else if !is_primary_cgu { return; } - create_entry_fn(tcx, module, unwind_context, main_def_id, use_start_lang_item); + create_entry_fn(tcx, module, unwind_context, main_def_id, is_jit, is_main_fn); fn create_entry_fn( tcx: TyCtxt<'_>, m: &mut impl Module, - unwind_context: &mut UnwindContext<'_>, + unwind_context: &mut UnwindContext, rust_main_def_id: DefId, - use_start_lang_item: bool, + ignore_lang_start_wrapper: bool, + is_main_fn: bool, ) { let main_ret_ty = tcx.fn_sig(rust_main_def_id).output(); // Given that `main()` has no arguments, @@ -57,9 +67,9 @@ pub(crate) fn maybe_create_entry_wrapper( let instance = Instance::mono(tcx, rust_main_def_id).polymorphize(tcx); - let main_name = tcx.symbol_name(instance).name.to_string(); + let main_name = tcx.symbol_name(instance).name; let main_sig = get_function_sig(tcx, m.isa().triple(), instance); - let main_func_id = m.declare_function(&main_name, Linkage::Import, &main_sig).unwrap(); + let main_func_id = m.declare_function(main_name, Linkage::Import, &main_sig).unwrap(); let mut ctx = Context::new(); ctx.func = Function::with_name_signature(ExternalName::user(0, 0), cmain_sig); @@ -74,7 +84,47 @@ pub(crate) fn maybe_create_entry_wrapper( let main_func_ref = m.declare_func_in_func(main_func_id, &mut bcx.func); - let call_inst = if use_start_lang_item { + let result = if is_main_fn && ignore_lang_start_wrapper { + // regular main fn, but ignoring #[lang = "start"] as we are running in the jit + // FIXME set program arguments somehow + let call_inst = bcx.ins().call(main_func_ref, &[]); + let call_results = bcx.func.dfg.inst_results(call_inst).to_owned(); + + let termination_trait = tcx.require_lang_item(LangItem::Termination, None); + let report = tcx + .associated_items(termination_trait) + .find_by_name_and_kind( + tcx, + Ident::from_str("report"), + AssocKind::Fn, + termination_trait, + ) + .unwrap(); + let report = Instance::resolve( + tcx, + ParamEnv::reveal_all(), + report.def_id, + tcx.mk_substs([GenericArg::from(main_ret_ty)].iter()), + ) + .unwrap() + .unwrap(); + + let report_name = tcx.symbol_name(report).name; + let report_sig = get_function_sig(tcx, m.isa().triple(), report); + let report_func_id = + m.declare_function(report_name, Linkage::Import, &report_sig).unwrap(); + let report_func_ref = m.declare_func_in_func(report_func_id, &mut bcx.func); + + // FIXME do proper abi handling instead of expecting the pass mode to be identical + // for returns and arguments. + let report_call_inst = bcx.ins().call(report_func_ref, &call_results); + let res = bcx.func.dfg.inst_results(report_call_inst)[0]; + match m.target_config().pointer_type() { + types::I32 => res, + types::I64 => bcx.ins().sextend(types::I64, res), + _ => unimplemented!("16bit systems are not yet supported"), + } + } else if is_main_fn { let start_def_id = tcx.require_lang_item(LangItem::Start, None); let start_instance = Instance::resolve( tcx, @@ -90,13 +140,14 @@ pub(crate) fn maybe_create_entry_wrapper( let main_val = bcx.ins().func_addr(m.target_config().pointer_type(), main_func_ref); let func_ref = m.declare_func_in_func(start_func_id, &mut bcx.func); - bcx.ins().call(func_ref, &[main_val, arg_argc, arg_argv]) + let call_inst = bcx.ins().call(func_ref, &[main_val, arg_argc, arg_argv]); + bcx.inst_results(call_inst)[0] } else { // using user-defined start fn - bcx.ins().call(main_func_ref, &[arg_argc, arg_argv]) + let call_inst = bcx.ins().call(main_func_ref, &[arg_argc, arg_argv]); + bcx.inst_results(call_inst)[0] }; - let result = bcx.inst_results(call_inst)[0]; bcx.ins().return_(&[result]); bcx.seal_all_blocks(); bcx.finalize(); diff --git a/compiler/rustc_codegen_cranelift/src/metadata.rs b/compiler/rustc_codegen_cranelift/src/metadata.rs index dbdc8cbad4..db24bf65eb 100644 --- a/compiler/rustc_codegen_cranelift/src/metadata.rs +++ b/compiler/rustc_codegen_cranelift/src/metadata.rs @@ -1,111 +1,20 @@ -//! Reading and writing of the rustc metadata for rlibs and dylibs +//! Writing of the rustc metadata for dylibs -use std::fs::File; -use std::path::Path; - -use rustc_codegen_ssa::METADATA_FILENAME; -use rustc_data_structures::memmap::Mmap; -use rustc_data_structures::owning_ref::OwningRef; -use rustc_data_structures::rustc_erase_owner; -use rustc_data_structures::sync::MetadataRef; -use rustc_middle::middle::cstore::{EncodedMetadata, MetadataLoader}; use rustc_middle::ty::TyCtxt; -use rustc_session::config; -use rustc_target::spec::Target; use crate::backend::WriteMetadata; -pub(crate) struct CraneliftMetadataLoader; - -fn load_metadata_with( - path: &Path, - f: impl for<'a> FnOnce(&'a [u8]) -> Result<&'a [u8], String>, -) -> Result { - let file = File::open(path).map_err(|e| format!("{:?}", e))?; - let data = unsafe { Mmap::map(file) }.map_err(|e| format!("{:?}", e))?; - let metadata = OwningRef::new(data).try_map(f)?; - return Ok(rustc_erase_owner!(metadata.map_owner_box())); -} - -impl MetadataLoader for CraneliftMetadataLoader { - fn get_rlib_metadata(&self, _target: &Target, path: &Path) -> Result { - load_metadata_with(path, |data| { - let archive = object::read::archive::ArchiveFile::parse(&*data) - .map_err(|e| format!("{:?}", e))?; - - for entry_result in archive.members() { - let entry = entry_result.map_err(|e| format!("{:?}", e))?; - if entry.name() == METADATA_FILENAME.as_bytes() { - return Ok(entry.data()); - } - } - - Err("couldn't find metadata entry".to_string()) - }) - } - - fn get_dylib_metadata(&self, _target: &Target, path: &Path) -> Result { - use object::{Object, ObjectSection}; - - load_metadata_with(path, |data| { - let file = object::File::parse(&data).map_err(|e| format!("parse: {:?}", e))?; - file.section_by_name(".rustc") - .ok_or("no .rustc section")? - .data() - .map_err(|e| format!("failed to read .rustc section: {:?}", e)) - }) - } -} - // Adapted from https://github.com/rust-lang/rust/blob/da573206f87b5510de4b0ee1a9c044127e409bd3/src/librustc_codegen_llvm/base.rs#L47-L112 -pub(crate) fn write_metadata( - tcx: TyCtxt<'_>, - product: &mut P, -) -> EncodedMetadata { +pub(crate) fn write_metadata(tcx: TyCtxt<'_>, object: &mut O) { use snap::write::FrameEncoder; use std::io::Write; - #[derive(PartialEq, Eq, PartialOrd, Ord)] - enum MetadataKind { - None, - Uncompressed, - Compressed, - } - - let kind = tcx - .sess - .crate_types() - .iter() - .map(|ty| match *ty { - config::CrateType::Executable - | config::CrateType::Staticlib - | config::CrateType::Cdylib => MetadataKind::None, - - config::CrateType::Rlib => MetadataKind::Uncompressed, - - config::CrateType::Dylib | config::CrateType::ProcMacro => MetadataKind::Compressed, - }) - .max() - .unwrap_or(MetadataKind::None); - - if kind == MetadataKind::None { - return EncodedMetadata::new(); - } - let metadata = tcx.encode_metadata(); - if kind == MetadataKind::Uncompressed { - return metadata; - } - - assert!(kind == MetadataKind::Compressed); - let mut compressed = tcx.metadata_encoding_version(); + let mut compressed = rustc_metadata::METADATA_HEADER.to_vec(); FrameEncoder::new(&mut compressed).write_all(&metadata.raw_data).unwrap(); - product.add_rustc_section( + object.add_rustc_section( rustc_middle::middle::exported_symbols::metadata_symbol_name(tcx), compressed, - tcx.sess.target.is_like_osx, ); - - metadata } diff --git a/compiler/rustc_codegen_cranelift/src/num.rs b/compiler/rustc_codegen_cranelift/src/num.rs index 2ebf30da2d..b6d378a5fe 100644 --- a/compiler/rustc_codegen_cranelift/src/num.rs +++ b/compiler/rustc_codegen_cranelift/src/num.rs @@ -271,14 +271,17 @@ pub(crate) fn codegen_checked_int_binop<'tcx>( let val_hi = fx.bcx.ins().umulhi(lhs, rhs); fx.bcx.ins().icmp_imm(IntCC::NotEqual, val_hi, 0) } else { + // Based on LLVM's instruction sequence for compiling + // a.checked_mul(b).is_some() to riscv64gc: + // mulh a2, a0, a1 + // mul a0, a0, a1 + // srai a0, a0, 63 + // xor a0, a0, a2 + // snez a0, a0 let val_hi = fx.bcx.ins().smulhi(lhs, rhs); - let not_all_zero = fx.bcx.ins().icmp_imm(IntCC::NotEqual, val_hi, 0); - let not_all_ones = fx.bcx.ins().icmp_imm( - IntCC::NotEqual, - val_hi, - u64::try_from((1u128 << ty.bits()) - 1).unwrap() as i64, - ); - fx.bcx.ins().band(not_all_zero, not_all_ones) + let val_sign = fx.bcx.ins().sshr_imm(val, i64::from(ty.bits() - 1)); + let xor = fx.bcx.ins().bxor(val_hi, val_sign); + fx.bcx.ins().icmp_imm(IntCC::NotEqual, xor, 0) }; (val, has_overflow) } diff --git a/compiler/rustc_codegen_cranelift/src/optimize/code_layout.rs b/compiler/rustc_codegen_cranelift/src/optimize/code_layout.rs deleted file mode 100644 index ca9ff15ec1..0000000000 --- a/compiler/rustc_codegen_cranelift/src/optimize/code_layout.rs +++ /dev/null @@ -1,34 +0,0 @@ -//! This optimization moves cold code to the end of the function. -//! -//! Some code is executed much less often than other code. For example panicking or the -//! landingpads for unwinding. By moving this cold code to the end of the function the average -//! amount of jumps is reduced and the code locality is improved. -//! -//! # Undefined behaviour -//! -//! This optimization doesn't assume anything that isn't already assumed by Cranelift itself. - -use crate::prelude::*; - -pub(super) fn optimize_function(ctx: &mut Context, cold_blocks: &EntitySet) { - // FIXME Move the block in place instead of remove and append once - // bytecodealliance/cranelift#1339 is implemented. - - let mut block_insts = FxHashMap::default(); - for block in cold_blocks.keys().filter(|&block| cold_blocks.contains(block)) { - let insts = ctx.func.layout.block_insts(block).collect::>(); - for &inst in &insts { - ctx.func.layout.remove_inst(inst); - } - block_insts.insert(block, insts); - ctx.func.layout.remove_block(block); - } - - // And then append them at the back again. - for block in cold_blocks.keys().filter(|&block| cold_blocks.contains(block)) { - ctx.func.layout.append_block(block); - for inst in block_insts.remove(&block).unwrap() { - ctx.func.layout.append_inst(inst, block); - } - } -} diff --git a/compiler/rustc_codegen_cranelift/src/optimize/mod.rs b/compiler/rustc_codegen_cranelift/src/optimize/mod.rs index 389f50e797..137fb5f773 100644 --- a/compiler/rustc_codegen_cranelift/src/optimize/mod.rs +++ b/compiler/rustc_codegen_cranelift/src/optimize/mod.rs @@ -2,29 +2,16 @@ use crate::prelude::*; -mod code_layout; pub(crate) mod peephole; -mod stack2reg; pub(crate) fn optimize_function<'tcx>( tcx: TyCtxt<'tcx>, instance: Instance<'tcx>, ctx: &mut Context, - cold_blocks: &EntitySet, clif_comments: &mut crate::pretty_clif::CommentWriter, ) { - // The code_layout optimization is very cheap. - self::code_layout::optimize_function(ctx, cold_blocks); + // FIXME classify optimizations over opt levels once we have more - if tcx.sess.opts.optimize == rustc_session::config::OptLevel::No { - return; // FIXME classify optimizations over opt levels - } - - // FIXME(#1142) stack2reg miscompiles lewton - if false { - self::stack2reg::optimize_function(ctx, clif_comments); - } - - crate::pretty_clif::write_clif_file(tcx, "stack2reg", None, instance, &ctx, &*clif_comments); + crate::pretty_clif::write_clif_file(tcx, "preopt", None, instance, &ctx, &*clif_comments); crate::base::verify_func(tcx, &*clif_comments, &ctx.func); } diff --git a/compiler/rustc_codegen_cranelift/src/optimize/stack2reg.rs b/compiler/rustc_codegen_cranelift/src/optimize/stack2reg.rs deleted file mode 100644 index 8bb02a3e55..0000000000 --- a/compiler/rustc_codegen_cranelift/src/optimize/stack2reg.rs +++ /dev/null @@ -1,486 +0,0 @@ -//! This optimization replaces stack accesses with SSA variables and removes dead stores when possible. -//! -//! # Undefined behaviour -//! -//! This optimization is based on the assumption that stack slots which don't have their address -//! leaked through `stack_addr` are only accessed using `stack_load` and `stack_store` in the -//! function which has the stack slots. This optimization also assumes that stack slot accesses -//! are never out of bounds. If these assumptions are not correct, then this optimization may remove -//! `stack_store` instruction incorrectly, or incorrectly use a previously stored value as the value -//! being loaded by a `stack_load`. - -use std::collections::BTreeMap; -use std::fmt; -use std::ops::Not; - -use rustc_data_structures::fx::FxHashSet; - -use cranelift_codegen::cursor::{Cursor, FuncCursor}; -use cranelift_codegen::ir::immediates::Offset32; -use cranelift_codegen::ir::{InstructionData, Opcode, ValueDef}; - -use crate::prelude::*; - -/// Workaround for `StackSlot` not implementing `Ord`. -#[derive(Copy, Clone, PartialEq, Eq)] -struct OrdStackSlot(StackSlot); - -impl fmt::Debug for OrdStackSlot { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{:?}", self.0) - } -} - -impl PartialOrd for OrdStackSlot { - fn partial_cmp(&self, rhs: &Self) -> Option { - self.0.as_u32().partial_cmp(&rhs.0.as_u32()) - } -} - -impl Ord for OrdStackSlot { - fn cmp(&self, rhs: &Self) -> std::cmp::Ordering { - self.0.as_u32().cmp(&rhs.0.as_u32()) - } -} - -#[derive(Debug, Default)] -struct StackSlotUsage { - stack_addr: FxHashSet, - stack_load: FxHashSet, - stack_store: FxHashSet, -} - -impl StackSlotUsage { - fn potential_stores_for_load(&self, ctx: &Context, load: Inst) -> Vec { - self.stack_store - .iter() - .cloned() - .filter(|&store| { - match spatial_overlap(&ctx.func, store, load) { - SpatialOverlap::No => false, // Can never be the source of the loaded value. - SpatialOverlap::Partial | SpatialOverlap::Full => true, - } - }) - .filter(|&store| { - match temporal_order(ctx, store, load) { - TemporalOrder::NeverBefore => false, // Can never be the source of the loaded value. - TemporalOrder::MaybeBefore | TemporalOrder::DefinitivelyBefore => true, - } - }) - .collect::>() - } - - fn potential_loads_of_store(&self, ctx: &Context, store: Inst) -> Vec { - self.stack_load - .iter() - .cloned() - .filter(|&load| { - match spatial_overlap(&ctx.func, store, load) { - SpatialOverlap::No => false, // Can never be the source of the loaded value. - SpatialOverlap::Partial | SpatialOverlap::Full => true, - } - }) - .filter(|&load| { - match temporal_order(ctx, store, load) { - TemporalOrder::NeverBefore => false, // Can never be the source of the loaded value. - TemporalOrder::MaybeBefore | TemporalOrder::DefinitivelyBefore => true, - } - }) - .collect::>() - } - - fn remove_unused_stack_addr(func: &mut Function, inst: Inst) { - func.dfg.detach_results(inst); - func.dfg.replace(inst).nop(); - } - - fn remove_unused_load(func: &mut Function, load: Inst) { - func.dfg.detach_results(load); - func.dfg.replace(load).nop(); - } - - fn remove_dead_store(&mut self, func: &mut Function, store: Inst) { - func.dfg.replace(store).nop(); - self.stack_store.remove(&store); - } - - fn change_load_to_alias(&mut self, func: &mut Function, load: Inst, value: Value) { - let loaded_value = func.dfg.inst_results(load)[0]; - let loaded_type = func.dfg.value_type(loaded_value); - - if func.dfg.value_type(value) == loaded_type { - func.dfg.detach_results(load); - func.dfg.replace(load).nop(); - func.dfg.change_to_alias(loaded_value, value); - } else { - func.dfg.replace(load).bitcast(loaded_type, value); - } - - self.stack_load.remove(&load); - } -} - -struct OptimizeContext<'a> { - ctx: &'a mut Context, - stack_slot_usage_map: BTreeMap, -} - -impl<'a> OptimizeContext<'a> { - fn for_context(ctx: &'a mut Context) -> Self { - ctx.flowgraph(); // Compute cfg and domtree. - - // Record all stack_addr, stack_load and stack_store instructions. - let mut stack_slot_usage_map = BTreeMap::::new(); - - let mut cursor = FuncCursor::new(&mut ctx.func); - while let Some(_block) = cursor.next_block() { - while let Some(inst) = cursor.next_inst() { - match cursor.func.dfg[inst] { - InstructionData::StackLoad { - opcode: Opcode::StackAddr, - stack_slot, - offset: _, - } => { - stack_slot_usage_map - .entry(OrdStackSlot(stack_slot)) - .or_insert_with(StackSlotUsage::default) - .stack_addr - .insert(inst); - } - InstructionData::StackLoad { - opcode: Opcode::StackLoad, - stack_slot, - offset: _, - } => { - stack_slot_usage_map - .entry(OrdStackSlot(stack_slot)) - .or_insert_with(StackSlotUsage::default) - .stack_load - .insert(inst); - } - InstructionData::StackStore { - opcode: Opcode::StackStore, - arg: _, - stack_slot, - offset: _, - } => { - stack_slot_usage_map - .entry(OrdStackSlot(stack_slot)) - .or_insert_with(StackSlotUsage::default) - .stack_store - .insert(inst); - } - _ => {} - } - } - } - - OptimizeContext { ctx, stack_slot_usage_map } - } -} - -pub(super) fn optimize_function( - ctx: &mut Context, - clif_comments: &mut crate::pretty_clif::CommentWriter, -) { - combine_stack_addr_with_load_store(&mut ctx.func); - - let mut opt_ctx = OptimizeContext::for_context(ctx); - - // FIXME Repeat following instructions until fixpoint. - - remove_unused_stack_addr_and_stack_load(&mut opt_ctx); - - if clif_comments.enabled() { - for (&OrdStackSlot(stack_slot), usage) in &opt_ctx.stack_slot_usage_map { - clif_comments.add_comment(stack_slot, format!("used by: {:?}", usage)); - } - } - - for (stack_slot, users) in opt_ctx.stack_slot_usage_map.iter_mut() { - if users.stack_addr.is_empty().not() { - // Stack addr leaked; there may be unknown loads and stores. - // FIXME use stacked borrows to optimize - continue; - } - - for load in users.stack_load.clone().into_iter() { - let potential_stores = users.potential_stores_for_load(&opt_ctx.ctx, load); - - if clif_comments.enabled() { - for &store in &potential_stores { - clif_comments.add_comment( - load, - format!( - "Potential store -> load forwarding {} -> {} ({:?}, {:?})", - opt_ctx.ctx.func.dfg.display_inst(store, None), - opt_ctx.ctx.func.dfg.display_inst(load, None), - spatial_overlap(&opt_ctx.ctx.func, store, load), - temporal_order(&opt_ctx.ctx, store, load), - ), - ); - } - } - - match *potential_stores { - [] => { - if clif_comments.enabled() { - clif_comments - .add_comment(load, "[BUG?] Reading uninitialized memory".to_string()); - } - } - [store] - if spatial_overlap(&opt_ctx.ctx.func, store, load) == SpatialOverlap::Full - && temporal_order(&opt_ctx.ctx, store, load) - == TemporalOrder::DefinitivelyBefore => - { - // Only one store could have been the origin of the value. - let stored_value = opt_ctx.ctx.func.dfg.inst_args(store)[0]; - - if clif_comments.enabled() { - clif_comments.add_comment( - load, - format!("Store to load forward {} -> {}", store, load), - ); - } - - users.change_load_to_alias(&mut opt_ctx.ctx.func, load, stored_value); - } - _ => {} // FIXME implement this - } - } - - for store in users.stack_store.clone().into_iter() { - let potential_loads = users.potential_loads_of_store(&opt_ctx.ctx, store); - - if clif_comments.enabled() { - for &load in &potential_loads { - clif_comments.add_comment( - store, - format!( - "Potential load from store {} <- {} ({:?}, {:?})", - opt_ctx.ctx.func.dfg.display_inst(load, None), - opt_ctx.ctx.func.dfg.display_inst(store, None), - spatial_overlap(&opt_ctx.ctx.func, store, load), - temporal_order(&opt_ctx.ctx, store, load), - ), - ); - } - } - - if potential_loads.is_empty() { - // Never loaded; can safely remove all stores and the stack slot. - // FIXME also remove stores when there is always a next store before a load. - - if clif_comments.enabled() { - clif_comments.add_comment( - store, - format!( - "Remove dead stack store {} of {}", - opt_ctx.ctx.func.dfg.display_inst(store, None), - stack_slot.0 - ), - ); - } - - users.remove_dead_store(&mut opt_ctx.ctx.func, store); - } - } - - if users.stack_store.is_empty() && users.stack_load.is_empty() { - opt_ctx.ctx.func.stack_slots[stack_slot.0].size = 0; - } - } -} - -fn combine_stack_addr_with_load_store(func: &mut Function) { - // Turn load and store into stack_load and stack_store when possible. - let mut cursor = FuncCursor::new(func); - while let Some(_block) = cursor.next_block() { - while let Some(inst) = cursor.next_inst() { - match cursor.func.dfg[inst] { - InstructionData::Load { opcode: Opcode::Load, arg: addr, flags: _, offset } => { - if cursor.func.dfg.ctrl_typevar(inst) == types::I128 - || cursor.func.dfg.ctrl_typevar(inst).is_vector() - { - continue; // WORKAROUD: stack_load.i128 not yet implemented - } - if let Some((stack_slot, stack_addr_offset)) = - try_get_stack_slot_and_offset_for_addr(cursor.func, addr) - { - if let Some(combined_offset) = offset.try_add_i64(stack_addr_offset.into()) - { - let ty = cursor.func.dfg.ctrl_typevar(inst); - cursor.func.dfg.replace(inst).stack_load( - ty, - stack_slot, - combined_offset, - ); - } - } - } - InstructionData::Store { - opcode: Opcode::Store, - args: [value, addr], - flags: _, - offset, - } => { - if cursor.func.dfg.ctrl_typevar(inst) == types::I128 - || cursor.func.dfg.ctrl_typevar(inst).is_vector() - { - continue; // WORKAROUND: stack_store.i128 not yet implemented - } - if let Some((stack_slot, stack_addr_offset)) = - try_get_stack_slot_and_offset_for_addr(cursor.func, addr) - { - if let Some(combined_offset) = offset.try_add_i64(stack_addr_offset.into()) - { - cursor.func.dfg.replace(inst).stack_store( - value, - stack_slot, - combined_offset, - ); - } - } - } - _ => {} - } - } - } -} - -fn remove_unused_stack_addr_and_stack_load(opt_ctx: &mut OptimizeContext<'_>) { - // FIXME incrementally rebuild on each call? - let mut stack_addr_load_insts_users = FxHashMap::>::default(); - - let mut cursor = FuncCursor::new(&mut opt_ctx.ctx.func); - while let Some(_block) = cursor.next_block() { - while let Some(inst) = cursor.next_inst() { - for &arg in cursor.func.dfg.inst_args(inst) { - if let ValueDef::Result(arg_origin, 0) = cursor.func.dfg.value_def(arg) { - match cursor.func.dfg[arg_origin].opcode() { - Opcode::StackAddr | Opcode::StackLoad => { - stack_addr_load_insts_users - .entry(arg_origin) - .or_insert_with(FxHashSet::default) - .insert(inst); - } - _ => {} - } - } - } - } - } - - #[cfg(debug_assertions)] - for inst in stack_addr_load_insts_users.keys() { - let mut is_recorded_stack_addr_or_stack_load = false; - for stack_slot_users in opt_ctx.stack_slot_usage_map.values() { - is_recorded_stack_addr_or_stack_load |= stack_slot_users.stack_addr.contains(inst) - || stack_slot_users.stack_load.contains(inst); - } - assert!(is_recorded_stack_addr_or_stack_load); - } - - // Replace all unused stack_addr and stack_load instructions with nop. - let mut func = &mut opt_ctx.ctx.func; - - for stack_slot_users in opt_ctx.stack_slot_usage_map.values_mut() { - stack_slot_users - .stack_addr - .drain_filter(|inst| { - stack_addr_load_insts_users.get(inst).map(|users| users.is_empty()).unwrap_or(true) - }) - .for_each(|inst| StackSlotUsage::remove_unused_stack_addr(&mut func, inst)); - - stack_slot_users - .stack_load - .drain_filter(|inst| { - stack_addr_load_insts_users.get(inst).map(|users| users.is_empty()).unwrap_or(true) - }) - .for_each(|inst| StackSlotUsage::remove_unused_load(&mut func, inst)); - } -} - -fn try_get_stack_slot_and_offset_for_addr( - func: &Function, - addr: Value, -) -> Option<(StackSlot, Offset32)> { - if let ValueDef::Result(addr_inst, 0) = func.dfg.value_def(addr) { - if let InstructionData::StackLoad { opcode: Opcode::StackAddr, stack_slot, offset } = - func.dfg[addr_inst] - { - return Some((stack_slot, offset)); - } - } - None -} - -#[derive(Copy, Clone, Debug, PartialEq, Eq)] -enum SpatialOverlap { - No, - Partial, - Full, -} - -fn spatial_overlap(func: &Function, src: Inst, dest: Inst) -> SpatialOverlap { - fn inst_info(func: &Function, inst: Inst) -> (StackSlot, Offset32, u32) { - match func.dfg[inst] { - InstructionData::StackLoad { opcode: Opcode::StackAddr, stack_slot, offset } - | InstructionData::StackLoad { opcode: Opcode::StackLoad, stack_slot, offset } - | InstructionData::StackStore { - opcode: Opcode::StackStore, - stack_slot, - offset, - arg: _, - } => (stack_slot, offset, func.dfg.ctrl_typevar(inst).bytes()), - _ => unreachable!("{:?}", func.dfg[inst]), - } - } - - debug_assert_ne!(src, dest); - - let (src_ss, src_offset, src_size) = inst_info(func, src); - let (dest_ss, dest_offset, dest_size) = inst_info(func, dest); - - if src_ss != dest_ss { - return SpatialOverlap::No; - } - - if src_offset == dest_offset && src_size == dest_size { - return SpatialOverlap::Full; - } - - let src_end: i64 = src_offset.try_add_i64(i64::from(src_size)).unwrap().into(); - let dest_end: i64 = dest_offset.try_add_i64(i64::from(dest_size)).unwrap().into(); - if src_end <= dest_offset.into() || dest_end <= src_offset.into() { - return SpatialOverlap::No; - } - - SpatialOverlap::Partial -} - -#[derive(Copy, Clone, Debug, PartialEq, Eq)] -enum TemporalOrder { - /// `src` will never be executed before `dest`. - NeverBefore, - - /// `src` may be executed before `dest`. - MaybeBefore, - - /// `src` will always be executed before `dest`. - /// There may still be other instructions in between. - DefinitivelyBefore, -} - -fn temporal_order(ctx: &Context, src: Inst, dest: Inst) -> TemporalOrder { - debug_assert_ne!(src, dest); - - if ctx.domtree.dominates(src, dest, &ctx.func.layout) { - TemporalOrder::DefinitivelyBefore - } else if ctx.domtree.dominates(src, dest, &ctx.func.layout) { - TemporalOrder::NeverBefore - } else { - TemporalOrder::MaybeBefore - } -} diff --git a/compiler/rustc_codegen_cranelift/src/pretty_clif.rs b/compiler/rustc_codegen_cranelift/src/pretty_clif.rs index d22ea3772e..cd8c5b5160 100644 --- a/compiler/rustc_codegen_cranelift/src/pretty_clif.rs +++ b/compiler/rustc_codegen_cranelift/src/pretty_clif.rs @@ -207,14 +207,14 @@ pub(crate) fn should_write_ir(tcx: TyCtxt<'_>) -> bool { pub(crate) fn write_ir_file( tcx: TyCtxt<'_>, - name: &str, + name: impl FnOnce() -> String, write: impl FnOnce(&mut dyn Write) -> std::io::Result<()>, ) { if !should_write_ir(tcx) { return; } - let clif_output_dir = tcx.output_filenames(LOCAL_CRATE).with_extension("clif"); + let clif_output_dir = tcx.output_filenames(()).with_extension("clif"); match std::fs::create_dir(&clif_output_dir) { Ok(()) => {} @@ -222,7 +222,7 @@ pub(crate) fn write_ir_file( res @ Err(_) => res.unwrap(), } - let clif_file_name = clif_output_dir.join(name); + let clif_file_name = clif_output_dir.join(name()); let res = std::fs::File::create(clif_file_name).and_then(|mut file| write(&mut file)); if let Err(err) = res { @@ -238,30 +238,31 @@ pub(crate) fn write_clif_file<'tcx>( context: &cranelift_codegen::Context, mut clif_comments: &CommentWriter, ) { - write_ir_file(tcx, &format!("{}.{}.clif", tcx.symbol_name(instance).name, postfix), |file| { - let value_ranges = - isa.map(|isa| context.build_value_labels_ranges(isa).expect("value location ranges")); + write_ir_file( + tcx, + || format!("{}.{}.clif", tcx.symbol_name(instance).name, postfix), + |file| { + let value_ranges = isa + .map(|isa| context.build_value_labels_ranges(isa).expect("value location ranges")); - let mut clif = String::new(); - cranelift_codegen::write::decorate_function( - &mut clif_comments, - &mut clif, - &context.func, - &DisplayFunctionAnnotations { - isa: Some(&*crate::build_isa(tcx.sess)), - value_ranges: value_ranges.as_ref(), - }, - ) - .unwrap(); + let mut clif = String::new(); + cranelift_codegen::write::decorate_function( + &mut clif_comments, + &mut clif, + &context.func, + &DisplayFunctionAnnotations { isa, value_ranges: value_ranges.as_ref() }, + ) + .unwrap(); - writeln!(file, "test compile")?; - writeln!(file, "set is_pic")?; - writeln!(file, "set enable_simd")?; - writeln!(file, "target {} haswell", crate::target_triple(tcx.sess))?; - writeln!(file)?; - file.write_all(clif.as_bytes())?; - Ok(()) - }); + writeln!(file, "test compile")?; + writeln!(file, "set is_pic")?; + writeln!(file, "set enable_simd")?; + writeln!(file, "target {} haswell", crate::target_triple(tcx.sess))?; + writeln!(file)?; + file.write_all(clif.as_bytes())?; + Ok(()) + }, + ); } impl fmt::Debug for FunctionCx<'_, '_, '_> { diff --git a/compiler/rustc_codegen_cranelift/src/toolchain.rs b/compiler/rustc_codegen_cranelift/src/toolchain.rs index 484a9b699a..f86236ef3e 100644 --- a/compiler/rustc_codegen_cranelift/src/toolchain.rs +++ b/compiler/rustc_codegen_cranelift/src/toolchain.rs @@ -2,9 +2,8 @@ use std::path::PathBuf; -use rustc_middle::bug; +use rustc_codegen_ssa::back::link::linker_and_flavor; use rustc_session::Session; -use rustc_target::spec::LinkerFlavor; /// Tries to infer the path of a binary for the target toolchain from the linker name. pub(crate) fn get_toolchain_binary(sess: &Session, tool: &str) -> PathBuf { @@ -30,89 +29,3 @@ pub(crate) fn get_toolchain_binary(sess: &Session, tool: &str) -> PathBuf { linker } - -// Adapted from https://github.com/rust-lang/rust/blob/5db778affee7c6600c8e7a177c48282dab3f6292/src/librustc_codegen_ssa/back/link.rs#L848-L931 -fn linker_and_flavor(sess: &Session) -> (PathBuf, LinkerFlavor) { - fn infer_from( - sess: &Session, - linker: Option, - flavor: Option, - ) -> Option<(PathBuf, LinkerFlavor)> { - match (linker, flavor) { - (Some(linker), Some(flavor)) => Some((linker, flavor)), - // only the linker flavor is known; use the default linker for the selected flavor - (None, Some(flavor)) => Some(( - PathBuf::from(match flavor { - LinkerFlavor::Em => { - if cfg!(windows) { - "emcc.bat" - } else { - "emcc" - } - } - LinkerFlavor::Gcc => { - if cfg!(any(target_os = "solaris", target_os = "illumos")) { - // On historical Solaris systems, "cc" may have - // been Sun Studio, which is not flag-compatible - // with "gcc". This history casts a long shadow, - // and many modern illumos distributions today - // ship GCC as "gcc" without also making it - // available as "cc". - "gcc" - } else { - "cc" - } - } - LinkerFlavor::Ld => "ld", - LinkerFlavor::Msvc => "link.exe", - LinkerFlavor::Lld(_) => "lld", - LinkerFlavor::PtxLinker => "rust-ptx-linker", - }), - flavor, - )), - (Some(linker), None) => { - let stem = linker.file_stem().and_then(|stem| stem.to_str()).unwrap_or_else(|| { - sess.fatal("couldn't extract file stem from specified linker") - }); - - let flavor = if stem == "emcc" { - LinkerFlavor::Em - } else if stem == "gcc" - || stem.ends_with("-gcc") - || stem == "clang" - || stem.ends_with("-clang") - { - LinkerFlavor::Gcc - } else if stem == "ld" || stem == "ld.lld" || stem.ends_with("-ld") { - LinkerFlavor::Ld - } else if stem == "link" || stem == "lld-link" { - LinkerFlavor::Msvc - } else if stem == "lld" || stem == "rust-lld" { - LinkerFlavor::Lld(sess.target.lld_flavor) - } else { - // fall back to the value in the target spec - sess.target.linker_flavor - }; - - Some((linker, flavor)) - } - (None, None) => None, - } - } - - // linker and linker flavor specified via command line have precedence over what the target - // specification specifies - if let Some(ret) = infer_from(sess, sess.opts.cg.linker.clone(), sess.opts.cg.linker_flavor) { - return ret; - } - - if let Some(ret) = infer_from( - sess, - sess.target.linker.clone().map(PathBuf::from), - Some(sess.target.linker_flavor), - ) { - return ret; - } - - bug!("Not enough information provided to determine how to invoke the linker"); -} diff --git a/compiler/rustc_codegen_cranelift/src/trap.rs b/compiler/rustc_codegen_cranelift/src/trap.rs index 1ab0703e98..21d3e68dbc 100644 --- a/compiler/rustc_codegen_cranelift/src/trap.rs +++ b/compiler/rustc_codegen_cranelift/src/trap.rs @@ -4,7 +4,6 @@ use crate::prelude::*; fn codegen_print(fx: &mut FunctionCx<'_, '_, '_>, msg: &str) { let puts = fx - .cx .module .declare_function( "puts", @@ -16,14 +15,13 @@ fn codegen_print(fx: &mut FunctionCx<'_, '_, '_>, msg: &str) { }, ) .unwrap(); - let puts = fx.cx.module.declare_func_in_func(puts, &mut fx.bcx.func); + let puts = fx.module.declare_func_in_func(puts, &mut fx.bcx.func); if fx.clif_comments.enabled() { fx.add_comment(puts, "puts"); } - let symbol_name = fx.tcx.symbol_name(fx.instance); - let real_msg = format!("trap at {:?} ({}): {}\0", fx.instance, symbol_name, msg); - let msg_ptr = fx.anonymous_str("trap", &real_msg); + let real_msg = format!("trap at {:?} ({}): {}\0", fx.instance, fx.symbol_name, msg); + let msg_ptr = fx.anonymous_str(&real_msg); fx.bcx.ins().call(puts, &[msg_ptr]); } diff --git a/compiler/rustc_codegen_cranelift/src/value_and_place.rs b/compiler/rustc_codegen_cranelift/src/value_and_place.rs index b97d390098..171f39805f 100644 --- a/compiler/rustc_codegen_cranelift/src/value_and_place.rs +++ b/compiler/rustc_codegen_cranelift/src/value_and_place.rs @@ -554,13 +554,14 @@ impl<'tcx> CPlace<'tcx> { let src_align = src_layout.align.abi.bytes() as u8; let dst_align = dst_layout.align.abi.bytes() as u8; fx.bcx.emit_small_memory_copy( - fx.cx.module.target_config(), + fx.module.target_config(), to_addr, from_addr, size, dst_align, src_align, true, + MemFlags::trusted(), ); } CValueInner::ByRef(_, Some(_)) => todo!(), diff --git a/compiler/rustc_codegen_cranelift/src/vtable.rs b/compiler/rustc_codegen_cranelift/src/vtable.rs index 9053d1aa1b..bbf07ffc85 100644 --- a/compiler/rustc_codegen_cranelift/src/vtable.rs +++ b/compiler/rustc_codegen_cranelift/src/vtable.rs @@ -72,15 +72,15 @@ pub(crate) fn get_vtable<'tcx>( layout: TyAndLayout<'tcx>, trait_ref: Option>, ) -> Value { - let data_id = if let Some(data_id) = fx.cx.vtables.get(&(layout.ty, trait_ref)) { + let data_id = if let Some(data_id) = fx.vtables.get(&(layout.ty, trait_ref)) { *data_id } else { let data_id = build_vtable(fx, layout, trait_ref); - fx.cx.vtables.insert((layout.ty, trait_ref), data_id); + fx.vtables.insert((layout.ty, trait_ref), data_id); data_id }; - let local_data_id = fx.cx.module.declare_data_in_func(data_id, &mut fx.bcx.func); + let local_data_id = fx.module.declare_data_in_func(data_id, &mut fx.bcx.func); fx.bcx.ins().global_value(fx.pointer_type, local_data_id) } @@ -94,7 +94,7 @@ fn build_vtable<'tcx>( let drop_in_place_fn = import_function( tcx, - fx.cx.module, + fx.module, Instance::resolve_drop_in_place(tcx, layout.ty).polymorphize(fx.tcx), ); @@ -111,7 +111,7 @@ fn build_vtable<'tcx>( opt_mth.map(|(def_id, substs)| { import_function( tcx, - fx.cx.module, + fx.module, Instance::resolve_for_vtable(tcx, ParamEnv::reveal_all(), def_id, substs) .unwrap() .polymorphize(fx.tcx), @@ -132,34 +132,16 @@ fn build_vtable<'tcx>( for (i, component) in components.into_iter().enumerate() { if let Some(func_id) = component { - let func_ref = fx.cx.module.declare_func_in_data(func_id, &mut data_ctx); + let func_ref = fx.module.declare_func_in_data(func_id, &mut data_ctx); data_ctx.write_function_addr((i * usize_size) as u32, func_ref); } } data_ctx.set_align(fx.tcx.data_layout.pointer_align.pref.bytes()); - let data_id = fx - .cx - .module - .declare_data( - &format!( - "__vtable.{}.for.{:?}.{}", - trait_ref - .as_ref() - .map(|trait_ref| format!("{:?}", trait_ref.skip_binder()).into()) - .unwrap_or(std::borrow::Cow::Borrowed("???")), - layout.ty, - fx.cx.vtables.len(), - ), - Linkage::Local, - false, - false, - ) - .unwrap(); - - // FIXME don't duplicate definitions in lazy jit mode - let _ = fx.cx.module.define_data(data_id, &data_ctx); + let data_id = fx.module.declare_anonymous_data(false, false).unwrap(); + + fx.module.define_data(data_id, &data_ctx).unwrap(); data_id } diff --git a/compiler/rustc_codegen_llvm/Cargo.toml b/compiler/rustc_codegen_llvm/Cargo.toml index 4999cb3c7a..d0eb6913ac 100644 --- a/compiler/rustc_codegen_llvm/Cargo.toml +++ b/compiler/rustc_codegen_llvm/Cargo.toml @@ -27,6 +27,7 @@ rustc_hir = { path = "../rustc_hir" } rustc_incremental = { path = "../rustc_incremental" } rustc_index = { path = "../rustc_index" } rustc_llvm = { path = "../rustc_llvm" } +rustc_metadata = { path = "../rustc_metadata" } rustc_session = { path = "../rustc_session" } rustc_serialize = { path = "../rustc_serialize" } rustc_target = { path = "../rustc_target" } diff --git a/compiler/rustc_codegen_llvm/src/abi.rs b/compiler/rustc_codegen_llvm/src/abi.rs index fba5667784..854e3ccc21 100644 --- a/compiler/rustc_codegen_llvm/src/abi.rs +++ b/compiler/rustc_codegen_llvm/src/abi.rs @@ -1,6 +1,7 @@ use crate::builder::Builder; use crate::context::CodegenCx; use crate::llvm::{self, AttributePlace}; +use crate::llvm_util; use crate::type_::Type; use crate::type_of::LayoutLlvmExt; use crate::value::Value; @@ -51,9 +52,15 @@ pub trait ArgAttributesExt { } fn should_use_mutable_noalias(cx: &CodegenCx<'_, '_>) -> bool { - // While #84958 has been fixed, mutable noalias is not enabled by default - // in Rust 1.53 out of an abundance of caution. - cx.tcx.sess.opts.debugging_opts.mutable_noalias.unwrap_or(false) + // LLVM prior to version 12 has known miscompiles in the presence of + // noalias attributes (see #54878). Only enable mutable noalias by + // default for versions we believe to be safe. + cx.tcx + .sess + .opts + .debugging_opts + .mutable_noalias + .unwrap_or_else(|| llvm_util::get_version() >= (12, 0, 0)) } impl ArgAttributesExt for ArgAttributes { diff --git a/compiler/rustc_codegen_llvm/src/asm.rs b/compiler/rustc_codegen_llvm/src/asm.rs index 84b091d8d4..ecf62ed213 100644 --- a/compiler/rustc_codegen_llvm/src/asm.rs +++ b/compiler/rustc_codegen_llvm/src/asm.rs @@ -283,10 +283,12 @@ impl AsmBuilderMethods<'tcx> for Builder<'a, 'll, 'tcx> { } InlineAsmArch::RiscV32 | InlineAsmArch::RiscV64 => {} InlineAsmArch::Nvptx64 => {} + InlineAsmArch::PowerPC | InlineAsmArch::PowerPC64 => {} InlineAsmArch::Hexagon => {} InlineAsmArch::Mips | InlineAsmArch::Mips64 => {} InlineAsmArch::SpirV => {} InlineAsmArch::Wasm32 => {} + InlineAsmArch::Bpf => {} } } if !options.contains(InlineAsmOptions::NOMEM) { @@ -355,10 +357,49 @@ impl AsmBuilderMethods<'tcx> for Builder<'a, 'll, 'tcx> { } impl AsmMethods for CodegenCx<'ll, 'tcx> { - fn codegen_global_asm(&self, ga: &hir::GlobalAsm) { - let asm = ga.asm.as_str(); + fn codegen_global_asm( + &self, + template: &[InlineAsmTemplatePiece], + operands: &[GlobalAsmOperandRef], + options: InlineAsmOptions, + _line_spans: &[Span], + ) { + let asm_arch = self.tcx.sess.asm_arch.unwrap(); + + // Default to Intel syntax on x86 + let intel_syntax = matches!(asm_arch, InlineAsmArch::X86 | InlineAsmArch::X86_64) + && !options.contains(InlineAsmOptions::ATT_SYNTAX); + + // Build the template string + let mut template_str = String::new(); + if intel_syntax { + template_str.push_str(".intel_syntax\n"); + } + for piece in template { + match *piece { + InlineAsmTemplatePiece::String(ref s) => template_str.push_str(s), + InlineAsmTemplatePiece::Placeholder { operand_idx, modifier: _, span: _ } => { + match operands[operand_idx] { + GlobalAsmOperandRef::Const { ref string } => { + // Const operands get injected directly into the + // template. Note that we don't need to escape $ + // here unlike normal inline assembly. + template_str.push_str(string); + } + } + } + } + } + if intel_syntax { + template_str.push_str("\n.att_syntax\n"); + } + unsafe { - llvm::LLVMRustAppendModuleInlineAsm(self.llmod, asm.as_ptr().cast(), asm.len()); + llvm::LLVMRustAppendModuleInlineAsm( + self.llmod, + template_str.as_ptr().cast(), + template_str.len(), + ); } } } @@ -540,6 +581,9 @@ fn reg_to_llvm(reg: InlineAsmRegOrRegClass, layout: Option<&TyAndLayout<'tcx>>) InlineAsmRegClass::Nvptx(NvptxInlineAsmRegClass::reg16) => "h", InlineAsmRegClass::Nvptx(NvptxInlineAsmRegClass::reg32) => "r", InlineAsmRegClass::Nvptx(NvptxInlineAsmRegClass::reg64) => "l", + InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::reg) => "r", + InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::reg_nonzero) => "b", + InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::freg) => "f", InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::reg) => "r", InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::freg) => "f", InlineAsmRegClass::X86(X86InlineAsmRegClass::reg) => "r", @@ -550,6 +594,8 @@ fn reg_to_llvm(reg: InlineAsmRegOrRegClass, layout: Option<&TyAndLayout<'tcx>>) InlineAsmRegClass::X86(X86InlineAsmRegClass::zmm_reg) => "v", InlineAsmRegClass::X86(X86InlineAsmRegClass::kreg) => "^Yk", InlineAsmRegClass::Wasm(WasmInlineAsmRegClass::local) => "r", + InlineAsmRegClass::Bpf(BpfInlineAsmRegClass::reg) => "r", + InlineAsmRegClass::Bpf(BpfInlineAsmRegClass::wreg) => "w", InlineAsmRegClass::SpirV(SpirVInlineAsmRegClass::reg) => { bug!("LLVM backend does not support SPIR-V") } @@ -590,6 +636,7 @@ fn modifier_to_llvm( InlineAsmRegClass::Hexagon(_) => None, InlineAsmRegClass::Mips(_) => None, InlineAsmRegClass::Nvptx(_) => None, + InlineAsmRegClass::PowerPC(_) => None, InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::reg) | InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::freg) => None, InlineAsmRegClass::X86(X86InlineAsmRegClass::reg) @@ -617,6 +664,7 @@ fn modifier_to_llvm( }, InlineAsmRegClass::X86(X86InlineAsmRegClass::kreg) => None, InlineAsmRegClass::Wasm(WasmInlineAsmRegClass::local) => None, + InlineAsmRegClass::Bpf(_) => None, InlineAsmRegClass::SpirV(SpirVInlineAsmRegClass::reg) => { bug!("LLVM backend does not support SPIR-V") } @@ -651,6 +699,9 @@ fn dummy_output_type(cx: &CodegenCx<'ll, 'tcx>, reg: InlineAsmRegClass) -> &'ll InlineAsmRegClass::Nvptx(NvptxInlineAsmRegClass::reg16) => cx.type_i16(), InlineAsmRegClass::Nvptx(NvptxInlineAsmRegClass::reg32) => cx.type_i32(), InlineAsmRegClass::Nvptx(NvptxInlineAsmRegClass::reg64) => cx.type_i64(), + InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::reg) => cx.type_i32(), + InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::reg_nonzero) => cx.type_i32(), + InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::freg) => cx.type_f64(), InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::reg) => cx.type_i32(), InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::freg) => cx.type_f32(), InlineAsmRegClass::X86(X86InlineAsmRegClass::reg) @@ -661,6 +712,8 @@ fn dummy_output_type(cx: &CodegenCx<'ll, 'tcx>, reg: InlineAsmRegClass) -> &'ll | InlineAsmRegClass::X86(X86InlineAsmRegClass::zmm_reg) => cx.type_f32(), InlineAsmRegClass::X86(X86InlineAsmRegClass::kreg) => cx.type_i16(), InlineAsmRegClass::Wasm(WasmInlineAsmRegClass::local) => cx.type_i32(), + InlineAsmRegClass::Bpf(BpfInlineAsmRegClass::reg) => cx.type_i64(), + InlineAsmRegClass::Bpf(BpfInlineAsmRegClass::wreg) => cx.type_i32(), InlineAsmRegClass::SpirV(SpirVInlineAsmRegClass::reg) => { bug!("LLVM backend does not support SPIR-V") } diff --git a/compiler/rustc_codegen_llvm/src/attributes.rs b/compiler/rustc_codegen_llvm/src/attributes.rs index ede38b723c..f1c45d8dc3 100644 --- a/compiler/rustc_codegen_llvm/src/attributes.rs +++ b/compiler/rustc_codegen_llvm/src/attributes.rs @@ -4,12 +4,10 @@ use std::ffi::CString; use cstr::cstr; use rustc_codegen_ssa::traits::*; -use rustc_data_structures::fx::FxHashMap; use rustc_data_structures::small_c_str::SmallCStr; use rustc_hir::def_id::DefId; use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags; use rustc_middle::ty::layout::HasTyCtxt; -use rustc_middle::ty::query::Providers; use rustc_middle::ty::{self, TyCtxt}; use rustc_session::config::OptLevel; use rustc_session::Session; @@ -352,35 +350,6 @@ pub fn from_fn_attrs(cx: &CodegenCx<'ll, 'tcx>, llfn: &'ll Value, instance: ty:: } } -pub fn provide_both(providers: &mut Providers) { - providers.wasm_import_module_map = |tcx, cnum| { - // Build up a map from DefId to a `NativeLib` structure, where - // `NativeLib` internally contains information about - // `#[link(wasm_import_module = "...")]` for example. - let native_libs = tcx.native_libraries(cnum); - - let def_id_to_native_lib = native_libs - .iter() - .filter_map(|lib| lib.foreign_module.map(|id| (id, lib))) - .collect::>(); - - let mut ret = FxHashMap::default(); - for (def_id, lib) in tcx.foreign_modules(cnum).iter() { - let module = def_id_to_native_lib.get(&def_id).and_then(|s| s.wasm_import_module); - let module = match module { - Some(s) => s, - None => continue, - }; - ret.extend(lib.foreign_items.iter().map(|id| { - assert_eq!(id.krate, cnum); - (*id, module.to_string()) - })); - } - - ret - }; -} - fn wasm_import_module(tcx: TyCtxt<'_>, id: DefId) -> Option { tcx.wasm_import_module_map(id.krate).get(&id).map(|s| CString::new(&s[..]).unwrap()) } diff --git a/compiler/rustc_codegen_llvm/src/back/archive.rs b/compiler/rustc_codegen_llvm/src/back/archive.rs index 4e7213853b..64416bced3 100644 --- a/compiler/rustc_codegen_llvm/src/back/archive.rs +++ b/compiler/rustc_codegen_llvm/src/back/archive.rs @@ -8,9 +8,11 @@ use std::ptr; use std::str; use crate::llvm::archive_ro::{ArchiveRO, Child}; -use crate::llvm::{self, ArchiveKind}; +use crate::llvm::{self, ArchiveKind, LLVMMachineType, LLVMRustCOFFShortExport}; use rustc_codegen_ssa::back::archive::{find_library, ArchiveBuilder}; use rustc_codegen_ssa::{looks_like_rust_object_file, METADATA_FILENAME}; +use rustc_data_structures::temp_dir::MaybeTempDir; +use rustc_middle::middle::cstore::DllImport; use rustc_session::Session; use rustc_span::symbol::Symbol; @@ -61,6 +63,17 @@ fn archive_config<'a>(sess: &'a Session, output: &Path, input: Option<&Path>) -> } } +/// Map machine type strings to values of LLVM's MachineTypes enum. +fn llvm_machine_type(cpu: &str) -> LLVMMachineType { + match cpu { + "x86_64" => LLVMMachineType::AMD64, + "x86" => LLVMMachineType::I386, + "aarch64" => LLVMMachineType::ARM64, + "arm" => LLVMMachineType::ARM, + _ => panic!("unsupported cpu type {}", cpu), + } +} + impl<'a> ArchiveBuilder<'a> for LlvmArchiveBuilder<'a> { /// Creates a new static archive, ready for modifying the archive specified /// by `config`. @@ -100,8 +113,9 @@ impl<'a> ArchiveBuilder<'a> for LlvmArchiveBuilder<'a> { /// Adds all of the contents of a native library to this archive. This will /// search in the relevant locations for a library named `name`. - fn add_native_library(&mut self, name: Symbol) { - let location = find_library(name, &self.config.lib_search_paths, self.config.sess); + fn add_native_library(&mut self, name: Symbol, verbatim: bool) { + let location = + find_library(name, verbatim, &self.config.lib_search_paths, self.config.sess); self.add_archive(&location, |_| false).unwrap_or_else(|e| { self.config.sess.fatal(&format!( "failed to add native library {}: {}", @@ -174,6 +188,74 @@ impl<'a> ArchiveBuilder<'a> for LlvmArchiveBuilder<'a> { self.config.sess.fatal(&format!("failed to build archive: {}", e)); } } + + fn inject_dll_import_lib( + &mut self, + lib_name: &str, + dll_imports: &[DllImport], + tmpdir: &MaybeTempDir, + ) { + let output_path = { + let mut output_path: PathBuf = tmpdir.as_ref().to_path_buf(); + output_path.push(format!("{}_imports", lib_name)); + output_path.with_extension("lib") + }; + + // we've checked for \0 characters in the library name already + let dll_name_z = CString::new(lib_name).unwrap(); + // All import names are Rust identifiers and therefore cannot contain \0 characters. + // FIXME: when support for #[link_name] implemented, ensure that import.name values don't + // have any \0 characters + let import_name_vector: Vec = dll_imports + .iter() + .map(if self.config.sess.target.arch == "x86" { + |import: &DllImport| CString::new(format!("_{}", import.name.to_string())).unwrap() + } else { + |import: &DllImport| CString::new(import.name.to_string()).unwrap() + }) + .collect(); + + let output_path_z = rustc_fs_util::path_to_c_string(&output_path); + + tracing::trace!("invoking LLVMRustWriteImportLibrary"); + tracing::trace!(" dll_name {:#?}", dll_name_z); + tracing::trace!(" output_path {}", output_path.display()); + tracing::trace!( + " import names: {}", + dll_imports.iter().map(|import| import.name.to_string()).collect::>().join(", "), + ); + + let ffi_exports: Vec = import_name_vector + .iter() + .map(|name_z| LLVMRustCOFFShortExport::from_name(name_z.as_ptr())) + .collect(); + let result = unsafe { + crate::llvm::LLVMRustWriteImportLibrary( + dll_name_z.as_ptr(), + output_path_z.as_ptr(), + ffi_exports.as_ptr(), + ffi_exports.len(), + llvm_machine_type(&self.config.sess.target.arch) as u16, + !self.config.sess.target.is_like_msvc, + ) + }; + + if result == crate::llvm::LLVMRustResult::Failure { + self.config.sess.fatal(&format!( + "Error creating import library for {}: {}", + lib_name, + llvm::last_error().unwrap_or("unknown LLVM error".to_string()) + )); + } + + self.add_archive(&output_path, |_| false).unwrap_or_else(|e| { + self.config.sess.fatal(&format!( + "failed to add native library {}: {}", + output_path.display(), + e + )); + }); + } } impl<'a> LlvmArchiveBuilder<'a> { diff --git a/compiler/rustc_codegen_llvm/src/back/lto.rs b/compiler/rustc_codegen_llvm/src/back/lto.rs index 4226ed7d99..f612785e5a 100644 --- a/compiler/rustc_codegen_llvm/src/back/lto.rs +++ b/compiler/rustc_codegen_llvm/src/back/lto.rs @@ -568,10 +568,11 @@ fn thin_lto( pub(crate) fn run_pass_manager( cgcx: &CodegenContext, + diag_handler: &Handler, module: &ModuleCodegen, config: &ModuleConfig, thin: bool, -) { +) -> Result<(), FatalError> { let _timer = cgcx.prof.extra_verbose_generic_activity("LLVM_lto_optimize", &module.name[..]); // Now we have one massive module inside of llmod. Time to run the @@ -584,15 +585,16 @@ pub(crate) fn run_pass_manager( if write::should_use_new_llvm_pass_manager(config) { let opt_stage = if thin { llvm::OptStage::ThinLTO } else { llvm::OptStage::FatLTO }; let opt_level = config.opt_level.unwrap_or(config::OptLevel::No); - // See comment below for why this is necessary. - let opt_level = if let config::OptLevel::No = opt_level { - config::OptLevel::Less - } else { - opt_level - }; - write::optimize_with_new_llvm_pass_manager(cgcx, module, config, opt_level, opt_stage); + write::optimize_with_new_llvm_pass_manager( + cgcx, + diag_handler, + module, + config, + opt_level, + opt_stage, + )?; debug!("lto done"); - return; + return Ok(()); } let pm = llvm::LLVMCreatePassManager(); @@ -603,26 +605,10 @@ pub(crate) fn run_pass_manager( llvm::LLVMRustAddPass(pm, pass.unwrap()); } - // When optimizing for LTO we don't actually pass in `-O0`, but we force - // it to always happen at least with `-O1`. - // - // With ThinLTO we mess around a lot with symbol visibility in a way - // that will actually cause linking failures if we optimize at O0 which - // notable is lacking in dead code elimination. To ensure we at least - // get some optimizations and correctly link we forcibly switch to `-O1` - // to get dead code elimination. - // - // Note that in general this shouldn't matter too much as you typically - // only turn on ThinLTO when you're compiling with optimizations - // otherwise. let opt_level = config .opt_level .map(|x| to_llvm_opt_settings(x).0) .unwrap_or(llvm::CodeGenOptLevel::None); - let opt_level = match opt_level { - llvm::CodeGenOptLevel::None => llvm::CodeGenOptLevel::Less, - level => level, - }; with_llvm_pmb(module.module_llvm.llmod(), config, opt_level, false, &mut |b| { if thin { llvm::LLVMRustPassManagerBuilderPopulateThinLTOPassManager(b, pm); @@ -650,6 +636,7 @@ pub(crate) fn run_pass_manager( llvm::LLVMDisposePassManager(pm); } debug!("lto done"); + Ok(()) } pub struct ModuleBuffer(&'static mut llvm::ModuleBuffer); @@ -872,7 +859,7 @@ pub unsafe fn optimize_thin_module( { info!("running thin lto passes over {}", module.name); let config = cgcx.config(module.kind); - run_pass_manager(cgcx, &module, config, true); + run_pass_manager(cgcx, &diag_handler, &module, config, true)?; save_temp_bitcode(cgcx, &module, "thin-lto-after-pm"); } } diff --git a/compiler/rustc_codegen_llvm/src/back/write.rs b/compiler/rustc_codegen_llvm/src/back/write.rs index b628ae3ae3..5b4a187a1d 100644 --- a/compiler/rustc_codegen_llvm/src/back/write.rs +++ b/compiler/rustc_codegen_llvm/src/back/write.rs @@ -20,7 +20,6 @@ use rustc_codegen_ssa::{CompiledModule, ModuleCodegen}; use rustc_data_structures::small_c_str::SmallCStr; use rustc_errors::{FatalError, Handler, Level}; use rustc_fs_util::{link_or_copy, path_to_c_string}; -use rustc_hir::def_id::LOCAL_CRATE; use rustc_middle::bug; use rustc_middle::ty::TyCtxt; use rustc_session::config::{self, Lto, OutputType, Passes, SwitchWithOptPath}; @@ -92,13 +91,12 @@ pub fn create_informational_target_machine(sess: &Session) -> &'static mut llvm: pub fn create_target_machine(tcx: TyCtxt<'_>, mod_name: &str) -> &'static mut llvm::TargetMachine { let split_dwarf_file = if tcx.sess.target_can_use_split_dwarf() { - tcx.output_filenames(LOCAL_CRATE) - .split_dwarf_path(tcx.sess.split_debuginfo(), Some(mod_name)) + tcx.output_filenames(()).split_dwarf_path(tcx.sess.split_debuginfo(), Some(mod_name)) } else { None }; let config = TargetMachineFactoryConfig { split_dwarf_file }; - target_machine_factory(&tcx.sess, tcx.backend_optimization_level(LOCAL_CRATE))(config) + target_machine_factory(&tcx.sess, tcx.backend_optimization_level(()))(config) .unwrap_or_else(|err| llvm_err(tcx.sess.diagnostic(), &err).raise()) } @@ -410,16 +408,17 @@ fn get_pgo_use_path(config: &ModuleConfig) -> Option { pub(crate) fn should_use_new_llvm_pass_manager(config: &ModuleConfig) -> bool { // The new pass manager is disabled by default. - config.new_llvm_pass_manager + config.new_llvm_pass_manager.unwrap_or(false) } pub(crate) unsafe fn optimize_with_new_llvm_pass_manager( cgcx: &CodegenContext, + diag_handler: &Handler, module: &ModuleCodegen, config: &ModuleConfig, opt_level: config::OptLevel, opt_stage: llvm::OptStage, -) { +) -> Result<(), FatalError> { let unroll_loops = opt_level != config::OptLevel::Size && opt_level != config::OptLevel::SizeMin; let using_thin_buffers = opt_stage == llvm::OptStage::PreLinkThinLTO || config.bitcode_needed(); @@ -449,13 +448,12 @@ pub(crate) unsafe fn optimize_with_new_llvm_pass_manager( std::ptr::null_mut() }; + let extra_passes = config.passes.join(","); + // FIXME: NewPM doesn't provide a facility to pass custom InlineParams. // We would have to add upstream support for this first, before we can support // config.inline_threshold and our more aggressive default thresholds. - // FIXME: NewPM uses an different and more explicit way to textually represent - // pass pipelines. It would probably make sense to expose this, but it would - // require a different format than the current -C passes. - llvm::LLVMRustOptimizeWithNewPassManager( + let result = llvm::LLVMRustOptimizeWithNewPassManager( module.module_llvm.llmod(), &*module.module_llvm.tm, to_pass_builder_opt_level(opt_level), @@ -472,10 +470,15 @@ pub(crate) unsafe fn optimize_with_new_llvm_pass_manager( sanitizer_options.as_ref(), pgo_gen_path.as_ref().map_or(std::ptr::null(), |s| s.as_ptr()), pgo_use_path.as_ref().map_or(std::ptr::null(), |s| s.as_ptr()), + config.instrument_coverage, + config.instrument_gcov, llvm_selfprofiler, selfprofile_before_pass_callback, selfprofile_after_pass_callback, + extra_passes.as_ptr().cast(), + extra_passes.len(), ); + result.into_result().map_err(|()| llvm_err(diag_handler, "failed to run LLVM passes")) } // Unsafe due to LLVM calls. @@ -484,7 +487,7 @@ pub(crate) unsafe fn optimize( diag_handler: &Handler, module: &ModuleCodegen, config: &ModuleConfig, -) { +) -> Result<(), FatalError> { let _timer = cgcx.prof.generic_activity_with_arg("LLVM_module_optimize", &module.name[..]); let llmod = module.module_llvm.llmod(); @@ -509,8 +512,14 @@ pub(crate) unsafe fn optimize( _ if cgcx.opts.cg.linker_plugin_lto.enabled() => llvm::OptStage::PreLinkThinLTO, _ => llvm::OptStage::PreLinkNoLTO, }; - optimize_with_new_llvm_pass_manager(cgcx, module, config, opt_level, opt_stage); - return; + return optimize_with_new_llvm_pass_manager( + cgcx, + diag_handler, + module, + config, + opt_level, + opt_stage, + ); } if cgcx.prof.llvm_recording_enabled() { @@ -545,15 +554,6 @@ pub(crate) unsafe fn optimize( llvm::LLVMRustAddPass(fpm, find_pass("lint").unwrap()); continue; } - if pass_name == "insert-gcov-profiling" || pass_name == "instrprof" { - // Instrumentation must be inserted before optimization, - // otherwise LLVM may optimize some functions away which - // breaks llvm-cov. - // - // This mirrors what Clang does in lib/CodeGen/BackendUtil.cpp. - llvm::LLVMRustAddPass(mpm, find_pass(pass_name).unwrap()); - continue; - } if let Some(pass) = find_pass(pass_name) { extra_passes.push(pass); @@ -566,6 +566,18 @@ pub(crate) unsafe fn optimize( } } + // Instrumentation must be inserted before optimization, + // otherwise LLVM may optimize some functions away which + // breaks llvm-cov. + // + // This mirrors what Clang does in lib/CodeGen/BackendUtil.cpp. + if config.instrument_gcov { + llvm::LLVMRustAddPass(mpm, find_pass("insert-gcov-profiling").unwrap()); + } + if config.instrument_coverage { + llvm::LLVMRustAddPass(mpm, find_pass("instrprof").unwrap()); + } + add_sanitizer_passes(config, &mut extra_passes); // Some options cause LLVM bitcode to be emitted, which uses ThinLTOBuffers, so we need @@ -642,6 +654,7 @@ pub(crate) unsafe fn optimize( llvm::LLVMDisposePassManager(fpm); llvm::LLVMDisposePassManager(mpm); } + Ok(()) } unsafe fn add_sanitizer_passes(config: &ModuleConfig, passes: &mut Vec<&'static mut llvm::Pass>) { diff --git a/compiler/rustc_codegen_llvm/src/base.rs b/compiler/rustc_codegen_llvm/src/base.rs index 6f6c649bb0..cc3cbea4de 100644 --- a/compiler/rustc_codegen_llvm/src/base.rs +++ b/compiler/rustc_codegen_llvm/src/base.rs @@ -18,7 +18,6 @@ use crate::builder::Builder; use crate::common; use crate::context::CodegenCx; use crate::llvm; -use crate::metadata; use crate::value::Value; use rustc_codegen_ssa::base::maybe_create_entry_wrapper; @@ -47,8 +46,24 @@ pub fn write_compressed_metadata<'tcx>( use snap::write::FrameEncoder; use std::io::Write; + // Historical note: + // + // When using link.exe it was seen that the section name `.note.rustc` + // was getting shortened to `.note.ru`, and according to the PE and COFF + // specification: + // + // > Executable images do not use a string table and do not support + // > section names longer than 8 characters + // + // https://docs.microsoft.com/en-us/windows/win32/debug/pe-format + // + // As a result, we choose a slightly shorter name! As to why + // `.note.rustc` works on MinGW, see + // https://github.com/llvm/llvm-project/blob/llvmorg-12.0.0/lld/COFF/Writer.cpp#L1190-L1197 + let section_name = if tcx.sess.target.is_like_osx { "__DATA,.rustc" } else { ".rustc" }; + let (metadata_llcx, metadata_llmod) = (&*llvm_module.llcx, llvm_module.llmod()); - let mut compressed = tcx.metadata_encoding_version(); + let mut compressed = rustc_metadata::METADATA_HEADER.to_vec(); FrameEncoder::new(&mut compressed).write_all(&metadata.raw_data).unwrap(); let llmeta = common::bytes_in_context(metadata_llcx, &compressed); @@ -59,7 +74,6 @@ pub fn write_compressed_metadata<'tcx>( unsafe { llvm::LLVMAddGlobal(metadata_llmod, common::val_ty(llconst), buf.as_ptr()) }; unsafe { llvm::LLVMSetInitializer(llglobal, llconst); - let section_name = metadata::metadata_section_name(&tcx.sess.target); let name = SmallCStr::new(section_name); llvm::LLVMSetSection(llglobal, name.as_ptr()); diff --git a/compiler/rustc_codegen_llvm/src/builder.rs b/compiler/rustc_codegen_llvm/src/builder.rs index 053cda1e7c..c8cf0116c6 100644 --- a/compiler/rustc_codegen_llvm/src/builder.rs +++ b/compiler/rustc_codegen_llvm/src/builder.rs @@ -69,6 +69,7 @@ impl abi::HasDataLayout for Builder<'_, '_, '_> { } impl ty::layout::HasTyCtxt<'tcx> for Builder<'_, '_, 'tcx> { + #[inline] fn tcx(&self) -> TyCtxt<'tcx> { self.cx.tcx } @@ -81,6 +82,7 @@ impl ty::layout::HasParamEnv<'tcx> for Builder<'_, '_, 'tcx> { } impl HasTargetSpec for Builder<'_, '_, 'tcx> { + #[inline] fn target_spec(&self) -> &Target { &self.cx.target_spec() } @@ -98,6 +100,7 @@ impl abi::LayoutOf for Builder<'_, '_, 'tcx> { impl Deref for Builder<'_, 'll, 'tcx> { type Target = CodegenCx<'ll, 'tcx>; + #[inline] fn deref(&self) -> &Self::Target { self.cx } @@ -118,24 +121,16 @@ macro_rules! builder_methods_for_value_instructions { } impl BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { - fn new_block<'b>(cx: &'a CodegenCx<'ll, 'tcx>, llfn: &'ll Value, name: &'b str) -> Self { - let mut bx = Builder::with_cx(cx); - let llbb = unsafe { - let name = SmallCStr::new(name); - llvm::LLVMAppendBasicBlockInContext(cx.llcx, llfn, name.as_ptr()) - }; - bx.position_at_end(llbb); + fn build(cx: &'a CodegenCx<'ll, 'tcx>, llbb: &'ll BasicBlock) -> Self { + let bx = Builder::with_cx(cx); + unsafe { + llvm::LLVMPositionBuilderAtEnd(bx.llbuilder, llbb); + } bx } - fn with_cx(cx: &'a CodegenCx<'ll, 'tcx>) -> Self { - // Create a fresh builder from the crate context. - let llbuilder = unsafe { llvm::LLVMCreateBuilderInContext(cx.llcx) }; - Builder { llbuilder, cx } - } - - fn build_sibling_block(&self, name: &str) -> Self { - Builder::new_block(self.cx, self.llfn(), name) + fn cx(&self) -> &CodegenCx<'ll, 'tcx> { + self.cx } fn llbb(&self) -> &'ll BasicBlock { @@ -144,12 +139,22 @@ impl BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { fn set_span(&mut self, _span: Span) {} - fn position_at_end(&mut self, llbb: &'ll BasicBlock) { + fn append_block(cx: &'a CodegenCx<'ll, 'tcx>, llfn: &'ll Value, name: &str) -> &'ll BasicBlock { unsafe { - llvm::LLVMPositionBuilderAtEnd(self.llbuilder, llbb); + let name = SmallCStr::new(name); + llvm::LLVMAppendBasicBlockInContext(cx.llcx, llfn, name.as_ptr()) } } + fn append_sibling_block(&mut self, name: &str) -> &'ll BasicBlock { + Self::append_block(self.cx, self.llfn(), name) + } + + fn build_sibling_block(&mut self, name: &str) -> Self { + let llbb = self.append_sibling_block(name); + Self::build(self.cx, llbb) + } + fn ret_void(&mut self) { unsafe { llvm::LLVMBuildRetVoid(self.llbuilder); @@ -1144,14 +1149,6 @@ impl BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { unsafe { llvm::LLVMBuildZExt(self.llbuilder, val, dest_ty, UNNAMED) } } - fn cx(&self) -> &CodegenCx<'ll, 'tcx> { - self.cx - } - - unsafe fn delete_basic_block(&mut self, bb: &'ll BasicBlock) { - llvm::LLVMDeleteBasicBlock(bb); - } - fn do_not_inline(&mut self, llret: &'ll Value) { llvm::Attribute::NoInline.apply_callsite(llvm::AttributePlace::Function, llret); } @@ -1165,6 +1162,12 @@ impl StaticBuilderMethods for Builder<'a, 'll, 'tcx> { } impl Builder<'a, 'll, 'tcx> { + fn with_cx(cx: &'a CodegenCx<'ll, 'tcx>) -> Self { + // Create a fresh builder from the crate context. + let llbuilder = unsafe { llvm::LLVMCreateBuilderInContext(cx.llcx) }; + Builder { llbuilder, cx } + } + pub fn llfn(&self) -> &'ll Value { unsafe { llvm::LLVMGetBasicBlockParent(self.llbb()) } } diff --git a/compiler/rustc_codegen_llvm/src/callee.rs b/compiler/rustc_codegen_llvm/src/callee.rs index b26969a501..bb16c90cd1 100644 --- a/compiler/rustc_codegen_llvm/src/callee.rs +++ b/compiler/rustc_codegen_llvm/src/callee.rs @@ -14,7 +14,6 @@ use tracing::debug; use rustc_middle::ty::layout::{FnAbiExt, HasTyCtxt}; use rustc_middle::ty::{self, Instance, TypeFoldable}; -use rustc_target::spec::RelocModel; /// Codegens a reference to a fn/method item, monomorphizing and /// inlining as it goes. @@ -181,7 +180,7 @@ pub fn get_fn(cx: &CodegenCx<'ll, 'tcx>, instance: Instance<'tcx>) -> &'ll Value llvm::LLVMSetDLLStorageClass(llfn, llvm::DLLStorageClass::DllImport); } - if cx.tcx.sess.relocation_model() == RelocModel::Static { + if cx.should_assume_dso_local(llfn, true) { llvm::LLVMRustSetDSOLocal(llfn, true); } } diff --git a/compiler/rustc_codegen_llvm/src/consts.rs b/compiler/rustc_codegen_llvm/src/consts.rs index 9904683997..e50d5506e2 100644 --- a/compiler/rustc_codegen_llvm/src/consts.rs +++ b/compiler/rustc_codegen_llvm/src/consts.rs @@ -282,6 +282,12 @@ impl CodegenCx<'ll, 'tcx> { } } + unsafe { + if self.should_assume_dso_local(g, true) { + llvm::LLVMRustSetDSOLocal(g, true); + } + } + self.instances.borrow_mut().insert(instance, g); g } @@ -363,6 +369,10 @@ impl StaticMethods for CodegenCx<'ll, 'tcx> { set_global_alignment(&self, g, self.align_of(ty)); llvm::LLVMSetInitializer(g, v); + if self.should_assume_dso_local(g, true) { + llvm::LLVMRustSetDSOLocal(g, true); + } + // As an optimization, all shared statics which do not have interior // mutability are placed into read-only memory. if !is_mutable && self.type_is_freeze(ty) { diff --git a/compiler/rustc_codegen_llvm/src/context.rs b/compiler/rustc_codegen_llvm/src/context.rs index f5c54b11c0..6aa952462f 100644 --- a/compiler/rustc_codegen_llvm/src/context.rs +++ b/compiler/rustc_codegen_llvm/src/context.rs @@ -765,18 +765,21 @@ impl<'b, 'tcx> CodegenCx<'b, 'tcx> { } impl HasDataLayout for CodegenCx<'ll, 'tcx> { + #[inline] fn data_layout(&self) -> &TargetDataLayout { &self.tcx.data_layout } } impl HasTargetSpec for CodegenCx<'ll, 'tcx> { + #[inline] fn target_spec(&self) -> &Target { &self.tcx.sess.target } } impl ty::layout::HasTyCtxt<'tcx> for CodegenCx<'ll, 'tcx> { + #[inline] fn tcx(&self) -> TyCtxt<'tcx> { self.tcx } diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs index 1faaa7e86f..d2a2e739ff 100644 --- a/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs +++ b/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs @@ -6,9 +6,8 @@ use llvm::coverageinfo::CounterMappingRegion; use rustc_codegen_ssa::coverageinfo::map::{Counter, CounterExpression}; use rustc_codegen_ssa::traits::{ConstMethods, CoverageInfoMethods}; use rustc_data_structures::fx::{FxHashMap, FxHashSet, FxIndexSet}; -use rustc_hir::def_id::{DefId, DefIdSet, LOCAL_CRATE}; +use rustc_hir::def_id::{DefId, DefIdSet}; use rustc_llvm::RustString; -use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags; use rustc_middle::mir::coverage::CodeRegion; use rustc_span::Symbol; @@ -249,7 +248,7 @@ fn save_function_record( /// /// We can find the unused functions (including generic functions) by the set difference of all MIR /// `DefId`s (`tcx` query `mir_keys`) minus the codegenned `DefId`s (`tcx` query -/// `collect_and_partition_mono_items`). +/// `codegened_and_inlined_items`). /// /// *HOWEVER* the codegenned `DefId`s are partitioned across multiple `CodegenUnit`s (CGUs), and /// this function is processing a `function_coverage_map` for the functions (`Instance`/`DefId`) @@ -266,7 +265,7 @@ fn add_unused_functions<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>) { let ignore_unused_generics = tcx.sess.instrument_coverage_except_unused_generics(); let all_def_ids: DefIdSet = tcx - .mir_keys(LOCAL_CRATE) + .mir_keys(()) .iter() .filter_map(|local_def_id| { let def_id = local_def_id.to_def_id(); @@ -277,15 +276,12 @@ fn add_unused_functions<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>) { }) .collect(); - let codegenned_def_ids = tcx.codegened_and_inlined_items(LOCAL_CRATE); + let codegenned_def_ids = tcx.codegened_and_inlined_items(()); let mut unused_def_ids_by_file: FxHashMap> = FxHashMap::default(); for &non_codegenned_def_id in all_def_ids.difference(codegenned_def_ids) { - let codegen_fn_attrs = tcx.codegen_fn_attrs(non_codegenned_def_id); - if codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::NO_COVERAGE) { - continue; - } - // Make sure the non-codegenned (unused) function has a file_name + // Make sure the non-codegenned (unused) function has at least one MIR + // `Coverage` statement with a code region, and return its file name. if let Some(non_codegenned_file_name) = tcx.covered_file_name(non_codegenned_def_id) { let def_ids = unused_def_ids_by_file.entry(*non_codegenned_file_name).or_insert_with(Vec::new); diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs index afc2bdbfd5..019bf4a09a 100644 --- a/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs +++ b/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs @@ -223,7 +223,8 @@ fn declare_unused_fn(cx: &CodegenCx<'ll, 'tcx>, def_id: &DefId) -> Instance<'tcx fn codegen_unused_fn_and_counter(cx: &CodegenCx<'ll, 'tcx>, instance: Instance<'tcx>) { let llfn = cx.get_fn(instance); - let mut bx = Builder::new_block(cx, llfn, "unused_function"); + let llbb = Builder::append_block(cx, llfn, "unused_function"); + let mut bx = Builder::build(cx, llbb); let fn_name = bx.get_pgo_func_name_var(instance); let hash = bx.const_u64(0); let num_counters = bx.const_u32(1); diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs index e6fa852155..1e70664e64 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs @@ -309,6 +309,7 @@ impl RecursiveTypeDescription<'ll, 'tcx> { unfinished_type, member_holding_stub, member_descriptions, + None, ); MetadataCreationResult::new(metadata_stub, true) } @@ -759,12 +760,12 @@ fn hex_encode(data: &[u8]) -> String { } pub fn file_metadata(cx: &CodegenCx<'ll, '_>, source_file: &SourceFile) -> &'ll DIFile { - debug!("file_metadata: file_name: {}", source_file.name); + debug!("file_metadata: file_name: {:?}", source_file.name); let hash = Some(&source_file.src_hash); - let file_name = Some(source_file.name.to_string()); + let file_name = Some(source_file.name.prefer_remapped().to_string()); let directory = if source_file.is_real_file() && !source_file.is_imported() { - Some(cx.sess().working_dir.0.to_string_lossy().to_string()) + Some(cx.sess().working_dir.to_string_lossy(false).to_string()) } else { // If the path comes from an upstream crate we assume it has been made // independent of the compiler's working directory one way or another. @@ -992,11 +993,12 @@ pub fn compile_unit_metadata( let producer = format!("clang LLVM ({})", rustc_producer); let name_in_debuginfo = name_in_debuginfo.to_string_lossy(); - let work_dir = tcx.sess.working_dir.0.to_string_lossy(); + let work_dir = tcx.sess.working_dir.to_string_lossy(false); let flags = "\0"; - let out_dir = &tcx.output_filenames(LOCAL_CRATE).out_directory; + let output_filenames = tcx.output_filenames(()); + let out_dir = &output_filenames.out_directory; let split_name = if tcx.sess.target_can_use_split_dwarf() { - tcx.output_filenames(LOCAL_CRATE) + output_filenames .split_dwarf_path(tcx.sess.split_debuginfo(), Some(codegen_unit_name)) .map(|f| out_dir.join(f)) } else { @@ -1057,15 +1059,12 @@ pub fn compile_unit_metadata( if tcx.sess.opts.debugging_opts.profile { let cu_desc_metadata = llvm::LLVMRustMetadataAsValue(debug_context.llcontext, unit_metadata); - let default_gcda_path = &tcx.output_filenames(LOCAL_CRATE).with_extension("gcda"); + let default_gcda_path = &output_filenames.with_extension("gcda"); let gcda_path = tcx.sess.opts.debugging_opts.profile_emit.as_ref().unwrap_or(default_gcda_path); let gcov_cu_info = [ - path_to_mdstring( - debug_context.llcontext, - &tcx.output_filenames(LOCAL_CRATE).with_extension("gcno"), - ), + path_to_mdstring(debug_context.llcontext, &output_filenames.with_extension("gcno")), path_to_mdstring(debug_context.llcontext, &gcda_path), cu_desc_metadata, ]; @@ -1458,7 +1457,7 @@ struct EnumMemberDescriptionFactory<'ll, 'tcx> { enum_type: Ty<'tcx>, layout: TyAndLayout<'tcx>, tag_type_metadata: Option<&'ll DIType>, - containing_scope: &'ll DIScope, + common_members: Vec>, span: Span, } @@ -1486,17 +1485,9 @@ impl EnumMemberDescriptionFactory<'ll, 'tcx> { _ => bug!(), }; - // This will always find the metadata in the type map. let fallback = use_enum_fallback(cx); - let self_metadata = if fallback { - self.containing_scope - } else { - type_metadata(cx, self.enum_type, self.span) - }; - let flags = match self.enum_type.kind() { - ty::Generator(..) => DIFlags::FlagArtificial, - _ => DIFlags::FlagZero, - }; + // This will always find the metadata in the type map. + let self_metadata = type_metadata(cx, self.enum_type, self.span); match self.layout.variants { Variants::Single { index } => { @@ -1511,7 +1502,7 @@ impl EnumMemberDescriptionFactory<'ll, 'tcx> { cx, self.layout, variant_info, - NoTag, + None, self_metadata, self.span, ); @@ -1523,6 +1514,7 @@ impl EnumMemberDescriptionFactory<'ll, 'tcx> { self.enum_type, variant_type_metadata, member_descriptions, + Some(&self.common_members), ); vec![MemberDescription { name: if fallback { String::new() } else { variant_info.variant_name() }, @@ -1530,7 +1522,7 @@ impl EnumMemberDescriptionFactory<'ll, 'tcx> { offset: Size::ZERO, size: self.layout.size, align: self.layout.align.abi, - flags, + flags: DIFlags::FlagZero, discriminant: None, source_info: variant_info.source_info(cx), }] @@ -1542,13 +1534,26 @@ impl EnumMemberDescriptionFactory<'ll, 'tcx> { .. } => { let tag_info = if fallback { - RegularTag { + // For MSVC, we generate a union of structs for each variant with an explicit + // discriminant field roughly equivalent to the following C: + // ```c + // union enum$<{name}> { + // struct {variant 0 name} { + // tag$ variant$; + // + // } variant0; + // + // } + // ``` + // The natvis in `intrinsic.nativs` then matches on `this.variant0.variant$` to + // determine which variant is active and then displays it. + Some(DirectTag { tag_field: Field::from(tag_field), tag_type_metadata: self.tag_type_metadata.unwrap(), - } + }) } else { // This doesn't matter in this case. - NoTag + None }; variants .iter_enumerated() @@ -1572,11 +1577,12 @@ impl EnumMemberDescriptionFactory<'ll, 'tcx> { self.enum_type, variant_type_metadata, member_descriptions, + Some(&self.common_members), ); MemberDescription { name: if fallback { - String::new() + format!("variant{}", i.as_u32()) } else { variant_info.variant_name() }, @@ -1584,7 +1590,7 @@ impl EnumMemberDescriptionFactory<'ll, 'tcx> { offset: Size::ZERO, size: self.layout.size, align: self.layout.align.abi, - flags, + flags: DIFlags::FlagZero, discriminant: Some( self.layout.ty.discriminant_for_variant(cx.tcx, i).unwrap().val as u64, @@ -1601,76 +1607,135 @@ impl EnumMemberDescriptionFactory<'ll, 'tcx> { ref variants, tag_field, } => { + let calculate_niche_value = |i: VariantIdx| { + if i == dataful_variant { + None + } else { + let value = (i.as_u32() as u128) + .wrapping_sub(niche_variants.start().as_u32() as u128) + .wrapping_add(niche_start); + let value = tag.value.size(cx).truncate(value); + // NOTE(eddyb) do *NOT* remove this assert, until + // we pass the full 128-bit value to LLVM, otherwise + // truncation will be silent and remain undetected. + assert_eq!(value as u64 as u128, value); + Some(value as u64) + } + }; + + // For MSVC, we will generate a union of two fields, one for the dataful variant + // and one that just points to the discriminant. We also create an enum that + // contains tag values for the non-dataful variants and make the discriminant field + // that type. We then use natvis to render the enum type correctly in Windbg/VS. + // This will generate debuginfo roughly equivalent to the following C: + // ```c + // union enum$<{name}, {min niche}, {max niche}, {dataful variant name}> { + // struct { + // + // } dataful_variant; + // enum Discriminant$ { + // + // } discriminant; + // } + // ``` + // The natvis in `intrinsic.natvis` matches on the type name `enum$<*, *, *, *>` + // and evaluates `this.discriminant`. If the value is between the min niche and max + // niche, then the enum is in the dataful variant and `this.dataful_variant` is + // rendered. Otherwise, the enum is in one of the non-dataful variants. In that + // case, we just need to render the name of the `this.discriminant` enum. if fallback { - let variant = self.layout.for_variant(cx, dataful_variant); - // Create a description of the non-null variant. - let (variant_type_metadata, member_description_factory) = describe_enum_variant( + let dataful_variant_layout = self.layout.for_variant(cx, dataful_variant); + + let mut discr_enum_ty = tag.value.to_ty(cx.tcx); + // If the niche is the NULL value of a reference, then `discr_enum_ty` will be a RawPtr. + // CodeView doesn't know what to do with enums whose base type is a pointer so we fix this up + // to just be `usize`. + if let ty::RawPtr(_) = discr_enum_ty.kind() { + discr_enum_ty = cx.tcx.types.usize; + } + + let tags: Vec<_> = variants + .iter_enumerated() + .filter_map(|(variant_idx, _)| { + calculate_niche_value(variant_idx).map(|tag| { + let variant = variant_info_for(variant_idx); + let name = variant.variant_name(); + + Some(unsafe { + llvm::LLVMRustDIBuilderCreateEnumerator( + DIB(cx), + name.as_ptr().cast(), + name.len(), + tag as i64, + !discr_enum_ty.is_signed(), + ) + }) + }) + }) + .collect(); + + let discr_enum = unsafe { + llvm::LLVMRustDIBuilderCreateEnumerationType( + DIB(cx), + self_metadata, + "Discriminant$".as_ptr().cast(), + "Discriminant$".len(), + unknown_file_metadata(cx), + UNKNOWN_LINE_NUMBER, + tag.value.size(cx).bits(), + tag.value.align(cx).abi.bits() as u32, + create_DIArray(DIB(cx), &tags), + type_metadata(cx, discr_enum_ty, self.span), + true, + ) + }; + + let variant_info = variant_info_for(dataful_variant); + let (variant_type_metadata, member_desc_factory) = describe_enum_variant( cx, - variant, - variant_info_for(dataful_variant), - OptimizedTag, - self.containing_scope, + dataful_variant_layout, + variant_info, + Some(NicheTag), + self_metadata, self.span, ); - let variant_member_descriptions = - member_description_factory.create_member_descriptions(cx); + let member_descriptions = member_desc_factory.create_member_descriptions(cx); set_members_of_composite_type( cx, self.enum_type, variant_type_metadata, - variant_member_descriptions, + member_descriptions, + Some(&self.common_members), ); - // Encode the information about the null variant in the union - // member's name. - let mut name = String::from("RUST$ENCODED$ENUM$"); - // Right now it's not even going to work for `niche_start > 0`, - // and for multiple niche variants it only supports the first. - fn compute_field_path<'a, 'tcx>( - cx: &CodegenCx<'a, 'tcx>, - name: &mut String, - layout: TyAndLayout<'tcx>, - offset: Size, - size: Size, - ) { - for i in 0..layout.fields.count() { - let field_offset = layout.fields.offset(i); - if field_offset > offset { - continue; - } - let inner_offset = offset - field_offset; - let field = layout.field(cx, i); - if inner_offset + size <= field.size { - write!(name, "{}$", i).unwrap(); - compute_field_path(cx, name, field, inner_offset, size); - } - } - } - compute_field_path( - cx, - &mut name, - self.layout, - self.layout.fields.offset(tag_field), - self.layout.field(cx, tag_field).size, - ); - let variant_info = variant_info_for(*niche_variants.start()); - variant_info.map_struct_name(|variant_name| { - name.push_str(variant_name); - }); - - // Create the (singleton) list of descriptions of union members. - vec![MemberDescription { - name, - type_metadata: variant_type_metadata, - offset: Size::ZERO, - size: variant.size, - align: variant.align.abi, - flags, - discriminant: None, - source_info: variant_info.source_info(cx), - }] + let (size, align) = + cx.size_and_align_of(dataful_variant_layout.field(cx, tag_field).ty); + + vec![ + MemberDescription { + // Name the dataful variant so that we can identify it for natvis + name: "dataful_variant".to_string(), + type_metadata: variant_type_metadata, + offset: Size::ZERO, + size: self.layout.size, + align: self.layout.align.abi, + flags: DIFlags::FlagZero, + discriminant: None, + source_info: variant_info.source_info(cx), + }, + MemberDescription { + name: "discriminant".into(), + type_metadata: discr_enum, + offset: dataful_variant_layout.fields.offset(tag_field), + size, + align, + flags: DIFlags::FlagZero, + discriminant: None, + source_info: None, + }, + ] } else { variants .iter_enumerated() @@ -1682,7 +1747,7 @@ impl EnumMemberDescriptionFactory<'ll, 'tcx> { cx, variant, variant_info, - OptimizedTag, + Some(NicheTag), self_metadata, self.span, ); @@ -1695,21 +1760,10 @@ impl EnumMemberDescriptionFactory<'ll, 'tcx> { self.enum_type, variant_type_metadata, member_descriptions, + Some(&self.common_members), ); - let niche_value = if i == dataful_variant { - None - } else { - let value = (i.as_u32() as u128) - .wrapping_sub(niche_variants.start().as_u32() as u128) - .wrapping_add(niche_start); - let value = tag.value.size(cx).truncate(value); - // NOTE(eddyb) do *NOT* remove this assert, until - // we pass the full 128-bit value to LLVM, otherwise - // truncation will be silent and remain undetected. - assert_eq!(value as u64 as u128, value); - Some(value as u64) - }; + let niche_value = calculate_niche_value(i); MemberDescription { name: variant_info.variant_name(), @@ -1717,7 +1771,7 @@ impl EnumMemberDescriptionFactory<'ll, 'tcx> { offset: Size::ZERO, size: self.layout.size, align: self.layout.align.abi, - flags, + flags: DIFlags::FlagZero, discriminant: niche_value, source_info: variant_info.source_info(cx), } @@ -1771,14 +1825,10 @@ impl VariantMemberDescriptionFactory<'ll, 'tcx> { } } -// FIXME: terminology here should be aligned with `abi::TagEncoding`. -// `OptimizedTag` is `TagEncoding::Niche`, `RegularTag` is `TagEncoding::Direct`. -// `NoTag` should be removed; users should use `Option` instead. #[derive(Copy, Clone)] enum EnumTagInfo<'ll> { - RegularTag { tag_field: Field, tag_type_metadata: &'ll DIType }, - OptimizedTag, - NoTag, + DirectTag { tag_field: Field, tag_type_metadata: &'ll DIType }, + NicheTag, } #[derive(Copy, Clone)] @@ -1849,13 +1899,6 @@ impl<'tcx> VariantInfo<'_, 'tcx> { } None } - - fn is_artificial(&self) -> bool { - match self { - VariantInfo::Generator { .. } => true, - VariantInfo::Adt(..) => false, - } - } } /// Returns a tuple of (1) `type_metadata_stub` of the variant, (2) a @@ -1866,7 +1909,7 @@ fn describe_enum_variant( cx: &CodegenCx<'ll, 'tcx>, layout: layout::TyAndLayout<'tcx>, variant: VariantInfo<'_, 'tcx>, - discriminant_info: EnumTagInfo<'ll>, + discriminant_info: Option>, containing_scope: &'ll DIScope, span: Span, ) -> (&'ll DICompositeType, MemberDescriptionFactory<'ll, 'tcx>) { @@ -1881,8 +1924,7 @@ fn describe_enum_variant( &variant_name, unique_type_id, Some(containing_scope), - // FIXME(tmandry): This doesn't seem to have any effect. - if variant.is_artificial() { DIFlags::FlagArtificial } else { DIFlags::FlagZero }, + DIFlags::FlagZero, ) }); @@ -1890,12 +1932,11 @@ fn describe_enum_variant( let (offsets, args) = if use_enum_fallback(cx) { // If this is not a univariant enum, there is also the discriminant field. let (discr_offset, discr_arg) = match discriminant_info { - RegularTag { tag_field, .. } => { + Some(DirectTag { tag_field, .. }) => { // We have the layout of an enum variant, we need the layout of the outer enum let enum_layout = cx.layout_of(layout.ty); let offset = enum_layout.fields.offset(tag_field.as_usize()); - let args = - ("RUST$ENUM$DISR".to_owned(), enum_layout.field(cx, tag_field.as_usize()).ty); + let args = ("variant$".to_owned(), enum_layout.field(cx, tag_field.as_usize()).ty); (Some(offset), Some(args)) } _ => (None, None), @@ -1926,7 +1967,7 @@ fn describe_enum_variant( offsets, args, tag_type_metadata: match discriminant_info { - RegularTag { tag_type_metadata, .. } => Some(tag_type_metadata), + Some(DirectTag { tag_type_metadata, .. }) => Some(tag_type_metadata), _ => None, }, span, @@ -1945,11 +1986,6 @@ fn prepare_enum_metadata( ) -> RecursiveTypeDescription<'ll, 'tcx> { let tcx = cx.tcx; let enum_name = compute_debuginfo_type_name(tcx, enum_type, false); - // FIXME(tmandry): This doesn't seem to have any effect. - let enum_flags = match enum_type.kind() { - ty::Generator(..) => DIFlags::FlagArtificial, - _ => DIFlags::FlagZero, - }; let containing_scope = get_namespace_for_item(cx, enum_def_id); // FIXME: This should emit actual file metadata for the enum, but we @@ -2061,9 +2097,9 @@ fn prepare_enum_metadata( if use_enum_fallback(cx) { let discriminant_type_metadata = match layout.variants { - Variants::Single { .. } - | Variants::Multiple { tag_encoding: TagEncoding::Niche { .. }, .. } => None, - Variants::Multiple { tag_encoding: TagEncoding::Direct, ref tag, .. } => { + Variants::Single { .. } => None, + Variants::Multiple { tag_encoding: TagEncoding::Niche { .. }, ref tag, .. } + | Variants::Multiple { tag_encoding: TagEncoding::Direct, ref tag, .. } => { Some(discriminant_type_metadata(tag.value)) } }; @@ -2075,14 +2111,14 @@ fn prepare_enum_metadata( unsafe { llvm::LLVMRustDIBuilderCreateUnionType( DIB(cx), - containing_scope, + None, enum_name.as_ptr().cast(), enum_name.len(), file_metadata, UNKNOWN_LINE_NUMBER, layout.size.bits(), layout.align.abi.bits() as u32, - enum_flags, + DIFlags::FlagZero, None, 0, // RuntimeLang unique_type_id_str.as_ptr().cast(), @@ -2101,7 +2137,7 @@ fn prepare_enum_metadata( enum_type, layout, tag_type_metadata: discriminant_type_metadata, - containing_scope, + common_members: vec![], span, }), ); @@ -2171,7 +2207,7 @@ fn prepare_enum_metadata( } }; - let mut outer_fields = match layout.variants { + let outer_fields = match layout.variants { Variants::Single { .. } => vec![], Variants::Multiple { .. } => { let tuple_mdf = TupleMemberDescriptionFactory { @@ -2203,18 +2239,21 @@ fn prepare_enum_metadata( UNKNOWN_LINE_NUMBER, layout.size.bits(), layout.align.abi.bits() as u32, - enum_flags, + DIFlags::FlagZero, discriminator_metadata, empty_array, variant_part_unique_type_id_str.as_ptr().cast(), variant_part_unique_type_id_str.len(), ) }; - outer_fields.push(Some(variant_part)); let struct_wrapper = { // The variant part must be wrapped in a struct according to DWARF. - let type_array = create_DIArray(DIB(cx), &outer_fields); + // All fields except the discriminant (including `outer_fields`) + // should be put into structures inside the variant part, which gives + // an equivalent layout but offers us much better integration with + // debuggers. + let type_array = create_DIArray(DIB(cx), &[Some(variant_part)]); let type_map = debug_context(cx).type_map.borrow(); let unique_type_id_str = type_map.get_unique_type_id_as_string(unique_type_id); @@ -2229,7 +2268,7 @@ fn prepare_enum_metadata( UNKNOWN_LINE_NUMBER, layout.size.bits(), layout.align.abi.bits() as u32, - enum_flags, + DIFlags::FlagZero, None, type_array, 0, @@ -2250,7 +2289,7 @@ fn prepare_enum_metadata( enum_type, layout, tag_type_metadata: None, - containing_scope, + common_members: outer_fields, span, }), ) @@ -2283,7 +2322,13 @@ fn composite_type_metadata( DIFlags::FlagZero, ); // ... and immediately create and add the member descriptions. - set_members_of_composite_type(cx, composite_type, composite_type_metadata, member_descriptions); + set_members_of_composite_type( + cx, + composite_type, + composite_type_metadata, + member_descriptions, + None, + ); composite_type_metadata } @@ -2293,6 +2338,7 @@ fn set_members_of_composite_type( composite_type: Ty<'tcx>, composite_type_metadata: &'ll DICompositeType, member_descriptions: Vec>, + common_members: Option<&Vec>>, ) { // In some rare cases LLVM metadata uniquing would lead to an existing type // description being used instead of a new one created in @@ -2311,10 +2357,13 @@ fn set_members_of_composite_type( } } - let member_metadata: Vec<_> = member_descriptions + let mut member_metadata: Vec<_> = member_descriptions .into_iter() .map(|desc| Some(desc.into_metadata(cx, composite_type_metadata))) .collect(); + if let Some(other_members) = common_members { + member_metadata.extend(other_members.iter()); + } let type_params = compute_type_parameters(cx, composite_type); unsafe { @@ -2435,7 +2484,7 @@ fn create_union_stub( llvm::LLVMRustDIBuilderCreateUnionType( DIB(cx), - containing_scope, + Some(containing_scope), union_type_name.as_ptr().cast(), union_type_name.len(), unknown_file_metadata(cx), diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs b/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs index b928e90373..2b99a2ebad 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs @@ -23,7 +23,7 @@ use rustc_codegen_ssa::mir::debuginfo::{DebugScope, FunctionDebugContext, Variab use rustc_codegen_ssa::traits::*; use rustc_data_structures::fx::{FxHashMap, FxHashSet}; use rustc_data_structures::sync::Lrc; -use rustc_hir::def_id::{DefId, DefIdMap, LOCAL_CRATE}; +use rustc_hir::def_id::{DefId, DefIdMap}; use rustc_index::vec::IndexVec; use rustc_middle::mir; use rustc_middle::ty::layout::HasTyCtxt; @@ -343,7 +343,7 @@ impl DebugInfoMethods<'tcx> for CodegenCx<'ll, 'tcx> { if self.sess().opts.optimize != config::OptLevel::No { spflags |= DISPFlags::SPFlagOptimized; } - if let Some((id, _)) = self.tcx.entry_fn(LOCAL_CRATE) { + if let Some((id, _)) = self.tcx.entry_fn(()) { if id == def_id { spflags |= DISPFlags::SPFlagMainSubprogram; } diff --git a/compiler/rustc_codegen_llvm/src/intrinsic.rs b/compiler/rustc_codegen_llvm/src/intrinsic.rs index fc6c1abf4a..22d513d66d 100644 --- a/compiler/rustc_codegen_llvm/src/intrinsic.rs +++ b/compiler/rustc_codegen_llvm/src/intrinsic.rs @@ -678,7 +678,8 @@ fn gen_fn<'ll, 'tcx>( cx.apply_target_cpu_attr(llfn); // FIXME(eddyb) find a nicer way to do this. unsafe { llvm::LLVMRustSetLinkage(llfn, llvm::Linkage::InternalLinkage) }; - let bx = Builder::new_block(cx, llfn, "entry-block"); + let llbb = Builder::append_block(cx, llfn, "entry-block"); + let bx = Builder::build(cx, llbb); codegen(bx); llfn } diff --git a/compiler/rustc_codegen_llvm/src/lib.rs b/compiler/rustc_codegen_llvm/src/lib.rs index 5ca4b226c3..776cb2ee99 100644 --- a/compiler/rustc_codegen_llvm/src/lib.rs +++ b/compiler/rustc_codegen_llvm/src/lib.rs @@ -8,12 +8,11 @@ #![feature(bool_to_option)] #![feature(const_cstr_unchecked)] #![feature(crate_visibility_modifier)] -#![feature(extended_key_value_attributes)] +#![cfg_attr(bootstrap, feature(extended_key_value_attributes))] #![feature(extern_types)] #![feature(in_band_lifetimes)] #![feature(iter_zip)] #![feature(nll)] -#![cfg_attr(bootstrap, feature(or_patterns))] #![recursion_limit = "256"] use back::write::{create_informational_target_machine, create_target_machine}; @@ -30,8 +29,8 @@ use rustc_codegen_ssa::{CodegenResults, CompiledModule}; use rustc_data_structures::fx::FxHashMap; use rustc_errors::{ErrorReported, FatalError, Handler}; use rustc_middle::dep_graph::{WorkProduct, WorkProductId}; -use rustc_middle::middle::cstore::{EncodedMetadata, MetadataLoaderDyn}; -use rustc_middle::ty::{self, TyCtxt}; +use rustc_middle::middle::cstore::EncodedMetadata; +use rustc_middle::ty::TyCtxt; use rustc_session::config::{OptLevel, OutputFilenames, PrintRequest}; use rustc_session::Session; use rustc_span::symbol::Symbol; @@ -69,7 +68,6 @@ pub mod llvm { } mod llvm_util; -mod metadata; mod mono_item; mod type_; mod type_of; @@ -162,7 +160,7 @@ impl WriteBackendMethods for LlvmCodegenBackend { module: &ModuleCodegen, config: &ModuleConfig, ) -> Result<(), FatalError> { - Ok(back::write::optimize(cgcx, diag_handler, module, config)) + back::write::optimize(cgcx, diag_handler, module, config) } unsafe fn optimize_thin( cgcx: &CodegenContext, @@ -189,8 +187,9 @@ impl WriteBackendMethods for LlvmCodegenBackend { module: &ModuleCodegen, config: &ModuleConfig, thin: bool, - ) { - back::lto::run_pass_manager(cgcx, module, config, thin) + ) -> Result<(), FatalError> { + let diag_handler = cgcx.create_diag_handler(); + back::lto::run_pass_manager(cgcx, &diag_handler, module, config, thin) } } @@ -249,18 +248,6 @@ impl CodegenBackend for LlvmCodegenBackend { target_features(sess) } - fn metadata_loader(&self) -> Box { - Box::new(metadata::LlvmMetadataLoader) - } - - fn provide(&self, providers: &mut ty::query::Providers) { - attributes::provide_both(providers); - } - - fn provide_extern(&self, providers: &mut ty::query::Providers) { - attributes::provide_both(providers); - } - fn codegen_crate<'tcx>( &self, tcx: TyCtxt<'tcx>, @@ -270,6 +257,7 @@ impl CodegenBackend for LlvmCodegenBackend { Box::new(rustc_codegen_ssa::base::codegen_crate( LlvmCodegenBackend(()), tcx, + crate::llvm_util::target_cpu(tcx.sess).to_string(), metadata, need_metadata_module, )) @@ -305,13 +293,11 @@ impl CodegenBackend for LlvmCodegenBackend { // Run the linker on any artifacts that resulted from the LLVM run. // This should produce either a finished executable or library. - let target_cpu = crate::llvm_util::target_cpu(sess); link_binary::>( sess, &codegen_results, outputs, - &codegen_results.crate_name.as_str(), - target_cpu, + &codegen_results.crate_info.local_crate_name.as_str(), ); Ok(()) diff --git a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs index 32b1526f6e..9192325101 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs @@ -29,6 +29,31 @@ pub enum LLVMRustResult { Success, Failure, } + +// Rust version of the C struct with the same name in rustc_llvm/llvm-wrapper/RustWrapper.cpp. +#[repr(C)] +pub struct LLVMRustCOFFShortExport { + pub name: *const c_char, +} + +impl LLVMRustCOFFShortExport { + pub fn from_name(name: *const c_char) -> LLVMRustCOFFShortExport { + LLVMRustCOFFShortExport { name } + } +} + +/// Translation of LLVM's MachineTypes enum, defined in llvm\include\llvm\BinaryFormat\COFF.h. +/// +/// We include only architectures supported on Windows. +#[derive(Copy, Clone, PartialEq)] +#[repr(C)] +pub enum LLVMMachineType { + AMD64 = 0x8664, + I386 = 0x14c, + ARM64 = 0xaa64, + ARM = 0x01c0, +} + // Consts for the LLVM CallConv type, pre-cast to usize. /// LLVM CallingConv::ID. Should we wrap this? @@ -54,7 +79,7 @@ pub enum CallConv { } /// LLVMRustLinkage -#[derive(PartialEq)] +#[derive(Copy, Clone, PartialEq)] #[repr(C)] pub enum Linkage { ExternalLinkage = 0, @@ -72,6 +97,7 @@ pub enum Linkage { // LLVMRustVisibility #[repr(C)] +#[derive(Copy, Clone, PartialEq)] pub enum Visibility { Default = 0, Hidden = 1, @@ -581,11 +607,6 @@ pub struct PassManager<'a>(InvariantOpaque<'a>); extern "C" { pub type PassManagerBuilder; } -extern "C" { - pub type ObjectFile; -} -#[repr(C)] -pub struct SectionIterator<'a>(InvariantOpaque<'a>); extern "C" { pub type Pass; } @@ -1034,6 +1055,7 @@ extern "C" { pub fn LLVMDeleteGlobal(GlobalVar: &Value); pub fn LLVMGetInitializer(GlobalVar: &Value) -> Option<&Value>; pub fn LLVMSetInitializer(GlobalVar: &'a Value, ConstantVal: &'a Value); + pub fn LLVMIsThreadLocal(GlobalVar: &Value) -> Bool; pub fn LLVMSetThreadLocal(GlobalVar: &Value, IsThreadLocal: Bool); pub fn LLVMSetThreadLocalMode(GlobalVar: &Value, Mode: ThreadLocalMode); pub fn LLVMIsGlobalConstant(GlobalVar: &Value) -> Bool; @@ -1079,7 +1101,6 @@ extern "C" { Fn: &'a Value, Name: *const c_char, ) -> &'a BasicBlock; - pub fn LLVMDeleteBasicBlock(BB: &BasicBlock); // Operations on instructions pub fn LLVMIsAInstruction(Val: &Value) -> Option<&Value>; @@ -1702,35 +1723,6 @@ extern "C" { pub fn LLVMDisposeMessage(message: *mut c_char); - // Stuff that's in llvm-wrapper/ because it's not upstream yet. - - /// Opens an object file. - pub fn LLVMCreateObjectFile( - MemBuf: &'static mut MemoryBuffer, - ) -> Option<&'static mut ObjectFile>; - /// Closes an object file. - pub fn LLVMDisposeObjectFile(ObjFile: &'static mut ObjectFile); - - /// Enumerates the sections in an object file. - pub fn LLVMGetSections(ObjFile: &'a ObjectFile) -> &'a mut SectionIterator<'a>; - /// Destroys a section iterator. - pub fn LLVMDisposeSectionIterator(SI: &'a mut SectionIterator<'a>); - /// Returns `true` if the section iterator is at the end of the section - /// list: - pub fn LLVMIsSectionIteratorAtEnd(ObjFile: &'a ObjectFile, SI: &SectionIterator<'a>) -> Bool; - /// Moves the section iterator to point to the next section. - pub fn LLVMMoveToNextSection(SI: &SectionIterator<'_>); - /// Returns the current section size. - pub fn LLVMGetSectionSize(SI: &SectionIterator<'_>) -> c_ulonglong; - /// Returns the current section contents as a string buffer. - pub fn LLVMGetSectionContents(SI: &SectionIterator<'_>) -> *const c_char; - - /// Reads the given file and returns it as a memory buffer. Use - /// LLVMDisposeMemoryBuffer() to get rid of it. - pub fn LLVMRustCreateMemoryBufferWithContentsOfFile( - Path: *const c_char, - ) -> Option<&'static mut MemoryBuffer>; - pub fn LLVMStartMultithreaded() -> Bool; /// Returns a string describing the last error caused by an LLVMRust* call. @@ -2037,7 +2029,7 @@ extern "C" { pub fn LLVMRustDIBuilderCreateUnionType( Builder: &DIBuilder<'a>, - Scope: &'a DIScope, + Scope: Option<&'a DIScope>, Name: *const c_char, NameLen: size_t, File: &'a DIFile, @@ -2203,10 +2195,14 @@ extern "C" { SanitizerOptions: Option<&SanitizerOptions>, PGOGenPath: *const c_char, PGOUsePath: *const c_char, + InstrumentCoverage: bool, + InstrumentGCOV: bool, llvm_selfprofiler: *mut c_void, begin_callback: SelfProfileBeforePassCallback, end_callback: SelfProfileAfterPassCallback, - ); + ExtraPasses: *const c_char, + ExtraPassesLen: size_t, + ) -> LLVMRustResult; pub fn LLVMRustPrintModule( M: &'a Module, Output: *const c_char, @@ -2231,12 +2227,6 @@ extern "C" { pub fn LLVMRustArchiveIteratorFree(AIR: &'a mut ArchiveIterator<'a>); pub fn LLVMRustDestroyArchive(AR: &'static mut Archive); - #[allow(improper_ctypes)] - pub fn LLVMRustGetSectionName( - SI: &SectionIterator<'_>, - data: &mut Option>, - ) -> size_t; - #[allow(improper_ctypes)] pub fn LLVMRustWriteTwineToString(T: &Twine, s: &RustString); @@ -2300,6 +2290,15 @@ extern "C" { ) -> &'a mut RustArchiveMember<'a>; pub fn LLVMRustArchiveMemberFree(Member: &'a mut RustArchiveMember<'a>); + pub fn LLVMRustWriteImportLibrary( + ImportName: *const c_char, + Path: *const c_char, + Exports: *const LLVMRustCOFFShortExport, + NumExports: usize, + Machine: u16, + MinGW: bool, + ) -> LLVMRustResult; + pub fn LLVMRustSetDataLayoutFromTargetMachine(M: &'a Module, TM: &'a TargetMachine); pub fn LLVMRustBuildOperandBundleDef( diff --git a/compiler/rustc_codegen_llvm/src/llvm/mod.rs b/compiler/rustc_codegen_llvm/src/llvm/mod.rs index bb9c6d4737..38d56f8721 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/mod.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/mod.rs @@ -150,50 +150,6 @@ impl Attribute { } } -// Memory-managed interface to object files. - -pub struct ObjectFile { - pub llof: &'static mut ffi::ObjectFile, -} - -unsafe impl Send for ObjectFile {} - -impl ObjectFile { - // This will take ownership of llmb - pub fn new(llmb: &'static mut MemoryBuffer) -> Option { - unsafe { - let llof = LLVMCreateObjectFile(llmb)?; - Some(ObjectFile { llof }) - } - } -} - -impl Drop for ObjectFile { - fn drop(&mut self) { - unsafe { - LLVMDisposeObjectFile(&mut *(self.llof as *mut _)); - } - } -} - -// Memory-managed interface to section iterators. - -pub struct SectionIter<'a> { - pub llsi: &'a mut SectionIterator<'a>, -} - -impl Drop for SectionIter<'a> { - fn drop(&mut self) { - unsafe { - LLVMDisposeSectionIterator(&mut *(self.llsi as *mut _)); - } - } -} - -pub fn mk_section_iter(llof: &ffi::ObjectFile) -> SectionIter<'_> { - unsafe { SectionIter { llsi: LLVMGetSections(llof) } } -} - pub fn set_section(llglobal: &Value, section_name: &str) { let section_name_cstr = CString::new(section_name).expect("unexpected CString error"); unsafe { diff --git a/compiler/rustc_codegen_llvm/src/llvm_util.rs b/compiler/rustc_codegen_llvm/src/llvm_util.rs index 97684ca6c1..0dd3d2ae15 100644 --- a/compiler/rustc_codegen_llvm/src/llvm_util.rs +++ b/compiler/rustc_codegen_llvm/src/llvm_util.rs @@ -163,6 +163,12 @@ pub fn to_llvm_feature<'a>(sess: &Session, s: &'a str) -> &'a str { ("x86", "avx512vpclmulqdq") => "vpclmulqdq", ("aarch64", "fp") => "fp-armv8", ("aarch64", "fp16") => "fullfp16", + ("aarch64", "fhm") => "fp16fml", + ("aarch64", "rcpc2") => "rcpc-immo", + ("aarch64", "dpb") => "ccpp", + ("aarch64", "dpb2") => "ccdp", + ("aarch64", "frintts") => "fptoint", + ("aarch64", "fcma") => "complxnum", (_, s) => s, } } @@ -350,24 +356,32 @@ pub fn llvm_global_features(sess: &Session) -> Vec { Some(_) | None => {} }; + let filter = |s: &str| { + if s.is_empty() { + return None; + } + let feature = if s.starts_with("+") || s.starts_with("-") { + &s[1..] + } else { + return Some(s.to_string()); + }; + // Rustc-specific feature requests like `+crt-static` or `-crt-static` + // are not passed down to LLVM. + if RUSTC_SPECIFIC_FEATURES.contains(&feature) { + return None; + } + // ... otherwise though we run through `to_llvm_feature` feature when + // passing requests down to LLVM. This means that all in-language + // features also work on the command line instead of having two + // different names when the LLVM name and the Rust name differ. + Some(format!("{}{}", &s[..1], to_llvm_feature(sess, feature))) + }; + // Features implied by an implicit or explicit `--target`. - features.extend( - sess.target - .features - .split(',') - .filter(|f| !f.is_empty() && !RUSTC_SPECIFIC_FEATURES.iter().any(|s| f.contains(s))) - .map(String::from), - ); + features.extend(sess.target.features.split(',').filter_map(&filter)); // -Ctarget-features - features.extend( - sess.opts - .cg - .target_feature - .split(',') - .filter(|f| !f.is_empty() && !RUSTC_SPECIFIC_FEATURES.iter().any(|s| f.contains(s))) - .map(String::from), - ); + features.extend(sess.opts.cg.target_feature.split(',').filter_map(&filter)); features } diff --git a/compiler/rustc_codegen_llvm/src/metadata.rs b/compiler/rustc_codegen_llvm/src/metadata.rs deleted file mode 100644 index b007df5730..0000000000 --- a/compiler/rustc_codegen_llvm/src/metadata.rs +++ /dev/null @@ -1,112 +0,0 @@ -use crate::llvm; -use crate::llvm::archive_ro::ArchiveRO; -use crate::llvm::{mk_section_iter, False, ObjectFile}; -use rustc_middle::middle::cstore::MetadataLoader; -use rustc_target::spec::Target; - -use rustc_codegen_ssa::METADATA_FILENAME; -use rustc_data_structures::owning_ref::OwningRef; -use rustc_data_structures::rustc_erase_owner; -use tracing::debug; - -use rustc_fs_util::path_to_c_string; -use std::path::Path; -use std::slice; - -pub use rustc_data_structures::sync::MetadataRef; - -pub struct LlvmMetadataLoader; - -impl MetadataLoader for LlvmMetadataLoader { - fn get_rlib_metadata(&self, _: &Target, filename: &Path) -> Result { - // Use ArchiveRO for speed here, it's backed by LLVM and uses mmap - // internally to read the file. We also avoid even using a memcpy by - // just keeping the archive along while the metadata is in use. - let archive = - ArchiveRO::open(filename).map(|ar| OwningRef::new(Box::new(ar))).map_err(|e| { - debug!("llvm didn't like `{}`: {}", filename.display(), e); - format!("failed to read rlib metadata in '{}': {}", filename.display(), e) - })?; - let buf: OwningRef<_, [u8]> = archive.try_map(|ar| { - ar.iter() - .filter_map(|s| s.ok()) - .find(|sect| sect.name() == Some(METADATA_FILENAME)) - .map(|s| s.data()) - .ok_or_else(|| { - debug!("didn't find '{}' in the archive", METADATA_FILENAME); - format!("failed to read rlib metadata: '{}'", filename.display()) - }) - })?; - Ok(rustc_erase_owner!(buf)) - } - - fn get_dylib_metadata(&self, target: &Target, filename: &Path) -> Result { - unsafe { - let buf = path_to_c_string(filename); - let mb = llvm::LLVMRustCreateMemoryBufferWithContentsOfFile(buf.as_ptr()) - .ok_or_else(|| format!("error reading library: '{}'", filename.display()))?; - let of = - ObjectFile::new(mb).map(|of| OwningRef::new(Box::new(of))).ok_or_else(|| { - format!("provided path not an object file: '{}'", filename.display()) - })?; - let buf = of.try_map(|of| search_meta_section(of, target, filename))?; - Ok(rustc_erase_owner!(buf)) - } - } -} - -fn search_meta_section<'a>( - of: &'a ObjectFile, - target: &Target, - filename: &Path, -) -> Result<&'a [u8], String> { - unsafe { - let si = mk_section_iter(of.llof); - while llvm::LLVMIsSectionIteratorAtEnd(of.llof, si.llsi) == False { - let mut name_buf = None; - let name_len = llvm::LLVMRustGetSectionName(si.llsi, &mut name_buf); - let name = name_buf.map_or_else( - String::new, // We got a NULL ptr, ignore `name_len`. - |buf| { - String::from_utf8( - slice::from_raw_parts(buf.as_ptr() as *const u8, name_len as usize) - .to_vec(), - ) - .unwrap() - }, - ); - debug!("get_metadata_section: name {}", name); - if read_metadata_section_name(target) == name { - let cbuf = llvm::LLVMGetSectionContents(si.llsi); - let csz = llvm::LLVMGetSectionSize(si.llsi) as usize; - // The buffer is valid while the object file is around - let buf: &'a [u8] = slice::from_raw_parts(cbuf as *const u8, csz); - return Ok(buf); - } - llvm::LLVMMoveToNextSection(si.llsi); - } - } - Err(format!("metadata not found: '{}'", filename.display())) -} - -pub fn metadata_section_name(target: &Target) -> &'static str { - // Historical note: - // - // When using link.exe it was seen that the section name `.note.rustc` - // was getting shortened to `.note.ru`, and according to the PE and COFF - // specification: - // - // > Executable images do not use a string table and do not support - // > section names longer than 8 characters - // - // https://docs.microsoft.com/en-us/windows/win32/debug/pe-format - // - // As a result, we choose a slightly shorter name! As to why - // `.note.rustc` works on MinGW, that's another good question... - - if target.is_like_osx { "__DATA,.rustc" } else { ".rustc" } -} - -fn read_metadata_section_name(_target: &Target) -> &'static str { - ".rustc" -} diff --git a/compiler/rustc_codegen_llvm/src/mono_item.rs b/compiler/rustc_codegen_llvm/src/mono_item.rs index fc1f364e9c..93456443aa 100644 --- a/compiler/rustc_codegen_llvm/src/mono_item.rs +++ b/compiler/rustc_codegen_llvm/src/mono_item.rs @@ -37,7 +37,7 @@ impl PreDefineMethods<'tcx> for CodegenCx<'ll, 'tcx> { unsafe { llvm::LLVMRustSetLinkage(g, base::linkage_to_llvm(linkage)); llvm::LLVMRustSetVisibility(g, base::visibility_to_llvm(visibility)); - if self.should_assume_dso_local(linkage, visibility) { + if self.should_assume_dso_local(g, false) { llvm::LLVMRustSetDSOLocal(g, true); } } @@ -85,7 +85,7 @@ impl PreDefineMethods<'tcx> for CodegenCx<'ll, 'tcx> { attributes::from_fn_attrs(self, lldecl, instance); unsafe { - if self.should_assume_dso_local(linkage, visibility) { + if self.should_assume_dso_local(lldecl, false) { llvm::LLVMRustSetDSOLocal(lldecl, true); } } @@ -95,28 +95,48 @@ impl PreDefineMethods<'tcx> for CodegenCx<'ll, 'tcx> { } impl CodegenCx<'ll, 'tcx> { - /// Whether a definition (NB: not declaration!) can be assumed to be local to a group of + /// Whether a definition or declaration can be assumed to be local to a group of /// libraries that form a single DSO or executable. pub(crate) unsafe fn should_assume_dso_local( &self, - linkage: Linkage, - visibility: Visibility, + llval: &llvm::Value, + is_declaration: bool, ) -> bool { - if matches!(linkage, Linkage::Internal | Linkage::Private) { + let linkage = llvm::LLVMRustGetLinkage(llval); + let visibility = llvm::LLVMRustGetVisibility(llval); + + if matches!(linkage, llvm::Linkage::InternalLinkage | llvm::Linkage::PrivateLinkage) { return true; } - if visibility != Visibility::Default && linkage != Linkage::ExternalWeak { + if visibility != llvm::Visibility::Default && linkage != llvm::Linkage::ExternalWeakLinkage + { return true; } - // Static relocation model should force copy relocations everywhere. - if self.tcx.sess.relocation_model() == RelocModel::Static { + // Symbols from executables can't really be imported any further. + let all_exe = self.tcx.sess.crate_types().iter().all(|ty| *ty == CrateType::Executable); + let is_declaration_for_linker = + is_declaration || linkage == llvm::Linkage::AvailableExternallyLinkage; + if all_exe && !is_declaration_for_linker { return true; } - // Symbols from executables can't really be imported any further. - if self.tcx.sess.crate_types().iter().all(|ty| *ty == CrateType::Executable) { + // PowerPC64 prefers TOC indirection to avoid copy relocations. + if matches!(&*self.tcx.sess.target.arch, "powerpc64" | "powerpc64le") { + return false; + } + + // Thread-local variables generally don't support copy relocations. + let is_thread_local_var = llvm::LLVMIsAGlobalVariable(llval) + .map(|v| llvm::LLVMIsThreadLocal(v) == llvm::True) + .unwrap_or(false); + if is_thread_local_var { + return false; + } + + // Static relocation model should force copy relocations everywhere. + if self.tcx.sess.relocation_model() == RelocModel::Static { return true; } diff --git a/compiler/rustc_codegen_ssa/Cargo.toml b/compiler/rustc_codegen_ssa/Cargo.toml index 7a3d715df6..9bd5764f07 100644 --- a/compiler/rustc_codegen_ssa/Cargo.toml +++ b/compiler/rustc_codegen_ssa/Cargo.toml @@ -9,7 +9,7 @@ test = false [dependencies] bitflags = "1.2.1" -cc = "1.0.1" +cc = "1.0.68" itertools = "0.9" tracing = "0.1" libc = "0.2.50" @@ -24,7 +24,7 @@ rustc_middle = { path = "../rustc_middle" } rustc_apfloat = { path = "../rustc_apfloat" } rustc_attr = { path = "../rustc_attr" } rustc_symbol_mangling = { path = "../rustc_symbol_mangling" } -rustc_data_structures = { path = "../rustc_data_structures"} +rustc_data_structures = { path = "../rustc_data_structures" } rustc_errors = { path = "../rustc_errors" } rustc_fs_util = { path = "../rustc_fs_util" } rustc_hir = { path = "../rustc_hir" } @@ -33,3 +33,8 @@ rustc_index = { path = "../rustc_index" } rustc_macros = { path = "../rustc_macros" } rustc_target = { path = "../rustc_target" } rustc_session = { path = "../rustc_session" } + +[dependencies.object] +version = "0.25.2" +default-features = false +features = ["read_core", "elf", "macho", "pe", "unaligned", "archive", "write"] diff --git a/compiler/rustc_codegen_ssa/src/back/archive.rs b/compiler/rustc_codegen_ssa/src/back/archive.rs index c477ac6462..63f457bb97 100644 --- a/compiler/rustc_codegen_ssa/src/back/archive.rs +++ b/compiler/rustc_codegen_ssa/src/back/archive.rs @@ -1,14 +1,24 @@ +use rustc_data_structures::temp_dir::MaybeTempDir; +use rustc_middle::middle::cstore::DllImport; use rustc_session::Session; use rustc_span::symbol::Symbol; use std::io; use std::path::{Path, PathBuf}; -pub fn find_library(name: Symbol, search_paths: &[PathBuf], sess: &Session) -> PathBuf { +pub fn find_library( + name: Symbol, + verbatim: bool, + search_paths: &[PathBuf], + sess: &Session, +) -> PathBuf { // On Windows, static libraries sometimes show up as libfoo.a and other // times show up as foo.lib - let oslibname = - format!("{}{}{}", sess.target.staticlib_prefix, name, sess.target.staticlib_suffix); + let oslibname = if verbatim { + name.to_string() + } else { + format!("{}{}{}", sess.target.staticlib_prefix, name, sess.target.staticlib_suffix) + }; let unixlibname = format!("lib{}.a", name); for path in search_paths { @@ -45,8 +55,15 @@ pub trait ArchiveBuilder<'a> { lto: bool, skip_objects: bool, ) -> io::Result<()>; - fn add_native_library(&mut self, name: Symbol); + fn add_native_library(&mut self, name: Symbol, verbatim: bool); fn update_symbols(&mut self); fn build(self); + + fn inject_dll_import_lib( + &mut self, + lib_name: &str, + dll_imports: &[DllImport], + tmpdir: &MaybeTempDir, + ); } diff --git a/compiler/rustc_codegen_ssa/src/back/link.rs b/compiler/rustc_codegen_ssa/src/back/link.rs index ea75943d6f..6c9ec9e7b0 100644 --- a/compiler/rustc_codegen_ssa/src/back/link.rs +++ b/compiler/rustc_codegen_ssa/src/back/link.rs @@ -1,11 +1,11 @@ -use rustc_data_structures::fx::FxHashSet; +use rustc_data_structures::fx::{FxHashMap, FxHashSet}; use rustc_data_structures::temp_dir::MaybeTempDir; use rustc_errors::Handler; use rustc_fs_util::fix_windows_verbatim_for_gcc; use rustc_hir::def_id::CrateNum; -use rustc_middle::middle::cstore::{EncodedMetadata, LibSource}; +use rustc_middle::middle::cstore::{DllImport, LibSource}; use rustc_middle::middle::dependency_format::Linkage; -use rustc_session::config::{self, CFGuard, CrateType, DebugInfo}; +use rustc_session::config::{self, CFGuard, CrateType, DebugInfo, LdImpl, Strip}; use rustc_session::config::{OutputFilenames, OutputType, PrintRequest}; use rustc_session::output::{check_file_is_writeable, invalid_output_for_target, out_filename}; use rustc_session::search_paths::PathKind; @@ -14,6 +14,7 @@ use rustc_session::utils::NativeLibKind; /// need out of the shared crate context before we get rid of it. use rustc_session::{filesearch, Session}; use rustc_span::symbol::Symbol; +use rustc_target::abi::Endian; use rustc_target::spec::crt_objects::{CrtObjects, CrtObjectsFallback}; use rustc_target::spec::{LinkOutputKind, LinkerFlavor, LldFlavor, SplitDebuginfo}; use rustc_target::spec::{PanicStrategy, RelocModel, RelroLevel, SanitizerSet, Target}; @@ -28,8 +29,12 @@ use crate::{ }; use cc::windows_registry; +use object::elf; +use object::write::Object; +use object::{Architecture, BinaryFormat, Endianness, FileFlags, SectionFlags, SectionKind}; use tempfile::Builder as TempFileBuilder; +use std::cmp::Ordering; use std::ffi::OsString; use std::path::{Path, PathBuf}; use std::process::{ExitStatus, Output, Stdio}; @@ -50,7 +55,6 @@ pub fn link_binary<'a, B: ArchiveBuilder<'a>>( codegen_results: &CodegenResults, outputs: &OutputFilenames, crate_name: &str, - target_cpu: &str, ) { let _timer = sess.timer("link_binary"); let output_metadata = sess.opts.output_types.contains_key(&OutputType::Metadata); @@ -100,7 +104,6 @@ pub fn link_binary<'a, B: ArchiveBuilder<'a>>( &out_filename, codegen_results, path.as_ref(), - target_cpu, ); } } @@ -280,9 +283,9 @@ pub fn each_linked_rlib( /// building an `.rlib` (stomping over one another), or writing an `.rmeta` into a /// directory being searched for `extern crate` (observing an incomplete file). /// The returned path is the temporary file containing the complete metadata. -pub fn emit_metadata(sess: &Session, metadata: &EncodedMetadata, tmpdir: &MaybeTempDir) -> PathBuf { +pub fn emit_metadata(sess: &Session, metadata: &[u8], tmpdir: &MaybeTempDir) -> PathBuf { let out_filename = tmpdir.as_ref().join(METADATA_FILENAME); - let result = fs::write(&out_filename, &metadata.raw_data); + let result = fs::write(&out_filename, metadata); if let Err(e) = result { sess.fatal(&format!("failed to write {}: {}", out_filename.display(), e)); @@ -329,18 +332,24 @@ fn link_rlib<'a, B: ArchiveBuilder<'a>>( // metadata of the rlib we're generating somehow. for lib in codegen_results.crate_info.used_libraries.iter() { match lib.kind { - NativeLibKind::StaticBundle => {} - NativeLibKind::StaticNoBundle - | NativeLibKind::Dylib - | NativeLibKind::Framework + NativeLibKind::Static { bundle: None | Some(true), .. } => {} + NativeLibKind::Static { bundle: Some(false), .. } + | NativeLibKind::Dylib { .. } + | NativeLibKind::Framework { .. } | NativeLibKind::RawDylib | NativeLibKind::Unspecified => continue, } if let Some(name) = lib.name { - ab.add_native_library(name); + ab.add_native_library(name, lib.verbatim.unwrap_or(false)); } } + for (raw_dylib_name, raw_dylib_imports) in + collate_raw_dylibs(&codegen_results.crate_info.used_libraries) + { + ab.inject_dll_import_lib(&raw_dylib_name, &raw_dylib_imports, tmpdir); + } + // After adding all files to the archive, we need to update the // symbol table of the archive. ab.update_symbols(); @@ -368,9 +377,11 @@ fn link_rlib<'a, B: ArchiveBuilder<'a>>( // code above. match flavor { RlibFlavor::Normal => { - // Instead of putting the metadata in an object file section, rlibs - // contain the metadata in a separate file. - ab.add_file(&emit_metadata(sess, &codegen_results.metadata, tmpdir)); + // metadata in rlib files is wrapped in a "dummy" object file for + // the target platform so the rlib can be processed entirely by + // normal linkers for the platform. + let metadata = create_metadata_file(sess, &codegen_results.metadata.raw_data); + ab.add_file(&emit_metadata(sess, &metadata, tmpdir)); // After adding all files to the archive, we need to update the // symbol table of the archive. This currently dies on macOS (see @@ -387,8 +398,188 @@ fn link_rlib<'a, B: ArchiveBuilder<'a>>( } } } + return ab; + + // For rlibs we "pack" rustc metadata into a dummy object file. When rustc + // creates a dylib crate type it will pass `--whole-archive` (or the + // platform equivalent) to include all object files from an rlib into the + // final dylib itself. This causes linkers to iterate and try to include all + // files located in an archive, so if metadata is stored in an archive then + // it needs to be of a form that the linker will be able to process. + // + // Note, though, that we don't actually want this metadata to show up in any + // final output of the compiler. Instead this is purely for rustc's own + // metadata tracking purposes. + // + // With the above in mind, each "flavor" of object format gets special + // handling here depending on the target: + // + // * MachO - macos-like targets will insert the metadata into a section that + // is sort of fake dwarf debug info. Inspecting the source of the macos + // linker this causes these sections to be skipped automatically because + // it's not in an allowlist of otherwise well known dwarf section names to + // go into the final artifact. + // + // * WebAssembly - we actually don't have any container format for this + // target. WebAssembly doesn't support the `dylib` crate type anyway so + // there's no need for us to support this at this time. Consequently the + // metadata bytes are simply stored as-is into an rlib. + // + // * COFF - Windows-like targets create an object with a section that has + // the `IMAGE_SCN_LNK_REMOVE` flag set which ensures that if the linker + // ever sees the section it doesn't process it and it's removed. + // + // * ELF - All other targets are similar to Windows in that there's a + // `SHF_EXCLUDE` flag we can set on sections in an object file to get + // automatically removed from the final output. + // + // Note that this metdata format is kept in sync with + // `rustc_codegen_ssa/src/back/metadata.rs`. + fn create_metadata_file(sess: &Session, metadata: &[u8]) -> Vec { + let endianness = match sess.target.options.endian { + Endian::Little => Endianness::Little, + Endian::Big => Endianness::Big, + }; + let architecture = match &sess.target.arch[..] { + "arm" => Architecture::Arm, + "aarch64" => Architecture::Aarch64, + "x86" => Architecture::I386, + "s390x" => Architecture::S390x, + "mips" => Architecture::Mips, + "mips64" => Architecture::Mips64, + "x86_64" => { + if sess.target.pointer_width == 32 { + Architecture::X86_64_X32 + } else { + Architecture::X86_64 + } + } + "powerpc" => Architecture::PowerPc, + "powerpc64" => Architecture::PowerPc64, + "riscv32" => Architecture::Riscv32, + "riscv64" => Architecture::Riscv64, + "sparc64" => Architecture::Sparc64, + + // This is used to handle all "other" targets. This includes targets + // in two categories: + // + // * Some targets don't have support in the `object` crate just yet + // to write an object file. These targets are likely to get filled + // out over time. + // + // * Targets like WebAssembly don't support dylibs, so the purpose + // of putting metadata in object files, to support linking rlibs + // into dylibs, is moot. + // + // In both of these cases it means that linking into dylibs will + // not be supported by rustc. This doesn't matter for targets like + // WebAssembly and for targets not supported by the `object` crate + // yet it means that work will need to be done in the `object` crate + // to add a case above. + _ => return metadata.to_vec(), + }; + + if sess.target.is_like_osx { + let mut file = Object::new(BinaryFormat::MachO, architecture, endianness); + + let section = + file.add_section(b"__DWARF".to_vec(), b".rmeta".to_vec(), SectionKind::Debug); + file.append_section_data(section, metadata, 1); + file.write().unwrap() + } else if sess.target.is_like_windows { + const IMAGE_SCN_LNK_REMOVE: u32 = 0; + let mut file = Object::new(BinaryFormat::Coff, architecture, endianness); + + let section = file.add_section(Vec::new(), b".rmeta".to_vec(), SectionKind::Debug); + file.section_mut(section).flags = + SectionFlags::Coff { characteristics: IMAGE_SCN_LNK_REMOVE }; + file.append_section_data(section, metadata, 1); + file.write().unwrap() + } else { + const SHF_EXCLUDE: u64 = 0x80000000; + let mut file = Object::new(BinaryFormat::Elf, architecture, endianness); + + match &sess.target.arch[..] { + // copied from `mipsel-linux-gnu-gcc foo.c -c` and + // inspecting the resulting `e_flags` field. + "mips" => { + let e_flags = elf::EF_MIPS_ARCH_32R2 | elf::EF_MIPS_CPIC | elf::EF_MIPS_PIC; + file.flags = FileFlags::Elf { e_flags }; + } + // copied from `mips64el-linux-gnuabi64-gcc foo.c -c` + "mips64" => { + let e_flags = elf::EF_MIPS_ARCH_64R2 | elf::EF_MIPS_CPIC | elf::EF_MIPS_PIC; + file.flags = FileFlags::Elf { e_flags }; + } + + // copied from `riscv64-linux-gnu-gcc foo.c -c`, note though + // that the `+d` target feature represents whether the double + // float abi is enabled. + "riscv64" if sess.target.options.features.contains("+d") => { + let e_flags = elf::EF_RISCV_RVC | elf::EF_RISCV_FLOAT_ABI_DOUBLE; + file.flags = FileFlags::Elf { e_flags }; + } + + _ => {} + } + + let section = file.add_section(Vec::new(), b".rmeta".to_vec(), SectionKind::Debug); + file.section_mut(section).flags = SectionFlags::Elf { sh_flags: SHF_EXCLUDE }; + file.append_section_data(section, metadata, 1); + file.write().unwrap() + } + } +} + +/// Extract all symbols defined in raw-dylib libraries, collated by library name. +/// +/// If we have multiple extern blocks that specify symbols defined in the same raw-dylib library, +/// then the CodegenResults value contains one NativeLib instance for each block. However, the +/// linker appears to expect only a single import library for each library used, so we need to +/// collate the symbols together by library name before generating the import libraries. +fn collate_raw_dylibs(used_libraries: &[NativeLib]) -> Vec<(String, Vec)> { + let mut dylib_table: FxHashMap> = FxHashMap::default(); + + for lib in used_libraries { + if lib.kind == NativeLibKind::RawDylib { + let name = lib.name.unwrap_or_else(|| + bug!("`link` attribute with kind = \"raw-dylib\" and no name should have caused error earlier") + ); + let name = if matches!(lib.verbatim, Some(true)) { + name.to_string() + } else { + format!("{}.dll", name) + }; + dylib_table + .entry(name) + .or_default() + .extend(lib.dll_imports.iter().map(|import| import.name)); + } + } - ab + // FIXME: when we add support for ordinals, fix this to propagate ordinals. Also figure out + // what we should do if we have two DllImport values with the same name but different + // ordinals. + let mut result = dylib_table + .into_iter() + .map(|(lib_name, imported_names)| { + let mut names = imported_names + .iter() + .map(|name| DllImport { name: *name, ordinal: None }) + .collect::>(); + names.sort_unstable_by(|a: &DllImport, b: &DllImport| { + match a.name.as_str().cmp(&b.name.as_str()) { + Ordering::Equal => a.ordinal.cmp(&b.ordinal), + x => x, + } + }); + (lib_name, names) + }) + .collect::>(); + result.sort_unstable_by(|a: &(String, Vec), b: &(String, Vec)| { + a.0.cmp(&b.0) + }); + result } /// Create a static archive. @@ -430,9 +621,10 @@ fn link_staticlib<'a, B: ArchiveBuilder<'a>>( // Clearly this is not sufficient for a general purpose feature, and // we'd want to read from the library's metadata to determine which // object files come from where and selectively skip them. - let skip_object_files = native_libs - .iter() - .any(|lib| lib.kind == NativeLibKind::StaticBundle && !relevant_lib(sess, lib)); + let skip_object_files = native_libs.iter().any(|lib| { + matches!(lib.kind, NativeLibKind::Static { bundle: None | Some(true), .. }) + && !relevant_lib(sess, lib) + }); ab.add_rlib( path, &name.as_str(), @@ -531,7 +723,6 @@ fn link_natively<'a, B: ArchiveBuilder<'a>>( out_filename: &Path, codegen_results: &CodegenResults, tmpdir: &Path, - target_cpu: &str, ) { info!("preparing {:?} to {:?}", crate_type, out_filename); let (linker_path, flavor) = linker_and_flavor(sess); @@ -543,7 +734,6 @@ fn link_natively<'a, B: ArchiveBuilder<'a>>( tmpdir, out_filename, codegen_results, - target_cpu, ); linker::disable_localization(&mut cmd); @@ -717,14 +907,6 @@ fn link_natively<'a, B: ArchiveBuilder<'a>>( } } - fn escape_string(s: &[u8]) -> String { - str::from_utf8(s).map(|s| s.to_owned()).unwrap_or_else(|_| { - let mut x = "Non-UTF-8 output: ".to_string(); - x.extend(s.iter().flat_map(|&b| ascii::escape_default(b)).map(char::from)); - x - }) - } - match prog { Ok(prog) => { if !prog.status.success() { @@ -866,9 +1048,50 @@ fn link_natively<'a, B: ArchiveBuilder<'a>>( // ... and otherwise we're processing a `*.dwp` packed dwarf file. SplitDebuginfo::Packed => link_dwarf_object(sess, &out_filename), } + + if sess.target.is_like_osx { + if let Some(option) = osx_strip_opt(sess.opts.debugging_opts.strip) { + strip_symbols_in_osx(sess, &out_filename, option); + } + } +} + +fn strip_symbols_in_osx<'a>(sess: &'a Session, out_filename: &Path, option: &str) { + let prog = Command::new("strip").arg(option).arg(out_filename).output(); + match prog { + Ok(prog) => { + if !prog.status.success() { + let mut output = prog.stderr.clone(); + output.extend_from_slice(&prog.stdout); + sess.struct_warn(&format!( + "stripping debug info with `strip` failed: {}", + prog.status + )) + .note(&escape_string(&output)) + .emit(); + } + } + Err(e) => sess.fatal(&format!("unable to run `strip`: {}", e)), + } +} + +fn osx_strip_opt<'a>(strip: Strip) -> Option<&'a str> { + match strip { + Strip::Debuginfo => Some("-S"), + Strip::Symbols => Some("-x"), + Strip::None => None, + } +} + +fn escape_string(s: &[u8]) -> String { + str::from_utf8(s).map(|s| s.to_owned()).unwrap_or_else(|_| { + let mut x = "Non-UTF-8 output: ".to_string(); + x.extend(s.iter().flat_map(|&b| ascii::escape_default(b)).map(char::from)); + x + }) } -fn link_sanitizers(sess: &Session, crate_type: CrateType, linker: &mut dyn Linker) { +fn add_sanitizer_libraries(sess: &Session, crate_type: CrateType, linker: &mut dyn Linker) { // On macOS the runtimes are distributed as dylibs which should be linked to // both executables and dynamic shared objects. Everywhere else the runtimes // are currently distributed as static liraries which should be linked to @@ -931,7 +1154,7 @@ fn link_sanitizer_runtime(sess: &Session, linker: &mut dyn Linker, name: &str) { let path = find_sanitizer_runtime(&sess, &filename); let rpath = path.to_str().expect("non-utf8 component in path"); linker.args(&["-Wl,-rpath", "-Xlinker", rpath]); - linker.link_dylib(Symbol::intern(&filename)); + linker.link_dylib(Symbol::intern(&filename), false, true); } else { let filename = format!("librustc{}_rt.{}.a", channel, name); let path = find_sanitizer_runtime(&sess, &filename).join(&filename); @@ -957,7 +1180,8 @@ pub fn ignored_for_lto(sess: &Session, info: &CrateInfo, cnum: CrateNum) -> bool && (info.compiler_builtins == Some(cnum) || info.is_no_builtins.contains(&cnum)) } -fn linker_and_flavor(sess: &Session) -> (PathBuf, LinkerFlavor) { +// This functions tries to determine the appropriate linker (and corresponding LinkerFlavor) to use +pub fn linker_and_flavor(sess: &Session) -> (PathBuf, LinkerFlavor) { fn infer_from( sess: &Session, linker: Option, @@ -992,6 +1216,7 @@ fn linker_and_flavor(sess: &Session) -> (PathBuf, LinkerFlavor) { LinkerFlavor::Msvc => "link.exe", LinkerFlavor::Lld(_) => "lld", LinkerFlavor::PtxLinker => "rust-ptx-linker", + LinkerFlavor::BpfLinker => "bpf-linker", }), flavor, )), @@ -1008,6 +1233,8 @@ fn linker_and_flavor(sess: &Session) -> (PathBuf, LinkerFlavor) { || stem.ends_with("-clang") { LinkerFlavor::Gcc + } else if stem == "wasm-ld" || stem.ends_with("-wasm-ld") { + LinkerFlavor::Lld(LldFlavor::Wasm) } else if stem == "ld" || stem == "ld.lld" || stem.ends_with("-ld") { LinkerFlavor::Ld } else if stem == "link" || stem == "lld-link" { @@ -1080,21 +1307,25 @@ fn print_native_static_libs(sess: &Session, all_native_libs: &[NativeLib]) { .filter_map(|lib| { let name = lib.name?; match lib.kind { - NativeLibKind::StaticNoBundle - | NativeLibKind::Dylib + NativeLibKind::Static { bundle: Some(false), .. } + | NativeLibKind::Dylib { .. } | NativeLibKind::Unspecified => { + let verbatim = lib.verbatim.unwrap_or(false); if sess.target.is_like_msvc { - Some(format!("{}.lib", name)) + Some(format!("{}{}", name, if verbatim { "" } else { ".lib" })) + } else if sess.target.linker_is_gnu { + Some(format!("-l{}{}", if verbatim { ":" } else { "" }, name)) } else { Some(format!("-l{}", name)) } } - NativeLibKind::Framework => { + NativeLibKind::Framework { .. } => { // ld-only syntax, since there are no frameworks in MSVC Some(format!("-framework {}", name)) } // These are included, no need to print them - NativeLibKind::StaticBundle | NativeLibKind::RawDylib => None, + NativeLibKind::Static { bundle: None | Some(true), .. } + | NativeLibKind::RawDylib => None, } }) .collect(); @@ -1483,55 +1714,6 @@ fn add_local_crate_metadata_objects( } } -/// Link native libraries corresponding to the current crate and all libraries corresponding to -/// all its dependency crates. -/// FIXME: Consider combining this with the functions above adding object files for the local crate. -fn link_local_crate_native_libs_and_dependent_crate_libs<'a, B: ArchiveBuilder<'a>>( - cmd: &mut dyn Linker, - sess: &'a Session, - crate_type: CrateType, - codegen_results: &CodegenResults, - tmpdir: &Path, -) { - // Take careful note of the ordering of the arguments we pass to the linker - // here. Linkers will assume that things on the left depend on things to the - // right. Things on the right cannot depend on things on the left. This is - // all formally implemented in terms of resolving symbols (libs on the right - // resolve unknown symbols of libs on the left, but not vice versa). - // - // For this reason, we have organized the arguments we pass to the linker as - // such: - // - // 1. The local object that LLVM just generated - // 2. Local native libraries - // 3. Upstream rust libraries - // 4. Upstream native libraries - // - // The rationale behind this ordering is that those items lower down in the - // list can't depend on items higher up in the list. For example nothing can - // depend on what we just generated (e.g., that'd be a circular dependency). - // Upstream rust libraries are not allowed to depend on our local native - // libraries as that would violate the structure of the DAG, in that - // scenario they are required to link to them as well in a shared fashion. - // - // Note that upstream rust libraries may contain native dependencies as - // well, but they also can't depend on what we just started to add to the - // link line. And finally upstream native libraries can't depend on anything - // in this DAG so far because they're only dylibs and dylibs can only depend - // on other dylibs (e.g., other native deps). - // - // If -Zlink-native-libraries=false is set, then the assumption is that an - // external build system already has the native dependencies defined, and it - // will provide them to the linker itself. - if sess.opts.debugging_opts.link_native_libraries { - add_local_native_libraries(cmd, sess, codegen_results); - } - add_upstream_rust_crates::(cmd, sess, codegen_results, crate_type, tmpdir); - if sess.opts.debugging_opts.link_native_libraries { - add_upstream_native_libraries(cmd, sess, codegen_results, crate_type); - } -} - /// Add sysroot and other globally set directories to the directory search list. fn add_library_search_dirs(cmd: &mut dyn Linker, sess: &Session, self_contained: bool) { // The default library location, we need this to find the runtime. @@ -1566,36 +1748,27 @@ fn add_rpath_args( ) { // FIXME (#2397): At some point we want to rpath our guesses as to // where extern libraries might live, based on the - // addl_lib_search_paths + // add_lib_search_paths if sess.opts.cg.rpath { - let target_triple = sess.opts.target_triple.triple(); - let mut get_install_prefix_lib_path = || { - let install_prefix = option_env!("CFG_PREFIX").expect("CFG_PREFIX"); - let tlib = filesearch::relative_target_lib_path(&sess.sysroot, target_triple); - let mut path = PathBuf::from(install_prefix); - path.push(&tlib); - - path - }; let mut rpath_config = RPathConfig { used_crates: &codegen_results.crate_info.used_crates_dynamic, out_filename: out_filename.to_path_buf(), has_rpath: sess.target.has_rpath, is_like_osx: sess.target.is_like_osx, linker_is_gnu: sess.target.linker_is_gnu, - get_install_prefix_lib_path: &mut get_install_prefix_lib_path, }; cmd.args(&rpath::get_rpath_flags(&mut rpath_config)); } } /// Produce the linker command line containing linker path and arguments. -/// `NO-OPT-OUT` marks the arguments that cannot be removed from the command line -/// by the user without creating a custom target specification. -/// `OBJECT-FILES` specify whether the arguments can add object files. -/// `CUSTOMIZATION-POINT` means that arbitrary arguments defined by the user -/// or by the target spec can be inserted here. -/// `AUDIT-ORDER` - need to figure out whether the option is order-dependent or not. +/// +/// When comments in the function say "order-(in)dependent" they mean order-dependence between +/// options and libraries/object files. For example `--whole-archive` (order-dependent) applies +/// to specific libraries passed after it, and `-o` (output file, order-independent) applies +/// to the linking process as a whole. +/// Order-independent options may still override each other in order-dependent fashion, +/// e.g `--foo=yes --foo=no` may be equivalent to `--foo=no`. fn linker_with_args<'a, B: ArchiveBuilder<'a>>( path: &Path, flavor: LinkerFlavor, @@ -1604,26 +1777,162 @@ fn linker_with_args<'a, B: ArchiveBuilder<'a>>( tmpdir: &Path, out_filename: &Path, codegen_results: &CodegenResults, - target_cpu: &str, ) -> Command { let crt_objects_fallback = crt_objects_fallback(sess, crate_type); let base_cmd = get_linker(sess, path, flavor, crt_objects_fallback); // FIXME: Move `/LIBPATH` addition for uwp targets from the linker construction // to the linker args construction. assert!(base_cmd.get_args().is_empty() || sess.target.vendor == "uwp"); - let cmd = &mut *codegen_results.linker_info.to_linker(base_cmd, &sess, flavor, target_cpu); + let cmd = &mut *codegen_results.linker_info.to_linker(base_cmd, &sess, flavor); let link_output_kind = link_output_kind(sess, crate_type); - // NO-OPT-OUT, OBJECT-FILES-MAYBE, CUSTOMIZATION-POINT + // ------------ Early order-dependent options ------------ + + // If we're building something like a dynamic library then some platforms + // need to make sure that all symbols are exported correctly from the + // dynamic library. + // Must be passed before any libraries to prevent the symbols to export from being thrown away, + // at least on some platforms (e.g. windows-gnu). + cmd.export_symbols(tmpdir, crate_type); + + // Can be used for adding custom CRT objects or overriding order-dependent options above. + // FIXME: In practice built-in target specs use this for arbitrary order-independent options, + // introduce a target spec option for order-independent linker options and migrate built-in + // specs to it. add_pre_link_args(cmd, sess, flavor); - // NO-OPT-OUT, OBJECT-FILES-NO + // ------------ Object code and libraries, order-dependent ------------ + + // Pre-link CRT objects. + add_pre_link_objects(cmd, sess, link_output_kind, crt_objects_fallback); + + // Sanitizer libraries. + add_sanitizer_libraries(sess, crate_type, cmd); + + // Object code from the current crate. + // Take careful note of the ordering of the arguments we pass to the linker + // here. Linkers will assume that things on the left depend on things to the + // right. Things on the right cannot depend on things on the left. This is + // all formally implemented in terms of resolving symbols (libs on the right + // resolve unknown symbols of libs on the left, but not vice versa). + // + // For this reason, we have organized the arguments we pass to the linker as + // such: + // + // 1. The local object that LLVM just generated + // 2. Local native libraries + // 3. Upstream rust libraries + // 4. Upstream native libraries + // + // The rationale behind this ordering is that those items lower down in the + // list can't depend on items higher up in the list. For example nothing can + // depend on what we just generated (e.g., that'd be a circular dependency). + // Upstream rust libraries are not supposed to depend on our local native + // libraries as that would violate the structure of the DAG, in that + // scenario they are required to link to them as well in a shared fashion. + // (The current implementation still doesn't prevent it though, see the FIXME below.) + // + // Note that upstream rust libraries may contain native dependencies as + // well, but they also can't depend on what we just started to add to the + // link line. And finally upstream native libraries can't depend on anything + // in this DAG so far because they can only depend on other native libraries + // and such dependencies are also required to be specified. + add_local_crate_regular_objects(cmd, codegen_results); + add_local_crate_metadata_objects(cmd, crate_type, codegen_results); + add_local_crate_allocator_objects(cmd, codegen_results); + + // Avoid linking to dynamic libraries unless they satisfy some undefined symbols + // at the point at which they are specified on the command line. + // Must be passed before any (dynamic) libraries to have effect on them. + // On Solaris-like systems, `-z ignore` acts as both `--as-needed` and `--gc-sections` + // so it will ignore unreferenced ELF sections from relocatable objects. + // For that reason, we put this flag after metadata objects as they would otherwise be removed. + // FIXME: Support more fine-grained dead code removal on Solaris/illumos + // and move this option back to the top. + cmd.add_as_needed(); + + // FIXME: Move this below to other native libraries + // (or alternatively link all native libraries after their respective crates). + // This change is somewhat breaking in practice due to local static libraries being linked + // as whole-archive (#85144), so removing whole-archive may be a pre-requisite. + if sess.opts.debugging_opts.link_native_libraries { + add_local_native_libraries(cmd, sess, codegen_results); + } + + // Rust libraries. + add_upstream_rust_crates::(cmd, sess, codegen_results, crate_type, tmpdir); + + // Native libraries linked with `#[link]` attributes at and `-l` command line options. + // If -Zlink-native-libraries=false is set, then the assumption is that an + // external build system already has the native dependencies defined, and it + // will provide them to the linker itself. + if sess.opts.debugging_opts.link_native_libraries { + add_upstream_native_libraries(cmd, sess, codegen_results, crate_type); + } + + // Library linking above uses some global state for things like `-Bstatic`/`-Bdynamic` to make + // command line shorter, reset it to default here before adding more libraries. + cmd.reset_per_library_state(); + + // FIXME: Built-in target specs occasionally use this for linking system libraries, + // eliminate all such uses by migrating them to `#[link]` attributes in `lib(std,c,unwind)` + // and remove the option. + add_late_link_args(cmd, sess, flavor, crate_type, codegen_results); + + // ------------ Arbitrary order-independent options ------------ + + // Add order-independent options determined by rustc from its compiler options, + // target properties and source code. + add_order_independent_options( + cmd, + sess, + link_output_kind, + crt_objects_fallback, + flavor, + crate_type, + codegen_results, + out_filename, + tmpdir, + ); + + // Can be used for arbitrary order-independent options. + // In practice may also be occasionally used for linking native libraries. + // Passed after compiler-generated options to support manual overriding when necessary. + add_user_defined_link_args(cmd, sess); + + // ------------ Object code and libraries, order-dependent ------------ + + // Post-link CRT objects. + add_post_link_objects(cmd, sess, link_output_kind, crt_objects_fallback); + + // ------------ Late order-dependent options ------------ + + // Doesn't really make sense. + // FIXME: In practice built-in target specs use this for arbitrary order-independent options, + // introduce a target spec option for order-independent linker options, migrate built-in specs + // to it and remove the option. + add_post_link_args(cmd, sess, flavor); + + cmd.take_cmd() +} + +fn add_order_independent_options( + cmd: &mut dyn Linker, + sess: &Session, + link_output_kind: LinkOutputKind, + crt_objects_fallback: bool, + flavor: LinkerFlavor, + crate_type: CrateType, + codegen_results: &CodegenResults, + out_filename: &Path, + tmpdir: &Path, +) { + add_gcc_ld_path(cmd, sess, flavor); + add_apple_sdk(cmd, sess, flavor); - // NO-OPT-OUT add_link_script(cmd, sess, tmpdir, crate_type); - // NO-OPT-OUT, OBJECT-FILES-NO, AUDIT-ORDER if sess.target.is_like_fuchsia && crate_type == CrateType::Executable { let prefix = if sess.opts.debugging_opts.sanitizer.contains(SanitizerSet::ADDRESS) { "asan/" @@ -1633,30 +1942,17 @@ fn linker_with_args<'a, B: ArchiveBuilder<'a>>( cmd.arg(format!("--dynamic-linker={}ld.so.1", prefix)); } - // NO-OPT-OUT, OBJECT-FILES-NO, AUDIT-ORDER if sess.target.eh_frame_header { cmd.add_eh_frame_header(); } - // NO-OPT-OUT, OBJECT-FILES-NO, AUDIT-ORDER // Make the binary compatible with data execution prevention schemes. cmd.add_no_exec(); - // NO-OPT-OUT, OBJECT-FILES-NO - // Avoid linking to dynamic libraries unless they satisfy some undefined symbols - // at the point at which they are specified on the command line. - // Must be passed before any dynamic libraries. - cmd.add_as_needed(); - - // NO-OPT-OUT, OBJECT-FILES-NO if crt_objects_fallback { cmd.no_crt_objects(); } - // NO-OPT-OUT, OBJECT-FILES-YES - add_pre_link_objects(cmd, sess, link_output_kind, crt_objects_fallback); - - // NO-OPT-OUT, OBJECT-FILES-NO, AUDIT-ORDER if sess.target.is_like_emscripten { cmd.arg("-s"); cmd.arg(if sess.panic_strategy() == PanicStrategy::Abort { @@ -1666,45 +1962,32 @@ fn linker_with_args<'a, B: ArchiveBuilder<'a>>( }); } - // OBJECT-FILES-YES, AUDIT-ORDER - link_sanitizers(sess, crate_type, cmd); + if flavor == LinkerFlavor::PtxLinker { + // Provide the linker with fallback to internal `target-cpu`. + cmd.arg("--fallback-arch"); + cmd.arg(&codegen_results.linker_info.target_cpu); + } else if flavor == LinkerFlavor::BpfLinker { + cmd.arg("--cpu"); + cmd.arg(&codegen_results.linker_info.target_cpu); + cmd.arg("--cpu-features"); + cmd.arg(match &sess.opts.cg.target_feature { + feat if !feat.is_empty() => feat, + _ => &sess.target.options.features, + }); + } - // OBJECT-FILES-NO, AUDIT-ORDER - // Linker plugins should be specified early in the list of arguments - // FIXME: How "early" exactly? cmd.linker_plugin_lto(); - // NO-OPT-OUT, OBJECT-FILES-NO, AUDIT-ORDER - // FIXME: Order-dependent, at least relatively to other args adding searh directories. add_library_search_dirs(cmd, sess, crt_objects_fallback); - // OBJECT-FILES-YES - add_local_crate_regular_objects(cmd, codegen_results); - - // NO-OPT-OUT, OBJECT-FILES-NO, AUDIT-ORDER cmd.output_filename(out_filename); - // OBJECT-FILES-NO, AUDIT-ORDER if crate_type == CrateType::Executable && sess.target.is_like_windows { - if let Some(ref s) = codegen_results.windows_subsystem { + if let Some(ref s) = codegen_results.crate_info.windows_subsystem { cmd.subsystem(s); } } - // NO-OPT-OUT, OBJECT-FILES-NO, AUDIT-ORDER - // If we're building something like a dynamic library then some platforms - // need to make sure that all symbols are exported correctly from the - // dynamic library. - cmd.export_symbols(tmpdir, crate_type); - - // OBJECT-FILES-YES - add_local_crate_metadata_objects(cmd, crate_type, codegen_results); - - // OBJECT-FILES-YES - add_local_crate_allocator_objects(cmd, codegen_results); - - // OBJECT-FILES-NO, AUDIT-ORDER - // FIXME: Order dependent, applies to the following objects. Where should it be placed? // Try to strip as much out of the generated object by removing unused // sections if possible. See more comments in linker.rs if !sess.link_dead_code() { @@ -1712,65 +1995,31 @@ fn linker_with_args<'a, B: ArchiveBuilder<'a>>( cmd.gc_sections(keep_metadata); } - // NO-OPT-OUT, OBJECT-FILES-NO, AUDIT-ORDER cmd.set_output_kind(link_output_kind, out_filename); - // OBJECT-FILES-NO, AUDIT-ORDER add_relro_args(cmd, sess); - // OBJECT-FILES-NO, AUDIT-ORDER // Pass optimization flags down to the linker. cmd.optimize(); - // OBJECT-FILES-NO, AUDIT-ORDER // Pass debuginfo and strip flags down to the linker. cmd.debuginfo(sess.opts.debugging_opts.strip); - // OBJECT-FILES-NO, AUDIT-ORDER // We want to prevent the compiler from accidentally leaking in any system libraries, // so by default we tell linkers not to link to any default libraries. if !sess.opts.cg.default_linker_libraries && sess.target.no_default_libraries { cmd.no_default_libraries(); } - // OBJECT-FILES-YES - link_local_crate_native_libs_and_dependent_crate_libs::( - cmd, - sess, - crate_type, - codegen_results, - tmpdir, - ); - - // OBJECT-FILES-NO, AUDIT-ORDER if sess.opts.cg.profile_generate.enabled() || sess.instrument_coverage() { cmd.pgo_gen(); } - // OBJECT-FILES-NO, AUDIT-ORDER if sess.opts.cg.control_flow_guard != CFGuard::Disabled { cmd.control_flow_guard(); } - // OBJECT-FILES-NO, AUDIT-ORDER add_rpath_args(cmd, sess, codegen_results, out_filename); - - // OBJECT-FILES-MAYBE, CUSTOMIZATION-POINT - add_user_defined_link_args(cmd, sess); - - // NO-OPT-OUT, OBJECT-FILES-NO, AUDIT-ORDER - cmd.finalize(); - - // NO-OPT-OUT, OBJECT-FILES-MAYBE, CUSTOMIZATION-POINT - add_late_link_args(cmd, sess, flavor, crate_type, codegen_results); - - // NO-OPT-OUT, OBJECT-FILES-YES - add_post_link_objects(cmd, sess, link_output_kind, crt_objects_fallback); - - // NO-OPT-OUT, OBJECT-FILES-MAYBE, CUSTOMIZATION-POINT - add_post_link_args(cmd, sess, flavor); - - cmd.take_cmd() } /// # Native library linking @@ -1803,16 +2052,30 @@ fn add_local_native_libraries( codegen_results.crate_info.used_libraries.iter().filter(|l| relevant_lib(sess, l)); let search_path = archive_search_paths(sess); + let mut last = (NativeLibKind::Unspecified, None); for lib in relevant_libs { let name = match lib.name { Some(l) => l, None => continue, }; + + // Skip if this library is the same as the last. + last = if (lib.kind, lib.name) == last { continue } else { (lib.kind, lib.name) }; + + let verbatim = lib.verbatim.unwrap_or(false); match lib.kind { - NativeLibKind::Dylib | NativeLibKind::Unspecified => cmd.link_dylib(name), - NativeLibKind::Framework => cmd.link_framework(name), - NativeLibKind::StaticNoBundle => cmd.link_staticlib(name), - NativeLibKind::StaticBundle => cmd.link_whole_staticlib(name, &search_path), + NativeLibKind::Dylib { as_needed } => { + cmd.link_dylib(name, verbatim, as_needed.unwrap_or(true)) + } + NativeLibKind::Unspecified => cmd.link_dylib(name, verbatim, true), + NativeLibKind::Framework { as_needed } => { + cmd.link_framework(name, as_needed.unwrap_or(true)) + } + NativeLibKind::Static { bundle: None | Some(true), .. } + | NativeLibKind::Static { whole_archive: Some(true), .. } => { + cmd.link_whole_staticlib(name, verbatim, &search_path); + } + NativeLibKind::Static { .. } => cmd.link_staticlib(name, verbatim), NativeLibKind::RawDylib => { // FIXME(#58713): Proper handling for raw dylibs. bug!("raw_dylib feature not yet implemented"); @@ -1950,11 +2213,8 @@ fn add_upstream_rust_crates<'a, B: ArchiveBuilder<'a>>( } // Adds the static "rlib" versions of all crates to the command line. - // There's a bit of magic which happens here specifically related to LTO and - // dynamic libraries. Specifically: - // - // * For LTO, we remove upstream object files. - // * For dylibs we remove metadata and bytecode from upstream rlibs + // There's a bit of magic which happens here specifically related to LTO, + // namely that we remove upstream object files. // // When performing LTO, almost(*) all of the bytecode from the upstream // libraries has already been included in our object file output. As a @@ -1967,20 +2227,9 @@ fn add_upstream_rust_crates<'a, B: ArchiveBuilder<'a>>( // their bytecode wasn't included. The object files in those libraries must // still be passed to the linker. // - // When making a dynamic library, linkers by default don't include any - // object files in an archive if they're not necessary to resolve the link. - // We basically want to convert the archive (rlib) to a dylib, though, so we - // *do* want everything included in the output, regardless of whether the - // linker thinks it's needed or not. As a result we must use the - // --whole-archive option (or the platform equivalent). When using this - // option the linker will fail if there are non-objects in the archive (such - // as our own metadata and/or bytecode). All in all, for rlibs to be - // entirely included in dylibs, we need to remove all non-object files. - // - // Note, however, that if we're not doing LTO or we're not producing a dylib - // (aka we're making an executable), we can just pass the rlib blindly to - // the linker (fast) because it's fine if it's not actually included as - // we're at the end of the dependency chain. + // Note, however, that if we're not doing LTO we can just pass the rlib + // blindly to the linker (fast) because it's fine if it's not actually + // included as we're at the end of the dependency chain. fn add_static_crate<'a, B: ArchiveBuilder<'a>>( cmd: &mut dyn Linker, sess: &'a Session, @@ -1992,20 +2241,38 @@ fn add_upstream_rust_crates<'a, B: ArchiveBuilder<'a>>( let src = &codegen_results.crate_info.used_crate_source[&cnum]; let cratepath = &src.rlib.as_ref().unwrap().0; + let mut link_upstream = |path: &Path| { + // If we're creating a dylib, then we need to include the + // whole of each object in our archive into that artifact. This is + // because a `dylib` can be reused as an intermediate artifact. + // + // Note, though, that we don't want to include the whole of a + // compiler-builtins crate (e.g., compiler-rt) because it'll get + // repeatedly linked anyway. + let path = fix_windows_verbatim_for_gcc(path); + if crate_type == CrateType::Dylib + && codegen_results.crate_info.compiler_builtins != Some(cnum) + { + cmd.link_whole_rlib(&path); + } else { + cmd.link_rlib(&path); + } + }; + // See the comment above in `link_staticlib` and `link_rlib` for why if // there's a static library that's not relevant we skip all object // files. let native_libs = &codegen_results.crate_info.native_libraries[&cnum]; - let skip_native = native_libs - .iter() - .any(|lib| lib.kind == NativeLibKind::StaticBundle && !relevant_lib(sess, lib)); + let skip_native = native_libs.iter().any(|lib| { + matches!(lib.kind, NativeLibKind::Static { bundle: None | Some(true), .. }) + && !relevant_lib(sess, lib) + }); if (!are_upstream_rust_objects_already_included(sess) || ignored_for_lto(sess, &codegen_results.crate_info, cnum)) - && crate_type != CrateType::Dylib && !skip_native { - cmd.link_rlib(&fix_windows_verbatim_for_gcc(cratepath)); + link_upstream(cratepath); return; } @@ -2055,21 +2322,7 @@ fn add_upstream_rust_crates<'a, B: ArchiveBuilder<'a>>( return; } archive.build(); - - // If we're creating a dylib, then we need to include the - // whole of each object in our archive into that artifact. This is - // because a `dylib` can be reused as an intermediate artifact. - // - // Note, though, that we don't want to include the whole of a - // compiler-builtins crate (e.g., compiler-rt) because it'll get - // repeatedly linked anyway. - if crate_type == CrateType::Dylib - && codegen_results.crate_info.compiler_builtins != Some(cnum) - { - cmd.link_whole_rlib(&fix_windows_verbatim_for_gcc(&dst)); - } else { - cmd.link_rlib(&fix_windows_verbatim_for_gcc(&dst)); - } + link_upstream(&dst); }); } @@ -2127,6 +2380,7 @@ fn add_upstream_native_libraries( .expect("failed to find crate type in dependency format list"); let crates = &codegen_results.crate_info.used_crates_static; + let mut last = (NativeLibKind::Unspecified, None); for &(cnum, _) in crates { for lib in codegen_results.crate_info.native_libraries[&cnum].iter() { let name = match lib.name { @@ -2136,26 +2390,33 @@ fn add_upstream_native_libraries( if !relevant_lib(sess, &lib) { continue; } + + // Skip if this library is the same as the last. + last = if (lib.kind, lib.name) == last { continue } else { (lib.kind, lib.name) }; + + let verbatim = lib.verbatim.unwrap_or(false); match lib.kind { - NativeLibKind::Dylib | NativeLibKind::Unspecified => cmd.link_dylib(name), - NativeLibKind::Framework => cmd.link_framework(name), - NativeLibKind::StaticNoBundle => { + NativeLibKind::Dylib { as_needed } => { + cmd.link_dylib(name, verbatim, as_needed.unwrap_or(true)) + } + NativeLibKind::Unspecified => cmd.link_dylib(name, verbatim, true), + NativeLibKind::Framework { as_needed } => { + cmd.link_framework(name, as_needed.unwrap_or(true)) + } + NativeLibKind::Static { bundle: Some(false), .. } => { // Link "static-nobundle" native libs only if the crate they originate from // is being linked statically to the current crate. If it's linked dynamically // or is an rlib already included via some other dylib crate, the symbols from // native libs will have already been included in that dylib. if data[cnum.as_usize() - 1] == Linkage::Static { - cmd.link_staticlib(name) + cmd.link_staticlib(name, verbatim) } } // ignore statically included native libraries here as we've // already included them when we included the rust library // previously - NativeLibKind::StaticBundle => {} - NativeLibKind::RawDylib => { - // FIXME(#58713): Proper handling for raw dylibs. - bug!("raw_dylib feature not yet implemented"); - } + NativeLibKind::Static { bundle: None | Some(true), .. } => {} + NativeLibKind::RawDylib => {} } } } @@ -2269,3 +2530,30 @@ fn get_apple_sdk_root(sdk_name: &str) -> Result { Err(e) => Err(format!("failed to get {} SDK path: {}", sdk_name, e)), } } + +fn add_gcc_ld_path(cmd: &mut dyn Linker, sess: &Session, flavor: LinkerFlavor) { + if let Some(ld_impl) = sess.opts.debugging_opts.gcc_ld { + if let LinkerFlavor::Gcc = flavor { + match ld_impl { + LdImpl::Lld => { + let tools_path = + sess.host_filesearch(PathKind::All).get_tools_search_paths(false); + let lld_path = tools_path + .into_iter() + .map(|p| p.join("gcc-ld")) + .find(|p| { + p.join(if sess.host.is_like_windows { "ld.exe" } else { "ld" }).exists() + }) + .unwrap_or_else(|| sess.fatal("rust-lld (as ld) not found")); + cmd.cmd().arg({ + let mut arg = OsString::from("-B"); + arg.push(lld_path); + arg + }); + } + } + } else { + sess.fatal("option `-Z gcc-ld` is used even though linker flavor is not gcc"); + } + } +} diff --git a/compiler/rustc_codegen_ssa/src/back/linker.rs b/compiler/rustc_codegen_ssa/src/back/linker.rs index 77d8ab49ff..43ff664c3e 100644 --- a/compiler/rustc_codegen_ssa/src/back/linker.rs +++ b/compiler/rustc_codegen_ssa/src/back/linker.rs @@ -37,12 +37,14 @@ pub fn disable_localization(linker: &mut Command) { /// need out of the shared crate context before we get rid of it. #[derive(Encodable, Decodable)] pub struct LinkerInfo { + pub(super) target_cpu: String, exports: FxHashMap>, } impl LinkerInfo { - pub fn new(tcx: TyCtxt<'_>) -> LinkerInfo { + pub fn new(tcx: TyCtxt<'_>, target_cpu: String) -> LinkerInfo { LinkerInfo { + target_cpu, exports: tcx .sess .crate_types() @@ -57,38 +59,33 @@ impl LinkerInfo { cmd: Command, sess: &'a Session, flavor: LinkerFlavor, - target_cpu: &'a str, ) -> Box { match flavor { LinkerFlavor::Lld(LldFlavor::Link) | LinkerFlavor::Msvc => { Box::new(MsvcLinker { cmd, sess, info: self }) as Box } LinkerFlavor::Em => Box::new(EmLinker { cmd, sess, info: self }) as Box, - LinkerFlavor::Gcc => Box::new(GccLinker { - cmd, - sess, - info: self, - hinted_static: false, - is_ld: false, - target_cpu, - }) as Box, + LinkerFlavor::Gcc => { + Box::new(GccLinker { cmd, sess, info: self, hinted_static: false, is_ld: false }) + as Box + } LinkerFlavor::Lld(LldFlavor::Ld) | LinkerFlavor::Lld(LldFlavor::Ld64) - | LinkerFlavor::Ld => Box::new(GccLinker { - cmd, - sess, - info: self, - hinted_static: false, - is_ld: true, - target_cpu, - }) as Box, + | LinkerFlavor::Ld => { + Box::new(GccLinker { cmd, sess, info: self, hinted_static: false, is_ld: true }) + as Box + } LinkerFlavor::Lld(LldFlavor::Wasm) => { Box::new(WasmLd::new(cmd, sess, self)) as Box } LinkerFlavor::PtxLinker => Box::new(PtxLinker { cmd, sess }) as Box, + + LinkerFlavor::BpfLinker => { + Box::new(BpfLinker { cmd, sess, info: self }) as Box + } } } } @@ -103,18 +100,19 @@ impl LinkerInfo { pub trait Linker { fn cmd(&mut self) -> &mut Command; fn set_output_kind(&mut self, output_kind: LinkOutputKind, out_filename: &Path); - fn link_dylib(&mut self, lib: Symbol); + fn link_dylib(&mut self, lib: Symbol, verbatim: bool, as_needed: bool); fn link_rust_dylib(&mut self, lib: Symbol, path: &Path); - fn link_framework(&mut self, framework: Symbol); - fn link_staticlib(&mut self, lib: Symbol); + fn link_framework(&mut self, framework: Symbol, as_needed: bool); + fn link_staticlib(&mut self, lib: Symbol, verbatim: bool); fn link_rlib(&mut self, lib: &Path); fn link_whole_rlib(&mut self, lib: &Path); - fn link_whole_staticlib(&mut self, lib: Symbol, search_path: &[PathBuf]); + fn link_whole_staticlib(&mut self, lib: Symbol, verbatim: bool, search_path: &[PathBuf]); fn include_path(&mut self, path: &Path); fn framework_path(&mut self, path: &Path); fn output_filename(&mut self, path: &Path); fn add_object(&mut self, path: &Path); fn gc_sections(&mut self, keep_metadata: bool); + fn no_gc_sections(&mut self); fn full_relro(&mut self); fn partial_relro(&mut self); fn no_relro(&mut self); @@ -132,7 +130,7 @@ pub trait Linker { fn add_eh_frame_header(&mut self) {} fn add_no_exec(&mut self) {} fn add_as_needed(&mut self) {} - fn finalize(&mut self); + fn reset_per_library_state(&mut self) {} } impl dyn Linker + '_ { @@ -156,7 +154,6 @@ pub struct GccLinker<'a> { hinted_static: bool, // Keeps track of the current hinting mode. // Link as ld is_ld: bool, - target_cpu: &'a str, } impl<'a> GccLinker<'a> { @@ -228,8 +225,7 @@ impl<'a> GccLinker<'a> { }; self.linker_arg(&format!("-plugin-opt={}", opt_level)); - let target_cpu = self.target_cpu; - self.linker_arg(&format!("-plugin-opt=mcpu={}", target_cpu)); + self.linker_arg(&format!("-plugin-opt=mcpu={}", self.info.target_cpu)); } fn build_dylib(&mut self, out_filename: &Path) { @@ -287,8 +283,11 @@ impl<'a> Linker for GccLinker<'a> { } } LinkOutputKind::DynamicPicExe => { - // `-pie` works for both gcc wrapper and ld. - self.cmd.arg("-pie"); + // noop on windows w/ gcc & ld, error w/ lld + if !self.sess.target.is_like_windows { + // `-pie` works for both gcc wrapper and ld. + self.cmd.arg("-pie"); + } } LinkOutputKind::StaticNoPicExe => { // `-static` works for both gcc wrapper and ld. @@ -338,13 +337,40 @@ impl<'a> Linker for GccLinker<'a> { } } - fn link_dylib(&mut self, lib: Symbol) { + fn link_dylib(&mut self, lib: Symbol, verbatim: bool, as_needed: bool) { + if self.sess.target.os == "illumos" && lib.as_str() == "c" { + // libc will be added via late_link_args on illumos so that it will + // appear last in the library search order. + // FIXME: This should be replaced by a more complete and generic + // mechanism for controlling the order of library arguments passed + // to the linker. + return; + } + if !as_needed { + if self.sess.target.is_like_osx { + // FIXME(81490): ld64 doesn't support these flags but macOS 11 + // has -needed-l{} / -needed_library {} + // but we have no way to detect that here. + self.sess.warn("`as-needed` modifier not implemented yet for ld64"); + } else if self.sess.target.linker_is_gnu && !self.sess.target.is_like_windows { + self.linker_arg("--no-as-needed"); + } else { + self.sess.warn("`as-needed` modifier not supported for current linker"); + } + } self.hint_dynamic(); - self.cmd.arg(format!("-l{}", lib)); + self.cmd.arg(format!("-l{}{}", if verbatim { ":" } else { "" }, lib)); + if !as_needed { + if self.sess.target.is_like_osx { + // See above FIXME comment + } else if self.sess.target.linker_is_gnu && !self.sess.target.is_like_windows { + self.linker_arg("--as-needed"); + } + } } - fn link_staticlib(&mut self, lib: Symbol) { + fn link_staticlib(&mut self, lib: Symbol, verbatim: bool) { self.hint_static(); - self.cmd.arg(format!("-l{}", lib)); + self.cmd.arg(format!("-l{}{}", if verbatim { ":" } else { "" }, lib)); } fn link_rlib(&mut self, lib: &Path) { self.hint_static(); @@ -378,8 +404,14 @@ impl<'a> Linker for GccLinker<'a> { self.cmd.arg(format!("-l{}", lib)); } - fn link_framework(&mut self, framework: Symbol) { + fn link_framework(&mut self, framework: Symbol, as_needed: bool) { self.hint_dynamic(); + if !as_needed { + // FIXME(81490): ld64 as of macOS 11 supports the -needed_framework + // flag but we have no way to detect that here. + // self.cmd.arg("-needed_framework").sym_arg(framework); + self.sess.warn("`as-needed` modifier not implemented yet for ld64"); + } self.cmd.arg("-framework").sym_arg(framework); } @@ -389,17 +421,21 @@ impl<'a> Linker for GccLinker<'a> { // don't otherwise explicitly reference them. This can occur for // libraries which are just providing bindings, libraries with generic // functions, etc. - fn link_whole_staticlib(&mut self, lib: Symbol, search_path: &[PathBuf]) { + fn link_whole_staticlib(&mut self, lib: Symbol, verbatim: bool, search_path: &[PathBuf]) { self.hint_static(); let target = &self.sess.target; if !target.is_like_osx { - self.linker_arg("--whole-archive").cmd.arg(format!("-l{}", lib)); + self.linker_arg("--whole-archive").cmd.arg(format!( + "-l{}{}", + if verbatim { ":" } else { "" }, + lib + )); self.linker_arg("--no-whole-archive"); } else { // -force_load is the macOS equivalent of --whole-archive, but it // involves passing the full path to the library to link. self.linker_arg("-force_load"); - let lib = archive::find_library(lib, search_path, &self.sess); + let lib = archive::find_library(lib, verbatim, search_path, &self.sess); self.linker_arg(&lib); } } @@ -432,21 +468,29 @@ impl<'a> Linker for GccLinker<'a> { // insert it here. if self.sess.target.is_like_osx { self.linker_arg("-dead_strip"); - } else if self.sess.target.is_like_solaris { - self.linker_arg("-zignore"); // If we're building a dylib, we don't use --gc-sections because LLVM // has already done the best it can do, and we also don't want to // eliminate the metadata. If we're building an executable, however, // --gc-sections drops the size of hello world from 1.8MB to 597K, a 67% // reduction. - } else if !keep_metadata { + } else if (self.sess.target.linker_is_gnu || self.sess.target.is_like_wasm) + && !keep_metadata + { self.linker_arg("--gc-sections"); } } + fn no_gc_sections(&mut self) { + if self.sess.target.is_like_osx { + self.linker_arg("-no_dead_strip"); + } else if self.sess.target.linker_is_gnu || self.sess.target.is_like_wasm { + self.linker_arg("--no-gc-sections"); + } + } + fn optimize(&mut self) { - if !self.sess.target.linker_is_gnu { + if !self.sess.target.linker_is_gnu && !self.sess.target.is_like_wasm { return; } @@ -482,15 +526,18 @@ impl<'a> Linker for GccLinker<'a> { fn control_flow_guard(&mut self) {} fn debuginfo(&mut self, strip: Strip) { + // MacOS linker doesn't support stripping symbols directly anymore. + if self.sess.target.is_like_osx { + return; + } + match strip { Strip::None => {} Strip::Debuginfo => { - // MacOS linker does not support longhand argument --strip-debug - self.linker_arg("-S"); + self.linker_arg("--strip-debug"); } Strip::Symbols => { - // MacOS linker does not support longhand argument --strip-all - self.linker_arg("-s"); + self.linker_arg("--strip-all"); } } } @@ -607,7 +654,7 @@ impl<'a> Linker for GccLinker<'a> { self.linker_arg(&subsystem); } - fn finalize(&mut self) { + fn reset_per_library_state(&mut self) { self.hint_dynamic(); // Reset to default before returning the composed command line. } @@ -653,8 +700,12 @@ impl<'a> Linker for GccLinker<'a> { } fn add_as_needed(&mut self) { - if self.sess.target.linker_is_gnu { + if self.sess.target.linker_is_gnu && !self.sess.target.is_like_windows { self.linker_arg("--as-needed"); + } else if self.sess.target.is_like_solaris { + // -z ignore is the Solaris equivalent to the GNU ld --as-needed option + self.linker_arg("-z"); + self.linker_arg("ignore"); } } } @@ -708,8 +759,12 @@ impl<'a> Linker for MsvcLinker<'a> { } } - fn link_dylib(&mut self, lib: Symbol) { - self.cmd.arg(&format!("{}.lib", lib)); + fn no_gc_sections(&mut self) { + self.cmd.arg("/OPT:NOREF,NOICF"); + } + + fn link_dylib(&mut self, lib: Symbol, verbatim: bool, _as_needed: bool) { + self.cmd.arg(format!("{}{}", lib, if verbatim { "" } else { ".lib" })); } fn link_rust_dylib(&mut self, lib: Symbol, path: &Path) { @@ -718,13 +773,13 @@ impl<'a> Linker for MsvcLinker<'a> { // check to see if the file is there and just omit linking to it if it's // not present. let name = format!("{}.dll.lib", lib); - if fs::metadata(&path.join(&name)).is_ok() { + if path.join(&name).exists() { self.cmd.arg(name); } } - fn link_staticlib(&mut self, lib: Symbol) { - self.cmd.arg(&format!("{}.lib", lib)); + fn link_staticlib(&mut self, lib: Symbol, verbatim: bool) { + self.cmd.arg(format!("{}{}", lib, if verbatim { "" } else { ".lib" })); } fn full_relro(&mut self) { @@ -762,16 +817,14 @@ impl<'a> Linker for MsvcLinker<'a> { fn framework_path(&mut self, _path: &Path) { bug!("frameworks are not supported on windows") } - fn link_framework(&mut self, _framework: Symbol) { + fn link_framework(&mut self, _framework: Symbol, _as_needed: bool) { bug!("frameworks are not supported on windows") } - fn link_whole_staticlib(&mut self, lib: Symbol, _search_path: &[PathBuf]) { - self.link_staticlib(lib); - self.cmd.arg(format!("/WHOLEARCHIVE:{}.lib", lib)); + fn link_whole_staticlib(&mut self, lib: Symbol, verbatim: bool, _search_path: &[PathBuf]) { + self.cmd.arg(format!("/WHOLEARCHIVE:{}{}", lib, if verbatim { "" } else { ".lib" })); } fn link_whole_rlib(&mut self, path: &Path) { - self.link_rlib(path); let mut arg = OsString::from("/WHOLEARCHIVE:"); arg.push(path); self.cmd.arg(arg); @@ -885,8 +938,6 @@ impl<'a> Linker for MsvcLinker<'a> { } } - fn finalize(&mut self) {} - // MSVC doesn't need group indicators fn group_start(&mut self) {} fn group_end(&mut self) {} @@ -917,7 +968,7 @@ impl<'a> Linker for EmLinker<'a> { self.cmd.arg("-L").arg(path); } - fn link_staticlib(&mut self, lib: Symbol) { + fn link_staticlib(&mut self, lib: Symbol, _verbatim: bool) { self.cmd.arg("-l").sym_arg(lib); } @@ -929,14 +980,14 @@ impl<'a> Linker for EmLinker<'a> { self.cmd.arg(path); } - fn link_dylib(&mut self, lib: Symbol) { + fn link_dylib(&mut self, lib: Symbol, verbatim: bool, _as_needed: bool) { // Emscripten always links statically - self.link_staticlib(lib); + self.link_staticlib(lib, verbatim); } - fn link_whole_staticlib(&mut self, lib: Symbol, _search_path: &[PathBuf]) { + fn link_whole_staticlib(&mut self, lib: Symbol, verbatim: bool, _search_path: &[PathBuf]) { // not supported? - self.link_staticlib(lib); + self.link_staticlib(lib, verbatim); } fn link_whole_rlib(&mut self, lib: &Path) { @@ -945,7 +996,7 @@ impl<'a> Linker for EmLinker<'a> { } fn link_rust_dylib(&mut self, lib: Symbol, _path: &Path) { - self.link_dylib(lib); + self.link_dylib(lib, false, true); } fn link_rlib(&mut self, lib: &Path) { @@ -968,7 +1019,7 @@ impl<'a> Linker for EmLinker<'a> { bug!("frameworks are not supported on Emscripten") } - fn link_framework(&mut self, _framework: Symbol) { + fn link_framework(&mut self, _framework: Symbol, _as_needed: bool) { bug!("frameworks are not supported on Emscripten") } @@ -976,6 +1027,10 @@ impl<'a> Linker for EmLinker<'a> { // noop } + fn no_gc_sections(&mut self) { + // noop + } + fn optimize(&mut self) { // Emscripten performs own optimizations self.cmd.arg(match self.sess.opts.optimize { @@ -1043,8 +1098,6 @@ impl<'a> Linker for EmLinker<'a> { // noop } - fn finalize(&mut self) {} - // Appears not necessary on Emscripten fn group_start(&mut self) {} fn group_end(&mut self) {} @@ -1119,11 +1172,11 @@ impl<'a> Linker for WasmLd<'a> { } } - fn link_dylib(&mut self, lib: Symbol) { + fn link_dylib(&mut self, lib: Symbol, _verbatim: bool, _as_needed: bool) { self.cmd.arg("-l").sym_arg(lib); } - fn link_staticlib(&mut self, lib: Symbol) { + fn link_staticlib(&mut self, lib: Symbol, _verbatim: bool) { self.cmd.arg("-l").sym_arg(lib); } @@ -1157,11 +1210,11 @@ impl<'a> Linker for WasmLd<'a> { self.cmd.arg("-l").sym_arg(lib); } - fn link_framework(&mut self, _framework: Symbol) { + fn link_framework(&mut self, _framework: Symbol, _as_needed: bool) { panic!("frameworks not supported") } - fn link_whole_staticlib(&mut self, lib: Symbol, _search_path: &[PathBuf]) { + fn link_whole_staticlib(&mut self, lib: Symbol, _verbatim: bool, _search_path: &[PathBuf]) { self.cmd.arg("-l").sym_arg(lib); } @@ -1173,6 +1226,10 @@ impl<'a> Linker for WasmLd<'a> { self.cmd.arg("--gc-sections"); } + fn no_gc_sections(&mut self) { + self.cmd.arg("--no-gc-sections"); + } + fn optimize(&mut self) { self.cmd.arg(match self.sess.opts.optimize { OptLevel::No => "-O0", @@ -1221,8 +1278,6 @@ impl<'a> Linker for WasmLd<'a> { fn subsystem(&mut self, _subsystem: &str) {} - fn finalize(&mut self) {} - // Not needed for now with LLD fn group_start(&mut self) {} fn group_end(&mut self) {} @@ -1250,7 +1305,7 @@ fn exported_symbols(tcx: TyCtxt<'_>, crate_type: CrateType) -> Vec { } } - let formats = tcx.dependency_formats(LOCAL_CRATE); + let formats = tcx.dependency_formats(()); let deps = formats.iter().find_map(|(t, list)| (*t == crate_type).then_some(list)).unwrap(); for (index, dep_format) in deps.iter().enumerate() { @@ -1319,15 +1374,108 @@ impl<'a> Linker for PtxLinker<'a> { self.cmd.arg("-o").arg(path); } - fn finalize(&mut self) { - // Provide the linker with fallback to internal `target-cpu`. - self.cmd.arg("--fallback-arch").arg(match self.sess.opts.cg.target_cpu { - Some(ref s) => s, - None => &self.sess.target.cpu, + fn link_dylib(&mut self, _lib: Symbol, _verbatim: bool, _as_needed: bool) { + panic!("external dylibs not supported") + } + + fn link_rust_dylib(&mut self, _lib: Symbol, _path: &Path) { + panic!("external dylibs not supported") + } + + fn link_staticlib(&mut self, _lib: Symbol, _verbatim: bool) { + panic!("staticlibs not supported") + } + + fn link_whole_staticlib(&mut self, _lib: Symbol, _verbatim: bool, _search_path: &[PathBuf]) { + panic!("staticlibs not supported") + } + + fn framework_path(&mut self, _path: &Path) { + panic!("frameworks not supported") + } + + fn link_framework(&mut self, _framework: Symbol, _as_needed: bool) { + panic!("frameworks not supported") + } + + fn full_relro(&mut self) {} + + fn partial_relro(&mut self) {} + + fn no_relro(&mut self) {} + + fn gc_sections(&mut self, _keep_metadata: bool) {} + + fn no_gc_sections(&mut self) {} + + fn pgo_gen(&mut self) {} + + fn no_crt_objects(&mut self) {} + + fn no_default_libraries(&mut self) {} + + fn control_flow_guard(&mut self) {} + + fn export_symbols(&mut self, _tmpdir: &Path, _crate_type: CrateType) {} + + fn subsystem(&mut self, _subsystem: &str) {} + + fn group_start(&mut self) {} + + fn group_end(&mut self) {} + + fn linker_plugin_lto(&mut self) {} +} + +pub struct BpfLinker<'a> { + cmd: Command, + sess: &'a Session, + info: &'a LinkerInfo, +} + +impl<'a> Linker for BpfLinker<'a> { + fn cmd(&mut self) -> &mut Command { + &mut self.cmd + } + + fn set_output_kind(&mut self, _output_kind: LinkOutputKind, _out_filename: &Path) {} + + fn link_rlib(&mut self, path: &Path) { + self.cmd.arg(path); + } + + fn link_whole_rlib(&mut self, path: &Path) { + self.cmd.arg(path); + } + + fn include_path(&mut self, path: &Path) { + self.cmd.arg("-L").arg(path); + } + + fn debuginfo(&mut self, _strip: Strip) { + self.cmd.arg("--debug"); + } + + fn add_object(&mut self, path: &Path) { + self.cmd.arg(path); + } + + fn optimize(&mut self) { + self.cmd.arg(match self.sess.opts.optimize { + OptLevel::No => "-O0", + OptLevel::Less => "-O1", + OptLevel::Default => "-O2", + OptLevel::Aggressive => "-O3", + OptLevel::Size => "-Os", + OptLevel::SizeMin => "-Oz", }); } - fn link_dylib(&mut self, _lib: Symbol) { + fn output_filename(&mut self, path: &Path) { + self.cmd.arg("-o").arg(path); + } + + fn link_dylib(&mut self, _lib: Symbol, _verbatim: bool, _as_needed: bool) { panic!("external dylibs not supported") } @@ -1335,11 +1483,11 @@ impl<'a> Linker for PtxLinker<'a> { panic!("external dylibs not supported") } - fn link_staticlib(&mut self, _lib: Symbol) { + fn link_staticlib(&mut self, _lib: Symbol, _verbatim: bool) { panic!("staticlibs not supported") } - fn link_whole_staticlib(&mut self, _lib: Symbol, _search_path: &[PathBuf]) { + fn link_whole_staticlib(&mut self, _lib: Symbol, _verbatim: bool, _search_path: &[PathBuf]) { panic!("staticlibs not supported") } @@ -1347,7 +1495,7 @@ impl<'a> Linker for PtxLinker<'a> { panic!("frameworks not supported") } - fn link_framework(&mut self, _framework: Symbol) { + fn link_framework(&mut self, _framework: Symbol, _as_needed: bool) { panic!("frameworks not supported") } @@ -1359,6 +1507,8 @@ impl<'a> Linker for PtxLinker<'a> { fn gc_sections(&mut self, _keep_metadata: bool) {} + fn no_gc_sections(&mut self) {} + fn pgo_gen(&mut self) {} fn no_crt_objects(&mut self) {} @@ -1367,7 +1517,20 @@ impl<'a> Linker for PtxLinker<'a> { fn control_flow_guard(&mut self) {} - fn export_symbols(&mut self, _tmpdir: &Path, _crate_type: CrateType) {} + fn export_symbols(&mut self, tmpdir: &Path, crate_type: CrateType) { + let path = tmpdir.join("symbols"); + let res: io::Result<()> = try { + let mut f = BufWriter::new(File::create(&path)?); + for sym in self.info.exports[&crate_type].iter() { + writeln!(f, "{}", sym)?; + } + }; + if let Err(e) = res { + self.sess.fatal(&format!("failed to write symbols file: {}", e)); + } else { + self.cmd.arg("--export-symbols").arg(&path); + } + } fn subsystem(&mut self, _subsystem: &str) {} diff --git a/compiler/rustc_codegen_ssa/src/back/lto.rs b/compiler/rustc_codegen_ssa/src/back/lto.rs index 0ff0522946..d6ae689f25 100644 --- a/compiler/rustc_codegen_ssa/src/back/lto.rs +++ b/compiler/rustc_codegen_ssa/src/back/lto.rs @@ -72,7 +72,7 @@ impl LtoModuleCodegen { let module = module.take().unwrap(); { let config = cgcx.config(module.kind); - B::run_lto_pass_manager(cgcx, &module, config, false); + B::run_lto_pass_manager(cgcx, &module, config, false)?; } Ok(module) } diff --git a/compiler/rustc_codegen_ssa/src/back/metadata.rs b/compiler/rustc_codegen_ssa/src/back/metadata.rs new file mode 100644 index 0000000000..0fff319580 --- /dev/null +++ b/compiler/rustc_codegen_ssa/src/back/metadata.rs @@ -0,0 +1,85 @@ +//! Reading of the rustc metadata for rlibs and dylibs + +use std::fs::File; +use std::path::Path; + +use object::{Object, ObjectSection}; +use rustc_data_structures::memmap::Mmap; +use rustc_data_structures::owning_ref::OwningRef; +use rustc_data_structures::rustc_erase_owner; +use rustc_data_structures::sync::MetadataRef; +use rustc_middle::middle::cstore::MetadataLoader; +use rustc_target::spec::Target; + +use crate::METADATA_FILENAME; + +/// The default metadata loader. This is used by cg_llvm and cg_clif. +/// +/// # Metadata location +/// +///
+///
rlib
+///
The metadata can be found in the `lib.rmeta` file inside of the ar archive.
+///
dylib
+///
The metadata can be found in the `.rustc` section of the shared library.
+///
+pub struct DefaultMetadataLoader; + +fn load_metadata_with( + path: &Path, + f: impl for<'a> FnOnce(&'a [u8]) -> Result<&'a [u8], String>, +) -> Result { + let file = + File::open(path).map_err(|e| format!("failed to open file '{}': {}", path.display(), e))?; + let data = unsafe { Mmap::map(file) } + .map_err(|e| format!("failed to mmap file '{}': {}", path.display(), e))?; + let metadata = OwningRef::new(data).try_map(f)?; + return Ok(rustc_erase_owner!(metadata.map_owner_box())); +} + +impl MetadataLoader for DefaultMetadataLoader { + fn get_rlib_metadata(&self, _target: &Target, path: &Path) -> Result { + load_metadata_with(path, |data| { + let archive = object::read::archive::ArchiveFile::parse(&*data) + .map_err(|e| format!("failed to parse rlib '{}': {}", path.display(), e))?; + + for entry_result in archive.members() { + let entry = entry_result + .map_err(|e| format!("failed to parse rlib '{}': {}", path.display(), e))?; + if entry.name() == METADATA_FILENAME.as_bytes() { + let data = entry + .data(data) + .map_err(|e| format!("failed to parse rlib '{}': {}", path.display(), e))?; + return search_for_metadata(path, data, ".rmeta"); + } + } + + Err(format!("metadata not found in rlib '{}'", path.display())) + }) + } + + fn get_dylib_metadata(&self, _target: &Target, path: &Path) -> Result { + load_metadata_with(path, |data| search_for_metadata(path, data, ".rustc")) + } +} + +fn search_for_metadata<'a>( + path: &Path, + bytes: &'a [u8], + section: &str, +) -> Result<&'a [u8], String> { + let file = match object::File::parse(bytes) { + Ok(f) => f, + // The parse above could fail for odd reasons like corruption, but for + // now we just interpret it as this target doesn't support metadata + // emission in object files so the entire byte slice itself is probably + // a metadata file. Ideally though if necessary we could at least check + // the prefix of bytes to see if it's an actual metadata object and if + // not forward the error along here. + Err(_) => return Ok(bytes), + }; + file.section_by_name(section) + .ok_or_else(|| format!("no `{}` section in '{}'", section, path.display()))? + .data() + .map_err(|e| format!("failed to read {} section in '{}': {}", section, path.display(), e)) +} diff --git a/compiler/rustc_codegen_ssa/src/back/mod.rs b/compiler/rustc_codegen_ssa/src/back/mod.rs index 20ca503d43..d11ed54eb2 100644 --- a/compiler/rustc_codegen_ssa/src/back/mod.rs +++ b/compiler/rustc_codegen_ssa/src/back/mod.rs @@ -3,6 +3,7 @@ pub mod command; pub mod link; pub mod linker; pub mod lto; +pub mod metadata; pub mod rpath; pub mod symbol_export; pub mod write; diff --git a/compiler/rustc_codegen_ssa/src/back/rpath.rs b/compiler/rustc_codegen_ssa/src/back/rpath.rs index 5f21046b05..39b0ccd120 100644 --- a/compiler/rustc_codegen_ssa/src/back/rpath.rs +++ b/compiler/rustc_codegen_ssa/src/back/rpath.rs @@ -13,7 +13,6 @@ pub struct RPathConfig<'a> { pub is_like_osx: bool, pub has_rpath: bool, pub linker_is_gnu: bool, - pub get_install_prefix_lib_path: &'a mut dyn FnMut() -> PathBuf, } pub fn get_rpath_flags(config: &mut RPathConfig<'_>) -> Vec { @@ -63,24 +62,13 @@ fn get_rpaths(config: &mut RPathConfig<'_>, libs: &[PathBuf]) -> Vec { // Use relative paths to the libraries. Binaries can be moved // as long as they maintain the relative relationship to the // crates they depend on. - let rel_rpaths = get_rpaths_relative_to_output(config, libs); + let rpaths = get_rpaths_relative_to_output(config, libs); - // And a final backup rpath to the global library location. - let fallback_rpaths = vec![get_install_prefix_rpath(config)]; - - fn log_rpaths(desc: &str, rpaths: &[String]) { - debug!("{} rpaths:", desc); - for rpath in rpaths { - debug!(" {}", *rpath); - } + debug!("rpaths:"); + for rpath in &rpaths { + debug!(" {}", rpath); } - log_rpaths("relative", &rel_rpaths); - log_rpaths("fallback", &fallback_rpaths); - - let mut rpaths = rel_rpaths; - rpaths.extend_from_slice(&fallback_rpaths); - // Remove duplicates minimize_rpaths(&rpaths) } @@ -113,13 +101,6 @@ fn path_relative_from(path: &Path, base: &Path) -> Option { diff_paths(path, base) } -fn get_install_prefix_rpath(config: &mut RPathConfig<'_>) -> String { - let path = (config.get_install_prefix_lib_path)(); - let path = env::current_dir().unwrap().join(&path); - // FIXME (#9639): This needs to handle non-utf8 paths - path.to_str().expect("non-utf8 component in rpath").to_owned() -} - fn minimize_rpaths(rpaths: &[String]) -> Vec { let mut set = FxHashSet::default(); let mut minimized = Vec::new(); diff --git a/compiler/rustc_codegen_ssa/src/back/rpath/tests.rs b/compiler/rustc_codegen_ssa/src/back/rpath/tests.rs index 35836ae719..24c362db12 100644 --- a/compiler/rustc_codegen_ssa/src/back/rpath/tests.rs +++ b/compiler/rustc_codegen_ssa/src/back/rpath/tests.rs @@ -40,7 +40,6 @@ fn test_rpath_relative() { is_like_osx: true, linker_is_gnu: false, out_filename: PathBuf::from("bin/rustc"), - get_install_prefix_lib_path: &mut || panic!(), }; let res = get_rpath_relative_to_output(config, Path::new("lib/libstd.so")); assert_eq!(res, "@loader_path/../lib"); @@ -48,7 +47,6 @@ fn test_rpath_relative() { let config = &mut RPathConfig { used_crates: &[], out_filename: PathBuf::from("bin/rustc"), - get_install_prefix_lib_path: &mut || panic!(), has_rpath: true, is_like_osx: false, linker_is_gnu: true, diff --git a/compiler/rustc_codegen_ssa/src/back/symbol_export.rs b/compiler/rustc_codegen_ssa/src/back/symbol_export.rs index b8f277c8ff..b2ecc3b0f3 100644 --- a/compiler/rustc_codegen_ssa/src/back/symbol_export.rs +++ b/compiler/rustc_codegen_ssa/src/back/symbol_export.rs @@ -4,7 +4,7 @@ use rustc_ast::expand::allocator::ALLOCATOR_METHODS; use rustc_data_structures::fingerprint::Fingerprint; use rustc_data_structures::fx::FxHashMap; use rustc_hir as hir; -use rustc_hir::def_id::{CrateNum, DefId, DefIdMap, CRATE_DEF_INDEX, LOCAL_CRATE}; +use rustc_hir::def_id::{CrateNum, DefId, DefIdMap, LocalDefId, CRATE_DEF_INDEX, LOCAL_CRATE}; use rustc_hir::Node; use rustc_index::vec::IndexVec; use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags; @@ -60,7 +60,7 @@ fn reachable_non_generics_provider(tcx: TyCtxt<'_>, cnum: CrateNum) -> DefIdMap< tcx.is_panic_runtime(LOCAL_CRATE) || tcx.is_compiler_builtins(LOCAL_CRATE); let mut reachable_non_generics: DefIdMap<_> = tcx - .reachable_set(LOCAL_CRATE) + .reachable_set(()) .iter() .filter_map(|&def_id| { // We want to ignore some FFI functions that are not exposed from @@ -133,12 +133,12 @@ fn reachable_non_generics_provider(tcx: TyCtxt<'_>, cnum: CrateNum) -> DefIdMap< }) .collect(); - if let Some(id) = tcx.proc_macro_decls_static(LOCAL_CRATE) { - reachable_non_generics.insert(id, SymbolExportLevel::C); + if let Some(id) = tcx.proc_macro_decls_static(()) { + reachable_non_generics.insert(id.to_def_id(), SymbolExportLevel::C); } - if let Some(id) = tcx.plugin_registrar_fn(LOCAL_CRATE) { - reachable_non_generics.insert(id, SymbolExportLevel::C); + if let Some(id) = tcx.plugin_registrar_fn(()) { + reachable_non_generics.insert(id.to_def_id(), SymbolExportLevel::C); } reachable_non_generics @@ -174,7 +174,7 @@ fn exported_symbols_provider_local( .map(|(&def_id, &level)| (ExportedSymbol::NonGeneric(def_id), level)) .collect(); - if tcx.entry_fn(LOCAL_CRATE).is_some() { + if tcx.entry_fn(()).is_some() { let exported_symbol = ExportedSymbol::NoDefId(SymbolName::new(tcx, "main")); symbols.push((exported_symbol, SymbolExportLevel::C)); @@ -230,7 +230,7 @@ fn exported_symbols_provider_local( // external linkage is enough for monomorphization to be linked to. let need_visibility = tcx.sess.target.dynamic_linking && !tcx.sess.target.only_cdylib; - let (_, cgus) = tcx.collect_and_partition_mono_items(LOCAL_CRATE); + let (_, cgus) = tcx.collect_and_partition_mono_items(()); for (mono_item, &(linkage, visibility)) in cgus.iter().flat_map(|cgu| cgu.items().iter()) { if linkage != Linkage::External { @@ -275,11 +275,9 @@ fn exported_symbols_provider_local( fn upstream_monomorphizations_provider( tcx: TyCtxt<'_>, - cnum: CrateNum, + (): (), ) -> DefIdMap, CrateNum>> { - debug_assert!(cnum == LOCAL_CRATE); - - let cnums = tcx.all_crate_nums(LOCAL_CRATE); + let cnums = tcx.all_crate_nums(()); let mut instances: DefIdMap> = Default::default(); @@ -341,7 +339,7 @@ fn upstream_monomorphizations_for_provider( def_id: DefId, ) -> Option<&FxHashMap, CrateNum>> { debug_assert!(!def_id.is_local()); - tcx.upstream_monomorphizations(LOCAL_CRATE).get(&def_id) + tcx.upstream_monomorphizations(()).get(&def_id) } fn upstream_drop_glue_for_provider<'tcx>( @@ -355,12 +353,8 @@ fn upstream_drop_glue_for_provider<'tcx>( } } -fn is_unreachable_local_definition_provider(tcx: TyCtxt<'_>, def_id: DefId) -> bool { - if let Some(def_id) = def_id.as_local() { - !tcx.reachable_set(LOCAL_CRATE).contains(&def_id) - } else { - bug!("is_unreachable_local_definition called with non-local DefId: {:?}", def_id) - } +fn is_unreachable_local_definition_provider(tcx: TyCtxt<'_>, def_id: LocalDefId) -> bool { + !tcx.reachable_set(()).contains(&def_id) } pub fn provide(providers: &mut Providers) { @@ -370,6 +364,7 @@ pub fn provide(providers: &mut Providers) { providers.upstream_monomorphizations = upstream_monomorphizations_provider; providers.is_unreachable_local_definition = is_unreachable_local_definition_provider; providers.upstream_drop_glue_for = upstream_drop_glue_for_provider; + providers.wasm_import_module_map = wasm_import_module_map; } pub fn provide_extern(providers: &mut Providers) { @@ -442,3 +437,30 @@ pub fn symbol_name_for_instance_in_crate<'tcx>( ExportedSymbol::NoDefId(symbol_name) => symbol_name.to_string(), } } + +fn wasm_import_module_map(tcx: TyCtxt<'_>, cnum: CrateNum) -> FxHashMap { + // Build up a map from DefId to a `NativeLib` structure, where + // `NativeLib` internally contains information about + // `#[link(wasm_import_module = "...")]` for example. + let native_libs = tcx.native_libraries(cnum); + + let def_id_to_native_lib = native_libs + .iter() + .filter_map(|lib| lib.foreign_module.map(|id| (id, lib))) + .collect::>(); + + let mut ret = FxHashMap::default(); + for (def_id, lib) in tcx.foreign_modules(cnum).iter() { + let module = def_id_to_native_lib.get(&def_id).and_then(|s| s.wasm_import_module); + let module = match module { + Some(s) => s, + None => continue, + }; + ret.extend(lib.foreign_items.iter().map(|id| { + assert_eq!(id.krate, cnum); + (*id, module.to_string()) + })); + } + + ret +} diff --git a/compiler/rustc_codegen_ssa/src/back/write.rs b/compiler/rustc_codegen_ssa/src/back/write.rs index c8688faa80..ff4e640957 100644 --- a/compiler/rustc_codegen_ssa/src/back/write.rs +++ b/compiler/rustc_codegen_ssa/src/back/write.rs @@ -31,7 +31,7 @@ use rustc_session::config::{self, CrateType, Lto, OutputFilenames, OutputType}; use rustc_session::config::{Passes, SwitchWithOptPath}; use rustc_session::Session; use rustc_span::source_map::SourceMap; -use rustc_span::symbol::{sym, Symbol}; +use rustc_span::symbol::sym; use rustc_span::{BytePos, FileName, InnerSpan, Pos, Span}; use rustc_target::spec::{MergeFunctions, PanicStrategy, SanitizerSet}; @@ -84,6 +84,8 @@ pub struct ModuleConfig { pub pgo_gen: SwitchWithOptPath, pub pgo_use: Option, + pub instrument_coverage: bool, + pub instrument_gcov: bool, pub sanitizer: SanitizerSet, pub sanitizer_recover: SanitizerSet, @@ -108,7 +110,7 @@ pub struct ModuleConfig { pub vectorize_slp: bool, pub merge_functions: bool, pub inline_threshold: Option, - pub new_llvm_pass_manager: bool, + pub new_llvm_pass_manager: Option, pub emit_lifetime_markers: bool, } @@ -165,25 +167,7 @@ impl ModuleConfig { }; ModuleConfig { - passes: if_regular!( - { - let mut passes = sess.opts.cg.passes.clone(); - // compiler_builtins overrides the codegen-units settings, - // which is incompatible with -Zprofile which requires that - // only a single codegen unit is used per crate. - if sess.opts.debugging_opts.profile && !is_compiler_builtins { - passes.push("insert-gcov-profiling".to_owned()); - } - - // The rustc option `-Zinstrument_coverage` injects intrinsic calls to - // `llvm.instrprof.increment()`, which requires the LLVM `instrprof` pass. - if sess.instrument_coverage() { - passes.push("instrprof".to_owned()); - } - passes - }, - vec![] - ), + passes: if_regular!(sess.opts.cg.passes.clone(), vec![]), opt_level: opt_level_and_size, opt_size: opt_level_and_size, @@ -193,6 +177,14 @@ impl ModuleConfig { SwitchWithOptPath::Disabled ), pgo_use: if_regular!(sess.opts.cg.profile_use.clone(), None), + instrument_coverage: if_regular!(sess.instrument_coverage(), false), + instrument_gcov: if_regular!( + // compiler_builtins overrides the codegen-units settings, + // which is incompatible with -Zprofile which requires that + // only a single codegen unit is used per crate. + sess.opts.debugging_opts.profile && !is_compiler_builtins, + false + ), sanitizer: if_regular!(sess.opts.debugging_opts.sanitizer, SanitizerSet::empty()), sanitizer_recover: if_regular!( @@ -427,29 +419,18 @@ fn need_pre_lto_bitcode_for_incr_comp(sess: &Session) -> bool { pub fn start_async_codegen( backend: B, tcx: TyCtxt<'_>, + target_cpu: String, metadata: EncodedMetadata, total_cgus: usize, ) -> OngoingCodegen { let (coordinator_send, coordinator_receive) = channel(); let sess = tcx.sess; - let crate_name = tcx.crate_name(LOCAL_CRATE); let crate_attrs = tcx.hir().attrs(rustc_hir::CRATE_HIR_ID); let no_builtins = tcx.sess.contains_name(crate_attrs, sym::no_builtins); let is_compiler_builtins = tcx.sess.contains_name(crate_attrs, sym::compiler_builtins); - let subsystem = tcx.sess.first_attr_value_str_by_name(crate_attrs, sym::windows_subsystem); - let windows_subsystem = subsystem.map(|subsystem| { - if subsystem != sym::windows && subsystem != sym::console { - tcx.sess.fatal(&format!( - "invalid windows subsystem `{}`, only \ - `windows` and `console` are allowed", - subsystem - )); - } - subsystem.to_string() - }); - let linker_info = LinkerInfo::new(tcx); + let linker_info = LinkerInfo::new(tcx, target_cpu); let crate_info = CrateInfo::new(tcx); let regular_config = @@ -479,9 +460,7 @@ pub fn start_async_codegen( OngoingCodegen { backend, - crate_name, metadata, - windows_subsystem, linker_info, crate_info, @@ -489,7 +468,7 @@ pub fn start_async_codegen( codegen_worker_receive, shared_emitter_main, future: coordinator_thread, - output_filenames: tcx.output_filenames(LOCAL_CRATE), + output_filenames: tcx.output_filenames(()), } } @@ -1049,7 +1028,7 @@ fn start_executing_work( // If we know that we won’t be doing codegen, create target machines without optimisation. config::OptLevel::No } else { - tcx.backend_optimization_level(LOCAL_CRATE) + tcx.backend_optimization_level(()) }; let cgcx = CodegenContext:: { backend: backend.clone(), @@ -1068,7 +1047,7 @@ fn start_executing_work( cgu_reuse_tracker: sess.cgu_reuse_tracker.clone(), coordinator_send, diag_emitter: shared_emitter.clone(), - output_filenames: tcx.output_filenames(LOCAL_CRATE), + output_filenames: tcx.output_filenames(()), regular_module_config: regular_config, metadata_module_config: metadata_config, allocator_module_config: allocator_config, @@ -1819,9 +1798,7 @@ impl SharedEmitterMain { pub struct OngoingCodegen { pub backend: B, - pub crate_name: Symbol, pub metadata: EncodedMetadata, - pub windows_subsystem: Option, pub linker_info: LinkerInfo, pub crate_info: CrateInfo, pub coordinator_send: Sender>, @@ -1864,9 +1841,7 @@ impl OngoingCodegen { ( CodegenResults { - crate_name: self.crate_name, metadata: self.metadata, - windows_subsystem: self.windows_subsystem, linker_info: self.linker_info, crate_info: self.crate_info, diff --git a/compiler/rustc_codegen_ssa/src/base.rs b/compiler/rustc_codegen_ssa/src/base.rs index e045a23eb0..38ab39febe 100644 --- a/compiler/rustc_codegen_ssa/src/base.rs +++ b/compiler/rustc_codegen_ssa/src/base.rs @@ -30,6 +30,7 @@ use rustc_middle::ty::{self, Instance, Ty, TyCtxt}; use rustc_session::cgu_reuse_tracker::CguReuse; use rustc_session::config::{self, EntryFnType}; use rustc_session::Session; +use rustc_span::symbol::sym; use rustc_target::abi::{Align, LayoutOf, VariantIdx}; use std::ops::{Deref, DerefMut}; @@ -347,7 +348,7 @@ pub fn codegen_instance<'a, 'tcx: 'a, Bx: BuilderMethods<'a, 'tcx>>( pub fn maybe_create_entry_wrapper<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( cx: &'a Bx::CodegenCx, ) -> Option { - let main_def_id = cx.tcx().entry_fn(LOCAL_CRATE).map(|(def_id, _)| def_id)?; + let (main_def_id, entry_type) = cx.tcx().entry_fn(())?; let main_is_local = main_def_id.is_local(); let instance = Instance::mono(cx.tcx(), main_def_id); @@ -357,28 +358,16 @@ pub fn maybe_create_entry_wrapper<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( if !cx.codegen_unit().contains_item(&MonoItem::Fn(instance)) { return None; } - } else { - // FIXME: Add support for non-local main fn codegen - let span = cx.tcx().main_def.unwrap().span; - let n = 28937; - cx.sess() - .struct_span_err(span, "entry symbol `main` from foreign crate is not yet supported.") - .note(&format!( - "see issue #{} \ - for more information", - n, n, - )) - .emit(); - cx.sess().abort_if_errors(); - bug!(); + } else if !cx.codegen_unit().is_primary() { + // We want to create the wrapper only when the codegen unit is the primary one + return None; } let main_llfn = cx.get_fn_addr(instance); - return cx.tcx().entry_fn(LOCAL_CRATE).map(|(_, et)| { - let use_start_lang_item = EntryFnType::Start != et; - create_entry_fn::(cx, main_llfn, main_def_id, use_start_lang_item) - }); + let use_start_lang_item = EntryFnType::Start != entry_type; + let entry_fn = create_entry_fn::(cx, main_llfn, main_def_id, use_start_lang_item); + return Some(entry_fn); fn create_entry_fn<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( cx: &'a Bx::CodegenCx, @@ -420,7 +409,8 @@ pub fn maybe_create_entry_wrapper<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( cx.set_frame_pointer_elimination(llfn); cx.apply_target_cpu_attr(llfn); - let mut bx = Bx::new_block(&cx, llfn, "top"); + let llbb = Bx::append_block(&cx, llfn, "top"); + let mut bx = Bx::build(&cx, llbb); bx.insert_reference_to_gdb_debug_scripts_section_global(); @@ -478,12 +468,13 @@ fn get_argc_argv<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( pub fn codegen_crate( backend: B, tcx: TyCtxt<'tcx>, + target_cpu: String, metadata: EncodedMetadata, need_metadata_module: bool, ) -> OngoingCodegen { // Skip crate items and just output metadata in -Z no-codegen mode. if tcx.sess.opts.debugging_opts.no_codegen || !tcx.sess.opts.output_types.should_codegen() { - let ongoing_codegen = start_async_codegen(backend, tcx, metadata, 1); + let ongoing_codegen = start_async_codegen(backend, tcx, target_cpu, metadata, 1); ongoing_codegen.codegen_finished(tcx); @@ -496,7 +487,7 @@ pub fn codegen_crate( // Run the monomorphization collector and partition the collected items into // codegen units. - let codegen_units = tcx.collect_and_partition_mono_items(LOCAL_CRATE).1; + let codegen_units = tcx.collect_and_partition_mono_items(()).1; // Force all codegen_unit queries so they are already either red or green // when compile_codegen_unit accesses them. We are not able to re-execute @@ -509,7 +500,8 @@ pub fn codegen_crate( } } - let ongoing_codegen = start_async_codegen(backend.clone(), tcx, metadata, codegen_units.len()); + let ongoing_codegen = + start_async_codegen(backend.clone(), tcx, target_cpu, metadata, codegen_units.len()); let ongoing_codegen = AbortCodegenOnDrop::(Some(ongoing_codegen)); // Codegen an allocator shim, if necessary. @@ -520,7 +512,7 @@ pub fn codegen_crate( // linkage, then it's already got an allocator shim and we'll be using that // one instead. If nothing exists then it's our job to generate the // allocator! - let any_dynamic_crate = tcx.dependency_formats(LOCAL_CRATE).iter().any(|(_, list)| { + let any_dynamic_crate = tcx.dependency_formats(()).iter().any(|(_, list)| { use rustc_middle::middle::dependency_format::Linkage; list.iter().any(|&linkage| linkage == Linkage::Dynamic) }); @@ -764,7 +756,22 @@ impl Drop for AbortCodegenOnDrop { impl CrateInfo { pub fn new(tcx: TyCtxt<'_>) -> CrateInfo { + let local_crate_name = tcx.crate_name(LOCAL_CRATE); + let crate_attrs = tcx.hir().attrs(rustc_hir::CRATE_HIR_ID); + let subsystem = tcx.sess.first_attr_value_str_by_name(crate_attrs, sym::windows_subsystem); + let windows_subsystem = subsystem.map(|subsystem| { + if subsystem != sym::windows && subsystem != sym::console { + tcx.sess.fatal(&format!( + "invalid windows subsystem `{}`, only \ + `windows` and `console` are allowed", + subsystem + )); + } + subsystem.to_string() + }); + let mut info = CrateInfo { + local_crate_name, panic_runtime: None, compiler_builtins: None, profiler_runtime: None, @@ -777,7 +784,8 @@ impl CrateInfo { used_crate_source: Default::default(), lang_item_to_crate: Default::default(), missing_lang_items: Default::default(), - dependency_formats: tcx.dependency_formats(LOCAL_CRATE), + dependency_formats: tcx.dependency_formats(()), + windows_subsystem, }; let lang_items = tcx.lang_items(); diff --git a/compiler/rustc_codegen_ssa/src/common.rs b/compiler/rustc_codegen_ssa/src/common.rs index afd83bfcb5..1fa60612d2 100644 --- a/compiler/rustc_codegen_ssa/src/common.rs +++ b/compiler/rustc_codegen_ssa/src/common.rs @@ -4,7 +4,8 @@ use rustc_errors::struct_span_err; use rustc_hir as hir; use rustc_hir::def_id::DefId; use rustc_hir::LangItem; -use rustc_middle::ty::{Ty, TyCtxt}; +use rustc_middle::mir::interpret::ConstValue; +use rustc_middle::ty::{self, layout::TyAndLayout, Ty, TyCtxt}; use rustc_session::Session; use rustc_span::Span; @@ -194,3 +195,30 @@ pub fn shift_mask_val<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( pub fn span_invalid_monomorphization_error(a: &Session, b: Span, c: &str) { struct_span_err!(a, b, E0511, "{}", c).emit(); } + +pub fn asm_const_to_str<'tcx>( + tcx: TyCtxt<'tcx>, + sp: Span, + const_value: ConstValue<'tcx>, + ty_and_layout: TyAndLayout<'tcx>, +) -> String { + let scalar = match const_value { + ConstValue::Scalar(s) => s, + _ => { + span_bug!(sp, "expected Scalar for promoted asm const, but got {:#?}", const_value) + } + }; + let value = scalar.assert_bits(ty_and_layout.size); + match ty_and_layout.ty.kind() { + ty::Uint(_) => value.to_string(), + ty::Int(int_ty) => match int_ty.normalize(tcx.sess.target.pointer_width) { + ty::IntTy::I8 => (value as i8).to_string(), + ty::IntTy::I16 => (value as i16).to_string(), + ty::IntTy::I32 => (value as i32).to_string(), + ty::IntTy::I64 => (value as i64).to_string(), + ty::IntTy::I128 => (value as i128).to_string(), + ty::IntTy::Isize => unreachable!(), + }, + _ => span_bug!(sp, "asm const has bad type {}", ty_and_layout.ty), + } +} diff --git a/compiler/rustc_codegen_ssa/src/coverageinfo/map.rs b/compiler/rustc_codegen_ssa/src/coverageinfo/map.rs index 4458fd6867..c1dfe1ef85 100644 --- a/compiler/rustc_codegen_ssa/src/coverageinfo/map.rs +++ b/compiler/rustc_codegen_ssa/src/coverageinfo/map.rs @@ -28,6 +28,7 @@ pub struct Expression { /// only whitespace or comments). According to LLVM Code Coverage Mapping documentation, "A count /// for a gap area is only used as the line execution count if there are no other regions on a /// line." +#[derive(Debug)] pub struct FunctionCoverage<'tcx> { instance: Instance<'tcx>, source_hash: u64, @@ -49,9 +50,9 @@ impl<'tcx> FunctionCoverage<'tcx> { } fn create(tcx: TyCtxt<'tcx>, instance: Instance<'tcx>, is_used: bool) -> Self { - let coverageinfo = tcx.coverageinfo(instance.def_id()); + let coverageinfo = tcx.coverageinfo(instance.def); debug!( - "FunctionCoverage::new(instance={:?}) has coverageinfo={:?}. is_used={}", + "FunctionCoverage::create(instance={:?}) has coverageinfo={:?}. is_used={}", instance, coverageinfo, is_used ); Self { @@ -113,6 +114,14 @@ impl<'tcx> FunctionCoverage<'tcx> { expression_id, lhs, op, rhs, region ); let expression_index = self.expression_index(u32::from(expression_id)); + debug_assert!( + expression_index.as_usize() < self.expressions.len(), + "expression_index {} is out of range for expressions.len() = {} + for {:?}", + expression_index.as_usize(), + self.expressions.len(), + self, + ); if let Some(previous_expression) = self.expressions[expression_index].replace(Expression { lhs, op, diff --git a/compiler/rustc_codegen_ssa/src/debuginfo/type_names.rs b/compiler/rustc_codegen_ssa/src/debuginfo/type_names.rs index d1bbf74307..7b4b0821c4 100644 --- a/compiler/rustc_codegen_ssa/src/debuginfo/type_names.rs +++ b/compiler/rustc_codegen_ssa/src/debuginfo/type_names.rs @@ -3,7 +3,8 @@ use rustc_data_structures::fx::FxHashSet; use rustc_hir as hir; use rustc_hir::def_id::DefId; -use rustc_middle::ty::{self, subst::SubstsRef, Ty, TyCtxt}; +use rustc_middle::ty::{self, subst::SubstsRef, AdtDef, Ty, TyCtxt}; +use rustc_target::abi::{TagEncoding, Variants}; use std::fmt::Write; @@ -45,8 +46,12 @@ pub fn push_debuginfo_type_name<'tcx>( ty::Float(float_ty) => output.push_str(float_ty.name_str()), ty::Foreign(def_id) => push_item_name(tcx, def_id, qualified, output), ty::Adt(def, substs) => { - push_item_name(tcx, def.did, qualified, output); - push_type_params(tcx, substs, output, visited); + if def.is_enum() && cpp_like_names { + msvc_enum_fallback(tcx, t, def, substs, output, visited); + } else { + push_item_name(tcx, def.did, qualified, output); + push_type_params(tcx, substs, output, visited); + } } ty::Tuple(component_types) => { if cpp_like_names { @@ -233,6 +238,54 @@ pub fn push_debuginfo_type_name<'tcx>( } } + /// MSVC names enums differently than other platforms so that the debugging visualization + // format (natvis) is able to understand enums and render the active variant correctly in the + // debugger. For more information, look in `src/etc/natvis/intrinsic.natvis` and + // `EnumMemberDescriptionFactor::create_member_descriptions`. + fn msvc_enum_fallback( + tcx: TyCtxt<'tcx>, + ty: Ty<'tcx>, + def: &AdtDef, + substs: SubstsRef<'tcx>, + output: &mut String, + visited: &mut FxHashSet>, + ) { + let layout = tcx.layout_of(tcx.param_env(def.did).and(ty)).expect("layout error"); + + if let Variants::Multiple { + tag_encoding: TagEncoding::Niche { dataful_variant, .. }, + tag, + variants, + .. + } = &layout.variants + { + let dataful_variant_layout = &variants[*dataful_variant]; + + // calculate the range of values for the dataful variant + let dataful_discriminant_range = + &dataful_variant_layout.largest_niche.as_ref().unwrap().scalar.valid_range; + + let min = dataful_discriminant_range.start(); + let min = tag.value.size(&tcx).truncate(*min); + + let max = dataful_discriminant_range.end(); + let max = tag.value.size(&tcx).truncate(*max); + + output.push_str("enum$<"); + push_item_name(tcx, def.did, true, output); + push_type_params(tcx, substs, output, visited); + + let dataful_variant_name = def.variants[*dataful_variant].ident.as_str(); + + output.push_str(&format!(", {}, {}, {}>", min, max, dataful_variant_name)); + } else { + output.push_str("enum$<"); + push_item_name(tcx, def.did, true, output); + push_type_params(tcx, substs, output, visited); + output.push('>'); + } + } + fn push_item_name(tcx: TyCtxt<'tcx>, def_id: DefId, qualified: bool, output: &mut String) { if qualified { output.push_str(&tcx.crate_name(def_id.krate).as_str()); diff --git a/compiler/rustc_codegen_ssa/src/lib.rs b/compiler/rustc_codegen_ssa/src/lib.rs index b761bb4e19..b6de12fa35 100644 --- a/compiler/rustc_codegen_ssa/src/lib.rs +++ b/compiler/rustc_codegen_ssa/src/lib.rs @@ -1,15 +1,11 @@ #![doc(html_root_url = "https://doc.rust-lang.org/nightly/nightly-rustc/")] #![feature(bool_to_option)] #![feature(box_patterns)] -#![feature(drain_filter)] #![feature(try_blocks)] #![feature(in_band_lifetimes)] #![feature(nll)] -#![cfg_attr(bootstrap, feature(or_patterns))] #![feature(associated_type_bounds)] -#![feature(iter_zip)] #![recursion_limit = "256"] -#![feature(box_syntax)] //! This crate contains codegen code that is used by all codegen backends (LLVM and others). //! The backend-agnostic functions of this crate use functions defined in various traits that @@ -113,11 +109,19 @@ pub struct NativeLib { pub kind: NativeLibKind, pub name: Option, pub cfg: Option, + pub verbatim: Option, + pub dll_imports: Vec, } impl From<&cstore::NativeLib> for NativeLib { fn from(lib: &cstore::NativeLib) -> Self { - NativeLib { kind: lib.kind, name: lib.name, cfg: lib.cfg.clone() } + NativeLib { + kind: lib.kind, + name: lib.name, + cfg: lib.cfg.clone(), + verbatim: lib.verbatim, + dll_imports: lib.dll_imports.clone(), + } } } @@ -131,6 +135,7 @@ impl From<&cstore::NativeLib> for NativeLib { /// and the corresponding properties without referencing information outside of a `CrateInfo`. #[derive(Debug, Encodable, Decodable)] pub struct CrateInfo { + pub local_crate_name: Symbol, pub panic_runtime: Option, pub compiler_builtins: Option, pub profiler_runtime: Option, @@ -144,16 +149,15 @@ pub struct CrateInfo { pub lang_item_to_crate: FxHashMap, pub missing_lang_items: FxHashMap>, pub dependency_formats: Lrc, + pub windows_subsystem: Option, } #[derive(Encodable, Decodable)] pub struct CodegenResults { - pub crate_name: Symbol, pub modules: Vec, pub allocator_module: Option, pub metadata_module: Option, pub metadata: rustc_middle::middle::cstore::EncodedMetadata, - pub windows_subsystem: Option, pub linker_info: back::linker::LinkerInfo, pub crate_info: CrateInfo, } diff --git a/compiler/rustc_codegen_ssa/src/mir/analyze.rs b/compiler/rustc_codegen_ssa/src/mir/analyze.rs index 38e928145a..49b5e8466b 100644 --- a/compiler/rustc_codegen_ssa/src/mir/analyze.rs +++ b/compiler/rustc_codegen_ssa/src/mir/analyze.rs @@ -7,11 +7,8 @@ use rustc_data_structures::graph::dominators::Dominators; use rustc_index::bit_set::BitSet; use rustc_index::vec::{Idx, IndexVec}; use rustc_middle::mir::traversal; -use rustc_middle::mir::visit::{ - MutatingUseContext, NonMutatingUseContext, NonUseContext, PlaceContext, Visitor, -}; +use rustc_middle::mir::visit::{MutatingUseContext, NonMutatingUseContext, PlaceContext, Visitor}; use rustc_middle::mir::{self, Location, TerminatorKind}; -use rustc_middle::ty; use rustc_middle::ty::layout::HasTyCtxt; use rustc_target::abi::LayoutOf; @@ -21,7 +18,12 @@ pub fn non_ssa_locals<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( let mir = fx.mir; let mut analyzer = LocalAnalyzer::new(fx); - analyzer.visit_body(&mir); + // If there exists a local definition that dominates all uses of that local, + // the definition should be visited first. Traverse blocks in preorder which + // is a topological sort of dominance partial order. + for (bb, data) in traversal::preorder(&mir) { + analyzer.visit_basic_block_data(bb, data); + } for (local, decl) in mir.local_decls.iter_enumerated() { let ty = fx.monomorphize(decl.ty); @@ -142,36 +144,7 @@ impl> LocalAnalyzer<'mir, 'a, 'tcx, Bx> { if let mir::ProjectionElem::Deref = elem { // Deref projections typically only read the pointer. - // (the exception being `VarDebugInfo` contexts, handled below) base_context = PlaceContext::NonMutatingUse(NonMutatingUseContext::Copy); - - // Indirect debuginfo requires going through memory, that only - // the debugger accesses, following our emitted DWARF pointer ops. - // - // FIXME(eddyb) Investigate the possibility of relaxing this, but - // note that `llvm.dbg.declare` *must* be used for indirect places, - // even if we start using `llvm.dbg.value` for all other cases, - // as we don't necessarily know when the value changes, but only - // where it lives in memory. - // - // It's possible `llvm.dbg.declare` could support starting from - // a pointer that doesn't point to an `alloca`, but this would - // only be useful if we know the pointer being `Deref`'d comes - // from an immutable place, and if `llvm.dbg.declare` calls - // must be at the very start of the function, then only function - // arguments could contain such pointers. - if context == PlaceContext::NonUse(NonUseContext::VarDebugInfo) { - // We use `NonUseContext::VarDebugInfo` for the base, - // which might not force the base local to memory, - // so we have to do it manually. - self.visit_local(&place_ref.local, context, location); - } - } - - // `NonUseContext::VarDebugInfo` needs to flow all the - // way down to the base local (see `visit_local`). - if context == PlaceContext::NonUse(NonUseContext::VarDebugInfo) { - base_context = context; } self.process_place(&place_base, base_context, location); @@ -186,20 +159,7 @@ impl> LocalAnalyzer<'mir, 'a, 'tcx, Bx> { ); } } else { - // FIXME this is super_place code, is repeated here to avoid cloning place or changing - // visit_place API - let mut context = context; - - if !place_ref.projection.is_empty() { - context = if context.is_mutating_use() { - PlaceContext::MutatingUse(MutatingUseContext::Projection) - } else { - PlaceContext::NonMutatingUse(NonMutatingUseContext::Projection) - }; - } - self.visit_local(&place_ref.local, context, location); - self.visit_projection(*place_ref, context, location); } } } @@ -228,34 +188,6 @@ impl<'mir, 'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> Visitor<'tcx> self.visit_rvalue(rvalue, location); } - fn visit_terminator(&mut self, terminator: &mir::Terminator<'tcx>, location: Location) { - let check = match terminator.kind { - mir::TerminatorKind::Call { func: mir::Operand::Constant(ref c), ref args, .. } => { - match *c.ty().kind() { - ty::FnDef(did, _) => Some((did, args)), - _ => None, - } - } - _ => None, - }; - if let Some((def_id, args)) = check { - if Some(def_id) == self.fx.cx.tcx().lang_items().box_free_fn() { - // box_free(x) shares with `drop x` the property that it - // is not guaranteed to be statically dominated by the - // definition of x, so x must always be in an alloca. - if let mir::Operand::Move(ref place) = args[0] { - self.visit_place( - place, - PlaceContext::MutatingUse(MutatingUseContext::Drop), - location, - ); - } - } - } - - self.super_terminator(terminator, location); - } - fn visit_place(&mut self, place: &mir::Place<'tcx>, context: PlaceContext, location: Location) { debug!("visit_place(place={:?}, context={:?})", place, context); self.process_place(&place.as_ref(), context, location); diff --git a/compiler/rustc_codegen_ssa/src/mir/block.rs b/compiler/rustc_codegen_ssa/src/mir/block.rs index fd3f89a2ae..93200bd1f2 100644 --- a/compiler/rustc_codegen_ssa/src/mir/block.rs +++ b/compiler/rustc_codegen_ssa/src/mir/block.rs @@ -12,7 +12,6 @@ use crate::MemFlags; use rustc_ast as ast; use rustc_hir::lang_items::LangItem; use rustc_index::vec::Idx; -use rustc_middle::mir::interpret::ConstValue; use rustc_middle::mir::AssertKind; use rustc_middle::mir::{self, SwitchTargets}; use rustc_middle::ty::layout::{FnAbiExt, HasTyCtxt}; @@ -21,7 +20,7 @@ use rustc_middle::ty::{self, Instance, Ty, TypeFoldable}; use rustc_span::source_map::Span; use rustc_span::{sym, Symbol}; use rustc_target::abi::call::{ArgAbi, FnAbi, PassMode}; -use rustc_target::abi::{self, LayoutOf}; +use rustc_target::abi::{self, HasDataLayout, LayoutOf}; use rustc_target::spec::abi::Abi; /// Used by `FunctionCx::codegen_terminator` for emitting common patterns @@ -33,13 +32,34 @@ struct TerminatorCodegenHelper<'tcx> { } impl<'a, 'tcx> TerminatorCodegenHelper<'tcx> { - /// Returns the associated funclet from `FunctionCx::funclets` for the - /// `funclet_bb` member if it is not `None`. + /// Returns the appropriate `Funclet` for the current funclet, if on MSVC, + /// either already previously cached, or newly created, by `landing_pad_for`. fn funclet<'b, Bx: BuilderMethods<'a, 'tcx>>( &self, - fx: &'b FunctionCx<'a, 'tcx, Bx>, + fx: &'b mut FunctionCx<'a, 'tcx, Bx>, ) -> Option<&'b Bx::Funclet> { - self.funclet_bb.and_then(|funcl| fx.funclets[funcl].as_ref()) + let funclet_bb = self.funclet_bb?; + if base::wants_msvc_seh(fx.cx.tcx().sess) { + // If `landing_pad_for` hasn't been called yet to create the `Funclet`, + // it has to be now. This may not seem necessary, as RPO should lead + // to all the unwind edges being visited (and so to `landing_pad_for` + // getting called for them), before building any of the blocks inside + // the funclet itself - however, if MIR contains edges that end up not + // being needed in the LLVM IR after monomorphization, the funclet may + // be unreachable, and we don't have yet a way to skip building it in + // such an eventuality (which may be a better solution than this). + if fx.funclets[funclet_bb].is_none() { + fx.landing_pad_for(funclet_bb); + } + + Some( + fx.funclets[funclet_bb] + .as_ref() + .expect("landing_pad_for didn't also create funclets entry"), + ) + } else { + None + } } fn lltarget>( @@ -48,17 +68,17 @@ impl<'a, 'tcx> TerminatorCodegenHelper<'tcx> { target: mir::BasicBlock, ) -> (Bx::BasicBlock, bool) { let span = self.terminator.source_info.span; - let lltarget = fx.blocks[target]; + let lltarget = fx.llbb(target); let target_funclet = fx.cleanup_kinds[target].funclet_bb(target); match (self.funclet_bb, target_funclet) { (None, None) => (lltarget, false), (Some(f), Some(t_f)) if f == t_f || !base::wants_msvc_seh(fx.cx.tcx().sess) => { (lltarget, false) } - // jump *into* cleanup - need a landing pad if GNU - (None, Some(_)) => (fx.landing_pad_to(target), false), + // jump *into* cleanup - need a landing pad if GNU, cleanup pad if MSVC + (None, Some(_)) => (fx.landing_pad_for(target), false), (Some(_), None) => span_bug!(span, "{:?} - jump out of cleanup?", self.terminator), - (Some(_), Some(_)) => (fx.landing_pad_to(target), true), + (Some(_), Some(_)) => (fx.landing_pad_for(target), true), } } @@ -113,13 +133,13 @@ impl<'a, 'tcx> TerminatorCodegenHelper<'tcx> { // If there is a cleanup block and the function we're calling can unwind, then // do an invoke, otherwise do a call. if let Some(cleanup) = cleanup.filter(|_| fn_abi.can_unwind) { - let ret_bx = if let Some((_, target)) = destination { - fx.blocks[target] + let ret_llbb = if let Some((_, target)) = destination { + fx.llbb(target) } else { fx.unreachable_block() }; let invokeret = - bx.invoke(fn_ptr, &llargs, ret_bx, self.llblock(fx, cleanup), self.funclet(fx)); + bx.invoke(fn_ptr, &llargs, ret_llbb, self.llblock(fx, cleanup), self.funclet(fx)); bx.apply_attrs_callsite(&fn_abi, invokeret); if let Some((ret_dest, target)) = destination { @@ -366,7 +386,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { // Create the failure block and the conditional branch to it. let lltarget = helper.llblock(self, target); - let panic_block = self.new_block("panic"); + let panic_block = bx.build_sibling_block("panic"); if expected { bx.cond_br(cond, lltarget, panic_block.llbb()); } else { @@ -825,33 +845,12 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { let const_value = self .eval_mir_constant(value) .unwrap_or_else(|_| span_bug!(span, "asm const cannot be resolved")); - let ty = value.ty(); - let size = bx.layout_of(ty).size; - let scalar = match const_value { - ConstValue::Scalar(s) => s, - _ => span_bug!( - span, - "expected Scalar for promoted asm const, but got {:#?}", - const_value - ), - }; - let value = scalar.assert_bits(size); - let string = match ty.kind() { - ty::Uint(_) => value.to_string(), - ty::Int(int_ty) => { - match int_ty.normalize(bx.tcx().sess.target.pointer_width) { - ty::IntTy::I8 => (value as i8).to_string(), - ty::IntTy::I16 => (value as i16).to_string(), - ty::IntTy::I32 => (value as i32).to_string(), - ty::IntTy::I64 => (value as i64).to_string(), - ty::IntTy::I128 => (value as i128).to_string(), - ty::IntTy::Isize => unreachable!(), - } - } - ty::Float(ty::FloatTy::F32) => f32::from_bits(value as u32).to_string(), - ty::Float(ty::FloatTy::F64) => f64::from_bits(value as u64).to_string(), - _ => span_bug!(span, "asm const has bad type {}", ty), - }; + let string = common::asm_const_to_str( + bx.tcx(), + span, + const_value, + bx.layout_of(value.ty()), + ); InlineAsmOperandRef::Const { string } } mir::InlineAsmOperand::SymFn { ref value } => { @@ -1144,7 +1143,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { let topmost = span.ctxt().outer_expn().expansion_cause().unwrap_or(span); let caller = tcx.sess.source_map().lookup_char_pos(topmost.lo()); let const_loc = tcx.const_caller_location(( - Symbol::intern(&caller.file.name.to_string()), + Symbol::intern(&caller.file.name.prefer_remapped().to_string_lossy()), caller.line as u32, caller.col_display as u32 + 1, )); @@ -1192,38 +1191,88 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { } } - /// Returns the landing-pad wrapper around the given basic block. - /// - /// No-op in MSVC SEH scheme. - fn landing_pad_to(&mut self, target_bb: mir::BasicBlock) -> Bx::BasicBlock { - if let Some(block) = self.landing_pads[target_bb] { - return block; + /// Returns the landing/cleanup pad wrapper around the given basic block. + // FIXME(eddyb) rename this to `eh_pad_for`. + fn landing_pad_for(&mut self, bb: mir::BasicBlock) -> Bx::BasicBlock { + if let Some(landing_pad) = self.landing_pads[bb] { + return landing_pad; } - let block = self.blocks[target_bb]; - let landing_pad = self.landing_pad_uncached(block); - self.landing_pads[target_bb] = Some(landing_pad); + let landing_pad = self.landing_pad_for_uncached(bb); + self.landing_pads[bb] = Some(landing_pad); landing_pad } - fn landing_pad_uncached(&mut self, target_bb: Bx::BasicBlock) -> Bx::BasicBlock { + // FIXME(eddyb) rename this to `eh_pad_for_uncached`. + fn landing_pad_for_uncached(&mut self, bb: mir::BasicBlock) -> Bx::BasicBlock { + let llbb = self.llbb(bb); if base::wants_msvc_seh(self.cx.sess()) { - span_bug!(self.mir.span, "landing pad was not inserted?") - } - - let mut bx = self.new_block("cleanup"); + let funclet; + let ret_llbb; + match self.mir[bb].terminator.as_ref().map(|t| &t.kind) { + // This is a basic block that we're aborting the program for, + // notably in an `extern` function. These basic blocks are inserted + // so that we assert that `extern` functions do indeed not panic, + // and if they do we abort the process. + // + // On MSVC these are tricky though (where we're doing funclets). If + // we were to do a cleanuppad (like below) the normal functions like + // `longjmp` would trigger the abort logic, terminating the + // program. Instead we insert the equivalent of `catch(...)` for C++ + // which magically doesn't trigger when `longjmp` files over this + // frame. + // + // Lots more discussion can be found on #48251 but this codegen is + // modeled after clang's for: + // + // try { + // foo(); + // } catch (...) { + // bar(); + // } + Some(&mir::TerminatorKind::Abort) => { + let mut cs_bx = self.new_block(&format!("cs_funclet{:?}", bb)); + let mut cp_bx = self.new_block(&format!("cp_funclet{:?}", bb)); + ret_llbb = cs_bx.llbb(); + + let cs = cs_bx.catch_switch(None, None, 1); + cs_bx.add_handler(cs, cp_bx.llbb()); + + // The "null" here is actually a RTTI type descriptor for the + // C++ personality function, but `catch (...)` has no type so + // it's null. The 64 here is actually a bitfield which + // represents that this is a catch-all block. + let null = cp_bx.const_null( + cp_bx.type_i8p_ext(cp_bx.cx().data_layout().instruction_address_space), + ); + let sixty_four = cp_bx.const_i32(64); + funclet = cp_bx.catch_pad(cs, &[null, sixty_four, null]); + cp_bx.br(llbb); + } + _ => { + let mut cleanup_bx = self.new_block(&format!("funclet_{:?}", bb)); + ret_llbb = cleanup_bx.llbb(); + funclet = cleanup_bx.cleanup_pad(None, &[]); + cleanup_bx.br(llbb); + } + } + self.funclets[bb] = Some(funclet); + ret_llbb + } else { + let mut bx = self.new_block("cleanup"); - let llpersonality = self.cx.eh_personality(); - let llretty = self.landing_pad_type(); - let lp = bx.landing_pad(llretty, llpersonality, 1); - bx.set_cleanup(lp); + let llpersonality = self.cx.eh_personality(); + let llretty = self.landing_pad_type(); + let lp = bx.landing_pad(llretty, llpersonality, 1); + bx.set_cleanup(lp); - let slot = self.get_personality_slot(&mut bx); - slot.storage_live(&mut bx); - Pair(bx.extract_value(lp, 0), bx.extract_value(lp, 1)).store(&mut bx, slot); + let slot = self.get_personality_slot(&mut bx); + slot.storage_live(&mut bx); + Pair(bx.extract_value(lp, 0), bx.extract_value(lp, 1)).store(&mut bx, slot); - bx.br(target_bb); - bx.llbb() + bx.br(llbb); + bx.llbb() + } } fn landing_pad_type(&self) -> Bx::Type { @@ -1240,14 +1289,29 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { }) } - pub fn new_block(&self, name: &str) -> Bx { - Bx::new_block(self.cx, self.llfn, name) + // FIXME(eddyb) replace with `build_sibling_block`/`append_sibling_block` + // (which requires having a `Bx` already, and not all callers do). + fn new_block(&self, name: &str) -> Bx { + let llbb = Bx::append_block(self.cx, self.llfn, name); + Bx::build(self.cx, llbb) + } + + /// Get the backend `BasicBlock` for a MIR `BasicBlock`, either already + /// cached in `self.cached_llbbs`, or created on demand (and cached). + // FIXME(eddyb) rename `llbb` and other `ll`-prefixed things to use a + // more backend-agnostic prefix such as `cg` (i.e. this would be `cgbb`). + pub fn llbb(&mut self, bb: mir::BasicBlock) -> Bx::BasicBlock { + self.cached_llbbs[bb].unwrap_or_else(|| { + // FIXME(eddyb) only name the block if `fewer_names` is `false`. + let llbb = Bx::append_block(self.cx, self.llfn, &format!("{:?}", bb)); + self.cached_llbbs[bb] = Some(llbb); + llbb + }) } - pub fn build_block(&self, bb: mir::BasicBlock) -> Bx { - let mut bx = Bx::with_cx(self.cx); - bx.position_at_end(self.blocks[bb]); - bx + pub fn build_block(&mut self, bb: mir::BasicBlock) -> Bx { + let llbb = self.llbb(bb); + Bx::build(self.cx, llbb) } fn make_return_dest( diff --git a/compiler/rustc_codegen_ssa/src/mir/coverageinfo.rs b/compiler/rustc_codegen_ssa/src/mir/coverageinfo.rs index 621ec0519c..a283bf1de7 100644 --- a/compiler/rustc_codegen_ssa/src/mir/coverageinfo.rs +++ b/compiler/rustc_codegen_ssa/src/mir/coverageinfo.rs @@ -31,7 +31,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { bx.add_coverage_counter(instance, id, code_region); } - let coverageinfo = bx.tcx().coverageinfo(instance.def_id()); + let coverageinfo = bx.tcx().coverageinfo(instance.def); let fn_name = bx.get_pgo_func_name_var(instance); let hash = bx.const_u64(function_source_hash); diff --git a/compiler/rustc_codegen_ssa/src/mir/mod.rs b/compiler/rustc_codegen_ssa/src/mir/mod.rs index 91df67b53d..3bbc481b61 100644 --- a/compiler/rustc_codegen_ssa/src/mir/mod.rs +++ b/compiler/rustc_codegen_ssa/src/mir/mod.rs @@ -1,4 +1,3 @@ -use crate::base; use crate::traits::*; use rustc_errors::ErrorReported; use rustc_middle::mir; @@ -6,14 +5,12 @@ use rustc_middle::mir::interpret::ErrorHandled; use rustc_middle::ty::layout::{FnAbiExt, HasTyCtxt, TyAndLayout}; use rustc_middle::ty::{self, Instance, Ty, TypeFoldable}; use rustc_target::abi::call::{FnAbi, PassMode}; -use rustc_target::abi::HasDataLayout; use std::iter; use rustc_index::bit_set::BitSet; use rustc_index::vec::IndexVec; -use self::analyze::CleanupKind; use self::debuginfo::{FunctionDebugContext, PerLocalVarDebugInfo}; use self::place::PlaceRef; use rustc_middle::mir::traversal; @@ -43,18 +40,22 @@ pub struct FunctionCx<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> { /// then later loaded when generating the DIVERGE_BLOCK. personality_slot: Option>, - /// A `Block` for each MIR `BasicBlock` - blocks: IndexVec, + /// A backend `BasicBlock` for each MIR `BasicBlock`, created lazily + /// as-needed (e.g. RPO reaching it or another block branching to it). + // FIXME(eddyb) rename `llbbs` and other `ll`-prefixed things to use a + // more backend-agnostic prefix such as `cg` (i.e. this would be `cgbbs`). + cached_llbbs: IndexVec>, /// The funclet status of each basic block cleanup_kinds: IndexVec, - /// When targeting MSVC, this stores the cleanup info for each funclet - /// BB. This is initialized as we compute the funclets' head block in RPO. + /// When targeting MSVC, this stores the cleanup info for each funclet BB. + /// This is initialized at the same time as the `landing_pads` entry for the + /// funclets' head block, i.e. when needed by an unwind / `cleanup_ret` edge. funclets: IndexVec>, - /// This stores the landing-pad block for a given BB, computed lazily on GNU - /// and eagerly on MSVC. + /// This stores the cached landing/cleanup pad block for a given BB. + // FIXME(eddyb) rename this to `eh_pads`. landing_pads: IndexVec>, /// Cached unreachable block @@ -143,7 +144,8 @@ pub fn codegen_mir<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( let debug_context = cx.create_function_debug_context(instance, &fn_abi, llfn, &mir); - let mut bx = Bx::new_block(cx, llfn, "start"); + let start_llbb = Bx::append_block(cx, llfn, "start"); + let mut bx = Bx::build(cx, start_llbb); if mir.basic_blocks().iter().any(|bb| bb.is_cleanup) { bx.set_personality_fn(cx.eh_personality()); @@ -153,19 +155,18 @@ pub fn codegen_mir<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( // Allocate a `Block` for every basic block, except // the start block, if nothing loops back to it. let reentrant_start_block = !mir.predecessors()[mir::START_BLOCK].is_empty(); - let block_bxs: IndexVec = mir - .basic_blocks() - .indices() - .map(|bb| { - if bb == mir::START_BLOCK && !reentrant_start_block { - bx.llbb() - } else { - bx.build_sibling_block(&format!("{:?}", bb)).llbb() - } - }) - .collect(); + let cached_llbbs: IndexVec> = + mir.basic_blocks() + .indices() + .map(|bb| { + if bb == mir::START_BLOCK && !reentrant_start_block { + Some(start_llbb) + } else { + None + } + }) + .collect(); - let (landing_pads, funclets) = create_funclets(&mir, &mut bx, &cleanup_kinds, &block_bxs); let mut fx = FunctionCx { instance, mir, @@ -173,11 +174,11 @@ pub fn codegen_mir<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( fn_abi, cx, personality_slot: None, - blocks: block_bxs, + cached_llbbs, unreachable_block: None, cleanup_kinds, - landing_pads, - funclets, + landing_pads: IndexVec::from_elem(None, mir.basic_blocks()), + funclets: IndexVec::from_fn_n(|_| None, mir.basic_blocks().len()), locals: IndexVec::new(), debug_context, per_local_var_debug_info: None, @@ -248,100 +249,14 @@ pub fn codegen_mir<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( // Branch to the START block, if it's not the entry block. if reentrant_start_block { - bx.br(fx.blocks[mir::START_BLOCK]); + bx.br(fx.llbb(mir::START_BLOCK)); } - let rpo = traversal::reverse_postorder(&mir); - let mut visited = BitSet::new_empty(mir.basic_blocks().len()); - // Codegen the body of each block using reverse postorder - for (bb, _) in rpo { - visited.insert(bb.index()); + // FIXME(eddyb) reuse RPO iterator between `analysis` and this. + for (bb, _) in traversal::reverse_postorder(&mir) { fx.codegen_block(bb); } - - // Remove blocks that haven't been visited, or have no - // predecessors. - for bb in mir.basic_blocks().indices() { - // Unreachable block - if !visited.contains(bb.index()) { - debug!("codegen_mir: block {:?} was not visited", bb); - unsafe { - bx.delete_basic_block(fx.blocks[bb]); - } - } - } -} - -fn create_funclets<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( - mir: &'tcx mir::Body<'tcx>, - bx: &mut Bx, - cleanup_kinds: &IndexVec, - block_bxs: &IndexVec, -) -> ( - IndexVec>, - IndexVec>, -) { - iter::zip(block_bxs.iter_enumerated(), cleanup_kinds) - .map(|((bb, &llbb), cleanup_kind)| { - match *cleanup_kind { - CleanupKind::Funclet if base::wants_msvc_seh(bx.sess()) => {} - _ => return (None, None), - } - - let funclet; - let ret_llbb; - match mir[bb].terminator.as_ref().map(|t| &t.kind) { - // This is a basic block that we're aborting the program for, - // notably in an `extern` function. These basic blocks are inserted - // so that we assert that `extern` functions do indeed not panic, - // and if they do we abort the process. - // - // On MSVC these are tricky though (where we're doing funclets). If - // we were to do a cleanuppad (like below) the normal functions like - // `longjmp` would trigger the abort logic, terminating the - // program. Instead we insert the equivalent of `catch(...)` for C++ - // which magically doesn't trigger when `longjmp` files over this - // frame. - // - // Lots more discussion can be found on #48251 but this codegen is - // modeled after clang's for: - // - // try { - // foo(); - // } catch (...) { - // bar(); - // } - Some(&mir::TerminatorKind::Abort) => { - let mut cs_bx = bx.build_sibling_block(&format!("cs_funclet{:?}", bb)); - let mut cp_bx = bx.build_sibling_block(&format!("cp_funclet{:?}", bb)); - ret_llbb = cs_bx.llbb(); - - let cs = cs_bx.catch_switch(None, None, 1); - cs_bx.add_handler(cs, cp_bx.llbb()); - - // The "null" here is actually a RTTI type descriptor for the - // C++ personality function, but `catch (...)` has no type so - // it's null. The 64 here is actually a bitfield which - // represents that this is a catch-all block. - let null = bx.const_null( - bx.type_i8p_ext(bx.cx().data_layout().instruction_address_space), - ); - let sixty_four = bx.const_i32(64); - funclet = cp_bx.catch_pad(cs, &[null, sixty_four, null]); - cp_bx.br(llbb); - } - _ => { - let mut cleanup_bx = bx.build_sibling_block(&format!("funclet_{:?}", bb)); - ret_llbb = cleanup_bx.llbb(); - funclet = cleanup_bx.cleanup_pad(None, &[]); - cleanup_bx.br(llbb); - } - }; - - (Some(ret_llbb), Some(funclet)) - }) - .unzip() } /// Produces, for each argument, a `Value` pointing at the diff --git a/compiler/rustc_codegen_ssa/src/mir/rvalue.rs b/compiler/rustc_codegen_ssa/src/mir/rvalue.rs index 9917c23f12..530de3de9e 100644 --- a/compiler/rustc_codegen_ssa/src/mir/rvalue.rs +++ b/compiler/rustc_codegen_ssa/src/mir/rvalue.rs @@ -14,7 +14,6 @@ use rustc_middle::ty::cast::{CastTy, IntTy}; use rustc_middle::ty::layout::HasTyCtxt; use rustc_middle::ty::{self, adjustment::PointerCast, Instance, Ty, TyCtxt}; use rustc_span::source_map::{Span, DUMMY_SP}; -use rustc_span::symbol::sym; use rustc_target::abi::{Abi, Int, LayoutOf, Variants}; impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { @@ -187,9 +186,6 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { mir::CastKind::Pointer(PointerCast::ReifyFnPointer) => { match *operand.layout.ty.kind() { ty::FnDef(def_id, substs) => { - if bx.cx().tcx().has_attr(def_id, sym::rustc_args_required_const) { - bug!("reifying a fn ptr that requires const arguments"); - } let instance = ty::Instance::resolve_for_fn_ptr( bx.tcx(), ty::ParamEnv::reveal_all(), diff --git a/compiler/rustc_codegen_ssa/src/mono_item.rs b/compiler/rustc_codegen_ssa/src/mono_item.rs index 8e79193759..48d753e0d8 100644 --- a/compiler/rustc_codegen_ssa/src/mono_item.rs +++ b/compiler/rustc_codegen_ssa/src/mono_item.rs @@ -1,10 +1,11 @@ use crate::base; +use crate::common; use crate::traits::*; use rustc_hir as hir; +use rustc_middle::mir::mono::MonoItem; use rustc_middle::mir::mono::{Linkage, Visibility}; use rustc_middle::ty::layout::HasTyCtxt; - -use rustc_middle::mir::mono::MonoItem; +use rustc_target::abi::LayoutOf; pub trait MonoItemExt<'a, 'tcx> { fn define>(&self, cx: &'a Bx::CodegenCx); @@ -32,8 +33,35 @@ impl<'a, 'tcx: 'a> MonoItemExt<'a, 'tcx> for MonoItem<'tcx> { } MonoItem::GlobalAsm(item_id) => { let item = cx.tcx().hir().item(item_id); - if let hir::ItemKind::GlobalAsm(ref ga) = item.kind { - cx.codegen_global_asm(ga); + if let hir::ItemKind::GlobalAsm(ref asm) = item.kind { + let operands: Vec<_> = asm + .operands + .iter() + .map(|(op, op_sp)| match *op { + hir::InlineAsmOperand::Const { ref anon_const } => { + let anon_const_def_id = + cx.tcx().hir().local_def_id(anon_const.hir_id).to_def_id(); + let const_value = + cx.tcx().const_eval_poly(anon_const_def_id).unwrap_or_else( + |_| span_bug!(*op_sp, "asm const cannot be resolved"), + ); + let ty = cx + .tcx() + .typeck_body(anon_const.body) + .node_type(anon_const.hir_id); + let string = common::asm_const_to_str( + cx.tcx(), + *op_sp, + const_value, + cx.layout_of(ty), + ); + GlobalAsmOperandRef::Const { string } + } + _ => span_bug!(*op_sp, "invalid operand type for global_asm!"), + }) + .collect(); + + cx.codegen_global_asm(asm.template, &operands, asm.options, asm.line_spans); } else { span_bug!(item.span, "Mismatch between hir::Item type and MonoItem type") } diff --git a/compiler/rustc_codegen_ssa/src/target_features.rs b/compiler/rustc_codegen_ssa/src/target_features.rs index 4e987908b4..c89d42ecc5 100644 --- a/compiler/rustc_codegen_ssa/src/target_features.rs +++ b/compiler/rustc_codegen_ssa/src/target_features.rs @@ -17,6 +17,8 @@ const ARM_ALLOWED_FEATURES: &[(&str, Option)] = &[ ("neon", Some(sym::arm_target_feature)), ("crc", Some(sym::arm_target_feature)), ("crypto", Some(sym::arm_target_feature)), + ("aes", Some(sym::arm_target_feature)), + ("sha2", Some(sym::arm_target_feature)), ("v5te", Some(sym::arm_target_feature)), ("v6", Some(sym::arm_target_feature)), ("v6k", Some(sym::arm_target_feature)), @@ -33,22 +35,95 @@ const ARM_ALLOWED_FEATURES: &[(&str, Option)] = &[ ("thumb-mode", Some(sym::arm_target_feature)), ]; +// Commented features are not available in LLVM 10.0, or have since been renamed const AARCH64_ALLOWED_FEATURES: &[(&str, Option)] = &[ - ("fp", Some(sym::aarch64_target_feature)), + // FEAT_AdvSimd ("neon", Some(sym::aarch64_target_feature)), + // FEAT_FP + ("fp", Some(sym::aarch64_target_feature)), + // FEAT_FP16 + ("fp16", Some(sym::aarch64_target_feature)), + // FEAT_SVE ("sve", Some(sym::aarch64_target_feature)), + // FEAT_CRC ("crc", Some(sym::aarch64_target_feature)), + // Cryptographic extension ("crypto", Some(sym::aarch64_target_feature)), + // FEAT_RAS ("ras", Some(sym::aarch64_target_feature)), + // FEAT_LSE ("lse", Some(sym::aarch64_target_feature)), + // FEAT_RDM ("rdm", Some(sym::aarch64_target_feature)), - ("fp16", Some(sym::aarch64_target_feature)), + // FEAT_RCPC ("rcpc", Some(sym::aarch64_target_feature)), + // FEAT_RCPC2 + ("rcpc2", Some(sym::aarch64_target_feature)), + // FEAT_DotProd ("dotprod", Some(sym::aarch64_target_feature)), + // FEAT_TME ("tme", Some(sym::aarch64_target_feature)), + // FEAT_FHM + ("fhm", Some(sym::aarch64_target_feature)), + // FEAT_DIT + ("dit", Some(sym::aarch64_target_feature)), + // FEAT_FLAGM + // ("flagm", Some(sym::aarch64_target_feature)), + // FEAT_SSBS + ("ssbs", Some(sym::aarch64_target_feature)), + // FEAT_SB + ("sb", Some(sym::aarch64_target_feature)), + // FEAT_PAUTH + // ("pauth", Some(sym::aarch64_target_feature)), + // FEAT_DPB + ("dpb", Some(sym::aarch64_target_feature)), + // FEAT_DPB2 + ("dpb2", Some(sym::aarch64_target_feature)), + // FEAT_SVE2 + ("sve2", Some(sym::aarch64_target_feature)), + // FEAT_SVE2_AES + ("sve2-aes", Some(sym::aarch64_target_feature)), + // FEAT_SVE2_SM4 + ("sve2-sm4", Some(sym::aarch64_target_feature)), + // FEAT_SVE2_SHA3 + ("sve2-sha3", Some(sym::aarch64_target_feature)), + // FEAT_SVE2_BitPerm + ("sve2-bitperm", Some(sym::aarch64_target_feature)), + // FEAT_FRINTTS + ("frintts", Some(sym::aarch64_target_feature)), + // FEAT_I8MM + // ("i8mm", Some(sym::aarch64_target_feature)), + // FEAT_F32MM + // ("f32mm", Some(sym::aarch64_target_feature)), + // FEAT_F64MM + // ("f64mm", Some(sym::aarch64_target_feature)), + // FEAT_BF16 + // ("bf16", Some(sym::aarch64_target_feature)), + // FEAT_RAND + ("rand", Some(sym::aarch64_target_feature)), + // FEAT_BTI + ("bti", Some(sym::aarch64_target_feature)), + // FEAT_MTE + ("mte", Some(sym::aarch64_target_feature)), + // FEAT_JSCVT + ("jsconv", Some(sym::aarch64_target_feature)), + // FEAT_FCMA + ("fcma", Some(sym::aarch64_target_feature)), + // FEAT_AES + ("aes", Some(sym::aarch64_target_feature)), + // FEAT_SHA1 & FEAT_SHA256 + ("sha2", Some(sym::aarch64_target_feature)), + // FEAT_SHA512 & FEAT_SHA3 + ("sha3", Some(sym::aarch64_target_feature)), + // FEAT_SM3 & FEAT_SM4 + ("sm4", Some(sym::aarch64_target_feature)), ("v8.1a", Some(sym::aarch64_target_feature)), ("v8.2a", Some(sym::aarch64_target_feature)), ("v8.3a", Some(sym::aarch64_target_feature)), + ("v8.4a", Some(sym::aarch64_target_feature)), + ("v8.5a", Some(sym::aarch64_target_feature)), + // ("v8.6a", Some(sym::aarch64_target_feature)), + // ("v8.7a", Some(sym::aarch64_target_feature)), ]; const X86_ALLOWED_FEATURES: &[(&str, Option)] = &[ @@ -130,11 +205,13 @@ const RISCV_ALLOWED_FEATURES: &[(&str, Option)] = &[ ]; const WASM_ALLOWED_FEATURES: &[(&str, Option)] = &[ - ("simd128", Some(sym::wasm_target_feature)), + ("simd128", None), ("atomics", Some(sym::wasm_target_feature)), ("nontrapping-fptoint", Some(sym::wasm_target_feature)), ]; +const BPF_ALLOWED_FEATURES: &[(&str, Option)] = &[("alu32", Some(sym::bpf_target_feature))]; + /// When rustdoc is running, provide a list of all known features so that all their respective /// primitives may be documented. /// @@ -149,6 +226,7 @@ pub fn all_known_features() -> impl Iterator &'static [(&'static str, Opt "powerpc" | "powerpc64" => POWERPC_ALLOWED_FEATURES, "riscv32" | "riscv64" => RISCV_ALLOWED_FEATURES, "wasm32" | "wasm64" => WASM_ALLOWED_FEATURES, + "bpf" => BPF_ALLOWED_FEATURES, _ => &[], } } diff --git a/compiler/rustc_codegen_ssa/src/traits/asm.rs b/compiler/rustc_codegen_ssa/src/traits/asm.rs index 69931935c4..86f2781a76 100644 --- a/compiler/rustc_codegen_ssa/src/traits/asm.rs +++ b/compiler/rustc_codegen_ssa/src/traits/asm.rs @@ -3,7 +3,7 @@ use crate::mir::operand::OperandRef; use crate::mir::place::PlaceRef; use rustc_ast::{InlineAsmOptions, InlineAsmTemplatePiece}; use rustc_hir::def_id::DefId; -use rustc_hir::{GlobalAsm, LlvmInlineAsmInner}; +use rustc_hir::LlvmInlineAsmInner; use rustc_middle::ty::Instance; use rustc_span::Span; use rustc_target::asm::InlineAsmRegOrRegClass; @@ -36,6 +36,11 @@ pub enum InlineAsmOperandRef<'tcx, B: BackendTypes + ?Sized> { }, } +#[derive(Debug)] +pub enum GlobalAsmOperandRef { + Const { string: String }, +} + pub trait AsmBuilderMethods<'tcx>: BackendTypes { /// Take an inline assembly expression and splat it out via LLVM fn codegen_llvm_inline_asm( @@ -57,5 +62,11 @@ pub trait AsmBuilderMethods<'tcx>: BackendTypes { } pub trait AsmMethods { - fn codegen_global_asm(&self, ga: &GlobalAsm); + fn codegen_global_asm( + &self, + template: &[InlineAsmTemplatePiece], + operands: &[GlobalAsmOperandRef], + options: InlineAsmOptions, + line_spans: &[Span], + ); } diff --git a/compiler/rustc_codegen_ssa/src/traits/backend.rs b/compiler/rustc_codegen_ssa/src/traits/backend.rs index f28db2fe84..dc4146ec7b 100644 --- a/compiler/rustc_codegen_ssa/src/traits/backend.rs +++ b/compiler/rustc_codegen_ssa/src/traits/backend.rs @@ -63,9 +63,16 @@ pub trait CodegenBackend { None } - fn metadata_loader(&self) -> Box; - fn provide(&self, _providers: &mut Providers); - fn provide_extern(&self, _providers: &mut Providers); + /// The metadata loader used to load rlib and dylib metadata. + /// + /// Alternative codegen backends may want to use different rlib or dylib formats than the + /// default native static archives and dynamic libraries. + fn metadata_loader(&self) -> Box { + Box::new(crate::back::metadata::DefaultMetadataLoader) + } + + fn provide(&self, _providers: &mut Providers) {} + fn provide_extern(&self, _providers: &mut Providers) {} fn codegen_crate<'tcx>( &self, tcx: TyCtxt<'tcx>, diff --git a/compiler/rustc_codegen_ssa/src/traits/builder.rs b/compiler/rustc_codegen_ssa/src/traits/builder.rs index 1bc05f30e5..29b2db5d4d 100644 --- a/compiler/rustc_codegen_ssa/src/traits/builder.rs +++ b/compiler/rustc_codegen_ssa/src/traits/builder.rs @@ -40,14 +40,21 @@ pub trait BuilderMethods<'a, 'tcx>: + HasParamEnv<'tcx> + HasTargetSpec { - fn new_block<'b>(cx: &'a Self::CodegenCx, llfn: Self::Function, name: &'b str) -> Self; - fn with_cx(cx: &'a Self::CodegenCx) -> Self; - fn build_sibling_block(&self, name: &str) -> Self; + fn build(cx: &'a Self::CodegenCx, llbb: Self::BasicBlock) -> Self; + fn cx(&self) -> &Self::CodegenCx; fn llbb(&self) -> Self::BasicBlock; + fn set_span(&mut self, span: Span); - fn position_at_end(&mut self, llbb: Self::BasicBlock); + // FIXME(eddyb) replace uses of this with `append_sibling_block`. + fn append_block(cx: &'a Self::CodegenCx, llfn: Self::Function, name: &str) -> Self::BasicBlock; + + fn append_sibling_block(&mut self, name: &str) -> Self::BasicBlock; + + // FIXME(eddyb) replace with callers using `append_sibling_block`. + fn build_sibling_block(&mut self, name: &str) -> Self; + fn ret_void(&mut self); fn ret(&mut self, v: Self::Value); fn br(&mut self, dest: Self::BasicBlock); @@ -291,6 +298,5 @@ pub trait BuilderMethods<'a, 'tcx>: ) -> Self::Value; fn zext(&mut self, val: Self::Value, dest_ty: Self::Type) -> Self::Value; - unsafe fn delete_basic_block(&mut self, bb: Self::BasicBlock); fn do_not_inline(&mut self, llret: Self::Value); } diff --git a/compiler/rustc_codegen_ssa/src/traits/mod.rs b/compiler/rustc_codegen_ssa/src/traits/mod.rs index be2e0ea230..c529fbbf51 100644 --- a/compiler/rustc_codegen_ssa/src/traits/mod.rs +++ b/compiler/rustc_codegen_ssa/src/traits/mod.rs @@ -29,7 +29,7 @@ mod type_; mod write; pub use self::abi::AbiBuilderMethods; -pub use self::asm::{AsmBuilderMethods, AsmMethods, InlineAsmOperandRef}; +pub use self::asm::{AsmBuilderMethods, AsmMethods, GlobalAsmOperandRef, InlineAsmOperandRef}; pub use self::backend::{Backend, BackendTypes, CodegenBackend, ExtraBackendMethods}; pub use self::builder::{BuilderMethods, OverflowOp}; pub use self::consts::ConstMethods; diff --git a/compiler/rustc_codegen_ssa/src/traits/write.rs b/compiler/rustc_codegen_ssa/src/traits/write.rs index 264e7c2aa9..93fbee2b49 100644 --- a/compiler/rustc_codegen_ssa/src/traits/write.rs +++ b/compiler/rustc_codegen_ssa/src/traits/write.rs @@ -58,7 +58,7 @@ pub trait WriteBackendMethods: 'static + Sized + Clone { llmod: &ModuleCodegen, config: &ModuleConfig, thin: bool, - ); + ) -> Result<(), FatalError>; } pub trait ThinBufferMethods: Send + Sync { diff --git a/compiler/rustc_data_structures/Cargo.toml b/compiler/rustc_data_structures/Cargo.toml index aa95ecbdaf..c35a164bb3 100644 --- a/compiler/rustc_data_structures/Cargo.toml +++ b/compiler/rustc_data_structures/Cargo.toml @@ -34,7 +34,7 @@ tempfile = "3.2" version = "0.11" [target.'cfg(windows)'.dependencies] -winapi = { version = "0.3", features = ["fileapi", "psapi"] } +winapi = { version = "0.3", features = ["fileapi", "psapi", "winerror"] } [target.'cfg(not(target_arch = "wasm32"))'.dependencies] memmap2 = "0.2.1" diff --git a/compiler/rustc_data_structures/src/box_region.rs b/compiler/rustc_data_structures/src/box_region.rs deleted file mode 100644 index eb6f4e8213..0000000000 --- a/compiler/rustc_data_structures/src/box_region.rs +++ /dev/null @@ -1,169 +0,0 @@ -//! This module provides a way to deal with self-referential data. -//! -//! The main idea is to allocate such data in a generator frame and then -//! give access to it by executing user-provided closures inside that generator. -//! The module provides a safe abstraction for the latter task. -//! -//! The interface consists of two exported macros meant to be used together: -//! * `declare_box_region_type` wraps a generator inside a struct with `access` -//! method which accepts closures. -//! * `box_region_allow_access` is a helper which should be called inside -//! a generator to actually execute those closures. - -use std::marker::PhantomData; -use std::ops::{Generator, GeneratorState}; -use std::pin::Pin; - -#[derive(Copy, Clone)] -pub struct AccessAction(*mut dyn FnMut()); - -impl AccessAction { - pub fn get(self) -> *mut dyn FnMut() { - self.0 - } -} - -#[derive(Copy, Clone)] -pub enum Action { - Initial, - Access(AccessAction), - Complete, -} - -pub struct PinnedGenerator { - generator: Pin, Return = R>>>, -} - -impl PinnedGenerator { - pub fn new, Return = R> + 'static>( - generator: T, - ) -> (I, Self) { - let mut result = PinnedGenerator { generator: Box::pin(generator) }; - - // Run it to the first yield to set it up - let init = match Pin::new(&mut result.generator).resume(Action::Initial) { - GeneratorState::Yielded(YieldType::Initial(y)) => y, - _ => panic!(), - }; - - (init, result) - } - - pub unsafe fn access(&mut self, closure: *mut dyn FnMut()) { - // Call the generator, which in turn will call the closure - if let GeneratorState::Complete(_) = - Pin::new(&mut self.generator).resume(Action::Access(AccessAction(closure))) - { - panic!() - } - } - - pub fn complete(&mut self) -> R { - // Tell the generator we want it to complete, consuming it and yielding a result - let result = Pin::new(&mut self.generator).resume(Action::Complete); - if let GeneratorState::Complete(r) = result { r } else { panic!() } - } -} - -#[derive(PartialEq)] -pub struct Marker(PhantomData); - -impl Marker { - pub unsafe fn new() -> Self { - Marker(PhantomData) - } -} - -pub enum YieldType { - Initial(I), - Accessor(Marker), -} - -#[macro_export] -#[allow_internal_unstable(fn_traits)] -macro_rules! declare_box_region_type { - (impl $v:vis - $name: ident, - $yield_type:ty, - for($($lifetimes:tt)*), - ($($args:ty),*) -> ($reti:ty, $retc:ty) - ) => { - $v struct $name($crate::box_region::PinnedGenerator< - $reti, - for<$($lifetimes)*> fn(($($args,)*)), - $retc - >); - - impl $name { - fn new + 'static>( - generator: T - ) -> ($reti, Self) { - let (initial, pinned) = $crate::box_region::PinnedGenerator::new(generator); - (initial, $name(pinned)) - } - - $v fn access FnOnce($($args,)*) -> R, R>(&mut self, f: F) -> R { - // Turn the FnOnce closure into *mut dyn FnMut() - // so we can pass it in to the generator - let mut r = None; - let mut f = Some(f); - let mut_f: &mut dyn for<$($lifetimes)*> FnMut(($($args,)*)) = - &mut |args| { - let f = f.take().unwrap(); - r = Some(FnOnce::call_once(f, args)); - }; - let mut_f = mut_f as *mut dyn for<$($lifetimes)*> FnMut(($($args,)*)); - - // Get the generator to call our closure - unsafe { - self.0.access(::std::mem::transmute(mut_f)); - } - - // Unwrap the result - r.unwrap() - } - - $v fn complete(mut self) -> $retc { - self.0.complete() - } - - fn initial_yield(value: $reti) -> $yield_type { - $crate::box_region::YieldType::Initial(value) - } - } - }; - - ($v:vis $name: ident, for($($lifetimes:tt)*), ($($args:ty),*) -> ($reti:ty, $retc:ty)) => { - declare_box_region_type!( - impl $v $name, - $crate::box_region::YieldType<$reti, for<$($lifetimes)*> fn(($($args,)*))>, - for($($lifetimes)*), - ($($args),*) -> ($reti, $retc) - ); - }; -} - -#[macro_export] -#[allow_internal_unstable(fn_traits)] -macro_rules! box_region_allow_access { - (for($($lifetimes:tt)*), ($($args:ty),*), ($($exprs:expr),*), $action:ident) => { - loop { - match $action { - $crate::box_region::Action::Access(accessor) => { - let accessor: &mut dyn for<$($lifetimes)*> FnMut($($args),*) = unsafe { - ::std::mem::transmute(accessor.get()) - }; - (*accessor)(($($exprs),*)); - unsafe { - let marker = $crate::box_region::Marker::< - for<$($lifetimes)*> fn(($($args,)*)) - >::new(); - $action = yield $crate::box_region::YieldType::Accessor(marker); - }; - } - $crate::box_region::Action::Complete => break, - $crate::box_region::Action::Initial => panic!("unexpected box_region action: Initial"), - } - } - } -} diff --git a/compiler/rustc_data_structures/src/flock.rs b/compiler/rustc_data_structures/src/flock.rs index 9383be474f..4f5d8d7ea4 100644 --- a/compiler/rustc_data_structures/src/flock.rs +++ b/compiler/rustc_data_structures/src/flock.rs @@ -54,6 +54,10 @@ cfg_if! { Ok(Lock { _file: file }) } } + + pub fn error_unsupported(err: &io::Error) -> bool { + matches!(err.raw_os_error(), Some(libc::ENOTSUP) | Some(libc::ENOSYS)) + } } // Note that we don't need a Drop impl to execute `flock(fd, LOCK_UN)`. Lock acquired by @@ -103,6 +107,10 @@ cfg_if! { Ok(Lock { file }) } } + + pub fn error_unsupported(err: &io::Error) -> bool { + matches!(err.raw_os_error(), Some(libc::ENOTSUP) | Some(libc::ENOSYS)) + } } impl Drop for Lock { @@ -122,6 +130,7 @@ cfg_if! { use std::mem; use std::os::windows::prelude::*; + use winapi::shared::winerror::ERROR_INVALID_FUNCTION; use winapi::um::minwinbase::{OVERLAPPED, LOCKFILE_FAIL_IMMEDIATELY, LOCKFILE_EXCLUSIVE_LOCK}; use winapi::um::fileapi::LockFileEx; use winapi::um::winnt::{FILE_SHARE_DELETE, FILE_SHARE_READ, FILE_SHARE_WRITE}; @@ -194,6 +203,10 @@ cfg_if! { Ok(Lock { _file: file }) } } + + pub fn error_unsupported(err: &io::Error) -> bool { + err.raw_os_error() == Some(ERROR_INVALID_FUNCTION as i32) + } } // Note that we don't need a Drop impl on the Windows: The file is unlocked diff --git a/compiler/rustc_data_structures/src/lib.rs b/compiler/rustc_data_structures/src/lib.rs index adbb98fa75..16151e9dca 100644 --- a/compiler/rustc_data_structures/src/lib.rs +++ b/compiler/rustc_data_structures/src/lib.rs @@ -10,15 +10,11 @@ #![feature(array_windows)] #![feature(control_flow_enum)] #![feature(in_band_lifetimes)] -#![feature(unboxed_closures)] -#![feature(generator_trait)] -#![feature(fn_traits)] #![feature(min_specialization)] #![feature(auto_traits)] #![feature(nll)] #![feature(allow_internal_unstable)] #![feature(hash_raw_entry)] -#![feature(stmt_expr_attributes)] #![feature(core_intrinsics)] #![feature(test)] #![feature(associated_type_bounds)] @@ -66,7 +62,6 @@ macro_rules! unlikely { pub mod base_n; pub mod binary_search_util; -pub mod box_region; pub mod captures; pub mod flock; pub mod functor; @@ -99,6 +94,7 @@ pub mod thin_vec; pub mod tiny_list; pub mod transitive_relation; pub mod vec_linked_list; +pub mod vec_map; pub mod work_queue; pub use atomic_ref::AtomicRef; pub mod frozen; diff --git a/compiler/rustc_data_structures/src/obligation_forest/mod.rs b/compiler/rustc_data_structures/src/obligation_forest/mod.rs index a5b2df1da5..05b1a85381 100644 --- a/compiler/rustc_data_structures/src/obligation_forest/mod.rs +++ b/compiler/rustc_data_structures/src/obligation_forest/mod.rs @@ -336,12 +336,13 @@ impl ObligationForest { // Returns Err(()) if we already know this obligation failed. fn register_obligation_at(&mut self, obligation: O, parent: Option) -> Result<(), ()> { - if self.done_cache.contains(&obligation.as_cache_key()) { + let cache_key = obligation.as_cache_key(); + if self.done_cache.contains(&cache_key) { debug!("register_obligation_at: ignoring already done obligation: {:?}", obligation); return Ok(()); } - match self.active_cache.entry(obligation.as_cache_key()) { + match self.active_cache.entry(cache_key) { Entry::Occupied(o) => { let node = &mut self.nodes[*o.get()]; if let Some(parent_index) = parent { @@ -365,8 +366,7 @@ impl ObligationForest { && self .error_cache .get(&obligation_tree_id) - .map(|errors| errors.contains(&obligation.as_cache_key())) - .unwrap_or(false); + .map_or(false, |errors| errors.contains(v.key())); if already_failed { Err(()) @@ -596,7 +596,7 @@ impl ObligationForest { Some(rpos) => { // Cycle detected. processor.process_backedge( - stack[rpos..].iter().map(GetObligation(&self.nodes)), + stack[rpos..].iter().map(|&i| &self.nodes[i].obligation), PhantomData, ); } @@ -704,20 +704,3 @@ impl ObligationForest { }); } } - -// I need a Clone closure. -#[derive(Clone)] -struct GetObligation<'a, O>(&'a [Node]); - -impl<'a, 'b, O> FnOnce<(&'b usize,)> for GetObligation<'a, O> { - type Output = &'a O; - extern "rust-call" fn call_once(self, args: (&'b usize,)) -> &'a O { - &self.0[*args.0].obligation - } -} - -impl<'a, 'b, O> FnMut<(&'b usize,)> for GetObligation<'a, O> { - extern "rust-call" fn call_mut(&mut self, args: (&'b usize,)) -> &'a O { - &self.0[*args.0].obligation - } -} diff --git a/compiler/rustc_data_structures/src/stable_hasher.rs b/compiler/rustc_data_structures/src/stable_hasher.rs index ff28784a1d..18b352cf3b 100644 --- a/compiler/rustc_data_structures/src/stable_hasher.rs +++ b/compiler/rustc_data_structures/src/stable_hasher.rs @@ -550,35 +550,3 @@ pub fn hash_stable_hashmap( entries.sort_unstable_by(|&(ref sk1, _), &(ref sk2, _)| sk1.cmp(sk2)); entries.hash_stable(hcx, hasher); } - -/// A vector container that makes sure that its items are hashed in a stable -/// order. -#[derive(Debug)] -pub struct StableVec(Vec); - -impl StableVec { - pub fn new(v: Vec) -> Self { - StableVec(v) - } -} - -impl ::std::ops::Deref for StableVec { - type Target = Vec; - - fn deref(&self) -> &Vec { - &self.0 - } -} - -impl HashStable for StableVec -where - T: HashStable + ToStableHashKey, -{ - fn hash_stable(&self, hcx: &mut HCX, hasher: &mut StableHasher) { - let StableVec(ref v) = *self; - - let mut sorted: Vec<_> = v.iter().map(|x| x.to_stable_hash_key(hcx)).collect(); - sorted.sort_unstable(); - sorted.hash_stable(hcx, hasher); - } -} diff --git a/compiler/rustc_data_structures/src/sync.rs b/compiler/rustc_data_structures/src/sync.rs index 26706cd2b1..722ce6b636 100644 --- a/compiler/rustc_data_structures/src/sync.rs +++ b/compiler/rustc_data_structures/src/sync.rs @@ -43,49 +43,9 @@ cfg_if! { use std::ops::Add; use std::panic::{resume_unwind, catch_unwind, AssertUnwindSafe}; - /// This is a single threaded variant of AtomicCell provided by crossbeam. - /// Unlike `Atomic` this is intended for all `Copy` types, - /// but it lacks the explicit ordering arguments. - #[derive(Debug)] - pub struct AtomicCell(Cell); - - impl AtomicCell { - #[inline] - pub fn new(v: T) -> Self { - AtomicCell(Cell::new(v)) - } - - #[inline] - pub fn get_mut(&mut self) -> &mut T { - self.0.get_mut() - } - } - - impl AtomicCell { - #[inline] - pub fn into_inner(self) -> T { - self.0.into_inner() - } - - #[inline] - pub fn load(&self) -> T { - self.0.get() - } - - #[inline] - pub fn store(&self, val: T) { - self.0.set(val) - } - - #[inline] - pub fn swap(&self, val: T) -> T { - self.0.replace(val) - } - } - /// This is a single threaded variant of `AtomicU64`, `AtomicUsize`, etc. - /// It differs from `AtomicCell` in that it has explicit ordering arguments - /// and is only intended for use with the native atomic types. + /// It has explicit ordering arguments and is only intended for use with + /// the native atomic types. /// You should use this type through the `AtomicU64`, `AtomicUsize`, etc, type aliases /// as it's not intended to be used separately. #[derive(Debug)] @@ -159,22 +119,6 @@ cfg_if! { (oper_a(), oper_b()) } - pub struct SerialScope; - - impl SerialScope { - pub fn spawn(&self, f: F) - where F: FnOnce(&SerialScope) - { - f(self) - } - } - - pub fn scope(f: F) -> R - where F: FnOnce(&SerialScope) -> R - { - f(&SerialScope) - } - #[macro_export] macro_rules! parallel { ($($blocks:tt),*) => { @@ -318,8 +262,6 @@ cfg_if! { pub use std::sync::atomic::{AtomicBool, AtomicUsize, AtomicU32, AtomicU64}; - pub use crossbeam_utils::atomic::AtomicCell; - pub use std::sync::Arc as Lrc; pub use std::sync::Weak as Weak; diff --git a/compiler/rustc_data_structures/src/tagged_ptr.rs b/compiler/rustc_data_structures/src/tagged_ptr.rs index cd1e12ca45..324a8624dd 100644 --- a/compiler/rustc_data_structures/src/tagged_ptr.rs +++ b/compiler/rustc_data_structures/src/tagged_ptr.rs @@ -90,9 +90,11 @@ pub unsafe trait Tag: Copy { unsafe impl Pointer for Box { const BITS: usize = std::mem::align_of::().trailing_zeros() as usize; + #[inline] fn into_usize(self) -> usize { Box::into_raw(self) as usize } + #[inline] unsafe fn from_usize(ptr: usize) -> Self { Box::from_raw(ptr as *mut T) } @@ -104,9 +106,11 @@ unsafe impl Pointer for Box { unsafe impl Pointer for Rc { const BITS: usize = std::mem::align_of::().trailing_zeros() as usize; + #[inline] fn into_usize(self) -> usize { Rc::into_raw(self) as usize } + #[inline] unsafe fn from_usize(ptr: usize) -> Self { Rc::from_raw(ptr as *const T) } @@ -118,9 +122,11 @@ unsafe impl Pointer for Rc { unsafe impl Pointer for Arc { const BITS: usize = std::mem::align_of::().trailing_zeros() as usize; + #[inline] fn into_usize(self) -> usize { Arc::into_raw(self) as usize } + #[inline] unsafe fn from_usize(ptr: usize) -> Self { Arc::from_raw(ptr as *const T) } @@ -132,9 +138,11 @@ unsafe impl Pointer for Arc { unsafe impl<'a, T: 'a> Pointer for &'a T { const BITS: usize = std::mem::align_of::().trailing_zeros() as usize; + #[inline] fn into_usize(self) -> usize { self as *const T as usize } + #[inline] unsafe fn from_usize(ptr: usize) -> Self { &*(ptr as *const T) } @@ -145,9 +153,11 @@ unsafe impl<'a, T: 'a> Pointer for &'a T { unsafe impl<'a, T: 'a> Pointer for &'a mut T { const BITS: usize = std::mem::align_of::().trailing_zeros() as usize; + #[inline] fn into_usize(self) -> usize { self as *mut T as usize } + #[inline] unsafe fn from_usize(ptr: usize) -> Self { &mut *(ptr as *mut T) } diff --git a/compiler/rustc_data_structures/src/vec_map.rs b/compiler/rustc_data_structures/src/vec_map.rs new file mode 100644 index 0000000000..73b04d3329 --- /dev/null +++ b/compiler/rustc_data_structures/src/vec_map.rs @@ -0,0 +1,155 @@ +use std::borrow::Borrow; +use std::iter::FromIterator; +use std::slice::{Iter, IterMut}; +use std::vec::IntoIter; + +use crate::stable_hasher::{HashStable, StableHasher}; + +/// A map type implemented as a vector of pairs `K` (key) and `V` (value). +/// It currently provides a subset of all the map operations, the rest could be added as needed. +#[derive(Clone, Encodable, Decodable, Debug)] +pub struct VecMap(Vec<(K, V)>); + +impl VecMap +where + K: PartialEq, +{ + pub fn new() -> Self { + VecMap(Default::default()) + } + + /// Sets the value of the entry, and returns the entry's old value. + pub fn insert(&mut self, k: K, v: V) -> Option { + if let Some(elem) = self.0.iter_mut().find(|(key, _)| *key == k) { + Some(std::mem::replace(&mut elem.1, v)) + } else { + self.0.push((k, v)); + None + } + } + + /// Gets a reference to the value in the entry. + pub fn get(&self, k: &Q) -> Option<&V> + where + K: Borrow, + Q: Eq, + { + self.0.iter().find(|(key, _)| k == key.borrow()).map(|elem| &elem.1) + } + + /// Returns the value corresponding to the supplied predicate filter. + /// + /// The supplied predicate will be applied to each (key, value) pair and it will return a + /// reference to the values where the predicate returns `true`. + pub fn get_by(&self, mut predicate: impl FnMut(&(K, V)) -> bool) -> Option<&V> { + self.0.iter().find(|kv| predicate(kv)).map(|elem| &elem.1) + } + + /// Returns `true` if the map contains a value for the specified key. + /// + /// The key may be any borrowed form of the map's key type, + /// [`Eq`] on the borrowed form *must* match those for + /// the key type. + pub fn contains_key(&self, k: &Q) -> bool + where + K: Borrow, + Q: Eq, + { + self.get(k).is_some() + } + + /// Returns `true` if the map contains no elements. + pub fn is_empty(&self) -> bool { + self.0.is_empty() + } + + pub fn iter(&self) -> Iter<'_, (K, V)> { + self.into_iter() + } + + pub fn iter_mut(&mut self) -> IterMut<'_, (K, V)> { + self.into_iter() + } +} + +impl Default for VecMap { + #[inline] + fn default() -> Self { + Self(Default::default()) + } +} + +impl From> for VecMap { + fn from(vec: Vec<(K, V)>) -> Self { + Self(vec) + } +} + +impl Into> for VecMap { + fn into(self) -> Vec<(K, V)> { + self.0 + } +} + +impl FromIterator<(K, V)> for VecMap { + fn from_iter>(iter: I) -> Self { + Self(iter.into_iter().collect()) + } +} + +impl<'a, K, V> IntoIterator for &'a VecMap { + type Item = &'a (K, V); + type IntoIter = Iter<'a, (K, V)>; + + #[inline] + fn into_iter(self) -> Self::IntoIter { + self.0.iter() + } +} + +impl<'a, K, V> IntoIterator for &'a mut VecMap { + type Item = &'a mut (K, V); + type IntoIter = IterMut<'a, (K, V)>; + + #[inline] + fn into_iter(self) -> Self::IntoIter { + self.0.iter_mut() + } +} + +impl IntoIterator for VecMap { + type Item = (K, V); + type IntoIter = IntoIter<(K, V)>; + + #[inline] + fn into_iter(self) -> Self::IntoIter { + self.0.into_iter() + } +} + +impl Extend<(K, V)> for VecMap { + fn extend>(&mut self, iter: I) { + self.0.extend(iter); + } + + fn extend_one(&mut self, item: (K, V)) { + self.0.extend_one(item); + } + + fn extend_reserve(&mut self, additional: usize) { + self.0.extend_reserve(additional); + } +} + +impl HashStable for VecMap +where + K: HashStable + Eq, + V: HashStable, +{ + fn hash_stable(&self, hcx: &mut CTX, hasher: &mut StableHasher) { + self.0.hash_stable(hcx, hasher) + } +} + +#[cfg(test)] +mod tests; diff --git a/compiler/rustc_data_structures/src/vec_map/tests.rs b/compiler/rustc_data_structures/src/vec_map/tests.rs new file mode 100644 index 0000000000..9083de8598 --- /dev/null +++ b/compiler/rustc_data_structures/src/vec_map/tests.rs @@ -0,0 +1,48 @@ +use super::*; + +impl VecMap { + fn into_vec(self) -> Vec<(K, V)> { + self.0.into() + } +} + +#[test] +fn test_from_iterator() { + assert_eq!( + std::iter::empty().collect::>().into_vec(), + Vec::<(i32, bool)>::new() + ); + assert_eq!(std::iter::once((42, true)).collect::>().into_vec(), vec![(42, true)]); + assert_eq!( + vec![(1, true), (2, false)].into_iter().collect::>().into_vec(), + vec![(1, true), (2, false)] + ); +} + +#[test] +fn test_into_iterator_owned() { + assert_eq!(VecMap::new().into_iter().collect::>(), Vec::<(i32, bool)>::new()); + assert_eq!(VecMap::from(vec![(1, true)]).into_iter().collect::>(), vec![(1, true)]); + assert_eq!( + VecMap::from(vec![(1, true), (2, false)]).into_iter().collect::>(), + vec![(1, true), (2, false)] + ); +} + +#[test] +fn test_insert() { + let mut v = VecMap::new(); + assert_eq!(v.insert(1, true), None); + assert_eq!(v.insert(2, false), None); + assert_eq!(v.clone().into_vec(), vec![(1, true), (2, false)]); + assert_eq!(v.insert(1, false), Some(true)); + assert_eq!(v.into_vec(), vec![(1, false), (2, false)]); +} + +#[test] +fn test_get() { + let v = vec![(1, true), (2, false)].into_iter().collect::>(); + assert_eq!(v.get(&1), Some(&true)); + assert_eq!(v.get(&2), Some(&false)); + assert_eq!(v.get(&3), None); +} diff --git a/compiler/rustc_driver/src/lib.rs b/compiler/rustc_driver/src/lib.rs index 51699403a3..b943977e4c 100644 --- a/compiler/rustc_driver/src/lib.rs +++ b/compiler/rustc_driver/src/lib.rs @@ -21,8 +21,7 @@ use rustc_data_structures::sync::SeqCst; use rustc_errors::registry::{InvalidErrorCode, Registry}; use rustc_errors::{ErrorReported, PResult}; use rustc_feature::find_gated_cfg; -use rustc_hir::def_id::LOCAL_CRATE; -use rustc_interface::util::{self, collect_crate_types, get_builtin_codegen_backend}; +use rustc_interface::util::{self, collect_crate_types, get_codegen_backend}; use rustc_interface::{interface, Queries}; use rustc_lint::LintStore; use rustc_metadata::locator; @@ -35,7 +34,7 @@ use rustc_session::config::{ErrorOutputType, Input, OutputType, PrintRequest, Tr use rustc_session::getopts; use rustc_session::lint::{Lint, LintId}; use rustc_session::{config, DiagnosticOutput, Session}; -use rustc_session::{early_error, early_warn}; +use rustc_session::{early_error, early_error_no_abort, early_warn}; use rustc_span::source_map::{FileLoader, FileName}; use rustc_span::symbol::sym; @@ -133,6 +132,7 @@ pub fn diagnostics_registry() -> Registry { Registry::new(&rustc_error_codes::DIAGNOSTICS) } +/// This is the primary entry point for rustc. pub struct RunCompiler<'a, 'b> { at_args: &'a [String], callbacks: &'b mut (dyn Callbacks + Send), @@ -146,6 +146,9 @@ impl<'a, 'b> RunCompiler<'a, 'b> { pub fn new(at_args: &'a [String], callbacks: &'b mut (dyn Callbacks + Send)) -> Self { Self { at_args, callbacks, file_loader: None, emitter: None, make_codegen_backend: None } } + + /// Set a custom codegen backend. + /// /// Used by cg_clif. pub fn set_make_codegen_backend( &mut self, @@ -156,11 +159,17 @@ impl<'a, 'b> RunCompiler<'a, 'b> { self.make_codegen_backend = make_codegen_backend; self } + + /// Emit diagnostics to the specified location. + /// /// Used by RLS. pub fn set_emitter(&mut self, emitter: Option>) -> &mut Self { self.emitter = emitter; self } + + /// Load files from sources other than the file system. + /// /// Used by RLS. pub fn set_file_loader( &mut self, @@ -169,6 +178,8 @@ impl<'a, 'b> RunCompiler<'a, 'b> { self.file_loader = file_loader; self } + + /// Parse args and run the compiler. pub fn run(self) -> interface::Result<()> { run_compiler( self.at_args, @@ -179,8 +190,6 @@ impl<'a, 'b> RunCompiler<'a, 'b> { ) } } -// Parse args and run the compiler. This is the primary entry point for rustc. -// The FileLoader provides a way to load files from sources other than the file system. fn run_compiler( at_args: &[String], callbacks: &mut (dyn Callbacks + Send), @@ -199,46 +208,43 @@ fn run_compiler( }; let sopts = config::build_session_options(&matches); - let cfg = interface::parse_cfgspecs(matches.opt_strs("cfg")); - - // We wrap `make_codegen_backend` in another `Option` such that `dummy_config` can take - // ownership of it when necessary, while also allowing the non-dummy config to take ownership - // when `dummy_config` is not used. - let mut make_codegen_backend = Some(make_codegen_backend); - - let mut dummy_config = |sopts, cfg, diagnostic_output| { - let mut config = interface::Config { - opts: sopts, - crate_cfg: cfg, - input: Input::File(PathBuf::new()), - input_path: None, - output_file: None, - output_dir: None, - file_loader: None, - diagnostic_output, - stderr: None, - lint_caps: Default::default(), - parse_sess_created: None, - register_lints: None, - override_queries: None, - make_codegen_backend: make_codegen_backend.take().unwrap(), - registry: diagnostics_registry(), - }; - callbacks.config(&mut config); - config - }; if let Some(ref code) = matches.opt_str("explain") { handle_explain(diagnostics_registry(), code, sopts.error_format); return Ok(()); } + let cfg = interface::parse_cfgspecs(matches.opt_strs("cfg")); let (odir, ofile) = make_output(&matches); - let (input, input_file_path, input_err) = match make_input(&matches.free) { - Some(v) => v, - None => match matches.free.len() { + let mut config = interface::Config { + opts: sopts, + crate_cfg: cfg, + input: Input::File(PathBuf::new()), + input_path: None, + output_file: ofile, + output_dir: odir, + file_loader, + diagnostic_output, + stderr: None, + lint_caps: Default::default(), + parse_sess_created: None, + register_lints: None, + override_queries: None, + make_codegen_backend, + registry: diagnostics_registry(), + }; + + match make_input(config.opts.error_format, &matches.free) { + Err(ErrorReported) => return Err(ErrorReported), + Ok(Some((input, input_file_path))) => { + config.input = input; + config.input_path = input_file_path; + + callbacks.config(&mut config); + } + Ok(None) => match matches.free.len() { 0 => { - let config = dummy_config(sopts, cfg, diagnostic_output); + callbacks.config(&mut config); interface::run_compiler(config, |compiler| { let sopts = &compiler.session().opts; if sopts.describe_lints { @@ -260,8 +266,8 @@ fn run_compiler( &***compiler.codegen_backend(), compiler.session(), None, - &odir, - &ofile, + &compiler.output_dir(), + &compiler.output_file(), ); if should_stop == Compilation::Stop { @@ -273,7 +279,7 @@ fn run_compiler( } 1 => panic!("make_input should have provided valid inputs"), _ => early_error( - sopts.error_format, + config.opts.error_format, &format!( "multiple input filenames provided (first two filenames are `{}` and `{}`)", matches.free[0], matches.free[1], @@ -282,35 +288,6 @@ fn run_compiler( }, }; - if let Some(err) = input_err { - // Immediately stop compilation if there was an issue reading - // the input (for example if the input stream is not UTF-8). - interface::run_compiler(dummy_config(sopts, cfg, diagnostic_output), |compiler| { - compiler.session().err(&err.to_string()); - }); - return Err(ErrorReported); - } - - let mut config = interface::Config { - opts: sopts, - crate_cfg: cfg, - input, - input_path: input_file_path, - output_file: ofile, - output_dir: odir, - file_loader, - diagnostic_output, - stderr: None, - lint_caps: Default::default(), - parse_sess_created: None, - register_lints: None, - override_queries: None, - make_codegen_backend: make_codegen_backend.unwrap(), - registry: diagnostics_registry(), - }; - - callbacks.config(&mut config); - interface::run_compiler(config, |compiler| { let sess = compiler.session(); let should_stop = RustcDefaultCalls::print_crate_info( @@ -324,7 +301,6 @@ fn run_compiler( RustcDefaultCalls::list_metadata( sess, &*compiler.codegen_backend().metadata_loader(), - &matches, compiler.input(), ) }) @@ -411,11 +387,10 @@ fn run_compiler( return early_exit(); } - if sess.opts.debugging_opts.save_analysis { - let crate_name = queries.crate_name()?.peek().clone(); - queries.global_ctxt()?.peek_mut().enter(|tcx| { - let result = tcx.analysis(LOCAL_CRATE); - + queries.global_ctxt()?.peek_mut().enter(|tcx| { + let result = tcx.analysis(()); + if sess.opts.debugging_opts.save_analysis { + let crate_name = queries.crate_name()?.peek().clone(); sess.time("save_analysis", || { save::process_crate( tcx, @@ -428,12 +403,9 @@ fn run_compiler( ), ) }); - - result - })?; - } - - queries.global_ctxt()?.peek_mut().enter(|tcx| tcx.analysis(LOCAL_CRATE))?; + } + result + })?; if callbacks.after_analysis(compiler, queries) == Compilation::Stop { return early_exit(); @@ -490,19 +462,23 @@ fn make_output(matches: &getopts::Matches) -> (Option, Option) } // Extract input (string or file and optional path) from matches. -fn make_input(free_matches: &[String]) -> Option<(Input, Option, Option)> { +fn make_input( + error_format: ErrorOutputType, + free_matches: &[String], +) -> Result)>, ErrorReported> { if free_matches.len() == 1 { let ifile = &free_matches[0]; if ifile == "-" { let mut src = String::new(); - let err = if io::stdin().read_to_string(&mut src).is_err() { - Some(io::Error::new( - io::ErrorKind::InvalidData, + if io::stdin().read_to_string(&mut src).is_err() { + // Immediately stop compilation if there was an issue reading + // the input (for example if the input stream is not UTF-8). + early_error_no_abort( + error_format, "couldn't read from stdin, as it did not contain valid UTF-8", - )) - } else { - None - }; + ); + return Err(ErrorReported); + } if let Ok(path) = env::var("UNSTABLE_RUSTDOC_TEST_PATH") { let line = env::var("UNSTABLE_RUSTDOC_TEST_LINE").expect( "when UNSTABLE_RUSTDOC_TEST_PATH is set \ @@ -511,18 +487,19 @@ fn make_input(free_matches: &[String]) -> Option<(Input, Option, Option let line = isize::from_str_radix(&line, 10) .expect("UNSTABLE_RUSTDOC_TEST_LINE needs to be an number"); let file_name = FileName::doc_test_source_code(PathBuf::from(path), line); - return Some((Input::Str { name: file_name, input: src }, None, err)); + Ok(Some((Input::Str { name: file_name, input: src }, None))) + } else { + Ok(Some((Input::Str { name: FileName::anon_source_code(&src), input: src }, None))) } - Some((Input::Str { name: FileName::anon_source_code(&src), input: src }, None, err)) } else { - Some((Input::File(PathBuf::from(ifile)), Some(PathBuf::from(ifile)), None)) + Ok(Some((Input::File(PathBuf::from(ifile)), Some(PathBuf::from(ifile))))) } } else { - None + Ok(None) } } -// Whether to stop or continue compilation. +/// Whether to stop or continue compilation. #[derive(Copy, Clone, Debug, Eq, PartialEq)] pub enum Compilation { Stop, @@ -619,28 +596,24 @@ fn show_content_with_pager(content: &str) { } impl RustcDefaultCalls { - fn process_rlink(sess: &Session, compiler: &interface::Compiler) -> Result<(), ErrorReported> { - if let Input::File(file) = compiler.input() { - // FIXME: #![crate_type] and #![crate_name] support not implemented yet - let attrs = vec![]; - sess.init_crate_types(collect_crate_types(sess, &attrs)); - let outputs = compiler.build_output_filenames(&sess, &attrs); - let rlink_data = fs::read_to_string(file).unwrap_or_else(|err| { - sess.fatal(&format!("failed to read rlink file: {}", err)); - }); - let codegen_results: CodegenResults = json::decode(&rlink_data).unwrap_or_else(|err| { - sess.fatal(&format!("failed to decode rlink: {}", err)); - }); - compiler.codegen_backend().link(&sess, codegen_results, &outputs) - } else { - sess.fatal("rlink must be a file") - } - } - pub fn try_process_rlink(sess: &Session, compiler: &interface::Compiler) -> Compilation { if sess.opts.debugging_opts.link_only { - let result = RustcDefaultCalls::process_rlink(sess, compiler); - abort_on_err(result, sess); + if let Input::File(file) = compiler.input() { + // FIXME: #![crate_type] and #![crate_name] support not implemented yet + sess.init_crate_types(collect_crate_types(sess, &[])); + let outputs = compiler.build_output_filenames(&sess, &[]); + let rlink_data = fs::read_to_string(file).unwrap_or_else(|err| { + sess.fatal(&format!("failed to read rlink file: {}", err)); + }); + let codegen_results: CodegenResults = + json::decode(&rlink_data).unwrap_or_else(|err| { + sess.fatal(&format!("failed to decode rlink: {}", err)); + }); + let result = compiler.codegen_backend().link(&sess, codegen_results, &outputs); + abort_on_err(result, sess); + } else { + sess.fatal("rlink must be a file") + } Compilation::Stop } else { Compilation::Continue @@ -650,11 +623,9 @@ impl RustcDefaultCalls { pub fn list_metadata( sess: &Session, metadata_loader: &dyn MetadataLoader, - matches: &getopts::Matches, input: &Input, ) -> Compilation { - let r = matches.opt_strs("Z"); - if r.iter().any(|s| *s == "ls") { + if sess.opts.debugging_opts.ls { match *input { Input::File(ref ifile) => { let path = &(*ifile); @@ -794,9 +765,16 @@ pub fn version(binary: &str, matches: &getopts::Matches) { println!("commit-date: {}", unw(util::commit_date_str())); println!("host: {}", config::host_triple()); println!("release: {}", unw(util::release_str())); - if cfg!(feature = "llvm") { - get_builtin_codegen_backend(&None, "llvm")().print_version(); - } + + let debug_flags = matches.opt_strs("Z"); + let backend_name = debug_flags.iter().find_map(|x| { + if x.starts_with("codegen-backend=") { + Some(&x["codegen-backends=".len()..]) + } else { + None + } + }); + get_codegen_backend(&None, backend_name).print_version(); } } @@ -1068,8 +1046,8 @@ pub fn handle_options(args: &[String]) -> Option { } // Don't handle -W help here, because we might first load plugins. - let r = matches.opt_strs("Z"); - if r.iter().any(|x| *x == "help") { + let debug_flags = matches.opt_strs("Z"); + if debug_flags.iter().any(|x| *x == "help") { describe_debug_flags(); return None; } @@ -1089,9 +1067,14 @@ pub fn handle_options(args: &[String]) -> Option { } if cg_flags.iter().any(|x| *x == "passes=list") { - if cfg!(feature = "llvm") { - get_builtin_codegen_backend(&None, "llvm")().print_passes(); - } + let backend_name = debug_flags.iter().find_map(|x| { + if x.starts_with("codegen-backend=") { + Some(&x["codegen-backends=".len()..]) + } else { + None + } + }); + get_codegen_backend(&None, backend_name).print_passes(); return None; } diff --git a/compiler/rustc_driver/src/pretty.rs b/compiler/rustc_driver/src/pretty.rs index e0c140b143..a2b4f3fcf7 100644 --- a/compiler/rustc_driver/src/pretty.rs +++ b/compiler/rustc_driver/src/pretty.rs @@ -4,19 +4,16 @@ use rustc_ast as ast; use rustc_ast_pretty::pprust; use rustc_errors::ErrorReported; use rustc_hir as hir; -use rustc_hir::def_id::LOCAL_CRATE; use rustc_hir_pretty as pprust_hir; use rustc_middle::hir::map as hir_map; use rustc_middle::ty::{self, TyCtxt}; use rustc_mir::util::{write_mir_graphviz, write_mir_pretty}; -use rustc_mir_build::thir; use rustc_session::config::{Input, PpAstTreeMode, PpHirMode, PpMode, PpSourceMode}; use rustc_session::Session; use rustc_span::symbol::Ident; use rustc_span::FileName; use std::cell::Cell; -use std::fmt::Write; use std::path::Path; pub use self::PpMode::*; @@ -74,7 +71,7 @@ where f(&annotation, tcx.hir().krate()) } PpHirMode::Typed => { - abort_on_err(tcx.analysis(LOCAL_CRATE), tcx.sess); + abort_on_err(tcx.analysis(()), tcx.sess); let annotation = TypedAnnotation { tcx, maybe_typeck_results: Cell::new(None) }; tcx.dep_graph.with_ignore(|| f(&annotation, tcx.hir().krate())) @@ -475,7 +472,7 @@ fn print_with_analysis( ppm: PpMode, ofile: Option<&Path>, ) -> Result<(), ErrorReported> { - tcx.analysis(LOCAL_CRATE)?; + tcx.analysis(())?; let out = match ppm { Mir => { @@ -491,18 +488,8 @@ fn print_with_analysis( } ThirTree => { - let mut out = String::new(); - abort_on_err(rustc_typeck::check_crate(tcx), tcx.sess); - debug!("pretty printing THIR tree"); - for did in tcx.body_owners() { - let hir = tcx.hir(); - let body = hir.body(hir.body_owned_by(hir.local_def_id_to_hir_id(did))); - let arena = thir::Arena::default(); - let thir = - thir::build_thir(tcx, ty::WithOptConstParam::unknown(did), &arena, &body.value); - let _ = writeln!(out, "{:?}:\n{:#?}\n", did, thir); - } - out + // FIXME(rust-lang/project-thir-unsafeck#8) + todo!() } _ => unreachable!(), diff --git a/compiler/rustc_error_codes/src/error_codes.rs b/compiler/rustc_error_codes/src/error_codes.rs index 41a1fa488d..f10efd8323 100644 --- a/compiler/rustc_error_codes/src/error_codes.rs +++ b/compiler/rustc_error_codes/src/error_codes.rs @@ -157,6 +157,7 @@ E0308: include_str!("./error_codes/E0308.md"), E0309: include_str!("./error_codes/E0309.md"), E0310: include_str!("./error_codes/E0310.md"), E0312: include_str!("./error_codes/E0312.md"), +E0316: include_str!("./error_codes/E0316.md"), E0317: include_str!("./error_codes/E0317.md"), E0321: include_str!("./error_codes/E0321.md"), E0322: include_str!("./error_codes/E0322.md"), @@ -470,6 +471,8 @@ E0778: include_str!("./error_codes/E0778.md"), E0779: include_str!("./error_codes/E0779.md"), E0780: include_str!("./error_codes/E0780.md"), E0781: include_str!("./error_codes/E0781.md"), +E0782: include_str!("./error_codes/E0782.md"), +E0783: include_str!("./error_codes/E0783.md"), ; // E0006, // merged with E0005 // E0008, // cannot bind by-move into a pattern guard @@ -551,9 +554,8 @@ E0781: include_str!("./error_codes/E0781.md"), E0311, // thing may not live long enough E0313, // lifetime of borrowed pointer outlives lifetime of captured // variable - E0314, // closure outlives stack frame - E0315, // cannot invoke closure outside of its lifetime - E0316, // nested quantification of lifetimes +// E0314, // closure outlives stack frame +// E0315, // cannot invoke closure outside of its lifetime // E0319, // trait impls for defaulted traits allowed just for structs/enums E0320, // recursive overflow during dropck // E0372, // coherence not object safe @@ -582,21 +584,21 @@ E0781: include_str!("./error_codes/E0781.md"), // E0470, removed // E0471, // constant evaluation error (in pattern) E0472, // llvm_asm! is unsupported on this target - E0473, // dereference of reference outside its lifetime - E0474, // captured variable `..` does not outlive the enclosing closure - E0475, // index of slice outside its lifetime +// E0473, // dereference of reference outside its lifetime +// E0474, // captured variable `..` does not outlive the enclosing closure +// E0475, // index of slice outside its lifetime E0476, // lifetime of the source pointer does not outlive lifetime bound... - E0479, // the type `..` (provided as the value of a type parameter) is... - E0480, // lifetime of method receiver does not outlive the method call - E0481, // lifetime of function argument does not outlive the function call +// E0479, // the type `..` (provided as the value of a type parameter) is... +// E0480, // lifetime of method receiver does not outlive the method call +// E0481, // lifetime of function argument does not outlive the function call E0482, // lifetime of return value does not outlive the function call - E0483, // lifetime of operand does not outlive the operation - E0484, // reference is not valid at the time of borrow - E0485, // automatically reference is not valid at the time of borrow - E0486, // type of expression contains references that are not valid during.. - E0487, // unsafe use of destructor: destructor might be called while... - E0488, // lifetime of variable does not enclose its declaration - E0489, // type/lifetime parameter not in scope here +// E0483, // lifetime of operand does not outlive the operation +// E0484, // reference is not valid at the time of borrow +// E0485, // automatically reference is not valid at the time of borrow +// E0486, // type of expression contains references that are not valid during.. +// E0487, // unsafe use of destructor: destructor might be called while... +// E0488, // lifetime of variable does not enclose its declaration +// E0489, // type/lifetime parameter not in scope here E0490, // a value of type `..` is borrowed for too long E0498, // malformed plugin attribute E0514, // metadata version mismatch diff --git a/compiler/rustc_error_codes/src/error_codes/E0277.md b/compiler/rustc_error_codes/src/error_codes/E0277.md index 9f6db6ed7a..5f05b59d5a 100644 --- a/compiler/rustc_error_codes/src/error_codes/E0277.md +++ b/compiler/rustc_error_codes/src/error_codes/E0277.md @@ -29,16 +29,16 @@ trait Foo { fn bar(&self); } -fn some_func(foo: T) { - foo.bar(); // we can now use this method since i32 implements the - // Foo trait -} - // we implement the trait on the i32 type impl Foo for i32 { fn bar(&self) {} } +fn some_func(foo: T) { + foo.bar(); // we can now use this method since i32 implements the + // Foo trait +} + fn main() { some_func(5i32); // ok! } diff --git a/compiler/rustc_error_codes/src/error_codes/E0316.md b/compiler/rustc_error_codes/src/error_codes/E0316.md new file mode 100644 index 0000000000..4368c32173 --- /dev/null +++ b/compiler/rustc_error_codes/src/error_codes/E0316.md @@ -0,0 +1,32 @@ +A `where` clause contains a nested quantification over lifetimes. + +Erroneous code example: + +```compile_fail,E0316 +trait Tr<'a, 'b> {} + +fn foo(t: T) +where + for<'a> &'a T: for<'b> Tr<'a, 'b>, // error: nested quantification +{ +} +``` + +Rust syntax allows lifetime quantifications in two places within +`where` clauses: Quantifying over the trait bound only (as in +`Ty: for<'l> Trait<'l>`) and quantifying over the whole clause +(as in `for<'l> &'l Ty: Trait<'l>`). Using both in the same clause +leads to a nested lifetime quantification, which is not supported. + +The following example compiles, because the clause with the nested +quantification has been rewritten to use only one `for<>`: + +``` +trait Tr<'a, 'b> {} + +fn foo(t: T) +where + for<'a, 'b> &'a T: Tr<'a, 'b>, // ok +{ +} +``` diff --git a/compiler/rustc_error_codes/src/error_codes/E0759.md b/compiler/rustc_error_codes/src/error_codes/E0759.md index 2fe5ada257..6b16a7d415 100644 --- a/compiler/rustc_error_codes/src/error_codes/E0759.md +++ b/compiler/rustc_error_codes/src/error_codes/E0759.md @@ -16,13 +16,13 @@ fn bar(x: &i32) -> Box { // error! Add `'static` requirement to fix them: -```compile_fail,E0759 +``` # use std::fmt::Debug; -fn foo(x: &i32) -> impl Debug + 'static { // ok! +fn foo(x: &'static i32) -> impl Debug + 'static { // ok! x } -fn bar(x: &i32) -> Box { // ok! +fn bar(x: &'static i32) -> Box { // ok! Box::new(x) } ``` diff --git a/compiler/rustc_error_codes/src/error_codes/E0782.md b/compiler/rustc_error_codes/src/error_codes/E0782.md new file mode 100644 index 0000000000..0f3253c050 --- /dev/null +++ b/compiler/rustc_error_codes/src/error_codes/E0782.md @@ -0,0 +1,26 @@ +Trait objects must include the `dyn` keyword. + +Erroneous code example: + +```edition2021,compile_fail,E0782 +trait Foo {} +fn test(arg: Box) {} // error! +``` + +Trait objects are a way to call methods on types that are not known until +runtime but conform to some trait. + +Trait objects should be formed with `Box`, but in the code above +`dyn` is left off. + +This makes it harder to see that `arg` is a trait object and not a +simply a heap allocated type called `Foo`. + +To fix this issue, add `dyn` before the trait name. + +```edition2021 +trait Foo {} +fn test(arg: Box) {} // ok! +``` + +This used to be allowed before edition 2021, but is now an error. diff --git a/compiler/rustc_error_codes/src/error_codes/E0783.md b/compiler/rustc_error_codes/src/error_codes/E0783.md new file mode 100644 index 0000000000..73981e59e0 --- /dev/null +++ b/compiler/rustc_error_codes/src/error_codes/E0783.md @@ -0,0 +1,22 @@ +The range pattern `...` is no longer allowed. + +Erroneous code example: + +```edition2021,compile_fail,E0783 +match 2u8 { + 0...9 => println!("Got a number less than 10"), // error! + _ => println!("Got a number 10 or more"), +} +``` + +Older Rust code using previous editions allowed `...` to stand for exclusive +ranges which are now signified using `..=`. + +To make this code compile replace the `...` with `..=`. + +```edition2021 +match 2u8 { + 0..=9 => println!("Got a number less than 10"), // ok! + _ => println!("Got a number 10 or more"), +} +``` diff --git a/compiler/rustc_errors/src/annotate_snippet_emitter_writer.rs b/compiler/rustc_errors/src/annotate_snippet_emitter_writer.rs index 6f365c07f6..577baec21f 100644 --- a/compiler/rustc_errors/src/annotate_snippet_emitter_writer.rs +++ b/compiler/rustc_errors/src/annotate_snippet_emitter_writer.rs @@ -126,7 +126,8 @@ impl AnnotateSnippetEmitterWriter { } // owned: line source, line index, annotations type Owned = (String, usize, Vec); - let origin = primary_lo.file.name.to_string(); + let filename = primary_lo.file.name.prefer_local(); + let origin = filename.to_string_lossy(); let annotated_files: Vec = annotated_files .into_iter() .flat_map(|annotated_file| { diff --git a/compiler/rustc_errors/src/diagnostic.rs b/compiler/rustc_errors/src/diagnostic.rs index b2f6a0c101..14ccced2c6 100644 --- a/compiler/rustc_errors/src/diagnostic.rs +++ b/compiler/rustc_errors/src/diagnostic.rs @@ -282,6 +282,22 @@ impl Diagnostic { msg: &str, suggestion: Vec<(Span, String)>, applicability: Applicability, + ) -> &mut Self { + self.multipart_suggestion_with_style( + msg, + suggestion, + applicability, + SuggestionStyle::ShowCode, + ) + } + + /// [`Diagnostic::multipart_suggestion()`] but you can set the [`SuggestionStyle`]. + pub fn multipart_suggestion_with_style( + &mut self, + msg: &str, + suggestion: Vec<(Span, String)>, + applicability: Applicability, + style: SuggestionStyle, ) -> &mut Self { assert!(!suggestion.is_empty()); self.suggestions.push(CodeSuggestion { @@ -292,7 +308,7 @@ impl Diagnostic { .collect(), }], msg: msg.to_owned(), - style: SuggestionStyle::ShowCode, + style, applicability, tool_metadata: Default::default(), }); diff --git a/compiler/rustc_errors/src/emitter.rs b/compiler/rustc_errors/src/emitter.rs index a58caf2667..d3f92bf304 100644 --- a/compiler/rustc_errors/src/emitter.rs +++ b/compiler/rustc_errors/src/emitter.rs @@ -309,7 +309,9 @@ pub trait Emitter { // are some which do actually involve macros. ExpnKind::Inlined | ExpnKind::Desugaring(..) | ExpnKind::AstPass(..) => None, - ExpnKind::Macro(macro_kind, _) => Some(macro_kind), + ExpnKind::Macro { kind: macro_kind, name, proc_macro: _ } => { + Some((macro_kind, name)) + } } }); @@ -320,13 +322,12 @@ pub trait Emitter { self.render_multispans_macro_backtrace(span, children, backtrace); if !backtrace { - if let Some(macro_kind) = has_macro_spans { + if let Some((macro_kind, name)) = has_macro_spans { + let descr = macro_kind.descr(); + let msg = format!( - "this {} originates in {} {} \ + "this {level} originates in the {descr} `{name}` \ (in Nightly builds, run with -Z macro-backtrace for more info)", - level, - macro_kind.article(), - macro_kind.descr(), ); children.push(SubDiagnostic { @@ -371,10 +372,19 @@ pub trait Emitter { new_labels .push((trace.call_site, "in the inlined copy of this code".to_string())); } else if always_backtrace { + let proc_macro = if let ExpnKind::Macro { kind: _, name: _, proc_macro: true } = + trace.kind + { + "procedural macro " + } else { + "" + }; + new_labels.push(( trace.def_site, format!( - "in this expansion of `{}`{}", + "in this expansion of {}`{}`{}", + proc_macro, trace.kind.descr(), if macro_backtrace.len() > 1 { // if macro_backtrace.len() == 1 it'll be @@ -400,7 +410,11 @@ pub trait Emitter { // and it needs an "in this macro invocation" label to match that. let redundant_span = trace.call_site.contains(sp); - if !redundant_span && matches!(trace.kind, ExpnKind::Macro(MacroKind::Bang, _)) + if !redundant_span + && matches!( + trace.kind, + ExpnKind::Macro { kind: MacroKind::Bang, name: _, proc_macro: _ } + ) || always_backtrace { new_labels.push(( @@ -1309,7 +1323,7 @@ impl EmitterWriter { buffer_msg_line_offset, &format!( "{}:{}:{}", - loc.file.name, + loc.file.name.prefer_local(), sm.doctest_offset_line(&loc.file.name, loc.line), loc.col.0 + 1, ), @@ -1323,7 +1337,7 @@ impl EmitterWriter { 0, &format!( "{}:{}:{}: ", - loc.file.name, + loc.file.name.prefer_local(), sm.doctest_offset_line(&loc.file.name, loc.line), loc.col.0 + 1, ), @@ -1347,12 +1361,12 @@ impl EmitterWriter { }; format!( "{}:{}{}", - annotated_file.file.name, + annotated_file.file.name.prefer_local(), sm.doctest_offset_line(&annotated_file.file.name, first_line.line_index), col ) } else { - annotated_file.file.name.to_string() + format!("{}", annotated_file.file.name.prefer_local()) }; buffer.append(buffer_msg_line_offset + 1, &loc, Style::LineAndColumn); for _ in 0..max_line_num_len { diff --git a/compiler/rustc_errors/src/json.rs b/compiler/rustc_errors/src/json.rs index 4027700646..72395bd31e 100644 --- a/compiler/rustc_errors/src/json.rs +++ b/compiler/rustc_errors/src/json.rs @@ -216,7 +216,7 @@ macro_rules! encode_fields { $( $enc.emit_struct_field( stringify!($name), - idx, + idx == 0, |enc| $name.encode(enc), )?; idx += 1; @@ -229,7 +229,7 @@ macro_rules! encode_fields { // Special-case encoder to skip tool_metadata if not set impl Encodable for Diagnostic { fn encode(&self, s: &mut E) -> Result<(), E::Error> { - s.emit_struct("diagnostic", 7, |s| { + s.emit_struct(false, |s| { let mut idx = 0; idx = encode_fields!( @@ -468,7 +468,7 @@ impl DiagnosticSpan { }); DiagnosticSpan { - file_name: start.file.name.to_string(), + file_name: start.file.name.prefer_local().to_string(), byte_start: start.file.original_relative_byte_pos(span.lo()).0, byte_end: start.file.original_relative_byte_pos(span.hi()).0, line_start: start.line, diff --git a/compiler/rustc_errors/src/lib.rs b/compiler/rustc_errors/src/lib.rs index f1a31f0d4f..979f2d3b30 100644 --- a/compiler/rustc_errors/src/lib.rs +++ b/compiler/rustc_errors/src/lib.rs @@ -5,7 +5,8 @@ #![doc(html_root_url = "https://doc.rust-lang.org/nightly/nightly-rustc/")] #![feature(crate_visibility_modifier)] #![feature(backtrace)] -#![feature(extended_key_value_attributes)] +#![cfg_attr(bootstrap, feature(extended_key_value_attributes))] +#![feature(format_args_capture)] #![feature(iter_zip)] #![feature(nll)] @@ -294,6 +295,7 @@ impl error::Error for ExplicitBug {} pub use diagnostic::{Diagnostic, DiagnosticId, DiagnosticStyledString, SubDiagnostic}; pub use diagnostic_builder::DiagnosticBuilder; +use std::backtrace::Backtrace; /// A handler deals with errors and other compiler output. /// Certain errors (fatal, bug, unimpl) may cause immediate exit, @@ -317,7 +319,7 @@ struct HandlerInner { deduplicated_err_count: usize, emitter: Box, delayed_span_bugs: Vec, - delayed_good_path_bugs: Vec, + delayed_good_path_bugs: Vec, /// This set contains the `DiagnosticId` of all emitted diagnostics to avoid /// emitting the same diagnostic with extended help (`--teach`) twice, which @@ -388,7 +390,7 @@ impl Drop for HandlerInner { if !self.has_any_message() { let bugs = std::mem::replace(&mut self.delayed_good_path_bugs, Vec::new()); self.flush_delayed( - bugs, + bugs.into_iter().map(DelayedDiagnostic::decorate).collect(), "no warnings or errors encountered even though `delayed_good_path_bugs` issued", ); } @@ -633,9 +635,9 @@ impl Handler { DiagnosticBuilder::new(self, Level::Note, msg) } - pub fn span_fatal(&self, span: impl Into, msg: &str) -> FatalError { + pub fn span_fatal(&self, span: impl Into, msg: &str) -> ! { self.emit_diag_at_span(Diagnostic::new(Fatal, msg), span); - FatalError + FatalError.raise() } pub fn span_fatal_with_code( @@ -643,9 +645,9 @@ impl Handler { span: impl Into, msg: &str, code: DiagnosticId, - ) -> FatalError { + ) -> ! { self.emit_diag_at_span(Diagnostic::new_with_code(Fatal, Some(code), msg), span); - FatalError + FatalError.raise() } pub fn span_err(&self, span: impl Into, msg: &str) { @@ -691,6 +693,7 @@ impl Handler { db } + // NOTE: intentionally doesn't raise an error so rustc_codegen_ssa only reports fatal errors in the main thread pub fn fatal(&self, msg: &str) -> FatalError { self.inner.borrow_mut().fatal(msg) } @@ -712,6 +715,7 @@ impl Handler { self.inner.borrow_mut().bug(msg) } + #[inline] pub fn err_count(&self) -> usize { self.inner.borrow().err_count() } @@ -921,6 +925,7 @@ impl HandlerInner { } } + #[inline] fn err_count(&self) -> usize { self.err_count + self.stashed_diagnostics.len() } @@ -968,12 +973,12 @@ impl HandlerInner { } fn delay_good_path_bug(&mut self, msg: &str) { - let mut diagnostic = Diagnostic::new(Level::Bug, msg); + let diagnostic = Diagnostic::new(Level::Bug, msg); if self.flags.report_delayed_bugs { self.emit_diagnostic(&diagnostic); } - diagnostic.note(&format!("delayed at {}", std::backtrace::Backtrace::force_capture())); - self.delayed_good_path_bugs.push(diagnostic); + let backtrace = std::backtrace::Backtrace::force_capture(); + self.delayed_good_path_bugs.push(DelayedDiagnostic::with_backtrace(diagnostic, backtrace)); } fn failure(&mut self, msg: &str) { @@ -1042,6 +1047,22 @@ impl HandlerInner { } } +struct DelayedDiagnostic { + inner: Diagnostic, + note: Backtrace, +} + +impl DelayedDiagnostic { + fn with_backtrace(diagnostic: Diagnostic, backtrace: Backtrace) -> Self { + DelayedDiagnostic { inner: diagnostic, note: backtrace } + } + + fn decorate(mut self) -> Diagnostic { + self.inner.note(&format!("delayed at {}", self.note)); + self.inner + } +} + #[derive(Copy, PartialEq, Clone, Hash, Debug, Encodable, Decodable)] pub enum Level { Bug, diff --git a/compiler/rustc_expand/src/base.rs b/compiler/rustc_expand/src/base.rs index 5950584281..aab2741c85 100644 --- a/compiler/rustc_expand/src/base.rs +++ b/compiler/rustc_expand/src/base.rs @@ -14,7 +14,7 @@ use rustc_lint_defs::builtin::PROC_MACRO_BACK_COMPAT; use rustc_lint_defs::BuiltinLintDiagnostics; use rustc_parse::{self, nt_to_tokenstream, parser, MACRO_ARGUMENTS}; use rustc_session::{parse::ParseSess, Limit, Session}; -use rustc_span::def_id::DefId; +use rustc_span::def_id::{CrateNum, DefId}; use rustc_span::edition::Edition; use rustc_span::hygiene::{AstPass, ExpnData, ExpnId, ExpnKind}; use rustc_span::source_map::SourceMap; @@ -810,8 +810,16 @@ impl SyntaxExtension { descr: Symbol, macro_def_id: Option, ) -> ExpnData { + use SyntaxExtensionKind::*; + let proc_macro = match self.kind { + // User-defined proc macro + Bang(..) | Attr(..) | Derive(..) => true, + // Consider everthing else to be not a proc + // macro for diagnostic purposes + LegacyBang(..) | LegacyAttr(..) | NonMacroAttr { .. } | LegacyDerive(..) => false, + }; ExpnData::new( - ExpnKind::Macro(self.macro_kind(), descr), + ExpnKind::Macro { kind: self.macro_kind(), name: descr, proc_macro }, parent, call_site, self.span, @@ -873,6 +881,10 @@ pub trait ResolverExpand { fn take_derive_resolutions(&mut self, expn_id: ExpnId) -> Option; /// Path resolution logic for `#[cfg_accessible(path)]`. fn cfg_accessible(&mut self, expn_id: ExpnId, path: &ast::Path) -> Result; + + /// Decodes the proc-macro quoted span in the specified crate, with the specified id. + /// No caching is performed. + fn get_proc_macro_quoted_span(&self, krate: CrateNum, id: usize) -> Span; } #[derive(Clone, Default)] @@ -1056,11 +1068,11 @@ impl<'a> ExtCtxt<'a> { self.resolver.check_unused_macros(); } - /// Resolves a path mentioned inside Rust code. + /// Resolves a `path` mentioned inside Rust code, returning an absolute path. /// - /// This unifies the logic used for resolving `include_X!`, and `#[doc(include)]` file paths. + /// This unifies the logic used for resolving `include_X!`. /// - /// Returns an absolute path to the file that `path` refers to. + /// FIXME: move this to `rustc_builtin_macros` and make it private. pub fn resolve_path( &self, path: impl Into, @@ -1072,13 +1084,18 @@ impl<'a> ExtCtxt<'a> { // after macro expansion (that is, they are unhygienic). if !path.is_absolute() { let callsite = span.source_callsite(); - let mut result = match self.source_map().span_to_unmapped_path(callsite) { - FileName::Real(name) => name.into_local_path(), + let mut result = match self.source_map().span_to_filename(callsite) { + FileName::Real(name) => name + .into_local_path() + .expect("attempting to resolve a file path in an external file"), FileName::DocTest(path, _) => path, other => { return Err(self.struct_span_err( span, - &format!("cannot resolve relative path in non-file source `{}`", other), + &format!( + "cannot resolve relative path in non-file source `{}`", + other.prefer_local() + ), )); } }; diff --git a/compiler/rustc_expand/src/build.rs b/compiler/rustc_expand/src/build.rs index cb8b939828..ef5b97a946 100644 --- a/compiler/rustc_expand/src/build.rs +++ b/compiler/rustc_expand/src/build.rs @@ -275,7 +275,12 @@ impl<'a> ExtCtxt<'a> { ) -> P { self.expr( span, - ast::ExprKind::Struct(P(ast::StructExpr { path, fields, rest: ast::StructRest::None })), + ast::ExprKind::Struct(P(ast::StructExpr { + qself: None, + path, + fields, + rest: ast::StructRest::None, + })), ) } pub fn expr_struct_ident( @@ -405,7 +410,7 @@ impl<'a> ExtCtxt<'a> { path: ast::Path, subpats: Vec>, ) -> P { - self.pat(span, PatKind::TupleStruct(path, subpats)) + self.pat(span, PatKind::TupleStruct(None, path, subpats)) } pub fn pat_struct( &self, @@ -413,7 +418,7 @@ impl<'a> ExtCtxt<'a> { path: ast::Path, field_pats: Vec, ) -> P { - self.pat(span, PatKind::Struct(path, field_pats, false)) + self.pat(span, PatKind::Struct(None, path, field_pats, false)) } pub fn pat_tuple(&self, span: Span, pats: Vec>) -> P { self.pat(span, PatKind::Tuple(pats)) diff --git a/compiler/rustc_expand/src/config.rs b/compiler/rustc_expand/src/config.rs index 03c83f9c07..f9140609c0 100644 --- a/compiler/rustc_expand/src/config.rs +++ b/compiler/rustc_expand/src/config.rs @@ -464,31 +464,9 @@ impl<'a> StripUnconfigured<'a> { return true; } }; - let error = |span, msg, suggestion: &str| { - let mut err = self.sess.parse_sess.span_diagnostic.struct_span_err(span, msg); - if !suggestion.is_empty() { - err.span_suggestion( - span, - "expected syntax is", - suggestion.into(), - Applicability::MaybeIncorrect, - ); - } - err.emit(); - true - }; - let span = meta_item.span; - match meta_item.meta_item_list() { - None => error(span, "`cfg` is not followed by parentheses", "cfg(/* predicate */)"), - Some([]) => error(span, "`cfg` predicate is not specified", ""), - Some([_, .., l]) => error(l.span(), "multiple `cfg` predicates are specified", ""), - Some([single]) => match single.meta_item() { - Some(meta_item) => { - attr::cfg_matches(meta_item, &self.sess.parse_sess, self.features) - } - None => error(single.span(), "`cfg` predicate key cannot be a literal", ""), - }, - } + parse_cfg(&meta_item, &self.sess).map_or(true, |meta_item| { + attr::cfg_matches(&meta_item, &self.sess.parse_sess, self.features) + }) }) } @@ -532,6 +510,32 @@ impl<'a> StripUnconfigured<'a> { } } +pub fn parse_cfg<'a>(meta_item: &'a MetaItem, sess: &Session) -> Option<&'a MetaItem> { + let error = |span, msg, suggestion: &str| { + let mut err = sess.parse_sess.span_diagnostic.struct_span_err(span, msg); + if !suggestion.is_empty() { + err.span_suggestion( + span, + "expected syntax is", + suggestion.into(), + Applicability::HasPlaceholders, + ); + } + err.emit(); + None + }; + let span = meta_item.span; + match meta_item.meta_item_list() { + None => error(span, "`cfg` is not followed by parentheses", "cfg(/* predicate */)"), + Some([]) => error(span, "`cfg` predicate is not specified", ""), + Some([_, .., l]) => error(l.span(), "multiple `cfg` predicates are specified", ""), + Some([single]) => match single.meta_item() { + Some(meta_item) => Some(meta_item), + None => error(single.span(), "`cfg` predicate key cannot be a literal", ""), + }, + } +} + fn is_cfg(sess: &Session, attr: &Attribute) -> bool { sess.check_name(attr, sym::cfg) } diff --git a/compiler/rustc_expand/src/expand.rs b/compiler/rustc_expand/src/expand.rs index 3347c93948..39c0447bd0 100644 --- a/compiler/rustc_expand/src/expand.rs +++ b/compiler/rustc_expand/src/expand.rs @@ -12,11 +12,11 @@ use rustc_ast::ptr::P; use rustc_ast::token; use rustc_ast::tokenstream::TokenStream; use rustc_ast::visit::{self, AssocCtxt, Visitor}; -use rustc_ast::{AstLike, AttrItem, Block, Inline, ItemKind, LitKind, MacArgs}; +use rustc_ast::{AstLike, Block, Inline, ItemKind, MacArgs}; use rustc_ast::{MacCallStmt, MacStmtStyle, MetaItemKind, ModKind, NestedMetaItem}; use rustc_ast::{NodeId, PatKind, Path, StmtKind, Unsafe}; use rustc_ast_pretty::pprust; -use rustc_attr::{self as attr, is_builtin_attr}; +use rustc_attr::is_builtin_attr; use rustc_data_structures::map_in_place::MapInPlace; use rustc_data_structures::stack::ensure_sufficient_stack; use rustc_data_structures::sync::Lrc; @@ -28,15 +28,14 @@ use rustc_session::lint::builtin::UNUSED_DOC_COMMENTS; use rustc_session::lint::BuiltinLintDiagnostics; use rustc_session::parse::{feature_err, ParseSess}; use rustc_session::Limit; -use rustc_span::symbol::{sym, Ident, Symbol}; -use rustc_span::{ExpnId, FileName, Span, DUMMY_SP}; +use rustc_span::symbol::{sym, Ident}; +use rustc_span::{ExpnId, FileName, Span}; use smallvec::{smallvec, SmallVec}; -use std::io::ErrorKind; use std::ops::DerefMut; use std::path::PathBuf; use std::rc::Rc; -use std::{iter, mem, slice}; +use std::{iter, mem}; macro_rules! ast_fragments { ( @@ -361,9 +360,11 @@ impl<'a, 'b> MacroExpander<'a, 'b> { // FIXME: Avoid visiting the crate as a `Mod` item, // make crate a first class expansion target instead. pub fn expand_crate(&mut self, mut krate: ast::Crate) -> ast::Crate { - let file_path = match self.cx.source_map().span_to_unmapped_path(krate.span) { - FileName::Real(name) => name.into_local_path(), - other => PathBuf::from(other.to_string()), + let file_path = match self.cx.source_map().span_to_filename(krate.span) { + FileName::Real(name) => name + .into_local_path() + .expect("attempting to resolve a file path in an external file"), + other => PathBuf::from(other.prefer_local().to_string()), }; let dir_path = file_path.parent().unwrap_or(&file_path).to_owned(); self.cx.root_path = dir_path.clone(); @@ -1522,139 +1523,6 @@ impl<'a, 'b> MutVisitor for InvocationCollector<'a, 'b> { noop_flat_map_generic_param(param, self) } - fn visit_attribute(&mut self, at: &mut ast::Attribute) { - // turn `#[doc(include="filename")]` attributes into `#[doc(include(file="filename", - // contents="file contents")]` attributes - if !self.cx.sess.check_name(at, sym::doc) { - return noop_visit_attribute(at, self); - } - - if let Some(list) = at.meta_item_list() { - if !list.iter().any(|it| it.has_name(sym::include)) { - return noop_visit_attribute(at, self); - } - - let mut items = vec![]; - - for mut it in list { - if !it.has_name(sym::include) { - items.push({ - noop_visit_meta_list_item(&mut it, self); - it - }); - continue; - } - - if let Some(file) = it.value_str() { - let err_count = self.cx.sess.parse_sess.span_diagnostic.err_count(); - self.check_attributes(slice::from_ref(at)); - if self.cx.sess.parse_sess.span_diagnostic.err_count() > err_count { - // avoid loading the file if they haven't enabled the feature - return noop_visit_attribute(at, self); - } - - let filename = match self.cx.resolve_path(&*file.as_str(), it.span()) { - Ok(filename) => filename, - Err(mut err) => { - err.emit(); - continue; - } - }; - - match self.cx.source_map().load_file(&filename) { - Ok(source_file) => { - let src = source_file - .src - .as_ref() - .expect("freshly loaded file should have a source"); - let src_interned = Symbol::intern(src.as_str()); - - let include_info = vec![ - ast::NestedMetaItem::MetaItem(attr::mk_name_value_item_str( - Ident::with_dummy_span(sym::file), - file, - DUMMY_SP, - )), - ast::NestedMetaItem::MetaItem(attr::mk_name_value_item_str( - Ident::with_dummy_span(sym::contents), - src_interned, - DUMMY_SP, - )), - ]; - - let include_ident = Ident::with_dummy_span(sym::include); - let item = attr::mk_list_item(include_ident, include_info); - items.push(ast::NestedMetaItem::MetaItem(item)); - } - Err(e) => { - let lit_span = it.name_value_literal_span().unwrap(); - - if e.kind() == ErrorKind::InvalidData { - self.cx - .struct_span_err( - lit_span, - &format!("{} wasn't a utf-8 file", filename.display()), - ) - .span_label(lit_span, "contains invalid utf-8") - .emit(); - } else { - let mut err = self.cx.struct_span_err( - lit_span, - &format!("couldn't read {}: {}", filename.display(), e), - ); - err.span_label(lit_span, "couldn't read file"); - - err.emit(); - } - } - } - } else { - let mut err = self - .cx - .struct_span_err(it.span(), "expected path to external documentation"); - - // Check if the user erroneously used `doc(include(...))` syntax. - let literal = it.meta_item_list().and_then(|list| { - if list.len() == 1 { - list[0].literal().map(|literal| &literal.kind) - } else { - None - } - }); - - let (path, applicability) = match &literal { - Some(LitKind::Str(path, ..)) => { - (path.to_string(), Applicability::MachineApplicable) - } - _ => (String::from(""), Applicability::HasPlaceholders), - }; - - err.span_suggestion( - it.span(), - "provide a file path with `=`", - format!("include = \"{}\"", path), - applicability, - ); - - err.emit(); - } - } - - let meta = attr::mk_list_item(Ident::with_dummy_span(sym::doc), items); - *at = ast::Attribute { - kind: ast::AttrKind::Normal( - AttrItem { path: meta.path, args: meta.kind.mac_args(meta.span), tokens: None }, - None, - ), - span: at.span, - id: at.id, - style: at.style, - }; - } else { - noop_visit_attribute(at, self) - } - } - fn visit_id(&mut self, id: &mut ast::NodeId) { if self.monotonic { debug_assert_eq!(*id, ast::DUMMY_NODE_ID); diff --git a/compiler/rustc_expand/src/lib.rs b/compiler/rustc_expand/src/lib.rs index 5fb8586750..efed41de23 100644 --- a/compiler/rustc_expand/src/lib.rs +++ b/compiler/rustc_expand/src/lib.rs @@ -1,9 +1,8 @@ -#![feature(bool_to_option)] #![feature(crate_visibility_modifier)] #![feature(decl_macro)] #![feature(destructuring_assignment)] +#![feature(format_args_capture)] #![feature(iter_zip)] -#![cfg_attr(bootstrap, feature(or_patterns))] #![feature(proc_macro_diagnostic)] #![feature(proc_macro_internals)] #![feature(proc_macro_span)] diff --git a/compiler/rustc_expand/src/mbe/macro_parser.rs b/compiler/rustc_expand/src/mbe/macro_parser.rs index 1aed42a24e..a7434d73ab 100644 --- a/compiler/rustc_expand/src/mbe/macro_parser.rs +++ b/compiler/rustc_expand/src/mbe/macro_parser.rs @@ -85,6 +85,7 @@ use smallvec::{smallvec, SmallVec}; use rustc_data_structures::fx::FxHashMap; use rustc_data_structures::sync::Lrc; +use rustc_span::symbol::Ident; use std::borrow::Cow; use std::collections::hash_map::Entry::{Occupied, Vacant}; use std::mem; @@ -615,7 +616,11 @@ fn inner_parse_loop<'root, 'tt>( /// Use the given sequence of token trees (`ms`) as a matcher. Match the token /// stream from the given `parser` against it and return the match. -pub(super) fn parse_tt(parser: &mut Cow<'_, Parser<'_>>, ms: &[TokenTree]) -> NamedParseResult { +pub(super) fn parse_tt( + parser: &mut Cow<'_, Parser<'_>>, + ms: &[TokenTree], + macro_name: Ident, +) -> NamedParseResult { // A queue of possible matcher positions. We initialize it with the matcher position in which // the "dot" is before the first token of the first token tree in `ms`. `inner_parse_loop` then // processes all of these possible matcher positions and produces possible next positions into @@ -711,7 +716,7 @@ pub(super) fn parse_tt(parser: &mut Cow<'_, Parser<'_>>, ms: &[TokenTree]) -> Na return Error( parser.token.span, format!( - "local ambiguity: multiple parsing options: {}", + "local ambiguity when calling macro `{macro_name}`: multiple parsing options: {}", match next_items.len() { 0 => format!("built-in NTs {}.", nts), 1 => format!("built-in NTs {} or 1 other option.", nts), diff --git a/compiler/rustc_expand/src/mbe/macro_rules.rs b/compiler/rustc_expand/src/mbe/macro_rules.rs index 91d4a0f0d6..abad190b07 100644 --- a/compiler/rustc_expand/src/mbe/macro_rules.rs +++ b/compiler/rustc_expand/src/mbe/macro_rules.rs @@ -245,7 +245,7 @@ fn generic_extension<'cx>( // are not recorded. On the first `Success(..)`ful matcher, the spans are merged. let mut gated_spans_snapshot = mem::take(&mut *sess.gated_spans.spans.borrow_mut()); - match parse_tt(&mut Cow::Borrowed(&parser), lhs_tt) { + match parse_tt(&mut Cow::Borrowed(&parser), lhs_tt, name) { Success(named_matches) => { // The matcher was `Success(..)`ful. // Merge the gated spans from parsing the matcher with the pre-existing ones. @@ -338,7 +338,7 @@ fn generic_extension<'cx>( _ => continue, }; if let Success(_) = - parse_tt(&mut Cow::Borrowed(&parser_from_cx(sess, arg.clone())), lhs_tt) + parse_tt(&mut Cow::Borrowed(&parser_from_cx(sess, arg.clone())), lhs_tt, name) { if comma_span.is_dummy() { err.note("you might be missing a comma"); @@ -432,7 +432,7 @@ pub fn compile_declarative_macro( ]; let parser = Parser::new(&sess.parse_sess, body, true, rustc_parse::MACRO_ARGUMENTS); - let argument_map = match parse_tt(&mut Cow::Borrowed(&parser), &argument_gram) { + let argument_map = match parse_tt(&mut Cow::Borrowed(&parser), &argument_gram, def.ident) { Success(m) => m, Failure(token, msg) => { let s = parse_failure_msg(&token); @@ -467,6 +467,7 @@ pub fn compile_declarative_macro( &sess.parse_sess, def.id, features, + edition, ) .pop() .unwrap(); @@ -492,6 +493,7 @@ pub fn compile_declarative_macro( &sess.parse_sess, def.id, features, + edition, ) .pop() .unwrap(); diff --git a/compiler/rustc_expand/src/mbe/quoted.rs b/compiler/rustc_expand/src/mbe/quoted.rs index aca02ef93f..fb7479eafc 100644 --- a/compiler/rustc_expand/src/mbe/quoted.rs +++ b/compiler/rustc_expand/src/mbe/quoted.rs @@ -9,7 +9,8 @@ use rustc_feature::Features; use rustc_session::parse::ParseSess; use rustc_span::symbol::{kw, Ident}; -use rustc_span::Span; +use rustc_span::edition::Edition; +use rustc_span::{Span, SyntaxContext}; use rustc_data_structures::sync::Lrc; @@ -32,6 +33,7 @@ const VALID_FRAGMENT_NAMES_MSG: &str = "valid fragment specifiers are \ /// - `sess`: the parsing session. Any errors will be emitted to this session. /// - `node_id`: the NodeId of the macro we are parsing. /// - `features`: language features so we can do feature gating. +/// - `edition`: the edition of the crate defining the macro /// /// # Returns /// @@ -42,6 +44,7 @@ pub(super) fn parse( sess: &ParseSess, node_id: NodeId, features: &Features, + edition: Edition, ) -> Vec { // Will contain the final collection of `self::TokenTree` let mut result = Vec::new(); @@ -52,7 +55,7 @@ pub(super) fn parse( while let Some(tree) = trees.next() { // Given the parsed tree, if there is a metavar and we are expecting matchers, actually // parse out the matcher (i.e., in `$id:ident` this would parse the `:` and `ident`). - let tree = parse_tree(tree, &mut trees, expect_matchers, sess, node_id, features); + let tree = parse_tree(tree, &mut trees, expect_matchers, sess, node_id, features, edition); match tree { TokenTree::MetaVar(start_sp, ident) if expect_matchers => { let span = match trees.next() { @@ -64,7 +67,19 @@ pub(super) fn parse( let kind = token::NonterminalKind::from_symbol(frag.name, || { - span.edition() + // FIXME(#85708) - once we properly decode a foreign + // crate's `SyntaxContext::root`, then we can replace + // this with just `span.edition()`. A + // `SyntaxContext::root()` from the current crate will + // have the edition of the current crate, and a + // `SyntaxxContext::root()` from a foreign crate will + // have the edition of that crate (which we manually + // retrieve via the `edition` parameter). + if span.ctxt() == SyntaxContext::root() { + edition + } else { + span.edition() + } }) .unwrap_or_else( || { @@ -117,6 +132,7 @@ pub(super) fn parse( /// - `expect_matchers`: same as for `parse` (see above). /// - `sess`: the parsing session. Any errors will be emitted to this session. /// - `features`: language features so we can do feature gating. +/// - `edition` - the edition of the crate defining the macro fn parse_tree( tree: tokenstream::TokenTree, outer_trees: &mut impl Iterator, @@ -124,6 +140,7 @@ fn parse_tree( sess: &ParseSess, node_id: NodeId, features: &Features, + edition: Edition, ) -> TokenTree { // Depending on what `tree` is, we could be parsing different parts of a macro match tree { @@ -151,7 +168,7 @@ fn parse_tree( sess.span_diagnostic.span_err(span.entire(), &msg); } // Parse the contents of the sequence itself - let sequence = parse(tts, expect_matchers, sess, node_id, features); + let sequence = parse(tts, expect_matchers, sess, node_id, features, edition); // Get the Kleene operator and optional separator let (separator, kleene) = parse_sep_and_kleene_op(&mut trees, span.entire(), sess); @@ -204,7 +221,7 @@ fn parse_tree( span, Lrc::new(Delimited { delim, - tts: parse(tts, expect_matchers, sess, node_id, features), + tts: parse(tts, expect_matchers, sess, node_id, features, edition), }), ), } diff --git a/compiler/rustc_expand/src/module.rs b/compiler/rustc_expand/src/module.rs index c5ce0baaa8..4d777049f0 100644 --- a/compiler/rustc_expand/src/module.rs +++ b/compiler/rustc_expand/src/module.rs @@ -36,8 +36,8 @@ crate struct ParsedExternalMod { pub enum ModError<'a> { CircularInclusion(Vec), ModInBlock(Option), - FileNotFound(Ident, PathBuf), - MultipleCandidates(Ident, String, String), + FileNotFound(Ident, PathBuf, PathBuf), + MultipleCandidates(Ident, PathBuf, PathBuf), ParserError(DiagnosticBuilder<'a>), } @@ -219,10 +219,8 @@ pub fn default_submod_path<'a>( file_path: secondary_path, dir_ownership: DirOwnership::Owned { relative: None }, }), - (false, false) => Err(ModError::FileNotFound(ident, default_path)), - (true, true) => { - Err(ModError::MultipleCandidates(ident, default_path_str, secondary_path_str)) - } + (false, false) => Err(ModError::FileNotFound(ident, default_path, secondary_path)), + (true, true) => Err(ModError::MultipleCandidates(ident, default_path, secondary_path)), } } @@ -249,7 +247,7 @@ impl ModError<'_> { } err } - ModError::FileNotFound(ident, default_path) => { + ModError::FileNotFound(ident, default_path, secondary_path) => { let mut err = struct_span_err!( diag, span, @@ -258,21 +256,22 @@ impl ModError<'_> { ident, ); err.help(&format!( - "to create the module `{}`, create file \"{}\"", + "to create the module `{}`, create file \"{}\" or \"{}\"", ident, default_path.display(), + secondary_path.display(), )); err } - ModError::MultipleCandidates(ident, default_path_short, secondary_path_short) => { + ModError::MultipleCandidates(ident, default_path, secondary_path) => { let mut err = struct_span_err!( diag, span, E0761, - "file for module `{}` found at both {} and {}", + "file for module `{}` found at both \"{}\" and \"{}\"", ident, - default_path_short, - secondary_path_short, + default_path.display(), + secondary_path.display(), ); err.help("delete or rename one of them to remove the ambiguity"); err diff --git a/compiler/rustc_expand/src/proc_macro.rs b/compiler/rustc_expand/src/proc_macro.rs index 3f84979ac0..494b3fb61e 100644 --- a/compiler/rustc_expand/src/proc_macro.rs +++ b/compiler/rustc_expand/src/proc_macro.rs @@ -9,12 +9,14 @@ use rustc_data_structures::sync::Lrc; use rustc_errors::ErrorReported; use rustc_parse::nt_to_tokenstream; use rustc_parse::parser::ForceCollect; +use rustc_span::def_id::CrateNum; use rustc_span::{Span, DUMMY_SP}; const EXEC_STRATEGY: pm::bridge::server::SameThread = pm::bridge::server::SameThread; pub struct BangProcMacro { pub client: pm::bridge::client::Client pm::TokenStream>, + pub krate: CrateNum, } impl base::ProcMacro for BangProcMacro { @@ -24,7 +26,7 @@ impl base::ProcMacro for BangProcMacro { span: Span, input: TokenStream, ) -> Result { - let server = proc_macro_server::Rustc::new(ecx); + let server = proc_macro_server::Rustc::new(ecx, self.krate); self.client.run(&EXEC_STRATEGY, server, input, ecx.ecfg.proc_macro_backtrace).map_err(|e| { let mut err = ecx.struct_span_err(span, "proc macro panicked"); if let Some(s) = e.as_str() { @@ -38,6 +40,7 @@ impl base::ProcMacro for BangProcMacro { pub struct AttrProcMacro { pub client: pm::bridge::client::Client pm::TokenStream>, + pub krate: CrateNum, } impl base::AttrProcMacro for AttrProcMacro { @@ -48,7 +51,7 @@ impl base::AttrProcMacro for AttrProcMacro { annotation: TokenStream, annotated: TokenStream, ) -> Result { - let server = proc_macro_server::Rustc::new(ecx); + let server = proc_macro_server::Rustc::new(ecx, self.krate); self.client .run(&EXEC_STRATEGY, server, annotation, annotated, ecx.ecfg.proc_macro_backtrace) .map_err(|e| { @@ -64,6 +67,7 @@ impl base::AttrProcMacro for AttrProcMacro { pub struct ProcMacroDerive { pub client: pm::bridge::client::Client pm::TokenStream>, + pub krate: CrateNum, } impl MultiItemModifier for ProcMacroDerive { @@ -97,7 +101,7 @@ impl MultiItemModifier for ProcMacroDerive { nt_to_tokenstream(&item, &ecx.sess.parse_sess, CanSynthesizeMissingTokens::No) }; - let server = proc_macro_server::Rustc::new(ecx); + let server = proc_macro_server::Rustc::new(ecx, self.krate); let stream = match self.client.run(&EXEC_STRATEGY, server, input, ecx.ecfg.proc_macro_backtrace) { Ok(stream) => stream, diff --git a/compiler/rustc_expand/src/proc_macro_server.rs b/compiler/rustc_expand/src/proc_macro_server.rs index 1ea26b4eab..92315c4d4f 100644 --- a/compiler/rustc_expand/src/proc_macro_server.rs +++ b/compiler/rustc_expand/src/proc_macro_server.rs @@ -1,12 +1,11 @@ -use crate::base::ExtCtxt; +use crate::base::{ExtCtxt, ResolverExpand}; use rustc_ast as ast; -use rustc_ast::token; -use rustc_ast::token::Nonterminal; -use rustc_ast::token::NtIdent; +use rustc_ast::token::{self, Nonterminal, NtIdent, TokenKind}; use rustc_ast::tokenstream::{self, CanSynthesizeMissingTokens}; use rustc_ast::tokenstream::{DelimSpan, Spacing::*, TokenStream, TreeAndSpacing}; use rustc_ast_pretty::pprust; +use rustc_data_structures::fx::FxHashMap; use rustc_data_structures::sync::Lrc; use rustc_errors::Diagnostic; use rustc_lint_defs::builtin::PROC_MACRO_BACK_COMPAT; @@ -14,6 +13,8 @@ use rustc_lint_defs::BuiltinLintDiagnostics; use rustc_parse::lexer::nfc_normalize; use rustc_parse::{nt_to_tokenstream, parse_stream_from_source_str}; use rustc_session::parse::ParseSess; +use rustc_span::def_id::CrateNum; +use rustc_span::hygiene::ExpnId; use rustc_span::hygiene::ExpnKind; use rustc_span::symbol::{self, kw, sym, Symbol}; use rustc_span::{BytePos, FileName, MultiSpan, Pos, RealFileName, SourceFile, Span}; @@ -355,22 +356,34 @@ pub struct Literal { } pub(crate) struct Rustc<'a> { + resolver: &'a dyn ResolverExpand, sess: &'a ParseSess, def_site: Span, call_site: Span, mixed_site: Span, span_debug: bool, + krate: CrateNum, + expn_id: ExpnId, + rebased_spans: FxHashMap, } impl<'a> Rustc<'a> { - pub fn new(cx: &'a ExtCtxt<'_>) -> Self { + pub fn new(cx: &'a ExtCtxt<'_>, krate: CrateNum) -> Self { let expn_data = cx.current_expansion.id.expn_data(); + let def_site = cx.with_def_site_ctxt(expn_data.def_site); + let call_site = cx.with_call_site_ctxt(expn_data.call_site); + let mixed_site = cx.with_mixed_site_ctxt(expn_data.call_site); + let sess = cx.parse_sess(); Rustc { - sess: &cx.sess.parse_sess, - def_site: cx.with_def_site_ctxt(expn_data.def_site), - call_site: cx.with_call_site_ctxt(expn_data.call_site), - mixed_site: cx.with_mixed_site_ctxt(expn_data.call_site), + resolver: cx.resolver, + sess, + def_site, + call_site, + mixed_site, span_debug: cx.ecfg.span_debug, + krate, + expn_id: cx.current_expansion.id, + rebased_spans: FxHashMap::default(), } } @@ -526,6 +539,33 @@ impl server::Ident for Rustc<'_> { } impl server::Literal for Rustc<'_> { + fn from_str(&mut self, s: &str) -> Result { + let override_span = None; + let stream = parse_stream_from_source_str( + FileName::proc_macro_source_code(s), + s.to_owned(), + self.sess, + override_span, + ); + if stream.len() != 1 { + return Err(()); + } + let tree = stream.into_trees().next().unwrap(); + let token = match tree { + tokenstream::TokenTree::Token(token) => token, + tokenstream::TokenTree::Delimited { .. } => return Err(()), + }; + let span_data = token.span.data(); + if (span_data.hi.0 - span_data.lo.0) as usize != s.len() { + // There is a comment or whitespace adjacent to the literal. + return Err(()); + } + let lit = match token.kind { + TokenKind::Literal(lit) => lit, + _ => return Err(()), + }; + Ok(Literal { lit, span: self.call_site }) + } fn debug_kind(&mut self, literal: &Self::Literal) -> String { format!("{:?}", literal.lit.kind) } @@ -623,10 +663,11 @@ impl server::SourceFile for Rustc<'_> { match file.name { FileName::Real(ref name) => name .local_path() + .expect("attempting to get a file path in an imported file in `proc_macro::SourceFile::path`") .to_str() .expect("non-UTF8 file path in `proc_macro::SourceFile::path`") .to_string(), - _ => file.name.to_string(), + _ => file.name.prefer_local().to_string(), } } fn is_real(&mut self, file: &Self::SourceFile) -> bool { @@ -713,6 +754,51 @@ impl server::Span for Rustc<'_> { fn source_text(&mut self, span: Self::Span) -> Option { self.sess.source_map().span_to_snippet(span).ok() } + /// Saves the provided span into the metadata of + /// *the crate we are currently compiling*, which must + /// be a proc-macro crate. This id can be passed to + /// `recover_proc_macro_span` when our current crate + /// is *run* as a proc-macro. + /// + /// Let's suppose that we have two crates - `my_client` + /// and `my_proc_macro`. The `my_proc_macro` crate + /// contains a procedural macro `my_macro`, which + /// is implemented as: `quote! { "hello" }` + /// + /// When we *compile* `my_proc_macro`, we will execute + /// the `quote` proc-macro. This will save the span of + /// "hello" into the metadata of `my_proc_macro`. As a result, + /// the body of `my_proc_macro` (after expansion) will end + /// up containg a call that looks like this: + /// `proc_macro::Ident::new("hello", proc_macro::Span::recover_proc_macro_span(0))` + /// + /// where `0` is the id returned by this function. + /// When `my_proc_macro` *executes* (during the compilation of `my_client`), + /// the call to `recover_proc_macro_span` will load the corresponding + /// span from the metadata of `my_proc_macro` (which we have access to, + /// since we've loaded `my_proc_macro` from disk in order to execute it). + /// In this way, we have obtained a span pointing into `my_proc_macro` + fn save_span(&mut self, mut span: Self::Span) -> usize { + // Throw away the `SyntaxContext`, since we currently + // skip serializing `SyntaxContext`s for proc-macro crates + span = span.with_ctxt(rustc_span::SyntaxContext::root()); + self.sess.save_proc_macro_span(span) + } + fn recover_proc_macro_span(&mut self, id: usize) -> Self::Span { + let resolver = self.resolver; + let krate = self.krate; + let expn_id = self.expn_id; + *self.rebased_spans.entry(id).or_insert_with(|| { + let raw_span = resolver.get_proc_macro_quoted_span(krate, id); + // Ignore the deserialized `SyntaxContext` entirely. + // FIXME: Preserve the macro backtrace from the serialized span + // For example, if a proc-macro crate has code like + // `macro_one!() -> macro_two!() -> quote!()`, we might + // want to 'concatenate' this backtrace with the backtrace from + // our current call site. + raw_span.with_def_site_ctxt(expn_id) + }) + } } // See issue #74616 for details @@ -722,10 +808,10 @@ fn ident_name_compatibility_hack( rustc: &mut Rustc<'_>, ) -> Option<(rustc_span::symbol::Ident, bool)> { if let NtIdent(ident, is_raw) = nt { - if let ExpnKind::Macro(_, macro_name) = orig_span.ctxt().outer_expn_data().kind { + if let ExpnKind::Macro { name: macro_name, .. } = orig_span.ctxt().outer_expn_data().kind { let source_map = rustc.sess.source_map(); let filename = source_map.span_to_filename(orig_span); - if let FileName::Real(RealFileName::Named(path)) = filename { + if let FileName::Real(RealFileName::LocalPath(path)) = filename { let matches_prefix = |prefix, filename| { // Check for a path that ends with 'prefix*/src/' let mut iter = path.components().rev(); @@ -788,7 +874,7 @@ fn ident_name_compatibility_hack( if macro_name == sym::tuple_from_req && matches_prefix("actix-web", "extract.rs") { let snippet = source_map.span_to_snippet(orig_span); if snippet.as_deref() == Ok("$T") { - if let FileName::Real(RealFileName::Named(macro_path)) = + if let FileName::Real(RealFileName::LocalPath(macro_path)) = source_map.span_to_filename(rustc.def_site) { if macro_path.to_string_lossy().contains("pin-project-internal-0.") { diff --git a/compiler/rustc_feature/src/accepted.rs b/compiler/rustc_feature/src/accepted.rs index e8642a5274..95504723e7 100644 --- a/compiler/rustc_feature/src/accepted.rs +++ b/compiler/rustc_feature/src/accepted.rs @@ -281,6 +281,12 @@ declare_features! ( (accepted, or_patterns, "1.53.0", Some(54883), None), /// Allows defining identifiers beyond ASCII. (accepted, non_ascii_idents, "1.53.0", Some(55467), None), + /// Allows arbitrary expressions in key-value attributes at parse time. + (accepted, extended_key_value_attributes, "1.54.0", Some(78835), None), + /// Allows unsizing coercions in `const fn`. + (accepted, const_fn_unsize, "1.54.0", Some(64992), None), + /// Allows `impl Trait` with multiple unrelated lifetimes. + (accepted, member_constraints, "1.54.0", Some(61997), None), // ------------------------------------------------------------------------- // feature-group-end: accepted features diff --git a/compiler/rustc_feature/src/active.rs b/compiler/rustc_feature/src/active.rs index bc9f1039d6..16f67783c1 100644 --- a/compiler/rustc_feature/src/active.rs +++ b/compiler/rustc_feature/src/active.rs @@ -250,6 +250,7 @@ declare_features! ( (active, f16c_target_feature, "1.36.0", Some(44839), None), (active, riscv_target_feature, "1.45.0", Some(44839), None), (active, ermsb_target_feature, "1.49.0", Some(44839), None), + (active, bpf_target_feature, "1.54.0", Some(44839), None), // ------------------------------------------------------------------------- // feature-group-end: actual feature gates (target features) @@ -274,9 +275,6 @@ declare_features! ( /// Allows using non lexical lifetimes (RFC 2094). (active, nll, "1.0.0", Some(43234), None), - /// Allows the definition of `const` functions with some advanced features. - (active, const_fn, "1.2.0", Some(57563), None), - /// Allows associated type defaults. (active, associated_type_defaults, "1.2.0", Some(29661), None), @@ -373,9 +371,6 @@ declare_features! ( /// Allows `#[doc(masked)]`. (active, doc_masked, "1.21.0", Some(44027), None), - /// Allows `#[doc(include = "some-file")]`. - (active, external_doc, "1.22.0", Some(44732), None), - /// Allows using `crate` as visibility modifier, synonymous with `pub(crate)`. (active, crate_visibility_modifier, "1.23.0", Some(53120), None), @@ -475,9 +470,6 @@ declare_features! ( /// Allows explicit discriminants on non-unit enum variants. (active, arbitrary_enum_discriminant, "1.37.0", Some(60553), None), - /// Allows `impl Trait` with multiple unrelated lifetimes. - (active, member_constraints, "1.37.0", Some(61997), None), - /// Allows `async || body` closures. (active, async_closure, "1.37.0", Some(62290), None), @@ -604,9 +596,6 @@ declare_features! ( /// Allows capturing disjoint fields in a closure/generator (RFC 2229). (active, capture_disjoint_fields, "1.49.0", Some(53488), None), - /// Allows arbitrary expressions in key-value attributes at parse time. - (active, extended_key_value_attributes, "1.50.0", Some(78835), None), - /// Allows const generics to have default values (e.g. `struct Foo(...);`). (active, const_generics_defaults, "1.51.0", Some(44580), None), @@ -647,12 +636,33 @@ declare_features! ( /// Allows trait bounds in `const fn`. (active, const_fn_trait_bound, "1.53.0", Some(57563), None), - /// Allows unsizing coercions in `const fn`. - (active, const_fn_unsize, "1.53.0", Some(64992), None), + /// Allows `async {}` expressions in const contexts. + (active, const_async_blocks, "1.53.0", Some(85368), None), /// Allows using imported `main` function (active, imported_main, "1.53.0", Some(28937), None), + /// Allows specifying modifiers in the link attribute: `#[link(modifiers = "...")]` + (active, native_link_modifiers, "1.53.0", Some(81490), None), + + /// Allows specifying the bundle link modifier + (active, native_link_modifiers_bundle, "1.53.0", Some(81490), None), + + /// Allows specifying the verbatim link modifier + (active, native_link_modifiers_verbatim, "1.53.0", Some(81490), None), + + /// Allows specifying the whole-archive link modifier + (active, native_link_modifiers_whole_archive, "1.53.0", Some(81490), None), + + /// Allows specifying the as-needed link modifier + (active, native_link_modifiers_as_needed, "1.53.0", Some(81490), None), + + /// Allows unnamed fields of struct and union type + (active, unnamed_fields, "1.53.0", Some(49804), None), + + /// Allows qualified paths in struct expressions, struct patterns and tuple struct patterns. + (active, more_qualified_paths, "1.54.0", Some(80080), None), + // ------------------------------------------------------------------------- // feature-group-end: actual feature gates // ------------------------------------------------------------------------- @@ -677,9 +687,10 @@ pub const INCOMPLETE_FEATURES: &[Symbol] = &[ sym::repr128, sym::unsized_locals, sym::capture_disjoint_fields, - sym::const_generics_defaults, sym::inherent_associated_types, sym::type_alias_impl_trait, + sym::rustc_insignificant_dtor, + sym::unnamed_fields, ]; /// Some features are not allowed to be used together at the same time, if diff --git a/compiler/rustc_feature/src/builtin_attrs.rs b/compiler/rustc_feature/src/builtin_attrs.rs index a8719be84c..259a6328a2 100644 --- a/compiler/rustc_feature/src/builtin_attrs.rs +++ b/compiler/rustc_feature/src/builtin_attrs.rs @@ -469,7 +469,6 @@ pub const BUILTIN_ATTRIBUTES: &[BuiltinAttribute] = &[ // ========================================================================== rustc_attr!(rustc_promotable, AssumedUsed, template!(Word), IMPL_DETAIL), - rustc_attr!(rustc_args_required_const, AssumedUsed, template!(List: "N"), INTERNAL_UNSTABLE), rustc_attr!(rustc_legacy_const_generics, AssumedUsed, template!(List: "N"), INTERNAL_UNSTABLE), // ========================================================================== @@ -557,6 +556,7 @@ pub const BUILTIN_ATTRIBUTES: &[BuiltinAttribute] = &[ rustc_attr!(TEST, rustc_outlives, Normal, template!(Word)), rustc_attr!(TEST, rustc_capture_analysis, Normal, template!(Word)), + rustc_attr!(TEST, rustc_insignificant_dtor, Normal, template!(Word)), rustc_attr!(TEST, rustc_variance, Normal, template!(Word)), rustc_attr!(TEST, rustc_layout, Normal, template!(List: "field1, field2, ...")), rustc_attr!(TEST, rustc_regions, Normal, template!(Word)), @@ -565,12 +565,9 @@ pub const BUILTIN_ATTRIBUTES: &[BuiltinAttribute] = &[ template!(Word, List: "delay_span_bug_from_inside_query") ), rustc_attr!(TEST, rustc_dump_user_substs, AssumedUsed, template!(Word)), + rustc_attr!(TEST, rustc_evaluate_where_clauses, AssumedUsed, template!(Word)), rustc_attr!(TEST, rustc_if_this_changed, AssumedUsed, template!(Word, List: "DepNode")), rustc_attr!(TEST, rustc_then_this_would_need, AssumedUsed, template!(List: "DepNode")), - rustc_attr!( - TEST, rustc_dirty, AssumedUsed, - template!(List: r#"cfg = "...", /*opt*/ label = "...", /*opt*/ except = "...""#), - ), rustc_attr!( TEST, rustc_clean, AssumedUsed, template!(List: r#"cfg = "...", /*opt*/ label = "...", /*opt*/ except = "...""#), diff --git a/compiler/rustc_feature/src/removed.rs b/compiler/rustc_feature/src/removed.rs index fa8ef182ae..71c10eb650 100644 --- a/compiler/rustc_feature/src/removed.rs +++ b/compiler/rustc_feature/src/removed.rs @@ -136,6 +136,13 @@ declare_features! ( (removed, main, "1.53.0", Some(29634), None, None), (removed, pub_macro_rules, "1.53.0", Some(78855), None, Some("removed due to being incomplete, in particular it does not work across crates")), + /// Allows the definition of `const` functions with some advanced features. + (removed, const_fn, "1.54.0", Some(57563), None, + Some("split into finer-grained feature gates")), + + /// Allows `#[doc(include = "some-file")]`. + (removed, external_doc, "1.54.0", Some(44732), None, + Some("use #[doc = include_str!(\"filename\")] instead, which handles macro invocations")), // ------------------------------------------------------------------------- // feature-group-end: removed features diff --git a/compiler/rustc_hir/src/arena.rs b/compiler/rustc_hir/src/arena.rs index ddf8218616..b05ca381b8 100644 --- a/compiler/rustc_hir/src/arena.rs +++ b/compiler/rustc_hir/src/arena.rs @@ -19,7 +19,6 @@ macro_rules! arena_types { [] attribute: rustc_ast::Attribute, [] block: rustc_hir::Block<$tcx>, [] bare_fn_ty: rustc_hir::BareFnTy<$tcx>, - [few] global_asm: rustc_hir::GlobalAsm, [] generic_arg: rustc_hir::GenericArg<$tcx>, [] generic_args: rustc_hir::GenericArgs<$tcx>, [] generic_bound: rustc_hir::GenericBound<$tcx>, diff --git a/compiler/rustc_hir/src/definitions.rs b/compiler/rustc_hir/src/definitions.rs index 0f77de9fb2..753b8c8567 100644 --- a/compiler/rustc_hir/src/definitions.rs +++ b/compiler/rustc_hir/src/definitions.rs @@ -87,11 +87,6 @@ impl DefPathTable { hash } - /// Used by librustdoc for fake DefIds. - pub fn num_def_ids(&self) -> usize { - self.index_to_key.len() - } - pub fn enumerated_keys_and_path_hashes( &self, ) -> impl Iterator + '_ { @@ -99,15 +94,6 @@ impl DefPathTable { .iter_enumerated() .map(move |(index, key)| (index, key, &self.def_path_hashes[index])) } - - pub fn all_def_path_hashes_and_def_ids( - &self, - krate: CrateNum, - ) -> impl Iterator + '_ { - self.def_path_hashes - .iter_enumerated() - .map(move |(index, hash)| (*hash, DefId { krate, index })) - } } /// The definition table containing node definitions. @@ -311,6 +297,7 @@ impl Definitions { self.table.index_to_key.len() } + #[inline] pub fn def_key(&self, id: LocalDefId) -> DefKey { self.table.def_key(id.local_def_index) } @@ -444,6 +431,14 @@ impl Definitions { pub fn iter_local_def_id(&self) -> impl Iterator + '_ { self.def_id_to_hir_id.iter_enumerated().map(|(k, _)| k) } + + #[inline(always)] + pub fn local_def_path_hash_to_def_id(&self, hash: DefPathHash) -> Option { + self.table + .def_path_hash_to_index + .get(&hash) + .map(|&local_def_index| LocalDefId { local_def_index }) + } } #[derive(Copy, Clone, PartialEq, Debug)] diff --git a/compiler/rustc_hir/src/hir.rs b/compiler/rustc_hir/src/hir.rs index 5baaaad737..577d43b1c8 100644 --- a/compiler/rustc_hir/src/hir.rs +++ b/compiler/rustc_hir/src/hir.rs @@ -1,7 +1,7 @@ // ignore-tidy-filelength use crate::def::{CtorKind, DefKind, Res}; use crate::def_id::DefId; -crate use crate::hir_id::HirId; +crate use crate::hir_id::{HirId, ItemLocalId}; use crate::{itemlikevisit, LangItem}; use rustc_ast::util::parser::ExprPrecedence; @@ -10,9 +10,10 @@ use rustc_ast::{Attribute, FloatTy, IntTy, Label, LitKind, StrStyle, TraitObject pub use rustc_ast::{BorrowKind, ImplPolarity, IsAuto}; pub use rustc_ast::{CaptureBy, Movability, Mutability}; use rustc_ast::{InlineAsmOptions, InlineAsmTemplatePiece}; +use rustc_data_structures::fx::FxHashMap; use rustc_data_structures::sync::{par_for_each_in, Send, Sync}; use rustc_macros::HashStable_Generic; -use rustc_span::source_map::{SourceMap, Spanned}; +use rustc_span::source_map::Spanned; use rustc_span::symbol::{kw, sym, Ident, Symbol}; use rustc_span::{def_id::LocalDefId, BytePos}; use rustc_span::{MultiSpan, Span, DUMMY_SP}; @@ -314,11 +315,18 @@ pub struct GenericArgs<'hir> { /// This is required mostly for pretty-printing and diagnostics, /// but also for changing lifetime elision rules to be "function-like". pub parenthesized: bool, + /// The span encompassing arguments and the surrounding brackets `<>` or `()` + /// Foo Fn(T, U, V) -> W + /// ^^^^^^^^^^^^^^^^^^^ ^^^^^^^^^ + /// Note that this may be: + /// - empty, if there are no generic brackets (but there may be hidden lifetimes) + /// - dummy, if this was generated while desugaring + pub span_ext: Span, } impl GenericArgs<'_> { pub const fn none() -> Self { - Self { args: &[], bindings: &[], parenthesized: false } + Self { args: &[], bindings: &[], parenthesized: false, span_ext: DUMMY_SP } } pub fn inputs(&self) -> &[Ty<'_>] { @@ -356,33 +364,17 @@ impl GenericArgs<'_> { own_counts } + /// The span encompassing the text inside the surrounding brackets. + /// It will also include bindings if they aren't in the form `-> Ret` + /// Returns `None` if the span is empty (e.g. no brackets) or dummy pub fn span(&self) -> Option { - self.args - .iter() - .filter(|arg| !arg.is_synthetic()) - .map(|arg| arg.span()) - .reduce(|span1, span2| span1.to(span2)) + let span_ext = self.span_ext()?; + Some(span_ext.with_lo(span_ext.lo() + BytePos(1)).with_hi(span_ext.hi() - BytePos(1))) } /// Returns span encompassing arguments and their surrounding `<>` or `()` - pub fn span_ext(&self, sm: &SourceMap) -> Option { - let mut span = self.span()?; - - let (o, c) = if self.parenthesized { ('(', ')') } else { ('<', '>') }; - - if let Ok(snippet) = sm.span_to_snippet(span) { - let snippet = snippet.as_bytes(); - - if snippet[0] != (o as u8) || snippet[snippet.len() - 1] != (c as u8) { - span = sm.span_extend_to_prev_char(span, o, true); - span = span.with_lo(span.lo() - BytePos(1)); - - span = sm.span_extend_to_next_char(span, c, true); - span = span.with_hi(span.hi() + BytePos(1)); - } - } - - Some(span) + pub fn span_ext(&self) -> Option { + Some(self.span_ext).filter(|span| !span.is_empty()) } pub fn is_empty(&self) -> bool { @@ -667,7 +659,9 @@ pub struct Crate<'hir> { /// they are declared in the static array generated by proc_macro_harness. pub proc_macros: Vec, - pub trait_map: BTreeMap>, + /// Map indicating what traits are in scope for places where this + /// is relevant; generated by resolve. + pub trait_map: FxHashMap>>, /// Collected attributes from HIR nodes. pub attrs: BTreeMap, @@ -2494,6 +2488,7 @@ pub enum FnRetTy<'hir> { } impl FnRetTy<'_> { + #[inline] pub fn span(&self) -> Span { match *self { Self::DefaultReturn(span) => span, @@ -2511,11 +2506,6 @@ pub struct Mod<'hir> { pub item_ids: &'hir [ItemId], } -#[derive(Encodable, Debug, HashStable_Generic)] -pub struct GlobalAsm { - pub asm: Symbol, -} - #[derive(Debug, HashStable_Generic)] pub struct EnumDef<'hir> { pub variants: &'hir [Variant<'hir>], @@ -2775,7 +2765,7 @@ pub enum ItemKind<'hir> { /// An external module, e.g. `extern { .. }`. ForeignMod { abi: Abi, items: &'hir [ForeignItemRef<'hir>] }, /// Module-level inline assembly (from `global_asm!`). - GlobalAsm(&'hir GlobalAsm), + GlobalAsm(&'hir InlineAsm<'hir>), /// A type alias, e.g., `type Foo = Bar`. TyAlias(&'hir Ty<'hir>, Generics<'hir>), /// An opaque `impl Trait` type alias, e.g., `type Foo = impl Bar;`. diff --git a/compiler/rustc_hir/src/intravisit.rs b/compiler/rustc_hir/src/intravisit.rs index 0ce04a77a5..c08f1f5321 100644 --- a/compiler/rustc_hir/src/intravisit.rs +++ b/compiler/rustc_hir/src/intravisit.rs @@ -589,8 +589,9 @@ pub fn walk_item<'v, V: Visitor<'v>>(visitor: &mut V, item: &'v Item<'v>) { visitor.visit_id(item.hir_id()); walk_list!(visitor, visit_foreign_item_ref, items); } - ItemKind::GlobalAsm(_) => { + ItemKind::GlobalAsm(asm) => { visitor.visit_id(item.hir_id()); + walk_inline_asm(visitor, asm); } ItemKind::TyAlias(ref ty, ref generics) => { visitor.visit_id(item.hir_id()); @@ -650,6 +651,28 @@ pub fn walk_item<'v, V: Visitor<'v>>(visitor: &mut V, item: &'v Item<'v>) { } } +fn walk_inline_asm<'v, V: Visitor<'v>>(visitor: &mut V, asm: &'v InlineAsm<'v>) { + for (op, _op_sp) in asm.operands { + match op { + InlineAsmOperand::In { expr, .. } + | InlineAsmOperand::InOut { expr, .. } + | InlineAsmOperand::Sym { expr, .. } => visitor.visit_expr(expr), + InlineAsmOperand::Out { expr, .. } => { + if let Some(expr) = expr { + visitor.visit_expr(expr); + } + } + InlineAsmOperand::SplitInOut { in_expr, out_expr, .. } => { + visitor.visit_expr(in_expr); + if let Some(out_expr) = out_expr { + visitor.visit_expr(out_expr); + } + } + InlineAsmOperand::Const { anon_const } => visitor.visit_anon_const(anon_const), + } + } +} + pub fn walk_use<'v, V: Visitor<'v>>(visitor: &mut V, path: &'v Path<'v>, hir_id: HirId) { visitor.visit_id(hir_id); visitor.visit_path(path, hir_id); @@ -1185,27 +1208,7 @@ pub fn walk_expr<'v, V: Visitor<'v>>(visitor: &mut V, expression: &'v Expr<'v>) walk_list!(visitor, visit_expr, optional_expression); } ExprKind::InlineAsm(ref asm) => { - for (op, _op_sp) in asm.operands { - match op { - InlineAsmOperand::In { expr, .. } - | InlineAsmOperand::InOut { expr, .. } - | InlineAsmOperand::Sym { expr, .. } => visitor.visit_expr(expr), - InlineAsmOperand::Out { expr, .. } => { - if let Some(expr) = expr { - visitor.visit_expr(expr); - } - } - InlineAsmOperand::SplitInOut { in_expr, out_expr, .. } => { - visitor.visit_expr(in_expr); - if let Some(out_expr) = out_expr { - visitor.visit_expr(out_expr); - } - } - InlineAsmOperand::Const { anon_const, .. } => { - visitor.visit_anon_const(anon_const) - } - } - } + walk_inline_asm(visitor, asm); } ExprKind::LlvmInlineAsm(ref asm) => { walk_list!(visitor, visit_expr, asm.outputs_exprs); diff --git a/compiler/rustc_hir/src/lang_items.rs b/compiler/rustc_hir/src/lang_items.rs index 498000db50..0b8535f8ca 100644 --- a/compiler/rustc_hir/src/lang_items.rs +++ b/compiler/rustc_hir/src/lang_items.rs @@ -308,12 +308,12 @@ language_item_table! { Termination, sym::termination, termination, Target::Trait; - Try, kw::Try, try_trait, Target::Trait; + Try, sym::Try, try_trait, Target::Trait; // Language items from AST lowering - TryFromError, sym::from_error, from_error_fn, Target::Method(MethodKind::Trait { body: false }); - TryFromOk, sym::from_ok, from_ok_fn, Target::Method(MethodKind::Trait { body: false }); - TryIntoResult, sym::into_result, into_result_fn, Target::Method(MethodKind::Trait { body: false }); + TryTraitFromResidual, sym::from_residual, from_residual_fn, Target::Method(MethodKind::Trait { body: false }); + TryTraitFromOutput, sym::from_output, from_output_fn, Target::Method(MethodKind::Trait { body: false }); + TryTraitBranch, sym::branch, branch_fn, Target::Method(MethodKind::Trait { body: false }); PollReady, sym::Ready, poll_ready_variant, Target::Variant; PollPending, sym::Pending, poll_pending_variant, Target::Variant; @@ -331,6 +331,9 @@ language_item_table! { ResultOk, sym::Ok, result_ok_variant, Target::Variant; ResultErr, sym::Err, result_err_variant, Target::Variant; + ControlFlowContinue, sym::Continue, cf_continue_variant, Target::Variant; + ControlFlowBreak, sym::Break, cf_break_variant, Target::Variant; + IntoIterIntoIter, sym::into_iter, into_iter_fn, Target::Method(MethodKind::Trait { body: false }); IteratorNext, sym::next, next_fn, Target::Method(MethodKind::Trait { body: false}); @@ -343,4 +346,7 @@ language_item_table! { Range, sym::Range, range_struct, Target::Struct; RangeToInclusive, sym::RangeToInclusive, range_to_inclusive_struct, Target::Struct; RangeTo, sym::RangeTo, range_to_struct, Target::Struct; + Send, sym::send, send_trait, Target::Trait; + UnwindSafe, sym::unwind_safe, unwind_safe_trait, Target::Trait; + RefUnwindSafe, sym::ref_unwind_safe, ref_unwind_safe_trait, Target::Trait; } diff --git a/compiler/rustc_hir/src/lib.rs b/compiler/rustc_hir/src/lib.rs index 65c99535c4..ad2ecae923 100644 --- a/compiler/rustc_hir/src/lib.rs +++ b/compiler/rustc_hir/src/lib.rs @@ -3,11 +3,10 @@ //! [rustc dev guide]: https://rustc-dev-guide.rust-lang.org/hir.html #![feature(crate_visibility_modifier)] -#![feature(const_panic)] -#![feature(extended_key_value_attributes)] +#![cfg_attr(bootstrap, feature(extended_key_value_attributes))] #![feature(in_band_lifetimes)] #![feature(once_cell)] -#![cfg_attr(bootstrap, feature(or_patterns))] +#![feature(min_specialization)] #![recursion_limit = "256"] #[macro_use] diff --git a/compiler/rustc_hir_pretty/src/lib.rs b/compiler/rustc_hir_pretty/src/lib.rs index 77d083fc5e..2b932b7c95 100644 --- a/compiler/rustc_hir_pretty/src/lib.rs +++ b/compiler/rustc_hir_pretty/src/lib.rs @@ -1,4 +1,3 @@ -#![cfg_attr(bootstrap, feature(or_patterns))] #![recursion_limit = "256"] use rustc_ast as ast; @@ -660,9 +659,9 @@ impl<'a> State<'a> { } self.bclose(item.span); } - hir::ItemKind::GlobalAsm(ref ga) => { - self.head(visibility_qualified(&item.vis, "global asm")); - self.s.word(ga.asm.to_string()); + hir::ItemKind::GlobalAsm(ref asm) => { + self.head(visibility_qualified(&item.vis, "global_asm!")); + self.print_inline_asm(asm); self.end() } hir::ItemKind::TyAlias(ref ty, ref generics) => { @@ -1352,6 +1351,110 @@ impl<'a> State<'a> { self.word(lit.node.to_lit_token().to_string()) } + fn print_inline_asm(&mut self, asm: &hir::InlineAsm<'_>) { + enum AsmArg<'a> { + Template(String), + Operand(&'a hir::InlineAsmOperand<'a>), + Options(ast::InlineAsmOptions), + } + + let mut args = vec![]; + args.push(AsmArg::Template(ast::InlineAsmTemplatePiece::to_string(&asm.template))); + args.extend(asm.operands.iter().map(|(o, _)| AsmArg::Operand(o))); + if !asm.options.is_empty() { + args.push(AsmArg::Options(asm.options)); + } + + self.popen(); + self.commasep(Consistent, &args, |s, arg| match arg { + AsmArg::Template(template) => s.print_string(&template, ast::StrStyle::Cooked), + AsmArg::Operand(op) => match op { + hir::InlineAsmOperand::In { reg, expr } => { + s.word("in"); + s.popen(); + s.word(format!("{}", reg)); + s.pclose(); + s.space(); + s.print_expr(expr); + } + hir::InlineAsmOperand::Out { reg, late, expr } => { + s.word(if *late { "lateout" } else { "out" }); + s.popen(); + s.word(format!("{}", reg)); + s.pclose(); + s.space(); + match expr { + Some(expr) => s.print_expr(expr), + None => s.word("_"), + } + } + hir::InlineAsmOperand::InOut { reg, late, expr } => { + s.word(if *late { "inlateout" } else { "inout" }); + s.popen(); + s.word(format!("{}", reg)); + s.pclose(); + s.space(); + s.print_expr(expr); + } + hir::InlineAsmOperand::SplitInOut { reg, late, in_expr, out_expr } => { + s.word(if *late { "inlateout" } else { "inout" }); + s.popen(); + s.word(format!("{}", reg)); + s.pclose(); + s.space(); + s.print_expr(in_expr); + s.space(); + s.word_space("=>"); + match out_expr { + Some(out_expr) => s.print_expr(out_expr), + None => s.word("_"), + } + } + hir::InlineAsmOperand::Const { anon_const } => { + s.word("const"); + s.space(); + s.print_anon_const(anon_const); + } + hir::InlineAsmOperand::Sym { expr } => { + s.word("sym"); + s.space(); + s.print_expr(expr); + } + }, + AsmArg::Options(opts) => { + s.word("options"); + s.popen(); + let mut options = vec![]; + if opts.contains(ast::InlineAsmOptions::PURE) { + options.push("pure"); + } + if opts.contains(ast::InlineAsmOptions::NOMEM) { + options.push("nomem"); + } + if opts.contains(ast::InlineAsmOptions::READONLY) { + options.push("readonly"); + } + if opts.contains(ast::InlineAsmOptions::PRESERVES_FLAGS) { + options.push("preserves_flags"); + } + if opts.contains(ast::InlineAsmOptions::NORETURN) { + options.push("noreturn"); + } + if opts.contains(ast::InlineAsmOptions::NOSTACK) { + options.push("nostack"); + } + if opts.contains(ast::InlineAsmOptions::ATT_SYNTAX) { + options.push("att_syntax"); + } + s.commasep(Inconsistent, &options, |s, &opt| { + s.word(opt); + }); + s.pclose(); + } + }); + self.pclose(); + } + pub fn print_expr(&mut self, expr: &hir::Expr<'_>) { self.maybe_print_comment(expr.span.lo()); self.print_outer_attributes(self.attrs(expr.hir_id)); @@ -1530,109 +1633,9 @@ impl<'a> State<'a> { self.print_expr_maybe_paren(&expr, parser::PREC_JUMP); } } - hir::ExprKind::InlineAsm(ref a) => { - enum AsmArg<'a> { - Template(String), - Operand(&'a hir::InlineAsmOperand<'a>), - Options(ast::InlineAsmOptions), - } - - let mut args = vec![]; - args.push(AsmArg::Template(ast::InlineAsmTemplatePiece::to_string(&a.template))); - args.extend(a.operands.iter().map(|(o, _)| AsmArg::Operand(o))); - if !a.options.is_empty() { - args.push(AsmArg::Options(a.options)); - } - + hir::ExprKind::InlineAsm(ref asm) => { self.word("asm!"); - self.popen(); - self.commasep(Consistent, &args, |s, arg| match arg { - AsmArg::Template(template) => s.print_string(&template, ast::StrStyle::Cooked), - AsmArg::Operand(op) => match op { - hir::InlineAsmOperand::In { reg, expr } => { - s.word("in"); - s.popen(); - s.word(format!("{}", reg)); - s.pclose(); - s.space(); - s.print_expr(expr); - } - hir::InlineAsmOperand::Out { reg, late, expr } => { - s.word(if *late { "lateout" } else { "out" }); - s.popen(); - s.word(format!("{}", reg)); - s.pclose(); - s.space(); - match expr { - Some(expr) => s.print_expr(expr), - None => s.word("_"), - } - } - hir::InlineAsmOperand::InOut { reg, late, expr } => { - s.word(if *late { "inlateout" } else { "inout" }); - s.popen(); - s.word(format!("{}", reg)); - s.pclose(); - s.space(); - s.print_expr(expr); - } - hir::InlineAsmOperand::SplitInOut { reg, late, in_expr, out_expr } => { - s.word(if *late { "inlateout" } else { "inout" }); - s.popen(); - s.word(format!("{}", reg)); - s.pclose(); - s.space(); - s.print_expr(in_expr); - s.space(); - s.word_space("=>"); - match out_expr { - Some(out_expr) => s.print_expr(out_expr), - None => s.word("_"), - } - } - hir::InlineAsmOperand::Const { anon_const } => { - s.word("const"); - s.space(); - s.print_anon_const(anon_const); - } - hir::InlineAsmOperand::Sym { expr } => { - s.word("sym"); - s.space(); - s.print_expr(expr); - } - }, - AsmArg::Options(opts) => { - s.word("options"); - s.popen(); - let mut options = vec![]; - if opts.contains(ast::InlineAsmOptions::PURE) { - options.push("pure"); - } - if opts.contains(ast::InlineAsmOptions::NOMEM) { - options.push("nomem"); - } - if opts.contains(ast::InlineAsmOptions::READONLY) { - options.push("readonly"); - } - if opts.contains(ast::InlineAsmOptions::PRESERVES_FLAGS) { - options.push("preserves_flags"); - } - if opts.contains(ast::InlineAsmOptions::NORETURN) { - options.push("noreturn"); - } - if opts.contains(ast::InlineAsmOptions::NOSTACK) { - options.push("nostack"); - } - if opts.contains(ast::InlineAsmOptions::ATT_SYNTAX) { - options.push("att_syntax"); - } - s.commasep(Inconsistent, &options, |s, &opt| { - s.word(opt); - }); - s.pclose(); - } - }); - self.pclose(); + self.print_inline_asm(asm); } hir::ExprKind::LlvmInlineAsm(ref a) => { let i = &a.inner; diff --git a/compiler/rustc_incremental/Cargo.toml b/compiler/rustc_incremental/Cargo.toml index 049e5b8b72..85bf4dc176 100644 --- a/compiler/rustc_incremental/Cargo.toml +++ b/compiler/rustc_incremental/Cargo.toml @@ -20,3 +20,4 @@ rustc_macros = { path = "../rustc_macros" } rustc_span = { path = "../rustc_span" } rustc_fs_util = { path = "../rustc_fs_util" } rustc_session = { path = "../rustc_session" } +rustc_errors = { path = "../rustc_errors" } diff --git a/compiler/rustc_incremental/src/assert_module_sources.rs b/compiler/rustc_incremental/src/assert_module_sources.rs index 5fb2c1cb9c..8220eb6c65 100644 --- a/compiler/rustc_incremental/src/assert_module_sources.rs +++ b/compiler/rustc_incremental/src/assert_module_sources.rs @@ -36,7 +36,7 @@ pub fn assert_module_sources(tcx: TyCtxt<'_>) { } let available_cgus = tcx - .collect_and_partition_mono_items(LOCAL_CRATE) + .collect_and_partition_mono_items(()) .1 .iter() .map(|cgu| cgu.name().to_string()) diff --git a/compiler/rustc_incremental/src/persist/dirty_clean.rs b/compiler/rustc_incremental/src/persist/dirty_clean.rs index e7bd488af8..9abd4eae91 100644 --- a/compiler/rustc_incremental/src/persist/dirty_clean.rs +++ b/compiler/rustc_incremental/src/persist/dirty_clean.rs @@ -1,6 +1,5 @@ -//! Debugging code to test fingerprints computed for query results. -//! For each node marked with `#[rustc_clean]` or `#[rustc_dirty]`, -//! we will compare the fingerprint from the current and from the previous +//! Debugging code to test fingerprints computed for query results. For each node marked with +//! `#[rustc_clean]` we will compare the fingerprint from the current and from the previous //! compilation session as appropriate: //! //! - `#[rustc_clean(cfg="rev2", except="typeck")]` if we are @@ -30,7 +29,6 @@ use std::iter::FromIterator; use std::vec::Vec; const EXCEPT: Symbol = sym::except; -const LABEL: Symbol = sym::label; const CFG: Symbol = sym::cfg; // Base and Extra labels to build up the labels @@ -101,6 +99,12 @@ const LABELS_FN_IN_TRAIT: &[&[&str]] = /// For generic cases like inline-assembly, modules, etc. const LABELS_HIR_ONLY: &[&[&str]] = &[BASE_HIR]; +/// Impl `DepNode`s. +const LABELS_TRAIT: &[&[&str]] = &[ + BASE_HIR, + &[label_strs::associated_item_def_ids, label_strs::predicates_of, label_strs::generics_of], +]; + /// Impl `DepNode`s. const LABELS_IMPL: &[&[&str]] = &[BASE_HIR, BASE_IMPL]; @@ -122,22 +126,12 @@ struct Assertion { dirty: Labels, } -impl Assertion { - fn from_clean_labels(labels: Labels) -> Assertion { - Assertion { clean: labels, dirty: Labels::default() } - } - - fn from_dirty_labels(labels: Labels) -> Assertion { - Assertion { clean: Labels::default(), dirty: labels } - } -} - pub fn check_dirty_clean_annotations(tcx: TyCtxt<'_>) { if !tcx.sess.opts.debugging_opts.query_dep_graph { return; } - // can't add `#[rustc_dirty]` etc without opting in to this feature + // can't add `#[rustc_clean]` etc without opting in to this feature if !tcx.features().rustc_attrs { return; } @@ -147,11 +141,7 @@ pub fn check_dirty_clean_annotations(tcx: TyCtxt<'_>) { let mut dirty_clean_visitor = DirtyCleanVisitor { tcx, checked_attrs: Default::default() }; krate.visit_all_item_likes(&mut dirty_clean_visitor); - let mut all_attrs = FindAllAttrs { - tcx, - attr_names: &[sym::rustc_dirty, sym::rustc_clean], - found_attrs: vec![], - }; + let mut all_attrs = FindAllAttrs { tcx, found_attrs: vec![] }; intravisit::walk_crate(&mut all_attrs, krate); // Note that we cannot use the existing "unused attribute"-infrastructure @@ -169,37 +159,20 @@ pub struct DirtyCleanVisitor<'tcx> { impl DirtyCleanVisitor<'tcx> { /// Possibly "deserialize" the attribute into a clean/dirty assertion fn assertion_maybe(&mut self, item_id: LocalDefId, attr: &Attribute) -> Option { - let is_clean = if self.tcx.sess.check_name(attr, sym::rustc_dirty) { - false - } else if self.tcx.sess.check_name(attr, sym::rustc_clean) { - true - } else { + if !self.tcx.sess.check_name(attr, sym::rustc_clean) { // skip: not rustc_clean/dirty return None; - }; + } if !check_config(self.tcx, attr) { // skip: not the correct `cfg=` return None; } - let assertion = if let Some(labels) = self.labels(attr) { - if is_clean { - Assertion::from_clean_labels(labels) - } else { - Assertion::from_dirty_labels(labels) - } - } else { - self.assertion_auto(item_id, attr, is_clean) - }; + let assertion = self.assertion_auto(item_id, attr); Some(assertion) } /// Gets the "auto" assertion on pre-validated attr, along with the `except` labels. - fn assertion_auto( - &mut self, - item_id: LocalDefId, - attr: &Attribute, - is_clean: bool, - ) -> Assertion { + fn assertion_auto(&mut self, item_id: LocalDefId, attr: &Attribute) -> Assertion { let (name, mut auto) = self.auto_labels(item_id, attr); let except = self.except(attr); for e in except.iter() { @@ -211,21 +184,7 @@ impl DirtyCleanVisitor<'tcx> { self.tcx.sess.span_fatal(attr.span, &msg); } } - if is_clean { - Assertion { clean: auto, dirty: except } - } else { - Assertion { clean: except, dirty: auto } - } - } - - fn labels(&self, attr: &Attribute) -> Option { - for item in attr.meta_item_list().unwrap_or_else(Vec::new) { - if item.has_name(LABEL) { - let value = expect_associated_value(self.tcx, &item); - return Some(self.resolve_labels(&item, value)); - } - } - None + Assertion { clean: auto, dirty: except } } /// `except=` attribute value @@ -288,20 +247,7 @@ impl DirtyCleanVisitor<'tcx> { HirItem::Union(..) => ("ItemUnion", LABELS_ADT), // Represents a Trait Declaration - // FIXME(michaelwoerister): trait declaration is buggy because sometimes some of - // the depnodes don't exist (because they legitimately didn't need to be - // calculated) - // - // michaelwoerister and vitiral came up with a possible solution, - // to just do this before every query - // ``` - // ::rustc_middle::ty::query::plumbing::force_from_dep_node(tcx, dep_node) - // ``` - // - // However, this did not seem to work effectively and more bugs were hit. - // Nebie @vitiral gave up :) - // - //HirItem::Trait(..) => ("ItemTrait", LABELS_TRAIT), + HirItem::Trait(..) => ("ItemTrait", LABELS_TRAIT), // An implementation, eg `impl Trait for Foo { .. }` HirItem::Impl { .. } => ("ItemKind::Impl", LABELS_IMPL), @@ -434,35 +380,23 @@ impl ItemLikeVisitor<'tcx> for DirtyCleanVisitor<'tcx> { } } -/// Given a `#[rustc_dirty]` or `#[rustc_clean]` attribute, scan -/// for a `cfg="foo"` attribute and check whether we have a cfg -/// flag called `foo`. -/// -/// Also make sure that the `label` and `except` fields do not -/// both exist. +/// Given a `#[rustc_clean]` attribute, scan for a `cfg="foo"` attribute and check whether we have +/// a cfg flag called `foo`. fn check_config(tcx: TyCtxt<'_>, attr: &Attribute) -> bool { debug!("check_config(attr={:?})", attr); let config = &tcx.sess.parse_sess.config; debug!("check_config: config={:?}", config); - let (mut cfg, mut except, mut label) = (None, false, false); + let mut cfg = None; for item in attr.meta_item_list().unwrap_or_else(Vec::new) { if item.has_name(CFG) { let value = expect_associated_value(tcx, &item); debug!("check_config: searching for cfg {:?}", value); cfg = Some(config.contains(&(value, None))); - } - if item.has_name(LABEL) { - label = true; - } - if item.has_name(EXCEPT) { - except = true; + } else if !item.has_name(EXCEPT) { + tcx.sess.span_err(attr.span, &format!("unknown item `{}`", item.name_or_empty())); } } - if label && except { - tcx.sess.span_fatal(attr.span, "must specify only one of: `label`, `except`"); - } - match cfg { None => tcx.sess.span_fatal(attr.span, "no cfg attribute"), Some(c) => c, @@ -483,21 +417,18 @@ fn expect_associated_value(tcx: TyCtxt<'_>, item: &NestedMetaItem) -> Symbol { } } -// A visitor that collects all #[rustc_dirty]/#[rustc_clean] attributes from +// A visitor that collects all #[rustc_clean] attributes from // the HIR. It is used to verify that we really ran checks for all annotated // nodes. -pub struct FindAllAttrs<'a, 'tcx> { +pub struct FindAllAttrs<'tcx> { tcx: TyCtxt<'tcx>, - attr_names: &'a [Symbol], found_attrs: Vec<&'tcx Attribute>, } -impl FindAllAttrs<'_, 'tcx> { +impl FindAllAttrs<'tcx> { fn is_active_attr(&mut self, attr: &Attribute) -> bool { - for attr_name in self.attr_names { - if self.tcx.sess.check_name(attr, *attr_name) && check_config(self.tcx, attr) { - return true; - } + if self.tcx.sess.check_name(attr, sym::rustc_clean) && check_config(self.tcx, attr) { + return true; } false @@ -506,17 +437,14 @@ impl FindAllAttrs<'_, 'tcx> { fn report_unchecked_attrs(&self, mut checked_attrs: FxHashSet) { for attr in &self.found_attrs { if !checked_attrs.contains(&attr.id) { - self.tcx.sess.span_err( - attr.span, - "found unchecked `#[rustc_dirty]` / `#[rustc_clean]` attribute", - ); + self.tcx.sess.span_err(attr.span, "found unchecked `#[rustc_clean]` attribute"); checked_attrs.insert(attr.id); } } } } -impl intravisit::Visitor<'tcx> for FindAllAttrs<'_, 'tcx> { +impl intravisit::Visitor<'tcx> for FindAllAttrs<'tcx> { type Map = Map<'tcx>; fn nested_visit_map(&mut self) -> intravisit::NestedVisitorMap { diff --git a/compiler/rustc_incremental/src/persist/fs.rs b/compiler/rustc_incremental/src/persist/fs.rs index 30c6c408bc..83e80b55da 100644 --- a/compiler/rustc_incremental/src/persist/fs.rs +++ b/compiler/rustc_incremental/src/persist/fs.rs @@ -106,6 +106,7 @@ use rustc_data_structures::fx::{FxHashMap, FxHashSet}; use rustc_data_structures::svh::Svh; use rustc_data_structures::{base_n, flock}; +use rustc_errors::ErrorReported; use rustc_fs_util::{link_or_copy, LinkOrCopy}; use rustc_session::{CrateDisambiguator, Session}; @@ -189,9 +190,9 @@ pub fn prepare_session_directory( sess: &Session, crate_name: &str, crate_disambiguator: CrateDisambiguator, -) { +) -> Result<(), ErrorReported> { if sess.opts.incremental.is_none() { - return; + return Ok(()); } let _timer = sess.timer("incr_comp_prepare_session_directory"); @@ -201,9 +202,7 @@ pub fn prepare_session_directory( // {incr-comp-dir}/{crate-name-and-disambiguator} let crate_dir = crate_path(sess, crate_name, crate_disambiguator); debug!("crate-dir: {}", crate_dir.display()); - if create_dir(sess, &crate_dir, "crate").is_err() { - return; - } + create_dir(sess, &crate_dir, "crate")?; // Hack: canonicalize the path *after creating the directory* // because, on windows, long paths can cause problems; @@ -217,7 +216,7 @@ pub fn prepare_session_directory( crate_dir.display(), err )); - return; + return Err(ErrorReported); } }; @@ -232,16 +231,11 @@ pub fn prepare_session_directory( // Lock the new session directory. If this fails, return an // error without retrying - let (directory_lock, lock_file_path) = match lock_directory(sess, &session_dir) { - Ok(e) => e, - Err(_) => return, - }; + let (directory_lock, lock_file_path) = lock_directory(sess, &session_dir)?; // Now that we have the lock, we can actually create the session // directory - if create_dir(sess, &session_dir, "session").is_err() { - return; - } + create_dir(sess, &session_dir, "session")?; // Find a suitable source directory to copy from. Ignore those that we // have already tried before. @@ -257,7 +251,7 @@ pub fn prepare_session_directory( ); sess.init_incr_comp_session(session_dir, directory_lock, false); - return; + return Ok(()); }; debug!("attempting to copy data from source: {}", source_directory.display()); @@ -278,7 +272,7 @@ pub fn prepare_session_directory( } sess.init_incr_comp_session(session_dir, directory_lock, true); - return; + return Ok(()); } else { debug!("copying failed - trying next directory"); @@ -478,7 +472,7 @@ fn generate_session_dir_path(crate_dir: &Path) -> PathBuf { directory_path } -fn create_dir(sess: &Session, path: &Path, dir_tag: &str) -> Result<(), ()> { +fn create_dir(sess: &Session, path: &Path, dir_tag: &str) -> Result<(), ErrorReported> { match std_fs::create_dir_all(path) { Ok(()) => { debug!("{} directory created successfully", dir_tag); @@ -492,13 +486,16 @@ fn create_dir(sess: &Session, path: &Path, dir_tag: &str) -> Result<(), ()> { path.display(), err )); - Err(()) + Err(ErrorReported) } } } /// Allocate the lock-file and lock it. -fn lock_directory(sess: &Session, session_dir: &Path) -> Result<(flock::Lock, PathBuf), ()> { +fn lock_directory( + sess: &Session, + session_dir: &Path, +) -> Result<(flock::Lock, PathBuf), ErrorReported> { let lock_file_path = lock_file_path(session_dir); debug!("lock_directory() - lock_file: {}", lock_file_path.display()); @@ -510,13 +507,36 @@ fn lock_directory(sess: &Session, session_dir: &Path) -> Result<(flock::Lock, Pa ) { // the lock should be exclusive Ok(lock) => Ok((lock, lock_file_path)), - Err(err) => { - sess.err(&format!( + Err(lock_err) => { + let mut err = sess.struct_err(&format!( "incremental compilation: could not create \ - session directory lock file: {}", - err + session directory lock file: {}", + lock_err )); - Err(()) + if flock::Lock::error_unsupported(&lock_err) { + err.note(&format!( + "the filesystem for the incremental path at {} \ + does not appear to support locking, consider changing the \ + incremental path to a filesystem that supports locking \ + or disable incremental compilation", + session_dir.display() + )); + if std::env::var_os("CARGO").is_some() { + err.help( + "incremental compilation can be disabled by setting the \ + environment variable CARGO_INCREMENTAL=0 (see \ + https://doc.rust-lang.org/cargo/reference/profiles.html#incremental)", + ); + err.help( + "the entire build directory can be changed to a different \ + filesystem by setting the environment variable CARGO_TARGET_DIR \ + to a different path (see \ + https://doc.rust-lang.org/cargo/reference/config.html#buildtarget-dir)", + ); + } + } + err.emit(); + Err(ErrorReported) } } } diff --git a/compiler/rustc_incremental/src/persist/load.rs b/compiler/rustc_incremental/src/persist/load.rs index 2661afd7ff..8539cc6937 100644 --- a/compiler/rustc_incremental/src/persist/load.rs +++ b/compiler/rustc_incremental/src/persist/load.rs @@ -1,8 +1,7 @@ //! Code to save/load the dep-graph from files. use rustc_data_structures::fx::FxHashMap; -use rustc_hir::definitions::Definitions; -use rustc_middle::dep_graph::{PreviousDepGraph, SerializedDepGraph, WorkProduct, WorkProductId}; +use rustc_middle::dep_graph::{SerializedDepGraph, WorkProduct, WorkProductId}; use rustc_middle::ty::query::OnDiskCache; use rustc_serialize::opaque::Decoder; use rustc_serialize::Decodable; @@ -22,8 +21,8 @@ pub enum LoadResult { Error { message: String }, } -impl LoadResult<(PreviousDepGraph, WorkProductMap)> { - pub fn open(self, sess: &Session) -> (PreviousDepGraph, WorkProductMap) { +impl LoadResult<(SerializedDepGraph, WorkProductMap)> { + pub fn open(self, sess: &Session) -> (SerializedDepGraph, WorkProductMap) { match self { LoadResult::Error { message } => { sess.warn(&message); @@ -84,7 +83,7 @@ impl MaybeAsync { } } -pub type DepGraphFuture = MaybeAsync>; +pub type DepGraphFuture = MaybeAsync>; /// Launch a thread and load the dependency graph in the background. pub fn load_dep_graph(sess: &Session) -> DepGraphFuture { @@ -185,7 +184,7 @@ pub fn load_dep_graph(sess: &Session) -> DepGraphFuture { let dep_graph = SerializedDepGraph::decode(&mut decoder) .expect("Error reading cached dep-graph"); - LoadResult::Ok { data: (PreviousDepGraph::new(dep_graph), prev_work_products) } + LoadResult::Ok { data: (dep_graph, prev_work_products) } } } })) @@ -196,10 +195,7 @@ pub fn load_dep_graph(sess: &Session) -> DepGraphFuture { /// If we are not in incremental compilation mode, returns `None`. /// Otherwise, tries to load the query result cache from disk, /// creating an empty cache if it could not be loaded. -pub fn load_query_result_cache<'a>( - sess: &'a Session, - definitions: &Definitions, -) -> Option> { +pub fn load_query_result_cache<'a>(sess: &'a Session) -> Option> { if sess.opts.incremental.is_none() { return None; } @@ -212,7 +208,7 @@ pub fn load_query_result_cache<'a>( sess.is_nightly_build(), ) { LoadResult::Ok { data: (bytes, start_pos) } => { - Some(OnDiskCache::new(sess, bytes, start_pos, definitions)) + Some(OnDiskCache::new(sess, bytes, start_pos)) } _ => Some(OnDiskCache::new_empty(sess.source_map())), } diff --git a/compiler/rustc_incremental/src/persist/save.rs b/compiler/rustc_incremental/src/persist/save.rs index 1484088837..a8455854eb 100644 --- a/compiler/rustc_incremental/src/persist/save.rs +++ b/compiler/rustc_incremental/src/persist/save.rs @@ -1,6 +1,6 @@ use rustc_data_structures::fx::FxHashMap; use rustc_data_structures::sync::join; -use rustc_middle::dep_graph::{DepGraph, PreviousDepGraph, WorkProduct, WorkProductId}; +use rustc_middle::dep_graph::{DepGraph, SerializedDepGraph, WorkProduct, WorkProductId}; use rustc_middle::ty::TyCtxt; use rustc_serialize::opaque::{FileEncodeResult, FileEncoder}; use rustc_serialize::Encodable as RustcEncodable; @@ -186,7 +186,7 @@ fn encode_query_cache(tcx: TyCtxt<'_>, encoder: &mut FileEncoder) -> FileEncodeR pub fn build_dep_graph( sess: &Session, - prev_graph: PreviousDepGraph, + prev_graph: SerializedDepGraph, prev_work_products: FxHashMap, ) -> Option { if sess.opts.incremental.is_none() { @@ -229,6 +229,7 @@ pub fn build_dep_graph( } Some(DepGraph::new( + &sess.prof, prev_graph, prev_work_products, encoder, diff --git a/compiler/rustc_index/src/bit_set.rs b/compiler/rustc_index/src/bit_set.rs index d26ab1939e..df777502c4 100644 --- a/compiler/rustc_index/src/bit_set.rs +++ b/compiler/rustc_index/src/bit_set.rs @@ -355,14 +355,18 @@ where Op: Fn(Word, Word) -> Word, { assert_eq!(out_vec.len(), in_vec.len()); - let mut changed = false; + let mut changed = 0; for (out_elem, in_elem) in iter::zip(out_vec, in_vec) { let old_val = *out_elem; let new_val = op(old_val, *in_elem); *out_elem = new_val; - changed |= old_val != new_val; + // This is essentially equivalent to a != with changed being a bool, but + // in practice this code gets auto-vectorized by the compiler for most + // operators. Using != here causes us to generate quite poor code as the + // compiler tries to go back to a boolean on each loop iteration. + changed |= old_val ^ new_val; } - changed + changed != 0 } const SPARSE_MAX: usize = 8; diff --git a/compiler/rustc_index/src/lib.rs b/compiler/rustc_index/src/lib.rs index 4c73b7bf61..0093fa5e56 100644 --- a/compiler/rustc_index/src/lib.rs +++ b/compiler/rustc_index/src/lib.rs @@ -1,6 +1,5 @@ #![feature(allow_internal_unstable)] #![feature(bench_black_box)] -#![feature(const_panic)] #![feature(extend_one)] #![feature(iter_zip)] #![feature(unboxed_closures)] diff --git a/compiler/rustc_index/src/vec.rs b/compiler/rustc_index/src/vec.rs index 1b1a59a254..246fa28d98 100644 --- a/compiler/rustc_index/src/vec.rs +++ b/compiler/rustc_index/src/vec.rs @@ -65,7 +65,7 @@ impl Idx for u32 { /// `u32::MAX`. You can also customize things like the `Debug` impl, /// what traits are derived, and so forth via the macro. #[macro_export] -#[allow_internal_unstable(step_trait, step_trait_ext, rustc_attrs)] +#[allow_internal_unstable(step_trait, rustc_attrs, trusted_step)] macro_rules! newtype_index { // ---- public rules ---- @@ -184,7 +184,7 @@ macro_rules! newtype_index { } } - unsafe impl ::std::iter::Step for $type { + impl ::std::iter::Step for $type { #[inline] fn steps_between(start: &Self, end: &Self) -> Option { ::steps_between( @@ -204,6 +204,9 @@ macro_rules! newtype_index { } } + // Safety: The implementation of `Step` upholds all invariants. + unsafe impl ::std::iter::TrustedStep for $type {} + impl From<$type> for u32 { #[inline] fn from(v: $type) -> u32 { diff --git a/compiler/rustc_infer/src/infer/canonical/query_response.rs b/compiler/rustc_infer/src/infer/canonical/query_response.rs index b8ecc94958..c3c28d7008 100644 --- a/compiler/rustc_infer/src/infer/canonical/query_response.rs +++ b/compiler/rustc_infer/src/infer/canonical/query_response.rs @@ -660,7 +660,12 @@ impl<'tcx> TypeRelatingDelegate<'tcx> for QueryTypeRelatingDelegate<'_, 'tcx> { ) } - fn push_outlives(&mut self, sup: ty::Region<'tcx>, sub: ty::Region<'tcx>) { + fn push_outlives( + &mut self, + sup: ty::Region<'tcx>, + sub: ty::Region<'tcx>, + _info: ty::VarianceDiagInfo<'tcx>, + ) { self.obligations.push(Obligation { cause: self.cause.clone(), param_env: self.param_env, diff --git a/compiler/rustc_infer/src/infer/combine.rs b/compiler/rustc_infer/src/infer/combine.rs index 30214e9420..3a11b5a214 100644 --- a/compiler/rustc_infer/src/infer/combine.rs +++ b/compiler/rustc_infer/src/infer/combine.rs @@ -371,9 +371,12 @@ impl<'infcx, 'tcx> CombineFields<'infcx, 'tcx> { match dir { EqTo => self.equate(a_is_expected).relate(a_ty, b_ty), SubtypeOf => self.sub(a_is_expected).relate(a_ty, b_ty), - SupertypeOf => { - self.sub(a_is_expected).relate_with_variance(ty::Contravariant, a_ty, b_ty) - } + SupertypeOf => self.sub(a_is_expected).relate_with_variance( + ty::Contravariant, + ty::VarianceDiagInfo::default(), + a_ty, + b_ty, + ), }?; Ok(()) @@ -574,6 +577,7 @@ impl TypeRelation<'tcx> for Generalizer<'_, 'tcx> { fn relate_with_variance>( &mut self, variance: ty::Variance, + _info: ty::VarianceDiagInfo<'tcx>, a: T, b: T, ) -> RelateResult<'tcx, T> { @@ -737,7 +741,12 @@ impl TypeRelation<'tcx> for Generalizer<'_, 'tcx> { if self.tcx().lazy_normalization() => { assert_eq!(promoted, None); - let substs = self.relate_with_variance(ty::Variance::Invariant, substs, substs)?; + let substs = self.relate_with_variance( + ty::Variance::Invariant, + ty::VarianceDiagInfo::default(), + substs, + substs, + )?; Ok(self.tcx().mk_const(ty::Const { ty: c.ty, val: ty::ConstKind::Unevaluated(ty::Unevaluated { def, substs, promoted }), @@ -831,6 +840,7 @@ impl TypeRelation<'tcx> for ConstInferUnifier<'_, 'tcx> { fn relate_with_variance>( &mut self, _variance: ty::Variance, + _info: ty::VarianceDiagInfo<'tcx>, a: T, b: T, ) -> RelateResult<'tcx, T> { @@ -965,7 +975,12 @@ impl TypeRelation<'tcx> for ConstInferUnifier<'_, 'tcx> { if self.tcx().lazy_normalization() => { assert_eq!(promoted, None); - let substs = self.relate_with_variance(ty::Variance::Invariant, substs, substs)?; + let substs = self.relate_with_variance( + ty::Variance::Invariant, + ty::VarianceDiagInfo::default(), + substs, + substs, + )?; Ok(self.tcx().mk_const(ty::Const { ty: c.ty, val: ty::ConstKind::Unevaluated(ty::Unevaluated { def, substs, promoted }), diff --git a/compiler/rustc_infer/src/infer/equate.rs b/compiler/rustc_infer/src/infer/equate.rs index 45ba50bb63..0c93271a1a 100644 --- a/compiler/rustc_infer/src/infer/equate.rs +++ b/compiler/rustc_infer/src/infer/equate.rs @@ -59,6 +59,7 @@ impl TypeRelation<'tcx> for Equate<'combine, 'infcx, 'tcx> { fn relate_with_variance>( &mut self, _: ty::Variance, + _info: ty::VarianceDiagInfo<'tcx>, a: T, b: T, ) -> RelateResult<'tcx, T> { diff --git a/compiler/rustc_infer/src/infer/error_reporting/mod.rs b/compiler/rustc_infer/src/infer/error_reporting/mod.rs index a91bd9ce2f..e3a79fe265 100644 --- a/compiler/rustc_infer/src/infer/error_reporting/mod.rs +++ b/compiler/rustc_infer/src/infer/error_reporting/mod.rs @@ -64,6 +64,7 @@ use rustc_hir as hir; use rustc_hir::def_id::DefId; use rustc_hir::lang_items::LangItem; use rustc_hir::{Item, ItemKind, Node}; +use rustc_middle::dep_graph::DepContext; use rustc_middle::ty::error::TypeError; use rustc_middle::ty::{ self, @@ -524,7 +525,7 @@ impl<'a, 'tcx> InferCtxt<'a, 'tcx> { } fn path_crate(self, cnum: CrateNum) -> Result { - Ok(vec![self.tcx.original_crate_name(cnum).to_string()]) + Ok(vec![self.tcx.crate_name(cnum).to_string()]) } fn path_qualified( self, @@ -1604,13 +1605,19 @@ impl<'a, 'tcx> InferCtxt<'a, 'tcx> { match (&terr, expected == found) { (TypeError::Sorts(values), extra) => { let sort_string = |ty: Ty<'tcx>| match (extra, ty.kind()) { - (true, ty::Opaque(def_id, _)) => format!( - " (opaque type at {})", - self.tcx + (true, ty::Opaque(def_id, _)) => { + let pos = self + .tcx .sess .source_map() - .mk_substr_filename(self.tcx.def_span(*def_id)), - ), + .lookup_char_pos(self.tcx.def_span(*def_id).lo()); + format!( + " (opaque type at <{}:{}:{}>)", + pos.file.name.prefer_local(), + pos.line, + pos.col.to_usize() + 1, + ) + } (true, _) => format!(" ({})", ty.sort_string(self.tcx)), (false, _) => "".to_string(), }; @@ -1959,7 +1966,33 @@ impl<'a, 'tcx> InferCtxt<'a, 'tcx> { struct_span_err!(self.tcx.sess, span, E0580, "{}", failure_str) } FailureCode::Error0308(failure_str) => { - struct_span_err!(self.tcx.sess, span, E0308, "{}", failure_str) + let mut err = struct_span_err!(self.tcx.sess, span, E0308, "{}", failure_str); + if let ValuePairs::Types(ty::error::ExpectedFound { expected, found }) = + trace.values + { + // If a tuple of length one was expected and the found expression has + // parentheses around it, perhaps the user meant to write `(expr,)` to + // build a tuple (issue #86100) + match (expected.kind(), found.kind()) { + (ty::Tuple(_), ty::Tuple(_)) => {} + (ty::Tuple(_), _) if expected.tuple_fields().count() == 1 => { + if let Ok(code) = self.tcx.sess().source_map().span_to_snippet(span) { + if let Some(code) = + code.strip_prefix('(').and_then(|s| s.strip_suffix(')')) + { + err.span_suggestion( + span, + "use a trailing comma to create a tuple with one element", + format!("({},)", code), + Applicability::MaybeIncorrect, + ); + } + } + } + _ => {} + } + } + err } FailureCode::Error0644(failure_str) => { struct_span_err!(self.tcx.sess, span, E0644, "{}", failure_str) @@ -2398,9 +2431,6 @@ impl<'a, 'tcx> InferCtxt<'a, 'tcx> { self.tcx.associated_item(def_id).ident ), infer::EarlyBoundRegion(_, name) => format!(" for lifetime parameter `{}`", name), - infer::BoundRegionInCoherence(name) => { - format!(" for lifetime parameter `{}` in coherence check", name) - } infer::UpvarRegion(ref upvar_id, _) => { let var_name = self.tcx.hir().name(upvar_id.var_path.hir_id); format!(" for capture of `{}` by closure", var_name) diff --git a/compiler/rustc_infer/src/infer/error_reporting/nice_region_error/named_anon_conflict.rs b/compiler/rustc_infer/src/infer/error_reporting/nice_region_error/named_anon_conflict.rs index 2f3c0d6957..0878f8550d 100644 --- a/compiler/rustc_infer/src/infer/error_reporting/nice_region_error/named_anon_conflict.rs +++ b/compiler/rustc_infer/src/infer/error_reporting/nice_region_error/named_anon_conflict.rs @@ -114,12 +114,16 @@ impl<'a, 'tcx> NiceRegionError<'a, 'tcx> { ); diag.span_label(span, format!("lifetime `{}` required", named)); - diag.span_suggestion( - new_ty_span, - &format!("add explicit lifetime `{}` to {}", named, span_label_var), - new_ty.to_string(), - Applicability::Unspecified, - ); + // Suggesting `'static` is nearly always incorrect, and can steer users + // down the wrong path. + if *named != ty::ReStatic { + diag.span_suggestion( + new_ty_span, + &format!("add explicit lifetime `{}` to {}", named, span_label_var), + new_ty.to_string(), + Applicability::Unspecified, + ); + } Some(diag) } diff --git a/compiler/rustc_infer/src/infer/glb.rs b/compiler/rustc_infer/src/infer/glb.rs index 02662043db..60f02b84aa 100644 --- a/compiler/rustc_infer/src/infer/glb.rs +++ b/compiler/rustc_infer/src/infer/glb.rs @@ -43,6 +43,7 @@ impl TypeRelation<'tcx> for Glb<'combine, 'infcx, 'tcx> { fn relate_with_variance>( &mut self, variance: ty::Variance, + _info: ty::VarianceDiagInfo<'tcx>, a: T, b: T, ) -> RelateResult<'tcx, T> { @@ -96,7 +97,7 @@ impl TypeRelation<'tcx> for Glb<'combine, 'infcx, 'tcx> { // When higher-ranked types are involved, computing the LUB is // very challenging, switch to invariance. This is obviously // overly conservative but works ok in practice. - self.relate_with_variance(ty::Variance::Invariant, a, b)?; + self.relate_with_variance(ty::Variance::Invariant, ty::VarianceDiagInfo::default(), a, b)?; Ok(a) } } diff --git a/compiler/rustc_infer/src/infer/lub.rs b/compiler/rustc_infer/src/infer/lub.rs index 4fa8f2f1a6..a08323535c 100644 --- a/compiler/rustc_infer/src/infer/lub.rs +++ b/compiler/rustc_infer/src/infer/lub.rs @@ -43,6 +43,7 @@ impl TypeRelation<'tcx> for Lub<'combine, 'infcx, 'tcx> { fn relate_with_variance>( &mut self, variance: ty::Variance, + _info: ty::VarianceDiagInfo<'tcx>, a: T, b: T, ) -> RelateResult<'tcx, T> { @@ -96,7 +97,7 @@ impl TypeRelation<'tcx> for Lub<'combine, 'infcx, 'tcx> { // When higher-ranked types are involved, computing the LUB is // very challenging, switch to invariance. This is obviously // overly conservative but works ok in practice. - self.relate_with_variance(ty::Variance::Invariant, a, b)?; + self.relate_with_variance(ty::Variance::Invariant, ty::VarianceDiagInfo::default(), a, b)?; Ok(a) } } diff --git a/compiler/rustc_infer/src/infer/mod.rs b/compiler/rustc_infer/src/infer/mod.rs index eaec6b46bc..f39431f249 100644 --- a/compiler/rustc_infer/src/infer/mod.rs +++ b/compiler/rustc_infer/src/infer/mod.rs @@ -453,8 +453,6 @@ pub enum RegionVariableOrigin { UpvarRegion(ty::UpvarId, Span), - BoundRegionInCoherence(Symbol), - /// This origin is used for the inference variables that we create /// during NLL region processing. Nll(NllRegionVariableOrigin), @@ -1749,7 +1747,6 @@ impl RegionVariableOrigin { | EarlyBoundRegion(a, ..) | LateBoundRegion(a, ..) | UpvarRegion(_, a) => a, - BoundRegionInCoherence(_) => rustc_span::DUMMY_SP, Nll(..) => bug!("NLL variable used with `span`"), } } diff --git a/compiler/rustc_infer/src/infer/nll_relate/mod.rs b/compiler/rustc_infer/src/infer/nll_relate/mod.rs index 077d2cc20a..20be06adfd 100644 --- a/compiler/rustc_infer/src/infer/nll_relate/mod.rs +++ b/compiler/rustc_infer/src/infer/nll_relate/mod.rs @@ -55,6 +55,8 @@ where /// - Bivariant means that it doesn't matter. ambient_variance: ty::Variance, + ambient_variance_info: ty::VarianceDiagInfo<'tcx>, + /// When we pass through a set of binders (e.g., when looking into /// a `fn` type), we push a new bound region scope onto here. This /// will contain the instantiated region for each region in those @@ -78,7 +80,12 @@ pub trait TypeRelatingDelegate<'tcx> { /// satisfied for the two types to be related. `sub` and `sup` may /// be regions from the type or new variables created through the /// delegate. - fn push_outlives(&mut self, sup: ty::Region<'tcx>, sub: ty::Region<'tcx>); + fn push_outlives( + &mut self, + sup: ty::Region<'tcx>, + sub: ty::Region<'tcx>, + info: ty::VarianceDiagInfo<'tcx>, + ); fn const_equate(&mut self, a: &'tcx ty::Const<'tcx>, b: &'tcx ty::Const<'tcx>); @@ -138,7 +145,14 @@ where delegate: D, ambient_variance: ty::Variance, ) -> Self { - Self { infcx, delegate, ambient_variance, a_scopes: vec![], b_scopes: vec![] } + Self { + infcx, + delegate, + ambient_variance, + ambient_variance_info: ty::VarianceDiagInfo::default(), + a_scopes: vec![], + b_scopes: vec![], + } } fn ambient_covariance(&self) -> bool { @@ -239,10 +253,15 @@ where /// Push a new outlives requirement into our output set of /// constraints. - fn push_outlives(&mut self, sup: ty::Region<'tcx>, sub: ty::Region<'tcx>) { + fn push_outlives( + &mut self, + sup: ty::Region<'tcx>, + sub: ty::Region<'tcx>, + info: ty::VarianceDiagInfo<'tcx>, + ) { debug!("push_outlives({:?}: {:?})", sup, sub); - self.delegate.push_outlives(sup, sub); + self.delegate.push_outlives(sup, sub, info); } /// Relate a projection type and some value type lazily. This will always @@ -490,6 +509,7 @@ where fn relate_with_variance>( &mut self, variance: ty::Variance, + info: ty::VarianceDiagInfo<'tcx>, a: T, b: T, ) -> RelateResult<'tcx, T> { @@ -497,6 +517,7 @@ where let old_ambient_variance = self.ambient_variance; self.ambient_variance = self.ambient_variance.xform(variance); + self.ambient_variance_info = self.ambient_variance_info.clone().xform(info); debug!("relate_with_variance: ambient_variance = {:?}", self.ambient_variance); @@ -574,12 +595,12 @@ where if self.ambient_covariance() { // Covariance: a <= b. Hence, `b: a`. - self.push_outlives(v_b, v_a); + self.push_outlives(v_b, v_a, self.ambient_variance_info.clone()); } if self.ambient_contravariance() { // Contravariant: b <= a. Hence, `a: b`. - self.push_outlives(v_a, v_b); + self.push_outlives(v_a, v_b, self.ambient_variance_info.clone()); } Ok(a) @@ -835,6 +856,7 @@ where fn relate_with_variance>( &mut self, variance: ty::Variance, + _info: ty::VarianceDiagInfo<'tcx>, a: T, b: T, ) -> RelateResult<'tcx, T> { diff --git a/compiler/rustc_infer/src/infer/region_constraints/mod.rs b/compiler/rustc_infer/src/infer/region_constraints/mod.rs index 9ffcddfae9..4935913016 100644 --- a/compiler/rustc_infer/src/infer/region_constraints/mod.rs +++ b/compiler/rustc_infer/src/infer/region_constraints/mod.rs @@ -3,7 +3,6 @@ use self::CombineMapType::*; use self::UndoLog::*; -use super::unify_key; use super::{ InferCtxtUndoLogs, MiscVariable, RegionVariableOrigin, Rollback, Snapshot, SubregionOrigin, }; @@ -12,9 +11,9 @@ use rustc_data_structures::fx::{FxHashMap, FxHashSet}; use rustc_data_structures::sync::Lrc; use rustc_data_structures::undo_log::UndoLogs; use rustc_data_structures::unify as ut; -use rustc_data_structures::unify::UnifyKey; use rustc_hir::def_id::DefId; use rustc_index::vec::IndexVec; +use rustc_middle::infer::unify_key::{RegionVidKey, UnifiedRegion}; use rustc_middle::ty::ReStatic; use rustc_middle::ty::{self, Ty, TyCtxt}; use rustc_middle::ty::{ReLateBound, ReVar}; @@ -54,7 +53,7 @@ pub struct RegionConstraintStorage<'tcx> { /// code is iterating to a fixed point, because otherwise we sometimes /// would wind up with a fresh stream of region variables that have been /// equated but appear distinct. - pub(super) unification_table: ut::UnificationTableStorage, + pub(super) unification_table: ut::UnificationTableStorage>, /// a flag set to true when we perform any unifications; this is used /// to micro-optimize `take_and_reset_data` @@ -407,8 +406,7 @@ impl<'tcx> RegionConstraintCollector<'_, 'tcx> { // `RegionConstraintData` contains the relationship here. if *any_unifications { *any_unifications = false; - self.unification_table() - .reset_unifications(|vid| unify_key::RegionVidKey { min_vid: vid }); + self.unification_table().reset_unifications(|_| UnifiedRegion(None)); } data @@ -435,8 +433,8 @@ impl<'tcx> RegionConstraintCollector<'_, 'tcx> { ) -> RegionVid { let vid = self.var_infos.push(RegionVariableInfo { origin, universe }); - let u_vid = self.unification_table().new_key(unify_key::RegionVidKey { min_vid: vid }); - assert_eq!(vid, u_vid); + let u_vid = self.unification_table().new_key(UnifiedRegion(None)); + assert_eq!(vid, u_vid.vid); self.undo_log.push(AddVar(vid)); debug!("created new region variable {:?} in {:?} with origin {:?}", vid, universe, origin); vid @@ -498,10 +496,18 @@ impl<'tcx> RegionConstraintCollector<'_, 'tcx> { self.make_subregion(origin.clone(), sub, sup); self.make_subregion(origin, sup, sub); - if let (ty::ReVar(sub), ty::ReVar(sup)) = (*sub, *sup) { - debug!("make_eqregion: uniying {:?} with {:?}", sub, sup); - self.unification_table().union(sub, sup); - self.any_unifications = true; + match (sub, sup) { + (&ty::ReVar(sub), &ty::ReVar(sup)) => { + debug!("make_eqregion: unifying {:?} with {:?}", sub, sup); + self.unification_table().union(sub, sup); + self.any_unifications = true; + } + (&ty::ReVar(vid), value) | (value, &ty::ReVar(vid)) => { + debug!("make_eqregion: unifying {:?} with {:?}", vid, value); + self.unification_table().union_value(vid, UnifiedRegion(Some(value))); + self.any_unifications = true; + } + (_, _) => {} } } } @@ -617,8 +623,29 @@ impl<'tcx> RegionConstraintCollector<'_, 'tcx> { } } - pub fn opportunistic_resolve_var(&mut self, rid: RegionVid) -> ty::RegionVid { - self.unification_table().probe_value(rid).min_vid + /// Resolves the passed RegionVid to the root RegionVid in the unification table + pub fn opportunistic_resolve_var(&mut self, rid: ty::RegionVid) -> ty::RegionVid { + self.unification_table().find(rid).vid + } + + /// If the Region is a `ReVar`, then resolves it either to the root value in + /// the unification table, if it exists, or to the root `ReVar` in the table. + /// If the Region is not a `ReVar`, just returns the Region itself. + pub fn opportunistic_resolve_region( + &mut self, + tcx: TyCtxt<'tcx>, + region: ty::Region<'tcx>, + ) -> ty::Region<'tcx> { + match region { + ty::ReVar(rid) => { + let unified_region = self.unification_table().probe_value(*rid); + unified_region.0.unwrap_or_else(|| { + let root = self.unification_table().find(*rid).vid; + tcx.reuse_or_mk_region(region, ty::ReVar(root)) + }) + } + _ => region, + } } fn combine_map(&mut self, t: CombineMapType) -> &mut CombineMap<'tcx> { @@ -673,8 +700,7 @@ impl<'tcx> RegionConstraintCollector<'_, 'tcx> { &self, value_count: usize, ) -> (Range, Vec) { - let range = RegionVid::from_index(value_count as u32) - ..RegionVid::from_index(self.unification_table.len() as u32); + let range = RegionVid::from(value_count)..RegionVid::from(self.unification_table.len()); ( range.clone(), (range.start.index()..range.end.index()) @@ -696,7 +722,7 @@ impl<'tcx> RegionConstraintCollector<'_, 'tcx> { } #[inline] - fn unification_table(&mut self) -> super::UnificationTable<'_, 'tcx, ty::RegionVid> { + fn unification_table(&mut self) -> super::UnificationTable<'_, 'tcx, RegionVidKey<'tcx>> { ut::UnificationTable::with_log(&mut self.storage.unification_table, self.undo_log) } } diff --git a/compiler/rustc_infer/src/infer/sub.rs b/compiler/rustc_infer/src/infer/sub.rs index bf5f328233..b3131936ae 100644 --- a/compiler/rustc_infer/src/infer/sub.rs +++ b/compiler/rustc_infer/src/infer/sub.rs @@ -62,6 +62,7 @@ impl TypeRelation<'tcx> for Sub<'combine, 'infcx, 'tcx> { fn relate_with_variance>( &mut self, variance: ty::Variance, + _info: ty::VarianceDiagInfo<'tcx>, a: T, b: T, ) -> RelateResult<'tcx, T> { diff --git a/compiler/rustc_infer/src/infer/undo_log.rs b/compiler/rustc_infer/src/infer/undo_log.rs index f41e872e00..5ad2519a93 100644 --- a/compiler/rustc_infer/src/infer/undo_log.rs +++ b/compiler/rustc_infer/src/infer/undo_log.rs @@ -3,6 +3,7 @@ use std::marker::PhantomData; use rustc_data_structures::snapshot_vec as sv; use rustc_data_structures::undo_log::{Rollback, UndoLogs}; use rustc_data_structures::unify as ut; +use rustc_middle::infer::unify_key::RegionVidKey; use rustc_middle::ty; use crate::{ @@ -22,7 +23,7 @@ pub(crate) enum UndoLog<'tcx> { IntUnificationTable(sv::UndoLog>), FloatUnificationTable(sv::UndoLog>), RegionConstraintCollector(region_constraints::UndoLog<'tcx>), - RegionUnificationTable(sv::UndoLog>), + RegionUnificationTable(sv::UndoLog>>), ProjectionCache(traits::UndoLog<'tcx>), PushRegionObligation, } @@ -55,7 +56,7 @@ impl_from! { ConstUnificationTable(sv::UndoLog>>), - RegionUnificationTable(sv::UndoLog>), + RegionUnificationTable(sv::UndoLog>>), ProjectionCache(traits::UndoLog<'tcx>), } diff --git a/compiler/rustc_infer/src/lib.rs b/compiler/rustc_infer/src/lib.rs index 15b4a7ed20..ee358c52c2 100644 --- a/compiler/rustc_infer/src/lib.rs +++ b/compiler/rustc_infer/src/lib.rs @@ -16,13 +16,12 @@ #![feature(bool_to_option)] #![feature(box_patterns)] #![feature(box_syntax)] -#![feature(const_panic)] #![feature(extend_one)] #![feature(iter_zip)] #![feature(never_type)] -#![cfg_attr(bootstrap, feature(or_patterns))] #![feature(in_band_lifetimes)] #![feature(control_flow_enum)] +#![feature(min_specialization)] #![recursion_limit = "512"] // For rustdoc #[macro_use] diff --git a/compiler/rustc_interface/src/lib.rs b/compiler/rustc_interface/src/lib.rs index f2b69da3f8..c7424b9e2a 100644 --- a/compiler/rustc_interface/src/lib.rs +++ b/compiler/rustc_interface/src/lib.rs @@ -1,10 +1,7 @@ #![feature(bool_to_option)] #![feature(box_patterns)] -#![feature(box_syntax)] #![feature(internal_output_capture)] #![feature(nll)] -#![feature(generator_trait)] -#![feature(generators)] #![feature(once_cell)] #![recursion_limit = "256"] diff --git a/compiler/rustc_interface/src/passes.rs b/compiler/rustc_interface/src/passes.rs index 02e62a2cee..9e3e96df3a 100644 --- a/compiler/rustc_interface/src/passes.rs +++ b/compiler/rustc_interface/src/passes.rs @@ -6,14 +6,13 @@ use rustc_ast::mut_visit::MutVisitor; use rustc_ast::{self as ast, visit}; use rustc_codegen_ssa::back::link::emit_metadata; use rustc_codegen_ssa::traits::CodegenBackend; +use rustc_data_structures::parallel; use rustc_data_structures::steal::Steal; use rustc_data_structures::sync::{par_iter, Lrc, OnceCell, ParallelIterator, WorkerLocal}; use rustc_data_structures::temp_dir::MaybeTempDir; -use rustc_data_structures::{box_region_allow_access, declare_box_region_type, parallel}; use rustc_errors::{ErrorReported, PResult}; use rustc_expand::base::ExtCtxt; -use rustc_hir::def_id::{CrateNum, LOCAL_CRATE}; -use rustc_hir::definitions::Definitions; +use rustc_hir::def_id::LOCAL_CRATE; use rustc_hir::Crate; use rustc_lint::LintStore; use rustc_metadata::creader::CStore; @@ -36,7 +35,6 @@ use rustc_session::output::{filename_for_input, filename_for_metadata}; use rustc_session::search_paths::PathKind; use rustc_session::Session; use rustc_span::symbol::{Ident, Symbol}; -use rustc_span::{FileName, RealFileName}; use rustc_trait_selection::traits; use rustc_typeck as typeck; use tracing::{info, warn}; @@ -49,9 +47,11 @@ use std::cell::RefCell; use std::ffi::OsString; use std::io::{self, BufWriter, Write}; use std::lazy::SyncLazy; +use std::marker::PhantomPinned; use std::path::PathBuf; +use std::pin::Pin; use std::rc::Rc; -use std::{env, fs, iter, mem}; +use std::{env, fs, iter}; pub fn parse<'a>(sess: &'a Session, input: &Input) -> PResult<'a, ast::Crate> { let krate = sess.time("parse_crate", || match input { @@ -87,11 +87,83 @@ fn count_nodes(krate: &ast::Crate) -> usize { counter.count } -declare_box_region_type!( - pub BoxedResolver, - for(), - (&mut Resolver<'_>) -> (Result, ResolverOutputs) -); +pub use boxed_resolver::BoxedResolver; +mod boxed_resolver { + use super::*; + + pub struct BoxedResolver(Pin>); + + struct BoxedResolverInner { + session: Lrc, + resolver_arenas: Option>, + resolver: Option>, + _pin: PhantomPinned, + } + + // Note: Drop order is important to prevent dangling references. Resolver must be dropped first, + // then resolver_arenas and finally session. + impl Drop for BoxedResolverInner { + fn drop(&mut self) { + self.resolver.take(); + self.resolver_arenas.take(); + } + } + + impl BoxedResolver { + pub(super) fn new(session: Lrc, make_resolver: F) -> Result<(ast::Crate, Self)> + where + F: for<'a> FnOnce( + &'a Session, + &'a ResolverArenas<'a>, + ) -> Result<(ast::Crate, Resolver<'a>)>, + { + let mut boxed_resolver = Box::new(BoxedResolverInner { + session, + resolver_arenas: Some(Resolver::arenas()), + resolver: None, + _pin: PhantomPinned, + }); + // SAFETY: `make_resolver` takes a resolver arena with an arbitrary lifetime and + // returns a resolver with the same lifetime as the arena. We ensure that the arena + // outlives the resolver in the drop impl and elsewhere so these transmutes are sound. + unsafe { + let (crate_, resolver) = make_resolver( + std::mem::transmute::<&Session, &Session>(&boxed_resolver.session), + std::mem::transmute::<&ResolverArenas<'_>, &ResolverArenas<'_>>( + boxed_resolver.resolver_arenas.as_ref().unwrap(), + ), + )?; + boxed_resolver.resolver = Some(resolver); + Ok((crate_, BoxedResolver(Pin::new_unchecked(boxed_resolver)))) + } + } + + pub fn access FnOnce(&mut Resolver<'a>) -> R, R>(&mut self, f: F) -> R { + // SAFETY: The resolver doesn't need to be pinned. + let mut resolver = unsafe { + self.0.as_mut().map_unchecked_mut(|boxed_resolver| &mut boxed_resolver.resolver) + }; + f((&mut *resolver).as_mut().unwrap()) + } + + pub fn to_resolver_outputs(resolver: Rc>) -> ResolverOutputs { + match Rc::try_unwrap(resolver) { + Ok(resolver) => { + let mut resolver = resolver.into_inner(); + // SAFETY: The resolver doesn't need to be pinned. + let mut resolver = unsafe { + resolver + .0 + .as_mut() + .map_unchecked_mut(|boxed_resolver| &mut boxed_resolver.resolver) + }; + resolver.take().unwrap().into_outputs() + } + Err(resolver) => resolver.borrow_mut().access(|resolver| resolver.clone_outputs()), + } + } + } +} /// Runs the "early phases" of the compiler: initial `cfg` processing, loading compiler plugins, /// syntax expansion, secondary `cfg` expansion, synthesis of a test @@ -113,41 +185,16 @@ pub fn configure_and_expand( // its contents but the results of name resolution on those contents. Hopefully we'll push // this back at some point. let crate_name = crate_name.to_string(); - let (result, resolver) = BoxedResolver::new(static move |mut action| { - let _ = action; - let sess = &*sess; - let resolver_arenas = Resolver::arenas(); - let res = configure_and_expand_inner( + BoxedResolver::new(sess, move |sess, resolver_arenas| { + configure_and_expand_inner( sess, &lint_store, krate, &crate_name, &resolver_arenas, - &*metadata_loader, - ); - let mut resolver = match res { - Err(v) => { - yield BoxedResolver::initial_yield(Err(v)); - panic!() - } - Ok((krate, resolver)) => { - action = yield BoxedResolver::initial_yield(Ok(krate)); - resolver - } - }; - box_region_allow_access!(for(), (&mut Resolver<'_>), (&mut resolver), action); - resolver.into_outputs() - }); - result.map(|k| (k, resolver)) -} - -impl BoxedResolver { - pub fn to_resolver_outputs(resolver: Rc>) -> ResolverOutputs { - match Rc::try_unwrap(resolver) { - Ok(resolver) => resolver.into_inner().complete(), - Err(resolver) => resolver.borrow_mut().access(|resolver| resolver.clone_outputs()), - } - } + metadata_loader, + ) + }) } pub fn register_plugins<'a>( @@ -174,7 +221,7 @@ pub fn register_plugins<'a>( let disambiguator = util::compute_crate_disambiguator(sess); sess.crate_disambiguator.set(disambiguator).expect("not yet initialized"); - rustc_incremental::prepare_session_directory(sess, &crate_name, disambiguator); + rustc_incremental::prepare_session_directory(sess, &crate_name, disambiguator)?; if sess.opts.incremental.is_some() { sess.time("incr_comp_garbage_collect_session_directories", || { @@ -233,11 +280,11 @@ fn pre_expansion_lint( fn configure_and_expand_inner<'a>( sess: &'a Session, - lint_store: &'a LintStore, + lint_store: &LintStore, mut krate: ast::Crate, crate_name: &str, resolver_arenas: &'a ResolverArenas<'a>, - metadata_loader: &'a MetadataLoaderDyn, + metadata_loader: Box, ) -> Result<(ast::Crate, Resolver<'a>)> { tracing::trace!("configure_and_expand_inner"); pre_expansion_lint(sess, lint_store, &krate, crate_name); @@ -532,10 +579,10 @@ fn output_conflicts_with_dir(output_paths: &[PathBuf]) -> Option { check_output(output_paths, check) } -fn escape_dep_filename(filename: &FileName) -> String { +fn escape_dep_filename(filename: &String) -> String { // Apparently clang and gcc *only* escape spaces: // http://llvm.org/klaus/clang/commit/9d50634cfc268ecc9a7250226dd5ca0e945240d4 - filename.to_string().replace(" ", "\\ ") + filename.replace(" ", "\\ ") } // Makefile comments only need escaping newlines and `\`. @@ -575,7 +622,7 @@ fn write_out_deps( .iter() .filter(|fmap| fmap.is_real_file()) .filter(|fmap| !fmap.is_imported()) - .map(|fmap| escape_dep_filename(&fmap.unmapped_path.as_ref().unwrap_or(&fmap.name))) + .map(|fmap| escape_dep_filename(&fmap.name.prefer_local().to_string())) .collect(); if let Some(ref backend) = sess.opts.debugging_opts.codegen_backend { @@ -587,16 +634,13 @@ fn write_out_deps( for cnum in resolver.cstore().crates_untracked() { let source = resolver.cstore().crate_source_untracked(cnum); if let Some((path, _)) = source.dylib { - let file_name = FileName::Real(RealFileName::Named(path)); - files.push(escape_dep_filename(&file_name)); + files.push(escape_dep_filename(&path.display().to_string())); } if let Some((path, _)) = source.rlib { - let file_name = FileName::Real(RealFileName::Named(path)); - files.push(escape_dep_filename(&file_name)); + files.push(escape_dep_filename(&path.display().to_string())); } if let Some((path, _)) = source.rmeta { - let file_name = FileName::Real(RealFileName::Named(path)); - files.push(escape_dep_filename(&file_name)); + files.push(escape_dep_filename(&path.display().to_string())); } } }); @@ -761,7 +805,7 @@ pub fn create_global_ctxt<'tcx>( lint_store: Lrc, krate: &'tcx Crate<'tcx>, dep_graph: DepGraph, - mut resolver_outputs: ResolverOutputs, + resolver_outputs: ResolverOutputs, outputs: OutputFilenames, crate_name: &str, queries: &'tcx OnceCell>, @@ -769,12 +813,8 @@ pub fn create_global_ctxt<'tcx>( arena: &'tcx WorkerLocal>, ) -> QueryContext<'tcx> { let sess = &compiler.session(); - let defs: &'tcx Definitions = arena.alloc(mem::replace( - &mut resolver_outputs.definitions, - Definitions::new(crate_name, sess.local_crate_disambiguator()), - )); - let query_result_on_disk_cache = rustc_incremental::load_query_result_cache(sess, defs); + let query_result_on_disk_cache = rustc_incremental::load_query_result_cache(sess); let codegen_backend = compiler.codegen_backend(); let mut local_providers = *DEFAULT_QUERY_PROVIDERS; @@ -798,12 +838,11 @@ pub fn create_global_ctxt<'tcx>( arena, resolver_outputs, krate, - defs, dep_graph, query_result_on_disk_cache, queries.as_dyn(), &crate_name, - &outputs, + outputs, ) }) }); @@ -813,9 +852,7 @@ pub fn create_global_ctxt<'tcx>( /// Runs the resolution, type-checking, region checking and other /// miscellaneous analysis passes on the crate. -fn analysis(tcx: TyCtxt<'_>, cnum: CrateNum) -> Result<()> { - assert_eq!(cnum, LOCAL_CRATE); - +fn analysis(tcx: TyCtxt<'_>, (): ()) -> Result<()> { rustc_passes::hir_id_validator::check_crate(tcx); let sess = tcx.sess; @@ -824,14 +861,13 @@ fn analysis(tcx: TyCtxt<'_>, cnum: CrateNum) -> Result<()> { sess.time("misc_checking_1", || { parallel!( { - entry_point = sess - .time("looking_for_entry_point", || rustc_passes::entry::find_entry_point(tcx)); + entry_point = sess.time("looking_for_entry_point", || tcx.entry_fn(())); - sess.time("looking_for_plugin_registrar", || { - plugin::build::find_plugin_registrar(tcx) - }); + sess.time("looking_for_plugin_registrar", || tcx.ensure().plugin_registrar_fn(())); - sess.time("looking_for_derive_registrar", || proc_macro_decls::find(tcx)); + sess.time("looking_for_derive_registrar", || { + tcx.ensure().proc_macro_decls_static(()) + }); let cstore = tcx .cstore_as_any() @@ -884,7 +920,10 @@ fn analysis(tcx: TyCtxt<'_>, cnum: CrateNum) -> Result<()> { sess.time("MIR_effect_checking", || { for def_id in tcx.body_owners() { - mir::transform::check_unsafety::check_unsafety(tcx, def_id); + tcx.ensure().thir_check_unsafety(def_id); + if !tcx.sess.opts.debugging_opts.thir_unsafeck { + mir::transform::check_unsafety::check_unsafety(tcx, def_id); + } if tcx.hir().body_const_context(def_id).is_some() { tcx.ensure() @@ -907,11 +946,11 @@ fn analysis(tcx: TyCtxt<'_>, cnum: CrateNum) -> Result<()> { sess.time("misc_checking_3", || { parallel!( { - tcx.ensure().privacy_access_levels(LOCAL_CRATE); + tcx.ensure().privacy_access_levels(()); parallel!( { - tcx.ensure().check_private_in_public(LOCAL_CRATE); + tcx.ensure().check_private_in_public(()); }, { sess.time("death_checking", || rustc_passes::dead::check_crate(tcx)); @@ -989,7 +1028,7 @@ fn encode_and_write_metadata( .tempdir_in(out_filename.parent().unwrap()) .unwrap_or_else(|err| tcx.sess.fatal(&format!("couldn't create a temp dir: {}", err))); let metadata_tmpdir = MaybeTempDir::new(metadata_tmpdir, tcx.sess.opts.cg.save_temps); - let metadata_filename = emit_metadata(tcx.sess, &metadata, &metadata_tmpdir); + let metadata_filename = emit_metadata(tcx.sess, &metadata.raw_data, &metadata_tmpdir); if let Err(e) = util::non_durable_rename(&metadata_filename, &out_filename) { tcx.sess.fatal(&format!("failed to write {}: {}", out_filename.display(), e)); } diff --git a/compiler/rustc_interface/src/proc_macro_decls.rs b/compiler/rustc_interface/src/proc_macro_decls.rs index 4637055a82..88cf6275eb 100644 --- a/compiler/rustc_interface/src/proc_macro_decls.rs +++ b/compiler/rustc_interface/src/proc_macro_decls.rs @@ -1,21 +1,15 @@ use rustc_hir as hir; -use rustc_hir::def_id::{CrateNum, DefId, LOCAL_CRATE}; +use rustc_hir::def_id::LocalDefId; use rustc_hir::itemlikevisit::ItemLikeVisitor; use rustc_middle::ty::query::Providers; use rustc_middle::ty::TyCtxt; use rustc_span::symbol::sym; -pub fn find(tcx: TyCtxt<'_>) -> Option { - tcx.proc_macro_decls_static(LOCAL_CRATE) -} - -fn proc_macro_decls_static(tcx: TyCtxt<'_>, cnum: CrateNum) -> Option { - assert_eq!(cnum, LOCAL_CRATE); - +fn proc_macro_decls_static(tcx: TyCtxt<'_>, (): ()) -> Option { let mut finder = Finder { tcx, decls: None }; tcx.hir().krate().visit_all_item_likes(&mut finder); - finder.decls.map(|id| tcx.hir().local_def_id(id).to_def_id()) + finder.decls.map(|id| tcx.hir().local_def_id(id)) } struct Finder<'tcx> { diff --git a/compiler/rustc_interface/src/queries.rs b/compiler/rustc_interface/src/queries.rs index bc94fb67ac..2320f0b47d 100644 --- a/compiler/rustc_interface/src/queries.rs +++ b/compiler/rustc_interface/src/queries.rs @@ -148,7 +148,7 @@ impl<'tcx> Queries<'tcx> { self.compiler.register_lints.as_deref().unwrap_or_else(|| empty), krate, &crate_name, - ); + )?; // Compute the dependency graph (in the background). We want to do // this as early as possible, to give the DepGraph maximum time to @@ -157,7 +157,7 @@ impl<'tcx> Queries<'tcx> { // called, which happens within passes::register_plugins(). self.dep_graph_future().ok(); - result + Ok(result) }) } @@ -245,8 +245,7 @@ impl<'tcx> Queries<'tcx> { self.prepare_outputs.compute(|| { let expansion_result = self.expansion()?; let (krate, boxed_resolver, _) = &*expansion_result.peek(); - let crate_name = self.crate_name()?; - let crate_name = crate_name.peek(); + let crate_name = self.crate_name()?.peek(); passes::prepare_outputs( self.session(), self.compiler, @@ -285,7 +284,7 @@ impl<'tcx> Queries<'tcx> { self.ongoing_codegen.compute(|| { let outputs = self.prepare_outputs()?; self.global_ctxt()?.peek_mut().enter(|tcx| { - tcx.analysis(LOCAL_CRATE).ok(); + tcx.analysis(()).ok(); // Don't do code generation if there were any errors self.session().compile_status()?; @@ -302,7 +301,7 @@ impl<'tcx> Queries<'tcx> { /// to write UI tests that actually test that compilation succeeds without reporting /// an error. fn check_for_rustc_errors_attr(tcx: TyCtxt<'_>) { - let def_id = match tcx.entry_fn(LOCAL_CRATE) { + let def_id = match tcx.entry_fn(()) { Some((def_id, _)) => def_id, _ => return, }; @@ -343,32 +342,36 @@ impl<'tcx> Queries<'tcx> { } pub fn linker(&'tcx self) -> Result { - let dep_graph = self.dep_graph()?; - let prepare_outputs = self.prepare_outputs()?; - let crate_hash = self.global_ctxt()?.peek_mut().enter(|tcx| tcx.crate_hash(LOCAL_CRATE)); - let ongoing_codegen = self.ongoing_codegen()?; - let sess = self.session().clone(); let codegen_backend = self.codegen_backend().clone(); + let dep_graph = self.dep_graph()?.peek().clone(); + let prepare_outputs = self.prepare_outputs()?.take(); + let crate_hash = self.global_ctxt()?.peek_mut().enter(|tcx| tcx.crate_hash(LOCAL_CRATE)); + let ongoing_codegen = self.ongoing_codegen()?.take(); + Ok(Linker { sess, - dep_graph: dep_graph.peek().clone(), - prepare_outputs: prepare_outputs.take(), - crate_hash, - ongoing_codegen: ongoing_codegen.take(), codegen_backend, + + dep_graph, + prepare_outputs, + crate_hash, + ongoing_codegen, }) } } pub struct Linker { + // compilation inputs sess: Lrc, + codegen_backend: Lrc>, + + // compilation outputs dep_graph: DepGraph, prepare_outputs: OutputFilenames, crate_hash: Svh, ongoing_codegen: Box, - codegen_backend: Lrc>, } impl Linker { diff --git a/compiler/rustc_interface/src/tests.rs b/compiler/rustc_interface/src/tests.rs index d8c1a7a268..5d8a6084f2 100644 --- a/compiler/rustc_interface/src/tests.rs +++ b/compiler/rustc_interface/src/tests.rs @@ -12,7 +12,7 @@ use rustc_session::config::{ }; use rustc_session::lint::Level; use rustc_session::search_paths::SearchPath; -use rustc_session::utils::{CanonicalizedPath, NativeLibKind}; +use rustc_session::utils::{CanonicalizedPath, NativeLib, NativeLibKind}; use rustc_session::{build_session, getopts, DiagnosticOutput, Session}; use rustc_span::edition::{Edition, DEFAULT_EDITION}; use rustc_span::symbol::sym; @@ -252,7 +252,8 @@ fn test_lints_tracking_hash_different_construction_order() { (String::from("d"), Level::Forbid), ]; - assert_same_hash(&v1, &v2); + // The hash should be order-dependent + assert_different_hash(&v1, &v2); } #[test] @@ -303,38 +304,122 @@ fn test_native_libs_tracking_hash_different_values() { let mut v2 = Options::default(); let mut v3 = Options::default(); let mut v4 = Options::default(); + let mut v5 = Options::default(); // Reference v1.libs = vec![ - (String::from("a"), None, NativeLibKind::StaticBundle), - (String::from("b"), None, NativeLibKind::Framework), - (String::from("c"), None, NativeLibKind::Unspecified), + NativeLib { + name: String::from("a"), + new_name: None, + kind: NativeLibKind::Static { bundle: None, whole_archive: None }, + verbatim: None, + }, + NativeLib { + name: String::from("b"), + new_name: None, + kind: NativeLibKind::Framework { as_needed: None }, + verbatim: None, + }, + NativeLib { + name: String::from("c"), + new_name: None, + kind: NativeLibKind::Unspecified, + verbatim: None, + }, ]; // Change label v2.libs = vec![ - (String::from("a"), None, NativeLibKind::StaticBundle), - (String::from("X"), None, NativeLibKind::Framework), - (String::from("c"), None, NativeLibKind::Unspecified), + NativeLib { + name: String::from("a"), + new_name: None, + kind: NativeLibKind::Static { bundle: None, whole_archive: None }, + verbatim: None, + }, + NativeLib { + name: String::from("X"), + new_name: None, + kind: NativeLibKind::Framework { as_needed: None }, + verbatim: None, + }, + NativeLib { + name: String::from("c"), + new_name: None, + kind: NativeLibKind::Unspecified, + verbatim: None, + }, ]; // Change kind v3.libs = vec![ - (String::from("a"), None, NativeLibKind::StaticBundle), - (String::from("b"), None, NativeLibKind::StaticBundle), - (String::from("c"), None, NativeLibKind::Unspecified), + NativeLib { + name: String::from("a"), + new_name: None, + kind: NativeLibKind::Static { bundle: None, whole_archive: None }, + verbatim: None, + }, + NativeLib { + name: String::from("b"), + new_name: None, + kind: NativeLibKind::Static { bundle: None, whole_archive: None }, + verbatim: None, + }, + NativeLib { + name: String::from("c"), + new_name: None, + kind: NativeLibKind::Unspecified, + verbatim: None, + }, ]; // Change new-name v4.libs = vec![ - (String::from("a"), None, NativeLibKind::StaticBundle), - (String::from("b"), Some(String::from("X")), NativeLibKind::Framework), - (String::from("c"), None, NativeLibKind::Unspecified), + NativeLib { + name: String::from("a"), + new_name: None, + kind: NativeLibKind::Static { bundle: None, whole_archive: None }, + verbatim: None, + }, + NativeLib { + name: String::from("b"), + new_name: Some(String::from("X")), + kind: NativeLibKind::Framework { as_needed: None }, + verbatim: None, + }, + NativeLib { + name: String::from("c"), + new_name: None, + kind: NativeLibKind::Unspecified, + verbatim: None, + }, + ]; + + // Change verbatim + v5.libs = vec![ + NativeLib { + name: String::from("a"), + new_name: None, + kind: NativeLibKind::Static { bundle: None, whole_archive: None }, + verbatim: None, + }, + NativeLib { + name: String::from("b"), + new_name: None, + kind: NativeLibKind::Framework { as_needed: None }, + verbatim: Some(true), + }, + NativeLib { + name: String::from("c"), + new_name: None, + kind: NativeLibKind::Unspecified, + verbatim: None, + }, ]; assert_different_hash(&v1, &v2); assert_different_hash(&v1, &v3); assert_different_hash(&v1, &v4); + assert_different_hash(&v1, &v5); } #[test] @@ -345,26 +430,72 @@ fn test_native_libs_tracking_hash_different_order() { // Reference v1.libs = vec![ - (String::from("a"), None, NativeLibKind::StaticBundle), - (String::from("b"), None, NativeLibKind::Framework), - (String::from("c"), None, NativeLibKind::Unspecified), + NativeLib { + name: String::from("a"), + new_name: None, + kind: NativeLibKind::Static { bundle: None, whole_archive: None }, + verbatim: None, + }, + NativeLib { + name: String::from("b"), + new_name: None, + kind: NativeLibKind::Framework { as_needed: None }, + verbatim: None, + }, + NativeLib { + name: String::from("c"), + new_name: None, + kind: NativeLibKind::Unspecified, + verbatim: None, + }, ]; v2.libs = vec![ - (String::from("b"), None, NativeLibKind::Framework), - (String::from("a"), None, NativeLibKind::StaticBundle), - (String::from("c"), None, NativeLibKind::Unspecified), + NativeLib { + name: String::from("b"), + new_name: None, + kind: NativeLibKind::Framework { as_needed: None }, + verbatim: None, + }, + NativeLib { + name: String::from("a"), + new_name: None, + kind: NativeLibKind::Static { bundle: None, whole_archive: None }, + verbatim: None, + }, + NativeLib { + name: String::from("c"), + new_name: None, + kind: NativeLibKind::Unspecified, + verbatim: None, + }, ]; v3.libs = vec![ - (String::from("c"), None, NativeLibKind::Unspecified), - (String::from("a"), None, NativeLibKind::StaticBundle), - (String::from("b"), None, NativeLibKind::Framework), + NativeLib { + name: String::from("c"), + new_name: None, + kind: NativeLibKind::Unspecified, + verbatim: None, + }, + NativeLib { + name: String::from("a"), + new_name: None, + kind: NativeLibKind::Static { bundle: None, whole_archive: None }, + verbatim: None, + }, + NativeLib { + name: String::from("b"), + new_name: None, + kind: NativeLibKind::Framework { as_needed: None }, + verbatim: None, + }, ]; - assert_same_hash(&v1, &v2); - assert_same_hash(&v1, &v3); - assert_same_hash(&v2, &v3); + // The hash should be order-dependent + assert_different_hash(&v1, &v2); + assert_different_hash(&v1, &v3); + assert_different_hash(&v2, &v3); } #[test] @@ -580,7 +711,7 @@ fn test_debugging_options_tracking_hash() { tracked!(mir_emit_retag, true); tracked!(mir_opt_level, Some(4)); tracked!(mutable_noalias, Some(true)); - tracked!(new_llvm_pass_manager, true); + tracked!(new_llvm_pass_manager, Some(true)); tracked!(no_codegen, true); tracked!(no_generate_arange_section, true); tracked!(no_link, true); @@ -595,6 +726,7 @@ fn test_debugging_options_tracking_hash() { tracked!(profile_emit, Some(PathBuf::from("abc"))); tracked!(relax_elf_relocations, Some(true)); tracked!(relro_level, Some(RelroLevel::Full)); + tracked!(simulate_remapped_rust_src_base, Some(PathBuf::from("/rustc/abc"))); tracked!(report_delayed_bugs, true); tracked!(sanitizer, SanitizerSet::ADDRESS); tracked!(sanitizer_memory_track_origins, 2); @@ -606,6 +738,7 @@ fn test_debugging_options_tracking_hash() { tracked!(symbol_mangling_version, Some(SymbolManglingVersion::V0)); tracked!(teach, true); tracked!(thinlto, Some(true)); + tracked!(thir_unsafeck, true); tracked!(tune_cpu, Some(String::from("abc"))); tracked!(tls_model, Some(TlsModel::GeneralDynamic)); tracked!(trap_unreachable, Some(false)); diff --git a/compiler/rustc_interface/src/util.rs b/compiler/rustc_interface/src/util.rs index 59488fc80a..6485fbebd6 100644 --- a/compiler/rustc_interface/src/util.rs +++ b/compiler/rustc_interface/src/util.rs @@ -35,7 +35,7 @@ use std::ops::DerefMut; use std::panic; use std::path::{Path, PathBuf}; use std::sync::atomic::{AtomicBool, Ordering}; -use std::sync::{Arc, Mutex, Once}; +use std::sync::{Arc, Mutex}; use std::thread; use tracing::info; @@ -76,7 +76,10 @@ pub fn create_session( let codegen_backend = if let Some(make_codegen_backend) = make_codegen_backend { make_codegen_backend(&sopts) } else { - get_codegen_backend(&sopts) + get_codegen_backend( + &sopts.maybe_sysroot, + sopts.debugging_opts.codegen_backend.as_ref().map(|name| &name[..]), + ) }; // target_override is documented to be called before init(), so this is okay @@ -244,35 +247,34 @@ fn load_backend_from_dylib(path: &Path) -> fn() -> Box { } } -pub fn get_codegen_backend(sopts: &config::Options) -> Box { - static INIT: Once = Once::new(); - - static mut LOAD: fn() -> Box = || unreachable!(); +/// Get the codegen backend based on the name and specified sysroot. +/// +/// A name of `None` indicates that the default backend should be used. +pub fn get_codegen_backend( + maybe_sysroot: &Option, + backend_name: Option<&str>, +) -> Box { + static LOAD: SyncOnceCell Box> = SyncOnceCell::new(); - INIT.call_once(|| { + let load = LOAD.get_or_init(|| { #[cfg(feature = "llvm")] const DEFAULT_CODEGEN_BACKEND: &str = "llvm"; #[cfg(not(feature = "llvm"))] const DEFAULT_CODEGEN_BACKEND: &str = "cranelift"; - let codegen_name = sopts - .debugging_opts - .codegen_backend - .as_ref() - .map(|name| &name[..]) - .unwrap_or(DEFAULT_CODEGEN_BACKEND); - - let backend = match codegen_name { + match backend_name.unwrap_or(DEFAULT_CODEGEN_BACKEND) { filename if filename.contains('.') => load_backend_from_dylib(filename.as_ref()), - codegen_name => get_builtin_codegen_backend(&sopts.maybe_sysroot, codegen_name), - }; - - unsafe { - LOAD = backend; + #[cfg(feature = "llvm")] + "llvm" => rustc_codegen_llvm::LlvmCodegenBackend::new, + backend_name => get_codegen_sysroot(maybe_sysroot, backend_name), } }); - unsafe { LOAD() } + + // SAFETY: In case of a builtin codegen backend this is safe. In case of an external codegen + // backend we hope that the backend links against the same rustc_driver version. If this is not + // the case, we get UB. + unsafe { load() } } // This is used for rustdoc, but it uses similar machinery to codegen backend @@ -390,17 +392,6 @@ fn sysroot_candidates() -> Vec { } } -pub fn get_builtin_codegen_backend( - maybe_sysroot: &Option, - backend_name: &str, -) -> fn() -> Box { - match backend_name { - #[cfg(feature = "llvm")] - "llvm" => rustc_codegen_llvm::LlvmCodegenBackend::new, - _ => get_codegen_sysroot(maybe_sysroot, backend_name), - } -} - pub fn get_codegen_sysroot( maybe_sysroot: &Option, backend_name: &str, @@ -423,8 +414,7 @@ pub fn get_codegen_sysroot( .iter() .chain(sysroot_candidates.iter()) .map(|sysroot| { - let libdir = filesearch::relative_target_lib_path(&sysroot, &target); - sysroot.join(libdir).with_file_name("codegen-backends") + filesearch::make_target_lib_path(&sysroot, &target).with_file_name("codegen-backends") }) .find(|f| { info!("codegen backend candidate: {}", f.display()); @@ -923,7 +913,7 @@ impl<'a> MutVisitor for ReplaceBodyWithLoop<'a, '_> { } } -/// Returns a version string such as "rustc 1.46.0 (04488afe3 2020-08-24)" +/// Returns a version string such as "1.46.0 (04488afe3 2020-08-24)" pub fn version_str() -> Option<&'static str> { option_env!("CFG_VERSION") } diff --git a/compiler/rustc_lint/src/builtin.rs b/compiler/rustc_lint/src/builtin.rs index 3965a3dcdf..f6a84966f7 100644 --- a/compiler/rustc_lint/src/builtin.rs +++ b/compiler/rustc_lint/src/builtin.rs @@ -489,7 +489,7 @@ fn has_doc(sess: &Session, attr: &ast::Attribute) -> bool { if let Some(list) = attr.meta_item_list() { for meta in list { - if meta.has_name(sym::include) || meta.has_name(sym::hidden) { + if meta.has_name(sym::hidden) { return true; } } @@ -1660,7 +1660,11 @@ declare_lint! { /// [`..` range expression]: https://doc.rust-lang.org/reference/expressions/range-expr.html pub ELLIPSIS_INCLUSIVE_RANGE_PATTERNS, Warn, - "`...` range patterns are deprecated" + "`...` range patterns are deprecated", + @future_incompatible = FutureIncompatibleInfo { + reference: "issue #80165 ", + edition: Some(Edition::Edition2021), + }; } #[derive(Default)] @@ -1704,32 +1708,57 @@ impl EarlyLintPass for EllipsisInclusiveRangePatterns { let suggestion = "use `..=` for an inclusive range"; if parenthesise { self.node_id = Some(pat.id); - cx.struct_span_lint(ELLIPSIS_INCLUSIVE_RANGE_PATTERNS, pat.span, |lint| { - let end = expr_to_string(&end); - let replace = match start { - Some(start) => format!("&({}..={})", expr_to_string(&start), end), - None => format!("&(..={})", end), - }; - lint.build(msg) - .span_suggestion( - pat.span, - suggestion, - replace, - Applicability::MachineApplicable, - ) - .emit(); - }); + let end = expr_to_string(&end); + let replace = match start { + Some(start) => format!("&({}..={})", expr_to_string(&start), end), + None => format!("&(..={})", end), + }; + if join.edition() >= Edition::Edition2021 { + let mut err = + rustc_errors::struct_span_err!(cx.sess, pat.span, E0783, "{}", msg,); + err.span_suggestion( + pat.span, + suggestion, + replace, + Applicability::MachineApplicable, + ) + .emit(); + } else { + cx.struct_span_lint(ELLIPSIS_INCLUSIVE_RANGE_PATTERNS, pat.span, |lint| { + lint.build(msg) + .span_suggestion( + pat.span, + suggestion, + replace, + Applicability::MachineApplicable, + ) + .emit(); + }); + } } else { - cx.struct_span_lint(ELLIPSIS_INCLUSIVE_RANGE_PATTERNS, join, |lint| { - lint.build(msg) - .span_suggestion_short( - join, - suggestion, - "..=".to_owned(), - Applicability::MachineApplicable, - ) - .emit(); - }); + let replace = "..=".to_owned(); + if join.edition() >= Edition::Edition2021 { + let mut err = + rustc_errors::struct_span_err!(cx.sess, pat.span, E0783, "{}", msg,); + err.span_suggestion_short( + join, + suggestion, + replace, + Applicability::MachineApplicable, + ) + .emit(); + } else { + cx.struct_span_lint(ELLIPSIS_INCLUSIVE_RANGE_PATTERNS, join, |lint| { + lint.build(msg) + .span_suggestion_short( + join, + suggestion, + replace, + Applicability::MachineApplicable, + ) + .emit(); + }); + } }; } } @@ -2328,7 +2357,7 @@ const HAS_MIN_FEATURES: &[Symbol] = &[sym::specialization]; declare_lint! { /// The `invalid_value` lint detects creating a value that is not valid, - /// such as a NULL reference. + /// such as a null reference. /// /// ### Example /// @@ -2359,7 +2388,7 @@ declare_lint! { /// [undefined behavior]: https://doc.rust-lang.org/reference/behavior-considered-undefined.html pub INVALID_VALUE, Warn, - "an invalid value is being created (such as a NULL reference)" + "an invalid value is being created (such as a null reference)" } declare_lint_pass!(InvalidValue => [INVALID_VALUE]); diff --git a/compiler/rustc_lint/src/context.rs b/compiler/rustc_lint/src/context.rs index c1d6a4f1de..a8df1b0952 100644 --- a/compiler/rustc_lint/src/context.rs +++ b/compiler/rustc_lint/src/context.rs @@ -334,8 +334,14 @@ impl LintStore { } } - /// Checks the validity of lint names derived from the command line - pub fn check_lint_name_cmdline(&self, sess: &Session, lint_name: &str, level: Level) { + /// Checks the validity of lint names derived from the command line. Returns + /// true if the lint is valid, false otherwise. + pub fn check_lint_name_cmdline( + &self, + sess: &Session, + lint_name: &str, + level: Option, + ) -> bool { let db = match self.check_lint_name(lint_name, None) { CheckLintNameResult::Ok(_) => None, CheckLintNameResult::Warning(ref msg, _) => Some(sess.struct_warn(msg)), @@ -361,18 +367,23 @@ impl LintStore { }; if let Some(mut db) = db { - let msg = format!( - "requested on the command line with `{} {}`", - match level { - Level::Allow => "-A", - Level::Warn => "-W", - Level::Deny => "-D", - Level::Forbid => "-F", - }, - lint_name - ); - db.note(&msg); + if let Some(level) = level { + let msg = format!( + "requested on the command line with `{} {}`", + match level { + Level::Allow => "-A", + Level::Warn => "-W", + Level::Deny => "-D", + Level::Forbid => "-F", + }, + lint_name + ); + db.note(&msg); + } db.emit(); + false + } else { + true } } @@ -922,7 +933,7 @@ impl<'tcx> LateContext<'tcx> { } fn path_crate(self, cnum: CrateNum) -> Result { - Ok(vec![self.tcx.original_crate_name(cnum)]) + Ok(vec![self.tcx.crate_name(cnum)]) } fn path_qualified( diff --git a/compiler/rustc_lint/src/early.rs b/compiler/rustc_lint/src/early.rs index 54fcaef414..eb2e495f73 100644 --- a/compiler/rustc_lint/src/early.rs +++ b/compiler/rustc_lint/src/early.rs @@ -367,11 +367,11 @@ pub fn check_ast_crate( krate, EarlyLintPassObjects { lints: &mut passes[..] }, buffered, - pre_expansion, + false, ); } } else { - for pass in &mut passes { + for (i, pass) in passes.iter_mut().enumerate() { buffered = sess.prof.extra_verbose_generic_activity("run_lint", pass.name()).run(|| { early_lint_crate( @@ -380,7 +380,7 @@ pub fn check_ast_crate( krate, EarlyLintPassObjects { lints: slice::from_mut(pass) }, buffered, - pre_expansion, + pre_expansion && i == 0, ) }); } diff --git a/compiler/rustc_lint/src/internal.rs b/compiler/rustc_lint/src/internal.rs index 9b1a339572..0398d4a996 100644 --- a/compiler/rustc_lint/src/internal.rs +++ b/compiler/rustc_lint/src/internal.rs @@ -248,10 +248,21 @@ impl EarlyLintPass for LintPassImpl { if last.ident.name == sym::LintPass { let expn_data = lint_pass.path.span.ctxt().outer_expn_data(); let call_site = expn_data.call_site; - if expn_data.kind != ExpnKind::Macro(MacroKind::Bang, sym::impl_lint_pass) - && call_site.ctxt().outer_expn_data().kind - != ExpnKind::Macro(MacroKind::Bang, sym::declare_lint_pass) - { + if !matches!( + expn_data.kind, + ExpnKind::Macro { + kind: MacroKind::Bang, + name: sym::impl_lint_pass, + proc_macro: _ + } + ) && !matches!( + call_site.ctxt().outer_expn_data().kind, + ExpnKind::Macro { + kind: MacroKind::Bang, + name: sym::declare_lint_pass, + proc_macro: _ + } + ) { cx.struct_span_lint( LINT_PASS_IMPL_WITHOUT_MACRO, lint_pass.path.span, diff --git a/compiler/rustc_lint/src/late.rs b/compiler/rustc_lint/src/late.rs index d325b5fe7f..e504662bb5 100644 --- a/compiler/rustc_lint/src/late.rs +++ b/compiler/rustc_lint/src/late.rs @@ -18,7 +18,7 @@ use crate::{passes::LateLintPassObject, LateContext, LateLintPass, LintStore}; use rustc_ast as ast; use rustc_data_structures::sync::{join, par_iter, ParallelIterator}; use rustc_hir as hir; -use rustc_hir::def_id::{LocalDefId, LOCAL_CRATE}; +use rustc_hir::def_id::LocalDefId; use rustc_hir::intravisit as hir_visit; use rustc_hir::intravisit::Visitor; use rustc_middle::hir::map::Map; @@ -375,7 +375,7 @@ fn late_lint_mod_pass<'tcx, T: LateLintPass<'tcx>>( module_def_id: LocalDefId, pass: T, ) { - let access_levels = &tcx.privacy_access_levels(LOCAL_CRATE); + let access_levels = &tcx.privacy_access_levels(()); let context = LateContext { tcx, @@ -423,7 +423,7 @@ pub fn late_lint_mod<'tcx, T: LateLintPass<'tcx>>( } fn late_lint_pass_crate<'tcx, T: LateLintPass<'tcx>>(tcx: TyCtxt<'tcx>, pass: T) { - let access_levels = &tcx.privacy_access_levels(LOCAL_CRATE); + let access_levels = &tcx.privacy_access_levels(()); let krate = tcx.hir().krate(); @@ -448,6 +448,10 @@ fn late_lint_pass_crate<'tcx, T: LateLintPass<'tcx>>(tcx: TyCtxt<'tcx>, pass: T) lint_callback!(cx, check_crate, krate); hir_visit::walk_crate(cx, krate); + for attr in krate.non_exported_macro_attrs { + // This HIR ID is a lie, since the macro ID isn't available. + cx.visit_attribute(hir::CRATE_HIR_ID, attr); + } lint_callback!(cx, check_crate_post, krate); }) diff --git a/compiler/rustc_lint/src/levels.rs b/compiler/rustc_lint/src/levels.rs index 54909381a1..0ee434f5fb 100644 --- a/compiler/rustc_lint/src/levels.rs +++ b/compiler/rustc_lint/src/levels.rs @@ -6,8 +6,7 @@ use rustc_ast_pretty::pprust; use rustc_data_structures::fx::FxHashMap; use rustc_errors::{struct_span_err, Applicability, DiagnosticBuilder}; use rustc_hir as hir; -use rustc_hir::def_id::{CrateNum, DefId, CRATE_DEF_INDEX, LOCAL_CRATE}; -use rustc_hir::{intravisit, HirId}; +use rustc_hir::{intravisit, HirId, CRATE_HIR_ID}; use rustc_middle::hir::map::Map; use rustc_middle::lint::LevelAndSource; use rustc_middle::lint::LintDiagnosticBuilder; @@ -28,10 +27,9 @@ use tracing::debug; use std::cmp; -fn lint_levels(tcx: TyCtxt<'_>, cnum: CrateNum) -> LintLevelMap { - assert_eq!(cnum, LOCAL_CRATE); +fn lint_levels(tcx: TyCtxt<'_>, (): ()) -> LintLevelMap { let store = unerased_lint_store(tcx); - let crate_attrs = tcx.get_attrs(DefId { krate: cnum, index: CRATE_DEF_INDEX }); + let crate_attrs = tcx.hir().attrs(CRATE_HIR_ID); let levels = LintLevelsBuilder::new(tcx.sess, false, &store, crate_attrs); let mut builder = LintLevelMapBuilder { levels, tcx, store }; let krate = tcx.hir().krate(); @@ -90,7 +88,7 @@ impl<'s> LintLevelsBuilder<'s> { self.sets.lint_cap = sess.opts.lint_cap.unwrap_or(Level::Forbid); for &(ref lint_name, level) in &sess.opts.lint_opts { - store.check_lint_name_cmdline(sess, &lint_name, level); + store.check_lint_name_cmdline(sess, &lint_name, Some(level)); let orig_level = level; // If the cap is less than this specified level, e.g., if we've got @@ -111,6 +109,16 @@ impl<'s> LintLevelsBuilder<'s> { } } + for lint_name in &sess.opts.force_warns { + let valid = store.check_lint_name_cmdline(sess, lint_name, None); + if valid { + let lints = store + .find_lints(lint_name) + .unwrap_or_else(|_| bug!("A valid lint failed to produce a lint ids")); + self.sets.force_warns.extend(&lints); + } + } + self.sets.list.push(LintSet::CommandLine { specs }); } @@ -144,6 +152,9 @@ impl<'s> LintLevelsBuilder<'s> { LintLevelSource::Default => false, LintLevelSource::Node(symbol, _, _) => self.store.is_lint_group(symbol), LintLevelSource::CommandLine(symbol, _) => self.store.is_lint_group(symbol), + LintLevelSource::ForceWarn(_symbol) => { + bug!("forced warn lint returned a forbid lint level") + } }; debug!( "fcw_warning={:?}, specs.get(&id) = {:?}, old_src={:?}, id_name={:?}", @@ -168,6 +179,7 @@ impl<'s> LintLevelsBuilder<'s> { LintLevelSource::CommandLine(_, _) => { diag_builder.note("`forbid` lint level was set on command line"); } + _ => bug!("forced warn lint returned a forbid lint level"), } diag_builder.emit(); }; diff --git a/compiler/rustc_lint/src/lib.rs b/compiler/rustc_lint/src/lib.rs index 2f46969b02..4f59460aa8 100644 --- a/compiler/rustc_lint/src/lib.rs +++ b/compiler/rustc_lint/src/lib.rs @@ -36,9 +36,6 @@ #![feature(iter_zip)] #![feature(never_type)] #![feature(nll)] -#![cfg_attr(bootstrap, feature(or_patterns))] -#![feature(half_open_range_patterns)] -#![feature(exclusive_range_pattern)] #![feature(control_flow_enum)] #![recursion_limit = "256"] diff --git a/compiler/rustc_lint/src/non_fmt_panic.rs b/compiler/rustc_lint/src/non_fmt_panic.rs index 5a27135581..c91dc37b37 100644 --- a/compiler/rustc_lint/src/non_fmt_panic.rs +++ b/compiler/rustc_lint/src/non_fmt_panic.rs @@ -4,7 +4,7 @@ use rustc_errors::{pluralize, Applicability}; use rustc_hir as hir; use rustc_middle::ty; use rustc_parse_format::{ParseMode, Parser, Piece}; -use rustc_span::{sym, symbol::kw, InnerSpan, Span, Symbol}; +use rustc_span::{hygiene, sym, symbol::kw, symbol::SymbolStr, InnerSpan, Span, Symbol}; declare_lint! { /// The `non_fmt_panic` lint detects `panic!(..)` invocations where the first @@ -67,7 +67,7 @@ fn check_panic<'tcx>(cx: &LateContext<'tcx>, f: &'tcx hir::Expr<'tcx>, arg: &'tc // The argument is *not* a string literal. - let (span, panic) = panic_call(cx, f); + let (span, panic, symbol_str) = panic_call(cx, f); // Find the span of the argument to `panic!()`, before expansion in the // case of `panic!(some_macro!())`. @@ -95,7 +95,7 @@ fn check_panic<'tcx>(cx: &LateContext<'tcx>, f: &'tcx hir::Expr<'tcx>, arg: &'tc } if arg_macro.map_or(false, |id| cx.tcx.is_diagnostic_item(sym::format_macro, id)) { // A case of `panic!(format!(..))`. - l.note("the panic!() macro supports formatting, so there's no need for the format!() macro here"); + l.note(format!("the {}!() macro supports formatting, so there's no need for the format!() macro here", symbol_str).as_str()); if let Some((open, close, _)) = find_delimiters(cx, arg_span) { l.multipart_suggestion( "remove the `format!(..)` macro call", @@ -160,7 +160,7 @@ fn check_panic_str<'tcx>( Parser::new(fmt.as_ref(), style, snippet.clone(), false, ParseMode::Format); let n_arguments = (&mut fmt_parser).filter(|a| matches!(a, Piece::NextArgument(_))).count(); - let (span, _) = panic_call(cx, f); + let (span, _, _) = panic_call(cx, f); if n_arguments > 0 && fmt_parser.errors.is_empty() { let arg_spans: Vec<_> = match &fmt_parser.arg_places[..] { @@ -230,7 +230,7 @@ fn find_delimiters<'tcx>(cx: &LateContext<'tcx>, span: Span) -> Option<(Span, Sp )) } -fn panic_call<'tcx>(cx: &LateContext<'tcx>, f: &'tcx hir::Expr<'tcx>) -> (Span, Symbol) { +fn panic_call<'tcx>(cx: &LateContext<'tcx>, f: &'tcx hir::Expr<'tcx>) -> (Span, Symbol, SymbolStr) { let mut expn = f.span.ctxt().outer_expn_data(); let mut panic_macro = kw::Empty; @@ -248,5 +248,11 @@ fn panic_call<'tcx>(cx: &LateContext<'tcx>, f: &'tcx hir::Expr<'tcx>) -> (Span, } } - (expn.call_site, panic_macro) + let macro_symbol = + if let hygiene::ExpnKind::Macro { kind: _, name: symbol, proc_macro: _ } = expn.kind { + symbol + } else { + Symbol::intern("panic") + }; + (expn.call_site, panic_macro, macro_symbol.as_str()) } diff --git a/compiler/rustc_lint/src/nonstandard_style.rs b/compiler/rustc_lint/src/nonstandard_style.rs index be9c6eafb6..7146dd51aa 100644 --- a/compiler/rustc_lint/src/nonstandard_style.rs +++ b/compiler/rustc_lint/src/nonstandard_style.rs @@ -176,6 +176,7 @@ impl EarlyLintPass for NonCamelCaseTypes { | ast::ItemKind::Struct(..) | ast::ItemKind::Union(..) => self.check_case(cx, "type", &it.ident), ast::ItemKind::Trait(..) => self.check_case(cx, "trait", &it.ident), + ast::ItemKind::TraitAlias(..) => self.check_case(cx, "trait alias", &it.ident), _ => (), } } diff --git a/compiler/rustc_lint/src/types.rs b/compiler/rustc_lint/src/types.rs index 9c94bab04e..a3a87a4876 100644 --- a/compiler/rustc_lint/src/types.rs +++ b/compiler/rustc_lint/src/types.rs @@ -5,7 +5,6 @@ use rustc_data_structures::fx::FxHashSet; use rustc_errors::Applicability; use rustc_hir as hir; use rustc_hir::{is_range_literal, ExprKind, Node}; -use rustc_index::vec::Idx; use rustc_middle::ty::layout::{IntegerExt, SizeSkeleton}; use rustc_middle::ty::subst::SubstsRef; use rustc_middle::ty::{self, AdtKind, Ty, TyCtxt, TypeFoldable}; @@ -13,7 +12,7 @@ use rustc_span::source_map; use rustc_span::symbol::sym; use rustc_span::{Span, DUMMY_SP}; use rustc_target::abi::Abi; -use rustc_target::abi::{Integer, LayoutOf, TagEncoding, VariantIdx, Variants}; +use rustc_target::abi::{Integer, LayoutOf, TagEncoding, Variants}; use rustc_target::spec::abi::Abi as SpecAbi; use std::cmp; @@ -680,16 +679,11 @@ pub fn transparent_newtype_field<'a, 'tcx>( variant: &'a ty::VariantDef, ) -> Option<&'a ty::FieldDef> { let param_env = tcx.param_env(variant.def_id); - for field in &variant.fields { + variant.fields.iter().find(|field| { let field_ty = tcx.type_of(field.did); let is_zst = tcx.layout_of(param_env.and(field_ty)).map_or(false, |layout| layout.is_zst()); - - if !is_zst { - return Some(field); - } - } - - None + !is_zst + }) } /// Is type known to be non-null? @@ -712,15 +706,10 @@ fn ty_is_known_nonnull<'tcx>(cx: &LateContext<'tcx>, ty: Ty<'tcx>, mode: CItemKi return false; } - for variant in &def.variants { - if let Some(field) = transparent_newtype_field(cx.tcx, variant) { - if ty_is_known_nonnull(cx, field.ty(tcx, substs), mode) { - return true; - } - } - } - - false + def.variants + .iter() + .filter_map(|variant| transparent_newtype_field(cx.tcx, variant)) + .any(|field| ty_is_known_nonnull(cx, field.ty(tcx, substs), mode)) } _ => false, } @@ -783,25 +772,14 @@ crate fn repr_nullable_ptr<'tcx>( ) -> Option> { debug!("is_repr_nullable_ptr(cx, ty = {:?})", ty); if let ty::Adt(ty_def, substs) = ty.kind() { - if ty_def.variants.len() != 2 { - return None; - } - - let get_variant_fields = |index| &ty_def.variants[VariantIdx::new(index)].fields; - let variant_fields = [get_variant_fields(0), get_variant_fields(1)]; - let fields = if variant_fields[0].is_empty() { - &variant_fields[1] - } else if variant_fields[1].is_empty() { - &variant_fields[0] - } else { - return None; + let field_ty = match &ty_def.variants.raw[..] { + [var_one, var_two] => match (&var_one.fields[..], &var_two.fields[..]) { + ([], [field]) | ([field], []) => field.ty(cx.tcx, substs), + _ => return None, + }, + _ => return None, }; - if fields.len() != 1 { - return None; - } - - let field_ty = fields[0].ty(cx.tcx, substs); if !ty_is_known_nonnull(cx, field_ty, ckind) { return None; } @@ -921,11 +899,18 @@ impl<'a, 'tcx> ImproperCTypesVisitor<'a, 'tcx> { } match *ty.kind() { - ty::Adt(def, _) if def.is_box() && matches!(self.mode, CItemKind::Definition) => { - FfiSafe - } - ty::Adt(def, substs) => { + if def.is_box() && matches!(self.mode, CItemKind::Definition) { + if ty.boxed_ty().is_sized(tcx.at(DUMMY_SP), self.cx.param_env) { + return FfiSafe; + } else { + return FfiUnsafe { + ty, + reason: format!("box cannot be represented as a single pointer"), + help: None, + }; + } + } if def.is_phantom_data() { return FfiPhantom(ty); } diff --git a/compiler/rustc_lint/src/unused.rs b/compiler/rustc_lint/src/unused.rs index 67946dfb29..44c2a550c3 100644 --- a/compiler/rustc_lint/src/unused.rs +++ b/compiler/rustc_lint/src/unused.rs @@ -858,10 +858,10 @@ impl EarlyLintPass for UnusedParens { // The other cases do not contain sub-patterns. | Wild | Rest | Lit(..) | MacCall(..) | Range(..) | Ident(.., None) | Path(..) => {}, // These are list-like patterns; parens can always be removed. - TupleStruct(_, ps) | Tuple(ps) | Slice(ps) | Or(ps) => for p in ps { + TupleStruct(_, _, ps) | Tuple(ps) | Slice(ps) | Or(ps) => for p in ps { self.check_unused_parens_pat(cx, p, false, false); }, - Struct(_, fps, _) => for f in fps { + Struct(_, _, fps, _) => for f in fps { self.check_unused_parens_pat(cx, &f.pat, false, false); }, // Avoid linting on `i @ (p0 | .. | pn)` and `box (p0 | .. | pn)`, #64106. diff --git a/compiler/rustc_lint_defs/src/builtin.rs b/compiler/rustc_lint_defs/src/builtin.rs index 04e45e2351..352146d646 100644 --- a/compiler/rustc_lint_defs/src/builtin.rs +++ b/compiler/rustc_lint_defs/src/builtin.rs @@ -1618,7 +1618,11 @@ declare_lint! { /// [`impl Trait`]: https://doc.rust-lang.org/book/ch10-02-traits.html#traits-as-parameters pub BARE_TRAIT_OBJECTS, Warn, - "suggest using `dyn Trait` for trait objects" + "suggest using `dyn Trait` for trait objects", + @future_incompatible = FutureIncompatibleInfo { + reference: "issue #80165 ", + edition: Some(Edition::Edition2021), + }; } declare_lint! { @@ -2990,8 +2994,9 @@ declare_lint_pass! { USELESS_DEPRECATED, UNSUPPORTED_NAKED_FUNCTIONS, MISSING_ABI, + INVALID_DOC_ATTRIBUTES, SEMICOLON_IN_EXPRESSIONS_FROM_MACROS, - DISJOINT_CAPTURE_DROP_REORDER, + DISJOINT_CAPTURE_MIGRATION, LEGACY_DERIVE_HELPERS, PROC_MACRO_BACK_COMPAT, OR_PATTERNS_BACK_COMPAT, @@ -3023,14 +3028,17 @@ declare_lint! { } declare_lint! { - /// The `disjoint_capture_drop_reorder` lint detects variables that aren't completely + /// The `disjoint_capture_migration` lint detects variables that aren't completely /// captured when the feature `capture_disjoint_fields` is enabled and it affects the Drop /// order of at least one path starting at this variable. + /// It can also detect when a variable implements a trait, but one of its field does not and + /// the field is captured by a closure and used with the assumption that said field implements + /// the same trait as the root variable. /// - /// ### Example + /// ### Example of drop reorder /// /// ```rust,compile_fail - /// # #![deny(disjoint_capture_drop_reorder)] + /// # #![deny(disjoint_capture_migration)] /// # #![allow(unused)] /// struct FancyInteger(i32); /// @@ -3061,10 +3069,35 @@ declare_lint! { /// /// In the above example `p.y` will be dropped at the end of `f` instead of with `c` if /// the feature `capture_disjoint_fields` is enabled. - pub DISJOINT_CAPTURE_DROP_REORDER, + /// + /// ### Example of auto-trait + /// + /// ```rust,compile_fail + /// #![deny(disjoint_capture_migration)] + /// use std::thread; + /// + /// struct Pointer (*mut i32); + /// unsafe impl Send for Pointer {} + /// + /// fn main() { + /// let mut f = 10; + /// let fptr = Pointer(&mut f as *mut i32); + /// thread::spawn(move || unsafe { + /// *fptr.0 = 20; + /// }); + /// } + /// ``` + /// + /// {{produces}} + /// + /// ### Explanation + /// + /// In the above example `fptr.0` is captured when feature `capture_disjoint_fields` is enabled. + /// The field is of type *mut i32 which doesn't implement Send, making the code invalid as the + /// field cannot be sent between thread safely. + pub DISJOINT_CAPTURE_MIGRATION, Allow, - "Drop reorder because of `capture_disjoint_fields`" - + "Drop reorder and auto traits error because of `capture_disjoint_fields`" } declare_lint_pass!(UnusedDocComment => [UNUSED_DOC_COMMENTS]); diff --git a/compiler/rustc_lint_defs/src/lib.rs b/compiler/rustc_lint_defs/src/lib.rs index 70475563a4..f1c4e5fb4a 100644 --- a/compiler/rustc_lint_defs/src/lib.rs +++ b/compiler/rustc_lint_defs/src/lib.rs @@ -25,7 +25,11 @@ macro_rules! pluralize { /// before applying the suggestion. #[derive(Copy, Clone, Debug, PartialEq, Hash, Encodable, Decodable)] pub enum Applicability { - /// The suggestion is definitely what the user intended. This suggestion should be + /// The suggestion is definitely what the user intended, or maintains the exact meaning of the code. + /// This suggestion should be automatically applied. + /// + /// In case of multiple `MachineApplicable` suggestions (whether as part of + /// the same `multipart_suggestion` or not), all of them should be /// automatically applied. MachineApplicable, diff --git a/compiler/rustc_llvm/Cargo.toml b/compiler/rustc_llvm/Cargo.toml index e29af05328..3fca2e1ccb 100644 --- a/compiler/rustc_llvm/Cargo.toml +++ b/compiler/rustc_llvm/Cargo.toml @@ -13,4 +13,4 @@ libc = "0.2.73" [build-dependencies] build_helper = { path = "../../src/build_helper" } -cc = "1.0.60" +cc = "1.0.68" diff --git a/compiler/rustc_llvm/build.rs b/compiler/rustc_llvm/build.rs index 301ed639f3..452d1b19a1 100644 --- a/compiler/rustc_llvm/build.rs +++ b/compiler/rustc_llvm/build.rs @@ -86,6 +86,7 @@ fn main() { "nvptx", "hexagon", "riscv", + "bpf", ]; let required_components = &[ diff --git a/compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp b/compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp index 617b2ed970..3595e37cf9 100644 --- a/compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp +++ b/compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp @@ -32,6 +32,8 @@ #include "llvm/Transforms/Instrumentation.h" #include "llvm/Transforms/Instrumentation/AddressSanitizer.h" #include "llvm/Support/TimeProfiler.h" +#include "llvm/Transforms/Instrumentation/GCOVProfiler.h" +#include "llvm/Transforms/Instrumentation/InstrProfiling.h" #include "llvm/Transforms/Instrumentation/ThreadSanitizer.h" #include "llvm/Transforms/Instrumentation/MemorySanitizer.h" #include "llvm/Transforms/Instrumentation/HWAddressSanitizer.h" @@ -609,7 +611,7 @@ LLVMRustWriteOutputFile(LLVMTargetMachineRef Target, LLVMPassManagerRef PMR, std::string ErrorInfo; std::error_code EC; - raw_fd_ostream OS(Path, EC, sys::fs::F_None); + raw_fd_ostream OS(Path, EC, sys::fs::OF_None); if (EC) ErrorInfo = EC.message(); if (ErrorInfo != "") { @@ -619,7 +621,7 @@ LLVMRustWriteOutputFile(LLVMTargetMachineRef Target, LLVMPassManagerRef PMR, buffer_ostream BOS(OS); if (DwoPath) { - raw_fd_ostream DOS(DwoPath, EC, sys::fs::F_None); + raw_fd_ostream DOS(DwoPath, EC, sys::fs::OF_None); EC.clear(); if (EC) ErrorInfo = EC.message(); @@ -734,7 +736,7 @@ struct LLVMRustSanitizerOptions { bool SanitizeHWAddressRecover; }; -extern "C" void +extern "C" LLVMRustResult LLVMRustOptimizeWithNewPassManager( LLVMModuleRef ModuleRef, LLVMTargetMachineRef TMRef, @@ -745,9 +747,11 @@ LLVMRustOptimizeWithNewPassManager( bool DisableSimplifyLibCalls, bool EmitLifetimeMarkers, LLVMRustSanitizerOptions *SanitizerOptions, const char *PGOGenPath, const char *PGOUsePath, + bool InstrumentCoverage, bool InstrumentGCOV, void* LlvmSelfProfiler, LLVMRustSelfProfileBeforePassCallback BeforePassCallback, - LLVMRustSelfProfileAfterPassCallback AfterPassCallback) { + LLVMRustSelfProfileAfterPassCallback AfterPassCallback, + const char *ExtraPasses, size_t ExtraPassesLen) { Module *TheModule = unwrap(ModuleRef); TargetMachine *TM = unwrap(TMRef); PassBuilder::OptimizationLevel OptLevel = fromRust(OptLevelRust); @@ -789,16 +793,23 @@ LLVMRustOptimizeWithNewPassManager( PGOOpt = PGOOptions(PGOUsePath, "", "", PGOOptions::IRUse); } -#if LLVM_VERSION_GE(12, 0) +#if LLVM_VERSION_GE(12, 0) && !LLVM_VERSION_GE(13,0) PassBuilder PB(DebugPassManager, TM, PTO, PGOOpt, &PIC); #else PassBuilder PB(TM, PTO, PGOOpt, &PIC); #endif +#if LLVM_VERSION_GE(13, 0) + LoopAnalysisManager LAM; + FunctionAnalysisManager FAM; + CGSCCAnalysisManager CGAM; + ModuleAnalysisManager MAM; +#else LoopAnalysisManager LAM(DebugPassManager); FunctionAnalysisManager FAM(DebugPassManager); CGSCCAnalysisManager CGAM(DebugPassManager); ModuleAnalysisManager MAM(DebugPassManager); +#endif FAM.registerPass([&] { return PB.buildDefaultAAPipeline(); }); @@ -834,6 +845,23 @@ LLVMRustOptimizeWithNewPassManager( ); } + if (InstrumentGCOV) { + PipelineStartEPCallbacks.push_back( + [](ModulePassManager &MPM, PassBuilder::OptimizationLevel Level) { + MPM.addPass(GCOVProfilerPass(GCOVOptions::getDefault())); + } + ); + } + + if (InstrumentCoverage) { + PipelineStartEPCallbacks.push_back( + [](ModulePassManager &MPM, PassBuilder::OptimizationLevel Level) { + InstrProfOptions Options; + MPM.addPass(InstrProfiling(Options, false)); + } + ); + } + if (SanitizerOptions) { if (SanitizerOptions->SanitizeMemory) { MemorySanitizerOptions Options( @@ -935,7 +963,11 @@ LLVMRustOptimizeWithNewPassManager( } } +#if LLVM_VERSION_GE(13, 0) + ModulePassManager MPM; +#else ModulePassManager MPM(DebugPassManager); +#endif bool NeedThinLTOBufferPasses = UseThinLTOBuffers; if (!NoPrepopulatePasses) { if (OptLevel == PassBuilder::OptimizationLevel::O0) { @@ -1042,6 +1074,14 @@ LLVMRustOptimizeWithNewPassManager( } } + if (ExtraPassesLen) { + if (auto Err = PB.parsePassPipeline(MPM, StringRef(ExtraPasses, ExtraPassesLen))) { + std::string ErrMsg = toString(std::move(Err)); + LLVMRustSetLastError(ErrMsg.c_str()); + return LLVMRustResult::Failure; + } + } + if (NeedThinLTOBufferPasses) { MPM.addPass(CanonicalizeAliasesPass()); MPM.addPass(NameAnonGlobalPass()); @@ -1052,6 +1092,7 @@ LLVMRustOptimizeWithNewPassManager( UpgradeCallsToIntrinsic(&*I++); // must be post-increment, as we remove MPM.run(*TheModule, MAM); + return LLVMRustResult::Success; } // Callback to demangle function name @@ -1146,7 +1187,7 @@ extern "C" LLVMRustResult LLVMRustPrintModule(LLVMModuleRef M, const char *Path, DemangleFn Demangle) { std::string ErrorInfo; std::error_code EC; - raw_fd_ostream OS(Path, EC, sys::fs::F_None); + raw_fd_ostream OS(Path, EC, sys::fs::OF_None); if (EC) ErrorInfo = EC.message(); if (ErrorInfo != "") { diff --git a/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp b/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp index 2e135fbe2b..9b757eb40c 100644 --- a/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp +++ b/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp @@ -6,6 +6,7 @@ #include "llvm/IR/Instructions.h" #include "llvm/IR/Intrinsics.h" #include "llvm/Object/Archive.h" +#include "llvm/Object/COFFImportFile.h" #include "llvm/Object/ObjectFile.h" #include "llvm/Bitcode/BitcodeWriterPass.h" #include "llvm/Support/Signals.h" @@ -70,17 +71,6 @@ extern "C" void LLVMRustInstallFatalErrorHandler() { install_fatal_error_handler(FatalErrorHandler); } -extern "C" LLVMMemoryBufferRef -LLVMRustCreateMemoryBufferWithContentsOfFile(const char *Path) { - ErrorOr> BufOr = - MemoryBuffer::getFile(Path, -1, false); - if (!BufOr) { - LLVMRustSetLastError(BufOr.getError().message().c_str()); - return nullptr; - } - return wrap(BufOr.get().release()); -} - extern "C" char *LLVMRustGetLastError(void) { char *Ret = LastError; LastError = nullptr; @@ -1077,30 +1067,6 @@ extern "C" void LLVMRustWriteValueToString(LLVMValueRef V, } } -// Note that the two following functions look quite similar to the -// LLVMGetSectionName function. Sadly, it appears that this function only -// returns a char* pointer, which isn't guaranteed to be null-terminated. The -// function provided by LLVM doesn't return the length, so we've created our own -// function which returns the length as well as the data pointer. -// -// For an example of this not returning a null terminated string, see -// lib/Object/COFFObjectFile.cpp in the getSectionName function. One of the -// branches explicitly creates a StringRef without a null terminator, and then -// that's returned. - -inline section_iterator *unwrap(LLVMSectionIteratorRef SI) { - return reinterpret_cast(SI); -} - -extern "C" size_t LLVMRustGetSectionName(LLVMSectionIteratorRef SI, - const char **Ptr) { - auto NameOrErr = (*unwrap(SI))->getName(); - if (!NameOrErr) - report_fatal_error(NameOrErr.takeError()); - *Ptr = NameOrErr->data(); - return NameOrErr->size(); -} - // LLVMArrayType function does not support 64-bit ElementCount extern "C" LLVMTypeRef LLVMRustArrayType(LLVMTypeRef ElementTy, uint64_t ElementCount) { @@ -1757,3 +1723,54 @@ extern "C" LLVMValueRef LLVMRustBuildMaxNum(LLVMBuilderRef B, LLVMValueRef LHS, LLVMValueRef RHS) { return wrap(unwrap(B)->CreateMaxNum(unwrap(LHS),unwrap(RHS))); } + +// This struct contains all necessary info about a symbol exported from a DLL. +// At the moment, it's just the symbol's name, but we use a separate struct to +// make it easier to add other information like ordinal later. +struct LLVMRustCOFFShortExport { + const char* name; +}; + +// Machine must be a COFF machine type, as defined in PE specs. +extern "C" LLVMRustResult LLVMRustWriteImportLibrary( + const char* ImportName, + const char* Path, + const LLVMRustCOFFShortExport* Exports, + size_t NumExports, + uint16_t Machine, + bool MinGW) +{ + std::vector ConvertedExports; + ConvertedExports.reserve(NumExports); + + for (size_t i = 0; i < NumExports; ++i) { + ConvertedExports.push_back(llvm::object::COFFShortExport{ + Exports[i].name, // Name + std::string{}, // ExtName + std::string{}, // SymbolName + std::string{}, // AliasTarget + 0, // Ordinal + false, // Noname + false, // Data + false, // Private + false // Constant + }); + } + + auto Error = llvm::object::writeImportLibrary( + ImportName, + Path, + ConvertedExports, + static_cast(Machine), + MinGW); + if (Error) { + std::string errorString; + llvm::raw_string_ostream stream(errorString); + stream << Error; + stream.flush(); + LLVMRustSetLastError(errorString.c_str()); + return LLVMRustResult::Failure; + } else { + return LLVMRustResult::Success; + } +} diff --git a/compiler/rustc_llvm/src/lib.rs b/compiler/rustc_llvm/src/lib.rs index 555aefb192..122627eb50 100644 --- a/compiler/rustc_llvm/src/lib.rs +++ b/compiler/rustc_llvm/src/lib.rs @@ -167,4 +167,12 @@ pub fn initialize_available_targets() { LLVMInitializeWebAssemblyAsmPrinter, LLVMInitializeWebAssemblyAsmParser ); + init_target!( + llvm_component = "bpf", + LLVMInitializeBPFTargetInfo, + LLVMInitializeBPFTarget, + LLVMInitializeBPFTargetMC, + LLVMInitializeBPFAsmPrinter, + LLVMInitializeBPFAsmParser + ); } diff --git a/compiler/rustc_macros/src/serialize.rs b/compiler/rustc_macros/src/serialize.rs index 72bd4804e9..7bc669f2b0 100644 --- a/compiler/rustc_macros/src/serialize.rs +++ b/compiler/rustc_macros/src/serialize.rs @@ -43,12 +43,9 @@ fn decodable_body( let decode_body = match s.variants() { [vi] => { let construct = vi.construct(|field, index| decode_field(field, index, true)); - let n_fields = vi.ast().fields.len(); quote! { ::rustc_serialize::Decoder::read_struct( __decoder, - #ty_name, - #n_fields, |__decoder| { ::std::result::Result::Ok(#construct) }, ) } @@ -77,7 +74,6 @@ fn decodable_body( quote! { ::rustc_serialize::Decoder::read_enum( __decoder, - #ty_name, |__decoder| { ::rustc_serialize::Decoder::read_enum_variant( __decoder, @@ -128,7 +124,7 @@ fn decode_field(field: &syn::Field, index: usize, is_struct: bool) -> proc_macro quote! { match ::rustc_serialize::Decoder::#decode_method( - __decoder, #opt_field_name #index, #decode_inner_method) { + __decoder, #opt_field_name #decode_inner_method) { ::std::result::Result::Ok(__res) => __res, ::std::result::Result::Err(__err) => return ::std::result::Result::Err(__err), } @@ -183,7 +179,6 @@ fn encodable_body( } }); - let ty_name = s.ast().ident.to_string(); let encode_body = match s.variants() { [_] => { let mut field_idx = 0usize; @@ -197,11 +192,12 @@ fn encodable_body( .ident .as_ref() .map_or_else(|| field_idx.to_string(), |i| i.to_string()); + let first = field_idx == 0; let result = quote! { match ::rustc_serialize::Encoder::emit_struct_field( __encoder, #field_name, - #field_idx, + #first, |__encoder| ::rustc_serialize::Encodable::<#encoder_ty>::encode(#bind_ident, __encoder), ) { @@ -215,8 +211,9 @@ fn encodable_body( }) .collect::() }); + let no_fields = field_idx == 0; quote! { - ::rustc_serialize::Encoder::emit_struct(__encoder, #ty_name, #field_idx, |__encoder| { + ::rustc_serialize::Encoder::emit_struct(__encoder, #no_fields, |__encoder| { ::std::result::Result::Ok(match *self { #encode_inner }) }) } @@ -232,10 +229,11 @@ fn encodable_body( .iter() .map(|binding| { let bind_ident = &binding.binding; + let first = field_idx == 0; let result = quote! { match ::rustc_serialize::Encoder::emit_enum_variant_arg( __encoder, - #field_idx, + #first, |__encoder| ::rustc_serialize::Encodable::<#encoder_ty>::encode(#bind_ident, __encoder), ) { @@ -260,7 +258,7 @@ fn encodable_body( result }); quote! { - ::rustc_serialize::Encoder::emit_enum(__encoder, #ty_name, |__encoder| { + ::rustc_serialize::Encoder::emit_enum(__encoder, |__encoder| { match *self { #encode_inner } diff --git a/compiler/rustc_metadata/src/creader.rs b/compiler/rustc_metadata/src/creader.rs index e9ae22f8ce..d73cfe35dc 100644 --- a/compiler/rustc_metadata/src/creader.rs +++ b/compiler/rustc_metadata/src/creader.rs @@ -54,7 +54,7 @@ pub struct CStore { pub struct CrateLoader<'a> { // Immutable configuration. sess: &'a Session, - metadata_loader: &'a MetadataLoaderDyn, + metadata_loader: Box, local_crate_name: Symbol, // Mutable output. cstore: CStore, @@ -219,7 +219,7 @@ impl CStore { impl<'a> CrateLoader<'a> { pub fn new( sess: &'a Session, - metadata_loader: &'a MetadataLoaderDyn, + metadata_loader: Box, local_crate_name: &str, ) -> Self { let local_crate_stable_id = @@ -544,7 +544,7 @@ impl<'a> CrateLoader<'a> { info!("falling back to a load"); let mut locator = CrateLocator::new( self.sess, - self.metadata_loader, + &*self.metadata_loader, name, hash, host_hash, diff --git a/compiler/rustc_metadata/src/dynamic_lib.rs b/compiler/rustc_metadata/src/dynamic_lib.rs index bdb53e3f75..1a900ccbf6 100644 --- a/compiler/rustc_metadata/src/dynamic_lib.rs +++ b/compiler/rustc_metadata/src/dynamic_lib.rs @@ -105,7 +105,7 @@ mod dl { return Ok(ret.cast()); } - // A NULL return from `dlopen` indicates that an error has definitely occurred, so if + // A null return from `dlopen` indicates that an error has definitely occurred, so if // nothing is in `dlerror`, we are racing with another thread that has stolen our error // message. See the explanation on the `dl::error` module for more information. dlerror.get().and_then(|()| Err("Unknown error".to_string())) @@ -117,7 +117,7 @@ mod dl { ) -> Result<*mut u8, String> { let mut dlerror = error::lock(); - // Unlike `dlopen`, it's possible for `dlsym` to return NULL without overwriting `dlerror`. + // Unlike `dlopen`, it's possible for `dlsym` to return null without overwriting `dlerror`. // Because of this, we clear `dlerror` before calling `dlsym` to avoid picking up a stale // error message by accident. dlerror.clear(); @@ -128,7 +128,7 @@ mod dl { return Ok(ret.cast()); } - // If `dlsym` returns NULL but there is nothing in `dlerror` it means one of two things: + // If `dlsym` returns null but there is nothing in `dlerror` it means one of two things: // - We tried to load a symbol mapped to address 0. This is not technically an error but is // unlikely to occur in practice and equally unlikely to be handled correctly by calling // code. Therefore we treat it as an error anyway. diff --git a/compiler/rustc_metadata/src/lib.rs b/compiler/rustc_metadata/src/lib.rs index c4d9e3f77f..2c9bad7e5c 100644 --- a/compiler/rustc_metadata/src/lib.rs +++ b/compiler/rustc_metadata/src/lib.rs @@ -1,14 +1,11 @@ #![doc(html_root_url = "https://doc.rust-lang.org/nightly/nightly-rustc/")] -#![feature(core_intrinsics)] #![feature(crate_visibility_modifier)] #![feature(drain_filter)] #![feature(in_band_lifetimes)] #![feature(nll)] #![feature(once_cell)] -#![cfg_attr(bootstrap, feature(or_patterns))] #![feature(proc_macro_internals)] #![feature(min_specialization)] -#![feature(stmt_expr_attributes)] #![feature(try_blocks)] #![feature(never_type)] #![recursion_limit = "256"] @@ -32,3 +29,5 @@ mod rmeta; pub mod creader; pub mod dynamic_lib; pub mod locator; + +pub use rmeta::METADATA_HEADER; diff --git a/compiler/rustc_metadata/src/native_libs.rs b/compiler/rustc_metadata/src/native_libs.rs index 523e016eeb..cd4c394ae1 100644 --- a/compiler/rustc_metadata/src/native_libs.rs +++ b/compiler/rustc_metadata/src/native_libs.rs @@ -3,13 +3,13 @@ use rustc_data_structures::fx::FxHashSet; use rustc_errors::struct_span_err; use rustc_hir as hir; use rustc_hir::itemlikevisit::ItemLikeVisitor; -use rustc_middle::middle::cstore::NativeLib; +use rustc_middle::middle::cstore::{DllImport, NativeLib}; use rustc_middle::ty::TyCtxt; use rustc_session::parse::feature_err; use rustc_session::utils::NativeLibKind; use rustc_session::Session; -use rustc_span::source_map::Span; use rustc_span::symbol::{kw, sym, Symbol}; +use rustc_span::Span; use rustc_target::spec::abi::Abi; crate fn collect(tcx: TyCtxt<'_>) -> Vec { @@ -33,8 +33,8 @@ struct Collector<'tcx> { impl ItemLikeVisitor<'tcx> for Collector<'tcx> { fn visit_item(&mut self, it: &'tcx hir::Item<'tcx>) { - let abi = match it.kind { - hir::ItemKind::ForeignMod { abi, .. } => abi, + let (abi, foreign_mod_items) = match it.kind { + hir::ItemKind::ForeignMod { abi, items } => (abi, items), _ => return, }; @@ -56,6 +56,8 @@ impl ItemLikeVisitor<'tcx> for Collector<'tcx> { cfg: None, foreign_module: Some(it.def_id.to_def_id()), wasm_import_module: None, + verbatim: None, + dll_imports: Vec::new(), }; let mut kind_specified = false; @@ -67,10 +69,18 @@ impl ItemLikeVisitor<'tcx> for Collector<'tcx> { None => continue, // skip like historical compilers }; lib.kind = match &*kind.as_str() { - "static" => NativeLibKind::StaticBundle, - "static-nobundle" => NativeLibKind::StaticNoBundle, - "dylib" => NativeLibKind::Dylib, - "framework" => NativeLibKind::Framework, + "static" => NativeLibKind::Static { bundle: None, whole_archive: None }, + "static-nobundle" => { + sess.struct_span_warn( + item.span(), + "library kind `static-nobundle` has been superseded by specifying \ + modifier `-bundle` with library kind `static`", + ) + .emit(); + NativeLibKind::Static { bundle: Some(false), whole_archive: None } + } + "dylib" => NativeLibKind::Dylib { as_needed: None }, + "framework" => NativeLibKind::Framework { as_needed: None }, "raw-dylib" => NativeLibKind::RawDylib, k => { struct_span_err!(sess, item.span(), E0458, "unknown kind: `{}`", k) @@ -108,6 +118,71 @@ impl ItemLikeVisitor<'tcx> for Collector<'tcx> { } } + // Do this outside the above loop so we don't depend on modifiers coming + // after kinds + if let Some(item) = items.iter().find(|item| item.has_name(sym::modifiers)) { + if let Some(modifiers) = item.value_str() { + let span = item.name_value_literal_span().unwrap(); + for modifier in modifiers.as_str().split(',') { + let (modifier, value) = match modifier.strip_prefix(&['+', '-'][..]) { + Some(m) => (m, modifier.starts_with('+')), + None => { + sess.span_err( + span, + "invalid linking modifier syntax, expected '+' or '-' prefix \ + before one of: bundle, verbatim, whole-archive, as-needed", + ); + continue; + } + }; + + match (modifier, &mut lib.kind) { + ("bundle", NativeLibKind::Static { bundle, .. }) => { + *bundle = Some(value); + } + ("bundle", _) => sess.span_err( + span, + "bundle linking modifier is only compatible with \ + `static` linking kind", + ), + + ("verbatim", _) => lib.verbatim = Some(value), + + ("whole-archive", NativeLibKind::Static { whole_archive, .. }) => { + *whole_archive = Some(value); + } + ("whole-archive", _) => sess.span_err( + span, + "whole-archive linking modifier is only compatible with \ + `static` linking kind", + ), + + ("as-needed", NativeLibKind::Dylib { as_needed }) + | ("as-needed", NativeLibKind::Framework { as_needed }) => { + *as_needed = Some(value); + } + ("as-needed", _) => sess.span_err( + span, + "as-needed linking modifier is only compatible with \ + `dylib` and `framework` linking kinds", + ), + + _ => sess.span_err( + span, + &format!( + "unrecognized linking modifier `{}`, expected one \ + of: bundle, verbatim, whole-archive, as-needed", + modifier + ), + ), + } + } + } else { + let msg = "must be of the form `#[link(modifiers = \"...\")]`"; + sess.span_err(item.span(), msg); + } + } + // In general we require #[link(name = "...")] but we allow // #[link(wasm_import_module = "...")] without the `name`. let requires_name = kind_specified || lib.wasm_import_module.is_none(); @@ -122,6 +197,27 @@ impl ItemLikeVisitor<'tcx> for Collector<'tcx> { .span_label(m.span, "missing `name` argument") .emit(); } + + if lib.kind == NativeLibKind::RawDylib { + match abi { + Abi::C { .. } => (), + Abi::Cdecl => (), + _ => { + if sess.target.arch == "x86" { + sess.span_fatal( + it.span, + r#"`#[link(kind = "raw-dylib")]` only supports C and Cdecl ABIs"#, + ); + } + } + }; + lib.dll_imports.extend( + foreign_mod_items + .iter() + .map(|child_item| DllImport { name: child_item.ident.name, ordinal: None }), + ); + } + self.register_native_lib(Some(m.span), lib); } } @@ -152,7 +248,7 @@ impl Collector<'tcx> { return; } let is_osx = self.tcx.sess.target.is_like_osx; - if lib.kind == NativeLibKind::Framework && !is_osx { + if matches!(lib.kind, NativeLibKind::Framework { .. }) && !is_osx { let msg = "native frameworks are only available on macOS targets"; match span { Some(span) => struct_span_err!(self.tcx.sess, span, E0455, "{}", msg).emit(), @@ -168,7 +264,9 @@ impl Collector<'tcx> { ) .emit(); } - if lib.kind == NativeLibKind::StaticNoBundle && !self.tcx.features().static_nobundle { + if matches!(lib.kind, NativeLibKind::Static { bundle: Some(false), .. }) + && !self.tcx.features().static_nobundle + { feature_err( &self.tcx.sess.parse_sess, sym::static_nobundle, @@ -177,15 +275,42 @@ impl Collector<'tcx> { ) .emit(); } - if lib.kind == NativeLibKind::RawDylib && !self.tcx.features().raw_dylib { - feature_err( - &self.tcx.sess.parse_sess, - sym::raw_dylib, - span.unwrap_or(rustc_span::DUMMY_SP), - "kind=\"raw-dylib\" is unstable", - ) - .emit(); + // this just unwraps lib.name; we already established that it isn't empty above. + if let (NativeLibKind::RawDylib, Some(lib_name)) = (lib.kind, lib.name) { + let span = match span { + Some(s) => s, + None => { + bug!("raw-dylib libraries are not supported on the command line"); + } + }; + + if !self.tcx.sess.target.options.is_like_windows { + self.tcx.sess.span_fatal( + span, + "`#[link(...)]` with `kind = \"raw-dylib\"` only supported on Windows", + ); + } else if !self.tcx.sess.target.options.is_like_msvc { + self.tcx.sess.span_warn( + span, + "`#[link(...)]` with `kind = \"raw-dylib\"` not supported on windows-gnu", + ); + } + + if lib_name.as_str().contains('\0') { + self.tcx.sess.span_err(span, "library name may not contain NUL characters"); + } + + if !self.tcx.features().raw_dylib { + feature_err( + &self.tcx.sess.parse_sess, + sym::raw_dylib, + span, + "kind=\"raw-dylib\" is unstable", + ) + .emit(); + } } + self.libs.push(lib); } @@ -193,30 +318,30 @@ impl Collector<'tcx> { fn process_command_line(&mut self) { // First, check for errors let mut renames = FxHashSet::default(); - for (name, new_name, _) in &self.tcx.sess.opts.libs { - if let Some(ref new_name) = new_name { + for lib in &self.tcx.sess.opts.libs { + if let Some(ref new_name) = lib.new_name { let any_duplicate = self .libs .iter() .filter_map(|lib| lib.name.as_ref()) - .any(|n| &n.as_str() == name); + .any(|n| &n.as_str() == &lib.name); if new_name.is_empty() { self.tcx.sess.err(&format!( "an empty renaming target was specified for library `{}`", - name + lib.name )); } else if !any_duplicate { self.tcx.sess.err(&format!( "renaming of the library `{}` was specified, \ however this crate contains no `#[link(...)]` \ attributes referencing this library.", - name + lib.name )); - } else if !renames.insert(name) { + } else if !renames.insert(&lib.name) { self.tcx.sess.err(&format!( "multiple renamings were \ specified for library `{}` .", - name + lib.name )); } } @@ -229,7 +354,7 @@ impl Collector<'tcx> { // it. (This ensures that the linker is able to see symbols from // all possible dependent libraries before linking in the library // in question.) - for &(ref name, ref new_name, kind) in &self.tcx.sess.opts.libs { + for passed_lib in &self.tcx.sess.opts.libs { // If we've already added any native libraries with the same // name, they will be pulled out into `existing`, so that we // can move them to the end of the list below. @@ -237,13 +362,14 @@ impl Collector<'tcx> { .libs .drain_filter(|lib| { if let Some(lib_name) = lib.name { - if lib_name.as_str() == *name { - if kind != NativeLibKind::Unspecified { - lib.kind = kind; + if lib_name.as_str() == passed_lib.name { + if passed_lib.kind != NativeLibKind::Unspecified { + lib.kind = passed_lib.kind; } - if let Some(new_name) = new_name { + if let Some(new_name) = &passed_lib.new_name { lib.name = Some(Symbol::intern(new_name)); } + lib.verbatim = passed_lib.verbatim; return true; } } @@ -252,13 +378,15 @@ impl Collector<'tcx> { .collect::>(); if existing.is_empty() { // Add if not found - let new_name = new_name.as_ref().map(|s| &**s); // &Option -> Option<&str> + let new_name = passed_lib.new_name.as_ref().map(|s| &**s); // &Option -> Option<&str> let lib = NativeLib { - name: Some(Symbol::intern(new_name.unwrap_or(name))), - kind, + name: Some(Symbol::intern(new_name.unwrap_or(&passed_lib.name))), + kind: passed_lib.kind, cfg: None, foreign_module: None, wasm_import_module: None, + verbatim: passed_lib.verbatim, + dll_imports: Vec::new(), }; self.register_native_lib(None, lib); } else { diff --git a/compiler/rustc_metadata/src/rmeta/decoder.rs b/compiler/rustc_metadata/src/rmeta/decoder.rs index 6671cf6fea..48900fecd3 100644 --- a/compiler/rustc_metadata/src/rmeta/decoder.rs +++ b/compiler/rustc_metadata/src/rmeta/decoder.rs @@ -27,7 +27,7 @@ use rustc_middle::middle::exported_symbols::{ExportedSymbol, SymbolExportLevel}; use rustc_middle::mir::interpret::{AllocDecodingSession, AllocDecodingState}; use rustc_middle::mir::{self, Body, Promoted}; use rustc_middle::ty::codec::TyDecoder; -use rustc_middle::ty::{self, Ty, TyCtxt}; +use rustc_middle::ty::{self, Ty, TyCtxt, Visibility}; use rustc_serialize::{opaque, Decodable, Decoder}; use rustc_session::Session; use rustc_span::hygiene::ExpnDataDecodeMode; @@ -252,6 +252,10 @@ impl<'a, 'tcx> DecodeContext<'a, 'tcx> { self.cdata.expect("missing CrateMetadata in DecodeContext") } + fn map_encoded_cnum_to_current(&self, cnum: CrateNum) -> CrateNum { + if cnum == LOCAL_CRATE { self.cdata().cnum } else { self.cdata().cnum_map[cnum] } + } + fn read_lazy_with_meta( &mut self, meta: T::Meta, @@ -300,7 +304,7 @@ impl<'a, 'tcx> TyDecoder<'tcx> for DecodeContext<'a, 'tcx> { { let tcx = self.tcx(); - let key = ty::CReaderCacheKey { cnum: self.cdata().cnum, pos: shorthand }; + let key = ty::CReaderCacheKey { cnum: Some(self.cdata().cnum), pos: shorthand }; if let Some(&ty) = tcx.ty_rcache.borrow().get(&key) { return Ok(ty); @@ -324,10 +328,6 @@ impl<'a, 'tcx> TyDecoder<'tcx> for DecodeContext<'a, 'tcx> { r } - fn map_encoded_cnum_to_current(&self, cnum: CrateNum) -> CrateNum { - if cnum == LOCAL_CRATE { self.cdata().cnum } else { self.cdata().cnum_map[cnum] } - } - fn decode_alloc_id(&mut self) -> Result { if let Some(alloc_decoding_session) = self.alloc_decoding_session { alloc_decoding_session.decode_alloc_id(self) @@ -716,30 +716,37 @@ impl<'a, 'tcx> CrateMetadataRef<'a> { .decode((self, sess)) } - fn load_proc_macro(&self, id: DefIndex, sess: &Session) -> SyntaxExtension { - let (name, kind, helper_attrs) = match *self.raw_proc_macro(id) { + fn load_proc_macro(&self, def_id: DefId, sess: &Session) -> SyntaxExtension { + let (name, kind, helper_attrs) = match *self.raw_proc_macro(def_id.index) { ProcMacro::CustomDerive { trait_name, attributes, client } => { let helper_attrs = attributes.iter().cloned().map(Symbol::intern).collect::>(); ( trait_name, - SyntaxExtensionKind::Derive(Box::new(ProcMacroDerive { client })), + SyntaxExtensionKind::Derive(Box::new(ProcMacroDerive { + client, + krate: def_id.krate, + })), helper_attrs, ) } - ProcMacro::Attr { name, client } => { - (name, SyntaxExtensionKind::Attr(Box::new(AttrProcMacro { client })), Vec::new()) - } - ProcMacro::Bang { name, client } => { - (name, SyntaxExtensionKind::Bang(Box::new(BangProcMacro { client })), Vec::new()) - } + ProcMacro::Attr { name, client } => ( + name, + SyntaxExtensionKind::Attr(Box::new(AttrProcMacro { client, krate: def_id.krate })), + Vec::new(), + ), + ProcMacro::Bang { name, client } => ( + name, + SyntaxExtensionKind::Bang(Box::new(BangProcMacro { client, krate: def_id.krate })), + Vec::new(), + ), }; - let attrs: Vec<_> = self.get_item_attrs(id, sess).collect(); + let attrs: Vec<_> = self.get_item_attrs(def_id.index, sess).collect(); SyntaxExtension::new( sess, kind, - self.get_span(id, sess), + self.get_span(def_id.index, sess), helper_attrs, self.root.edition, Symbol::intern(name), @@ -1305,6 +1312,17 @@ impl<'a, 'tcx> CrateMetadataRef<'a> { .collect() } + fn get_struct_field_visibilities(&self, id: DefIndex) -> Vec { + self.root + .tables + .children + .get(self, id) + .unwrap_or_else(Lazy::empty) + .decode(self) + .map(|field_index| self.get_visibility(field_index)) + .collect() + } + fn get_inherent_implementations_for_type( &self, tcx: TyCtxt<'tcx>, @@ -1379,6 +1397,15 @@ impl<'a, 'tcx> CrateMetadataRef<'a> { } } + fn get_proc_macro_quoted_span(&self, index: usize, sess: &Session) -> Span { + self.root + .tables + .proc_macro_quoted_spans + .get(self, index) + .unwrap_or_else(|| panic!("Missing proc macro quoted span: {:?}", index)) + .decode((self, sess)) + } + fn get_foreign_modules(&self, tcx: TyCtxt<'tcx>) -> Lrc> { if self.root.is_proc_macro_crate() { // Proc macro crates do not have any *target* foreign modules. @@ -1635,9 +1662,11 @@ impl<'a, 'tcx> CrateMetadataRef<'a> { if let Some(virtual_dir) = virtual_rust_source_base_dir { if let Some(real_dir) = &sess.opts.real_rust_source_base_dir { if let rustc_span::FileName::Real(old_name) = name { - if let rustc_span::RealFileName::Named(one_path) = old_name { - if let Ok(rest) = one_path.strip_prefix(virtual_dir) { - let virtual_name = one_path.clone(); + if let rustc_span::RealFileName::Remapped { local_path: _, virtual_name } = + old_name + { + if let Ok(rest) = virtual_name.strip_prefix(virtual_dir) { + let virtual_name = virtual_name.clone(); // The std library crates are in // `$sysroot/lib/rustlib/src/rust/library`, whereas other crates @@ -1673,8 +1702,8 @@ impl<'a, 'tcx> CrateMetadataRef<'a> { virtual_name.display(), new_path.display(), ); - let new_name = rustc_span::RealFileName::Devirtualized { - local_path: new_path, + let new_name = rustc_span::RealFileName::Remapped { + local_path: Some(new_path), virtual_name, }; *old_name = new_name; @@ -1694,7 +1723,6 @@ impl<'a, 'tcx> CrateMetadataRef<'a> { // containing the information we need. let rustc_span::SourceFile { mut name, - name_was_remapped, src_hash, start_pos, end_pos, @@ -1706,11 +1734,34 @@ impl<'a, 'tcx> CrateMetadataRef<'a> { .. } = source_file_to_import; + // If this file is under $sysroot/lib/rustlib/src/ but has not been remapped + // during rust bootstrapping by `remap-debuginfo = true`, and the user + // wish to simulate that behaviour by -Z simulate-remapped-rust-src-base, + // then we change `name` to a similar state as if the rust was bootstrapped + // with `remap-debuginfo = true`. + // This is useful for testing so that tests about the effects of + // `try_to_translate_virtual_to_real` don't have to worry about how the + // compiler is bootstrapped. + if let Some(virtual_dir) = + &sess.opts.debugging_opts.simulate_remapped_rust_src_base + { + if let Some(real_dir) = &sess.opts.real_rust_source_base_dir { + if let rustc_span::FileName::Real(ref mut old_name) = name { + if let rustc_span::RealFileName::LocalPath(local) = old_name { + if let Ok(rest) = local.strip_prefix(real_dir) { + *old_name = rustc_span::RealFileName::Remapped { + local_path: None, + virtual_name: virtual_dir.join(rest), + }; + } + } + } + } + } + // If this file's path has been remapped to `/rustc/$hash`, // we might be able to reverse that (also see comments above, // on `try_to_translate_virtual_to_real`). - // FIXME(eddyb) we could check `name_was_remapped` here, - // but in practice it seems to be always `false`. try_to_translate_virtual_to_real(&mut name); let source_length = (end_pos - start_pos).to_usize(); @@ -1735,7 +1786,6 @@ impl<'a, 'tcx> CrateMetadataRef<'a> { let local_version = sess.source_map().new_imported_source_file( name, - name_was_remapped, src_hash, name_hash, source_length, diff --git a/compiler/rustc_metadata/src/rmeta/decoder/cstore_impl.rs b/compiler/rustc_metadata/src/rmeta/decoder/cstore_impl.rs index bebee9dac3..9a97835d9c 100644 --- a/compiler/rustc_metadata/src/rmeta/decoder/cstore_impl.rs +++ b/compiler/rustc_metadata/src/rmeta/decoder/cstore_impl.rs @@ -1,14 +1,14 @@ use crate::creader::{CStore, LoadedMacro}; use crate::foreign_modules; use crate::native_libs; -use crate::rmeta::{self, encoder}; +use crate::rmeta::encoder; use rustc_ast as ast; use rustc_ast::expand::allocator::AllocatorKind; use rustc_data_structures::stable_map::FxHashMap; use rustc_data_structures::svh::Svh; use rustc_hir as hir; -use rustc_hir::def::DefKind; +use rustc_hir::def::{CtorKind, DefKind}; use rustc_hir::def_id::{CrateNum, DefId, DefIdMap, CRATE_DEF_INDEX, LOCAL_CRATE}; use rustc_hir::definitions::{DefKey, DefPath, DefPathHash}; use rustc_middle::hir::exports::Export; @@ -17,7 +17,7 @@ use rustc_middle::middle::cstore::{CrateSource, CrateStore, EncodedMetadata}; use rustc_middle::middle::exported_symbols::ExportedSymbol; use rustc_middle::middle::stability::DeprecationEntry; use rustc_middle::ty::query::Providers; -use rustc_middle::ty::{self, TyCtxt}; +use rustc_middle::ty::{self, TyCtxt, Visibility}; use rustc_session::utils::NativeLibKind; use rustc_session::{CrateDisambiguator, Session}; use rustc_span::source_map::{Span, Spanned}; @@ -155,6 +155,7 @@ provide! { <'tcx> tcx, def_id, other, cdata, is_ctfe_mir_available => { cdata.is_ctfe_mir_available(def_id.index) } dylib_dependency_formats => { cdata.get_dylib_dependency_formats(tcx) } + is_private_dep => { cdata.private_dep } is_panic_runtime => { cdata.root.panic_runtime } is_compiler_builtins => { cdata.root.compiler_builtins } has_global_allocator => { cdata.root.has_global_allocator } @@ -185,23 +186,10 @@ provide! { <'tcx> tcx, def_id, other, cdata, } native_libraries => { Lrc::new(cdata.get_native_libraries(tcx.sess)) } foreign_modules => { cdata.get_foreign_modules(tcx) } - plugin_registrar_fn => { - cdata.root.plugin_registrar_fn.map(|index| { - DefId { krate: def_id.krate, index } - }) - } - proc_macro_decls_static => { - cdata.root.proc_macro_data.as_ref().map(|data| { - DefId { - krate: def_id.krate, - index: data.proc_macro_decls_static, - } - }) - } crate_disambiguator => { cdata.root.disambiguator } crate_hash => { cdata.root.hash } crate_host_hash => { cdata.host_hash } - original_crate_name => { cdata.root.name } + crate_name => { cdata.root.name } extra_filename => { cdata.root.extra_filename.clone() } @@ -218,7 +206,6 @@ provide! { <'tcx> tcx, def_id, other, cdata, let r = *cdata.dep_kind.lock(); r } - crate_name => { cdata.root.name } item_children => { let mut result = SmallVec::<[_; 8]>::new(); cdata.each_child_of_item(def_id.index, |child| result.push(child), tcx.sess); @@ -256,16 +243,17 @@ pub fn provide(providers: &mut Providers) { // resolve! Does this work? Unsure! That's what the issue is about *providers = Providers { is_dllimport_foreign_item: |tcx, id| match tcx.native_library_kind(id) { - Some(NativeLibKind::Dylib | NativeLibKind::RawDylib | NativeLibKind::Unspecified) => { - true - } + Some( + NativeLibKind::Dylib { .. } | NativeLibKind::RawDylib | NativeLibKind::Unspecified, + ) => true, _ => false, }, is_statically_included_foreign_item: |tcx, id| { - matches!( - tcx.native_library_kind(id), - Some(NativeLibKind::StaticBundle | NativeLibKind::StaticNoBundle) - ) + matches!(tcx.native_library_kind(id), Some(NativeLibKind::Static { .. })) + }, + is_private_dep: |_tcx, cnum| { + assert_eq!(cnum, LOCAL_CRATE); + false }, native_library_kind: |tcx, id| { tcx.native_libraries(id.krate) @@ -299,11 +287,10 @@ pub fn provide(providers: &mut Providers) { // external item that is visible from at least one local module) to a // sufficiently visible parent (considering modules that re-export the // external item to be parents). - visible_parent_map: |tcx, cnum| { + visible_parent_map: |tcx, ()| { use std::collections::hash_map::Entry; use std::collections::vec_deque::VecDeque; - assert_eq!(cnum, LOCAL_CRATE); let mut visible_parent_map: DefIdMap = Default::default(); // Issue 46112: We want the map to prefer the shortest @@ -351,7 +338,7 @@ pub fn provide(providers: &mut Providers) { Entry::Occupied(mut entry) => { // If `child` is defined in crate `cnum`, ensure // that it is mapped to a parent in `cnum`. - if child.krate == cnum && entry.get().krate != cnum { + if child.is_local() && entry.get().is_local() { entry.insert(parent); } } @@ -373,17 +360,14 @@ pub fn provide(providers: &mut Providers) { visible_parent_map }, - dependency_formats: |tcx, cnum| { - assert_eq!(cnum, LOCAL_CRATE); - Lrc::new(crate::dependency_format::calculate(tcx)) - }, + dependency_formats: |tcx, ()| Lrc::new(crate::dependency_format::calculate(tcx)), has_global_allocator: |tcx, cnum| { assert_eq!(cnum, LOCAL_CRATE); CStore::from_tcx(tcx).has_global_allocator() }, - postorder_cnums: |tcx, cnum| { - assert_eq!(cnum, LOCAL_CRATE); - tcx.arena.alloc_slice(&CStore::from_tcx(tcx).crate_dependencies_in_postorder(cnum)) + postorder_cnums: |tcx, ()| { + tcx.arena + .alloc_slice(&CStore::from_tcx(tcx).crate_dependencies_in_postorder(LOCAL_CRATE)) }, ..*providers @@ -395,6 +379,20 @@ impl CStore { self.get_crate_data(def.krate).get_struct_field_names(def.index, sess) } + pub fn struct_field_visibilities_untracked(&self, def: DefId) -> Vec { + self.get_crate_data(def.krate).get_struct_field_visibilities(def.index) + } + + pub fn ctor_def_id_and_kind_untracked(&self, def: DefId) -> Option<(DefId, CtorKind)> { + self.get_crate_data(def.krate).get_ctor_def_id(def.index).map(|ctor_def_id| { + (ctor_def_id, self.get_crate_data(def.krate).get_ctor_kind(def.index)) + }) + } + + pub fn visibility_untracked(&self, def: DefId) -> Visibility { + self.get_crate_data(def.krate).get_visibility(def.index) + } + pub fn item_children_untracked( &self, def_id: DefId, @@ -414,7 +412,7 @@ impl CStore { let data = self.get_crate_data(id.krate); if data.root.is_proc_macro_crate() { - return LoadedMacro::ProcMacro(data.load_proc_macro(id.index, sess)); + return LoadedMacro::ProcMacro(data.load_proc_macro(id, sess)); } let span = data.get_span(id.index, sess); @@ -461,13 +459,25 @@ impl CStore { self.get_crate_data(def_id.krate).module_expansion(def_id.index, sess) } - pub fn num_def_ids(&self, cnum: CrateNum) -> usize { + /// Only public-facing way to traverse all the definitions in a non-local crate. + /// Critically useful for this third-party project: . + /// See for context. + pub fn num_def_ids_untracked(&self, cnum: CrateNum) -> usize { self.get_crate_data(cnum).num_def_ids() } pub fn item_attrs(&self, def_id: DefId, sess: &Session) -> Vec { self.get_crate_data(def_id.krate).get_item_attrs(def_id.index, sess).collect() } + + pub fn get_proc_macro_quoted_span_untracked( + &self, + cnum: CrateNum, + id: usize, + sess: &Session, + ) -> Span { + self.get_crate_data(cnum).get_proc_macro_quoted_span(id, sess) + } } impl CrateStore for CStore { @@ -479,10 +489,6 @@ impl CrateStore for CStore { self.get_crate_data(cnum).root.name } - fn crate_is_private_dep_untracked(&self, cnum: CrateNum) -> bool { - self.get_crate_data(cnum).private_dep - } - fn crate_disambiguator_untracked(&self, cnum: CrateNum) -> CrateDisambiguator { self.get_crate_data(cnum).root.disambiguator } @@ -530,10 +536,6 @@ impl CrateStore for CStore { encoder::encode_metadata(tcx) } - fn metadata_encoding_version(&self) -> &[u8] { - rmeta::METADATA_HEADER - } - fn allocator_kind(&self) -> Option { self.allocator_kind() } diff --git a/compiler/rustc_metadata/src/rmeta/encoder.rs b/compiler/rustc_metadata/src/rmeta/encoder.rs index 02fa6905a0..7600739800 100644 --- a/compiler/rustc_metadata/src/rmeta/encoder.rs +++ b/compiler/rustc_metadata/src/rmeta/encoder.rs @@ -28,9 +28,12 @@ use rustc_middle::ty::codec::TyEncoder; use rustc_middle::ty::{self, SymbolName, Ty, TyCtxt}; use rustc_serialize::{opaque, Encodable, Encoder}; use rustc_session::config::CrateType; -use rustc_span::hygiene::{ExpnDataEncodeMode, HygieneEncodeContext, MacroKind}; use rustc_span::symbol::{sym, Ident, Symbol}; use rustc_span::{self, ExternalSource, FileName, SourceFile, Span, SyntaxContext}; +use rustc_span::{ + hygiene::{ExpnDataEncodeMode, HygieneEncodeContext, MacroKind}, + RealFileName, +}; use rustc_target::abi::VariantIdx; use std::hash::Hash; use std::num::NonZeroUsize; @@ -466,7 +469,6 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> { let source_map = self.tcx.sess.source_map(); let all_source_files = source_map.files(); - let (working_dir, _cwd_remapped) = self.tcx.sess.working_dir.clone(); // By replacing the `Option` with `None`, we ensure that we can't // accidentally serialize any more `Span`s after the source map encoding // is done. @@ -485,18 +487,41 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> { }) .map(|(_, source_file)| { let mut adapted = match source_file.name { - // This path of this SourceFile has been modified by - // path-remapping, so we use it verbatim (and avoid - // cloning the whole map in the process). - _ if source_file.name_was_remapped => source_file.clone(), - - // Otherwise expand all paths to absolute paths because - // any relative paths are potentially relative to a - // wrong directory. - FileName::Real(ref name) => { - let name = name.stable_name(); + FileName::Real(ref realname) => { let mut adapted = (**source_file).clone(); - adapted.name = Path::new(&working_dir).join(name).into(); + adapted.name = FileName::Real(match realname { + RealFileName::LocalPath(path_to_file) => { + // Prepend path of working directory onto potentially + // relative paths, because they could become relative + // to a wrong directory. + let working_dir = &self.tcx.sess.working_dir; + match working_dir { + RealFileName::LocalPath(absolute) => { + // If working_dir has not been remapped, then we emit a + // LocalPath variant as it's likely to be a valid path + RealFileName::LocalPath( + Path::new(absolute).join(path_to_file), + ) + } + RealFileName::Remapped { local_path: _, virtual_name } => { + // If working_dir has been remapped, then we emit + // Remapped variant as the expanded path won't be valid + RealFileName::Remapped { + local_path: None, + virtual_name: Path::new(virtual_name) + .join(path_to_file), + } + } + } + } + RealFileName::Remapped { local_path: _, virtual_name } => { + RealFileName::Remapped { + // We do not want any local path to be exported into metadata + local_path: None, + virtual_name: virtual_name.clone(), + } + } + }); adapted.name_hash = { let mut hasher: StableHasher = StableHasher::new(); adapted.name.hash(&mut hasher); @@ -653,7 +678,6 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> { has_global_allocator: tcx.has_global_allocator(LOCAL_CRATE), has_panic_handler: tcx.has_panic_handler(LOCAL_CRATE), has_default_lib_allocator, - plugin_registrar_fn: tcx.plugin_registrar_fn(LOCAL_CRATE).map(|id| id.index), proc_macro_data, compiler_builtins: tcx.sess.contains_name(&attrs, sym::compiler_builtins), needs_allocator: tcx.sess.contains_name(&attrs, sym::needs_allocator), @@ -919,7 +943,7 @@ impl EncodeContext<'a, 'tcx> { }); record!(self.tables.span[def_id] <- tcx.def_span(def_id)); record!(self.tables.attributes[def_id] <- tcx.get_attrs(def_id)); - record!(self.tables.expn_that_defined[def_id] <- self.tcx.expansion_that_defined(def_id)); + record!(self.tables.expn_that_defined[def_id] <- self.tcx.expn_that_defined(def_id)); if should_encode_visibility(def_kind) { record!(self.tables.visibility[def_id] <- self.tcx.visibility(def_id)); } @@ -945,13 +969,12 @@ impl EncodeContext<'a, 'tcx> { record!(self.tables.super_predicates[def_id] <- self.tcx.super_predicates_of(def_id)); } } - let inherent_impls = tcx.crate_inherent_impls(LOCAL_CRATE); + let inherent_impls = tcx.crate_inherent_impls(()); for (def_id, implementations) in inherent_impls.inherent_impls.iter() { - assert!(def_id.is_local()); if implementations.is_empty() { continue; } - record!(self.tables.inherent_impls[def_id] <- implementations.iter().map(|&def_id| { + record!(self.tables.inherent_impls[def_id.to_def_id()] <- implementations.iter().map(|&def_id| { assert!(def_id.is_local()); def_id.index })); @@ -1238,7 +1261,7 @@ impl EncodeContext<'a, 'tcx> { let mut keys_and_jobs = self .tcx - .mir_keys(LOCAL_CRATE) + .mir_keys(()) .iter() .filter_map(|&def_id| { let (encode_const, encode_opt) = should_encode_mir(self.tcx, def_id); @@ -1576,9 +1599,14 @@ impl EncodeContext<'a, 'tcx> { let tcx = self.tcx; let hir = tcx.hir(); - let proc_macro_decls_static = tcx.proc_macro_decls_static(LOCAL_CRATE).unwrap().index; + let proc_macro_decls_static = tcx.proc_macro_decls_static(()).unwrap().local_def_index; let stability = tcx.lookup_stability(DefId::local(CRATE_DEF_INDEX)).copied(); let macros = self.lazy(hir.krate().proc_macros.iter().map(|p| p.owner.local_def_index)); + let spans = self.tcx.sess.parse_sess.proc_macro_quoted_spans(); + for (i, span) in spans.into_iter().enumerate() { + let span = self.lazy(span); + self.tables.proc_macro_quoted_spans.set(i, span); + } record!(self.tables.def_kind[LOCAL_CRATE.as_def_id()] <- DefKind::Mod); record!(self.tables.span[LOCAL_CRATE.as_def_id()] <- tcx.def_span(LOCAL_CRATE.as_def_id())); @@ -1646,7 +1674,7 @@ impl EncodeContext<'a, 'tcx> { .iter() .map(|&cnum| { let dep = CrateDep { - name: self.tcx.original_crate_name(cnum), + name: self.tcx.crate_name(cnum), hash: self.tcx.crate_hash(cnum), host_hash: self.tcx.crate_host_hash(cnum), kind: self.tcx.dep_kind(cnum), @@ -1768,7 +1796,7 @@ impl EncodeContext<'a, 'tcx> { fn encode_dylib_dependency_formats(&mut self) -> Lazy<[Option]> { empty_proc_macro!(self); - let formats = self.tcx.dependency_formats(LOCAL_CRATE); + let formats = self.tcx.dependency_formats(()); for (ty, arr) in formats.iter() { if *ty != CrateType::Dylib { continue; @@ -1998,7 +2026,7 @@ fn prefetch_mir(tcx: TyCtxt<'_>) { return; } - par_iter(tcx.mir_keys(LOCAL_CRATE)).for_each(|&def_id| { + par_iter(tcx.mir_keys(())).for_each(|&def_id| { let (encode_const, encode_opt) = should_encode_mir(tcx, def_id); if encode_const { diff --git a/compiler/rustc_metadata/src/rmeta/mod.rs b/compiler/rustc_metadata/src/rmeta/mod.rs index 959aa1719a..99ea0cc8f2 100644 --- a/compiler/rustc_metadata/src/rmeta/mod.rs +++ b/compiler/rustc_metadata/src/rmeta/mod.rs @@ -52,7 +52,7 @@ const METADATA_VERSION: u8 = 5; /// This header is followed by the position of the `CrateRoot`, /// which is encoded as a 32-bit big-endian unsigned integer, /// and further followed by the rustc version string. -crate const METADATA_HEADER: &[u8; 8] = &[b'r', b'u', b's', b't', 0, 0, 0, METADATA_VERSION]; +pub const METADATA_HEADER: &[u8] = &[b'r', b'u', b's', b't', 0, 0, 0, METADATA_VERSION]; /// Additional metadata for a `Lazy` where `T` may not be `Sized`, /// e.g. for `Lazy<[T]>`, this is the length (count of `T` values). @@ -209,7 +209,6 @@ crate struct CrateRoot<'tcx> { has_global_allocator: bool, has_panic_handler: bool, has_default_lib_allocator: bool, - plugin_registrar_fn: Option, crate_deps: Lazy<[CrateDep]>, dylib_dependency_formats: Lazy<[Option]>, @@ -258,15 +257,15 @@ crate struct TraitImpls { /// Define `LazyTables` and `TableBuilders` at the same time. macro_rules! define_tables { - ($($name:ident: Table),+ $(,)?) => { + ($($name:ident: Table<$IDX:ty, $T:ty>),+ $(,)?) => { #[derive(MetadataEncodable, MetadataDecodable)] crate struct LazyTables<'tcx> { - $($name: Lazy!(Table)),+ + $($name: Lazy!(Table<$IDX, $T>)),+ } #[derive(Default)] struct TableBuilders<'tcx> { - $($name: TableBuilder),+ + $($name: TableBuilder<$IDX, $T>),+ } impl TableBuilders<'tcx> { @@ -315,6 +314,7 @@ define_tables! { // definitions from any given crate. def_keys: Table>, def_path_hashes: Table>, + proc_macro_quoted_spans: Table>, } #[derive(Copy, Clone, MetadataEncodable, MetadataDecodable)] diff --git a/compiler/rustc_middle/src/arena.rs b/compiler/rustc_middle/src/arena.rs index 9a42bbe7ba..a89d00e26a 100644 --- a/compiler/rustc_middle/src/arena.rs +++ b/compiler/rustc_middle/src/arena.rs @@ -14,6 +14,7 @@ macro_rules! arena_types { [] layouts: rustc_target::abi::Layout, // AdtDef are interned and compared by address [] adt_def: rustc_middle::ty::AdtDef, + [] steal_thir: rustc_data_structures::steal::Steal>, [] steal_mir: rustc_data_structures::steal::Steal>, [decode] mir: rustc_middle::mir::Body<$tcx>, [] steal_promoted: @@ -91,7 +92,7 @@ macro_rules! arena_types { [] predicates: rustc_middle::ty::PredicateInner<$tcx>, // HIR query types - [few] indexed_hir: rustc_middle::hir::map::IndexedHir<$tcx>, + [few] indexed_hir: rustc_middle::hir::IndexedHir<$tcx>, [few] hir_definitions: rustc_hir::definitions::Definitions, [] hir_owner: rustc_middle::hir::Owner<$tcx>, [] hir_owner_nodes: rustc_middle::hir::OwnerNodes<$tcx>, diff --git a/compiler/rustc_middle/src/dep_graph/dep_node.rs b/compiler/rustc_middle/src/dep_graph/dep_node.rs index aa54d1ae7b..8476929eae 100644 --- a/compiler/rustc_middle/src/dep_graph/dep_node.rs +++ b/compiler/rustc_middle/src/dep_graph/dep_node.rs @@ -285,7 +285,7 @@ pub type DepNode = rustc_query_system::dep_graph::DepNode; // required that their size stay the same, but we don't want to change // it inadvertently. This assert just ensures we're aware of any change. #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] -static_assert_size!(DepNode, 17); +static_assert_size!(DepNode, 18); #[cfg(not(any(target_arch = "x86", target_arch = "x86_64")))] static_assert_size!(DepNode, 24); diff --git a/compiler/rustc_middle/src/dep_graph/mod.rs b/compiler/rustc_middle/src/dep_graph/mod.rs index 31bea83295..aa61219ad7 100644 --- a/compiler/rustc_middle/src/dep_graph/mod.rs +++ b/compiler/rustc_middle/src/dep_graph/mod.rs @@ -18,7 +18,6 @@ crate use dep_node::{make_compile_codegen_unit, make_compile_mono_item}; pub type DepGraph = rustc_query_system::dep_graph::DepGraph; pub type TaskDeps = rustc_query_system::dep_graph::TaskDeps; pub type DepGraphQuery = rustc_query_system::dep_graph::DepGraphQuery; -pub type PreviousDepGraph = rustc_query_system::dep_graph::PreviousDepGraph; pub type SerializedDepGraph = rustc_query_system::dep_graph::SerializedDepGraph; pub type EdgeFilter = rustc_query_system::dep_graph::debug::EdgeFilter; diff --git a/compiler/rustc_middle/src/hir/map/collector.rs b/compiler/rustc_middle/src/hir/map/collector.rs index 57c1b9b82e..8ffd98326f 100644 --- a/compiler/rustc_middle/src/hir/map/collector.rs +++ b/compiler/rustc_middle/src/hir/map/collector.rs @@ -1,22 +1,20 @@ use crate::arena::Arena; -use crate::hir::map::{Entry, HirOwnerData, Map}; -use crate::hir::{AttributeMap, Owner, OwnerNodes, ParentedNode}; +use crate::hir::map::Map; +use crate::hir::{IndexedHir, OwnerNodes, ParentedNode}; use crate::ich::StableHashingContext; -use crate::middle::cstore::CrateStore; use rustc_data_structures::fingerprint::Fingerprint; use rustc_data_structures::fx::FxHashMap; use rustc_data_structures::stable_hasher::{HashStable, StableHasher}; -use rustc_data_structures::svh::Svh; use rustc_hir as hir; +use rustc_hir::def_id::LocalDefId; use rustc_hir::def_id::CRATE_DEF_INDEX; -use rustc_hir::def_id::{LocalDefId, CRATE_DEF_ID, LOCAL_CRATE}; -use rustc_hir::definitions::{self, DefPathHash}; +use rustc_hir::definitions; use rustc_hir::intravisit::{self, NestedVisitorMap, Visitor}; use rustc_hir::*; use rustc_index::vec::{Idx, IndexVec}; -use rustc_session::{CrateDisambiguator, Session}; +use rustc_session::Session; use rustc_span::source_map::SourceMap; -use rustc_span::{Span, Symbol, DUMMY_SP}; +use rustc_span::{Span, DUMMY_SP}; use std::iter::repeat; @@ -30,7 +28,8 @@ pub(super) struct NodeCollector<'a, 'hir> { /// Source map source_map: &'a SourceMap, - map: IndexVec>, + map: IndexVec>>, + parenting: FxHashMap, /// The parent of this node parent_node: hir::HirId, @@ -40,10 +39,6 @@ pub(super) struct NodeCollector<'a, 'hir> { definitions: &'a definitions::Definitions, hcx: StableHashingContext<'a>, - - // We are collecting HIR hashes here so we can compute the - // crate hash from them later on. - hir_body_nodes: Vec<(DefPathHash, Fingerprint)>, } fn insert_vec_map(map: &mut IndexVec>, k: K, v: V) { @@ -58,44 +53,20 @@ fn insert_vec_map(map: &mut IndexVec>, k: K, v: V fn hash_body( hcx: &mut StableHashingContext<'_>, - def_id: LocalDefId, - def_path_hash: DefPathHash, item_like: impl for<'a> HashStable>, - krate: &Crate<'_>, - hir_body_nodes: &mut Vec<(DefPathHash, Fingerprint)>, ) -> Fingerprint { - // Hash of nodes. - let hash: Fingerprint = { - let mut stable_hasher = StableHasher::new(); - hcx.while_hashing_hir_bodies(true, |hcx| { - item_like.hash_stable(hcx, &mut stable_hasher); - }); - stable_hasher.finish() - }; - // Hash for crate_hash. - let hash_with_attrs: Fingerprint = { - let mut hasher = StableHasher::new(); - hash.hash_stable(hcx, &mut hasher); - AttributeMap { map: &krate.attrs, prefix: def_id }.hash_stable(hcx, &mut hasher); - hasher.finish() - }; - hir_body_nodes.push((def_path_hash, hash_with_attrs)); - hash + let mut stable_hasher = StableHasher::new(); + hcx.while_hashing_hir_bodies(true, |hcx| { + item_like.hash_stable(hcx, &mut stable_hasher); + }); + stable_hasher.finish() } -fn upstream_crates(cstore: &dyn CrateStore) -> Vec<(Symbol, Fingerprint, Svh)> { - let mut upstream_crates: Vec<_> = cstore - .crates_untracked() - .iter() - .map(|&cnum| { - let name = cstore.crate_name_untracked(cnum); - let disambiguator = cstore.crate_disambiguator_untracked(cnum).to_fingerprint(); - let hash = cstore.crate_hash_untracked(cnum); - (name, disambiguator, hash) - }) - .collect(); - upstream_crates.sort_unstable_by_key(|&(name, dis, _)| (name.as_str(), dis)); - upstream_crates +/// Represents an entry and its parent `HirId`. +#[derive(Copy, Clone, Debug)] +pub struct Entry<'hir> { + parent: HirId, + node: Node<'hir>, } impl<'a, 'hir> NodeCollector<'a, 'hir> { @@ -106,11 +77,6 @@ impl<'a, 'hir> NodeCollector<'a, 'hir> { definitions: &'a definitions::Definitions, mut hcx: StableHashingContext<'a>, ) -> NodeCollector<'a, 'hir> { - let root_mod_def_path_hash = - definitions.def_path_hash(LocalDefId { local_def_index: CRATE_DEF_INDEX }); - - let mut hir_body_nodes = Vec::new(); - let hash = { let Crate { ref item, @@ -130,14 +96,7 @@ impl<'a, 'hir> NodeCollector<'a, 'hir> { attrs: _, } = *krate; - hash_body( - &mut hcx, - CRATE_DEF_ID, - root_mod_def_path_hash, - item, - krate, - &mut hir_body_nodes, - ) + hash_body(&mut hcx, item) }; let mut collector = NodeCollector { @@ -148,10 +107,8 @@ impl<'a, 'hir> NodeCollector<'a, 'hir> { current_dep_node_owner: LocalDefId { local_def_index: CRATE_DEF_INDEX }, definitions, hcx, - hir_body_nodes, - map: (0..definitions.def_index_count()) - .map(|_| HirOwnerData { signature: None, with_bodies: None }) - .collect(), + map: IndexVec::from_fn_n(|_| None, definitions.def_index_count()), + parenting: FxHashMap::default(), }; collector.insert_entry( hir::CRATE_HIR_ID, @@ -162,56 +119,13 @@ impl<'a, 'hir> NodeCollector<'a, 'hir> { collector } - pub(super) fn finalize_and_compute_crate_hash( - mut self, - crate_disambiguator: CrateDisambiguator, - cstore: &dyn CrateStore, - commandline_args_hash: u64, - ) -> (IndexVec>, Svh) { + pub(super) fn finalize_and_compute_crate_hash(mut self) -> IndexedHir<'hir> { // Insert bodies into the map for (id, body) in self.krate.bodies.iter() { - let bodies = &mut self.map[id.hir_id.owner].with_bodies.as_mut().unwrap().bodies; + let bodies = &mut self.map[id.hir_id.owner].as_mut().unwrap().bodies; assert!(bodies.insert(id.hir_id.local_id, body).is_none()); } - - self.hir_body_nodes.sort_unstable_by_key(|bn| bn.0); - - let node_hashes = self.hir_body_nodes.iter().fold( - Fingerprint::ZERO, - |combined_fingerprint, &(def_path_hash, fingerprint)| { - combined_fingerprint.combine(def_path_hash.0.combine(fingerprint)) - }, - ); - - let upstream_crates = upstream_crates(cstore); - - // We hash the final, remapped names of all local source files so we - // don't have to include the path prefix remapping commandline args. - // If we included the full mapping in the SVH, we could only have - // reproducible builds by compiling from the same directory. So we just - // hash the result of the mapping instead of the mapping itself. - let mut source_file_names: Vec<_> = self - .source_map - .files() - .iter() - .filter(|source_file| source_file.cnum == LOCAL_CRATE) - .map(|source_file| source_file.name_hash) - .collect(); - - source_file_names.sort_unstable(); - - let crate_hash_input = ( - ((node_hashes, upstream_crates), source_file_names), - (commandline_args_hash, crate_disambiguator.to_fingerprint()), - &self.krate.non_exported_macro_attrs, - ); - - let mut stable_hasher = StableHasher::new(); - crate_hash_input.hash_stable(&mut self.hcx, &mut stable_hasher); - let crate_hash: Fingerprint = stable_hasher.finish(); - - let svh = Svh::new(crate_hash.to_smaller_hash()); - (self.map, svh) + IndexedHir { map: self.map, parenting: self.parenting } } fn insert_entry(&mut self, id: HirId, entry: Entry<'hir>, hash: Fingerprint) { @@ -221,23 +135,13 @@ impl<'a, 'hir> NodeCollector<'a, 'hir> { let data = &mut self.map[id.owner]; - if data.with_bodies.is_none() { - data.with_bodies = Some(arena.alloc(OwnerNodes { + if i == 0 { + debug_assert!(data.is_none()); + *data = Some(arena.alloc(OwnerNodes { hash, nodes: IndexVec::new(), bodies: FxHashMap::default(), })); - } - - let nodes = data.with_bodies.as_mut().unwrap(); - - if i == 0 { - // Overwrite the dummy hash with the real HIR owner hash. - nodes.hash = hash; - - debug_assert!(data.signature.is_none()); - data.signature = - Some(self.arena.alloc(Owner { parent: entry.parent, node: entry.node })); let dk_parent = self.definitions.def_key(id.owner).parent; if let Some(dk_parent) = dk_parent { @@ -249,15 +153,20 @@ impl<'a, 'hir> NodeCollector<'a, 'hir> { id.owner, dk_parent, entry.parent, ) } + + debug_assert_eq!(self.parenting.get(&id.owner), Some(&entry.parent)); } } else { - assert_eq!(entry.parent.owner, id.owner); - insert_vec_map( - &mut nodes.nodes, - id.local_id, - ParentedNode { parent: entry.parent.local_id, node: entry.node }, - ); + debug_assert_eq!(entry.parent.owner, id.owner); } + + let data = data.as_mut().unwrap(); + + insert_vec_map( + &mut data.nodes, + id.local_id, + ParentedNode { parent: entry.parent.local_id, node: entry.node }, + ); } fn insert(&mut self, span: Span, hir_id: HirId, node: Node<'hir>) { @@ -280,7 +189,7 @@ impl<'a, 'hir> NodeCollector<'a, 'hir> { span, "inconsistent DepNode at `{:?}` for `{}`: \ current_dep_node_owner={} ({:?}), hir_id.owner={} ({:?})", - self.source_map.span_to_string(span), + self.source_map.span_to_diagnostic_string(span), node_str, self.definitions .def_path(self.current_dep_node_owner) @@ -312,22 +221,28 @@ impl<'a, 'hir> NodeCollector<'a, 'hir> { f: F, ) { let prev_owner = self.current_dep_node_owner; - - let def_path_hash = self.definitions.def_path_hash(dep_node_owner); - - let hash = hash_body( - &mut self.hcx, - dep_node_owner, - def_path_hash, - item_like, - self.krate, - &mut self.hir_body_nodes, - ); + let hash = hash_body(&mut self.hcx, item_like); self.current_dep_node_owner = dep_node_owner; f(self, hash); self.current_dep_node_owner = prev_owner; } + + fn insert_nested(&mut self, item: LocalDefId) { + #[cfg(debug_assertions)] + { + let dk_parent = self.definitions.def_key(item).parent.unwrap(); + let dk_parent = LocalDefId { local_def_index: dk_parent }; + let dk_parent = self.definitions.local_def_id_to_hir_id(dk_parent); + debug_assert_eq!( + dk_parent.owner, self.parent_node.owner, + "Different parents for {:?}", + item + ) + } + + assert_eq!(self.parenting.insert(item, self.parent_node), None); + } } impl<'a, 'hir> Visitor<'hir> for NodeCollector<'a, 'hir> { @@ -343,18 +258,22 @@ impl<'a, 'hir> Visitor<'hir> for NodeCollector<'a, 'hir> { fn visit_nested_item(&mut self, item: ItemId) { debug!("visit_nested_item: {:?}", item); + self.insert_nested(item.def_id); self.visit_item(self.krate.item(item)); } fn visit_nested_trait_item(&mut self, item_id: TraitItemId) { + self.insert_nested(item_id.def_id); self.visit_trait_item(self.krate.trait_item(item_id)); } fn visit_nested_impl_item(&mut self, item_id: ImplItemId) { + self.insert_nested(item_id.def_id); self.visit_impl_item(self.krate.impl_item(item_id)); } fn visit_nested_foreign_item(&mut self, foreign_id: ForeignItemId) { + self.insert_nested(foreign_id.def_id); self.visit_foreign_item(self.krate.foreign_item(foreign_id)); } @@ -542,6 +461,7 @@ impl<'a, 'hir> Visitor<'hir> for NodeCollector<'a, 'hir> { self.definitions.local_def_id_to_hir_id(LocalDefId { local_def_index }) }); self.with_parent(parent, |this| { + this.insert_nested(macro_def.def_id); this.with_dep_node_owner(macro_def.def_id, macro_def, |this, hash| { this.insert_with_hash( macro_def.span, diff --git a/compiler/rustc_middle/src/hir/map/mod.rs b/compiler/rustc_middle/src/hir/map/mod.rs index 4cd126988f..20bbf9097f 100644 --- a/compiler/rustc_middle/src/hir/map/mod.rs +++ b/compiler/rustc_middle/src/hir/map/mod.rs @@ -1,8 +1,11 @@ use self::collector::NodeCollector; -use crate::hir::{Owner, OwnerNodes}; +use crate::hir::{AttributeMap, IndexedHir}; +use crate::middle::cstore::CrateStore; use crate::ty::TyCtxt; use rustc_ast as ast; +use rustc_data_structures::fingerprint::Fingerprint; +use rustc_data_structures::stable_hasher::{HashStable, StableHasher}; use rustc_data_structures::svh::Svh; use rustc_hir::def::{DefKind, Res}; use rustc_hir::def_id::{CrateNum, DefId, LocalDefId, CRATE_DEF_INDEX, LOCAL_CRATE}; @@ -11,7 +14,7 @@ use rustc_hir::intravisit; use rustc_hir::intravisit::Visitor; use rustc_hir::itemlikevisit::ItemLikeVisitor; use rustc_hir::*; -use rustc_index::vec::IndexVec; +use rustc_index::vec::Idx; use rustc_span::hygiene::MacroKind; use rustc_span::source_map::Spanned; use rustc_span::symbol::{kw, Ident, Symbol}; @@ -21,22 +24,6 @@ use rustc_target::spec::abi::Abi; pub mod blocks; mod collector; -/// Represents an entry and its parent `HirId`. -#[derive(Copy, Clone, Debug)] -pub struct Entry<'hir> { - parent: HirId, - node: Node<'hir>, -} - -impl<'hir> Entry<'hir> { - fn parent_node(self) -> Option { - match self.node { - Node::Crate(_) => None, - _ => Some(self.parent), - } - } -} - fn fn_decl<'hir>(node: Node<'hir>) -> Option<&'hir FnDecl<'hir>> { match node { Node::Item(Item { kind: ItemKind::Fn(sig, _, _), .. }) @@ -86,20 +73,6 @@ fn is_body_owner<'hir>(node: Node<'hir>, hir_id: HirId) -> bool { } } -#[derive(Debug)] -pub(super) struct HirOwnerData<'hir> { - pub(super) signature: Option<&'hir Owner<'hir>>, - pub(super) with_bodies: Option<&'hir mut OwnerNodes<'hir>>, -} - -#[derive(Debug)] -pub struct IndexedHir<'hir> { - /// The SVH of the local crate. - pub crate_hash: Svh, - - pub(super) map: IndexVec>, -} - #[derive(Copy, Clone)] pub struct Map<'hir> { pub(super) tcx: TyCtxt<'hir>, @@ -129,17 +102,55 @@ impl<'hir> Iterator for ParentHirIterator<'_, 'hir> { } self.current_id = parent_id; - if let Some(entry) = self.map.find_entry(parent_id) { - return Some((parent_id, entry.node)); + if let Some(node) = self.map.find(parent_id) { + return Some((parent_id, node)); + } + // If this `HirId` doesn't have an entry, skip it and look for its `parent_id`. + } + } +} + +/// An iterator that walks up the ancestor tree of a given `HirId`. +/// Constructed using `tcx.hir().parent_owner_iter(hir_id)`. +pub struct ParentOwnerIterator<'map, 'hir> { + current_id: HirId, + map: &'map Map<'hir>, +} + +impl<'hir> Iterator for ParentOwnerIterator<'_, 'hir> { + type Item = (HirId, Node<'hir>); + + fn next(&mut self) -> Option { + if self.current_id.local_id.index() != 0 { + self.current_id.local_id = ItemLocalId::new(0); + if let Some(node) = self.map.find(self.current_id) { + return Some((self.current_id, node)); + } + } + if self.current_id == CRATE_HIR_ID { + return None; + } + loop { + // There are nodes that do not have entries, so we need to skip them. + let parent_id = self.map.def_key(self.current_id.owner).parent; + + let parent_id = parent_id.map_or(CRATE_HIR_ID.owner, |local_def_index| { + let def_id = LocalDefId { local_def_index }; + self.map.local_def_id_to_hir_id(def_id).owner + }); + self.current_id = HirId::make_owner(parent_id); + + // If this `HirId` doesn't have an entry, skip it and look for its `parent_id`. + if let Some(node) = self.map.find(self.current_id) { + return Some((self.current_id, node)); } - // If this `HirId` doesn't have an `Entry`, skip it and look for its `parent_id`. } } } impl<'hir> Map<'hir> { pub fn krate(&self) -> &'hir Crate<'hir> { - self.tcx.hir_crate(LOCAL_CRATE) + self.tcx.hir_crate(()) } #[inline] @@ -165,7 +176,7 @@ impl<'hir> Map<'hir> { bug!( "local_def_id: no entry for `{:?}`, which has a map of `{:?}`", hir_id, - self.find_entry(hir_id) + self.find(hir_id) ) }) } @@ -272,27 +283,60 @@ impl<'hir> Map<'hir> { .unwrap_or_else(|| bug!("def_kind: unsupported node: {:?}", local_def_id)) } - fn find_entry(&self, id: HirId) -> Option> { + pub fn find_parent_node(&self, id: HirId) -> Option { if id.local_id == ItemLocalId::from_u32(0) { - let owner = self.tcx.hir_owner(id.owner); - owner.map(|owner| Entry { parent: owner.parent, node: owner.node }) + Some(self.tcx.hir_owner_parent(id.owner)) } else { - let owner = self.tcx.hir_owner_nodes(id.owner); - owner.and_then(|owner| { - let node = owner.nodes[id.local_id].as_ref(); - // FIXME(eddyb) use a single generic type instead of having both - // `Entry` and `ParentedNode`, which are effectively the same. - // Alternatively, rewrite code using `Entry` to use `ParentedNode`. - node.map(|node| Entry { - parent: HirId { owner: id.owner, local_id: node.parent }, - node: node.node, - }) - }) + let owner = self.tcx.hir_owner_nodes(id.owner)?; + let node = owner.nodes[id.local_id].as_ref()?; + let hir_id = HirId { owner: id.owner, local_id: node.parent }; + Some(hir_id) + } + } + + pub fn get_parent_node(&self, hir_id: HirId) -> HirId { + self.find_parent_node(hir_id).unwrap_or(CRATE_HIR_ID) + } + + /// Retrieves the `Node` corresponding to `id`, returning `None` if cannot be found. + pub fn find(&self, id: HirId) -> Option> { + if id.local_id == ItemLocalId::from_u32(0) { + let owner = self.tcx.hir_owner(id.owner)?; + Some(owner.node) + } else { + let owner = self.tcx.hir_owner_nodes(id.owner)?; + let node = owner.nodes[id.local_id].as_ref()?; + Some(node.node) } } - fn get_entry(&self, id: HirId) -> Entry<'hir> { - self.find_entry(id).unwrap() + /// Retrieves the `Node` corresponding to `id`, panicking if it cannot be found. + pub fn get(&self, id: HirId) -> Node<'hir> { + self.find(id).unwrap_or_else(|| bug!("couldn't find hir id {} in the HIR map", id)) + } + + pub fn get_if_local(&self, id: DefId) -> Option> { + id.as_local().and_then(|id| self.find(self.local_def_id_to_hir_id(id))) + } + + pub fn get_generics(&self, id: DefId) -> Option<&'hir Generics<'hir>> { + self.get_if_local(id).and_then(|node| match &node { + Node::ImplItem(impl_item) => Some(&impl_item.generics), + Node::TraitItem(trait_item) => Some(&trait_item.generics), + Node::Item(Item { + kind: + ItemKind::Fn(_, generics, _) + | ItemKind::TyAlias(_, generics) + | ItemKind::Enum(_, generics) + | ItemKind::Struct(_, generics) + | ItemKind::Union(_, generics) + | ItemKind::Trait(_, _, generics, ..) + | ItemKind::TraitAlias(generics, _) + | ItemKind::Impl(Impl { generics, .. }), + .. + }) => Some(generics), + _ => None, + }) } pub fn item(&self, id: ItemId) -> &'hir Item<'hir> { @@ -445,7 +489,7 @@ impl<'hir> Map<'hir> { } pub fn trait_impls(&self, trait_did: DefId) -> &'hir [LocalDefId] { - self.tcx.all_local_trait_impls(LOCAL_CRATE).get(&trait_did).map_or(&[], |xs| &xs[..]) + self.tcx.all_local_trait_impls(()).get(&trait_did).map_or(&[], |xs| &xs[..]) } /// Gets the attributes on the crate. This is preferable to @@ -457,7 +501,7 @@ impl<'hir> Map<'hir> { pub fn get_module(&self, module: LocalDefId) -> (&'hir Mod<'hir>, Span, HirId) { let hir_id = self.local_def_id_to_hir_id(module); - match self.get_entry(hir_id).node { + match self.get(hir_id) { Node::Item(&Item { span, kind: ItemKind::Mod(ref m), .. }) => (m, span, hir_id), Node::Crate(item) => (&item, item.inner, hir_id), node => panic!("not a module: {:?}", node), @@ -496,60 +540,18 @@ impl<'hir> Map<'hir> { } } - /// Retrieves the `Node` corresponding to `id`, panicking if it cannot be found. - pub fn get(&self, id: HirId) -> Node<'hir> { - self.find(id).unwrap_or_else(|| bug!("couldn't find hir id {} in the HIR map", id)) - } - - pub fn get_if_local(&self, id: DefId) -> Option> { - id.as_local().and_then(|id| self.find(self.local_def_id_to_hir_id(id))) - } - - pub fn get_generics(&self, id: DefId) -> Option<&'hir Generics<'hir>> { - self.get_if_local(id).and_then(|node| match &node { - Node::ImplItem(impl_item) => Some(&impl_item.generics), - Node::TraitItem(trait_item) => Some(&trait_item.generics), - Node::Item(Item { - kind: - ItemKind::Fn(_, generics, _) - | ItemKind::TyAlias(_, generics) - | ItemKind::Enum(_, generics) - | ItemKind::Struct(_, generics) - | ItemKind::Union(_, generics) - | ItemKind::Trait(_, _, generics, ..) - | ItemKind::TraitAlias(generics, _) - | ItemKind::Impl(Impl { generics, .. }), - .. - }) => Some(generics), - _ => None, - }) - } - - /// Retrieves the `Node` corresponding to `id`, returning `None` if cannot be found. - pub fn find(&self, hir_id: HirId) -> Option> { - self.find_entry(hir_id).map(|entry| entry.node) - } - - /// Similar to `get_parent`; returns the parent HIR Id, or just `hir_id` if there - /// is no parent. Note that the parent may be `CRATE_HIR_ID`, which is not itself - /// present in the map, so passing the return value of `get_parent_node` to - /// `get` may in fact panic. - /// This function returns the immediate parent in the HIR, whereas `get_parent` - /// returns the enclosing item. Note that this might not be the actual parent - /// node in the HIR -- some kinds of nodes are not in the map and these will - /// never appear as the parent node. Thus, you can always walk the parent nodes - /// from a node to the root of the HIR (unless you get back the same ID here, - /// which can happen if the ID is not in the map itself or is just weird). - pub fn get_parent_node(&self, hir_id: HirId) -> HirId { - self.get_entry(hir_id).parent_node().unwrap_or(hir_id) - } - /// Returns an iterator for the nodes in the ancestor tree of the `current_id` /// until the crate root is reached. Prefer this over your own loop using `get_parent_node`. pub fn parent_iter(&self, current_id: HirId) -> ParentHirIterator<'_, 'hir> { ParentHirIterator { current_id, map: self } } + /// Returns an iterator for the nodes in the ancestor tree of the `current_id` + /// until the crate root is reached. Prefer this over your own loop using `get_parent_node`. + pub fn parent_owner_iter(&self, current_id: HirId) -> ParentOwnerIterator<'_, 'hir> { + ParentOwnerIterator { current_id, map: self } + } + /// Checks if the node is left-hand side of an assignment. pub fn is_lhs(&self, id: HirId) -> bool { match self.find(self.get_parent_node(id)) { @@ -570,7 +572,7 @@ impl<'hir> Map<'hir> { /// Whether `hir_id` corresponds to a `mod` or a crate. pub fn is_hir_id_module(&self, hir_id: HirId) -> bool { matches!( - self.get_entry(hir_id).node, + self.get(hir_id), Node::Item(Item { kind: ItemKind::Mod(_), .. }) | Node::Crate(..) ) } @@ -600,8 +602,8 @@ impl<'hir> Map<'hir> { pub fn get_return_block(&self, id: HirId) -> Option { let mut iter = self.parent_iter(id).peekable(); let mut ignore_tail = false; - if let Some(entry) = self.find_entry(id) { - if let Node::Expr(Expr { kind: ExprKind::Ret(_), .. }) = entry.node { + if let Some(node) = self.find(id) { + if let Node::Expr(Expr { kind: ExprKind::Ret(_), .. }) = node { // When dealing with `return` statements, we don't care about climbing only tail // expressions. ignore_tail = true; @@ -638,23 +640,23 @@ impl<'hir> Map<'hir> { /// in the HIR which is recorded by the map and is an item, either an item /// in a module, trait, or impl. pub fn get_parent_item(&self, hir_id: HirId) -> HirId { - for (hir_id, node) in self.parent_iter(hir_id) { - match node { - Node::Crate(_) - | Node::Item(_) - | Node::ForeignItem(_) - | Node::TraitItem(_) - | Node::ImplItem(_) => return hir_id, - _ => {} + for (hir_id, node) in self.parent_owner_iter(hir_id) { + if let Node::Crate(_) + | Node::Item(_) + | Node::ForeignItem(_) + | Node::TraitItem(_) + | Node::ImplItem(_) = node + { + return hir_id; } } - hir_id + CRATE_HIR_ID } /// Returns the `HirId` of `id`'s nearest module parent, or `id` itself if no /// module parent is in this map. pub(super) fn get_module_parent_node(&self, hir_id: HirId) -> HirId { - for (hir_id, node) in self.parent_iter(hir_id) { + for (hir_id, node) in self.parent_owner_iter(hir_id) { if let Node::Item(&Item { kind: ItemKind::Mod(_), .. }) = node { return hir_id; } @@ -728,12 +730,8 @@ impl<'hir> Map<'hir> { pub fn get_foreign_abi(&self, hir_id: HirId) -> Abi { let parent = self.get_parent_item(hir_id); - if let Some(entry) = self.find_entry(parent) { - if let Entry { - node: Node::Item(Item { kind: ItemKind::ForeignMod { abi, .. }, .. }), - .. - } = entry - { + if let Some(node) = self.find(parent) { + if let Node::Item(Item { kind: ItemKind::ForeignMod { abi, .. }, .. }) = node { return *abi; } } @@ -827,7 +825,7 @@ impl<'hir> Map<'hir> { } pub fn opt_span(&self, hir_id: HirId) -> Option { - let span = match self.find_entry(hir_id)?.node { + let span = match self.find(hir_id)? { Node::Param(param) => param.span, Node::Item(item) => match &item.kind { ItemKind::Fn(sig, _, _) => sig.span, @@ -876,7 +874,7 @@ impl<'hir> Map<'hir> { /// Like `hir.span()`, but includes the body of function items /// (instead of just the function header) pub fn span_with_body(&self, hir_id: HirId) -> Span { - match self.find_entry(hir_id).map(|entry| entry.node) { + match self.find(hir_id) { Some(Node::TraitItem(item)) => item.span, Some(Node::ImplItem(impl_item)) => impl_item.span, Some(Node::Item(item)) => item.span, @@ -930,24 +928,88 @@ impl<'hir> intravisit::Map<'hir> for Map<'hir> { } } -pub(super) fn index_hir<'tcx>(tcx: TyCtxt<'tcx>, cnum: CrateNum) -> &'tcx IndexedHir<'tcx> { - assert_eq!(cnum, LOCAL_CRATE); - +pub(super) fn index_hir<'tcx>(tcx: TyCtxt<'tcx>, (): ()) -> &'tcx IndexedHir<'tcx> { let _prof_timer = tcx.sess.prof.generic_activity("build_hir_map"); - let (map, crate_hash) = { - let hcx = tcx.create_stable_hashing_context(); + let hcx = tcx.create_stable_hashing_context(); + let mut collector = + NodeCollector::root(tcx.sess, &**tcx.arena, tcx.untracked_crate, &tcx.definitions, hcx); + intravisit::walk_crate(&mut collector, tcx.untracked_crate); - let mut collector = - NodeCollector::root(tcx.sess, &**tcx.arena, tcx.untracked_crate, &tcx.definitions, hcx); - intravisit::walk_crate(&mut collector, tcx.untracked_crate); + let map = collector.finalize_and_compute_crate_hash(); + tcx.arena.alloc(map) +} - let crate_disambiguator = tcx.sess.local_crate_disambiguator(); - let cmdline_args = tcx.sess.opts.dep_tracking_hash(true); - collector.finalize_and_compute_crate_hash(crate_disambiguator, &*tcx.cstore, cmdline_args) - }; +pub(super) fn crate_hash(tcx: TyCtxt<'_>, crate_num: CrateNum) -> Svh { + assert_eq!(crate_num, LOCAL_CRATE); + + let mut hcx = tcx.create_stable_hashing_context(); + + let mut hir_body_nodes: Vec<_> = tcx + .index_hir(()) + .map + .iter_enumerated() + .filter_map(|(def_id, hod)| { + let def_path_hash = tcx.definitions.def_path_hash(def_id); + let mut hasher = StableHasher::new(); + hod.as_ref()?.hash_stable(&mut hcx, &mut hasher); + AttributeMap { map: &tcx.untracked_crate.attrs, prefix: def_id } + .hash_stable(&mut hcx, &mut hasher); + Some((def_path_hash, hasher.finish())) + }) + .collect(); + hir_body_nodes.sort_unstable_by_key(|bn| bn.0); - tcx.arena.alloc(IndexedHir { crate_hash, map }) + let node_hashes = hir_body_nodes.iter().fold( + Fingerprint::ZERO, + |combined_fingerprint, &(def_path_hash, fingerprint)| { + combined_fingerprint.combine(def_path_hash.0.combine(fingerprint)) + }, + ); + + let upstream_crates = upstream_crates(&*tcx.cstore); + + // We hash the final, remapped names of all local source files so we + // don't have to include the path prefix remapping commandline args. + // If we included the full mapping in the SVH, we could only have + // reproducible builds by compiling from the same directory. So we just + // hash the result of the mapping instead of the mapping itself. + let mut source_file_names: Vec<_> = tcx + .sess + .source_map() + .files() + .iter() + .filter(|source_file| source_file.cnum == LOCAL_CRATE) + .map(|source_file| source_file.name_hash) + .collect(); + + source_file_names.sort_unstable(); + + let mut stable_hasher = StableHasher::new(); + node_hashes.hash_stable(&mut hcx, &mut stable_hasher); + upstream_crates.hash_stable(&mut hcx, &mut stable_hasher); + source_file_names.hash_stable(&mut hcx, &mut stable_hasher); + tcx.sess.opts.dep_tracking_hash(true).hash_stable(&mut hcx, &mut stable_hasher); + tcx.sess.local_crate_disambiguator().to_fingerprint().hash_stable(&mut hcx, &mut stable_hasher); + tcx.untracked_crate.non_exported_macro_attrs.hash_stable(&mut hcx, &mut stable_hasher); + + let crate_hash: Fingerprint = stable_hasher.finish(); + Svh::new(crate_hash.to_smaller_hash()) +} + +fn upstream_crates(cstore: &dyn CrateStore) -> Vec<(Symbol, Fingerprint, Svh)> { + let mut upstream_crates: Vec<_> = cstore + .crates_untracked() + .iter() + .map(|&cnum| { + let name = cstore.crate_name_untracked(cnum); + let disambiguator = cstore.crate_disambiguator_untracked(cnum).to_fingerprint(); + let hash = cstore.crate_hash_untracked(cnum); + (name, disambiguator, hash) + }) + .collect(); + upstream_crates.sort_unstable_by_key(|&(name, dis, _)| (name.as_str(), dis)); + upstream_crates } fn hir_id_to_string(map: &Map<'_>, id: HirId) -> String { diff --git a/compiler/rustc_middle/src/hir/mod.rs b/compiler/rustc_middle/src/hir/mod.rs index cf4e473d8a..087f772c81 100644 --- a/compiler/rustc_middle/src/hir/mod.rs +++ b/compiler/rustc_middle/src/hir/mod.rs @@ -13,28 +13,41 @@ use rustc_ast::Attribute; use rustc_data_structures::fingerprint::Fingerprint; use rustc_data_structures::fx::FxHashMap; use rustc_data_structures::stable_hasher::{HashStable, StableHasher}; -use rustc_hir::def_id::{LocalDefId, LOCAL_CRATE}; +use rustc_hir::def_id::LocalDefId; use rustc_hir::*; -use rustc_index::vec::IndexVec; +use rustc_index::vec::{Idx, IndexVec}; use rustc_span::DUMMY_SP; use std::collections::BTreeMap; +/// Result of HIR indexing. #[derive(Debug)] +pub struct IndexedHir<'hir> { + /// Contents of the HIR owned by each definition. None for definitions that are not HIR owners. + // The `mut` comes from construction time, and is harmless since we only ever hand out + // immutable refs to IndexedHir. + map: IndexVec>>, + /// Map from each owner to its parent's HirId inside another owner. + // This map is separate from `map` to eventually allow for per-owner indexing. + parenting: FxHashMap, +} + +/// Top-level HIR node for current owner. This only contains the node for which +/// `HirId::local_id == 0`, and excludes bodies. +#[derive(Copy, Clone, Debug)] pub struct Owner<'tcx> { - parent: HirId, node: Node<'tcx>, } impl<'a, 'tcx> HashStable> for Owner<'tcx> { fn hash_stable(&self, hcx: &mut StableHashingContext<'a>, hasher: &mut StableHasher) { - let Owner { parent, node } = self; - hcx.while_hashing_hir_bodies(false, |hcx| { - parent.hash_stable(hcx, hasher); - node.hash_stable(hcx, hasher); - }); + let Owner { node } = self; + hcx.while_hashing_hir_bodies(false, |hcx| node.hash_stable(hcx, hasher)); } } +/// HIR node coupled with its parent's id in the same HIR owner. +/// +/// The parent is trash when the node is a HIR owner. #[derive(Clone, Debug)] pub struct ParentedNode<'tcx> { parent: ItemLocalId, @@ -43,8 +56,12 @@ pub struct ParentedNode<'tcx> { #[derive(Debug)] pub struct OwnerNodes<'tcx> { + /// Pre-computed hash of the full HIR. hash: Fingerprint, + /// Full HIR for the current owner. + // The zeroth node's parent is trash, but is never accessed. nodes: IndexVec>>, + /// Content of local bodies. bodies: FxHashMap>, } @@ -57,6 +74,8 @@ impl<'a, 'tcx> HashStable> for OwnerNodes<'tcx> { } } +/// Attributes owner by a HIR owner. It is build as a slice inside the attributes map, restricted +/// to the nodes whose `HirId::owner` is `prefix`. #[derive(Copy, Clone)] pub struct AttributeMap<'tcx> { map: &'tcx BTreeMap, @@ -115,11 +134,20 @@ pub fn provide(providers: &mut Providers) { let hir = tcx.hir(); hir.local_def_id(hir.get_module_parent_node(hir.local_def_id_to_hir_id(id))) }; - providers.hir_crate = |tcx, _| tcx.untracked_crate; + providers.hir_crate = |tcx, ()| tcx.untracked_crate; providers.index_hir = map::index_hir; + providers.crate_hash = map::crate_hash; providers.hir_module_items = |tcx, id| &tcx.untracked_crate.modules[&id]; - providers.hir_owner = |tcx, id| tcx.index_hir(LOCAL_CRATE).map[id].signature; - providers.hir_owner_nodes = |tcx, id| tcx.index_hir(LOCAL_CRATE).map[id].with_bodies.as_deref(); + providers.hir_owner = |tcx, id| { + let owner = tcx.index_hir(()).map[id].as_ref()?; + let node = owner.nodes[ItemLocalId::new(0)].as_ref()?.node; + Some(Owner { node }) + }; + providers.hir_owner_nodes = |tcx, id| tcx.index_hir(()).map[id].as_deref(); + providers.hir_owner_parent = |tcx, id| { + let index = tcx.index_hir(()); + index.parenting.get(&id).copied().unwrap_or(CRATE_HIR_ID) + }; providers.hir_attrs = |tcx, id| AttributeMap { map: &tcx.untracked_crate.attrs, prefix: id }; providers.def_span = |tcx, def_id| tcx.hir().span_if_local(def_id).unwrap_or(DUMMY_SP); providers.fn_arg_names = |tcx, id| { @@ -138,4 +166,9 @@ pub fn provide(providers: &mut Providers) { } }; providers.opt_def_kind = |tcx, def_id| tcx.hir().opt_def_kind(def_id.expect_local()); + providers.all_local_trait_impls = |tcx, ()| &tcx.hir_crate(()).trait_impls; + providers.expn_that_defined = |tcx, id| { + let id = id.expect_local(); + tcx.definitions.expansion_that_defined(id) + }; } diff --git a/compiler/rustc_middle/src/ich/impls_syntax.rs b/compiler/rustc_middle/src/ich/impls_syntax.rs index f53992172b..acf2990b64 100644 --- a/compiler/rustc_middle/src/ich/impls_syntax.rs +++ b/compiler/rustc_middle/src/ich/impls_syntax.rs @@ -6,6 +6,7 @@ use crate::ich::StableHashingContext; use rustc_ast as ast; use rustc_data_structures::stable_hasher::{HashStable, StableHasher}; use rustc_span::{BytePos, NormalizedPos, SourceFile}; +use std::assert_matches::assert_matches; use smallvec::SmallVec; @@ -45,7 +46,11 @@ impl<'ctx> rustc_ast::HashStableContext for StableHashingContext<'ctx> { item.hash_stable(self, hasher); style.hash_stable(self, hasher); span.hash_stable(self, hasher); - assert!(tokens.as_ref().is_none(), "Tokens should have been removed during lowering!"); + assert_matches!( + tokens.as_ref(), + None, + "Tokens should have been removed during lowering!" + ); } else { unreachable!(); } @@ -57,8 +62,6 @@ impl<'a> HashStable> for SourceFile { let SourceFile { name: _, // We hash the smaller name_hash instead of this name_hash, - name_was_remapped, - unmapped_path: _, cnum, // Do not hash the source as it is not encoded src: _, @@ -73,7 +76,6 @@ impl<'a> HashStable> for SourceFile { } = *self; (name_hash as u64).hash_stable(hcx, hasher); - name_was_remapped.hash_stable(hcx, hasher); src_hash.hash_stable(hcx, hasher); diff --git a/compiler/rustc_middle/src/infer/canonical.rs b/compiler/rustc_middle/src/infer/canonical.rs index 5df2f91f09..d764d45ba7 100644 --- a/compiler/rustc_middle/src/infer/canonical.rs +++ b/compiler/rustc_middle/src/infer/canonical.rs @@ -294,6 +294,7 @@ TrivialTypeFoldableImpls! { } impl<'tcx> CanonicalVarValues<'tcx> { + #[inline] pub fn len(&self) -> usize { self.var_values.len() } diff --git a/compiler/rustc_middle/src/infer/unify_key.rs b/compiler/rustc_middle/src/infer/unify_key.rs index 641cf23781..0b05dd5c0b 100644 --- a/compiler/rustc_middle/src/infer/unify_key.rs +++ b/compiler/rustc_middle/src/infer/unify_key.rs @@ -16,37 +16,48 @@ pub trait ToType { } #[derive(PartialEq, Copy, Clone, Debug)] -pub struct RegionVidKey { - /// The minimum region vid in the unification set. This is needed - /// to have a canonical name for a type to prevent infinite - /// recursion. - pub min_vid: ty::RegionVid, -} - -impl UnifyValue for RegionVidKey { - type Error = NoError; +pub struct UnifiedRegion<'tcx>(pub Option>); - fn unify_values(value1: &Self, value2: &Self) -> Result { - let min_vid = if value1.min_vid.index() < value2.min_vid.index() { - value1.min_vid - } else { - value2.min_vid - }; +#[derive(PartialEq, Copy, Clone, Debug)] +pub struct RegionVidKey<'tcx> { + pub vid: ty::RegionVid, + pub phantom: PhantomData>, +} - Ok(RegionVidKey { min_vid }) +impl<'tcx> From for RegionVidKey<'tcx> { + fn from(vid: ty::RegionVid) -> Self { + RegionVidKey { vid, phantom: PhantomData } } } -impl UnifyKey for ty::RegionVid { - type Value = RegionVidKey; +impl<'tcx> UnifyKey for RegionVidKey<'tcx> { + type Value = UnifiedRegion<'tcx>; fn index(&self) -> u32 { - u32::from(*self) + self.vid.as_u32() } - fn from_index(i: u32) -> ty::RegionVid { - ty::RegionVid::from(i) + fn from_index(i: u32) -> Self { + RegionVidKey::from(ty::RegionVid::from_u32(i)) } fn tag() -> &'static str { - "RegionVid" + "RegionVidKey" + } +} + +impl<'tcx> UnifyValue for UnifiedRegion<'tcx> { + type Error = NoError; + + fn unify_values(value1: &Self, value2: &Self) -> Result { + Ok(match (value1.0, value2.0) { + // Here we can just pick one value, because the full constraints graph + // will be handled later. Ideally, we might want a `MultipleValues` + // variant or something. For now though, this is fine. + (Some(_), Some(_)) => *value1, + + (Some(_), _) => *value1, + (_, Some(_)) => *value2, + + (None, None) => *value1, + }) } } diff --git a/compiler/rustc_middle/src/lib.rs b/compiler/rustc_middle/src/lib.rs index 2ff271f398..e1d7bc4be5 100644 --- a/compiler/rustc_middle/src/lib.rs +++ b/compiler/rustc_middle/src/lib.rs @@ -24,18 +24,17 @@ #![doc(html_root_url = "https://doc.rust-lang.org/nightly/nightly-rustc/")] #![feature(array_windows)] +#![feature(assert_matches)] #![feature(backtrace)] #![feature(bool_to_option)] #![feature(box_patterns)] #![feature(box_syntax)] -#![feature(const_panic)] #![feature(core_intrinsics)] #![feature(discriminant_kind)] #![feature(never_type)] #![feature(extern_types)] #![feature(nll)] #![feature(once_cell)] -#![cfg_attr(bootstrap, feature(or_patterns))] #![feature(min_specialization)] #![feature(trusted_len)] #![feature(test)] @@ -49,6 +48,7 @@ #![feature(control_flow_enum)] #![feature(associated_type_defaults)] #![feature(iter_zip)] +#![feature(thread_local_const_init)] #![recursion_limit = "512"] #[macro_use] @@ -81,6 +81,7 @@ pub mod infer; pub mod lint; pub mod middle; pub mod mir; +pub mod thir; pub mod traits; pub mod ty; diff --git a/compiler/rustc_middle/src/lint.rs b/compiler/rustc_middle/src/lint.rs index 26e61ec8cf..4c7ea937ce 100644 --- a/compiler/rustc_middle/src/lint.rs +++ b/compiler/rustc_middle/src/lint.rs @@ -1,7 +1,7 @@ use std::cmp; use crate::ich::StableHashingContext; -use rustc_data_structures::fx::FxHashMap; +use rustc_data_structures::fx::{FxHashMap, FxHashSet}; use rustc_data_structures::stable_hasher::{HashStable, StableHasher}; use rustc_errors::{DiagnosticBuilder, DiagnosticId}; use rustc_hir::HirId; @@ -28,6 +28,9 @@ pub enum LintLevelSource { /// The provided `Level` is the level specified on the command line. /// (The actual level may be lower due to `--cap-lints`.) CommandLine(Symbol, Level), + + /// Lint is being forced to warn no matter what. + ForceWarn(Symbol), } impl LintLevelSource { @@ -36,6 +39,7 @@ impl LintLevelSource { LintLevelSource::Default => symbol::kw::Default, LintLevelSource::Node(name, _, _) => name, LintLevelSource::CommandLine(name, _) => name, + LintLevelSource::ForceWarn(name) => name, } } @@ -44,6 +48,7 @@ impl LintLevelSource { LintLevelSource::Default => DUMMY_SP, LintLevelSource::Node(_, span, _) => span, LintLevelSource::CommandLine(_, _) => DUMMY_SP, + LintLevelSource::ForceWarn(_) => DUMMY_SP, } } } @@ -55,6 +60,7 @@ pub type LevelAndSource = (Level, LintLevelSource); pub struct LintLevelSets { pub list: Vec, pub lint_cap: Level, + pub force_warns: FxHashSet, } #[derive(Debug)] @@ -73,7 +79,11 @@ pub enum LintSet { impl LintLevelSets { pub fn new() -> Self { - LintLevelSets { list: Vec::new(), lint_cap: Level::Forbid } + LintLevelSets { + list: Vec::new(), + lint_cap: Level::Forbid, + force_warns: FxHashSet::default(), + } } pub fn get_lint_level( @@ -83,6 +93,11 @@ impl LintLevelSets { aux: Option<&FxHashMap>, sess: &Session, ) -> LevelAndSource { + // Check whether we should always warn + if self.force_warns.contains(&LintId::of(lint)) { + return (Level::Warn, LintLevelSource::ForceWarn(Symbol::intern(lint.name))); + } + let (level, mut src) = self.get_lint_id_level(LintId::of(lint), idx, aux); // If `level` is none then we actually assume the default level for this @@ -176,11 +191,11 @@ impl LintLevelMap { impl<'a> HashStable> for LintLevelMap { #[inline] fn hash_stable(&self, hcx: &mut StableHashingContext<'a>, hasher: &mut StableHasher) { - let LintLevelMap { ref sets, ref id_to_set } = *self; + let LintLevelMap { ref sets, ref id_to_set, .. } = *self; id_to_set.hash_stable(hcx, hasher); - let LintLevelSets { ref list, lint_cap } = *sets; + let LintLevelSets { ref list, lint_cap, .. } = *sets; lint_cap.hash_stable(hcx, hasher); @@ -272,11 +287,13 @@ pub fn struct_lint_level<'s, 'd>( // emit shouldn't be automatically fixed by rustfix. err.allow_suggestions(false); - // If this is a future incompatible lint it'll become a hard error, so - // we have to emit *something*. Also, if this lint occurs in the - // expansion of a macro from an external crate, allow individual lints - // to opt-out from being reported. - if future_incompatible.is_none() && !lint.report_in_external_macro { + // If this is a future incompatible that is not an edition fixing lint + // it'll become a hard error, so we have to emit *something*. Also, + // if this lint occurs in the expansion of a macro from an external crate, + // allow individual lints to opt-out from being reported. + let not_future_incompatible = + future_incompatible.map(|f| f.edition.is_some()).unwrap_or(true); + if not_future_incompatible && !lint.report_in_external_macro { err.cancel(); // Don't continue further, since we don't want to have // `diag_span_note_once` called for a diagnostic that isn't emitted. @@ -344,6 +361,13 @@ pub fn struct_lint_level<'s, 'd>( ); } } + LintLevelSource::ForceWarn(_) => { + sess.diag_note_once( + &mut err, + DiagnosticMessageId::from(lint), + "warning forced by `force-warns` commandline option", + ); + } } err.code(DiagnosticId::Lint { name, has_future_breakage }); @@ -387,7 +411,7 @@ pub fn in_external_macro(sess: &Session, span: Span) -> bool { false } ExpnKind::AstPass(_) | ExpnKind::Desugaring(_) => true, // well, it's "external" - ExpnKind::Macro(MacroKind::Bang, _) => { + ExpnKind::Macro { kind: MacroKind::Bang, name: _, proc_macro: _ } => { // Dummy span for the `def_site` means it's an external macro. expn_data.def_site.is_dummy() || sess.source_map().is_imported(expn_data.def_site) } diff --git a/compiler/rustc_middle/src/middle/cstore.rs b/compiler/rustc_middle/src/middle/cstore.rs index 4f1ca968c3..a7ab43d106 100644 --- a/compiler/rustc_middle/src/middle/cstore.rs +++ b/compiler/rustc_middle/src/middle/cstore.rs @@ -94,6 +94,14 @@ pub struct NativeLib { pub cfg: Option, pub foreign_module: Option, pub wasm_import_module: Option, + pub verbatim: Option, + pub dll_imports: Vec, +} + +#[derive(Clone, Debug, Encodable, Decodable, HashStable)] +pub struct DllImport { + pub name: Symbol, + pub ordinal: Option, } #[derive(Clone, TyEncodable, TyDecodable, HashStable, Debug)] @@ -198,7 +206,6 @@ pub trait CrateStore { // "queries" used in resolve that aren't tracked for incremental compilation fn crate_name_untracked(&self, cnum: CrateNum) -> Symbol; - fn crate_is_private_dep_untracked(&self, cnum: CrateNum) -> bool; fn crate_disambiguator_untracked(&self, cnum: CrateNum) -> CrateDisambiguator; fn crate_hash_untracked(&self, cnum: CrateNum) -> Svh; @@ -208,7 +215,6 @@ pub trait CrateStore { // utility functions fn encode_metadata(&self, tcx: TyCtxt<'_>) -> EncodedMetadata; - fn metadata_encoding_version(&self) -> &[u8]; fn allocator_kind(&self) -> Option; } @@ -250,7 +256,7 @@ pub fn used_crates(tcx: TyCtxt<'_>, prefer: LinkagePreference) -> Vec<(CrateNum, Some((cnum, path)) }) .collect::>(); - let mut ordering = tcx.postorder_cnums(LOCAL_CRATE).to_owned(); + let mut ordering = tcx.postorder_cnums(()).to_owned(); ordering.reverse(); libs.sort_by_cached_key(|&(a, _)| ordering.iter().position(|x| *x == a)); libs diff --git a/compiler/rustc_middle/src/middle/exported_symbols.rs b/compiler/rustc_middle/src/middle/exported_symbols.rs index 276e45ce99..288dd03944 100644 --- a/compiler/rustc_middle/src/middle/exported_symbols.rs +++ b/compiler/rustc_middle/src/middle/exported_symbols.rs @@ -49,7 +49,7 @@ impl<'tcx> ExportedSymbol<'tcx> { pub fn metadata_symbol_name(tcx: TyCtxt<'_>) -> String { format!( "rust_metadata_{}_{}", - tcx.original_crate_name(LOCAL_CRATE), + tcx.crate_name(LOCAL_CRATE), tcx.crate_disambiguator(LOCAL_CRATE).to_fingerprint().to_hex() ) } diff --git a/compiler/rustc_middle/src/middle/resolve_lifetime.rs b/compiler/rustc_middle/src/middle/resolve_lifetime.rs index aa6488b329..2665ea8d7f 100644 --- a/compiler/rustc_middle/src/middle/resolve_lifetime.rs +++ b/compiler/rustc_middle/src/middle/resolve_lifetime.rs @@ -49,6 +49,20 @@ pub enum Region { Free(DefId, /* lifetime decl */ DefId), } +/// This is used in diagnostics to improve suggestions for missing generic arguments. +/// It gives information on the type of lifetimes that are in scope for a particular `PathSegment`, +/// so that we can e.g. suggest elided-lifetimes-in-paths of the form <'_, '_> e.g. +#[derive(Clone, PartialEq, Eq, Hash, TyEncodable, TyDecodable, Debug, HashStable)] +pub enum LifetimeScopeForPath { + // Contains all lifetime names that are in scope and could possibly be used in generics + // arguments of path. + NonElided(Vec), + + // Information that allows us to suggest args of the form `<'_>` in case + // no generic arguments were provided for a path. + Elided, +} + /// A set containing, at most, one known element. /// If two distinct values are inserted into a set, then it /// becomes `Many`, which can be used to detect ambiguities. diff --git a/compiler/rustc_middle/src/middle/stability.rs b/compiler/rustc_middle/src/middle/stability.rs index fc9a2970e0..aa0f18846d 100644 --- a/compiler/rustc_middle/src/middle/stability.rs +++ b/compiler/rustc_middle/src/middle/stability.rs @@ -281,7 +281,13 @@ impl<'tcx> TyCtxt<'tcx> { /// If `id` is `Some(_)`, this function will also check if the item at `def_id` has been /// deprecated. If the item is indeed deprecated, we will emit a deprecation lint attached to /// `id`. - pub fn eval_stability(self, def_id: DefId, id: Option, span: Span) -> EvalResult { + pub fn eval_stability( + self, + def_id: DefId, + id: Option, + span: Span, + method_span: Option, + ) -> EvalResult { // Deprecated attributes apply in-crate and cross-crate. if let Some(id) = id { if let Some(depr_entry) = self.lookup_deprecation_entry(def_id) { @@ -300,6 +306,7 @@ impl<'tcx> TyCtxt<'tcx> { let path = &with_no_trimmed_paths(|| self.def_path_str(def_id)); let kind = self.def_kind(def_id).descr(def_id); let (message, lint) = deprecation_message(&depr_entry.attr, kind, path); + let span = method_span.unwrap_or(span); late_report_deprecation( self, &message, @@ -382,8 +389,14 @@ impl<'tcx> TyCtxt<'tcx> { /// /// This function will also check if the item is deprecated. /// If so, and `id` is not `None`, a deprecated lint attached to `id` will be emitted. - pub fn check_stability(self, def_id: DefId, id: Option, span: Span) { - self.check_optional_stability(def_id, id, span, |span, def_id| { + pub fn check_stability( + self, + def_id: DefId, + id: Option, + span: Span, + method_span: Option, + ) { + self.check_optional_stability(def_id, id, span, method_span, |span, def_id| { // The API could be uncallable for other reasons, for example when a private module // was referenced. self.sess.delay_span_bug(span, &format!("encountered unmarked API: {:?}", def_id)); @@ -399,6 +412,7 @@ impl<'tcx> TyCtxt<'tcx> { def_id: DefId, id: Option, span: Span, + method_span: Option, unmarked: impl FnOnce(Span, DefId), ) { let soft_handler = |lint, span, msg: &_| { @@ -406,7 +420,7 @@ impl<'tcx> TyCtxt<'tcx> { lint.build(msg).emit() }) }; - match self.eval_stability(def_id, id, span) { + match self.eval_stability(def_id, id, span, method_span) { EvalResult::Allow => {} EvalResult::Deny { feature, reason, issue, is_soft } => { report_unstable(self.sess, feature, reason, issue, is_soft, span, soft_handler) diff --git a/compiler/rustc_middle/src/mir/interpret/allocation.rs b/compiler/rustc_middle/src/mir/interpret/allocation.rs index 898c375e9a..ee3902991e 100644 --- a/compiler/rustc_middle/src/mir/interpret/allocation.rs +++ b/compiler/rustc_middle/src/mir/interpret/allocation.rs @@ -4,16 +4,22 @@ use std::borrow::Cow; use std::convert::TryFrom; use std::iter; use std::ops::{Deref, DerefMut, Range}; +use std::ptr; use rustc_ast::Mutability; use rustc_data_structures::sorted_map::SortedMap; use rustc_target::abi::{Align, HasDataLayout, Size}; use super::{ - read_target_uint, write_target_uint, AllocId, InterpResult, Pointer, Scalar, ScalarMaybeUninit, - UninitBytesAccess, + read_target_uint, write_target_uint, AllocId, InterpError, Pointer, Scalar, ScalarMaybeUninit, + UndefinedBehaviorInfo, UninitBytesAccess, UnsupportedOpInfo, }; +/// This type represents an Allocation in the Miri/CTFE core engine. +/// +/// Its public API is rather low-level, working directly with allocation offsets and a custom error +/// type to account for the lack of an AllocId on this level. The Miri/CTFE core engine `memory` +/// module provides higher-level access. #[derive(Clone, Debug, Eq, PartialEq, PartialOrd, Ord, Hash, TyEncodable, TyDecodable)] #[derive(HashStable)] pub struct Allocation { @@ -27,8 +33,6 @@ pub struct Allocation { relocations: Relocations, /// Denotes which part of this allocation is initialized. init_mask: InitMask, - /// The size of the allocation. Currently, must always equal `bytes.len()`. - pub size: Size, /// The alignment of the allocation to detect unaligned reads. /// (`Align` guarantees that this is a power of two.) pub align: Align, @@ -40,69 +44,81 @@ pub struct Allocation { pub extra: Extra, } -pub trait AllocationExtra: std::fmt::Debug + Clone { - // There is no constructor in here because the constructor's type depends - // on `MemoryKind`, and making things sufficiently generic leads to painful - // inference failure. +/// We have our own error type that does not know about the `AllocId`; that information +/// is added when converting to `InterpError`. +#[derive(Debug)] +pub enum AllocError { + /// Encountered a pointer where we needed raw bytes. + ReadPointerAsBytes, + /// Using uninitialized data where it is not allowed. + InvalidUninitBytes(Option), +} +pub type AllocResult = Result; - /// Hook for performing extra checks on a memory read access. - /// - /// Takes read-only access to the allocation so we can keep all the memory read - /// operations take `&self`. Use a `RefCell` in `AllocExtra` if you - /// need to mutate. - #[inline(always)] - fn memory_read( - _alloc: &Allocation, - _ptr: Pointer, - _size: Size, - ) -> InterpResult<'tcx> { - Ok(()) +impl AllocError { + pub fn to_interp_error<'tcx>(self, alloc_id: AllocId) -> InterpError<'tcx> { + match self { + AllocError::ReadPointerAsBytes => { + InterpError::Unsupported(UnsupportedOpInfo::ReadPointerAsBytes) + } + AllocError::InvalidUninitBytes(info) => InterpError::UndefinedBehavior( + UndefinedBehaviorInfo::InvalidUninitBytes(info.map(|b| (alloc_id, b))), + ), + } } +} + +/// The information that makes up a memory access: offset and size. +#[derive(Copy, Clone, Debug)] +pub struct AllocRange { + pub start: Size, + pub size: Size, +} - /// Hook for performing extra checks on a memory write access. +/// Free-starting constructor for less syntactic overhead. +#[inline(always)] +pub fn alloc_range(start: Size, size: Size) -> AllocRange { + AllocRange { start, size } +} + +impl AllocRange { #[inline(always)] - fn memory_written( - _alloc: &mut Allocation, - _ptr: Pointer, - _size: Size, - ) -> InterpResult<'tcx> { - Ok(()) + pub fn end(self) -> Size { + self.start + self.size // This does overflow checking. } - /// Hook for performing extra checks on a memory deallocation. - /// `size` will be the size of the allocation. - #[inline(always)] - fn memory_deallocated( - _alloc: &mut Allocation, - _ptr: Pointer, - _size: Size, - ) -> InterpResult<'tcx> { - Ok(()) + /// Returns the `subrange` within this range; panics if it is not a subrange. + #[inline] + pub fn subrange(self, subrange: AllocRange) -> AllocRange { + let sub_start = self.start + subrange.start; + let range = alloc_range(sub_start, subrange.size); + assert!(range.end() <= self.end(), "access outside the bounds for given AllocRange"); + range } } -// For `Tag = ()` and no extra state, we have a trivial implementation. -impl AllocationExtra<()> for () {} - // The constructors are all without extra; the extra gets added by a machine hook later. impl Allocation { - /// Creates a read-only allocation initialized by the given bytes - pub fn from_bytes<'a>(slice: impl Into>, align: Align) -> Self { + /// Creates an allocation initialized by the given bytes + pub fn from_bytes<'a>( + slice: impl Into>, + align: Align, + mutability: Mutability, + ) -> Self { let bytes = slice.into().into_owned(); let size = Size::from_bytes(bytes.len()); Self { bytes, relocations: Relocations::new(), init_mask: InitMask::new(size, true), - size, align, - mutability: Mutability::Not, + mutability, extra: (), } } - pub fn from_byte_aligned_bytes<'a>(slice: impl Into>) -> Self { - Allocation::from_bytes(slice, Align::from_bytes(1).unwrap()) + pub fn from_bytes_byte_aligned_immutable<'a>(slice: impl Into>) -> Self { + Allocation::from_bytes(slice, Align::ONE, Mutability::Not) } pub fn uninit(size: Size, align: Align) -> Self { @@ -110,7 +126,6 @@ impl Allocation { bytes: vec![0; size.bytes_usize()], relocations: Relocations::new(), init_mask: InitMask::new(size, false), - size, align, mutability: Mutability::Mut, extra: (), @@ -118,7 +133,7 @@ impl Allocation { } } -impl Allocation<(), ()> { +impl Allocation<()> { /// Add Tag and Extra fields pub fn with_tags_and_extra( self, @@ -127,7 +142,6 @@ impl Allocation<(), ()> { ) -> Allocation { Allocation { bytes: self.bytes, - size: self.size, relocations: Relocations::from_presorted( self.relocations .iter() @@ -150,12 +164,16 @@ impl Allocation<(), ()> { /// Raw accessors. Provide access to otherwise private bytes. impl Allocation { pub fn len(&self) -> usize { - self.size.bytes_usize() + self.bytes.len() + } + + pub fn size(&self) -> Size { + Size::from_bytes(self.len()) } /// Looks at a slice which may describe uninitialized bytes or describe a relocation. This differs /// from `get_bytes_with_uninit_and_ptr` in that it does no relocation checks (even on the - /// edges) at all. It further ignores `AllocationExtra` callbacks. + /// edges) at all. /// This must not be used for reads affecting the interpreter execution. pub fn inspect_with_uninit_and_ptr_outside_interpreter(&self, range: Range) -> &[u8] { &self.bytes[range] @@ -173,23 +191,7 @@ impl Allocation { } /// Byte accessors. -impl<'tcx, Tag: Copy, Extra: AllocationExtra> Allocation { - /// Just a small local helper function to avoid a bit of code repetition. - /// Returns the range of this allocation that was meant. - #[inline] - fn check_bounds(&self, offset: Size, size: Size) -> Range { - let end = offset + size; // This does overflow checking. - let end = usize::try_from(end.bytes()).expect("access too big for this host architecture"); - assert!( - end <= self.len(), - "Out-of-bounds access at offset {}, size {} in allocation of size {}", - offset.bytes(), - size.bytes(), - self.len() - ); - offset.bytes_usize()..end - } - +impl Allocation { /// The last argument controls whether we error out when there are uninitialized /// or pointer bytes. You should never call this, call `get_bytes` or /// `get_bytes_with_uninit_and_ptr` instead, @@ -202,23 +204,18 @@ impl<'tcx, Tag: Copy, Extra: AllocationExtra> Allocation { fn get_bytes_internal( &self, cx: &impl HasDataLayout, - ptr: Pointer, - size: Size, + range: AllocRange, check_init_and_ptr: bool, - ) -> InterpResult<'tcx, &[u8]> { - let range = self.check_bounds(ptr.offset, size); - + ) -> AllocResult<&[u8]> { if check_init_and_ptr { - self.check_init(ptr, size)?; - self.check_relocations(cx, ptr, size)?; + self.check_init(range)?; + self.check_relocations(cx, range)?; } else { // We still don't want relocations on the *edges*. - self.check_relocation_edges(cx, ptr, size)?; + self.check_relocation_edges(cx, range)?; } - AllocationExtra::memory_read(self, ptr, size)?; - - Ok(&self.bytes[range]) + Ok(&self.bytes[range.start.bytes_usize()..range.end().bytes_usize()]) } /// Checks that these bytes are initialized and not pointer bytes, and then return them @@ -228,13 +225,8 @@ impl<'tcx, Tag: Copy, Extra: AllocationExtra> Allocation { /// Most likely, you want to use the `PlaceTy` and `OperandTy`-based methods /// on `InterpCx` instead. #[inline] - pub fn get_bytes( - &self, - cx: &impl HasDataLayout, - ptr: Pointer, - size: Size, - ) -> InterpResult<'tcx, &[u8]> { - self.get_bytes_internal(cx, ptr, size, true) + pub fn get_bytes(&self, cx: &impl HasDataLayout, range: AllocRange) -> AllocResult<&[u8]> { + self.get_bytes_internal(cx, range, true) } /// It is the caller's responsibility to handle uninitialized and pointer bytes. @@ -245,10 +237,9 @@ impl<'tcx, Tag: Copy, Extra: AllocationExtra> Allocation { pub fn get_bytes_with_uninit_and_ptr( &self, cx: &impl HasDataLayout, - ptr: Pointer, - size: Size, - ) -> InterpResult<'tcx, &[u8]> { - self.get_bytes_internal(cx, ptr, size, false) + range: AllocRange, + ) -> AllocResult<&[u8]> { + self.get_bytes_internal(cx, range, false) } /// Just calling this already marks everything as defined and removes relocations, @@ -257,89 +248,43 @@ impl<'tcx, Tag: Copy, Extra: AllocationExtra> Allocation { /// It is the caller's responsibility to check bounds and alignment beforehand. /// Most likely, you want to use the `PlaceTy` and `OperandTy`-based methods /// on `InterpCx` instead. - pub fn get_bytes_mut( - &mut self, - cx: &impl HasDataLayout, - ptr: Pointer, - size: Size, - ) -> InterpResult<'tcx, &mut [u8]> { - let range = self.check_bounds(ptr.offset, size); + pub fn get_bytes_mut(&mut self, cx: &impl HasDataLayout, range: AllocRange) -> &mut [u8] { + self.mark_init(range, true); + self.clear_relocations(cx, range); - self.mark_init(ptr, size, true); - self.clear_relocations(cx, ptr, size); + &mut self.bytes[range.start.bytes_usize()..range.end().bytes_usize()] + } - AllocationExtra::memory_written(self, ptr, size)?; + /// A raw pointer variant of `get_bytes_mut` that avoids invalidating existing aliases into this memory. + pub fn get_bytes_mut_ptr(&mut self, cx: &impl HasDataLayout, range: AllocRange) -> *mut [u8] { + self.mark_init(range, true); + self.clear_relocations(cx, range); - Ok(&mut self.bytes[range]) + assert!(range.end().bytes_usize() <= self.bytes.len()); // need to do our own bounds-check + let begin_ptr = self.bytes.as_mut_ptr().wrapping_add(range.start.bytes_usize()); + let len = range.end().bytes_usize() - range.start.bytes_usize(); + ptr::slice_from_raw_parts_mut(begin_ptr, len) } } /// Reading and writing. -impl<'tcx, Tag: Copy, Extra: AllocationExtra> Allocation { - /// Reads bytes until a `0` is encountered. Will error if the end of the allocation is reached - /// before a `0` is found. - /// - /// Most likely, you want to call `Memory::read_c_str` instead of this method. - pub fn read_c_str( - &self, - cx: &impl HasDataLayout, - ptr: Pointer, - ) -> InterpResult<'tcx, &[u8]> { - let offset = ptr.offset.bytes_usize(); - Ok(match self.bytes[offset..].iter().position(|&c| c == 0) { - Some(size) => { - let size_with_null = Size::from_bytes(size) + Size::from_bytes(1); - // Go through `get_bytes` for checks and AllocationExtra hooks. - // We read the null, so we include it in the request, but we want it removed - // from the result, so we do subslicing. - &self.get_bytes(cx, ptr, size_with_null)?[..size] - } - // This includes the case where `offset` is out-of-bounds to begin with. - None => throw_ub!(UnterminatedCString(ptr.erase_tag())), - }) - } - +impl Allocation { /// Validates that `ptr.offset` and `ptr.offset + size` do not point to the middle of a /// relocation. If `allow_uninit_and_ptr` is `false`, also enforces that the memory in the /// given range contains neither relocations nor uninitialized bytes. pub fn check_bytes( &self, cx: &impl HasDataLayout, - ptr: Pointer, - size: Size, + range: AllocRange, allow_uninit_and_ptr: bool, - ) -> InterpResult<'tcx> { + ) -> AllocResult { // Check bounds and relocations on the edges. - self.get_bytes_with_uninit_and_ptr(cx, ptr, size)?; + self.get_bytes_with_uninit_and_ptr(cx, range)?; // Check uninit and ptr. if !allow_uninit_and_ptr { - self.check_init(ptr, size)?; - self.check_relocations(cx, ptr, size)?; - } - Ok(()) - } - - /// Writes `src` to the memory starting at `ptr.offset`. - /// - /// It is the caller's responsibility to check bounds and alignment beforehand. - /// Most likely, you want to call `Memory::write_bytes` instead of this method. - pub fn write_bytes( - &mut self, - cx: &impl HasDataLayout, - ptr: Pointer, - src: impl IntoIterator, - ) -> InterpResult<'tcx> { - let mut src = src.into_iter(); - let (lower, upper) = src.size_hint(); - let len = upper.expect("can only write bounded iterators"); - assert_eq!(lower, len, "can only write iterators with a precise length"); - let bytes = self.get_bytes_mut(cx, ptr, Size::from_bytes(len))?; - // `zip` would stop when the first iterator ends; we want to definitely - // cover all of `bytes`. - for dest in bytes { - *dest = src.next().expect("iterator was shorter than it said it would be"); + self.check_init(range)?; + self.check_relocations(cx, range)?; } - assert!(src.next().is_none(), "iterator was longer than it said it would be"); Ok(()) } @@ -353,14 +298,13 @@ impl<'tcx, Tag: Copy, Extra: AllocationExtra> Allocation { pub fn read_scalar( &self, cx: &impl HasDataLayout, - ptr: Pointer, - size: Size, - ) -> InterpResult<'tcx, ScalarMaybeUninit> { + range: AllocRange, + ) -> AllocResult> { // `get_bytes_unchecked` tests relocation edges. - let bytes = self.get_bytes_with_uninit_and_ptr(cx, ptr, size)?; + let bytes = self.get_bytes_with_uninit_and_ptr(cx, range)?; // Uninit check happens *after* we established that the alignment is correct. // We must not return `Ok()` for unaligned pointers! - if self.is_init(ptr, size).is_err() { + if self.is_init(range).is_err() { // This inflates uninitialized bytes to the entire scalar, even if only a few // bytes are uninitialized. return Ok(ScalarMaybeUninit::Uninit); @@ -368,29 +312,19 @@ impl<'tcx, Tag: Copy, Extra: AllocationExtra> Allocation { // Now we do the actual reading. let bits = read_target_uint(cx.data_layout().endian, bytes).unwrap(); // See if we got a pointer. - if size != cx.data_layout().pointer_size { + if range.size != cx.data_layout().pointer_size { + // Not a pointer. // *Now*, we better make sure that the inside is free of relocations too. - self.check_relocations(cx, ptr, size)?; + self.check_relocations(cx, range)?; } else { - if let Some(&(tag, alloc_id)) = self.relocations.get(&ptr.offset) { + // Maybe a pointer. + if let Some(&(tag, alloc_id)) = self.relocations.get(&range.start) { let ptr = Pointer::new_with_tag(alloc_id, Size::from_bytes(bits), tag); return Ok(ScalarMaybeUninit::Scalar(ptr.into())); } } // We don't. Just return the bits. - Ok(ScalarMaybeUninit::Scalar(Scalar::from_uint(bits, size))) - } - - /// Reads a pointer-sized scalar. - /// - /// It is the caller's responsibility to check bounds and alignment beforehand. - /// Most likely, you want to call `InterpCx::read_scalar` instead of this method. - pub fn read_ptr_sized( - &self, - cx: &impl HasDataLayout, - ptr: Pointer, - ) -> InterpResult<'tcx, ScalarMaybeUninit> { - self.read_scalar(cx, ptr, cx.data_layout().pointer_size) + Ok(ScalarMaybeUninit::Scalar(Scalar::from_uint(bits, range.size))) } /// Writes a *non-ZST* scalar. @@ -403,78 +337,56 @@ impl<'tcx, Tag: Copy, Extra: AllocationExtra> Allocation { pub fn write_scalar( &mut self, cx: &impl HasDataLayout, - ptr: Pointer, + range: AllocRange, val: ScalarMaybeUninit, - type_size: Size, - ) -> InterpResult<'tcx> { + ) -> AllocResult { let val = match val { ScalarMaybeUninit::Scalar(scalar) => scalar, ScalarMaybeUninit::Uninit => { - self.mark_init(ptr, type_size, false); + self.mark_init(range, false); return Ok(()); } }; - let bytes = match val.to_bits_or_ptr(type_size, cx) { + let bytes = match val.to_bits_or_ptr(range.size, cx) { Err(val) => u128::from(val.offset.bytes()), Ok(data) => data, }; let endian = cx.data_layout().endian; - let dst = self.get_bytes_mut(cx, ptr, type_size)?; + let dst = self.get_bytes_mut(cx, range); write_target_uint(endian, dst, bytes).unwrap(); // See if we have to also write a relocation. if let Scalar::Ptr(val) = val { - self.relocations.insert(ptr.offset, (val.tag, val.alloc_id)); + self.relocations.insert(range.start, (val.tag, val.alloc_id)); } Ok(()) } - - /// Writes a pointer-sized scalar. - /// - /// It is the caller's responsibility to check bounds and alignment beforehand. - /// Most likely, you want to call `InterpCx::write_scalar` instead of this method. - pub fn write_ptr_sized( - &mut self, - cx: &impl HasDataLayout, - ptr: Pointer, - val: ScalarMaybeUninit, - ) -> InterpResult<'tcx> { - let ptr_size = cx.data_layout().pointer_size; - self.write_scalar(cx, ptr, val, ptr_size) - } } /// Relocations. -impl<'tcx, Tag: Copy, Extra> Allocation { +impl Allocation { /// Returns all relocations overlapping with the given pointer-offset pair. pub fn get_relocations( &self, cx: &impl HasDataLayout, - ptr: Pointer, - size: Size, + range: AllocRange, ) -> &[(Size, (Tag, AllocId))] { // We have to go back `pointer_size - 1` bytes, as that one would still overlap with // the beginning of this range. - let start = ptr.offset.bytes().saturating_sub(cx.data_layout().pointer_size.bytes() - 1); - let end = ptr.offset + size; // This does overflow checking. - self.relocations.range(Size::from_bytes(start)..end) + let start = range.start.bytes().saturating_sub(cx.data_layout().pointer_size.bytes() - 1); + self.relocations.range(Size::from_bytes(start)..range.end()) } /// Checks that there are no relocations overlapping with the given range. #[inline(always)] - fn check_relocations( - &self, - cx: &impl HasDataLayout, - ptr: Pointer, - size: Size, - ) -> InterpResult<'tcx> { - if self.get_relocations(cx, ptr, size).is_empty() { + fn check_relocations(&self, cx: &impl HasDataLayout, range: AllocRange) -> AllocResult { + if self.get_relocations(cx, range).is_empty() { Ok(()) } else { - throw_unsup!(ReadPointerAsBytes) + Err(AllocError::ReadPointerAsBytes) } } @@ -484,11 +396,11 @@ impl<'tcx, Tag: Copy, Extra> Allocation { /// uninitialized. This is a somewhat odd "spooky action at a distance", /// but it allows strictly more code to run than if we would just error /// immediately in that case. - fn clear_relocations(&mut self, cx: &impl HasDataLayout, ptr: Pointer, size: Size) { + fn clear_relocations(&mut self, cx: &impl HasDataLayout, range: AllocRange) { // Find the start and end of the given range and its outermost relocations. let (first, last) = { // Find all relocations overlapping the given range. - let relocations = self.get_relocations(cx, ptr, size); + let relocations = self.get_relocations(cx, range); if relocations.is_empty() { return; } @@ -498,8 +410,8 @@ impl<'tcx, Tag: Copy, Extra> Allocation { relocations.last().unwrap().0 + cx.data_layout().pointer_size, ) }; - let start = ptr.offset; - let end = start + size; // `Size` addition + let start = range.start; + let end = range.end(); // Mark parts of the outermost relocations as uninitialized if they partially fall outside the // given range. @@ -517,46 +429,41 @@ impl<'tcx, Tag: Copy, Extra> Allocation { /// Errors if there are relocations overlapping with the edges of the /// given memory range. #[inline] - fn check_relocation_edges( - &self, - cx: &impl HasDataLayout, - ptr: Pointer, - size: Size, - ) -> InterpResult<'tcx> { - self.check_relocations(cx, ptr, Size::ZERO)?; - self.check_relocations(cx, ptr.offset(size, cx)?, Size::ZERO)?; + fn check_relocation_edges(&self, cx: &impl HasDataLayout, range: AllocRange) -> AllocResult { + self.check_relocations(cx, alloc_range(range.start, Size::ZERO))?; + self.check_relocations(cx, alloc_range(range.end(), Size::ZERO))?; Ok(()) } } /// Uninitialized bytes. -impl<'tcx, Tag: Copy, Extra> Allocation { +impl Allocation { /// Checks whether the given range is entirely initialized. /// /// Returns `Ok(())` if it's initialized. Otherwise returns the range of byte /// indexes of the first contiguous uninitialized access. - fn is_init(&self, ptr: Pointer, size: Size) -> Result<(), Range> { - self.init_mask.is_range_initialized(ptr.offset, ptr.offset + size) // `Size` addition + fn is_init(&self, range: AllocRange) -> Result<(), Range> { + self.init_mask.is_range_initialized(range.start, range.end()) // `Size` addition } /// Checks that a range of bytes is initialized. If not, returns the `InvalidUninitBytes` /// error which will report the first range of bytes which is uninitialized. - fn check_init(&self, ptr: Pointer, size: Size) -> InterpResult<'tcx> { - self.is_init(ptr, size).or_else(|idx_range| { - throw_ub!(InvalidUninitBytes(Some(UninitBytesAccess { - access_ptr: ptr.erase_tag(), - access_size: size, - uninit_ptr: Pointer::new(ptr.alloc_id, idx_range.start), + fn check_init(&self, range: AllocRange) -> AllocResult { + self.is_init(range).or_else(|idx_range| { + Err(AllocError::InvalidUninitBytes(Some(UninitBytesAccess { + access_offset: range.start, + access_size: range.size, + uninit_offset: idx_range.start, uninit_size: idx_range.end - idx_range.start, // `Size` subtraction }))) }) } - pub fn mark_init(&mut self, ptr: Pointer, size: Size, is_init: bool) { - if size.bytes() == 0 { + pub fn mark_init(&mut self, range: AllocRange, is_init: bool) { + if range.size.bytes() == 0 { return; } - self.init_mask.set_range(ptr.offset, ptr.offset + size, is_init); + self.init_mask.set_range(range.start, range.end(), is_init); } } @@ -691,25 +598,25 @@ impl Allocation { pub fn prepare_relocation_copy( &self, cx: &impl HasDataLayout, - src: Pointer, - size: Size, - dest: Pointer, - length: u64, + src: AllocRange, + dest: Size, + count: u64, ) -> AllocationRelocations { - let relocations = self.get_relocations(cx, src, size); + let relocations = self.get_relocations(cx, src); if relocations.is_empty() { return AllocationRelocations { relative_relocations: Vec::new() }; } - let mut new_relocations = Vec::with_capacity(relocations.len() * (length as usize)); + let size = src.size; + let mut new_relocations = Vec::with_capacity(relocations.len() * (count as usize)); - for i in 0..length { + for i in 0..count { new_relocations.extend(relocations.iter().map(|&(offset, reloc)| { // compute offset for current repetition - let dest_offset = dest.offset + size * i; // `Size` operations + let dest_offset = dest + size * i; // `Size` operations ( // shift offsets from source allocation to destination allocation - (offset + dest_offset) - src.offset, // `Size` operations + (offset + dest_offset) - src.start, // `Size` operations reloc, ) })); diff --git a/compiler/rustc_middle/src/mir/interpret/error.rs b/compiler/rustc_middle/src/mir/interpret/error.rs index cc0df12743..65d9c1dd90 100644 --- a/compiler/rustc_middle/src/mir/interpret/error.rs +++ b/compiler/rustc_middle/src/mir/interpret/error.rs @@ -170,24 +170,25 @@ impl fmt::Display for InvalidProgramInfo<'_> { /// Details of why a pointer had to be in-bounds. #[derive(Debug, Copy, Clone, TyEncodable, TyDecodable, HashStable)] pub enum CheckInAllocMsg { + /// We are access memory. MemoryAccessTest, - NullPointerTest, + /// We are doing pointer arithmetic. PointerArithmeticTest, + /// None of the above -- generic/unspecific inbounds test. InboundsTest, } impl fmt::Display for CheckInAllocMsg { /// When this is printed as an error the context looks like this - /// "{test name} failed: pointer must be in-bounds at offset..." + /// "{msg}pointer must be in-bounds at offset..." fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!( f, "{}", match *self { - CheckInAllocMsg::MemoryAccessTest => "memory access", - CheckInAllocMsg::NullPointerTest => "NULL pointer test", - CheckInAllocMsg::PointerArithmeticTest => "pointer arithmetic", - CheckInAllocMsg::InboundsTest => "inbounds test", + CheckInAllocMsg::MemoryAccessTest => "memory access failed: ", + CheckInAllocMsg::PointerArithmeticTest => "pointer arithmetic failed: ", + CheckInAllocMsg::InboundsTest => "", } ) } @@ -197,11 +198,11 @@ impl fmt::Display for CheckInAllocMsg { #[derive(Debug)] pub struct UninitBytesAccess { /// Location of the original memory access. - pub access_ptr: Pointer, + pub access_offset: Size, /// Size of the original memory access. pub access_size: Size, /// Location of the first uninitialized byte that was accessed. - pub uninit_ptr: Pointer, + pub uninit_offset: Size, /// Number of consecutive uninitialized bytes that were accessed. pub uninit_size: Size, } @@ -263,7 +264,7 @@ pub enum UndefinedBehaviorInfo<'tcx> { /// Using a string that is not valid UTF-8, InvalidStr(std::str::Utf8Error), /// Using uninitialized data where it is not allowed. - InvalidUninitBytes(Option), + InvalidUninitBytes(Option<(AllocId, UninitBytesAccess)>), /// Working with a local that is not currently live. DeadLocal, /// Data size is not equal to target size. @@ -301,18 +302,18 @@ impl fmt::Display for UndefinedBehaviorInfo<'_> { } PointerOutOfBounds { ptr, msg, allocation_size } => write!( f, - "{} failed: pointer must be in-bounds at offset {}, \ + "{}pointer must be in-bounds at offset {}, \ but is outside bounds of {} which has size {}", msg, ptr.offset.bytes(), ptr.alloc_id, allocation_size.bytes() ), - DanglingIntPointer(_, CheckInAllocMsg::NullPointerTest) => { - write!(f, "NULL pointer is not allowed for this operation") + DanglingIntPointer(0, CheckInAllocMsg::InboundsTest) => { + write!(f, "null pointer is not a valid pointer for this operation") } DanglingIntPointer(i, msg) => { - write!(f, "{} failed: 0x{:x} is not a valid pointer", msg, i) + write!(f, "{}0x{:x} is not a valid pointer", msg, i) } AlignmentCheckFailed { required, has } => write!( f, @@ -334,18 +335,18 @@ impl fmt::Display for UndefinedBehaviorInfo<'_> { write!(f, "using {} as function pointer but it does not point to a function", p) } InvalidStr(err) => write!(f, "this string is not valid UTF-8: {}", err), - InvalidUninitBytes(Some(access)) => write!( + InvalidUninitBytes(Some((alloc, access))) => write!( f, "reading {} byte{} of memory starting at {}, \ but {} byte{} {} uninitialized starting at {}, \ and this operation requires initialized memory", access.access_size.bytes(), pluralize!(access.access_size.bytes()), - access.access_ptr, + Pointer::new(*alloc, access.access_offset), access.uninit_size.bytes(), pluralize!(access.uninit_size.bytes()), if access.uninit_size.bytes() != 1 { "are" } else { "is" }, - access.uninit_ptr, + Pointer::new(*alloc, access.uninit_offset), ), InvalidUninitBytes(None) => write!( f, @@ -434,8 +435,12 @@ impl AsAny for T { } /// A trait for machine-specific errors (or other "machine stop" conditions). -pub trait MachineStopType: AsAny + fmt::Display + Send {} -impl MachineStopType for String {} +pub trait MachineStopType: AsAny + fmt::Display + Send { + /// If `true`, emit a hard error instead of going through the `CONST_ERR` lint + fn is_hard_err(&self) -> bool { + false + } +} impl dyn MachineStopType { #[inline(always)] @@ -445,7 +450,7 @@ impl dyn MachineStopType { } #[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))] -static_assert_size!(InterpError<'_>, 72); +static_assert_size!(InterpError<'_>, 64); pub enum InterpError<'tcx> { /// The program caused undefined behavior. diff --git a/compiler/rustc_middle/src/mir/interpret/mod.rs b/compiler/rustc_middle/src/mir/interpret/mod.rs index 55fe5f971e..14bdb0a5a2 100644 --- a/compiler/rustc_middle/src/mir/interpret/mod.rs +++ b/compiler/rustc_middle/src/mir/interpret/mod.rs @@ -125,7 +125,7 @@ pub use self::error::{ pub use self::value::{get_slice_bytes, ConstAlloc, ConstValue, Scalar, ScalarMaybeUninit}; -pub use self::allocation::{Allocation, AllocationExtra, InitMask, Relocations}; +pub use self::allocation::{alloc_range, AllocRange, Allocation, InitMask, Relocations}; pub use self::pointer::{Pointer, PointerArithmetic}; @@ -246,6 +246,7 @@ pub struct AllocDecodingState { } impl AllocDecodingState { + #[inline] pub fn new_decoding_session(&self) -> AllocDecodingSession<'_> { static DECODER_SESSION_ID: AtomicU32 = AtomicU32::new(0); let counter = DECODER_SESSION_ID.fetch_add(1, Ordering::SeqCst); diff --git a/compiler/rustc_middle/src/mir/interpret/value.rs b/compiler/rustc_middle/src/mir/interpret/value.rs index 888777a941..66ff6990e8 100644 --- a/compiler/rustc_middle/src/mir/interpret/value.rs +++ b/compiler/rustc_middle/src/mir/interpret/value.rs @@ -10,7 +10,7 @@ use rustc_target::abi::{HasDataLayout, Size, TargetDataLayout}; use crate::ty::{Lift, ParamEnv, ScalarInt, Ty, TyCtxt}; -use super::{AllocId, Allocation, InterpResult, Pointer, PointerArithmetic}; +use super::{AllocId, AllocRange, Allocation, InterpResult, Pointer, PointerArithmetic}; /// Represents the result of const evaluation via the `eval_to_allocation` query. #[derive(Copy, Clone, HashStable, TyEncodable, TyDecodable, Debug, Hash, Eq, PartialEq)] @@ -661,9 +661,7 @@ pub fn get_slice_bytes<'tcx>(cx: &impl HasDataLayout, val: ConstValue<'tcx>) -> let len = end - start; data.get_bytes( cx, - // invent a pointer, only the offset is relevant anyway - Pointer::new(AllocId(0), Size::from_bytes(start)), - Size::from_bytes(len), + AllocRange { start: Size::from_bytes(start), size: Size::from_bytes(len) }, ) .unwrap_or_else(|err| bug!("const slice is invalid: {:?}", err)) } else { diff --git a/compiler/rustc_middle/src/mir/mod.rs b/compiler/rustc_middle/src/mir/mod.rs index e22c0b40d5..7ae7eab6e5 100644 --- a/compiler/rustc_middle/src/mir/mod.rs +++ b/compiler/rustc_middle/src/mir/mod.rs @@ -461,7 +461,7 @@ impl<'tcx> Body<'tcx> { } #[inline] - pub fn predecessors(&self) -> impl std::ops::Deref + '_ { + pub fn predecessors(&self) -> &Predecessors { self.predecessor_cache.compute(&self.basic_blocks) } @@ -683,6 +683,15 @@ impl BorrowKind { BorrowKind::Mut { allow_two_phase_borrow } => allow_two_phase_borrow, } } + + pub fn describe_mutability(&self) -> String { + match *self { + BorrowKind::Shared | BorrowKind::Shallow | BorrowKind::Unique => { + "immutable".to_string() + } + BorrowKind::Mut { .. } => "mutable".to_string(), + } + } } /////////////////////////////////////////////////////////////////////////// @@ -1240,10 +1249,12 @@ impl<'tcx> BasicBlockData<'tcx> { /// /// Terminator may not be None after construction of the basic block is complete. This accessor /// provides a convenience way to reach the terminator. + #[inline] pub fn terminator(&self) -> &Terminator<'tcx> { self.terminator.as_ref().expect("invalid terminator state") } + #[inline] pub fn terminator_mut(&mut self) -> &mut Terminator<'tcx> { self.terminator.as_mut().expect("invalid terminator state") } @@ -1340,7 +1351,7 @@ impl AssertKind { } /// Format the message arguments for the `assert(cond, msg..)` terminator in MIR printing. - fn fmt_assert_args(&self, f: &mut W) -> fmt::Result + pub fn fmt_assert_args(&self, f: &mut W) -> fmt::Result where O: Debug, { @@ -1861,6 +1872,7 @@ impl<'tcx> PlaceRef<'tcx> { /// If this place represents a local variable like `_X` with no /// projections, return `Some(_X)`. + #[inline] pub fn as_local(&self) -> Option { match *self { PlaceRef { local, projection: [] } => Some(local), @@ -1868,6 +1880,7 @@ impl<'tcx> PlaceRef<'tcx> { } } + #[inline] pub fn last_projection(&self) -> Option<(PlaceRef<'tcx>, PlaceElem<'tcx>)> { if let &[ref proj_base @ .., elem] = self.projection { Some((PlaceRef { local: self.local, projection: proj_base }, elem)) @@ -2365,10 +2378,14 @@ impl<'tcx> Debug for Rvalue<'tcx> { ) } else { let span = tcx.hir().span(hir_id); - format!("[closure@{}]", tcx.sess.source_map().span_to_string(span)) + format!( + "[closure@{}]", + tcx.sess.source_map().span_to_diagnostic_string(span) + ) }; let mut struct_fmt = fmt.debug_struct(&name); + // FIXME(project-rfc-2229#48): This should be a list of capture names/places if let Some(upvars) = tcx.upvars_mentioned(def_id) { for (&var_id, place) in iter::zip(upvars.keys(), places) { let var_name = tcx.hir().name(var_id); @@ -2388,6 +2405,7 @@ impl<'tcx> Debug for Rvalue<'tcx> { let name = format!("[generator@{:?}]", tcx.hir().span(hir_id)); let mut struct_fmt = fmt.debug_struct(&name); + // FIXME(project-rfc-2229#48): This should be a list of capture names/places if let Some(upvars) = tcx.upvars_mentioned(def_id) { for (&var_id, place) in iter::zip(upvars.keys(), places) { let var_name = tcx.hir().name(var_id); @@ -2450,12 +2468,14 @@ impl Constant<'tcx> { _ => None, } } + #[inline] pub fn ty(&self) -> Ty<'tcx> { self.literal.ty() } } impl From<&'tcx ty::Const<'tcx>> for ConstantKind<'tcx> { + #[inline] fn from(ct: &'tcx ty::Const<'tcx>) -> Self { Self::Ty(ct) } @@ -2804,13 +2824,13 @@ impl<'a, 'b> graph::GraphSuccessors<'b> for Body<'a> { impl graph::GraphPredecessors<'graph> for Body<'tcx> { type Item = BasicBlock; - type Iter = smallvec::IntoIter<[BasicBlock; 4]>; + type Iter = std::iter::Copied>; } impl graph::WithPredecessors for Body<'tcx> { #[inline] fn predecessors(&self, node: Self::Node) -> >::Iter { - self.predecessors()[node].clone().into_iter() + self.predecessors()[node].iter().copied() } } diff --git a/compiler/rustc_middle/src/mir/mono.rs b/compiler/rustc_middle/src/mir/mono.rs index 77f38e52ad..74650f50a1 100644 --- a/compiler/rustc_middle/src/mir/mono.rs +++ b/compiler/rustc_middle/src/mir/mono.rs @@ -88,7 +88,7 @@ impl<'tcx> MonoItem<'tcx> { match *self { MonoItem::Fn(ref instance) => { - let entry_def_id = tcx.entry_fn(LOCAL_CRATE).map(|(id, _)| id); + let entry_def_id = tcx.entry_fn(()).map(|(id, _)| id); // If this function isn't inlined or otherwise has an extern // indicator, then we'll be creating a globally shared version. if tcx.codegen_fn_attrs(instance.def_id()).contains_extern_indicator() @@ -186,6 +186,15 @@ impl<'tcx> MonoItem<'tcx> { pub fn codegen_dep_node(&self, tcx: TyCtxt<'tcx>) -> DepNode { crate::dep_graph::make_compile_mono_item(tcx, self) } + + /// Returns the item's `CrateNum` + pub fn krate(&self) -> CrateNum { + match self { + MonoItem::Fn(ref instance) => instance.def_id().krate, + MonoItem::Static(def_id) => def_id.krate, + MonoItem::GlobalAsm(..) => LOCAL_CRATE, + } + } } impl<'a, 'tcx> HashStable> for MonoItem<'tcx> { @@ -229,6 +238,7 @@ pub struct CodegenUnit<'tcx> { name: Symbol, items: FxHashMap, (Linkage, Visibility)>, size_estimate: Option, + primary: bool, } /// Specifies the linkage type for a `MonoItem`. @@ -257,8 +267,9 @@ pub enum Visibility { } impl<'tcx> CodegenUnit<'tcx> { + #[inline] pub fn new(name: Symbol) -> CodegenUnit<'tcx> { - CodegenUnit { name, items: Default::default(), size_estimate: None } + CodegenUnit { name, items: Default::default(), size_estimate: None, primary: false } } pub fn name(&self) -> Symbol { @@ -269,6 +280,14 @@ impl<'tcx> CodegenUnit<'tcx> { self.name = name; } + pub fn is_primary(&self) -> bool { + self.primary + } + + pub fn make_primary(&mut self) { + self.primary = true; + } + pub fn items(&self) -> &FxHashMap, (Linkage, Visibility)> { &self.items } @@ -293,6 +312,7 @@ impl<'tcx> CodegenUnit<'tcx> { self.size_estimate = Some(self.items.keys().map(|mi| mi.size_estimate(tcx)).sum()); } + #[inline] pub fn size_estimate(&self) -> usize { // Should only be called if `estimate_size` has previously been called. self.size_estimate.expect("estimate_size must be called before getting a size_estimate") @@ -378,6 +398,7 @@ impl<'a, 'tcx> HashStable> for CodegenUnit<'tcx> { name, // The size estimate is not relevant to the hash size_estimate: _, + primary: _, } = *self; name.hash_stable(hcx, hasher); diff --git a/compiler/rustc_middle/src/mir/query.rs b/compiler/rustc_middle/src/mir/query.rs index fdd874c6f6..4fb737f463 100644 --- a/compiler/rustc_middle/src/mir/query.rs +++ b/compiler/rustc_middle/src/mir/query.rs @@ -2,13 +2,14 @@ use crate::mir::{abstract_const, Body, Promoted}; use crate::ty::{self, Ty, TyCtxt}; -use rustc_data_structures::fx::FxHashMap; use rustc_data_structures::sync::Lrc; +use rustc_data_structures::vec_map::VecMap; use rustc_errors::ErrorReported; use rustc_hir as hir; use rustc_hir::def_id::{DefId, LocalDefId}; use rustc_index::bit_set::BitMatrix; use rustc_index::vec::IndexVec; +use rustc_middle::ty::OpaqueTypeKey; use rustc_span::Span; use rustc_target::abi::VariantIdx; use smallvec::SmallVec; @@ -32,7 +33,6 @@ pub enum UnsafetyViolationDetails { UseOfInlineAssembly, InitializingTypeWith, CastOfPointerToInt, - BorrowOfPackedField, UseOfMutableStatic, UseOfExternStatic, DerefOfRawPointer, @@ -64,11 +64,6 @@ impl UnsafetyViolationDetails { CastOfPointerToInt => { ("cast of pointer to int", "casting pointers to integers in constants") } - BorrowOfPackedField => ( - "borrow of packed field", - "fields of packed structs might be misaligned: dereferencing a misaligned pointer \ - or even just creating a misaligned reference is undefined behavior", - ), UseOfMutableStatic => ( "use of mutable static", "mutable statics can be mutated by multiple threads: aliasing violations or data \ @@ -81,7 +76,7 @@ impl UnsafetyViolationDetails { ), DerefOfRawPointer => ( "dereference of raw pointer", - "raw pointers may be NULL, dangling or unaligned; they can violate aliasing rules \ + "raw pointers may be null, dangling or unaligned; they can violate aliasing rules \ and cause data races: all of these are undefined behavior", ), AssignToDroppingUnionField => ( @@ -216,7 +211,7 @@ pub struct BorrowCheckResult<'tcx> { /// All the opaque types that are restricted to concrete types /// by this function. Unlike the value in `TypeckResults`, this has /// unerased regions. - pub concrete_opaque_types: FxHashMap>, + pub concrete_opaque_types: VecMap, Ty<'tcx>>, pub closure_requirements: Option>, pub used_mut_upvars: SmallVec<[Field; 8]>, } diff --git a/compiler/rustc_middle/src/query/mod.rs b/compiler/rustc_middle/src/query/mod.rs index 7e62e10821..0860520ef9 100644 --- a/compiler/rustc_middle/src/query/mod.rs +++ b/compiler/rustc_middle/src/query/mod.rs @@ -20,7 +20,7 @@ rustc_queries! { /// This is because the `hir_crate` query gives you access to all other items. /// To avoid this fate, do not call `tcx.hir().krate()`; instead, /// prefer wrappers like `tcx.visit_all_items_in_krate()`. - query hir_crate(key: CrateNum) -> &'tcx Crate<'tcx> { + query hir_crate(key: ()) -> &'tcx Crate<'tcx> { eval_always no_hash desc { "get the crate HIR" } @@ -28,7 +28,7 @@ rustc_queries! { /// The indexed HIR. This can be conveniently accessed by `tcx.hir()`. /// Avoid calling this query directly. - query index_hir(_: CrateNum) -> &'tcx map::IndexedHir<'tcx> { + query index_hir(_: ()) -> &'tcx crate::hir::IndexedHir<'tcx> { eval_always no_hash desc { "index HIR" } @@ -47,11 +47,20 @@ rustc_queries! { /// /// This can be conveniently accessed by methods on `tcx.hir()`. /// Avoid calling this query directly. - query hir_owner(key: LocalDefId) -> Option<&'tcx crate::hir::Owner<'tcx>> { + query hir_owner(key: LocalDefId) -> Option> { eval_always desc { |tcx| "HIR owner of `{}`", tcx.def_path_str(key.to_def_id()) } } + /// Gives access to the HIR node's parent for the HIR owner `key`. + /// + /// This can be conveniently accessed by methods on `tcx.hir()`. + /// Avoid calling this query directly. + query hir_owner_parent(key: LocalDefId) -> hir::HirId { + eval_always + desc { |tcx| "HIR parent of `{}`", tcx.def_path_str(key.to_def_id()) } + } + /// Gives access to the HIR nodes and bodies inside the HIR owner `key`. /// /// This can be conveniently accessed by methods on `tcx.hir()`. @@ -105,7 +114,7 @@ rustc_queries! { cache_on_disk_if { key.is_local() } } - query analysis(key: CrateNum) -> Result<(), ErrorReported> { + query analysis(key: ()) -> Result<(), ErrorReported> { eval_always desc { "running analysis passes on this crate" } } @@ -190,7 +199,7 @@ rustc_queries! { desc { "looking up the native libraries of a linked crate" } } - query lint_levels(_: CrateNum) -> LintLevelMap { + query lint_levels(_: ()) -> LintLevelMap { storage(ArenaCacheSelector<'tcx>) eval_always desc { "computing the lint levels for items in this crate" } @@ -201,8 +210,8 @@ rustc_queries! { desc { |tcx| "parent module of `{}`", tcx.def_path_str(key.to_def_id()) } } - /// Internal helper query. Use `tcx.expansion_that_defined` instead query expn_that_defined(key: DefId) -> rustc_span::ExpnId { + eval_always desc { |tcx| "expansion that defined `{}`", tcx.def_path_str(key) } } @@ -211,10 +220,17 @@ rustc_queries! { desc { "checking if the crate is_panic_runtime" } } + /// Fetch the THIR for a given body. If typeck for that body failed, returns an empty `Thir`. + query thir_body(key: ty::WithOptConstParam) -> (&'tcx Steal>, thir::ExprId) { + // Perf tests revealed that hashing THIR is inefficient (see #85729). + no_hash + desc { |tcx| "building THIR for `{}`", tcx.def_path_str(key.did.to_def_id()) } + } + /// Set of all the `DefId`s in this crate that have MIR associated with /// them. This includes all the body owners, but also things like struct /// constructors. - query mir_keys(_: CrateNum) -> FxHashSet { + query mir_keys(_: ()) -> FxHashSet { storage(ArenaCacheSelector<'tcx>) desc { "getting a list of all mir_keys" } } @@ -326,10 +342,9 @@ rustc_queries! { /// Returns coverage summary info for a function, after executing the `InstrumentCoverage` /// MIR pass (assuming the -Zinstrument-coverage option is enabled). - query coverageinfo(key: DefId) -> mir::CoverageInfo { - desc { |tcx| "retrieving coverage info from MIR for `{}`", tcx.def_path_str(key) } + query coverageinfo(key: ty::InstanceDef<'tcx>) -> mir::CoverageInfo { + desc { |tcx| "retrieving coverage info from MIR for `{}`", tcx.def_path_str(key.def_id()) } storage(ArenaCacheSelector<'tcx>) - cache_on_disk_if { key.is_local() } } /// Returns the name of the file that contains the function body, if instrumented for coverage. @@ -535,7 +550,7 @@ rustc_queries! { } /// Gets a map with the variance of every item; use `item_variance` instead. - query crate_variances(_: CrateNum) -> ty::CrateVariancesMap<'tcx> { + query crate_variances(_: ()) -> ty::CrateVariancesMap<'tcx> { storage(ArenaCacheSelector<'tcx>) desc { "computing the variances for items in this crate" } } @@ -546,8 +561,7 @@ rustc_queries! { } /// Maps from thee `DefId` of a type to its (inferred) outlives. - query inferred_outlives_crate(_: CrateNum) - -> ty::CratePredicatesMap<'tcx> { + query inferred_outlives_crate(_: ()) -> ty::CratePredicatesMap<'tcx> { storage(ArenaCacheSelector<'tcx>) desc { "computing the inferred outlives predicates for items in this crate" } } @@ -602,6 +616,19 @@ rustc_queries! { } } + /// Unsafety-check this `LocalDefId` with THIR unsafeck. This should be + /// used with `-Zthir-unsafeck`. + query thir_check_unsafety(key: LocalDefId) { + desc { |tcx| "unsafety-checking `{}`", tcx.def_path_str(key.to_def_id()) } + cache_on_disk_if { true } + } + query thir_check_unsafety_for_const_arg(key: (LocalDefId, DefId)) { + desc { + |tcx| "unsafety-checking the const argument `{}`", + tcx.def_path_str(key.0.to_def_id()) + } + } + /// HACK: when evaluated, this reports a "unsafe derive on repr(packed)" error. /// /// Unsafety checking is executed for each method separately, but we only want @@ -673,7 +700,7 @@ rustc_queries! { desc { |tcx| "computing CoerceUnsized info for `{}`", tcx.def_path_str(key) } } - query typeck_item_bodies(_: CrateNum) -> () { + query typeck_item_bodies(_: ()) -> () { desc { "type-checking all item bodies" } } @@ -732,18 +759,15 @@ rustc_queries! { /// Gets a complete map from all types to their inherent impls. /// Not meant to be used directly outside of coherence. - /// (Defined only for `LOCAL_CRATE`.) - query crate_inherent_impls(k: CrateNum) - -> CrateInherentImpls { + query crate_inherent_impls(k: ()) -> CrateInherentImpls { storage(ArenaCacheSelector<'tcx>) eval_always - desc { "all inherent impls defined in crate `{:?}`", k } + desc { "all inherent impls defined in crate" } } /// Checks all types in the crate for overlap in their inherent impls. Reports errors. /// Not meant to be used directly outside of coherence. - /// (Defined only for `LOCAL_CRATE`.) - query crate_inherent_impls_overlap_check(_: CrateNum) + query crate_inherent_impls_overlap_check(_: ()) -> () { eval_always desc { "check for overlap between inherent impls defined in this crate" } @@ -837,16 +861,16 @@ rustc_queries! { } /// Performs part of the privacy check and computes "access levels". - query privacy_access_levels(_: CrateNum) -> &'tcx AccessLevels { + query privacy_access_levels(_: ()) -> &'tcx AccessLevels { eval_always desc { "privacy access levels" } } - query check_private_in_public(_: CrateNum) -> () { + query check_private_in_public(_: ()) -> () { eval_always desc { "checking for private elements in public interfaces" } } - query reachable_set(_: CrateNum) -> FxHashSet { + query reachable_set(_: ()) -> FxHashSet { storage(ArenaCacheSelector<'tcx>) desc { "reachability" } } @@ -956,7 +980,7 @@ rustc_queries! { /// Passing in any other crate will cause an ICE. /// /// [`LOCAL_CRATE`]: rustc_hir::def_id::LOCAL_CRATE - query all_local_trait_impls(local_crate: CrateNum) -> &'tcx BTreeMap> { + query all_local_trait_impls(_: ()) -> &'tcx BTreeMap> { desc { "local trait impls" } } @@ -1013,6 +1037,10 @@ rustc_queries! { query needs_drop_raw(env: ty::ParamEnvAnd<'tcx, Ty<'tcx>>) -> bool { desc { "computing whether `{}` needs drop", env.value } } + /// Query backing `TyS::has_significant_drop_raw`. + query has_significant_drop_raw(env: ty::ParamEnvAnd<'tcx, Ty<'tcx>>) -> bool { + desc { "computing whether `{}` has a significant drop", env.value } + } /// Query backing `TyS::is_structural_eq_shallow`. /// @@ -1033,6 +1061,17 @@ rustc_queries! { cache_on_disk_if { true } } + /// A list of types where the ADT requires drop if and only if any of those types + /// has significant drop. A type marked with the attribute `rustc_insignificant_dtor` + /// is considered to not be significant. A drop is significant if it is implemented + /// by the user or does anything that will have any observable behavior (other than + /// freeing up memory). If the ADT is known to have a significant destructor then + /// `Err(AlwaysRequiresDrop)` is returned. + query adt_significant_drop_tys(def_id: DefId) -> Result<&'tcx ty::List>, AlwaysRequiresDrop> { + desc { |tcx| "computing when `{}` has a significant destructor", tcx.def_path_str(def_id) } + cache_on_disk_if { false } + } + query layout_raw( env: ty::ParamEnvAnd<'tcx, Ty<'tcx>> ) -> Result<&'tcx rustc_target::abi::Layout, ty::layout::LayoutError<'tcx>> { @@ -1044,9 +1083,7 @@ rustc_queries! { desc { "dylib dependency formats of crate" } } - query dependency_formats(_: CrateNum) - -> Lrc - { + query dependency_formats(_: ()) -> Lrc { desc { "get the linkage format of all dependencies" } } @@ -1090,8 +1127,7 @@ rustc_queries! { desc { "computing whether impls specialize one another" } } query in_scope_traits_map(_: LocalDefId) - -> Option<&'tcx FxHashMap>> { - eval_always + -> Option<&'tcx FxHashMap>> { desc { "traits in scope at a block" } } @@ -1134,10 +1170,10 @@ rustc_queries! { query is_reachable_non_generic(def_id: DefId) -> bool { desc { |tcx| "checking whether `{}` is an exported symbol", tcx.def_path_str(def_id) } } - query is_unreachable_local_definition(def_id: DefId) -> bool { + query is_unreachable_local_definition(def_id: LocalDefId) -> bool { desc { |tcx| "checking whether `{}` is reachable from outside the crate", - tcx.def_path_str(def_id), + tcx.def_path_str(def_id.to_def_id()), } } @@ -1147,11 +1183,9 @@ rustc_queries! { /// added or removed in any upstream crate. Instead use the narrower /// `upstream_monomorphizations_for`, `upstream_drop_glue_for`, or, even /// better, `Instance::upstream_monomorphization()`. - query upstream_monomorphizations( - k: CrateNum - ) -> DefIdMap, CrateNum>> { + query upstream_monomorphizations(_: ()) -> DefIdMap, CrateNum>> { storage(ArenaCacheSelector<'tcx>) - desc { "collecting available upstream monomorphizations `{:?}`", k } + desc { "collecting available upstream monomorphizations" } } /// Returns the set of upstream monomorphizations available for the @@ -1194,13 +1228,13 @@ rustc_queries! { /// Identifies the entry-point (e.g., the `main` function) for a given /// crate, returning `None` if there is no entry point (such as for library crates). - query entry_fn(_: CrateNum) -> Option<(DefId, EntryFnType)> { + query entry_fn(_: ()) -> Option<(DefId, EntryFnType)> { desc { "looking up the entry function of a crate" } } - query plugin_registrar_fn(_: CrateNum) -> Option { + query plugin_registrar_fn(_: ()) -> Option { desc { "looking up the plugin registrar for a crate" } } - query proc_macro_decls_static(_: CrateNum) -> Option { + query proc_macro_decls_static(_: ()) -> Option { desc { "looking up the derive registrar for a crate" } } query crate_disambiguator(_: CrateNum) -> CrateDisambiguator { @@ -1217,10 +1251,6 @@ rustc_queries! { eval_always desc { "looking up the hash of a host version of a crate" } } - query original_crate_name(_: CrateNum) -> Symbol { - eval_always - desc { "looking up the original name a crate" } - } query extra_filename(_: CrateNum) -> String { eval_always desc { "looking up the extra filename for a crate" } @@ -1292,6 +1322,10 @@ rustc_queries! { desc { "looking up late bound vars" } } + query lifetime_scope_map(_: LocalDefId) -> Option> { + desc { "finds the lifetime scope for an HirId of a PathSegment" } + } + query visibility(def_id: DefId) -> ty::Visibility { eval_always desc { |tcx| "computing visibility of `{}`", tcx.def_path_str(def_id) } @@ -1323,7 +1357,7 @@ rustc_queries! { desc { |tcx| "computing crate imported by `{}`", tcx.def_path_str(def_id.to_def_id()) } } - query get_lib_features(_: CrateNum) -> LibFeatures { + query get_lib_features(_: ()) -> LibFeatures { storage(ArenaCacheSelector<'tcx>) eval_always desc { "calculating the lib features map" } @@ -1333,16 +1367,14 @@ rustc_queries! { desc { "calculating the lib features defined in a crate" } } /// Returns the lang items defined in another crate by loading it from metadata. - // FIXME: It is illegal to pass a `CrateNum` other than `LOCAL_CRATE` here, just get rid - // of that argument? - query get_lang_items(_: CrateNum) -> LanguageItems { + query get_lang_items(_: ()) -> LanguageItems { storage(ArenaCacheSelector<'tcx>) eval_always desc { "calculating the lang items map" } } /// Returns all diagnostic items defined in all crates. - query all_diagnostic_items(_: CrateNum) -> FxHashMap { + query all_diagnostic_items(_: ()) -> FxHashMap { storage(ArenaCacheSelector<'tcx>) eval_always desc { "calculating the diagnostic items map" } @@ -1362,13 +1394,11 @@ rustc_queries! { query missing_lang_items(_: CrateNum) -> &'tcx [LangItem] { desc { "calculating the missing lang items in a crate" } } - query visible_parent_map(_: CrateNum) - -> DefIdMap { + query visible_parent_map(_: ()) -> DefIdMap { storage(ArenaCacheSelector<'tcx>) desc { "calculating the visible parent map" } } - query trimmed_def_paths(_: CrateNum) - -> FxHashMap { + query trimmed_def_paths(_: ()) -> FxHashMap { storage(ArenaCacheSelector<'tcx>) desc { "calculating trimmed def paths" } } @@ -1380,10 +1410,16 @@ rustc_queries! { eval_always desc { "looking at the source for a crate" } } - query postorder_cnums(_: CrateNum) -> &'tcx [CrateNum] { + query postorder_cnums(_: ()) -> &'tcx [CrateNum] { eval_always desc { "generating a postorder list of CrateNums" } } + /// Returns whether or not the crate with CrateNum 'cnum' + /// is marked as a private dependency + query is_private_dep(c: CrateNum) -> bool { + eval_always + desc { "check whether crate {} is a private dependency", c } + } query upvars_mentioned(def_id: DefId) -> Option<&'tcx FxIndexMap> { desc { |tcx| "collecting upvars mentioned in `{}`", tcx.def_path_str(def_id) } @@ -1393,8 +1429,7 @@ rustc_queries! { eval_always desc { |tcx| "maybe_unused_trait_import for `{}`", tcx.def_path_str(def_id.to_def_id()) } } - query maybe_unused_extern_crates(_: CrateNum) - -> &'tcx [(LocalDefId, Span)] { + query maybe_unused_extern_crates(_: ()) -> &'tcx [(LocalDefId, Span)] { eval_always desc { "looking up all possibly unused extern crates" } } @@ -1404,12 +1439,12 @@ rustc_queries! { desc { |tcx| "names_imported_by_glob_use for `{}`", tcx.def_path_str(def_id.to_def_id()) } } - query stability_index(_: CrateNum) -> stability::Index<'tcx> { + query stability_index(_: ()) -> stability::Index<'tcx> { storage(ArenaCacheSelector<'tcx>) eval_always desc { "calculating the stability index for the local crate" } } - query all_crate_nums(_: CrateNum) -> &'tcx [CrateNum] { + query all_crate_nums(_: ()) -> &'tcx [CrateNum] { eval_always desc { "fetching all foreign CrateNum instances" } } @@ -1417,7 +1452,7 @@ rustc_queries! { /// A vector of every trait accessible in the whole crate /// (i.e., including those from subcrates). This is used only for /// error reporting. - query all_traits(_: CrateNum) -> &'tcx [DefId] { + query all_traits(_: ()) -> &'tcx [DefId] { desc { "fetching all foreign and local traits" } } @@ -1431,8 +1466,7 @@ rustc_queries! { desc { "exported_symbols" } } - query collect_and_partition_mono_items(_: CrateNum) - -> (&'tcx DefIdSet, &'tcx [CodegenUnit<'tcx>]) { + query collect_and_partition_mono_items(_: ()) -> (&'tcx DefIdSet, &'tcx [CodegenUnit<'tcx>]) { eval_always desc { "collect_and_partition_mono_items" } } @@ -1441,8 +1475,7 @@ rustc_queries! { } /// All items participating in code generation together with items inlined into them. - query codegened_and_inlined_items(_: CrateNum) - -> &'tcx DefIdSet { + query codegened_and_inlined_items(_: ()) -> &'tcx DefIdSet { eval_always desc { "codegened_and_inlined_items" } } @@ -1457,11 +1490,11 @@ rustc_queries! { tcx.def_path_str(key) } } - query backend_optimization_level(_: CrateNum) -> OptLevel { + query backend_optimization_level(_: ()) -> OptLevel { desc { "optimization level used by backend" } } - query output_filenames(_: CrateNum) -> Arc { + query output_filenames(_: ()) -> Arc { eval_always desc { "output_filenames" } } @@ -1637,7 +1670,7 @@ rustc_queries! { desc { |tcx| "estimating size for `{}`", tcx.def_path_str(def.def_id()) } } - query features_query(_: CrateNum) -> &'tcx rustc_feature::Features { + query features_query(_: ()) -> &'tcx rustc_feature::Features { eval_always desc { "looking up enabled feature gates" } } diff --git a/compiler/rustc_middle/src/thir.rs b/compiler/rustc_middle/src/thir.rs new file mode 100644 index 0000000000..a506911370 --- /dev/null +++ b/compiler/rustc_middle/src/thir.rs @@ -0,0 +1,747 @@ +use rustc_ast::{InlineAsmOptions, InlineAsmTemplatePiece}; +use rustc_hir as hir; +use rustc_hir::def::CtorKind; +use rustc_hir::def_id::DefId; +use rustc_hir::RangeEnd; +use rustc_index::newtype_index; +use rustc_index::vec::{Idx, IndexVec}; +use rustc_middle::infer::canonical::Canonical; +use rustc_middle::middle::region; +use rustc_middle::mir::{ + BinOp, BorrowKind, FakeReadCause, Field, Mutability, UnOp, UserTypeProjection, +}; +use rustc_middle::ty::adjustment::PointerCast; +use rustc_middle::ty::subst::SubstsRef; +use rustc_middle::ty::{self, AdtDef, Const, Ty, UpvarSubsts, UserType}; +use rustc_middle::ty::{ + CanonicalUserType, CanonicalUserTypeAnnotation, CanonicalUserTypeAnnotations, +}; +use rustc_span::{Span, Symbol, DUMMY_SP}; +use rustc_target::abi::VariantIdx; +use rustc_target::asm::InlineAsmRegOrRegClass; + +use std::fmt; +use std::ops::Index; + +newtype_index! { + #[derive(HashStable)] + pub struct ArmId { + DEBUG_FORMAT = "a{}" + } +} + +newtype_index! { + #[derive(HashStable)] + pub struct ExprId { + DEBUG_FORMAT = "e{}" + } +} + +newtype_index! { + #[derive(HashStable)] + pub struct StmtId { + DEBUG_FORMAT = "s{}" + } +} + +macro_rules! thir_with_elements { + ($($name:ident: $id:ty => $value:ty,)*) => { + #[derive(Debug, HashStable)] + pub struct Thir<'tcx> { + $( + pub $name: IndexVec<$id, $value>, + )* + } + + impl<'tcx> Thir<'tcx> { + pub fn new() -> Thir<'tcx> { + Thir { + $( + $name: IndexVec::new(), + )* + } + } + } + + $( + impl<'tcx> Index<$id> for Thir<'tcx> { + type Output = $value; + fn index(&self, index: $id) -> &Self::Output { + &self.$name[index] + } + } + )* + } +} + +thir_with_elements! { + arms: ArmId => Arm<'tcx>, + exprs: ExprId => Expr<'tcx>, + stmts: StmtId => Stmt<'tcx>, +} + +#[derive(Copy, Clone, Debug, HashStable)] +pub enum LintLevel { + Inherited, + Explicit(hir::HirId), +} + +#[derive(Debug, HashStable)] +pub struct Block { + pub targeted_by_break: bool, + pub region_scope: region::Scope, + pub opt_destruction_scope: Option, + pub span: Span, + pub stmts: Box<[StmtId]>, + pub expr: Option, + pub safety_mode: BlockSafety, +} + +#[derive(Copy, Clone, Debug, HashStable)] +pub enum BlockSafety { + Safe, + ExplicitUnsafe(hir::HirId), + PushUnsafe, + PopUnsafe, +} + +#[derive(Debug, HashStable)] +pub struct Stmt<'tcx> { + pub kind: StmtKind<'tcx>, + pub opt_destruction_scope: Option, +} + +#[derive(Debug, HashStable)] +pub enum StmtKind<'tcx> { + Expr { + /// scope for this statement; may be used as lifetime of temporaries + scope: region::Scope, + + /// expression being evaluated in this statement + expr: ExprId, + }, + + Let { + /// scope for variables bound in this let; covers this and + /// remaining statements in block + remainder_scope: region::Scope, + + /// scope for the initialization itself; might be used as + /// lifetime of temporaries + init_scope: region::Scope, + + /// `let = ...` + /// + /// if a type is included, it is added as an ascription pattern + pattern: Pat<'tcx>, + + /// let pat: ty = ... + initializer: Option, + + /// the lint level for this let-statement + lint_level: LintLevel, + }, +} + +// `Expr` is used a lot. Make sure it doesn't unintentionally get bigger. +#[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))] +rustc_data_structures::static_assert_size!(Expr<'_>, 144); + +/// The Thir trait implementor lowers their expressions (`&'tcx H::Expr`) +/// into instances of this `Expr` enum. This lowering can be done +/// basically as lazily or as eagerly as desired: every recursive +/// reference to an expression in this enum is an `ExprId`, which +/// may in turn be another instance of this enum (boxed), or else an +/// unlowered `&'tcx H::Expr`. Note that instances of `Expr` are very +/// short-lived. They are created by `Thir::to_expr`, analyzed and +/// converted into MIR, and then discarded. +/// +/// If you compare `Expr` to the full compiler AST, you will see it is +/// a good bit simpler. In fact, a number of the more straight-forward +/// MIR simplifications are already done in the impl of `Thir`. For +/// example, method calls and overloaded operators are absent: they are +/// expected to be converted into `Expr::Call` instances. +#[derive(Debug, HashStable)] +pub struct Expr<'tcx> { + /// type of this expression + pub ty: Ty<'tcx>, + + /// lifetime of this expression if it should be spilled into a + /// temporary; should be None only if in a constant context + pub temp_lifetime: Option, + + /// span of the expression in the source + pub span: Span, + + /// kind of expression + pub kind: ExprKind<'tcx>, +} + +#[derive(Debug, HashStable)] +pub enum ExprKind<'tcx> { + Scope { + region_scope: region::Scope, + lint_level: LintLevel, + value: ExprId, + }, + Box { + value: ExprId, + }, + If { + cond: ExprId, + then: ExprId, + else_opt: Option, + }, + Call { + ty: Ty<'tcx>, + fun: ExprId, + args: Box<[ExprId]>, + /// Whether this is from a call in HIR, rather than from an overloaded + /// operator. `true` for overloaded function call. + from_hir_call: bool, + /// This `Span` is the span of the function, without the dot and receiver + /// (e.g. `foo(a, b)` in `x.foo(a, b)` + fn_span: Span, + }, + Deref { + arg: ExprId, + }, // NOT overloaded! + Binary { + op: BinOp, + lhs: ExprId, + rhs: ExprId, + }, // NOT overloaded! + LogicalOp { + op: LogicalOp, + lhs: ExprId, + rhs: ExprId, + }, // NOT overloaded! + // LogicalOp is distinct from BinaryOp because of lazy evaluation of the operands. + Unary { + op: UnOp, + arg: ExprId, + }, // NOT overloaded! + Cast { + source: ExprId, + }, + Use { + source: ExprId, + }, // Use a lexpr to get a vexpr. + NeverToAny { + source: ExprId, + }, + Pointer { + cast: PointerCast, + source: ExprId, + }, + Loop { + body: ExprId, + }, + Match { + scrutinee: ExprId, + arms: Box<[ArmId]>, + }, + Block { + body: Block, + }, + Assign { + lhs: ExprId, + rhs: ExprId, + }, + AssignOp { + op: BinOp, + lhs: ExprId, + rhs: ExprId, + }, + Field { + lhs: ExprId, + name: Field, + }, + Index { + lhs: ExprId, + index: ExprId, + }, + VarRef { + id: hir::HirId, + }, + /// Used to represent upvars mentioned in a closure/generator + UpvarRef { + /// DefId of the closure/generator + closure_def_id: DefId, + + /// HirId of the root variable + var_hir_id: hir::HirId, + }, + Borrow { + borrow_kind: BorrowKind, + arg: ExprId, + }, + /// A `&raw [const|mut] $place_expr` raw borrow resulting in type `*[const|mut] T`. + AddressOf { + mutability: hir::Mutability, + arg: ExprId, + }, + Break { + label: region::Scope, + value: Option, + }, + Continue { + label: region::Scope, + }, + Return { + value: Option, + }, + ConstBlock { + value: &'tcx Const<'tcx>, + }, + Repeat { + value: ExprId, + count: &'tcx Const<'tcx>, + }, + Array { + fields: Box<[ExprId]>, + }, + Tuple { + fields: Box<[ExprId]>, + }, + Adt { + adt_def: &'tcx AdtDef, + variant_index: VariantIdx, + substs: SubstsRef<'tcx>, + + /// Optional user-given substs: for something like `let x = + /// Bar:: { ... }`. + user_ty: Option>>, + + fields: Box<[FieldExpr]>, + base: Option>, + }, + PlaceTypeAscription { + source: ExprId, + /// Type that the user gave to this expression + user_ty: Option>>, + }, + ValueTypeAscription { + source: ExprId, + /// Type that the user gave to this expression + user_ty: Option>>, + }, + Closure { + closure_id: DefId, + substs: UpvarSubsts<'tcx>, + upvars: Box<[ExprId]>, + movability: Option, + fake_reads: Vec<(ExprId, FakeReadCause, hir::HirId)>, + }, + Literal { + literal: &'tcx Const<'tcx>, + user_ty: Option>>, + /// The `DefId` of the `const` item this literal + /// was produced from, if this is not a user-written + /// literal value. + const_id: Option, + }, + /// A literal containing the address of a `static`. + /// + /// This is only distinguished from `Literal` so that we can register some + /// info for diagnostics. + StaticRef { + literal: &'tcx Const<'tcx>, + def_id: DefId, + }, + InlineAsm { + template: &'tcx [InlineAsmTemplatePiece], + operands: Box<[InlineAsmOperand<'tcx>]>, + options: InlineAsmOptions, + line_spans: &'tcx [Span], + }, + /// An expression taking a reference to a thread local. + ThreadLocalRef(DefId), + LlvmInlineAsm { + asm: &'tcx hir::LlvmInlineAsmInner, + outputs: Box<[ExprId]>, + inputs: Box<[ExprId]>, + }, + Yield { + value: ExprId, + }, +} + +#[derive(Debug, HashStable)] +pub struct FieldExpr { + pub name: Field, + pub expr: ExprId, +} + +#[derive(Debug, HashStable)] +pub struct FruInfo<'tcx> { + pub base: ExprId, + pub field_types: Box<[Ty<'tcx>]>, +} + +#[derive(Debug, HashStable)] +pub struct Arm<'tcx> { + pub pattern: Pat<'tcx>, + pub guard: Option>, + pub body: ExprId, + pub lint_level: LintLevel, + pub scope: region::Scope, + pub span: Span, +} + +#[derive(Debug, HashStable)] +pub enum Guard<'tcx> { + If(ExprId), + IfLet(Pat<'tcx>, ExprId), +} + +#[derive(Copy, Clone, Debug, HashStable)] +pub enum LogicalOp { + And, + Or, +} + +#[derive(Debug, HashStable)] +pub enum InlineAsmOperand<'tcx> { + In { + reg: InlineAsmRegOrRegClass, + expr: ExprId, + }, + Out { + reg: InlineAsmRegOrRegClass, + late: bool, + expr: Option, + }, + InOut { + reg: InlineAsmRegOrRegClass, + late: bool, + expr: ExprId, + }, + SplitInOut { + reg: InlineAsmRegOrRegClass, + late: bool, + in_expr: ExprId, + out_expr: Option, + }, + Const { + value: &'tcx Const<'tcx>, + span: Span, + }, + SymFn { + expr: ExprId, + }, + SymStatic { + def_id: DefId, + }, +} + +#[derive(Copy, Clone, Debug, PartialEq, HashStable)] +pub enum BindingMode { + ByValue, + ByRef(BorrowKind), +} + +#[derive(Clone, Debug, PartialEq, HashStable)] +pub struct FieldPat<'tcx> { + pub field: Field, + pub pattern: Pat<'tcx>, +} + +#[derive(Clone, Debug, PartialEq, HashStable)] +pub struct Pat<'tcx> { + pub ty: Ty<'tcx>, + pub span: Span, + pub kind: Box>, +} + +impl<'tcx> Pat<'tcx> { + pub fn wildcard_from_ty(ty: Ty<'tcx>) -> Self { + Pat { ty, span: DUMMY_SP, kind: Box::new(PatKind::Wild) } + } +} + +#[derive(Copy, Clone, Debug, PartialEq, HashStable)] +pub struct PatTyProj<'tcx> { + pub user_ty: CanonicalUserType<'tcx>, +} + +impl<'tcx> PatTyProj<'tcx> { + pub fn from_user_type(user_annotation: CanonicalUserType<'tcx>) -> Self { + Self { user_ty: user_annotation } + } + + pub fn user_ty( + self, + annotations: &mut CanonicalUserTypeAnnotations<'tcx>, + inferred_ty: Ty<'tcx>, + span: Span, + ) -> UserTypeProjection { + UserTypeProjection { + base: annotations.push(CanonicalUserTypeAnnotation { + span, + user_ty: self.user_ty, + inferred_ty, + }), + projs: Vec::new(), + } + } +} + +#[derive(Copy, Clone, Debug, PartialEq, HashStable)] +pub struct Ascription<'tcx> { + pub user_ty: PatTyProj<'tcx>, + /// Variance to use when relating the type `user_ty` to the **type of the value being + /// matched**. Typically, this is `Variance::Covariant`, since the value being matched must + /// have a type that is some subtype of the ascribed type. + /// + /// Note that this variance does not apply for any bindings within subpatterns. The type + /// assigned to those bindings must be exactly equal to the `user_ty` given here. + /// + /// The only place where this field is not `Covariant` is when matching constants, where + /// we currently use `Contravariant` -- this is because the constant type just needs to + /// be "comparable" to the type of the input value. So, for example: + /// + /// ```text + /// match x { "foo" => .. } + /// ``` + /// + /// requires that `&'static str <: T_x`, where `T_x` is the type of `x`. Really, we should + /// probably be checking for a `PartialEq` impl instead, but this preserves the behavior + /// of the old type-check for now. See #57280 for details. + pub variance: ty::Variance, + pub user_ty_span: Span, +} + +#[derive(Clone, Debug, PartialEq, HashStable)] +pub enum PatKind<'tcx> { + Wild, + + AscribeUserType { + ascription: Ascription<'tcx>, + subpattern: Pat<'tcx>, + }, + + /// `x`, `ref x`, `x @ P`, etc. + Binding { + mutability: Mutability, + name: Symbol, + mode: BindingMode, + var: hir::HirId, + ty: Ty<'tcx>, + subpattern: Option>, + /// Is this the leftmost occurrence of the binding, i.e., is `var` the + /// `HirId` of this pattern? + is_primary: bool, + }, + + /// `Foo(...)` or `Foo{...}` or `Foo`, where `Foo` is a variant name from an ADT with + /// multiple variants. + Variant { + adt_def: &'tcx AdtDef, + substs: SubstsRef<'tcx>, + variant_index: VariantIdx, + subpatterns: Vec>, + }, + + /// `(...)`, `Foo(...)`, `Foo{...}`, or `Foo`, where `Foo` is a variant name from an ADT with + /// a single variant. + Leaf { + subpatterns: Vec>, + }, + + /// `box P`, `&P`, `&mut P`, etc. + Deref { + subpattern: Pat<'tcx>, + }, + + /// One of the following: + /// * `&str`, which will be handled as a string pattern and thus exhaustiveness + /// checking will detect if you use the same string twice in different patterns. + /// * integer, bool, char or float, which will be handled by exhaustivenes to cover exactly + /// its own value, similar to `&str`, but these values are much simpler. + /// * Opaque constants, that must not be matched structurally. So anything that does not derive + /// `PartialEq` and `Eq`. + Constant { + value: &'tcx ty::Const<'tcx>, + }, + + Range(PatRange<'tcx>), + + /// Matches against a slice, checking the length and extracting elements. + /// irrefutable when there is a slice pattern and both `prefix` and `suffix` are empty. + /// e.g., `&[ref xs @ ..]`. + Slice { + prefix: Vec>, + slice: Option>, + suffix: Vec>, + }, + + /// Fixed match against an array; irrefutable. + Array { + prefix: Vec>, + slice: Option>, + suffix: Vec>, + }, + + /// An or-pattern, e.g. `p | q`. + /// Invariant: `pats.len() >= 2`. + Or { + pats: Vec>, + }, +} + +#[derive(Copy, Clone, Debug, PartialEq, HashStable)] +pub struct PatRange<'tcx> { + pub lo: &'tcx ty::Const<'tcx>, + pub hi: &'tcx ty::Const<'tcx>, + pub end: RangeEnd, +} + +impl<'tcx> fmt::Display for Pat<'tcx> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + // Printing lists is a chore. + let mut first = true; + let mut start_or_continue = |s| { + if first { + first = false; + "" + } else { + s + } + }; + let mut start_or_comma = || start_or_continue(", "); + + match *self.kind { + PatKind::Wild => write!(f, "_"), + PatKind::AscribeUserType { ref subpattern, .. } => write!(f, "{}: _", subpattern), + PatKind::Binding { mutability, name, mode, ref subpattern, .. } => { + let is_mut = match mode { + BindingMode::ByValue => mutability == Mutability::Mut, + BindingMode::ByRef(bk) => { + write!(f, "ref ")?; + matches!(bk, BorrowKind::Mut { .. }) + } + }; + if is_mut { + write!(f, "mut ")?; + } + write!(f, "{}", name)?; + if let Some(ref subpattern) = *subpattern { + write!(f, " @ {}", subpattern)?; + } + Ok(()) + } + PatKind::Variant { ref subpatterns, .. } | PatKind::Leaf { ref subpatterns } => { + let variant = match *self.kind { + PatKind::Variant { adt_def, variant_index, .. } => { + Some(&adt_def.variants[variant_index]) + } + _ => { + if let ty::Adt(adt, _) = self.ty.kind() { + if !adt.is_enum() { + Some(&adt.variants[VariantIdx::new(0)]) + } else { + None + } + } else { + None + } + } + }; + + if let Some(variant) = variant { + write!(f, "{}", variant.ident)?; + + // Only for Adt we can have `S {...}`, + // which we handle separately here. + if variant.ctor_kind == CtorKind::Fictive { + write!(f, " {{ ")?; + + let mut printed = 0; + for p in subpatterns { + if let PatKind::Wild = *p.pattern.kind { + continue; + } + let name = variant.fields[p.field.index()].ident; + write!(f, "{}{}: {}", start_or_comma(), name, p.pattern)?; + printed += 1; + } + + if printed < variant.fields.len() { + write!(f, "{}..", start_or_comma())?; + } + + return write!(f, " }}"); + } + } + + let num_fields = variant.map_or(subpatterns.len(), |v| v.fields.len()); + if num_fields != 0 || variant.is_none() { + write!(f, "(")?; + for i in 0..num_fields { + write!(f, "{}", start_or_comma())?; + + // Common case: the field is where we expect it. + if let Some(p) = subpatterns.get(i) { + if p.field.index() == i { + write!(f, "{}", p.pattern)?; + continue; + } + } + + // Otherwise, we have to go looking for it. + if let Some(p) = subpatterns.iter().find(|p| p.field.index() == i) { + write!(f, "{}", p.pattern)?; + } else { + write!(f, "_")?; + } + } + write!(f, ")")?; + } + + Ok(()) + } + PatKind::Deref { ref subpattern } => { + match self.ty.kind() { + ty::Adt(def, _) if def.is_box() => write!(f, "box ")?, + ty::Ref(_, _, mutbl) => { + write!(f, "&{}", mutbl.prefix_str())?; + } + _ => bug!("{} is a bad Deref pattern type", self.ty), + } + write!(f, "{}", subpattern) + } + PatKind::Constant { value } => write!(f, "{}", value), + PatKind::Range(PatRange { lo, hi, end }) => { + write!(f, "{}", lo)?; + write!(f, "{}", end)?; + write!(f, "{}", hi) + } + PatKind::Slice { ref prefix, ref slice, ref suffix } + | PatKind::Array { ref prefix, ref slice, ref suffix } => { + write!(f, "[")?; + for p in prefix { + write!(f, "{}{}", start_or_comma(), p)?; + } + if let Some(ref slice) = *slice { + write!(f, "{}", start_or_comma())?; + match *slice.kind { + PatKind::Wild => {} + _ => write!(f, "{}", slice)?, + } + write!(f, "..")?; + } + for p in suffix { + write!(f, "{}{}", start_or_comma(), p)?; + } + write!(f, "]") + } + PatKind::Or { ref pats } => { + for pat in pats { + write!(f, "{}{}", start_or_continue(" | "), pat)?; + } + Ok(()) + } + } + } +} diff --git a/compiler/rustc_middle/src/ty/_match.rs b/compiler/rustc_middle/src/ty/_match.rs index 8e2c79701a..02ff1b9f4d 100644 --- a/compiler/rustc_middle/src/ty/_match.rs +++ b/compiler/rustc_middle/src/ty/_match.rs @@ -46,6 +46,7 @@ impl TypeRelation<'tcx> for Match<'tcx> { fn relate_with_variance>( &mut self, _: ty::Variance, + _: ty::VarianceDiagInfo<'tcx>, a: T, b: T, ) -> RelateResult<'tcx, T> { diff --git a/compiler/rustc_middle/src/ty/adjustment.rs b/compiler/rustc_middle/src/ty/adjustment.rs index a50dda69a0..8f648b2113 100644 --- a/compiler/rustc_middle/src/ty/adjustment.rs +++ b/compiler/rustc_middle/src/ty/adjustment.rs @@ -47,7 +47,7 @@ pub enum PointerCast { /// 1. The simplest cases are where a pointer is not adjusted fat vs thin. /// Here the pointer will be dereferenced N times (where a dereference can /// happen to raw or borrowed pointers or any smart pointer which implements -/// Deref, including Box<_>). The types of dereferences is given by +/// `Deref`, including `Box<_>`). The types of dereferences is given by /// `autoderefs`. It can then be auto-referenced zero or one times, indicated /// by `autoref`, to either a raw or borrowed pointer. In these cases unsize is /// `false`. @@ -56,7 +56,7 @@ pub enum PointerCast { /// with a thin pointer, deref a number of times, unsize the underlying data, /// then autoref. The 'unsize' phase may change a fixed length array to a /// dynamically sized one, a concrete object to a trait object, or statically -/// sized struct to a dynamically sized one. E.g., &[i32; 4] -> &[i32] is +/// sized struct to a dynamically sized one. E.g., `&[i32; 4]` -> `&[i32]` is /// represented by: /// /// ``` @@ -66,7 +66,7 @@ pub enum PointerCast { /// ``` /// /// Note that for a struct, the 'deep' unsizing of the struct is not recorded. -/// E.g., `struct Foo { x: T }` we can coerce &Foo<[i32; 4]> to &Foo<[i32]> +/// E.g., `struct Foo { x: T }` we can coerce `&Foo<[i32; 4]>` to `&Foo<[i32]>` /// The autoderef and -ref are the same as in the above example, but the type /// stored in `unsize` is `Foo<[i32]>`, we don't store any further detail about /// the underlying conversions from `[i32; 4]` to `[i32]`. @@ -75,8 +75,8 @@ pub enum PointerCast { /// that case, we have the pointer we need coming in, so there are no /// autoderefs, and no autoref. Instead we just do the `Unsize` transformation. /// At some point, of course, `Box` should move out of the compiler, in which -/// case this is analogous to transforming a struct. E.g., Box<[i32; 4]> -> -/// Box<[i32]> is an `Adjust::Unsize` with the target `Box<[i32]>`. +/// case this is analogous to transforming a struct. E.g., `Box<[i32; 4]>` -> +/// `Box<[i32]>` is an `Adjust::Unsize` with the target `Box<[i32]>`. #[derive(Clone, TyEncodable, TyDecodable, HashStable, TypeFoldable)] pub struct Adjustment<'tcx> { pub kind: Adjust<'tcx>, diff --git a/compiler/rustc_middle/src/ty/closure.rs b/compiler/rustc_middle/src/ty/closure.rs index 887a5831cd..0706a057dd 100644 --- a/compiler/rustc_middle/src/ty/closure.rs +++ b/compiler/rustc_middle/src/ty/closure.rs @@ -1,7 +1,7 @@ use crate::hir::place::{ Place as HirPlace, PlaceBase as HirPlaceBase, ProjectionKind as HirProjectionKind, }; -use crate::ty; +use crate::{mir, ty}; use rustc_data_structures::fx::{FxHashMap, FxIndexMap}; use rustc_hir as hir; @@ -12,6 +12,10 @@ use super::{Ty, TyCtxt}; use self::BorrowKind::*; +// Captures are represented using fields inside a structure. +// This represents accessing self in the closure structure +pub const CAPTURE_STRUCT_LOCAL: mir::Local = mir::Local::from_u32(1); + #[derive( Clone, Copy, @@ -151,6 +155,10 @@ pub struct CapturedPlace<'tcx> { } impl CapturedPlace<'tcx> { + pub fn to_string(&self, tcx: TyCtxt<'tcx>) -> String { + place_to_string_for_capture(tcx, &self.place) + } + /// Returns the hir-id of the root variable for the captured place. /// e.g., if `a.b.c` was captured, would return the hir-id for `a`. pub fn get_root_variable(&self) -> hir::HirId { @@ -168,6 +176,22 @@ impl CapturedPlace<'tcx> { } } + /// Return span pointing to use that resulted in selecting the captured path + pub fn get_path_span(&self, tcx: TyCtxt<'tcx>) -> Span { + if let Some(path_expr_id) = self.info.path_expr_id { + tcx.hir().span(path_expr_id) + } else if let Some(capture_kind_expr_id) = self.info.capture_kind_expr_id { + tcx.hir().span(capture_kind_expr_id) + } else { + // Fallback on upvars mentioned if neither path or capture expr id is captured + + // Safe to unwrap since we know this place is captured by the closure, therefore the closure must have upvars. + tcx.upvars_mentioned(self.get_closure_local_def_id()).unwrap() + [&self.get_root_variable()] + .span + } + } + /// Return span pointing to use that resulted in selecting the current capture kind pub fn get_capture_kind_span(&self, tcx: TyCtxt<'tcx>) -> Span { if let Some(capture_kind_expr_id) = self.info.capture_kind_expr_id { diff --git a/compiler/rustc_middle/src/ty/codec.rs b/compiler/rustc_middle/src/ty/codec.rs index d7767dc39c..5ec665e913 100644 --- a/compiler/rustc_middle/src/ty/codec.rs +++ b/compiler/rustc_middle/src/ty/codec.rs @@ -15,7 +15,7 @@ use crate::mir::{ use crate::ty::subst::SubstsRef; use crate::ty::{self, List, Ty, TyCtxt}; use rustc_data_structures::fx::FxHashMap; -use rustc_hir::def_id::{CrateNum, DefId}; +use rustc_hir::def_id::DefId; use rustc_serialize::{Decodable, Decoder, Encodable, Encoder}; use rustc_span::Span; use std::hash::Hash; @@ -179,8 +179,6 @@ pub trait TyDecoder<'tcx>: Decoder { where F: FnOnce(&mut Self) -> R; - fn map_encoded_cnum_to_current(&self, cnum: CrateNum) -> CrateNum; - fn positioned_at_shorthand(&self) -> bool { (self.peek_byte() & (SHORTHAND_OFFSET as u8)) != 0 } diff --git a/compiler/rustc_middle/src/ty/context.rs b/compiler/rustc_middle/src/ty/context.rs index b414618f7d..73991436b7 100644 --- a/compiler/rustc_middle/src/ty/context.rs +++ b/compiler/rustc_middle/src/ty/context.rs @@ -9,10 +9,11 @@ use crate::infer::canonical::{Canonical, CanonicalVarInfo, CanonicalVarInfos}; use crate::lint::{struct_lint_level, LintDiagnosticBuilder, LintLevelSource}; use crate::middle; use crate::middle::cstore::{CrateStoreDyn, EncodedMetadata}; -use crate::middle::resolve_lifetime::{self, ObjectLifetimeDefault}; +use crate::middle::resolve_lifetime::{self, LifetimeScopeForPath, ObjectLifetimeDefault}; use crate::middle::stability; use crate::mir::interpret::{self, Allocation, ConstValue, Scalar}; use crate::mir::{Body, Field, Local, Place, PlaceElem, ProjectionKind, Promoted}; +use crate::thir::Thir; use crate::traits; use crate::ty::query::{self, OnDiskCache, TyCtxtAt}; use crate::ty::subst::{GenericArg, GenericArgKind, InternalSubsts, Subst, SubstsRef, UserSubsts}; @@ -30,9 +31,10 @@ use rustc_attr as attr; use rustc_data_structures::fx::{FxHashMap, FxHashSet}; use rustc_data_structures::profiling::SelfProfilerRef; use rustc_data_structures::sharded::{IntoPointer, ShardedHashMap}; -use rustc_data_structures::stable_hasher::{HashStable, StableHasher, StableVec}; +use rustc_data_structures::stable_hasher::{HashStable, StableHasher}; use rustc_data_structures::steal::Steal; use rustc_data_structures::sync::{self, Lock, Lrc, WorkerLocal}; +use rustc_data_structures::vec_map::VecMap; use rustc_errors::ErrorReported; use rustc_hir as hir; use rustc_hir::def::{DefKind, Res}; @@ -46,10 +48,12 @@ use rustc_hir::{ use rustc_index::vec::{Idx, IndexVec}; use rustc_macros::HashStable; use rustc_middle::mir::FakeReadCause; +use rustc_middle::ty::OpaqueTypeKey; use rustc_serialize::opaque::{FileEncodeResult, FileEncoder}; use rustc_session::config::{BorrowckMode, CrateType, OutputFilenames}; use rustc_session::lint::{Level, Lint}; use rustc_session::Session; +use rustc_span::def_id::StableCrateId; use rustc_span::source_map::MultiSpan; use rustc_span::symbol::{kw, sym, Ident, Symbol}; use rustc_span::{Span, DUMMY_SP}; @@ -285,17 +289,6 @@ impl<'a, V> LocalTableInContextMut<'a, V> { } } -/// All information necessary to validate and reveal an `impl Trait`. -#[derive(TyEncodable, TyDecodable, Debug, HashStable)] -pub struct ResolvedOpaqueTy<'tcx> { - /// The revealed type as seen by this function. - pub concrete_type: Ty<'tcx>, - /// Generic parameters on the opaque type as passed by this function. - /// For `type Foo = impl Bar; fn foo() -> Foo { .. }` - /// this is `[T, U]`, not `[A, B]`. - pub substs: SubstsRef<'tcx>, -} - /// Whenever a value may be live across a generator yield, the type of that value winds up in the /// `GeneratorInteriorTypeCause` struct. This struct adds additional information about such /// captured types that can be useful for diagnostics. In particular, it stores the span that @@ -423,7 +416,7 @@ pub struct TypeckResults<'tcx> { /// All the opaque types that are restricted to concrete types /// by this function. - pub concrete_opaque_types: FxHashMap>, + pub concrete_opaque_types: VecMap, Ty<'tcx>>, /// Tracks the minimum captures required for a closure; /// see `MinCaptureInformationMap` for more details. @@ -965,15 +958,11 @@ pub struct GlobalCtxt<'tcx> { /// Resolutions of `extern crate` items produced by resolver. extern_crate_map: FxHashMap, - /// Map indicating what traits are in scope for places where this - /// is relevant; generated by resolve. - trait_map: FxHashMap>>, - /// Export map produced by name resolution. export_map: ExportMap, pub(crate) untracked_crate: &'tcx hir::Crate<'tcx>, - pub(crate) definitions: &'tcx Definitions, + pub(crate) definitions: Definitions, /// This provides access to the incremental compilation on-disk cache for query results. /// Do not access this directly. It is only meant to be used by @@ -1008,7 +997,7 @@ pub struct GlobalCtxt<'tcx> { /// The definite name of the current crate after taking into account /// attributes, commandline parameters, etc. - pub crate_name: Symbol, + crate_name: Symbol, /// Data layout specification for the current target. pub data_layout: TargetDataLayout, @@ -1041,6 +1030,10 @@ impl<'tcx> TyCtxt<'tcx> { } } + pub fn alloc_steal_thir(self, thir: Thir<'tcx>) -> &'tcx Steal> { + self.arena.alloc(Steal::new(thir)) + } + pub fn alloc_steal_mir(self, mir: Body<'tcx>) -> &'tcx Steal> { self.arena.alloc(Steal::new(mir)) } @@ -1072,7 +1065,7 @@ impl<'tcx> TyCtxt<'tcx> { /// Allocates a read-only byte or string literal for `mir::interpret`. pub fn allocate_bytes(self, bytes: &[u8]) -> interpret::AllocId { // Create an allocation that just contains these bytes. - let alloc = interpret::Allocation::from_byte_aligned_bytes(bytes); + let alloc = interpret::Allocation::from_bytes_byte_aligned_immutable(bytes); let alloc = self.intern_const_alloc(alloc); self.create_memory_alloc(alloc) } @@ -1130,12 +1123,11 @@ impl<'tcx> TyCtxt<'tcx> { arena: &'tcx WorkerLocal>, resolutions: ty::ResolverOutputs, krate: &'tcx hir::Crate<'tcx>, - definitions: &'tcx Definitions, dep_graph: DepGraph, on_disk_cache: Option>, queries: &'tcx dyn query::QueryEngine<'tcx>, crate_name: &str, - output_filenames: &OutputFilenames, + output_filenames: OutputFilenames, ) -> GlobalCtxt<'tcx> { let data_layout = TargetDataLayout::parse(&s.target).unwrap_or_else(|err| { s.fatal(&err); @@ -1146,12 +1138,6 @@ impl<'tcx> TyCtxt<'tcx> { let common_consts = CommonConsts::new(&interners, &common_types); let cstore = resolutions.cstore; - let mut trait_map: FxHashMap<_, FxHashMap<_, _>> = FxHashMap::default(); - for (hir_id, v) in krate.trait_map.iter() { - let map = trait_map.entry(hir_id.owner).or_default(); - map.insert(hir_id.local_id, StableVec::new(v.to_vec())); - } - GlobalCtxt { sess: s, lint_store, @@ -1165,14 +1151,13 @@ impl<'tcx> TyCtxt<'tcx> { consts: common_consts, visibilities: resolutions.visibilities, extern_crate_map: resolutions.extern_crate_map, - trait_map, export_map: resolutions.export_map, maybe_unused_trait_imports: resolutions.maybe_unused_trait_imports, maybe_unused_extern_crates: resolutions.maybe_unused_extern_crates, glob_map: resolutions.glob_map, extern_prelude: resolutions.extern_prelude, untracked_crate: krate, - definitions, + definitions: resolutions.definitions, on_disk_cache, queries, query_caches: query::QueryCaches::default(), @@ -1186,7 +1171,7 @@ impl<'tcx> TyCtxt<'tcx> { stability_interner: Default::default(), const_stability_interner: Default::default(), alloc_map: Lock::new(interpret::AllocMap::new()), - output_filenames: Arc::new(output_filenames.clone()), + output_filenames: Arc::new(output_filenames), main_def: resolutions.main_def, } } @@ -1219,18 +1204,18 @@ impl<'tcx> TyCtxt<'tcx> { } pub fn lib_features(self) -> &'tcx middle::lib_features::LibFeatures { - self.get_lib_features(LOCAL_CRATE) + self.get_lib_features(()) } /// Obtain all lang items of this crate and all dependencies (recursively) pub fn lang_items(self) -> &'tcx rustc_hir::lang_items::LanguageItems { - self.get_lang_items(LOCAL_CRATE) + self.get_lang_items(()) } /// Obtain the given diagnostic item's `DefId`. Use `is_diagnostic_item` if you just want to /// compare against another `DefId`, since `is_diagnostic_item` is cheaper. pub fn get_diagnostic_item(self, name: Symbol) -> Option { - self.all_diagnostic_items(LOCAL_CRATE).get(&name).copied() + self.all_diagnostic_items(()).get(&name).copied() } /// Check whether the diagnostic item with the given `name` has the given `DefId`. @@ -1239,11 +1224,11 @@ impl<'tcx> TyCtxt<'tcx> { } pub fn stability(self) -> &'tcx stability::Index<'tcx> { - self.stability_index(LOCAL_CRATE) + self.stability_index(()) } pub fn crates(self) -> &'tcx [CrateNum] { - self.all_crate_nums(LOCAL_CRATE) + self.all_crate_nums(()) } pub fn allocator_kind(self) -> Option { @@ -1251,7 +1236,7 @@ impl<'tcx> TyCtxt<'tcx> { } pub fn features(self) -> &'tcx rustc_feature::Features { - self.features_query(LOCAL_CRATE) + self.features_query(()) } pub fn def_key(self, id: DefId) -> rustc_hir::definitions::DefKey { @@ -1271,12 +1256,6 @@ impl<'tcx> TyCtxt<'tcx> { } } - /// Returns whether or not the crate with CrateNum 'cnum' - /// is marked as a private dependency - pub fn is_private_dep(self, cnum: CrateNum) -> bool { - if cnum == LOCAL_CRATE { false } else { self.cstore.crate_is_private_dep_untracked(cnum) } - } - #[inline] pub fn def_path_hash(self, def_id: DefId) -> rustc_hir::definitions::DefPathHash { if let Some(def_id) = def_id.as_local() { @@ -1286,6 +1265,11 @@ impl<'tcx> TyCtxt<'tcx> { } } + #[inline] + pub fn stable_crate_id(self, cnum: CrateNum) -> StableCrateId { + self.def_path_hash(cnum.as_def_id()).stable_crate_id() + } + pub fn def_path_debug_str(self, def_id: DefId) -> String { // We are explicitly not going through queries here in order to get // crate name and disambiguator since this code is called from debug!() @@ -1310,10 +1294,6 @@ impl<'tcx> TyCtxt<'tcx> { ) } - pub fn metadata_encoding_version(self) -> Vec { - self.cstore.metadata_encoding_version().to_vec() - } - pub fn encode_metadata(self) -> EncodedMetadata { let _prof_timer = self.prof.verbose_generic_activity("generate_crate_metadata"); self.cstore.encode_metadata(self) @@ -1329,14 +1309,14 @@ impl<'tcx> TyCtxt<'tcx> { pub fn create_stable_hashing_context(self) -> StableHashingContext<'tcx> { let krate = self.gcx.untracked_crate; - StableHashingContext::new(self.sess, krate, self.definitions, &*self.cstore) + StableHashingContext::new(self.sess, krate, &self.definitions, &*self.cstore) } #[inline(always)] pub fn create_no_span_stable_hashing_context(self) -> StableHashingContext<'tcx> { let krate = self.gcx.untracked_crate; - StableHashingContext::ignore_spans(self.sess, krate, self.definitions, &*self.cstore) + StableHashingContext::ignore_spans(self.sess, krate, &self.definitions, &*self.cstore) } pub fn serialize_query_result_cache(self, encoder: &mut FileEncoder) -> FileEncodeResult { @@ -1705,7 +1685,7 @@ pub mod tls { #[cfg(not(parallel_compiler))] thread_local! { /// A thread local variable that stores a pointer to the current `ImplicitCtxt`. - static TLV: Cell = Cell::new(0); + static TLV: Cell = const { Cell::new(0) }; } /// Sets TLV to `value` during the call to `f`. @@ -2624,7 +2604,7 @@ impl<'tcx> TyCtxt<'tcx> { lint: &'static Lint, mut id: hir::HirId, ) -> (Level, LintLevelSource) { - let sets = self.lint_levels(LOCAL_CRATE); + let sets = self.lint_levels(()); loop { if let Some(pair) = sets.level_and_source(lint, id, self.sess) { return pair; @@ -2658,8 +2638,10 @@ impl<'tcx> TyCtxt<'tcx> { struct_lint_level(self.sess, lint, level, src, None, decorate); } - pub fn in_scope_traits(self, id: HirId) -> Option<&'tcx StableVec> { - self.in_scope_traits_map(id.owner).and_then(|map| map.get(&id.local_id)) + pub fn in_scope_traits(self, id: HirId) -> Option<&'tcx [TraitCandidate]> { + let map = self.in_scope_traits_map(id.owner)?; + let candidates = map.get(&id.local_id)?; + Some(&*candidates) } pub fn named_region(self, id: HirId) -> Option { @@ -2686,6 +2668,10 @@ impl<'tcx> TyCtxt<'tcx> { .iter(), ) } + + pub fn lifetime_scope(self, id: HirId) -> Option { + self.lifetime_scope_map(id.owner).and_then(|mut map| map.remove(&id.local_id)) + } } impl TyCtxtAt<'tcx> { @@ -2785,17 +2771,14 @@ fn ptr_eq(t: *const T, u: *const U) -> bool { } pub fn provide(providers: &mut ty::query::Providers) { - providers.in_scope_traits_map = |tcx, id| tcx.gcx.trait_map.get(&id); + providers.in_scope_traits_map = |tcx, id| tcx.hir_crate(()).trait_map.get(&id); providers.module_exports = |tcx, id| tcx.gcx.export_map.get(&id).map(|v| &v[..]); providers.crate_name = |tcx, id| { assert_eq!(id, LOCAL_CRATE); tcx.crate_name }; providers.maybe_unused_trait_import = |tcx, id| tcx.maybe_unused_trait_imports.contains(&id); - providers.maybe_unused_extern_crates = |tcx, cnum| { - assert_eq!(cnum, LOCAL_CRATE); - &tcx.maybe_unused_extern_crates[..] - }; + providers.maybe_unused_extern_crates = |tcx, ()| &tcx.maybe_unused_extern_crates[..]; providers.names_imported_by_glob_use = |tcx, id| tcx.arena.alloc(tcx.glob_map.get(&id).cloned().unwrap_or_default()); @@ -2812,18 +2795,9 @@ pub fn provide(providers: &mut ty::query::Providers) { tcx.stability().local_deprecation_entry(id) }; providers.extern_mod_stmt_cnum = |tcx, id| tcx.extern_crate_map.get(&id).cloned(); - providers.all_crate_nums = |tcx, cnum| { - assert_eq!(cnum, LOCAL_CRATE); - tcx.arena.alloc_slice(&tcx.cstore.crates_untracked()) - }; - providers.output_filenames = |tcx, cnum| { - assert_eq!(cnum, LOCAL_CRATE); - tcx.output_filenames.clone() - }; - providers.features_query = |tcx, cnum| { - assert_eq!(cnum, LOCAL_CRATE); - tcx.sess.features_untracked() - }; + providers.all_crate_nums = |tcx, ()| tcx.arena.alloc_slice(&tcx.cstore.crates_untracked()); + providers.output_filenames = |tcx, ()| tcx.output_filenames.clone(); + providers.features_query = |tcx, ()| tcx.sess.features_untracked(); providers.is_panic_runtime = |tcx, cnum| { assert_eq!(cnum, LOCAL_CRATE); tcx.sess.contains_name(tcx.hir().krate_attrs(), sym::panic_runtime) diff --git a/compiler/rustc_middle/src/ty/diagnostics.rs b/compiler/rustc_middle/src/ty/diagnostics.rs index 982c8a354b..bfb4c0cb53 100644 --- a/compiler/rustc_middle/src/ty/diagnostics.rs +++ b/compiler/rustc_middle/src/ty/diagnostics.rs @@ -54,7 +54,7 @@ impl<'tcx> TyS<'tcx> { /// ADTs with no type arguments. pub fn is_simple_text(&self) -> bool { match self.kind() { - Adt(_, substs) => substs.types().next().is_none(), + Adt(_, substs) => substs.non_erasable_generics().next().is_none(), Ref(_, ty, _) => ty.is_simple_text(), _ => self.is_simple_ty(), } diff --git a/compiler/rustc_middle/src/ty/error.rs b/compiler/rustc_middle/src/ty/error.rs index 008e6d015e..96aae3bd70 100644 --- a/compiler/rustc_middle/src/ty/error.rs +++ b/compiler/rustc_middle/src/ty/error.rs @@ -159,10 +159,23 @@ impl<'tcx> fmt::Display for TypeError<'tcx> { ) }), IntMismatch(ref values) => { - write!(f, "expected `{:?}`, found `{:?}`", values.expected, values.found) + let expected = match values.expected { + ty::IntVarValue::IntType(ty) => ty.name_str(), + ty::IntVarValue::UintType(ty) => ty.name_str(), + }; + let found = match values.found { + ty::IntVarValue::IntType(ty) => ty.name_str(), + ty::IntVarValue::UintType(ty) => ty.name_str(), + }; + write!(f, "expected `{}`, found `{}`", expected, found) } FloatMismatch(ref values) => { - write!(f, "expected `{:?}`, found `{:?}`", values.expected, values.found) + write!( + f, + "expected `{}`, found `{}`", + values.expected.name_str(), + values.found.name_str() + ) } VariadicMismatch(ref values) => write!( f, diff --git a/compiler/rustc_middle/src/ty/flags.rs b/compiler/rustc_middle/src/ty/flags.rs index 92288c8982..9faa172a49 100644 --- a/compiler/rustc_middle/src/ty/flags.rs +++ b/compiler/rustc_middle/src/ty/flags.rs @@ -141,7 +141,9 @@ impl FlagComputation { &ty::Infer(infer) => { self.add_flags(TypeFlags::STILL_FURTHER_SPECIALIZABLE); match infer { - ty::FreshTy(_) | ty::FreshIntTy(_) | ty::FreshFloatTy(_) => {} + ty::FreshTy(_) | ty::FreshIntTy(_) | ty::FreshFloatTy(_) => { + self.add_flags(TypeFlags::HAS_TY_FRESH) + } ty::TyVar(_) | ty::IntVar(_) | ty::FloatVar(_) => { self.add_flags(TypeFlags::HAS_TY_INFER) @@ -278,7 +280,7 @@ impl FlagComputation { ty::ConstKind::Infer(infer) => { self.add_flags(TypeFlags::STILL_FURTHER_SPECIALIZABLE); match infer { - InferConst::Fresh(_) => {} + InferConst::Fresh(_) => self.add_flags(TypeFlags::HAS_CT_FRESH), InferConst::Var(_) => self.add_flags(TypeFlags::HAS_CT_INFER), } } diff --git a/compiler/rustc_middle/src/ty/generics.rs b/compiler/rustc_middle/src/ty/generics.rs index c8fdbc30d1..4e3f475a91 100644 --- a/compiler/rustc_middle/src/ty/generics.rs +++ b/compiler/rustc_middle/src/ty/generics.rs @@ -93,6 +93,7 @@ pub struct Generics { } impl<'tcx> Generics { + #[inline] pub fn count(&self) -> usize { self.parent_count + self.params.len() } diff --git a/compiler/rustc_middle/src/ty/layout.rs b/compiler/rustc_middle/src/ty/layout.rs index c2e9dba6c8..28a44b09de 100644 --- a/compiler/rustc_middle/src/ty/layout.rs +++ b/compiler/rustc_middle/src/ty/layout.rs @@ -367,7 +367,14 @@ impl<'tcx> LayoutCx<'tcx, TyCtxt<'tcx>> { for &i in &inverse_memory_index { let field = fields[i as usize]; if !sized { - bug!("univariant: field #{} of `{}` comes after unsized field", offsets.len(), ty); + self.tcx.sess.delay_span_bug( + DUMMY_SP, + &format!( + "univariant: field #{} of `{}` comes after unsized field", + offsets.len(), + ty + ), + ); } if field.is_unsized() { @@ -2579,7 +2586,7 @@ where fn adjust_for_abi(&mut self, cx: &C, abi: SpecAbi); } -fn fn_can_unwind( +pub fn fn_can_unwind( panic_strategy: PanicStrategy, codegen_fn_attr_flags: CodegenFnAttrFlags, call_conv: Conv, @@ -2641,6 +2648,43 @@ fn fn_can_unwind( } } +pub fn conv_from_spec_abi(tcx: TyCtxt<'_>, abi: SpecAbi) -> Conv { + use rustc_target::spec::abi::Abi::*; + match tcx.sess.target.adjust_abi(abi) { + RustIntrinsic | PlatformIntrinsic | Rust | RustCall => Conv::Rust, + + // It's the ABI's job to select this, not ours. + System { .. } => bug!("system abi should be selected elsewhere"), + EfiApi => bug!("eficall abi should be selected elsewhere"), + + Stdcall { .. } => Conv::X86Stdcall, + Fastcall => Conv::X86Fastcall, + Vectorcall => Conv::X86VectorCall, + Thiscall { .. } => Conv::X86ThisCall, + C { .. } => Conv::C, + Unadjusted => Conv::C, + Win64 => Conv::X86_64Win64, + SysV64 => Conv::X86_64SysV, + Aapcs => Conv::ArmAapcs, + CCmseNonSecureCall => Conv::CCmseNonSecureCall, + PtxKernel => Conv::PtxKernel, + Msp430Interrupt => Conv::Msp430Intr, + X86Interrupt => Conv::X86Intr, + AmdGpuKernel => Conv::AmdGpuKernel, + AvrInterrupt => Conv::AvrInterrupt, + AvrNonBlockingInterrupt => Conv::AvrNonBlockingInterrupt, + Wasm => Conv::C, + + // These API constants ought to be more specific... + Cdecl => Conv::C, + } +} + +pub fn fn_ptr_codegen_fn_attr_flags() -> CodegenFnAttrFlags { + // Assume that fn pointers may always unwind + CodegenFnAttrFlags::UNWIND +} + impl<'tcx, C> FnAbiExt<'tcx, C> for call::FnAbi<'tcx, Ty<'tcx>> where C: LayoutOf, TyAndLayout = TyAndLayout<'tcx>> @@ -2650,10 +2694,7 @@ where + HasParamEnv<'tcx>, { fn of_fn_ptr(cx: &C, sig: ty::PolyFnSig<'tcx>, extra_args: &[Ty<'tcx>]) -> Self { - // Assume that fn pointers may always unwind - let codegen_fn_attr_flags = CodegenFnAttrFlags::UNWIND; - - call::FnAbi::new_internal(cx, sig, extra_args, None, codegen_fn_attr_flags, false) + call::FnAbi::new_internal(cx, sig, extra_args, None, fn_ptr_codegen_fn_attr_flags(), false) } fn of_instance(cx: &C, instance: ty::Instance<'tcx>, extra_args: &[Ty<'tcx>]) -> Self { @@ -2689,35 +2730,7 @@ where let sig = cx.tcx().normalize_erasing_late_bound_regions(ty::ParamEnv::reveal_all(), sig); - use rustc_target::spec::abi::Abi::*; - let conv = match cx.tcx().sess.target.adjust_abi(sig.abi) { - RustIntrinsic | PlatformIntrinsic | Rust | RustCall => Conv::Rust, - - // It's the ABI's job to select this, not ours. - System { .. } => bug!("system abi should be selected elsewhere"), - EfiApi => bug!("eficall abi should be selected elsewhere"), - - Stdcall { .. } => Conv::X86Stdcall, - Fastcall => Conv::X86Fastcall, - Vectorcall => Conv::X86VectorCall, - Thiscall { .. } => Conv::X86ThisCall, - C { .. } => Conv::C, - Unadjusted => Conv::C, - Win64 => Conv::X86_64Win64, - SysV64 => Conv::X86_64SysV, - Aapcs => Conv::ArmAapcs, - CCmseNonSecureCall => Conv::CCmseNonSecureCall, - PtxKernel => Conv::PtxKernel, - Msp430Interrupt => Conv::Msp430Intr, - X86Interrupt => Conv::X86Intr, - AmdGpuKernel => Conv::AmdGpuKernel, - AvrInterrupt => Conv::AvrInterrupt, - AvrNonBlockingInterrupt => Conv::AvrNonBlockingInterrupt, - Wasm => Conv::C, - - // These API constants ought to be more specific... - Cdecl => Conv::C, - }; + let conv = conv_from_spec_abi(cx.tcx(), sig.abi); let mut inputs = sig.inputs(); let extra_args = if sig.abi == RustCall { @@ -2753,6 +2766,7 @@ where target.os == "linux" && target.arch == "sparc64" && target_env_gnu_like; let linux_powerpc_gnu_like = target.os == "linux" && target.arch == "powerpc" && target_env_gnu_like; + use SpecAbi::*; let rust_abi = matches!(sig.abi, RustIntrinsic | PlatformIntrinsic | Rust | RustCall); // Handle safe Rust thin and fat pointers. diff --git a/compiler/rustc_middle/src/ty/list.rs b/compiler/rustc_middle/src/ty/list.rs index e657088a5e..44dfcbf186 100644 --- a/compiler/rustc_middle/src/ty/list.rs +++ b/compiler/rustc_middle/src/ty/list.rs @@ -37,9 +37,11 @@ pub struct List { unsafe impl<'a, T: 'a> rustc_data_structures::tagged_ptr::Pointer for &'a List { const BITS: usize = std::mem::align_of::().trailing_zeros() as usize; + #[inline] fn into_usize(self) -> usize { self as *const List as usize } + #[inline] unsafe fn from_usize(ptr: usize) -> Self { &*(ptr as *const List) } diff --git a/compiler/rustc_middle/src/ty/mod.rs b/compiler/rustc_middle/src/ty/mod.rs index af49533753..227aa8dc28 100644 --- a/compiler/rustc_middle/src/ty/mod.rs +++ b/compiler/rustc_middle/src/ty/mod.rs @@ -36,10 +36,9 @@ use rustc_data_structures::sync::{self, par_iter, ParallelIterator}; use rustc_data_structures::tagged_ptr::CopyTaggedPtr; use rustc_hir as hir; use rustc_hir::def::{CtorKind, CtorOf, DefKind, Res}; -use rustc_hir::def_id::{CrateNum, DefId, DefIdMap, LocalDefId, CRATE_DEF_INDEX}; +use rustc_hir::def_id::{CrateNum, DefId, LocalDefId, LocalDefIdMap, CRATE_DEF_INDEX}; use rustc_hir::{Constness, Node}; use rustc_macros::HashStable; -use rustc_span::hygiene::ExpnId; use rustc_span::symbol::{kw, Ident, Symbol}; use rustc_span::Span; use rustc_target::abi::Align; @@ -59,7 +58,7 @@ pub use self::consts::{Const, ConstInt, ConstKind, InferConst, ScalarInt, Uneval pub use self::context::{ tls, CanonicalUserType, CanonicalUserTypeAnnotation, CanonicalUserTypeAnnotations, CtxtInterners, DelaySpanBugEmitted, FreeRegionInfo, GeneratorInteriorTypeCause, GlobalCtxt, - Lift, ResolvedOpaqueTy, TyCtxt, TypeckResults, UserType, UserTypeAnnotationIndex, + Lift, TyCtxt, TypeckResults, UserType, UserTypeAnnotationIndex, }; pub use self::instance::{Instance, InstanceDef}; pub use self::list::List; @@ -72,7 +71,7 @@ pub use self::sty::{ ExistentialPredicate, ExistentialProjection, ExistentialTraitRef, FnSig, FreeRegion, GenSig, GeneratorSubsts, GeneratorSubstsParts, ParamConst, ParamTy, PolyExistentialProjection, PolyExistentialTraitRef, PolyFnSig, PolyGenSig, PolyTraitRef, ProjectionTy, Region, RegionKind, - RegionVid, TraitRef, TyKind, TypeAndMut, UpvarSubsts, + RegionVid, TraitRef, TyKind, TypeAndMut, UpvarSubsts, VarianceDiagInfo, VarianceDiagMutKind, }; pub use self::trait_def::TraitDef; @@ -269,7 +268,7 @@ pub struct CrateVariancesMap<'tcx> { // the types of AST nodes. #[derive(Copy, Clone, PartialEq, Eq, Hash)] pub struct CReaderCacheKey { - pub cnum: CrateNum, + pub cnum: Option, pub pos: usize, } @@ -836,6 +835,12 @@ impl<'tcx> InstantiatedPredicates<'tcx> { } } +#[derive(Copy, Clone, Debug, PartialEq, Eq, HashStable, TyEncodable, TyDecodable)] +pub struct OpaqueTypeKey<'tcx> { + pub def_id: DefId, + pub substs: SubstsRef<'tcx>, +} + rustc_index::newtype_index! { /// "Universes" are used during type- and trait-checking in the /// presence of `for<..>` binders to control what sets of names are @@ -1097,12 +1102,14 @@ pub struct ParamEnv<'tcx> { unsafe impl rustc_data_structures::tagged_ptr::Tag for traits::Reveal { const BITS: usize = 1; + #[inline] fn into_usize(self) -> usize { match self { traits::Reveal::UserFacing => 0, traits::Reveal::All => 1, } } + #[inline] unsafe fn from_usize(ptr: usize) -> Self { match ptr { 0 => traits::Reveal::UserFacing, @@ -1200,6 +1207,7 @@ impl<'tcx> ParamEnv<'tcx> { } /// Returns this same environment but with no caller bounds. + #[inline] pub fn without_caller_bounds(self) -> Self { Self::new(List::empty(), self.reveal()) } @@ -1618,7 +1626,7 @@ impl<'tcx> TyCtxt<'tcx> { fn item_name_from_def_id(self, def_id: DefId) -> Option { if def_id.index == CRATE_DEF_INDEX { - Some(self.original_crate_name(def_id.krate)) + Some(self.crate_name(def_id.krate)) } else { let def_key = self.def_key(def_id); match def_key.disambiguated_data.data { @@ -1859,20 +1867,11 @@ impl<'tcx> TyCtxt<'tcx> { && use_name .span .ctxt() - .hygienic_eq(def_name.span.ctxt(), self.expansion_that_defined(def_parent_def_id)) - } - - pub fn expansion_that_defined(self, scope: DefId) -> ExpnId { - match scope.as_local() { - // Parsing and expansion aren't incremental, so we don't - // need to go through a query for the same-crate case. - Some(scope) => self.hir().definitions().expansion_that_defined(scope), - None => self.expn_that_defined(scope), - } + .hygienic_eq(def_name.span.ctxt(), self.expn_that_defined(def_parent_def_id)) } pub fn adjust_ident(self, mut ident: Ident, scope: DefId) -> Ident { - ident.span.normalize_to_macros_2_0_and_adjust(self.expansion_that_defined(scope)); + ident.span.normalize_to_macros_2_0_and_adjust(self.expn_that_defined(scope)); ident } @@ -1883,8 +1882,7 @@ impl<'tcx> TyCtxt<'tcx> { block: hir::HirId, ) -> (Ident, DefId) { let scope = - match ident.span.normalize_to_macros_2_0_and_adjust(self.expansion_that_defined(scope)) - { + match ident.span.normalize_to_macros_2_0_and_adjust(self.expn_that_defined(scope)) { Some(actual_expansion) => { self.hir().definitions().parent_module_of_macro_def(actual_expansion) } @@ -1970,7 +1968,6 @@ pub fn provide(providers: &mut ty::query::Providers) { super::util::bug::provide(providers); *providers = ty::query::Providers { trait_impls_of: trait_def::trait_impls_of_provider, - all_local_trait_impls: trait_def::all_local_trait_impls, type_uninhabited_from: inhabitedness::type_uninhabited_from, const_param_default: consts::const_param_default, ..*providers @@ -1984,7 +1981,7 @@ pub fn provide(providers: &mut ty::query::Providers) { /// (constructing this map requires touching the entire crate). #[derive(Clone, Debug, Default, HashStable)] pub struct CrateInherentImpls { - pub inherent_impls: DefIdMap>, + pub inherent_impls: LocalDefIdMap>, } #[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, TyEncodable, HashStable)] diff --git a/compiler/rustc_middle/src/ty/print/pretty.rs b/compiler/rustc_middle/src/ty/print/pretty.rs index 1989c91a87..25557bdd10 100644 --- a/compiler/rustc_middle/src/ty/print/pretty.rs +++ b/compiler/rustc_middle/src/ty/print/pretty.rs @@ -1,5 +1,5 @@ use crate::middle::cstore::{ExternCrate, ExternCrateSource}; -use crate::mir::interpret::{AllocId, ConstValue, GlobalAlloc, Pointer, Scalar}; +use crate::mir::interpret::{AllocRange, ConstValue, GlobalAlloc, Pointer, Scalar}; use crate::ty::subst::{GenericArg, GenericArgKind, Subst}; use crate::ty::{self, ConstInt, DefIdTree, ParamConst, ScalarInt, Ty, TyCtxt, TypeFoldable}; use rustc_apfloat::ieee::{Double, Single}; @@ -7,7 +7,7 @@ use rustc_data_structures::fx::FxHashMap; use rustc_data_structures::sso::SsoHashSet; use rustc_hir as hir; use rustc_hir::def::{self, CtorKind, DefKind, Namespace}; -use rustc_hir::def_id::{CrateNum, DefId, DefIdSet, CRATE_DEF_INDEX, LOCAL_CRATE}; +use rustc_hir::def_id::{DefId, DefIdSet, CRATE_DEF_INDEX, LOCAL_CRATE}; use rustc_hir::definitions::{DefPathData, DefPathDataName, DisambiguatedDefPathData}; use rustc_hir::ItemKind; use rustc_session::config::TrimmedDefPaths; @@ -55,10 +55,10 @@ macro_rules! define_scoped_cx { } thread_local! { - static FORCE_IMPL_FILENAME_LINE: Cell = Cell::new(false); - static SHOULD_PREFIX_WITH_CRATE: Cell = Cell::new(false); - static NO_TRIMMED_PATH: Cell = Cell::new(false); - static NO_QUERIES: Cell = Cell::new(false); + static FORCE_IMPL_FILENAME_LINE: Cell = const { Cell::new(false) }; + static SHOULD_PREFIX_WITH_CRATE: Cell = const { Cell::new(false) }; + static NO_TRIMMED_PATH: Cell = const { Cell::new(false) }; + static NO_QUERIES: Cell = const { Cell::new(false) }; } /// Avoids running any queries during any prints that occur @@ -285,7 +285,7 @@ pub trait PrettyPrinter<'tcx>: return Ok((self, false)); } - match self.tcx().trimmed_def_paths(LOCAL_CRATE).get(&def_id) { + match self.tcx().trimmed_def_paths(()).get(&def_id) { None => Ok((self, false)), Some(symbol) => { self.write_str(&symbol.as_str())?; @@ -361,7 +361,7 @@ pub trait PrettyPrinter<'tcx>: return Ok((self, false)); } - let visible_parent_map = self.tcx().visible_parent_map(LOCAL_CRATE); + let visible_parent_map = self.tcx().visible_parent_map(()); let mut cur_def_key = self.tcx().def_key(def_id); debug!("try_print_visible_def_path: cur_def_key={:?}", cur_def_key); @@ -452,7 +452,7 @@ pub trait PrettyPrinter<'tcx>: } // Re-exported `extern crate` (#43189). DefPathData::CrateRoot => { - data = DefPathData::TypeNs(self.tcx().original_crate_name(def_id.krate)); + data = DefPathData::TypeNs(self.tcx().crate_name(def_id.krate)); } _ => {} } @@ -667,7 +667,12 @@ pub trait PrettyPrinter<'tcx>: if let Some(did) = did.as_local() { let hir_id = self.tcx().hir().local_def_id_to_hir_id(did); let span = self.tcx().hir().span(hir_id); - p!(write("@{}", self.tcx().sess.source_map().span_to_string(span))); + p!(write( + "@{}", + // This may end up in stderr diagnostics but it may also be emitted + // into MIR. Hence we use the remapped path if available + self.tcx().sess.source_map().span_to_embeddable_string(span) + )); } else { p!(write("@"), print_def_path(did, substs)); } @@ -702,7 +707,12 @@ pub trait PrettyPrinter<'tcx>: p!("@", print_def_path(did.to_def_id(), substs)); } else { let span = self.tcx().hir().span(hir_id); - p!(write("@{}", self.tcx().sess.source_map().span_to_string(span))); + p!(write( + "@{}", + // This may end up in stderr diagnostics but it may also be emitted + // into MIR. Hence we use the remapped path if available + self.tcx().sess.source_map().span_to_embeddable_string(span) + )); } } else { p!(write("@"), print_def_path(did, substs)); @@ -994,9 +1004,9 @@ pub trait PrettyPrinter<'tcx>: _, ) => match self.tcx().get_global_alloc(ptr.alloc_id) { Some(GlobalAlloc::Memory(alloc)) => { - let bytes = int.assert_bits(self.tcx().data_layout.pointer_size); - let size = Size::from_bytes(bytes); - if let Ok(byte_str) = alloc.get_bytes(&self.tcx(), ptr, size) { + let len = int.assert_bits(self.tcx().data_layout.pointer_size); + let range = AllocRange { start: ptr.offset, size: Size::from_bytes(len) }; + if let Ok(byte_str) = alloc.get_bytes(&self.tcx(), range) { p!(pretty_print_byte_str(byte_str)) } else { p!("") @@ -1171,10 +1181,9 @@ pub trait PrettyPrinter<'tcx>: (ConstValue::ByRef { alloc, offset }, ty::Array(t, n)) if *t == u8_type => { let n = n.val.try_to_bits(self.tcx().data_layout.pointer_size).unwrap(); // cast is ok because we already checked for pointer size (32 or 64 bit) above - let n = Size::from_bytes(n); - let ptr = Pointer::new(AllocId(0), offset); + let range = AllocRange { start: offset, size: Size::from_bytes(n) }; - let byte_str = alloc.get_bytes(&self.tcx(), ptr, n).unwrap(); + let byte_str = alloc.get_bytes(&self.tcx(), range).unwrap(); p!("*"); p!(pretty_print_byte_str(byte_str)); Ok(self) @@ -1407,7 +1416,13 @@ impl Printer<'tcx> for FmtPrinter<'_, 'tcx, F> { if !self.empty_path { write!(self, "::")?; } - write!(self, "", self.tcx.sess.source_map().span_to_string(span))?; + write!( + self, + "", + // This may end up in stderr diagnostics but it may also be emitted + // into MIR. Hence we use the remapped path if available + self.tcx.sess.source_map().span_to_embeddable_string(span) + )?; self.empty_path = false; return Ok(self); @@ -2286,9 +2301,7 @@ fn for_each_def(tcx: TyCtxt<'_>, mut collect_fn: impl for<'b> FnMut(&'b Ident, N /// `std::vec::Vec` to just `Vec`, as long as there is no other `Vec` importable anywhere. /// /// The implementation uses similar import discovery logic to that of 'use' suggestions. -fn trimmed_def_paths(tcx: TyCtxt<'_>, crate_num: CrateNum) -> FxHashMap { - assert_eq!(crate_num, LOCAL_CRATE); - +fn trimmed_def_paths(tcx: TyCtxt<'_>, (): ()) -> FxHashMap { let mut map = FxHashMap::default(); if let TrimmedDefPaths::GoodPath = tcx.sess.opts.trimmed_def_paths { diff --git a/compiler/rustc_middle/src/ty/query/mod.rs b/compiler/rustc_middle/src/ty/query/mod.rs index c170858ba8..297110ee3e 100644 --- a/compiler/rustc_middle/src/ty/query/mod.rs +++ b/compiler/rustc_middle/src/ty/query/mod.rs @@ -1,6 +1,5 @@ use crate::dep_graph; use crate::hir::exports::Export; -use crate::hir::map; use crate::infer::canonical::{self, Canonical}; use crate::lint::LintLevelMap; use crate::middle::codegen_fn_attrs::CodegenFnAttrs; @@ -10,13 +9,16 @@ use crate::middle::exported_symbols::{ExportedSymbol, SymbolExportLevel}; use crate::middle::lib_features::LibFeatures; use crate::middle::privacy::AccessLevels; use crate::middle::region; -use crate::middle::resolve_lifetime::{ObjectLifetimeDefault, Region, ResolveLifetimes}; +use crate::middle::resolve_lifetime::{ + LifetimeScopeForPath, ObjectLifetimeDefault, Region, ResolveLifetimes, +}; use crate::middle::stability::{self, DeprecationEntry}; use crate::mir; use crate::mir::interpret::GlobalId; use crate::mir::interpret::{ConstAlloc, LitToConstError, LitToConstInput}; use crate::mir::interpret::{ConstValue, EvalToAllocationRawResult, EvalToConstValueResult}; use crate::mir::mono::CodegenUnit; +use crate::thir; use crate::traits::query::{ CanonicalPredicateGoal, CanonicalProjectionGoal, CanonicalTyGoal, CanonicalTypeOpAscribeUserTypeGoal, CanonicalTypeOpEqGoal, CanonicalTypeOpNormalizeGoal, @@ -32,7 +34,6 @@ use crate::ty::subst::{GenericArg, SubstsRef}; use crate::ty::util::AlwaysRequiresDrop; use crate::ty::{self, AdtSizedConstraint, CrateInherentImpls, ParamEnvAnd, Ty, TyCtxt}; use rustc_data_structures::fx::{FxHashMap, FxHashSet, FxIndexMap}; -use rustc_data_structures::stable_hasher::StableVec; use rustc_data_structures::steal::Steal; use rustc_data_structures::svh::Svh; use rustc_data_structures::sync::Lrc; @@ -233,6 +234,7 @@ macro_rules! define_callbacks { } pub trait QueryEngine<'tcx>: rustc_data_structures::sync::Sync { + #[cfg(parallel_compiler)] unsafe fn deadlock(&'tcx self, tcx: TyCtxt<'tcx>, registry: &rustc_rayon_core::Registry); fn encode_query_results( diff --git a/compiler/rustc_middle/src/ty/query/on_disk_cache.rs b/compiler/rustc_middle/src/ty/query/on_disk_cache.rs index 43b775f808..6df8e64fad 100644 --- a/compiler/rustc_middle/src/ty/query/on_disk_cache.rs +++ b/compiler/rustc_middle/src/ty/query/on_disk_cache.rs @@ -9,9 +9,8 @@ use rustc_data_structures::sync::{HashMapExt, Lock, Lrc, OnceCell}; use rustc_data_structures::thin_vec::ThinVec; use rustc_data_structures::unhash::UnhashMap; use rustc_errors::Diagnostic; -use rustc_hir::def_id::{CrateNum, DefId, DefIndex, LocalDefId, LOCAL_CRATE}; +use rustc_hir::def_id::{CrateNum, DefId, DefIndex, LocalDefId, StableCrateId, LOCAL_CRATE}; use rustc_hir::definitions::DefPathHash; -use rustc_hir::definitions::Definitions; use rustc_index::vec::{Idx, IndexVec}; use rustc_query_system::dep_graph::DepContext; use rustc_query_system::query::QueryContext; @@ -19,7 +18,7 @@ use rustc_serialize::{ opaque::{self, FileEncodeResult, FileEncoder, IntEncodedWithFixedSize}, Decodable, Decoder, Encodable, Encoder, }; -use rustc_session::{CrateDisambiguator, Session}; +use rustc_session::Session; use rustc_span::hygiene::{ ExpnDataDecodeMode, ExpnDataEncodeMode, ExpnId, HygieneDecodeContext, HygieneEncodeContext, SyntaxContext, SyntaxContextData, @@ -28,7 +27,6 @@ use rustc_span::source_map::{SourceMap, StableSourceFileId}; use rustc_span::CachingSourceMapView; use rustc_span::{BytePos, ExpnData, SourceFile, Span, DUMMY_SP}; use std::collections::hash_map::Entry; -use std::iter::FromIterator; use std::mem; const TAG_FILE_FOOTER: u128 = 0xC0FFEE_C0FFEE_C0FFEE_C0FFEE_C0FFEE; @@ -53,11 +51,10 @@ pub struct OnDiskCache<'sess> { // session. current_diagnostics: Lock>>, - prev_cnums: Vec<(u32, String, CrateDisambiguator)>, - cnum_map: OnceCell>>, + cnum_map: OnceCell>, source_map: &'sess SourceMap, - file_index_to_stable_id: FxHashMap, + file_index_to_stable_id: FxHashMap, // Caches that are populated lazily during decoding. file_index_to_file: Lock>>, @@ -104,12 +101,6 @@ pub struct OnDiskCache<'sess> { // during the next compilation session. latest_foreign_def_path_hashes: Lock>, - // Maps `DefPathHashes` to their corresponding `LocalDefId`s for all - // local items in the current compilation session. This is only populated - // when we are in incremental mode and have loaded a pre-existing cache - // from disk, since this map is only used when deserializing a `DefPathHash` - // from the incremental cache. - local_def_path_hash_to_def_id: UnhashMap, // Caches all lookups of `DefPathHashes`, both for local and foreign // definitions. A definition from the previous compilation session // may no longer exist in the current compilation session, so @@ -120,8 +111,7 @@ pub struct OnDiskCache<'sess> { // This type is used only for serialization and deserialization. #[derive(Encodable, Decodable)] struct Footer { - file_index_to_stable_id: FxHashMap, - prev_cnums: Vec<(u32, String, CrateDisambiguator)>, + file_index_to_stable_id: FxHashMap, query_result_index: EncodedQueryResultIndex, diagnostics_index: EncodedQueryResultIndex, // The location of all allocations. @@ -167,23 +157,35 @@ crate struct RawDefId { pub index: u32, } -fn make_local_def_path_hash_map(definitions: &Definitions) -> UnhashMap { - UnhashMap::from_iter( - definitions - .def_path_table() - .all_def_path_hashes_and_def_ids(LOCAL_CRATE) - .map(|(hash, def_id)| (hash, def_id.as_local().unwrap())), - ) +/// An `EncodedSourceFileId` is the same as a `StableSourceFileId` except that +/// the source crate is represented as a [StableCrateId] instead of as a +/// `CrateNum`. This way `EncodedSourceFileId` can be encoded and decoded +/// without any additional context, i.e. with a simple `opaque::Decoder` (which +/// is the only thing available when decoding the cache's [Footer]. +#[derive(Encodable, Decodable, Clone, Debug)] +struct EncodedSourceFileId { + file_name_hash: u64, + stable_crate_id: StableCrateId, +} + +impl EncodedSourceFileId { + fn translate(&self, cnum_map: &UnhashMap) -> StableSourceFileId { + let cnum = cnum_map[&self.stable_crate_id]; + StableSourceFileId { file_name_hash: self.file_name_hash, cnum } + } + + fn new(tcx: TyCtxt<'_>, file: &SourceFile) -> EncodedSourceFileId { + let source_file_id = StableSourceFileId::new(file); + EncodedSourceFileId { + file_name_hash: source_file_id.file_name_hash, + stable_crate_id: tcx.stable_crate_id(source_file_id.cnum), + } + } } impl<'sess> OnDiskCache<'sess> { /// Creates a new `OnDiskCache` instance from the serialized data in `data`. - pub fn new( - sess: &'sess Session, - data: Vec, - start_pos: usize, - definitions: &Definitions, - ) -> Self { + pub fn new(sess: &'sess Session, data: Vec, start_pos: usize) -> Self { debug_assert!(sess.opts.incremental.is_some()); // Wrap in a scope so we can borrow `data`. @@ -208,7 +210,6 @@ impl<'sess> OnDiskCache<'sess> { serialized_data: data, file_index_to_stable_id: footer.file_index_to_stable_id, file_index_to_file: Default::default(), - prev_cnums: footer.prev_cnums, cnum_map: OnceCell::new(), source_map: sess.source_map(), current_diagnostics: Default::default(), @@ -220,7 +221,6 @@ impl<'sess> OnDiskCache<'sess> { hygiene_context: Default::default(), foreign_def_path_hashes: footer.foreign_def_path_hashes, latest_foreign_def_path_hashes: Default::default(), - local_def_path_hash_to_def_id: make_local_def_path_hash_map(definitions), def_path_hash_to_def_id_cache: Default::default(), } } @@ -230,7 +230,6 @@ impl<'sess> OnDiskCache<'sess> { serialized_data: Vec::new(), file_index_to_stable_id: Default::default(), file_index_to_file: Default::default(), - prev_cnums: vec![], cnum_map: OnceCell::new(), source_map, current_diagnostics: Default::default(), @@ -242,7 +241,6 @@ impl<'sess> OnDiskCache<'sess> { hygiene_context: Default::default(), foreign_def_path_hashes: Default::default(), latest_foreign_def_path_hashes: Default::default(), - local_def_path_hash_to_def_id: Default::default(), def_path_hash_to_def_id_cache: Default::default(), } } @@ -266,7 +264,8 @@ impl<'sess> OnDiskCache<'sess> { let index = SourceFileIndex(index as u32); let file_ptr: *const SourceFile = &**file as *const _; file_to_file_index.insert(file_ptr, index); - file_index_to_stable_id.insert(index, StableSourceFileId::new(&file)); + let source_file_id = EncodedSourceFileId::new(tcx, &file); + file_index_to_stable_id.insert(index, source_file_id); } (file_to_file_index, file_index_to_stable_id) @@ -351,16 +350,6 @@ impl<'sess> OnDiskCache<'sess> { interpret_alloc_index }; - let sorted_cnums = sorted_cnums_including_local_crate(tcx); - let prev_cnums: Vec<_> = sorted_cnums - .iter() - .map(|&cnum| { - let crate_name = tcx.original_crate_name(cnum).to_string(); - let crate_disambiguator = tcx.crate_disambiguator(cnum); - (cnum.as_u32(), crate_name, crate_disambiguator) - }) - .collect(); - let mut syntax_contexts = FxHashMap::default(); let mut expn_ids = FxHashMap::default(); @@ -392,7 +381,6 @@ impl<'sess> OnDiskCache<'sess> { TAG_FILE_FOOTER, &Footer { file_index_to_stable_id, - prev_cnums, query_result_index, diagnostics_index, interpret_alloc_index, @@ -409,16 +397,7 @@ impl<'sess> OnDiskCache<'sess> { // DO NOT WRITE ANYTHING TO THE ENCODER AFTER THIS POINT! The address // of the footer must be the last thing in the data stream. - return Ok(()); - - fn sorted_cnums_including_local_crate(tcx: TyCtxt<'_>) -> Vec { - let mut cnums = vec![LOCAL_CRATE]; - cnums.extend_from_slice(tcx.crates()); - cnums.sort_unstable(); - // Just to be sure... - cnums.dedup(); - cnums - } + Ok(()) }) } @@ -453,12 +432,11 @@ impl<'sess> OnDiskCache<'sess> { self.foreign_def_path_hashes.get(hash).copied() } - fn try_remap_cnum(&self, tcx: TyCtxt<'_>, cnum: u32) -> Option { - let cnum_map = - self.cnum_map.get_or_init(|| Self::compute_cnum_map(tcx, &self.prev_cnums[..])); - debug!("try_remap_cnum({}): cnum_map={:?}", cnum, cnum_map); + fn try_remap_cnum(&self, tcx: TyCtxt<'_>, stable_crate_id: StableCrateId) -> Option { + let cnum_map = self.cnum_map.get_or_init(|| Self::compute_cnum_map(tcx)); + debug!("try_remap_cnum({:?}): cnum_map={:?}", stable_crate_id, cnum_map); - cnum_map[CrateNum::from_u32(cnum)] + cnum_map.get(&stable_crate_id).copied() } pub(crate) fn store_foreign_def_id_hash(&self, def_id: DefId, hash: DefPathHash) { @@ -557,8 +535,7 @@ impl<'sess> OnDiskCache<'sess> { where T: Decodable>, { - let cnum_map = - self.cnum_map.get_or_init(|| Self::compute_cnum_map(tcx, &self.prev_cnums[..])); + let cnum_map = self.cnum_map.get_or_init(|| Self::compute_cnum_map(tcx)); let mut decoder = CacheDecoder { tcx, @@ -579,31 +556,16 @@ impl<'sess> OnDiskCache<'sess> { // current-session-`CrateNum`. There might be `CrateNum`s from the previous // `Session` that don't occur in the current one. For these, the mapping // maps to None. - fn compute_cnum_map( - tcx: TyCtxt<'_>, - prev_cnums: &[(u32, String, CrateDisambiguator)], - ) -> IndexVec> { + fn compute_cnum_map(tcx: TyCtxt<'_>) -> UnhashMap { tcx.dep_graph.with_ignore(|| { - let current_cnums = tcx - .all_crate_nums(LOCAL_CRATE) + tcx.all_crate_nums(()) .iter() + .chain(std::iter::once(&LOCAL_CRATE)) .map(|&cnum| { - let crate_name = tcx.original_crate_name(cnum).to_string(); - let crate_disambiguator = tcx.crate_disambiguator(cnum); - ((crate_name, crate_disambiguator), cnum) + let hash = tcx.def_path_hash(cnum.as_def_id()).stable_crate_id(); + (hash, cnum) }) - .collect::>(); - - let map_size = prev_cnums.iter().map(|&(cnum, ..)| cnum).max().unwrap_or(0) + 1; - let mut map = IndexVec::from_elem_n(None, map_size as usize); - - for &(prev_cnum, ref crate_name, crate_disambiguator) in prev_cnums { - let key = (crate_name.clone(), crate_disambiguator); - map[CrateNum::from_u32(prev_cnum)] = current_cnums.get(&key).cloned(); - } - - map[LOCAL_CRATE] = Some(LOCAL_CRATE); - map + .collect() }) } @@ -622,7 +584,7 @@ impl<'sess> OnDiskCache<'sess> { debug!("def_path_hash_to_def_id({:?})", hash); // Check if the `DefPathHash` corresponds to a definition in the current // crate - if let Some(def_id) = self.local_def_path_hash_to_def_id.get(&hash).cloned() { + if let Some(def_id) = tcx.definitions.local_def_path_hash_to_def_id(hash) { let def_id = def_id.to_def_id(); e.insert(Some(def_id)); return Some(def_id); @@ -636,7 +598,7 @@ impl<'sess> OnDiskCache<'sess> { debug!("def_path_hash_to_def_id({:?}): raw_def_id = {:?}", hash, raw_def_id); // If the owning crate no longer exists, the corresponding definition definitely // no longer exists. - let krate = self.try_remap_cnum(tcx, raw_def_id.krate)?; + let krate = self.try_remap_cnum(tcx, hash.stable_crate_id())?; debug!("def_path_hash_to_def_id({:?}): krate = {:?}", hash, krate); // If our `DefPathHash` corresponded to a definition in the local crate, // we should have either found it in `local_def_path_hash_to_def_id`, or @@ -668,9 +630,9 @@ pub struct CacheDecoder<'a, 'tcx> { tcx: TyCtxt<'tcx>, opaque: opaque::Decoder<'a>, source_map: &'a SourceMap, - cnum_map: &'a IndexVec>, + cnum_map: &'a UnhashMap, file_index_to_file: &'a Lock>>, - file_index_to_stable_id: &'a FxHashMap, + file_index_to_stable_id: &'a FxHashMap, alloc_decoding_session: AllocDecodingSession<'a>, syntax_contexts: &'a FxHashMap, expn_data: &'a FxHashMap, @@ -683,6 +645,7 @@ impl<'a, 'tcx> CacheDecoder<'a, 'tcx> { ref file_index_to_file, ref file_index_to_stable_id, ref source_map, + ref cnum_map, .. } = *self; @@ -690,7 +653,7 @@ impl<'a, 'tcx> CacheDecoder<'a, 'tcx> { .borrow_mut() .entry(index) .or_insert_with(|| { - let stable_id = file_index_to_stable_id[&index]; + let stable_id = file_index_to_stable_id[&index].translate(cnum_map); source_map .source_file_by_stable_id(stable_id) .expect("failed to lookup `SourceFile` in new context") @@ -764,8 +727,7 @@ impl<'a, 'tcx> TyDecoder<'tcx> for CacheDecoder<'a, 'tcx> { { let tcx = self.tcx(); - let cache_key = - ty::CReaderCacheKey { cnum: CrateNum::ReservedForIncrCompCache, pos: shorthand }; + let cache_key = ty::CReaderCacheKey { cnum: None, pos: shorthand }; if let Some(&ty) = tcx.ty_rcache.borrow().get(&cache_key) { return Ok(ty); @@ -790,10 +752,6 @@ impl<'a, 'tcx> TyDecoder<'tcx> for CacheDecoder<'a, 'tcx> { r } - fn map_encoded_cnum_to_current(&self, cnum: CrateNum) -> CrateNum { - self.cnum_map[cnum].unwrap_or_else(|| bug!("could not find new `CrateNum` for {:?}", cnum)) - } - fn decode_alloc_id(&mut self) -> Result { let alloc_decoding_session = self.alloc_decoding_session; alloc_decoding_session.decode_alloc_id(self) @@ -875,8 +833,9 @@ impl<'a, 'tcx> Decodable> for Span { impl<'a, 'tcx> Decodable> for CrateNum { fn decode(d: &mut CacheDecoder<'a, 'tcx>) -> Result { - let cnum = CrateNum::from_u32(u32::decode(d)?); - Ok(d.map_encoded_cnum_to_current(cnum)) + let stable_id = StableCrateId::decode(d)?; + let cnum = d.cnum_map[&stable_id]; + Ok(cnum) } } @@ -1086,6 +1045,15 @@ where } } +impl<'a, 'tcx, E> Encodable> for CrateNum +where + E: 'a + OpaqueEncoder, +{ + fn encode(&self, s: &mut CacheEncoder<'a, 'tcx, E>) -> Result<(), E::Error> { + s.tcx.stable_crate_id(*self).encode(s) + } +} + impl<'a, 'tcx, E> Encodable> for DefId where E: 'a + OpaqueEncoder, diff --git a/compiler/rustc_middle/src/ty/relate.rs b/compiler/rustc_middle/src/ty/relate.rs index b6f93c9bd5..3f426b1368 100644 --- a/compiler/rustc_middle/src/ty/relate.rs +++ b/compiler/rustc_middle/src/ty/relate.rs @@ -67,6 +67,7 @@ pub trait TypeRelation<'tcx>: Sized { fn relate_with_variance>( &mut self, variance: ty::Variance, + info: ty::VarianceDiagInfo<'tcx>, a: T, b: T, ) -> RelateResult<'tcx, T>; @@ -111,24 +112,23 @@ pub trait Relate<'tcx>: TypeFoldable<'tcx> + Copy { /////////////////////////////////////////////////////////////////////////// // Relate impls -impl<'tcx> Relate<'tcx> for ty::TypeAndMut<'tcx> { - fn relate>( - relation: &mut R, - a: ty::TypeAndMut<'tcx>, - b: ty::TypeAndMut<'tcx>, - ) -> RelateResult<'tcx, ty::TypeAndMut<'tcx>> { - debug!("{}.mts({:?}, {:?})", relation.tag(), a, b); - if a.mutbl != b.mutbl { - Err(TypeError::Mutability) - } else { - let mutbl = a.mutbl; - let variance = match mutbl { - ast::Mutability::Not => ty::Covariant, - ast::Mutability::Mut => ty::Invariant, - }; - let ty = relation.relate_with_variance(variance, a.ty, b.ty)?; - Ok(ty::TypeAndMut { ty, mutbl }) - } +pub fn relate_type_and_mut<'tcx, R: TypeRelation<'tcx>>( + relation: &mut R, + a: ty::TypeAndMut<'tcx>, + b: ty::TypeAndMut<'tcx>, + kind: ty::VarianceDiagMutKind, +) -> RelateResult<'tcx, ty::TypeAndMut<'tcx>> { + debug!("{}.mts({:?}, {:?})", relation.tag(), a, b); + if a.mutbl != b.mutbl { + Err(TypeError::Mutability) + } else { + let mutbl = a.mutbl; + let (variance, info) = match mutbl { + ast::Mutability::Not => (ty::Covariant, ty::VarianceDiagInfo::None), + ast::Mutability::Mut => (ty::Invariant, ty::VarianceDiagInfo::Mut { kind, ty: a.ty }), + }; + let ty = relation.relate_with_variance(variance, info, a.ty, b.ty)?; + Ok(ty::TypeAndMut { ty, mutbl }) } } @@ -142,7 +142,7 @@ pub fn relate_substs>( let params = iter::zip(a_subst, b_subst).enumerate().map(|(i, (a, b))| { let variance = variances.map_or(ty::Invariant, |v| v[i]); - relation.relate_with_variance(variance, a, b) + relation.relate_with_variance(variance, ty::VarianceDiagInfo::default(), a, b) }); tcx.mk_substs(params) @@ -177,7 +177,12 @@ impl<'tcx> Relate<'tcx> for ty::FnSig<'tcx> { if is_output { relation.relate(a, b) } else { - relation.relate_with_variance(ty::Contravariant, a, b) + relation.relate_with_variance( + ty::Contravariant, + ty::VarianceDiagInfo::default(), + a, + b, + ) } }) .enumerate() @@ -251,8 +256,18 @@ impl<'tcx> Relate<'tcx> for ty::ExistentialProjection<'tcx> { b.item_def_id, ))) } else { - let ty = relation.relate_with_variance(ty::Invariant, a.ty, b.ty)?; - let substs = relation.relate_with_variance(ty::Invariant, a.substs, b.substs)?; + let ty = relation.relate_with_variance( + ty::Invariant, + ty::VarianceDiagInfo::default(), + a.ty, + b.ty, + )?; + let substs = relation.relate_with_variance( + ty::Invariant, + ty::VarianceDiagInfo::default(), + a.substs, + b.substs, + )?; Ok(ty::ExistentialProjection { item_def_id: a.item_def_id, substs, ty }) } } @@ -364,7 +379,12 @@ pub fn super_relate_tys>( (&ty::Dynamic(a_obj, a_region), &ty::Dynamic(b_obj, b_region)) => { let region_bound = relation.with_cause(Cause::ExistentialRegionBound, |relation| { - relation.relate_with_variance(ty::Contravariant, a_region, b_region) + relation.relate_with_variance( + ty::Contravariant, + ty::VarianceDiagInfo::default(), + a_region, + b_region, + ) })?; Ok(tcx.mk_dynamic(relation.relate(a_obj, b_obj)?, region_bound)) } @@ -398,15 +418,20 @@ pub fn super_relate_tys>( } (&ty::RawPtr(a_mt), &ty::RawPtr(b_mt)) => { - let mt = relation.relate(a_mt, b_mt)?; + let mt = relate_type_and_mut(relation, a_mt, b_mt, ty::VarianceDiagMutKind::RawPtr)?; Ok(tcx.mk_ptr(mt)) } (&ty::Ref(a_r, a_ty, a_mutbl), &ty::Ref(b_r, b_ty, b_mutbl)) => { - let r = relation.relate_with_variance(ty::Contravariant, a_r, b_r)?; + let r = relation.relate_with_variance( + ty::Contravariant, + ty::VarianceDiagInfo::default(), + a_r, + b_r, + )?; let a_mt = ty::TypeAndMut { ty: a_ty, mutbl: a_mutbl }; let b_mt = ty::TypeAndMut { ty: b_ty, mutbl: b_mutbl }; - let mt = relation.relate(a_mt, b_mt)?; + let mt = relate_type_and_mut(relation, a_mt, b_mt, ty::VarianceDiagMutKind::Ref)?; Ok(tcx.mk_ref(r, mt)) } @@ -536,8 +561,12 @@ pub fn super_relate_consts>( (ty::ConstKind::Unevaluated(au), ty::ConstKind::Unevaluated(bu)) if au.def == bu.def && au.promoted == bu.promoted => { - let substs = - relation.relate_with_variance(ty::Variance::Invariant, au.substs, bu.substs)?; + let substs = relation.relate_with_variance( + ty::Variance::Invariant, + ty::VarianceDiagInfo::default(), + au.substs, + bu.substs, + )?; return Ok(tcx.mk_const(ty::Const { val: ty::ConstKind::Unevaluated(ty::Unevaluated { def: au.def, diff --git a/compiler/rustc_middle/src/ty/sty.rs b/compiler/rustc_middle/src/ty/sty.rs index 691bfcc98d..1d9ff51228 100644 --- a/compiler/rustc_middle/src/ty/sty.rs +++ b/compiler/rustc_middle/src/ty/sty.rs @@ -669,7 +669,7 @@ impl<'tcx> GeneratorSubsts<'tcx> { } } -#[derive(Debug, Copy, Clone)] +#[derive(Debug, Copy, Clone, HashStable)] pub enum UpvarSubsts<'tcx> { Closure(SubstsRef<'tcx>), Generator(SubstsRef<'tcx>), @@ -1837,10 +1837,12 @@ impl<'tcx> TyS<'tcx> { #[inline] pub fn is_enum(&self) -> bool { - match self.kind() { - Adt(adt_def, _) => adt_def.is_enum(), - _ => false, - } + matches!(self.kind(), Adt(adt_def, _) if adt_def.is_enum()) + } + + #[inline] + pub fn is_union(&self) -> bool { + matches!(self.kind(), Adt(adt_def, _) if adt_def.is_union()) } #[inline] @@ -1888,11 +1890,6 @@ impl<'tcx> TyS<'tcx> { matches!(self.kind(), Int(ty::IntTy::Isize) | Uint(ty::UintTy::Usize)) } - #[inline] - pub fn is_machine(&self) -> bool { - matches!(self.kind(), Int(..) | Uint(..) | Float(..)) - } - #[inline] pub fn has_concrete_skeleton(&self) -> bool { !matches!(self.kind(), Param(_) | Infer(_) | Error(_)) @@ -2184,3 +2181,55 @@ impl<'tcx> TyS<'tcx> { } } } + +/// Extra information about why we ended up with a particular variance. +/// This is only used to add more information to error messages, and +/// has no effect on soundness. While choosing the 'wrong' `VarianceDiagInfo` +/// may lead to confusing notes in error messages, it will never cause +/// a miscompilation or unsoundness. +/// +/// When in doubt, use `VarianceDiagInfo::default()` +#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)] +pub enum VarianceDiagInfo<'tcx> { + /// No additional information - this is the default. + /// We will not add any additional information to error messages. + None, + /// We switched our variance because a type occurs inside + /// the generic argument of a mutable reference or pointer + /// (`*mut T` or `&mut T`). In either case, our variance + /// will always be `Invariant`. + Mut { + /// Tracks whether we had a mutable pointer or reference, + /// for better error messages + kind: VarianceDiagMutKind, + /// The type parameter of the mutable pointer/reference + /// (the `T` in `&mut T` or `*mut T`). + ty: Ty<'tcx>, + }, +} + +#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)] +pub enum VarianceDiagMutKind { + /// A mutable raw pointer (`*mut T`) + RawPtr, + /// A mutable reference (`&mut T`) + Ref, +} + +impl<'tcx> VarianceDiagInfo<'tcx> { + /// Mirrors `Variance::xform` - used to 'combine' the existing + /// and new `VarianceDiagInfo`s when our variance changes. + pub fn xform(self, other: VarianceDiagInfo<'tcx>) -> VarianceDiagInfo<'tcx> { + // For now, just use the first `VarianceDiagInfo::Mut` that we see + match self { + VarianceDiagInfo::None => other, + VarianceDiagInfo::Mut { .. } => self, + } + } +} + +impl<'tcx> Default for VarianceDiagInfo<'tcx> { + fn default() -> Self { + Self::None + } +} diff --git a/compiler/rustc_middle/src/ty/subst.rs b/compiler/rustc_middle/src/ty/subst.rs index c84ca61122..9b8d22d8ea 100644 --- a/compiler/rustc_middle/src/ty/subst.rs +++ b/compiler/rustc_middle/src/ty/subst.rs @@ -197,7 +197,7 @@ impl<'a, 'tcx> InternalSubsts<'tcx> { } /// Interpret these substitutions as the substitutions of a generator type. - /// Closure substitutions have a particular structure controlled by the + /// Generator substitutions have a particular structure controlled by the /// compiler that encodes information like the signature and generator kind; /// see `ty::GeneratorSubsts` struct for more comments. pub fn as_generator(&'tcx self) -> GeneratorSubsts<'tcx> { diff --git a/compiler/rustc_middle/src/ty/trait_def.rs b/compiler/rustc_middle/src/ty/trait_def.rs index e9b8883f29..79166472ac 100644 --- a/compiler/rustc_middle/src/ty/trait_def.rs +++ b/compiler/rustc_middle/src/ty/trait_def.rs @@ -4,14 +4,13 @@ use crate::ty::fast_reject; use crate::ty::fold::TypeFoldable; use crate::ty::{Ty, TyCtxt}; use rustc_hir as hir; -use rustc_hir::def_id::{CrateNum, DefId, LocalDefId}; +use rustc_hir::def_id::DefId; use rustc_hir::definitions::DefPathHash; use rustc_data_structures::fx::FxHashMap; use rustc_data_structures::stable_hasher::{HashStable, StableHasher}; use rustc_errors::ErrorReported; use rustc_macros::HashStable; -use std::collections::BTreeMap; /// A trait's definition with type information. #[derive(HashStable)] @@ -209,14 +208,6 @@ impl<'tcx> TyCtxt<'tcx> { } } -// Query provider for `all_local_trait_impls`. -pub(super) fn all_local_trait_impls<'tcx>( - tcx: TyCtxt<'tcx>, - krate: CrateNum, -) -> &'tcx BTreeMap> { - &tcx.hir_crate(krate).trait_impls -} - // Query provider for `trait_impls_of`. pub(super) fn trait_impls_of_provider(tcx: TyCtxt<'_>, trait_id: DefId) -> TraitImpls { let mut impls = TraitImpls::default(); diff --git a/compiler/rustc_middle/src/ty/util.rs b/compiler/rustc_middle/src/ty/util.rs index e365928c15..7bf69b9e63 100644 --- a/compiler/rustc_middle/src/ty/util.rs +++ b/compiler/rustc_middle/src/ty/util.rs @@ -791,6 +791,39 @@ impl<'tcx> ty::TyS<'tcx> { } } + /// Checks if `ty` has has a significant drop. + /// + /// Note that this method can return false even if `ty` has a destructor + /// attached; even if that is the case then the adt has been marked with + /// the attribute `rustc_insignificant_dtor`. + /// + /// Note that this method is used to check for change in drop order for + /// 2229 drop reorder migration analysis. + #[inline] + pub fn has_significant_drop( + &'tcx self, + tcx: TyCtxt<'tcx>, + param_env: ty::ParamEnv<'tcx>, + ) -> bool { + // Avoid querying in simple cases. + match needs_drop_components(self, &tcx.data_layout) { + Err(AlwaysRequiresDrop) => true, + Ok(components) => { + let query_ty = match *components { + [] => return false, + // If we've got a single component, call the query with that + // to increase the chance that we hit the query cache. + [component_ty] => component_ty, + _ => self, + }; + // This doesn't depend on regions, so try to minimize distinct + // query keys used. + let erased = tcx.normalize_erasing_regions(param_env, query_ty); + tcx.has_significant_drop_raw(param_env.and(erased)) + } + } + } + /// Returns `true` if equality for this type is both reflexive and structural. /// /// Reflexive equality for a type is indicated by an `Eq` impl for that type. diff --git a/compiler/rustc_mir/src/borrow_check/constraints/graph.rs b/compiler/rustc_mir/src/borrow_check/constraints/graph.rs index f3f6b8c10d..9e4cfb2cc0 100644 --- a/compiler/rustc_mir/src/borrow_check/constraints/graph.rs +++ b/compiler/rustc_mir/src/borrow_check/constraints/graph.rs @@ -1,7 +1,7 @@ use rustc_data_structures::graph; use rustc_index::vec::IndexVec; use rustc_middle::mir::ConstraintCategory; -use rustc_middle::ty::RegionVid; +use rustc_middle::ty::{RegionVid, VarianceDiagInfo}; use rustc_span::DUMMY_SP; use crate::borrow_check::{ @@ -26,8 +26,8 @@ crate type ReverseConstraintGraph = ConstraintGraph; /// Marker trait that controls whether a `R1: R2` constraint /// represents an edge `R1 -> R2` or `R2 -> R1`. crate trait ConstraintGraphDirecton: Copy + 'static { - fn start_region(c: &OutlivesConstraint) -> RegionVid; - fn end_region(c: &OutlivesConstraint) -> RegionVid; + fn start_region(c: &OutlivesConstraint<'_>) -> RegionVid; + fn end_region(c: &OutlivesConstraint<'_>) -> RegionVid; fn is_normal() -> bool; } @@ -39,11 +39,11 @@ crate trait ConstraintGraphDirecton: Copy + 'static { crate struct Normal; impl ConstraintGraphDirecton for Normal { - fn start_region(c: &OutlivesConstraint) -> RegionVid { + fn start_region(c: &OutlivesConstraint<'_>) -> RegionVid { c.sup } - fn end_region(c: &OutlivesConstraint) -> RegionVid { + fn end_region(c: &OutlivesConstraint<'_>) -> RegionVid { c.sub } @@ -60,11 +60,11 @@ impl ConstraintGraphDirecton for Normal { crate struct Reverse; impl ConstraintGraphDirecton for Reverse { - fn start_region(c: &OutlivesConstraint) -> RegionVid { + fn start_region(c: &OutlivesConstraint<'_>) -> RegionVid { c.sub } - fn end_region(c: &OutlivesConstraint) -> RegionVid { + fn end_region(c: &OutlivesConstraint<'_>) -> RegionVid { c.sup } @@ -78,7 +78,7 @@ impl ConstraintGraph { /// R2` is treated as an edge `R1 -> R2`. We use this graph to /// construct SCCs for region inference but also for error /// reporting. - crate fn new(direction: D, set: &OutlivesConstraintSet, num_region_vars: usize) -> Self { + crate fn new(direction: D, set: &OutlivesConstraintSet<'_>, num_region_vars: usize) -> Self { let mut first_constraints = IndexVec::from_elem_n(None, num_region_vars); let mut next_constraints = IndexVec::from_elem(None, &set.outlives); @@ -96,21 +96,21 @@ impl ConstraintGraph { /// Given the constraint set from which this graph was built /// creates a region graph so that you can iterate over *regions* /// and not constraints. - crate fn region_graph<'rg>( + crate fn region_graph<'rg, 'tcx>( &'rg self, - set: &'rg OutlivesConstraintSet, + set: &'rg OutlivesConstraintSet<'tcx>, static_region: RegionVid, - ) -> RegionGraph<'rg, D> { + ) -> RegionGraph<'rg, 'tcx, D> { RegionGraph::new(set, self, static_region) } /// Given a region `R`, iterate over all constraints `R: R1`. - crate fn outgoing_edges<'a>( + crate fn outgoing_edges<'a, 'tcx>( &'a self, region_sup: RegionVid, - constraints: &'a OutlivesConstraintSet, + constraints: &'a OutlivesConstraintSet<'tcx>, static_region: RegionVid, - ) -> Edges<'a, D> { + ) -> Edges<'a, 'tcx, D> { //if this is the `'static` region and the graph's direction is normal, //then setup the Edges iterator to return all regions #53178 if region_sup == static_region && D::is_normal() { @@ -129,22 +129,22 @@ impl ConstraintGraph { } } -crate struct Edges<'s, D: ConstraintGraphDirecton> { +crate struct Edges<'s, 'tcx, D: ConstraintGraphDirecton> { graph: &'s ConstraintGraph, - constraints: &'s OutlivesConstraintSet, + constraints: &'s OutlivesConstraintSet<'tcx>, pointer: Option, next_static_idx: Option, static_region: RegionVid, } -impl<'s, D: ConstraintGraphDirecton> Iterator for Edges<'s, D> { - type Item = OutlivesConstraint; +impl<'s, 'tcx, D: ConstraintGraphDirecton> Iterator for Edges<'s, 'tcx, D> { + type Item = OutlivesConstraint<'tcx>; fn next(&mut self) -> Option { if let Some(p) = self.pointer { self.pointer = self.graph.next_constraints[p]; - Some(self.constraints[p]) + Some(self.constraints[p].clone()) } else if let Some(next_static_idx) = self.next_static_idx { self.next_static_idx = if next_static_idx == (self.graph.first_constraints.len() - 1) { None @@ -157,6 +157,7 @@ impl<'s, D: ConstraintGraphDirecton> Iterator for Edges<'s, D> { sub: next_static_idx.into(), locations: Locations::All(DUMMY_SP), category: ConstraintCategory::Internal, + variance_info: VarianceDiagInfo::default(), }) } else { None @@ -167,19 +168,19 @@ impl<'s, D: ConstraintGraphDirecton> Iterator for Edges<'s, D> { /// This struct brings together a constraint set and a (normal, not /// reverse) constraint graph. It implements the graph traits and is /// usd for doing the SCC computation. -crate struct RegionGraph<'s, D: ConstraintGraphDirecton> { - set: &'s OutlivesConstraintSet, +crate struct RegionGraph<'s, 'tcx, D: ConstraintGraphDirecton> { + set: &'s OutlivesConstraintSet<'tcx>, constraint_graph: &'s ConstraintGraph, static_region: RegionVid, } -impl<'s, D: ConstraintGraphDirecton> RegionGraph<'s, D> { +impl<'s, 'tcx, D: ConstraintGraphDirecton> RegionGraph<'s, 'tcx, D> { /// Creates a "dependency graph" where each region constraint `R1: /// R2` is treated as an edge `R1 -> R2`. We use this graph to /// construct SCCs for region inference but also for error /// reporting. crate fn new( - set: &'s OutlivesConstraintSet, + set: &'s OutlivesConstraintSet<'tcx>, constraint_graph: &'s ConstraintGraph, static_region: RegionVid, ) -> Self { @@ -188,18 +189,18 @@ impl<'s, D: ConstraintGraphDirecton> RegionGraph<'s, D> { /// Given a region `R`, iterate over all regions `R1` such that /// there exists a constraint `R: R1`. - crate fn outgoing_regions(&self, region_sup: RegionVid) -> Successors<'_, D> { + crate fn outgoing_regions(&self, region_sup: RegionVid) -> Successors<'_, 'tcx, D> { Successors { edges: self.constraint_graph.outgoing_edges(region_sup, self.set, self.static_region), } } } -crate struct Successors<'s, D: ConstraintGraphDirecton> { - edges: Edges<'s, D>, +crate struct Successors<'s, 'tcx, D: ConstraintGraphDirecton> { + edges: Edges<'s, 'tcx, D>, } -impl<'s, D: ConstraintGraphDirecton> Iterator for Successors<'s, D> { +impl<'s, 'tcx, D: ConstraintGraphDirecton> Iterator for Successors<'s, 'tcx, D> { type Item = RegionVid; fn next(&mut self) -> Option { @@ -207,23 +208,26 @@ impl<'s, D: ConstraintGraphDirecton> Iterator for Successors<'s, D> { } } -impl<'s, D: ConstraintGraphDirecton> graph::DirectedGraph for RegionGraph<'s, D> { +impl<'s, 'tcx, D: ConstraintGraphDirecton> graph::DirectedGraph for RegionGraph<'s, 'tcx, D> { type Node = RegionVid; } -impl<'s, D: ConstraintGraphDirecton> graph::WithNumNodes for RegionGraph<'s, D> { +impl<'s, 'tcx, D: ConstraintGraphDirecton> graph::WithNumNodes for RegionGraph<'s, 'tcx, D> { fn num_nodes(&self) -> usize { self.constraint_graph.first_constraints.len() } } -impl<'s, D: ConstraintGraphDirecton> graph::WithSuccessors for RegionGraph<'s, D> { +impl<'s, 'tcx, D: ConstraintGraphDirecton> graph::WithSuccessors for RegionGraph<'s, 'tcx, D> { fn successors(&self, node: Self::Node) -> >::Iter { self.outgoing_regions(node) } } -impl<'s, 'graph, D: ConstraintGraphDirecton> graph::GraphSuccessors<'graph> for RegionGraph<'s, D> { +impl<'s, 'graph, 'tcx, D: ConstraintGraphDirecton> graph::GraphSuccessors<'graph> + for RegionGraph<'s, 'tcx, D> +{ type Item = RegionVid; - type Iter = Successors<'graph, D>; + // FIXME - why can't this be `'graph, 'tcx` + type Iter = Successors<'graph, 'graph, D>; } diff --git a/compiler/rustc_mir/src/borrow_check/constraints/mod.rs b/compiler/rustc_mir/src/borrow_check/constraints/mod.rs index 3772b7d8f9..b944479ca4 100644 --- a/compiler/rustc_mir/src/borrow_check/constraints/mod.rs +++ b/compiler/rustc_mir/src/borrow_check/constraints/mod.rs @@ -1,7 +1,7 @@ use rustc_data_structures::graph::scc::Sccs; use rustc_index::vec::IndexVec; use rustc_middle::mir::ConstraintCategory; -use rustc_middle::ty::RegionVid; +use rustc_middle::ty::{RegionVid, VarianceDiagInfo}; use std::fmt; use std::ops::Index; @@ -14,12 +14,12 @@ crate mod graph; /// a unique `OutlivesConstraintIndex` and you can index into the set /// (`constraint_set[i]`) to access the constraint details. #[derive(Clone, Default)] -crate struct OutlivesConstraintSet { - outlives: IndexVec, +crate struct OutlivesConstraintSet<'tcx> { + outlives: IndexVec>, } -impl OutlivesConstraintSet { - crate fn push(&mut self, constraint: OutlivesConstraint) { +impl<'tcx> OutlivesConstraintSet<'tcx> { + crate fn push(&mut self, constraint: OutlivesConstraint<'tcx>) { debug!( "OutlivesConstraintSet::push({:?}: {:?} @ {:?}", constraint.sup, constraint.sub, constraint.locations @@ -59,21 +59,21 @@ impl OutlivesConstraintSet { Sccs::new(region_graph) } - crate fn outlives(&self) -> &IndexVec { + crate fn outlives(&self) -> &IndexVec> { &self.outlives } } -impl Index for OutlivesConstraintSet { - type Output = OutlivesConstraint; +impl<'tcx> Index for OutlivesConstraintSet<'tcx> { + type Output = OutlivesConstraint<'tcx>; fn index(&self, i: OutlivesConstraintIndex) -> &Self::Output { &self.outlives[i] } } -#[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord)] -pub struct OutlivesConstraint { +#[derive(Clone, PartialEq, Eq, PartialOrd, Ord)] +pub struct OutlivesConstraint<'tcx> { // NB. The ordering here is not significant for correctness, but // it is for convenience. Before we dump the constraints in the // debugging logs, we sort them, and we'd like the "super region" @@ -89,11 +89,18 @@ pub struct OutlivesConstraint { /// What caused this constraint? pub category: ConstraintCategory, + + /// Variance diagnostic information + pub variance_info: VarianceDiagInfo<'tcx>, } -impl fmt::Debug for OutlivesConstraint { +impl<'tcx> fmt::Debug for OutlivesConstraint<'tcx> { fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(formatter, "({:?}: {:?}) due to {:?}", self.sup, self.sub, self.locations) + write!( + formatter, + "({:?}: {:?}) due to {:?} ({:?})", + self.sup, self.sub, self.locations, self.variance_info + ) } } diff --git a/compiler/rustc_mir/src/borrow_check/diagnostics/conflict_errors.rs b/compiler/rustc_mir/src/borrow_check/diagnostics/conflict_errors.rs index 9f19a474ca..8b0761889b 100644 --- a/compiler/rustc_mir/src/borrow_check/diagnostics/conflict_errors.rs +++ b/compiler/rustc_mir/src/borrow_check/diagnostics/conflict_errors.rs @@ -4,10 +4,9 @@ use rustc_errors::{Applicability, DiagnosticBuilder}; use rustc_hir as hir; use rustc_hir::def_id::DefId; use rustc_hir::{AsyncGeneratorKind, GeneratorKind}; -use rustc_index::vec::Idx; use rustc_middle::mir::{ self, AggregateKind, BindingForm, BorrowKind, ClearCrossCrate, ConstraintCategory, - FakeReadCause, Local, LocalDecl, LocalInfo, LocalKind, Location, Operand, Place, PlaceRef, + FakeReadCause, LocalDecl, LocalInfo, LocalKind, Location, Operand, Place, PlaceRef, ProjectionElem, Rvalue, Statement, StatementKind, Terminator, TerminatorKind, VarBindingForm, }; use rustc_middle::ty::{self, suggest_constraining_type_param, Ty, TypeFoldable}; @@ -99,7 +98,7 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> { ); err.span_label(span, format!("use of possibly-uninitialized {}", item_msg)); - use_spans.var_span_label( + use_spans.var_span_label_path_only( &mut err, format!("{} occurs due to use{}", desired_action.as_noun(), use_spans.describe()), ); @@ -255,6 +254,7 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> { partially_str, move_spans.describe() ), + "moved", ); } } @@ -304,7 +304,7 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> { } } - use_spans.var_span_label( + use_spans.var_span_label_path_only( &mut err, format!("{} occurs due to use{}", desired_action.as_noun(), use_spans.describe()), ); @@ -434,13 +434,16 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> { err.span_label(borrow_span, format!("borrow of {} occurs here", borrow_msg)); err.span_label(span, format!("move out of {} occurs here", value_msg)); - borrow_spans.var_span_label( + borrow_spans.var_span_label_path_only( &mut err, format!("borrow occurs due to use{}", borrow_spans.describe()), ); - move_spans - .var_span_label(&mut err, format!("move occurs due to use{}", move_spans.describe())); + move_spans.var_span_label( + &mut err, + format!("move occurs due to use{}", move_spans.describe()), + "moved", + ); self.explain_why_borrow_contains_point(location, borrow, None) .add_explanation_to_diagnostic( @@ -468,6 +471,8 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> { let use_spans = self.move_spans(place.as_ref(), location); let span = use_spans.var_or_use(); + // If the attempted use is in a closure then we do not care about the path span of the place we are currently trying to use + // we call `var_span_label` on `borrow_spans` to annotate if the existing borrow was in a closure let mut err = self.cannot_use_when_mutably_borrowed( span, &self.describe_any_place(place.as_ref()), @@ -475,11 +480,15 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> { &self.describe_any_place(borrow.borrowed_place.as_ref()), ); - borrow_spans.var_span_label(&mut err, { - let place = &borrow.borrowed_place; - let desc_place = self.describe_any_place(place.as_ref()); - format!("borrow occurs due to use of {}{}", desc_place, borrow_spans.describe()) - }); + borrow_spans.var_span_label( + &mut err, + { + let place = &borrow.borrowed_place; + let desc_place = self.describe_any_place(place.as_ref()); + format!("borrow occurs due to use of {}{}", desc_place, borrow_spans.describe()) + }, + "mutable", + ); self.explain_why_borrow_contains_point(location, borrow, None) .add_explanation_to_diagnostic( @@ -591,6 +600,7 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> { desc_place, borrow_spans.describe(), ), + "immutable", ); return err; @@ -667,7 +677,8 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> { if issued_spans == borrow_spans { borrow_spans.var_span_label( &mut err, - format!("borrows occur due to use of {}{}", desc_place, borrow_spans.describe()), + format!("borrows occur due to use of {}{}", desc_place, borrow_spans.describe(),), + gen_borrow_kind.describe_mutability(), ); } else { let borrow_place = &issued_borrow.borrowed_place; @@ -679,6 +690,7 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> { borrow_place_desc, issued_spans.describe(), ), + issued_borrow.kind.describe_mutability(), ); borrow_spans.var_span_label( @@ -688,6 +700,7 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> { desc_place, borrow_spans.describe(), ), + gen_borrow_kind.describe_mutability(), ); } @@ -847,7 +860,7 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> { self.prefixes(borrow.borrowed_place.as_ref(), PrefixSet::All).last().unwrap(); let borrow_spans = self.retrieve_borrow_spans(borrow); - let borrow_span = borrow_spans.var_or_use(); + let borrow_span = borrow_spans.var_or_use_path_span(); assert!(root_place.projection.is_empty()); let proper_span = self.body.local_decls[root_place.local].source_info.span; @@ -987,7 +1000,7 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> { location, name, borrow, drop_span, borrow_spans ); - let borrow_span = borrow_spans.var_or_use(); + let borrow_span = borrow_spans.var_or_use_path_span(); if let BorrowExplanation::MustBeValidFor { category, span, @@ -1260,7 +1273,9 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> { bug!("temporary or return pointer with a name") } LocalKind::Var => "local variable ", - LocalKind::Arg if !self.upvars.is_empty() && local == Local::new(1) => { + LocalKind::Arg + if !self.upvars.is_empty() && local == ty::CAPTURE_STRUCT_LOCAL => + { "variable captured by `move` " } LocalKind::Arg => "function parameter ", @@ -1575,6 +1590,7 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> { loan_spans.var_span_label( &mut err, format!("borrow occurs due to use{}", loan_spans.describe()), + loan.kind.describe_mutability(), ); err.buffer(&mut self.errors_buffer); @@ -1585,8 +1601,11 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> { let mut err = self.cannot_assign_to_borrowed(span, loan_span, &descr_place); - loan_spans - .var_span_label(&mut err, format!("borrow occurs due to use{}", loan_spans.describe())); + loan_spans.var_span_label( + &mut err, + format!("borrow occurs due to use{}", loan_spans.describe()), + loan.kind.describe_mutability(), + ); self.explain_why_borrow_contains_point(location, loan, None).add_explanation_to_diagnostic( self.infcx.tcx, diff --git a/compiler/rustc_mir/src/borrow_check/diagnostics/explain_borrow.rs b/compiler/rustc_mir/src/borrow_check/diagnostics/explain_borrow.rs index 2a388b8a72..e9f1ecb9bb 100644 --- a/compiler/rustc_mir/src/borrow_check/diagnostics/explain_borrow.rs +++ b/compiler/rustc_mir/src/borrow_check/diagnostics/explain_borrow.rs @@ -15,6 +15,7 @@ use rustc_middle::ty::{self, RegionVid, TyCtxt}; use rustc_span::symbol::Symbol; use rustc_span::Span; +use crate::borrow_check::region_infer::BlameConstraint; use crate::borrow_check::{ borrow_set::BorrowData, nll::ConstraintDescription, region_infer::Cause, MirBorrowckCtxt, WriteKind, @@ -24,8 +25,8 @@ use super::{find_use, RegionName, UseSpans}; #[derive(Debug)] pub(in crate::borrow_check) enum BorrowExplanation { - UsedLater(LaterUseKind, Span), - UsedLaterInLoop(LaterUseKind, Span), + UsedLater(LaterUseKind, Span, Option), + UsedLaterInLoop(LaterUseKind, Span, Option), UsedLaterWhenDropped { drop_loc: Location, dropped_local: Local, @@ -67,7 +68,7 @@ impl BorrowExplanation { borrow_span: Option, ) { match *self { - BorrowExplanation::UsedLater(later_use_kind, var_or_use_span) => { + BorrowExplanation::UsedLater(later_use_kind, var_or_use_span, path_span) => { let message = match later_use_kind { LaterUseKind::TraitCapture => "captured here by trait object", LaterUseKind::ClosureCapture => "captured here by closure", @@ -75,14 +76,31 @@ impl BorrowExplanation { LaterUseKind::FakeLetRead => "stored here", LaterUseKind::Other => "used here", }; - if !borrow_span.map_or(false, |sp| sp.overlaps(var_or_use_span)) { - err.span_label( - var_or_use_span, - format!("{}borrow later {}", borrow_desc, message), - ); + // We can use `var_or_use_span` if either `path_span` is not present, or both spans are the same + if path_span.map(|path_span| path_span == var_or_use_span).unwrap_or(true) { + if borrow_span.map(|sp| !sp.overlaps(var_or_use_span)).unwrap_or(true) { + err.span_label( + var_or_use_span, + format!("{}borrow later {}", borrow_desc, message), + ); + } + } else { + // path_span must be `Some` as otherwise the if condition is true + let path_span = path_span.unwrap(); + // path_span is only present in the case of closure capture + assert!(matches!(later_use_kind, LaterUseKind::ClosureCapture)); + if !borrow_span.map_or(false, |sp| sp.overlaps(var_or_use_span)) { + let path_label = "used here by closure"; + let capture_kind_label = message; + err.span_label( + var_or_use_span, + format!("{}borrow later {}", borrow_desc, capture_kind_label), + ); + err.span_label(path_span, path_label); + } } } - BorrowExplanation::UsedLaterInLoop(later_use_kind, var_or_use_span) => { + BorrowExplanation::UsedLaterInLoop(later_use_kind, var_or_use_span, path_span) => { let message = match later_use_kind { LaterUseKind::TraitCapture => { "borrow captured here by trait object, in later iteration of loop" @@ -94,7 +112,24 @@ impl BorrowExplanation { LaterUseKind::FakeLetRead => "borrow later stored here", LaterUseKind::Other => "borrow used here, in later iteration of loop", }; - err.span_label(var_or_use_span, format!("{}{}", borrow_desc, message)); + // We can use `var_or_use_span` if either `path_span` is not present, or both spans are the same + if path_span.map(|path_span| path_span == var_or_use_span).unwrap_or(true) { + err.span_label(var_or_use_span, format!("{}{}", borrow_desc, message)); + } else { + // path_span must be `Some` as otherwise the if condition is true + let path_span = path_span.unwrap(); + // path_span is only present in the case of closure capture + assert!(matches!(later_use_kind, LaterUseKind::ClosureCapture)); + if borrow_span.map(|sp| !sp.overlaps(var_or_use_span)).unwrap_or(true) { + let path_label = "used here by closure"; + let capture_kind_label = message; + err.span_label( + var_or_use_span, + format!("{}borrow later {}", borrow_desc, capture_kind_label), + ); + err.span_label(path_span, path_label); + } + } } BorrowExplanation::UsedLaterWhenDropped { drop_loc, @@ -255,12 +290,13 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> { borrow_region: RegionVid, outlived_region: RegionVid, ) -> (ConstraintCategory, bool, Span, Option) { - let (category, from_closure, span) = self.regioncx.best_blame_constraint( - &self.body, - borrow_region, - NllRegionVariableOrigin::FreeRegion, - |r| self.regioncx.provides_universal_region(r, borrow_region, outlived_region), - ); + let BlameConstraint { category, from_closure, span, variance_info: _ } = + self.regioncx.best_blame_constraint( + &self.body, + borrow_region, + NllRegionVariableOrigin::FreeRegion, + |r| self.regioncx.provides_universal_region(r, borrow_region, outlived_region), + ); let outlived_fr_name = self.give_region_a_name(outlived_region); @@ -311,13 +347,13 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> { let borrow_location = location; if self.is_use_in_later_iteration_of_loop(borrow_location, location) { let later_use = self.later_use_kind(borrow, spans, location); - BorrowExplanation::UsedLaterInLoop(later_use.0, later_use.1) + BorrowExplanation::UsedLaterInLoop(later_use.0, later_use.1, later_use.2) } else { // Check if the location represents a `FakeRead`, and adapt the error // message to the `FakeReadCause` it is from: in particular, // the ones inserted in optimized `let var = ` patterns. let later_use = self.later_use_kind(borrow, spans, location); - BorrowExplanation::UsedLater(later_use.0, later_use.1) + BorrowExplanation::UsedLater(later_use.0, later_use.1, later_use.2) } } @@ -498,16 +534,19 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> { } /// Determine how the borrow was later used. + /// First span returned points to the location of the conflicting use + /// Second span if `Some` is returned in the case of closures and points + /// to the use of the path fn later_use_kind( &self, borrow: &BorrowData<'tcx>, use_spans: UseSpans<'tcx>, location: Location, - ) -> (LaterUseKind, Span) { + ) -> (LaterUseKind, Span, Option) { match use_spans { - UseSpans::ClosureUse { var_span, .. } => { + UseSpans::ClosureUse { capture_kind_span, path_span, .. } => { // Used in a closure. - (LaterUseKind::ClosureCapture, var_span) + (LaterUseKind::ClosureCapture, capture_kind_span, Some(path_span)) } UseSpans::PatUse(span) | UseSpans::OtherUse(span) @@ -542,7 +581,7 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> { } } }; - return (LaterUseKind::Call, function_span); + return (LaterUseKind::Call, function_span, None); } else { LaterUseKind::Other } @@ -550,7 +589,7 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> { LaterUseKind::Other }; - (kind, span) + (kind, span, None) } } } diff --git a/compiler/rustc_mir/src/borrow_check/diagnostics/mod.rs b/compiler/rustc_mir/src/borrow_check/diagnostics/mod.rs index aa9f18d999..1bb8c7ebe5 100644 --- a/compiler/rustc_mir/src/borrow_check/diagnostics/mod.rs +++ b/compiler/rustc_mir/src/borrow_check/diagnostics/mod.rs @@ -18,7 +18,6 @@ use rustc_span::{ Span, }; use rustc_target::abi::VariantIdx; -use std::iter; use super::borrow_set::BorrowData; use super::MirBorrowckCtxt; @@ -216,11 +215,10 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> { PlaceRef { local, projection: [proj_base @ .., elem] } => { match elem { ProjectionElem::Deref => { - // FIXME(project-rfc_2229#36): print capture precisely here. let upvar_field_projection = self.is_upvar_field_projection(place); if let Some(field) = upvar_field_projection { let var_index = field.index(); - let name = self.upvars[var_index].name.to_string(); + let name = self.upvars[var_index].place.to_string(self.infcx.tcx); if self.upvars[var_index].by_ref { buf.push_str(&name); } else { @@ -265,7 +263,7 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> { let upvar_field_projection = self.is_upvar_field_projection(place); if let Some(field) = upvar_field_projection { let var_index = field.index(); - let name = self.upvars[var_index].name.to_string(); + let name = self.upvars[var_index].place.to_string(self.infcx.tcx); buf.push_str(&name); } else { let field_name = self @@ -550,8 +548,12 @@ pub(super) enum UseSpans<'tcx> { /// The span of the args of the closure, including the `move` keyword if /// it's present. args_span: Span, - /// The span of the first use of the captured variable inside the closure. - var_span: Span, + /// The span of the use resulting in capture kind + /// Check `ty::CaptureInfo` for more details + capture_kind_span: Span, + /// The span of the use resulting in the captured path + /// Check `ty::CaptureInfo` for more details + path_span: Span, }, /// The access is caused by using a variable as the receiver of a method /// that takes 'self' @@ -606,9 +608,23 @@ impl UseSpans<'_> { } } + /// Returns the span of `self`, in the case of a `ClosureUse` returns the `path_span` + pub(super) fn var_or_use_path_span(self) -> Span { + match self { + UseSpans::ClosureUse { path_span: span, .. } + | UseSpans::PatUse(span) + | UseSpans::OtherUse(span) => span, + UseSpans::FnSelfUse { + fn_call_span, kind: FnSelfUseKind::DerefCoercion { .. }, .. + } => fn_call_span, + UseSpans::FnSelfUse { var_span, .. } => var_span, + } + } + + /// Returns the span of `self`, in the case of a `ClosureUse` returns the `capture_kind_span` pub(super) fn var_or_use(self) -> Span { match self { - UseSpans::ClosureUse { var_span: span, .. } + UseSpans::ClosureUse { capture_kind_span: span, .. } | UseSpans::PatUse(span) | UseSpans::OtherUse(span) => span, UseSpans::FnSelfUse { @@ -636,14 +652,35 @@ impl UseSpans<'_> { } } + // Add a span label to the use of the captured variable, if it exists. + // only adds label to the `path_span` + pub(super) fn var_span_label_path_only( + self, + err: &mut DiagnosticBuilder<'_>, + message: impl Into, + ) { + if let UseSpans::ClosureUse { path_span, .. } = self { + err.span_label(path_span, message); + } + } + // Add a span label to the use of the captured variable, if it exists. pub(super) fn var_span_label( self, err: &mut DiagnosticBuilder<'_>, message: impl Into, + kind_desc: impl Into, ) { - if let UseSpans::ClosureUse { var_span, .. } = self { - err.span_label(var_span, message); + if let UseSpans::ClosureUse { capture_kind_span, path_span, .. } = self { + if capture_kind_span == path_span { + err.span_label(capture_kind_span, message); + } else { + let capture_kind_label = + format!("capture is {} because of use here", kind_desc.into()); + let path_label = message; + err.span_label(capture_kind_span, capture_kind_label); + err.span_label(path_span, path_label); + } } } @@ -791,10 +828,15 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> { box AggregateKind::Closure(def_id, _) | box AggregateKind::Generator(def_id, _, _) => { debug!("move_spans: def_id={:?} places={:?}", def_id, places); - if let Some((args_span, generator_kind, var_span)) = + if let Some((args_span, generator_kind, capture_kind_span, path_span)) = self.closure_span(*def_id, moved_place, places) { - return ClosureUse { generator_kind, args_span, var_span }; + return ClosureUse { + generator_kind, + args_span, + capture_kind_span, + path_span, + }; } } _ => {} @@ -809,10 +851,15 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> { | FakeReadCause::ForLet(Some(closure_def_id)) => { debug!("move_spans: def_id={:?} place={:?}", closure_def_id, place); let places = &[Operand::Move(*place)]; - if let Some((args_span, generator_kind, var_span)) = + if let Some((args_span, generator_kind, capture_kind_span, path_span)) = self.closure_span(closure_def_id, moved_place, places) { - return ClosureUse { generator_kind, args_span, var_span }; + return ClosureUse { + generator_kind, + args_span, + capture_kind_span, + path_span, + }; } } _ => {} @@ -972,10 +1019,10 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> { "borrow_spans: def_id={:?} is_generator={:?} places={:?}", def_id, is_generator, places ); - if let Some((args_span, generator_kind, var_span)) = + if let Some((args_span, generator_kind, capture_kind_span, path_span)) = self.closure_span(*def_id, Place::from(target).as_ref(), places) { - return ClosureUse { generator_kind, args_span, var_span }; + return ClosureUse { generator_kind, args_span, capture_kind_span, path_span }; } else { return OtherUse(use_span); } @@ -989,13 +1036,15 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> { OtherUse(use_span) } - /// Finds the span of a captured variable within a closure or generator. + /// Finds the spans of a captured place within a closure or generator. + /// The first span is the location of the use resulting in the capture kind of the capture + /// The second span is the location the use resulting in the captured path of the capture fn closure_span( &self, def_id: DefId, target_place: PlaceRef<'tcx>, places: &[Operand<'tcx>], - ) -> Option<(Span, Option, Span)> { + ) -> Option<(Span, Option, Span, Span)> { debug!( "closure_span: def_id={:?} target_place={:?} places={:?}", def_id, target_place, places @@ -1005,13 +1054,13 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> { let expr = &self.infcx.tcx.hir().expect_expr(hir_id).kind; debug!("closure_span: hir_id={:?} expr={:?}", hir_id, expr); if let hir::ExprKind::Closure(.., body_id, args_span, _) = expr { - for (captured_place, place) in iter::zip( - self.infcx.tcx.typeck(def_id.expect_local()).closure_min_captures_flattened(def_id), - places, - ) { - let upvar_hir_id = captured_place.get_root_variable(); - //FIXME(project-rfc-2229#8): Use better span from captured_place - let span = self.infcx.tcx.upvars_mentioned(local_did)?[&upvar_hir_id].span; + for (captured_place, place) in self + .infcx + .tcx + .typeck(def_id.expect_local()) + .closure_min_captures_flattened(def_id) + .zip(places) + { match place { Operand::Copy(place) | Operand::Move(place) if target_place == place.as_ref() => @@ -1020,18 +1069,12 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> { let body = self.infcx.tcx.hir().body(*body_id); let generator_kind = body.generator_kind(); - // If we have a more specific span available, point to that. - // We do this even though this span might be part of a borrow error - // message rather than a move error message. Our goal is to point - // to a span that shows why the upvar is used in the closure, - // so a move-related span is as good as any (and potentially better, - // if the overall error is due to a move of the upvar). - - let usage_span = match captured_place.info.capture_kind { - ty::UpvarCapture::ByValue(Some(span)) => span, - _ => span, - }; - return Some((*args_span, generator_kind, usage_span)); + return Some(( + *args_span, + generator_kind, + captured_place.get_capture_kind_span(self.infcx.tcx), + captured_place.get_path_span(self.infcx.tcx), + )); } _ => {} } diff --git a/compiler/rustc_mir/src/borrow_check/diagnostics/move_errors.rs b/compiler/rustc_mir/src/borrow_check/diagnostics/move_errors.rs index fb7694b7d8..3f87d9c7ac 100644 --- a/compiler/rustc_mir/src/borrow_check/diagnostics/move_errors.rs +++ b/compiler/rustc_mir/src/borrow_check/diagnostics/move_errors.rs @@ -345,10 +345,8 @@ impl<'a, 'tcx> MirBorrowckCtxt<'a, 'tcx> { }; let upvar = &self.upvars[upvar_field.unwrap().index()]; - // FIXME(project-rfc-2229#8): Improve borrow-check diagnostics in case of precise - // capture. let upvar_hir_id = upvar.place.get_root_variable(); - let upvar_name = upvar.name; + let upvar_name = upvar.place.to_string(self.infcx.tcx); let upvar_span = self.infcx.tcx.hir().span(upvar_hir_id); let place_name = self.describe_any_place(move_place.as_ref()); @@ -478,8 +476,11 @@ impl<'a, 'tcx> MirBorrowckCtxt<'a, 'tcx> { self.note_type_does_not_implement_copy(err, &place_desc, place_ty, Some(span), ""); use_spans.args_span_label(err, format!("move out of {} occurs here", place_desc)); - use_spans - .var_span_label(err, format!("move occurs due to use{}", use_spans.describe())); + use_spans.var_span_label( + err, + format!("move occurs due to use{}", use_spans.describe()), + "moved", + ); } } } diff --git a/compiler/rustc_mir/src/borrow_check/diagnostics/mutability_errors.rs b/compiler/rustc_mir/src/borrow_check/diagnostics/mutability_errors.rs index d1fb999e51..bf5f2c0eec 100644 --- a/compiler/rustc_mir/src/borrow_check/diagnostics/mutability_errors.rs +++ b/compiler/rustc_mir/src/borrow_check/diagnostics/mutability_errors.rs @@ -1,12 +1,11 @@ use rustc_hir as hir; use rustc_hir::Node; -use rustc_index::vec::Idx; use rustc_middle::hir::map::Map; use rustc_middle::mir::{Mutability, Place, PlaceRef, ProjectionElem}; use rustc_middle::ty::{self, Ty, TyCtxt}; use rustc_middle::{ hir::place::PlaceBase, - mir::{self, ClearCrossCrate, Local, LocalDecl, LocalInfo, Location}, + mir::{self, ClearCrossCrate, Local, LocalDecl, LocalInfo, LocalKind, Location}, }; use rustc_span::source_map::DesugaringKind; use rustc_span::symbol::{kw, Symbol}; @@ -85,7 +84,7 @@ impl<'a, 'tcx> MirBorrowckCtxt<'a, 'tcx> { if self.is_upvar_field_projection(access_place.as_ref()).is_some() { reason = ", as it is not declared as mutable".to_string(); } else { - let name = self.upvars[upvar_index.index()].name; + let name = self.upvars[upvar_index.index()].place.to_string(self.infcx.tcx); reason = format!(", as `{}` is not declared as mutable", name); } } @@ -115,12 +114,14 @@ impl<'a, 'tcx> MirBorrowckCtxt<'a, 'tcx> { } } PlaceRef { local: _, projection: [proj_base @ .., ProjectionElem::Deref] } => { - if the_place_err.local == Local::new(1) + if the_place_err.local == ty::CAPTURE_STRUCT_LOCAL && proj_base.is_empty() && !self.upvars.is_empty() { item_msg = format!("`{}`", access_place_desc.unwrap()); - debug_assert!(self.body.local_decls[Local::new(1)].ty.is_region_ptr()); + debug_assert!( + self.body.local_decls[ty::CAPTURE_STRUCT_LOCAL].ty.is_region_ptr() + ); debug_assert!(is_closure_or_generator( Place::ty_from( the_place_err.local, @@ -195,6 +196,7 @@ impl<'a, 'tcx> MirBorrowckCtxt<'a, 'tcx> { "mutable borrow occurs due to use of {} in closure", self.describe_any_place(access_place.as_ref()), ), + "mutable", ); borrow_span } @@ -423,15 +425,28 @@ impl<'a, 'tcx> MirBorrowckCtxt<'a, 'tcx> { match label { Some((true, err_help_span, suggested_code)) => { - err.span_suggestion( - err_help_span, - &format!( - "consider changing this to be a mutable {}", - pointer_desc - ), - suggested_code, - Applicability::MachineApplicable, - ); + let (is_trait_sig, local_trait) = self.is_error_in_trait(local); + if !is_trait_sig { + err.span_suggestion( + err_help_span, + &format!( + "consider changing this to be a mutable {}", + pointer_desc + ), + suggested_code, + Applicability::MachineApplicable, + ); + } else if let Some(x) = local_trait { + err.span_suggestion( + x, + &format!( + "consider changing that to be a mutable {}", + pointer_desc + ), + suggested_code, + Applicability::MachineApplicable, + ); + } } Some((false, err_label_span, message)) => { err.span_label(err_label_span, &message); @@ -464,11 +479,9 @@ impl<'a, 'tcx> MirBorrowckCtxt<'a, 'tcx> { } } - PlaceRef { - local, - projection: [ProjectionElem::Deref], - // FIXME document what is this 1 magic number about - } if local == Local::new(1) && !self.upvars.is_empty() => { + PlaceRef { local, projection: [ProjectionElem::Deref] } + if local == ty::CAPTURE_STRUCT_LOCAL && !self.upvars.is_empty() => + { self.expected_fn_found_fn_mut_call(&mut err, span, act); } @@ -502,6 +515,65 @@ impl<'a, 'tcx> MirBorrowckCtxt<'a, 'tcx> { err.buffer(&mut self.errors_buffer); } + /// User cannot make signature of a trait mutable without changing the + /// trait. So we find if this error belongs to a trait and if so we move + /// suggestion to the trait or disable it if it is out of scope of this crate + fn is_error_in_trait(&self, local: Local) -> (bool, Option) { + if self.body.local_kind(local) != LocalKind::Arg { + return (false, None); + } + let hir_map = self.infcx.tcx.hir(); + let my_def = self.body.source.def_id(); + let my_hir = hir_map.local_def_id_to_hir_id(my_def.as_local().unwrap()); + let td = if let Some(a) = + self.infcx.tcx.impl_of_method(my_def).and_then(|x| self.infcx.tcx.trait_id_of_impl(x)) + { + a + } else { + return (false, None); + }; + ( + true, + td.as_local().and_then(|tld| { + let h = hir_map.local_def_id_to_hir_id(tld); + match hir_map.find(h) { + Some(Node::Item(hir::Item { + kind: hir::ItemKind::Trait(_, _, _, _, items), + .. + })) => { + let mut f_in_trait_opt = None; + for hir::TraitItemRef { id: fi, kind: k, .. } in *items { + let hi = fi.hir_id(); + if !matches!(k, hir::AssocItemKind::Fn { .. }) { + continue; + } + if hir_map.name(hi) != hir_map.name(my_hir) { + continue; + } + f_in_trait_opt = Some(hi); + break; + } + f_in_trait_opt.and_then(|f_in_trait| match hir_map.find(f_in_trait) { + Some(Node::TraitItem(hir::TraitItem { + kind: + hir::TraitItemKind::Fn( + hir::FnSig { decl: hir::FnDecl { inputs, .. }, .. }, + _, + ), + .. + })) => { + let hir::Ty { span, .. } = inputs[local.index() - 1]; + Some(span) + } + _ => None, + }) + } + _ => None, + } + }), + ) + } + // point to span of upvar making closure call require mutable borrow fn show_mutating_upvar( &self, @@ -830,9 +902,13 @@ fn suggest_ampmut<'tcx>( { let lt_name = &src[1..ws_pos]; let ty = &src[ws_pos..]; - return (assignment_rhs_span, format!("&{} mut {}", lt_name, ty)); + if !ty.trim_start().starts_with("mut") { + return (assignment_rhs_span, format!("&{} mut {}", lt_name, ty)); + } } else if let Some(stripped) = src.strip_prefix('&') { - return (assignment_rhs_span, format!("&mut {}", stripped)); + if !stripped.trim_start().starts_with("mut") { + return (assignment_rhs_span, format!("&mut {}", stripped)); + } } } } diff --git a/compiler/rustc_mir/src/borrow_check/diagnostics/region_errors.rs b/compiler/rustc_mir/src/borrow_check/diagnostics/region_errors.rs index 058986593a..feb7672f65 100644 --- a/compiler/rustc_mir/src/borrow_check/diagnostics/region_errors.rs +++ b/compiler/rustc_mir/src/borrow_check/diagnostics/region_errors.rs @@ -13,6 +13,7 @@ use rustc_span::Span; use crate::util::borrowck_errors; +use crate::borrow_check::region_infer::BlameConstraint; use crate::borrow_check::{ nll::ConstraintDescription, region_infer::{values::RegionElement, TypeTest}, @@ -275,12 +276,12 @@ impl<'a, 'tcx> MirBorrowckCtxt<'a, 'tcx> { ) { debug!("report_region_error(fr={:?}, outlived_fr={:?})", fr, outlived_fr); - let (category, _, span) = + let BlameConstraint { category, span, variance_info, from_closure: _ } = self.regioncx.best_blame_constraint(&self.body, fr, fr_origin, |r| { self.regioncx.provides_universal_region(r, fr, outlived_fr) }); - debug!("report_region_error: category={:?} {:?}", category, span); + debug!("report_region_error: category={:?} {:?} {:?}", category, span, variance_info); // Check if we can use one of the "nice region errors". if let (Some(f), Some(o)) = (self.to_error_region(fr), self.to_error_region(outlived_fr)) { let nice = NiceRegionError::new_from_span(self.infcx, span, o, f); @@ -309,7 +310,7 @@ impl<'a, 'tcx> MirBorrowckCtxt<'a, 'tcx> { span, }; - let diag = match (category, fr_is_local, outlived_fr_is_local) { + let mut diag = match (category, fr_is_local, outlived_fr_is_local) { (ConstraintCategory::Return(kind), true, false) if self.is_closure_fn_mut(fr) => { self.report_fnmut_error(&errci, kind) } @@ -332,6 +333,19 @@ impl<'a, 'tcx> MirBorrowckCtxt<'a, 'tcx> { } }; + match variance_info { + ty::VarianceDiagInfo::None => {} + ty::VarianceDiagInfo::Mut { kind, ty } => { + let kind_name = match kind { + ty::VarianceDiagMutKind::Ref => "reference", + ty::VarianceDiagMutKind::RawPtr => "pointer", + }; + diag.note(&format!("requirement occurs because of a mutable {kind_name} to {ty}",)); + diag.note(&format!("mutable {kind_name}s are invariant over their type parameter")); + diag.help("see for more information about variance"); + } + } + diag.buffer(&mut self.errors_buffer); } @@ -385,6 +399,7 @@ impl<'a, 'tcx> MirBorrowckCtxt<'a, 'tcx> { diag.span_label(*span, message); + // FIXME(project-rfc-2229#48): This should store a captured_place not a hir id if let ReturnConstraint::ClosureUpvar(upvar) = kind { let def_id = match self.regioncx.universal_regions().defining_ty { DefiningTy::Closure(def_id, _) => def_id, diff --git a/compiler/rustc_mir/src/borrow_check/mod.rs b/compiler/rustc_mir/src/borrow_check/mod.rs index 2d1d83b165..36eb8a4baa 100644 --- a/compiler/rustc_mir/src/borrow_check/mod.rs +++ b/compiler/rustc_mir/src/borrow_check/mod.rs @@ -74,9 +74,6 @@ crate use region_infer::RegionInferenceContext; // FIXME(eddyb) perhaps move this somewhere more centrally. #[derive(Debug)] crate struct Upvar<'tcx> { - // FIXME(project-rfc_2229#36): print capture precisely here. - name: Symbol, - place: CapturedPlace<'tcx>, /// If true, the capture is behind a reference. @@ -159,13 +156,12 @@ fn do_mir_borrowck<'a, 'tcx>( let upvars: Vec<_> = tables .closure_min_captures_flattened(def.did.to_def_id()) .map(|captured_place| { - let var_hir_id = captured_place.get_root_variable(); let capture = captured_place.info.capture_kind; let by_ref = match capture { ty::UpvarCapture::ByValue(_) => false, ty::UpvarCapture::ByRef(..) => true, }; - Upvar { name: tcx.hir().name(var_hir_id), place: captured_place.clone(), by_ref } + Upvar { place: captured_place.clone(), by_ref } }) .collect(); @@ -1969,13 +1965,11 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> { // no move out from an earlier location) then this is an attempt at initialization // of the union - we should error in that case. let tcx = this.infcx.tcx; - if let ty::Adt(def, _) = base.ty(this.body(), tcx).ty.kind() { - if def.is_union() { - if this.move_data.path_map[mpi].iter().any(|moi| { - this.move_data.moves[*moi].source.is_predecessor_of(location, this.body) - }) { - return; - } + if base.ty(this.body(), tcx).ty.is_union() { + if this.move_data.path_map[mpi].iter().any(|moi| { + this.move_data.moves[*moi].source.is_predecessor_of(location, this.body) + }) { + return; } } diff --git a/compiler/rustc_mir/src/borrow_check/nll.rs b/compiler/rustc_mir/src/borrow_check/nll.rs index a0265b20d1..bfeafa33a9 100644 --- a/compiler/rustc_mir/src/borrow_check/nll.rs +++ b/compiler/rustc_mir/src/borrow_check/nll.rs @@ -1,15 +1,14 @@ //! The entry point of the NLL borrow checker. -use rustc_data_structures::fx::FxHashMap; +use rustc_data_structures::vec_map::VecMap; use rustc_errors::Diagnostic; -use rustc_hir::def_id::DefId; use rustc_index::vec::IndexVec; use rustc_infer::infer::InferCtxt; use rustc_middle::mir::{ BasicBlock, Body, ClosureOutlivesSubject, ClosureRegionRequirements, LocalKind, Location, Promoted, }; -use rustc_middle::ty::{self, RegionKind, RegionVid}; +use rustc_middle::ty::{self, OpaqueTypeKey, RegionKind, RegionVid, Ty}; use rustc_span::symbol::sym; use std::env; use std::fmt::Debug; @@ -47,7 +46,7 @@ crate type PoloniusOutput = Output; /// closure requirements to propagate, and any generated errors. crate struct NllOutput<'tcx> { pub regioncx: RegionInferenceContext<'tcx>, - pub opaque_type_values: FxHashMap>, + pub opaque_type_values: VecMap, Ty<'tcx>>, pub polonius_output: Option>, pub opt_closure_req: Option>, pub nll_errors: RegionErrors<'tcx>, @@ -367,7 +366,7 @@ pub(super) fn dump_annotation<'a, 'tcx>( body: &Body<'tcx>, regioncx: &RegionInferenceContext<'tcx>, closure_region_requirements: &Option>, - opaque_type_values: &FxHashMap>, + opaque_type_values: &VecMap, Ty<'tcx>>, errors_buffer: &mut Vec, ) { let tcx = infcx.tcx; diff --git a/compiler/rustc_mir/src/borrow_check/places_conflict.rs b/compiler/rustc_mir/src/borrow_check/places_conflict.rs index 3654b51949..d21550a8e1 100644 --- a/compiler/rustc_mir/src/borrow_check/places_conflict.rs +++ b/compiler/rustc_mir/src/borrow_check/places_conflict.rs @@ -331,17 +331,14 @@ fn place_projection_conflict<'tcx>( Overlap::EqualOrDisjoint } else { let ty = Place::ty_from(pi1_local, pi1_proj_base, body, tcx).ty; - match ty.kind() { - ty::Adt(def, _) if def.is_union() => { - // Different fields of a union, we are basically stuck. - debug!("place_element_conflict: STUCK-UNION"); - Overlap::Arbitrary - } - _ => { - // Different fields of a struct (`a.x` vs. `a.y`). Disjoint! - debug!("place_element_conflict: DISJOINT-FIELD"); - Overlap::Disjoint - } + if ty.is_union() { + // Different fields of a union, we are basically stuck. + debug!("place_element_conflict: STUCK-UNION"); + Overlap::Arbitrary + } else { + // Different fields of a struct (`a.x` vs. `a.y`). Disjoint! + debug!("place_element_conflict: DISJOINT-FIELD"); + Overlap::Disjoint } } } diff --git a/compiler/rustc_mir/src/borrow_check/region_infer/dump_mir.rs b/compiler/rustc_mir/src/borrow_check/region_infer/dump_mir.rs index 86d9db294b..213ebff12a 100644 --- a/compiler/rustc_mir/src/borrow_check/region_infer/dump_mir.rs +++ b/compiler/rustc_mir/src/borrow_check/region_infer/dump_mir.rs @@ -74,9 +74,11 @@ impl<'tcx> RegionInferenceContext<'tcx> { let mut constraints: Vec<_> = self.constraints.outlives().iter().collect(); constraints.sort(); for constraint in &constraints { - let OutlivesConstraint { sup, sub, locations, category } = constraint; + let OutlivesConstraint { sup, sub, locations, category, variance_info: _ } = constraint; let (name, arg) = match locations { - Locations::All(span) => ("All", tcx.sess.source_map().span_to_string(*span)), + Locations::All(span) => { + ("All", tcx.sess.source_map().span_to_embeddable_string(*span)) + } Locations::Single(loc) => ("Single", format!("{:?}", loc)), }; with_msg(&format!("{:?}: {:?} due to {:?} at {}({})", sup, sub, category, name, arg))?; diff --git a/compiler/rustc_mir/src/borrow_check/region_infer/graphviz.rs b/compiler/rustc_mir/src/borrow_check/region_infer/graphviz.rs index 7156612f47..b944d74e6f 100644 --- a/compiler/rustc_mir/src/borrow_check/region_infer/graphviz.rs +++ b/compiler/rustc_mir/src/borrow_check/region_infer/graphviz.rs @@ -35,7 +35,7 @@ struct RawConstraints<'a, 'tcx> { impl<'a, 'this, 'tcx> dot::Labeller<'this> for RawConstraints<'a, 'tcx> { type Node = RegionVid; - type Edge = OutlivesConstraint; + type Edge = OutlivesConstraint<'tcx>; fn graph_id(&'this self) -> dot::Id<'this> { dot::Id::new("RegionInferenceContext").unwrap() @@ -49,31 +49,31 @@ impl<'a, 'this, 'tcx> dot::Labeller<'this> for RawConstraints<'a, 'tcx> { fn node_label(&'this self, n: &RegionVid) -> dot::LabelText<'this> { dot::LabelText::LabelStr(format!("{:?}", n).into()) } - fn edge_label(&'this self, e: &OutlivesConstraint) -> dot::LabelText<'this> { + fn edge_label(&'this self, e: &OutlivesConstraint<'tcx>) -> dot::LabelText<'this> { dot::LabelText::LabelStr(format!("{:?}", e.locations).into()) } } impl<'a, 'this, 'tcx> dot::GraphWalk<'this> for RawConstraints<'a, 'tcx> { type Node = RegionVid; - type Edge = OutlivesConstraint; + type Edge = OutlivesConstraint<'tcx>; fn nodes(&'this self) -> dot::Nodes<'this, RegionVid> { let vids: Vec = self.regioncx.definitions.indices().collect(); vids.into() } - fn edges(&'this self) -> dot::Edges<'this, OutlivesConstraint> { + fn edges(&'this self) -> dot::Edges<'this, OutlivesConstraint<'tcx>> { (&self.regioncx.constraints.outlives().raw[..]).into() } // Render `a: b` as `a -> b`, indicating the flow // of data during inference. - fn source(&'this self, edge: &OutlivesConstraint) -> RegionVid { + fn source(&'this self, edge: &OutlivesConstraint<'tcx>) -> RegionVid { edge.sup } - fn target(&'this self, edge: &OutlivesConstraint) -> RegionVid { + fn target(&'this self, edge: &OutlivesConstraint<'tcx>) -> RegionVid { edge.sub } } diff --git a/compiler/rustc_mir/src/borrow_check/region_infer/mod.rs b/compiler/rustc_mir/src/borrow_check/region_infer/mod.rs index bbd512fd36..dded7a7e3c 100644 --- a/compiler/rustc_mir/src/borrow_check/region_infer/mod.rs +++ b/compiler/rustc_mir/src/borrow_check/region_infer/mod.rs @@ -54,7 +54,7 @@ pub struct RegionInferenceContext<'tcx> { liveness_constraints: LivenessValues, /// The outlives constraints computed by the type-check. - constraints: Frozen, + constraints: Frozen>, /// The constraint-set, but in graph form, making it easy to traverse /// the constraints adjacent to a particular region. Used to construct @@ -227,10 +227,10 @@ enum RegionRelationCheckResult { Error, } -#[derive(Copy, Clone, PartialEq, Eq, Debug)] -enum Trace { +#[derive(Clone, PartialEq, Eq, Debug)] +enum Trace<'tcx> { StartRegion, - FromOutlivesConstraint(OutlivesConstraint), + FromOutlivesConstraint(OutlivesConstraint<'tcx>), NotVisited, } @@ -247,7 +247,7 @@ impl<'tcx> RegionInferenceContext<'tcx> { universal_regions: Rc>, placeholder_indices: Rc, universal_region_relations: Frozen>, - outlives_constraints: OutlivesConstraintSet, + outlives_constraints: OutlivesConstraintSet<'tcx>, member_constraints_in: MemberConstraintSet<'tcx, RegionVid>, closure_bounds_mapping: FxHashMap< Location, @@ -1241,7 +1241,7 @@ impl<'tcx> RegionInferenceContext<'tcx> { /// it. However, it works pretty well in practice. In particular, /// this is needed to deal with projection outlives bounds like /// - /// ```ignore (internal compiler representation so lifetime syntax is invalid) + /// ```text /// >::Item: '1 /// ``` /// @@ -1750,20 +1750,35 @@ impl<'tcx> RegionInferenceContext<'tcx> { crate fn retrieve_closure_constraint_info( &self, body: &Body<'tcx>, - constraint: &OutlivesConstraint, - ) -> (ConstraintCategory, bool, Span) { + constraint: &OutlivesConstraint<'tcx>, + ) -> BlameConstraint<'tcx> { let loc = match constraint.locations { - Locations::All(span) => return (constraint.category, false, span), + Locations::All(span) => { + return BlameConstraint { + category: constraint.category, + from_closure: false, + span, + variance_info: constraint.variance_info.clone(), + }; + } Locations::Single(loc) => loc, }; let opt_span_category = self.closure_bounds_mapping[&loc].get(&(constraint.sup, constraint.sub)); - opt_span_category.map(|&(category, span)| (category, true, span)).unwrap_or(( - constraint.category, - false, - body.source_info(loc).span, - )) + opt_span_category + .map(|&(category, span)| BlameConstraint { + category, + from_closure: true, + span: span, + variance_info: constraint.variance_info.clone(), + }) + .unwrap_or(BlameConstraint { + category: constraint.category, + from_closure: false, + span: body.source_info(loc).span, + variance_info: constraint.variance_info.clone(), + }) } /// Finds a good span to blame for the fact that `fr1` outlives `fr2`. @@ -1774,9 +1789,10 @@ impl<'tcx> RegionInferenceContext<'tcx> { fr1_origin: NllRegionVariableOrigin, fr2: RegionVid, ) -> (ConstraintCategory, Span) { - let (category, _, span) = self.best_blame_constraint(body, fr1, fr1_origin, |r| { - self.provides_universal_region(r, fr1, fr2) - }); + let BlameConstraint { category, span, .. } = + self.best_blame_constraint(body, fr1, fr1_origin, |r| { + self.provides_universal_region(r, fr1, fr2) + }); (category, span) } @@ -1792,7 +1808,7 @@ impl<'tcx> RegionInferenceContext<'tcx> { &self, from_region: RegionVid, target_test: impl Fn(RegionVid) -> bool, - ) -> Option<(Vec, RegionVid)> { + ) -> Option<(Vec>, RegionVid)> { let mut context = IndexVec::from_elem(Trace::NotVisited, &self.definitions); context[from_region] = Trace::StartRegion; @@ -1816,14 +1832,14 @@ impl<'tcx> RegionInferenceContext<'tcx> { let mut result = vec![]; let mut p = r; loop { - match context[p] { + match context[p].clone() { Trace::NotVisited => { bug!("found unvisited region {:?} on path to {:?}", p, r) } Trace::FromOutlivesConstraint(c) => { - result.push(c); p = c.sup; + result.push(c); } Trace::StartRegion => { @@ -1846,7 +1862,7 @@ impl<'tcx> RegionInferenceContext<'tcx> { // Always inline this closure because it can be hot. let mut handle_constraint = #[inline(always)] - |constraint: OutlivesConstraint| { + |constraint: OutlivesConstraint<'tcx>| { debug_assert_eq!(constraint.sup, r); let sub_region = constraint.sub; if let Trace::NotVisited = context[sub_region] { @@ -1870,6 +1886,7 @@ impl<'tcx> RegionInferenceContext<'tcx> { sub: constraint.min_choice, locations: Locations::All(p_c.definition_span), category: ConstraintCategory::OpaqueType, + variance_info: ty::VarianceDiagInfo::default(), }; handle_constraint(constraint); } @@ -1967,7 +1984,7 @@ impl<'tcx> RegionInferenceContext<'tcx> { from_region: RegionVid, from_region_origin: NllRegionVariableOrigin, target_test: impl Fn(RegionVid) -> bool, - ) -> (ConstraintCategory, bool, Span) { + ) -> BlameConstraint<'tcx> { debug!( "best_blame_constraint(from_region={:?}, from_region_origin={:?})", from_region, from_region_origin @@ -1979,7 +1996,7 @@ impl<'tcx> RegionInferenceContext<'tcx> { debug!( "best_blame_constraint: path={:#?}", path.iter() - .map(|&c| format!( + .map(|c| format!( "{:?} ({:?}: {:?})", c, self.constraint_sccs.scc(c.sup), @@ -1989,13 +2006,18 @@ impl<'tcx> RegionInferenceContext<'tcx> { ); // Classify each of the constraints along the path. - let mut categorized_path: Vec<(ConstraintCategory, bool, Span)> = path + let mut categorized_path: Vec> = path .iter() .map(|constraint| { if constraint.category == ConstraintCategory::ClosureBounds { self.retrieve_closure_constraint_info(body, &constraint) } else { - (constraint.category, false, constraint.locations.span(body)) + BlameConstraint { + category: constraint.category, + from_closure: false, + span: constraint.locations.span(body), + variance_info: constraint.variance_info.clone(), + } } }) .collect(); @@ -2067,12 +2089,12 @@ impl<'tcx> RegionInferenceContext<'tcx> { }; let find_region = |i: &usize| { - let constraint = path[*i]; + let constraint = &path[*i]; let constraint_sup_scc = self.constraint_sccs.scc(constraint.sup); if blame_source { - match categorized_path[*i].0 { + match categorized_path[*i].category { ConstraintCategory::OpaqueType | ConstraintCategory::Boring | ConstraintCategory::BoringNoLocation @@ -2083,7 +2105,7 @@ impl<'tcx> RegionInferenceContext<'tcx> { _ => constraint_sup_scc != target_scc, } } else { - match categorized_path[*i].0 { + match categorized_path[*i].category { ConstraintCategory::OpaqueType | ConstraintCategory::Boring | ConstraintCategory::BoringNoLocation @@ -2103,37 +2125,42 @@ impl<'tcx> RegionInferenceContext<'tcx> { if let Some(i) = best_choice { if let Some(next) = categorized_path.get(i + 1) { - if matches!(categorized_path[i].0, ConstraintCategory::Return(_)) - && next.0 == ConstraintCategory::OpaqueType + if matches!(categorized_path[i].category, ConstraintCategory::Return(_)) + && next.category == ConstraintCategory::OpaqueType { // The return expression is being influenced by the return type being // impl Trait, point at the return type and not the return expr. - return *next; + return next.clone(); } } - if categorized_path[i].0 == ConstraintCategory::Return(ReturnConstraint::Normal) { + if categorized_path[i].category == ConstraintCategory::Return(ReturnConstraint::Normal) + { let field = categorized_path.iter().find_map(|p| { - if let ConstraintCategory::ClosureUpvar(f) = p.0 { Some(f) } else { None } + if let ConstraintCategory::ClosureUpvar(f) = p.category { + Some(f) + } else { + None + } }); if let Some(field) = field { - categorized_path[i].0 = + categorized_path[i].category = ConstraintCategory::Return(ReturnConstraint::ClosureUpvar(field)); } } - return categorized_path[i]; + return categorized_path[i].clone(); } // If that search fails, that is.. unusual. Maybe everything // is in the same SCC or something. In that case, find what // appears to be the most interesting point to report to the // user via an even more ad-hoc guess. - categorized_path.sort_by(|p0, p1| p0.0.cmp(&p1.0)); + categorized_path.sort_by(|p0, p1| p0.category.cmp(&p1.category)); debug!("`: sorted_path={:#?}", categorized_path); - *categorized_path.first().unwrap() + categorized_path.remove(0) } } @@ -2228,3 +2255,11 @@ impl<'tcx> ClosureRegionRequirementsExt<'tcx> for ClosureRegionRequirements<'tcx .collect() } } + +#[derive(Clone, Debug)] +pub struct BlameConstraint<'tcx> { + pub category: ConstraintCategory, + pub from_closure: bool, + pub span: Span, + pub variance_info: ty::VarianceDiagInfo<'tcx>, +} diff --git a/compiler/rustc_mir/src/borrow_check/region_infer/opaque_types.rs b/compiler/rustc_mir/src/borrow_check/region_infer/opaque_types.rs index 0d1d255104..3ec24156f2 100644 --- a/compiler/rustc_mir/src/borrow_check/region_infer/opaque_types.rs +++ b/compiler/rustc_mir/src/borrow_check/region_infer/opaque_types.rs @@ -1,7 +1,6 @@ -use rustc_data_structures::fx::FxHashMap; -use rustc_hir::def_id::DefId; +use rustc_data_structures::vec_map::VecMap; use rustc_infer::infer::InferCtxt; -use rustc_middle::ty::{self, TyCtxt, TypeFoldable}; +use rustc_middle::ty::{self, OpaqueTypeKey, Ty, TyCtxt, TypeFoldable}; use rustc_span::Span; use rustc_trait_selection::opaque_types::InferCtxtExt; @@ -51,12 +50,13 @@ impl<'tcx> RegionInferenceContext<'tcx> { pub(in crate::borrow_check) fn infer_opaque_types( &self, infcx: &InferCtxt<'_, 'tcx>, - opaque_ty_decls: FxHashMap>, + opaque_ty_decls: VecMap, Ty<'tcx>>, span: Span, - ) -> FxHashMap> { + ) -> VecMap, Ty<'tcx>> { opaque_ty_decls .into_iter() - .map(|(opaque_def_id, ty::ResolvedOpaqueTy { concrete_type, substs })| { + .map(|(opaque_type_key, concrete_type)| { + let substs = opaque_type_key.substs; debug!(?concrete_type, ?substs); let mut subst_regions = vec![self.universal_regions.fr_static]; @@ -110,16 +110,14 @@ impl<'tcx> RegionInferenceContext<'tcx> { debug!(?universal_concrete_type, ?universal_substs); + let opaque_type_key = + OpaqueTypeKey { def_id: opaque_type_key.def_id, substs: universal_substs }; let remapped_type = infcx.infer_opaque_definition_from_instantiation( - opaque_def_id, - universal_substs, + opaque_type_key, universal_concrete_type, span, ); - ( - opaque_def_id, - ty::ResolvedOpaqueTy { concrete_type: remapped_type, substs: universal_substs }, - ) + (opaque_type_key, remapped_type) }) .collect() } diff --git a/compiler/rustc_mir/src/borrow_check/type_check/constraint_conversion.rs b/compiler/rustc_mir/src/borrow_check/type_check/constraint_conversion.rs index 8513e5e531..eb11b93714 100644 --- a/compiler/rustc_mir/src/borrow_check/type_check/constraint_conversion.rs +++ b/compiler/rustc_mir/src/borrow_check/type_check/constraint_conversion.rs @@ -143,6 +143,7 @@ impl<'a, 'tcx> ConstraintConversion<'a, 'tcx> { category: self.category, sub, sup, + variance_info: ty::VarianceDiagInfo::default(), }); } diff --git a/compiler/rustc_mir/src/borrow_check/type_check/liveness/mod.rs b/compiler/rustc_mir/src/borrow_check/type_check/liveness/mod.rs index bddcd34ed3..a34ae281b7 100644 --- a/compiler/rustc_mir/src/borrow_check/type_check/liveness/mod.rs +++ b/compiler/rustc_mir/src/borrow_check/type_check/liveness/mod.rs @@ -107,7 +107,7 @@ fn compute_live_locals( fn regions_that_outlive_free_regions( num_region_vars: usize, universal_regions: &UniversalRegions<'tcx>, - constraint_set: &OutlivesConstraintSet, + constraint_set: &OutlivesConstraintSet<'tcx>, ) -> FxHashSet { // Build a graph of the outlives constraints thus far. This is // a reverse graph, so for each constraint `R1: R2` we have an diff --git a/compiler/rustc_mir/src/borrow_check/type_check/mod.rs b/compiler/rustc_mir/src/borrow_check/type_check/mod.rs index d27fcb2f26..09cafddeef 100644 --- a/compiler/rustc_mir/src/borrow_check/type_check/mod.rs +++ b/compiler/rustc_mir/src/borrow_check/type_check/mod.rs @@ -7,9 +7,10 @@ use either::Either; use rustc_data_structures::frozen::Frozen; use rustc_data_structures::fx::{FxHashMap, FxHashSet}; +use rustc_data_structures::vec_map::VecMap; use rustc_errors::struct_span_err; use rustc_hir as hir; -use rustc_hir::def_id::{DefId, LocalDefId}; +use rustc_hir::def_id::LocalDefId; use rustc_hir::lang_items::LangItem; use rustc_index::vec::{Idx, IndexVec}; use rustc_infer::infer::canonical::QueryRegionConstraints; @@ -27,8 +28,8 @@ use rustc_middle::ty::cast::CastTy; use rustc_middle::ty::fold::TypeFoldable; use rustc_middle::ty::subst::{GenericArgKind, Subst, SubstsRef, UserSubsts}; use rustc_middle::ty::{ - self, CanonicalUserTypeAnnotation, CanonicalUserTypeAnnotations, RegionVid, ToPredicate, Ty, - TyCtxt, UserType, UserTypeAnnotationIndex, WithConstness, + self, CanonicalUserTypeAnnotation, CanonicalUserTypeAnnotations, OpaqueTypeKey, RegionVid, + ToPredicate, Ty, TyCtxt, UserType, UserTypeAnnotationIndex, WithConstness, }; use rustc_span::{Span, DUMMY_SP}; use rustc_target::abi::VariantIdx; @@ -226,7 +227,7 @@ fn translate_outlives_facts(typeck: &mut TypeChecker<'_, '_>) { let _prof_timer = typeck.infcx.tcx.prof.generic_activity("polonius_fact_generation"); let location_table = cx.location_table; facts.outlives.extend(cx.constraints.outlives_constraints.outlives().iter().flat_map( - |constraint: &OutlivesConstraint| { + |constraint: &OutlivesConstraint<'_>| { if let Some(from_location) = constraint.locations.from_location() { Either::Left(iter::once(( constraint.sup, @@ -572,7 +573,7 @@ impl<'a, 'b, 'tcx> TypeVerifier<'a, 'b, 'tcx> { let locations = location.to_locations(); for constraint in constraints.outlives().iter() { - let mut constraint = *constraint; + let mut constraint = constraint.clone(); constraint.locations = locations; if let ConstraintCategory::Return(_) | ConstraintCategory::UseAsConst @@ -818,7 +819,7 @@ struct TypeChecker<'a, 'tcx> { reported_errors: FxHashSet<(Ty<'tcx>, Span)>, borrowck_context: &'a mut BorrowCheckContext<'a, 'tcx>, universal_region_relations: &'a UniversalRegionRelations<'tcx>, - opaque_type_values: FxHashMap>, + opaque_type_values: VecMap, Ty<'tcx>>, } struct BorrowCheckContext<'a, 'tcx> { @@ -833,7 +834,7 @@ struct BorrowCheckContext<'a, 'tcx> { crate struct MirTypeckResults<'tcx> { crate constraints: MirTypeckRegionConstraints<'tcx>, pub(in crate::borrow_check) universal_region_relations: Frozen>, - crate opaque_type_values: FxHashMap>, + crate opaque_type_values: VecMap, Ty<'tcx>>, } /// A collection of region constraints that must be satisfied for the @@ -862,7 +863,7 @@ crate struct MirTypeckRegionConstraints<'tcx> { /// hence it must report on their liveness constraints. crate liveness_constraints: LivenessValues, - crate outlives_constraints: OutlivesConstraintSet, + crate outlives_constraints: OutlivesConstraintSet<'tcx>, crate member_constraints: MemberConstraintSet<'tcx, RegionVid>, @@ -978,7 +979,7 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> { borrowck_context, reported_errors: Default::default(), universal_region_relations, - opaque_type_values: FxHashMap::default(), + opaque_type_values: VecMap::default(), }; checker.check_user_type_annotations(); checker @@ -1240,7 +1241,7 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> { let param_env = self.param_env; let body = self.body; let concrete_opaque_types = &tcx.typeck(anon_owner_def_id).concrete_opaque_types; - let mut opaque_type_values = Vec::new(); + let mut opaque_type_values = VecMap::new(); debug!("eq_opaque_type_and_type: mir_def_id={:?}", body.source.def_id()); let opaque_type_map = self.fully_perform_op( @@ -1281,37 +1282,39 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> { .eq(output_ty, revealed_ty)?, ); - for (&opaque_def_id, opaque_decl) in &opaque_type_map { + for &(opaque_type_key, opaque_decl) in &opaque_type_map { let resolved_ty = infcx.resolve_vars_if_possible(opaque_decl.concrete_ty); let concrete_is_opaque = if let ty::Opaque(def_id, _) = resolved_ty.kind() { - *def_id == opaque_def_id + *def_id == opaque_type_key.def_id } else { false }; - let opaque_defn_ty = match concrete_opaque_types.get(&opaque_def_id) { + + let concrete_ty = match concrete_opaque_types + .get_by(|(key, _)| key.def_id == opaque_type_key.def_id) + { None => { if !concrete_is_opaque { tcx.sess.delay_span_bug( body.span, &format!( "Non-defining use of {:?} with revealed type", - opaque_def_id, + opaque_type_key.def_id, ), ); } continue; } - Some(opaque_defn_ty) => opaque_defn_ty, + Some(concrete_ty) => concrete_ty, }; - debug!("opaque_defn_ty = {:?}", opaque_defn_ty); - let subst_opaque_defn_ty = - opaque_defn_ty.concrete_type.subst(tcx, opaque_decl.substs); + debug!("concrete_ty = {:?}", concrete_ty); + let subst_opaque_defn_ty = concrete_ty.subst(tcx, opaque_type_key.substs); let renumbered_opaque_defn_ty = renumber::renumber_regions(infcx, subst_opaque_defn_ty); debug!( "eq_opaque_type_and_type: concrete_ty={:?}={:?} opaque_defn_ty={:?}", - opaque_decl.concrete_ty, resolved_ty, renumbered_opaque_defn_ty, + concrete_ty, resolved_ty, renumbered_opaque_defn_ty, ); if !concrete_is_opaque { @@ -1322,13 +1325,7 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> { .at(&ObligationCause::dummy(), param_env) .eq(opaque_decl.concrete_ty, renumbered_opaque_defn_ty)?, ); - opaque_type_values.push(( - opaque_def_id, - ty::ResolvedOpaqueTy { - concrete_type: renumbered_opaque_defn_ty, - substs: opaque_decl.substs, - }, - )); + opaque_type_values.insert(opaque_type_key, renumbered_opaque_defn_ty); } else { // We're using an opaque `impl Trait` type without // 'revealing' it. For example, code like this: @@ -1351,7 +1348,7 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> { // gets 'revealed' into debug!( "eq_opaque_type_and_type: non-defining use of {:?}", - opaque_def_id, + opaque_type_key.def_id, ); } } @@ -1376,14 +1373,14 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> { // prove that `T: Iterator` where `T` is the type we // instantiated it with). if let Some(opaque_type_map) = opaque_type_map { - for (opaque_def_id, opaque_decl) in opaque_type_map { + for (opaque_type_key, opaque_decl) in opaque_type_map { self.fully_perform_op( locations, ConstraintCategory::OpaqueType, CustomTypeOp::new( |_cx| { infcx.constrain_opaque_type( - opaque_def_id, + opaque_type_key, &opaque_decl, GenerateMemberConstraints::IfNoStaticBound, universal_region_relations, @@ -2535,6 +2532,7 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> { sub: borrow_region.to_region_vid(), locations: location.to_locations(), category, + variance_info: ty::VarianceDiagInfo::default(), }); match mutbl { diff --git a/compiler/rustc_mir/src/borrow_check/type_check/relate_tys.rs b/compiler/rustc_mir/src/borrow_check/type_check/relate_tys.rs index 249945f04b..f97252a117 100644 --- a/compiler/rustc_mir/src/borrow_check/type_check/relate_tys.rs +++ b/compiler/rustc_mir/src/borrow_check/type_check/relate_tys.rs @@ -94,7 +94,12 @@ impl TypeRelatingDelegate<'tcx> for NllTypeRelatingDelegate<'_, '_, 'tcx> { ) } - fn push_outlives(&mut self, sup: ty::Region<'tcx>, sub: ty::Region<'tcx>) { + fn push_outlives( + &mut self, + sup: ty::Region<'tcx>, + sub: ty::Region<'tcx>, + info: ty::VarianceDiagInfo<'tcx>, + ) { if let Some(borrowck_context) = &mut self.borrowck_context { let sub = borrowck_context.universal_regions.to_region_vid(sub); let sup = borrowck_context.universal_regions.to_region_vid(sup); @@ -103,6 +108,7 @@ impl TypeRelatingDelegate<'tcx> for NllTypeRelatingDelegate<'_, '_, 'tcx> { sub, locations: self.locations, category: self.category, + variance_info: info, }); } } diff --git a/compiler/rustc_mir/src/const_eval/error.rs b/compiler/rustc_mir/src/const_eval/error.rs index 754ed0bea8..fc21047ab7 100644 --- a/compiler/rustc_mir/src/const_eval/error.rs +++ b/compiler/rustc_mir/src/const_eval/error.rs @@ -9,7 +9,7 @@ use rustc_span::{Span, Symbol}; use super::InterpCx; use crate::interpret::{ - struct_error, ErrorHandled, FrameInfo, InterpError, InterpErrorInfo, Machine, + struct_error, ErrorHandled, FrameInfo, InterpError, InterpErrorInfo, Machine, MachineStopType, }; /// The CTFE machine has some custom error kinds. @@ -24,12 +24,21 @@ pub enum ConstEvalErrKind { Abort(String), } +impl MachineStopType for ConstEvalErrKind { + fn is_hard_err(&self) -> bool { + match self { + Self::Panic { .. } => true, + _ => false, + } + } +} + // The errors become `MachineStop` with plain strings when being raised. // `ConstEvalErr` (in `librustc_middle/mir/interpret/error.rs`) knows to // handle these. impl<'tcx> Into> for ConstEvalErrKind { fn into(self) -> InterpErrorInfo<'tcx> { - err_machine_stop!(self.to_string()).into() + err_machine_stop!(self).into() } } @@ -148,31 +157,10 @@ impl<'tcx> ConstEvalErr<'tcx> { tcx: TyCtxtAt<'tcx>, message: &str, emit: impl FnOnce(DiagnosticBuilder<'_>), - lint_root: Option, + mut lint_root: Option, ) -> ErrorHandled { - let must_error = match self.error { - err_inval!(Layout(LayoutError::Unknown(_))) | err_inval!(TooGeneric) => { - return ErrorHandled::TooGeneric; - } - err_inval!(AlreadyReported(error_reported)) => { - return ErrorHandled::Reported(error_reported); - } - // We must *always* hard error on these, even if the caller wants just a lint. - err_inval!(Layout(LayoutError::SizeOverflow(_))) => true, - _ => false, - }; - trace!("reporting const eval failure at {:?}", self.span); - - let err_msg = match &self.error { - InterpError::MachineStop(msg) => { - // A custom error (`ConstEvalErrKind` in `librustc_mir/interp/const_eval/error.rs`). - // Should be turned into a string by now. - msg.downcast_ref::().expect("invalid MachineStop payload").clone() - } - err => err.to_string(), - }; - let finish = |mut err: DiagnosticBuilder<'_>, span_msg: Option| { + trace!("reporting const eval failure at {:?}", self.span); if let Some(span_msg) = span_msg { err.span_label(self.span, span_msg); } @@ -186,34 +174,50 @@ impl<'tcx> ConstEvalErr<'tcx> { emit(err) }; - if must_error { - // The `message` makes little sense here, this is a more serious error than the - // caller thinks anyway. - // See . - finish(struct_error(tcx, &err_msg), None); - ErrorHandled::Reported(ErrorReported) - } else { - // Regular case. - if let Some(lint_root) = lint_root { - // Report as lint. - let hir_id = self - .stacktrace - .iter() - .rev() - .find_map(|frame| frame.lint_root) - .unwrap_or(lint_root); - tcx.struct_span_lint_hir( - rustc_session::lint::builtin::CONST_ERR, - hir_id, - tcx.span, - |lint| finish(lint.build(message), Some(err_msg)), - ); - ErrorHandled::Linted - } else { - // Report as hard error. - finish(struct_error(tcx, message), Some(err_msg)); - ErrorHandled::Reported(ErrorReported) + // Special handling for certain errors + match &self.error { + // Don't emit a new diagnostic for these errors + err_inval!(Layout(LayoutError::Unknown(_))) | err_inval!(TooGeneric) => { + return ErrorHandled::TooGeneric; + } + err_inval!(AlreadyReported(error_reported)) => { + return ErrorHandled::Reported(*error_reported); + } + err_inval!(Layout(LayoutError::SizeOverflow(_))) => { + // We must *always* hard error on these, even if the caller wants just a lint. + // The `message` makes little sense here, this is a more serious error than the + // caller thinks anyway. + // See . + finish(struct_error(tcx, &self.error.to_string()), None); + return ErrorHandled::Reported(ErrorReported); } + _ => {} + }; + + // If we have a 'hard error', then set `lint_root` to `None` so that we don't + // emit a lint. + if matches!(&self.error, InterpError::MachineStop(err) if err.is_hard_err()) { + lint_root = None; + } + + let err_msg = self.error.to_string(); + + // Regular case - emit a lint. + if let Some(lint_root) = lint_root { + // Report as lint. + let hir_id = + self.stacktrace.iter().rev().find_map(|frame| frame.lint_root).unwrap_or(lint_root); + tcx.struct_span_lint_hir( + rustc_session::lint::builtin::CONST_ERR, + hir_id, + tcx.span, + |lint| finish(lint.build(message), Some(err_msg)), + ); + ErrorHandled::Linted + } else { + // Report as hard error. + finish(struct_error(tcx, message), Some(err_msg)); + ErrorHandled::Reported(ErrorReported) } } } diff --git a/compiler/rustc_mir/src/const_eval/eval_queries.rs b/compiler/rustc_mir/src/const_eval/eval_queries.rs index d51adc8864..460fea3746 100644 --- a/compiler/rustc_mir/src/const_eval/eval_queries.rs +++ b/compiler/rustc_mir/src/const_eval/eval_queries.rs @@ -16,6 +16,7 @@ use rustc_middle::ty::print::with_no_trimmed_paths; use rustc_middle::ty::{self, subst::Subst, TyCtxt}; use rustc_span::source_map::Span; use rustc_target::abi::{Abi, LayoutOf}; +use std::borrow::Cow; use std::convert::TryInto; pub fn note_on_undefined_behavior_error() -> &'static str { @@ -169,8 +170,9 @@ pub(super) fn op_to_const<'tcx>( (ecx.tcx.global_alloc(ptr.alloc_id).unwrap_memory(), ptr.offset.bytes()) } Scalar::Int { .. } => ( - ecx.tcx - .intern_const_alloc(Allocation::from_byte_aligned_bytes(b"" as &[u8])), + ecx.tcx.intern_const_alloc(Allocation::from_bytes_byte_aligned_immutable( + b"" as &[u8], + )), 0, ), }; @@ -327,11 +329,22 @@ pub fn eval_to_allocation_raw_provider<'tcx>( )) } else { let msg = if is_static { - "could not evaluate static initializer" + Cow::from("could not evaluate static initializer") } else { - "evaluation of constant value failed" + // If the current item has generics, we'd like to enrich the message with the + // instance and its substs: to show the actual compile-time values, in addition to + // the expression, leading to the const eval error. + let instance = &key.value.instance; + if !instance.substs.is_empty() { + let instance = with_no_trimmed_paths(|| instance.to_string()); + let msg = format!("evaluation of `{}` failed", instance); + Cow::from(msg) + } else { + Cow::from("evaluation of constant value failed") + } }; - Err(err.report_as_error(ecx.tcx.at(ecx.cur_span()), msg)) + + Err(err.report_as_error(ecx.tcx.at(ecx.cur_span()), &msg)) } } Ok(mplace) => { diff --git a/compiler/rustc_mir/src/const_eval/machine.rs b/compiler/rustc_mir/src/const_eval/machine.rs index 8e9148f5b6..773df7d7b6 100644 --- a/compiler/rustc_mir/src/const_eval/machine.rs +++ b/compiler/rustc_mir/src/const_eval/machine.rs @@ -17,7 +17,7 @@ use rustc_target::spec::abi::Abi; use crate::interpret::{ self, compile_time_machine, AllocId, Allocation, Frame, ImmTy, InterpCx, InterpResult, Memory, - OpTy, PlaceTy, Pointer, Scalar, + OpTy, PlaceTy, Pointer, Scalar, StackPopUnwind, }; use super::error::*; @@ -223,7 +223,7 @@ impl<'mir, 'tcx> interpret::Machine<'mir, 'tcx> for CompileTimeInterpreter<'mir, _abi: Abi, args: &[OpTy<'tcx>], _ret: Option<(&PlaceTy<'tcx>, mir::BasicBlock)>, - _unwind: Option, // unwinding is not supported in consts + _unwind: StackPopUnwind, // unwinding is not supported in consts ) -> InterpResult<'tcx, Option<&'mir mir::Body<'tcx>>> { debug!("find_mir_or_eval_fn: {:?}", instance); @@ -263,7 +263,7 @@ impl<'mir, 'tcx> interpret::Machine<'mir, 'tcx> for CompileTimeInterpreter<'mir, instance: ty::Instance<'tcx>, args: &[OpTy<'tcx>], ret: Option<(&PlaceTy<'tcx>, mir::BasicBlock)>, - _unwind: Option, + _unwind: StackPopUnwind, ) -> InterpResult<'tcx> { // Shared intrinsics. if ecx.emulate_intrinsic(instance, args, ret)? { diff --git a/compiler/rustc_mir/src/const_eval/mod.rs b/compiler/rustc_mir/src/const_eval/mod.rs index 3f14efc920..6a514e9f62 100644 --- a/compiler/rustc_mir/src/const_eval/mod.rs +++ b/compiler/rustc_mir/src/const_eval/mod.rs @@ -181,7 +181,7 @@ pub(crate) fn deref_const<'tcx>( let mplace = ecx.deref_operand(&op).unwrap(); if let Scalar::Ptr(ptr) = mplace.ptr { assert_eq!( - ecx.memory.get_raw(ptr.alloc_id).unwrap().mutability, + tcx.get_global_alloc(ptr.alloc_id).unwrap().unwrap_memory().mutability, Mutability::Not, "deref_const cannot be used with mutable allocations as \ that could allow pattern matching to observe mutable statics", diff --git a/compiler/rustc_mir/src/dataflow/move_paths/builder.rs b/compiler/rustc_mir/src/dataflow/move_paths/builder.rs index 994b403abf..cea465ea1e 100644 --- a/compiler/rustc_mir/src/dataflow/move_paths/builder.rs +++ b/compiler/rustc_mir/src/dataflow/move_paths/builder.rs @@ -137,10 +137,7 @@ impl<'b, 'a, 'tcx> Gatherer<'b, 'a, 'tcx> { self.loc, InteriorOfSliceOrArray { ty: place_ty, - is_index: match elem { - ProjectionElem::Index(..) => true, - _ => false, - }, + is_index: matches!(elem, ProjectionElem::Index(..)), }, )); } @@ -522,10 +519,8 @@ impl<'b, 'a, 'tcx> Gatherer<'b, 'a, 'tcx> { // Check if we are assigning into a field of a union, if so, lookup the place // of the union so it is marked as initialized again. if let Some((place_base, ProjectionElem::Field(_, _))) = place.last_projection() { - if let ty::Adt(def, _) = place_base.ty(self.builder.body, self.builder.tcx).ty.kind() { - if def.is_union() { - place = place_base; - } + if place_base.ty(self.builder.body, self.builder.tcx).ty.is_union() { + place = place_base; } } diff --git a/compiler/rustc_mir/src/interpret/cast.rs b/compiler/rustc_mir/src/interpret/cast.rs index 2d9e6df0ab..848b44d13a 100644 --- a/compiler/rustc_mir/src/interpret/cast.rs +++ b/compiler/rustc_mir/src/interpret/cast.rs @@ -7,7 +7,6 @@ use rustc_middle::mir::CastKind; use rustc_middle::ty::adjustment::PointerCast; use rustc_middle::ty::layout::{IntegerExt, TyAndLayout}; use rustc_middle::ty::{self, FloatTy, Ty, TypeAndMut}; -use rustc_span::symbol::sym; use rustc_target::abi::{Integer, LayoutOf, Variants}; use super::{ @@ -49,13 +48,6 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { // All reifications must be monomorphic, bail out otherwise. ensure_monomorphic_enough(*self.tcx, src.layout.ty)?; - if self.tcx.has_attr(def_id, sym::rustc_args_required_const) { - span_bug!( - self.cur_span(), - "reifying a fn ptr that requires const arguments" - ); - } - let instance = ty::Instance::resolve_for_fn_ptr( *self.tcx, self.param_env, diff --git a/compiler/rustc_mir/src/interpret/eval_context.rs b/compiler/rustc_mir/src/interpret/eval_context.rs index 2d83d6cfbd..6f7519e615 100644 --- a/compiler/rustc_mir/src/interpret/eval_context.rs +++ b/compiler/rustc_mir/src/interpret/eval_context.rs @@ -134,14 +134,25 @@ pub struct FrameInfo<'tcx> { pub lint_root: Option, } -#[derive(Clone, Eq, PartialEq, Debug, HashStable)] // Miri debug-prints these +/// Unwind information. +#[derive(Clone, Copy, Eq, PartialEq, Debug, HashStable)] +pub enum StackPopUnwind { + /// The cleanup block. + Cleanup(mir::BasicBlock), + /// No cleanup needs to be done. + Skip, + /// Unwinding is not allowed (UB). + NotAllowed, +} + +#[derive(Clone, Copy, Eq, PartialEq, Debug, HashStable)] // Miri debug-prints these pub enum StackPopCleanup { /// Jump to the next block in the caller, or cause UB if None (that's a function /// that may never return). Also store layout of return place so /// we can validate it at that layout. /// `ret` stores the block we jump to on a normal return, while `unwind` /// stores the block used for cleanup during unwinding. - Goto { ret: Option, unwind: Option }, + Goto { ret: Option, unwind: StackPopUnwind }, /// Just do nothing: Used by Main and for the `box_alloc` hook in miri. /// `cleanup` says whether locals are deallocated. Static computation /// wants them leaked to intern what they need (and just throw away @@ -263,7 +274,13 @@ impl<'tcx> fmt::Display for FrameInfo<'tcx> { } if !self.span.is_dummy() { let lo = tcx.sess.source_map().lookup_char_pos(self.span.lo()); - write!(f, " at {}:{}:{}", lo.file.name, lo.line, lo.col.to_usize() + 1)?; + write!( + f, + " at {}:{}:{}", + lo.file.name.prefer_local(), + lo.line, + lo.col.to_usize() + 1 + )?; } Ok(()) }) @@ -740,13 +757,20 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { /// *Unwind* to the given `target` basic block. /// Do *not* use for returning! Use `return_to_block` instead. /// - /// If `target` is `None`, that indicates the function does not need cleanup during - /// unwinding, and we will just keep propagating that upwards. - pub fn unwind_to_block(&mut self, target: Option) { + /// If `target` is `StackPopUnwind::Skip`, that indicates the function does not need cleanup + /// during unwinding, and we will just keep propagating that upwards. + /// + /// If `target` is `StackPopUnwind::NotAllowed`, that indicates the function does not allow + /// unwinding, and doing so is UB. + pub fn unwind_to_block(&mut self, target: StackPopUnwind) -> InterpResult<'tcx> { self.frame_mut().loc = match target { - Some(block) => Ok(mir::Location { block, statement_index: 0 }), - None => Err(self.frame_mut().body.span), + StackPopUnwind::Cleanup(block) => Ok(mir::Location { block, statement_index: 0 }), + StackPopUnwind::Skip => Err(self.frame_mut().body.span), + StackPopUnwind::NotAllowed => { + throw_ub_format!("unwinding past a stack frame that does not allow unwinding") + } }; + Ok(()) } /// Pops the current frame from the stack, deallocating the @@ -795,21 +819,20 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { } } + let return_to_block = frame.return_to_block; + // Now where do we jump next? // Usually we want to clean up (deallocate locals), but in a few rare cases we don't. // In that case, we return early. We also avoid validation in that case, // because this is CTFE and the final value will be thoroughly validated anyway. - let (cleanup, next_block) = match frame.return_to_block { - StackPopCleanup::Goto { ret, unwind } => { - (true, Some(if unwinding { unwind } else { ret })) - } - StackPopCleanup::None { cleanup, .. } => (cleanup, None), + let cleanup = match return_to_block { + StackPopCleanup::Goto { .. } => true, + StackPopCleanup::None { cleanup, .. } => cleanup, }; if !cleanup { assert!(self.stack().is_empty(), "only the topmost frame should ever be leaked"); - assert!(next_block.is_none(), "tried to skip cleanup when we have a next block!"); assert!(!unwinding, "tried to skip cleanup during unwinding"); // Leak the locals, skip validation, skip machine hook. return Ok(()); @@ -828,16 +851,20 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { // Normal return, figure out where to jump. if unwinding { // Follow the unwind edge. - let unwind = next_block.expect("Encountered StackPopCleanup::None when unwinding!"); - self.unwind_to_block(unwind); + let unwind = match return_to_block { + StackPopCleanup::Goto { unwind, .. } => unwind, + StackPopCleanup::None { .. } => { + panic!("Encountered StackPopCleanup::None when unwinding!") + } + }; + self.unwind_to_block(unwind) } else { // Follow the normal return edge. - if let Some(ret) = next_block { - self.return_to_block(ret)?; + match return_to_block { + StackPopCleanup::Goto { ret, .. } => self.return_to_block(ret), + StackPopCleanup::None { .. } => Ok(()), } } - - Ok(()) } /// Mark a storage as live, killing the previous content. diff --git a/compiler/rustc_mir/src/interpret/intern.rs b/compiler/rustc_mir/src/interpret/intern.rs index 95464da145..23c0fe97c5 100644 --- a/compiler/rustc_mir/src/interpret/intern.rs +++ b/compiler/rustc_mir/src/interpret/intern.rs @@ -305,7 +305,6 @@ where let base_intern_mode = match intern_kind { InternKind::Static(mutbl) => InternMode::Static(mutbl), // `Constant` includes array lengths. - // `Promoted` includes non-`Copy` array initializers and `rustc_args_required_const` arguments. InternKind::Constant | InternKind::Promoted => InternMode::Const, }; diff --git a/compiler/rustc_mir/src/interpret/intrinsics.rs b/compiler/rustc_mir/src/interpret/intrinsics.rs index d74ef66a4b..4e4166dad5 100644 --- a/compiler/rustc_mir/src/interpret/intrinsics.rs +++ b/compiler/rustc_mir/src/interpret/intrinsics.rs @@ -14,7 +14,7 @@ use rustc_middle::ty; use rustc_middle::ty::subst::SubstsRef; use rustc_middle::ty::{Ty, TyCtxt}; use rustc_span::symbol::{sym, Symbol}; -use rustc_target::abi::{Abi, LayoutOf as _, Primitive, Size}; +use rustc_target::abi::{Abi, Align, LayoutOf as _, Primitive, Size}; use super::{ util::ensure_monomorphic_enough, CheckInAllocMsg, ImmTy, InterpCx, Machine, OpTy, PlaceTy, @@ -56,8 +56,12 @@ crate fn eval_nullary_intrinsic<'tcx>( let alloc = type_name::alloc_type_name(tcx, tp_ty); ConstValue::Slice { data: alloc, start: 0, end: alloc.len() } } - sym::needs_drop => ConstValue::from_bool(tp_ty.needs_drop(tcx, param_env)), + sym::needs_drop => { + ensure_monomorphic_enough(tcx, tp_ty)?; + ConstValue::from_bool(tp_ty.needs_drop(tcx, param_env)) + } sym::min_align_of | sym::pref_align_of => { + // Correctly handles non-monomorphic calls, so there is no need for ensure_monomorphic_enough. let layout = tcx.layout_of(param_env.and(tp_ty)).map_err(|e| err_inval!(Layout(e)))?; let n = match name { sym::pref_align_of => layout.align.pref.bytes(), @@ -71,6 +75,7 @@ crate fn eval_nullary_intrinsic<'tcx>( ConstValue::from_u64(tcx.type_id_hash(tp_ty)) } sym::variant_count => match tp_ty.kind() { + // Correctly handles non-monomorphic calls, so there is no need for ensure_monomorphic_enough. ty::Adt(ref adt, _) => ConstValue::from_machine_usize(adt.variants.len() as u64, &tcx), ty::Projection(_) | ty::Opaque(_, _) @@ -323,7 +328,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { self.write_scalar(result, dest)?; } sym::copy => { - self.copy(&args[0], &args[1], &args[2], /*nonoverlapping*/ false)?; + self.copy_intrinsic(&args[0], &args[1], &args[2], /*nonoverlapping*/ false)?; } sym::offset => { let ptr = self.read_scalar(&args[0])?.check_init()?; @@ -348,8 +353,8 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { let b = self.read_immediate(&args[1])?.to_scalar()?; // Special case: if both scalars are *equal integers* - // and not NULL, we pretend there is an allocation of size 0 right there, - // and their offset is 0. (There's never a valid object at NULL, making it an + // and not null, we pretend there is an allocation of size 0 right there, + // and their offset is 0. (There's never a valid object at null, making it an // exception from the exception.) // This is the dual to the special exception for offset-by-0 // in the inbounds pointer offset operation (see the Miri code, `src/operator.rs`). @@ -501,7 +506,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { /// Offsets a pointer by some multiple of its type, returning an error if the pointer leaves its /// allocation. For integer pointers, we consider each of them their own tiny allocation of size - /// 0, so offset-by-0 (and only 0) is okay -- except that NULL cannot be offset by _any_ value. + /// 0, so offset-by-0 (and only 0) is okay -- except that null cannot be offset by _any_ value. pub fn ptr_offset_inbounds( &self, ptr: Scalar, @@ -521,13 +526,37 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { // pointers to be properly aligned (unlike a read/write operation). let min_ptr = if offset_bytes >= 0 { ptr } else { offset_ptr }; let size = offset_bytes.unsigned_abs(); - // This call handles checking for integer/NULL pointers. + // This call handles checking for integer/null pointers. self.memory.check_ptr_access_align( min_ptr, Size::from_bytes(size), - None, - CheckInAllocMsg::InboundsTest, + Align::ONE, + CheckInAllocMsg::PointerArithmeticTest, )?; Ok(offset_ptr) } + + /// Copy `count*size_of::()` many bytes from `*src` to `*dst`. + pub(crate) fn copy_intrinsic( + &mut self, + src: &OpTy<'tcx, >::PointerTag>, + dst: &OpTy<'tcx, >::PointerTag>, + count: &OpTy<'tcx, >::PointerTag>, + nonoverlapping: bool, + ) -> InterpResult<'tcx> { + let count = self.read_scalar(&count)?.to_machine_usize(self)?; + let layout = self.layout_of(src.layout.ty.builtin_deref(true).unwrap().ty)?; + let (size, align) = (layout.size, layout.align.abi); + let size = size.checked_mul(count, self).ok_or_else(|| { + err_ub_format!( + "overflow computing total size of `{}`", + if nonoverlapping { "copy_nonoverlapping" } else { "copy" } + ) + })?; + + let src = self.read_scalar(&src)?.check_init()?; + let dst = self.read_scalar(&dst)?.check_init()?; + + self.memory.copy(src, align, dst, align, size, nonoverlapping) + } } diff --git a/compiler/rustc_mir/src/interpret/intrinsics/caller_location.rs b/compiler/rustc_mir/src/interpret/intrinsics/caller_location.rs index 4dfdc08b87..792a474910 100644 --- a/compiler/rustc_mir/src/interpret/intrinsics/caller_location.rs +++ b/compiler/rustc_mir/src/interpret/intrinsics/caller_location.rs @@ -1,5 +1,6 @@ use std::convert::TryFrom; +use rustc_ast::Mutability; use rustc_hir::lang_items::LangItem; use rustc_middle::mir::TerminatorKind; use rustc_middle::ty::subst::Subst; @@ -79,7 +80,8 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { line: u32, col: u32, ) -> MPlaceTy<'tcx, M::PointerTag> { - let file = self.allocate_str(&filename.as_str(), MemoryKind::CallerLocation); + let file = + self.allocate_str(&filename.as_str(), MemoryKind::CallerLocation, Mutability::Not); let line = Scalar::from_u32(line); let col = Scalar::from_u32(col); @@ -106,7 +108,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { let topmost = span.ctxt().outer_expn().expansion_cause().unwrap_or(span); let caller = self.tcx.sess.source_map().lookup_char_pos(topmost.lo()); ( - Symbol::intern(&caller.file.name.to_string()), + Symbol::intern(&caller.file.name.prefer_remapped().to_string_lossy()), u32::try_from(caller.line).unwrap(), u32::try_from(caller.col_display).unwrap().checked_add(1).unwrap(), ) diff --git a/compiler/rustc_mir/src/interpret/intrinsics/type_name.rs b/compiler/rustc_mir/src/interpret/intrinsics/type_name.rs index ae5e78ee33..a7012cd63f 100644 --- a/compiler/rustc_mir/src/interpret/intrinsics/type_name.rs +++ b/compiler/rustc_mir/src/interpret/intrinsics/type_name.rs @@ -1,5 +1,5 @@ use rustc_hir::def_id::CrateNum; -use rustc_hir::definitions::{DefPathData, DisambiguatedDefPathData}; +use rustc_hir::definitions::DisambiguatedDefPathData; use rustc_middle::mir::interpret::Allocation; use rustc_middle::ty::{ self, @@ -88,7 +88,7 @@ impl<'tcx> Printer<'tcx> for AbsolutePathPrinter<'tcx> { } fn path_crate(mut self, cnum: CrateNum) -> Result { - self.path.push_str(&self.tcx.original_crate_name(cnum).as_str()); + self.path.push_str(&self.tcx.crate_name(cnum).as_str()); Ok(self) } @@ -127,11 +127,6 @@ impl<'tcx> Printer<'tcx> for AbsolutePathPrinter<'tcx> { ) -> Result { self = print_prefix(self)?; - // Skip `::{{constructor}}` on tuple/unit structs. - if disambiguated_data.data == DefPathData::Ctor { - return Ok(self); - } - write!(self.path, "::{}", disambiguated_data.data).unwrap(); Ok(self) @@ -197,6 +192,6 @@ impl Write for AbsolutePathPrinter<'_> { /// Directly returns an `Allocation` containing an absolute path representation of the given type. crate fn alloc_type_name<'tcx>(tcx: TyCtxt<'tcx>, ty: Ty<'tcx>) -> &'tcx Allocation { let path = AbsolutePathPrinter { tcx, path: String::new() }.print_type(ty).unwrap().path; - let alloc = Allocation::from_byte_aligned_bytes(path.into_bytes()); + let alloc = Allocation::from_bytes_byte_aligned_immutable(path.into_bytes()); tcx.intern_const_alloc(alloc) } diff --git a/compiler/rustc_mir/src/interpret/machine.rs b/compiler/rustc_mir/src/interpret/machine.rs index 65869f9563..0d01dc3c21 100644 --- a/compiler/rustc_mir/src/interpret/machine.rs +++ b/compiler/rustc_mir/src/interpret/machine.rs @@ -13,8 +13,8 @@ use rustc_target::abi::Size; use rustc_target::spec::abi::Abi; use super::{ - AllocId, Allocation, AllocationExtra, CheckInAllocMsg, Frame, ImmTy, InterpCx, InterpResult, - LocalValue, MemPlace, Memory, MemoryKind, OpTy, Operand, PlaceTy, Pointer, Scalar, + AllocId, Allocation, CheckInAllocMsg, Frame, ImmTy, InterpCx, InterpResult, LocalValue, + MemPlace, Memory, MemoryKind, OpTy, Operand, PlaceTy, Pointer, Scalar, StackPopUnwind, }; /// Data returned by Machine::stack_pop, @@ -105,7 +105,7 @@ pub trait Machine<'mir, 'tcx>: Sized { type MemoryExtra; /// Extra data stored in every allocation. - type AllocExtra: AllocationExtra + 'static; + type AllocExtra: Debug + Clone + 'static; /// Memory's allocation map type MemoryMap: AllocMap< @@ -132,6 +132,11 @@ pub trait Machine<'mir, 'tcx>: Sized { /// Whether to enforce the validity invariant fn enforce_validity(ecx: &InterpCx<'mir, 'tcx, Self>) -> bool; + /// Whether function calls should be [ABI](Abi)-checked. + fn enforce_abi(_ecx: &InterpCx<'mir, 'tcx, Self>) -> bool { + true + } + /// Entry point for obtaining the MIR of anything that should get evaluated. /// So not just functions and shims, but also const/static initializers, anonymous /// constants, ... @@ -158,7 +163,7 @@ pub trait Machine<'mir, 'tcx>: Sized { abi: Abi, args: &[OpTy<'tcx, Self::PointerTag>], ret: Option<(&PlaceTy<'tcx, Self::PointerTag>, mir::BasicBlock)>, - unwind: Option, + unwind: StackPopUnwind, ) -> InterpResult<'tcx, Option<&'mir mir::Body<'tcx>>>; /// Execute `fn_val`. It is the hook's responsibility to advance the instruction @@ -169,7 +174,7 @@ pub trait Machine<'mir, 'tcx>: Sized { abi: Abi, args: &[OpTy<'tcx, Self::PointerTag>], ret: Option<(&PlaceTy<'tcx, Self::PointerTag>, mir::BasicBlock)>, - unwind: Option, + unwind: StackPopUnwind, ) -> InterpResult<'tcx>; /// Directly process an intrinsic without pushing a stack frame. It is the hook's @@ -179,7 +184,7 @@ pub trait Machine<'mir, 'tcx>: Sized { instance: ty::Instance<'tcx>, args: &[OpTy<'tcx, Self::PointerTag>], ret: Option<(&PlaceTy<'tcx, Self::PointerTag>, mir::BasicBlock)>, - unwind: Option, + unwind: StackPopUnwind, ) -> InterpResult<'tcx>; /// Called to evaluate `Assert` MIR terminators that trigger a panic. @@ -305,10 +310,39 @@ pub trait Machine<'mir, 'tcx>: Sized { kind: Option>, ) -> (Cow<'b, Allocation>, Self::PointerTag); - /// Called to notify the machine before a deallocation occurs. - fn before_deallocation( + /// Hook for performing extra checks on a memory read access. + /// + /// Takes read-only access to the allocation so we can keep all the memory read + /// operations take `&self`. Use a `RefCell` in `AllocExtra` if you + /// need to mutate. + #[inline(always)] + fn memory_read( + _memory_extra: &Self::MemoryExtra, + _alloc_extra: &Self::AllocExtra, + _ptr: Pointer, + _size: Size, + ) -> InterpResult<'tcx> { + Ok(()) + } + + /// Hook for performing extra checks on a memory write access. + #[inline(always)] + fn memory_written( _memory_extra: &mut Self::MemoryExtra, - _id: AllocId, + _alloc_extra: &mut Self::AllocExtra, + _ptr: Pointer, + _size: Size, + ) -> InterpResult<'tcx> { + Ok(()) + } + + /// Hook for performing extra operations on a memory deallocation. + #[inline(always)] + fn memory_deallocated( + _memory_extra: &mut Self::MemoryExtra, + _alloc_extra: &mut Self::AllocExtra, + _ptr: Pointer, + _size: Size, ) -> InterpResult<'tcx> { Ok(()) } @@ -322,7 +356,7 @@ pub trait Machine<'mir, 'tcx>: Sized { Ok(()) } - /// Executes a retagging operation + /// Executes a retagging operation. #[inline] fn retag( _ecx: &mut InterpCx<'mir, 'tcx, Self>, @@ -369,6 +403,7 @@ pub trait Machine<'mir, 'tcx>: Sized { ) -> InterpResult<'tcx, Pointer> { Err((if int == 0 { // This is UB, seriously. + // (`DanglingIntPointer` with these exact arguments has special printing code.) err_ub!(DanglingIntPointer(0, CheckInAllocMsg::InboundsTest)) } else { // This is just something we cannot support during const-eval. @@ -421,7 +456,7 @@ pub macro compile_time_machine(<$mir: lifetime, $tcx: lifetime>) { _abi: Abi, _args: &[OpTy<$tcx>], _ret: Option<(&PlaceTy<$tcx>, mir::BasicBlock)>, - _unwind: Option, + _unwind: StackPopUnwind, ) -> InterpResult<$tcx> { match fn_val {} } diff --git a/compiler/rustc_mir/src/interpret/memory.rs b/compiler/rustc_mir/src/interpret/memory.rs index 3648748a90..76073599e2 100644 --- a/compiler/rustc_mir/src/interpret/memory.rs +++ b/compiler/rustc_mir/src/interpret/memory.rs @@ -2,10 +2,11 @@ //! //! Generally, we use `Pointer` to denote memory addresses. However, some operations //! have a "size"-like parameter, and they take `Scalar` for the address because -//! if the size is 0, then the pointer can also be a (properly aligned, non-NULL) +//! if the size is 0, then the pointer can also be a (properly aligned, non-null) //! integer. It is crucial that these operations call `check_align` *before* //! short-circuiting the empty case! +use std::assert_matches::assert_matches; use std::borrow::Cow; use std::collections::VecDeque; use std::convert::{TryFrom, TryInto}; @@ -18,8 +19,8 @@ use rustc_middle::ty::{Instance, ParamEnv, TyCtxt}; use rustc_target::abi::{Align, HasDataLayout, Size, TargetDataLayout}; use super::{ - AllocId, AllocMap, Allocation, AllocationExtra, CheckInAllocMsg, GlobalAlloc, InterpResult, - Machine, MayLeak, Pointer, PointerArithmetic, Scalar, + alloc_range, AllocId, AllocMap, AllocRange, Allocation, CheckInAllocMsg, GlobalAlloc, + InterpResult, Machine, MayLeak, Pointer, PointerArithmetic, Scalar, ScalarMaybeUninit, }; use crate::util::pretty; @@ -105,7 +106,7 @@ pub struct Memory<'mir, 'tcx, M: Machine<'mir, 'tcx>> { /// Map for "extra" function pointers. extra_fn_ptr_map: FxHashMap, - /// To be able to compare pointers with NULL, and to check alignment for accesses + /// To be able to compare pointers with null, and to check alignment for accesses /// to ZSTs (where pointers may dangle), we keep track of the size even for allocations /// that do not exist any more. // FIXME: this should not be public, but interning currently needs access to it @@ -125,6 +126,24 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> HasDataLayout for Memory<'mir, 'tcx, M> } } +/// A reference to some allocation that was already bounds-checked for the given region +/// and had the on-access machine hooks run. +#[derive(Copy, Clone)] +pub struct AllocRef<'a, 'tcx, Tag, Extra> { + alloc: &'a Allocation, + range: AllocRange, + tcx: TyCtxt<'tcx>, + alloc_id: AllocId, +} +/// A reference to some allocation that was already bounds-checked for the given region +/// and had the on-access machine hooks run. +pub struct AllocRefMut<'a, 'tcx, Tag, Extra> { + alloc: &'a mut Allocation, + range: AllocRange, + tcx: TyCtxt<'tcx>, + alloc_id: AllocId, +} + impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> Memory<'mir, 'tcx, M> { pub fn new(tcx: TyCtxt<'tcx>, extra: M::MemoryExtra) -> Self { Memory { @@ -201,9 +220,11 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> Memory<'mir, 'tcx, M> { pub fn allocate_bytes( &mut self, bytes: &[u8], + align: Align, kind: MemoryKind, + mutability: Mutability, ) -> Pointer { - let alloc = Allocation::from_byte_aligned_bytes(bytes); + let alloc = Allocation::from_bytes(bytes, align, mutability); self.allocate_with(alloc, kind) } @@ -244,9 +265,17 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> Memory<'mir, 'tcx, M> { let new_ptr = self.allocate(new_size, new_align, kind); let old_size = match old_size_and_align { Some((size, _align)) => size, - None => self.get_raw(ptr.alloc_id)?.size, + None => self.get_raw(ptr.alloc_id)?.size(), }; - self.copy(ptr, new_ptr, old_size.min(new_size), /*nonoverlapping*/ true)?; + // This will also call the access hooks. + self.copy( + ptr.into(), + Align::ONE, + new_ptr.into(), + Align::ONE, + old_size.min(new_size), + /*nonoverlapping*/ true, + )?; self.deallocate(ptr, old_size_and_align, kind)?; Ok(new_ptr) @@ -278,8 +307,6 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> Memory<'mir, 'tcx, M> { ); } - M::before_deallocation(&mut self.extra, ptr.alloc_id)?; - let (alloc_kind, mut alloc) = match self.alloc_map.remove(&ptr.alloc_id) { Some(alloc) => alloc, None => { @@ -297,6 +324,9 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> Memory<'mir, 'tcx, M> { } }; + if alloc.mutability == Mutability::Not { + throw_ub_format!("deallocating immutable allocation {}", ptr.alloc_id); + } if alloc_kind != kind { throw_ub_format!( "deallocating {}, which is {} memory, using {} deallocation operation", @@ -306,11 +336,11 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> Memory<'mir, 'tcx, M> { ); } if let Some((size, align)) = old_size_and_align { - if size != alloc.size || align != alloc.align { + if size != alloc.size() || align != alloc.align { throw_ub_format!( "incorrect layout on deallocation: {} has size {} and alignment {}, but gave size {} and alignment {}", ptr.alloc_id, - alloc.size.bytes(), + alloc.size().bytes(), alloc.align.bytes(), size.bytes(), align.bytes(), @@ -319,11 +349,11 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> Memory<'mir, 'tcx, M> { } // Let the machine take some extra action - let size = alloc.size; - AllocationExtra::memory_deallocated(&mut alloc, ptr, size)?; + let size = alloc.size(); + M::memory_deallocated(&mut self.extra, &mut alloc.extra, ptr, size)?; // Don't forget to remember size and align of this now-dead allocation - let old = self.dead_alloc_map.insert(ptr.alloc_id, (alloc.size, alloc.align)); + let old = self.dead_alloc_map.insert(ptr.alloc_id, (size, alloc.align)); if old.is_some() { bug!("Nothing can be deallocated twice"); } @@ -331,40 +361,53 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> Memory<'mir, 'tcx, M> { Ok(()) } - /// Check if the given scalar is allowed to do a memory access of given `size` - /// and `align`. On success, returns `None` for zero-sized accesses (where - /// nothing else is left to do) and a `Pointer` to use for the actual access otherwise. - /// Crucially, if the input is a `Pointer`, we will test it for liveness - /// *even if* the size is 0. - /// - /// Everyone accessing memory based on a `Scalar` should use this method to get the - /// `Pointer` they need. And even if you already have a `Pointer`, call this method - /// to make sure it is sufficiently aligned and not dangling. Not doing that may - /// cause ICEs. - /// - /// Most of the time you should use `check_mplace_access`, but when you just have a pointer, - /// this method is still appropriate. + /// Internal helper function for APIs that offer memory access based on `Scalar` pointers. #[inline(always)] - pub fn check_ptr_access( + pub(super) fn check_ptr_access( &self, sptr: Scalar, size: Size, align: Align, ) -> InterpResult<'tcx, Option>> { let align = M::enforce_alignment(&self.extra).then_some(align); - self.check_ptr_access_align(sptr, size, align, CheckInAllocMsg::MemoryAccessTest) + self.check_and_deref_ptr(sptr, size, align, CheckInAllocMsg::MemoryAccessTest, |ptr| { + let (size, align) = + self.get_size_and_align(ptr.alloc_id, AllocCheck::Dereferenceable)?; + Ok((size, align, ptr)) + }) } - /// Like `check_ptr_access`, but *definitely* checks alignment when `align` - /// is `Some` (overriding `M::enforce_alignment`). Also lets the caller control - /// the error message for the out-of-bounds case. + /// Check if the given scalar is allowed to do a memory access of given `size` and `align` + /// (ignoring `M::enforce_alignment`). The caller can control the error message for the + /// out-of-bounds case. + #[inline(always)] pub fn check_ptr_access_align( + &self, + sptr: Scalar, + size: Size, + align: Align, + msg: CheckInAllocMsg, + ) -> InterpResult<'tcx> { + self.check_and_deref_ptr(sptr, size, Some(align), msg, |ptr| { + let (size, align) = + self.get_size_and_align(ptr.alloc_id, AllocCheck::Dereferenceable)?; + Ok((size, align, ())) + })?; + Ok(()) + } + + /// Low-level helper function to check if a ptr is in-bounds and potentially return a reference + /// to the allocation it points to. Supports both shared and mutable references, to the actual + /// checking is offloaded to a helper closure. `align` defines whether and which alignment check + /// is done. Returns `None` for size 0, and otherwise `Some` of what `alloc_size` returned. + fn check_and_deref_ptr( &self, sptr: Scalar, size: Size, align: Option, msg: CheckInAllocMsg, - ) -> InterpResult<'tcx, Option>> { + alloc_size: impl FnOnce(Pointer) -> InterpResult<'tcx, (Size, Align, T)>, + ) -> InterpResult<'tcx, Option> { fn check_offset_align(offset: u64, align: Align) -> InterpResult<'static> { if offset % align.bytes() == 0 { Ok(()) @@ -391,7 +434,7 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> Memory<'mir, 'tcx, M> { Ok(bits) => { let bits = u64::try_from(bits).unwrap(); // it's ptr-sized assert!(size.bytes() == 0); - // Must be non-NULL. + // Must be non-null. if bits == 0 { throw_ub!(DanglingIntPointer(0, msg)) } @@ -402,9 +445,8 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> Memory<'mir, 'tcx, M> { None } Err(ptr) => { - let (allocation_size, alloc_align) = - self.get_size_and_align(ptr.alloc_id, AllocCheck::Dereferenceable)?; - // Test bounds. This also ensures non-NULL. + let (allocation_size, alloc_align, ret_val) = alloc_size(ptr)?; + // Test bounds. This also ensures non-null. // It is sufficient to check this for the end pointer. The addition // checks for overflow. let end_ptr = ptr.offset(size, self)?; @@ -431,12 +473,12 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> Memory<'mir, 'tcx, M> { // We can still be zero-sized in this branch, in which case we have to // return `None`. - if size.bytes() == 0 { None } else { Some(ptr) } + if size.bytes() == 0 { None } else { Some(ret_val) } } }) } - /// Test if the pointer might be NULL. + /// Test if the pointer might be null. pub fn ptr_may_be_null(&self, ptr: Pointer) -> bool { let (size, _align) = self .get_size_and_align(ptr.alloc_id, AllocCheck::MaybeDead) @@ -502,8 +544,8 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> Memory<'mir, 'tcx, M> { } /// Gives raw access to the `Allocation`, without bounds or alignment checks. - /// Use the higher-level, `PlaceTy`- and `OpTy`-based APIs in `InterpCx` instead! - pub fn get_raw( + /// The caller is responsible for calling the access hooks! + fn get_raw( &self, id: AllocId, ) -> InterpResult<'tcx, &Allocation> { @@ -537,21 +579,58 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> Memory<'mir, 'tcx, M> { } } + /// "Safe" (bounds and align-checked) allocation access. + pub fn get<'a>( + &'a self, + sptr: Scalar, + size: Size, + align: Align, + ) -> InterpResult<'tcx, Option>> { + let align = M::enforce_alignment(&self.extra).then_some(align); + let ptr_and_alloc = self.check_and_deref_ptr( + sptr, + size, + align, + CheckInAllocMsg::MemoryAccessTest, + |ptr| { + let alloc = self.get_raw(ptr.alloc_id)?; + Ok((alloc.size(), alloc.align, (ptr, alloc))) + }, + )?; + if let Some((ptr, alloc)) = ptr_and_alloc { + M::memory_read(&self.extra, &alloc.extra, ptr, size)?; + let range = alloc_range(ptr.offset, size); + Ok(Some(AllocRef { alloc, range, tcx: self.tcx, alloc_id: ptr.alloc_id })) + } else { + // Even in this branch we have to be sure that we actually access the allocation, in + // order to ensure that `static FOO: Type = FOO;` causes a cycle error instead of + // magically pulling *any* ZST value from the ether. However, the `get_raw` above is + // always called when `sptr` is truly a `Pointer`, so we are good. + Ok(None) + } + } + + /// Return the `extra` field of the given allocation. + pub fn get_alloc_extra<'a>(&'a self, id: AllocId) -> InterpResult<'tcx, &'a M::AllocExtra> { + Ok(&self.get_raw(id)?.extra) + } + /// Gives raw mutable access to the `Allocation`, without bounds or alignment checks. - /// Use the higher-level, `PlaceTy`- and `OpTy`-based APIs in `InterpCx` instead! - pub fn get_raw_mut( + /// The caller is responsible for calling the access hooks! + /// + /// Also returns a ptr to `self.extra` so that the caller can use it in parallel with the + /// allocation. + fn get_raw_mut( &mut self, id: AllocId, - ) -> InterpResult<'tcx, &mut Allocation> { + ) -> InterpResult<'tcx, (&mut Allocation, &mut M::MemoryExtra)> + { let tcx = self.tcx; - let memory_extra = &self.extra; + let memory_extra = &mut self.extra; let a = self.alloc_map.get_mut_or(id, || { // Need to make a copy, even if `get_global_alloc` is able // to give us a cheap reference. let alloc = Self::get_global_alloc(memory_extra, tcx, id, /*is_write*/ true)?; - if alloc.mutability == Mutability::Not { - throw_ub!(WriteToReadOnly(id)) - } let kind = M::GLOBAL_KIND.expect( "I got a global allocation that I have to copy but the machine does \ not expect that to happen", @@ -567,11 +646,41 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> Memory<'mir, 'tcx, M> { if a.mutability == Mutability::Not { throw_ub!(WriteToReadOnly(id)) } - Ok(a) + Ok((a, memory_extra)) } } } + /// "Safe" (bounds and align-checked) allocation access. + pub fn get_mut<'a>( + &'a mut self, + sptr: Scalar, + size: Size, + align: Align, + ) -> InterpResult<'tcx, Option>> { + let ptr = self.check_ptr_access(sptr, size, align)?; + if let Some(ptr) = ptr { + let tcx = self.tcx; + // FIXME: can we somehow avoid looking up the allocation twice here? + // We cannot call `get_raw_mut` inside `check_and_deref_ptr` as that would duplicate `&mut self`. + let (alloc, extra) = self.get_raw_mut(ptr.alloc_id)?; + M::memory_written(extra, &mut alloc.extra, ptr, size)?; + let range = alloc_range(ptr.offset, size); + Ok(Some(AllocRefMut { alloc, range, tcx, alloc_id: ptr.alloc_id })) + } else { + Ok(None) + } + } + + /// Return the `extra` field of the given allocation. + pub fn get_alloc_extra_mut<'a>( + &'a mut self, + id: AllocId, + ) -> InterpResult<'tcx, (&'a mut M::AllocExtra, &'a mut M::MemoryExtra)> { + let (alloc, memory_extra) = self.get_raw_mut(id)?; + Ok((&mut alloc.extra, memory_extra)) + } + /// Obtain the size and alignment of an allocation, even if that allocation has /// been deallocated. /// @@ -586,7 +695,7 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> Memory<'mir, 'tcx, M> { // a) cause cycles in case `id` refers to a static // b) duplicate a global's allocation in miri if let Some((_, alloc)) = self.alloc_map.get(id) { - return Ok((alloc.size, alloc.align)); + return Ok((alloc.size(), alloc.align)); } // # Function pointers @@ -596,7 +705,7 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> Memory<'mir, 'tcx, M> { // The caller requested no function pointers. throw_ub!(DerefFunctionPointer(id)) } else { - Ok((Size::ZERO, Align::from_bytes(1).unwrap())) + Ok((Size::ZERO, Align::ONE)) }; } @@ -614,7 +723,7 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> Memory<'mir, 'tcx, M> { Some(GlobalAlloc::Memory(alloc)) => { // Need to duplicate the logic here, because the global allocations have // different associated types than the interpreter-local ones. - Ok((alloc.size, alloc.align)) + Ok((alloc.size(), alloc.align)) } Some(GlobalAlloc::Function(_)) => bug!("We already checked function pointers above"), // The rest must be dead. @@ -658,7 +767,7 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> Memory<'mir, 'tcx, M> { } pub fn mark_immutable(&mut self, id: AllocId) -> InterpResult<'tcx> { - self.get_raw_mut(id)?.mutability = Mutability::Not; + self.get_raw_mut(id)?.0.mutability = Mutability::Not; Ok(()) } @@ -792,158 +901,172 @@ impl<'a, 'mir, 'tcx, M: Machine<'mir, 'tcx>> std::fmt::Debug for DumpAllocs<'a, } /// Reading and writing. -impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> Memory<'mir, 'tcx, M> { - /// Reads the given number of bytes from memory. Returns them as a slice. - /// - /// Performs appropriate bounds checks. - pub fn read_bytes(&self, ptr: Scalar, size: Size) -> InterpResult<'tcx, &[u8]> { - let ptr = match self.check_ptr_access(ptr, size, Align::from_bytes(1).unwrap())? { - Some(ptr) => ptr, - None => return Ok(&[]), // zero-sized access - }; - self.get_raw(ptr.alloc_id)?.get_bytes(self, ptr, size) +impl<'tcx, 'a, Tag: Copy, Extra> AllocRefMut<'a, 'tcx, Tag, Extra> { + pub fn write_scalar( + &mut self, + range: AllocRange, + val: ScalarMaybeUninit, + ) -> InterpResult<'tcx> { + Ok(self + .alloc + .write_scalar(&self.tcx, self.range.subrange(range), val) + .map_err(|e| e.to_interp_error(self.alloc_id))?) } - /// Reads a 0-terminated sequence of bytes from memory. Returns them as a slice. - /// - /// Performs appropriate bounds checks. - pub fn read_c_str(&self, ptr: Scalar) -> InterpResult<'tcx, &[u8]> { - let ptr = self.force_ptr(ptr)?; // We need to read at least 1 byte, so we *need* a ptr. - self.get_raw(ptr.alloc_id)?.read_c_str(self, ptr) + pub fn write_ptr_sized( + &mut self, + offset: Size, + val: ScalarMaybeUninit, + ) -> InterpResult<'tcx> { + self.write_scalar(alloc_range(offset, self.tcx.data_layout().pointer_size), val) } +} - /// Reads a 0x0000-terminated u16-sequence from memory. Returns them as a Vec. - /// Terminator 0x0000 is not included in the returned Vec. - /// - /// Performs appropriate bounds checks. - pub fn read_wide_str(&self, ptr: Scalar) -> InterpResult<'tcx, Vec> { - let size_2bytes = Size::from_bytes(2); - let align_2bytes = Align::from_bytes(2).unwrap(); - // We need to read at least 2 bytes, so we *need* a ptr. - let mut ptr = self.force_ptr(ptr)?; - let allocation = self.get_raw(ptr.alloc_id)?; - let mut u16_seq = Vec::new(); - - loop { - ptr = self - .check_ptr_access(ptr.into(), size_2bytes, align_2bytes)? - .expect("cannot be a ZST"); - let single_u16 = allocation.read_scalar(self, ptr, size_2bytes)?.to_u16()?; - if single_u16 != 0x0000 { - u16_seq.push(single_u16); - ptr = ptr.offset(size_2bytes, self)?; - } else { - break; - } - } - Ok(u16_seq) +impl<'tcx, 'a, Tag: Copy, Extra> AllocRef<'a, 'tcx, Tag, Extra> { + pub fn read_scalar(&self, range: AllocRange) -> InterpResult<'tcx, ScalarMaybeUninit> { + Ok(self + .alloc + .read_scalar(&self.tcx, self.range.subrange(range)) + .map_err(|e| e.to_interp_error(self.alloc_id))?) } - /// Writes the given stream of bytes into memory. + pub fn read_ptr_sized(&self, offset: Size) -> InterpResult<'tcx, ScalarMaybeUninit> { + self.read_scalar(alloc_range(offset, self.tcx.data_layout().pointer_size)) + } + + pub fn check_bytes(&self, range: AllocRange, allow_uninit_and_ptr: bool) -> InterpResult<'tcx> { + Ok(self + .alloc + .check_bytes(&self.tcx, self.range.subrange(range), allow_uninit_and_ptr) + .map_err(|e| e.to_interp_error(self.alloc_id))?) + } +} + +impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> Memory<'mir, 'tcx, M> { + /// Reads the given number of bytes from memory. Returns them as a slice. /// /// Performs appropriate bounds checks. - pub fn write_bytes( - &mut self, - ptr: Scalar, - src: impl IntoIterator, - ) -> InterpResult<'tcx> { - let mut src = src.into_iter(); - let size = Size::from_bytes(src.size_hint().0); - // `write_bytes` checks that this lower bound `size` matches the upper bound and reality. - let ptr = match self.check_ptr_access(ptr, size, Align::from_bytes(1).unwrap())? { - Some(ptr) => ptr, - None => { - // zero-sized access - assert!(src.next().is_none(), "iterator said it was empty but returned an element"); - return Ok(()); - } + pub fn read_bytes(&self, sptr: Scalar, size: Size) -> InterpResult<'tcx, &[u8]> { + let alloc_ref = match self.get(sptr, size, Align::ONE)? { + Some(a) => a, + None => return Ok(&[]), // zero-sized access }; - let tcx = self.tcx; - self.get_raw_mut(ptr.alloc_id)?.write_bytes(&tcx, ptr, src) + // Side-step AllocRef and directly access the underlying bytes more efficiently. + // (We are staying inside the bounds here so all is good.) + Ok(alloc_ref + .alloc + .get_bytes(&alloc_ref.tcx, alloc_ref.range) + .map_err(|e| e.to_interp_error(alloc_ref.alloc_id))?) } - /// Writes the given stream of u16s into memory. + /// Writes the given stream of bytes into memory. /// /// Performs appropriate bounds checks. - pub fn write_u16s( + pub fn write_bytes( &mut self, - ptr: Scalar, - src: impl IntoIterator, + sptr: Scalar, + src: impl IntoIterator, ) -> InterpResult<'tcx> { let mut src = src.into_iter(); let (lower, upper) = src.size_hint(); let len = upper.expect("can only write bounded iterators"); assert_eq!(lower, len, "can only write iterators with a precise length"); - let size = Size::from_bytes(lower); - let ptr = match self.check_ptr_access(ptr, size, Align::from_bytes(2).unwrap())? { - Some(ptr) => ptr, + let size = Size::from_bytes(len); + let alloc_ref = match self.get_mut(sptr, size, Align::ONE)? { + Some(alloc_ref) => alloc_ref, None => { // zero-sized access - assert!(src.next().is_none(), "iterator said it was empty but returned an element"); + assert_matches!( + src.next(), + None, + "iterator said it was empty but returned an element" + ); return Ok(()); } }; - let tcx = self.tcx; - let allocation = self.get_raw_mut(ptr.alloc_id)?; - for idx in 0..len { - let val = Scalar::from_u16( - src.next().expect("iterator was shorter than it said it would be"), - ); - let offset_ptr = ptr.offset(Size::from_bytes(idx) * 2, &tcx)?; // `Size` multiplication - allocation.write_scalar(&tcx, offset_ptr, val.into(), Size::from_bytes(2))?; + // Side-step AllocRef and directly access the underlying bytes more efficiently. + // (We are staying inside the bounds here so all is good.) + let bytes = alloc_ref.alloc.get_bytes_mut(&alloc_ref.tcx, alloc_ref.range); + // `zip` would stop when the first iterator ends; we want to definitely + // cover all of `bytes`. + for dest in bytes { + *dest = src.next().expect("iterator was shorter than it said it would be"); } - assert!(src.next().is_none(), "iterator was longer than it said it would be"); + assert_matches!(src.next(), None, "iterator was longer than it said it would be"); Ok(()) } - /// Expects the caller to have checked bounds and alignment. pub fn copy( &mut self, - src: Pointer, - dest: Pointer, + src: Scalar, + src_align: Align, + dest: Scalar, + dest_align: Align, size: Size, nonoverlapping: bool, ) -> InterpResult<'tcx> { - self.copy_repeatedly(src, dest, size, 1, nonoverlapping) + self.copy_repeatedly(src, src_align, dest, dest_align, size, 1, nonoverlapping) } - /// Expects the caller to have checked bounds and alignment. pub fn copy_repeatedly( &mut self, - src: Pointer, - dest: Pointer, + src: Scalar, + src_align: Align, + dest: Scalar, + dest_align: Align, size: Size, - length: u64, + num_copies: u64, nonoverlapping: bool, ) -> InterpResult<'tcx> { + let tcx = self.tcx; + // We need to do our own bounds-checks. + let src = self.check_ptr_access(src, size, src_align)?; + let dest = self.check_ptr_access(dest, size * num_copies, dest_align)?; // `Size` multiplication + + // FIXME: we look up both allocations twice here, once ebfore for the `check_ptr_access` + // and once below to get the underlying `&[mut] Allocation`. + + // Source alloc preparations and access hooks. + let src = match src { + None => return Ok(()), // Zero-sized *source*, that means dst is also zero-sized and we have nothing to do. + Some(src_ptr) => src_ptr, + }; + let src_alloc = self.get_raw(src.alloc_id)?; + M::memory_read(&self.extra, &src_alloc.extra, src, size)?; + // We need the `dest` ptr for the next operation, so we get it now. + // We already did the source checks and called the hooks so we are good to return early. + let dest = match dest { + None => return Ok(()), // Zero-sized *destiantion*. + Some(dest_ptr) => dest_ptr, + }; + // first copy the relocations to a temporary buffer, because // `get_bytes_mut` will clear the relocations, which is correct, // since we don't want to keep any relocations at the target. // (`get_bytes_with_uninit_and_ptr` below checks that there are no // relocations overlapping the edges; those would not be handled correctly). - let relocations = - self.get_raw(src.alloc_id)?.prepare_relocation_copy(self, src, size, dest, length); - - let tcx = self.tcx; - - // This checks relocation edges on the src. - let src_bytes = - self.get_raw(src.alloc_id)?.get_bytes_with_uninit_and_ptr(&tcx, src, size)?.as_ptr(); - let dest_bytes = - self.get_raw_mut(dest.alloc_id)?.get_bytes_mut(&tcx, dest, size * length)?; // `Size` multiplication - - // If `dest_bytes` is empty we just optimize to not run anything for zsts. - // See #67539 - if dest_bytes.is_empty() { - return Ok(()); - } - - let dest_bytes = dest_bytes.as_mut_ptr(); - + let relocations = src_alloc.prepare_relocation_copy( + self, + alloc_range(src.offset, size), + dest.offset, + num_copies, + ); // Prepare a copy of the initialization mask. - let compressed = self.get_raw(src.alloc_id)?.compress_uninit_range(src, size); + let compressed = src_alloc.compress_uninit_range(src, size); + // This checks relocation edges on the src. + let src_bytes = src_alloc + .get_bytes_with_uninit_and_ptr(&tcx, alloc_range(src.offset, size)) + .map_err(|e| e.to_interp_error(src.alloc_id))? + .as_ptr(); // raw ptr, so we can also get a ptr to the destination allocation + + // Destination alloc preparations and access hooks. + let (dest_alloc, extra) = self.get_raw_mut(dest.alloc_id)?; + M::memory_written(extra, &mut dest_alloc.extra, dest, size * num_copies)?; + let dest_bytes = dest_alloc + .get_bytes_mut_ptr(&tcx, alloc_range(dest.offset, size * num_copies)) + .as_mut_ptr(); if compressed.no_bytes_init() { // Fast path: If all bytes are `uninit` then there is nothing to copy. The target range @@ -952,8 +1075,7 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> Memory<'mir, 'tcx, M> { // This also avoids writing to the target bytes so that the backing allocation is never // touched if the bytes stay uninitialized for the whole interpreter execution. On contemporary // operating system this can avoid physically allocating the page. - let dest_alloc = self.get_raw_mut(dest.alloc_id)?; - dest_alloc.mark_init(dest, size * length, false); // `Size` multiplication + dest_alloc.mark_init(alloc_range(dest.offset, size * num_copies), false); // `Size` multiplication dest_alloc.mark_relocation_range(relocations); return Ok(()); } @@ -974,7 +1096,7 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> Memory<'mir, 'tcx, M> { } } - for i in 0..length { + for i in 0..num_copies { ptr::copy( src_bytes, dest_bytes.add((size * i).bytes_usize()), // `Size` multiplication @@ -982,7 +1104,7 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> Memory<'mir, 'tcx, M> { ); } } else { - for i in 0..length { + for i in 0..num_copies { ptr::copy_nonoverlapping( src_bytes, dest_bytes.add((size * i).bytes_usize()), // `Size` multiplication @@ -992,16 +1114,10 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> Memory<'mir, 'tcx, M> { } } - // now fill in all the data - self.get_raw_mut(dest.alloc_id)?.mark_compressed_init_range( - &compressed, - dest, - size, - length, - ); - + // now fill in all the "init" data + dest_alloc.mark_compressed_init_range(&compressed, dest, size, num_copies); // copy the relocations to the destination - self.get_raw_mut(dest.alloc_id)?.mark_relocation_range(relocations); + dest_alloc.mark_relocation_range(relocations); Ok(()) } diff --git a/compiler/rustc_mir/src/interpret/mod.rs b/compiler/rustc_mir/src/interpret/mod.rs index a29ef117ac..2b9fe56599 100644 --- a/compiler/rustc_mir/src/interpret/mod.rs +++ b/compiler/rustc_mir/src/interpret/mod.rs @@ -18,10 +18,12 @@ mod visitor; pub use rustc_middle::mir::interpret::*; // have all the `interpret` symbols in one place: here -pub use self::eval_context::{Frame, FrameInfo, InterpCx, LocalState, LocalValue, StackPopCleanup}; +pub use self::eval_context::{ + Frame, FrameInfo, InterpCx, LocalState, LocalValue, StackPopCleanup, StackPopUnwind, +}; pub use self::intern::{intern_const_alloc_recursive, InternKind}; pub use self::machine::{compile_time_machine, AllocMap, Machine, MayLeak, StackPopJump}; -pub use self::memory::{AllocCheck, FnVal, Memory, MemoryKind}; +pub use self::memory::{AllocCheck, AllocRef, AllocRefMut, FnVal, Memory, MemoryKind}; pub use self::operand::{ImmTy, Immediate, OpTy, Operand}; pub use self::place::{MPlaceTy, MemPlace, MemPlaceMeta, Place, PlaceTy}; pub use self::validity::{CtfeValidationMode, RefTracking}; diff --git a/compiler/rustc_mir/src/interpret/operand.rs b/compiler/rustc_mir/src/interpret/operand.rs index e5bc932026..06432a8b90 100644 --- a/compiler/rustc_mir/src/interpret/operand.rs +++ b/compiler/rustc_mir/src/interpret/operand.rs @@ -15,8 +15,8 @@ use rustc_target::abi::{Abi, HasDataLayout, LayoutOf, Size, TagEncoding}; use rustc_target::abi::{VariantIdx, Variants}; use super::{ - from_known_layout, mir_assign_valid_types, ConstValue, GlobalId, InterpCx, InterpResult, - MPlaceTy, Machine, MemPlace, Place, PlaceTy, Pointer, Scalar, ScalarMaybeUninit, + alloc_range, from_known_layout, mir_assign_valid_types, ConstValue, GlobalId, InterpCx, + InterpResult, MPlaceTy, Machine, MemPlace, Place, PlaceTy, Pointer, Scalar, ScalarMaybeUninit, }; /// An `Immediate` represents a single immediate self-contained Rust value. @@ -249,19 +249,9 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { return Ok(None); } - let ptr = match self - .check_mplace_access(mplace, None) - .expect("places should be checked on creation") - { + let alloc = match self.get_alloc(mplace)? { Some(ptr) => ptr, None => { - if let Scalar::Ptr(ptr) = mplace.ptr { - // We may be reading from a static. - // In order to ensure that `static FOO: Type = FOO;` causes a cycle error - // instead of magically pulling *any* ZST value from the ether, we need to - // actually access the referenced allocation. - self.memory.get_raw(ptr.alloc_id)?; - } return Ok(Some(ImmTy { // zero-sized type imm: Scalar::ZST.into(), @@ -270,11 +260,9 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { } }; - let alloc = self.memory.get_raw(ptr.alloc_id)?; - match mplace.layout.abi { Abi::Scalar(..) => { - let scalar = alloc.read_scalar(self, ptr, mplace.layout.size)?; + let scalar = alloc.read_scalar(alloc_range(Size::ZERO, mplace.layout.size))?; Ok(Some(ImmTy { imm: scalar.into(), layout: mplace.layout })) } Abi::ScalarPair(ref a, ref b) => { @@ -283,12 +271,10 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { // which `ptr.offset(b_offset)` cannot possibly fail to satisfy. let (a, b) = (&a.value, &b.value); let (a_size, b_size) = (a.size(self), b.size(self)); - let a_ptr = ptr; let b_offset = a_size.align_to(b.align(self).abi); assert!(b_offset.bytes() > 0); // we later use the offset to tell apart the fields - let b_ptr = ptr.offset(b_offset, self)?; - let a_val = alloc.read_scalar(self, a_ptr, a_size)?; - let b_val = alloc.read_scalar(self, b_ptr, b_size)?; + let a_val = alloc.read_scalar(alloc_range(Size::ZERO, a_size))?; + let b_val = alloc.read_scalar(alloc_range(b_offset, b_size))?; Ok(Some(ImmTy { imm: Immediate::ScalarPair(a_val, b_val), layout: mplace.layout })) } _ => Ok(None), diff --git a/compiler/rustc_mir/src/interpret/place.rs b/compiler/rustc_mir/src/interpret/place.rs index 699b531f50..4c53510ed0 100644 --- a/compiler/rustc_mir/src/interpret/place.rs +++ b/compiler/rustc_mir/src/interpret/place.rs @@ -6,6 +6,7 @@ use std::convert::TryFrom; use std::fmt::Debug; use std::hash::Hash; +use rustc_ast::Mutability; use rustc_macros::HashStable; use rustc_middle::mir; use rustc_middle::ty::layout::{PrimitiveExt, TyAndLayout}; @@ -14,8 +15,8 @@ use rustc_target::abi::{Abi, Align, FieldsShape, TagEncoding}; use rustc_target::abi::{HasDataLayout, LayoutOf, Size, VariantIdx, Variants}; use super::{ - mir_assign_valid_types, AllocId, AllocMap, Allocation, AllocationExtra, ConstAlloc, ImmTy, - Immediate, InterpCx, InterpResult, LocalValue, Machine, MemoryKind, OpTy, Operand, Pointer, + alloc_range, mir_assign_valid_types, AllocRef, AllocRefMut, ConstAlloc, ImmTy, Immediate, + InterpCx, InterpResult, LocalValue, Machine, MemoryKind, OpTy, Operand, Pointer, PointerArithmetic, Scalar, ScalarMaybeUninit, }; @@ -291,9 +292,6 @@ where // FIXME: Working around https://github.com/rust-lang/rust/issues/54385 Tag: Debug + Copy + Eq + Hash + 'static, M: Machine<'mir, 'tcx, PointerTag = Tag>, - // FIXME: Working around https://github.com/rust-lang/rust/issues/24159 - M::MemoryMap: AllocMap, Allocation)>, - M::AllocExtra: AllocationExtra, { /// Take a value, which represents a (thin or wide) reference, and make it a place. /// Alignment is just based on the type. This is the inverse of `MemPlace::to_ref()`. @@ -339,24 +337,26 @@ where self.mplace_access_checked(place, None) } - /// Check if the given place is good for memory access with the given - /// size, falling back to the layout's size if `None` (in the latter case, - /// this must be a statically sized type). - /// - /// On success, returns `None` for zero-sized accesses (where nothing else is - /// left to do) and a `Pointer` to use for the actual access otherwise. #[inline] - pub(super) fn check_mplace_access( + pub(super) fn get_alloc( &self, place: &MPlaceTy<'tcx, M::PointerTag>, - size: Option, - ) -> InterpResult<'tcx, Option>> { - let size = size.unwrap_or_else(|| { - assert!(!place.layout.is_unsized()); - assert!(!place.meta.has_meta()); - place.layout.size - }); - self.memory.check_ptr_access(place.ptr, size, place.align) + ) -> InterpResult<'tcx, Option>> { + assert!(!place.layout.is_unsized()); + assert!(!place.meta.has_meta()); + let size = place.layout.size; + self.memory.get(place.ptr, size, place.align) + } + + #[inline] + pub(super) fn get_alloc_mut( + &mut self, + place: &MPlaceTy<'tcx, M::PointerTag>, + ) -> InterpResult<'tcx, Option>> { + assert!(!place.layout.is_unsized()); + assert!(!place.meta.has_meta()); + let size = place.layout.size; + self.memory.get_mut(place.ptr, size, place.align) } /// Return the "access-checked" version of this `MPlace`, where for non-ZST @@ -373,10 +373,11 @@ where .size_and_align_of_mplace(&place)? .unwrap_or((place.layout.size, place.layout.align.abi)); assert!(place.mplace.align <= align, "dynamic alignment less strict than static one?"); - // Check (stricter) dynamic alignment, unless forced otherwise. - place.mplace.align = force_align.unwrap_or(align); - // When dereferencing a pointer, it must be non-NULL, aligned, and live. - if let Some(ptr) = self.check_mplace_access(&place, Some(size))? { + let align = force_align.unwrap_or(align); + // Record new (stricter, unless forced) alignment requirement in place. + place.mplace.align = align; + // When dereferencing a pointer, it must be non-null, aligned, and live. + if let Some(ptr) = self.memory.check_ptr_access(place.ptr, size, align)? { place.mplace.ptr = ptr.into(); } Ok(place) @@ -786,12 +787,12 @@ where // wrong type. // Invalid places are a thing: the return place of a diverging function - let ptr = match self.check_mplace_access(dest, None)? { - Some(ptr) => ptr, + let tcx = *self.tcx; + let mut alloc = match self.get_alloc_mut(dest)? { + Some(a) => a, None => return Ok(()), // zero-sized access }; - let tcx = *self.tcx; // FIXME: We should check that there are dest.layout.size many bytes available in // memory. The code below is not sufficient, with enough padding it might not // cover all the bytes! @@ -805,12 +806,7 @@ where dest.layout ), } - self.memory.get_raw_mut(ptr.alloc_id)?.write_scalar( - &tcx, - ptr, - scalar, - dest.layout.size, - ) + alloc.write_scalar(alloc_range(Size::ZERO, dest.layout.size), scalar) } Immediate::ScalarPair(a_val, b_val) => { // We checked `ptr_align` above, so all fields will have the alignment they need. @@ -824,16 +820,15 @@ where dest.layout ), }; - let (a_size, b_size) = (a.size(self), b.size(self)); - let b_offset = a_size.align_to(b.align(self).abi); - let b_ptr = ptr.offset(b_offset, self)?; + let (a_size, b_size) = (a.size(&tcx), b.size(&tcx)); + let b_offset = a_size.align_to(b.align(&tcx).abi); // It is tempting to verify `b_offset` against `layout.fields.offset(1)`, // but that does not work: We could be a newtype around a pair, then the // fields do not match the `ScalarPair` components. - self.memory.get_raw_mut(ptr.alloc_id)?.write_scalar(&tcx, ptr, a_val, a_size)?; - self.memory.get_raw_mut(b_ptr.alloc_id)?.write_scalar(&tcx, b_ptr, b_val, b_size) + alloc.write_scalar(alloc_range(Size::ZERO, a_size), a_val)?; + alloc.write_scalar(alloc_range(b_offset, b_size), b_val) } } } @@ -902,19 +897,8 @@ where }); assert_eq!(src.meta, dest.meta, "Can only copy between equally-sized instances"); - let src = self - .check_mplace_access(&src, Some(size)) - .expect("places should be checked on creation"); - let dest = self - .check_mplace_access(&dest, Some(size)) - .expect("places should be checked on creation"); - let (src_ptr, dest_ptr) = match (src, dest) { - (Some(src_ptr), Some(dest_ptr)) => (src_ptr, dest_ptr), - (None, None) => return Ok(()), // zero-sized copy - _ => bug!("The pointers should both be Some or both None"), - }; - - self.memory.copy(src_ptr, dest_ptr, size, /*nonoverlapping*/ true) + self.memory + .copy(src.ptr, src.align, dest.ptr, dest.align, size, /*nonoverlapping*/ true) } /// Copies the data from an operand to a place. The layouts may disagree, but they must @@ -1039,21 +1023,23 @@ where MPlaceTy::from_aligned_ptr(ptr, layout) } - /// Returns a wide MPlace. + /// Returns a wide MPlace of type `&'static [mut] str` to a new 1-aligned allocation. pub fn allocate_str( &mut self, str: &str, kind: MemoryKind, + mutbl: Mutability, ) -> MPlaceTy<'tcx, M::PointerTag> { - let ptr = self.memory.allocate_bytes(str.as_bytes(), kind); + let ptr = self.memory.allocate_bytes(str.as_bytes(), Align::ONE, kind, mutbl); let meta = Scalar::from_machine_usize(u64::try_from(str.len()).unwrap(), self); - let mplace = MemPlace { - ptr: ptr.into(), - align: Align::from_bytes(1).unwrap(), - meta: MemPlaceMeta::Meta(meta), - }; + let mplace = + MemPlace { ptr: ptr.into(), align: Align::ONE, meta: MemPlaceMeta::Meta(meta) }; - let layout = self.layout_of(self.tcx.mk_static_str()).unwrap(); + let ty = self.tcx.mk_ref( + self.tcx.lifetimes.re_static, + ty::TypeAndMut { ty: self.tcx.types.str_, mutbl }, + ); + let layout = self.layout_of(ty).unwrap(); MPlaceTy { mplace, layout } } diff --git a/compiler/rustc_mir/src/interpret/step.rs b/compiler/rustc_mir/src/interpret/step.rs index 6084f67abd..129dd8f8e0 100644 --- a/compiler/rustc_mir/src/interpret/step.rs +++ b/compiler/rustc_mir/src/interpret/step.rs @@ -2,7 +2,6 @@ //! //! The main entry point is the `step` method. -use crate::interpret::OpTy; use rustc_middle::mir; use rustc_middle::mir::interpret::{InterpResult, Scalar}; use rustc_target::abi::LayoutOf; @@ -119,7 +118,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { let src = self.eval_operand(src, None)?; let dst = self.eval_operand(dst, None)?; let count = self.eval_operand(count, None)?; - self.copy(&src, &dst, &count, /* nonoverlapping */ true)?; + self.copy_intrinsic(&src, &dst, &count, /* nonoverlapping */ true)?; } // Statements we do not track. @@ -149,37 +148,6 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { Ok(()) } - pub(crate) fn copy( - &mut self, - src: &OpTy<'tcx, >::PointerTag>, - dst: &OpTy<'tcx, >::PointerTag>, - count: &OpTy<'tcx, >::PointerTag>, - nonoverlapping: bool, - ) -> InterpResult<'tcx> { - let count = self.read_scalar(&count)?.to_machine_usize(self)?; - let layout = self.layout_of(src.layout.ty.builtin_deref(true).unwrap().ty)?; - let (size, align) = (layout.size, layout.align.abi); - let size = size.checked_mul(count, self).ok_or_else(|| { - err_ub_format!( - "overflow computing total size of `{}`", - if nonoverlapping { "copy_nonoverlapping" } else { "copy" } - ) - })?; - - // Make sure we check both pointers for an access of the total size and aligment, - // *even if* the total size is 0. - let src = - self.memory.check_ptr_access(self.read_scalar(&src)?.check_init()?, size, align)?; - - let dst = - self.memory.check_ptr_access(self.read_scalar(&dst)?.check_init()?, size, align)?; - - if let (Some(src), Some(dst)) = (src, dst) { - self.memory.copy(src, dst, size, nonoverlapping)?; - } - Ok(()) - } - /// Evaluate an assignment statement. /// /// There is no separate `eval_rvalue` function. Instead, the code for handling each rvalue @@ -254,28 +222,34 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { } Repeat(ref operand, _) => { - let op = self.eval_operand(operand, None)?; + let src = self.eval_operand(operand, None)?; + assert!(!src.layout.is_unsized()); let dest = self.force_allocation(&dest)?; let length = dest.len(self)?; - if let Some(first_ptr) = self.check_mplace_access(&dest, None)? { - // Write the first. + if length == 0 { + // Nothing to copy... but let's still make sure that `dest` as a place is valid. + self.get_alloc_mut(&dest)?; + } else { + // Write the src to the first element. let first = self.mplace_field(&dest, 0)?; - self.copy_op(&op, &first.into())?; - - if length > 1 { - let elem_size = first.layout.size; - // Copy the rest. This is performance-sensitive code - // for big static/const arrays! - let rest_ptr = first_ptr.offset(elem_size, self)?; - self.memory.copy_repeatedly( - first_ptr, - rest_ptr, - elem_size, - length - 1, - /*nonoverlapping:*/ true, - )?; - } + self.copy_op(&src, &first.into())?; + + // This is performance-sensitive code for big static/const arrays! So we + // avoid writing each operand individually and instead just make many copies + // of the first element. + let elem_size = first.layout.size; + let first_ptr = first.ptr; + let rest_ptr = first_ptr.ptr_offset(elem_size, self)?; + self.memory.copy_repeatedly( + first_ptr, + first.align, + rest_ptr, + first.align, + elem_size, + length - 1, + /*nonoverlapping:*/ true, + )?; } } diff --git a/compiler/rustc_mir/src/interpret/terminator.rs b/compiler/rustc_mir/src/interpret/terminator.rs index 4aa1360d53..a5bdeb55e7 100644 --- a/compiler/rustc_mir/src/interpret/terminator.rs +++ b/compiler/rustc_mir/src/interpret/terminator.rs @@ -1,17 +1,31 @@ use std::borrow::Cow; use std::convert::TryFrom; -use rustc_middle::ty::layout::TyAndLayout; +use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags; +use rustc_middle::ty::layout::{self, TyAndLayout}; use rustc_middle::ty::Instance; -use rustc_middle::{mir, ty}; +use rustc_middle::{ + mir, + ty::{self, Ty}, +}; use rustc_target::abi::{self, LayoutOf as _}; use rustc_target::spec::abi::Abi; use super::{ FnVal, ImmTy, InterpCx, InterpResult, MPlaceTy, Machine, OpTy, PlaceTy, StackPopCleanup, + StackPopUnwind, }; impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { + fn fn_can_unwind(&self, attrs: CodegenFnAttrFlags, abi: Abi) -> bool { + layout::fn_can_unwind( + self.tcx.sess.panic_strategy(), + attrs, + layout::conv_from_spec_abi(*self.tcx, abi), + abi, + ) + } + pub(super) fn eval_terminator( &mut self, terminator: &mir::Terminator<'tcx>, @@ -55,12 +69,16 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { let old_stack = self.frame_idx(); let old_loc = self.frame().loc; let func = self.eval_operand(func, None)?; - let (fn_val, abi) = match *func.layout.ty.kind() { + let (fn_val, abi, caller_can_unwind) = match *func.layout.ty.kind() { ty::FnPtr(sig) => { let caller_abi = sig.abi(); let fn_ptr = self.read_scalar(&func)?.check_init()?; let fn_val = self.memory.get_fn(fn_ptr)?; - (fn_val, caller_abi) + ( + fn_val, + caller_abi, + self.fn_can_unwind(layout::fn_ptr_codegen_fn_attr_flags(), caller_abi), + ) } ty::FnDef(def_id, substs) => { let sig = func.layout.ty.fn_sig(*self.tcx); @@ -69,6 +87,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { self.resolve(ty::WithOptConstParam::unknown(def_id), substs)?, ), sig.abi(), + self.fn_can_unwind(self.tcx.codegen_fn_attrs(def_id).flags, sig.abi()), ) } _ => span_bug!( @@ -86,7 +105,17 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { } None => None, }; - self.eval_fn_call(fn_val, abi, &args[..], ret, *cleanup)?; + self.eval_fn_call( + fn_val, + abi, + &args[..], + ret, + match (cleanup, caller_can_unwind) { + (Some(cleanup), true) => StackPopUnwind::Cleanup(*cleanup), + (None, true) => StackPopUnwind::Skip, + (_, false) => StackPopUnwind::NotAllowed, + }, + )?; // Sanity-check that `eval_fn_call` either pushed a new frame or // did a jump to another block. if self.frame_idx() == old_stack && self.frame().loc == old_loc { @@ -216,7 +245,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { caller_abi: Abi, args: &[OpTy<'tcx, M::PointerTag>], ret: Option<(&PlaceTy<'tcx, M::PointerTag>, mir::BasicBlock)>, - unwind: Option, + mut unwind: StackPopUnwind, ) -> InterpResult<'tcx> { trace!("eval_fn_call: {:#?}", fn_val); @@ -227,17 +256,15 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { } }; + let get_abi = |this: &Self, instance_ty: Ty<'tcx>| match instance_ty.kind() { + ty::FnDef(..) => instance_ty.fn_sig(*this.tcx).abi(), + ty::Closure(..) => Abi::RustCall, + ty::Generator(..) => Abi::Rust, + _ => span_bug!(this.cur_span(), "unexpected callee ty: {:?}", instance_ty), + }; + // ABI check - { - let callee_abi = { - let instance_ty = instance.ty(*self.tcx, self.param_env); - match instance_ty.kind() { - ty::FnDef(..) => instance_ty.fn_sig(*self.tcx).abi(), - ty::Closure(..) => Abi::RustCall, - ty::Generator(..) => Abi::Rust, - _ => span_bug!(self.cur_span(), "unexpected callee ty: {:?}", instance_ty), - } - }; + let check_abi = |callee_abi: Abi| -> InterpResult<'tcx> { let normalize_abi = |abi| match abi { Abi::Rust | Abi::RustCall | Abi::RustIntrinsic | Abi::PlatformIntrinsic => // These are all the same ABI, really. @@ -253,10 +280,14 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { caller_abi.name() ) } - } + Ok(()) + }; match instance.def { ty::InstanceDef::Intrinsic(..) => { + if M::enforce_abi(self) { + check_abi(get_abi(self, instance.ty(*self.tcx, self.param_env)))?; + } assert!(caller_abi == Abi::RustIntrinsic || caller_abi == Abi::PlatformIntrinsic); M::call_intrinsic(self, instance, args, ret, unwind) } @@ -274,6 +305,24 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { None => return Ok(()), }; + // Check against the ABI of the MIR body we are calling (not the ABI of `instance`; + // these can differ when `find_mir_or_eval_fn` does something clever like resolve + // exported symbol names). + let callee_def_id = body.source.def_id(); + let callee_abi = get_abi(self, self.tcx.type_of(callee_def_id)); + + if M::enforce_abi(self) { + check_abi(callee_abi)?; + } + + if !matches!(unwind, StackPopUnwind::NotAllowed) + && !self + .fn_can_unwind(self.tcx.codegen_fn_attrs(callee_def_id).flags, callee_abi) + { + // The callee cannot unwind. + unwind = StackPopUnwind::NotAllowed; + } + self.push_stack_frame( instance, body, @@ -462,7 +511,10 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { Abi::Rust, &[arg.into()], Some((&dest.into(), target)), - unwind, + match unwind { + Some(cleanup) => StackPopUnwind::Cleanup(cleanup), + None => StackPopUnwind::Skip, + }, ) } } diff --git a/compiler/rustc_mir/src/interpret/traits.rs b/compiler/rustc_mir/src/interpret/traits.rs index 50603bdd45..d0c04b5b41 100644 --- a/compiler/rustc_mir/src/interpret/traits.rs +++ b/compiler/rustc_mir/src/interpret/traits.rs @@ -62,32 +62,32 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { let drop = Instance::resolve_drop_in_place(tcx, ty); let drop = self.memory.create_fn_alloc(FnVal::Instance(drop)); + // Prepare the fn ptrs we will write into the vtable later. + let fn_ptrs = methods + .iter() + .enumerate() // remember the original position + .filter_map(|(i, method)| { + if let Some((def_id, substs)) = method { Some((i, def_id, substs)) } else { None } + }) + .map(|(i, def_id, substs)| { + let instance = + ty::Instance::resolve_for_vtable(tcx, self.param_env, *def_id, substs) + .ok_or_else(|| err_inval!(TooGeneric))?; + Ok((i, self.memory.create_fn_alloc(FnVal::Instance(instance)))) + }) + .collect::)>>>()?; + // No need to do any alignment checks on the memory accesses below, because we know the // allocation is correctly aligned as we created it above. Also we're only offsetting by // multiples of `ptr_align`, which means that it will stay aligned to `ptr_align`. - let vtable_alloc = self.memory.get_raw_mut(vtable.alloc_id)?; - vtable_alloc.write_ptr_sized(&tcx, vtable, drop.into())?; - - let size_ptr = vtable.offset(ptr_size, &tcx)?; - vtable_alloc.write_ptr_sized(&tcx, size_ptr, Scalar::from_uint(size, ptr_size).into())?; - let align_ptr = vtable.offset(ptr_size * 2, &tcx)?; - vtable_alloc.write_ptr_sized(&tcx, align_ptr, Scalar::from_uint(align, ptr_size).into())?; - - for (i, method) in methods.iter().enumerate() { - if let Some((def_id, substs)) = *method { - // resolve for vtable: insert shims where needed - let instance = - ty::Instance::resolve_for_vtable(tcx, self.param_env, def_id, substs) - .ok_or_else(|| err_inval!(TooGeneric))?; - let fn_ptr = self.memory.create_fn_alloc(FnVal::Instance(instance)); - // We cannot use `vtable_allic` as we are creating fn ptrs in this loop. - let method_ptr = vtable.offset(ptr_size * (3 + i as u64), &tcx)?; - self.memory.get_raw_mut(vtable.alloc_id)?.write_ptr_sized( - &tcx, - method_ptr, - fn_ptr.into(), - )?; - } + let mut vtable_alloc = + self.memory.get_mut(vtable.into(), vtable_size, ptr_align)?.expect("not a ZST"); + vtable_alloc.write_ptr_sized(ptr_size * 0, drop.into())?; + vtable_alloc.write_ptr_sized(ptr_size * 1, Scalar::from_uint(size, ptr_size).into())?; + vtable_alloc.write_ptr_sized(ptr_size * 2, Scalar::from_uint(align, ptr_size).into())?; + + for (i, fn_ptr) in fn_ptrs.into_iter() { + vtable_alloc.write_ptr_sized(ptr_size * (3 + i as u64), fn_ptr.into())?; } M::after_static_mem_initialized(self, vtable, vtable_size)?; @@ -111,13 +111,9 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { let vtable_slot = vtable.ptr_offset(ptr_size * idx.checked_add(3).unwrap(), self)?; let vtable_slot = self .memory - .check_ptr_access(vtable_slot, ptr_size, self.tcx.data_layout.pointer_align.abi)? + .get(vtable_slot, ptr_size, self.tcx.data_layout.pointer_align.abi)? .expect("cannot be a ZST"); - let fn_ptr = self - .memory - .get_raw(vtable_slot.alloc_id)? - .read_ptr_sized(self, vtable_slot)? - .check_init()?; + let fn_ptr = vtable_slot.read_ptr_sized(Size::ZERO)?.check_init()?; self.memory.get_fn(fn_ptr) } @@ -129,14 +125,9 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { // We don't care about the pointee type; we just want a pointer. let vtable = self .memory - .check_ptr_access( - vtable, - self.tcx.data_layout.pointer_size, - self.tcx.data_layout.pointer_align.abi, - )? + .get(vtable, self.tcx.data_layout.pointer_size, self.tcx.data_layout.pointer_align.abi)? .expect("cannot be a ZST"); - let drop_fn = - self.memory.get_raw(vtable.alloc_id)?.read_ptr_sized(self, vtable)?.check_init()?; + let drop_fn = vtable.read_ptr_sized(Size::ZERO)?.check_init()?; // We *need* an instance here, no other kind of function value, to be able // to determine the type. let drop_instance = self.memory.get_fn(drop_fn)?.as_instance()?; @@ -161,14 +152,14 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { // the size, and the align (which we read below). let vtable = self .memory - .check_ptr_access(vtable, 3 * pointer_size, self.tcx.data_layout.pointer_align.abi)? + .get(vtable, 3 * pointer_size, self.tcx.data_layout.pointer_align.abi)? .expect("cannot be a ZST"); - let alloc = self.memory.get_raw(vtable.alloc_id)?; - let size = alloc.read_ptr_sized(self, vtable.offset(pointer_size, self)?)?.check_init()?; + let size = vtable.read_ptr_sized(pointer_size)?.check_init()?; let size = u64::try_from(self.force_bits(size, pointer_size)?).unwrap(); - let align = - alloc.read_ptr_sized(self, vtable.offset(pointer_size * 2, self)?)?.check_init()?; + let align = vtable.read_ptr_sized(pointer_size * 2)?.check_init()?; let align = u64::try_from(self.force_bits(align, pointer_size)?).unwrap(); + let align = Align::from_bytes(align) + .map_err(|e| err_ub_format!("invalid vtable: alignment {}", e))?; if size >= self.tcx.data_layout.obj_size_bound() { throw_ub_format!( @@ -176,6 +167,6 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { size is bigger than largest supported object" ); } - Ok((Size::from_bytes(size), Align::from_bytes(align).unwrap())) + Ok((Size::from_bytes(size), align)) } } diff --git a/compiler/rustc_mir/src/interpret/validity.rs b/compiler/rustc_mir/src/interpret/validity.rs index 062ef7d8b4..fb165a991b 100644 --- a/compiler/rustc_mir/src/interpret/validity.rs +++ b/compiler/rustc_mir/src/interpret/validity.rs @@ -15,13 +15,13 @@ use rustc_middle::mir::interpret::InterpError; use rustc_middle::ty; use rustc_middle::ty::layout::TyAndLayout; use rustc_span::symbol::{sym, Symbol}; -use rustc_target::abi::{Abi, LayoutOf, Scalar, Size, VariantIdx, Variants}; +use rustc_target::abi::{Abi, LayoutOf, Scalar as ScalarAbi, Size, VariantIdx, Variants}; use std::hash::Hash; use super::{ - CheckInAllocMsg, GlobalAlloc, InterpCx, InterpResult, MPlaceTy, Machine, MemPlaceMeta, OpTy, - ScalarMaybeUninit, ValueVisitor, + alloc_range, CheckInAllocMsg, GlobalAlloc, InterpCx, InterpResult, MPlaceTy, Machine, + MemPlaceMeta, OpTy, Scalar, ScalarMaybeUninit, ValueVisitor, }; macro_rules! throw_validation_failure { @@ -329,8 +329,8 @@ impl<'rt, 'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> ValidityVisitor<'rt, 'mir, ' self.ecx.memory.check_ptr_access_align( vtable, 3 * self.ecx.tcx.data_layout.pointer_size, // drop, size, align - Some(self.ecx.tcx.data_layout.pointer_align.abi), - CheckInAllocMsg::InboundsTest, + self.ecx.tcx.data_layout.pointer_align.abi, + CheckInAllocMsg::InboundsTest, // will anyway be replaced by validity message ), self.path, err_ub!(DanglingIntPointer(..)) | @@ -411,12 +411,12 @@ impl<'rt, 'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> ValidityVisitor<'rt, 'mir, ' // alignment should take attributes into account). .unwrap_or_else(|| (place.layout.size, place.layout.align.abi)); // Direct call to `check_ptr_access_align` checks alignment even on CTFE machines. - let ptr: Option<_> = try_validation!( + try_validation!( self.ecx.memory.check_ptr_access_align( place.ptr, size, - Some(align), - CheckInAllocMsg::InboundsTest, + align, + CheckInAllocMsg::InboundsTest, // will anyway be replaced by validity message ), self.path, err_ub!(AlignmentCheckFailed { required, has }) => @@ -427,7 +427,7 @@ impl<'rt, 'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> ValidityVisitor<'rt, 'mir, ' has.bytes() }, err_ub!(DanglingIntPointer(0, _)) => - { "a NULL {}", kind }, + { "a null {}", kind }, err_ub!(DanglingIntPointer(i, _)) => { "a dangling {} (address 0x{:x} is unallocated)", kind, i }, err_ub!(PointerOutOfBounds { .. }) => @@ -441,9 +441,18 @@ impl<'rt, 'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> ValidityVisitor<'rt, 'mir, ' ); // Recursive checking if let Some(ref mut ref_tracking) = self.ref_tracking { - if let Some(ptr) = ptr { + // Proceed recursively even for ZST, no reason to skip them! + // `!` is a ZST and we want to validate it. + // Normalize before handing `place` to tracking because that will + // check for duplicates. + let place = if size.bytes() > 0 { + self.ecx.force_mplace_ptr(place).expect("we already bounds-checked") + } else { + place + }; + // Skip validation entirely for some external statics + if let Scalar::Ptr(ptr) = place.ptr { // not a ZST - // Skip validation entirely for some external statics let alloc_kind = self.ecx.tcx.get_global_alloc(ptr.alloc_id); if let Some(GlobalAlloc::Static(did)) = alloc_kind { assert!(!self.ecx.tcx.is_thread_local_static(did)); @@ -473,15 +482,6 @@ impl<'rt, 'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> ValidityVisitor<'rt, 'mir, ' return Ok(()); } } - // Proceed recursively even for ZST, no reason to skip them! - // `!` is a ZST and we want to validate it. - // Normalize before handing `place` to tracking because that will - // check for duplicates. - let place = if size.bytes() > 0 { - self.ecx.force_mplace_ptr(place).expect("we already bounds-checked") - } else { - place - }; let path = &self.path; ref_tracking.track(place, || { // We need to clone the path anyway, make sure it gets created @@ -638,7 +638,7 @@ impl<'rt, 'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> ValidityVisitor<'rt, 'mir, ' fn visit_scalar( &mut self, op: &OpTy<'tcx, M::PointerTag>, - scalar_layout: &Scalar, + scalar_layout: &ScalarAbi, ) -> InterpResult<'tcx> { let value = self.read_scalar(op)?; let valid_range = &scalar_layout.valid_range; @@ -662,10 +662,10 @@ impl<'rt, 'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> ValidityVisitor<'rt, 'mir, ' let bits = match value.to_bits_or_ptr(op.layout.size, self.ecx) { Err(ptr) => { if lo == 1 && hi == max_hi { - // Only NULL is the niche. So make sure the ptr is NOT NULL. + // Only null is the niche. So make sure the ptr is NOT null. if self.ecx.memory.ptr_may_be_null(ptr) { throw_validation_failure!(self.path, - { "a potentially NULL pointer" } + { "a potentially null pointer" } expected { "something that cannot possibly fail to be {}", wrapping_range_format(valid_range, max_hi) @@ -833,6 +833,7 @@ impl<'rt, 'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> ValueVisitor<'mir, 'tcx, M> self.ecx.memory.read_bytes(mplace.ptr, Size::from_bytes(len)), self.path, err_ub!(InvalidUninitBytes(..)) => { "uninitialized data in `str`" }, + err_unsup!(ReadPointerAsBytes) => { "a pointer in `str`" }, ); } ty::Array(tys, ..) | ty::Slice(tys) @@ -850,16 +851,10 @@ impl<'rt, 'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> ValueVisitor<'mir, 'tcx, M> let mplace = op.assert_mem_place(self.ecx); // This is the length of the array/slice. let len = mplace.len(self.ecx)?; - // Zero length slices have nothing to be checked. - if len == 0 { - return Ok(()); - } // This is the element type size. let layout = self.ecx.layout_of(tys)?; // This is the size in bytes of the whole array. (This checks for overflow.) let size = layout.size * len; - // Size is not 0, get a pointer. - let ptr = self.ecx.force_ptr(mplace.ptr)?; // Optimization: we just check the entire range at once. // NOTE: Keep this in sync with the handling of integer and float @@ -871,10 +866,16 @@ impl<'rt, 'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> ValueVisitor<'mir, 'tcx, M> // to reject those pointers, we just do not have the machinery to // talk about parts of a pointer. // We also accept uninit, for consistency with the slow path. - match self.ecx.memory.get_raw(ptr.alloc_id)?.check_bytes( - self.ecx, - ptr, - size, + let alloc = match self.ecx.memory.get(mplace.ptr, size, mplace.align)? { + Some(a) => a, + None => { + // Size 0, nothing more to check. + return Ok(()); + } + }; + + match alloc.check_bytes( + alloc_range(Size::ZERO, size), /*allow_uninit_and_ptr*/ self.ctfe_mode.is_none(), ) { // In the happy case, we needn't check anything else. @@ -884,12 +885,12 @@ impl<'rt, 'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> ValueVisitor<'mir, 'tcx, M> // For some errors we might be able to provide extra information. // (This custom logic does not fit the `try_validation!` macro.) match err.kind() { - err_ub!(InvalidUninitBytes(Some(access))) => { + err_ub!(InvalidUninitBytes(Some((_alloc_id, access)))) => { // Some byte was uninitialized, determine which // element that byte belongs to so we can // provide an index. let i = usize::try_from( - access.uninit_ptr.offset.bytes() / layout.size.bytes(), + access.uninit_offset.bytes() / layout.size.bytes(), ) .unwrap(); self.path.push(PathElem::ArrayElem(i)); diff --git a/compiler/rustc_mir/src/lib.rs b/compiler/rustc_mir/src/lib.rs index f9adc27116..1da17bddcb 100644 --- a/compiler/rustc_mir/src/lib.rs +++ b/compiler/rustc_mir/src/lib.rs @@ -7,26 +7,26 @@ Rust MIR: a lowered representation of Rust. #![feature(nll)] #![feature(in_band_lifetimes)] #![feature(array_windows)] +#![feature(assert_matches)] #![feature(bindings_after_at)] #![feature(bool_to_option)] #![feature(box_patterns)] #![feature(box_syntax)] -#![feature(const_panic)] #![feature(crate_visibility_modifier)] #![feature(decl_macro)] #![feature(exact_size_is_empty)] -#![feature(exhaustive_patterns)] +#![feature(format_args_capture)] #![feature(iter_zip)] #![feature(never_type)] #![feature(map_try_insert)] #![feature(min_specialization)] +#![feature(slice_ptr_get)] #![feature(trusted_len)] #![feature(try_blocks)] #![feature(associated_type_defaults)] #![feature(stmt_expr_attributes)] #![feature(trait_alias)] #![feature(option_get_or_insert_default)] -#![cfg_attr(bootstrap, feature(or_patterns))] #![feature(once_cell)] #![feature(control_flow_enum)] #![recursion_limit = "256"] diff --git a/compiler/rustc_mir/src/monomorphize/collector.rs b/compiler/rustc_mir/src/monomorphize/collector.rs index fdefc89067..31cb5484bc 100644 --- a/compiler/rustc_mir/src/monomorphize/collector.rs +++ b/compiler/rustc_mir/src/monomorphize/collector.rs @@ -194,6 +194,7 @@ use rustc_middle::mir::mono::{InstantiationMode, MonoItem}; use rustc_middle::mir::visit::Visitor as MirVisitor; use rustc_middle::mir::{self, Local, Location}; use rustc_middle::ty::adjustment::{CustomCoerceUnsized, PointerCast}; +use rustc_middle::ty::print::with_no_trimmed_paths; use rustc_middle::ty::subst::{GenericArgKind, InternalSubsts}; use rustc_middle::ty::{self, GenericParamDefKind, Instance, Ty, TyCtxt, TypeFoldable}; use rustc_middle::{middle::codegen_fn_attrs::CodegenFnAttrFlags, mir::visit::TyContext}; @@ -322,7 +323,7 @@ fn collect_roots(tcx: TyCtxt<'_>, mode: MonoItemCollectionMode) -> Vec, mode: MonoItemCollectionMode) -> Vec( tcx: TyCtxt<'tcx>, starting_point: Spanned>, @@ -359,6 +361,31 @@ fn collect_items_rec<'tcx>( let mut neighbors = Vec::new(); let recursion_depth_reset; + // + // Post-monomorphization errors MVP + // + // We can encounter errors while monomorphizing an item, but we don't have a good way of + // showing a complete stack of spans ultimately leading to collecting the erroneous one yet. + // (It's also currently unclear exactly which diagnostics and information would be interesting + // to report in such cases) + // + // This leads to suboptimal error reporting: a post-monomorphization error (PME) will be + // shown with just a spanned piece of code causing the error, without information on where + // it was called from. This is especially obscure if the erroneous mono item is in a + // dependency. See for example issue #85155, where, before minimization, a PME happened two + // crates downstream from libcore's stdarch, without a way to know which dependency was the + // cause. + // + // If such an error occurs in the current crate, its span will be enough to locate the + // source. If the cause is in another crate, the goal here is to quickly locate which mono + // item in the current crate is ultimately responsible for causing the error. + // + // To give at least _some_ context to the user: while collecting mono items, we check the + // error count. If it has changed, a PME occurred, and we trigger some diagnostics about the + // current step of mono items collection. + // + let error_count = tcx.sess.diagnostic().err_count(); + match starting_point.node { MonoItem::Static(def_id) => { let instance = Instance::mono(tcx, def_id); @@ -390,11 +417,41 @@ fn collect_items_rec<'tcx>( collect_neighbours(tcx, instance, &mut neighbors); }); } - MonoItem::GlobalAsm(..) => { + MonoItem::GlobalAsm(item_id) => { recursion_depth_reset = None; + + let item = tcx.hir().item(item_id); + if let hir::ItemKind::GlobalAsm(asm) = item.kind { + for (op, op_sp) in asm.operands { + match op { + hir::InlineAsmOperand::Const { .. } => { + // Only constants which resolve to a plain integer + // are supported. Therefore the value should not + // depend on any other items. + } + _ => span_bug!(*op_sp, "invalid operand type for global_asm!"), + } + } + } else { + span_bug!(item.span, "Mismatch between hir::Item type and MonoItem type") + } } } + // Check for PMEs and emit a diagnostic if one happened. To try to show relevant edges of the + // mono item graph where the PME diagnostics are currently the most problematic (e.g. ones + // involving a dependency, and the lack of context is confusing) in this MVP, we focus on + // diagnostics on edges crossing a crate boundary: the collected mono items which are not + // defined in the local crate. + if tcx.sess.diagnostic().err_count() > error_count && starting_point.node.krate() != LOCAL_CRATE + { + let formatted_item = with_no_trimmed_paths(|| starting_point.node.to_string()); + tcx.sess.span_note_without_error( + starting_point.span, + &format!("the above error was encountered while instantiating `{}`", formatted_item), + ); + } + record_accesses(tcx, starting_point.node, neighbors.iter().map(|i| &i.node), inlining_map); for neighbour in neighbors { @@ -452,7 +509,7 @@ fn shrunk_instance_name( after = &s[positions().rev().nth(after).unwrap_or(0)..], ); - let path = tcx.output_filenames(LOCAL_CRATE).temp_path_ext("long-type.txt", None); + let path = tcx.output_filenames(()).temp_path_ext("long-type.txt", None); let written_to_path = std::fs::write(&path, s).ok().map(|_| path); (shrunk, written_to_path) diff --git a/compiler/rustc_mir/src/monomorphize/partitioning/default.rs b/compiler/rustc_mir/src/monomorphize/partitioning/default.rs index edd46310f2..541c825416 100644 --- a/compiler/rustc_mir/src/monomorphize/partitioning/default.rs +++ b/compiler/rustc_mir/src/monomorphize/partitioning/default.rs @@ -451,7 +451,9 @@ fn mono_item_visibility( let is_generic = instance.substs.non_erasable_generics().next().is_some(); // Upstream `DefId` instances get different handling than local ones. - if !def_id.is_local() { + let def_id = if let Some(def_id) = def_id.as_local() { + def_id + } else { return if export_generics && is_generic { // If it is a upstream monomorphization and we export generics, we must make // it available to downstream crates. @@ -460,7 +462,7 @@ fn mono_item_visibility( } else { Visibility::Hidden }; - } + }; if is_generic { if export_generics { @@ -470,7 +472,7 @@ fn mono_item_visibility( } else { // This instance might be useful in a downstream crate. *can_be_internalized = false; - default_visibility(tcx, def_id, true) + default_visibility(tcx, def_id.to_def_id(), true) } } else { // We are not exporting generics or the definition is not reachable @@ -481,10 +483,10 @@ fn mono_item_visibility( // If this isn't a generic function then we mark this a `Default` if // this is a reachable item, meaning that it's a symbol other crates may // access when they link to us. - if tcx.is_reachable_non_generic(def_id) { + if tcx.is_reachable_non_generic(def_id.to_def_id()) { *can_be_internalized = false; debug_assert!(!is_generic); - return default_visibility(tcx, def_id, false); + return default_visibility(tcx, def_id.to_def_id(), false); } // If this isn't reachable then we're gonna tag this with `Hidden` diff --git a/compiler/rustc_mir/src/monomorphize/partitioning/mod.rs b/compiler/rustc_mir/src/monomorphize/partitioning/mod.rs index dc2379fd92..6ed0ab8be4 100644 --- a/compiler/rustc_mir/src/monomorphize/partitioning/mod.rs +++ b/compiler/rustc_mir/src/monomorphize/partitioning/mod.rs @@ -97,7 +97,7 @@ mod merging; use rustc_data_structures::fx::{FxHashMap, FxHashSet}; use rustc_data_structures::sync; -use rustc_hir::def_id::{CrateNum, DefIdSet, LOCAL_CRATE}; +use rustc_hir::def_id::DefIdSet; use rustc_middle::mir::mono::MonoItem; use rustc_middle::mir::mono::{CodegenUnit, Linkage}; use rustc_middle::ty::print::with_no_trimmed_paths; @@ -311,10 +311,8 @@ where fn collect_and_partition_mono_items<'tcx>( tcx: TyCtxt<'tcx>, - cnum: CrateNum, + (): (), ) -> (&'tcx DefIdSet, &'tcx [CodegenUnit<'tcx>]) { - assert_eq!(cnum, LOCAL_CRATE); - let collection_mode = match tcx.sess.opts.debugging_opts.print_mono_items { Some(ref s) => { let mode_string = s.to_lowercase(); @@ -350,12 +348,14 @@ fn collect_and_partition_mono_items<'tcx>( let (codegen_units, _) = tcx.sess.time("partition_and_assert_distinct_symbols", || { sync::join( || { - &*tcx.arena.alloc_from_iter(partition( + let mut codegen_units = partition( tcx, &mut items.iter().cloned(), tcx.sess.codegen_units(), &inlining_map, - )) + ); + codegen_units[0].make_primary(); + &*tcx.arena.alloc_from_iter(codegen_units) }, || assert_symbols_are_distinct(tcx, items.iter()), ) @@ -424,8 +424,8 @@ fn collect_and_partition_mono_items<'tcx>( (tcx.arena.alloc(mono_items), codegen_units) } -fn codegened_and_inlined_items<'tcx>(tcx: TyCtxt<'tcx>, cnum: CrateNum) -> &'tcx DefIdSet { - let (items, cgus) = tcx.collect_and_partition_mono_items(cnum); +fn codegened_and_inlined_items<'tcx>(tcx: TyCtxt<'tcx>, (): ()) -> &'tcx DefIdSet { + let (items, cgus) = tcx.collect_and_partition_mono_items(()); let mut visited = DefIdSet::default(); let mut result = items.clone(); @@ -453,12 +453,12 @@ pub fn provide(providers: &mut Providers) { providers.codegened_and_inlined_items = codegened_and_inlined_items; providers.is_codegened_item = |tcx, def_id| { - let (all_mono_items, _) = tcx.collect_and_partition_mono_items(LOCAL_CRATE); + let (all_mono_items, _) = tcx.collect_and_partition_mono_items(()); all_mono_items.contains(&def_id) }; providers.codegen_unit = |tcx, name| { - let (_, all) = tcx.collect_and_partition_mono_items(LOCAL_CRATE); + let (_, all) = tcx.collect_and_partition_mono_items(()); all.iter() .find(|cgu| cgu.name() == name) .unwrap_or_else(|| panic!("failed to find cgu with name {:?}", name)) diff --git a/compiler/rustc_mir/src/transform/check_consts/ops.rs b/compiler/rustc_mir/src/transform/check_consts/ops.rs index ffeaaf60a3..dcbc9c523d 100644 --- a/compiler/rustc_mir/src/transform/check_consts/ops.rs +++ b/compiler/rustc_mir/src/transform/check_consts/ops.rs @@ -141,12 +141,20 @@ impl NonConstOp for FnPtrCast { pub struct Generator(pub hir::GeneratorKind); impl NonConstOp for Generator { fn status_in_item(&self, _: &ConstCx<'_, '_>) -> Status { - Status::Forbidden + if let hir::GeneratorKind::Async(hir::AsyncGeneratorKind::Block) = self.0 { + Status::Unstable(sym::const_async_blocks) + } else { + Status::Forbidden + } } fn build_error(&self, ccx: &ConstCx<'_, 'tcx>, span: Span) -> DiagnosticBuilder<'tcx> { let msg = format!("{}s are not allowed in {}s", self.0, ccx.const_kind()); - ccx.tcx.sess.struct_span_err(span, &msg) + if let hir::GeneratorKind::Async(hir::AsyncGeneratorKind::Block) = self.0 { + feature_err(&ccx.tcx.sess.parse_sess, sym::const_async_blocks, span, &msg) + } else { + ccx.tcx.sess.struct_span_err(span, &msg) + } } } @@ -533,30 +541,6 @@ impl NonConstOp for UnionAccess { } } -/// See [#64992]. -/// -/// [#64992]: https://github.com/rust-lang/rust/issues/64992 -#[derive(Debug)] -pub struct UnsizingCast; -impl NonConstOp for UnsizingCast { - fn status_in_item(&self, ccx: &ConstCx<'_, '_>) -> Status { - if ccx.const_kind() != hir::ConstContext::ConstFn { - Status::Allowed - } else { - Status::Unstable(sym::const_fn_unsize) - } - } - - fn build_error(&self, ccx: &ConstCx<'_, 'tcx>, span: Span) -> DiagnosticBuilder<'tcx> { - feature_err( - &ccx.tcx.sess.parse_sess, - sym::const_fn_unsize, - span, - "unsizing casts to types besides slices are not allowed in const fn", - ) - } -} - // Types that cannot appear in the signature or locals of a `const fn`. pub mod ty { use super::*; diff --git a/compiler/rustc_mir/src/transform/check_consts/validation.rs b/compiler/rustc_mir/src/transform/check_consts/validation.rs index 63fc66f2b9..4fbd27c89d 100644 --- a/compiler/rustc_mir/src/transform/check_consts/validation.rs +++ b/compiler/rustc_mir/src/transform/check_consts/validation.rs @@ -10,9 +10,7 @@ use rustc_middle::mir::visit::{MutatingUseContext, NonMutatingUseContext, PlaceC use rustc_middle::mir::*; use rustc_middle::ty::cast::CastTy; use rustc_middle::ty::subst::GenericArgKind; -use rustc_middle::ty::{ - self, adjustment::PointerCast, Instance, InstanceDef, Ty, TyCtxt, TypeAndMut, -}; +use rustc_middle::ty::{self, adjustment::PointerCast, Instance, InstanceDef, Ty, TyCtxt}; use rustc_middle::ty::{Binder, TraitPredicate, TraitRef}; use rustc_span::{sym, Span, Symbol}; use rustc_trait_selection::traits::error_reporting::InferCtxtExt; @@ -358,10 +356,9 @@ impl Validator<'mir, 'tcx> { } fn check_static(&mut self, def_id: DefId, span: Span) { - assert!( - !self.tcx.is_thread_local_static(def_id), - "tls access is checked in `Rvalue::ThreadLocalRef" - ); + if self.tcx.is_thread_local_static(def_id) { + self.tcx.sess.delay_span_bug(span, "tls access is checked in `Rvalue::ThreadLocalRef"); + } self.check_op_spanned(ops::StaticAccess, span) } @@ -636,17 +633,9 @@ impl Visitor<'tcx> for Validator<'mir, 'tcx> { _, ) => self.check_op(ops::FnPtrCast), - Rvalue::Cast(CastKind::Pointer(PointerCast::Unsize), _, cast_ty) => { - if let Some(TypeAndMut { ty, .. }) = cast_ty.builtin_deref(true) { - let unsized_ty = self.tcx.struct_tail_erasing_lifetimes(ty, self.param_env); - - // Casting/coercing things to slices is fine. - if let ty::Slice(_) | ty::Str = unsized_ty.kind() { - return; - } - } - - self.check_op(ops::UnsizingCast); + Rvalue::Cast(CastKind::Pointer(PointerCast::Unsize), _, _) => { + // Nothing to check here (`check_local_or_return_ty` ensures no trait objects occur + // in the type of any local, which also excludes casts). } Rvalue::Cast(CastKind::Misc, ref operand, cast_ty) => { @@ -740,13 +729,11 @@ impl Visitor<'tcx> for Validator<'mir, 'tcx> { let base_ty = Place::ty_from(place_local, proj_base, self.body, self.tcx).ty; if let ty::RawPtr(_) = base_ty.kind() { if proj_base.is_empty() { - if let (local, []) = (place_local, proj_base) { - let decl = &self.body.local_decls[local]; - if let Some(box LocalInfo::StaticRef { def_id, .. }) = decl.local_info { - let span = decl.source_info.span; - self.check_static(def_id, span); - return; - } + let decl = &self.body.local_decls[place_local]; + if let Some(box LocalInfo::StaticRef { def_id, .. }) = decl.local_info { + let span = decl.source_info.span; + self.check_static(def_id, span); + return; } } self.check_op(ops::RawPtrDeref); @@ -763,12 +750,8 @@ impl Visitor<'tcx> for Validator<'mir, 'tcx> { | ProjectionElem::Field(..) | ProjectionElem::Index(_) => { let base_ty = Place::ty_from(place_local, proj_base, self.body, self.tcx).ty; - match base_ty.ty_adt_def() { - Some(def) if def.is_union() => { - self.check_op(ops::UnionAccess); - } - - _ => {} + if base_ty.is_union() { + self.check_op(ops::UnionAccess); } } } diff --git a/compiler/rustc_mir/src/transform/check_unsafety.rs b/compiler/rustc_mir/src/transform/check_unsafety.rs index 955be8cc81..103ddda1a1 100644 --- a/compiler/rustc_mir/src/transform/check_unsafety.rs +++ b/compiler/rustc_mir/src/transform/check_unsafety.rs @@ -221,7 +221,7 @@ impl<'a, 'tcx> Visitor<'tcx> for UnsafetyChecker<'a, 'tcx> { } let base_ty = base.ty(self.body, self.tcx).ty; - if base_ty.ty_adt_def().map_or(false, |adt| adt.is_union()) { + if base_ty.is_union() { // If we did not hit a `Deref` yet and the overall place use is an assignment, the // rules are different. let assign_to_field = !saw_deref @@ -376,6 +376,12 @@ impl<'a, 'tcx> UnsafetyChecker<'a, 'tcx> { /// Checks whether calling `func_did` needs an `unsafe` context or not, i.e. whether /// the called function has target features the calling function hasn't. fn check_target_features(&mut self, func_did: DefId) { + // Unsafety isn't required on wasm targets. For more information see + // the corresponding check in typeck/src/collect.rs + if self.tcx.sess.target.options.is_like_wasm { + return; + } + let callee_features = &self.tcx.codegen_fn_attrs(func_did).target_features; let self_features = &self.tcx.codegen_fn_attrs(self.body_did).target_features; diff --git a/compiler/rustc_mir/src/transform/const_goto.rs b/compiler/rustc_mir/src/transform/const_goto.rs index b5c8b4bebc..ba10b54c5a 100644 --- a/compiler/rustc_mir/src/transform/const_goto.rs +++ b/compiler/rustc_mir/src/transform/const_goto.rs @@ -47,7 +47,7 @@ impl<'tcx> MirPass<'tcx> for ConstGoto { // if we applied optimizations, we potentially have some cfg to cleanup to // make it easier for further passes if should_simplify { - simplify_cfg(body); + simplify_cfg(tcx, body); simplify_locals(body, tcx); } } diff --git a/compiler/rustc_mir/src/transform/const_prop.rs b/compiler/rustc_mir/src/transform/const_prop.rs index 5968bbbfca..681d63c6fc 100644 --- a/compiler/rustc_mir/src/transform/const_prop.rs +++ b/compiler/rustc_mir/src/transform/const_prop.rs @@ -33,6 +33,7 @@ use crate::interpret::{ self, compile_time_machine, AllocId, Allocation, ConstValue, CtfeValidationMode, Frame, ImmTy, Immediate, InterpCx, InterpResult, LocalState, LocalValue, MemPlace, Memory, MemoryKind, OpTy, Operand as InterpOperand, PlaceTy, Pointer, Scalar, ScalarMaybeUninit, StackPopCleanup, + StackPopUnwind, }; use crate::transform::MirPass; @@ -198,7 +199,7 @@ impl<'mir, 'tcx> interpret::Machine<'mir, 'tcx> for ConstPropMachine<'mir, 'tcx> _abi: Abi, _args: &[OpTy<'tcx>], _ret: Option<(&PlaceTy<'tcx>, BasicBlock)>, - _unwind: Option, + _unwind: StackPopUnwind, ) -> InterpResult<'tcx, Option<&'mir Body<'tcx>>> { Ok(None) } @@ -208,7 +209,7 @@ impl<'mir, 'tcx> interpret::Machine<'mir, 'tcx> for ConstPropMachine<'mir, 'tcx> _instance: ty::Instance<'tcx>, _args: &[OpTy<'tcx>], _ret: Option<(&PlaceTy<'tcx>, BasicBlock)>, - _unwind: Option, + _unwind: StackPopUnwind, ) -> InterpResult<'tcx> { throw_machine_stop_str!("calling intrinsics isn't supported in ConstProp") } diff --git a/compiler/rustc_mir/src/transform/coverage/debug.rs b/compiler/rustc_mir/src/transform/coverage/debug.rs index 4836148309..f6672335cb 100644 --- a/compiler/rustc_mir/src/transform/coverage/debug.rs +++ b/compiler/rustc_mir/src/transform/coverage/debug.rs @@ -116,10 +116,10 @@ use crate::util::pretty; use crate::util::spanview::{self, SpanViewable}; use rustc_data_structures::fx::FxHashMap; -use rustc_index::vec::Idx; use rustc_middle::mir::coverage::*; use rustc_middle::mir::{self, BasicBlock, TerminatorKind}; use rustc_middle::ty::TyCtxt; +use rustc_span::Span; use std::iter; use std::lazy::SyncOnceCell; @@ -636,6 +636,7 @@ pub(super) fn dump_coverage_spanview( mir_body: &mir::Body<'tcx>, basic_coverage_blocks: &CoverageGraph, pass_name: &str, + body_span: Span, coverage_spans: &Vec, ) { let mir_source = mir_body.source; @@ -647,7 +648,7 @@ pub(super) fn dump_coverage_spanview( let crate_name = tcx.crate_name(def_id.krate); let item_name = tcx.def_path(def_id).to_filename_friendly_no_crate(); let title = format!("{}.{} - Coverage Spans", crate_name, item_name); - spanview::write_document(tcx, def_id, span_viewables, &title, &mut file) + spanview::write_document(tcx, body_span, span_viewables, &title, &mut file) .expect("Unexpected IO error dumping coverage spans as HTML"); } diff --git a/compiler/rustc_mir/src/transform/coverage/graph.rs b/compiler/rustc_mir/src/transform/coverage/graph.rs index 6f5fa858e2..0521f507ec 100644 --- a/compiler/rustc_mir/src/transform/coverage/graph.rs +++ b/compiler/rustc_mir/src/transform/coverage/graph.rs @@ -269,13 +269,13 @@ impl graph::WithSuccessors for CoverageGraph { impl graph::GraphPredecessors<'graph> for CoverageGraph { type Item = BasicCoverageBlock; - type Iter = std::vec::IntoIter; + type Iter = std::iter::Copied>; } impl graph::WithPredecessors for CoverageGraph { #[inline] fn predecessors(&self, node: Self::Node) -> >::Iter { - self.predecessors[node].clone().into_iter() + self.predecessors[node].iter().copied() } } diff --git a/compiler/rustc_mir/src/transform/coverage/mod.rs b/compiler/rustc_mir/src/transform/coverage/mod.rs index eaeb44289c..71c244fdd4 100644 --- a/compiler/rustc_mir/src/transform/coverage/mod.rs +++ b/compiler/rustc_mir/src/transform/coverage/mod.rs @@ -32,7 +32,7 @@ use rustc_middle::mir::{ use rustc_middle::ty::TyCtxt; use rustc_span::def_id::DefId; use rustc_span::source_map::SourceMap; -use rustc_span::{CharPos, Pos, SourceFile, Span, Symbol}; +use rustc_span::{CharPos, ExpnKind, Pos, SourceFile, Span, Symbol}; /// A simple error message wrapper for `coverage::Error`s. #[derive(Debug)] @@ -95,7 +95,7 @@ impl<'tcx> MirPass<'tcx> for InstrumentCoverage { trace!("InstrumentCoverage starting for {:?}", mir_source.def_id()); Instrumentor::new(&self.name(), tcx, mir_body).inject_counters(); - trace!("InstrumentCoverage starting for {:?}", mir_source.def_id()); + trace!("InstrumentCoverage done for {:?}", mir_source.def_id()); } } @@ -113,8 +113,11 @@ struct Instrumentor<'a, 'tcx> { impl<'a, 'tcx> Instrumentor<'a, 'tcx> { fn new(pass_name: &'a str, tcx: TyCtxt<'tcx>, mir_body: &'a mut mir::Body<'tcx>) -> Self { let source_map = tcx.sess.source_map(); - let (some_fn_sig, hir_body) = fn_sig_and_body(tcx, mir_body.source.def_id()); - let body_span = hir_body.value.span; + let def_id = mir_body.source.def_id(); + let (some_fn_sig, hir_body) = fn_sig_and_body(tcx, def_id); + + let body_span = get_body_span(tcx, hir_body, mir_body); + let source_file = source_map.lookup_source_file(body_span.lo()); let fn_sig_span = match some_fn_sig.filter(|fn_sig| { fn_sig.span.ctxt() == body_span.ctxt() @@ -123,6 +126,15 @@ impl<'a, 'tcx> Instrumentor<'a, 'tcx> { Some(fn_sig) => fn_sig.span.with_hi(body_span.lo()), None => body_span.shrink_to_lo(), }; + + debug!( + "instrumenting {}: {:?}, fn sig span: {:?}, body span: {:?}", + if tcx.is_closure(def_id) { "closure" } else { "function" }, + def_id, + fn_sig_span, + body_span + ); + let function_source_hash = hash_mir_source(tcx, hir_body); let basic_coverage_blocks = CoverageGraph::from_mir(mir_body); Self { @@ -139,19 +151,11 @@ impl<'a, 'tcx> Instrumentor<'a, 'tcx> { fn inject_counters(&'a mut self) { let tcx = self.tcx; - let source_map = tcx.sess.source_map(); let mir_source = self.mir_body.source; let def_id = mir_source.def_id(); let fn_sig_span = self.fn_sig_span; let body_span = self.body_span; - debug!( - "instrumenting {:?}, fn sig span: {}, body span: {}", - def_id, - source_map.span_to_string(fn_sig_span), - source_map.span_to_string(body_span) - ); - let mut graphviz_data = debug::GraphvizData::new(); let mut debug_used_expressions = debug::UsedExpressions::new(); @@ -183,6 +187,7 @@ impl<'a, 'tcx> Instrumentor<'a, 'tcx> { self.mir_body, &self.basic_coverage_blocks, self.pass_name, + body_span, &coverage_spans, ); } @@ -290,7 +295,7 @@ impl<'a, 'tcx> Instrumentor<'a, 'tcx> { let tcx = self.tcx; let source_map = tcx.sess.source_map(); let body_span = self.body_span; - let file_name = Symbol::intern(&self.source_file.name.to_string()); + let file_name = Symbol::intern(&self.source_file.name.prefer_remapped().to_string_lossy()); let mut bcb_counters = IndexVec::from_elem_n(None, self.basic_coverage_blocks.num_nodes()); for covspan in coverage_spans { @@ -311,8 +316,8 @@ impl<'a, 'tcx> Instrumentor<'a, 'tcx> { "Calling make_code_region(file_name={}, source_file={:?}, span={}, body_span={})", file_name, self.source_file, - source_map.span_to_string(span), - source_map.span_to_string(body_span) + source_map.span_to_diagnostic_string(span), + source_map.span_to_diagnostic_string(body_span) ); inject_statement( @@ -539,6 +544,35 @@ fn fn_sig_and_body<'tcx>( (hir::map::fn_sig(hir_node), tcx.hir().body(fn_body_id)) } +fn get_body_span<'tcx>( + tcx: TyCtxt<'tcx>, + hir_body: &rustc_hir::Body<'tcx>, + mir_body: &mut mir::Body<'tcx>, +) -> Span { + let mut body_span = hir_body.value.span; + let def_id = mir_body.source.def_id(); + + if tcx.is_closure(def_id) { + // If the MIR function is a closure, and if the closure body span + // starts from a macro, but it's content is not in that macro, try + // to find a non-macro callsite, and instrument the spans there + // instead. + loop { + let expn_data = body_span.ctxt().outer_expn_data(); + if expn_data.is_root() { + break; + } + if let ExpnKind::Macro { .. } = expn_data.kind { + body_span = expn_data.call_site; + } else { + break; + } + } + } + + body_span +} + fn hash_mir_source<'tcx>(tcx: TyCtxt<'tcx>, hir_body: &'tcx rustc_hir::Body<'tcx>) -> u64 { let mut hcx = tcx.create_no_span_stable_hashing_context(); hash(&mut hcx, &hir_body.value).to_smaller_hash() diff --git a/compiler/rustc_mir/src/transform/coverage/query.rs b/compiler/rustc_mir/src/transform/coverage/query.rs index 2ba9d1bdc0..760f16eae6 100644 --- a/compiler/rustc_mir/src/transform/coverage/query.rs +++ b/compiler/rustc_mir/src/transform/coverage/query.rs @@ -120,8 +120,8 @@ impl CoverageVisitor { } } -fn coverageinfo<'tcx>(tcx: TyCtxt<'tcx>, def_id: DefId) -> CoverageInfo { - let mir_body = mir_body(tcx, def_id); +fn coverageinfo<'tcx>(tcx: TyCtxt<'tcx>, instance_def: ty::InstanceDef<'tcx>) -> CoverageInfo { + let mir_body = tcx.instance_mir(instance_def); let mut coverage_visitor = CoverageVisitor { // num_counters always has at least the `ZERO` counter. diff --git a/compiler/rustc_mir/src/transform/coverage/spans.rs b/compiler/rustc_mir/src/transform/coverage/spans.rs index 2041109eb3..f62171b3c5 100644 --- a/compiler/rustc_mir/src/transform/coverage/spans.rs +++ b/compiler/rustc_mir/src/transform/coverage/spans.rs @@ -11,8 +11,9 @@ use rustc_middle::mir::{ use rustc_middle::ty::TyCtxt; use rustc_span::source_map::original_sp; -use rustc_span::{BytePos, Span}; +use rustc_span::{BytePos, ExpnKind, MacroKind, Span, Symbol}; +use std::cell::RefCell; use std::cmp::Ordering; #[derive(Debug, Copy, Clone)] @@ -67,6 +68,8 @@ impl CoverageStatement { #[derive(Debug, Clone)] pub(super) struct CoverageSpan { pub span: Span, + pub expn_span: Span, + pub current_macro_or_none: RefCell>>, pub bcb: BasicCoverageBlock, pub coverage_statements: Vec, pub is_closure: bool, @@ -74,12 +77,20 @@ pub(super) struct CoverageSpan { impl CoverageSpan { pub fn for_fn_sig(fn_sig_span: Span) -> Self { - Self { span: fn_sig_span, bcb: START_BCB, coverage_statements: vec![], is_closure: false } + Self { + span: fn_sig_span, + expn_span: fn_sig_span, + current_macro_or_none: Default::default(), + bcb: START_BCB, + coverage_statements: vec![], + is_closure: false, + } } pub fn for_statement( statement: &Statement<'tcx>, span: Span, + expn_span: Span, bcb: BasicCoverageBlock, bb: BasicBlock, stmt_index: usize, @@ -94,15 +105,24 @@ impl CoverageSpan { Self { span, + expn_span, + current_macro_or_none: Default::default(), bcb, coverage_statements: vec![CoverageStatement::Statement(bb, span, stmt_index)], is_closure, } } - pub fn for_terminator(span: Span, bcb: BasicCoverageBlock, bb: BasicBlock) -> Self { + pub fn for_terminator( + span: Span, + expn_span: Span, + bcb: BasicCoverageBlock, + bb: BasicBlock, + ) -> Self { Self { span, + expn_span, + current_macro_or_none: Default::default(), bcb, coverage_statements: vec![CoverageStatement::Terminator(bb, span)], is_closure: false, @@ -158,6 +178,41 @@ impl CoverageSpan { .collect::>() .join("\n") } + + /// If the span is part of a macro, returns the macro name symbol. + pub fn current_macro(&self) -> Option { + self.current_macro_or_none + .borrow_mut() + .get_or_insert_with(|| { + if let ExpnKind::Macro { + kind: MacroKind::Bang, + name: current_macro, + proc_macro: _, + } = self.expn_span.ctxt().outer_expn_data().kind + { + return Some(current_macro); + } + None + }) + .map(|symbol| symbol) + } + + /// If the span is part of a macro, and the macro is visible (expands directly to the given + /// body_span), returns the macro name symbol. + pub fn visible_macro(&self, body_span: Span) -> Option { + if let Some(current_macro) = self.current_macro() { + if self.expn_span.parent().unwrap_or_else(|| bug!("macro must have a parent")).ctxt() + == body_span.ctxt() + { + return Some(current_macro); + } + } + None + } + + pub fn is_macro_expansion(&self) -> bool { + self.current_macro().is_some() + } } /// Converts the initial set of `CoverageSpan`s (one per MIR `Statement` or `Terminator`) into a @@ -191,16 +246,23 @@ pub struct CoverageSpans<'a, 'tcx> { /// iteration. some_curr: Option, - /// The original `span` for `curr`, in case the `curr` span is modified. + /// The original `span` for `curr`, in case `curr.span()` is modified. The `curr_original_span` + /// **must not be mutated** (except when advancing to the next `curr`), even if `curr.span()` + /// is mutated. curr_original_span: Span, /// The CoverageSpan from a prior iteration; typically assigned from that iteration's `curr`. /// If that `curr` was discarded, `prev` retains its value from the previous iteration. some_prev: Option, - /// Assigned from `curr_original_span` from the previous iteration. + /// Assigned from `curr_original_span` from the previous iteration. The `prev_original_span` + /// **must not be mutated** (except when advancing to the next `prev`), even if `prev.span()` + /// is mutated. prev_original_span: Span, + /// A copy of the expn_span from the prior iteration. + prev_expn_span: Option, + /// One or more `CoverageSpan`s with the same `Span` but different `BasicCoverageBlock`s, and /// no `BasicCoverageBlock` in this list dominates another `BasicCoverageBlock` in the list. /// If a new `curr` span also fits this criteria (compared to an existing list of @@ -255,15 +317,13 @@ impl<'a, 'tcx> CoverageSpans<'a, 'tcx> { curr_original_span: Span::with_root_ctxt(BytePos(0), BytePos(0)), some_prev: None, prev_original_span: Span::with_root_ctxt(BytePos(0), BytePos(0)), + prev_expn_span: None, pending_dups: Vec::new(), }; let sorted_spans = coverage_spans.mir_to_initial_sorted_coverage_spans(); coverage_spans.sorted_spans_iter = Some(sorted_spans.into_iter()); - coverage_spans.some_prev = coverage_spans.sorted_spans_iter.as_mut().unwrap().next(); - coverage_spans.prev_original_span = - coverage_spans.some_prev.as_ref().expect("at least one span").span; coverage_spans.to_refined_spans() } @@ -317,10 +377,14 @@ impl<'a, 'tcx> CoverageSpans<'a, 'tcx> { /// de-duplicated `CoverageSpan`s. fn to_refined_spans(mut self) -> Vec { while self.next_coverage_span() { - if self.curr().is_mergeable(self.prev()) { + if self.some_prev.is_none() { + debug!(" initial span"); + self.check_invoked_macro_name_span(); + } else if self.curr().is_mergeable(self.prev()) { debug!(" same bcb (and neither is a closure), merge with prev={:?}", self.prev()); let prev = self.take_prev(); self.curr_mut().merge_from(prev); + self.check_invoked_macro_name_span(); // Note that curr.span may now differ from curr_original_span } else if self.prev_ends_before_curr() { debug!( @@ -329,7 +393,8 @@ impl<'a, 'tcx> CoverageSpans<'a, 'tcx> { self.prev() ); let prev = self.take_prev(); - self.refined_spans.push(prev); + self.push_refined_span(prev); + self.check_invoked_macro_name_span(); } else if self.prev().is_closure { // drop any equal or overlapping span (`curr`) and keep `prev` to test again in the // next iter @@ -342,20 +407,45 @@ impl<'a, 'tcx> CoverageSpans<'a, 'tcx> { } else if self.curr().is_closure { self.carve_out_span_for_closure(); } else if self.prev_original_span == self.curr().span { - // Note that this compares the new span to `prev_original_span`, which may not - // be the full `prev.span` (if merged during the previous iteration). - self.hold_pending_dups_unless_dominated(); + // Note that this compares the new (`curr`) span to `prev_original_span`. + // In this branch, the actual span byte range of `prev_original_span` is not + // important. What is important is knowing whether the new `curr` span was + // **originally** the same as the original span of `prev()`. The original spans + // reflect their original sort order, and for equal spans, conveys a partial + // ordering based on CFG dominator priority. + if self.prev().is_macro_expansion() && self.curr().is_macro_expansion() { + // Macros that expand to include branching (such as + // `assert_eq!()`, `assert_ne!()`, `info!()`, `debug!()`, or + // `trace!()) typically generate callee spans with identical + // ranges (typically the full span of the macro) for all + // `BasicBlocks`. This makes it impossible to distinguish + // the condition (`if val1 != val2`) from the optional + // branched statements (such as the call to `panic!()` on + // assert failure). In this case it is better (or less + // worse) to drop the optional branch bcbs and keep the + // non-conditional statements, to count when reached. + debug!( + " curr and prev are part of a macro expansion, and curr has the same span \ + as prev, but is in a different bcb. Drop curr and keep prev for next iter. \ + prev={:?}", + self.prev() + ); + self.take_curr(); + } else { + self.hold_pending_dups_unless_dominated(); + } } else { self.cutoff_prev_at_overlapping_curr(); + self.check_invoked_macro_name_span(); } } debug!(" AT END, adding last prev={:?}", self.prev()); let prev = self.take_prev(); - let CoverageSpans { pending_dups, mut refined_spans, .. } = self; + let pending_dups = self.pending_dups.split_off(0); for dup in pending_dups { debug!(" ...adding at least one pending dup={:?}", dup); - refined_spans.push(dup); + self.push_refined_span(dup); } // Async functions wrap a closure that implements the body to be executed. The enclosing @@ -365,21 +455,60 @@ impl<'a, 'tcx> CoverageSpans<'a, 'tcx> { // excluded. The closure's `Return` is the only one that will be counted. This provides // adequate coverage, and more intuitive counts. (Avoids double-counting the closing brace // of the function body.) - let body_ends_with_closure = if let Some(last_covspan) = refined_spans.last() { + let body_ends_with_closure = if let Some(last_covspan) = self.refined_spans.last() { last_covspan.is_closure && last_covspan.span.hi() == self.body_span.hi() } else { false }; if !body_ends_with_closure { - refined_spans.push(prev); + self.push_refined_span(prev); } // Remove `CoverageSpan`s derived from closures, originally added to ensure the coverage // regions for the current function leave room for the closure's own coverage regions // (injected separately, from the closure's own MIR). - refined_spans.retain(|covspan| !covspan.is_closure); - refined_spans + self.refined_spans.retain(|covspan| !covspan.is_closure); + self.refined_spans + } + + fn push_refined_span(&mut self, covspan: CoverageSpan) { + let len = self.refined_spans.len(); + if len > 0 { + let last = &mut self.refined_spans[len - 1]; + if last.is_mergeable(&covspan) { + debug!( + "merging new refined span with last refined span, last={:?}, covspan={:?}", + last, covspan + ); + last.merge_from(covspan); + return; + } + } + self.refined_spans.push(covspan) + } + + fn check_invoked_macro_name_span(&mut self) { + if let Some(visible_macro) = self.curr().visible_macro(self.body_span) { + if self.prev_expn_span.map_or(true, |prev_expn_span| { + self.curr().expn_span.ctxt() != prev_expn_span.ctxt() + }) { + let merged_prefix_len = self.curr_original_span.lo() - self.curr().span.lo(); + let after_macro_bang = + merged_prefix_len + BytePos(visible_macro.as_str().bytes().count() as u32 + 1); + let mut macro_name_cov = self.curr().clone(); + self.curr_mut().span = + self.curr().span.with_lo(self.curr().span.lo() + after_macro_bang); + macro_name_cov.span = + macro_name_cov.span.with_hi(macro_name_cov.span.lo() + after_macro_bang); + debug!( + " and curr starts a new macro expansion, so add a new span just for \ + the macro `{}!`, new span={:?}", + visible_macro, macro_name_cov + ); + self.push_refined_span(macro_name_cov); + } + } } // Generate a set of `CoverageSpan`s from the filtered set of `Statement`s and `Terminator`s of @@ -401,14 +530,25 @@ impl<'a, 'tcx> CoverageSpans<'a, 'tcx> { .iter() .enumerate() .filter_map(move |(index, statement)| { - filtered_statement_span(statement, self.body_span).map(|span| { - CoverageSpan::for_statement(statement, span, bcb, bb, index) + filtered_statement_span(statement).map(|span| { + CoverageSpan::for_statement( + statement, + function_source_span(span, self.body_span), + span, + bcb, + bb, + index, + ) }) }) - .chain( - filtered_terminator_span(data.terminator(), self.body_span) - .map(|span| CoverageSpan::for_terminator(span, bcb, bb)), - ) + .chain(filtered_terminator_span(data.terminator()).map(|span| { + CoverageSpan::for_terminator( + function_source_span(span, self.body_span), + span, + bcb, + bb, + ) + })) }) .collect() } @@ -461,7 +601,7 @@ impl<'a, 'tcx> CoverageSpans<'a, 'tcx> { let pending_dups = self.pending_dups.split_off(0); for dup in pending_dups.into_iter() { debug!(" ...adding at least one pending={:?}", dup); - self.refined_spans.push(dup); + self.push_refined_span(dup); } } else { self.pending_dups.clear(); @@ -473,12 +613,13 @@ impl<'a, 'tcx> CoverageSpans<'a, 'tcx> { /// Advance `prev` to `curr` (if any), and `curr` to the next `CoverageSpan` in sorted order. fn next_coverage_span(&mut self) -> bool { if let Some(curr) = self.some_curr.take() { + self.prev_expn_span = Some(curr.expn_span); self.some_prev = Some(curr); self.prev_original_span = self.curr_original_span; } while let Some(curr) = self.sorted_spans_iter.as_mut().unwrap().next() { debug!("FOR curr={:?}", curr); - if self.prev_starts_after_next(&curr) { + if self.some_prev.is_some() && self.prev_starts_after_next(&curr) { debug!( " prev.span starts after curr.span, so curr will be dropped (skipping past \ closure?); prev={:?}", @@ -535,29 +676,38 @@ impl<'a, 'tcx> CoverageSpans<'a, 'tcx> { for mut dup in pending_dups.iter().cloned() { dup.span = dup.span.with_hi(left_cutoff); debug!(" ...and at least one pre_closure dup={:?}", dup); - self.refined_spans.push(dup); + self.push_refined_span(dup); } } - self.refined_spans.push(pre_closure); + self.push_refined_span(pre_closure); } if has_post_closure_span { - // Update prev.span to start after the closure (and discard curr) + // Mutate `prev.span()` to start after the closure (and discard curr). + // (**NEVER** update `prev_original_span` because it affects the assumptions + // about how the `CoverageSpan`s are ordered.) self.prev_mut().span = self.prev().span.with_lo(right_cutoff); - self.prev_original_span = self.prev().span; + debug!(" Mutated prev.span to start after the closure. prev={:?}", self.prev()); for dup in pending_dups.iter_mut() { + debug!(" ...and at least one overlapping dup={:?}", dup); dup.span = dup.span.with_lo(right_cutoff); } self.pending_dups.append(&mut pending_dups); let closure_covspan = self.take_curr(); - self.refined_spans.push(closure_covspan); // since self.prev() was already updated + self.push_refined_span(closure_covspan); // since self.prev() was already updated } else { pending_dups.clear(); } } /// Called if `curr.span` equals `prev_original_span` (and potentially equal to all - /// `pending_dups` spans, if any); but keep in mind, `prev.span` may start at a `Span.lo()` that - /// is less than (further left of) `prev_original_span.lo()`. + /// `pending_dups` spans, if any). Keep in mind, `prev.span()` may have been changed. + /// If prev.span() was merged into other spans (with matching BCB, for instance), + /// `prev.span.hi()` will be greater than (further right of) `prev_original_span.hi()`. + /// If prev.span() was split off to the right of a closure, prev.span().lo() will be + /// greater than prev_original_span.lo(). The actual span of `prev_original_span` is + /// not as important as knowing that `prev()` **used to have the same span** as `curr(), + /// which means their sort order is still meaningful for determinating the dominator + /// relationship. /// /// When two `CoverageSpan`s have the same `Span`, dominated spans can be discarded; but if /// neither `CoverageSpan` dominates the other, both (or possibly more than two) are held, @@ -640,7 +790,7 @@ impl<'a, 'tcx> CoverageSpans<'a, 'tcx> { } else { debug!(" ... adding modified prev={:?}", self.prev()); let prev = self.take_prev(); - self.refined_spans.push(prev); + self.push_refined_span(prev); } } else { // with `pending_dups`, `prev` cannot have any statements that don't overlap @@ -653,10 +803,9 @@ impl<'a, 'tcx> CoverageSpans<'a, 'tcx> { } } -pub(super) fn filtered_statement_span( - statement: &'a Statement<'tcx>, - body_span: Span, -) -> Option { +/// If the MIR `Statement` has a span contributive to computing coverage spans, +/// return it; otherwise return `None`. +pub(super) fn filtered_statement_span(statement: &'a Statement<'tcx>) -> Option { match statement.kind { // These statements have spans that are often outside the scope of the executed source code // for their parent `BasicBlock`. @@ -693,15 +842,14 @@ pub(super) fn filtered_statement_span( | StatementKind::LlvmInlineAsm(_) | StatementKind::Retag(_, _) | StatementKind::AscribeUserType(_, _) => { - Some(function_source_span(statement.source_info.span, body_span)) + Some(statement.source_info.span) } } } -pub(super) fn filtered_terminator_span( - terminator: &'a Terminator<'tcx>, - body_span: Span, -) -> Option { +/// If the MIR `Terminator` has a span contributive to computing coverage spans, +/// return it; otherwise return `None`. +pub(super) fn filtered_terminator_span(terminator: &'a Terminator<'tcx>) -> Option { match terminator.kind { // These terminators have spans that don't positively contribute to computing a reasonable // span of actually executed source code. (For example, SwitchInt terminators extracted from @@ -725,7 +873,7 @@ pub(super) fn filtered_terminator_span( span = span.with_lo(constant.span.lo()); } } - Some(function_source_span(span, body_span)) + Some(span) } // Retain spans from all other terminators @@ -736,13 +884,20 @@ pub(super) fn filtered_terminator_span( | TerminatorKind::GeneratorDrop | TerminatorKind::FalseUnwind { .. } | TerminatorKind::InlineAsm { .. } => { - Some(function_source_span(terminator.source_info.span, body_span)) + Some(terminator.source_info.span) } } } +/// Returns an extrapolated span (pre-expansion[^1]) corresponding to a range +/// within the function's body source. This span is guaranteed to be contained +/// within, or equal to, the `body_span`. If the extrapolated span is not +/// contained within the `body_span`, the `body_span` is returned. +/// +/// [^1]Expansions result from Rust syntax including macros, syntactic sugar, +/// etc.). #[inline] -fn function_source_span(span: Span, body_span: Span) -> Span { - let span = original_sp(span, body_span).with_ctxt(body_span.ctxt()); - if body_span.contains(span) { span } else { body_span } +pub(super) fn function_source_span(span: Span, body_span: Span) -> Span { + let original_span = original_sp(span, body_span).with_ctxt(body_span.ctxt()); + if body_span.contains(original_span) { original_span } else { body_span } } diff --git a/compiler/rustc_mir/src/transform/coverage/tests.rs b/compiler/rustc_mir/src/transform/coverage/tests.rs index dee112443d..b04c2d542d 100644 --- a/compiler/rustc_mir/src/transform/coverage/tests.rs +++ b/compiler/rustc_mir/src/transform/coverage/tests.rs @@ -1,6 +1,10 @@ //! This crate hosts a selection of "unit tests" for components of the `InstrumentCoverage` MIR //! pass. //! +//! ```shell +//! ./x.py test --keep-stage 1 compiler/rustc_mir --test-args '--show-output coverage' +//! ``` +//! //! The tests construct a few "mock" objects, as needed, to support the `InstrumentCoverage` //! functions and algorithms. Mocked objects include instances of `mir::Body`; including //! `Terminator`s of various `kind`s, and `Span` objects. Some functions used by or used on @@ -679,10 +683,13 @@ fn test_make_bcb_counters() { let mut basic_coverage_blocks = graph::CoverageGraph::from_mir(&mir_body); let mut coverage_spans = Vec::new(); for (bcb, data) in basic_coverage_blocks.iter_enumerated() { - if let Some(span) = - spans::filtered_terminator_span(data.terminator(&mir_body), body_span) - { - coverage_spans.push(spans::CoverageSpan::for_terminator(span, bcb, data.last_bb())); + if let Some(span) = spans::filtered_terminator_span(data.terminator(&mir_body)) { + coverage_spans.push(spans::CoverageSpan::for_terminator( + spans::function_source_span(span, body_span), + span, + bcb, + data.last_bb(), + )); } } let mut coverage_counters = counters::CoverageCounters::new(0); diff --git a/compiler/rustc_mir/src/transform/deduplicate_blocks.rs b/compiler/rustc_mir/src/transform/deduplicate_blocks.rs index c41e71e09a..912505c659 100644 --- a/compiler/rustc_mir/src/transform/deduplicate_blocks.rs +++ b/compiler/rustc_mir/src/transform/deduplicate_blocks.rs @@ -26,7 +26,7 @@ impl<'tcx> MirPass<'tcx> for DeduplicateBlocks { if has_opts_to_apply { let mut opt_applier = OptApplier { tcx, duplicates }; opt_applier.visit_body(body); - simplify_cfg(body); + simplify_cfg(tcx, body); } } } diff --git a/compiler/rustc_mir/src/transform/dest_prop.rs b/compiler/rustc_mir/src/transform/dest_prop.rs index 29df86ca6c..4f5a467a6e 100644 --- a/compiler/rustc_mir/src/transform/dest_prop.rs +++ b/compiler/rustc_mir/src/transform/dest_prop.rs @@ -114,7 +114,7 @@ use rustc_middle::mir::{ traversal, Body, InlineAsmOperand, Local, LocalKind, Location, Operand, Place, PlaceElem, Rvalue, Statement, StatementKind, Terminator, TerminatorKind, }; -use rustc_middle::ty::{self, Ty, TyCtxt}; +use rustc_middle::ty::TyCtxt; // Empirical measurements have resulted in some observations: // - Running on a body with a single block and 500 locals takes barely any time @@ -910,17 +910,8 @@ impl<'a, 'tcx> Visitor<'tcx> for FindAssignments<'a, 'tcx> { // Handle the "subtle case" described above by rejecting any `dest` that is or // projects through a union. - let is_union = |ty: Ty<'_>| { - if let ty::Adt(def, _) = ty.kind() { - if def.is_union() { - return true; - } - } - - false - }; let mut place_ty = PlaceTy::from_ty(self.body.local_decls[dest.local].ty); - if is_union(place_ty.ty) { + if place_ty.ty.is_union() { return; } for elem in dest.projection { @@ -930,7 +921,7 @@ impl<'a, 'tcx> Visitor<'tcx> for FindAssignments<'a, 'tcx> { } place_ty = place_ty.projection_ty(self.tcx, elem); - if is_union(place_ty.ty) { + if place_ty.ty.is_union() { return; } } diff --git a/compiler/rustc_mir/src/transform/early_otherwise_branch.rs b/compiler/rustc_mir/src/transform/early_otherwise_branch.rs index f7ea9faec4..07127042fa 100644 --- a/compiler/rustc_mir/src/transform/early_otherwise_branch.rs +++ b/compiler/rustc_mir/src/transform/early_otherwise_branch.rs @@ -164,16 +164,13 @@ impl<'tcx> MirPass<'tcx> for EarlyOtherwiseBranch { // Since this optimization adds new basic blocks and invalidates others, // clean up the cfg to make it nicer for other passes if should_cleanup { - simplify_cfg(body); + simplify_cfg(tcx, body); } } } fn is_switch<'tcx>(terminator: &Terminator<'tcx>) -> bool { - match terminator.kind { - TerminatorKind::SwitchInt { .. } => true, - _ => false, - } + matches!(terminator.kind, TerminatorKind::SwitchInt { .. }) } struct Helper<'a, 'tcx> { diff --git a/compiler/rustc_mir/src/transform/generator.rs b/compiler/rustc_mir/src/transform/generator.rs index 003003a8ab..3560b4b1e8 100644 --- a/compiler/rustc_mir/src/transform/generator.rs +++ b/compiler/rustc_mir/src/transform/generator.rs @@ -964,7 +964,7 @@ fn create_generator_drop_shim<'tcx>( // Make sure we remove dead blocks to remove // unrelated code from the resume part of the function - simplify::remove_dead_blocks(&mut body); + simplify::remove_dead_blocks(tcx, &mut body); dump_mir(tcx, None, "generator_drop", &0, &body, |_, _| Ok(())); @@ -1137,7 +1137,7 @@ fn create_generator_resume_function<'tcx>( // Make sure we remove dead blocks to remove // unrelated code from the drop part of the function - simplify::remove_dead_blocks(body); + simplify::remove_dead_blocks(tcx, body); dump_mir(tcx, None, "generator_resume", &0, body, |_, _| Ok(())); } diff --git a/compiler/rustc_mir/src/transform/inline.rs b/compiler/rustc_mir/src/transform/inline.rs index b6f80763bc..f1c95a84ad 100644 --- a/compiler/rustc_mir/src/transform/inline.rs +++ b/compiler/rustc_mir/src/transform/inline.rs @@ -57,7 +57,7 @@ impl<'tcx> MirPass<'tcx> for Inline { if inline(tcx, body) { debug!("running simplify cfg on {:?}", body.source); CfgSimplifier::new(body).simplify(); - remove_dead_blocks(body); + remove_dead_blocks(tcx, body); } } } diff --git a/compiler/rustc_mir/src/transform/instcombine.rs b/compiler/rustc_mir/src/transform/instcombine.rs index 7aaf022416..b64189a7f3 100644 --- a/compiler/rustc_mir/src/transform/instcombine.rs +++ b/compiler/rustc_mir/src/transform/instcombine.rs @@ -4,7 +4,7 @@ use crate::transform::MirPass; use rustc_hir::Mutability; use rustc_middle::mir::{ BinOp, Body, Constant, LocalDecls, Operand, Place, ProjectionElem, Rvalue, SourceInfo, - StatementKind, + StatementKind, UnOp, }; use rustc_middle::ty::{self, TyCtxt}; @@ -47,28 +47,35 @@ impl<'tcx, 'a> InstCombineContext<'tcx, 'a> { Rvalue::BinaryOp(op @ (BinOp::Eq | BinOp::Ne), box (a, b)) => { let new = match (op, self.try_eval_bool(a), self.try_eval_bool(b)) { // Transform "Eq(a, true)" ==> "a" - (BinOp::Eq, _, Some(true)) => Some(a.clone()), + (BinOp::Eq, _, Some(true)) => Some(Rvalue::Use(a.clone())), // Transform "Ne(a, false)" ==> "a" - (BinOp::Ne, _, Some(false)) => Some(a.clone()), + (BinOp::Ne, _, Some(false)) => Some(Rvalue::Use(a.clone())), // Transform "Eq(true, b)" ==> "b" - (BinOp::Eq, Some(true), _) => Some(b.clone()), + (BinOp::Eq, Some(true), _) => Some(Rvalue::Use(b.clone())), // Transform "Ne(false, b)" ==> "b" - (BinOp::Ne, Some(false), _) => Some(b.clone()), + (BinOp::Ne, Some(false), _) => Some(Rvalue::Use(b.clone())), - // FIXME: Consider combining remaining comparisons into logical operations: // Transform "Eq(false, b)" ==> "Not(b)" + (BinOp::Eq, Some(false), _) => Some(Rvalue::UnaryOp(UnOp::Not, b.clone())), + // Transform "Ne(true, b)" ==> "Not(b)" + (BinOp::Ne, Some(true), _) => Some(Rvalue::UnaryOp(UnOp::Not, b.clone())), + // Transform "Eq(a, false)" ==> "Not(a)" + (BinOp::Eq, _, Some(false)) => Some(Rvalue::UnaryOp(UnOp::Not, a.clone())), + // Transform "Ne(a, true)" ==> "Not(a)" + (BinOp::Ne, _, Some(true)) => Some(Rvalue::UnaryOp(UnOp::Not, a.clone())), + _ => None, }; if let Some(new) = new { if self.should_combine(source_info, rvalue) { - *rvalue = Rvalue::Use(new); + *rvalue = new; } } } diff --git a/compiler/rustc_mir/src/transform/lower_intrinsics.rs b/compiler/rustc_mir/src/transform/lower_intrinsics.rs index e6ee474285..6d7e4cdb1c 100644 --- a/compiler/rustc_mir/src/transform/lower_intrinsics.rs +++ b/compiler/rustc_mir/src/transform/lower_intrinsics.rs @@ -5,6 +5,7 @@ use rustc_middle::mir::*; use rustc_middle::ty::subst::SubstsRef; use rustc_middle::ty::{self, Ty, TyCtxt}; use rustc_span::symbol::{sym, Symbol}; +use rustc_span::Span; use rustc_target::spec::abi::Abi; pub struct LowerIntrinsics; @@ -119,6 +120,9 @@ impl<'tcx> MirPass<'tcx> for LowerIntrinsics { terminator.kind = TerminatorKind::Goto { target }; } } + _ if intrinsic_name.as_str().starts_with("simd_shuffle") => { + validate_simd_shuffle(tcx, args, terminator.source_info.span); + } _ => {} } } @@ -132,9 +136,19 @@ fn resolve_rust_intrinsic( ) -> Option<(Symbol, SubstsRef<'tcx>)> { if let ty::FnDef(def_id, substs) = *func_ty.kind() { let fn_sig = func_ty.fn_sig(tcx); - if fn_sig.abi() == Abi::RustIntrinsic { + if let Abi::RustIntrinsic | Abi::PlatformIntrinsic = fn_sig.abi() { return Some((tcx.item_name(def_id), substs)); } } None } + +fn validate_simd_shuffle(tcx: TyCtxt<'tcx>, args: &[Operand<'tcx>], span: Span) { + match &args[2] { + Operand::Constant(_) => {} // all good + _ => { + let msg = format!("last argument of `simd_shuffle` is required to be a `const` item"); + tcx.sess.span_err(span, &msg); + } + } +} diff --git a/compiler/rustc_mir/src/transform/match_branches.rs b/compiler/rustc_mir/src/transform/match_branches.rs index f7a9835353..21b208a08c 100644 --- a/compiler/rustc_mir/src/transform/match_branches.rs +++ b/compiler/rustc_mir/src/transform/match_branches.rs @@ -167,7 +167,7 @@ impl<'tcx> MirPass<'tcx> for MatchBranchSimplification { } if should_cleanup { - simplify_cfg(body); + simplify_cfg(tcx, body); } } } diff --git a/compiler/rustc_mir/src/transform/mod.rs b/compiler/rustc_mir/src/transform/mod.rs index 5c49ee69ed..2201223e13 100644 --- a/compiler/rustc_mir/src/transform/mod.rs +++ b/compiler/rustc_mir/src/transform/mod.rs @@ -3,7 +3,7 @@ use required_consts::RequiredConstsVisitor; use rustc_data_structures::fx::FxHashSet; use rustc_data_structures::steal::Steal; use rustc_hir as hir; -use rustc_hir::def_id::{CrateNum, DefId, LocalDefId, LOCAL_CRATE}; +use rustc_hir::def_id::{DefId, LocalDefId}; use rustc_hir::intravisit::{self, NestedVisitorMap, Visitor}; use rustc_index::vec::IndexVec; use rustc_middle::mir::visit::Visitor as _; @@ -98,14 +98,13 @@ pub(crate) fn provide(providers: &mut Providers) { } fn is_mir_available(tcx: TyCtxt<'_>, def_id: DefId) -> bool { - tcx.mir_keys(def_id.krate).contains(&def_id.expect_local()) + let def_id = def_id.expect_local(); + tcx.mir_keys(()).contains(&def_id) } /// Finds the full set of `DefId`s within the current crate that have /// MIR associated with them. -fn mir_keys(tcx: TyCtxt<'_>, krate: CrateNum) -> FxHashSet { - assert_eq!(krate, LOCAL_CRATE); - +fn mir_keys(tcx: TyCtxt<'_>, (): ()) -> FxHashSet { let mut set = FxHashSet::default(); // All body-owners have MIR associated with them. diff --git a/compiler/rustc_mir/src/transform/multiple_return_terminators.rs b/compiler/rustc_mir/src/transform/multiple_return_terminators.rs index 4aaa0baa9f..cd2db18055 100644 --- a/compiler/rustc_mir/src/transform/multiple_return_terminators.rs +++ b/compiler/rustc_mir/src/transform/multiple_return_terminators.rs @@ -38,6 +38,6 @@ impl<'tcx> MirPass<'tcx> for MultipleReturnTerminators { } } - simplify::remove_dead_blocks(body) + simplify::remove_dead_blocks(tcx, body) } } diff --git a/compiler/rustc_mir/src/transform/promote_consts.rs b/compiler/rustc_mir/src/transform/promote_consts.rs index 1bbaf833c4..78e84419c6 100644 --- a/compiler/rustc_mir/src/transform/promote_consts.rs +++ b/compiler/rustc_mir/src/transform/promote_consts.rs @@ -12,20 +12,16 @@ //! initialization and can otherwise silence errors, if //! move analysis runs after promotion on broken MIR. -use rustc_ast::LitKind; use rustc_hir as hir; -use rustc_hir::def_id::DefId; use rustc_middle::mir::traversal::ReversePostorder; use rustc_middle::mir::visit::{MutVisitor, MutatingUseContext, PlaceContext, Visitor}; use rustc_middle::mir::*; use rustc_middle::ty::cast::CastTy; use rustc_middle::ty::subst::InternalSubsts; use rustc_middle::ty::{self, List, TyCtxt, TypeFoldable}; -use rustc_span::symbol::sym; use rustc_span::Span; use rustc_index::vec::{Idx, IndexVec}; -use rustc_target::spec::abi::Abi; use std::cell::Cell; use std::{cmp, iter, mem}; @@ -36,8 +32,8 @@ use crate::transform::MirPass; /// A `MirPass` for promotion. /// -/// Promotion is the extraction of promotable temps into separate MIR bodies. This pass also emits -/// errors when promotion of `#[rustc_args_required_const]` arguments fails. +/// Promotion is the extraction of promotable temps into separate MIR bodies so they can have +/// `'static` lifetime. /// /// After this pass is run, `promoted_fragments` will hold the MIR body corresponding to each /// newly created `Constant`. @@ -101,47 +97,16 @@ impl TempState { pub enum Candidate { /// Borrow of a constant temporary, candidate for lifetime extension. Ref(Location), - - /// Currently applied to function calls where the callee has the unstable - /// `#[rustc_args_required_const]` attribute as well as the SIMD shuffle - /// intrinsic. The intrinsic requires the arguments are indeed constant and - /// the attribute currently provides the semantic requirement that arguments - /// must be constant. - Argument { bb: BasicBlock, index: usize }, } impl Candidate { - /// Returns `true` if we should use the "explicit" rules for promotability for this `Candidate`. - fn forces_explicit_promotion(&self) -> bool { - match self { - Candidate::Ref(_) => false, - Candidate::Argument { .. } => true, - } - } - fn source_info(&self, body: &Body<'_>) -> SourceInfo { match self { Candidate::Ref(location) => *body.source_info(*location), - Candidate::Argument { bb, .. } => *body.source_info(body.terminator_loc(*bb)), } } } -fn args_required_const(tcx: TyCtxt<'_>, def_id: DefId) -> Option> { - let attrs = tcx.get_attrs(def_id); - let attr = attrs.iter().find(|a| tcx.sess.check_name(a, sym::rustc_args_required_const))?; - let mut ret = vec![]; - for meta in attr.meta_item_list()? { - match meta.literal()?.kind { - LitKind::Int(a, _) => { - ret.push(a as usize); - } - _ => bug!("invalid arg index"), - } - } - Some(ret) -} - struct Collector<'a, 'tcx> { ccx: &'a ConstCx<'a, 'tcx>, temps: IndexVec, @@ -208,31 +173,6 @@ impl<'tcx> Visitor<'tcx> for Collector<'_, 'tcx> { _ => {} } } - - fn visit_terminator(&mut self, terminator: &Terminator<'tcx>, location: Location) { - self.super_terminator(terminator, location); - - if let TerminatorKind::Call { ref func, .. } = terminator.kind { - if let ty::FnDef(def_id, _) = *func.ty(self.ccx.body, self.ccx.tcx).kind() { - let fn_sig = self.ccx.tcx.fn_sig(def_id); - if let Abi::RustIntrinsic | Abi::PlatformIntrinsic = fn_sig.abi() { - let name = self.ccx.tcx.item_name(def_id); - // FIXME(eddyb) use `#[rustc_args_required_const(2)]` for shuffles. - if name.as_str().starts_with("simd_shuffle") { - self.candidates.push(Candidate::Argument { bb: location.block, index: 2 }); - - return; // Don't double count `simd_shuffle` candidates - } - } - - if let Some(constant_args) = args_required_const(self.ccx.tcx, def_id) { - for index in constant_args { - self.candidates.push(Candidate::Argument { bb: location.block, index }); - } - } - } - } - } } pub fn collect_temps_and_candidates( @@ -256,14 +196,6 @@ pub fn collect_temps_and_candidates( struct Validator<'a, 'tcx> { ccx: &'a ConstCx<'a, 'tcx>, temps: &'a IndexVec, - - /// Explicit promotion happens e.g. for constant arguments declared via - /// `rustc_args_required_const`. - /// Implicit promotion has almost the same rules, except that disallows `const fn` - /// except for those marked `#[rustc_promotable]`. This is to avoid changing - /// a legitimate run-time operation into a failing compile-time operation - /// e.g. due to addresses being compared inside the function. - explicit: bool, } impl std::ops::Deref for Validator<'a, 'tcx> { @@ -280,8 +212,6 @@ impl<'tcx> Validator<'_, 'tcx> { fn validate_candidate(&self, candidate: Candidate) -> Result<(), Unpromotable> { match candidate { Candidate::Ref(loc) => { - assert!(!self.explicit); - let statement = &self.body[loc.block].statements[loc.statement_index]; match &statement.kind { StatementKind::Assign(box (_, Rvalue::Ref(_, kind, place))) => { @@ -310,15 +240,6 @@ impl<'tcx> Validator<'_, 'tcx> { _ => bug!(), } } - Candidate::Argument { bb, index } => { - assert!(self.explicit); - - let terminator = self.body[bb].terminator(); - match &terminator.kind { - TerminatorKind::Call { args, .. } => self.validate_operand(&args[index]), - _ => bug!(), - } - } } } @@ -448,12 +369,10 @@ impl<'tcx> Validator<'_, 'tcx> { ProjectionElem::ConstantIndex { .. } | ProjectionElem::Subslice { .. } => {} ProjectionElem::Index(local) => { - if !self.explicit { - let mut promotable = false; - // Only accept if we can predict the index and are indexing an array. - let val = if let TempState::Defined { location: loc, .. } = - self.temps[local] - { + let mut promotable = false; + // Only accept if we can predict the index and are indexing an array. + let val = + if let TempState::Defined { location: loc, .. } = self.temps[local] { let block = &self.body[loc.block]; if loc.statement_index < block.statements.len() { let statement = &block.statements[loc.statement_index]; @@ -470,38 +389,35 @@ impl<'tcx> Validator<'_, 'tcx> { } else { None }; - if let Some(idx) = val { - // Determine the type of the thing we are indexing. - let ty = place_base.ty(self.body, self.tcx).ty; - match ty.kind() { - ty::Array(_, len) => { - // It's an array; determine its length. - if let Some(len) = - len.try_eval_usize(self.tcx, self.param_env) - { - // If the index is in-bounds, go ahead. - if idx < len { - promotable = true; - } + if let Some(idx) = val { + // Determine the type of the thing we are indexing. + let ty = place_base.ty(self.body, self.tcx).ty; + match ty.kind() { + ty::Array(_, len) => { + // It's an array; determine its length. + if let Some(len) = len.try_eval_usize(self.tcx, self.param_env) + { + // If the index is in-bounds, go ahead. + if idx < len { + promotable = true; } } - _ => {} } - } - if !promotable { - return Err(Unpromotable); + _ => {} } } + if !promotable { + return Err(Unpromotable); + } + self.validate_local(local)?; } ProjectionElem::Field(..) => { let base_ty = place_base.ty(self.body, self.tcx).ty; - if let Some(def) = base_ty.ty_adt_def() { + if base_ty.is_union() { // No promotion of union field accesses. - if def.is_union() { - return Err(Unpromotable); - } + return Err(Unpromotable); } } } @@ -636,7 +552,7 @@ impl<'tcx> Validator<'_, 'tcx> { match op { BinOp::Div | BinOp::Rem => { - if !self.explicit && lhs_ty.is_integral() { + if lhs_ty.is_integral() { // Integer division: the RHS must be a non-zero const. let const_val = match rhs { Operand::Constant(c) => { @@ -721,13 +637,12 @@ impl<'tcx> Validator<'_, 'tcx> { ) -> Result<(), Unpromotable> { let fn_ty = callee.ty(self.body, self.tcx); - // When doing explicit promotion and inside const/static items, we promote all (eligible) function calls. + // Inside const/static items, we promote all (eligible) function calls. // Everywhere else, we require `#[rustc_promotable]` on the callee. - let promote_all_const_fn = self.explicit - || matches!( - self.const_kind, - Some(hir::ConstContext::Static(_) | hir::ConstContext::Const) - ); + let promote_all_const_fn = matches!( + self.const_kind, + Some(hir::ConstContext::Static(_) | hir::ConstContext::Const) + ); if !promote_all_const_fn { if let ty::FnDef(def_id, _) = *fn_ty.kind() { // Never promote runtime `const fn` calls of @@ -765,41 +680,12 @@ pub fn validate_candidates( temps: &IndexVec, candidates: &[Candidate], ) -> Vec { - let mut validator = Validator { ccx, temps, explicit: false }; + let validator = Validator { ccx, temps }; candidates .iter() .copied() - .filter(|&candidate| { - validator.explicit = candidate.forces_explicit_promotion(); - - // FIXME(eddyb) also emit the errors for shuffle indices - // and `#[rustc_args_required_const]` arguments here. - - let is_promotable = validator.validate_candidate(candidate).is_ok(); - - // If we use explicit validation, we carry the risk of turning a legitimate run-time - // operation into a failing compile-time operation. Make sure that does not happen - // by asserting that there is no possible run-time behavior here in case promotion - // fails. - if validator.explicit && !is_promotable { - ccx.tcx.sess.delay_span_bug( - ccx.body.span, - "Explicit promotion requested, but failed to promote", - ); - } - - match candidate { - Candidate::Argument { bb, index } if !is_promotable => { - let span = ccx.body[bb].terminator().source_info.span; - let msg = format!("argument {} is required to be a constant", index + 1); - ccx.tcx.sess.span_err(span, &msg); - } - _ => (), - } - - is_promotable - }) + .filter(|&candidate| validator.validate_candidate(candidate).is_ok()) .collect() } @@ -1039,26 +925,6 @@ impl<'a, 'tcx> Promoter<'a, 'tcx> { _ => bug!(), } } - Candidate::Argument { bb, index } => { - let terminator = blocks[bb].terminator_mut(); - match terminator.kind { - TerminatorKind::Call { ref mut args, .. } => { - let ty = args[index].ty(local_decls, self.tcx); - let span = terminator.source_info.span; - - Rvalue::Use(mem::replace(&mut args[index], promoted_operand(ty, span))) - } - // We expected a `TerminatorKind::Call` for which we'd like to promote an - // argument. `qualify_consts` saw a `TerminatorKind::Call` here, but - // we are seeing a `Goto`. That means that the `promote_temps` method - // already promoted this call away entirely. This case occurs when calling - // a function requiring a constant argument and as that constant value - // providing a value whose computation contains another call to a function - // requiring a constant argument. - TerminatorKind::Goto { .. } => return None, - _ => bug!(), - } - } } }; @@ -1113,7 +979,6 @@ pub fn promote_candidates<'tcx>( } } } - Candidate::Argument { .. } => {} } // Declare return place local so that `mir::Body::new` doesn't complain. diff --git a/compiler/rustc_mir/src/transform/remove_unneeded_drops.rs b/compiler/rustc_mir/src/transform/remove_unneeded_drops.rs index 5144d48750..02e45021a0 100644 --- a/compiler/rustc_mir/src/transform/remove_unneeded_drops.rs +++ b/compiler/rustc_mir/src/transform/remove_unneeded_drops.rs @@ -36,7 +36,7 @@ impl<'tcx> MirPass<'tcx> for RemoveUnneededDrops { // if we applied optimizations, we potentially have some cfg to cleanup to // make it easier for further passes if should_simplify { - simplify_cfg(body); + simplify_cfg(tcx, body); } } } diff --git a/compiler/rustc_mir/src/transform/remove_zsts.rs b/compiler/rustc_mir/src/transform/remove_zsts.rs index 70f7538dd5..40b1a8a2da 100644 --- a/compiler/rustc_mir/src/transform/remove_zsts.rs +++ b/compiler/rustc_mir/src/transform/remove_zsts.rs @@ -16,32 +16,29 @@ impl<'tcx> MirPass<'tcx> for RemoveZsts { let (basic_blocks, local_decls) = body.basic_blocks_and_local_decls_mut(); for block in basic_blocks.iter_mut() { for statement in block.statements.iter_mut() { - match statement.kind { - StatementKind::Assign(box (place, _)) => { - let place_ty = place.ty(local_decls, tcx).ty; - if !maybe_zst(place_ty) { - continue; - } - let layout = match tcx.layout_of(param_env.and(place_ty)) { - Ok(layout) => layout, - Err(_) => continue, - }; - if !layout.is_zst() { - continue; - } - if involves_a_union(place, local_decls, tcx) { - continue; - } - if tcx.consider_optimizing(|| { - format!( - "RemoveZsts - Place: {:?} SourceInfo: {:?}", - place, statement.source_info - ) - }) { - statement.make_nop(); - } + if let StatementKind::Assign(box (place, _)) = statement.kind { + let place_ty = place.ty(local_decls, tcx).ty; + if !maybe_zst(place_ty) { + continue; + } + let layout = match tcx.layout_of(param_env.and(place_ty)) { + Ok(layout) => layout, + Err(_) => continue, + }; + if !layout.is_zst() { + continue; + } + if involves_a_union(place, local_decls, tcx) { + continue; + } + if tcx.consider_optimizing(|| { + format!( + "RemoveZsts - Place: {:?} SourceInfo: {:?}", + place, statement.source_info + ) + }) { + statement.make_nop(); } - _ => {} } } } @@ -69,21 +66,14 @@ fn involves_a_union<'tcx>( tcx: TyCtxt<'tcx>, ) -> bool { let mut place_ty = PlaceTy::from_ty(local_decls[place.local].ty); - if is_union(place_ty.ty) { + if place_ty.ty.is_union() { return true; } for elem in place.projection { place_ty = place_ty.projection_ty(tcx, elem); - if is_union(place_ty.ty) { + if place_ty.ty.is_union() { return true; } } return false; } - -fn is_union(ty: Ty<'_>) -> bool { - match ty.kind() { - ty::Adt(def, _) if def.is_union() => true, - _ => false, - } -} diff --git a/compiler/rustc_mir/src/transform/simplify.rs b/compiler/rustc_mir/src/transform/simplify.rs index 65e2d096b2..7aebca77e6 100644 --- a/compiler/rustc_mir/src/transform/simplify.rs +++ b/compiler/rustc_mir/src/transform/simplify.rs @@ -29,6 +29,7 @@ use crate::transform::MirPass; use rustc_index::vec::{Idx, IndexVec}; +use rustc_middle::mir::coverage::*; use rustc_middle::mir::visit::{MutVisitor, MutatingUseContext, PlaceContext, Visitor}; use rustc_middle::mir::*; use rustc_middle::ty::TyCtxt; @@ -46,9 +47,9 @@ impl SimplifyCfg { } } -pub fn simplify_cfg(body: &mut Body<'_>) { +pub fn simplify_cfg(tcx: TyCtxt<'tcx>, body: &mut Body<'_>) { CfgSimplifier::new(body).simplify(); - remove_dead_blocks(body); + remove_dead_blocks(tcx, body); // FIXME: Should probably be moved into some kind of pass manager body.basic_blocks_mut().raw.shrink_to_fit(); @@ -59,9 +60,9 @@ impl<'tcx> MirPass<'tcx> for SimplifyCfg { Cow::Borrowed(&self.label) } - fn run_pass(&self, _tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { + fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { debug!("SimplifyCfg({:?}) - simplifying {:?}", self.label, body.source); - simplify_cfg(body); + simplify_cfg(tcx, body); } } @@ -286,7 +287,7 @@ impl<'a, 'tcx> CfgSimplifier<'a, 'tcx> { } } -pub fn remove_dead_blocks(body: &mut Body<'_>) { +pub fn remove_dead_blocks(tcx: TyCtxt<'tcx>, body: &mut Body<'_>) { let reachable = traversal::reachable_as_bitset(body); let num_blocks = body.basic_blocks().len(); if num_blocks == reachable.count() { @@ -306,6 +307,11 @@ pub fn remove_dead_blocks(body: &mut Body<'_>) { } used_blocks += 1; } + + if tcx.sess.instrument_coverage() { + save_unreachable_coverage(basic_blocks, used_blocks); + } + basic_blocks.raw.truncate(used_blocks); for block in basic_blocks { @@ -315,6 +321,75 @@ pub fn remove_dead_blocks(body: &mut Body<'_>) { } } +/// Some MIR transforms can determine at compile time that a sequences of +/// statements will never be executed, so they can be dropped from the MIR. +/// For example, an `if` or `else` block that is guaranteed to never be executed +/// because its condition can be evaluated at compile time, such as by const +/// evaluation: `if false { ... }`. +/// +/// Those statements are bypassed by redirecting paths in the CFG around the +/// `dead blocks`; but with `-Z instrument-coverage`, the dead blocks usually +/// include `Coverage` statements representing the Rust source code regions to +/// be counted at runtime. Without these `Coverage` statements, the regions are +/// lost, and the Rust source code will show no coverage information. +/// +/// What we want to show in a coverage report is the dead code with coverage +/// counts of `0`. To do this, we need to save the code regions, by injecting +/// `Unreachable` coverage statements. These are non-executable statements whose +/// code regions are still recorded in the coverage map, representing regions +/// with `0` executions. +fn save_unreachable_coverage( + basic_blocks: &mut IndexVec>, + first_dead_block: usize, +) { + let has_live_counters = basic_blocks.raw[0..first_dead_block].iter().any(|live_block| { + live_block.statements.iter().any(|statement| { + if let StatementKind::Coverage(coverage) = &statement.kind { + matches!(coverage.kind, CoverageKind::Counter { .. }) + } else { + false + } + }) + }); + if !has_live_counters { + // If there are no live `Counter` `Coverage` statements anymore, don't + // move dead coverage to the `START_BLOCK`. Just allow the dead + // `Coverage` statements to be dropped with the dead blocks. + // + // The `generator::StateTransform` MIR pass can create atypical + // conditions, where all live `Counter`s are dropped from the MIR. + // + // At least one Counter per function is required by LLVM (and necessary, + // to add the `function_hash` to the counter's call to the LLVM + // intrinsic `instrprof.increment()`). + return; + } + + // Retain coverage info for dead blocks, so coverage reports will still + // report `0` executions for the uncovered code regions. + let mut dropped_coverage = Vec::new(); + for dead_block in basic_blocks.raw[first_dead_block..].iter() { + for statement in dead_block.statements.iter() { + if let StatementKind::Coverage(coverage) = &statement.kind { + if let Some(code_region) = &coverage.code_region { + dropped_coverage.push((statement.source_info, code_region.clone())); + } + } + } + } + + let start_block = &mut basic_blocks[START_BLOCK]; + for (source_info, code_region) in dropped_coverage { + start_block.statements.push(Statement { + source_info, + kind: StatementKind::Coverage(box Coverage { + kind: CoverageKind::Unreachable, + code_region: Some(code_region), + }), + }) + } +} + pub struct SimplifyLocals; impl<'tcx> MirPass<'tcx> for SimplifyLocals { diff --git a/compiler/rustc_mir/src/transform/simplify_try.rs b/compiler/rustc_mir/src/transform/simplify_try.rs index b42543c04e..dd2ec39c06 100644 --- a/compiler/rustc_mir/src/transform/simplify_try.rs +++ b/compiler/rustc_mir/src/transform/simplify_try.rs @@ -558,7 +558,7 @@ impl<'tcx> MirPass<'tcx> for SimplifyBranchSame { if did_remove_blocks { // We have dead blocks now, so remove those. - simplify::remove_dead_blocks(body); + simplify::remove_dead_blocks(tcx, body); } } } @@ -628,10 +628,7 @@ impl<'a, 'tcx> SimplifyBranchSameOptimizationFinder<'a, 'tcx> { // But `asm!(...)` could abort the program, // so we cannot assume that the `unreachable` terminator itself is reachable. // FIXME(Centril): use a normalization pass instead of a check. - || bb.statements.iter().any(|stmt| match stmt.kind { - StatementKind::LlvmInlineAsm(..) => true, - _ => false, - }) + || bb.statements.iter().any(|stmt| matches!(stmt.kind, StatementKind::LlvmInlineAsm(..))) }) .peekable(); diff --git a/compiler/rustc_mir/src/transform/unreachable_prop.rs b/compiler/rustc_mir/src/transform/unreachable_prop.rs index 658c6b6e9d..e7fb6b4f6b 100644 --- a/compiler/rustc_mir/src/transform/unreachable_prop.rs +++ b/compiler/rustc_mir/src/transform/unreachable_prop.rs @@ -60,7 +60,7 @@ impl MirPass<'_> for UnreachablePropagation { } if replaced { - simplify::remove_dead_blocks(body); + simplify::remove_dead_blocks(tcx, body); } } } diff --git a/compiler/rustc_mir/src/transform/validate.rs b/compiler/rustc_mir/src/transform/validate.rs index d009b0b1b2..835789069b 100644 --- a/compiler/rustc_mir/src/transform/validate.rs +++ b/compiler/rustc_mir/src/transform/validate.rs @@ -11,8 +11,9 @@ use rustc_middle::mir::interpret::Scalar; use rustc_middle::mir::traversal; use rustc_middle::mir::visit::{PlaceContext, Visitor}; use rustc_middle::mir::{ - AggregateKind, BasicBlock, Body, BorrowKind, Local, Location, MirPhase, Operand, PlaceRef, - Rvalue, SourceScope, Statement, StatementKind, Terminator, TerminatorKind, + AggregateKind, BasicBlock, Body, BorrowKind, Local, Location, MirPhase, Operand, PlaceElem, + PlaceRef, ProjectionElem, Rvalue, SourceScope, Statement, StatementKind, Terminator, + TerminatorKind, }; use rustc_middle::ty::fold::BottomUpFolder; use rustc_middle::ty::{self, ParamEnv, Ty, TyCtxt, TypeFoldable}; @@ -217,6 +218,23 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> { self.super_operand(operand, location); } + fn visit_projection_elem( + &mut self, + local: Local, + proj_base: &[PlaceElem<'tcx>], + elem: PlaceElem<'tcx>, + context: PlaceContext, + location: Location, + ) { + if let ProjectionElem::Index(index) = elem { + let index_ty = self.body.local_decls[index].ty; + if index_ty != self.tcx.types.usize { + self.fail(location, format!("bad index ({:?} != usize)", index_ty)) + } + } + self.super_projection_elem(local, proj_base, elem, context, location); + } + fn visit_statement(&mut self, statement: &Statement<'tcx>, location: Location) { match &statement.kind { StatementKind::Assign(box (dest, rvalue)) => { diff --git a/compiler/rustc_mir/src/util/generic_graph.rs b/compiler/rustc_mir/src/util/generic_graph.rs index 6ce305a482..770b52a4d4 100644 --- a/compiler/rustc_mir/src/util/generic_graph.rs +++ b/compiler/rustc_mir/src/util/generic_graph.rs @@ -1,6 +1,5 @@ use gsgdt::{Edge, Graph, Node, NodeStyle}; use rustc_hir::def_id::DefId; -use rustc_index::vec::Idx; use rustc_middle::mir::*; use rustc_middle::ty::TyCtxt; diff --git a/compiler/rustc_mir/src/util/pretty.rs b/compiler/rustc_mir/src/util/pretty.rs index 3b88aec16b..c4e1e184ac 100644 --- a/compiler/rustc_mir/src/util/pretty.rs +++ b/compiler/rustc_mir/src/util/pretty.rs @@ -10,7 +10,7 @@ use super::spanview::write_mir_fn_spanview; use crate::transform::MirSource; use either::Either; use rustc_data_structures::fx::FxHashMap; -use rustc_hir::def_id::{DefId, LOCAL_CRATE}; +use rustc_hir::def_id::DefId; use rustc_index::vec::Idx; use rustc_middle::mir::interpret::{ read_target_uint, AllocId, Allocation, ConstValue, GlobalAlloc, Pointer, @@ -445,7 +445,10 @@ impl Visitor<'tcx> for ExtraComments<'tcx> { ty::Tuple(tys) if tys.is_empty() => {} _ => { self.push("mir::Constant"); - self.push(&format!("+ span: {}", self.tcx.sess.source_map().span_to_string(*span))); + self.push(&format!( + "+ span: {}", + self.tcx.sess.source_map().span_to_embeddable_string(*span) + )); if let Some(user_ty) = user_ty { self.push(&format!("+ user_ty: {:?}", user_ty)); } @@ -516,7 +519,7 @@ impl Visitor<'tcx> for ExtraComments<'tcx> { } fn comment(tcx: TyCtxt<'_>, SourceInfo { span, scope }: SourceInfo) -> String { - format!("scope {} at {}", scope.index(), tcx.sess.source_map().span_to_string(span)) + format!("scope {} at {}", scope.index(), tcx.sess.source_map().span_to_embeddable_string(span)) } /// Prints local variables in a scope tree. @@ -617,7 +620,7 @@ fn write_scope_tree( "{0:1$} // at {2}", indented_header, ALIGN, - tcx.sess.source_map().span_to_string(span), + tcx.sess.source_map().span_to_embeddable_string(span), )?; } else { writeln!(w, "{}", indented_header)?; @@ -773,8 +776,8 @@ pub struct RenderAllocation<'a, 'tcx, Tag, Extra> { impl std::fmt::Display for RenderAllocation<'a, 'tcx, Tag, Extra> { fn fmt(&self, w: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { let RenderAllocation { tcx, alloc } = *self; - write!(w, "size: {}, align: {})", alloc.size.bytes(), alloc.align.bytes())?; - if alloc.size == Size::ZERO { + write!(w, "size: {}, align: {})", alloc.size().bytes(), alloc.align.bytes())?; + if alloc.size() == Size::ZERO { // We are done. return write!(w, " {{}}"); } @@ -819,9 +822,9 @@ fn write_allocation_bytes( w: &mut dyn std::fmt::Write, prefix: &str, ) -> std::fmt::Result { - let num_lines = alloc.size.bytes_usize().saturating_sub(BYTES_PER_LINE); + let num_lines = alloc.size().bytes_usize().saturating_sub(BYTES_PER_LINE); // Number of chars needed to represent all line numbers. - let pos_width = format!("{:x}", alloc.size.bytes()).len(); + let pos_width = format!("{:x}", alloc.size().bytes()).len(); if num_lines > 0 { write!(w, "{}0x{:02$x} │ ", prefix, 0, pos_width)?; @@ -842,7 +845,7 @@ fn write_allocation_bytes( } }; - while i < alloc.size { + while i < alloc.size() { // The line start already has a space. While we could remove that space from the line start // printing and unconditionally print a space here, that would cause the single-line case // to have a single space before it, which looks weird. @@ -926,7 +929,7 @@ fn write_allocation_bytes( i += Size::from_bytes(1); } // Print a new line header if the next line still has some bytes to print. - if i == line_start + Size::from_bytes(BYTES_PER_LINE) && i != alloc.size { + if i == line_start + Size::from_bytes(BYTES_PER_LINE) && i != alloc.size() { line_start = write_allocation_newline(w, line_start, &ascii, pos_width, prefix)?; ascii.clear(); } @@ -1004,7 +1007,7 @@ fn write_user_type_annotations( "| {:?}: {:?} at {}", index.index(), annotation.user_ty, - tcx.sess.source_map().span_to_string(annotation.span) + tcx.sess.source_map().span_to_embeddable_string(annotation.span) )?; } if !body.user_type_annotations.is_empty() { @@ -1017,6 +1020,6 @@ pub fn dump_mir_def_ids(tcx: TyCtxt<'_>, single: Option) -> Vec { if let Some(i) = single { vec![i] } else { - tcx.mir_keys(LOCAL_CRATE).iter().map(|def_id| def_id.to_def_id()).collect() + tcx.mir_keys(()).iter().map(|def_id| def_id.to_def_id()).collect() } } diff --git a/compiler/rustc_mir/src/util/spanview.rs b/compiler/rustc_mir/src/util/spanview.rs index a9a30e407b..42683dac42 100644 --- a/compiler/rustc_mir/src/util/spanview.rs +++ b/compiler/rustc_mir/src/util/spanview.rs @@ -99,7 +99,11 @@ where W: Write, { let def_id = body.source.def_id(); - let body_span = hir_body(tcx, def_id).value.span; + let hir_body = hir_body(tcx, def_id); + if hir_body.is_none() { + return Ok(()); + } + let body_span = hir_body.unwrap().value.span; let mut span_viewables = Vec::new(); for (bb, data) in body.basic_blocks().iter_enumerated() { match spanview { @@ -127,7 +131,7 @@ where } } } - write_document(tcx, def_id, span_viewables, title, w)?; + write_document(tcx, fn_span(tcx, def_id), span_viewables, title, w)?; Ok(()) } @@ -135,7 +139,7 @@ where /// list `SpanViewable`s. pub fn write_document<'tcx, W>( tcx: TyCtxt<'tcx>, - def_id: DefId, + spanview_span: Span, mut span_viewables: Vec, title: &str, w: &mut W, @@ -143,16 +147,16 @@ pub fn write_document<'tcx, W>( where W: Write, { - let fn_span = fn_span(tcx, def_id); - let mut from_pos = fn_span.lo(); - let end_pos = fn_span.hi(); + let mut from_pos = spanview_span.lo(); + let end_pos = spanview_span.hi(); let source_map = tcx.sess.source_map(); let start = source_map.lookup_char_pos(from_pos); let indent_to_initial_start_col = " ".repeat(start.col.to_usize()); debug!( - "fn_span source is:\n{}{}", + "spanview_span={:?}; source is:\n{}{}", + spanview_span, indent_to_initial_start_col, - source_map.span_to_snippet(fn_span).expect("function should have printable source") + source_map.span_to_snippet(spanview_span).expect("function should have printable source") ); writeln!(w, "{}", HEADER)?; writeln!(w, "{}", title)?; @@ -624,7 +628,7 @@ fn tooltip<'tcx>( ) -> String { let source_map = tcx.sess.source_map(); let mut text = Vec::new(); - text.push(format!("{}: {}:", spanview_id, &source_map.span_to_string(span))); + text.push(format!("{}: {}:", spanview_id, &source_map.span_to_embeddable_string(span))); for statement in statements { let source_range = source_range_no_file(tcx, &statement.source_info.span); text.push(format!( @@ -664,19 +668,16 @@ fn fn_span<'tcx>(tcx: TyCtxt<'tcx>, def_id: DefId) -> Span { let hir_id = tcx.hir().local_def_id_to_hir_id(def_id.as_local().expect("expected DefId is local")); let fn_decl_span = tcx.hir().span(hir_id); - let body_span = hir_body(tcx, def_id).value.span; - if fn_decl_span.ctxt() == body_span.ctxt() { - fn_decl_span.to(body_span) + if let Some(body_span) = hir_body(tcx, def_id).map(|hir_body| hir_body.value.span) { + if fn_decl_span.ctxt() == body_span.ctxt() { fn_decl_span.to(body_span) } else { body_span } } else { - // This probably occurs for functions defined via macros - body_span + fn_decl_span } } -fn hir_body<'tcx>(tcx: TyCtxt<'tcx>, def_id: DefId) -> &'tcx rustc_hir::Body<'tcx> { +fn hir_body<'tcx>(tcx: TyCtxt<'tcx>, def_id: DefId) -> Option<&'tcx rustc_hir::Body<'tcx>> { let hir_node = tcx.hir().get_if_local(def_id).expect("expected DefId is local"); - let fn_body_id = hir::map::associated_body(hir_node).expect("HIR node is a function with body"); - tcx.hir().body(fn_body_id) + hir::map::associated_body(hir_node).map(|fn_body_id| tcx.hir().body(fn_body_id)) } fn escape_html(s: &str) -> String { diff --git a/compiler/rustc_mir_build/src/build/block.rs b/compiler/rustc_mir_build/src/build/block.rs index 808c6e3ff6..8426b24270 100644 --- a/compiler/rustc_mir_build/src/build/block.rs +++ b/compiler/rustc_mir_build/src/build/block.rs @@ -1,8 +1,8 @@ use crate::build::matches::ArmHasGuard; use crate::build::ForGuard::OutsideGuard; use crate::build::{BlockAnd, BlockAndExtension, BlockFrame, Builder}; -use crate::thir::*; use rustc_middle::mir::*; +use rustc_middle::thir::*; use rustc_session::lint::builtin::UNSAFE_OP_IN_UNSAFE_FN; use rustc_session::lint::Level; use rustc_span::Span; @@ -12,18 +12,19 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { &mut self, destination: Place<'tcx>, block: BasicBlock, - ast_block: &Block<'_, 'tcx>, + ast_block: &Block, source_info: SourceInfo, ) -> BlockAnd<()> { let Block { region_scope, opt_destruction_scope, span, - stmts, + ref stmts, expr, targeted_by_break, safety_mode, } = *ast_block; + let expr = expr.map(|expr| &self.thir[expr]); self.in_opt_scope(opt_destruction_scope.map(|de| (de, source_info)), move |this| { this.in_scope((region_scope, source_info), LintLevel::Inherited, move |this| { if targeted_by_break { @@ -32,13 +33,13 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { destination, block, span, - stmts, + &stmts, expr, safety_mode, )) }) } else { - this.ast_block_stmts(destination, block, span, stmts, expr, safety_mode) + this.ast_block_stmts(destination, block, span, &stmts, expr, safety_mode) } }) }) @@ -49,8 +50,8 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { destination: Place<'tcx>, mut block: BasicBlock, span: Span, - stmts: &[Stmt<'_, 'tcx>], - expr: Option<&Expr<'_, 'tcx>>, + stmts: &[StmtId], + expr: Option<&Expr<'tcx>>, safety_mode: BlockSafety, ) -> BlockAnd<()> { let this = self; @@ -78,23 +79,30 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { this.update_source_scope_for_safety_mode(span, safety_mode); let source_info = this.source_info(span); - for Stmt { kind, opt_destruction_scope } in stmts { + for stmt in stmts { + let Stmt { ref kind, opt_destruction_scope } = this.thir[*stmt]; match kind { - &StmtKind::Expr { scope, expr } => { + StmtKind::Expr { scope, expr } => { this.block_context.push(BlockFrame::Statement { ignores_expr_result: true }); unpack!( block = this.in_opt_scope( opt_destruction_scope.map(|de| (de, source_info)), |this| { - let si = (scope, source_info); + let si = (*scope, source_info); this.in_scope(si, LintLevel::Inherited, |this| { - this.stmt_expr(block, expr, Some(scope)) + this.stmt_expr(block, &this.thir[*expr], Some(*scope)) }) } ) ); } - StmtKind::Let { remainder_scope, init_scope, pattern, initializer, lint_level } => { + StmtKind::Let { + remainder_scope, + init_scope, + ref pattern, + initializer, + lint_level, + } => { let ignores_expr_result = matches!(*pattern.kind, PatKind::Wild); this.block_context.push(BlockFrame::Statement { ignores_expr_result }); @@ -110,6 +118,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { // Evaluate the initializer, if present. if let Some(init) = initializer { + let init = &this.thir[*init]; let initializer_span = init.span; unpack!( @@ -145,7 +154,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { debug!("ast_block_stmts: pattern={:?}", pattern); this.visit_primary_bindings( - &pattern, + pattern, UserTypeProjections::none(), &mut |this, _, _, _, node, span, _, _| { this.storage_live_binding(block, node, span, OutsideGuard, true); diff --git a/compiler/rustc_mir_build/src/build/expr/as_constant.rs b/compiler/rustc_mir_build/src/build/expr/as_constant.rs index 57f56e2d09..5e305ebba2 100644 --- a/compiler/rustc_mir_build/src/build/expr/as_constant.rs +++ b/compiler/rustc_mir_build/src/build/expr/as_constant.rs @@ -1,18 +1,20 @@ //! See docs in build/expr/mod.rs use crate::build::Builder; -use crate::thir::*; use rustc_middle::mir::*; +use rustc_middle::thir::*; use rustc_middle::ty::CanonicalUserTypeAnnotation; impl<'a, 'tcx> Builder<'a, 'tcx> { /// Compile `expr`, yielding a compile-time constant. Assumes that /// `expr` is a valid compile-time constant! - crate fn as_constant(&mut self, expr: &Expr<'_, 'tcx>) -> Constant<'tcx> { + crate fn as_constant(&mut self, expr: &Expr<'tcx>) -> Constant<'tcx> { let this = self; let Expr { ty, temp_lifetime: _, span, ref kind } = *expr; match *kind { - ExprKind::Scope { region_scope: _, lint_level: _, value } => this.as_constant(value), + ExprKind::Scope { region_scope: _, lint_level: _, value } => { + this.as_constant(&this.thir[value]) + } ExprKind::Literal { literal, user_ty, const_id: _ } => { let user_ty = user_ty.map(|user_ty| { this.canonical_user_type_annotations.push(CanonicalUserTypeAnnotation { diff --git a/compiler/rustc_mir_build/src/build/expr/as_operand.rs b/compiler/rustc_mir_build/src/build/expr/as_operand.rs index c393878e0b..b2a1dbf4c5 100644 --- a/compiler/rustc_mir_build/src/build/expr/as_operand.rs +++ b/compiler/rustc_mir_build/src/build/expr/as_operand.rs @@ -2,9 +2,9 @@ use crate::build::expr::category::Category; use crate::build::{BlockAnd, BlockAndExtension, Builder}; -use crate::thir::*; use rustc_middle::middle::region; use rustc_middle::mir::*; +use rustc_middle::thir::*; impl<'a, 'tcx> Builder<'a, 'tcx> { /// Returns an operand suitable for use until the end of the current @@ -17,7 +17,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { crate fn as_local_operand( &mut self, block: BasicBlock, - expr: &Expr<'_, 'tcx>, + expr: &Expr<'tcx>, ) -> BlockAnd> { let local_scope = self.local_scope(); self.as_operand(block, Some(local_scope), expr) @@ -74,7 +74,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { crate fn as_local_call_operand( &mut self, block: BasicBlock, - expr: &Expr<'_, 'tcx>, + expr: &Expr<'tcx>, ) -> BlockAnd> { let local_scope = self.local_scope(); self.as_call_operand(block, Some(local_scope), expr) @@ -93,7 +93,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { &mut self, mut block: BasicBlock, scope: Option, - expr: &Expr<'_, 'tcx>, + expr: &Expr<'tcx>, ) -> BlockAnd> { debug!("as_operand(block={:?}, expr={:?})", block, expr); let this = self; @@ -101,8 +101,9 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { if let ExprKind::Scope { region_scope, lint_level, value } = expr.kind { let source_info = this.source_info(expr.span); let region_scope = (region_scope, source_info); - return this - .in_scope(region_scope, lint_level, |this| this.as_operand(block, scope, value)); + return this.in_scope(region_scope, lint_level, |this| { + this.as_operand(block, scope, &this.thir[value]) + }); } let category = Category::of(&expr.kind).unwrap(); @@ -123,7 +124,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { &mut self, mut block: BasicBlock, scope: Option, - expr: &Expr<'_, 'tcx>, + expr: &Expr<'tcx>, ) -> BlockAnd> { debug!("as_call_operand(block={:?}, expr={:?})", block, expr); let this = self; @@ -132,7 +133,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { let source_info = this.source_info(expr.span); let region_scope = (region_scope, source_info); return this.in_scope(region_scope, lint_level, |this| { - this.as_call_operand(block, scope, value) + this.as_call_operand(block, scope, &this.thir[value]) }); } @@ -151,7 +152,9 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { // type, and that value is coming from the deref of a box. if let ExprKind::Deref { arg } = expr.kind { // Generate let tmp0 = arg0 - let operand = unpack!(block = this.as_temp(block, scope, arg, Mutability::Mut)); + let operand = unpack!( + block = this.as_temp(block, scope, &this.thir[arg], Mutability::Mut) + ); // Return the operand *tmp0 to be used as the call argument let place = Place { diff --git a/compiler/rustc_mir_build/src/build/expr/as_place.rs b/compiler/rustc_mir_build/src/build/expr/as_place.rs index 1053890e61..5511cd4c73 100644 --- a/compiler/rustc_mir_build/src/build/expr/as_place.rs +++ b/compiler/rustc_mir_build/src/build/expr/as_place.rs @@ -3,13 +3,13 @@ use crate::build::expr::category::Category; use crate::build::ForGuard::{OutsideGuard, RefWithinGuard}; use crate::build::{BlockAnd, BlockAndExtension, Builder}; -use crate::thir::*; use rustc_hir::def_id::DefId; use rustc_hir::HirId; use rustc_middle::hir::place::ProjectionKind as HirProjectionKind; use rustc_middle::middle::region; use rustc_middle::mir::AssertKind::BoundsCheck; use rustc_middle::mir::*; +use rustc_middle::thir::*; use rustc_middle::ty::AdtDef; use rustc_middle::ty::{self, CanonicalUserTypeAnnotation, Ty, TyCtxt, Variance}; use rustc_span::Span; @@ -209,9 +209,7 @@ fn to_upvars_resolved_place_builder<'a, 'tcx>( match from_builder.base { PlaceBase::Local(_) => Ok(from_builder), PlaceBase::Upvar { var_hir_id, closure_def_id, closure_kind } => { - // Captures are represented using fields inside a structure. - // This represents accessing self in the closure structure - let mut upvar_resolved_place_builder = PlaceBuilder::from(Local::new(1)); + let mut upvar_resolved_place_builder = PlaceBuilder::from(ty::CAPTURE_STRUCT_LOCAL); match closure_kind { ty::ClosureKind::Fn | ty::ClosureKind::FnMut => { upvar_resolved_place_builder = upvar_resolved_place_builder.deref(); @@ -381,7 +379,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { crate fn as_place( &mut self, mut block: BasicBlock, - expr: &Expr<'_, 'tcx>, + expr: &Expr<'tcx>, ) -> BlockAnd> { let place_builder = unpack!(block = self.as_place_builder(block, expr)); block.and(place_builder.into_place(self.tcx, self.typeck_results)) @@ -392,7 +390,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { crate fn as_place_builder( &mut self, block: BasicBlock, - expr: &Expr<'_, 'tcx>, + expr: &Expr<'tcx>, ) -> BlockAnd> { self.expr_as_place(block, expr, Mutability::Mut, None) } @@ -405,7 +403,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { crate fn as_read_only_place( &mut self, mut block: BasicBlock, - expr: &Expr<'_, 'tcx>, + expr: &Expr<'tcx>, ) -> BlockAnd> { let place_builder = unpack!(block = self.as_read_only_place_builder(block, expr)); block.and(place_builder.into_place(self.tcx, self.typeck_results)) @@ -420,7 +418,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { fn as_read_only_place_builder( &mut self, block: BasicBlock, - expr: &Expr<'_, 'tcx>, + expr: &Expr<'tcx>, ) -> BlockAnd> { self.expr_as_place(block, expr, Mutability::Not, None) } @@ -428,7 +426,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { fn expr_as_place( &mut self, mut block: BasicBlock, - expr: &Expr<'_, 'tcx>, + expr: &Expr<'tcx>, mutability: Mutability, fake_borrow_temps: Option<&mut Vec>, ) -> BlockAnd> { @@ -440,23 +438,27 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { match expr.kind { ExprKind::Scope { region_scope, lint_level, value } => { this.in_scope((region_scope, source_info), lint_level, |this| { - this.expr_as_place(block, value, mutability, fake_borrow_temps) + this.expr_as_place(block, &this.thir[value], mutability, fake_borrow_temps) }) } ExprKind::Field { lhs, name } => { - let place_builder = - unpack!(block = this.expr_as_place(block, lhs, mutability, fake_borrow_temps,)); + let place_builder = unpack!( + block = + this.expr_as_place(block, &this.thir[lhs], mutability, fake_borrow_temps,) + ); block.and(place_builder.field(name, expr.ty)) } ExprKind::Deref { arg } => { - let place_builder = - unpack!(block = this.expr_as_place(block, arg, mutability, fake_borrow_temps,)); + let place_builder = unpack!( + block = + this.expr_as_place(block, &this.thir[arg], mutability, fake_borrow_temps,) + ); block.and(place_builder.deref()) } ExprKind::Index { lhs, index } => this.lower_index_expression( block, - lhs, - index, + &this.thir[lhs], + &this.thir[index], mutability, fake_borrow_temps, expr.temp_lifetime, @@ -481,7 +483,12 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { ExprKind::PlaceTypeAscription { source, user_ty } => { let place_builder = unpack!( - block = this.expr_as_place(block, source, mutability, fake_borrow_temps,) + block = this.expr_as_place( + block, + &this.thir[source], + mutability, + fake_borrow_temps, + ) ); if let Some(user_ty) = user_ty { let annotation_index = @@ -509,6 +516,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { block.and(place_builder) } ExprKind::ValueTypeAscription { source, user_ty } => { + let source = &this.thir[source]; let temp = unpack!(block = this.as_temp(block, source.temp_lifetime, source, mutability)); if let Some(user_ty) = user_ty { @@ -613,8 +621,8 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { fn lower_index_expression( &mut self, mut block: BasicBlock, - base: &Expr<'_, 'tcx>, - index: &Expr<'_, 'tcx>, + base: &Expr<'tcx>, + index: &Expr<'tcx>, mutability: Mutability, fake_borrow_temps: Option<&mut Vec>, temp_lifetime: Option, diff --git a/compiler/rustc_mir_build/src/build/expr/as_rvalue.rs b/compiler/rustc_mir_build/src/build/expr/as_rvalue.rs index 689b27150e..69786c14ee 100644 --- a/compiler/rustc_mir_build/src/build/expr/as_rvalue.rs +++ b/compiler/rustc_mir_build/src/build/expr/as_rvalue.rs @@ -5,11 +5,11 @@ use rustc_index::vec::Idx; use crate::build::expr::as_place::PlaceBase; use crate::build::expr::category::{Category, RvalueFunc}; use crate::build::{BlockAnd, BlockAndExtension, Builder}; -use crate::thir::*; use rustc_middle::middle::region; use rustc_middle::mir::AssertKind; use rustc_middle::mir::Place; use rustc_middle::mir::*; +use rustc_middle::thir::*; use rustc_middle::ty::{self, Ty, UpvarSubsts}; use rustc_span::Span; @@ -23,7 +23,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { crate fn as_local_rvalue( &mut self, block: BasicBlock, - expr: &Expr<'_, 'tcx>, + expr: &Expr<'tcx>, ) -> BlockAnd> { let local_scope = self.local_scope(); self.as_rvalue(block, Some(local_scope), expr) @@ -34,7 +34,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { &mut self, mut block: BasicBlock, scope: Option, - expr: &Expr<'_, 'tcx>, + expr: &Expr<'tcx>, ) -> BlockAnd> { debug!("expr_as_rvalue(block={:?}, scope={:?}, expr={:?})", block, scope, expr); @@ -46,19 +46,22 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { ExprKind::ThreadLocalRef(did) => block.and(Rvalue::ThreadLocalRef(did)), ExprKind::Scope { region_scope, lint_level, value } => { let region_scope = (region_scope, source_info); - this.in_scope(region_scope, lint_level, |this| this.as_rvalue(block, scope, value)) + this.in_scope(region_scope, lint_level, |this| { + this.as_rvalue(block, scope, &this.thir[value]) + }) } ExprKind::Repeat { value, count } => { - let value_operand = unpack!(block = this.as_operand(block, scope, value)); + let value_operand = + unpack!(block = this.as_operand(block, scope, &this.thir[value])); block.and(Rvalue::Repeat(value_operand, count)) } ExprKind::Binary { op, lhs, rhs } => { - let lhs = unpack!(block = this.as_operand(block, scope, lhs)); - let rhs = unpack!(block = this.as_operand(block, scope, rhs)); + let lhs = unpack!(block = this.as_operand(block, scope, &this.thir[lhs])); + let rhs = unpack!(block = this.as_operand(block, scope, &this.thir[rhs])); this.build_binary_op(block, op, expr_span, expr.ty, lhs, rhs) } ExprKind::Unary { op, arg } => { - let arg = unpack!(block = this.as_operand(block, scope, arg)); + let arg = unpack!(block = this.as_operand(block, scope, &this.thir[arg])); // Check for -MIN on signed integers if this.check_overflow && op == UnOp::Neg && expr.ty.is_signed() { let bool_ty = this.tcx.types.bool; @@ -84,6 +87,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { block.and(Rvalue::UnaryOp(op, arg)) } ExprKind::Box { value } => { + let value = &this.thir[value]; // The `Box` temporary created here is not a part of the HIR, // and therefore is not considered during generator auto-trait // determination. See the comment about `box` at `yield_in_scope`. @@ -112,14 +116,14 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { block.and(Rvalue::Use(Operand::Move(Place::from(result)))) } ExprKind::Cast { source } => { - let source = unpack!(block = this.as_operand(block, scope, source)); + let source = unpack!(block = this.as_operand(block, scope, &this.thir[source])); block.and(Rvalue::Cast(CastKind::Misc, source, expr.ty)) } ExprKind::Pointer { cast, source } => { - let source = unpack!(block = this.as_operand(block, scope, source)); + let source = unpack!(block = this.as_operand(block, scope, &this.thir[source])); block.and(Rvalue::Cast(CastKind::Pointer(cast), source, expr.ty)) } - ExprKind::Array { fields } => { + ExprKind::Array { ref fields } => { // (*) We would (maybe) be closer to codegen if we // handled this and other aggregate cases via // `into()`, not `as_rvalue` -- in that case, instead @@ -150,22 +154,24 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { let el_ty = expr.ty.sequence_element_type(this.tcx); let fields: Vec<_> = fields .into_iter() - .map(|f| unpack!(block = this.as_operand(block, scope, f))) + .copied() + .map(|f| unpack!(block = this.as_operand(block, scope, &this.thir[f]))) .collect(); block.and(Rvalue::Aggregate(box AggregateKind::Array(el_ty), fields)) } - ExprKind::Tuple { fields } => { + ExprKind::Tuple { ref fields } => { // see (*) above // first process the set of fields let fields: Vec<_> = fields .into_iter() - .map(|f| unpack!(block = this.as_operand(block, scope, f))) + .copied() + .map(|f| unpack!(block = this.as_operand(block, scope, &this.thir[f]))) .collect(); block.and(Rvalue::Aggregate(box AggregateKind::Tuple, fields)) } - ExprKind::Closure { closure_id, substs, upvars, movability, ref fake_reads } => { + ExprKind::Closure { closure_id, substs, ref upvars, movability, ref fake_reads } => { // Convert the closure fake reads, if any, from `ExprRef` to mir `Place` // and push the fake reads. // This must come before creating the operands. This is required in case @@ -180,32 +186,30 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { // }; // ``` // - // FIXME(RFC2229, rust#85435): Remove feature gate once diagnostics are - // improved and unsafe checking works properly in closure bodies again. - if this.tcx.features().capture_disjoint_fields { - for (thir_place, cause, hir_id) in fake_reads.into_iter() { - let place_builder = - unpack!(block = this.as_place_builder(block, thir_place)); - - if let Ok(place_builder_resolved) = - place_builder.try_upvars_resolved(this.tcx, this.typeck_results) - { - let mir_place = - place_builder_resolved.into_place(this.tcx, this.typeck_results); - this.cfg.push_fake_read( - block, - this.source_info(this.tcx.hir().span(*hir_id)), - *cause, - mir_place, - ); - } + for (thir_place, cause, hir_id) in fake_reads.into_iter() { + let place_builder = + unpack!(block = this.as_place_builder(block, &this.thir[*thir_place])); + + if let Ok(place_builder_resolved) = + place_builder.try_upvars_resolved(this.tcx, this.typeck_results) + { + let mir_place = + place_builder_resolved.into_place(this.tcx, this.typeck_results); + this.cfg.push_fake_read( + block, + this.source_info(this.tcx.hir().span(*hir_id)), + *cause, + mir_place, + ); } } // see (*) above let operands: Vec<_> = upvars .into_iter() + .copied() .map(|upvar| { + let upvar = &this.thir[upvar]; match Category::of(&upvar.kind) { // Use as_place to avoid creating a temporary when // moving a variable into a closure, so that @@ -231,7 +235,11 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { arg, } => unpack!( block = this.limit_capture_mutability( - upvar.span, upvar.ty, scope, block, arg, + upvar.span, + upvar.ty, + scope, + block, + &this.thir[arg], ) ), _ => unpack!(block = this.as_operand(block, scope, upvar)), @@ -404,7 +412,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { upvar_ty: Ty<'tcx>, temp_lifetime: Option, mut block: BasicBlock, - arg: &Expr<'_, 'tcx>, + arg: &Expr<'tcx>, ) -> BlockAnd> { let this = self; @@ -439,7 +447,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { } => { // Not in a closure debug_assert!( - local == Local::new(1), + local == ty::CAPTURE_STRUCT_LOCAL, "Expected local to be Local(1), found {:?}", local ); diff --git a/compiler/rustc_mir_build/src/build/expr/as_temp.rs b/compiler/rustc_mir_build/src/build/expr/as_temp.rs index 98b910ab21..45e0243c88 100644 --- a/compiler/rustc_mir_build/src/build/expr/as_temp.rs +++ b/compiler/rustc_mir_build/src/build/expr/as_temp.rs @@ -2,10 +2,10 @@ use crate::build::scope::DropKind; use crate::build::{BlockAnd, BlockAndExtension, Builder}; -use crate::thir::*; use rustc_data_structures::stack::ensure_sufficient_stack; use rustc_middle::middle::region; use rustc_middle::mir::*; +use rustc_middle::thir::*; impl<'a, 'tcx> Builder<'a, 'tcx> { /// Compile `expr` into a fresh temporary. This is used when building @@ -14,7 +14,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { &mut self, block: BasicBlock, temp_lifetime: Option, - expr: &Expr<'_, 'tcx>, + expr: &Expr<'tcx>, mutability: Mutability, ) -> BlockAnd { // this is the only place in mir building that we need to truly need to worry about @@ -27,7 +27,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { &mut self, mut block: BasicBlock, temp_lifetime: Option, - expr: &Expr<'_, 'tcx>, + expr: &Expr<'tcx>, mutability: Mutability, ) -> BlockAnd { debug!( @@ -40,7 +40,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { let source_info = this.source_info(expr_span); if let ExprKind::Scope { region_scope, lint_level, value } = expr.kind { return this.in_scope((region_scope, source_info), lint_level, |this| { - this.as_temp(block, temp_lifetime, value, mutability) + this.as_temp(block, temp_lifetime, &this.thir[value], mutability) }); } diff --git a/compiler/rustc_mir_build/src/build/expr/category.rs b/compiler/rustc_mir_build/src/build/expr/category.rs index 0cadfa2f0a..c834ce6ce6 100644 --- a/compiler/rustc_mir_build/src/build/expr/category.rs +++ b/compiler/rustc_mir_build/src/build/expr/category.rs @@ -1,4 +1,4 @@ -use crate::thir::*; +use rustc_middle::thir::*; #[derive(Debug, PartialEq)] crate enum Category { @@ -31,7 +31,7 @@ crate enum RvalueFunc { /// Determines the category for a given expression. Note that scope /// and paren expressions have no category. impl Category { - crate fn of(ek: &ExprKind<'_, '_>) -> Option { + crate fn of(ek: &ExprKind<'_>) -> Option { match *ek { ExprKind::Scope { .. } => None, diff --git a/compiler/rustc_mir_build/src/build/expr/into.rs b/compiler/rustc_mir_build/src/build/expr/into.rs index 1e7ed3d95d..f2b00f0f6e 100644 --- a/compiler/rustc_mir_build/src/build/expr/into.rs +++ b/compiler/rustc_mir_build/src/build/expr/into.rs @@ -2,13 +2,13 @@ use crate::build::expr::category::{Category, RvalueFunc}; use crate::build::{BlockAnd, BlockAndExtension, BlockFrame, Builder}; -use crate::thir::*; use rustc_ast::InlineAsmOptions; use rustc_data_structures::fx::FxHashMap; use rustc_data_structures::stack::ensure_sufficient_stack; use rustc_hir as hir; use rustc_index::vec::Idx; use rustc_middle::mir::*; +use rustc_middle::thir::*; use rustc_middle::ty::{self, CanonicalUserTypeAnnotation}; use std::iter; @@ -19,7 +19,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { &mut self, destination: Place<'tcx>, mut block: BasicBlock, - expr: &Expr<'_, 'tcx>, + expr: &Expr<'tcx>, ) -> BlockAnd<()> { debug!("expr_into_dest(destination={:?}, block={:?}, expr={:?})", destination, block, expr); @@ -42,19 +42,24 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { let region_scope = (region_scope, source_info); ensure_sufficient_stack(|| { this.in_scope(region_scope, lint_level, |this| { - this.expr_into_dest(destination, block, value) + this.expr_into_dest(destination, block, &this.thir[value]) }) }) } ExprKind::Block { body: ref ast_block } => { this.ast_block(destination, block, ast_block, source_info) } - ExprKind::Match { scrutinee, arms } => { - this.match_expr(destination, expr_span, block, scrutinee, arms) + ExprKind::Match { scrutinee, ref arms } => { + this.match_expr(destination, expr_span, block, &this.thir[scrutinee], arms) } ExprKind::If { cond, then, else_opt } => { let place = unpack!( - block = this.as_temp(block, Some(this.local_scope()), cond, Mutability::Mut) + block = this.as_temp( + block, + Some(this.local_scope()), + &this.thir[cond], + Mutability::Mut + ) ); let operand = Operand::Move(Place::from(place)); @@ -63,9 +68,11 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { let term = TerminatorKind::if_(this.tcx, operand, then_block, else_block); this.cfg.terminate(block, source_info, term); - unpack!(then_block = this.expr_into_dest(destination, then_block, then)); + unpack!( + then_block = this.expr_into_dest(destination, then_block, &this.thir[then]) + ); else_block = if let Some(else_opt) = else_opt { - unpack!(this.expr_into_dest(destination, else_block, else_opt)) + unpack!(this.expr_into_dest(destination, else_block, &this.thir[else_opt])) } else { // Body of the `if` expression without an `else` clause must return `()`, thus // we implicitly generate a `else {}` if it is not specified. @@ -89,6 +96,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { join_block.unit() } ExprKind::NeverToAny { source } => { + let source = &this.thir[source]; let is_call = matches!(source.kind, ExprKind::Call { .. } | ExprKind::InlineAsm { .. }); @@ -127,7 +135,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { this.cfg.start_new_block(), ); - let lhs = unpack!(block = this.as_local_operand(block, lhs)); + let lhs = unpack!(block = this.as_local_operand(block, &this.thir[lhs])); let blocks = match op { LogicalOp::And => (else_block, shortcircuit_block), LogicalOp::Or => (shortcircuit_block, else_block), @@ -150,7 +158,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { ); this.cfg.goto(shortcircuit_block, source_info, join_block); - let rhs = unpack!(else_block = this.as_local_operand(else_block, rhs)); + let rhs = unpack!(else_block = this.as_local_operand(else_block, &this.thir[rhs])); this.cfg.push_assign(else_block, source_info, destination, Rvalue::Use(rhs)); this.cfg.goto(else_block, source_info, join_block); @@ -186,18 +194,20 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { // introduce a unit temporary as the destination for the loop body. let tmp = this.get_unit_temp(); // Execute the body, branching back to the test. - let body_block_end = unpack!(this.expr_into_dest(tmp, body_block, body)); + let body_block_end = + unpack!(this.expr_into_dest(tmp, body_block, &this.thir[body])); this.cfg.goto(body_block_end, source_info, loop_block); // Loops are only exited by `break` expressions. None }) } - ExprKind::Call { ty: _, fun, args, from_hir_call, fn_span } => { - let fun = unpack!(block = this.as_local_operand(block, fun)); + ExprKind::Call { ty: _, fun, ref args, from_hir_call, fn_span } => { + let fun = unpack!(block = this.as_local_operand(block, &this.thir[fun])); let args: Vec<_> = args .into_iter() - .map(|arg| unpack!(block = this.as_local_call_operand(block, arg))) + .copied() + .map(|arg| unpack!(block = this.as_local_call_operand(block, &this.thir[arg]))) .collect(); let success = this.cfg.start_new_block(); @@ -228,8 +238,9 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { this.diverge_from(block); success.unit() } - ExprKind::Use { source } => this.expr_into_dest(destination, block, source), + ExprKind::Use { source } => this.expr_into_dest(destination, block, &this.thir[source]), ExprKind::Borrow { arg, borrow_kind } => { + let arg = &this.thir[arg]; // We don't do this in `as_rvalue` because we use `as_place` // for borrow expressions, so we cannot create an `RValue` that // remains valid across user code. `as_rvalue` is usually called @@ -244,6 +255,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { block.unit() } ExprKind::AddressOf { mutability, arg } => { + let arg = &this.thir[arg]; let place = match mutability { hir::Mutability::Not => this.as_read_only_place(block, arg), hir::Mutability::Mut => this.as_place(block, arg), @@ -252,7 +264,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { this.cfg.push_assign(block, source_info, destination, address_of); block.unit() } - ExprKind::Adt { adt_def, variant_index, substs, user_ty, fields, ref base } => { + ExprKind::Adt { adt_def, variant_index, substs, user_ty, ref fields, ref base } => { // See the notes for `ExprKind::Array` in `as_rvalue` and for // `ExprKind::Borrow` above. let is_union = adt_def.is_union(); @@ -264,19 +276,27 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { // (evaluating them in order given by user) let fields_map: FxHashMap<_, _> = fields .into_iter() - .map(|f| (f.name, unpack!(block = this.as_operand(block, Some(scope), f.expr)))) + .map(|f| { + ( + f.name, + unpack!( + block = this.as_operand(block, Some(scope), &this.thir[f.expr]) + ), + ) + }) .collect(); let field_names: Vec<_> = (0..adt_def.variants[variant_index].fields.len()).map(Field::new).collect(); let fields: Vec<_> = if let Some(FruInfo { base, field_types }) = base { - let place_builder = unpack!(block = this.as_place_builder(block, base)); + let place_builder = + unpack!(block = this.as_place_builder(block, &this.thir[*base])); // MIR does not natively support FRU, so for each // base-supplied field, generate an operand that // reads it from the base. - iter::zip(field_names, *field_types) + iter::zip(field_names, &**field_types) .map(|(n, ty)| match fields_map.get(&n) { Some(v) => v.clone(), None => { @@ -316,27 +336,26 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { ); block.unit() } - ExprKind::InlineAsm { template, operands, options, line_spans } => { - use crate::thir; - use rustc_middle::mir; + ExprKind::InlineAsm { template, ref operands, options, line_spans } => { + use rustc_middle::{mir, thir}; let operands = operands .into_iter() .map(|op| match *op { thir::InlineAsmOperand::In { reg, expr } => mir::InlineAsmOperand::In { reg, - value: unpack!(block = this.as_local_operand(block, expr)), + value: unpack!(block = this.as_local_operand(block, &this.thir[expr])), }, thir::InlineAsmOperand::Out { reg, late, expr } => { mir::InlineAsmOperand::Out { reg, late, - place: expr - .as_ref() - .map(|expr| unpack!(block = this.as_place(block, expr))), + place: expr.map(|expr| { + unpack!(block = this.as_place(block, &this.thir[expr])) + }), } } thir::InlineAsmOperand::InOut { reg, late, expr } => { - let place = unpack!(block = this.as_place(block, expr)); + let place = unpack!(block = this.as_place(block, &this.thir[expr])); mir::InlineAsmOperand::InOut { reg, late, @@ -349,9 +368,11 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { mir::InlineAsmOperand::InOut { reg, late, - in_value: unpack!(block = this.as_local_operand(block, in_expr)), - out_place: out_expr.as_ref().map(|out_expr| { - unpack!(block = this.as_place(block, out_expr)) + in_value: unpack!( + block = this.as_local_operand(block, &this.thir[in_expr]) + ), + out_place: out_expr.map(|out_expr| { + unpack!(block = this.as_place(block, &this.thir[out_expr])) }), } } @@ -360,9 +381,9 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { value: box Constant { span, user_ty: None, literal: value.into() }, } } - thir::InlineAsmOperand::SymFn { expr } => { - mir::InlineAsmOperand::SymFn { value: box this.as_constant(expr) } - } + thir::InlineAsmOperand::SymFn { expr } => mir::InlineAsmOperand::SymFn { + value: box this.as_constant(&this.thir[expr]), + }, thir::InlineAsmOperand::SymStatic { def_id } => { mir::InlineAsmOperand::SymStatic { def_id } } @@ -434,7 +455,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { ExprKind::Yield { value } => { let scope = this.local_scope(); - let value = unpack!(block = this.as_operand(block, Some(scope), value)); + let value = unpack!(block = this.as_operand(block, Some(scope), &this.thir[value])); let resume = this.cfg.start_new_block(); this.cfg.terminate( block, diff --git a/compiler/rustc_mir_build/src/build/expr/stmt.rs b/compiler/rustc_mir_build/src/build/expr/stmt.rs index f01315fc5d..b03a6bb1a2 100644 --- a/compiler/rustc_mir_build/src/build/expr/stmt.rs +++ b/compiler/rustc_mir_build/src/build/expr/stmt.rs @@ -1,8 +1,8 @@ use crate::build::scope::BreakableTarget; use crate::build::{BlockAnd, BlockAndExtension, BlockFrame, Builder}; -use crate::thir::*; use rustc_middle::middle::region; use rustc_middle::mir::*; +use rustc_middle::thir::*; impl<'a, 'tcx> Builder<'a, 'tcx> { /// Builds a block of MIR statements to evaluate the THIR `expr`. @@ -13,7 +13,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { crate fn stmt_expr( &mut self, mut block: BasicBlock, - expr: &Expr<'_, 'tcx>, + expr: &Expr<'tcx>, statement_scope: Option, ) -> BlockAnd<()> { let this = self; @@ -24,10 +24,12 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { match expr.kind { ExprKind::Scope { region_scope, lint_level, value } => { this.in_scope((region_scope, source_info), lint_level, |this| { - this.stmt_expr(block, value, statement_scope) + this.stmt_expr(block, &this.thir[value], statement_scope) }) } ExprKind::Assign { lhs, rhs } => { + let lhs = &this.thir[lhs]; + let rhs = &this.thir[rhs]; let lhs_span = lhs.span; // Note: we evaluate assignments right-to-left. This @@ -61,6 +63,8 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { // only affects weird things like `x += {x += 1; x}` // -- is that equal to `x + (x + 1)` or `2*(x+1)`? + let lhs = &this.thir[lhs]; + let rhs = &this.thir[rhs]; let lhs_ty = lhs.ty; debug!("stmt_expr AssignOp block_context.push(SubExpr) : {:?}", expr); @@ -87,24 +91,30 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { } ExprKind::Break { label, value } => this.break_scope( block, - value.as_deref(), + value.map(|value| &this.thir[value]), BreakableTarget::Break(label), source_info, ), - ExprKind::Return { value } => { - this.break_scope(block, value.as_deref(), BreakableTarget::Return, source_info) - } - ExprKind::LlvmInlineAsm { asm, outputs, inputs } => { + ExprKind::Return { value } => this.break_scope( + block, + value.map(|value| &this.thir[value]), + BreakableTarget::Return, + source_info, + ), + ExprKind::LlvmInlineAsm { asm, ref outputs, ref inputs } => { debug!("stmt_expr LlvmInlineAsm block_context.push(SubExpr) : {:?}", expr); this.block_context.push(BlockFrame::SubExpr); let outputs = outputs .into_iter() - .map(|output| unpack!(block = this.as_place(block, &output))) + .copied() + .map(|output| unpack!(block = this.as_place(block, &this.thir[output]))) .collect::>() .into_boxed_slice(); let inputs = inputs .into_iter() + .copied() .map(|input| { + let input = &this.thir[input]; (input.span, unpack!(block = this.as_local_operand(block, &input))) }) .collect::>() @@ -139,14 +149,14 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { // than the entirety of block(s) surrounding it. let adjusted_span = (|| { if let ExprKind::Block { body } = &expr.kind { - if let Some(tail_expr) = &body.expr { - let mut expr = &*tail_expr; + if let Some(tail_expr) = body.expr { + let mut expr = &this.thir[tail_expr]; while let ExprKind::Block { body: Block { expr: Some(nested_expr), .. }, } - | ExprKind::Scope { value: nested_expr, .. } = &expr.kind + | ExprKind::Scope { value: nested_expr, .. } = expr.kind { - expr = nested_expr; + expr = &this.thir[nested_expr]; } this.block_context.push(BlockFrame::TailExpr { tail_result_is_ignored: true, diff --git a/compiler/rustc_mir_build/src/build/matches/mod.rs b/compiler/rustc_mir_build/src/build/matches/mod.rs index 0e422dc3c6..8164529dd1 100644 --- a/compiler/rustc_mir_build/src/build/matches/mod.rs +++ b/compiler/rustc_mir_build/src/build/matches/mod.rs @@ -10,7 +10,6 @@ use crate::build::scope::DropKind; use crate::build::ForGuard::{self, OutsideGuard, RefWithinGuard}; use crate::build::{BlockAnd, BlockAndExtension, Builder}; use crate::build::{GuardFrame, GuardFrameLocal, LocalsForNode}; -use crate::thir::{self, *}; use rustc_data_structures::{ fx::{FxHashSet, FxIndexMap}, stack::ensure_sufficient_stack, @@ -19,6 +18,7 @@ use rustc_hir::HirId; use rustc_index::bit_set::BitSet; use rustc_middle::middle::region; use rustc_middle::mir::*; +use rustc_middle::thir::{self, *}; use rustc_middle::ty::{self, CanonicalUserTypeAnnotation, Ty}; use rustc_span::symbol::Symbol; use rustc_span::Span; @@ -90,8 +90,8 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { destination: Place<'tcx>, span: Span, mut block: BasicBlock, - scrutinee: &Expr<'_, 'tcx>, - arms: &[Arm<'_, 'tcx>], + scrutinee: &Expr<'tcx>, + arms: &[ArmId], ) -> BlockAnd<()> { let scrutinee_span = scrutinee.span; let scrutinee_place = @@ -99,7 +99,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { let mut arm_candidates = self.create_match_candidates(scrutinee_place.clone(), &arms); - let match_has_guard = arms.iter().any(|arm| arm.guard.is_some()); + let match_has_guard = arms.iter().copied().any(|arm| self.thir[arm].guard.is_some()); let mut candidates = arm_candidates.iter_mut().map(|(_, candidate)| candidate).collect::>(); @@ -120,7 +120,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { fn lower_scrutinee( &mut self, mut block: BasicBlock, - scrutinee: &Expr<'_, 'tcx>, + scrutinee: &Expr<'tcx>, scrutinee_span: Span, ) -> BlockAnd> { let scrutinee_place_builder = unpack!(block = self.as_place_builder(block, scrutinee)); @@ -156,12 +156,17 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { fn create_match_candidates<'pat>( &mut self, scrutinee: PlaceBuilder<'tcx>, - arms: &'pat [Arm<'pat, 'tcx>], - ) -> Vec<(&'pat Arm<'pat, 'tcx>, Candidate<'pat, 'tcx>)> { + arms: &'pat [ArmId], + ) -> Vec<(&'pat Arm<'tcx>, Candidate<'pat, 'tcx>)> + where + 'a: 'pat, + { // Assemble a list of candidates: there is one candidate per pattern, // which means there may be more than one candidate *per arm*. arms.iter() + .copied() .map(|arm| { + let arm = &self.thir[arm]; let arm_has_guard = arm.guard.is_some(); let arm_candidate = Candidate::new(scrutinee.clone(), &arm.pattern, arm_has_guard); (arm, arm_candidate) @@ -231,7 +236,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { destination: Place<'tcx>, scrutinee_place_builder: PlaceBuilder<'tcx>, scrutinee_span: Span, - arm_candidates: Vec<(&'_ Arm<'_, 'tcx>, Candidate<'_, 'tcx>)>, + arm_candidates: Vec<(&'_ Arm<'tcx>, Candidate<'_, 'tcx>)>, outer_source_info: SourceInfo, fake_borrow_temps: Vec<(Place<'tcx>, Local)>, ) -> BlockAnd<()> { @@ -286,7 +291,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { this.source_scope = source_scope; } - this.expr_into_dest(destination, arm_block, &arm.body) + this.expr_into_dest(destination, arm_block, &&this.thir[arm.body]) }) }) .collect(); @@ -313,7 +318,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { &mut self, outer_source_info: SourceInfo, candidate: Candidate<'_, 'tcx>, - guard: Option<&Guard<'_, 'tcx>>, + guard: Option<&Guard<'tcx>>, fake_borrow_temps: &Vec<(Place<'tcx>, Local)>, scrutinee_span: Span, arm_span: Option, @@ -389,7 +394,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { &mut self, mut block: BasicBlock, irrefutable_pat: Pat<'tcx>, - initializer: &Expr<'_, 'tcx>, + initializer: &Expr<'tcx>, ) -> BlockAnd<()> { match *irrefutable_pat.kind { // Optimize the case of `let x = ...` to write directly into `x` @@ -427,7 +432,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { .. }, ascription: - thir::pattern::Ascription { user_ty: pat_ascription_ty, variance: _, user_ty_span }, + thir::Ascription { user_ty: pat_ascription_ty, variance: _, user_ty_span }, } => { let place = self.storage_live_binding(block, var, irrefutable_pat.span, OutsideGuard, true); @@ -682,7 +687,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { PatKind::AscribeUserType { ref subpattern, - ascription: thir::pattern::Ascription { ref user_ty, user_ty_span, variance: _ }, + ascription: thir::Ascription { ref user_ty, user_ty_span, variance: _ }, } => { // This corresponds to something like // @@ -1665,7 +1670,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { &mut self, candidate: Candidate<'pat, 'tcx>, parent_bindings: &[(Vec>, Vec>)], - guard: Option<&Guard<'_, 'tcx>>, + guard: Option<&Guard<'tcx>>, fake_borrows: &Vec<(Place<'tcx>, Local)>, scrutinee_span: Span, arm_span: Option, @@ -1799,12 +1804,14 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { self.cfg.push_assign(block, scrutinee_source_info, Place::from(temp), borrow); } - let (guard_span, (post_guard_block, otherwise_post_guard_block)) = match guard { + let (guard_span, (post_guard_block, otherwise_post_guard_block)) = match *guard { Guard::If(e) => { + let e = &self.thir[e]; let source_info = self.source_info(e.span); (e.span, self.test_bool(block, e, source_info)) } - Guard::IfLet(pat, scrutinee) => { + Guard::IfLet(ref pat, scrutinee) => { + let scrutinee = &self.thir[scrutinee]; let scrutinee_span = scrutinee.span; let scrutinee_place_builder = unpack!(block = self.lower_scrutinee(block, scrutinee, scrutinee_span)); @@ -1840,7 +1847,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { guard_candidate, None, &fake_borrow_temps, - scrutinee.span, + scrutinee_span, None, None, ); diff --git a/compiler/rustc_mir_build/src/build/matches/simplify.rs b/compiler/rustc_mir_build/src/build/matches/simplify.rs index 3ad143a57f..13cfc3695c 100644 --- a/compiler/rustc_mir_build/src/build/matches/simplify.rs +++ b/compiler/rustc_mir_build/src/build/matches/simplify.rs @@ -15,8 +15,8 @@ use crate::build::expr::as_place::PlaceBuilder; use crate::build::matches::{Ascription, Binding, Candidate, MatchPair}; use crate::build::Builder; -use crate::thir::{self, *}; use rustc_hir::RangeEnd; +use rustc_middle::thir::{self, *}; use rustc_middle::ty; use rustc_middle::ty::layout::IntegerExt; use rustc_target::abi::{Integer, Size}; @@ -152,7 +152,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { match *match_pair.pattern.kind { PatKind::AscribeUserType { ref subpattern, - ascription: thir::pattern::Ascription { variance, user_ty, user_ty_span }, + ascription: thir::Ascription { variance, user_ty, user_ty_span }, } => { // Apply the type ascription to the value at `match_pair.place`, which is the candidate.ascriptions.push(Ascription { diff --git a/compiler/rustc_mir_build/src/build/matches/test.rs b/compiler/rustc_mir_build/src/build/matches/test.rs index b082169cd6..c87f42738c 100644 --- a/compiler/rustc_mir_build/src/build/matches/test.rs +++ b/compiler/rustc_mir_build/src/build/matches/test.rs @@ -9,11 +9,11 @@ use crate::build::expr::as_place::PlaceBuilder; use crate::build::matches::{Candidate, MatchPair, Test, TestKind}; use crate::build::Builder; use crate::thir::pattern::compare_const_vals; -use crate::thir::*; use rustc_data_structures::fx::FxIndexMap; use rustc_hir::{LangItem, RangeEnd}; use rustc_index::bit_set::BitSet; use rustc_middle::mir::*; +use rustc_middle::thir::*; use rustc_middle::ty::subst::{GenericArg, Subst}; use rustc_middle::ty::util::IntTypeExt; use rustc_middle::ty::{self, adjustment::PointerCast, Ty, TyCtxt}; diff --git a/compiler/rustc_mir_build/src/build/matches/util.rs b/compiler/rustc_mir_build/src/build/matches/util.rs index d49a00a566..3cf8ae6efd 100644 --- a/compiler/rustc_mir_build/src/build/matches/util.rs +++ b/compiler/rustc_mir_build/src/build/matches/util.rs @@ -1,8 +1,8 @@ use crate::build::expr::as_place::PlaceBuilder; use crate::build::matches::MatchPair; use crate::build::Builder; -use crate::thir::*; use rustc_middle::mir::*; +use rustc_middle::thir::*; use rustc_middle::ty; use smallvec::SmallVec; use std::convert::TryInto; diff --git a/compiler/rustc_mir_build/src/build/mod.rs b/compiler/rustc_mir_build/src/build/mod.rs index f944e5f8f0..10d6521e7d 100644 --- a/compiler/rustc_mir_build/src/build/mod.rs +++ b/compiler/rustc_mir_build/src/build/mod.rs @@ -1,7 +1,7 @@ use crate::build; use crate::build::expr::as_place::PlaceBuilder; use crate::build::scope::DropKind; -use crate::thir::{build_thir, Arena, BindingMode, Expr, LintLevel, Pat, PatKind}; +use crate::thir::pattern::pat_from_hir; use rustc_attr::{self as attr, UnwindAttr}; use rustc_errors::ErrorReported; use rustc_hir as hir; @@ -13,6 +13,7 @@ use rustc_infer::infer::{InferCtxt, TyCtxtInferExt}; use rustc_middle::hir::place::PlaceBase as HirPlaceBase; use rustc_middle::middle::region; use rustc_middle::mir::*; +use rustc_middle::thir::{BindingMode, Expr, ExprId, LintLevel, PatKind, Thir}; use rustc_middle::ty::subst::Subst; use rustc_middle::ty::{self, Ty, TyCtxt, TypeFoldable, TypeckResults}; use rustc_span::symbol::{kw, sym}; @@ -45,6 +46,16 @@ fn mir_build(tcx: TyCtxt<'_>, def: ty::WithOptConstParam) -> Body<'_ let body_owner_kind = tcx.hir().body_owner_kind(id); let typeck_results = tcx.typeck_opt_const_arg(def); + // Ensure unsafeck is ran before we steal the THIR. + match def { + ty::WithOptConstParam { did, const_param_did: Some(const_param_did) } => { + tcx.ensure().thir_check_unsafety_for_const_arg((did, const_param_did)) + } + ty::WithOptConstParam { did, const_param_did: None } => { + tcx.ensure().thir_check_unsafety(did) + } + } + // Figure out what primary body this item has. let (body_id, return_ty_span, span_with_body) = match tcx.hir().get(id) { Node::Expr(hir::Expr { kind: hir::ExprKind::Closure(_, decl, body_id, _, _), .. }) => { @@ -88,7 +99,6 @@ fn mir_build(tcx: TyCtxt<'_>, def: ty::WithOptConstParam) -> Body<'_ // If we don't have a specialized span for the body, just use the // normal def span. let span_with_body = span_with_body.unwrap_or_else(|| tcx.hir().span(id)); - let arena = Arena::default(); tcx.infer_ctxt().enter(|infcx| { let body = if let Some(ErrorReported) = typeck_results.tainted_by_errors { @@ -105,7 +115,10 @@ fn mir_build(tcx: TyCtxt<'_>, def: ty::WithOptConstParam) -> Body<'_ }; let body = tcx.hir().body(body_id); - let thir = build_thir(tcx, def, &arena, &body.value); + let (thir, expr) = tcx.thir_body(def); + // We ran all queries that depended on THIR at the beginning + // of `mir_build`, so now we can steal it + let thir = thir.steal(); let ty = tcx.type_of(fn_def_id); let mut abi = fn_sig.abi; let implicit_argument = match ty.kind() { @@ -181,6 +194,7 @@ fn mir_build(tcx: TyCtxt<'_>, def: ty::WithOptConstParam) -> Body<'_ }; let mut mir = build::construct_fn( + &thir, &infcx, def, id, @@ -190,7 +204,7 @@ fn mir_build(tcx: TyCtxt<'_>, def: ty::WithOptConstParam) -> Body<'_ return_ty, return_ty_span, body, - thir, + expr, span_with_body, ); if yield_ty.is_some() { @@ -212,10 +226,12 @@ fn mir_build(tcx: TyCtxt<'_>, def: ty::WithOptConstParam) -> Body<'_ let return_ty = typeck_results.node_type(id); - let ast_expr = &tcx.hir().body(body_id).value; - let thir = build_thir(tcx, def, &arena, ast_expr); + let (thir, expr) = tcx.thir_body(def); + // We ran all queries that depended on THIR at the beginning + // of `mir_build`, so now we can steal it + let thir = thir.steal(); - build::construct_const(&infcx, thir, def, id, return_ty, return_ty_span) + build::construct_const(&thir, &infcx, expr, def, id, return_ty, return_ty_span) }; lints::check(tcx, &body); @@ -323,6 +339,7 @@ struct Builder<'a, 'tcx> { region_scope_tree: &'tcx region::ScopeTree, param_env: ty::ParamEnv<'tcx>, + thir: &'a Thir<'tcx>, cfg: CFG<'tcx>, def_id: DefId, @@ -633,6 +650,7 @@ struct ArgInfo<'tcx>( ); fn construct_fn<'tcx, A>( + thir: &Thir<'tcx>, infcx: &InferCtxt<'_, 'tcx>, fn_def: ty::WithOptConstParam, fn_id: hir::HirId, @@ -642,7 +660,7 @@ fn construct_fn<'tcx, A>( return_ty: Ty<'tcx>, return_ty_span: Span, body: &'tcx hir::Body<'tcx>, - expr: &Expr<'_, 'tcx>, + expr: ExprId, span_with_body: Span, ) -> Body<'tcx> where @@ -654,6 +672,7 @@ where let span = tcx.hir().span(fn_id); let mut builder = Builder::new( + thir, infcx, fn_def, fn_id, @@ -683,7 +702,7 @@ where fn_def.did.to_def_id(), &arguments, arg_scope, - expr, + &thir[expr], ) })) })); @@ -708,8 +727,9 @@ where } fn construct_const<'a, 'tcx>( + thir: &'a Thir<'tcx>, infcx: &'a InferCtxt<'a, 'tcx>, - expr: &Expr<'_, 'tcx>, + expr: ExprId, def: ty::WithOptConstParam, hir_id: hir::HirId, const_ty: Ty<'tcx>, @@ -717,11 +737,21 @@ fn construct_const<'a, 'tcx>( ) -> Body<'tcx> { let tcx = infcx.tcx; let span = tcx.hir().span(hir_id); - let mut builder = - Builder::new(infcx, def, hir_id, span, 0, Safety::Safe, const_ty, const_ty_span, None); + let mut builder = Builder::new( + thir, + infcx, + def, + hir_id, + span, + 0, + Safety::Safe, + const_ty, + const_ty_span, + None, + ); let mut block = START_BLOCK; - unpack!(block = builder.expr_into_dest(Place::return_place(), block, &expr)); + unpack!(block = builder.expr_into_dest(Place::return_place(), block, &thir[expr])); let source_info = builder.source_info(span); builder.cfg.terminate(block, source_info, TerminatorKind::Return); @@ -761,22 +791,48 @@ fn construct_error<'a, 'tcx>( hir::BodyOwnerKind::Const => 0, hir::BodyOwnerKind::Static(_) => 0, }; - let mut builder = - Builder::new(infcx, def, hir_id, span, num_params, Safety::Safe, ty, span, generator_kind); - let source_info = builder.source_info(span); + let mut cfg = CFG { basic_blocks: IndexVec::new() }; + let mut source_scopes = IndexVec::new(); + let mut local_decls = IndexVec::from_elem_n(LocalDecl::new(ty, span), 1); + + cfg.start_new_block(); + source_scopes.push(SourceScopeData { + span, + parent_scope: None, + inlined: None, + inlined_parent_scope: None, + local_data: ClearCrossCrate::Set(SourceScopeLocalData { + lint_root: hir_id, + safety: Safety::Safe, + }), + }); + let source_info = SourceInfo { span, scope: OUTERMOST_SOURCE_SCOPE }; + // Some MIR passes will expect the number of parameters to match the // function declaration. for _ in 0..num_params { - builder.local_decls.push(LocalDecl::with_source_info(ty, source_info)); + local_decls.push(LocalDecl::with_source_info(ty, source_info)); } - builder.cfg.terminate(START_BLOCK, source_info, TerminatorKind::Unreachable); - let mut body = builder.finish(); + cfg.terminate(START_BLOCK, source_info, TerminatorKind::Unreachable); + + let mut body = Body::new( + MirSource::item(def.did.to_def_id()), + cfg.basic_blocks, + source_scopes, + local_decls, + IndexVec::new(), + num_params, + vec![], + span, + generator_kind, + ); body.generator.as_mut().map(|gen| gen.yield_ty = Some(ty)); body } impl<'a, 'tcx> Builder<'a, 'tcx> { fn new( + thir: &'a Thir<'tcx>, infcx: &'a InferCtxt<'a, 'tcx>, def: ty::WithOptConstParam, hir_id: hir::HirId, @@ -803,6 +859,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { let lint_level = LintLevel::Explicit(hir_id); let mut builder = Builder { + thir, tcx, infcx, typeck_results: tcx.typeck_opt_const_arg(def), @@ -866,7 +923,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { fn_def_id: DefId, arguments: &[ArgInfo<'tcx>], argument_scope: region::Scope, - expr: &Expr<'_, 'tcx>, + expr: &Expr<'tcx>, ) -> BlockAnd<()> { // Allocate locals for the function arguments for &ArgInfo(ty, _, arg_opt, _) in arguments.iter() { @@ -896,9 +953,8 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { // the given closure and use the necessary information to create upvar // debuginfo and to fill `self.upvar_mutbls`. if hir_typeck_results.closure_min_captures.get(&fn_def_id).is_some() { - let closure_env_arg = Local::new(1); let mut closure_env_projs = vec![]; - let mut closure_ty = self.local_decls[closure_env_arg].ty; + let mut closure_ty = self.local_decls[ty::CAPTURE_STRUCT_LOCAL].ty; if let ty::Ref(_, ty, _) = closure_ty.kind() { closure_env_projs.push(ProjectionElem::Deref); closure_ty = ty; @@ -944,7 +1000,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { name, source_info: SourceInfo::outermost(tcx_hir.span(var_id)), value: VarDebugInfoContents::Place(Place { - local: closure_env_arg, + local: ty::CAPTURE_STRUCT_LOCAL, projection: tcx.intern_place_elems(&projs), }), }); @@ -975,7 +1031,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { Node::Pat(pat) | Node::Binding(pat) => pat, node => bug!("pattern became {:?}", node), }; - let pattern = Pat::from_hir(tcx, self.param_env, self.typeck_results, pat); + let pattern = pat_from_hir(tcx, self.param_env, self.typeck_results, pat); let original_source_scope = self.source_scope; let span = pattern.span; self.set_correct_source_scope_for_arg(arg.hir_id, original_source_scope, span); diff --git a/compiler/rustc_mir_build/src/build/scope.rs b/compiler/rustc_mir_build/src/build/scope.rs index 41fc925c04..3de894bd37 100644 --- a/compiler/rustc_mir_build/src/build/scope.rs +++ b/compiler/rustc_mir_build/src/build/scope.rs @@ -82,11 +82,12 @@ that contains only loops and breakable blocks. It tracks where a `break`, */ use crate::build::{BlockAnd, BlockAndExtension, BlockFrame, Builder, CFG}; -use crate::thir::{Expr, LintLevel}; use rustc_data_structures::fx::FxHashMap; use rustc_index::vec::IndexVec; use rustc_middle::middle::region; use rustc_middle::mir::*; +use rustc_middle::thir::{Expr, LintLevel}; + use rustc_span::{Span, DUMMY_SP}; #[derive(Debug)] @@ -574,7 +575,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { crate fn break_scope( &mut self, mut block: BasicBlock, - value: Option<&Expr<'_, 'tcx>>, + value: Option<&Expr<'tcx>>, target: BreakableTarget, source_info: SourceInfo, ) -> BlockAnd<()> { @@ -933,7 +934,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { crate fn test_bool( &mut self, mut block: BasicBlock, - condition: &Expr<'_, 'tcx>, + condition: &Expr<'tcx>, source_info: SourceInfo, ) -> (BasicBlock, BasicBlock) { let cond = unpack!(block = self.as_local_operand(block, condition)); diff --git a/compiler/rustc_mir_build/src/check_unsafety.rs b/compiler/rustc_mir_build/src/check_unsafety.rs new file mode 100644 index 0000000000..2d52577829 --- /dev/null +++ b/compiler/rustc_mir_build/src/check_unsafety.rs @@ -0,0 +1,425 @@ +use crate::thir::visit::{self, Visitor}; + +use rustc_errors::struct_span_err; +use rustc_hir as hir; +use rustc_middle::thir::*; +use rustc_middle::ty::{self, TyCtxt}; +use rustc_session::lint::builtin::{UNSAFE_OP_IN_UNSAFE_FN, UNUSED_UNSAFE}; +use rustc_session::lint::Level; +use rustc_span::def_id::{DefId, LocalDefId}; +use rustc_span::symbol::Symbol; +use rustc_span::Span; + +use std::ops::Bound; + +struct UnsafetyVisitor<'a, 'tcx> { + tcx: TyCtxt<'tcx>, + thir: &'a Thir<'tcx>, + /// The `HirId` of the current scope, which would be the `HirId` + /// of the current HIR node, modulo adjustments. Used for lint levels. + hir_context: hir::HirId, + /// The current "safety context". This notably tracks whether we are in an + /// `unsafe` block, and whether it has been used. + safety_context: SafetyContext, + body_unsafety: BodyUnsafety, + /// The `#[target_feature]` attributes of the body. Used for checking + /// calls to functions with `#[target_feature]` (RFC 2396). + body_target_features: &'tcx Vec, + is_const: bool, +} + +impl<'tcx> UnsafetyVisitor<'_, 'tcx> { + fn in_safety_context( + &mut self, + safety_context: SafetyContext, + f: impl FnOnce(&mut Self) -> R, + ) { + if let ( + SafetyContext::UnsafeBlock { span: enclosing_span, .. }, + SafetyContext::UnsafeBlock { span: block_span, hir_id, .. }, + ) = (self.safety_context, safety_context) + { + self.warn_unused_unsafe( + hir_id, + block_span, + Some((self.tcx.sess.source_map().guess_head_span(enclosing_span), "block")), + ); + f(self); + } else { + let prev_context = self.safety_context; + self.safety_context = safety_context; + + f(self); + + if let SafetyContext::UnsafeBlock { used: false, span, hir_id } = self.safety_context { + self.warn_unused_unsafe( + hir_id, + span, + if self.unsafe_op_in_unsafe_fn_allowed() { + self.body_unsafety.unsafe_fn_sig_span().map(|span| (span, "fn")) + } else { + None + }, + ); + } + self.safety_context = prev_context; + return; + } + } + + fn requires_unsafe(&mut self, span: Span, kind: UnsafeOpKind) { + let (description, note) = kind.description_and_note(); + let unsafe_op_in_unsafe_fn_allowed = self.unsafe_op_in_unsafe_fn_allowed(); + match self.safety_context { + SafetyContext::UnsafeBlock { ref mut used, .. } => { + if !self.body_unsafety.is_unsafe() || !unsafe_op_in_unsafe_fn_allowed { + // Mark this block as useful + *used = true; + } + } + SafetyContext::UnsafeFn if unsafe_op_in_unsafe_fn_allowed => {} + SafetyContext::UnsafeFn => { + // unsafe_op_in_unsafe_fn is disallowed + self.tcx.struct_span_lint_hir( + UNSAFE_OP_IN_UNSAFE_FN, + self.hir_context, + span, + |lint| { + lint.build(&format!( + "{} is unsafe and requires unsafe block (error E0133)", + description, + )) + .span_label(span, description) + .note(note) + .emit(); + }, + ) + } + SafetyContext::Safe => { + let fn_sugg = if unsafe_op_in_unsafe_fn_allowed { " function or" } else { "" }; + struct_span_err!( + self.tcx.sess, + span, + E0133, + "{} is unsafe and requires unsafe{} block", + description, + fn_sugg, + ) + .span_label(span, description) + .note(note) + .emit(); + } + } + } + + fn warn_unused_unsafe( + &self, + hir_id: hir::HirId, + block_span: Span, + enclosing_unsafe: Option<(Span, &'static str)>, + ) { + let block_span = self.tcx.sess.source_map().guess_head_span(block_span); + self.tcx.struct_span_lint_hir(UNUSED_UNSAFE, hir_id, block_span, |lint| { + let msg = "unnecessary `unsafe` block"; + let mut db = lint.build(msg); + db.span_label(block_span, msg); + if let Some((span, kind)) = enclosing_unsafe { + db.span_label(span, format!("because it's nested under this `unsafe` {}", kind)); + } + db.emit(); + }); + } + + /// Whether the `unsafe_op_in_unsafe_fn` lint is `allow`ed at the current HIR node. + fn unsafe_op_in_unsafe_fn_allowed(&self) -> bool { + self.tcx.lint_level_at_node(UNSAFE_OP_IN_UNSAFE_FN, self.hir_context).0 == Level::Allow + } +} + +impl<'a, 'tcx> Visitor<'a, 'tcx> for UnsafetyVisitor<'a, 'tcx> { + fn thir(&self) -> &'a Thir<'tcx> { + &self.thir + } + + fn visit_block(&mut self, block: &Block) { + if let BlockSafety::ExplicitUnsafe(hir_id) = block.safety_mode { + self.in_safety_context( + SafetyContext::UnsafeBlock { span: block.span, hir_id, used: false }, + |this| visit::walk_block(this, block), + ); + } else { + visit::walk_block(self, block); + } + } + + fn visit_expr(&mut self, expr: &Expr<'tcx>) { + match expr.kind { + ExprKind::Scope { value, lint_level: LintLevel::Explicit(hir_id), region_scope: _ } => { + let prev_id = self.hir_context; + self.hir_context = hir_id; + self.visit_expr(&self.thir[value]); + self.hir_context = prev_id; + return; + } + ExprKind::Call { fun, ty: _, args: _, from_hir_call: _, fn_span: _ } => { + if self.thir[fun].ty.fn_sig(self.tcx).unsafety() == hir::Unsafety::Unsafe { + self.requires_unsafe(expr.span, CallToUnsafeFunction); + } else if let &ty::FnDef(func_did, _) = self.thir[fun].ty.kind() { + // If the called function has target features the calling function hasn't, + // the call requires `unsafe`. Don't check this on wasm + // targets, though. For more information on wasm see the + // is_like_wasm check in typeck/src/collect.rs + if !self.tcx.sess.target.options.is_like_wasm + && !self + .tcx + .codegen_fn_attrs(func_did) + .target_features + .iter() + .all(|feature| self.body_target_features.contains(feature)) + { + self.requires_unsafe(expr.span, CallToFunctionWith); + } + } + } + ExprKind::Deref { arg } => { + if let ExprKind::StaticRef { def_id, .. } = self.thir[arg].kind { + if self.tcx.is_mutable_static(def_id) { + self.requires_unsafe(expr.span, UseOfMutableStatic); + } else if self.tcx.is_foreign_item(def_id) { + self.requires_unsafe(expr.span, UseOfExternStatic); + } + } else if self.thir[arg].ty.is_unsafe_ptr() { + self.requires_unsafe(expr.span, DerefOfRawPointer); + } + } + ExprKind::InlineAsm { .. } | ExprKind::LlvmInlineAsm { .. } => { + self.requires_unsafe(expr.span, UseOfInlineAssembly); + } + ExprKind::Adt { + adt_def, + variant_index: _, + substs: _, + user_ty: _, + fields: _, + base: _, + } => match self.tcx.layout_scalar_valid_range(adt_def.did) { + (Bound::Unbounded, Bound::Unbounded) => {} + _ => self.requires_unsafe(expr.span, InitializingTypeWith), + }, + ExprKind::Cast { source } => { + let source = &self.thir[source]; + if self.tcx.features().const_raw_ptr_to_usize_cast + && self.is_const + && (source.ty.is_unsafe_ptr() || source.ty.is_fn_ptr()) + && expr.ty.is_integral() + { + self.requires_unsafe(expr.span, CastOfPointerToInt); + } + } + ExprKind::Closure { + closure_id, + substs: _, + upvars: _, + movability: _, + fake_reads: _, + } => { + let closure_id = closure_id.expect_local(); + let closure_def = if let Some((did, const_param_id)) = + ty::WithOptConstParam::try_lookup(closure_id, self.tcx) + { + ty::WithOptConstParam { did, const_param_did: Some(const_param_id) } + } else { + ty::WithOptConstParam::unknown(closure_id) + }; + let (closure_thir, expr) = self.tcx.thir_body(closure_def); + let closure_thir = &closure_thir.borrow(); + let hir_context = self.tcx.hir().local_def_id_to_hir_id(closure_id); + let mut closure_visitor = + UnsafetyVisitor { thir: closure_thir, hir_context, ..*self }; + closure_visitor.visit_expr(&closure_thir[expr]); + // Unsafe blocks can be used in closures, make sure to take it into account + self.safety_context = closure_visitor.safety_context; + } + _ => {} + } + + visit::walk_expr(self, expr); + } +} + +#[derive(Clone, Copy)] +enum SafetyContext { + Safe, + UnsafeFn, + UnsafeBlock { span: Span, hir_id: hir::HirId, used: bool }, +} + +#[derive(Clone, Copy)] +enum BodyUnsafety { + /// The body is not unsafe. + Safe, + /// The body is an unsafe function. The span points to + /// the signature of the function. + Unsafe(Span), +} + +impl BodyUnsafety { + /// Returns whether the body is unsafe. + fn is_unsafe(&self) -> bool { + matches!(self, BodyUnsafety::Unsafe(_)) + } + + /// If the body is unsafe, returns the `Span` of its signature. + fn unsafe_fn_sig_span(self) -> Option { + match self { + BodyUnsafety::Unsafe(span) => Some(span), + BodyUnsafety::Safe => None, + } + } +} + +#[derive(Clone, Copy, PartialEq)] +enum UnsafeOpKind { + CallToUnsafeFunction, + UseOfInlineAssembly, + InitializingTypeWith, + CastOfPointerToInt, + UseOfMutableStatic, + UseOfExternStatic, + DerefOfRawPointer, + #[allow(dead_code)] // FIXME + AssignToDroppingUnionField, + #[allow(dead_code)] // FIXME + AccessToUnionField, + #[allow(dead_code)] // FIXME + MutationOfLayoutConstrainedField, + #[allow(dead_code)] // FIXME + BorrowOfLayoutConstrainedField, + CallToFunctionWith, +} + +use UnsafeOpKind::*; + +impl UnsafeOpKind { + pub fn description_and_note(&self) -> (&'static str, &'static str) { + match self { + CallToUnsafeFunction => ( + "call to unsafe function", + "consult the function's documentation for information on how to avoid undefined \ + behavior", + ), + UseOfInlineAssembly => ( + "use of inline assembly", + "inline assembly is entirely unchecked and can cause undefined behavior", + ), + InitializingTypeWith => ( + "initializing type with `rustc_layout_scalar_valid_range` attr", + "initializing a layout restricted type's field with a value outside the valid \ + range is undefined behavior", + ), + CastOfPointerToInt => { + ("cast of pointer to int", "casting pointers to integers in constants") + } + UseOfMutableStatic => ( + "use of mutable static", + "mutable statics can be mutated by multiple threads: aliasing violations or data \ + races will cause undefined behavior", + ), + UseOfExternStatic => ( + "use of extern static", + "extern statics are not controlled by the Rust type system: invalid data, \ + aliasing violations or data races will cause undefined behavior", + ), + DerefOfRawPointer => ( + "dereference of raw pointer", + "raw pointers may be null, dangling or unaligned; they can violate aliasing rules \ + and cause data races: all of these are undefined behavior", + ), + AssignToDroppingUnionField => ( + "assignment to union field that might need dropping", + "the previous content of the field will be dropped, which causes undefined \ + behavior if the field was not properly initialized", + ), + AccessToUnionField => ( + "access to union field", + "the field may not be properly initialized: using uninitialized data will cause \ + undefined behavior", + ), + MutationOfLayoutConstrainedField => ( + "mutation of layout constrained field", + "mutating layout constrained fields cannot statically be checked for valid values", + ), + BorrowOfLayoutConstrainedField => ( + "borrow of layout constrained field with interior mutability", + "references to fields of layout constrained fields lose the constraints. Coupled \ + with interior mutability, the field can be changed to invalid values", + ), + CallToFunctionWith => ( + "call to function with `#[target_feature]`", + "can only be called if the required target features are available", + ), + } + } +} + +pub fn check_unsafety<'tcx>(tcx: TyCtxt<'tcx>, def: ty::WithOptConstParam) { + // THIR unsafeck is gated under `-Z thir-unsafeck` + if !tcx.sess.opts.debugging_opts.thir_unsafeck { + return; + } + + // Closures are handled by their parent function + if tcx.is_closure(def.did.to_def_id()) { + tcx.ensure().thir_check_unsafety(tcx.hir().local_def_id_to_hir_id(def.did).owner); + return; + } + + let (thir, expr) = tcx.thir_body(def); + let thir = &thir.borrow(); + // If `thir` is empty, a type error occured, skip this body. + if thir.exprs.is_empty() { + return; + } + + let hir_id = tcx.hir().local_def_id_to_hir_id(def.did); + let body_unsafety = tcx.hir().fn_sig_by_hir_id(hir_id).map_or(BodyUnsafety::Safe, |fn_sig| { + if fn_sig.header.unsafety == hir::Unsafety::Unsafe { + BodyUnsafety::Unsafe(fn_sig.span) + } else { + BodyUnsafety::Safe + } + }); + let body_target_features = &tcx.codegen_fn_attrs(def.did).target_features; + let safety_context = + if body_unsafety.is_unsafe() { SafetyContext::UnsafeFn } else { SafetyContext::Safe }; + let is_const = match tcx.hir().body_owner_kind(hir_id) { + hir::BodyOwnerKind::Closure => false, + hir::BodyOwnerKind::Fn => tcx.is_const_fn_raw(def.did.to_def_id()), + hir::BodyOwnerKind::Const | hir::BodyOwnerKind::Static(_) => true, + }; + let mut visitor = UnsafetyVisitor { + tcx, + thir, + safety_context, + hir_context: hir_id, + body_unsafety, + body_target_features, + is_const, + }; + visitor.visit_expr(&thir[expr]); +} + +crate fn thir_check_unsafety<'tcx>(tcx: TyCtxt<'tcx>, def_id: LocalDefId) { + if let Some(def) = ty::WithOptConstParam::try_lookup(def_id, tcx) { + tcx.thir_check_unsafety_for_const_arg(def) + } else { + check_unsafety(tcx, ty::WithOptConstParam::unknown(def_id)) + } +} + +crate fn thir_check_unsafety_for_const_arg<'tcx>( + tcx: TyCtxt<'tcx>, + (did, param_did): (LocalDefId, DefId), +) { + check_unsafety(tcx, ty::WithOptConstParam { did, const_param_did: Some(param_did) }) +} diff --git a/compiler/rustc_mir_build/src/lib.rs b/compiler/rustc_mir_build/src/lib.rs index da9a0b08e8..d2992f0bf1 100644 --- a/compiler/rustc_mir_build/src/lib.rs +++ b/compiler/rustc_mir_build/src/lib.rs @@ -1,16 +1,14 @@ //! Construction of MIR from HIR. //! //! This crate also contains the match exhaustiveness and usefulness checking. -#![feature(array_windows)] #![feature(box_patterns)] #![feature(box_syntax)] -#![feature(const_panic)] #![feature(control_flow_enum)] #![feature(crate_visibility_modifier)] #![feature(bool_to_option)] #![feature(iter_zip)] #![feature(once_cell)] -#![cfg_attr(bootstrap, feature(or_patterns))] +#![feature(min_specialization)] #![recursion_limit = "256"] #[macro_use] @@ -19,6 +17,7 @@ extern crate tracing; extern crate rustc_middle; mod build; +mod check_unsafety; mod lints; pub mod thir; @@ -28,4 +27,7 @@ pub fn provide(providers: &mut Providers) { providers.check_match = thir::pattern::check_match; providers.lit_to_const = thir::constant::lit_to_const; providers.mir_built = build::mir_built; + providers.thir_check_unsafety = check_unsafety::thir_check_unsafety; + providers.thir_check_unsafety_for_const_arg = check_unsafety::thir_check_unsafety_for_const_arg; + providers.thir_body = thir::cx::thir_body; } diff --git a/compiler/rustc_mir_build/src/thir/arena.rs b/compiler/rustc_mir_build/src/thir/arena.rs deleted file mode 100644 index aacc7b12a4..0000000000 --- a/compiler/rustc_mir_build/src/thir/arena.rs +++ /dev/null @@ -1,98 +0,0 @@ -use crate::thir::*; - -macro_rules! declare_arena { - ([], [$($a:tt $name:ident: $ty:ty,)*]) => { - #[derive(Default)] - pub struct Arena<'thir, 'tcx> { - pub dropless: rustc_arena::DroplessArena, - drop: rustc_arena::DropArena, - $($name: rustc_arena::arena_for_type!($a[$ty]),)* - } - - pub trait ArenaAllocatable<'thir, 'tcx, T = Self>: Sized { - fn allocate_on(self, arena: &'thir Arena<'thir, 'tcx>) -> &'thir mut Self; - fn allocate_from_iter( - arena: &'thir Arena<'thir, 'tcx>, - iter: impl ::std::iter::IntoIterator, - ) -> &'thir mut [Self]; - } - - impl<'thir, 'tcx, T: Copy> ArenaAllocatable<'thir, 'tcx, ()> for T { - #[inline] - fn allocate_on(self, arena: &'thir Arena<'thir, 'tcx>) -> &'thir mut Self { - arena.dropless.alloc(self) - } - #[inline] - fn allocate_from_iter( - arena: &'thir Arena<'thir, 'tcx>, - iter: impl ::std::iter::IntoIterator, - ) -> &'thir mut [Self] { - arena.dropless.alloc_from_iter(iter) - } - - } - $( - impl<'thir, 'tcx> ArenaAllocatable<'thir, 'tcx, $ty> for $ty { - #[inline] - fn allocate_on(self, arena: &'thir Arena<'thir, 'tcx>) -> &'thir mut Self { - if !::std::mem::needs_drop::() { - return arena.dropless.alloc(self); - } - match rustc_arena::which_arena_for_type!($a[&arena.$name]) { - ::std::option::Option::<&rustc_arena::TypedArena>::Some(ty_arena) => { - ty_arena.alloc(self) - } - ::std::option::Option::None => unsafe { arena.drop.alloc(self) }, - } - } - - #[inline] - fn allocate_from_iter( - arena: &'thir Arena<'thir, 'tcx>, - iter: impl ::std::iter::IntoIterator, - ) -> &'thir mut [Self] { - if !::std::mem::needs_drop::() { - return arena.dropless.alloc_from_iter(iter); - } - match rustc_arena::which_arena_for_type!($a[&arena.$name]) { - ::std::option::Option::<&rustc_arena::TypedArena>::Some(ty_arena) => { - ty_arena.alloc_from_iter(iter) - } - ::std::option::Option::None => unsafe { arena.drop.alloc_from_iter(iter) }, - } - } - } - )* - - impl<'thir, 'tcx> Arena<'thir, 'tcx> { - #[inline] - pub fn alloc, U>(&'thir self, value: T) -> &'thir mut T { - value.allocate_on(self) - } - - #[allow(dead_code)] // function is never used - #[inline] - pub fn alloc_slice(&'thir self, value: &[T]) -> &'thir mut [T] { - if value.is_empty() { - return &mut []; - } - self.dropless.alloc_slice(value) - } - - pub fn alloc_from_iter, U>( - &'thir self, - iter: impl ::std::iter::IntoIterator, - ) -> &'thir mut [T] { - T::allocate_from_iter(self, iter) - } - } - } -} - -declare_arena!([], [ - [] arm: Arm<'thir, 'tcx>, - [] expr: Expr<'thir, 'tcx>, - [] field_expr: FieldExpr<'thir, 'tcx>, - [few] inline_asm_operand: InlineAsmOperand<'thir, 'tcx>, - [] stmt: Stmt<'thir, 'tcx>, -]); diff --git a/compiler/rustc_mir_build/src/thir/constant.rs b/compiler/rustc_mir_build/src/thir/constant.rs index ac93d04297..d62fd161e2 100644 --- a/compiler/rustc_mir_build/src/thir/constant.rs +++ b/compiler/rustc_mir_build/src/thir/constant.rs @@ -25,14 +25,14 @@ crate fn lit_to_const<'tcx>( let lit = match (lit, &ty.kind()) { (ast::LitKind::Str(s, _), ty::Ref(_, inner_ty, _)) if inner_ty.is_str() => { let s = s.as_str(); - let allocation = Allocation::from_byte_aligned_bytes(s.as_bytes()); + let allocation = Allocation::from_bytes_byte_aligned_immutable(s.as_bytes()); let allocation = tcx.intern_const_alloc(allocation); ConstValue::Slice { data: allocation, start: 0, end: s.len() } } (ast::LitKind::ByteStr(data), ty::Ref(_, inner_ty, _)) if matches!(inner_ty.kind(), ty::Slice(_)) => { - let allocation = Allocation::from_byte_aligned_bytes(data as &[u8]); + let allocation = Allocation::from_bytes_byte_aligned_immutable(data as &[u8]); let allocation = tcx.intern_const_alloc(allocation); ConstValue::Slice { data: allocation, start: 0, end: data.len() } } diff --git a/compiler/rustc_mir_build/src/thir/cx/block.rs b/compiler/rustc_mir_build/src/thir/cx/block.rs index d450f8a265..77235fe9ab 100644 --- a/compiler/rustc_mir_build/src/thir/cx/block.rs +++ b/compiler/rustc_mir_build/src/thir/cx/block.rs @@ -1,14 +1,14 @@ use crate::thir::cx::Cx; -use crate::thir::{self, *}; use rustc_hir as hir; use rustc_middle::middle::region; +use rustc_middle::thir::*; use rustc_middle::ty; use rustc_index::vec::Idx; -impl<'thir, 'tcx> Cx<'thir, 'tcx> { - crate fn mirror_block(&mut self, block: &'tcx hir::Block<'tcx>) -> Block<'thir, 'tcx> { +impl<'tcx> Cx<'tcx> { + crate fn mirror_block(&mut self, block: &'tcx hir::Block<'tcx>) -> Block { // We have to eagerly lower the "spine" of the statements // in order to get the lexical scoping correctly. let stmts = self.mirror_stmts(block.hir_id.local_id, block.stmts); @@ -37,65 +37,78 @@ impl<'thir, 'tcx> Cx<'thir, 'tcx> { &mut self, block_id: hir::ItemLocalId, stmts: &'tcx [hir::Stmt<'tcx>], - ) -> &'thir [Stmt<'thir, 'tcx>] { - self.arena.alloc_from_iter(stmts.iter().enumerate().filter_map(|(index, stmt)| { - let hir_id = stmt.hir_id; - let opt_dxn_ext = self.region_scope_tree.opt_destruction_scope(hir_id.local_id); - match stmt.kind { - hir::StmtKind::Expr(ref expr) | hir::StmtKind::Semi(ref expr) => Some(Stmt { - kind: StmtKind::Expr { - scope: region::Scope { id: hir_id.local_id, data: region::ScopeData::Node }, - expr: self.mirror_expr(expr), - }, - opt_destruction_scope: opt_dxn_ext, - }), - hir::StmtKind::Item(..) => { - // ignore for purposes of the MIR - None - } - hir::StmtKind::Local(ref local) => { - let remainder_scope = region::Scope { - id: block_id, - data: region::ScopeData::Remainder(region::FirstStatementIndex::new(index)), - }; + ) -> Box<[StmtId]> { + stmts + .iter() + .enumerate() + .filter_map(|(index, stmt)| { + let hir_id = stmt.hir_id; + let opt_dxn_ext = self.region_scope_tree.opt_destruction_scope(hir_id.local_id); + match stmt.kind { + hir::StmtKind::Expr(ref expr) | hir::StmtKind::Semi(ref expr) => { + let stmt = Stmt { + kind: StmtKind::Expr { + scope: region::Scope { + id: hir_id.local_id, + data: region::ScopeData::Node, + }, + expr: self.mirror_expr(expr), + }, + opt_destruction_scope: opt_dxn_ext, + }; + Some(self.thir.stmts.push(stmt)) + } + hir::StmtKind::Item(..) => { + // ignore for purposes of the MIR + None + } + hir::StmtKind::Local(ref local) => { + let remainder_scope = region::Scope { + id: block_id, + data: region::ScopeData::Remainder(region::FirstStatementIndex::new( + index, + )), + }; - let mut pattern = self.pattern_from_hir(local.pat); + let mut pattern = self.pattern_from_hir(local.pat); - if let Some(ty) = &local.ty { - if let Some(&user_ty) = - self.typeck_results.user_provided_types().get(ty.hir_id) - { - debug!("mirror_stmts: user_ty={:?}", user_ty); - pattern = Pat { - ty: pattern.ty, - span: pattern.span, - kind: Box::new(PatKind::AscribeUserType { - ascription: thir::pattern::Ascription { - user_ty: PatTyProj::from_user_type(user_ty), - user_ty_span: ty.span, - variance: ty::Variance::Covariant, - }, - subpattern: pattern, - }), - }; + if let Some(ty) = &local.ty { + if let Some(&user_ty) = + self.typeck_results.user_provided_types().get(ty.hir_id) + { + debug!("mirror_stmts: user_ty={:?}", user_ty); + pattern = Pat { + ty: pattern.ty, + span: pattern.span, + kind: Box::new(PatKind::AscribeUserType { + ascription: Ascription { + user_ty: PatTyProj::from_user_type(user_ty), + user_ty_span: ty.span, + variance: ty::Variance::Covariant, + }, + subpattern: pattern, + }), + }; + } } - } - Some(Stmt { - kind: StmtKind::Let { - remainder_scope, - init_scope: region::Scope { - id: hir_id.local_id, - data: region::ScopeData::Node, + let stmt = Stmt { + kind: StmtKind::Let { + remainder_scope, + init_scope: region::Scope { + id: hir_id.local_id, + data: region::ScopeData::Node, + }, + pattern, + initializer: local.init.map(|init| self.mirror_expr(init)), + lint_level: LintLevel::Explicit(local.hir_id), }, - pattern, - initializer: local.init.map(|init| self.mirror_expr(init)), - lint_level: LintLevel::Explicit(local.hir_id), - }, - opt_destruction_scope: opt_dxn_ext, - }) + opt_destruction_scope: opt_dxn_ext, + }; + Some(self.thir.stmts.push(stmt)) + } } - } - })) + }) + .collect() } } diff --git a/compiler/rustc_mir_build/src/thir/cx/expr.rs b/compiler/rustc_mir_build/src/thir/cx/expr.rs index 924278e1a7..aa4acfab5c 100644 --- a/compiler/rustc_mir_build/src/thir/cx/expr.rs +++ b/compiler/rustc_mir_build/src/thir/cx/expr.rs @@ -1,6 +1,5 @@ use crate::thir::cx::Cx; use crate::thir::util::UserAnnotatedTyHelpers; -use crate::thir::*; use rustc_data_structures::stack::ensure_sufficient_stack; use rustc_hir as hir; use rustc_hir::def::{CtorKind, CtorOf, DefKind, Res}; @@ -8,43 +7,30 @@ use rustc_index::vec::Idx; use rustc_middle::hir::place::Place as HirPlace; use rustc_middle::hir::place::PlaceBase as HirPlaceBase; use rustc_middle::hir::place::ProjectionKind as HirProjectionKind; +use rustc_middle::middle::region; use rustc_middle::mir::interpret::Scalar; -use rustc_middle::mir::BorrowKind; +use rustc_middle::mir::{BinOp, BorrowKind, Field, UnOp}; +use rustc_middle::thir::*; use rustc_middle::ty::adjustment::{ Adjust, Adjustment, AutoBorrow, AutoBorrowMutability, PointerCast, }; use rustc_middle::ty::subst::{InternalSubsts, SubstsRef}; -use rustc_middle::ty::{self, AdtKind, Ty}; +use rustc_middle::ty::{self, AdtKind, Ty, UpvarSubsts, UserType}; +use rustc_span::def_id::DefId; use rustc_span::Span; +use rustc_target::abi::VariantIdx; -use std::iter; - -impl<'thir, 'tcx> Cx<'thir, 'tcx> { - /// Mirrors and allocates a single [`hir::Expr`]. If you need to mirror a whole slice - /// of expressions, prefer using [`mirror_exprs`]. - /// - /// [`mirror_exprs`]: Self::mirror_exprs - crate fn mirror_expr(&mut self, expr: &'tcx hir::Expr<'tcx>) -> &'thir Expr<'thir, 'tcx> { +impl<'tcx> Cx<'tcx> { + crate fn mirror_expr(&mut self, expr: &'tcx hir::Expr<'tcx>) -> ExprId { // `mirror_expr` is recursing very deep. Make sure the stack doesn't overflow. - ensure_sufficient_stack(|| self.arena.alloc(self.mirror_expr_inner(expr))) + ensure_sufficient_stack(|| self.mirror_expr_inner(expr)) } - /// Mirrors and allocates a slice of [`hir::Expr`]s. They will be allocated as a - /// contiguous sequence in memory. - crate fn mirror_exprs(&mut self, exprs: &'tcx [hir::Expr<'tcx>]) -> &'thir [Expr<'thir, 'tcx>] { - self.arena.alloc_from_iter(exprs.iter().map(|expr| self.mirror_expr_inner(expr))) + crate fn mirror_exprs(&mut self, exprs: &'tcx [hir::Expr<'tcx>]) -> Box<[ExprId]> { + exprs.iter().map(|expr| self.mirror_expr_inner(expr)).collect() } - /// Mirrors a [`hir::Expr`] without allocating it into the arena. - /// This is a separate, private function so that [`mirror_expr`] and [`mirror_exprs`] can - /// decide how to allocate this expression (alone or within a slice). - /// - /// [`mirror_expr`]: Self::mirror_expr - /// [`mirror_exprs`]: Self::mirror_exprs - pub(super) fn mirror_expr_inner( - &mut self, - hir_expr: &'tcx hir::Expr<'tcx>, - ) -> Expr<'thir, 'tcx> { + pub(super) fn mirror_expr_inner(&mut self, hir_expr: &'tcx hir::Expr<'tcx>) -> ExprId { let temp_lifetime = self.region_scope_tree.temporary_scope(hir_expr.hir_id.local_id); let expr_scope = region::Scope { id: hir_expr.hir_id.local_id, data: region::ScopeData::Node }; @@ -66,7 +52,7 @@ impl<'thir, 'tcx> Cx<'thir, 'tcx> { span: hir_expr.span, kind: ExprKind::Scope { region_scope: expr_scope, - value: self.arena.alloc(expr), + value: self.thir.exprs.push(expr), lint_level: LintLevel::Explicit(hir_expr.hir_id), }, }; @@ -81,22 +67,22 @@ impl<'thir, 'tcx> Cx<'thir, 'tcx> { span: hir_expr.span, kind: ExprKind::Scope { region_scope, - value: self.arena.alloc(expr), + value: self.thir.exprs.push(expr), lint_level: LintLevel::Inherited, }, }; } // OK, all done! - expr + self.thir.exprs.push(expr) } fn apply_adjustment( &mut self, hir_expr: &'tcx hir::Expr<'tcx>, - mut expr: Expr<'thir, 'tcx>, + mut expr: Expr<'tcx>, adjustment: &Adjustment<'tcx>, - ) -> Expr<'thir, 'tcx> { + ) -> Expr<'tcx> { let Expr { temp_lifetime, mut span, .. } = expr; // Adjust the span from the block, to the last expression of the @@ -109,10 +95,10 @@ impl<'thir, 'tcx> Cx<'thir, 'tcx> { // x // // ^ error message points at this expression. // } - let mut adjust_span = |expr: &mut Expr<'thir, 'tcx>| { + let mut adjust_span = |expr: &mut Expr<'tcx>| { if let ExprKind::Block { body } = &expr.kind { - if let Some(ref last_expr) = body.expr { - span = last_expr.span; + if let Some(last_expr) = body.expr { + span = self.thir[last_expr].span; expr.span = span; } } @@ -121,13 +107,13 @@ impl<'thir, 'tcx> Cx<'thir, 'tcx> { let kind = match adjustment.kind { Adjust::Pointer(PointerCast::Unsize) => { adjust_span(&mut expr); - ExprKind::Pointer { cast: PointerCast::Unsize, source: self.arena.alloc(expr) } + ExprKind::Pointer { cast: PointerCast::Unsize, source: self.thir.exprs.push(expr) } } - Adjust::Pointer(cast) => ExprKind::Pointer { cast, source: self.arena.alloc(expr) }, - Adjust::NeverToAny => ExprKind::NeverToAny { source: self.arena.alloc(expr) }, + Adjust::Pointer(cast) => ExprKind::Pointer { cast, source: self.thir.exprs.push(expr) }, + Adjust::NeverToAny => ExprKind::NeverToAny { source: self.thir.exprs.push(expr) }, Adjust::Deref(None) => { adjust_span(&mut expr); - ExprKind::Deref { arg: self.arena.alloc(expr) } + ExprKind::Deref { arg: self.thir.exprs.push(expr) } } Adjust::Deref(Some(deref)) => { // We don't need to do call adjust_span here since @@ -142,30 +128,27 @@ impl<'thir, 'tcx> Cx<'thir, 'tcx> { span, kind: ExprKind::Borrow { borrow_kind: deref.mutbl.to_borrow_kind(), - arg: self.arena.alloc(expr), + arg: self.thir.exprs.push(expr), }, }; - self.overloaded_place( - hir_expr, - adjustment.target, - Some(call), - self.arena.alloc_from_iter(iter::once(expr)), - deref.span, - ) - } - Adjust::Borrow(AutoBorrow::Ref(_, m)) => { - ExprKind::Borrow { borrow_kind: m.to_borrow_kind(), arg: self.arena.alloc(expr) } + let expr = box [self.thir.exprs.push(expr)]; + + self.overloaded_place(hir_expr, adjustment.target, Some(call), expr, deref.span) } + Adjust::Borrow(AutoBorrow::Ref(_, m)) => ExprKind::Borrow { + borrow_kind: m.to_borrow_kind(), + arg: self.thir.exprs.push(expr), + }, Adjust::Borrow(AutoBorrow::RawPtr(mutability)) => { - ExprKind::AddressOf { mutability, arg: self.arena.alloc(expr) } + ExprKind::AddressOf { mutability, arg: self.thir.exprs.push(expr) } } }; Expr { temp_lifetime, ty: adjustment.target, span, kind } } - fn make_mirror_unadjusted(&mut self, expr: &'tcx hir::Expr<'tcx>) -> Expr<'thir, 'tcx> { + fn make_mirror_unadjusted(&mut self, expr: &'tcx hir::Expr<'tcx>) -> Expr<'tcx> { let expr_ty = self.typeck_results().expr_ty(expr); let temp_lifetime = self.region_scope_tree.temporary_scope(expr.hir_id.local_id); @@ -177,7 +160,7 @@ impl<'thir, 'tcx> Cx<'thir, 'tcx> { let args = self.mirror_exprs(args); ExprKind::Call { ty: expr.ty, - fun: self.arena.alloc(expr), + fun: self.thir.exprs.push(expr), args, from_hir_call: true, fn_span, @@ -202,13 +185,12 @@ impl<'thir, 'tcx> Cx<'thir, 'tcx> { span: expr.span, kind: ExprKind::Tuple { fields: self.mirror_exprs(args) }, }; + let tupled_args = self.thir.exprs.push(tupled_args); ExprKind::Call { ty: method.ty, - fun: self.arena.alloc(method), - args: self - .arena - .alloc_from_iter(vec![self.mirror_expr_inner(fun), tupled_args]), + fun: self.thir.exprs.push(method), + args: box [self.mirror_expr(fun), tupled_args], from_hir_call: true, fn_span: expr.span, } @@ -238,10 +220,14 @@ impl<'thir, 'tcx> Cx<'thir, 'tcx> { }); debug!("make_mirror_unadjusted: (call) user_ty={:?}", user_ty); - let field_refs = - self.arena.alloc_from_iter(args.iter().enumerate().map(|(idx, e)| { - FieldExpr { name: Field::new(idx), expr: self.mirror_expr(e) } - })); + let field_refs = args + .iter() + .enumerate() + .map(|(idx, e)| FieldExpr { + name: Field::new(idx), + expr: self.mirror_expr(e), + }) + .collect(); ExprKind::Adt { adt_def, substs, @@ -278,9 +264,9 @@ impl<'thir, 'tcx> Cx<'thir, 'tcx> { hir::ExprKind::AssignOp(op, ref lhs, ref rhs) => { if self.typeck_results().is_method_call(expr) { - let lhs = self.mirror_expr_inner(lhs); - let rhs = self.mirror_expr_inner(rhs); - self.overloaded_operator(expr, self.arena.alloc_from_iter(vec![lhs, rhs])) + let lhs = self.mirror_expr(lhs); + let rhs = self.mirror_expr(rhs); + self.overloaded_operator(expr, box [lhs, rhs]) } else { ExprKind::AssignOp { op: bin_op(op.node), @@ -298,9 +284,9 @@ impl<'thir, 'tcx> Cx<'thir, 'tcx> { hir::ExprKind::Binary(op, ref lhs, ref rhs) => { if self.typeck_results().is_method_call(expr) { - let lhs = self.mirror_expr_inner(lhs); - let rhs = self.mirror_expr_inner(rhs); - self.overloaded_operator(expr, self.arena.alloc_from_iter(vec![lhs, rhs])) + let lhs = self.mirror_expr(lhs); + let rhs = self.mirror_expr(rhs); + self.overloaded_operator(expr, box [lhs, rhs]) } else { // FIXME overflow match op.node { @@ -329,15 +315,9 @@ impl<'thir, 'tcx> Cx<'thir, 'tcx> { hir::ExprKind::Index(ref lhs, ref index) => { if self.typeck_results().is_method_call(expr) { - let lhs = self.mirror_expr_inner(lhs); - let index = self.mirror_expr_inner(index); - self.overloaded_place( - expr, - expr_ty, - None, - self.arena.alloc_from_iter(vec![lhs, index]), - expr.span, - ) + let lhs = self.mirror_expr(lhs); + let index = self.mirror_expr(index); + self.overloaded_place(expr, expr_ty, None, box [lhs, index], expr.span) } else { ExprKind::Index { lhs: self.mirror_expr(lhs), index: self.mirror_expr(index) } } @@ -345,14 +325,8 @@ impl<'thir, 'tcx> Cx<'thir, 'tcx> { hir::ExprKind::Unary(hir::UnOp::Deref, ref arg) => { if self.typeck_results().is_method_call(expr) { - let arg = self.mirror_expr_inner(arg); - self.overloaded_place( - expr, - expr_ty, - None, - self.arena.alloc_from_iter(iter::once(arg)), - expr.span, - ) + let arg = self.mirror_expr(arg); + self.overloaded_place(expr, expr_ty, None, box [arg], expr.span) } else { ExprKind::Deref { arg: self.mirror_expr(arg) } } @@ -360,8 +334,8 @@ impl<'thir, 'tcx> Cx<'thir, 'tcx> { hir::ExprKind::Unary(hir::UnOp::Not, ref arg) => { if self.typeck_results().is_method_call(expr) { - let arg = self.mirror_expr_inner(arg); - self.overloaded_operator(expr, self.arena.alloc_from_iter(iter::once(arg))) + let arg = self.mirror_expr(arg); + self.overloaded_operator(expr, box [arg]) } else { ExprKind::Unary { op: UnOp::Not, arg: self.mirror_expr(arg) } } @@ -369,8 +343,8 @@ impl<'thir, 'tcx> Cx<'thir, 'tcx> { hir::ExprKind::Unary(hir::UnOp::Neg, ref arg) => { if self.typeck_results().is_method_call(expr) { - let arg = self.mirror_expr_inner(arg); - self.overloaded_operator(expr, self.arena.alloc_from_iter(iter::once(arg))) + let arg = self.mirror_expr(arg); + self.overloaded_operator(expr, box [arg]) } else if let hir::ExprKind::Lit(ref lit) = arg.kind { ExprKind::Literal { literal: self.const_eval_literal(&lit.node, expr_ty, lit.span, true), @@ -396,11 +370,10 @@ impl<'thir, 'tcx> Cx<'thir, 'tcx> { fields: self.field_refs(fields), base: base.as_ref().map(|base| FruInfo { base: self.mirror_expr(base), - field_types: self.arena.alloc_from_iter( - self.typeck_results().fru_field_types()[expr.hir_id] - .iter() - .cloned(), - ), + field_types: self.typeck_results().fru_field_types()[expr.hir_id] + .iter() + .copied() + .collect(), }), } } @@ -447,12 +420,15 @@ impl<'thir, 'tcx> Cx<'thir, 'tcx> { } }; - let upvars = self.arena.alloc_from_iter( - self.typeck_results - .closure_min_captures_flattened(def_id) - .zip(substs.upvar_tys()) - .map(|(captured_place, ty)| self.capture_upvar(expr, captured_place, ty)), - ); + let upvars = self + .typeck_results + .closure_min_captures_flattened(def_id) + .zip(substs.upvar_tys()) + .map(|(captured_place, ty)| { + let upvars = self.capture_upvar(expr, captured_place, ty); + self.thir.exprs.push(upvars) + }) + .collect(); // Convert the closure fake reads, if any, from hir `Place` to ExprRef let fake_reads = match self.typeck_results.closure_fake_reads.get(&def_id) { @@ -460,8 +436,7 @@ impl<'thir, 'tcx> Cx<'thir, 'tcx> { .iter() .map(|(place, cause, hir_id)| { let expr = self.convert_captured_hir_place(expr, place.clone()); - let expr_ref: &'thir Expr<'thir, 'tcx> = self.arena.alloc(expr); - (expr_ref, *cause, *hir_id) + (self.thir.exprs.push(expr), *cause, *hir_id) }) .collect(), None => Vec::new(), @@ -477,99 +452,105 @@ impl<'thir, 'tcx> Cx<'thir, 'tcx> { hir::ExprKind::InlineAsm(ref asm) => ExprKind::InlineAsm { template: asm.template, - operands: self.arena.alloc_from_iter(asm.operands.iter().map(|(op, _op_sp)| { - match *op { - hir::InlineAsmOperand::In { reg, ref expr } => { - InlineAsmOperand::In { reg, expr: self.mirror_expr(expr) } - } - hir::InlineAsmOperand::Out { reg, late, ref expr } => { - InlineAsmOperand::Out { + operands: asm + .operands + .iter() + .map(|(op, _op_sp)| { + match *op { + hir::InlineAsmOperand::In { reg, ref expr } => { + InlineAsmOperand::In { reg, expr: self.mirror_expr(expr) } + } + hir::InlineAsmOperand::Out { reg, late, ref expr } => { + InlineAsmOperand::Out { + reg, + late, + expr: expr.as_ref().map(|expr| self.mirror_expr(expr)), + } + } + hir::InlineAsmOperand::InOut { reg, late, ref expr } => { + InlineAsmOperand::InOut { reg, late, expr: self.mirror_expr(expr) } + } + hir::InlineAsmOperand::SplitInOut { reg, late, - expr: expr.as_ref().map(|expr| self.mirror_expr(expr)), - } - } - hir::InlineAsmOperand::InOut { reg, late, ref expr } => { - InlineAsmOperand::InOut { reg, late, expr: self.mirror_expr(expr) } - } - hir::InlineAsmOperand::SplitInOut { - reg, - late, - ref in_expr, - ref out_expr, - } => InlineAsmOperand::SplitInOut { - reg, - late, - in_expr: self.mirror_expr(in_expr), - out_expr: out_expr.as_ref().map(|expr| self.mirror_expr(expr)), - }, - hir::InlineAsmOperand::Const { ref anon_const } => { - let anon_const_def_id = self.tcx.hir().local_def_id(anon_const.hir_id); - let value = ty::Const::from_anon_const(self.tcx, anon_const_def_id); - let span = self.tcx.hir().span(anon_const.hir_id); + ref in_expr, + ref out_expr, + } => InlineAsmOperand::SplitInOut { + reg, + late, + in_expr: self.mirror_expr(in_expr), + out_expr: out_expr.as_ref().map(|expr| self.mirror_expr(expr)), + }, + hir::InlineAsmOperand::Const { ref anon_const } => { + let anon_const_def_id = + self.tcx.hir().local_def_id(anon_const.hir_id); + let value = ty::Const::from_anon_const(self.tcx, anon_const_def_id); + let span = self.tcx.hir().span(anon_const.hir_id); - InlineAsmOperand::Const { value, span } - } - hir::InlineAsmOperand::Sym { ref expr } => { - let qpath = match expr.kind { - hir::ExprKind::Path(ref qpath) => qpath, - _ => span_bug!( - expr.span, - "asm `sym` operand should be a path, found {:?}", - expr.kind - ), - }; - let temp_lifetime = - self.region_scope_tree.temporary_scope(expr.hir_id.local_id); - let res = self.typeck_results().qpath_res(qpath, expr.hir_id); - let ty; - match res { - Res::Def(DefKind::Fn, _) | Res::Def(DefKind::AssocFn, _) => { - ty = self.typeck_results().node_type(expr.hir_id); - let user_ty = self.user_substs_applied_to_res(expr.hir_id, res); - InlineAsmOperand::SymFn { - expr: self.arena.alloc(Expr { - ty, - temp_lifetime, - span: expr.span, - kind: ExprKind::Literal { - literal: ty::Const::zero_sized(self.tcx, ty), - user_ty, - const_id: None, - }, - }), + InlineAsmOperand::Const { value, span } + } + hir::InlineAsmOperand::Sym { ref expr } => { + let qpath = match expr.kind { + hir::ExprKind::Path(ref qpath) => qpath, + _ => span_bug!( + expr.span, + "asm `sym` operand should be a path, found {:?}", + expr.kind + ), + }; + let temp_lifetime = + self.region_scope_tree.temporary_scope(expr.hir_id.local_id); + let res = self.typeck_results().qpath_res(qpath, expr.hir_id); + let ty; + match res { + Res::Def(DefKind::Fn, _) | Res::Def(DefKind::AssocFn, _) => { + ty = self.typeck_results().node_type(expr.hir_id); + let user_ty = + self.user_substs_applied_to_res(expr.hir_id, res); + InlineAsmOperand::SymFn { + expr: self.thir.exprs.push(Expr { + ty, + temp_lifetime, + span: expr.span, + kind: ExprKind::Literal { + literal: ty::Const::zero_sized(self.tcx, ty), + user_ty, + const_id: None, + }, + }), + } } - } - Res::Def(DefKind::Static, def_id) => { - InlineAsmOperand::SymStatic { def_id } - } + Res::Def(DefKind::Static, def_id) => { + InlineAsmOperand::SymStatic { def_id } + } - _ => { - self.tcx.sess.span_err( - expr.span, - "asm `sym` operand must point to a fn or static", - ); - - // Not a real fn, but we're not reaching codegen anyways... - ty = self.tcx.ty_error(); - InlineAsmOperand::SymFn { - expr: self.arena.alloc(Expr { - ty, - temp_lifetime, - span: expr.span, - kind: ExprKind::Literal { - literal: ty::Const::zero_sized(self.tcx, ty), - user_ty: None, - const_id: None, - }, - }), + _ => { + self.tcx.sess.span_err( + expr.span, + "asm `sym` operand must point to a fn or static", + ); + + // Not a real fn, but we're not reaching codegen anyways... + ty = self.tcx.ty_error(); + InlineAsmOperand::SymFn { + expr: self.thir.exprs.push(Expr { + ty, + temp_lifetime, + span: expr.span, + kind: ExprKind::Literal { + literal: ty::Const::zero_sized(self.tcx, ty), + user_ty: None, + const_id: None, + }, + }), + } } } } } - } - })), + }) + .collect(), options: asm.options, line_spans: asm.line_spans, }, @@ -616,13 +597,13 @@ impl<'thir, 'tcx> Cx<'thir, 'tcx> { }, hir::ExprKind::Match(ref discr, ref arms, _) => ExprKind::Match { scrutinee: self.mirror_expr(discr), - arms: self.arena.alloc_from_iter(arms.iter().map(|a| self.convert_arm(a))), + arms: arms.iter().map(|a| self.convert_arm(a)).collect(), }, hir::ExprKind::Loop(ref body, ..) => { let block_ty = self.typeck_results().node_type(body.hir_id); let temp_lifetime = self.region_scope_tree.temporary_scope(body.hir_id.local_id); let block = self.mirror_block(body); - let body = self.arena.alloc(Expr { + let body = self.thir.exprs.push(Expr { ty: block_ty, temp_lifetime, span: block.span, @@ -692,35 +673,34 @@ impl<'thir, 'tcx> Cx<'thir, 'tcx> { }; let source = if let Some((did, offset, var_ty)) = var { - let mk_const = |literal| { - self.arena.alloc(Expr { - temp_lifetime, - ty: var_ty, - span: expr.span, - kind: ExprKind::Literal { literal, user_ty: None, const_id: None }, - }) + let mk_const = |literal| Expr { + temp_lifetime, + ty: var_ty, + span: expr.span, + kind: ExprKind::Literal { literal, user_ty: None, const_id: None }, }; - let offset = mk_const(ty::Const::from_bits( + let offset = self.thir.exprs.push(mk_const(ty::Const::from_bits( self.tcx, offset as u128, self.param_env.and(var_ty), - )); + ))); match did { Some(did) => { // in case we are offsetting from a computed discriminant // and not the beginning of discriminants (which is always `0`) let substs = InternalSubsts::identity_for_item(self.tcx(), did); - let lhs = mk_const(self.tcx().mk_const(ty::Const { + let lhs = ty::Const { val: ty::ConstKind::Unevaluated(ty::Unevaluated { def: ty::WithOptConstParam::unknown(did), substs, promoted: None, }), ty: var_ty, - })); + }; + let lhs = self.thir.exprs.push(mk_const(self.tcx().mk_const(lhs))); let bin = ExprKind::Binary { op: BinOp::Add, lhs: lhs, rhs: offset }; - self.arena.alloc(Expr { + self.thir.exprs.push(Expr { temp_lifetime, ty: var_ty, span: expr.span, @@ -739,7 +719,7 @@ impl<'thir, 'tcx> Cx<'thir, 'tcx> { if let Some(user_ty) = user_ty { // NOTE: Creating a new Expr and wrapping a Cast inside of it may be // inefficient, revisit this when performance becomes an issue. - let cast_expr = self.arena.alloc(Expr { + let cast_expr = self.thir.exprs.push(Expr { temp_lifetime, ty: expr_ty, span: expr.span, @@ -819,7 +799,7 @@ impl<'thir, 'tcx> Cx<'thir, 'tcx> { expr: &hir::Expr<'_>, span: Span, overloaded_callee: Option<(DefId, SubstsRef<'tcx>)>, - ) -> Expr<'thir, 'tcx> { + ) -> Expr<'tcx> { let temp_lifetime = self.region_scope_tree.temporary_scope(expr.hir_id.local_id); let (def_id, substs, user_ty) = match overloaded_callee { Some((def_id, substs)) => (def_id, substs, None), @@ -846,8 +826,8 @@ impl<'thir, 'tcx> Cx<'thir, 'tcx> { } } - fn convert_arm(&mut self, arm: &'tcx hir::Arm<'tcx>) -> Arm<'thir, 'tcx> { - Arm { + fn convert_arm(&mut self, arm: &'tcx hir::Arm<'tcx>) -> ArmId { + let arm = Arm { pattern: self.pattern_from_hir(&arm.pat), guard: arm.guard.as_ref().map(|g| match g { hir::Guard::If(ref e) => Guard::If(self.mirror_expr(e)), @@ -859,14 +839,11 @@ impl<'thir, 'tcx> Cx<'thir, 'tcx> { lint_level: LintLevel::Explicit(arm.hir_id), scope: region::Scope { id: arm.hir_id.local_id, data: region::ScopeData::Node }, span: arm.span, - } + }; + self.thir.arms.push(arm) } - fn convert_path_expr( - &mut self, - expr: &'tcx hir::Expr<'tcx>, - res: Res, - ) -> ExprKind<'thir, 'tcx> { + fn convert_path_expr(&mut self, expr: &'tcx hir::Expr<'tcx>, res: Res) -> ExprKind<'tcx> { let substs = self.typeck_results().node_substs(expr.hir_id); match res { // A regular function, constructor function or a constant. @@ -934,7 +911,7 @@ impl<'thir, 'tcx> Cx<'thir, 'tcx> { variant_index: adt_def.variant_index_with_ctor_id(def_id), substs, user_ty: user_provided_type, - fields: self.arena.alloc_from_iter(iter::empty()), + fields: box [], base: None, }, _ => bug!("unexpected ty: {:?}", ty), @@ -956,7 +933,7 @@ impl<'thir, 'tcx> Cx<'thir, 'tcx> { } }; ExprKind::Deref { - arg: self.arena.alloc(Expr { ty, temp_lifetime, span: expr.span, kind }), + arg: self.thir.exprs.push(Expr { ty, temp_lifetime, span: expr.span, kind }), } } @@ -966,7 +943,7 @@ impl<'thir, 'tcx> Cx<'thir, 'tcx> { } } - fn convert_var(&mut self, var_hir_id: hir::HirId) -> ExprKind<'thir, 'tcx> { + fn convert_var(&mut self, var_hir_id: hir::HirId) -> ExprKind<'tcx> { // We want upvars here not captures. // Captures will be handled in MIR. let is_upvar = self @@ -989,10 +966,17 @@ impl<'thir, 'tcx> Cx<'thir, 'tcx> { fn overloaded_operator( &mut self, expr: &'tcx hir::Expr<'tcx>, - args: &'thir [Expr<'thir, 'tcx>], - ) -> ExprKind<'thir, 'tcx> { - let fun = self.arena.alloc(self.method_callee(expr, expr.span, None)); - ExprKind::Call { ty: fun.ty, fun, args, from_hir_call: false, fn_span: expr.span } + args: Box<[ExprId]>, + ) -> ExprKind<'tcx> { + let fun = self.method_callee(expr, expr.span, None); + let fun = self.thir.exprs.push(fun); + ExprKind::Call { + ty: self.thir[fun].ty, + fun, + args, + from_hir_call: false, + fn_span: expr.span, + } } fn overloaded_place( @@ -1000,9 +984,9 @@ impl<'thir, 'tcx> Cx<'thir, 'tcx> { expr: &'tcx hir::Expr<'tcx>, place_ty: Ty<'tcx>, overloaded_callee: Option<(DefId, SubstsRef<'tcx>)>, - args: &'thir [Expr<'thir, 'tcx>], + args: Box<[ExprId]>, span: Span, - ) -> ExprKind<'thir, 'tcx> { + ) -> ExprKind<'tcx> { // For an overloaded *x or x[y] expression of type T, the method // call returns an &T and we must add the deref so that the types // line up (this is because `*x` and `x[y]` represent places): @@ -1010,7 +994,7 @@ impl<'thir, 'tcx> Cx<'thir, 'tcx> { // Reconstruct the output assuming it's a reference with the // same region and mutability as the receiver. This holds for // `Deref(Mut)::Deref(_mut)` and `Index(Mut)::index(_mut)`. - let (region, mutbl) = match *args[0].ty.kind() { + let (region, mutbl) = match *self.thir[args[0]].ty.kind() { ty::Ref(region, _, mutbl) => (region, mutbl), _ => span_bug!(span, "overloaded_place: receiver is not a reference"), }; @@ -1019,12 +1003,14 @@ impl<'thir, 'tcx> Cx<'thir, 'tcx> { // construct the complete expression `foo()` for the overloaded call, // which will yield the &T type let temp_lifetime = self.region_scope_tree.temporary_scope(expr.hir_id.local_id); - let fun = self.arena.alloc(self.method_callee(expr, span, overloaded_callee)); - let ref_expr = self.arena.alloc(Expr { + let fun = self.method_callee(expr, span, overloaded_callee); + let fun = self.thir.exprs.push(fun); + let fun_ty = self.thir[fun].ty; + let ref_expr = self.thir.exprs.push(Expr { temp_lifetime, ty: ref_ty, span, - kind: ExprKind::Call { ty: fun.ty, fun, args, from_hir_call: false, fn_span: span }, + kind: ExprKind::Call { ty: fun_ty, fun, args, from_hir_call: false, fn_span: span }, }); // construct and return a deref wrapper `*foo()` @@ -1035,7 +1021,7 @@ impl<'thir, 'tcx> Cx<'thir, 'tcx> { &mut self, closure_expr: &'tcx hir::Expr<'tcx>, place: HirPlace<'tcx>, - ) -> Expr<'thir, 'tcx> { + ) -> Expr<'tcx> { let temp_lifetime = self.region_scope_tree.temporary_scope(closure_expr.hir_id.local_id); let var_ty = place.base_ty; @@ -1059,13 +1045,13 @@ impl<'thir, 'tcx> Cx<'thir, 'tcx> { for proj in place.projections.iter() { let kind = match proj.kind { HirProjectionKind::Deref => { - ExprKind::Deref { arg: self.arena.alloc(captured_place_expr) } + ExprKind::Deref { arg: self.thir.exprs.push(captured_place_expr) } } HirProjectionKind::Field(field, ..) => { // Variant index will always be 0, because for multi-variant // enums, we capture the enum entirely. ExprKind::Field { - lhs: self.arena.alloc(captured_place_expr), + lhs: self.thir.exprs.push(captured_place_expr), name: Field::new(field as usize), } } @@ -1087,7 +1073,7 @@ impl<'thir, 'tcx> Cx<'thir, 'tcx> { closure_expr: &'tcx hir::Expr<'tcx>, captured_place: &'tcx ty::CapturedPlace<'tcx>, upvar_ty: Ty<'tcx>, - ) -> Expr<'thir, 'tcx> { + ) -> Expr<'tcx> { let upvar_capture = captured_place.info.capture_kind; let captured_place_expr = self.convert_captured_hir_place(closure_expr, captured_place.place.clone()); @@ -1107,7 +1093,7 @@ impl<'thir, 'tcx> Cx<'thir, 'tcx> { span: closure_expr.span, kind: ExprKind::Borrow { borrow_kind, - arg: self.arena.alloc(captured_place_expr), + arg: self.thir.exprs.push(captured_place_expr), }, } } @@ -1115,14 +1101,14 @@ impl<'thir, 'tcx> Cx<'thir, 'tcx> { } /// Converts a list of named fields (i.e., for struct-like struct/enum ADTs) into FieldExpr. - fn field_refs( - &mut self, - fields: &'tcx [hir::ExprField<'tcx>], - ) -> &'thir [FieldExpr<'thir, 'tcx>] { - self.arena.alloc_from_iter(fields.iter().map(|field| FieldExpr { - name: Field::new(self.tcx.field_index(field.hir_id, self.typeck_results)), - expr: self.mirror_expr(field.expr), - })) + fn field_refs(&mut self, fields: &'tcx [hir::ExprField<'tcx>]) -> Box<[FieldExpr]> { + fields + .iter() + .map(|field| FieldExpr { + name: Field::new(self.tcx.field_index(field.hir_id, self.typeck_results)), + expr: self.mirror_expr(field.expr), + }) + .collect() } } diff --git a/compiler/rustc_mir_build/src/thir/cx/mod.rs b/compiler/rustc_mir_build/src/thir/cx/mod.rs index c0433604f8..49ba71e352 100644 --- a/compiler/rustc_mir_build/src/thir/cx/mod.rs +++ b/compiler/rustc_mir_build/src/thir/cx/mod.rs @@ -2,30 +2,37 @@ //! structures into the THIR. The `builder` is generally ignorant of the tcx, //! etc., and instead goes through the `Cx` for most of its work. -use crate::thir::arena::Arena; +use crate::thir::pattern::pat_from_hir; use crate::thir::util::UserAnnotatedTyHelpers; -use crate::thir::*; use rustc_ast as ast; +use rustc_data_structures::steal::Steal; use rustc_hir as hir; use rustc_hir::def_id::{DefId, LocalDefId}; use rustc_hir::Node; use rustc_middle::middle::region; use rustc_middle::mir::interpret::{LitToConstError, LitToConstInput}; +use rustc_middle::thir::*; use rustc_middle::ty::{self, Ty, TyCtxt}; +use rustc_span::Span; -pub fn build_thir<'thir, 'tcx>( +crate fn thir_body<'tcx>( tcx: TyCtxt<'tcx>, owner_def: ty::WithOptConstParam, - arena: &'thir Arena<'thir, 'tcx>, - expr: &'tcx hir::Expr<'tcx>, -) -> &'thir Expr<'thir, 'tcx> { - Cx::new(tcx, owner_def, &arena).mirror_expr(expr) +) -> (&'tcx Steal>, ExprId) { + let hir = tcx.hir(); + let body = hir.body(hir.body_owned_by(hir.local_def_id_to_hir_id(owner_def.did))); + let mut cx = Cx::new(tcx, owner_def); + if cx.typeck_results.tainted_by_errors.is_some() { + return (tcx.alloc_steal_thir(Thir::new()), ExprId::from_u32(0)); + } + let expr = cx.mirror_expr(&body.value); + (tcx.alloc_steal_thir(cx.thir), expr) } -struct Cx<'thir, 'tcx> { +struct Cx<'tcx> { tcx: TyCtxt<'tcx>, - arena: &'thir Arena<'thir, 'tcx>, + thir: Thir<'tcx>, crate param_env: ty::ParamEnv<'tcx>, @@ -36,16 +43,12 @@ struct Cx<'thir, 'tcx> { body_owner: DefId, } -impl<'thir, 'tcx> Cx<'thir, 'tcx> { - fn new( - tcx: TyCtxt<'tcx>, - def: ty::WithOptConstParam, - arena: &'thir Arena<'thir, 'tcx>, - ) -> Cx<'thir, 'tcx> { +impl<'tcx> Cx<'tcx> { + fn new(tcx: TyCtxt<'tcx>, def: ty::WithOptConstParam) -> Cx<'tcx> { let typeck_results = tcx.typeck_opt_const_arg(def); Cx { tcx, - arena, + thir: Thir::new(), param_env: tcx.param_env(def.did), region_scope_tree: tcx.region_scope_tree(def.did), typeck_results, @@ -83,11 +86,11 @@ impl<'thir, 'tcx> Cx<'thir, 'tcx> { Node::Pat(p) | Node::Binding(p) => p, node => bug!("pattern became {:?}", node), }; - Pat::from_hir(self.tcx, self.param_env, self.typeck_results(), p) + pat_from_hir(self.tcx, self.param_env, self.typeck_results(), p) } } -impl<'tcx> UserAnnotatedTyHelpers<'tcx> for Cx<'_, 'tcx> { +impl<'tcx> UserAnnotatedTyHelpers<'tcx> for Cx<'tcx> { fn tcx(&self) -> TyCtxt<'tcx> { self.tcx } diff --git a/compiler/rustc_mir_build/src/thir/mod.rs b/compiler/rustc_mir_build/src/thir/mod.rs index f4596d523d..e5123d8ef0 100644 --- a/compiler/rustc_mir_build/src/thir/mod.rs +++ b/compiler/rustc_mir_build/src/thir/mod.rs @@ -4,383 +4,11 @@ //! unit-tested and separated from the Rust source and compiler data //! structures. -use rustc_ast::{InlineAsmOptions, InlineAsmTemplatePiece}; -use rustc_hir as hir; -use rustc_hir::def_id::DefId; -use rustc_middle::infer::canonical::Canonical; -use rustc_middle::middle::region; -use rustc_middle::mir::{BinOp, BorrowKind, FakeReadCause, Field, UnOp}; -use rustc_middle::ty::adjustment::PointerCast; -use rustc_middle::ty::subst::SubstsRef; -use rustc_middle::ty::{AdtDef, Const, Ty, UpvarSubsts, UserType}; -use rustc_span::Span; -use rustc_target::abi::VariantIdx; -use rustc_target::asm::InlineAsmRegOrRegClass; - crate mod constant; crate mod cx; -pub use cx::build_thir; crate mod pattern; -pub use self::pattern::{Ascription, BindingMode, FieldPat, Pat, PatKind, PatRange, PatTyProj}; - -mod arena; -pub use arena::Arena; mod util; - -#[derive(Copy, Clone, Debug)] -pub enum LintLevel { - Inherited, - Explicit(hir::HirId), -} - -#[derive(Debug)] -pub struct Block<'thir, 'tcx> { - pub targeted_by_break: bool, - pub region_scope: region::Scope, - pub opt_destruction_scope: Option, - pub span: Span, - pub stmts: &'thir [Stmt<'thir, 'tcx>], - pub expr: Option<&'thir Expr<'thir, 'tcx>>, - pub safety_mode: BlockSafety, -} - -#[derive(Copy, Clone, Debug)] -pub enum BlockSafety { - Safe, - ExplicitUnsafe(hir::HirId), - PushUnsafe, - PopUnsafe, -} - -#[derive(Debug)] -pub struct Stmt<'thir, 'tcx> { - pub kind: StmtKind<'thir, 'tcx>, - pub opt_destruction_scope: Option, -} - -#[derive(Debug)] -pub enum StmtKind<'thir, 'tcx> { - Expr { - /// scope for this statement; may be used as lifetime of temporaries - scope: region::Scope, - - /// expression being evaluated in this statement - expr: &'thir Expr<'thir, 'tcx>, - }, - - Let { - /// scope for variables bound in this let; covers this and - /// remaining statements in block - remainder_scope: region::Scope, - - /// scope for the initialization itself; might be used as - /// lifetime of temporaries - init_scope: region::Scope, - - /// `let = ...` - /// - /// if a type is included, it is added as an ascription pattern - pattern: Pat<'tcx>, - - /// let pat: ty = ... - initializer: Option<&'thir Expr<'thir, 'tcx>>, - - /// the lint level for this let-statement - lint_level: LintLevel, - }, -} - -// `Expr` is used a lot. Make sure it doesn't unintentionally get bigger. -#[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))] -rustc_data_structures::static_assert_size!(Expr<'_, '_>, 144); - -/// The Thir trait implementor lowers their expressions (`&'tcx H::Expr`) -/// into instances of this `Expr` enum. This lowering can be done -/// basically as lazily or as eagerly as desired: every recursive -/// reference to an expression in this enum is an `&'thir Expr<'thir, 'tcx>`, which -/// may in turn be another instance of this enum (boxed), or else an -/// unlowered `&'tcx H::Expr`. Note that instances of `Expr` are very -/// short-lived. They are created by `Thir::to_expr`, analyzed and -/// converted into MIR, and then discarded. -/// -/// If you compare `Expr` to the full compiler AST, you will see it is -/// a good bit simpler. In fact, a number of the more straight-forward -/// MIR simplifications are already done in the impl of `Thir`. For -/// example, method calls and overloaded operators are absent: they are -/// expected to be converted into `Expr::Call` instances. -#[derive(Debug)] -pub struct Expr<'thir, 'tcx> { - /// type of this expression - pub ty: Ty<'tcx>, - - /// lifetime of this expression if it should be spilled into a - /// temporary; should be None only if in a constant context - pub temp_lifetime: Option, - - /// span of the expression in the source - pub span: Span, - - /// kind of expression - pub kind: ExprKind<'thir, 'tcx>, -} - -#[derive(Debug)] -pub enum ExprKind<'thir, 'tcx> { - Scope { - region_scope: region::Scope, - lint_level: LintLevel, - value: &'thir Expr<'thir, 'tcx>, - }, - Box { - value: &'thir Expr<'thir, 'tcx>, - }, - If { - cond: &'thir Expr<'thir, 'tcx>, - then: &'thir Expr<'thir, 'tcx>, - else_opt: Option<&'thir Expr<'thir, 'tcx>>, - }, - Call { - ty: Ty<'tcx>, - fun: &'thir Expr<'thir, 'tcx>, - args: &'thir [Expr<'thir, 'tcx>], - /// Whether this is from a call in HIR, rather than from an overloaded - /// operator. `true` for overloaded function call. - from_hir_call: bool, - /// This `Span` is the span of the function, without the dot and receiver - /// (e.g. `foo(a, b)` in `x.foo(a, b)` - fn_span: Span, - }, - Deref { - arg: &'thir Expr<'thir, 'tcx>, - }, // NOT overloaded! - Binary { - op: BinOp, - lhs: &'thir Expr<'thir, 'tcx>, - rhs: &'thir Expr<'thir, 'tcx>, - }, // NOT overloaded! - LogicalOp { - op: LogicalOp, - lhs: &'thir Expr<'thir, 'tcx>, - rhs: &'thir Expr<'thir, 'tcx>, - }, // NOT overloaded! - // LogicalOp is distinct from BinaryOp because of lazy evaluation of the operands. - Unary { - op: UnOp, - arg: &'thir Expr<'thir, 'tcx>, - }, // NOT overloaded! - Cast { - source: &'thir Expr<'thir, 'tcx>, - }, - Use { - source: &'thir Expr<'thir, 'tcx>, - }, // Use a lexpr to get a vexpr. - NeverToAny { - source: &'thir Expr<'thir, 'tcx>, - }, - Pointer { - cast: PointerCast, - source: &'thir Expr<'thir, 'tcx>, - }, - Loop { - body: &'thir Expr<'thir, 'tcx>, - }, - Match { - scrutinee: &'thir Expr<'thir, 'tcx>, - arms: &'thir [Arm<'thir, 'tcx>], - }, - Block { - body: Block<'thir, 'tcx>, - }, - Assign { - lhs: &'thir Expr<'thir, 'tcx>, - rhs: &'thir Expr<'thir, 'tcx>, - }, - AssignOp { - op: BinOp, - lhs: &'thir Expr<'thir, 'tcx>, - rhs: &'thir Expr<'thir, 'tcx>, - }, - Field { - lhs: &'thir Expr<'thir, 'tcx>, - name: Field, - }, - Index { - lhs: &'thir Expr<'thir, 'tcx>, - index: &'thir Expr<'thir, 'tcx>, - }, - VarRef { - id: hir::HirId, - }, - /// Used to represent upvars mentioned in a closure/generator - UpvarRef { - /// DefId of the closure/generator - closure_def_id: DefId, - - /// HirId of the root variable - var_hir_id: hir::HirId, - }, - Borrow { - borrow_kind: BorrowKind, - arg: &'thir Expr<'thir, 'tcx>, - }, - /// A `&raw [const|mut] $place_expr` raw borrow resulting in type `*[const|mut] T`. - AddressOf { - mutability: hir::Mutability, - arg: &'thir Expr<'thir, 'tcx>, - }, - Break { - label: region::Scope, - value: Option<&'thir Expr<'thir, 'tcx>>, - }, - Continue { - label: region::Scope, - }, - Return { - value: Option<&'thir Expr<'thir, 'tcx>>, - }, - ConstBlock { - value: &'tcx Const<'tcx>, - }, - Repeat { - value: &'thir Expr<'thir, 'tcx>, - count: &'tcx Const<'tcx>, - }, - Array { - fields: &'thir [Expr<'thir, 'tcx>], - }, - Tuple { - fields: &'thir [Expr<'thir, 'tcx>], - }, - Adt { - adt_def: &'tcx AdtDef, - variant_index: VariantIdx, - substs: SubstsRef<'tcx>, - - /// Optional user-given substs: for something like `let x = - /// Bar:: { ... }`. - user_ty: Option>>, - - fields: &'thir [FieldExpr<'thir, 'tcx>], - base: Option>, - }, - PlaceTypeAscription { - source: &'thir Expr<'thir, 'tcx>, - /// Type that the user gave to this expression - user_ty: Option>>, - }, - ValueTypeAscription { - source: &'thir Expr<'thir, 'tcx>, - /// Type that the user gave to this expression - user_ty: Option>>, - }, - Closure { - closure_id: DefId, - substs: UpvarSubsts<'tcx>, - upvars: &'thir [Expr<'thir, 'tcx>], - movability: Option, - fake_reads: Vec<(&'thir Expr<'thir, 'tcx>, FakeReadCause, hir::HirId)>, - }, - Literal { - literal: &'tcx Const<'tcx>, - user_ty: Option>>, - /// The `DefId` of the `const` item this literal - /// was produced from, if this is not a user-written - /// literal value. - const_id: Option, - }, - /// A literal containing the address of a `static`. - /// - /// This is only distinguished from `Literal` so that we can register some - /// info for diagnostics. - StaticRef { - literal: &'tcx Const<'tcx>, - def_id: DefId, - }, - InlineAsm { - template: &'tcx [InlineAsmTemplatePiece], - operands: &'thir [InlineAsmOperand<'thir, 'tcx>], - options: InlineAsmOptions, - line_spans: &'tcx [Span], - }, - /// An expression taking a reference to a thread local. - ThreadLocalRef(DefId), - LlvmInlineAsm { - asm: &'tcx hir::LlvmInlineAsmInner, - outputs: &'thir [Expr<'thir, 'tcx>], - inputs: &'thir [Expr<'thir, 'tcx>], - }, - Yield { - value: &'thir Expr<'thir, 'tcx>, - }, -} - -#[derive(Debug)] -pub struct FieldExpr<'thir, 'tcx> { - pub name: Field, - pub expr: &'thir Expr<'thir, 'tcx>, -} - -#[derive(Debug)] -pub struct FruInfo<'thir, 'tcx> { - pub base: &'thir Expr<'thir, 'tcx>, - pub field_types: &'thir [Ty<'tcx>], -} - -#[derive(Debug)] -pub struct Arm<'thir, 'tcx> { - pub pattern: Pat<'tcx>, - pub guard: Option>, - pub body: &'thir Expr<'thir, 'tcx>, - pub lint_level: LintLevel, - pub scope: region::Scope, - pub span: Span, -} - -#[derive(Debug)] -pub enum Guard<'thir, 'tcx> { - If(&'thir Expr<'thir, 'tcx>), - IfLet(Pat<'tcx>, &'thir Expr<'thir, 'tcx>), -} - -#[derive(Copy, Clone, Debug)] -pub enum LogicalOp { - And, - Or, -} - -#[derive(Debug)] -pub enum InlineAsmOperand<'thir, 'tcx> { - In { - reg: InlineAsmRegOrRegClass, - expr: &'thir Expr<'thir, 'tcx>, - }, - Out { - reg: InlineAsmRegOrRegClass, - late: bool, - expr: Option<&'thir Expr<'thir, 'tcx>>, - }, - InOut { - reg: InlineAsmRegOrRegClass, - late: bool, - expr: &'thir Expr<'thir, 'tcx>, - }, - SplitInOut { - reg: InlineAsmRegOrRegClass, - late: bool, - in_expr: &'thir Expr<'thir, 'tcx>, - out_expr: Option<&'thir Expr<'thir, 'tcx>>, - }, - Const { - value: &'tcx Const<'tcx>, - span: Span, - }, - SymFn { - expr: &'thir Expr<'thir, 'tcx>, - }, - SymStatic { - def_id: DefId, - }, -} +pub mod visit; diff --git a/compiler/rustc_mir_build/src/thir/pattern/check_match.rs b/compiler/rustc_mir_build/src/thir/pattern/check_match.rs index fdecbb9478..389a759531 100644 --- a/compiler/rustc_mir_build/src/thir/pattern/check_match.rs +++ b/compiler/rustc_mir_build/src/thir/pattern/check_match.rs @@ -1,8 +1,8 @@ use super::usefulness::{ - compute_match_usefulness, expand_pattern, MatchArm, MatchCheckCtxt, Reachability, + compute_match_usefulness, expand_pattern, is_wildcard, MatchArm, MatchCheckCtxt, Reachability, UsefulnessReport, }; -use super::{PatCtxt, PatKind, PatternError}; +use super::{PatCtxt, PatternError}; use rustc_arena::TypedArena; use rustc_ast::Mutability; @@ -12,6 +12,7 @@ use rustc_hir::def::*; use rustc_hir::def_id::DefId; use rustc_hir::intravisit::{self, NestedVisitorMap, Visitor}; use rustc_hir::{HirId, Pat}; +use rustc_middle::thir::PatKind; use rustc_middle::ty::{self, Ty, TyCtxt}; use rustc_session::lint::builtin::BINDINGS_WITH_VARIANT_NAME; use rustc_session::lint::builtin::{IRREFUTABLE_LET_PATTERNS, UNREACHABLE_PATTERNS}; @@ -344,7 +345,7 @@ fn check_for_bindings_named_same_as_variants(cx: &MatchVisitor<'_, '_>, pat: &Pa /// Checks for common cases of "catchall" patterns that may not be intended as such. fn pat_is_catchall(pat: &super::Pat<'_>) -> bool { - use super::PatKind::*; + use PatKind::*; match &*pat.kind { Binding { subpattern: None, .. } => true, Binding { subpattern: Some(s), .. } | Deref { subpattern: s } => pat_is_catchall(s), @@ -496,16 +497,25 @@ fn non_exhaustive_match<'p, 'tcx>( err.span_label(sp, pattern_not_covered_label(&witnesses, &joined_patterns)); }; + let is_variant_list_non_exhaustive = match scrut_ty.kind() { + ty::Adt(def, _) if def.is_variant_list_non_exhaustive() && !def.did.is_local() => true, + _ => false, + }; + adt_defined_here(cx, &mut err, scrut_ty, &witnesses); err.help( "ensure that all possible cases are being handled, \ possibly by adding wildcards or more match arms", ); - err.note(&format!("the matched value is of type `{}`", scrut_ty)); + err.note(&format!( + "the matched value is of type `{}`{}", + scrut_ty, + if is_variant_list_non_exhaustive { ", which is marked as non-exhaustive" } else { "" } + )); if (scrut_ty == cx.tcx.types.usize || scrut_ty == cx.tcx.types.isize) && !is_empty_match && witnesses.len() == 1 - && witnesses[0].is_wildcard() + && is_wildcard(&witnesses[0]) { err.note(&format!( "`{}` does not have a fixed maximum value, \ diff --git a/compiler/rustc_mir_build/src/thir/pattern/const_to_pat.rs b/compiler/rustc_mir_build/src/thir/pattern/const_to_pat.rs index c0624c805a..369fff0045 100644 --- a/compiler/rustc_mir_build/src/thir/pattern/const_to_pat.rs +++ b/compiler/rustc_mir_build/src/thir/pattern/const_to_pat.rs @@ -2,6 +2,7 @@ use rustc_hir as hir; use rustc_index::vec::Idx; use rustc_infer::infer::{InferCtxt, TyCtxtInferExt}; use rustc_middle::mir::Field; +use rustc_middle::thir::{FieldPat, Pat, PatKind}; use rustc_middle::ty::print::with_no_trimmed_paths; use rustc_middle::ty::{self, AdtDef, Ty, TyCtxt}; use rustc_session::lint; @@ -12,7 +13,7 @@ use rustc_trait_selection::traits::{self, ObligationCause, PredicateObligation}; use std::cell::Cell; -use super::{FieldPat, Pat, PatCtxt, PatKind}; +use super::PatCtxt; impl<'a, 'tcx> PatCtxt<'a, 'tcx> { /// Converts an evaluated constant to a pattern (if possible). diff --git a/compiler/rustc_mir_build/src/thir/pattern/deconstruct_pat.rs b/compiler/rustc_mir_build/src/thir/pattern/deconstruct_pat.rs index 8c740a7ec1..4b5b648c50 100644 --- a/compiler/rustc_mir_build/src/thir/pattern/deconstruct_pat.rs +++ b/compiler/rustc_mir_build/src/thir/pattern/deconstruct_pat.rs @@ -46,16 +46,15 @@ use self::Constructor::*; use self::SliceKind::*; use super::compare_const_vals; -use super::usefulness::{MatchCheckCtxt, PatCtxt}; -use super::{FieldPat, Pat, PatKind, PatRange}; +use super::usefulness::{is_wildcard, MatchCheckCtxt, PatCtxt}; use rustc_data_structures::captures::Captures; use rustc_index::vec::Idx; -use rustc_hir::def_id::DefId; use rustc_hir::{HirId, RangeEnd}; use rustc_middle::mir::interpret::ConstValue; use rustc_middle::mir::Field; +use rustc_middle::thir::{FieldPat, Pat, PatKind, PatRange}; use rustc_middle::ty::layout::IntegerExt; use rustc_middle::ty::{self, Const, Ty, TyCtxt}; use rustc_session::lint; @@ -590,7 +589,7 @@ pub(super) enum Constructor<'tcx> { /// and fixed-length arrays. Single, /// Enum variants. - Variant(DefId), + Variant(VariantIdx), /// Ranges of integer literal values (`2`, `2..=5` or `2..5`). IntRange(IntRange), /// Ranges of floating-point literal values (`2.0..=5.2`). @@ -634,7 +633,7 @@ impl<'tcx> Constructor<'tcx> { fn variant_index_for_adt(&self, adt: &'tcx ty::AdtDef) -> VariantIdx { match *self { - Variant(id) => adt.variant_index_with_id(id), + Variant(idx) => idx, Single => { assert!(!adt.is_enum()); VariantIdx::new(0) @@ -649,9 +648,7 @@ impl<'tcx> Constructor<'tcx> { PatKind::AscribeUserType { .. } => bug!(), // Handled by `expand_pattern` PatKind::Binding { .. } | PatKind::Wild => Wildcard, PatKind::Leaf { .. } | PatKind::Deref { .. } => Single, - &PatKind::Variant { adt_def, variant_index, .. } => { - Variant(adt_def.variants[variant_index].def_id) - } + &PatKind::Variant { variant_index, .. } => Variant(variant_index), PatKind::Constant { value } => { if let Some(int_range) = IntRange::from_const(cx.tcx, cx.param_env, value) { IntRange(int_range) @@ -928,15 +925,15 @@ impl<'tcx> SplitWildcard<'tcx> { // If `exhaustive_patterns` is enabled, we exclude variants known to be // uninhabited. def.variants - .iter() - .filter(|v| { + .iter_enumerated() + .filter(|(_, v)| { !v.uninhabited_from(cx.tcx, substs, def.adt_kind(), cx.param_env) .contains(cx.tcx, cx.module) }) - .map(|v| Variant(v.def_id)) + .map(|(idx, _)| Variant(idx)) .collect() } else { - def.variants.iter().map(|v| Variant(v.def_id)).collect() + def.variants.indices().map(|idx| Variant(idx)).collect() } } ty::Char => { @@ -1248,13 +1245,13 @@ impl<'p, 'tcx> Fields<'p, 'tcx> { // of reporting `[x, _, .., _, y]`, we prefer to report `[x, .., y]`. // This is incorrect if the size is not known, since `[_, ..]` captures // arrays of lengths `>= 1` whereas `[..]` captures any length. - while !prefix.is_empty() && prefix.last().unwrap().is_wildcard() { + while !prefix.is_empty() && is_wildcard(prefix.last().unwrap()) { prefix.pop(); } } let suffix: Vec<_> = if slice.array_len.is_some() { // Same as above. - subpatterns.skip_while(Pat::is_wildcard).collect() + subpatterns.skip_while(is_wildcard).collect() } else { subpatterns.collect() }; diff --git a/compiler/rustc_mir_build/src/thir/pattern/mod.rs b/compiler/rustc_mir_build/src/thir/pattern/mod.rs index 9ac79a37ac..3225d302cb 100644 --- a/compiler/rustc_mir_build/src/thir/pattern/mod.rs +++ b/compiler/rustc_mir_build/src/thir/pattern/mod.rs @@ -11,7 +11,7 @@ use crate::thir::util::UserAnnotatedTyHelpers; use rustc_errors::struct_span_err; use rustc_hir as hir; -use rustc_hir::def::{CtorKind, CtorOf, DefKind, Res}; +use rustc_hir::def::{CtorOf, DefKind, Res}; use rustc_hir::pat_util::EnumerateAndAdjustIterator; use rustc_hir::RangeEnd; use rustc_index::vec::Idx; @@ -19,16 +19,12 @@ use rustc_middle::mir::interpret::{get_slice_bytes, ConstValue}; use rustc_middle::mir::interpret::{ErrorHandled, LitToConstError, LitToConstInput}; use rustc_middle::mir::UserTypeProjection; use rustc_middle::mir::{BorrowKind, Field, Mutability}; +use rustc_middle::thir::{Ascription, BindingMode, FieldPat, Pat, PatKind, PatRange, PatTyProj}; use rustc_middle::ty::subst::{GenericArg, SubstsRef}; use rustc_middle::ty::{self, AdtDef, DefIdTree, Region, Ty, TyCtxt, UserType}; -use rustc_middle::ty::{ - CanonicalUserType, CanonicalUserTypeAnnotation, CanonicalUserTypeAnnotations, -}; -use rustc_span::{Span, Symbol, DUMMY_SP}; -use rustc_target::abi::VariantIdx; +use rustc_span::{Span, Symbol}; use std::cmp::Ordering; -use std::fmt; #[derive(Clone, Debug)] crate enum PatternError { @@ -39,317 +35,6 @@ crate enum PatternError { NonConstPath(Span), } -#[derive(Copy, Clone, Debug, PartialEq)] -pub enum BindingMode { - ByValue, - ByRef(BorrowKind), -} - -#[derive(Clone, Debug, PartialEq)] -pub struct FieldPat<'tcx> { - pub field: Field, - pub pattern: Pat<'tcx>, -} - -#[derive(Clone, Debug, PartialEq)] -pub struct Pat<'tcx> { - pub ty: Ty<'tcx>, - pub span: Span, - pub kind: Box>, -} - -impl<'tcx> Pat<'tcx> { - pub(crate) fn wildcard_from_ty(ty: Ty<'tcx>) -> Self { - Pat { ty, span: DUMMY_SP, kind: Box::new(PatKind::Wild) } - } -} - -#[derive(Copy, Clone, Debug, PartialEq)] -pub struct PatTyProj<'tcx> { - pub user_ty: CanonicalUserType<'tcx>, -} - -impl<'tcx> PatTyProj<'tcx> { - pub(crate) fn from_user_type(user_annotation: CanonicalUserType<'tcx>) -> Self { - Self { user_ty: user_annotation } - } - - pub(crate) fn user_ty( - self, - annotations: &mut CanonicalUserTypeAnnotations<'tcx>, - inferred_ty: Ty<'tcx>, - span: Span, - ) -> UserTypeProjection { - UserTypeProjection { - base: annotations.push(CanonicalUserTypeAnnotation { - span, - user_ty: self.user_ty, - inferred_ty, - }), - projs: Vec::new(), - } - } -} - -#[derive(Copy, Clone, Debug, PartialEq)] -pub struct Ascription<'tcx> { - pub user_ty: PatTyProj<'tcx>, - /// Variance to use when relating the type `user_ty` to the **type of the value being - /// matched**. Typically, this is `Variance::Covariant`, since the value being matched must - /// have a type that is some subtype of the ascribed type. - /// - /// Note that this variance does not apply for any bindings within subpatterns. The type - /// assigned to those bindings must be exactly equal to the `user_ty` given here. - /// - /// The only place where this field is not `Covariant` is when matching constants, where - /// we currently use `Contravariant` -- this is because the constant type just needs to - /// be "comparable" to the type of the input value. So, for example: - /// - /// ```text - /// match x { "foo" => .. } - /// ``` - /// - /// requires that `&'static str <: T_x`, where `T_x` is the type of `x`. Really, we should - /// probably be checking for a `PartialEq` impl instead, but this preserves the behavior - /// of the old type-check for now. See #57280 for details. - pub variance: ty::Variance, - pub user_ty_span: Span, -} - -#[derive(Clone, Debug, PartialEq)] -pub enum PatKind<'tcx> { - Wild, - - AscribeUserType { - ascription: Ascription<'tcx>, - subpattern: Pat<'tcx>, - }, - - /// `x`, `ref x`, `x @ P`, etc. - Binding { - mutability: Mutability, - name: Symbol, - mode: BindingMode, - var: hir::HirId, - ty: Ty<'tcx>, - subpattern: Option>, - /// Is this the leftmost occurrence of the binding, i.e., is `var` the - /// `HirId` of this pattern? - is_primary: bool, - }, - - /// `Foo(...)` or `Foo{...}` or `Foo`, where `Foo` is a variant name from an ADT with - /// multiple variants. - Variant { - adt_def: &'tcx AdtDef, - substs: SubstsRef<'tcx>, - variant_index: VariantIdx, - subpatterns: Vec>, - }, - - /// `(...)`, `Foo(...)`, `Foo{...}`, or `Foo`, where `Foo` is a variant name from an ADT with - /// a single variant. - Leaf { - subpatterns: Vec>, - }, - - /// `box P`, `&P`, `&mut P`, etc. - Deref { - subpattern: Pat<'tcx>, - }, - - /// One of the following: - /// * `&str`, which will be handled as a string pattern and thus exhaustiveness - /// checking will detect if you use the same string twice in different patterns. - /// * integer, bool, char or float, which will be handled by exhaustivenes to cover exactly - /// its own value, similar to `&str`, but these values are much simpler. - /// * Opaque constants, that must not be matched structurally. So anything that does not derive - /// `PartialEq` and `Eq`. - Constant { - value: &'tcx ty::Const<'tcx>, - }, - - Range(PatRange<'tcx>), - - /// Matches against a slice, checking the length and extracting elements. - /// irrefutable when there is a slice pattern and both `prefix` and `suffix` are empty. - /// e.g., `&[ref xs @ ..]`. - Slice { - prefix: Vec>, - slice: Option>, - suffix: Vec>, - }, - - /// Fixed match against an array; irrefutable. - Array { - prefix: Vec>, - slice: Option>, - suffix: Vec>, - }, - - /// An or-pattern, e.g. `p | q`. - /// Invariant: `pats.len() >= 2`. - Or { - pats: Vec>, - }, -} - -#[derive(Copy, Clone, Debug, PartialEq)] -pub struct PatRange<'tcx> { - pub lo: &'tcx ty::Const<'tcx>, - pub hi: &'tcx ty::Const<'tcx>, - pub end: RangeEnd, -} - -impl<'tcx> fmt::Display for Pat<'tcx> { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - // Printing lists is a chore. - let mut first = true; - let mut start_or_continue = |s| { - if first { - first = false; - "" - } else { - s - } - }; - let mut start_or_comma = || start_or_continue(", "); - - match *self.kind { - PatKind::Wild => write!(f, "_"), - PatKind::AscribeUserType { ref subpattern, .. } => write!(f, "{}: _", subpattern), - PatKind::Binding { mutability, name, mode, ref subpattern, .. } => { - let is_mut = match mode { - BindingMode::ByValue => mutability == Mutability::Mut, - BindingMode::ByRef(bk) => { - write!(f, "ref ")?; - matches!(bk, BorrowKind::Mut { .. }) - } - }; - if is_mut { - write!(f, "mut ")?; - } - write!(f, "{}", name)?; - if let Some(ref subpattern) = *subpattern { - write!(f, " @ {}", subpattern)?; - } - Ok(()) - } - PatKind::Variant { ref subpatterns, .. } | PatKind::Leaf { ref subpatterns } => { - let variant = match *self.kind { - PatKind::Variant { adt_def, variant_index, .. } => { - Some(&adt_def.variants[variant_index]) - } - _ => { - if let ty::Adt(adt, _) = self.ty.kind() { - if !adt.is_enum() { - Some(&adt.variants[VariantIdx::new(0)]) - } else { - None - } - } else { - None - } - } - }; - - if let Some(variant) = variant { - write!(f, "{}", variant.ident)?; - - // Only for Adt we can have `S {...}`, - // which we handle separately here. - if variant.ctor_kind == CtorKind::Fictive { - write!(f, " {{ ")?; - - let mut printed = 0; - for p in subpatterns { - if let PatKind::Wild = *p.pattern.kind { - continue; - } - let name = variant.fields[p.field.index()].ident; - write!(f, "{}{}: {}", start_or_comma(), name, p.pattern)?; - printed += 1; - } - - if printed < variant.fields.len() { - write!(f, "{}..", start_or_comma())?; - } - - return write!(f, " }}"); - } - } - - let num_fields = variant.map_or(subpatterns.len(), |v| v.fields.len()); - if num_fields != 0 || variant.is_none() { - write!(f, "(")?; - for i in 0..num_fields { - write!(f, "{}", start_or_comma())?; - - // Common case: the field is where we expect it. - if let Some(p) = subpatterns.get(i) { - if p.field.index() == i { - write!(f, "{}", p.pattern)?; - continue; - } - } - - // Otherwise, we have to go looking for it. - if let Some(p) = subpatterns.iter().find(|p| p.field.index() == i) { - write!(f, "{}", p.pattern)?; - } else { - write!(f, "_")?; - } - } - write!(f, ")")?; - } - - Ok(()) - } - PatKind::Deref { ref subpattern } => { - match self.ty.kind() { - ty::Adt(def, _) if def.is_box() => write!(f, "box ")?, - ty::Ref(_, _, mutbl) => { - write!(f, "&{}", mutbl.prefix_str())?; - } - _ => bug!("{} is a bad Deref pattern type", self.ty), - } - write!(f, "{}", subpattern) - } - PatKind::Constant { value } => write!(f, "{}", value), - PatKind::Range(PatRange { lo, hi, end }) => { - write!(f, "{}", lo)?; - write!(f, "{}", end)?; - write!(f, "{}", hi) - } - PatKind::Slice { ref prefix, ref slice, ref suffix } - | PatKind::Array { ref prefix, ref slice, ref suffix } => { - write!(f, "[")?; - for p in prefix { - write!(f, "{}{}", start_or_comma(), p)?; - } - if let Some(ref slice) = *slice { - write!(f, "{}", start_or_comma())?; - match *slice.kind { - PatKind::Wild => {} - _ => write!(f, "{}", slice)?, - } - write!(f, "..")?; - } - for p in suffix { - write!(f, "{}{}", start_or_comma(), p)?; - } - write!(f, "]") - } - PatKind::Or { ref pats } => { - for pat in pats { - write!(f, "{}{}", start_or_continue(" | "), pat)?; - } - Ok(()) - } - } - } -} - crate struct PatCtxt<'a, 'tcx> { crate tcx: TyCtxt<'tcx>, crate param_env: ty::ParamEnv<'tcx>, @@ -358,22 +43,20 @@ crate struct PatCtxt<'a, 'tcx> { include_lint_checks: bool, } -impl<'a, 'tcx> Pat<'tcx> { - crate fn from_hir( - tcx: TyCtxt<'tcx>, - param_env: ty::ParamEnv<'tcx>, - typeck_results: &'a ty::TypeckResults<'tcx>, - pat: &'tcx hir::Pat<'tcx>, - ) -> Self { - let mut pcx = PatCtxt::new(tcx, param_env, typeck_results); - let result = pcx.lower_pattern(pat); - if !pcx.errors.is_empty() { - let msg = format!("encountered errors lowering pattern: {:?}", pcx.errors); - tcx.sess.delay_span_bug(pat.span, &msg); - } - debug!("Pat::from_hir({:?}) = {:?}", pat, result); - result - } +crate fn pat_from_hir<'a, 'tcx>( + tcx: TyCtxt<'tcx>, + param_env: ty::ParamEnv<'tcx>, + typeck_results: &'a ty::TypeckResults<'tcx>, + pat: &'tcx hir::Pat<'tcx>, +) -> Pat<'tcx> { + let mut pcx = PatCtxt::new(tcx, param_env, typeck_results); + let result = pcx.lower_pattern(pat); + if !pcx.errors.is_empty() { + let msg = format!("encountered errors lowering pattern: {:?}", pcx.errors); + tcx.sess.delay_span_bug(pat.span, &msg); + } + debug!("pat_from_hir({:?}) = {:?}", pat, result); + result } impl<'a, 'tcx> PatCtxt<'a, 'tcx> { diff --git a/compiler/rustc_mir_build/src/thir/pattern/usefulness.rs b/compiler/rustc_mir_build/src/thir/pattern/usefulness.rs index dce0df8473..5d4eb75155 100644 --- a/compiler/rustc_mir_build/src/thir/pattern/usefulness.rs +++ b/compiler/rustc_mir_build/src/thir/pattern/usefulness.rs @@ -284,7 +284,6 @@ use self::Usefulness::*; use self::WitnessPreference::*; use super::deconstruct_pat::{Constructor, Fields, SplitWildcard}; -use super::{Pat, PatKind}; use super::{PatternFoldable, PatternFolder}; use rustc_data_structures::captures::Captures; @@ -293,6 +292,7 @@ use rustc_data_structures::fx::FxHashMap; use rustc_arena::TypedArena; use rustc_hir::def_id::DefId; use rustc_hir::HirId; +use rustc_middle::thir::{Pat, PatKind}; use rustc_middle::ty::{self, Ty, TyCtxt}; use rustc_span::Span; @@ -382,31 +382,29 @@ impl<'tcx> PatternFolder<'tcx> for LiteralExpander { } } -impl<'tcx> Pat<'tcx> { - pub(super) fn is_wildcard(&self) -> bool { - matches!(*self.kind, PatKind::Binding { subpattern: None, .. } | PatKind::Wild) - } +pub(super) fn is_wildcard(pat: &Pat<'_>) -> bool { + matches!(*pat.kind, PatKind::Binding { subpattern: None, .. } | PatKind::Wild) +} - fn is_or_pat(&self) -> bool { - matches!(*self.kind, PatKind::Or { .. }) - } +fn is_or_pat(pat: &Pat<'_>) -> bool { + matches!(*pat.kind, PatKind::Or { .. }) +} - /// Recursively expand this pattern into its subpatterns. Only useful for or-patterns. - fn expand_or_pat(&self) -> Vec<&Self> { - fn expand<'p, 'tcx>(pat: &'p Pat<'tcx>, vec: &mut Vec<&'p Pat<'tcx>>) { - if let PatKind::Or { pats } = pat.kind.as_ref() { - for pat in pats { - expand(pat, vec); - } - } else { - vec.push(pat) +/// Recursively expand this pattern into its subpatterns. Only useful for or-patterns. +fn expand_or_pat<'p, 'tcx>(pat: &'p Pat<'tcx>) -> Vec<&'p Pat<'tcx>> { + fn expand<'p, 'tcx>(pat: &'p Pat<'tcx>, vec: &mut Vec<&'p Pat<'tcx>>) { + if let PatKind::Or { pats } = pat.kind.as_ref() { + for pat in pats { + expand(pat, vec); } + } else { + vec.push(pat) } - - let mut pats = Vec::new(); - expand(self, &mut pats); - pats } + + let mut pats = Vec::new(); + expand(pat, &mut pats); + pats } /// A row of a matrix. Rows of len 1 are very common, which is why `SmallVec[_; 2]` @@ -451,7 +449,7 @@ impl<'p, 'tcx> PatStack<'p, 'tcx> { // Recursively expand the first pattern into its subpatterns. Only useful if the pattern is an // or-pattern. Panics if `self` is empty. fn expand_or_pat<'a>(&'a self) -> impl Iterator> + Captures<'a> { - self.head().expand_or_pat().into_iter().map(move |pat| { + expand_or_pat(self.head()).into_iter().map(move |pat| { let mut new_patstack = PatStack::from_pattern(pat); new_patstack.pats.extend_from_slice(&self.pats[1..]); new_patstack @@ -525,7 +523,7 @@ impl<'p, 'tcx> Matrix<'p, 'tcx> { /// Pushes a new row to the matrix. If the row starts with an or-pattern, this recursively /// expands it. fn push(&mut self, row: PatStack<'p, 'tcx>) { - if !row.is_empty() && row.head().is_or_pat() { + if !row.is_empty() && is_or_pat(row.head()) { for row in row.expand_or_pat() { self.patterns.push(row); } @@ -760,7 +758,7 @@ impl<'p, 'tcx> SubPatSet<'p, 'tcx> { } } SubPatSet::Alt { subpats, pat, alt_count, .. } => { - let expanded = pat.expand_or_pat(); + let expanded = expand_or_pat(pat); for i in 0..*alt_count { let sub_set = subpats.get(&i).unwrap_or(&SubPatSet::Empty); if sub_set.is_empty() { @@ -1118,7 +1116,7 @@ fn is_useful<'p, 'tcx>( let pcx = PatCtxt { cx, ty, span: v.head().span, is_top_level }; // If the first pattern is an or-pattern, expand it. - let ret = if v.head().is_or_pat() { + let ret = if is_or_pat(v.head()) { debug!("expanding or-pattern"); let v_head = v.head(); let vs: Vec<_> = v.expand_or_pat().collect(); @@ -1174,7 +1172,7 @@ fn is_useful<'p, 'tcx>( #[derive(Clone, Copy)] crate struct MatchArm<'p, 'tcx> { /// The pattern must have been lowered through `check_match::MatchVisitor::lower_pattern`. - crate pat: &'p super::Pat<'tcx>, + crate pat: &'p Pat<'tcx>, crate hir_id: HirId, crate has_guard: bool, } @@ -1196,7 +1194,7 @@ crate struct UsefulnessReport<'p, 'tcx> { crate arm_usefulness: Vec<(MatchArm<'p, 'tcx>, Reachability)>, /// If the match is exhaustive, this is empty. If not, this contains witnesses for the lack of /// exhaustiveness. - crate non_exhaustiveness_witnesses: Vec>, + crate non_exhaustiveness_witnesses: Vec>, } /// The entrypoint for the usefulness algorithm. Computes whether a match is exhaustive and which @@ -1232,7 +1230,7 @@ crate fn compute_match_usefulness<'p, 'tcx>( }) .collect(); - let wild_pattern = cx.pattern_arena.alloc(super::Pat::wildcard_from_ty(scrut_ty)); + let wild_pattern = cx.pattern_arena.alloc(Pat::wildcard_from_ty(scrut_ty)); let v = PatStack::from_pattern(wild_pattern); let usefulness = is_useful(cx, &matrix, &v, ConstructWitness, scrut_hir_id, false, true); let non_exhaustiveness_witnesses = match usefulness { diff --git a/compiler/rustc_mir_build/src/thir/visit.rs b/compiler/rustc_mir_build/src/thir/visit.rs new file mode 100644 index 0000000000..1a60b1de7f --- /dev/null +++ b/compiler/rustc_mir_build/src/thir/visit.rs @@ -0,0 +1,179 @@ +use rustc_middle::thir::*; +use rustc_middle::ty::Const; + +pub trait Visitor<'a, 'tcx: 'a>: Sized { + fn thir(&self) -> &'a Thir<'tcx>; + + fn visit_expr(&mut self, expr: &Expr<'tcx>) { + walk_expr(self, expr); + } + + fn visit_stmt(&mut self, stmt: &Stmt<'tcx>) { + walk_stmt(self, stmt); + } + + fn visit_block(&mut self, block: &Block) { + walk_block(self, block); + } + + fn visit_arm(&mut self, arm: &Arm<'tcx>) { + walk_arm(self, arm); + } + + fn visit_const(&mut self, _cnst: &'tcx Const<'tcx>) {} +} + +pub fn walk_expr<'a, 'tcx: 'a, V: Visitor<'a, 'tcx>>(visitor: &mut V, expr: &Expr<'tcx>) { + use ExprKind::*; + match expr.kind { + Scope { value, region_scope: _, lint_level: _ } => { + visitor.visit_expr(&visitor.thir()[value]) + } + Box { value } => visitor.visit_expr(&visitor.thir()[value]), + If { cond, then, else_opt } => { + visitor.visit_expr(&visitor.thir()[cond]); + visitor.visit_expr(&visitor.thir()[then]); + if let Some(else_expr) = else_opt { + visitor.visit_expr(&visitor.thir()[else_expr]); + } + } + Call { fun, ref args, ty: _, from_hir_call: _, fn_span: _ } => { + visitor.visit_expr(&visitor.thir()[fun]); + for &arg in &**args { + visitor.visit_expr(&visitor.thir()[arg]); + } + } + Deref { arg } => visitor.visit_expr(&visitor.thir()[arg]), + Binary { lhs, rhs, op: _ } | LogicalOp { lhs, rhs, op: _ } => { + visitor.visit_expr(&visitor.thir()[lhs]); + visitor.visit_expr(&visitor.thir()[rhs]); + } + Unary { arg, op: _ } => visitor.visit_expr(&visitor.thir()[arg]), + Cast { source } => visitor.visit_expr(&visitor.thir()[source]), + Use { source } => visitor.visit_expr(&visitor.thir()[source]), + NeverToAny { source } => visitor.visit_expr(&visitor.thir()[source]), + Pointer { source, cast: _ } => visitor.visit_expr(&visitor.thir()[source]), + Loop { body } => visitor.visit_expr(&visitor.thir()[body]), + Match { scrutinee, ref arms } => { + visitor.visit_expr(&visitor.thir()[scrutinee]); + for &arm in &**arms { + visitor.visit_arm(&visitor.thir()[arm]); + } + } + Block { ref body } => visitor.visit_block(body), + Assign { lhs, rhs } | AssignOp { lhs, rhs, op: _ } => { + visitor.visit_expr(&visitor.thir()[lhs]); + visitor.visit_expr(&visitor.thir()[rhs]); + } + Field { lhs, name: _ } => visitor.visit_expr(&visitor.thir()[lhs]), + Index { lhs, index } => { + visitor.visit_expr(&visitor.thir()[lhs]); + visitor.visit_expr(&visitor.thir()[index]); + } + VarRef { id: _ } | UpvarRef { closure_def_id: _, var_hir_id: _ } => {} + Borrow { arg, borrow_kind: _ } => visitor.visit_expr(&visitor.thir()[arg]), + AddressOf { arg, mutability: _ } => visitor.visit_expr(&visitor.thir()[arg]), + Break { value, label: _ } => { + if let Some(value) = value { + visitor.visit_expr(&visitor.thir()[value]) + } + } + Continue { label: _ } => {} + Return { value } => { + if let Some(value) = value { + visitor.visit_expr(&visitor.thir()[value]) + } + } + ConstBlock { value } => visitor.visit_const(value), + Repeat { value, count } => { + visitor.visit_expr(&visitor.thir()[value]); + visitor.visit_const(count); + } + Array { ref fields } | Tuple { ref fields } => { + for &field in &**fields { + visitor.visit_expr(&visitor.thir()[field]); + } + } + Adt { ref fields, ref base, adt_def: _, variant_index: _, substs: _, user_ty: _ } => { + for field in &**fields { + visitor.visit_expr(&visitor.thir()[field.expr]); + } + if let Some(base) = base { + visitor.visit_expr(&visitor.thir()[base.base]); + } + } + PlaceTypeAscription { source, user_ty: _ } | ValueTypeAscription { source, user_ty: _ } => { + visitor.visit_expr(&visitor.thir()[source]) + } + Closure { closure_id: _, substs: _, upvars: _, movability: _, fake_reads: _ } => {} + Literal { literal, user_ty: _, const_id: _ } => visitor.visit_const(literal), + StaticRef { literal, def_id: _ } => visitor.visit_const(literal), + InlineAsm { ref operands, template: _, options: _, line_spans: _ } => { + for op in &**operands { + use InlineAsmOperand::*; + match op { + In { expr, reg: _ } + | Out { expr: Some(expr), reg: _, late: _ } + | InOut { expr, reg: _, late: _ } + | SymFn { expr } => visitor.visit_expr(&visitor.thir()[*expr]), + SplitInOut { in_expr, out_expr, reg: _, late: _ } => { + visitor.visit_expr(&visitor.thir()[*in_expr]); + if let Some(out_expr) = out_expr { + visitor.visit_expr(&visitor.thir()[*out_expr]); + } + } + Out { expr: None, reg: _, late: _ } + | Const { value: _, span: _ } + | SymStatic { def_id: _ } => {} + } + } + } + ThreadLocalRef(_) => {} + LlvmInlineAsm { ref outputs, ref inputs, asm: _ } => { + for &out_expr in &**outputs { + visitor.visit_expr(&visitor.thir()[out_expr]); + } + for &in_expr in &**inputs { + visitor.visit_expr(&visitor.thir()[in_expr]); + } + } + Yield { value } => visitor.visit_expr(&visitor.thir()[value]), + } +} + +pub fn walk_stmt<'a, 'tcx: 'a, V: Visitor<'a, 'tcx>>(visitor: &mut V, stmt: &Stmt<'tcx>) { + match stmt.kind { + StmtKind::Expr { expr, scope: _ } => visitor.visit_expr(&visitor.thir()[expr]), + StmtKind::Let { + initializer, + remainder_scope: _, + init_scope: _, + pattern: _, + lint_level: _, + } => { + if let Some(init) = initializer { + visitor.visit_expr(&visitor.thir()[init]); + } + } + } +} + +pub fn walk_block<'a, 'tcx: 'a, V: Visitor<'a, 'tcx>>(visitor: &mut V, block: &Block) { + for &stmt in &*block.stmts { + visitor.visit_stmt(&visitor.thir()[stmt]); + } + if let Some(expr) = block.expr { + visitor.visit_expr(&visitor.thir()[expr]); + } +} + +pub fn walk_arm<'a, 'tcx: 'a, V: Visitor<'a, 'tcx>>(visitor: &mut V, arm: &Arm<'tcx>) { + match arm.guard { + Some(Guard::If(expr)) => visitor.visit_expr(&visitor.thir()[expr]), + Some(Guard::IfLet(ref _pat, expr)) => { + visitor.visit_expr(&visitor.thir()[expr]); + } + None => {} + } + visitor.visit_expr(&visitor.thir()[arm.body]); +} diff --git a/compiler/rustc_parse/src/lexer/mod.rs b/compiler/rustc_parse/src/lexer/mod.rs index bd8dfd678a..1c2f9a9645 100644 --- a/compiler/rustc_parse/src/lexer/mod.rs +++ b/compiler/rustc_parse/src/lexer/mod.rs @@ -148,15 +148,11 @@ impl<'a> StringReader<'a> { None => "unterminated block comment", }; let last_bpos = self.pos; - self.sess - .span_diagnostic - .struct_span_fatal_with_code( - self.mk_sp(start, last_bpos), - msg, - error_code!(E0758), - ) - .emit(); - FatalError.raise(); + self.sess.span_diagnostic.span_fatal_with_code( + self.mk_sp(start, last_bpos), + msg, + error_code!(E0758), + ); } // Skip non-doc comments @@ -315,57 +311,41 @@ impl<'a> StringReader<'a> { let (lit_kind, mode, prefix_len, postfix_len) = match kind { rustc_lexer::LiteralKind::Char { terminated } => { if !terminated { - self.sess - .span_diagnostic - .struct_span_fatal_with_code( - self.mk_sp(start, suffix_start), - "unterminated character literal", - error_code!(E0762), - ) - .emit(); - FatalError.raise(); + self.sess.span_diagnostic.span_fatal_with_code( + self.mk_sp(start, suffix_start), + "unterminated character literal", + error_code!(E0762), + ) } (token::Char, Mode::Char, 1, 1) // ' ' } rustc_lexer::LiteralKind::Byte { terminated } => { if !terminated { - self.sess - .span_diagnostic - .struct_span_fatal_with_code( - self.mk_sp(start + BytePos(1), suffix_start), - "unterminated byte constant", - error_code!(E0763), - ) - .emit(); - FatalError.raise(); + self.sess.span_diagnostic.span_fatal_with_code( + self.mk_sp(start + BytePos(1), suffix_start), + "unterminated byte constant", + error_code!(E0763), + ) } (token::Byte, Mode::Byte, 2, 1) // b' ' } rustc_lexer::LiteralKind::Str { terminated } => { if !terminated { - self.sess - .span_diagnostic - .struct_span_fatal_with_code( - self.mk_sp(start, suffix_start), - "unterminated double quote string", - error_code!(E0765), - ) - .emit(); - FatalError.raise(); + self.sess.span_diagnostic.span_fatal_with_code( + self.mk_sp(start, suffix_start), + "unterminated double quote string", + error_code!(E0765), + ) } (token::Str, Mode::Str, 1, 1) // " " } rustc_lexer::LiteralKind::ByteStr { terminated } => { if !terminated { - self.sess - .span_diagnostic - .struct_span_fatal_with_code( - self.mk_sp(start + BytePos(1), suffix_start), - "unterminated double quote byte string", - error_code!(E0766), - ) - .emit(); - FatalError.raise(); + self.sess.span_diagnostic.span_fatal_with_code( + self.mk_sp(start + BytePos(1), suffix_start), + "unterminated double quote byte string", + error_code!(E0766), + ) } (token::ByteStr, Mode::ByteStr, 2, 1) // b" " } diff --git a/compiler/rustc_parse/src/lib.rs b/compiler/rustc_parse/src/lib.rs index 905077a48e..51df06bd98 100644 --- a/compiler/rustc_parse/src/lib.rs +++ b/compiler/rustc_parse/src/lib.rs @@ -3,8 +3,6 @@ #![feature(array_windows)] #![feature(crate_visibility_modifier)] #![feature(bindings_after_at)] -#![feature(iter_order_by)] -#![cfg_attr(bootstrap, feature(or_patterns))] #![feature(box_syntax)] #![feature(box_patterns)] #![recursion_limit = "256"] @@ -188,8 +186,10 @@ pub fn maybe_file_to_stream( override_span: Option, ) -> Result<(TokenStream, Vec), Vec> { let src = source_file.src.as_ref().unwrap_or_else(|| { - sess.span_diagnostic - .bug(&format!("cannot lex `source_file` without source: {}", source_file.name)); + sess.span_diagnostic.bug(&format!( + "cannot lex `source_file` without source: {}", + source_file.name.prefer_local() + )); }); let (token_trees, unmatched_braces) = diff --git a/compiler/rustc_parse/src/parser/attr.rs b/compiler/rustc_parse/src/parser/attr.rs index ee6ff4dba3..8b05038907 100644 --- a/compiler/rustc_parse/src/parser/attr.rs +++ b/compiler/rustc_parse/src/parser/attr.rs @@ -32,7 +32,6 @@ impl<'a> Parser<'a> { let mut just_parsed_doc_comment = false; let start_pos = self.token_cursor.num_next_calls; loop { - debug!("parse_outer_attributes: self.token={:?}", self.token); let attr = if self.check(&token::Pound) { let inner_error_reason = if just_parsed_doc_comment { "an inner attribute is not permitted following an outer doc comment" diff --git a/compiler/rustc_parse/src/parser/attr_wrapper.rs b/compiler/rustc_parse/src/parser/attr_wrapper.rs index 35759a396e..e1d0b84f41 100644 --- a/compiler/rustc_parse/src/parser/attr_wrapper.rs +++ b/compiler/rustc_parse/src/parser/attr_wrapper.rs @@ -342,16 +342,10 @@ impl<'a> Parser<'a> { // If we support tokens at all if let Some(target_tokens) = ret.tokens_mut() { - if let Some(target_tokens) = target_tokens { - assert!( - !self.capture_cfg, - "Encountered existing tokens with capture_cfg set: {:?}", - target_tokens - ); - } else { + if target_tokens.is_none() { // Store se our newly captured tokens into the AST node *target_tokens = Some(tokens.clone()); - }; + } } let final_attrs = ret.attrs(); diff --git a/compiler/rustc_parse/src/parser/diagnostics.rs b/compiler/rustc_parse/src/parser/diagnostics.rs index 70a5ac6f15..b37caaebfb 100644 --- a/compiler/rustc_parse/src/parser/diagnostics.rs +++ b/compiler/rustc_parse/src/parser/diagnostics.rs @@ -144,11 +144,7 @@ impl AttemptLocalParseRecovery { } impl<'a> Parser<'a> { - pub(super) fn span_fatal_err>( - &self, - sp: S, - err: Error, - ) -> DiagnosticBuilder<'a> { + pub(super) fn span_err>(&self, sp: S, err: Error) -> DiagnosticBuilder<'a> { err.span_err(sp, self.diagnostic()) } @@ -370,7 +366,7 @@ impl<'a> Parser<'a> { let mut snapshot = self.clone(); let path = Path { segments: vec![], span: self.prev_token.span.shrink_to_lo(), tokens: None }; - let struct_expr = snapshot.parse_struct_expr(path, AttrVec::new(), false); + let struct_expr = snapshot.parse_struct_expr(None, path, AttrVec::new(), false); let block_tail = self.parse_block_tail(lo, s, AttemptLocalParseRecovery::No); return Some(match (struct_expr, block_tail) { (Ok(expr), Err(mut err)) => { diff --git a/compiler/rustc_parse/src/parser/expr.rs b/compiler/rustc_parse/src/parser/expr.rs index e155b3fa77..0c814ccfdc 100644 --- a/compiler/rustc_parse/src/parser/expr.rs +++ b/compiler/rustc_parse/src/parser/expr.rs @@ -94,17 +94,7 @@ impl<'a> Parser<'a> { /// Parses an expression, forcing tokens to be collected pub fn parse_expr_force_collect(&mut self) -> PResult<'a, P> { - // If we have outer attributes, then the call to `collect_tokens_trailing_token` - // will be made for us. - if matches!(self.token.kind, TokenKind::Pound | TokenKind::DocComment(..)) { - self.parse_expr() - } else { - // If we don't have outer attributes, then we need to ensure - // that collection happens by using `collect_tokens_no_attrs`. - // Expression don't support custom inner attributes, so `parse_expr` - // will never try to collect tokens if we don't have outer attributes. - self.collect_tokens_no_attrs(|this| this.parse_expr()) - } + self.collect_tokens_no_attrs(|this| this.parse_expr()) } pub fn parse_anon_const_expr(&mut self) -> PResult<'a, AnonConst> { @@ -1118,9 +1108,6 @@ impl<'a> Parser<'a> { self.parse_closure_expr(attrs) } else if self.check(&token::OpenDelim(token::Bracket)) { self.parse_array_or_repeat_expr(attrs) - } else if self.eat_lt() { - let (qself, path) = self.parse_qpath(PathStyle::Expr)?; - Ok(self.mk_expr(lo.to(path.span), ExprKind::Path(Some(qself), path), attrs)) } else if self.check_path() { self.parse_path_start_expr(attrs) } else if self.check_keyword(kw::Move) || self.check_keyword(kw::Static) { @@ -1216,10 +1203,9 @@ impl<'a> Parser<'a> { } } - fn parse_tuple_parens_expr(&mut self, mut attrs: AttrVec) -> PResult<'a, P> { + fn parse_tuple_parens_expr(&mut self, attrs: AttrVec) -> PResult<'a, P> { let lo = self.token.span; self.expect(&token::OpenDelim(token::Paren))?; - attrs.extend(self.parse_inner_attributes()?); // `(#![foo] a, b, ...)` is OK. let (es, trailing_comma) = match self.parse_seq_to_end( &token::CloseDelim(token::Paren), SeqSep::trailing_allowed(token::Comma), @@ -1239,12 +1225,10 @@ impl<'a> Parser<'a> { self.maybe_recover_from_bad_qpath(expr, true) } - fn parse_array_or_repeat_expr(&mut self, mut attrs: AttrVec) -> PResult<'a, P> { + fn parse_array_or_repeat_expr(&mut self, attrs: AttrVec) -> PResult<'a, P> { let lo = self.token.span; self.bump(); // `[` - attrs.extend(self.parse_inner_attributes()?); - let close = &token::CloseDelim(token::Bracket); let kind = if self.eat(close) { // Empty vector @@ -1275,12 +1259,20 @@ impl<'a> Parser<'a> { } fn parse_path_start_expr(&mut self, attrs: AttrVec) -> PResult<'a, P> { - let path = self.parse_path(PathStyle::Expr)?; + let (qself, path) = if self.eat_lt() { + let (qself, path) = self.parse_qpath(PathStyle::Expr)?; + (Some(qself), path) + } else { + (None, self.parse_path(PathStyle::Expr)?) + }; let lo = path.span; // `!`, as an operator, is prefix, so we know this isn't that. let (hi, kind) = if self.eat(&token::Not) { // MACRO INVOCATION expression + if qself.is_some() { + self.struct_span_err(path.span, "macros cannot use qualified paths").emit(); + } let mac = MacCall { path, args: self.parse_mac_args()?, @@ -1288,13 +1280,16 @@ impl<'a> Parser<'a> { }; (self.prev_token.span, ExprKind::MacCall(mac)) } else if self.check(&token::OpenDelim(token::Brace)) { - if let Some(expr) = self.maybe_parse_struct_expr(&path, &attrs) { + if let Some(expr) = self.maybe_parse_struct_expr(qself.as_ref(), &path, &attrs) { + if qself.is_some() { + self.sess.gated_spans.gate(sym::more_qualified_paths, path.span); + } return expr; } else { - (path.span, ExprKind::Path(None, path)) + (path.span, ExprKind::Path(qself, path)) } } else { - (path.span, ExprKind::Path(None, path)) + (path.span, ExprKind::Path(qself, path)) }; let expr = self.mk_expr(lo.to(hi), kind, attrs); @@ -2261,6 +2256,7 @@ impl<'a> Parser<'a> { fn maybe_parse_struct_expr( &mut self, + qself: Option<&ast::QSelf>, path: &ast::Path, attrs: &AttrVec, ) -> Option>> { @@ -2269,7 +2265,7 @@ impl<'a> Parser<'a> { if let Err(err) = self.expect(&token::OpenDelim(token::Brace)) { return Some(Err(err)); } - let expr = self.parse_struct_expr(path.clone(), attrs.clone(), true); + let expr = self.parse_struct_expr(qself.cloned(), path.clone(), attrs.clone(), true); if let (Ok(expr), false) = (&expr, struct_allowed) { // This is a struct literal, but we don't can't accept them here. self.error_struct_lit_not_allowed_here(path.span, expr.span); @@ -2292,16 +2288,15 @@ impl<'a> Parser<'a> { /// Precondition: already parsed the '{'. pub(super) fn parse_struct_expr( &mut self, + qself: Option, pth: ast::Path, - mut attrs: AttrVec, + attrs: AttrVec, recover: bool, ) -> PResult<'a, P> { let mut fields = Vec::new(); let mut base = ast::StructRest::None; let mut recover_async = false; - attrs.extend(self.parse_inner_attributes()?); - let mut async_block_err = |e: &mut DiagnosticBuilder<'_>, span: Span| { recover_async = true; e.span_label(span, "`async` blocks are only allowed in Rust 2018 or later"); @@ -2391,7 +2386,7 @@ impl<'a> Parser<'a> { let expr = if recover_async { ExprKind::Err } else { - ExprKind::Struct(P(ast::StructExpr { path: pth, fields, rest: base })) + ExprKind::Struct(P(ast::StructExpr { qself, path: pth, fields, rest: base })) }; Ok(self.mk_expr(span, expr, attrs)) } diff --git a/compiler/rustc_parse/src/parser/item.rs b/compiler/rustc_parse/src/parser/item.rs index acf3867cf8..54e6ff6272 100644 --- a/compiler/rustc_parse/src/parser/item.rs +++ b/compiler/rustc_parse/src/parser/item.rs @@ -1124,11 +1124,11 @@ impl<'a> Parser<'a> { if !this.recover_nested_adt_item(kw::Enum)? { return Ok((None, TrailingToken::None)); } - let ident = this.parse_ident()?; + let ident = this.parse_field_ident("enum", vlo)?; let struct_def = if this.check(&token::OpenDelim(token::Brace)) { // Parse a struct variant. - let (fields, recovered) = this.parse_record_struct_body()?; + let (fields, recovered) = this.parse_record_struct_body("struct")?; VariantData::Struct(fields, recovered) } else if this.check(&token::OpenDelim(token::Paren)) { VariantData::Tuple(this.parse_tuple_struct_body()?, DUMMY_NODE_ID) @@ -1182,7 +1182,7 @@ impl<'a> Parser<'a> { VariantData::Unit(DUMMY_NODE_ID) } else { // If we see: `struct Foo where T: Copy { ... }` - let (fields, recovered) = self.parse_record_struct_body()?; + let (fields, recovered) = self.parse_record_struct_body("struct")?; VariantData::Struct(fields, recovered) } // No `where` so: `struct Foo;` @@ -1190,7 +1190,7 @@ impl<'a> Parser<'a> { VariantData::Unit(DUMMY_NODE_ID) // Record-style struct definition } else if self.token == token::OpenDelim(token::Brace) { - let (fields, recovered) = self.parse_record_struct_body()?; + let (fields, recovered) = self.parse_record_struct_body("struct")?; VariantData::Struct(fields, recovered) // Tuple-style struct definition with optional where-clause. } else if self.token == token::OpenDelim(token::Paren) { @@ -1220,10 +1220,10 @@ impl<'a> Parser<'a> { let vdata = if self.token.is_keyword(kw::Where) { generics.where_clause = self.parse_where_clause()?; - let (fields, recovered) = self.parse_record_struct_body()?; + let (fields, recovered) = self.parse_record_struct_body("union")?; VariantData::Struct(fields, recovered) } else if self.token == token::OpenDelim(token::Brace) { - let (fields, recovered) = self.parse_record_struct_body()?; + let (fields, recovered) = self.parse_record_struct_body("union")?; VariantData::Struct(fields, recovered) } else { let token_str = super::token_descr(&self.token); @@ -1236,12 +1236,15 @@ impl<'a> Parser<'a> { Ok((class_name, ItemKind::Union(vdata, generics))) } - fn parse_record_struct_body(&mut self) -> PResult<'a, (Vec, /* recovered */ bool)> { + pub(super) fn parse_record_struct_body( + &mut self, + adt_ty: &str, + ) -> PResult<'a, (Vec, /* recovered */ bool)> { let mut fields = Vec::new(); let mut recovered = false; if self.eat(&token::OpenDelim(token::Brace)) { while self.token != token::CloseDelim(token::Brace) { - let field = self.parse_field_def().map_err(|e| { + let field = self.parse_field_def(adt_ty).map_err(|e| { self.consume_block(token::Brace, ConsumeClosingDelim::No); recovered = true; e @@ -1294,24 +1297,25 @@ impl<'a> Parser<'a> { } /// Parses an element of a struct declaration. - fn parse_field_def(&mut self) -> PResult<'a, FieldDef> { + fn parse_field_def(&mut self, adt_ty: &str) -> PResult<'a, FieldDef> { let attrs = self.parse_outer_attributes()?; self.collect_tokens_trailing_token(attrs, ForceCollect::No, |this, attrs| { let lo = this.token.span; let vis = this.parse_visibility(FollowedByType::No)?; - Ok((this.parse_single_struct_field(lo, vis, attrs)?, TrailingToken::None)) + Ok((this.parse_single_struct_field(adt_ty, lo, vis, attrs)?, TrailingToken::None)) }) } /// Parses a structure field declaration. fn parse_single_struct_field( &mut self, + adt_ty: &str, lo: Span, vis: Visibility, attrs: Vec, ) -> PResult<'a, FieldDef> { let mut seen_comma: bool = false; - let a_var = self.parse_name_and_ty(lo, vis, attrs)?; + let a_var = self.parse_name_and_ty(adt_ty, lo, vis, attrs)?; if self.token == token::Comma { seen_comma = true; } @@ -1322,7 +1326,7 @@ impl<'a> Parser<'a> { token::CloseDelim(token::Brace) => {} token::DocComment(..) => { let previous_span = self.prev_token.span; - let mut err = self.span_fatal_err(self.token.span, Error::UselessDocComment); + let mut err = self.span_err(self.token.span, Error::UselessDocComment); self.bump(); // consume the doc comment let comma_after_doc_seen = self.eat(&token::Comma); // `seen_comma` is always false, because we are inside doc block @@ -1395,16 +1399,61 @@ impl<'a> Parser<'a> { Ok(a_var) } + fn expect_field_ty_separator(&mut self) -> PResult<'a, ()> { + if let Err(mut err) = self.expect(&token::Colon) { + let sm = self.sess.source_map(); + let eq_typo = self.token.kind == token::Eq && self.look_ahead(1, |t| t.is_path_start()); + let semi_typo = self.token.kind == token::Semi + && self.look_ahead(1, |t| { + t.is_path_start() + // We check that we are in a situation like `foo; bar` to avoid bad suggestions + // when there's no type and `;` was used instead of a comma. + && match (sm.lookup_line(self.token.span.hi()), sm.lookup_line(t.span.lo())) { + (Ok(l), Ok(r)) => l.line == r.line, + _ => true, + } + }); + if eq_typo || semi_typo { + self.bump(); + // Gracefully handle small typos. + err.span_suggestion_short( + self.prev_token.span, + "field names and their types are separated with `:`", + ":".to_string(), + Applicability::MachineApplicable, + ); + err.emit(); + } else { + return Err(err); + } + } + Ok(()) + } + /// Parses a structure field. fn parse_name_and_ty( &mut self, + adt_ty: &str, lo: Span, vis: Visibility, attrs: Vec, ) -> PResult<'a, FieldDef> { - let name = self.parse_ident_common(false)?; - self.expect(&token::Colon)?; + let name = self.parse_field_ident(adt_ty, lo)?; + self.expect_field_ty_separator()?; let ty = self.parse_ty()?; + if self.token.kind == token::Eq { + self.bump(); + let const_expr = self.parse_anon_const_expr()?; + let sp = ty.span.shrink_to_hi().to(const_expr.value.span); + self.struct_span_err(sp, "default values on `struct` fields aren't supported") + .span_suggestion( + sp, + "remove this unsupported default value", + String::new(), + Applicability::MachineApplicable, + ) + .emit(); + } Ok(FieldDef { span: lo.to(self.prev_token.span), ident: Some(name), @@ -1416,6 +1465,38 @@ impl<'a> Parser<'a> { }) } + /// Parses a field identifier. Specialized version of `parse_ident_common` + /// for better diagnostics and suggestions. + fn parse_field_ident(&mut self, adt_ty: &str, lo: Span) -> PResult<'a, Ident> { + let (ident, is_raw) = self.ident_or_err()?; + if !is_raw && ident.is_reserved() { + if ident.name == kw::Underscore { + self.sess.gated_spans.gate(sym::unnamed_fields, lo); + } else { + let err = if self.check_fn_front_matter(false) { + // We use `parse_fn` to get a span for the function + if let Err(mut db) = self.parse_fn(&mut Vec::new(), |_| true, lo) { + db.delay_as_bug(); + } + let mut err = self.struct_span_err( + lo.to(self.prev_token.span), + &format!("functions are not allowed in {} definitions", adt_ty), + ); + err.help( + "unlike in C++, Java, and C#, functions are declared in `impl` blocks", + ); + err.help("see https://doc.rust-lang.org/book/ch05-03-method-syntax.html for more information"); + err + } else { + self.expected_ident_found() + }; + return Err(err); + } + } + self.bump(); + Ok(ident) + } + /// Parses a declarative macro 2.0 definition. /// The `macro` keyword has already been parsed. /// ``` diff --git a/compiler/rustc_parse/src/parser/mod.rs b/compiler/rustc_parse/src/parser/mod.rs index a55eb3cf75..cd9f84db5e 100644 --- a/compiler/rustc_parse/src/parser/mod.rs +++ b/compiler/rustc_parse/src/parser/mod.rs @@ -63,6 +63,7 @@ enum BlockMode { /// Whether or not we should force collection of tokens for an AST node, /// regardless of whether or not it has attributes +#[derive(Clone, Copy, PartialEq)] pub enum ForceCollect { Yes, No, @@ -522,27 +523,27 @@ impl<'a> Parser<'a> { self.parse_ident_common(true) } + fn ident_or_err(&mut self) -> PResult<'a, (Ident, /* is_raw */ bool)> { + self.token.ident().ok_or_else(|| match self.prev_token.kind { + TokenKind::DocComment(..) => { + self.span_err(self.prev_token.span, Error::UselessDocComment) + } + _ => self.expected_ident_found(), + }) + } + fn parse_ident_common(&mut self, recover: bool) -> PResult<'a, Ident> { - match self.token.ident() { - Some((ident, is_raw)) => { - if !is_raw && ident.is_reserved() { - let mut err = self.expected_ident_found(); - if recover { - err.emit(); - } else { - return Err(err); - } - } - self.bump(); - Ok(ident) + let (ident, is_raw) = self.ident_or_err()?; + if !is_raw && ident.is_reserved() { + let mut err = self.expected_ident_found(); + if recover { + err.emit(); + } else { + return Err(err); } - _ => Err(match self.prev_token.kind { - TokenKind::DocComment(..) => { - self.span_fatal_err(self.prev_token.span, Error::UselessDocComment) - } - _ => self.expected_ident_found(), - }), } + self.bump(); + Ok(ident) } /// Checks if the next token is `tok`, and returns `true` if so. @@ -1065,27 +1066,11 @@ impl<'a> Parser<'a> { } else if !delimited_only { if self.eat(&token::Eq) { let eq_span = self.prev_token.span; - let mut is_interpolated_expr = false; - if let token::Interpolated(nt) = &self.token.kind { - if let token::NtExpr(..) = **nt { - is_interpolated_expr = true; - } - } // Collect tokens because they are used during lowering to HIR. let expr = self.parse_expr_force_collect()?; let span = expr.span; - match &expr.kind { - // Not gated to support things like `doc = $expr` that work on stable. - // Do not gate in `capture_cfg` mode, since we flatten all nontemrinals - // before parsing. `capture_cfg` mode is only used to reparse existing - // tokens, so the gating will be performed by the initial parse - _ if is_interpolated_expr || self.capture_cfg => {} - ExprKind::Lit(lit) if lit.kind.is_unsuffixed() => {} - _ => self.sess.gated_spans.gate(sym::extended_key_value_attributes, span), - } - let token_kind = token::Interpolated(Lrc::new(token::NtExpr(expr))); MacArgs::Eq(eq_span, Token::new(token_kind, span)) } else { diff --git a/compiler/rustc_parse/src/parser/nonterminal.rs b/compiler/rustc_parse/src/parser/nonterminal.rs index 0c43e304f1..30a6b61407 100644 --- a/compiler/rustc_parse/src/parser/nonterminal.rs +++ b/compiler/rustc_parse/src/parser/nonterminal.rs @@ -1,5 +1,6 @@ use rustc_ast::ptr::P; use rustc_ast::token::{self, Nonterminal, NonterminalKind, Token}; +use rustc_ast::AstLike; use rustc_ast_pretty::pprust; use rustc_errors::PResult; use rustc_span::symbol::{kw, Ident}; @@ -102,7 +103,7 @@ impl<'a> Parser<'a> { // which requires having captured tokens available. Since we cannot determine // in advance whether or not a proc-macro will be (transitively) invoked, // we always capture tokens for any `Nonterminal` which needs them. - Ok(match kind { + let mut nt = match kind { NonterminalKind::Item => match self.parse_item(ForceCollect::Yes)? { Some(item) => token::NtItem(item), None => { @@ -169,7 +170,19 @@ impl<'a> Parser<'a> { return Err(self.struct_span_err(self.token.span, msg)); } } - }) + }; + + // If tokens are supported at all, they should be collected. + if matches!(nt.tokens_mut(), Some(None)) { + panic!( + "Missing tokens for nt {:?} at {:?}: {:?}", + nt, + nt.span(), + pprust::nonterminal_to_string(&nt) + ); + } + + Ok(nt) } } diff --git a/compiler/rustc_parse/src/parser/pat.rs b/compiler/rustc_parse/src/parser/pat.rs index 0abefbd6a1..418122202b 100644 --- a/compiler/rustc_parse/src/parser/pat.rs +++ b/compiler/rustc_parse/src/parser/pat.rs @@ -859,7 +859,8 @@ impl<'a> Parser<'a> { /// Parse a struct ("record") pattern (e.g. `Foo { ... }` or `Foo::Bar { ... }`). fn parse_pat_struct(&mut self, qself: Option, path: Path) -> PResult<'a, PatKind> { if qself.is_some() { - return self.error_qpath_before_pat(&path, "{"); + // Feature gate the use of qualified paths in patterns + self.sess.gated_spans.gate(sym::more_qualified_paths, path.span); } self.bump(); let (fields, etc) = self.parse_pat_fields().unwrap_or_else(|mut e| { @@ -869,27 +870,17 @@ impl<'a> Parser<'a> { (vec![], true) }); self.bump(); - Ok(PatKind::Struct(path, fields, etc)) + Ok(PatKind::Struct(qself, path, fields, etc)) } /// Parse tuple struct or tuple variant pattern (e.g. `Foo(...)` or `Foo::Bar(...)`). fn parse_pat_tuple_struct(&mut self, qself: Option, path: Path) -> PResult<'a, PatKind> { - if qself.is_some() { - return self.error_qpath_before_pat(&path, "("); - } let (fields, _) = self.parse_paren_comma_seq(|p| p.parse_pat_allow_top_alt(None, RecoverComma::No))?; - Ok(PatKind::TupleStruct(path, fields)) - } - - /// Error when there's a qualified path, e.g. `::Baz` - /// as the path of e.g., a tuple or record struct pattern. - fn error_qpath_before_pat(&mut self, path: &Path, token: &str) -> PResult<'a, PatKind> { - let msg = &format!("unexpected `{}` after qualified path", token); - let mut err = self.struct_span_err(self.token.span, msg); - err.span_label(self.token.span, msg); - err.span_label(path.span, "the qualified path"); - Err(err) + if qself.is_some() { + self.sess.gated_spans.gate(sym::more_qualified_paths, path.span); + } + Ok(PatKind::TupleStruct(qself, path, fields)) } /// Parses the fields of a struct-like pattern. diff --git a/compiler/rustc_parse/src/parser/path.rs b/compiler/rustc_parse/src/parser/path.rs index 9cc600d9ed..953c691506 100644 --- a/compiler/rustc_parse/src/parser/path.rs +++ b/compiler/rustc_parse/src/parser/path.rs @@ -352,49 +352,59 @@ impl<'a> Parser<'a> { debug!("parse_generic_args_with_leading_angle_bracket_recovery: (snapshotting)"); match self.parse_angle_args() { Ok(args) => Ok(args), - Err(ref mut e) if is_first_invocation && self.unmatched_angle_bracket_count > 0 => { - // Cancel error from being unable to find `>`. We know the error - // must have been this due to a non-zero unmatched angle bracket - // count. - e.cancel(); - + Err(mut e) if is_first_invocation && self.unmatched_angle_bracket_count > 0 => { // Swap `self` with our backup of the parser state before attempting to parse // generic arguments. let snapshot = mem::replace(self, snapshot.unwrap()); - debug!( - "parse_generic_args_with_leading_angle_bracket_recovery: (snapshot failure) \ - snapshot.count={:?}", - snapshot.unmatched_angle_bracket_count, - ); - // Eat the unmatched angle brackets. - for _ in 0..snapshot.unmatched_angle_bracket_count { - self.eat_lt(); - } - - // Make a span over ${unmatched angle bracket count} characters. - let span = lo.with_hi(lo.lo() + BytePos(snapshot.unmatched_angle_bracket_count)); - self.struct_span_err( - span, - &format!( - "unmatched angle bracket{}", - pluralize!(snapshot.unmatched_angle_bracket_count) - ), - ) - .span_suggestion( - span, - &format!( - "remove extra angle bracket{}", - pluralize!(snapshot.unmatched_angle_bracket_count) - ), - String::new(), - Applicability::MachineApplicable, - ) - .emit(); + let all_angle_brackets = (0..snapshot.unmatched_angle_bracket_count) + .fold(true, |a, _| a && self.eat_lt()); + + if !all_angle_brackets { + // If there are other tokens in between the extraneous `<`s, we cannot simply + // suggest to remove them. This check also prevents us from accidentally ending + // up in the middle of a multibyte character (issue #84104). + let _ = mem::replace(self, snapshot); + Err(e) + } else { + // Cancel error from being unable to find `>`. We know the error + // must have been this due to a non-zero unmatched angle bracket + // count. + e.cancel(); + + debug!( + "parse_generic_args_with_leading_angle_bracket_recovery: (snapshot failure) \ + snapshot.count={:?}", + snapshot.unmatched_angle_bracket_count, + ); + + // Make a span over ${unmatched angle bracket count} characters. + // This is safe because `all_angle_brackets` ensures that there are only `<`s, + // i.e. no multibyte characters, in this range. + let span = + lo.with_hi(lo.lo() + BytePos(snapshot.unmatched_angle_bracket_count)); + self.struct_span_err( + span, + &format!( + "unmatched angle bracket{}", + pluralize!(snapshot.unmatched_angle_bracket_count) + ), + ) + .span_suggestion( + span, + &format!( + "remove extra angle bracket{}", + pluralize!(snapshot.unmatched_angle_bracket_count) + ), + String::new(), + Applicability::MachineApplicable, + ) + .emit(); - // Try again without unmatched angle bracket characters. - self.parse_angle_args() + // Try again without unmatched angle bracket characters. + self.parse_angle_args() + } } Err(e) => Err(e), } diff --git a/compiler/rustc_parse/src/parser/stmt.rs b/compiler/rustc_parse/src/parser/stmt.rs index 592f64f4a3..9ef3f61ec3 100644 --- a/compiler/rustc_parse/src/parser/stmt.rs +++ b/compiler/rustc_parse/src/parser/stmt.rs @@ -73,7 +73,11 @@ impl<'a> Parser<'a> { // or `auto trait` items. We aim to parse an arbitrary path `a::b` but not something // that starts like a path (1 token), but it fact not a path. // Also, we avoid stealing syntax from `parse_item_`. - self.parse_stmt_path_start(lo, attrs, force_collect)? + if force_collect == ForceCollect::Yes { + self.collect_tokens_no_attrs(|this| this.parse_stmt_path_start(lo, attrs)) + } else { + self.parse_stmt_path_start(lo, attrs) + }? } else if let Some(item) = self.parse_item_common(attrs.clone(), false, true, |_| true, force_collect)? { @@ -85,7 +89,13 @@ impl<'a> Parser<'a> { self.mk_stmt(lo, StmtKind::Empty) } else if self.token != token::CloseDelim(token::Brace) { // Remainder are line-expr stmts. - let e = self.parse_expr_res(Restrictions::STMT_EXPR, Some(attrs))?; + let e = if force_collect == ForceCollect::Yes { + self.collect_tokens_no_attrs(|this| { + this.parse_expr_res(Restrictions::STMT_EXPR, Some(attrs)) + }) + } else { + self.parse_expr_res(Restrictions::STMT_EXPR, Some(attrs)) + }?; self.mk_stmt(lo.to(e.span), StmtKind::Expr(e)) } else { self.error_outer_attrs(&attrs.take_for_recovery()); @@ -93,13 +103,8 @@ impl<'a> Parser<'a> { })) } - fn parse_stmt_path_start( - &mut self, - lo: Span, - attrs: AttrWrapper, - force_collect: ForceCollect, - ) -> PResult<'a, Stmt> { - let stmt = self.collect_tokens_trailing_token(attrs, force_collect, |this, attrs| { + fn parse_stmt_path_start(&mut self, lo: Span, attrs: AttrWrapper) -> PResult<'a, Stmt> { + let stmt = self.collect_tokens_trailing_token(attrs, ForceCollect::No, |this, attrs| { let path = this.parse_path(PathStyle::Expr)?; if this.eat(&token::Not) { @@ -112,7 +117,7 @@ impl<'a> Parser<'a> { } let expr = if this.eat(&token::OpenDelim(token::Brace)) { - this.parse_struct_expr(path, AttrVec::new(), true)? + this.parse_struct_expr(None, path, AttrVec::new(), true)? } else { let hi = this.prev_token.span; this.mk_expr(lo.to(hi), ExprKind::Path(None, path), AttrVec::new()) @@ -168,7 +173,7 @@ impl<'a> Parser<'a> { fn error_outer_attrs(&self, attrs: &[Attribute]) { if let [.., last] = attrs { if last.is_doc_comment() { - self.span_fatal_err(last.span, Error::UselessDocComment).emit(); + self.span_err(last.span, Error::UselessDocComment).emit(); } else if attrs.iter().any(|a| a.style == AttrStyle::Outer) { self.struct_span_err(last.span, "expected statement after outer attribute").emit(); } diff --git a/compiler/rustc_parse/src/parser/ty.rs b/compiler/rustc_parse/src/parser/ty.rs index 0f7b8ebd37..de5a563260 100644 --- a/compiler/rustc_parse/src/parser/ty.rs +++ b/compiler/rustc_parse/src/parser/ty.rs @@ -226,6 +226,19 @@ impl<'a> Parser<'a> { } } else if self.eat_keyword(kw::Impl) { self.parse_impl_ty(&mut impl_dyn_multi)? + } else if self.token.is_keyword(kw::Union) + && self.look_ahead(1, |t| t == &token::OpenDelim(token::Brace)) + { + self.bump(); + let (fields, recovered) = self.parse_record_struct_body("union")?; + let span = lo.to(self.prev_token.span); + self.sess.gated_spans.gate(sym::unnamed_fields, span); + TyKind::AnonymousUnion(fields, recovered) + } else if self.eat_keyword(kw::Struct) { + let (fields, recovered) = self.parse_record_struct_body("struct")?; + let span = lo.to(self.prev_token.span); + self.sess.gated_spans.gate(sym::unnamed_fields, span); + TyKind::AnonymousStruct(fields, recovered) } else if self.is_explicit_dyn_type() { self.parse_dyn_ty(&mut impl_dyn_multi)? } else if self.eat_lt() { @@ -321,7 +334,6 @@ impl<'a> Parser<'a> { mut bounds: GenericBounds, plus: bool, ) -> PResult<'a, TyKind> { - assert_ne!(self.token, token::Question); if plus { self.eat_plus(); // `+`, or `+=` gets split and `+` is discarded bounds.append(&mut self.parse_generic_bounds(Some(self.prev_token.span))?); @@ -470,7 +482,7 @@ impl<'a> Parser<'a> { /// Is a `dyn B0 + ... + Bn` type allowed here? fn is_explicit_dyn_type(&mut self) -> bool { self.check_keyword(kw::Dyn) - && (self.token.uninterpolated_span().rust_2018() + && (!self.token.uninterpolated_span().rust_2015() || self.look_ahead(1, |t| { t.can_begin_bound() && !can_continue_type_after_non_fn_ident(t) })) @@ -539,7 +551,21 @@ impl<'a> Parser<'a> { ) -> PResult<'a, GenericBounds> { let mut bounds = Vec::new(); let mut negative_bounds = Vec::new(); - while self.can_begin_bound() { + + while self.can_begin_bound() || self.token.is_keyword(kw::Dyn) { + if self.token.is_keyword(kw::Dyn) { + // Account for `&dyn Trait + dyn Other`. + self.struct_span_err(self.token.span, "invalid `dyn` keyword") + .help("`dyn` is only needed at the start of a trait `+`-separated list") + .span_suggestion( + self.token.span, + "remove this keyword", + String::new(), + Applicability::MachineApplicable, + ) + .emit(); + self.bump(); + } match self.parse_generic_bound()? { Ok(bound) => bounds.push(bound), Err(neg_sp) => negative_bounds.push(neg_sp), @@ -721,7 +747,26 @@ impl<'a> Parser<'a> { let lifetime_defs = self.parse_late_bound_lifetime_defs()?; let path = self.parse_path(PathStyle::Type)?; if has_parens { - self.expect(&token::CloseDelim(token::Paren))?; + if self.token.is_like_plus() { + // Someone has written something like `&dyn (Trait + Other)`. The correct code + // would be `&(dyn Trait + Other)`, but we don't have access to the appropriate + // span to suggest that. When written as `&dyn Trait + Other`, an appropriate + // suggestion is given. + let bounds = vec![]; + self.parse_remaining_bounds(bounds, true)?; + self.expect(&token::CloseDelim(token::Paren))?; + let sp = vec![lo, self.prev_token.span]; + let sugg: Vec<_> = sp.iter().map(|sp| (*sp, String::new())).collect(); + self.struct_span_err(sp, "incorrect braces around trait bounds") + .multipart_suggestion( + "remove the parentheses", + sugg, + Applicability::MachineApplicable, + ) + .emit(); + } else { + self.expect(&token::CloseDelim(token::Paren))?; + } } let modifier = modifiers.to_trait_bound_modifier(); diff --git a/compiler/rustc_parse_format/src/lib.rs b/compiler/rustc_parse_format/src/lib.rs index 50db69f420..9d653de910 100644 --- a/compiler/rustc_parse_format/src/lib.rs +++ b/compiler/rustc_parse_format/src/lib.rs @@ -10,7 +10,6 @@ test(attr(deny(warnings))) )] #![feature(nll)] -#![cfg_attr(bootstrap, feature(or_patterns))] #![feature(bool_to_option)] pub use Alignment::*; diff --git a/compiler/rustc_passes/src/check_attr.rs b/compiler/rustc_passes/src/check_attr.rs index 3ffe488cae..9c42a42fe6 100644 --- a/compiler/rustc_passes/src/check_attr.rs +++ b/compiler/rustc_passes/src/check_attr.rs @@ -8,21 +8,19 @@ use rustc_middle::hir::map::Map; use rustc_middle::ty::query::Providers; use rustc_middle::ty::TyCtxt; -use rustc_ast::{Attribute, Lit, LitKind, NestedMetaItem}; +use rustc_ast::{AttrStyle, Attribute, Lit, LitKind, NestedMetaItem}; use rustc_errors::{pluralize, struct_span_err, Applicability}; use rustc_hir as hir; use rustc_hir::def_id::LocalDefId; use rustc_hir::intravisit::{self, NestedVisitorMap, Visitor}; -use rustc_hir::{ - self, FnSig, ForeignItem, ForeignItemKind, HirId, Item, ItemKind, TraitItem, CRATE_HIR_ID, -}; +use rustc_hir::{self, FnSig, ForeignItem, HirId, Item, ItemKind, TraitItem, CRATE_HIR_ID}; use rustc_hir::{MethodKind, Target}; use rustc_session::lint::builtin::{ CONFLICTING_REPR_HINTS, INVALID_DOC_ATTRIBUTES, UNUSED_ATTRIBUTES, }; use rustc_session::parse::feature_err; use rustc_span::symbol::{sym, Symbol}; -use rustc_span::{Span, DUMMY_SP}; +use rustc_span::{MultiSpan, Span, DUMMY_SP}; pub(crate) fn target_from_impl_item<'tcx>( tcx: TyCtxt<'tcx>, @@ -67,6 +65,7 @@ impl CheckAttrVisitor<'tcx> { item: Option>, ) { let mut is_valid = true; + let mut specified_inline = None; let attrs = self.tcx.hir().attrs(hir_id); for attr in attrs { is_valid &= match attr.name_or_empty() { @@ -77,12 +76,9 @@ impl CheckAttrVisitor<'tcx> { sym::track_caller => { self.check_track_caller(hir_id, &attr.span, attrs, span, target) } - sym::doc => self.check_doc_attrs(attr, hir_id, target), + sym::doc => self.check_doc_attrs(attr, hir_id, target, &mut specified_inline), sym::no_link => self.check_no_link(hir_id, &attr, span, target), sym::export_name => self.check_export_name(hir_id, &attr, span, target), - sym::rustc_args_required_const => { - self.check_rustc_args_required_const(&attr, span, target, item) - } sym::rustc_layout_scalar_valid_range_start | sym::rustc_layout_scalar_valid_range_end => { self.check_rustc_layout_scalar_valid_range(&attr, span, target) @@ -564,7 +560,71 @@ impl CheckAttrVisitor<'tcx> { true } - fn check_attr_crate_level( + /// Checks `#[doc(inline)]`/`#[doc(no_inline)]` attributes. Returns `true` if valid. + /// + /// A doc inlining attribute is invalid if it is applied to a non-`use` item, or + /// if there are conflicting attributes for one item. + /// + /// `specified_inline` is used to keep track of whether we have + /// already seen an inlining attribute for this item. + /// If so, `specified_inline` holds the value and the span of + /// the first `inline`/`no_inline` attribute. + fn check_doc_inline( + &self, + attr: &Attribute, + meta: &NestedMetaItem, + hir_id: HirId, + target: Target, + specified_inline: &mut Option<(bool, Span)>, + ) -> bool { + if target == Target::Use || target == Target::ExternCrate { + let do_inline = meta.name_or_empty() == sym::inline; + if let Some((prev_inline, prev_span)) = *specified_inline { + if do_inline != prev_inline { + let mut spans = MultiSpan::from_spans(vec![prev_span, meta.span()]); + spans.push_span_label(prev_span, String::from("this attribute...")); + spans.push_span_label( + meta.span(), + String::from("...conflicts with this attribute"), + ); + self.tcx + .sess + .struct_span_err(spans, "conflicting doc inlining attributes") + .help("remove one of the conflicting attributes") + .emit(); + return false; + } + true + } else { + *specified_inline = Some((do_inline, meta.span())); + true + } + } else { + self.tcx.struct_span_lint_hir( + INVALID_DOC_ATTRIBUTES, + hir_id, + meta.span(), + |lint| { + let mut err = lint.build( + "this attribute can only be applied to a `use` item", + ); + err.span_label(meta.span(), "only applicable on `use` items"); + if attr.style == AttrStyle::Outer { + err.span_label( + self.tcx.hir().span(hir_id), + "not a `use` item", + ); + } + err.note("read https://doc.rust-lang.org/nightly/rustdoc/the-doc-attribute.html#docno_inlinedocinline for more information") + .emit(); + }, + ); + false + } + } + + /// Checks that an attribute is *not* used at the crate level. Returns `true` if valid. + fn check_attr_not_crate_level( &self, meta: &NestedMetaItem, hir_id: HirId, @@ -586,40 +646,103 @@ impl CheckAttrVisitor<'tcx> { true } - fn check_doc_attrs(&self, attr: &Attribute, hir_id: HirId, target: Target) -> bool { + /// Checks that an attribute is used at the crate level. Returns `true` if valid. + fn check_attr_crate_level( + &self, + attr: &Attribute, + meta: &NestedMetaItem, + hir_id: HirId, + ) -> bool { + if hir_id != CRATE_HIR_ID { + self.tcx.struct_span_lint_hir( + INVALID_DOC_ATTRIBUTES, + hir_id, + meta.span(), + |lint| { + let mut err = lint.build( + "this attribute can only be applied at the crate level", + ); + if attr.style == AttrStyle::Outer && self.tcx.hir().get_parent_item(hir_id) == CRATE_HIR_ID { + if let Ok(mut src) = + self.tcx.sess.source_map().span_to_snippet(attr.span) + { + src.insert(1, '!'); + err.span_suggestion_verbose( + attr.span, + "to apply to the crate, use an inner attribute", + src, + Applicability::MaybeIncorrect, + ); + } else { + err.span_help( + attr.span, + "to apply to the crate, use an inner attribute", + ); + } + } + err.note("read https://doc.rust-lang.org/nightly/rustdoc/the-doc-attribute.html#at-the-crate-level for more information") + .emit(); + }, + ); + return false; + } + true + } + + /// Runs various checks on `#[doc]` attributes. Returns `true` if valid. + /// + /// `specified_inline` should be initialized to `None` and kept for the scope + /// of one item. Read the documentation of [`check_doc_inline`] for more information. + /// + /// [`check_doc_inline`]: Self::check_doc_inline + fn check_doc_attrs( + &self, + attr: &Attribute, + hir_id: HirId, + target: Target, + specified_inline: &mut Option<(bool, Span)>, + ) -> bool { let mut is_valid = true; if let Some(list) = attr.meta().and_then(|mi| mi.meta_item_list().map(|l| l.to_vec())) { - for meta in list { + for meta in &list { if let Some(i_meta) = meta.meta_item() { match i_meta.name_or_empty() { sym::alias - if !self.check_attr_crate_level(&meta, hir_id, "alias") + if !self.check_attr_not_crate_level(&meta, hir_id, "alias") || !self.check_doc_alias(&meta, hir_id, target) => { is_valid = false } sym::keyword - if !self.check_attr_crate_level(&meta, hir_id, "keyword") + if !self.check_attr_not_crate_level(&meta, hir_id, "keyword") || !self.check_doc_keyword(&meta, hir_id) => { is_valid = false } - sym::test if CRATE_HIR_ID != hir_id => { - self.tcx.struct_span_lint_hir( - INVALID_DOC_ATTRIBUTES, + sym::html_favicon_url + | sym::html_logo_url + | sym::html_playground_url + | sym::issue_tracker_base_url + | sym::html_root_url + | sym::html_no_source + | sym::test + if !self.check_attr_crate_level(&attr, &meta, hir_id) => + { + is_valid = false; + } + + sym::inline | sym::no_inline + if !self.check_doc_inline( + &attr, + &meta, hir_id, - meta.span(), - |lint| { - lint.build( - "`#![doc(test(...)]` is only allowed \ - as a crate-level attribute", - ) - .emit(); - }, - ); + target, + specified_inline, + ) => + { is_valid = false; } @@ -634,7 +757,6 @@ impl CheckAttrVisitor<'tcx> { | sym::html_no_source | sym::html_playground_url | sym::html_root_url - | sym::include | sym::inline | sym::issue_tracker_base_url | sym::keyword @@ -669,6 +791,30 @@ impl CheckAttrVisitor<'tcx> { ); diag.note("`doc(spotlight)` is now a no-op"); } + if i_meta.has_name(sym::include) { + if let Some(value) = i_meta.value_str() { + // if there are multiple attributes, the suggestion would suggest deleting all of them, which is incorrect + let applicability = if list.len() == 1 { + Applicability::MachineApplicable + } else { + Applicability::MaybeIncorrect + }; + let inner = if attr.style == AttrStyle::Inner { + "!" + } else { + "" + }; + diag.span_suggestion( + attr.meta().unwrap().span, + "use `doc = include_str!` instead", + format!( + "#{}[doc = include_str!(\"{}\")]", + inner, value + ), + applicability, + ); + } + } diag.emit(); }, ); @@ -820,79 +966,6 @@ impl CheckAttrVisitor<'tcx> { } } - /// Checks if `#[rustc_args_required_const]` is applied to a function and has a valid argument. - fn check_rustc_args_required_const( - &self, - attr: &Attribute, - span: &Span, - target: Target, - item: Option>, - ) -> bool { - let is_function = matches!(target, Target::Fn | Target::Method(..) | Target::ForeignFn); - if !is_function { - self.tcx - .sess - .struct_span_err(attr.span, "attribute should be applied to a function") - .span_label(*span, "not a function") - .emit(); - return false; - } - - let list = match attr.meta_item_list() { - // The attribute form is validated on AST. - None => return false, - Some(it) => it, - }; - - let mut invalid_args = vec![]; - for meta in list { - if let Some(LitKind::Int(val, _)) = meta.literal().map(|lit| &lit.kind) { - if let Some(ItemLike::Item(Item { - kind: ItemKind::Fn(FnSig { decl, .. }, ..), - .. - })) - | Some(ItemLike::ForeignItem(ForeignItem { - kind: ForeignItemKind::Fn(decl, ..), - .. - })) = item - { - let arg_count = decl.inputs.len() as u128; - if *val >= arg_count { - let span = meta.span(); - self.tcx - .sess - .struct_span_err(span, "index exceeds number of arguments") - .span_label( - span, - format!( - "there {} only {} argument{}", - if arg_count != 1 { "are" } else { "is" }, - arg_count, - pluralize!(arg_count) - ), - ) - .emit(); - return false; - } - } else { - bug!("should be a function item"); - } - } else { - invalid_args.push(meta.span()); - } - } - - if !invalid_args.is_empty() { - self.tcx - .sess - .struct_span_err(invalid_args, "arguments should be non-negative integers") - .emit(); - false - } else { - true - } - } - fn check_rustc_layout_scalar_valid_range( &self, attr: &Attribute, diff --git a/compiler/rustc_passes/src/dead.rs b/compiler/rustc_passes/src/dead.rs index c671035bb7..c1b6da97f4 100644 --- a/compiler/rustc_passes/src/dead.rs +++ b/compiler/rustc_passes/src/dead.rs @@ -5,7 +5,7 @@ use rustc_data_structures::fx::{FxHashMap, FxHashSet}; use rustc_hir as hir; use rustc_hir::def::{CtorOf, DefKind, Res}; -use rustc_hir::def_id::{DefId, LOCAL_CRATE}; +use rustc_hir::def_id::DefId; use rustc_hir::intravisit::{self, NestedVisitorMap, Visitor}; use rustc_hir::itemlikevisit::ItemLikeVisitor; use rustc_hir::{Node, PatKind, TyKind}; @@ -44,6 +44,7 @@ struct MarkSymbolVisitor<'tcx> { repr_has_repr_c: bool, in_pat: bool, inherited_pub_visibility: bool, + pub_visibility: bool, ignore_variant_stack: Vec, // maps from tuple struct constructors to tuple struct items struct_constructors: FxHashMap, @@ -133,6 +134,23 @@ impl<'tcx> MarkSymbolVisitor<'tcx> { } } + #[allow(dead_code)] // FIXME(81658): should be used + lint reinstated after #83171 relands. + fn handle_assign(&mut self, expr: &'tcx hir::Expr<'tcx>) { + if self + .typeck_results() + .expr_adjustments(expr) + .iter() + .any(|adj| matches!(adj.kind, ty::adjustment::Adjust::Deref(_))) + { + self.visit_expr(expr); + } else if let hir::ExprKind::Field(base, ..) = expr.kind { + // Ignore write to field + self.handle_assign(base); + } else { + self.visit_expr(expr); + } + } + fn handle_field_pattern_match( &mut self, lhs: &hir::Pat<'_>, @@ -172,27 +190,33 @@ impl<'tcx> MarkSymbolVisitor<'tcx> { fn visit_node(&mut self, node: Node<'tcx>) { let had_repr_c = self.repr_has_repr_c; - self.repr_has_repr_c = false; let had_inherited_pub_visibility = self.inherited_pub_visibility; + let had_pub_visibility = self.pub_visibility; + self.repr_has_repr_c = false; self.inherited_pub_visibility = false; + self.pub_visibility = false; match node { - Node::Item(item) => match item.kind { - hir::ItemKind::Struct(..) | hir::ItemKind::Union(..) => { - let def = self.tcx.adt_def(item.def_id); - self.repr_has_repr_c = def.repr.c(); + Node::Item(item) => { + self.pub_visibility = item.vis.node.is_pub(); - intravisit::walk_item(self, &item); - } - hir::ItemKind::Enum(..) => { - self.inherited_pub_visibility = item.vis.node.is_pub(); + match item.kind { + hir::ItemKind::Struct(..) | hir::ItemKind::Union(..) => { + let def = self.tcx.adt_def(item.def_id); + self.repr_has_repr_c = def.repr.c(); - intravisit::walk_item(self, &item); - } - hir::ItemKind::ForeignMod { .. } => {} - _ => { - intravisit::walk_item(self, &item); + intravisit::walk_item(self, &item); + } + hir::ItemKind::Enum(..) => { + self.inherited_pub_visibility = self.pub_visibility; + + intravisit::walk_item(self, &item); + } + hir::ItemKind::ForeignMod { .. } => {} + _ => { + intravisit::walk_item(self, &item); + } } - }, + } Node::TraitItem(trait_item) => { intravisit::walk_trait_item(self, trait_item); } @@ -204,8 +228,9 @@ impl<'tcx> MarkSymbolVisitor<'tcx> { } _ => {} } - self.repr_has_repr_c = had_repr_c; + self.pub_visibility = had_pub_visibility; self.inherited_pub_visibility = had_inherited_pub_visibility; + self.repr_has_repr_c = had_repr_c; } fn mark_as_used_if_union(&mut self, adt: &ty::AdtDef, fields: &[hir::ExprField<'_>]) { @@ -243,10 +268,10 @@ impl<'tcx> Visitor<'tcx> for MarkSymbolVisitor<'tcx> { ) { let has_repr_c = self.repr_has_repr_c; let inherited_pub_visibility = self.inherited_pub_visibility; - let live_fields = def - .fields() - .iter() - .filter(|f| has_repr_c || inherited_pub_visibility || f.vis.node.is_pub()); + let pub_visibility = self.pub_visibility; + let live_fields = def.fields().iter().filter(|f| { + has_repr_c || (pub_visibility && (inherited_pub_visibility || f.vis.node.is_pub())) + }); self.live_symbols.extend(live_fields.map(|f| f.hir_id)); intravisit::walk_struct_def(self, def); @@ -451,7 +476,7 @@ fn create_and_seed_worklist<'tcx>( ) .chain( // Seed entry point - tcx.entry_fn(LOCAL_CRATE).and_then(|(def_id, _)| { + tcx.entry_fn(()).and_then(|(def_id, _)| { def_id.as_local().map(|def_id| tcx.hir().local_def_id_to_hir_id(def_id)) }), ) @@ -479,6 +504,7 @@ fn find_live<'tcx>( repr_has_repr_c: false, in_pat: false, inherited_pub_visibility: false, + pub_visibility: false, ignore_variant_stack: vec![], struct_constructors, }; @@ -687,7 +713,7 @@ impl Visitor<'tcx> for DeadVisitor<'tcx> { } pub fn check_crate(tcx: TyCtxt<'_>) { - let access_levels = &tcx.privacy_access_levels(LOCAL_CRATE); + let access_levels = &tcx.privacy_access_levels(()); let krate = tcx.hir().krate(); let live_symbols = find_live(tcx, access_levels, krate); let mut visitor = DeadVisitor { tcx, live_symbols }; diff --git a/compiler/rustc_passes/src/diagnostic_items.rs b/compiler/rustc_passes/src/diagnostic_items.rs index 8dd3700e5b..f345d45d17 100644 --- a/compiler/rustc_passes/src/diagnostic_items.rs +++ b/compiler/rustc_passes/src/diagnostic_items.rs @@ -16,7 +16,7 @@ use rustc_hir::itemlikevisit::ItemLikeVisitor; use rustc_middle::ty::query::Providers; use rustc_middle::ty::TyCtxt; use rustc_session::Session; -use rustc_span::def_id::{DefId, LocalDefId, LOCAL_CRATE}; +use rustc_span::def_id::{CrateNum, DefId, LocalDefId, LOCAL_CRATE}; use rustc_span::symbol::{sym, Symbol}; struct DiagnosticItemCollector<'tcx> { @@ -99,7 +99,9 @@ fn extract(sess: &Session, attrs: &[ast::Attribute]) -> Option { } /// Traverse and collect the diagnostic items in the current -fn collect<'tcx>(tcx: TyCtxt<'tcx>) -> FxHashMap { +fn diagnostic_items<'tcx>(tcx: TyCtxt<'tcx>, cnum: CrateNum) -> FxHashMap { + assert_eq!(cnum, LOCAL_CRATE); + // Initialize the collector. let mut collector = DiagnosticItemCollector::new(tcx); @@ -114,7 +116,7 @@ fn collect<'tcx>(tcx: TyCtxt<'tcx>) -> FxHashMap { } /// Traverse and collect all the diagnostic items in all crates. -fn collect_all<'tcx>(tcx: TyCtxt<'tcx>) -> FxHashMap { +fn all_diagnostic_items<'tcx>(tcx: TyCtxt<'tcx>, (): ()) -> FxHashMap { // Initialize the collector. let mut collector = FxHashMap::default(); @@ -129,12 +131,6 @@ fn collect_all<'tcx>(tcx: TyCtxt<'tcx>) -> FxHashMap { } pub fn provide(providers: &mut Providers) { - providers.diagnostic_items = |tcx, id| { - assert_eq!(id, LOCAL_CRATE); - collect(tcx) - }; - providers.all_diagnostic_items = |tcx, id| { - assert_eq!(id, LOCAL_CRATE); - collect_all(tcx) - }; + providers.diagnostic_items = diagnostic_items; + providers.all_diagnostic_items = all_diagnostic_items; } diff --git a/compiler/rustc_passes/src/entry.rs b/compiler/rustc_passes/src/entry.rs index 65cfe986ad..ca6a7561f3 100644 --- a/compiler/rustc_passes/src/entry.rs +++ b/compiler/rustc_passes/src/entry.rs @@ -1,6 +1,6 @@ use rustc_ast::entry::EntryPointType; use rustc_errors::struct_span_err; -use rustc_hir::def_id::{CrateNum, DefId, CRATE_DEF_INDEX, LOCAL_CRATE}; +use rustc_hir::def_id::{DefId, CRATE_DEF_INDEX, LOCAL_CRATE}; use rustc_hir::itemlikevisit::ItemLikeVisitor; use rustc_hir::{ForeignItem, HirId, ImplItem, Item, ItemKind, TraitItem, CRATE_HIR_ID}; use rustc_middle::hir::map::Map; @@ -48,9 +48,7 @@ impl<'a, 'tcx> ItemLikeVisitor<'tcx> for EntryContext<'a, 'tcx> { } } -fn entry_fn(tcx: TyCtxt<'_>, cnum: CrateNum) -> Option<(DefId, EntryFnType)> { - assert_eq!(cnum, LOCAL_CRATE); - +fn entry_fn(tcx: TyCtxt<'_>, (): ()) -> Option<(DefId, EntryFnType)> { let any_exe = tcx.sess.crate_types().iter().any(|ty| *ty == CrateType::Executable); if !any_exe { // No need to find a main function. @@ -227,10 +225,6 @@ fn no_main_err(tcx: TyCtxt<'_>, visitor: &EntryContext<'_, '_>) { err.emit(); } -pub fn find_entry_point(tcx: TyCtxt<'_>) -> Option<(DefId, EntryFnType)> { - tcx.entry_fn(LOCAL_CRATE) -} - pub fn provide(providers: &mut Providers) { *providers = Providers { entry_fn, ..*providers }; } diff --git a/compiler/rustc_passes/src/intrinsicck.rs b/compiler/rustc_passes/src/intrinsicck.rs index 3f095d0e82..012d97ef10 100644 --- a/compiler/rustc_passes/src/intrinsicck.rs +++ b/compiler/rustc_passes/src/intrinsicck.rs @@ -347,7 +347,7 @@ impl ExprVisitor<'tcx> { } fn check_asm(&self, asm: &hir::InlineAsm<'tcx>) { - for (idx, (op, op_sp)) in asm.operands.iter().enumerate() { + for (idx, (op, _)) in asm.operands.iter().enumerate() { match *op { hir::InlineAsmOperand::In { reg, ref expr } => { self.check_asm_operand_type(idx, reg, expr, asm.template, None); @@ -372,19 +372,7 @@ impl ExprVisitor<'tcx> { ); } } - hir::InlineAsmOperand::Const { ref anon_const } => { - let anon_const_def_id = self.tcx.hir().local_def_id(anon_const.hir_id); - let value = ty::Const::from_anon_const(self.tcx, anon_const_def_id); - match value.ty.kind() { - ty::Int(_) | ty::Uint(_) | ty::Float(_) => {} - _ => { - let msg = - "asm `const` arguments must be integer or floating-point values"; - self.tcx.sess.span_err(*op_sp, msg); - } - } - } - hir::InlineAsmOperand::Sym { .. } => {} + hir::InlineAsmOperand::Const { .. } | hir::InlineAsmOperand::Sym { .. } => {} } } } diff --git a/compiler/rustc_passes/src/lang_items.rs b/compiler/rustc_passes/src/lang_items.rs index 7e6bb9779f..118fcca450 100644 --- a/compiler/rustc_passes/src/lang_items.rs +++ b/compiler/rustc_passes/src/lang_items.rs @@ -13,12 +13,13 @@ use crate::weak_lang_items; use rustc_middle::middle::cstore::ExternCrate; use rustc_middle::ty::TyCtxt; -use rustc_errors::struct_span_err; +use rustc_errors::{pluralize, struct_span_err}; use rustc_hir as hir; -use rustc_hir::def_id::{DefId, LOCAL_CRATE}; +use rustc_hir::def_id::DefId; use rustc_hir::itemlikevisit::ItemLikeVisitor; use rustc_hir::lang_items::{extract, ITEM_REFS}; use rustc_hir::{HirId, LangItem, LanguageItems, Target}; +use rustc_span::Span; use rustc_middle::ty::query::Providers; @@ -61,8 +62,7 @@ impl LanguageItemCollector<'tcx> { match ITEM_REFS.get(&value).cloned() { // Known lang item with attribute on correct target. Some((item_index, expected_target)) if actual_target == expected_target => { - let def_id = self.tcx.hir().local_def_id(hir_id); - self.collect_item(item_index, def_id.to_def_id()); + self.collect_item_extended(item_index, hir_id, span); } // Known lang item with attribute on incorrect target. Some((_, expected_target)) => { @@ -180,10 +180,131 @@ impl LanguageItemCollector<'tcx> { self.items.groups[group as usize].push(item_def_id); } } + + // Like collect_item() above, but also checks whether the lang item is declared + // with the right number of generic arguments if it is a trait. + fn collect_item_extended(&mut self, item_index: usize, hir_id: HirId, span: Span) { + let item_def_id = self.tcx.hir().local_def_id(hir_id).to_def_id(); + let lang_item = LangItem::from_u32(item_index as u32).unwrap(); + let name = lang_item.name(); + + self.collect_item(item_index, item_def_id); + + // Now check whether the lang_item has the expected number of generic + // arguments if it is a trait. Generally speaking, binary and indexing + // operations have one (for the RHS/index), unary operations have none, + // and the rest also have none except for the closure traits (one for + // the argument list), generators (one for the resume argument), + // ordering/equality relations (one for the RHS), and various conversion + // traits. + + let expected_num = match lang_item { + // Binary operations + LangItem::Add + | LangItem::Sub + | LangItem::Mul + | LangItem::Div + | LangItem::Rem + | LangItem::BitXor + | LangItem::BitAnd + | LangItem::BitOr + | LangItem::Shl + | LangItem::Shr + | LangItem::AddAssign + | LangItem::SubAssign + | LangItem::MulAssign + | LangItem::DivAssign + | LangItem::RemAssign + | LangItem::BitXorAssign + | LangItem::BitAndAssign + | LangItem::BitOrAssign + | LangItem::ShlAssign + | LangItem::ShrAssign + | LangItem::Index + | LangItem::IndexMut + + // Miscellaneous + | LangItem::Unsize + | LangItem::CoerceUnsized + | LangItem::DispatchFromDyn + | LangItem::Fn + | LangItem::FnMut + | LangItem::FnOnce + | LangItem::Generator + | LangItem::PartialEq + | LangItem::PartialOrd + => Some(1), + + // Unary operations + LangItem::Neg + | LangItem::Not + + // Miscellaneous + | LangItem::Deref + | LangItem::DerefMut + | LangItem::Sized + | LangItem::StructuralPeq + | LangItem::StructuralTeq + | LangItem::Copy + | LangItem::Clone + | LangItem::Sync + | LangItem::DiscriminantKind + | LangItem::PointeeTrait + | LangItem::Freeze + | LangItem::Drop + | LangItem::Receiver + | LangItem::Future + | LangItem::Unpin + | LangItem::Termination + | LangItem::Try + | LangItem::Send + | LangItem::UnwindSafe + | LangItem::RefUnwindSafe + => Some(0), + + // Not a trait + _ => None, + }; + + if let Some(expected_num) = expected_num { + let (actual_num, generics_span) = match self.tcx.hir().get(hir_id) { + hir::Node::Item(hir::Item { + kind: hir::ItemKind::Trait(_, _, generics, ..), + .. + }) => (generics.params.len(), generics.span), + _ => bug!("op/index/deref lang item target is not a trait: {:?}", lang_item), + }; + + if expected_num != actual_num { + // We are issuing E0718 "incorrect target" here, because while the + // item kind of the target is correct, the target is still wrong + // because of the wrong number of generic arguments. + struct_span_err!( + self.tcx.sess, + span, + E0718, + "`{}` language item must be applied to a trait with {} generic argument{}", + name, + expected_num, + pluralize!(expected_num) + ) + .span_label( + generics_span, + format!( + "this trait has {} generic argument{}, not {}", + actual_num, + pluralize!(actual_num), + expected_num + ), + ) + .emit(); + } + } + } } /// Traverses and collects all the lang items in all crates. -fn collect(tcx: TyCtxt<'_>) -> LanguageItems { +fn get_lang_items(tcx: TyCtxt<'_>, (): ()) -> LanguageItems { // Initialize the collector. let mut collector = LanguageItemCollector::new(tcx); @@ -207,8 +328,5 @@ fn collect(tcx: TyCtxt<'_>) -> LanguageItems { } pub fn provide(providers: &mut Providers) { - providers.get_lang_items = |tcx, id| { - assert_eq!(id, LOCAL_CRATE); - collect(tcx) - }; + providers.get_lang_items = get_lang_items; } diff --git a/compiler/rustc_passes/src/lib.rs b/compiler/rustc_passes/src/lib.rs index 0be7ef7e12..28633faa20 100644 --- a/compiler/rustc_passes/src/lib.rs +++ b/compiler/rustc_passes/src/lib.rs @@ -5,12 +5,11 @@ //! This API is completely unstable and subject to change. #![doc(html_root_url = "https://doc.rust-lang.org/nightly/nightly-rustc/")] -#![feature(const_panic)] #![feature(crate_visibility_modifier)] #![feature(in_band_lifetimes)] #![feature(iter_zip)] #![feature(nll)] -#![cfg_attr(bootstrap, feature(or_patterns))] +#![feature(min_specialization)] #![recursion_limit = "256"] #[macro_use] diff --git a/compiler/rustc_passes/src/lib_features.rs b/compiler/rustc_passes/src/lib_features.rs index 3dfe317a4b..363a6417f9 100644 --- a/compiler/rustc_passes/src/lib_features.rs +++ b/compiler/rustc_passes/src/lib_features.rs @@ -6,7 +6,6 @@ use rustc_ast::{Attribute, MetaItem, MetaItemKind}; use rustc_errors::struct_span_err; -use rustc_hir::def_id::LOCAL_CRATE; use rustc_hir::intravisit::{self, NestedVisitorMap, Visitor}; use rustc_middle::hir::map::Map; use rustc_middle::middle::lib_features::LibFeatures; @@ -127,7 +126,7 @@ impl Visitor<'tcx> for LibFeatureCollector<'tcx> { } } -fn collect(tcx: TyCtxt<'_>) -> LibFeatures { +fn get_lib_features(tcx: TyCtxt<'_>, (): ()) -> LibFeatures { let mut collector = LibFeatureCollector::new(tcx); let krate = tcx.hir().krate(); for attr in krate.non_exported_macro_attrs { @@ -138,8 +137,5 @@ fn collect(tcx: TyCtxt<'_>) -> LibFeatures { } pub fn provide(providers: &mut Providers) { - providers.get_lib_features = |tcx, id| { - assert_eq!(id, LOCAL_CRATE); - collect(tcx) - }; + providers.get_lib_features = get_lib_features; } diff --git a/compiler/rustc_passes/src/liveness.rs b/compiler/rustc_passes/src/liveness.rs index fa930471c2..4ceefa17bc 100644 --- a/compiler/rustc_passes/src/liveness.rs +++ b/compiler/rustc_passes/src/liveness.rs @@ -132,9 +132,9 @@ enum LiveNodeKind { fn live_node_kind_to_string(lnk: LiveNodeKind, tcx: TyCtxt<'_>) -> String { let sm = tcx.sess.source_map(); match lnk { - UpvarNode(s) => format!("Upvar node [{}]", sm.span_to_string(s)), - ExprNode(s) => format!("Expr node [{}]", sm.span_to_string(s)), - VarDefNode(s) => format!("Var def node [{}]", sm.span_to_string(s)), + UpvarNode(s) => format!("Upvar node [{}]", sm.span_to_diagnostic_string(s)), + ExprNode(s) => format!("Expr node [{}]", sm.span_to_diagnostic_string(s)), + VarDefNode(s) => format!("Var def node [{}]", sm.span_to_diagnostic_string(s)), ClosureNode => "Closure node".to_owned(), ExitNode => "Exit node".to_owned(), } diff --git a/compiler/rustc_passes/src/reachable.rs b/compiler/rustc_passes/src/reachable.rs index 20aaaea5b9..0b3227abb5 100644 --- a/compiler/rustc_passes/src/reachable.rs +++ b/compiler/rustc_passes/src/reachable.rs @@ -8,8 +8,7 @@ use rustc_data_structures::fx::FxHashSet; use rustc_hir as hir; use rustc_hir::def::{DefKind, Res}; -use rustc_hir::def_id::LOCAL_CRATE; -use rustc_hir::def_id::{CrateNum, DefId, LocalDefId}; +use rustc_hir::def_id::{DefId, LocalDefId}; use rustc_hir::intravisit::{self, NestedVisitorMap, Visitor}; use rustc_hir::itemlikevisit::ItemLikeVisitor; use rustc_hir::Node; @@ -250,7 +249,7 @@ impl<'tcx> ReachableContext<'tcx> { // Reachable constants will be inlined into other crates // unconditionally, so we need to make sure that their // contents are also reachable. - hir::ItemKind::Const(_, init) => { + hir::ItemKind::Const(_, init) | hir::ItemKind::Static(_, _, init) => { self.visit_nested_body(init); } @@ -261,7 +260,6 @@ impl<'tcx> ReachableContext<'tcx> { | hir::ItemKind::Use(..) | hir::ItemKind::OpaqueTy(..) | hir::ItemKind::TyAlias(..) - | hir::ItemKind::Static(..) | hir::ItemKind::Mod(..) | hir::ItemKind::ForeignMod { .. } | hir::ItemKind::Impl { .. } @@ -386,10 +384,8 @@ impl<'a, 'tcx> ItemLikeVisitor<'tcx> for CollectPrivateImplItemsVisitor<'a, 'tcx } } -fn reachable_set<'tcx>(tcx: TyCtxt<'tcx>, crate_num: CrateNum) -> FxHashSet { - debug_assert!(crate_num == LOCAL_CRATE); - - let access_levels = &tcx.privacy_access_levels(LOCAL_CRATE); +fn reachable_set<'tcx>(tcx: TyCtxt<'tcx>, (): ()) -> FxHashSet { + let access_levels = &tcx.privacy_access_levels(()); let any_library = tcx.sess.crate_types().iter().any(|ty| { diff --git a/compiler/rustc_passes/src/region.rs b/compiler/rustc_passes/src/region.rs index 14a373c594..c133f1a041 100644 --- a/compiler/rustc_passes/src/region.rs +++ b/compiler/rustc_passes/src/region.rs @@ -717,7 +717,7 @@ impl<'tcx> Visitor<'tcx> for RegionResolutionVisitor<'tcx> { debug!( "visit_body(id={:?}, span={:?}, body.id={:?}, cx.parent={:?})", owner_id, - self.tcx.sess.source_map().span_to_string(body.value.span), + self.tcx.sess.source_map().span_to_diagnostic_string(body.value.span), body_id, self.cx.parent ); diff --git a/compiler/rustc_passes/src/stability.rs b/compiler/rustc_passes/src/stability.rs index 9c4f9b1198..f41e0e0370 100644 --- a/compiler/rustc_passes/src/stability.rs +++ b/compiler/rustc_passes/src/stability.rs @@ -629,7 +629,7 @@ impl<'tcx> Visitor<'tcx> for MissingStabilityAnnotations<'tcx> { // stable (assuming they have not inherited instability from their parent). } -fn new_index(tcx: TyCtxt<'tcx>) -> Index<'tcx> { +fn stability_index(tcx: TyCtxt<'tcx>, (): ()) -> Index<'tcx> { let is_staged_api = tcx.sess.opts.debugging_opts.force_unstable_if_unmarked || tcx.features().staged_api; let mut staged_api = FxHashMap::default(); @@ -704,11 +704,7 @@ fn check_mod_unstable_api_usage(tcx: TyCtxt<'_>, module_def_id: LocalDefId) { } pub(crate) fn provide(providers: &mut Providers) { - *providers = Providers { check_mod_unstable_api_usage, ..*providers }; - providers.stability_index = |tcx, cnum| { - assert_eq!(cnum, LOCAL_CRATE); - new_index(tcx) - }; + *providers = Providers { check_mod_unstable_api_usage, stability_index, ..*providers }; } struct Checker<'tcx> { @@ -739,7 +735,7 @@ impl Visitor<'tcx> for Checker<'tcx> { None => return, }; let def_id = DefId { krate: cnum, index: CRATE_DEF_INDEX }; - self.tcx.check_stability(def_id, Some(item.hir_id()), item.span); + self.tcx.check_stability(def_id, Some(item.hir_id()), item.span, None); } // For implementations of traits, check the stability of each item @@ -783,7 +779,7 @@ impl Visitor<'tcx> for Checker<'tcx> { .map(|item| item.def_id); if let Some(def_id) = trait_item_def_id { // Pass `None` to skip deprecation warnings. - self.tcx.check_stability(def_id, None, impl_item.span); + self.tcx.check_stability(def_id, None, impl_item.span, None); } } } @@ -832,7 +828,7 @@ impl Visitor<'tcx> for Checker<'tcx> { fn visit_path(&mut self, path: &'tcx hir::Path<'tcx>, id: hir::HirId) { if let Some(def_id) = path.res.opt_def_id() { - self.tcx.check_stability(def_id, Some(id), path.span) + self.tcx.check_stability(def_id, Some(id), path.span, None) } intravisit::walk_path(self, path) } @@ -880,7 +876,7 @@ impl Visitor<'tcx> for CheckTraitImplStable<'tcx> { /// were expected to be library features), and the list of features used from /// libraries, identify activated features that don't exist and error about them. pub fn check_unused_or_stable_features(tcx: TyCtxt<'_>) { - let access_levels = &tcx.privacy_access_levels(LOCAL_CRATE); + let access_levels = &tcx.privacy_access_levels(()); if tcx.stability().staged_api[&LOCAL_CRATE] { let krate = tcx.hir().krate(); diff --git a/compiler/rustc_plugin_impl/src/build.rs b/compiler/rustc_plugin_impl/src/build.rs index a49afa35e4..b95c4a7201 100644 --- a/compiler/rustc_plugin_impl/src/build.rs +++ b/compiler/rustc_plugin_impl/src/build.rs @@ -1,7 +1,7 @@ //! Used by `rustc` when compiling a plugin crate. use rustc_hir as hir; -use rustc_hir::def_id::{CrateNum, DefId, LocalDefId, LOCAL_CRATE}; +use rustc_hir::def_id::LocalDefId; use rustc_hir::itemlikevisit::ItemLikeVisitor; use rustc_middle::ty::query::Providers; use rustc_middle::ty::TyCtxt; @@ -31,33 +31,25 @@ impl<'v, 'tcx> ItemLikeVisitor<'v> for RegistrarFinder<'tcx> { } /// Finds the function marked with `#[plugin_registrar]`, if any. -pub fn find_plugin_registrar(tcx: TyCtxt<'_>) -> Option { - tcx.plugin_registrar_fn(LOCAL_CRATE) -} - -fn plugin_registrar_fn(tcx: TyCtxt<'_>, cnum: CrateNum) -> Option { - assert_eq!(cnum, LOCAL_CRATE); - +fn plugin_registrar_fn(tcx: TyCtxt<'_>, (): ()) -> Option { let mut finder = RegistrarFinder { tcx, registrars: Vec::new() }; tcx.hir().krate().visit_all_item_likes(&mut finder); - match finder.registrars.len() { - 0 => None, - 1 => { - let (def_id, _) = finder.registrars.pop().unwrap(); - Some(def_id.to_def_id()) - } - _ => { - let diagnostic = tcx.sess.diagnostic(); - let mut e = diagnostic.struct_err("multiple plugin registration functions found"); - for &(_, span) in &finder.registrars { - e.span_note(span, "one is here"); - } - e.emit(); - diagnostic.abort_if_errors(); - unreachable!(); + let (def_id, span) = finder.registrars.pop()?; + + if !finder.registrars.is_empty() { + let diagnostic = tcx.sess.diagnostic(); + let mut e = diagnostic.struct_err("multiple plugin registration functions found"); + e.span_note(span, "one is here"); + for &(_, span) in &finder.registrars { + e.span_note(span, "one is here"); } + e.emit(); + diagnostic.abort_if_errors(); + unreachable!(); } + + Some(def_id) } pub fn provide(providers: &mut Providers) { diff --git a/compiler/rustc_privacy/src/lib.rs b/compiler/rustc_privacy/src/lib.rs index d37a5be2fe..e64f12ef48 100644 --- a/compiler/rustc_privacy/src/lib.rs +++ b/compiler/rustc_privacy/src/lib.rs @@ -1,7 +1,6 @@ #![doc(html_root_url = "https://doc.rust-lang.org/nightly/nightly-rustc/")] #![feature(in_band_lifetimes)] #![feature(nll)] -#![cfg_attr(bootstrap, feature(or_patterns))] #![feature(control_flow_enum)] #![feature(try_blocks)] #![feature(associated_type_defaults)] @@ -12,7 +11,7 @@ use rustc_data_structures::fx::FxHashSet; use rustc_errors::struct_span_err; use rustc_hir as hir; use rustc_hir::def::{DefKind, Res}; -use rustc_hir::def_id::{CrateNum, DefId, LocalDefId, CRATE_DEF_INDEX, LOCAL_CRATE}; +use rustc_hir::def_id::{DefId, LocalDefId, CRATE_DEF_INDEX, LOCAL_CRATE}; use rustc_hir::intravisit::{self, DeepVisitor, NestedVisitorMap, Visitor}; use rustc_hir::{AssocItemKind, HirIdSet, Node, PatKind}; use rustc_middle::bug; @@ -2092,9 +2091,7 @@ fn check_mod_privacy(tcx: TyCtxt<'_>, module_def_id: LocalDefId) { intravisit::walk_mod(&mut visitor, module, hir_id); } -fn privacy_access_levels(tcx: TyCtxt<'_>, krate: CrateNum) -> &AccessLevels { - assert_eq!(krate, LOCAL_CRATE); - +fn privacy_access_levels(tcx: TyCtxt<'_>, (): ()) -> &AccessLevels { // Build up a set of all exported items in the AST. This is a set of all // items which are reachable from external crates based on visibility. let mut visitor = EmbargoVisitor { @@ -2117,10 +2114,8 @@ fn privacy_access_levels(tcx: TyCtxt<'_>, krate: CrateNum) -> &AccessLevels { tcx.arena.alloc(visitor.access_levels) } -fn check_private_in_public(tcx: TyCtxt<'_>, krate: CrateNum) { - assert_eq!(krate, LOCAL_CRATE); - - let access_levels = tcx.privacy_access_levels(LOCAL_CRATE); +fn check_private_in_public(tcx: TyCtxt<'_>, (): ()) { + let access_levels = tcx.privacy_access_levels(()); let krate = tcx.hir().krate(); diff --git a/compiler/rustc_query_impl/src/keys.rs b/compiler/rustc_query_impl/src/keys.rs index 1fdb37398f..27a0dc4768 100644 --- a/compiler/rustc_query_impl/src/keys.rs +++ b/compiler/rustc_query_impl/src/keys.rs @@ -21,6 +21,16 @@ pub trait Key { fn default_span(&self, tcx: TyCtxt<'_>) -> Span; } +impl Key for () { + fn query_crate(&self) -> CrateNum { + LOCAL_CRATE + } + + fn default_span(&self, _: TyCtxt<'_>) -> Span { + DUMMY_SP + } +} + impl<'tcx> Key for ty::InstanceDef<'tcx> { fn query_crate(&self) -> CrateNum { LOCAL_CRATE diff --git a/compiler/rustc_query_impl/src/lib.rs b/compiler/rustc_query_impl/src/lib.rs index 00d886000f..4175fb6925 100644 --- a/compiler/rustc_query_impl/src/lib.rs +++ b/compiler/rustc_query_impl/src/lib.rs @@ -2,13 +2,9 @@ #![doc(html_root_url = "https://doc.rust-lang.org/nightly/nightly-rustc/")] #![feature(in_band_lifetimes)] -#![feature(exhaustive_patterns)] #![feature(nll)] #![feature(min_specialization)] -#![feature(crate_visibility_modifier)] -#![feature(once_cell)] #![feature(rustc_attrs)] -#![feature(never_type)] #![recursion_limit = "256"] #[macro_use] @@ -26,7 +22,7 @@ use rustc_middle::ty::query::{query_keys, query_storage, query_stored, query_val use rustc_middle::ty::query::{Providers, QueryEngine}; use rustc_middle::ty::{self, TyCtxt}; use rustc_serialize::opaque; -use rustc_span::{Span, DUMMY_SP}; +use rustc_span::Span; #[macro_use] mod plumbing; diff --git a/compiler/rustc_query_impl/src/plumbing.rs b/compiler/rustc_query_impl/src/plumbing.rs index ee914fa1ba..b4191c135b 100644 --- a/compiler/rustc_query_impl/src/plumbing.rs +++ b/compiler/rustc_query_impl/src/plumbing.rs @@ -3,7 +3,7 @@ //! manage the caches, and so forth. use super::queries; -use rustc_middle::dep_graph::{DepKind, DepNode, DepNodeExt, DepNodeIndex, SerializedDepNodeIndex}; +use rustc_middle::dep_graph::{DepKind, DepNode, DepNodeIndex, SerializedDepNodeIndex}; use rustc_middle::ty::query::on_disk_cache; use rustc_middle::ty::tls::{self, ImplicitCtxt}; use rustc_middle::ty::{self, TyCtxt}; @@ -14,7 +14,7 @@ use rustc_data_structures::sync::Lock; use rustc_data_structures::thin_vec::ThinVec; use rustc_errors::Diagnostic; use rustc_serialize::opaque; -use rustc_span::def_id::{DefId, LocalDefId}; +use rustc_span::def_id::LocalDefId; #[derive(Copy, Clone)] pub struct QueryCtxt<'tcx> { @@ -25,6 +25,7 @@ pub struct QueryCtxt<'tcx> { impl<'tcx> std::ops::Deref for QueryCtxt<'tcx> { type Target = TyCtxt<'tcx>; + #[inline] fn deref(&self) -> &Self::Target { &self.tcx } @@ -42,10 +43,6 @@ impl HasDepContext for QueryCtxt<'tcx> { } impl QueryContext for QueryCtxt<'tcx> { - fn def_path_str(&self, def_id: DefId) -> String { - self.tcx.def_path_str(def_id) - } - fn current_query_job(&self) -> Option> { tls::with_related_context(**self, |icx| icx.query) } @@ -60,39 +57,6 @@ impl QueryContext for QueryCtxt<'tcx> { } fn try_force_from_dep_node(&self, dep_node: &DepNode) -> bool { - // FIXME: This match is just a workaround for incremental bugs and should - // be removed. https://github.com/rust-lang/rust/issues/62649 is one such - // bug that must be fixed before removing this. - match dep_node.kind { - DepKind::hir_owner | DepKind::hir_owner_nodes => { - if let Some(def_id) = dep_node.extract_def_id(**self) { - let def_id = def_id.expect_local(); - let hir_id = self.tcx.hir().local_def_id_to_hir_id(def_id); - if def_id != hir_id.owner { - // This `DefPath` does not have a - // corresponding `DepNode` (e.g. a - // struct field), and the ` DefPath` - // collided with the `DefPath` of a - // proper item that existed in the - // previous compilation session. - // - // Since the given `DefPath` does not - // denote the item that previously - // existed, we just fail to mark green. - return false; - } - } else { - // If the node does not exist anymore, we - // just fail to mark green. - return false; - } - } - _ => { - // For other kinds of nodes it's OK to be - // forced. - } - } - debug!("try_force_from_dep_node({:?}) --- trying to force", dep_node); // We must avoid ever having to call `force_from_dep_node()` for a @@ -457,20 +421,7 @@ macro_rules! define_queries { } fn force_from_dep_node(tcx: QueryCtxt<'_>, dep_node: &DepNode) -> bool { - if is_anon { - return false; - } - - if !can_reconstruct_query_key() { - return false; - } - - if let Some(key) = recover(*tcx, dep_node) { - force_query::, _>(tcx, key, DUMMY_SP, *dep_node); - return true; - } - - false + force_query::, _>(tcx, dep_node) } fn try_load_from_on_disk_cache(tcx: QueryCtxt<'_>, dep_node: &DepNode) { @@ -550,12 +501,10 @@ macro_rules! define_queries_struct { } impl QueryEngine<'tcx> for Queries<'tcx> { - unsafe fn deadlock(&'tcx self, _tcx: TyCtxt<'tcx>, _registry: &rustc_rayon_core::Registry) { - #[cfg(parallel_compiler)] - { - let tcx = QueryCtxt { tcx: _tcx, queries: self }; - rustc_query_system::query::deadlock(tcx, _registry) - } + #[cfg(parallel_compiler)] + unsafe fn deadlock(&'tcx self, tcx: TyCtxt<'tcx>, registry: &rustc_rayon_core::Registry) { + let tcx = QueryCtxt { tcx, queries: self }; + rustc_query_system::query::deadlock(tcx, registry) } fn encode_query_results( diff --git a/compiler/rustc_query_impl/src/profiling_support.rs b/compiler/rustc_query_impl/src/profiling_support.rs index 2517793ece..95edc1e93a 100644 --- a/compiler/rustc_query_impl/src/profiling_support.rs +++ b/compiler/rustc_query_impl/src/profiling_support.rs @@ -61,7 +61,7 @@ impl<'p, 'c, 'tcx> QueryKeyStringBuilder<'p, 'c, 'tcx> { match def_key.disambiguated_data.data { DefPathData::CrateRoot => { - crate_name = self.tcx.original_crate_name(def_id.krate).as_str(); + crate_name = self.tcx.crate_name(def_id.krate).as_str(); name = &*crate_name; dis = ""; end_index = 3; diff --git a/compiler/rustc_query_system/src/dep_graph/graph.rs b/compiler/rustc_query_system/src/dep_graph/graph.rs index 7a0fc32066..71e67dfee5 100644 --- a/compiler/rustc_query_system/src/dep_graph/graph.rs +++ b/compiler/rustc_query_system/src/dep_graph/graph.rs @@ -19,9 +19,8 @@ use std::marker::PhantomData; use std::mem; use std::sync::atomic::Ordering::Relaxed; -use super::prev::PreviousDepGraph; use super::query::DepGraphQuery; -use super::serialized::{GraphEncoder, SerializedDepNodeIndex}; +use super::serialized::{GraphEncoder, SerializedDepGraph, SerializedDepNodeIndex}; use super::{DepContext, DepKind, DepNode, HasDepContext, WorkProductId}; use crate::query::QueryContext; @@ -45,6 +44,7 @@ rustc_index::newtype_index! { impl DepNodeIndex { pub const INVALID: DepNodeIndex = DepNodeIndex::MAX; + pub const SINGLETON_DEPENDENCYLESS_ANON_NODE: DepNodeIndex = DepNodeIndex::from_u32(0); } impl std::convert::From for QueryInvocationId { @@ -78,7 +78,7 @@ struct DepGraphData { /// The dep-graph from the previous compilation session. It contains all /// nodes and edges as well as all fingerprints of nodes that have them. - previous: PreviousDepGraph, + previous: SerializedDepGraph, colors: DepNodeColorMap, @@ -109,7 +109,8 @@ where impl DepGraph { pub fn new( - prev_graph: PreviousDepGraph, + profiler: &SelfProfilerRef, + prev_graph: SerializedDepGraph, prev_work_products: FxHashMap, encoder: FileEncoder, record_graph: bool, @@ -117,16 +118,23 @@ impl DepGraph { ) -> DepGraph { let prev_graph_node_count = prev_graph.node_count(); + let current = + CurrentDepGraph::new(prev_graph_node_count, encoder, record_graph, record_stats); + + // Instantiate a dependy-less node only once for anonymous queries. + let _green_node_index = current.intern_new_node( + profiler, + DepNode { kind: DepKind::NULL, hash: current.anon_id_seed.into() }, + smallvec![], + Fingerprint::ZERO, + ); + debug_assert_eq!(_green_node_index, DepNodeIndex::SINGLETON_DEPENDENCYLESS_ANON_NODE); + DepGraph { data: Some(Lrc::new(DepGraphData { previous_work_products: prev_work_products, dep_node_debug: Default::default(), - current: CurrentDepGraph::new( - prev_graph_node_count, - encoder, - record_graph, - record_stats, - ), + current, emitting_diagnostics: Default::default(), emitting_diagnostics_cond_var: Condvar::new(), previous: prev_graph, @@ -288,30 +296,47 @@ impl DepGraph { let task_deps = Lock::new(TaskDeps::default()); let result = K::with_deps(Some(&task_deps), op); let task_deps = task_deps.into_inner(); + let task_deps = task_deps.reads; + + let dep_node_index = match task_deps.len() { + 0 => { + // Because the dep-node id of anon nodes is computed from the sets of its + // dependencies we already know what the ID of this dependency-less node is + // going to be (i.e. equal to the precomputed + // `SINGLETON_DEPENDENCYLESS_ANON_NODE`). As a consequence we can skip creating + // a `StableHasher` and sending the node through interning. + DepNodeIndex::SINGLETON_DEPENDENCYLESS_ANON_NODE + } + 1 => { + // When there is only one dependency, don't bother creating a node. + task_deps[0] + } + _ => { + // The dep node indices are hashed here instead of hashing the dep nodes of the + // dependencies. These indices may refer to different nodes per session, but this isn't + // a problem here because we that ensure the final dep node hash is per session only by + // combining it with the per session random number `anon_id_seed`. This hash only need + // to map the dependencies to a single value on a per session basis. + let mut hasher = StableHasher::new(); + task_deps.hash(&mut hasher); + + let target_dep_node = DepNode { + kind: dep_kind, + // Fingerprint::combine() is faster than sending Fingerprint + // through the StableHasher (at least as long as StableHasher + // is so slow). + hash: data.current.anon_id_seed.combine(hasher.finish()).into(), + }; - // The dep node indices are hashed here instead of hashing the dep nodes of the - // dependencies. These indices may refer to different nodes per session, but this isn't - // a problem here because we that ensure the final dep node hash is per session only by - // combining it with the per session random number `anon_id_seed`. This hash only need - // to map the dependencies to a single value on a per session basis. - let mut hasher = StableHasher::new(); - task_deps.reads.hash(&mut hasher); - - let target_dep_node = DepNode { - kind: dep_kind, - // Fingerprint::combine() is faster than sending Fingerprint - // through the StableHasher (at least as long as StableHasher - // is so slow). - hash: data.current.anon_id_seed.combine(hasher.finish()).into(), + data.current.intern_new_node( + cx.profiler(), + target_dep_node, + task_deps, + Fingerprint::ZERO, + ) + } }; - let dep_node_index = data.current.intern_new_node( - cx.profiler(), - target_dep_node, - task_deps.reads, - Fingerprint::ZERO, - ); - (result, dep_node_index) } else { (op(), self.next_virtual_depnode_index()) @@ -488,6 +513,117 @@ impl DepGraph { } } + fn try_mark_parent_green>( + &self, + tcx: Ctxt, + data: &DepGraphData, + parent_dep_node_index: SerializedDepNodeIndex, + dep_node: &DepNode, + ) -> Option<()> { + let dep_dep_node_color = data.colors.get(parent_dep_node_index); + let dep_dep_node = &data.previous.index_to_node(parent_dep_node_index); + + match dep_dep_node_color { + Some(DepNodeColor::Green(_)) => { + // This dependency has been marked as green before, we are + // still fine and can continue with checking the other + // dependencies. + debug!( + "try_mark_previous_green({:?}) --- found dependency {:?} to \ + be immediately green", + dep_node, dep_dep_node, + ); + return Some(()); + } + Some(DepNodeColor::Red) => { + // We found a dependency the value of which has changed + // compared to the previous compilation session. We cannot + // mark the DepNode as green and also don't need to bother + // with checking any of the other dependencies. + debug!( + "try_mark_previous_green({:?}) - END - dependency {:?} was immediately red", + dep_node, dep_dep_node, + ); + return None; + } + None => {} + } + + // We don't know the state of this dependency. If it isn't + // an eval_always node, let's try to mark it green recursively. + if !dep_dep_node.kind.is_eval_always() { + debug!( + "try_mark_previous_green({:?}) --- state of dependency {:?} ({}) \ + is unknown, trying to mark it green", + dep_node, dep_dep_node, dep_dep_node.hash, + ); + + let node_index = + self.try_mark_previous_green(tcx, data, parent_dep_node_index, dep_dep_node); + if node_index.is_some() { + debug!( + "try_mark_previous_green({:?}) --- managed to MARK dependency {:?} as green", + dep_node, dep_dep_node + ); + return Some(()); + } + } + + // We failed to mark it green, so we try to force the query. + debug!( + "try_mark_previous_green({:?}) --- trying to force dependency {:?}", + dep_node, dep_dep_node + ); + if !tcx.try_force_from_dep_node(dep_dep_node) { + // The DepNode could not be forced. + debug!( + "try_mark_previous_green({:?}) - END - dependency {:?} could not be forced", + dep_node, dep_dep_node + ); + return None; + } + + let dep_dep_node_color = data.colors.get(parent_dep_node_index); + + match dep_dep_node_color { + Some(DepNodeColor::Green(_)) => { + debug!( + "try_mark_previous_green({:?}) --- managed to FORCE dependency {:?} to green", + dep_node, dep_dep_node + ); + return Some(()); + } + Some(DepNodeColor::Red) => { + debug!( + "try_mark_previous_green({:?}) - END - dependency {:?} was red after forcing", + dep_node, dep_dep_node + ); + return None; + } + None => {} + } + + if !tcx.dep_context().sess().has_errors_or_delayed_span_bugs() { + panic!("try_mark_previous_green() - Forcing the DepNode should have set its color") + } + + // If the query we just forced has resulted in + // some kind of compilation error, we cannot rely on + // the dep-node color having been properly updated. + // This means that the query system has reached an + // invalid state. We let the compiler continue (by + // returning `None`) so it can emit error messages + // and wind down, but rely on the fact that this + // invalid state will not be persisted to the + // incremental compilation cache because of + // compilation errors being present. + debug!( + "try_mark_previous_green({:?}) - END - dependency {:?} resulted in compilation error", + dep_node, dep_dep_node + ); + return None; + } + /// Try to mark a dep-node which existed in the previous compilation session as green. fn try_mark_previous_green>( &self, @@ -512,123 +648,7 @@ impl DepGraph { let prev_deps = data.previous.edge_targets_from(prev_dep_node_index); for &dep_dep_node_index in prev_deps { - let dep_dep_node_color = data.colors.get(dep_dep_node_index); - - match dep_dep_node_color { - Some(DepNodeColor::Green(_)) => { - // This dependency has been marked as green before, we are - // still fine and can continue with checking the other - // dependencies. - debug!( - "try_mark_previous_green({:?}) --- found dependency {:?} to \ - be immediately green", - dep_node, - data.previous.index_to_node(dep_dep_node_index) - ); - } - Some(DepNodeColor::Red) => { - // We found a dependency the value of which has changed - // compared to the previous compilation session. We cannot - // mark the DepNode as green and also don't need to bother - // with checking any of the other dependencies. - debug!( - "try_mark_previous_green({:?}) - END - dependency {:?} was \ - immediately red", - dep_node, - data.previous.index_to_node(dep_dep_node_index) - ); - return None; - } - None => { - let dep_dep_node = &data.previous.index_to_node(dep_dep_node_index); - - // We don't know the state of this dependency. If it isn't - // an eval_always node, let's try to mark it green recursively. - if !dep_dep_node.kind.is_eval_always() { - debug!( - "try_mark_previous_green({:?}) --- state of dependency {:?} ({}) \ - is unknown, trying to mark it green", - dep_node, dep_dep_node, dep_dep_node.hash, - ); - - let node_index = self.try_mark_previous_green( - tcx, - data, - dep_dep_node_index, - dep_dep_node, - ); - if node_index.is_some() { - debug!( - "try_mark_previous_green({:?}) --- managed to MARK \ - dependency {:?} as green", - dep_node, dep_dep_node - ); - continue; - } - } - - // We failed to mark it green, so we try to force the query. - debug!( - "try_mark_previous_green({:?}) --- trying to force \ - dependency {:?}", - dep_node, dep_dep_node - ); - if tcx.try_force_from_dep_node(dep_dep_node) { - let dep_dep_node_color = data.colors.get(dep_dep_node_index); - - match dep_dep_node_color { - Some(DepNodeColor::Green(_)) => { - debug!( - "try_mark_previous_green({:?}) --- managed to \ - FORCE dependency {:?} to green", - dep_node, dep_dep_node - ); - } - Some(DepNodeColor::Red) => { - debug!( - "try_mark_previous_green({:?}) - END - \ - dependency {:?} was red after forcing", - dep_node, dep_dep_node - ); - return None; - } - None => { - if !tcx.dep_context().sess().has_errors_or_delayed_span_bugs() { - panic!( - "try_mark_previous_green() - Forcing the DepNode \ - should have set its color" - ) - } else { - // If the query we just forced has resulted in - // some kind of compilation error, we cannot rely on - // the dep-node color having been properly updated. - // This means that the query system has reached an - // invalid state. We let the compiler continue (by - // returning `None`) so it can emit error messages - // and wind down, but rely on the fact that this - // invalid state will not be persisted to the - // incremental compilation cache because of - // compilation errors being present. - debug!( - "try_mark_previous_green({:?}) - END - \ - dependency {:?} resulted in compilation error", - dep_node, dep_dep_node - ); - return None; - } - } - } - } else { - // The DepNode could not be forced. - debug!( - "try_mark_previous_green({:?}) - END - dependency {:?} \ - could not be forced", - dep_node, dep_dep_node - ); - return None; - } - } - } + self.try_mark_parent_green(tcx, data, dep_dep_node_index, dep_node)? } // If we got here without hitting a `return` that means that all @@ -797,7 +817,7 @@ impl DepGraph { } } - fn next_virtual_depnode_index(&self) -> DepNodeIndex { + pub(crate) fn next_virtual_depnode_index(&self) -> DepNodeIndex { let index = self.virtual_dep_node_index.fetch_add(1, Relaxed); DepNodeIndex::from_u32(index) } @@ -857,7 +877,7 @@ rustc_index::newtype_index! { /// For this reason, we avoid storing `DepNode`s more than once as map /// keys. The `new_node_to_index` map only contains nodes not in the previous /// graph, and we map nodes in the previous graph to indices via a two-step -/// mapping. `PreviousDepGraph` maps from `DepNode` to `SerializedDepNodeIndex`, +/// mapping. `SerializedDepGraph` maps from `DepNode` to `SerializedDepNodeIndex`, /// and the `prev_index_to_index` vector (which is more compact and faster than /// using a map) maps from `SerializedDepNodeIndex` to `DepNodeIndex`. /// @@ -982,7 +1002,7 @@ impl CurrentDepGraph { fn intern_node( &self, profiler: &SelfProfilerRef, - prev_graph: &PreviousDepGraph, + prev_graph: &SerializedDepGraph, key: DepNode, edges: EdgesVec, fingerprint: Option, @@ -1080,7 +1100,7 @@ impl CurrentDepGraph { fn promote_node_and_deps_to_current( &self, profiler: &SelfProfilerRef, - prev_graph: &PreviousDepGraph, + prev_graph: &SerializedDepGraph, prev_index: SerializedDepNodeIndex, ) -> DepNodeIndex { self.debug_assert_not_in_new_nodes(prev_graph, prev_index); @@ -1112,7 +1132,7 @@ impl CurrentDepGraph { #[inline] fn debug_assert_not_in_new_nodes( &self, - prev_graph: &PreviousDepGraph, + prev_graph: &SerializedDepGraph, prev_index: SerializedDepNodeIndex, ) { let node = &prev_graph.index_to_node(prev_index); diff --git a/compiler/rustc_query_system/src/dep_graph/mod.rs b/compiler/rustc_query_system/src/dep_graph/mod.rs index 1b6ecf3e63..15e2633c4f 100644 --- a/compiler/rustc_query_system/src/dep_graph/mod.rs +++ b/compiler/rustc_query_system/src/dep_graph/mod.rs @@ -1,13 +1,11 @@ pub mod debug; mod dep_node; mod graph; -mod prev; mod query; mod serialized; pub use dep_node::{DepNode, DepNodeParams, WorkProductId}; pub use graph::{hash_result, DepGraph, DepNodeColor, DepNodeIndex, TaskDeps, WorkProduct}; -pub use prev::PreviousDepGraph; pub use query::DepGraphQuery; pub use serialized::{SerializedDepGraph, SerializedDepNodeIndex}; diff --git a/compiler/rustc_query_system/src/dep_graph/prev.rs b/compiler/rustc_query_system/src/dep_graph/prev.rs deleted file mode 100644 index 6303bbf53b..0000000000 --- a/compiler/rustc_query_system/src/dep_graph/prev.rs +++ /dev/null @@ -1,56 +0,0 @@ -use super::serialized::{SerializedDepGraph, SerializedDepNodeIndex}; -use super::{DepKind, DepNode}; -use rustc_data_structures::fingerprint::Fingerprint; -use rustc_data_structures::fx::FxHashMap; - -#[derive(Debug)] -pub struct PreviousDepGraph { - data: SerializedDepGraph, - index: FxHashMap, SerializedDepNodeIndex>, -} - -impl Default for PreviousDepGraph { - fn default() -> Self { - PreviousDepGraph { data: Default::default(), index: Default::default() } - } -} - -impl PreviousDepGraph { - pub fn new(data: SerializedDepGraph) -> PreviousDepGraph { - let index: FxHashMap<_, _> = - data.nodes.iter_enumerated().map(|(idx, &dep_node)| (dep_node, idx)).collect(); - PreviousDepGraph { data, index } - } - - #[inline] - pub fn edge_targets_from( - &self, - dep_node_index: SerializedDepNodeIndex, - ) -> &[SerializedDepNodeIndex] { - self.data.edge_targets_from(dep_node_index) - } - - #[inline] - pub fn index_to_node(&self, dep_node_index: SerializedDepNodeIndex) -> DepNode { - self.data.nodes[dep_node_index] - } - - #[inline] - pub fn node_to_index_opt(&self, dep_node: &DepNode) -> Option { - self.index.get(dep_node).cloned() - } - - #[inline] - pub fn fingerprint_of(&self, dep_node: &DepNode) -> Option { - self.index.get(dep_node).map(|&node_index| self.data.fingerprints[node_index]) - } - - #[inline] - pub fn fingerprint_by_index(&self, dep_node_index: SerializedDepNodeIndex) -> Fingerprint { - self.data.fingerprints[dep_node_index] - } - - pub fn node_count(&self) -> usize { - self.index.len() - } -} diff --git a/compiler/rustc_query_system/src/dep_graph/serialized.rs b/compiler/rustc_query_system/src/dep_graph/serialized.rs index 6f3d1fb719..73c00fc49b 100644 --- a/compiler/rustc_query_system/src/dep_graph/serialized.rs +++ b/compiler/rustc_query_system/src/dep_graph/serialized.rs @@ -37,17 +37,19 @@ rustc_index::newtype_index! { #[derive(Debug)] pub struct SerializedDepGraph { /// The set of all DepNodes in the graph - pub nodes: IndexVec>, + nodes: IndexVec>, /// The set of all Fingerprints in the graph. Each Fingerprint corresponds to /// the DepNode at the same index in the nodes vector. - pub fingerprints: IndexVec, + fingerprints: IndexVec, /// For each DepNode, stores the list of edges originating from that /// DepNode. Encoded as a [start, end) pair indexing into edge_list_data, /// which holds the actual DepNodeIndices of the target nodes. - pub edge_list_indices: IndexVec, + edge_list_indices: IndexVec, /// A flattened list of all edge targets in the graph. Edge sources are /// implicit in edge_list_indices. - pub edge_list_data: Vec, + edge_list_data: Vec, + /// Reciprocal map to `nodes`. + index: FxHashMap, SerializedDepNodeIndex>, } impl Default for SerializedDepGraph { @@ -57,6 +59,7 @@ impl Default for SerializedDepGraph { fingerprints: Default::default(), edge_list_indices: Default::default(), edge_list_data: Default::default(), + index: Default::default(), } } } @@ -67,6 +70,30 @@ impl SerializedDepGraph { let targets = self.edge_list_indices[source]; &self.edge_list_data[targets.0 as usize..targets.1 as usize] } + + #[inline] + pub fn index_to_node(&self, dep_node_index: SerializedDepNodeIndex) -> DepNode { + self.nodes[dep_node_index] + } + + #[inline] + pub fn node_to_index_opt(&self, dep_node: &DepNode) -> Option { + self.index.get(dep_node).cloned() + } + + #[inline] + pub fn fingerprint_of(&self, dep_node: &DepNode) -> Option { + self.index.get(dep_node).map(|&node_index| self.fingerprints[node_index]) + } + + #[inline] + pub fn fingerprint_by_index(&self, dep_node_index: SerializedDepNodeIndex) -> Fingerprint { + self.fingerprints[dep_node_index] + } + + pub fn node_count(&self) -> usize { + self.index.len() + } } impl<'a, K: DepKind + Decodable>> Decodable> @@ -95,21 +122,21 @@ impl<'a, K: DepKind + Decodable>> Decodable = d.read_struct_field("node", 0, Decodable::decode)?; + d.read_struct(|d| { + let dep_node: DepNode = d.read_struct_field("node", Decodable::decode)?; let _i: SerializedDepNodeIndex = nodes.push(dep_node); debug_assert_eq!(_i.index(), _index); let fingerprint: Fingerprint = - d.read_struct_field("fingerprint", 1, Decodable::decode)?; + d.read_struct_field("fingerprint", Decodable::decode)?; let _i: SerializedDepNodeIndex = fingerprints.push(fingerprint); debug_assert_eq!(_i.index(), _index); - d.read_struct_field("edges", 2, |d| { + d.read_struct_field("edges", |d| { d.read_seq(|d, len| { let start = edge_list_data.len().try_into().unwrap(); - for e in 0..len { - let edge = d.read_seq_elt(e, Decodable::decode)?; + for _ in 0..len { + let edge = d.read_seq_elt(Decodable::decode)?; edge_list_data.push(edge); } let end = edge_list_data.len().try_into().unwrap(); @@ -121,7 +148,10 @@ impl<'a, K: DepKind + Decodable>> Decodable = + nodes.iter_enumerated().map(|(idx, &dep_node)| (dep_node, idx)).collect(); + + Ok(SerializedDepGraph { nodes, fingerprints, edge_list_indices, edge_list_data, index }) } } diff --git a/compiler/rustc_query_system/src/lib.rs b/compiler/rustc_query_system/src/lib.rs index be72baefb9..0d4fb34265 100644 --- a/compiler/rustc_query_system/src/lib.rs +++ b/compiler/rustc_query_system/src/lib.rs @@ -1,11 +1,8 @@ #![feature(bool_to_option)] -#![feature(const_panic)] #![feature(core_intrinsics)] -#![feature(drain_filter)] #![feature(hash_raw_entry)] #![feature(iter_zip)] #![feature(min_specialization)] -#![feature(stmt_expr_attributes)] #[macro_use] extern crate tracing; diff --git a/compiler/rustc_query_system/src/query/caches.rs b/compiler/rustc_query_system/src/query/caches.rs index 2e4c8d0565..011c2ceebb 100644 --- a/compiler/rustc_query_system/src/query/caches.rs +++ b/compiler/rustc_query_system/src/query/caches.rs @@ -14,7 +14,7 @@ pub trait CacheSelector { type Cache; } -pub trait QueryStorage: Default { +pub trait QueryStorage { type Value: Debug; type Stored: Clone; @@ -23,7 +23,7 @@ pub trait QueryStorage: Default { fn store_nocache(&self, value: Self::Value) -> Self::Stored; } -pub trait QueryCache: QueryStorage { +pub trait QueryCache: QueryStorage + Sized { type Key: Hash + Eq + Clone + Debug; type Sharded: Default; diff --git a/compiler/rustc_query_system/src/query/config.rs b/compiler/rustc_query_system/src/query/config.rs index 4e2515c3ac..f2a6b6df4b 100644 --- a/compiler/rustc_query_system/src/query/config.rs +++ b/compiler/rustc_query_system/src/query/config.rs @@ -52,10 +52,6 @@ impl QueryVtable { (self.hash_result)(hcx, value) } - pub(crate) fn handle_cycle_error(&self, tcx: CTX, diag: DiagnosticBuilder<'_>) -> V { - (self.handle_cycle_error)(tcx, diag) - } - pub(crate) fn cache_on_disk(&self, tcx: CTX, key: &K, value: Option<&V>) -> bool { (self.cache_on_disk)(tcx, key, value) } diff --git a/compiler/rustc_query_system/src/query/job.rs b/compiler/rustc_query_system/src/query/job.rs index 21f580db04..a967670280 100644 --- a/compiler/rustc_query_system/src/query/job.rs +++ b/compiler/rustc_query_system/src/query/job.rs @@ -9,7 +9,6 @@ use rustc_span::Span; use std::convert::TryFrom; use std::hash::Hash; -use std::marker::PhantomData; use std::num::NonZeroU32; #[cfg(parallel_compiler)] @@ -100,8 +99,6 @@ pub struct QueryJob { /// The latch that is used to wait on this job. #[cfg(parallel_compiler)] latch: Option>, - - dummy: PhantomData>, } impl QueryJob @@ -116,23 +113,17 @@ where parent, #[cfg(parallel_compiler)] latch: None, - dummy: PhantomData, } } #[cfg(parallel_compiler)] - pub(super) fn latch(&mut self, _id: QueryJobId) -> QueryLatch { + pub(super) fn latch(&mut self) -> QueryLatch { if self.latch.is_none() { self.latch = Some(QueryLatch::new()); } self.latch.as_ref().unwrap().clone() } - #[cfg(not(parallel_compiler))] - pub(super) fn latch(&mut self, id: QueryJobId) -> QueryLatch { - QueryLatch { id } - } - /// Signals to waiters that the query is complete. /// /// This does nothing for single threaded rustc, @@ -148,13 +139,7 @@ where } #[cfg(not(parallel_compiler))] -#[derive(Clone)] -pub(super) struct QueryLatch { - id: QueryJobId, -} - -#[cfg(not(parallel_compiler))] -impl QueryLatch +impl QueryJobId where D: Copy + Clone + Eq + Hash, { @@ -172,7 +157,7 @@ where let info = query_map.get(&job).unwrap(); cycle.push(info.info.clone()); - if job == self.id { + if job == *self { cycle.reverse(); // This is the end of the cycle diff --git a/compiler/rustc_query_system/src/query/mod.rs b/compiler/rustc_query_system/src/query/mod.rs index aef8a13cce..927e8117f0 100644 --- a/compiler/rustc_query_system/src/query/mod.rs +++ b/compiler/rustc_query_system/src/query/mod.rs @@ -19,7 +19,6 @@ use crate::dep_graph::{DepNode, DepNodeIndex, HasDepContext, SerializedDepNodeIn use rustc_data_structures::sync::Lock; use rustc_data_structures::thin_vec::ThinVec; use rustc_errors::Diagnostic; -use rustc_span::def_id::DefId; use rustc_span::Span; /// Description of a frame in the query stack. @@ -64,9 +63,6 @@ impl QueryStackFrame { } pub trait QueryContext: HasDepContext { - /// Get string representation from DefPath. - fn def_path_str(&self, def_id: DefId) -> String; - /// Get the query information from the TLS context. fn current_query_job(&self) -> Option>; diff --git a/compiler/rustc_query_system/src/query/plumbing.rs b/compiler/rustc_query_system/src/query/plumbing.rs index 06a364691d..c1f9fa39e9 100644 --- a/compiler/rustc_query_system/src/query/plumbing.rs +++ b/compiler/rustc_query_system/src/query/plumbing.rs @@ -2,7 +2,7 @@ //! generate the actual methods on tcx which find and execute the provider, //! manage the caches, and so forth. -use crate::dep_graph::{DepContext, DepKind, DepNode}; +use crate::dep_graph::{DepContext, DepKind, DepNode, DepNodeParams}; use crate::dep_graph::{DepNodeIndex, SerializedDepNodeIndex}; use crate::query::caches::QueryCache; use crate::query::config::{QueryDescription, QueryVtable, QueryVtableExt}; @@ -11,15 +11,15 @@ use crate::query::job::{ }; use crate::query::{QueryContext, QueryMap, QueryStackFrame}; -#[cfg(not(parallel_compiler))] -use rustc_data_structures::cold_path; use rustc_data_structures::fingerprint::Fingerprint; use rustc_data_structures::fx::{FxHashMap, FxHasher}; use rustc_data_structures::sharded::{get_shard_index_by_hash, Sharded}; use rustc_data_structures::sync::{Lock, LockGuard}; use rustc_data_structures::thin_vec::ThinVec; +#[cfg(not(parallel_compiler))] +use rustc_errors::DiagnosticBuilder; use rustc_errors::{Diagnostic, FatalError}; -use rustc_span::Span; +use rustc_span::{Span, DUMMY_SP}; use std::collections::hash_map::Entry; use std::fmt::Debug; use std::hash::{Hash, Hasher}; @@ -36,7 +36,7 @@ pub struct QueryCacheStore { pub cache_hits: AtomicUsize, } -impl Default for QueryCacheStore { +impl Default for QueryCacheStore { fn default() -> Self { Self { cache: C::default(), @@ -125,18 +125,15 @@ where // We use try_lock_shards here since we are called from the // deadlock handler, and this shouldn't be locked. let shards = self.shards.try_lock_shards()?; - let shards = shards.iter().enumerate(); - jobs.extend(shards.flat_map(|(shard_id, shard)| { - shard.active.iter().filter_map(move |(k, v)| { + for (shard_id, shard) in shards.iter().enumerate() { + for (k, v) in shard.active.iter() { if let QueryResult::Started(ref job) = *v { let id = QueryJobId::new(job.id, shard_id, kind); let info = QueryInfo { span: job.span, query: make_query(tcx, k.clone()) }; - Some((id, QueryJobInfo { info, job: job.clone() })) - } else { - None + jobs.insert(id, QueryJobInfo { info, job: job.clone() }); } - }) - })); + } + } Some(()) } @@ -161,6 +158,31 @@ where id: QueryJobId, } +#[cold] +#[inline(never)] +#[cfg(not(parallel_compiler))] +fn mk_cycle( + tcx: CTX, + root: QueryJobId, + span: Span, + handle_cycle_error: fn(CTX, DiagnosticBuilder<'_>) -> V, + cache: &dyn crate::query::QueryStorage, +) -> R +where + CTX: QueryContext, + V: std::fmt::Debug, + R: Clone, +{ + let error: CycleError = root.find_cycle_in_stack( + tcx.try_collect_active_jobs().unwrap(), + &tcx.current_query_job(), + span, + ); + let error = report_cycle(tcx.dep_context().sess(), error); + let value = handle_cycle_error(tcx, error); + cache.store_nocache(value) +} + impl<'tcx, D, C> JobOwner<'tcx, D, C> where D: Copy + Clone + Eq + Hash, @@ -180,7 +202,7 @@ where state: &'b QueryState, cache: &'b QueryCacheStore, span: Span, - key: &C::Key, + key: C::Key, lookup: QueryLookup, query: &QueryVtable, ) -> TryGetJob<'b, CTX::DepKind, C> @@ -191,94 +213,86 @@ where let mut state_lock = state.shards.get_shard_by_index(shard).lock(); let lock = &mut *state_lock; - let (latch, mut _query_blocked_prof_timer) = match lock.active.entry((*key).clone()) { - Entry::Occupied(mut entry) => { - match entry.get_mut() { - QueryResult::Started(job) => { - // For parallel queries, we'll block and wait until the query running - // in another thread has completed. Record how long we wait in the - // self-profiler. - let _query_blocked_prof_timer = if cfg!(parallel_compiler) { - Some(tcx.dep_context().profiler().query_blocked()) - } else { - None - }; - - // Create the id of the job we're waiting for - let id = QueryJobId::new(job.id, shard, query.dep_kind); - - (job.latch(id), _query_blocked_prof_timer) - } - QueryResult::Poisoned => FatalError.raise(), - } - } + match lock.active.entry(key) { Entry::Vacant(entry) => { - // No job entry for this query. Return a new one to be started later. - // Generate an id unique within this shard. let id = lock.jobs.checked_add(1).unwrap(); lock.jobs = id; let id = QueryShardJobId(NonZeroU32::new(id).unwrap()); - let global_id = QueryJobId::new(id, shard, query.dep_kind); - let job = tcx.current_query_job(); let job = QueryJob::new(id, span, job); + let key = entry.key().clone(); entry.insert(QueryResult::Started(job)); - let owner = JobOwner { state, cache, id: global_id, key: (*key).clone() }; + let global_id = QueryJobId::new(id, shard, query.dep_kind); + let owner = JobOwner { state, cache, id: global_id, key }; return TryGetJob::NotYetStarted(owner); } - }; - mem::drop(state_lock); - - // If we are single-threaded we know that we have cycle error, - // so we just return the error. - #[cfg(not(parallel_compiler))] - return TryGetJob::Cycle(cold_path(|| { - let error: CycleError = latch.find_cycle_in_stack( - tcx.try_collect_active_jobs().unwrap(), - &tcx.current_query_job(), - span, - ); - let error = report_cycle(tcx.dep_context().sess(), error); - let value = query.handle_cycle_error(tcx, error); - cache.cache.store_nocache(value) - })); - - // With parallel queries we might just have to wait on some other - // thread. - #[cfg(parallel_compiler)] - { - let result = latch.wait_on(tcx.current_query_job(), span); - - if let Err(cycle) = result { - let cycle = report_cycle(tcx.dep_context().sess(), cycle); - let value = query.handle_cycle_error(tcx, cycle); - let value = cache.cache.store_nocache(value); - return TryGetJob::Cycle(value); - } + Entry::Occupied(mut entry) => { + match entry.get_mut() { + #[cfg(not(parallel_compiler))] + QueryResult::Started(job) => { + let id = QueryJobId::new(job.id, shard, query.dep_kind); - let cached = cache - .cache - .lookup(cache, &key, |value, index| { - if unlikely!(tcx.dep_context().profiler().enabled()) { - tcx.dep_context().profiler().query_cache_hit(index.into()); + drop(state_lock); + + // If we are single-threaded we know that we have cycle error, + // so we just return the error. + return TryGetJob::Cycle(mk_cycle( + tcx, + id, + span, + query.handle_cycle_error, + &cache.cache, + )); } - #[cfg(debug_assertions)] - { - cache.cache_hits.fetch_add(1, Ordering::Relaxed); + #[cfg(parallel_compiler)] + QueryResult::Started(job) => { + // For parallel queries, we'll block and wait until the query running + // in another thread has completed. Record how long we wait in the + // self-profiler. + let query_blocked_prof_timer = tcx.dep_context().profiler().query_blocked(); + + // Get the latch out + let latch = job.latch(); + let key = entry.key().clone(); + + drop(state_lock); + + // With parallel queries we might just have to wait on some other + // thread. + let result = latch.wait_on(tcx.current_query_job(), span); + + if let Err(cycle) = result { + let cycle = report_cycle(tcx.dep_context().sess(), cycle); + let value = (query.handle_cycle_error)(tcx, cycle); + let value = cache.cache.store_nocache(value); + return TryGetJob::Cycle(value); + } + + let cached = cache + .cache + .lookup(cache, &key, |value, index| { + if unlikely!(tcx.dep_context().profiler().enabled()) { + tcx.dep_context().profiler().query_cache_hit(index.into()); + } + #[cfg(debug_assertions)] + { + cache.cache_hits.fetch_add(1, Ordering::Relaxed); + } + (value.clone(), index) + }) + .unwrap_or_else(|_| panic!("value must be in cache after waiting")); + + query_blocked_prof_timer.finish_with_query_invocation_id(cached.1.into()); + + return TryGetJob::JobCompleted(cached); } - (value.clone(), index) - }) - .unwrap_or_else(|_| panic!("value must be in cache after waiting")); - - if let Some(prof_timer) = _query_blocked_prof_timer.take() { - prof_timer.finish_with_query_invocation_id(cached.1.into()); + QueryResult::Poisoned => FatalError.raise(), + } } - - return TryGetJob::JobCompleted(cached); } } @@ -417,11 +431,17 @@ fn try_execute_query( ) -> C::Stored where C: QueryCache, - C::Key: crate::dep_graph::DepNodeParams, + C::Key: DepNodeParams, CTX: QueryContext, { let job = match JobOwner::<'_, CTX::DepKind, C>::try_start( - tcx, state, cache, span, &key, lookup, query, + tcx, + state, + cache, + span, + key.clone(), + lookup, + query, ) { TryGetJob::NotYetStarted(job) => job, TryGetJob::Cycle(result) => return result, @@ -432,11 +452,15 @@ where } }; - // Fast path for when incr. comp. is off. `to_dep_node` is - // expensive for some `DepKind`s. - if !tcx.dep_context().dep_graph().is_fully_enabled() { - let null_dep_node = DepNode::new_no_params(DepKind::NULL); - return force_query_with_job(tcx, key, job, null_dep_node, query).0; + let dep_graph = tcx.dep_context().dep_graph(); + + // Fast path for when incr. comp. is off. + if !dep_graph.is_fully_enabled() { + let prof_timer = tcx.dep_context().profiler().query_provider(); + let result = tcx.start_query(job.id, None, || query.compute(tcx, key)); + let dep_node_index = dep_graph.next_virtual_depnode_index(); + prof_timer.finish_with_query_invocation_id(dep_node_index.into()); + return job.complete(result, dep_node_index); } if query.anon { @@ -444,17 +468,14 @@ where let ((result, dep_node_index), diagnostics) = with_diagnostics(|diagnostics| { tcx.start_query(job.id, diagnostics, || { - tcx.dep_context().dep_graph().with_anon_task( - *tcx.dep_context(), - query.dep_kind, - || query.compute(tcx, key), - ) + dep_graph + .with_anon_task(*tcx.dep_context(), query.dep_kind, || query.compute(tcx, key)) }) }); prof_timer.finish_with_query_invocation_id(dep_node_index.into()); - tcx.dep_context().dep_graph().read_index(dep_node_index); + dep_graph.read_index(dep_node_index); if unlikely!(!diagnostics.is_empty()) { tcx.store_diagnostics_for_anon_node(dep_node_index, diagnostics); @@ -470,7 +491,7 @@ where // promoted to the current session during // `try_mark_green()`, so we can ignore them here. let loaded = tcx.start_query(job.id, None, || { - let marked = tcx.dep_context().dep_graph().try_mark_green_and_read(tcx, &dep_node); + let marked = dep_graph.try_mark_green_and_read(tcx, &dep_node); marked.map(|(prev_dep_node_index, dep_node_index)| { ( load_from_disk_and_cache_in_memory( @@ -491,7 +512,7 @@ where } let (result, dep_node_index) = force_query_with_job(tcx, key, job, dep_node, query); - tcx.dep_context().dep_graph().read_index(dep_node_index); + dep_graph.read_index(dep_node_index); result } @@ -594,7 +615,7 @@ fn incremental_verify_ich( tcx.sess().struct_err(&format!("internal compiler error: encountered incremental compilation error with {:?}", dep_node)) .help(&format!("This is a known issue with the compiler. Run {} to allow your project to compile", run_cmd)) .note(&format!("Please follow the instructions below to create a bug report with the provided information")) - .note(&format!("See for more information.")) + .note(&format!("See for more information")) .emit(); panic!("Found unstable fingerprints for {:?}: {:?}", dep_node, result); } @@ -673,7 +694,7 @@ fn get_query_impl( where CTX: QueryContext, C: QueryCache, - C::Key: crate::dep_graph::DepNodeParams, + C::Key: DepNodeParams, { try_execute_query(tcx, state, cache, span, key, lookup, query) } @@ -723,15 +744,28 @@ fn force_query_impl( tcx: CTX, state: &QueryState, cache: &QueryCacheStore, - key: C::Key, - span: Span, dep_node: DepNode, query: &QueryVtable, -) where +) -> bool +where C: QueryCache, - C::Key: crate::dep_graph::DepNodeParams, + C::Key: DepNodeParams, CTX: QueryContext, { + debug_assert!(!query.anon); + + if !>::can_reconstruct_query_key() { + return false; + } + + let key = if let Some(key) = + >::recover(*tcx.dep_context(), &dep_node) + { + key + } else { + return false; + }; + // We may be concurrently trying both execute and force a query. // Ensure that only one of them runs the query. let cached = cache.cache.lookup(cache, &key, |_, index| { @@ -745,19 +779,28 @@ fn force_query_impl( }); let lookup = match cached { - Ok(()) => return, + Ok(()) => return true, Err(lookup) => lookup, }; let job = match JobOwner::<'_, CTX::DepKind, C>::try_start( - tcx, state, cache, span, &key, lookup, query, + tcx, + state, + cache, + DUMMY_SP, + key.clone(), + lookup, + query, ) { TryGetJob::NotYetStarted(job) => job, - TryGetJob::Cycle(_) => return, + TryGetJob::Cycle(_) => return true, #[cfg(parallel_compiler)] - TryGetJob::JobCompleted(_) => return, + TryGetJob::JobCompleted(_) => return true, }; + force_query_with_job(tcx, key, job, dep_node, query); + + true } pub enum QueryMode { @@ -774,7 +817,7 @@ pub fn get_query( ) -> Option where Q: QueryDescription, - Q::Key: crate::dep_graph::DepNodeParams, + Q::Key: DepNodeParams, CTX: QueryContext, { let query = &Q::VTABLE; @@ -790,11 +833,15 @@ where Some(value) } -pub fn force_query(tcx: CTX, key: Q::Key, span: Span, dep_node: DepNode) +pub fn force_query(tcx: CTX, dep_node: &DepNode) -> bool where Q: QueryDescription, - Q::Key: crate::dep_graph::DepNodeParams, + Q::Key: DepNodeParams, CTX: QueryContext, { - force_query_impl(tcx, Q::query_state(tcx), Q::query_cache(tcx), key, span, dep_node, &Q::VTABLE) + if Q::ANON { + return false; + } + + force_query_impl(tcx, Q::query_state(tcx), Q::query_cache(tcx), *dep_node, &Q::VTABLE) } diff --git a/compiler/rustc_resolve/src/build_reduced_graph.rs b/compiler/rustc_resolve/src/build_reduced_graph.rs index b5c95cfcb2..e10314a11f 100644 --- a/compiler/rustc_resolve/src/build_reduced_graph.rs +++ b/compiler/rustc_resolve/src/build_reduced_graph.rs @@ -995,7 +995,20 @@ impl<'a, 'b> BuildReducedGraphVisitor<'a, 'b> { // Record some extra data for better diagnostics. let cstore = self.r.cstore(); match res { - Res::Def(DefKind::Struct | DefKind::Union, def_id) => { + Res::Def(DefKind::Struct, def_id) => { + let field_names = cstore.struct_field_names_untracked(def_id, self.r.session); + let ctor = cstore.ctor_def_id_and_kind_untracked(def_id); + if let Some((ctor_def_id, ctor_kind)) = ctor { + let ctor_res = Res::Def(DefKind::Ctor(CtorOf::Struct, ctor_kind), ctor_def_id); + let ctor_vis = cstore.visibility_untracked(ctor_def_id); + let field_visibilities = cstore.struct_field_visibilities_untracked(def_id); + self.r + .struct_constructors + .insert(def_id, (ctor_res, ctor_vis, field_visibilities)); + } + self.insert_field_names(def_id, field_names); + } + Res::Def(DefKind::Union, def_id) => { let field_names = cstore.struct_field_names_untracked(def_id, self.r.session); self.insert_field_names(def_id, field_names); } @@ -1007,12 +1020,6 @@ impl<'a, 'b> BuildReducedGraphVisitor<'a, 'b> { self.r.has_self.insert(def_id); } } - Res::Def(DefKind::Ctor(CtorOf::Struct, ..), def_id) => { - let parent = cstore.def_key(def_id).parent; - if let Some(struct_def_id) = parent.map(|index| DefId { index, ..def_id }) { - self.r.struct_constructors.insert(struct_def_id, (res, vis, vec![])); - } - } _ => {} } } diff --git a/compiler/rustc_resolve/src/diagnostics.rs b/compiler/rustc_resolve/src/diagnostics.rs index 6ea46f5c52..03d94f4389 100644 --- a/compiler/rustc_resolve/src/diagnostics.rs +++ b/compiler/rustc_resolve/src/diagnostics.rs @@ -425,27 +425,32 @@ impl<'a> Resolver<'a> { } err } - ResolutionError::BindingShadowsSomethingUnacceptable(what_binding, name, binding) => { - let res = binding.res(); - let shadows_what = res.descr(); + ResolutionError::BindingShadowsSomethingUnacceptable { + shadowing_binding_descr, + name, + participle, + article, + shadowed_binding_descr, + shadowed_binding_span, + } => { let mut err = struct_span_err!( self.session, span, E0530, "{}s cannot shadow {}s", - what_binding, - shadows_what + shadowing_binding_descr, + shadowed_binding_descr, ); err.span_label( span, - format!("cannot be named the same as {} {}", res.article(), shadows_what), + format!("cannot be named the same as {} {}", article, shadowed_binding_descr), ); - let participle = if binding.is_import() { "imported" } else { "defined" }; - let msg = format!("the {} `{}` is {} here", shadows_what, name, participle); - err.span_label(binding.span, msg); + let msg = + format!("the {} `{}` is {} here", shadowed_binding_descr, name, participle); + err.span_label(shadowed_binding_span, msg); err } - ResolutionError::ForwardDeclaredTyParam => { + ResolutionError::ForwardDeclaredGenericParam => { let mut err = struct_span_err!( self.session, span, diff --git a/compiler/rustc_resolve/src/late.rs b/compiler/rustc_resolve/src/late.rs index 92f21191de..a21d8197bd 100644 --- a/compiler/rustc_resolve/src/late.rs +++ b/compiler/rustc_resolve/src/late.rs @@ -1066,10 +1066,14 @@ impl<'a: 'ast, 'b, 'ast> LateResolutionVisitor<'a, 'b, 'ast> { self.future_proof_import(use_tree); } - ItemKind::ExternCrate(..) | ItemKind::MacroDef(..) | ItemKind::GlobalAsm(..) => { + ItemKind::ExternCrate(..) | ItemKind::MacroDef(..) => { // do nothing, these are just around to be encoded } + ItemKind::GlobalAsm(_) => { + visit::walk_item(self, item); + } + ItemKind::MacCall(_) => panic!("unexpanded macro in resolve!"), } } @@ -1609,10 +1613,10 @@ impl<'a: 'ast, 'b, 'ast> LateResolutionVisitor<'a, 'b, 'ast> { self.r.record_partial_res(pat.id, PartialRes::new(res)); self.r.record_pat_span(pat.id, pat.span); } - PatKind::TupleStruct(ref path, ref sub_patterns) => { + PatKind::TupleStruct(ref qself, ref path, ref sub_patterns) => { self.smart_resolve_path( pat.id, - None, + qself.as_ref(), path, PathSource::TupleStruct( pat.span, @@ -1623,8 +1627,8 @@ impl<'a: 'ast, 'b, 'ast> LateResolutionVisitor<'a, 'b, 'ast> { PatKind::Path(ref qself, ref path) => { self.smart_resolve_path(pat.id, qself.as_ref(), path, PathSource::Pat); } - PatKind::Struct(ref path, ..) => { - self.smart_resolve_path(pat.id, None, path, PathSource::Struct); + PatKind::Struct(ref qself, ref path, ..) => { + self.smart_resolve_path(pat.id, qself.as_ref(), path, PathSource::Struct); } PatKind::Or(ref ps) => { // Add a new set of bindings to the stack. `Or` here records that when a @@ -1759,13 +1763,33 @@ impl<'a: 'ast, 'b, 'ast> LateResolutionVisitor<'a, 'b, 'ast> { // to something unusable as a pattern (e.g., constructor function), // but we still conservatively report an error, see // issues/33118#issuecomment-233962221 for one reason why. + let binding = binding.expect("no binding for a ctor or static"); + self.report_error( + ident.span, + ResolutionError::BindingShadowsSomethingUnacceptable { + shadowing_binding_descr: pat_src.descr(), + name: ident.name, + participle: if binding.is_import() { "imported" } else { "defined" }, + article: binding.res().article(), + shadowed_binding_descr: binding.res().descr(), + shadowed_binding_span: binding.span, + }, + ); + None + } + Res::Def(DefKind::ConstParam, def_id) => { + // Same as for DefKind::Const above, but here, `binding` is `None`, so we + // have to construct the error differently self.report_error( ident.span, - ResolutionError::BindingShadowsSomethingUnacceptable( - pat_src.descr(), - ident.name, - binding.expect("no binding for a ctor or static"), - ), + ResolutionError::BindingShadowsSomethingUnacceptable { + shadowing_binding_descr: pat_src.descr(), + name: ident.name, + participle: "defined", + article: res.article(), + shadowed_binding_descr: res.descr(), + shadowed_binding_span: self.r.opt_span(def_id).expect("const parameter defined outside of local crate"), + } ); None } @@ -1935,7 +1959,7 @@ impl<'a: 'ast, 'b, 'ast> LateResolutionVisitor<'a, 'b, 'ast> { if ns == ValueNS { let item_name = path.last().unwrap().ident; let traits = self.traits_in_scope(item_name, ns); - self.r.trait_map.insert(id, traits); + self.r.trait_map.as_mut().unwrap().insert(id, traits); } if PrimTy::from_name(path[0].ident.name).is_some() { @@ -2264,7 +2288,7 @@ impl<'a: 'ast, 'b, 'ast> LateResolutionVisitor<'a, 'b, 'ast> { } ExprKind::Struct(ref se) => { - self.smart_resolve_path(expr.id, None, &se.path, PathSource::Struct); + self.smart_resolve_path(expr.id, se.qself.as_ref(), &se.path, PathSource::Struct); visit::walk_expr(self, expr); } @@ -2411,12 +2435,12 @@ impl<'a: 'ast, 'b, 'ast> LateResolutionVisitor<'a, 'b, 'ast> { // the field name so that we can do some nice error reporting // later on in typeck. let traits = self.traits_in_scope(ident, ValueNS); - self.r.trait_map.insert(expr.id, traits); + self.r.trait_map.as_mut().unwrap().insert(expr.id, traits); } ExprKind::MethodCall(ref segment, ..) => { debug!("(recording candidate traits for expr) recording traits for {}", expr.id); let traits = self.traits_in_scope(segment.ident, ValueNS); - self.r.trait_map.insert(expr.id, traits); + self.r.trait_map.as_mut().unwrap().insert(expr.id, traits); } _ => { // Nothing to do. diff --git a/compiler/rustc_resolve/src/late/diagnostics.rs b/compiler/rustc_resolve/src/late/diagnostics.rs index e33c374f56..03b578d4ad 100644 --- a/compiler/rustc_resolve/src/late/diagnostics.rs +++ b/compiler/rustc_resolve/src/late/diagnostics.rs @@ -6,10 +6,13 @@ use crate::{CrateLint, Module, ModuleKind, ModuleOrUniformRoot}; use crate::{PathResult, PathSource, Segment}; use rustc_ast::visit::FnKind; -use rustc_ast::{self as ast, Expr, ExprKind, Item, ItemKind, NodeId, Path, Ty, TyKind}; +use rustc_ast::{ + self as ast, Expr, ExprKind, GenericParam, GenericParamKind, Item, ItemKind, NodeId, Path, Ty, + TyKind, +}; use rustc_ast_pretty::pprust::path_segment_to_string; use rustc_data_structures::fx::FxHashSet; -use rustc_errors::{pluralize, struct_span_err, Applicability, DiagnosticBuilder}; +use rustc_errors::{pluralize, struct_span_err, Applicability, DiagnosticBuilder, SuggestionStyle}; use rustc_hir as hir; use rustc_hir::def::Namespace::{self, *}; use rustc_hir::def::{self, CtorKind, CtorOf, DefKind}; @@ -819,6 +822,19 @@ impl<'a: 'ast, 'ast> LateResolutionVisitor<'a, '_, 'ast> { _ => false, }; + let find_span = |source: &PathSource<'_>, err: &mut DiagnosticBuilder<'_>| { + match source { + PathSource::Expr(Some(Expr { span, kind: ExprKind::Call(_, _), .. })) + | PathSource::TupleStruct(span, _) => { + // We want the main underline to cover the suggested code as well for + // cleaner output. + err.set_span(*span); + *span + } + _ => span, + } + }; + let mut bad_struct_syntax_suggestion = |def_id: DefId| { let (followed_by_brace, closing_brace) = self.followed_by_brace(span); @@ -862,18 +878,7 @@ impl<'a: 'ast, 'ast> LateResolutionVisitor<'a, '_, 'ast> { } } PathSource::Expr(_) | PathSource::TupleStruct(..) | PathSource::Pat => { - let span = match &source { - PathSource::Expr(Some(Expr { - span, kind: ExprKind::Call(_, _), .. - })) - | PathSource::TupleStruct(span, _) => { - // We want the main underline to cover the suggested code as well for - // cleaner output. - err.set_span(*span); - *span - } - _ => span, - }; + let span = find_span(&source, err); if let Some(span) = self.def_span(def_id) { err.span_label(span, &format!("`{}` defined here", path_str)); } @@ -1047,6 +1052,23 @@ impl<'a: 'ast, 'ast> LateResolutionVisitor<'a, '_, 'ast> { ) if ns == ValueNS => { bad_struct_syntax_suggestion(def_id); } + (Res::Def(DefKind::Ctor(_, CtorKind::Const), def_id), _) if ns == ValueNS => { + match source { + PathSource::Expr(_) | PathSource::TupleStruct(..) | PathSource::Pat => { + let span = find_span(&source, err); + if let Some(span) = self.def_span(def_id) { + err.span_label(span, &format!("`{}` defined here", path_str)); + } + err.span_suggestion( + span, + &format!("use this syntax instead"), + format!("{path_str}"), + Applicability::MaybeIncorrect, + ); + } + _ => return false, + } + } (Res::Def(DefKind::Ctor(_, CtorKind::Fn), def_id), _) if ns == ValueNS => { if let Some(span) = self.def_span(def_id) { err.span_label(span, &format!("`{}` defined here", path_str)); @@ -1581,8 +1603,8 @@ impl<'a: 'ast, 'ast> LateResolutionVisitor<'a, '_, 'ast> { if !self.diagnostic_metadata.currently_processing_generics && !single_uppercase_char { return None; } - match (self.diagnostic_metadata.current_item, single_uppercase_char) { - (Some(Item { kind: ItemKind::Fn(..), ident, .. }), _) if ident.name == sym::main => { + match (self.diagnostic_metadata.current_item, single_uppercase_char, self.diagnostic_metadata.currently_processing_generics) { + (Some(Item { kind: ItemKind::Fn(..), ident, .. }), _, _) if ident.name == sym::main => { // Ignore `fn main()` as we don't want to suggest `fn main()` } ( @@ -1594,9 +1616,11 @@ impl<'a: 'ast, 'ast> LateResolutionVisitor<'a, '_, 'ast> { | kind @ ItemKind::Union(..), .. }), - true, + true, _ ) - | (Some(Item { kind, .. }), false) => { + // Without the 2nd `true`, we'd suggest `impl ` for `impl T` when a type `T` isn't found + | (Some(Item { kind: kind @ ItemKind::Impl(..), .. }), true, true) + | (Some(Item { kind, .. }), false, _) => { // Likely missing type parameter. if let Some(generics) = kind.generics() { if span.overlaps(generics.span) { @@ -1614,6 +1638,10 @@ impl<'a: 'ast, 'ast> LateResolutionVisitor<'a, '_, 'ast> { let (span, sugg) = if let [.., param] = &generics.params[..] { let span = if let [.., bound] = ¶m.bounds[..] { bound.span() + } else if let GenericParam { + kind: GenericParamKind::Const { ty, kw_span: _, default }, .. + } = param { + default.as_ref().map(|def| def.value.span).unwrap_or(ty.span) } else { param.ident.span }; @@ -1668,12 +1696,12 @@ impl<'a: 'ast, 'ast> LateResolutionVisitor<'a, '_, 'ast> { impl<'tcx> LifetimeContext<'_, 'tcx> { crate fn report_missing_lifetime_specifiers( &self, - span: Span, + spans: Vec, count: usize, ) -> DiagnosticBuilder<'tcx> { struct_span_err!( self.tcx.sess, - span, + spans, E0106, "missing lifetime specifier{}", pluralize!(count) @@ -1802,81 +1830,115 @@ impl<'tcx> LifetimeContext<'_, 'tcx> { crate fn add_missing_lifetime_specifiers_label( &self, err: &mut DiagnosticBuilder<'_>, - span: Span, - count: usize, + mut spans_with_counts: Vec<(Span, usize)>, lifetime_names: &FxHashSet, lifetime_spans: Vec, params: &[ElisionFailureInfo], ) { - let snippet = self.tcx.sess.source_map().span_to_snippet(span).ok(); - - err.span_label( - span, - &format!( - "expected {} lifetime parameter{}", - if count == 1 { "named".to_string() } else { count.to_string() }, - pluralize!(count) - ), - ); + let snippets: Vec> = spans_with_counts + .iter() + .map(|(span, _)| self.tcx.sess.source_map().span_to_snippet(*span).ok()) + .collect(); + + // Empty generics are marked with a span of "<", but since from now on + // that information is in the snippets it can be removed from the spans. + for ((span, _), snippet) in spans_with_counts.iter_mut().zip(&snippets) { + if snippet.as_deref() == Some("<") { + *span = span.shrink_to_hi(); + } + } - let suggest_existing = |err: &mut DiagnosticBuilder<'_>, - name: &str, - formatter: &dyn Fn(&str) -> String| { - if let Some(MissingLifetimeSpot::HigherRanked { span: for_span, span_type }) = - self.missing_named_lifetime_spots.iter().rev().next() - { - // When we have `struct S<'a>(&'a dyn Fn(&X) -> &X);` we want to not only suggest - // using `'a`, but also introduce the concept of HRLTs by suggesting - // `struct S<'a>(&'a dyn for<'b> Fn(&X) -> &'b X);`. (#72404) - let mut introduce_suggestion = vec![]; + for &(span, count) in &spans_with_counts { + err.span_label( + span, + format!( + "expected {} lifetime parameter{}", + if count == 1 { "named".to_string() } else { count.to_string() }, + pluralize!(count), + ), + ); + } - let a_to_z_repeat_n = |n| { - (b'a'..=b'z').map(move |c| { - let mut s = '\''.to_string(); - s.extend(std::iter::repeat(char::from(c)).take(n)); - s - }) - }; + let suggest_existing = + |err: &mut DiagnosticBuilder<'_>, + name: &str, + formatters: Vec String>>>| { + if let Some(MissingLifetimeSpot::HigherRanked { span: for_span, span_type }) = + self.missing_named_lifetime_spots.iter().rev().next() + { + // When we have `struct S<'a>(&'a dyn Fn(&X) -> &X);` we want to not only suggest + // using `'a`, but also introduce the concept of HRLTs by suggesting + // `struct S<'a>(&'a dyn for<'b> Fn(&X) -> &'b X);`. (#72404) + let mut introduce_suggestion = vec![]; + + let a_to_z_repeat_n = |n| { + (b'a'..=b'z').map(move |c| { + let mut s = '\''.to_string(); + s.extend(std::iter::repeat(char::from(c)).take(n)); + s + }) + }; - // If all single char lifetime names are present, we wrap around and double the chars. - let lt_name = (1..) - .flat_map(a_to_z_repeat_n) - .find(|lt| !lifetime_names.contains(&Symbol::intern(<))) - .unwrap(); - let msg = format!( - "consider making the {} lifetime-generic with a new `{}` lifetime", - span_type.descr(), - lt_name, - ); - err.note( - "for more information on higher-ranked polymorphism, visit \ + // If all single char lifetime names are present, we wrap around and double the chars. + let lt_name = (1..) + .flat_map(a_to_z_repeat_n) + .find(|lt| !lifetime_names.contains(&Symbol::intern(<))) + .unwrap(); + let msg = format!( + "consider making the {} lifetime-generic with a new `{}` lifetime", + span_type.descr(), + lt_name, + ); + err.note( + "for more information on higher-ranked polymorphism, visit \ https://doc.rust-lang.org/nomicon/hrtb.html", - ); - let for_sugg = span_type.suggestion(<_name); - for param in params { - if let Ok(snippet) = self.tcx.sess.source_map().span_to_snippet(param.span) { - if snippet.starts_with('&') && !snippet.starts_with("&'") { - introduce_suggestion - .push((param.span, format!("&{} {}", lt_name, &snippet[1..]))); - } else if let Some(stripped) = snippet.strip_prefix("&'_ ") { - introduce_suggestion - .push((param.span, format!("&{} {}", lt_name, stripped))); + ); + let for_sugg = span_type.suggestion(<_name); + for param in params { + if let Ok(snippet) = self.tcx.sess.source_map().span_to_snippet(param.span) + { + if snippet.starts_with('&') && !snippet.starts_with("&'") { + introduce_suggestion + .push((param.span, format!("&{} {}", lt_name, &snippet[1..]))); + } else if let Some(stripped) = snippet.strip_prefix("&'_ ") { + introduce_suggestion + .push((param.span, format!("&{} {}", lt_name, stripped))); + } + } + } + introduce_suggestion.push((*for_span, for_sugg)); + for ((span, _), formatter) in spans_with_counts.iter().zip(formatters.iter()) { + if let Some(formatter) = formatter { + introduce_suggestion.push((*span, formatter(<_name))); } } + err.multipart_suggestion_with_style( + &msg, + introduce_suggestion, + Applicability::MaybeIncorrect, + SuggestionStyle::ShowAlways, + ); } - introduce_suggestion.push((*for_span, for_sugg)); - introduce_suggestion.push((span, formatter(<_name))); - err.multipart_suggestion(&msg, introduce_suggestion, Applicability::MaybeIncorrect); - } - err.span_suggestion_verbose( - span, - &format!("consider using the `{}` lifetime", lifetime_names.iter().next().unwrap()), - formatter(name), - Applicability::MaybeIncorrect, - ); - }; - let suggest_new = |err: &mut DiagnosticBuilder<'_>, sugg: &str| { + let spans_suggs: Vec<_> = formatters + .into_iter() + .zip(spans_with_counts.iter()) + .filter_map(|(fmt, (span, _))| { + if let Some(formatter) = fmt { Some((formatter, span)) } else { None } + }) + .map(|(formatter, span)| (*span, formatter(name))) + .collect(); + err.multipart_suggestion_with_style( + &format!( + "consider using the `{}` lifetime", + lifetime_names.iter().next().unwrap() + ), + spans_suggs, + Applicability::MaybeIncorrect, + SuggestionStyle::ShowAlways, + ); + }; + let suggest_new = |err: &mut DiagnosticBuilder<'_>, suggs: Vec>| { for missing in self.missing_named_lifetime_spots.iter().rev() { let mut introduce_suggestion = vec![]; let msg; @@ -1921,38 +1983,52 @@ impl<'tcx> LifetimeContext<'_, 'tcx> { (*span, span_type.suggestion("'a")) } MissingLifetimeSpot::Static => { - let (span, sugg) = match snippet.as_deref() { - Some("&") => (span.shrink_to_hi(), "'static ".to_owned()), - Some("'_") => (span, "'static".to_owned()), - Some(snippet) if !snippet.ends_with('>') => { - if snippet == "" { - ( - span, - std::iter::repeat("'static") - .take(count) - .collect::>() - .join(", "), - ) - } else { - ( - span.shrink_to_hi(), - format!( - "<{}>", + let mut spans_suggs = Vec::new(); + for ((span, count), snippet) in + spans_with_counts.iter().copied().zip(snippets.iter()) + { + let (span, sugg) = match snippet.as_deref() { + Some("&") => (span.shrink_to_hi(), "'static ".to_owned()), + Some("'_") => (span, "'static".to_owned()), + Some(snippet) if !snippet.ends_with('>') => { + if snippet == "" { + ( + span, + std::iter::repeat("'static") + .take(count) + .collect::>() + .join(", "), + ) + } else if snippet == "<" || snippet == "(" { + ( + span.shrink_to_hi(), std::iter::repeat("'static") .take(count) .collect::>() - .join(", ") - ), - ) + .join(", "), + ) + } else { + ( + span.shrink_to_hi(), + format!( + "<{}>", + std::iter::repeat("'static") + .take(count) + .collect::>() + .join(", "), + ), + ) + } } - } - _ => continue, - }; - err.span_suggestion_verbose( - span, + _ => continue, + }; + spans_suggs.push((span, sugg.to_string())); + } + err.multipart_suggestion_with_style( "consider using the `'static` lifetime", - sugg.to_string(), + spans_suggs, Applicability::MaybeIncorrect, + SuggestionStyle::ShowAlways, ); continue; } @@ -1967,8 +2043,17 @@ impl<'tcx> LifetimeContext<'_, 'tcx> { } } } - introduce_suggestion.push((span, sugg.to_string())); - err.multipart_suggestion(&msg, introduce_suggestion, Applicability::MaybeIncorrect); + for ((span, _), sugg) in spans_with_counts.iter().copied().zip(suggs.iter()) { + if let Some(sugg) = sugg { + introduce_suggestion.push((span, sugg.to_string())); + } + } + err.multipart_suggestion_with_style( + &msg, + introduce_suggestion, + Applicability::MaybeIncorrect, + SuggestionStyle::ShowAlways, + ); if should_break { break; } @@ -1976,68 +2061,81 @@ impl<'tcx> LifetimeContext<'_, 'tcx> { }; let lifetime_names: Vec<_> = lifetime_names.iter().collect(); - match (&lifetime_names[..], snippet.as_deref()) { - ([name], Some("&")) => { - suggest_existing(err, &name.as_str()[..], &|name| format!("&{} ", name)); - } - ([name], Some("'_")) => { - suggest_existing(err, &name.as_str()[..], &|n| n.to_string()); - } - ([name], Some("")) => { - suggest_existing(err, &name.as_str()[..], &|n| format!("{}, ", n).repeat(count)); - } - ([name], Some(snippet)) if !snippet.ends_with('>') => { - let f = |name: &str| { - format!( - "{}<{}>", - snippet, - std::iter::repeat(name.to_string()) - .take(count) - .collect::>() - .join(", ") - ) - }; - suggest_existing(err, &name.as_str()[..], &f); - } - ([], Some("&")) if count == 1 => { - suggest_new(err, "&'a "); - } - ([], Some("'_")) if count == 1 => { - suggest_new(err, "'a"); + match &lifetime_names[..] { + [name] => { + let mut suggs: Vec String>>> = Vec::new(); + for (snippet, (_, count)) in snippets.iter().zip(spans_with_counts.iter().copied()) + { + suggs.push(match snippet.as_deref() { + Some("&") => Some(Box::new(|name| format!("&{} ", name))), + Some("'_") => Some(Box::new(|n| n.to_string())), + Some("") => Some(Box::new(move |n| format!("{}, ", n).repeat(count))), + Some("<") => Some(Box::new(move |n| { + std::iter::repeat(n).take(count).collect::>().join(", ") + })), + Some(snippet) if !snippet.ends_with('>') => Some(Box::new(move |name| { + format!( + "{}<{}>", + snippet, + std::iter::repeat(name.to_string()) + .take(count) + .collect::>() + .join(", ") + ) + })), + _ => None, + }); + } + suggest_existing(err, &name.as_str()[..], suggs); } - ([], Some(snippet)) if !snippet.ends_with('>') => { - if snippet == "" { - // This happens when we have `type Bar<'a> = Foo` where we point at the space - // before `T`. We will suggest `type Bar<'a> = Foo<'a, T>`. - suggest_new( - err, - &std::iter::repeat("'a, ").take(count).collect::>().join(""), - ); - } else { - suggest_new( - err, - &format!( + [] => { + let mut suggs = Vec::new(); + for (snippet, (_, count)) in + snippets.iter().cloned().zip(spans_with_counts.iter().copied()) + { + suggs.push(match snippet.as_deref() { + Some("&") => Some("&'a ".to_string()), + Some("'_") => Some("'a".to_string()), + Some("") => { + Some(std::iter::repeat("'a, ").take(count).collect::>().join("")) + } + Some("<") => { + Some(std::iter::repeat("'a").take(count).collect::>().join(", ")) + } + Some(snippet) => Some(format!( "{}<{}>", snippet, - std::iter::repeat("'a").take(count).collect::>().join(", ") - ), - ); + std::iter::repeat("'a").take(count).collect::>().join(", "), + )), + None => None, + }); } + suggest_new(err, suggs); } - (lts, ..) if lts.len() > 1 => { + lts if lts.len() > 1 => { err.span_note(lifetime_spans, "these named lifetimes are available to use"); - if Some("") == snippet.as_deref() { + + let mut spans_suggs: Vec<_> = Vec::new(); + for ((span, _), snippet) in spans_with_counts.iter().copied().zip(snippets.iter()) { + match snippet.as_deref() { + Some("") => spans_suggs.push((span, "'lifetime, ".to_string())), + Some("&") => spans_suggs.push((span, "&'lifetime ".to_string())), + _ => {} + } + } + + if spans_suggs.len() > 0 { // This happens when we have `Foo` where we point at the space before `T`, // but this can be confusing so we give a suggestion with placeholders. - err.span_suggestion_verbose( - span, + err.multipart_suggestion_with_style( "consider using one of the available lifetimes here", - "'lifetime, ".repeat(count), + spans_suggs, Applicability::HasPlaceholders, + SuggestionStyle::ShowAlways, ); } } - _ => {} + _ => unreachable!(), } } diff --git a/compiler/rustc_resolve/src/late/lifetimes.rs b/compiler/rustc_resolve/src/late/lifetimes.rs index 174df09cbd..ca7cdc4caf 100644 --- a/compiler/rustc_resolve/src/late/lifetimes.rs +++ b/compiler/rustc_resolve/src/late/lifetimes.rs @@ -8,11 +8,11 @@ use crate::late::diagnostics::{ForLifetimeSpanType, MissingLifetimeSpot}; use rustc_ast::walk_list; -use rustc_data_structures::fx::{FxHashMap, FxHashSet}; +use rustc_data_structures::fx::{FxHashMap, FxHashSet, FxIndexMap}; use rustc_errors::{struct_span_err, Applicability, DiagnosticBuilder}; use rustc_hir as hir; use rustc_hir::def::{DefKind, Res}; -use rustc_hir::def_id::DefIdMap; +use rustc_hir::def_id::{DefIdMap, LocalDefId}; use rustc_hir::hir_id::ItemLocalId; use rustc_hir::intravisit::{self, NestedVisitorMap, Visitor}; use rustc_hir::{GenericArg, GenericParam, LifetimeName, Node, ParamName, QPath}; @@ -22,7 +22,7 @@ use rustc_middle::middle::resolve_lifetime::*; use rustc_middle::ty::{self, DefIdTree, GenericParamDefKind, TyCtxt}; use rustc_middle::{bug, span_bug}; use rustc_session::lint; -use rustc_span::def_id::{DefId, LocalDefId}; +use rustc_span::def_id::DefId; use rustc_span::symbol::{kw, sym, Ident, Symbol}; use rustc_span::Span; use std::borrow::Cow; @@ -158,6 +158,9 @@ struct NamedRegionMap { // - trait refs // - bound types (like `T` in `for<'a> T<'a>: Foo`) late_bound_vars: HirIdMap>, + + // maps `PathSegment` `HirId`s to lifetime scopes. + scope_for_path: Option>>, } crate struct LifetimeContext<'a, 'tcx> { @@ -195,7 +198,9 @@ enum Scope<'a> { /// it should be shifted by the number of `Binder`s in between the /// declaration `Binder` and the location it's referenced from. Binder { - lifetimes: FxHashMap, + /// We use an IndexMap here because we want these lifetimes in order + /// for diagnostics. + lifetimes: FxIndexMap, /// if we extend this scope with another scope, what is the next index /// we should use for an early-bound region? @@ -379,6 +384,10 @@ pub fn provide(providers: &mut ty::query::Providers) { } }, late_bound_vars_map: |tcx, id| resolve_lifetimes_for(tcx, id).late_bound_vars.get(&id), + lifetime_scope_map: |tcx, id| { + let item_id = item_for(tcx, id); + do_resolve(tcx, item_id, false, true).scope_for_path.unwrap().remove(&id) + }, ..*providers }; @@ -419,7 +428,7 @@ fn resolve_lifetimes_trait_definition( tcx: TyCtxt<'_>, local_def_id: LocalDefId, ) -> ResolveLifetimes { - do_resolve(tcx, local_def_id, true) + convert_named_region_map(do_resolve(tcx, local_def_id, true, false)) } /// Computes the `ResolveLifetimes` map that contains data for an entire `Item`. @@ -427,19 +436,21 @@ fn resolve_lifetimes_trait_definition( /// `named_region_map`, `is_late_bound_map`, etc. #[tracing::instrument(level = "debug", skip(tcx))] fn resolve_lifetimes(tcx: TyCtxt<'_>, local_def_id: LocalDefId) -> ResolveLifetimes { - do_resolve(tcx, local_def_id, false) + convert_named_region_map(do_resolve(tcx, local_def_id, false, false)) } fn do_resolve( tcx: TyCtxt<'_>, local_def_id: LocalDefId, trait_definition_only: bool, -) -> ResolveLifetimes { + with_scope_for_path: bool, +) -> NamedRegionMap { let item = tcx.hir().expect_item(tcx.hir().local_def_id_to_hir_id(local_def_id)); let mut named_region_map = NamedRegionMap { defs: Default::default(), late_bound: Default::default(), late_bound_vars: Default::default(), + scope_for_path: with_scope_for_path.then(|| Default::default()), }; let mut visitor = LifetimeContext { tcx, @@ -455,6 +466,10 @@ fn do_resolve( }; visitor.visit_item(item); + named_region_map +} + +fn convert_named_region_map(named_region_map: NamedRegionMap) -> ResolveLifetimes { let mut rl = ResolveLifetimes::default(); for (hir_id, v) in named_region_map.defs { @@ -567,6 +582,41 @@ fn late_region_as_bound_region<'tcx>(tcx: TyCtxt<'tcx>, region: &Region) -> ty:: } } +#[tracing::instrument(level = "debug")] +fn get_lifetime_scopes_for_path(mut scope: &Scope<'_>) -> LifetimeScopeForPath { + let mut available_lifetimes = vec![]; + loop { + match scope { + Scope::Binder { lifetimes, s, .. } => { + available_lifetimes.extend(lifetimes.keys().filter_map(|p| match p { + hir::ParamName::Plain(ident) => Some(ident.name.to_string()), + _ => None, + })); + scope = s; + } + Scope::Body { s, .. } => { + scope = s; + } + Scope::Elision { elide, s } => { + if let Elide::Exact(_) = elide { + return LifetimeScopeForPath::Elided; + } else { + scope = s; + } + } + Scope::ObjectLifetimeDefault { s, .. } => { + scope = s; + } + Scope::Root => { + return LifetimeScopeForPath::NonElided(available_lifetimes); + } + Scope::Supertrait { s, .. } | Scope::TraitRefBoundary { s, .. } => { + scope = s; + } + } + } +} + impl<'a, 'tcx> LifetimeContext<'a, 'tcx> { /// Returns the binders in scope and the type of `Binder` that should be created for a poly trait ref. fn poly_trait_ref_binder_info(&mut self) -> (Vec, BinderScopeType) { @@ -656,7 +706,7 @@ impl<'a, 'tcx> Visitor<'tcx> for LifetimeContext<'a, 'tcx> { self.map.late_bound_vars.insert(hir_id, vec![]); let scope = Scope::Binder { hir_id, - lifetimes: FxHashMap::default(), + lifetimes: FxIndexMap::default(), next_early_index: self.next_early_index(), s: self.scope, track_lifetime_uses: true, @@ -720,9 +770,7 @@ impl<'a, 'tcx> Visitor<'tcx> for LifetimeContext<'a, 'tcx> { // We need to add *all* deps, since opaque tys may want them from *us* for (&owner, defs) in resolved_lifetimes.defs.iter() { defs.iter().for_each(|(&local_id, region)| { - self.map - .defs - .insert(hir::HirId { owner, local_id }, region.clone()); + self.map.defs.insert(hir::HirId { owner, local_id }, *region); }); } for (&owner, late_bound) in resolved_lifetimes.late_bound.iter() { @@ -836,7 +884,7 @@ impl<'a, 'tcx> Visitor<'tcx> for LifetimeContext<'a, 'tcx> { }; self.missing_named_lifetime_spots .push(MissingLifetimeSpot::HigherRanked { span, span_type }); - let (lifetimes, binders): (FxHashMap, Vec<_>) = c + let (lifetimes, binders): (FxIndexMap, Vec<_>) = c .generic_params .iter() .filter_map(|param| match param.kind { @@ -1010,7 +1058,7 @@ impl<'a, 'tcx> Visitor<'tcx> for LifetimeContext<'a, 'tcx> { debug!(?index); let mut elision = None; - let mut lifetimes = FxHashMap::default(); + let mut lifetimes = FxIndexMap::default(); let mut non_lifetime_count = 0; for param in generics.params { match param.kind { @@ -1181,7 +1229,7 @@ impl<'a, 'tcx> Visitor<'tcx> for LifetimeContext<'a, 'tcx> { let mut index = self.next_early_index(); let mut non_lifetime_count = 0; debug!("visit_ty: index = {}", index); - let lifetimes: FxHashMap = generics + let lifetimes: FxIndexMap = generics .params .iter() .filter_map(|param| match param.kind { @@ -1241,13 +1289,51 @@ impl<'a, 'tcx> Visitor<'tcx> for LifetimeContext<'a, 'tcx> { self.resolve_lifetime_ref(lifetime_ref); } + fn visit_assoc_type_binding(&mut self, type_binding: &'tcx hir::TypeBinding<'_>) { + let scope = self.scope; + if let Some(scope_for_path) = self.map.scope_for_path.as_mut() { + // We add lifetime scope information for `Ident`s in associated type bindings and use + // the `HirId` of the type binding as the key in `LifetimeMap` + let lifetime_scope = get_lifetime_scopes_for_path(scope); + let map = scope_for_path.entry(type_binding.hir_id.owner).or_default(); + map.insert(type_binding.hir_id.local_id, lifetime_scope); + } + hir::intravisit::walk_assoc_type_binding(self, type_binding); + } + fn visit_path(&mut self, path: &'tcx hir::Path<'tcx>, _: hir::HirId) { for (i, segment) in path.segments.iter().enumerate() { let depth = path.segments.len() - i - 1; if let Some(ref args) = segment.args { self.visit_segment_args(path.res, depth, args); } + + let scope = self.scope; + if let Some(scope_for_path) = self.map.scope_for_path.as_mut() { + // Add lifetime scope information to path segment. Note we cannot call `visit_path_segment` + // here because that call would yield to resolution problems due to `walk_path_segment` + // being called, which processes the path segments generic args, which we have already + // processed using `visit_segment_args`. + let lifetime_scope = get_lifetime_scopes_for_path(scope); + if let Some(hir_id) = segment.hir_id { + let map = scope_for_path.entry(hir_id.owner).or_default(); + map.insert(hir_id.local_id, lifetime_scope); + } + } + } + } + + fn visit_path_segment(&mut self, path_span: Span, path_segment: &'tcx hir::PathSegment<'tcx>) { + let scope = self.scope; + if let Some(scope_for_path) = self.map.scope_for_path.as_mut() { + let lifetime_scope = get_lifetime_scopes_for_path(scope); + if let Some(hir_id) = path_segment.hir_id { + let map = scope_for_path.entry(hir_id.owner).or_default(); + map.insert(hir_id.local_id, lifetime_scope); + } } + + intravisit::walk_path_segment(self, path_span, path_segment); } fn visit_fn_decl(&mut self, fd: &'tcx hir::FnDecl<'tcx>) { @@ -1290,7 +1376,7 @@ impl<'a, 'tcx> Visitor<'tcx> for LifetimeContext<'a, 'tcx> { ref bound_generic_params, .. }) => { - let (lifetimes, binders): (FxHashMap, Vec<_>) = + let (lifetimes, binders): (FxIndexMap, Vec<_>) = bound_generic_params .iter() .filter_map(|param| match param.kind { @@ -1360,7 +1446,7 @@ impl<'a, 'tcx> Visitor<'tcx> for LifetimeContext<'a, 'tcx> { self.map.late_bound_vars.insert(*hir_id, binders); let scope = Scope::Binder { hir_id: *hir_id, - lifetimes: FxHashMap::default(), + lifetimes: FxIndexMap::default(), s: self.scope, next_early_index: self.next_early_index(), track_lifetime_uses: true, @@ -1388,7 +1474,7 @@ impl<'a, 'tcx> Visitor<'tcx> for LifetimeContext<'a, 'tcx> { let (mut binders, scope_type) = self.poly_trait_ref_binder_info(); let initial_bound_vars = binders.len() as u32; - let mut lifetimes: FxHashMap = FxHashMap::default(); + let mut lifetimes: FxIndexMap = FxIndexMap::default(); let binders_iter = trait_ref .bound_generic_params .iter() @@ -1755,14 +1841,6 @@ fn object_lifetime_defaults_for_item( } impl<'a, 'tcx> LifetimeContext<'a, 'tcx> { - // FIXME(#37666) this works around a limitation in the region inferencer - fn hack(&mut self, f: F) - where - F: for<'b> FnOnce(&mut LifetimeContext<'b, 'tcx>), - { - f(self) - } - fn with(&mut self, wrap_scope: Scope<'_>, f: F) where F: for<'b> FnOnce(ScopeRef<'_>, &mut LifetimeContext<'b, 'tcx>), @@ -2115,7 +2193,7 @@ impl<'a, 'tcx> LifetimeContext<'a, 'tcx> { let mut non_lifetime_count = 0; let mut named_late_bound_vars = 0; - let lifetimes: FxHashMap = generics + let lifetimes: FxIndexMap = generics .params .iter() .filter_map(|param| match param.kind { @@ -2166,7 +2244,7 @@ impl<'a, 'tcx> LifetimeContext<'a, 'tcx> { }; self.with(scope, move |old_scope, this| { this.check_lifetime_params(old_scope, &generics.params); - this.hack(walk); // FIXME(#37666) workaround in place of `walk(this)` + walk(this); }); } @@ -2956,7 +3034,6 @@ impl<'a, 'tcx> LifetimeContext<'a, 'tcx> { return; } - let span = lifetime_refs[0].span; let mut late_depth = 0; let mut scope = self.scope; let mut lifetime_names = FxHashSet::default(); @@ -3035,7 +3112,26 @@ impl<'a, 'tcx> LifetimeContext<'a, 'tcx> { } }; - let mut err = self.report_missing_lifetime_specifiers(span, lifetime_refs.len()); + // If we specifically need the `scope_for_path` map, then we're in the + // diagnostic pass and we don't want to emit more errors. + if self.map.scope_for_path.is_some() { + self.tcx.sess.delay_span_bug( + rustc_span::DUMMY_SP, + "Encountered unexpected errors during diagnostics related part", + ); + return; + } + + let mut spans: Vec<_> = lifetime_refs.iter().map(|lt| lt.span).collect(); + spans.sort(); + let mut spans_dedup = spans.clone(); + spans_dedup.dedup(); + let spans_with_counts: Vec<_> = spans_dedup + .into_iter() + .map(|sp| (sp, spans.iter().filter(|nsp| *nsp == &sp).count())) + .collect(); + + let mut err = self.report_missing_lifetime_specifiers(spans.clone(), lifetime_refs.len()); if let Some(params) = error { // If there's no lifetime available, suggest `'static`. @@ -3043,10 +3139,10 @@ impl<'a, 'tcx> LifetimeContext<'a, 'tcx> { lifetime_names.insert(kw::StaticLifetime); } } + self.add_missing_lifetime_specifiers_label( &mut err, - span, - lifetime_refs.len(), + spans_with_counts, &lifetime_names, lifetime_spans, error.unwrap_or(&[]), @@ -3347,7 +3443,7 @@ impl<'a, 'tcx> LifetimeContext<'a, 'tcx> { fn insert_lifetime(&mut self, lifetime_ref: &'tcx hir::Lifetime, def: Region) { debug!( node = ?self.tcx.hir().node_to_string(lifetime_ref.hir_id), - span = ?self.tcx.sess.source_map().span_to_string(lifetime_ref.span) + span = ?self.tcx.sess.source_map().span_to_diagnostic_string(lifetime_ref.span) ); self.map.defs.insert(lifetime_ref.hir_id, def); diff --git a/compiler/rustc_resolve/src/lib.rs b/compiler/rustc_resolve/src/lib.rs index 9197f4059c..6d5531d330 100644 --- a/compiler/rustc_resolve/src/lib.rs +++ b/compiler/rustc_resolve/src/lib.rs @@ -16,7 +16,6 @@ #![feature(format_args_capture)] #![feature(iter_zip)] #![feature(nll)] -#![cfg_attr(bootstrap, feature(or_patterns))] #![recursion_limit = "256"] #![allow(rustdoc::private_intra_doc_links)] @@ -234,9 +233,16 @@ enum ResolutionError<'a> { /* current */ &'static str, ), /// Error E0530: `X` bindings cannot shadow `Y`s. - BindingShadowsSomethingUnacceptable(&'static str, Symbol, &'a NameBinding<'a>), + BindingShadowsSomethingUnacceptable { + shadowing_binding_descr: &'static str, + name: Symbol, + participle: &'static str, + article: &'static str, + shadowed_binding_descr: &'static str, + shadowed_binding_span: Span, + }, /// Error E0128: generic parameters with a default cannot use forward-declared identifiers. - ForwardDeclaredTyParam, // FIXME(const_generics_defaults) + ForwardDeclaredGenericParam, /// ERROR E0770: the type of const parameters must not depend on other generic parameters. ParamInTyOfConstParam(Symbol), /// generic parameters must not be used inside const evaluations. @@ -903,7 +909,7 @@ pub struct Resolver<'a> { /// `CrateNum` resolutions of `extern crate` items. extern_crate_map: FxHashMap, export_map: ExportMap, - trait_map: NodeMap>, + trait_map: Option>>, /// A map from nodes to anonymous modules. /// Anonymous modules are pseudo-modules that are implicitly created around items @@ -1132,8 +1138,8 @@ impl ResolverAstLowering for Resolver<'_> { self.next_node_id() } - fn trait_map(&self) -> &NodeMap> { - &self.trait_map + fn take_trait_map(&mut self) -> NodeMap> { + std::mem::replace(&mut self.trait_map, None).unwrap() } fn opt_local_def_id(&self, node: NodeId) -> Option { @@ -1192,7 +1198,7 @@ impl<'a> Resolver<'a> { session: &'a Session, krate: &Crate, crate_name: &str, - metadata_loader: &'a MetadataLoaderDyn, + metadata_loader: Box, arenas: &'a ResolverArenas<'a>, ) -> Resolver<'a> { let root_local_def_id = LocalDefId { local_def_index: CRATE_DEF_INDEX }; @@ -1280,7 +1286,7 @@ impl<'a> Resolver<'a> { label_res_map: Default::default(), extern_crate_map: Default::default(), export_map: FxHashMap::default(), - trait_map: Default::default(), + trait_map: Some(NodeMap::default()), underscore_disambiguator: 0, empty_module, module_map, @@ -1770,9 +1776,11 @@ impl<'a> Resolver<'a> { let expn_data = expn_id.expn_data(); match expn_data.kind { ExpnKind::Root - | ExpnKind::Macro(MacroKind::Bang | MacroKind::Derive, _) => { - Scope::DeriveHelpersCompat - } + | ExpnKind::Macro { + kind: MacroKind::Bang | MacroKind::Derive, + name: _, + proc_macro: _, + } => Scope::DeriveHelpersCompat, _ => Scope::DeriveHelpers(expn_data.parent), } } @@ -2600,7 +2608,7 @@ impl<'a> Resolver<'a> { let res_error = if rib_ident.name == kw::SelfUpper { ResolutionError::SelfInTyParamDefault } else { - ResolutionError::ForwardDeclaredTyParam + ResolutionError::ForwardDeclaredGenericParam }; self.report_error(span, res_error); } diff --git a/compiler/rustc_resolve/src/macros.rs b/compiler/rustc_resolve/src/macros.rs index 10e27f33c2..3f7db2b696 100644 --- a/compiler/rustc_resolve/src/macros.rs +++ b/compiler/rustc_resolve/src/macros.rs @@ -20,7 +20,7 @@ use rustc_expand::compile_declarative_macro; use rustc_expand::expand::{AstFragment, Invocation, InvocationKind, SupportsMacroExpansion}; use rustc_feature::is_builtin_attr_name; use rustc_hir::def::{self, DefKind, NonMacroAttrKind}; -use rustc_hir::def_id; +use rustc_hir::def_id::{self, CrateNum}; use rustc_hir::PrimTy; use rustc_middle::middle::stability; use rustc_middle::ty; @@ -325,7 +325,11 @@ impl<'a> ResolverExpand for Resolver<'a> { let expn_data = expn_id.expn_data(); match expn_data.kind { ExpnKind::Root - | ExpnKind::Macro(MacroKind::Bang | MacroKind::Derive, _) => { + | ExpnKind::Macro { + name: _, + kind: MacroKind::Bang | MacroKind::Derive, + proc_macro: _, + } => { break; } _ => expn_id = expn_data.parent, @@ -462,6 +466,10 @@ impl<'a> ResolverExpand for Resolver<'a> { .emit(); Ok(false) } + + fn get_proc_macro_quoted_span(&self, krate: CrateNum, id: usize) -> Span { + self.crate_loader.cstore().get_proc_macro_quoted_span_untracked(krate, id, self.session) + } } impl<'a> Resolver<'a> { diff --git a/compiler/rustc_save_analysis/src/dump_visitor.rs b/compiler/rustc_save_analysis/src/dump_visitor.rs index 12c77e0c8a..54b6a12158 100644 --- a/compiler/rustc_save_analysis/src/dump_visitor.rs +++ b/compiler/rustc_save_analysis/src/dump_visitor.rs @@ -190,7 +190,7 @@ impl<'tcx> DumpVisitor<'tcx> { }; let data = CompilationOptions { - directory: self.tcx.sess.working_dir.0.clone(), + directory: self.tcx.sess.working_dir.remapped_path_if_available().into(), program, arguments, output: self.save_ctxt.compilation_output(crate_name), @@ -1112,7 +1112,7 @@ impl<'tcx> DumpVisitor<'tcx> { name: String::new(), qualname, span, - value: filename.to_string(), + value: filename.prefer_remapped().to_string(), children, parent: None, decl_id: None, diff --git a/compiler/rustc_save_analysis/src/lib.rs b/compiler/rustc_save_analysis/src/lib.rs index c19c16b88a..297383bfed 100644 --- a/compiler/rustc_save_analysis/src/lib.rs +++ b/compiler/rustc_save_analysis/src/lib.rs @@ -1,6 +1,5 @@ #![doc(html_root_url = "https://doc.rust-lang.org/nightly/nightly-rustc/")] #![feature(nll)] -#![cfg_attr(bootstrap, feature(or_patterns))] #![recursion_limit = "256"] mod dump_visitor; @@ -80,7 +79,7 @@ impl<'tcx> SaveContext<'tcx> { let end = sm.lookup_char_pos(span.hi()); SpanData { - file_name: start.file.name.to_string().into(), + file_name: start.file.name.prefer_remapped().to_string().into(), byte_start: span.lo().0, byte_end: span.hi().0, line_start: Row::new_one_indexed(start.line as u32), @@ -95,7 +94,7 @@ impl<'tcx> SaveContext<'tcx> { let sess = &self.tcx.sess; // Save-analysis is emitted per whole session, not per each crate type let crate_type = sess.crate_types()[0]; - let outputs = &*self.tcx.output_filenames(LOCAL_CRATE); + let outputs = &*self.tcx.output_filenames(()); if outputs.outputs.contains_key(&OutputType::Metadata) { filename_for_metadata(sess, crate_name, outputs) @@ -290,7 +289,7 @@ impl<'tcx> SaveContext<'tcx> { name: item.ident.to_string(), qualname, span: self.span_from_span(item.ident.span), - value: filename.to_string(), + value: filename.prefer_remapped().to_string(), parent: None, children: m .item_ids @@ -786,7 +785,7 @@ impl<'tcx> SaveContext<'tcx> { let callee = span.source_callee()?; let mac_name = match callee.kind { - ExpnKind::Macro(kind, name) => match kind { + ExpnKind::Macro { kind, name, proc_macro: _ } => match kind { MacroKind::Bang => name, // Ignore attribute macros, their spans are usually mangled @@ -824,20 +823,6 @@ impl<'tcx> SaveContext<'tcx> { // FIXME: Should save-analysis beautify doc strings itself or leave it to users? result.push_str(&beautify_doc_string(val).as_str()); result.push('\n'); - } else if self.tcx.sess.check_name(attr, sym::doc) { - if let Some(meta_list) = attr.meta_item_list() { - meta_list - .into_iter() - .filter(|it| it.has_name(sym::include)) - .filter_map(|it| it.meta_item_list().map(|l| l.to_owned())) - .flat_map(|it| it) - .filter(|meta| meta.has_name(sym::contents)) - .filter_map(|meta| meta.value_str()) - .for_each(|val| { - result.push_str(&val.as_str()); - result.push('\n'); - }); - } } } @@ -1000,7 +985,7 @@ pub fn process_crate<'l, 'tcx, H: SaveHandler>( // Privacy checking requires and is done after type checking; use a // fallback in case the access levels couldn't have been correctly computed. let access_levels = match tcx.sess.compile_status() { - Ok(..) => tcx.privacy_access_levels(LOCAL_CRATE), + Ok(..) => tcx.privacy_access_levels(()), Err(..) => tcx.arena.alloc(AccessLevels::default()), }; diff --git a/compiler/rustc_save_analysis/src/span_utils.rs b/compiler/rustc_save_analysis/src/span_utils.rs index edcd492577..1947b04f44 100644 --- a/compiler/rustc_save_analysis/src/span_utils.rs +++ b/compiler/rustc_save_analysis/src/span_utils.rs @@ -16,8 +16,7 @@ impl<'a> SpanUtils<'a> { pub fn make_filename_string(&self, file: &SourceFile) -> String { match &file.name { - FileName::Real(name) if !file.name_was_remapped => { - let path = name.local_path(); + FileName::Real(RealFileName::LocalPath(path)) => { if path.is_absolute() { self.sess .source_map() @@ -27,12 +26,15 @@ impl<'a> SpanUtils<'a> { .display() .to_string() } else { - self.sess.working_dir.0.join(&path).display().to_string() + self.sess + .working_dir + .remapped_path_if_available() + .join(&path) + .display() + .to_string() } } - // If the file name is already remapped, we assume the user - // configured it the way they wanted to, so use that directly - filename => filename.to_string(), + filename => filename.prefer_remapped().to_string(), } } diff --git a/compiler/rustc_serialize/src/collection_impls.rs b/compiler/rustc_serialize/src/collection_impls.rs index ae6d27e037..80a7f65018 100644 --- a/compiler/rustc_serialize/src/collection_impls.rs +++ b/compiler/rustc_serialize/src/collection_impls.rs @@ -21,8 +21,8 @@ impl>> Decodable for SmallVec { d.read_seq(|d, len| { let mut vec = SmallVec::with_capacity(len); // FIXME(#48994) - could just be collected into a Result - for i in 0..len { - vec.push(d.read_seq_elt(i, |d| Decodable::decode(d))?); + for _ in 0..len { + vec.push(d.read_seq_elt(|d| Decodable::decode(d))?); } Ok(vec) }) @@ -44,8 +44,8 @@ impl> Decodable for LinkedList { fn decode(d: &mut D) -> Result, D::Error> { d.read_seq(|d, len| { let mut list = LinkedList::new(); - for i in 0..len { - list.push_back(d.read_seq_elt(i, |d| Decodable::decode(d))?); + for _ in 0..len { + list.push_back(d.read_seq_elt(|d| Decodable::decode(d))?); } Ok(list) }) @@ -67,8 +67,8 @@ impl> Decodable for VecDeque { fn decode(d: &mut D) -> Result, D::Error> { d.read_seq(|d, len| { let mut deque: VecDeque = VecDeque::with_capacity(len); - for i in 0..len { - deque.push_back(d.read_seq_elt(i, |d| Decodable::decode(d))?); + for _ in 0..len { + deque.push_back(d.read_seq_elt(|d| Decodable::decode(d))?); } Ok(deque) }) @@ -84,7 +84,7 @@ where e.emit_map(self.len(), |e| { for (i, (key, val)) in self.iter().enumerate() { e.emit_map_elt_key(i, |e| key.encode(e))?; - e.emit_map_elt_val(i, |e| val.encode(e))?; + e.emit_map_elt_val(|e| val.encode(e))?; } Ok(()) }) @@ -99,9 +99,9 @@ where fn decode(d: &mut D) -> Result, D::Error> { d.read_map(|d, len| { let mut map = BTreeMap::new(); - for i in 0..len { - let key = d.read_map_elt_key(i, |d| Decodable::decode(d))?; - let val = d.read_map_elt_val(i, |d| Decodable::decode(d))?; + for _ in 0..len { + let key = d.read_map_elt_key(|d| Decodable::decode(d))?; + let val = d.read_map_elt_val(|d| Decodable::decode(d))?; map.insert(key, val); } Ok(map) @@ -130,8 +130,8 @@ where fn decode(d: &mut D) -> Result, D::Error> { d.read_seq(|d, len| { let mut set = BTreeSet::new(); - for i in 0..len { - set.insert(d.read_seq_elt(i, |d| Decodable::decode(d))?); + for _ in 0..len { + set.insert(d.read_seq_elt(|d| Decodable::decode(d))?); } Ok(set) }) @@ -148,7 +148,7 @@ where e.emit_map(self.len(), |e| { for (i, (key, val)) in self.iter().enumerate() { e.emit_map_elt_key(i, |e| key.encode(e))?; - e.emit_map_elt_val(i, |e| val.encode(e))?; + e.emit_map_elt_val(|e| val.encode(e))?; } Ok(()) }) @@ -165,9 +165,9 @@ where d.read_map(|d, len| { let state = Default::default(); let mut map = HashMap::with_capacity_and_hasher(len, state); - for i in 0..len { - let key = d.read_map_elt_key(i, |d| Decodable::decode(d))?; - let val = d.read_map_elt_val(i, |d| Decodable::decode(d))?; + for _ in 0..len { + let key = d.read_map_elt_key(|d| Decodable::decode(d))?; + let val = d.read_map_elt_val(|d| Decodable::decode(d))?; map.insert(key, val); } Ok(map) @@ -209,8 +209,8 @@ where d.read_seq(|d, len| { let state = Default::default(); let mut set = HashSet::with_capacity_and_hasher(len, state); - for i in 0..len { - set.insert(d.read_seq_elt(i, |d| Decodable::decode(d))?); + for _ in 0..len { + set.insert(d.read_seq_elt(|d| Decodable::decode(d))?); } Ok(set) }) @@ -227,7 +227,7 @@ where e.emit_map(self.len(), |e| { for (i, (key, val)) in self.iter().enumerate() { e.emit_map_elt_key(i, |e| key.encode(e))?; - e.emit_map_elt_val(i, |e| val.encode(e))?; + e.emit_map_elt_val(|e| val.encode(e))?; } Ok(()) }) @@ -244,9 +244,9 @@ where d.read_map(|d, len| { let state = Default::default(); let mut map = indexmap::IndexMap::with_capacity_and_hasher(len, state); - for i in 0..len { - let key = d.read_map_elt_key(i, |d| Decodable::decode(d))?; - let val = d.read_map_elt_val(i, |d| Decodable::decode(d))?; + for _ in 0..len { + let key = d.read_map_elt_key(|d| Decodable::decode(d))?; + let val = d.read_map_elt_val(|d| Decodable::decode(d))?; map.insert(key, val); } Ok(map) @@ -278,8 +278,8 @@ where d.read_seq(|d, len| { let state = Default::default(); let mut set = indexmap::IndexSet::with_capacity_and_hasher(len, state); - for i in 0..len { - set.insert(d.read_seq_elt(i, |d| Decodable::decode(d))?); + for _ in 0..len { + set.insert(d.read_seq_elt(|d| Decodable::decode(d))?); } Ok(set) }) diff --git a/compiler/rustc_serialize/src/json.rs b/compiler/rustc_serialize/src/json.rs index 78a102c5c2..b79adb6f7b 100644 --- a/compiler/rustc_serialize/src/json.rs +++ b/compiler/rustc_serialize/src/json.rs @@ -560,7 +560,7 @@ impl<'a> crate::Encoder for Encoder<'a> { Ok(()) } - fn emit_enum(&mut self, _name: &str, f: F) -> EncodeResult + fn emit_enum(&mut self, f: F) -> EncodeResult where F: FnOnce(&mut Encoder<'a>) -> EncodeResult, { @@ -589,46 +589,20 @@ impl<'a> crate::Encoder for Encoder<'a> { } } - fn emit_enum_variant_arg(&mut self, idx: usize, f: F) -> EncodeResult + fn emit_enum_variant_arg(&mut self, first: bool, f: F) -> EncodeResult where F: FnOnce(&mut Encoder<'a>) -> EncodeResult, { if self.is_emitting_map_key { return Err(EncoderError::BadHashmapKey); } - if idx != 0 { + if !first { write!(self.writer, ",")?; } f(self) } - fn emit_enum_struct_variant( - &mut self, - name: &str, - id: usize, - cnt: usize, - f: F, - ) -> EncodeResult - where - F: FnOnce(&mut Encoder<'a>) -> EncodeResult, - { - if self.is_emitting_map_key { - return Err(EncoderError::BadHashmapKey); - } - self.emit_enum_variant(name, id, cnt, f) - } - - fn emit_enum_struct_variant_field(&mut self, _: &str, idx: usize, f: F) -> EncodeResult - where - F: FnOnce(&mut Encoder<'a>) -> EncodeResult, - { - if self.is_emitting_map_key { - return Err(EncoderError::BadHashmapKey); - } - self.emit_enum_variant_arg(idx, f) - } - - fn emit_struct(&mut self, _: &str, _: usize, f: F) -> EncodeResult + fn emit_struct(&mut self, _: bool, f: F) -> EncodeResult where F: FnOnce(&mut Encoder<'a>) -> EncodeResult, { @@ -641,14 +615,14 @@ impl<'a> crate::Encoder for Encoder<'a> { Ok(()) } - fn emit_struct_field(&mut self, name: &str, idx: usize, f: F) -> EncodeResult + fn emit_struct_field(&mut self, name: &str, first: bool, f: F) -> EncodeResult where F: FnOnce(&mut Encoder<'a>) -> EncodeResult, { if self.is_emitting_map_key { return Err(EncoderError::BadHashmapKey); } - if idx != 0 { + if !first { write!(self.writer, ",")?; } escape_str(self.writer, name)?; @@ -675,25 +649,6 @@ impl<'a> crate::Encoder for Encoder<'a> { self.emit_seq_elt(idx, f) } - fn emit_tuple_struct(&mut self, _name: &str, len: usize, f: F) -> EncodeResult - where - F: FnOnce(&mut Encoder<'a>) -> EncodeResult, - { - if self.is_emitting_map_key { - return Err(EncoderError::BadHashmapKey); - } - self.emit_seq(len, f) - } - fn emit_tuple_struct_arg(&mut self, idx: usize, f: F) -> EncodeResult - where - F: FnOnce(&mut Encoder<'a>) -> EncodeResult, - { - if self.is_emitting_map_key { - return Err(EncoderError::BadHashmapKey); - } - self.emit_seq_elt(idx, f) - } - fn emit_option(&mut self, f: F) -> EncodeResult where F: FnOnce(&mut Encoder<'a>) -> EncodeResult, @@ -774,7 +729,7 @@ impl<'a> crate::Encoder for Encoder<'a> { Ok(()) } - fn emit_map_elt_val(&mut self, _idx: usize, f: F) -> EncodeResult + fn emit_map_elt_val(&mut self, f: F) -> EncodeResult where F: FnOnce(&mut Encoder<'a>) -> EncodeResult, { @@ -892,7 +847,7 @@ impl<'a> crate::Encoder for PrettyEncoder<'a> { Ok(()) } - fn emit_enum(&mut self, _name: &str, f: F) -> EncodeResult + fn emit_enum(&mut self, f: F) -> EncodeResult where F: FnOnce(&mut PrettyEncoder<'a>) -> EncodeResult, { @@ -930,54 +885,28 @@ impl<'a> crate::Encoder for PrettyEncoder<'a> { } } - fn emit_enum_variant_arg(&mut self, idx: usize, f: F) -> EncodeResult + fn emit_enum_variant_arg(&mut self, first: bool, f: F) -> EncodeResult where F: FnOnce(&mut PrettyEncoder<'a>) -> EncodeResult, { if self.is_emitting_map_key { return Err(EncoderError::BadHashmapKey); } - if idx != 0 { + if !first { writeln!(self.writer, ",")?; } spaces(self.writer, self.curr_indent)?; f(self) } - fn emit_enum_struct_variant( - &mut self, - name: &str, - id: usize, - cnt: usize, - f: F, - ) -> EncodeResult + fn emit_struct(&mut self, no_fields: bool, f: F) -> EncodeResult where F: FnOnce(&mut PrettyEncoder<'a>) -> EncodeResult, { if self.is_emitting_map_key { return Err(EncoderError::BadHashmapKey); } - self.emit_enum_variant(name, id, cnt, f) - } - - fn emit_enum_struct_variant_field(&mut self, _: &str, idx: usize, f: F) -> EncodeResult - where - F: FnOnce(&mut PrettyEncoder<'a>) -> EncodeResult, - { - if self.is_emitting_map_key { - return Err(EncoderError::BadHashmapKey); - } - self.emit_enum_variant_arg(idx, f) - } - - fn emit_struct(&mut self, _: &str, len: usize, f: F) -> EncodeResult - where - F: FnOnce(&mut PrettyEncoder<'a>) -> EncodeResult, - { - if self.is_emitting_map_key { - return Err(EncoderError::BadHashmapKey); - } - if len == 0 { + if no_fields { write!(self.writer, "{{}}")?; } else { write!(self.writer, "{{")?; @@ -991,14 +920,14 @@ impl<'a> crate::Encoder for PrettyEncoder<'a> { Ok(()) } - fn emit_struct_field(&mut self, name: &str, idx: usize, f: F) -> EncodeResult + fn emit_struct_field(&mut self, name: &str, first: bool, f: F) -> EncodeResult where F: FnOnce(&mut PrettyEncoder<'a>) -> EncodeResult, { if self.is_emitting_map_key { return Err(EncoderError::BadHashmapKey); } - if idx == 0 { + if first { writeln!(self.writer)?; } else { writeln!(self.writer, ",")?; @@ -1028,25 +957,6 @@ impl<'a> crate::Encoder for PrettyEncoder<'a> { self.emit_seq_elt(idx, f) } - fn emit_tuple_struct(&mut self, _: &str, len: usize, f: F) -> EncodeResult - where - F: FnOnce(&mut PrettyEncoder<'a>) -> EncodeResult, - { - if self.is_emitting_map_key { - return Err(EncoderError::BadHashmapKey); - } - self.emit_seq(len, f) - } - fn emit_tuple_struct_arg(&mut self, idx: usize, f: F) -> EncodeResult - where - F: FnOnce(&mut PrettyEncoder<'a>) -> EncodeResult, - { - if self.is_emitting_map_key { - return Err(EncoderError::BadHashmapKey); - } - self.emit_seq_elt(idx, f) - } - fn emit_option(&mut self, f: F) -> EncodeResult where F: FnOnce(&mut PrettyEncoder<'a>) -> EncodeResult, @@ -1149,7 +1059,7 @@ impl<'a> crate::Encoder for PrettyEncoder<'a> { Ok(()) } - fn emit_map_elt_val(&mut self, _idx: usize, f: F) -> EncodeResult + fn emit_map_elt_val(&mut self, f: F) -> EncodeResult where F: FnOnce(&mut PrettyEncoder<'a>) -> EncodeResult, { @@ -2373,7 +2283,7 @@ impl crate::Decoder for Decoder { Ok(()) } - fn read_enum(&mut self, _name: &str, f: F) -> DecodeResult + fn read_enum(&mut self, f: F) -> DecodeResult where F: FnOnce(&mut Decoder) -> DecodeResult, { @@ -2410,33 +2320,14 @@ impl crate::Decoder for Decoder { f(self, idx) } - fn read_enum_variant_arg(&mut self, _idx: usize, f: F) -> DecodeResult + fn read_enum_variant_arg(&mut self, f: F) -> DecodeResult where F: FnOnce(&mut Decoder) -> DecodeResult, { f(self) } - fn read_enum_struct_variant(&mut self, names: &[&str], f: F) -> DecodeResult - where - F: FnMut(&mut Decoder, usize) -> DecodeResult, - { - self.read_enum_variant(names, f) - } - - fn read_enum_struct_variant_field( - &mut self, - _name: &str, - idx: usize, - f: F, - ) -> DecodeResult - where - F: FnOnce(&mut Decoder) -> DecodeResult, - { - self.read_enum_variant_arg(idx, f) - } - - fn read_struct(&mut self, _name: &str, _len: usize, f: F) -> DecodeResult + fn read_struct(&mut self, f: F) -> DecodeResult where F: FnOnce(&mut Decoder) -> DecodeResult, { @@ -2445,7 +2336,7 @@ impl crate::Decoder for Decoder { Ok(value) } - fn read_struct_field(&mut self, name: &str, _idx: usize, f: F) -> DecodeResult + fn read_struct_field(&mut self, name: &str, f: F) -> DecodeResult where F: FnOnce(&mut Decoder) -> DecodeResult, { @@ -2483,25 +2374,11 @@ impl crate::Decoder for Decoder { }) } - fn read_tuple_arg(&mut self, idx: usize, f: F) -> DecodeResult - where - F: FnOnce(&mut Decoder) -> DecodeResult, - { - self.read_seq_elt(idx, f) - } - - fn read_tuple_struct(&mut self, _name: &str, len: usize, f: F) -> DecodeResult - where - F: FnOnce(&mut Decoder) -> DecodeResult, - { - self.read_tuple(len, f) - } - - fn read_tuple_struct_arg(&mut self, idx: usize, f: F) -> DecodeResult + fn read_tuple_arg(&mut self, f: F) -> DecodeResult where F: FnOnce(&mut Decoder) -> DecodeResult, { - self.read_tuple_arg(idx, f) + self.read_seq_elt(f) } fn read_option(&mut self, mut f: F) -> DecodeResult @@ -2527,7 +2404,7 @@ impl crate::Decoder for Decoder { f(self, len) } - fn read_seq_elt(&mut self, _idx: usize, f: F) -> DecodeResult + fn read_seq_elt(&mut self, f: F) -> DecodeResult where F: FnOnce(&mut Decoder) -> DecodeResult, { @@ -2547,14 +2424,14 @@ impl crate::Decoder for Decoder { f(self, len) } - fn read_map_elt_key(&mut self, _idx: usize, f: F) -> DecodeResult + fn read_map_elt_key(&mut self, f: F) -> DecodeResult where F: FnOnce(&mut Decoder) -> DecodeResult, { f(self) } - fn read_map_elt_val(&mut self, _idx: usize, f: F) -> DecodeResult + fn read_map_elt_val(&mut self, f: F) -> DecodeResult where F: FnOnce(&mut Decoder) -> DecodeResult, { diff --git a/compiler/rustc_serialize/src/lib.rs b/compiler/rustc_serialize/src/lib.rs index cf5a911827..c79786a839 100644 --- a/compiler/rustc_serialize/src/lib.rs +++ b/compiler/rustc_serialize/src/lib.rs @@ -14,10 +14,7 @@ Core encoding and decoding interfaces. #![feature(nll)] #![feature(associated_type_bounds)] #![feature(min_specialization)] -#![feature(vec_spare_capacity)] #![feature(core_intrinsics)] -#![feature(maybe_uninit_array_assume_init)] -#![feature(maybe_uninit_uninit_array)] #![feature(maybe_uninit_slice)] #![feature(new_uninit)] #![cfg_attr(test, feature(test))] diff --git a/compiler/rustc_serialize/src/serialize.rs b/compiler/rustc_serialize/src/serialize.rs index d3e5f30697..bb3c537ef1 100644 --- a/compiler/rustc_serialize/src/serialize.rs +++ b/compiler/rustc_serialize/src/serialize.rs @@ -37,7 +37,7 @@ pub trait Encoder { // Compound types: #[inline] - fn emit_enum(&mut self, _name: &str, f: F) -> Result<(), Self::Error> + fn emit_enum(&mut self, f: F) -> Result<(), Self::Error> where F: FnOnce(&mut Self) -> Result<(), Self::Error>, { @@ -59,40 +59,15 @@ pub trait Encoder { } #[inline] - fn emit_enum_variant_arg(&mut self, _a_idx: usize, f: F) -> Result<(), Self::Error> + fn emit_enum_variant_arg(&mut self, _first: bool, f: F) -> Result<(), Self::Error> where F: FnOnce(&mut Self) -> Result<(), Self::Error>, { f(self) } - fn emit_enum_struct_variant( - &mut self, - v_name: &str, - v_id: usize, - len: usize, - f: F, - ) -> Result<(), Self::Error> - where - F: FnOnce(&mut Self) -> Result<(), Self::Error>, - { - self.emit_enum_variant(v_name, v_id, len, f) - } - - fn emit_enum_struct_variant_field( - &mut self, - _f_name: &str, - f_idx: usize, - f: F, - ) -> Result<(), Self::Error> - where - F: FnOnce(&mut Self) -> Result<(), Self::Error>, - { - self.emit_enum_variant_arg(f_idx, f) - } - #[inline] - fn emit_struct(&mut self, _name: &str, _len: usize, f: F) -> Result<(), Self::Error> + fn emit_struct(&mut self, _no_fields: bool, f: F) -> Result<(), Self::Error> where F: FnOnce(&mut Self) -> Result<(), Self::Error>, { @@ -100,12 +75,7 @@ pub trait Encoder { } #[inline] - fn emit_struct_field( - &mut self, - _f_name: &str, - _f_idx: usize, - f: F, - ) -> Result<(), Self::Error> + fn emit_struct_field(&mut self, _f_name: &str, _first: bool, f: F) -> Result<(), Self::Error> where F: FnOnce(&mut Self) -> Result<(), Self::Error>, { @@ -128,26 +98,12 @@ pub trait Encoder { f(self) } - fn emit_tuple_struct(&mut self, _name: &str, len: usize, f: F) -> Result<(), Self::Error> - where - F: FnOnce(&mut Self) -> Result<(), Self::Error>, - { - self.emit_tuple(len, f) - } - - fn emit_tuple_struct_arg(&mut self, f_idx: usize, f: F) -> Result<(), Self::Error> - where - F: FnOnce(&mut Self) -> Result<(), Self::Error>, - { - self.emit_tuple_arg(f_idx, f) - } - // Specialized types: fn emit_option(&mut self, f: F) -> Result<(), Self::Error> where F: FnOnce(&mut Self) -> Result<(), Self::Error>, { - self.emit_enum("Option", f) + self.emit_enum(f) } #[inline] @@ -195,7 +151,7 @@ pub trait Encoder { } #[inline] - fn emit_map_elt_val(&mut self, _idx: usize, f: F) -> Result<(), Self::Error> + fn emit_map_elt_val(&mut self, f: F) -> Result<(), Self::Error> where F: FnOnce(&mut Self) -> Result<(), Self::Error>, { @@ -229,7 +185,7 @@ pub trait Decoder { // Compound types: #[inline] - fn read_enum(&mut self, _name: &str, f: F) -> Result + fn read_enum(&mut self, f: F) -> Result where F: FnOnce(&mut Self) -> Result, { @@ -246,34 +202,15 @@ pub trait Decoder { } #[inline] - fn read_enum_variant_arg(&mut self, _a_idx: usize, f: F) -> Result + fn read_enum_variant_arg(&mut self, f: F) -> Result where F: FnOnce(&mut Self) -> Result, { f(self) } - fn read_enum_struct_variant(&mut self, names: &[&str], f: F) -> Result - where - F: FnMut(&mut Self, usize) -> Result, - { - self.read_enum_variant(names, f) - } - - fn read_enum_struct_variant_field( - &mut self, - _f_name: &str, - f_idx: usize, - f: F, - ) -> Result - where - F: FnOnce(&mut Self) -> Result, - { - self.read_enum_variant_arg(f_idx, f) - } - #[inline] - fn read_struct(&mut self, _s_name: &str, _len: usize, f: F) -> Result + fn read_struct(&mut self, f: F) -> Result where F: FnOnce(&mut Self) -> Result, { @@ -281,12 +218,7 @@ pub trait Decoder { } #[inline] - fn read_struct_field( - &mut self, - _f_name: &str, - _f_idx: usize, - f: F, - ) -> Result + fn read_struct_field(&mut self, _f_name: &str, f: F) -> Result where F: FnOnce(&mut Self) -> Result, { @@ -302,33 +234,19 @@ pub trait Decoder { } #[inline] - fn read_tuple_arg(&mut self, _a_idx: usize, f: F) -> Result + fn read_tuple_arg(&mut self, f: F) -> Result where F: FnOnce(&mut Self) -> Result, { f(self) } - fn read_tuple_struct(&mut self, _s_name: &str, len: usize, f: F) -> Result - where - F: FnOnce(&mut Self) -> Result, - { - self.read_tuple(len, f) - } - - fn read_tuple_struct_arg(&mut self, a_idx: usize, f: F) -> Result - where - F: FnOnce(&mut Self) -> Result, - { - self.read_tuple_arg(a_idx, f) - } - // Specialized types: fn read_option(&mut self, mut f: F) -> Result where F: FnMut(&mut Self, bool) -> Result, { - self.read_enum("Option", move |this| { + self.read_enum(move |this| { this.read_enum_variant(&["None", "Some"], move |this, idx| match idx { 0 => f(this, false), 1 => f(this, true), @@ -346,7 +264,7 @@ pub trait Decoder { } #[inline] - fn read_seq_elt(&mut self, _idx: usize, f: F) -> Result + fn read_seq_elt(&mut self, f: F) -> Result where F: FnOnce(&mut Self) -> Result, { @@ -362,7 +280,7 @@ pub trait Decoder { } #[inline] - fn read_map_elt_key(&mut self, _idx: usize, f: F) -> Result + fn read_map_elt_key(&mut self, f: F) -> Result where F: FnOnce(&mut Self) -> Result, { @@ -370,7 +288,7 @@ pub trait Decoder { } #[inline] - fn read_map_elt_val(&mut self, _idx: usize, f: F) -> Result + fn read_map_elt_val(&mut self, f: F) -> Result where F: FnOnce(&mut Self) -> Result, { @@ -550,8 +468,8 @@ impl> Decodable for Vec { default fn decode(d: &mut D) -> Result, D::Error> { d.read_seq(|d, len| { let mut v = Vec::with_capacity(len); - for i in 0..len { - v.push(d.read_seq_elt(i, |d| Decodable::decode(d))?); + for _ in 0..len { + v.push(d.read_seq_elt(|d| Decodable::decode(d))?); } Ok(v) }) @@ -571,7 +489,7 @@ impl Decodable for [u8; N] { assert!(len == N); let mut v = [0u8; N]; for i in 0..len { - v[i] = d.read_seq_elt(i, |d| Decodable::decode(d))?; + v[i] = d.read_seq_elt(|d| Decodable::decode(d))?; } Ok(v) }) @@ -615,12 +533,12 @@ impl> Decodable for Option { impl, T2: Encodable> Encodable for Result { fn encode(&self, s: &mut S) -> Result<(), S::Error> { - s.emit_enum("Result", |s| match *self { + s.emit_enum(|s| match *self { Ok(ref v) => { - s.emit_enum_variant("Ok", 0, 1, |s| s.emit_enum_variant_arg(0, |s| v.encode(s))) + s.emit_enum_variant("Ok", 0, 1, |s| s.emit_enum_variant_arg(true, |s| v.encode(s))) } Err(ref v) => { - s.emit_enum_variant("Err", 1, 1, |s| s.emit_enum_variant_arg(0, |s| v.encode(s))) + s.emit_enum_variant("Err", 1, 1, |s| s.emit_enum_variant_arg(true, |s| v.encode(s))) } }) } @@ -628,10 +546,10 @@ impl, T2: Encodable> Encodable for Result, T2: Decodable> Decodable for Result { fn decode(d: &mut D) -> Result, D::Error> { - d.read_enum("Result", |d| { + d.read_enum(|d| { d.read_enum_variant(&["Ok", "Err"], |d, disr| match disr { - 0 => Ok(Ok(d.read_enum_variant_arg(0, |d| T1::decode(d))?)), - 1 => Ok(Err(d.read_enum_variant_arg(0, |d| T2::decode(d))?)), + 0 => Ok(Ok(d.read_enum_variant_arg(|d| T1::decode(d))?)), + 1 => Ok(Err(d.read_enum_variant_arg(|d| T2::decode(d))?)), _ => { panic!( "Encountered invalid discriminant while \ @@ -668,8 +586,7 @@ macro_rules! tuple { fn decode(d: &mut D) -> Result<($($name,)+), D::Error> { let len: usize = count!($($name)+); d.read_tuple(len, |d| { - let mut i = 0; - let ret = ($(d.read_tuple_arg({ i+=1; i-1 }, |d| -> Result<$name, D::Error> { + let ret = ($(d.read_tuple_arg(|d| -> Result<$name, D::Error> { Decodable::decode(d) })?,)+); Ok(ret) diff --git a/compiler/rustc_session/src/cgu_reuse_tracker.rs b/compiler/rustc_session/src/cgu_reuse_tracker.rs index 0eec12aa03..a9e1475433 100644 --- a/compiler/rustc_session/src/cgu_reuse_tracker.rs +++ b/compiler/rustc_session/src/cgu_reuse_tracker.rs @@ -112,7 +112,7 @@ impl CguReuseTracker { not recorded", cgu_user_name, cgu_name ); - diag.span_fatal(error_span.0, &msg).raise(); + diag.span_fatal(error_span.0, &msg) } } } diff --git a/compiler/rustc_session/src/config.rs b/compiler/rustc_session/src/config.rs index ad2b36691b..2b547f8be9 100644 --- a/compiler/rustc_session/src/config.rs +++ b/compiler/rustc_session/src/config.rs @@ -5,7 +5,7 @@ pub use crate::options::*; use crate::lint; use crate::search_paths::SearchPath; -use crate::utils::{CanonicalizedPath, NativeLibKind}; +use crate::utils::{CanonicalizedPath, NativeLib, NativeLibKind}; use crate::{early_error, early_warn, Session}; use rustc_data_structures::fx::FxHashSet; @@ -677,6 +677,7 @@ impl Default for Options { optimize: OptLevel::No, debuginfo: DebugInfo::None, lint_opts: Vec::new(), + force_warns: Vec::new(), lint_cap: None, describe_lints: false, output_types: OutputTypes(BTreeMap::new()), @@ -685,10 +686,10 @@ impl Default for Options { target_triple: TargetTriple::from_triple(host_triple()), test: false, incremental: None, - debugging_opts: basic_debugging_options(), + debugging_opts: Default::default(), prints: Vec::new(), borrowck_mode: BorrowckMode::Migrate, - cg: basic_codegen_options(), + cg: Default::default(), error_format: ErrorOutputType::default(), externs: Externs(BTreeMap::new()), extern_dep_specs: ExternDepSpecs(BTreeMap::new()), @@ -816,7 +817,7 @@ fn default_configuration(sess: &Session) -> CrateConfig { ret.reserve(6); // the minimum number of insertions // Target bindings. ret.insert((sym::target_os, Some(Symbol::intern(os)))); - if let Some(ref fam) = sess.target.os_family { + for fam in &sess.target.families { ret.insert((sym::target_family, Some(Symbol::intern(fam)))); if fam == "windows" { ret.insert((sym::windows, None)); @@ -893,8 +894,13 @@ pub fn build_configuration(sess: &Session, mut user_cfg: CrateConfig) -> CrateCo user_cfg } -pub(super) fn build_target_config(opts: &Options, target_override: Option) -> Target { - let target_result = target_override.map_or_else(|| Target::search(&opts.target_triple), Ok); +pub(super) fn build_target_config( + opts: &Options, + target_override: Option, + sysroot: &PathBuf, +) -> Target { + let target_result = + target_override.map_or_else(|| Target::search(&opts.target_triple, sysroot), Ok); let target = target_result.unwrap_or_else(|e| { early_error( opts.error_format, @@ -1027,8 +1033,11 @@ pub fn rustc_short_optgroups() -> Vec { "", "Link the generated crate(s) to the specified native library NAME. The optional KIND can be one of - static, framework, or dylib (the default).", - "[KIND=]NAME", + static, framework, or dylib (the default). + Optional comma separated MODIFIERS (bundle|verbatim|whole-archive|as-needed) + may be specified each with a prefix of either '+' to + enable or '-' to disable.", + "[KIND[:MODIFIERS]=]NAME[:RENAME]", ), make_crate_type_option(), opt::opt_s("", "crate-name", "Specify the name of the crate being built", "NAME"), @@ -1084,6 +1093,13 @@ pub fn rustc_short_optgroups() -> Vec { level", "LEVEL", ), + opt::multi_s( + "", + "force-warns", + "Specifiy lints that should warn even if \ + they are allowed somewhere else", + "LINT", + ), opt::multi_s("C", "codegen", "Set a codegen option", "OPT[=VALUE]"), opt::flag_s("V", "version", "Print version info and exit"), opt::flag_s("v", "verbose", "Use verbose output"), @@ -1148,7 +1164,8 @@ pub fn rustc_optgroups() -> Vec { pub fn get_cmd_lint_options( matches: &getopts::Matches, error_format: ErrorOutputType, -) -> (Vec<(String, lint::Level)>, bool, Option) { + debugging_opts: &DebuggingOptions, +) -> (Vec<(String, lint::Level)>, bool, Option, Vec) { let mut lint_opts_with_position = vec![]; let mut describe_lints = false; @@ -1181,7 +1198,18 @@ pub fn get_cmd_lint_options( lint::Level::from_str(&cap) .unwrap_or_else(|| early_error(error_format, &format!("unknown lint level: `{}`", cap))) }); - (lint_opts, describe_lints, lint_cap) + + if !debugging_opts.unstable_options && matches.opt_present("force-warns") { + early_error( + error_format, + "the `-Z unstable-options` flag must also be passed to enable \ + the flag `--force-warns=lints`", + ); + } + + let force_warns = matches.opt_strs("force-warns"); + + (lint_opts, describe_lints, lint_cap, force_warns) } /// Parses the `--color` flag. @@ -1499,7 +1527,10 @@ fn collect_print_requests( prints } -fn parse_target_triple(matches: &getopts::Matches, error_format: ErrorOutputType) -> TargetTriple { +pub fn parse_target_triple( + matches: &getopts::Matches, + error_format: ErrorOutputType, +) -> TargetTriple { match matches.opt_str("target") { Some(target) if target.ends_with(".json") => { let path = Path::new(&target); @@ -1591,52 +1622,127 @@ fn select_debuginfo( } } -fn parse_libs( - matches: &getopts::Matches, +fn parse_native_lib_kind(kind: &str, error_format: ErrorOutputType) -> NativeLibKind { + match kind { + "dylib" => NativeLibKind::Dylib { as_needed: None }, + "framework" => NativeLibKind::Framework { as_needed: None }, + "static" => NativeLibKind::Static { bundle: None, whole_archive: None }, + "static-nobundle" => { + early_warn( + error_format, + "library kind `static-nobundle` has been superseded by specifying \ + `-bundle` on library kind `static`. Try `static:-bundle`", + ); + NativeLibKind::Static { bundle: Some(false), whole_archive: None } + } + s => early_error( + error_format, + &format!("unknown library kind `{}`, expected one of dylib, framework, or static", s), + ), + } +} + +fn parse_native_lib_modifiers( + is_nightly: bool, + mut kind: NativeLibKind, + modifiers: &str, error_format: ErrorOutputType, -) -> Vec<(String, Option, NativeLibKind)> { +) -> (NativeLibKind, Option) { + let mut verbatim = None; + for modifier in modifiers.split(',') { + let (modifier, value) = match modifier.strip_prefix(&['+', '-'][..]) { + Some(m) => (m, modifier.starts_with('+')), + None => early_error( + error_format, + "invalid linking modifier syntax, expected '+' or '-' prefix \ + before one of: bundle, verbatim, whole-archive, as-needed", + ), + }; + + if !is_nightly { + early_error( + error_format, + "linking modifiers are currently unstable and only accepted on \ + the nightly compiler", + ); + } + + match (modifier, &mut kind) { + ("bundle", NativeLibKind::Static { bundle, .. }) => { + *bundle = Some(value); + } + ("bundle", _) => early_error( + error_format, + "bundle linking modifier is only compatible with \ + `static` linking kind", + ), + + ("verbatim", _) => verbatim = Some(value), + + ("whole-archive", NativeLibKind::Static { whole_archive, .. }) => { + *whole_archive = Some(value); + } + ("whole-archive", _) => early_error( + error_format, + "whole-archive linking modifier is only compatible with \ + `static` linking kind", + ), + + ("as-needed", NativeLibKind::Dylib { as_needed }) + | ("as-needed", NativeLibKind::Framework { as_needed }) => { + *as_needed = Some(value); + } + ("as-needed", _) => early_error( + error_format, + "as-needed linking modifier is only compatible with \ + `dylib` and `framework` linking kinds", + ), + + _ => early_error( + error_format, + &format!( + "unrecognized linking modifier `{}`, expected one \ + of: bundle, verbatim, whole-archive, as-needed", + modifier + ), + ), + } + } + + (kind, verbatim) +} + +fn parse_libs(matches: &getopts::Matches, error_format: ErrorOutputType) -> Vec { + let is_nightly = nightly_options::match_is_nightly_build(matches); matches .opt_strs("l") .into_iter() .map(|s| { - // Parse string of the form "[KIND=]lib[:new_name]", - // where KIND is one of "dylib", "framework", "static". - let (name, kind) = match s.split_once('=') { - None => (s, NativeLibKind::Unspecified), + // Parse string of the form "[KIND[:MODIFIERS]=]lib[:new_name]", + // where KIND is one of "dylib", "framework", "static" and + // where MODIFIERS are a comma separated list of supported modifiers + // (bundle, verbatim, whole-archive, as-needed). Each modifier is prefixed + // with either + or - to indicate whether it is enabled or disabled. + // The last value specified for a given modifier wins. + let (name, kind, verbatim) = match s.split_once('=') { + None => (s, NativeLibKind::Unspecified, None), Some((kind, name)) => { - let kind = match kind { - "dylib" => NativeLibKind::Dylib, - "framework" => NativeLibKind::Framework, - "static" => NativeLibKind::StaticBundle, - "static-nobundle" => NativeLibKind::StaticNoBundle, - s => { - early_error( - error_format, - &format!( - "unknown library kind `{}`, expected \ - one of dylib, framework, or static", - s - ), - ); + let (kind, verbatim) = match kind.split_once(':') { + None => (parse_native_lib_kind(kind, error_format), None), + Some((kind, modifiers)) => { + let kind = parse_native_lib_kind(kind, error_format); + parse_native_lib_modifiers(is_nightly, kind, modifiers, error_format) } }; - (name.to_string(), kind) + (name.to_string(), kind, verbatim) } }; - if kind == NativeLibKind::StaticNoBundle - && !nightly_options::match_is_nightly_build(matches) - { - early_error( - error_format, - "the library kind 'static-nobundle' is only \ - accepted on the nightly compiler", - ); - } + let (name, new_name) = match name.split_once(':') { None => (name, None), Some((name, new_name)) => (name.to_string(), Some(new_name.to_owned())), }; - (name, new_name, kind) + NativeLib { name, new_name, kind, verbatim } }) .collect() } @@ -1840,9 +1946,10 @@ pub fn build_session_options(matches: &getopts::Matches) -> Options { let crate_types = parse_crate_types_from_list(unparsed_crate_types) .unwrap_or_else(|e| early_error(error_format, &e[..])); - let (lint_opts, describe_lints, lint_cap) = get_cmd_lint_options(matches, error_format); + let mut debugging_opts = DebuggingOptions::build(matches, error_format); + let (lint_opts, describe_lints, lint_cap, force_warns) = + get_cmd_lint_options(matches, error_format, &debugging_opts); - let mut debugging_opts = build_debugging_options(matches, error_format); check_debug_option_stability(&debugging_opts, error_format, json_rendered); if !debugging_opts.unstable_options && json_unused_externs { @@ -1855,7 +1962,7 @@ pub fn build_session_options(matches: &getopts::Matches) -> Options { let output_types = parse_output_types(&debugging_opts, matches, error_format); - let mut cg = build_codegen_options(matches, error_format); + let mut cg = CodegenOptions::build(matches, error_format); let (disable_thinlto, mut codegen_units) = should_override_cgus_and_disable_thinlto( &output_types, matches, @@ -1865,12 +1972,7 @@ pub fn build_session_options(matches: &getopts::Matches) -> Options { check_thread_count(&debugging_opts, error_format); - let incremental = - if std::env::var_os("RUSTC_FORCE_INCREMENTAL").map(|v| v == "1").unwrap_or(false) { - cg.incremental.as_ref().map(PathBuf::from) - } else { - None - }; + let incremental = cg.incremental.as_ref().map(PathBuf::from); if debugging_opts.profile && incremental.is_some() { early_error( @@ -2019,6 +2121,7 @@ pub fn build_session_options(matches: &getopts::Matches) -> Options { optimize: opt_level, debuginfo, lint_opts, + force_warns, lint_cap, describe_lints, output_types, @@ -2314,6 +2417,7 @@ impl PpMode { /// we have an opt-in scheme here, so one is hopefully forced to think about /// how the hash should be calculated when adding a new command-line argument. crate mod dep_tracking { + use super::LdImpl; use super::{ CFGuard, CrateType, DebugInfo, ErrorOutputType, InstrumentCoverage, LinkerPluginLto, LtoCli, OptLevel, OutputTypes, Passes, SourceFileHashAlgorithm, SwitchWithOptPath, @@ -2321,7 +2425,7 @@ crate mod dep_tracking { }; use crate::lint; use crate::options::WasiExecModel; - use crate::utils::NativeLibKind; + use crate::utils::{NativeLib, NativeLibKind}; use rustc_feature::UnstableFeatures; use rustc_span::edition::Edition; use rustc_target::spec::{CodeModel, MergeFunctions, PanicStrategy, RelocModel}; @@ -2337,83 +2441,65 @@ crate mod dep_tracking { } macro_rules! impl_dep_tracking_hash_via_hash { - ($t:ty) => { + ($($t:ty),+ $(,)?) => {$( impl DepTrackingHash for $t { fn hash(&self, hasher: &mut DefaultHasher, _: ErrorOutputType) { Hash::hash(self, hasher); } } - }; + )+}; } - macro_rules! impl_dep_tracking_hash_for_sortable_vec_of { - ($t:ty) => { - impl DepTrackingHash for Vec<$t> { - fn hash(&self, hasher: &mut DefaultHasher, error_format: ErrorOutputType) { - let mut elems: Vec<&$t> = self.iter().collect(); - elems.sort(); - Hash::hash(&elems.len(), hasher); - for (index, elem) in elems.iter().enumerate() { - Hash::hash(&index, hasher); - DepTrackingHash::hash(*elem, hasher, error_format); - } + impl DepTrackingHash for Option { + fn hash(&self, hasher: &mut DefaultHasher, error_format: ErrorOutputType) { + match self { + Some(x) => { + Hash::hash(&1, hasher); + DepTrackingHash::hash(x, hasher, error_format); } + None => Hash::hash(&0, hasher), } - }; + } } - impl_dep_tracking_hash_via_hash!(bool); - impl_dep_tracking_hash_via_hash!(usize); - impl_dep_tracking_hash_via_hash!(u64); - impl_dep_tracking_hash_via_hash!(String); - impl_dep_tracking_hash_via_hash!(PathBuf); - impl_dep_tracking_hash_via_hash!(lint::Level); - impl_dep_tracking_hash_via_hash!(Option); - impl_dep_tracking_hash_via_hash!(Option); - impl_dep_tracking_hash_via_hash!(Option); - impl_dep_tracking_hash_via_hash!(Option); - impl_dep_tracking_hash_via_hash!(Option); - impl_dep_tracking_hash_via_hash!(Option<(String, u64)>); - impl_dep_tracking_hash_via_hash!(Option>); - impl_dep_tracking_hash_via_hash!(Option); - impl_dep_tracking_hash_via_hash!(Option); - impl_dep_tracking_hash_via_hash!(Option); - impl_dep_tracking_hash_via_hash!(Option); - impl_dep_tracking_hash_via_hash!(Option); - impl_dep_tracking_hash_via_hash!(Option); - impl_dep_tracking_hash_via_hash!(Option); - impl_dep_tracking_hash_via_hash!(Option); - impl_dep_tracking_hash_via_hash!(Option); - impl_dep_tracking_hash_via_hash!(Option); - impl_dep_tracking_hash_via_hash!(CrateType); - impl_dep_tracking_hash_via_hash!(MergeFunctions); - impl_dep_tracking_hash_via_hash!(PanicStrategy); - impl_dep_tracking_hash_via_hash!(RelroLevel); - impl_dep_tracking_hash_via_hash!(Passes); - impl_dep_tracking_hash_via_hash!(OptLevel); - impl_dep_tracking_hash_via_hash!(LtoCli); - impl_dep_tracking_hash_via_hash!(DebugInfo); - impl_dep_tracking_hash_via_hash!(UnstableFeatures); - impl_dep_tracking_hash_via_hash!(OutputTypes); - impl_dep_tracking_hash_via_hash!(NativeLibKind); - impl_dep_tracking_hash_via_hash!(SanitizerSet); - impl_dep_tracking_hash_via_hash!(CFGuard); - impl_dep_tracking_hash_via_hash!(TargetTriple); - impl_dep_tracking_hash_via_hash!(Edition); - impl_dep_tracking_hash_via_hash!(LinkerPluginLto); - impl_dep_tracking_hash_via_hash!(Option); - impl_dep_tracking_hash_via_hash!(SwitchWithOptPath); - impl_dep_tracking_hash_via_hash!(Option); - impl_dep_tracking_hash_via_hash!(Option); - impl_dep_tracking_hash_via_hash!(TrimmedDefPaths); - - impl_dep_tracking_hash_for_sortable_vec_of!(String); - impl_dep_tracking_hash_for_sortable_vec_of!(PathBuf); - impl_dep_tracking_hash_for_sortable_vec_of!((PathBuf, PathBuf)); - impl_dep_tracking_hash_for_sortable_vec_of!(CrateType); - impl_dep_tracking_hash_for_sortable_vec_of!((String, lint::Level)); - impl_dep_tracking_hash_for_sortable_vec_of!((String, Option, NativeLibKind)); - impl_dep_tracking_hash_for_sortable_vec_of!((String, u64)); + impl_dep_tracking_hash_via_hash!( + bool, + usize, + NonZeroUsize, + u64, + String, + PathBuf, + lint::Level, + WasiExecModel, + u32, + RelocModel, + CodeModel, + TlsModel, + InstrumentCoverage, + CrateType, + MergeFunctions, + PanicStrategy, + RelroLevel, + Passes, + OptLevel, + LtoCli, + DebugInfo, + UnstableFeatures, + OutputTypes, + NativeLib, + NativeLibKind, + SanitizerSet, + CFGuard, + TargetTriple, + Edition, + LinkerPluginLto, + SplitDebuginfo, + SwitchWithOptPath, + SymbolManglingVersion, + SourceFileHashAlgorithm, + TrimmedDefPaths, + Option, + ); impl DepTrackingHash for (T1, T2) where @@ -2444,6 +2530,16 @@ crate mod dep_tracking { } } + impl DepTrackingHash for Vec { + fn hash(&self, hasher: &mut DefaultHasher, error_format: ErrorOutputType) { + Hash::hash(&self.len(), hasher); + for (index, elem) in self.iter().enumerate() { + Hash::hash(&index, hasher); + DepTrackingHash::hash(elem, hasher, error_format); + } + } + } + // This is a stable hash because BTreeMap is a sorted container crate fn stable_hash( sub_hashes: BTreeMap<&'static str, &dyn DepTrackingHash>, diff --git a/compiler/rustc_session/src/filesearch.rs b/compiler/rustc_session/src/filesearch.rs index 2df326628e..6fe6a555f1 100644 --- a/compiler/rustc_session/src/filesearch.rs +++ b/compiler/rustc_session/src/filesearch.rs @@ -1,6 +1,5 @@ pub use self::FileMatch::*; -use std::borrow::Cow; use std::env; use std::fs; use std::path::{Path, PathBuf}; @@ -91,26 +90,21 @@ impl<'a> FileSearch<'a> { // Returns a list of directories where target-specific tool binaries are located. pub fn get_tools_search_paths(&self, self_contained: bool) -> Vec { - let mut p = PathBuf::from(self.sysroot); - p.push(find_libdir(self.sysroot).as_ref()); - p.push(RUST_LIB_DIR); - p.push(&self.triple); - p.push("bin"); + let rustlib_path = rustc_target::target_rustlib_path(self.sysroot, &self.triple); + let p = std::array::IntoIter::new([ + Path::new(&self.sysroot), + Path::new(&rustlib_path), + Path::new("bin"), + ]) + .collect::(); if self_contained { vec![p.clone(), p.join("self-contained")] } else { vec![p] } } } -pub fn relative_target_lib_path(sysroot: &Path, target_triple: &str) -> PathBuf { - let mut p = PathBuf::from(find_libdir(sysroot).as_ref()); - assert!(p.is_relative()); - p.push(RUST_LIB_DIR); - p.push(target_triple); - p.push("lib"); - p -} - pub fn make_target_lib_path(sysroot: &Path, target_triple: &str) -> PathBuf { - sysroot.join(&relative_target_lib_path(sysroot, target_triple)) + let rustlib_path = rustc_target::target_rustlib_path(sysroot, target_triple); + std::array::IntoIter::new([sysroot, Path::new(&rustlib_path), Path::new("lib")]) + .collect::() } // This function checks if sysroot is found using env::args().next(), and if it @@ -157,11 +151,13 @@ pub fn get_or_default_sysroot() -> PathBuf { return None; } + // Pop off `bin/rustc`, obtaining the suspected sysroot. p.pop(); p.pop(); - let mut libdir = PathBuf::from(&p); - libdir.push(find_libdir(&p).as_ref()); - if libdir.exists() { Some(p) } else { None } + // Look for the target rustlib directory in the suspected sysroot. + let mut rustlib_path = rustc_target::target_rustlib_path(&p, "dummy"); + rustlib_path.pop(); // pop off the dummy target. + if rustlib_path.exists() { Some(p) } else { None } } None => None, } @@ -171,37 +167,3 @@ pub fn get_or_default_sysroot() -> PathBuf { // use env::current_exe() to imply sysroot. from_env_args_next().unwrap_or_else(from_current_exe) } - -// The name of the directory rustc expects libraries to be located. -fn find_libdir(sysroot: &Path) -> Cow<'static, str> { - // FIXME: This is a quick hack to make the rustc binary able to locate - // Rust libraries in Linux environments where libraries might be installed - // to lib64/lib32. This would be more foolproof by basing the sysroot off - // of the directory where `librustc_driver` is located, rather than - // where the rustc binary is. - // If --libdir is set during configuration to the value other than - // "lib" (i.e., non-default), this value is used (see issue #16552). - - #[cfg(target_pointer_width = "64")] - const PRIMARY_LIB_DIR: &str = "lib64"; - - #[cfg(target_pointer_width = "32")] - const PRIMARY_LIB_DIR: &str = "lib32"; - - const SECONDARY_LIB_DIR: &str = "lib"; - - match option_env!("CFG_LIBDIR_RELATIVE") { - None | Some("lib") => { - if sysroot.join(PRIMARY_LIB_DIR).join(RUST_LIB_DIR).exists() { - PRIMARY_LIB_DIR.into() - } else { - SECONDARY_LIB_DIR.into() - } - } - Some(libdir) => libdir.into(), - } -} - -// The name of rustc's own place to organize libraries. -// Used to be "rustc", now the default is "rustlib" -const RUST_LIB_DIR: &str = "rustlib"; diff --git a/compiler/rustc_session/src/lib.rs b/compiler/rustc_session/src/lib.rs index 7971f7ef9e..8d00a9a959 100644 --- a/compiler/rustc_session/src/lib.rs +++ b/compiler/rustc_session/src/lib.rs @@ -1,6 +1,5 @@ #![feature(crate_visibility_modifier)] #![feature(once_cell)] -#![cfg_attr(bootstrap, feature(or_patterns))] #![recursion_limit = "256"] #[macro_use] diff --git a/compiler/rustc_session/src/options.rs b/compiler/rustc_session/src/options.rs index 1c2a7f7716..1946bfd78c 100644 --- a/compiler/rustc_session/src/options.rs +++ b/compiler/rustc_session/src/options.rs @@ -3,7 +3,7 @@ use crate::config::*; use crate::early_error; use crate::lint; use crate::search_paths::SearchPath; -use crate::utils::NativeLibKind; +use crate::utils::NativeLib; use rustc_target::spec::{CodeModel, LinkerFlavor, MergeFunctions, PanicStrategy, SanitizerSet}; use rustc_target::spec::{RelocModel, RelroLevel, SplitDebuginfo, TargetTriple, TlsModel}; @@ -130,10 +130,11 @@ top_level_options!( debuginfo: DebugInfo [TRACKED], lint_opts: Vec<(String, lint::Level)> [TRACKED], lint_cap: Option [TRACKED], + force_warns: Vec [TRACKED], describe_lints: bool [UNTRACKED], output_types: OutputTypes [TRACKED], search_paths: Vec [UNTRACKED], - libs: Vec<(String, Option, NativeLibKind)> [TRACKED], + libs: Vec [TRACKED], maybe_sysroot: Option [UNTRACKED], target_triple: TargetTriple [TRACKED], @@ -210,9 +211,7 @@ top_level_options!( /// generated code to parse an option into its respective field in the struct. There are a few /// hand-written parsers for parsing specific types of values in this module. macro_rules! options { - ($struct_name:ident, $setter_name:ident, $defaultfn:ident, - $buildfn:ident, $prefix:expr, $outputname:expr, - $stat:ident, $mod_desc:ident, $mod_set:ident, + ($struct_name:ident, $stat:ident, $prefix:expr, $outputname:expr, $($( #[$attr:meta] )* $opt:ident : $t:ty = ( $init:expr, $parse:ident, @@ -223,50 +222,20 @@ macro_rules! options { #[derive(Clone)] pub struct $struct_name { $(pub $opt: $t),* } - pub fn $defaultfn() -> $struct_name { - $struct_name { $( $( #[$attr] )* $opt: $init),* } - } - - pub fn $buildfn(matches: &getopts::Matches, error_format: ErrorOutputType) -> $struct_name - { - let mut op = $defaultfn(); - for option in matches.opt_strs($prefix) { - let (key, value) = match option.split_once('=') { - None => (option, None), - Some((k, v)) => (k.to_string(), Some(v)), - }; - let option_to_lookup = key.replace("-", "_"); - let mut found = false; - for &(candidate, setter, type_desc, _) in $stat { - if option_to_lookup != candidate { continue } - if !setter(&mut op, value) { - match value { - None => { - early_error(error_format, &format!("{0} option `{1}` requires \ - {2} ({3} {1}=)", - $outputname, key, - type_desc, $prefix)) - } - Some(value) => { - early_error(error_format, &format!("incorrect value `{}` for {} \ - option `{}` - {} was expected", - value, $outputname, - key, type_desc)) - } - } - } - found = true; - break; - } - if !found { - early_error(error_format, &format!("unknown {} option: `{}`", - $outputname, key)); - } + impl Default for $struct_name { + fn default() -> $struct_name { + $struct_name { $( $( #[$attr] )* $opt: $init),* } } - return op; } impl $struct_name { + pub fn build( + matches: &getopts::Matches, + error_format: ErrorOutputType, + ) -> $struct_name { + build_options(matches, $stat, $prefix, $outputname, error_format) + } + fn dep_tracking_hash(&self, _for_crate_hash: bool, error_format: ErrorOutputType) -> u64 { let mut sub_hashes = BTreeMap::new(); $({ @@ -284,548 +253,631 @@ macro_rules! options { } } - pub type $setter_name = fn(&mut $struct_name, v: Option<&str>) -> bool; - pub const $stat: &[(&str, $setter_name, &str, &str)] = - &[ $( (stringify!($opt), $mod_set::$opt, $mod_desc::$parse, $desc) ),* ]; - - #[allow(non_upper_case_globals, dead_code)] - mod $mod_desc { - pub const parse_no_flag: &str = "no value"; - pub const parse_bool: &str = "one of: `y`, `yes`, `on`, `n`, `no`, or `off`"; - pub const parse_opt_bool: &str = parse_bool; - pub const parse_string: &str = "a string"; - pub const parse_opt_string: &str = parse_string; - pub const parse_string_push: &str = parse_string; - pub const parse_opt_pathbuf: &str = "a path"; - pub const parse_pathbuf_push: &str = parse_opt_pathbuf; - pub const parse_list: &str = "a space-separated list of strings"; - pub const parse_opt_list: &str = parse_list; - pub const parse_opt_comma_list: &str = "a comma-separated list of strings"; - pub const parse_number: &str = "a number"; - pub const parse_opt_number: &str = parse_number; - pub const parse_threads: &str = parse_number; - pub const parse_passes: &str = "a space-separated list of passes, or `all`"; - pub const parse_panic_strategy: &str = "either `unwind` or `abort`"; - pub const parse_relro_level: &str = "one of: `full`, `partial`, or `off`"; - pub const parse_sanitizers: &str = "comma separated list of sanitizers: `address`, `hwaddress`, `leak`, `memory` or `thread`"; - pub const parse_sanitizer_memory_track_origins: &str = "0, 1, or 2"; - pub const parse_cfguard: &str = - "either a boolean (`yes`, `no`, `on`, `off`, etc), `checks`, or `nochecks`"; - pub const parse_strip: &str = "either `none`, `debuginfo`, or `symbols`"; - pub const parse_linker_flavor: &str = ::rustc_target::spec::LinkerFlavor::one_of(); - pub const parse_optimization_fuel: &str = "crate=integer"; - pub const parse_mir_spanview: &str = "`statement` (default), `terminator`, or `block`"; - pub const parse_instrument_coverage: &str = "`all` (default), `except-unused-generics`, `except-unused-functions`, or `off`"; - pub const parse_unpretty: &str = "`string` or `string=string`"; - pub const parse_treat_err_as_bug: &str = "either no value or a number bigger than 0"; - pub const parse_lto: &str = - "either a boolean (`yes`, `no`, `on`, `off`, etc), `thin`, `fat`, or omitted"; - pub const parse_linker_plugin_lto: &str = - "either a boolean (`yes`, `no`, `on`, `off`, etc), or the path to the linker plugin"; - pub const parse_switch_with_opt_path: &str = - "an optional path to the profiling data output directory"; - pub const parse_merge_functions: &str = "one of: `disabled`, `trampolines`, or `aliases`"; - pub const parse_symbol_mangling_version: &str = "either `legacy` or `v0` (RFC 2603)"; - pub const parse_src_file_hash: &str = "either `md5` or `sha1`"; - pub const parse_relocation_model: &str = - "one of supported relocation models (`rustc --print relocation-models`)"; - pub const parse_code_model: &str = - "one of supported code models (`rustc --print code-models`)"; - pub const parse_tls_model: &str = - "one of supported TLS models (`rustc --print tls-models`)"; - pub const parse_target_feature: &str = parse_string; - pub const parse_wasi_exec_model: &str = "either `command` or `reactor`"; - pub const parse_split_debuginfo: &str = - "one of supported split-debuginfo modes (`off` or `dsymutil`)"; - } + pub const $stat: OptionDescrs<$struct_name> = + &[ $( (stringify!($opt), $opt, desc::$parse, $desc) ),* ]; - #[allow(dead_code)] - mod $mod_set { - use super::*; - use std::str::FromStr; - - // Sometimes different options need to build a common structure. - // That structure can kept in one of the options' fields, the others become dummy. - macro_rules! redirect_field { - ($cg:ident.link_arg) => { $cg.link_args }; - ($cg:ident.pre_link_arg) => { $cg.pre_link_args }; - ($cg:ident.$field:ident) => { $cg.$field }; + $( + fn $opt(cg: &mut $struct_name, v: Option<&str>) -> bool { + parse::$parse(&mut redirect_field!(cg.$opt), v) } + )* - $( - pub fn $opt(cg: &mut $struct_name, v: Option<&str>) -> bool { - $parse(&mut redirect_field!(cg.$opt), v) - } - )* - - /// This is for boolean options that don't take a value and start with - /// `no-`. This style of option is deprecated. - fn parse_no_flag(slot: &mut bool, v: Option<&str>) -> bool { - match v { - None => { *slot = true; true } - Some(_) => false, - } - } +) } - /// Use this for any boolean option that has a static default. - fn parse_bool(slot: &mut bool, v: Option<&str>) -> bool { - match v { - Some("y") | Some("yes") | Some("on") | None => { *slot = true; true } - Some("n") | Some("no") | Some("off") => { *slot = false; true } - _ => false, +// Sometimes different options need to build a common structure. +// That structure can be kept in one of the options' fields, the others become dummy. +macro_rules! redirect_field { + ($cg:ident.link_arg) => { + $cg.link_args + }; + ($cg:ident.pre_link_arg) => { + $cg.pre_link_args + }; + ($cg:ident.$field:ident) => { + $cg.$field + }; +} + +type OptionSetter = fn(&mut O, v: Option<&str>) -> bool; +type OptionDescrs = &'static [(&'static str, OptionSetter, &'static str, &'static str)]; + +fn build_options( + matches: &getopts::Matches, + descrs: OptionDescrs, + prefix: &str, + outputname: &str, + error_format: ErrorOutputType, +) -> O { + let mut op = O::default(); + for option in matches.opt_strs(prefix) { + let (key, value) = match option.split_once('=') { + None => (option, None), + Some((k, v)) => (k.to_string(), Some(v)), + }; + + let option_to_lookup = key.replace("-", "_"); + match descrs.iter().find(|(name, ..)| *name == option_to_lookup) { + Some((_, setter, type_desc, _)) => { + if !setter(&mut op, value) { + match value { + None => early_error( + error_format, + &format!( + "{0} option `{1}` requires {2} ({3} {1}=)", + outputname, key, type_desc, prefix + ), + ), + Some(value) => early_error( + error_format, + &format!( + "incorrect value `{}` for {} option `{}` - {} was expected", + value, outputname, key, type_desc + ), + ), + } + } } + None => early_error(error_format, &format!("unknown {} option: `{}`", outputname, key)), } + } + return op; +} + +#[allow(non_upper_case_globals)] +mod desc { + pub const parse_no_flag: &str = "no value"; + pub const parse_bool: &str = "one of: `y`, `yes`, `on`, `n`, `no`, or `off`"; + pub const parse_opt_bool: &str = parse_bool; + pub const parse_string: &str = "a string"; + pub const parse_opt_string: &str = parse_string; + pub const parse_string_push: &str = parse_string; + pub const parse_opt_pathbuf: &str = "a path"; + pub const parse_list: &str = "a space-separated list of strings"; + pub const parse_opt_comma_list: &str = "a comma-separated list of strings"; + pub const parse_number: &str = "a number"; + pub const parse_opt_number: &str = parse_number; + pub const parse_threads: &str = parse_number; + pub const parse_passes: &str = "a space-separated list of passes, or `all`"; + pub const parse_panic_strategy: &str = "either `unwind` or `abort`"; + pub const parse_relro_level: &str = "one of: `full`, `partial`, or `off`"; + pub const parse_sanitizers: &str = + "comma separated list of sanitizers: `address`, `hwaddress`, `leak`, `memory` or `thread`"; + pub const parse_sanitizer_memory_track_origins: &str = "0, 1, or 2"; + pub const parse_cfguard: &str = + "either a boolean (`yes`, `no`, `on`, `off`, etc), `checks`, or `nochecks`"; + pub const parse_strip: &str = "either `none`, `debuginfo`, or `symbols`"; + pub const parse_linker_flavor: &str = ::rustc_target::spec::LinkerFlavor::one_of(); + pub const parse_optimization_fuel: &str = "crate=integer"; + pub const parse_mir_spanview: &str = "`statement` (default), `terminator`, or `block`"; + pub const parse_instrument_coverage: &str = + "`all` (default), `except-unused-generics`, `except-unused-functions`, or `off`"; + pub const parse_unpretty: &str = "`string` or `string=string`"; + pub const parse_treat_err_as_bug: &str = "either no value or a number bigger than 0"; + pub const parse_lto: &str = + "either a boolean (`yes`, `no`, `on`, `off`, etc), `thin`, `fat`, or omitted"; + pub const parse_linker_plugin_lto: &str = + "either a boolean (`yes`, `no`, `on`, `off`, etc), or the path to the linker plugin"; + pub const parse_switch_with_opt_path: &str = + "an optional path to the profiling data output directory"; + pub const parse_merge_functions: &str = "one of: `disabled`, `trampolines`, or `aliases`"; + pub const parse_symbol_mangling_version: &str = "either `legacy` or `v0` (RFC 2603)"; + pub const parse_src_file_hash: &str = "either `md5` or `sha1`"; + pub const parse_relocation_model: &str = + "one of supported relocation models (`rustc --print relocation-models`)"; + pub const parse_code_model: &str = "one of supported code models (`rustc --print code-models`)"; + pub const parse_tls_model: &str = "one of supported TLS models (`rustc --print tls-models`)"; + pub const parse_target_feature: &str = parse_string; + pub const parse_wasi_exec_model: &str = "either `command` or `reactor`"; + pub const parse_split_debuginfo: &str = + "one of supported split-debuginfo modes (`off`, `packed`, or `unpacked`)"; + pub const parse_gcc_ld: &str = "one of: no value, `lld`"; +} + +mod parse { + crate use super::*; + use std::str::FromStr; - /// Use this for any boolean option that lacks a static default. (The - /// actions taken when such an option is not specified will depend on - /// other factors, such as other options, or target options.) - fn parse_opt_bool(slot: &mut Option, v: Option<&str>) -> bool { - match v { - Some("y") | Some("yes") | Some("on") | None => { *slot = Some(true); true } - Some("n") | Some("no") | Some("off") => { *slot = Some(false); true } - _ => false, + /// This is for boolean options that don't take a value and start with + /// `no-`. This style of option is deprecated. + crate fn parse_no_flag(slot: &mut bool, v: Option<&str>) -> bool { + match v { + None => { + *slot = true; + true } + Some(_) => false, } + } - /// Use this for any string option that has a static default. - fn parse_string(slot: &mut String, v: Option<&str>) -> bool { - match v { - Some(s) => { *slot = s.to_string(); true }, - None => false, + /// Use this for any boolean option that has a static default. + crate fn parse_bool(slot: &mut bool, v: Option<&str>) -> bool { + match v { + Some("y") | Some("yes") | Some("on") | None => { + *slot = true; + true + } + Some("n") | Some("no") | Some("off") => { + *slot = false; + true } + _ => false, } + } - /// Use this for any string option that lacks a static default. - fn parse_opt_string(slot: &mut Option, v: Option<&str>) -> bool { - match v { - Some(s) => { *slot = Some(s.to_string()); true }, - None => false, + /// Use this for any boolean option that lacks a static default. (The + /// actions taken when such an option is not specified will depend on + /// other factors, such as other options, or target options.) + crate fn parse_opt_bool(slot: &mut Option, v: Option<&str>) -> bool { + match v { + Some("y") | Some("yes") | Some("on") | None => { + *slot = Some(true); + true + } + Some("n") | Some("no") | Some("off") => { + *slot = Some(false); + true } + _ => false, } + } - fn parse_opt_pathbuf(slot: &mut Option, v: Option<&str>) -> bool { - match v { - Some(s) => { *slot = Some(PathBuf::from(s)); true }, - None => false, + /// Use this for any string option that has a static default. + crate fn parse_string(slot: &mut String, v: Option<&str>) -> bool { + match v { + Some(s) => { + *slot = s.to_string(); + true } + None => false, } + } - fn parse_string_push(slot: &mut Vec, v: Option<&str>) -> bool { - match v { - Some(s) => { slot.push(s.to_string()); true }, - None => false, + /// Use this for any string option that lacks a static default. + crate fn parse_opt_string(slot: &mut Option, v: Option<&str>) -> bool { + match v { + Some(s) => { + *slot = Some(s.to_string()); + true } + None => false, } + } - fn parse_pathbuf_push(slot: &mut Vec, v: Option<&str>) -> bool { - match v { - Some(s) => { slot.push(PathBuf::from(s)); true }, - None => false, + crate fn parse_opt_pathbuf(slot: &mut Option, v: Option<&str>) -> bool { + match v { + Some(s) => { + *slot = Some(PathBuf::from(s)); + true } + None => false, } + } - fn parse_list(slot: &mut Vec, v: Option<&str>) - -> bool { - match v { - Some(s) => { - slot.extend(s.split_whitespace().map(|s| s.to_string())); - true - }, - None => false, + crate fn parse_string_push(slot: &mut Vec, v: Option<&str>) -> bool { + match v { + Some(s) => { + slot.push(s.to_string()); + true } + None => false, } + } - fn parse_opt_list(slot: &mut Option>, v: Option<&str>) - -> bool { - match v { - Some(s) => { - let v = s.split_whitespace().map(|s| s.to_string()).collect(); - *slot = Some(v); - true - }, - None => false, + crate fn parse_list(slot: &mut Vec, v: Option<&str>) -> bool { + match v { + Some(s) => { + slot.extend(s.split_whitespace().map(|s| s.to_string())); + true } + None => false, } + } - fn parse_opt_comma_list(slot: &mut Option>, v: Option<&str>) - -> bool { - match v { - Some(s) => { - let v = s.split(',').map(|s| s.to_string()).collect(); - *slot = Some(v); - true - }, - None => false, + crate fn parse_opt_comma_list(slot: &mut Option>, v: Option<&str>) -> bool { + match v { + Some(s) => { + let mut v: Vec<_> = s.split(',').map(|s| s.to_string()).collect(); + v.sort_unstable(); + *slot = Some(v); + true } + None => false, } + } - fn parse_threads(slot: &mut usize, v: Option<&str>) -> bool { - match v.and_then(|s| s.parse().ok()) { - Some(0) => { *slot = ::num_cpus::get(); true }, - Some(i) => { *slot = i; true }, - None => false + crate fn parse_threads(slot: &mut usize, v: Option<&str>) -> bool { + match v.and_then(|s| s.parse().ok()) { + Some(0) => { + *slot = ::num_cpus::get(); + true + } + Some(i) => { + *slot = i; + true } + None => false, } + } - /// Use this for any numeric option that has a static default. - fn parse_number(slot: &mut T, v: Option<&str>) -> bool { - match v.and_then(|s| s.parse().ok()) { - Some(i) => { *slot = i; true }, - None => false + /// Use this for any numeric option that has a static default. + crate fn parse_number(slot: &mut T, v: Option<&str>) -> bool { + match v.and_then(|s| s.parse().ok()) { + Some(i) => { + *slot = i; + true } + None => false, } + } - /// Use this for any numeric option that lacks a static default. - fn parse_opt_number(slot: &mut Option, v: Option<&str>) -> bool { - match v { - Some(s) => { *slot = s.parse().ok(); slot.is_some() } - None => false + /// Use this for any numeric option that lacks a static default. + crate fn parse_opt_number(slot: &mut Option, v: Option<&str>) -> bool { + match v { + Some(s) => { + *slot = s.parse().ok(); + slot.is_some() } + None => false, } + } - fn parse_passes(slot: &mut Passes, v: Option<&str>) -> bool { - match v { - Some("all") => { - *slot = Passes::All; + crate fn parse_passes(slot: &mut Passes, v: Option<&str>) -> bool { + match v { + Some("all") => { + *slot = Passes::All; + true + } + v => { + let mut passes = vec![]; + if parse_list(&mut passes, v) { + *slot = Passes::Some(passes); true - } - v => { - let mut passes = vec![]; - if parse_list(&mut passes, v) { - *slot = Passes::Some(passes); - true - } else { - false - } + } else { + false } } } + } - fn parse_panic_strategy(slot: &mut Option, v: Option<&str>) -> bool { - match v { - Some("unwind") => *slot = Some(PanicStrategy::Unwind), - Some("abort") => *slot = Some(PanicStrategy::Abort), - _ => return false - } - true + crate fn parse_panic_strategy(slot: &mut Option, v: Option<&str>) -> bool { + match v { + Some("unwind") => *slot = Some(PanicStrategy::Unwind), + Some("abort") => *slot = Some(PanicStrategy::Abort), + _ => return false, } + true + } - fn parse_relro_level(slot: &mut Option, v: Option<&str>) -> bool { - match v { - Some(s) => { - match s.parse::() { - Ok(level) => *slot = Some(level), - _ => return false - } - }, - _ => return false + crate fn parse_relro_level(slot: &mut Option, v: Option<&str>) -> bool { + match v { + Some(s) => match s.parse::() { + Ok(level) => *slot = Some(level), + _ => return false, + }, + _ => return false, + } + true + } + + crate fn parse_sanitizers(slot: &mut SanitizerSet, v: Option<&str>) -> bool { + if let Some(v) = v { + for s in v.split(',') { + *slot |= match s { + "address" => SanitizerSet::ADDRESS, + "leak" => SanitizerSet::LEAK, + "memory" => SanitizerSet::MEMORY, + "thread" => SanitizerSet::THREAD, + "hwaddress" => SanitizerSet::HWADDRESS, + _ => return false, + } } true + } else { + false } + } - fn parse_sanitizers(slot: &mut SanitizerSet, v: Option<&str>) -> bool { - if let Some(v) = v { - for s in v.split(',') { - *slot |= match s { - "address" => SanitizerSet::ADDRESS, - "leak" => SanitizerSet::LEAK, - "memory" => SanitizerSet::MEMORY, - "thread" => SanitizerSet::THREAD, - "hwaddress" => SanitizerSet::HWADDRESS, - _ => return false, - } - } + crate fn parse_sanitizer_memory_track_origins(slot: &mut usize, v: Option<&str>) -> bool { + match v { + Some("2") | None => { + *slot = 2; true - } else { - false } + Some("1") => { + *slot = 1; + true + } + Some("0") => { + *slot = 0; + true + } + Some(_) => false, } + } - fn parse_sanitizer_memory_track_origins(slot: &mut usize, v: Option<&str>) -> bool { - match v { - Some("2") | None => { *slot = 2; true } - Some("1") => { *slot = 1; true } - Some("0") => { *slot = 0; true } - Some(_) => false, - } + crate fn parse_strip(slot: &mut Strip, v: Option<&str>) -> bool { + match v { + Some("none") => *slot = Strip::None, + Some("debuginfo") => *slot = Strip::Debuginfo, + Some("symbols") => *slot = Strip::Symbols, + _ => return false, } + true + } - fn parse_strip(slot: &mut Strip, v: Option<&str>) -> bool { - match v { - Some("none") => *slot = Strip::None, - Some("debuginfo") => *slot = Strip::Debuginfo, - Some("symbols") => *slot = Strip::Symbols, - _ => return false, + crate fn parse_cfguard(slot: &mut CFGuard, v: Option<&str>) -> bool { + if v.is_some() { + let mut bool_arg = None; + if parse_opt_bool(&mut bool_arg, v) { + *slot = if bool_arg.unwrap() { CFGuard::Checks } else { CFGuard::Disabled }; + return true; } - true } - fn parse_cfguard(slot: &mut CFGuard, v: Option<&str>) -> bool { - if v.is_some() { - let mut bool_arg = None; - if parse_opt_bool(&mut bool_arg, v) { - *slot = if bool_arg.unwrap() { - CFGuard::Checks - } else { - CFGuard::Disabled - }; - return true - } - } + *slot = match v { + None => CFGuard::Checks, + Some("checks") => CFGuard::Checks, + Some("nochecks") => CFGuard::NoChecks, + Some(_) => return false, + }; + true + } - *slot = match v { - None => CFGuard::Checks, - Some("checks") => CFGuard::Checks, - Some("nochecks") => CFGuard::NoChecks, - Some(_) => return false, - }; - true + crate fn parse_linker_flavor(slot: &mut Option, v: Option<&str>) -> bool { + match v.and_then(LinkerFlavor::from_str) { + Some(lf) => *slot = Some(lf), + _ => return false, } + true + } - fn parse_linker_flavor(slote: &mut Option, v: Option<&str>) -> bool { - match v.and_then(LinkerFlavor::from_str) { - Some(lf) => *slote = Some(lf), - _ => return false, + crate fn parse_optimization_fuel(slot: &mut Option<(String, u64)>, v: Option<&str>) -> bool { + match v { + None => false, + Some(s) => { + let parts = s.split('=').collect::>(); + if parts.len() != 2 { + return false; + } + let crate_name = parts[0].to_string(); + let fuel = parts[1].parse::(); + if fuel.is_err() { + return false; + } + *slot = Some((crate_name, fuel.unwrap())); + true } - true } + } - fn parse_optimization_fuel(slot: &mut Option<(String, u64)>, v: Option<&str>) -> bool { - match v { - None => false, - Some(s) => { - let parts = s.split('=').collect::>(); - if parts.len() != 2 { return false; } - let crate_name = parts[0].to_string(); - let fuel = parts[1].parse::(); - if fuel.is_err() { return false; } - *slot = Some((crate_name, fuel.unwrap())); - true - } + crate fn parse_unpretty(slot: &mut Option, v: Option<&str>) -> bool { + match v { + None => false, + Some(s) if s.split('=').count() <= 2 => { + *slot = Some(s.to_string()); + true } + _ => false, } + } - fn parse_unpretty(slot: &mut Option, v: Option<&str>) -> bool { - match v { - None => false, - Some(s) if s.split('=').count() <= 2 => { - *slot = Some(s.to_string()); - true - } - _ => false, + crate fn parse_mir_spanview(slot: &mut Option, v: Option<&str>) -> bool { + if v.is_some() { + let mut bool_arg = None; + if parse_opt_bool(&mut bool_arg, v) { + *slot = if bool_arg.unwrap() { Some(MirSpanview::Statement) } else { None }; + return true; } } - fn parse_mir_spanview(slot: &mut Option, v: Option<&str>) -> bool { - if v.is_some() { - let mut bool_arg = None; - if parse_opt_bool(&mut bool_arg, v) { - *slot = if bool_arg.unwrap() { - Some(MirSpanview::Statement) - } else { - None - }; - return true - } + let v = match v { + None => { + *slot = Some(MirSpanview::Statement); + return true; } + Some(v) => v, + }; + + *slot = Some(match v.trim_end_matches("s") { + "statement" | "stmt" => MirSpanview::Statement, + "terminator" | "term" => MirSpanview::Terminator, + "block" | "basicblock" => MirSpanview::Block, + _ => return false, + }); + true + } - let v = match v { - None => { - *slot = Some(MirSpanview::Statement); - return true; - } - Some(v) => v, - }; - - *slot = Some(match v.trim_end_matches("s") { - "statement" | "stmt" => MirSpanview::Statement, - "terminator" | "term" => MirSpanview::Terminator, - "block" | "basicblock" => MirSpanview::Block, - _ => return false, - }); - true + crate fn parse_instrument_coverage( + slot: &mut Option, + v: Option<&str>, + ) -> bool { + if v.is_some() { + let mut bool_arg = None; + if parse_opt_bool(&mut bool_arg, v) { + *slot = if bool_arg.unwrap() { Some(InstrumentCoverage::All) } else { None }; + return true; + } } - fn parse_instrument_coverage(slot: &mut Option, v: Option<&str>) -> bool { - if v.is_some() { - let mut bool_arg = None; - if parse_opt_bool(&mut bool_arg, v) { - *slot = if bool_arg.unwrap() { - Some(InstrumentCoverage::All) - } else { - None - }; - return true - } + let v = match v { + None => { + *slot = Some(InstrumentCoverage::All); + return true; } + Some(v) => v, + }; - let v = match v { - None => { - *slot = Some(InstrumentCoverage::All); - return true; - } - Some(v) => v, - }; + *slot = Some(match v { + "all" => InstrumentCoverage::All, + "except-unused-generics" | "except_unused_generics" => { + InstrumentCoverage::ExceptUnusedGenerics + } + "except-unused-functions" | "except_unused_functions" => { + InstrumentCoverage::ExceptUnusedFunctions + } + "off" | "no" | "n" | "false" | "0" => InstrumentCoverage::Off, + _ => return false, + }); + true + } - *slot = Some(match v { - "all" => InstrumentCoverage::All, - "except-unused-generics" | "except_unused_generics" => { - InstrumentCoverage::ExceptUnusedGenerics - } - "except-unused-functions" | "except_unused_functions" => { - InstrumentCoverage::ExceptUnusedFunctions - } - "off" | "no" | "n" | "false" | "0" => InstrumentCoverage::Off, - _ => return false, - }); - true + crate fn parse_treat_err_as_bug(slot: &mut Option, v: Option<&str>) -> bool { + match v { + Some(s) => { + *slot = s.parse().ok(); + slot.is_some() + } + None => { + *slot = NonZeroUsize::new(1); + true + } } + } - fn parse_treat_err_as_bug(slot: &mut Option, v: Option<&str>) -> bool { - match v { - Some(s) => { *slot = s.parse().ok(); slot.is_some() } - None => { *slot = NonZeroUsize::new(1); true } + crate fn parse_lto(slot: &mut LtoCli, v: Option<&str>) -> bool { + if v.is_some() { + let mut bool_arg = None; + if parse_opt_bool(&mut bool_arg, v) { + *slot = if bool_arg.unwrap() { LtoCli::Yes } else { LtoCli::No }; + return true; } } - fn parse_lto(slot: &mut LtoCli, v: Option<&str>) -> bool { - if v.is_some() { - let mut bool_arg = None; - if parse_opt_bool(&mut bool_arg, v) { - *slot = if bool_arg.unwrap() { - LtoCli::Yes - } else { - LtoCli::No - }; - return true - } - } + *slot = match v { + None => LtoCli::NoParam, + Some("thin") => LtoCli::Thin, + Some("fat") => LtoCli::Fat, + Some(_) => return false, + }; + true + } - *slot = match v { - None => LtoCli::NoParam, - Some("thin") => LtoCli::Thin, - Some("fat") => LtoCli::Fat, - Some(_) => return false, - }; - true + crate fn parse_linker_plugin_lto(slot: &mut LinkerPluginLto, v: Option<&str>) -> bool { + if v.is_some() { + let mut bool_arg = None; + if parse_opt_bool(&mut bool_arg, v) { + *slot = if bool_arg.unwrap() { + LinkerPluginLto::LinkerPluginAuto + } else { + LinkerPluginLto::Disabled + }; + return true; + } } - fn parse_linker_plugin_lto(slot: &mut LinkerPluginLto, v: Option<&str>) -> bool { - if v.is_some() { - let mut bool_arg = None; - if parse_opt_bool(&mut bool_arg, v) { - *slot = if bool_arg.unwrap() { - LinkerPluginLto::LinkerPluginAuto - } else { - LinkerPluginLto::Disabled - }; - return true - } - } + *slot = match v { + None => LinkerPluginLto::LinkerPluginAuto, + Some(path) => LinkerPluginLto::LinkerPlugin(PathBuf::from(path)), + }; + true + } - *slot = match v { - None => LinkerPluginLto::LinkerPluginAuto, - Some(path) => LinkerPluginLto::LinkerPlugin(PathBuf::from(path)), - }; - true - } + crate fn parse_switch_with_opt_path(slot: &mut SwitchWithOptPath, v: Option<&str>) -> bool { + *slot = match v { + None => SwitchWithOptPath::Enabled(None), + Some(path) => SwitchWithOptPath::Enabled(Some(PathBuf::from(path))), + }; + true + } - fn parse_switch_with_opt_path(slot: &mut SwitchWithOptPath, v: Option<&str>) -> bool { - *slot = match v { - None => SwitchWithOptPath::Enabled(None), - Some(path) => SwitchWithOptPath::Enabled(Some(PathBuf::from(path))), - }; - true + crate fn parse_merge_functions(slot: &mut Option, v: Option<&str>) -> bool { + match v.and_then(|s| MergeFunctions::from_str(s).ok()) { + Some(mergefunc) => *slot = Some(mergefunc), + _ => return false, } + true + } - fn parse_merge_functions(slot: &mut Option, v: Option<&str>) -> bool { - match v.and_then(|s| MergeFunctions::from_str(s).ok()) { - Some(mergefunc) => *slot = Some(mergefunc), - _ => return false, - } - true + crate fn parse_relocation_model(slot: &mut Option, v: Option<&str>) -> bool { + match v.and_then(|s| RelocModel::from_str(s).ok()) { + Some(relocation_model) => *slot = Some(relocation_model), + None if v == Some("default") => *slot = None, + _ => return false, } + true + } - fn parse_relocation_model(slot: &mut Option, v: Option<&str>) -> bool { - match v.and_then(|s| RelocModel::from_str(s).ok()) { - Some(relocation_model) => *slot = Some(relocation_model), - None if v == Some("default") => *slot = None, - _ => return false, - } - true + crate fn parse_code_model(slot: &mut Option, v: Option<&str>) -> bool { + match v.and_then(|s| CodeModel::from_str(s).ok()) { + Some(code_model) => *slot = Some(code_model), + _ => return false, } + true + } - fn parse_code_model(slot: &mut Option, v: Option<&str>) -> bool { - match v.and_then(|s| CodeModel::from_str(s).ok()) { - Some(code_model) => *slot = Some(code_model), - _ => return false, - } - true + crate fn parse_tls_model(slot: &mut Option, v: Option<&str>) -> bool { + match v.and_then(|s| TlsModel::from_str(s).ok()) { + Some(tls_model) => *slot = Some(tls_model), + _ => return false, } + true + } - fn parse_tls_model(slot: &mut Option, v: Option<&str>) -> bool { - match v.and_then(|s| TlsModel::from_str(s).ok()) { - Some(tls_model) => *slot = Some(tls_model), - _ => return false, - } - true - } + crate fn parse_symbol_mangling_version( + slot: &mut Option, + v: Option<&str>, + ) -> bool { + *slot = match v { + Some("legacy") => Some(SymbolManglingVersion::Legacy), + Some("v0") => Some(SymbolManglingVersion::V0), + _ => return false, + }; + true + } - fn parse_symbol_mangling_version( - slot: &mut Option, - v: Option<&str>, - ) -> bool { - *slot = match v { - Some("legacy") => Some(SymbolManglingVersion::Legacy), - Some("v0") => Some(SymbolManglingVersion::V0), - _ => return false, - }; - true + crate fn parse_src_file_hash( + slot: &mut Option, + v: Option<&str>, + ) -> bool { + match v.and_then(|s| SourceFileHashAlgorithm::from_str(s).ok()) { + Some(hash_kind) => *slot = Some(hash_kind), + _ => return false, } + true + } - fn parse_src_file_hash(slot: &mut Option, v: Option<&str>) -> bool { - match v.and_then(|s| SourceFileHashAlgorithm::from_str(s).ok()) { - Some(hash_kind) => *slot = Some(hash_kind), - _ => return false, + crate fn parse_target_feature(slot: &mut String, v: Option<&str>) -> bool { + match v { + Some(s) => { + if !slot.is_empty() { + slot.push_str(","); + } + slot.push_str(s); + true } - true + None => false, } + } - fn parse_target_feature(slot: &mut String, v: Option<&str>) -> bool { - match v { - Some(s) => { - if !slot.is_empty() { - slot.push_str(","); - } - slot.push_str(s); - true - } - None => false, - } + crate fn parse_wasi_exec_model(slot: &mut Option, v: Option<&str>) -> bool { + match v { + Some("command") => *slot = Some(WasiExecModel::Command), + Some("reactor") => *slot = Some(WasiExecModel::Reactor), + _ => return false, } + true + } - fn parse_wasi_exec_model(slot: &mut Option, v: Option<&str>) -> bool { - match v { - Some("command") => *slot = Some(WasiExecModel::Command), - Some("reactor") => *slot = Some(WasiExecModel::Reactor), - _ => return false, - } - true + crate fn parse_split_debuginfo(slot: &mut Option, v: Option<&str>) -> bool { + match v.and_then(|s| SplitDebuginfo::from_str(s).ok()) { + Some(e) => *slot = Some(e), + _ => return false, } + true + } - fn parse_split_debuginfo(slot: &mut Option, v: Option<&str>) -> bool { - match v.and_then(|s| SplitDebuginfo::from_str(s).ok()) { - Some(e) => *slot = Some(e), - _ => return false, - } - true + crate fn parse_gcc_ld(slot: &mut Option, v: Option<&str>) -> bool { + match v { + None => *slot = None, + Some("lld") => *slot = Some(LdImpl::Lld), + _ => return false, } + true } -) } +} -options! {CodegenOptions, CodegenSetter, basic_codegen_options, - build_codegen_options, "C", "codegen", - CG_OPTIONS, cg_type_desc, cgsetters, +options! { + CodegenOptions, CG_OPTIONS, "C", "codegen", // This list is in alphabetical order. // @@ -933,9 +985,8 @@ options! {CodegenOptions, CodegenSetter, basic_codegen_options, // - src/doc/rustc/src/codegen-options/index.md } -options! {DebuggingOptions, DebuggingSetter, basic_debugging_options, - build_debugging_options, "Z", "debugging", - DB_OPTIONS, db_type_desc, dbsetters, +options! { + DebuggingOptions, DB_OPTIONS, "Z", "debugging", // This list is in alphabetical order. // @@ -1026,6 +1077,7 @@ options! {DebuggingOptions, DebuggingSetter, basic_debugging_options, "set the optimization fuel quota for a crate"), function_sections: Option = (None, parse_opt_bool, [TRACKED], "whether each function should go in its own section"), + gcc_ld: Option = (None, parse_gcc_ld, [TRACKED], "implementation of ld used by cc"), graphviz_dark_mode: bool = (false, parse_bool, [UNTRACKED], "use dark-themed colors in graphviz output (default: no)"), graphviz_font: String = ("Courier, monospace".to_string(), parse_string, [UNTRACKED], @@ -1056,12 +1108,12 @@ options! {DebuggingOptions, DebuggingSetter, basic_debugging_options, "gather statistics about the input (default: no)"), instrument_coverage: Option = (None, parse_instrument_coverage, [TRACKED], "instrument the generated code to support LLVM source-based code coverage \ - reports (note, the compiler build config must include `profiler = true`, \ - and is mutually exclusive with `-C profile-generate`/`-C profile-use`); \ - implies `-Z symbol-mangling-version=v0`; disables/overrides some Rust \ - optimizations. Optional values are: `=all` (default coverage), \ - `=except-unused-generics`, `=except-unused-functions`, or `=off` \ - (default: instrument-coverage=off)"), + reports (note, the compiler build config must include `profiler = true`); \ + implies `-Z symbol-mangling-version=v0`. Optional values are: + `=all` (implicit value) + `=except-unused-generics` + `=except-unused-functions` + `=off` (default)"), instrument_mcount: bool = (false, parse_bool, [TRACKED], "insert function instrument code for mcount-based tracing (default: no)"), keep_hygiene_data: bool = (false, parse_bool, [UNTRACKED], @@ -1088,7 +1140,7 @@ options! {DebuggingOptions, DebuggingSetter, basic_debugging_options, "MIR optimization level (0-4; default: 1 in non optimized builds and 2 in optimized builds)"), mutable_noalias: Option = (None, parse_opt_bool, [TRACKED], "emit noalias metadata for mutable references (default: yes for LLVM >= 12, otherwise no)"), - new_llvm_pass_manager: bool = (false, parse_bool, [TRACKED], + new_llvm_pass_manager: Option = (None, parse_opt_bool, [TRACKED], "use new LLVM pass manager (default: no)"), nll_facts: bool = (false, parse_bool, [UNTRACKED], "dump facts from NLL analysis into side files (default: no)"), @@ -1161,6 +1213,9 @@ options! {DebuggingOptions, DebuggingSetter, basic_debugging_options, "whether ELF relocations can be relaxed"), relro_level: Option = (None, parse_relro_level, [TRACKED], "choose which RELRO level to use"), + simulate_remapped_rust_src_base: Option = (None, parse_opt_pathbuf, [TRACKED], + "simulate the effect of remap-debuginfo = true at bootstrapping by remapping path \ + to rust's source base directory. only meant for testing purposes"), report_delayed_bugs: bool = (false, parse_bool, [TRACKED], "immediately print bugs registered with `delay_span_bug` (default: no)"), sanitizer: SanitizerSet = (SanitizerSet::empty(), parse_sanitizers, [TRACKED], @@ -1211,6 +1266,8 @@ options! {DebuggingOptions, DebuggingSetter, basic_debugging_options, "select processor to schedule for (`rustc --print target-cpus` for details)"), thinlto: Option = (None, parse_opt_bool, [TRACKED], "enable ThinLTO when possible"), + thir_unsafeck: bool = (false, parse_bool, [TRACKED], + "use the work-in-progress THIR unsafety checker. NOTE: this is unsound (default: no)"), /// We default to 1 here since we want to behave like /// a sequential compiler for now. This'll likely be adjusted /// in the future. Note that -Zthreads=0 is the way to get @@ -1275,3 +1332,8 @@ pub enum WasiExecModel { Command, Reactor, } + +#[derive(Clone, Copy, Hash)] +pub enum LdImpl { + Lld, +} diff --git a/compiler/rustc_session/src/output.rs b/compiler/rustc_session/src/output.rs index 777eea3f68..cc1e4bb198 100644 --- a/compiler/rustc_session/src/output.rs +++ b/compiler/rustc_session/src/output.rs @@ -127,6 +127,11 @@ pub fn filename_for_metadata( crate_name: &str, outputs: &OutputFilenames, ) -> PathBuf { + // If the command-line specified the path, use that directly. + if let Some(Some(out_filename)) = sess.opts.output_types.get(&OutputType::Metadata) { + return out_filename.clone(); + } + let libname = format!("{}{}", crate_name, sess.opts.cg.extra_filename); let out_filename = outputs diff --git a/compiler/rustc_session/src/parse.rs b/compiler/rustc_session/src/parse.rs index 65d5d96aba..7b7878e9c7 100644 --- a/compiler/rustc_session/src/parse.rs +++ b/compiler/rustc_session/src/parse.rs @@ -137,6 +137,9 @@ pub struct ParseSess { pub type_ascription_path_suggestions: Lock>, /// Whether cfg(version) should treat the current release as incomplete pub assume_incomplete_release: bool, + /// Spans passed to `proc_macro::quote_span`. Each span has a numerical + /// identifier represented by its position in the vector. + pub proc_macro_quoted_spans: Lock>, } impl ParseSess { @@ -164,6 +167,7 @@ impl ParseSess { env_depinfo: Default::default(), type_ascription_path_suggestions: Default::default(), assume_incomplete_release: false, + proc_macro_quoted_spans: Default::default(), } } @@ -236,4 +240,14 @@ impl ParseSess { ); } } + + pub fn save_proc_macro_span(&self, span: Span) -> usize { + let mut spans = self.proc_macro_quoted_spans.lock(); + spans.push(span); + return spans.len() - 1; + } + + pub fn proc_macro_quoted_spans(&self) -> Vec { + self.proc_macro_quoted_spans.lock().clone() + } } diff --git a/compiler/rustc_session/src/session.rs b/compiler/rustc_session/src/session.rs index e7dfc4b8c4..86b8389a67 100644 --- a/compiler/rustc_session/src/session.rs +++ b/compiler/rustc_session/src/session.rs @@ -23,8 +23,8 @@ use rustc_errors::registry::Registry; use rustc_errors::{Diagnostic, DiagnosticBuilder, DiagnosticId, ErrorReported}; use rustc_lint_defs::FutureBreakage; pub use rustc_span::crate_disambiguator::CrateDisambiguator; -use rustc_span::edition::Edition; use rustc_span::source_map::{FileLoader, MultiSpan, RealFileLoader, SourceMap, Span}; +use rustc_span::{edition::Edition, RealFileName}; use rustc_span::{sym, SourceFileHashAlgorithm, Symbol}; use rustc_target::asm::InlineAsmArch; use rustc_target::spec::{CodeModel, PanicStrategy, RelocModel, RelroLevel}; @@ -125,9 +125,8 @@ pub struct Session { /// The name of the root source file of the crate, in the local file system. /// `None` means that there is no source file. pub local_crate_source_file: Option, - /// The directory the compiler has been executed in plus a flag indicating - /// if the value stored here has been affected by path remapping. - pub working_dir: (PathBuf, bool), + /// The directory the compiler has been executed in + pub working_dir: RealFileName, /// Set of `(DiagnosticId, Option, message)` tuples tracking /// (sub)diagnostics that have been set once, but should not be set again, @@ -421,7 +420,7 @@ impl Session { } pub fn span_fatal>(&self, sp: S, msg: &str) -> ! { - self.diagnostic().span_fatal(sp, msg).raise() + self.diagnostic().span_fatal(sp, msg) } pub fn span_fatal_with_code>( &self, @@ -429,7 +428,7 @@ impl Session { msg: &str, code: DiagnosticId, ) -> ! { - self.diagnostic().span_fatal_with_code(sp, msg, code).raise() + self.diagnostic().span_fatal_with_code(sp, msg, code) } pub fn fatal(&self, msg: &str) -> ! { self.diagnostic().fatal(msg).raise() @@ -453,6 +452,7 @@ impl Session { pub fn emit_err<'a>(&'a self, err: impl SessionDiagnostic<'a>) { err.into_diagnostic(self).emit() } + #[inline] pub fn err_count(&self) -> usize { self.diagnostic().err_count() } @@ -492,12 +492,6 @@ impl Session { pub fn warn(&self, msg: &str) { self.diagnostic().warn(msg) } - pub fn opt_span_warn>(&self, opt_sp: Option, msg: &str) { - match opt_sp { - Some(sp) => self.span_warn(sp, msg), - None => self.warn(msg), - } - } /// Delay a span_bug() call until abort_if_errors() #[track_caller] pub fn delay_span_bug>(&self, sp: S, msg: &str) { @@ -531,6 +525,7 @@ impl Session { self.diagnostic().struct_note_without_error(msg) } + #[inline] pub fn diagnostic(&self) -> &rustc_errors::Handler { &self.parse_sess.span_diagnostic } @@ -1282,9 +1277,14 @@ pub fn build_session( DiagnosticOutput::Raw(write) => Some(write), }; - let target_cfg = config::build_target_config(&sopts, target_override); + let sysroot = match &sopts.maybe_sysroot { + Some(sysroot) => sysroot.clone(), + None => filesearch::get_or_default_sysroot(), + }; + + let target_cfg = config::build_target_config(&sopts, target_override, &sysroot); let host_triple = TargetTriple::from_triple(config::host_triple()); - let host = Target::search(&host_triple).unwrap_or_else(|e| { + let host = Target::search(&host_triple, &sysroot).unwrap_or_else(|e| { early_error(sopts.error_format, &format!("Error loading host specification: {}", e)) }); @@ -1331,10 +1331,6 @@ pub fn build_session( let mut parse_sess = ParseSess::with_span_handler(span_diagnostic, source_map); parse_sess.assume_incomplete_release = sopts.debugging_opts.assume_incomplete_release; - let sysroot = match &sopts.maybe_sysroot { - Some(sysroot) => sysroot.clone(), - None => filesearch::get_or_default_sysroot(), - }; let host_triple = config::host_triple(); let target_triple = sopts.target_triple.triple(); @@ -1361,7 +1357,12 @@ pub fn build_session( let working_dir = env::current_dir().unwrap_or_else(|e| { parse_sess.span_diagnostic.fatal(&format!("Current directory is invalid: {}", e)).raise() }); - let working_dir = file_path_mapping.map_prefix(working_dir); + let (path, remapped) = file_path_mapping.map_prefix(working_dir.clone()); + let working_dir = if remapped { + RealFileName::Remapped { local_path: Some(working_dir), virtual_name: path } + } else { + RealFileName::LocalPath(path) + }; let cgu_reuse_tracker = if sopts.debugging_opts.query_dep_graph { CguReuseTracker::new() @@ -1513,6 +1514,14 @@ fn validate_commandline_args_with_session_available(sess: &Session) { if let (Some(first), Some(second)) = (sanitizer_iter.next(), sanitizer_iter.next()) { sess.err(&format!("`-Zsanitizer={}` is incompatible with `-Zsanitizer={}`", first, second)); } + + // Cannot enable crt-static with sanitizers on Linux + if sess.crt_static(None) && !sess.opts.debugging_opts.sanitizer.is_empty() { + sess.err( + "Sanitizer is incompatible with statically linked libc, \ + disable it using `-C target-feature=-crt-static`", + ); + } } /// Holds data on the current incremental compilation session, if there is one. @@ -1533,7 +1542,7 @@ pub enum IncrCompSession { InvalidBecauseOfErrors { session_directory: PathBuf }, } -pub fn early_error(output: config::ErrorOutputType, msg: &str) -> ! { +pub fn early_error_no_abort(output: config::ErrorOutputType, msg: &str) { let emitter: Box = match output { config::ErrorOutputType::HumanReadable(kind) => { let (short, color_config) = kind.unzip(); @@ -1545,6 +1554,10 @@ pub fn early_error(output: config::ErrorOutputType, msg: &str) -> ! { }; let handler = rustc_errors::Handler::with_emitter(true, None, emitter); handler.struct_fatal(msg).emit(); +} + +pub fn early_error(output: config::ErrorOutputType, msg: &str) -> ! { + early_error_no_abort(output, msg); rustc_errors::FatalError.raise(); } diff --git a/compiler/rustc_session/src/utils.rs b/compiler/rustc_session/src/utils.rs index e9d597d1ba..1a044e677a 100644 --- a/compiler/rustc_session/src/utils.rs +++ b/compiler/rustc_session/src/utils.rs @@ -19,25 +19,42 @@ impl Session { #[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Encodable, Decodable)] pub enum NativeLibKind { - /// Static library (e.g. `libfoo.a` on Linux or `foo.lib` on Windows/MSVC) included - /// when linking a final binary, but not when archiving an rlib. - StaticNoBundle, - /// Static library (e.g. `libfoo.a` on Linux or `foo.lib` on Windows/MSVC) included - /// when linking a final binary, but also included when archiving an rlib. - StaticBundle, + /// Static library (e.g. `libfoo.a` on Linux or `foo.lib` on Windows/MSVC) + Static { + /// Whether to bundle objects from static library into produced rlib + bundle: Option, + /// Whether to link static library without throwing any object files away + whole_archive: Option, + }, /// Dynamic library (e.g. `libfoo.so` on Linux) /// or an import library corresponding to a dynamic library (e.g. `foo.lib` on Windows/MSVC). - Dylib, + Dylib { + /// Whether the dynamic library will be linked only if it satifies some undefined symbols + as_needed: Option, + }, /// Dynamic library (e.g. `foo.dll` on Windows) without a corresponding import library. RawDylib, /// A macOS-specific kind of dynamic libraries. - Framework, + Framework { + /// Whether the framework will be linked only if it satifies some undefined symbols + as_needed: Option, + }, /// The library kind wasn't specified, `Dylib` is currently used as a default. Unspecified, } rustc_data_structures::impl_stable_hash_via_hash!(NativeLibKind); +#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Encodable, Decodable)] +pub struct NativeLib { + pub name: String, + pub new_name: Option, + pub kind: NativeLibKind, + pub verbatim: Option, +} + +rustc_data_structures::impl_stable_hash_via_hash!(NativeLib); + /// A path that has been canonicalized along with its original, non-canonicalized form #[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)] pub struct CanonicalizedPath { diff --git a/compiler/rustc_span/src/def_id.rs b/compiler/rustc_span/src/def_id.rs index 95bb0ad7ba..6ee75376ad 100644 --- a/compiler/rustc_span/src/def_id.rs +++ b/compiler/rustc_span/src/def_id.rs @@ -10,65 +10,23 @@ use std::borrow::Borrow; use std::fmt; rustc_index::newtype_index! { - pub struct CrateId { + pub struct CrateNum { ENCODABLE = custom + DEBUG_FORMAT = "crate{}" } } -#[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] -pub enum CrateNum { - /// A special `CrateNum` that we use for the `tcx.rcache` when decoding from - /// the incr. comp. cache. - ReservedForIncrCompCache, - Index(CrateId), -} - /// Item definitions in the currently-compiled crate would have the `CrateNum` /// `LOCAL_CRATE` in their `DefId`. -pub const LOCAL_CRATE: CrateNum = CrateNum::Index(CrateId::from_u32(0)); - -impl Idx for CrateNum { - #[inline] - fn new(value: usize) -> Self { - CrateNum::Index(Idx::new(value)) - } - - #[inline] - fn index(self) -> usize { - match self { - CrateNum::Index(idx) => Idx::index(idx), - _ => panic!("Tried to get crate index of {:?}", self), - } - } -} +pub const LOCAL_CRATE: CrateNum = CrateNum::from_u32(0); impl CrateNum { + #[inline] pub fn new(x: usize) -> CrateNum { CrateNum::from_usize(x) } - pub fn from_usize(x: usize) -> CrateNum { - CrateNum::Index(CrateId::from_usize(x)) - } - - pub fn from_u32(x: u32) -> CrateNum { - CrateNum::Index(CrateId::from_u32(x)) - } - - pub fn as_usize(self) -> usize { - match self { - CrateNum::Index(id) => id.as_usize(), - _ => panic!("tried to get index of non-standard crate {:?}", self), - } - } - - pub fn as_u32(self) -> u32 { - match self { - CrateNum::Index(id) => id.as_u32(), - _ => panic!("tried to get index of non-standard crate {:?}", self), - } - } - + #[inline] pub fn as_def_id(&self) -> DefId { DefId { krate: *self, index: CRATE_DEF_INDEX } } @@ -76,10 +34,7 @@ impl CrateNum { impl fmt::Display for CrateNum { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - CrateNum::Index(id) => fmt::Display::fmt(&id.private, f), - CrateNum::ReservedForIncrCompCache => write!(f, "crate for decoding incr comp cache"), - } + fmt::Display::fmt(&self.private, f) } } @@ -97,15 +52,6 @@ impl Decodable for CrateNum { } } -impl ::std::fmt::Debug for CrateNum { - fn fmt(&self, fmt: &mut ::std::fmt::Formatter<'_>) -> ::std::fmt::Result { - match self { - CrateNum::Index(id) => write!(fmt, "crate{}", id.private), - CrateNum::ReservedForIncrCompCache => write!(fmt, "crate for decoding incr comp cache"), - } - } -} - /// A `DefPathHash` is a fixed-size representation of a `DefPath` that is /// stable across crate and compilation session boundaries. It consists of two /// separate 64-bit hashes. The first uniquely identifies the crate this @@ -271,20 +217,20 @@ impl DefId { impl Encodable for DefId { default fn encode(&self, s: &mut E) -> Result<(), E::Error> { - s.emit_struct("DefId", 2, |s| { - s.emit_struct_field("krate", 0, |s| self.krate.encode(s))?; + s.emit_struct(false, |s| { + s.emit_struct_field("krate", true, |s| self.krate.encode(s))?; - s.emit_struct_field("index", 1, |s| self.index.encode(s)) + s.emit_struct_field("index", false, |s| self.index.encode(s)) }) } } impl Decodable for DefId { default fn decode(d: &mut D) -> Result { - d.read_struct("DefId", 2, |d| { + d.read_struct(|d| { Ok(DefId { - krate: d.read_struct_field("krate", 0, Decodable::decode)?, - index: d.read_struct_field("index", 1, Decodable::decode)?, + krate: d.read_struct_field("krate", Decodable::decode)?, + index: d.read_struct_field("index", Decodable::decode)?, }) }) } diff --git a/compiler/rustc_span/src/hygiene.rs b/compiler/rustc_span/src/hygiene.rs index f75fe22767..8f3b8cc2d0 100644 --- a/compiler/rustc_span/src/hygiene.rs +++ b/compiler/rustc_span/src/hygiene.rs @@ -144,7 +144,10 @@ impl ExpnId { let expn_data = self.expn_data(); // Stop going up the backtrace once include! is encountered if expn_data.is_root() - || expn_data.kind == ExpnKind::Macro(MacroKind::Bang, sym::include) + || matches!( + expn_data.kind, + ExpnKind::Macro { kind: MacroKind::Bang, name: sym::include, proc_macro: _ } + ) { break; } @@ -839,7 +842,13 @@ pub enum ExpnKind { /// No expansion, aka root expansion. Only `ExpnId::root()` has this kind. Root, /// Expansion produced by a macro. - Macro(MacroKind, Symbol), + Macro { + kind: MacroKind, + name: Symbol, + /// If `true`, this macro is a procedural macro. This + /// flag is only used for diagnostic purposes + proc_macro: bool, + }, /// Transform done by the compiler on the AST. AstPass(AstPass), /// Desugaring done by the compiler during HIR lowering. @@ -852,7 +861,7 @@ impl ExpnKind { pub fn descr(&self) -> String { match *self { ExpnKind::Root => kw::PathRoot.to_string(), - ExpnKind::Macro(macro_kind, name) => match macro_kind { + ExpnKind::Macro { kind, name, proc_macro: _ } => match kind { MacroKind::Bang => format!("{}!", name), MacroKind::Attr => format!("#[{}]", name), MacroKind::Derive => format!("#[derive({})]", name), @@ -1330,7 +1339,7 @@ fn update_disambiguator(expn_id: ExpnId) { // This cache is only used by `DummyHashStableContext`, // so we won't pollute the cache values of the normal `StableHashingContext` thread_local! { - static CACHE: ExpnIdCache = Default::default(); + static CACHE: ExpnIdCache = const { ExpnIdCache::new(Vec::new()) }; } &CACHE diff --git a/compiler/rustc_span/src/lib.rs b/compiler/rustc_span/src/lib.rs index d30236ec3e..9cf68cbd23 100644 --- a/compiler/rustc_span/src/lib.rs +++ b/compiler/rustc_span/src/lib.rs @@ -16,10 +16,10 @@ #![doc(html_root_url = "https://doc.rust-lang.org/nightly/nightly-rustc/")] #![feature(array_windows)] #![feature(crate_visibility_modifier)] -#![feature(const_panic)] #![feature(negative_impls)] #![feature(nll)] #![feature(min_specialization)] +#![feature(thread_local_const_init)] #[macro_use] extern crate rustc_macros; @@ -113,52 +113,115 @@ pub fn with_default_session_globals(f: impl FnOnce() -> R) -> R { // deserialization. scoped_tls::scoped_thread_local!(pub static SESSION_GLOBALS: SessionGlobals); -// FIXME: Perhaps this should not implement Rustc{Decodable, Encodable} -// // FIXME: We should use this enum or something like it to get rid of the // use of magic `/rust/1.x/...` paths across the board. -#[derive(Debug, Eq, PartialEq, Clone, Ord, PartialOrd, Hash)] -#[derive(HashStable_Generic, Decodable, Encodable)] +#[derive(Debug, Eq, PartialEq, Clone, Ord, PartialOrd)] +#[derive(HashStable_Generic, Decodable)] pub enum RealFileName { - Named(PathBuf), - /// For de-virtualized paths (namely paths into libstd that have been mapped - /// to the appropriate spot on the local host's file system), - Devirtualized { - /// `local_path` is the (host-dependent) local path to the file. - local_path: PathBuf, + LocalPath(PathBuf), + /// For remapped paths (namely paths into libstd that have been mapped + /// to the appropriate spot on the local host's file system, and local file + /// system paths that have been remapped with `FilePathMapping`), + Remapped { + /// `local_path` is the (host-dependent) local path to the file. This is + /// None if the file was imported from another crate + local_path: Option, /// `virtual_name` is the stable path rustc will store internally within /// build artifacts. virtual_name: PathBuf, }, } +impl Hash for RealFileName { + fn hash(&self, state: &mut H) { + // To prevent #70924 from happening again we should only hash the + // remapped (virtualized) path if that exists. This is because + // virtualized paths to sysroot crates (/rust/$hash or /rust/$version) + // remain stable even if the corresponding local_path changes + self.remapped_path_if_available().hash(state) + } +} + +// This is functionally identical to #[derive(Encodable)], with the exception of +// an added assert statement +impl Encodable for RealFileName { + fn encode(&self, encoder: &mut S) -> Result<(), S::Error> { + encoder.emit_enum(|encoder| match *self { + RealFileName::LocalPath(ref local_path) => { + encoder.emit_enum_variant("LocalPath", 0, 1, |encoder| { + Ok({ + encoder + .emit_enum_variant_arg(true, |encoder| local_path.encode(encoder))?; + }) + }) + } + + RealFileName::Remapped { ref local_path, ref virtual_name } => encoder + .emit_enum_variant("Remapped", 1, 2, |encoder| { + // For privacy and build reproducibility, we must not embed host-dependant path in artifacts + // if they have been remapped by --remap-path-prefix + assert!(local_path.is_none()); + Ok({ + encoder + .emit_enum_variant_arg(true, |encoder| local_path.encode(encoder))?; + encoder + .emit_enum_variant_arg(false, |encoder| virtual_name.encode(encoder))?; + }) + }), + }) + } +} + impl RealFileName { - /// Returns the path suitable for reading from the file system on the local host. - /// Avoid embedding this in build artifacts; see `stable_name()` for that. - pub fn local_path(&self) -> &Path { + /// Returns the path suitable for reading from the file system on the local host, + /// if this information exists. + /// Avoid embedding this in build artifacts; see `remapped_path_if_available()` for that. + pub fn local_path(&self) -> Option<&Path> { match self { - RealFileName::Named(p) - | RealFileName::Devirtualized { local_path: p, virtual_name: _ } => &p, + RealFileName::LocalPath(p) => Some(p), + RealFileName::Remapped { local_path: p, virtual_name: _ } => { + p.as_ref().map(PathBuf::as_path) + } + } + } + + /// Returns the path suitable for reading from the file system on the local host, + /// if this information exists. + /// Avoid embedding this in build artifacts; see `remapped_path_if_available()` for that. + pub fn into_local_path(self) -> Option { + match self { + RealFileName::LocalPath(p) => Some(p), + RealFileName::Remapped { local_path: p, virtual_name: _ } => p, } } - /// Returns the path suitable for reading from the file system on the local host. - /// Avoid embedding this in build artifacts; see `stable_name()` for that. - pub fn into_local_path(self) -> PathBuf { + /// Returns the path suitable for embedding into build artifacts. This would still + /// be a local path if it has not been remapped. A remapped path will not correspond + /// to a valid file system path: see `local_path_if_available()` for something that + /// is more likely to return paths into the local host file system. + pub fn remapped_path_if_available(&self) -> &Path { match self { - RealFileName::Named(p) - | RealFileName::Devirtualized { local_path: p, virtual_name: _ } => p, + RealFileName::LocalPath(p) + | RealFileName::Remapped { local_path: _, virtual_name: p } => &p, } } - /// Returns the path suitable for embedding into build artifacts. Note that - /// a virtualized path will not correspond to a valid file system path; see - /// `local_path()` for something that is more likely to return paths into the - /// local host file system. - pub fn stable_name(&self) -> &Path { + /// Returns the path suitable for reading from the file system on the local host, + /// if this information exists. Otherwise returns the remapped name. + /// Avoid embedding this in build artifacts; see `remapped_path_if_available()` for that. + pub fn local_path_if_available(&self) -> &Path { match self { - RealFileName::Named(p) - | RealFileName::Devirtualized { local_path: _, virtual_name: p } => &p, + RealFileName::LocalPath(path) + | RealFileName::Remapped { local_path: None, virtual_name: path } + | RealFileName::Remapped { local_path: Some(path), virtual_name: _ } => path, + } + } + + pub fn to_string_lossy(&self, prefer_local: bool) -> Cow<'_, str> { + if prefer_local { + self.local_path_if_available().to_string_lossy() + } else { + self.remapped_path_if_available().to_string_lossy() } } } @@ -187,16 +250,24 @@ pub enum FileName { InlineAsm(u64), } -impl std::fmt::Display for FileName { +impl From for FileName { + fn from(p: PathBuf) -> Self { + assert!(!p.to_string_lossy().ends_with('>')); + FileName::Real(RealFileName::LocalPath(p)) + } +} + +pub struct FileNameDisplay<'a> { + inner: &'a FileName, + prefer_local: bool, +} + +impl fmt::Display for FileNameDisplay<'_> { fn fmt(&self, fmt: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { use FileName::*; - match *self { - Real(RealFileName::Named(ref path)) => write!(fmt, "{}", path.display()), - // FIXME: might be nice to display both components of Devirtualized. - // But for now (to backport fix for issue #70924), best to not - // perturb diagnostics so its obvious test suite still works. - Real(RealFileName::Devirtualized { ref local_path, virtual_name: _ }) => { - write!(fmt, "{}", local_path.display()) + match *self.inner { + Real(ref name) => { + write!(fmt, "{}", name.to_string_lossy(self.prefer_local)) } QuoteExpansion(_) => write!(fmt, ""), MacroExpansion(_) => write!(fmt, ""), @@ -211,10 +282,12 @@ impl std::fmt::Display for FileName { } } -impl From for FileName { - fn from(p: PathBuf) -> Self { - assert!(!p.to_string_lossy().ends_with('>')); - FileName::Real(RealFileName::Named(p)) +impl FileNameDisplay<'_> { + pub fn to_string_lossy(&self) -> Cow<'_, str> { + match self.inner { + FileName::Real(ref inner) => inner.to_string_lossy(self.prefer_local), + _ => Cow::from(format!("{}", self)), + } } } @@ -235,6 +308,16 @@ impl FileName { } } + pub fn prefer_remapped(&self) -> FileNameDisplay<'_> { + FileNameDisplay { inner: self, prefer_local: false } + } + + // This may include transient local filesystem information. + // Must not be embedded in build outputs. + pub fn prefer_local(&self) -> FileNameDisplay<'_> { + FileNameDisplay { inner: self, prefer_local: true } + } + pub fn macro_expansion_source_code(src: &str) -> FileName { let mut hasher = StableHasher::new(); src.hash(&mut hasher); @@ -393,7 +476,10 @@ impl Span { /// Returns `true` if `span` originates in a derive-macro's expansion. pub fn in_derive_expansion(self) -> bool { - matches!(self.ctxt().outer_expn_data().kind, ExpnKind::Macro(MacroKind::Derive, _)) + matches!( + self.ctxt().outer_expn_data().kind, + ExpnKind::Macro { kind: MacroKind::Derive, name: _, proc_macro: _ } + ) } #[inline] @@ -744,17 +830,17 @@ impl Default for Span { impl Encodable for Span { default fn encode(&self, s: &mut E) -> Result<(), E::Error> { let span = self.data(); - s.emit_struct("Span", 2, |s| { - s.emit_struct_field("lo", 0, |s| span.lo.encode(s))?; - s.emit_struct_field("hi", 1, |s| span.hi.encode(s)) + s.emit_struct(false, |s| { + s.emit_struct_field("lo", true, |s| span.lo.encode(s))?; + s.emit_struct_field("hi", false, |s| span.hi.encode(s)) }) } } impl Decodable for Span { default fn decode(s: &mut D) -> Result { - s.read_struct("Span", 2, |d| { - let lo = d.read_struct_field("lo", 0, Decodable::decode)?; - let hi = d.read_struct_field("hi", 1, Decodable::decode)?; + s.read_struct(|d| { + let lo = d.read_struct_field("lo", Decodable::decode)?; + let hi = d.read_struct_field("hi", Decodable::decode)?; Ok(Span::new(lo, hi, SyntaxContext::root())) }) @@ -792,7 +878,7 @@ pub fn debug_with_source_map( f: &mut fmt::Formatter<'_>, source_map: &SourceMap, ) -> fmt::Result { - write!(f, "{} ({:?})", source_map.span_to_string(span), span.ctxt()) + write!(f, "{} ({:?})", source_map.span_to_diagnostic_string(span), span.ctxt()) } pub fn default_span_debug(span: Span, f: &mut fmt::Formatter<'_>) -> fmt::Result { @@ -1124,11 +1210,6 @@ pub struct SourceFile { /// originate from files has names between angle brackets by convention /// (e.g., ``). pub name: FileName, - /// `true` if the `name` field above has been modified by `--remap-path-prefix`. - pub name_was_remapped: bool, - /// The unmapped path of the file that the source came from. - /// Set to `None` if the `SourceFile` was imported from an external crate. - pub unmapped_path: Option, /// The complete source code. pub src: Option>, /// The source code's hash. @@ -1156,13 +1237,12 @@ pub struct SourceFile { impl Encodable for SourceFile { fn encode(&self, s: &mut S) -> Result<(), S::Error> { - s.emit_struct("SourceFile", 8, |s| { - s.emit_struct_field("name", 0, |s| self.name.encode(s))?; - s.emit_struct_field("name_was_remapped", 1, |s| self.name_was_remapped.encode(s))?; - s.emit_struct_field("src_hash", 2, |s| self.src_hash.encode(s))?; - s.emit_struct_field("start_pos", 3, |s| self.start_pos.encode(s))?; - s.emit_struct_field("end_pos", 4, |s| self.end_pos.encode(s))?; - s.emit_struct_field("lines", 5, |s| { + s.emit_struct(false, |s| { + s.emit_struct_field("name", true, |s| self.name.encode(s))?; + s.emit_struct_field("src_hash", false, |s| self.src_hash.encode(s))?; + s.emit_struct_field("start_pos", false, |s| self.start_pos.encode(s))?; + s.emit_struct_field("end_pos", false, |s| self.end_pos.encode(s))?; + s.emit_struct_field("lines", false, |s| { let lines = &self.lines[..]; // Store the length. s.emit_u32(lines.len() as u32)?; @@ -1220,27 +1300,24 @@ impl Encodable for SourceFile { Ok(()) })?; - s.emit_struct_field("multibyte_chars", 6, |s| self.multibyte_chars.encode(s))?; - s.emit_struct_field("non_narrow_chars", 7, |s| self.non_narrow_chars.encode(s))?; - s.emit_struct_field("name_hash", 8, |s| self.name_hash.encode(s))?; - s.emit_struct_field("normalized_pos", 9, |s| self.normalized_pos.encode(s))?; - s.emit_struct_field("cnum", 10, |s| self.cnum.encode(s)) + s.emit_struct_field("multibyte_chars", false, |s| self.multibyte_chars.encode(s))?; + s.emit_struct_field("non_narrow_chars", false, |s| self.non_narrow_chars.encode(s))?; + s.emit_struct_field("name_hash", false, |s| self.name_hash.encode(s))?; + s.emit_struct_field("normalized_pos", false, |s| self.normalized_pos.encode(s))?; + s.emit_struct_field("cnum", false, |s| self.cnum.encode(s)) }) } } impl Decodable for SourceFile { fn decode(d: &mut D) -> Result { - d.read_struct("SourceFile", 8, |d| { - let name: FileName = d.read_struct_field("name", 0, |d| Decodable::decode(d))?; - let name_was_remapped: bool = - d.read_struct_field("name_was_remapped", 1, |d| Decodable::decode(d))?; + d.read_struct(|d| { + let name: FileName = d.read_struct_field("name", |d| Decodable::decode(d))?; let src_hash: SourceFileHash = - d.read_struct_field("src_hash", 2, |d| Decodable::decode(d))?; - let start_pos: BytePos = - d.read_struct_field("start_pos", 3, |d| Decodable::decode(d))?; - let end_pos: BytePos = d.read_struct_field("end_pos", 4, |d| Decodable::decode(d))?; - let lines: Vec = d.read_struct_field("lines", 5, |d| { + d.read_struct_field("src_hash", |d| Decodable::decode(d))?; + let start_pos: BytePos = d.read_struct_field("start_pos", |d| Decodable::decode(d))?; + let end_pos: BytePos = d.read_struct_field("end_pos", |d| Decodable::decode(d))?; + let lines: Vec = d.read_struct_field("lines", |d| { let num_lines: u32 = Decodable::decode(d)?; let mut lines = Vec::with_capacity(num_lines as usize); @@ -1269,17 +1346,15 @@ impl Decodable for SourceFile { Ok(lines) })?; let multibyte_chars: Vec = - d.read_struct_field("multibyte_chars", 6, |d| Decodable::decode(d))?; + d.read_struct_field("multibyte_chars", |d| Decodable::decode(d))?; let non_narrow_chars: Vec = - d.read_struct_field("non_narrow_chars", 7, |d| Decodable::decode(d))?; - let name_hash: u128 = d.read_struct_field("name_hash", 8, |d| Decodable::decode(d))?; + d.read_struct_field("non_narrow_chars", |d| Decodable::decode(d))?; + let name_hash: u128 = d.read_struct_field("name_hash", |d| Decodable::decode(d))?; let normalized_pos: Vec = - d.read_struct_field("normalized_pos", 9, |d| Decodable::decode(d))?; - let cnum: CrateNum = d.read_struct_field("cnum", 10, |d| Decodable::decode(d))?; + d.read_struct_field("normalized_pos", |d| Decodable::decode(d))?; + let cnum: CrateNum = d.read_struct_field("cnum", |d| Decodable::decode(d))?; Ok(SourceFile { name, - name_was_remapped, - unmapped_path: None, start_pos, end_pos, src: None, @@ -1300,15 +1375,13 @@ impl Decodable for SourceFile { impl fmt::Debug for SourceFile { fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(fmt, "SourceFile({})", self.name) + write!(fmt, "SourceFile({:?})", self.name) } } impl SourceFile { pub fn new( name: FileName, - name_was_remapped: bool, - unmapped_path: FileName, mut src: String, start_pos: BytePos, hash_kind: SourceFileHashAlgorithm, @@ -1330,8 +1403,6 @@ impl SourceFile { SourceFile { name, - name_was_remapped, - unmapped_path: Some(unmapped_path), src: Some(Lrc::new(src)), src_hash, external_src: Lock::new(ExternalSource::Unneeded), diff --git a/compiler/rustc_span/src/source_map.rs b/compiler/rustc_span/src/source_map.rs index f612d1425b..32031ac707 100644 --- a/compiler/rustc_span/src/source_map.rs +++ b/compiler/rustc_span/src/source_map.rs @@ -15,11 +15,11 @@ pub use crate::*; use rustc_data_structures::fx::FxHashMap; use rustc_data_structures::stable_hasher::StableHasher; use rustc_data_structures::sync::{AtomicU32, Lrc, MappedReadGuard, ReadGuard, RwLock}; -use std::cmp; -use std::convert::TryFrom; use std::hash::Hash; use std::path::{Path, PathBuf}; use std::sync::atomic::Ordering; +use std::{clone::Clone, cmp}; +use std::{convert::TryFrom, unreachable}; use std::fs; use std::io; @@ -109,7 +109,7 @@ pub struct RealFileLoader; impl FileLoader for RealFileLoader { fn file_exists(&self, path: &Path) -> bool { - fs::metadata(path).is_ok() + path.exists() } fn read_file(&self, path: &Path) -> io::Result { @@ -117,42 +117,42 @@ impl FileLoader for RealFileLoader { } } -// This is a `SourceFile` identifier that is used to correlate `SourceFile`s between -// subsequent compilation sessions (which is something we need to do during -// incremental compilation). +/// This is a [SourceFile] identifier that is used to correlate source files between +/// subsequent compilation sessions (which is something we need to do during +/// incremental compilation). +/// +/// The [StableSourceFileId] also contains the CrateNum of the crate the source +/// file was originally parsed for. This way we get two separate entries in +/// the [SourceMap] if the same file is part of both the local and an upstream +/// crate. Trying to only have one entry for both cases is problematic because +/// at the point where we discover that there's a local use of the file in +/// addition to the upstream one, we might already have made decisions based on +/// the assumption that it's an upstream file. Treating the two files as +/// different has no real downsides. #[derive(Copy, Clone, PartialEq, Eq, Hash, Encodable, Decodable, Debug)] -pub struct StableSourceFileId(u128); +pub struct StableSourceFileId { + // A hash of the source file's FileName. This is hash so that it's size + // is more predictable than if we included the actual FileName value. + pub file_name_hash: u64, + + // The CrateNum of the crate this source file was originally parsed for. + // We cannot include this information in the hash because at the time + // of hashing we don't have the context to map from the CrateNum's numeric + // value to a StableCrateId. + pub cnum: CrateNum, +} // FIXME: we need a more globally consistent approach to the problem solved by // StableSourceFileId, perhaps built atop source_file.name_hash. impl StableSourceFileId { pub fn new(source_file: &SourceFile) -> StableSourceFileId { - StableSourceFileId::new_from_pieces( - &source_file.name, - source_file.name_was_remapped, - source_file.unmapped_path.as_ref(), - ) + StableSourceFileId::new_from_name(&source_file.name, source_file.cnum) } - fn new_from_pieces( - name: &FileName, - name_was_remapped: bool, - unmapped_path: Option<&FileName>, - ) -> StableSourceFileId { + fn new_from_name(name: &FileName, cnum: CrateNum) -> StableSourceFileId { let mut hasher = StableHasher::new(); - - if let FileName::Real(real_name) = name { - // rust-lang/rust#70924: Use the stable (virtualized) name when - // available. (We do not want artifacts from transient file system - // paths for libstd to leak into our build artifacts.) - real_name.stable_name().hash(&mut hasher) - } else { - name.hash(&mut hasher); - } - name_was_remapped.hash(&mut hasher); - unmapped_path.hash(&mut hasher); - - StableSourceFileId(hasher.finish()) + name.hash(&mut hasher); + StableSourceFileId { file_name_hash: hasher.finish(), cnum } } } @@ -283,35 +283,15 @@ impl SourceMap { fn try_new_source_file( &self, - mut filename: FileName, + filename: FileName, src: String, ) -> Result, OffsetOverflowError> { - // The path is used to determine the directory for loading submodules and - // include files, so it must be before remapping. // Note that filename may not be a valid path, eg it may be `` etc, // but this is okay because the directory determined by `path.pop()` will // be empty, so the working directory will be used. - let unmapped_path = filename.clone(); - - let was_remapped; - if let FileName::Real(real_filename) = &mut filename { - match real_filename { - RealFileName::Named(path_to_be_remapped) - | RealFileName::Devirtualized { - local_path: path_to_be_remapped, - virtual_name: _, - } => { - let mapped = self.path_mapping.map_prefix(path_to_be_remapped.clone()); - was_remapped = mapped.1; - *path_to_be_remapped = mapped.0; - } - } - } else { - was_remapped = false; - } + let (filename, _) = self.path_mapping.map_filename_prefix(&filename); - let file_id = - StableSourceFileId::new_from_pieces(&filename, was_remapped, Some(&unmapped_path)); + let file_id = StableSourceFileId::new_from_name(&filename, LOCAL_CRATE); let lrc_sf = match self.source_file_by_stable_id(file_id) { Some(lrc_sf) => lrc_sf, @@ -320,13 +300,15 @@ impl SourceMap { let source_file = Lrc::new(SourceFile::new( filename, - was_remapped, - unmapped_path, src, Pos::from_usize(start_pos), self.hash_kind, )); + // Let's make sure the file_id we generated above actually matches + // the ID we generate for the SourceFile we just created. + debug_assert_eq!(StableSourceFileId::new(&source_file), file_id); + let mut files = self.files.borrow_mut(); files.source_files.push(source_file.clone()); @@ -345,7 +327,6 @@ impl SourceMap { pub fn new_imported_source_file( &self, filename: FileName, - name_was_remapped: bool, src_hash: SourceFileHash, name_hash: u128, source_len: usize, @@ -382,8 +363,6 @@ impl SourceMap { let source_file = Lrc::new(SourceFile { name: filename, - name_was_remapped, - unmapped_path: None, src: None, src_hash, external_src: Lock::new(ExternalSource::Foreign { @@ -411,11 +390,6 @@ impl SourceMap { source_file } - pub fn mk_substr_filename(&self, sp: Span) -> String { - let pos = self.lookup_char_pos(sp.lo()); - format!("<{}:{}:{}>", pos.file.name, pos.line, pos.col.to_usize() + 1) - } - // If there is a doctest offset, applies it to the line. pub fn doctest_offset_line(&self, file: &FileName, orig: usize) -> usize { match file { @@ -453,8 +427,8 @@ impl SourceMap { } } - pub fn span_to_string(&self, sp: Span) -> String { - if self.files.borrow().source_files.is_empty() && sp.is_dummy() { + fn span_to_string(&self, sp: Span, prefer_local: bool) -> String { + if self.files.borrow().source_files.is_empty() || sp.is_dummy() { return "no-location".to_string(); } @@ -462,7 +436,7 @@ impl SourceMap { let hi = self.lookup_char_pos(sp.hi()); format!( "{}:{}:{}: {}:{}", - lo.file.name, + if prefer_local { lo.file.name.prefer_local() } else { lo.file.name.prefer_remapped() }, lo.line, lo.col.to_usize() + 1, hi.line, @@ -470,16 +444,20 @@ impl SourceMap { ) } - pub fn span_to_filename(&self, sp: Span) -> FileName { - self.lookup_char_pos(sp.lo()).file.name.clone() + /// Format the span location suitable for embedding in build artifacts + pub fn span_to_embeddable_string(&self, sp: Span) -> String { + self.span_to_string(sp, false) } - pub fn span_to_unmapped_path(&self, sp: Span) -> FileName { - self.lookup_char_pos(sp.lo()) - .file - .unmapped_path - .clone() - .expect("`SourceMap::span_to_unmapped_path` called for imported `SourceFile`?") + /// Format the span location to be printed in diagnostics. Must not be emitted + /// to build artifacts as this may leak local file paths. Use span_to_embeddable_string + /// for string suitable for embedding. + pub fn span_to_diagnostic_string(&self, sp: Span) -> String { + self.span_to_string(sp, true) + } + + pub fn span_to_filename(&self, sp: Span) -> FileName { + self.lookup_char_pos(sp.lo()).file.name.clone() } pub fn is_multiline(&self, sp: Span) -> bool { @@ -1001,7 +979,13 @@ impl SourceMap { } pub fn ensure_source_file_source_present(&self, source_file: Lrc) -> bool { source_file.add_external_src(|| match source_file.name { - FileName::Real(ref name) => self.file_loader.read_file(name.local_path()).ok(), + FileName::Real(ref name) => { + if let Some(local_path) = name.local_path() { + self.file_loader.read_file(local_path).ok() + } else { + None + } + } _ => None, }) } @@ -1046,9 +1030,20 @@ impl FilePathMapping { fn map_filename_prefix(&self, file: &FileName) -> (FileName, bool) { match file { FileName::Real(realfile) => { - let path = realfile.local_path(); - let (path, mapped) = self.map_prefix(path.to_path_buf()); - (FileName::Real(RealFileName::Named(path)), mapped) + if let RealFileName::LocalPath(local_path) = realfile { + let (mapped_path, mapped) = self.map_prefix(local_path.to_path_buf()); + let realfile = if mapped { + RealFileName::Remapped { + local_path: Some(local_path.clone()), + virtual_name: mapped_path, + } + } else { + realfile.clone() + }; + (FileName::Real(realfile), mapped) + } else { + unreachable!("attempted to remap an already remapped filename"); + } } other => (other.clone(), false), } diff --git a/compiler/rustc_span/src/source_map/tests.rs b/compiler/rustc_span/src/source_map/tests.rs index 7d814f1d82..f13979941a 100644 --- a/compiler/rustc_span/src/source_map/tests.rs +++ b/compiler/rustc_span/src/source_map/tests.rs @@ -193,7 +193,7 @@ fn t8() { fn t9() { let sm = init_source_map(); let span = Span::with_root_ctxt(BytePos(12), BytePos(23)); - let sstr = sm.span_to_string(span); + let sstr = sm.span_to_diagnostic_string(span); assert_eq!(sstr, "blork.rs:2:1: 2:12"); } @@ -229,7 +229,6 @@ fn t10() { let SourceFile { name, - name_was_remapped, src_hash, start_pos, end_pos, @@ -243,7 +242,6 @@ fn t10() { let imported_src_file = sm.new_imported_source_file( name, - name_was_remapped, src_hash, name_hash, (end_pos - start_pos).to_usize(), diff --git a/compiler/rustc_span/src/span_encoding.rs b/compiler/rustc_span/src/span_encoding.rs index ceb9b59b13..5ea39b343b 100644 --- a/compiler/rustc_span/src/span_encoding.rs +++ b/compiler/rustc_span/src/span_encoding.rs @@ -102,7 +102,7 @@ impl Span { // Interned format. debug_assert!(self.ctxt_or_zero == 0); let index = self.base_or_index; - with_span_interner(|interner| *interner.get(index)) + with_span_interner(|interner| interner.spans[index as usize]) } } } @@ -117,11 +117,6 @@ impl SpanInterner { let (index, _) = self.spans.insert_full(*span_data); index as u32 } - - #[inline] - fn get(&self, index: u32) -> &SpanData { - &self.spans[index as usize] - } } // If an interner exists, return it. Otherwise, prepare a fresh one. diff --git a/compiler/rustc_span/src/symbol.rs b/compiler/rustc_span/src/symbol.rs index c4007f74bd..a31ab5b350 100644 --- a/compiler/rustc_span/src/symbol.rs +++ b/compiler/rustc_span/src/symbol.rs @@ -130,10 +130,13 @@ symbols! { BTreeSet, BinaryHeap, Borrow, + Break, C, + CStr, CString, Center, Clone, + Continue, Copy, Count, Debug, @@ -325,7 +328,9 @@ symbols! { box_free, box_patterns, box_syntax, + bpf_target_feature, braced_empty_structs, + branch, breakpoint, bridge, bswap, @@ -374,6 +379,7 @@ symbols! { conservative_impl_trait, console, const_allocate, + const_async_blocks, const_compare_raw_pointers, const_constructor, const_eval_limit, @@ -410,6 +416,7 @@ symbols! { constructor, contents, context, + control_flow_enum, convert, copy, copy_closures, @@ -510,7 +517,6 @@ symbols! { env, eq, ermsb_target_feature, - err, exact_div, except, exchange_malloc, @@ -579,10 +585,10 @@ symbols! { frem_fast, from, from_desugaring, - from_error, from_generator, from_method, - from_ok, + from_output, + from_residual, from_size_align_unchecked, from_trait, from_usize, @@ -651,7 +657,6 @@ symbols! { instruction_set, intel, into_iter, - into_result, into_trait, intra_doc_pointers, intrinsics, @@ -747,8 +752,10 @@ symbols! { minnumf64, mips_target_feature, misc, + modifiers, module, module_path, + more_qualified_paths, more_struct_aliases, movbe_target_feature, move_ref_pattern, @@ -762,6 +769,11 @@ symbols! { naked, naked_functions, name, + native_link_modifiers, + native_link_modifiers_as_needed, + native_link_modifiers_bundle, + native_link_modifiers_verbatim, + native_link_modifiers_whole_archive, ne, nearbyintf32, nearbyintf64, @@ -932,6 +944,7 @@ symbols! { receiver, recursion_limit, reexport_test_harness_main, + ref_unwind_safe, reference, reflect, reg, @@ -940,6 +953,7 @@ symbols! { reg64, reg_abcd, reg_byte, + reg_nonzero, reg_thumb, register_attr, register_tool, @@ -955,6 +969,7 @@ symbols! { repr_packed, repr_simd, repr_transparent, + residual, result, result_type, rhs, @@ -985,7 +1000,6 @@ symbols! { rustc_allocator, rustc_allocator_nounwind, rustc_allow_const_fn_unstable, - rustc_args_required_const, rustc_attrs, rustc_builtin_macro, rustc_capture_analysis, @@ -1003,9 +1017,11 @@ symbols! { rustc_dump_program_clauses, rustc_dump_user_substs, rustc_error, + rustc_evaluate_where_clauses, rustc_expected_cgu_reuse, rustc_if_this_changed, rustc_inherit_overflow_checks, + rustc_insignificant_dtor, rustc_layout, rustc_layout_scalar_valid_range_end, rustc_layout_scalar_valid_range_start, @@ -1053,6 +1069,7 @@ symbols! { self_in_typedefs, self_struct_ctor, semitransparent, + send, send_trait, shl, shl_assign, @@ -1220,7 +1237,7 @@ symbols! { try_blocks, try_from_trait, try_into_trait, - try_trait, + try_trait_v2, tt, tuple, tuple_from_req, @@ -1258,6 +1275,7 @@ symbols! { unix, unlikely, unmarked_api, + unnamed_fields, unpin, unreachable, unreachable_code, @@ -1274,6 +1292,7 @@ symbols! { unused_qualifications, unwind, unwind_attributes, + unwind_safe, unwrap, unwrap_or, use_extern_macros, @@ -1314,6 +1333,7 @@ symbols! { wrapping_add, wrapping_mul, wrapping_sub, + wreg, write_bytes, xmm_reg, ymm_reg, diff --git a/compiler/rustc_symbol_mangling/src/legacy.rs b/compiler/rustc_symbol_mangling/src/legacy.rs index 7d186c330b..025eaffcbd 100644 --- a/compiler/rustc_symbol_mangling/src/legacy.rs +++ b/compiler/rustc_symbol_mangling/src/legacy.rs @@ -126,9 +126,7 @@ fn get_symbol_hash<'tcx>( substs.hash_stable(&mut hcx, &mut hasher); if let Some(instantiating_crate) = instantiating_crate { - tcx.original_crate_name(instantiating_crate) - .as_str() - .hash_stable(&mut hcx, &mut hasher); + tcx.crate_name(instantiating_crate).as_str().hash_stable(&mut hcx, &mut hasher); tcx.crate_disambiguator(instantiating_crate).hash_stable(&mut hcx, &mut hasher); } @@ -255,7 +253,7 @@ impl Printer<'tcx> for SymbolPrinter<'tcx> { } fn path_crate(mut self, cnum: CrateNum) -> Result { - self.write_str(&self.tcx.original_crate_name(cnum).as_str())?; + self.write_str(&self.tcx.crate_name(cnum).as_str())?; Ok(self) } fn path_qualified( diff --git a/compiler/rustc_symbol_mangling/src/lib.rs b/compiler/rustc_symbol_mangling/src/lib.rs index c050bbc9b9..f716ce1efc 100644 --- a/compiler/rustc_symbol_mangling/src/lib.rs +++ b/compiler/rustc_symbol_mangling/src/lib.rs @@ -90,7 +90,6 @@ #![doc(html_root_url = "https://doc.rust-lang.org/nightly/nightly-rustc/")] #![feature(never_type)] #![feature(nll)] -#![cfg_attr(bootstrap, feature(or_patterns))] #![feature(in_band_lifetimes)] #![recursion_limit = "256"] @@ -165,11 +164,11 @@ fn compute_symbol_name( // FIXME(eddyb) Precompute a custom symbol name based on attributes. let is_foreign = if let Some(def_id) = def_id.as_local() { - if tcx.plugin_registrar_fn(LOCAL_CRATE) == Some(def_id.to_def_id()) { + if tcx.plugin_registrar_fn(()) == Some(def_id) { let disambiguator = tcx.sess.local_crate_disambiguator(); return tcx.sess.generate_plugin_registrar_symbol(disambiguator); } - if tcx.proc_macro_decls_static(LOCAL_CRATE) == Some(def_id.to_def_id()) { + if tcx.proc_macro_decls_static(()) == Some(def_id) { let disambiguator = tcx.sess.local_crate_disambiguator(); return tcx.sess.generate_proc_macro_decls_symbol(disambiguator); } diff --git a/compiler/rustc_symbol_mangling/src/v0.rs b/compiler/rustc_symbol_mangling/src/v0.rs index 37a834043f..828f1ac0a7 100644 --- a/compiler/rustc_symbol_mangling/src/v0.rs +++ b/compiler/rustc_symbol_mangling/src/v0.rs @@ -485,9 +485,39 @@ impl Printer<'tcx> for SymbolMangler<'tcx> { mut self, predicates: &'tcx ty::List>>, ) -> Result { - for predicate in predicates { - self = self.in_binder(&predicate, |mut cx, predicate| { - match predicate { + // Okay, so this is a bit tricky. Imagine we have a trait object like + // `dyn for<'a> Foo<'a, Bar = &'a ()>`. When we mangle this, the + // output looks really close to the syntax, where the `Bar = &'a ()` bit + // is under the same binders (`['a]`) as the `Foo<'a>` bit. However, we + // actually desugar these into two separate `ExistentialPredicate`s. We + // can't enter/exit the "binder scope" twice though, because then we + // would mangle the binders twice. (Also, side note, we merging these + // two is kind of difficult, because of potential HRTBs in the Projection + // predicate.) + // + // Also worth mentioning: imagine that we instead had + // `dyn for<'a> Foo<'a, Bar = &'a ()> + Send`. In this case, `Send` is + // under the same binders as `Foo`. Currently, this doesn't matter, + // because only *auto traits* are allowed other than the principal trait + // and all auto traits don't have any generics. Two things could + // make this not an "okay" mangling: + // 1) Instead of mangling only *used* + // bound vars, we want to mangle *all* bound vars (`for<'b> Send` is a + // valid trait predicate); + // 2) We allow multiple "principal" traits in the future, or at least + // allow in any form another trait predicate that can take generics. + // + // Here we assume that predicates have the following structure: + // [ [{}]] [{}] + // Since any predicates after the first one shouldn't change the binders, + // just put them all in the binders of the first. + self = self.in_binder(&predicates[0], |mut cx, _| { + for predicate in predicates.iter() { + // It would be nice to be able to validate bound vars here, but + // projections can actually include bound vars from super traits + // because of HRTBs (only in the `Self` type). Also, auto traits + // could have different bound vars *anyways*. + match predicate.as_ref().skip_binder() { ty::ExistentialPredicate::Trait(trait_ref) => { // Use a type that can't appear in defaults of type parameters. let dummy_self = cx.tcx.mk_ty_infer(ty::FreshTy(0)); @@ -504,9 +534,10 @@ impl Printer<'tcx> for SymbolMangler<'tcx> { cx = cx.print_def_path(*def_id, &[])?; } } - Ok(cx) - })?; - } + } + Ok(cx) + })?; + self.push("E"); Ok(self) } @@ -563,7 +594,7 @@ impl Printer<'tcx> for SymbolMangler<'tcx> { self.push("C"); let fingerprint = self.tcx.crate_disambiguator(cnum).to_fingerprint(); self.push_disambiguator(fingerprint.to_smaller_hash()); - let name = self.tcx.original_crate_name(cnum).as_str(); + let name = self.tcx.crate_name(cnum).as_str(); self.push_ident(&name); Ok(self) } diff --git a/compiler/rustc_target/src/abi/call/bpf.rs b/compiler/rustc_target/src/abi/call/bpf.rs new file mode 100644 index 0000000000..466c525531 --- /dev/null +++ b/compiler/rustc_target/src/abi/call/bpf.rs @@ -0,0 +1,31 @@ +// see https://github.com/llvm/llvm-project/blob/main/llvm/lib/Target/BPF/BPFCallingConv.td +use crate::abi::call::{ArgAbi, FnAbi}; + +fn classify_ret(ret: &mut ArgAbi<'_, Ty>) { + if ret.layout.is_aggregate() || ret.layout.size.bits() > 64 { + ret.make_indirect(); + } else { + ret.extend_integer_width_to(32); + } +} + +fn classify_arg(arg: &mut ArgAbi<'_, Ty>) { + if arg.layout.is_aggregate() || arg.layout.size.bits() > 64 { + arg.make_indirect(); + } else { + arg.extend_integer_width_to(32); + } +} + +pub fn compute_abi_info(fn_abi: &mut FnAbi<'_, Ty>) { + if !fn_abi.ret.is_ignore() { + classify_ret(&mut fn_abi.ret); + } + + for arg in &mut fn_abi.args { + if arg.is_ignore() { + continue; + } + classify_arg(arg); + } +} diff --git a/compiler/rustc_target/src/abi/call/mod.rs b/compiler/rustc_target/src/abi/call/mod.rs index 0cf2441d84..6e0e140374 100644 --- a/compiler/rustc_target/src/abi/call/mod.rs +++ b/compiler/rustc_target/src/abi/call/mod.rs @@ -6,6 +6,7 @@ mod aarch64; mod amdgpu; mod arm; mod avr; +mod bpf; mod hexagon; mod mips; mod mips64; @@ -654,6 +655,7 @@ impl<'a, Ty> FnAbi<'a, Ty> { } } "asmjs" => wasm::compute_c_abi_info(cx, self), + "bpf" => bpf::compute_abi_info(self), a => return Err(format!("unrecognized arch \"{}\" in target specification", a)), } diff --git a/compiler/rustc_target/src/abi/mod.rs b/compiler/rustc_target/src/abi/mod.rs index e2618da749..8e71ded3d1 100644 --- a/compiler/rustc_target/src/abi/mod.rs +++ b/compiler/rustc_target/src/abi/mod.rs @@ -222,6 +222,7 @@ pub trait HasDataLayout { } impl HasDataLayout for TargetDataLayout { + #[inline] fn data_layout(&self) -> &TargetDataLayout { self } @@ -441,6 +442,8 @@ pub struct Align { } impl Align { + pub const ONE: Align = Align { pow2: 0 }; + #[inline] pub fn from_bits(bits: u64) -> Result { Align::from_bytes(Size::from_bits(bits).bytes()) @@ -450,7 +453,7 @@ impl Align { pub fn from_bytes(align: u64) -> Result { // Treat an alignment of 0 bytes like 1-byte alignment. if align == 0 { - return Ok(Align { pow2: 0 }); + return Ok(Align::ONE); } #[cold] @@ -750,11 +753,7 @@ impl FieldsShape { match *self { FieldsShape::Primitive => 0, FieldsShape::Union(count) => count.get(), - FieldsShape::Array { count, .. } => { - let usize_count = count as usize; - assert_eq!(usize_count as u64, count); - usize_count - } + FieldsShape::Array { count, .. } => count.try_into().unwrap(), FieldsShape::Arbitrary { ref offsets, .. } => offsets.len(), } } @@ -788,11 +787,7 @@ impl FieldsShape { unreachable!("FieldsShape::memory_index: `Primitive`s have no fields") } FieldsShape::Union(_) | FieldsShape::Array { .. } => i, - FieldsShape::Arbitrary { ref memory_index, .. } => { - let r = memory_index[i]; - assert_eq!(r as usize as u32, r); - r as usize - } + FieldsShape::Arbitrary { ref memory_index, .. } => memory_index[i].try_into().unwrap(), } } @@ -860,6 +855,7 @@ pub enum Abi { impl Abi { /// Returns `true` if the layout corresponds to an unsized type. + #[inline] pub fn is_unsized(&self) -> bool { match *self { Abi::Uninhabited | Abi::Scalar(_) | Abi::ScalarPair(..) | Abi::Vector { .. } => false, @@ -879,11 +875,13 @@ impl Abi { } /// Returns `true` if this is an uninhabited type + #[inline] pub fn is_uninhabited(&self) -> bool { matches!(*self, Abi::Uninhabited) } /// Returns `true` is this is a scalar type + #[inline] pub fn is_scalar(&self) -> bool { matches!(*self, Abi::Scalar(_)) } diff --git a/compiler/rustc_target/src/asm/aarch64.rs b/compiler/rustc_target/src/asm/aarch64.rs index e7c9edea76..f180eea01a 100644 --- a/compiler/rustc_target/src/asm/aarch64.rs +++ b/compiler/rustc_target/src/asm/aarch64.rs @@ -85,8 +85,6 @@ def_regs! { x15: reg = ["x15", "w15"], x16: reg = ["x16", "w16"], x17: reg = ["x17", "w17"], - x18: reg = ["x18", "w18"], - x19: reg = ["x19", "w19"], x20: reg = ["x20", "w20"], x21: reg = ["x21", "w21"], x22: reg = ["x22", "w22"], @@ -96,7 +94,7 @@ def_regs! { x26: reg = ["x26", "w26"], x27: reg = ["x27", "w27"], x28: reg = ["x28", "w28"], - x30: reg = ["x30", "w30", "lr"], + x30: reg = ["x30", "w30", "lr", "wlr"], v0: vreg, vreg_low16 = ["v0", "b0", "h0", "s0", "d0", "q0"], v1: vreg, vreg_low16 = ["v1", "b1", "h1", "s1", "d1", "q1"], v2: vreg, vreg_low16 = ["v2", "b2", "h2", "s2", "d2", "q2"], @@ -129,7 +127,11 @@ def_regs! { v29: vreg = ["v29", "b29", "h29", "s29", "d29", "q29"], v30: vreg = ["v30", "b30", "h30", "s30", "d30", "q30"], v31: vreg = ["v31", "b31", "h31", "s31", "d31", "q31"], - #error = ["x29", "fp"] => + #error = ["x18", "w18"] => + "x18 is used as a reserved register on some targets and cannot be used as an operand for inline asm", + #error = ["x19", "w19"] => + "x19 is used internally by LLVM and cannot be used as an operand for inline asm", + #error = ["x29", "w29", "fp", "wfp"] => "the frame pointer cannot be used as an operand for inline asm", #error = ["sp", "wsp"] => "the stack pointer cannot be used as an operand for inline asm", diff --git a/compiler/rustc_target/src/asm/arm.rs b/compiler/rustc_target/src/asm/arm.rs index a7a708fe7d..4c323fc35d 100644 --- a/compiler/rustc_target/src/asm/arm.rs +++ b/compiler/rustc_target/src/asm/arm.rs @@ -98,7 +98,6 @@ def_regs! { r5: reg, reg_thumb = ["r5", "v2"], r7: reg, reg_thumb = ["r7", "v4"] % frame_pointer_r7, r8: reg = ["r8", "v5"], - r9: reg = ["r9", "v6", "rfp"], r10: reg = ["r10", "sl"], r11: reg = ["r11", "fp"] % frame_pointer_r11, r12: reg = ["r12", "ip"], @@ -185,6 +184,8 @@ def_regs! { q15: qreg = ["q15"], #error = ["r6", "v3"] => "r6 is used internally by LLVM and cannot be used as an operand for inline asm", + #error = ["r9", "v6", "rfp"] => + "r9 is used internally by LLVM and cannot be used as an operand for inline asm", #error = ["r13", "sp"] => "the stack pointer cannot be used as an operand for inline asm", #error = ["r15", "pc"] => diff --git a/compiler/rustc_target/src/asm/bpf.rs b/compiler/rustc_target/src/asm/bpf.rs new file mode 100644 index 0000000000..ecb6bdc95c --- /dev/null +++ b/compiler/rustc_target/src/asm/bpf.rs @@ -0,0 +1,129 @@ +use super::{InlineAsmArch, InlineAsmType, Target}; +use rustc_macros::HashStable_Generic; +use std::fmt; + +def_reg_class! { + Bpf BpfInlineAsmRegClass { + reg, + wreg, + } +} + +impl BpfInlineAsmRegClass { + pub fn valid_modifiers(self, _arch: InlineAsmArch) -> &'static [char] { + &[] + } + + pub fn suggest_class(self, _arch: InlineAsmArch, _ty: InlineAsmType) -> Option { + None + } + + pub fn suggest_modifier( + self, + _arch: InlineAsmArch, + _ty: InlineAsmType, + ) -> Option<(char, &'static str)> { + None + } + + pub fn default_modifier(self, _arch: InlineAsmArch) -> Option<(char, &'static str)> { + None + } + + pub fn supported_types( + self, + _arch: InlineAsmArch, + ) -> &'static [(InlineAsmType, Option<&'static str>)] { + match self { + Self::reg => types! { _: I8, I16, I32, I64; }, + Self::wreg => types! { "alu32": I8, I16, I32; }, + } + } +} + +fn only_alu32( + _arch: InlineAsmArch, + mut has_feature: impl FnMut(&str) -> bool, + _target: &Target, +) -> Result<(), &'static str> { + if !has_feature("alu32") { + Err("register can't be used without the `alu32` target feature") + } else { + Ok(()) + } +} + +def_regs! { + Bpf BpfInlineAsmReg BpfInlineAsmRegClass { + r0: reg = ["r0"], + r1: reg = ["r1"], + r2: reg = ["r2"], + r3: reg = ["r3"], + r4: reg = ["r4"], + r5: reg = ["r5"], + r6: reg = ["r6"], + r7: reg = ["r7"], + r8: reg = ["r8"], + r9: reg = ["r9"], + w0: wreg = ["w0"] % only_alu32, + w1: wreg = ["w1"] % only_alu32, + w2: wreg = ["w2"] % only_alu32, + w3: wreg = ["w3"] % only_alu32, + w4: wreg = ["w4"] % only_alu32, + w5: wreg = ["w5"] % only_alu32, + w6: wreg = ["w6"] % only_alu32, + w7: wreg = ["w7"] % only_alu32, + w8: wreg = ["w8"] % only_alu32, + w9: wreg = ["w9"] % only_alu32, + + #error = ["r10", "w10"] => + "the stack pointer cannot be used as an operand for inline asm", + } +} + +impl BpfInlineAsmReg { + pub fn emit( + self, + out: &mut dyn fmt::Write, + _arch: InlineAsmArch, + _modifier: Option, + ) -> fmt::Result { + out.write_str(self.name()) + } + + pub fn overlapping_regs(self, mut cb: impl FnMut(BpfInlineAsmReg)) { + cb(self); + + macro_rules! reg_conflicts { + ( + $( + $r:ident : $w:ident + ),* + ) => { + match self { + $( + Self::$r => { + cb(Self::$w); + } + Self::$w => { + cb(Self::$r); + } + )* + } + }; + } + + reg_conflicts! { + r0 : w0, + r1 : w1, + r2 : w2, + r3 : w3, + r4 : w4, + r5 : w5, + r6 : w6, + r7 : w7, + r8 : w8, + r9 : w9 + } + } +} diff --git a/compiler/rustc_target/src/asm/hexagon.rs b/compiler/rustc_target/src/asm/hexagon.rs index d41941d0b4..74afddb69d 100644 --- a/compiler/rustc_target/src/asm/hexagon.rs +++ b/compiler/rustc_target/src/asm/hexagon.rs @@ -60,7 +60,6 @@ def_regs! { r16: reg = ["r16"], r17: reg = ["r17"], r18: reg = ["r18"], - r19: reg = ["r19"], r20: reg = ["r20"], r21: reg = ["r21"], r22: reg = ["r22"], @@ -70,6 +69,8 @@ def_regs! { r26: reg = ["r26"], r27: reg = ["r27"], r28: reg = ["r28"], + #error = ["r19"] => + "r19 is used internally by LLVM and cannot be used as an operand for inline asm", #error = ["r29", "sp"] => "the stack pointer cannot be used as an operand for inline asm", #error = ["r30", "fr"] => diff --git a/compiler/rustc_target/src/asm/mod.rs b/compiler/rustc_target/src/asm/mod.rs index e2268a61a4..305ea7d50e 100644 --- a/compiler/rustc_target/src/asm/mod.rs +++ b/compiler/rustc_target/src/asm/mod.rs @@ -6,7 +6,6 @@ use rustc_span::Symbol; use std::fmt; use std::str::FromStr; -#[macro_use] macro_rules! def_reg_class { ($arch:ident $arch_regclass:ident { $( @@ -51,7 +50,6 @@ macro_rules! def_reg_class { } } -#[macro_use] macro_rules! def_regs { ($arch:ident $arch_reg:ident $arch_regclass:ident { $( @@ -129,7 +127,6 @@ macro_rules! def_regs { } } -#[macro_use] macro_rules! types { ( $(_ : $($ty:expr),+;)? @@ -151,9 +148,11 @@ macro_rules! types { mod aarch64; mod arm; +mod bpf; mod hexagon; mod mips; mod nvptx; +mod powerpc; mod riscv; mod spirv; mod wasm; @@ -161,9 +160,11 @@ mod x86; pub use aarch64::{AArch64InlineAsmReg, AArch64InlineAsmRegClass}; pub use arm::{ArmInlineAsmReg, ArmInlineAsmRegClass}; +pub use bpf::{BpfInlineAsmReg, BpfInlineAsmRegClass}; pub use hexagon::{HexagonInlineAsmReg, HexagonInlineAsmRegClass}; pub use mips::{MipsInlineAsmReg, MipsInlineAsmRegClass}; pub use nvptx::{NvptxInlineAsmReg, NvptxInlineAsmRegClass}; +pub use powerpc::{PowerPCInlineAsmReg, PowerPCInlineAsmRegClass}; pub use riscv::{RiscVInlineAsmReg, RiscVInlineAsmRegClass}; pub use spirv::{SpirVInlineAsmReg, SpirVInlineAsmRegClass}; pub use wasm::{WasmInlineAsmReg, WasmInlineAsmRegClass}; @@ -181,8 +182,11 @@ pub enum InlineAsmArch { Hexagon, Mips, Mips64, + PowerPC, + PowerPC64, SpirV, Wasm32, + Bpf, } impl FromStr for InlineAsmArch { @@ -197,11 +201,14 @@ impl FromStr for InlineAsmArch { "riscv32" => Ok(Self::RiscV32), "riscv64" => Ok(Self::RiscV64), "nvptx64" => Ok(Self::Nvptx64), + "powerpc" => Ok(Self::PowerPC), + "powerpc64" => Ok(Self::PowerPC64), "hexagon" => Ok(Self::Hexagon), "mips" => Ok(Self::Mips), "mips64" => Ok(Self::Mips64), "spirv" => Ok(Self::SpirV), "wasm32" => Ok(Self::Wasm32), + "bpf" => Ok(Self::Bpf), _ => Err(()), } } @@ -225,10 +232,12 @@ pub enum InlineAsmReg { AArch64(AArch64InlineAsmReg), RiscV(RiscVInlineAsmReg), Nvptx(NvptxInlineAsmReg), + PowerPC(PowerPCInlineAsmReg), Hexagon(HexagonInlineAsmReg), Mips(MipsInlineAsmReg), SpirV(SpirVInlineAsmReg), Wasm(WasmInlineAsmReg), + Bpf(BpfInlineAsmReg), // Placeholder for invalid register constraints for the current target Err, } @@ -240,8 +249,10 @@ impl InlineAsmReg { Self::Arm(r) => r.name(), Self::AArch64(r) => r.name(), Self::RiscV(r) => r.name(), + Self::PowerPC(r) => r.name(), Self::Hexagon(r) => r.name(), Self::Mips(r) => r.name(), + Self::Bpf(r) => r.name(), Self::Err => "", } } @@ -252,8 +263,10 @@ impl InlineAsmReg { Self::Arm(r) => InlineAsmRegClass::Arm(r.reg_class()), Self::AArch64(r) => InlineAsmRegClass::AArch64(r.reg_class()), Self::RiscV(r) => InlineAsmRegClass::RiscV(r.reg_class()), + Self::PowerPC(r) => InlineAsmRegClass::PowerPC(r.reg_class()), Self::Hexagon(r) => InlineAsmRegClass::Hexagon(r.reg_class()), Self::Mips(r) => InlineAsmRegClass::Mips(r.reg_class()), + Self::Bpf(r) => InlineAsmRegClass::Bpf(r.reg_class()), Self::Err => InlineAsmRegClass::Err, } } @@ -283,6 +296,9 @@ impl InlineAsmReg { InlineAsmArch::Nvptx64 => { Self::Nvptx(NvptxInlineAsmReg::parse(arch, has_feature, target, &name)?) } + InlineAsmArch::PowerPC | InlineAsmArch::PowerPC64 => { + Self::PowerPC(PowerPCInlineAsmReg::parse(arch, has_feature, target, &name)?) + } InlineAsmArch::Hexagon => { Self::Hexagon(HexagonInlineAsmReg::parse(arch, has_feature, target, &name)?) } @@ -295,6 +311,9 @@ impl InlineAsmReg { InlineAsmArch::Wasm32 => { Self::Wasm(WasmInlineAsmReg::parse(arch, has_feature, target, &name)?) } + InlineAsmArch::Bpf => { + Self::Bpf(BpfInlineAsmReg::parse(arch, has_feature, target, &name)?) + } }) } @@ -311,8 +330,10 @@ impl InlineAsmReg { Self::Arm(r) => r.emit(out, arch, modifier), Self::AArch64(r) => r.emit(out, arch, modifier), Self::RiscV(r) => r.emit(out, arch, modifier), + Self::PowerPC(r) => r.emit(out, arch, modifier), Self::Hexagon(r) => r.emit(out, arch, modifier), Self::Mips(r) => r.emit(out, arch, modifier), + Self::Bpf(r) => r.emit(out, arch, modifier), Self::Err => unreachable!("Use of InlineAsmReg::Err"), } } @@ -323,8 +344,10 @@ impl InlineAsmReg { Self::Arm(r) => r.overlapping_regs(|r| cb(Self::Arm(r))), Self::AArch64(_) => cb(self), Self::RiscV(_) => cb(self), + Self::PowerPC(_) => cb(self), Self::Hexagon(r) => r.overlapping_regs(|r| cb(Self::Hexagon(r))), Self::Mips(_) => cb(self), + Self::Bpf(r) => r.overlapping_regs(|r| cb(Self::Bpf(r))), Self::Err => unreachable!("Use of InlineAsmReg::Err"), } } @@ -348,10 +371,12 @@ pub enum InlineAsmRegClass { AArch64(AArch64InlineAsmRegClass), RiscV(RiscVInlineAsmRegClass), Nvptx(NvptxInlineAsmRegClass), + PowerPC(PowerPCInlineAsmRegClass), Hexagon(HexagonInlineAsmRegClass), Mips(MipsInlineAsmRegClass), SpirV(SpirVInlineAsmRegClass), Wasm(WasmInlineAsmRegClass), + Bpf(BpfInlineAsmRegClass), // Placeholder for invalid register constraints for the current target Err, } @@ -364,10 +389,12 @@ impl InlineAsmRegClass { Self::AArch64(r) => r.name(), Self::RiscV(r) => r.name(), Self::Nvptx(r) => r.name(), + Self::PowerPC(r) => r.name(), Self::Hexagon(r) => r.name(), Self::Mips(r) => r.name(), Self::SpirV(r) => r.name(), Self::Wasm(r) => r.name(), + Self::Bpf(r) => r.name(), Self::Err => rustc_span::symbol::sym::reg, } } @@ -382,10 +409,12 @@ impl InlineAsmRegClass { Self::AArch64(r) => r.suggest_class(arch, ty).map(InlineAsmRegClass::AArch64), Self::RiscV(r) => r.suggest_class(arch, ty).map(InlineAsmRegClass::RiscV), Self::Nvptx(r) => r.suggest_class(arch, ty).map(InlineAsmRegClass::Nvptx), + Self::PowerPC(r) => r.suggest_class(arch, ty).map(InlineAsmRegClass::PowerPC), Self::Hexagon(r) => r.suggest_class(arch, ty).map(InlineAsmRegClass::Hexagon), Self::Mips(r) => r.suggest_class(arch, ty).map(InlineAsmRegClass::Mips), Self::SpirV(r) => r.suggest_class(arch, ty).map(InlineAsmRegClass::SpirV), Self::Wasm(r) => r.suggest_class(arch, ty).map(InlineAsmRegClass::Wasm), + Self::Bpf(r) => r.suggest_class(arch, ty).map(InlineAsmRegClass::Bpf), Self::Err => unreachable!("Use of InlineAsmRegClass::Err"), } } @@ -407,10 +436,12 @@ impl InlineAsmRegClass { Self::AArch64(r) => r.suggest_modifier(arch, ty), Self::RiscV(r) => r.suggest_modifier(arch, ty), Self::Nvptx(r) => r.suggest_modifier(arch, ty), + Self::PowerPC(r) => r.suggest_modifier(arch, ty), Self::Hexagon(r) => r.suggest_modifier(arch, ty), Self::Mips(r) => r.suggest_modifier(arch, ty), Self::SpirV(r) => r.suggest_modifier(arch, ty), Self::Wasm(r) => r.suggest_modifier(arch, ty), + Self::Bpf(r) => r.suggest_modifier(arch, ty), Self::Err => unreachable!("Use of InlineAsmRegClass::Err"), } } @@ -428,10 +459,12 @@ impl InlineAsmRegClass { Self::AArch64(r) => r.default_modifier(arch), Self::RiscV(r) => r.default_modifier(arch), Self::Nvptx(r) => r.default_modifier(arch), + Self::PowerPC(r) => r.default_modifier(arch), Self::Hexagon(r) => r.default_modifier(arch), Self::Mips(r) => r.default_modifier(arch), Self::SpirV(r) => r.default_modifier(arch), Self::Wasm(r) => r.default_modifier(arch), + Self::Bpf(r) => r.default_modifier(arch), Self::Err => unreachable!("Use of InlineAsmRegClass::Err"), } } @@ -448,10 +481,12 @@ impl InlineAsmRegClass { Self::AArch64(r) => r.supported_types(arch), Self::RiscV(r) => r.supported_types(arch), Self::Nvptx(r) => r.supported_types(arch), + Self::PowerPC(r) => r.supported_types(arch), Self::Hexagon(r) => r.supported_types(arch), Self::Mips(r) => r.supported_types(arch), Self::SpirV(r) => r.supported_types(arch), Self::Wasm(r) => r.supported_types(arch), + Self::Bpf(r) => r.supported_types(arch), Self::Err => unreachable!("Use of InlineAsmRegClass::Err"), } } @@ -467,12 +502,16 @@ impl InlineAsmRegClass { Self::RiscV(RiscVInlineAsmRegClass::parse(arch, name)?) } InlineAsmArch::Nvptx64 => Self::Nvptx(NvptxInlineAsmRegClass::parse(arch, name)?), + InlineAsmArch::PowerPC | InlineAsmArch::PowerPC64 => { + Self::PowerPC(PowerPCInlineAsmRegClass::parse(arch, name)?) + } InlineAsmArch::Hexagon => Self::Hexagon(HexagonInlineAsmRegClass::parse(arch, name)?), InlineAsmArch::Mips | InlineAsmArch::Mips64 => { Self::Mips(MipsInlineAsmRegClass::parse(arch, name)?) } InlineAsmArch::SpirV => Self::SpirV(SpirVInlineAsmRegClass::parse(arch, name)?), InlineAsmArch::Wasm32 => Self::Wasm(WasmInlineAsmRegClass::parse(arch, name)?), + InlineAsmArch::Bpf => Self::Bpf(BpfInlineAsmRegClass::parse(arch, name)?), }) } @@ -485,10 +524,12 @@ impl InlineAsmRegClass { Self::AArch64(r) => r.valid_modifiers(arch), Self::RiscV(r) => r.valid_modifiers(arch), Self::Nvptx(r) => r.valid_modifiers(arch), + Self::PowerPC(r) => r.valid_modifiers(arch), Self::Hexagon(r) => r.valid_modifiers(arch), Self::Mips(r) => r.valid_modifiers(arch), Self::SpirV(r) => r.valid_modifiers(arch), Self::Wasm(r) => r.valid_modifiers(arch), + Self::Bpf(r) => r.valid_modifiers(arch), Self::Err => unreachable!("Use of InlineAsmRegClass::Err"), } } @@ -633,6 +674,11 @@ pub fn allocatable_registers( nvptx::fill_reg_map(arch, has_feature, target, &mut map); map } + InlineAsmArch::PowerPC | InlineAsmArch::PowerPC64 => { + let mut map = powerpc::regclass_map(); + powerpc::fill_reg_map(arch, has_feature, target, &mut map); + map + } InlineAsmArch::Hexagon => { let mut map = hexagon::regclass_map(); hexagon::fill_reg_map(arch, has_feature, target, &mut map); @@ -653,5 +699,10 @@ pub fn allocatable_registers( wasm::fill_reg_map(arch, has_feature, target, &mut map); map } + InlineAsmArch::Bpf => { + let mut map = bpf::regclass_map(); + bpf::fill_reg_map(arch, has_feature, target, &mut map); + map + } } } diff --git a/compiler/rustc_target/src/asm/powerpc.rs b/compiler/rustc_target/src/asm/powerpc.rs new file mode 100644 index 0000000000..42fc25c4ff --- /dev/null +++ b/compiler/rustc_target/src/asm/powerpc.rs @@ -0,0 +1,152 @@ +use super::{InlineAsmArch, InlineAsmType}; +use rustc_macros::HashStable_Generic; +use std::fmt; + +def_reg_class! { + PowerPC PowerPCInlineAsmRegClass { + reg, + reg_nonzero, + freg, + } +} + +impl PowerPCInlineAsmRegClass { + pub fn valid_modifiers(self, _arch: super::InlineAsmArch) -> &'static [char] { + &[] + } + + pub fn suggest_class(self, _arch: InlineAsmArch, _ty: InlineAsmType) -> Option { + None + } + + pub fn suggest_modifier( + self, + _arch: InlineAsmArch, + _ty: InlineAsmType, + ) -> Option<(char, &'static str)> { + None + } + + pub fn default_modifier(self, _arch: InlineAsmArch) -> Option<(char, &'static str)> { + None + } + + pub fn supported_types( + self, + arch: InlineAsmArch, + ) -> &'static [(InlineAsmType, Option<&'static str>)] { + match self { + Self::reg | Self::reg_nonzero => { + if arch == InlineAsmArch::PowerPC { + types! { _: I8, I16, I32; } + } else { + types! { _: I8, I16, I32, I64; } + } + } + Self::freg => types! { _: F32, F64; }, + } + } +} + +def_regs! { + PowerPC PowerPCInlineAsmReg PowerPCInlineAsmRegClass { + r0: reg = ["r0", "0"], + r3: reg, reg_nonzero = ["r3", "3"], + r4: reg, reg_nonzero = ["r4", "4"], + r5: reg, reg_nonzero = ["r5", "5"], + r6: reg, reg_nonzero = ["r6", "6"], + r7: reg, reg_nonzero = ["r7", "7"], + r8: reg, reg_nonzero = ["r8", "8"], + r9: reg, reg_nonzero = ["r9", "9"], + r10: reg, reg_nonzero = ["r10", "10"], + r11: reg, reg_nonzero = ["r11", "11"], + r12: reg, reg_nonzero = ["r12", "12"], + r14: reg, reg_nonzero = ["r14", "14"], + r15: reg, reg_nonzero = ["r15", "15"], + r16: reg, reg_nonzero = ["r16", "16"], + r17: reg, reg_nonzero = ["r17", "17"], + r18: reg, reg_nonzero = ["r18", "18"], + r19: reg, reg_nonzero = ["r19", "19"], + r20: reg, reg_nonzero = ["r20", "20"], + r21: reg, reg_nonzero = ["r21", "21"], + r22: reg, reg_nonzero = ["r22", "22"], + r23: reg, reg_nonzero = ["r23", "23"], + r24: reg, reg_nonzero = ["r24", "24"], + r25: reg, reg_nonzero = ["r25", "25"], + r26: reg, reg_nonzero = ["r26", "26"], + r27: reg, reg_nonzero = ["r27", "27"], + r28: reg, reg_nonzero = ["r28", "28"], + f0: freg = ["f0", "fr0"], + f1: freg = ["f1", "fr1"], + f2: freg = ["f2", "fr2"], + f3: freg = ["f3", "fr3"], + f4: freg = ["f4", "fr4"], + f5: freg = ["f5", "fr5"], + f6: freg = ["f6", "fr6"], + f7: freg = ["f7", "fr7"], + f8: freg = ["f8", "fr8"], + f9: freg = ["f9", "fr9"], + f10: freg = ["f10", "fr10"], + f11: freg = ["f11", "fr11"], + f12: freg = ["f12", "fr12"], + f13: freg = ["f13", "fr13"], + f14: freg = ["f14", "fr14"], + f15: freg = ["f15", "fr15"], + f16: freg = ["f16", "fr16"], + f17: freg = ["f17", "fr17"], + f18: freg = ["f18", "fr18"], + f19: freg = ["f19", "fr19"], + f20: freg = ["f20", "fr20"], + f21: freg = ["f21", "fr21"], + f22: freg = ["f22", "fr22"], + f23: freg = ["f23", "fr23"], + f24: freg = ["f24", "fr24"], + f25: freg = ["f25", "fr25"], + f26: freg = ["f26", "fr26"], + f27: freg = ["f27", "fr27"], + f28: freg = ["f28", "fr28"], + f29: freg = ["f29", "fr29"], + f30: freg = ["f30", "fr30"], + f31: freg = ["f31", "fr31"], + #error = ["r1", "1", "sp"] => + "the stack pointer cannot be used as an operand for inline asm", + #error = ["r2", "2"] => + "r2 is a system reserved register and cannot be used as an operand for inline asm", + #error = ["r13", "13"] => + "r13 is a system reserved register and cannot be used as an operand for inline asm", + #error = ["r29", "29"] => + "r29 is used internally by LLVM and cannot be used as an operand for inline asm", + #error = ["r30", "30"] => + "r30 is used internally by LLVM and cannot be used as an operand for inline asm", + #error = ["r31", "31", "fp"] => + "the frame pointer cannot be used as an operand for inline asm", + #error = ["lr"] => + "the link register cannot be used as an operand for inline asm", + #error = ["ctr"] => + "the counter register cannot be used as an operand for inline asm", + #error = ["vrsave"] => + "the vrsave register cannot be used as an operand for inline asm", + } +} + +impl PowerPCInlineAsmReg { + pub fn emit( + self, + out: &mut dyn fmt::Write, + _arch: InlineAsmArch, + _modifier: Option, + ) -> fmt::Result { + // Strip off the leading prefix. + if self as u32 <= Self::r28 as u32 { + let index = self as u32 - Self::r28 as u32; + write!(out, "{}", index) + } else if self as u32 >= Self::f0 as u32 && self as u32 <= Self::f31 as u32 { + let index = self as u32 - Self::f31 as u32; + write!(out, "{}", index) + } else { + unreachable!() + } + } + + pub fn overlapping_regs(self, mut _cb: impl FnMut(PowerPCInlineAsmReg)) {} +} diff --git a/compiler/rustc_target/src/asm/riscv.rs b/compiler/rustc_target/src/asm/riscv.rs index 185d6ac824..e276a9175f 100644 --- a/compiler/rustc_target/src/asm/riscv.rs +++ b/compiler/rustc_target/src/asm/riscv.rs @@ -66,7 +66,6 @@ def_regs! { x5: reg = ["x5", "t0"], x6: reg = ["x6", "t1"], x7: reg = ["x7", "t2"], - x9: reg = ["x9", "s1"], x10: reg = ["x10", "a0"], x11: reg = ["x11", "a1"], x12: reg = ["x12", "a2"], @@ -121,6 +120,8 @@ def_regs! { f29: freg = ["f29", "ft9"], f30: freg = ["f30", "ft10"], f31: freg = ["f31", "ft11"], + #error = ["x9", "s1"] => + "s1 is used internally by LLVM and cannot be used as an operand for inline asm", #error = ["x8", "s0", "fp"] => "the frame pointer cannot be used as an operand for inline asm", #error = ["x2", "sp"] => diff --git a/compiler/rustc_target/src/asm/x86.rs b/compiler/rustc_target/src/asm/x86.rs index 90660dad4c..48f83ca7cd 100644 --- a/compiler/rustc_target/src/asm/x86.rs +++ b/compiler/rustc_target/src/asm/x86.rs @@ -152,13 +152,41 @@ fn high_byte( } } +fn rbx_reserved( + arch: InlineAsmArch, + _has_feature: impl FnMut(&str) -> bool, + _target: &Target, +) -> Result<(), &'static str> { + match arch { + InlineAsmArch::X86 => Ok(()), + InlineAsmArch::X86_64 => { + Err("rbx is used internally by LLVM and cannot be used as an operand for inline asm") + } + _ => unreachable!(), + } +} + +fn esi_reserved( + arch: InlineAsmArch, + _has_feature: impl FnMut(&str) -> bool, + _target: &Target, +) -> Result<(), &'static str> { + match arch { + InlineAsmArch::X86 => { + Err("esi is used internally by LLVM and cannot be used as an operand for inline asm") + } + InlineAsmArch::X86_64 => Ok(()), + _ => unreachable!(), + } +} + def_regs! { X86 X86InlineAsmReg X86InlineAsmRegClass { ax: reg, reg_abcd = ["ax", "eax", "rax"], - bx: reg, reg_abcd = ["bx", "ebx", "rbx"], + bx: reg, reg_abcd = ["bx", "ebx", "rbx"] % rbx_reserved, cx: reg, reg_abcd = ["cx", "ecx", "rcx"], dx: reg, reg_abcd = ["dx", "edx", "rdx"], - si: reg = ["si", "esi", "rsi"], + si: reg = ["si", "esi", "rsi"] % esi_reserved, di: reg = ["di", "edi", "rdi"], r8: reg = ["r8", "r8w", "r8d"] % x86_64_only, r9: reg = ["r9", "r9w", "r9d"] % x86_64_only, diff --git a/compiler/rustc_target/src/lib.rs b/compiler/rustc_target/src/lib.rs index 6702538874..cb8f6b9656 100644 --- a/compiler/rustc_target/src/lib.rs +++ b/compiler/rustc_target/src/lib.rs @@ -9,11 +9,13 @@ #![doc(html_root_url = "https://doc.rust-lang.org/nightly/nightly-rustc/")] #![feature(bool_to_option)] -#![feature(const_panic)] #![feature(nll)] #![feature(never_type)] #![feature(associated_type_bounds)] #![feature(exhaustive_patterns)] +#![feature(min_specialization)] + +use std::path::{Path, PathBuf}; #[macro_use] extern crate rustc_macros; @@ -29,3 +31,52 @@ pub mod spec; /// This is a hack to allow using the `HashStable_Generic` derive macro /// instead of implementing everything in `rustc_middle`. pub trait HashStableContext {} + +/// The name of rustc's own place to organize libraries. +/// +/// Used to be `rustc`, now the default is `rustlib`. +const RUST_LIB_DIR: &str = "rustlib"; + +/// Returns a `rustlib` path for this particular target, relative to the provided sysroot. +/// +/// For example: `target_sysroot_path("/usr", "x86_64-unknown-linux-gnu")` => +/// `"lib*/rustlib/x86_64-unknown-linux-gnu"`. +pub fn target_rustlib_path(sysroot: &Path, target_triple: &str) -> PathBuf { + let libdir = find_libdir(sysroot); + std::array::IntoIter::new([ + Path::new(libdir.as_ref()), + Path::new(RUST_LIB_DIR), + Path::new(target_triple), + ]) + .collect::() +} + +/// The name of the directory rustc expects libraries to be located. +fn find_libdir(sysroot: &Path) -> std::borrow::Cow<'static, str> { + // FIXME: This is a quick hack to make the rustc binary able to locate + // Rust libraries in Linux environments where libraries might be installed + // to lib64/lib32. This would be more foolproof by basing the sysroot off + // of the directory where `librustc_driver` is located, rather than + // where the rustc binary is. + // If --libdir is set during configuration to the value other than + // "lib" (i.e., non-default), this value is used (see issue #16552). + + #[cfg(target_pointer_width = "64")] + const PRIMARY_LIB_DIR: &str = "lib64"; + + #[cfg(target_pointer_width = "32")] + const PRIMARY_LIB_DIR: &str = "lib32"; + + const SECONDARY_LIB_DIR: &str = "lib"; + + match option_env!("CFG_LIBDIR_RELATIVE") { + None | Some("lib") => { + if sysroot.join(PRIMARY_LIB_DIR).join(RUST_LIB_DIR).exists() { + PRIMARY_LIB_DIR.into() + } else { + SECONDARY_LIB_DIR.into() + } + } + Some(libdir) => libdir.into(), + } +} diff --git a/compiler/rustc_target/src/spec/aarch64_apple_ios.rs b/compiler/rustc_target/src/spec/aarch64_apple_ios.rs index 2218c6c6da..5682039b86 100644 --- a/compiler/rustc_target/src/spec/aarch64_apple_ios.rs +++ b/compiler/rustc_target/src/spec/aarch64_apple_ios.rs @@ -10,7 +10,6 @@ pub fn target() -> Target { arch: "aarch64".to_string(), options: TargetOptions { features: "+neon,+fp-armv8,+apple-a7".to_string(), - eliminate_frame_pointer: false, max_atomic_width: Some(128), unsupported_abis: super::arm_base::unsupported_abis(), forces_embed_bitcode: true, diff --git a/compiler/rustc_target/src/spec/aarch64_apple_ios_macabi.rs b/compiler/rustc_target/src/spec/aarch64_apple_ios_macabi.rs index 758950bd34..8a832546d0 100644 --- a/compiler/rustc_target/src/spec/aarch64_apple_ios_macabi.rs +++ b/compiler/rustc_target/src/spec/aarch64_apple_ios_macabi.rs @@ -10,7 +10,6 @@ pub fn target() -> Target { arch: "aarch64".to_string(), options: TargetOptions { features: "+neon,+fp-armv8,+apple-a12".to_string(), - eliminate_frame_pointer: false, max_atomic_width: Some(128), unsupported_abis: super::arm_base::unsupported_abis(), forces_embed_bitcode: true, diff --git a/compiler/rustc_target/src/spec/aarch64_apple_ios_sim.rs b/compiler/rustc_target/src/spec/aarch64_apple_ios_sim.rs index e594ceec1b..2187015b62 100644 --- a/compiler/rustc_target/src/spec/aarch64_apple_ios_sim.rs +++ b/compiler/rustc_target/src/spec/aarch64_apple_ios_sim.rs @@ -18,7 +18,6 @@ pub fn target() -> Target { arch: "aarch64".to_string(), options: TargetOptions { features: "+neon,+fp-armv8,+apple-a7".to_string(), - eliminate_frame_pointer: false, max_atomic_width: Some(128), unsupported_abis: super::arm_base::unsupported_abis(), forces_embed_bitcode: true, diff --git a/compiler/rustc_target/src/spec/aarch64_apple_tvos.rs b/compiler/rustc_target/src/spec/aarch64_apple_tvos.rs index a83de77dc2..cb6c06b371 100644 --- a/compiler/rustc_target/src/spec/aarch64_apple_tvos.rs +++ b/compiler/rustc_target/src/spec/aarch64_apple_tvos.rs @@ -10,7 +10,6 @@ pub fn target() -> Target { arch: "aarch64".to_string(), options: TargetOptions { features: "+neon,+fp-armv8,+apple-a7".to_string(), - eliminate_frame_pointer: false, max_atomic_width: Some(128), unsupported_abis: super::arm_base::unsupported_abis(), forces_embed_bitcode: true, diff --git a/compiler/rustc_target/src/spec/aarch64_unknown_none.rs b/compiler/rustc_target/src/spec/aarch64_unknown_none.rs index c9f622820d..de92b167f0 100644 --- a/compiler/rustc_target/src/spec/aarch64_unknown_none.rs +++ b/compiler/rustc_target/src/spec/aarch64_unknown_none.rs @@ -16,7 +16,6 @@ pub fn target() -> Target { executables: true, relocation_model: RelocModel::Static, disable_redzone: true, - linker_is_gnu: true, max_atomic_width: Some(128), panic_strategy: PanicStrategy::Abort, unsupported_abis: super::arm_base::unsupported_abis(), diff --git a/compiler/rustc_target/src/spec/aarch64_unknown_none_softfloat.rs b/compiler/rustc_target/src/spec/aarch64_unknown_none_softfloat.rs index 0811871c99..2566eeae14 100644 --- a/compiler/rustc_target/src/spec/aarch64_unknown_none_softfloat.rs +++ b/compiler/rustc_target/src/spec/aarch64_unknown_none_softfloat.rs @@ -16,7 +16,6 @@ pub fn target() -> Target { executables: true, relocation_model: RelocModel::Static, disable_redzone: true, - linker_is_gnu: true, max_atomic_width: Some(128), panic_strategy: PanicStrategy::Abort, unsupported_abis: super::arm_base::unsupported_abis(), diff --git a/compiler/rustc_target/src/spec/apple_base.rs b/compiler/rustc_target/src/spec/apple_base.rs index 6fa0b34545..8530db179d 100644 --- a/compiler/rustc_target/src/spec/apple_base.rs +++ b/compiler/rustc_target/src/spec/apple_base.rs @@ -22,10 +22,12 @@ pub fn opts(os: &str) -> TargetOptions { // macOS has -dead_strip, which doesn't rely on function_sections function_sections: false, dynamic_linking: true, + linker_is_gnu: false, executables: true, - os_family: Some("unix".to_string()), + families: vec!["unix".to_string()], is_like_osx: true, dwarf_version: Some(2), + eliminate_frame_pointer: false, has_rpath: true, dll_suffix: ".dylib".to_string(), archive_format: "darwin".to_string(), diff --git a/compiler/rustc_target/src/spec/apple_sdk_base.rs b/compiler/rustc_target/src/spec/apple_sdk_base.rs index 538c4ca869..e7f7bb343d 100644 --- a/compiler/rustc_target/src/spec/apple_sdk_base.rs +++ b/compiler/rustc_target/src/spec/apple_sdk_base.rs @@ -44,7 +44,6 @@ pub fn opts(os: &str, arch: Arch) -> TargetOptions { executables: true, link_env_remove: link_env_remove(arch), has_elf_tls: false, - eliminate_frame_pointer: false, ..super::apple_base::opts(os) } } diff --git a/compiler/rustc_target/src/spec/avr_gnu_base.rs b/compiler/rustc_target/src/spec/avr_gnu_base.rs index 69ccce875a..2cb2661a52 100644 --- a/compiler/rustc_target/src/spec/avr_gnu_base.rs +++ b/compiler/rustc_target/src/spec/avr_gnu_base.rs @@ -16,7 +16,6 @@ pub fn target(target_cpu: String) -> Target { linker: Some("avr-gcc".to_owned()), executables: true, - linker_is_gnu: true, eh_frame_header: false, pre_link_args: vec![(LinkerFlavor::Gcc, vec![format!("-mmcu={}", target_cpu)])] .into_iter() diff --git a/compiler/rustc_target/src/spec/bpf_base.rs b/compiler/rustc_target/src/spec/bpf_base.rs new file mode 100644 index 0000000000..764cc735d7 --- /dev/null +++ b/compiler/rustc_target/src/spec/bpf_base.rs @@ -0,0 +1,42 @@ +use crate::spec::{LinkerFlavor, MergeFunctions, PanicStrategy, TargetOptions}; +use crate::{abi::Endian, spec::abi::Abi}; + +pub fn opts(endian: Endian) -> TargetOptions { + TargetOptions { + allow_asm: true, + endian, + linker_flavor: LinkerFlavor::BpfLinker, + atomic_cas: false, + executables: true, + dynamic_linking: true, + no_builtins: true, + panic_strategy: PanicStrategy::Abort, + position_independent_executables: true, + // Disable MergeFunctions since: + // - older kernels don't support bpf-to-bpf calls + // - on newer kernels, userspace still needs to relocate before calling + // BPF_PROG_LOAD and not all BPF libraries do that yet + merge_functions: MergeFunctions::Disabled, + obj_is_bitcode: true, + requires_lto: false, + singlethread: true, + max_atomic_width: Some(64), + unsupported_abis: vec![ + Abi::Cdecl, + Abi::Stdcall { unwind: false }, + Abi::Stdcall { unwind: true }, + Abi::Fastcall, + Abi::Vectorcall, + Abi::Thiscall { unwind: false }, + Abi::Thiscall { unwind: true }, + Abi::Aapcs, + Abi::Win64, + Abi::SysV64, + Abi::PtxKernel, + Abi::Msp430Interrupt, + Abi::X86Interrupt, + Abi::AmdGpuKernel, + ], + ..Default::default() + } +} diff --git a/compiler/rustc_target/src/spec/bpfeb_unknown_none.rs b/compiler/rustc_target/src/spec/bpfeb_unknown_none.rs new file mode 100644 index 0000000000..a45da82eb4 --- /dev/null +++ b/compiler/rustc_target/src/spec/bpfeb_unknown_none.rs @@ -0,0 +1,12 @@ +use crate::spec::Target; +use crate::{abi::Endian, spec::bpf_base}; + +pub fn target() -> Target { + Target { + llvm_target: "bpfeb".to_string(), + data_layout: "E-m:e-p:64:64-i64:64-i128:128-n32:64-S128".to_string(), + pointer_width: 64, + arch: "bpf".to_string(), + options: bpf_base::opts(Endian::Big), + } +} diff --git a/compiler/rustc_target/src/spec/bpfel_unknown_none.rs b/compiler/rustc_target/src/spec/bpfel_unknown_none.rs new file mode 100644 index 0000000000..6c9afdf35a --- /dev/null +++ b/compiler/rustc_target/src/spec/bpfel_unknown_none.rs @@ -0,0 +1,12 @@ +use crate::spec::Target; +use crate::{abi::Endian, spec::bpf_base}; + +pub fn target() -> Target { + Target { + llvm_target: "bpfel".to_string(), + data_layout: "e-m:e-p:64:64-i64:64-i128:128-n32:64-S128".to_string(), + pointer_width: 64, + arch: "bpf".to_string(), + options: bpf_base::opts(Endian::Little), + } +} diff --git a/compiler/rustc_target/src/spec/dragonfly_base.rs b/compiler/rustc_target/src/spec/dragonfly_base.rs index dd01709878..e13a640d4d 100644 --- a/compiler/rustc_target/src/spec/dragonfly_base.rs +++ b/compiler/rustc_target/src/spec/dragonfly_base.rs @@ -5,8 +5,7 @@ pub fn opts() -> TargetOptions { os: "dragonfly".to_string(), dynamic_linking: true, executables: true, - os_family: Some("unix".to_string()), - linker_is_gnu: true, + families: vec!["unix".to_string()], has_rpath: true, position_independent_executables: true, relro_level: RelroLevel::Full, diff --git a/compiler/rustc_target/src/spec/freebsd_base.rs b/compiler/rustc_target/src/spec/freebsd_base.rs index ad3383cc5f..bef2fce7c8 100644 --- a/compiler/rustc_target/src/spec/freebsd_base.rs +++ b/compiler/rustc_target/src/spec/freebsd_base.rs @@ -5,8 +5,7 @@ pub fn opts() -> TargetOptions { os: "freebsd".to_string(), dynamic_linking: true, executables: true, - os_family: Some("unix".to_string()), - linker_is_gnu: true, + families: vec!["unix".to_string()], has_rpath: true, position_independent_executables: true, eliminate_frame_pointer: false, // FIXME 43575 diff --git a/compiler/rustc_target/src/spec/fuchsia_base.rs b/compiler/rustc_target/src/spec/fuchsia_base.rs index 2b925f8b94..23a2e65749 100644 --- a/compiler/rustc_target/src/spec/fuchsia_base.rs +++ b/compiler/rustc_target/src/spec/fuchsia_base.rs @@ -25,9 +25,8 @@ pub fn opts() -> TargetOptions { linker: Some("rust-lld".to_owned()), dynamic_linking: true, executables: true, - os_family: Some("unix".to_string()), + families: vec!["unix".to_string()], is_like_fuchsia: true, - linker_is_gnu: true, pre_link_args, pre_link_objects: crt_objects::new(&[ (LinkOutputKind::DynamicNoPicExe, &["Scrt1.o"]), diff --git a/compiler/rustc_target/src/spec/haiku_base.rs b/compiler/rustc_target/src/spec/haiku_base.rs index 956e4ed4bf..2b95523d6f 100644 --- a/compiler/rustc_target/src/spec/haiku_base.rs +++ b/compiler/rustc_target/src/spec/haiku_base.rs @@ -5,9 +5,8 @@ pub fn opts() -> TargetOptions { os: "haiku".to_string(), dynamic_linking: true, executables: true, - os_family: Some("unix".to_string()), + families: vec!["unix".to_string()], relro_level: RelroLevel::Full, - linker_is_gnu: true, ..Default::default() } } diff --git a/compiler/rustc_target/src/spec/hermit_base.rs b/compiler/rustc_target/src/spec/hermit_base.rs index ad013047e6..75ca1f79b1 100644 --- a/compiler/rustc_target/src/spec/hermit_base.rs +++ b/compiler/rustc_target/src/spec/hermit_base.rs @@ -13,7 +13,6 @@ pub fn opts() -> TargetOptions { linker: Some("rust-lld".to_owned()), executables: true, has_elf_tls: true, - linker_is_gnu: true, pre_link_args, panic_strategy: PanicStrategy::Abort, position_independent_executables: true, diff --git a/compiler/rustc_target/src/spec/hermit_kernel_base.rs b/compiler/rustc_target/src/spec/hermit_kernel_base.rs index 6d18a14d6a..c55a46e69a 100644 --- a/compiler/rustc_target/src/spec/hermit_kernel_base.rs +++ b/compiler/rustc_target/src/spec/hermit_kernel_base.rs @@ -14,7 +14,6 @@ pub fn opts() -> TargetOptions { linker: Some("rust-lld".to_owned()), executables: true, has_elf_tls: true, - linker_is_gnu: true, pre_link_args, panic_strategy: PanicStrategy::Abort, position_independent_executables: true, diff --git a/compiler/rustc_target/src/spec/illumos_base.rs b/compiler/rustc_target/src/spec/illumos_base.rs index d9b5716c04..9d9da50be7 100644 --- a/compiler/rustc_target/src/spec/illumos_base.rs +++ b/compiler/rustc_target/src/spec/illumos_base.rs @@ -6,6 +6,17 @@ pub fn opts() -> TargetOptions { late_link_args.insert( LinkerFlavor::Gcc, vec![ + // The illumos libc contains a stack unwinding implementation, as + // does libgcc_s. The latter implementation includes several + // additional symbols that are not always in base libc. To force + // the consistent use of just one unwinder, we ensure libc appears + // after libgcc_s in the NEEDED list for the resultant binary by + // ignoring any attempts to add it as a dynamic dependency until the + // very end. + // FIXME: This should be replaced by a more complete and generic + // mechanism for controlling the order of library arguments passed + // to the linker. + "-lc".to_string(), // LLVM will insert calls to the stack protector functions // "__stack_chk_fail" and "__stack_chk_guard" into code in native // object files. Some platforms include these symbols directly in @@ -20,8 +31,9 @@ pub fn opts() -> TargetOptions { dynamic_linking: true, executables: true, has_rpath: true, - os_family: Some("unix".to_string()), + families: vec!["unix".to_string()], is_like_solaris: true, + linker_is_gnu: false, limit_rdylib_exports: false, // Linker doesn't support this eliminate_frame_pointer: false, eh_frame_header: false, diff --git a/compiler/rustc_target/src/spec/l4re_base.rs b/compiler/rustc_target/src/spec/l4re_base.rs index db6b74eff6..f6e3102f61 100644 --- a/compiler/rustc_target/src/spec/l4re_base.rs +++ b/compiler/rustc_target/src/spec/l4re_base.rs @@ -20,7 +20,8 @@ pub fn opts() -> TargetOptions { executables: true, panic_strategy: PanicStrategy::Abort, linker: Some("ld".to_string()), - os_family: Some("unix".to_string()), + linker_is_gnu: false, + families: vec!["unix".to_string()], ..Default::default() } } diff --git a/compiler/rustc_target/src/spec/linux_base.rs b/compiler/rustc_target/src/spec/linux_base.rs index eeefd056e4..af81bc714c 100644 --- a/compiler/rustc_target/src/spec/linux_base.rs +++ b/compiler/rustc_target/src/spec/linux_base.rs @@ -5,8 +5,7 @@ pub fn opts() -> TargetOptions { os: "linux".to_string(), dynamic_linking: true, executables: true, - os_family: Some("unix".to_string()), - linker_is_gnu: true, + families: vec!["unix".to_string()], has_rpath: true, position_independent_executables: true, relro_level: RelroLevel::Full, diff --git a/compiler/rustc_target/src/spec/linux_kernel_base.rs b/compiler/rustc_target/src/spec/linux_kernel_base.rs index 64f47b4aa9..145aa4a589 100644 --- a/compiler/rustc_target/src/spec/linux_kernel_base.rs +++ b/compiler/rustc_target/src/spec/linux_kernel_base.rs @@ -8,7 +8,6 @@ pub fn opts() -> TargetOptions { // don't use probe-stack=inline-asm until rust#83139 and rust#84667 are resolved stack_probes: StackProbeType::Call, eliminate_frame_pointer: false, - linker_is_gnu: true, position_independent_executables: true, needs_plt: true, relro_level: RelroLevel::Full, diff --git a/compiler/rustc_target/src/spec/mipsel_sony_psp.rs b/compiler/rustc_target/src/spec/mipsel_sony_psp.rs index 08c290e6ff..dc14e4bdf9 100644 --- a/compiler/rustc_target/src/spec/mipsel_sony_psp.rs +++ b/compiler/rustc_target/src/spec/mipsel_sony_psp.rs @@ -21,7 +21,6 @@ pub fn target() -> Target { cpu: "mips2".to_string(), executables: true, linker: Some("rust-lld".to_owned()), - linker_is_gnu: true, relocation_model: RelocModel::Static, // PSP FPU only supports single precision floats. diff --git a/compiler/rustc_target/src/spec/mod.rs b/compiler/rustc_target/src/spec/mod.rs index 2af4669344..0f2aaeb533 100644 --- a/compiler/rustc_target/src/spec/mod.rs +++ b/compiler/rustc_target/src/spec/mod.rs @@ -57,6 +57,7 @@ mod apple_base; mod apple_sdk_base; mod arm_base; mod avr_gnu_base; +mod bpf_base; mod dragonfly_base; mod freebsd_base; mod fuchsia_base; @@ -93,6 +94,7 @@ pub enum LinkerFlavor { Msvc, Lld(LldFlavor), PtxLinker, + BpfLinker, } #[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd)] @@ -161,6 +163,7 @@ flavor_mappings! { ((LinkerFlavor::Ld), "ld"), ((LinkerFlavor::Msvc), "msvc"), ((LinkerFlavor::PtxLinker), "ptx-linker"), + ((LinkerFlavor::BpfLinker), "bpf-linker"), ((LinkerFlavor::Lld(LldFlavor::Wasm)), "wasm-ld"), ((LinkerFlavor::Lld(LldFlavor::Ld64)), "ld64.lld"), ((LinkerFlavor::Lld(LldFlavor::Ld)), "ld.lld"), @@ -897,6 +900,9 @@ supported_targets! { ("aarch64_be-unknown-linux-gnu", aarch64_be_unknown_linux_gnu), ("aarch64-unknown-linux-gnu_ilp32", aarch64_unknown_linux_gnu_ilp32), ("aarch64_be-unknown-linux-gnu_ilp32", aarch64_be_unknown_linux_gnu_ilp32), + + ("bpfeb-unknown-none", bpfeb_unknown_none), + ("bpfel-unknown-none", bpfel_unknown_none), } /// Everything `rustc` knows about how to compile for a specific target. @@ -922,6 +928,7 @@ pub trait HasTargetSpec { } impl HasTargetSpec for Target { + #[inline] fn target_spec(&self) -> &Target { self } @@ -1042,8 +1049,12 @@ pub struct TargetOptions { pub staticlib_prefix: String, /// String to append to the name of every static library. Defaults to ".a". pub staticlib_suffix: String, - /// OS family to use for conditional compilation. Valid options: "unix", "windows". - pub os_family: Option, + /// Values of the `target_family` cfg set for this target. + /// + /// Common options are: "unix", "windows". Defaults to no families. + /// + /// See . + pub families: Vec, /// Whether the target toolchain's ABI supports returning small structs as an integer. pub abi_return_struct_as_int: bool, /// Whether the target toolchain is like macOS's. Only useful for compiling against iOS/macOS, @@ -1082,7 +1093,7 @@ pub struct TargetOptions { /// Version of DWARF to use if not using the default. /// Useful because some platforms (osx, bsd) only want up to DWARF2. pub dwarf_version: Option, - /// Whether the linker support GNU-like arguments such as -O. Defaults to false. + /// Whether the linker support GNU-like arguments such as -O. Defaults to true. pub linker_is_gnu: bool, /// The MinGW toolchain has a known issue that prevents it from correctly /// handling COFF object files with more than 215 sections. Since each weak @@ -1293,7 +1304,7 @@ impl Default for TargetOptions { exe_suffix: String::new(), staticlib_prefix: "lib".to_string(), staticlib_suffix: ".a".to_string(), - os_family: None, + families: Vec::new(), abi_return_struct_as_int: false, is_like_osx: false, is_like_solaris: false, @@ -1303,7 +1314,7 @@ impl Default for TargetOptions { is_like_fuchsia: false, is_like_wasm: false, dwarf_version: None, - linker_is_gnu: false, + linker_is_gnu: true, allows_weak_linkage: true, has_rpath: false, no_default_libraries: true, @@ -1605,14 +1616,6 @@ impl Target { .map(|s| s.to_string() ); } } ); - ($key_name:ident = $json_name:expr, optional) => ( { - let name = $json_name; - if let Some(o) = obj.find(name) { - base.$key_name = o - .as_string() - .map(|s| s.to_string() ); - } - } ); ($key_name:ident, LldFlavor) => ( { let name = (stringify!($key_name)).replace("_", "-"); obj.find(&name[..]).and_then(|o| o.as_string().and_then(|s| { @@ -1759,6 +1762,16 @@ impl Target { Some(Ok(())) })).unwrap_or(Ok(())) } ); + ($key_name:ident, TargetFamilies) => ( { + let value = obj.find("target-family"); + if let Some(v) = value.and_then(Json::as_array) { + base.$key_name = v.iter() + .map(|a| a.as_string().unwrap().to_string()) + .collect(); + } else if let Some(v) = value.and_then(Json::as_string) { + base.$key_name = vec![v.to_string()]; + } + } ); } if let Some(s) = obj.find("target-endian").and_then(Json::as_string) { @@ -1802,7 +1815,7 @@ impl Target { key!(exe_suffix); key!(staticlib_prefix); key!(staticlib_suffix); - key!(os_family = "target-family", optional); + key!(families, TargetFamilies); key!(abi_return_struct_as_int, bool); key!(is_like_osx, bool); key!(is_like_solaris, bool); @@ -1891,14 +1904,16 @@ impl Target { Ok(base) } - /// Search RUST_TARGET_PATH for a JSON file specifying the given target - /// triple. Note that it could also just be a bare filename already, so also - /// check for that. If one of the hardcoded targets we know about, just - /// return it directly. + /// Search for a JSON file specifying the given target triple. + /// + /// If none is found in `$RUST_TARGET_PATH`, look for a file called `target.json` inside the + /// sysroot under the target-triple's `rustlib` directory. Note that it could also just be a + /// bare filename already, so also check for that. If one of the hardcoded targets we know + /// about, just return it directly. /// - /// The error string could come from any of the APIs called, including - /// filesystem access and JSON decoding. - pub fn search(target_triple: &TargetTriple) -> Result { + /// The error string could come from any of the APIs called, including filesystem access and + /// JSON decoding. + pub fn search(target_triple: &TargetTriple, sysroot: &PathBuf) -> Result { use rustc_serialize::json; use std::env; use std::fs; @@ -1925,14 +1940,26 @@ impl Target { let target_path = env::var_os("RUST_TARGET_PATH").unwrap_or_default(); - // FIXME 16351: add a sane default search path? - for dir in env::split_paths(&target_path) { let p = dir.join(&path); if p.is_file() { return load_file(&p); } } + + // Additionally look in the sysroot under `lib/rustlib//target.json` + // as a fallback. + let rustlib_path = crate::target_rustlib_path(&sysroot, &target_triple); + let p = std::array::IntoIter::new([ + Path::new(sysroot), + Path::new(&rustlib_path), + Path::new("target.json"), + ]) + .collect::(); + if p.is_file() { + return load_file(&p); + } + Err(format!("Could not find specification for target {:?}", target_triple)) } TargetTriple::TargetPath(ref target_path) => { @@ -2042,7 +2069,7 @@ impl ToJson for Target { target_option_val!(exe_suffix); target_option_val!(staticlib_prefix); target_option_val!(staticlib_suffix); - target_option_val!(os_family, "target-family"); + target_option_val!(families, "target-family"); target_option_val!(abi_return_struct_as_int); target_option_val!(is_like_osx); target_option_val!(is_like_solaris); diff --git a/compiler/rustc_target/src/spec/msp430_none_elf.rs b/compiler/rustc_target/src/spec/msp430_none_elf.rs index cc2578aa57..6e3a241a86 100644 --- a/compiler/rustc_target/src/spec/msp430_none_elf.rs +++ b/compiler/rustc_target/src/spec/msp430_none_elf.rs @@ -17,6 +17,7 @@ pub fn target() -> Target { // dependency on this specific gcc. asm_args: vec!["-mcpu=msp430".to_string()], linker: Some("msp430-elf-gcc".to_string()), + linker_is_gnu: false, // There are no atomic CAS instructions available in the MSP430 // instruction set, and the LLVM backend doesn't currently support diff --git a/compiler/rustc_target/src/spec/msvc_base.rs b/compiler/rustc_target/src/spec/msvc_base.rs index 4ed7685ca0..f1ed4c154d 100644 --- a/compiler/rustc_target/src/spec/msvc_base.rs +++ b/compiler/rustc_target/src/spec/msvc_base.rs @@ -16,6 +16,7 @@ pub fn opts() -> TargetOptions { is_like_windows: true, is_like_msvc: true, lld_flavor: LldFlavor::Link, + linker_is_gnu: false, pre_link_args, abi_return_struct_as_int: true, emit_debug_gdb_scripts: false, diff --git a/compiler/rustc_target/src/spec/netbsd_base.rs b/compiler/rustc_target/src/spec/netbsd_base.rs index 680cd60788..6bb6083d47 100644 --- a/compiler/rustc_target/src/spec/netbsd_base.rs +++ b/compiler/rustc_target/src/spec/netbsd_base.rs @@ -5,8 +5,7 @@ pub fn opts() -> TargetOptions { os: "netbsd".to_string(), dynamic_linking: true, executables: true, - os_family: Some("unix".to_string()), - linker_is_gnu: true, + families: vec!["unix".to_string()], no_default_libraries: false, has_rpath: true, position_independent_executables: true, diff --git a/compiler/rustc_target/src/spec/nvptx64_nvidia_cuda.rs b/compiler/rustc_target/src/spec/nvptx64_nvidia_cuda.rs index 15d8e4843f..97960a75b0 100644 --- a/compiler/rustc_target/src/spec/nvptx64_nvidia_cuda.rs +++ b/compiler/rustc_target/src/spec/nvptx64_nvidia_cuda.rs @@ -14,6 +14,7 @@ pub fn target() -> Target { linker_flavor: LinkerFlavor::PtxLinker, // The linker can be installed from `crates.io`. linker: Some("rust-ptx-linker".to_string()), + linker_is_gnu: false, // With `ptx-linker` approach, it can be later overridden via link flags. cpu: "sm_30".to_string(), diff --git a/compiler/rustc_target/src/spec/openbsd_base.rs b/compiler/rustc_target/src/spec/openbsd_base.rs index a6fd01ab11..29b415e772 100644 --- a/compiler/rustc_target/src/spec/openbsd_base.rs +++ b/compiler/rustc_target/src/spec/openbsd_base.rs @@ -5,8 +5,7 @@ pub fn opts() -> TargetOptions { os: "openbsd".to_string(), dynamic_linking: true, executables: true, - os_family: Some("unix".to_string()), - linker_is_gnu: true, + families: vec!["unix".to_string()], has_rpath: true, abi_return_struct_as_int: true, position_independent_executables: true, diff --git a/compiler/rustc_target/src/spec/redox_base.rs b/compiler/rustc_target/src/spec/redox_base.rs index 0afb4a72ac..fcf5db3746 100644 --- a/compiler/rustc_target/src/spec/redox_base.rs +++ b/compiler/rustc_target/src/spec/redox_base.rs @@ -6,8 +6,7 @@ pub fn opts() -> TargetOptions { env: "relibc".to_string(), dynamic_linking: true, executables: true, - os_family: Some("unix".to_string()), - linker_is_gnu: true, + families: vec!["unix".to_string()], has_rpath: true, position_independent_executables: true, relro_level: RelroLevel::Full, diff --git a/compiler/rustc_target/src/spec/solaris_base.rs b/compiler/rustc_target/src/spec/solaris_base.rs index 59731f2582..bc32b50168 100644 --- a/compiler/rustc_target/src/spec/solaris_base.rs +++ b/compiler/rustc_target/src/spec/solaris_base.rs @@ -6,8 +6,9 @@ pub fn opts() -> TargetOptions { dynamic_linking: true, executables: true, has_rpath: true, - os_family: Some("unix".to_string()), + families: vec!["unix".to_string()], is_like_solaris: true, + linker_is_gnu: false, limit_rdylib_exports: false, // Linker doesn't support this eh_frame_header: false, diff --git a/compiler/rustc_target/src/spec/thumbv4t_none_eabi.rs b/compiler/rustc_target/src/spec/thumbv4t_none_eabi.rs index ef58824f38..f996009f83 100644 --- a/compiler/rustc_target/src/spec/thumbv4t_none_eabi.rs +++ b/compiler/rustc_target/src/spec/thumbv4t_none_eabi.rs @@ -28,7 +28,6 @@ pub fn target() -> Target { options: TargetOptions { linker_flavor: LinkerFlavor::Ld, linker: Some("arm-none-eabi-ld".to_string()), - linker_is_gnu: true, // extra args passed to the external assembler (assuming `arm-none-eabi-as`): // * activate t32/a32 interworking diff --git a/compiler/rustc_target/src/spec/vxworks_base.rs b/compiler/rustc_target/src/spec/vxworks_base.rs index 41c4d7625a..a91e771786 100644 --- a/compiler/rustc_target/src/spec/vxworks_base.rs +++ b/compiler/rustc_target/src/spec/vxworks_base.rs @@ -9,8 +9,7 @@ pub fn opts() -> TargetOptions { exe_suffix: ".vxe".to_string(), dynamic_linking: true, executables: true, - os_family: Some("unix".to_string()), - linker_is_gnu: true, + families: vec!["unix".to_string()], has_rpath: true, has_elf_tls: true, crt_static_default: true, diff --git a/compiler/rustc_target/src/spec/wasm32_unknown_emscripten.rs b/compiler/rustc_target/src/spec/wasm32_unknown_emscripten.rs index e028dbaa32..302139395d 100644 --- a/compiler/rustc_target/src/spec/wasm32_unknown_emscripten.rs +++ b/compiler/rustc_target/src/spec/wasm32_unknown_emscripten.rs @@ -34,11 +34,10 @@ pub fn target() -> Target { // functionality, and a .wasm file. exe_suffix: ".js".to_string(), linker: None, - linker_is_gnu: true, is_like_emscripten: true, panic_strategy: PanicStrategy::Unwind, post_link_args, - os_family: Some("unix".to_string()), + families: vec!["unix".to_string()], ..options }; Target { diff --git a/compiler/rustc_target/src/spec/wasm_base.rs b/compiler/rustc_target/src/spec/wasm_base.rs index b208eb92f8..4c954a1e56 100644 --- a/compiler/rustc_target/src/spec/wasm_base.rs +++ b/compiler/rustc_target/src/spec/wasm_base.rs @@ -61,6 +61,7 @@ pub fn options() -> TargetOptions { TargetOptions { is_like_wasm: true, + families: vec!["wasm".to_string()], // we allow dynamic linking, but only cdylibs. Basically we allow a // final library artifact that exports some symbols (a wasm module) but @@ -101,12 +102,7 @@ pub fn options() -> TargetOptions { // we use the LLD shipped with the Rust toolchain by default linker: Some("rust-lld".to_owned()), lld_flavor: LldFlavor::Wasm, - - // No need for indirection here, simd types can always be passed by - // value as the whole module either has simd or not, which is different - // from x86 (for example) where programs can have functions that don't - // enable simd features. - simd_types_indirect: false, + linker_is_gnu: false, pre_link_args, diff --git a/compiler/rustc_target/src/spec/windows_gnu_base.rs b/compiler/rustc_target/src/spec/windows_gnu_base.rs index 478c567a93..35a52896f6 100644 --- a/compiler/rustc_target/src/spec/windows_gnu_base.rs +++ b/compiler/rustc_target/src/spec/windows_gnu_base.rs @@ -71,7 +71,7 @@ pub fn opts() -> TargetOptions { dll_prefix: String::new(), dll_suffix: ".dll".to_string(), exe_suffix: ".exe".to_string(), - os_family: Some("windows".to_string()), + families: vec!["windows".to_string()], is_like_windows: true, allows_weak_linkage: false, pre_link_args, diff --git a/compiler/rustc_target/src/spec/windows_msvc_base.rs b/compiler/rustc_target/src/spec/windows_msvc_base.rs index c041245e32..0d58618a44 100644 --- a/compiler/rustc_target/src/spec/windows_msvc_base.rs +++ b/compiler/rustc_target/src/spec/windows_msvc_base.rs @@ -13,7 +13,7 @@ pub fn opts() -> TargetOptions { exe_suffix: ".exe".to_string(), staticlib_prefix: String::new(), staticlib_suffix: ".lib".to_string(), - os_family: Some("windows".to_string()), + families: vec!["windows".to_string()], crt_static_allows_dylibs: true, crt_static_respected: true, requires_uwtable: true, diff --git a/compiler/rustc_target/src/spec/x86_64_fortanix_unknown_sgx.rs b/compiler/rustc_target/src/spec/x86_64_fortanix_unknown_sgx.rs index 90705c526f..06eb33d8d8 100644 --- a/compiler/rustc_target/src/spec/x86_64_fortanix_unknown_sgx.rs +++ b/compiler/rustc_target/src/spec/x86_64_fortanix_unknown_sgx.rs @@ -57,7 +57,6 @@ pub fn target() -> Target { vendor: "fortanix".into(), linker_flavor: LinkerFlavor::Lld(LldFlavor::Ld), executables: true, - linker_is_gnu: true, linker: Some("rust-lld".to_owned()), max_atomic_width: Some(64), cpu: "x86-64".into(), diff --git a/compiler/rustc_target/src/spec/x86_64_unknown_none_hermitkernel.rs b/compiler/rustc_target/src/spec/x86_64_unknown_none_hermitkernel.rs index 28d9801b78..359cb0f688 100644 --- a/compiler/rustc_target/src/spec/x86_64_unknown_none_hermitkernel.rs +++ b/compiler/rustc_target/src/spec/x86_64_unknown_none_hermitkernel.rs @@ -11,7 +11,7 @@ pub fn target() -> Target { base.stack_probes = StackProbeType::Call; Target { - llvm_target: "x86_64-unknown-none-elf".to_string(), + llvm_target: "x86_64-unknown-hermit".to_string(), pointer_width: 64, data_layout: "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128" .to_string(), diff --git a/compiler/rustc_trait_selection/src/autoderef.rs b/compiler/rustc_trait_selection/src/autoderef.rs index 3f24a33f7d..ac2e0ebae3 100644 --- a/compiler/rustc_trait_selection/src/autoderef.rs +++ b/compiler/rustc_trait_selection/src/autoderef.rs @@ -6,6 +6,7 @@ use rustc_infer::infer::InferCtxt; use rustc_middle::ty::{self, TraitRef, Ty, TyCtxt, WithConstness}; use rustc_middle::ty::{ToPredicate, TypeFoldable}; use rustc_session::DiagnosticMessageId; +use rustc_span::def_id::LOCAL_CRATE; use rustc_span::Span; #[derive(Copy, Clone, Debug)] @@ -231,7 +232,8 @@ pub fn report_autoderef_recursion_limit_error<'tcx>(tcx: TyCtxt<'tcx>, span: Spa .span_label(span, "deref recursion limit reached") .help(&format!( "consider adding a `#![recursion_limit=\"{}\"]` attribute to your crate (`{}`)", - suggested_limit, tcx.crate_name, + suggested_limit, + tcx.crate_name(LOCAL_CRATE), )) .emit(); } diff --git a/compiler/rustc_trait_selection/src/lib.rs b/compiler/rustc_trait_selection/src/lib.rs index 4097e1577e..e932b1bca7 100644 --- a/compiler/rustc_trait_selection/src/lib.rs +++ b/compiler/rustc_trait_selection/src/lib.rs @@ -14,11 +14,11 @@ #![feature(bool_to_option)] #![feature(box_patterns)] #![feature(drain_filter)] +#![feature(hash_drain_filter)] #![feature(in_band_lifetimes)] #![feature(iter_zip)] #![feature(never_type)] #![feature(crate_visibility_modifier)] -#![cfg_attr(bootstrap, feature(or_patterns))] #![feature(control_flow_enum)] #![recursion_limit = "512"] // For rustdoc diff --git a/compiler/rustc_trait_selection/src/opaque_types.rs b/compiler/rustc_trait_selection/src/opaque_types.rs index fb4a8ce687..89ec211f26 100644 --- a/compiler/rustc_trait_selection/src/opaque_types.rs +++ b/compiler/rustc_trait_selection/src/opaque_types.rs @@ -2,21 +2,22 @@ use crate::infer::InferCtxtExt as _; use crate::traits::{self, PredicateObligation}; use rustc_data_structures::fx::FxHashMap; use rustc_data_structures::sync::Lrc; +use rustc_data_structures::vec_map::VecMap; use rustc_hir as hir; -use rustc_hir::def_id::{DefId, DefIdMap, LocalDefId}; +use rustc_hir::def_id::{DefId, LocalDefId}; use rustc_hir::Node; use rustc_infer::infer::error_reporting::unexpected_hidden_region_diagnostic; use rustc_infer::infer::free_regions::FreeRegionRelations; use rustc_infer::infer::type_variable::{TypeVariableOrigin, TypeVariableOriginKind}; use rustc_infer::infer::{self, InferCtxt, InferOk}; use rustc_middle::ty::fold::{BottomUpFolder, TypeFoldable, TypeFolder, TypeVisitor}; -use rustc_middle::ty::subst::{GenericArg, GenericArgKind, InternalSubsts, Subst, SubstsRef}; -use rustc_middle::ty::{self, Ty, TyCtxt}; +use rustc_middle::ty::subst::{GenericArg, GenericArgKind, InternalSubsts, Subst}; +use rustc_middle::ty::{self, OpaqueTypeKey, Ty, TyCtxt}; use rustc_span::Span; use std::ops::ControlFlow; -pub type OpaqueTypeMap<'tcx> = DefIdMap>; +pub type OpaqueTypeMap<'tcx> = VecMap, OpaqueTypeDecl<'tcx>>; /// Information about the opaque types whose values we /// are inferring in this function (these are the `impl Trait` that @@ -26,19 +27,6 @@ pub struct OpaqueTypeDecl<'tcx> { /// The opaque type (`ty::Opaque`) for this declaration. pub opaque_type: Ty<'tcx>, - /// The substitutions that we apply to the opaque type that this - /// `impl Trait` desugars to. e.g., if: - /// - /// fn foo<'a, 'b, T>() -> impl Trait<'a> - /// - /// winds up desugared to: - /// - /// type Foo<'x, X> = impl Trait<'x> - /// fn foo<'a, 'b, T>() -> Foo<'a, T> - /// - /// then `substs` would be `['a, T]`. - pub substs: SubstsRef<'tcx>, - /// The span of this particular definition of the opaque type. So /// for example: /// @@ -46,6 +34,7 @@ pub struct OpaqueTypeDecl<'tcx> { /// type Foo = impl Baz; /// fn bar() -> Foo { /// // ^^^ This is the span we are looking for! + /// } /// ``` /// /// In cases where the fn returns `(impl Trait, impl Trait)` or @@ -124,7 +113,7 @@ pub trait InferCtxtExt<'tcx> { fn constrain_opaque_type>( &self, - def_id: DefId, + opaque_type_key: OpaqueTypeKey<'tcx>, opaque_defn: &OpaqueTypeDecl<'tcx>, mode: GenerateMemberConstraints, free_region_relations: &FRR, @@ -135,23 +124,13 @@ pub trait InferCtxtExt<'tcx> { &self, concrete_ty: Ty<'tcx>, opaque_defn: &OpaqueTypeDecl<'tcx>, - opaque_type_def_id: DefId, + opaque_type_key: OpaqueTypeKey<'tcx>, first_own_region_index: usize, ); - /*private*/ - fn member_constraint_feature_gate( - &self, - opaque_defn: &OpaqueTypeDecl<'tcx>, - opaque_type_def_id: DefId, - conflict1: ty::Region<'tcx>, - conflict2: ty::Region<'tcx>, - ) -> bool; - fn infer_opaque_definition_from_instantiation( &self, - def_id: DefId, - substs: SubstsRef<'tcx>, + opaque_type_key: OpaqueTypeKey<'tcx>, instantiated_ty: Ty<'tcx>, span: Span, ) -> Ty<'tcx>; @@ -378,10 +357,10 @@ impl<'a, 'tcx> InferCtxtExt<'tcx> for InferCtxt<'a, 'tcx> { ) { debug!("constrain_opaque_types()"); - for (&def_id, opaque_defn) in opaque_types { + for &(opaque_type_key, opaque_defn) in opaque_types { self.constrain_opaque_type( - def_id, - opaque_defn, + opaque_type_key, + &opaque_defn, GenerateMemberConstraints::WhenRequired, free_region_relations, ); @@ -391,11 +370,13 @@ impl<'a, 'tcx> InferCtxtExt<'tcx> for InferCtxt<'a, 'tcx> { /// See `constrain_opaque_types` for documentation. fn constrain_opaque_type>( &self, - def_id: DefId, + opaque_type_key: OpaqueTypeKey<'tcx>, opaque_defn: &OpaqueTypeDecl<'tcx>, mode: GenerateMemberConstraints, free_region_relations: &FRR, ) { + let def_id = opaque_type_key.def_id; + debug!("constrain_opaque_type()"); debug!("constrain_opaque_type: def_id={:?}", def_id); debug!("constrain_opaque_type: opaque_defn={:#?}", opaque_defn); @@ -434,9 +415,9 @@ impl<'a, 'tcx> InferCtxtExt<'tcx> for InferCtxt<'a, 'tcx> { let bounds = tcx.explicit_item_bounds(def_id); debug!("constrain_opaque_type: predicates: {:#?}", bounds); let bounds: Vec<_> = - bounds.iter().map(|(bound, _)| bound.subst(tcx, opaque_defn.substs)).collect(); + bounds.iter().map(|(bound, _)| bound.subst(tcx, opaque_type_key.substs)).collect(); debug!("constrain_opaque_type: bounds={:#?}", bounds); - let opaque_type = tcx.mk_opaque(def_id, opaque_defn.substs); + let opaque_type = tcx.mk_opaque(def_id, opaque_type_key.substs); let required_region_bounds = required_region_bounds(tcx, opaque_type, bounds.into_iter()); @@ -448,7 +429,12 @@ impl<'a, 'tcx> InferCtxtExt<'tcx> for InferCtxt<'a, 'tcx> { }); } if let GenerateMemberConstraints::IfNoStaticBound = mode { - self.generate_member_constraint(concrete_ty, opaque_defn, def_id, first_own_region); + self.generate_member_constraint( + concrete_ty, + opaque_defn, + opaque_type_key, + first_own_region, + ); } return; } @@ -462,7 +448,7 @@ impl<'a, 'tcx> InferCtxtExt<'tcx> for InferCtxt<'a, 'tcx> { // second. let mut least_region = None; - for subst_arg in &opaque_defn.substs[first_own_region..] { + for subst_arg in &opaque_type_key.substs[first_own_region..] { let subst_region = match subst_arg.unpack() { GenericArgKind::Lifetime(r) => r, GenericArgKind::Type(_) | GenericArgKind::Const(_) => continue, @@ -489,13 +475,10 @@ impl<'a, 'tcx> InferCtxtExt<'tcx> for InferCtxt<'a, 'tcx> { // ['a, 'b, 'c]`, where `'a..'c` are the // regions that appear in the impl trait. - // For now, enforce a feature gate outside of async functions. - self.member_constraint_feature_gate(opaque_defn, def_id, lr, subst_region); - return self.generate_member_constraint( concrete_ty, opaque_defn, - def_id, + opaque_type_key, first_own_region, ); } @@ -508,7 +491,12 @@ impl<'a, 'tcx> InferCtxtExt<'tcx> for InferCtxt<'a, 'tcx> { if let GenerateMemberConstraints::IfNoStaticBound = mode { if least_region != tcx.lifetimes.re_static { - self.generate_member_constraint(concrete_ty, opaque_defn, def_id, first_own_region); + self.generate_member_constraint( + concrete_ty, + opaque_defn, + opaque_type_key, + first_own_region, + ); } } concrete_ty.visit_with(&mut ConstrainOpaqueTypeRegionVisitor { @@ -528,14 +516,14 @@ impl<'a, 'tcx> InferCtxtExt<'tcx> for InferCtxt<'a, 'tcx> { &self, concrete_ty: Ty<'tcx>, opaque_defn: &OpaqueTypeDecl<'tcx>, - opaque_type_def_id: DefId, + opaque_type_key: OpaqueTypeKey<'tcx>, first_own_region: usize, ) { // Create the set of choice regions: each region in the hidden // type can be equal to any of the region parameters of the // opaque type definition. let choice_regions: Lrc>> = Lrc::new( - opaque_defn.substs[first_own_region..] + opaque_type_key.substs[first_own_region..] .iter() .filter_map(|arg| match arg.unpack() { GenericArgKind::Lifetime(r) => Some(r), @@ -548,7 +536,7 @@ impl<'a, 'tcx> InferCtxtExt<'tcx> for InferCtxt<'a, 'tcx> { concrete_ty.visit_with(&mut ConstrainOpaqueTypeRegionVisitor { op: |r| { self.member_constraint( - opaque_type_def_id, + opaque_type_key.def_id, opaque_defn.definition_span, concrete_ty, r, @@ -558,60 +546,6 @@ impl<'a, 'tcx> InferCtxtExt<'tcx> for InferCtxt<'a, 'tcx> { }); } - /// Member constraints are presently feature-gated except for - /// async-await. We expect to lift this once we've had a bit more - /// time. - fn member_constraint_feature_gate( - &self, - opaque_defn: &OpaqueTypeDecl<'tcx>, - opaque_type_def_id: DefId, - conflict1: ty::Region<'tcx>, - conflict2: ty::Region<'tcx>, - ) -> bool { - // If we have `#![feature(member_constraints)]`, no problems. - if self.tcx.features().member_constraints { - return false; - } - - let span = self.tcx.def_span(opaque_type_def_id); - - // Without a feature-gate, we only generate member-constraints for async-await. - let context_name = match opaque_defn.origin { - // No feature-gate required for `async fn`. - hir::OpaqueTyOrigin::AsyncFn => return false, - - // Otherwise, generate the label we'll use in the error message. - hir::OpaqueTyOrigin::Binding - | hir::OpaqueTyOrigin::FnReturn - | hir::OpaqueTyOrigin::TyAlias - | hir::OpaqueTyOrigin::Misc => "impl Trait", - }; - let msg = format!("ambiguous lifetime bound in `{}`", context_name); - let mut err = self.tcx.sess.struct_span_err(span, &msg); - - let conflict1_name = conflict1.to_string(); - let conflict2_name = conflict2.to_string(); - let label_owned; - let label = match (&*conflict1_name, &*conflict2_name) { - ("'_", "'_") => "the elided lifetimes here do not outlive one another", - _ => { - label_owned = format!( - "neither `{}` nor `{}` outlives the other", - conflict1_name, conflict2_name, - ); - &label_owned - } - }; - err.span_label(span, label); - - if self.tcx.sess.is_nightly_build() { - err.help("add #![feature(member_constraints)] to the crate attributes to enable"); - } - - err.emit(); - true - } - /// Given the fully resolved, instantiated type for an opaque /// type, i.e., the value of an inference variable like C1 or C2 /// (*), computes the "definition type" for an opaque type @@ -637,11 +571,12 @@ impl<'a, 'tcx> InferCtxtExt<'tcx> for InferCtxt<'a, 'tcx> { /// `opaque_defn.concrete_ty` fn infer_opaque_definition_from_instantiation( &self, - def_id: DefId, - substs: SubstsRef<'tcx>, + opaque_type_key: OpaqueTypeKey<'tcx>, instantiated_ty: Ty<'tcx>, span: Span, ) -> Ty<'tcx> { + let OpaqueTypeKey { def_id, substs } = opaque_type_key; + debug!( "infer_opaque_definition_from_instantiation(def_id={:?}, instantiated_ty={:?})", def_id, instantiated_ty @@ -1072,7 +1007,9 @@ impl<'a, 'tcx> Instantiator<'a, 'tcx> { ), }; if in_definition_scope { - return self.fold_opaque_ty(ty, def_id.to_def_id(), substs, origin); + let opaque_type_key = + OpaqueTypeKey { def_id: def_id.to_def_id(), substs }; + return self.fold_opaque_ty(ty, opaque_type_key, origin); } debug!( @@ -1094,18 +1031,18 @@ impl<'a, 'tcx> Instantiator<'a, 'tcx> { fn fold_opaque_ty( &mut self, ty: Ty<'tcx>, - def_id: DefId, - substs: SubstsRef<'tcx>, + opaque_type_key: OpaqueTypeKey<'tcx>, origin: hir::OpaqueTyOrigin, ) -> Ty<'tcx> { let infcx = self.infcx; let tcx = infcx.tcx; + let OpaqueTypeKey { def_id, substs } = opaque_type_key; debug!("instantiate_opaque_types: Opaque(def_id={:?}, substs={:?})", def_id, substs); // Use the same type variable if the exact same opaque type appears more // than once in the return type (e.g., if it's passed to a type alias). - if let Some(opaque_defn) = self.opaque_types.get(&def_id) { + if let Some(opaque_defn) = self.opaque_types.get(&opaque_type_key) { debug!("instantiate_opaque_types: returning concrete ty {:?}", opaque_defn.concrete_ty); return opaque_defn.concrete_ty; } @@ -1143,10 +1080,9 @@ impl<'a, 'tcx> Instantiator<'a, 'tcx> { let definition_span = self.value_span; self.opaque_types.insert( - def_id, + OpaqueTypeKey { def_id, substs }, OpaqueTypeDecl { opaque_type: ty, - substs, definition_span, concrete_ty: ty_var, has_required_region_bounds: !required_region_bounds.is_empty(), diff --git a/compiler/rustc_trait_selection/src/traits/error_reporting/mod.rs b/compiler/rustc_trait_selection/src/traits/error_reporting/mod.rs index da5a1af7f7..19c3385dd4 100644 --- a/compiler/rustc_trait_selection/src/traits/error_reporting/mod.rs +++ b/compiler/rustc_trait_selection/src/traits/error_reporting/mod.rs @@ -14,7 +14,7 @@ use crate::infer::{self, InferCtxt, TyCtxtInferExt}; use rustc_data_structures::fx::FxHashMap; use rustc_errors::{pluralize, struct_span_err, Applicability, DiagnosticBuilder, ErrorReported}; use rustc_hir as hir; -use rustc_hir::def_id::{DefId, LOCAL_CRATE}; +use rustc_hir::def_id::DefId; use rustc_hir::intravisit::Visitor; use rustc_hir::Node; use rustc_middle::mir::abstract_const::NotConstEvaluatable; @@ -1427,7 +1427,7 @@ impl<'a, 'tcx> InferCtxtPrivExt<'tcx> for InferCtxt<'a, 'tcx> { self.tcx.find_map_relevant_impl(trait_def_id, trait_ref.skip_binder().self_ty(), Some) }; let required_trait_path = self.tcx.def_path_str(trait_ref.def_id()); - let all_traits = self.tcx.all_traits(LOCAL_CRATE); + let all_traits = self.tcx.all_traits(()); let traits_with_same_path: std::collections::BTreeSet<_> = all_traits .iter() .filter(|trait_def_id| **trait_def_id != trait_ref.def_id()) @@ -1878,6 +1878,10 @@ impl<'v> Visitor<'v> for FindTypeParam { hir::intravisit::NestedVisitorMap::None } + fn visit_where_predicate(&mut self, _: &'v hir::WherePredicate<'v>) { + // Skip where-clauses, to avoid suggesting indirection for type parameters found there. + } + fn visit_ty(&mut self, ty: &hir::Ty<'_>) { // We collect the spans of all uses of the "bare" type param, like in `field: T` or // `field: (T, T)` where we could make `T: ?Sized` while skipping cases that are known to be diff --git a/compiler/rustc_trait_selection/src/traits/error_reporting/on_unimplemented.rs b/compiler/rustc_trait_selection/src/traits/error_reporting/on_unimplemented.rs index 1ea34e5814..0ca0245a20 100644 --- a/compiler/rustc_trait_selection/src/traits/error_reporting/on_unimplemented.rs +++ b/compiler/rustc_trait_selection/src/traits/error_reporting/on_unimplemented.rs @@ -186,6 +186,15 @@ impl<'a, 'tcx> InferCtxtExt<'tcx> for InferCtxt<'a, 'tcx> { }; let name = param.name; flags.push((name, Some(value))); + + if let GenericParamDefKind::Type { .. } = param.kind { + let param_ty = trait_ref.substs[param.index as usize].expect_ty(); + if let Some(def) = param_ty.ty_adt_def() { + // We also want to be able to select the parameter's + // original signature with no type arguments resolved + flags.push((name, Some(self.tcx.type_of(def.did).to_string()))); + } + } } if let Some(true) = self_ty.ty_adt_def().map(|def| def.did.is_local()) { diff --git a/compiler/rustc_trait_selection/src/traits/error_reporting/suggestions.rs b/compiler/rustc_trait_selection/src/traits/error_reporting/suggestions.rs index 6a4d41ffc1..5c35b515f3 100644 --- a/compiler/rustc_trait_selection/src/traits/error_reporting/suggestions.rs +++ b/compiler/rustc_trait_selection/src/traits/error_reporting/suggestions.rs @@ -21,6 +21,7 @@ use rustc_middle::ty::{ Infer, InferTy, ToPredicate, Ty, TyCtxt, TypeFoldable, WithConstness, }; use rustc_middle::ty::{TypeAndMut, TypeckResults}; +use rustc_span::def_id::LOCAL_CRATE; use rustc_span::symbol::{kw, sym, Ident, Symbol}; use rustc_span::{BytePos, MultiSpan, Span, DUMMY_SP}; use rustc_target::spec::abi; @@ -686,17 +687,36 @@ impl<'a, 'tcx> InferCtxtExt<'tcx> for InferCtxt<'a, 'tcx> { return false; } + // Blacklist traits for which it would be nonsensical to suggest borrowing. + // For instance, immutable references are always Copy, so suggesting to + // borrow would always succeed, but it's probably not what the user wanted. + let blacklist: Vec<_> = + [LangItem::Copy, LangItem::Clone, LangItem::Unpin, LangItem::Sized, LangItem::Send] + .iter() + .filter_map(|lang_item| self.tcx.lang_items().require(*lang_item).ok()) + .collect(); + let span = obligation.cause.span; let param_env = obligation.param_env; let trait_ref = trait_ref.skip_binder(); - if let ObligationCauseCode::ImplDerivedObligation(obligation) = &obligation.cause.code { - // Try to apply the original trait binding obligation by borrowing. - let self_ty = trait_ref.self_ty(); - let found = self_ty.to_string(); - let new_self_ty = self.tcx.mk_imm_ref(self.tcx.lifetimes.re_static, self_ty); - let substs = self.tcx.mk_substs_trait(new_self_ty, &[]); - let new_trait_ref = ty::TraitRef::new(obligation.parent_trait_ref.def_id(), substs); + let found_ty = trait_ref.self_ty(); + let found_ty_str = found_ty.to_string(); + let imm_borrowed_found_ty = self.tcx.mk_imm_ref(self.tcx.lifetimes.re_static, found_ty); + let imm_substs = self.tcx.mk_substs_trait(imm_borrowed_found_ty, &[]); + let mut_borrowed_found_ty = self.tcx.mk_mut_ref(self.tcx.lifetimes.re_static, found_ty); + let mut_substs = self.tcx.mk_substs_trait(mut_borrowed_found_ty, &[]); + + // Try to apply the original trait binding obligation by borrowing. + let mut try_borrowing = |new_trait_ref: ty::TraitRef<'tcx>, + expected_trait_ref: ty::TraitRef<'tcx>, + mtbl: bool, + blacklist: &[DefId]| + -> bool { + if blacklist.contains(&expected_trait_ref.def_id) { + return false; + } + let new_obligation = Obligation::new( ObligationCause::dummy(), param_env, @@ -713,8 +733,8 @@ impl<'a, 'tcx> InferCtxtExt<'tcx> for InferCtxt<'a, 'tcx> { let msg = format!( "the trait bound `{}: {}` is not satisfied", - found, - obligation.parent_trait_ref.skip_binder().print_only_trait_path(), + found_ty_str, + expected_trait_ref.print_only_trait_path(), ); if has_custom_message { err.note(&msg); @@ -730,7 +750,7 @@ impl<'a, 'tcx> InferCtxtExt<'tcx> for InferCtxt<'a, 'tcx> { span, &format!( "expected an implementor of trait `{}`", - obligation.parent_trait_ref.skip_binder().print_only_trait_path(), + expected_trait_ref.print_only_trait_path(), ), ); @@ -745,16 +765,52 @@ impl<'a, 'tcx> InferCtxtExt<'tcx> for InferCtxt<'a, 'tcx> { err.span_suggestion( span, - "consider borrowing here", - format!("&{}", snippet), + &format!( + "consider{} borrowing here", + if mtbl { " mutably" } else { "" } + ), + format!("&{}{}", if mtbl { "mut " } else { "" }, snippet), Applicability::MaybeIncorrect, ); } return true; } } + return false; + }; + + if let ObligationCauseCode::ImplDerivedObligation(obligation) = &obligation.cause.code { + let expected_trait_ref = obligation.parent_trait_ref.skip_binder(); + let new_imm_trait_ref = + ty::TraitRef::new(obligation.parent_trait_ref.def_id(), imm_substs); + let new_mut_trait_ref = + ty::TraitRef::new(obligation.parent_trait_ref.def_id(), mut_substs); + if try_borrowing(new_imm_trait_ref, expected_trait_ref, false, &[]) { + return true; + } else { + return try_borrowing(new_mut_trait_ref, expected_trait_ref, true, &[]); + } + } else if let ObligationCauseCode::BindingObligation(_, _) + | ObligationCauseCode::ItemObligation(_) = &obligation.cause.code + { + if try_borrowing( + ty::TraitRef::new(trait_ref.def_id, imm_substs), + trait_ref, + false, + &blacklist[..], + ) { + return true; + } else { + return try_borrowing( + ty::TraitRef::new(trait_ref.def_id, mut_substs), + trait_ref, + true, + &blacklist[..], + ); + } + } else { + false } - false } /// Whenever references are used by mistake, like `for (i, e) in &vec.iter().enumerate()`, @@ -2258,7 +2314,8 @@ impl<'a, 'tcx> InferCtxtExt<'tcx> for InferCtxt<'a, 'tcx> { let suggested_limit = current_limit * 2; err.help(&format!( "consider adding a `#![recursion_limit=\"{}\"]` attribute to your crate (`{}`)", - suggested_limit, self.tcx.crate_name, + suggested_limit, + self.tcx.crate_name(LOCAL_CRATE), )); } diff --git a/compiler/rustc_trait_selection/src/traits/fulfill.rs b/compiler/rustc_trait_selection/src/traits/fulfill.rs index fc9739f70d..120680092b 100644 --- a/compiler/rustc_trait_selection/src/traits/fulfill.rs +++ b/compiler/rustc_trait_selection/src/traits/fulfill.rs @@ -6,7 +6,7 @@ use rustc_errors::ErrorReported; use rustc_infer::traits::{SelectionError, TraitEngine, TraitEngineExt as _, TraitObligation}; use rustc_middle::mir::abstract_const::NotConstEvaluatable; use rustc_middle::mir::interpret::ErrorHandled; -use rustc_middle::ty::error::ExpectedFound; +use rustc_middle::ty::error::{ExpectedFound, TypeError}; use rustc_middle::ty::subst::SubstsRef; use rustc_middle::ty::ToPredicate; use rustc_middle::ty::{self, Binder, Const, Ty, TypeFoldable}; @@ -591,7 +591,16 @@ impl<'a, 'b, 'tcx> FulfillProcessor<'a, 'b, 'tcx> { ) } (Err(ErrorHandled::TooGeneric), _) | (_, Err(ErrorHandled::TooGeneric)) => { - ProcessResult::Unchanged + if c1.has_infer_types_or_consts() || c2.has_infer_types_or_consts() { + ProcessResult::Unchanged + } else { + // Two different constants using generic parameters ~> error. + let expected_found = ExpectedFound::new(true, c1, c2); + ProcessResult::Error(FulfillmentErrorCode::CodeConstEquateError( + expected_found, + TypeError::ConstMismatch(expected_found), + )) + } } } } diff --git a/compiler/rustc_trait_selection/src/traits/project.rs b/compiler/rustc_trait_selection/src/traits/project.rs index b3e5df4da0..388413ae06 100644 --- a/compiler/rustc_trait_selection/src/traits/project.rs +++ b/compiler/rustc_trait_selection/src/traits/project.rs @@ -529,15 +529,7 @@ fn opt_normalize_projection_type<'a, 'b, 'tcx>( // evaluation this is not the case, and dropping the trait // evaluations can causes ICEs (e.g., #43132). debug!(?ty, "found normalized ty"); - - // Once we have inferred everything we need to know, we - // can ignore the `obligations` from that point on. - if infcx.unresolved_type_vars(&ty.value).is_none() { - infcx.inner.borrow_mut().projection_cache().complete_normalized(cache_key, &ty); - // No need to extend `obligations`. - } else { - obligations.extend(ty.obligations); - } + obligations.extend(ty.obligations); return Ok(Some(ty.value)); } Err(ProjectionCacheEntry::Error) => { diff --git a/compiler/rustc_trait_selection/src/traits/select/mod.rs b/compiler/rustc_trait_selection/src/traits/select/mod.rs index 727285e492..ea5eb2b686 100644 --- a/compiler/rustc_trait_selection/src/traits/select/mod.rs +++ b/compiler/rustc_trait_selection/src/traits/select/mod.rs @@ -557,6 +557,23 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> { ty::PredicateKind::ConstEquate(c1, c2) => { debug!(?c1, ?c2, "evaluate_predicate_recursively: equating consts"); + if self.tcx().features().const_evaluatable_checked { + // FIXME: we probably should only try to unify abstract constants + // if the constants depend on generic parameters. + // + // Let's just see where this breaks :shrug: + if let (ty::ConstKind::Unevaluated(a), ty::ConstKind::Unevaluated(b)) = + (c1.val, c2.val) + { + if self + .tcx() + .try_unify_abstract_consts(((a.def, a.substs), (b.def, b.substs))) + { + return Ok(EvaluatedToOk); + } + } + } + let evaluate = |c: &'tcx ty::Const<'tcx>| { if let ty::ConstKind::Unevaluated(unevaluated) = c.val { self.infcx @@ -591,7 +608,12 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> { ) } (Err(ErrorHandled::TooGeneric), _) | (_, Err(ErrorHandled::TooGeneric)) => { - Ok(EvaluatedToAmbig) + if c1.has_infer_types_or_consts() || c2.has_infer_types_or_consts() { + Ok(EvaluatedToAmbig) + } else { + // Two different constants using generic parameters ~> error. + Ok(EvaluatedToErr) + } } } } @@ -636,8 +658,8 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> { if let Some(result) = stack.cache().get_provisional(fresh_trait_ref) { debug!(?result, "PROVISIONAL CACHE HIT"); - stack.update_reached_depth(stack.cache().current_reached_depth()); - return Ok(result); + stack.update_reached_depth(result.reached_depth); + return Ok(result.result); } // Check if this is a match for something already on the @@ -661,7 +683,7 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> { debug!(?result, "CACHE MISS"); self.insert_evaluation_cache(obligation.param_env, fresh_trait_ref, dep_node, result); - stack.cache().on_completion(stack.depth, |fresh_trait_ref, provisional_result| { + stack.cache().on_completion(stack.dfn, |fresh_trait_ref, provisional_result| { self.insert_evaluation_cache( obligation.param_env, fresh_trait_ref, @@ -1044,8 +1066,6 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> { } /// Returns `true` if the global caches can be used. - /// Do note that if the type itself is not in the - /// global tcx, the local caches will be used. fn can_use_global_caches(&self, param_env: ty::ParamEnv<'tcx>) -> bool { // If there are any inference variables in the `ParamEnv`, then we // always use a cache local to this particular scope. Otherwise, we @@ -2164,7 +2184,7 @@ impl<'o, 'tcx> TraitObligationStack<'o, 'tcx> { /// required accessing something from the stack at depth `reached_depth`. fn update_reached_depth(&self, reached_depth: usize) { assert!( - self.depth > reached_depth, + self.depth >= reached_depth, "invoked `update_reached_depth` with something under this stack: \ self.depth={} reached_depth={}", self.depth, @@ -2237,23 +2257,6 @@ struct ProvisionalEvaluationCache<'tcx> { /// next "depth first number" to issue -- just a counter dfn: Cell, - /// Stores the "coldest" depth (bottom of stack) reached by any of - /// the evaluation entries. The idea here is that all things in the provisional - /// cache are always dependent on *something* that is colder in the stack: - /// therefore, if we add a new entry that is dependent on something *colder still*, - /// we have to modify the depth for all entries at once. - /// - /// Example: - /// - /// Imagine we have a stack `A B C D E` (with `E` being the top of - /// the stack). We cache something with depth 2, which means that - /// it was dependent on C. Then we pop E but go on and process a - /// new node F: A B C D F. Now F adds something to the cache with - /// depth 1, meaning it is dependent on B. Our original cache - /// entry is also dependent on B, because there is a path from E - /// to C and then from C to F and from F to B. - reached_depth: Cell, - /// Map from cache key to the provisionally evaluated thing. /// The cache entries contain the result but also the DFN in which they /// were added. The DFN is used to clear out values on failure. @@ -2277,12 +2280,13 @@ struct ProvisionalEvaluationCache<'tcx> { #[derive(Copy, Clone, Debug)] struct ProvisionalEvaluation { from_dfn: usize, + reached_depth: usize, result: EvaluationResult, } impl<'tcx> Default for ProvisionalEvaluationCache<'tcx> { fn default() -> Self { - Self { dfn: Cell::new(0), reached_depth: Cell::new(usize::MAX), map: Default::default() } + Self { dfn: Cell::new(0), map: Default::default() } } } @@ -2297,22 +2301,17 @@ impl<'tcx> ProvisionalEvaluationCache<'tcx> { /// Check the provisional cache for any result for /// `fresh_trait_ref`. If there is a hit, then you must consider /// it an access to the stack slots at depth - /// `self.current_reached_depth()` and above. - fn get_provisional(&self, fresh_trait_ref: ty::PolyTraitRef<'tcx>) -> Option { + /// `reached_depth` (from the returned value). + fn get_provisional( + &self, + fresh_trait_ref: ty::PolyTraitRef<'tcx>, + ) -> Option { debug!( ?fresh_trait_ref, - reached_depth = ?self.reached_depth.get(), "get_provisional = {:#?}", self.map.borrow().get(&fresh_trait_ref), ); - Some(self.map.borrow().get(&fresh_trait_ref)?.result) - } - - /// Current value of the `reached_depth` counter -- all the - /// provisional cache entries are dependent on the item at this - /// depth. - fn current_reached_depth(&self) -> usize { - self.reached_depth.get() + Some(self.map.borrow().get(&fresh_trait_ref)?.clone()) } /// Insert a provisional result into the cache. The result came @@ -2326,13 +2325,31 @@ impl<'tcx> ProvisionalEvaluationCache<'tcx> { fresh_trait_ref: ty::PolyTraitRef<'tcx>, result: EvaluationResult, ) { - debug!(?from_dfn, ?reached_depth, ?fresh_trait_ref, ?result, "insert_provisional"); - let r_d = self.reached_depth.get(); - self.reached_depth.set(r_d.min(reached_depth)); + debug!(?from_dfn, ?fresh_trait_ref, ?result, "insert_provisional"); - debug!(reached_depth = self.reached_depth.get()); + let mut map = self.map.borrow_mut(); + + // Subtle: when we complete working on the DFN `from_dfn`, anything + // that remains in the provisional cache must be dependent on some older + // stack entry than `from_dfn`. We have to update their depth with our transitive + // depth in that case or else it would be referring to some popped note. + // + // Example: + // A (reached depth 0) + // ... + // B // depth 1 -- reached depth = 0 + // C // depth 2 -- reached depth = 1 (should be 0) + // B + // A // depth 0 + // D (reached depth 1) + // C (cache -- reached depth = 2) + for (_k, v) in &mut *map { + if v.from_dfn >= from_dfn { + v.reached_depth = reached_depth.min(v.reached_depth); + } + } - self.map.borrow_mut().insert(fresh_trait_ref, ProvisionalEvaluation { from_dfn, result }); + map.insert(fresh_trait_ref, ProvisionalEvaluation { from_dfn, reached_depth, result }); } /// Invoked when the node with dfn `dfn` does not get a successful @@ -2360,25 +2377,40 @@ impl<'tcx> ProvisionalEvaluationCache<'tcx> { /// was a failure, then `on_failure` should have been invoked /// already). The callback `op` will be invoked for each /// provisional entry that we can now confirm. + /// + /// Note that we may still have provisional cache items remaining + /// in the cache when this is done. For example, if there is a + /// cycle: + /// + /// * A depends on... + /// * B depends on A + /// * C depends on... + /// * D depends on C + /// * ... + /// + /// Then as we complete the C node we will have a provisional cache + /// with results for A, B, C, and D. This method would clear out + /// the C and D results, but leave A and B provisional. + /// + /// This is determined based on the DFN: we remove any provisional + /// results created since `dfn` started (e.g., in our example, dfn + /// would be 2, representing the C node, and hence we would + /// remove the result for D, which has DFN 3, but not the results for + /// A and B, which have DFNs 0 and 1 respectively). fn on_completion( &self, - depth: usize, + dfn: usize, mut op: impl FnMut(ty::PolyTraitRef<'tcx>, EvaluationResult), ) { - debug!(?depth, reached_depth = ?self.reached_depth.get(), "on_completion"); + debug!(?dfn, "on_completion"); - if self.reached_depth.get() < depth { - debug!("on_completion: did not yet reach depth to complete"); - return; - } - - for (fresh_trait_ref, eval) in self.map.borrow_mut().drain() { + for (fresh_trait_ref, eval) in + self.map.borrow_mut().drain_filter(|_k, eval| eval.from_dfn >= dfn) + { debug!(?fresh_trait_ref, ?eval, "on_completion"); op(fresh_trait_ref, eval.result); } - - self.reached_depth.set(usize::MAX); } } diff --git a/compiler/rustc_ty_utils/src/needs_drop.rs b/compiler/rustc_ty_utils/src/needs_drop.rs index 64f82817d3..bc8f10e15d 100644 --- a/compiler/rustc_ty_utils/src/needs_drop.rs +++ b/compiler/rustc_ty_utils/src/needs_drop.rs @@ -6,7 +6,7 @@ use rustc_middle::ty::subst::Subst; use rustc_middle::ty::util::{needs_drop_components, AlwaysRequiresDrop}; use rustc_middle::ty::{self, Ty, TyCtxt}; use rustc_session::Limit; -use rustc_span::DUMMY_SP; +use rustc_span::{sym, DUMMY_SP}; type NeedsDropResult = Result; @@ -21,6 +21,19 @@ fn needs_drop_raw<'tcx>(tcx: TyCtxt<'tcx>, query: ty::ParamEnvAnd<'tcx, Ty<'tcx> res } +fn has_significant_drop_raw<'tcx>( + tcx: TyCtxt<'tcx>, + query: ty::ParamEnvAnd<'tcx, Ty<'tcx>>, +) -> bool { + let significant_drop_fields = + move |adt_def: &ty::AdtDef| tcx.adt_significant_drop_tys(adt_def.did).map(|tys| tys.iter()); + let res = NeedsDropTypes::new(tcx, query.param_env, query.value, significant_drop_fields) + .next() + .is_some(); + debug!("has_significant_drop_raw({:?}) = {:?}", query, res); + res +} + struct NeedsDropTypes<'tcx, F> { tcx: TyCtxt<'tcx>, param_env: ty::ParamEnv<'tcx>, @@ -155,12 +168,20 @@ where } } -fn adt_drop_tys(tcx: TyCtxt<'_>, def_id: DefId) -> Result<&ty::List>, AlwaysRequiresDrop> { +// This is a helper function for `adt_drop_tys` and `adt_significant_drop_tys`. +// Depending on the implentation of `adt_has_dtor`, it is used to check if the +// ADT has a destructor or if the ADT only has a significant destructor. For +// understanding significant destructor look at `adt_significant_drop_tys`. +fn adt_drop_tys_helper( + tcx: TyCtxt<'_>, + def_id: DefId, + adt_has_dtor: impl Fn(&ty::AdtDef) -> bool, +) -> Result<&ty::List>, AlwaysRequiresDrop> { let adt_components = move |adt_def: &ty::AdtDef| { if adt_def.is_manually_drop() { debug!("adt_drop_tys: `{:?}` is manually drop", adt_def); return Ok(Vec::new().into_iter()); - } else if adt_def.destructor(tcx).is_some() { + } else if adt_has_dtor(adt_def) { debug!("adt_drop_tys: `{:?}` implements `Drop`", adt_def); return Err(AlwaysRequiresDrop); } else if adt_def.is_union() { @@ -179,6 +200,30 @@ fn adt_drop_tys(tcx: TyCtxt<'_>, def_id: DefId) -> Result<&ty::List>, Alw res.map(|components| tcx.intern_type_list(&components)) } +fn adt_drop_tys(tcx: TyCtxt<'_>, def_id: DefId) -> Result<&ty::List>, AlwaysRequiresDrop> { + let adt_has_dtor = |adt_def: &ty::AdtDef| adt_def.destructor(tcx).is_some(); + adt_drop_tys_helper(tcx, def_id, adt_has_dtor) +} + +fn adt_significant_drop_tys( + tcx: TyCtxt<'_>, + def_id: DefId, +) -> Result<&ty::List>, AlwaysRequiresDrop> { + let adt_has_dtor = |adt_def: &ty::AdtDef| { + adt_def + .destructor(tcx) + .map(|dtor| !tcx.has_attr(dtor.did, sym::rustc_insignificant_dtor)) + .unwrap_or(false) + }; + adt_drop_tys_helper(tcx, def_id, adt_has_dtor) +} + pub(crate) fn provide(providers: &mut ty::query::Providers) { - *providers = ty::query::Providers { needs_drop_raw, adt_drop_tys, ..*providers }; + *providers = ty::query::Providers { + needs_drop_raw, + has_significant_drop_raw, + adt_drop_tys, + adt_significant_drop_tys, + ..*providers + }; } diff --git a/compiler/rustc_ty_utils/src/representability.rs b/compiler/rustc_ty_utils/src/representability.rs index ca001635a3..d3eb9fd955 100644 --- a/compiler/rustc_ty_utils/src/representability.rs +++ b/compiler/rustc_ty_utils/src/representability.rs @@ -25,11 +25,26 @@ pub enum Representability { pub fn ty_is_representable<'tcx>(tcx: TyCtxt<'tcx>, ty: Ty<'tcx>, sp: Span) -> Representability { debug!("is_type_representable: {:?}", ty); // To avoid a stack overflow when checking an enum variant or struct that - // contains a different, structurally recursive type, maintain a stack - // of seen types and check recursion for each of them (issues #3008, #3779). + // contains a different, structurally recursive type, maintain a stack of + // seen types and check recursion for each of them (issues #3008, #3779, + // #74224, #84611). `shadow_seen` contains the full stack and `seen` only + // the one for the current type (e.g. if we have structs A and B, B contains + // a field of type A, and we're currently looking at B, then `seen` will be + // cleared when recursing to check A, but `shadow_seen` won't, so that we + // can catch cases of mutual recursion where A also contains B). let mut seen: Vec> = Vec::new(); + let mut shadow_seen: Vec<&'tcx ty::AdtDef> = Vec::new(); let mut representable_cache = FxHashMap::default(); - let r = is_type_structurally_recursive(tcx, sp, &mut seen, &mut representable_cache, ty); + let mut force_result = false; + let r = is_type_structurally_recursive( + tcx, + sp, + &mut seen, + &mut shadow_seen, + &mut representable_cache, + ty, + &mut force_result, + ); debug!("is_type_representable: {:?} is {:?}", ty, r); r } @@ -48,21 +63,38 @@ fn are_inner_types_recursive<'tcx>( tcx: TyCtxt<'tcx>, sp: Span, seen: &mut Vec>, + shadow_seen: &mut Vec<&'tcx ty::AdtDef>, representable_cache: &mut FxHashMap, Representability>, ty: Ty<'tcx>, + force_result: &mut bool, ) -> Representability { + debug!("are_inner_types_recursive({:?}, {:?}, {:?})", ty, seen, shadow_seen); match ty.kind() { ty::Tuple(..) => { // Find non representable - fold_repr( - ty.tuple_fields().map(|ty| { - is_type_structurally_recursive(tcx, sp, seen, representable_cache, ty) - }), - ) + fold_repr(ty.tuple_fields().map(|ty| { + is_type_structurally_recursive( + tcx, + sp, + seen, + shadow_seen, + representable_cache, + ty, + force_result, + ) + })) } // Fixed-length vectors. // FIXME(#11924) Behavior undecided for zero-length vectors. - ty::Array(ty, _) => is_type_structurally_recursive(tcx, sp, seen, representable_cache, ty), + ty::Array(ty, _) => is_type_structurally_recursive( + tcx, + sp, + seen, + shadow_seen, + representable_cache, + ty, + force_result, + ), ty::Adt(def, substs) => { // Find non representable fields with their spans fold_repr(def.all_fields().map(|field| { @@ -76,12 +108,128 @@ fn are_inner_types_recursive<'tcx>( Some(hir::Node::Field(field)) => field.ty.span, _ => sp, }; - match is_type_structurally_recursive(tcx, span, seen, representable_cache, ty) { - Representability::SelfRecursive(_) => { - Representability::SelfRecursive(vec![span]) + + let mut result = None; + + // First, we check whether the field type per se is representable. + // This catches cases as in #74224 and #84611. There is a special + // case related to mutual recursion, though; consider this example: + // + // struct A { + // z: T, + // x: B, + // } + // + // struct B { + // y: A + // } + // + // Here, without the following special case, both A and B are + // ContainsRecursive, which is a problem because we only report + // errors for SelfRecursive. We fix this by detecting this special + // case (shadow_seen.first() is the type we are originally + // interested in, and if we ever encounter the same AdtDef again, + // we know that it must be SelfRecursive) and "forcibly" returning + // SelfRecursive (by setting force_result, which tells the calling + // invocations of are_inner_types_representable to forward the + // result without adjusting). + if shadow_seen.len() > seen.len() && shadow_seen.first() == Some(def) { + *force_result = true; + result = Some(Representability::SelfRecursive(vec![span])); + } + + if result == None { + result = Some(Representability::Representable); + + // Now, we check whether the field types per se are representable, e.g. + // for struct Foo { x: Option }, we first check whether Option<_> + // by itself is representable (which it is), and the nesting of Foo + // will be detected later. This is necessary for #74224 and #84611. + + // If we have encountered an ADT definition that we have not seen + // before (no need to check them twice), recurse to see whether that + // definition is SelfRecursive. If so, we must be ContainsRecursive. + if shadow_seen.len() > 1 + && !shadow_seen + .iter() + .take(shadow_seen.len() - 1) + .any(|seen_def| seen_def == def) + { + let adt_def_id = def.did; + let raw_adt_ty = tcx.type_of(adt_def_id); + debug!("are_inner_types_recursive: checking nested type: {:?}", raw_adt_ty); + + // Check independently whether the ADT is SelfRecursive. If so, + // we must be ContainsRecursive (except for the special case + // mentioned above). + let mut nested_seen: Vec> = vec![]; + result = Some( + match is_type_structurally_recursive( + tcx, + span, + &mut nested_seen, + shadow_seen, + representable_cache, + raw_adt_ty, + force_result, + ) { + Representability::SelfRecursive(_) => { + if *force_result { + Representability::SelfRecursive(vec![span]) + } else { + Representability::ContainsRecursive + } + } + x => x, + }, + ); + } + + // We only enter the following block if the type looks representable + // so far. This is necessary for cases such as this one (#74224): + // + // struct A { + // x: T, + // y: A>, + // } + // + // struct B { + // z: A + // } + // + // When checking B, we recurse into A and check field y of type + // A>. We haven't seen this exact type before, so we recurse + // into A>, which contains, A>>, and so forth, + // ad infinitum. We can prevent this from happening by first checking + // A separately (the code above) and only checking for nested Bs if + // A actually looks representable (which it wouldn't in this example). + if result == Some(Representability::Representable) { + // Now, even if the type is representable (e.g. Option<_>), + // it might still contribute to a recursive type, e.g.: + // struct Foo { x: Option> } + // These cases are handled by passing the full `seen` + // stack to is_type_structurally_recursive (instead of the + // empty `nested_seen` above): + result = Some( + match is_type_structurally_recursive( + tcx, + span, + seen, + shadow_seen, + representable_cache, + ty, + force_result, + ) { + Representability::SelfRecursive(_) => { + Representability::SelfRecursive(vec![span]) + } + x => x, + }, + ); } - x => x, } + + result.unwrap() })) } ty::Closure(..) => { @@ -106,8 +254,10 @@ fn is_type_structurally_recursive<'tcx>( tcx: TyCtxt<'tcx>, sp: Span, seen: &mut Vec>, + shadow_seen: &mut Vec<&'tcx ty::AdtDef>, representable_cache: &mut FxHashMap, Representability>, ty: Ty<'tcx>, + force_result: &mut bool, ) -> Representability { debug!("is_type_structurally_recursive: {:?} {:?}", ty, sp); if let Some(representability) = representable_cache.get(ty) { @@ -118,8 +268,15 @@ fn is_type_structurally_recursive<'tcx>( return representability.clone(); } - let representability = - is_type_structurally_recursive_inner(tcx, sp, seen, representable_cache, ty); + let representability = is_type_structurally_recursive_inner( + tcx, + sp, + seen, + shadow_seen, + representable_cache, + ty, + force_result, + ); representable_cache.insert(ty, representability.clone()); representability @@ -129,12 +286,16 @@ fn is_type_structurally_recursive_inner<'tcx>( tcx: TyCtxt<'tcx>, sp: Span, seen: &mut Vec>, + shadow_seen: &mut Vec<&'tcx ty::AdtDef>, representable_cache: &mut FxHashMap, Representability>, ty: Ty<'tcx>, + force_result: &mut bool, ) -> Representability { match ty.kind() { ty::Adt(def, _) => { { + debug!("is_type_structurally_recursive_inner: adt: {:?}, seen: {:?}", ty, seen); + // Iterate through stack of previously seen types. let mut iter = seen.iter(); @@ -158,8 +319,10 @@ fn is_type_structurally_recursive_inner<'tcx>( // will recurse infinitely for some inputs. // // It is important that we DO take generic parameters into account - // here, so that code like this is considered SelfRecursive, not - // ContainsRecursive: + // here, because nesting e.g. Options is allowed (as long as the + // definition of Option doesn't itself include an Option field, which + // would be a case of SelfRecursive above). The following, too, counts + // as SelfRecursive: // // struct Foo { Option> } @@ -174,13 +337,31 @@ fn is_type_structurally_recursive_inner<'tcx>( // For structs and enums, track all previously seen types by pushing them // onto the 'seen' stack. seen.push(ty); - let out = are_inner_types_recursive(tcx, sp, seen, representable_cache, ty); + shadow_seen.push(def); + let out = are_inner_types_recursive( + tcx, + sp, + seen, + shadow_seen, + representable_cache, + ty, + force_result, + ); + shadow_seen.pop(); seen.pop(); out } _ => { // No need to push in other cases. - are_inner_types_recursive(tcx, sp, seen, representable_cache, ty) + are_inner_types_recursive( + tcx, + sp, + seen, + shadow_seen, + representable_cache, + ty, + force_result, + ) } } } diff --git a/compiler/rustc_ty_utils/src/ty.rs b/compiler/rustc_ty_utils/src/ty.rs index 38e5ce6fd8..ebc7b0d0d9 100644 --- a/compiler/rustc_ty_utils/src/ty.rs +++ b/compiler/rustc_ty_utils/src/ty.rs @@ -1,5 +1,4 @@ use rustc_data_structures::fx::FxIndexSet; -use rustc_data_structures::svh::Svh; use rustc_hir as hir; use rustc_hir::def_id::{CrateNum, DefId, LocalDefId, LOCAL_CRATE}; use rustc_middle::hir::map as hir_map; @@ -8,7 +7,6 @@ use rustc_middle::ty::{ self, Binder, Predicate, PredicateKind, ToPredicate, Ty, TyCtxt, WithConstness, }; use rustc_session::CrateDisambiguator; -use rustc_span::symbol::Symbol; use rustc_span::Span; use rustc_trait_selection::traits; @@ -395,15 +393,6 @@ fn crate_disambiguator(tcx: TyCtxt<'_>, crate_num: CrateNum) -> CrateDisambiguat tcx.sess.local_crate_disambiguator() } -fn original_crate_name(tcx: TyCtxt<'_>, crate_num: CrateNum) -> Symbol { - assert_eq!(crate_num, LOCAL_CRATE); - tcx.crate_name -} - -fn crate_hash(tcx: TyCtxt<'_>, crate_num: CrateNum) -> Svh { - tcx.index_hir(crate_num).crate_hash -} - fn instance_def_size_estimate<'tcx>( tcx: TyCtxt<'tcx>, instance_def: ty::InstanceDef<'tcx>, @@ -550,8 +539,6 @@ pub fn provide(providers: &mut ty::query::Providers) { param_env_reveal_all_normalized, trait_of_item, crate_disambiguator, - original_crate_name, - crate_hash, instance_def_size_estimate, issue33140_self_ty, impl_defaultness, diff --git a/compiler/rustc_type_ir/src/lib.rs b/compiler/rustc_type_ir/src/lib.rs index 8fcdf813b4..2d102127dd 100644 --- a/compiler/rustc_type_ir/src/lib.rs +++ b/compiler/rustc_type_ir/src/lib.rs @@ -1,3 +1,5 @@ +#![feature(min_specialization)] + #[macro_use] extern crate bitflags; #[macro_use] @@ -59,6 +61,15 @@ bitflags! { | TypeFlags::HAS_CT_INFER.bits | TypeFlags::HAS_TY_PLACEHOLDER.bits | TypeFlags::HAS_CT_PLACEHOLDER.bits + // We consider 'freshened' types and constants + // to depend on a particular fn. + // The freshening process throws away information, + // which can make things unsuitable for use in a global + // cache. Note that there is no 'fresh lifetime' flag - + // freshening replaces all lifetimes with `ReErased`, + // which is different from how types/const are freshened. + | TypeFlags::HAS_TY_FRESH.bits + | TypeFlags::HAS_CT_FRESH.bits | TypeFlags::HAS_FREE_LOCAL_REGIONS.bits; /// Does this have `Projection`? @@ -90,6 +101,12 @@ bitflags! { /// Does this value have parameters/placeholders/inference variables which could be /// replaced later, in a way that would change the results of `impl` specialization? const STILL_FURTHER_SPECIALIZABLE = 1 << 17; + + /// Does this value have `InferTy::FreshTy/FreshIntTy/FreshFloatTy`? + const HAS_TY_FRESH = 1 << 18; + + /// Does this value have `InferConst::Fresh`? + const HAS_CT_FRESH = 1 << 19; } } diff --git a/compiler/rustc_typeck/src/astconv/generics.rs b/compiler/rustc_typeck/src/astconv/generics.rs index a3804e468d..456f2a908a 100644 --- a/compiler/rustc_typeck/src/astconv/generics.rs +++ b/compiler/rustc_typeck/src/astconv/generics.rs @@ -4,7 +4,7 @@ use crate::astconv::{ GenericArgCountResult, GenericArgPosition, }; use crate::errors::AssocTypeBindingNotAllowed; -use crate::structured_errors::{StructuredDiagnostic, WrongNumberOfGenericArgs}; +use crate::structured_errors::{GenericArgsInfo, StructuredDiagnostic, WrongNumberOfGenericArgs}; use rustc_ast::ast::ParamKindOrd; use rustc_errors::{struct_span_err, Applicability, DiagnosticBuilder, ErrorReported}; use rustc_hir as hir; @@ -278,9 +278,7 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o { // another. This is an error. However, if we already know that // the arguments don't match up with the parameters, we won't issue // an additional error, as the user already knows what's wrong. - if arg_count.correct.is_ok() - && arg_count.explicit_late_bound == ExplicitLateBound::No - { + if arg_count.correct.is_ok() { // We're going to iterate over the parameters to sort them out, and // show that order to the user as a possible order for the parameters let mut param_types_present = defs @@ -440,6 +438,11 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o { has_self: bool, infer_args: bool, ) -> GenericArgCountResult { + debug!( + "check_generic_arg_count(span: {:?}, def_id: {:?}, seg: {:?}, gen_params: {:?}, gen_args: {:?})", + span, def_id, seg, gen_params, gen_args + ); + let default_counts = gen_params.own_defaults(); let param_counts = gen_params.own_counts(); let named_type_param_count = param_counts.types - has_self as usize; @@ -455,63 +458,116 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o { let mut invalid_args = vec![]; - let mut check_generics = - |kind, expected_min, expected_max, provided, params_offset, args_offset, silent| { + let mut check_lifetime_args = |min_expected_args: usize, + max_expected_args: usize, + provided_args: usize, + late_bounds_ignore: bool| + -> bool { + if (min_expected_args..=max_expected_args).contains(&provided_args) { + return true; + } + + if late_bounds_ignore { + return true; + } + + if provided_args > max_expected_args { + invalid_args.extend( + gen_args.args[max_expected_args..provided_args].iter().map(|arg| arg.span()), + ); + }; + + let gen_args_info = if provided_args > min_expected_args { + invalid_args.extend( + gen_args.args[min_expected_args..provided_args].iter().map(|arg| arg.span()), + ); + let num_redundant_args = provided_args - min_expected_args; + GenericArgsInfo::ExcessLifetimes { num_redundant_args } + } else { + let num_missing_args = min_expected_args - provided_args; + GenericArgsInfo::MissingLifetimes { num_missing_args } + }; + + WrongNumberOfGenericArgs::new( + tcx, + gen_args_info, + seg, + gen_params, + has_self as usize, + gen_args, + def_id, + ) + .diagnostic() + .emit(); + + false + }; + + let min_expected_lifetime_args = if infer_lifetimes { 0 } else { param_counts.lifetimes }; + let max_expected_lifetime_args = param_counts.lifetimes; + let num_provided_lifetime_args = arg_counts.lifetimes; + + let lifetimes_correct = check_lifetime_args( + min_expected_lifetime_args, + max_expected_lifetime_args, + num_provided_lifetime_args, + explicit_late_bound == ExplicitLateBound::Yes, + ); + + let mut check_types_and_consts = + |expected_min, expected_max, provided, params_offset, args_offset| { + debug!( + "check_types_and_consts(expected_min: {:?}, expected_max: {:?}, \ + provided: {:?}, params_offset: {:?}, args_offset: {:?}", + expected_min, expected_max, provided, params_offset, args_offset + ); if (expected_min..=expected_max).contains(&provided) { return true; } - if silent { - return false; - } + let num_default_params = expected_max - expected_min; - if provided > expected_max { + let gen_args_info = if provided > expected_max { invalid_args.extend( gen_args.args[args_offset + expected_max..args_offset + provided] .iter() .map(|arg| arg.span()), ); + let num_redundant_args = provided - expected_max; + + GenericArgsInfo::ExcessTypesOrConsts { + num_redundant_args, + num_default_params, + args_offset, + } + } else { + let num_missing_args = expected_max - provided; + + GenericArgsInfo::MissingTypesOrConsts { + num_missing_args, + num_default_params, + args_offset, + } }; - WrongNumberOfGenericArgs { + debug!("gen_args_info: {:?}", gen_args_info); + + WrongNumberOfGenericArgs::new( tcx, - kind, - expected_min, - expected_max, - provided, - params_offset, - args_offset, - path_segment: seg, + gen_args_info, + seg, gen_params, + params_offset, gen_args, def_id, - span, - } + ) .diagnostic() .emit(); false }; - let lifetimes_correct = check_generics( - "lifetime", - if infer_lifetimes { 0 } else { param_counts.lifetimes }, - param_counts.lifetimes, - arg_counts.lifetimes, - has_self as usize, - 0, - explicit_late_bound == ExplicitLateBound::Yes, - ); - let args_correct = { - let kind = if param_counts.consts + arg_counts.consts == 0 { - "type" - } else if named_type_param_count + arg_counts.types == 0 { - "const" - } else { - "generic" - }; - let expected_min = if infer_args { 0 } else { @@ -519,15 +575,15 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o { - default_counts.types - default_counts.consts }; + debug!("expected_min: {:?}", expected_min); + debug!("arg_counts.lifetimes: {:?}", arg_counts.lifetimes); - check_generics( - kind, + check_types_and_consts( expected_min, param_counts.consts + named_type_param_count, arg_counts.consts + arg_counts.types, param_counts.lifetimes + has_self as usize, arg_counts.lifetimes, - false, ) }; diff --git a/compiler/rustc_typeck/src/astconv/mod.rs b/compiler/rustc_typeck/src/astconv/mod.rs index 2f2e90e4bd..ef19562104 100644 --- a/compiler/rustc_typeck/src/astconv/mod.rs +++ b/compiler/rustc_typeck/src/astconv/mod.rs @@ -120,6 +120,7 @@ pub enum SizedByDefault { #[derive(Debug)] struct ConvertedBinding<'a, 'tcx> { + hir_id: hir::HirId, item_name: Ident, kind: ConvertedBindingKind<'a, 'tcx>, gen_args: &'a GenericArgs<'a>, @@ -431,6 +432,7 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o { param.def_id, Some(arg.id()), arg.span(), + None, |_, _| { // Default generic parameters may not be marked // with stability attributes, i.e. when the @@ -590,6 +592,7 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o { } }; ConvertedBinding { + hir_id: binding.hir_id, item_name: binding.ident, kind, gen_args: binding.gen_args, @@ -609,6 +612,10 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o { item_segment: &hir::PathSegment<'_>, parent_substs: SubstsRef<'tcx>, ) -> SubstsRef<'tcx> { + debug!( + "create_substs_for_associated_item(span: {:?}, item_def_id: {:?}, item_segment: {:?}", + span, item_def_id, item_segment + ); if tcx.generics_of(item_def_id).params.is_empty() { self.prohibit_generics(slice::from_ref(item_segment)); @@ -1053,7 +1060,7 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o { .span_label(binding.span, "private associated type") .emit(); } - tcx.check_stability(assoc_ty.def_id, Some(hir_ref_id), binding.span); + tcx.check_stability(assoc_ty.def_id, Some(hir_ref_id), binding.span, None); if !speculative { dup_bindings @@ -1071,9 +1078,10 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o { // Include substitutions for generic parameters of associated types let projection_ty = candidate.map_bound(|trait_ref| { + let ident = Ident::new(assoc_ty.ident.name, binding.item_name.span); let item_segment = hir::PathSegment { - ident: assoc_ty.ident, - hir_id: None, + ident, + hir_id: Some(binding.hir_id), res: None, args: Some(binding.gen_args), infer_args: false, @@ -1386,11 +1394,13 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o { let auto_trait_predicates = auto_traits.into_iter().map(|trait_ref| { ty::Binder::dummy(ty::ExistentialPredicate::AutoTrait(trait_ref.trait_ref().def_id())) }); + // N.b. principal, projections, auto traits + // FIXME: This is actually wrong with multiple principals in regards to symbol mangling let mut v = regular_trait_predicates - .chain(auto_trait_predicates) .chain( existential_projections.map(|x| x.map_bound(ty::ExistentialPredicate::Projection)), ) + .chain(auto_trait_predicates) .collect::>(); v.sort_by(|a, b| a.skip_binder().stable_cmp(tcx, &b.skip_binder())); v.dedup(); @@ -1659,7 +1669,7 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o { .find(|vd| tcx.hygienic_eq(assoc_ident, vd.ident, adt_def.did)); if let Some(variant_def) = variant_def { if permit_variants { - tcx.check_stability(variant_def.def_id, Some(hir_ref_id), span); + tcx.check_stability(variant_def.def_id, Some(hir_ref_id), span, None); self.prohibit_generics(slice::from_ref(assoc_segment)); return Ok((qself_ty, DefKind::Variant, variant_def.def_id)); } else { @@ -1779,7 +1789,7 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o { .span_label(span, &format!("private {}", kind)) .emit(); } - tcx.check_stability(item.def_id, Some(hir_ref_id), span); + tcx.check_stability(item.def_id, Some(hir_ref_id), span, None); if let Some(variant_def_id) = variant_resolution { tcx.struct_span_lint_hir(AMBIGUOUS_ASSOCIATED_ITEMS, hir_ref_id, span, |lint| { diff --git a/compiler/rustc_typeck/src/check/callee.rs b/compiler/rustc_typeck/src/check/callee.rs index b48102e0fc..cb8f336721 100644 --- a/compiler/rustc_typeck/src/check/callee.rs +++ b/compiler/rustc_typeck/src/check/callee.rs @@ -6,8 +6,14 @@ use rustc_errors::{struct_span_err, Applicability, DiagnosticBuilder}; use rustc_hir as hir; use rustc_hir::def::{Namespace, Res}; use rustc_hir::def_id::{DefId, LOCAL_CRATE}; -use rustc_infer::infer::type_variable::{TypeVariableOrigin, TypeVariableOriginKind}; -use rustc_infer::{infer, traits}; +use rustc_infer::{ + infer, + traits::{self, Obligation}, +}; +use rustc_infer::{ + infer::type_variable::{TypeVariableOrigin, TypeVariableOriginKind}, + traits::ObligationCause, +}; use rustc_middle::ty::adjustment::{ Adjust, Adjustment, AllowTwoPhase, AutoBorrow, AutoBorrowMutability, }; @@ -17,6 +23,7 @@ use rustc_span::symbol::{sym, Ident}; use rustc_span::Span; use rustc_target::spec::abi; use rustc_trait_selection::autoderef::Autoderef; +use rustc_trait_selection::traits::query::evaluate_obligation::InferCtxtExt; use std::iter; /// Checks that it is legal to call methods of the trait corresponding @@ -294,7 +301,34 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> { expected: Expectation<'tcx>, ) -> Ty<'tcx> { let (fn_sig, def_id) = match *callee_ty.kind() { - ty::FnDef(def_id, _) => (callee_ty.fn_sig(self.tcx), Some(def_id)), + ty::FnDef(def_id, subst) => { + // Unit testing: function items annotated with + // `#[rustc_evaluate_where_clauses]` trigger special output + // to let us test the trait evaluation system. + if self.tcx.has_attr(def_id, sym::rustc_evaluate_where_clauses) { + let predicates = self.tcx.predicates_of(def_id); + let predicates = predicates.instantiate(self.tcx, subst); + for (predicate, predicate_span) in + predicates.predicates.iter().zip(&predicates.spans) + { + let obligation = Obligation::new( + ObligationCause::dummy_with_span(callee_expr.span), + self.param_env, + predicate.clone(), + ); + let result = self.infcx.evaluate_obligation(&obligation); + self.tcx + .sess + .struct_span_err( + callee_expr.span, + &format!("evaluate({:?}) = {:?}", predicate, result), + ) + .span_label(*predicate_span, "predicate") + .emit(); + } + } + (callee_ty.fn_sig(self.tcx), Some(def_id)) + } ty::FnPtr(sig) => (sig, None), ref t => { let mut unit_variant = None; diff --git a/compiler/rustc_typeck/src/check/cast.rs b/compiler/rustc_typeck/src/check/cast.rs index b760a54f08..3cbc3d231f 100644 --- a/compiler/rustc_typeck/src/check/cast.rs +++ b/compiler/rustc_typeck/src/check/cast.rs @@ -35,6 +35,7 @@ use crate::type_error_struct; use rustc_errors::{struct_span_err, Applicability, DiagnosticBuilder, ErrorReported}; use rustc_hir as hir; use rustc_hir::lang_items::LangItem; +use rustc_middle::mir::Mutability; use rustc_middle::ty::adjustment::AllowTwoPhase; use rustc_middle::ty::cast::{CastKind, CastTy}; use rustc_middle::ty::error::TypeError; @@ -347,15 +348,52 @@ impl<'a, 'tcx> CastCheck<'tcx> { fcx.ty_to_string(self.cast_ty) ); let mut sugg = None; + let mut sugg_mutref = false; if let ty::Ref(reg, _, mutbl) = *self.cast_ty.kind() { - if fcx - .try_coerce( - self.expr, - fcx.tcx.mk_ref(reg, TypeAndMut { ty: self.expr_ty, mutbl }), - self.cast_ty, - AllowTwoPhase::No, - ) - .is_ok() + if let ty::RawPtr(TypeAndMut { ty: expr_ty, .. }) = *self.expr_ty.kind() { + if fcx + .try_coerce( + self.expr, + fcx.tcx.mk_ref( + &ty::RegionKind::ReErased, + TypeAndMut { ty: expr_ty, mutbl }, + ), + self.cast_ty, + AllowTwoPhase::No, + ) + .is_ok() + { + sugg = Some(format!("&{}*", mutbl.prefix_str())); + } + } else if let ty::Ref(expr_reg, expr_ty, expr_mutbl) = *self.expr_ty.kind() { + if expr_mutbl == Mutability::Not + && mutbl == Mutability::Mut + && fcx + .try_coerce( + self.expr, + fcx.tcx.mk_ref( + expr_reg, + TypeAndMut { ty: expr_ty, mutbl: Mutability::Mut }, + ), + self.cast_ty, + AllowTwoPhase::No, + ) + .is_ok() + { + sugg_mutref = true; + } + } + + if !sugg_mutref + && sugg == None + && fcx + .try_coerce( + self.expr, + fcx.tcx.mk_ref(reg, TypeAndMut { ty: self.expr_ty, mutbl }), + self.cast_ty, + AllowTwoPhase::No, + ) + .is_ok() { sugg = Some(format!("&{}", mutbl.prefix_str())); } @@ -375,11 +413,15 @@ impl<'a, 'tcx> CastCheck<'tcx> { sugg = Some(format!("&{}", mutbl.prefix_str())); } } - if let Some(sugg) = sugg { + if sugg_mutref { + err.span_label(self.span, "invalid cast"); + err.span_note(self.expr.span, "this reference is immutable"); + err.span_note(self.cast_span, "trying to cast to a mutable reference type"); + } else if let Some(sugg) = sugg { err.span_label(self.span, "invalid cast"); err.span_suggestion_verbose( self.expr.span.shrink_to_lo(), - "borrow the value for the cast to be valid", + "consider borrowing the value", sugg, Applicability::MachineApplicable, ); diff --git a/compiler/rustc_typeck/src/check/check.rs b/compiler/rustc_typeck/src/check/check.rs index 92d7ea2600..70d85796d0 100644 --- a/compiler/rustc_typeck/src/check/check.rs +++ b/compiler/rustc_typeck/src/check/check.rs @@ -16,7 +16,7 @@ use rustc_middle::ty::fold::TypeFoldable; use rustc_middle::ty::layout::MAX_SIMD_LANES; use rustc_middle::ty::subst::GenericArgKind; use rustc_middle::ty::util::{Discr, IntTypeExt}; -use rustc_middle::ty::{self, ParamEnv, RegionKind, Ty, TyCtxt}; +use rustc_middle::ty::{self, OpaqueTypeKey, ParamEnv, RegionKind, Ty, TyCtxt}; use rustc_session::lint::builtin::UNINHABITED_STATIC; use rustc_span::symbol::sym; use rustc_span::{self, MultiSpan, Span}; @@ -716,10 +716,10 @@ fn check_opaque_meets_bounds<'tcx>( infcx.instantiate_opaque_types(def_id, hir_id, param_env, opaque_ty, span), ); - for (def_id, opaque_defn) in opaque_type_map { + for (OpaqueTypeKey { def_id, substs }, opaque_defn) in opaque_type_map { match infcx .at(&misc_cause, param_env) - .eq(opaque_defn.concrete_ty, tcx.type_of(def_id).subst(tcx, opaque_defn.substs)) + .eq(opaque_defn.concrete_ty, tcx.type_of(def_id).subst(tcx, substs)) { Ok(infer_ok) => inh.register_infer_ok_obligations(infer_ok), Err(ty_err) => tcx.sess.delay_span_bug( @@ -1214,10 +1214,19 @@ pub fn check_simd(tcx: TyCtxt<'_>, sp: Span, def_id: LocalDefId) { } } + // Check that we use types valid for use in the lanes of a SIMD "vector register" + // These are scalar types which directly match a "machine" type + // Yes: Integers, floats, "thin" pointers + // No: char, "fat" pointers, compound types match e.kind() { - ty::Param(_) => { /* struct(T, T, T, T) is ok */ } - _ if e.is_machine() => { /* struct(u8, u8, u8, u8) is ok */ } - ty::Array(ty, _c) if ty.is_machine() => { /* struct([f32; 4]) */ } + ty::Param(_) => (), // pass struct(T, T, T, T) through, let monomorphization catch errors + ty::Int(_) | ty::Uint(_) | ty::Float(_) | ty::RawPtr(_) => (), // struct(u8, u8, u8, u8) is ok + ty::Array(t, _clen) + if matches!( + t.kind(), + ty::Int(_) | ty::Uint(_) | ty::Float(_) | ty::RawPtr(_) + ) => + { /* struct([f32; 4]) is ok */ } _ => { struct_span_err!( tcx.sess, diff --git a/compiler/rustc_typeck/src/check/demand.rs b/compiler/rustc_typeck/src/check/demand.rs index e5fcdcfa74..33bc25accb 100644 --- a/compiler/rustc_typeck/src/check/demand.rs +++ b/compiler/rustc_typeck/src/check/demand.rs @@ -8,6 +8,7 @@ use rustc_errors::{Applicability, DiagnosticBuilder}; use rustc_hir as hir; use rustc_hir::lang_items::LangItem; use rustc_hir::{is_range_literal, Node}; +use rustc_middle::lint::in_external_macro; use rustc_middle::ty::adjustment::AllowTwoPhase; use rustc_middle::ty::{self, AssocItem, Ty, TypeAndMut}; use rustc_span::symbol::sym; @@ -16,6 +17,7 @@ use rustc_span::Span; use super::method::probe; use std::fmt; +use std::iter; impl<'a, 'tcx> FnCtxt<'a, 'tcx> { pub fn emit_coerce_suggestions( @@ -412,14 +414,16 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> { checked_ty: Ty<'tcx>, expected: Ty<'tcx>, ) -> Option<(Span, &'static str, String, Applicability)> { - let sm = self.sess().source_map(); + let sess = self.sess(); let sp = expr.span; - if sm.is_imported(sp) { - // Ignore if span is from within a macro #41858, #58298. We previously used the macro - // call span, but that breaks down when the type error comes from multiple calls down. + + // If the span is from an external macro, there's no suggestion we can make. + if in_external_macro(sess, sp) { return None; } + let sm = sess.source_map(); + let replace_prefix = |s: &str, old: &str, new: &str| { s.strip_prefix(old).map(|stripped| new.to_string() + stripped) }; @@ -427,10 +431,6 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> { let is_struct_pat_shorthand_field = self.is_hir_id_from_struct_pattern_shorthand_field(expr.hir_id, sp); - // If the span is from a macro, then it's hard to extract the text - // and make a good suggestion, so don't bother. - let is_macro = sp.from_expansion() && sp.desugaring_kind().is_none(); - // `ExprKind::DropTemps` is semantically irrelevant for these suggestions. let expr = expr.peel_drop_temps(); @@ -570,32 +570,38 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> { hir::ExprKind::AddrOf(hir::BorrowKind::Ref, _, ref expr), _, &ty::Ref(_, checked, _), - ) if { - self.infcx.can_sub(self.param_env, checked, &expected).is_ok() && !is_macro - } => - { + ) if self.infcx.can_sub(self.param_env, checked, &expected).is_ok() => { // We have `&T`, check if what was expected was `T`. If so, // we may want to suggest removing a `&`. if sm.is_imported(expr.span) { - if let Ok(src) = sm.span_to_snippet(sp) { - if let Some(src) = src.strip_prefix('&') { + // Go through the spans from which this span was expanded, + // and find the one that's pointing inside `sp`. + // + // E.g. for `&format!("")`, where we want the span to the + // `format!()` invocation instead of its expansion. + if let Some(call_span) = + iter::successors(Some(expr.span), |s| s.parent()).find(|&s| sp.contains(s)) + { + if let Ok(code) = sm.span_to_snippet(call_span) { return Some(( sp, "consider removing the borrow", - src.to_string(), + code, Applicability::MachineApplicable, )); } } return None; } - if let Ok(code) = sm.span_to_snippet(expr.span) { - return Some(( - sp, - "consider removing the borrow", - code, - Applicability::MachineApplicable, - )); + if sp.contains(expr.span) { + if let Ok(code) = sm.span_to_snippet(expr.span) { + return Some(( + sp, + "consider removing the borrow", + code, + Applicability::MachineApplicable, + )); + } } } ( @@ -643,7 +649,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> { } } } - _ if sp == expr.span && !is_macro => { + _ if sp == expr.span => { if let Some(steps) = self.deref_steps(checked_ty, expected) { let expr = expr.peel_blocks(); diff --git a/compiler/rustc_typeck/src/check/dropck.rs b/compiler/rustc_typeck/src/check/dropck.rs index de6336b254..01276495c1 100644 --- a/compiler/rustc_typeck/src/check/dropck.rs +++ b/compiler/rustc_typeck/src/check/dropck.rs @@ -310,6 +310,7 @@ impl TypeRelation<'tcx> for SimpleEqRelation<'tcx> { fn relate_with_variance>( &mut self, _: ty::Variance, + _info: ty::VarianceDiagInfo<'tcx>, a: T, b: T, ) -> RelateResult<'tcx, T> { diff --git a/compiler/rustc_typeck/src/check/expr.rs b/compiler/rustc_typeck/src/check/expr.rs index 991c2ba693..d0cbb58fb1 100644 --- a/compiler/rustc_typeck/src/check/expr.rs +++ b/compiler/rustc_typeck/src/check/expr.rs @@ -1230,7 +1230,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> { // struct-like enums (yet...), but it's definitely not // a bug to have constructed one. if adt_kind != AdtKind::Enum { - tcx.check_stability(v_field.did, Some(expr_id), field.span); + tcx.check_stability(v_field.did, Some(expr_id), field.span, None); } self.field_ty(field.span, v_field, substs) @@ -1571,7 +1571,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> { self.apply_adjustments(base, adjustments); self.register_predicates(autoderef.into_obligations()); - self.tcx.check_stability(field.did, Some(expr.hir_id), expr.span); + self.tcx.check_stability(field.did, Some(expr.hir_id), expr.span, None); return field_ty; } private_candidate = Some((base_def.did, field_ty)); diff --git a/compiler/rustc_typeck/src/check/fn_ctxt/_impl.rs b/compiler/rustc_typeck/src/check/fn_ctxt/_impl.rs index a50f8e1c65..96569ae0e7 100644 --- a/compiler/rustc_typeck/src/check/fn_ctxt/_impl.rs +++ b/compiler/rustc_typeck/src/check/fn_ctxt/_impl.rs @@ -30,6 +30,7 @@ use rustc_middle::ty::{ use rustc_session::lint; use rustc_session::lint::builtin::BARE_TRAIT_OBJECTS; use rustc_session::parse::feature_err; +use rustc_span::edition::Edition; use rustc_span::source_map::{original_sp, DUMMY_SP}; use rustc_span::symbol::{kw, sym, Ident}; use rustc_span::{self, BytePos, MultiSpan, Span}; @@ -719,11 +720,11 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> { pub(in super::super) fn select_obligations_where_possible( &self, fallback_has_occurred: bool, - mutate_fullfillment_errors: impl Fn(&mut Vec>), + mutate_fulfillment_errors: impl Fn(&mut Vec>), ) { let result = self.fulfillment_cx.borrow_mut().select_where_possible(self); if let Err(mut errors) = result { - mutate_fullfillment_errors(&mut errors); + mutate_fulfillment_errors(&mut errors); self.report_fulfillment_errors(&errors, self.inh.body_id, fallback_has_occurred); } } @@ -969,20 +970,42 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> { if let TyKind::TraitObject([poly_trait_ref, ..], _, TraitObjectSyntax::None) = self_ty.kind { - self.tcx.struct_span_lint_hir(BARE_TRAIT_OBJECTS, hir_id, self_ty.span, |lint| { - let mut db = lint - .build(&format!("trait objects without an explicit `dyn` are deprecated")); - let (sugg, app) = match self.tcx.sess.source_map().span_to_snippet(self_ty.span) - { - Ok(s) if poly_trait_ref.trait_ref.path.is_global() => { - (format!("", s), Applicability::MachineApplicable) - } - Ok(s) => (format!("", s), Applicability::MachineApplicable), - Err(_) => (">".to_string(), Applicability::HasPlaceholders), - }; - db.span_suggestion(self_ty.span, "use `dyn`", sugg, app); - db.emit() - }); + let msg = "trait objects without an explicit `dyn` are deprecated"; + let (sugg, app) = match self.tcx.sess.source_map().span_to_snippet(self_ty.span) { + Ok(s) if poly_trait_ref.trait_ref.path.is_global() => { + (format!("", s), Applicability::MachineApplicable) + } + Ok(s) => (format!("", s), Applicability::MachineApplicable), + Err(_) => (">".to_string(), Applicability::HasPlaceholders), + }; + let replace = String::from("use `dyn`"); + if self.sess().edition() >= Edition::Edition2021 { + let mut err = rustc_errors::struct_span_err!( + self.sess(), + self_ty.span, + E0783, + "{}", + msg, + ); + err.span_suggestion( + self_ty.span, + &sugg, + replace, + Applicability::MachineApplicable, + ) + .emit(); + } else { + self.tcx.struct_span_lint_hir( + BARE_TRAIT_OBJECTS, + hir_id, + self_ty.span, + |lint| { + let mut db = lint.build(msg); + db.span_suggestion(self_ty.span, &replace, sugg, app); + db.emit() + }, + ); + } } } } @@ -1282,6 +1305,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> { let mut infer_args_for_err = FxHashSet::default(); + let mut explicit_late_bound = ExplicitLateBound::No; for &PathSeg(def_id, index) in &path_segs { let seg = &segments[index]; let generics = tcx.generics_of(def_id); @@ -1290,17 +1314,20 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> { // parameter internally, but we don't allow users to specify the // parameter's value explicitly, so we have to do some error- // checking here. - if let GenericArgCountResult { - correct: Err(GenericArgCountMismatch { reported: Some(_), .. }), - .. - } = >::check_generic_arg_count_for_call( + let arg_count = >::check_generic_arg_count_for_call( tcx, span, def_id, &generics, seg, IsMethodCall::No, - ) { + ); + + if let ExplicitLateBound::Yes = arg_count.explicit_late_bound { + explicit_late_bound = ExplicitLateBound::Yes; + } + + if let Err(GenericArgCountMismatch { reported: Some(_), .. }) = arg_count.correct { infer_args_for_err.insert(index); self.set_tainted_by_errors(); // See issue #53251. } @@ -1357,7 +1384,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> { let ty = tcx.type_of(def_id); let arg_count = GenericArgCountResult { - explicit_late_bound: ExplicitLateBound::No, + explicit_late_bound, correct: if infer_args_for_err.is_empty() { Ok(()) } else { @@ -1495,7 +1522,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> { let ty = tcx.type_of(impl_def_id); let impl_ty = self.instantiate_type_scheme(span, &substs, ty); - match self.at(&self.misc(span), self.param_env).sup(impl_ty, self_ty) { + match self.at(&self.misc(span), self.param_env).eq(impl_ty, self_ty) { Ok(ok) => self.register_infer_ok_obligations(ok), Err(_) => { self.tcx.sess.delay_span_bug( @@ -1510,8 +1537,6 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> { } } - self.check_rustc_args_require_const(def_id, hir_id, span); - debug!("instantiate_value_path: type of {:?} is {:?}", hir_id, ty_substituted); self.write_substs(hir_id, substs); diff --git a/compiler/rustc_typeck/src/check/fn_ctxt/checks.rs b/compiler/rustc_typeck/src/check/fn_ctxt/checks.rs index 80b5a9d4e6..49aea19c8d 100644 --- a/compiler/rustc_typeck/src/check/fn_ctxt/checks.rs +++ b/compiler/rustc_typeck/src/check/fn_ctxt/checks.rs @@ -18,7 +18,7 @@ use rustc_middle::ty::adjustment::AllowTwoPhase; use rustc_middle::ty::fold::TypeFoldable; use rustc_middle::ty::{self, Ty}; use rustc_session::Session; -use rustc_span::symbol::{sym, Ident}; +use rustc_span::symbol::Ident; use rustc_span::{self, MultiSpan, Span}; use rustc_trait_selection::traits::{self, ObligationCauseCode, StatementAsExpression}; @@ -720,34 +720,6 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> { ty } - pub(in super::super) fn check_rustc_args_require_const( - &self, - def_id: DefId, - hir_id: hir::HirId, - span: Span, - ) { - // We're only interested in functions tagged with - // #[rustc_args_required_const], so ignore anything that's not. - if !self.tcx.has_attr(def_id, sym::rustc_args_required_const) { - return; - } - - // If our calling expression is indeed the function itself, we're good! - // If not, generate an error that this can only be called directly. - if let Node::Expr(expr) = self.tcx.hir().get(self.tcx.hir().get_parent_node(hir_id)) { - if let ExprKind::Call(ref callee, ..) = expr.kind { - if callee.hir_id == hir_id { - return; - } - } - } - - self.tcx.sess.span_err( - span, - "this function can only be invoked directly, not through a function pointer", - ); - } - /// A common error is to add an extra semicolon: /// /// ``` @@ -986,7 +958,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> { error.obligation.predicate.kind().skip_binder() { // If any of the type arguments in this path segment caused the - // `FullfillmentError`, point at its span (#61860). + // `FulfillmentError`, point at its span (#61860). for arg in path .segments .iter() diff --git a/compiler/rustc_typeck/src/check/generator_interior.rs b/compiler/rustc_typeck/src/check/generator_interior.rs index e40aa91485..5f26e701c0 100644 --- a/compiler/rustc_typeck/src/check/generator_interior.rs +++ b/compiler/rustc_typeck/src/check/generator_interior.rs @@ -89,19 +89,31 @@ impl<'a, 'tcx> InteriorVisitor<'a, 'tcx> { if let Some((unresolved_type, unresolved_type_span)) = self.fcx.unresolved_type_vars(&ty) { - let note = format!( - "the type is part of the {} because of this {}", - self.kind, yield_data.source - ); - // If unresolved type isn't a ty_var then unresolved_type_span is None let span = self .prev_unresolved_span .unwrap_or_else(|| unresolved_type_span.unwrap_or(source_span)); - self.fcx - .need_type_info_err_in_generator(self.kind, span, unresolved_type) - .span_note(yield_data.span, &*note) - .emit(); + + // If we encounter an int/float variable, then inference fallback didn't + // finish due to some other error. Don't emit spurious additional errors. + if let ty::Infer(ty::InferTy::IntVar(_) | ty::InferTy::FloatVar(_)) = + unresolved_type.kind() + { + self.fcx + .tcx + .sess + .delay_span_bug(span, &format!("Encountered var {:?}", unresolved_type)); + } else { + let note = format!( + "the type is part of the {} because of this {}", + self.kind, yield_data.source + ); + + self.fcx + .need_type_info_err_in_generator(self.kind, span, unresolved_type) + .span_note(yield_data.span, &*note) + .emit(); + } } else { // Insert the type into the ordered set. let scope_span = scope.map(|s| s.span(self.fcx.tcx, self.region_scope_tree)); diff --git a/compiler/rustc_typeck/src/check/inherited.rs b/compiler/rustc_typeck/src/check/inherited.rs index 1dacbade1b..2e9bef15f9 100644 --- a/compiler/rustc_typeck/src/check/inherited.rs +++ b/compiler/rustc_typeck/src/check/inherited.rs @@ -2,13 +2,14 @@ use super::callee::DeferredCallResolution; use super::MaybeInProgressTables; use rustc_data_structures::fx::FxHashMap; +use rustc_data_structures::vec_map::VecMap; use rustc_hir as hir; use rustc_hir::def_id::{DefIdMap, LocalDefId}; use rustc_hir::HirIdMap; use rustc_infer::infer; use rustc_infer::infer::{InferCtxt, InferOk, TyCtxtInferExt}; use rustc_middle::ty::fold::TypeFoldable; -use rustc_middle::ty::{self, Ty, TyCtxt}; +use rustc_middle::ty::{self, OpaqueTypeKey, Ty, TyCtxt}; use rustc_span::{self, Span}; use rustc_trait_selection::infer::InferCtxtExt as _; use rustc_trait_selection::opaque_types::OpaqueTypeDecl; @@ -58,7 +59,7 @@ pub struct Inherited<'a, 'tcx> { // associated fresh inference variable. Writeback resolves these // variables to get the concrete type, which can be used to // 'de-opaque' OpaqueTypeDecl, after typeck is done with all functions. - pub(super) opaque_types: RefCell>>, + pub(super) opaque_types: RefCell, OpaqueTypeDecl<'tcx>>>, /// A map from inference variables created from opaque /// type instantiations (`ty::Infer`) to the actual opaque diff --git a/compiler/rustc_typeck/src/check/method/mod.rs b/compiler/rustc_typeck/src/check/method/mod.rs index bd7ffd057b..427102afee 100644 --- a/compiler/rustc_typeck/src/check/method/mod.rs +++ b/compiler/rustc_typeck/src/check/method/mod.rs @@ -205,7 +205,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> { .insert(*import_id); } - self.tcx.check_stability(pick.item.def_id, Some(call_expr.hir_id), span); + self.tcx.check_stability(pick.item.def_id, Some(call_expr.hir_id), span, None); let result = self.confirm_method(span, self_expr, call_expr, self_ty, pick.clone(), segment); @@ -303,8 +303,8 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> { opt_input_types: Option<&[Ty<'tcx>]>, ) -> Option>> { debug!( - "lookup_in_trait_adjusted(self_ty={:?}, m_name={}, trait_def_id={:?})", - self_ty, m_name, trait_def_id + "lookup_in_trait_adjusted(self_ty={:?}, m_name={}, trait_def_id={:?}, opt_input_types={:?})", + self_ty, m_name, trait_def_id, opt_input_types ); // Construct a trait-reference `self_ty : Trait` @@ -445,7 +445,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> { // them as well. It's ok to use the variant's id as a ctor id since an // error will be reported on any use of such resolution anyway. let ctor_def_id = variant_def.ctor_def_id.unwrap_or(variant_def.def_id); - tcx.check_stability(ctor_def_id, Some(expr_id), span); + tcx.check_stability(ctor_def_id, Some(expr_id), span, Some(method_name.span)); return Ok(( DefKind::Ctor(CtorOf::Variant, variant_def.ctor_kind), ctor_def_id, @@ -475,7 +475,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> { let def_kind = pick.item.kind.as_def_kind(); debug!("resolve_ufcs: def_kind={:?}, def_id={:?}", def_kind, pick.item.def_id); - tcx.check_stability(pick.item.def_id, Some(expr_id), span); + tcx.check_stability(pick.item.def_id, Some(expr_id), span, Some(method_name.span)); Ok((def_kind, pick.item.def_id)) } diff --git a/compiler/rustc_typeck/src/check/method/probe.rs b/compiler/rustc_typeck/src/check/method/probe.rs index c79743f2d7..440e0f4e1a 100644 --- a/compiler/rustc_typeck/src/check/method/probe.rs +++ b/compiler/rustc_typeck/src/check/method/probe.rs @@ -1286,7 +1286,7 @@ impl<'a, 'tcx> ProbeContext<'a, 'tcx> { if let Some(uc) = unstable_candidates { applicable_candidates.retain(|&(p, _)| { if let stability::EvalResult::Deny { feature, .. } = - self.tcx.eval_stability(p.item.def_id, None, self.span) + self.tcx.eval_stability(p.item.def_id, None, self.span, None) { uc.push((p, feature)); return false; diff --git a/compiler/rustc_typeck/src/check/method/suggest.rs b/compiler/rustc_typeck/src/check/method/suggest.rs index b2e4e7a981..16382c7e7a 100644 --- a/compiler/rustc_typeck/src/check/method/suggest.rs +++ b/compiler/rustc_typeck/src/check/method/suggest.rs @@ -6,7 +6,7 @@ use rustc_data_structures::fx::{FxHashMap, FxHashSet}; use rustc_errors::{pluralize, struct_span_err, Applicability, DiagnosticBuilder}; use rustc_hir as hir; use rustc_hir::def::{DefKind, Namespace, Res}; -use rustc_hir::def_id::{DefId, CRATE_DEF_INDEX, LOCAL_CRATE}; +use rustc_hir::def_id::{DefId, CRATE_DEF_INDEX}; use rustc_hir::intravisit; use rustc_hir::lang_items::LangItem; use rustc_hir::{ExprKind, Node, QPath}; @@ -383,6 +383,42 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> { return None; } else { span = item_name.span; + + // Don't show generic arguments when the method can't be found in any implementation (#81576). + let mut ty_str_reported = ty_str.clone(); + if let ty::Adt(_, ref generics) = actual.kind() { + if generics.len() > 0 { + let mut autoderef = self.autoderef(span, actual); + let candidate_found = autoderef.any(|(ty, _)| { + if let ty::Adt(ref adt_deref, _) = ty.kind() { + self.tcx + .inherent_impls(adt_deref.did) + .iter() + .filter_map(|def_id| { + self.associated_item( + *def_id, + item_name, + Namespace::ValueNS, + ) + }) + .count() + >= 1 + } else { + false + } + }); + let has_deref = autoderef.step_count() > 0; + if !candidate_found + && !has_deref + && unsatisfied_predicates.is_empty() + { + if let Some((path_string, _)) = ty_str.split_once('<') { + ty_str_reported = path_string.to_string(); + } + } + } + } + let mut err = struct_span_err!( tcx.sess, span, @@ -391,7 +427,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> { item_kind, item_name, actual.prefix_string(self.tcx), - ty_str, + ty_str_reported, ); if let Mode::MethodCall = mode { if let SelfSource::MethodCall(call) = source { @@ -449,6 +485,63 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> { let mut label_span_not_found = || { if unsatisfied_predicates.is_empty() { err.span_label(span, format!("{item_kind} not found in `{ty_str}`")); + if let ty::Adt(ref adt, _) = rcvr_ty.kind() { + let mut inherent_impls_candidate = self + .tcx + .inherent_impls(adt.did) + .iter() + .copied() + .filter(|def_id| { + if let Some(assoc) = + self.associated_item(*def_id, item_name, Namespace::ValueNS) + { + // Check for both mode is the same so we avoid suggesting + // incorrect associated item. + match (mode, assoc.fn_has_self_parameter, source) { + (Mode::MethodCall, true, SelfSource::MethodCall(_)) => { + // We check that the suggest type is actually + // different from the received one + // So we avoid suggestion method with Box + // for instance + self.tcx.at(span).type_of(*def_id) != actual + && self.tcx.at(span).type_of(*def_id) != rcvr_ty + } + (Mode::Path, false, _) => true, + _ => false, + } + } else { + false + } + }) + .collect::>(); + if inherent_impls_candidate.len() > 0 { + inherent_impls_candidate.sort(); + inherent_impls_candidate.dedup(); + + // number of type to shows at most. + let limit = if inherent_impls_candidate.len() == 5 { 5 } else { 4 }; + let type_candidates = inherent_impls_candidate + .iter() + .take(limit) + .map(|impl_item| { + format!("- `{}`", self.tcx.at(span).type_of(*impl_item)) + }) + .collect::>() + .join("\n"); + let additional_types = if inherent_impls_candidate.len() > limit { + format!( + "\nand {} more types", + inherent_impls_candidate.len() - limit + ) + } else { + "".to_string() + }; + err.note(&format!( + "the {item_kind} was found for\n{}{}", + type_candidates, additional_types + )); + } + } } else { err.span_label(span, format!("{item_kind} cannot be called on `{ty_str}` due to unsatisfied trait bounds")); } @@ -1440,11 +1533,11 @@ impl Ord for TraitInfo { /// Retrieves all traits in this crate and any dependent crates. pub fn all_traits(tcx: TyCtxt<'_>) -> Vec { - tcx.all_traits(LOCAL_CRATE).iter().map(|&def_id| TraitInfo { def_id }).collect() + tcx.all_traits(()).iter().map(|&def_id| TraitInfo { def_id }).collect() } /// Computes all traits in this crate and any dependent crates. -fn compute_all_traits(tcx: TyCtxt<'_>) -> Vec { +fn compute_all_traits(tcx: TyCtxt<'_>, (): ()) -> &[DefId] { use hir::itemlikevisit; let mut traits = vec![]; @@ -1503,14 +1596,11 @@ fn compute_all_traits(tcx: TyCtxt<'_>) -> Vec { handle_external_res(tcx, &mut traits, &mut external_mods, Res::Def(DefKind::Mod, def_id)); } - traits + tcx.arena.alloc_from_iter(traits) } pub fn provide(providers: &mut ty::query::Providers) { - providers.all_traits = |tcx, cnum| { - assert_eq!(cnum, LOCAL_CRATE); - &tcx.arena.alloc(compute_all_traits(tcx))[..] - } + providers.all_traits = compute_all_traits; } struct UsePlacementFinder<'tcx> { diff --git a/compiler/rustc_typeck/src/check/mod.rs b/compiler/rustc_typeck/src/check/mod.rs index cb7589318d..ad7853b7cd 100644 --- a/compiler/rustc_typeck/src/check/mod.rs +++ b/compiler/rustc_typeck/src/check/mod.rs @@ -105,7 +105,7 @@ use rustc_data_structures::fx::{FxHashMap, FxHashSet}; use rustc_errors::{pluralize, struct_span_err, Applicability}; use rustc_hir as hir; use rustc_hir::def::Res; -use rustc_hir::def_id::{CrateNum, DefId, LocalDefId, LOCAL_CRATE}; +use rustc_hir::def_id::{DefId, LocalDefId}; use rustc_hir::intravisit::Visitor; use rustc_hir::itemlikevisit::ItemLikeVisitor; use rustc_hir::{HirIdMap, ImplicitSelfKind, Node}; @@ -545,18 +545,17 @@ fn typeck_with_fallback<'tcx>( kind: TypeVariableOriginKind::TypeInference, span, }), - Node::Expr(&hir::Expr { kind: hir::ExprKind::InlineAsm(ia), .. }) - if ia.operands.iter().any(|(op, _op_sp)| match op { + Node::Expr(&hir::Expr { kind: hir::ExprKind::InlineAsm(asm), .. }) + | Node::Item(&hir::Item { kind: hir::ItemKind::GlobalAsm(asm), .. }) + if asm.operands.iter().any(|(op, _op_sp)| match op { hir::InlineAsmOperand::Const { anon_const } => { anon_const.hir_id == id } _ => false, }) => { - fcx.next_ty_var(TypeVariableOrigin { - kind: TypeVariableOriginKind::MiscVariable, - span, - }) + // Inline assembly constants must be integers. + fcx.next_int_var() } _ => fallback(), }, @@ -1161,8 +1160,7 @@ impl ItemLikeVisitor<'tcx> for CheckItemTypesVisitor<'tcx> { fn visit_foreign_item(&mut self, _: &'tcx hir::ForeignItem<'tcx>) {} } -fn typeck_item_bodies(tcx: TyCtxt<'_>, crate_num: CrateNum) { - debug_assert!(crate_num == LOCAL_CRATE); +fn typeck_item_bodies(tcx: TyCtxt<'_>, (): ()) { tcx.par_body_owners(|body_owner_def_id| { tcx.ensure().typeck(body_owner_def_id); }); @@ -1189,3 +1187,14 @@ fn fatally_break_rust(sess: &Session) { fn potentially_plural_count(count: usize, word: &str) -> String { format!("{} {}{}", count, word, pluralize!(count)) } + +fn has_expected_num_generic_args<'tcx>( + tcx: TyCtxt<'tcx>, + trait_did: Option, + expected: usize, +) -> bool { + trait_did.map_or(true, |trait_did| { + let generics = tcx.generics_of(trait_did); + generics.count() == expected + if generics.has_self { 1 } else { 0 } + }) +} diff --git a/compiler/rustc_typeck/src/check/op.rs b/compiler/rustc_typeck/src/check/op.rs index 567cb1a90d..963436d05d 100644 --- a/compiler/rustc_typeck/src/check/op.rs +++ b/compiler/rustc_typeck/src/check/op.rs @@ -1,7 +1,7 @@ //! Code related to processing overloaded binary and unary operators. use super::method::MethodCallee; -use super::FnCtxt; +use super::{has_expected_num_generic_args, FnCtxt}; use rustc_ast as ast; use rustc_errors::{self, struct_span_err, Applicability, DiagnosticBuilder}; use rustc_hir as hir; @@ -795,6 +795,23 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> { lhs_ty, op, opname, trait_did ); + // Catches cases like #83893, where a lang item is declared with the + // wrong number of generic arguments. Should have yielded an error + // elsewhere by now, but we have to catch it here so that we do not + // index `other_tys` out of bounds (if the lang item has too many + // generic arguments, `other_tys` is too short). + if !has_expected_num_generic_args( + self.tcx, + trait_did, + match op { + // Binary ops have a generic right-hand side, unary ops don't + Op::Binary(..) => 1, + Op::Unary(..) => 0, + }, + ) { + return Err(()); + } + let method = trait_did.and_then(|trait_did| { let opname = Ident::with_dummy_span(opname); self.lookup_method_in_trait(span, opname, trait_did, lhs_ty, Some(other_tys)) diff --git a/compiler/rustc_typeck/src/check/pat.rs b/compiler/rustc_typeck/src/check/pat.rs index 53593b9bab..3ac760e236 100644 --- a/compiler/rustc_typeck/src/check/pat.rs +++ b/compiler/rustc_typeck/src/check/pat.rs @@ -958,7 +958,12 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> { let field_ty = self.field_ty(subpat.span, &variant.fields[i], substs); self.check_pat(&subpat, field_ty, def_bm, TopInfo { parent_pat: Some(&pat), ..ti }); - self.tcx.check_stability(variant.fields[i].did, Some(pat.hir_id), subpat.span); + self.tcx.check_stability( + variant.fields[i].did, + Some(pat.hir_id), + subpat.span, + None, + ); } } else { // Pattern has wrong number of fields. @@ -1192,7 +1197,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> { .get(&ident) .map(|(i, f)| { self.write_field_index(field.hir_id, *i); - self.tcx.check_stability(f.did, Some(pat.hir_id), span); + self.tcx.check_stability(f.did, Some(pat.hir_id), span, None); self.field_ty(span, f, substs) }) .unwrap_or_else(|| { diff --git a/compiler/rustc_typeck/src/check/place_op.rs b/compiler/rustc_typeck/src/check/place_op.rs index 5bd385107c..652b82f106 100644 --- a/compiler/rustc_typeck/src/check/place_op.rs +++ b/compiler/rustc_typeck/src/check/place_op.rs @@ -1,5 +1,5 @@ use crate::check::method::MethodCallee; -use crate::check::{FnCtxt, PlaceOp}; +use crate::check::{has_expected_num_generic_args, FnCtxt, PlaceOp}; use rustc_hir as hir; use rustc_infer::infer::type_variable::{TypeVariableOrigin, TypeVariableOriginKind}; use rustc_infer::infer::InferOk; @@ -153,6 +153,21 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> { PlaceOp::Deref => (self.tcx.lang_items().deref_trait(), sym::deref), PlaceOp::Index => (self.tcx.lang_items().index_trait(), sym::index), }; + + // If the lang item was declared incorrectly, stop here so that we don't + // run into an ICE (#83893). The error is reported where the lang item is + // declared. + if !has_expected_num_generic_args( + self.tcx, + imm_tr, + match op { + PlaceOp::Deref => 0, + PlaceOp::Index => 1, + }, + ) { + return None; + } + imm_tr.and_then(|trait_did| { self.lookup_method_in_trait( span, @@ -177,6 +192,21 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> { PlaceOp::Deref => (self.tcx.lang_items().deref_mut_trait(), sym::deref_mut), PlaceOp::Index => (self.tcx.lang_items().index_mut_trait(), sym::index_mut), }; + + // If the lang item was declared incorrectly, stop here so that we don't + // run into an ICE (#83893). The error is reported where the lang item is + // declared. + if !has_expected_num_generic_args( + self.tcx, + mut_tr, + match op { + PlaceOp::Deref => 0, + PlaceOp::Index => 1, + }, + ) { + return None; + } + mut_tr.and_then(|trait_did| { self.lookup_method_in_trait( span, @@ -218,7 +248,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> { // Clear previous flag; after a pointer indirection it does not apply any more. inside_union = false; } - if source.ty_adt_def().map_or(false, |adt| adt.is_union()) { + if source.is_union() { inside_union = true; } // Fix up the autoderefs. Autorefs can only occur immediately preceding diff --git a/compiler/rustc_typeck/src/check/upvar.rs b/compiler/rustc_typeck/src/check/upvar.rs index 751eebb9f9..6baa185406 100644 --- a/compiler/rustc_typeck/src/check/upvar.rs +++ b/compiler/rustc_typeck/src/check/upvar.rs @@ -42,11 +42,13 @@ use rustc_hir::intravisit::{self, NestedVisitorMap, Visitor}; use rustc_infer::infer::UpvarRegion; use rustc_middle::hir::place::{Place, PlaceBase, PlaceWithHirId, Projection, ProjectionKind}; use rustc_middle::mir::FakeReadCause; -use rustc_middle::ty::{self, Ty, TyCtxt, TypeckResults, UpvarSubsts}; +use rustc_middle::ty::{self, TraitRef, Ty, TyCtxt, TypeckResults, UpvarSubsts}; use rustc_session::lint; use rustc_span::sym; use rustc_span::{MultiSpan, Span, Symbol}; +use rustc_trait_selection::traits::{Obligation, ObligationCause}; +use rustc_data_structures::stable_set::FxHashSet; use rustc_index::vec::Idx; use rustc_target::abi::VariantIdx; @@ -168,7 +170,8 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> { self.compute_min_captures(closure_def_id, delegate.capture_information); let closure_hir_id = self.tcx.hir().local_def_id_to_hir_id(local_def_id); - if should_do_migration_analysis(self.tcx, closure_hir_id) { + + if should_do_disjoint_capture_migration_analysis(self.tcx, closure_hir_id) { self.perform_2229_migration_anaysis(closure_def_id, body_id, capture_clause, span); } @@ -320,7 +323,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> { /// /// InferBorrowKind results in a structure like this: /// - /// ``` + /// ```text /// { /// Place(base: hir_id_s, projections: [], ....) -> { /// capture_kind_expr: hir_id_L5, @@ -345,7 +348,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> { /// ``` /// /// After the min capture analysis, we get: - /// ``` + /// ```text /// { /// hir_id_s -> [ /// Place(base: hir_id_s, projections: [], ....) -> { @@ -471,7 +474,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> { capture_clause: hir::CaptureBy, span: Span, ) { - let need_migrations = self.compute_2229_migrations( + let (need_migrations, reasons) = self.compute_2229_migrations( closure_def_id, span, capture_clause, @@ -485,12 +488,16 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> { let local_def_id = closure_def_id.expect_local(); let closure_hir_id = self.tcx.hir().local_def_id_to_hir_id(local_def_id); self.tcx.struct_span_lint_hir( - lint::builtin::DISJOINT_CAPTURE_DROP_REORDER, + lint::builtin::DISJOINT_CAPTURE_MIGRATION, closure_hir_id, span, |lint| { let mut diagnostics_builder = lint.build( - "drop order affected for closure because of `capture_disjoint_fields`", + format!( + "{} affected for closure because of `capture_disjoint_fields`", + reasons + ) + .as_str(), ); let closure_body_span = self.tcx.hir().span(body_id.hir_id); let (sugg, app) = @@ -527,6 +534,169 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> { } } + /// Combines all the reasons for 2229 migrations + fn compute_2229_migrations_reasons( + &self, + auto_trait_reasons: FxHashSet<&str>, + drop_reason: bool, + ) -> String { + let mut reasons = String::new(); + + if auto_trait_reasons.len() > 0 { + reasons = format!( + "{} trait implementation", + auto_trait_reasons.clone().into_iter().collect::>().join(", ") + ); + } + + if auto_trait_reasons.len() > 0 && drop_reason { + reasons = format!("{}, and ", reasons); + } + + if drop_reason { + reasons = format!("{}drop order", reasons); + } + + reasons + } + + /// Returns true if `ty` may implement `trait_def_id` + fn ty_impls_trait( + &self, + ty: Ty<'tcx>, + cause: &ObligationCause<'tcx>, + trait_def_id: DefId, + ) -> bool { + use crate::rustc_middle::ty::ToPredicate; + use crate::rustc_middle::ty::WithConstness; + use rustc_trait_selection::traits::query::evaluate_obligation::InferCtxtExt; + let tcx = self.infcx.tcx; + + let trait_ref = TraitRef { def_id: trait_def_id, substs: tcx.mk_substs_trait(ty, &[]) }; + + let obligation = Obligation::new( + cause.clone(), + self.param_env, + trait_ref.without_const().to_predicate(tcx), + ); + + self.infcx.predicate_may_hold(&obligation) + } + + /// Returns true if migration is needed for trait for the provided var_hir_id + fn need_2229_migrations_for_trait( + &self, + min_captures: Option<&ty::RootVariableMinCaptureList<'tcx>>, + var_hir_id: hir::HirId, + check_trait: Option, + ) -> bool { + let root_var_min_capture_list = if let Some(root_var_min_capture_list) = + min_captures.and_then(|m| m.get(&var_hir_id)) + { + root_var_min_capture_list + } else { + return false; + }; + + let ty = self.infcx.resolve_vars_if_possible(self.node_ty(var_hir_id)); + + let cause = ObligationCause::misc(self.tcx.hir().span(var_hir_id), self.body_id); + + let obligation_should_hold = check_trait + .map(|check_trait| self.ty_impls_trait(ty, &cause, check_trait)) + .unwrap_or(false); + + // Check whether catpured fields also implement the trait + + for capture in root_var_min_capture_list.iter() { + let ty = capture.place.ty(); + + let obligation_holds_for_capture = check_trait + .map(|check_trait| self.ty_impls_trait(ty, &cause, check_trait)) + .unwrap_or(false); + + if !obligation_holds_for_capture && obligation_should_hold { + return true; + } + } + false + } + + /// Figures out the list of root variables (and their types) that aren't completely + /// captured by the closure when `capture_disjoint_fields` is enabled and auto-traits + /// differ between the root variable and the captured paths. + /// + /// The output list would include a root variable if: + /// - It would have been captured into the closure when `capture_disjoint_fields` wasn't + /// enabled, **and** + /// - It wasn't completely captured by the closure, **and** + /// - One of the paths captured does not implement all the auto-traits its root variable + /// implements. + fn compute_2229_migrations_for_trait( + &self, + min_captures: Option<&ty::RootVariableMinCaptureList<'tcx>>, + var_hir_id: hir::HirId, + ) -> Option> { + let tcx = self.infcx.tcx; + + // Check whether catpured fields also implement the trait + let mut auto_trait_reasons = FxHashSet::default(); + + if self.need_2229_migrations_for_trait( + min_captures, + var_hir_id, + tcx.lang_items().clone_trait(), + ) { + auto_trait_reasons.insert("`Clone`"); + } + + if self.need_2229_migrations_for_trait( + min_captures, + var_hir_id, + tcx.lang_items().sync_trait(), + ) { + auto_trait_reasons.insert("`Sync`"); + } + + if self.need_2229_migrations_for_trait( + min_captures, + var_hir_id, + tcx.lang_items().send_trait(), + ) { + auto_trait_reasons.insert("`Send`"); + } + + if self.need_2229_migrations_for_trait( + min_captures, + var_hir_id, + tcx.lang_items().unpin_trait(), + ) { + auto_trait_reasons.insert("`Unpin`"); + } + + if self.need_2229_migrations_for_trait( + min_captures, + var_hir_id, + tcx.lang_items().unwind_safe_trait(), + ) { + auto_trait_reasons.insert("`UnwindSafe`"); + } + + if self.need_2229_migrations_for_trait( + min_captures, + var_hir_id, + tcx.lang_items().ref_unwind_safe_trait(), + ) { + auto_trait_reasons.insert("`RefUnwindSafe`"); + } + + if auto_trait_reasons.len() > 0 { + return Some(auto_trait_reasons); + } + + return None; + } + /// Figures out the list of root variables (and their types) that aren't completely /// captured by the closure when `capture_disjoint_fields` is enabled and drop order of /// some path starting at that root variable **might** be affected. @@ -536,76 +706,131 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> { /// enabled, **and** /// - It wasn't completely captured by the closure, **and** /// - One of the paths starting at this root variable, that is not captured needs Drop. - fn compute_2229_migrations( + /// + /// This function only returns true for significant drops. A type is considerent to have a + /// significant drop if it's Drop implementation is not annotated by `rustc_insignificant_dtor`. + fn compute_2229_migrations_for_drop( &self, closure_def_id: DefId, closure_span: Span, - closure_clause: hir::CaptureBy, min_captures: Option<&ty::RootVariableMinCaptureList<'tcx>>, - ) -> Vec { - let upvars = if let Some(upvars) = self.tcx.upvars_mentioned(closure_def_id) { - upvars - } else { - return vec![]; - }; + closure_clause: hir::CaptureBy, + var_hir_id: hir::HirId, + ) -> bool { + let ty = self.infcx.resolve_vars_if_possible(self.node_ty(var_hir_id)); - let mut need_migrations = Vec::new(); + if !ty.has_significant_drop(self.tcx, self.tcx.param_env(closure_def_id.expect_local())) { + return false; + } - for (&var_hir_id, _) in upvars.iter() { - let ty = self.infcx.resolve_vars_if_possible(self.node_ty(var_hir_id)); + let root_var_min_capture_list = if let Some(root_var_min_capture_list) = + min_captures.and_then(|m| m.get(&var_hir_id)) + { + root_var_min_capture_list + } else { + // The upvar is mentioned within the closure but no path starting from it is + // used. - if !ty.needs_drop(self.tcx, self.tcx.param_env(closure_def_id.expect_local())) { - continue; + match closure_clause { + // Only migrate if closure is a move closure + hir::CaptureBy::Value => return true, + hir::CaptureBy::Ref => {} } - let root_var_min_capture_list = if let Some(root_var_min_capture_list) = - min_captures.and_then(|m| m.get(&var_hir_id)) - { - root_var_min_capture_list - } else { - // The upvar is mentioned within the closure but no path starting from it is - // used. + return false; + }; - match closure_clause { - // Only migrate if closure is a move closure - hir::CaptureBy::Value => need_migrations.push(var_hir_id), + let projections_list = root_var_min_capture_list + .iter() + .filter_map(|captured_place| match captured_place.info.capture_kind { + // Only care about captures that are moved into the closure + ty::UpvarCapture::ByValue(..) => Some(captured_place.place.projections.as_slice()), + ty::UpvarCapture::ByRef(..) => None, + }) + .collect::>(); - hir::CaptureBy::Ref => {} - } + let is_moved = !projections_list.is_empty(); - continue; - }; + let is_not_completely_captured = + root_var_min_capture_list.iter().any(|capture| capture.place.projections.len() > 0); - let projections_list = root_var_min_capture_list - .iter() - .filter_map(|captured_place| match captured_place.info.capture_kind { - // Only care about captures that are moved into the closure - ty::UpvarCapture::ByValue(..) => { - Some(captured_place.place.projections.as_slice()) - } - ty::UpvarCapture::ByRef(..) => None, - }) - .collect::>(); + if is_moved + && is_not_completely_captured + && self.has_significant_drop_outside_of_captures( + closure_def_id, + closure_span, + ty, + projections_list, + ) + { + return true; + } - let is_moved = !projections_list.is_empty(); + return false; + } - let is_not_completely_captured = - root_var_min_capture_list.iter().any(|capture| capture.place.projections.len() > 0); + /// Figures out the list of root variables (and their types) that aren't completely + /// captured by the closure when `capture_disjoint_fields` is enabled and either drop + /// order of some path starting at that root variable **might** be affected or auto-traits + /// differ between the root variable and the captured paths. + /// + /// The output list would include a root variable if: + /// - It would have been moved into the closure when `capture_disjoint_fields` wasn't + /// enabled, **and** + /// - It wasn't completely captured by the closure, **and** + /// - One of the paths starting at this root variable, that is not captured needs Drop **or** + /// - One of the paths captured does not implement all the auto-traits its root variable + /// implements. + /// + /// Returns a tuple containing a vector of HirIds as well as a String containing the reason + /// why root variables whose HirId is contained in the vector should be fully captured. + fn compute_2229_migrations( + &self, + closure_def_id: DefId, + closure_span: Span, + closure_clause: hir::CaptureBy, + min_captures: Option<&ty::RootVariableMinCaptureList<'tcx>>, + ) -> (Vec, String) { + let upvars = if let Some(upvars) = self.tcx.upvars_mentioned(closure_def_id) { + upvars + } else { + return (Vec::new(), format!("")); + }; - if is_moved - && is_not_completely_captured - && self.has_significant_drop_outside_of_captures( - closure_def_id, - closure_span, - ty, - projections_list, - ) + let mut need_migrations = Vec::new(); + let mut auto_trait_reasons = FxHashSet::default(); + let mut drop_reorder_reason = false; + + // Perform auto-trait analysis + for (&var_hir_id, _) in upvars.iter() { + let mut need_migration = false; + if let Some(trait_migration_cause) = + self.compute_2229_migrations_for_trait(min_captures, var_hir_id) { + need_migration = true; + auto_trait_reasons.extend(trait_migration_cause); + } + + if self.compute_2229_migrations_for_drop( + closure_def_id, + closure_span, + min_captures, + closure_clause, + var_hir_id, + ) { + need_migration = true; + drop_reorder_reason = true; + } + + if need_migration { need_migrations.push(var_hir_id); } } - need_migrations + ( + need_migrations, + self.compute_2229_migrations_reasons(auto_trait_reasons, drop_reorder_reason), + ) } /// This is a helper function to `compute_2229_migrations_precise_pass`. Provided the type @@ -613,11 +838,6 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> { /// using list of `Projection` slices), it returns true if there is a path that is not /// captured starting at this root variable that implements Drop. /// - /// FIXME(project-rfc-2229#35): This should return true only for significant drops. - /// A drop is significant if it's implemented by the user or does - /// anything that will have any observable behavior (other than - /// freeing up memory). - /// /// The way this function works is at a given call it looks at type `base_path_ty` of some base /// path say P and then list of projection slices which represent the different captures moved /// into the closure starting off of P. @@ -673,7 +893,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> { /// (Ty((w.p).x), [ &[] ]) (Ty((w.p).y), []) // IMP 2 /// | | /// v v - /// false NeedsDrop(Ty(w.p.y)) + /// false NeedsSignificantDrop(Ty(w.p.y)) /// | /// v /// true @@ -717,7 +937,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> { captured_by_move_projs: Vec<&[Projection<'tcx>]>, ) -> bool { let needs_drop = |ty: Ty<'tcx>| { - ty.needs_drop(self.tcx, self.tcx.param_env(closure_def_id.expect_local())) + ty.has_significant_drop(self.tcx, self.tcx.param_env(closure_def_id.expect_local())) }; let is_drop_defined_for_ty = |ty: Ty<'tcx>| { @@ -1368,6 +1588,11 @@ impl<'a, 'tcx> InferBorrowKind<'a, 'tcx> { impl<'a, 'tcx> euv::Delegate<'tcx> for InferBorrowKind<'a, 'tcx> { fn fake_read(&mut self, place: Place<'tcx>, cause: FakeReadCause, diag_expr_id: hir::HirId) { if let PlaceBase::Upvar(_) = place.base { + // We need to restrict Fake Read precision to avoid fake reading unsafe code, + // such as deref of a raw pointer. + let place = restrict_capture_precision(place); + let place = + restrict_repr_packed_field_ref_capture(self.fcx.tcx, self.fcx.param_env, &place); self.fake_reads.push((place, cause, diag_expr_id)); } } @@ -1544,9 +1769,8 @@ fn var_name(tcx: TyCtxt<'_>, var_hir_id: hir::HirId) -> Symbol { tcx.hir().name(var_hir_id) } -fn should_do_migration_analysis(tcx: TyCtxt<'_>, closure_id: hir::HirId) -> bool { - let (level, _) = - tcx.lint_level_at_node(lint::builtin::DISJOINT_CAPTURE_DROP_REORDER, closure_id); +fn should_do_disjoint_capture_migration_analysis(tcx: TyCtxt<'_>, closure_id: hir::HirId) -> bool { + let (level, _) = tcx.lint_level_at_node(lint::builtin::DISJOINT_CAPTURE_MIGRATION, closure_id); !matches!(level, lint::Level::Allow) } diff --git a/compiler/rustc_typeck/src/check/wfcheck.rs b/compiler/rustc_typeck/src/check/wfcheck.rs index d25dd9a6e8..7436edccf8 100644 --- a/compiler/rustc_typeck/src/check/wfcheck.rs +++ b/compiler/rustc_typeck/src/check/wfcheck.rs @@ -43,7 +43,7 @@ struct CheckWfFcxBuilder<'tcx> { impl<'tcx> CheckWfFcxBuilder<'tcx> { fn with_fcx(&mut self, f: F) where - F: for<'b> FnOnce(&FnCtxt<'b, 'tcx>, TyCtxt<'tcx>) -> Vec>, + F: for<'b> FnOnce(&FnCtxt<'b, 'tcx>) -> Vec>, { let id = self.id; let span = self.span; @@ -56,7 +56,7 @@ impl<'tcx> CheckWfFcxBuilder<'tcx> { // empty `param_env`. check_false_global_bounds(&fcx, span, id); } - let wf_tys = f(&fcx, fcx.tcx); + let wf_tys = f(&fcx); fcx.select_all_obligations_or_error(); fcx.regionck_item(id, span, &wf_tys); }); @@ -388,7 +388,7 @@ fn check_associated_item( debug!("check_associated_item: {:?}", item_id); let code = ObligationCauseCode::MiscObligation; - for_id(tcx, item_id, span).with_fcx(|fcx, tcx| { + for_id(tcx, item_id, span).with_fcx(|fcx| { let item = fcx.tcx.associated_item(fcx.tcx.hir().local_def_id(item_id)); let (mut implied_bounds, self_ty) = match item.container { @@ -409,7 +409,6 @@ fn check_associated_item( let sig = fcx.normalize_associated_types_in(span, sig); let hir_sig = sig_if_method.expect("bad signature for method"); check_fn_or_method( - tcx, fcx, item.ident.span, sig, @@ -467,9 +466,9 @@ fn check_type_defn<'tcx, F>( ) where F: for<'fcx> FnMut(&FnCtxt<'fcx, 'tcx>) -> Vec>, { - for_item(tcx, item).with_fcx(|fcx, fcx_tcx| { + for_item(tcx, item).with_fcx(|fcx| { let variants = lookup_fields(fcx); - let packed = fcx.tcx.adt_def(item.def_id).repr.packed(); + let packed = tcx.adt_def(item.def_id).repr.packed(); for variant in &variants { // For DST, or when drop needs to copy things around, all @@ -477,15 +476,14 @@ fn check_type_defn<'tcx, F>( let needs_drop_copy = || { packed && { let ty = variant.fields.last().unwrap().ty; - let ty = fcx.tcx.erase_regions(ty); + let ty = tcx.erase_regions(ty); if ty.needs_infer() { - fcx_tcx - .sess + tcx.sess .delay_span_bug(item.span, &format!("inference variables in {:?}", ty)); // Just treat unresolved type expression as if it needs drop. true } else { - ty.needs_drop(fcx_tcx, fcx_tcx.param_env(item.def_id)) + ty.needs_drop(tcx, tcx.param_env(item.def_id)) } } }; @@ -497,7 +495,7 @@ fn check_type_defn<'tcx, F>( let last = idx == variant.fields.len() - 1; fcx.register_bound( field.ty, - fcx.tcx.require_lang_item(LangItem::Sized, None), + tcx.require_lang_item(LangItem::Sized, None), traits::ObligationCause::new( field.span, fcx.body_id, @@ -524,11 +522,10 @@ fn check_type_defn<'tcx, F>( // Explicit `enum` discriminant values must const-evaluate successfully. if let Some(discr_def_id) = variant.explicit_discr { - let discr_substs = - InternalSubsts::identity_for_item(fcx.tcx, discr_def_id.to_def_id()); + let discr_substs = InternalSubsts::identity_for_item(tcx, discr_def_id.to_def_id()); let cause = traits::ObligationCause::new( - fcx.tcx.def_span(discr_def_id), + tcx.def_span(discr_def_id), fcx.body_id, traits::MiscObligation, ); @@ -539,12 +536,12 @@ fn check_type_defn<'tcx, F>( ty::WithOptConstParam::unknown(discr_def_id.to_def_id()), discr_substs, ) - .to_predicate(fcx.tcx), + .to_predicate(tcx), )); } } - check_where_clauses(tcx, fcx, item.span, item.def_id.to_def_id(), None); + check_where_clauses(fcx, item.span, item.def_id.to_def_id(), None); // No implied bounds in a struct definition. vec![] @@ -569,8 +566,9 @@ fn check_trait(tcx: TyCtxt<'_>, item: &hir::Item<'_>) { } } - for_item(tcx, item).with_fcx(|fcx, _| { - check_where_clauses(tcx, fcx, item.span, item.def_id.to_def_id(), None); + // FIXME: this shouldn't use an `FnCtxt` at all. + for_item(tcx, item).with_fcx(|fcx| { + check_where_clauses(fcx, item.span, item.def_id.to_def_id(), None); vec![] }); @@ -610,20 +608,12 @@ fn check_item_fn( span: Span, decl: &hir::FnDecl<'_>, ) { - for_id(tcx, item_id, span).with_fcx(|fcx, tcx| { - let def_id = fcx.tcx.hir().local_def_id(item_id); - let sig = fcx.tcx.fn_sig(def_id); + for_id(tcx, item_id, span).with_fcx(|fcx| { + let def_id = tcx.hir().local_def_id(item_id); + let sig = tcx.fn_sig(def_id); let sig = fcx.normalize_associated_types_in(span, sig); let mut implied_bounds = vec![]; - check_fn_or_method( - tcx, - fcx, - ident.span, - sig, - decl, - def_id.to_def_id(), - &mut implied_bounds, - ); + check_fn_or_method(fcx, ident.span, sig, decl, def_id.to_def_id(), &mut implied_bounds); implied_bounds }) } @@ -631,7 +621,7 @@ fn check_item_fn( fn check_item_type(tcx: TyCtxt<'_>, item_id: hir::HirId, ty_span: Span, allow_foreign_ty: bool) { debug!("check_item_type: {:?}", item_id); - for_id(tcx, item_id, ty_span).with_fcx(|fcx, tcx| { + for_id(tcx, item_id, ty_span).with_fcx(|fcx| { let ty = tcx.type_of(tcx.hir().local_def_id(item_id)); let item_ty = fcx.normalize_associated_types_in(ty_span, ty); @@ -647,7 +637,7 @@ fn check_item_type(tcx: TyCtxt<'_>, item_id: hir::HirId, ty_span: Span, allow_fo if forbid_unsized { fcx.register_bound( item_ty, - fcx.tcx.require_lang_item(LangItem::Sized, None), + tcx.require_lang_item(LangItem::Sized, None), traits::ObligationCause::new(ty_span, fcx.body_id, traits::MiscObligation), ); } @@ -665,13 +655,13 @@ fn check_impl<'tcx>( ) { debug!("check_impl: {:?}", item); - for_item(tcx, item).with_fcx(|fcx, tcx| { + for_item(tcx, item).with_fcx(|fcx| { match *ast_trait_ref { Some(ref ast_trait_ref) => { // `#[rustc_reservation_impl]` impls are not real impls and // therefore don't need to be WF (the trait's `Self: Trait` predicate // won't hold). - let trait_ref = fcx.tcx.impl_trait_ref(item.def_id).unwrap(); + let trait_ref = tcx.impl_trait_ref(item.def_id).unwrap(); let trait_ref = fcx.normalize_associated_types_in(ast_trait_ref.path.span, trait_ref); let obligations = traits::wf::trait_obligations( @@ -687,7 +677,7 @@ fn check_impl<'tcx>( } } None => { - let self_ty = fcx.tcx.type_of(item.def_id); + let self_ty = tcx.type_of(item.def_id); let self_ty = fcx.normalize_associated_types_in(item.span, self_ty); fcx.register_wf_obligation( self_ty.into(), @@ -697,7 +687,7 @@ fn check_impl<'tcx>( } } - check_where_clauses(tcx, fcx, item.span, item.def_id.to_def_id(), None); + check_where_clauses(fcx, item.span, item.def_id.to_def_id(), None); fcx.impl_implied_bounds(item.def_id.to_def_id(), item.span) }); @@ -705,15 +695,15 @@ fn check_impl<'tcx>( /// Checks where-clauses and inline bounds that are declared on `def_id`. fn check_where_clauses<'tcx, 'fcx>( - tcx: TyCtxt<'tcx>, fcx: &FnCtxt<'fcx, 'tcx>, span: Span, def_id: DefId, return_ty: Option<(Ty<'tcx>, Span)>, ) { debug!("check_where_clauses(def_id={:?}, return_ty={:?})", def_id, return_ty); + let tcx = fcx.tcx; - let predicates = fcx.tcx.predicates_of(def_id); + let predicates = tcx.predicates_of(def_id); let generics = tcx.generics_of(def_id); let is_our_default = |def: &ty::GenericParamDef| match def.kind { @@ -734,14 +724,14 @@ fn check_where_clauses<'tcx, 'fcx>( match param.kind { GenericParamDefKind::Type { .. } => { if is_our_default(¶m) { - let ty = fcx.tcx.type_of(param.def_id); + let ty = tcx.type_of(param.def_id); // Ignore dependent defaults -- that is, where the default of one type // parameter includes another (e.g., ``). In those cases, we can't // be sure if it will error or not as user might always specify the other. if !ty.needs_subst() { fcx.register_wf_obligation( ty.into(), - fcx.tcx.def_span(param.def_id), + tcx.def_span(param.def_id), ObligationCauseCode::MiscObligation, ); } @@ -754,7 +744,7 @@ fn check_where_clauses<'tcx, 'fcx>( let default_ct = tcx.const_param_default(param.def_id); fcx.register_wf_obligation( default_ct.into(), - fcx.tcx.def_span(param.def_id), + tcx.def_span(param.def_id), ObligationCauseCode::MiscObligation, ); } @@ -772,17 +762,17 @@ fn check_where_clauses<'tcx, 'fcx>( // For more examples see tests `defaults-well-formedness.rs` and `type-check-defaults.rs`. // // First we build the defaulted substitution. - let substs = InternalSubsts::for_item(fcx.tcx, def_id, |param, _| { + let substs = InternalSubsts::for_item(tcx, def_id, |param, _| { match param.kind { GenericParamDefKind::Lifetime => { // All regions are identity. - fcx.tcx.mk_param_from_def(param) + tcx.mk_param_from_def(param) } GenericParamDefKind::Type { .. } => { // If the param has a default, ... if is_our_default(param) { - let default_ty = fcx.tcx.type_of(param.def_id); + let default_ty = tcx.type_of(param.def_id); // ... and it's not a dependent default, ... if !default_ty.needs_subst() { // ... then substitute it with the default. @@ -790,7 +780,7 @@ fn check_where_clauses<'tcx, 'fcx>( } } - fcx.tcx.mk_param_from_def(param) + tcx.mk_param_from_def(param) } GenericParamDefKind::Const { .. } => { // FIXME(const_generics_defaults): I(@lcnr) feel like always @@ -811,7 +801,7 @@ fn check_where_clauses<'tcx, 'fcx>( } } - fcx.tcx.mk_param_from_def(param) + tcx.mk_param_from_def(param) } } }); @@ -848,7 +838,7 @@ fn check_where_clauses<'tcx, 'fcx>( } let mut param_count = CountParams::default(); let has_region = pred.visit_with(&mut param_count).is_break(); - let substituted_pred = pred.subst(fcx.tcx, substs); + let substituted_pred = pred.subst(tcx, substs); // Don't check non-defaulted params, dependent defaults (including lifetimes) // or preds with multiple params. if substituted_pred.has_param_types_or_consts() @@ -879,14 +869,14 @@ fn check_where_clauses<'tcx, 'fcx>( traits::Obligation::new(cause, fcx.param_env, pred) }); - let predicates = predicates.instantiate_identity(fcx.tcx); + let predicates = predicates.instantiate_identity(tcx); if let Some((mut return_ty, span)) = return_ty { if return_ty.has_infer_types_or_consts() { fcx.select_obligations_where_possible(false, |_| {}); return_ty = fcx.resolve_vars_if_possible(return_ty); } - check_opaque_types(tcx, fcx, def_id.expect_local(), span, return_ty); + check_opaque_types(fcx, def_id.expect_local(), span, return_ty); } let predicates = fcx.normalize_associated_types_in(span, predicates); @@ -905,7 +895,6 @@ fn check_where_clauses<'tcx, 'fcx>( } fn check_fn_or_method<'fcx, 'tcx>( - tcx: TyCtxt<'tcx>, fcx: &FnCtxt<'fcx, 'tcx>, span: Span, sig: ty::PolyFnSig<'tcx>, @@ -930,7 +919,7 @@ fn check_fn_or_method<'fcx, 'tcx>( // FIXME(#25759) return types should not be implied bounds implied_bounds.push(sig.output()); - check_where_clauses(tcx, fcx, span, def_id, Some((sig.output(), hir_decl.output.span()))); + check_where_clauses(fcx, span, def_id, Some((sig.output(), hir_decl.output.span()))); } /// Checks "defining uses" of opaque `impl Trait` types to ensure that they meet the restrictions @@ -953,15 +942,16 @@ fn check_fn_or_method<'fcx, 'tcx>( /// ``` /// fn check_opaque_types<'fcx, 'tcx>( - tcx: TyCtxt<'tcx>, fcx: &FnCtxt<'fcx, 'tcx>, fn_def_id: LocalDefId, span: Span, ty: Ty<'tcx>, ) { - trace!("check_opaque_types(ty={:?})", ty); + trace!("check_opaque_types(fn_def_id={:?}, ty={:?})", fn_def_id, ty); + let tcx = fcx.tcx; + ty.fold_with(&mut ty::fold::BottomUpFolder { - tcx: fcx.tcx, + tcx, ty_op: |ty| { if let ty::Opaque(def_id, substs) = *ty.kind() { trace!("check_opaque_types: opaque_ty, {:?}, {:?}", def_id, substs); @@ -1298,12 +1288,14 @@ fn check_variances_for_type_defn<'tcx>( match param.name { hir::ParamName::Error => {} - _ => report_bivariance(tcx, param.span, param.name.ident().name), + _ => report_bivariance(tcx, param), } } } -fn report_bivariance(tcx: TyCtxt<'_>, span: Span, param_name: Symbol) { +fn report_bivariance(tcx: TyCtxt<'_>, param: &rustc_hir::GenericParam<'_>) { + let span = param.span; + let param_name = param.name.ident().name; let mut err = error_392(tcx, span, param_name); let suggested_marker_id = tcx.lang_items().phantom_data(); @@ -1318,7 +1310,14 @@ fn report_bivariance(tcx: TyCtxt<'_>, span: Span, param_name: Symbol) { format!("consider removing `{}` or referring to it in a field", param_name) }; err.help(&msg); - err.emit(); + + if matches!(param.kind, rustc_hir::GenericParamKind::Type { .. }) { + err.help(&format!( + "if you intended `{0}` to be a const parameter, use `const {0}: usize` instead", + param_name + )); + } + err.emit() } /// Feature gates RFC 2056 -- trivial bounds, checking for global bounds that diff --git a/compiler/rustc_typeck/src/check/writeback.rs b/compiler/rustc_typeck/src/check/writeback.rs index e472add6e8..032cc7ee23 100644 --- a/compiler/rustc_typeck/src/check/writeback.rs +++ b/compiler/rustc_typeck/src/check/writeback.rs @@ -475,8 +475,9 @@ impl<'cx, 'tcx> WritebackCx<'cx, 'tcx> { } fn visit_opaque_types(&mut self, span: Span) { - for (&def_id, opaque_defn) in self.fcx.opaque_types.borrow().iter() { - let hir_id = self.tcx().hir().local_def_id_to_hir_id(def_id.expect_local()); + for &(opaque_type_key, opaque_defn) in self.fcx.opaque_types.borrow().iter() { + let hir_id = + self.tcx().hir().local_def_id_to_hir_id(opaque_type_key.def_id.expect_local()); let instantiated_ty = self.resolve(opaque_defn.concrete_ty, &hir_id); debug_assert!(!instantiated_ty.has_escaping_bound_vars()); @@ -494,50 +495,47 @@ impl<'cx, 'tcx> WritebackCx<'cx, 'tcx> { // ``` // figures out the concrete type with `U`, but the stored type is with `T`. let definition_ty = self.fcx.infer_opaque_definition_from_instantiation( - def_id, - opaque_defn.substs, + opaque_type_key, instantiated_ty, span, ); let mut skip_add = false; - if let ty::Opaque(defin_ty_def_id, _substs) = *definition_ty.kind() { + if let ty::Opaque(definition_ty_def_id, _substs) = *definition_ty.kind() { if let hir::OpaqueTyOrigin::Misc | hir::OpaqueTyOrigin::TyAlias = opaque_defn.origin { - if def_id == defin_ty_def_id { + if opaque_type_key.def_id == definition_ty_def_id { debug!( "skipping adding concrete definition for opaque type {:?} {:?}", - opaque_defn, defin_ty_def_id + opaque_defn, opaque_type_key.def_id ); skip_add = true; } } } - if !opaque_defn.substs.needs_infer() { + if !opaque_type_key.substs.needs_infer() { // We only want to add an entry into `concrete_opaque_types` // if we actually found a defining usage of this opaque type. // Otherwise, we do nothing - we'll either find a defining usage // in some other location, or we'll end up emitting an error due // to the lack of defining usage if !skip_add { - let new = ty::ResolvedOpaqueTy { - concrete_type: definition_ty, - substs: opaque_defn.substs, - }; - - let old = self.typeck_results.concrete_opaque_types.insert(def_id, new); - if let Some(old) = old { - if old.concrete_type != definition_ty || old.substs != opaque_defn.substs { + let old_concrete_ty = self + .typeck_results + .concrete_opaque_types + .insert(opaque_type_key, definition_ty); + if let Some(old_concrete_ty) = old_concrete_ty { + if old_concrete_ty != definition_ty { span_bug!( span, "`visit_opaque_types` tried to write different types for the same \ opaque type: {:?}, {:?}, {:?}, {:?}", - def_id, + opaque_type_key.def_id, definition_ty, opaque_defn, - old, + old_concrete_ty, ); } } diff --git a/compiler/rustc_typeck/src/check_unused.rs b/compiler/rustc_typeck/src/check_unused.rs index e1743a5dfc..836bed2a15 100644 --- a/compiler/rustc_typeck/src/check_unused.rs +++ b/compiler/rustc_typeck/src/check_unused.rs @@ -1,7 +1,7 @@ use rustc_data_structures::fx::{FxHashMap, FxHashSet}; use rustc_errors::Applicability; use rustc_hir as hir; -use rustc_hir::def_id::{DefId, LocalDefId, LOCAL_CRATE}; +use rustc_hir::def_id::{DefId, LocalDefId}; use rustc_hir::itemlikevisit::ItemLikeVisitor; use rustc_middle::ty::TyCtxt; use rustc_session::lint; @@ -77,7 +77,7 @@ fn unused_crates_lint(tcx: TyCtxt<'_>) { // can always suggest removing (no matter which edition we are // in). let unused_extern_crates: FxHashMap = tcx - .maybe_unused_extern_crates(LOCAL_CRATE) + .maybe_unused_extern_crates(()) .iter() .filter(|&&(def_id, _)| { // The `def_id` here actually was calculated during resolution (at least diff --git a/compiler/rustc_typeck/src/coherence/inherent_impls.rs b/compiler/rustc_typeck/src/coherence/inherent_impls.rs index cc592c7a26..51698437a3 100644 --- a/compiler/rustc_typeck/src/coherence/inherent_impls.rs +++ b/compiler/rustc_typeck/src/coherence/inherent_impls.rs @@ -9,16 +9,14 @@ use rustc_errors::struct_span_err; use rustc_hir as hir; -use rustc_hir::def_id::{CrateNum, DefId, LocalDefId, LOCAL_CRATE}; +use rustc_hir::def_id::{DefId, LocalDefId}; use rustc_hir::itemlikevisit::ItemLikeVisitor; use rustc_middle::ty::{self, CrateInherentImpls, TyCtxt}; use rustc_span::Span; /// On-demand query: yields a map containing all types mapped to their inherent impls. -pub fn crate_inherent_impls(tcx: TyCtxt<'_>, crate_num: CrateNum) -> CrateInherentImpls { - assert_eq!(crate_num, LOCAL_CRATE); - +pub fn crate_inherent_impls(tcx: TyCtxt<'_>, (): ()) -> CrateInherentImpls { let krate = tcx.hir().krate(); let mut collect = InherentCollect { tcx, impls_map: Default::default() }; krate.visit_all_item_likes(&mut collect); @@ -27,9 +25,9 @@ pub fn crate_inherent_impls(tcx: TyCtxt<'_>, crate_num: CrateNum) -> CrateInhere /// On-demand query: yields a vector of the inherent impls for a specific type. pub fn inherent_impls(tcx: TyCtxt<'_>, ty_def_id: DefId) -> &[DefId] { - assert!(ty_def_id.is_local()); + let ty_def_id = ty_def_id.expect_local(); - let crate_map = tcx.crate_inherent_impls(ty_def_id.krate); + let crate_map = tcx.crate_inherent_impls(()); match crate_map.inherent_impls.get(&ty_def_id) { Some(v) => &v[..], None => &[], @@ -364,7 +362,7 @@ impl ItemLikeVisitor<'v> for InherentCollect<'tcx> { impl InherentCollect<'tcx> { fn check_def_id(&mut self, item: &hir::Item<'_>, def_id: DefId) { - if def_id.is_local() { + if let Some(def_id) = def_id.as_local() { // Add the implementation to the mapping from implementation to base // type def ID, if there is a base type for this implementation and // the implementation does not have any associated traits. diff --git a/compiler/rustc_typeck/src/coherence/inherent_impls_overlap.rs b/compiler/rustc_typeck/src/coherence/inherent_impls_overlap.rs index c69389e7b4..f039790eca 100644 --- a/compiler/rustc_typeck/src/coherence/inherent_impls_overlap.rs +++ b/compiler/rustc_typeck/src/coherence/inherent_impls_overlap.rs @@ -1,7 +1,7 @@ use rustc_data_structures::fx::{FxHashMap, FxHashSet}; use rustc_errors::struct_span_err; use rustc_hir as hir; -use rustc_hir::def_id::{CrateNum, DefId, LOCAL_CRATE}; +use rustc_hir::def_id::DefId; use rustc_hir::itemlikevisit::ItemLikeVisitor; use rustc_middle::ty::{self, TyCtxt}; use rustc_span::Symbol; @@ -9,8 +9,7 @@ use rustc_trait_selection::traits::{self, SkipLeakCheck}; use smallvec::SmallVec; use std::collections::hash_map::Entry; -pub fn crate_inherent_impls_overlap_check(tcx: TyCtxt<'_>, crate_num: CrateNum) { - assert_eq!(crate_num, LOCAL_CRATE); +pub fn crate_inherent_impls_overlap_check(tcx: TyCtxt<'_>, (): ()) { let krate = tcx.hir().krate(); krate.visit_all_item_likes(&mut InherentOverlapChecker { tcx }); } diff --git a/compiler/rustc_typeck/src/coherence/mod.rs b/compiler/rustc_typeck/src/coherence/mod.rs index f04782a1f4..03a9fe0179 100644 --- a/compiler/rustc_typeck/src/coherence/mod.rs +++ b/compiler/rustc_typeck/src/coherence/mod.rs @@ -6,7 +6,7 @@ // mappings. That mapping code resides here. use rustc_errors::struct_span_err; -use rustc_hir::def_id::{DefId, LocalDefId, LOCAL_CRATE}; +use rustc_hir::def_id::{DefId, LocalDefId}; use rustc_middle::ty::query::Providers; use rustc_middle::ty::{self, TyCtxt, TypeFoldable}; use rustc_span::Span; @@ -203,8 +203,8 @@ pub fn check_coherence(tcx: TyCtxt<'_>) { tcx.sess.time("orphan_checking", || orphan::check(tcx)); // these queries are executed for side-effects (error reporting): - tcx.ensure().crate_inherent_impls(LOCAL_CRATE); - tcx.ensure().crate_inherent_impls_overlap_check(LOCAL_CRATE); + tcx.ensure().crate_inherent_impls(()); + tcx.ensure().crate_inherent_impls_overlap_check(()); } /// Checks whether an impl overlaps with the automatic `impl Trait for dyn Trait`. diff --git a/compiler/rustc_typeck/src/collect.rs b/compiler/rustc_typeck/src/collect.rs index d073262cad..aebae0d47c 100644 --- a/compiler/rustc_typeck/src/collect.rs +++ b/compiler/rustc_typeck/src/collect.rs @@ -2595,6 +2595,7 @@ fn from_target_feature( Some(sym::rtm_target_feature) => rust_features.rtm_target_feature, Some(sym::f16c_target_feature) => rust_features.f16c_target_feature, Some(sym::ermsb_target_feature) => rust_features.ermsb_target_feature, + Some(sym::bpf_target_feature) => rust_features.bpf_target_feature, Some(name) => bug!("unknown target feature gate {}", name), None => true, }; @@ -2768,7 +2769,25 @@ fn codegen_fn_attrs(tcx: TyCtxt<'_>, id: DefId) -> CodegenFnAttrs { } } else if tcx.sess.check_name(attr, sym::target_feature) { if !tcx.is_closure(id) && tcx.fn_sig(id).unsafety() == hir::Unsafety::Normal { - if !tcx.features().target_feature_11 { + if tcx.sess.target.is_like_wasm || tcx.sess.opts.actually_rustdoc { + // The `#[target_feature]` attribute is allowed on + // WebAssembly targets on all functions, including safe + // ones. Other targets require that `#[target_feature]` is + // only applied to unsafe funtions (pending the + // `target_feature_11` feature) because on most targets + // execution of instructions that are not supported is + // considered undefined behavior. For WebAssembly which is a + // 100% safe target at execution time it's not possible to + // execute undefined instructions, and even if a future + // feature was added in some form for this it would be a + // deterministic trap. There is no undefined behavior when + // executing WebAssembly so `#[target_feature]` is allowed + // on safe functions (but again, only for WebAssembly) + // + // Note that this is also allowed if `actually_rustdoc` so + // if a target is documenting some wasm-specific code then + // it's not spuriously denied. + } else if !tcx.features().target_feature_11 { let mut err = feature_err( &tcx.sess.parse_sess, sym::target_feature_11, diff --git a/compiler/rustc_typeck/src/collect/type_of.rs b/compiler/rustc_typeck/src/collect/type_of.rs index 97b6f5cf41..29a87b18a9 100644 --- a/compiler/rustc_typeck/src/collect/type_of.rs +++ b/compiler/rustc_typeck/src/collect/type_of.rs @@ -349,8 +349,8 @@ pub(super) fn type_of(tcx: TyCtxt<'_>, def_id: DefId) -> Ty<'_> { let concrete_ty = tcx .mir_borrowck(owner.expect_local()) .concrete_opaque_types - .get(&def_id.to_def_id()) - .map(|opaque| opaque.concrete_type) + .get_by(|(key, _)| key.def_id == def_id.to_def_id()) + .map(|concrete_ty| *concrete_ty) .unwrap_or_else(|| { tcx.sess.delay_span_bug( DUMMY_SP, @@ -450,8 +450,9 @@ pub(super) fn type_of(tcx: TyCtxt<'_>, def_id: DefId) -> Ty<'_> { tcx.typeck(def_id).node_type(anon_const.hir_id) } - Node::Expr(&Expr { kind: ExprKind::InlineAsm(ia), .. }) - if ia.operands.iter().any(|(op, _op_sp)| match op { + Node::Expr(&Expr { kind: ExprKind::InlineAsm(asm), .. }) + | Node::Item(&Item { kind: ItemKind::GlobalAsm(asm), .. }) + if asm.operands.iter().any(|(op, _op_sp)| match op { hir::InlineAsmOperand::Const { anon_const } => anon_const.hir_id == hir_id, _ => false, }) => @@ -514,7 +515,13 @@ fn find_opaque_ty_constraints(tcx: TyCtxt<'_>, def_id: LocalDefId) -> Ty<'_> { } // Calling `mir_borrowck` can lead to cycle errors through // const-checking, avoid calling it if we don't have to. - if !self.tcx.typeck(def_id).concrete_opaque_types.contains_key(&self.def_id) { + if self + .tcx + .typeck(def_id) + .concrete_opaque_types + .get_by(|(key, _)| key.def_id == self.def_id) + .is_none() + { debug!( "find_opaque_ty_constraints: no constraint for `{:?}` at `{:?}`", self.def_id, def_id, @@ -522,11 +529,13 @@ fn find_opaque_ty_constraints(tcx: TyCtxt<'_>, def_id: LocalDefId) -> Ty<'_> { return; } // Use borrowck to get the type with unerased regions. - let ty = self.tcx.mir_borrowck(def_id).concrete_opaque_types.get(&self.def_id); - if let Some(ty::ResolvedOpaqueTy { concrete_type, substs }) = ty { + let concrete_opaque_types = &self.tcx.mir_borrowck(def_id).concrete_opaque_types; + if let Some((opaque_type_key, concrete_type)) = + concrete_opaque_types.iter().find(|(key, _)| key.def_id == self.def_id) + { debug!( "find_opaque_ty_constraints: found constraint for `{:?}` at `{:?}`: {:?}", - self.def_id, def_id, ty, + self.def_id, def_id, concrete_type, ); // FIXME(oli-obk): trace the actual span from inference to improve errors. @@ -537,7 +546,7 @@ fn find_opaque_ty_constraints(tcx: TyCtxt<'_>, def_id: LocalDefId) -> Ty<'_> { // using `delay_span_bug`, just in case `wfcheck` slips up. let opaque_generics = self.tcx.generics_of(self.def_id); let mut used_params: FxHashSet<_> = FxHashSet::default(); - for (i, arg) in substs.iter().enumerate() { + for (i, arg) in opaque_type_key.substs.iter().enumerate() { let arg_is_param = match arg.unpack() { GenericArgKind::Type(ty) => matches!(ty.kind(), ty::Param(_)), GenericArgKind::Lifetime(lt) => { @@ -698,8 +707,8 @@ fn let_position_impl_trait_type(tcx: TyCtxt<'_>, opaque_ty_id: LocalDefId) -> Ty let owner_typeck_results = tcx.typeck(scope_def_id); let concrete_ty = owner_typeck_results .concrete_opaque_types - .get(&opaque_ty_def_id) - .map(|opaque| opaque.concrete_type) + .get_by(|(key, _)| key.def_id == opaque_ty_def_id) + .map(|concrete_ty| *concrete_ty) .unwrap_or_else(|| { tcx.sess.delay_span_bug( DUMMY_SP, @@ -765,7 +774,7 @@ fn infer_placeholder_type( if !ty.references_error() { diag.span_suggestion( span, - "replace `_` with the correct type", + "replace with the correct type", ty.to_string(), Applicability::MaybeIncorrect, ); diff --git a/compiler/rustc_typeck/src/expr_use_visitor.rs b/compiler/rustc_typeck/src/expr_use_visitor.rs index 532ee00daf..f174941279 100644 --- a/compiler/rustc_typeck/src/expr_use_visitor.rs +++ b/compiler/rustc_typeck/src/expr_use_visitor.rs @@ -763,7 +763,9 @@ impl<'a, 'tcx> ExprUseVisitor<'a, 'tcx> { PlaceBase::Local(*var_hir_id) }; let place_with_id = PlaceWithHirId::new( - capture_info.path_expr_id.unwrap_or(closure_expr.hir_id), + capture_info.path_expr_id.unwrap_or( + capture_info.capture_kind_expr_id.unwrap_or(closure_expr.hir_id), + ), place.base_ty, place_base, place.projections.clone(), diff --git a/compiler/rustc_typeck/src/lib.rs b/compiler/rustc_typeck/src/lib.rs index 4e07e52347..92ef829747 100644 --- a/compiler/rustc_typeck/src/lib.rs +++ b/compiler/rustc_typeck/src/lib.rs @@ -65,7 +65,6 @@ This API is completely unstable and subject to change. #![feature(is_sorted)] #![feature(iter_zip)] #![feature(nll)] -#![cfg_attr(bootstrap, feature(or_patterns))] #![feature(try_blocks)] #![feature(never_type)] #![feature(slice_partition_dedup)] @@ -97,7 +96,7 @@ mod variance; use rustc_errors::{struct_span_err, ErrorReported}; use rustc_hir as hir; -use rustc_hir::def_id::{DefId, LOCAL_CRATE}; +use rustc_hir::def_id::DefId; use rustc_hir::{Node, CRATE_HIR_ID}; use rustc_infer::infer::{InferOk, TyCtxtInferExt}; use rustc_infer::traits::TraitEngineExt as _; @@ -449,7 +448,7 @@ fn check_start_fn_ty(tcx: TyCtxt<'_>, start_def_id: DefId) { } fn check_for_entry_fn(tcx: TyCtxt<'_>) { - match tcx.entry_fn(LOCAL_CRATE) { + match tcx.entry_fn(()) { Some((def_id, EntryFnType::Main)) => check_main_fn_ty(tcx, def_id), Some((def_id, EntryFnType::Start)) => check_start_fn_ty(tcx, def_id), _ => {} @@ -510,7 +509,7 @@ pub fn check_crate(tcx: TyCtxt<'_>) -> Result<(), ErrorReported> { } }); - tcx.sess.time("item_bodies_checking", || tcx.typeck_item_bodies(LOCAL_CRATE)); + tcx.sess.time("item_bodies_checking", || tcx.typeck_item_bodies(())); check_unused::check_crate(tcx); check_for_entry_fn(tcx); diff --git a/compiler/rustc_typeck/src/outlives/mod.rs b/compiler/rustc_typeck/src/outlives/mod.rs index e94b8450bf..d7eb31c2ab 100644 --- a/compiler/rustc_typeck/src/outlives/mod.rs +++ b/compiler/rustc_typeck/src/outlives/mod.rs @@ -1,6 +1,6 @@ use hir::Node; use rustc_hir as hir; -use rustc_hir::def_id::{CrateNum, DefId, LOCAL_CRATE}; +use rustc_hir::def_id::DefId; use rustc_middle::ty::query::Providers; use rustc_middle::ty::subst::GenericArgKind; use rustc_middle::ty::{self, CratePredicatesMap, ToPredicate, TyCtxt}; @@ -23,7 +23,7 @@ fn inferred_outlives_of(tcx: TyCtxt<'_>, item_def_id: DefId) -> &[(ty::Predicate match tcx.hir().get(id) { Node::Item(item) => match item.kind { hir::ItemKind::Struct(..) | hir::ItemKind::Enum(..) | hir::ItemKind::Union(..) => { - let crate_map = tcx.inferred_outlives_crate(LOCAL_CRATE); + let crate_map = tcx.inferred_outlives_crate(()); let predicates = crate_map.predicates.get(&item_def_id).copied().unwrap_or(&[]); @@ -58,9 +58,7 @@ fn inferred_outlives_of(tcx: TyCtxt<'_>, item_def_id: DefId) -> &[(ty::Predicate } } -fn inferred_outlives_crate(tcx: TyCtxt<'_>, crate_num: CrateNum) -> CratePredicatesMap<'_> { - assert_eq!(crate_num, LOCAL_CRATE); - +fn inferred_outlives_crate(tcx: TyCtxt<'_>, (): ()) -> CratePredicatesMap<'_> { // Compute a map from each struct/enum/union S to the **explicit** // outlives predicates (`T: 'a`, `'a: 'b`) that the user wrote. // Typically there won't be many of these, except in older code where diff --git a/compiler/rustc_typeck/src/structured_errors/wrong_number_of_generic_args.rs b/compiler/rustc_typeck/src/structured_errors/wrong_number_of_generic_args.rs index e35c155746..bccc19774e 100644 --- a/compiler/rustc_typeck/src/structured_errors/wrong_number_of_generic_args.rs +++ b/compiler/rustc_typeck/src/structured_errors/wrong_number_of_generic_args.rs @@ -1,34 +1,20 @@ use crate::structured_errors::StructuredDiagnostic; -use hir::def::DefKind; use rustc_errors::{pluralize, Applicability, DiagnosticBuilder, DiagnosticId}; use rustc_hir as hir; +use rustc_middle::middle::resolve_lifetime::LifetimeScopeForPath; use rustc_middle::ty::{self as ty, TyCtxt}; use rustc_session::Session; -use rustc_span::Span; use rustc_span::{def_id::DefId, MultiSpan}; +use GenericArgsInfo::*; + /// Handles the `wrong number of type / lifetime / ... arguments` family of error messages. pub struct WrongNumberOfGenericArgs<'a, 'tcx> { crate tcx: TyCtxt<'tcx>, - /// "type", "lifetime" etc., put verbatim into the message - crate kind: &'static str, - - /// Minimum number of expected generic arguments (e.g. `2` for `HashMap`) - crate expected_min: usize, + crate angle_brackets: AngleBrackets, - /// Maximum number of expected generic arguments (e.g. `3` for `HashMap`) - crate expected_max: usize, - - /// Number of generic arguments provided by the user - crate provided: usize, - - /// Offset into `gen_params` - depends on the `kind`; might be different than `args_offset` when - /// user passed e.g. more arguments than was actually expected - crate params_offset: usize, - - /// Offset into `gen_args` - depends on the `kind` - crate args_offset: usize, + crate gen_args_info: GenericArgsInfo, /// Offending path segment crate path_segment: &'a hir::PathSegment<'a>, @@ -36,64 +22,344 @@ pub struct WrongNumberOfGenericArgs<'a, 'tcx> { /// Generic parameters as expected by type or trait crate gen_params: &'a ty::Generics, + /// Index offset into parameters. Depends on whether `Self` is included and on + /// number of lifetime parameters in case we're processing missing or redundant + /// type or constant arguments. + crate params_offset: usize, + /// Generic arguments as provided by user crate gen_args: &'a hir::GenericArgs<'a>, /// DefId of the generic type crate def_id: DefId, +} + +// Provides information about the kind of arguments that were provided for +// the PathSegment, for which missing generic arguments were detected +#[derive(Debug)] +pub(crate) enum AngleBrackets { + // No angle brackets were provided, but generic arguments exist in elided form + Implied, + + // No angle brackets were provided + Missing, + + // Angle brackets are available, but missing some generic arguments + Available, +} - /// Offending place where the generic type has been misused - crate span: Span, +// Information about the kind of arguments that are either missing or are unexpected +#[derive(Debug)] +pub enum GenericArgsInfo { + MissingLifetimes { + num_missing_args: usize, + }, + ExcessLifetimes { + num_redundant_args: usize, + }, + MissingTypesOrConsts { + num_missing_args: usize, + + // type or const generic arguments can have default values + num_default_params: usize, + + // lifetime arguments precede type and const parameters, this + // field gives the number of generic lifetime arguments to let + // us infer the position of type and const generic arguments + // in the angle brackets + args_offset: usize, + }, + + ExcessTypesOrConsts { + num_redundant_args: usize, + + // type or const generic arguments can have default values + num_default_params: usize, + + // lifetime arguments precede type and const parameters, this + // field gives the number of generic lifetime arguments to let + // us infer the position of type and const generic arguments + // in the angle brackets + args_offset: usize, + }, } -impl<'tcx> WrongNumberOfGenericArgs<'_, 'tcx> { - fn quantifier_and_bound(&self) -> (&'static str, usize) { - if self.expected_min == self.expected_max { - ("", self.expected_min) - } else if self.provided < self.expected_min { - ("at least ", self.expected_min) +impl<'a, 'tcx> WrongNumberOfGenericArgs<'a, 'tcx> { + pub fn new( + tcx: TyCtxt<'tcx>, + gen_args_info: GenericArgsInfo, + path_segment: &'a hir::PathSegment<'_>, + gen_params: &'a ty::Generics, + params_offset: usize, + gen_args: &'a hir::GenericArgs<'a>, + def_id: DefId, + ) -> Self { + let angle_brackets = if gen_args.span_ext().is_none() { + if gen_args.is_empty() { AngleBrackets::Missing } else { AngleBrackets::Implied } } else { - ("at most ", self.expected_max) + AngleBrackets::Available + }; + + Self { + tcx, + angle_brackets, + gen_args_info, + path_segment, + gen_params, + params_offset, + gen_args, + def_id, } } - fn start_diagnostics(&self) -> DiagnosticBuilder<'tcx> { - let span = self.path_segment.ident.span; + fn missing_lifetimes(&self) -> bool { + match self.gen_args_info { + MissingLifetimes { .. } | ExcessLifetimes { .. } => true, + MissingTypesOrConsts { .. } | ExcessTypesOrConsts { .. } => false, + } + } - let msg = { - let def_path = self.tcx.def_path_str(self.def_id); - let def_kind = self.tcx.def_kind(self.def_id).descr(self.def_id); - let (quantifier, bound) = self.quantifier_and_bound(); + fn kind(&self) -> String { + if self.missing_lifetimes() { "lifetime".to_string() } else { "generic".to_string() } + } - if self.gen_args.span().is_some() { - format!( - "this {} takes {}{} {} argument{} but {}{} {} argument{} {} supplied", - def_kind, - quantifier, - bound, - self.kind, - pluralize!(bound), - if self.provided > 0 && self.provided < self.expected_min { - "only " - } else { - "" - }, - self.provided, - self.kind, - pluralize!(self.provided), - if self.provided == 1 { "was" } else { "were" }, - ) - } else { - format!("missing generics for {} `{}`", def_kind, def_path) + fn num_provided_args(&self) -> usize { + if self.missing_lifetimes() { + self.num_provided_lifetime_args() + } else { + self.num_provided_type_or_const_args() + } + } + + fn num_provided_lifetime_args(&self) -> usize { + match self.angle_brackets { + AngleBrackets::Missing => 0, + // Only lifetime arguments can be implied + AngleBrackets::Implied => self.gen_args.args.len(), + AngleBrackets::Available => self.gen_args.args.iter().fold(0, |acc, arg| match arg { + hir::GenericArg::Lifetime(_) => acc + 1, + _ => acc, + }), + } + } + + fn num_provided_type_or_const_args(&self) -> usize { + match self.angle_brackets { + AngleBrackets::Missing => 0, + // Only lifetime arguments can be implied + AngleBrackets::Implied => 0, + AngleBrackets::Available => self.gen_args.args.iter().fold(0, |acc, arg| match arg { + hir::GenericArg::Type(_) | hir::GenericArg::Const(_) => acc + 1, + _ => acc, + }), + } + } + + fn num_expected_lifetime_args(&self) -> usize { + let num_provided_args = self.num_provided_lifetime_args(); + match self.gen_args_info { + MissingLifetimes { num_missing_args } => num_provided_args + num_missing_args, + ExcessLifetimes { num_redundant_args } => num_provided_args - num_redundant_args, + _ => 0, + } + } + + fn num_expected_type_or_const_args(&self) -> usize { + let num_provided_args = self.num_provided_type_or_const_args(); + match self.gen_args_info { + MissingTypesOrConsts { num_missing_args, .. } => num_provided_args + num_missing_args, + ExcessTypesOrConsts { num_redundant_args, .. } => { + num_provided_args - num_redundant_args } + _ => 0, + } + } + + // Gives the number of expected arguments taking into account default arguments + fn num_expected_type_or_const_args_including_defaults(&self) -> usize { + let provided_args = self.num_provided_type_or_const_args(); + match self.gen_args_info { + MissingTypesOrConsts { num_missing_args, num_default_params, .. } => { + provided_args + num_missing_args - num_default_params + } + ExcessTypesOrConsts { num_redundant_args, num_default_params, .. } => { + provided_args - num_redundant_args - num_default_params + } + _ => 0, + } + } + + fn num_missing_lifetime_args(&self) -> usize { + let missing_args = self.num_expected_lifetime_args() - self.num_provided_lifetime_args(); + assert!(missing_args > 0); + missing_args + } + + fn num_missing_type_or_const_args(&self) -> usize { + let missing_args = self.num_expected_type_or_const_args_including_defaults() + - self.num_provided_type_or_const_args(); + assert!(missing_args > 0); + missing_args + } + + fn num_excess_lifetime_args(&self) -> usize { + match self.gen_args_info { + ExcessLifetimes { num_redundant_args } => num_redundant_args, + _ => 0, + } + } + + fn num_excess_type_or_const_args(&self) -> usize { + match self.gen_args_info { + ExcessTypesOrConsts { num_redundant_args, .. } => num_redundant_args, + _ => 0, + } + } + + fn too_many_args_provided(&self) -> bool { + match self.gen_args_info { + MissingLifetimes { .. } | MissingTypesOrConsts { .. } => false, + ExcessLifetimes { num_redundant_args } + | ExcessTypesOrConsts { num_redundant_args, .. } => { + assert!(num_redundant_args > 0); + true + } + } + } + + fn not_enough_args_provided(&self) -> bool { + match self.gen_args_info { + MissingLifetimes { num_missing_args } + | MissingTypesOrConsts { num_missing_args, .. } => { + assert!(num_missing_args > 0); + true + } + ExcessLifetimes { .. } | ExcessTypesOrConsts { .. } => false, + } + } + + // Helper method to get the index offset in angle brackets, at which type or const arguments + // start appearing + fn get_lifetime_args_offset(&self) -> usize { + match self.gen_args_info { + MissingLifetimes { .. } | ExcessLifetimes { .. } => 0, + MissingTypesOrConsts { args_offset, .. } | ExcessTypesOrConsts { args_offset, .. } => { + args_offset + } + } + } + + fn get_num_default_params(&self) -> usize { + match self.gen_args_info { + MissingTypesOrConsts { num_default_params, .. } + | ExcessTypesOrConsts { num_default_params, .. } => num_default_params, + _ => 0, + } + } + + // Helper function to choose a quantifier word for the number of expected arguments + // and to give a bound for the number of expected arguments + fn get_quantifier_and_bound(&self) -> (&'static str, usize) { + if self.get_num_default_params() == 0 { + match self.gen_args_info { + MissingLifetimes { .. } | ExcessLifetimes { .. } => { + ("", self.num_expected_lifetime_args()) + } + MissingTypesOrConsts { .. } | ExcessTypesOrConsts { .. } => { + ("", self.num_expected_type_or_const_args()) + } + } + } else { + match self.gen_args_info { + MissingLifetimes { .. } => ("at least ", self.num_expected_lifetime_args()), + MissingTypesOrConsts { .. } => { + ("at least ", self.num_expected_type_or_const_args_including_defaults()) + } + ExcessLifetimes { .. } => ("at most ", self.num_expected_lifetime_args()), + ExcessTypesOrConsts { .. } => ("at most ", self.num_expected_type_or_const_args()), + } + } + } + + // Creates lifetime name suggestions from the lifetime parameter names + fn get_lifetime_args_suggestions_from_param_names(&self, num_params_to_take: usize) -> String { + self.gen_params + .params + .iter() + .skip(self.params_offset + self.num_provided_lifetime_args()) + .take(num_params_to_take) + .map(|param| param.name.to_string()) + .collect::>() + .join(", ") + } + + // Creates type or constant name suggestions from the provided parameter names + fn get_type_or_const_args_suggestions_from_param_names( + &self, + num_params_to_take: usize, + ) -> String { + self.gen_params + .params + .iter() + .skip(self.params_offset + self.num_provided_type_or_const_args()) + .take(num_params_to_take) + .map(|param| param.name.to_string()) + .collect::>() + .join(", ") + } + + fn create_error_message(&self) -> String { + let def_path = self.tcx.def_path_str(self.def_id); + let def_kind = self.tcx.def_kind(self.def_id).descr(self.def_id); + let (quantifier, bound) = self.get_quantifier_and_bound(); + let kind = self.kind(); + let provided_lt_args = self.num_provided_lifetime_args(); + let provided_type_or_const_args = self.num_provided_type_or_const_args(); + + let get_verb = |num_args| if num_args == 1 { "was" } else { "were" }; + + let (provided_args_str, verb) = match self.gen_args_info { + MissingLifetimes { .. } | ExcessLifetimes { .. } => ( + format!("{} lifetime argument{}", provided_lt_args, pluralize!(provided_lt_args)), + get_verb(provided_lt_args), + ), + MissingTypesOrConsts { .. } | ExcessTypesOrConsts { .. } => ( + format!( + "{} generic argument{}", + provided_type_or_const_args, + pluralize!(provided_type_or_const_args) + ), + get_verb(provided_type_or_const_args), + ), }; + if self.gen_args.span_ext().is_some() { + format!( + "this {} takes {}{} {} argument{} but {} {} supplied", + def_kind, + quantifier, + bound, + kind, + pluralize!(bound), + provided_args_str.as_str(), + verb + ) + } else { + format!("missing generics for {} `{}`", def_kind, def_path) + } + } + + fn start_diagnostics(&self) -> DiagnosticBuilder<'tcx> { + let span = self.path_segment.ident.span; + let msg = self.create_error_message(); + self.tcx.sess.struct_span_err_with_code(span, &msg, self.code()) } /// Builds the `expected 1 type argument / supplied 2 type arguments` message. fn notify(&self, err: &mut DiagnosticBuilder<'_>) { - let (quantifier, bound) = self.quantifier_and_bound(); + let (quantifier, bound) = self.get_quantifier_and_bound(); + let provided_args = self.num_provided_args(); err.span_label( self.path_segment.ident.span, @@ -101,12 +367,12 @@ impl<'tcx> WrongNumberOfGenericArgs<'_, 'tcx> { "expected {}{} {} argument{}", quantifier, bound, - self.kind, + self.kind(), pluralize!(bound), ), ); - // When user's provided too many arguments, we don't highlight each of them, because it + // When too many arguments were provided, we don't highlight each of them, because it // would overlap with the suggestion to remove them: // // ``` @@ -114,21 +380,27 @@ impl<'tcx> WrongNumberOfGenericArgs<'_, 'tcx> { // ----- ----- supplied 2 type arguments // ^^^^^^^ remove this type argument // ``` - if self.provided > self.expected_max { + if self.too_many_args_provided() { return; } - let args = self.gen_args.args.iter().skip(self.args_offset).take(self.provided).enumerate(); + let args = self + .gen_args + .args + .iter() + .skip(self.get_lifetime_args_offset()) + .take(provided_args) + .enumerate(); for (i, arg) in args { err.span_label( arg.span(), - if i + 1 == self.provided { + if i + 1 == provided_args { format!( "supplied {} {} argument{}", - self.provided, - self.kind, - pluralize!(self.provided) + provided_args, + self.kind(), + pluralize!(provided_args) ) } else { String::new() @@ -138,56 +410,24 @@ impl<'tcx> WrongNumberOfGenericArgs<'_, 'tcx> { } fn suggest(&self, err: &mut DiagnosticBuilder<'_>) { - if self.provided == 0 { - if self.gen_args.span().is_some() { - self.suggest_adding_args(err); - } else { - self.suggest_creating_generics(err); - } - } else if self.provided < self.expected_min { - self.suggest_adding_args(err); - } else { - self.suggest_removing_args_or_generics(err); - } - } - - /// Suggests to create generics (`<...>`) when current invocation site contains no generics at - /// all: - /// - /// ```text - /// type Map = HashMap; - /// ``` - fn suggest_creating_generics(&self, err: &mut DiagnosticBuilder<'_>) { - let params = self - .gen_params - .params - .iter() - .skip(self.params_offset) - .take(self.expected_min) - .map(|param| param.name.to_string()) - .collect::>() - .join(", "); - - let def_kind = self.tcx.def_kind(self.def_id); - - let sugg = if matches!(def_kind, DefKind::Fn | DefKind::AssocFn) { - format!("::<{}>", params) - } else { - format!("<{}>", params) - }; - - let msg = format!( - "use angle brackets to add missing {} argument{}", - self.kind, - pluralize!(self.expected_min), + debug!( + "suggest(self.provided {:?}, self.gen_args.span(): {:?})", + self.num_provided_args(), + self.gen_args.span(), ); - err.span_suggestion_verbose( - self.path_segment.ident.span.shrink_to_hi(), - &msg, - sugg, - Applicability::HasPlaceholders, - ); + match self.angle_brackets { + AngleBrackets::Missing | AngleBrackets::Implied => self.suggest_adding_args(err), + AngleBrackets::Available => { + if self.not_enough_args_provided() { + self.suggest_adding_args(err); + } else if self.too_many_args_provided() { + self.suggest_removing_args_or_generics(err); + } else { + unreachable!(); + } + } + } } /// Suggests to add missing argument(s) when current invocation site already contains some @@ -197,38 +437,151 @@ impl<'tcx> WrongNumberOfGenericArgs<'_, 'tcx> { /// type Map = HashMap; /// ``` fn suggest_adding_args(&self, err: &mut DiagnosticBuilder<'_>) { - assert!(!self.gen_args.is_empty()); - if self.gen_args.parenthesized { return; } - let missing_arg_count = self.expected_min - self.provided; + match self.gen_args_info { + MissingLifetimes { .. } => { + self.suggest_adding_lifetime_args(err); + } + MissingTypesOrConsts { .. } => { + self.suggest_adding_type_and_const_args(err); + } + _ => unreachable!(), + } + } - let (span, sugg_prefix) = if self.args_offset + self.provided == 0 { - let span = self.gen_args.args[0].span().shrink_to_lo(); - (span, "") + fn suggest_adding_lifetime_args(&self, err: &mut DiagnosticBuilder<'_>) { + debug!("suggest_adding_lifetime_args(path_segment: {:?})", self.path_segment); + let num_missing_args = self.num_missing_lifetime_args(); + let num_params_to_take = num_missing_args; + let msg = format!("add missing {} argument{}", self.kind(), pluralize!(num_missing_args)); + + // we first try to get lifetime name suggestions from scope or elision information. If none is + // available we use the parameter defintions + let suggested_args = if let Some(hir_id) = self.path_segment.hir_id { + if let Some(lifetimes_in_scope) = self.tcx.lifetime_scope(hir_id) { + match lifetimes_in_scope { + LifetimeScopeForPath::NonElided(param_names) => { + debug!("NonElided(param_names: {:?})", param_names); + + if param_names.len() >= num_params_to_take { + // use lifetime parameters in scope for suggestions + param_names + .iter() + .take(num_params_to_take) + .map(|p| (*p).clone()) + .collect::>() + .join(", ") + } else { + // Not enough lifetime arguments in scope -> create suggestions from + // lifetime parameter names in definition. An error for the incorrect + // lifetime scope will be output later. + self.get_lifetime_args_suggestions_from_param_names(num_params_to_take) + } + } + LifetimeScopeForPath::Elided => { + debug!("Elided"); + // use suggestions of the form `<'_, '_>` in case lifetime can be elided + ["'_"].repeat(num_params_to_take).join(",") + } + } + } else { + self.get_lifetime_args_suggestions_from_param_names(num_params_to_take) + } } else { - let span = - self.gen_args.args[self.args_offset + self.provided - 1].span().shrink_to_hi(); - (span, ", ") + self.get_lifetime_args_suggestions_from_param_names(num_params_to_take) }; - let msg = format!("add missing {} argument{}", self.kind, pluralize!(missing_arg_count)); + debug!("suggested_args: {:?}", &suggested_args); - let sugg = self - .gen_params - .params - .iter() - .skip(self.params_offset + self.provided) - .take(missing_arg_count) - .map(|param| param.name.to_string()) - .collect::>() - .join(", "); + match self.angle_brackets { + AngleBrackets::Missing => { + let span = self.path_segment.ident.span; + + // insert a suggestion of the form "Y<'a, 'b>" + let ident = self.path_segment.ident.name.to_ident_string(); + let sugg = format!("{}<{}>", ident, suggested_args); + debug!("sugg: {:?}", sugg); + + err.span_suggestion_verbose(span, &msg, sugg, Applicability::HasPlaceholders); + } + + AngleBrackets::Available => { + let (sugg_span, is_first) = if self.num_provided_lifetime_args() == 0 { + (self.gen_args.span().unwrap().shrink_to_lo(), true) + } else { + let last_lt = &self.gen_args.args[self.num_provided_lifetime_args() - 1]; + (last_lt.span().shrink_to_hi(), false) + }; + let has_non_lt_args = self.num_provided_type_or_const_args() != 0; + let has_bindings = !self.gen_args.bindings.is_empty(); + + let sugg_prefix = if is_first { "" } else { ", " }; + let sugg_suffix = + if is_first && (has_non_lt_args || has_bindings) { ", " } else { "" }; - let sugg = format!("{}{}", sugg_prefix, sugg); + let sugg = format!("{}{}{}", sugg_prefix, suggested_args, sugg_suffix); + debug!("sugg: {:?}", sugg); - err.span_suggestion_verbose(span, &msg, sugg, Applicability::HasPlaceholders); + err.span_suggestion_verbose(sugg_span, &msg, sugg, Applicability::HasPlaceholders); + } + AngleBrackets::Implied => { + // We never encounter missing lifetimes in situations in which lifetimes are elided + unreachable!(); + } + } + } + + fn suggest_adding_type_and_const_args(&self, err: &mut DiagnosticBuilder<'_>) { + let num_missing_args = self.num_missing_type_or_const_args(); + let msg = format!("add missing {} argument{}", self.kind(), pluralize!(num_missing_args)); + + let suggested_args = + self.get_type_or_const_args_suggestions_from_param_names(num_missing_args); + debug!("suggested_args: {:?}", suggested_args); + + match self.angle_brackets { + AngleBrackets::Missing | AngleBrackets::Implied => { + let span = self.path_segment.ident.span; + + // insert a suggestion of the form "Y" + let ident = self.path_segment.ident.name.to_ident_string(); + let sugg = format!("{}<{}>", ident, suggested_args); + debug!("sugg: {:?}", sugg); + + err.span_suggestion_verbose(span, &msg, sugg, Applicability::HasPlaceholders); + } + AngleBrackets::Available => { + let gen_args_span = self.gen_args.span().unwrap(); + let sugg_offset = + self.get_lifetime_args_offset() + self.num_provided_type_or_const_args(); + + let (sugg_span, is_first) = if sugg_offset == 0 { + (gen_args_span.shrink_to_lo(), true) + } else { + let arg_span = self.gen_args.args[sugg_offset - 1].span(); + // If we came here then inferred lifetimes's spans can only point + // to either the opening bracket or to the space right after. + // Both of these spans have an `hi` lower than or equal to the span + // of the generics excluding the brackets. + // This allows us to check if `arg_span` is the artificial span of + // an inferred lifetime, in which case the generic we're suggesting to + // add will be the first visible, even if it isn't the actual first generic. + (arg_span.shrink_to_hi(), arg_span.hi() <= gen_args_span.lo()) + }; + + let sugg_prefix = if is_first { "" } else { ", " }; + let sugg_suffix = + if is_first && !self.gen_args.bindings.is_empty() { ", " } else { "" }; + + let sugg = format!("{}{}{}", sugg_prefix, suggested_args, sugg_suffix); + debug!("sugg: {:?}", sugg); + + err.span_suggestion_verbose(sugg_span, &msg, sugg, Applicability::HasPlaceholders); + } + } } /// Suggests to remove redundant argument(s): @@ -237,19 +590,109 @@ impl<'tcx> WrongNumberOfGenericArgs<'_, 'tcx> { /// type Map = HashMap; /// ``` fn suggest_removing_args_or_generics(&self, err: &mut DiagnosticBuilder<'_>) { - assert!(self.provided > 0); + let num_provided_lt_args = self.num_provided_lifetime_args(); + let num_provided_type_const_args = self.num_provided_type_or_const_args(); + let num_provided_args = num_provided_lt_args + num_provided_type_const_args; + assert!(num_provided_args > 0); + + let num_redundant_lt_args = self.num_excess_lifetime_args(); + let num_redundant_type_or_const_args = self.num_excess_type_or_const_args(); + let num_redundant_args = num_redundant_lt_args + num_redundant_type_or_const_args; + + let redundant_lifetime_args = num_redundant_lt_args > 0; + let redundant_type_or_const_args = num_redundant_type_or_const_args > 0; + + let remove_entire_generics = num_redundant_args >= self.gen_args.args.len(); + + let remove_lifetime_args = |err: &mut DiagnosticBuilder<'_>| { + let mut lt_arg_spans = Vec::new(); + let mut found_redundant = false; + for arg in self.gen_args.args { + if let hir::GenericArg::Lifetime(_) = arg { + lt_arg_spans.push(arg.span()); + if lt_arg_spans.len() > self.num_expected_lifetime_args() { + found_redundant = true; + } + } else if found_redundant { + // Argument which is redundant and separated like this `'c` + // is not included to avoid including `Bar` in span. + // ``` + // type Foo<'a, T> = &'a T; + // let _: Foo<'a, 'b, Bar, 'c>; + // ``` + break; + } + } + + let span_lo_redundant_lt_args = lt_arg_spans[self.num_expected_lifetime_args()]; + let span_hi_redundant_lt_args = lt_arg_spans[lt_arg_spans.len() - 1]; + + let span_redundant_lt_args = span_lo_redundant_lt_args.to(span_hi_redundant_lt_args); + debug!("span_redundant_lt_args: {:?}", span_redundant_lt_args); + + let num_redundant_lt_args = lt_arg_spans.len() - self.num_expected_lifetime_args(); + let msg_lifetimes = format!( + "remove {} {} argument{}", + if num_redundant_lt_args == 1 { "this" } else { "these" }, + "lifetime", + pluralize!(num_redundant_lt_args), + ); + + err.span_suggestion( + span_redundant_lt_args, + &msg_lifetimes, + String::new(), + Applicability::MaybeIncorrect, + ); + }; + + let remove_type_or_const_args = |err: &mut DiagnosticBuilder<'_>| { + let mut gen_arg_spans = Vec::new(); + let mut found_redundant = false; + for arg in self.gen_args.args { + match arg { + hir::GenericArg::Type(_) | hir::GenericArg::Const(_) => { + gen_arg_spans.push(arg.span()); + if gen_arg_spans.len() > self.num_expected_type_or_const_args() { + found_redundant = true; + } + } + _ if found_redundant => break, + _ => {} + } + } - let redundant_args_count = self.provided - self.expected_max; - let remove_entire_generics = redundant_args_count >= self.gen_args.args.len(); + let span_lo_redundant_type_or_const_args = + gen_arg_spans[self.num_expected_type_or_const_args()]; + let span_hi_redundant_type_or_const_args = gen_arg_spans[gen_arg_spans.len() - 1]; - let (span, msg) = if remove_entire_generics { - let sm = self.tcx.sess.source_map(); + let span_redundant_type_or_const_args = + span_lo_redundant_type_or_const_args.to(span_hi_redundant_type_or_const_args); + debug!("span_redundant_type_or_const_args: {:?}", span_redundant_type_or_const_args); + let num_redundant_gen_args = + gen_arg_spans.len() - self.num_expected_type_or_const_args(); + let msg_types_or_consts = format!( + "remove {} {} argument{}", + if num_redundant_gen_args == 1 { "this" } else { "these" }, + "generic", + pluralize!(num_redundant_type_or_const_args), + ); + + err.span_suggestion( + span_redundant_type_or_const_args, + &msg_types_or_consts, + String::new(), + Applicability::MaybeIncorrect, + ); + }; + + if remove_entire_generics { let span = self .path_segment .args .unwrap() - .span_ext(sm) + .span_ext() .unwrap() .with_lo(self.path_segment.ident.span.hi()); @@ -258,70 +701,16 @@ impl<'tcx> WrongNumberOfGenericArgs<'_, 'tcx> { if self.gen_args.parenthesized { "parenthetical " } else { "" }, ); - (span, msg) + err.span_suggestion(span, &msg, String::new(), Applicability::MaybeIncorrect); + } else if redundant_lifetime_args && redundant_type_or_const_args { + remove_lifetime_args(err); + remove_type_or_const_args(err); + } else if redundant_lifetime_args { + remove_lifetime_args(err); } else { - // When it comes to removing particular argument(s) from the generics, there are two - // edge cases we have to consider: - // - // When the first redundant argument is at the beginning or in the middle of the - // generics, like so: - // - // ``` - // type Map = HashMap; - // ^^^^^^^^^^^^^^^^ - // | span must start with the argument - // ``` - // - // When the last redundant argument is at the ending of the generics, like so: - // - // ``` - // type Map = HashMap; - // ^^^^^^^^^^^^^^^^ - // | span must start with the comma - // ``` - - // Index of the first redundant argument - let from_idx = self.args_offset + self.expected_max; - - // Index of the last redundant argument - let to_idx = self.args_offset + self.provided - 1; - - assert!(from_idx <= to_idx); - - let (from, comma_eaten) = { - let first_argument_starts_generics = from_idx == 0; - let last_argument_ends_generics = to_idx + 1 == self.gen_args.args.len(); - - if !first_argument_starts_generics && last_argument_ends_generics { - (self.gen_args.args[from_idx - 1].span().hi(), true) - } else { - (self.gen_args.args[from_idx].span().lo(), false) - } - }; - - let to = { - let hi = self.gen_args.args[to_idx].span().hi(); - - if comma_eaten { - hi - } else { - self.gen_args.args.get(to_idx + 1).map(|arg| arg.span().lo()).unwrap_or(hi) - } - }; - - let span = Span::new(from, to, self.span.ctxt()); - - let msg = format!( - "remove {} {} argument{}", - if redundant_args_count == 1 { "this" } else { "these" }, - self.kind, - pluralize!(redundant_args_count), - ); - - (span, msg) - }; - - err.span_suggestion(span, &msg, String::new(), Applicability::MaybeIncorrect); + assert!(redundant_type_or_const_args); + remove_type_or_const_args(err); + } } /// Builds the `type defined here` message. @@ -334,7 +723,7 @@ impl<'tcx> WrongNumberOfGenericArgs<'_, 'tcx> { let msg = { let def_kind = self.tcx.def_kind(self.def_id).descr(self.def_id); - let (quantifier, bound) = self.quantifier_and_bound(); + let (quantifier, bound) = self.get_quantifier_and_bound(); let params = if bound == 0 { String::new() @@ -362,7 +751,7 @@ impl<'tcx> WrongNumberOfGenericArgs<'_, 'tcx> { def_kind, quantifier, bound, - self.kind, + self.kind(), pluralize!(bound), params, ) diff --git a/compiler/rustc_typeck/src/variance/mod.rs b/compiler/rustc_typeck/src/variance/mod.rs index 1565efbb02..66fb9eb869 100644 --- a/compiler/rustc_typeck/src/variance/mod.rs +++ b/compiler/rustc_typeck/src/variance/mod.rs @@ -6,7 +6,7 @@ use hir::Node; use rustc_arena::DroplessArena; use rustc_hir as hir; -use rustc_hir::def_id::{CrateNum, DefId, LOCAL_CRATE}; +use rustc_hir::def_id::DefId; use rustc_middle::ty::query::Providers; use rustc_middle::ty::{self, CrateVariancesMap, TyCtxt}; @@ -30,8 +30,7 @@ pub fn provide(providers: &mut Providers) { *providers = Providers { variances_of, crate_variances, ..*providers }; } -fn crate_variances(tcx: TyCtxt<'_>, crate_num: CrateNum) -> CrateVariancesMap<'_> { - assert_eq!(crate_num, LOCAL_CRATE); +fn crate_variances(tcx: TyCtxt<'_>, (): ()) -> CrateVariancesMap<'_> { let arena = DroplessArena::default(); let terms_cx = terms::determine_parameters_to_be_inferred(tcx, &arena); let constraints_cx = constraints::add_constraints_from_crate(terms_cx); @@ -79,6 +78,6 @@ fn variances_of(tcx: TyCtxt<'_>, item_def_id: DefId) -> &[ty::Variance] { // Everything else must be inferred. - let crate_map = tcx.crate_variances(LOCAL_CRATE); + let crate_map = tcx.crate_variances(()); crate_map.variances.get(&item_def_id).copied().unwrap_or(&[]) } diff --git a/config.toml.example b/config.toml.example index 6e5584797b..7fa5353d09 100644 --- a/config.toml.example +++ b/config.toml.example @@ -94,7 +94,7 @@ changelog-seen = 2 # support. You'll need to write a target specification at least, and most # likely, teach rustc about the C ABI of the target. Get in touch with the # Rust team and file an issue if you need assistance in porting! -#targets = "AArch64;ARM;Hexagon;MSP430;Mips;NVPTX;PowerPC;RISCV;Sparc;SystemZ;WebAssembly;X86" +#targets = "AArch64;ARM;BPF;Hexagon;MSP430;Mips;NVPTX;PowerPC;RISCV;Sparc;SystemZ;WebAssembly;X86" # LLVM experimental targets to build support for. These targets are specified in # the same format as above, but since these targets are experimental, they are @@ -372,8 +372,6 @@ changelog-seen = 2 # This is mostly useful for tools; if you have changes to `compiler/` they will be ignored. # # You can set this to "if-unchanged" to only download if `compiler/` has not been modified. -# -# FIXME(#82739): currently, this also uses the downloaded compiler for stage0, but that causes unnecessary rebuilds. #download-rustc = false # Number of codegen units to use for each compiler invocation. A value of 0 @@ -565,6 +563,14 @@ changelog-seen = 2 # Use LLVM libunwind as the implementation for Rust's unwinder. # Accepted values are 'in-tree' (formerly true), 'system' or 'no' (formerly false). +# This option only applies for Linux and Fuchsia targets. +# On Linux target, if crt-static is not enabled, 'no' means dynamic link to +# `libgcc_s.so`, 'in-tree' means static link to the in-tree build of llvm libunwind +# and 'system' means dynamic link to `libunwind.so`. If crt-static is enabled, +# the behavior is depend on the libc. On musl target, 'no' and 'in-tree' both +# means static link to the in-tree build of llvm libunwind, and 'system' means +# static link to `libunwind.a` provided by system. Due to the limitation of glibc, +# it must link to `libgcc_eh.a` to get a working output, and this option have no effect. #llvm-libunwind = 'no' # Enable Windows Control Flow Guard checks in the standard library. diff --git a/git-commit-hash b/git-commit-hash index a1526191f7..c7d63488d8 100644 --- a/git-commit-hash +++ b/git-commit-hash @@ -1 +1 @@ -53cb7b09b00cbea8754ffb78e7e3cb521cb8af4b \ No newline at end of file +a178d0322ce20e33eac124758e837cbd80a6f633 \ No newline at end of file diff --git a/library/alloc/Cargo.toml b/library/alloc/Cargo.toml index 4f97c95bcb..5793f5e681 100644 --- a/library/alloc/Cargo.toml +++ b/library/alloc/Cargo.toml @@ -11,7 +11,7 @@ edition = "2018" [dependencies] core = { path = "../core" } -compiler_builtins = { version = "0.1.39", features = ['rustc-dep-of-std'] } +compiler_builtins = { version = "0.1.40", features = ['rustc-dep-of-std'] } [dev-dependencies] rand = "0.7" diff --git a/library/alloc/benches/vec.rs b/library/alloc/benches/vec.rs index c9bdcaa78f..91eec10d57 100644 --- a/library/alloc/benches/vec.rs +++ b/library/alloc/benches/vec.rs @@ -551,19 +551,13 @@ const LEN: usize = 16384; #[bench] fn bench_chain_collect(b: &mut Bencher) { let data = black_box([0; LEN]); - b.iter(|| data.iter().cloned().chain([1].iter().cloned()).collect::>()); + b.iter(|| data.iter().cloned().chain([1]).collect::>()); } #[bench] fn bench_chain_chain_collect(b: &mut Bencher) { let data = black_box([0; LEN]); - b.iter(|| { - data.iter() - .cloned() - .chain([1].iter().cloned()) - .chain([2].iter().cloned()) - .collect::>() - }); + b.iter(|| data.iter().cloned().chain([1]).chain([2]).collect::>()); } #[bench] diff --git a/library/alloc/src/alloc.rs b/library/alloc/src/alloc.rs index cb9daaea00..a10a027ffd 100644 --- a/library/alloc/src/alloc.rs +++ b/library/alloc/src/alloc.rs @@ -308,7 +308,7 @@ unsafe impl Allocator for Global { /// The allocator for unique pointers. // This function must not unwind. If it does, MIR codegen will fail. -#[cfg(not(test))] +#[cfg(all(not(no_global_oom_handling), not(test)))] #[lang = "exchange_malloc"] #[inline] unsafe fn exchange_malloc(size: usize, align: usize) -> *mut u8 { @@ -337,6 +337,7 @@ pub(crate) unsafe fn box_free(ptr: Unique, alloc: A) // # Allocation error handler +#[cfg(not(no_global_oom_handling))] extern "Rust" { // This is the magic symbol to call the global alloc error handler. rustc generates // it to call `__rg_oom` if there is a `#[alloc_error_handler]`, or to call the @@ -358,7 +359,7 @@ extern "Rust" { /// [`set_alloc_error_hook`]: ../../std/alloc/fn.set_alloc_error_hook.html /// [`take_alloc_error_hook`]: ../../std/alloc/fn.take_alloc_error_hook.html #[stable(feature = "global_alloc", since = "1.28.0")] -#[cfg(not(test))] +#[cfg(all(not(no_global_oom_handling), not(test)))] #[rustc_allocator_nounwind] #[cold] pub fn handle_alloc_error(layout: Layout) -> ! { @@ -368,10 +369,10 @@ pub fn handle_alloc_error(layout: Layout) -> ! { } // For alloc test `std::alloc::handle_alloc_error` can be used directly. -#[cfg(test)] +#[cfg(all(not(no_global_oom_handling), test))] pub use std::alloc::handle_alloc_error; -#[cfg(not(any(target_os = "hermit", test)))] +#[cfg(all(not(no_global_oom_handling), not(any(target_os = "hermit", test))))] #[doc(hidden)] #[allow(unused_attributes)] #[unstable(feature = "alloc_internals", issue = "none")] diff --git a/library/alloc/src/borrow.rs b/library/alloc/src/borrow.rs index bdb2d67347..9d61b3684b 100644 --- a/library/alloc/src/borrow.rs +++ b/library/alloc/src/borrow.rs @@ -4,12 +4,15 @@ use core::cmp::Ordering; use core::hash::{Hash, Hasher}; -use core::ops::{Add, AddAssign, Deref}; +use core::ops::Deref; +#[cfg(not(no_global_oom_handling))] +use core::ops::{Add, AddAssign}; #[stable(feature = "rust1", since = "1.0.0")] pub use core::borrow::{Borrow, BorrowMut}; use crate::fmt; +#[cfg(not(no_global_oom_handling))] use crate::string::String; use Cow::*; @@ -429,6 +432,7 @@ impl AsRef for Cow<'_, T> { } } +#[cfg(not(no_global_oom_handling))] #[stable(feature = "cow_add", since = "1.14.0")] impl<'a> Add<&'a str> for Cow<'a, str> { type Output = Cow<'a, str>; @@ -440,6 +444,7 @@ impl<'a> Add<&'a str> for Cow<'a, str> { } } +#[cfg(not(no_global_oom_handling))] #[stable(feature = "cow_add", since = "1.14.0")] impl<'a> Add> for Cow<'a, str> { type Output = Cow<'a, str>; @@ -451,6 +456,7 @@ impl<'a> Add> for Cow<'a, str> { } } +#[cfg(not(no_global_oom_handling))] #[stable(feature = "cow_add", since = "1.14.0")] impl<'a> AddAssign<&'a str> for Cow<'a, str> { fn add_assign(&mut self, rhs: &'a str) { @@ -467,6 +473,7 @@ impl<'a> AddAssign<&'a str> for Cow<'a, str> { } } +#[cfg(not(no_global_oom_handling))] #[stable(feature = "cow_add", since = "1.14.0")] impl<'a> AddAssign> for Cow<'a, str> { fn add_assign(&mut self, rhs: Cow<'a, str>) { diff --git a/library/alloc/src/boxed.rs b/library/alloc/src/boxed.rs index ef37fef045..13b42442dc 100644 --- a/library/alloc/src/boxed.rs +++ b/library/alloc/src/boxed.rs @@ -84,7 +84,7 @@ //! /* Returns ownership to the caller */ //! struct Foo* foo_new(void); //! -//! /* Takes ownership from the caller; no-op when invoked with NULL */ +//! /* Takes ownership from the caller; no-op when invoked with null */ //! void foo_delete(struct Foo*); //! ``` //! @@ -139,7 +139,9 @@ use core::convert::{From, TryFrom}; use core::fmt; use core::future::Future; use core::hash::{Hash, Hasher}; -use core::iter::{FromIterator, FusedIterator, Iterator}; +#[cfg(not(no_global_oom_handling))] +use core::iter::FromIterator; +use core::iter::{FusedIterator, Iterator}; use core::marker::{Unpin, Unsize}; use core::mem; use core::ops::{ @@ -150,10 +152,16 @@ use core::ptr::{self, Unique}; use core::stream::Stream; use core::task::{Context, Poll}; -use crate::alloc::{handle_alloc_error, AllocError, Allocator, Global, Layout, WriteCloneIntoRaw}; +#[cfg(not(no_global_oom_handling))] +use crate::alloc::{handle_alloc_error, WriteCloneIntoRaw}; +use crate::alloc::{AllocError, Allocator, Global, Layout}; +#[cfg(not(no_global_oom_handling))] use crate::borrow::Cow; +#[cfg(not(no_global_oom_handling))] use crate::raw_vec::RawVec; +#[cfg(not(no_global_oom_handling))] use crate::str::from_boxed_utf8_unchecked; +#[cfg(not(no_global_oom_handling))] use crate::vec::Vec; /// A pointer type for heap allocation. @@ -177,6 +185,7 @@ impl Box { /// ``` /// let five = Box::new(5); /// ``` + #[cfg(not(no_global_oom_handling))] #[inline(always)] #[doc(alias = "alloc")] #[doc(alias = "malloc")] @@ -203,6 +212,7 @@ impl Box { /// /// assert_eq!(*five, 5) /// ``` + #[cfg(not(no_global_oom_handling))] #[unstable(feature = "new_uninit", issue = "63291")] #[inline] pub fn new_uninit() -> Box> { @@ -227,6 +237,7 @@ impl Box { /// ``` /// /// [zeroed]: mem::MaybeUninit::zeroed + #[cfg(not(no_global_oom_handling))] #[inline] #[doc(alias = "calloc")] #[unstable(feature = "new_uninit", issue = "63291")] @@ -236,6 +247,7 @@ impl Box { /// Constructs a new `Pin>`. If `T` does not implement `Unpin`, then /// `x` will be pinned in memory and unable to be moved. + #[cfg(not(no_global_oom_handling))] #[stable(feature = "pin", since = "1.33.0")] #[inline(always)] pub fn pin(x: T) -> Pin> { @@ -329,6 +341,7 @@ impl Box { /// /// let five = Box::new_in(5, System); /// ``` + #[cfg(not(no_global_oom_handling))] #[unstable(feature = "allocator_api", issue = "32838")] #[inline] pub fn new_in(x: T, alloc: A) -> Self { @@ -385,6 +398,7 @@ impl Box { /// assert_eq!(*five, 5) /// ``` #[unstable(feature = "allocator_api", issue = "32838")] + #[cfg(not(no_global_oom_handling))] // #[unstable(feature = "new_uninit", issue = "63291")] pub fn new_uninit_in(alloc: A) -> Box, A> { let layout = Layout::new::>(); @@ -447,6 +461,7 @@ impl Box { /// /// [zeroed]: mem::MaybeUninit::zeroed #[unstable(feature = "allocator_api", issue = "32838")] + #[cfg(not(no_global_oom_handling))] // #[unstable(feature = "new_uninit", issue = "63291")] pub fn new_zeroed_in(alloc: A) -> Box, A> { let layout = Layout::new::>(); @@ -490,6 +505,7 @@ impl Box { /// Constructs a new `Pin>`. If `T` does not implement `Unpin`, then /// `x` will be pinned in memory and unable to be moved. + #[cfg(not(no_global_oom_handling))] #[unstable(feature = "allocator_api", issue = "32838")] #[inline(always)] pub fn pin_in(x: T, alloc: A) -> Pin @@ -547,6 +563,7 @@ impl Box<[T]> { /// /// assert_eq!(*values, [1, 2, 3]) /// ``` + #[cfg(not(no_global_oom_handling))] #[unstable(feature = "new_uninit", issue = "63291")] pub fn new_uninit_slice(len: usize) -> Box<[mem::MaybeUninit]> { unsafe { RawVec::with_capacity(len).into_box(len) } @@ -570,6 +587,7 @@ impl Box<[T]> { /// ``` /// /// [zeroed]: mem::MaybeUninit::zeroed + #[cfg(not(no_global_oom_handling))] #[unstable(feature = "new_uninit", issue = "63291")] pub fn new_zeroed_slice(len: usize) -> Box<[mem::MaybeUninit]> { unsafe { RawVec::with_capacity_zeroed(len).into_box(len) } @@ -599,6 +617,7 @@ impl Box<[T], A> { /// /// assert_eq!(*values, [1, 2, 3]) /// ``` + #[cfg(not(no_global_oom_handling))] #[unstable(feature = "allocator_api", issue = "32838")] // #[unstable(feature = "new_uninit", issue = "63291")] pub fn new_uninit_slice_in(len: usize, alloc: A) -> Box<[mem::MaybeUninit], A> { @@ -625,6 +644,7 @@ impl Box<[T], A> { /// ``` /// /// [zeroed]: mem::MaybeUninit::zeroed + #[cfg(not(no_global_oom_handling))] #[unstable(feature = "allocator_api", issue = "32838")] // #[unstable(feature = "new_uninit", issue = "63291")] pub fn new_zeroed_slice_in(len: usize, alloc: A) -> Box<[mem::MaybeUninit], A> { @@ -1013,6 +1033,7 @@ impl Default for Box { } } +#[cfg(not(no_global_oom_handling))] #[stable(feature = "rust1", since = "1.0.0")] impl Default for Box<[T]> { fn default() -> Self { @@ -1020,6 +1041,7 @@ impl Default for Box<[T]> { } } +#[cfg(not(no_global_oom_handling))] #[stable(feature = "default_box_extra", since = "1.17.0")] impl Default for Box { fn default() -> Self { @@ -1027,6 +1049,7 @@ impl Default for Box { } } +#[cfg(not(no_global_oom_handling))] #[stable(feature = "rust1", since = "1.0.0")] impl Clone for Box { /// Returns a new box with a `clone()` of this box's contents. @@ -1076,6 +1099,7 @@ impl Clone for Box { } } +#[cfg(not(no_global_oom_handling))] #[stable(feature = "box_slice_clone", since = "1.3.0")] impl Clone for Box { fn clone(&self) -> Self { @@ -1182,9 +1206,10 @@ impl Hasher for Box { } } +#[cfg(not(no_global_oom_handling))] #[stable(feature = "from_for_ptrs", since = "1.6.0")] impl From for Box { - /// Converts a generic type `T` into a `Box` + /// Converts a `T` into a `Box` /// /// The conversion allocates on the heap and moves `t` /// from the stack into it. @@ -1214,6 +1239,7 @@ where } } +#[cfg(not(no_global_oom_handling))] #[stable(feature = "box_from_slice", since = "1.17.0")] impl From<&[T]> for Box<[T]> { /// Converts a `&[T]` into a `Box<[T]>` @@ -1239,6 +1265,7 @@ impl From<&[T]> for Box<[T]> { } } +#[cfg(not(no_global_oom_handling))] #[stable(feature = "box_from_cow", since = "1.45.0")] impl From> for Box<[T]> { #[inline] @@ -1250,6 +1277,7 @@ impl From> for Box<[T]> { } } +#[cfg(not(no_global_oom_handling))] #[stable(feature = "box_from_slice", since = "1.17.0")] impl From<&str> for Box { /// Converts a `&str` into a `Box` @@ -1268,6 +1296,7 @@ impl From<&str> for Box { } } +#[cfg(not(no_global_oom_handling))] #[stable(feature = "box_from_cow", since = "1.45.0")] impl From> for Box { #[inline] @@ -1567,6 +1596,7 @@ impl, U: ?Sized, A: Allocator> CoerceUnsized> fo #[unstable(feature = "dispatch_from_dyn", issue = "none")] impl, U: ?Sized> DispatchFromDyn> for Box {} +#[cfg(not(no_global_oom_handling))] #[stable(feature = "boxed_slice_from_iter", since = "1.32.0")] impl FromIterator for Box<[I]> { fn from_iter>(iter: T) -> Self { @@ -1574,6 +1604,7 @@ impl FromIterator for Box<[I]> { } } +#[cfg(not(no_global_oom_handling))] #[stable(feature = "box_slice_clone", since = "1.3.0")] impl Clone for Box<[T], A> { fn clone(&self) -> Self { diff --git a/library/alloc/src/collections/binary_heap.rs b/library/alloc/src/collections/binary_heap.rs index a201af0103..544e18d1ff 100644 --- a/library/alloc/src/collections/binary_heap.rs +++ b/library/alloc/src/collections/binary_heap.rs @@ -1299,6 +1299,7 @@ impl ExactSizeIterator for IntoIter { impl FusedIterator for IntoIter {} #[unstable(issue = "none", feature = "inplace_iteration")] +#[doc(hidden)] unsafe impl SourceIter for IntoIter { type Source = IntoIter; @@ -1309,6 +1310,7 @@ unsafe impl SourceIter for IntoIter { } #[unstable(issue = "none", feature = "inplace_iteration")] +#[doc(hidden)] unsafe impl InPlaceIterable for IntoIter {} impl AsIntoIter for IntoIter { diff --git a/library/alloc/src/collections/btree/map.rs b/library/alloc/src/collections/btree/map.rs index 971244718b..30194aa446 100644 --- a/library/alloc/src/collections/btree/map.rs +++ b/library/alloc/src/collections/btree/map.rs @@ -398,12 +398,12 @@ impl fmt::Debug for ValuesMut<'_, K, V> { /// See its documentation for more. /// /// [`into_keys`]: BTreeMap::into_keys -#[unstable(feature = "map_into_keys_values", issue = "75294")] +#[stable(feature = "map_into_keys_values", since = "1.54.0")] pub struct IntoKeys { inner: IntoIter, } -#[unstable(feature = "map_into_keys_values", issue = "75294")] +#[stable(feature = "map_into_keys_values", since = "1.54.0")] impl fmt::Debug for IntoKeys { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_list().entries(self.inner.iter().map(|(key, _)| key)).finish() @@ -416,12 +416,12 @@ impl fmt::Debug for IntoKeys { /// See its documentation for more. /// /// [`into_values`]: BTreeMap::into_values -#[unstable(feature = "map_into_keys_values", issue = "75294")] +#[stable(feature = "map_into_keys_values", since = "1.54.0")] pub struct IntoValues { inner: IntoIter, } -#[unstable(feature = "map_into_keys_values", issue = "75294")] +#[stable(feature = "map_into_keys_values", since = "1.54.0")] impl fmt::Debug for IntoValues { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_list().entries(self.inner.iter().map(|(_, val)| val)).finish() @@ -1242,7 +1242,6 @@ impl BTreeMap { /// # Examples /// /// ``` - /// #![feature(map_into_keys_values)] /// use std::collections::BTreeMap; /// /// let mut a = BTreeMap::new(); @@ -1253,7 +1252,7 @@ impl BTreeMap { /// assert_eq!(keys, [1, 2]); /// ``` #[inline] - #[unstable(feature = "map_into_keys_values", issue = "75294")] + #[stable(feature = "map_into_keys_values", since = "1.54.0")] pub fn into_keys(self) -> IntoKeys { IntoKeys { inner: self.into_iter() } } @@ -1265,7 +1264,6 @@ impl BTreeMap { /// # Examples /// /// ``` - /// #![feature(map_into_keys_values)] /// use std::collections::BTreeMap; /// /// let mut a = BTreeMap::new(); @@ -1276,7 +1274,7 @@ impl BTreeMap { /// assert_eq!(values, ["hello", "goodbye"]); /// ``` #[inline] - #[unstable(feature = "map_into_keys_values", issue = "75294")] + #[stable(feature = "map_into_keys_values", since = "1.54.0")] pub fn into_values(self) -> IntoValues { IntoValues { inner: self.into_iter() } } @@ -1441,7 +1439,10 @@ impl IntoIterator for BTreeMap { impl Drop for Dropper { fn drop(&mut self) { // Similar to advancing a non-fusing iterator. - fn next_or_end(this: &mut Dropper) -> Option<(K, V)> { + fn next_or_end( + this: &mut Dropper, + ) -> Option, marker::KV>> + { if this.remaining_length == 0 { unsafe { ptr::read(&this.front).deallocating_end() } None @@ -1457,13 +1458,15 @@ impl Drop for Dropper { fn drop(&mut self) { // Continue the same loop we perform below. This only runs when unwinding, so we // don't have to care about panics this time (they'll abort). - while let Some(_pair) = next_or_end(&mut self.0) {} + while let Some(kv) = next_or_end(&mut self.0) { + kv.drop_key_val(); + } } } - while let Some(pair) = next_or_end(self) { + while let Some(kv) = next_or_end(self) { let guard = DropGuard(self); - drop(pair); + kv.drop_key_val(); mem::forget(guard); } } @@ -1487,7 +1490,9 @@ impl Iterator for IntoIter { None } else { self.length -= 1; - Some(unsafe { self.range.front.as_mut().unwrap().deallocating_next_unchecked() }) + let front = self.range.front.as_mut().unwrap(); + let kv = unsafe { front.deallocating_next_unchecked() }; + Some(kv.into_key_val()) } } @@ -1503,7 +1508,9 @@ impl DoubleEndedIterator for IntoIter { None } else { self.length -= 1; - Some(unsafe { self.range.back.as_mut().unwrap().deallocating_next_back_unchecked() }) + let back = self.range.back.as_mut().unwrap(); + let kv = unsafe { back.deallocating_next_back_unchecked() }; + Some(kv.into_key_val()) } } } @@ -1776,7 +1783,7 @@ impl<'a, K, V> Range<'a, K, V> { } } -#[unstable(feature = "map_into_keys_values", issue = "75294")] +#[stable(feature = "map_into_keys_values", since = "1.54.0")] impl Iterator for IntoKeys { type Item = K; @@ -1801,24 +1808,24 @@ impl Iterator for IntoKeys { } } -#[unstable(feature = "map_into_keys_values", issue = "75294")] +#[stable(feature = "map_into_keys_values", since = "1.54.0")] impl DoubleEndedIterator for IntoKeys { fn next_back(&mut self) -> Option { self.inner.next_back().map(|(k, _)| k) } } -#[unstable(feature = "map_into_keys_values", issue = "75294")] +#[stable(feature = "map_into_keys_values", since = "1.54.0")] impl ExactSizeIterator for IntoKeys { fn len(&self) -> usize { self.inner.len() } } -#[unstable(feature = "map_into_keys_values", issue = "75294")] +#[stable(feature = "map_into_keys_values", since = "1.54.0")] impl FusedIterator for IntoKeys {} -#[unstable(feature = "map_into_keys_values", issue = "75294")] +#[stable(feature = "map_into_keys_values", since = "1.54.0")] impl Iterator for IntoValues { type Item = V; @@ -1835,21 +1842,21 @@ impl Iterator for IntoValues { } } -#[unstable(feature = "map_into_keys_values", issue = "75294")] +#[stable(feature = "map_into_keys_values", since = "1.54.0")] impl DoubleEndedIterator for IntoValues { fn next_back(&mut self) -> Option { self.inner.next_back().map(|(_, v)| v) } } -#[unstable(feature = "map_into_keys_values", issue = "75294")] +#[stable(feature = "map_into_keys_values", since = "1.54.0")] impl ExactSizeIterator for IntoValues { fn len(&self) -> usize { self.inner.len() } } -#[unstable(feature = "map_into_keys_values", issue = "75294")] +#[stable(feature = "map_into_keys_values", since = "1.54.0")] impl FusedIterator for IntoValues {} #[stable(feature = "btree_range", since = "1.17.0")] diff --git a/library/alloc/src/collections/btree/navigate.rs b/library/alloc/src/collections/btree/navigate.rs index 4399feaccc..563c070dd0 100644 --- a/library/alloc/src/collections/btree/navigate.rs +++ b/library/alloc/src/collections/btree/navigate.rs @@ -237,25 +237,27 @@ impl impl Handle, marker::Edge> { /// Given a leaf edge handle into a dying tree, returns the next leaf edge - /// on the right side, and the key-value pair in between, which is either - /// in the same leaf node, in an ancestor node, or non-existent. + /// on the right side, and the key-value pair in between, if they exist. /// - /// This method also deallocates any node(s) it reaches the end of. This - /// implies that if no more key-value pair exists, the entire remainder of - /// the tree will have been deallocated and there is nothing left to return. + /// If the given edge is the last one in a leaf, this method deallocates + /// the leaf, as well as any ancestor nodes whose last edge was reached. + /// This implies that if no more key-value pair follows, the entire tree + /// will have been deallocated and there is nothing left to return. /// /// # Safety - /// The given edge must not have been previously returned by counterpart - /// `deallocating_next_back`. - unsafe fn deallocating_next(self) -> Option<(Self, (K, V))> { + /// - The given edge must not have been previously returned by counterpart + /// `deallocating_next_back`. + /// - The returned KV handle is only valid to access the key and value, + /// and only valid until the next call to this method or counterpart + /// `deallocating_next_back`. + pub unsafe fn deallocating_next( + self, + ) -> Option<(Self, Handle, marker::KV>)> + { let mut edge = self.forget_node_type(); loop { edge = match edge.right_kv() { - Ok(kv) => { - let k = unsafe { ptr::read(kv.reborrow().into_kv().0) }; - let v = unsafe { ptr::read(kv.reborrow().into_kv().1) }; - return Some((kv.next_leaf_edge(), (k, v))); - } + Ok(kv) => return Some((unsafe { ptr::read(&kv) }.next_leaf_edge(), kv)), Err(last_edge) => match unsafe { last_edge.into_node().deallocate_and_ascend() } { Some(parent_edge) => parent_edge.forget_node_type(), None => return None, @@ -265,25 +267,27 @@ impl Handle, marker::Edge> { } /// Given a leaf edge handle into a dying tree, returns the next leaf edge - /// on the left side, and the key-value pair in between, which is either - /// in the same leaf node, in an ancestor node, or non-existent. + /// on the left side, and the key-value pair in between, if they exist. /// - /// This method also deallocates any node(s) it reaches the end of. This - /// implies that if no more key-value pair exists, the entire remainder of - /// the tree will have been deallocated and there is nothing left to return. + /// If the given edge is the first one in a leaf, this method deallocates + /// the leaf, as well as any ancestor nodes whose first edge was reached. + /// This implies that if no more key-value pair follows, the entire tree + /// will have been deallocated and there is nothing left to return. /// /// # Safety - /// The given edge must not have been previously returned by counterpart - /// `deallocating_next`. - unsafe fn deallocating_next_back(self) -> Option<(Self, (K, V))> { + /// - The given edge must not have been previously returned by counterpart + /// `deallocating_next`. + /// - The returned KV handle is only valid to access the key and value, + /// and only valid until the next call to this method or counterpart + /// `deallocating_next`. + unsafe fn deallocating_next_back( + self, + ) -> Option<(Self, Handle, marker::KV>)> + { let mut edge = self.forget_node_type(); loop { edge = match edge.left_kv() { - Ok(kv) => { - let k = unsafe { ptr::read(kv.reborrow().into_kv().0) }; - let v = unsafe { ptr::read(kv.reborrow().into_kv().1) }; - return Some((kv.next_back_leaf_edge(), (k, v))); - } + Ok(kv) => return Some((unsafe { ptr::read(&kv) }.next_back_leaf_edge(), kv)), Err(last_edge) => match unsafe { last_edge.into_node().deallocate_and_ascend() } { Some(parent_edge) => parent_edge.forget_node_type(), None => return None, @@ -373,13 +377,15 @@ impl Handle, marker::Edge> { /// /// # Safety /// - There must be another KV in the direction travelled. - /// - That KV was not previously returned by counterpart `next_back_unchecked` - /// on any copy of the handles being used to traverse the tree. + /// - That KV was not previously returned by counterpart + /// `deallocating_next_back_unchecked` on any copy of the handles + /// being used to traverse the tree. /// /// The only safe way to proceed with the updated handle is to compare it, drop it, - /// call this method again subject to its safety conditions, or call counterpart - /// `next_back_unchecked` subject to its safety conditions. - pub unsafe fn deallocating_next_unchecked(&mut self) -> (K, V) { + /// or call this method or counterpart `deallocating_next_back_unchecked` again. + pub unsafe fn deallocating_next_unchecked( + &mut self, + ) -> Handle, marker::KV> { super::mem::replace(self, |leaf_edge| unsafe { leaf_edge.deallocating_next().unwrap_unchecked() }) @@ -391,13 +397,15 @@ impl Handle, marker::Edge> { /// /// # Safety /// - There must be another KV in the direction travelled. - /// - That leaf edge was not previously returned by counterpart `next_unchecked` - /// on any copy of the handles being used to traverse the tree. + /// - That leaf edge was not previously returned by counterpart + /// `deallocating_next_unchecked` on any copy of the handles + /// being used to traverse the tree. /// /// The only safe way to proceed with the updated handle is to compare it, drop it, - /// call this method again subject to its safety conditions, or call counterpart - /// `next_unchecked` subject to its safety conditions. - pub unsafe fn deallocating_next_back_unchecked(&mut self) -> (K, V) { + /// or call this method or counterpart `deallocating_next_unchecked` again. + pub unsafe fn deallocating_next_back_unchecked( + &mut self, + ) -> Handle, marker::KV> { super::mem::replace(self, |leaf_edge| unsafe { leaf_edge.deallocating_next_back().unwrap_unchecked() }) diff --git a/library/alloc/src/collections/btree/node.rs b/library/alloc/src/collections/btree/node.rs index af403496e3..3c453529ba 100644 --- a/library/alloc/src/collections/btree/node.rs +++ b/library/alloc/src/collections/btree/node.rs @@ -422,14 +422,14 @@ impl<'a, K, V, Type> NodeRef, K, V, Type> { NodeRef { height: self.height, node: self.node, _marker: PhantomData } } - /// Borrows exclusive access to the leaf portion of any leaf or internal node. + /// Borrows exclusive access to the leaf portion of a leaf or internal node. fn as_leaf_mut(&mut self) -> &mut LeafNode { let ptr = Self::as_leaf_ptr(self); // SAFETY: we have exclusive access to the entire node. unsafe { &mut *ptr } } - /// Offers exclusive access to the leaf portion of any leaf or internal node. + /// Offers exclusive access to the leaf portion of a leaf or internal node. fn into_leaf_mut(mut self) -> &'a mut LeafNode { let ptr = Self::as_leaf_ptr(&mut self); // SAFETY: we have exclusive access to the entire node. @@ -437,6 +437,15 @@ impl<'a, K, V, Type> NodeRef, K, V, Type> { } } +impl NodeRef { + /// Borrows exclusive access to the leaf portion of a dying leaf or internal node. + fn as_leaf_dying(&mut self) -> &mut LeafNode { + let ptr = Self::as_leaf_ptr(self); + // SAFETY: we have exclusive access to the entire node. + unsafe { &mut *ptr } + } +} + impl<'a, K: 'a, V: 'a, Type> NodeRef, K, V, Type> { /// Borrows exclusive access to an element of the key storage area. /// @@ -1040,13 +1049,37 @@ impl<'a, K: 'a, V: 'a, NodeType> Handle, K, V, NodeType> } } - /// Replace the key and value that the KV handle refers to. + /// Replaces the key and value that the KV handle refers to. pub fn replace_kv(&mut self, k: K, v: V) -> (K, V) { let (key, val) = self.kv_mut(); (mem::replace(key, k), mem::replace(val, v)) } } +impl Handle, marker::KV> { + /// Extracts the key and value that the KV handle refers to. + pub fn into_key_val(mut self) -> (K, V) { + debug_assert!(self.idx < self.node.len()); + let leaf = self.node.as_leaf_dying(); + unsafe { + let key = leaf.keys.get_unchecked_mut(self.idx).assume_init_read(); + let val = leaf.vals.get_unchecked_mut(self.idx).assume_init_read(); + (key, val) + } + } + + /// Drops the key and value that the KV handle refers to. + #[inline] + pub fn drop_key_val(mut self) { + debug_assert!(self.idx < self.node.len()); + let leaf = self.node.as_leaf_dying(); + unsafe { + leaf.keys.get_unchecked_mut(self.idx).assume_init_drop(); + leaf.vals.get_unchecked_mut(self.idx).assume_init_drop(); + } + } +} + impl<'a, K: 'a, V: 'a, NodeType> Handle, K, V, NodeType>, marker::KV> { /// Helps implementations of `split` for a particular `NodeType`, /// by taking care of leaf data. diff --git a/library/alloc/src/collections/linked_list.rs b/library/alloc/src/collections/linked_list.rs index a5481fd175..1a58ad51f7 100644 --- a/library/alloc/src/collections/linked_list.rs +++ b/library/alloc/src/collections/linked_list.rs @@ -64,7 +64,15 @@ pub struct Iter<'a, T: 'a> { #[stable(feature = "collection_debug", since = "1.17.0")] impl fmt::Debug for Iter<'_, T> { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_tuple("Iter").field(&self.len).finish() + f.debug_tuple("Iter") + .field(&*mem::ManuallyDrop::new(LinkedList { + head: self.head, + tail: self.tail, + len: self.len, + marker: PhantomData, + })) + .field(&self.len) + .finish() } } @@ -82,19 +90,24 @@ impl Clone for Iter<'_, T> { /// documentation for more. #[stable(feature = "rust1", since = "1.0.0")] pub struct IterMut<'a, T: 'a> { - // We do *not* exclusively own the entire list here, references to node's `element` - // have been handed out by the iterator! So be careful when using this; the methods - // called must be aware that there can be aliasing pointers to `element`. - list: &'a mut LinkedList, head: Option>>, tail: Option>>, len: usize, + marker: PhantomData<&'a mut Node>, } #[stable(feature = "collection_debug", since = "1.17.0")] impl fmt::Debug for IterMut<'_, T> { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_tuple("IterMut").field(&self.list).field(&self.len).finish() + f.debug_tuple("IterMut") + .field(&*mem::ManuallyDrop::new(LinkedList { + head: self.head, + tail: self.tail, + len: self.len, + marker: PhantomData, + })) + .field(&self.len) + .finish() } } @@ -442,27 +455,6 @@ impl LinkedList { } } - /// Moves all elements from `other` to the begin of the list. - #[unstable(feature = "linked_list_prepend", issue = "none")] - pub fn prepend(&mut self, other: &mut Self) { - match self.head { - None => mem::swap(self, other), - Some(mut head) => { - // `as_mut` is okay here because we have exclusive access to the entirety - // of both lists. - if let Some(mut other_tail) = other.tail.take() { - unsafe { - head.as_mut().prev = Some(other_tail); - other_tail.as_mut().next = Some(head); - } - - self.head = other.head.take(); - self.len += mem::replace(&mut other.len, 0); - } - } - } - } - /// Provides a forward iterator. /// /// # Examples @@ -514,7 +506,7 @@ impl LinkedList { #[inline] #[stable(feature = "rust1", since = "1.0.0")] pub fn iter_mut(&mut self) -> IterMut<'_, T> { - IterMut { head: self.head, tail: self.tail, len: self.len, list: self } + IterMut { head: self.head, tail: self.tail, len: self.len, marker: PhantomData } } /// Provides a cursor at the front element. diff --git a/library/alloc/src/collections/mod.rs b/library/alloc/src/collections/mod.rs index 8213e904fb..b9b3d650ea 100644 --- a/library/alloc/src/collections/mod.rs +++ b/library/alloc/src/collections/mod.rs @@ -2,11 +2,16 @@ #![stable(feature = "rust1", since = "1.0.0")] +#[cfg(not(no_global_oom_handling))] pub mod binary_heap; +#[cfg(not(no_global_oom_handling))] mod btree; +#[cfg(not(no_global_oom_handling))] pub mod linked_list; +#[cfg(not(no_global_oom_handling))] pub mod vec_deque; +#[cfg(not(no_global_oom_handling))] #[stable(feature = "rust1", since = "1.0.0")] pub mod btree_map { //! A map based on a B-Tree. @@ -14,6 +19,7 @@ pub mod btree_map { pub use super::btree::map::*; } +#[cfg(not(no_global_oom_handling))] #[stable(feature = "rust1", since = "1.0.0")] pub mod btree_set { //! A set based on a B-Tree. @@ -21,22 +27,27 @@ pub mod btree_set { pub use super::btree::set::*; } +#[cfg(not(no_global_oom_handling))] #[stable(feature = "rust1", since = "1.0.0")] #[doc(no_inline)] pub use binary_heap::BinaryHeap; +#[cfg(not(no_global_oom_handling))] #[stable(feature = "rust1", since = "1.0.0")] #[doc(no_inline)] pub use btree_map::BTreeMap; +#[cfg(not(no_global_oom_handling))] #[stable(feature = "rust1", since = "1.0.0")] #[doc(no_inline)] pub use btree_set::BTreeSet; +#[cfg(not(no_global_oom_handling))] #[stable(feature = "rust1", since = "1.0.0")] #[doc(no_inline)] pub use linked_list::LinkedList; +#[cfg(not(no_global_oom_handling))] #[stable(feature = "rust1", since = "1.0.0")] #[doc(no_inline)] pub use vec_deque::VecDeque; diff --git a/library/alloc/src/collections/vec_deque/iter.rs b/library/alloc/src/collections/vec_deque/iter.rs index e4cfb3acdf..f3eb228c9e 100644 --- a/library/alloc/src/collections/vec_deque/iter.rs +++ b/library/alloc/src/collections/vec_deque/iter.rs @@ -66,7 +66,7 @@ impl<'a, T> Iterator for Iter<'a, T> { where Self: Sized, F: FnMut(B, Self::Item) -> R, - R: Try, + R: Try, { let (mut iter, final_res); if self.tail <= self.head { @@ -140,7 +140,7 @@ impl<'a, T> DoubleEndedIterator for Iter<'a, T> { where Self: Sized, F: FnMut(B, Self::Item) -> R, - R: Try, + R: Try, { let (mut iter, final_res); if self.tail <= self.head { diff --git a/library/alloc/src/collections/vec_deque/mod.rs b/library/alloc/src/collections/vec_deque/mod.rs index 7d6fbf1c43..5d03be35e4 100644 --- a/library/alloc/src/collections/vec_deque/mod.rs +++ b/library/alloc/src/collections/vec_deque/mod.rs @@ -2416,7 +2416,6 @@ impl VecDeque { /// found; the fourth could match any position in `[1, 4]`. /// /// ``` - /// #![feature(vecdeque_binary_search)] /// use std::collections::VecDeque; /// /// let deque: VecDeque<_> = vec![0, 1, 1, 1, 1, 2, 3, 5, 8, 13, 21, 34, 55].into(); @@ -2432,7 +2431,6 @@ impl VecDeque { /// sort order: /// /// ``` - /// #![feature(vecdeque_binary_search)] /// use std::collections::VecDeque; /// /// let mut deque: VecDeque<_> = vec![0, 1, 1, 1, 1, 2, 3, 5, 8, 13, 21, 34, 55].into(); @@ -2441,7 +2439,7 @@ impl VecDeque { /// deque.insert(idx, num); /// assert_eq!(deque, &[0, 1, 1, 1, 1, 2, 3, 5, 8, 13, 21, 34, 42, 55]); /// ``` - #[unstable(feature = "vecdeque_binary_search", issue = "78021")] + #[stable(feature = "vecdeque_binary_search", since = "1.54.0")] #[inline] pub fn binary_search(&self, x: &T) -> Result where @@ -2476,7 +2474,6 @@ impl VecDeque { /// found; the fourth could match any position in `[1, 4]`. /// /// ``` - /// #![feature(vecdeque_binary_search)] /// use std::collections::VecDeque; /// /// let deque: VecDeque<_> = vec![0, 1, 1, 1, 1, 2, 3, 5, 8, 13, 21, 34, 55].into(); @@ -2487,7 +2484,7 @@ impl VecDeque { /// let r = deque.binary_search_by(|x| x.cmp(&1)); /// assert!(matches!(r, Ok(1..=4))); /// ``` - #[unstable(feature = "vecdeque_binary_search", issue = "78021")] + #[stable(feature = "vecdeque_binary_search", since = "1.54.0")] pub fn binary_search_by<'a, F>(&'a self, mut f: F) -> Result where F: FnMut(&'a T) -> Ordering, @@ -2530,7 +2527,6 @@ impl VecDeque { /// fourth could match any position in `[1, 4]`. /// /// ``` - /// #![feature(vecdeque_binary_search)] /// use std::collections::VecDeque; /// /// let deque: VecDeque<_> = vec![(0, 0), (2, 1), (4, 1), (5, 1), @@ -2543,7 +2539,7 @@ impl VecDeque { /// let r = deque.binary_search_by_key(&1, |&(a, b)| b); /// assert!(matches!(r, Ok(1..=4))); /// ``` - #[unstable(feature = "vecdeque_binary_search", issue = "78021")] + #[stable(feature = "vecdeque_binary_search", since = "1.54.0")] #[inline] pub fn binary_search_by_key<'a, B, F>(&'a self, b: &B, mut f: F) -> Result where @@ -2574,7 +2570,6 @@ impl VecDeque { /// # Examples /// /// ``` - /// #![feature(vecdeque_binary_search)] /// use std::collections::VecDeque; /// /// let deque: VecDeque<_> = vec![1, 2, 3, 3, 5, 6, 7].into(); @@ -2584,7 +2579,7 @@ impl VecDeque { /// assert!(deque.iter().take(i).all(|&x| x < 5)); /// assert!(deque.iter().skip(i).all(|&x| !(x < 5))); /// ``` - #[unstable(feature = "vecdeque_binary_search", issue = "78021")] + #[stable(feature = "vecdeque_binary_search", since = "1.54.0")] pub fn partition_point

`](https://doc.rust-lang.org/std/pin/struct.Pin.html), a wrapper - around a kind of pointer which makes that pointer "pin" its value in place, - preventing the value referenced by that pointer from being moved. -* [`Unpin`](https://doc.rust-lang.org/std/marker/trait.Unpin.html), types that - are safe to be moved, even if they're pinned. - -Most users will not interact with pinning directly, and so we won't explain -more here. For the details, see the [documentation for -`std::pin`](https://doc.rust-lang.org/std/pin/index.html). - -What *is* useful to know about pinning is that it's a pre-requisite for -`async`/`await`. Folks who write async libraries may need to learn about -pinning, but folks using them generally shouldn't need to interact with this -feature at all. - diff --git a/src/doc/edition-guide/src/rust-next/qustion-mark-operator-in-macros.md b/src/doc/edition-guide/src/rust-next/qustion-mark-operator-in-macros.md deleted file mode 100644 index bed0381dd1..0000000000 --- a/src/doc/edition-guide/src/rust-next/qustion-mark-operator-in-macros.md +++ /dev/null @@ -1,14 +0,0 @@ -# ? operator in macros - -![Minimum Rust version: 1.32](https://img.shields.io/badge/Minimum%20Rust%20Version-1.32-brightgreen.svg) - -`macro_rules` macros can use `?`, like this: - -```rust -macro_rules! bar { - ($(a)?) => {} -} -``` - -The `?` will match zero or one repetitions of the pattern, similar to the -already-existing `*` for "zero or more" and `+` for "one or more." diff --git a/src/doc/edition-guide/src/rust-next/tryfrom-and-tryinto.md b/src/doc/edition-guide/src/rust-next/tryfrom-and-tryinto.md deleted file mode 100644 index 3c4be08734..0000000000 --- a/src/doc/edition-guide/src/rust-next/tryfrom-and-tryinto.md +++ /dev/null @@ -1,23 +0,0 @@ -# TryFrom and TryInto - -Initially added: ![Minimum Rust version: 1.34](https://img.shields.io/badge/Minimum%20Rust%20Version-1.34-brightgreen.svg) - -The [`TryFrom`](../../std/convert/trait.TryFrom.html) and -[`TryInto`](../../std/convert/trait.TryInto.html) traits are like the -[`From`](../../std/convert/trait.From.html) and -[`Into`](../../std/convert/trait.Into.html) traits, except that they return a -result, meaning that they may fail. - -For example, the `from_be_bytes` and related methods on integer types take -arrays, but data is often read in via slices. Converting between slices and -arrays is tedious to do manually. With the new traits, it can be done inline -with `.try_into()`: - -```rust -use std::convert::TryInto; -# fn main() -> Result<(), Box> { -# let slice = &[1, 2, 3, 4][..]; -let num = u32::from_be_bytes(slice.try_into()?); -# Ok(()) -# } -``` diff --git a/src/doc/edition-guide/src/rust-next/uniform-paths.md b/src/doc/edition-guide/src/rust-next/uniform-paths.md deleted file mode 100644 index 27a91abf4d..0000000000 --- a/src/doc/edition-guide/src/rust-next/uniform-paths.md +++ /dev/null @@ -1,23 +0,0 @@ -# Uniform Paths - -![Minimum Rust version: 1.32](https://img.shields.io/badge/Minimum%20Rust%20Version-1.32-brightgreen.svg) - -Rust 2018 added several improvements to the module system. We have one last -tweak landing in 1.32.0. Nicknamed "uniform paths", it permits previously -invalid import path statements to be resolved exactly the same way as -non-import paths. For example: - -```rust,edition2018 -enum Color { - Red, - Green, - Blue, -} - -use Color::*; -``` - -This code did not previously compile, as use statements had to start with -`super`, `self`, or `crate`. Now that the compiler supports uniform paths, -this code will work, and do what you probably expect: import the variants of -the Color enum defined above the `use` statement. diff --git a/src/doc/embedded-book/src/intro/install.md b/src/doc/embedded-book/src/intro/install.md index 53bb68851e..d2510925bb 100644 --- a/src/doc/embedded-book/src/intro/install.md +++ b/src/doc/embedded-book/src/intro/install.md @@ -58,9 +58,9 @@ rustup target add thumbv8m.main-none-eabihf ### `cargo-binutils` ``` text -$ cargo install cargo-binutils +cargo install cargo-binutils -$ rustup component add llvm-tools-preview +rustup component add llvm-tools-preview ``` ### `cargo-generate` diff --git a/src/doc/nomicon/src/atomics.md b/src/doc/nomicon/src/atomics.md index 05ab2070c8..2883107474 100644 --- a/src/doc/nomicon/src/atomics.md +++ b/src/doc/nomicon/src/atomics.md @@ -13,7 +13,7 @@ received some bugfixes since then.) Trying to fully explain the model in this book is fairly hopeless. It's defined in terms of madness-inducing causality graphs that require a full book to properly understand in a practical way. If you want all the nitty-gritty -details, you should check out the [C++20 draft specification (Section 31)][C++-model]. +details, you should check out the [C++ specification][C++-model]. Still, we'll try to cover the basics and some of the problems Rust developers face. @@ -256,4 +256,4 @@ relaxed operations can be cheaper on weakly-ordered platforms. [C11-busted]: http://plv.mpi-sws.org/c11comp/popl15.pdf -[C++-model]: http://eel.is/c++draft/atomics.order +[C++-model]: https://en.cppreference.com/w/cpp/atomic/memory_order diff --git a/src/doc/nomicon/src/exception-safety.md b/src/doc/nomicon/src/exception-safety.md index a3cbc6abd6..0a63764aaf 100644 --- a/src/doc/nomicon/src/exception-safety.md +++ b/src/doc/nomicon/src/exception-safety.md @@ -48,7 +48,7 @@ impl Vec { self.set_len(self.len() + to_push.len()); for (i, x) in to_push.iter().enumerate() { - self.ptr().offset(i as isize).write(x.clone()); + self.ptr().add(i).write(x.clone()); } } } @@ -58,7 +58,7 @@ impl Vec { We bypass `push` in order to avoid redundant capacity and `len` checks on the Vec that we definitely know has capacity. The logic is totally correct, except there's a subtle problem with our code: it's not exception-safe! `set_len`, -`offset`, and `write` are all fine; `clone` is the panic bomb we over-looked. +`add`, and `write` are all fine; `clone` is the panic bomb we over-looked. Clone is completely out of our control, and is totally free to panic. If it does, our function will exit early with the length of the Vec set too large. If diff --git a/src/doc/nomicon/src/safe-unsafe-meaning.md b/src/doc/nomicon/src/safe-unsafe-meaning.md index d5f2d4e4bf..6f55622e26 100644 --- a/src/doc/nomicon/src/safe-unsafe-meaning.md +++ b/src/doc/nomicon/src/safe-unsafe-meaning.md @@ -121,16 +121,19 @@ trait expects. In this situation, the Unsafe Rust in the internals of implementation is correct. If it isn't, it's the fault of the unsafe trait implementation, which is consistent with Rust's safety guarantees. -The decision of whether to mark a trait `unsafe` is an API design choice. -Rust has traditionally avoided doing this because it makes Unsafe -Rust pervasive, which isn't desirable. `Send` and `Sync` are marked unsafe -because thread safety is a *fundamental property* that unsafe code can't -possibly hope to defend against in the way it could defend against a buggy -`Ord` implementation. Similarly, `GlobalAllocator` is keeping accounts of all -the memory in the program and other things like `Box` or `Vec` build on top of -it. If it does something weird (giving the same chunk of memory to another -request when it is still in use), there's no chance to detect that and do -anything about it. +The decision of whether to mark a trait `unsafe` is an API design choice. A +safe trait is easier to implement, but any unsafe code that relies on it must +defend against incorrect behavior. Marking a trait `unsafe` shifts this +responsibility to the implementor. Rust has traditionally avoided marking +traits `unsafe` because it makes Unsafe Rust pervasive, which isn't desirable. + +`Send` and `Sync` are marked unsafe because thread safety is a *fundamental +property* that unsafe code can't possibly hope to defend against in the way it +could defend against a buggy `Ord` implementation. Similarly, `GlobalAllocator` +is keeping accounts of all the memory in the program and other things like +`Box` or `Vec` build on top of it. If it does something weird (giving the same +chunk of memory to another request when it is still in use), there's no chance +to detect that and do anything about it. The decision of whether to mark your own traits `unsafe` depends on the same sort of consideration. If `unsafe` code can't reasonably expect to defend diff --git a/src/doc/nomicon/src/vec-alloc.md b/src/doc/nomicon/src/vec-alloc.md index 0d2705702a..cf3844bdfd 100644 --- a/src/doc/nomicon/src/vec-alloc.md +++ b/src/doc/nomicon/src/vec-alloc.md @@ -1,41 +1,52 @@ # Allocating Memory -Using Unique throws a wrench in an important feature of Vec (and indeed all of -the std collections): an empty Vec doesn't actually allocate at all. So if we -can't allocate, but also can't put a null pointer in `ptr`, what do we do in -`Vec::new`? Well, we just put some other garbage in there! +Using `NonNull` throws a wrench in an important feature of Vec (and indeed all of +the std collections): creating an empty Vec doesn't actually allocate at all. This +is not the same as allocating a zero-sized memory block, which is not allowed by +the global allocator (it results in undefined behavior!). So if we can't allocate, +but also can't put a null pointer in `ptr`, what do we do in `Vec::new`? Well, we +just put some other garbage in there! This is perfectly fine because we already have `cap == 0` as our sentinel for no allocation. We don't even need to handle it specially in almost any code because we usually need to check if `cap > len` or `len > 0` anyway. The recommended -Rust value to put here is `mem::align_of::()`. Unique provides a convenience -for this: `Unique::dangling()`. There are quite a few places where we'll +Rust value to put here is `mem::align_of::()`. `NonNull` provides a convenience +for this: `NonNull::dangling()`. There are quite a few places where we'll want to use `dangling` because there's no real allocation to talk about but `null` would make the compiler do bad things. So: ```rust,ignore +use std::mem; + impl Vec { fn new() -> Self { assert!(mem::size_of::() != 0, "We're not ready to handle ZSTs"); - Vec { ptr: Unique::dangling(), len: 0, cap: 0 } + Vec { + ptr: NonNull::dangling(), + len: 0, + cap: 0, + _marker: PhantomData, + } } } +# fn main() {} ``` I slipped in that assert there because zero-sized types will require some special handling throughout our code, and I want to defer the issue for now. Without this assert, some of our early drafts will do some Very Bad Things. -Next we need to figure out what to actually do when we *do* want space. For -that, we'll need to use the rest of the heap APIs. These basically allow us to -talk directly to Rust's allocator (`malloc` on Unix platforms and `HeapAlloc` -on Windows by default). +Next we need to figure out what to actually do when we *do* want space. For that, +we use the global allocation functions [`alloc`][alloc], [`realloc`][realloc], +and [`dealloc`][dealloc] which are available in stable Rust in +[`std::alloc`][std_alloc]. These functions are expected to become deprecated in +favor of the methods of [`std::alloc::Global`][Global] after this type is stabilized. We'll also need a way to handle out-of-memory (OOM) conditions. The standard -library calls `std::alloc::oom()`, which in turn calls the `oom` langitem, -which aborts the program in a platform-specific manner. +library provides a function [`alloc::handle_alloc_error`][handle_alloc_error], +which will abort the program in a platform-specific manner. The reason we abort and don't panic is because unwinding can cause allocations to happen, and that seems like a bad thing to do when your allocator just came back with "hey I don't have any more memory". @@ -152,52 +163,48 @@ such we will guard against this case explicitly. Ok with all the nonsense out of the way, let's actually allocate some memory: ```rust,ignore -fn grow(&mut self) { - // this is all pretty delicate, so let's say it's all unsafe - unsafe { - let elem_size = mem::size_of::(); - - let (new_cap, ptr) = if self.cap == 0 { - let ptr = Global.allocate(Layout::array::(1).unwrap()); - (1, ptr) +use std::alloc::{self, Layout}; + +impl Vec { + fn grow(&mut self) { + let (new_cap, new_layout) = if self.cap == 0 { + (1, Layout::array::(1).unwrap()) } else { - // as an invariant, we can assume that `self.cap < isize::MAX`, - // so this doesn't need to be checked. + // This can't overflow since self.cap <= isize::MAX. let new_cap = 2 * self.cap; - // Similarly this can't overflow due to previously allocating this - let old_num_bytes = self.cap * elem_size; - - // check that the new allocation doesn't exceed `isize::MAX` at all - // regardless of the actual size of the capacity. This combines the - // `new_cap <= isize::MAX` and `new_num_bytes <= usize::MAX` checks - // we need to make. We lose the ability to allocate e.g. 2/3rds of - // the address space with a single Vec of i16's on 32-bit though. - // Alas, poor Yorick -- I knew him, Horatio. - assert!(old_num_bytes <= (isize::MAX as usize) / 2, - "capacity overflow"); - - let c: NonNull = self.ptr.into(); - let ptr = Global.grow(c.cast(), - Layout::array::(self.cap).unwrap(), - Layout::array::(new_cap).unwrap()); - (new_cap, ptr) + + // `Layout::array` checks that the number of bytes is <= usize::MAX, + // but this is redundant since old_layout.size() <= isize::MAX, + // so the `unwrap` should never fail. + let new_layout = Layout::array::(new_cap).unwrap(); + (new_cap, new_layout) }; - // If allocate or reallocate fail, oom - if ptr.is_err() { - handle_alloc_error(Layout::from_size_align_unchecked( - new_cap * elem_size, - mem::align_of::(), - )) - } + // Ensure that the new allocation doesn't exceed `isize::MAX` bytes. + assert!(new_layout.size() <= isize::MAX as usize, "Allocation too large"); - let ptr = ptr.unwrap(); + let new_ptr = if self.cap == 0 { + unsafe { alloc::alloc(new_layout) } + } else { + let old_layout = Layout::array::(self.cap).unwrap(); + let old_ptr = self.ptr.as_ptr() as *mut u8; + unsafe { alloc::realloc(old_ptr, old_layout, new_layout.size()) } + }; - self.ptr = Unique::new_unchecked(ptr.as_ptr() as *mut _); + // If allocation fails, `new_ptr` will be null, in which case we abort. + self.ptr = match NonNull::new(new_ptr as *mut T) { + Some(p) => p, + None => alloc::handle_alloc_error(new_layout), + }; self.cap = new_cap; } } +# fn main() {} ``` -Nothing particularly tricky here. Just computing sizes and alignments and doing -some careful multiplication checks. +[Global]: ../std/alloc/struct.Global.html +[handle_alloc_error]: ../alloc/alloc/fn.handle_alloc_error.html +[alloc]: ../alloc/alloc/fn.alloc.html +[realloc]: ../alloc/alloc/fn.realloc.html +[dealloc]: ../alloc/alloc/fn.dealloc.html +[std_alloc]: ../alloc/alloc/index.html diff --git a/src/doc/nomicon/src/vec-dealloc.md b/src/doc/nomicon/src/vec-dealloc.md index 34ddc2b5cc..45fe9412d3 100644 --- a/src/doc/nomicon/src/vec-dealloc.md +++ b/src/doc/nomicon/src/vec-dealloc.md @@ -7,20 +7,17 @@ ask Rust if `T` `needs_drop` and omit the calls to `pop`. However in practice LLVM is *really* good at removing simple side-effect free code like this, so I wouldn't bother unless you notice it's not being stripped (in this case it is). -We must not call `Global.deallocate` when `self.cap == 0`, as in this case we +We must not call `alloc::dealloc` when `self.cap == 0`, as in this case we haven't actually allocated any memory. - ```rust,ignore impl Drop for Vec { fn drop(&mut self) { if self.cap != 0 { while let Some(_) = self.pop() { } - + let layout = Layout::array::(self.cap).unwrap(); unsafe { - let c: NonNull = self.ptr.into(); - Global.deallocate(c.cast(), - Layout::array::(self.cap).unwrap()); + alloc::dealloc(self.ptr.as_ptr() as *mut u8, layout); } } } diff --git a/src/doc/nomicon/src/vec-drain.md b/src/doc/nomicon/src/vec-drain.md index a2fd0ebd18..9b38dfff63 100644 --- a/src/doc/nomicon/src/vec-drain.md +++ b/src/doc/nomicon/src/vec-drain.md @@ -46,7 +46,7 @@ impl RawValIter { // information to LLVM via GEP. slice.as_ptr() } else { - slice.as_ptr().offset(slice.len() as isize) + slice.as_ptr().add(slice.len()) } } } diff --git a/src/doc/nomicon/src/vec-final.md b/src/doc/nomicon/src/vec-final.md index e5f352964c..786cfab287 100644 --- a/src/doc/nomicon/src/vec-final.md +++ b/src/doc/nomicon/src/vec-final.md @@ -1,79 +1,85 @@ # The Final Code ```rust -#![feature(ptr_internals)] -#![feature(allocator_api)] -#![feature(alloc_layout_extra)] - -use std::ptr::{Unique, NonNull, self}; +use std::alloc::{self, Layout}; +use std::marker::PhantomData; use std::mem; use std::ops::{Deref, DerefMut}; -use std::marker::PhantomData; -use std::alloc::{ - Allocator, - Global, - GlobalAlloc, - Layout, - handle_alloc_error -}; +use std::ptr::{self, NonNull}; struct RawVec { - ptr: Unique, + ptr: NonNull, cap: usize, + _marker: PhantomData, } +unsafe impl Send for RawVec {} +unsafe impl Sync for RawVec {} + impl RawVec { fn new() -> Self { // !0 is usize::MAX. This branch should be stripped at compile time. let cap = if mem::size_of::() == 0 { !0 } else { 0 }; - // Unique::dangling() doubles as "unallocated" and "zero-sized allocation" - RawVec { ptr: Unique::dangling(), cap: cap } + // `NonNull::dangling()` doubles as "unallocated" and "zero-sized allocation" + RawVec { + ptr: NonNull::dangling(), + cap: cap, + _marker: PhantomData, + } } fn grow(&mut self) { - unsafe { - let elem_size = mem::size_of::(); - - // since we set the capacity to usize::MAX when elem_size is - // 0, getting to here necessarily means the Vec is overfull. - assert!(elem_size != 0, "capacity overflow"); + // since we set the capacity to usize::MAX when T has size 0, + // getting to here necessarily means the Vec is overfull. + assert!(mem::size_of::() != 0, "capacity overflow"); - let (new_cap, ptr) = if self.cap == 0 { - let ptr = Global.allocate(Layout::array::(1).unwrap()); - (1, ptr) - } else { - let new_cap = 2 * self.cap; - let c: NonNull = self.ptr.into(); - let ptr = Global.grow(c.cast(), - Layout::array::(self.cap).unwrap(), - Layout::array::(new_cap).unwrap()); - (new_cap, ptr) - }; - - // If allocate or reallocate fail, oom - if ptr.is_err() { - handle_alloc_error(Layout::from_size_align_unchecked( - new_cap * elem_size, - mem::align_of::(), - )) - } - let ptr = ptr.unwrap(); - - self.ptr = Unique::new_unchecked(ptr.as_ptr() as *mut _); - self.cap = new_cap; - } + let (new_cap, new_layout) = if self.cap == 0 { + (1, Layout::array::(1).unwrap()) + } else { + // This can't overflow because we ensure self.cap <= isize::MAX. + let new_cap = 2 * self.cap; + + // `Layout::array` checks that the number of bytes is <= usize::MAX, + // but this is redundant since old_layout.size() <= isize::MAX, + // so the `unwrap` should never fail. + let new_layout = Layout::array::(new_cap).unwrap(); + (new_cap, new_layout) + }; + + // Ensure that the new allocation doesn't exceed `isize::MAX` bytes. + assert!( + new_layout.size() <= isize::MAX as usize, + "Allocation too large" + ); + + let new_ptr = if self.cap == 0 { + unsafe { alloc::alloc(new_layout) } + } else { + let old_layout = Layout::array::(self.cap).unwrap(); + let old_ptr = self.ptr.as_ptr() as *mut u8; + unsafe { alloc::realloc(old_ptr, old_layout, new_layout.size()) } + }; + + // If allocation fails, `new_ptr` will be null, in which case we abort. + self.ptr = match NonNull::new(new_ptr as *mut T) { + Some(p) => p, + None => alloc::handle_alloc_error(new_layout), + }; + self.cap = new_cap; } } impl Drop for RawVec { fn drop(&mut self) { let elem_size = mem::size_of::(); + if self.cap != 0 && elem_size != 0 { unsafe { - let c: NonNull = self.ptr.into(); - Global.deallocate(c.cast(), - Layout::array::(self.cap).unwrap()); + alloc::dealloc( + self.ptr.as_ptr() as *mut u8, + Layout::array::(self.cap).unwrap(), + ); } } } @@ -85,21 +91,30 @@ pub struct Vec { } impl Vec { - fn ptr(&self) -> *mut T { self.buf.ptr.as_ptr() } + fn ptr(&self) -> *mut T { + self.buf.ptr.as_ptr() + } - fn cap(&self) -> usize { self.buf.cap } + fn cap(&self) -> usize { + self.buf.cap + } pub fn new() -> Self { - Vec { buf: RawVec::new(), len: 0 } + Vec { + buf: RawVec::new(), + len: 0, + } } pub fn push(&mut self, elem: T) { - if self.len == self.cap() { self.buf.grow(); } + if self.len == self.cap() { + self.buf.grow(); + } unsafe { - ptr::write(self.ptr().offset(self.len as isize), elem); + ptr::write(self.ptr().add(self.len), elem); } - // Can't fail, we'll OOM first. + // Can't overflow, we'll OOM first. self.len += 1; } @@ -108,23 +123,23 @@ impl Vec { None } else { self.len -= 1; - unsafe { - Some(ptr::read(self.ptr().offset(self.len as isize))) - } + unsafe { Some(ptr::read(self.ptr().add(self.len))) } } } pub fn insert(&mut self, index: usize, elem: T) { assert!(index <= self.len, "index out of bounds"); - if self.cap() == self.len { self.buf.grow(); } + if self.cap() == self.len { + self.buf.grow(); + } unsafe { - if index < self.len { - ptr::copy(self.ptr().offset(index as isize), - self.ptr().offset(index as isize + 1), - self.len - index); - } - ptr::write(self.ptr().offset(index as isize), elem); + ptr::copy( + self.ptr().add(index), + self.ptr().add(index + 1), + self.len - index, + ); + ptr::write(self.ptr().add(index), elem); self.len += 1; } } @@ -133,10 +148,12 @@ impl Vec { assert!(index < self.len, "index out of bounds"); unsafe { self.len -= 1; - let result = ptr::read(self.ptr().offset(index as isize)); - ptr::copy(self.ptr().offset(index as isize + 1), - self.ptr().offset(index as isize), - self.len - index); + let result = ptr::read(self.ptr().add(index)); + ptr::copy( + self.ptr().add(index + 1), + self.ptr().add(index), + self.len - index, + ); result } } @@ -181,24 +198,16 @@ impl Drop for Vec { impl Deref for Vec { type Target = [T]; fn deref(&self) -> &[T] { - unsafe { - std::slice::from_raw_parts(self.ptr(), self.len) - } + unsafe { std::slice::from_raw_parts(self.ptr(), self.len) } } } impl DerefMut for Vec { fn deref_mut(&mut self) -> &mut [T] { - unsafe { - std::slice::from_raw_parts_mut(self.ptr(), self.len) - } + unsafe { std::slice::from_raw_parts_mut(self.ptr(), self.len) } } } - - - - struct RawValIter { start: *const T, end: *const T, @@ -213,7 +222,7 @@ impl RawValIter { } else if slice.len() == 0 { slice.as_ptr() } else { - slice.as_ptr().offset(slice.len() as isize) + slice.as_ptr().add(slice.len()) }, } } @@ -239,8 +248,8 @@ impl Iterator for RawValIter { fn size_hint(&self) -> (usize, Option) { let elem_size = mem::size_of::(); - let len = (self.end as usize - self.start as usize) - / if elem_size == 0 { 1 } else { elem_size }; + let len = (self.end as usize - self.start as usize) / + if elem_size == 0 { 1 } else { elem_size }; (len, Some(len)) } } @@ -262,9 +271,6 @@ impl DoubleEndedIterator for RawValIter { } } - - - pub struct IntoIter { _buf: RawVec, // we don't actually care about this. Just need it to live. iter: RawValIter, @@ -272,12 +278,18 @@ pub struct IntoIter { impl Iterator for IntoIter { type Item = T; - fn next(&mut self) -> Option { self.iter.next() } - fn size_hint(&self) -> (usize, Option) { self.iter.size_hint() } + fn next(&mut self) -> Option { + self.iter.next() + } + fn size_hint(&self) -> (usize, Option) { + self.iter.size_hint() + } } impl DoubleEndedIterator for IntoIter { - fn next_back(&mut self) -> Option { self.iter.next_back() } + fn next_back(&mut self) -> Option { + self.iter.next_back() + } } impl Drop for IntoIter { @@ -286,9 +298,6 @@ impl Drop for IntoIter { } } - - - pub struct Drain<'a, T: 'a> { vec: PhantomData<&'a mut Vec>, iter: RawValIter, @@ -296,12 +305,18 @@ pub struct Drain<'a, T: 'a> { impl<'a, T> Iterator for Drain<'a, T> { type Item = T; - fn next(&mut self) -> Option { self.iter.next() } - fn size_hint(&self) -> (usize, Option) { self.iter.size_hint() } + fn next(&mut self) -> Option { + self.iter.next() + } + fn size_hint(&self) -> (usize, Option) { + self.iter.size_hint() + } } impl<'a, T> DoubleEndedIterator for Drain<'a, T> { - fn next_back(&mut self) -> Option { self.iter.next_back() } + fn next_back(&mut self) -> Option { + self.iter.next_back() + } } impl<'a, T> Drop for Drain<'a, T> { @@ -321,6 +336,7 @@ impl<'a, T> Drop for Drain<'a, T> { # # mod tests { # use super::*; +# # pub fn create_push_pop() { # let mut v = Vec::new(); # v.push(1); diff --git a/src/doc/nomicon/src/vec-insert-remove.md b/src/doc/nomicon/src/vec-insert-remove.md index c02a16d821..9138b3ec35 100644 --- a/src/doc/nomicon/src/vec-insert-remove.md +++ b/src/doc/nomicon/src/vec-insert-remove.md @@ -20,13 +20,11 @@ pub fn insert(&mut self, index: usize, elem: T) { if self.cap == self.len { self.grow(); } unsafe { - if index < self.len { - // ptr::copy(src, dest, len): "copy from source to dest len elems" - ptr::copy(self.ptr.as_ptr().offset(index as isize), - self.ptr.as_ptr().offset(index as isize + 1), - self.len - index); - } - ptr::write(self.ptr.as_ptr().offset(index as isize), elem); + // ptr::copy(src, dest, len): "copy from src to dest len elems" + ptr::copy(self.ptr.as_ptr().add(index), + self.ptr.as_ptr().add(index + 1), + self.len - index); + ptr::write(self.ptr.as_ptr().add(index), elem); self.len += 1; } } @@ -41,9 +39,9 @@ pub fn remove(&mut self, index: usize) -> T { assert!(index < self.len, "index out of bounds"); unsafe { self.len -= 1; - let result = ptr::read(self.ptr.as_ptr().offset(index as isize)); - ptr::copy(self.ptr.as_ptr().offset(index as isize + 1), - self.ptr.as_ptr().offset(index as isize), + let result = ptr::read(self.ptr.as_ptr().add(index)); + ptr::copy(self.ptr.as_ptr().add(index + 1), + self.ptr.as_ptr().add(index), self.len - index); result } diff --git a/src/doc/nomicon/src/vec-into-iter.md b/src/doc/nomicon/src/vec-into-iter.md index 02326f68d4..03c2a9d3e8 100644 --- a/src/doc/nomicon/src/vec-into-iter.md +++ b/src/doc/nomicon/src/vec-into-iter.md @@ -44,10 +44,11 @@ So we're going to use the following struct: ```rust,ignore pub struct IntoIter { - buf: Unique, + buf: NonNull, cap: usize, start: *const T, end: *const T, + _marker: PhantomData, } ``` @@ -73,8 +74,9 @@ impl Vec { // can't offset off this pointer, it's not allocated! ptr.as_ptr() } else { - ptr.as_ptr().offset(len as isize) + ptr.as_ptr().add(len) }, + _marker: PhantomData, } } } @@ -134,11 +136,9 @@ impl Drop for IntoIter { if self.cap != 0 { // drop any remaining elements for _ in &mut *self {} - + let layout = Layout::array::(self.cap).unwrap(); unsafe { - let c: NonNull = self.buf.into(); - Global.deallocate(c.cast(), - Layout::array::(self.cap).unwrap()); + alloc::dealloc(self.buf.as_ptr() as *mut u8, layout); } } } diff --git a/src/doc/nomicon/src/vec-layout.md b/src/doc/nomicon/src/vec-layout.md index 26c979f7f4..20e23065ed 100644 --- a/src/doc/nomicon/src/vec-layout.md +++ b/src/doc/nomicon/src/vec-layout.md @@ -22,64 +22,39 @@ conservatively assume we don't own any values of type `T`. See [the chapter on ownership and lifetimes][ownership] for all the details on variance and drop check. -As we saw in the ownership chapter, we should use `Unique` in place of -`*mut T` when we have a raw pointer to an allocation we own. Unique is unstable, +As we saw in the ownership chapter, the standard library uses `Unique` in place of +`*mut T` when it has a raw pointer to an allocation that it owns. Unique is unstable, so we'd like to not use it if possible, though. As a recap, Unique is a wrapper around a raw pointer that declares that: -* We are variant over `T` +* We are covariant over `T` * We may own a value of type `T` (for drop check) * We are Send/Sync if `T` is Send/Sync * Our pointer is never null (so `Option>` is null-pointer-optimized) -We can implement all of the above requirements except for the last -one in stable Rust: +We can implement all of the above requirements in stable Rust. To do this, instead +of using `Unique` we will use [`NonNull`][NonNull], another wrapper around a +raw pointer, which gives us two of the above properties, namely it is covariant +over `T` and is declared to never be null. By adding a `PhantomData` (for drop +check) and implementing Send/Sync if `T` is, we get the same results as using +`Unique`: -```rust,ignore +```rust +use std::ptr::NonNull; use std::marker::PhantomData; -use std::ops::Deref; -use std::mem; - -struct Unique { - ptr: *const T, // *const for variance - _marker: PhantomData, // For the drop checker -} - -// Deriving Send and Sync is safe because we are the Unique owners -// of this data. It's like Unique is "just" T. -unsafe impl Send for Unique {} -unsafe impl Sync for Unique {} -impl Unique { - pub fn new(ptr: *mut T) -> Self { - Unique { ptr: ptr, _marker: PhantomData } - } - - pub fn as_ptr(&self) -> *mut T { - self.ptr as *mut T - } -} -``` - -Unfortunately the mechanism for stating that your value is non-zero is -unstable and unlikely to be stabilized soon. As such we're just going to -take the hit and use std's Unique: - - -```rust,ignore pub struct Vec { - ptr: Unique, + ptr: NonNull, cap: usize, len: usize, + _marker: PhantomData, } -``` -If you don't care about the null-pointer optimization, then you can use the -stable code. However we will be designing the rest of the code around enabling -this optimization. It should be noted that `Unique::new` is unsafe to call, because -putting `null` inside of it is Undefined Behavior. Our stable Unique doesn't -need `new` to be unsafe because it doesn't make any interesting guarantees about -its contents. +unsafe impl Send for Vec {} +unsafe impl Sync for Vec {} +# fn main() {} +``` [ownership]: ownership.html +[NonNull]: ../std/ptr/struct.NonNull.html diff --git a/src/doc/nomicon/src/vec-push-pop.md b/src/doc/nomicon/src/vec-push-pop.md index 6f39a05d09..2366e36d7d 100644 --- a/src/doc/nomicon/src/vec-push-pop.md +++ b/src/doc/nomicon/src/vec-push-pop.md @@ -22,7 +22,7 @@ pub fn push(&mut self, elem: T) { if self.len == self.cap { self.grow(); } unsafe { - ptr::write(self.ptr.as_ptr().offset(self.len as isize), elem); + ptr::write(self.ptr.as_ptr().add(self.len), elem); } // Can't fail, we'll OOM first. @@ -48,7 +48,7 @@ pub fn pop(&mut self) -> Option { } else { self.len -= 1; unsafe { - Some(ptr::read(self.ptr.as_ptr().offset(self.len as isize))) + Some(ptr::read(self.ptr.as_ptr().add(self.len))) } } } diff --git a/src/doc/nomicon/src/vec-raw.md b/src/doc/nomicon/src/vec-raw.md index 7303a549fc..3fb62b72cc 100644 --- a/src/doc/nomicon/src/vec-raw.md +++ b/src/doc/nomicon/src/vec-raw.md @@ -10,56 +10,64 @@ allocating, growing, and freeing: ```rust,ignore struct RawVec { - ptr: Unique, + ptr: NonNull, cap: usize, + _marker: PhantomData, } +unsafe impl Send for RawVec {} +unsafe impl Sync for RawVec {} + impl RawVec { fn new() -> Self { - assert!(mem::size_of::() != 0, "We're not ready to handle ZSTs"); - RawVec { ptr: Unique::dangling(), cap: 0 } + assert!(mem::size_of::() != 0, "TODO: implement ZST support"); + RawVec { + ptr: NonNull::dangling(), + cap: 0, + _marker: PhantomData, + } } - // unchanged from Vec fn grow(&mut self) { - unsafe { - let elem_size = mem::size_of::(); - - let (new_cap, ptr) = if self.cap == 0 { - let ptr = Global.allocate(Layout::array::(1).unwrap()); - (1, ptr) - } else { - let new_cap = 2 * self.cap; - let c: NonNull = self.ptr.into(); - let ptr = Global.grow(c.cast(), - Layout::array::(self.cap).unwrap(), - Layout::array::(new_cap).unwrap()); - (new_cap, ptr) - }; - - // If allocate or reallocate fail, oom - if ptr.is_err() { - handle_alloc_error(Layout::from_size_align_unchecked( - new_cap * elem_size, - mem::align_of::(), - )) - } - - let ptr = ptr.unwrap(); - - self.ptr = Unique::new_unchecked(ptr.as_ptr() as *mut _); - self.cap = new_cap; - } + let (new_cap, new_layout) = if self.cap == 0 { + (1, Layout::array::(1).unwrap()) + } else { + // This can't overflow because we ensure self.cap <= isize::MAX. + let new_cap = 2 * self.cap; + + // Layout::array checks that the number of bytes is <= usize::MAX, + // but this is redundant since old_layout.size() <= isize::MAX, + // so the `unwrap` should never fail. + let new_layout = Layout::array::(new_cap).unwrap(); + (new_cap, new_layout) + }; + + // Ensure that the new allocation doesn't exceed `isize::MAX` bytes. + assert!(new_layout.size() <= isize::MAX as usize, "Allocation too large"); + + let new_ptr = if self.cap == 0 { + unsafe { alloc::alloc(new_layout) } + } else { + let old_layout = Layout::array::(self.cap).unwrap(); + let old_ptr = self.ptr.as_ptr() as *mut u8; + unsafe { alloc::realloc(old_ptr, old_layout, new_layout.size()) } + }; + + // If allocation fails, `new_ptr` will be null, in which case we abort. + self.ptr = match NonNull::new(new_ptr as *mut T) { + Some(p) => p, + None => alloc::handle_alloc_error(new_layout), + }; + self.cap = new_cap; } } impl Drop for RawVec { fn drop(&mut self) { if self.cap != 0 { + let layout = Layout::array::(self.cap).unwrap(); unsafe { - let c: NonNull = self.ptr.into(); - Global.deallocate(c.cast(), - Layout::array::(self.cap).unwrap()); + alloc::dealloc(self.ptr.as_ptr() as *mut u8, layout); } } } @@ -75,18 +83,25 @@ pub struct Vec { } impl Vec { - fn ptr(&self) -> *mut T { self.buf.ptr.as_ptr() } + fn ptr(&self) -> *mut T { + self.buf.ptr.as_ptr() + } - fn cap(&self) -> usize { self.buf.cap } + fn cap(&self) -> usize { + self.buf.cap + } pub fn new() -> Self { - Vec { buf: RawVec::new(), len: 0 } + Vec { + buf: RawVec::new(), + len: 0, + } } // push/pop/insert/remove largely unchanged: // * `self.ptr.as_ptr() -> self.ptr()` // * `self.cap -> self.cap()` - // * `self.grow -> self.buf.grow()` + // * `self.grow() -> self.buf.grow()` } impl Drop for Vec { @@ -127,7 +142,7 @@ impl Vec { IntoIter { start: buf.ptr.as_ptr(), - end: buf.ptr.as_ptr().offset(len as isize), + end: buf.ptr.as_ptr().add(len), _buf: buf, } } diff --git a/src/doc/nomicon/src/vec-zsts.md b/src/doc/nomicon/src/vec-zsts.md index 418f557a77..d1b8fe85e7 100644 --- a/src/doc/nomicon/src/vec-zsts.md +++ b/src/doc/nomicon/src/vec-zsts.md @@ -19,7 +19,7 @@ Thankfully we abstracted out pointer-iterators and allocating handling into ## Allocating Zero-Sized Types So if the allocator API doesn't support zero-sized allocations, what on earth -do we store as our allocation? `Unique::dangling()` of course! Almost every operation +do we store as our allocation? `NonNull::dangling()` of course! Almost every operation with a ZST is a no-op since ZSTs have exactly one value, and therefore no state needs to be considered to store or load them. This actually extends to `ptr::read` and `ptr::write`: they won't actually look at the pointer at all. As such we never need @@ -38,43 +38,49 @@ impl RawVec { // !0 is usize::MAX. This branch should be stripped at compile time. let cap = if mem::size_of::() == 0 { !0 } else { 0 }; - // Unique::dangling() doubles as "unallocated" and "zero-sized allocation" - RawVec { ptr: Unique::dangling(), cap: cap } + // `NonNull::dangling()` doubles as "unallocated" and "zero-sized allocation" + RawVec { + ptr: NonNull::dangling(), + cap: cap, + _marker: PhantomData, + } } fn grow(&mut self) { - unsafe { - let elem_size = mem::size_of::(); + // since we set the capacity to usize::MAX when T has size 0, + // getting to here necessarily means the Vec is overfull. + assert!(mem::size_of::() != 0, "capacity overflow"); - // since we set the capacity to usize::MAX when elem_size is - // 0, getting to here necessarily means the Vec is overfull. - assert!(elem_size != 0, "capacity overflow"); + let (new_cap, new_layout) = if self.cap == 0 { + (1, Layout::array::(1).unwrap()) + } else { + // This can't overflow because we ensure self.cap <= isize::MAX. + let new_cap = 2 * self.cap; - let (new_cap, ptr) = if self.cap == 0 { - let ptr = Global.allocate(Layout::array::(1).unwrap()); - (1, ptr) - } else { - let new_cap = 2 * self.cap; - let c: NonNull = self.ptr.into(); - let ptr = Global.grow(c.cast(), - Layout::array::(self.cap).unwrap(), - Layout::array::(new_cap).unwrap()); - (new_cap, ptr) - }; - - // If allocate or reallocate fail, oom - if ptr.is_err() { - handle_alloc_error(Layout::from_size_align_unchecked( - new_cap * elem_size, - mem::align_of::(), - )) - } + // `Layout::array` checks that the number of bytes is <= usize::MAX, + // but this is redundant since old_layout.size() <= isize::MAX, + // so the `unwrap` should never fail. + let new_layout = Layout::array::(new_cap).unwrap(); + (new_cap, new_layout) + }; - let ptr = ptr.unwrap(); + // Ensure that the new allocation doesn't exceed `isize::MAX` bytes. + assert!(new_layout.size() <= isize::MAX as usize, "Allocation too large"); - self.ptr = Unique::new_unchecked(ptr.as_ptr() as *mut _); - self.cap = new_cap; - } + let new_ptr = if self.cap == 0 { + unsafe { alloc::alloc(new_layout) } + } else { + let old_layout = Layout::array::(self.cap).unwrap(); + let old_ptr = self.ptr.as_ptr() as *mut u8; + unsafe { alloc::realloc(old_ptr, old_layout, new_layout.size()) } + }; + + // If allocation fails, `new_ptr` will be null, in which case we abort. + self.ptr = match NonNull::new(new_ptr as *mut T) { + Some(p) => p, + None => alloc::handle_alloc_error(new_layout), + }; + self.cap = new_cap; } } @@ -82,12 +88,12 @@ impl Drop for RawVec { fn drop(&mut self) { let elem_size = mem::size_of::(); - // don't free zero-sized allocations, as they were never allocated. if self.cap != 0 && elem_size != 0 { unsafe { - let c: NonNull = self.ptr.into(); - Global.deallocate(c.cast(), - Layout::array::(self.cap).unwrap()); + alloc::dealloc( + self.ptr.as_ptr() as *mut u8, + Layout::array::(self.cap).unwrap(), + ); } } } @@ -117,7 +123,7 @@ impl RawValIter { } else if slice.len() == 0 { slice.as_ptr() } else { - slice.as_ptr().offset(slice.len() as isize) + slice.as_ptr().add(slice.len()) }, } } diff --git a/src/doc/nomicon/src/vec.md b/src/doc/nomicon/src/vec.md index d2d47e981a..b033863313 100644 --- a/src/doc/nomicon/src/vec.md +++ b/src/doc/nomicon/src/vec.md @@ -1,16 +1,10 @@ # Example: Implementing Vec To bring everything together, we're going to write `std::Vec` from scratch. -Because all the best tools for writing unsafe code are unstable, this -project will only work on nightly (as of Rust 1.9.0). With the exception of the -allocator API, much of the unstable code we'll use is expected to be stabilized -in a similar form as it is today. - -However we will generally try to avoid unstable code where possible. In -particular we won't use any intrinsics that could make a code a little -bit nicer or efficient because intrinsics are permanently unstable. Although -many intrinsics *do* become stabilized elsewhere (`std::ptr` and `std::mem` -consist of many intrinsics). +We will limit ourselves to stable Rust. In particular we won't use any +intrinsics that could make our code a little bit nicer or efficient because +intrinsics are permanently unstable. Although many intrinsics *do* become +stabilized elsewhere (`std::ptr` and `std::mem` consist of many intrinsics). Ultimately this means our implementation may not take advantage of all possible optimizations, though it will be by no means *naive*. We will diff --git a/src/doc/nomicon/src/working-with-unsafe.md b/src/doc/nomicon/src/working-with-unsafe.md index c29a75a531..935ce580c2 100644 --- a/src/doc/nomicon/src/working-with-unsafe.md +++ b/src/doc/nomicon/src/working-with-unsafe.md @@ -71,7 +71,7 @@ impl Vec { self.reallocate(); } unsafe { - ptr::write(self.ptr.offset(self.len as isize), elem); + ptr::write(self.ptr.add(self.len), elem); self.len += 1; } } diff --git a/src/doc/reference/src/attributes.md b/src/doc/reference/src/attributes.md index d6ce2531b8..857cd7d720 100644 --- a/src/doc/reference/src/attributes.md +++ b/src/doc/reference/src/attributes.md @@ -13,7 +13,7 @@ > > _AttrInput_ :\ >       [_DelimTokenTree_]\ ->    | `=` [_LiteralExpression_]_without suffix_ +>    | `=` [_Expression_] An _attribute_ is a general, free-form metadatum that is interpreted according to name, convention, language, and compiler version. Attributes are modeled @@ -26,7 +26,7 @@ the bang after the hash, apply to the thing that follows the attribute. The attribute consists of a path to the attribute, followed by an optional delimited token tree whose interpretation is defined by the attribute. Attributes other than macro attributes also allow the input to be an equals -sign (`=`) followed by a literal expression. See the [meta item +sign (`=`) followed by an expression. See the [meta item syntax](#meta-item-attribute-syntax) below for more details. Attributes can be classified into the following kinds: @@ -94,7 +94,7 @@ attributes]. It has the following grammar: > **Syntax**\ > _MetaItem_ :\ >       [_SimplePath_]\ ->    | [_SimplePath_] `=` [_LiteralExpression_]_without suffix_\ +>    | [_SimplePath_] `=` [_Expression_]\ >    | [_SimplePath_] `(` _MetaSeq_? `)` > > _MetaSeq_ :\ @@ -102,10 +102,34 @@ attributes]. It has the following grammar: > > _MetaItemInner_ :\ >       _MetaItem_\ ->    | [_LiteralExpression_]_without suffix_ +>    | [_Expression_] + +Expressions in meta items must macro-expand to literal expressions, which must not +include integer or float type suffixes. Expressions which are not literal expressions +will be syntactically accepted (and can be passed to proc-macros), but will be rejected after parsing. + +Note that if the attribute appears within another macro, it will be expanded +after that outer macro. For example, the following code will expand the +`Serialize` proc-macro first, which must preserve the `include_str!` call in +order for it to be expanded: + +```rust ignore +#[derive(Serialize)] +struct Foo { + #[doc = include_str!("x.md")] + x: u32 +} +``` -Literal expressions in meta items must not include integer or float type -suffixes. +Additionally, macros in attributes will be expanded only after all other attributes applied to the item: + +```rust ignore +#[macro_attr1] // expanded first +#[doc = mac!()] // `mac!` is expanded fourth. +#[macro_attr2] // expanded second +#[derive(MacroDerive1, MacroDerive2)] // expanded third +fn foo() {} +``` Various built-in attributes use different subsets of the meta item syntax to specify their inputs. The following grammar rules show some commonly used @@ -255,7 +279,7 @@ The following is an index of all built-in attributes. [The Rustdoc Book]: ../rustdoc/the-doc-attribute.html [The Unstable Book]: ../unstable-book/index.html [_DelimTokenTree_]: macros.md -[_LiteralExpression_]: expressions/literal-expr.md +[_Expression_]: expressions.md [_SimplePath_]: paths.md#simple-paths [`allow`]: attributes/diagnostics.md#lint-check-attributes [`automatically_derived`]: attributes/derive.md#the-automatically_derived-attribute diff --git a/src/doc/reference/src/attributes/limits.md b/src/doc/reference/src/attributes/limits.md index 446c33093a..517637e5dc 100644 --- a/src/doc/reference/src/attributes/limits.md +++ b/src/doc/reference/src/attributes/limits.md @@ -42,19 +42,14 @@ to set the limit based on the number of type substitutions. > Note: The default in `rustc` is 1048576. - - -```rust,compile_fail,ignore -#![type_length_limit = "8"] +```rust,compile_fail +#![type_length_limit = "4"] fn f(x: T) {} // This fails to compile because monomorphizing to -// `f::<(i32, i32, i32, i32, i32, i32, i32, i32, i32)>>` requires more -// than 8 type elements. -f((1, 2, 3, 4, 5, 6, 7, 8, 9)); +// `f::<((((i32,), i32), i32), i32)>` requires more than 4 type elements. +f(((((1,), 2), 3), 4)); ``` [_MetaNameValueStr_]: ../attributes.md#meta-item-attribute-syntax diff --git a/src/doc/reference/src/conditional-compilation.md b/src/doc/reference/src/conditional-compilation.md index 0ec20f3f65..8085cbfab9 100644 --- a/src/doc/reference/src/conditional-compilation.md +++ b/src/doc/reference/src/conditional-compilation.md @@ -132,12 +132,15 @@ Example values: ### `target_family` -Key-value option set at most once with the target's operating system value. +Key-value option providing a more generic description of a target, such as the family of the +operating systems or architectures that the target generally falls into. Any number of +`target_family` key-value pairs can be set. Example values: * `"unix"` * `"windows"` +* `"wasm"` ### `unix` and `windows` diff --git a/src/doc/reference/src/expressions/operator-expr.md b/src/doc/reference/src/expressions/operator-expr.md index 75f59636df..be7c5f12e5 100644 --- a/src/doc/reference/src/expressions/operator-expr.md +++ b/src/doc/reference/src/expressions/operator-expr.md @@ -337,35 +337,29 @@ reference types and `mut` or `const` in pointer types. ### Semantics -* Numeric cast - * Casting between two integers of the same size (e.g. i32 -> u32) is a no-op - * Casting from a larger integer to a smaller integer (e.g. u32 -> u8) will - truncate - * Casting from a smaller integer to a larger integer (e.g. u8 -> u32) will - * zero-extend if the source is unsigned - * sign-extend if the source is signed - * Casting from a float to an integer will round the float towards zero - * `NaN` will return `0` - * Values larger than the maximum integer value will saturate to the - maximum value of the integer type. - * Values smaller than the minimum integer value will saturate to the - minimum value of the integer type. - * Casting from an integer to float will produce the closest possible float \* - * if necessary, rounding is according to `roundTiesToEven` mode \*\*\* - * on overflow, infinity (of the same sign as the input) is produced - * note: with the current set of numeric types, overflow can only happen - on `u128 as f32` for values greater or equal to `f32::MAX + (0.5 ULP)` - * Casting from an f32 to an f64 is perfect and lossless - * Casting from an f64 to an f32 will produce the closest possible f32 \*\* - * if necessary, rounding is according to `roundTiesToEven` mode \*\*\* - * on overflow, infinity (of the same sign as the input) is produced -* Enum cast - * Casts an enum to its discriminant, then uses a numeric cast if needed. -* Primitive to integer cast - * `false` casts to `0`, `true` casts to `1` - * `char` casts to the value of the code point, then uses a numeric cast if needed. -* `u8` to `char` cast - * Casts to the `char` with the corresponding code point. +#### Numeric cast + +* Casting between two integers of the same size (e.g. i32 -> u32) is a no-op +* Casting from a larger integer to a smaller integer (e.g. u32 -> u8) will + truncate +* Casting from a smaller integer to a larger integer (e.g. u8 -> u32) will + * zero-extend if the source is unsigned + * sign-extend if the source is signed +* Casting from a float to an integer will round the float towards zero + * `NaN` will return `0` + * Values larger than the maximum integer value will saturate to the + maximum value of the integer type. + * Values smaller than the minimum integer value will saturate to the + minimum value of the integer type. +* Casting from an integer to float will produce the closest possible float \* + * if necessary, rounding is according to `roundTiesToEven` mode \*\*\* + * on overflow, infinity (of the same sign as the input) is produced + * note: with the current set of numeric types, overflow can only happen + on `u128 as f32` for values greater or equal to `f32::MAX + (0.5 ULP)` +* Casting from an f32 to an f64 is perfect and lossless +* Casting from an f64 to an f32 will produce the closest possible f32 \*\* + * if necessary, rounding is according to `roundTiesToEven` mode \*\*\* + * on overflow, infinity (of the same sign as the input) is produced \* if integer-to-float casts with this rounding mode and overflow behavior are not supported natively by the hardware, these casts will likely be slower than @@ -379,6 +373,51 @@ expected. number, preferring the one with an even least significant digit if exactly halfway between two floating point numbers. +#### Enum cast + +Casts an enum to its discriminant, then uses a numeric cast if needed. + +#### Primitive to integer cast + +* `false` casts to `0`, `true` casts to `1` +* `char` casts to the value of the code point, then uses a numeric cast if needed. + +#### `u8` to `char` cast + +Casts to the `char` with the corresponding code point. + +#### Pointer to address cast + +Casting from a raw pointer to an integer produces the machine address of the referenced memory. +If the integer type is smaller than the pointer type, the address may be truncated; using `usize` avoids this. + +#### Address to pointer cast + +Casting from an integer to a raw pointer interprets the integer as a memory address and produces a pointer referencing that memory. + +

+ +Warning: +This interacts with the Rust memory model, which is still under development. +A pointer obtained from this cast may suffer additional restrictions even if it is bitwise equal to a valid pointer. +Dereferencing such a pointer may be [undefined behavior] if aliasing rules are not followed. + +
+ +A trivial example of sound address arithmetic: + +```rust +let mut values: [i32; 2] = [1, 2]; +let p1: *mut i32 = values.as_mut_ptr(); +let first_address = p1 as usize; +let second_address = first_address + 4; // 4 == size_of::() +let p2 = second_address as *mut i32; +unsafe { + *p2 += 1; +} +assert_eq!(values[1], 3); +``` + ## Assignment expressions > **Syntax**\ @@ -470,6 +509,8 @@ fn example() { } ``` +Like assignment expressions, compound assignment expressions always produce [the unit value][unit]. +
Warning: The evaluation order of operands swaps depending on the types of the operands: @@ -487,6 +528,7 @@ See [this test] for an example of using this dependency. [logical xor]: ../types/boolean.md#logical-xor [mutable]: ../expressions.md#mutability [place expression]: ../expressions.md#place-expressions-and-value-expressions +[undefined behavior]: ../behavior-considered-undefined.md [unit]: ../types/tuple.md [value expression]: ../expressions.md#place-expressions-and-value-expressions [temporary value]: ../expressions.md#temporaries diff --git a/src/doc/reference/src/glossary.md b/src/doc/reference/src/glossary.md index f88ba2f0fd..e345634a84 100644 --- a/src/doc/reference/src/glossary.md +++ b/src/doc/reference/src/glossary.md @@ -50,6 +50,16 @@ Combinators are higher-order functions that apply only functions and earlier defined combinators to provide a result from its arguments. They can be used to manage control flow in a modular fashion. +### Crate + +A crate is the unit of compilation and linking. There are different [types of +crates], such as libraries or executables. Crates may link and refer to other +library crates, called external crates. A crate has a self-contained tree of +[modules], starting from an unnamed root module called the crate root. [Items] +may be made visible to other crates by marking them as public in the crate +root, including through [paths] of public modules. +[More][crate]. + ### Dispatch Dispatch is the mechanism to determine which specific version of code is actually @@ -130,6 +140,14 @@ This is not affected by applied type arguments. `struct Foo` is considered local `Vec` is not. `LocalType` is local. Type aliases do not affect locality. +### Module + +A module is a container for zero or more [items]. Modules are organized in a +tree, starting from an unnamed module at the root called the crate root or the +root module. [Paths] may be used to refer to items from other modules, which +may be restricted by [visibility rules]. +[More][modules] + ### Name A [*name*] is an [identifier] or [lifetime or loop label] that refers to an @@ -274,6 +292,7 @@ example of an uninhabited type is the [never type] `!`, or an enum with no varia [associated item]: #associated-item [attributes]: attributes.md [*entity*]: names.md +[crate]: crates-and-source-files.md [enums]: items/enumerations.md [fields]: expressions/field-expr.md [free item]: #free-item @@ -291,6 +310,7 @@ example of an uninhabited type is the [never type] `!`, or an enum with no varia [lints]: attributes/diagnostics.md#lint-check-attributes [loop labels]: tokens.md#lifetimes-and-loop-labels [method]: items/associated-items.md#methods +[modules]: items/modules.md [*Name resolution*]: names/name-resolution.md [*name*]: names.md [*namespace*]: names/namespaces.md @@ -303,7 +323,9 @@ example of an uninhabited type is the [never type] `!`, or an enum with no varia [trait objects]: types/trait-object.md [traits]: items/traits.md [turbofish test]: https://github.com/rust-lang/rust/blob/master/src/test/ui/bastion-of-the-turbofish.rs +[types of crates]: linkage.md [types]: types.md [undefined-behavior]: behavior-considered-undefined.md [unions]: items/unions.md [variable bindings]: patterns.md +[visibility rules]: visibility-and-privacy.md diff --git a/src/doc/reference/src/items/generics.md b/src/doc/reference/src/items/generics.md index 946b5e9a50..686b6661dd 100644 --- a/src/doc/reference/src/items/generics.md +++ b/src/doc/reference/src/items/generics.md @@ -221,10 +221,7 @@ fn generic() { >    [_Lifetime_] `:` [_LifetimeBounds_] > > _TypeBoundWhereClauseItem_ :\ ->    _ForLifetimes_? [_Type_] `:` [_TypeParamBounds_]? -> -> _ForLifetimes_ :\ ->    `for` [_GenericParams_](#generic-parameters) +>    [_ForLifetimes_]? [_Type_] `:` [_TypeParamBounds_]? *Where clauses* provide another way to specify bounds on type and lifetime parameters as well as a way to specify bounds on types that aren't type @@ -277,6 +274,7 @@ struct Foo<#[my_flexible_clone(unbounded)] H> { [IDENTIFIER]: ../identifiers.md [LIFETIME_OR_LABEL]: ../tokens.md#lifetimes-and-loop-labels +[_ForLifetimes_]: ../trait-bounds.md#higher-ranked-trait-bounds [_LifetimeParam_]: #generic-parameters [_LifetimeBounds_]: ../trait-bounds.md [_Lifetime_]: ../trait-bounds.md diff --git a/src/doc/reference/src/items/implementations.md b/src/doc/reference/src/items/implementations.md index f1baad74e0..cb3598e8e6 100644 --- a/src/doc/reference/src/items/implementations.md +++ b/src/doc/reference/src/items/implementations.md @@ -111,7 +111,7 @@ keyword. # struct Point {x: f64, y: f64}; # type Surface = i32; # struct BoundingBox {x: f64, y: f64, width: f64, height: f64}; -# trait Shape { fn draw(&self, Surface); fn bounding_box(&self) -> BoundingBox; } +# trait Shape { fn draw(&self, s: Surface); fn bounding_box(&self) -> BoundingBox; } # fn do_draw_circle(s: Surface, c: Circle) { } struct Circle { radius: f64, diff --git a/src/doc/reference/src/items/traits.md b/src/doc/reference/src/items/traits.md index e5f766f5f0..a4b68815b0 100644 --- a/src/doc/reference/src/items/traits.md +++ b/src/doc/reference/src/items/traits.md @@ -81,9 +81,9 @@ Object safe traits can be the base trait of a [trait object]. A trait is * [`Rc`] * [`Arc`] * [`Pin

`] where `P` is one of the types above - * Does not have a `where Self: Sized` bound (reciever type of `Self` (i.e. `self`) implies this). + * Does not have a `where Self: Sized` bound (receiver type of `Self` (i.e. `self`) implies this). * Explicitly non-dispatchable functions require: - * Have a `where Self: Sized` bound (reciever type of `Self` (i.e. `self`) implies this). + * Have a `where Self: Sized` bound (receiver type of `Self` (i.e. `self`) implies this). ```rust # use std::rc::Rc; diff --git a/src/doc/reference/src/macros-by-example.md b/src/doc/reference/src/macros-by-example.md index 2682059dda..7d2b447096 100644 --- a/src/doc/reference/src/macros-by-example.md +++ b/src/doc/reference/src/macros-by-example.md @@ -28,7 +28,7 @@ > > _MacroFragSpec_ :\ >       `block` | `expr` | `ident` | `item` | `lifetime` | `literal`\ ->    | `meta` | `pat` | `path` | `stmt` | `tt` | `ty` | `vis` +>    | `meta` | `pat` | `pat_param` | `path` | `stmt` | `tt` | `ty` | `vis` > > _MacroRepSep_ :\ >    [_Token_]_except delimiters and repetition operators_ @@ -122,7 +122,8 @@ fragment specifiers are: * `block`: a [_BlockExpression_] * `stmt`: a [_Statement_] without the trailing semicolon (except for item statements that require semicolons) - * `pat`: a [_PatternNoTopAlt_] + * `pat_param`: a [_PatternNoTopAlt_] + * `pat`: equivalent to `pat_param` * `expr`: an [_Expression_] * `ty`: a [_Type_] * `ident`: an [IDENTIFIER_OR_KEYWORD] @@ -450,7 +451,7 @@ Matchers like `$i:expr,` or `$i:expr;` would be legal, however, because `,` and `;` are legal expression separators. The specific rules are: * `expr` and `stmt` may only be followed by one of: `=>`, `,`, or `;`. - * `pat` may only be followed by one of: `=>`, `,`, `=`, `|`, `if`, or `in`. + * `pat` and `pat_param` may only be followed by one of: `=>`, `,`, `=`, `|`, `if`, or `in`. * `path` and `ty` may only be followed by one of: `=>`, `,`, `=`, `|`, `;`, `:`, `>`, `>>`, `[`, `{`, `as`, `where`, or a macro variable of `block` fragment specifier. diff --git a/src/doc/reference/src/trait-bounds.md b/src/doc/reference/src/trait-bounds.md index fe677a9a1d..4d1d5d345c 100644 --- a/src/doc/reference/src/trait-bounds.md +++ b/src/doc/reference/src/trait-bounds.md @@ -100,6 +100,9 @@ fn f<'a, 'b>(x: &'a i32, mut y: &'b i32) where 'a: 'b { ## Higher-ranked trait bounds +> _ForLifetimes_ :\ +>    `for` [_GenericParams_] + Type bounds may be *higher ranked* over lifetimes. These bounds specify a bound is true *for all* lifetimes. For example, a bound such as `for<'a> &'a T: PartialEq` would require an implementation like @@ -137,6 +140,7 @@ fn call_on_ref_zero(f: F) where F: for<'a> Fn(&'a i32) { ``` [LIFETIME_OR_LABEL]: tokens.md#lifetimes-and-loop-labels +[_GenericParams_]: items/generics.md [_TypePath_]: paths.md#paths-in-types [`Sized`]: special-types-and-traits.md#sized diff --git a/src/doc/reference/src/types/function-pointer.md b/src/doc/reference/src/types/function-pointer.md index 2059cf48bc..a51f761352 100644 --- a/src/doc/reference/src/types/function-pointer.md +++ b/src/doc/reference/src/types/function-pointer.md @@ -54,7 +54,7 @@ restrictions as [regular function parameters]. [IDENTIFIER]: ../identifiers.md [_Abi_]: ../items/functions.md -[_ForLifetimes_]: ../items/generics.md#where-clauses +[_ForLifetimes_]: ../trait-bounds.md#higher-ranked-trait-bounds [_TypeNoBounds_]: ../types.md#type-expressions [_Type_]: ../types.md#type-expressions [_OuterAttribute_]: ../attributes.md diff --git a/src/doc/reference/src/types/impl-trait.md b/src/doc/reference/src/types/impl-trait.md index d7beeda7a0..413f999f85 100644 --- a/src/doc/reference/src/types/impl-trait.md +++ b/src/doc/reference/src/types/impl-trait.md @@ -5,33 +5,120 @@ > > _ImplTraitTypeOneBound_ : `impl` [_TraitBound_] -## Anonymous type parameters +`impl Trait` provides ways to specify unnamed but concrete types that +implement a specific trait. +It can appear in two sorts of places: argument position (where it can act as an anonymous type parameter to functions), and return position (where it can act as an abstract return type). + +```rust +trait Trait {} +# impl Trait for () {} -> Note: This section is a placeholder for more comprehensive reference -> material. +// argument position: anonymous type parameter +fn foo(arg: impl Trait) { +} + +// return position: abstract return type +fn bar() -> impl Trait { +} +``` +## Anonymous type parameters > Note: This is often called "impl Trait in argument position". +(The term "parameter" is more correct here, but "impl Trait in argument position" is the phrasing used during the development of this feature, and it remains in parts of the implementation.) -Functions can declare an argument to be an anonymous type parameter where the -callee must provide a type that has the bounds declared by the anonymous type -parameter and the function can only use the methods available by the trait -bounds of the anonymous type parameter. +Functions can use `impl` followed by a set of trait bounds to declare a parameter as having an anonymous type. +The caller must provide a type that satisfies the bounds declared by the anonymous type parameter, and the function can only use the methods available through the trait bounds of the anonymous type parameter. -They are written as `impl` followed by a set of trait bounds. +For example, these two forms are almost equivalent: -## Abstract return types +```rust,ignore +trait Trait {} + +// generic type parameter +fn foo(arg: T) { +} + +// impl Trait in argument position +fn foo(arg: impl Trait) { +} +``` -> Note: This section is a placeholder for more comprehensive reference -> material. +That is, `impl Trait` in argument position is syntactic sugar for a generic type parameter like ``, except that the type is anonymous and doesn't appear in the [_GenericParams_] list. + +> **Note:** +> For function parameters, generic type parameters and `impl Trait` are not exactly equivalent. +> With a generic parameter such as ``, the caller has the option to explicitly specify the generic argument for `T` at the call site using [_GenericArgs_], for example, `foo::(1)`. +> If `impl Trait` is the type of *any* function parameter, then the caller can't ever provide any generic arguments when calling that function. +This includes generic arguments for the return type or any const generics. +> +> Therefore, changing the function signature from either one to the other can constitute a breaking change for the callers of a function. + +## Abstract return types > Note: This is often called "impl Trait in return position". -Functions, except for associated trait functions, can return an abstract -return type. These types stand in for another concrete type where the -use-site may only use the trait methods declared by the trait bounds of the -type. +Functions can use `impl Trait` to return an abstract return type. +These types stand in for another concrete type where the caller may only use the methods declared by the specified `Trait`. +Each possible return value from the function must resolve to the same concrete type. + +`impl Trait` in return position allows a function to return an unboxed abstract type. +This is particularly useful with [closures] and iterators. +For example, closures have a unique, un-writable type. +Previously, the only way to return a closure from a function was to use a [trait object]: + +```rust +fn returns_closure() -> Box i32> { + Box::new(|x| x + 1) +} +``` + +This could incur performance penalties from heap allocation and dynamic dispatch. +It wasn't possible to fully specify the type of the closure, only to use the `Fn` trait. +That means that the trait object is necessary. +However, with `impl Trait`, it is possible to write this more simply: + +```rust +fn returns_closure() -> impl Fn(i32) -> i32 { + |x| x + 1 +} +``` + +which also avoids the drawbacks of using a boxed trait object. + +Similarly, the concrete types of iterators could become very complex, incorporating the types of all previous iterators in a chain. +Returning `impl Iterator` means that a function only exposes the `Iterator` trait as a bound on its return type, instead of explicitly specifying all of the other iterator types involved. + +### Differences between generics and `impl Trait` in return position + +In argument position, `impl Trait` is very similar in semantics to a generic type parameter. +However, there are significant differences between the two in return position. +With `impl Trait`, unlike with a generic type parameter, the function chooses the return type, and the caller cannot choose the return type. + +The function: + +```rust,ignore +fn foo() -> T { +``` + +allows the caller to determine the return type, `T`, and the function returns that type. + +The function: + +```rust,ignore +fn foo() -> impl Trait { +``` + +doesn't allow the caller to determine the return type. +Instead, the function chooses the return type, but only promises that it will implement `Trait`. + +## Limitations -They are written as `impl` followed by a set of trait bounds. +`impl Trait` can only appear as a parameter or return type of a free or inherent function. +It cannot appear inside implementations of traits, nor can it be the type of a let binding or appear inside a type alias. +[closures]: closure.md +[_GenericArgs_]: ../paths.md#paths-in-expressions +[_GenericParams_]: ../items/generics.md [_TraitBound_]: ../trait-bounds.md +[trait object]: trait-object.md [_TypeParamBounds_]: ../trait-bounds.md diff --git a/src/doc/rust-by-example/src/cargo/test.md b/src/doc/rust-by-example/src/cargo/test.md index b4bc69d5c5..01884a09af 100644 --- a/src/doc/rust-by-example/src/cargo/test.md +++ b/src/doc/rust-by-example/src/cargo/test.md @@ -14,12 +14,19 @@ foo ├── Cargo.toml ├── src │ └── main.rs +│ └── lib.rs └── tests ├── my_test.rs └── my_other_test.rs ``` -Each file in `tests` is a separate integration test. +Each file in `tests` is a separate +[integration test](https://doc.rust-lang.org/book/ch11-03-test-organization.html#integration-tests), +i.e. a test that is meant to test your library as if it were being called from a dependent +crate. + +The [Testing][testing] chapter elaborates on the three different testing styles: +[Unit][unit_testing], [Doc][doc_testing], and [Integration][integration_testing]. `cargo` naturally provides an easy way to run all of your tests! @@ -64,5 +71,84 @@ test result: ok. 2 passed; 0 failed; 0 ignored; 0 measured; 2 filtered out ``` One word of caution: Cargo may run multiple tests concurrently, so make sure -that they don't race with each other. For example, if they all output to a -file, you should make them write to different files. +that they don't race with each other. + +One example of this concurrency causing issues is if two tests output to a +file, such as below: + +```rust +#[cfg(test)] +mod tests { + // Import the necessary modules + use std::fs::OpenOptions; + use std::io::Write; + + // This test writes to a file + #[test] + fn test_file() { + // Opens the file ferris.txt or creates one if it doesn't exist. + let mut file = OpenOptions::new() + .append(true) + .create(true) + .open("ferris.txt") + .expect("Failed to open ferris.txt"); + + // Print "Ferris" 5 times. + for _ in 0..5 { + file.write_all("Ferris\n".as_bytes()) + .expect("Could not write to ferris.txt"); + } + } + + // This test tries to write to the same file + #[test] + fn test_file_also() { + // Opens the file ferris.txt or creates one if it doesn't exist. + let mut file = OpenOptions::new() + .append(true) + .create(true) + .open("ferris.txt") + .expect("Failed to open ferris.txt"); + + // Print "Corro" 5 times. + for _ in 0..5 { + file.write_all("Corro\n".as_bytes()) + .expect("Could not write to ferris.txt"); + } + } +} +``` + +Although the intent is to get the following: +```shell +$ cat ferris.txt +Ferris +Ferris +Ferris +Ferris +Ferris +Corro +Corro +Corro +Corro +Corro +``` +What actually gets put into `ferris.txt` is this: +```shell +$ cargo test test_foo +Corro +Ferris +Corro +Ferris +Corro +Ferris +Corro +Ferris +Corro +Ferris +``` + +[testing]: ../testing.md +[unit_testing]: ../testing/unit_testing.md +[integration_testing]: ../testing/unit_testing.md +[doc_testing]: ../testing/doc_testing.md diff --git a/src/doc/rust-by-example/src/custom_types/structs.md b/src/doc/rust-by-example/src/custom_types/structs.md index 227a261fb5..90dac52b6f 100644 --- a/src/doc/rust-by-example/src/custom_types/structs.md +++ b/src/doc/rust-by-example/src/custom_types/structs.md @@ -59,7 +59,7 @@ fn main() { println!("second point: ({}, {})", bottom_right.x, bottom_right.y); // Destructure the point using a `let` binding - let Point { x: top_edge, y: left_edge } = point; + let Point { x: left_edge, y: top_edge } = point; let _rectangle = Rectangle { // struct instantiation is an expression too diff --git a/src/doc/rust-by-example/src/hello/print.md b/src/doc/rust-by-example/src/hello/print.md index e09f128ebc..89d2aa0de3 100644 --- a/src/doc/rust-by-example/src/hello/print.md +++ b/src/doc/rust-by-example/src/hello/print.md @@ -39,7 +39,7 @@ fn main() { println!("{number:>width$}", number=1, width=6); // You can pad numbers with extra zeroes. This will output "000001". - println!("{number:>0width$}", number=1, width=6); + println!("{number:0>width$}", number=1, width=6); // Rust even checks to make sure the correct number of arguments are // used. diff --git a/src/doc/rustc-dev-guide/ci/check-in.sh b/src/doc/rustc-dev-guide/ci/check-in.sh index 323b75d67b..4477789c76 100755 --- a/src/doc/rustc-dev-guide/ci/check-in.sh +++ b/src/doc/rustc-dev-guide/ci/check-in.sh @@ -5,7 +5,10 @@ set -eu # This is not a very smart script if [ $# != 2 ]; then echo "usage: $0 " - if [ $# = 1 ] ; then + if [ $# = 0 ]; then + echo "help: you can find the last check-in at" \ + "https://rust-lang.zulipchat.com/#narrow/stream/238009-t-compiler.2Fmeetings/search/wg-rustc-dev-guide" + elif [ $# = 1 ] ; then echo "help: you can find the number of PRs merged at" \ "https://github.com/rust-lang/rustc-dev-guide/pulls?q=is%3Apr+is%3Amerged+updated%3A%3E$1" fi diff --git a/src/doc/rustc-dev-guide/src/appendix/bibliography.md b/src/doc/rustc-dev-guide/src/appendix/bibliography.md index 0caed3fafb..27529314c5 100644 --- a/src/doc/rustc-dev-guide/src/appendix/bibliography.md +++ b/src/doc/rustc-dev-guide/src/appendix/bibliography.md @@ -11,7 +11,7 @@ Rust, as well as publications about Rust. * [Making ad-hoc polymorphism less ad hoc](https://dl.acm.org/doi/10.1145/75277.75283) * [Macros that work together](https://www.cs.utah.edu/plt/publications/jfp12-draft-fcdf.pdf) * [Traits: composable units of behavior](http://scg.unibe.ch/archive/papers/Scha03aTraits.pdf) -* [Alias burying](http://www.cs.uwm.edu/faculty/boyland/papers/unique-preprint.ps) - We tried something similar and abandoned it. +* [Alias burying](https://dl.acm.org/doi/10.1002/spe.370) - We tried something similar and abandoned it. * [External uniqueness is unique enough](http://www.cs.uu.nl/research/techreps/UU-CS-2002-048.html) * [Uniqueness and Reference Immutability for Safe Parallelism](https://research.microsoft.com/pubs/170528/msr-tr-2012-79.pdf) * [Region Based Memory Management](https://www.cs.ucla.edu/~palsberg/tba/papers/tofte-talpin-iandc97.pdf) diff --git a/src/doc/rustc-dev-guide/src/building/prerequisites.md b/src/doc/rustc-dev-guide/src/building/prerequisites.md index f7f47d40c5..be5d96ad9b 100644 --- a/src/doc/rustc-dev-guide/src/building/prerequisites.md +++ b/src/doc/rustc-dev-guide/src/building/prerequisites.md @@ -47,7 +47,7 @@ These are not so much requirements as _recommendations_: * ~15GB of free disk space (~25GB or more if doing incremental builds). * \>= 8GB RAM -* \>= 4 cores +* \>= 2 cores * Internet access Beefier machines will lead to much faster builds. If your machine is not very diff --git a/src/doc/rustc-dev-guide/src/compiler-team.md b/src/doc/rustc-dev-guide/src/compiler-team.md index 8cfcc5bb66..aac2634843 100644 --- a/src/doc/rustc-dev-guide/src/compiler-team.md +++ b/src/doc/rustc-dev-guide/src/compiler-team.md @@ -11,7 +11,7 @@ contributions to rustc and its design. Currently the compiler team chats in Zulip: -- Team chat occurs in the `t-compiler` stream on [the Zulip instance][zulip] +- Team chat occurs in the [`t-compiler`][zulip-t-compiler] stream on the Zulip instance - There are also a number of other associated Zulip streams, such as [`t-compiler/help`][zulip-help], where people can ask for help with rustc development, or [`t-compiler/meetings`][zulip-meetings], @@ -60,8 +60,8 @@ The meeting currently takes place on Thursdays at 10am Boston time (UTC-4 typically, but daylight savings time sometimes makes things complicated). - [procedure]: ./bug-fix-procedure.md +[zulip-t-compiler]: https://rust-lang.zulipchat.com/#narrow/stream/131828-t-compiler [zulip-help]: https://rust-lang.zulipchat.com/#narrow/stream/182449-t-compiler.2Fhelp [zulip-meetings]: https://rust-lang.zulipchat.com/#narrow/stream/238009-t-compiler.2Fmeetings diff --git a/src/doc/rustc-dev-guide/src/contributing.md b/src/doc/rustc-dev-guide/src/contributing.md index abe1bb8390..cf049015d9 100644 --- a/src/doc/rustc-dev-guide/src/contributing.md +++ b/src/doc/rustc-dev-guide/src/contributing.md @@ -147,8 +147,12 @@ rebase when bringing the latest changes from the master branch to your feature branch. Also, please make sure that fixup commits are squashed into other related commits with meaningful commit messages. -If you encounter merge conflicts, your PR will get marked as `S-waiting-on-author`. -When you resolve them, you should use `@rustbot` to mark it as `S-waiting-on-review`. +If you encounter merge conflicts or when a reviewer asks you to perform some +changes, your PR will get marked as `S-waiting-on-author`. When you resolve +them, you should use `@rustbot` to mark it as `S-waiting-on-review`: + + @rustbot label -S-waiting-on-author +S-waiting-on-review + See [this chapter][labeling] for more details. GitHub allows [closing issues using keywords][closing-keywords]. This feature @@ -165,7 +169,8 @@ the issue in question. As a developer to this repository, you don't have to treat the following external projects differently from other crates that are directly in this repo: -* Clippy +* [Clippy](https://github.com/rust-lang/rust-clippy) +* [rustfmt](https://github.com/rust-lang/rustfmt) They are just regular files and directories. This is in contrast to `submodule` dependencies (see below for those). Only tool authors will actually use any operations here. @@ -229,7 +234,6 @@ subtrees) actually needs to use `git subtree`. Currently building Rust will also build the following external projects: * [miri](https://github.com/rust-lang/miri) -* [rustfmt](https://github.com/rust-lang/rustfmt) * [rls](https://github.com/rust-lang/rls/) We allow breakage of these tools in the nightly channel. Maintainers of these @@ -254,29 +258,30 @@ before the PR is merged. #### Breaking Tools Built With The Compiler -Rust's build system builds a number of tools that make use of the -internals of the compiler. This includes -[RLS](https://github.com/rust-lang/rls) and -[rustfmt](https://github.com/rust-lang/rustfmt). If these tools -break because of your changes, you may run into a sort of "chicken and egg" -problem. These tools rely on the latest compiler to be built so you can't update -them to reflect your changes to the compiler until those changes are merged into -the compiler. At the same time, you can't get your changes merged into the compiler -because the rust-lang/rust build won't pass until those tools build and pass their -tests. - -That means that, in the default state, you can't update the compiler without first -fixing rustfmt, rls and the other tools that the compiler builds. - -Luckily, a feature was [added to Rust's build](https://github.com/rust-lang/rust/issues/45861) -to make all of this easy to handle. The idea is that we allow these tools to be "broken", -so that the rust-lang/rust build passes without trying to build them, then land the change -in the compiler, wait for a nightly, and go update the tools that you broke. Once you're done -and the tools are working again, you go back in the compiler and update the tools -so they can be distributed again. +Rust's build system builds a number of tools that make use of the internals of +the compiler and that are hosted in a separate repository, and included in Rust +via git submodules. This includes [RLS](https://github.com/rust-lang/rls) and +[Miri](https://github.com/rust-lang/Miri). If these tools break because of your +changes, you may run into a sort of "chicken and egg" problem. These tools rely +on the latest compiler to be built so you can't update them (in their own +repositories) to reflect your changes to the compiler until those changes are +merged into the compiler. At the same time, you can't get your changes merged +into the compiler because the rust-lang/rust build won't pass until those tools +build and pass their tests. + +Luckily, a feature was +[added to Rust's build](https://github.com/rust-lang/rust/issues/45861) to make +all of this easy to handle. The idea is that we allow these tools to be +"broken", so that the rust-lang/rust build passes without trying to build them, +then land the change in the compiler, and go update the tools that you +broke. Some tools will require waiting for a nightly release before this can +happen, while others use the builds uploaded after each bors merge and thus can +be updated immediately (check the tool's documentation for details). Once you're +done and the tools are working again, you go back in the compiler and update the +tools so they can be distributed again. This should avoid a bunch of synchronization dances and is also much easier on contributors as -there's no need to block on rls/rustfmt/other tools changes going upstream. +there's no need to block on rls/miri/other tools changes going upstream. Here are those same steps in detail: @@ -286,8 +291,8 @@ Here are those same steps in detail: from resetting to the original branch after you make your changes. If you need to [update any submodules to their latest versions](#updating-submodules), see the section of this file about that for more information. -2. (optional) Run `./x.py test src/tools/rustfmt` (substituting the submodule - that broke for `rustfmt`). Fix any errors in the submodule (and possibly others). +2. (optional) Run `./x.py test src/tools/rls` (substituting the submodule + that broke for `rls`). Fix any errors in the submodule (and possibly others). 3. (optional) Make commits for your changes and send them to upstream repositories as a PR. 4. (optional) Maintainers of these submodules will **not** merge the PR. The PR can't be merged because CI will be broken. You'll want to write a message on the PR referencing @@ -299,16 +304,16 @@ Here are those same steps in detail: #### Updating submodules -These instructions are specific to updating `rustfmt`, however they may apply +These instructions are specific to updating `rls`, however they may apply to the other submodules as well. Please help by improving these instructions if you find any discrepancies or special cases that need to be addressed. -To update the `rustfmt` submodule, start by running the appropriate +To update the `rls` submodule, start by running the appropriate [`git submodule` command](https://git-scm.com/book/en/v2/Git-Tools-Submodules). For example, to update to the latest commit on the remote master branch, you may want to run: ``` -git submodule update --remote src/tools/rustfmt +git submodule update --remote src/tools/rls ``` If you run `./x.py build` now, and you are lucky, it may just work. If you see an error message about patches that did not resolve to any crates, you will need @@ -316,10 +321,10 @@ to complete a few more steps which are outlined with their rationale below. *(This error may change in the future to include more information.)* ``` -error: failed to resolve patches for `https://github.com/rust-lang/rustfmt` +error: failed to resolve patches for `https://github.com/rust-lang/rls` Caused by: - patch for `rustfmt-nightly` in `https://github.com/rust-lang/rustfmt` did not resolve to any crates + patch for `rls` in `https://github.com/rust-lang/rls` did not resolve to any crates failed to run: ~/rust/build/x86_64-unknown-linux-gnu/stage0/bin/cargo build --manifest-path ~/rust/src/bootstrap/Cargo.toml ``` @@ -350,14 +355,14 @@ reveals what the problem is: > crates.io, but it's important to keep this in mind! This says that when we updated the submodule, the version number in our -`src/tools/rustfmt/Cargo.toml` changed. The new version is different from +`src/tools/rls/Cargo.toml` changed. The new version is different from the version in `Cargo.lock`, so the build can no longer continue. To resolve this, we need to update `Cargo.lock`. Luckily, cargo provides a command to do this easily. ``` -$ cargo update -p rustfmt-nightly +$ cargo update -p rls ``` This should change the version listed in `Cargo.lock` to the new version you updated diff --git a/src/doc/rustc-dev-guide/src/diagnostics.md b/src/doc/rustc-dev-guide/src/diagnostics.md index f4e71bbdc4..38683f98db 100644 --- a/src/doc/rustc-dev-guide/src/diagnostics.md +++ b/src/doc/rustc-dev-guide/src/diagnostics.md @@ -484,9 +484,13 @@ module. ### Declaring a lint The built-in compiler lints are defined in the [`rustc_lint`][builtin] -crate. +crate. Lints that need to be implemented in other crates are defined in +[`rustc_lint_defs`]. You should prefer to place lints in `rustc_lint` if +possible. One benefit is that it is close to the dependency root, so it can be +much faster to work on. [builtin]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_lint/index.html +[`rustc_lint_defs`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_lint_defs/index.html Every lint is implemented via a `struct` that implements the `LintPass` `trait` (you also implement one of the more specific lint pass traits, either diff --git a/src/doc/rustc-dev-guide/src/git.md b/src/doc/rustc-dev-guide/src/git.md index e1b0e02926..e10f1d7050 100644 --- a/src/doc/rustc-dev-guide/src/git.md +++ b/src/doc/rustc-dev-guide/src/git.md @@ -139,7 +139,6 @@ Changes not staged for commit: (use "git restore ..." to discard changes in working directory) modified: src/tools/cargo (new commits) modified: src/tools/rls (new commits) - modified: src/tools/rustfmt (new commits) no changes added to commit (use "git add" and/or "git commit -a") ``` @@ -347,9 +346,9 @@ you might want to get used to the main concepts of Git before reading this secti The `rust-lang/rust` repository uses [Git submodules] as a way to use other Rust projects from within the `rust` repo. Examples include Rust's fork of -`llvm-project` and many devtools such as `cargo`, `rust-analyzer` and `rustfmt`. +`llvm-project` and many devtools such as `cargo`, `rust-analyzer` and `rls`. -Those projects are developped and maintained in an separate Git (and GitHub) +Those projects are developed and maintained in an separate Git (and GitHub) repository, and they have their own Git history/commits, issue tracker and PRs. Submodules allow us to create some sort of embedded sub-repository inside the `rust` repository and use them like they were directories in the `rust` repository. diff --git a/src/doc/rustc-dev-guide/src/identifiers.md b/src/doc/rustc-dev-guide/src/identifiers.md index 37e6a1c929..50d0c76e2f 100644 --- a/src/doc/rustc-dev-guide/src/identifiers.md +++ b/src/doc/rustc-dev-guide/src/identifiers.md @@ -53,7 +53,7 @@ See the [HIR chapter][hir-map] for more detailed information. [`LocalDefId`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_hir/def_id/struct.LocalDefId.html [`HirId`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_hir/hir_id/struct.HirId.html [`BodyId`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_hir/struct.BodyId.html -[`CrateNum`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_hir/def_id/enum.CrateNum.html +[`CrateNum`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_hir/def_id/struct.CrateNum.html [`DefIndex`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_hir/def_id/struct.DefIndex.html [`Body`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_hir/struct.Body.html [Node]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_hir/hir/enum.Node.html diff --git a/src/doc/rustc-dev-guide/src/llvm-coverage-instrumentation.md b/src/doc/rustc-dev-guide/src/llvm-coverage-instrumentation.md index 8a044849f5..728805387a 100644 --- a/src/doc/rustc-dev-guide/src/llvm-coverage-instrumentation.md +++ b/src/doc/rustc-dev-guide/src/llvm-coverage-instrumentation.md @@ -28,11 +28,12 @@ them), and generate various reports for analysis, for example:
Detailed instructions and examples are documented in the -[Rust Unstable Book (under _source-based-code-coverage_)][unstable-book-sbcc]. +[Rust Unstable Book (under +_compiler-flags/instrument-coverage_)][unstable-book-instrument-coverage]. [llvm-instrprof-increment]: https://llvm.org/docs/LangRef.html#llvm-instrprof-increment-intrinsic -[Coverage Map]: https://llvm.org/docs/CoverageMappingFormat.html -[unstable-book-sbcc]: https://doc.rust-lang.org/nightly/unstable-book/compiler-flags/source-based-code-coverage.html +[coverage map]: https://llvm.org/docs/CoverageMappingFormat.html +[unstable-book-instrument-coverage]: https://doc.rust-lang.org/nightly/unstable-book/compiler-flags/instrument-coverage.html ## Rust symbol mangling @@ -82,7 +83,7 @@ a span of code ([`CodeRegion`][code-region]). It counts the number of times a branch is executed, and also specifies the exact location of that code span in the Rust source code. -Note that many of these `Coverage` statements will *not* be converted into +Note that many of these `Coverage` statements will _not_ be converted into physical counters (or any other executable instructions) in the final binary. Some of them will be (see `CoverageKind::`[`Counter`][counter-coverage-kind]), but other counters can be computed on the fly, when generating a coverage @@ -111,7 +112,7 @@ fn some_func(flag: bool) { In this example, four contiguous code regions are counted while only incrementing two counters. -CFG analysis is used to not only determine *where* the branches are, for +CFG analysis is used to not only determine _where_ the branches are, for conditional expressions like `if`, `else`, `match`, and `loop`, but also to determine where expressions can be used in place of physical counters. @@ -150,40 +151,41 @@ MIR `Statement` into some backend-specific action or instruction. match statement.kind { ... mir::StatementKind::Coverage(box ref coverage) => { - self.codegen_coverage(&mut bx, coverage.clone()); + self.codegen_coverage(&mut bx, coverage.clone(), statement.source_info.scope); bx } ``` - `codegen_coverage()` handles each `CoverageKind` as follows: -* For all `CoverageKind`s, Coverage data (counter ID, expression equation +- For all `CoverageKind`s, Coverage data (counter ID, expression equation and ID, and code regions) are passed to the backend's `Builder`, to populate data structures that will be used to generate the crate's "Coverage Map". (See the [`FunctionCoverage`][function-coverage] `struct`.) -* For `CoverageKind::Counter`s, an instruction is injected in the backend +- For `CoverageKind::Counter`s, an instruction is injected in the backend IR to increment the physical counter, by calling the `BuilderMethod` [`instrprof_increment()`][instrprof-increment]. ```rust - pub fn codegen_coverage(&self, bx: &mut Bx, coverage: Coverage) { + pub fn codegen_coverage(&self, bx: &mut Bx, coverage: Coverage, scope: SourceScope) { + ... + let instance = ... // the scoped instance (current or inlined function) let Coverage { kind, code_region } = coverage; match kind { CoverageKind::Counter { function_source_hash, id } => { - if let Some(code_region) = code_region { - bx.add_coverage_counter(self.instance, id, code_region); - } + ... + bx.add_coverage_counter(instance, id, code_region); ... bx.instrprof_increment(fn_name, hash, num_counters, index); } CoverageKind::Expression { id, lhs, op, rhs } => { - bx.add_coverage_counter_expression(self.instance, id, lhs, op, rhs, code_region); + bx.add_coverage_counter_expression(instance, id, lhs, op, rhs, code_region); } CoverageKind::Unreachable => { - ... + bx.add_coverage_unreachable( + instance, + code_region.expect(... ``` -_code snippet trimmed for brevity_ > The function name `instrprof_increment()` is taken from the LLVM intrinsic call of the same name ([`llvm.instrprof.increment`][llvm-instrprof-increment]), @@ -221,7 +223,7 @@ properly-configured variables in LLVM IR, according to very specific details of the [_LLVM Coverage Mapping Format_][coverage-mapping-format] (Version 4).[^llvm-and-covmap-versions] -[^llvm-and-covmap-versions]: The Rust compiler (as of +[^llvm-and-covmap-versions]: The Rust compiler (as of January 2021) supports _LLVM Coverage Mapping Format_ Version 4 (the most up-to-date version of the format, at the time of this writing) for improved compatibility with other LLVM-based compilers (like _Clang_), and to take @@ -233,13 +235,16 @@ instrument-coverage` will generate an error message. ```rust pub fn finalize<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>) { + ... + if !tcx.sess.instrument_coverage_except_unused_functions() { + add_unused_functions(cx); + } + let mut function_coverage_map = match cx.coverage_context() { Some(ctx) => ctx.take_function_coverage_map(), None => return, }; ... - add_unreachable_coverage(tcx, &mut function_coverage_map); - let mut mapgen = CoverageMapGenerator::new(); for (instance, function_coverage) in function_coverage_map { @@ -250,56 +255,51 @@ pub fn finalize<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>) { ``` _code snippet trimmed for brevity_ -One notable step, performed by `mapgen::finalize()` before processing the -`Instance`s and their `FunctionCoverage`s, is the call to -[`add_unreachable_functions()`][add-unreachable-coverage]. +One notable first step performed by `mapgen::finalize()` is the call to +[`add_unused_functions()`][add-unused-functions]: -When finalizing the coverage map, `FunctionCoverage` only has the `CodeRegion`s and counters for -the functions that went through codegen; such as public functions and "used" functions -(functions referenced by other "used" or public items). Any other functions (considered unused -or "Unreachable") were still parsed and processed through the MIR stage. +When finalizing the coverage map, `FunctionCoverage` only has the `CodeRegion`s +and counters for the functions that went through codegen; such as public +functions and "used" functions (functions referenced by other "used" or public +items). Any other functions (considered unused) were still parsed and processed +through the MIR stage. -The set of unreachable functions is computed via the set difference of all MIR -`DefId`s (`tcx` query `mir_keys`) minus the codegenned `DefId`s -(`tcx` query `collect_and_partition_mono_items`). `add_unreachable_functions()` -computes the set of unreachable functions, queries the `tcx` for the -previously-computed `CodeRegions`, for each unreachable MIR, and adds those code -regions to one of the non-generic codegenned functions (non-generic avoids -potentially injecting the unreachable coverage multiple times for multiple -instantiations). +The set of unused functions is computed via the set difference of all MIR +`DefId`s (`tcx` query `mir_keys`) minus the codegenned `DefId`s (`tcx` query +`codegened_and_inlined_items`). `add_unused_functions()` computes the set of +unused functions, queries the `tcx` for the previously-computed `CodeRegions`, +for each unused MIR, synthesizes an LLVM function (with no internal statements, +since it will not be called), and adds a new `FunctionCoverage`, with +`Unreachable` code regions. [compile-codegen-unit]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_codegen_llvm/base/fn.compile_codegen_unit.html [coverageinfo-finalize]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_codegen_llvm/context/struct.CodegenCx.html#method.coverageinfo_finalize [mapgen-finalize]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_codegen_llvm/coverageinfo/mapgen/fn.finalize.html [coverage-mapping-format]: https://llvm.org/docs/CoverageMappingFormat.html -[add-unreachable-coverage]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_codegen_llvm/coverageinfo/mapgen/fn.add_unreachable_coverage.html +[add-unused-functions]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_codegen_llvm/coverageinfo/mapgen/fn.add_unused_functions.html ## Testing LLVM Coverage Coverage instrumentation in the MIR is validated by a `mir-opt` test: [`instrument-coverage`][mir-opt-test]. -More complete testing of end-to-end coverage instrumentation and reports are done -in the `run-make-fulldeps` tests, with sample Rust programs (to be instrumented) -in the [`coverage`][coverage-test-samples] directory, and the actual tests and expected -results in [`coverage-reports`]. - -In addition to testing the final result, two intermediate results are also validated -to catch potential regression errors early: Minimum `CoverageSpan`s computed during -the `InstrumentCoverage` MIR pass are saved in `mir_dump` [Spanview][spanview-debugging] -files and compared to expected results in [`coverage-spanview`]. +More complete testing of end-to-end coverage instrumentation and reports are +done in the `run-make-fulldeps` tests, with sample Rust programs (to be +instrumented) in the [`coverage`][coverage-test-samples] directory, and the +actual tests and expected results in [`coverage-reports`]. -Finally, the [`coverage-llvmir`] test compares compiles a simple Rust program with -`-Z instrument-coverage` and compares the compiled program's LLVM IR to expected -LLVM IR instructions and structured data for a coverage-enabled program, including -various checks for Coverage Map-related metadata and the LLVM intrinsic calls to -increment the runtime counters. +Finally, the [`coverage-llvmir`] test compares compiles a simple Rust program +with `-Z instrument-coverage` and compares the compiled program's LLVM IR to +expected LLVM IR instructions and structured data for a coverage-enabled +program, including various checks for Coverage Map-related metadata and the LLVM +intrinsic calls to increment the runtime counters. Expected results for both the `mir-opt` tests and the `coverage*` tests under `run-make-fulldeps` can be refreshed by running: ```shell -$ ./x.py test src/test/ --blessed +$ ./x.py test mir-opt --blessed +$ ./x.py test src/test/run-make-fulldeps/coverage --blessed ``` [mir-opt-test]: https://github.com/rust-lang/rust/blob/master/src/test/mir-opt/instrument_coverage.rs @@ -396,16 +396,18 @@ contrast with the [`SimplifyCfg`][simplify-cfg] MIR pass, this step does not alter the MIR itself, because the `CoverageGraph` aggressively simplifies the CFG, and ignores nodes that are not relevant to coverage. For example: - * The BCB CFG ignores (excludes) branches considered not relevant + - The BCB CFG ignores (excludes) branches considered not relevant to the current coverage solution. It excludes unwind-related code[^78544] that is injected by the Rust compiler but has no physical source code to count, which allows a `Call`-terminated BasicBlock to be merged with its successor, within a single BCB. - * A `Goto`-terminated `BasicBlock` can be merged with its successor - ***as long as*** it has the only incoming edge to the successor `BasicBlock`. - * Some BasicBlock terminators support Rust-specific concerns--like borrow-checking--that are - not relevant to coverage analysis. `FalseUnwind`, for example, can be treated the same as - a `Goto` (potentially merged with its successor into the same BCB). + - A `Goto`-terminated `BasicBlock` can be merged with its successor + **_as long as_** it has the only incoming edge to the successor + `BasicBlock`. + - Some BasicBlock terminators support Rust-specific concerns--like + borrow-checking--that are not relevant to coverage analysis. `FalseUnwind`, + for example, can be treated the same as a `Goto` (potentially merged with + its successor into the same BCB). [^78544]: (Note, however, that Issue [#78544][rust-lang/rust#78544] considers providing future support for coverage of programs that intentionally @@ -448,24 +450,24 @@ directional edges (the arrows) leading from each node to its `successors()`. The nodes contain information in sections: 1. The gray header has a label showing the BCB ID (or _index_ for looking up -its `BasicCoverageBlockData`). + its `BasicCoverageBlockData`). 2. The first content section shows the assigned `Counter` or `Expression` for -each contiguous section of code. (There may be more than one `Expression` -incremented by the same `Counter` for discontiguous sections of code representing -the same sequential actions.) Note the code is represented by the line and -column ranges (for example: `52:28-52:33`, representing the original source -line 52, for columns 28-33). These are followed by the MIR `Statement` or -`Terminator` represented by that source range. (How these coverage regions -are determined is discussed in the following section.) + each contiguous section of code. (There may be more than one `Expression` + incremented by the same `Counter` for discontiguous sections of code + representing the same sequential actions.) Note the code is represented by + the line and column ranges (for example: `52:28-52:33`, representing the + original source line 52, for columns 28-33). These are followed by the MIR + `Statement` or `Terminator` represented by that source range. (How these + coverage regions are determined is discussed in the following section.) 3. The final section(s) show the MIR `BasicBlock`s (by ID/index and its -`TerminatorKind`) contained in this BCB. The last BCB is separated out because -its `successors()` determine the edges leading out of the BCB, and into -the `leading_bb()` (first `BasicBlock`) of each successor BCB. + `TerminatorKind`) contained in this BCB. The last BCB is separated out + because its `successors()` determine the edges leading out of the BCB, and + into the `leading_bb()` (first `BasicBlock`) of each successor BCB. Note, to find the `BasicCoverageBlock` from a final BCB `Terminator`'s successor `BasicBlock`, there is an index and helper -function--[`bcb_from_bb()`][bcb-from-bb]--to look up a `BasicCoverageBlock` from _any_ -contained `BasicBlock`. +function--[`bcb_from_bb()`][bcb-from-bb]--to look up a `BasicCoverageBlock` from +*any* contained `BasicBlock`. [directed-graph]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_data_structures/graph/trait.DirectedGraph.html [graph-traits]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_data_structures/graph/index.html#traits @@ -572,7 +574,7 @@ incoming edges. Given the following graph, for example, the count for In this situation, BCB node `B` may require an edge counter for its "edge from A", and that edge might be computed from an `Expression`, -`Counter(A) - Counter(C)`. But an expression for the BCB _node_ `B` +`Counter(A) - Counter(C)`. But an expression for the BCB _node_ `B` would be the sum of all incoming edges: ```text diff --git a/src/doc/rustc-dev-guide/src/miri.md b/src/doc/rustc-dev-guide/src/miri.md index fd091d565b..18cd6ed392 100644 --- a/src/doc/rustc-dev-guide/src/miri.md +++ b/src/doc/rustc-dev-guide/src/miri.md @@ -2,8 +2,13 @@ -Miri (**MIR** **I**nterpreter) is a virtual machine for executing MIR without +The Miri (**MIR** **I**nterpreter) engine is a virtual machine for executing MIR without compiling to machine code. It is usually invoked via `tcx.const_eval_*` functions. +In the following, we will refer to the Miri engine as just "Miri", but note that +there also is a stand-alone +[tool called "Miri"](https://github.com/rust-lang/miri/) that is based on the +engine (sometimes referred to as Miri-the-tool to disambiguate it from the +engine). If you start out with a constant: diff --git a/src/doc/rustc-dev-guide/src/queries/incremental-compilation-in-detail.md b/src/doc/rustc-dev-guide/src/queries/incremental-compilation-in-detail.md index f0c6dcacd6..f81e02d28f 100644 --- a/src/doc/rustc-dev-guide/src/queries/incremental-compilation-in-detail.md +++ b/src/doc/rustc-dev-guide/src/queries/incremental-compilation-in-detail.md @@ -433,6 +433,26 @@ respect to incremental compilation: the result of the query has changed. As a consequence anything depending on a `no_hash` query will always be re-executed. + Using `no_hash` for a query can make sense in two circumstances: + + - If the result of the query is very likely to change whenever one of its + inputs changes, e.g. a function like `|a, b, c| -> (a * b * c)`. In such + a case recomputing the query will always yield a red node if one of the + inputs is red so we can spare us the trouble and default to red immediately. + A counter example would be a function like `|a| -> (a == 42)` where the + result does not change for most changes of `a`. + + - If the result of a query is a big, monolithic collection (e.g. `index_hir`) + and there are "projection queries" reading from that collection + (e.g. `hir_owner`). In such a case the big collection will likely fulfill the + condition above (any changed input means recomputing the whole collection) + and the results of the projection queries will be hashed anyway. If we also + hashed the collection query it would mean that we effectively hash the same + data twice: once when hashing the collection and another time when hashing all + the projection query results. `no_hash` allows us to avoid that redundancy + and the projection queries act as a "firewall", shielding their dependents + from the unconditionally red `no_hash` node. + - `cache_on_disk_if` - This attribute is what determines which query results are persisted in the incremental compilation query result cache. The attribute takes an expression that allows per query invocation diff --git a/src/doc/rustc-dev-guide/src/query.md b/src/doc/rustc-dev-guide/src/query.md index 294d61fb3d..aa3a813283 100644 --- a/src/doc/rustc-dev-guide/src/query.md +++ b/src/doc/rustc-dev-guide/src/query.md @@ -155,10 +155,10 @@ providers**. Almost all **extern providers** wind up going through the from the crate metadata. But in some cases there are crates that provide queries for *both* local and external crates, in which case they define both a `provide` and a `provide_extern` function, through -[`provide_both`][ext_provide_both], that `rustc_driver` can invoke. +[`wasm_import_module_map`][wasm_import_module_map], that `rustc_driver` can invoke. [rustc_metadata]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_metadata/index.html -[ext_provide_both]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_codegen_llvm/attributes/fn.provide_both.html +[wasm_import_module_map]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_codegen_ssa/back/symbol_export/fn.wasm_import_module_map.html ### Adding a new kind of query diff --git a/src/doc/rustc-dev-guide/src/rustdoc-internals.md b/src/doc/rustc-dev-guide/src/rustdoc-internals.md index b2dc2f7fd1..7ae68cc21c 100644 --- a/src/doc/rustc-dev-guide/src/rustdoc-internals.md +++ b/src/doc/rustc-dev-guide/src/rustdoc-internals.md @@ -245,7 +245,7 @@ server. To test these features locally, you can run a local HTTP server, like this: ```bash -$ ./x.py doc library/std --stage 1 +$ ./x.py doc library/std # The documentation has been generated into `build/[YOUR ARCH]/doc`. $ python3 -m http.server -d build/[YOUR ARCH]/doc ``` diff --git a/src/doc/rustc-dev-guide/src/rustdoc.md b/src/doc/rustc-dev-guide/src/rustdoc.md index 88177ccba8..2123d2b748 100644 --- a/src/doc/rustc-dev-guide/src/rustdoc.md +++ b/src/doc/rustc-dev-guide/src/rustdoc.md @@ -42,7 +42,7 @@ does is call the `main()` that's in this crate's `lib.rs`, though.) * If you've used `rustup toolchain link local /path/to/build/$TARGET/stage1` previously, then after the previous build command, `cargo +local doc` will Just Work. -* Use `./x.py doc --stage 1 library/std` to use this rustdoc to generate the +* Use `./x.py doc library/std` to use this rustdoc to generate the standard library docs. * The completed docs will be available in `build/$TARGET/doc/std`, though the bundle is meant to be used as though you would copy out the `doc` folder to diff --git a/src/doc/rustc-dev-guide/src/thir.md b/src/doc/rustc-dev-guide/src/thir.md index b023eb2e2a..a1d3899079 100644 --- a/src/doc/rustc-dev-guide/src/thir.md +++ b/src/doc/rustc-dev-guide/src/thir.md @@ -34,11 +34,14 @@ But it has some other interesting features that distinguish it from the HIR: [HIR]: ./hir.md The THIR lives in [`rustc_mir_build::thir`][thir-docs]. To construct a [`thir::Expr`], -you can use the [`build_thir`] function, passing in the memory arena where the THIR +you can use the [`thir_body`] function, passing in the memory arena where the THIR will be allocated. Dropping this arena will result in the THIR being destroyed, which is useful to keep peak memory in check. Having a THIR representation of all bodies of a crate in memory at the same time would be very heavy. +You can get a debug representation of the THIR by passing the `-Zunpretty=thir-tree` flag +to `rustc`. + [thir-docs]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_mir_build/thir/index.html -[`thir::Expr`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_mir_build/thir/struct.Expr.html -[`build_thir`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_mir_build/thir/fn.build_thir.html +[`thir::Expr`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/thir/struct.Expr.html +[`thir_body`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/context/struct.TyCtxt.html#method.thir_body diff --git a/src/doc/rustc/src/SUMMARY.md b/src/doc/rustc/src/SUMMARY.md index b269aab981..bc1873b683 100644 --- a/src/doc/rustc/src/SUMMARY.md +++ b/src/doc/rustc/src/SUMMARY.md @@ -13,6 +13,7 @@ - [JSON Output](json.md) - [Tests](tests/index.md) - [Platform Support](platform-support.md) + - [aarch64-apple-ios-sim](platform-support/aarch64-apple-ios-sim.md) - [Target Tier Policy](target-tier-policy.md) - [Targets](targets/index.md) - [Built-in Targets](targets/built-in.md) diff --git a/src/doc/rustc/src/codegen-options/index.md b/src/doc/rustc/src/codegen-options/index.md index d9083a5c4a..05384117ac 100644 --- a/src/doc/rustc/src/codegen-options/index.md +++ b/src/doc/rustc/src/codegen-options/index.md @@ -149,8 +149,7 @@ values: * `y`, `yes`, `on`, or no value: Unwind tables are forced to be generated. * `n`, `no`, or `off`: Unwind tables are not forced to be generated. If unwind - tables are required by the target or `-C panic=unwind`, an error will be - emitted. + tables are required by the target an error will be emitted. The default if not specified depends on the target. @@ -161,9 +160,6 @@ to save information after compiling a crate to be reused when recompiling the crate, improving re-compile times. This takes a path to a directory where incremental files will be stored. -Note that this option currently does not take effect unless -`RUSTC_FORCE_INCREMENTAL=1` in the environment. - ## inline-threshold This option lets you set the default threshold for inlining a function. It @@ -237,6 +233,8 @@ flavor. Valid options are: * `ptx-linker`: use [`rust-ptx-linker`](https://github.com/denzp/rust-ptx-linker) for Nvidia NVPTX GPGPU support. +* `bpf-linker`: use + [`bpf-linker`](https://github.com/alessandrod/bpf-linker) for eBPF support. * `wasm-ld`: use the [`wasm-ld`](https://lld.llvm.org/WebAssembly.html) executable, a port of LLVM `lld` for WebAssembly. * `ld64.lld`: use the LLVM `lld` executable with the [`-flavor darwin` diff --git a/src/doc/rustc/src/platform-support.md b/src/doc/rustc/src/platform-support.md index 7f57d476aa..3225e95941 100644 --- a/src/doc/rustc/src/platform-support.md +++ b/src/doc/rustc/src/platform-support.md @@ -6,43 +6,38 @@ } -Support for different platforms are organized into three tiers, each with a -different set of guarantees. For more information on the policies for targets -at each tier, see the [Target Tier Policy](target-tier-policy.md). +Support for different platforms ("targets") are organized into three tiers, +each with a different set of guarantees. For more information on the policies +for targets at each tier, see the [Target Tier Policy](target-tier-policy.md). -Platforms are identified by their "target triple" which is the string to -inform the compiler what kind of output should be produced. The columns in the -tables below have the following meanings: +Targets are identified by their "target triple" which is the string to inform +the compiler what kind of output should be produced. -* std: - * ✓ indicates the full standard library is available. - * \* indicates the target only supports [`no_std`] development. - * ? indicates the standard library support is unknown or a work-in-progress. -* host: A ✓ indicates that `rustc` and `cargo` can run on the host platform. +## Tier 1 with Host Tools -[`no_std`]: https://rust-embedded.github.io/book/intro/no-std.html +Tier 1 targets can be thought of as "guaranteed to work". The Rust project +builds official binary releases for each tier 1 target, and automated testing +ensures that each tier 1 target builds and passes tests after each change. -## Tier 1 +Tier 1 targets with host tools additionally support running tools like `rustc` +and `cargo` natively on the target, and automated testing ensures that tests +pass for the host tools as well. This allows the target to be used as a +development platform, not just a compilation target. For the full requirements, +see [Tier 1 with Host Tools](target-tier-policy.md#tier-1-with-host-tools) in +the Target Tier Policy. -Tier 1 platforms can be thought of as "guaranteed to work". -Specifically they will each satisfy the following requirements: +All tier 1 targets with host tools support the full standard library. -* Official binary releases are provided for the platform. -* Automated testing is set up to run tests for the platform. -* Landing changes to the `rust-lang/rust` repository's master branch is gated - on tests passing. -* Documentation for how to use and how to build the platform is available. - -target | std | host | notes --------|-----|------|------- -`aarch64-unknown-linux-gnu` | ✓ | ✓ | ARM64 Linux (kernel 4.2, glibc 2.17+) [^missing-stack-probes] -`i686-pc-windows-gnu` | ✓ | ✓ | 32-bit MinGW (Windows 7+) -`i686-pc-windows-msvc` | ✓ | ✓ | 32-bit MSVC (Windows 7+) -`i686-unknown-linux-gnu` | ✓ | ✓ | 32-bit Linux (kernel 2.6.32+, glibc 2.11+) -`x86_64-apple-darwin` | ✓ | ✓ | 64-bit macOS (10.7+, Lion+) -`x86_64-pc-windows-gnu` | ✓ | ✓ | 64-bit MinGW (Windows 7+) -`x86_64-pc-windows-msvc` | ✓ | ✓ | 64-bit MSVC (Windows 7+) -`x86_64-unknown-linux-gnu` | ✓ | ✓ | 64-bit Linux (kernel 2.6.32+, glibc 2.11+) +target | notes +-------|------- +`aarch64-unknown-linux-gnu` | ARM64 Linux (kernel 4.2, glibc 2.17+) [^missing-stack-probes] +`i686-pc-windows-gnu` | 32-bit MinGW (Windows 7+) +`i686-pc-windows-msvc` | 32-bit MSVC (Windows 7+) +`i686-unknown-linux-gnu` | 32-bit Linux (kernel 2.6.32+, glibc 2.11+) +`x86_64-apple-darwin` | 64-bit macOS (10.7+, Lion+) +`x86_64-pc-windows-gnu` | 64-bit MinGW (Windows 7+) +`x86_64-pc-windows-msvc` | 64-bit MSVC (Windows 7+) +`x86_64-unknown-linux-gnu` | 64-bit Linux (kernel 2.6.32+, glibc 2.11+) [^missing-stack-probes]: Stack probes support is missing on `aarch64-unknown-linux-gnu`, but it's planned to be implemented in the near @@ -50,110 +45,158 @@ target | std | host | notes [77071]: https://github.com/rust-lang/rust/issues/77071 +## Tier 1 + +Tier 1 targets can be thought of as "guaranteed to work". The Rust project +builds official binary releases for each tier 1 target, and automated testing +ensures that each tier 1 target builds and passes tests after each change. For +the full requirements, see [Tier 1 target +policy](target-tier-policy.md#tier-1-target-policy) in the Target Tier Policy. + +At this time, all Tier 1 targets are [Tier 1 with Host +Tools](#tier-1-with-host-tools). + +## Tier 2 with Host Tools + +Tier 2 targets can be thought of as "guaranteed to build". The Rust project +builds official binary releases for each tier 2 target, and automated builds +ensure that each tier 2 target builds after each change. Automated tests are +not always run so it's not guaranteed to produce a working build, but tier 2 +targets often work to quite a good degree and patches are always welcome! + +Tier 2 targets with host tools additionally support running tools like `rustc` +and `cargo` natively on the target, and automated builds ensure that the host +tools build as well. This allows the target to be used as a development +platform, not just a compilation target. For the full requirements, see [Tier 2 +with Host Tools](target-tier-policy.md#tier-2-with-host-tools) in the Target +Tier Policy. + +All tier 2 targets with host tools support the full standard library. + +target | notes +-------|------- +`aarch64-apple-darwin` | ARM64 macOS (11.0+, Big Sur+) +`aarch64-pc-windows-msvc` | ARM64 Windows MSVC +`aarch64-unknown-linux-musl` | ARM64 Linux with MUSL +`arm-unknown-linux-gnueabi` | ARMv6 Linux (kernel 3.2, glibc 2.17) +`arm-unknown-linux-gnueabihf` | ARMv6 Linux, hardfloat (kernel 3.2, glibc 2.17) +`armv7-unknown-linux-gnueabihf` | ARMv7 Linux, hardfloat (kernel 3.2, glibc 2.17) +`mips-unknown-linux-gnu` | MIPS Linux (kernel 4.4, glibc 2.23) +`mips64-unknown-linux-gnuabi64` | MIPS64 Linux, n64 ABI (kernel 4.4, glibc 2.23) +`mips64el-unknown-linux-gnuabi64` | MIPS64 (LE) Linux, n64 ABI (kernel 4.4, glibc 2.23) +`mipsel-unknown-linux-gnu` | MIPS (LE) Linux (kernel 4.4, glibc 2.23) +`powerpc-unknown-linux-gnu` | PowerPC Linux (kernel 2.6.32, glibc 2.11) +`powerpc64-unknown-linux-gnu` | PPC64 Linux (kernel 2.6.32, glibc 2.11) +`powerpc64le-unknown-linux-gnu` | PPC64LE Linux (kernel 3.10, glibc 2.17) +`riscv64gc-unknown-linux-gnu` | RISC-V Linux (kernel 4.20, glibc 2.29) +`s390x-unknown-linux-gnu` | S390x Linux (kernel 2.6.32, glibc 2.11) +`x86_64-unknown-freebsd` | 64-bit FreeBSD +`x86_64-unknown-illumos` | illumos +`x86_64-unknown-linux-musl` | 64-bit Linux with MUSL +`x86_64-unknown-netbsd` | NetBSD/amd64 + ## Tier 2 -Tier 2 platforms can be thought of as "guaranteed to build". Automated tests -are not run so it's not guaranteed to produce a working build, but platforms -often work to quite a good degree and patches are always welcome! -Specifically, these platforms are required to have each of the following: +Tier 2 targets can be thought of as "guaranteed to build". The Rust project +builds official binary releases for each tier 2 target, and automated builds +ensure that each tier 2 target builds after each change. Automated tests are +not always run so it's not guaranteed to produce a working build, but tier 2 +targets often work to quite a good degree and patches are always welcome! For +the full requirements, see [Tier 2 target +policy](target-tier-policy.md#tier-2-target-policy) in the Target Tier Policy. -* Official binary releases are provided for the platform. -* Automated building is set up, but may not be running tests. -* Landing changes to the `rust-lang/rust` repository's master branch is gated on - platforms **building**. For some platforms only the standard library is - compiled, but for others `rustc` and `cargo` are too. +The `std` column in the table below has the following meanings: -target | std | host | notes --------|-----|------|------- -`aarch64-apple-darwin` | ✓ | ✓ | ARM64 macOS (11.0+, Big Sur+) -`aarch64-apple-ios` | ✓ | | ARM64 iOS -`aarch64-fuchsia` | ✓ | | ARM64 Fuchsia -`aarch64-linux-android` | ✓ | | ARM64 Android -`aarch64-pc-windows-msvc` | ✓ | ✓ | ARM64 Windows MSVC -`aarch64-unknown-linux-musl` | ✓ | ✓ | ARM64 Linux with MUSL -`aarch64-unknown-none` | * | | Bare ARM64, hardfloat -`aarch64-unknown-none-softfloat` | * | | Bare ARM64, softfloat -`arm-linux-androideabi` | ✓ | | ARMv7 Android -`arm-unknown-linux-gnueabi` | ✓ | ✓ | ARMv6 Linux (kernel 3.2, glibc 2.17) -`arm-unknown-linux-gnueabihf` | ✓ | ✓ | ARMv6 Linux, hardfloat (kernel 3.2, glibc 2.17) -`arm-unknown-linux-musleabi` | ✓ | | ARMv6 Linux with MUSL -`arm-unknown-linux-musleabihf` | ✓ | | ARMv6 Linux with MUSL, hardfloat -`armebv7r-none-eabi` | * | | Bare ARMv7-R, Big Endian -`armebv7r-none-eabihf` | * | | Bare ARMv7-R, Big Endian, hardfloat -`armv5te-unknown-linux-gnueabi` | ✓ | | ARMv5TE Linux (kernel 4.4, glibc 2.23) -`armv5te-unknown-linux-musleabi` | ✓ | | ARMv5TE Linux with MUSL -`armv7-linux-androideabi` | ✓ | | ARMv7a Android -`armv7a-none-eabi` | * | | Bare ARMv7-A -`armv7r-none-eabi` | * | | Bare ARMv7-R -`armv7r-none-eabihf` | * | | Bare ARMv7-R, hardfloat -`armv7-unknown-linux-gnueabi` | ✓ | | ARMv7 Linux (kernel 4.15, glibc 2.27) -`armv7-unknown-linux-gnueabihf` | ✓ | ✓ | ARMv7 Linux, hardfloat (kernel 3.2, glibc 2.17) -`armv7-unknown-linux-musleabi` | ✓ | | ARMv7 Linux, MUSL -`armv7-unknown-linux-musleabihf` | ✓ | | ARMv7 Linux with MUSL -`asmjs-unknown-emscripten` | ✓ | | asm.js via Emscripten -`i586-pc-windows-msvc` | ✓ | | 32-bit Windows w/o SSE -`i586-unknown-linux-gnu` | ✓ | | 32-bit Linux w/o SSE (kernel 4.4, glibc 2.23) -`i586-unknown-linux-musl` | ✓ | | 32-bit Linux w/o SSE, MUSL -`i686-linux-android` | ✓ | | 32-bit x86 Android -`i686-unknown-freebsd` | ✓ | | 32-bit FreeBSD -`i686-unknown-linux-musl` | ✓ | | 32-bit Linux with MUSL -`mips-unknown-linux-gnu` | ✓ | ✓ | MIPS Linux (kernel 4.4, glibc 2.23) -`mips-unknown-linux-musl` | ✓ | | MIPS Linux with MUSL -`mips64-unknown-linux-gnuabi64` | ✓ | ✓ | MIPS64 Linux, n64 ABI (kernel 4.4, glibc 2.23) -`mips64-unknown-linux-muslabi64` | ✓ | | MIPS64 Linux, n64 ABI, MUSL -`mips64el-unknown-linux-gnuabi64` | ✓ | ✓ | MIPS64 (LE) Linux, n64 ABI (kernel 4.4, glibc 2.23) -`mips64el-unknown-linux-muslabi64` | ✓ | | MIPS64 (LE) Linux, n64 ABI, MUSL -`mipsel-unknown-linux-gnu` | ✓ | ✓ | MIPS (LE) Linux (kernel 4.4, glibc 2.23) -`mipsel-unknown-linux-musl` | ✓ | | MIPS (LE) Linux with MUSL -`nvptx64-nvidia-cuda` | ✓ | | --emit=asm generates PTX code that [runs on NVIDIA GPUs] -`powerpc-unknown-linux-gnu` | ✓ | ✓ | PowerPC Linux (kernel 2.6.32, glibc 2.11) -`powerpc64-unknown-linux-gnu` | ✓ | ✓ | PPC64 Linux (kernel 2.6.32, glibc 2.11) -`powerpc64le-unknown-linux-gnu` | ✓ | ✓ | PPC64LE Linux (kernel 3.10, glibc 2.17) -`riscv32i-unknown-none-elf` | * | | Bare RISC-V (RV32I ISA) -`riscv32imac-unknown-none-elf` | * | | Bare RISC-V (RV32IMAC ISA) -`riscv32imc-unknown-none-elf` | * | | Bare RISC-V (RV32IMC ISA) -`riscv64gc-unknown-linux-gnu` | ✓ | ✓ | RISC-V Linux (kernel 4.20, glibc 2.29) -`riscv64gc-unknown-none-elf` | * | | Bare RISC-V (RV64IMAFDC ISA) -`riscv64imac-unknown-none-elf` | * | | Bare RISC-V (RV64IMAC ISA) -`s390x-unknown-linux-gnu` | ✓ | ✓ | S390x Linux (kernel 2.6.32, glibc 2.11) -`sparc64-unknown-linux-gnu` | ✓ | | SPARC Linux (kernel 4.4, glibc 2.23) -`sparcv9-sun-solaris` | ✓ | | SPARC Solaris 10/11, illumos -`thumbv6m-none-eabi` | * | | Bare Cortex-M0, M0+, M1 -`thumbv7em-none-eabi` | * | | Bare Cortex-M4, M7 -`thumbv7em-none-eabihf` | * | | Bare Cortex-M4F, M7F, FPU, hardfloat -`thumbv7m-none-eabi` | * | | Bare Cortex-M3 -`thumbv7neon-linux-androideabi` | ✓ | | Thumb2-mode ARMv7a Android with NEON -`thumbv7neon-unknown-linux-gnueabihf` | ✓ | | Thumb2-mode ARMv7a Linux with NEON (kernel 4.4, glibc 2.23) -`thumbv8m.base-none-eabi` | * | | ARMv8-M Baseline -`thumbv8m.main-none-eabi` | * | | ARMv8-M Mainline -`thumbv8m.main-none-eabihf` | * | | ARMv8-M Mainline, hardfloat -`wasm32-unknown-emscripten` | ✓ | | WebAssembly via Emscripten -`wasm32-unknown-unknown` | ✓ | | WebAssembly -`wasm32-wasi` | ✓ | | WebAssembly with WASI -`x86_64-apple-ios` | ✓ | | 64-bit x86 iOS -`x86_64-fortanix-unknown-sgx` | ✓ | | [Fortanix ABI] for 64-bit Intel SGX -`x86_64-fuchsia` | ✓ | | 64-bit Fuchsia -`x86_64-linux-android` | ✓ | | 64-bit x86 Android -`x86_64-pc-solaris` | ✓ | | 64-bit Solaris 10/11, illumos -`x86_64-unknown-freebsd` | ✓ | ✓ | 64-bit FreeBSD -`x86_64-unknown-illumos` | ✓ | ✓ | illumos -`x86_64-unknown-linux-gnux32` | ✓ | | 64-bit Linux (x32 ABI) (kernel 4.15, glibc 2.27) -`x86_64-unknown-linux-musl` | ✓ | ✓ | 64-bit Linux with MUSL -`x86_64-unknown-netbsd` | ✓ | ✓ | NetBSD/amd64 -`x86_64-unknown-redox` | ✓ | | Redox OS +* ✓ indicates the full standard library is available. +* \* indicates the target only supports [`no_std`] development. + +[`no_std`]: https://rust-embedded.github.io/book/intro/no-std.html + +target | std | notes +-------|:---:|------- +`aarch64-apple-ios` | ✓ | ARM64 iOS +`aarch64-fuchsia` | ✓ | ARM64 Fuchsia +`aarch64-linux-android` | ✓ | ARM64 Android +`aarch64-unknown-none-softfloat` | * | Bare ARM64, softfloat +`aarch64-unknown-none` | * | Bare ARM64, hardfloat +`arm-linux-androideabi` | ✓ | ARMv7 Android +`arm-unknown-linux-musleabi` | ✓ | ARMv6 Linux with MUSL +`arm-unknown-linux-musleabihf` | ✓ | ARMv6 Linux with MUSL, hardfloat +`armebv7r-none-eabi` | * | Bare ARMv7-R, Big Endian +`armebv7r-none-eabihf` | * | Bare ARMv7-R, Big Endian, hardfloat +`armv5te-unknown-linux-gnueabi` | ✓ | ARMv5TE Linux (kernel 4.4, glibc 2.23) +`armv5te-unknown-linux-musleabi` | ✓ | ARMv5TE Linux with MUSL +`armv7-linux-androideabi` | ✓ | ARMv7a Android +`armv7-unknown-linux-gnueabi` | ✓ |ARMv7 Linux (kernel 4.15, glibc 2.27) +`armv7-unknown-linux-musleabi` | ✓ |ARMv7 Linux, MUSL +`armv7-unknown-linux-musleabihf` | ✓ | ARMv7 Linux with MUSL +`armv7a-none-eabi` | * | Bare ARMv7-A +`armv7r-none-eabi` | * | Bare ARMv7-R +`armv7r-none-eabihf` | * | Bare ARMv7-R, hardfloat +`asmjs-unknown-emscripten` | ✓ | asm.js via Emscripten +`i586-pc-windows-msvc` | ✓ | 32-bit Windows w/o SSE +`i586-unknown-linux-gnu` | ✓ | 32-bit Linux w/o SSE (kernel 4.4, glibc 2.23) +`i586-unknown-linux-musl` | ✓ | 32-bit Linux w/o SSE, MUSL +`i686-linux-android` | ✓ | 32-bit x86 Android +`i686-unknown-freebsd` | ✓ | 32-bit FreeBSD +`i686-unknown-linux-musl` | ✓ | 32-bit Linux with MUSL +`mips-unknown-linux-musl` | ✓ | MIPS Linux with MUSL +`mips64-unknown-linux-muslabi64` | ✓ | MIPS64 Linux, n64 ABI, MUSL +`mips64el-unknown-linux-muslabi64` | ✓ | MIPS64 (LE) Linux, n64 ABI, MUSL +`mipsel-unknown-linux-musl` | ✓ | MIPS (LE) Linux with MUSL +`nvptx64-nvidia-cuda` | * | --emit=asm generates PTX code that [runs on NVIDIA GPUs] +`riscv32i-unknown-none-elf` | * | Bare RISC-V (RV32I ISA) +`riscv32imac-unknown-none-elf` | * | Bare RISC-V (RV32IMAC ISA) +`riscv32imc-unknown-none-elf` | * | Bare RISC-V (RV32IMC ISA) +`riscv64gc-unknown-none-elf` | * | Bare RISC-V (RV64IMAFDC ISA) +`riscv64imac-unknown-none-elf` | * | Bare RISC-V (RV64IMAC ISA) +`sparc64-unknown-linux-gnu` | ✓ | SPARC Linux (kernel 4.4, glibc 2.23) +`sparcv9-sun-solaris` | ✓ | SPARC Solaris 10/11, illumos +`thumbv6m-none-eabi` | * | Bare Cortex-M0, M0+, M1 +`thumbv7em-none-eabi` | * | Bare Cortex-M4, M7 +`thumbv7em-none-eabihf` | * | Bare Cortex-M4F, M7F, FPU, hardfloat +`thumbv7m-none-eabi` | * | Bare Cortex-M3 +`thumbv7neon-linux-androideabi` | ✓ | Thumb2-mode ARMv7a Android with NEON +`thumbv7neon-unknown-linux-gnueabihf` | ✓ | Thumb2-mode ARMv7a Linux with NEON (kernel 4.4, glibc 2.23) +`thumbv8m.base-none-eabi` | * | ARMv8-M Baseline +`thumbv8m.main-none-eabi` | * | ARMv8-M Mainline +`thumbv8m.main-none-eabihf` | * | ARMv8-M Mainline, hardfloat +`wasm32-unknown-emscripten` | ✓ | WebAssembly via Emscripten +`wasm32-unknown-unknown` | ✓ | WebAssembly +`wasm32-wasi` | ✓ | WebAssembly with WASI +`x86_64-apple-ios` | ✓ | 64-bit x86 iOS +`x86_64-fortanix-unknown-sgx` | ✓ | [Fortanix ABI] for 64-bit Intel SGX +`x86_64-fuchsia` | ✓ | 64-bit Fuchsia +`x86_64-linux-android` | ✓ | 64-bit x86 Android +`x86_64-pc-solaris` | ✓ | 64-bit Solaris 10/11, illumos +`x86_64-unknown-linux-gnux32` | ✓ | 64-bit Linux (x32 ABI) (kernel 4.15, glibc 2.27) +`x86_64-unknown-redox` | ✓ | Redox OS [Fortanix ABI]: https://edp.fortanix.com/ ## Tier 3 -Tier 3 platforms are those which the Rust codebase has support for, but which -are not built or tested automatically, and may not work. Official builds are -not available. +Tier 3 targets are those which the Rust codebase has support for, but which the +Rust project does not build or test automatically, so they may or may not work. +Official builds are not available. For the full requirements, see [Tier 3 +target policy](target-tier-policy.md#tier-3-target-policy) in the Target Tier +Policy. + +The `std` column in the table below has the following meanings: + +* ✓ indicates the full standard library is available. +* \* indicates the target only supports [`no_std`] development. +* ? indicates the standard library support is unknown or a work-in-progress. + +[`no_std`]: https://rust-embedded.github.io/book/intro/no-std.html + +The `host` column indicates whether the codebase includes support for building +host tools. target | std | host | notes --------|-----|------|------- +-------|:---:|:----:|------- `aarch64-apple-ios-macabi` | ? | | Apple Catalyst on ARM64 -`aarch64-apple-ios-sim` | ? | | Apple iOS Simulator on ARM64 +[`aarch64-apple-ios-sim`](platform-support/aarch64-apple-ios-sim.md) | ✓ | | Apple iOS Simulator on ARM64 `aarch64-apple-tvos` | * | | ARM64 tvOS `aarch64-unknown-freebsd` | ✓ | ✓ | ARM64 FreeBSD `aarch64-unknown-hermit` | ? | | @@ -163,8 +206,8 @@ target | std | host | notes `aarch64-unknown-redox` | ? | | ARM64 Redox OS `aarch64-uwp-windows-msvc` | ? | | `aarch64-wrs-vxworks` | ? | | -`aarch64_be-unknown-linux-gnu` | ✓ | ✓ | ARM64 Linux (big-endian) `aarch64_be-unknown-linux-gnu_ilp32` | ✓ | ✓ | ARM64 Linux (big-endian, ILP32 ABI) +`aarch64_be-unknown-linux-gnu` | ✓ | ✓ | ARM64 Linux (big-endian) `armv4t-unknown-linux-gnueabi` | ? | | `armv5te-unknown-linux-uclibceabi` | ? | | ARMv5TE Linux with uClibc `armv6-unknown-freebsd` | ✓ | ✓ | ARMv6 FreeBSD @@ -175,22 +218,24 @@ target | std | host | notes `armv7-wrs-vxworks-eabihf` | ? | | `armv7a-none-eabihf` | * | | ARM Cortex-A, hardfloat `armv7s-apple-ios` | ✓ | | -`avr-unknown-gnu-atmega328` | ✗ | | AVR. Requires `-Z build-std=core` +`avr-unknown-gnu-atmega328` | * | | AVR. Requires `-Z build-std=core` +`bpfeb-unknown-none` | * | | BPF (big endian) +`bpfel-unknown-none` | * | | BPF (little endian) `hexagon-unknown-linux-musl` | ? | | `i386-apple-ios` | ✓ | | 32-bit x86 iOS `i686-apple-darwin` | ✓ | ✓ | 32-bit macOS (10.7+, Lion+) `i686-pc-windows-msvc` | ✓ | | 32-bit Windows XP support -`i686-unknown-uefi` | * | | 32-bit UEFI `i686-unknown-haiku` | ✓ | ✓ | 32-bit Haiku `i686-unknown-netbsd` | ✓ | ✓ | NetBSD/i386 with SSE2 `i686-unknown-openbsd` | ✓ | ✓ | 32-bit OpenBSD +`i686-unknown-uefi` | * | | 32-bit UEFI `i686-uwp-windows-gnu` | ? | | `i686-uwp-windows-msvc` | ? | | `i686-wrs-vxworks` | ? | | `mips-unknown-linux-uclibc` | ✓ | | MIPS Linux with uClibc +`mipsel-sony-psp` | * | | MIPS (LE) Sony PlayStation Portable (PSP) `mipsel-unknown-linux-uclibc` | ✓ | | MIPS (LE) Linux with uClibc `mipsel-unknown-none` | * | | Bare MIPS (LE) softfloat -`mipsel-sony-psp` | * | | MIPS (LE) Sony PlayStation Portable (PSP) `mipsisa32r6-unknown-linux-gnu` | ? | | `mipsisa32r6el-unknown-linux-gnu` | ? | | `mipsisa64r6-unknown-linux-gnuabi64` | ? | | @@ -200,34 +245,34 @@ target | std | host | notes `powerpc-unknown-linux-musl` | ? | | `powerpc-unknown-netbsd` | ✓ | ✓ | `powerpc-unknown-openbsd` | ? | | -`powerpc-wrs-vxworks` | ? | | `powerpc-wrs-vxworks-spe` | ? | | +`powerpc-wrs-vxworks` | ? | | `powerpc64-unknown-freebsd` | ✓ | ✓ | PPC64 FreeBSD (ELFv1 and ELFv2) `powerpc64-unknown-linux-musl` | ? | | `powerpc64-wrs-vxworks` | ? | | `powerpc64le-unknown-linux-musl` | ? | | -`riscv64gc-unknown-linux-musl` | | | RISC-V Linux (kernel 4.20, musl 1.2.0) `riscv32gc-unknown-linux-gnu` | | | RISC-V Linux (kernel 5.4, glibc 2.33) `riscv32gc-unknown-linux-musl` | | | RISC-V Linux (kernel 5.4, musl + RISCV32 support patches) +`riscv64gc-unknown-linux-musl` | | | RISC-V Linux (kernel 4.20, musl 1.2.0) `s390x-unknown-linux-musl` | | | S390x Linux (kernel 2.6.32, MUSL) `sparc-unknown-linux-gnu` | ✓ | | 32-bit SPARC Linux `sparc64-unknown-netbsd` | ✓ | ✓ | NetBSD/sparc64 `sparc64-unknown-openbsd` | ? | | +`thumbv4t-none-eabi` | * | | ARMv4T T32 `thumbv7a-pc-windows-msvc` | ? | | `thumbv7a-uwp-windows-msvc` | ✓ | | `thumbv7neon-unknown-linux-musleabihf` | ? | | Thumb2-mode ARMv7a Linux with NEON, MUSL -`thumbv4t-none-eabi` | * | | ARMv4T T32 `wasm64-unknown-unknown` | * | | WebAssembly `x86_64-apple-ios-macabi` | ✓ | | Apple Catalyst on x86_64 `x86_64-apple-tvos` | * | | x86 64-bit tvOS -`x86_64-unknown-none-linuxkernel` | * | | Linux kernel modules -`x86_64-sun-solaris` | ? | | Deprecated target for 64-bit Solaris 10/11, illumos `x86_64-pc-windows-msvc` | ✓ | | 64-bit Windows XP support +`x86_64-sun-solaris` | ? | | Deprecated target for 64-bit Solaris 10/11, illumos `x86_64-unknown-dragonfly` | ✓ | ✓ | 64-bit DragonFlyBSD `x86_64-unknown-haiku` | ✓ | ✓ | 64-bit Haiku `x86_64-unknown-hermit` | ? | | -`x86_64-unknown-none-hermitkernel` | ? | | HermitCore kernel `x86_64-unknown-l4re-uclibc` | ? | | +`x86_64-unknown-none-hermitkernel` | ? | | HermitCore kernel +`x86_64-unknown-none-linuxkernel` | * | | Linux kernel modules `x86_64-unknown-openbsd` | ✓ | ✓ | 64-bit OpenBSD `x86_64-unknown-uefi` | * | | 64-bit UEFI `x86_64-uwp-windows-gnu` | ✓ | | diff --git a/src/doc/rustc/src/platform-support/aarch64-apple-ios-sim.md b/src/doc/rustc/src/platform-support/aarch64-apple-ios-sim.md new file mode 100644 index 0000000000..9aa5db26f9 --- /dev/null +++ b/src/doc/rustc/src/platform-support/aarch64-apple-ios-sim.md @@ -0,0 +1,56 @@ +# aarch64-apple-ios-sim + +**Tier: 3** + +Apple iOS Simulator on ARM64. + +## Designated Developers + +* [@badboy](https://github.com/badboy) +* [@deg4uss3r](https://github.com/deg4uss3r) + +## Requirements + +This target is cross-compiled. +To build this target Xcode 12 or higher on macOS is required. + +## Building + +The target can be built by enabling it for a `rustc` build: + +```toml +[build] +build-stage = 1 +target = ["aarch64-apple-ios-sim"] +``` + +## Cross-compilation + +This target can be cross-compiled from `x86_64` or `aarch64` macOS hosts. + +Other hosts are not supported for cross-compilation, but might work when also providing the required Xcode SDK. + +## Testing + +Currently there is no support to run the rustc test suite for this target. + + +## Building Rust programs + +*Note: Building for this target requires the corresponding iOS SDK, as provided by Xcode 12+.* + +If `rustc` has support for that target and the library artifacts are available, +then Rust programs can be built for that target: + +```text +rustc --target aarch64-apple-ios-sim your-code.rs +``` + +On Rust Nightly it is possible to build without the target artifacts available: + +```text +cargo build -Z build-std --target aarch64-apple-ios-sim +``` + +There is no easy way to run simple programs in the iOS simulator. +Static library builds can be embedded into iOS applications. diff --git a/src/doc/rustc/src/target-tier-policy.md b/src/doc/rustc/src/target-tier-policy.md index 463f56099f..cc02b294b4 100644 --- a/src/doc/rustc/src/target-tier-policy.md +++ b/src/doc/rustc/src/target-tier-policy.md @@ -1,5 +1,16 @@ # Target Tier Policy +## Table of Contents + +* [General](#general) +* [Tier 3 target policy](#tier-3-target-policy) +* [Tier 2 target policy](#tier-2-target-policy) + * [Tier 2 with host tools](#tier-2-with-host-tools) +* [Tier 1 target policy](#tier-1-target-policy) + * [Tier 1 with host tools](#tier-1-with-host-tools) + +## General + Rust provides three tiers of target support: - Rust provides no guarantees about tier 3 targets; they exist in the codebase, @@ -91,7 +102,7 @@ place minimal requirements on the introduction of targets. A proposed new tier 3 target must be reviewed and approved by a member of the compiler team based on these requirements. The reviewer may choose to gauge -broader compiler team consensus via a Major Change Proposal (MCP). +broader compiler team consensus via a [Major Change Proposal (MCP)][MCP]. A proposed target or target-specific patch that substantially changes code shared with other targets (not just target-specific code) must be reviewed and @@ -212,8 +223,8 @@ patches that fail to build on a target. Thus, we place requirements that ensure the target will not block forward progress of the Rust project. A proposed new tier 2 target must be reviewed and approved by the compiler team -based on these requirements. Such review and approval may occur via a Major -Change Proposal (MCP). +based on these requirements. Such review and approval may occur via a [Major +Change Proposal (MCP)][MCP]. In addition, the infrastructure team must approve the integration of the target into Continuous Integration (CI), and the tier 2 CI-related requirements. This @@ -380,7 +391,7 @@ development platform, not just a compilation target. A proposed new tier 2 target with host tools must be reviewed and approved by the compiler team based on these requirements. Such review and approval may -occur via a Major Change Proposal (MCP). +occur via a [Major Change Proposal (MCP)][MCP]. In addition, the infrastructure team must approve the integration of the target's host tools into Continuous Integration (CI), and the CI-related @@ -637,3 +648,5 @@ for demotion of a tier 1 target (with or without host tools) requires a full RFC process, with approval by the compiler and release teams. Any such proposal will be communicated widely to the Rust community, both when initially proposed and before being dropped from a stable release. + +[MCP]: https://forge.rust-lang.org/compiler/mcp.html diff --git a/src/doc/rustdoc/README.md b/src/doc/rustdoc/README.md new file mode 100644 index 0000000000..7d97d5e4ab --- /dev/null +++ b/src/doc/rustdoc/README.md @@ -0,0 +1,5 @@ +# Rustdoc + +This is documentation for rustdoc itself, written in mdbook format. +To build the book, use `x.py doc src/doc/rustdoc`. +To run doctests, use `x.py test src/doc/rustdoc`. diff --git a/src/doc/rustdoc/src/documentation-tests.md b/src/doc/rustdoc/src/documentation-tests.md index 8e3c622818..cb7b85655b 100644 --- a/src/doc/rustdoc/src/documentation-tests.md +++ b/src/doc/rustdoc/src/documentation-tests.md @@ -424,9 +424,7 @@ without including it in your main documentation. For example, you could write th `lib.rs` to test your README as part of your doctests: ```rust,no_run -#![feature(external_doc)] - -#[doc(include = "../README.md")] +#[doc = include_str!("../README.md")] #[cfg(doctest)] pub struct ReadmeDoctests; ``` diff --git a/src/doc/rustdoc/src/how-to-write-documentation.md b/src/doc/rustdoc/src/how-to-write-documentation.md index 688be7aede..f89495cca3 100644 --- a/src/doc/rustdoc/src/how-to-write-documentation.md +++ b/src/doc/rustdoc/src/how-to-write-documentation.md @@ -229,15 +229,13 @@ Example: ```md - [x] Complete task -- [ ] IncComplete task +- [ ] Incomplete task ``` -This will render as +This will render as: -

    -
    • -
    • -
+> - [x] Complete task +> - [ ] Incomplete task See the specification for the [task list extension] for more details. diff --git a/src/doc/rustdoc/src/lints.md b/src/doc/rustdoc/src/lints.md index a6626679a7..16b091eb25 100644 --- a/src/doc/rustdoc/src/lints.md +++ b/src/doc/rustdoc/src/lints.md @@ -294,6 +294,50 @@ warning: unclosed HTML tag `h1` warning: 2 warnings emitted ``` +## invalid_rust_codeblocks + +This lint **warns by default**. It detects Rust code blocks in documentation +examples that are invalid (e.g. empty, not parsable as Rust). For example: + +```rust +/// Empty code blocks (with and without the `rust` marker): +/// +/// ```rust +/// ``` +/// +/// Invalid syntax in code blocks: +/// +/// ```rust +/// '< +/// ``` +pub fn foo() {} +``` + +Which will give: + +```text +warning: Rust code block is empty + --> lint.rs:3:5 + | +3 | /// ```rust + | _____^ +4 | | /// ``` + | |_______^ + | + = note: `#[warn(rustdoc::invalid_rust_codeblocks)]` on by default + +warning: could not parse code block as Rust code + --> lint.rs:8:5 + | +8 | /// ```rust + | _____^ +9 | | /// '< +10 | | /// ``` + | |_______^ + | + = note: error from rustc: unterminated character literal +``` + ## bare_urls This lint is **warn-by-default**. It detects URLs which are not links. diff --git a/src/doc/rustdoc/src/the-doc-attribute.md b/src/doc/rustdoc/src/the-doc-attribute.md index 52f2a3728f..d192f7d5ce 100644 --- a/src/doc/rustdoc/src/the-doc-attribute.md +++ b/src/doc/rustdoc/src/the-doc-attribute.md @@ -35,6 +35,13 @@ Which can feel more flexible. Note that this would generate this: but given that docs are rendered via Markdown, it will remove these newlines. +Another use case is for including external files as documentation: + +```rust,no_run +#[doc = include_str!("../README.md")] +# fn f() {} +``` + The `doc` attribute has more options though! These don't involve the text of the output, but instead, various aspects of the presentation of the output. We've split them into two kinds below: attributes that are useful at the diff --git a/src/doc/rustdoc/src/unstable-features.md b/src/doc/rustdoc/src/unstable-features.md index 28b81a4026..e9b15666bb 100644 --- a/src/doc/rustdoc/src/unstable-features.md +++ b/src/doc/rustdoc/src/unstable-features.md @@ -131,22 +131,6 @@ Book][unstable-masked] and [its tracking issue][issue-masked]. [unstable-masked]: ../unstable-book/language-features/doc-masked.html [issue-masked]: https://github.com/rust-lang/rust/issues/44027 -### Include external files as API documentation - -As designed in [RFC 1990], Rustdoc can read an external file to use as a type's documentation. This -is useful if certain documentation is so long that it would break the flow of reading the source. -Instead of writing it all inline, writing `#[doc(include = "sometype.md")]` will ask Rustdoc to -instead read that file and use it as if it were written inline. - -[RFC 1990]: https://github.com/rust-lang/rfcs/pull/1990 - -`#[doc(include = "...")]` currently requires the `#![feature(external_doc)]` feature gate. For more -information, see [its chapter in the Unstable Book][unstable-include] and [its tracking -issue][issue-include]. - -[unstable-include]: ../unstable-book/language-features/external-doc.html -[issue-include]: https://github.com/rust-lang/rust/issues/44732 - ## Unstable command-line arguments These features are enabled by passing a command-line flag to Rustdoc, but the flags in question are diff --git a/src/doc/unstable-book/src/compiler-flags/force-warns.md b/src/doc/unstable-book/src/compiler-flags/force-warns.md new file mode 100644 index 0000000000..0a205be096 --- /dev/null +++ b/src/doc/unstable-book/src/compiler-flags/force-warns.md @@ -0,0 +1,21 @@ +# `force-warns` + +The tracking issue for this feature is: [#85512](https://github.com/rust-lang/rust/issues/85512). + +------------------------ + +This feature allows you to cause any lint to produce a warning even if the lint has a different level by default or another level is set somewhere else. For instance, the `force-warns` option can be used to make a lint (e.g., `dead_code`) produce a warning even if that lint is allowed in code with `#![allow(dead_code)]`. + +## Example + +```rust,ignore (partial-example) +#![allow(dead_code)] + +fn dead_function() {} +// This would normally not produce a warning even though the +// function is not used, because dead code is being allowed + +fn main() {} +``` + +We can force a warning to be produced by providing `--force-warns dead_code` to rustc. diff --git a/src/doc/unstable-book/src/compiler-flags/instrument-coverage.md b/src/doc/unstable-book/src/compiler-flags/instrument-coverage.md new file mode 100644 index 0000000000..f7c2a26f01 --- /dev/null +++ b/src/doc/unstable-book/src/compiler-flags/instrument-coverage.md @@ -0,0 +1,346 @@ +# `instrument-coverage` + +The tracking issue for this feature is: [#79121]. + +[#79121]: https://github.com/rust-lang/rust/issues/79121 + +--- + +## Introduction + +The Rust compiler includes two code coverage implementations: + +- A GCC-compatible, gcov-based coverage implementation, enabled with `-Z profile`, which derives coverage data based on DebugInfo. +- A source-based code coverage implementation, enabled with `-Z instrument-coverage`, which uses LLVM's native, efficient coverage instrumentation to generate very precise coverage data. + +This document describes how to enable and use the LLVM instrumentation-based coverage, via the `-Z instrument-coverage` compiler flag. + +## How it works + +When `-Z instrument-coverage` is enabled, the Rust compiler enhances rust-based libraries and binaries by: + +- Automatically injecting calls to an LLVM intrinsic ([`llvm.instrprof.increment`]), at functions and branches in compiled code, to increment counters when conditional sections of code are executed. +- Embedding additional information in the data section of each library and binary (using the [LLVM Code Coverage Mapping Format] _Version 4_, supported _only_ in LLVM 11 and up), to define the code regions (start and end positions in the source code) being counted. + +When running a coverage-instrumented program, the counter values are written to a `profraw` file at program termination. LLVM bundles tools that read the counter results, combine those results with the coverage map (embedded in the program binary), and generate coverage reports in multiple formats. + +[`llvm.instrprof.increment`]: https://llvm.org/docs/LangRef.html#llvm-instrprof-increment-intrinsic +[llvm code coverage mapping format]: https://llvm.org/docs/CoverageMappingFormat.html + +> **Note**: `-Z instrument-coverage` also automatically enables `-Z symbol-mangling-version=v0` (tracking issue [#60705]). The `v0` symbol mangler is strongly recommended, but be aware that this demangler is also experimental. The `v0` demangler can be overridden by explicitly adding `-Z symbol-mangling-version=legacy`. + +[#60705]: https://github.com/rust-lang/rust/issues/60705 + +## Enable coverage profiling in the Rust compiler + +Rust's source-based code coverage requires the Rust "profiler runtime". Without it, compiling with `-Z instrument-coverage` generates an error that the profiler runtime is missing. + +The Rust `nightly` distribution channel includes the profiler runtime, by default. + +> **Important**: If you are building the Rust compiler from the source distribution, the profiler runtime is _not_ enabled in the default `config.toml.example`. Edit your `config.toml` file and ensure the `profiler` feature is set it to `true` (either under the `[build]` section, or under the settings for an individual `[target.]`): +> +> ```toml +> # Build the profiler runtime (required when compiling with options that depend +> # on this runtime, such as `-C profile-generate` or `-Z instrument-coverage`). +> profiler = true +> ``` + +### Building the demangler + +LLVM coverage reporting tools generate results that can include function names and other symbol references, and the raw coverage results report symbols using the compiler's "mangled" version of the symbol names, which can be difficult to interpret. To work around this issue, LLVM coverage tools also support a user-specified symbol name demangler. + +One option for a Rust demangler is [`rustfilt`], which can be installed with: + +```shell +cargo install rustfilt +``` + +Another option, if you are building from the Rust compiler source distribution, is to use the `rust-demangler` tool included in the Rust source distribution, which can be built with: + +```shell +$ ./x.py build rust-demangler +``` + +[`rustfilt`]: https://crates.io/crates/rustfilt + +## Compiling with coverage enabled + +Set the `-Z instrument-coverage` compiler flag in order to enable LLVM source-based code coverage profiling. + +The default option generates coverage for all functions, including unused (never called) functions and generics. The compiler flag supports an optional value to tailor this behavior. (See [`-Z instrument-coverage=`](#-z-instrument-coverageoptions), below.) + +With `cargo`, you can instrument your program binary _and_ dependencies at the same time. + +For example (if your project's Cargo.toml builds a binary by default): + +```shell +$ cd your-project +$ cargo clean +$ RUSTFLAGS="-Z instrument-coverage" cargo build +``` + +If `cargo` is not configured to use your `profiler`-enabled version of `rustc`, set the path explicitly via the `RUSTC` environment variable. Here is another example, using a `stage1` build of `rustc` to compile an `example` binary (from the [`json5format`] crate): + +```shell +$ RUSTC=$HOME/rust/build/x86_64-unknown-linux-gnu/stage1/bin/rustc \ + RUSTFLAGS="-Z instrument-coverage" \ + cargo build --example formatjson5 +``` + +> **Note**: that some compiler options, combined with `-Z instrument-coverage`, can produce LLVM IR and/or linked binaries that are incompatible with LLVM coverage maps. For example, coverage requires references to actual functions in LLVM IR. If any covered function is optimized out, the coverage tools may not be able to process the coverage results. If you need to pass additional options, with coverage enabled, test them early, to confirm you will get the coverage results you expect. + +## Running the instrumented binary to generate raw coverage profiling data + +In the previous example, `cargo` generated the coverage-instrumented binary `formatjson5`: + +```shell +$ echo "{some: 'thing'}" | target/debug/examples/formatjson5 - +``` + +```json5 +{ + some: "thing", +} +``` + +After running this program, a new file, `default.profraw`, should be in the current working directory. It's often preferable to set a specific file name or path. You can change the output file using the environment variable `LLVM_PROFILE_FILE`: + +```shell +$ echo "{some: 'thing'}" \ + | LLVM_PROFILE_FILE="formatjson5.profraw" target/debug/examples/formatjson5 - +... +$ ls formatjson5.profraw +formatjson5.profraw +``` + +If `LLVM_PROFILE_FILE` contains a path to a non-existent directory, the missing directory structure will be created. Additionally, the following special pattern strings are rewritten: + +- `%p` - The process ID. +- `%h` - The hostname of the machine running the program. +- `%t` - The value of the TMPDIR environment variable. +- `%Nm` - the instrumented binary’s signature: The runtime creates a pool of N raw profiles, used for on-line profile merging. The runtime takes care of selecting a raw profile from the pool, locking it, and updating it before the program exits. `N` must be between `1` and `9`, and defaults to `1` if omitted (with simply `%m`). +- `%c` - Does not add anything to the filename, but enables a mode (on some platforms, including Darwin) in which profile counter updates are continuously synced to a file. This means that if the instrumented program crashes, or is killed by a signal, perfect coverage information can still be recovered. + +## Installing LLVM coverage tools + +LLVM's supplies two tools—`llvm-profdata` and `llvm-cov`—that process coverage data and generate reports. There are several ways to find and/or install these tools, but note that the coverage mapping data generated by the Rust compiler requires LLVM version 11 or higher. (`llvm-cov --version` typically shows the tool's LLVM version number.): + +- The LLVM tools may be installed (or installable) directly to your OS (such as via `apt-get`, for Linux). +- If you are building the Rust compiler from source, you can optionally use the bundled LLVM tools, built from source. Those tool binaries can typically be found in your build platform directory at something like: `rust/build/x86_64-unknown-linux-gnu/llvm/bin/llvm-*`. +- You can install compatible versions of these tools via `rustup`. + +The `rustup` option is guaranteed to install a compatible version of the LLVM tools, but they can be hard to find. We recommend [`cargo-binutils`], which installs Rust-specific wrappers around these and other LLVM tools, so you can invoke them via `cargo` commands! + +```shell +$ rustup component add llvm-tools-preview +$ cargo install cargo-binutils +$ cargo profdata -- --help # note the additional "--" preceding the tool-specific arguments +``` + +[`cargo-binutils`]: https://crates.io/crates/cargo-binutils + +## Creating coverage reports + +Raw profiles have to be indexed before they can be used to generate coverage reports. This is done using [`llvm-profdata merge`] (or `cargo profdata -- merge`), which can combine multiple raw profiles and index them at the same time: + +```shell +$ llvm-profdata merge -sparse formatjson5.profraw -o formatjson5.profdata +``` + +Finally, the `.profdata` file is used, in combination with the coverage map (from the program binary) to generate coverage reports using [`llvm-cov report`] (or `cargo cov -- report`), for a coverage summaries; and [`llvm-cov show`] (or `cargo cov -- show`), to see detailed coverage of lines and regions (character ranges) overlaid on the original source code. + +These commands have several display and filtering options. For example: + +```shell +$ llvm-cov show -Xdemangler=rustfilt target/debug/examples/formatjson5 \ + -instr-profile=formatjson5.profdata \ + -show-line-counts-or-regions \ + -show-instantiations \ + -name=add_quoted_string +``` + +Screenshot of sample `llvm-cov show` result, for function add_quoted_string +
+
+ +Some of the more notable options in this example include: + +- `--Xdemangler=rustfilt` - the command name or path used to demangle Rust symbols (`rustfilt` in the example, but this could also be a path to the `rust-demangler` tool) +- `target/debug/examples/formatjson5` - the instrumented binary (from which to extract the coverage map) +- `--instr-profile=.profdata` - the location of the `.profdata` file created by `llvm-profdata merge` (from the `.profraw` file generated by the instrumented binary) +- `--name=` - to show coverage for a specific function (or, consider using another filter option, such as `--name-regex=`) + +[`llvm-profdata merge`]: https://llvm.org/docs/CommandGuide/llvm-profdata.html#profdata-merge +[`llvm-cov report`]: https://llvm.org/docs/CommandGuide/llvm-cov.html#llvm-cov-report +[`llvm-cov show`]: https://llvm.org/docs/CommandGuide/llvm-cov.html#llvm-cov-show + +> **Note**: Coverage can also be disabled on an individual function by annotating the function with the [`no_coverage` attribute] (which requires the feature flag `#![feature(no_coverage)]`). + +[`no_coverage` attribute]: ../language-features/no-coverage.md + +## Interpreting reports + +There are four statistics tracked in a coverage summary: + +- Function coverage is the percentage of functions that have been executed at least once. A function is considered to be executed if any of its instantiations are executed. +- Instantiation coverage is the percentage of function instantiations that have been executed at least once. Generic functions and functions generated from macros are two kinds of functions that may have multiple instantiations. +- Line coverage is the percentage of code lines that have been executed at least once. Only executable lines within function bodies are considered to be code lines. +- Region coverage is the percentage of code regions that have been executed at least once. A code region may span multiple lines: for example, in a large function body with no control flow. In other cases, a single line can contain multiple code regions: `return x || (y && z)` has countable code regions for `x` (which may resolve the expression, if `x` is `true`), `|| (y && z)` (executed only if `x` was `false`), and `return` (executed in either situation). + +Of these four statistics, function coverage is usually the least granular while region coverage is the most granular. The project-wide totals for each statistic are listed in the summary. + +## Test coverage + +A typical use case for coverage analysis is test coverage. Rust's source-based coverage tools can both measure your tests' code coverage as percentage, and pinpoint functions and branches not tested. + +The following example (using the [`json5format`] crate, for demonstration purposes) show how to generate and analyze coverage results for all tests in a crate. + +Since `cargo test` both builds and runs the tests, we set both the additional `RUSTFLAGS`, to add the `-Z instrument-coverage` flag, and `LLVM_PROFILE_FILE`, to set a custom filename for the raw profiling data generated during the test runs. Since there may be more than one test binary, apply `%m` in the filename pattern. This generates unique names for each test binary. (Otherwise, each executed test binary would overwrite the coverage results from the previous binary.) + +```shell +$ RUSTFLAGS="-Z instrument-coverage" \ + LLVM_PROFILE_FILE="json5format-%m.profraw" \ + cargo test --tests +``` + +Make note of the test binary file paths, displayed after the word "`Running`" in the test output: + +```text + ... + Compiling json5format v0.1.3 ($HOME/json5format) + Finished test [unoptimized + debuginfo] target(s) in 14.60s + + Running target/debug/deps/json5format-fececd4653271682 +running 25 tests +... +test result: ok. 25 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out + + Running target/debug/deps/lib-30768f9c53506dc5 +running 31 tests +... +test result: ok. 31 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out +``` + +You should have one or more `.profraw` files now, one for each test binary. Run the `profdata` tool to merge them: + +```shell +$ cargo profdata -- merge \ + -sparse json5format-*.profraw -o json5format.profdata +``` + +Then run the `cov` tool, with the `profdata` file and all test binaries: + +```shell +$ cargo cov -- report \ + --use-color --ignore-filename-regex='/.cargo/registry' \ + --instr-profile=json5format.profdata \ + --object target/debug/deps/lib-30768f9c53506dc5 \ + --object target/debug/deps/json5format-fececd4653271682 +$ cargo cov -- show \ + --use-color --ignore-filename-regex='/.cargo/registry' \ + --instr-profile=json5format.profdata \ + --object target/debug/deps/lib-30768f9c53506dc5 \ + --object target/debug/deps/json5format-fececd4653271682 \ + --show-instantiations --show-line-counts-or-regions \ + --Xdemangler=rustfilt | less -R +``` + +> **Note**: The command line option `--ignore-filename-regex=/.cargo/registry`, which excludes the sources for dependencies from the coverage results.\_ + +### Tips for listing the binaries automatically + +For `bash` users, one suggested way to automatically complete the `cov` command with the list of binaries is with a command like: + +```bash +$ cargo cov -- report \ + $( \ + for file in \ + $( \ + RUSTFLAGS="-Z instrument-coverage" \ + cargo test --tests --no-run --message-format=json \ + | jq -r "select(.profile.test == true) | .filenames[]" \ + | grep -v dSYM - \ + ); \ + do \ + printf "%s %s " -object $file; \ + done \ + ) \ + --instr-profile=json5format.profdata --summary-only # and/or other options +``` + +Adding `--no-run --message-format=json` to the _same_ `cargo test` command used to run +the tests (including the same environment variables and flags) generates output in a JSON +format that `jq` can easily query. + +The `printf` command takes this list and generates the `--object ` arguments +for each listed test binary. + +### Including doc tests + +The previous examples run `cargo test` with `--tests`, which excludes doc tests.[^79417] + +To include doc tests in the coverage results, drop the `--tests` flag, and apply the +`-Z instrument-coverage` flag, and some doc-test-specific options in the +`RUSTDOCFLAGS` environment variable. (The `cargo profdata` command does not change.) + +```bash +$ RUSTFLAGS="-Z instrument-coverage" \ + RUSTDOCFLAGS="-Z instrument-coverage -Z unstable-options --persist-doctests target/debug/doctestbins" \ + LLVM_PROFILE_FILE="json5format-%m.profraw" \ + cargo test +$ cargo profdata -- merge \ + -sparse json5format-*.profraw -o json5format.profdata +``` + +The `-Z unstable-options --persist-doctests` flag is required, to save the test binaries +(with their coverage maps) for `llvm-cov`. + +```bash +$ cargo cov -- report \ + $( \ + for file in \ + $( \ + RUSTFLAGS="-Z instrument-coverage" \ + RUSTDOCFLAGS="-Z instrument-coverage -Z unstable-options --persist-doctests target/debug/doctestbins" \ + cargo test --no-run --message-format=json \ + | jq -r "select(.profile.test == true) | .filenames[]" \ + | grep -v dSYM - \ + ) \ + target/debug/doctestbins/*/rust_out; \ + do \ + [[ -x $file ]] && printf "%s %s " -object $file; \ + done \ + ) \ + --instr-profile=json5format.profdata --summary-only # and/or other options +``` + +> **Note**: The differences in this `cargo cov` command, compared with the version without +> doc tests, include: + +- The `cargo test ... --no-run` command is updated with the same environment variables + and flags used to _build_ the tests, _including_ the doc tests. (`LLVM_PROFILE_FILE` + is only used when _running_ the tests.) +- The file glob pattern `target/debug/doctestbins/*/rust_out` adds the `rust_out` + binaries generated for doc tests (note, however, that some `rust_out` files may not + be executable binaries). +- `[[ -x $file ]] &&` filters the files passed on to the `printf`, to include only + executable binaries. + +[^79417]: + There is ongoing work to resolve a known issue + [(#79417)](https://github.com/rust-lang/rust/issues/79417) that doc test coverage + generates incorrect source line numbers in `llvm-cov show` results. + +## `-Z instrument-coverage=` + +- `-Z instrument-coverage=all`: Instrument all functions, including unused functions and unused generics. (This is the same as `-Z instrument-coverage`, with no value.) +- `-Z instrument-coverage=except-unused-generics`: Instrument all functions except unused generics. +- `-Z instrument-coverage=except-unused-functions`: Instrument only used (called) functions and instantiated generic functions. +- `-Z instrument-coverage=off`: Do not instrument any functions. (This is the same as simply not including the `-Z instrument-coverage` option.) + +## Other references + +Rust's implementation and workflow for source-based code coverage is based on the same library and tools used to implement [source-based code coverage in Clang]. (This document is partially based on the Clang guide.) + +[source-based code coverage in clang]: https://clang.llvm.org/docs/SourceBasedCodeCoverage.html +[`json5format`]: https://crates.io/crates/json5format diff --git a/src/doc/unstable-book/src/compiler-flags/source-based-code-coverage.md b/src/doc/unstable-book/src/compiler-flags/source-based-code-coverage.md index 0d05b80e21..cb65978e0a 100644 --- a/src/doc/unstable-book/src/compiler-flags/source-based-code-coverage.md +++ b/src/doc/unstable-book/src/compiler-flags/source-based-code-coverage.md @@ -1,327 +1,5 @@ # `source-based-code-coverage` -The tracking issue for this feature is: [#79121]. +See compiler flag [`-Z instrument-coverage`]. ------------------------- - -## Introduction - -The Rust compiler includes two code coverage implementations: - -* A GCC-compatible, gcov-based coverage implementation, enabled with [`-Zprofile`], which operates on DebugInfo. -* A source-based code coverage implementation, enabled with `-Zinstrument-coverage`, which uses LLVM's native coverage instrumentation to generate very precise coverage data. - -This document describes how to enable and use the LLVM instrumentation-based coverage, via the `-Zinstrument-coverage` compiler flag. - -## How it works - -When `-Zinstrument-coverage` is enabled, the Rust compiler enhances rust-based libraries and binaries by: - -* Automatically injecting calls to an LLVM intrinsic ([`llvm.instrprof.increment`]), at functions and branches in compiled code, to increment counters when conditional sections of code are executed. -* Embedding additional information in the data section of each library and binary (using the [LLVM Code Coverage Mapping Format] _Version 4_, supported _only_ in LLVM 11 and up), to define the code regions (start and end positions in the source code) being counted. - -When running a coverage-instrumented program, the counter values are written to a `profraw` file at program termination. LLVM bundles tools that read the counter results, combine those results with the coverage map (embedded in the program binary), and generate coverage reports in multiple formats. - -## Enable coverage profiling in the Rust compiler - -Rust's source-based code coverage requires the Rust "profiler runtime". Without it, compiling with `-Zinstrument-coverage` generates an error that the profiler runtime is missing. - -The Rust `nightly` distribution channel should include the profiler runtime, by default. - -*IMPORTANT:* If you are building the Rust compiler from the source distribution, the profiler runtime is *not* enabled in the default `config.toml.example`. Edit your `config.toml` file and ensure the `profiler` feature is set it to `true`: - -```toml -# Build the profiler runtime (required when compiling with options that depend -# on this runtime, such as `-C profile-generate` or `-Z instrument-coverage`). -profiler = true -``` - -If changed, rebuild the Rust compiler (see [rustc-dev-guide-how-to-build-and-run]). - -### Building the demangler - -LLVM coverage reporting tools generate results that can include function names and other symbol references, and the raw coverage results report symbols using the compiler's "mangled" version of the symbol names, which can be difficult to interpret. To work around this issue, LLVM coverage tools also support a user-specified symbol name demangler. - -One option for a Rust demangler is [`rustfilt`], which can be installed with: - -```shell -cargo install rustfilt -``` - -Another option, if you are building from the Rust compiler source distribution, is to use the `rust-demangler` tool included in the Rust source distribution, which can be built with: - -```shell -$ ./x.py build rust-demangler -``` - -## Compiling with coverage enabled - -Set the `-Zinstrument-coverage` compiler flag in order to enable LLVM source-based code coverage profiling. - -With `cargo`, you can instrument your program binary *and* dependencies at the same time. - -For example (if your project's Cargo.toml builds a binary by default): - -```shell -$ cd your-project -$ cargo clean -$ RUSTFLAGS="-Zinstrument-coverage" cargo build -``` - -If `cargo` is not configured to use your `profiler`-enabled version of `rustc`, set the path explicitly via the `RUSTC` environment variable. Here is another example, using a `stage1` build of `rustc` to compile an `example` binary (from the [`json5format`] crate): - -```shell -$ RUSTC=$HOME/rust/build/x86_64-unknown-linux-gnu/stage1/bin/rustc \ - RUSTFLAGS="-Zinstrument-coverage" \ - cargo build --example formatjson5 -``` - -Note that some compiler options, combined with `-Zinstrument-coverage`, can produce LLVM IR and/or linked binaries that are incompatible with LLVM coverage maps. For example, coverage requires references to actual functions in LLVM IR. If any covered function is optimized out, the coverage tools may not be able to process the coverage results. If you need to pass additional options, with coverage enabled, test them early, to confirm you will get the coverage results you expect. - -## Running the instrumented binary to generate raw coverage profiling data - -In the previous example, `cargo` generated the coverage-instrumented binary `formatjson5`: - -```shell -$ echo "{some: 'thing'}" | target/debug/examples/formatjson5 - -``` -```json5 -{ - some: 'thing', -} -``` - -After running this program, a new file, `default.profraw`, should be in the current working directory. It's often preferable to set a specific file name or path. You can change the output file using the environment variable `LLVM_PROFILE_FILE`: - - -```shell -$ echo "{some: 'thing'}" \ - | LLVM_PROFILE_FILE="formatjson5.profraw" target/debug/examples/formatjson5 - -... -$ ls formatjson5.profraw -formatjson5.profraw -``` - -If `LLVM_PROFILE_FILE` contains a path to a non-existent directory, the missing directory structure will be created. Additionally, the following special pattern strings are rewritten: - -* `%p` - The process ID. -* `%h` - The hostname of the machine running the program. -* `%t` - The value of the TMPDIR environment variable. -* `%Nm` - the instrumented binary’s signature: The runtime creates a pool of N raw profiles, used for on-line profile merging. The runtime takes care of selecting a raw profile from the pool, locking it, and updating it before the program exits. `N` must be between `1` and `9`, and defaults to `1` if omitted (with simply `%m`). -* `%c` - Does not add anything to the filename, but enables a mode (on some platforms, including Darwin) in which profile counter updates are continuously synced to a file. This means that if the instrumented program crashes, or is killed by a signal, perfect coverage information can still be recovered. - -## Installing LLVM coverage tools - -LLVM's supplies two tools—`llvm-profdata` and `llvm-cov`—that process coverage data and generate reports. There are several ways to find and/or install these tools, but note that the coverage mapping data generated by the Rust compiler requires LLVM version 11 or higher. (`llvm-cov --version` typically shows the tool's LLVM version number.): - -* The LLVM tools may be installed (or installable) directly to your OS (such as via `apt-get`, for Linux). -* If you are building the Rust compiler from source, you can optionally use the bundled LLVM tools, built from source. Those tool binaries can typically be found in your build platform directory at something like: `rust/build/x86_64-unknown-linux-gnu/llvm/bin/llvm-*`. -* You can install compatible versions of these tools via `rustup`. - -The `rustup` option is guaranteed to install a compatible version of the LLVM tools, but they can be hard to find. We recommend [`cargo-binutils`], which installs Rust-specific wrappers around these and other LLVM tools, so you can invoke them via `cargo` commands! - -```shell -$ rustup component add llvm-tools-preview -$ cargo install cargo-binutils -$ cargo profdata -- --help # note the additional "--" preceding the tool-specific arguments -``` - -## Creating coverage reports - -Raw profiles have to be indexed before they can be used to generate coverage reports. This is done using [`llvm-profdata merge`] (or `cargo profdata -- merge`), which can combine multiple raw profiles and index them at the same time: - -```shell -$ llvm-profdata merge -sparse formatjson5.profraw -o formatjson5.profdata -``` - -Finally, the `.profdata` file is used, in combination with the coverage map (from the program binary) to generate coverage reports using [`llvm-cov report`] (or `cargo cov -- report`), for a coverage summaries; and [`llvm-cov show`] (or `cargo cov -- show`), to see detailed coverage of lines and regions (character ranges) overlaid on the original source code. - -These commands have several display and filtering options. For example: - -```shell -$ llvm-cov show -Xdemangler=rustfilt target/debug/examples/formatjson5 \ - -instr-profile=formatjson5.profdata \ - -show-line-counts-or-regions \ - -show-instantiations \ - -name=add_quoted_string -``` - -Screenshot of sample `llvm-cov show` result, for function add_quoted_string -
-
- -Some of the more notable options in this example include: - -* `--Xdemangler=rustfilt` - the command name or path used to demangle Rust symbols (`rustfilt` in the example, but this could also be a path to the `rust-demangler` tool) -* `target/debug/examples/formatjson5` - the instrumented binary (from which to extract the coverage map) -* `--instr-profile=.profdata` - the location of the `.profdata` file created by `llvm-profdata merge` (from the `.profraw` file generated by the instrumented binary) -* `--name=` - to show coverage for a specific function (or, consider using another filter option, such as `--name-regex=`) - -## Interpreting reports - -There are four statistics tracked in a coverage summary: - -* Function coverage is the percentage of functions that have been executed at least once. A function is considered to be executed if any of its instantiations are executed. -* Instantiation coverage is the percentage of function instantiations that have been executed at least once. Generic functions and functions generated from macros are two kinds of functions that may have multiple instantiations. -* Line coverage is the percentage of code lines that have been executed at least once. Only executable lines within function bodies are considered to be code lines. -* Region coverage is the percentage of code regions that have been executed at least once. A code region may span multiple lines: for example, in a large function body with no control flow. In other cases, a single line can contain multiple code regions: `return x || (y && z)` has countable code regions for `x` (which may resolve the expression, if `x` is `true`), `|| (y && z)` (executed only if `x` was `false`), and `return` (executed in either situation). - -Of these four statistics, function coverage is usually the least granular while region coverage is the most granular. The project-wide totals for each statistic are listed in the summary. - -## Test coverage - -A typical use case for coverage analysis is test coverage. Rust's source-based coverage tools can both measure your tests' code coverage as percentage, and pinpoint functions and branches not tested. - -The following example (using the [`json5format`] crate, for demonstration purposes) show how to generate and analyze coverage results for all tests in a crate. - -Since `cargo test` both builds and runs the tests, we set both the additional `RUSTFLAGS`, to add the `-Zinstrument-coverage` flag, and `LLVM_PROFILE_FILE`, to set a custom filename for the raw profiling data generated during the test runs. Since there may be more than one test binary, apply `%m` in the filename pattern. This generates unique names for each test binary. (Otherwise, each executed test binary would overwrite the coverage results from the previous binary.) - -```shell -$ RUSTFLAGS="-Zinstrument-coverage" \ - LLVM_PROFILE_FILE="json5format-%m.profraw" \ - cargo test --tests -``` - -Make note of the test binary file paths, displayed after the word "`Running`" in the test output: - -```text - ... - Compiling json5format v0.1.3 ($HOME/json5format) - Finished test [unoptimized + debuginfo] target(s) in 14.60s - - Running target/debug/deps/json5format-fececd4653271682 -running 25 tests -... -test result: ok. 25 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out - - Running target/debug/deps/lib-30768f9c53506dc5 -running 31 tests -... -test result: ok. 31 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out -``` - -You should have one ore more `.profraw` files now, one for each test binary. Run the `profdata` tool to merge them: - -```shell -$ cargo profdata -- merge \ - -sparse json5format-*.profraw -o json5format.profdata -``` - -Then run the `cov` tool, with the `profdata` file and all test binaries: - -```shell -$ cargo cov -- report \ - --use-color --ignore-filename-regex='/.cargo/registry' \ - --instr-profile=json5format.profdata \ - --object target/debug/deps/lib-30768f9c53506dc5 \ - --object target/debug/deps/json5format-fececd4653271682 -$ cargo cov -- show \ - --use-color --ignore-filename-regex='/.cargo/registry' \ - --instr-profile=json5format.profdata \ - --object target/debug/deps/lib-30768f9c53506dc5 \ - --object target/debug/deps/json5format-fececd4653271682 \ - --show-instantiations --show-line-counts-or-regions \ - --Xdemangler=rustfilt | less -R -``` - -_Note the command line option `--ignore-filename-regex=/.cargo/registry`, which excludes the sources for dependencies from the coverage results._ - -### Tips for listing the binaries automatically - -For `bash` users, one suggested way to automatically complete the `cov` command with the list of binaries is with a command like: - -```bash -$ cargo cov -- report \ - $( \ - for file in \ - $( \ - RUSTFLAGS="-Zinstrument-coverage" \ - cargo test --tests --no-run --message-format=json \ - | jq -r "select(.profile.test == true) | .filenames[]" \ - | grep -v dSYM - \ - ); \ - do \ - printf "%s %s " -object $file; \ - done \ - ) \ - --instr-profile=json5format.profdata --summary-only # and/or other options -``` - -Adding `--no-run --message-format=json` to the _same_ `cargo test` command used to run -the tests (including the same environment variables and flags) generates output in a JSON -format that `jq` can easily query. - -The `printf` command takes this list and generates the `--object ` arguments -for each listed test binary. - -### Including doc tests - -The previous examples run `cargo test` with `--tests`, which excludes doc tests.[^79417] - -To include doc tests in the coverage results, drop the `--tests` flag, and apply the -`-Zinstrument-coverage` flag, and some doc-test-specific options in the -`RUSTDOCFLAGS` environment variable. (The `cargo profdata` command does not change.) - -```bash -$ RUSTFLAGS="-Zinstrument-coverage" \ - RUSTDOCFLAGS="-Zinstrument-coverage -Zunstable-options --persist-doctests target/debug/doctestbins" \ - LLVM_PROFILE_FILE="json5format-%m.profraw" \ - cargo test -$ cargo profdata -- merge \ - -sparse json5format-*.profraw -o json5format.profdata -``` - -The `-Zunstable-options --persist-doctests` flag is required, to save the test binaries -(with their coverage maps) for `llvm-cov`. - -```bash -$ cargo cov -- report \ - $( \ - for file in \ - $( \ - RUSTFLAGS="-Zinstrument-coverage" \ - RUSTDOCFLAGS="-Zinstrument-coverage -Zunstable-options --persist-doctests target/debug/doctestbins" \ - cargo test --no-run --message-format=json \ - | jq -r "select(.profile.test == true) | .filenames[]" \ - | grep -v dSYM - \ - ) \ - target/debug/doctestbins/*/rust_out; \ - do \ - [[ -x $file ]] && printf "%s %s " -object $file; \ - done \ - ) \ - --instr-profile=json5format.profdata --summary-only # and/or other options -``` - -Note, the differences in this `cargo cov` command, compared with the version without -doc tests, include: - -* The `cargo test ... --no-run` command is updated with the same environment variables - and flags used to _build_ the tests, _including_ the doc tests. (`LLVM_PROFILE_FILE` - is only used when _running_ the tests.) -* The file glob pattern `target/debug/doctestbins/*/rust_out` adds the `rust_out` - binaries generated for doc tests (note, however, that some `rust_out` files may not - be executable binaries). -* `[[ -x $file ]] &&` filters the files passed on to the `printf`, to include only - executable binaries. - -[^79417]: There is ongoing work to resolve a known issue -[(#79417)](https://github.com/rust-lang/rust/issues/79417) that doc test coverage -generates incorrect source line numbers in `llvm-cov show` results. - -## Other references - -Rust's implementation and workflow for source-based code coverage is based on the same library and tools used to implement [source-based code coverage in Clang]. (This document is partially based on the Clang guide.) - -[#79121]: https://github.com/rust-lang/rust/issues/79121 -[`-Zprofile`]: profile.md -[`llvm.instrprof.increment`]: https://llvm.org/docs/LangRef.html#llvm-instrprof-increment-intrinsic -[LLVM Code Coverage Mapping Format]: https://llvm.org/docs/CoverageMappingFormat.html -[rustc-dev-guide-how-to-build-and-run]: https://rustc-dev-guide.rust-lang.org/building/how-to-build-and-run.html -[`rustfilt`]: https://crates.io/crates/rustfilt -[`json5format`]: https://crates.io/crates/json5format -[`cargo-binutils`]: https://crates.io/crates/cargo-binutils -[`llvm-profdata merge`]: https://llvm.org/docs/CommandGuide/llvm-profdata.html#profdata-merge -[`llvm-cov report`]: https://llvm.org/docs/CommandGuide/llvm-cov.html#llvm-cov-report -[`llvm-cov show`]: https://llvm.org/docs/CommandGuide/llvm-cov.html#llvm-cov-show -[source-based code coverage in Clang]: https://clang.llvm.org/docs/SourceBasedCodeCoverage.html +[`-z instrument-coverage`]: ./instrument-coverage.html diff --git a/src/doc/unstable-book/src/language-features/const-fn.md b/src/doc/unstable-book/src/language-features/const-fn.md deleted file mode 100644 index bcf7f78b8f..0000000000 --- a/src/doc/unstable-book/src/language-features/const-fn.md +++ /dev/null @@ -1,10 +0,0 @@ -# `const_fn` - -The tracking issue for this feature is: [#57563] - -[#57563]: https://github.com/rust-lang/rust/issues/57563 - ------------------------- - -The `const_fn` feature enables additional functionality not stabilized in the -[minimal subset of `const_fn`](https://github.com/rust-lang/rust/issues/53555) diff --git a/src/doc/unstable-book/src/language-features/external-doc.md b/src/doc/unstable-book/src/language-features/external-doc.md deleted file mode 100644 index effae5d299..0000000000 --- a/src/doc/unstable-book/src/language-features/external-doc.md +++ /dev/null @@ -1,40 +0,0 @@ -# `external_doc` - -The tracking issue for this feature is: [#44732] - -The `external_doc` feature allows the use of the `include` parameter to the `#[doc]` attribute, to -include external files in documentation. Use the attribute in place of, or in addition to, regular -doc comments and `#[doc]` attributes, and `rustdoc` will load the given file when it renders -documentation for your crate. - -With the following files in the same directory: - -`external-doc.md`: - -```markdown -# My Awesome Type - -This is the documentation for this spectacular type. -``` - -`lib.rs`: - -```no_run (needs-external-files) -#![feature(external_doc)] - -#[doc(include = "external-doc.md")] -pub struct MyAwesomeType; -``` - -`rustdoc` will load the file `external-doc.md` and use it as the documentation for the `MyAwesomeType` -struct. - -When locating files, `rustdoc` will base paths in the `src/` directory, as if they were alongside the -`lib.rs` for your crate. So if you want a `docs/` folder to live alongside the `src/` directory, -start your paths with `../docs/` for `rustdoc` to properly find the file. - -This feature was proposed in [RFC #1990] and initially implemented in PR [#44781]. - -[#44732]: https://github.com/rust-lang/rust/issues/44732 -[RFC #1990]: https://github.com/rust-lang/rfcs/pull/1990 -[#44781]: https://github.com/rust-lang/rust/pull/44781 diff --git a/src/doc/unstable-book/src/language-features/member-constraints.md b/src/doc/unstable-book/src/language-features/member-constraints.md deleted file mode 100644 index 3ba4a3e6b1..0000000000 --- a/src/doc/unstable-book/src/language-features/member-constraints.md +++ /dev/null @@ -1,29 +0,0 @@ -# `member_constraints` - -The tracking issue for this feature is: [#61997] - -[#61997]: https://github.com/rust-lang/rust/issues/61997 - ------------------------- - -The `member_constraints` feature gate lets you use `impl Trait` syntax with -multiple unrelated lifetime parameters. - -A simple example is: - -```rust -#![feature(member_constraints)] - -trait Trait<'a, 'b> { } -impl Trait<'_, '_> for T {} - -fn foo<'a, 'b>(x: &'a u32, y: &'b u32) -> impl Trait<'a, 'b> { - (x, y) -} - -fn main() { } -``` - -Without the `member_constraints` feature gate, the above example is an -error because both `'a` and `'b` appear in the impl Trait bounds, but -neither outlives the other. diff --git a/src/doc/unstable-book/src/language-features/more-qualified-paths.md b/src/doc/unstable-book/src/language-features/more-qualified-paths.md new file mode 100644 index 0000000000..857af577a6 --- /dev/null +++ b/src/doc/unstable-book/src/language-features/more-qualified-paths.md @@ -0,0 +1,29 @@ +# `more_qualified_paths` + +The `more_qualified_paths` feature can be used in order to enable the +use of qualified paths in patterns. + +## Example + +```rust +#![feature(more_qualified_paths)] + +fn main() { + // destructure through a qualified path + let ::Assoc { br } = StructStruct { br: 2 }; +} + +struct StructStruct { + br: i8, +} + +struct Foo; + +trait A { + type Assoc; +} + +impl A for Foo { + type Assoc = StructStruct; +} +``` diff --git a/src/doc/unstable-book/src/language-features/native-link-modifiers-as-needed.md b/src/doc/unstable-book/src/language-features/native-link-modifiers-as-needed.md new file mode 100644 index 0000000000..1757673612 --- /dev/null +++ b/src/doc/unstable-book/src/language-features/native-link-modifiers-as-needed.md @@ -0,0 +1,18 @@ +# `native_link_modifiers_as_needed` + +The tracking issue for this feature is: [#81490] + +[#81490]: https://github.com/rust-lang/rust/issues/81490 + +------------------------ + +The `native_link_modifiers_as_needed` feature allows you to use the `as-needed` modifier. + +`as-needed` is only compatible with the `dynamic` and `framework` linking kinds. Using any other kind will result in a compiler error. + +`+as-needed` means that the library will be actually linked only if it satisfies some undefined symbols at the point at which it is specified on the command line, making it similar to static libraries in this regard. + +This modifier translates to `--as-needed` for ld-like linkers, and to `-dead_strip_dylibs` / `-needed_library` / `-needed_framework` for ld64. +The modifier does nothing for linkers that don't support it (e.g. `link.exe`). + +The default for this modifier is unclear, some targets currently specify it as `+as-needed`, some do not. We may want to try making `+as-needed` a default for all targets. diff --git a/src/doc/unstable-book/src/language-features/native-link-modifiers-bundle.md b/src/doc/unstable-book/src/language-features/native-link-modifiers-bundle.md new file mode 100644 index 0000000000..ac192cff13 --- /dev/null +++ b/src/doc/unstable-book/src/language-features/native-link-modifiers-bundle.md @@ -0,0 +1,19 @@ +# `native_link_modifiers_bundle` + +The tracking issue for this feature is: [#81490] + +[#81490]: https://github.com/rust-lang/rust/issues/81490 + +------------------------ + +The `native_link_modifiers_bundle` feature allows you to use the `bundle` modifier. + +Only compatible with the `static` linking kind. Using any other kind will result in a compiler error. + +`+bundle` means objects from the static library are bundled into the produced crate (a rlib, for example) and are used from this crate later during linking of the final binary. + +`-bundle` means the static library is included into the produced rlib "by name" and object files from it are included only during linking of the final binary, the file search by that name is also performed during final linking. + +This modifier is supposed to supersede the `static-nobundle` linking kind defined by [RFC 1717](https://github.com/rust-lang/rfcs/pull/1717). + +The default for this modifier is currently `+bundle`, but it could be changed later on some future edition boundary. diff --git a/src/doc/unstable-book/src/language-features/native-link-modifiers-verbatim.md b/src/doc/unstable-book/src/language-features/native-link-modifiers-verbatim.md new file mode 100644 index 0000000000..02bd87e509 --- /dev/null +++ b/src/doc/unstable-book/src/language-features/native-link-modifiers-verbatim.md @@ -0,0 +1,20 @@ +# `native_link_modifiers_verbatim` + +The tracking issue for this feature is: [#81490] + +[#81490]: https://github.com/rust-lang/rust/issues/81490 + +------------------------ + +The `native_link_modifiers_verbatim` feature allows you to use the `verbatim` modifier. + +`+verbatim` means that rustc itself won't add any target-specified library prefixes or suffixes (like `lib` or `.a`) to the library name, and will try its best to ask for the same thing from the linker. + +For `ld`-like linkers rustc will use the `-l:filename` syntax (note the colon) when passing the library, so the linker won't add any prefixes or suffixes as well. +See [`-l namespec`](https://sourceware.org/binutils/docs/ld/Options.html) in ld documentation for more details. +For linkers not supporting any verbatim modifiers (e.g. `link.exe` or `ld64`) the library name will be passed as is. + +The default for this modifier is `-verbatim`. + +This RFC changes the behavior of `raw-dylib` linking kind specified by [RFC 2627](https://github.com/rust-lang/rfcs/pull/2627). The `.dll` suffix (or other target-specified suffixes for other targets) is now added automatically. +If your DLL doesn't have the `.dll` suffix, it can be specified with `+verbatim`. diff --git a/src/doc/unstable-book/src/language-features/native-link-modifiers-whole-archive.md b/src/doc/unstable-book/src/language-features/native-link-modifiers-whole-archive.md new file mode 100644 index 0000000000..4961e88cad --- /dev/null +++ b/src/doc/unstable-book/src/language-features/native-link-modifiers-whole-archive.md @@ -0,0 +1,18 @@ +# `native_link_modifiers_whole_archive` + +The tracking issue for this feature is: [#81490] + +[#81490]: https://github.com/rust-lang/rust/issues/81490 + +------------------------ + +The `native_link_modifiers_whole_archive` feature allows you to use the `whole-archive` modifier. + +Only compatible with the `static` linking kind. Using any other kind will result in a compiler error. + +`+whole-archive` means that the static library is linked as a whole archive without throwing any object files away. + +This modifier translates to `--whole-archive` for `ld`-like linkers, to `/WHOLEARCHIVE` for `link.exe`, and to `-force_load` for `ld64`. +The modifier does nothing for linkers that don't support it. + +The default for this modifier is `-whole-archive`. diff --git a/src/doc/unstable-book/src/language-features/native-link-modifiers.md b/src/doc/unstable-book/src/language-features/native-link-modifiers.md new file mode 100644 index 0000000000..fc8b575462 --- /dev/null +++ b/src/doc/unstable-book/src/language-features/native-link-modifiers.md @@ -0,0 +1,11 @@ +# `native_link_modifiers` + +The tracking issue for this feature is: [#81490] + +[#81490]: https://github.com/rust-lang/rust/issues/81490 + +------------------------ + +The `native_link_modifiers` feature allows you to use the `modifiers` syntax with the `#[link(..)]` attribute. + +Modifiers are specified as a comma-delimited string with each modifier prefixed with either a `+` or `-` to indicate that the modifier is enabled or disabled, respectively. The last boolean value specified for a given modifier wins. diff --git a/src/doc/unstable-book/src/language-features/no-coverage.md b/src/doc/unstable-book/src/language-features/no-coverage.md new file mode 100644 index 0000000000..327cdb3979 --- /dev/null +++ b/src/doc/unstable-book/src/language-features/no-coverage.md @@ -0,0 +1,30 @@ +# `no_coverage` + +The tracking issue for this feature is: [#84605] + +[#84605]: https://github.com/rust-lang/rust/issues/84605 + +--- + +The `no_coverage` attribute can be used to selectively disable coverage +instrumentation in an annotated function. This might be useful to: + +- Avoid instrumentation overhead in a performance critical function +- Avoid generating coverage for a function that is not meant to be executed, + but still target 100% coverage for the rest of the program. + +## Example + +```rust +#![feature(no_coverage)] + +// `foo()` will get coverage instrumentation (by default) +fn foo() { + // ... +} + +#[no_coverage] +fn bar() { + // ... +} +``` diff --git a/src/doc/unstable-book/src/library-features/asm.md b/src/doc/unstable-book/src/library-features/asm.md index 4f9033cedc..03dbf4fb61 100644 --- a/src/doc/unstable-book/src/library-features/asm.md +++ b/src/doc/unstable-book/src/library-features/asm.md @@ -26,9 +26,11 @@ Inline assembly is currently supported on the following architectures: - AArch64 - RISC-V - NVPTX +- PowerPC - Hexagon - MIPS32r2 and MIPS64r2 - wasm32 +- BPF ## Basic usage @@ -459,7 +461,7 @@ options := "options(" option *["," option] [","] ")" asm := "asm!(" format_string *("," format_string) *("," [ident "="] operand) ["," options] [","] ")" ``` -The macro will initially be supported only on ARM, AArch64, Hexagon, x86, x86-64 and RISC-V targets. Support for more targets may be added in the future. The compiler will emit an error if `asm!` is used on an unsupported target. +The macro will initially be supported only on ARM, AArch64, Hexagon, PowerPC, x86, x86-64 and RISC-V targets. Support for more targets may be added in the future. The compiler will emit an error if `asm!` is used on an unsupported target. [format-syntax]: https://doc.rust-lang.org/std/fmt/#syntax @@ -508,7 +510,7 @@ Several types of operands are supported: - Identical to `inout` except that the register allocator can reuse a register allocated to an `in` (this can happen if the compiler knows the `in` has the same initial value as the `inlateout`). - You should only write to the register after all inputs are read, otherwise you may clobber an input. * `const ` - - `` must be an integer or floating-point constant expression. + - `` must be an integer constant expression. - The value of the expression is formatted as a string and substituted directly into the asm template string. * `sym ` - `` must refer to a `fn` or `static`. @@ -535,20 +537,20 @@ Here is the list of currently supported register classes: | Architecture | Register class | Registers | LLVM constraint code | | ------------ | -------------- | --------- | -------------------- | -| x86 | `reg` | `ax`, `bx`, `cx`, `dx`, `si`, `di`, `r[8-15]` (x86-64 only) | `r` | +| x86 | `reg` | `ax`, `bx`, `cx`, `dx`, `si`, `di`, `bp`, `r[8-15]` (x86-64 only) | `r` | | x86 | `reg_abcd` | `ax`, `bx`, `cx`, `dx` | `Q` | | x86-32 | `reg_byte` | `al`, `bl`, `cl`, `dl`, `ah`, `bh`, `ch`, `dh` | `q` | -| x86-64 | `reg_byte`\* | `al`, `bl`, `cl`, `dl`, `sil`, `dil`, `r[8-15]b` | `q` | +| x86-64 | `reg_byte`\* | `al`, `bl`, `cl`, `dl`, `sil`, `dil`, `bpl`, `r[8-15]b` | `q` | | x86 | `xmm_reg` | `xmm[0-7]` (x86) `xmm[0-15]` (x86-64) | `x` | | x86 | `ymm_reg` | `ymm[0-7]` (x86) `ymm[0-15]` (x86-64) | `x` | | x86 | `zmm_reg` | `zmm[0-7]` (x86) `zmm[0-31]` (x86-64) | `v` | | x86 | `kreg` | `k[1-7]` | `Yk` | -| AArch64 | `reg` | `x[0-28]`, `x30` | `r` | +| AArch64 | `reg` | `x[0-30]` | `r` | | AArch64 | `vreg` | `v[0-31]` | `w` | | AArch64 | `vreg_low16` | `v[0-15]` | `x` | -| ARM | `reg` | `r[0-5]` `r7`\*, `r[8-10]`, `r11`\*, `r12`, `r14` | `r` | +| ARM | `reg` | `r[0-12]`, `r14` | `r` | | ARM (Thumb) | `reg_thumb` | `r[0-r7]` | `l` | -| ARM (ARM) | `reg_thumb` | `r[0-r10]`, `r12`, `r14` | `l` | +| ARM (ARM) | `reg_thumb` | `r[0-r12]`, `r14` | `l` | | ARM | `sreg` | `s[0-31]` | `t` | | ARM | `sreg_low16` | `s[0-15]` | `x` | | ARM | `dreg` | `d[0-31]` | `w` | @@ -565,7 +567,12 @@ Here is the list of currently supported register classes: | RISC-V | `reg` | `x1`, `x[5-7]`, `x[9-15]`, `x[16-31]` (non-RV32E) | `r` | | RISC-V | `freg` | `f[0-31]` | `f` | | Hexagon | `reg` | `r[0-28]` | `r` | +| PowerPC | `reg` | `r[0-31]` | `r` | +| PowerPC | `reg_nonzero` | | `r[1-31]` | `b` | +| PowerPC | `freg` | `f[0-31]` | `f` | | wasm32 | `local` | None\* | `r` | +| BPF | `reg` | `r[0-10]` | `r` | +| BPF | `wreg` | `w[0-10]` | `w` | > **Note**: On x86 we treat `reg_byte` differently from `reg` because the compiler can allocate `al` and `ah` separately whereas `reg` reserves the whole register. > @@ -573,9 +580,7 @@ Here is the list of currently supported register classes: > > Note #3: NVPTX doesn't have a fixed register set, so named registers are not supported. > -> Note #4: On ARM the frame pointer is either `r7` or `r11` depending on the platform. -> -> Note #5: WebAssembly doesn't have registers, so named registers are not supported. +> Note #4: WebAssembly doesn't have registers, so named registers are not supported. Additional register classes may be added in the future based on demand (e.g. MMX, x87, etc). @@ -609,7 +614,12 @@ Each register class has constraints on which value types they can be used with. | RISC-V | `freg` | `f` | `f32` | | RISC-V | `freg` | `d` | `f64` | | Hexagon | `reg` | None | `i8`, `i16`, `i32`, `f32` | +| PowerPC | `reg` | None | `i8`, `i16`, `i32` | +| PowerPC | `reg_nonzero` | None | `i8`, `i16`, `i32` | +| PowerPC | `freg` | None | `f32`, `f64` | | wasm32 | `local` | None | `i8` `i16` `i32` `i64` `f32` `f64` | +| BPF | `reg` | None | `i8` `i16` `i32` `i64` | +| BPF | `wreg` | `alu32` | `i8` `i16` `i32` | > **Note**: For the purposes of the above table pointers, function pointers and `isize`/`usize` are treated as the equivalent integer type (`i16`/`i32`/`i64` depending on the target). @@ -669,6 +679,7 @@ Some registers have multiple names. These are all treated by the compiler as ide | Hexagon | `r29` | `sp` | | Hexagon | `r30` | `fr` | | Hexagon | `r31` | `lr` | +| BPF | `r[0-10]` | `w[0-10]` | Some registers cannot be used for input or output operands: @@ -677,13 +688,14 @@ Some registers cannot be used for input or output operands: | All | `sp` | The stack pointer must be restored to its original value at the end of an asm code block. | | All | `bp` (x86), `x29` (AArch64), `x8` (RISC-V), `fr` (Hexagon), `$fp` (MIPS) | The frame pointer cannot be used as an input or output. | | ARM | `r7` or `r11` | On ARM the frame pointer can be either `r7` or `r11` depending on the target. The frame pointer cannot be used as an input or output. | -| ARM | `r6` | `r6` is used internally by LLVM as a base pointer and therefore cannot be used as an input or output. | +| All | `si` (x86-32), `bx` (x86-64), `r6` (ARM), `x19` (AArch64), `r19` (Hexagon), `x9` (RISC-V) | This is used internally by LLVM as a "base pointer" for functions with complex stack frames. | | x86 | `k0` | This is a constant zero register which can't be modified. | | x86 | `ip` | This is the program counter, not a real register. | | x86 | `mm[0-7]` | MMX registers are not currently supported (but may be in the future). | | x86 | `st([0-7])` | x87 registers are not currently supported (but may be in the future). | | AArch64 | `xzr` | This is a constant zero register which can't be modified. | | ARM | `pc` | This is the program counter, not a real register. | +| ARM | `r9` | This is a reserved register on some ARM targets. | | MIPS | `$0` or `$zero` | This is a constant zero register which can't be modified. | | MIPS | `$1` or `$at` | Reserved for assembler. | | MIPS | `$26`/`$k0`, `$27`/`$k1` | OS-reserved registers. | @@ -693,9 +705,10 @@ Some registers cannot be used for input or output operands: | RISC-V | `gp`, `tp` | These registers are reserved and cannot be used as inputs or outputs. | | Hexagon | `lr` | This is the link register which cannot be used as an input or output. | -In some cases LLVM will allocate a "reserved register" for `reg` operands even though this register cannot be explicitly specified. Assembly code making use of reserved registers should be careful since `reg` operands may alias with those registers. Reserved registers are: -- The frame pointer on all architectures. -- `r6` on ARM. +In some cases LLVM will allocate a "reserved register" for `reg` operands even though this register cannot be explicitly specified. Assembly code making use of reserved registers should be careful since `reg` operands may alias with those registers. Reserved registers are the frame pointer and base pointer +- The frame pointer and LLVM base pointer on all architectures. +- `r9` on ARM. +- `x18` on AArch64. ## Template modifiers @@ -744,6 +757,9 @@ The supported modifiers are a subset of LLVM's (and GCC's) [asm template argumen | RISC-V | `reg` | None | `x1` | None | | RISC-V | `freg` | None | `f0` | None | | Hexagon | `reg` | None | `r0` | None | +| PowerPC | `reg` | None | `0` | None | +| PowerPC | `reg_nonzero` | None | `3` | `b` | +| PowerPC | `freg` | None | `0` | None | > Notes: > - on ARM `e` / `f`: this prints the low or high doubleword register name of a NEON quad (128-bit) register. diff --git a/src/doc/unstable-book/src/library-features/global-asm.md b/src/doc/unstable-book/src/library-features/global-asm.md index ce1155a977..146d17b463 100644 --- a/src/doc/unstable-book/src/library-features/global-asm.md +++ b/src/doc/unstable-book/src/library-features/global-asm.md @@ -8,12 +8,9 @@ The tracking issue for this feature is: [#35119] The `global_asm!` macro allows the programmer to write arbitrary assembly outside the scope of a function body, passing it through -`rustc` and `llvm` to the assembler. The macro is a no-frills -interface to LLVM's concept of [module-level inline assembly]. That is, -all caveats applicable to LLVM's module-level inline assembly apply -to `global_asm!`. - -[module-level inline assembly]: http://llvm.org/docs/LangRef.html#module-level-inline-assembly +`rustc` and `llvm` to the assembler. That is to say, `global_asm!` is +equivalent to assembling the asm with an external assembler and then +linking the resulting object file with the current crate. `global_asm!` fills a role not currently satisfied by either `asm!` or `#[naked]` functions. The programmer has _all_ features of the @@ -38,11 +35,11 @@ And a more complicated usage looks like this: # mod x86 { pub mod sally { - global_asm!(r#" - .global foo - foo: - jmp baz - "#); + global_asm!( + ".global foo", + "foo:", + "jmp baz", + ); #[no_mangle] pub unsafe extern "C" fn baz() {} @@ -56,11 +53,11 @@ extern "C" { } pub mod harry { - global_asm!(r#" - .global bar - bar: - jmp quux - "#); + global_asm!( + ".global bar", + "bar:", + "jmp quux", + ); #[no_mangle] pub unsafe extern "C" fn quux() {} @@ -69,8 +66,44 @@ pub mod harry { ``` You may use `global_asm!` multiple times, anywhere in your crate, in -whatever way suits you. The effect is as if you concatenated all -usages and placed the larger, single usage in the crate root. +whatever way suits you. However, you should not rely on assembler state +(e.g. assembler macros) defined in one `global_asm!` to be available in +another one. It is implementation-defined whether the multiple usages +are concatenated into one or assembled separately. + +`global_asm!` also supports `const` operands like `asm!`, which allows +constants defined in Rust to be used in assembly code: + +```rust,no_run +#![feature(global_asm)] +# #[cfg(any(target_arch="x86", target_arch="x86_64"))] +# mod x86 { +const C: i32 = 1234; +global_asm!( + ".global bar", + "bar: .word {c}", + c = const C, +); +# } +``` + +The syntax for passing operands is the same as `asm!` except that only +`const` operands are allowed. Refer to the [asm](asm.md) documentation +for more details. + +On x86, the assembly code will use intel syntax by default. You can +override this by adding `options(att_syntax)` at the end of the macro +arguments list: + +```rust,no_run +#![feature(global_asm)] +# #[cfg(any(target_arch="x86", target_arch="x86_64"))] +# mod x86 { +global_asm!("movl ${}, %ecx", const 5, options(att_syntax)); +// is equivalent to +global_asm!("mov ecx, {}", const 5); +# } +``` ------------------------ diff --git a/src/etc/htmldocck.py b/src/etc/htmldocck.py index 2f7233685d..8647db5a45 100644 --- a/src/etc/htmldocck.py +++ b/src/etc/htmldocck.py @@ -135,6 +135,8 @@ except NameError: unichr = chr +channel = os.environ["DOC_RUST_LANG_ORG_CHANNEL"] + class CustomHTMLParser(HTMLParser): """simplified HTML parser. @@ -270,6 +272,7 @@ def flatten(node): def normalize_xpath(path): + path = path.replace("{{channel}}", channel) if path.startswith('//'): return '.' + path # avoid warnings elif path.startswith('.//'): @@ -334,6 +337,7 @@ class CachedFiles(object): def check_string(data, pat, regexp): + pat = pat.replace("{{channel}}", channel) if not pat: return True # special case a presence testing elif regexp: diff --git a/src/etc/natvis/intrinsic.natvis b/src/etc/natvis/intrinsic.natvis index 030892a432..89280149a0 100644 --- a/src/etc/natvis/intrinsic.natvis +++ b/src/etc/natvis/intrinsic.natvis @@ -149,4 +149,57 @@ ... + + + {tag(),en} + {tag(),en} + {tag(),en} + {tag(),en} + {tag(),en} + {tag(),en} + {tag(),en} + {tag(),en} + {tag(),en} + {tag(),en} + {tag(),en} + {tag(),en} + {tag(),en} + {tag(),en} + {tag(),en} + {tag(),en} + + + variant0 + variant1 + variant2 + variant3 + variant4 + variant5 + variant6 + variant7 + variant8 + variant9 + variant10 + variant11 + variant12 + variant13 + variant14 + variant15 + + + + + + + + {"$T4",sb}({dataful_variant}) + {discriminant,en} + + dataful_variant + + {"$T4",sb} + + + diff --git a/src/etc/natvis/libcore.natvis b/src/etc/natvis/libcore.natvis index 9c3c26f597..1766777052 100644 --- a/src/etc/natvis/libcore.natvis +++ b/src/etc/natvis/libcore.natvis @@ -14,14 +14,6 @@ - - None - Some({__0}) - - __0 - - - None Some({($T1 *)this}) @@ -30,15 +22,6 @@ - - Ok({__0}) - Err({(*($T2*) &__0)}) - - __0 - (*($T2*) &__0) - - - {(void*) pointer} diff --git a/src/librustdoc/Cargo.toml b/src/librustdoc/Cargo.toml index b816b9d718..5ce7594945 100644 --- a/src/librustdoc/Cargo.toml +++ b/src/librustdoc/Cargo.toml @@ -10,7 +10,7 @@ path = "lib.rs" [dependencies] arrayvec = { version = "0.7", default-features = false } pulldown-cmark = { version = "0.8", default-features = false } -minifier = "0.0.39" +minifier = "0.0.41" rayon = { version = "0.3.0", package = "rustc-rayon" } serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" diff --git a/src/librustdoc/clean/auto_trait.rs b/src/librustdoc/clean/auto_trait.rs index 92eb6214f7..a3f63ea104 100644 --- a/src/librustdoc/clean/auto_trait.rs +++ b/src/librustdoc/clean/auto_trait.rs @@ -113,12 +113,11 @@ impl<'a, 'tcx> AutoTraitFinder<'a, 'tcx> { name: None, attrs: Default::default(), visibility: Inherited, - def_id: self.cx.next_def_id(item_def_id.krate), + def_id: FakeDefId::new_fake(item_def_id.krate), kind: box ImplItem(Impl { span: Span::dummy(), unsafety: hir::Unsafety::Normal, generics: new_generics, - provided_trait_methods: Default::default(), trait_: Some(trait_ref.clean(self.cx).get_trait_type().unwrap()), for_: ty.clean(self.cx), items: Vec::new(), @@ -562,7 +561,7 @@ impl<'a, 'tcx> AutoTraitFinder<'a, 'tcx> { } WherePredicate::EqPredicate { lhs, rhs } => { match lhs { - Type::QPath { name: left_name, ref self_type, ref trait_ } => { + Type::QPath { name: left_name, ref self_type, ref trait_, .. } => { let ty = &*self_type; match **trait_ { Type::ResolvedPath { @@ -665,7 +664,10 @@ impl<'a, 'tcx> AutoTraitFinder<'a, 'tcx> { } } GenericParamDefKind::Lifetime => {} - GenericParamDefKind::Const { .. } => {} + GenericParamDefKind::Const { ref mut default, .. } => { + // We never want something like `impl` + default.take(); + } } } diff --git a/src/librustdoc/clean/blanket_impl.rs b/src/librustdoc/clean/blanket_impl.rs index 3a14a1d23f..3f2fae1aca 100644 --- a/src/librustdoc/clean/blanket_impl.rs +++ b/src/librustdoc/clean/blanket_impl.rs @@ -1,6 +1,5 @@ use crate::rustc_trait_selection::traits::query::evaluate_obligation::InferCtxtExt; use rustc_hir as hir; -use rustc_hir::def_id::LOCAL_CRATE; use rustc_infer::infer::{InferOk, TyCtxtInferExt}; use rustc_infer::traits; use rustc_middle::ty::subst::Subst; @@ -20,7 +19,7 @@ impl<'a, 'tcx> BlanketImplFinder<'a, 'tcx> { debug!("get_blanket_impls({:?})", ty); let mut impls = Vec::new(); - for &trait_def_id in self.cx.tcx.all_traits(LOCAL_CRATE).iter() { + for &trait_def_id in self.cx.tcx.all_traits(()).iter() { if !self.cx.cache.access_levels.is_public(trait_def_id) || self.cx.generated_synthetics.get(&(ty, trait_def_id)).is_some() { @@ -92,18 +91,12 @@ impl<'a, 'tcx> BlanketImplFinder<'a, 'tcx> { } self.cx.generated_synthetics.insert((ty, trait_def_id)); - let provided_trait_methods = self - .cx - .tcx - .provided_trait_methods(trait_def_id) - .map(|meth| meth.ident.name) - .collect(); impls.push(Item { name: None, attrs: Default::default(), visibility: Inherited, - def_id: self.cx.next_def_id(impl_def_id.krate), + def_id: FakeDefId::new_fake(item_def_id.krate), kind: box ImplItem(Impl { span: self.cx.tcx.def_span(impl_def_id).clean(self.cx), unsafety: hir::Unsafety::Normal, @@ -112,7 +105,6 @@ impl<'a, 'tcx> BlanketImplFinder<'a, 'tcx> { self.cx.tcx.explicit_predicates_of(impl_def_id), ) .clean(self.cx), - provided_trait_methods, // FIXME(eddyb) compute both `trait_` and `for_` from // the post-inference `trait_ref`, as it's more accurate. trait_: Some(trait_ref.clean(self.cx).get_trait_type().unwrap()), @@ -126,7 +118,7 @@ impl<'a, 'tcx> BlanketImplFinder<'a, 'tcx> { .clean(self.cx), negative_polarity: false, synthetic: false, - blanket_impl: Some(trait_ref.self_ty().clean(self.cx)), + blanket_impl: Some(box trait_ref.self_ty().clean(self.cx)), }), cfg: None, }); diff --git a/src/librustdoc/clean/inline.rs b/src/librustdoc/clean/inline.rs index 3e89c1ac4c..111827aacd 100644 --- a/src/librustdoc/clean/inline.rs +++ b/src/librustdoc/clean/inline.rs @@ -15,11 +15,11 @@ use rustc_span::hygiene::MacroKind; use rustc_span::symbol::{kw, sym, Symbol}; use rustc_span::Span; -use crate::clean::{self, Attributes, AttributesExt, GetDefId, ToSource}; +use crate::clean::{self, Attributes, AttributesExt, FakeDefId, GetDefId, ToSource}; use crate::core::DocContext; use crate::formats::item_type::ItemType; -use super::Clean; +use super::{Clean, Visibility}; type Attrs<'hir> = rustc_middle::ty::Attributes<'hir>; @@ -121,7 +121,7 @@ crate fn try_inline( }; let (attrs, cfg) = merge_attrs(cx, Some(parent_module), load_attrs(cx, did), attrs_clone); - cx.inlined.insert(did); + cx.inlined.insert(did.into()); ret.push(clean::Item::from_def_id_and_attrs_and_parts( did, Some(name), @@ -188,13 +188,23 @@ crate fn record_extern_fqn(cx: &mut DocContext<'_>, did: DefId, kind: ItemType) if did.is_local() { cx.cache.exact_paths.insert(did, fqn); } else { - cx.cache.external_paths.insert(did, (fqn, ItemType::from(kind))); + cx.cache.external_paths.insert(did, (fqn, kind)); } } crate fn build_external_trait(cx: &mut DocContext<'_>, did: DefId) -> clean::Trait { - let trait_items = - cx.tcx.associated_items(did).in_definition_order().map(|item| item.clean(cx)).collect(); + let trait_items = cx + .tcx + .associated_items(did) + .in_definition_order() + .map(|item| { + // When building an external trait, the cleaned trait will have all items public, + // which causes methods to have a `pub` prefix, which is invalid since items in traits + // can not have a visibility prefix. Thus we override the visibility here manually. + // See https://github.com/rust-lang/rust/issues/81274 + clean::Item { visibility: Visibility::Inherited, ..item.clean(cx) } + }) + .collect(); let predicates = cx.tcx.predicates_of(did); let generics = (cx.tcx.generics_of(did), predicates).clean(cx); @@ -307,10 +317,10 @@ fn merge_attrs( } else { Attributes::from_ast(&both, None) }, - both.cfg(cx.sess().diagnostic()), + both.cfg(cx.sess()), ) } else { - (old_attrs.clean(cx), old_attrs.cfg(cx.sess().diagnostic())) + (old_attrs.clean(cx), old_attrs.cfg(cx.sess())) } } @@ -322,7 +332,7 @@ crate fn build_impl( attrs: Option>, ret: &mut Vec, ) { - if !cx.inlined.insert(did) { + if !cx.inlined.insert(did.into()) { return; } @@ -414,16 +424,10 @@ crate fn build_impl( record_extern_trait(cx, trait_did); } - let provided = trait_ - .def_id() - .map(|did| tcx.provided_trait_methods(did).map(|meth| meth.ident.name).collect()) - .unwrap_or_default(); - - debug!("build_impl: impl {:?} for {:?}", trait_.def_id(), for_.def_id()); - let (merged_attrs, cfg) = merge_attrs(cx, parent_module.into(), load_attrs(cx, did), attrs); debug!("merged_attrs={:?}", merged_attrs); + debug!("build_impl: impl {:?} for {:?}", trait_.def_id(), for_.def_id()); ret.push(clean::Item::from_def_id_and_attrs_and_parts( did, None, @@ -431,7 +435,6 @@ crate fn build_impl( span: clean::types::rustc_span(did, cx.tcx), unsafety: hir::Unsafety::Normal, generics, - provided_trait_methods: provided, trait_, for_, items: trait_items, @@ -467,7 +470,7 @@ fn build_module( items.push(clean::Item { name: None, attrs: box clean::Attributes::default(), - def_id: cx.next_def_id(did.krate), + def_id: FakeDefId::new_fake(did.krate), visibility: clean::Public, kind: box clean::ImportItem(clean::Import::new_simple( item.ident.name, @@ -524,7 +527,7 @@ fn build_static(cx: &mut DocContext<'_>, did: DefId, mutable: bool) -> clean::St } fn build_macro(cx: &mut DocContext<'_>, did: DefId, name: Symbol) -> clean::ItemKind { - let imported_from = cx.tcx.original_crate_name(did.krate); + let imported_from = cx.tcx.crate_name(did.krate); match cx.enter_resolver(|r| r.cstore().load_macro_untracked(did, cx.sess())) { LoadedMacro::MacroDef(def, _) => { let matchers: Vec = if let ast::ItemKind::MacroDef(ref def) = def.kind { @@ -585,6 +588,7 @@ fn filter_non_trait_generics(trait_did: DefId, mut g: clean::Generics) -> clean: self_type: box clean::Generic(ref s), trait_: box clean::ResolvedPath { did, .. }, name: ref _name, + .. }, ref bounds, } => !(bounds.is_empty() || *s == kw::SelfUpper && did == trait_did), diff --git a/src/librustdoc/clean/mod.rs b/src/librustdoc/clean/mod.rs index 6563f398ed..d1c18821ea 100644 --- a/src/librustdoc/clean/mod.rs +++ b/src/librustdoc/clean/mod.rs @@ -17,11 +17,11 @@ use rustc_hir::def::{CtorKind, DefKind, Res}; use rustc_hir::def_id::{CrateNum, DefId, CRATE_DEF_INDEX, LOCAL_CRATE}; use rustc_index::vec::{Idx, IndexVec}; use rustc_infer::infer::region_constraints::{Constraint, RegionConstraintData}; -use rustc_middle::bug; use rustc_middle::middle::resolve_lifetime as rl; use rustc_middle::ty::fold::TypeFolder; use rustc_middle::ty::subst::{InternalSubsts, Subst}; use rustc_middle::ty::{self, AdtKind, Lift, Ty, TyCtxt}; +use rustc_middle::{bug, span_bug}; use rustc_mir::const_eval::{is_const_fn, is_unstable_const_fn}; use rustc_span::hygiene::{AstPass, MacroKind}; use rustc_span::symbol::{kw, sym, Ident, Symbol}; @@ -100,12 +100,13 @@ impl Clean for doctree::Module<'_> { // determine if we should display the inner contents or // the outer `mod` item for the source code. let span = Span::from_rustc_span({ + let where_outer = self.where_outer(cx.tcx); let sm = cx.sess().source_map(); - let outer = sm.lookup_char_pos(self.where_outer.lo()); + let outer = sm.lookup_char_pos(where_outer.lo()); let inner = sm.lookup_char_pos(self.where_inner.lo()); if outer.file.start_pos == inner.file.start_pos { // mod foo { ... } - self.where_outer + where_outer } else { // mod foo; (and a separate SourceFile for the contents) self.where_inner @@ -157,7 +158,15 @@ impl Clean for hir::GenericBound<'_> { impl Clean for (ty::TraitRef<'_>, &[TypeBinding]) { fn clean(&self, cx: &mut DocContext<'_>) -> Type { let (trait_ref, bounds) = *self; - inline::record_extern_fqn(cx, trait_ref.def_id, ItemType::Trait); + let kind = cx.tcx.def_kind(trait_ref.def_id).into(); + if !matches!(kind, ItemType::Trait | ItemType::TraitAlias) { + span_bug!( + cx.tcx.def_span(trait_ref.def_id), + "`TraitRef` had unexpected kind {:?}", + kind + ); + } + inline::record_extern_fqn(cx, trait_ref.def_id, kind); let path = external_path( cx, cx.tcx.item_name(trait_ref.def_id), @@ -409,9 +418,11 @@ impl<'tcx> Clean for ty::ProjectionTy<'tcx> { GenericBound::TraitBound(t, _) => t.trait_, GenericBound::Outlives(_) => panic!("cleaning a trait got a lifetime"), }; + let self_type = self.self_ty().clean(cx); Type::QPath { name: cx.tcx.associated_item(self.item_def_id).ident.name, - self_type: box self.self_ty().clean(cx), + self_def_id: self_type.def_id(), + self_type: box self_type, trait_: box trait_, } } @@ -434,11 +445,15 @@ impl Clean for ty::GenericParamDef { }, ) } - ty::GenericParamDefKind::Const { .. } => ( + ty::GenericParamDefKind::Const { has_default, .. } => ( self.name, GenericParamDefKind::Const { did: self.def_id, ty: cx.tcx.type_of(self.def_id).clean(cx), + default: match has_default { + true => Some(cx.tcx.const_param_default(self.def_id).to_string()), + false => None, + }, }, ), }; @@ -476,12 +491,15 @@ impl Clean for hir::GenericParam<'_> { synthetic, }, ), - hir::GenericParamKind::Const { ref ty, default: _ } => ( + hir::GenericParamKind::Const { ref ty, default } => ( self.name.ident().name, GenericParamDefKind::Const { did: cx.tcx.hir().local_def_id(self.hir_id).to_def_id(), ty: ty.clean(cx), - // FIXME(const_generics_defaults): add `default` field here for docs + default: default.map(|ct| { + let def_id = cx.tcx.hir().local_def_id(ct.hir_id); + ty::Const::from_anon_const(cx.tcx, def_id).to_string() + }), }, ), }; @@ -654,10 +672,11 @@ impl<'a, 'tcx> Clean for (&'a ty::Generics, ty::GenericPredicates<'tcx if let Some(((_, trait_did, name), rhs)) = proj.as_ref().and_then(|(lhs, rhs)| Some((lhs.projection()?, rhs))) { - impl_trait_proj - .entry(param_idx) - .or_default() - .push((trait_did, name, rhs)); + impl_trait_proj.entry(param_idx).or_default().push(( + trait_did.into(), + name, + rhs, + )); } return None; @@ -1094,7 +1113,7 @@ impl Clean for ty::AssocItem { .filter_map(|pred| { let (name, self_type, trait_, bounds) = match *pred { WherePredicate::BoundPredicate { - ty: QPath { ref name, ref self_type, ref trait_ }, + ty: QPath { ref name, ref self_type, ref trait_, .. }, ref bounds, } => (name, self_type, trait_, bounds), _ => return None, @@ -1272,16 +1291,15 @@ fn clean_qpath(hir_ty: &hir::Ty<'_>, cx: &mut DocContext<'_>) -> Type { let segments = if p.is_global() { &p.segments[1..] } else { &p.segments }; let trait_segments = &segments[..segments.len() - 1]; + let trait_def = cx.tcx.associated_item(p.res.def_id()).container.id(); let trait_path = self::Path { global: p.is_global(), - res: Res::Def( - DefKind::Trait, - cx.tcx.associated_item(p.res.def_id()).container.id(), - ), + res: Res::Def(DefKind::Trait, trait_def), segments: trait_segments.clean(cx), }; Type::QPath { name: p.segments.last().expect("segments were empty").ident.name, + self_def_id: Some(DefId::local(qself.hir_id.owner.local_def_index)), self_type: box qself.clean(cx), trait_: box resolve_type(cx, trait_path, hir_id), } @@ -1296,6 +1314,7 @@ fn clean_qpath(hir_ty: &hir::Ty<'_>, cx: &mut DocContext<'_>) -> Type { let trait_path = hir::Path { span, res, segments: &[] }.clean(cx); Type::QPath { name: segment.ident.name, + self_def_id: res.opt_def_id(), self_type: box qself.clean(cx), trait_: box resolve_type(cx, trait_path, hir_id), } @@ -1930,11 +1949,6 @@ fn clean_impl(impl_: &hir::Impl<'_>, hir_id: hir::HirId, cx: &mut DocContext<'_> build_deref_target_impls(cx, &items, &mut ret); } - let provided: FxHashSet = trait_ - .def_id() - .map(|did| tcx.provided_trait_methods(did).map(|meth| meth.ident.name).collect()) - .unwrap_or_default(); - let for_ = impl_.self_ty.clean(cx); let type_alias = for_.def_id().and_then(|did| match tcx.def_kind(did) { DefKind::TyAlias => Some(tcx.type_of(did).clean(cx)), @@ -1945,7 +1959,6 @@ fn clean_impl(impl_: &hir::Impl<'_>, hir_id: hir::HirId, cx: &mut DocContext<'_> span: types::rustc_span(tcx.hir().local_def_id(hir_id).to_def_id(), tcx), unsafety: impl_.unsafety, generics: impl_.generics.clean(cx), - provided_trait_methods: provided.clone(), trait_, for_, items, @@ -2003,10 +2016,10 @@ fn clean_extern_crate( vec![Item { name: Some(name), attrs: box attrs.clean(cx), - def_id: crate_def_id, + def_id: crate_def_id.into(), visibility: krate.vis.clean(cx), kind: box ExternCrateItem { src: orig_name }, - cfg: attrs.cfg(cx.sess().diagnostic()), + cfg: attrs.cfg(cx.sess()), }] } diff --git a/src/librustdoc/clean/types.rs b/src/librustdoc/clean/types.rs index ad4a09d45f..6a7c3f8caa 100644 --- a/src/librustdoc/clean/types.rs +++ b/src/librustdoc/clean/types.rs @@ -1,9 +1,9 @@ -use std::cell::RefCell; +use std::cell::{Cell, RefCell}; use std::default::Default; -use std::fmt; use std::hash::{Hash, Hasher}; use std::iter::FromIterator; use std::lazy::SyncOnceCell as OnceCell; +use std::path::PathBuf; use std::rc::Rc; use std::sync::Arc; use std::{slice, vec}; @@ -18,7 +18,7 @@ use rustc_data_structures::fx::{FxHashMap, FxHashSet}; use rustc_data_structures::thin_vec::ThinVec; use rustc_hir as hir; use rustc_hir::def::{CtorKind, DefKind, Res}; -use rustc_hir::def_id::{CrateNum, DefId, DefIndex, CRATE_DEF_INDEX}; +use rustc_hir::def_id::{CrateNum, DefId, DefIndex, CRATE_DEF_INDEX, LOCAL_CRATE}; use rustc_hir::lang_items::LangItem; use rustc_hir::{BodyId, Mutability}; use rustc_index::vec::IndexVec; @@ -48,7 +48,75 @@ use self::ItemKind::*; use self::SelfTy::*; use self::Type::*; -thread_local!(crate static MAX_DEF_IDX: RefCell> = Default::default()); +crate type FakeDefIdSet = FxHashSet; + +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Copy)] +crate enum FakeDefId { + Real(DefId), + Fake(DefIndex, CrateNum), +} + +impl FakeDefId { + #[cfg(parallel_compiler)] + crate fn new_fake(crate: CrateNum) -> Self { + unimplemented!("") + } + + #[cfg(not(parallel_compiler))] + crate fn new_fake(krate: CrateNum) -> Self { + thread_local!(static FAKE_DEF_ID_COUNTER: Cell = Cell::new(0)); + let id = FAKE_DEF_ID_COUNTER.with(|id| { + let tmp = id.get(); + id.set(tmp + 1); + tmp + }); + Self::Fake(DefIndex::from(id), krate) + } + + #[inline] + crate fn is_local(self) -> bool { + match self { + FakeDefId::Real(id) => id.is_local(), + FakeDefId::Fake(_, krate) => krate == LOCAL_CRATE, + } + } + + #[inline] + #[track_caller] + crate fn expect_real(self) -> rustc_hir::def_id::DefId { + self.as_real().unwrap_or_else(|| panic!("FakeDefId::expect_real: `{:?}` isn't real", self)) + } + + #[inline] + crate fn as_real(self) -> Option { + match self { + FakeDefId::Real(id) => Some(id), + FakeDefId::Fake(_, _) => None, + } + } + + #[inline] + crate fn krate(self) -> CrateNum { + match self { + FakeDefId::Real(id) => id.krate, + FakeDefId::Fake(_, krate) => krate, + } + } + + #[inline] + crate fn index(self) -> Option { + match self { + FakeDefId::Real(id) => Some(id.index), + FakeDefId::Fake(_, _) => None, + } + } +} + +impl From for FakeDefId { + fn from(id: DefId) -> Self { + Self::Real(id) + } +} #[derive(Clone, Debug)] crate struct Crate { @@ -90,6 +158,58 @@ impl ExternalCrate { tcx.crate_name(self.crate_num) } + crate fn src_root(&self, tcx: TyCtxt<'_>) -> PathBuf { + match self.src(tcx) { + FileName::Real(ref p) => match p.local_path_if_available().parent() { + Some(p) => p.to_path_buf(), + None => PathBuf::new(), + }, + _ => PathBuf::new(), + } + } + + /// Attempts to find where an external crate is located, given that we're + /// rendering in to the specified source destination. + crate fn location( + &self, + extern_url: Option<&str>, + dst: &std::path::Path, + tcx: TyCtxt<'_>, + ) -> ExternalLocation { + use ExternalLocation::*; + + fn to_remote(url: impl ToString) -> ExternalLocation { + let mut url = url.to_string(); + if !url.ends_with('/') { + url.push('/'); + } + Remote(url) + } + + // See if there's documentation generated into the local directory + // WARNING: since rustdoc creates these directories as it generates documentation, this check is only accurate before rendering starts. + // Make sure to call `location()` by that time. + let local_location = dst.join(&*self.name(tcx).as_str()); + if local_location.is_dir() { + return Local; + } + + if let Some(url) = extern_url { + return to_remote(url); + } + + // Failing that, see if there's an attribute specifying where to find this + // external crate + let did = DefId { krate: self.crate_num, index: CRATE_DEF_INDEX }; + tcx.get_attrs(did) + .lists(sym::doc) + .filter(|a| a.has_name(sym::html_root_url)) + .filter_map(|a| a.value_str()) + .map(to_remote) + .next() + .unwrap_or(Unknown) // Well, at least we tried. + } + crate fn keywords(&self, tcx: TyCtxt<'_>) -> ThinVec<(DefId, Symbol)> { let root = self.def_id(); @@ -208,7 +328,7 @@ impl ExternalCrate { /// Anything with a source location and set of attributes and, optionally, a /// name. That is, anything that can be documented. This doesn't correspond /// directly to the AST's concept of an item; it's a strict superset. -#[derive(Clone)] +#[derive(Clone, Debug)] crate struct Item { /// The name of this item. /// Optional because not every item has a name, e.g. impls. @@ -218,7 +338,7 @@ crate struct Item { /// Information about this item that is specific to what kind of item it is. /// E.g., struct vs enum vs function. crate kind: Box, - crate def_id: DefId, + crate def_id: FakeDefId, crate cfg: Option>, } @@ -227,21 +347,6 @@ crate struct Item { #[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))] rustc_data_structures::static_assert_size!(Item, 48); -impl fmt::Debug for Item { - fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { - let def_id: &dyn fmt::Debug = if self.is_fake() { &"**FAKE**" } else { &self.def_id }; - - fmt.debug_struct("Item") - .field("name", &self.name) - .field("attrs", &self.attrs) - .field("kind", &self.kind) - .field("visibility", &self.visibility) - .field("def_id", def_id) - .field("cfg", &self.cfg) - .finish() - } -} - crate fn rustc_span(def_id: DefId, tcx: TyCtxt<'_>) -> Span { Span::from_rustc_span(def_id.as_local().map_or_else( || tcx.def_span(def_id), @@ -254,19 +359,19 @@ crate fn rustc_span(def_id: DefId, tcx: TyCtxt<'_>) -> Span { impl Item { crate fn stability<'tcx>(&self, tcx: TyCtxt<'tcx>) -> Option<&'tcx Stability> { - if self.is_fake() { None } else { tcx.lookup_stability(self.def_id) } + if self.is_fake() { None } else { tcx.lookup_stability(self.def_id.expect_real()) } } crate fn const_stability<'tcx>(&self, tcx: TyCtxt<'tcx>) -> Option<&'tcx ConstStability> { - if self.is_fake() { None } else { tcx.lookup_const_stability(self.def_id) } + if self.is_fake() { None } else { tcx.lookup_const_stability(self.def_id.expect_real()) } } crate fn deprecation(&self, tcx: TyCtxt<'_>) -> Option { - if self.is_fake() { None } else { tcx.lookup_deprecation(self.def_id) } + if self.is_fake() { None } else { tcx.lookup_deprecation(self.def_id.expect_real()) } } crate fn inner_docs(&self, tcx: TyCtxt<'_>) -> bool { - if self.is_fake() { false } else { tcx.get_attrs(self.def_id).inner_docs() } + if self.is_fake() { false } else { tcx.get_attrs(self.def_id.expect_real()).inner_docs() } } crate fn span(&self, tcx: TyCtxt<'_>) -> Span { @@ -281,7 +386,7 @@ impl Item { } else if self.is_fake() { Span::dummy() } else { - rustc_span(self.def_id, tcx) + rustc_span(self.def_id.expect_real(), tcx) } } @@ -320,7 +425,7 @@ impl Item { kind, box ast_attrs.clean(cx), cx, - ast_attrs.cfg(cx.sess().diagnostic()), + ast_attrs.cfg(cx.sess()), ) } @@ -332,10 +437,10 @@ impl Item { cx: &mut DocContext<'_>, cfg: Option>, ) -> Item { - debug!("name={:?}, def_id={:?}", name, def_id); + trace!("name={:?}, def_id={:?}", name, def_id); Item { - def_id, + def_id: def_id.into(), kind: box kind, name, attrs, @@ -359,9 +464,9 @@ impl Item { .map_or(&[][..], |v| v.as_slice()) .iter() .filter_map(|ItemLink { link: s, link_text, did, ref fragment }| { - match *did { + match did { Some(did) => { - if let Some((mut href, ..)) = href(did, cx) { + if let Some((mut href, ..)) = href(did.clone(), cx) { if let Some(ref fragment) = *fragment { href.push('#'); href.push_str(fragment); @@ -380,8 +485,8 @@ impl Item { None => { let relative_to = &cx.current; if let Some(ref fragment) = *fragment { - let url = match cx.cache().extern_locations.get(&self.def_id.krate) { - Some(&(_, _, ExternalLocation::Local)) => { + let url = match cx.cache().extern_locations.get(&self.def_id.krate()) { + Some(&ExternalLocation::Local) => { if relative_to[0] == "std" { let depth = relative_to.len() - 1; "../".repeat(depth) @@ -390,11 +495,11 @@ impl Item { format!("{}std/", "../".repeat(depth)) } } - Some(&(_, _, ExternalLocation::Remote(ref s))) => { + Some(ExternalLocation::Remote(ref s)) => { format!("{}/std/", s.trim_end_matches('/')) } - Some(&(_, _, ExternalLocation::Unknown)) | None => { - "https://doc.rust-lang.org/nightly/std/".to_string() + Some(ExternalLocation::Unknown) | None => { + format!("{}/std/", crate::DOC_RUST_LANG_ORG_CHANNEL) } }; // This is a primitive so the url is done "by hand". @@ -419,9 +524,8 @@ impl Item { } crate fn is_crate(&self) -> bool { - self.is_mod() && self.def_id.index == CRATE_DEF_INDEX + self.is_mod() && self.def_id.as_real().map_or(false, |did| did.index == CRATE_DEF_INDEX) } - crate fn is_mod(&self) -> bool { self.type_() == ItemType::Module } @@ -532,13 +636,8 @@ impl Item { } } - /// See the documentation for [`next_def_id()`]. - /// - /// [`next_def_id()`]: DocContext::next_def_id() crate fn is_fake(&self) -> bool { - MAX_DEF_IDX.with(|m| { - m.borrow().get(&self.def_id.krate).map(|&idx| idx <= self.def_id.index).unwrap_or(false) - }) + matches!(self.def_id, FakeDefId::Fake(_, _)) } } @@ -624,10 +723,6 @@ impl ItemKind { | KeywordItem(_) => [].iter(), } } - - crate fn is_type_alias(&self) -> bool { - matches!(self, ItemKind::TypedefItem(..) | ItemKind::AssocTypeItem(..)) - } } #[derive(Clone, Debug)] @@ -680,7 +775,7 @@ crate trait AttributesExt { fn other_attrs(&self) -> Vec; - fn cfg(&self, diagnostic: &::rustc_errors::Handler) -> Option>; + fn cfg(&self, sess: &Session) -> Option>; } impl AttributesExt for [ast::Attribute] { @@ -705,17 +800,28 @@ impl AttributesExt for [ast::Attribute] { self.iter().filter(|attr| attr.doc_str().is_none()).cloned().collect() } - fn cfg(&self, diagnostic: &::rustc_errors::Handler) -> Option> { + fn cfg(&self, sess: &Session) -> Option> { let mut cfg = Cfg::True; for attr in self.iter() { + // #[doc] if attr.doc_str().is_none() && attr.has_name(sym::doc) { - if let Some(mi) = attr.meta() { - if let Some(cfg_mi) = Attributes::extract_cfg(&mi) { - // Extracted #[doc(cfg(...))] - match Cfg::parse(cfg_mi) { - Ok(new_cfg) => cfg &= new_cfg, - Err(e) => diagnostic.span_err(e.span, e.msg), + // #[doc(...)] + if let Some(list) = attr.meta().as_ref().and_then(|mi| mi.meta_item_list()) { + for item in list { + // #[doc(hidden)] + if !item.has_name(sym::cfg) { + continue; + } + // #[doc(cfg(...))] + if let Some(cfg_mi) = item + .meta_item() + .and_then(|item| rustc_expand::config::parse_cfg(&item, sess)) + { + match Cfg::parse(&cfg_mi) { + Ok(new_cfg) => cfg &= new_cfg, + Err(e) => sess.span_err(e.span, e.msg), + } } } } @@ -788,9 +894,6 @@ crate enum DocFragmentKind { SugaredDoc, /// A doc fragment created from a "raw" `#[doc=""]` attribute. RawDoc, - /// A doc fragment created from a `#[doc(include="filename")]` attribute. Contains both the - /// given filename and the file contents. - Include { filename: Symbol }, } // The goal of this function is to apply the `DocFragment` transformations that are required when @@ -824,18 +927,8 @@ impl<'a> FromIterator<&'a DocFragment> for String { where T: IntoIterator, { - let mut prev_kind: Option = None; iter.into_iter().fold(String::new(), |mut acc, frag| { - if !acc.is_empty() - && prev_kind - .take() - .map(|p| matches!(p, DocFragmentKind::Include { .. }) && p != frag.kind) - .unwrap_or(false) - { - acc.push('\n'); - } add_doc_fragment(&mut acc, &frag); - prev_kind = Some(frag.kind); acc }) } @@ -882,68 +975,6 @@ impl Attributes { self.other_attrs.lists(name) } - /// Extracts the content from an attribute `#[doc(cfg(content))]`. - crate fn extract_cfg(mi: &ast::MetaItem) -> Option<&ast::MetaItem> { - use rustc_ast::NestedMetaItem::MetaItem; - - if let ast::MetaItemKind::List(ref nmis) = mi.kind { - if nmis.len() == 1 { - if let MetaItem(ref cfg_mi) = nmis[0] { - if cfg_mi.has_name(sym::cfg) { - if let ast::MetaItemKind::List(ref cfg_nmis) = cfg_mi.kind { - if cfg_nmis.len() == 1 { - if let MetaItem(ref content_mi) = cfg_nmis[0] { - return Some(content_mi); - } - } - } - } - } - } - } - - None - } - - /// Reads a `MetaItem` from within an attribute, looks for whether it is a - /// `#[doc(include="file")]`, and returns the filename and contents of the file as loaded from - /// its expansion. - crate fn extract_include(mi: &ast::MetaItem) -> Option<(Symbol, Symbol)> { - mi.meta_item_list().and_then(|list| { - for meta in list { - if meta.has_name(sym::include) { - // the actual compiled `#[doc(include="filename")]` gets expanded to - // `#[doc(include(file="filename", contents="file contents")]` so we need to - // look for that instead - return meta.meta_item_list().and_then(|list| { - let mut filename: Option = None; - let mut contents: Option = None; - - for it in list { - if it.has_name(sym::file) { - if let Some(name) = it.value_str() { - filename = Some(name); - } - } else if it.has_name(sym::contents) { - if let Some(docs) = it.value_str() { - contents = Some(docs); - } - } - } - - if let (Some(filename), Some(contents)) = (filename, contents) { - Some((filename, contents)) - } else { - None - } - }); - } - } - - None - }) - } - crate fn has_doc_flag(&self, flag: Symbol) -> bool { for attr in &self.other_attrs { if !attr.has_name(sym::doc) { @@ -967,22 +998,13 @@ impl Attributes { let mut doc_strings: Vec = vec![]; let mut doc_line = 0; - fn update_need_backline(doc_strings: &mut Vec, frag: &DocFragment) { + fn update_need_backline(doc_strings: &mut Vec) { if let Some(prev) = doc_strings.last_mut() { - if matches!(prev.kind, DocFragmentKind::Include { .. }) - || prev.kind != frag.kind - || prev.parent_module != frag.parent_module - { - // add a newline for extra padding between segments - prev.need_backline = prev.kind == DocFragmentKind::SugaredDoc - || prev.kind == DocFragmentKind::RawDoc - } else { - prev.need_backline = true; - } + prev.need_backline = true; } } - let clean_attr = |(attr, parent_module): (&ast::Attribute, _)| { + let clean_attr = |(attr, parent_module): (&ast::Attribute, Option)| { if let Some(value) = attr.doc_str() { trace!("got doc_str={:?}", value); let value = beautify_doc_string(value); @@ -1004,31 +1026,12 @@ impl Attributes { indent: 0, }; - update_need_backline(&mut doc_strings, &frag); + update_need_backline(&mut doc_strings); doc_strings.push(frag); None } else { - if attr.has_name(sym::doc) { - if let Some(mi) = attr.meta() { - if let Some((filename, contents)) = Attributes::extract_include(&mi) { - let line = doc_line; - doc_line += contents.as_str().lines().count(); - let frag = DocFragment { - line, - span: attr.span, - doc: contents, - kind: DocFragmentKind::Include { filename }, - parent_module, - need_backline: false, - indent: 0, - }; - update_need_backline(&mut doc_strings, &frag); - doc_strings.push(frag); - } - } - } Some(attr.clone()) } }; @@ -1054,10 +1057,7 @@ impl Attributes { let mut out = String::new(); add_doc_fragment(&mut out, &ori); while let Some(new_frag) = iter.next() { - if matches!(ori.kind, DocFragmentKind::Include { .. }) - || new_frag.kind != ori.kind - || new_frag.parent_module != ori.parent_module - { + if new_frag.kind != ori.kind || new_frag.parent_module != ori.parent_module { break; } add_doc_fragment(&mut out, &new_frag); @@ -1220,6 +1220,7 @@ crate enum GenericParamDefKind { Const { did: DefId, ty: Type, + default: Option, }, } @@ -1435,6 +1436,7 @@ crate enum Type { QPath { name: Symbol, self_type: Box, + self_def_id: Option, trait_: Box, }, @@ -1518,7 +1520,6 @@ impl Type { } } RawPointer(..) => Some(PrimitiveType::RawPointer), - BorrowedRef { type_: box Generic(..), .. } => Some(PrimitiveType::Reference), BareFunction(..) => Some(PrimitiveType::Fn), Never => Some(PrimitiveType::Never), _ => None, @@ -1577,18 +1578,12 @@ impl Type { } crate fn is_primitive(&self) -> bool { - match self { - Self::Primitive(_) => true, - Self::BorrowedRef { ref type_, .. } | Self::RawPointer(_, ref type_) => { - type_.is_primitive() - } - _ => false, - } + self.primitive_type().is_some() } crate fn projection(&self) -> Option<(&Type, DefId, Symbol)> { let (self_, trait_, name) = match self { - QPath { self_type, trait_, name } => (self_type, trait_, name), + QPath { self_type, trait_, name, .. } => (self_type, trait_, name), _ => return None, }; let trait_did = match **trait_ { @@ -1602,7 +1597,7 @@ impl Type { impl Type { fn inner_def_id(&self, cache: Option<&Cache>) -> Option { let t: PrimitiveType = match *self { - ResolvedPath { did, .. } => return Some(did), + ResolvedPath { did, .. } => return Some(did.into()), Primitive(p) => return cache.and_then(|c| c.primitive_locations.get(&p).cloned()), BorrowedRef { type_: box Generic(..), .. } => PrimitiveType::Reference, BorrowedRef { ref type_, .. } => return type_.inner_def_id(cache), @@ -1690,37 +1685,6 @@ impl PrimitiveType { } } - crate fn as_str(&self) -> &'static str { - use self::PrimitiveType::*; - match *self { - Isize => "isize", - I8 => "i8", - I16 => "i16", - I32 => "i32", - I64 => "i64", - I128 => "i128", - Usize => "usize", - U8 => "u8", - U16 => "u16", - U32 => "u32", - U64 => "u64", - U128 => "u128", - F32 => "f32", - F64 => "f64", - Str => "str", - Bool => "bool", - Char => "char", - Array => "array", - Slice => "slice", - Tuple => "tuple", - Unit => "unit", - RawPointer => "pointer", - Reference => "reference", - Fn => "fn", - Never => "never", - } - } - crate fn impls(&self, tcx: TyCtxt<'_>) -> &'static ArrayVec { Self::all_impls(tcx).get(self).expect("missing impl for primitive type") } @@ -1783,10 +1747,6 @@ impl PrimitiveType { }) } - crate fn to_url_str(&self) -> &'static str { - self.as_str() - } - crate fn as_sym(&self) -> Symbol { use PrimitiveType::*; match self { @@ -2149,13 +2109,21 @@ crate struct Impl { crate span: Span, crate unsafety: hir::Unsafety, crate generics: Generics, - crate provided_trait_methods: FxHashSet, crate trait_: Option, crate for_: Type, crate items: Vec, crate negative_polarity: bool, crate synthetic: bool, - crate blanket_impl: Option, + crate blanket_impl: Option>, +} + +impl Impl { + crate fn provided_trait_methods(&self, tcx: TyCtxt<'_>) -> FxHashSet { + self.trait_ + .def_id() + .map(|did| tcx.provided_trait_methods(did).map(|meth| meth.ident.name).collect()) + .unwrap_or_default() + } } #[derive(Clone, Debug)] diff --git a/src/librustdoc/clean/utils.rs b/src/librustdoc/clean/utils.rs index b5953d2dc4..706a56fbcb 100644 --- a/src/librustdoc/clean/utils.rs +++ b/src/librustdoc/clean/utils.rs @@ -2,7 +2,7 @@ use crate::clean::auto_trait::AutoTraitFinder; use crate::clean::blanket_impl::BlanketImplFinder; use crate::clean::{ inline, Clean, Crate, Generic, GenericArg, GenericArgs, ImportSource, Item, ItemKind, Lifetime, - MacroKind, Path, PathSegment, Primitive, PrimitiveType, ResolvedPath, Type, TypeBinding, + Path, PathSegment, Primitive, PrimitiveType, ResolvedPath, Type, TypeBinding, }; use crate::core::DocContext; use crate::formats::item_type::ItemType; @@ -16,6 +16,9 @@ use rustc_middle::ty::{self, DefIdTree, TyCtxt}; use rustc_span::symbol::{kw, sym, Symbol}; use std::mem; +#[cfg(test)] +mod tests; + crate fn krate(cx: &mut DocContext<'_>) -> Crate { use crate::visit_lib::LibEmbargoVisitor; @@ -45,9 +48,9 @@ crate fn krate(cx: &mut DocContext<'_>) -> Crate { // `#[doc(masked)]` to the injected `extern crate` because it's unstable. if it.is_extern_crate() && (it.attrs.has_doc_flag(sym::masked) - || cx.tcx.is_compiler_builtins(it.def_id.krate)) + || cx.tcx.is_compiler_builtins(it.def_id.krate())) { - cx.cache.masked_crates.insert(it.def_id.krate); + cx.cache.masked_crates.insert(it.def_id.krate()); } } } @@ -172,8 +175,9 @@ crate fn strip_type(ty: Type) -> Type { Type::BorrowedRef { lifetime, mutability, type_ } => { Type::BorrowedRef { lifetime, mutability, type_: Box::new(strip_type(*type_)) } } - Type::QPath { name, self_type, trait_ } => Type::QPath { + Type::QPath { name, self_type, trait_, self_def_id } => Type::QPath { name, + self_def_id, self_type: Box::new(strip_type(*self_type)), trait_: Box::new(strip_type(*trait_)), }, @@ -335,11 +339,27 @@ crate fn print_evaluated_const(tcx: TyCtxt<'_>, def_id: DefId) -> Option fn format_integer_with_underscore_sep(num: &str) -> String { let num_chars: Vec<_> = num.chars().collect(); - let num_start_index = if num_chars.get(0) == Some(&'-') { 1 } else { 0 }; + let mut num_start_index = if num_chars.get(0) == Some(&'-') { 1 } else { 0 }; + let chunk_size = match num[num_start_index..].as_bytes() { + [b'0', b'b' | b'x', ..] => { + num_start_index += 2; + 4 + } + [b'0', b'o', ..] => { + num_start_index += 2; + let remaining_chars = num_chars.len() - num_start_index; + if remaining_chars <= 6 { + // don't add underscores to Unix permissions like 0755 or 100755 + return num.to_string(); + } + 3 + } + _ => 3, + }; num_chars[..num_start_index] .iter() - .chain(num_chars[num_start_index..].rchunks(3).rev().intersperse(&['_']).flatten()) + .chain(num_chars[num_start_index..].rchunks(chunk_size).rev().intersperse(&['_']).flatten()) .collect() } @@ -431,35 +451,48 @@ crate fn get_auto_trait_and_blanket_impls( auto_impls.into_iter().chain(blanket_impls) } +/// If `res` has a documentation page associated, store it in the cache. +/// +/// This is later used by [`href()`] to determine the HTML link for the item. +/// +/// [`href()`]: crate::html::format::href crate fn register_res(cx: &mut DocContext<'_>, res: Res) -> DefId { + use DefKind::*; debug!("register_res({:?})", res); let (did, kind) = match res { - Res::Def(DefKind::Fn, i) => (i, ItemType::Function), - Res::Def(DefKind::TyAlias, i) => (i, ItemType::Typedef), - Res::Def(DefKind::Enum, i) => (i, ItemType::Enum), - Res::Def(DefKind::Trait, i) => (i, ItemType::Trait), Res::Def(DefKind::AssocTy | DefKind::AssocFn | DefKind::AssocConst, i) => { + // associated items are documented, but on the page of their parent (cx.tcx.parent(i).unwrap(), ItemType::Trait) } - Res::Def(DefKind::Struct, i) => (i, ItemType::Struct), - Res::Def(DefKind::Union, i) => (i, ItemType::Union), - Res::Def(DefKind::Mod, i) => (i, ItemType::Module), - Res::Def(DefKind::ForeignTy, i) => (i, ItemType::ForeignType), - Res::Def(DefKind::Const, i) => (i, ItemType::Constant), - Res::Def(DefKind::Static, i) => (i, ItemType::Static), Res::Def(DefKind::Variant, i) => { + // variant items are documented, but on the page of their parent (cx.tcx.parent(i).expect("cannot get parent def id"), ItemType::Enum) } - Res::Def(DefKind::Macro(mac_kind), i) => match mac_kind { - MacroKind::Bang => (i, ItemType::Macro), - MacroKind::Attr => (i, ItemType::ProcAttribute), - MacroKind::Derive => (i, ItemType::ProcDerive), - }, - Res::Def(DefKind::TraitAlias, i) => (i, ItemType::TraitAlias), - Res::SelfTy(Some(def_id), _) => (def_id, ItemType::Trait), - Res::SelfTy(_, Some((impl_def_id, _))) => return impl_def_id, - _ => return res.def_id(), + // Each of these have their own page. + Res::Def( + kind + @ + (Fn | TyAlias | Enum | Trait | Struct | Union | Mod | ForeignTy | Const | Static + | Macro(..) | TraitAlias), + i, + ) => (i, kind.into()), + // This is part of a trait definition; document the trait. + Res::SelfTy(Some(trait_def_id), _) => (trait_def_id, ItemType::Trait), + // This is an inherent impl; it doesn't have its own page. + Res::SelfTy(None, Some((impl_def_id, _))) => return impl_def_id, + Res::SelfTy(None, None) + | Res::PrimTy(_) + | Res::ToolMod + | Res::SelfCtor(_) + | Res::Local(_) + | Res::NonMacroAttr(_) + | Res::Err => return res.def_id(), + Res::Def( + TyParam | ConstParam | Ctor(..) | ExternCrate | Use | ForeignMod | AnonConst | OpaqueTy + | Field | LifetimeParam | GlobalAsm | Impl | Closure | Generator, + id, + ) => return id, }; if did.is_local() { return did; @@ -490,8 +523,6 @@ where } /// Find the nearest parent module of a [`DefId`]. -/// -/// **Panics if the item it belongs to [is fake][Item::is_fake].** crate fn find_nearest_parent_module(tcx: TyCtxt<'_>, def_id: DefId) -> Option { if def_id.is_top_level_module() { // The crate root has no parent. Use it as the root instead. @@ -525,3 +556,9 @@ crate fn has_doc_flag(attrs: ty::Attributes<'_>, flag: Symbol) -> bool { && attr.meta_item_list().map_or(false, |l| rustc_attr::list_contains_name(&l, flag)) }) } + +/// A link to `doc.rust-lang.org` that includes the channel name. Use this instead of manual links +/// so that the channel is consistent. +/// +/// Set by `bootstrap::Builder::doc_rust_lang_org_channel` in order to keep tests passing on beta/stable. +crate const DOC_RUST_LANG_ORG_CHANNEL: &'static str = env!("DOC_RUST_LANG_ORG_CHANNEL"); diff --git a/src/librustdoc/clean/utils/tests.rs b/src/librustdoc/clean/utils/tests.rs new file mode 100644 index 0000000000..ebf4b49548 --- /dev/null +++ b/src/librustdoc/clean/utils/tests.rs @@ -0,0 +1,41 @@ +use super::*; + +#[test] +fn int_format_decimal() { + assert_eq!(format_integer_with_underscore_sep("12345678"), "12_345_678"); + assert_eq!(format_integer_with_underscore_sep("123"), "123"); + assert_eq!(format_integer_with_underscore_sep("123459"), "123_459"); + assert_eq!(format_integer_with_underscore_sep("-12345678"), "-12_345_678"); + assert_eq!(format_integer_with_underscore_sep("-123"), "-123"); + assert_eq!(format_integer_with_underscore_sep("-123459"), "-123_459"); +} + +#[test] +fn int_format_hex() { + assert_eq!(format_integer_with_underscore_sep("0xab3"), "0xab3"); + assert_eq!(format_integer_with_underscore_sep("0xa2345b"), "0xa2_345b"); + assert_eq!(format_integer_with_underscore_sep("0xa2e6345b"), "0xa2e6_345b"); + assert_eq!(format_integer_with_underscore_sep("-0xab3"), "-0xab3"); + assert_eq!(format_integer_with_underscore_sep("-0xa2345b"), "-0xa2_345b"); + assert_eq!(format_integer_with_underscore_sep("-0xa2e6345b"), "-0xa2e6_345b"); +} + +#[test] +fn int_format_binary() { + assert_eq!(format_integer_with_underscore_sep("0o12345671"), "0o12_345_671"); + assert_eq!(format_integer_with_underscore_sep("0o123"), "0o123"); + assert_eq!(format_integer_with_underscore_sep("0o123451"), "0o123451"); + assert_eq!(format_integer_with_underscore_sep("-0o12345671"), "-0o12_345_671"); + assert_eq!(format_integer_with_underscore_sep("-0o123"), "-0o123"); + assert_eq!(format_integer_with_underscore_sep("-0o123451"), "-0o123451"); +} + +#[test] +fn int_format_octal() { + assert_eq!(format_integer_with_underscore_sep("0b101"), "0b101"); + assert_eq!(format_integer_with_underscore_sep("0b101101011"), "0b1_0110_1011"); + assert_eq!(format_integer_with_underscore_sep("0b01101011"), "0b0110_1011"); + assert_eq!(format_integer_with_underscore_sep("-0b101"), "-0b101"); + assert_eq!(format_integer_with_underscore_sep("-0b101101011"), "-0b1_0110_1011"); + assert_eq!(format_integer_with_underscore_sep("-0b01101011"), "-0b0110_1011"); +} diff --git a/src/librustdoc/config.rs b/src/librustdoc/config.rs index 8f10ab2d3a..1b5a00dde5 100644 --- a/src/librustdoc/config.rs +++ b/src/librustdoc/config.rs @@ -6,11 +6,10 @@ use std::path::PathBuf; use std::str::FromStr; use rustc_data_structures::fx::FxHashMap; -use rustc_session::config::{self, parse_crate_types_from_list, parse_externs, CrateType}; use rustc_session::config::{ - build_codegen_options, build_debugging_options, get_cmd_lint_options, host_triple, - nightly_options, + self, parse_crate_types_from_list, parse_externs, parse_target_triple, CrateType, }; +use rustc_session::config::{get_cmd_lint_options, nightly_options}; use rustc_session::config::{CodegenOptions, DebuggingOptions, ErrorOutputType, Externs}; use rustc_session::getopts; use rustc_session::lint::Level; @@ -120,6 +119,8 @@ crate struct Options { /// For example, using ignore-foo to ignore running the doctest on any target that /// contains "foo" as a substring crate enable_per_target_ignores: bool, + /// Do not run doctests, compile them if should_test is active. + crate no_run: bool, /// The path to a rustc-like binary to build tests with. If not set, we /// default to loading from `$sysroot/bin/rustc`. @@ -197,6 +198,7 @@ impl fmt::Debug for Options { .field("runtool_args", &self.runtool_args) .field("enable-per-target-ignores", &self.enable_per_target_ignores) .field("run_check", &self.run_check) + .field("no_run", &self.no_run) .finish() } } @@ -267,6 +269,8 @@ crate struct RenderOptions { crate document_hidden: bool, /// If `true`, generate a JSON file in the crate folder instead of HTML redirection files. crate generate_redirect_map: bool, + /// Show the memory layout of types in the docs. + crate show_type_layout: bool, crate unstable_features: rustc_feature::UnstableFeatures, crate emit: Vec, } @@ -345,20 +349,13 @@ impl Options { return Err(0); } - if matches.opt_strs("print").iter().any(|opt| opt == "unversioned-files") { - for file in crate::html::render::FILES_UNVERSIONED.keys() { - println!("{}", file); - } - return Err(0); - } - let color = config::parse_color(&matches); let config::JsonConfig { json_rendered, json_unused_externs, .. } = config::parse_json(&matches); let error_format = config::parse_error_format(&matches, color, json_rendered); - let codegen_options = build_codegen_options(matches, error_format); - let debugging_opts = build_debugging_options(matches, error_format); + let codegen_options = CodegenOptions::build(matches, error_format); + let debugging_opts = DebuggingOptions::build(matches, error_format); let diag = new_handler(error_format, None, &debugging_opts); @@ -466,6 +463,12 @@ impl Options { test_args.iter().flat_map(|s| s.split_whitespace()).map(|s| s.to_string()).collect(); let should_test = matches.opt_present("test"); + let no_run = matches.opt_present("no-run"); + + if !should_test && no_run { + diag.err("the `--test` flag must be passed to enable `--no-run`"); + return Err(1); + } let output = matches.opt_str("o").map(|s| PathBuf::from(&s)).unwrap_or_else(|| PathBuf::from("doc")); @@ -554,14 +557,7 @@ impl Options { } } - let target = - matches.opt_str("target").map_or(TargetTriple::from_triple(host_triple()), |target| { - if target.ends_with(".json") { - TargetTriple::TargetPath(PathBuf::from(target)) - } else { - TargetTriple::TargetTriple(target) - } - }); + let target = parse_target_triple(matches, error_format); let show_coverage = matches.opt_present("show-coverage"); @@ -630,8 +626,10 @@ impl Options { let document_hidden = matches.opt_present("document-hidden-items"); let run_check = matches.opt_present("check"); let generate_redirect_map = matches.opt_present("generate-redirect-map"); + let show_type_layout = matches.opt_present("show-type-layout"); - let (lint_opts, describe_lints, lint_cap) = get_cmd_lint_options(matches, error_format); + let (lint_opts, describe_lints, lint_cap, _) = + get_cmd_lint_options(matches, error_format, &debugging_opts); Ok(Options { input, @@ -666,6 +664,7 @@ impl Options { enable_per_target_ignores, test_builder, run_check, + no_run, render_options: RenderOptions { output, external_html, @@ -688,6 +687,7 @@ impl Options { document_private, document_hidden, generate_redirect_map, + show_type_layout, unstable_features: rustc_feature::UnstableFeatures::from_environment( crate_name.as_deref(), ), diff --git a/src/librustdoc/core.rs b/src/librustdoc/core.rs index 1ec4757a91..97be79a655 100644 --- a/src/librustdoc/core.rs +++ b/src/librustdoc/core.rs @@ -1,12 +1,12 @@ -use rustc_ast as ast; use rustc_data_structures::fx::{FxHashMap, FxHashSet}; use rustc_data_structures::sync::{self, Lrc}; use rustc_driver::abort_on_err; use rustc_errors::emitter::{Emitter, EmitterWriter}; use rustc_errors::json::JsonEmitter; use rustc_feature::UnstableFeatures; +use rustc_hir::def::Namespace::TypeNS; use rustc_hir::def::Res; -use rustc_hir::def_id::{CrateNum, DefId, DefIndex, LocalDefId, LOCAL_CRATE}; +use rustc_hir::def_id::{DefId, LocalDefId, CRATE_DEF_INDEX}; use rustc_hir::HirId; use rustc_hir::{ intravisit::{self, NestedVisitorMap, Visitor}, @@ -23,16 +23,14 @@ use rustc_session::DiagnosticOutput; use rustc_session::Session; use rustc_span::source_map; use rustc_span::symbol::sym; -use rustc_span::Span; +use rustc_span::{Span, DUMMY_SP}; use std::cell::RefCell; -use std::collections::hash_map::Entry; use std::mem; use std::rc::Rc; -use crate::clean; use crate::clean::inline::build_external_trait; -use crate::clean::{TraitWithExtraInfo, MAX_DEF_IDX}; +use crate::clean::{self, FakeDefId, TraitWithExtraInfo}; use crate::config::{Options as RustdocOptions, OutputFormat, RenderOptions}; use crate::formats::cache::Cache; use crate::passes::{self, Condition::*, ConditionalPass}; @@ -66,7 +64,6 @@ crate struct DocContext<'tcx> { crate ct_substs: FxHashMap, /// Table synthetic type parameter for `impl Trait` in argument position -> bounds crate impl_trait_bounds: FxHashMap>, - crate fake_def_ids: FxHashMap, /// Auto-trait or blanket impls processed so far, as `(self_ty, trait_def_id)`. // FIXME(eddyb) make this a `ty::TraitRef<'tcx>` set. crate generated_synthetics: FxHashSet<(Ty<'tcx>, DefId)>, @@ -81,7 +78,7 @@ crate struct DocContext<'tcx> { /// This same cache is used throughout rustdoc, including in [`crate::html::render`]. crate cache: Cache, /// Used by [`clean::inline`] to tell if an item has already been inlined. - crate inlined: FxHashSet, + crate inlined: FxHashSet, /// Used by `calculate_doc_coverage`. crate output_format: OutputFormat, } @@ -129,54 +126,14 @@ impl<'tcx> DocContext<'tcx> { r } - /// Create a new "fake" [`DefId`]. - /// - /// This is an ugly hack, but it's the simplest way to handle synthetic impls without greatly - /// refactoring either rustdoc or [`rustc_middle`]. In particular, allowing new [`DefId`]s - /// to be registered after the AST is constructed would require storing the [`DefId`] mapping - /// in a [`RefCell`], decreasing the performance for normal compilation for very little gain. - /// - /// Instead, we construct "fake" [`DefId`]s, which start immediately after the last `DefId`. - /// In the [`Debug`] impl for [`clean::Item`], we explicitly check for fake `DefId`s, - /// as we'll end up with a panic if we use the `DefId` `Debug` impl for fake `DefId`s. - /// - /// [`RefCell`]: std::cell::RefCell - /// [`Debug`]: std::fmt::Debug - /// [`clean::Item`]: crate::clean::types::Item - crate fn next_def_id(&mut self, crate_num: CrateNum) -> DefId { - let def_index = match self.fake_def_ids.entry(crate_num) { - Entry::Vacant(e) => { - let num_def_idx = { - let num_def_idx = if crate_num == LOCAL_CRATE { - self.tcx.hir().definitions().def_path_table().num_def_ids() - } else { - self.resolver.borrow_mut().access(|r| r.cstore().num_def_ids(crate_num)) - }; - - DefIndex::from_usize(num_def_idx) - }; - - MAX_DEF_IDX.with(|m| { - m.borrow_mut().insert(crate_num, num_def_idx); - }); - e.insert(num_def_idx) - } - Entry::Occupied(e) => e.into_mut(), - }; - *def_index = *def_index + 1; - - DefId { krate: crate_num, index: *def_index } - } - /// Like `hir().local_def_id_to_hir_id()`, but skips calling it on fake DefIds. /// (This avoids a slice-index-out-of-bounds panic.) - crate fn as_local_hir_id(tcx: TyCtxt<'_>, def_id: DefId) -> Option { - if MAX_DEF_IDX.with(|m| { - m.borrow().get(&def_id.krate).map(|&idx| idx <= def_id.index).unwrap_or(false) - }) { - None - } else { - def_id.as_local().map(|def_id| tcx.hir().local_def_id_to_hir_id(def_id)) + crate fn as_local_hir_id(tcx: TyCtxt<'_>, def_id: FakeDefId) -> Option { + match def_id { + FakeDefId::Real(real_id) => { + real_id.as_local().map(|def_id| tcx.hir().local_def_id_to_hir_id(def_id)) + } + FakeDefId::Fake(_, _) => None, } } } @@ -260,8 +217,9 @@ crate fn create_config( // By default, rustdoc ignores all lints. // Specifically unblock lints relevant to documentation or the lint machinery itself. let mut lints_to_show = vec![ - // it's unclear whether this should be part of rustdoc directly (#77364) + // it's unclear whether these should be part of rustdoc directly (#77364) rustc_lint::builtin::MISSING_DOCS.name.to_string(), + rustc_lint::builtin::INVALID_DOC_ATTRIBUTES.name.to_string(), // these are definitely not part of rustdoc, but we want to warn on them anyway. rustc_lint::builtin::RENAMED_AND_REMOVED_LINTS.name.to_string(), rustc_lint::builtin::UNKNOWN_LINTS.name.to_string(), @@ -269,14 +227,7 @@ crate fn create_config( lints_to_show.extend(crate::lint::RUSTDOC_LINTS.iter().map(|lint| lint.name.to_string())); let (lint_opts, lint_caps) = crate::lint::init_lints(lints_to_show, lint_opts, |lint| { - // FIXME: why is this necessary? - if lint.name == crate::lint::BROKEN_INTRA_DOC_LINKS.name - || lint.name == crate::lint::INVALID_CODEBLOCK_ATTRIBUTES.name - { - None - } else { - Some((lint.name_lower(), lint::Allow)) - } + Some((lint.name_lower(), lint::Allow)) }); let crate_types = @@ -349,17 +300,42 @@ crate fn create_config( } crate fn create_resolver<'a>( + externs: config::Externs, queries: &Queries<'a>, sess: &Session, ) -> Rc> { - let parts = abort_on_err(queries.expansion(), sess).peek(); - let (krate, resolver, _) = &*parts; - let resolver = resolver.borrow().clone(); - - let mut loader = crate::passes::collect_intra_doc_links::IntraLinkCrateLoader::new(resolver); - ast::visit::walk_crate(&mut loader, krate); + let extern_names: Vec = externs + .iter() + .filter(|(_, entry)| entry.add_prelude) + .map(|(name, _)| name) + .cloned() + .collect(); + + let (_, resolver, _) = &*abort_on_err(queries.expansion(), sess).peek(); + let resolver = resolver.borrow(); + + // Before we actually clone it, let's force all the extern'd crates to + // actually be loaded, just in case they're only referred to inside + // intra-doc links + resolver.borrow_mut().access(|resolver| { + sess.time("load_extern_crates", || { + for extern_name in &extern_names { + debug!("loading extern crate {}", extern_name); + if let Err(()) = resolver + .resolve_str_path_error( + DUMMY_SP, + extern_name, + TypeNS, + LocalDefId { local_def_index: CRATE_DEF_INDEX }.to_def_id(), + ) { + warn!("unable to resolve external crate {} (do you have an unused `--extern` crate?)", extern_name) + } + } + }); + }); - loader.resolver + // Now we're good to clone the resolver because everything should be loaded + resolver.clone() } crate fn run_global_ctxt( @@ -398,7 +374,7 @@ crate fn run_global_ctxt( }); rustc_passes::stability::check_unused_or_stable_features(tcx); - let access_levels = tcx.privacy_access_levels(LOCAL_CRATE); + let access_levels = tcx.privacy_access_levels(()); // Convert from a HirId set to a DefId set since we don't always have easy access // to the map from defid -> hirid let access_levels = AccessLevels { @@ -419,10 +395,9 @@ crate fn run_global_ctxt( lt_substs: Default::default(), ct_substs: Default::default(), impl_trait_bounds: Default::default(), - fake_def_ids: Default::default(), generated_synthetics: Default::default(), auto_traits: tcx - .all_traits(LOCAL_CRATE) + .all_traits(()) .iter() .cloned() .filter(|trait_def_id| tcx.trait_is_auto(*trait_def_id)) @@ -450,15 +425,18 @@ crate fn run_global_ctxt( let mut krate = tcx.sess.time("clean_crate", || clean::krate(&mut ctxt)); if krate.module.doc_value().map(|d| d.is_empty()).unwrap_or(true) { - let help = "The following guide may be of use:\n\ - https://doc.rust-lang.org/nightly/rustdoc/how-to-write-documentation.html"; + let help = format!( + "The following guide may be of use:\n\ + {}/rustdoc/how-to-write-documentation.html", + crate::DOC_RUST_LANG_ORG_CHANNEL + ); tcx.struct_lint_node( crate::lint::MISSING_CRATE_LEVEL_DOCS, DocContext::as_local_hir_id(tcx, krate.module.def_id).unwrap(), |lint| { let mut diag = lint.build("no documentation found for this crate's top-level module"); - diag.help(help); + diag.help(&help); diag.emit(); }, ); diff --git a/src/librustdoc/doctest.rs b/src/librustdoc/doctest.rs index 466d1b6540..88e2f6048e 100644 --- a/src/librustdoc/doctest.rs +++ b/src/librustdoc/doctest.rs @@ -76,7 +76,6 @@ crate fn run(options: Options) -> Result<(), ErrorReported> { externs: options.externs.clone(), unstable_features: options.render_options.unstable_features, actually_rustdoc: true, - debugging_opts: config::DebuggingOptions { ..config::basic_debugging_options() }, edition: options.edition, target_triple: options.target.clone(), crate_name: options.crate_name.clone(), @@ -840,7 +839,7 @@ impl Collector { if !item_path.is_empty() { item_path.push(' '); } - format!("{} - {}(line {})", filename, item_path, line) + format!("{} - {}(line {})", filename.prefer_local(), item_path, line) } crate fn set_position(&mut self, position: Span) { @@ -852,8 +851,10 @@ impl Collector { let filename = source_map.span_to_filename(self.position); if let FileName::Real(ref filename) = filename { if let Ok(cur_dir) = env::current_dir() { - if let Ok(path) = filename.local_path().strip_prefix(&cur_dir) { - return path.to_owned().into(); + if let Some(local_path) = filename.local_path() { + if let Ok(path) = local_path.strip_prefix(&cur_dir) { + return path.to_owned().into(); + } } } } @@ -879,20 +880,27 @@ impl Tester for Collector { let target = self.options.target.clone(); let target_str = target.to_string(); let unused_externs = self.unused_extern_reports.clone(); + let no_run = config.no_run || options.no_run; if !config.compile_fail { self.compiling_test_count.fetch_add(1, Ordering::SeqCst); } - // FIXME(#44940): if doctests ever support path remapping, then this filename - // needs to be the result of `SourceMap::span_to_unmapped_path`. let path = match &filename { - FileName::Real(path) => path.local_path().to_path_buf(), + FileName::Real(path) => { + if let Some(local_path) = path.local_path() { + local_path.to_path_buf() + } else { + // Somehow we got the filename from the metadata of another crate, should never happen + unreachable!("doctest from a different crate"); + } + } _ => PathBuf::from(r"doctest.rs"), }; // For example `module/file.rs` would become `module_file_rs` let file = filename - .to_string() + .prefer_local() + .to_string_lossy() .chars() .map(|c| if c.is_ascii_alphanumeric() { c } else { '_' }) .collect::(); @@ -934,6 +942,10 @@ impl Tester for Collector { // compiler failures are test failures should_panic: testing::ShouldPanic::No, allow_fail: config.allow_fail, + #[cfg(not(bootstrap))] + compile_fail: config.compile_fail, + #[cfg(not(bootstrap))] + no_run, test_type: testing::TestType::DocTest, }, testfn: testing::DynTestFn(box move || { @@ -946,7 +958,7 @@ impl Tester for Collector { line, options, config.should_panic, - config.no_run, + no_run, config.test_harness, runtool, runtool_args, @@ -1095,7 +1107,7 @@ impl<'a, 'hir, 'tcx> HirCollector<'a, 'hir, 'tcx> { let ast_attrs = self.tcx.hir().attrs(hir_id); let mut attrs = Attributes::from_ast(ast_attrs, None); - if let Some(ref cfg) = ast_attrs.cfg(self.sess.diagnostic()) { + if let Some(ref cfg) = ast_attrs.cfg(self.sess) { if !cfg.matches(&self.sess.parse_sess, Some(&self.sess.features_untracked())) { return; } diff --git a/src/librustdoc/doctree.rs b/src/librustdoc/doctree.rs index d3f4353a58..eadac89f79 100644 --- a/src/librustdoc/doctree.rs +++ b/src/librustdoc/doctree.rs @@ -1,12 +1,12 @@ //! This module is used to store stuff from Rust's AST in a more convenient //! manner (and with prettier names) before cleaning. +use rustc_middle::ty::TyCtxt; use rustc_span::{self, Span, Symbol}; use rustc_hir as hir; crate struct Module<'hir> { crate name: Symbol, - crate where_outer: Span, crate where_inner: Span, crate mods: Vec>, crate id: hir::HirId, @@ -17,16 +17,19 @@ crate struct Module<'hir> { } impl Module<'hir> { - crate fn new(name: Symbol) -> Module<'hir> { + crate fn new(name: Symbol, id: hir::HirId, where_inner: Span) -> Module<'hir> { Module { name, - id: hir::CRATE_HIR_ID, - where_outer: rustc_span::DUMMY_SP, - where_inner: rustc_span::DUMMY_SP, + id, + where_inner, mods: Vec::new(), items: Vec::new(), foreigns: Vec::new(), macros: Vec::new(), } } + + crate fn where_outer(&self, tcx: TyCtxt<'_>) -> Span { + tcx.hir().span(self.id) + } } diff --git a/src/librustdoc/fold.rs b/src/librustdoc/fold.rs index 37d11d4ed4..723792381e 100644 --- a/src/librustdoc/fold.rs +++ b/src/librustdoc/fold.rs @@ -1,17 +1,10 @@ use crate::clean::*; -crate struct StripItem(pub Item); - -impl StripItem { - crate fn strip(self) -> Item { - match self.0 { - Item { kind: box StrippedItem(..), .. } => self.0, - mut i => { - i.kind = box StrippedItem(i.kind); - i - } - } +crate fn strip_item(mut item: Item) -> Item { + if !matches!(*item.kind, StrippedItem(..)) { + item.kind = box StrippedItem(item.kind); } + item } crate trait DocFolder: Sized { diff --git a/src/librustdoc/formats/cache.rs b/src/librustdoc/formats/cache.rs index 8f8bca64e1..5734a4a98e 100644 --- a/src/librustdoc/formats/cache.rs +++ b/src/librustdoc/formats/cache.rs @@ -1,21 +1,19 @@ use std::collections::BTreeMap; use std::mem; -use std::path::{Path, PathBuf}; +use std::path::Path; use rustc_data_structures::fx::{FxHashMap, FxHashSet}; use rustc_hir::def_id::{CrateNum, DefId, CRATE_DEF_INDEX}; use rustc_middle::middle::privacy::AccessLevels; use rustc_middle::ty::TyCtxt; -use rustc_span::source_map::FileName; use rustc_span::symbol::sym; -use rustc_span::Symbol; -use crate::clean::{self, GetDefId}; +use crate::clean::{self, FakeDefId, GetDefId}; use crate::fold::DocFolder; use crate::formats::item_type::ItemType; use crate::formats::Impl; use crate::html::markdown::short_markdown_summary; -use crate::html::render::cache::{extern_location, get_index_search_type, ExternalLocation}; +use crate::html::render::cache::{get_index_search_type, ExternalLocation}; use crate::html::render::IndexItem; /// This cache is used to store information about the [`clean::Crate`] being @@ -72,7 +70,7 @@ crate struct Cache { crate implementors: FxHashMap>, /// Cache of where external crate documentation can be found. - crate extern_locations: FxHashMap, + crate extern_locations: FxHashMap, /// Cache of where documentation for primitives can be found. crate primitive_locations: FxHashMap, @@ -124,7 +122,7 @@ crate struct Cache { /// All intra-doc links resolved so far. /// /// Links are indexed by the DefId of the item they document. - crate intra_doc_links: BTreeMap>, + crate intra_doc_links: BTreeMap>, } /// This struct is used to wrap the `cache` and `tcx` in order to run `DocFolder`. @@ -155,21 +153,10 @@ impl Cache { // Cache where all our extern crates are located // FIXME: this part is specific to HTML so it'd be nice to remove it from the common code for &(n, ref e) in &krate.externs { - let src_root = match e.src(tcx) { - FileName::Real(ref p) => match p.local_path().parent() { - Some(p) => p.to_path_buf(), - None => PathBuf::new(), - }, - _ => PathBuf::new(), - }; let name = e.name(tcx); let extern_url = extern_html_root_urls.get(&*name.as_str()).map(|u| &**u); let did = DefId { krate: n, index: CRATE_DEF_INDEX }; - self.extern_locations.insert( - n, - (name, src_root, extern_location(e, extern_url, tcx.get_attrs(did), &dst, tcx)), - ); - + self.extern_locations.insert(n, e.location(extern_url, &dst, tcx)); self.external_paths.insert(did, (vec![name.to_string()], ItemType::Module)); } @@ -218,7 +205,7 @@ impl<'a, 'tcx> DocFolder for CacheBuilder<'a, 'tcx> { // If the impl is from a masked crate or references something from a // masked crate then remove it completely. if let clean::ImplItem(ref i) = *item.kind { - if self.cache.masked_crates.contains(&item.def_id.krate) + if self.cache.masked_crates.contains(&item.def_id.krate()) || i.trait_.def_id().map_or(false, |d| self.cache.masked_crates.contains(&d.krate)) || i.for_.def_id().map_or(false, |d| self.cache.masked_crates.contains(&d.krate)) { @@ -229,9 +216,11 @@ impl<'a, 'tcx> DocFolder for CacheBuilder<'a, 'tcx> { // Propagate a trait method's documentation to all implementors of the // trait. if let clean::TraitItem(ref t) = *item.kind { - self.cache.traits.entry(item.def_id).or_insert_with(|| clean::TraitWithExtraInfo { - trait_: t.clone(), - is_notable: item.attrs.has_doc_flag(sym::notable_trait), + self.cache.traits.entry(item.def_id.expect_real()).or_insert_with(|| { + clean::TraitWithExtraInfo { + trait_: t.clone(), + is_notable: item.attrs.has_doc_flag(sym::notable_trait), + } }); } @@ -241,7 +230,7 @@ impl<'a, 'tcx> DocFolder for CacheBuilder<'a, 'tcx> { if i.blanket_impl.is_none() { self.cache .implementors - .entry(did) + .entry(did.into()) .or_default() .push(Impl { impl_item: item.clone() }); } @@ -302,7 +291,7 @@ impl<'a, 'tcx> DocFolder for CacheBuilder<'a, 'tcx> { // A crate has a module at its root, containing all items, // which should not be indexed. The crate-item itself is // inserted later on when serializing the search-index. - if item.def_id.index != CRATE_DEF_INDEX { + if item.def_id.index().map_or(false, |idx| idx != CRATE_DEF_INDEX) { self.cache.search_index.push(IndexItem { ty: item.type_(), name: s.to_string(), @@ -340,6 +329,7 @@ impl<'a, 'tcx> DocFolder for CacheBuilder<'a, 'tcx> { | clean::EnumItem(..) | clean::TypedefItem(..) | clean::TraitItem(..) + | clean::TraitAliasItem(..) | clean::FunctionItem(..) | clean::ModuleItem(..) | clean::ForeignFunctionItem(..) @@ -350,26 +340,46 @@ impl<'a, 'tcx> DocFolder for CacheBuilder<'a, 'tcx> { | clean::ForeignTypeItem | clean::MacroItem(..) | clean::ProcMacroItem(..) - | clean::VariantItem(..) - if !self.cache.stripped_mod => - { - // Re-exported items mean that the same id can show up twice - // in the rustdoc ast that we're looking at. We know, - // however, that a re-exported item doesn't show up in the - // `public_items` map, so we can skip inserting into the - // paths map if there was already an entry present and we're - // not a public item. - if !self.cache.paths.contains_key(&item.def_id) - || self.cache.access_levels.is_public(item.def_id) - { - self.cache.paths.insert(item.def_id, (self.cache.stack.clone(), item.type_())); + | clean::VariantItem(..) => { + if !self.cache.stripped_mod { + // Re-exported items mean that the same id can show up twice + // in the rustdoc ast that we're looking at. We know, + // however, that a re-exported item doesn't show up in the + // `public_items` map, so we can skip inserting into the + // paths map if there was already an entry present and we're + // not a public item. + if !self.cache.paths.contains_key(&item.def_id.expect_real()) + || self.cache.access_levels.is_public(item.def_id.expect_real()) + { + self.cache.paths.insert( + item.def_id.expect_real(), + (self.cache.stack.clone(), item.type_()), + ); + } } } clean::PrimitiveItem(..) => { - self.cache.paths.insert(item.def_id, (self.cache.stack.clone(), item.type_())); + self.cache + .paths + .insert(item.def_id.expect_real(), (self.cache.stack.clone(), item.type_())); } - _ => {} + clean::ExternCrateItem { .. } + | clean::ImportItem(..) + | clean::OpaqueTyItem(..) + | clean::ImplItem(..) + | clean::TyMethodItem(..) + | clean::MethodItem(..) + | clean::StructFieldItem(..) + | clean::AssocConstItem(..) + | clean::AssocTypeItem(..) + | clean::StrippedItem(..) + | clean::KeywordItem(..) => { + // FIXME: Do these need handling? + // The person writing this comment doesn't know. + // So would rather leave them to an expert, + // as at least the list is better than `_ => {}`. + } } // Maintain the parent stack @@ -381,7 +391,7 @@ impl<'a, 'tcx> DocFolder for CacheBuilder<'a, 'tcx> { | clean::StructItem(..) | clean::UnionItem(..) | clean::VariantItem(..) => { - self.cache.parent_stack.push(item.def_id); + self.cache.parent_stack.push(item.def_id.expect_real()); self.cache.parent_is_trait_impl = false; true } diff --git a/src/librustdoc/formats/mod.rs b/src/librustdoc/formats/mod.rs index 1ce6572bbe..6060b0560c 100644 --- a/src/librustdoc/formats/mod.rs +++ b/src/librustdoc/formats/mod.rs @@ -2,9 +2,9 @@ crate mod cache; crate mod item_type; crate mod renderer; -crate use renderer::{run_format, FormatRenderer}; +use rustc_hir::def_id::DefId; -use rustc_span::def_id::DefId; +crate use renderer::{run_format, FormatRenderer}; use crate::clean; use crate::clean::types::GetDefId; diff --git a/src/librustdoc/html/format.rs b/src/librustdoc/html/format.rs index 45412f55c1..918a5cb509 100644 --- a/src/librustdoc/html/format.rs +++ b/src/librustdoc/html/format.rs @@ -12,11 +12,14 @@ use std::iter; use rustc_data_structures::captures::Captures; use rustc_data_structures::fx::FxHashSet; use rustc_hir as hir; +use rustc_hir::def_id::DefId; use rustc_middle::ty::TyCtxt; -use rustc_span::def_id::{DefId, CRATE_DEF_INDEX}; +use rustc_span::def_id::CRATE_DEF_INDEX; use rustc_target::spec::abi::Abi; -use crate::clean::{self, utils::find_nearest_parent_module, PrimitiveType}; +use crate::clean::{ + self, utils::find_nearest_parent_module, ExternalCrate, FakeDefId, GetDefId, PrimitiveType, +}; use crate::formats::item_type::ItemType; use crate::html::escape::Escape; use crate::html::render::cache::ExternalLocation; @@ -174,12 +177,22 @@ impl clean::GenericParamDef { Ok(()) } - clean::GenericParamDefKind::Const { ref ty, .. } => { + clean::GenericParamDefKind::Const { ref ty, ref default, .. } => { if f.alternate() { - write!(f, "const {}: {:#}", self.name, ty.print(cx)) + write!(f, "const {}: {:#}", self.name, ty.print(cx))?; } else { - write!(f, "const {}: {}", self.name, ty.print(cx)) + write!(f, "const {}: {}", self.name, ty.print(cx))?; } + + if let Some(default) = default { + if f.alternate() { + write!(f, " = {:#}", default)?; + } else { + write!(f, " = {}", default)?; + } + } + + Ok(()) } }) } @@ -465,14 +478,14 @@ crate fn href(did: DefId, cx: &Context<'_>) -> Option<(String, ItemType, Vec { + ExternalLocation::Remote(ref s) => { let s = s.trim_end_matches('/'); let mut s = vec![&s[..]]; s.extend(module_fqp[..].iter().map(String::as_str)); s } - (.., ExternalLocation::Local) => href_relative_parts(module_fqp, relative_to), - (.., ExternalLocation::Unknown) => return None, + ExternalLocation::Local => href_relative_parts(module_fqp, relative_to), + ExternalLocation::Unknown => return None, }, ) } @@ -571,19 +584,21 @@ fn primitive_link( f, "", "../".repeat(len), - prim.to_url_str() + prim.as_sym() )?; needs_termination = true; } Some(&def_id) => { let cname_str; let loc = match m.extern_locations[&def_id.krate] { - (ref cname, _, ExternalLocation::Remote(ref s)) => { - cname_str = cname.as_str(); + ExternalLocation::Remote(ref s) => { + cname_str = + ExternalCrate { crate_num: def_id.krate }.name(cx.tcx()).as_str(); Some(vec![s.trim_end_matches('/'), &cname_str[..]]) } - (ref cname, _, ExternalLocation::Local) => { - cname_str = cname.as_str(); + ExternalLocation::Local => { + cname_str = + ExternalCrate { crate_num: def_id.krate }.name(cx.tcx()).as_str(); Some(if cx.current.first().map(|x| &x[..]) == Some(&cname_str[..]) { iter::repeat("..").take(cx.current.len() - 1).collect() } else { @@ -591,14 +606,14 @@ fn primitive_link( iter::repeat("..").take(cx.current.len()).chain(cname).collect() }) } - (.., ExternalLocation::Unknown) => None, + ExternalLocation::Unknown => None, }; if let Some(loc) = loc { write!( f, "", loc.join("/"), - prim.to_url_str() + prim.as_sym() )?; needs_termination = true; } @@ -635,7 +650,7 @@ crate fn anchor<'a, 'cx: 'a>( text: &'a str, cx: &'cx Context<'_>, ) -> impl fmt::Display + 'a { - let parts = href(did, cx); + let parts = href(did.into(), cx); display_fn(move |f| { if let Some((url, short_ty, fqp)) = parts { write!( @@ -672,7 +687,7 @@ fn fmt_type<'cx>( fmt::Display::fmt(&tybounds(param_names, cx), f) } clean::Infer => write!(f, "_"), - clean::Primitive(prim) => primitive_link(f, prim, prim.as_str(), cx), + clean::Primitive(prim) => primitive_link(f, prim, &*prim.as_sym().as_str(), cx), clean::BareFunction(ref decl) => { if f.alternate() { write!( @@ -831,10 +846,13 @@ fn fmt_type<'cx>( write!(f, "impl {}", print_generic_bounds(bounds, cx)) } } - clean::QPath { ref name, ref self_type, ref trait_ } => { + clean::QPath { ref name, ref self_type, ref trait_, ref self_def_id } => { let should_show_cast = match *trait_ { box clean::ResolvedPath { ref path, .. } => { - !path.segments.is_empty() && !self_type.is_self_type() + !path.segments.is_empty() + && self_def_id + .zip(trait_.def_id()) + .map_or(!self_type.is_self_type(), |(id, trait_)| id != trait_) } _ => true, }; @@ -863,7 +881,7 @@ fn fmt_type<'cx>( // everything comes in as a fully resolved QPath (hard to // look at). box clean::ResolvedPath { did, ref param_names, .. } => { - match href(did, cx) { + match href(did.into(), cx) { Some((ref url, _, ref path)) if !f.alternate() => { write!( f, @@ -1141,7 +1159,7 @@ impl clean::FnDecl { impl clean::Visibility { crate fn print_with_space<'a, 'tcx: 'a>( self, - item_did: DefId, + item_did: FakeDefId, cx: &'a Context<'tcx>, ) -> impl fmt::Display + 'a + Captures<'tcx> { let to_print = match self { @@ -1151,7 +1169,7 @@ impl clean::Visibility { // FIXME(camelid): This may not work correctly if `item_did` is a module. // However, rustdoc currently never displays a module's // visibility, so it shouldn't matter. - let parent_module = find_nearest_parent_module(cx.tcx(), item_did); + let parent_module = find_nearest_parent_module(cx.tcx(), item_did.expect_real()); if vis_did.index == CRATE_DEF_INDEX { "pub(crate) ".to_owned() diff --git a/src/librustdoc/html/highlight.rs b/src/librustdoc/html/highlight.rs index 0030ef67c0..51392ca119 100644 --- a/src/librustdoc/html/highlight.rs +++ b/src/librustdoc/html/highlight.rs @@ -13,7 +13,6 @@ use std::iter::Peekable; use rustc_lexer::{LiteralKind, TokenKind}; use rustc_span::edition::Edition; use rustc_span::symbol::Symbol; -use rustc_span::with_default_session_globals; use super::format::Buffer; @@ -25,6 +24,7 @@ crate fn render_with_highlighting( playground_button: Option<&str>, tooltip: Option<(Option, &str)>, edition: Edition, + extra_content: Option, ) { debug!("highlighting: ================\n{}\n==============", src); if let Some((edition_info, class)) = tooltip { @@ -40,13 +40,21 @@ crate fn render_with_highlighting( ); } - write_header(out, class); + write_header(out, class, extra_content); write_code(out, &src, edition); write_footer(out, playground_button); } -fn write_header(out: &mut Buffer, class: Option<&str>) { - write!(out, "
\n", class.unwrap_or_default());
+fn write_header(out: &mut Buffer, class: Option<&str>, extra_content: Option) {
+    write!(out, "
");
+    if let Some(extra) = extra_content {
+        out.push_buffer(extra);
+    }
+    if let Some(class) = class {
+        writeln!(out, "
", class);
+    } else {
+        writeln!(out, "
");
+    }
 }
 
 fn write_code(out: &mut Buffer, src: &str, edition: Edition) {
@@ -62,7 +70,7 @@ fn write_code(out: &mut Buffer, src: &str, edition: Edition) {
 }
 
 fn write_footer(out: &mut Buffer, playground_button: Option<&str>) {
-    write!(out, "
{}
\n", playground_button.unwrap_or_default());
+    writeln!(out, "
{}
", playground_button.unwrap_or_default()); } /// How a span of text is classified. Mostly corresponds to token kinds. @@ -238,28 +246,26 @@ impl<'a> Classifier<'a> { /// possibly giving it an HTML span with a class specifying what flavor of /// token is used. fn highlight(mut self, sink: &mut dyn FnMut(Highlight<'a>)) { - with_default_session_globals(|| { - loop { - if self - .tokens - .peek() - .map(|t| matches!(t.0, TokenKind::Colon | TokenKind::Ident)) - .unwrap_or(false) - { - let tokens = self.get_full_ident_path(); - for (token, start, end) in tokens { - let text = &self.src[start..end]; - self.advance(token, text, sink); - self.byte_pos += text.len() as u32; - } - } - if let Some((token, text)) = self.next() { + loop { + if self + .tokens + .peek() + .map(|t| matches!(t.0, TokenKind::Colon | TokenKind::Ident)) + .unwrap_or(false) + { + let tokens = self.get_full_ident_path(); + for (token, start, end) in tokens { + let text = &self.src[start..end]; self.advance(token, text, sink); - } else { - break; + self.byte_pos += text.len() as u32; } } - }) + if let Some((token, text)) = self.next() { + self.advance(token, text, sink); + } else { + break; + } + } } /// Single step of highlighting. This will classify `token`, but maybe also diff --git a/src/librustdoc/html/highlight/tests.rs b/src/librustdoc/html/highlight/tests.rs index 305cf61091..a0da2c963d 100644 --- a/src/librustdoc/html/highlight/tests.rs +++ b/src/librustdoc/html/highlight/tests.rs @@ -2,6 +2,7 @@ use super::write_code; use crate::html::format::Buffer; use expect_test::expect_file; use rustc_span::edition::Edition; +use rustc_span::with_default_session_globals; const STYLE: &str = r#"

(&self, mut pred: P) -> usize where P: FnMut(&T) -> bool, diff --git a/library/alloc/src/collections/vec_deque/pair_slices.rs b/library/alloc/src/collections/vec_deque/pair_slices.rs index 812765d0b0..7b87090fb0 100644 --- a/library/alloc/src/collections/vec_deque/pair_slices.rs +++ b/library/alloc/src/collections/vec_deque/pair_slices.rs @@ -1,4 +1,3 @@ -use core::array; use core::cmp::{self}; use core::mem::replace; @@ -37,7 +36,7 @@ impl<'a, 'b, T> PairSlices<'a, 'b, T> { } pub fn remainder(self) -> impl Iterator { - array::IntoIter::new([self.b0, self.b1]) + IntoIterator::into_iter([self.b0, self.b1]) } } diff --git a/library/alloc/src/fmt.rs b/library/alloc/src/fmt.rs index 3ef55f06e5..fd5ee189fb 100644 --- a/library/alloc/src/fmt.rs +++ b/library/alloc/src/fmt.rs @@ -546,6 +546,7 @@ pub use core::fmt::{LowerExp, UpperExp}; #[stable(feature = "rust1", since = "1.0.0")] pub use core::fmt::{LowerHex, Pointer, UpperHex}; +#[cfg(not(no_global_oom_handling))] use crate::string; /// The `format` function takes an [`Arguments`] struct and returns the resulting @@ -574,6 +575,7 @@ use crate::string; /// /// [`format_args!`]: core::format_args /// [`format!`]: crate::format +#[cfg(not(no_global_oom_handling))] #[stable(feature = "rust1", since = "1.0.0")] pub fn format(args: Arguments<'_>) -> string::String { let capacity = args.estimated_capacity(); diff --git a/library/alloc/src/lib.rs b/library/alloc/src/lib.rs index 15308a4469..a04e7c8a49 100644 --- a/library/alloc/src/lib.rs +++ b/library/alloc/src/lib.rs @@ -59,7 +59,6 @@ #![allow(unused_attributes)] #![stable(feature = "alloc", since = "1.36.0")] #![doc( - html_root_url = "https://doc.rust-lang.org/nightly/", html_playground_url = "https://play.rust-lang.org/", issue_tracker_base_url = "https://github.com/rust-lang/rust/issues/", test(no_crate_inject, attr(allow(unused_variables), deny(warnings))) @@ -87,9 +86,8 @@ #![feature(cfg_sanitize)] #![feature(cfg_target_has_atomic)] #![feature(coerce_unsized)] -#![feature(const_btree_new)] -#![cfg_attr(bootstrap, feature(const_fn))] -#![cfg_attr(not(bootstrap), feature(const_fn_trait_bound))] +#![cfg_attr(not(no_global_oom_handling), feature(const_btree_new))] +#![feature(const_fn_trait_bound)] #![feature(cow_is_borrowed)] #![feature(const_cow_is_borrowed)] #![feature(destructuring_assignment)] @@ -118,7 +116,6 @@ #![feature(nonnull_slice_from_raw_parts)] #![feature(auto_traits)] #![feature(option_result_unwrap_unchecked)] -#![cfg_attr(bootstrap, feature(or_patterns))] #![feature(pattern)] #![feature(ptr_internals)] #![feature(rustc_attrs)] @@ -140,11 +137,13 @@ #![feature(maybe_uninit_extra, maybe_uninit_slice, maybe_uninit_uninit_array)] #![feature(alloc_layout_extra)] #![feature(trusted_random_access)] -#![feature(try_trait)] +#![cfg_attr(bootstrap, feature(try_trait))] +#![cfg_attr(not(bootstrap), feature(try_trait_v2))] #![feature(min_type_alias_impl_trait)] #![feature(associated_type_bounds)] #![feature(slice_group_by)] #![feature(decl_macro)] +#![feature(bindings_after_at)] // Allow testing this library #[cfg(test)] @@ -183,7 +182,7 @@ pub mod str; pub mod string; #[cfg(target_has_atomic = "ptr")] pub mod sync; -#[cfg(target_has_atomic = "ptr")] +#[cfg(all(not(no_global_oom_handling), target_has_atomic = "ptr"))] pub mod task; #[cfg(test)] mod tests; diff --git a/library/alloc/src/raw_vec.rs b/library/alloc/src/raw_vec.rs index fe87a97bac..2e2c9b76bd 100644 --- a/library/alloc/src/raw_vec.rs +++ b/library/alloc/src/raw_vec.rs @@ -9,13 +9,16 @@ use core::ops::Drop; use core::ptr::{self, NonNull, Unique}; use core::slice; -use crate::alloc::{handle_alloc_error, Allocator, Global, Layout}; +#[cfg(not(no_global_oom_handling))] +use crate::alloc::handle_alloc_error; +use crate::alloc::{Allocator, Global, Layout}; use crate::boxed::Box; use crate::collections::TryReserveError::{self, *}; #[cfg(test)] mod tests; +#[cfg(not(no_global_oom_handling))] enum AllocInit { /// The contents of the new memory are uninitialized. Uninitialized, @@ -82,12 +85,14 @@ impl RawVec { /// # Aborts /// /// Aborts on OOM. + #[cfg(not(no_global_oom_handling))] #[inline] pub fn with_capacity(capacity: usize) -> Self { Self::with_capacity_in(capacity, Global) } /// Like `with_capacity`, but guarantees the buffer is zeroed. + #[cfg(not(no_global_oom_handling))] #[inline] pub fn with_capacity_zeroed(capacity: usize) -> Self { Self::with_capacity_zeroed_in(capacity, Global) @@ -131,6 +136,7 @@ impl RawVec { /// Like `with_capacity`, but parameterized over the choice of /// allocator for the returned `RawVec`. + #[cfg(not(no_global_oom_handling))] #[inline] pub fn with_capacity_in(capacity: usize, alloc: A) -> Self { Self::allocate_in(capacity, AllocInit::Uninitialized, alloc) @@ -138,6 +144,7 @@ impl RawVec { /// Like `with_capacity_zeroed`, but parameterized over the choice /// of allocator for the returned `RawVec`. + #[cfg(not(no_global_oom_handling))] #[inline] pub fn with_capacity_zeroed_in(capacity: usize, alloc: A) -> Self { Self::allocate_in(capacity, AllocInit::Zeroed, alloc) @@ -177,6 +184,7 @@ impl RawVec { } } + #[cfg(not(no_global_oom_handling))] fn allocate_in(capacity: usize, init: AllocInit, alloc: A) -> Self { if mem::size_of::() == 0 { Self::new_in(alloc) @@ -309,6 +317,7 @@ impl RawVec { /// # vector.push_all(&[1, 3, 5, 7, 9]); /// # } /// ``` + #[cfg(not(no_global_oom_handling))] #[inline] pub fn reserve(&mut self, len: usize, additional: usize) { // Callers expect this function to be very cheap when there is already sufficient capacity. @@ -355,6 +364,7 @@ impl RawVec { /// # Aborts /// /// Aborts on OOM. + #[cfg(not(no_global_oom_handling))] pub fn reserve_exact(&mut self, len: usize, additional: usize) { handle_reserve(self.try_reserve_exact(len, additional)); } @@ -378,6 +388,7 @@ impl RawVec { /// # Aborts /// /// Aborts on OOM. + #[cfg(not(no_global_oom_handling))] pub fn shrink_to_fit(&mut self, amount: usize) { handle_reserve(self.shrink(amount)); } @@ -452,6 +463,7 @@ impl RawVec { Ok(()) } + #[cfg(not(no_global_oom_handling))] fn shrink(&mut self, amount: usize) -> Result<(), TryReserveError> { assert!(amount <= self.capacity(), "Tried to shrink to a larger capacity"); @@ -512,6 +524,7 @@ unsafe impl<#[may_dangle] T, A: Allocator> Drop for RawVec { } // Central function for reserve error handling. +#[cfg(not(no_global_oom_handling))] #[inline] fn handle_reserve(result: Result<(), TryReserveError>) { match result { @@ -542,6 +555,7 @@ fn alloc_guard(alloc_size: usize) -> Result<(), TryReserveError> { // One central function responsible for reporting capacity overflows. This'll // ensure that the code generation related to these panics is minimal as there's // only one location which panics rather than a bunch throughout the module. +#[cfg(not(no_global_oom_handling))] fn capacity_overflow() -> ! { panic!("capacity overflow"); } diff --git a/library/alloc/src/rc.rs b/library/alloc/src/rc.rs index cb4af7c5cd..f131182a89 100644 --- a/library/alloc/src/rc.rs +++ b/library/alloc/src/rc.rs @@ -255,19 +255,27 @@ use core::convert::{From, TryFrom}; use core::fmt; use core::hash::{Hash, Hasher}; use core::intrinsics::abort; +#[cfg(not(no_global_oom_handling))] use core::iter; use core::marker::{self, PhantomData, Unpin, Unsize}; -use core::mem::{self, align_of_val_raw, forget, size_of_val}; +#[cfg(not(no_global_oom_handling))] +use core::mem::size_of_val; +use core::mem::{self, align_of_val_raw, forget}; use core::ops::{CoerceUnsized, Deref, DispatchFromDyn, Receiver}; use core::pin::Pin; use core::ptr::{self, NonNull}; +#[cfg(not(no_global_oom_handling))] use core::slice::from_raw_parts_mut; -use crate::alloc::{ - box_free, handle_alloc_error, AllocError, Allocator, Global, Layout, WriteCloneIntoRaw, -}; +#[cfg(not(no_global_oom_handling))] +use crate::alloc::handle_alloc_error; +#[cfg(not(no_global_oom_handling))] +use crate::alloc::{box_free, WriteCloneIntoRaw}; +use crate::alloc::{AllocError, Allocator, Global, Layout}; use crate::borrow::{Cow, ToOwned}; +#[cfg(not(no_global_oom_handling))] use crate::string::String; +#[cfg(not(no_global_oom_handling))] use crate::vec::Vec; #[cfg(test)] @@ -434,6 +442,7 @@ impl Rc { /// /// assert_eq!(*five, 5) /// ``` + #[cfg(not(no_global_oom_handling))] #[unstable(feature = "new_uninit", issue = "63291")] pub fn new_uninit() -> Rc> { unsafe { @@ -465,6 +474,7 @@ impl Rc { /// ``` /// /// [zeroed]: mem::MaybeUninit::zeroed + #[cfg(not(no_global_oom_handling))] #[unstable(feature = "new_uninit", issue = "63291")] pub fn new_zeroed() -> Rc> { unsafe { @@ -637,6 +647,7 @@ impl Rc<[T]> { /// /// assert_eq!(*values, [1, 2, 3]) /// ``` + #[cfg(not(no_global_oom_handling))] #[unstable(feature = "new_uninit", issue = "63291")] pub fn new_uninit_slice(len: usize) -> Rc<[mem::MaybeUninit]> { unsafe { Rc::from_ptr(Rc::allocate_for_slice(len)) } @@ -662,6 +673,7 @@ impl Rc<[T]> { /// ``` /// /// [zeroed]: mem::MaybeUninit::zeroed + #[cfg(not(no_global_oom_handling))] #[unstable(feature = "new_uninit", issue = "63291")] pub fn new_zeroed_slice(len: usize) -> Rc<[mem::MaybeUninit]> { unsafe { @@ -1122,6 +1134,7 @@ impl Rc { /// assert!(76 == *data); /// assert!(weak.upgrade().is_none()); /// ``` + #[cfg(not(no_global_oom_handling))] #[inline] #[stable(feature = "rc_unique", since = "1.4.0")] pub fn make_mut(this: &mut Self) -> &mut T { @@ -1195,6 +1208,7 @@ impl Rc { /// /// The function `mem_to_rcbox` is called with the data pointer /// and must return back a (potentially fat)-pointer for the `RcBox`. + #[cfg(not(no_global_oom_handling))] unsafe fn allocate_for_layout( value_layout: Layout, allocate: impl FnOnce(Layout) -> Result, AllocError>, @@ -1245,6 +1259,7 @@ impl Rc { } /// Allocates an `RcBox` with sufficient space for an unsized inner value + #[cfg(not(no_global_oom_handling))] unsafe fn allocate_for_ptr(ptr: *const T) -> *mut RcBox { // Allocate for the `RcBox` using the given value. unsafe { @@ -1256,6 +1271,7 @@ impl Rc { } } + #[cfg(not(no_global_oom_handling))] fn from_box(v: Box) -> Rc { unsafe { let (box_unique, alloc) = Box::into_unique(v); @@ -1281,6 +1297,7 @@ impl Rc { impl Rc<[T]> { /// Allocates an `RcBox<[T]>` with the given length. + #[cfg(not(no_global_oom_handling))] unsafe fn allocate_for_slice(len: usize) -> *mut RcBox<[T]> { unsafe { Self::allocate_for_layout( @@ -1294,6 +1311,7 @@ impl Rc<[T]> { /// Copy elements from slice into newly allocated Rc<\[T\]> /// /// Unsafe because the caller must either take ownership or bind `T: Copy` + #[cfg(not(no_global_oom_handling))] unsafe fn copy_from_slice(v: &[T]) -> Rc<[T]> { unsafe { let ptr = Self::allocate_for_slice(v.len()); @@ -1305,6 +1323,7 @@ impl Rc<[T]> { /// Constructs an `Rc<[T]>` from an iterator known to be of a certain size. /// /// Behavior is undefined should the size be wrong. + #[cfg(not(no_global_oom_handling))] unsafe fn from_iter_exact(iter: impl iter::Iterator, len: usize) -> Rc<[T]> { // Panic guard while cloning T elements. // In the event of a panic, elements that have been written @@ -1356,6 +1375,7 @@ trait RcFromSlice { fn from_slice(slice: &[T]) -> Self; } +#[cfg(not(no_global_oom_handling))] impl RcFromSlice for Rc<[T]> { #[inline] default fn from_slice(v: &[T]) -> Self { @@ -1363,6 +1383,7 @@ impl RcFromSlice for Rc<[T]> { } } +#[cfg(not(no_global_oom_handling))] impl RcFromSlice for Rc<[T]> { #[inline] fn from_slice(v: &[T]) -> Self { @@ -1712,11 +1733,25 @@ impl fmt::Pointer for Rc { #[stable(feature = "from_for_ptrs", since = "1.6.0")] impl From for Rc { + /// Converts a generic type `T` into a `Rc` + /// + /// The conversion allocates on the heap and moves `t` + /// from the stack into it. + /// + /// # Example + /// ```rust + /// # use std::rc::Rc; + /// let x = 5; + /// let rc = Rc::new(5); + /// + /// assert_eq!(Rc::from(x), rc); + /// ``` fn from(t: T) -> Self { Rc::new(t) } } +#[cfg(not(no_global_oom_handling))] #[stable(feature = "shared_from_slice", since = "1.21.0")] impl From<&[T]> for Rc<[T]> { /// Allocate a reference-counted slice and fill it by cloning `v`'s items. @@ -1735,6 +1770,7 @@ impl From<&[T]> for Rc<[T]> { } } +#[cfg(not(no_global_oom_handling))] #[stable(feature = "shared_from_slice", since = "1.21.0")] impl From<&str> for Rc { /// Allocate a reference-counted string slice and copy `v` into it. @@ -1753,6 +1789,7 @@ impl From<&str> for Rc { } } +#[cfg(not(no_global_oom_handling))] #[stable(feature = "shared_from_slice", since = "1.21.0")] impl From for Rc { /// Allocate a reference-counted string slice and copy `v` into it. @@ -1771,6 +1808,7 @@ impl From for Rc { } } +#[cfg(not(no_global_oom_handling))] #[stable(feature = "shared_from_slice", since = "1.21.0")] impl From> for Rc { /// Move a boxed object to a new, reference counted, allocation. @@ -1789,6 +1827,7 @@ impl From> for Rc { } } +#[cfg(not(no_global_oom_handling))] #[stable(feature = "shared_from_slice", since = "1.21.0")] impl From> for Rc<[T]> { /// Allocate a reference-counted slice and move `v`'s items into it. @@ -1820,6 +1859,18 @@ where B: ToOwned + ?Sized, Rc: From<&'a B> + From, { + /// Create a reference-counted pointer from + /// a clone-on-write pointer by copying its content. + /// + /// # Example + /// + /// ```rust + /// # use std::rc::Rc; + /// # use std::borrow::Cow; + /// let cow: Cow = Cow::Borrowed("eggplant"); + /// let shared: Rc = Rc::from(cow); + /// assert_eq!("eggplant", &shared[..]); + /// ``` #[inline] fn from(cow: Cow<'a, B>) -> Rc { match cow { @@ -1842,6 +1893,7 @@ impl TryFrom> for Rc<[T; N]> { } } +#[cfg(not(no_global_oom_handling))] #[stable(feature = "shared_from_iter", since = "1.37.0")] impl iter::FromIterator for Rc<[T]> { /// Takes each element in the `Iterator` and collects it into an `Rc<[T]>`. @@ -1888,16 +1940,19 @@ impl iter::FromIterator for Rc<[T]> { } /// Specialization trait used for collecting into `Rc<[T]>`. +#[cfg(not(no_global_oom_handling))] trait ToRcSlice: Iterator + Sized { fn to_rc_slice(self) -> Rc<[T]>; } +#[cfg(not(no_global_oom_handling))] impl> ToRcSlice for I { default fn to_rc_slice(self) -> Rc<[T]> { self.collect::>().into() } } +#[cfg(not(no_global_oom_handling))] impl> ToRcSlice for I { fn to_rc_slice(self) -> Rc<[T]> { // This is the case for a `TrustedLen` iterator. @@ -2260,7 +2315,7 @@ impl Weak { } #[stable(feature = "rc_weak", since = "1.4.0")] -impl Drop for Weak { +unsafe impl<#[may_dangle] T: ?Sized> Drop for Weak { /// Drops the `Weak` pointer. /// /// # Examples diff --git a/library/alloc/src/slice.rs b/library/alloc/src/slice.rs index f5d0e911b6..dcd6489920 100644 --- a/library/alloc/src/slice.rs +++ b/library/alloc/src/slice.rs @@ -83,11 +83,19 @@ #![cfg_attr(test, allow(unused_imports, dead_code))] use core::borrow::{Borrow, BorrowMut}; +#[cfg(not(no_global_oom_handling))] use core::cmp::Ordering::{self, Less}; -use core::mem::{self, size_of}; +#[cfg(not(no_global_oom_handling))] +use core::mem; +#[cfg(not(no_global_oom_handling))] +use core::mem::size_of; +#[cfg(not(no_global_oom_handling))] use core::ptr; -use crate::alloc::{Allocator, Global}; +use crate::alloc::Allocator; +#[cfg(not(no_global_oom_handling))] +use crate::alloc::Global; +#[cfg(not(no_global_oom_handling))] use crate::borrow::ToOwned; use crate::boxed::Box; use crate::vec::Vec; @@ -158,17 +166,20 @@ mod hack { } } + #[cfg(not(no_global_oom_handling))] #[inline] pub fn to_vec(s: &[T], alloc: A) -> Vec { T::to_vec(s, alloc) } + #[cfg(not(no_global_oom_handling))] pub trait ConvertVec { fn to_vec(s: &[Self], alloc: A) -> Vec where Self: Sized; } + #[cfg(not(no_global_oom_handling))] impl ConvertVec for T { #[inline] default fn to_vec(s: &[Self], alloc: A) -> Vec { @@ -205,6 +216,7 @@ mod hack { } } + #[cfg(not(no_global_oom_handling))] impl ConvertVec for T { #[inline] fn to_vec(s: &[Self], alloc: A) -> Vec { @@ -250,6 +262,7 @@ impl [T] { /// v.sort(); /// assert!(v == [-5, -3, 1, 2, 4]); /// ``` + #[cfg(not(no_global_oom_handling))] #[stable(feature = "rust1", since = "1.0.0")] #[inline] pub fn sort(&mut self) @@ -304,6 +317,7 @@ impl [T] { /// v.sort_by(|a, b| b.cmp(a)); /// assert!(v == [5, 4, 3, 2, 1]); /// ``` + #[cfg(not(no_global_oom_handling))] #[stable(feature = "rust1", since = "1.0.0")] #[inline] pub fn sort_by(&mut self, mut compare: F) @@ -344,6 +358,7 @@ impl [T] { /// v.sort_by_key(|k| k.abs()); /// assert!(v == [1, 2, -3, 4, -5]); /// ``` + #[cfg(not(no_global_oom_handling))] #[stable(feature = "slice_sort_by_key", since = "1.7.0")] #[inline] pub fn sort_by_key(&mut self, mut f: F) @@ -386,6 +401,7 @@ impl [T] { /// ``` /// /// [pdqsort]: https://github.com/orlp/pdqsort + #[cfg(not(no_global_oom_handling))] #[stable(feature = "slice_sort_by_cached_key", since = "1.34.0")] #[inline] pub fn sort_by_cached_key(&mut self, f: F) @@ -443,6 +459,7 @@ impl [T] { /// let x = s.to_vec(); /// // Here, `s` and `x` can be modified independently. /// ``` + #[cfg(not(no_global_oom_handling))] #[rustc_conversion_suggestion] #[stable(feature = "rust1", since = "1.0.0")] #[inline] @@ -466,6 +483,7 @@ impl [T] { /// let x = s.to_vec_in(System); /// // Here, `s` and `x` can be modified independently. /// ``` + #[cfg(not(no_global_oom_handling))] #[inline] #[unstable(feature = "allocator_api", issue = "32838")] pub fn to_vec_in(&self, alloc: A) -> Vec @@ -517,6 +535,7 @@ impl [T] { /// // this will panic at runtime /// b"0123456789abcdef".repeat(usize::MAX); /// ``` + #[cfg(not(no_global_oom_handling))] #[stable(feature = "repeat_generic_slice", since = "1.40.0")] pub fn repeat(&self, n: usize) -> Vec where @@ -642,6 +661,7 @@ impl [u8] { /// To uppercase the value in-place, use [`make_ascii_uppercase`]. /// /// [`make_ascii_uppercase`]: slice::make_ascii_uppercase + #[cfg(not(no_global_oom_handling))] #[stable(feature = "ascii_methods_on_intrinsics", since = "1.23.0")] #[inline] pub fn to_ascii_uppercase(&self) -> Vec { @@ -659,6 +679,7 @@ impl [u8] { /// To lowercase the value in-place, use [`make_ascii_lowercase`]. /// /// [`make_ascii_lowercase`]: slice::make_ascii_lowercase + #[cfg(not(no_global_oom_handling))] #[stable(feature = "ascii_methods_on_intrinsics", since = "1.23.0")] #[inline] pub fn to_ascii_lowercase(&self) -> Vec { @@ -724,6 +745,7 @@ pub trait Join { fn join(slice: &Self, sep: Separator) -> Self::Output; } +#[cfg(not(no_global_oom_handling))] #[unstable(feature = "slice_concat_ext", issue = "27747")] impl> Concat for [V] { type Output = Vec; @@ -738,6 +760,7 @@ impl> Concat for [V] { } } +#[cfg(not(no_global_oom_handling))] #[unstable(feature = "slice_concat_ext", issue = "27747")] impl> Join<&T> for [V] { type Output = Vec; @@ -760,6 +783,7 @@ impl> Join<&T> for [V] { } } +#[cfg(not(no_global_oom_handling))] #[unstable(feature = "slice_concat_ext", issue = "27747")] impl> Join<&[T]> for [V] { type Output = Vec; @@ -801,6 +825,7 @@ impl BorrowMut<[T]> for Vec { } } +#[cfg(not(no_global_oom_handling))] #[stable(feature = "rust1", since = "1.0.0")] impl ToOwned for [T] { type Owned = Vec; @@ -835,6 +860,7 @@ impl ToOwned for [T] { /// Inserts `v[0]` into pre-sorted sequence `v[1..]` so that whole `v[..]` becomes sorted. /// /// This is the integral subroutine of insertion sort. +#[cfg(not(no_global_oom_handling))] fn insert_head(v: &mut [T], is_less: &mut F) where F: FnMut(&T, &T) -> bool, @@ -906,6 +932,7 @@ where /// /// The two slices must be non-empty and `mid` must be in bounds. Buffer `buf` must be long enough /// to hold a copy of the shorter slice. Also, `T` must not be a zero-sized type. +#[cfg(not(no_global_oom_handling))] unsafe fn merge(v: &mut [T], mid: usize, buf: *mut T, is_less: &mut F) where F: FnMut(&T, &T) -> bool, @@ -1026,6 +1053,7 @@ where /// 2. for every `i` in `2..runs.len()`: `runs[i - 2].len > runs[i - 1].len + runs[i].len` /// /// The invariants ensure that the total running time is *O*(*n* \* log(*n*)) worst-case. +#[cfg(not(no_global_oom_handling))] fn merge_sort(v: &mut [T], mut is_less: F) where F: FnMut(&T, &T) -> bool, diff --git a/library/alloc/src/str.rs b/library/alloc/src/str.rs index 879af7cf4d..57279e81a9 100644 --- a/library/alloc/src/str.rs +++ b/library/alloc/src/str.rs @@ -74,6 +74,7 @@ pub use core::str::{RSplitTerminator, SplitTerminator}; /// Note: `str` in `Concat` is not meaningful here. /// This type parameter of the trait only exists to enable another impl. +#[cfg(not(no_global_oom_handling))] #[unstable(feature = "slice_concat_ext", issue = "27747")] impl> Concat for [S] { type Output = String; @@ -83,6 +84,7 @@ impl> Concat for [S] { } } +#[cfg(not(no_global_oom_handling))] #[unstable(feature = "slice_concat_ext", issue = "27747")] impl> Join<&str> for [S] { type Output = String; @@ -92,6 +94,7 @@ impl> Join<&str> for [S] { } } +#[cfg(not(no_global_oom_handling))] macro_rules! specialize_for_lengths { ($separator:expr, $target:expr, $iter:expr; $($num:expr),*) => {{ let mut target = $target; @@ -122,6 +125,7 @@ macro_rules! specialize_for_lengths { }} } +#[cfg(not(no_global_oom_handling))] macro_rules! copy_slice_and_advance { ($target:expr, $bytes:expr) => { let len = $bytes.len(); @@ -139,6 +143,7 @@ macro_rules! copy_slice_and_advance { // the bounds for String-join are S: Borrow and for Vec-join Borrow<[T]> // [T] and str both impl AsRef<[T]> for some T // => s.borrow().as_ref() and we always have slices +#[cfg(not(no_global_oom_handling))] fn join_generic_copy(slice: &[S], sep: &[T]) -> Vec where T: Copy, @@ -205,6 +210,7 @@ impl BorrowMut for String { } } +#[cfg(not(no_global_oom_handling))] #[stable(feature = "rust1", since = "1.0.0")] impl ToOwned for str { type Owned = String; @@ -264,6 +270,7 @@ impl str { /// let s = "this is old"; /// assert_eq!(s, s.replace("cookie monster", "little lamb")); /// ``` + #[cfg(not(no_global_oom_handling))] #[must_use = "this returns the replaced string as a new allocation, \ without modifying the original"] #[stable(feature = "rust1", since = "1.0.0")] @@ -303,6 +310,7 @@ impl str { /// let s = "this is old"; /// assert_eq!(s, s.replacen("cookie monster", "little lamb", 10)); /// ``` + #[cfg(not(no_global_oom_handling))] #[must_use = "this returns the replaced string as a new allocation, \ without modifying the original"] #[stable(feature = "str_replacen", since = "1.16.0")] @@ -358,6 +366,7 @@ impl str { /// /// assert_eq!(new_year, new_year.to_lowercase()); /// ``` + #[cfg(not(no_global_oom_handling))] #[stable(feature = "unicode_case_mapping", since = "1.2.0")] pub fn to_lowercase(&self) -> String { let mut s = String::with_capacity(self.len()); @@ -437,6 +446,7 @@ impl str { /// /// assert_eq!("TSCHÜSS", s.to_uppercase()); /// ``` + #[cfg(not(no_global_oom_handling))] #[stable(feature = "unicode_case_mapping", since = "1.2.0")] pub fn to_uppercase(&self) -> String { let mut s = String::with_capacity(self.len()); @@ -496,6 +506,7 @@ impl str { /// // this will panic at runtime /// "0123456789abcdef".repeat(usize::MAX); /// ``` + #[cfg(not(no_global_oom_handling))] #[stable(feature = "repeat_str", since = "1.16.0")] pub fn repeat(&self, n: usize) -> String { unsafe { String::from_utf8_unchecked(self.as_bytes().repeat(n)) } @@ -522,6 +533,7 @@ impl str { /// /// [`make_ascii_uppercase`]: str::make_ascii_uppercase /// [`to_uppercase`]: #method.to_uppercase + #[cfg(not(no_global_oom_handling))] #[stable(feature = "ascii_methods_on_intrinsics", since = "1.23.0")] #[inline] pub fn to_ascii_uppercase(&self) -> String { @@ -552,6 +564,7 @@ impl str { /// /// [`make_ascii_lowercase`]: str::make_ascii_lowercase /// [`to_lowercase`]: #method.to_lowercase + #[cfg(not(no_global_oom_handling))] #[stable(feature = "ascii_methods_on_intrinsics", since = "1.23.0")] #[inline] pub fn to_ascii_lowercase(&self) -> String { diff --git a/library/alloc/src/string.rs b/library/alloc/src/string.rs index f4ec4a36ff..93f5fe45cd 100644 --- a/library/alloc/src/string.rs +++ b/library/alloc/src/string.rs @@ -42,20 +42,33 @@ #![stable(feature = "rust1", since = "1.0.0")] +#[cfg(not(no_global_oom_handling))] use core::char::{decode_utf16, REPLACEMENT_CHARACTER}; use core::fmt; use core::hash; -use core::iter::{FromIterator, FusedIterator}; +#[cfg(not(no_global_oom_handling))] +use core::iter::FromIterator; +use core::iter::{from_fn, FusedIterator}; +#[cfg(not(no_global_oom_handling))] +use core::ops::Add; +#[cfg(not(no_global_oom_handling))] +use core::ops::AddAssign; +#[cfg(not(no_global_oom_handling))] use core::ops::Bound::{Excluded, Included, Unbounded}; -use core::ops::{self, Add, AddAssign, Index, IndexMut, Range, RangeBounds}; +use core::ops::{self, Index, IndexMut, Range, RangeBounds}; use core::ptr; use core::slice; -use core::str::{lossy, pattern::Pattern}; +#[cfg(not(no_global_oom_handling))] +use core::str::lossy; +use core::str::pattern::Pattern; +#[cfg(not(no_global_oom_handling))] use crate::borrow::{Cow, ToOwned}; use crate::boxed::Box; use crate::collections::TryReserveError; -use crate::str::{self, from_boxed_utf8_unchecked, Chars, FromStr, Utf8Error}; +use crate::str::{self, Chars, Utf8Error}; +#[cfg(not(no_global_oom_handling))] +use crate::str::{from_boxed_utf8_unchecked, FromStr}; use crate::vec::Vec; /// A UTF-8–encoded, growable string. @@ -314,7 +327,8 @@ pub struct String { /// assert_eq!(vec![0, 159], value.unwrap_err().into_bytes()); /// ``` #[stable(feature = "rust1", since = "1.0.0")] -#[derive(Debug, Clone, PartialEq, Eq)] +#[cfg_attr(not(no_global_oom_handling), derive(Clone))] +#[derive(Debug, PartialEq, Eq)] pub struct FromUtf8Error { bytes: Vec, error: Utf8Error, @@ -403,6 +417,7 @@ impl String { /// // ...but this may make the string reallocate /// s.push('a'); /// ``` + #[cfg(not(no_global_oom_handling))] #[inline] #[doc(alias = "alloc")] #[doc(alias = "malloc")] @@ -535,6 +550,7 @@ impl String { /// /// assert_eq!("Hello �World", output); /// ``` + #[cfg(not(no_global_oom_handling))] #[stable(feature = "rust1", since = "1.0.0")] pub fn from_utf8_lossy(v: &[u8]) -> Cow<'_, str> { let mut iter = lossy::Utf8Lossy::from_bytes(v).chunks(); @@ -587,6 +603,7 @@ impl String { /// 0xD800, 0x0069, 0x0063]; /// assert!(String::from_utf16(v).is_err()); /// ``` + #[cfg(not(no_global_oom_handling))] #[stable(feature = "rust1", since = "1.0.0")] pub fn from_utf16(v: &[u16]) -> Result { // This isn't done via collect::>() for performance reasons. @@ -626,6 +643,7 @@ impl String { /// assert_eq!(String::from("𝄞mus\u{FFFD}ic\u{FFFD}"), /// String::from_utf16_lossy(v)); /// ``` + #[cfg(not(no_global_oom_handling))] #[inline] #[stable(feature = "rust1", since = "1.0.0")] pub fn from_utf16_lossy(v: &[u16]) -> String { @@ -818,12 +836,49 @@ impl String { /// /// assert_eq!("foobar", s); /// ``` + #[cfg(not(no_global_oom_handling))] #[inline] #[stable(feature = "rust1", since = "1.0.0")] pub fn push_str(&mut self, string: &str) { self.vec.extend_from_slice(string.as_bytes()) } + /// Copies elements from `src` range to the end of the string. + /// + /// ## Panics + /// + /// Panics if the starting point or end point do not lie on a [`char`] + /// boundary, or if they're out of bounds. + /// + /// ## Examples + /// + /// ``` + /// #![feature(string_extend_from_within)] + /// let mut string = String::from("abcde"); + /// + /// string.extend_from_within(2..); + /// assert_eq!(string, "abcdecde"); + /// + /// string.extend_from_within(..2); + /// assert_eq!(string, "abcdecdeab"); + /// + /// string.extend_from_within(4..8); + /// assert_eq!(string, "abcdecdeabecde"); + /// ``` + #[cfg(not(no_global_oom_handling))] + #[unstable(feature = "string_extend_from_within", issue = "none")] + pub fn extend_from_within(&mut self, src: R) + where + R: RangeBounds, + { + let src @ Range { start, end } = slice::range(src, ..self.len()); + + assert!(self.is_char_boundary(start)); + assert!(self.is_char_boundary(end)); + + self.vec.extend_from_within(src); + } + /// Returns this `String`'s capacity, in bytes. /// /// # Examples @@ -885,6 +940,7 @@ impl String { /// // ... doesn't actually increase. /// assert_eq!(10, s.capacity()); /// ``` + #[cfg(not(no_global_oom_handling))] #[inline] #[stable(feature = "rust1", since = "1.0.0")] pub fn reserve(&mut self, additional: usize) { @@ -932,6 +988,7 @@ impl String { /// // ... doesn't actually increase. /// assert_eq!(10, s.capacity()); /// ``` + #[cfg(not(no_global_oom_handling))] #[inline] #[stable(feature = "rust1", since = "1.0.0")] pub fn reserve_exact(&mut self, additional: usize) { @@ -1026,6 +1083,7 @@ impl String { /// s.shrink_to_fit(); /// assert_eq!(3, s.capacity()); /// ``` + #[cfg(not(no_global_oom_handling))] #[inline] #[stable(feature = "rust1", since = "1.0.0")] pub fn shrink_to_fit(&mut self) { @@ -1053,6 +1111,7 @@ impl String { /// s.shrink_to(0); /// assert!(s.capacity() >= 3); /// ``` + #[cfg(not(no_global_oom_handling))] #[inline] #[unstable(feature = "shrink_to", reason = "new API", issue = "56431")] pub fn shrink_to(&mut self, min_capacity: usize) { @@ -1074,6 +1133,7 @@ impl String { /// /// assert_eq!("abc123", s); /// ``` + #[cfg(not(no_global_oom_handling))] #[inline] #[stable(feature = "rust1", since = "1.0.0")] pub fn push(&mut self, ch: char) { @@ -1222,6 +1282,7 @@ impl String { /// s.remove_matches("ana"); /// assert_eq!("bna", s); /// ``` + #[cfg(not(no_global_oom_handling))] #[unstable(feature = "string_remove_matches", reason = "new API", issue = "72826")] pub fn remove_matches<'a, P>(&'a mut self, pat: P) where @@ -1229,32 +1290,49 @@ impl String { { use core::str::pattern::Searcher; - let matches = { + let rejections = { let mut searcher = pat.into_searcher(self); - let mut matches = Vec::new(); - - while let Some(m) = searcher.next_match() { - matches.push(m); - } - - matches + // Per Searcher::next: + // + // A Match result needs to contain the whole matched pattern, + // however Reject results may be split up into arbitrary many + // adjacent fragments. Both ranges may have zero length. + // + // In practice the implementation of Searcher::next_match tends to + // be more efficient, so we use it here and do some work to invert + // matches into rejections since that's what we want to copy below. + let mut front = 0; + let rejections: Vec<_> = from_fn(|| { + let (start, end) = searcher.next_match()?; + let prev_front = front; + front = end; + Some((prev_front, start)) + }) + .collect(); + rejections.into_iter().chain(core::iter::once((front, self.len()))) }; - let len = self.len(); - let mut shrunk_by = 0; + let mut len = 0; + let ptr = self.vec.as_mut_ptr(); + + for (start, end) in rejections { + let count = end - start; + if start != len { + // SAFETY: per Searcher::next: + // + // The stream of Match and Reject values up to a Done will + // contain index ranges that are adjacent, non-overlapping, + // covering the whole haystack, and laying on utf8 + // boundaries. + unsafe { + ptr::copy(ptr.add(start), ptr.add(len), count); + } + } + len += count; + } - // SAFETY: start and end will be on utf8 byte boundaries per - // the Searcher docs unsafe { - for (start, end) in matches { - ptr::copy( - self.vec.as_mut_ptr().add(end - shrunk_by), - self.vec.as_mut_ptr().add(start - shrunk_by), - len - end, - ); - shrunk_by += end - start; - } - self.vec.set_len(len - shrunk_by); + self.vec.set_len(len); } } @@ -1352,6 +1430,7 @@ impl String { /// /// assert_eq!("foo", s); /// ``` + #[cfg(not(no_global_oom_handling))] #[inline] #[stable(feature = "rust1", since = "1.0.0")] pub fn insert(&mut self, idx: usize, ch: char) { @@ -1364,6 +1443,7 @@ impl String { } } + #[cfg(not(no_global_oom_handling))] unsafe fn insert_bytes(&mut self, idx: usize, bytes: &[u8]) { let len = self.len(); let amt = bytes.len(); @@ -1397,6 +1477,7 @@ impl String { /// /// assert_eq!("foobar", s); /// ``` + #[cfg(not(no_global_oom_handling))] #[inline] #[stable(feature = "insert_str", since = "1.16.0")] pub fn insert_str(&mut self, idx: usize, string: &str) { @@ -1502,6 +1583,7 @@ impl String { /// assert_eq!(world, "World!"); /// # } /// ``` + #[cfg(not(no_global_oom_handling))] #[inline] #[stable(feature = "string_split_off", since = "1.16.0")] #[must_use = "use `.truncate()` if you don't need the other half"] @@ -1608,6 +1690,7 @@ impl String { /// s.replace_range(..beta_offset, "Α is capital alpha; "); /// assert_eq!(s, "Α is capital alpha; β is beta"); /// ``` + #[cfg(not(no_global_oom_handling))] #[stable(feature = "splice", since = "1.27.0")] pub fn replace_range(&mut self, range: R, replace_with: &str) where @@ -1654,6 +1737,7 @@ impl String { /// /// let b = s.into_boxed_str(); /// ``` + #[cfg(not(no_global_oom_handling))] #[stable(feature = "box_str", since = "1.4.0")] #[inline] pub fn into_boxed_str(self) -> Box { @@ -1748,6 +1832,7 @@ impl fmt::Display for FromUtf16Error { } } +#[cfg(not(no_global_oom_handling))] #[stable(feature = "rust1", since = "1.0.0")] impl Clone for String { fn clone(&self) -> Self { @@ -1759,6 +1844,7 @@ impl Clone for String { } } +#[cfg(not(no_global_oom_handling))] #[stable(feature = "rust1", since = "1.0.0")] impl FromIterator for String { fn from_iter>(iter: I) -> String { @@ -1768,6 +1854,7 @@ impl FromIterator for String { } } +#[cfg(not(no_global_oom_handling))] #[stable(feature = "string_from_iter_by_ref", since = "1.17.0")] impl<'a> FromIterator<&'a char> for String { fn from_iter>(iter: I) -> String { @@ -1777,6 +1864,7 @@ impl<'a> FromIterator<&'a char> for String { } } +#[cfg(not(no_global_oom_handling))] #[stable(feature = "rust1", since = "1.0.0")] impl<'a> FromIterator<&'a str> for String { fn from_iter>(iter: I) -> String { @@ -1786,6 +1874,7 @@ impl<'a> FromIterator<&'a str> for String { } } +#[cfg(not(no_global_oom_handling))] #[stable(feature = "extend_string", since = "1.4.0")] impl FromIterator for String { fn from_iter>(iter: I) -> String { @@ -1804,6 +1893,7 @@ impl FromIterator for String { } } +#[cfg(not(no_global_oom_handling))] #[stable(feature = "box_str2", since = "1.45.0")] impl FromIterator> for String { fn from_iter>>(iter: I) -> String { @@ -1813,6 +1903,7 @@ impl FromIterator> for String { } } +#[cfg(not(no_global_oom_handling))] #[stable(feature = "herd_cows", since = "1.19.0")] impl<'a> FromIterator> for String { fn from_iter>>(iter: I) -> String { @@ -1832,6 +1923,7 @@ impl<'a> FromIterator> for String { } } +#[cfg(not(no_global_oom_handling))] #[stable(feature = "rust1", since = "1.0.0")] impl Extend for String { fn extend>(&mut self, iter: I) { @@ -1852,6 +1944,7 @@ impl Extend for String { } } +#[cfg(not(no_global_oom_handling))] #[stable(feature = "extend_ref", since = "1.2.0")] impl<'a> Extend<&'a char> for String { fn extend>(&mut self, iter: I) { @@ -1869,6 +1962,7 @@ impl<'a> Extend<&'a char> for String { } } +#[cfg(not(no_global_oom_handling))] #[stable(feature = "rust1", since = "1.0.0")] impl<'a> Extend<&'a str> for String { fn extend>(&mut self, iter: I) { @@ -1881,6 +1975,7 @@ impl<'a> Extend<&'a str> for String { } } +#[cfg(not(no_global_oom_handling))] #[stable(feature = "box_str2", since = "1.45.0")] impl Extend> for String { fn extend>>(&mut self, iter: I) { @@ -1888,6 +1983,7 @@ impl Extend> for String { } } +#[cfg(not(no_global_oom_handling))] #[stable(feature = "extend_string", since = "1.4.0")] impl Extend for String { fn extend>(&mut self, iter: I) { @@ -1900,6 +1996,7 @@ impl Extend for String { } } +#[cfg(not(no_global_oom_handling))] #[stable(feature = "herd_cows", since = "1.19.0")] impl<'a> Extend> for String { fn extend>>(&mut self, iter: I) { @@ -2001,8 +2098,11 @@ macro_rules! impl_eq { impl_eq! { String, str } impl_eq! { String, &'a str } +#[cfg(not(no_global_oom_handling))] impl_eq! { Cow<'a, str>, str } +#[cfg(not(no_global_oom_handling))] impl_eq! { Cow<'a, str>, &'b str } +#[cfg(not(no_global_oom_handling))] impl_eq! { Cow<'a, str>, String } #[stable(feature = "rust1", since = "1.0.0")] @@ -2075,6 +2175,7 @@ impl hash::Hash for String { /// let b = " world"; /// let c = a.to_string() + b; /// ``` +#[cfg(not(no_global_oom_handling))] #[stable(feature = "rust1", since = "1.0.0")] impl Add<&str> for String { type Output = String; @@ -2089,6 +2190,7 @@ impl Add<&str> for String { /// Implements the `+=` operator for appending to a `String`. /// /// This has the same behavior as the [`push_str`][String::push_str] method. +#[cfg(not(no_global_oom_handling))] #[stable(feature = "stringaddassign", since = "1.12.0")] impl AddAssign<&str> for String { #[inline] @@ -2221,6 +2323,7 @@ impl ops::DerefMut for String { #[stable(feature = "str_parse_error", since = "1.5.0")] pub type ParseError = core::convert::Infallible; +#[cfg(not(no_global_oom_handling))] #[stable(feature = "rust1", since = "1.0.0")] impl FromStr for String { type Err = core::convert::Infallible; @@ -2264,6 +2367,7 @@ pub trait ToString { /// if the `Display` implementation returns an error. /// This indicates an incorrect `Display` implementation /// since `fmt::Write for String` never returns an error itself. +#[cfg(not(no_global_oom_handling))] #[stable(feature = "rust1", since = "1.0.0")] impl ToString for T { // A common guideline is to not inline generic functions. However, @@ -2272,14 +2376,16 @@ impl ToString for T { // to try to remove it. #[inline] default fn to_string(&self) -> String { - use fmt::Write; let mut buf = String::new(); - buf.write_fmt(format_args!("{}", self)) + let mut formatter = core::fmt::Formatter::new(&mut buf); + // Bypass format_args!() to avoid write_str with zero-length strs + fmt::Display::fmt(self, &mut formatter) .expect("a Display implementation returned an error unexpectedly"); buf } } +#[cfg(not(no_global_oom_handling))] #[stable(feature = "char_to_string_specialization", since = "1.46.0")] impl ToString for char { #[inline] @@ -2288,6 +2394,50 @@ impl ToString for char { } } +#[cfg(not(no_global_oom_handling))] +#[stable(feature = "u8_to_string_specialization", since = "1.54.0")] +impl ToString for u8 { + #[inline] + fn to_string(&self) -> String { + let mut buf = String::with_capacity(3); + let mut n = *self; + if n >= 10 { + if n >= 100 { + buf.push((b'0' + n / 100) as char); + n %= 100; + } + buf.push((b'0' + n / 10) as char); + n %= 10; + } + buf.push((b'0' + n) as char); + buf + } +} + +#[cfg(not(no_global_oom_handling))] +#[stable(feature = "i8_to_string_specialization", since = "1.54.0")] +impl ToString for i8 { + #[inline] + fn to_string(&self) -> String { + let mut buf = String::with_capacity(4); + if self.is_negative() { + buf.push('-'); + } + let mut n = self.unsigned_abs(); + if n >= 10 { + if n >= 100 { + buf.push('1'); + n -= 100; + } + buf.push((b'0' + n / 10) as char); + n %= 10; + } + buf.push((b'0' + n) as char); + buf + } +} + +#[cfg(not(no_global_oom_handling))] #[stable(feature = "str_to_string_specialization", since = "1.9.0")] impl ToString for str { #[inline] @@ -2296,6 +2446,7 @@ impl ToString for str { } } +#[cfg(not(no_global_oom_handling))] #[stable(feature = "cow_str_to_string_specialization", since = "1.17.0")] impl ToString for Cow<'_, str> { #[inline] @@ -2304,6 +2455,7 @@ impl ToString for Cow<'_, str> { } } +#[cfg(not(no_global_oom_handling))] #[stable(feature = "string_to_string_specialization", since = "1.17.0")] impl ToString for String { #[inline] @@ -2336,17 +2488,22 @@ impl AsRef<[u8]> for String { } } +#[cfg(not(no_global_oom_handling))] #[stable(feature = "rust1", since = "1.0.0")] impl From<&str> for String { + /// Converts a `&str` into a [`String`]. + /// + /// The result is allocated on the heap. #[inline] fn from(s: &str) -> String { s.to_owned() } } +#[cfg(not(no_global_oom_handling))] #[stable(feature = "from_mut_str_for_string", since = "1.44.0")] impl From<&mut str> for String { - /// Converts a `&mut str` into a `String`. + /// Converts a `&mut str` into a [`String`]. /// /// The result is allocated on the heap. #[inline] @@ -2355,8 +2512,12 @@ impl From<&mut str> for String { } } +#[cfg(not(no_global_oom_handling))] #[stable(feature = "from_ref_string", since = "1.35.0")] impl From<&String> for String { + /// Converts a `&String` into a [`String`]. + /// + /// This clones `s` and returns the clone. #[inline] fn from(s: &String) -> String { s.clone() @@ -2367,7 +2528,7 @@ impl From<&String> for String { #[cfg(not(test))] #[stable(feature = "string_from_box", since = "1.18.0")] impl From> for String { - /// Converts the given boxed `str` slice to a `String`. + /// Converts the given boxed `str` slice to a [`String`]. /// It is notable that the `str` slice is owned. /// /// # Examples @@ -2386,9 +2547,10 @@ impl From> for String { } } +#[cfg(not(no_global_oom_handling))] #[stable(feature = "box_from_str", since = "1.20.0")] impl From for Box { - /// Converts the given `String` to a boxed `str` slice that is owned. + /// Converts the given [`String`] to a boxed `str` slice that is owned. /// /// # Examples /// @@ -2406,16 +2568,34 @@ impl From for Box { } } +#[cfg(not(no_global_oom_handling))] #[stable(feature = "string_from_cow_str", since = "1.14.0")] impl<'a> From> for String { + /// Converts a clone-on-write string to an owned + /// instance of [`String`]. + /// + /// This extracts the owned string, + /// clones the string if it is not already owned. + /// + /// # Example + /// + /// ``` + /// # use std::borrow::Cow; + /// // If the string is not owned... + /// let cow: Cow = Cow::Borrowed("eggplant"); + /// // It will allocate on the heap and copy the string. + /// let owned: String = String::from(cow); + /// assert_eq!(&owned[..], "eggplant"); + /// ``` fn from(s: Cow<'a, str>) -> String { s.into_owned() } } +#[cfg(not(no_global_oom_handling))] #[stable(feature = "rust1", since = "1.0.0")] impl<'a> From<&'a str> for Cow<'a, str> { - /// Converts a string slice into a Borrowed variant. + /// Converts a string slice into a [`Borrowed`] variant. /// No heap allocation is performed, and the string /// is not copied. /// @@ -2425,15 +2605,18 @@ impl<'a> From<&'a str> for Cow<'a, str> { /// # use std::borrow::Cow; /// assert_eq!(Cow::from("eggplant"), Cow::Borrowed("eggplant")); /// ``` + /// + /// [`Borrowed`]: crate::borrow::Cow::Borrowed #[inline] fn from(s: &'a str) -> Cow<'a, str> { Cow::Borrowed(s) } } +#[cfg(not(no_global_oom_handling))] #[stable(feature = "rust1", since = "1.0.0")] impl<'a> From for Cow<'a, str> { - /// Converts a String into an Owned variant. + /// Converts a [`String`] into an [`Owned`] variant. /// No heap allocation is performed, and the string /// is not copied. /// @@ -2445,15 +2628,18 @@ impl<'a> From for Cow<'a, str> { /// let s2 = "eggplant".to_string(); /// assert_eq!(Cow::from(s), Cow::<'static, str>::Owned(s2)); /// ``` + /// + /// [`Owned`]: crate::borrow::Cow::Owned #[inline] fn from(s: String) -> Cow<'a, str> { Cow::Owned(s) } } +#[cfg(not(no_global_oom_handling))] #[stable(feature = "cow_from_string_ref", since = "1.28.0")] impl<'a> From<&'a String> for Cow<'a, str> { - /// Converts a String reference into a Borrowed variant. + /// Converts a [`String`] reference into a [`Borrowed`] variant. /// No heap allocation is performed, and the string /// is not copied. /// @@ -2464,12 +2650,15 @@ impl<'a> From<&'a String> for Cow<'a, str> { /// let s = "eggplant".to_string(); /// assert_eq!(Cow::from(&s), Cow::Borrowed("eggplant")); /// ``` + /// + /// [`Borrowed`]: crate::borrow::Cow::Borrowed #[inline] fn from(s: &'a String) -> Cow<'a, str> { Cow::Borrowed(s.as_str()) } } +#[cfg(not(no_global_oom_handling))] #[stable(feature = "cow_str_from_iter", since = "1.12.0")] impl<'a> FromIterator for Cow<'a, str> { fn from_iter>(it: I) -> Cow<'a, str> { @@ -2477,6 +2666,7 @@ impl<'a> FromIterator for Cow<'a, str> { } } +#[cfg(not(no_global_oom_handling))] #[stable(feature = "cow_str_from_iter", since = "1.12.0")] impl<'a, 'b> FromIterator<&'b str> for Cow<'a, str> { fn from_iter>(it: I) -> Cow<'a, str> { @@ -2484,6 +2674,7 @@ impl<'a, 'b> FromIterator<&'b str> for Cow<'a, str> { } } +#[cfg(not(no_global_oom_handling))] #[stable(feature = "cow_str_from_iter", since = "1.12.0")] impl<'a> FromIterator for Cow<'a, str> { fn from_iter>(it: I) -> Cow<'a, str> { @@ -2493,7 +2684,7 @@ impl<'a> FromIterator for Cow<'a, str> { #[stable(feature = "from_string_for_vec_u8", since = "1.14.0")] impl From for Vec { - /// Converts the given `String` to a vector `Vec` that holds values of type `u8`. + /// Converts the given [`String`] to a vector [`Vec`] that holds values of type [`u8`]. /// /// # Examples /// @@ -2512,6 +2703,7 @@ impl From for Vec { } } +#[cfg(not(no_global_oom_handling))] #[stable(feature = "rust1", since = "1.0.0")] impl fmt::Write for String { #[inline] @@ -2635,8 +2827,17 @@ impl DoubleEndedIterator for Drain<'_> { #[stable(feature = "fused", since = "1.26.0")] impl FusedIterator for Drain<'_> {} +#[cfg(not(no_global_oom_handling))] #[stable(feature = "from_char_for_string", since = "1.46.0")] impl From for String { + /// Allocates an owned [`String`] from a single character. + /// + /// # Example + /// ```rust + /// let c: char = 'a'; + /// let s: String = String::from(c); + /// assert_eq!("a", &s[..]); + /// ``` #[inline] fn from(c: char) -> Self { c.to_string() diff --git a/library/alloc/src/sync.rs b/library/alloc/src/sync.rs index 1b7e656cef..a8fa028fc9 100644 --- a/library/alloc/src/sync.rs +++ b/library/alloc/src/sync.rs @@ -12,23 +12,31 @@ use core::fmt; use core::hash::{Hash, Hasher}; use core::hint; use core::intrinsics::abort; +#[cfg(not(no_global_oom_handling))] use core::iter; use core::marker::{PhantomData, Unpin, Unsize}; -use core::mem::{self, align_of_val_raw, size_of_val}; +#[cfg(not(no_global_oom_handling))] +use core::mem::size_of_val; +use core::mem::{self, align_of_val_raw}; use core::ops::{CoerceUnsized, Deref, DispatchFromDyn, Receiver}; use core::pin::Pin; use core::ptr::{self, NonNull}; +#[cfg(not(no_global_oom_handling))] use core::slice::from_raw_parts_mut; use core::sync::atomic; use core::sync::atomic::Ordering::{Acquire, Relaxed, Release, SeqCst}; -use crate::alloc::{ - box_free, handle_alloc_error, AllocError, Allocator, Global, Layout, WriteCloneIntoRaw, -}; +#[cfg(not(no_global_oom_handling))] +use crate::alloc::handle_alloc_error; +#[cfg(not(no_global_oom_handling))] +use crate::alloc::{box_free, WriteCloneIntoRaw}; +use crate::alloc::{AllocError, Allocator, Global, Layout}; use crate::borrow::{Cow, ToOwned}; use crate::boxed::Box; use crate::rc::is_dangling; +#[cfg(not(no_global_oom_handling))] use crate::string::String; +#[cfg(not(no_global_oom_handling))] use crate::vec::Vec; #[cfg(test)] @@ -431,6 +439,7 @@ impl Arc { /// /// assert_eq!(*five, 5) /// ``` + #[cfg(not(no_global_oom_handling))] #[unstable(feature = "new_uninit", issue = "63291")] pub fn new_uninit() -> Arc> { unsafe { @@ -462,6 +471,7 @@ impl Arc { /// ``` /// /// [zeroed]: ../../std/mem/union.MaybeUninit.html#method.zeroed + #[cfg(not(no_global_oom_handling))] #[unstable(feature = "new_uninit", issue = "63291")] pub fn new_zeroed() -> Arc> { unsafe { @@ -635,6 +645,7 @@ impl Arc<[T]> { /// /// assert_eq!(*values, [1, 2, 3]) /// ``` + #[cfg(not(no_global_oom_handling))] #[unstable(feature = "new_uninit", issue = "63291")] pub fn new_uninit_slice(len: usize) -> Arc<[mem::MaybeUninit]> { unsafe { Arc::from_ptr(Arc::allocate_for_slice(len)) } @@ -660,6 +671,7 @@ impl Arc<[T]> { /// ``` /// /// [zeroed]: ../../std/mem/union.MaybeUninit.html#method.zeroed + #[cfg(not(no_global_oom_handling))] #[unstable(feature = "new_uninit", issue = "63291")] pub fn new_zeroed_slice(len: usize) -> Arc<[mem::MaybeUninit]> { unsafe { @@ -1072,6 +1084,7 @@ impl Arc { /// /// The function `mem_to_arcinner` is called with the data pointer /// and must return back a (potentially fat)-pointer for the `ArcInner`. + #[cfg(not(no_global_oom_handling))] unsafe fn allocate_for_layout( value_layout: Layout, allocate: impl FnOnce(Layout) -> Result, AllocError>, @@ -1120,6 +1133,7 @@ impl Arc { } /// Allocates an `ArcInner` with sufficient space for an unsized inner value. + #[cfg(not(no_global_oom_handling))] unsafe fn allocate_for_ptr(ptr: *const T) -> *mut ArcInner { // Allocate for the `ArcInner` using the given value. unsafe { @@ -1131,6 +1145,7 @@ impl Arc { } } + #[cfg(not(no_global_oom_handling))] fn from_box(v: Box) -> Arc { unsafe { let (box_unique, alloc) = Box::into_unique(v); @@ -1156,6 +1171,7 @@ impl Arc { impl Arc<[T]> { /// Allocates an `ArcInner<[T]>` with the given length. + #[cfg(not(no_global_oom_handling))] unsafe fn allocate_for_slice(len: usize) -> *mut ArcInner<[T]> { unsafe { Self::allocate_for_layout( @@ -1169,6 +1185,7 @@ impl Arc<[T]> { /// Copy elements from slice into newly allocated Arc<\[T\]> /// /// Unsafe because the caller must either take ownership or bind `T: Copy`. + #[cfg(not(no_global_oom_handling))] unsafe fn copy_from_slice(v: &[T]) -> Arc<[T]> { unsafe { let ptr = Self::allocate_for_slice(v.len()); @@ -1182,6 +1199,7 @@ impl Arc<[T]> { /// Constructs an `Arc<[T]>` from an iterator known to be of a certain size. /// /// Behavior is undefined should the size be wrong. + #[cfg(not(no_global_oom_handling))] unsafe fn from_iter_exact(iter: impl iter::Iterator, len: usize) -> Arc<[T]> { // Panic guard while cloning T elements. // In the event of a panic, elements that have been written @@ -1229,10 +1247,12 @@ impl Arc<[T]> { } /// Specialization trait used for `From<&[T]>`. +#[cfg(not(no_global_oom_handling))] trait ArcFromSlice { fn from_slice(slice: &[T]) -> Self; } +#[cfg(not(no_global_oom_handling))] impl ArcFromSlice for Arc<[T]> { #[inline] default fn from_slice(v: &[T]) -> Self { @@ -1240,6 +1260,7 @@ impl ArcFromSlice for Arc<[T]> { } } +#[cfg(not(no_global_oom_handling))] impl ArcFromSlice for Arc<[T]> { #[inline] fn from_slice(v: &[T]) -> Self { @@ -1341,6 +1362,7 @@ impl Arc { /// assert_eq!(*data, 8); /// assert_eq!(*other_data, 12); /// ``` + #[cfg(not(no_global_oom_handling))] #[inline] #[stable(feature = "arc_unique", since = "1.4.0")] pub fn make_mut(this: &mut Self) -> &mut T { @@ -1993,7 +2015,7 @@ impl Default for Weak { } #[stable(feature = "arc_weak", since = "1.4.0")] -impl Drop for Weak { +unsafe impl<#[may_dangle] T: ?Sized> Drop for Weak { /// Drops the `Weak` pointer. /// /// # Examples @@ -2283,6 +2305,7 @@ impl From for Arc { } } +#[cfg(not(no_global_oom_handling))] #[stable(feature = "shared_from_slice", since = "1.21.0")] impl From<&[T]> for Arc<[T]> { /// Allocate a reference-counted slice and fill it by cloning `v`'s items. @@ -2301,6 +2324,7 @@ impl From<&[T]> for Arc<[T]> { } } +#[cfg(not(no_global_oom_handling))] #[stable(feature = "shared_from_slice", since = "1.21.0")] impl From<&str> for Arc { /// Allocate a reference-counted `str` and copy `v` into it. @@ -2319,6 +2343,7 @@ impl From<&str> for Arc { } } +#[cfg(not(no_global_oom_handling))] #[stable(feature = "shared_from_slice", since = "1.21.0")] impl From for Arc { /// Allocate a reference-counted `str` and copy `v` into it. @@ -2337,6 +2362,7 @@ impl From for Arc { } } +#[cfg(not(no_global_oom_handling))] #[stable(feature = "shared_from_slice", since = "1.21.0")] impl From> for Arc { /// Move a boxed object to a new, reference-counted allocation. @@ -2355,6 +2381,7 @@ impl From> for Arc { } } +#[cfg(not(no_global_oom_handling))] #[stable(feature = "shared_from_slice", since = "1.21.0")] impl From> for Arc<[T]> { /// Allocate a reference-counted slice and move `v`'s items into it. @@ -2386,6 +2413,18 @@ where B: ToOwned + ?Sized, Arc: From<&'a B> + From, { + /// Create an atomically reference-counted pointer from + /// a clone-on-write pointer by copying its content. + /// + /// # Example + /// + /// ```rust + /// # use std::sync::Arc; + /// # use std::borrow::Cow; + /// let cow: Cow = Cow::Borrowed("eggplant"); + /// let shared: Arc = Arc::from(cow); + /// assert_eq!("eggplant", &shared[..]); + /// ``` #[inline] fn from(cow: Cow<'a, B>) -> Arc { match cow { @@ -2408,6 +2447,7 @@ impl TryFrom> for Arc<[T; N]> { } } +#[cfg(not(no_global_oom_handling))] #[stable(feature = "shared_from_iter", since = "1.37.0")] impl iter::FromIterator for Arc<[T]> { /// Takes each element in the `Iterator` and collects it into an `Arc<[T]>`. @@ -2458,12 +2498,14 @@ trait ToArcSlice: Iterator + Sized { fn to_arc_slice(self) -> Arc<[T]>; } +#[cfg(not(no_global_oom_handling))] impl> ToArcSlice for I { default fn to_arc_slice(self) -> Arc<[T]> { self.collect::>().into() } } +#[cfg(not(no_global_oom_handling))] impl> ToArcSlice for I { fn to_arc_slice(self) -> Arc<[T]> { // This is the case for a `TrustedLen` iterator. diff --git a/library/alloc/src/vec/cow.rs b/library/alloc/src/vec/cow.rs index 73d15d3064..64943a273c 100644 --- a/library/alloc/src/vec/cow.rs +++ b/library/alloc/src/vec/cow.rs @@ -5,6 +5,12 @@ use super::Vec; #[stable(feature = "cow_from_vec", since = "1.8.0")] impl<'a, T: Clone> From<&'a [T]> for Cow<'a, [T]> { + /// Creates a [`Borrowed`] variant of [`Cow`] + /// from a slice. + /// + /// This conversion does not allocate or clone the data. + /// + /// [`Borrowed`]: crate::borrow::Cow::Borrowed fn from(s: &'a [T]) -> Cow<'a, [T]> { Cow::Borrowed(s) } @@ -12,6 +18,12 @@ impl<'a, T: Clone> From<&'a [T]> for Cow<'a, [T]> { #[stable(feature = "cow_from_vec", since = "1.8.0")] impl<'a, T: Clone> From> for Cow<'a, [T]> { + /// Creates an [`Owned`] variant of [`Cow`] + /// from an owned instance of [`Vec`]. + /// + /// This conversion does not allocate or clone the data. + /// + /// [`Owned`]: crate::borrow::Cow::Owned fn from(v: Vec) -> Cow<'a, [T]> { Cow::Owned(v) } @@ -19,6 +31,12 @@ impl<'a, T: Clone> From> for Cow<'a, [T]> { #[stable(feature = "cow_from_vec_ref", since = "1.28.0")] impl<'a, T: Clone> From<&'a Vec> for Cow<'a, [T]> { + /// Creates a [`Borrowed`] variant of [`Cow`] + /// from a reference to [`Vec`]. + /// + /// This conversion does not allocate or clone the data. + /// + /// [`Borrowed`]: crate::borrow::Cow::Borrowed fn from(v: &'a Vec) -> Cow<'a, [T]> { Cow::Borrowed(v.as_slice()) } diff --git a/library/alloc/src/vec/into_iter.rs b/library/alloc/src/vec/into_iter.rs index 324e894baf..8da4d995ba 100644 --- a/library/alloc/src/vec/into_iter.rs +++ b/library/alloc/src/vec/into_iter.rs @@ -94,6 +94,7 @@ impl IntoIter { /// (&mut into_iter).for_each(core::mem::drop); /// unsafe { core::ptr::write(&mut into_iter, Vec::new().into_iter()); } /// ``` + #[cfg(not(no_global_oom_handling))] pub(super) fn forget_allocation_drop_remaining(&mut self) { let remaining = self.as_raw_mut_slice(); @@ -224,6 +225,7 @@ where const MAY_HAVE_SIDE_EFFECT: bool = false; } +#[cfg(not(no_global_oom_handling))] #[stable(feature = "vec_into_iter_clone", since = "1.8.0")] impl Clone for IntoIter { #[cfg(not(test))] @@ -262,9 +264,11 @@ unsafe impl<#[may_dangle] T, A: Allocator> Drop for IntoIter { } #[unstable(issue = "none", feature = "inplace_iteration")] +#[doc(hidden)] unsafe impl InPlaceIterable for IntoIter {} #[unstable(issue = "none", feature = "inplace_iteration")] +#[doc(hidden)] unsafe impl SourceIter for IntoIter { type Source = Self; diff --git a/library/alloc/src/vec/is_zero.rs b/library/alloc/src/vec/is_zero.rs index b5739970b6..0efc4893c3 100644 --- a/library/alloc/src/vec/is_zero.rs +++ b/library/alloc/src/vec/is_zero.rs @@ -69,3 +69,36 @@ unsafe impl IsZero for Option> { self.is_none() } } + +// `Option` and similar have a representation guarantee that +// they're the same size as the corresponding `u32` type, as well as a guarantee +// that transmuting between `NonZeroU32` and `Option` works. +// While the documentation officially makes it UB to transmute from `None`, +// we're the standard library so we can make extra inferences, and we know that +// the only niche available to represent `None` is the one that's all zeros. + +macro_rules! impl_is_zero_option_of_nonzero { + ($($t:ident,)+) => {$( + unsafe impl IsZero for Option { + #[inline] + fn is_zero(&self) -> bool { + self.is_none() + } + } + )+}; +} + +impl_is_zero_option_of_nonzero!( + NonZeroU8, + NonZeroU16, + NonZeroU32, + NonZeroU64, + NonZeroU128, + NonZeroI8, + NonZeroI16, + NonZeroI32, + NonZeroI64, + NonZeroI128, + NonZeroUsize, + NonZeroIsize, +); diff --git a/library/alloc/src/vec/mod.rs b/library/alloc/src/vec/mod.rs index 85c9446689..4a1d564e2a 100644 --- a/library/alloc/src/vec/mod.rs +++ b/library/alloc/src/vec/mod.rs @@ -53,12 +53,16 @@ #![stable(feature = "rust1", since = "1.0.0")] -use core::cmp::{self, Ordering}; +#[cfg(not(no_global_oom_handling))] +use core::cmp; +use core::cmp::Ordering; use core::convert::TryFrom; use core::fmt; use core::hash::{Hash, Hasher}; use core::intrinsics::{arith_offset, assume}; -use core::iter::{self, FromIterator}; +use core::iter; +#[cfg(not(no_global_oom_handling))] +use core::iter::FromIterator; use core::marker::PhantomData; use core::mem::{self, ManuallyDrop, MaybeUninit}; use core::ops::{self, Index, IndexMut, Range, RangeBounds}; @@ -76,9 +80,11 @@ pub use self::drain_filter::DrainFilter; mod drain_filter; +#[cfg(not(no_global_oom_handling))] #[stable(feature = "vec_splice", since = "1.21.0")] pub use self::splice::Splice; +#[cfg(not(no_global_oom_handling))] mod splice; #[stable(feature = "drain", since = "1.6.0")] @@ -86,44 +92,60 @@ pub use self::drain::Drain; mod drain; +#[cfg(not(no_global_oom_handling))] mod cow; +#[cfg(not(no_global_oom_handling))] pub(crate) use self::into_iter::AsIntoIter; #[stable(feature = "rust1", since = "1.0.0")] pub use self::into_iter::IntoIter; mod into_iter; +#[cfg(not(no_global_oom_handling))] use self::is_zero::IsZero; mod is_zero; +#[cfg(not(no_global_oom_handling))] mod source_iter_marker; mod partial_eq; +#[cfg(not(no_global_oom_handling))] use self::spec_from_elem::SpecFromElem; +#[cfg(not(no_global_oom_handling))] mod spec_from_elem; +#[cfg(not(no_global_oom_handling))] use self::set_len_on_drop::SetLenOnDrop; +#[cfg(not(no_global_oom_handling))] mod set_len_on_drop; +#[cfg(not(no_global_oom_handling))] use self::in_place_drop::InPlaceDrop; +#[cfg(not(no_global_oom_handling))] mod in_place_drop; +#[cfg(not(no_global_oom_handling))] use self::spec_from_iter_nested::SpecFromIterNested; +#[cfg(not(no_global_oom_handling))] mod spec_from_iter_nested; +#[cfg(not(no_global_oom_handling))] use self::spec_from_iter::SpecFromIter; +#[cfg(not(no_global_oom_handling))] mod spec_from_iter; +#[cfg(not(no_global_oom_handling))] use self::spec_extend::SpecExtend; +#[cfg(not(no_global_oom_handling))] mod spec_extend; /// A contiguous growable array type, written as `Vec` and pronounced 'vector'. @@ -435,6 +457,7 @@ impl Vec { /// assert_eq!(vec.len(), 11); /// assert!(vec.capacity() >= 11); /// ``` + #[cfg(not(no_global_oom_handling))] #[inline] #[doc(alias = "malloc")] #[stable(feature = "rust1", since = "1.0.0")] @@ -574,6 +597,7 @@ impl Vec { /// assert_eq!(vec.len(), 11); /// assert!(vec.capacity() >= 11); /// ``` + #[cfg(not(no_global_oom_handling))] #[inline] #[unstable(feature = "allocator_api", issue = "32838")] pub fn with_capacity_in(capacity: usize, alloc: A) -> Self { @@ -774,6 +798,7 @@ impl Vec { /// vec.reserve(10); /// assert!(vec.capacity() >= 11); /// ``` + #[cfg(not(no_global_oom_handling))] #[doc(alias = "realloc")] #[stable(feature = "rust1", since = "1.0.0")] pub fn reserve(&mut self, additional: usize) { @@ -800,6 +825,7 @@ impl Vec { /// vec.reserve_exact(10); /// assert!(vec.capacity() >= 11); /// ``` + #[cfg(not(no_global_oom_handling))] #[doc(alias = "realloc")] #[stable(feature = "rust1", since = "1.0.0")] pub fn reserve_exact(&mut self, additional: usize) { @@ -895,11 +921,12 @@ impl Vec { /// /// ``` /// let mut vec = Vec::with_capacity(10); - /// vec.extend([1, 2, 3].iter().cloned()); + /// vec.extend([1, 2, 3]); /// assert_eq!(vec.capacity(), 10); /// vec.shrink_to_fit(); /// assert!(vec.capacity() >= 3); /// ``` + #[cfg(not(no_global_oom_handling))] #[doc(alias = "realloc")] #[stable(feature = "rust1", since = "1.0.0")] pub fn shrink_to_fit(&mut self) { @@ -923,13 +950,14 @@ impl Vec { /// ``` /// #![feature(shrink_to)] /// let mut vec = Vec::with_capacity(10); - /// vec.extend([1, 2, 3].iter().cloned()); + /// vec.extend([1, 2, 3]); /// assert_eq!(vec.capacity(), 10); /// vec.shrink_to(4); /// assert!(vec.capacity() >= 4); /// vec.shrink_to(0); /// assert!(vec.capacity() >= 3); /// ``` + #[cfg(not(no_global_oom_handling))] #[doc(alias = "realloc")] #[unstable(feature = "shrink_to", reason = "new API", issue = "56431")] pub fn shrink_to(&mut self, min_capacity: usize) { @@ -956,12 +984,13 @@ impl Vec { /// /// ``` /// let mut vec = Vec::with_capacity(10); - /// vec.extend([1, 2, 3].iter().cloned()); + /// vec.extend([1, 2, 3]); /// /// assert_eq!(vec.capacity(), 10); /// let slice = vec.into_boxed_slice(); /// assert_eq!(slice.into_vec().capacity(), 3); /// ``` + #[cfg(not(no_global_oom_handling))] #[stable(feature = "rust1", since = "1.0.0")] pub fn into_boxed_slice(mut self) -> Box<[T], A> { unsafe { @@ -1299,6 +1328,7 @@ impl Vec { /// vec.insert(4, 5); /// assert_eq!(vec, [1, 4, 2, 3, 5]); /// ``` + #[cfg(not(no_global_oom_handling))] #[stable(feature = "rust1", since = "1.0.0")] pub fn insert(&mut self, index: usize, element: T) { #[cold] @@ -1589,6 +1619,8 @@ impl Vec { let prev_ptr = ptr.add(gap.write.wrapping_sub(1)); if same_bucket(&mut *read_ptr, &mut *prev_ptr) { + // Increase `gap.read` now since the drop may panic. + gap.read += 1; /* We have found duplicate, drop it in-place */ ptr::drop_in_place(read_ptr); } else { @@ -1601,9 +1633,8 @@ impl Vec { /* We have filled that place, so go further */ gap.write += 1; + gap.read += 1; } - - gap.read += 1; } /* Technically we could let `gap` clean up with its Drop, but @@ -1627,6 +1658,7 @@ impl Vec { /// vec.push(3); /// assert_eq!(vec, [1, 2, 3]); /// ``` + #[cfg(not(no_global_oom_handling))] #[inline] #[stable(feature = "rust1", since = "1.0.0")] pub fn push(&mut self, value: T) { @@ -1680,6 +1712,7 @@ impl Vec { /// assert_eq!(vec, [1, 2, 3, 4, 5, 6]); /// assert_eq!(vec2, []); /// ``` + #[cfg(not(no_global_oom_handling))] #[inline] #[stable(feature = "append", since = "1.4.0")] pub fn append(&mut self, other: &mut Self) { @@ -1690,6 +1723,7 @@ impl Vec { } /// Appends elements to `Self` from other buffer. + #[cfg(not(no_global_oom_handling))] #[inline] unsafe fn append_elements(&mut self, other: *const [T]) { let count = unsafe { (*other).len() }; @@ -1827,6 +1861,7 @@ impl Vec { /// assert_eq!(vec, [1]); /// assert_eq!(vec2, [2, 3]); /// ``` + #[cfg(not(no_global_oom_handling))] #[inline] #[must_use = "use `.truncate()` if you don't need the other half"] #[stable(feature = "split_off", since = "1.4.0")] @@ -1891,6 +1926,7 @@ impl Vec { /// vec.resize_with(4, || { p *= 2; p }); /// assert_eq!(vec, [2, 4, 8, 16]); /// ``` + #[cfg(not(no_global_oom_handling))] #[stable(feature = "vec_resize_with", since = "1.33.0")] pub fn resize_with(&mut self, new_len: usize, f: F) where @@ -1926,6 +1962,7 @@ impl Vec { /// static_ref[0] += 1; /// assert_eq!(static_ref, &[2, 2, 3]); /// ``` + #[cfg(not(no_global_oom_handling))] #[stable(feature = "vec_leak", since = "1.47.0")] #[inline] pub fn leak<'a>(self) -> &'a mut [T] @@ -2084,6 +2121,7 @@ impl Vec { /// vec.resize(2, 0); /// assert_eq!(vec, [1, 2]); /// ``` + #[cfg(not(no_global_oom_handling))] #[stable(feature = "vec_resize", since = "1.5.0")] pub fn resize(&mut self, new_len: usize, value: T) { let len = self.len(); @@ -2114,6 +2152,7 @@ impl Vec { /// ``` /// /// [`extend`]: Vec::extend + #[cfg(not(no_global_oom_handling))] #[stable(feature = "vec_extend_from_slice", since = "1.6.0")] pub fn extend_from_slice(&mut self, other: &[T]) { self.spec_extend(other.iter()) @@ -2135,6 +2174,7 @@ impl Vec { /// vec.extend_from_within(4..8); /// assert_eq!(vec, [0, 1, 2, 3, 4, 2, 3, 4, 0, 1, 4, 2, 3, 4]); /// ``` + #[cfg(not(no_global_oom_handling))] #[stable(feature = "vec_extend_from_within", since = "1.53.0")] pub fn extend_from_within(&mut self, src: R) where @@ -2188,6 +2228,7 @@ impl T> ExtendWith for ExtendFunc { } impl Vec { + #[cfg(not(no_global_oom_handling))] /// Extend the vector by `n` values, using the given generator. fn extend_with>(&mut self, n: usize, mut value: E) { self.reserve(n); @@ -2245,12 +2286,14 @@ impl Vec { //////////////////////////////////////////////////////////////////////////////// #[doc(hidden)] +#[cfg(not(no_global_oom_handling))] #[stable(feature = "rust1", since = "1.0.0")] pub fn from_elem(elem: T, n: usize) -> Vec { ::from_elem(elem, n, Global) } #[doc(hidden)] +#[cfg(not(no_global_oom_handling))] #[unstable(feature = "allocator_api", issue = "32838")] pub fn from_elem_in(elem: T, n: usize, alloc: A) -> Vec { ::from_elem(elem, n, alloc) @@ -2331,6 +2374,7 @@ impl ops::DerefMut for Vec { } } +#[cfg(not(no_global_oom_handling))] #[stable(feature = "rust1", since = "1.0.0")] impl Clone for Vec { #[cfg(not(test))] @@ -2397,6 +2441,7 @@ impl, A: Allocator> IndexMut for Vec { } } +#[cfg(not(no_global_oom_handling))] #[stable(feature = "rust1", since = "1.0.0")] impl FromIterator for Vec { #[inline] @@ -2467,6 +2512,7 @@ impl<'a, T, A: Allocator> IntoIterator for &'a mut Vec { } } +#[cfg(not(no_global_oom_handling))] #[stable(feature = "rust1", since = "1.0.0")] impl Extend for Vec { #[inline] @@ -2488,6 +2534,7 @@ impl Extend for Vec { impl Vec { // leaf method to which various SpecFrom/SpecExtend implementations delegate when // they have no further optimizations to apply + #[cfg(not(no_global_oom_handling))] fn extend_desugared>(&mut self, mut iterator: I) { // This is the case for a general iterator. // @@ -2539,10 +2586,11 @@ impl Vec { /// ``` /// let mut v = vec![1, 2, 3]; /// let new = [7, 8]; - /// let u: Vec<_> = v.splice(..2, new.iter().cloned()).collect(); + /// let u: Vec<_> = v.splice(..2, new).collect(); /// assert_eq!(v, &[7, 8, 3]); /// assert_eq!(u, &[1, 2]); /// ``` + #[cfg(not(no_global_oom_handling))] #[inline] #[stable(feature = "vec_splice", since = "1.21.0")] pub fn splice(&mut self, range: R, replace_with: I) -> Splice<'_, I::IntoIter, A> @@ -2619,6 +2667,7 @@ impl Vec { /// append the entire slice at once. /// /// [`copy_from_slice`]: slice::copy_from_slice +#[cfg(not(no_global_oom_handling))] #[stable(feature = "extend_ref", since = "1.2.0")] impl<'a, T: Copy + 'a, A: Allocator + 'a> Extend<&'a T> for Vec { fn extend>(&mut self, iter: I) { @@ -2713,6 +2762,7 @@ impl AsMut<[T]> for Vec { } } +#[cfg(not(no_global_oom_handling))] #[stable(feature = "rust1", since = "1.0.0")] impl From<&[T]> for Vec { /// Allocate a `Vec` and fill it by cloning `s`'s items. @@ -2732,6 +2782,7 @@ impl From<&[T]> for Vec { } } +#[cfg(not(no_global_oom_handling))] #[stable(feature = "vec_from_mut", since = "1.19.0")] impl From<&mut [T]> for Vec { /// Allocate a `Vec` and fill it by cloning `s`'s items. @@ -2813,6 +2864,7 @@ impl From> for Vec { } // note: test pulls in libstd, which causes errors here +#[cfg(not(no_global_oom_handling))] #[cfg(not(test))] #[stable(feature = "box_from_vec", since = "1.20.0")] impl From> for Box<[T], A> { @@ -2831,6 +2883,7 @@ impl From> for Box<[T], A> { } } +#[cfg(not(no_global_oom_handling))] #[stable(feature = "rust1", since = "1.0.0")] impl From<&str> for Vec { /// Allocate a `Vec` and fill it with a UTF-8 string. diff --git a/library/alloc/src/vec/partial_eq.rs b/library/alloc/src/vec/partial_eq.rs index ff90b6caf4..50e1409610 100644 --- a/library/alloc/src/vec/partial_eq.rs +++ b/library/alloc/src/vec/partial_eq.rs @@ -1,4 +1,5 @@ use crate::alloc::Allocator; +#[cfg(not(no_global_oom_handling))] use crate::borrow::Cow; use super::Vec; @@ -26,8 +27,11 @@ __impl_slice_eq1! { [A: Allocator] &[T], Vec, #[stable(feature = "partiale __impl_slice_eq1! { [A: Allocator] &mut [T], Vec, #[stable(feature = "partialeq_vec_for_ref_slice", since = "1.46.0")] } __impl_slice_eq1! { [A: Allocator] Vec, [U], #[stable(feature = "partialeq_vec_for_slice", since = "1.48.0")] } __impl_slice_eq1! { [A: Allocator] [T], Vec, #[stable(feature = "partialeq_vec_for_slice", since = "1.48.0")] } +#[cfg(not(no_global_oom_handling))] __impl_slice_eq1! { [A: Allocator] Cow<'_, [T]>, Vec where T: Clone, #[stable(feature = "rust1", since = "1.0.0")] } +#[cfg(not(no_global_oom_handling))] __impl_slice_eq1! { [] Cow<'_, [T]>, &[U] where T: Clone, #[stable(feature = "rust1", since = "1.0.0")] } +#[cfg(not(no_global_oom_handling))] __impl_slice_eq1! { [] Cow<'_, [T]>, &mut [U] where T: Clone, #[stable(feature = "rust1", since = "1.0.0")] } __impl_slice_eq1! { [A: Allocator, const N: usize] Vec, [U; N], #[stable(feature = "rust1", since = "1.0.0")] } __impl_slice_eq1! { [A: Allocator, const N: usize] Vec, &[U; N], #[stable(feature = "rust1", since = "1.0.0")] } diff --git a/library/alloc/src/vec/spec_from_iter.rs b/library/alloc/src/vec/spec_from_iter.rs index bbfcc68dae..efa6868473 100644 --- a/library/alloc/src/vec/spec_from_iter.rs +++ b/library/alloc/src/vec/spec_from_iter.rs @@ -1,6 +1,5 @@ use core::mem::ManuallyDrop; use core::ptr::{self}; -use core::slice::{self}; use super::{IntoIter, SpecExtend, SpecFromIterNested, Vec}; @@ -19,9 +18,7 @@ use super::{IntoIter, SpecExtend, SpecFromIterNested, Vec}; /// |where I: | | |where I: | /// | Iterator (default)----------+ | | Iterator (default) | /// | vec::IntoIter | | | TrustedLen | -/// | SourceIterMarker---fallback-+ | | | -/// | slice::Iter | | | -/// | Iterator | +---------------------+ +/// | SourceIterMarker---fallback-+ | +---------------------+ /// +---------------------------------+ /// ``` pub(super) trait SpecFromIter { @@ -65,33 +62,3 @@ impl SpecFromIter> for Vec { vec } } - -impl<'a, T: 'a, I> SpecFromIter<&'a T, I> for Vec -where - I: Iterator, - T: Clone, -{ - default fn from_iter(iterator: I) -> Self { - SpecFromIter::from_iter(iterator.cloned()) - } -} - -// This utilizes `iterator.as_slice().to_vec()` since spec_extend -// must take more steps to reason about the final capacity + length -// and thus do more work. `to_vec()` directly allocates the correct amount -// and fills it exactly. -impl<'a, T: 'a + Clone> SpecFromIter<&'a T, slice::Iter<'a, T>> for Vec { - #[cfg(not(test))] - fn from_iter(iterator: slice::Iter<'a, T>) -> Self { - iterator.as_slice().to_vec() - } - - // HACK(japaric): with cfg(test) the inherent `[T]::to_vec` method, which is - // required for this method definition, is not available. Instead use the - // `slice::to_vec` function which is only available with cfg(test) - // NB see the slice::hack module in slice.rs for more information - #[cfg(test)] - fn from_iter(iterator: slice::Iter<'a, T>) -> Self { - crate::slice::to_vec(iterator.as_slice(), crate::alloc::Global) - } -} diff --git a/library/alloc/src/vec/splice.rs b/library/alloc/src/vec/splice.rs index 0a27b5b62e..bad765c7f5 100644 --- a/library/alloc/src/vec/splice.rs +++ b/library/alloc/src/vec/splice.rs @@ -14,7 +14,7 @@ use super::{Drain, Vec}; /// ``` /// let mut v = vec![0, 1, 2]; /// let new = [7, 8]; -/// let iter: std::vec::Splice<_> = v.splice(1.., new.iter().cloned()); +/// let iter: std::vec::Splice<_> = v.splice(1.., new); /// ``` #[derive(Debug)] #[stable(feature = "vec_splice", since = "1.21.0")] diff --git a/library/alloc/tests/arc.rs b/library/alloc/tests/arc.rs index c02ba26705..ce40b5c9b0 100644 --- a/library/alloc/tests/arc.rs +++ b/library/alloc/tests/arc.rs @@ -195,3 +195,18 @@ fn shared_from_iter_trustedlen_no_fuse() { assert_trusted_len(&iter); assert_eq!(&[Box::new(42), Box::new(24)], &*iter.collect::>()); } + +#[test] +fn weak_may_dangle() { + fn hmm<'a>(val: &'a mut Weak<&'a str>) -> Weak<&'a str> { + val.clone() + } + + // Without #[may_dangle] we get: + let mut val = Weak::new(); + hmm(&mut val); + // ~~~~~~~~ borrowed value does not live long enough + // + // `val` dropped here while still borrowed + // borrow might be used here, when `val` is dropped and runs the `Drop` code for type `std::sync::Weak` +} diff --git a/library/alloc/tests/lib.rs b/library/alloc/tests/lib.rs index 25a83a0b01..3143afa269 100644 --- a/library/alloc/tests/lib.rs +++ b/library/alloc/tests/lib.rs @@ -17,7 +17,6 @@ #![feature(binary_heap_as_slice)] #![feature(inplace_iteration)] #![feature(iter_map_while)] -#![feature(vecdeque_binary_search)] #![feature(slice_group_by)] #![feature(slice_partition_dedup)] #![feature(vec_spare_capacity)] diff --git a/library/alloc/tests/rc.rs b/library/alloc/tests/rc.rs index 501b4f0f81..efb39a6096 100644 --- a/library/alloc/tests/rc.rs +++ b/library/alloc/tests/rc.rs @@ -191,3 +191,18 @@ fn shared_from_iter_trustedlen_no_fuse() { assert_trusted_len(&iter); assert_eq!(&[Box::new(42), Box::new(24)], &*iter.collect::>()); } + +#[test] +fn weak_may_dangle() { + fn hmm<'a>(val: &'a mut Weak<&'a str>) -> Weak<&'a str> { + val.clone() + } + + // Without #[may_dangle] we get: + let mut val = Weak::new(); + hmm(&mut val); + // ~~~~~~~~ borrowed value does not live long enough + // + // `val` dropped here while still borrowed + // borrow might be used here, when `val` is dropped and runs the `Drop` code for type `std::rc::Weak` +} diff --git a/library/alloc/tests/vec.rs b/library/alloc/tests/vec.rs index ad69234403..c203cdafec 100644 --- a/library/alloc/tests/vec.rs +++ b/library/alloc/tests/vec.rs @@ -793,7 +793,7 @@ fn test_drain_leak() { fn test_splice() { let mut v = vec![1, 2, 3, 4, 5]; let a = [10, 11, 12]; - v.splice(2..4, a.iter().cloned()); + v.splice(2..4, a); assert_eq!(v, &[1, 2, 10, 11, 12, 5]); v.splice(1..3, Some(20)); assert_eq!(v, &[1, 20, 11, 12, 5]); @@ -803,7 +803,7 @@ fn test_splice() { fn test_splice_inclusive_range() { let mut v = vec![1, 2, 3, 4, 5]; let a = [10, 11, 12]; - let t1: Vec<_> = v.splice(2..=3, a.iter().cloned()).collect(); + let t1: Vec<_> = v.splice(2..=3, a).collect(); assert_eq!(v, &[1, 2, 10, 11, 12, 5]); assert_eq!(t1, &[3, 4]); let t2: Vec<_> = v.splice(1..=2, Some(20)).collect(); @@ -816,7 +816,7 @@ fn test_splice_inclusive_range() { fn test_splice_out_of_bounds() { let mut v = vec![1, 2, 3, 4, 5]; let a = [10, 11, 12]; - v.splice(5..6, a.iter().cloned()); + v.splice(5..6, a); } #[test] @@ -824,7 +824,7 @@ fn test_splice_out_of_bounds() { fn test_splice_inclusive_out_of_bounds() { let mut v = vec![1, 2, 3, 4, 5]; let a = [10, 11, 12]; - v.splice(5..=5, a.iter().cloned()); + v.splice(5..=5, a); } #[test] @@ -848,7 +848,7 @@ fn test_splice_unbounded() { fn test_splice_forget() { let mut v = vec![1, 2, 3, 4, 5]; let a = [10, 11, 12]; - std::mem::forget(v.splice(2..4, a.iter().cloned())); + std::mem::forget(v.splice(2..4, a)); assert_eq!(v, &[1, 2]); } @@ -2234,48 +2234,50 @@ fn test_vec_dedup() { #[test] fn test_vec_dedup_panicking() { #[derive(Debug)] - struct Panic { - drop_counter: &'static AtomicU32, + struct Panic<'a> { + drop_counter: &'a Cell, value: bool, index: usize, } - impl PartialEq for Panic { + impl<'a> PartialEq for Panic<'a> { fn eq(&self, other: &Self) -> bool { self.value == other.value } } - impl Drop for Panic { + impl<'a> Drop for Panic<'a> { fn drop(&mut self) { - let x = self.drop_counter.fetch_add(1, Ordering::SeqCst); - assert!(x != 4); + self.drop_counter.set(self.drop_counter.get() + 1); + if !std::thread::panicking() { + assert!(self.index != 4); + } } } - static DROP_COUNTER: AtomicU32 = AtomicU32::new(0); + let drop_counter = &Cell::new(0); let expected = [ - Panic { drop_counter: &DROP_COUNTER, value: false, index: 0 }, - Panic { drop_counter: &DROP_COUNTER, value: false, index: 5 }, - Panic { drop_counter: &DROP_COUNTER, value: true, index: 6 }, - Panic { drop_counter: &DROP_COUNTER, value: true, index: 7 }, + Panic { drop_counter, value: false, index: 0 }, + Panic { drop_counter, value: false, index: 5 }, + Panic { drop_counter, value: true, index: 6 }, + Panic { drop_counter, value: true, index: 7 }, ]; let mut vec = vec![ - Panic { drop_counter: &DROP_COUNTER, value: false, index: 0 }, + Panic { drop_counter, value: false, index: 0 }, // these elements get deduplicated - Panic { drop_counter: &DROP_COUNTER, value: false, index: 1 }, - Panic { drop_counter: &DROP_COUNTER, value: false, index: 2 }, - Panic { drop_counter: &DROP_COUNTER, value: false, index: 3 }, - Panic { drop_counter: &DROP_COUNTER, value: false, index: 4 }, - // here it panics - Panic { drop_counter: &DROP_COUNTER, value: false, index: 5 }, - Panic { drop_counter: &DROP_COUNTER, value: true, index: 6 }, - Panic { drop_counter: &DROP_COUNTER, value: true, index: 7 }, + Panic { drop_counter, value: false, index: 1 }, + Panic { drop_counter, value: false, index: 2 }, + Panic { drop_counter, value: false, index: 3 }, + Panic { drop_counter, value: false, index: 4 }, + // here it panics while dropping the item with index==4 + Panic { drop_counter, value: false, index: 5 }, + Panic { drop_counter, value: true, index: 6 }, + Panic { drop_counter, value: true, index: 7 }, ]; - let _ = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { - vec.dedup(); - })); + let _ = catch_unwind(AssertUnwindSafe(|| vec.dedup())).unwrap_err(); + + assert_eq!(drop_counter.get(), 4); let ok = vec.iter().zip(expected.iter()).all(|(x, y)| x.index == y.index); diff --git a/library/core/src/array/mod.rs b/library/core/src/array/mod.rs index 8b56c9560a..37af3557fd 100644 --- a/library/core/src/array/mod.rs +++ b/library/core/src/array/mod.rs @@ -1,6 +1,4 @@ -//! Implementations of things like `Eq` for fixed-length arrays -//! up to a certain length. Eventually, we should be able to generalize -//! to all lengths. +//! Helper functions and types for fixed-length arrays. //! //! *[See also the array primitive type](array).* @@ -158,7 +156,6 @@ impl fmt::Debug for [T; N] { // Note: the `#[rustc_skip_array_during_method_dispatch]` on `trait IntoIterator` // hides this implementation from explicit `.into_iter()` calls on editions < 2021, // so those calls will still resolve to the slice implementation, by reference. -#[cfg(not(bootstrap))] #[stable(feature = "array_into_iter_impl", since = "1.53.0")] impl IntoIterator for [T; N] { type Item = T; @@ -419,7 +416,7 @@ impl [T; N] { { // SAFETY: we know for certain that this iterator will yield exactly `N` // items. - unsafe { collect_into_array_unchecked(&mut IntoIter::new(self).map(f)) } + unsafe { collect_into_array_unchecked(&mut IntoIterator::into_iter(self).map(f)) } } /// 'Zips up' two arrays into a single array of pairs. @@ -440,7 +437,7 @@ impl [T; N] { /// ``` #[unstable(feature = "array_zip", issue = "80094")] pub fn zip(self, rhs: [U; N]) -> [(T, U); N] { - let mut iter = IntoIter::new(self).zip(IntoIter::new(rhs)); + let mut iter = IntoIterator::into_iter(self).zip(rhs); // SAFETY: we know for certain that this iterator will yield exactly `N` // items. diff --git a/library/core/src/cell.rs b/library/core/src/cell.rs index 4820588df2..f88a6e418c 100644 --- a/library/core/src/cell.rs +++ b/library/core/src/cell.rs @@ -1123,6 +1123,15 @@ impl Clone for RefCell { fn clone(&self) -> RefCell { RefCell::new(self.borrow().clone()) } + + /// # Panics + /// + /// Panics if `other` is currently mutably borrowed. + #[inline] + #[track_caller] + fn clone_from(&mut self, other: &Self) { + self.get_mut().clone_from(&other.borrow()) + } } #[stable(feature = "rust1", since = "1.0.0")] diff --git a/library/core/src/char/methods.rs b/library/core/src/char/methods.rs index dcab2cd2d9..80d0890551 100644 --- a/library/core/src/char/methods.rs +++ b/library/core/src/char/methods.rs @@ -1,6 +1,5 @@ //! impl char {} -use crate::intrinsics::likely; use crate::slice; use crate::str::from_utf8_unchecked_mut; use crate::unicode::printable::is_printable; @@ -58,7 +57,7 @@ impl char { /// ]; /// /// assert_eq!( - /// decode_utf16(v.iter().cloned()) + /// decode_utf16(v) /// .map(|r| r.map_err(|e| e.unpaired_surrogate())) /// .collect::>(), /// vec![ @@ -82,7 +81,7 @@ impl char { /// ]; /// /// assert_eq!( - /// decode_utf16(v.iter().cloned()) + /// decode_utf16(v) /// .map(|r| r.unwrap_or(REPLACEMENT_CHARACTER)) /// .collect::(), /// "𝄞mus�ic�" @@ -332,21 +331,16 @@ impl char { #[inline] pub fn to_digit(self, radix: u32) -> Option { assert!(radix <= 36, "to_digit: radix is too high (maximum 36)"); - // the code is split up here to improve execution speed for cases where - // the `radix` is constant and 10 or smaller - let val = if likely(radix <= 10) { - // If not a digit, a number greater than radix will be created. - (self as u32).wrapping_sub('0' as u32) - } else { - match self { - '0'..='9' => self as u32 - '0' as u32, - 'a'..='z' => self as u32 - 'a' as u32 + 10, - 'A'..='Z' => self as u32 - 'A' as u32 + 10, - _ => return None, + // If not a digit, a number greater than radix will be created. + let mut digit = (self as u32).wrapping_sub('0' as u32); + if radix > 10 { + if digit < 10 { + return Some(digit); } - }; - - if val < radix { Some(val) } else { None } + // Force the 6th bit to be set to ensure ascii is lower case. + digit = (self as u32 | 0b10_0000).wrapping_sub('a' as u32).saturating_add(10); + } + (digit < radix).then_some(digit) } /// Returns an iterator that yields the hexadecimal Unicode escape of a diff --git a/library/core/src/cmp.rs b/library/core/src/cmp.rs index f8b16b6f92..ecea898504 100644 --- a/library/core/src/cmp.rs +++ b/library/core/src/cmp.rs @@ -274,7 +274,7 @@ pub trait Eq: PartialEq { // // This should never be implemented by hand. #[doc(hidden)] - #[cfg_attr(not(bootstrap), no_coverage)] // rust-lang/rust#84605 + #[no_coverage] // rust-lang/rust#84605 #[inline] #[stable(feature = "rust1", since = "1.0.0")] fn assert_receiver_is_total_eq(&self) {} @@ -578,7 +578,7 @@ impl Ordering { /// v.sort_by_key(|&num| (num > 3, Reverse(num))); /// assert_eq!(v, vec![3, 2, 1, 6, 5, 4]); /// ``` -#[derive(PartialEq, Eq, Debug, Copy, Clone, Default, Hash)] +#[derive(PartialEq, Eq, Debug, Copy, Default, Hash)] #[stable(feature = "reverse_cmp_key", since = "1.19.0")] #[repr(transparent)] pub struct Reverse(#[stable(feature = "reverse_cmp_key", since = "1.19.0")] pub T); @@ -616,6 +616,19 @@ impl Ord for Reverse { } } +#[stable(feature = "reverse_cmp_key", since = "1.19.0")] +impl Clone for Reverse { + #[inline] + fn clone(&self) -> Reverse { + Reverse(self.0.clone()) + } + + #[inline] + fn clone_from(&mut self, other: &Self) { + self.0.clone_from(&other.0) + } +} + /// Trait for types that form a [total order](https://en.wikipedia.org/wiki/Total_order). /// /// An order is a total order if it is (for all `a`, `b` and `c`): diff --git a/library/core/src/fmt/mod.rs b/library/core/src/fmt/mod.rs index 87042d95fb..02ac4fb800 100644 --- a/library/core/src/fmt/mod.rs +++ b/library/core/src/fmt/mod.rs @@ -221,6 +221,28 @@ pub struct Formatter<'a> { buf: &'a mut (dyn Write + 'a), } +impl<'a> Formatter<'a> { + /// Creates a new formatter with default settings. + /// + /// This can be used as a micro-optimization in cases where a full `Arguments` + /// structure (as created by `format_args!`) is not necessary; `Arguments` + /// is a little more expensive to use in simple formatting scenarios. + /// + /// Currently not intended for use outside of the standard library. + #[unstable(feature = "fmt_internals", reason = "internal to standard library", issue = "none")] + #[doc(hidden)] + pub fn new(buf: &'a mut (dyn Write + 'a)) -> Formatter<'a> { + Formatter { + flags: 0, + fill: ' ', + align: rt::v1::Alignment::Unknown, + width: None, + precision: None, + buf, + } + } +} + // NB. Argument is essentially an optimized partially applied formatting function, // equivalent to `exists T.(&T, fn(&T, &mut Formatter<'_>) -> Result`. @@ -1075,22 +1097,16 @@ pub trait UpperExp { /// [`write!`]: crate::write! #[stable(feature = "rust1", since = "1.0.0")] pub fn write(output: &mut dyn Write, args: Arguments<'_>) -> Result { - let mut formatter = Formatter { - flags: 0, - width: None, - precision: None, - buf: output, - align: rt::v1::Alignment::Unknown, - fill: ' ', - }; - + let mut formatter = Formatter::new(output); let mut idx = 0; match args.fmt { None => { // We can use default formatting parameters for all arguments. for (arg, piece) in iter::zip(args.args, args.pieces) { - formatter.buf.write_str(*piece)?; + if !piece.is_empty() { + formatter.buf.write_str(*piece)?; + } (arg.formatter)(arg.value, &mut formatter)?; idx += 1; } @@ -1099,7 +1115,9 @@ pub fn write(output: &mut dyn Write, args: Arguments<'_>) -> Result { // Every spec has a corresponding argument that is preceded by // a string piece. for (arg, piece) in iter::zip(fmt, args.pieces) { - formatter.buf.write_str(*piece)?; + if !piece.is_empty() { + formatter.buf.write_str(*piece)?; + } // SAFETY: arg and args.args come from the same Arguments, // which guarantees the indexes are always within bounds. unsafe { run(&mut formatter, arg, &args.args) }?; @@ -1227,12 +1245,13 @@ impl<'a> Formatter<'a> { /// // We need to remove "-" from the number output. /// let tmp = self.nb.abs().to_string(); /// - /// formatter.pad_integral(self.nb > 0, "Foo ", &tmp) + /// formatter.pad_integral(self.nb >= 0, "Foo ", &tmp) /// } /// } /// /// assert_eq!(&format!("{}", Foo::new(2)), "2"); /// assert_eq!(&format!("{}", Foo::new(-1)), "-1"); + /// assert_eq!(&format!("{}", Foo::new(0)), "0"); /// assert_eq!(&format!("{:#}", Foo::new(-1)), "-Foo 1"); /// assert_eq!(&format!("{:0>#8}", Foo::new(-1)), "00-Foo 1"); /// ``` diff --git a/library/core/src/future/future.rs b/library/core/src/future/future.rs index cc4cf54a2f..15952c6806 100644 --- a/library/core/src/future/future.rs +++ b/library/core/src/future/future.rs @@ -24,8 +24,7 @@ use crate::task::{Context, Poll}; /// `.await` the value. /// /// [`Waker`]: crate::task::Waker -#[cfg_attr(bootstrap, doc(spotlight))] -#[cfg_attr(not(bootstrap), doc(notable_trait))] +#[doc(notable_trait)] #[must_use = "futures do nothing unless you `.await` or poll them"] #[stable(feature = "futures_api", since = "1.36.0")] #[lang = "future_trait"] diff --git a/library/core/src/hint.rs b/library/core/src/hint.rs index f7aec73644..a0b65399da 100644 --- a/library/core/src/hint.rs +++ b/library/core/src/hint.rs @@ -128,7 +128,7 @@ pub fn spin_loop() { #[cfg(target_arch = "aarch64")] { // SAFETY: the `cfg` attr ensures that we only execute this on aarch64 targets. - unsafe { crate::arch::aarch64::__yield() }; + unsafe { crate::arch::aarch64::__isb(crate::arch::aarch64::SY) }; } #[cfg(target_arch = "arm")] { diff --git a/library/core/src/intrinsics.rs b/library/core/src/intrinsics.rs index 175fc2e04c..70ab27cbfa 100644 --- a/library/core/src/intrinsics.rs +++ b/library/core/src/intrinsics.rs @@ -723,7 +723,7 @@ extern "rust-intrinsic" { /// macro, which panics when it is executed, it is *undefined behavior* to /// reach code marked with this function. /// - /// The stabilized version of this intrinsic is [`core::hint::unreachable_unchecked`](crate::hint::unreachable_unchecked). + /// The stabilized version of this intrinsic is [`core::hint::unreachable_unchecked`]. #[rustc_const_unstable(feature = "const_unreachable_unchecked", issue = "53188")] pub fn unreachable() -> !; @@ -768,13 +768,13 @@ extern "rust-intrinsic" { /// More specifically, this is the offset in bytes between successive /// items of the same type, including alignment padding. /// - /// The stabilized version of this intrinsic is [`core::mem::size_of`](crate::mem::size_of). + /// The stabilized version of this intrinsic is [`core::mem::size_of`]. #[rustc_const_stable(feature = "const_size_of", since = "1.40.0")] pub fn size_of() -> usize; /// The minimum alignment of a type. /// - /// The stabilized version of this intrinsic is [`core::mem::align_of`](crate::mem::align_of). + /// The stabilized version of this intrinsic is [`core::mem::align_of`]. #[rustc_const_stable(feature = "const_min_align_of", since = "1.40.0")] pub fn min_align_of() -> usize; /// The preferred alignment of a type. @@ -790,13 +790,13 @@ extern "rust-intrinsic" { pub fn size_of_val(_: *const T) -> usize; /// The required alignment of the referenced value. /// - /// The stabilized version of this intrinsic is [`core::mem::align_of_val`](crate::mem::align_of_val). + /// The stabilized version of this intrinsic is [`core::mem::align_of_val`]. #[rustc_const_unstable(feature = "const_align_of_val", issue = "46571")] pub fn min_align_of_val(_: *const T) -> usize; /// Gets a static string slice containing the name of a type. /// - /// The stabilized version of this intrinsic is [`core::any::type_name`](crate::any::type_name). + /// The stabilized version of this intrinsic is [`core::any::type_name`]. #[rustc_const_unstable(feature = "const_type_name", issue = "63084")] pub fn type_name() -> &'static str; @@ -804,7 +804,7 @@ extern "rust-intrinsic" { /// function will return the same value for a type regardless of whichever /// crate it is invoked in. /// - /// The stabilized version of this intrinsic is [`core::any::TypeId::of`](crate::any::TypeId::of). + /// The stabilized version of this intrinsic is [`core::any::TypeId::of`]. #[rustc_const_unstable(feature = "const_type_id", issue = "77125")] pub fn type_id() -> u64; @@ -829,7 +829,7 @@ extern "rust-intrinsic" { /// Gets a reference to a static `Location` indicating where it was called. /// - /// Consider using [`core::panic::Location::caller`](crate::panic::Location::caller) instead. + /// Consider using [`core::panic::Location::caller`] instead. #[rustc_const_unstable(feature = "const_caller_location", issue = "76156")] pub fn caller_location() -> &'static crate::panic::Location<'static>; @@ -1158,11 +1158,11 @@ extern "rust-intrinsic" { /// Performs a volatile load from the `src` pointer. /// - /// The stabilized version of this intrinsic is [`core::ptr::read_volatile`](crate::ptr::read_volatile). + /// The stabilized version of this intrinsic is [`core::ptr::read_volatile`]. pub fn volatile_load(src: *const T) -> T; /// Performs a volatile store to the `dst` pointer. /// - /// The stabilized version of this intrinsic is [`core::ptr::write_volatile`](crate::ptr::write_volatile). + /// The stabilized version of this intrinsic is [`core::ptr::write_volatile`]. pub fn volatile_store(dst: *mut T, val: T); /// Performs a volatile load from the `src` pointer @@ -1703,7 +1703,7 @@ extern "rust-intrinsic" { /// Returns the value of the discriminant for the variant in 'v'; /// if `T` has no discriminant, returns `0`. /// - /// The stabilized version of this intrinsic is [`core::mem::discriminant`](crate::mem::discriminant). + /// The stabilized version of this intrinsic is [`core::mem::discriminant`]. #[rustc_const_unstable(feature = "const_discriminant", issue = "69821")] pub fn discriminant_value(v: &T) -> ::Discriminant; @@ -1785,7 +1785,7 @@ pub(crate) fn is_aligned_and_not_null(ptr: *const T) -> bool { /// [violate memory safety][read-ownership]. /// /// Note that even if the effectively copied size (`count * size_of::()`) is -/// `0`, the pointers must be non-NULL and properly aligned. +/// `0`, the pointers must be non-null and properly aligned. /// /// [`read`]: crate::ptr::read /// [read-ownership]: crate::ptr::read#ownership-of-the-returned-value @@ -1889,7 +1889,7 @@ pub const unsafe fn copy_nonoverlapping(src: *const T, dst: *mut T, count: us /// [violate memory safety][read-ownership]. /// /// Note that even if the effectively copied size (`count * size_of::()`) is -/// `0`, the pointers must be non-NULL and properly aligned. +/// `0`, the pointers must be non-null and properly aligned. /// /// [`read`]: crate::ptr::read /// [read-ownership]: crate::ptr::read#ownership-of-the-returned-value @@ -1963,7 +1963,7 @@ pub const unsafe fn copy(src: *const T, dst: *mut T, count: usize) { /// invalid value of `T` is undefined behavior. /// /// Note that even if the effectively copied size (`count * size_of::()`) is -/// `0`, the pointer must be non-NULL and properly aligned. +/// `0`, the pointer must be non-null and properly aligned. /// /// [valid]: crate::ptr#safety /// diff --git a/library/core/src/iter/adapters/chain.rs b/library/core/src/iter/adapters/chain.rs index ce5e9936bb..53e48500e3 100644 --- a/library/core/src/iter/adapters/chain.rs +++ b/library/core/src/iter/adapters/chain.rs @@ -98,7 +98,7 @@ where where Self: Sized, F: FnMut(Acc, Self::Item) -> R, - R: Try, + R: Try, { if let Some(ref mut a) = self.a { acc = a.try_fold(acc, &mut f)?; @@ -281,7 +281,7 @@ where where Self: Sized, F: FnMut(Acc, Self::Item) -> R, - R: Try, + R: Try, { if let Some(ref mut b) = self.b { acc = b.try_rfold(acc, &mut f)?; diff --git a/library/core/src/iter/adapters/cloned.rs b/library/core/src/iter/adapters/cloned.rs index 0abdbba2ef..7efc155175 100644 --- a/library/core/src/iter/adapters/cloned.rs +++ b/library/core/src/iter/adapters/cloned.rs @@ -46,7 +46,7 @@ where where Self: Sized, F: FnMut(B, Self::Item) -> R, - R: Try, + R: Try, { self.it.try_fold(init, clone_try_fold(f)) } @@ -82,7 +82,7 @@ where where Self: Sized, F: FnMut(B, Self::Item) -> R, - R: Try, + R: Try, { self.it.try_rfold(init, clone_try_fold(f)) } diff --git a/library/core/src/iter/adapters/copied.rs b/library/core/src/iter/adapters/copied.rs index 0a5822452a..def2408927 100644 --- a/library/core/src/iter/adapters/copied.rs +++ b/library/core/src/iter/adapters/copied.rs @@ -50,7 +50,7 @@ where where Self: Sized, F: FnMut(B, Self::Item) -> R, - R: Try, + R: Try, { self.it.try_fold(init, copy_try_fold(f)) } @@ -98,7 +98,7 @@ where where Self: Sized, F: FnMut(B, Self::Item) -> R, - R: Try, + R: Try, { self.it.try_rfold(init, copy_try_fold(f)) } diff --git a/library/core/src/iter/adapters/cycle.rs b/library/core/src/iter/adapters/cycle.rs index 6e9a011f81..815e708f9e 100644 --- a/library/core/src/iter/adapters/cycle.rs +++ b/library/core/src/iter/adapters/cycle.rs @@ -53,7 +53,7 @@ where fn try_fold(&mut self, mut acc: Acc, mut f: F) -> R where F: FnMut(Acc, Self::Item) -> R, - R: Try, + R: Try, { // fully iterate the current iterator. this is necessary because // `self.iter` may be empty even when `self.orig` isn't diff --git a/library/core/src/iter/adapters/enumerate.rs b/library/core/src/iter/adapters/enumerate.rs index 73cee1df30..91722a4b62 100644 --- a/library/core/src/iter/adapters/enumerate.rs +++ b/library/core/src/iter/adapters/enumerate.rs @@ -71,7 +71,7 @@ where where Self: Sized, Fold: FnMut(Acc, Self::Item) -> R, - R: Try, + R: Try, { #[inline] fn enumerate<'a, T, Acc, R>( @@ -150,7 +150,7 @@ where where Self: Sized, Fold: FnMut(Acc, Self::Item) -> R, - R: Try, + R: Try, { // Can safely add and subtract the count, as `ExactSizeIterator` promises // that the number of elements fits into a `usize`. diff --git a/library/core/src/iter/adapters/filter.rs b/library/core/src/iter/adapters/filter.rs index 0337892b9e..d5f19f1274 100644 --- a/library/core/src/iter/adapters/filter.rs +++ b/library/core/src/iter/adapters/filter.rs @@ -37,7 +37,7 @@ fn filter_fold( move |acc, item| if predicate(&item) { fold(acc, item) } else { acc } } -fn filter_try_fold<'a, T, Acc, R: Try>( +fn filter_try_fold<'a, T, Acc, R: Try>( predicate: &'a mut impl FnMut(&T) -> bool, mut fold: impl FnMut(Acc, T) -> R + 'a, ) -> impl FnMut(Acc, T) -> R + 'a { @@ -88,7 +88,7 @@ where where Self: Sized, Fold: FnMut(Acc, Self::Item) -> R, - R: Try, + R: Try, { self.iter.try_fold(init, filter_try_fold(&mut self.predicate, fold)) } @@ -117,7 +117,7 @@ where where Self: Sized, Fold: FnMut(Acc, Self::Item) -> R, - R: Try, + R: Try, { self.iter.try_rfold(init, filter_try_fold(&mut self.predicate, fold)) } diff --git a/library/core/src/iter/adapters/filter_map.rs b/library/core/src/iter/adapters/filter_map.rs index 0dccf2c533..01b7be9d52 100644 --- a/library/core/src/iter/adapters/filter_map.rs +++ b/library/core/src/iter/adapters/filter_map.rs @@ -39,7 +39,7 @@ fn filter_map_fold( } } -fn filter_map_try_fold<'a, T, B, Acc, R: Try>( +fn filter_map_try_fold<'a, T, B, Acc, R: Try>( f: &'a mut impl FnMut(T) -> Option, mut fold: impl FnMut(Acc, B) -> R + 'a, ) -> impl FnMut(Acc, T) -> R + 'a { @@ -72,7 +72,7 @@ where where Self: Sized, Fold: FnMut(Acc, Self::Item) -> R, - R: Try, + R: Try, { self.iter.try_fold(init, filter_map_try_fold(&mut self.f, fold)) } @@ -111,7 +111,7 @@ where where Self: Sized, Fold: FnMut(Acc, Self::Item) -> R, - R: Try, + R: Try, { self.iter.try_rfold(init, filter_map_try_fold(&mut self.f, fold)) } diff --git a/library/core/src/iter/adapters/flatten.rs b/library/core/src/iter/adapters/flatten.rs index 0114d7af4f..3315d34659 100644 --- a/library/core/src/iter/adapters/flatten.rs +++ b/library/core/src/iter/adapters/flatten.rs @@ -61,7 +61,7 @@ where where Self: Sized, Fold: FnMut(Acc, Self::Item) -> R, - R: Try, + R: Try, { self.inner.try_fold(init, fold) } @@ -91,7 +91,7 @@ where where Self: Sized, Fold: FnMut(Acc, Self::Item) -> R, - R: Try, + R: Try, { self.inner.try_rfold(init, fold) } @@ -178,7 +178,7 @@ where where Self: Sized, Fold: FnMut(Acc, Self::Item) -> R, - R: Try, + R: Try, { self.inner.try_fold(init, fold) } @@ -208,7 +208,7 @@ where where Self: Sized, Fold: FnMut(Acc, Self::Item) -> R, - R: Try, + R: Try, { self.inner.try_rfold(init, fold) } @@ -293,10 +293,10 @@ where where Self: Sized, Fold: FnMut(Acc, Self::Item) -> R, - R: Try, + R: Try, { #[inline] - fn flatten<'a, T: IntoIterator, Acc, R: Try>( + fn flatten<'a, T: IntoIterator, Acc, R: Try>( frontiter: &'a mut Option, fold: &'a mut impl FnMut(Acc, T::Item) -> R, ) -> impl FnMut(Acc, T) -> R + 'a { @@ -382,10 +382,10 @@ where where Self: Sized, Fold: FnMut(Acc, Self::Item) -> R, - R: Try, + R: Try, { #[inline] - fn flatten<'a, T: IntoIterator, Acc, R: Try>( + fn flatten<'a, T: IntoIterator, Acc, R: Try>( backiter: &'a mut Option, fold: &'a mut impl FnMut(Acc, T::Item) -> R, ) -> impl FnMut(Acc, T) -> R + 'a diff --git a/library/core/src/iter/adapters/fuse.rs b/library/core/src/iter/adapters/fuse.rs index c01f384dec..aff48b1b22 100644 --- a/library/core/src/iter/adapters/fuse.rs +++ b/library/core/src/iter/adapters/fuse.rs @@ -92,7 +92,7 @@ where where Self: Sized, Fold: FnMut(Acc, Self::Item) -> R, - R: Try, + R: Try, { FuseImpl::try_fold(self, acc, fold) } @@ -148,7 +148,7 @@ where where Self: Sized, Fold: FnMut(Acc, Self::Item) -> R, - R: Try, + R: Try, { FuseImpl::try_rfold(self, acc, fold) } @@ -219,7 +219,7 @@ trait FuseImpl { where Self: Sized, Fold: FnMut(Acc, Self::Item) -> R, - R: Try; + R: Try; fn fold(self, acc: Acc, fold: Fold) -> Acc where Fold: FnMut(Acc, Self::Item) -> Acc; @@ -238,7 +238,7 @@ trait FuseImpl { where Self: Sized, Fold: FnMut(Acc, Self::Item) -> R, - R: Try, + R: Try, I: DoubleEndedIterator; fn rfold(self, acc: Acc, fold: Fold) -> Acc where @@ -305,7 +305,7 @@ where where Self: Sized, Fold: FnMut(Acc, Self::Item) -> R, - R: Try, + R: Try, { if let Some(ref mut iter) = self.iter { acc = iter.try_fold(acc, fold)?; @@ -354,7 +354,7 @@ where where Self: Sized, Fold: FnMut(Acc, Self::Item) -> R, - R: Try, + R: Try, I: DoubleEndedIterator, { if let Some(ref mut iter) = self.iter { @@ -443,7 +443,7 @@ where where Self: Sized, Fold: FnMut(Acc, Self::Item) -> R, - R: Try, + R: Try, { unchecked!(self).try_fold(init, fold) } @@ -485,7 +485,7 @@ where where Self: Sized, Fold: FnMut(Acc, Self::Item) -> R, - R: Try, + R: Try, I: DoubleEndedIterator, { unchecked!(self).try_rfold(init, fold) diff --git a/library/core/src/iter/adapters/inspect.rs b/library/core/src/iter/adapters/inspect.rs index 88f5ee61b6..36835d12e5 100644 --- a/library/core/src/iter/adapters/inspect.rs +++ b/library/core/src/iter/adapters/inspect.rs @@ -87,7 +87,7 @@ where where Self: Sized, Fold: FnMut(Acc, Self::Item) -> R, - R: Try, + R: Try, { self.iter.try_fold(init, inspect_try_fold(&mut self.f, fold)) } @@ -117,7 +117,7 @@ where where Self: Sized, Fold: FnMut(Acc, Self::Item) -> R, - R: Try, + R: Try, { self.iter.try_rfold(init, inspect_try_fold(&mut self.f, fold)) } diff --git a/library/core/src/iter/adapters/map.rs b/library/core/src/iter/adapters/map.rs index 2a4b7efd5e..0bf9f4b032 100644 --- a/library/core/src/iter/adapters/map.rs +++ b/library/core/src/iter/adapters/map.rs @@ -110,7 +110,7 @@ where where Self: Sized, G: FnMut(Acc, Self::Item) -> R, - R: Try, + R: Try, { self.iter.try_fold(init, map_try_fold(&mut self.f, g)) } @@ -146,7 +146,7 @@ where where Self: Sized, G: FnMut(Acc, Self::Item) -> R, - R: Try, + R: Try, { self.iter.try_rfold(init, map_try_fold(&mut self.f, g)) } diff --git a/library/core/src/iter/adapters/map_while.rs b/library/core/src/iter/adapters/map_while.rs index 26114d5328..8f89e15880 100644 --- a/library/core/src/iter/adapters/map_while.rs +++ b/library/core/src/iter/adapters/map_while.rs @@ -54,7 +54,7 @@ where where Self: Sized, Fold: FnMut(Acc, Self::Item) -> R, - R: Try, + R: Try, { let Self { iter, predicate } = self; iter.try_fold(init, |acc, x| match predicate(x) { diff --git a/library/core/src/iter/adapters/mod.rs b/library/core/src/iter/adapters/mod.rs index 61d8351d59..9fdd4fca04 100644 --- a/library/core/src/iter/adapters/mod.rs +++ b/library/core/src/iter/adapters/mod.rs @@ -88,6 +88,7 @@ pub use self::zip::zip; /// [`FromIterator`]: crate::iter::FromIterator /// [`as_inner`]: SourceIter::as_inner #[unstable(issue = "none", feature = "inplace_iteration")] +#[doc(hidden)] pub unsafe trait SourceIter { /// A source stage in an iterator pipeline. type Source: Iterator; @@ -167,7 +168,7 @@ where fn try_fold(&mut self, init: B, mut f: F) -> R where F: FnMut(B, Self::Item) -> R, - R: Try, + R: Try, { let error = &mut *self.error; self.iter diff --git a/library/core/src/iter/adapters/peekable.rs b/library/core/src/iter/adapters/peekable.rs index 0d860aec49..69bd2996ef 100644 --- a/library/core/src/iter/adapters/peekable.rs +++ b/library/core/src/iter/adapters/peekable.rs @@ -1,5 +1,5 @@ use crate::iter::{adapters::SourceIter, FusedIterator, TrustedLen}; -use crate::ops::Try; +use crate::ops::{ControlFlow, Try}; /// An iterator with a `peek()` that returns an optional reference to the next /// element. @@ -91,7 +91,7 @@ impl Iterator for Peekable { where Self: Sized, F: FnMut(B, Self::Item) -> R, - R: Try, + R: Try, { let acc = match self.peeked.take() { Some(None) => return try { init }, @@ -130,19 +130,42 @@ where } #[inline] + #[cfg(not(bootstrap))] fn try_rfold(&mut self, init: B, mut f: F) -> R where Self: Sized, F: FnMut(B, Self::Item) -> R, - R: Try, + R: Try, { + match self.peeked.take() { + Some(None) => try { init }, + Some(Some(v)) => match self.iter.try_rfold(init, &mut f).branch() { + ControlFlow::Continue(acc) => f(acc, v), + ControlFlow::Break(r) => { + self.peeked = Some(Some(v)); + R::from_residual(r) + } + }, + None => self.iter.try_rfold(init, f), + } + } + + #[inline] + #[cfg(bootstrap)] + fn try_rfold(&mut self, init: B, mut f: F) -> R + where + Self: Sized, + F: FnMut(B, Self::Item) -> R, + R: Try, + { + let _use_the_import: ControlFlow<()>; match self.peeked.take() { Some(None) => try { init }, Some(Some(v)) => match self.iter.try_rfold(init, &mut f).into_result() { Ok(acc) => f(acc, v), Err(e) => { self.peeked = Some(Some(v)); - Try::from_error(e) + R::from_error(e) } }, None => self.iter.try_rfold(init, f), diff --git a/library/core/src/iter/adapters/rev.rs b/library/core/src/iter/adapters/rev.rs index 105ed40a3e..139fb7bbdd 100644 --- a/library/core/src/iter/adapters/rev.rs +++ b/library/core/src/iter/adapters/rev.rs @@ -51,7 +51,7 @@ where where Self: Sized, F: FnMut(B, Self::Item) -> R, - R: Try, + R: Try, { self.iter.try_rfold(init, f) } @@ -96,7 +96,7 @@ where where Self: Sized, F: FnMut(B, Self::Item) -> R, - R: Try, + R: Try, { self.iter.try_fold(init, f) } diff --git a/library/core/src/iter/adapters/scan.rs b/library/core/src/iter/adapters/scan.rs index 0214899295..96705b01f6 100644 --- a/library/core/src/iter/adapters/scan.rs +++ b/library/core/src/iter/adapters/scan.rs @@ -56,9 +56,9 @@ where where Self: Sized, Fold: FnMut(Acc, Self::Item) -> R, - R: Try, + R: Try, { - fn scan<'a, T, St, B, Acc, R: Try>( + fn scan<'a, T, St, B, Acc, R: Try>( state: &'a mut St, f: &'a mut impl FnMut(&mut St, T) -> Option, mut fold: impl FnMut(Acc, B) -> R + 'a, diff --git a/library/core/src/iter/adapters/skip.rs b/library/core/src/iter/adapters/skip.rs index e55c7a6bf5..c358a6d12b 100644 --- a/library/core/src/iter/adapters/skip.rs +++ b/library/core/src/iter/adapters/skip.rs @@ -88,7 +88,7 @@ where where Self: Sized, Fold: FnMut(Acc, Self::Item) -> R, - R: Try, + R: Try, { let n = self.n; self.n = 0; @@ -146,9 +146,9 @@ where where Self: Sized, Fold: FnMut(Acc, Self::Item) -> R, - R: Try, + R: Try, { - fn check>( + fn check>( mut n: usize, mut fold: impl FnMut(Acc, T) -> R, ) -> impl FnMut(Acc, T) -> ControlFlow { diff --git a/library/core/src/iter/adapters/skip_while.rs b/library/core/src/iter/adapters/skip_while.rs index efcb469fc9..93e29edc8d 100644 --- a/library/core/src/iter/adapters/skip_while.rs +++ b/library/core/src/iter/adapters/skip_while.rs @@ -70,7 +70,7 @@ where where Self: Sized, Fold: FnMut(Acc, Self::Item) -> R, - R: Try, + R: Try, { if !self.flag { match self.next() { diff --git a/library/core/src/iter/adapters/step_by.rs b/library/core/src/iter/adapters/step_by.rs index 2ba56eeccb..4252c34a0e 100644 --- a/library/core/src/iter/adapters/step_by.rs +++ b/library/core/src/iter/adapters/step_by.rs @@ -111,7 +111,7 @@ where fn try_fold(&mut self, mut acc: Acc, mut f: F) -> R where F: FnMut(Acc, Self::Item) -> R, - R: Try, + R: Try, { #[inline] fn nth(iter: &mut I, step: usize) -> impl FnMut() -> Option + '_ { @@ -187,7 +187,7 @@ where fn try_rfold(&mut self, init: Acc, mut f: F) -> R where F: FnMut(Acc, Self::Item) -> R, - R: Try, + R: Try, { #[inline] fn nth_back( diff --git a/library/core/src/iter/adapters/take.rs b/library/core/src/iter/adapters/take.rs index 9efc7a480a..beda8c32c6 100644 --- a/library/core/src/iter/adapters/take.rs +++ b/library/core/src/iter/adapters/take.rs @@ -77,9 +77,9 @@ where where Self: Sized, Fold: FnMut(Acc, Self::Item) -> R, - R: Try, + R: Try, { - fn check<'a, T, Acc, R: Try>( + fn check<'a, T, Acc, R: Try>( n: &'a mut usize, mut fold: impl FnMut(Acc, T) -> R + 'a, ) -> impl FnMut(Acc, T) -> ControlFlow + 'a { @@ -166,7 +166,7 @@ where where Self: Sized, Fold: FnMut(Acc, Self::Item) -> R, - R: Try, + R: Try, { if self.n == 0 { try { init } diff --git a/library/core/src/iter/adapters/take_while.rs b/library/core/src/iter/adapters/take_while.rs index 746eb41f4c..93457d20f7 100644 --- a/library/core/src/iter/adapters/take_while.rs +++ b/library/core/src/iter/adapters/take_while.rs @@ -68,9 +68,9 @@ where where Self: Sized, Fold: FnMut(Acc, Self::Item) -> R, - R: Try, + R: Try, { - fn check<'a, T, Acc, R: Try>( + fn check<'a, T, Acc, R: Try>( flag: &'a mut bool, p: &'a mut impl FnMut(&T) -> bool, mut fold: impl FnMut(Acc, T) -> R + 'a, diff --git a/library/core/src/iter/adapters/zip.rs b/library/core/src/iter/adapters/zip.rs index 2f8f504d8f..c95324c80b 100644 --- a/library/core/src/iter/adapters/zip.rs +++ b/library/core/src/iter/adapters/zip.rs @@ -434,7 +434,7 @@ impl ZipFmt {$( + #[unstable(feature = "trusted_step", issue = "85731")] + unsafe impl TrustedStep for $type {} + )*}; +} +unsafe_impl_trusted_step![char i8 i16 i32 i64 i128 isize u8 u16 u32 u64 u128 usize]; /// Objects that have a notion of *successor* and *predecessor* operations. /// /// The *successor* operation moves towards values that compare greater. /// The *predecessor* operation moves towards values that compare lesser. -/// -/// # Safety -/// -/// This trait is `unsafe` because its implementation must be correct for -/// the safety of `unsafe trait TrustedLen` implementations, and the results -/// of using this trait can otherwise be trusted by `unsafe` code to be correct -/// and fulfill the listed obligations. #[unstable(feature = "step_trait", reason = "recently redesigned", issue = "42168")] -pub unsafe trait Step: Clone + PartialOrd + Sized { +pub trait Step: Clone + PartialOrd + Sized { /// Returns the number of *successor* steps required to get from `start` to `end`. /// /// Returns `None` if the number of steps would overflow `usize` @@ -55,7 +57,6 @@ pub unsafe trait Step: Clone + PartialOrd + Sized { /// /// * `Step::forward_checked(a, n) == (0..n).try_fold(a, |x, _| Step::forward_checked(&x, 1))` /// * Corollary: `Step::forward_checked(&a, 0) == Some(a)` - #[unstable(feature = "step_trait_ext", reason = "recently added", issue = "42168")] fn forward_checked(start: Self, count: usize) -> Option; /// Returns the value that would be obtained by taking the *successor* @@ -81,7 +82,6 @@ pub unsafe trait Step: Clone + PartialOrd + Sized { /// * Corollary: `Step::forward(a, 0) == a` /// * `Step::forward(a, n) >= a` /// * `Step::backward(Step::forward(a, n), n) == a` - #[unstable(feature = "step_trait_ext", reason = "recently added", issue = "42168")] fn forward(start: Self, count: usize) -> Self { Step::forward_checked(start, count).expect("overflow in `Step::forward`") } @@ -106,7 +106,6 @@ pub unsafe trait Step: Clone + PartialOrd + Sized { /// For any `a` and `n`, where no overflow occurs: /// /// * `Step::forward_unchecked(a, n)` is equivalent to `Step::forward(a, n)` - #[unstable(feature = "unchecked_math", reason = "niche optimization path", issue = "none")] unsafe fn forward_unchecked(start: Self, count: usize) -> Self { Step::forward(start, count) } @@ -127,7 +126,6 @@ pub unsafe trait Step: Clone + PartialOrd + Sized { /// /// * `Step::backward_checked(a, n) == (0..n).try_fold(a, |x, _| Step::backward_checked(&x, 1))` /// * Corollary: `Step::backward_checked(&a, 0) == Some(a)` - #[unstable(feature = "step_trait_ext", reason = "recently added", issue = "42168")] fn backward_checked(start: Self, count: usize) -> Option; /// Returns the value that would be obtained by taking the *predecessor* @@ -153,7 +151,6 @@ pub unsafe trait Step: Clone + PartialOrd + Sized { /// * Corollary: `Step::backward(a, 0) == a` /// * `Step::backward(a, n) <= a` /// * `Step::forward(Step::backward(a, n), n) == a` - #[unstable(feature = "step_trait_ext", reason = "recently added", issue = "42168")] fn backward(start: Self, count: usize) -> Self { Step::backward_checked(start, count).expect("overflow in `Step::backward`") } @@ -178,7 +175,6 @@ pub unsafe trait Step: Clone + PartialOrd + Sized { /// For any `a` and `n`, where no overflow occurs: /// /// * `Step::backward_unchecked(a, n)` is equivalent to `Step::backward(a, n)` - #[unstable(feature = "unchecked_math", reason = "niche optimization path", issue = "none")] unsafe fn backward_unchecked(start: Self, count: usize) -> Self { Step::backward(start, count) } @@ -237,7 +233,7 @@ macro_rules! step_integer_impls { $( #[allow(unreachable_patterns)] #[unstable(feature = "step_trait", reason = "recently redesigned", issue = "42168")] - unsafe impl Step for $u_narrower { + impl Step for $u_narrower { step_identical_methods!(); #[inline] @@ -269,7 +265,7 @@ macro_rules! step_integer_impls { #[allow(unreachable_patterns)] #[unstable(feature = "step_trait", reason = "recently redesigned", issue = "42168")] - unsafe impl Step for $i_narrower { + impl Step for $i_narrower { step_identical_methods!(); #[inline] @@ -333,7 +329,7 @@ macro_rules! step_integer_impls { $( #[allow(unreachable_patterns)] #[unstable(feature = "step_trait", reason = "recently redesigned", issue = "42168")] - unsafe impl Step for $u_wider { + impl Step for $u_wider { step_identical_methods!(); #[inline] @@ -358,7 +354,7 @@ macro_rules! step_integer_impls { #[allow(unreachable_patterns)] #[unstable(feature = "step_trait", reason = "recently redesigned", issue = "42168")] - unsafe impl Step for $i_wider { + impl Step for $i_wider { step_identical_methods!(); #[inline] @@ -408,7 +404,7 @@ step_integer_impls! { } #[unstable(feature = "step_trait", reason = "recently redesigned", issue = "42168")] -unsafe impl Step for char { +impl Step for char { #[inline] fn steps_between(&start: &char, &end: &char) -> Option { let start = start as u32; @@ -512,15 +508,27 @@ macro_rules! range_incl_exact_iter_impl { )*) } -#[stable(feature = "rust1", since = "1.0.0")] -impl Iterator for ops::Range { +/// Specialization implementations for `Range`. +trait RangeIteratorImpl { + type Item; + + // Iterator + fn spec_next(&mut self) -> Option; + fn spec_nth(&mut self, n: usize) -> Option; + + // DoubleEndedIterator + fn spec_next_back(&mut self) -> Option; + fn spec_nth_back(&mut self, n: usize) -> Option; +} + +impl RangeIteratorImpl for ops::Range { type Item = A; #[inline] - fn next(&mut self) -> Option { + default fn spec_next(&mut self) -> Option { if self.start < self.end { - // SAFETY: just checked precondition - let n = unsafe { Step::forward_unchecked(self.start.clone(), 1) }; + let n = + Step::forward_checked(self.start.clone(), 1).expect("`Step` invariants not upheld"); Some(mem::replace(&mut self.start, n)) } else { None @@ -528,17 +536,59 @@ impl Iterator for ops::Range { } #[inline] - fn size_hint(&self) -> (usize, Option) { + default fn spec_nth(&mut self, n: usize) -> Option { + if let Some(plus_n) = Step::forward_checked(self.start.clone(), n) { + if plus_n < self.end { + self.start = + Step::forward_checked(plus_n.clone(), 1).expect("`Step` invariants not upheld"); + return Some(plus_n); + } + } + + self.start = self.end.clone(); + None + } + + #[inline] + default fn spec_next_back(&mut self) -> Option { if self.start < self.end { - let hint = Step::steps_between(&self.start, &self.end); - (hint.unwrap_or(usize::MAX), hint) + self.end = + Step::backward_checked(self.end.clone(), 1).expect("`Step` invariants not upheld"); + Some(self.end.clone()) } else { - (0, Some(0)) + None } } #[inline] - fn nth(&mut self, n: usize) -> Option { + default fn spec_nth_back(&mut self, n: usize) -> Option { + if let Some(minus_n) = Step::backward_checked(self.end.clone(), n) { + if minus_n > self.start { + self.end = + Step::backward_checked(minus_n, 1).expect("`Step` invariants not upheld"); + return Some(self.end.clone()); + } + } + + self.end = self.start.clone(); + None + } +} + +impl RangeIteratorImpl for ops::Range { + #[inline] + fn spec_next(&mut self) -> Option { + if self.start < self.end { + // SAFETY: just checked precondition + let n = unsafe { Step::forward_unchecked(self.start.clone(), 1) }; + Some(mem::replace(&mut self.start, n)) + } else { + None + } + } + + #[inline] + fn spec_nth(&mut self, n: usize) -> Option { if let Some(plus_n) = Step::forward_checked(self.start.clone(), n) { if plus_n < self.end { // SAFETY: just checked precondition @@ -551,6 +601,56 @@ impl Iterator for ops::Range { None } + #[inline] + fn spec_next_back(&mut self) -> Option { + if self.start < self.end { + // SAFETY: just checked precondition + self.end = unsafe { Step::backward_unchecked(self.end.clone(), 1) }; + Some(self.end.clone()) + } else { + None + } + } + + #[inline] + fn spec_nth_back(&mut self, n: usize) -> Option { + if let Some(minus_n) = Step::backward_checked(self.end.clone(), n) { + if minus_n > self.start { + // SAFETY: just checked precondition + self.end = unsafe { Step::backward_unchecked(minus_n, 1) }; + return Some(self.end.clone()); + } + } + + self.end = self.start.clone(); + None + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl Iterator for ops::Range { + type Item = A; + + #[inline] + fn next(&mut self) -> Option { + self.spec_next() + } + + #[inline] + fn size_hint(&self) -> (usize, Option) { + if self.start < self.end { + let hint = Step::steps_between(&self.start, &self.end); + (hint.unwrap_or(usize::MAX), hint) + } else { + (0, Some(0)) + } + } + + #[inline] + fn nth(&mut self, n: usize) -> Option { + self.spec_nth(n) + } + #[inline] fn last(mut self) -> Option { self.next_back() @@ -631,32 +731,36 @@ range_incl_exact_iter_impl! { impl DoubleEndedIterator for ops::Range { #[inline] fn next_back(&mut self) -> Option { - if self.start < self.end { - // SAFETY: just checked precondition - self.end = unsafe { Step::backward_unchecked(self.end.clone(), 1) }; - Some(self.end.clone()) - } else { - None - } + self.spec_next_back() } #[inline] fn nth_back(&mut self, n: usize) -> Option { - if let Some(minus_n) = Step::backward_checked(self.end.clone(), n) { - if minus_n > self.start { - // SAFETY: just checked precondition - self.end = unsafe { Step::backward_unchecked(minus_n, 1) }; - return Some(self.end.clone()); - } - } - - self.end = self.start.clone(); - None + self.spec_nth_back(n) } } +// Safety: +// The following invariants for `Step::steps_between` exist: +// +// > * `steps_between(&a, &b) == Some(n)` only if `a <= b` +// > * Note that `a <= b` does _not_ imply `steps_between(&a, &b) != None`; +// > this is the case when it would require more than `usize::MAX` steps to +// > get to `b` +// > * `steps_between(&a, &b) == None` if `a > b` +// +// The first invariant is what is generally required for `TrustedLen` to be +// sound. The note addendum satisfies an additional `TrustedLen` invariant. +// +// > The upper bound must only be `None` if the actual iterator length is larger +// > than `usize::MAX` +// +// The second invariant logically follows the first so long as the `PartialOrd` +// implementation is correct; regardless it is explicitly stated. If `a < b` +// then `(0, Some(0))` is returned by `ops::Range::size_hint`. As such +// the second invariant is upheld. #[unstable(feature = "trusted_len", issue = "37572")] -unsafe impl TrustedLen for ops::Range {} +unsafe impl TrustedLen for ops::Range {} #[stable(feature = "fused", since = "1.26.0")] impl FusedIterator for ops::Range {} @@ -684,18 +788,130 @@ impl Iterator for ops::RangeFrom { } } +// Safety: See above implementation for `ops::Range` +#[unstable(feature = "trusted_len", issue = "37572")] +unsafe impl TrustedLen for ops::RangeFrom {} + #[stable(feature = "fused", since = "1.26.0")] impl FusedIterator for ops::RangeFrom {} -#[unstable(feature = "trusted_len", issue = "37572")] -unsafe impl TrustedLen for ops::RangeFrom {} +trait RangeInclusiveIteratorImpl { + type Item; -#[stable(feature = "inclusive_range", since = "1.26.0")] -impl Iterator for ops::RangeInclusive { + // Iterator + fn spec_next(&mut self) -> Option; + fn spec_try_fold(&mut self, init: B, f: F) -> R + where + Self: Sized, + F: FnMut(B, Self::Item) -> R, + R: Try; + + // DoubleEndedIterator + fn spec_next_back(&mut self) -> Option; + fn spec_try_rfold(&mut self, init: B, f: F) -> R + where + Self: Sized, + F: FnMut(B, Self::Item) -> R, + R: Try; +} + +impl RangeInclusiveIteratorImpl for ops::RangeInclusive { type Item = A; #[inline] - fn next(&mut self) -> Option { + default fn spec_next(&mut self) -> Option { + if self.is_empty() { + return None; + } + let is_iterating = self.start < self.end; + Some(if is_iterating { + let n = + Step::forward_checked(self.start.clone(), 1).expect("`Step` invariants not upheld"); + mem::replace(&mut self.start, n) + } else { + self.exhausted = true; + self.start.clone() + }) + } + + #[inline] + default fn spec_try_fold(&mut self, init: B, mut f: F) -> R + where + Self: Sized, + F: FnMut(B, A) -> R, + R: Try, + { + if self.is_empty() { + return try { init }; + } + + let mut accum = init; + + while self.start < self.end { + let n = + Step::forward_checked(self.start.clone(), 1).expect("`Step` invariants not upheld"); + let n = mem::replace(&mut self.start, n); + accum = f(accum, n)?; + } + + self.exhausted = true; + + if self.start == self.end { + accum = f(accum, self.start.clone())?; + } + + try { accum } + } + + #[inline] + default fn spec_next_back(&mut self) -> Option { + if self.is_empty() { + return None; + } + let is_iterating = self.start < self.end; + Some(if is_iterating { + let n = + Step::backward_checked(self.end.clone(), 1).expect("`Step` invariants not upheld"); + mem::replace(&mut self.end, n) + } else { + self.exhausted = true; + self.end.clone() + }) + } + + #[inline] + default fn spec_try_rfold(&mut self, init: B, mut f: F) -> R + where + Self: Sized, + F: FnMut(B, A) -> R, + R: Try, + { + if self.is_empty() { + return try { init }; + } + + let mut accum = init; + + while self.start < self.end { + let n = + Step::backward_checked(self.end.clone(), 1).expect("`Step` invariants not upheld"); + let n = mem::replace(&mut self.end, n); + accum = f(accum, n)?; + } + + self.exhausted = true; + + if self.start == self.end { + accum = f(accum, self.start.clone())?; + } + + try { accum } + } +} + +impl RangeInclusiveIteratorImpl for ops::RangeInclusive { + #[inline] + fn spec_next(&mut self) -> Option { if self.is_empty() { return None; } @@ -710,6 +926,90 @@ impl Iterator for ops::RangeInclusive { }) } + #[inline] + fn spec_try_fold(&mut self, init: B, mut f: F) -> R + where + Self: Sized, + F: FnMut(B, T) -> R, + R: Try, + { + if self.is_empty() { + return try { init }; + } + + let mut accum = init; + + while self.start < self.end { + // SAFETY: just checked precondition + let n = unsafe { Step::forward_unchecked(self.start.clone(), 1) }; + let n = mem::replace(&mut self.start, n); + accum = f(accum, n)?; + } + + self.exhausted = true; + + if self.start == self.end { + accum = f(accum, self.start.clone())?; + } + + try { accum } + } + + #[inline] + fn spec_next_back(&mut self) -> Option { + if self.is_empty() { + return None; + } + let is_iterating = self.start < self.end; + Some(if is_iterating { + // SAFETY: just checked precondition + let n = unsafe { Step::backward_unchecked(self.end.clone(), 1) }; + mem::replace(&mut self.end, n) + } else { + self.exhausted = true; + self.end.clone() + }) + } + + #[inline] + fn spec_try_rfold(&mut self, init: B, mut f: F) -> R + where + Self: Sized, + F: FnMut(B, T) -> R, + R: Try, + { + if self.is_empty() { + return try { init }; + } + + let mut accum = init; + + while self.start < self.end { + // SAFETY: just checked precondition + let n = unsafe { Step::backward_unchecked(self.end.clone(), 1) }; + let n = mem::replace(&mut self.end, n); + accum = f(accum, n)?; + } + + self.exhausted = true; + + if self.start == self.end { + accum = f(accum, self.start.clone())?; + } + + try { accum } + } +} + +#[stable(feature = "inclusive_range", since = "1.26.0")] +impl Iterator for ops::RangeInclusive { + type Item = A; + + #[inline] + fn next(&mut self) -> Option { + self.spec_next() + } + #[inline] fn size_hint(&self) -> (usize, Option) { if self.is_empty() { @@ -751,32 +1051,13 @@ impl Iterator for ops::RangeInclusive { } #[inline] - fn try_fold(&mut self, init: B, mut f: F) -> R + fn try_fold(&mut self, init: B, f: F) -> R where Self: Sized, F: FnMut(B, Self::Item) -> R, - R: Try, + R: Try, { - if self.is_empty() { - return try { init }; - } - - let mut accum = init; - - while self.start < self.end { - // SAFETY: just checked precondition - let n = unsafe { Step::forward_unchecked(self.start.clone(), 1) }; - let n = mem::replace(&mut self.start, n); - accum = f(accum, n)?; - } - - self.exhausted = true; - - if self.start == self.end { - accum = f(accum, self.start.clone())?; - } - - try { accum } + self.spec_try_fold(init, f) } #[inline] @@ -813,18 +1094,7 @@ impl Iterator for ops::RangeInclusive { impl DoubleEndedIterator for ops::RangeInclusive { #[inline] fn next_back(&mut self) -> Option { - if self.is_empty() { - return None; - } - let is_iterating = self.start < self.end; - Some(if is_iterating { - // SAFETY: just checked precondition - let n = unsafe { Step::backward_unchecked(self.end.clone(), 1) }; - mem::replace(&mut self.end, n) - } else { - self.exhausted = true; - self.end.clone() - }) + self.spec_next_back() } #[inline] @@ -856,32 +1126,13 @@ impl DoubleEndedIterator for ops::RangeInclusive { } #[inline] - fn try_rfold(&mut self, init: B, mut f: F) -> R + fn try_rfold(&mut self, init: B, f: F) -> R where Self: Sized, F: FnMut(B, Self::Item) -> R, - R: Try, + R: Try, { - if self.is_empty() { - return try { init }; - } - - let mut accum = init; - - while self.start < self.end { - // SAFETY: just checked precondition - let n = unsafe { Step::backward_unchecked(self.end.clone(), 1) }; - let n = mem::replace(&mut self.end, n); - accum = f(accum, n)?; - } - - self.exhausted = true; - - if self.start == self.end { - accum = f(accum, self.start.clone())?; - } - - try { accum } + self.spec_try_rfold(init, f) } #[inline] @@ -899,8 +1150,9 @@ impl DoubleEndedIterator for ops::RangeInclusive { } } +// Safety: See above implementation for `ops::Range` #[unstable(feature = "trusted_len", issue = "37572")] -unsafe impl TrustedLen for ops::RangeInclusive {} +unsafe impl TrustedLen for ops::RangeInclusive {} #[stable(feature = "fused", since = "1.26.0")] impl FusedIterator for ops::RangeInclusive {} diff --git a/library/core/src/iter/sources/repeat.rs b/library/core/src/iter/sources/repeat.rs index d1f2879235..a9478041c6 100644 --- a/library/core/src/iter/sources/repeat.rs +++ b/library/core/src/iter/sources/repeat.rs @@ -72,10 +72,32 @@ impl Iterator for Repeat { fn next(&mut self) -> Option { Some(self.element.clone()) } + #[inline] fn size_hint(&self) -> (usize, Option) { (usize::MAX, None) } + + #[inline] + fn advance_by(&mut self, n: usize) -> Result<(), usize> { + // Advancing an infinite iterator of a single element is a no-op. + let _ = n; + Ok(()) + } + + #[inline] + fn nth(&mut self, n: usize) -> Option { + let _ = n; + Some(self.element.clone()) + } + + fn last(self) -> Option { + loop {} + } + + fn count(self) -> usize { + loop {} + } } #[stable(feature = "rust1", since = "1.0.0")] @@ -84,6 +106,19 @@ impl DoubleEndedIterator for Repeat { fn next_back(&mut self) -> Option { Some(self.element.clone()) } + + #[inline] + fn advance_back_by(&mut self, n: usize) -> Result<(), usize> { + // Advancing an infinite iterator of a single element is a no-op. + let _ = n; + Ok(()) + } + + #[inline] + fn nth_back(&mut self, n: usize) -> Option { + let _ = n; + Some(self.element.clone()) + } } #[stable(feature = "fused", since = "1.26.0")] diff --git a/library/core/src/iter/traits/collect.rs b/library/core/src/iter/traits/collect.rs index 13a2e24cad..e2a407509b 100644 --- a/library/core/src/iter/traits/collect.rs +++ b/library/core/src/iter/traits/collect.rs @@ -198,7 +198,7 @@ pub trait FromIterator: Sized { /// } /// ``` #[rustc_diagnostic_item = "IntoIterator"] -#[cfg_attr(not(bootstrap), rustc_skip_array_during_method_dispatch)] +#[rustc_skip_array_during_method_dispatch] #[stable(feature = "rust1", since = "1.0.0")] pub trait IntoIterator { /// The type of the elements being iterated over. diff --git a/library/core/src/iter/traits/double_ended.rs b/library/core/src/iter/traits/double_ended.rs index 6f8cb6b5a6..c302502b3b 100644 --- a/library/core/src/iter/traits/double_ended.rs +++ b/library/core/src/iter/traits/double_ended.rs @@ -218,7 +218,7 @@ pub trait DoubleEndedIterator: Iterator { where Self: Sized, F: FnMut(B, Self::Item) -> R, - R: Try, + R: Try, { let mut accum = init; while let Some(x) = self.next_back() { diff --git a/library/core/src/iter/traits/iterator.rs b/library/core/src/iter/traits/iterator.rs index da9e5fde7c..11dea400a4 100644 --- a/library/core/src/iter/traits/iterator.rs +++ b/library/core/src/iter/traits/iterator.rs @@ -25,40 +25,6 @@ fn _assert_is_object_safe(_: &dyn Iterator) {} /// [impl]: crate::iter#implementing-iterator #[stable(feature = "rust1", since = "1.0.0")] #[rustc_on_unimplemented( - on( - _Self = "[std::ops::Range; 1]", - label = "if you meant to iterate between two values, remove the square brackets", - note = "`[start..end]` is an array of one `Range`; you might have meant to have a `Range` \ - without the brackets: `start..end`" - ), - on( - _Self = "[std::ops::RangeFrom; 1]", - label = "if you meant to iterate from a value onwards, remove the square brackets", - note = "`[start..]` is an array of one `RangeFrom`; you might have meant to have a \ - `RangeFrom` without the brackets: `start..`, keeping in mind that iterating over an \ - unbounded iterator will run forever unless you `break` or `return` from within the \ - loop" - ), - on( - _Self = "[std::ops::RangeTo; 1]", - label = "if you meant to iterate until a value, remove the square brackets and add a \ - starting value", - note = "`[..end]` is an array of one `RangeTo`; you might have meant to have a bounded \ - `Range` without the brackets: `0..end`" - ), - on( - _Self = "[std::ops::RangeInclusive; 1]", - label = "if you meant to iterate between two values, remove the square brackets", - note = "`[start..=end]` is an array of one `RangeInclusive`; you might have meant to have a \ - `RangeInclusive` without the brackets: `start..=end`" - ), - on( - _Self = "[std::ops::RangeToInclusive; 1]", - label = "if you meant to iterate until a value (including it), remove the square brackets \ - and add a starting value", - note = "`[..=end]` is an array of one `RangeToInclusive`; you might have meant to have a \ - bounded `RangeInclusive` without the brackets: `0..=end`" - ), on( _Self = "std::ops::RangeTo", label = "if you meant to iterate until a value, add a starting value", @@ -79,11 +45,6 @@ fn _assert_is_object_safe(_: &dyn Iterator) {} _Self = "std::string::String", label = "`{Self}` is not an iterator; try calling `.chars()` or `.bytes()`" ), - on( - _Self = "[]", - label = "arrays do not yet implement `IntoIterator`; try using `std::array::IntoIter::new(arr)`", - note = "see for more details" - ), on( _Self = "{integral}", note = "if you want to iterate between `start` until a value `end`, use the exclusive range \ @@ -92,8 +53,7 @@ fn _assert_is_object_safe(_: &dyn Iterator) {} label = "`{Self}` is not an iterator", message = "`{Self}` is not an iterator" )] -#[cfg_attr(bootstrap, doc(spotlight))] -#[cfg_attr(not(bootstrap), doc(notable_trait))] +#[doc(notable_trait)] #[rustc_diagnostic_item = "Iterator"] #[must_use = "iterators are lazy and do nothing unless consumed"] pub trait Iterator { @@ -178,7 +138,7 @@ pub trait Iterator { /// A more complex example: /// /// ``` - /// // The even numbers from zero to ten. + /// // The even numbers in the range of zero to nine. /// let iter = (0..10).filter(|x| x % 2 == 0); /// /// // We might iterate from zero to ten times. Knowing that it's five @@ -1849,6 +1809,12 @@ pub trait Iterator { /// /// The relative order of partitioned items is not maintained. /// + /// # Current implementation + /// Current algorithms tries finding the first element for which the predicate evaluates + /// to false, and the last element for which it evaluates to true and repeatedly swaps them. + /// + /// Time Complexity: *O*(*N*) + /// /// See also [`is_partitioned()`] and [`partition()`]. /// /// [`is_partitioned()`]: Iterator::is_partitioned @@ -1999,7 +1965,7 @@ pub trait Iterator { where Self: Sized, F: FnMut(B, Self::Item) -> R, - R: Try, + R: Try, { let mut accum = init; while let Some(x) = self.next() { @@ -2041,7 +2007,7 @@ pub trait Iterator { where Self: Sized, F: FnMut(Self::Item) -> R, - R: Try, + R: Try, { #[inline] fn call(mut f: impl FnMut(T) -> R) -> impl FnMut((), T) -> R { @@ -2412,17 +2378,48 @@ pub trait Iterator { /// ``` #[inline] #[unstable(feature = "try_find", reason = "new API", issue = "63178")] + #[cfg(not(bootstrap))] + fn try_find(&mut self, f: F) -> Result, E> + where + Self: Sized, + F: FnMut(&Self::Item) -> R, + R: Try, + // FIXME: This bound is rather strange, but means minimal breakage on nightly. + // See #85115 for the issue tracking a holistic solution for this and try_map. + R: crate::ops::TryV2>, + { + #[inline] + fn check(mut f: F) -> impl FnMut((), T) -> ControlFlow> + where + F: FnMut(&T) -> R, + R: Try, + R: crate::ops::TryV2>, + { + move |(), x| match f(&x).branch() { + ControlFlow::Continue(false) => ControlFlow::CONTINUE, + ControlFlow::Continue(true) => ControlFlow::Break(Ok(x)), + ControlFlow::Break(Err(x)) => ControlFlow::Break(Err(x)), + } + } + + self.try_fold((), check(f)).break_value().transpose() + } + + /// We're bootstrapping. + #[inline] + #[unstable(feature = "try_find", reason = "new API", issue = "63178")] + #[cfg(bootstrap)] fn try_find(&mut self, f: F) -> Result, R::Error> where Self: Sized, F: FnMut(&Self::Item) -> R, - R: Try, + R: Try, { #[inline] fn check(mut f: F) -> impl FnMut((), T) -> ControlFlow> where F: FnMut(&T) -> R, - R: Try, + R: Try, { move |(), x| match f(&x).into_result() { Ok(false) => ControlFlow::CONTINUE, @@ -2571,6 +2568,18 @@ pub trait Iterator { /// If several elements are equally maximum, the last element is /// returned. If the iterator is empty, [`None`] is returned. /// + /// Note that [`f32`]/[`f64`] doesn't implement [`Ord`] due to NaN being + /// incomparable. You can work around this by using [`Iterator::reduce`]: + /// ``` + /// assert_eq!( + /// vec![2.4, f32::NAN, 1.3] + /// .into_iter() + /// .reduce(f32::max) + /// .unwrap(), + /// 2.4 + /// ); + /// ``` + /// /// # Examples /// /// Basic usage: @@ -2594,8 +2603,20 @@ pub trait Iterator { /// Returns the minimum element of an iterator. /// - /// If several elements are equally minimum, the first element is - /// returned. If the iterator is empty, [`None`] is returned. + /// If several elements are equally minimum, the first element is returned. + /// If the iterator is empty, [`None`] is returned. + /// + /// Note that [`f32`]/[`f64`] doesn't implement [`Ord`] due to NaN being + /// incomparable. You can work around this by using [`Iterator::reduce`]: + /// ``` + /// assert_eq!( + /// vec![2.4, f32::NAN, 1.3] + /// .into_iter() + /// .reduce(f32::min) + /// .unwrap(), + /// 1.3 + /// ); + /// ``` /// /// # Examples /// diff --git a/library/core/src/iter/traits/marker.rs b/library/core/src/iter/traits/marker.rs index c4e21b8486..ebf37f97bc 100644 --- a/library/core/src/iter/traits/marker.rs +++ b/library/core/src/iter/traits/marker.rs @@ -1,3 +1,5 @@ +use crate::iter::Step; + /// An iterator that always continues to yield `None` when exhausted. /// /// Calling next on a fused iterator that has returned `None` once is guaranteed @@ -53,4 +55,20 @@ unsafe impl TrustedLen for &mut I {} /// [`next()`]: Iterator::next /// [`try_fold()`]: Iterator::try_fold #[unstable(issue = "none", feature = "inplace_iteration")] +#[doc(hidden)] pub unsafe trait InPlaceIterable: Iterator {} + +/// A type that upholds all invariants of [`Step`]. +/// +/// The invariants of [`Step::steps_between()`] are a superset of the invariants +/// of [`TrustedLen`]. As such, [`TrustedLen`] is implemented for all range +/// types with the same generic type argument. +/// +/// # Safety +/// +/// The implementation of [`Step`] for the given type must guarantee all +/// invariants of all methods are upheld. See the [`Step`] trait's documentation +/// for details. Consumers are free to rely on the invariants in unsafe code. +#[unstable(feature = "trusted_step", issue = "85731")] +#[rustc_specialization_trait] +pub unsafe trait TrustedStep: Step {} diff --git a/library/core/src/iter/traits/mod.rs b/library/core/src/iter/traits/mod.rs index 880f8d831f..ffd745a46b 100644 --- a/library/core/src/iter/traits/mod.rs +++ b/library/core/src/iter/traits/mod.rs @@ -13,5 +13,7 @@ pub use self::exact_size::ExactSizeIterator; pub use self::iterator::Iterator; #[unstable(issue = "none", feature = "inplace_iteration")] pub use self::marker::InPlaceIterable; +#[unstable(feature = "trusted_step", issue = "85731")] +pub use self::marker::TrustedStep; #[stable(feature = "rust1", since = "1.0.0")] pub use self::marker::{FusedIterator, TrustedLen}; diff --git a/library/core/src/lib.rs b/library/core/src/lib.rs index d1329b8f22..0bb0f8768a 100644 --- a/library/core/src/lib.rs +++ b/library/core/src/lib.rs @@ -51,7 +51,6 @@ #![cfg(not(test))] #![stable(feature = "core", since = "1.6.0")] #![doc( - html_root_url = "https://doc.rust-lang.org/nightly/", html_playground_url = "https://play.rust-lang.org/", issue_tracker_base_url = "https://github.com/rust-lang/rust/issues/", test(no_crate_inject, attr(deny(warnings))), @@ -66,6 +65,7 @@ #![feature(allow_internal_unstable)] #![feature(arbitrary_self_types)] #![feature(asm)] +#![feature(bool_to_option)] #![feature(cfg_target_has_atomic)] #![feature(const_heap)] #![feature(const_alloc_layout)] @@ -77,16 +77,17 @@ #![feature(const_float_classify)] #![feature(const_float_bits_conv)] #![feature(const_int_unchecked_arith)] +#![feature(const_inherent_unchecked_arith)] #![feature(const_mut_refs)] #![feature(const_refs_to_cell)] #![feature(const_panic)] #![feature(const_pin)] -#![cfg_attr(bootstrap, feature(const_fn))] #![feature(const_fn_union)] #![feature(const_impl_trait)] #![feature(const_fn_floating_point_arithmetic)] #![feature(const_fn_fn_ptr_basics)] -#![cfg_attr(not(bootstrap), feature(const_fn_trait_bound))] +#![feature(const_fn_trait_bound)] +#![cfg_attr(bootstrap, feature(const_fn))] #![feature(const_option)] #![feature(const_precise_live_drops)] #![feature(const_ptr_offset)] @@ -109,10 +110,9 @@ #![feature(custom_inner_attributes)] #![feature(decl_macro)] #![feature(doc_cfg)] -#![cfg_attr(bootstrap, feature(doc_spotlight))] -#![cfg_attr(not(bootstrap), feature(doc_notable_trait))] +#![feature(doc_notable_trait)] #![feature(duration_consts_2)] -#![feature(extended_key_value_attributes)] +#![cfg_attr(bootstrap, feature(extended_key_value_attributes))] #![feature(extern_types)] #![feature(fundamental)] #![feature(intra_doc_pointers)] @@ -126,7 +126,6 @@ #![feature(exhaustive_patterns)] #![feature(no_core)] #![feature(auto_traits)] -#![cfg_attr(bootstrap, feature(or_patterns))] #![feature(prelude_import)] #![feature(ptr_metadata)] #![feature(repr_simd, platform_intrinsics)] @@ -161,14 +160,19 @@ #![feature(const_fn_transmute)] #![feature(abi_unadjusted)] #![feature(adx_target_feature)] -#![feature(external_doc)] #![feature(associated_type_bounds)] #![feature(const_caller_location)] #![feature(slice_ptr_get)] #![feature(no_niche)] // rust-lang/rust#68303 -#![cfg_attr(not(bootstrap), feature(no_coverage))] // rust-lang/rust#84605 +#![feature(no_coverage)] // rust-lang/rust#84605 #![feature(int_error_matching)] +#![cfg_attr(bootstrap, feature(target_feature_11))] #![deny(unsafe_op_in_unsafe_fn)] +#![deny(or_patterns_back_compat)] + +// allow using `core::` in intra-doc links +#[allow(unused_extern_crates)] +extern crate self as core; #[prelude_import] #[allow(unused)] @@ -178,6 +182,16 @@ use prelude::v1::*; #[macro_use] mod macros; +// We don't export this through #[macro_export] for now, to avoid breakage. +// See https://github.com/rust-lang/rust/issues/82913 +#[cfg(not(test))] +#[unstable(feature = "assert_matches", issue = "82775")] +/// Unstable module containing the unstable `assert_matches` macro. +pub mod assert_matches { + #[unstable(feature = "assert_matches", issue = "82775")] + pub use crate::macros::{assert_matches, debug_assert_matches}; +} + #[macro_use] mod internal_macros; @@ -299,8 +313,7 @@ pub mod primitive; unused_imports, unsafe_op_in_unsafe_fn )] -#[cfg_attr(bootstrap, allow(rustdoc::non_autolinks))] -#[cfg_attr(not(bootstrap), allow(rustdoc::bare_urls))] +#[allow(rustdoc::bare_urls)] // FIXME: This annotation should be moved into rust-lang/stdarch after clashing_extern_declarations is // merged. It currently cannot because bootstrap fails as the lint hasn't been defined yet. #[allow(clashing_extern_declarations)] diff --git a/library/core/src/macros/mod.rs b/library/core/src/macros/mod.rs index 66efa3c20b..1a3c088e92 100644 --- a/library/core/src/macros/mod.rs +++ b/library/core/src/macros/mod.rs @@ -110,6 +110,62 @@ macro_rules! assert_ne { }); } +/// Asserts that an expression matches any of the given patterns. +/// +/// Like in a `match` expression, the pattern can be optionally followed by `if` +/// and a guard expression that has access to names bound by the pattern. +/// +/// On panic, this macro will print the value of the expression with its +/// debug representation. +/// +/// Like [`assert!`], this macro has a second form, where a custom +/// panic message can be provided. +/// +/// # Examples +/// +/// ``` +/// #![feature(assert_matches)] +/// +/// use std::assert_matches::assert_matches; +/// +/// let a = 1u32.checked_add(2); +/// let b = 1u32.checked_sub(2); +/// assert_matches!(a, Some(_)); +/// assert_matches!(b, None); +/// +/// let c = Ok("abc".to_string()); +/// assert_matches!(c, Ok(x) | Err(x) if x.len() < 100); +/// ``` +#[unstable(feature = "assert_matches", issue = "82775")] +#[allow_internal_unstable(core_panic)] +#[rustc_macro_transparency = "semitransparent"] +pub macro assert_matches { + ($left:expr, $( $pattern:pat_param )|+ $( if $guard: expr )? $(,)?) => ({ + match $left { + $( $pattern )|+ $( if $guard )? => {} + ref left_val => { + $crate::panicking::assert_matches_failed( + left_val, + $crate::stringify!($($pattern)|+ $(if $guard)?), + $crate::option::Option::None + ); + } + } + }), + ($left:expr, $( $pattern:pat_param )|+ $( if $guard: expr )?, $($arg:tt)+) => ({ + match $left { + $( $pattern )|+ $( if $guard )? => {} + ref left_val => { + $crate::panicking::assert_matches_failed( + left_val, + $crate::stringify!($($pattern)|+ $(if $guard)?), + $crate::option::Option::Some($crate::format_args!($($arg)+)) + ); + } + } + }), +} + /// Asserts that a boolean expression is `true` at runtime. /// /// This will invoke the [`panic!`] macro if the provided expression cannot be @@ -208,6 +264,45 @@ macro_rules! debug_assert_ne { ($($arg:tt)*) => (if $crate::cfg!(debug_assertions) { $crate::assert_ne!($($arg)*); }) } +/// Asserts that an expression matches any of the given patterns. +/// +/// Like in a `match` expression, the pattern can be optionally followed by `if` +/// and a guard expression that has access to names bound by the pattern. +/// +/// On panic, this macro will print the value of the expression with its +/// debug representation. +/// +/// Unlike [`assert_matches!`], `debug_assert_matches!` statements are only +/// enabled in non optimized builds by default. An optimized build will not +/// execute `debug_assert_matches!` statements unless `-C debug-assertions` is +/// passed to the compiler. This makes `debug_assert_matches!` useful for +/// checks that are too expensive to be present in a release build but may be +/// helpful during development. The result of expanding `debug_assert_matches!` +/// is always type checked. +/// +/// # Examples +/// +/// ``` +/// #![feature(assert_matches)] +/// +/// use std::assert_matches::debug_assert_matches; +/// +/// let a = 1u32.checked_add(2); +/// let b = 1u32.checked_sub(2); +/// debug_assert_matches!(a, Some(_)); +/// debug_assert_matches!(b, None); +/// +/// let c = Ok("abc".to_string()); +/// debug_assert_matches!(c, Ok(x) | Err(x) if x.len() < 100); +/// ``` +#[macro_export] +#[unstable(feature = "assert_matches", issue = "82775")] +#[allow_internal_unstable(assert_matches)] +#[rustc_macro_transparency = "semitransparent"] +pub macro debug_assert_matches($($arg:tt)*) { + if $crate::cfg!(debug_assertions) { $crate::assert_matches::assert_matches!($($arg)*); } +} + /// Returns whether the given expression matches any of the given patterns. /// /// Like in a `match` expression, the pattern can be optionally followed by `if` @@ -225,7 +320,7 @@ macro_rules! debug_assert_ne { #[macro_export] #[stable(feature = "matches_macro", since = "1.42.0")] macro_rules! matches { - ($expression:expr, $( $pattern:pat )|+ $( if $guard: expr )? $(,)?) => { + ($expression:expr, $( $pattern:pat_param )|+ $( if $guard: expr )? $(,)?) => { match $expression { $( $pattern )|+ $( if $guard )? => true, _ => false @@ -505,7 +600,7 @@ macro_rules! unreachable { /// Indicates unimplemented code by panicking with a message of "not implemented". /// /// This allows your code to type-check, which is useful if you are prototyping or -/// implementing a trait that requires multiple methods which you don't plan of using all of. +/// implementing a trait that requires multiple methods which you don't plan to use all of. /// /// The difference between `unimplemented!` and [`todo!`] is that while `todo!` /// conveys an intent of implementing the functionality later and the message is "not yet @@ -1268,7 +1363,10 @@ pub(crate) mod builtin { #[rustc_builtin_macro] #[macro_export] macro_rules! global_asm { - ("assembly") => { + ("assembly template", + $(operands,)* + $(options($(option),*))? + ) => { /* compiler built-in */ }; } diff --git a/library/core/src/marker.rs b/library/core/src/marker.rs index fb957348be..2b24045504 100644 --- a/library/core/src/marker.rs +++ b/library/core/src/marker.rs @@ -31,6 +31,7 @@ use crate::hash::Hasher; /// [ub]: ../../reference/behavior-considered-undefined.html #[stable(feature = "rust1", since = "1.0.0")] #[cfg_attr(not(test), rustc_diagnostic_item = "send_trait")] +#[cfg_attr(not(bootstrap), lang = "send")] #[rustc_on_unimplemented( message = "`{Self}` cannot be sent between threads safely", label = "`{Self}` cannot be sent between threads safely" diff --git a/library/core/src/mem/maybe_uninit.rs b/library/core/src/mem/maybe_uninit.rs index 4d7d47579e..10219201a4 100644 --- a/library/core/src/mem/maybe_uninit.rs +++ b/library/core/src/mem/maybe_uninit.rs @@ -10,7 +10,7 @@ use crate::ptr; /// /// The compiler, in general, assumes that a variable is properly initialized /// according to the requirements of the variable's type. For example, a variable of -/// reference type must be aligned and non-NULL. This is an invariant that must +/// reference type must be aligned and non-null. This is an invariant that must /// *always* be upheld, even in unsafe code. As a consequence, zero-initializing a /// variable of reference type causes instantaneous [undefined behavior][ub], /// no matter whether that reference ever gets used to access memory: @@ -870,7 +870,7 @@ impl MaybeUninit { // SAFETY: // * The caller guarantees that all elements of the array are initialized // * `MaybeUninit` and T are guaranteed to have the same layout - // * MaybeUnint does not drop, so there are no double-frees + // * `MaybeUninit` does not drop, so there are no double-frees // And thus the conversion is safe unsafe { intrinsics::assert_inhabited::<[T; N]>(); diff --git a/library/core/src/num/bignum.rs b/library/core/src/num/bignum.rs index 197b85fba1..8a06a09888 100644 --- a/library/core/src/num/bignum.rs +++ b/library/core/src/num/bignum.rs @@ -119,7 +119,7 @@ macro_rules! define_bignum { pub fn from_small(v: $ty) -> $name { let mut base = [0; $n]; base[0] = v; - $name { size: 1, base: base } + $name { size: 1, base } } /// Makes a bignum from `u64` value. @@ -131,7 +131,7 @@ macro_rules! define_bignum { v >>= <$ty>::BITS; sz += 1; } - $name { size: sz, base: base } + $name { size: sz, base } } /// Returns the internal digits as a slice `[a, b, c, ...]` such that the numeric diff --git a/library/core/src/num/f32.rs b/library/core/src/num/f32.rs index 0d6d919d99..c47a2e8b05 100644 --- a/library/core/src/num/f32.rs +++ b/library/core/src/num/f32.rs @@ -727,8 +727,8 @@ impl f32 { /// /// This is currently identical to `transmute::(self)` on all platforms. /// - /// See `from_bits` for some discussion of the portability of this operation - /// (there are almost no issues). + /// See [`from_bits`](Self::from_bits) for some discussion of the + /// portability of this operation (there are almost no issues). /// /// Note that this function is distinct from `as` casting, which attempts to /// preserve the *numeric* value, and not the bitwise value. @@ -854,35 +854,6 @@ impl f32 { self.to_bits().to_ne_bytes() } - /// Return the memory representation of this floating point number as a byte array in - /// native byte order. - /// - /// [`to_ne_bytes`] should be preferred over this whenever possible. - /// - /// [`to_ne_bytes`]: f32::to_ne_bytes - /// - /// # Examples - /// - /// ``` - /// #![feature(num_as_ne_bytes)] - /// let num = 12.5f32; - /// let bytes = num.as_ne_bytes(); - /// assert_eq!( - /// bytes, - /// if cfg!(target_endian = "big") { - /// &[0x41, 0x48, 0x00, 0x00] - /// } else { - /// &[0x00, 0x00, 0x48, 0x41] - /// } - /// ); - /// ``` - #[unstable(feature = "num_as_ne_bytes", issue = "76976")] - #[inline] - pub fn as_ne_bytes(&self) -> &[u8; 4] { - // SAFETY: `f32` is a plain old datatype so we can always transmute to it - unsafe { &*(self as *const Self as *const _) } - } - /// Create a floating point value from its representation as a byte array in big endian. /// /// # Examples diff --git a/library/core/src/num/f64.rs b/library/core/src/num/f64.rs index 42214e7b50..cfcc08b9ad 100644 --- a/library/core/src/num/f64.rs +++ b/library/core/src/num/f64.rs @@ -741,8 +741,8 @@ impl f64 { /// /// This is currently identical to `transmute::(self)` on all platforms. /// - /// See `from_bits` for some discussion of the portability of this operation - /// (there are almost no issues). + /// See [`from_bits`](Self::from_bits) for some discussion of the + /// portability of this operation (there are almost no issues). /// /// Note that this function is distinct from `as` casting, which attempts to /// preserve the *numeric* value, and not the bitwise value. @@ -868,35 +868,6 @@ impl f64 { self.to_bits().to_ne_bytes() } - /// Return the memory representation of this floating point number as a byte array in - /// native byte order. - /// - /// [`to_ne_bytes`] should be preferred over this whenever possible. - /// - /// [`to_ne_bytes`]: f64::to_ne_bytes - /// - /// # Examples - /// - /// ``` - /// #![feature(num_as_ne_bytes)] - /// let num = 12.5f64; - /// let bytes = num.as_ne_bytes(); - /// assert_eq!( - /// bytes, - /// if cfg!(target_endian = "big") { - /// &[0x40, 0x29, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] - /// } else { - /// &[0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x29, 0x40] - /// } - /// ); - /// ``` - #[unstable(feature = "num_as_ne_bytes", issue = "76976")] - #[inline] - pub fn as_ne_bytes(&self) -> &[u8; 8] { - // SAFETY: `f64` is a plain old datatype so we can always transmute to it - unsafe { &*(self as *const Self as *const _) } - } - /// Create a floating point value from its representation as a byte array in big endian. /// /// # Examples diff --git a/library/core/src/num/int_macros.rs b/library/core/src/num/int_macros.rs index 4b341132e3..a0efe68128 100644 --- a/library/core/src/num/int_macros.rs +++ b/library/core/src/num/int_macros.rs @@ -407,17 +407,25 @@ macro_rules! int_impl { } /// Unchecked integer addition. Computes `self + rhs`, assuming overflow - /// cannot occur. This results in undefined behavior when - #[doc = concat!("`self + rhs > ", stringify!($SelfT), "::MAX` or `self + rhs < ", stringify!($SelfT), "::MIN`.")] + /// cannot occur. + /// + /// # Safety + /// + /// This results in undefined behavior when + #[doc = concat!("`self + rhs > ", stringify!($SelfT), "::MAX` or `self + rhs < ", stringify!($SelfT), "::MIN`,")] + /// i.e. when [`checked_add`] would return `None`. + /// + #[doc = concat!("[`checked_add`]: ", stringify!($SelfT), "::checked_add")] #[unstable( feature = "unchecked_math", reason = "niche optimization path", - issue = "none", + issue = "85122", )] #[must_use = "this returns the result of the operation, \ without modifying the original"] + #[rustc_const_unstable(feature = "const_inherent_unchecked_arith", issue = "85122")] #[inline(always)] - pub unsafe fn unchecked_add(self, rhs: Self) -> Self { + pub const unsafe fn unchecked_add(self, rhs: Self) -> Self { // SAFETY: the caller must uphold the safety contract for // `unchecked_add`. unsafe { intrinsics::unchecked_add(self, rhs) } @@ -445,17 +453,25 @@ macro_rules! int_impl { } /// Unchecked integer subtraction. Computes `self - rhs`, assuming overflow - /// cannot occur. This results in undefined behavior when - #[doc = concat!("`self - rhs > ", stringify!($SelfT), "::MAX` or `self - rhs < ", stringify!($SelfT), "::MIN`.")] + /// cannot occur. + /// + /// # Safety + /// + /// This results in undefined behavior when + #[doc = concat!("`self - rhs > ", stringify!($SelfT), "::MAX` or `self - rhs < ", stringify!($SelfT), "::MIN`,")] + /// i.e. when [`checked_sub`] would return `None`. + /// + #[doc = concat!("[`checked_sub`]: ", stringify!($SelfT), "::checked_sub")] #[unstable( feature = "unchecked_math", reason = "niche optimization path", - issue = "none", + issue = "85122", )] #[must_use = "this returns the result of the operation, \ without modifying the original"] + #[rustc_const_unstable(feature = "const_inherent_unchecked_arith", issue = "85122")] #[inline(always)] - pub unsafe fn unchecked_sub(self, rhs: Self) -> Self { + pub const unsafe fn unchecked_sub(self, rhs: Self) -> Self { // SAFETY: the caller must uphold the safety contract for // `unchecked_sub`. unsafe { intrinsics::unchecked_sub(self, rhs) } @@ -483,17 +499,25 @@ macro_rules! int_impl { } /// Unchecked integer multiplication. Computes `self * rhs`, assuming overflow - /// cannot occur. This results in undefined behavior when - #[doc = concat!("`self * rhs > ", stringify!($SelfT), "::MAX` or `self * rhs < ", stringify!($SelfT), "::MIN`.")] + /// cannot occur. + /// + /// # Safety + /// + /// This results in undefined behavior when + #[doc = concat!("`self * rhs > ", stringify!($SelfT), "::MAX` or `self * rhs < ", stringify!($SelfT), "::MIN`,")] + /// i.e. when [`checked_mul`] would return `None`. + /// + #[doc = concat!("[`checked_mul`]: ", stringify!($SelfT), "::checked_mul")] #[unstable( feature = "unchecked_math", reason = "niche optimization path", - issue = "none", + issue = "85122", )] #[must_use = "this returns the result of the operation, \ without modifying the original"] + #[rustc_const_unstable(feature = "const_inherent_unchecked_arith", issue = "85122")] #[inline(always)] - pub unsafe fn unchecked_mul(self, rhs: Self) -> Self { + pub const unsafe fn unchecked_mul(self, rhs: Self) -> Self { // SAFETY: the caller must uphold the safety contract for // `unchecked_mul`. unsafe { intrinsics::unchecked_mul(self, rhs) } @@ -642,6 +666,31 @@ macro_rules! int_impl { if unlikely!(b) {None} else {Some(a)} } + /// Unchecked shift left. Computes `self << rhs`, assuming that + /// `rhs` is less than the number of bits in `self`. + /// + /// # Safety + /// + /// This results in undefined behavior if `rhs` is larger than + /// or equal to the number of bits in `self`, + /// i.e. when [`checked_shl`] would return `None`. + /// + #[doc = concat!("[`checked_shl`]: ", stringify!($SelfT), "::checked_shl")] + #[unstable( + feature = "unchecked_math", + reason = "niche optimization path", + issue = "85122", + )] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[rustc_const_unstable(feature = "const_inherent_unchecked_arith", issue = "85122")] + #[inline(always)] + pub const unsafe fn unchecked_shl(self, rhs: Self) -> Self { + // SAFETY: the caller must uphold the safety contract for + // `unchecked_shl`. + unsafe { intrinsics::unchecked_shl(self, rhs) } + } + /// Checked shift right. Computes `self >> rhs`, returning `None` if `rhs` is /// larger than or equal to the number of bits in `self`. /// @@ -663,6 +712,31 @@ macro_rules! int_impl { if unlikely!(b) {None} else {Some(a)} } + /// Unchecked shift right. Computes `self >> rhs`, assuming that + /// `rhs` is less than the number of bits in `self`. + /// + /// # Safety + /// + /// This results in undefined behavior if `rhs` is larger than + /// or equal to the number of bits in `self`, + /// i.e. when [`checked_shr`] would return `None`. + /// + #[doc = concat!("[`checked_shr`]: ", stringify!($SelfT), "::checked_shr")] + #[unstable( + feature = "unchecked_math", + reason = "niche optimization path", + issue = "85122", + )] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[rustc_const_unstable(feature = "const_inherent_unchecked_arith", issue = "85122")] + #[inline(always)] + pub const unsafe fn unchecked_shr(self, rhs: Self) -> Self { + // SAFETY: the caller must uphold the safety contract for + // `unchecked_shr`. + unsafe { intrinsics::unchecked_shr(self, rhs) } + } + /// Checked absolute value. Computes `self.abs()`, returning `None` if /// `self == MIN`. /// @@ -1839,36 +1913,6 @@ macro_rules! int_impl { unsafe { mem::transmute(self) } } - /// Return the memory representation of this integer as a byte array in - /// native byte order. - /// - /// [`to_ne_bytes`] should be preferred over this whenever possible. - /// - /// [`to_ne_bytes`]: Self::to_ne_bytes - /// - /// # Examples - /// - /// ``` - /// #![feature(num_as_ne_bytes)] - #[doc = concat!("let num = ", $swap_op, stringify!($SelfT), ";")] - /// let bytes = num.as_ne_bytes(); - /// assert_eq!( - /// bytes, - /// if cfg!(target_endian = "big") { - #[doc = concat!(" &", $be_bytes)] - /// } else { - #[doc = concat!(" &", $le_bytes)] - /// } - /// ); - /// ``` - #[unstable(feature = "num_as_ne_bytes", issue = "76976")] - #[inline] - pub fn as_ne_bytes(&self) -> &[u8; mem::size_of::()] { - // SAFETY: integers are plain old datatypes so we can always transmute them to - // arrays of bytes - unsafe { &*(self as *const Self as *const _) } - } - /// Create an integer value from its representation as a byte array in /// big endian. /// diff --git a/library/core/src/num/uint_macros.rs b/library/core/src/num/uint_macros.rs index 08d9161eff..e512d90ef3 100644 --- a/library/core/src/num/uint_macros.rs +++ b/library/core/src/num/uint_macros.rs @@ -417,17 +417,25 @@ macro_rules! uint_impl { } /// Unchecked integer addition. Computes `self + rhs`, assuming overflow - /// cannot occur. This results in undefined behavior when - #[doc = concat!("`self + rhs > ", stringify!($SelfT), "::MAX` or `self + rhs < ", stringify!($SelfT), "::MIN`.")] + /// cannot occur. + /// + /// # Safety + /// + /// This results in undefined behavior when + #[doc = concat!("`self + rhs > ", stringify!($SelfT), "::MAX` or `self + rhs < ", stringify!($SelfT), "::MIN`,")] + /// i.e. when [`checked_add`] would return `None`. + /// + #[doc = concat!("[`checked_add`]: ", stringify!($SelfT), "::checked_add")] #[unstable( feature = "unchecked_math", reason = "niche optimization path", - issue = "none", + issue = "85122", )] #[must_use = "this returns the result of the operation, \ without modifying the original"] + #[rustc_const_unstable(feature = "const_inherent_unchecked_arith", issue = "85122")] #[inline(always)] - pub unsafe fn unchecked_add(self, rhs: Self) -> Self { + pub const unsafe fn unchecked_add(self, rhs: Self) -> Self { // SAFETY: the caller must uphold the safety contract for // `unchecked_add`. unsafe { intrinsics::unchecked_add(self, rhs) } @@ -455,17 +463,25 @@ macro_rules! uint_impl { } /// Unchecked integer subtraction. Computes `self - rhs`, assuming overflow - /// cannot occur. This results in undefined behavior when - #[doc = concat!("`self - rhs > ", stringify!($SelfT), "::MAX` or `self - rhs < ", stringify!($SelfT), "::MIN`.")] + /// cannot occur. + /// + /// # Safety + /// + /// This results in undefined behavior when + #[doc = concat!("`self - rhs > ", stringify!($SelfT), "::MAX` or `self - rhs < ", stringify!($SelfT), "::MIN`,")] + /// i.e. when [`checked_sub`] would return `None`. + /// + #[doc = concat!("[`checked_sub`]: ", stringify!($SelfT), "::checked_sub")] #[unstable( feature = "unchecked_math", reason = "niche optimization path", - issue = "none", + issue = "85122", )] #[must_use = "this returns the result of the operation, \ without modifying the original"] + #[rustc_const_unstable(feature = "const_inherent_unchecked_arith", issue = "85122")] #[inline(always)] - pub unsafe fn unchecked_sub(self, rhs: Self) -> Self { + pub const unsafe fn unchecked_sub(self, rhs: Self) -> Self { // SAFETY: the caller must uphold the safety contract for // `unchecked_sub`. unsafe { intrinsics::unchecked_sub(self, rhs) } @@ -493,17 +509,25 @@ macro_rules! uint_impl { } /// Unchecked integer multiplication. Computes `self * rhs`, assuming overflow - /// cannot occur. This results in undefined behavior when - #[doc = concat!("`self * rhs > ", stringify!($SelfT), "::MAX` or `self * rhs < ", stringify!($SelfT), "::MIN`.")] + /// cannot occur. + /// + /// # Safety + /// + /// This results in undefined behavior when + #[doc = concat!("`self * rhs > ", stringify!($SelfT), "::MAX` or `self * rhs < ", stringify!($SelfT), "::MIN`,")] + /// i.e. when [`checked_mul`] would return `None`. + /// + #[doc = concat!("[`checked_mul`]: ", stringify!($SelfT), "::checked_mul")] #[unstable( feature = "unchecked_math", reason = "niche optimization path", - issue = "none", + issue = "85122", )] #[must_use = "this returns the result of the operation, \ without modifying the original"] + #[rustc_const_unstable(feature = "const_inherent_unchecked_arith", issue = "85122")] #[inline(always)] - pub unsafe fn unchecked_mul(self, rhs: Self) -> Self { + pub const unsafe fn unchecked_mul(self, rhs: Self) -> Self { // SAFETY: the caller must uphold the safety contract for // `unchecked_mul`. unsafe { intrinsics::unchecked_mul(self, rhs) } @@ -652,6 +676,31 @@ macro_rules! uint_impl { if unlikely!(b) {None} else {Some(a)} } + /// Unchecked shift left. Computes `self << rhs`, assuming that + /// `rhs` is less than the number of bits in `self`. + /// + /// # Safety + /// + /// This results in undefined behavior if `rhs` is larger than + /// or equal to the number of bits in `self`, + /// i.e. when [`checked_shl`] would return `None`. + /// + #[doc = concat!("[`checked_shl`]: ", stringify!($SelfT), "::checked_shl")] + #[unstable( + feature = "unchecked_math", + reason = "niche optimization path", + issue = "85122", + )] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[rustc_const_unstable(feature = "const_inherent_unchecked_arith", issue = "85122")] + #[inline(always)] + pub const unsafe fn unchecked_shl(self, rhs: Self) -> Self { + // SAFETY: the caller must uphold the safety contract for + // `unchecked_shl`. + unsafe { intrinsics::unchecked_shl(self, rhs) } + } + /// Checked shift right. Computes `self >> rhs`, returning `None` /// if `rhs` is larger than or equal to the number of bits in `self`. /// @@ -673,6 +722,31 @@ macro_rules! uint_impl { if unlikely!(b) {None} else {Some(a)} } + /// Unchecked shift right. Computes `self >> rhs`, assuming that + /// `rhs` is less than the number of bits in `self`. + /// + /// # Safety + /// + /// This results in undefined behavior if `rhs` is larger than + /// or equal to the number of bits in `self`, + /// i.e. when [`checked_shr`] would return `None`. + /// + #[doc = concat!("[`checked_shr`]: ", stringify!($SelfT), "::checked_shr")] + #[unstable( + feature = "unchecked_math", + reason = "niche optimization path", + issue = "85122", + )] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[rustc_const_unstable(feature = "const_inherent_unchecked_arith", issue = "85122")] + #[inline(always)] + pub const unsafe fn unchecked_shr(self, rhs: Self) -> Self { + // SAFETY: the caller must uphold the safety contract for + // `unchecked_shr`. + unsafe { intrinsics::unchecked_shr(self, rhs) } + } + /// Checked exponentiation. Computes `self.pow(exp)`, returning `None` if /// overflow occurred. /// @@ -1669,36 +1743,6 @@ macro_rules! uint_impl { unsafe { mem::transmute(self) } } - /// Return the memory representation of this integer as a byte array in - /// native byte order. - /// - /// [`to_ne_bytes`] should be preferred over this whenever possible. - /// - /// [`to_ne_bytes`]: Self::to_ne_bytes - /// - /// # Examples - /// - /// ``` - /// #![feature(num_as_ne_bytes)] - #[doc = concat!("let num = ", $swap_op, stringify!($SelfT), ";")] - /// let bytes = num.as_ne_bytes(); - /// assert_eq!( - /// bytes, - /// if cfg!(target_endian = "big") { - #[doc = concat!(" &", $be_bytes)] - /// } else { - #[doc = concat!(" &", $le_bytes)] - /// } - /// ); - /// ``` - #[unstable(feature = "num_as_ne_bytes", issue = "76976")] - #[inline] - pub fn as_ne_bytes(&self) -> &[u8; mem::size_of::()] { - // SAFETY: integers are plain old datatypes so we can always transmute them to - // arrays of bytes - unsafe { &*(self as *const Self as *const _) } - } - /// Create a native endian integer value from its representation /// as a byte array in big endian. /// diff --git a/library/core/src/num/wrapping.rs b/library/core/src/num/wrapping.rs index f1b9dabe7d..be6d70320d 100644 --- a/library/core/src/num/wrapping.rs +++ b/library/core/src/num/wrapping.rs @@ -433,6 +433,21 @@ macro_rules! wrapping_int_impl { #[unstable(feature = "wrapping_int_impl", issue = "32463")] pub const MAX: Self = Self(<$t>::MAX); + /// Returns the size of this integer type in bits. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + /// #![feature(wrapping_int_impl)] + /// use std::num::Wrapping; + /// + #[doc = concat!("assert_eq!(>::BITS, ", stringify!($t), "::BITS);")] + /// ``` + #[unstable(feature = "wrapping_int_impl", issue = "32463")] + pub const BITS: u32 = <$t>::BITS; + /// Returns the number of ones in the binary representation of `self`. /// /// # Examples diff --git a/library/core/src/ops/control_flow.rs b/library/core/src/ops/control_flow.rs index ecaff053bd..9d9398fb56 100644 --- a/library/core/src/ops/control_flow.rs +++ b/library/core/src/ops/control_flow.rs @@ -1,5 +1,4 @@ -use crate::convert; -use crate::ops::{self, Try}; +use crate::{convert, ops}; /// Used to tell an operation whether it should exit early or go on as usual. /// @@ -53,8 +52,10 @@ use crate::ops::{self, Try}; #[derive(Debug, Clone, Copy, PartialEq)] pub enum ControlFlow { /// Move on to the next phase of the operation as normal. + #[cfg_attr(not(bootstrap), lang = "Continue")] Continue(C), /// Exit the operation without running subsequent phases. + #[cfg_attr(not(bootstrap), lang = "Break")] Break(B), // Yes, the order of the variants doesn't match the type parameters. // They're in this order so that `ControlFlow` <-> `Result` @@ -62,11 +63,12 @@ pub enum ControlFlow { } #[unstable(feature = "control_flow_enum", reason = "new API", issue = "75744")] -impl Try for ControlFlow { - type Ok = C; +#[cfg(bootstrap)] +impl ops::TryV1 for ControlFlow { + type Output = C; type Error = B; #[inline] - fn into_result(self) -> Result { + fn into_result(self) -> Result { match self { ControlFlow::Continue(y) => Ok(y), ControlFlow::Break(x) => Err(x), @@ -77,7 +79,7 @@ impl Try for ControlFlow { ControlFlow::Break(v) } #[inline] - fn from_ok(v: Self::Ok) -> Self { + fn from_ok(v: Self::Output) -> Self { ControlFlow::Continue(v) } } @@ -182,23 +184,46 @@ impl ControlFlow { } } -impl ControlFlow { +#[cfg(bootstrap)] +impl ControlFlow { /// Create a `ControlFlow` from any type implementing `Try`. - #[unstable(feature = "control_flow_enum", reason = "new API", issue = "75744")] #[inline] - pub fn from_try(r: R) -> Self { - match Try::into_result(r) { + pub(crate) fn from_try(r: R) -> Self { + match R::into_result(r) { Ok(v) => ControlFlow::Continue(v), - Err(v) => ControlFlow::Break(Try::from_error(v)), + Err(v) => ControlFlow::Break(R::from_error(v)), + } + } + + /// Convert a `ControlFlow` into any type implementing `Try`; + #[inline] + pub(crate) fn into_try(self) -> R { + match self { + ControlFlow::Continue(v) => R::from_ok(v), + ControlFlow::Break(v) => v, + } + } +} + +/// These are used only as part of implementing the iterator adapters. +/// They have mediocre names and non-obvious semantics, so aren't +/// currently on a path to potential stabilization. +#[cfg(not(bootstrap))] +impl ControlFlow { + /// Create a `ControlFlow` from any type implementing `Try`. + #[inline] + pub(crate) fn from_try(r: R) -> Self { + match R::branch(r) { + ControlFlow::Continue(v) => ControlFlow::Continue(v), + ControlFlow::Break(v) => ControlFlow::Break(R::from_residual(v)), } } /// Convert a `ControlFlow` into any type implementing `Try`; - #[unstable(feature = "control_flow_enum", reason = "new API", issue = "75744")] #[inline] - pub fn into_try(self) -> R { + pub(crate) fn into_try(self) -> R { match self { - ControlFlow::Continue(v) => Try::from_ok(v), + ControlFlow::Continue(v) => R::from_output(v), ControlFlow::Break(v) => v, } } diff --git a/library/core/src/ops/mod.rs b/library/core/src/ops/mod.rs index 1b07936ccd..139a8c0eec 100644 --- a/library/core/src/ops/mod.rs +++ b/library/core/src/ops/mod.rs @@ -147,6 +147,7 @@ mod function; mod generator; mod index; mod range; +#[cfg(bootstrap)] mod r#try; mod try_trait; mod unsize; @@ -183,13 +184,22 @@ pub use self::range::{Range, RangeFrom, RangeFull, RangeTo}; pub use self::range::{Bound, RangeBounds, RangeInclusive, RangeToInclusive}; #[unstable(feature = "try_trait", issue = "42327")] +#[cfg(bootstrap)] pub use self::r#try::Try; +#[unstable(feature = "try_trait_transition", reason = "for bootstrap", issue = "none")] +#[cfg(bootstrap)] +pub(crate) use self::r#try::Try as TryV1; + #[unstable(feature = "try_trait_v2", issue = "84277")] pub use self::try_trait::FromResidual; +#[unstable(feature = "try_trait_v2", issue = "84277")] +#[cfg(not(bootstrap))] +pub use self::try_trait::Try; + #[unstable(feature = "try_trait_transition", reason = "for bootstrap", issue = "none")] -pub use self::try_trait::Try as TryV2; +pub(crate) use self::try_trait::Try as TryV2; #[unstable(feature = "generator_trait", issue = "43122")] pub use self::generator::{Generator, GeneratorState}; diff --git a/library/core/src/ops/range.rs b/library/core/src/ops/range.rs index dbeb391213..bb948376bc 100644 --- a/library/core/src/ops/range.rs +++ b/library/core/src/ops/range.rs @@ -674,10 +674,10 @@ pub enum Bound { Unbounded, } -#[unstable(feature = "bound_as_ref", issue = "80996")] impl Bound { /// Converts from `&Bound` to `Bound<&T>`. #[inline] + #[unstable(feature = "bound_as_ref", issue = "80996")] pub fn as_ref(&self) -> Bound<&T> { match *self { Included(ref x) => Included(x), @@ -686,8 +686,9 @@ impl Bound { } } - /// Converts from `&mut Bound` to `Bound<&T>`. + /// Converts from `&mut Bound` to `Bound<&mut T>`. #[inline] + #[unstable(feature = "bound_as_ref", issue = "80996")] pub fn as_mut(&mut self) -> Bound<&mut T> { match *self { Included(ref mut x) => Included(x), @@ -695,6 +696,39 @@ impl Bound { Unbounded => Unbounded, } } + + /// Maps a `Bound` to a `Bound` by applying a function to the contained value (including + /// both `Included` and `Excluded`), returning a `Bound` of the same kind. + /// + /// # Examples + /// + /// ``` + /// #![feature(bound_map)] + /// use std::ops::Bound::*; + /// + /// let bound_string = Included("Hello, World!"); + /// + /// assert_eq!(bound_string.map(|s| s.len()), Included(13)); + /// ``` + /// + /// ``` + /// #![feature(bound_map)] + /// use std::ops::Bound; + /// use Bound::*; + /// + /// let unbounded_string: Bound = Unbounded; + /// + /// assert_eq!(unbounded_string.map(|s| s.len()), Unbounded); + /// ``` + #[inline] + #[unstable(feature = "bound_map", issue = "86026")] + pub fn map U>(self, f: F) -> Bound { + match self { + Unbounded => Unbounded, + Included(x) => Included(f(x)), + Excluded(x) => Excluded(f(x)), + } + } } impl Bound<&T> { diff --git a/library/core/src/ops/try.rs b/library/core/src/ops/try.rs index 3bede56997..9d659e78d3 100644 --- a/library/core/src/ops/try.rs +++ b/library/core/src/ops/try.rs @@ -25,11 +25,11 @@ ) )] #[doc(alias = "?")] -#[lang = "try"] +#[cfg_attr(bootstrap, lang = "try")] pub trait Try { /// The type of this value when viewed as successful. #[unstable(feature = "try_trait", issue = "42327")] - type Ok; + type Output; // This no longer follows its RFC, but is only used in bootstrap. /// The type of this value when viewed as failed. #[unstable(feature = "try_trait", issue = "42327")] type Error; @@ -43,19 +43,19 @@ pub trait Try { /// in the return type of the enclosing scope (which must itself implement /// `Try`). Specifically, the value `X::from_error(From::from(e))` /// is returned, where `X` is the return type of the enclosing function. - #[lang = "into_result"] + #[cfg_attr(bootstrap, lang = "into_result")] #[unstable(feature = "try_trait", issue = "42327")] - fn into_result(self) -> Result; + fn into_result(self) -> Result; /// Wrap an error value to construct the composite result. For example, /// `Result::Err(x)` and `Result::from_error(x)` are equivalent. - #[lang = "from_error"] + #[cfg_attr(bootstrap, lang = "from_error")] #[unstable(feature = "try_trait", issue = "42327")] fn from_error(v: Self::Error) -> Self; /// Wrap an OK value to construct the composite result. For example, /// `Result::Ok(x)` and `Result::from_ok(x)` are equivalent. - #[lang = "from_ok"] + #[cfg_attr(bootstrap, lang = "from_ok")] #[unstable(feature = "try_trait", issue = "42327")] - fn from_ok(v: Self::Ok) -> Self; + fn from_ok(v: Self::Output) -> Self; } diff --git a/library/core/src/ops/try_trait.rs b/library/core/src/ops/try_trait.rs index 0c819b000a..1d9bc45261 100644 --- a/library/core/src/ops/try_trait.rs +++ b/library/core/src/ops/try_trait.rs @@ -41,8 +41,7 @@ use crate::ops::ControlFlow; /// output type that we want: /// ``` /// # #![feature(try_trait_v2)] -/// # #![feature(try_trait_transition)] -/// # use std::ops::TryV2 as Try; +/// # use std::ops::Try; /// fn simple_try_fold_1>( /// iter: impl Iterator, /// mut accum: A, @@ -56,9 +55,8 @@ use crate::ops::ControlFlow; /// into the return type using [`Try::from_output`]: /// ``` /// # #![feature(try_trait_v2)] -/// # #![feature(try_trait_transition)] /// # #![feature(control_flow_enum)] -/// # use std::ops::{ControlFlow, TryV2 as Try}; +/// # use std::ops::{ControlFlow, Try}; /// fn simple_try_fold_2>( /// iter: impl Iterator, /// mut accum: A, @@ -81,9 +79,8 @@ use crate::ops::ControlFlow; /// recreated from their corresponding residual, so we'll just call it: /// ``` /// # #![feature(try_trait_v2)] -/// # #![feature(try_trait_transition)] /// # #![feature(control_flow_enum)] -/// # use std::ops::{ControlFlow, TryV2 as Try}; +/// # use std::ops::{ControlFlow, Try}; /// pub fn simple_try_fold_3>( /// iter: impl Iterator, /// mut accum: A, @@ -103,10 +100,9 @@ use crate::ops::ControlFlow; /// But this "call `branch`, then `match` on it, and `return` if it was a /// `Break`" is exactly what happens inside the `?` operator. So rather than /// do all this manually, we can just use `?` instead: -/// ```compile_fail (enable again once ? converts to the new trait) +/// ``` /// # #![feature(try_trait_v2)] -/// # #![feature(try_trait_transition)] -/// # use std::ops::TryV2 as Try; +/// # use std::ops::Try; /// fn simple_try_fold>( /// iter: impl Iterator, /// mut accum: A, @@ -119,6 +115,22 @@ use crate::ops::ControlFlow; /// } /// ``` #[unstable(feature = "try_trait_v2", issue = "84277")] +#[rustc_on_unimplemented( + on( + all(from_method = "from_output", from_desugaring = "TryBlock"), + message = "a `try` block must return `Result` or `Option` \ + (or another type that implements `{Try}`)", + label = "could not wrap the final value of the block as `{Self}` doesn't implement `Try`", + ), + on( + all(from_method = "branch", from_desugaring = "QuestionMark"), + message = "the `?` operator can only be applied to values \ + that implement `{Try}`", + label = "the `?` operator cannot be applied to type `{Self}`" + ) +)] +#[doc(alias = "?")] +#[cfg_attr(not(bootstrap), lang = "Try")] pub trait Try: FromResidual { /// The type of the value produced by `?` when *not* short-circuiting. #[unstable(feature = "try_trait_v2", issue = "84277")] @@ -159,8 +171,7 @@ pub trait Try: FromResidual { /// ``` /// #![feature(try_trait_v2)] /// #![feature(control_flow_enum)] - /// #![feature(try_trait_transition)] - /// use std::ops::TryV2 as Try; + /// use std::ops::Try; /// /// assert_eq!( as Try>::from_output(3), Ok(3)); /// assert_eq!( as Try>::from_output(4), Some(4)); @@ -178,6 +189,7 @@ pub trait Try: FromResidual { /// let r = std::iter::empty().try_fold(4, |_, ()| -> Option<_> { unreachable!() }); /// assert_eq!(r, Some(4)); /// ``` + #[cfg_attr(not(bootstrap), lang = "from_output")] #[unstable(feature = "try_trait_v2", issue = "84277")] fn from_output(output: Self::Output) -> Self; @@ -191,8 +203,7 @@ pub trait Try: FromResidual { /// ``` /// #![feature(try_trait_v2)] /// #![feature(control_flow_enum)] - /// #![feature(try_trait_transition)] - /// use std::ops::{ControlFlow, TryV2 as Try}; + /// use std::ops::{ControlFlow, Try}; /// /// assert_eq!(Ok::<_, String>(3).branch(), ControlFlow::Continue(3)); /// assert_eq!(Err::(3).branch(), ControlFlow::Break(Err(3))); @@ -206,15 +217,105 @@ pub trait Try: FromResidual { /// ControlFlow::Break(ControlFlow::Break(3)), /// ); /// ``` + #[cfg_attr(not(bootstrap), lang = "branch")] #[unstable(feature = "try_trait_v2", issue = "84277")] fn branch(self) -> ControlFlow; } -/// Used to specify which residuals can be converted into which [`Try`] types. +/// Used to specify which residuals can be converted into which [`crate::ops::Try`] types. /// /// Every `Try` type needs to be recreatable from its own associated /// `Residual` type, but can also have additional `FromResidual` implementations /// to support interconversion with other `Try` types. +#[rustc_on_unimplemented( + on( + all( + from_method = "from_residual", + from_desugaring = "QuestionMark", + _Self = "std::result::Result", + R = "std::option::Option" + ), + message = "the `?` operator can only be used on `Result`s, not `Option`s, \ + in {ItemContext} that returns `Result`", + label = "use `.ok_or(...)?` to provide an error compatible with `{Self}`", + enclosing_scope = "this function returns a `Result`" + ), + on( + all( + from_method = "from_residual", + from_desugaring = "QuestionMark", + _Self = "std::result::Result", + ), + // There's a special error message in the trait selection code for + // `From` in `?`, so this is not shown for result-in-result errors, + // and thus it can be phrased more strongly than `ControlFlow`'s. + message = "the `?` operator can only be used on `Result`s \ + in {ItemContext} that returns `Result`", + label = "this `?` produces `{R}`, which is incompatible with `{Self}`", + enclosing_scope = "this function returns a `Result`" + ), + on( + all( + from_method = "from_residual", + from_desugaring = "QuestionMark", + _Self = "std::option::Option", + R = "std::result::Result", + ), + message = "the `?` operator can only be used on `Option`s, not `Result`s, \ + in {ItemContext} that returns `Option`", + label = "use `.ok()?` if you want to discard the `{R}` error information", + enclosing_scope = "this function returns an `Option`" + ), + on( + all( + from_method = "from_residual", + from_desugaring = "QuestionMark", + _Self = "std::option::Option", + ), + // `Option`-in-`Option` always works, as there's only one possible + // residual, so this can also be phrased strongly. + message = "the `?` operator can only be used on `Option`s \ + in {ItemContext} that returns `Option`", + label = "this `?` produces `{R}`, which is incompatible with `{Self}`", + enclosing_scope = "this function returns an `Option`" + ), + on( + all( + from_method = "from_residual", + from_desugaring = "QuestionMark", + _Self = "std::ops::ControlFlow", + R = "std::ops::ControlFlow", + ), + message = "the `?` operator in {ItemContext} that returns `ControlFlow` \ + can only be used on other `ControlFlow`s (with the same Break type)", + label = "this `?` produces `{R}`, which is incompatible with `{Self}`", + enclosing_scope = "this function returns a `ControlFlow`", + note = "unlike `Result`, there's no `From`-conversion performed for `ControlFlow`" + ), + on( + all( + from_method = "from_residual", + from_desugaring = "QuestionMark", + _Self = "std::ops::ControlFlow", + // `R` is not a `ControlFlow`, as that case was matched previously + ), + message = "the `?` operator can only be used on `ControlFlow`s \ + in {ItemContext} that returns `ControlFlow`", + label = "this `?` produces `{R}`, which is incompatible with `{Self}`", + enclosing_scope = "this function returns a `ControlFlow`", + ), + on( + all( + from_method = "from_residual", + from_desugaring = "QuestionMark" + ), + message = "the `?` operator can only be used in {ItemContext} \ + that returns `Result` or `Option` \ + (or another type that implements `{FromResidual}`)", + label = "cannot use the `?` operator in {ItemContext} that returns `{Self}`", + enclosing_scope = "this function should return `Result` or `Option` to accept `?`" + ), +)] #[unstable(feature = "try_trait_v2", issue = "84277")] pub trait FromResidual::Residual> { /// Constructs the type from a compatible `Residual` type. @@ -238,6 +339,7 @@ pub trait FromResidual::Residual> { /// ControlFlow::Break(5), /// ); /// ``` + #[cfg_attr(not(bootstrap), lang = "from_residual")] #[unstable(feature = "try_trait_v2", issue = "84277")] fn from_residual(residual: R) -> Self; } diff --git a/library/core/src/option.rs b/library/core/src/option.rs index 04551dded8..4e7afca6a4 100644 --- a/library/core/src/option.rs +++ b/library/core/src/option.rs @@ -83,6 +83,8 @@ //! * [`ptr::NonNull`] //! * `#[repr(transparent)]` struct around one of the types in this list. //! +//! This is called the "null pointer optimization" or NPO. +//! //! It is further guaranteed that, for the cases above, one can //! [`mem::transmute`] from all valid values of `T` to `Option` and //! from `Some::(_)` to `T` (but transmuting `None::` to `T` @@ -209,7 +211,7 @@ impl Option { /// assert_eq!(x.is_none(), true); /// ``` #[must_use = "if you intended to assert that this doesn't have a value, consider \ - `.and_then(|| panic!(\"`Option` had a value when expected `None`\"))` instead"] + `.and_then(|_| panic!(\"`Option` had a value when expected `None`\"))` instead"] #[inline] #[rustc_const_stable(feature = "const_option", since = "1.48.0")] #[stable(feature = "rust1", since = "1.0.0")] @@ -1641,11 +1643,13 @@ impl> FromIterator> for Option { #[rustc_diagnostic_item = "none_error"] #[unstable(feature = "try_trait", issue = "42327")] #[derive(Clone, Copy, PartialEq, PartialOrd, Eq, Ord, Debug, Hash)] +#[cfg(bootstrap)] pub struct NoneError; #[unstable(feature = "try_trait", issue = "42327")] -impl ops::Try for Option { - type Ok = T; +#[cfg(bootstrap)] +impl ops::TryV1 for Option { + type Output = T; type Error = NoneError; #[inline] diff --git a/library/core/src/panicking.rs b/library/core/src/panicking.rs index 35d9b2c5d2..3e3e96fcd7 100644 --- a/library/core/src/panicking.rs +++ b/library/core/src/panicking.rs @@ -97,6 +97,7 @@ pub fn panic_fmt(fmt: fmt::Arguments<'_>) -> ! { pub enum AssertKind { Eq, Ne, + Match, } /// Internal function for `assert_eq!` and `assert_ne!` macros @@ -113,32 +114,54 @@ where T: fmt::Debug + ?Sized, U: fmt::Debug + ?Sized, { - #[track_caller] - fn inner( - kind: AssertKind, - left: &dyn fmt::Debug, - right: &dyn fmt::Debug, - args: Option>, - ) -> ! { - let op = match kind { - AssertKind::Eq => "==", - AssertKind::Ne => "!=", - }; - - match args { - Some(args) => panic!( - r#"assertion failed: `(left {} right)` + assert_failed_inner(kind, &left, &right, args) +} + +/// Internal function for `assert_match!` +#[cold] +#[track_caller] +#[doc(hidden)] +pub fn assert_matches_failed( + left: &T, + right: &str, + args: Option>, +) -> ! { + // Use the Display implementation to display the pattern. + struct Pattern<'a>(&'a str); + impl fmt::Debug for Pattern<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Display::fmt(self.0, f) + } + } + assert_failed_inner(AssertKind::Match, &left, &Pattern(right), args); +} + +/// Non-generic version of the above functions, to avoid code bloat. +#[track_caller] +fn assert_failed_inner( + kind: AssertKind, + left: &dyn fmt::Debug, + right: &dyn fmt::Debug, + args: Option>, +) -> ! { + let op = match kind { + AssertKind::Eq => "==", + AssertKind::Ne => "!=", + AssertKind::Match => "matches", + }; + + match args { + Some(args) => panic!( + r#"assertion failed: `(left {} right)` left: `{:?}`, right: `{:?}`: {}"#, - op, left, right, args - ), - None => panic!( - r#"assertion failed: `(left {} right)` + op, left, right, args + ), + None => panic!( + r#"assertion failed: `(left {} right)` left: `{:?}`, right: `{:?}`"#, - op, left, right, - ), - } + op, left, right, + ), } - inner(kind, &left, &right, args) } diff --git a/library/core/src/prelude/mod.rs b/library/core/src/prelude/mod.rs index 8f57db4949..79753c1fb6 100644 --- a/library/core/src/prelude/mod.rs +++ b/library/core/src/prelude/mod.rs @@ -11,9 +11,9 @@ pub mod v1; /// The 2015 version of the core prelude. /// /// See the [module-level documentation](self) for more. -#[unstable(feature = "prelude_2015", issue = "none")] +#[unstable(feature = "prelude_2015", issue = "85684")] pub mod rust_2015 { - #[unstable(feature = "prelude_2015", issue = "none")] + #[unstable(feature = "prelude_2015", issue = "85684")] #[doc(no_inline)] pub use super::v1::*; } @@ -21,9 +21,9 @@ pub mod rust_2015 { /// The 2018 version of the core prelude. /// /// See the [module-level documentation](self) for more. -#[unstable(feature = "prelude_2018", issue = "none")] +#[unstable(feature = "prelude_2018", issue = "85684")] pub mod rust_2018 { - #[unstable(feature = "prelude_2018", issue = "none")] + #[unstable(feature = "prelude_2018", issue = "85684")] #[doc(no_inline)] pub use super::v1::*; } @@ -31,11 +31,17 @@ pub mod rust_2018 { /// The 2021 version of the core prelude. /// /// See the [module-level documentation](self) for more. -#[unstable(feature = "prelude_2021", issue = "none")] +#[unstable(feature = "prelude_2021", issue = "85684")] pub mod rust_2021 { - #[unstable(feature = "prelude_2021", issue = "none")] + #[unstable(feature = "prelude_2021", issue = "85684")] #[doc(no_inline)] pub use super::v1::*; - // FIXME: Add more things. + #[unstable(feature = "prelude_2021", issue = "85684")] + #[doc(no_inline)] + pub use crate::iter::FromIterator; + + #[unstable(feature = "prelude_2021", issue = "85684")] + #[doc(no_inline)] + pub use crate::convert::{TryFrom, TryInto}; } diff --git a/library/core/src/ptr/const_ptr.rs b/library/core/src/ptr/const_ptr.rs index b9b2ba9ae6..8348afb2a5 100644 --- a/library/core/src/ptr/const_ptr.rs +++ b/library/core/src/ptr/const_ptr.rs @@ -66,7 +66,7 @@ impl *const T { /// /// # Safety /// - /// When calling this method, you have to ensure that *either* the pointer is NULL *or* + /// When calling this method, you have to ensure that *either* the pointer is null *or* /// all of the following is true: /// /// * The pointer must be properly aligned. @@ -130,7 +130,7 @@ impl *const T { /// /// # Safety /// - /// When calling this method, you have to ensure that *either* the pointer is NULL *or* + /// When calling this method, you have to ensure that *either* the pointer is null *or* /// all of the following is true: /// /// * The pointer must be properly aligned. @@ -974,7 +974,7 @@ impl *const [T] { /// /// # Safety /// - /// When calling this method, you have to ensure that *either* the pointer is NULL *or* + /// When calling this method, you have to ensure that *either* the pointer is null *or* /// all of the following is true: /// /// * The pointer must be [valid] for reads for `ptr.len() * mem::size_of::()` many bytes, diff --git a/library/core/src/ptr/mod.rs b/library/core/src/ptr/mod.rs index 28044e0651..6a6cee0911 100644 --- a/library/core/src/ptr/mod.rs +++ b/library/core/src/ptr/mod.rs @@ -149,7 +149,7 @@ mod mut_ptr; /// again. [`write()`] can be used to overwrite data without causing it to be /// dropped. /// -/// Note that even if `T` has size `0`, the pointer must be non-NULL and properly aligned. +/// Note that even if `T` has size `0`, the pointer must be non-null and properly aligned. /// /// [valid]: self#safety /// @@ -315,7 +315,7 @@ pub const fn slice_from_raw_parts_mut(data: *mut T, len: usize) -> *mut [T] { /// /// * Both `x` and `y` must be properly aligned. /// -/// Note that even if `T` has size `0`, the pointers must be non-NULL and properly aligned. +/// Note that even if `T` has size `0`, the pointers must be non-null and properly aligned. /// /// [valid]: self#safety /// @@ -342,10 +342,12 @@ pub const fn slice_from_raw_parts_mut(data: *mut T, len: usize) -> *mut [T] { /// ``` /// use std::ptr; /// -/// let mut array = [0, 1, 2, 3]; +/// let mut array: [i32; 4] = [0, 1, 2, 3]; +/// +/// let array_ptr: *mut i32 = array.as_mut_ptr(); /// -/// let x = array[0..].as_mut_ptr() as *mut [u32; 3]; // this is `array[0..3]` -/// let y = array[1..].as_mut_ptr() as *mut [u32; 3]; // this is `array[1..4]` +/// let x = array_ptr as *mut [i32; 3]; // this is `array[0..3]` +/// let y = unsafe { array_ptr.add(1) } as *mut [i32; 3]; // this is `array[1..4]` /// /// unsafe { /// ptr::swap(x, y); @@ -394,7 +396,7 @@ pub const unsafe fn swap(x: *mut T, y: *mut T) { /// beginning at `y` with the same size. /// /// Note that even if the effectively copied size (`count * size_of::()`) is `0`, -/// the pointers must be non-NULL and properly aligned. +/// the pointers must be non-null and properly aligned. /// /// [valid]: self#safety /// @@ -539,7 +541,7 @@ const unsafe fn swap_nonoverlapping_bytes(x: *mut u8, y: *mut u8, len: usize) { /// /// * `dst` must point to a properly initialized value of type `T`. /// -/// Note that even if `T` has size `0`, the pointer must be non-NULL and properly aligned. +/// Note that even if `T` has size `0`, the pointer must be non-null and properly aligned. /// /// [valid]: self#safety /// @@ -586,7 +588,7 @@ pub unsafe fn replace(dst: *mut T, mut src: T) -> T { /// /// * `src` must point to a properly initialized value of type `T`. /// -/// Note that even if `T` has size `0`, the pointer must be non-NULL and properly aligned. +/// Note that even if `T` has size `0`, the pointer must be non-null and properly aligned. /// /// # Examples /// @@ -711,16 +713,13 @@ pub const unsafe fn read(src: *const T) -> T { /// whether `T` is [`Copy`]. If `T` is not [`Copy`], using both the returned /// value and the value at `*src` can [violate memory safety][read-ownership]. /// -/// Note that even if `T` has size `0`, the pointer must be non-NULL. +/// Note that even if `T` has size `0`, the pointer must be non-null. /// /// [read-ownership]: read#ownership-of-the-returned-value /// [valid]: self#safety /// /// ## On `packed` structs /// -/// It is currently impossible to create raw pointers to unaligned fields -/// of a packed struct. -/// /// Attempting to create a raw pointer to an `unaligned` struct field with /// an expression such as `&packed.unaligned as *const FieldType` creates an /// intermediate unaligned reference before converting that to a raw pointer. @@ -729,9 +728,13 @@ pub const unsafe fn read(src: *const T) -> T { /// As a result, using `&packed.unaligned as *const FieldType` causes immediate /// *undefined behavior* in your program. /// +/// Instead you must use the [`ptr::addr_of!`](addr_of) macro to +/// create the pointer. You may use that returned pointer together with this +/// function. +/// /// An example of what not to do and how this relates to `read_unaligned` is: /// -/// ```no_run +/// ``` /// #[repr(packed, C)] /// struct Packed { /// _padding: u8, @@ -743,24 +746,15 @@ pub const unsafe fn read(src: *const T) -> T { /// unaligned: 0x01020304, /// }; /// -/// #[allow(unaligned_references)] -/// let v = unsafe { -/// // Here we attempt to take the address of a 32-bit integer which is not aligned. -/// let unaligned = -/// // A temporary unaligned reference is created here which results in -/// // undefined behavior regardless of whether the reference is used or not. -/// &packed.unaligned -/// // Casting to a raw pointer doesn't help; the mistake already happened. -/// as *const u32; -/// -/// let v = std::ptr::read_unaligned(unaligned); +/// // Take the address of a 32-bit integer which is not aligned. +/// // In contrast to `&packed.unaligned as *const _`, this has no undefined behavior. +/// let unaligned = std::ptr::addr_of!(packed.unaligned); /// -/// v -/// }; +/// let v = unsafe { std::ptr::read_unaligned(unaligned) }; +/// assert_eq!(v, 0x01020304); /// ``` /// /// Accessing unaligned fields directly with e.g. `packed.unaligned` is safe however. -// FIXME: Update docs based on outcome of RFC #2582 and friends. /// /// # Examples /// @@ -816,7 +810,7 @@ pub const unsafe fn read_unaligned(src: *const T) -> T { /// * `dst` must be properly aligned. Use [`write_unaligned`] if this is not the /// case. /// -/// Note that even if `T` has size `0`, the pointer must be non-NULL and properly aligned. +/// Note that even if `T` has size `0`, the pointer must be non-null and properly aligned. /// /// [valid]: self#safety /// @@ -912,15 +906,12 @@ pub unsafe fn write(dst: *mut T, src: T) { /// /// * `dst` must be [valid] for writes. /// -/// Note that even if `T` has size `0`, the pointer must be non-NULL. +/// Note that even if `T` has size `0`, the pointer must be non-null. /// /// [valid]: self#safety /// /// ## On `packed` structs /// -/// It is currently impossible to create raw pointers to unaligned fields -/// of a packed struct. -/// /// Attempting to create a raw pointer to an `unaligned` struct field with /// an expression such as `&packed.unaligned as *const FieldType` creates an /// intermediate unaligned reference before converting that to a raw pointer. @@ -929,36 +920,32 @@ pub unsafe fn write(dst: *mut T, src: T) { /// As a result, using `&packed.unaligned as *const FieldType` causes immediate /// *undefined behavior* in your program. /// -/// An example of what not to do and how this relates to `write_unaligned` is: +/// Instead you must use the [`ptr::addr_of_mut!`](addr_of_mut) +/// macro to create the pointer. You may use that returned pointer together with +/// this function. +/// +/// An example of how to do it and how this relates to `write_unaligned` is: /// -/// ```no_run +/// ``` /// #[repr(packed, C)] /// struct Packed { /// _padding: u8, /// unaligned: u32, /// } /// -/// let v = 0x01020304; /// let mut packed: Packed = unsafe { std::mem::zeroed() }; /// -/// #[allow(unaligned_references)] -/// let v = unsafe { -/// // Here we attempt to take the address of a 32-bit integer which is not aligned. -/// let unaligned = -/// // A temporary unaligned reference is created here which results in -/// // undefined behavior regardless of whether the reference is used or not. -/// &mut packed.unaligned -/// // Casting to a raw pointer doesn't help; the mistake already happened. -/// as *mut u32; +/// // Take the address of a 32-bit integer which is not aligned. +/// // In contrast to `&packed.unaligned as *mut _`, this has no undefined behavior. +/// let unaligned = std::ptr::addr_of_mut!(packed.unaligned); /// -/// std::ptr::write_unaligned(unaligned, v); +/// unsafe { std::ptr::write_unaligned(unaligned, 42) }; /// -/// v -/// }; +/// assert_eq!({packed.unaligned}, 42); // `{...}` forces copying the field instead of creating a reference. /// ``` /// -/// Accessing unaligned fields directly with e.g. `packed.unaligned` is safe however. -// FIXME: Update docs based on outcome of RFC #2582 and friends. +/// Accessing unaligned fields directly with e.g. `packed.unaligned` is safe however +/// (as can be seen in the `assert_eq!` above). /// /// # Examples /// @@ -1026,7 +1013,7 @@ pub const unsafe fn write_unaligned(dst: *mut T, src: T) { /// However, storing non-[`Copy`] types in volatile memory is almost certainly /// incorrect. /// -/// Note that even if `T` has size `0`, the pointer must be non-NULL and properly aligned. +/// Note that even if `T` has size `0`, the pointer must be non-null and properly aligned. /// /// [valid]: self#safety /// [read-ownership]: read#ownership-of-the-returned-value @@ -1096,7 +1083,7 @@ pub unsafe fn read_volatile(src: *const T) -> T { /// /// * `dst` must be properly aligned. /// -/// Note that even if `T` has size `0`, the pointer must be non-NULL and properly aligned. +/// Note that even if `T` has size `0`, the pointer must be non-null and properly aligned. /// /// [valid]: self#safety /// @@ -1498,7 +1485,7 @@ fnptr_impls_args! { A, B, C, D, E, F, G, H, I, J, K, L } /// /// Note, however, that the `expr` in `addr_of!(expr)` is still subject to all /// the usual rules. In particular, `addr_of!(*ptr::null())` is Undefined -/// Behavior because it dereferences a NULL pointer. +/// Behavior because it dereferences a null pointer. /// /// # Example /// @@ -1538,7 +1525,7 @@ pub macro addr_of($place:expr) { /// /// Note, however, that the `expr` in `addr_of_mut!(expr)` is still subject to all /// the usual rules. In particular, `addr_of_mut!(*ptr::null_mut())` is Undefined -/// Behavior because it dereferences a NULL pointer. +/// Behavior because it dereferences a null pointer. /// /// # Examples /// diff --git a/library/core/src/ptr/mut_ptr.rs b/library/core/src/ptr/mut_ptr.rs index 16f6efc374..a642404154 100644 --- a/library/core/src/ptr/mut_ptr.rs +++ b/library/core/src/ptr/mut_ptr.rs @@ -42,7 +42,7 @@ impl *mut T { /// Casts to a pointer of another type. #[stable(feature = "ptr_cast", since = "1.38.0")] #[rustc_const_stable(feature = "const_ptr_cast", since = "1.38.0")] - #[inline] + #[inline(always)] pub const fn cast(self) -> *mut U { self as _ } @@ -68,7 +68,7 @@ impl *mut T { /// /// # Safety /// - /// When calling this method, you have to ensure that *either* the pointer is NULL *or* + /// When calling this method, you have to ensure that *either* the pointer is null *or* /// all of the following is true: /// /// * The pointer must be properly aligned. @@ -135,7 +135,7 @@ impl *mut T { /// /// # Safety /// - /// When calling this method, you have to ensure that *either* the pointer is NULL *or* + /// When calling this method, you have to ensure that *either* the pointer is null *or* /// all of the following is true: /// /// * The pointer must be properly aligned. @@ -314,7 +314,7 @@ impl *mut T { /// /// # Safety /// - /// When calling this method, you have to ensure that *either* the pointer is NULL *or* + /// When calling this method, you have to ensure that *either* the pointer is null *or* /// all of the following is true: /// /// * The pointer must be properly aligned. @@ -380,7 +380,7 @@ impl *mut T { /// /// # Safety /// - /// When calling this method, you have to ensure that *either* the pointer is NULL *or* + /// When calling this method, you have to ensure that *either* the pointer is null *or* /// all of the following is true: /// /// * The pointer must be properly aligned. @@ -551,7 +551,7 @@ impl *mut T { /// ``` #[stable(feature = "ptr_offset_from", since = "1.47.0")] #[rustc_const_unstable(feature = "const_ptr_offset_from", issue = "41079")] - #[inline] + #[inline(always)] pub const unsafe fn offset_from(self, origin: *const T) -> isize where T: Sized, @@ -859,7 +859,7 @@ impl *mut T { /// [`ptr::read`]: crate::ptr::read() #[stable(feature = "pointer_methods", since = "1.26.0")] #[rustc_const_unstable(feature = "const_ptr_read", issue = "80377")] - #[inline] + #[inline(always)] pub const unsafe fn read(self) -> T where T: Sized, @@ -879,7 +879,7 @@ impl *mut T { /// /// [`ptr::read_volatile`]: crate::ptr::read_volatile() #[stable(feature = "pointer_methods", since = "1.26.0")] - #[inline] + #[inline(always)] pub unsafe fn read_volatile(self) -> T where T: Sized, @@ -898,7 +898,7 @@ impl *mut T { /// [`ptr::read_unaligned`]: crate::ptr::read_unaligned() #[stable(feature = "pointer_methods", since = "1.26.0")] #[rustc_const_unstable(feature = "const_ptr_read", issue = "80377")] - #[inline] + #[inline(always)] pub const unsafe fn read_unaligned(self) -> T where T: Sized, @@ -917,7 +917,7 @@ impl *mut T { /// [`ptr::copy`]: crate::ptr::copy() #[rustc_const_unstable(feature = "const_intrinsic_copy", issue = "80697")] #[stable(feature = "pointer_methods", since = "1.26.0")] - #[inline] + #[inline(always)] pub const unsafe fn copy_to(self, dest: *mut T, count: usize) where T: Sized, @@ -936,7 +936,7 @@ impl *mut T { /// [`ptr::copy_nonoverlapping`]: crate::ptr::copy_nonoverlapping() #[rustc_const_unstable(feature = "const_intrinsic_copy", issue = "80697")] #[stable(feature = "pointer_methods", since = "1.26.0")] - #[inline] + #[inline(always)] pub const unsafe fn copy_to_nonoverlapping(self, dest: *mut T, count: usize) where T: Sized, @@ -955,7 +955,7 @@ impl *mut T { /// [`ptr::copy`]: crate::ptr::copy() #[rustc_const_unstable(feature = "const_intrinsic_copy", issue = "80697")] #[stable(feature = "pointer_methods", since = "1.26.0")] - #[inline] + #[inline(always)] pub const unsafe fn copy_from(self, src: *const T, count: usize) where T: Sized, @@ -974,7 +974,7 @@ impl *mut T { /// [`ptr::copy_nonoverlapping`]: crate::ptr::copy_nonoverlapping() #[rustc_const_unstable(feature = "const_intrinsic_copy", issue = "80697")] #[stable(feature = "pointer_methods", since = "1.26.0")] - #[inline] + #[inline(always)] pub const unsafe fn copy_from_nonoverlapping(self, src: *const T, count: usize) where T: Sized, @@ -989,7 +989,7 @@ impl *mut T { /// /// [`ptr::drop_in_place`]: crate::ptr::drop_in_place() #[stable(feature = "pointer_methods", since = "1.26.0")] - #[inline] + #[inline(always)] pub unsafe fn drop_in_place(self) { // SAFETY: the caller must uphold the safety contract for `drop_in_place`. unsafe { drop_in_place(self) } @@ -1002,7 +1002,7 @@ impl *mut T { /// /// [`ptr::write`]: crate::ptr::write() #[stable(feature = "pointer_methods", since = "1.26.0")] - #[inline] + #[inline(always)] pub unsafe fn write(self, val: T) where T: Sized, @@ -1018,7 +1018,7 @@ impl *mut T { /// /// [`ptr::write_bytes`]: crate::ptr::write_bytes() #[stable(feature = "pointer_methods", since = "1.26.0")] - #[inline] + #[inline(always)] pub unsafe fn write_bytes(self, val: u8, count: usize) where T: Sized, @@ -1038,7 +1038,7 @@ impl *mut T { /// /// [`ptr::write_volatile`]: crate::ptr::write_volatile() #[stable(feature = "pointer_methods", since = "1.26.0")] - #[inline] + #[inline(always)] pub unsafe fn write_volatile(self, val: T) where T: Sized, @@ -1057,7 +1057,7 @@ impl *mut T { /// [`ptr::write_unaligned`]: crate::ptr::write_unaligned() #[stable(feature = "pointer_methods", since = "1.26.0")] #[rustc_const_unstable(feature = "const_ptr_write", issue = "none")] - #[inline] + #[inline(always)] pub const unsafe fn write_unaligned(self, val: T) where T: Sized, @@ -1073,7 +1073,7 @@ impl *mut T { /// /// [`ptr::replace`]: crate::ptr::replace() #[stable(feature = "pointer_methods", since = "1.26.0")] - #[inline] + #[inline(always)] pub unsafe fn replace(self, src: T) -> T where T: Sized, @@ -1090,7 +1090,7 @@ impl *mut T { /// /// [`ptr::swap`]: crate::ptr::swap() #[stable(feature = "pointer_methods", since = "1.26.0")] - #[inline] + #[inline(always)] pub unsafe fn swap(self, with: *mut T) where T: Sized, @@ -1169,7 +1169,7 @@ impl *mut [T] { /// let slice: *mut [i8] = ptr::slice_from_raw_parts_mut(ptr::null_mut(), 3); /// assert_eq!(slice.len(), 3); /// ``` - #[inline] + #[inline(always)] #[unstable(feature = "slice_ptr_len", issue = "71146")] #[rustc_const_unstable(feature = "const_slice_ptr_len", issue = "71146")] pub const fn len(self) -> usize { @@ -1189,7 +1189,7 @@ impl *mut [T] { /// let slice: *mut [i8] = ptr::slice_from_raw_parts_mut(ptr::null_mut(), 3); /// assert_eq!(slice.as_mut_ptr(), 0 as *mut i8); /// ``` - #[inline] + #[inline(always)] #[unstable(feature = "slice_ptr_get", issue = "74265")] #[rustc_const_unstable(feature = "slice_ptr_get", issue = "74265")] pub const fn as_mut_ptr(self) -> *mut T { @@ -1216,7 +1216,7 @@ impl *mut [T] { /// } /// ``` #[unstable(feature = "slice_ptr_get", issue = "74265")] - #[inline] + #[inline(always)] pub unsafe fn get_unchecked_mut(self, index: I) -> *mut I::Output where I: SliceIndex<[T]>, @@ -1236,7 +1236,7 @@ impl *mut [T] { /// /// # Safety /// - /// When calling this method, you have to ensure that *either* the pointer is NULL *or* + /// When calling this method, you have to ensure that *either* the pointer is null *or* /// all of the following is true: /// /// * The pointer must be [valid] for reads for `ptr.len() * mem::size_of::()` many bytes, @@ -1287,7 +1287,7 @@ impl *mut [T] { /// /// # Safety /// - /// When calling this method, you have to ensure that *either* the pointer is NULL *or* + /// When calling this method, you have to ensure that *either* the pointer is null *or* /// all of the following is true: /// /// * The pointer must be [valid] for reads and writes for `ptr.len() * mem::size_of::()` @@ -1331,7 +1331,7 @@ impl *mut [T] { // Equality for pointers #[stable(feature = "rust1", since = "1.0.0")] impl PartialEq for *mut T { - #[inline] + #[inline(always)] fn eq(&self, other: &*mut T) -> bool { *self == *other } @@ -1356,27 +1356,27 @@ impl Ord for *mut T { #[stable(feature = "rust1", since = "1.0.0")] impl PartialOrd for *mut T { - #[inline] + #[inline(always)] fn partial_cmp(&self, other: &*mut T) -> Option { Some(self.cmp(other)) } - #[inline] + #[inline(always)] fn lt(&self, other: &*mut T) -> bool { *self < *other } - #[inline] + #[inline(always)] fn le(&self, other: &*mut T) -> bool { *self <= *other } - #[inline] + #[inline(always)] fn gt(&self, other: &*mut T) -> bool { *self > *other } - #[inline] + #[inline(always)] fn ge(&self, other: &*mut T) -> bool { *self >= *other } diff --git a/library/core/src/ptr/non_null.rs b/library/core/src/ptr/non_null.rs index e525f61604..1c65518af0 100644 --- a/library/core/src/ptr/non_null.rs +++ b/library/core/src/ptr/non_null.rs @@ -519,7 +519,7 @@ impl NonNull<[T]> { I: SliceIndex<[T]>, { // SAFETY: the caller ensures that `self` is dereferencable and `index` in-bounds. - // As a consequence, the resulting pointer cannot be NULL. + // As a consequence, the resulting pointer cannot be null. unsafe { NonNull::new_unchecked(self.as_ptr().get_unchecked_mut(index)) } } } diff --git a/library/core/src/result.rs b/library/core/src/result.rs index e0071f806a..babd0a0b55 100644 --- a/library/core/src/result.rs +++ b/library/core/src/result.rs @@ -1627,8 +1627,9 @@ impl> FromIterator> for Result { } #[unstable(feature = "try_trait", issue = "42327")] -impl ops::Try for Result { - type Ok = T; +#[cfg(bootstrap)] +impl ops::TryV1 for Result { + type Output = T; type Error = E; #[inline] diff --git a/library/core/src/slice/ascii.rs b/library/core/src/slice/ascii.rs index 906dcb1e8b..b92ab7e347 100644 --- a/library/core/src/slice/ascii.rs +++ b/library/core/src/slice/ascii.rs @@ -110,7 +110,7 @@ impl<'a> iter::Iterator for EscapeAscii<'a> { fn try_fold(&mut self, init: Acc, fold: Fold) -> R where Fold: FnMut(Acc, Self::Item) -> R, - R: ops::Try, + R: ops::Try, { self.inner.try_fold(init, fold) } diff --git a/library/core/src/slice/mod.rs b/library/core/src/slice/mod.rs index ec28cdd1ba..0e5c5ee726 100644 --- a/library/core/src/slice/mod.rs +++ b/library/core/src/slice/mod.rs @@ -1948,8 +1948,9 @@ impl [T] { /// assert!(!v.contains(&50)); /// ``` /// - /// If you do not have an `&T`, but just an `&U` such that `T: Borrow` - /// (e.g. `String: Borrow`), you can use `iter().any`: + /// If you do not have a `&T`, but some other value that you can compare + /// with one (for example, `String` implements `PartialEq`), you can + /// use `iter().any`: /// /// ``` /// let v = [String::from("hello"), String::from("world")]; // slice of `String` @@ -2099,9 +2100,11 @@ impl [T] { /// /// If the value is found then [`Result::Ok`] is returned, containing the /// index of the matching element. If there are multiple matches, then any - /// one of the matches could be returned. If the value is not found then - /// [`Result::Err`] is returned, containing the index where a matching - /// element could be inserted while maintaining sorted order. + /// one of the matches could be returned. The index is chosen + /// deterministically, but is subject to change in future versions of Rust. + /// If the value is not found then [`Result::Err`] is returned, containing + /// the index where a matching element could be inserted while maintaining + /// sorted order. /// /// See also [`binary_search_by`], [`binary_search_by_key`], and [`partition_point`]. /// @@ -2152,9 +2155,11 @@ impl [T] { /// /// If the value is found then [`Result::Ok`] is returned, containing the /// index of the matching element. If there are multiple matches, then any - /// one of the matches could be returned. If the value is not found then - /// [`Result::Err`] is returned, containing the index where a matching - /// element could be inserted while maintaining sorted order. + /// one of the matches could be returned. The index is chosen + /// deterministically, but is subject to change in future versions of Rust. + /// If the value is not found then [`Result::Err`] is returned, containing + /// the index where a matching element could be inserted while maintaining + /// sorted order. /// /// See also [`binary_search`], [`binary_search_by_key`], and [`partition_point`]. /// @@ -2223,9 +2228,11 @@ impl [T] { /// /// If the value is found then [`Result::Ok`] is returned, containing the /// index of the matching element. If there are multiple matches, then any - /// one of the matches could be returned. If the value is not found then - /// [`Result::Err`] is returned, containing the index where a matching - /// element could be inserted while maintaining sorted order. + /// one of the matches could be returned. The index is chosen + /// deterministically, but is subject to change in future versions of Rust. + /// If the value is not found then [`Result::Err`] is returned, containing + /// the index where a matching element could be inserted while maintaining + /// sorted order. /// /// See also [`binary_search`], [`binary_search_by`], and [`partition_point`]. /// @@ -3095,7 +3102,11 @@ impl [T] { // SAFETY: the conditions for `ptr::copy` have all been checked above, // as have those for `ptr::add`. unsafe { - ptr::copy(self.as_ptr().add(src_start), self.as_mut_ptr().add(dest), count); + // Derive both `src_ptr` and `dest_ptr` from the same loan + let ptr = self.as_mut_ptr(); + let src_ptr = ptr.add(src_start); + let dest_ptr = ptr.add(dest); + ptr::copy(src_ptr, dest_ptr, count); } } diff --git a/library/core/src/str/iter.rs b/library/core/src/str/iter.rs index 724137aba9..6ec6b70b57 100644 --- a/library/core/src/str/iter.rs +++ b/library/core/src/str/iter.rs @@ -1467,7 +1467,7 @@ macro_rules! escape_types_impls { #[inline] fn try_fold(&mut self, init: Acc, fold: Fold) -> R where - Self: Sized, Fold: FnMut(Acc, Self::Item) -> R, R: Try + Self: Sized, Fold: FnMut(Acc, Self::Item) -> R, R: Try { self.inner.try_fold(init, fold) } diff --git a/library/core/src/str/mod.rs b/library/core/src/str/mod.rs index 95dd54976b..065acd3f38 100644 --- a/library/core/src/str/mod.rs +++ b/library/core/src/str/mod.rs @@ -192,14 +192,26 @@ impl str { #[stable(feature = "is_char_boundary", since = "1.9.0")] #[inline] pub fn is_char_boundary(&self, index: usize) -> bool { - // 0 and len are always ok. + // 0 is always ok. // Test for 0 explicitly so that it can optimize out the check // easily and skip reading string data for that case. - if index == 0 || index == self.len() { + // Note that optimizing `self.get(..index)` relies on this. + if index == 0 { return true; } + match self.as_bytes().get(index) { - None => false, + // For `None` we have two options: + // + // - index == self.len() + // Empty strings are valid, so return true + // - index > self.len() + // In this case return false + // + // The check is placed exactly here, because it improves generated + // code on higher opt-levels. See PR #84751 for more details. + None => index == self.len(), + // This is bit magic equivalent to: b < 128 || b >= 192 Some(&b) => (b as i8) >= -0x40, } diff --git a/library/core/src/task/poll.rs b/library/core/src/task/poll.rs index 2765c21a46..9cf89623d8 100644 --- a/library/core/src/task/poll.rs +++ b/library/core/src/task/poll.rs @@ -1,7 +1,7 @@ #![stable(feature = "futures_api", since = "1.36.0")] use crate::convert; -use crate::ops::{self, ControlFlow, Try}; +use crate::ops::{self, ControlFlow}; use crate::result::Result; /// Indicates whether a value is available or if the current task has been @@ -129,12 +129,13 @@ impl From for Poll { } #[stable(feature = "futures_api", since = "1.36.0")] -impl Try for Poll> { - type Ok = Poll; +#[cfg(bootstrap)] +impl ops::TryV1 for Poll> { + type Output = Poll; type Error = E; #[inline] - fn into_result(self) -> Result { + fn into_result(self) -> Result { match self { Poll::Ready(Ok(x)) => Ok(Poll::Ready(x)), Poll::Ready(Err(e)) => Err(e), @@ -148,7 +149,7 @@ impl Try for Poll> { } #[inline] - fn from_ok(x: Self::Ok) -> Self { + fn from_ok(x: Self::Output) -> Self { x.map(Ok) } } @@ -184,12 +185,13 @@ impl> ops::FromResidual> for Pol } #[stable(feature = "futures_api", since = "1.36.0")] -impl Try for Poll>> { - type Ok = Poll>; +#[cfg(bootstrap)] +impl ops::TryV1 for Poll>> { + type Output = Poll>; type Error = E; #[inline] - fn into_result(self) -> Result { + fn into_result(self) -> Result { match self { Poll::Ready(Some(Ok(x))) => Ok(Poll::Ready(Some(x))), Poll::Ready(Some(Err(e))) => Err(e), @@ -204,7 +206,7 @@ impl Try for Poll>> { } #[inline] - fn from_ok(x: Self::Ok) -> Self { + fn from_ok(x: Self::Output) -> Self { x.map(|x| x.map(Ok)) } } diff --git a/library/core/src/time.rs b/library/core/src/time.rs index bfea39e321..489b722440 100644 --- a/library/core/src/time.rs +++ b/library/core/src/time.rs @@ -518,13 +518,11 @@ impl Duration { if let Some(mut secs) = self.secs.checked_sub(rhs.secs) { let nanos = if self.nanos >= rhs.nanos { self.nanos - rhs.nanos + } else if let Some(sub_secs) = secs.checked_sub(1) { + secs = sub_secs; + self.nanos + NANOS_PER_SEC - rhs.nanos } else { - if let Some(sub_secs) = secs.checked_sub(1) { - secs = sub_secs; - self.nanos + NANOS_PER_SEC - rhs.nanos - } else { - return None; - } + return None; }; debug_assert!(nanos < NANOS_PER_SEC); Some(Duration { secs, nanos }) diff --git a/library/core/tests/any.rs b/library/core/tests/any.rs index b0dc990346..b36d6f0d40 100644 --- a/library/core/tests/any.rs +++ b/library/core/tests/any.rs @@ -114,3 +114,16 @@ fn any_unsized() { fn is_any() {} is_any::<[i32]>(); } + +#[test] +fn distinct_type_names() { + // https://github.com/rust-lang/rust/issues/84666 + + struct Velocity(f32, f32); + + fn type_name_of_val(_: T) -> &'static str { + type_name::() + } + + assert_ne!(type_name_of_val(Velocity), type_name_of_val(Velocity(0.0, -9.8)),); +} diff --git a/library/core/tests/array.rs b/library/core/tests/array.rs index ce7480ce2e..0ae625bdb6 100644 --- a/library/core/tests/array.rs +++ b/library/core/tests/array.rs @@ -1,4 +1,4 @@ -use core::array::{self, IntoIter}; +use core::array; use core::convert::TryFrom; #[test] @@ -41,14 +41,14 @@ fn array_try_from() { #[test] fn iterator_collect() { let arr = [0, 1, 2, 5, 9]; - let v: Vec<_> = IntoIter::new(arr.clone()).collect(); + let v: Vec<_> = IntoIterator::into_iter(arr.clone()).collect(); assert_eq!(&arr[..], &v[..]); } #[test] fn iterator_rev_collect() { let arr = [0, 1, 2, 5, 9]; - let v: Vec<_> = IntoIter::new(arr.clone()).rev().collect(); + let v: Vec<_> = IntoIterator::into_iter(arr.clone()).rev().collect(); assert_eq!(&v[..], &[9, 5, 2, 1, 0]); } @@ -56,11 +56,11 @@ fn iterator_rev_collect() { fn iterator_nth() { let v = [0, 1, 2, 3, 4]; for i in 0..v.len() { - assert_eq!(IntoIter::new(v.clone()).nth(i).unwrap(), v[i]); + assert_eq!(IntoIterator::into_iter(v.clone()).nth(i).unwrap(), v[i]); } - assert_eq!(IntoIter::new(v.clone()).nth(v.len()), None); + assert_eq!(IntoIterator::into_iter(v.clone()).nth(v.len()), None); - let mut iter = IntoIter::new(v); + let mut iter = IntoIterator::into_iter(v); assert_eq!(iter.nth(2).unwrap(), v[2]); assert_eq!(iter.nth(1).unwrap(), v[4]); } @@ -68,17 +68,17 @@ fn iterator_nth() { #[test] fn iterator_last() { let v = [0, 1, 2, 3, 4]; - assert_eq!(IntoIter::new(v).last().unwrap(), 4); - assert_eq!(IntoIter::new([0]).last().unwrap(), 0); + assert_eq!(IntoIterator::into_iter(v).last().unwrap(), 4); + assert_eq!(IntoIterator::into_iter([0]).last().unwrap(), 0); - let mut it = IntoIter::new([0, 9, 2, 4]); + let mut it = IntoIterator::into_iter([0, 9, 2, 4]); assert_eq!(it.next_back(), Some(4)); assert_eq!(it.last(), Some(2)); } #[test] fn iterator_clone() { - let mut it = IntoIter::new([0, 2, 4, 6, 8]); + let mut it = IntoIterator::into_iter([0, 2, 4, 6, 8]); assert_eq!(it.next(), Some(0)); assert_eq!(it.next_back(), Some(8)); let mut clone = it.clone(); @@ -92,7 +92,7 @@ fn iterator_clone() { #[test] fn iterator_fused() { - let mut it = IntoIter::new([0, 9, 2]); + let mut it = IntoIterator::into_iter([0, 9, 2]); assert_eq!(it.next(), Some(0)); assert_eq!(it.next(), Some(9)); assert_eq!(it.next(), Some(2)); @@ -105,7 +105,7 @@ fn iterator_fused() { #[test] fn iterator_len() { - let mut it = IntoIter::new([0, 1, 2, 5, 9]); + let mut it = IntoIterator::into_iter([0, 1, 2, 5, 9]); assert_eq!(it.size_hint(), (5, Some(5))); assert_eq!(it.len(), 5); assert_eq!(it.is_empty(), false); @@ -121,7 +121,7 @@ fn iterator_len() { assert_eq!(it.is_empty(), false); // Empty - let it = IntoIter::new([] as [String; 0]); + let it = IntoIterator::into_iter([] as [String; 0]); assert_eq!(it.size_hint(), (0, Some(0))); assert_eq!(it.len(), 0); assert_eq!(it.is_empty(), true); @@ -130,9 +130,9 @@ fn iterator_len() { #[test] fn iterator_count() { let v = [0, 1, 2, 3, 4]; - assert_eq!(IntoIter::new(v.clone()).count(), 5); + assert_eq!(IntoIterator::into_iter(v.clone()).count(), 5); - let mut iter2 = IntoIter::new(v); + let mut iter2 = IntoIterator::into_iter(v); iter2.next(); iter2.next(); assert_eq!(iter2.count(), 3); @@ -140,13 +140,13 @@ fn iterator_count() { #[test] fn iterator_flat_map() { - assert!((0..5).flat_map(|i| IntoIter::new([2 * i, 2 * i + 1])).eq(0..10)); + assert!((0..5).flat_map(|i| IntoIterator::into_iter([2 * i, 2 * i + 1])).eq(0..10)); } #[test] fn iterator_debug() { let arr = [0, 1, 2, 5, 9]; - assert_eq!(format!("{:?}", IntoIter::new(arr)), "IntoIter([0, 1, 2, 5, 9])",); + assert_eq!(format!("{:?}", IntoIterator::into_iter(arr)), "IntoIter([0, 1, 2, 5, 9])",); } #[test] @@ -176,14 +176,14 @@ fn iterator_drops() { // Simple: drop new iterator. let i = Cell::new(0); { - IntoIter::new(five(&i)); + IntoIterator::into_iter(five(&i)); } assert_eq!(i.get(), 5); // Call `next()` once. let i = Cell::new(0); { - let mut iter = IntoIter::new(five(&i)); + let mut iter = IntoIterator::into_iter(five(&i)); let _x = iter.next(); assert_eq!(i.get(), 0); assert_eq!(iter.count(), 4); @@ -194,7 +194,7 @@ fn iterator_drops() { // Check `clone` and calling `next`/`next_back`. let i = Cell::new(0); { - let mut iter = IntoIter::new(five(&i)); + let mut iter = IntoIterator::into_iter(five(&i)); iter.next(); assert_eq!(i.get(), 1); iter.next_back(); @@ -217,7 +217,7 @@ fn iterator_drops() { // Check via `nth`. let i = Cell::new(0); { - let mut iter = IntoIter::new(five(&i)); + let mut iter = IntoIterator::into_iter(five(&i)); let _x = iter.nth(2); assert_eq!(i.get(), 2); let _y = iter.last(); @@ -227,13 +227,13 @@ fn iterator_drops() { // Check every element. let i = Cell::new(0); - for (index, _x) in IntoIter::new(five(&i)).enumerate() { + for (index, _x) in IntoIterator::into_iter(five(&i)).enumerate() { assert_eq!(i.get(), index); } assert_eq!(i.get(), 5); let i = Cell::new(0); - for (index, _x) in IntoIter::new(five(&i)).rev().enumerate() { + for (index, _x) in IntoIterator::into_iter(five(&i)).rev().enumerate() { assert_eq!(i.get(), index); } assert_eq!(i.get(), 5); diff --git a/library/core/tests/char.rs b/library/core/tests/char.rs index c16f54081c..51eca1e05d 100644 --- a/library/core/tests/char.rs +++ b/library/core/tests/char.rs @@ -67,10 +67,20 @@ fn test_to_digit() { assert_eq!('A'.to_digit(16), Some(10)); assert_eq!('b'.to_digit(16), Some(11)); assert_eq!('B'.to_digit(16), Some(11)); + assert_eq!('A'.to_digit(36), Some(10)); assert_eq!('z'.to_digit(36), Some(35)); assert_eq!('Z'.to_digit(36), Some(35)); - assert_eq!(' '.to_digit(10), None); + assert_eq!('['.to_digit(36), None); + assert_eq!('`'.to_digit(36), None); + assert_eq!('{'.to_digit(36), None); assert_eq!('$'.to_digit(36), None); + assert_eq!('@'.to_digit(16), None); + assert_eq!('G'.to_digit(16), None); + assert_eq!('g'.to_digit(16), None); + assert_eq!(' '.to_digit(10), None); + assert_eq!('/'.to_digit(10), None); + assert_eq!(':'.to_digit(10), None); + assert_eq!(':'.to_digit(11), None); } #[test] diff --git a/library/core/tests/iter/adapters/zip.rs b/library/core/tests/iter/adapters/zip.rs index 000c15f72c..797bfd957f 100644 --- a/library/core/tests/iter/adapters/zip.rs +++ b/library/core/tests/iter/adapters/zip.rs @@ -236,9 +236,7 @@ fn test_zip_trusted_random_access_composition() { fn test_double_ended_zip() { let xs = [1, 2, 3, 4, 5, 6]; let ys = [1, 2, 3, 7]; - let a = xs.iter().cloned(); - let b = ys.iter().cloned(); - let mut it = a.zip(b); + let mut it = xs.iter().cloned().zip(ys); assert_eq!(it.next(), Some((1, 1))); assert_eq!(it.next(), Some((2, 2))); assert_eq!(it.next_back(), Some((4, 7))); diff --git a/library/core/tests/lib.rs b/library/core/tests/lib.rs index 35e4d213dd..16051b3bc3 100644 --- a/library/core/tests/lib.rs +++ b/library/core/tests/lib.rs @@ -40,11 +40,10 @@ #![feature(maybe_uninit_write_slice)] #![feature(min_specialization)] #![feature(step_trait)] -#![feature(step_trait_ext)] #![feature(str_internals)] #![feature(test)] #![feature(trusted_len)] -#![feature(try_trait)] +#![feature(try_trait_v2)] #![feature(slice_internals)] #![feature(slice_partition_dedup)] #![feature(int_error_matching)] diff --git a/library/core/tests/option.rs b/library/core/tests/option.rs index 9470451278..88ea15a3b3 100644 --- a/library/core/tests/option.rs +++ b/library/core/tests/option.rs @@ -301,18 +301,6 @@ fn test_try() { Some(val) } assert_eq!(try_option_none(), None); - - fn try_option_ok() -> Result { - let val = Some(1)?; - Ok(val) - } - assert_eq!(try_option_ok(), Ok(1)); - - fn try_option_err() -> Result { - let val = None?; - Ok(val) - } - assert_eq!(try_option_err(), Err(NoneError)); } #[test] diff --git a/library/core/tests/result.rs b/library/core/tests/result.rs index c461ab380a..f4e5e7751b 100644 --- a/library/core/tests/result.rs +++ b/library/core/tests/result.rs @@ -249,26 +249,14 @@ pub fn test_into_err() { #[test] fn test_try() { - fn try_result_some() -> Option { - let val = Ok(1)?; - Some(val) - } - assert_eq!(try_result_some(), Some(1)); - - fn try_result_none() -> Option { - let val = Err(NoneError)?; - Some(val) - } - assert_eq!(try_result_none(), None); - - fn try_result_ok() -> Result { + fn try_result_ok() -> Result { let result: Result = Ok(1); let val = result?; Ok(val) } assert_eq!(try_result_ok(), Ok(1)); - fn try_result_err() -> Result { + fn try_result_err() -> Result { let result: Result = Err(1); let val = result?; Ok(val) @@ -401,3 +389,17 @@ fn result_opt_conversions() { assert_eq!(res, Err(BadNumErr)) } + +#[test] +#[cfg(not(bootstrap))] // Needs the V2 trait +fn result_try_trait_v2_branch() { + use core::num::NonZeroU32; + use core::ops::{ControlFlow::*, Try}; + assert_eq!(Ok::(4).branch(), Continue(4)); + assert_eq!(Err::(4).branch(), Break(Err(4))); + let one = NonZeroU32::new(1).unwrap(); + assert_eq!(Ok::<(), NonZeroU32>(()).branch(), Continue(())); + assert_eq!(Err::<(), NonZeroU32>(one).branch(), Break(Err(one))); + assert_eq!(Ok::(one).branch(), Continue(one)); + assert_eq!(Err::(()).branch(), Break(Err(()))); +} diff --git a/library/panic_abort/src/lib.rs b/library/panic_abort/src/lib.rs index 5dcd1e6af3..d95ea6530c 100644 --- a/library/panic_abort/src/lib.rs +++ b/library/panic_abort/src/lib.rs @@ -5,10 +5,7 @@ #![no_std] #![unstable(feature = "panic_abort", issue = "32837")] -#![doc( - html_root_url = "https://doc.rust-lang.org/nightly/", - issue_tracker_base_url = "https://github.com/rust-lang/rust/issues/" -)] +#![doc(issue_tracker_base_url = "https://github.com/rust-lang/rust/issues/")] #![panic_runtime] #![allow(unused_features)] #![feature(core_intrinsics)] diff --git a/library/panic_unwind/src/lib.rs b/library/panic_unwind/src/lib.rs index 99a0c67fc1..d32a3f1f83 100644 --- a/library/panic_unwind/src/lib.rs +++ b/library/panic_unwind/src/lib.rs @@ -13,10 +13,7 @@ #![no_std] #![unstable(feature = "panic_unwind", issue = "32837")] -#![doc( - html_root_url = "https://doc.rust-lang.org/nightly/", - issue_tracker_base_url = "https://github.com/rust-lang/rust/issues/" -)] +#![doc(issue_tracker_base_url = "https://github.com/rust-lang/rust/issues/")] #![feature(core_intrinsics)] #![feature(lang_items)] #![feature(nll)] diff --git a/library/panic_unwind/src/seh.rs b/library/panic_unwind/src/seh.rs index 5597bbb93d..58028d4057 100644 --- a/library/panic_unwind/src/seh.rs +++ b/library/panic_unwind/src/seh.rs @@ -316,7 +316,7 @@ pub unsafe fn panic(data: Box) -> u32 { } pub unsafe fn cleanup(payload: *mut u8) -> Box { - // A NULL payload here means that we got here from the catch (...) of + // A null payload here means that we got here from the catch (...) of // __rust_try. This happens when a non-Rust foreign exception is caught. if payload.is_null() { super::__rust_foreign_exception(); diff --git a/library/proc_macro/src/bridge/buffer.rs b/library/proc_macro/src/bridge/buffer.rs index aeecbd4966..717201aef1 100644 --- a/library/proc_macro/src/bridge/buffer.rs +++ b/library/proc_macro/src/bridge/buffer.rs @@ -39,7 +39,7 @@ pub struct Buffer { data: *mut T, len: usize, capacity: usize, - extend_from_slice: extern "C" fn(Buffer, Slice<'_, T>) -> Buffer, + reserve: extern "C" fn(Buffer, usize) -> Buffer, drop: extern "C" fn(Buffer), } @@ -78,18 +78,44 @@ impl Buffer { mem::take(self) } + // We have the array method separate from extending from a slice. This is + // because in the case of small arrays, codegen can be more efficient + // (avoiding a memmove call). With extend_from_slice, LLVM at least + // currently is not able to make that optimization. + pub(super) fn extend_from_array(&mut self, xs: &[T; N]) { + if xs.len() > (self.capacity - self.len) { + let b = self.take(); + *self = (b.reserve)(b, xs.len()); + } + unsafe { + xs.as_ptr().copy_to_nonoverlapping(self.data.add(self.len), xs.len()); + self.len += xs.len(); + } + } + pub(super) fn extend_from_slice(&mut self, xs: &[T]) { - // Fast path to avoid going through an FFI call. - if let Some(final_len) = self.len.checked_add(xs.len()) { - if final_len <= self.capacity { - let dst = unsafe { slice::from_raw_parts_mut(self.data, self.capacity) }; - dst[self.len..][..xs.len()].copy_from_slice(xs); - self.len = final_len; - return; - } + if xs.len() > (self.capacity - self.len) { + let b = self.take(); + *self = (b.reserve)(b, xs.len()); + } + unsafe { + xs.as_ptr().copy_to_nonoverlapping(self.data.add(self.len), xs.len()); + self.len += xs.len(); + } + } + + pub(super) fn push(&mut self, v: T) { + // The code here is taken from Vec::push, and we know that reserve() + // will panic if we're exceeding isize::MAX bytes and so there's no need + // to check for overflow. + if self.len == self.capacity { + let b = self.take(); + *self = (b.reserve)(b, 1); + } + unsafe { + *self.data.add(self.len) = v; + self.len += 1; } - let b = self.take(); - *self = (b.extend_from_slice)(b, Slice::from(xs)); } } @@ -131,9 +157,9 @@ impl From> for Buffer { } } - extern "C" fn extend_from_slice(b: Buffer, xs: Slice<'_, T>) -> Buffer { + extern "C" fn reserve(b: Buffer, additional: usize) -> Buffer { let mut v = to_vec(b); - v.extend_from_slice(&xs); + v.reserve(additional); Buffer::from(v) } @@ -141,6 +167,6 @@ impl From> for Buffer { mem::drop(to_vec(b)); } - Buffer { data, len, capacity, extend_from_slice, drop } + Buffer { data, len, capacity, reserve, drop } } } diff --git a/library/proc_macro/src/bridge/mod.rs b/library/proc_macro/src/bridge/mod.rs index c898d483a8..a2953b6856 100644 --- a/library/proc_macro/src/bridge/mod.rs +++ b/library/proc_macro/src/bridge/mod.rs @@ -107,6 +107,7 @@ macro_rules! with_api { Literal { fn drop($self: $S::Literal); fn clone($self: &$S::Literal) -> $S::Literal; + fn from_str(s: &str) -> Result<$S::Literal, ()>; fn debug_kind($self: &$S::Literal) -> String; fn symbol($self: &$S::Literal) -> String; fn suffix($self: &$S::Literal) -> Option; @@ -162,6 +163,8 @@ macro_rules! with_api { fn join($self: $S::Span, other: $S::Span) -> Option<$S::Span>; fn resolved_at($self: $S::Span, at: $S::Span) -> $S::Span; fn source_text($self: $S::Span) -> Option; + fn save_span($self: $S::Span) -> usize; + fn recover_proc_macro_span(id: usize) -> $S::Span; }, } }; @@ -313,6 +316,19 @@ impl Unmark for Option { } } +impl Mark for Result { + type Unmarked = Result; + fn mark(unmarked: Self::Unmarked) -> Self { + unmarked.map(T::mark).map_err(E::mark) + } +} +impl Unmark for Result { + type Unmarked = Result; + fn unmark(self) -> Self::Unmarked { + self.map(T::unmark).map_err(E::unmark) + } +} + macro_rules! mark_noop { ($($ty:ty),* $(,)?) => { $( @@ -338,6 +354,7 @@ mark_noop! { &'a [u8], &'a str, String, + usize, Delimiter, Level, LineColumn, diff --git a/library/proc_macro/src/bridge/rpc.rs b/library/proc_macro/src/bridge/rpc.rs index 5c2f9ec984..588e6ded0f 100644 --- a/library/proc_macro/src/bridge/rpc.rs +++ b/library/proc_macro/src/bridge/rpc.rs @@ -27,7 +27,7 @@ macro_rules! rpc_encode_decode { (le $ty:ty) => { impl Encode for $ty { fn encode(self, w: &mut Writer, _: &mut S) { - w.write_all(&self.to_le_bytes()).unwrap(); + w.extend_from_array(&self.to_le_bytes()); } } @@ -114,7 +114,7 @@ impl DecodeMut<'_, '_, S> for () { impl Encode for u8 { fn encode(self, w: &mut Writer, _: &mut S) { - w.write_all(&[self]).unwrap(); + w.push(self); } } diff --git a/library/proc_macro/src/lib.rs b/library/proc_macro/src/lib.rs index c7f58f3615..26fbf50e2d 100644 --- a/library/proc_macro/src/lib.rs +++ b/library/proc_macro/src/lib.rs @@ -12,7 +12,6 @@ #![stable(feature = "proc_macro_lib", since = "1.15.0")] #![deny(missing_docs)] #![doc( - html_root_url = "https://doc.rust-lang.org/nightly/", html_playground_url = "https://play.rust-lang.org/", issue_tracker_base_url = "https://github.com/rust-lang/rust/issues/", test(no_crate_inject, attr(deny(warnings))), @@ -21,8 +20,7 @@ #![feature(rustc_allow_const_fn_unstable)] #![feature(nll)] #![feature(staged_api)] -#![cfg_attr(bootstrap, feature(const_fn))] -#![cfg_attr(not(bootstrap), feature(const_fn_trait_bound))] +#![feature(const_fn_trait_bound)] #![feature(const_fn_fn_ptr_basics)] #![feature(allow_internal_unstable)] #![feature(decl_macro)] @@ -33,6 +31,7 @@ #![feature(restricted_std)] #![feature(rustc_attrs)] #![feature(min_specialization)] +#![feature(bound_cloned)] #![recursion_limit = "256"] #[unstable(feature = "proc_macro_internals", issue = "27812")] @@ -45,7 +44,7 @@ mod diagnostic; pub use diagnostic::{Diagnostic, Level, MultiSpan}; use std::cmp::Ordering; -use std::ops::{Bound, RangeBounds}; +use std::ops::RangeBounds; use std::path::PathBuf; use std::str::FromStr; use std::{error, fmt, iter, mem}; @@ -91,6 +90,12 @@ pub struct LexError { _inner: (), } +impl LexError { + fn new() -> Self { + LexError { _inner: () } + } +} + #[stable(feature = "proc_macro_lexerror_impls", since = "1.44.0")] impl fmt::Display for LexError { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { @@ -265,7 +270,7 @@ pub mod token_stream { /// Unquoting is done with `$`, and works by taking the single next ident as the unquoted term. /// To quote `$` itself, use `$$`. #[unstable(feature = "proc_macro_quote", issue = "54722")] -#[allow_internal_unstable(proc_macro_def_site)] +#[allow_internal_unstable(proc_macro_def_site, proc_macro_internals)] #[rustc_builtin_macro] pub macro quote($($t:tt)*) { /* compiler built-in */ @@ -394,6 +399,20 @@ impl Span { self.0.source_text() } + // Used by the implementation of `Span::quote` + #[doc(hidden)] + #[unstable(feature = "proc_macro_internals", issue = "27812")] + pub fn save_span(&self) -> usize { + self.0.save_span() + } + + // Used by the implementation of `Span::quote` + #[doc(hidden)] + #[unstable(feature = "proc_macro_internals", issue = "27812")] + pub fn recover_proc_macro_span(id: usize) -> Span { + Span(bridge::client::Span::recover_proc_macro_span(id)) + } + diagnostic_method!(error, Level::Error); diagnostic_method!(warning, Level::Warning); diagnostic_method!(note, Level::Note); @@ -1144,16 +1163,29 @@ impl Literal { // was 'c' or whether it was '\u{63}'. #[unstable(feature = "proc_macro_span", issue = "54725")] pub fn subspan>(&self, range: R) -> Option { - // HACK(eddyb) something akin to `Option::cloned`, but for `Bound<&T>`. - fn cloned_bound(bound: Bound<&T>) -> Bound { - match bound { - Bound::Included(x) => Bound::Included(x.clone()), - Bound::Excluded(x) => Bound::Excluded(x.clone()), - Bound::Unbounded => Bound::Unbounded, - } - } + self.0.subspan(range.start_bound().cloned(), range.end_bound().cloned()).map(Span) + } +} + +/// Parse a single literal from its stringified representation. +/// +/// In order to parse successfully, the input string must not contain anything +/// but the literal token. Specifically, it must not contain whitespace or +/// comments in addition to the literal. +/// +/// The resulting literal token will have a `Span::call_site()` span. +/// +/// NOTE: some errors may cause panics instead of returning `LexError`. We +/// reserve the right to change these errors into `LexError`s later. +#[stable(feature = "proc_macro_literal_parse", since = "1.54.0")] +impl FromStr for Literal { + type Err = LexError; - self.0.subspan(cloned_bound(range.start_bound()), cloned_bound(range.end_bound())).map(Span) + fn from_str(src: &str) -> Result { + match bridge::client::Literal::from_str(src) { + Ok(literal) => Ok(Literal(literal)), + Err(()) => Err(LexError::new()), + } } } diff --git a/library/proc_macro/src/quote.rs b/library/proc_macro/src/quote.rs index 144e2d6bac..1fd5988970 100644 --- a/library/proc_macro/src/quote.rs +++ b/library/proc_macro/src/quote.rs @@ -65,6 +65,7 @@ pub fn quote(stream: TokenStream) -> TokenStream { if stream.is_empty() { return quote!(crate::TokenStream::new()); } + let proc_macro_crate = quote!(crate); let mut after_dollar = false; let tokens = stream .into_iter() @@ -105,7 +106,7 @@ pub fn quote(stream: TokenStream) -> TokenStream { ))), TokenTree::Ident(tt) => quote!(crate::TokenTree::Ident(crate::Ident::new( (@ TokenTree::from(Literal::string(&tt.to_string()))), - (@ quote_span(tt.span())), + (@ quote_span(proc_macro_crate.clone(), tt.span())), ))), TokenTree::Literal(tt) => quote!(crate::TokenTree::Literal({ let mut iter = (@ TokenTree::from(Literal::string(&tt.to_string()))) @@ -115,7 +116,7 @@ pub fn quote(stream: TokenStream) -> TokenStream { if let (Some(crate::TokenTree::Literal(mut lit)), None) = (iter.next(), iter.next()) { - lit.set_span((@ quote_span(tt.span()))); + lit.set_span((@ quote_span(proc_macro_crate.clone(), tt.span()))); lit } else { unreachable!() @@ -135,6 +136,7 @@ pub fn quote(stream: TokenStream) -> TokenStream { /// Quote a `Span` into a `TokenStream`. /// This is needed to implement a custom quoter. #[unstable(feature = "proc_macro_quote", issue = "54722")] -pub fn quote_span(_: Span) -> TokenStream { - quote!(crate::Span::def_site()) +pub fn quote_span(proc_macro_crate: TokenStream, span: Span) -> TokenStream { + let id = span.save_span(); + quote!((@ proc_macro_crate ) ::Span::recover_proc_macro_span((@ TokenTree::from(Literal::usize_unsuffixed(id))))) } diff --git a/library/profiler_builtins/Cargo.toml b/library/profiler_builtins/Cargo.toml index 695c847d2c..7b7ca8029b 100644 --- a/library/profiler_builtins/Cargo.toml +++ b/library/profiler_builtins/Cargo.toml @@ -14,4 +14,4 @@ core = { path = "../core" } compiler_builtins = { version = "0.1.0", features = ['rustc-dep-of-std'] } [build-dependencies] -cc = "1.0.1" +cc = "1.0.68" diff --git a/library/std/Cargo.toml b/library/std/Cargo.toml index ab7b142ef3..415d874c7f 100644 --- a/library/std/Cargo.toml +++ b/library/std/Cargo.toml @@ -17,7 +17,7 @@ panic_unwind = { path = "../panic_unwind", optional = true } panic_abort = { path = "../panic_abort" } core = { path = "../core" } libc = { version = "0.2.93", default-features = false, features = ['rustc-dep-of-std'] } -compiler_builtins = { version = "0.1.39" } +compiler_builtins = { version = "0.1.44" } profiler_builtins = { path = "../profiler_builtins", optional = true } unwind = { path = "../unwind" } hashbrown = { version = "0.11", default-features = false, features = ['rustc-dep-of-std'] } diff --git a/library/std/src/alloc.rs b/library/std/src/alloc.rs index 843ef09a58..8ee55234ce 100644 --- a/library/std/src/alloc.rs +++ b/library/std/src/alloc.rs @@ -63,8 +63,6 @@ use core::ptr::NonNull; use core::sync::atomic::{AtomicPtr, Ordering}; use core::{mem, ptr}; -use crate::sys_common::util::dumb_print; - #[stable(feature = "alloc_module", since = "1.28.0")] #[doc(inline)] pub use alloc_crate::alloc::*; @@ -317,7 +315,7 @@ pub fn take_alloc_error_hook() -> fn(Layout) { } fn default_alloc_error_hook(layout: Layout) { - dumb_print(format_args!("memory allocation of {} bytes failed\n", layout.size())); + rtprintpanic!("memory allocation of {} bytes failed\n", layout.size()); } #[cfg(not(test))] diff --git a/library/std/src/collections/hash/map.rs b/library/std/src/collections/hash/map.rs index bdf559847c..a1f52a9c2e 100644 --- a/library/std/src/collections/hash/map.rs +++ b/library/std/src/collections/hash/map.rs @@ -962,7 +962,6 @@ where /// # Examples /// /// ``` - /// #![feature(map_into_keys_values)] /// use std::collections::HashMap; /// /// let mut map = HashMap::new(); @@ -973,7 +972,7 @@ where /// let vec: Vec<&str> = map.into_keys().collect(); /// ``` #[inline] - #[unstable(feature = "map_into_keys_values", issue = "75294")] + #[stable(feature = "map_into_keys_values", since = "1.54.0")] pub fn into_keys(self) -> IntoKeys { IntoKeys { inner: self.into_iter() } } @@ -985,7 +984,6 @@ where /// # Examples /// /// ``` - /// #![feature(map_into_keys_values)] /// use std::collections::HashMap; /// /// let mut map = HashMap::new(); @@ -996,7 +994,7 @@ where /// let vec: Vec = map.into_values().collect(); /// ``` #[inline] - #[unstable(feature = "map_into_keys_values", issue = "75294")] + #[stable(feature = "map_into_keys_values", since = "1.54.0")] pub fn into_values(self) -> IntoValues { IntoValues { inner: self.into_iter() } } @@ -1405,15 +1403,13 @@ pub struct ValuesMut<'a, K: 'a, V: 'a> { /// # Example /// /// ``` -/// #![feature(map_into_keys_values)] -/// /// use std::collections::HashMap; /// /// let mut map = HashMap::new(); /// map.insert("a", 1); /// let iter_keys = map.into_keys(); /// ``` -#[unstable(feature = "map_into_keys_values", issue = "75294")] +#[stable(feature = "map_into_keys_values", since = "1.54.0")] pub struct IntoKeys { inner: IntoIter, } @@ -1428,15 +1424,13 @@ pub struct IntoKeys { /// # Example /// /// ``` -/// #![feature(map_into_keys_values)] -/// /// use std::collections::HashMap; /// /// let mut map = HashMap::new(); /// map.insert("a", 1); /// let iter_keys = map.into_values(); /// ``` -#[unstable(feature = "map_into_keys_values", issue = "75294")] +#[stable(feature = "map_into_keys_values", since = "1.54.0")] pub struct IntoValues { inner: IntoIter, } @@ -2137,7 +2131,7 @@ impl fmt::Debug for ValuesMut<'_, K, V> { } } -#[unstable(feature = "map_into_keys_values", issue = "75294")] +#[stable(feature = "map_into_keys_values", since = "1.54.0")] impl Iterator for IntoKeys { type Item = K; @@ -2150,24 +2144,24 @@ impl Iterator for IntoKeys { self.inner.size_hint() } } -#[unstable(feature = "map_into_keys_values", issue = "75294")] +#[stable(feature = "map_into_keys_values", since = "1.54.0")] impl ExactSizeIterator for IntoKeys { #[inline] fn len(&self) -> usize { self.inner.len() } } -#[unstable(feature = "map_into_keys_values", issue = "75294")] +#[stable(feature = "map_into_keys_values", since = "1.54.0")] impl FusedIterator for IntoKeys {} -#[unstable(feature = "map_into_keys_values", issue = "75294")] +#[stable(feature = "map_into_keys_values", since = "1.54.0")] impl fmt::Debug for IntoKeys { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_list().entries(self.inner.iter().map(|(k, _)| k)).finish() } } -#[unstable(feature = "map_into_keys_values", issue = "75294")] +#[stable(feature = "map_into_keys_values", since = "1.54.0")] impl Iterator for IntoValues { type Item = V; @@ -2180,17 +2174,17 @@ impl Iterator for IntoValues { self.inner.size_hint() } } -#[unstable(feature = "map_into_keys_values", issue = "75294")] +#[stable(feature = "map_into_keys_values", since = "1.54.0")] impl ExactSizeIterator for IntoValues { #[inline] fn len(&self) -> usize { self.inner.len() } } -#[unstable(feature = "map_into_keys_values", issue = "75294")] +#[stable(feature = "map_into_keys_values", since = "1.54.0")] impl FusedIterator for IntoValues {} -#[unstable(feature = "map_into_keys_values", issue = "75294")] +#[stable(feature = "map_into_keys_values", since = "1.54.0")] impl fmt::Debug for IntoValues { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_list().entries(self.inner.iter().map(|(_, v)| v)).finish() diff --git a/library/std/src/env.rs b/library/std/src/env.rs index 821e7d4cfe..4403280efc 100644 --- a/library/std/src/env.rs +++ b/library/std/src/env.rs @@ -61,6 +61,7 @@ pub fn current_dir() -> io::Result { /// assert!(env::set_current_dir(&root).is_ok()); /// println!("Successfully changed working directory to {}!", root.display()); /// ``` +#[doc(alias = "chdir")] #[stable(feature = "env", since = "1.0.0")] pub fn set_current_dir>(path: P) -> io::Result<()> { os_imp::chdir(path.as_ref()) @@ -323,13 +324,13 @@ impl Error for VarError { /// assert_eq!(env::var(key), Ok("VALUE".to_string())); /// ``` #[stable(feature = "env", since = "1.0.0")] -pub fn set_var, V: AsRef>(k: K, v: V) { - _set_var(k.as_ref(), v.as_ref()) +pub fn set_var, V: AsRef>(key: K, value: V) { + _set_var(key.as_ref(), value.as_ref()) } -fn _set_var(k: &OsStr, v: &OsStr) { - os_imp::setenv(k, v).unwrap_or_else(|e| { - panic!("failed to set environment variable `{:?}` to `{:?}`: {}", k, v, e) +fn _set_var(key: &OsStr, value: &OsStr) { + os_imp::setenv(key, value).unwrap_or_else(|e| { + panic!("failed to set environment variable `{:?}` to `{:?}`: {}", key, value, e) }) } @@ -365,13 +366,13 @@ fn _set_var(k: &OsStr, v: &OsStr) { /// assert!(env::var(key).is_err()); /// ``` #[stable(feature = "env", since = "1.0.0")] -pub fn remove_var>(k: K) { - _remove_var(k.as_ref()) +pub fn remove_var>(key: K) { + _remove_var(key.as_ref()) } -fn _remove_var(k: &OsStr) { - os_imp::unsetenv(k) - .unwrap_or_else(|e| panic!("failed to remove environment variable `{:?}`: {}", k, e)) +fn _remove_var(key: &OsStr) { + os_imp::unsetenv(key) + .unwrap_or_else(|e| panic!("failed to remove environment variable `{:?}`: {}", key, e)) } /// An iterator that splits an environment variable into paths according to diff --git a/library/std/src/ffi/c_str.rs b/library/std/src/ffi/c_str.rs index ed4950c57a..be7e099b73 100644 --- a/library/std/src/ffi/c_str.rs +++ b/library/std/src/ffi/c_str.rs @@ -10,7 +10,6 @@ use crate::error::Error; use crate::fmt::{self, Write}; use crate::io; use crate::mem; -use crate::memchr; use crate::num::NonZeroU8; use crate::ops; use crate::os::raw::c_char; @@ -20,6 +19,7 @@ use crate::slice; use crate::str::{self, Utf8Error}; use crate::sync::Arc; use crate::sys; +use crate::sys_common::memchr; /// A type representing an owned, C-compatible, nul-terminated string with no nul bytes in the /// middle. @@ -185,6 +185,7 @@ pub struct CString { /// /// [`&str`]: prim@str #[derive(Hash)] +#[cfg_attr(not(test), rustc_diagnostic_item = "CStr")] #[stable(feature = "rust1", since = "1.0.0")] // FIXME: // `fn from` in `impl From<&CStr> for Box` current implementation relies @@ -498,7 +499,7 @@ impl CString { /// Failure to call [`CString::from_raw`] will lead to a memory leak. /// /// The C side must **not** modify the length of the string (by writing a - /// `NULL` somewhere inside the string or removing the final one) before + /// `null` somewhere inside the string or removing the final one) before /// it makes it back into Rust using [`CString::from_raw`]. See the safety section /// in [`CString::from_raw`]. /// @@ -671,6 +672,7 @@ impl CString { } /// Bypass "move out of struct which implements [`Drop`] trait" restriction. + #[inline] fn into_inner(self) -> Box<[u8]> { // Rationale: `mem::forget(self)` invalidates the previous call to `ptr::read(&self.inner)` // so we use `ManuallyDrop` to ensure `self` is not dropped. diff --git a/library/std/src/ffi/os_str.rs b/library/std/src/ffi/os_str.rs index ea4b286601..ca391ffb3d 100644 --- a/library/std/src/ffi/os_str.rs +++ b/library/std/src/ffi/os_str.rs @@ -601,7 +601,7 @@ impl OsStr { /// // sequences simply through collecting user command line arguments, for /// // example. /// - /// #[cfg(any(unix, target_os = "redox"))] { + /// #[cfg(unix)] { /// use std::ffi::OsStr; /// use std::os::unix::ffi::OsStrExt; /// diff --git a/library/std/src/fs.rs b/library/std/src/fs.rs index e6120b8ee3..a1636e2f60 100644 --- a/library/std/src/fs.rs +++ b/library/std/src/fs.rs @@ -2208,3 +2208,29 @@ impl AsInnerMut for DirBuilder { &mut self.inner } } + +/// Returns `Ok(true)` if the path points at an existing entity. +/// +/// This function will traverse symbolic links to query information about the +/// destination file. In case of broken symbolic links this will return `Ok(false)`. +/// +/// As opposed to the `exists()` method, this one doesn't silently ignore errors +/// unrelated to the path not existing. (E.g. it will return `Err(_)` in case of permission +/// denied on some of the parent directories.) +/// +/// # Examples +/// +/// ```no_run +/// #![feature(path_try_exists)] +/// use std::fs; +/// +/// assert!(!fs::try_exists("does_not_exist.txt").expect("Can't check existence of file does_not_exist.txt")); +/// assert!(fs::try_exists("/root/secret_file.txt").is_err()); +/// ``` +// FIXME: stabilization should modify documentation of `exists()` to recommend this method +// instead. +#[unstable(feature = "path_try_exists", issue = "83186")] +#[inline] +pub fn try_exists>(path: P) -> io::Result { + fs_imp::try_exists(path.as_ref()) +} diff --git a/library/std/src/io/buffered/bufwriter.rs b/library/std/src/io/buffered/bufwriter.rs index 80f98bbbad..ef2769d431 100644 --- a/library/std/src/io/buffered/bufwriter.rs +++ b/library/std/src/io/buffered/bufwriter.rs @@ -4,6 +4,7 @@ use crate::io::{ self, Error, ErrorKind, IntoInnerError, IoSlice, Seek, SeekFrom, Write, DEFAULT_BUF_SIZE, }; use crate::mem; +use crate::ptr; /// Wraps a writer and buffers its output. /// @@ -68,6 +69,10 @@ use crate::mem; #[stable(feature = "rust1", since = "1.0.0")] pub struct BufWriter { inner: Option, + // The buffer. Avoid using this like a normal `Vec` in common code paths. + // That is, don't use `buf.push`, `buf.extend_from_slice`, or any other + // methods that require bounds checking or the like. This makes an enormous + // difference to performance (we may want to stop using a `Vec` entirely). buf: Vec, // #30888: If the inner writer panics in a call to write, we don't want to // write the buffered data a second time in BufWriter's destructor. This @@ -181,9 +186,14 @@ impl BufWriter { /// data. Writes as much as possible without exceeding capacity. Returns /// the number of bytes written. pub(super) fn write_to_buf(&mut self, buf: &[u8]) -> usize { - let available = self.buf.capacity() - self.buf.len(); + let available = self.spare_capacity(); let amt_to_buffer = available.min(buf.len()); - self.buf.extend_from_slice(&buf[..amt_to_buffer]); + + // SAFETY: `amt_to_buffer` is <= buffer's spare capacity by construction. + unsafe { + self.write_to_buffer_unchecked(&buf[..amt_to_buffer]); + } + amt_to_buffer } @@ -331,6 +341,103 @@ impl BufWriter { let buf = if !self.panicked { Ok(buf) } else { Err(WriterPanicked { buf }) }; (self.inner.take().unwrap(), buf) } + + // Ensure this function does not get inlined into `write`, so that it + // remains inlineable and its common path remains as short as possible. + // If this function ends up being called frequently relative to `write`, + // it's likely a sign that the client is using an improperly sized buffer + // or their write patterns are somewhat pathological. + #[cold] + #[inline(never)] + fn write_cold(&mut self, buf: &[u8]) -> io::Result { + if buf.len() > self.spare_capacity() { + self.flush_buf()?; + } + + // Why not len > capacity? To avoid a needless trip through the buffer when the input + // exactly fills it. We'd just need to flush it to the underlying writer anyway. + if buf.len() >= self.buf.capacity() { + self.panicked = true; + let r = self.get_mut().write(buf); + self.panicked = false; + r + } else { + // Write to the buffer. In this case, we write to the buffer even if it fills it + // exactly. Doing otherwise would mean flushing the buffer, then writing this + // input to the inner writer, which in many cases would be a worse strategy. + + // SAFETY: There was either enough spare capacity already, or there wasn't and we + // flushed the buffer to ensure that there is. In the latter case, we know that there + // is because flushing ensured that our entire buffer is spare capacity, and we entered + // this block because the input buffer length is less than that capacity. In either + // case, it's safe to write the input buffer to our buffer. + unsafe { + self.write_to_buffer_unchecked(buf); + } + + Ok(buf.len()) + } + } + + // Ensure this function does not get inlined into `write_all`, so that it + // remains inlineable and its common path remains as short as possible. + // If this function ends up being called frequently relative to `write_all`, + // it's likely a sign that the client is using an improperly sized buffer + // or their write patterns are somewhat pathological. + #[cold] + #[inline(never)] + fn write_all_cold(&mut self, buf: &[u8]) -> io::Result<()> { + // Normally, `write_all` just calls `write` in a loop. We can do better + // by calling `self.get_mut().write_all()` directly, which avoids + // round trips through the buffer in the event of a series of partial + // writes in some circumstances. + + if buf.len() > self.spare_capacity() { + self.flush_buf()?; + } + + // Why not len > capacity? To avoid a needless trip through the buffer when the input + // exactly fills it. We'd just need to flush it to the underlying writer anyway. + if buf.len() >= self.buf.capacity() { + self.panicked = true; + let r = self.get_mut().write_all(buf); + self.panicked = false; + r + } else { + // Write to the buffer. In this case, we write to the buffer even if it fills it + // exactly. Doing otherwise would mean flushing the buffer, then writing this + // input to the inner writer, which in many cases would be a worse strategy. + + // SAFETY: There was either enough spare capacity already, or there wasn't and we + // flushed the buffer to ensure that there is. In the latter case, we know that there + // is because flushing ensured that our entire buffer is spare capacity, and we entered + // this block because the input buffer length is less than that capacity. In either + // case, it's safe to write the input buffer to our buffer. + unsafe { + self.write_to_buffer_unchecked(buf); + } + + Ok(()) + } + } + + // SAFETY: Requires `buf.len() <= self.buf.capacity() - self.buf.len()`, + // i.e., that input buffer length is less than or equal to spare capacity. + #[inline] + unsafe fn write_to_buffer_unchecked(&mut self, buf: &[u8]) { + debug_assert!(buf.len() <= self.spare_capacity()); + let old_len = self.buf.len(); + let buf_len = buf.len(); + let src = buf.as_ptr(); + let dst = self.buf.as_mut_ptr().add(old_len); + ptr::copy_nonoverlapping(src, dst, buf_len); + self.buf.set_len(old_len + buf_len); + } + + #[inline] + fn spare_capacity(&self) -> usize { + self.buf.capacity() - self.buf.len() + } } #[unstable(feature = "bufwriter_into_raw_parts", issue = "80690")] @@ -402,63 +509,82 @@ impl fmt::Debug for WriterPanicked { #[stable(feature = "rust1", since = "1.0.0")] impl Write for BufWriter { + #[inline] fn write(&mut self, buf: &[u8]) -> io::Result { - if self.buf.len() + buf.len() > self.buf.capacity() { - self.flush_buf()?; - } - // FIXME: Why no len > capacity? Why not buffer len == capacity? #72919 - if buf.len() >= self.buf.capacity() { - self.panicked = true; - let r = self.get_mut().write(buf); - self.panicked = false; - r - } else { - self.buf.extend_from_slice(buf); + // Use < instead of <= to avoid a needless trip through the buffer in some cases. + // See `write_cold` for details. + if buf.len() < self.spare_capacity() { + // SAFETY: safe by above conditional. + unsafe { + self.write_to_buffer_unchecked(buf); + } + Ok(buf.len()) + } else { + self.write_cold(buf) } } + #[inline] fn write_all(&mut self, buf: &[u8]) -> io::Result<()> { - // Normally, `write_all` just calls `write` in a loop. We can do better - // by calling `self.get_mut().write_all()` directly, which avoids - // round trips through the buffer in the event of a series of partial - // writes in some circumstances. - if self.buf.len() + buf.len() > self.buf.capacity() { - self.flush_buf()?; - } - // FIXME: Why no len > capacity? Why not buffer len == capacity? #72919 - if buf.len() >= self.buf.capacity() { - self.panicked = true; - let r = self.get_mut().write_all(buf); - self.panicked = false; - r - } else { - self.buf.extend_from_slice(buf); + // Use < instead of <= to avoid a needless trip through the buffer in some cases. + // See `write_all_cold` for details. + if buf.len() < self.spare_capacity() { + // SAFETY: safe by above conditional. + unsafe { + self.write_to_buffer_unchecked(buf); + } + Ok(()) + } else { + self.write_all_cold(buf) } } fn write_vectored(&mut self, bufs: &[IoSlice<'_>]) -> io::Result { + // FIXME: Consider applying `#[inline]` / `#[inline(never)]` optimizations already applied + // to `write` and `write_all`. The performance benefits can be significant. See #79930. if self.get_ref().is_write_vectored() { - let total_len = bufs.iter().map(|b| b.len()).sum::(); - if self.buf.len() + total_len > self.buf.capacity() { + // We have to handle the possibility that the total length of the buffers overflows + // `usize` (even though this can only happen if multiple `IoSlice`s reference the + // same underlying buffer, as otherwise the buffers wouldn't fit in memory). If the + // computation overflows, then surely the input cannot fit in our buffer, so we forward + // to the inner writer's `write_vectored` method to let it handle it appropriately. + let saturated_total_len = + bufs.iter().fold(0usize, |acc, b| acc.saturating_add(b.len())); + + if saturated_total_len > self.spare_capacity() { + // Flush if the total length of the input exceeds our buffer's spare capacity. + // If we would have overflowed, this condition also holds, and we need to flush. self.flush_buf()?; } - if total_len >= self.buf.capacity() { + + if saturated_total_len >= self.buf.capacity() { + // Forward to our inner writer if the total length of the input is greater than or + // equal to our buffer capacity. If we would have overflowed, this condition also + // holds, and we punt to the inner writer. self.panicked = true; let r = self.get_mut().write_vectored(bufs); self.panicked = false; r } else { - bufs.iter().for_each(|b| self.buf.extend_from_slice(b)); - Ok(total_len) + // `saturated_total_len < self.buf.capacity()` implies that we did not saturate. + + // SAFETY: We checked whether or not the spare capacity was large enough above. If + // it was, then we're safe already. If it wasn't, we flushed, making sufficient + // room for any input <= the buffer size, which includes this input. + unsafe { + bufs.iter().for_each(|b| self.write_to_buffer_unchecked(b)); + }; + + Ok(saturated_total_len) } } else { let mut iter = bufs.iter(); let mut total_written = if let Some(buf) = iter.by_ref().find(|&buf| !buf.is_empty()) { // This is the first non-empty slice to write, so if it does // not fit in the buffer, we still get to flush and proceed. - if self.buf.len() + buf.len() > self.buf.capacity() { + if buf.len() > self.spare_capacity() { self.flush_buf()?; } if buf.len() >= self.buf.capacity() { @@ -469,7 +595,13 @@ impl Write for BufWriter { self.panicked = false; return r; } else { - self.buf.extend_from_slice(buf); + // SAFETY: We checked whether or not the spare capacity was large enough above. + // If it was, then we're safe already. If it wasn't, we flushed, making + // sufficient room for any input <= the buffer size, which includes this input. + unsafe { + self.write_to_buffer_unchecked(buf); + } + buf.len() } } else { @@ -477,11 +609,18 @@ impl Write for BufWriter { }; debug_assert!(total_written != 0); for buf in iter { - if self.buf.len() + buf.len() > self.buf.capacity() { - break; - } else { - self.buf.extend_from_slice(buf); + if buf.len() <= self.spare_capacity() { + // SAFETY: safe by above conditional. + unsafe { + self.write_to_buffer_unchecked(buf); + } + + // This cannot overflow `usize`. If we are here, we've written all of the bytes + // so far to our buffer, and we've ensured that we never exceed the buffer's + // capacity. Therefore, `total_written` <= `self.buf.capacity()` <= `usize::MAX`. total_written += buf.len(); + } else { + break; } } Ok(total_written) diff --git a/library/std/src/io/buffered/linewritershim.rs b/library/std/src/io/buffered/linewritershim.rs index d0c859d2e0..0175d2693e 100644 --- a/library/std/src/io/buffered/linewritershim.rs +++ b/library/std/src/io/buffered/linewritershim.rs @@ -1,5 +1,5 @@ use crate::io::{self, BufWriter, IoSlice, Write}; -use crate::memchr; +use crate::sys_common::memchr; /// Private helper struct for implementing the line-buffered writing logic. /// This shim temporarily wraps a BufWriter, and uses its internals to diff --git a/library/std/src/io/cursor.rs b/library/std/src/io/cursor.rs index 9021b47006..9527254c94 100644 --- a/library/std/src/io/cursor.rs +++ b/library/std/src/io/cursor.rs @@ -71,7 +71,7 @@ use core::convert::TryInto; /// } /// ``` #[stable(feature = "rust1", since = "1.0.0")] -#[derive(Clone, Debug, Default, Eq, PartialEq)] +#[derive(Debug, Default, Eq, PartialEq)] pub struct Cursor { inner: T, pos: u64, @@ -205,6 +205,23 @@ impl Cursor { } } +#[stable(feature = "rust1", since = "1.0.0")] +impl Clone for Cursor +where + T: Clone, +{ + #[inline] + fn clone(&self) -> Self { + Cursor { inner: self.inner.clone(), pos: self.pos } + } + + #[inline] + fn clone_from(&mut self, other: &Self) { + self.inner.clone_from(&other.inner); + self.pos = other.pos; + } +} + #[stable(feature = "rust1", since = "1.0.0")] impl io::Seek for Cursor where diff --git a/library/std/src/io/error.rs b/library/std/src/io/error.rs index 9bed12bf2a..56e6f08268 100644 --- a/library/std/src/io/error.rs +++ b/library/std/src/io/error.rs @@ -186,6 +186,11 @@ pub enum ErrorKind { /// This means that the operation can never succeed. #[stable(feature = "unsupported_error", since = "1.53.0")] Unsupported, + + /// An operation could not be completed, because it failed + /// to allocate enough memory. + #[stable(feature = "out_of_memory_error", since = "1.54.0")] + OutOfMemory, } impl ErrorKind { @@ -210,6 +215,7 @@ impl ErrorKind { ErrorKind::Other => "other os error", ErrorKind::UnexpectedEof => "unexpected end of file", ErrorKind::Unsupported => "unsupported", + ErrorKind::OutOfMemory => "out of memory", } } } diff --git a/library/std/src/io/impls.rs b/library/std/src/io/impls.rs index 6891bd8a66..7a2a49ba7d 100644 --- a/library/std/src/io/impls.rs +++ b/library/std/src/io/impls.rs @@ -87,6 +87,11 @@ impl Seek for &mut S { fn seek(&mut self, pos: SeekFrom) -> io::Result { (**self).seek(pos) } + + #[inline] + fn stream_position(&mut self) -> io::Result { + (**self).stream_position() + } } #[stable(feature = "rust1", since = "1.0.0")] impl BufRead for &mut B { @@ -186,6 +191,11 @@ impl Seek for Box { fn seek(&mut self, pos: SeekFrom) -> io::Result { (**self).seek(pos) } + + #[inline] + fn stream_position(&mut self) -> io::Result { + (**self).stream_position() + } } #[stable(feature = "rust1", since = "1.0.0")] impl BufRead for Box { diff --git a/library/std/src/io/mod.rs b/library/std/src/io/mod.rs index 94c70c4f26..4c154dbe01 100644 --- a/library/std/src/io/mod.rs +++ b/library/std/src/io/mod.rs @@ -253,12 +253,12 @@ mod tests; use crate::cmp; use crate::fmt; -use crate::memchr; use crate::ops::{Deref, DerefMut}; use crate::ptr; use crate::slice; use crate::str; use crate::sys; +use crate::sys_common::memchr; #[stable(feature = "rust1", since = "1.0.0")] pub use self::buffered::IntoInnerError; @@ -509,8 +509,7 @@ pub(crate) fn default_read_exact(this: &mut R, mut buf: &mut [ /// [`std::io`]: self /// [`File`]: crate::fs::File #[stable(feature = "rust1", since = "1.0.0")] -#[cfg_attr(bootstrap, doc(spotlight))] -#[cfg_attr(not(bootstrap), doc(notable_trait))] +#[doc(notable_trait)] pub trait Read { /// Pull some bytes from this source into the specified buffer, returning /// how many bytes were read. @@ -526,7 +525,12 @@ pub trait Read { /// /// 1. This reader has reached its "end of file" and will likely no longer /// be able to produce bytes. Note that this does not mean that the - /// reader will *always* no longer be able to produce bytes. + /// reader will *always* no longer be able to produce bytes. As an example, + /// on Linux, this method will call the `recv` syscall for a [`TcpStream`], + /// where returning zero indicates the connection was shut down correctly. While + /// for [`File`], it is possible to reach the end of file and get zero as result, + /// but if more data is appended to the file, future calls to `read` will return + /// more data. /// 2. The buffer specified was 0 bytes in length. /// /// It is not an error if the returned value `n` is smaller than the buffer size, @@ -568,6 +572,7 @@ pub trait Read { /// /// [`Ok(n)`]: Ok /// [`File`]: crate::fs::File + /// [`TcpStream`]: crate::net::TcpStream /// /// ```no_run /// use std::io; @@ -1301,8 +1306,7 @@ impl Initializer { /// /// [`write_all`]: Write::write_all #[stable(feature = "rust1", since = "1.0.0")] -#[cfg_attr(bootstrap, doc(spotlight))] -#[cfg_attr(not(bootstrap), doc(notable_trait))] +#[doc(notable_trait)] pub trait Write { /// Write a buffer into this writer, returning how many bytes were written. /// @@ -1663,10 +1667,47 @@ pub trait Seek { /// /// # Errors /// + /// Seeking can fail, for example because it might involve flushing a buffer. + /// /// Seeking to a negative offset is considered an error. #[stable(feature = "rust1", since = "1.0.0")] fn seek(&mut self, pos: SeekFrom) -> Result; + /// Rewind to the beginning of a stream. + /// + /// This is a convenience method, equivalent to `seek(SeekFrom::Start(0))`. + /// + /// # Errors + /// + /// Rewinding can fail, for example because it might involve flushing a buffer. + /// + /// # Example + /// + /// ```no_run + /// #![feature(seek_rewind)] + /// use std::io::{Read, Seek, Write}; + /// use std::fs::OpenOptions; + /// + /// let mut f = OpenOptions::new() + /// .write(true) + /// .read(true) + /// .create(true) + /// .open("foo.txt").unwrap(); + /// + /// let hello = "Hello!\n"; + /// write!(f, "{}", hello).unwrap(); + /// f.rewind().unwrap(); + /// + /// let mut buf = String::new(); + /// f.read_to_string(&mut buf).unwrap(); + /// assert_eq!(&buf, hello); + /// ``` + #[unstable(feature = "seek_rewind", issue = "85149")] + fn rewind(&mut self) -> Result<()> { + self.seek(SeekFrom::Start(0))?; + Ok(()) + } + /// Returns the length of this stream (in bytes). /// /// This method is implemented using up to three seek operations. If this diff --git a/library/std/src/keyword_docs.rs b/library/std/src/keyword_docs.rs index 39ed62425c..5b8e83766f 100644 --- a/library/std/src/keyword_docs.rs +++ b/library/std/src/keyword_docs.rs @@ -987,13 +987,13 @@ mod mod_keyword {} /// Capture a [closure]'s environment by value. /// /// `move` converts any variables captured by reference or mutable reference -/// to owned by value variables. +/// to variables captured by value. /// /// ```rust -/// let capture = "hello"; -/// let closure = move || { -/// println!("rust says {}", capture); -/// }; +/// let data = vec![1, 2, 3]; +/// let closure = move || println!("captured {:?} by value", data); +/// +/// // data is no longer available, it is owned by the closure /// ``` /// /// Note: `move` closures may still implement [`Fn`] or [`FnMut`], even though @@ -1004,31 +1004,29 @@ mod mod_keyword {} /// ```rust /// fn create_fn() -> impl Fn() { /// let text = "Fn".to_owned(); -/// /// move || println!("This is a: {}", text) /// } /// -/// let fn_plain = create_fn(); -/// -/// fn_plain(); +/// let fn_plain = create_fn(); +/// fn_plain(); /// ``` /// /// `move` is often used when [threads] are involved. /// /// ```rust -/// let x = 5; +/// let data = vec![1, 2, 3]; /// /// std::thread::spawn(move || { -/// println!("captured {} by value", x) +/// println!("captured {:?} by value", data) /// }).join().unwrap(); /// -/// // x is no longer available +/// // data was moved to the spawned thread, so we cannot use it here /// ``` /// /// `move` is also valid before an async block. /// /// ```rust -/// let capture = "hello"; +/// let capture = "hello".to_owned(); /// let block = async move { /// println!("rust says {} from async block", capture); /// }; diff --git a/library/std/src/lib.rs b/library/std/src/lib.rs index 1b764d5a69..31dcd9546b 100644 --- a/library/std/src/lib.rs +++ b/library/std/src/lib.rs @@ -190,7 +190,6 @@ #![cfg_attr(not(feature = "restricted-std"), stable(feature = "rust1", since = "1.0.0"))] #![cfg_attr(feature = "restricted-std", unstable(feature = "restricted_std", issue = "none"))] #![doc( - html_root_url = "https://doc.rust-lang.org/nightly/", html_playground_url = "https://play.rust-lang.org/", issue_tracker_base_url = "https://github.com/rust-lang/rust/issues/", test(no_crate_inject, attr(deny(warnings))), @@ -230,6 +229,7 @@ #![feature(arbitrary_self_types)] #![feature(array_error_internals)] #![feature(asm)] +#![feature(assert_matches)] #![feature(associated_type_bounds)] #![feature(atomic_mut_ptr)] #![feature(bench_black_box)] @@ -245,7 +245,6 @@ #![feature(const_cstr_unchecked)] #![feature(const_fn_floating_point_arithmetic)] #![feature(const_fn_transmute)] -#![feature(const_fn)] #![feature(const_fn_fn_ptr_basics)] #![feature(const_io_structs)] #![feature(const_ip)] @@ -260,15 +259,14 @@ #![feature(doc_cfg)] #![feature(doc_keyword)] #![feature(doc_masked)] -#![cfg_attr(bootstrap, feature(doc_spotlight))] -#![cfg_attr(not(bootstrap), feature(doc_notable_trait))] +#![feature(doc_notable_trait)] #![feature(dropck_eyepatch)] #![feature(duration_constants)] #![feature(edition_panic)] #![feature(exact_size_is_empty)] #![feature(exhaustive_patterns)] #![feature(extend_one)] -#![feature(extended_key_value_attributes)] +#![cfg_attr(bootstrap, feature(extended_key_value_attributes))] #![feature(fn_traits)] #![feature(format_args_nl)] #![feature(gen_future)] @@ -300,7 +298,6 @@ #![feature(nonnull_slice_from_raw_parts)] #![feature(once_cell)] #![feature(auto_traits)] -#![cfg_attr(bootstrap, feature(or_patterns))] #![feature(panic_info_message)] #![feature(panic_internals)] #![feature(panic_unwind)] @@ -530,7 +527,6 @@ mod sys; pub mod alloc; // Private support modules -mod memchr; mod panicking; // The runtime entry point and a few unstable public functions used by the @@ -564,9 +560,9 @@ pub use core::{ #[stable(feature = "builtin_macro_prelude", since = "1.38.0")] #[allow(deprecated)] pub use core::{ - asm, assert, cfg, column, compile_error, concat, concat_idents, env, file, format_args, - format_args_nl, global_asm, include, include_bytes, include_str, line, llvm_asm, log_syntax, - module_path, option_env, stringify, trace_macros, + asm, assert, assert_matches, cfg, column, compile_error, concat, concat_idents, env, file, + format_args, format_args_nl, global_asm, include, include_bytes, include_str, line, llvm_asm, + log_syntax, module_path, option_env, stringify, trace_macros, }; #[stable(feature = "core_primitive", since = "1.43.0")] diff --git a/library/std/src/macros.rs b/library/std/src/macros.rs index b729349cf5..b2c5df5410 100644 --- a/library/std/src/macros.rs +++ b/library/std/src/macros.rs @@ -185,9 +185,10 @@ macro_rules! eprintln { /// builds or when debugging in release mode is significantly faster. /// /// Note that the macro is intended as a debugging tool and therefore you -/// should avoid having uses of it in version control for long periods. -/// Use cases involving debug output that should be added to version control -/// are better served by macros such as [`debug!`] from the [`log`] crate. +/// should avoid having uses of it in version control for long periods +/// (other than in tests and similar). +/// Debug output from production code is better done with other facilities +/// such as the [`debug!`] macro from the [`log`] crate. /// /// # Stability /// diff --git a/library/std/src/net/ip.rs b/library/std/src/net/ip.rs index 9b629e19be..2b6d0d7d5d 100644 --- a/library/std/src/net/ip.rs +++ b/library/std/src/net/ip.rs @@ -1,11 +1,3 @@ -#![unstable( - feature = "ip", - reason = "extra functionality has not been \ - scrutinized to the level that it should \ - be to be stable", - issue = "27709" -)] - // Tests for this module #[cfg(all(test, not(target_os = "emscripten")))] mod tests; @@ -126,6 +118,7 @@ pub struct Ipv6Addr { #[allow(missing_docs)] #[derive(Copy, PartialEq, Eq, Clone, Hash, Debug)] +#[unstable(feature = "ip", issue = "27709")] pub enum Ipv6MulticastScope { InterfaceLocal, LinkLocal, @@ -199,6 +192,7 @@ impl IpAddr { /// assert_eq!(IpAddr::V6(Ipv6Addr::new(0, 0, 0x1c9, 0, 0, 0xafc8, 0, 0x1)).is_global(), true); /// ``` #[rustc_const_unstable(feature = "const_ip", issue = "76205")] + #[unstable(feature = "ip", issue = "27709")] #[inline] pub const fn is_global(&self) -> bool { match self { @@ -249,6 +243,7 @@ impl IpAddr { /// ); /// ``` #[rustc_const_unstable(feature = "const_ip", issue = "76205")] + #[unstable(feature = "ip", issue = "27709")] #[inline] pub const fn is_documentation(&self) -> bool { match self { @@ -319,7 +314,7 @@ impl Ipv4Addr { Ipv4Addr { inner: c::in_addr { s_addr: u32::from_ne_bytes([a, b, c, d]) } } } - /// An IPv4 address with the address pointing to localhost: 127.0.0.1. + /// An IPv4 address with the address pointing to localhost: `127.0.0.1` /// /// # Examples /// @@ -332,7 +327,7 @@ impl Ipv4Addr { #[stable(feature = "ip_constructors", since = "1.30.0")] pub const LOCALHOST: Self = Ipv4Addr::new(127, 0, 0, 1); - /// An IPv4 address representing an unspecified address: 0.0.0.0 + /// An IPv4 address representing an unspecified address: `0.0.0.0` /// /// This corresponds to the constant `INADDR_ANY` in other languages. /// @@ -348,7 +343,7 @@ impl Ipv4Addr { #[stable(feature = "ip_constructors", since = "1.30.0")] pub const UNSPECIFIED: Self = Ipv4Addr::new(0, 0, 0, 0); - /// An IPv4 address representing the broadcast address: 255.255.255.255 + /// An IPv4 address representing the broadcast address: `255.255.255.255` /// /// # Examples /// @@ -379,7 +374,7 @@ impl Ipv4Addr { self.inner.s_addr.to_ne_bytes() } - /// Returns [`true`] for the special 'unspecified' address (0.0.0.0). + /// Returns [`true`] for the special 'unspecified' address (`0.0.0.0`). /// /// This property is defined in _UNIX Network Programming, Second Edition_, /// W. Richard Stevens, p. 891; see also [ip7]. @@ -401,7 +396,7 @@ impl Ipv4Addr { self.inner.s_addr == 0 } - /// Returns [`true`] if this is a loopback address (127.0.0.0/8). + /// Returns [`true`] if this is a loopback address (`127.0.0.0/8`). /// /// This property is defined by [IETF RFC 1122]. /// @@ -426,9 +421,9 @@ impl Ipv4Addr { /// /// The private address ranges are defined in [IETF RFC 1918] and include: /// - /// - 10.0.0.0/8 - /// - 172.16.0.0/12 - /// - 192.168.0.0/16 + /// - `10.0.0.0/8` + /// - `172.16.0.0/12` + /// - `192.168.0.0/16` /// /// [IETF RFC 1918]: https://tools.ietf.org/html/rfc1918 /// @@ -457,7 +452,7 @@ impl Ipv4Addr { } } - /// Returns [`true`] if the address is link-local (169.254.0.0/16). + /// Returns [`true`] if the address is link-local (`169.254.0.0/16`). /// /// This property is defined by [IETF RFC 3927]. /// @@ -490,7 +485,7 @@ impl Ipv4Addr { /// - the broadcast address (see [`Ipv4Addr::is_broadcast()`]) /// - addresses used for documentation (see [`Ipv4Addr::is_documentation()`]) /// - the unspecified address (see [`Ipv4Addr::is_unspecified()`]), and the whole - /// 0.0.0.0/8 block + /// `0.0.0.0/8` block /// - addresses reserved for future protocols (see /// [`Ipv4Addr::is_ietf_protocol_assignment()`], except /// `192.0.0.9/32` and `192.0.0.10/32` which are globally routable @@ -549,6 +544,7 @@ impl Ipv4Addr { /// assert_eq!(Ipv4Addr::new(80, 9, 12, 3).is_global(), true); /// ``` #[rustc_const_unstable(feature = "const_ipv4", issue = "76205")] + #[unstable(feature = "ip", issue = "27709")] #[inline] pub const fn is_global(&self) -> bool { // check if this address is 192.0.0.9 or 192.0.0.10. These addresses are the only two @@ -587,6 +583,7 @@ impl Ipv4Addr { /// assert_eq!(Ipv4Addr::new(100, 128, 0, 0).is_shared(), false); /// ``` #[rustc_const_unstable(feature = "const_ipv4", issue = "76205")] + #[unstable(feature = "ip", issue = "27709")] #[inline] pub const fn is_shared(&self) -> bool { self.octets()[0] == 100 && (self.octets()[1] & 0b1100_0000 == 0b0100_0000) @@ -620,6 +617,7 @@ impl Ipv4Addr { /// assert_eq!(Ipv4Addr::new(191, 255, 255, 255).is_ietf_protocol_assignment(), false); /// ``` #[rustc_const_unstable(feature = "const_ipv4", issue = "76205")] + #[unstable(feature = "ip", issue = "27709")] #[inline] pub const fn is_ietf_protocol_assignment(&self) -> bool { self.octets()[0] == 192 && self.octets()[1] == 0 && self.octets()[2] == 0 @@ -644,6 +642,7 @@ impl Ipv4Addr { /// assert_eq!(Ipv4Addr::new(198, 20, 0, 0).is_benchmarking(), false); /// ``` #[rustc_const_unstable(feature = "const_ipv4", issue = "76205")] + #[unstable(feature = "ip", issue = "27709")] #[inline] pub const fn is_benchmarking(&self) -> bool { self.octets()[0] == 198 && (self.octets()[1] & 0xfe) == 18 @@ -677,14 +676,15 @@ impl Ipv4Addr { /// assert_eq!(Ipv4Addr::new(255, 255, 255, 255).is_reserved(), false); /// ``` #[rustc_const_unstable(feature = "const_ipv4", issue = "76205")] + #[unstable(feature = "ip", issue = "27709")] #[inline] pub const fn is_reserved(&self) -> bool { self.octets()[0] & 240 == 240 && !self.is_broadcast() } - /// Returns [`true`] if this is a multicast address (224.0.0.0/4). + /// Returns [`true`] if this is a multicast address (`224.0.0.0/4`). /// - /// Multicast addresses have a most significant octet between 224 and 239, + /// Multicast addresses have a most significant octet between `224` and `239`, /// and is defined by [IETF RFC 5771]. /// /// [IETF RFC 5771]: https://tools.ietf.org/html/rfc5771 @@ -705,9 +705,9 @@ impl Ipv4Addr { self.octets()[0] >= 224 && self.octets()[0] <= 239 } - /// Returns [`true`] if this is a broadcast address (255.255.255.255). + /// Returns [`true`] if this is a broadcast address (`255.255.255.255`). /// - /// A broadcast address has all octets set to 255 as defined in [IETF RFC 919]. + /// A broadcast address has all octets set to `255` as defined in [IETF RFC 919]. /// /// [IETF RFC 919]: https://tools.ietf.org/html/rfc919 /// @@ -730,9 +730,9 @@ impl Ipv4Addr { /// /// This is defined in [IETF RFC 5737]: /// - /// - 192.0.2.0/24 (TEST-NET-1) - /// - 198.51.100.0/24 (TEST-NET-2) - /// - 203.0.113.0/24 (TEST-NET-3) + /// - `192.0.2.0/24` (TEST-NET-1) + /// - `198.51.100.0/24` (TEST-NET-2) + /// - `203.0.113.0/24` (TEST-NET-3) /// /// [IETF RFC 5737]: https://tools.ietf.org/html/rfc5737 /// @@ -760,7 +760,7 @@ impl Ipv4Addr { /// Converts this address to an IPv4-compatible [`IPv6` address]. /// - /// a.b.c.d becomes ::a.b.c.d + /// `a.b.c.d` becomes `::a.b.c.d` /// /// This isn't typically the method you want; these addresses don't typically /// function on modern systems. Use `to_ipv6_mapped` instead. @@ -774,7 +774,7 @@ impl Ipv4Addr { /// /// assert_eq!( /// Ipv4Addr::new(192, 0, 2, 255).to_ipv6_compatible(), - /// Ipv6Addr::new(0, 0, 0, 0, 0, 0, 49152, 767) + /// Ipv6Addr::new(0, 0, 0, 0, 0, 0, 0xc000, 0x2ff) /// ); /// ``` #[rustc_const_stable(feature = "const_ipv4", since = "1.50.0")] @@ -789,7 +789,7 @@ impl Ipv4Addr { /// Converts this address to an IPv4-mapped [`IPv6` address]. /// - /// a.b.c.d becomes ::ffff:a.b.c.d + /// `a.b.c.d` becomes `::ffff:a.b.c.d` /// /// [`IPv6` address]: Ipv6Addr /// @@ -799,7 +799,7 @@ impl Ipv4Addr { /// use std::net::{Ipv4Addr, Ipv6Addr}; /// /// assert_eq!(Ipv4Addr::new(192, 0, 2, 255).to_ipv6_mapped(), - /// Ipv6Addr::new(0, 0, 0, 0, 0, 65535, 49152, 767)); + /// Ipv6Addr::new(0, 0, 0, 0, 0, 0xffff, 0xc000, 0x2ff)); /// ``` #[rustc_const_stable(feature = "const_ipv4", since = "1.50.0")] #[stable(feature = "rust1", since = "1.0.0")] @@ -1172,7 +1172,7 @@ impl Ipv6Addr { ] } - /// Returns [`true`] for the special 'unspecified' address (::). + /// Returns [`true`] for the special 'unspecified' address (`::`). /// /// This property is defined in [IETF RFC 4291]. /// @@ -1234,6 +1234,7 @@ impl Ipv6Addr { /// assert_eq!(Ipv6Addr::new(0, 0, 0x1c9, 0, 0, 0xafc8, 0, 0x1).is_global(), true); /// ``` #[rustc_const_unstable(feature = "const_ipv6", issue = "76205")] + #[unstable(feature = "ip", issue = "27709")] #[inline] pub const fn is_global(&self) -> bool { match self.multicast_scope() { @@ -1260,27 +1261,17 @@ impl Ipv6Addr { /// assert_eq!(Ipv6Addr::new(0xfc02, 0, 0, 0, 0, 0, 0, 0).is_unique_local(), true); /// ``` #[rustc_const_unstable(feature = "const_ipv6", issue = "76205")] + #[unstable(feature = "ip", issue = "27709")] #[inline] pub const fn is_unique_local(&self) -> bool { (self.segments()[0] & 0xfe00) == 0xfc00 } - /// Returns [`true`] if the address is a unicast link-local address (`fe80::/64`). - /// - /// A common misconception is to think that "unicast link-local addresses start with - /// `fe80::`", but [IETF RFC 4291] actually defines a stricter format for these addresses: + /// Returns [`true`] if this is a unicast address, as defined by [IETF RFC 4291]. + /// Any address that is not a [multicast address] (`ff00::/8`) is unicast. /// - /// ```no_rust - /// | 10 | - /// | bits | 54 bits | 64 bits | - /// +----------+-------------------------+----------------------------+ - /// |1111111010| 0 | interface ID | - /// +----------+-------------------------+----------------------------+ - /// ``` - /// - /// This method validates the format defined in the RFC and won't recognize addresses - /// like `fe80:0:0:1::` or `fe81::` as unicast link-local addresses. - /// If you need a less strict validation, use [`Ipv6Addr::is_unicast_link_local()`] instead. + /// [IETF RFC 4291]: https://tools.ietf.org/html/rfc4291 + /// [multicast address]: Ipv6Addr::is_multicast /// /// # Examples /// @@ -1289,54 +1280,46 @@ impl Ipv6Addr { /// /// use std::net::Ipv6Addr; /// - /// let ip = Ipv6Addr::new(0xfe80, 0, 0, 0, 0, 0, 0, 0); - /// assert!(ip.is_unicast_link_local_strict()); + /// // The unspecified and loopback addresses are unicast. + /// assert_eq!(Ipv6Addr::UNSPECIFIED.is_unicast(), true); + /// assert_eq!(Ipv6Addr::LOCALHOST.is_unicast(), true); /// - /// let ip = Ipv6Addr::new(0xfe80, 0, 0, 0, 0xffff, 0xffff, 0xffff, 0xffff); - /// assert!(ip.is_unicast_link_local_strict()); - /// - /// let ip = Ipv6Addr::new(0xfe80, 0, 0, 1, 0, 0, 0, 0); - /// assert!(!ip.is_unicast_link_local_strict()); - /// assert!(ip.is_unicast_link_local()); - /// - /// let ip = Ipv6Addr::new(0xfe81, 0, 0, 0, 0, 0, 0, 0); - /// assert!(!ip.is_unicast_link_local_strict()); - /// assert!(ip.is_unicast_link_local()); + /// // Any address that is not a multicast address (`ff00::/8`) is unicast. + /// assert_eq!(Ipv6Addr::new(0x2001, 0xdb8, 0, 0, 0, 0, 0, 0).is_unicast(), true); + /// assert_eq!(Ipv6Addr::new(0xff00, 0, 0, 0, 0, 0, 0, 0).is_unicast(), false); /// ``` - /// - /// # See also - /// - /// - [IETF RFC 4291 section 2.5.6] - /// - [RFC 4291 errata 4406] (which has been rejected but provides useful - /// insight) - /// - [`Ipv6Addr::is_unicast_link_local()`] - /// - /// [IETF RFC 4291]: https://tools.ietf.org/html/rfc4291 - /// [IETF RFC 4291 section 2.5.6]: https://tools.ietf.org/html/rfc4291#section-2.5.6 - /// [RFC 4291 errata 4406]: https://www.rfc-editor.org/errata/eid4406 #[rustc_const_unstable(feature = "const_ipv6", issue = "76205")] + #[unstable(feature = "ip", issue = "27709")] #[inline] - pub const fn is_unicast_link_local_strict(&self) -> bool { - matches!(self.segments(), [0xfe80, 0, 0, 0, ..]) + pub const fn is_unicast(&self) -> bool { + !self.is_multicast() } - /// Returns [`true`] if the address is a unicast link-local address (`fe80::/10`). + /// Returns `true` if the address is a unicast address with link-local scope, + /// as defined in [RFC 4291]. /// - /// This method returns [`true`] for addresses in the range reserved by [RFC 4291 section 2.4], - /// i.e. addresses with the following format: + /// A unicast address has link-local scope if it has the prefix `fe80::/10`, as per [RFC 4291 section 2.4]. + /// Note that this encompasses more addresses than those defined in [RFC 4291 section 2.5.6], + /// which describes "Link-Local IPv6 Unicast Addresses" as having the following stricter format: /// - /// ```no_rust - /// | 10 | - /// | bits | 54 bits | 64 bits | + /// ```text + /// | 10 bits | 54 bits | 64 bits | /// +----------+-------------------------+----------------------------+ - /// |1111111010| arbitratry value | interface ID | + /// |1111111010| 0 | interface ID | /// +----------+-------------------------+----------------------------+ /// ``` + /// So while currently the only addresses with link-local scope an application will encounter are all in `fe80::/64`, + /// this might change in the future with the publication of new standards. More addresses in `fe80::/10` could be allocated, + /// and those addresses will have link-local scope. /// - /// As a result, this method considers addresses such as `fe80:0:0:1::` or `fe81::` to be - /// unicast link-local addresses, whereas [`Ipv6Addr::is_unicast_link_local_strict()`] does not. - /// If you need a strict validation fully compliant with the RFC, use - /// [`Ipv6Addr::is_unicast_link_local_strict()`] instead. + /// Also note that while [RFC 4291 section 2.5.3] mentions about the [loopback address] (`::1`) that "it is treated as having Link-Local scope", + /// this does not mean that the loopback address actually has link-local scope and this method will return `false` on it. + /// + /// [RFC 4291]: https://tools.ietf.org/html/rfc4291 + /// [RFC 4291 section 2.4]: https://tools.ietf.org/html/rfc4291#section-2.4 + /// [RFC 4291 section 2.5.3]: https://tools.ietf.org/html/rfc4291#section-2.5.3 + /// [RFC 4291 section 2.5.6]: https://tools.ietf.org/html/rfc4291#section-2.5.6 + /// [loopback address]: Ipv6Addr::LOCALHOST /// /// # Examples /// @@ -1345,36 +1328,25 @@ impl Ipv6Addr { /// /// use std::net::Ipv6Addr; /// - /// let ip = Ipv6Addr::new(0xfe80, 0, 0, 0, 0, 0, 0, 0); - /// assert!(ip.is_unicast_link_local()); - /// - /// let ip = Ipv6Addr::new(0xfe80, 0, 0, 0, 0xffff, 0xffff, 0xffff, 0xffff); - /// assert!(ip.is_unicast_link_local()); + /// // The loopback address (`::1`) does not actually have link-local scope. + /// assert_eq!(Ipv6Addr::LOCALHOST.is_unicast_link_local(), false); /// - /// let ip = Ipv6Addr::new(0xfe80, 0, 0, 1, 0, 0, 0, 0); - /// assert!(ip.is_unicast_link_local()); - /// assert!(!ip.is_unicast_link_local_strict()); + /// // Only addresses in `fe80::/10` have link-local scope. + /// assert_eq!(Ipv6Addr::new(0x2001, 0xdb8, 0, 0, 0, 0, 0, 0).is_unicast_link_local(), false); + /// assert_eq!(Ipv6Addr::new(0xfe80, 0, 0, 0, 0, 0, 0, 0).is_unicast_link_local(), true); /// - /// let ip = Ipv6Addr::new(0xfe81, 0, 0, 0, 0, 0, 0, 0); - /// assert!(ip.is_unicast_link_local()); - /// assert!(!ip.is_unicast_link_local_strict()); + /// // Addresses outside the stricter `fe80::/64` also have link-local scope. + /// assert_eq!(Ipv6Addr::new(0xfe80, 0, 0, 1, 0, 0, 0, 0).is_unicast_link_local(), true); + /// assert_eq!(Ipv6Addr::new(0xfe81, 0, 0, 0, 0, 0, 0, 0).is_unicast_link_local(), true); /// ``` - /// - /// # See also - /// - /// - [IETF RFC 4291 section 2.4] - /// - [RFC 4291 errata 4406] (which has been rejected but provides useful - /// insight) - /// - /// [IETF RFC 4291 section 2.4]: https://tools.ietf.org/html/rfc4291#section-2.4 - /// [RFC 4291 errata 4406]: https://www.rfc-editor.org/errata/eid4406 #[rustc_const_unstable(feature = "const_ipv6", issue = "76205")] + #[unstable(feature = "ip", issue = "27709")] #[inline] pub const fn is_unicast_link_local(&self) -> bool { (self.segments()[0] & 0xffc0) == 0xfe80 } - /// Returns [`true`] if this is a deprecated unicast site-local address (fec0::/10). The + /// Returns [`true`] if this is a deprecated unicast site-local address (`fec0::/10`). The /// unicast site-local address format is defined in [RFC 4291 section 2.5.7] as: /// /// ```no_rust @@ -1403,12 +1375,13 @@ impl Ipv6Addr { /// /// # Warning /// - /// As per [RFC 3879], the whole `FEC0::/10` prefix is + /// As per [RFC 3879], the whole `fec0::/10` prefix is /// deprecated. New software must not support site-local /// addresses. /// /// [RFC 3879]: https://tools.ietf.org/html/rfc3879 #[rustc_const_unstable(feature = "const_ipv6", issue = "76205")] + #[unstable(feature = "ip", issue = "27709")] #[inline] pub const fn is_unicast_site_local(&self) -> bool { (self.segments()[0] & 0xffc0) == 0xfec0 @@ -1432,6 +1405,7 @@ impl Ipv6Addr { /// assert_eq!(Ipv6Addr::new(0x2001, 0xdb8, 0, 0, 0, 0, 0, 0).is_documentation(), true); /// ``` #[rustc_const_unstable(feature = "const_ipv6", issue = "76205")] + #[unstable(feature = "ip", issue = "27709")] #[inline] pub const fn is_documentation(&self) -> bool { (self.segments()[0] == 0x2001) && (self.segments()[1] == 0xdb8) @@ -1468,9 +1442,10 @@ impl Ipv6Addr { /// assert_eq!(Ipv6Addr::new(0, 0, 0, 0, 0, 0xffff, 0xc00a, 0x2ff).is_unicast_global(), true); /// ``` #[rustc_const_unstable(feature = "const_ipv6", issue = "76205")] + #[unstable(feature = "ip", issue = "27709")] #[inline] pub const fn is_unicast_global(&self) -> bool { - !self.is_multicast() + self.is_unicast() && !self.is_loopback() && !self.is_unicast_link_local() && !self.is_unique_local() @@ -1494,6 +1469,7 @@ impl Ipv6Addr { /// assert_eq!(Ipv6Addr::new(0, 0, 0, 0, 0, 0xffff, 0xc00a, 0x2ff).multicast_scope(), None); /// ``` #[rustc_const_unstable(feature = "const_ipv6", issue = "76205")] + #[unstable(feature = "ip", issue = "27709")] #[inline] pub const fn multicast_scope(&self) -> Option { if self.is_multicast() { @@ -1512,7 +1488,7 @@ impl Ipv6Addr { } } - /// Returns [`true`] if this is a multicast address (ff00::/8). + /// Returns [`true`] if this is a multicast address (`ff00::/8`). /// /// This property is defined by [IETF RFC 4291]. /// @@ -1555,6 +1531,7 @@ impl Ipv6Addr { /// assert_eq!(Ipv6Addr::new(0, 0, 0, 0, 0, 0, 0, 1).to_ipv4_mapped(), None); /// ``` #[rustc_const_unstable(feature = "const_ipv6", issue = "76205")] + #[unstable(feature = "ip", issue = "27709")] #[inline] pub const fn to_ipv4_mapped(&self) -> Option { match self.octets() { @@ -1568,7 +1545,7 @@ impl Ipv6Addr { /// Converts this address to an [`IPv4` address]. Returns [`None`] if this address is /// neither IPv4-compatible or IPv4-mapped. /// - /// ::a.b.c.d and ::ffff:a.b.c.d become a.b.c.d + /// `::a.b.c.d` and `::ffff:a.b.c.d` become `a.b.c.d` /// /// [`IPv4` address]: Ipv4Addr /// diff --git a/library/std/src/net/ip/tests.rs b/library/std/src/net/ip/tests.rs index ef0d4edc43..05f8dea0b7 100644 --- a/library/std/src/net/ip/tests.rs +++ b/library/std/src/net/ip/tests.rs @@ -480,7 +480,6 @@ fn ipv6_properties() { let unique_local: u16 = 1 << 2; let global: u16 = 1 << 3; let unicast_link_local: u16 = 1 << 4; - let unicast_link_local_strict: u16 = 1 << 5; let unicast_site_local: u16 = 1 << 6; let unicast_global: u16 = 1 << 7; let documentation: u16 = 1 << 8; @@ -524,11 +523,6 @@ fn ipv6_properties() { } else { assert!(!ip!($s).is_unicast_link_local()); } - if ($mask & unicast_link_local_strict) == unicast_link_local_strict { - assert!(ip!($s).is_unicast_link_local_strict()); - } else { - assert!(!ip!($s).is_unicast_link_local_strict()); - } if ($mask & unicast_site_local) == unicast_site_local { assert!(ip!($s).is_unicast_site_local()); } else { @@ -587,7 +581,6 @@ fn ipv6_properties() { let unique_local: u16 = 1 << 2; let global: u16 = 1 << 3; let unicast_link_local: u16 = 1 << 4; - let unicast_link_local_strict: u16 = 1 << 5; let unicast_site_local: u16 = 1 << 6; let unicast_global: u16 = 1 << 7; let documentation: u16 = 1 << 8; @@ -621,11 +614,7 @@ fn ipv6_properties() { unicast_link_local ); - check!( - "fe80::", - &[0xfe, 0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - unicast_link_local | unicast_link_local_strict - ); + check!("fe80::", &[0xfe, 0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], unicast_link_local); check!( "febf:ffff::", @@ -650,7 +639,7 @@ fn ipv6_properties() { 0xfe, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff ], - unicast_link_local | unicast_link_local_strict + unicast_link_local ); check!( @@ -897,9 +886,6 @@ fn ipv6_const() { const IS_UNIQUE_LOCAL: bool = IP_ADDRESS.is_unique_local(); assert!(!IS_UNIQUE_LOCAL); - const IS_UNICAST_LINK_LOCAL_STRICT: bool = IP_ADDRESS.is_unicast_link_local_strict(); - assert!(!IS_UNICAST_LINK_LOCAL_STRICT); - const IS_UNICAST_LINK_LOCAL: bool = IP_ADDRESS.is_unicast_link_local(); assert!(!IS_UNICAST_LINK_LOCAL); diff --git a/library/std/src/net/udp.rs b/library/std/src/net/udp.rs index b9af5992df..18297139b7 100644 --- a/library/std/src/net/udp.rs +++ b/library/std/src/net/udp.rs @@ -159,7 +159,7 @@ impl UdpSocket { /// This will return an error when the IP version of the local socket /// does not match that returned from [`ToSocketAddrs`]. /// - /// See issue #34202 for more details. + /// See [Issue #34202] for more details. /// /// # Examples /// @@ -169,6 +169,8 @@ impl UdpSocket { /// let socket = UdpSocket::bind("127.0.0.1:34254").expect("couldn't bind to address"); /// socket.send_to(&[0; 10], "127.0.0.1:4242").expect("couldn't send data"); /// ``` + /// + /// [Issue #34202]: https://github.com/rust-lang/rust/issues/34202 #[stable(feature = "rust1", since = "1.0.0")] pub fn send_to(&self, buf: &[u8], addr: A) -> io::Result { match addr.to_socket_addrs()?.next() { diff --git a/library/std/src/os/emscripten/raw.rs b/library/std/src/os/emscripten/raw.rs index dda7c82525..503645c08c 100644 --- a/library/std/src/os/emscripten/raw.rs +++ b/library/std/src/os/emscripten/raw.rs @@ -22,7 +22,6 @@ pub type mode_t = u32; #[stable(feature = "pthread_t", since = "1.8.0")] pub type pthread_t = c_ulong; -#[doc(inline)] #[stable(feature = "raw_ext", since = "1.1.0")] pub type blkcnt_t = u64; #[stable(feature = "raw_ext", since = "1.1.0")] diff --git a/library/std/src/sys/sgx/ext/arch.rs b/library/std/src/os/fortanix_sgx/arch.rs similarity index 88% rename from library/std/src/sys/sgx/ext/arch.rs rename to library/std/src/os/fortanix_sgx/arch.rs index 730db34e73..b0170e6744 100644 --- a/library/std/src/sys/sgx/ext/arch.rs +++ b/library/std/src/os/fortanix_sgx/arch.rs @@ -32,9 +32,12 @@ pub fn egetkey(request: &Align512<[u8; 512]>) -> Result, u32> let error; asm!( + // rbx is reserved by LLVM + "xchg {0}, rbx", "enclu", + "mov rbx, {0}", + inout(reg) request => _, inlateout("eax") ENCLU_EGETKEY => error, - in("rbx") request, in("rcx") out.as_mut_ptr(), options(nostack), ); @@ -60,9 +63,12 @@ pub fn ereport( let mut report = MaybeUninit::uninit(); asm!( + // rbx is reserved by LLVM + "xchg {0}, rbx", "enclu", + "mov rbx, {0}", + inout(reg) targetinfo => _, in("eax") ENCLU_EREPORT, - in("rbx") targetinfo, in("rcx") reportdata, in("rdx") report.as_mut_ptr(), options(preserves_flags, nostack), diff --git a/library/std/src/sys/sgx/ext/ffi.rs b/library/std/src/os/fortanix_sgx/ffi.rs similarity index 100% rename from library/std/src/sys/sgx/ext/ffi.rs rename to library/std/src/os/fortanix_sgx/ffi.rs diff --git a/library/std/src/sys/sgx/ext/io.rs b/library/std/src/os/fortanix_sgx/io.rs similarity index 100% rename from library/std/src/sys/sgx/ext/io.rs rename to library/std/src/os/fortanix_sgx/io.rs diff --git a/library/std/src/os/fortanix_sgx/mod.rs b/library/std/src/os/fortanix_sgx/mod.rs index 69923268e5..a40dabe190 100644 --- a/library/std/src/os/fortanix_sgx/mod.rs +++ b/library/std/src/os/fortanix_sgx/mod.rs @@ -3,7 +3,7 @@ //! This includes functions to deal with memory isolation, usercalls, and the //! SGX instruction set. -#![deny(missing_docs, missing_debug_implementations)] +#![deny(missing_docs)] #![unstable(feature = "sgx_platform", issue = "56975")] /// Low-level interfaces to usercalls. See the [ABI documentation] for more @@ -43,7 +43,9 @@ pub mod mem { pub use crate::sys::abi::mem::*; } -pub use crate::sys::ext::{arch, ffi, io}; +pub mod arch; +pub mod ffi; +pub mod io; /// Functions for querying thread-related information. pub mod thread { diff --git a/library/std/src/sys/hermit/ext/ffi.rs b/library/std/src/os/hermit/ffi.rs similarity index 100% rename from library/std/src/sys/hermit/ext/ffi.rs rename to library/std/src/os/hermit/ffi.rs diff --git a/library/std/src/sys/hermit/ext/mod.rs b/library/std/src/os/hermit/mod.rs similarity index 94% rename from library/std/src/sys/hermit/ext/mod.rs rename to library/std/src/os/hermit/mod.rs index ea87d0ad2c..4657b545a1 100644 --- a/library/std/src/sys/hermit/ext/mod.rs +++ b/library/std/src/os/hermit/mod.rs @@ -1,5 +1,4 @@ #![stable(feature = "rust1", since = "1.0.0")] -#![allow(missing_docs)] pub mod ffi; diff --git a/library/std/src/os/linux/mod.rs b/library/std/src/os/linux/mod.rs index f179a52433..94438defc2 100644 --- a/library/std/src/os/linux/mod.rs +++ b/library/std/src/os/linux/mod.rs @@ -1,6 +1,7 @@ //! Linux-specific definitions. #![stable(feature = "raw_ext", since = "1.1.0")] +#![doc(cfg(target_os = "linux"))] pub mod fs; pub mod raw; diff --git a/library/std/src/os/linux/raw.rs b/library/std/src/os/linux/raw.rs index 525102212c..5b68a7e126 100644 --- a/library/std/src/os/linux/raw.rs +++ b/library/std/src/os/linux/raw.rs @@ -9,7 +9,6 @@ definitions" )] #![allow(deprecated)] -#![allow(missing_debug_implementations)] use crate::os::raw::c_ulong; diff --git a/library/std/src/os/mod.rs b/library/std/src/os/mod.rs index b95511e43d..07e29ebf36 100644 --- a/library/std/src/os/mod.rs +++ b/library/std/src/os/mod.rs @@ -3,78 +3,119 @@ #![stable(feature = "os", since = "1.0.0")] #![allow(missing_docs, nonstandard_style, missing_debug_implementations)] -// When documenting libstd we want to show unix/windows/linux/wasi modules as these are the "main -// modules" that are used across platforms, so all modules are enabled when `cfg(doc)` is set. -// This should help show platform-specific functionality in a hopefully cross-platform way in the -// documentation. -// Note that we deliberately avoid `cfg_if!` here to work around a rust-analyzer bug that would make -// `std::os` submodules unusable: https://github.com/rust-analyzer/rust-analyzer/issues/6038 +pub mod raw; -#[cfg(doc)] -#[stable(feature = "rust1", since = "1.0.0")] -pub use crate::sys::unix_ext as unix; +// The code below could be written clearer using `cfg_if!`. However, the items below are +// publicly exported by `std` and external tools can have trouble analysing them because of the use +// of a macro that is not vendored by Rust and included in the toolchain. +// See https://github.com/rust-analyzer/rust-analyzer/issues/6038. -#[cfg(doc)] -#[stable(feature = "rust1", since = "1.0.0")] -pub use crate::sys::windows_ext as windows; +#[cfg(all( + doc, + not(any( + all(target_arch = "wasm32", not(target_os = "wasi")), + all(target_vendor = "fortanix", target_env = "sgx") + )) +))] +#[path = "."] +mod doc { + // When documenting std we want to show the `unix`, `windows`, `linux` and `wasi` + // modules as these are the "main modules" that are used across platforms, + // so these modules are enabled when `cfg(doc)` is set. + // This should help show platform-specific functionality in a hopefully cross-platform + // way in the documentation. -#[cfg(doc)] -#[doc(cfg(target_os = "linux"))] -pub mod linux; + pub mod unix; -#[cfg(doc)] -#[stable(feature = "wasi_ext_doc", since = "1.35.0")] -pub use crate::sys::wasi_ext as wasi; + pub mod linux; -// If we're not documenting libstd then we just expose the main modules as we otherwise would. + pub mod wasi; -#[cfg(not(doc))] -#[cfg(any(unix, target_os = "hermit"))] -#[stable(feature = "rust1", since = "1.0.0")] -pub use crate::sys::ext as unix; + pub mod windows; +} +#[cfg(all( + doc, + any( + all(target_arch = "wasm32", not(target_os = "wasi")), + all(target_vendor = "fortanix", target_env = "sgx") + ) +))] +mod doc { + // On certain platforms right now the "main modules" modules that are + // documented don't compile (missing things in `libc` which is empty), + // so just omit them with an empty module. -#[cfg(not(doc))] -#[cfg(windows)] -#[stable(feature = "rust1", since = "1.0.0")] -pub use crate::sys::ext as windows; + #[unstable(issue = "none", feature = "std_internals")] + pub mod unix {} -#[cfg(not(doc))] -#[cfg(any(target_os = "linux", target_os = "l4re"))] -pub mod linux; + #[unstable(issue = "none", feature = "std_internals")] + pub mod linux {} + + #[unstable(issue = "none", feature = "std_internals")] + pub mod wasi {} + + #[unstable(issue = "none", feature = "std_internals")] + pub mod windows {} +} +#[cfg(doc)] +#[stable(feature = "os", since = "1.0.0")] +pub use doc::*; #[cfg(not(doc))] -#[cfg(target_os = "wasi")] -pub mod wasi; - -#[cfg(target_os = "android")] -pub mod android; -#[cfg(target_os = "dragonfly")] -pub mod dragonfly; -#[cfg(target_os = "emscripten")] -pub mod emscripten; -#[cfg(all(target_vendor = "fortanix", target_env = "sgx"))] -pub mod fortanix_sgx; -#[cfg(target_os = "freebsd")] -pub mod freebsd; -#[cfg(target_os = "fuchsia")] -pub mod fuchsia; -#[cfg(target_os = "haiku")] -pub mod haiku; -#[cfg(target_os = "illumos")] -pub mod illumos; -#[cfg(target_os = "ios")] -pub mod ios; -#[cfg(target_os = "macos")] -pub mod macos; -#[cfg(target_os = "netbsd")] -pub mod netbsd; -#[cfg(target_os = "openbsd")] -pub mod openbsd; -#[cfg(target_os = "redox")] -pub mod redox; -#[cfg(target_os = "solaris")] -pub mod solaris; -#[cfg(target_os = "vxworks")] -pub mod vxworks; +#[path = "."] +mod imp { + // If we're not documenting std then we only expose modules appropriate for the + // current platform. -pub mod raw; + #[cfg(all(target_vendor = "fortanix", target_env = "sgx"))] + pub mod fortanix_sgx; + + #[cfg(target_os = "hermit")] + #[path = "hermit/mod.rs"] + pub mod unix; + + #[cfg(target_os = "android")] + pub mod android; + #[cfg(target_os = "dragonfly")] + pub mod dragonfly; + #[cfg(target_os = "emscripten")] + pub mod emscripten; + #[cfg(target_os = "freebsd")] + pub mod freebsd; + #[cfg(target_os = "fuchsia")] + pub mod fuchsia; + #[cfg(target_os = "haiku")] + pub mod haiku; + #[cfg(target_os = "illumos")] + pub mod illumos; + #[cfg(target_os = "ios")] + pub mod ios; + #[cfg(target_os = "l4re")] + pub mod linux; + #[cfg(target_os = "linux")] + pub mod linux; + #[cfg(target_os = "macos")] + pub mod macos; + #[cfg(target_os = "netbsd")] + pub mod netbsd; + #[cfg(target_os = "openbsd")] + pub mod openbsd; + #[cfg(target_os = "redox")] + pub mod redox; + #[cfg(target_os = "solaris")] + pub mod solaris; + #[cfg(unix)] + pub mod unix; + + #[cfg(target_os = "vxworks")] + pub mod vxworks; + + #[cfg(target_os = "wasi")] + pub mod wasi; + + #[cfg(windows)] + pub mod windows; +} +#[cfg(not(doc))] +#[stable(feature = "os", since = "1.0.0")] +pub use imp::*; diff --git a/library/std/src/os/redox/raw.rs b/library/std/src/os/redox/raw.rs index abe6dfc6b0..9a6b99684c 100644 --- a/library/std/src/os/redox/raw.rs +++ b/library/std/src/os/redox/raw.rs @@ -9,7 +9,6 @@ definitions" )] #![allow(deprecated)] -#![allow(missing_debug_implementations)] use crate::os::raw::{c_char, c_int, c_long, c_ulong, c_void}; diff --git a/library/std/src/sys/unix/ext/ffi.rs b/library/std/src/os/unix/ffi.rs similarity index 100% rename from library/std/src/sys/unix/ext/ffi.rs rename to library/std/src/os/unix/ffi.rs diff --git a/library/std/src/sys/unix/ext/fs.rs b/library/std/src/os/unix/fs.rs similarity index 99% rename from library/std/src/sys/unix/ext/fs.rs rename to library/std/src/os/unix/fs.rs index 9cf51be283..913c71d410 100644 --- a/library/std/src/sys/unix/ext/fs.rs +++ b/library/std/src/os/unix/fs.rs @@ -906,7 +906,7 @@ impl DirBuilderExt for fs::DirBuilder { /// } /// ``` #[unstable(feature = "unix_chroot", issue = "84715")] -#[cfg(not(target_os = "fuchsia"))] +#[cfg(not(any(target_os = "fuchsia", target_os = "vxworks")))] pub fn chroot>(dir: P) -> io::Result<()> { sys::fs::chroot(dir.as_ref()) } diff --git a/library/std/src/sys/unix/ext/io.rs b/library/std/src/os/unix/io.rs similarity index 100% rename from library/std/src/sys/unix/ext/io.rs rename to library/std/src/os/unix/io.rs diff --git a/library/std/src/sys/unix/ext/mod.rs b/library/std/src/os/unix/mod.rs similarity index 61% rename from library/std/src/sys/unix/ext/mod.rs rename to library/std/src/os/unix/mod.rs index 735bf35a3c..6fc1c89a2b 100644 --- a/library/std/src/sys/unix/ext/mod.rs +++ b/library/std/src/os/unix/mod.rs @@ -27,44 +27,43 @@ #![stable(feature = "rust1", since = "1.0.0")] #![doc(cfg(unix))] -#![allow(missing_docs)] -cfg_if::cfg_if! { - if #[cfg(doc)] { - // Use linux as the default platform when documenting on other platforms like Windows - use crate::os::linux as platform; - } else { - #[cfg(target_os = "android")] - use crate::os::android as platform; - #[cfg(target_os = "dragonfly")] - use crate::os::dragonfly as platform; - #[cfg(target_os = "emscripten")] - use crate::os::emscripten as platform; - #[cfg(target_os = "freebsd")] - use crate::os::freebsd as platform; - #[cfg(target_os = "fuchsia")] - use crate::os::fuchsia as platform; - #[cfg(target_os = "haiku")] - use crate::os::haiku as platform; - #[cfg(target_os = "illumos")] - use crate::os::illumos as platform; - #[cfg(target_os = "ios")] - use crate::os::ios as platform; - #[cfg(any(target_os = "linux", target_os = "l4re"))] - use crate::os::linux as platform; - #[cfg(target_os = "macos")] - use crate::os::macos as platform; - #[cfg(target_os = "netbsd")] - use crate::os::netbsd as platform; - #[cfg(target_os = "openbsd")] - use crate::os::openbsd as platform; - #[cfg(target_os = "redox")] - use crate::os::redox as platform; - #[cfg(target_os = "solaris")] - use crate::os::solaris as platform; - #[cfg(target_os = "vxworks")] - use crate::os::vxworks as platform; - } +// Use linux as the default platform when documenting on other platforms like Windows +#[cfg(doc)] +use crate::os::linux as platform; + +#[cfg(not(doc))] +mod platform { + #[cfg(target_os = "android")] + pub use crate::os::android::*; + #[cfg(target_os = "dragonfly")] + pub use crate::os::dragonfly::*; + #[cfg(target_os = "emscripten")] + pub use crate::os::emscripten::*; + #[cfg(target_os = "freebsd")] + pub use crate::os::freebsd::*; + #[cfg(target_os = "fuchsia")] + pub use crate::os::fuchsia::*; + #[cfg(target_os = "haiku")] + pub use crate::os::haiku::*; + #[cfg(target_os = "illumos")] + pub use crate::os::illumos::*; + #[cfg(target_os = "ios")] + pub use crate::os::ios::*; + #[cfg(any(target_os = "linux", target_os = "l4re"))] + pub use crate::os::linux::*; + #[cfg(target_os = "macos")] + pub use crate::os::macos::*; + #[cfg(target_os = "netbsd")] + pub use crate::os::netbsd::*; + #[cfg(target_os = "openbsd")] + pub use crate::os::openbsd::*; + #[cfg(target_os = "redox")] + pub use crate::os::redox::*; + #[cfg(target_os = "solaris")] + pub use crate::os::solaris::*; + #[cfg(target_os = "vxworks")] + pub use crate::os::vxworks::*; } pub mod ffi; diff --git a/library/std/src/sys/unix/ext/net/addr.rs b/library/std/src/os/unix/net/addr.rs similarity index 100% rename from library/std/src/sys/unix/ext/net/addr.rs rename to library/std/src/os/unix/net/addr.rs diff --git a/library/std/src/sys/unix/ext/net/ancillary.rs b/library/std/src/os/unix/net/ancillary.rs similarity index 99% rename from library/std/src/sys/unix/ext/net/ancillary.rs rename to library/std/src/os/unix/net/ancillary.rs index 011ae643f8..15ce7056fe 100644 --- a/library/std/src/sys/unix/ext/net/ancillary.rs +++ b/library/std/src/os/unix/net/ancillary.rs @@ -49,7 +49,7 @@ pub(super) fn recv_vectored_with_ancillary_from( msg.msg_controllen = ancillary.buffer.len() as libc::socklen_t; } } - // macos requires that the control pointer is NULL when the len is 0. + // macos requires that the control pointer is null when the len is 0. if msg.msg_controllen > 0 { msg.msg_control = ancillary.buffer.as_mut_ptr().cast(); } @@ -97,7 +97,7 @@ pub(super) fn send_vectored_with_ancillary_to( msg.msg_controllen = ancillary.length as libc::socklen_t; } } - // macos requires that the control pointer is NULL when the len is 0. + // macos requires that the control pointer is null when the len is 0. if msg.msg_controllen > 0 { msg.msg_control = ancillary.buffer.as_mut_ptr().cast(); } diff --git a/library/std/src/sys/unix/ext/net/datagram.rs b/library/std/src/os/unix/net/datagram.rs similarity index 100% rename from library/std/src/sys/unix/ext/net/datagram.rs rename to library/std/src/os/unix/net/datagram.rs diff --git a/library/std/src/sys/unix/ext/net/listener.rs b/library/std/src/os/unix/net/listener.rs similarity index 100% rename from library/std/src/sys/unix/ext/net/listener.rs rename to library/std/src/os/unix/net/listener.rs diff --git a/library/std/src/sys/unix/ext/net/mod.rs b/library/std/src/os/unix/net/mod.rs similarity index 100% rename from library/std/src/sys/unix/ext/net/mod.rs rename to library/std/src/os/unix/net/mod.rs diff --git a/library/std/src/sys/unix/ext/net/raw_fd.rs b/library/std/src/os/unix/net/raw_fd.rs similarity index 100% rename from library/std/src/sys/unix/ext/net/raw_fd.rs rename to library/std/src/os/unix/net/raw_fd.rs diff --git a/library/std/src/sys/unix/ext/net/stream.rs b/library/std/src/os/unix/net/stream.rs similarity index 100% rename from library/std/src/sys/unix/ext/net/stream.rs rename to library/std/src/os/unix/net/stream.rs diff --git a/library/std/src/sys/unix/ext/net/tests.rs b/library/std/src/os/unix/net/tests.rs similarity index 100% rename from library/std/src/sys/unix/ext/net/tests.rs rename to library/std/src/os/unix/net/tests.rs diff --git a/library/std/src/sys/unix/ext/process.rs b/library/std/src/os/unix/process.rs similarity index 86% rename from library/std/src/sys/unix/ext/process.rs rename to library/std/src/os/unix/process.rs index 355855bcd1..3dc389b758 100644 --- a/library/std/src/sys/unix/ext/process.rs +++ b/library/std/src/os/unix/process.rs @@ -75,6 +75,12 @@ pub trait CommandExt: Sealed { /// sure that the closure does not violate library invariants by making /// invalid use of these duplicates. /// + /// Panicking in the closure is safe only if all the format arguments for the + /// panic message can be safely formatted; this is because although + /// `Command` calls [`std::panic::always_abort`](crate::panic::always_abort) + /// before calling the pre_exec hook, panic will still try to format the + /// panic message. + /// /// When this closure is run, aspects such as the stdio file descriptors and /// working directory have successfully been changed, so output to these /// locations may not appear where intended. @@ -195,22 +201,32 @@ impl CommandExt for process::Command { } } -/// Unix-specific extensions to [`process::ExitStatus`]. +/// Unix-specific extensions to [`process::ExitStatus`] and +/// [`ExitStatusError`](process::ExitStatusError). /// -/// On Unix, `ExitStatus` **does not necessarily represent an exit status**, as passed to the -/// `exit` system call or returned by [`ExitStatus::code()`](crate::process::ExitStatus::code). -/// It represents **any wait status**, as returned by one of the `wait` family of system calls. +/// On Unix, `ExitStatus` **does not necessarily represent an exit status**, as +/// passed to the `exit` system call or returned by +/// [`ExitStatus::code()`](crate::process::ExitStatus::code). It represents **any wait status** +/// as returned by one of the `wait` family of system +/// calls. /// -/// This is because a Unix wait status (a Rust `ExitStatus`) can represent a Unix exit status, but -/// can also represent other kinds of process event. +/// A Unix wait status (a Rust `ExitStatus`) can represent a Unix exit status, but can also +/// represent other kinds of process event. /// /// This trait is sealed: it cannot be implemented outside the standard library. /// This is so that future additional methods are not breaking changes. #[stable(feature = "rust1", since = "1.0.0")] pub trait ExitStatusExt: Sealed { - /// Creates a new `ExitStatus` from the raw underlying integer status value from `wait` + /// Creates a new `ExitStatus` or `ExitStatusError` from the raw underlying integer status + /// value from `wait` /// /// The value should be a **wait status, not an exit status**. + /// + /// # Panics + /// + /// Panics on an attempt to make an `ExitStatusError` from a wait status of `0`. + /// + /// Making an `ExitStatus` always succeds and never panics. #[stable(feature = "exit_status_from", since = "1.12.0")] fn from_raw(raw: i32) -> Self; @@ -272,6 +288,35 @@ impl ExitStatusExt for process::ExitStatus { } } +#[unstable(feature = "exit_status_error", issue = "84908")] +impl ExitStatusExt for process::ExitStatusError { + fn from_raw(raw: i32) -> Self { + process::ExitStatus::from_raw(raw) + .exit_ok() + .expect_err("::from_raw(0) but zero is not an error") + } + + fn signal(&self) -> Option { + self.into_status().signal() + } + + fn core_dumped(&self) -> bool { + self.into_status().core_dumped() + } + + fn stopped_signal(&self) -> Option { + self.into_status().stopped_signal() + } + + fn continued(&self) -> bool { + self.into_status().continued() + } + + fn into_raw(self) -> i32 { + self.into_status().into_raw() + } +} + #[stable(feature = "process_extensions", since = "1.2.0")] impl FromRawFd for process::Stdio { #[inline] diff --git a/library/std/src/sys/unix/ext/raw.rs b/library/std/src/os/unix/raw.rs similarity index 100% rename from library/std/src/sys/unix/ext/raw.rs rename to library/std/src/os/unix/raw.rs diff --git a/library/std/src/sys/unix/ext/thread.rs b/library/std/src/os/unix/thread.rs similarity index 100% rename from library/std/src/sys/unix/ext/thread.rs rename to library/std/src/os/unix/thread.rs diff --git a/library/std/src/sys/unix/ext/ucred.rs b/library/std/src/os/unix/ucred.rs similarity index 100% rename from library/std/src/sys/unix/ext/ucred.rs rename to library/std/src/os/unix/ucred.rs diff --git a/library/std/src/sys/unix/ext/ucred/tests.rs b/library/std/src/os/unix/ucred/tests.rs similarity index 100% rename from library/std/src/sys/unix/ext/ucred/tests.rs rename to library/std/src/os/unix/ucred/tests.rs diff --git a/library/std/src/os/wasi.rs b/library/std/src/os/wasi.rs deleted file mode 100644 index d25b8d39ed..0000000000 --- a/library/std/src/os/wasi.rs +++ /dev/null @@ -1,6 +0,0 @@ -//! WASI-specific definitions - -#![stable(feature = "raw_ext", since = "1.1.0")] - -#[stable(feature = "rust1", since = "1.0.0")] -pub use crate::sys::ext::*; diff --git a/library/std/src/sys/wasi/ext/ffi.rs b/library/std/src/os/wasi/ffi.rs similarity index 100% rename from library/std/src/sys/wasi/ext/ffi.rs rename to library/std/src/os/wasi/ffi.rs diff --git a/library/std/src/sys/wasi/ext/fs.rs b/library/std/src/os/wasi/fs.rs similarity index 100% rename from library/std/src/sys/wasi/ext/fs.rs rename to library/std/src/os/wasi/fs.rs diff --git a/library/std/src/sys/wasi/ext/io.rs b/library/std/src/os/wasi/io.rs similarity index 100% rename from library/std/src/sys/wasi/ext/io.rs rename to library/std/src/os/wasi/io.rs diff --git a/library/std/src/sys/wasi/ext/mod.rs b/library/std/src/os/wasi/mod.rs similarity index 92% rename from library/std/src/sys/wasi/ext/mod.rs rename to library/std/src/os/wasi/mod.rs index b08402f077..44b7c32e95 100644 --- a/library/std/src/sys/wasi/ext/mod.rs +++ b/library/std/src/os/wasi/mod.rs @@ -1,4 +1,4 @@ -//! Platform-specific extensions to `std` for WASI. +//! Platform-specific extensions to `std` for the WebAssembly System Interface (WASI). //! //! Provides access to platform-level information on WASI, and exposes //! WASI-specific functions that would otherwise be inappropriate as @@ -25,6 +25,7 @@ //! } //! ``` +#![stable(feature = "rust1", since = "1.0.0")] #![deny(unsafe_op_in_unsafe_fn)] #![doc(cfg(target_os = "wasi"))] diff --git a/library/std/src/sys/windows/ext/ffi.rs b/library/std/src/os/windows/ffi.rs similarity index 100% rename from library/std/src/sys/windows/ext/ffi.rs rename to library/std/src/os/windows/ffi.rs diff --git a/library/std/src/sys/windows/ext/fs.rs b/library/std/src/os/windows/fs.rs similarity index 100% rename from library/std/src/sys/windows/ext/fs.rs rename to library/std/src/os/windows/fs.rs diff --git a/library/std/src/sys/windows/ext/io.rs b/library/std/src/os/windows/io.rs similarity index 100% rename from library/std/src/sys/windows/ext/io.rs rename to library/std/src/os/windows/io.rs diff --git a/library/std/src/sys/windows/ext/mod.rs b/library/std/src/os/windows/mod.rs similarity index 98% rename from library/std/src/sys/windows/ext/mod.rs rename to library/std/src/os/windows/mod.rs index 613d3dc189..52ac508f9f 100644 --- a/library/std/src/sys/windows/ext/mod.rs +++ b/library/std/src/os/windows/mod.rs @@ -8,7 +8,6 @@ #![stable(feature = "rust1", since = "1.0.0")] #![doc(cfg(windows))] -#![allow(missing_docs)] pub mod ffi; pub mod fs; diff --git a/library/std/src/sys/windows/ext/process.rs b/library/std/src/os/windows/process.rs similarity index 100% rename from library/std/src/sys/windows/ext/process.rs rename to library/std/src/os/windows/process.rs diff --git a/library/std/src/sys/windows/ext/raw.rs b/library/std/src/os/windows/raw.rs similarity index 100% rename from library/std/src/sys/windows/ext/raw.rs rename to library/std/src/os/windows/raw.rs diff --git a/library/std/src/sys/windows/ext/thread.rs b/library/std/src/os/windows/thread.rs similarity index 100% rename from library/std/src/sys/windows/ext/thread.rs rename to library/std/src/os/windows/thread.rs diff --git a/library/std/src/panic.rs b/library/std/src/panic.rs index 3e634239ad..9e3880dfd4 100644 --- a/library/std/src/panic.rs +++ b/library/std/src/panic.rs @@ -55,6 +55,7 @@ pub use core::panic::{Location, PanicInfo}; /// See the [`panic!`] macro for more information about panicking. #[stable(feature = "panic_any", since = "1.51.0")] #[inline] +#[track_caller] pub fn panic_any(msg: M) -> ! { crate::panicking::begin_panic(msg); } @@ -132,6 +133,7 @@ pub fn panic_any(msg: M) -> ! { /// [`AssertUnwindSafe`] wrapper struct can be used to force this trait to be /// implemented for any closed over variables passed to `catch_unwind`. #[stable(feature = "catch_unwind", since = "1.9.0")] +#[cfg_attr(all(not(bootstrap), not(test)), lang = "unwind_safe")] #[rustc_on_unimplemented( message = "the type `{Self}` may not be safely transferred across an unwind boundary", label = "`{Self}` may not be safely transferred across an unwind boundary" @@ -147,6 +149,7 @@ pub auto trait UnwindSafe {} /// This is a "helper marker trait" used to provide impl blocks for the /// [`UnwindSafe`] trait, for more information see that documentation. #[stable(feature = "catch_unwind", since = "1.9.0")] +#[cfg_attr(all(not(bootstrap), not(test)), lang = "ref_unwind_safe")] #[rustc_on_unimplemented( message = "the type `{Self}` may contain interior mutability and a reference may not be safely \ transferrable across a catch_unwind boundary", @@ -461,5 +464,41 @@ pub fn resume_unwind(payload: Box) -> ! { panicking::rust_panic_without_hook(payload) } +/// Make all future panics abort directly without running the panic hook or unwinding. +/// +/// There is no way to undo this; the effect lasts until the process exits or +/// execs (or the equivalent). +/// +/// # Use after fork +/// +/// This function is particularly useful for calling after `libc::fork`. After `fork`, in a +/// multithreaded program it is (on many platforms) not safe to call the allocator. It is also +/// generally highly undesirable for an unwind to unwind past the `fork`, because that results in +/// the unwind propagating to code that was only ever expecting to run in the parent. +/// +/// `panic::always_abort()` helps avoid both of these. It directly avoids any further unwinding, +/// and if there is a panic, the abort will occur without allocating provided that the arguments to +/// panic can be formatted without allocating. +/// +/// Examples +/// +/// ```no_run +/// #![feature(panic_always_abort)] +/// use std::panic; +/// +/// panic::always_abort(); +/// +/// let _ = panic::catch_unwind(|| { +/// panic!("inside the catch"); +/// }); +/// +/// // We will have aborted already, due to the panic. +/// unreachable!(); +/// ``` +#[unstable(feature = "panic_always_abort", issue = "84438")] +pub fn always_abort() { + crate::panicking::panic_count::set_always_abort(); +} + #[cfg(test)] mod tests; diff --git a/library/std/src/panicking.rs b/library/std/src/panicking.rs index 6cd572cbe8..0b9c9fb479 100644 --- a/library/std/src/panicking.rs +++ b/library/std/src/panicking.rs @@ -19,8 +19,8 @@ use crate::process; use crate::sync::atomic::{AtomicBool, Ordering}; use crate::sys::stdio::panic_output; use crate::sys_common::backtrace::{self, RustBacktrace}; -use crate::sys_common::rwlock::RWLock; -use crate::sys_common::{thread_info, util}; +use crate::sys_common::rwlock::StaticRWLock; +use crate::sys_common::thread_info; use crate::thread; #[cfg(not(test))] @@ -74,7 +74,7 @@ enum Hook { Custom(*mut (dyn Fn(&PanicInfo<'_>) + 'static + Sync + Send)), } -static HOOK_LOCK: RWLock = RWLock::new(); +static HOOK_LOCK: StaticRWLock = StaticRWLock::new(); static mut HOOK: Hook = Hook::Default; /// Registers a custom panic hook, replacing any that was previously registered. @@ -117,10 +117,10 @@ pub fn set_hook(hook: Box) + 'static + Sync + Send>) { } unsafe { - HOOK_LOCK.write(); + let guard = HOOK_LOCK.write(); let old_hook = HOOK; HOOK = Hook::Custom(Box::into_raw(hook)); - HOOK_LOCK.write_unlock(); + drop(guard); if let Hook::Custom(ptr) = old_hook { #[allow(unused_must_use)] @@ -165,10 +165,10 @@ pub fn take_hook() -> Box) + 'static + Sync + Send> { } unsafe { - HOOK_LOCK.write(); + let guard = HOOK_LOCK.write(); let hook = HOOK; HOOK = Hook::Default; - HOOK_LOCK.write_unlock(); + drop(guard); match hook { Hook::Default => Box::new(default_hook), @@ -180,7 +180,7 @@ pub fn take_hook() -> Box) + 'static + Sync + Send> { fn default_hook(info: &PanicInfo<'_>) { // If this is a double panic, make sure that we print a backtrace // for this panic. Otherwise only print it if logging is enabled. - let backtrace_env = if panic_count::get() >= 2 { + let backtrace_env = if panic_count::get_count() >= 2 { RustBacktrace::Print(crate::backtrace_rs::PrintFmt::Full) } else { backtrace::rust_backtrace_env() @@ -193,7 +193,7 @@ fn default_hook(info: &PanicInfo<'_>) { Some(s) => *s, None => match info.payload().downcast_ref::() { Some(s) => &s[..], - None => "Box", + None => "Box", }, }; let thread = thread_info::current_thread(); @@ -233,6 +233,8 @@ pub mod panic_count { use crate::cell::Cell; use crate::sync::atomic::{AtomicUsize, Ordering}; + pub const ALWAYS_ABORT_FLAG: usize = 1 << (usize::BITS - 1); + // Panic count for the current thread. thread_local! { static LOCAL_PANIC_COUNT: Cell = Cell::new(0) } @@ -241,33 +243,53 @@ pub mod panic_count { // thread, if that thread currently views `GLOBAL_PANIC_COUNT` as being zero, // then `LOCAL_PANIC_COUNT` in that thread is zero. This invariant holds before // and after increase and decrease, but not necessarily during their execution. + // + // Additionally, the top bit of GLOBAL_PANIC_COUNT (GLOBAL_ALWAYS_ABORT_FLAG) + // records whether panic::always_abort() has been called. This can only be + // set, never cleared. + // + // This could be viewed as a struct containing a single bit and an n-1-bit + // value, but if we wrote it like that it would be more than a single word, + // and even a newtype around usize would be clumsy because we need atomics. + // But we use such a tuple for the return type of increase(). + // + // Stealing a bit is fine because it just amounts to assuming that each + // panicking thread consumes at least 2 bytes of address space. static GLOBAL_PANIC_COUNT: AtomicUsize = AtomicUsize::new(0); - pub fn increase() -> usize { - GLOBAL_PANIC_COUNT.fetch_add(1, Ordering::Relaxed); - LOCAL_PANIC_COUNT.with(|c| { - let next = c.get() + 1; - c.set(next); - next - }) + pub fn increase() -> (bool, usize) { + ( + GLOBAL_PANIC_COUNT.fetch_add(1, Ordering::Relaxed) & ALWAYS_ABORT_FLAG != 0, + LOCAL_PANIC_COUNT.with(|c| { + let next = c.get() + 1; + c.set(next); + next + }), + ) } - pub fn decrease() -> usize { + pub fn decrease() { GLOBAL_PANIC_COUNT.fetch_sub(1, Ordering::Relaxed); LOCAL_PANIC_COUNT.with(|c| { let next = c.get() - 1; c.set(next); next - }) + }); + } + + pub fn set_always_abort() { + GLOBAL_PANIC_COUNT.fetch_or(ALWAYS_ABORT_FLAG, Ordering::Relaxed); } - pub fn get() -> usize { + // Disregards ALWAYS_ABORT_FLAG + pub fn get_count() -> usize { LOCAL_PANIC_COUNT.with(|c| c.get()) } + // Disregards ALWAYS_ABORT_FLAG #[inline] - pub fn is_zero() -> bool { - if GLOBAL_PANIC_COUNT.load(Ordering::Relaxed) == 0 { + pub fn count_is_zero() -> bool { + if GLOBAL_PANIC_COUNT.load(Ordering::Relaxed) & !ALWAYS_ABORT_FLAG == 0 { // Fast path: if `GLOBAL_PANIC_COUNT` is zero, all threads // (including the current one) will have `LOCAL_PANIC_COUNT` // equal to zero, so TLS access can be avoided. @@ -410,7 +432,7 @@ pub unsafe fn r#try R>(f: F) -> Result> /// Determines whether the current thread is unwinding because of panic. #[inline] pub fn panicking() -> bool { - !panic_count::is_zero() + !panic_count::count_is_zero() } /// The entry point for panicking with a formatted message. @@ -563,21 +585,30 @@ fn rust_panic_with_hook( message: Option<&fmt::Arguments<'_>>, location: &Location<'_>, ) -> ! { - let panics = panic_count::increase(); + let (must_abort, panics) = panic_count::increase(); // If this is the third nested call (e.g., panics == 2, this is 0-indexed), // the panic hook probably triggered the last panic, otherwise the // double-panic check would have aborted the process. In this case abort the // process real quickly as we don't want to try calling it again as it'll // probably just panic again. - if panics > 2 { - util::dumb_print(format_args!("thread panicked while processing panic. aborting.\n")); + if must_abort || panics > 2 { + if panics > 2 { + // Don't try to print the message in this case + // - perhaps that is causing the recursive panics. + rtprintpanic!("thread panicked while processing panic. aborting.\n"); + } else { + // Unfortunately, this does not print a backtrace, because creating + // a `Backtrace` will allocate, which we must to avoid here. + let panicinfo = PanicInfo::internal_constructor(message, location); + rtprintpanic!("{}\npanicked after panic::always_abort(), aborting.\n", panicinfo); + } intrinsics::abort() } unsafe { let mut info = PanicInfo::internal_constructor(message, location); - HOOK_LOCK.read(); + let _guard = HOOK_LOCK.read(); match HOOK { // Some platforms (like wasm) know that printing to stderr won't ever actually // print anything, and if that's the case we can skip the default @@ -595,7 +626,6 @@ fn rust_panic_with_hook( (*ptr)(&info); } }; - HOOK_LOCK.read_unlock(); } if panics > 1 { @@ -603,7 +633,7 @@ fn rust_panic_with_hook( // have limited options. Currently our preference is to // just abort. In the future we may consider resuming // unwinding or otherwise exiting the thread cleanly. - util::dumb_print(format_args!("thread panicked while panicking. aborting.\n")); + rtprintpanic!("thread panicked while panicking. aborting.\n"); intrinsics::abort() } diff --git a/library/std/src/path.rs b/library/std/src/path.rs index ed0987064e..ede147aca1 100644 --- a/library/std/src/path.rs +++ b/library/std/src/path.rs @@ -569,7 +569,7 @@ pub struct Components<'a> { prefix: Option>, // true if path *physically* has a root separator; for most Windows - // prefixes, it may have a "logical" rootseparator for the purposes of + // prefixes, it may have a "logical" root separator for the purposes of // normalization, e.g., \\server\share == \\server\share\. has_physical_root: bool, @@ -1420,6 +1420,9 @@ impl Clone for PathBuf { #[stable(feature = "box_from_path", since = "1.17.0")] impl From<&Path> for Box { + /// Creates a boxed [`Path`] from a reference. + /// + /// This will allocate and clone `path` to it. fn from(path: &Path) -> Box { let boxed: Box = path.inner.into(); let rw = Box::into_raw(boxed) as *mut Path; @@ -1429,6 +1432,9 @@ impl From<&Path> for Box { #[stable(feature = "box_from_cow", since = "1.45.0")] impl From> for Box { + /// Creates a boxed [`Path`] from a clone-on-write pointer. + /// + /// Converting from a `Cow::Owned` does not clone or allocate. #[inline] fn from(cow: Cow<'_, Path>) -> Box { match cow { @@ -1471,6 +1477,9 @@ impl Clone for Box { #[stable(feature = "rust1", since = "1.0.0")] impl> From<&T> for PathBuf { + /// Converts a borrowed `OsStr` to a `PathBuf`. + /// + /// Allocates a [`PathBuf`] and copies the data into it. #[inline] fn from(s: &T) -> PathBuf { PathBuf::from(s.as_ref().to_os_string()) @@ -1575,6 +1584,10 @@ impl Default for PathBuf { #[stable(feature = "cow_from_path", since = "1.6.0")] impl<'a> From<&'a Path> for Cow<'a, Path> { + /// Creates a clone-on-write pointer from a reference to + /// [`Path`]. + /// + /// This conversion does not clone or allocate. #[inline] fn from(s: &'a Path) -> Cow<'a, Path> { Cow::Borrowed(s) @@ -1583,6 +1596,10 @@ impl<'a> From<&'a Path> for Cow<'a, Path> { #[stable(feature = "cow_from_path", since = "1.6.0")] impl<'a> From for Cow<'a, Path> { + /// Creates a clone-on-write pointer from an owned + /// instance of [`PathBuf`]. + /// + /// This conversion does not clone or allocate. #[inline] fn from(s: PathBuf) -> Cow<'a, Path> { Cow::Owned(s) @@ -1591,6 +1608,10 @@ impl<'a> From for Cow<'a, Path> { #[stable(feature = "cow_from_pathbuf_ref", since = "1.28.0")] impl<'a> From<&'a PathBuf> for Cow<'a, Path> { + /// Creates a clone-on-write pointer from a reference to + /// [`PathBuf`]. + /// + /// This conversion does not clone or allocate. #[inline] fn from(p: &'a PathBuf) -> Cow<'a, Path> { Cow::Borrowed(p.as_path()) @@ -1599,6 +1620,9 @@ impl<'a> From<&'a PathBuf> for Cow<'a, Path> { #[stable(feature = "pathbuf_from_cow_path", since = "1.28.0")] impl<'a> From> for PathBuf { + /// Converts a clone-on-write pointer to an owned path. + /// + /// Converting from a `Cow::Owned` does not clone or allocate. #[inline] fn from(p: Cow<'a, Path>) -> Self { p.into_owned() @@ -2462,10 +2486,10 @@ impl Path { /// Returns `true` if the path points at an existing entity. /// /// This function will traverse symbolic links to query information about the - /// destination file. In case of broken symbolic links this will return `false`. + /// destination file. /// - /// If you cannot access the directory containing the file, e.g., because of a - /// permission error, this will return `false`. + /// If you cannot access the metadata of the file, e.g. because of a + /// permission error or broken symbolic links, this will return `false`. /// /// # Examples /// @@ -2507,20 +2531,16 @@ impl Path { #[unstable(feature = "path_try_exists", issue = "83186")] #[inline] pub fn try_exists(&self) -> io::Result { - match fs::metadata(self) { - Ok(_) => Ok(true), - Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(false), - Err(error) => Err(error), - } + fs::try_exists(self) } /// Returns `true` if the path exists on disk and is pointing at a regular file. /// /// This function will traverse symbolic links to query information about the - /// destination file. In case of broken symbolic links this will return `false`. + /// destination file. /// - /// If you cannot access the directory containing the file, e.g., because of a - /// permission error, this will return `false`. + /// If you cannot access the metadata of the file, e.g. because of a + /// permission error or broken symbolic links, this will return `false`. /// /// # Examples /// @@ -2549,10 +2569,10 @@ impl Path { /// Returns `true` if the path exists on disk and is pointing at a directory. /// /// This function will traverse symbolic links to query information about the - /// destination file. In case of broken symbolic links this will return `false`. + /// destination file. /// - /// If you cannot access the directory containing the file, e.g., because of a - /// permission error, this will return `false`. + /// If you cannot access the metadata of the file, e.g. because of a + /// permission error or broken symbolic links, this will return `false`. /// /// # Examples /// diff --git a/library/std/src/prelude/mod.rs b/library/std/src/prelude/mod.rs index 1b4facdd04..12d52cc8e0 100644 --- a/library/std/src/prelude/mod.rs +++ b/library/std/src/prelude/mod.rs @@ -88,9 +88,9 @@ pub mod v1; /// The 2015 version of the prelude of The Rust Standard Library. /// /// See the [module-level documentation](self) for more. -#[unstable(feature = "prelude_2015", issue = "none")] +#[unstable(feature = "prelude_2015", issue = "85684")] pub mod rust_2015 { - #[unstable(feature = "prelude_2015", issue = "none")] + #[unstable(feature = "prelude_2015", issue = "85684")] #[doc(no_inline)] pub use super::v1::*; } @@ -98,9 +98,9 @@ pub mod rust_2015 { /// The 2018 version of the prelude of The Rust Standard Library. /// /// See the [module-level documentation](self) for more. -#[unstable(feature = "prelude_2018", issue = "none")] +#[unstable(feature = "prelude_2018", issue = "85684")] pub mod rust_2018 { - #[unstable(feature = "prelude_2018", issue = "none")] + #[unstable(feature = "prelude_2018", issue = "85684")] #[doc(no_inline)] pub use super::v1::*; } @@ -108,13 +108,13 @@ pub mod rust_2018 { /// The 2021 version of the prelude of The Rust Standard Library. /// /// See the [module-level documentation](self) for more. -#[unstable(feature = "prelude_2021", issue = "none")] +#[unstable(feature = "prelude_2021", issue = "85684")] pub mod rust_2021 { - #[unstable(feature = "prelude_2021", issue = "none")] + #[unstable(feature = "prelude_2021", issue = "85684")] #[doc(no_inline)] pub use super::v1::*; - #[unstable(feature = "prelude_2021", issue = "none")] + #[unstable(feature = "prelude_2021", issue = "85684")] #[doc(no_inline)] pub use core::prelude::rust_2021::*; } diff --git a/library/std/src/primitive_docs.rs b/library/std/src/primitive_docs.rs index cd6b0b2d7e..8a3e425350 100644 --- a/library/std/src/primitive_docs.rs +++ b/library/std/src/primitive_docs.rs @@ -345,6 +345,9 @@ mod prim_never {} mod prim_char {} #[doc(primitive = "unit")] +#[doc(alias = "(")] +#[doc(alias = ")")] +#[doc(alias = "()")] // /// The `()` type, also called "unit". /// @@ -445,7 +448,27 @@ mod prim_unit {} /// Note that here the call to [`drop`] is for clarity - it indicates /// that we are done with the given value and it should be destroyed. /// -/// ## 3. Get it from C. +/// ## 3. Create it using `ptr::addr_of!` +/// +/// Instead of coercing a reference to a raw pointer, you can use the macros +/// [`ptr::addr_of!`] (for `*const T`) and [`ptr::addr_of_mut!`] (for `*mut T`). +/// These macros allow you to create raw pointers to fields to which you cannot +/// create a reference (without causing undefined behaviour), such as an +/// unaligned field. This might be necessary if packed structs or uninitialized +/// memory is involved. +/// +/// ``` +/// #[derive(Debug, Default, Copy, Clone)] +/// #[repr(C, packed)] +/// struct S { +/// aligned: u8, +/// unaligned: u32, +/// } +/// let s = S::default(); +/// let p = std::ptr::addr_of!(s.unaligned); // not allowed with coercion +/// ``` +/// +/// ## 4. Get it from C. /// /// ``` /// # #![feature(rustc_private)] @@ -517,8 +540,7 @@ mod prim_pointer {} /// /// # Examples /// -#[cfg_attr(bootstrap, doc = "```ignore")] -#[cfg_attr(not(bootstrap), doc = "```")] +/// ``` /// let mut array: [i32; 3] = [0; 3]; /// /// array[1] = 1; @@ -558,8 +580,7 @@ mod prim_pointer {} /// `IntoIterator` by value. In the future, the behavior on the 2015 and 2018 edition /// might be made consistent to the behavior of later editions. /// -#[cfg_attr(bootstrap, doc = "```rust,edition2018,ignore")] -#[cfg_attr(not(bootstrap), doc = "```rust,edition2018")] +/// ```rust,edition2018 /// # #![allow(array_into_iter)] // override our `deny(warnings)` /// let array: [i32; 3] = [0; 3]; /// @@ -614,8 +635,7 @@ mod prim_pointer {} /// * replace `for ... in array.into_iter() {` with `for ... in array {`, /// equivalent to the post-2021 behavior (Rust 1.53+) /// -#[cfg_attr(bootstrap, doc = "```rust,edition2018,ignore")] -#[cfg_attr(not(bootstrap), doc = "```rust,edition2018")] +/// ```rust,edition2018 /// use std::array::IntoIter; /// /// let array: [i32; 3] = [0; 3]; diff --git a/library/std/src/process.rs b/library/std/src/process.rs index b45c620fd0..6903ba9056 100644 --- a/library/std/src/process.rs +++ b/library/std/src/process.rs @@ -110,6 +110,7 @@ use crate::ffi::OsStr; use crate::fmt; use crate::fs; use crate::io::{self, Initializer, IoSlice, IoSliceMut}; +use crate::num::NonZeroI32; use crate::path::Path; use crate::str; use crate::sys::pipe::{read2, AnonPipe}; @@ -1387,8 +1388,8 @@ impl From for Stdio { /// An `ExitStatus` represents every possible disposition of a process. On Unix this /// is the **wait status**. It is *not* simply an *exit status* (a value passed to `exit`). /// -/// For proper error reporting of failed processes, print the value of `ExitStatus` using its -/// implementation of [`Display`](crate::fmt::Display). +/// For proper error reporting of failed processes, print the value of `ExitStatus` or +/// `ExitStatusError` using their implementations of [`Display`](crate::fmt::Display). /// /// [`status`]: Command::status /// [`wait`]: Child::wait @@ -1401,6 +1402,29 @@ pub struct ExitStatus(imp::ExitStatus); impl crate::sealed::Sealed for ExitStatus {} impl ExitStatus { + /// Was termination successful? Returns a `Result`. + /// + /// # Examples + /// + /// ``` + /// #![feature(exit_status_error)] + /// # if cfg!(unix) { + /// use std::process::Command; + /// + /// let status = Command::new("ls") + /// .arg("/dev/nonexistent") + /// .status() + /// .expect("ls could not be executed"); + /// + /// println!("ls: {}", status); + /// status.exit_ok().expect_err("/dev/nonexistent could be listed!"); + /// # } // cfg!(unix) + /// ``` + #[unstable(feature = "exit_status_error", issue = "84908")] + pub fn exit_ok(&self) -> Result<(), ExitStatusError> { + self.0.exit_ok().map_err(ExitStatusError) + } + /// Was termination successful? Signal termination is not considered a /// success, and success is defined as a zero exit status. /// @@ -1422,7 +1446,7 @@ impl ExitStatus { /// ``` #[stable(feature = "process", since = "1.0.0")] pub fn success(&self) -> bool { - self.0.success() + self.0.exit_ok().is_ok() } /// Returns the exit code of the process, if any. @@ -1476,6 +1500,120 @@ impl fmt::Display for ExitStatus { } } +/// Allows extension traits within `std`. +#[unstable(feature = "sealed", issue = "none")] +impl crate::sealed::Sealed for ExitStatusError {} + +/// Describes the result of a process after it has failed +/// +/// Produced by the [`.exit_ok`](ExitStatus::exit_ok) method on [`ExitStatus`]. +/// +/// # Examples +/// +/// ``` +/// #![feature(exit_status_error)] +/// # if cfg!(unix) { +/// use std::process::{Command, ExitStatusError}; +/// +/// fn run(cmd: &str) -> Result<(),ExitStatusError> { +/// Command::new(cmd).status().unwrap().exit_ok()?; +/// Ok(()) +/// } +/// +/// run("true").unwrap(); +/// run("false").unwrap_err(); +/// # } // cfg!(unix) +/// ``` +#[derive(PartialEq, Eq, Clone, Copy, Debug)] +#[unstable(feature = "exit_status_error", issue = "84908")] +// The definition of imp::ExitStatusError should ideally be such that +// Result<(), imp::ExitStatusError> has an identical representation to imp::ExitStatus. +pub struct ExitStatusError(imp::ExitStatusError); + +#[unstable(feature = "exit_status_error", issue = "84908")] +impl ExitStatusError { + /// Reports the exit code, if applicable, from an `ExitStatusError`. + /// + /// In Unix terms the return value is the **exit status**: the value passed to `exit`, if the + /// process finished by calling `exit`. Note that on Unix the exit status is truncated to 8 + /// bits, and that values that didn't come from a program's call to `exit` may be invented by the + /// runtime system (often, for example, 255, 254, 127 or 126). + /// + /// On Unix, this will return `None` if the process was terminated by a signal. If you want to + /// handle such situations specially, consider using methods from + /// [`ExitStatusExt`](crate::os::unix::process::ExitStatusExt). + /// + /// If the process finished by calling `exit` with a nonzero value, this will return + /// that exit status. + /// + /// If the error was something else, it will return `None`. + /// + /// If the process exited successfully (ie, by calling `exit(0)`), there is no + /// `ExitStatusError`. So the return value from `ExitStatusError::code()` is always nonzero. + /// + /// # Examples + /// + /// ``` + /// #![feature(exit_status_error)] + /// # #[cfg(unix)] { + /// use std::process::Command; + /// + /// let bad = Command::new("false").status().unwrap().exit_ok().unwrap_err(); + /// assert_eq!(bad.code(), Some(1)); + /// # } // #[cfg(unix)] + /// ``` + pub fn code(&self) -> Option { + self.code_nonzero().map(Into::into) + } + + /// Reports the exit code, if applicable, from an `ExitStatusError`, as a `NonZero` + /// + /// This is exaclty like [`code()`](Self::code), except that it returns a `NonZeroI32`. + /// + /// Plain `code`, returning a plain integer, is provided because is is often more convenient. + /// The returned value from `code()` is indeed also nonzero; use `code_nonzero()` when you want + /// a type-level guarantee of nonzeroness. + /// + /// # Examples + /// + /// ``` + /// #![feature(exit_status_error)] + /// # if cfg!(unix) { + /// use std::convert::TryFrom; + /// use std::num::NonZeroI32; + /// use std::process::Command; + /// + /// let bad = Command::new("false").status().unwrap().exit_ok().unwrap_err(); + /// assert_eq!(bad.code_nonzero().unwrap(), NonZeroI32::try_from(1).unwrap()); + /// # } // cfg!(unix) + /// ``` + pub fn code_nonzero(&self) -> Option { + self.0.code() + } + + /// Converts an `ExitStatusError` (back) to an `ExitStatus`. + pub fn into_status(&self) -> ExitStatus { + ExitStatus(self.0.into()) + } +} + +#[unstable(feature = "exit_status_error", issue = "84908")] +impl Into for ExitStatusError { + fn into(self) -> ExitStatus { + ExitStatus(self.0.into()) + } +} + +#[unstable(feature = "exit_status_error", issue = "84908")] +impl fmt::Display for ExitStatusError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "process exited unsuccessfully: {}", self.into_status()) + } +} + +#[unstable(feature = "exit_status_error", issue = "84908")] +impl crate::error::Error for ExitStatusError {} + /// This type represents the status code a process can return to its /// parent under normal termination. /// diff --git a/library/std/src/sync/mutex.rs b/library/std/src/sync/mutex.rs index 773ab18b2c..e7c5479ab9 100644 --- a/library/std/src/sync/mutex.rs +++ b/library/std/src/sync/mutex.rs @@ -294,8 +294,14 @@ impl Mutex { /// # Errors /// /// If another user of this mutex panicked while holding the mutex, then - /// this call will return an error if the mutex would otherwise be - /// acquired. + /// this call will return the [`Poisoned`] error if the mutex would + /// otherwise be acquired. + /// + /// If the mutex could not be acquired because it is already locked, then + /// this call will return the [`WouldBlock`] error. + /// + /// [`Poisoned`]: TryLockError::Poisoned + /// [`WouldBlock`]: TryLockError::WouldBlock /// /// # Examples /// diff --git a/library/std/src/sync/rwlock.rs b/library/std/src/sync/rwlock.rs index b01bcec136..0d00f74eaa 100644 --- a/library/std/src/sync/rwlock.rs +++ b/library/std/src/sync/rwlock.rs @@ -3,9 +3,7 @@ mod tests; use crate::cell::UnsafeCell; use crate::fmt; -use crate::mem; use crate::ops::{Deref, DerefMut}; -use crate::ptr; use crate::sync::{poison, LockResult, TryLockError, TryLockResult}; use crate::sys_common::rwlock as sys; @@ -66,7 +64,7 @@ use crate::sys_common::rwlock as sys; /// [`Mutex`]: super::Mutex #[stable(feature = "rust1", since = "1.0.0")] pub struct RwLock { - inner: Box, + inner: sys::MovableRWLock, poison: poison::Flag, data: UnsafeCell, } @@ -130,7 +128,7 @@ impl RwLock { #[stable(feature = "rust1", since = "1.0.0")] pub fn new(t: T) -> RwLock { RwLock { - inner: box sys::RWLock::new(), + inner: sys::MovableRWLock::new(), poison: poison::Flag::new(), data: UnsafeCell::new(t), } @@ -199,11 +197,17 @@ impl RwLock { /// /// # Errors /// - /// This function will return an error if the RwLock is poisoned. An RwLock - /// is poisoned whenever a writer panics while holding an exclusive lock. An - /// error will only be returned if the lock would have otherwise been + /// This function will return the [`Poisoned`] error if the RwLock is poisoned. + /// An RwLock is poisoned whenever a writer panics while holding an exclusive + /// lock. `Poisoned` will only be returned if the lock would have otherwise been /// acquired. /// + /// This function will return the [`WouldBlock`] error if the RwLock could not + /// be acquired because it was already locked exclusively. + /// + /// [`Poisoned`]: TryLockError::Poisoned + /// [`WouldBlock`]: TryLockError::WouldBlock + /// /// # Examples /// /// ``` @@ -281,10 +285,17 @@ impl RwLock { /// /// # Errors /// - /// This function will return an error if the RwLock is poisoned. An RwLock - /// is poisoned whenever a writer panics while holding an exclusive lock. An - /// error will only be returned if the lock would have otherwise been - /// acquired. + /// This function will return the [`Poisoned`] error if the RwLock is + /// poisoned. An RwLock is poisoned whenever a writer panics while holding + /// an exclusive lock. `Poisoned` will only be returned if the lock would have + /// otherwise been acquired. + /// + /// This function will return the [`WouldBlock`] error if the RwLock could not + /// be acquired because it was already locked exclusively. + /// + /// [`Poisoned`]: TryLockError::Poisoned + /// [`WouldBlock`]: TryLockError::WouldBlock + /// /// /// # Examples /// @@ -363,24 +374,8 @@ impl RwLock { where T: Sized, { - // We know statically that there are no outstanding references to - // `self` so there's no need to lock the inner lock. - // - // To get the inner value, we'd like to call `data.into_inner()`, - // but because `RwLock` impl-s `Drop`, we can't move out of it, so - // we'll have to destructure it manually instead. - unsafe { - // Like `let RwLock { inner, poison, data } = self`. - let (inner, poison, data) = { - let RwLock { ref inner, ref poison, ref data } = self; - (ptr::read(inner), ptr::read(poison), ptr::read(data)) - }; - mem::forget(self); - inner.destroy(); // Keep in sync with the `Drop` impl. - drop(inner); - - poison::map_result(poison.borrow(), |_| data.into_inner()) - } + let data = self.data.into_inner(); + poison::map_result(self.poison.borrow(), |_| data) } /// Returns a mutable reference to the underlying data. @@ -411,14 +406,6 @@ impl RwLock { } } -#[stable(feature = "rust1", since = "1.0.0")] -unsafe impl<#[may_dangle] T: ?Sized> Drop for RwLock { - fn drop(&mut self) { - // IMPORTANT: This code needs to be kept in sync with `RwLock::into_inner`. - unsafe { self.inner.destroy() } - } -} - #[stable(feature = "rust1", since = "1.0.0")] impl fmt::Debug for RwLock { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { diff --git a/library/std/src/sys/hermit/fs.rs b/library/std/src/sys/hermit/fs.rs index 5b3f2fa4e8..76ea70d997 100644 --- a/library/std/src/sys/hermit/fs.rs +++ b/library/std/src/sys/hermit/fs.rs @@ -12,7 +12,7 @@ use crate::sys::time::SystemTime; use crate::sys::unsupported; use crate::sys_common::os_str_bytes::OsStrExt; -pub use crate::sys_common::fs::copy; +pub use crate::sys_common::fs::{copy, try_exists}; //pub use crate::sys_common::fs::remove_dir_all; fn cstr(path: &Path) -> io::Result { diff --git a/library/std/src/sys/hermit/mod.rs b/library/std/src/sys/hermit/mod.rs index 0c49a6fb6d..15a76bbd2c 100644 --- a/library/std/src/sys/hermit/mod.rs +++ b/library/std/src/sys/hermit/mod.rs @@ -24,7 +24,6 @@ pub mod args; pub mod cmath; pub mod condvar; pub mod env; -pub mod ext; pub mod fd; pub mod fs; #[path = "../unsupported/io.rs"] diff --git a/library/std/src/sys/hermit/os.rs b/library/std/src/sys/hermit/os.rs index 81cd68a74e..40bd393098 100644 --- a/library/std/src/sys/hermit/os.rs +++ b/library/std/src/sys/hermit/os.rs @@ -4,11 +4,11 @@ use crate::ffi::{CStr, OsStr, OsString}; use crate::fmt; use crate::io; use crate::marker::PhantomData; -use crate::memchr; use crate::path::{self, PathBuf}; use crate::str; use crate::sync::Mutex; use crate::sys::hermit::abi; +use crate::sys::memchr; use crate::sys::unsupported; use crate::sys_common::os_str_bytes::*; use crate::vec; diff --git a/library/std/src/sys/hermit/rwlock.rs b/library/std/src/sys/hermit/rwlock.rs index 06442e925f..d205818012 100644 --- a/library/std/src/sys/hermit/rwlock.rs +++ b/library/std/src/sys/hermit/rwlock.rs @@ -8,6 +8,8 @@ pub struct RWLock { state: UnsafeCell, } +pub type MovableRWLock = Box; + enum State { Unlocked, Reading(usize), diff --git a/library/std/src/sys/mod.rs b/library/std/src/sys/mod.rs index 2450a7aac5..f813587b1b 100644 --- a/library/std/src/sys/mod.rs +++ b/library/std/src/sys/mod.rs @@ -49,80 +49,27 @@ cfg_if::cfg_if! { } } -// Import essential modules from both platforms when documenting. These are -// then later used in the `std::os` module when documenting, for example, -// Windows when we're compiling for Linux. +// Import essential modules from platforms used in `std::os` when documenting. +// +// Note that on some platforms those modules don't compile +// (missing things in `libc` which is empty), so they are not included in `std::os` and can be +// omitted here as well. #[cfg(doc)] +#[cfg(not(any( + all(target_arch = "wasm32", not(target_os = "wasi")), + all(target_vendor = "fortanix", target_env = "sgx") +)))] cfg_if::cfg_if! { - if #[cfg(unix)] { - // On unix we'll document what's already available - #[stable(feature = "rust1", since = "1.0.0")] - pub use self::ext as unix_ext; - } else if #[cfg(any(target_os = "hermit", - all(target_arch = "wasm32", not(target_os = "wasi")), - all(target_vendor = "fortanix", target_env = "sgx")))] { - // On non-WASI wasm right now the module below doesn't compile - // (missing things in `libc` which is empty) so just omit everything - // with an empty module - #[unstable(issue = "none", feature = "std_internals")] - #[allow(missing_docs)] - pub mod unix_ext {} - } else { - #[path = "unix/ext/mod.rs"] - pub mod unix_ext; - } -} - -#[cfg(doc)] -cfg_if::cfg_if! { - if #[cfg(windows)] { - // On windows we'll just be documenting what's already available - #[allow(missing_docs)] - #[stable(feature = "rust1", since = "1.0.0")] - pub use self::ext as windows_ext; - } else if #[cfg(any(target_os = "hermit", - all(target_arch = "wasm32", not(target_os = "wasi")), - all(target_vendor = "fortanix", target_env = "sgx")))] { - // On non-WASI wasm right now the shim below doesn't compile, so - // just omit it - #[unstable(issue = "none", feature = "std_internals")] - #[allow(missing_docs)] - pub mod windows_ext {} - } else { - // On all other platforms (aka linux/osx/etc) then pull in a "minimal" + if #[cfg(not(windows))] { + // On non-Windows platforms (aka linux/osx/etc) pull in a "minimal" // amount of windows goop which ends up compiling + #[macro_use] #[path = "windows/compat.rs"] - mod compat; + pub mod compat; #[path = "windows/c.rs"] - mod c; - - #[path = "windows/ext/mod.rs"] - pub mod windows_ext; - } -} - -#[cfg(doc)] -cfg_if::cfg_if! { - if #[cfg(target_os = "wasi")] { - // On WASI we'll document what's already available - #[stable(feature = "wasi_ext_doc", since = "1.35.0")] - pub use self::ext as wasi_ext; - } else if #[cfg(any(target_os = "hermit", - target_arch = "wasm32", - all(target_vendor = "fortanix", target_env = "sgx")))] { - // On non-WASI wasm right now the module below doesn't compile - // (missing things in `libc` which is empty) so just omit everything - // with an empty module - #[unstable(issue = "none", feature = "std_internals")] - #[allow(missing_docs)] - pub mod wasi_ext {} - } else { - // On other platforms like Windows document the bare bones of WASI - #[path = "wasi/ext/mod.rs"] - #[stable(feature = "wasi_ext_doc", since = "1.35.0")] - pub mod wasi_ext; + pub mod c; } } diff --git a/library/std/src/sys/sgx/abi/mod.rs b/library/std/src/sys/sgx/abi/mod.rs index f9536c4203..231cc15b84 100644 --- a/library/std/src/sys/sgx/abi/mod.rs +++ b/library/std/src/sys/sgx/abi/mod.rs @@ -15,7 +15,7 @@ pub mod tls; pub mod usercalls; #[cfg(not(test))] -global_asm!(include_str!("entry.S")); +global_asm!(include_str!("entry.S"), options(att_syntax)); #[repr(C)] struct EntryReturn(u64, u64); diff --git a/library/std/src/sys/sgx/abi/tls.rs b/library/std/src/sys/sgx/abi/tls/mod.rs similarity index 100% rename from library/std/src/sys/sgx/abi/tls.rs rename to library/std/src/sys/sgx/abi/tls/mod.rs diff --git a/library/std/src/sys/sgx/ext/mod.rs b/library/std/src/sys/sgx/ext/mod.rs deleted file mode 100644 index 258ad3cd21..0000000000 --- a/library/std/src/sys/sgx/ext/mod.rs +++ /dev/null @@ -1,5 +0,0 @@ -#![unstable(feature = "sgx_platform", issue = "56975")] - -pub mod arch; -pub mod ffi; -pub mod io; diff --git a/library/std/src/sys/sgx/mod.rs b/library/std/src/sys/sgx/mod.rs index bf3bd57e98..cdfceca19f 100644 --- a/library/std/src/sys/sgx/mod.rs +++ b/library/std/src/sys/sgx/mod.rs @@ -17,7 +17,6 @@ pub mod args; pub mod cmath; pub mod condvar; pub mod env; -pub mod ext; pub mod fd; #[path = "../unsupported/fs.rs"] pub mod fs; diff --git a/library/std/src/sys/sgx/mutex.rs b/library/std/src/sys/sgx/mutex.rs index 8874517dac..1b5ced4178 100644 --- a/library/std/src/sys/sgx/mutex.rs +++ b/library/std/src/sys/sgx/mutex.rs @@ -8,7 +8,7 @@ pub struct Mutex { inner: SpinMutex>, } -pub type MovableMutex = Box; +pub type MovableMutex = Mutex; // Implementation according to “Operating Systems: Three Easy Pieces”, chapter 28 impl Mutex { diff --git a/library/std/src/sys/sgx/rwlock.rs b/library/std/src/sys/sgx/rwlock.rs index 0c96e3fcdd..2d038b5189 100644 --- a/library/std/src/sys/sgx/rwlock.rs +++ b/library/std/src/sys/sgx/rwlock.rs @@ -13,6 +13,8 @@ pub struct RWLock { writer: SpinMutex>, } +pub type MovableRWLock = Box; + // Check at compile time that RWLock size matches C definition (see test_c_rwlock_initializer below) // // # Safety diff --git a/library/std/src/sys/sgx/waitqueue.rs b/library/std/src/sys/sgx/waitqueue/mod.rs similarity index 97% rename from library/std/src/sys/sgx/waitqueue.rs rename to library/std/src/sys/sgx/waitqueue/mod.rs index e464dc3ee9..61bb11d9a6 100644 --- a/library/std/src/sys/sgx/waitqueue.rs +++ b/library/std/src/sys/sgx/waitqueue/mod.rs @@ -13,13 +13,8 @@ #[cfg(test)] mod tests; -/// A doubly-linked list where callers are in charge of memory allocation -/// of the nodes in the list. -mod unsafe_list; - -/// Trivial spinlock-based implementation of `sync::Mutex`. -// FIXME: Perhaps use Intel TSX to avoid locking? mod spin_mutex; +mod unsafe_list; use crate::num::NonZeroUsize; use crate::ops::{Deref, DerefMut}; diff --git a/library/std/src/sys/sgx/waitqueue/spin_mutex.rs b/library/std/src/sys/sgx/waitqueue/spin_mutex.rs index 7f1a671bab..f6e851ccad 100644 --- a/library/std/src/sys/sgx/waitqueue/spin_mutex.rs +++ b/library/std/src/sys/sgx/waitqueue/spin_mutex.rs @@ -1,3 +1,6 @@ +//! Trivial spinlock-based implementation of `sync::Mutex`. +// FIXME: Perhaps use Intel TSX to avoid locking? + #[cfg(test)] mod tests; diff --git a/library/std/src/sys/sgx/waitqueue/unsafe_list.rs b/library/std/src/sys/sgx/waitqueue/unsafe_list.rs index 0834d2593f..cf2f0886c1 100644 --- a/library/std/src/sys/sgx/waitqueue/unsafe_list.rs +++ b/library/std/src/sys/sgx/waitqueue/unsafe_list.rs @@ -1,3 +1,6 @@ +//! A doubly-linked list where callers are in charge of memory allocation +//! of the nodes in the list. + #[cfg(test)] mod tests; diff --git a/library/std/src/sys/unix/fs.rs b/library/std/src/sys/unix/fs.rs index 45bae25a0c..f8ca67c844 100644 --- a/library/std/src/sys/unix/fs.rs +++ b/library/std/src/sys/unix/fs.rs @@ -48,7 +48,7 @@ use libc::{ dirent64, fstat64, ftruncate64, lseek64, lstat64, off64_t, open64, readdir64_r, stat64, }; -pub use crate::sys_common::fs::remove_dir_all; +pub use crate::sys_common::fs::{remove_dir_all, try_exists}; pub struct File(FileDesc); @@ -116,7 +116,7 @@ cfg_has_statx! {{ match STATX_STATE.load(Ordering::Relaxed) { 0 => { - // It is a trick to call `statx` with NULL pointers to check if the syscall + // It is a trick to call `statx` with null pointers to check if the syscall // is available. According to the manual, it is expected to fail with EFAULT. // We do this mainly for performance, since it is nearly hundreds times // faster than a normal successful call. @@ -450,7 +450,7 @@ impl Iterator for ReadDir { super::os::set_errno(0); let entry_ptr = libc::readdir(self.inner.dirp.0); if entry_ptr.is_null() { - // NULL can mean either the end is reached or an error occurred. + // null can mean either the end is reached or an error occurred. // So we had to clear errno beforehand to check for an error now. return match super::os::errno() { 0 => None, @@ -1329,7 +1329,7 @@ pub fn copy(from: &Path, to: &Path) -> io::Result { Ok(bytes_copied as u64) } -#[cfg(not(target_os = "fuchsia"))] +#[cfg(not(any(target_os = "fuchsia", target_os = "vxworks")))] pub fn chroot(dir: &Path) -> io::Result<()> { let dir = cstr(dir)?; cvt(unsafe { libc::chroot(dir.as_ptr()) })?; diff --git a/library/std/src/sys/unix/mod.rs b/library/std/src/sys/unix/mod.rs index 6c4fbaf273..ca9cc8ca7b 100644 --- a/library/std/src/sys/unix/mod.rs +++ b/library/std/src/sys/unix/mod.rs @@ -15,7 +15,6 @@ pub mod args; pub mod cmath; pub mod condvar; pub mod env; -pub mod ext; pub mod fd; pub mod fs; pub mod futex; @@ -149,6 +148,7 @@ pub fn decode_error_kind(errno: i32) -> ErrorKind { libc::ETIMEDOUT => ErrorKind::TimedOut, libc::EEXIST => ErrorKind::AlreadyExists, libc::ENOSYS => ErrorKind::Unsupported, + libc::ENOMEM => ErrorKind::OutOfMemory, // These two constants can have the same value on some systems, // but different values on others, so we can't use a match @@ -210,7 +210,6 @@ cfg_if::cfg_if! { if #[cfg(target_os = "android")] { #[link(name = "dl")] #[link(name = "log")] - #[link(name = "gcc")] extern "C" {} } else if #[cfg(target_os = "freebsd")] { #[link(name = "execinfo")] diff --git a/library/std/src/sys/unix/net.rs b/library/std/src/sys/unix/net.rs index e6b61062d1..d5a15964c0 100644 --- a/library/std/src/sys/unix/net.rs +++ b/library/std/src/sys/unix/net.rs @@ -62,7 +62,7 @@ impl Socket { target_os = "illumos", target_os = "linux", target_os = "netbsd", - target_os = "opensbd", + target_os = "openbsd", ))] { // On platforms that support it we pass the SOCK_CLOEXEC // flag to atomically create the socket and set it as @@ -99,7 +99,7 @@ impl Socket { target_os = "illumos", target_os = "linux", target_os = "netbsd", - target_os = "opensbd", + target_os = "openbsd", ))] { // Like above, set cloexec atomically cvt(libc::socketpair(fam, ty | libc::SOCK_CLOEXEC, 0, fds.as_mut_ptr()))?; @@ -204,7 +204,7 @@ impl Socket { target_os = "illumos", target_os = "linux", target_os = "netbsd", - target_os = "opensbd", + target_os = "openbsd", ))] { let fd = cvt_r(|| unsafe { libc::accept4(self.0.raw(), storage, len, libc::SOCK_CLOEXEC) diff --git a/library/std/src/sys/unix/os.rs b/library/std/src/sys/unix/os.rs index 984c08c2ad..41ca976239 100644 --- a/library/std/src/sys/unix/os.rs +++ b/library/std/src/sys/unix/os.rs @@ -13,15 +13,14 @@ use crate::fmt; use crate::io; use crate::iter; use crate::mem; -use crate::memchr; use crate::path::{self, PathBuf}; use crate::ptr; use crate::slice; use crate::str; use crate::sys::cvt; use crate::sys::fd; -use crate::sys::rwlock::{RWLockReadGuard, StaticRWLock}; -use crate::sys_common::mutex::{StaticMutex, StaticMutexGuard}; +use crate::sys::memchr; +use crate::sys_common::rwlock::{StaticRWLock, StaticRWLockReadGuard}; use crate::vec; use libc::{c_char, c_int, c_void}; @@ -155,12 +154,10 @@ pub fn getcwd() -> io::Result { pub fn chdir(p: &path::Path) -> io::Result<()> { let p: &OsStr = p.as_ref(); let p = CString::new(p.as_bytes())?; - unsafe { - match libc::chdir(p.as_ptr()) == (0 as c_int) { - true => Ok(()), - false => Err(io::Error::last_os_error()), - } + if unsafe { libc::chdir(p.as_ptr()) } != 0 { + return Err(io::Error::last_os_error()); } + Ok(()) } pub struct SplitPaths<'a> { @@ -492,8 +489,8 @@ pub unsafe fn environ() -> *mut *const *const c_char { static ENV_LOCK: StaticRWLock = StaticRWLock::new(); -pub fn env_read_lock() -> RWLockReadGuard { - ENV_LOCK.read_with_guard() +pub fn env_read_lock() -> StaticRWLockReadGuard { + ENV_LOCK.read() } /// Returns a vector of (variable, value) byte-vector pairs for all the @@ -553,7 +550,7 @@ pub fn setenv(k: &OsStr, v: &OsStr) -> io::Result<()> { let v = CString::new(v.as_bytes())?; unsafe { - let _guard = ENV_LOCK.write_with_guard(); + let _guard = ENV_LOCK.write(); cvt(libc::setenv(k.as_ptr(), v.as_ptr(), 1)).map(drop) } } @@ -562,7 +559,7 @@ pub fn unsetenv(n: &OsStr) -> io::Result<()> { let nbuf = CString::new(n.as_bytes())?; unsafe { - let _guard = ENV_LOCK.write_with_guard(); + let _guard = ENV_LOCK.write(); cvt(libc::unsetenv(nbuf.as_ptr())).map(drop) } } diff --git a/library/std/src/sys/unix/process/mod.rs b/library/std/src/sys/unix/process/mod.rs index f67c70c017..b5a19ed54a 100644 --- a/library/std/src/sys/unix/process/mod.rs +++ b/library/std/src/sys/unix/process/mod.rs @@ -1,5 +1,5 @@ pub use self::process_common::{Command, CommandArgs, ExitCode, Stdio, StdioPipes}; -pub use self::process_inner::{ExitStatus, Process}; +pub use self::process_inner::{ExitStatus, ExitStatusError, Process}; pub use crate::ffi::OsString as EnvKey; pub use crate::sys_common::process::CommandEnvs; diff --git a/library/std/src/sys/unix/process/process_common.rs b/library/std/src/sys/unix/process/process_common.rs index b9dcc4e4b9..c5bdd1bda4 100644 --- a/library/std/src/sys/unix/process/process_common.rs +++ b/library/std/src/sys/unix/process/process_common.rs @@ -153,7 +153,7 @@ impl Command { } pub fn arg(&mut self, arg: &OsStr) { - // Overwrite the trailing NULL pointer in `argv` and then add a new null + // Overwrite the trailing null pointer in `argv` and then add a new null // pointer. let arg = os2c(arg, &mut self.saw_nul); self.argv.0[self.args.len()] = arg.as_ptr(); diff --git a/library/std/src/sys/unix/process/process_fuchsia.rs b/library/std/src/sys/unix/process/process_fuchsia.rs index b19ad4ccdc..507abb2787 100644 --- a/library/std/src/sys/unix/process/process_fuchsia.rs +++ b/library/std/src/sys/unix/process/process_fuchsia.rs @@ -1,7 +1,8 @@ -use crate::convert::TryInto; +use crate::convert::{TryFrom, TryInto}; use crate::fmt; use crate::io; use crate::mem; +use crate::num::{NonZeroI32, NonZeroI64}; use crate::ptr; use crate::sys::process::process_common::*; @@ -236,8 +237,11 @@ impl Process { pub struct ExitStatus(i64); impl ExitStatus { - pub fn success(&self) -> bool { - self.code() == Some(0) + pub fn exit_ok(&self) -> Result<(), ExitStatusError> { + match NonZeroI64::try_from(self.0) { + /* was nonzero */ Ok(failure) => Err(ExitStatusError(failure)), + /* was zero, couldn't convert */ Err(_) => Ok(()), + } } pub fn code(&self) -> Option { @@ -306,3 +310,19 @@ impl fmt::Display for ExitStatus { write!(f, "exit code: {}", self.0) } } + +#[derive(PartialEq, Eq, Clone, Copy, Debug)] +pub struct ExitStatusError(NonZeroI64); + +impl Into for ExitStatusError { + fn into(self) -> ExitStatus { + ExitStatus(self.0.into()) + } +} + +impl ExitStatusError { + pub fn code(self) -> Option { + // fixme: affected by the same bug as ExitStatus::code() + ExitStatus(self.0.into()).code().map(|st| st.try_into().unwrap()) + } +} diff --git a/library/std/src/sys/unix/process/process_unix.rs b/library/std/src/sys/unix/process/process_unix.rs index ed9044382a..ed55e1aa71 100644 --- a/library/std/src/sys/unix/process/process_unix.rs +++ b/library/std/src/sys/unix/process/process_unix.rs @@ -1,7 +1,9 @@ -use crate::convert::TryInto; +use crate::convert::{TryFrom, TryInto}; use crate::fmt; use crate::io::{self, Error, ErrorKind}; use crate::mem; +use crate::num::NonZeroI32; +use crate::os::raw::NonZero_c_int; use crate::ptr; use crate::sys; use crate::sys::cvt; @@ -54,6 +56,7 @@ impl Command { let (env_lock, pid) = unsafe { (sys::os::env_read_lock(), cvt(libc::fork())?) }; if pid == 0 { + crate::panic::always_abort(); mem::forget(env_lock); drop(input); let Err(err) = unsafe { self.do_exec(theirs, envp.as_ref()) }; @@ -490,8 +493,16 @@ impl ExitStatus { libc::WIFEXITED(self.0) } - pub fn success(&self) -> bool { - self.code() == Some(0) + pub fn exit_ok(&self) -> Result<(), ExitStatusError> { + // This assumes that WIFEXITED(status) && WEXITSTATUS==0 corresponds to status==0. This is + // true on all actual versions of Unix, is widely assumed, and is specified in SuS + // https://pubs.opengroup.org/onlinepubs/9699919799/functions/wait.html . If it is not + // true for a platform pretending to be Unix, the tests (our doctests, and also + // procsss_unix/tests.rs) will spot it. `ExitStatusError::code` assumes this too. + match NonZero_c_int::try_from(self.0) { + /* was nonzero */ Ok(failure) => Err(ExitStatusError(failure)), + /* was zero, couldn't convert */ Err(_) => Ok(()), + } } pub fn code(&self) -> Option { @@ -546,6 +557,21 @@ impl fmt::Display for ExitStatus { } } +#[derive(PartialEq, Eq, Clone, Copy, Debug)] +pub struct ExitStatusError(NonZero_c_int); + +impl Into for ExitStatusError { + fn into(self) -> ExitStatus { + ExitStatus(self.0.into()) + } +} + +impl ExitStatusError { + pub fn code(self) -> Option { + ExitStatus(self.0.into()).code().map(|st| st.try_into().unwrap()) + } +} + #[cfg(test)] #[path = "process_unix/tests.rs"] mod tests; diff --git a/library/std/src/sys/unix/process/process_unix/tests.rs b/library/std/src/sys/unix/process/process_unix/tests.rs index 02c469fbcd..157debf2d2 100644 --- a/library/std/src/sys/unix/process/process_unix/tests.rs +++ b/library/std/src/sys/unix/process/process_unix/tests.rs @@ -1,3 +1,10 @@ +use crate::os::unix::process::{CommandExt, ExitStatusExt}; +use crate::panic::catch_unwind; +use crate::process::Command; + +// Many of the other aspects of this situation, including heap alloc concurrency +// safety etc., are tested in src/test/ui/process/process-panic-after-fork.rs + #[test] fn exitstatus_display_tests() { // In practice this is the same on every Unix. @@ -28,3 +35,23 @@ fn exitstatus_display_tests() { t(0x000ff, "unrecognised wait status: 255 0xff"); } } + +#[test] +#[cfg_attr(target_os = "emscripten", ignore)] +fn test_command_fork_no_unwind() { + let got = catch_unwind(|| { + let mut c = Command::new("echo"); + c.arg("hi"); + unsafe { + c.pre_exec(|| panic!("{}", "crash now!")); + } + let st = c.status().expect("failed to get command status"); + dbg!(st); + st + }); + dbg!(&got); + let status = got.expect("panic unexpectedly propagated"); + dbg!(status); + let signal = status.signal().expect("expected child process to die of signal"); + assert!(signal == libc::SIGABRT || signal == libc::SIGILL || signal == libc::SIGTRAP); +} diff --git a/library/std/src/sys/unix/process/process_vxworks.rs b/library/std/src/sys/unix/process/process_vxworks.rs index eecdb624b9..c17822f512 100644 --- a/library/std/src/sys/unix/process/process_vxworks.rs +++ b/library/std/src/sys/unix/process/process_vxworks.rs @@ -1,5 +1,8 @@ +use crate::convert::{TryFrom, TryInto}; use crate::fmt; use crate::io::{self, Error, ErrorKind}; +use crate::num::NonZeroI32; +use crate::os::raw::NonZero_c_int; use crate::sys; use crate::sys::cvt; use crate::sys::process::process_common::*; @@ -187,8 +190,16 @@ impl ExitStatus { libc::WIFEXITED(self.0) } - pub fn success(&self) -> bool { - self.code() == Some(0) + pub fn exit_ok(&self) -> Result<(), ExitStatusError> { + // This assumes that WIFEXITED(status) && WEXITSTATUS==0 corresponds to status==0. This is + // true on all actual versions of Unix, is widely assumed, and is specified in SuS + // https://pubs.opengroup.org/onlinepubs/9699919799/functions/wait.html . If it is not + // true for a platform pretending to be Unix, the tests (our doctests, and also + // procsss_unix/tests.rs) will spot it. `ExitStatusError::code` assumes this too. + match NonZero_c_int::try_from(self.0) { + Ok(failure) => Err(ExitStatusError(failure)), + Err(_) => Ok(()), + } } pub fn code(&self) -> Option { @@ -235,3 +246,18 @@ impl fmt::Display for ExitStatus { } } } + +#[derive(PartialEq, Eq, Clone, Copy, Debug)] +pub struct ExitStatusError(NonZero_c_int); + +impl Into for ExitStatusError { + fn into(self) -> ExitStatus { + ExitStatus(self.0.into()) + } +} + +impl ExitStatusError { + pub fn code(self) -> Option { + ExitStatus(self.0.into()).code().map(|st| st.try_into().unwrap()) + } +} diff --git a/library/std/src/sys/unix/rwlock.rs b/library/std/src/sys/unix/rwlock.rs index d97d9d712f..b1faf12c22 100644 --- a/library/std/src/sys/unix/rwlock.rs +++ b/library/std/src/sys/unix/rwlock.rs @@ -7,6 +7,8 @@ pub struct RWLock { num_readers: AtomicUsize, } +pub type MovableRWLock = Box; + unsafe impl Send for RWLock {} unsafe impl Sync for RWLock {} @@ -139,55 +141,3 @@ impl RWLock { } } } - -pub struct StaticRWLock(RWLock); - -impl StaticRWLock { - pub const fn new() -> StaticRWLock { - StaticRWLock(RWLock::new()) - } - - /// Acquires shared access to the underlying lock, blocking the current - /// thread to do so. - /// - /// The lock is automatically unlocked when the returned guard is dropped. - #[inline] - pub fn read_with_guard(&'static self) -> RWLockReadGuard { - // SAFETY: All methods require static references, therefore self - // cannot be moved between invocations. - unsafe { - self.0.read(); - } - RWLockReadGuard(&self.0) - } - - /// Acquires write access to the underlying lock, blocking the current thread - /// to do so. - /// - /// The lock is automatically unlocked when the returned guard is dropped. - #[inline] - pub fn write_with_guard(&'static self) -> RWLockWriteGuard { - // SAFETY: All methods require static references, therefore self - // cannot be moved between invocations. - unsafe { - self.0.write(); - } - RWLockWriteGuard(&self.0) - } -} - -pub struct RWLockReadGuard(&'static RWLock); - -impl Drop for RWLockReadGuard { - fn drop(&mut self) { - unsafe { self.0.read_unlock() } - } -} - -pub struct RWLockWriteGuard(&'static RWLock); - -impl Drop for RWLockWriteGuard { - fn drop(&mut self) { - unsafe { self.0.write_unlock() } - } -} diff --git a/library/std/src/sys/unix/stack_overflow.rs b/library/std/src/sys/unix/stack_overflow.rs index 2a487fff54..81f47a779d 100644 --- a/library/std/src/sys/unix/stack_overflow.rs +++ b/library/std/src/sys/unix/stack_overflow.rs @@ -42,6 +42,7 @@ mod imp { use crate::io; use crate::mem; use crate::ptr; + use crate::thread; use libc::MAP_FAILED; use libc::{mmap, munmap}; @@ -95,15 +96,16 @@ mod imp { info: *mut libc::siginfo_t, _data: *mut libc::c_void, ) { - use crate::sys_common::util::report_overflow; - let guard = thread_info::stack_guard().unwrap_or(0..0); let addr = siginfo_si_addr(info); // If the faulting address is within the guard page, then we print a // message saying so and abort. if guard.start <= addr && addr < guard.end { - report_overflow(); + rtprintpanic!( + "\nthread '{}' has overflowed its stack\n", + thread::current().name().unwrap_or("") + ); rtabort!("stack overflow"); } else { // Unregister ourselves by reverting back to the default behavior. diff --git a/library/std/src/sys/unsupported/args.rs b/library/std/src/sys/unsupported/args.rs index c924a7d8a2..a2d75a6197 100644 --- a/library/std/src/sys/unsupported/args.rs +++ b/library/std/src/sys/unsupported/args.rs @@ -1,4 +1,5 @@ use crate::ffi::OsString; +use crate::fmt; pub struct Args {} diff --git a/library/std/src/sys/unsupported/fs.rs b/library/std/src/sys/unsupported/fs.rs index cd533761e3..6b45e29c14 100644 --- a/library/std/src/sys/unsupported/fs.rs +++ b/library/std/src/sys/unsupported/fs.rs @@ -275,6 +275,10 @@ pub fn remove_dir_all(_path: &Path) -> io::Result<()> { unsupported() } +pub fn try_exists(_path: &Path) -> io::Result { + unsupported() +} + pub fn readlink(_p: &Path) -> io::Result { unsupported() } diff --git a/library/std/src/sys/unsupported/process.rs b/library/std/src/sys/unsupported/process.rs index 38ac0a1ddd..7846e43cfb 100644 --- a/library/std/src/sys/unsupported/process.rs +++ b/library/std/src/sys/unsupported/process.rs @@ -2,6 +2,7 @@ use crate::ffi::OsStr; use crate::fmt; use crate::io; use crate::marker::PhantomData; +use crate::num::NonZeroI32; use crate::path::Path; use crate::sys::fs::File; use crate::sys::pipe::AnonPipe; @@ -97,7 +98,7 @@ impl fmt::Debug for Command { pub struct ExitStatus(!); impl ExitStatus { - pub fn success(&self) -> bool { + pub fn exit_ok(&self) -> Result<(), ExitStatusError> { self.0 } @@ -134,6 +135,21 @@ impl fmt::Display for ExitStatus { } } +#[derive(PartialEq, Eq, Clone, Copy, Debug)] +pub struct ExitStatusError(ExitStatus); + +impl Into for ExitStatusError { + fn into(self) -> ExitStatus { + self.0.0 + } +} + +impl ExitStatusError { + pub fn code(self) -> Option { + self.0.0 + } +} + #[derive(PartialEq, Eq, Clone, Copy, Debug)] pub struct ExitCode(bool); diff --git a/library/std/src/sys/unsupported/rwlock.rs b/library/std/src/sys/unsupported/rwlock.rs index 6982b2b155..8438adeb5b 100644 --- a/library/std/src/sys/unsupported/rwlock.rs +++ b/library/std/src/sys/unsupported/rwlock.rs @@ -5,6 +5,8 @@ pub struct RWLock { mode: Cell, } +pub type MovableRWLock = RWLock; + unsafe impl Send for RWLock {} unsafe impl Sync for RWLock {} // no threads on this platform diff --git a/library/std/src/sys/wasi/fs.rs b/library/std/src/sys/wasi/fs.rs index ed0f03e4b7..45e38f68b8 100644 --- a/library/std/src/sys/wasi/fs.rs +++ b/library/std/src/sys/wasi/fs.rs @@ -14,7 +14,7 @@ use crate::sys::time::SystemTime; use crate::sys::unsupported; use crate::sys_common::FromInner; -pub use crate::sys_common::fs::remove_dir_all; +pub use crate::sys_common::fs::{remove_dir_all, try_exists}; pub struct File { fd: WasiFd, diff --git a/library/std/src/sys/wasi/mod.rs b/library/std/src/sys/wasi/mod.rs index 37f74fcc05..45a829c0cd 100644 --- a/library/std/src/sys/wasi/mod.rs +++ b/library/std/src/sys/wasi/mod.rs @@ -33,7 +33,6 @@ pub mod mutex; pub mod net; pub mod os; pub use crate::sys_common::os_str_bytes as os_str; -pub mod ext; #[path = "../unix/path.rs"] pub mod path; #[path = "../unsupported/pipe.rs"] @@ -77,6 +76,7 @@ pub fn decode_error_kind(errno: i32) -> std_io::ErrorKind { wasi::ERRNO_EXIST => AlreadyExists, wasi::ERRNO_AGAIN => WouldBlock, wasi::ERRNO_NOSYS => Unsupported, + wasi::ERRNO_NOMEM => OutOfMemory, _ => Other, } } diff --git a/library/std/src/sys/wasm/args.rs b/library/std/src/sys/wasm/args.rs deleted file mode 100644 index fde1ab79e1..0000000000 --- a/library/std/src/sys/wasm/args.rs +++ /dev/null @@ -1,42 +0,0 @@ -use crate::ffi::OsString; -use crate::fmt; -use crate::vec; - -pub fn args() -> Args { - Args { iter: Vec::new().into_iter() } -} - -pub struct Args { - iter: vec::IntoIter, -} - -impl !Send for Args {} -impl !Sync for Args {} - -impl fmt::Debug for Args { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - self.iter.as_slice().fmt(f) - } -} - -impl Iterator for Args { - type Item = OsString; - fn next(&mut self) -> Option { - self.iter.next() - } - fn size_hint(&self) -> (usize, Option) { - self.iter.size_hint() - } -} - -impl ExactSizeIterator for Args { - fn len(&self) -> usize { - self.iter.len() - } -} - -impl DoubleEndedIterator for Args { - fn next_back(&mut self) -> Option { - self.iter.next_back() - } -} diff --git a/library/std/src/sys/wasm/condvar_atomics.rs b/library/std/src/sys/wasm/atomics/condvar.rs similarity index 100% rename from library/std/src/sys/wasm/condvar_atomics.rs rename to library/std/src/sys/wasm/atomics/condvar.rs diff --git a/library/std/src/sys/wasm/futex_atomics.rs b/library/std/src/sys/wasm/atomics/futex.rs similarity index 100% rename from library/std/src/sys/wasm/futex_atomics.rs rename to library/std/src/sys/wasm/atomics/futex.rs diff --git a/library/std/src/sys/wasm/mutex_atomics.rs b/library/std/src/sys/wasm/atomics/mutex.rs similarity index 100% rename from library/std/src/sys/wasm/mutex_atomics.rs rename to library/std/src/sys/wasm/atomics/mutex.rs diff --git a/library/std/src/sys/wasm/rwlock_atomics.rs b/library/std/src/sys/wasm/atomics/rwlock.rs similarity index 99% rename from library/std/src/sys/wasm/rwlock_atomics.rs rename to library/std/src/sys/wasm/atomics/rwlock.rs index 06442e925f..64eaa2fc48 100644 --- a/library/std/src/sys/wasm/rwlock_atomics.rs +++ b/library/std/src/sys/wasm/atomics/rwlock.rs @@ -8,6 +8,8 @@ pub struct RWLock { state: UnsafeCell, } +pub type MovableRWLock = RWLock; + enum State { Unlocked, Reading(usize), diff --git a/library/std/src/sys/wasm/thread.rs b/library/std/src/sys/wasm/atomics/thread.rs similarity index 83% rename from library/std/src/sys/wasm/thread.rs rename to library/std/src/sys/wasm/atomics/thread.rs index b7bf95c89b..54bc877aa7 100644 --- a/library/std/src/sys/wasm/thread.rs +++ b/library/std/src/sys/wasm/atomics/thread.rs @@ -13,20 +13,10 @@ impl Thread { unsupported() } - pub fn yield_now() { - // do nothing - } + pub fn yield_now() {} - pub fn set_name(_name: &CStr) { - // nope - } + pub fn set_name(_name: &CStr) {} - #[cfg(not(target_feature = "atomics"))] - pub fn sleep(_dur: Duration) { - panic!("can't sleep"); - } - - #[cfg(target_feature = "atomics")] pub fn sleep(dur: Duration) { use crate::arch::wasm32; use crate::cmp; @@ -46,9 +36,7 @@ impl Thread { } } - pub fn join(self) { - self.0 - } + pub fn join(self) {} } pub mod guard { @@ -61,11 +49,9 @@ pub mod guard { } } -// This is only used by atomics primitives when the `atomics` feature is -// enabled. In that mode we currently just use our own thread-local to store our +// We currently just use our own thread-local to store our // current thread's ID, and then we lazily initialize it to something allocated // from a global counter. -#[cfg(target_feature = "atomics")] pub fn my_id() -> u32 { use crate::sync::atomic::{AtomicU32, Ordering::SeqCst}; diff --git a/library/std/src/sys/wasm/mod.rs b/library/std/src/sys/wasm/mod.rs index afcc5ca928..cd701a333f 100644 --- a/library/std/src/sys/wasm/mod.rs +++ b/library/std/src/sys/wasm/mod.rs @@ -17,6 +17,7 @@ #![deny(unsafe_op_in_unsafe_fn)] pub mod alloc; +#[path = "../unsupported/args.rs"] pub mod args; #[path = "../unix/cmath.rs"] pub mod cmath; @@ -37,7 +38,6 @@ pub mod pipe; pub mod process; #[path = "../unsupported/stdio.rs"] pub mod stdio; -pub mod thread; #[path = "../unsupported/thread_local_dtor.rs"] pub mod thread_local_dtor; #[path = "../unsupported/thread_local_key.rs"] @@ -49,14 +49,16 @@ pub use crate::sys_common::os_str_bytes as os_str; cfg_if::cfg_if! { if #[cfg(target_feature = "atomics")] { - #[path = "condvar_atomics.rs"] + #[path = "atomics/condvar.rs"] pub mod condvar; - #[path = "mutex_atomics.rs"] + #[path = "atomics/mutex.rs"] pub mod mutex; - #[path = "rwlock_atomics.rs"] + #[path = "atomics/rwlock.rs"] pub mod rwlock; - #[path = "futex_atomics.rs"] + #[path = "atomics/futex.rs"] pub mod futex; + #[path = "atomics/thread.rs"] + pub mod thread; } else { #[path = "../unsupported/condvar.rs"] pub mod condvar; @@ -64,6 +66,8 @@ cfg_if::cfg_if! { pub mod mutex; #[path = "../unsupported/rwlock.rs"] pub mod rwlock; + #[path = "../unsupported/thread.rs"] + pub mod thread; } } diff --git a/library/std/src/sys/windows/c.rs b/library/std/src/sys/windows/c.rs index 3e4176ef7f..b7efc88447 100644 --- a/library/std/src/sys/windows/c.rs +++ b/library/std/src/sys/windows/c.rs @@ -4,6 +4,7 @@ #![cfg_attr(test, allow(dead_code))] #![unstable(issue = "none", feature = "windows_c")] +use crate::os::raw::NonZero_c_ulong; use crate::os::raw::{c_char, c_int, c_long, c_longlong, c_uint, c_ulong, c_ushort}; use crate::ptr; @@ -13,6 +14,7 @@ pub use self::EXCEPTION_DISPOSITION::*; pub use self::FILE_INFO_BY_HANDLE_CLASS::*; pub type DWORD = c_ulong; +pub type NonZeroDWORD = NonZero_c_ulong; pub type HANDLE = LPVOID; pub type HINSTANCE = HANDLE; pub type HMODULE = HINSTANCE; @@ -168,7 +170,10 @@ pub const ERROR_FILE_NOT_FOUND: DWORD = 2; pub const ERROR_PATH_NOT_FOUND: DWORD = 3; pub const ERROR_ACCESS_DENIED: DWORD = 5; pub const ERROR_INVALID_HANDLE: DWORD = 6; +pub const ERROR_NOT_ENOUGH_MEMORY: DWORD = 8; +pub const ERROR_OUTOFMEMORY: DWORD = 14; pub const ERROR_NO_MORE_FILES: DWORD = 18; +pub const ERROR_SHARING_VIOLATION: u32 = 32; pub const ERROR_HANDLE_EOF: DWORD = 38; pub const ERROR_FILE_EXISTS: DWORD = 80; pub const ERROR_INVALID_PARAMETER: DWORD = 87; @@ -626,7 +631,7 @@ pub struct timeval { pub tv_usec: c_long, } -// Functions forbidden when targeting UWP +// Desktop specific functions & types cfg_if::cfg_if! { if #[cfg(not(target_vendor = "uwp"))] { pub const EXCEPTION_CONTINUE_SEARCH: LONG = 0; @@ -640,7 +645,7 @@ if #[cfg(not(target_vendor = "uwp"))] { pub ExceptionRecord: *mut EXCEPTION_RECORD, pub ExceptionAddress: LPVOID, pub NumberParameters: DWORD, - pub ExceptionInformation: [LPVOID; EXCEPTION_MAXIMUM_PARAMETERS] + pub ExceptionInformation: [LPVOID; EXCEPTION_MAXIMUM_PARAMETERS], } pub enum CONTEXT {} @@ -651,8 +656,8 @@ if #[cfg(not(target_vendor = "uwp"))] { pub ContextRecord: *mut CONTEXT, } - pub type PVECTORED_EXCEPTION_HANDLER = extern "system" - fn(ExceptionInfo: *mut EXCEPTION_POINTERS) -> LONG; + pub type PVECTORED_EXCEPTION_HANDLER = + extern "system" fn(ExceptionInfo: *mut EXCEPTION_POINTERS) -> LONG; #[repr(C)] #[derive(Copy, Clone)] @@ -686,44 +691,66 @@ if #[cfg(not(target_vendor = "uwp"))] { pub const TOKEN_READ: DWORD = 0x20008; + #[link(name = "advapi32")] extern "system" { + // Forbidden when targeting UWP #[link_name = "SystemFunction036"] pub fn RtlGenRandom(RandomBuffer: *mut u8, RandomBufferLength: ULONG) -> BOOLEAN; - pub fn ReadConsoleW(hConsoleInput: HANDLE, - lpBuffer: LPVOID, - nNumberOfCharsToRead: DWORD, - lpNumberOfCharsRead: LPDWORD, - pInputControl: PCONSOLE_READCONSOLE_CONTROL) -> BOOL; + // Allowed but unused by UWP + pub fn OpenProcessToken( + ProcessHandle: HANDLE, + DesiredAccess: DWORD, + TokenHandle: *mut HANDLE, + ) -> BOOL; + } - pub fn WriteConsoleW(hConsoleOutput: HANDLE, - lpBuffer: LPCVOID, - nNumberOfCharsToWrite: DWORD, - lpNumberOfCharsWritten: LPDWORD, - lpReserved: LPVOID) -> BOOL; + #[link(name = "userenv")] + extern "system" { + // Allowed but unused by UWP + pub fn GetUserProfileDirectoryW( + hToken: HANDLE, + lpProfileDir: LPWSTR, + lpcchSize: *mut DWORD, + ) -> BOOL; + } - pub fn GetConsoleMode(hConsoleHandle: HANDLE, - lpMode: LPDWORD) -> BOOL; + #[link(name = "kernel32")] + extern "system" { + // Functions forbidden when targeting UWP + pub fn ReadConsoleW( + hConsoleInput: HANDLE, + lpBuffer: LPVOID, + nNumberOfCharsToRead: DWORD, + lpNumberOfCharsRead: LPDWORD, + pInputControl: PCONSOLE_READCONSOLE_CONTROL, + ) -> BOOL; + + pub fn WriteConsoleW( + hConsoleOutput: HANDLE, + lpBuffer: LPCVOID, + nNumberOfCharsToWrite: DWORD, + lpNumberOfCharsWritten: LPDWORD, + lpReserved: LPVOID, + ) -> BOOL; + + pub fn GetConsoleMode(hConsoleHandle: HANDLE, lpMode: LPDWORD) -> BOOL; // Allowed but unused by UWP - pub fn OpenProcessToken(ProcessHandle: HANDLE, - DesiredAccess: DWORD, - TokenHandle: *mut HANDLE) -> BOOL; - pub fn GetUserProfileDirectoryW(hToken: HANDLE, - lpProfileDir: LPWSTR, - lpcchSize: *mut DWORD) -> BOOL; - pub fn GetFileInformationByHandle(hFile: HANDLE, - lpFileInformation: LPBY_HANDLE_FILE_INFORMATION) - -> BOOL; - pub fn SetHandleInformation(hObject: HANDLE, - dwMask: DWORD, - dwFlags: DWORD) -> BOOL; - pub fn AddVectoredExceptionHandler(FirstHandler: ULONG, - VectoredHandler: PVECTORED_EXCEPTION_HANDLER) - -> LPVOID; - pub fn CreateHardLinkW(lpSymlinkFileName: LPCWSTR, - lpTargetFileName: LPCWSTR, - lpSecurityAttributes: LPSECURITY_ATTRIBUTES) - -> BOOL; + pub fn GetFileInformationByHandle( + hFile: HANDLE, + lpFileInformation: LPBY_HANDLE_FILE_INFORMATION, + ) -> BOOL; + pub fn SetHandleInformation(hObject: HANDLE, dwMask: DWORD, dwFlags: DWORD) -> BOOL; + pub fn AddVectoredExceptionHandler( + FirstHandler: ULONG, + VectoredHandler: PVECTORED_EXCEPTION_HANDLER, + ) -> LPVOID; + pub fn CreateHardLinkW( + lpSymlinkFileName: LPCWSTR, + lpTargetFileName: LPCWSTR, + lpSecurityAttributes: LPSECURITY_ATTRIBUTES, + ) -> BOOL; + pub fn SetThreadStackGuarantee(_size: *mut c_ulong) -> BOOL; } } } @@ -742,55 +769,32 @@ if #[cfg(target_vendor = "uwp")] { pub Directory: BOOLEAN, } + #[link(name = "bcrypt")] extern "system" { - pub fn GetFileInformationByHandleEx(hFile: HANDLE, - fileInfoClass: FILE_INFO_BY_HANDLE_CLASS, - lpFileInformation: LPVOID, - dwBufferSize: DWORD) -> BOOL; - pub fn BCryptGenRandom(hAlgorithm: LPVOID, pBuffer: *mut u8, - cbBuffer: ULONG, dwFlags: ULONG) -> LONG; + pub fn BCryptGenRandom( + hAlgorithm: LPVOID, + pBuffer: *mut u8, + cbBuffer: ULONG, + dwFlags: ULONG, + ) -> LONG; + } + #[link(name = "kernel32")] + extern "system" { + pub fn GetFileInformationByHandleEx( + hFile: HANDLE, + fileInfoClass: FILE_INFO_BY_HANDLE_CLASS, + lpFileInformation: LPVOID, + dwBufferSize: DWORD, + ) -> BOOL; } } } // Shared between Desktop & UWP + +#[link(name = "kernel32")] extern "system" { - pub fn WSAStartup(wVersionRequested: WORD, lpWSAData: LPWSADATA) -> c_int; - pub fn WSACleanup() -> c_int; - pub fn WSAGetLastError() -> c_int; - pub fn WSADuplicateSocketW( - s: SOCKET, - dwProcessId: DWORD, - lpProtocolInfo: LPWSAPROTOCOL_INFO, - ) -> c_int; - pub fn WSASend( - s: SOCKET, - lpBuffers: LPWSABUF, - dwBufferCount: DWORD, - lpNumberOfBytesSent: LPDWORD, - dwFlags: DWORD, - lpOverlapped: LPWSAOVERLAPPED, - lpCompletionRoutine: LPWSAOVERLAPPED_COMPLETION_ROUTINE, - ) -> c_int; - pub fn WSARecv( - s: SOCKET, - lpBuffers: LPWSABUF, - dwBufferCount: DWORD, - lpNumberOfBytesRecvd: LPDWORD, - lpFlags: LPDWORD, - lpOverlapped: LPWSAOVERLAPPED, - lpCompletionRoutine: LPWSAOVERLAPPED_COMPLETION_ROUTINE, - ) -> c_int; pub fn GetCurrentProcessId() -> DWORD; - pub fn WSASocketW( - af: c_int, - kind: c_int, - protocol: c_int, - lpProtocolInfo: LPWSAPROTOCOL_INFO, - g: GROUP, - dwFlags: DWORD, - ) -> SOCKET; - pub fn ioctlsocket(s: SOCKET, cmd: c_long, argp: *mut c_ulong) -> c_int; pub fn InitializeCriticalSection(CriticalSection: *mut CRITICAL_SECTION); pub fn EnterCriticalSection(CriticalSection: *mut CRITICAL_SECTION); pub fn TryEnterCriticalSection(CriticalSection: *mut CRITICAL_SECTION) -> BOOL; @@ -877,28 +881,6 @@ extern "system" { pub fn DeleteFileW(lpPathName: LPCWSTR) -> BOOL; pub fn GetCurrentDirectoryW(nBufferLength: DWORD, lpBuffer: LPWSTR) -> DWORD; pub fn SetCurrentDirectoryW(lpPathName: LPCWSTR) -> BOOL; - - pub fn closesocket(socket: SOCKET) -> c_int; - pub fn recv(socket: SOCKET, buf: *mut c_void, len: c_int, flags: c_int) -> c_int; - pub fn send(socket: SOCKET, buf: *const c_void, len: c_int, flags: c_int) -> c_int; - pub fn recvfrom( - socket: SOCKET, - buf: *mut c_void, - len: c_int, - flags: c_int, - addr: *mut SOCKADDR, - addrlen: *mut c_int, - ) -> c_int; - pub fn sendto( - socket: SOCKET, - buf: *const c_void, - len: c_int, - flags: c_int, - addr: *const SOCKADDR, - addrlen: c_int, - ) -> c_int; - pub fn shutdown(socket: SOCKET, how: c_int) -> c_int; - pub fn accept(socket: SOCKET, address: *mut SOCKADDR, address_len: *mut c_int) -> SOCKET; pub fn DuplicateHandle( hSourceProcessHandle: HANDLE, hSourceHandle: HANDLE, @@ -945,32 +927,6 @@ extern "system" { pub fn FindFirstFileW(fileName: LPCWSTR, findFileData: LPWIN32_FIND_DATAW) -> HANDLE; pub fn FindNextFileW(findFile: HANDLE, findFileData: LPWIN32_FIND_DATAW) -> BOOL; pub fn FindClose(findFile: HANDLE) -> BOOL; - pub fn getsockopt( - s: SOCKET, - level: c_int, - optname: c_int, - optval: *mut c_char, - optlen: *mut c_int, - ) -> c_int; - pub fn setsockopt( - s: SOCKET, - level: c_int, - optname: c_int, - optval: *const c_void, - optlen: c_int, - ) -> c_int; - pub fn getsockname(socket: SOCKET, address: *mut SOCKADDR, address_len: *mut c_int) -> c_int; - pub fn getpeername(socket: SOCKET, address: *mut SOCKADDR, address_len: *mut c_int) -> c_int; - pub fn bind(socket: SOCKET, address: *const SOCKADDR, address_len: socklen_t) -> c_int; - pub fn listen(socket: SOCKET, backlog: c_int) -> c_int; - pub fn connect(socket: SOCKET, address: *const SOCKADDR, len: c_int) -> c_int; - pub fn getaddrinfo( - node: *const c_char, - service: *const c_char, - hints: *const ADDRINFOA, - res: *mut *mut ADDRINFOA, - ) -> c_int; - pub fn freeaddrinfo(res: *mut ADDRINFOA); pub fn GetProcAddress(handle: HMODULE, name: LPCSTR) -> *mut c_void; pub fn GetModuleHandleA(lpModuleName: LPCSTR) -> HMODULE; @@ -1007,47 +963,23 @@ extern "system" { lpNumberOfBytesTransferred: LPDWORD, bWait: BOOL, ) -> BOOL; - pub fn select( - nfds: c_int, - readfds: *mut fd_set, - writefds: *mut fd_set, - exceptfds: *mut fd_set, - timeout: *const timeval, - ) -> c_int; - - // >= Vista / Server 2008 - // https://docs.microsoft.com/en-us/windows/win32/api/winbase/nf-winbase-createsymboliclinkw pub fn CreateSymbolicLinkW( lpSymlinkFileName: LPCWSTR, lpTargetFileName: LPCWSTR, dwFlags: DWORD, ) -> BOOLEAN; - - // >= Vista / Server 2008 - // https://docs.microsoft.com/en-us/windows/win32/api/fileapi/nf-fileapi-getfinalpathnamebyhandlew pub fn GetFinalPathNameByHandleW( hFile: HANDLE, lpszFilePath: LPCWSTR, cchFilePath: DWORD, dwFlags: DWORD, ) -> DWORD; - - // >= Vista / Server 2003 - // https://docs.microsoft.com/en-us/windows/win32/api/processthreadsapi/nf-processthreadsapi-setthreadstackguarantee - #[cfg(not(target_vendor = "uwp"))] - pub fn SetThreadStackGuarantee(_size: *mut c_ulong) -> BOOL; - - // >= Vista / Server 2008 - // https://docs.microsoft.com/en-us/windows/win32/api/fileapi/nf-fileapi-setfileinformationbyhandle pub fn SetFileInformationByHandle( hFile: HANDLE, FileInformationClass: FILE_INFO_BY_HANDLE_CLASS, lpFileInformation: LPVOID, dwBufferSize: DWORD, ) -> BOOL; - - // >= Vista / Server 2008 - // https://docs.microsoft.com/en-us/windows/win32/api/synchapi/nf-synchapi-sleepconditionvariablesrw pub fn SleepConditionVariableSRW( ConditionVariable: PCONDITION_VARIABLE, SRWLock: PSRWLOCK, @@ -1055,13 +987,9 @@ extern "system" { Flags: ULONG, ) -> BOOL; - // >= Vista / Server 2008 - // https://docs.microsoft.com/en-us/windows/win32/api/synchapi/nf-synchapi-wakeconditionvariable pub fn WakeConditionVariable(ConditionVariable: PCONDITION_VARIABLE); pub fn WakeAllConditionVariable(ConditionVariable: PCONDITION_VARIABLE); - // >= Vista / Server 2008 - // https://docs.microsoft.com/en-us/windows/win32/api/synchapi/nf-synchapi-acquiresrwlockexclusive pub fn AcquireSRWLockExclusive(SRWLock: PSRWLOCK); pub fn AcquireSRWLockShared(SRWLock: PSRWLOCK); pub fn ReleaseSRWLockExclusive(SRWLock: PSRWLOCK); @@ -1070,6 +998,99 @@ extern "system" { pub fn TryAcquireSRWLockShared(SRWLock: PSRWLOCK) -> BOOLEAN; } +#[link(name = "ws2_32")] +extern "system" { + pub fn WSAStartup(wVersionRequested: WORD, lpWSAData: LPWSADATA) -> c_int; + pub fn WSACleanup() -> c_int; + pub fn WSAGetLastError() -> c_int; + pub fn WSADuplicateSocketW( + s: SOCKET, + dwProcessId: DWORD, + lpProtocolInfo: LPWSAPROTOCOL_INFO, + ) -> c_int; + pub fn WSASend( + s: SOCKET, + lpBuffers: LPWSABUF, + dwBufferCount: DWORD, + lpNumberOfBytesSent: LPDWORD, + dwFlags: DWORD, + lpOverlapped: LPWSAOVERLAPPED, + lpCompletionRoutine: LPWSAOVERLAPPED_COMPLETION_ROUTINE, + ) -> c_int; + pub fn WSARecv( + s: SOCKET, + lpBuffers: LPWSABUF, + dwBufferCount: DWORD, + lpNumberOfBytesRecvd: LPDWORD, + lpFlags: LPDWORD, + lpOverlapped: LPWSAOVERLAPPED, + lpCompletionRoutine: LPWSAOVERLAPPED_COMPLETION_ROUTINE, + ) -> c_int; + pub fn WSASocketW( + af: c_int, + kind: c_int, + protocol: c_int, + lpProtocolInfo: LPWSAPROTOCOL_INFO, + g: GROUP, + dwFlags: DWORD, + ) -> SOCKET; + pub fn ioctlsocket(s: SOCKET, cmd: c_long, argp: *mut c_ulong) -> c_int; + pub fn closesocket(socket: SOCKET) -> c_int; + pub fn recv(socket: SOCKET, buf: *mut c_void, len: c_int, flags: c_int) -> c_int; + pub fn send(socket: SOCKET, buf: *const c_void, len: c_int, flags: c_int) -> c_int; + pub fn recvfrom( + socket: SOCKET, + buf: *mut c_void, + len: c_int, + flags: c_int, + addr: *mut SOCKADDR, + addrlen: *mut c_int, + ) -> c_int; + pub fn sendto( + socket: SOCKET, + buf: *const c_void, + len: c_int, + flags: c_int, + addr: *const SOCKADDR, + addrlen: c_int, + ) -> c_int; + pub fn shutdown(socket: SOCKET, how: c_int) -> c_int; + pub fn accept(socket: SOCKET, address: *mut SOCKADDR, address_len: *mut c_int) -> SOCKET; + pub fn getsockopt( + s: SOCKET, + level: c_int, + optname: c_int, + optval: *mut c_char, + optlen: *mut c_int, + ) -> c_int; + pub fn setsockopt( + s: SOCKET, + level: c_int, + optname: c_int, + optval: *const c_void, + optlen: c_int, + ) -> c_int; + pub fn getsockname(socket: SOCKET, address: *mut SOCKADDR, address_len: *mut c_int) -> c_int; + pub fn getpeername(socket: SOCKET, address: *mut SOCKADDR, address_len: *mut c_int) -> c_int; + pub fn bind(socket: SOCKET, address: *const SOCKADDR, address_len: socklen_t) -> c_int; + pub fn listen(socket: SOCKET, backlog: c_int) -> c_int; + pub fn connect(socket: SOCKET, address: *const SOCKADDR, len: c_int) -> c_int; + pub fn getaddrinfo( + node: *const c_char, + service: *const c_char, + hints: *const ADDRINFOA, + res: *mut *mut ADDRINFOA, + ) -> c_int; + pub fn freeaddrinfo(res: *mut ADDRINFOA); + pub fn select( + nfds: c_int, + readfds: *mut fd_set, + writefds: *mut fd_set, + exceptfds: *mut fd_set, + timeout: *const timeval, + ) -> c_int; +} + // Functions that aren't available on every version of Windows that we support, // but we still use them and just provide some form of a fallback implementation. compat_fn! { diff --git a/library/std/src/sys/windows/fs.rs b/library/std/src/sys/windows/fs.rs index 8e6bd76f85..2b6143de96 100644 --- a/library/std/src/sys/windows/fs.rs +++ b/library/std/src/sys/windows/fs.rs @@ -944,3 +944,32 @@ fn symlink_junction_inner(original: &Path, junction: &Path) -> io::Result<()> { .map(drop) } } + +// Try to see if a file exists but, unlike `exists`, report I/O errors. +pub fn try_exists(path: &Path) -> io::Result { + // Open the file to ensure any symlinks are followed to their target. + let mut opts = OpenOptions::new(); + // No read, write, etc access rights are needed. + opts.access_mode(0); + // Backup semantics enables opening directories as well as files. + opts.custom_flags(c::FILE_FLAG_BACKUP_SEMANTICS); + match File::open(path, &opts) { + Err(e) => match e.kind() { + // The file definitely does not exist + io::ErrorKind::NotFound => Ok(false), + + // `ERROR_SHARING_VIOLATION` means that the file has been locked by + // another process. This is often temporary so we simply report it + // as the file existing. + io::ErrorKind::Other if e.raw_os_error() == Some(c::ERROR_SHARING_VIOLATION as i32) => { + Ok(true) + } + // Other errors such as `ERROR_ACCESS_DENIED` may indicate that the + // file exists. However, these types of errors are usually more + // permanent so we report them here. + _ => Err(e), + }, + // The file was opened successfully therefore it must exist, + Ok(_) => Ok(true), + } +} diff --git a/library/std/src/sys/windows/mod.rs b/library/std/src/sys/windows/mod.rs index ddb6ac5f55..f23e874f24 100644 --- a/library/std/src/sys/windows/mod.rs +++ b/library/std/src/sys/windows/mod.rs @@ -18,7 +18,6 @@ pub mod c; pub mod cmath; pub mod condvar; pub mod env; -pub mod ext; pub mod fs; pub mod handle; pub mod io; @@ -71,6 +70,7 @@ pub fn decode_error_kind(errno: i32) -> ErrorKind { c::ERROR_PATH_NOT_FOUND => return ErrorKind::NotFound, c::ERROR_NO_DATA => return ErrorKind::BrokenPipe, c::ERROR_INVALID_PARAMETER => return ErrorKind::InvalidInput, + c::ERROR_NOT_ENOUGH_MEMORY | c::ERROR_OUTOFMEMORY => return ErrorKind::OutOfMemory, c::ERROR_SEM_TIMEOUT | c::WAIT_TIMEOUT | c::ERROR_DRIVER_CANCEL_TIMEOUT @@ -281,17 +281,3 @@ pub fn abort_internal() -> ! { } crate::intrinsics::abort(); } - -cfg_if::cfg_if! { - if #[cfg(target_vendor = "uwp")] { - #[link(name = "ws2_32")] - // For BCryptGenRandom - #[link(name = "bcrypt")] - extern "C" {} - } else { - #[link(name = "advapi32")] - #[link(name = "ws2_32")] - #[link(name = "userenv")] - extern "C" {} - } -} diff --git a/library/std/src/sys/windows/process.rs b/library/std/src/sys/windows/process.rs index a579960614..81dbea4a06 100644 --- a/library/std/src/sys/windows/process.rs +++ b/library/std/src/sys/windows/process.rs @@ -5,6 +5,7 @@ mod tests; use crate::borrow::Borrow; use crate::collections::BTreeMap; +use crate::convert::{TryFrom, TryInto}; use crate::env; use crate::env::split_paths; use crate::ffi::{OsStr, OsString}; @@ -12,10 +13,12 @@ use crate::fmt; use crate::fs; use crate::io::{self, Error, ErrorKind}; use crate::mem; +use crate::num::NonZeroI32; use crate::os::windows::ffi::OsStrExt; use crate::path::Path; use crate::ptr; use crate::sys::c; +use crate::sys::c::NonZeroDWORD; use crate::sys::cvt; use crate::sys::fs::{File, OpenOptions}; use crate::sys::handle::Handle; @@ -376,8 +379,11 @@ impl Process { pub struct ExitStatus(c::DWORD); impl ExitStatus { - pub fn success(&self) -> bool { - self.0 == 0 + pub fn exit_ok(&self) -> Result<(), ExitStatusError> { + match NonZeroDWORD::try_from(self.0) { + /* was nonzero */ Ok(failure) => Err(ExitStatusError(failure)), + /* was zero, couldn't convert */ Err(_) => Ok(()), + } } pub fn code(&self) -> Option { Some(self.0 as i32) @@ -406,6 +412,21 @@ impl fmt::Display for ExitStatus { } } +#[derive(PartialEq, Eq, Clone, Copy, Debug)] +pub struct ExitStatusError(c::NonZeroDWORD); + +impl Into for ExitStatusError { + fn into(self) -> ExitStatus { + ExitStatus(self.0.into()) + } +} + +impl ExitStatusError { + pub fn code(self) -> Option { + Some((u32::from(self.0) as i32).try_into().unwrap()) + } +} + #[derive(PartialEq, Eq, Clone, Copy, Debug)] pub struct ExitCode(c::DWORD); diff --git a/library/std/src/sys/windows/rwlock.rs b/library/std/src/sys/windows/rwlock.rs index a769326352..b7a5b1e7ac 100644 --- a/library/std/src/sys/windows/rwlock.rs +++ b/library/std/src/sys/windows/rwlock.rs @@ -5,6 +5,8 @@ pub struct RWLock { inner: UnsafeCell, } +pub type MovableRWLock = RWLock; + unsafe impl Send for RWLock {} unsafe impl Sync for RWLock {} diff --git a/library/std/src/sys/windows/stack_overflow.rs b/library/std/src/sys/windows/stack_overflow.rs index 39efb77820..755dc0a6c8 100644 --- a/library/std/src/sys/windows/stack_overflow.rs +++ b/library/std/src/sys/windows/stack_overflow.rs @@ -1,7 +1,7 @@ #![cfg_attr(test, allow(dead_code))] use crate::sys::c; -use crate::sys_common::util::report_overflow; +use crate::thread; pub struct Handler; @@ -24,7 +24,10 @@ extern "system" fn vectored_handler(ExceptionInfo: *mut c::EXCEPTION_POINTERS) - let code = rec.ExceptionCode; if code == c::EXCEPTION_STACK_OVERFLOW { - report_overflow(); + rtprintpanic!( + "\nthread '{}' has overflowed its stack\n", + thread::current().name().unwrap_or("") + ); } c::EXCEPTION_CONTINUE_SEARCH } diff --git a/library/std/src/sys_common/fs.rs b/library/std/src/sys_common/fs.rs index 30908824dd..309f548387 100644 --- a/library/std/src/sys_common/fs.rs +++ b/library/std/src/sys_common/fs.rs @@ -41,3 +41,11 @@ fn remove_dir_all_recursive(path: &Path) -> io::Result<()> { } fs::remove_dir(path) } + +pub fn try_exists(path: &Path) -> io::Result { + match fs::metadata(path) { + Ok(_) => Ok(true), + Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(false), + Err(error) => Err(error), + } +} diff --git a/library/std/src/memchr.rs b/library/std/src/sys_common/memchr.rs similarity index 92% rename from library/std/src/memchr.rs rename to library/std/src/sys_common/memchr.rs index 86a08f75a8..b219e87891 100644 --- a/library/std/src/memchr.rs +++ b/library/std/src/sys_common/memchr.rs @@ -1,6 +1,8 @@ // Original implementation taken from rust-memchr. // Copyright 2015 Andrew Gallant, bluss and Nicolas Koch +use crate::sys::memchr as sys; + #[cfg(test)] mod tests; @@ -25,7 +27,7 @@ mod tests; /// ``` #[inline] pub fn memchr(needle: u8, haystack: &[u8]) -> Option { - crate::sys::memchr::memchr(needle, haystack) + sys::memchr(needle, haystack) } /// A safe interface to `memrchr`. @@ -45,5 +47,5 @@ pub fn memchr(needle: u8, haystack: &[u8]) -> Option { /// ``` #[inline] pub fn memrchr(needle: u8, haystack: &[u8]) -> Option { - crate::sys::memchr::memrchr(needle, haystack) + sys::memrchr(needle, haystack) } diff --git a/library/std/src/memchr/tests.rs b/library/std/src/sys_common/memchr/tests.rs similarity index 100% rename from library/std/src/memchr/tests.rs rename to library/std/src/sys_common/memchr/tests.rs diff --git a/library/std/src/sys_common/mod.rs b/library/std/src/sys_common/mod.rs index 7fa6977f2a..1a9caa22c9 100644 --- a/library/std/src/sys_common/mod.rs +++ b/library/std/src/sys_common/mod.rs @@ -25,6 +25,7 @@ pub mod bytestring; pub mod condvar; pub mod fs; pub mod io; +pub mod memchr; pub mod mutex; // `doc` is required because `sys/mod.rs` imports `unix/ext/mod.rs` on Windows // when generating documentation. @@ -40,7 +41,6 @@ pub mod thread_info; pub mod thread_local_dtor; pub mod thread_local_key; pub mod thread_parker; -pub mod util; pub mod wtf8; cfg_if::cfg_if! { diff --git a/library/std/src/sys_common/rt.rs b/library/std/src/sys_common/rt.rs index c70f2ecc04..02013ecc4c 100644 --- a/library/std/src/sys_common/rt.rs +++ b/library/std/src/sys_common/rt.rs @@ -1,4 +1,5 @@ #![deny(unsafe_op_in_unsafe_fn)] +#![allow(unused_macros)] use crate::sync::Once; use crate::sys; @@ -38,8 +39,25 @@ pub fn cleanup() { }); } +// Prints to the "panic output", depending on the platform this may be: +// - the standard error output +// - some dedicated platform specific output +// - nothing (so this macro is a no-op) +macro_rules! rtprintpanic { + ($($t:tt)*) => { + if let Some(mut out) = crate::sys::stdio::panic_output() { + let _ = crate::io::Write::write_fmt(&mut out, format_args!($($t)*)); + } + } +} + macro_rules! rtabort { - ($($t:tt)*) => (crate::sys_common::util::abort(format_args!($($t)*))) + ($($t:tt)*) => { + { + rtprintpanic!("fatal runtime error: {}\n", format_args!($($t)*)); + crate::sys::abort_internal(); + } + } } macro_rules! rtassert { @@ -50,7 +68,6 @@ macro_rules! rtassert { }; } -#[allow(unused_macros)] // not used on all platforms macro_rules! rtunwrap { ($ok:ident, $e:expr) => { match $e { diff --git a/library/std/src/sys_common/rwlock.rs b/library/std/src/sys_common/rwlock.rs index 3705d641a1..07ec20f4dc 100644 --- a/library/std/src/sys_common/rwlock.rs +++ b/library/std/src/sys_common/rwlock.rs @@ -1,63 +1,112 @@ use crate::sys::rwlock as imp; +/// An OS-based reader-writer lock, meant for use in static variables. +/// +/// This rwlock does not implement poisoning. +/// +/// This rwlock has a const constructor ([`StaticRWLock::new`]), does not +/// implement `Drop` to cleanup resources. +pub struct StaticRWLock(imp::RWLock); + +impl StaticRWLock { + /// Creates a new rwlock for use. + pub const fn new() -> Self { + Self(imp::RWLock::new()) + } + + /// Acquires shared access to the underlying lock, blocking the current + /// thread to do so. + /// + /// The lock is automatically unlocked when the returned guard is dropped. + #[inline] + pub fn read(&'static self) -> StaticRWLockReadGuard { + unsafe { self.0.read() }; + StaticRWLockReadGuard(&self.0) + } + + /// Acquires write access to the underlying lock, blocking the current thread + /// to do so. + /// + /// The lock is automatically unlocked when the returned guard is dropped. + #[inline] + pub fn write(&'static self) -> StaticRWLockWriteGuard { + unsafe { self.0.write() }; + StaticRWLockWriteGuard(&self.0) + } +} + +#[must_use] +pub struct StaticRWLockReadGuard(&'static imp::RWLock); + +impl Drop for StaticRWLockReadGuard { + #[inline] + fn drop(&mut self) { + unsafe { + self.0.read_unlock(); + } + } +} + +#[must_use] +pub struct StaticRWLockWriteGuard(&'static imp::RWLock); + +impl Drop for StaticRWLockWriteGuard { + #[inline] + fn drop(&mut self) { + unsafe { + self.0.write_unlock(); + } + } +} + /// An OS-based reader-writer lock. /// -/// This structure is entirely unsafe and serves as the lowest layer of a -/// cross-platform binding of system rwlocks. It is recommended to use the -/// safer types at the top level of this crate instead of this type. -pub struct RWLock(imp::RWLock); +/// This rwlock does *not* have a const constructor, cleans up its resources in +/// its `Drop` implementation and may safely be moved (when not borrowed). +/// +/// This rwlock does not implement poisoning. +/// +/// This is either a wrapper around `Box` or `imp::RWLock`, +/// depending on the platform. It is boxed on platforms where `imp::RWLock` may +/// not be moved. +pub struct MovableRWLock(imp::MovableRWLock); -impl RWLock { +impl MovableRWLock { /// Creates a new reader-writer lock for use. - /// - /// Behavior is undefined if the reader-writer lock is moved after it is - /// first used with any of the functions below. - pub const fn new() -> RWLock { - RWLock(imp::RWLock::new()) + pub fn new() -> Self { + Self(imp::MovableRWLock::from(imp::RWLock::new())) } /// Acquires shared access to the underlying lock, blocking the current /// thread to do so. - /// - /// Behavior is undefined if the rwlock has been moved between this and any - /// previous method call. #[inline] - pub unsafe fn read(&self) { - self.0.read() + pub fn read(&self) { + unsafe { self.0.read() } } /// Attempts to acquire shared access to this lock, returning whether it /// succeeded or not. /// /// This function does not block the current thread. - /// - /// Behavior is undefined if the rwlock has been moved between this and any - /// previous method call. #[inline] - pub unsafe fn try_read(&self) -> bool { - self.0.try_read() + pub fn try_read(&self) -> bool { + unsafe { self.0.try_read() } } /// Acquires write access to the underlying lock, blocking the current thread /// to do so. - /// - /// Behavior is undefined if the rwlock has been moved between this and any - /// previous method call. #[inline] - pub unsafe fn write(&self) { - self.0.write() + pub fn write(&self) { + unsafe { self.0.write() } } /// Attempts to acquire exclusive access to this lock, returning whether it /// succeeded or not. /// /// This function does not block the current thread. - /// - /// Behavior is undefined if the rwlock has been moved between this and any - /// previous method call. #[inline] - pub unsafe fn try_write(&self) -> bool { - self.0.try_write() + pub fn try_write(&self) -> bool { + unsafe { self.0.try_write() } } /// Unlocks previously acquired shared access to this lock. @@ -76,13 +125,10 @@ impl RWLock { pub unsafe fn write_unlock(&self) { self.0.write_unlock() } +} - /// Destroys OS-related resources with this RWLock. - /// - /// Behavior is undefined if there are any currently active users of this - /// lock. - #[inline] - pub unsafe fn destroy(&self) { - self.0.destroy() +impl Drop for MovableRWLock { + fn drop(&mut self) { + unsafe { self.0.destroy() }; } } diff --git a/library/std/src/sys_common/util.rs b/library/std/src/sys_common/util.rs deleted file mode 100644 index 9f7c3bd879..0000000000 --- a/library/std/src/sys_common/util.rs +++ /dev/null @@ -1,28 +0,0 @@ -use crate::fmt; -use crate::io::prelude::*; -use crate::sys::stdio::panic_output; -use crate::thread; - -pub fn dumb_print(args: fmt::Arguments<'_>) { - if let Some(mut out) = panic_output() { - let _ = out.write_fmt(args); - } -} - -// Other platforms should use the appropriate platform-specific mechanism for -// aborting the process. If no platform-specific mechanism is available, -// crate::intrinsics::abort() may be used instead. The above implementations cover -// all targets currently supported by libstd. - -pub fn abort(args: fmt::Arguments<'_>) -> ! { - dumb_print(format_args!("fatal runtime error: {}\n", args)); - crate::sys::abort_internal(); -} - -#[allow(dead_code)] // stack overflow detection not enabled on all platforms -pub unsafe fn report_overflow() { - dumb_print(format_args!( - "\nthread '{}' has overflowed its stack\n", - thread::current().name().unwrap_or("") - )); -} diff --git a/library/std/src/thread/local.rs b/library/std/src/thread/local.rs index abd5b77848..e62f4440b3 100644 --- a/library/std/src/thread/local.rs +++ b/library/std/src/thread/local.rs @@ -162,6 +162,7 @@ macro_rules! thread_local { macro_rules! __thread_local_inner { // used to generate the `LocalKey` value for const-initialized thread locals (@key $t:ty, const $init:expr) => {{ + #[cfg_attr(not(windows), inline)] // see comments below unsafe fn __getit() -> $crate::option::Option<&'static $t> { const _REQUIRE_UNSTABLE: () = $crate::thread::require_unstable_const_init_thread_local(); @@ -260,6 +261,29 @@ macro_rules! __thread_local_inner { #[inline] fn __init() -> $t { $init } + // When reading this function you might ask "why is this inlined + // everywhere other than Windows?", and that's a very reasonable + // question to ask. The short story is that it segfaults rustc if + // this function is inlined. The longer story is that Windows looks + // to not support `extern` references to thread locals across DLL + // boundaries. This appears to at least not be supported in the ABI + // that LLVM implements. + // + // Because of this we never inline on Windows, but we do inline on + // other platforms (where external references to thread locals + // across DLLs are supported). A better fix for this would be to + // inline this function on Windows, but only for "statically linked" + // components. For example if two separately compiled rlibs end up + // getting linked into a DLL then it's fine to inline this function + // across that boundary. It's only not fine to inline this function + // across a DLL boundary. Unfortunately rustc doesn't currently + // have this sort of logic available in an attribute, and it's not + // clear that rustc is even equipped to answer this (it's more of a + // Cargo question kinda). This means that, unfortunately, Windows + // gets the pessimistic path for now where it's never inlined. + // + // The issue of "should enable on Windows sometimes" is #84933 + #[cfg_attr(not(windows), inline)] unsafe fn __getit() -> $crate::option::Option<&'static $t> { #[cfg(all(target_arch = "wasm32", not(target_feature = "atomics")))] static __KEY: $crate::thread::__StaticLocalKeyInner<$t> = @@ -722,7 +746,7 @@ pub mod os { unsafe extern "C" fn destroy_value(ptr: *mut u8) { // SAFETY: // - // The OS TLS ensures that this key contains a NULL value when this + // The OS TLS ensures that this key contains a null value when this // destructor starts to run. We set it back to a sentinel value of 1 to // ensure that any future calls to `get` for this thread will return // `None`. diff --git a/library/std/src/time.rs b/library/std/src/time.rs index 89addae078..899cf6841e 100644 --- a/library/std/src/time.rs +++ b/library/std/src/time.rs @@ -82,7 +82,7 @@ pub use core::time::Duration; /// Currently, the following system calls are being used to get the current time using `now()`: /// /// | Platform | System call | -/// |:---------:|:--------------------------------------------------------------------:| +/// |-----------|----------------------------------------------------------------------| /// | SGX | [`insecure_time` usercall]. More information on [timekeeping in SGX] | /// | UNIX | [clock_gettime (Monotonic Clock)] | /// | Darwin | [mach_absolute_time] | @@ -158,7 +158,7 @@ pub struct Instant(time::Instant); /// Currently, the following system calls are being used to get the current time using `now()`: /// /// | Platform | System call | -/// |:---------:|:--------------------------------------------------------------------:| +/// |-----------|----------------------------------------------------------------------| /// | SGX | [`insecure_time` usercall]. More information on [timekeeping in SGX] | /// | UNIX | [clock_gettime (Realtime Clock)] | /// | Darwin | [gettimeofday] | diff --git a/library/std/tests/run-time-detect.rs b/library/std/tests/run-time-detect.rs index 61a04c4672..c7107b5d0a 100644 --- a/library/std/tests/run-time-detect.rs +++ b/library/std/tests/run-time-detect.rs @@ -27,7 +27,6 @@ fn aarch64_linux() { println!("asimd: {}", is_aarch64_feature_detected!("asimd")); println!("sve: {}", is_aarch64_feature_detected!("sve")); println!("crc: {}", is_aarch64_feature_detected!("crc")); - println!("crypto: {}", is_aarch64_feature_detected!("crypto")); println!("lse: {}", is_aarch64_feature_detected!("lse")); println!("rdm: {}", is_aarch64_feature_detected!("rdm")); println!("rcpc: {}", is_aarch64_feature_detected!("rcpc")); diff --git a/library/stdarch/.cirrus.yml b/library/stdarch/.cirrus.yml index 1b389a96df..7b75ba39dd 100644 --- a/library/stdarch/.cirrus.yml +++ b/library/stdarch/.cirrus.yml @@ -1,7 +1,11 @@ +env: + # Temporary fix for https://github.com/rust-lang/rustup/issues/2774. + RUSTUP_IO_THREADS: "1" + task: name: x86_64-unknown-freebsd freebsd_instance: - image: freebsd-12-1-release-amd64 + image: freebsd-12-2-release-amd64 setup_script: - pkg install -y curl - curl https://sh.rustup.rs -sSf --output rustup.sh diff --git a/library/stdarch/ci/docker/wasm32-wasi/Dockerfile b/library/stdarch/ci/docker/wasm32-wasi/Dockerfile index eca3f61c70..eceebb9ee1 100644 --- a/library/stdarch/ci/docker/wasm32-wasi/Dockerfile +++ b/library/stdarch/ci/docker/wasm32-wasi/Dockerfile @@ -7,10 +7,12 @@ RUN apt-get update -y && apt-get install -y --no-install-recommends \ xz-utils \ clang -RUN curl -L https://github.com/bytecodealliance/wasmtime/releases/download/v0.22.1/wasmtime-v0.22.1-x86_64-linux.tar.xz | tar xJf - -ENV PATH=$PATH:/wasmtime-v0.22.1-x86_64-linux +RUN curl -L https://github.com/bytecodealliance/wasmtime/releases/download/v0.26.0/wasmtime-v0.26.0-x86_64-linux.tar.xz | tar xJf - +ENV PATH=$PATH:/wasmtime-v0.26.0-x86_64-linux ENV CARGO_TARGET_WASM32_WASI_RUNNER="wasmtime \ --enable-simd \ + --enable-threads \ + --opt-level 0 \ --mapdir .::/checkout/target/wasm32-wasi/release/deps \ --" diff --git a/library/stdarch/ci/run.sh b/library/stdarch/ci/run.sh index 699c89cecb..1766a37ca3 100755 --- a/library/stdarch/ci/run.sh +++ b/library/stdarch/ci/run.sh @@ -87,12 +87,6 @@ case ${TARGET} in export RUSTFLAGS="${RUSTFLAGS} -C target-feature=+avx" cargo_test "--release" ;; - wasm32*) - prev="$RUSTFLAGS" - export RUSTFLAGS="${RUSTFLAGS} -C target-feature=+simd128,+unimplemented-simd128" - cargo_test "--release" - export RUSTFLAGS="$prev" - ;; # FIXME: don't build anymore #mips-*gnu* | mipsel-*gnu*) # export RUSTFLAGS="${RUSTFLAGS} -C target-feature=+msa,+fp64,+mips32r5" diff --git a/library/stdarch/crates/assert-instr-macro/Cargo.toml b/library/stdarch/crates/assert-instr-macro/Cargo.toml index eeefca2540..3d9b32067f 100644 --- a/library/stdarch/crates/assert-instr-macro/Cargo.toml +++ b/library/stdarch/crates/assert-instr-macro/Cargo.toml @@ -2,6 +2,7 @@ name = "assert-instr-macro" version = "0.1.0" authors = ["Alex Crichton "] +edition = "2018" [lib] proc-macro = true diff --git a/library/stdarch/crates/assert-instr-macro/src/lib.rs b/library/stdarch/crates/assert-instr-macro/src/lib.rs index 0b3d5b725d..ef76c1c0d4 100644 --- a/library/stdarch/crates/assert-instr-macro/src/lib.rs +++ b/library/stdarch/crates/assert-instr-macro/src/lib.rs @@ -9,11 +9,8 @@ //! function itself contains the relevant instruction. #![deny(rust_2018_idioms)] -extern crate proc_macro; -extern crate proc_macro2; #[macro_use] extern crate quote; -extern crate syn; use proc_macro2::TokenStream; use quote::ToTokens; @@ -63,6 +60,7 @@ pub fn assert_instr( ); let mut inputs = Vec::new(); let mut input_vals = Vec::new(); + let mut const_vals = Vec::new(); let ret = &func.sig.output; for arg in func.sig.inputs.iter() { let capture = match *arg { @@ -83,6 +81,20 @@ pub fn assert_instr( input_vals.push(quote! { #ident }); } } + for arg in func.sig.generics.params.iter() { + let c = match *arg { + syn::GenericParam::Const(ref c) => c, + ref v => panic!( + "only const generics are allowed: `{:?}`", + v.clone().into_token_stream() + ), + }; + if let Some(&(_, ref tokens)) = invoc.args.iter().find(|a| c.ident == a.0) { + const_vals.push(quote! { #tokens }); + } else { + panic!("const generics must have a value for tests"); + } + } let attrs = func .attrs @@ -134,7 +146,7 @@ pub fn assert_instr( std::mem::transmute(#shim_name_str.as_bytes().as_ptr()), std::sync::atomic::Ordering::Relaxed, ); - #name(#(#input_vals),*) + #name::<#(#const_vals),*>(#(#input_vals),*) } }; diff --git a/library/stdarch/crates/core_arch/build.rs b/library/stdarch/crates/core_arch/build.rs index 8a347e3f62..4d65e9ddc3 100644 --- a/library/stdarch/crates/core_arch/build.rs +++ b/library/stdarch/crates/core_arch/build.rs @@ -1,17 +1,3 @@ -use std::env; - fn main() { println!("cargo:rustc-cfg=core_arch_docs"); - - // Used to tell our `#[assert_instr]` annotations that all simd intrinsics - // are available to test their codegen, since some are gated behind an extra - // `-Ctarget-feature=+unimplemented-simd128` that doesn't have any - // equivalent in `#[target_feature]` right now. - println!("cargo:rerun-if-env-changed=RUSTFLAGS"); - if env::var("RUSTFLAGS") - .unwrap_or_default() - .contains("unimplemented-simd128") - { - println!("cargo:rustc-cfg=all_simd"); - } } diff --git a/library/stdarch/crates/core_arch/src/aarch64/armclang.rs b/library/stdarch/crates/core_arch/src/aarch64/armclang.rs new file mode 100644 index 0000000000..54847be7b1 --- /dev/null +++ b/library/stdarch/crates/core_arch/src/aarch64/armclang.rs @@ -0,0 +1,23 @@ +//! ARM compiler specific intrinsics +//! +//! # References +//! +//! - [ARM Compiler v 6.10 - armclang Reference Guide][arm_comp_ref] +//! +//! [arm_comp_ref]: https://developer.arm.com/docs/100067/0610 + +#[cfg(test)] +use stdarch_test::assert_instr; + +/// Inserts a breakpoint instruction. +/// +/// `VAL` is a compile-time constant integer in range `[0, 65535]`. +/// +/// The breakpoint instruction inserted is `BRK` on A64. +#[cfg_attr(test, assert_instr(brk, VAL = 0))] +#[inline(always)] +#[rustc_legacy_const_generics(0)] +pub unsafe fn __breakpoint() { + static_assert_imm16!(VAL); + asm!("brk {}", const VAL); +} diff --git a/library/stdarch/crates/core_arch/src/aarch64/mod.rs b/library/stdarch/crates/core_arch/src/aarch64/mod.rs index f6d0fc9dbd..0411fc1068 100644 --- a/library/stdarch/crates/core_arch/src/aarch64/mod.rs +++ b/library/stdarch/crates/core_arch/src/aarch64/mod.rs @@ -21,7 +21,11 @@ pub use self::crc::*; mod prefetch; pub use self::prefetch::*; -pub use super::acle::*; +pub use super::arm_shared::*; + +mod armclang; + +pub use self::armclang::*; #[cfg(test)] use stdarch_test::assert_instr; diff --git a/library/stdarch/crates/core_arch/src/aarch64/neon/generated.rs b/library/stdarch/crates/core_arch/src/aarch64/neon/generated.rs index db76c8721d..44d83d0131 100644 --- a/library/stdarch/crates/core_arch/src/aarch64/neon/generated.rs +++ b/library/stdarch/crates/core_arch/src/aarch64/neon/generated.rs @@ -9,6 +9,95 @@ use super::*; #[cfg(test)] use stdarch_test::assert_instr; +/// Absolute difference between the arguments of Floating +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fabd))] +pub unsafe fn vabd_f64(a: float64x1_t, b: float64x1_t) -> float64x1_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fabd.v1f64")] + fn vabd_f64_(a: float64x1_t, b: float64x1_t) -> float64x1_t; + } + vabd_f64_(a, b) +} + +/// Absolute difference between the arguments of Floating +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fabd))] +pub unsafe fn vabdq_f64(a: float64x2_t, b: float64x2_t) -> float64x2_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fabd.v2f64")] + fn vabdq_f64_(a: float64x2_t, b: float64x2_t) -> float64x2_t; + } + vabdq_f64_(a, b) +} + +/// Unsigned Absolute difference Long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uabdl))] +pub unsafe fn vabdl_high_u8(a: uint8x16_t, b: uint8x16_t) -> uint16x8_t { + let c: uint8x8_t = simd_shuffle8!(a, a, [8, 9, 10, 11, 12, 13, 14, 15]); + let d: uint8x8_t = simd_shuffle8!(b, b, [8, 9, 10, 11, 12, 13, 14, 15]); + simd_cast(vabd_u8(c, d)) +} + +/// Unsigned Absolute difference Long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uabdl))] +pub unsafe fn vabdl_high_u16(a: uint16x8_t, b: uint16x8_t) -> uint32x4_t { + let c: uint16x4_t = simd_shuffle4!(a, a, [4, 5, 6, 7]); + let d: uint16x4_t = simd_shuffle4!(b, b, [4, 5, 6, 7]); + simd_cast(vabd_u16(c, d)) +} + +/// Unsigned Absolute difference Long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uabdl))] +pub unsafe fn vabdl_high_u32(a: uint32x4_t, b: uint32x4_t) -> uint64x2_t { + let c: uint32x2_t = simd_shuffle2!(a, a, [2, 3]); + let d: uint32x2_t = simd_shuffle2!(b, b, [2, 3]); + simd_cast(vabd_u32(c, d)) +} + +/// Signed Absolute difference Long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sabdl))] +pub unsafe fn vabdl_high_s8(a: int8x16_t, b: int8x16_t) -> int16x8_t { + let c: int8x8_t = simd_shuffle8!(a, a, [8, 9, 10, 11, 12, 13, 14, 15]); + let d: int8x8_t = simd_shuffle8!(b, b, [8, 9, 10, 11, 12, 13, 14, 15]); + let e: uint8x8_t = simd_cast(vabd_s8(c, d)); + simd_cast(e) +} + +/// Signed Absolute difference Long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sabdl))] +pub unsafe fn vabdl_high_s16(a: int16x8_t, b: int16x8_t) -> int32x4_t { + let c: int16x4_t = simd_shuffle4!(a, a, [4, 5, 6, 7]); + let d: int16x4_t = simd_shuffle4!(b, b, [4, 5, 6, 7]); + let e: uint16x4_t = simd_cast(vabd_s16(c, d)); + simd_cast(e) +} + +/// Signed Absolute difference Long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sabdl))] +pub unsafe fn vabdl_high_s32(a: int32x4_t, b: int32x4_t) -> int64x2_t { + let c: int32x2_t = simd_shuffle2!(a, a, [2, 3]); + let d: int32x2_t = simd_shuffle2!(b, b, [2, 3]); + let e: uint32x2_t = simd_cast(vabd_s32(c, d)); + simd_cast(e) +} + /// Compare bitwise Equal (vector) #[inline] #[target_feature(enable = "neon")] @@ -73,6 +162,298 @@ pub unsafe fn vceqq_f64(a: float64x2_t, b: float64x2_t) -> uint64x2_t { simd_eq(a, b) } +/// Signed compare bitwise equal to zero +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(cmeq))] +pub unsafe fn vceqz_s8(a: int8x8_t) -> uint8x8_t { + let b: i8x8 = i8x8::new(0, 0, 0, 0, 0, 0, 0, 0); + simd_eq(a, transmute(b)) +} + +/// Signed compare bitwise equal to zero +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(cmeq))] +pub unsafe fn vceqzq_s8(a: int8x16_t) -> uint8x16_t { + let b: i8x16 = i8x16::new(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + simd_eq(a, transmute(b)) +} + +/// Signed compare bitwise equal to zero +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(cmeq))] +pub unsafe fn vceqz_s16(a: int16x4_t) -> uint16x4_t { + let b: i16x4 = i16x4::new(0, 0, 0, 0); + simd_eq(a, transmute(b)) +} + +/// Signed compare bitwise equal to zero +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(cmeq))] +pub unsafe fn vceqzq_s16(a: int16x8_t) -> uint16x8_t { + let b: i16x8 = i16x8::new(0, 0, 0, 0, 0, 0, 0, 0); + simd_eq(a, transmute(b)) +} + +/// Signed compare bitwise equal to zero +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(cmeq))] +pub unsafe fn vceqz_s32(a: int32x2_t) -> uint32x2_t { + let b: i32x2 = i32x2::new(0, 0); + simd_eq(a, transmute(b)) +} + +/// Signed compare bitwise equal to zero +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(cmeq))] +pub unsafe fn vceqzq_s32(a: int32x4_t) -> uint32x4_t { + let b: i32x4 = i32x4::new(0, 0, 0, 0); + simd_eq(a, transmute(b)) +} + +/// Signed compare bitwise equal to zero +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(cmeq))] +pub unsafe fn vceqz_s64(a: int64x1_t) -> uint64x1_t { + let b: i64x1 = i64x1::new(0); + simd_eq(a, transmute(b)) +} + +/// Signed compare bitwise equal to zero +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(cmeq))] +pub unsafe fn vceqzq_s64(a: int64x2_t) -> uint64x2_t { + let b: i64x2 = i64x2::new(0, 0); + simd_eq(a, transmute(b)) +} + +/// Signed compare bitwise equal to zero +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(cmeq))] +pub unsafe fn vceqz_p8(a: poly8x8_t) -> uint8x8_t { + let b: i8x8 = i8x8::new(0, 0, 0, 0, 0, 0, 0, 0); + simd_eq(a, transmute(b)) +} + +/// Signed compare bitwise equal to zero +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(cmeq))] +pub unsafe fn vceqzq_p8(a: poly8x16_t) -> uint8x16_t { + let b: i8x16 = i8x16::new(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + simd_eq(a, transmute(b)) +} + +/// Signed compare bitwise equal to zero +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(cmeq))] +pub unsafe fn vceqz_p64(a: poly64x1_t) -> uint64x1_t { + let b: i64x1 = i64x1::new(0); + simd_eq(a, transmute(b)) +} + +/// Signed compare bitwise equal to zero +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(cmeq))] +pub unsafe fn vceqzq_p64(a: poly64x2_t) -> uint64x2_t { + let b: i64x2 = i64x2::new(0, 0); + simd_eq(a, transmute(b)) +} + +/// Unsigned compare bitwise equal to zero +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(cmeq))] +pub unsafe fn vceqz_u8(a: uint8x8_t) -> uint8x8_t { + let b: u8x8 = u8x8::new(0, 0, 0, 0, 0, 0, 0, 0); + simd_eq(a, transmute(b)) +} + +/// Unsigned compare bitwise equal to zero +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(cmeq))] +pub unsafe fn vceqzq_u8(a: uint8x16_t) -> uint8x16_t { + let b: u8x16 = u8x16::new(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + simd_eq(a, transmute(b)) +} + +/// Unsigned compare bitwise equal to zero +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(cmeq))] +pub unsafe fn vceqz_u16(a: uint16x4_t) -> uint16x4_t { + let b: u16x4 = u16x4::new(0, 0, 0, 0); + simd_eq(a, transmute(b)) +} + +/// Unsigned compare bitwise equal to zero +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(cmeq))] +pub unsafe fn vceqzq_u16(a: uint16x8_t) -> uint16x8_t { + let b: u16x8 = u16x8::new(0, 0, 0, 0, 0, 0, 0, 0); + simd_eq(a, transmute(b)) +} + +/// Unsigned compare bitwise equal to zero +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(cmeq))] +pub unsafe fn vceqz_u32(a: uint32x2_t) -> uint32x2_t { + let b: u32x2 = u32x2::new(0, 0); + simd_eq(a, transmute(b)) +} + +/// Unsigned compare bitwise equal to zero +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(cmeq))] +pub unsafe fn vceqzq_u32(a: uint32x4_t) -> uint32x4_t { + let b: u32x4 = u32x4::new(0, 0, 0, 0); + simd_eq(a, transmute(b)) +} + +/// Unsigned compare bitwise equal to zero +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(cmeq))] +pub unsafe fn vceqz_u64(a: uint64x1_t) -> uint64x1_t { + let b: u64x1 = u64x1::new(0); + simd_eq(a, transmute(b)) +} + +/// Unsigned compare bitwise equal to zero +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(cmeq))] +pub unsafe fn vceqzq_u64(a: uint64x2_t) -> uint64x2_t { + let b: u64x2 = u64x2::new(0, 0); + simd_eq(a, transmute(b)) +} + +/// Floating-point compare bitwise equal to zero +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcmeq))] +pub unsafe fn vceqz_f32(a: float32x2_t) -> uint32x2_t { + let b: f32x2 = f32x2::new(0.0, 0.0); + simd_eq(a, transmute(b)) +} + +/// Floating-point compare bitwise equal to zero +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcmeq))] +pub unsafe fn vceqzq_f32(a: float32x4_t) -> uint32x4_t { + let b: f32x4 = f32x4::new(0.0, 0.0, 0.0, 0.0); + simd_eq(a, transmute(b)) +} + +/// Floating-point compare bitwise equal to zero +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcmeq))] +pub unsafe fn vceqz_f64(a: float64x1_t) -> uint64x1_t { + let b: f64 = 0.0; + simd_eq(a, transmute(b)) +} + +/// Floating-point compare bitwise equal to zero +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcmeq))] +pub unsafe fn vceqzq_f64(a: float64x2_t) -> uint64x2_t { + let b: f64x2 = f64x2::new(0.0, 0.0); + simd_eq(a, transmute(b)) +} + +/// Signed compare bitwise Test bits nonzero +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(cmtst))] +pub unsafe fn vtst_s64(a: int64x1_t, b: int64x1_t) -> uint64x1_t { + let c: int64x1_t = simd_and(a, b); + let d: i64x1 = i64x1::new(0); + simd_ne(c, transmute(d)) +} + +/// Signed compare bitwise Test bits nonzero +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(cmtst))] +pub unsafe fn vtstq_s64(a: int64x2_t, b: int64x2_t) -> uint64x2_t { + let c: int64x2_t = simd_and(a, b); + let d: i64x2 = i64x2::new(0, 0); + simd_ne(c, transmute(d)) +} + +/// Signed compare bitwise Test bits nonzero +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(cmtst))] +pub unsafe fn vtst_p64(a: poly64x1_t, b: poly64x1_t) -> uint64x1_t { + let c: poly64x1_t = simd_and(a, b); + let d: i64x1 = i64x1::new(0); + simd_ne(c, transmute(d)) +} + +/// Signed compare bitwise Test bits nonzero +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(cmtst))] +pub unsafe fn vtstq_p64(a: poly64x2_t, b: poly64x2_t) -> uint64x2_t { + let c: poly64x2_t = simd_and(a, b); + let d: i64x2 = i64x2::new(0, 0); + simd_ne(c, transmute(d)) +} + +/// Unsigned compare bitwise Test bits nonzero +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(cmtst))] +pub unsafe fn vtst_u64(a: uint64x1_t, b: uint64x1_t) -> uint64x1_t { + let c: uint64x1_t = simd_and(a, b); + let d: u64x1 = u64x1::new(0); + simd_ne(c, transmute(d)) +} + +/// Unsigned compare bitwise Test bits nonzero +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(cmtst))] +pub unsafe fn vtstq_u64(a: uint64x2_t, b: uint64x2_t) -> uint64x2_t { + let c: uint64x2_t = simd_and(a, b); + let d: u64x2 = u64x2::new(0, 0); + simd_ne(c, transmute(d)) +} + +/// Floating-point absolute value +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fabs))] +pub unsafe fn vabs_f64(a: float64x1_t) -> float64x1_t { + simd_fabs(a) +} + +/// Floating-point absolute value +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fabs))] +pub unsafe fn vabsq_f64(a: float64x2_t) -> float64x2_t { + simd_fabs(a) +} + /// Compare signed greater than #[inline] #[target_feature(enable = "neon")] @@ -265,490 +646,16872 @@ pub unsafe fn vcgeq_f64(a: float64x2_t, b: float64x2_t) -> uint64x2_t { simd_ge(a, b) } -/// Multiply +/// Compare signed greater than or equal to zero #[inline] #[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(fmul))] -pub unsafe fn vmul_f64(a: float64x1_t, b: float64x1_t) -> float64x1_t { - simd_mul(a, b) +#[cfg_attr(test, assert_instr(cmge))] +pub unsafe fn vcgez_s8(a: int8x8_t) -> uint8x8_t { + let b: i8x8 = i8x8::new(0, 0, 0, 0, 0, 0, 0, 0); + simd_ge(a, transmute(b)) } -/// Multiply +/// Compare signed greater than or equal to zero #[inline] #[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(fmul))] -pub unsafe fn vmulq_f64(a: float64x2_t, b: float64x2_t) -> float64x2_t { - simd_mul(a, b) +#[cfg_attr(test, assert_instr(cmge))] +pub unsafe fn vcgezq_s8(a: int8x16_t) -> uint8x16_t { + let b: i8x16 = i8x16::new(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + simd_ge(a, transmute(b)) } -/// Subtract +/// Compare signed greater than or equal to zero #[inline] #[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(fsub))] -pub unsafe fn vsub_f64(a: float64x1_t, b: float64x1_t) -> float64x1_t { - simd_sub(a, b) +#[cfg_attr(test, assert_instr(cmge))] +pub unsafe fn vcgez_s16(a: int16x4_t) -> uint16x4_t { + let b: i16x4 = i16x4::new(0, 0, 0, 0); + simd_ge(a, transmute(b)) } -/// Subtract +/// Compare signed greater than or equal to zero #[inline] #[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(fsub))] -pub unsafe fn vsubq_f64(a: float64x2_t, b: float64x2_t) -> float64x2_t { - simd_sub(a, b) +#[cfg_attr(test, assert_instr(cmge))] +pub unsafe fn vcgezq_s16(a: int16x8_t) -> uint16x8_t { + let b: i16x8 = i16x8::new(0, 0, 0, 0, 0, 0, 0, 0); + simd_ge(a, transmute(b)) } -/// Maximum (vector) +/// Compare signed greater than or equal to zero #[inline] #[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(fmax))] -pub unsafe fn vmax_f64(a: float64x1_t, b: float64x1_t) -> float64x1_t { - #[allow(improper_ctypes)] - extern "C" { - #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fmax.v1f64")] - fn vmax_f64_(a: float64x1_t, a: float64x1_t) -> float64x1_t; - } - vmax_f64_(a, b) +#[cfg_attr(test, assert_instr(cmge))] +pub unsafe fn vcgez_s32(a: int32x2_t) -> uint32x2_t { + let b: i32x2 = i32x2::new(0, 0); + simd_ge(a, transmute(b)) } -/// Maximum (vector) +/// Compare signed greater than or equal to zero #[inline] #[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(fmax))] -pub unsafe fn vmaxq_f64(a: float64x2_t, b: float64x2_t) -> float64x2_t { - #[allow(improper_ctypes)] - extern "C" { - #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fmax.v2f64")] - fn vmaxq_f64_(a: float64x2_t, a: float64x2_t) -> float64x2_t; - } - vmaxq_f64_(a, b) +#[cfg_attr(test, assert_instr(cmge))] +pub unsafe fn vcgezq_s32(a: int32x4_t) -> uint32x4_t { + let b: i32x4 = i32x4::new(0, 0, 0, 0); + simd_ge(a, transmute(b)) } -/// Minimum (vector) +/// Compare signed greater than or equal to zero #[inline] #[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(fmin))] -pub unsafe fn vmin_f64(a: float64x1_t, b: float64x1_t) -> float64x1_t { - #[allow(improper_ctypes)] - extern "C" { - #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fmin.v1f64")] - fn vmin_f64_(a: float64x1_t, a: float64x1_t) -> float64x1_t; - } - vmin_f64_(a, b) +#[cfg_attr(test, assert_instr(cmge))] +pub unsafe fn vcgez_s64(a: int64x1_t) -> uint64x1_t { + let b: i64x1 = i64x1::new(0); + simd_ge(a, transmute(b)) } -/// Minimum (vector) +/// Compare signed greater than or equal to zero #[inline] #[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(fmin))] -pub unsafe fn vminq_f64(a: float64x2_t, b: float64x2_t) -> float64x2_t { - #[allow(improper_ctypes)] - extern "C" { - #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fmin.v2f64")] - fn vminq_f64_(a: float64x2_t, a: float64x2_t) -> float64x2_t; - } - vminq_f64_(a, b) +#[cfg_attr(test, assert_instr(cmge))] +pub unsafe fn vcgezq_s64(a: int64x2_t) -> uint64x2_t { + let b: i64x2 = i64x2::new(0, 0); + simd_ge(a, transmute(b)) } -#[cfg(test)] -mod test { - use super::*; - use crate::core_arch::simd::*; - use std::mem::transmute; - use stdarch_test::simd_test; +/// Floating-point compare greater than or equal to zero +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcmge))] +pub unsafe fn vcgez_f32(a: float32x2_t) -> uint32x2_t { + let b: f32x2 = f32x2::new(0.0, 0.0); + simd_ge(a, transmute(b)) +} + +/// Floating-point compare greater than or equal to zero +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcmge))] +pub unsafe fn vcgezq_f32(a: float32x4_t) -> uint32x4_t { + let b: f32x4 = f32x4::new(0.0, 0.0, 0.0, 0.0); + simd_ge(a, transmute(b)) +} + +/// Floating-point compare greater than or equal to zero +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcmge))] +pub unsafe fn vcgez_f64(a: float64x1_t) -> uint64x1_t { + let b: f64 = 0.0; + simd_ge(a, transmute(b)) +} + +/// Floating-point compare greater than or equal to zero +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcmge))] +pub unsafe fn vcgezq_f64(a: float64x2_t) -> uint64x2_t { + let b: f64x2 = f64x2::new(0.0, 0.0); + simd_ge(a, transmute(b)) +} + +/// Compare signed greater than zero +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(cmgt))] +pub unsafe fn vcgtz_s8(a: int8x8_t) -> uint8x8_t { + let b: i8x8 = i8x8::new(0, 0, 0, 0, 0, 0, 0, 0); + simd_gt(a, transmute(b)) +} + +/// Compare signed greater than zero +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(cmgt))] +pub unsafe fn vcgtzq_s8(a: int8x16_t) -> uint8x16_t { + let b: i8x16 = i8x16::new(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + simd_gt(a, transmute(b)) +} + +/// Compare signed greater than zero +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(cmgt))] +pub unsafe fn vcgtz_s16(a: int16x4_t) -> uint16x4_t { + let b: i16x4 = i16x4::new(0, 0, 0, 0); + simd_gt(a, transmute(b)) +} + +/// Compare signed greater than zero +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(cmgt))] +pub unsafe fn vcgtzq_s16(a: int16x8_t) -> uint16x8_t { + let b: i16x8 = i16x8::new(0, 0, 0, 0, 0, 0, 0, 0); + simd_gt(a, transmute(b)) +} + +/// Compare signed greater than zero +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(cmgt))] +pub unsafe fn vcgtz_s32(a: int32x2_t) -> uint32x2_t { + let b: i32x2 = i32x2::new(0, 0); + simd_gt(a, transmute(b)) +} + +/// Compare signed greater than zero +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(cmgt))] +pub unsafe fn vcgtzq_s32(a: int32x4_t) -> uint32x4_t { + let b: i32x4 = i32x4::new(0, 0, 0, 0); + simd_gt(a, transmute(b)) +} + +/// Compare signed greater than zero +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(cmgt))] +pub unsafe fn vcgtz_s64(a: int64x1_t) -> uint64x1_t { + let b: i64x1 = i64x1::new(0); + simd_gt(a, transmute(b)) +} + +/// Compare signed greater than zero +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(cmgt))] +pub unsafe fn vcgtzq_s64(a: int64x2_t) -> uint64x2_t { + let b: i64x2 = i64x2::new(0, 0); + simd_gt(a, transmute(b)) +} + +/// Floating-point compare greater than zero +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcmgt))] +pub unsafe fn vcgtz_f32(a: float32x2_t) -> uint32x2_t { + let b: f32x2 = f32x2::new(0.0, 0.0); + simd_gt(a, transmute(b)) +} + +/// Floating-point compare greater than zero +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcmgt))] +pub unsafe fn vcgtzq_f32(a: float32x4_t) -> uint32x4_t { + let b: f32x4 = f32x4::new(0.0, 0.0, 0.0, 0.0); + simd_gt(a, transmute(b)) +} + +/// Floating-point compare greater than zero +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcmgt))] +pub unsafe fn vcgtz_f64(a: float64x1_t) -> uint64x1_t { + let b: f64 = 0.0; + simd_gt(a, transmute(b)) +} + +/// Floating-point compare greater than zero +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcmgt))] +pub unsafe fn vcgtzq_f64(a: float64x2_t) -> uint64x2_t { + let b: f64x2 = f64x2::new(0.0, 0.0); + simd_gt(a, transmute(b)) +} + +/// Compare signed less than or equal to zero +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(cmgt))] +pub unsafe fn vclez_s8(a: int8x8_t) -> uint8x8_t { + let b: i8x8 = i8x8::new(0, 0, 0, 0, 0, 0, 0, 0); + simd_le(a, transmute(b)) +} + +/// Compare signed less than or equal to zero +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(cmgt))] +pub unsafe fn vclezq_s8(a: int8x16_t) -> uint8x16_t { + let b: i8x16 = i8x16::new(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + simd_le(a, transmute(b)) +} + +/// Compare signed less than or equal to zero +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(cmgt))] +pub unsafe fn vclez_s16(a: int16x4_t) -> uint16x4_t { + let b: i16x4 = i16x4::new(0, 0, 0, 0); + simd_le(a, transmute(b)) +} + +/// Compare signed less than or equal to zero +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(cmgt))] +pub unsafe fn vclezq_s16(a: int16x8_t) -> uint16x8_t { + let b: i16x8 = i16x8::new(0, 0, 0, 0, 0, 0, 0, 0); + simd_le(a, transmute(b)) +} + +/// Compare signed less than or equal to zero +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(cmgt))] +pub unsafe fn vclez_s32(a: int32x2_t) -> uint32x2_t { + let b: i32x2 = i32x2::new(0, 0); + simd_le(a, transmute(b)) +} + +/// Compare signed less than or equal to zero +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(cmgt))] +pub unsafe fn vclezq_s32(a: int32x4_t) -> uint32x4_t { + let b: i32x4 = i32x4::new(0, 0, 0, 0); + simd_le(a, transmute(b)) +} + +/// Compare signed less than or equal to zero +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(cmgt))] +pub unsafe fn vclez_s64(a: int64x1_t) -> uint64x1_t { + let b: i64x1 = i64x1::new(0); + simd_le(a, transmute(b)) +} + +/// Compare signed less than or equal to zero +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(cmgt))] +pub unsafe fn vclezq_s64(a: int64x2_t) -> uint64x2_t { + let b: i64x2 = i64x2::new(0, 0); + simd_le(a, transmute(b)) +} + +/// Floating-point compare less than or equal to zero +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcmle))] +pub unsafe fn vclez_f32(a: float32x2_t) -> uint32x2_t { + let b: f32x2 = f32x2::new(0.0, 0.0); + simd_le(a, transmute(b)) +} + +/// Floating-point compare less than or equal to zero +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcmle))] +pub unsafe fn vclezq_f32(a: float32x4_t) -> uint32x4_t { + let b: f32x4 = f32x4::new(0.0, 0.0, 0.0, 0.0); + simd_le(a, transmute(b)) +} + +/// Floating-point compare less than or equal to zero +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcmle))] +pub unsafe fn vclez_f64(a: float64x1_t) -> uint64x1_t { + let b: f64 = 0.0; + simd_le(a, transmute(b)) +} + +/// Floating-point compare less than or equal to zero +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcmle))] +pub unsafe fn vclezq_f64(a: float64x2_t) -> uint64x2_t { + let b: f64x2 = f64x2::new(0.0, 0.0); + simd_le(a, transmute(b)) +} + +/// Compare signed less than zero +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sshr))] +pub unsafe fn vcltz_s8(a: int8x8_t) -> uint8x8_t { + let b: i8x8 = i8x8::new(0, 0, 0, 0, 0, 0, 0, 0); + simd_lt(a, transmute(b)) +} + +/// Compare signed less than zero +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sshr))] +pub unsafe fn vcltzq_s8(a: int8x16_t) -> uint8x16_t { + let b: i8x16 = i8x16::new(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + simd_lt(a, transmute(b)) +} + +/// Compare signed less than zero +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sshr))] +pub unsafe fn vcltz_s16(a: int16x4_t) -> uint16x4_t { + let b: i16x4 = i16x4::new(0, 0, 0, 0); + simd_lt(a, transmute(b)) +} + +/// Compare signed less than zero +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sshr))] +pub unsafe fn vcltzq_s16(a: int16x8_t) -> uint16x8_t { + let b: i16x8 = i16x8::new(0, 0, 0, 0, 0, 0, 0, 0); + simd_lt(a, transmute(b)) +} + +/// Compare signed less than zero +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sshr))] +pub unsafe fn vcltz_s32(a: int32x2_t) -> uint32x2_t { + let b: i32x2 = i32x2::new(0, 0); + simd_lt(a, transmute(b)) +} + +/// Compare signed less than zero +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sshr))] +pub unsafe fn vcltzq_s32(a: int32x4_t) -> uint32x4_t { + let b: i32x4 = i32x4::new(0, 0, 0, 0); + simd_lt(a, transmute(b)) +} + +/// Compare signed less than zero +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sshr))] +pub unsafe fn vcltz_s64(a: int64x1_t) -> uint64x1_t { + let b: i64x1 = i64x1::new(0); + simd_lt(a, transmute(b)) +} + +/// Compare signed less than zero +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sshr))] +pub unsafe fn vcltzq_s64(a: int64x2_t) -> uint64x2_t { + let b: i64x2 = i64x2::new(0, 0); + simd_lt(a, transmute(b)) +} + +/// Floating-point compare less than zero +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcmlt))] +pub unsafe fn vcltz_f32(a: float32x2_t) -> uint32x2_t { + let b: f32x2 = f32x2::new(0.0, 0.0); + simd_lt(a, transmute(b)) +} + +/// Floating-point compare less than zero +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcmlt))] +pub unsafe fn vcltzq_f32(a: float32x4_t) -> uint32x4_t { + let b: f32x4 = f32x4::new(0.0, 0.0, 0.0, 0.0); + simd_lt(a, transmute(b)) +} + +/// Floating-point compare less than zero +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcmlt))] +pub unsafe fn vcltz_f64(a: float64x1_t) -> uint64x1_t { + let b: f64 = 0.0; + simd_lt(a, transmute(b)) +} + +/// Floating-point compare less than zero +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcmlt))] +pub unsafe fn vcltzq_f64(a: float64x2_t) -> uint64x2_t { + let b: f64x2 = f64x2::new(0.0, 0.0); + simd_lt(a, transmute(b)) +} + +/// Floating-point absolute compare greater than +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(facgt))] +pub unsafe fn vcagt_f64(a: float64x1_t, b: float64x1_t) -> uint64x1_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.facgt.v1i64.v1f64")] + fn vcagt_f64_(a: float64x1_t, b: float64x1_t) -> uint64x1_t; + } + vcagt_f64_(a, b) +} + +/// Floating-point absolute compare greater than +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(facgt))] +pub unsafe fn vcagtq_f64(a: float64x2_t, b: float64x2_t) -> uint64x2_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.facgt.v2i64.v2f64")] + fn vcagtq_f64_(a: float64x2_t, b: float64x2_t) -> uint64x2_t; + } + vcagtq_f64_(a, b) +} + +/// Floating-point absolute compare greater than or equal +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(facge))] +pub unsafe fn vcage_f64(a: float64x1_t, b: float64x1_t) -> uint64x1_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.facge.v1i64.v1f64")] + fn vcage_f64_(a: float64x1_t, b: float64x1_t) -> uint64x1_t; + } + vcage_f64_(a, b) +} + +/// Floating-point absolute compare greater than or equal +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(facge))] +pub unsafe fn vcageq_f64(a: float64x2_t, b: float64x2_t) -> uint64x2_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.facge.v2i64.v2f64")] + fn vcageq_f64_(a: float64x2_t, b: float64x2_t) -> uint64x2_t; + } + vcageq_f64_(a, b) +} + +/// Floating-point absolute compare less than +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(facgt))] +pub unsafe fn vcalt_f64(a: float64x1_t, b: float64x1_t) -> uint64x1_t { + vcagt_f64(b, a) +} + +/// Floating-point absolute compare less than +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(facgt))] +pub unsafe fn vcaltq_f64(a: float64x2_t, b: float64x2_t) -> uint64x2_t { + vcagtq_f64(b, a) +} + +/// Floating-point absolute compare less than or equal +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(facge))] +pub unsafe fn vcale_f64(a: float64x1_t, b: float64x1_t) -> uint64x1_t { + vcage_f64(b, a) +} + +/// Floating-point absolute compare less than or equal +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(facge))] +pub unsafe fn vcaleq_f64(a: float64x2_t, b: float64x2_t) -> uint64x2_t { + vcageq_f64(b, a) +} + +/// Insert vector element from another vector element +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(mov, LANE1 = 0, LANE2 = 1))] +#[rustc_legacy_const_generics(1, 3)] +pub unsafe fn vcopy_lane_s8(a: int8x8_t, b: int8x8_t) -> int8x8_t { + static_assert_imm3!(LANE1); + static_assert_imm3!(LANE2); + match LANE1 & 0b111 { + 0 => simd_shuffle8!(a, b, [8 + LANE2 as u32, 1, 2, 3, 4, 5, 6, 7]), + 1 => simd_shuffle8!(a, b, [0, 8 + LANE2 as u32, 2, 3, 4, 5, 6, 7]), + 2 => simd_shuffle8!(a, b, [0, 1, 8 + LANE2 as u32, 3, 4, 5, 6, 7]), + 3 => simd_shuffle8!(a, b, [0, 1, 2, 8 + LANE2 as u32, 4, 5, 6, 7]), + 4 => simd_shuffle8!(a, b, [0, 1, 2, 3, 8 + LANE2 as u32, 5, 6, 7]), + 5 => simd_shuffle8!(a, b, [0, 1, 2, 3, 4, 8 + LANE2 as u32, 6, 7]), + 6 => simd_shuffle8!(a, b, [0, 1, 2, 3, 4, 5, 8 + LANE2 as u32, 7]), + 7 => simd_shuffle8!(a, b, [0, 1, 2, 3, 4, 5, 6, 8 + LANE2 as u32]), + _ => unreachable_unchecked(), + } +} + +/// Insert vector element from another vector element +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(mov, LANE1 = 0, LANE2 = 1))] +#[rustc_legacy_const_generics(1, 3)] +pub unsafe fn vcopyq_laneq_s8(a: int8x16_t, b: int8x16_t) -> int8x16_t { + static_assert_imm4!(LANE1); + static_assert_imm4!(LANE2); + match LANE1 & 0b1111 { + 0 => simd_shuffle16!(a, b, [16 + LANE2 as u32, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), + 1 => simd_shuffle16!(a, b, [0, 16 + LANE2 as u32, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), + 2 => simd_shuffle16!(a, b, [0, 1, 16 + LANE2 as u32, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), + 3 => simd_shuffle16!(a, b, [0, 1, 2, 16 + LANE2 as u32, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), + 4 => simd_shuffle16!(a, b, [0, 1, 2, 3, 16 + LANE2 as u32, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), + 5 => simd_shuffle16!(a, b, [0, 1, 2, 3, 4, 16 + LANE2 as u32, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), + 6 => simd_shuffle16!(a, b, [0, 1, 2, 3, 4, 5, 16 + LANE2 as u32, 7, 8, 9, 10, 11, 12, 13, 14, 15]), + 7 => simd_shuffle16!(a, b, [0, 1, 2, 3, 4, 5, 6, 16 + LANE2 as u32, 8, 9, 10, 11, 12, 13, 14, 15]), + 8 => simd_shuffle16!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 16 + LANE2 as u32, 9, 10, 11, 12, 13, 14, 15]), + 9 => simd_shuffle16!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 16 + LANE2 as u32, 10, 11, 12, 13, 14, 15]), + 10 => simd_shuffle16!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 16 + LANE2 as u32, 11, 12, 13, 14, 15]), + 11 => simd_shuffle16!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 16 + LANE2 as u32, 12, 13, 14, 15]), + 12 => simd_shuffle16!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 16 + LANE2 as u32, 13, 14, 15]), + 13 => simd_shuffle16!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 16 + LANE2 as u32, 14, 15]), + 14 => simd_shuffle16!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 16 + LANE2 as u32, 15]), + 15 => simd_shuffle16!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 16 + LANE2 as u32]), + _ => unreachable_unchecked(), + } +} + +/// Insert vector element from another vector element +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(mov, LANE1 = 0, LANE2 = 1))] +#[rustc_legacy_const_generics(1, 3)] +pub unsafe fn vcopy_lane_s16(a: int16x4_t, b: int16x4_t) -> int16x4_t { + static_assert_imm2!(LANE1); + static_assert_imm2!(LANE2); + match LANE1 & 0b11 { + 0 => simd_shuffle4!(a, b, [4 + LANE2 as u32, 1, 2, 3]), + 1 => simd_shuffle4!(a, b, [0, 4 + LANE2 as u32, 2, 3]), + 2 => simd_shuffle4!(a, b, [0, 1, 4 + LANE2 as u32, 3]), + 3 => simd_shuffle4!(a, b, [0, 1, 2, 4 + LANE2 as u32]), + _ => unreachable_unchecked(), + } +} + +/// Insert vector element from another vector element +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(mov, LANE1 = 0, LANE2 = 1))] +#[rustc_legacy_const_generics(1, 3)] +pub unsafe fn vcopyq_laneq_s16(a: int16x8_t, b: int16x8_t) -> int16x8_t { + static_assert_imm3!(LANE1); + static_assert_imm3!(LANE2); + match LANE1 & 0b111 { + 0 => simd_shuffle8!(a, b, [8 + LANE2 as u32, 1, 2, 3, 4, 5, 6, 7]), + 1 => simd_shuffle8!(a, b, [0, 8 + LANE2 as u32, 2, 3, 4, 5, 6, 7]), + 2 => simd_shuffle8!(a, b, [0, 1, 8 + LANE2 as u32, 3, 4, 5, 6, 7]), + 3 => simd_shuffle8!(a, b, [0, 1, 2, 8 + LANE2 as u32, 4, 5, 6, 7]), + 4 => simd_shuffle8!(a, b, [0, 1, 2, 3, 8 + LANE2 as u32, 5, 6, 7]), + 5 => simd_shuffle8!(a, b, [0, 1, 2, 3, 4, 8 + LANE2 as u32, 6, 7]), + 6 => simd_shuffle8!(a, b, [0, 1, 2, 3, 4, 5, 8 + LANE2 as u32, 7]), + 7 => simd_shuffle8!(a, b, [0, 1, 2, 3, 4, 5, 6, 8 + LANE2 as u32]), + _ => unreachable_unchecked(), + } +} + +/// Insert vector element from another vector element +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(mov, LANE1 = 0, LANE2 = 1))] +#[rustc_legacy_const_generics(1, 3)] +pub unsafe fn vcopy_lane_s32(a: int32x2_t, b: int32x2_t) -> int32x2_t { + static_assert_imm1!(LANE1); + static_assert_imm1!(LANE2); + match LANE1 & 0b1 { + 0 => simd_shuffle2!(a, b, [2 + LANE2 as u32, 1]), + 1 => simd_shuffle2!(a, b, [0, 2 + LANE2 as u32]), + _ => unreachable_unchecked(), + } +} + +/// Insert vector element from another vector element +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(mov, LANE1 = 0, LANE2 = 1))] +#[rustc_legacy_const_generics(1, 3)] +pub unsafe fn vcopyq_laneq_s32(a: int32x4_t, b: int32x4_t) -> int32x4_t { + static_assert_imm2!(LANE1); + static_assert_imm2!(LANE2); + match LANE1 & 0b11 { + 0 => simd_shuffle4!(a, b, [4 + LANE2 as u32, 1, 2, 3]), + 1 => simd_shuffle4!(a, b, [0, 4 + LANE2 as u32, 2, 3]), + 2 => simd_shuffle4!(a, b, [0, 1, 4 + LANE2 as u32, 3]), + 3 => simd_shuffle4!(a, b, [0, 1, 2, 4 + LANE2 as u32]), + _ => unreachable_unchecked(), + } +} + +/// Insert vector element from another vector element +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(mov, LANE1 = 0, LANE2 = 1))] +#[rustc_legacy_const_generics(1, 3)] +pub unsafe fn vcopyq_laneq_s64(a: int64x2_t, b: int64x2_t) -> int64x2_t { + static_assert_imm1!(LANE1); + static_assert_imm1!(LANE2); + match LANE1 & 0b1 { + 0 => simd_shuffle2!(a, b, [2 + LANE2 as u32, 1]), + 1 => simd_shuffle2!(a, b, [0, 2 + LANE2 as u32]), + _ => unreachable_unchecked(), + } +} + +/// Insert vector element from another vector element +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(mov, LANE1 = 0, LANE2 = 1))] +#[rustc_legacy_const_generics(1, 3)] +pub unsafe fn vcopy_lane_u8(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t { + static_assert_imm3!(LANE1); + static_assert_imm3!(LANE2); + match LANE1 & 0b111 { + 0 => simd_shuffle8!(a, b, [8 + LANE2 as u32, 1, 2, 3, 4, 5, 6, 7]), + 1 => simd_shuffle8!(a, b, [0, 8 + LANE2 as u32, 2, 3, 4, 5, 6, 7]), + 2 => simd_shuffle8!(a, b, [0, 1, 8 + LANE2 as u32, 3, 4, 5, 6, 7]), + 3 => simd_shuffle8!(a, b, [0, 1, 2, 8 + LANE2 as u32, 4, 5, 6, 7]), + 4 => simd_shuffle8!(a, b, [0, 1, 2, 3, 8 + LANE2 as u32, 5, 6, 7]), + 5 => simd_shuffle8!(a, b, [0, 1, 2, 3, 4, 8 + LANE2 as u32, 6, 7]), + 6 => simd_shuffle8!(a, b, [0, 1, 2, 3, 4, 5, 8 + LANE2 as u32, 7]), + 7 => simd_shuffle8!(a, b, [0, 1, 2, 3, 4, 5, 6, 8 + LANE2 as u32]), + _ => unreachable_unchecked(), + } +} + +/// Insert vector element from another vector element +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(mov, LANE1 = 0, LANE2 = 1))] +#[rustc_legacy_const_generics(1, 3)] +pub unsafe fn vcopyq_laneq_u8(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t { + static_assert_imm4!(LANE1); + static_assert_imm4!(LANE2); + match LANE1 & 0b1111 { + 0 => simd_shuffle16!(a, b, [16 + LANE2 as u32, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), + 1 => simd_shuffle16!(a, b, [0, 16 + LANE2 as u32, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), + 2 => simd_shuffle16!(a, b, [0, 1, 16 + LANE2 as u32, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), + 3 => simd_shuffle16!(a, b, [0, 1, 2, 16 + LANE2 as u32, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), + 4 => simd_shuffle16!(a, b, [0, 1, 2, 3, 16 + LANE2 as u32, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), + 5 => simd_shuffle16!(a, b, [0, 1, 2, 3, 4, 16 + LANE2 as u32, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), + 6 => simd_shuffle16!(a, b, [0, 1, 2, 3, 4, 5, 16 + LANE2 as u32, 7, 8, 9, 10, 11, 12, 13, 14, 15]), + 7 => simd_shuffle16!(a, b, [0, 1, 2, 3, 4, 5, 6, 16 + LANE2 as u32, 8, 9, 10, 11, 12, 13, 14, 15]), + 8 => simd_shuffle16!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 16 + LANE2 as u32, 9, 10, 11, 12, 13, 14, 15]), + 9 => simd_shuffle16!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 16 + LANE2 as u32, 10, 11, 12, 13, 14, 15]), + 10 => simd_shuffle16!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 16 + LANE2 as u32, 11, 12, 13, 14, 15]), + 11 => simd_shuffle16!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 16 + LANE2 as u32, 12, 13, 14, 15]), + 12 => simd_shuffle16!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 16 + LANE2 as u32, 13, 14, 15]), + 13 => simd_shuffle16!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 16 + LANE2 as u32, 14, 15]), + 14 => simd_shuffle16!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 16 + LANE2 as u32, 15]), + 15 => simd_shuffle16!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 16 + LANE2 as u32]), + _ => unreachable_unchecked(), + } +} + +/// Insert vector element from another vector element +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(mov, LANE1 = 0, LANE2 = 1))] +#[rustc_legacy_const_generics(1, 3)] +pub unsafe fn vcopy_lane_u16(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t { + static_assert_imm2!(LANE1); + static_assert_imm2!(LANE2); + match LANE1 & 0b11 { + 0 => simd_shuffle4!(a, b, [4 + LANE2 as u32, 1, 2, 3]), + 1 => simd_shuffle4!(a, b, [0, 4 + LANE2 as u32, 2, 3]), + 2 => simd_shuffle4!(a, b, [0, 1, 4 + LANE2 as u32, 3]), + 3 => simd_shuffle4!(a, b, [0, 1, 2, 4 + LANE2 as u32]), + _ => unreachable_unchecked(), + } +} + +/// Insert vector element from another vector element +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(mov, LANE1 = 0, LANE2 = 1))] +#[rustc_legacy_const_generics(1, 3)] +pub unsafe fn vcopyq_laneq_u16(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t { + static_assert_imm3!(LANE1); + static_assert_imm3!(LANE2); + match LANE1 & 0b111 { + 0 => simd_shuffle8!(a, b, [8 + LANE2 as u32, 1, 2, 3, 4, 5, 6, 7]), + 1 => simd_shuffle8!(a, b, [0, 8 + LANE2 as u32, 2, 3, 4, 5, 6, 7]), + 2 => simd_shuffle8!(a, b, [0, 1, 8 + LANE2 as u32, 3, 4, 5, 6, 7]), + 3 => simd_shuffle8!(a, b, [0, 1, 2, 8 + LANE2 as u32, 4, 5, 6, 7]), + 4 => simd_shuffle8!(a, b, [0, 1, 2, 3, 8 + LANE2 as u32, 5, 6, 7]), + 5 => simd_shuffle8!(a, b, [0, 1, 2, 3, 4, 8 + LANE2 as u32, 6, 7]), + 6 => simd_shuffle8!(a, b, [0, 1, 2, 3, 4, 5, 8 + LANE2 as u32, 7]), + 7 => simd_shuffle8!(a, b, [0, 1, 2, 3, 4, 5, 6, 8 + LANE2 as u32]), + _ => unreachable_unchecked(), + } +} + +/// Insert vector element from another vector element +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(mov, LANE1 = 0, LANE2 = 1))] +#[rustc_legacy_const_generics(1, 3)] +pub unsafe fn vcopy_lane_u32(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t { + static_assert_imm1!(LANE1); + static_assert_imm1!(LANE2); + match LANE1 & 0b1 { + 0 => simd_shuffle2!(a, b, [2 + LANE2 as u32, 1]), + 1 => simd_shuffle2!(a, b, [0, 2 + LANE2 as u32]), + _ => unreachable_unchecked(), + } +} + +/// Insert vector element from another vector element +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(mov, LANE1 = 0, LANE2 = 1))] +#[rustc_legacy_const_generics(1, 3)] +pub unsafe fn vcopyq_laneq_u32(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t { + static_assert_imm2!(LANE1); + static_assert_imm2!(LANE2); + match LANE1 & 0b11 { + 0 => simd_shuffle4!(a, b, [4 + LANE2 as u32, 1, 2, 3]), + 1 => simd_shuffle4!(a, b, [0, 4 + LANE2 as u32, 2, 3]), + 2 => simd_shuffle4!(a, b, [0, 1, 4 + LANE2 as u32, 3]), + 3 => simd_shuffle4!(a, b, [0, 1, 2, 4 + LANE2 as u32]), + _ => unreachable_unchecked(), + } +} + +/// Insert vector element from another vector element +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(mov, LANE1 = 0, LANE2 = 1))] +#[rustc_legacy_const_generics(1, 3)] +pub unsafe fn vcopyq_laneq_u64(a: uint64x2_t, b: uint64x2_t) -> uint64x2_t { + static_assert_imm1!(LANE1); + static_assert_imm1!(LANE2); + match LANE1 & 0b1 { + 0 => simd_shuffle2!(a, b, [2 + LANE2 as u32, 1]), + 1 => simd_shuffle2!(a, b, [0, 2 + LANE2 as u32]), + _ => unreachable_unchecked(), + } +} + +/// Insert vector element from another vector element +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(mov, LANE1 = 0, LANE2 = 1))] +#[rustc_legacy_const_generics(1, 3)] +pub unsafe fn vcopy_lane_p8(a: poly8x8_t, b: poly8x8_t) -> poly8x8_t { + static_assert_imm3!(LANE1); + static_assert_imm3!(LANE2); + match LANE1 & 0b111 { + 0 => simd_shuffle8!(a, b, [8 + LANE2 as u32, 1, 2, 3, 4, 5, 6, 7]), + 1 => simd_shuffle8!(a, b, [0, 8 + LANE2 as u32, 2, 3, 4, 5, 6, 7]), + 2 => simd_shuffle8!(a, b, [0, 1, 8 + LANE2 as u32, 3, 4, 5, 6, 7]), + 3 => simd_shuffle8!(a, b, [0, 1, 2, 8 + LANE2 as u32, 4, 5, 6, 7]), + 4 => simd_shuffle8!(a, b, [0, 1, 2, 3, 8 + LANE2 as u32, 5, 6, 7]), + 5 => simd_shuffle8!(a, b, [0, 1, 2, 3, 4, 8 + LANE2 as u32, 6, 7]), + 6 => simd_shuffle8!(a, b, [0, 1, 2, 3, 4, 5, 8 + LANE2 as u32, 7]), + 7 => simd_shuffle8!(a, b, [0, 1, 2, 3, 4, 5, 6, 8 + LANE2 as u32]), + _ => unreachable_unchecked(), + } +} + +/// Insert vector element from another vector element +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(mov, LANE1 = 0, LANE2 = 1))] +#[rustc_legacy_const_generics(1, 3)] +pub unsafe fn vcopyq_laneq_p8(a: poly8x16_t, b: poly8x16_t) -> poly8x16_t { + static_assert_imm4!(LANE1); + static_assert_imm4!(LANE2); + match LANE1 & 0b1111 { + 0 => simd_shuffle16!(a, b, [16 + LANE2 as u32, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), + 1 => simd_shuffle16!(a, b, [0, 16 + LANE2 as u32, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), + 2 => simd_shuffle16!(a, b, [0, 1, 16 + LANE2 as u32, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), + 3 => simd_shuffle16!(a, b, [0, 1, 2, 16 + LANE2 as u32, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), + 4 => simd_shuffle16!(a, b, [0, 1, 2, 3, 16 + LANE2 as u32, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), + 5 => simd_shuffle16!(a, b, [0, 1, 2, 3, 4, 16 + LANE2 as u32, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), + 6 => simd_shuffle16!(a, b, [0, 1, 2, 3, 4, 5, 16 + LANE2 as u32, 7, 8, 9, 10, 11, 12, 13, 14, 15]), + 7 => simd_shuffle16!(a, b, [0, 1, 2, 3, 4, 5, 6, 16 + LANE2 as u32, 8, 9, 10, 11, 12, 13, 14, 15]), + 8 => simd_shuffle16!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 16 + LANE2 as u32, 9, 10, 11, 12, 13, 14, 15]), + 9 => simd_shuffle16!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 16 + LANE2 as u32, 10, 11, 12, 13, 14, 15]), + 10 => simd_shuffle16!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 16 + LANE2 as u32, 11, 12, 13, 14, 15]), + 11 => simd_shuffle16!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 16 + LANE2 as u32, 12, 13, 14, 15]), + 12 => simd_shuffle16!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 16 + LANE2 as u32, 13, 14, 15]), + 13 => simd_shuffle16!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 16 + LANE2 as u32, 14, 15]), + 14 => simd_shuffle16!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 16 + LANE2 as u32, 15]), + 15 => simd_shuffle16!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 16 + LANE2 as u32]), + _ => unreachable_unchecked(), + } +} + +/// Insert vector element from another vector element +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(mov, LANE1 = 0, LANE2 = 1))] +#[rustc_legacy_const_generics(1, 3)] +pub unsafe fn vcopy_lane_p16(a: poly16x4_t, b: poly16x4_t) -> poly16x4_t { + static_assert_imm2!(LANE1); + static_assert_imm2!(LANE2); + match LANE1 & 0b11 { + 0 => simd_shuffle4!(a, b, [4 + LANE2 as u32, 1, 2, 3]), + 1 => simd_shuffle4!(a, b, [0, 4 + LANE2 as u32, 2, 3]), + 2 => simd_shuffle4!(a, b, [0, 1, 4 + LANE2 as u32, 3]), + 3 => simd_shuffle4!(a, b, [0, 1, 2, 4 + LANE2 as u32]), + _ => unreachable_unchecked(), + } +} + +/// Insert vector element from another vector element +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(mov, LANE1 = 0, LANE2 = 1))] +#[rustc_legacy_const_generics(1, 3)] +pub unsafe fn vcopyq_laneq_p16(a: poly16x8_t, b: poly16x8_t) -> poly16x8_t { + static_assert_imm3!(LANE1); + static_assert_imm3!(LANE2); + match LANE1 & 0b111 { + 0 => simd_shuffle8!(a, b, [8 + LANE2 as u32, 1, 2, 3, 4, 5, 6, 7]), + 1 => simd_shuffle8!(a, b, [0, 8 + LANE2 as u32, 2, 3, 4, 5, 6, 7]), + 2 => simd_shuffle8!(a, b, [0, 1, 8 + LANE2 as u32, 3, 4, 5, 6, 7]), + 3 => simd_shuffle8!(a, b, [0, 1, 2, 8 + LANE2 as u32, 4, 5, 6, 7]), + 4 => simd_shuffle8!(a, b, [0, 1, 2, 3, 8 + LANE2 as u32, 5, 6, 7]), + 5 => simd_shuffle8!(a, b, [0, 1, 2, 3, 4, 8 + LANE2 as u32, 6, 7]), + 6 => simd_shuffle8!(a, b, [0, 1, 2, 3, 4, 5, 8 + LANE2 as u32, 7]), + 7 => simd_shuffle8!(a, b, [0, 1, 2, 3, 4, 5, 6, 8 + LANE2 as u32]), + _ => unreachable_unchecked(), + } +} + +/// Insert vector element from another vector element +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(mov, LANE1 = 0, LANE2 = 1))] +#[rustc_legacy_const_generics(1, 3)] +pub unsafe fn vcopyq_laneq_p64(a: poly64x2_t, b: poly64x2_t) -> poly64x2_t { + static_assert_imm1!(LANE1); + static_assert_imm1!(LANE2); + match LANE1 & 0b1 { + 0 => simd_shuffle2!(a, b, [2 + LANE2 as u32, 1]), + 1 => simd_shuffle2!(a, b, [0, 2 + LANE2 as u32]), + _ => unreachable_unchecked(), + } +} + +/// Insert vector element from another vector element +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(mov, LANE1 = 0, LANE2 = 1))] +#[rustc_legacy_const_generics(1, 3)] +pub unsafe fn vcopy_lane_f32(a: float32x2_t, b: float32x2_t) -> float32x2_t { + static_assert_imm1!(LANE1); + static_assert_imm1!(LANE2); + match LANE1 & 0b1 { + 0 => simd_shuffle2!(a, b, [2 + LANE2 as u32, 1]), + 1 => simd_shuffle2!(a, b, [0, 2 + LANE2 as u32]), + _ => unreachable_unchecked(), + } +} + +/// Insert vector element from another vector element +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(mov, LANE1 = 0, LANE2 = 1))] +#[rustc_legacy_const_generics(1, 3)] +pub unsafe fn vcopyq_laneq_f32(a: float32x4_t, b: float32x4_t) -> float32x4_t { + static_assert_imm2!(LANE1); + static_assert_imm2!(LANE2); + match LANE1 & 0b11 { + 0 => simd_shuffle4!(a, b, [4 + LANE2 as u32, 1, 2, 3]), + 1 => simd_shuffle4!(a, b, [0, 4 + LANE2 as u32, 2, 3]), + 2 => simd_shuffle4!(a, b, [0, 1, 4 + LANE2 as u32, 3]), + 3 => simd_shuffle4!(a, b, [0, 1, 2, 4 + LANE2 as u32]), + _ => unreachable_unchecked(), + } +} + +/// Insert vector element from another vector element +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(mov, LANE1 = 0, LANE2 = 1))] +#[rustc_legacy_const_generics(1, 3)] +pub unsafe fn vcopyq_laneq_f64(a: float64x2_t, b: float64x2_t) -> float64x2_t { + static_assert_imm1!(LANE1); + static_assert_imm1!(LANE2); + match LANE1 & 0b1 { + 0 => simd_shuffle2!(a, b, [2 + LANE2 as u32, 1]), + 1 => simd_shuffle2!(a, b, [0, 2 + LANE2 as u32]), + _ => unreachable_unchecked(), + } +} + +/// Insert vector element from another vector element +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(mov, LANE1 = 0, LANE2 = 1))] +#[rustc_legacy_const_generics(1, 3)] +pub unsafe fn vcopy_laneq_s8(a: int8x8_t, b: int8x16_t) -> int8x8_t { + static_assert_imm3!(LANE1); + static_assert_imm4!(LANE2); + let a: int8x16_t = simd_shuffle16!(a, a, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]); + match LANE1 & 0b111 { + 0 => simd_shuffle8!(a, b, [16 + LANE2 as u32, 1, 2, 3, 4, 5, 6, 7]), + 1 => simd_shuffle8!(a, b, [0, 16 + LANE2 as u32, 2, 3, 4, 5, 6, 7]), + 2 => simd_shuffle8!(a, b, [0, 1, 16 + LANE2 as u32, 3, 4, 5, 6, 7]), + 3 => simd_shuffle8!(a, b, [0, 1, 2, 16 + LANE2 as u32, 4, 5, 6, 7]), + 4 => simd_shuffle8!(a, b, [0, 1, 2, 3, 16 + LANE2 as u32, 5, 6, 7]), + 5 => simd_shuffle8!(a, b, [0, 1, 2, 3, 4, 16 + LANE2 as u32, 6, 7]), + 6 => simd_shuffle8!(a, b, [0, 1, 2, 3, 4, 5, 16 + LANE2 as u32, 7]), + 7 => simd_shuffle8!(a, b, [0, 1, 2, 3, 4, 5, 6, 16 + LANE2 as u32]), + _ => unreachable_unchecked(), + } +} + +/// Insert vector element from another vector element +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(mov, LANE1 = 0, LANE2 = 1))] +#[rustc_legacy_const_generics(1, 3)] +pub unsafe fn vcopy_laneq_s16(a: int16x4_t, b: int16x8_t) -> int16x4_t { + static_assert_imm2!(LANE1); + static_assert_imm3!(LANE2); + let a: int16x8_t = simd_shuffle8!(a, a, [0, 1, 2, 3, 4, 5, 6, 7]); + match LANE1 & 0b11 { + 0 => simd_shuffle4!(a, b, [8 + LANE2 as u32, 1, 2, 3]), + 1 => simd_shuffle4!(a, b, [0, 8 + LANE2 as u32, 2, 3]), + 2 => simd_shuffle4!(a, b, [0, 1, 8 + LANE2 as u32, 3]), + 3 => simd_shuffle4!(a, b, [0, 1, 2, 8 + LANE2 as u32]), + _ => unreachable_unchecked(), + } +} + +/// Insert vector element from another vector element +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(mov, LANE1 = 0, LANE2 = 1))] +#[rustc_legacy_const_generics(1, 3)] +pub unsafe fn vcopy_laneq_s32(a: int32x2_t, b: int32x4_t) -> int32x2_t { + static_assert_imm1!(LANE1); + static_assert_imm2!(LANE2); + let a: int32x4_t = simd_shuffle4!(a, a, [0, 1, 2, 3]); + match LANE1 & 0b1 { + 0 => simd_shuffle2!(a, b, [4 + LANE2 as u32, 1]), + 1 => simd_shuffle2!(a, b, [0, 4 + LANE2 as u32]), + _ => unreachable_unchecked(), + } +} + +/// Insert vector element from another vector element +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(mov, LANE1 = 0, LANE2 = 1))] +#[rustc_legacy_const_generics(1, 3)] +pub unsafe fn vcopy_laneq_u8(a: uint8x8_t, b: uint8x16_t) -> uint8x8_t { + static_assert_imm3!(LANE1); + static_assert_imm4!(LANE2); + let a: uint8x16_t = simd_shuffle16!(a, a, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]); + match LANE1 & 0b111 { + 0 => simd_shuffle8!(a, b, [16 + LANE2 as u32, 1, 2, 3, 4, 5, 6, 7]), + 1 => simd_shuffle8!(a, b, [0, 16 + LANE2 as u32, 2, 3, 4, 5, 6, 7]), + 2 => simd_shuffle8!(a, b, [0, 1, 16 + LANE2 as u32, 3, 4, 5, 6, 7]), + 3 => simd_shuffle8!(a, b, [0, 1, 2, 16 + LANE2 as u32, 4, 5, 6, 7]), + 4 => simd_shuffle8!(a, b, [0, 1, 2, 3, 16 + LANE2 as u32, 5, 6, 7]), + 5 => simd_shuffle8!(a, b, [0, 1, 2, 3, 4, 16 + LANE2 as u32, 6, 7]), + 6 => simd_shuffle8!(a, b, [0, 1, 2, 3, 4, 5, 16 + LANE2 as u32, 7]), + 7 => simd_shuffle8!(a, b, [0, 1, 2, 3, 4, 5, 6, 16 + LANE2 as u32]), + _ => unreachable_unchecked(), + } +} + +/// Insert vector element from another vector element +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(mov, LANE1 = 0, LANE2 = 1))] +#[rustc_legacy_const_generics(1, 3)] +pub unsafe fn vcopy_laneq_u16(a: uint16x4_t, b: uint16x8_t) -> uint16x4_t { + static_assert_imm2!(LANE1); + static_assert_imm3!(LANE2); + let a: uint16x8_t = simd_shuffle8!(a, a, [0, 1, 2, 3, 4, 5, 6, 7]); + match LANE1 & 0b11 { + 0 => simd_shuffle4!(a, b, [8 + LANE2 as u32, 1, 2, 3]), + 1 => simd_shuffle4!(a, b, [0, 8 + LANE2 as u32, 2, 3]), + 2 => simd_shuffle4!(a, b, [0, 1, 8 + LANE2 as u32, 3]), + 3 => simd_shuffle4!(a, b, [0, 1, 2, 8 + LANE2 as u32]), + _ => unreachable_unchecked(), + } +} + +/// Insert vector element from another vector element +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(mov, LANE1 = 0, LANE2 = 1))] +#[rustc_legacy_const_generics(1, 3)] +pub unsafe fn vcopy_laneq_u32(a: uint32x2_t, b: uint32x4_t) -> uint32x2_t { + static_assert_imm1!(LANE1); + static_assert_imm2!(LANE2); + let a: uint32x4_t = simd_shuffle4!(a, a, [0, 1, 2, 3]); + match LANE1 & 0b1 { + 0 => simd_shuffle2!(a, b, [4 + LANE2 as u32, 1]), + 1 => simd_shuffle2!(a, b, [0, 4 + LANE2 as u32]), + _ => unreachable_unchecked(), + } +} + +/// Insert vector element from another vector element +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(mov, LANE1 = 0, LANE2 = 1))] +#[rustc_legacy_const_generics(1, 3)] +pub unsafe fn vcopy_laneq_p8(a: poly8x8_t, b: poly8x16_t) -> poly8x8_t { + static_assert_imm3!(LANE1); + static_assert_imm4!(LANE2); + let a: poly8x16_t = simd_shuffle16!(a, a, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]); + match LANE1 & 0b111 { + 0 => simd_shuffle8!(a, b, [16 + LANE2 as u32, 1, 2, 3, 4, 5, 6, 7]), + 1 => simd_shuffle8!(a, b, [0, 16 + LANE2 as u32, 2, 3, 4, 5, 6, 7]), + 2 => simd_shuffle8!(a, b, [0, 1, 16 + LANE2 as u32, 3, 4, 5, 6, 7]), + 3 => simd_shuffle8!(a, b, [0, 1, 2, 16 + LANE2 as u32, 4, 5, 6, 7]), + 4 => simd_shuffle8!(a, b, [0, 1, 2, 3, 16 + LANE2 as u32, 5, 6, 7]), + 5 => simd_shuffle8!(a, b, [0, 1, 2, 3, 4, 16 + LANE2 as u32, 6, 7]), + 6 => simd_shuffle8!(a, b, [0, 1, 2, 3, 4, 5, 16 + LANE2 as u32, 7]), + 7 => simd_shuffle8!(a, b, [0, 1, 2, 3, 4, 5, 6, 16 + LANE2 as u32]), + _ => unreachable_unchecked(), + } +} + +/// Insert vector element from another vector element +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(mov, LANE1 = 0, LANE2 = 1))] +#[rustc_legacy_const_generics(1, 3)] +pub unsafe fn vcopy_laneq_p16(a: poly16x4_t, b: poly16x8_t) -> poly16x4_t { + static_assert_imm2!(LANE1); + static_assert_imm3!(LANE2); + let a: poly16x8_t = simd_shuffle8!(a, a, [0, 1, 2, 3, 4, 5, 6, 7]); + match LANE1 & 0b11 { + 0 => simd_shuffle4!(a, b, [8 + LANE2 as u32, 1, 2, 3]), + 1 => simd_shuffle4!(a, b, [0, 8 + LANE2 as u32, 2, 3]), + 2 => simd_shuffle4!(a, b, [0, 1, 8 + LANE2 as u32, 3]), + 3 => simd_shuffle4!(a, b, [0, 1, 2, 8 + LANE2 as u32]), + _ => unreachable_unchecked(), + } +} + +/// Insert vector element from another vector element +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(mov, LANE1 = 0, LANE2 = 1))] +#[rustc_legacy_const_generics(1, 3)] +pub unsafe fn vcopy_laneq_f32(a: float32x2_t, b: float32x4_t) -> float32x2_t { + static_assert_imm1!(LANE1); + static_assert_imm2!(LANE2); + let a: float32x4_t = simd_shuffle4!(a, a, [0, 1, 2, 3]); + match LANE1 & 0b1 { + 0 => simd_shuffle2!(a, b, [4 + LANE2 as u32, 1]), + 1 => simd_shuffle2!(a, b, [0, 4 + LANE2 as u32]), + _ => unreachable_unchecked(), + } +} + +/// Insert vector element from another vector element +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(mov, LANE1 = 0, LANE2 = 1))] +#[rustc_legacy_const_generics(1, 3)] +pub unsafe fn vcopyq_lane_s8(a: int8x16_t, b: int8x8_t) -> int8x16_t { + static_assert_imm4!(LANE1); + static_assert_imm3!(LANE2); + let b: int8x16_t = simd_shuffle16!(b, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]); + match LANE1 & 0b1111 { + 0 => simd_shuffle16!(a, b, [16 + LANE2 as u32, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), + 1 => simd_shuffle16!(a, b, [0, 16 + LANE2 as u32, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), + 2 => simd_shuffle16!(a, b, [0, 1, 16 + LANE2 as u32, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), + 3 => simd_shuffle16!(a, b, [0, 1, 2, 16 + LANE2 as u32, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), + 4 => simd_shuffle16!(a, b, [0, 1, 2, 3, 16 + LANE2 as u32, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), + 5 => simd_shuffle16!(a, b, [0, 1, 2, 3, 4, 16 + LANE2 as u32, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), + 6 => simd_shuffle16!(a, b, [0, 1, 2, 3, 4, 5, 16 + LANE2 as u32, 7, 8, 9, 10, 11, 12, 13, 14, 15]), + 7 => simd_shuffle16!(a, b, [0, 1, 2, 3, 4, 5, 6, 16 + LANE2 as u32, 8, 9, 10, 11, 12, 13, 14, 15]), + 8 => simd_shuffle16!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 16 + LANE2 as u32, 9, 10, 11, 12, 13, 14, 15]), + 9 => simd_shuffle16!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 16 + LANE2 as u32, 10, 11, 12, 13, 14, 15]), + 10 => simd_shuffle16!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 16 + LANE2 as u32, 11, 12, 13, 14, 15]), + 11 => simd_shuffle16!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 16 + LANE2 as u32, 12, 13, 14, 15]), + 12 => simd_shuffle16!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 16 + LANE2 as u32, 13, 14, 15]), + 13 => simd_shuffle16!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 16 + LANE2 as u32, 14, 15]), + 14 => simd_shuffle16!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 16 + LANE2 as u32, 15]), + 15 => simd_shuffle16!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 16 + LANE2 as u32]), + _ => unreachable_unchecked(), + } +} + +/// Insert vector element from another vector element +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(mov, LANE1 = 0, LANE2 = 1))] +#[rustc_legacy_const_generics(1, 3)] +pub unsafe fn vcopyq_lane_s16(a: int16x8_t, b: int16x4_t) -> int16x8_t { + static_assert_imm3!(LANE1); + static_assert_imm2!(LANE2); + let b: int16x8_t = simd_shuffle8!(b, b, [0, 1, 2, 3, 4, 5, 6, 7]); + match LANE1 & 0b111 { + 0 => simd_shuffle8!(a, b, [8 + LANE2 as u32, 1, 2, 3, 4, 5, 6, 7]), + 1 => simd_shuffle8!(a, b, [0, 8 + LANE2 as u32, 2, 3, 4, 5, 6, 7]), + 2 => simd_shuffle8!(a, b, [0, 1, 8 + LANE2 as u32, 3, 4, 5, 6, 7]), + 3 => simd_shuffle8!(a, b, [0, 1, 2, 8 + LANE2 as u32, 4, 5, 6, 7]), + 4 => simd_shuffle8!(a, b, [0, 1, 2, 3, 8 + LANE2 as u32, 5, 6, 7]), + 5 => simd_shuffle8!(a, b, [0, 1, 2, 3, 4, 8 + LANE2 as u32, 6, 7]), + 6 => simd_shuffle8!(a, b, [0, 1, 2, 3, 4, 5, 8 + LANE2 as u32, 7]), + 7 => simd_shuffle8!(a, b, [0, 1, 2, 3, 4, 5, 6, 8 + LANE2 as u32]), + _ => unreachable_unchecked(), + } +} + +/// Insert vector element from another vector element +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(mov, LANE1 = 0, LANE2 = 1))] +#[rustc_legacy_const_generics(1, 3)] +pub unsafe fn vcopyq_lane_s32(a: int32x4_t, b: int32x2_t) -> int32x4_t { + static_assert_imm2!(LANE1); + static_assert_imm1!(LANE2); + let b: int32x4_t = simd_shuffle4!(b, b, [0, 1, 2, 3]); + match LANE1 & 0b11 { + 0 => simd_shuffle4!(a, b, [4 + LANE2 as u32, 1, 2, 3]), + 1 => simd_shuffle4!(a, b, [0, 4 + LANE2 as u32, 2, 3]), + 2 => simd_shuffle4!(a, b, [0, 1, 4 + LANE2 as u32, 3]), + 3 => simd_shuffle4!(a, b, [0, 1, 2, 4 + LANE2 as u32]), + _ => unreachable_unchecked(), + } +} + +/// Insert vector element from another vector element +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(mov, LANE1 = 0, LANE2 = 1))] +#[rustc_legacy_const_generics(1, 3)] +pub unsafe fn vcopyq_lane_u8(a: uint8x16_t, b: uint8x8_t) -> uint8x16_t { + static_assert_imm4!(LANE1); + static_assert_imm3!(LANE2); + let b: uint8x16_t = simd_shuffle16!(b, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]); + match LANE1 & 0b1111 { + 0 => simd_shuffle16!(a, b, [16 + LANE2 as u32, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), + 1 => simd_shuffle16!(a, b, [0, 16 + LANE2 as u32, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), + 2 => simd_shuffle16!(a, b, [0, 1, 16 + LANE2 as u32, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), + 3 => simd_shuffle16!(a, b, [0, 1, 2, 16 + LANE2 as u32, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), + 4 => simd_shuffle16!(a, b, [0, 1, 2, 3, 16 + LANE2 as u32, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), + 5 => simd_shuffle16!(a, b, [0, 1, 2, 3, 4, 16 + LANE2 as u32, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), + 6 => simd_shuffle16!(a, b, [0, 1, 2, 3, 4, 5, 16 + LANE2 as u32, 7, 8, 9, 10, 11, 12, 13, 14, 15]), + 7 => simd_shuffle16!(a, b, [0, 1, 2, 3, 4, 5, 6, 16 + LANE2 as u32, 8, 9, 10, 11, 12, 13, 14, 15]), + 8 => simd_shuffle16!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 16 + LANE2 as u32, 9, 10, 11, 12, 13, 14, 15]), + 9 => simd_shuffle16!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 16 + LANE2 as u32, 10, 11, 12, 13, 14, 15]), + 10 => simd_shuffle16!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 16 + LANE2 as u32, 11, 12, 13, 14, 15]), + 11 => simd_shuffle16!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 16 + LANE2 as u32, 12, 13, 14, 15]), + 12 => simd_shuffle16!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 16 + LANE2 as u32, 13, 14, 15]), + 13 => simd_shuffle16!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 16 + LANE2 as u32, 14, 15]), + 14 => simd_shuffle16!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 16 + LANE2 as u32, 15]), + 15 => simd_shuffle16!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 16 + LANE2 as u32]), + _ => unreachable_unchecked(), + } +} + +/// Insert vector element from another vector element +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(mov, LANE1 = 0, LANE2 = 1))] +#[rustc_legacy_const_generics(1, 3)] +pub unsafe fn vcopyq_lane_u16(a: uint16x8_t, b: uint16x4_t) -> uint16x8_t { + static_assert_imm3!(LANE1); + static_assert_imm2!(LANE2); + let b: uint16x8_t = simd_shuffle8!(b, b, [0, 1, 2, 3, 4, 5, 6, 7]); + match LANE1 & 0b111 { + 0 => simd_shuffle8!(a, b, [8 + LANE2 as u32, 1, 2, 3, 4, 5, 6, 7]), + 1 => simd_shuffle8!(a, b, [0, 8 + LANE2 as u32, 2, 3, 4, 5, 6, 7]), + 2 => simd_shuffle8!(a, b, [0, 1, 8 + LANE2 as u32, 3, 4, 5, 6, 7]), + 3 => simd_shuffle8!(a, b, [0, 1, 2, 8 + LANE2 as u32, 4, 5, 6, 7]), + 4 => simd_shuffle8!(a, b, [0, 1, 2, 3, 8 + LANE2 as u32, 5, 6, 7]), + 5 => simd_shuffle8!(a, b, [0, 1, 2, 3, 4, 8 + LANE2 as u32, 6, 7]), + 6 => simd_shuffle8!(a, b, [0, 1, 2, 3, 4, 5, 8 + LANE2 as u32, 7]), + 7 => simd_shuffle8!(a, b, [0, 1, 2, 3, 4, 5, 6, 8 + LANE2 as u32]), + _ => unreachable_unchecked(), + } +} + +/// Insert vector element from another vector element +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(mov, LANE1 = 0, LANE2 = 1))] +#[rustc_legacy_const_generics(1, 3)] +pub unsafe fn vcopyq_lane_u32(a: uint32x4_t, b: uint32x2_t) -> uint32x4_t { + static_assert_imm2!(LANE1); + static_assert_imm1!(LANE2); + let b: uint32x4_t = simd_shuffle4!(b, b, [0, 1, 2, 3]); + match LANE1 & 0b11 { + 0 => simd_shuffle4!(a, b, [4 + LANE2 as u32, 1, 2, 3]), + 1 => simd_shuffle4!(a, b, [0, 4 + LANE2 as u32, 2, 3]), + 2 => simd_shuffle4!(a, b, [0, 1, 4 + LANE2 as u32, 3]), + 3 => simd_shuffle4!(a, b, [0, 1, 2, 4 + LANE2 as u32]), + _ => unreachable_unchecked(), + } +} + +/// Insert vector element from another vector element +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(mov, LANE1 = 0, LANE2 = 1))] +#[rustc_legacy_const_generics(1, 3)] +pub unsafe fn vcopyq_lane_p8(a: poly8x16_t, b: poly8x8_t) -> poly8x16_t { + static_assert_imm4!(LANE1); + static_assert_imm3!(LANE2); + let b: poly8x16_t = simd_shuffle16!(b, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]); + match LANE1 & 0b1111 { + 0 => simd_shuffle16!(a, b, [16 + LANE2 as u32, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), + 1 => simd_shuffle16!(a, b, [0, 16 + LANE2 as u32, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), + 2 => simd_shuffle16!(a, b, [0, 1, 16 + LANE2 as u32, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), + 3 => simd_shuffle16!(a, b, [0, 1, 2, 16 + LANE2 as u32, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), + 4 => simd_shuffle16!(a, b, [0, 1, 2, 3, 16 + LANE2 as u32, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), + 5 => simd_shuffle16!(a, b, [0, 1, 2, 3, 4, 16 + LANE2 as u32, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), + 6 => simd_shuffle16!(a, b, [0, 1, 2, 3, 4, 5, 16 + LANE2 as u32, 7, 8, 9, 10, 11, 12, 13, 14, 15]), + 7 => simd_shuffle16!(a, b, [0, 1, 2, 3, 4, 5, 6, 16 + LANE2 as u32, 8, 9, 10, 11, 12, 13, 14, 15]), + 8 => simd_shuffle16!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 16 + LANE2 as u32, 9, 10, 11, 12, 13, 14, 15]), + 9 => simd_shuffle16!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 16 + LANE2 as u32, 10, 11, 12, 13, 14, 15]), + 10 => simd_shuffle16!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 16 + LANE2 as u32, 11, 12, 13, 14, 15]), + 11 => simd_shuffle16!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 16 + LANE2 as u32, 12, 13, 14, 15]), + 12 => simd_shuffle16!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 16 + LANE2 as u32, 13, 14, 15]), + 13 => simd_shuffle16!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 16 + LANE2 as u32, 14, 15]), + 14 => simd_shuffle16!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 16 + LANE2 as u32, 15]), + 15 => simd_shuffle16!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 16 + LANE2 as u32]), + _ => unreachable_unchecked(), + } +} + +/// Insert vector element from another vector element +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(mov, LANE1 = 0, LANE2 = 1))] +#[rustc_legacy_const_generics(1, 3)] +pub unsafe fn vcopyq_lane_p16(a: poly16x8_t, b: poly16x4_t) -> poly16x8_t { + static_assert_imm3!(LANE1); + static_assert_imm2!(LANE2); + let b: poly16x8_t = simd_shuffle8!(b, b, [0, 1, 2, 3, 4, 5, 6, 7]); + match LANE1 & 0b111 { + 0 => simd_shuffle8!(a, b, [8 + LANE2 as u32, 1, 2, 3, 4, 5, 6, 7]), + 1 => simd_shuffle8!(a, b, [0, 8 + LANE2 as u32, 2, 3, 4, 5, 6, 7]), + 2 => simd_shuffle8!(a, b, [0, 1, 8 + LANE2 as u32, 3, 4, 5, 6, 7]), + 3 => simd_shuffle8!(a, b, [0, 1, 2, 8 + LANE2 as u32, 4, 5, 6, 7]), + 4 => simd_shuffle8!(a, b, [0, 1, 2, 3, 8 + LANE2 as u32, 5, 6, 7]), + 5 => simd_shuffle8!(a, b, [0, 1, 2, 3, 4, 8 + LANE2 as u32, 6, 7]), + 6 => simd_shuffle8!(a, b, [0, 1, 2, 3, 4, 5, 8 + LANE2 as u32, 7]), + 7 => simd_shuffle8!(a, b, [0, 1, 2, 3, 4, 5, 6, 8 + LANE2 as u32]), + _ => unreachable_unchecked(), + } +} + +/// Insert vector element from another vector element +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(zip1, LANE1 = 1, LANE2 = 0))] +#[rustc_legacy_const_generics(1, 3)] +pub unsafe fn vcopyq_lane_s64(a: int64x2_t, b: int64x1_t) -> int64x2_t { + static_assert_imm1!(LANE1); + static_assert!(LANE2 : i32 where LANE2 == 0); + let b: int64x2_t = simd_shuffle2!(b, b, [0, 1]); + match LANE1 & 0b1 { + 0 => simd_shuffle2!(a, b, [2 + LANE2 as u32, 1]), + 1 => simd_shuffle2!(a, b, [0, 2 + LANE2 as u32]), + _ => unreachable_unchecked(), + } +} + +/// Insert vector element from another vector element +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(zip1, LANE1 = 1, LANE2 = 0))] +#[rustc_legacy_const_generics(1, 3)] +pub unsafe fn vcopyq_lane_u64(a: uint64x2_t, b: uint64x1_t) -> uint64x2_t { + static_assert_imm1!(LANE1); + static_assert!(LANE2 : i32 where LANE2 == 0); + let b: uint64x2_t = simd_shuffle2!(b, b, [0, 1]); + match LANE1 & 0b1 { + 0 => simd_shuffle2!(a, b, [2 + LANE2 as u32, 1]), + 1 => simd_shuffle2!(a, b, [0, 2 + LANE2 as u32]), + _ => unreachable_unchecked(), + } +} + +/// Insert vector element from another vector element +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(zip1, LANE1 = 1, LANE2 = 0))] +#[rustc_legacy_const_generics(1, 3)] +pub unsafe fn vcopyq_lane_p64(a: poly64x2_t, b: poly64x1_t) -> poly64x2_t { + static_assert_imm1!(LANE1); + static_assert!(LANE2 : i32 where LANE2 == 0); + let b: poly64x2_t = simd_shuffle2!(b, b, [0, 1]); + match LANE1 & 0b1 { + 0 => simd_shuffle2!(a, b, [2 + LANE2 as u32, 1]), + 1 => simd_shuffle2!(a, b, [0, 2 + LANE2 as u32]), + _ => unreachable_unchecked(), + } +} + +/// Insert vector element from another vector element +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(mov, LANE1 = 1, LANE2 = 0))] +#[rustc_legacy_const_generics(1, 3)] +pub unsafe fn vcopyq_lane_f32(a: float32x4_t, b: float32x2_t) -> float32x4_t { + static_assert_imm2!(LANE1); + static_assert_imm1!(LANE2); + let b: float32x4_t = simd_shuffle4!(b, b, [0, 1, 2, 3]); + match LANE1 & 0b11 { + 0 => simd_shuffle4!(a, b, [4 + LANE2 as u32, 1, 2, 3]), + 1 => simd_shuffle4!(a, b, [0, 4 + LANE2 as u32, 2, 3]), + 2 => simd_shuffle4!(a, b, [0, 1, 4 + LANE2 as u32, 3]), + 3 => simd_shuffle4!(a, b, [0, 1, 2, 4 + LANE2 as u32]), + _ => unreachable_unchecked(), + } +} + +/// Insert vector element from another vector element +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(zip1, LANE1 = 1, LANE2 = 0))] +#[rustc_legacy_const_generics(1, 3)] +pub unsafe fn vcopyq_lane_f64(a: float64x2_t, b: float64x1_t) -> float64x2_t { + static_assert_imm1!(LANE1); + static_assert!(LANE2 : i32 where LANE2 == 0); + let b: float64x2_t = simd_shuffle2!(b, b, [0, 1]); + match LANE1 & 0b1 { + 0 => simd_shuffle2!(a, b, [2 + LANE2 as u32, 1]), + 1 => simd_shuffle2!(a, b, [0, 2 + LANE2 as u32]), + _ => unreachable_unchecked(), + } +} + +/// Insert vector element from another vector element +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop))] +pub unsafe fn vcreate_f64(a: u64) -> float64x1_t { + transmute(a) +} + +/// Fixed-point convert to floating-point +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(scvtf))] +pub unsafe fn vcvt_f64_s64(a: int64x1_t) -> float64x1_t { + simd_cast(a) +} + +/// Fixed-point convert to floating-point +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(scvtf))] +pub unsafe fn vcvtq_f64_s64(a: int64x2_t) -> float64x2_t { + simd_cast(a) +} + +/// Fixed-point convert to floating-point +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ucvtf))] +pub unsafe fn vcvt_f64_u64(a: uint64x1_t) -> float64x1_t { + simd_cast(a) +} + +/// Fixed-point convert to floating-point +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ucvtf))] +pub unsafe fn vcvtq_f64_u64(a: uint64x2_t) -> float64x2_t { + simd_cast(a) +} + +/// Floating-point convert to higher precision long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcvtl))] +pub unsafe fn vcvt_f64_f32(a: float32x2_t) -> float64x2_t { + simd_cast(a) +} + +/// Floating-point convert to higher precision long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcvtl))] +pub unsafe fn vcvt_high_f64_f32(a: float32x4_t) -> float64x2_t { + let b: float32x2_t = simd_shuffle2!(a, a, [2, 3]); + simd_cast(b) +} + +/// Floating-point convert to lower precision narrow +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcvtn))] +pub unsafe fn vcvt_f32_f64(a: float64x2_t) -> float32x2_t { + simd_cast(a) +} + +/// Floating-point convert to lower precision narrow +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcvtn))] +pub unsafe fn vcvt_high_f32_f64(a: float32x2_t, b: float64x2_t) -> float32x4_t { + simd_shuffle4!(a, simd_cast(b), [0, 1, 2, 3]) +} + +/// Floating-point convert to lower precision narrow, rounding to odd +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcvtxn))] +pub unsafe fn vcvtx_f32_f64(a: float64x2_t) -> float32x2_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fcvtxn.v2f32.v2f64")] + fn vcvtx_f32_f64_(a: float64x2_t) -> float32x2_t; + } + vcvtx_f32_f64_(a) +} + +/// Floating-point convert to lower precision narrow, rounding to odd +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcvtxn))] +pub unsafe fn vcvtx_high_f32_f64(a: float32x2_t, b: float64x2_t) -> float32x4_t { + simd_shuffle4!(a, vcvtx_f32_f64(b), [0, 1, 2, 3]) +} + +/// Fixed-point convert to floating-point +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(scvtf, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vcvt_n_f64_s64(a: int64x1_t) -> float64x1_t { + static_assert!(N : i32 where N >= 1 && N <= 64); + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.vcvtfxs2fp.v1f64.v1i64")] + fn vcvt_n_f64_s64_(a: int64x1_t, n: i32) -> float64x1_t; + } + vcvt_n_f64_s64_(a, N) +} + +/// Fixed-point convert to floating-point +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(scvtf, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vcvtq_n_f64_s64(a: int64x2_t) -> float64x2_t { + static_assert!(N : i32 where N >= 1 && N <= 64); + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.vcvtfxs2fp.v2f64.v2i64")] + fn vcvtq_n_f64_s64_(a: int64x2_t, n: i32) -> float64x2_t; + } + vcvtq_n_f64_s64_(a, N) +} + +/// Fixed-point convert to floating-point +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(scvtf, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vcvts_n_f32_s32(a: i32) -> f32 { + static_assert!(N : i32 where N >= 1 && N <= 32); + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.vcvtfxs2fp.f32.i32")] + fn vcvts_n_f32_s32_(a: i32, n: i32) -> f32; + } + vcvts_n_f32_s32_(a, N) +} + +/// Fixed-point convert to floating-point +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(scvtf, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vcvtd_n_f64_s64(a: i64) -> f64 { + static_assert!(N : i32 where N >= 1 && N <= 64); + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.vcvtfxs2fp.f64.i64")] + fn vcvtd_n_f64_s64_(a: i64, n: i32) -> f64; + } + vcvtd_n_f64_s64_(a, N) +} + +/// Fixed-point convert to floating-point +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ucvtf, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vcvt_n_f64_u64(a: uint64x1_t) -> float64x1_t { + static_assert!(N : i32 where N >= 1 && N <= 64); + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.vcvtfxu2fp.v1f64.v1i64")] + fn vcvt_n_f64_u64_(a: uint64x1_t, n: i32) -> float64x1_t; + } + vcvt_n_f64_u64_(a, N) +} + +/// Fixed-point convert to floating-point +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ucvtf, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vcvtq_n_f64_u64(a: uint64x2_t) -> float64x2_t { + static_assert!(N : i32 where N >= 1 && N <= 64); + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.vcvtfxu2fp.v2f64.v2i64")] + fn vcvtq_n_f64_u64_(a: uint64x2_t, n: i32) -> float64x2_t; + } + vcvtq_n_f64_u64_(a, N) +} + +/// Fixed-point convert to floating-point +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ucvtf, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vcvts_n_f32_u32(a: u32) -> f32 { + static_assert!(N : i32 where N >= 1 && N <= 32); + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.vcvtfxu2fp.f32.i32")] + fn vcvts_n_f32_u32_(a: u32, n: i32) -> f32; + } + vcvts_n_f32_u32_(a, N) +} + +/// Fixed-point convert to floating-point +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ucvtf, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vcvtd_n_f64_u64(a: u64) -> f64 { + static_assert!(N : i32 where N >= 1 && N <= 64); + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.vcvtfxu2fp.f64.i64")] + fn vcvtd_n_f64_u64_(a: u64, n: i32) -> f64; + } + vcvtd_n_f64_u64_(a, N) +} + +/// Floating-point convert to fixed-point, rounding toward zero +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcvtzs, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vcvt_n_s64_f64(a: float64x1_t) -> int64x1_t { + static_assert!(N : i32 where N >= 1 && N <= 64); + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.vcvtfp2fxs.v1i64.v1f64")] + fn vcvt_n_s64_f64_(a: float64x1_t, n: i32) -> int64x1_t; + } + vcvt_n_s64_f64_(a, N) +} + +/// Floating-point convert to fixed-point, rounding toward zero +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcvtzs, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vcvtq_n_s64_f64(a: float64x2_t) -> int64x2_t { + static_assert!(N : i32 where N >= 1 && N <= 64); + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.vcvtfp2fxs.v2i64.v2f64")] + fn vcvtq_n_s64_f64_(a: float64x2_t, n: i32) -> int64x2_t; + } + vcvtq_n_s64_f64_(a, N) +} + +/// Floating-point convert to fixed-point, rounding toward zero +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcvtzs, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vcvts_n_s32_f32(a: f32) -> i32 { + static_assert!(N : i32 where N >= 1 && N <= 32); + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.vcvtfp2fxs.i32.f32")] + fn vcvts_n_s32_f32_(a: f32, n: i32) -> i32; + } + vcvts_n_s32_f32_(a, N) +} + +/// Floating-point convert to fixed-point, rounding toward zero +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcvtzs, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vcvtd_n_s64_f64(a: f64) -> i64 { + static_assert!(N : i32 where N >= 1 && N <= 64); + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.vcvtfp2fxs.i64.f64")] + fn vcvtd_n_s64_f64_(a: f64, n: i32) -> i64; + } + vcvtd_n_s64_f64_(a, N) +} + +/// Floating-point convert to fixed-point, rounding toward zero +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcvtzu, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vcvt_n_u64_f64(a: float64x1_t) -> uint64x1_t { + static_assert!(N : i32 where N >= 1 && N <= 64); + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.vcvtfp2fxu.v1i64.v1f64")] + fn vcvt_n_u64_f64_(a: float64x1_t, n: i32) -> uint64x1_t; + } + vcvt_n_u64_f64_(a, N) +} + +/// Floating-point convert to fixed-point, rounding toward zero +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcvtzu, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vcvtq_n_u64_f64(a: float64x2_t) -> uint64x2_t { + static_assert!(N : i32 where N >= 1 && N <= 64); + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.vcvtfp2fxu.v2i64.v2f64")] + fn vcvtq_n_u64_f64_(a: float64x2_t, n: i32) -> uint64x2_t; + } + vcvtq_n_u64_f64_(a, N) +} + +/// Floating-point convert to fixed-point, rounding toward zero +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcvtzu, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vcvts_n_u32_f32(a: f32) -> u32 { + static_assert!(N : i32 where N >= 1 && N <= 32); + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.vcvtfp2fxu.i32.f32")] + fn vcvts_n_u32_f32_(a: f32, n: i32) -> u32; + } + vcvts_n_u32_f32_(a, N) +} + +/// Floating-point convert to fixed-point, rounding toward zero +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcvtzu, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vcvtd_n_u64_f64(a: f64) -> u64 { + static_assert!(N : i32 where N >= 1 && N <= 64); + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.vcvtfp2fxu.i64.f64")] + fn vcvtd_n_u64_f64_(a: f64, n: i32) -> u64; + } + vcvtd_n_u64_f64_(a, N) +} + +/// Fixed-point convert to floating-point +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(scvtf))] +pub unsafe fn vcvts_f32_s32(a: i32) -> f32 { + a as f32 +} + +/// Fixed-point convert to floating-point +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(scvtf))] +pub unsafe fn vcvtd_f64_s64(a: i64) -> f64 { + a as f64 +} + +/// Fixed-point convert to floating-point +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ucvtf))] +pub unsafe fn vcvts_f32_u32(a: u32) -> f32 { + a as f32 +} + +/// Fixed-point convert to floating-point +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ucvtf))] +pub unsafe fn vcvtd_f64_u64(a: u64) -> f64 { + a as f64 +} + +/// Fixed-point convert to floating-point +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcvtzs))] +pub unsafe fn vcvts_s32_f32(a: f32) -> i32 { + a as i32 +} + +/// Fixed-point convert to floating-point +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcvtzs))] +pub unsafe fn vcvtd_s64_f64(a: f64) -> i64 { + a as i64 +} + +/// Fixed-point convert to floating-point +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcvtzu))] +pub unsafe fn vcvts_u32_f32(a: f32) -> u32 { + a as u32 +} + +/// Fixed-point convert to floating-point +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcvtzu))] +pub unsafe fn vcvtd_u64_f64(a: f64) -> u64 { + a as u64 +} + +/// Floating-point convert to signed fixed-point, rounding toward zero +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcvtzs))] +pub unsafe fn vcvt_s64_f64(a: float64x1_t) -> int64x1_t { + simd_cast(a) +} + +/// Floating-point convert to signed fixed-point, rounding toward zero +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcvtzs))] +pub unsafe fn vcvtq_s64_f64(a: float64x2_t) -> int64x2_t { + simd_cast(a) +} + +/// Floating-point convert to unsigned fixed-point, rounding toward zero +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcvtzu))] +pub unsafe fn vcvt_u64_f64(a: float64x1_t) -> uint64x1_t { + simd_cast(a) +} + +/// Floating-point convert to unsigned fixed-point, rounding toward zero +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcvtzu))] +pub unsafe fn vcvtq_u64_f64(a: float64x2_t) -> uint64x2_t { + simd_cast(a) +} + +/// Floating-point convert to signed integer, rounding to nearest with ties to away +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcvtas))] +pub unsafe fn vcvta_s32_f32(a: float32x2_t) -> int32x2_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fcvtas.v2i32.v2f32")] + fn vcvta_s32_f32_(a: float32x2_t) -> int32x2_t; + } + vcvta_s32_f32_(a) +} + +/// Floating-point convert to signed integer, rounding to nearest with ties to away +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcvtas))] +pub unsafe fn vcvtaq_s32_f32(a: float32x4_t) -> int32x4_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fcvtas.v4i32.v4f32")] + fn vcvtaq_s32_f32_(a: float32x4_t) -> int32x4_t; + } + vcvtaq_s32_f32_(a) +} + +/// Floating-point convert to signed integer, rounding to nearest with ties to away +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcvtas))] +pub unsafe fn vcvta_s64_f64(a: float64x1_t) -> int64x1_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fcvtas.v1i64.v1f64")] + fn vcvta_s64_f64_(a: float64x1_t) -> int64x1_t; + } + vcvta_s64_f64_(a) +} + +/// Floating-point convert to signed integer, rounding to nearest with ties to away +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcvtas))] +pub unsafe fn vcvtaq_s64_f64(a: float64x2_t) -> int64x2_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fcvtas.v2i64.v2f64")] + fn vcvtaq_s64_f64_(a: float64x2_t) -> int64x2_t; + } + vcvtaq_s64_f64_(a) +} + +/// Floating-point convert to integer, rounding to nearest with ties to away +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcvtas))] +pub unsafe fn vcvtas_s32_f32(a: f32) -> i32 { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fcvtas.i32.f32")] + fn vcvtas_s32_f32_(a: f32) -> i32; + } + vcvtas_s32_f32_(a) +} + +/// Floating-point convert to integer, rounding to nearest with ties to away +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcvtas))] +pub unsafe fn vcvtad_s64_f64(a: f64) -> i64 { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fcvtas.i64.f64")] + fn vcvtad_s64_f64_(a: f64) -> i64; + } + vcvtad_s64_f64_(a) +} + +/// Floating-point convert to integer, rounding to nearest with ties to away +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcvtau))] +pub unsafe fn vcvtas_u32_f32(a: f32) -> u32 { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fcvtau.i32.f32")] + fn vcvtas_u32_f32_(a: f32) -> u32; + } + vcvtas_u32_f32_(a) +} + +/// Floating-point convert to integer, rounding to nearest with ties to away +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcvtau))] +pub unsafe fn vcvtad_u64_f64(a: f64) -> u64 { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fcvtau.i64.f64")] + fn vcvtad_u64_f64_(a: f64) -> u64; + } + vcvtad_u64_f64_(a) +} + +/// Floating-point convert to signed integer, rounding to nearest with ties to even +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcvtns))] +pub unsafe fn vcvtn_s32_f32(a: float32x2_t) -> int32x2_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fcvtns.v2i32.v2f32")] + fn vcvtn_s32_f32_(a: float32x2_t) -> int32x2_t; + } + vcvtn_s32_f32_(a) +} + +/// Floating-point convert to signed integer, rounding to nearest with ties to even +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcvtns))] +pub unsafe fn vcvtnq_s32_f32(a: float32x4_t) -> int32x4_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fcvtns.v4i32.v4f32")] + fn vcvtnq_s32_f32_(a: float32x4_t) -> int32x4_t; + } + vcvtnq_s32_f32_(a) +} + +/// Floating-point convert to signed integer, rounding to nearest with ties to even +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcvtns))] +pub unsafe fn vcvtn_s64_f64(a: float64x1_t) -> int64x1_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fcvtns.v1i64.v1f64")] + fn vcvtn_s64_f64_(a: float64x1_t) -> int64x1_t; + } + vcvtn_s64_f64_(a) +} + +/// Floating-point convert to signed integer, rounding to nearest with ties to even +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcvtns))] +pub unsafe fn vcvtnq_s64_f64(a: float64x2_t) -> int64x2_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fcvtns.v2i64.v2f64")] + fn vcvtnq_s64_f64_(a: float64x2_t) -> int64x2_t; + } + vcvtnq_s64_f64_(a) +} + +/// Floating-point convert to signed integer, rounding to nearest with ties to even +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcvtns))] +pub unsafe fn vcvtns_s32_f32(a: f32) -> i32 { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fcvtns.i32.f32")] + fn vcvtns_s32_f32_(a: f32) -> i32; + } + vcvtns_s32_f32_(a) +} + +/// Floating-point convert to signed integer, rounding to nearest with ties to even +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcvtns))] +pub unsafe fn vcvtnd_s64_f64(a: f64) -> i64 { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fcvtns.i64.f64")] + fn vcvtnd_s64_f64_(a: f64) -> i64; + } + vcvtnd_s64_f64_(a) +} + +/// Floating-point convert to signed integer, rounding toward minus infinity +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcvtms))] +pub unsafe fn vcvtm_s32_f32(a: float32x2_t) -> int32x2_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fcvtms.v2i32.v2f32")] + fn vcvtm_s32_f32_(a: float32x2_t) -> int32x2_t; + } + vcvtm_s32_f32_(a) +} + +/// Floating-point convert to signed integer, rounding toward minus infinity +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcvtms))] +pub unsafe fn vcvtmq_s32_f32(a: float32x4_t) -> int32x4_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fcvtms.v4i32.v4f32")] + fn vcvtmq_s32_f32_(a: float32x4_t) -> int32x4_t; + } + vcvtmq_s32_f32_(a) +} + +/// Floating-point convert to signed integer, rounding toward minus infinity +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcvtms))] +pub unsafe fn vcvtm_s64_f64(a: float64x1_t) -> int64x1_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fcvtms.v1i64.v1f64")] + fn vcvtm_s64_f64_(a: float64x1_t) -> int64x1_t; + } + vcvtm_s64_f64_(a) +} + +/// Floating-point convert to signed integer, rounding toward minus infinity +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcvtms))] +pub unsafe fn vcvtmq_s64_f64(a: float64x2_t) -> int64x2_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fcvtms.v2i64.v2f64")] + fn vcvtmq_s64_f64_(a: float64x2_t) -> int64x2_t; + } + vcvtmq_s64_f64_(a) +} + +/// Floating-point convert to signed integer, rounding toward minus infinity +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcvtms))] +pub unsafe fn vcvtms_s32_f32(a: f32) -> i32 { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fcvtms.i32.f32")] + fn vcvtms_s32_f32_(a: f32) -> i32; + } + vcvtms_s32_f32_(a) +} + +/// Floating-point convert to signed integer, rounding toward minus infinity +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcvtms))] +pub unsafe fn vcvtmd_s64_f64(a: f64) -> i64 { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fcvtms.i64.f64")] + fn vcvtmd_s64_f64_(a: f64) -> i64; + } + vcvtmd_s64_f64_(a) +} + +/// Floating-point convert to signed integer, rounding toward plus infinity +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcvtps))] +pub unsafe fn vcvtp_s32_f32(a: float32x2_t) -> int32x2_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fcvtps.v2i32.v2f32")] + fn vcvtp_s32_f32_(a: float32x2_t) -> int32x2_t; + } + vcvtp_s32_f32_(a) +} + +/// Floating-point convert to signed integer, rounding toward plus infinity +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcvtps))] +pub unsafe fn vcvtpq_s32_f32(a: float32x4_t) -> int32x4_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fcvtps.v4i32.v4f32")] + fn vcvtpq_s32_f32_(a: float32x4_t) -> int32x4_t; + } + vcvtpq_s32_f32_(a) +} + +/// Floating-point convert to signed integer, rounding toward plus infinity +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcvtps))] +pub unsafe fn vcvtp_s64_f64(a: float64x1_t) -> int64x1_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fcvtps.v1i64.v1f64")] + fn vcvtp_s64_f64_(a: float64x1_t) -> int64x1_t; + } + vcvtp_s64_f64_(a) +} + +/// Floating-point convert to signed integer, rounding toward plus infinity +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcvtps))] +pub unsafe fn vcvtpq_s64_f64(a: float64x2_t) -> int64x2_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fcvtps.v2i64.v2f64")] + fn vcvtpq_s64_f64_(a: float64x2_t) -> int64x2_t; + } + vcvtpq_s64_f64_(a) +} + +/// Floating-point convert to signed integer, rounding toward plus infinity +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcvtps))] +pub unsafe fn vcvtps_s32_f32(a: f32) -> i32 { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fcvtps.i32.f32")] + fn vcvtps_s32_f32_(a: f32) -> i32; + } + vcvtps_s32_f32_(a) +} + +/// Floating-point convert to signed integer, rounding toward plus infinity +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcvtps))] +pub unsafe fn vcvtpd_s64_f64(a: f64) -> i64 { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fcvtps.i64.f64")] + fn vcvtpd_s64_f64_(a: f64) -> i64; + } + vcvtpd_s64_f64_(a) +} + +/// Floating-point convert to unsigned integer, rounding to nearest with ties to away +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcvtau))] +pub unsafe fn vcvta_u32_f32(a: float32x2_t) -> uint32x2_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fcvtau.v2i32.v2f32")] + fn vcvta_u32_f32_(a: float32x2_t) -> uint32x2_t; + } + vcvta_u32_f32_(a) +} + +/// Floating-point convert to unsigned integer, rounding to nearest with ties to away +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcvtau))] +pub unsafe fn vcvtaq_u32_f32(a: float32x4_t) -> uint32x4_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fcvtau.v4i32.v4f32")] + fn vcvtaq_u32_f32_(a: float32x4_t) -> uint32x4_t; + } + vcvtaq_u32_f32_(a) +} + +/// Floating-point convert to unsigned integer, rounding to nearest with ties to away +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcvtau))] +pub unsafe fn vcvta_u64_f64(a: float64x1_t) -> uint64x1_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fcvtau.v1i64.v1f64")] + fn vcvta_u64_f64_(a: float64x1_t) -> uint64x1_t; + } + vcvta_u64_f64_(a) +} + +/// Floating-point convert to unsigned integer, rounding to nearest with ties to away +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcvtau))] +pub unsafe fn vcvtaq_u64_f64(a: float64x2_t) -> uint64x2_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fcvtau.v2i64.v2f64")] + fn vcvtaq_u64_f64_(a: float64x2_t) -> uint64x2_t; + } + vcvtaq_u64_f64_(a) +} + +/// Floating-point convert to unsigned integer, rounding to nearest with ties to even +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcvtnu))] +pub unsafe fn vcvtn_u32_f32(a: float32x2_t) -> uint32x2_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fcvtnu.v2i32.v2f32")] + fn vcvtn_u32_f32_(a: float32x2_t) -> uint32x2_t; + } + vcvtn_u32_f32_(a) +} + +/// Floating-point convert to unsigned integer, rounding to nearest with ties to even +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcvtnu))] +pub unsafe fn vcvtnq_u32_f32(a: float32x4_t) -> uint32x4_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fcvtnu.v4i32.v4f32")] + fn vcvtnq_u32_f32_(a: float32x4_t) -> uint32x4_t; + } + vcvtnq_u32_f32_(a) +} + +/// Floating-point convert to unsigned integer, rounding to nearest with ties to even +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcvtnu))] +pub unsafe fn vcvtn_u64_f64(a: float64x1_t) -> uint64x1_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fcvtnu.v1i64.v1f64")] + fn vcvtn_u64_f64_(a: float64x1_t) -> uint64x1_t; + } + vcvtn_u64_f64_(a) +} + +/// Floating-point convert to unsigned integer, rounding to nearest with ties to even +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcvtnu))] +pub unsafe fn vcvtnq_u64_f64(a: float64x2_t) -> uint64x2_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fcvtnu.v2i64.v2f64")] + fn vcvtnq_u64_f64_(a: float64x2_t) -> uint64x2_t; + } + vcvtnq_u64_f64_(a) +} + +/// Floating-point convert to unsigned integer, rounding to nearest with ties to even +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcvtnu))] +pub unsafe fn vcvtns_u32_f32(a: f32) -> u32 { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fcvtnu.i32.f32")] + fn vcvtns_u32_f32_(a: f32) -> u32; + } + vcvtns_u32_f32_(a) +} + +/// Floating-point convert to unsigned integer, rounding to nearest with ties to even +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcvtnu))] +pub unsafe fn vcvtnd_u64_f64(a: f64) -> u64 { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fcvtnu.i64.f64")] + fn vcvtnd_u64_f64_(a: f64) -> u64; + } + vcvtnd_u64_f64_(a) +} + +/// Floating-point convert to unsigned integer, rounding toward minus infinity +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcvtmu))] +pub unsafe fn vcvtm_u32_f32(a: float32x2_t) -> uint32x2_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fcvtmu.v2i32.v2f32")] + fn vcvtm_u32_f32_(a: float32x2_t) -> uint32x2_t; + } + vcvtm_u32_f32_(a) +} + +/// Floating-point convert to unsigned integer, rounding toward minus infinity +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcvtmu))] +pub unsafe fn vcvtmq_u32_f32(a: float32x4_t) -> uint32x4_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fcvtmu.v4i32.v4f32")] + fn vcvtmq_u32_f32_(a: float32x4_t) -> uint32x4_t; + } + vcvtmq_u32_f32_(a) +} + +/// Floating-point convert to unsigned integer, rounding toward minus infinity +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcvtmu))] +pub unsafe fn vcvtm_u64_f64(a: float64x1_t) -> uint64x1_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fcvtmu.v1i64.v1f64")] + fn vcvtm_u64_f64_(a: float64x1_t) -> uint64x1_t; + } + vcvtm_u64_f64_(a) +} + +/// Floating-point convert to unsigned integer, rounding toward minus infinity +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcvtmu))] +pub unsafe fn vcvtmq_u64_f64(a: float64x2_t) -> uint64x2_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fcvtmu.v2i64.v2f64")] + fn vcvtmq_u64_f64_(a: float64x2_t) -> uint64x2_t; + } + vcvtmq_u64_f64_(a) +} + +/// Floating-point convert to unsigned integer, rounding toward minus infinity +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcvtmu))] +pub unsafe fn vcvtms_u32_f32(a: f32) -> u32 { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fcvtmu.i32.f32")] + fn vcvtms_u32_f32_(a: f32) -> u32; + } + vcvtms_u32_f32_(a) +} + +/// Floating-point convert to unsigned integer, rounding toward minus infinity +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcvtmu))] +pub unsafe fn vcvtmd_u64_f64(a: f64) -> u64 { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fcvtmu.i64.f64")] + fn vcvtmd_u64_f64_(a: f64) -> u64; + } + vcvtmd_u64_f64_(a) +} + +/// Floating-point convert to unsigned integer, rounding toward plus infinity +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcvtpu))] +pub unsafe fn vcvtp_u32_f32(a: float32x2_t) -> uint32x2_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fcvtpu.v2i32.v2f32")] + fn vcvtp_u32_f32_(a: float32x2_t) -> uint32x2_t; + } + vcvtp_u32_f32_(a) +} + +/// Floating-point convert to unsigned integer, rounding toward plus infinity +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcvtpu))] +pub unsafe fn vcvtpq_u32_f32(a: float32x4_t) -> uint32x4_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fcvtpu.v4i32.v4f32")] + fn vcvtpq_u32_f32_(a: float32x4_t) -> uint32x4_t; + } + vcvtpq_u32_f32_(a) +} + +/// Floating-point convert to unsigned integer, rounding toward plus infinity +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcvtpu))] +pub unsafe fn vcvtp_u64_f64(a: float64x1_t) -> uint64x1_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fcvtpu.v1i64.v1f64")] + fn vcvtp_u64_f64_(a: float64x1_t) -> uint64x1_t; + } + vcvtp_u64_f64_(a) +} + +/// Floating-point convert to unsigned integer, rounding toward plus infinity +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcvtpu))] +pub unsafe fn vcvtpq_u64_f64(a: float64x2_t) -> uint64x2_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fcvtpu.v2i64.v2f64")] + fn vcvtpq_u64_f64_(a: float64x2_t) -> uint64x2_t; + } + vcvtpq_u64_f64_(a) +} + +/// Floating-point convert to unsigned integer, rounding toward plus infinity +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcvtpu))] +pub unsafe fn vcvtps_u32_f32(a: f32) -> u32 { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fcvtpu.i32.f32")] + fn vcvtps_u32_f32_(a: f32) -> u32; + } + vcvtps_u32_f32_(a) +} + +/// Floating-point convert to unsigned integer, rounding toward plus infinity +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fcvtpu))] +pub unsafe fn vcvtpd_u64_f64(a: f64) -> u64 { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fcvtpu.i64.f64")] + fn vcvtpd_u64_f64_(a: f64) -> u64; + } + vcvtpd_u64_f64_(a) +} + +/// Set all vector lanes to the same value +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(dup, N = 1))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vdupq_laneq_p64(a: poly64x2_t) -> poly64x2_t { + static_assert_imm1!(N); + simd_shuffle2!(a, a, [N as u32, N as u32]) +} + +/// Set all vector lanes to the same value +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(dup, N = 0))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vdupq_lane_p64(a: poly64x1_t) -> poly64x2_t { + static_assert!(N : i32 where N == 0); + simd_shuffle2!(a, a, [N as u32, N as u32]) +} + +/// Set all vector lanes to the same value +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(dup, N = 1))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vdupq_laneq_f64(a: float64x2_t) -> float64x2_t { + static_assert_imm1!(N); + simd_shuffle2!(a, a, [N as u32, N as u32]) +} + +/// Set all vector lanes to the same value +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(dup, N = 0))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vdupq_lane_f64(a: float64x1_t) -> float64x2_t { + static_assert!(N : i32 where N == 0); + simd_shuffle2!(a, a, [N as u32, N as u32]) +} + +/// Set all vector lanes to the same value +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop, N = 0))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vdup_lane_p64(a: poly64x1_t) -> poly64x1_t { + static_assert!(N : i32 where N == 0); + a +} + +/// Set all vector lanes to the same value +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop, N = 0))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vdup_lane_f64(a: float64x1_t) -> float64x1_t { + static_assert!(N : i32 where N == 0); + a +} + +/// Set all vector lanes to the same value +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop, N = 1))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vdup_laneq_p64(a: poly64x2_t) -> poly64x1_t { + static_assert_imm1!(N); + transmute::(simd_extract(a, N as u32)) +} + +/// Set all vector lanes to the same value +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop, N = 1))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vdup_laneq_f64(a: float64x2_t) -> float64x1_t { + static_assert_imm1!(N); + transmute::(simd_extract(a, N as u32)) +} + +/// Set all vector lanes to the same value +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop, N = 4))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vdupb_lane_s8(a: int8x8_t) -> i8 { + static_assert_imm3!(N); + simd_extract(a, N as u32) +} + +/// Set all vector lanes to the same value +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop, N = 8))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vdupb_laneq_s8(a: int8x16_t) -> i8 { + static_assert_imm4!(N); + simd_extract(a, N as u32) +} + +/// Set all vector lanes to the same value +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vduph_lane_s16(a: int16x4_t) -> i16 { + static_assert_imm2!(N); + simd_extract(a, N as u32) +} + +/// Set all vector lanes to the same value +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop, N = 4))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vduph_laneq_s16(a: int16x8_t) -> i16 { + static_assert_imm3!(N); + simd_extract(a, N as u32) +} + +/// Set all vector lanes to the same value +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop, N = 1))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vdups_lane_s32(a: int32x2_t) -> i32 { + static_assert_imm1!(N); + simd_extract(a, N as u32) +} + +/// Set all vector lanes to the same value +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vdups_laneq_s32(a: int32x4_t) -> i32 { + static_assert_imm2!(N); + simd_extract(a, N as u32) +} + +/// Set all vector lanes to the same value +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop, N = 0))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vdupd_lane_s64(a: int64x1_t) -> i64 { + static_assert!(N : i32 where N == 0); + simd_extract(a, N as u32) +} + +/// Set all vector lanes to the same value +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop, N = 1))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vdupd_laneq_s64(a: int64x2_t) -> i64 { + static_assert_imm1!(N); + simd_extract(a, N as u32) +} + +/// Set all vector lanes to the same value +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop, N = 4))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vdupb_lane_u8(a: uint8x8_t) -> u8 { + static_assert_imm3!(N); + simd_extract(a, N as u32) +} + +/// Set all vector lanes to the same value +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop, N = 8))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vdupb_laneq_u8(a: uint8x16_t) -> u8 { + static_assert_imm4!(N); + simd_extract(a, N as u32) +} + +/// Set all vector lanes to the same value +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vduph_lane_u16(a: uint16x4_t) -> u16 { + static_assert_imm2!(N); + simd_extract(a, N as u32) +} + +/// Set all vector lanes to the same value +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop, N = 4))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vduph_laneq_u16(a: uint16x8_t) -> u16 { + static_assert_imm3!(N); + simd_extract(a, N as u32) +} + +/// Set all vector lanes to the same value +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop, N = 1))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vdups_lane_u32(a: uint32x2_t) -> u32 { + static_assert_imm1!(N); + simd_extract(a, N as u32) +} + +/// Set all vector lanes to the same value +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vdups_laneq_u32(a: uint32x4_t) -> u32 { + static_assert_imm2!(N); + simd_extract(a, N as u32) +} + +/// Set all vector lanes to the same value +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop, N = 0))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vdupd_lane_u64(a: uint64x1_t) -> u64 { + static_assert!(N : i32 where N == 0); + simd_extract(a, N as u32) +} + +/// Set all vector lanes to the same value +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop, N = 1))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vdupd_laneq_u64(a: uint64x2_t) -> u64 { + static_assert_imm1!(N); + simd_extract(a, N as u32) +} + +/// Set all vector lanes to the same value +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop, N = 4))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vdupb_lane_p8(a: poly8x8_t) -> p8 { + static_assert_imm3!(N); + simd_extract(a, N as u32) +} + +/// Set all vector lanes to the same value +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop, N = 8))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vdupb_laneq_p8(a: poly8x16_t) -> p8 { + static_assert_imm4!(N); + simd_extract(a, N as u32) +} + +/// Set all vector lanes to the same value +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vduph_lane_p16(a: poly16x4_t) -> p16 { + static_assert_imm2!(N); + simd_extract(a, N as u32) +} + +/// Set all vector lanes to the same value +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop, N = 4))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vduph_laneq_p16(a: poly16x8_t) -> p16 { + static_assert_imm3!(N); + simd_extract(a, N as u32) +} + +/// Set all vector lanes to the same value +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop, N = 1))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vdups_lane_f32(a: float32x2_t) -> f32 { + static_assert_imm1!(N); + simd_extract(a, N as u32) +} + +/// Set all vector lanes to the same value +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vdups_laneq_f32(a: float32x4_t) -> f32 { + static_assert_imm2!(N); + simd_extract(a, N as u32) +} + +/// Set all vector lanes to the same value +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop, N = 0))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vdupd_lane_f64(a: float64x1_t) -> f64 { + static_assert!(N : i32 where N == 0); + simd_extract(a, N as u32) +} + +/// Set all vector lanes to the same value +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop, N = 1))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vdupd_laneq_f64(a: float64x2_t) -> f64 { + static_assert_imm1!(N); + simd_extract(a, N as u32) +} + +/// Extract vector from pair of vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ext, N = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vextq_p64(a: poly64x2_t, b: poly64x2_t) -> poly64x2_t { + static_assert_imm1!(N); + match N & 0b1 { + 0 => simd_shuffle2!(a, b, [0, 1]), + 1 => simd_shuffle2!(a, b, [1, 2]), + _ => unreachable_unchecked(), + } +} + +/// Extract vector from pair of vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ext, N = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vextq_f64(a: float64x2_t, b: float64x2_t) -> float64x2_t { + static_assert_imm1!(N); + match N & 0b1 { + 0 => simd_shuffle2!(a, b, [0, 1]), + 1 => simd_shuffle2!(a, b, [1, 2]), + _ => unreachable_unchecked(), + } +} + +/// Floating-point multiply-add to accumulator +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmul))] +pub unsafe fn vmla_f64(a: float64x1_t, b: float64x1_t, c: float64x1_t) -> float64x1_t { + simd_add(a, simd_mul(b, c)) +} + +/// Floating-point multiply-add to accumulator +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmul))] +pub unsafe fn vmlaq_f64(a: float64x2_t, b: float64x2_t, c: float64x2_t) -> float64x2_t { + simd_add(a, simd_mul(b, c)) +} + +/// Signed multiply-add long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(smlal2))] +pub unsafe fn vmlal_high_s8(a: int16x8_t, b: int8x16_t, c: int8x16_t) -> int16x8_t { + let b: int8x8_t = simd_shuffle8!(b, b, [8, 9, 10, 11, 12, 13, 14, 15]); + let c: int8x8_t = simd_shuffle8!(c, c, [8, 9, 10, 11, 12, 13, 14, 15]); + vmlal_s8(a, b, c) +} + +/// Signed multiply-add long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(smlal2))] +pub unsafe fn vmlal_high_s16(a: int32x4_t, b: int16x8_t, c: int16x8_t) -> int32x4_t { + let b: int16x4_t = simd_shuffle4!(b, b, [4, 5, 6, 7]); + let c: int16x4_t = simd_shuffle4!(c, c, [4, 5, 6, 7]); + vmlal_s16(a, b, c) +} + +/// Signed multiply-add long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(smlal2))] +pub unsafe fn vmlal_high_s32(a: int64x2_t, b: int32x4_t, c: int32x4_t) -> int64x2_t { + let b: int32x2_t = simd_shuffle2!(b, b, [2, 3]); + let c: int32x2_t = simd_shuffle2!(c, c, [2, 3]); + vmlal_s32(a, b, c) +} + +/// Unsigned multiply-add long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(umlal2))] +pub unsafe fn vmlal_high_u8(a: uint16x8_t, b: uint8x16_t, c: uint8x16_t) -> uint16x8_t { + let b: uint8x8_t = simd_shuffle8!(b, b, [8, 9, 10, 11, 12, 13, 14, 15]); + let c: uint8x8_t = simd_shuffle8!(c, c, [8, 9, 10, 11, 12, 13, 14, 15]); + vmlal_u8(a, b, c) +} + +/// Unsigned multiply-add long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(umlal2))] +pub unsafe fn vmlal_high_u16(a: uint32x4_t, b: uint16x8_t, c: uint16x8_t) -> uint32x4_t { + let b: uint16x4_t = simd_shuffle4!(b, b, [4, 5, 6, 7]); + let c: uint16x4_t = simd_shuffle4!(c, c, [4, 5, 6, 7]); + vmlal_u16(a, b, c) +} + +/// Unsigned multiply-add long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(umlal2))] +pub unsafe fn vmlal_high_u32(a: uint64x2_t, b: uint32x4_t, c: uint32x4_t) -> uint64x2_t { + let b: uint32x2_t = simd_shuffle2!(b, b, [2, 3]); + let c: uint32x2_t = simd_shuffle2!(c, c, [2, 3]); + vmlal_u32(a, b, c) +} + +/// Multiply-add long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(smlal2))] +pub unsafe fn vmlal_high_n_s16(a: int32x4_t, b: int16x8_t, c: i16) -> int32x4_t { + vmlal_high_s16(a, b, vdupq_n_s16(c)) +} + +/// Multiply-add long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(smlal2))] +pub unsafe fn vmlal_high_n_s32(a: int64x2_t, b: int32x4_t, c: i32) -> int64x2_t { + vmlal_high_s32(a, b, vdupq_n_s32(c)) +} + +/// Multiply-add long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(umlal2))] +pub unsafe fn vmlal_high_n_u16(a: uint32x4_t, b: uint16x8_t, c: u16) -> uint32x4_t { + vmlal_high_u16(a, b, vdupq_n_u16(c)) +} + +/// Multiply-add long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(umlal2))] +pub unsafe fn vmlal_high_n_u32(a: uint64x2_t, b: uint32x4_t, c: u32) -> uint64x2_t { + vmlal_high_u32(a, b, vdupq_n_u32(c)) +} + +/// Multiply-add long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(smlal2, LANE = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vmlal_high_lane_s16(a: int32x4_t, b: int16x8_t, c: int16x4_t) -> int32x4_t { + static_assert_imm2!(LANE); + vmlal_high_s16(a, b, simd_shuffle8!(c, c, [LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Multiply-add long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(smlal2, LANE = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vmlal_high_laneq_s16(a: int32x4_t, b: int16x8_t, c: int16x8_t) -> int32x4_t { + static_assert_imm3!(LANE); + vmlal_high_s16(a, b, simd_shuffle8!(c, c, [LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Multiply-add long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(smlal2, LANE = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vmlal_high_lane_s32(a: int64x2_t, b: int32x4_t, c: int32x2_t) -> int64x2_t { + static_assert_imm1!(LANE); + vmlal_high_s32(a, b, simd_shuffle4!(c, c, [LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Multiply-add long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(smlal2, LANE = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vmlal_high_laneq_s32(a: int64x2_t, b: int32x4_t, c: int32x4_t) -> int64x2_t { + static_assert_imm2!(LANE); + vmlal_high_s32(a, b, simd_shuffle4!(c, c, [LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Multiply-add long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(umlal2, LANE = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vmlal_high_lane_u16(a: uint32x4_t, b: uint16x8_t, c: uint16x4_t) -> uint32x4_t { + static_assert_imm2!(LANE); + vmlal_high_u16(a, b, simd_shuffle8!(c, c, [LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Multiply-add long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(umlal2, LANE = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vmlal_high_laneq_u16(a: uint32x4_t, b: uint16x8_t, c: uint16x8_t) -> uint32x4_t { + static_assert_imm3!(LANE); + vmlal_high_u16(a, b, simd_shuffle8!(c, c, [LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Multiply-add long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(umlal2, LANE = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vmlal_high_lane_u32(a: uint64x2_t, b: uint32x4_t, c: uint32x2_t) -> uint64x2_t { + static_assert_imm1!(LANE); + vmlal_high_u32(a, b, simd_shuffle4!(c, c, [LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Multiply-add long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(umlal2, LANE = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vmlal_high_laneq_u32(a: uint64x2_t, b: uint32x4_t, c: uint32x4_t) -> uint64x2_t { + static_assert_imm2!(LANE); + vmlal_high_u32(a, b, simd_shuffle4!(c, c, [LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Floating-point multiply-subtract from accumulator +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmul))] +pub unsafe fn vmls_f64(a: float64x1_t, b: float64x1_t, c: float64x1_t) -> float64x1_t { + simd_sub(a, simd_mul(b, c)) +} + +/// Floating-point multiply-subtract from accumulator +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmul))] +pub unsafe fn vmlsq_f64(a: float64x2_t, b: float64x2_t, c: float64x2_t) -> float64x2_t { + simd_sub(a, simd_mul(b, c)) +} + +/// Signed multiply-subtract long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(smlsl2))] +pub unsafe fn vmlsl_high_s8(a: int16x8_t, b: int8x16_t, c: int8x16_t) -> int16x8_t { + let b: int8x8_t = simd_shuffle8!(b, b, [8, 9, 10, 11, 12, 13, 14, 15]); + let c: int8x8_t = simd_shuffle8!(c, c, [8, 9, 10, 11, 12, 13, 14, 15]); + vmlsl_s8(a, b, c) +} + +/// Signed multiply-subtract long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(smlsl2))] +pub unsafe fn vmlsl_high_s16(a: int32x4_t, b: int16x8_t, c: int16x8_t) -> int32x4_t { + let b: int16x4_t = simd_shuffle4!(b, b, [4, 5, 6, 7]); + let c: int16x4_t = simd_shuffle4!(c, c, [4, 5, 6, 7]); + vmlsl_s16(a, b, c) +} + +/// Signed multiply-subtract long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(smlsl2))] +pub unsafe fn vmlsl_high_s32(a: int64x2_t, b: int32x4_t, c: int32x4_t) -> int64x2_t { + let b: int32x2_t = simd_shuffle2!(b, b, [2, 3]); + let c: int32x2_t = simd_shuffle2!(c, c, [2, 3]); + vmlsl_s32(a, b, c) +} + +/// Unsigned multiply-subtract long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(umlsl2))] +pub unsafe fn vmlsl_high_u8(a: uint16x8_t, b: uint8x16_t, c: uint8x16_t) -> uint16x8_t { + let b: uint8x8_t = simd_shuffle8!(b, b, [8, 9, 10, 11, 12, 13, 14, 15]); + let c: uint8x8_t = simd_shuffle8!(c, c, [8, 9, 10, 11, 12, 13, 14, 15]); + vmlsl_u8(a, b, c) +} + +/// Unsigned multiply-subtract long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(umlsl2))] +pub unsafe fn vmlsl_high_u16(a: uint32x4_t, b: uint16x8_t, c: uint16x8_t) -> uint32x4_t { + let b: uint16x4_t = simd_shuffle4!(b, b, [4, 5, 6, 7]); + let c: uint16x4_t = simd_shuffle4!(c, c, [4, 5, 6, 7]); + vmlsl_u16(a, b, c) +} + +/// Unsigned multiply-subtract long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(umlsl2))] +pub unsafe fn vmlsl_high_u32(a: uint64x2_t, b: uint32x4_t, c: uint32x4_t) -> uint64x2_t { + let b: uint32x2_t = simd_shuffle2!(b, b, [2, 3]); + let c: uint32x2_t = simd_shuffle2!(c, c, [2, 3]); + vmlsl_u32(a, b, c) +} + +/// Multiply-subtract long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(smlsl2))] +pub unsafe fn vmlsl_high_n_s16(a: int32x4_t, b: int16x8_t, c: i16) -> int32x4_t { + vmlsl_high_s16(a, b, vdupq_n_s16(c)) +} + +/// Multiply-subtract long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(smlsl2))] +pub unsafe fn vmlsl_high_n_s32(a: int64x2_t, b: int32x4_t, c: i32) -> int64x2_t { + vmlsl_high_s32(a, b, vdupq_n_s32(c)) +} + +/// Multiply-subtract long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(umlsl2))] +pub unsafe fn vmlsl_high_n_u16(a: uint32x4_t, b: uint16x8_t, c: u16) -> uint32x4_t { + vmlsl_high_u16(a, b, vdupq_n_u16(c)) +} + +/// Multiply-subtract long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(umlsl2))] +pub unsafe fn vmlsl_high_n_u32(a: uint64x2_t, b: uint32x4_t, c: u32) -> uint64x2_t { + vmlsl_high_u32(a, b, vdupq_n_u32(c)) +} + +/// Multiply-subtract long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(smlsl2, LANE = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vmlsl_high_lane_s16(a: int32x4_t, b: int16x8_t, c: int16x4_t) -> int32x4_t { + static_assert_imm2!(LANE); + vmlsl_high_s16(a, b, simd_shuffle8!(c, c, [LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Multiply-subtract long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(smlsl2, LANE = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vmlsl_high_laneq_s16(a: int32x4_t, b: int16x8_t, c: int16x8_t) -> int32x4_t { + static_assert_imm3!(LANE); + vmlsl_high_s16(a, b, simd_shuffle8!(c, c, [LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Multiply-subtract long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(smlsl2, LANE = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vmlsl_high_lane_s32(a: int64x2_t, b: int32x4_t, c: int32x2_t) -> int64x2_t { + static_assert_imm1!(LANE); + vmlsl_high_s32(a, b, simd_shuffle4!(c, c, [LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Multiply-subtract long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(smlsl2, LANE = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vmlsl_high_laneq_s32(a: int64x2_t, b: int32x4_t, c: int32x4_t) -> int64x2_t { + static_assert_imm2!(LANE); + vmlsl_high_s32(a, b, simd_shuffle4!(c, c, [LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Multiply-subtract long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(umlsl2, LANE = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vmlsl_high_lane_u16(a: uint32x4_t, b: uint16x8_t, c: uint16x4_t) -> uint32x4_t { + static_assert_imm2!(LANE); + vmlsl_high_u16(a, b, simd_shuffle8!(c, c, [LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Multiply-subtract long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(umlsl2, LANE = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vmlsl_high_laneq_u16(a: uint32x4_t, b: uint16x8_t, c: uint16x8_t) -> uint32x4_t { + static_assert_imm3!(LANE); + vmlsl_high_u16(a, b, simd_shuffle8!(c, c, [LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Multiply-subtract long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(umlsl2, LANE = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vmlsl_high_lane_u32(a: uint64x2_t, b: uint32x4_t, c: uint32x2_t) -> uint64x2_t { + static_assert_imm1!(LANE); + vmlsl_high_u32(a, b, simd_shuffle4!(c, c, [LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Multiply-subtract long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(umlsl2, LANE = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vmlsl_high_laneq_u32(a: uint64x2_t, b: uint32x4_t, c: uint32x4_t) -> uint64x2_t { + static_assert_imm2!(LANE); + vmlsl_high_u32(a, b, simd_shuffle4!(c, c, [LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Extract narrow +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(xtn2))] +pub unsafe fn vmovn_high_s16(a: int8x8_t, b: int16x8_t) -> int8x16_t { + let c: int8x8_t = simd_cast(b); + simd_shuffle16!(a, c, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]) +} + +/// Extract narrow +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(xtn2))] +pub unsafe fn vmovn_high_s32(a: int16x4_t, b: int32x4_t) -> int16x8_t { + let c: int16x4_t = simd_cast(b); + simd_shuffle8!(a, c, [0, 1, 2, 3, 4, 5, 6, 7]) +} + +/// Extract narrow +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(xtn2))] +pub unsafe fn vmovn_high_s64(a: int32x2_t, b: int64x2_t) -> int32x4_t { + let c: int32x2_t = simd_cast(b); + simd_shuffle4!(a, c, [0, 1, 2, 3]) +} + +/// Extract narrow +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(xtn2))] +pub unsafe fn vmovn_high_u16(a: uint8x8_t, b: uint16x8_t) -> uint8x16_t { + let c: uint8x8_t = simd_cast(b); + simd_shuffle16!(a, c, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]) +} + +/// Extract narrow +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(xtn2))] +pub unsafe fn vmovn_high_u32(a: uint16x4_t, b: uint32x4_t) -> uint16x8_t { + let c: uint16x4_t = simd_cast(b); + simd_shuffle8!(a, c, [0, 1, 2, 3, 4, 5, 6, 7]) +} + +/// Extract narrow +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(xtn2))] +pub unsafe fn vmovn_high_u64(a: uint32x2_t, b: uint64x2_t) -> uint32x4_t { + let c: uint32x2_t = simd_cast(b); + simd_shuffle4!(a, c, [0, 1, 2, 3]) +} + +/// Negate +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(neg))] +pub unsafe fn vneg_s64(a: int64x1_t) -> int64x1_t { + simd_neg(a) +} + +/// Negate +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(neg))] +pub unsafe fn vnegq_s64(a: int64x2_t) -> int64x2_t { + simd_neg(a) +} + +/// Negate +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fneg))] +pub unsafe fn vneg_f64(a: float64x1_t) -> float64x1_t { + simd_neg(a) +} + +/// Negate +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fneg))] +pub unsafe fn vnegq_f64(a: float64x2_t) -> float64x2_t { + simd_neg(a) +} + +/// Signed saturating negate +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqneg))] +pub unsafe fn vqneg_s64(a: int64x1_t) -> int64x1_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqneg.v1i64")] + fn vqneg_s64_(a: int64x1_t) -> int64x1_t; + } + vqneg_s64_(a) +} + +/// Signed saturating negate +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqneg))] +pub unsafe fn vqnegq_s64(a: int64x2_t) -> int64x2_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqneg.v2i64")] + fn vqnegq_s64_(a: int64x2_t) -> int64x2_t; + } + vqnegq_s64_(a) +} + +/// Saturating subtract +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqsub))] +pub unsafe fn vqsubb_s8(a: i8, b: i8) -> i8 { + let a: int8x8_t = vdup_n_s8(a); + let b: int8x8_t = vdup_n_s8(b); + simd_extract(vqsub_s8(a, b), 0) +} + +/// Saturating subtract +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqsub))] +pub unsafe fn vqsubh_s16(a: i16, b: i16) -> i16 { + let a: int16x4_t = vdup_n_s16(a); + let b: int16x4_t = vdup_n_s16(b); + simd_extract(vqsub_s16(a, b), 0) +} + +/// Saturating subtract +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uqsub))] +pub unsafe fn vqsubb_u8(a: u8, b: u8) -> u8 { + let a: uint8x8_t = vdup_n_u8(a); + let b: uint8x8_t = vdup_n_u8(b); + simd_extract(vqsub_u8(a, b), 0) +} + +/// Saturating subtract +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uqsub))] +pub unsafe fn vqsubh_u16(a: u16, b: u16) -> u16 { + let a: uint16x4_t = vdup_n_u16(a); + let b: uint16x4_t = vdup_n_u16(b); + simd_extract(vqsub_u16(a, b), 0) +} + +/// Saturating subtract +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uqsub))] +pub unsafe fn vqsubs_u32(a: u32, b: u32) -> u32 { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uqsub.i32")] + fn vqsubs_u32_(a: u32, b: u32) -> u32; + } + vqsubs_u32_(a, b) +} + +/// Saturating subtract +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uqsub))] +pub unsafe fn vqsubd_u64(a: u64, b: u64) -> u64 { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uqsub.i64")] + fn vqsubd_u64_(a: u64, b: u64) -> u64; + } + vqsubd_u64_(a, b) +} + +/// Saturating subtract +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqsub))] +pub unsafe fn vqsubs_s32(a: i32, b: i32) -> i32 { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqsub.i32")] + fn vqsubs_s32_(a: i32, b: i32) -> i32; + } + vqsubs_s32_(a, b) +} + +/// Saturating subtract +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqsub))] +pub unsafe fn vqsubd_s64(a: i64, b: i64) -> i64 { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqsub.i64")] + fn vqsubd_s64_(a: i64, b: i64) -> i64; + } + vqsubd_s64_(a, b) +} + +/// Reverse bit order +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(rbit))] +pub unsafe fn vrbit_s8(a: int8x8_t) -> int8x8_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.rbit.v8i8")] + fn vrbit_s8_(a: int8x8_t) -> int8x8_t; + } + vrbit_s8_(a) +} + +/// Reverse bit order +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(rbit))] +pub unsafe fn vrbitq_s8(a: int8x16_t) -> int8x16_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.rbit.v16i8")] + fn vrbitq_s8_(a: int8x16_t) -> int8x16_t; + } + vrbitq_s8_(a) +} + +/// Reverse bit order +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(rbit))] +pub unsafe fn vrbit_u8(a: uint8x8_t) -> uint8x8_t { + transmute(vrbit_s8(transmute(a))) +} + +/// Reverse bit order +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(rbit))] +pub unsafe fn vrbitq_u8(a: uint8x16_t) -> uint8x16_t { + transmute(vrbitq_s8(transmute(a))) +} + +/// Reverse bit order +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(rbit))] +pub unsafe fn vrbit_p8(a: poly8x8_t) -> poly8x8_t { + transmute(vrbit_s8(transmute(a))) +} + +/// Reverse bit order +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(rbit))] +pub unsafe fn vrbitq_p8(a: poly8x16_t) -> poly8x16_t { + transmute(vrbitq_s8(transmute(a))) +} + +/// Floating-point round to integral exact, using current rounding mode +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(frintx))] +pub unsafe fn vrndx_f32(a: float32x2_t) -> float32x2_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.rint.v2f32")] + fn vrndx_f32_(a: float32x2_t) -> float32x2_t; + } + vrndx_f32_(a) +} + +/// Floating-point round to integral exact, using current rounding mode +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(frintx))] +pub unsafe fn vrndxq_f32(a: float32x4_t) -> float32x4_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.rint.v4f32")] + fn vrndxq_f32_(a: float32x4_t) -> float32x4_t; + } + vrndxq_f32_(a) +} + +/// Floating-point round to integral exact, using current rounding mode +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(frintx))] +pub unsafe fn vrndx_f64(a: float64x1_t) -> float64x1_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.rint.v1f64")] + fn vrndx_f64_(a: float64x1_t) -> float64x1_t; + } + vrndx_f64_(a) +} + +/// Floating-point round to integral exact, using current rounding mode +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(frintx))] +pub unsafe fn vrndxq_f64(a: float64x2_t) -> float64x2_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.rint.v2f64")] + fn vrndxq_f64_(a: float64x2_t) -> float64x2_t; + } + vrndxq_f64_(a) +} + +/// Floating-point round to integral, to nearest with ties to away +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(frinta))] +pub unsafe fn vrnda_f32(a: float32x2_t) -> float32x2_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.round.v2f32")] + fn vrnda_f32_(a: float32x2_t) -> float32x2_t; + } + vrnda_f32_(a) +} + +/// Floating-point round to integral, to nearest with ties to away +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(frinta))] +pub unsafe fn vrndaq_f32(a: float32x4_t) -> float32x4_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.round.v4f32")] + fn vrndaq_f32_(a: float32x4_t) -> float32x4_t; + } + vrndaq_f32_(a) +} + +/// Floating-point round to integral, to nearest with ties to away +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(frinta))] +pub unsafe fn vrnda_f64(a: float64x1_t) -> float64x1_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.round.v1f64")] + fn vrnda_f64_(a: float64x1_t) -> float64x1_t; + } + vrnda_f64_(a) +} + +/// Floating-point round to integral, to nearest with ties to away +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(frinta))] +pub unsafe fn vrndaq_f64(a: float64x2_t) -> float64x2_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.round.v2f64")] + fn vrndaq_f64_(a: float64x2_t) -> float64x2_t; + } + vrndaq_f64_(a) +} + +/// Floating-point round to integral, to nearest with ties to even +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(frintn))] +pub unsafe fn vrndn_f64(a: float64x1_t) -> float64x1_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.frintn.v1f64")] + fn vrndn_f64_(a: float64x1_t) -> float64x1_t; + } + vrndn_f64_(a) +} + +/// Floating-point round to integral, to nearest with ties to even +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(frintn))] +pub unsafe fn vrndnq_f64(a: float64x2_t) -> float64x2_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.frintn.v2f64")] + fn vrndnq_f64_(a: float64x2_t) -> float64x2_t; + } + vrndnq_f64_(a) +} + +/// Floating-point round to integral, toward minus infinity +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(frintm))] +pub unsafe fn vrndm_f32(a: float32x2_t) -> float32x2_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.floor.v2f32")] + fn vrndm_f32_(a: float32x2_t) -> float32x2_t; + } + vrndm_f32_(a) +} + +/// Floating-point round to integral, toward minus infinity +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(frintm))] +pub unsafe fn vrndmq_f32(a: float32x4_t) -> float32x4_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.floor.v4f32")] + fn vrndmq_f32_(a: float32x4_t) -> float32x4_t; + } + vrndmq_f32_(a) +} + +/// Floating-point round to integral, toward minus infinity +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(frintm))] +pub unsafe fn vrndm_f64(a: float64x1_t) -> float64x1_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.floor.v1f64")] + fn vrndm_f64_(a: float64x1_t) -> float64x1_t; + } + vrndm_f64_(a) +} + +/// Floating-point round to integral, toward minus infinity +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(frintm))] +pub unsafe fn vrndmq_f64(a: float64x2_t) -> float64x2_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.floor.v2f64")] + fn vrndmq_f64_(a: float64x2_t) -> float64x2_t; + } + vrndmq_f64_(a) +} + +/// Floating-point round to integral, toward plus infinity +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(frintp))] +pub unsafe fn vrndp_f32(a: float32x2_t) -> float32x2_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.ceil.v2f32")] + fn vrndp_f32_(a: float32x2_t) -> float32x2_t; + } + vrndp_f32_(a) +} + +/// Floating-point round to integral, toward plus infinity +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(frintp))] +pub unsafe fn vrndpq_f32(a: float32x4_t) -> float32x4_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.ceil.v4f32")] + fn vrndpq_f32_(a: float32x4_t) -> float32x4_t; + } + vrndpq_f32_(a) +} + +/// Floating-point round to integral, toward plus infinity +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(frintp))] +pub unsafe fn vrndp_f64(a: float64x1_t) -> float64x1_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.ceil.v1f64")] + fn vrndp_f64_(a: float64x1_t) -> float64x1_t; + } + vrndp_f64_(a) +} + +/// Floating-point round to integral, toward plus infinity +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(frintp))] +pub unsafe fn vrndpq_f64(a: float64x2_t) -> float64x2_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.ceil.v2f64")] + fn vrndpq_f64_(a: float64x2_t) -> float64x2_t; + } + vrndpq_f64_(a) +} + +/// Floating-point round to integral, toward zero +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(frintz))] +pub unsafe fn vrnd_f32(a: float32x2_t) -> float32x2_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.trunc.v2f32")] + fn vrnd_f32_(a: float32x2_t) -> float32x2_t; + } + vrnd_f32_(a) +} + +/// Floating-point round to integral, toward zero +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(frintz))] +pub unsafe fn vrndq_f32(a: float32x4_t) -> float32x4_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.trunc.v4f32")] + fn vrndq_f32_(a: float32x4_t) -> float32x4_t; + } + vrndq_f32_(a) +} + +/// Floating-point round to integral, toward zero +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(frintz))] +pub unsafe fn vrnd_f64(a: float64x1_t) -> float64x1_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.trunc.v1f64")] + fn vrnd_f64_(a: float64x1_t) -> float64x1_t; + } + vrnd_f64_(a) +} + +/// Floating-point round to integral, toward zero +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(frintz))] +pub unsafe fn vrndq_f64(a: float64x2_t) -> float64x2_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.trunc.v2f64")] + fn vrndq_f64_(a: float64x2_t) -> float64x2_t; + } + vrndq_f64_(a) +} + +/// Floating-point round to integral, using current rounding mode +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(frinti))] +pub unsafe fn vrndi_f32(a: float32x2_t) -> float32x2_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.nearbyint.v2f32")] + fn vrndi_f32_(a: float32x2_t) -> float32x2_t; + } + vrndi_f32_(a) +} + +/// Floating-point round to integral, using current rounding mode +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(frinti))] +pub unsafe fn vrndiq_f32(a: float32x4_t) -> float32x4_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.nearbyint.v4f32")] + fn vrndiq_f32_(a: float32x4_t) -> float32x4_t; + } + vrndiq_f32_(a) +} + +/// Floating-point round to integral, using current rounding mode +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(frinti))] +pub unsafe fn vrndi_f64(a: float64x1_t) -> float64x1_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.nearbyint.v1f64")] + fn vrndi_f64_(a: float64x1_t) -> float64x1_t; + } + vrndi_f64_(a) +} + +/// Floating-point round to integral, using current rounding mode +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(frinti))] +pub unsafe fn vrndiq_f64(a: float64x2_t) -> float64x2_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.nearbyint.v2f64")] + fn vrndiq_f64_(a: float64x2_t) -> float64x2_t; + } + vrndiq_f64_(a) +} + +/// Saturating add +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqadd))] +pub unsafe fn vqaddb_s8(a: i8, b: i8) -> i8 { + let a: int8x8_t = vdup_n_s8(a); + let b: int8x8_t = vdup_n_s8(b); + simd_extract(vqadd_s8(a, b), 0) +} + +/// Saturating add +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqadd))] +pub unsafe fn vqaddh_s16(a: i16, b: i16) -> i16 { + let a: int16x4_t = vdup_n_s16(a); + let b: int16x4_t = vdup_n_s16(b); + simd_extract(vqadd_s16(a, b), 0) +} + +/// Saturating add +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uqadd))] +pub unsafe fn vqaddb_u8(a: u8, b: u8) -> u8 { + let a: uint8x8_t = vdup_n_u8(a); + let b: uint8x8_t = vdup_n_u8(b); + simd_extract(vqadd_u8(a, b), 0) +} + +/// Saturating add +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uqadd))] +pub unsafe fn vqaddh_u16(a: u16, b: u16) -> u16 { + let a: uint16x4_t = vdup_n_u16(a); + let b: uint16x4_t = vdup_n_u16(b); + simd_extract(vqadd_u16(a, b), 0) +} + +/// Saturating add +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uqadd))] +pub unsafe fn vqadds_u32(a: u32, b: u32) -> u32 { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uqadd.i32")] + fn vqadds_u32_(a: u32, b: u32) -> u32; + } + vqadds_u32_(a, b) +} + +/// Saturating add +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uqadd))] +pub unsafe fn vqaddd_u64(a: u64, b: u64) -> u64 { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uqadd.i64")] + fn vqaddd_u64_(a: u64, b: u64) -> u64; + } + vqaddd_u64_(a, b) +} + +/// Saturating add +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqadd))] +pub unsafe fn vqadds_s32(a: i32, b: i32) -> i32 { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqadd.i32")] + fn vqadds_s32_(a: i32, b: i32) -> i32; + } + vqadds_s32_(a, b) +} + +/// Saturating add +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqadd))] +pub unsafe fn vqaddd_s64(a: i64, b: i64) -> i64 { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqadd.i64")] + fn vqaddd_s64_(a: i64, b: i64) -> i64; + } + vqaddd_s64_(a, b) +} + +/// Multiply +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmul))] +pub unsafe fn vmul_f64(a: float64x1_t, b: float64x1_t) -> float64x1_t { + simd_mul(a, b) +} + +/// Multiply +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmul))] +pub unsafe fn vmulq_f64(a: float64x2_t, b: float64x2_t) -> float64x2_t { + simd_mul(a, b) +} + +/// Vector multiply by scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmul))] +pub unsafe fn vmul_n_f64(a: float64x1_t, b: f64) -> float64x1_t { + simd_mul(a, vdup_n_f64(b)) +} + +/// Vector multiply by scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmul))] +pub unsafe fn vmulq_n_f64(a: float64x2_t, b: f64) -> float64x2_t { + simd_mul(a, vdupq_n_f64(b)) +} + +/// Floating-point multiply +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmul, LANE = 0))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vmul_lane_f64(a: float64x1_t, b: float64x1_t) -> float64x1_t { + static_assert!(LANE : i32 where LANE == 0); + simd_mul(a, transmute::(simd_extract(b, LANE as u32))) +} + +/// Floating-point multiply +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmul, LANE = 0))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vmul_laneq_f64(a: float64x1_t, b: float64x2_t) -> float64x1_t { + static_assert_imm1!(LANE); + simd_mul(a, transmute::(simd_extract(b, LANE as u32))) +} + +/// Floating-point multiply +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmul, LANE = 0))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vmulq_lane_f64(a: float64x2_t, b: float64x1_t) -> float64x2_t { + static_assert!(LANE : i32 where LANE == 0); + simd_mul(a, simd_shuffle2!(b, b, [LANE as u32, LANE as u32])) +} + +/// Floating-point multiply +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmul, LANE = 0))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vmulq_laneq_f64(a: float64x2_t, b: float64x2_t) -> float64x2_t { + static_assert_imm1!(LANE); + simd_mul(a, simd_shuffle2!(b, b, [LANE as u32, LANE as u32])) +} + +/// Floating-point multiply +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmul, LANE = 0))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vmuls_lane_f32(a: f32, b: float32x2_t) -> f32 { + static_assert_imm1!(LANE); + let b: f32 = simd_extract(b, LANE as u32); + a * b +} + +/// Floating-point multiply +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmul, LANE = 0))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vmuls_laneq_f32(a: f32, b: float32x4_t) -> f32 { + static_assert_imm2!(LANE); + let b: f32 = simd_extract(b, LANE as u32); + a * b +} + +/// Floating-point multiply +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmul, LANE = 0))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vmuld_lane_f64(a: f64, b: float64x1_t) -> f64 { + static_assert!(LANE : i32 where LANE == 0); + let b: f64 = simd_extract(b, LANE as u32); + a * b +} + +/// Floating-point multiply +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmul, LANE = 0))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vmuld_laneq_f64(a: f64, b: float64x2_t) -> f64 { + static_assert_imm1!(LANE); + let b: f64 = simd_extract(b, LANE as u32); + a * b +} + +/// Signed multiply long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(smull2))] +pub unsafe fn vmull_high_s8(a: int8x16_t, b: int8x16_t) -> int16x8_t { + let a: int8x8_t = simd_shuffle8!(a, a, [8, 9, 10, 11, 12, 13, 14, 15]); + let b: int8x8_t = simd_shuffle8!(b, b, [8, 9, 10, 11, 12, 13, 14, 15]); + vmull_s8(a, b) +} + +/// Signed multiply long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(smull2))] +pub unsafe fn vmull_high_s16(a: int16x8_t, b: int16x8_t) -> int32x4_t { + let a: int16x4_t = simd_shuffle4!(a, a, [4, 5, 6, 7]); + let b: int16x4_t = simd_shuffle4!(b, b, [4, 5, 6, 7]); + vmull_s16(a, b) +} + +/// Signed multiply long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(smull2))] +pub unsafe fn vmull_high_s32(a: int32x4_t, b: int32x4_t) -> int64x2_t { + let a: int32x2_t = simd_shuffle2!(a, a, [2, 3]); + let b: int32x2_t = simd_shuffle2!(b, b, [2, 3]); + vmull_s32(a, b) +} + +/// Unsigned multiply long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(umull2))] +pub unsafe fn vmull_high_u8(a: uint8x16_t, b: uint8x16_t) -> uint16x8_t { + let a: uint8x8_t = simd_shuffle8!(a, a, [8, 9, 10, 11, 12, 13, 14, 15]); + let b: uint8x8_t = simd_shuffle8!(b, b, [8, 9, 10, 11, 12, 13, 14, 15]); + vmull_u8(a, b) +} + +/// Unsigned multiply long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(umull2))] +pub unsafe fn vmull_high_u16(a: uint16x8_t, b: uint16x8_t) -> uint32x4_t { + let a: uint16x4_t = simd_shuffle4!(a, a, [4, 5, 6, 7]); + let b: uint16x4_t = simd_shuffle4!(b, b, [4, 5, 6, 7]); + vmull_u16(a, b) +} + +/// Unsigned multiply long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(umull2))] +pub unsafe fn vmull_high_u32(a: uint32x4_t, b: uint32x4_t) -> uint64x2_t { + let a: uint32x2_t = simd_shuffle2!(a, a, [2, 3]); + let b: uint32x2_t = simd_shuffle2!(b, b, [2, 3]); + vmull_u32(a, b) +} + +/// Polynomial multiply long +#[inline] +#[target_feature(enable = "neon,crypto")] +#[cfg_attr(test, assert_instr(pmull))] +pub unsafe fn vmull_p64(a: p64, b: p64) -> p128 { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.pmull64")] + fn vmull_p64_(a: p64, b: p64) -> int8x16_t; + } + transmute(vmull_p64_(a, b)) +} + +/// Polynomial multiply long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(pmull))] +pub unsafe fn vmull_high_p8(a: poly8x16_t, b: poly8x16_t) -> poly16x8_t { + let a: poly8x8_t = simd_shuffle8!(a, a, [8, 9, 10, 11, 12, 13, 14, 15]); + let b: poly8x8_t = simd_shuffle8!(b, b, [8, 9, 10, 11, 12, 13, 14, 15]); + vmull_p8(a, b) +} + +/// Polynomial multiply long +#[inline] +#[target_feature(enable = "neon,crypto")] +#[cfg_attr(test, assert_instr(pmull))] +pub unsafe fn vmull_high_p64(a: poly64x2_t, b: poly64x2_t) -> p128 { + vmull_p64(simd_extract(a, 1), simd_extract(b, 1)) +} + +/// Multiply long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(smull2))] +pub unsafe fn vmull_high_n_s16(a: int16x8_t, b: i16) -> int32x4_t { + vmull_high_s16(a, vdupq_n_s16(b)) +} + +/// Multiply long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(smull2))] +pub unsafe fn vmull_high_n_s32(a: int32x4_t, b: i32) -> int64x2_t { + vmull_high_s32(a, vdupq_n_s32(b)) +} + +/// Multiply long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(umull2))] +pub unsafe fn vmull_high_n_u16(a: uint16x8_t, b: u16) -> uint32x4_t { + vmull_high_u16(a, vdupq_n_u16(b)) +} + +/// Multiply long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(umull2))] +pub unsafe fn vmull_high_n_u32(a: uint32x4_t, b: u32) -> uint64x2_t { + vmull_high_u32(a, vdupq_n_u32(b)) +} + +/// Multiply long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(smull2, LANE = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vmull_high_lane_s16(a: int16x8_t, b: int16x4_t) -> int32x4_t { + static_assert_imm2!(LANE); + vmull_high_s16(a, simd_shuffle8!(b, b, [LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Multiply long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(smull2, LANE = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vmull_high_laneq_s16(a: int16x8_t, b: int16x8_t) -> int32x4_t { + static_assert_imm3!(LANE); + vmull_high_s16(a, simd_shuffle8!(b, b, [LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Multiply long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(smull2, LANE = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vmull_high_lane_s32(a: int32x4_t, b: int32x2_t) -> int64x2_t { + static_assert_imm1!(LANE); + vmull_high_s32(a, simd_shuffle4!(b, b, [LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Multiply long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(smull2, LANE = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vmull_high_laneq_s32(a: int32x4_t, b: int32x4_t) -> int64x2_t { + static_assert_imm2!(LANE); + vmull_high_s32(a, simd_shuffle4!(b, b, [LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Multiply long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(umull2, LANE = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vmull_high_lane_u16(a: uint16x8_t, b: uint16x4_t) -> uint32x4_t { + static_assert_imm2!(LANE); + vmull_high_u16(a, simd_shuffle8!(b, b, [LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Multiply long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(umull2, LANE = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vmull_high_laneq_u16(a: uint16x8_t, b: uint16x8_t) -> uint32x4_t { + static_assert_imm3!(LANE); + vmull_high_u16(a, simd_shuffle8!(b, b, [LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Multiply long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(umull2, LANE = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vmull_high_lane_u32(a: uint32x4_t, b: uint32x2_t) -> uint64x2_t { + static_assert_imm1!(LANE); + vmull_high_u32(a, simd_shuffle4!(b, b, [LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Multiply long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(umull2, LANE = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vmull_high_laneq_u32(a: uint32x4_t, b: uint32x4_t) -> uint64x2_t { + static_assert_imm2!(LANE); + vmull_high_u32(a, simd_shuffle4!(b, b, [LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Floating-point multiply extended +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmulx))] +pub unsafe fn vmulx_f32(a: float32x2_t, b: float32x2_t) -> float32x2_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fmulx.v2f32")] + fn vmulx_f32_(a: float32x2_t, b: float32x2_t) -> float32x2_t; + } + vmulx_f32_(a, b) +} + +/// Floating-point multiply extended +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmulx))] +pub unsafe fn vmulxq_f32(a: float32x4_t, b: float32x4_t) -> float32x4_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fmulx.v4f32")] + fn vmulxq_f32_(a: float32x4_t, b: float32x4_t) -> float32x4_t; + } + vmulxq_f32_(a, b) +} + +/// Floating-point multiply extended +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmulx))] +pub unsafe fn vmulx_f64(a: float64x1_t, b: float64x1_t) -> float64x1_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fmulx.v1f64")] + fn vmulx_f64_(a: float64x1_t, b: float64x1_t) -> float64x1_t; + } + vmulx_f64_(a, b) +} + +/// Floating-point multiply extended +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmulx))] +pub unsafe fn vmulxq_f64(a: float64x2_t, b: float64x2_t) -> float64x2_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fmulx.v2f64")] + fn vmulxq_f64_(a: float64x2_t, b: float64x2_t) -> float64x2_t; + } + vmulxq_f64_(a, b) +} + +/// Floating-point multiply extended +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmulx, LANE = 0))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vmulx_lane_f64(a: float64x1_t, b: float64x1_t) -> float64x1_t { + static_assert!(LANE : i32 where LANE == 0); + vmulx_f64(a, transmute::(simd_extract(b, LANE as u32))) +} + +/// Floating-point multiply extended +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmulx, LANE = 0))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vmulx_laneq_f64(a: float64x1_t, b: float64x2_t) -> float64x1_t { + static_assert_imm1!(LANE); + vmulx_f64(a, transmute::(simd_extract(b, LANE as u32))) +} + +/// Floating-point multiply extended +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmulx, LANE = 0))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vmulx_lane_f32(a: float32x2_t, b: float32x2_t) -> float32x2_t { + static_assert_imm1!(LANE); + vmulx_f32(a, simd_shuffle2!(b, b, [LANE as u32, LANE as u32])) +} + +/// Floating-point multiply extended +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmulx, LANE = 0))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vmulx_laneq_f32(a: float32x2_t, b: float32x4_t) -> float32x2_t { + static_assert_imm2!(LANE); + vmulx_f32(a, simd_shuffle2!(b, b, [LANE as u32, LANE as u32])) +} + +/// Floating-point multiply extended +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmulx, LANE = 0))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vmulxq_lane_f32(a: float32x4_t, b: float32x2_t) -> float32x4_t { + static_assert_imm1!(LANE); + vmulxq_f32(a, simd_shuffle4!(b, b, [LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Floating-point multiply extended +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmulx, LANE = 0))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vmulxq_laneq_f32(a: float32x4_t, b: float32x4_t) -> float32x4_t { + static_assert_imm2!(LANE); + vmulxq_f32(a, simd_shuffle4!(b, b, [LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Floating-point multiply extended +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmulx, LANE = 0))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vmulxq_lane_f64(a: float64x2_t, b: float64x1_t) -> float64x2_t { + static_assert!(LANE : i32 where LANE == 0); + vmulxq_f64(a, simd_shuffle2!(b, b, [LANE as u32, LANE as u32])) +} + +/// Floating-point multiply extended +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmulx, LANE = 0))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vmulxq_laneq_f64(a: float64x2_t, b: float64x2_t) -> float64x2_t { + static_assert_imm1!(LANE); + vmulxq_f64(a, simd_shuffle2!(b, b, [LANE as u32, LANE as u32])) +} + +/// Floating-point multiply extended +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmulx))] +pub unsafe fn vmulxs_f32(a: f32, b: f32) -> f32 { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fmulx.f32")] + fn vmulxs_f32_(a: f32, b: f32) -> f32; + } + vmulxs_f32_(a, b) +} + +/// Floating-point multiply extended +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmulx))] +pub unsafe fn vmulxd_f64(a: f64, b: f64) -> f64 { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fmulx.f64")] + fn vmulxd_f64_(a: f64, b: f64) -> f64; + } + vmulxd_f64_(a, b) +} + +/// Floating-point multiply extended +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmulx, LANE = 0))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vmulxs_lane_f32(a: f32, b: float32x2_t) -> f32 { + static_assert_imm1!(LANE); + vmulxs_f32(a, simd_extract(b, LANE as u32)) +} + +/// Floating-point multiply extended +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmulx, LANE = 0))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vmulxs_laneq_f32(a: f32, b: float32x4_t) -> f32 { + static_assert_imm2!(LANE); + vmulxs_f32(a, simd_extract(b, LANE as u32)) +} + +/// Floating-point multiply extended +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmulx, LANE = 0))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vmulxd_lane_f64(a: f64, b: float64x1_t) -> f64 { + static_assert!(LANE : i32 where LANE == 0); + vmulxd_f64(a, simd_extract(b, LANE as u32)) +} + +/// Floating-point multiply extended +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmulx, LANE = 0))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vmulxd_laneq_f64(a: f64, b: float64x2_t) -> f64 { + static_assert_imm1!(LANE); + vmulxd_f64(a, simd_extract(b, LANE as u32)) +} + +/// Floating-point fused Multiply-Add to accumulator(vector) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmadd))] +pub unsafe fn vfma_f64(a: float64x1_t, b: float64x1_t, c: float64x1_t) -> float64x1_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.fma.v1f64")] + fn vfma_f64_(a: float64x1_t, b: float64x1_t, c: float64x1_t) -> float64x1_t; + } + vfma_f64_(b, c, a) +} + +/// Floating-point fused Multiply-Add to accumulator(vector) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmla))] +pub unsafe fn vfmaq_f64(a: float64x2_t, b: float64x2_t, c: float64x2_t) -> float64x2_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.fma.v2f64")] + fn vfmaq_f64_(a: float64x2_t, b: float64x2_t, c: float64x2_t) -> float64x2_t; + } + vfmaq_f64_(b, c, a) +} + +/// Floating-point fused Multiply-Add to accumulator(vector) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmadd))] +pub unsafe fn vfma_n_f64(a: float64x1_t, b: float64x1_t, c: f64) -> float64x1_t { + vfma_f64(a, b, vdup_n_f64(c)) +} + +/// Floating-point fused Multiply-Add to accumulator(vector) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmla))] +pub unsafe fn vfmaq_n_f64(a: float64x2_t, b: float64x2_t, c: f64) -> float64x2_t { + vfmaq_f64(a, b, vdupq_n_f64(c)) +} + +/// Floating-point fused multiply-add to accumulator +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmla, LANE = 0))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vfma_lane_f32(a: float32x2_t, b: float32x2_t, c: float32x2_t) -> float32x2_t { + static_assert_imm1!(LANE); + vfma_f32(a, b, vdup_n_f32(simd_extract(c, LANE as u32))) +} + +/// Floating-point fused multiply-add to accumulator +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmla, LANE = 0))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vfma_laneq_f32(a: float32x2_t, b: float32x2_t, c: float32x4_t) -> float32x2_t { + static_assert_imm2!(LANE); + vfma_f32(a, b, vdup_n_f32(simd_extract(c, LANE as u32))) +} + +/// Floating-point fused multiply-add to accumulator +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmla, LANE = 0))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vfmaq_lane_f32(a: float32x4_t, b: float32x4_t, c: float32x2_t) -> float32x4_t { + static_assert_imm1!(LANE); + vfmaq_f32(a, b, vdupq_n_f32(simd_extract(c, LANE as u32))) +} + +/// Floating-point fused multiply-add to accumulator +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmla, LANE = 0))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vfmaq_laneq_f32(a: float32x4_t, b: float32x4_t, c: float32x4_t) -> float32x4_t { + static_assert_imm2!(LANE); + vfmaq_f32(a, b, vdupq_n_f32(simd_extract(c, LANE as u32))) +} + +/// Floating-point fused multiply-add to accumulator +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmadd, LANE = 0))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vfma_lane_f64(a: float64x1_t, b: float64x1_t, c: float64x1_t) -> float64x1_t { + static_assert!(LANE : i32 where LANE == 0); + vfma_f64(a, b, vdup_n_f64(simd_extract(c, LANE as u32))) +} + +/// Floating-point fused multiply-add to accumulator +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmla, LANE = 0))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vfma_laneq_f64(a: float64x1_t, b: float64x1_t, c: float64x2_t) -> float64x1_t { + static_assert_imm1!(LANE); + vfma_f64(a, b, vdup_n_f64(simd_extract(c, LANE as u32))) +} + +/// Floating-point fused multiply-add to accumulator +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmla, LANE = 0))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vfmaq_lane_f64(a: float64x2_t, b: float64x2_t, c: float64x1_t) -> float64x2_t { + static_assert!(LANE : i32 where LANE == 0); + vfmaq_f64(a, b, vdupq_n_f64(simd_extract(c, LANE as u32))) +} + +/// Floating-point fused multiply-add to accumulator +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmla, LANE = 0))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vfmaq_laneq_f64(a: float64x2_t, b: float64x2_t, c: float64x2_t) -> float64x2_t { + static_assert_imm1!(LANE); + vfmaq_f64(a, b, vdupq_n_f64(simd_extract(c, LANE as u32))) +} + +/// Floating-point fused multiply-add to accumulator +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmla, LANE = 0))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vfmas_lane_f32(a: f32, b: f32, c: float32x2_t) -> f32 { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.fma.f32")] + fn vfmas_lane_f32_(a: f32, b: f32, c: f32) -> f32; + } + static_assert_imm1!(LANE); + let c: f32 = simd_extract(c, LANE as u32); + vfmas_lane_f32_(b, c, a) +} + +/// Floating-point fused multiply-add to accumulator +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmla, LANE = 0))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vfmas_laneq_f32(a: f32, b: f32, c: float32x4_t) -> f32 { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.fma.f32")] + fn vfmas_laneq_f32_(a: f32, b: f32, c: f32) -> f32; + } + static_assert_imm2!(LANE); + let c: f32 = simd_extract(c, LANE as u32); + vfmas_laneq_f32_(b, c, a) +} + +/// Floating-point fused multiply-add to accumulator +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmadd, LANE = 0))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vfmad_lane_f64(a: f64, b: f64, c: float64x1_t) -> f64 { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.fma.f64")] + fn vfmad_lane_f64_(a: f64, b: f64, c: f64) -> f64; + } + static_assert!(LANE : i32 where LANE == 0); + let c: f64 = simd_extract(c, LANE as u32); + vfmad_lane_f64_(b, c, a) +} + +/// Floating-point fused multiply-add to accumulator +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmla, LANE = 0))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vfmad_laneq_f64(a: f64, b: f64, c: float64x2_t) -> f64 { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.fma.f64")] + fn vfmad_laneq_f64_(a: f64, b: f64, c: f64) -> f64; + } + static_assert_imm1!(LANE); + let c: f64 = simd_extract(c, LANE as u32); + vfmad_laneq_f64_(b, c, a) +} + +/// Floating-point fused multiply-subtract from accumulator +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmsub))] +pub unsafe fn vfms_f64(a: float64x1_t, b: float64x1_t, c: float64x1_t) -> float64x1_t { + let b: float64x1_t = simd_neg(b); + vfma_f64(a, b, c) +} + +/// Floating-point fused multiply-subtract from accumulator +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmls))] +pub unsafe fn vfmsq_f64(a: float64x2_t, b: float64x2_t, c: float64x2_t) -> float64x2_t { + let b: float64x2_t = simd_neg(b); + vfmaq_f64(a, b, c) +} + +/// Floating-point fused Multiply-subtract to accumulator(vector) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmsub))] +pub unsafe fn vfms_n_f64(a: float64x1_t, b: float64x1_t, c: f64) -> float64x1_t { + vfms_f64(a, b, vdup_n_f64(c)) +} + +/// Floating-point fused Multiply-subtract to accumulator(vector) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmls))] +pub unsafe fn vfmsq_n_f64(a: float64x2_t, b: float64x2_t, c: f64) -> float64x2_t { + vfmsq_f64(a, b, vdupq_n_f64(c)) +} + +/// Floating-point fused multiply-subtract to accumulator +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmls, LANE = 0))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vfms_lane_f32(a: float32x2_t, b: float32x2_t, c: float32x2_t) -> float32x2_t { + static_assert_imm1!(LANE); + vfms_f32(a, b, vdup_n_f32(simd_extract(c, LANE as u32))) +} + +/// Floating-point fused multiply-subtract to accumulator +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmls, LANE = 0))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vfms_laneq_f32(a: float32x2_t, b: float32x2_t, c: float32x4_t) -> float32x2_t { + static_assert_imm2!(LANE); + vfms_f32(a, b, vdup_n_f32(simd_extract(c, LANE as u32))) +} + +/// Floating-point fused multiply-subtract to accumulator +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmls, LANE = 0))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vfmsq_lane_f32(a: float32x4_t, b: float32x4_t, c: float32x2_t) -> float32x4_t { + static_assert_imm1!(LANE); + vfmsq_f32(a, b, vdupq_n_f32(simd_extract(c, LANE as u32))) +} + +/// Floating-point fused multiply-subtract to accumulator +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmls, LANE = 0))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vfmsq_laneq_f32(a: float32x4_t, b: float32x4_t, c: float32x4_t) -> float32x4_t { + static_assert_imm2!(LANE); + vfmsq_f32(a, b, vdupq_n_f32(simd_extract(c, LANE as u32))) +} + +/// Floating-point fused multiply-subtract to accumulator +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmsub, LANE = 0))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vfms_lane_f64(a: float64x1_t, b: float64x1_t, c: float64x1_t) -> float64x1_t { + static_assert!(LANE : i32 where LANE == 0); + vfms_f64(a, b, vdup_n_f64(simd_extract(c, LANE as u32))) +} + +/// Floating-point fused multiply-subtract to accumulator +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmls, LANE = 0))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vfms_laneq_f64(a: float64x1_t, b: float64x1_t, c: float64x2_t) -> float64x1_t { + static_assert_imm1!(LANE); + vfms_f64(a, b, vdup_n_f64(simd_extract(c, LANE as u32))) +} + +/// Floating-point fused multiply-subtract to accumulator +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmls, LANE = 0))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vfmsq_lane_f64(a: float64x2_t, b: float64x2_t, c: float64x1_t) -> float64x2_t { + static_assert!(LANE : i32 where LANE == 0); + vfmsq_f64(a, b, vdupq_n_f64(simd_extract(c, LANE as u32))) +} + +/// Floating-point fused multiply-subtract to accumulator +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmls, LANE = 0))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vfmsq_laneq_f64(a: float64x2_t, b: float64x2_t, c: float64x2_t) -> float64x2_t { + static_assert_imm1!(LANE); + vfmsq_f64(a, b, vdupq_n_f64(simd_extract(c, LANE as u32))) +} + +/// Floating-point fused multiply-subtract to accumulator +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmls, LANE = 0))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vfmss_lane_f32(a: f32, b: f32, c: float32x2_t) -> f32 { + vfmas_lane_f32::(a, -b, c) +} + +/// Floating-point fused multiply-subtract to accumulator +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmls, LANE = 0))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vfmss_laneq_f32(a: f32, b: f32, c: float32x4_t) -> f32 { + vfmas_laneq_f32::(a, -b, c) +} + +/// Floating-point fused multiply-subtract to accumulator +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmsub, LANE = 0))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vfmsd_lane_f64(a: f64, b: f64, c: float64x1_t) -> f64 { + vfmad_lane_f64::(a, -b, c) +} + +/// Floating-point fused multiply-subtract to accumulator +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmls, LANE = 0))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vfmsd_laneq_f64(a: f64, b: f64, c: float64x2_t) -> f64 { + vfmad_laneq_f64::(a, -b, c) +} + +/// Divide +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fdiv))] +pub unsafe fn vdiv_f32(a: float32x2_t, b: float32x2_t) -> float32x2_t { + simd_div(a, b) +} + +/// Divide +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fdiv))] +pub unsafe fn vdivq_f32(a: float32x4_t, b: float32x4_t) -> float32x4_t { + simd_div(a, b) +} + +/// Divide +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fdiv))] +pub unsafe fn vdiv_f64(a: float64x1_t, b: float64x1_t) -> float64x1_t { + simd_div(a, b) +} + +/// Divide +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fdiv))] +pub unsafe fn vdivq_f64(a: float64x2_t, b: float64x2_t) -> float64x2_t { + simd_div(a, b) +} + +/// Subtract +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fsub))] +pub unsafe fn vsub_f64(a: float64x1_t, b: float64x1_t) -> float64x1_t { + simd_sub(a, b) +} + +/// Subtract +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fsub))] +pub unsafe fn vsubq_f64(a: float64x2_t, b: float64x2_t) -> float64x2_t { + simd_sub(a, b) +} + +/// Signed Add Long across Vector +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(saddlv))] +pub unsafe fn vaddlv_s16(a: int16x4_t) -> i32 { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.saddlv.i32.v4i16")] + fn vaddlv_s16_(a: int16x4_t) -> i32; + } + vaddlv_s16_(a) +} + +/// Signed Add Long across Vector +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(saddlv))] +pub unsafe fn vaddlvq_s16(a: int16x8_t) -> i32 { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.saddlv.i32.v8i16")] + fn vaddlvq_s16_(a: int16x8_t) -> i32; + } + vaddlvq_s16_(a) +} + +/// Signed Add Long across Vector +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(saddlp))] +pub unsafe fn vaddlv_s32(a: int32x2_t) -> i64 { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.saddlv.i64.v2i32")] + fn vaddlv_s32_(a: int32x2_t) -> i64; + } + vaddlv_s32_(a) +} + +/// Signed Add Long across Vector +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(saddlv))] +pub unsafe fn vaddlvq_s32(a: int32x4_t) -> i64 { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.saddlv.i64.v4i32")] + fn vaddlvq_s32_(a: int32x4_t) -> i64; + } + vaddlvq_s32_(a) +} + +/// Unsigned Add Long across Vector +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uaddlv))] +pub unsafe fn vaddlv_u16(a: uint16x4_t) -> u32 { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uaddlv.i32.v4i16")] + fn vaddlv_u16_(a: uint16x4_t) -> u32; + } + vaddlv_u16_(a) +} + +/// Unsigned Add Long across Vector +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uaddlv))] +pub unsafe fn vaddlvq_u16(a: uint16x8_t) -> u32 { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uaddlv.i32.v8i16")] + fn vaddlvq_u16_(a: uint16x8_t) -> u32; + } + vaddlvq_u16_(a) +} + +/// Unsigned Add Long across Vector +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uaddlp))] +pub unsafe fn vaddlv_u32(a: uint32x2_t) -> u64 { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uaddlv.i64.v2i32")] + fn vaddlv_u32_(a: uint32x2_t) -> u64; + } + vaddlv_u32_(a) +} + +/// Unsigned Add Long across Vector +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uaddlv))] +pub unsafe fn vaddlvq_u32(a: uint32x4_t) -> u64 { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uaddlv.i64.v4i32")] + fn vaddlvq_u32_(a: uint32x4_t) -> u64; + } + vaddlvq_u32_(a) +} + +/// Signed Subtract Wide +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ssubw))] +pub unsafe fn vsubw_high_s8(a: int16x8_t, b: int8x16_t) -> int16x8_t { + let c: int8x8_t = simd_shuffle8!(b, b, [8, 9, 10, 11, 12, 13, 14, 15]); + simd_sub(a, simd_cast(c)) +} + +/// Signed Subtract Wide +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ssubw))] +pub unsafe fn vsubw_high_s16(a: int32x4_t, b: int16x8_t) -> int32x4_t { + let c: int16x4_t = simd_shuffle4!(b, b, [4, 5, 6, 7]); + simd_sub(a, simd_cast(c)) +} + +/// Signed Subtract Wide +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ssubw))] +pub unsafe fn vsubw_high_s32(a: int64x2_t, b: int32x4_t) -> int64x2_t { + let c: int32x2_t = simd_shuffle2!(b, b, [2, 3]); + simd_sub(a, simd_cast(c)) +} + +/// Unsigned Subtract Wide +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(usubw))] +pub unsafe fn vsubw_high_u8(a: uint16x8_t, b: uint8x16_t) -> uint16x8_t { + let c: uint8x8_t = simd_shuffle8!(b, b, [8, 9, 10, 11, 12, 13, 14, 15]); + simd_sub(a, simd_cast(c)) +} + +/// Unsigned Subtract Wide +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(usubw))] +pub unsafe fn vsubw_high_u16(a: uint32x4_t, b: uint16x8_t) -> uint32x4_t { + let c: uint16x4_t = simd_shuffle4!(b, b, [4, 5, 6, 7]); + simd_sub(a, simd_cast(c)) +} + +/// Unsigned Subtract Wide +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(usubw))] +pub unsafe fn vsubw_high_u32(a: uint64x2_t, b: uint32x4_t) -> uint64x2_t { + let c: uint32x2_t = simd_shuffle2!(b, b, [2, 3]); + simd_sub(a, simd_cast(c)) +} + +/// Signed Subtract Long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ssubl))] +pub unsafe fn vsubl_high_s8(a: int8x16_t, b: int8x16_t) -> int16x8_t { + let c: int8x8_t = simd_shuffle8!(a, a, [8, 9, 10, 11, 12, 13, 14, 15]); + let d: int16x8_t = simd_cast(c); + let e: int8x8_t = simd_shuffle8!(b, b, [8, 9, 10, 11, 12, 13, 14, 15]); + let f: int16x8_t = simd_cast(e); + simd_sub(d, f) +} + +/// Signed Subtract Long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ssubl))] +pub unsafe fn vsubl_high_s16(a: int16x8_t, b: int16x8_t) -> int32x4_t { + let c: int16x4_t = simd_shuffle4!(a, a, [4, 5, 6, 7]); + let d: int32x4_t = simd_cast(c); + let e: int16x4_t = simd_shuffle4!(b, b, [4, 5, 6, 7]); + let f: int32x4_t = simd_cast(e); + simd_sub(d, f) +} + +/// Signed Subtract Long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ssubl))] +pub unsafe fn vsubl_high_s32(a: int32x4_t, b: int32x4_t) -> int64x2_t { + let c: int32x2_t = simd_shuffle2!(a, a, [2, 3]); + let d: int64x2_t = simd_cast(c); + let e: int32x2_t = simd_shuffle2!(b, b, [2, 3]); + let f: int64x2_t = simd_cast(e); + simd_sub(d, f) +} + +/// Unsigned Subtract Long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(usubl))] +pub unsafe fn vsubl_high_u8(a: uint8x16_t, b: uint8x16_t) -> uint16x8_t { + let c: uint8x8_t = simd_shuffle8!(a, a, [8, 9, 10, 11, 12, 13, 14, 15]); + let d: uint16x8_t = simd_cast(c); + let e: uint8x8_t = simd_shuffle8!(b, b, [8, 9, 10, 11, 12, 13, 14, 15]); + let f: uint16x8_t = simd_cast(e); + simd_sub(d, f) +} + +/// Unsigned Subtract Long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(usubl))] +pub unsafe fn vsubl_high_u16(a: uint16x8_t, b: uint16x8_t) -> uint32x4_t { + let c: uint16x4_t = simd_shuffle4!(a, a, [4, 5, 6, 7]); + let d: uint32x4_t = simd_cast(c); + let e: uint16x4_t = simd_shuffle4!(b, b, [4, 5, 6, 7]); + let f: uint32x4_t = simd_cast(e); + simd_sub(d, f) +} + +/// Unsigned Subtract Long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(usubl))] +pub unsafe fn vsubl_high_u32(a: uint32x4_t, b: uint32x4_t) -> uint64x2_t { + let c: uint32x2_t = simd_shuffle2!(a, a, [2, 3]); + let d: uint64x2_t = simd_cast(c); + let e: uint32x2_t = simd_shuffle2!(b, b, [2, 3]); + let f: uint64x2_t = simd_cast(e); + simd_sub(d, f) +} + +/// Maximum (vector) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmax))] +pub unsafe fn vmax_f64(a: float64x1_t, b: float64x1_t) -> float64x1_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fmax.v1f64")] + fn vmax_f64_(a: float64x1_t, b: float64x1_t) -> float64x1_t; + } + vmax_f64_(a, b) +} + +/// Maximum (vector) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmax))] +pub unsafe fn vmaxq_f64(a: float64x2_t, b: float64x2_t) -> float64x2_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fmax.v2f64")] + fn vmaxq_f64_(a: float64x2_t, b: float64x2_t) -> float64x2_t; + } + vmaxq_f64_(a, b) +} + +/// Floating-point Maximun Number (vector) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmaxnm))] +pub unsafe fn vmaxnm_f64(a: float64x1_t, b: float64x1_t) -> float64x1_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fmaxnm.v1f64")] + fn vmaxnm_f64_(a: float64x1_t, b: float64x1_t) -> float64x1_t; + } + vmaxnm_f64_(a, b) +} + +/// Floating-point Maximun Number (vector) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmaxnm))] +pub unsafe fn vmaxnmq_f64(a: float64x2_t, b: float64x2_t) -> float64x2_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fmaxnm.v2f64")] + fn vmaxnmq_f64_(a: float64x2_t, b: float64x2_t) -> float64x2_t; + } + vmaxnmq_f64_(a, b) +} + +/// Floating-point Maximum Number Pairwise (vector). +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmaxnmp))] +pub unsafe fn vpmaxnm_f32(a: float32x2_t, b: float32x2_t) -> float32x2_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fmaxnmp.v2f32")] + fn vpmaxnm_f32_(a: float32x2_t, b: float32x2_t) -> float32x2_t; + } + vpmaxnm_f32_(a, b) +} + +/// Floating-point Maximum Number Pairwise (vector). +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmaxnmp))] +pub unsafe fn vpmaxnmq_f64(a: float64x2_t, b: float64x2_t) -> float64x2_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fmaxnmp.v2f64")] + fn vpmaxnmq_f64_(a: float64x2_t, b: float64x2_t) -> float64x2_t; + } + vpmaxnmq_f64_(a, b) +} + +/// Floating-point Maximum Number Pairwise (vector). +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmaxnmp))] +pub unsafe fn vpmaxnmq_f32(a: float32x4_t, b: float32x4_t) -> float32x4_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fmaxnmp.v4f32")] + fn vpmaxnmq_f32_(a: float32x4_t, b: float32x4_t) -> float32x4_t; + } + vpmaxnmq_f32_(a, b) +} + +/// Minimum (vector) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmin))] +pub unsafe fn vmin_f64(a: float64x1_t, b: float64x1_t) -> float64x1_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fmin.v1f64")] + fn vmin_f64_(a: float64x1_t, b: float64x1_t) -> float64x1_t; + } + vmin_f64_(a, b) +} + +/// Minimum (vector) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmin))] +pub unsafe fn vminq_f64(a: float64x2_t, b: float64x2_t) -> float64x2_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fmin.v2f64")] + fn vminq_f64_(a: float64x2_t, b: float64x2_t) -> float64x2_t; + } + vminq_f64_(a, b) +} + +/// Floating-point Minimun Number (vector) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fminnm))] +pub unsafe fn vminnm_f64(a: float64x1_t, b: float64x1_t) -> float64x1_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fminnm.v1f64")] + fn vminnm_f64_(a: float64x1_t, b: float64x1_t) -> float64x1_t; + } + vminnm_f64_(a, b) +} + +/// Floating-point Minimun Number (vector) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fminnm))] +pub unsafe fn vminnmq_f64(a: float64x2_t, b: float64x2_t) -> float64x2_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fminnm.v2f64")] + fn vminnmq_f64_(a: float64x2_t, b: float64x2_t) -> float64x2_t; + } + vminnmq_f64_(a, b) +} + +/// Floating-point Minimum Number Pairwise (vector). +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fminnmp))] +pub unsafe fn vpminnm_f32(a: float32x2_t, b: float32x2_t) -> float32x2_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fminnmp.v2f32")] + fn vpminnm_f32_(a: float32x2_t, b: float32x2_t) -> float32x2_t; + } + vpminnm_f32_(a, b) +} + +/// Floating-point Minimum Number Pairwise (vector). +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fminnmp))] +pub unsafe fn vpminnmq_f64(a: float64x2_t, b: float64x2_t) -> float64x2_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fminnmp.v2f64")] + fn vpminnmq_f64_(a: float64x2_t, b: float64x2_t) -> float64x2_t; + } + vpminnmq_f64_(a, b) +} + +/// Floating-point Minimum Number Pairwise (vector). +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fminnmp))] +pub unsafe fn vpminnmq_f32(a: float32x4_t, b: float32x4_t) -> float32x4_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fminnmp.v4f32")] + fn vpminnmq_f32_(a: float32x4_t, b: float32x4_t) -> float32x4_t; + } + vpminnmq_f32_(a, b) +} + +/// Signed saturating doubling multiply long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqdmull))] +pub unsafe fn vqdmullh_s16(a: i16, b: i16) -> i32 { + let a: int16x4_t = vdup_n_s16(a); + let b: int16x4_t = vdup_n_s16(b); + simd_extract(vqdmull_s16(a, b), 0) +} + +/// Signed saturating doubling multiply long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqdmull))] +pub unsafe fn vqdmulls_s32(a: i32, b: i32) -> i64 { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqdmulls.scalar")] + fn vqdmulls_s32_(a: i32, b: i32) -> i64; + } + vqdmulls_s32_(a, b) +} + +/// Signed saturating doubling multiply long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqdmull2))] +pub unsafe fn vqdmull_high_s16(a: int16x8_t, b: int16x8_t) -> int32x4_t { + let a: int16x4_t = simd_shuffle4!(a, a, [4, 5, 6, 7]); + let b: int16x4_t = simd_shuffle4!(b, b, [4, 5, 6, 7]); + vqdmull_s16(a, b) +} + +/// Signed saturating doubling multiply long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqdmull2))] +pub unsafe fn vqdmull_high_s32(a: int32x4_t, b: int32x4_t) -> int64x2_t { + let a: int32x2_t = simd_shuffle2!(a, a, [2, 3]); + let b: int32x2_t = simd_shuffle2!(b, b, [2, 3]); + vqdmull_s32(a, b) +} + +/// Signed saturating doubling multiply long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqdmull2))] +pub unsafe fn vqdmull_high_n_s16(a: int16x8_t, b: i16) -> int32x4_t { + let a: int16x4_t = simd_shuffle4!(a, a, [4, 5, 6, 7]); + let b: int16x4_t = vdup_n_s16(b); + vqdmull_s16(a, b) +} + +/// Signed saturating doubling multiply long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqdmull2))] +pub unsafe fn vqdmull_high_n_s32(a: int32x4_t, b: i32) -> int64x2_t { + let a: int32x2_t = simd_shuffle2!(a, a, [2, 3]); + let b: int32x2_t = vdup_n_s32(b); + vqdmull_s32(a, b) +} + +/// Vector saturating doubling long multiply by scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqdmull, N = 4))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vqdmull_laneq_s16(a: int16x4_t, b: int16x8_t) -> int32x4_t { + static_assert_imm3!(N); + let b: int16x4_t = simd_shuffle4!(b, b, [N as u32, N as u32, N as u32, N as u32]); + vqdmull_s16(a, b) +} + +/// Vector saturating doubling long multiply by scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqdmull, N = 2))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vqdmull_laneq_s32(a: int32x2_t, b: int32x4_t) -> int64x2_t { + static_assert_imm2!(N); + let b: int32x2_t = simd_shuffle2!(b, b, [N as u32, N as u32]); + vqdmull_s32(a, b) +} + +/// Signed saturating doubling multiply long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqdmull, N = 2))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vqdmullh_lane_s16(a: i16, b: int16x4_t) -> i32 { + static_assert_imm2!(N); + let b: i16 = simd_extract(b, N as u32); + vqdmullh_s16(a, b) +} + +/// Signed saturating doubling multiply long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqdmull, N = 4))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vqdmullh_laneq_s16(a: i16, b: int16x8_t) -> i32 { + static_assert_imm3!(N); + let b: i16 = simd_extract(b, N as u32); + vqdmullh_s16(a, b) +} + +/// Signed saturating doubling multiply long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqdmull, N = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vqdmulls_lane_s32(a: i32, b: int32x2_t) -> i64 { + static_assert_imm1!(N); + let b: i32 = simd_extract(b, N as u32); + vqdmulls_s32(a, b) +} + +/// Signed saturating doubling multiply long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqdmull, N = 2))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vqdmulls_laneq_s32(a: i32, b: int32x4_t) -> i64 { + static_assert_imm2!(N); + let b: i32 = simd_extract(b, N as u32); + vqdmulls_s32(a, b) +} + +/// Signed saturating doubling multiply long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqdmull2, N = 2))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vqdmull_high_lane_s16(a: int16x8_t, b: int16x4_t) -> int32x4_t { + static_assert_imm2!(N); + let a: int16x4_t = simd_shuffle4!(a, a, [4, 5, 6, 7]); + let b: int16x4_t = simd_shuffle4!(b, b, [N as u32, N as u32, N as u32, N as u32]); + vqdmull_s16(a, b) +} + +/// Signed saturating doubling multiply long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqdmull2, N = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vqdmull_high_lane_s32(a: int32x4_t, b: int32x2_t) -> int64x2_t { + static_assert_imm1!(N); + let a: int32x2_t = simd_shuffle2!(a, a, [2, 3]); + let b: int32x2_t = simd_shuffle2!(b, b, [N as u32, N as u32]); + vqdmull_s32(a, b) +} + +/// Signed saturating doubling multiply long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqdmull2, N = 4))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vqdmull_high_laneq_s16(a: int16x8_t, b: int16x8_t) -> int32x4_t { + static_assert_imm3!(N); + let a: int16x4_t = simd_shuffle4!(a, a, [4, 5, 6, 7]); + let b: int16x4_t = simd_shuffle4!(b, b, [N as u32, N as u32, N as u32, N as u32]); + vqdmull_s16(a, b) +} + +/// Signed saturating doubling multiply long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqdmull2, N = 2))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vqdmull_high_laneq_s32(a: int32x4_t, b: int32x4_t) -> int64x2_t { + static_assert_imm2!(N); + let a: int32x2_t = simd_shuffle2!(a, a, [2, 3]); + let b: int32x2_t = simd_shuffle2!(b, b, [N as u32, N as u32]); + vqdmull_s32(a, b) +} + +/// Signed saturating doubling multiply-add long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqdmlal2))] +pub unsafe fn vqdmlal_high_s16(a: int32x4_t, b: int16x8_t, c: int16x8_t) -> int32x4_t { + vqaddq_s32(a, vqdmull_high_s16(b, c)) +} + +/// Signed saturating doubling multiply-add long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqdmlal2))] +pub unsafe fn vqdmlal_high_s32(a: int64x2_t, b: int32x4_t, c: int32x4_t) -> int64x2_t { + vqaddq_s64(a, vqdmull_high_s32(b, c)) +} + +/// Signed saturating doubling multiply-add long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqdmlal2))] +pub unsafe fn vqdmlal_high_n_s16(a: int32x4_t, b: int16x8_t, c: i16) -> int32x4_t { + vqaddq_s32(a, vqdmull_high_n_s16(b, c)) +} + +/// Signed saturating doubling multiply-add long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqdmlal2))] +pub unsafe fn vqdmlal_high_n_s32(a: int64x2_t, b: int32x4_t, c: i32) -> int64x2_t { + vqaddq_s64(a, vqdmull_high_n_s32(b, c)) +} + +/// Vector widening saturating doubling multiply accumulate with scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqdmlal, N = 2))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vqdmlal_laneq_s16(a: int32x4_t, b: int16x4_t, c: int16x8_t) -> int32x4_t { + static_assert_imm3!(N); + vqaddq_s32(a, vqdmull_laneq_s16::(b, c)) +} + +/// Vector widening saturating doubling multiply accumulate with scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqdmlal, N = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vqdmlal_laneq_s32(a: int64x2_t, b: int32x2_t, c: int32x4_t) -> int64x2_t { + static_assert_imm2!(N); + vqaddq_s64(a, vqdmull_laneq_s32::(b, c)) +} + +/// Signed saturating doubling multiply-add long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqdmlal2, N = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vqdmlal_high_lane_s16(a: int32x4_t, b: int16x8_t, c: int16x4_t) -> int32x4_t { + static_assert_imm2!(N); + vqaddq_s32(a, vqdmull_high_lane_s16::(b, c)) +} + +/// Signed saturating doubling multiply-add long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqdmlal2, N = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vqdmlal_high_laneq_s16(a: int32x4_t, b: int16x8_t, c: int16x8_t) -> int32x4_t { + static_assert_imm3!(N); + vqaddq_s32(a, vqdmull_high_laneq_s16::(b, c)) +} + +/// Signed saturating doubling multiply-add long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqdmlal2, N = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vqdmlal_high_lane_s32(a: int64x2_t, b: int32x4_t, c: int32x2_t) -> int64x2_t { + static_assert_imm1!(N); + vqaddq_s64(a, vqdmull_high_lane_s32::(b, c)) +} + +/// Signed saturating doubling multiply-add long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqdmlal2, N = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vqdmlal_high_laneq_s32(a: int64x2_t, b: int32x4_t, c: int32x4_t) -> int64x2_t { + static_assert_imm2!(N); + vqaddq_s64(a, vqdmull_high_laneq_s32::(b, c)) +} + +/// Signed saturating doubling multiply-subtract long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqdmlsl2))] +pub unsafe fn vqdmlsl_high_s16(a: int32x4_t, b: int16x8_t, c: int16x8_t) -> int32x4_t { + vqsubq_s32(a, vqdmull_high_s16(b, c)) +} + +/// Signed saturating doubling multiply-subtract long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqdmlsl2))] +pub unsafe fn vqdmlsl_high_s32(a: int64x2_t, b: int32x4_t, c: int32x4_t) -> int64x2_t { + vqsubq_s64(a, vqdmull_high_s32(b, c)) +} + +/// Signed saturating doubling multiply-subtract long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqdmlsl2))] +pub unsafe fn vqdmlsl_high_n_s16(a: int32x4_t, b: int16x8_t, c: i16) -> int32x4_t { + vqsubq_s32(a, vqdmull_high_n_s16(b, c)) +} + +/// Signed saturating doubling multiply-subtract long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqdmlsl2))] +pub unsafe fn vqdmlsl_high_n_s32(a: int64x2_t, b: int32x4_t, c: i32) -> int64x2_t { + vqsubq_s64(a, vqdmull_high_n_s32(b, c)) +} + +/// Vector widening saturating doubling multiply subtract with scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqdmlsl, N = 2))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vqdmlsl_laneq_s16(a: int32x4_t, b: int16x4_t, c: int16x8_t) -> int32x4_t { + static_assert_imm3!(N); + vqsubq_s32(a, vqdmull_laneq_s16::(b, c)) +} + +/// Vector widening saturating doubling multiply subtract with scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqdmlsl, N = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vqdmlsl_laneq_s32(a: int64x2_t, b: int32x2_t, c: int32x4_t) -> int64x2_t { + static_assert_imm2!(N); + vqsubq_s64(a, vqdmull_laneq_s32::(b, c)) +} + +/// Signed saturating doubling multiply-subtract long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqdmlsl2, N = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vqdmlsl_high_lane_s16(a: int32x4_t, b: int16x8_t, c: int16x4_t) -> int32x4_t { + static_assert_imm2!(N); + vqsubq_s32(a, vqdmull_high_lane_s16::(b, c)) +} + +/// Signed saturating doubling multiply-subtract long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqdmlsl2, N = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vqdmlsl_high_laneq_s16(a: int32x4_t, b: int16x8_t, c: int16x8_t) -> int32x4_t { + static_assert_imm3!(N); + vqsubq_s32(a, vqdmull_high_laneq_s16::(b, c)) +} + +/// Signed saturating doubling multiply-subtract long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqdmlsl2, N = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vqdmlsl_high_lane_s32(a: int64x2_t, b: int32x4_t, c: int32x2_t) -> int64x2_t { + static_assert_imm1!(N); + vqsubq_s64(a, vqdmull_high_lane_s32::(b, c)) +} + +/// Signed saturating doubling multiply-subtract long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqdmlsl2, N = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vqdmlsl_high_laneq_s32(a: int64x2_t, b: int32x4_t, c: int32x4_t) -> int64x2_t { + static_assert_imm2!(N); + vqsubq_s64(a, vqdmull_high_laneq_s32::(b, c)) +} + +/// Signed saturating doubling multiply returning high half +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqdmulh))] +pub unsafe fn vqdmulhh_s16(a: i16, b: i16) -> i16 { + let a: int16x4_t = vdup_n_s16(a); + let b: int16x4_t = vdup_n_s16(b); + simd_extract(vqdmulh_s16(a, b), 0) +} + +/// Signed saturating doubling multiply returning high half +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqdmulh))] +pub unsafe fn vqdmulhs_s32(a: i32, b: i32) -> i32 { + let a: int32x2_t = vdup_n_s32(a); + let b: int32x2_t = vdup_n_s32(b); + simd_extract(vqdmulh_s32(a, b), 0) +} + +/// Signed saturating doubling multiply returning high half +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqdmulh, N = 2))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vqdmulhh_lane_s16(a: i16, b: int16x4_t) -> i16 { + static_assert_imm2!(N); + let b: i16 = simd_extract(b, N as u32); + vqdmulhh_s16(a, b) +} + +/// Signed saturating doubling multiply returning high half +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqdmulh, N = 2))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vqdmulhh_laneq_s16(a: i16, b: int16x8_t) -> i16 { + static_assert_imm3!(N); + let b: i16 = simd_extract(b, N as u32); + vqdmulhh_s16(a, b) +} + +/// Signed saturating doubling multiply returning high half +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqdmulh, N = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vqdmulhs_lane_s32(a: i32, b: int32x2_t) -> i32 { + static_assert_imm1!(N); + let b: i32 = simd_extract(b, N as u32); + vqdmulhs_s32(a, b) +} + +/// Signed saturating doubling multiply returning high half +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqdmulh, N = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vqdmulhs_laneq_s32(a: i32, b: int32x4_t) -> i32 { + static_assert_imm2!(N); + let b: i32 = simd_extract(b, N as u32); + vqdmulhs_s32(a, b) +} + +/// Saturating extract narrow +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqxtn))] +pub unsafe fn vqmovnh_s16(a: i16) -> i8 { + simd_extract(vqmovn_s16(vdupq_n_s16(a)), 0) +} + +/// Saturating extract narrow +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqxtn))] +pub unsafe fn vqmovns_s32(a: i32) -> i16 { + simd_extract(vqmovn_s32(vdupq_n_s32(a)), 0) +} + +/// Saturating extract narrow +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uqxtn))] +pub unsafe fn vqmovnh_u16(a: u16) -> u8 { + simd_extract(vqmovn_u16(vdupq_n_u16(a)), 0) +} + +/// Saturating extract narrow +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uqxtn))] +pub unsafe fn vqmovns_u32(a: u32) -> u16 { + simd_extract(vqmovn_u32(vdupq_n_u32(a)), 0) +} + +/// Saturating extract narrow +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqxtn))] +pub unsafe fn vqmovnd_s64(a: i64) -> i32 { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.scalar.sqxtn.i32.i64")] + fn vqmovnd_s64_(a: i64) -> i32; + } + vqmovnd_s64_(a) +} + +/// Saturating extract narrow +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uqxtn))] +pub unsafe fn vqmovnd_u64(a: u64) -> u32 { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.scalar.uqxtn.i32.i64")] + fn vqmovnd_u64_(a: u64) -> u32; + } + vqmovnd_u64_(a) +} + +/// Signed saturating extract narrow +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqxtn2))] +pub unsafe fn vqmovn_high_s16(a: int8x8_t, b: int16x8_t) -> int8x16_t { + simd_shuffle16!(a, vqmovn_s16(b), [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]) +} + +/// Signed saturating extract narrow +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqxtn2))] +pub unsafe fn vqmovn_high_s32(a: int16x4_t, b: int32x4_t) -> int16x8_t { + simd_shuffle8!(a, vqmovn_s32(b), [0, 1, 2, 3, 4, 5, 6, 7]) +} + +/// Signed saturating extract narrow +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqxtn2))] +pub unsafe fn vqmovn_high_s64(a: int32x2_t, b: int64x2_t) -> int32x4_t { + simd_shuffle4!(a, vqmovn_s64(b), [0, 1, 2, 3]) +} + +/// Signed saturating extract narrow +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uqxtn2))] +pub unsafe fn vqmovn_high_u16(a: uint8x8_t, b: uint16x8_t) -> uint8x16_t { + simd_shuffle16!(a, vqmovn_u16(b), [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]) +} + +/// Signed saturating extract narrow +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uqxtn2))] +pub unsafe fn vqmovn_high_u32(a: uint16x4_t, b: uint32x4_t) -> uint16x8_t { + simd_shuffle8!(a, vqmovn_u32(b), [0, 1, 2, 3, 4, 5, 6, 7]) +} + +/// Signed saturating extract narrow +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uqxtn2))] +pub unsafe fn vqmovn_high_u64(a: uint32x2_t, b: uint64x2_t) -> uint32x4_t { + simd_shuffle4!(a, vqmovn_u64(b), [0, 1, 2, 3]) +} + +/// Signed saturating extract unsigned narrow +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqxtun))] +pub unsafe fn vqmovunh_s16(a: i16) -> u8 { + simd_extract(vqmovun_s16(vdupq_n_s16(a)), 0) +} + +/// Signed saturating extract unsigned narrow +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqxtun))] +pub unsafe fn vqmovuns_s32(a: i32) -> u16 { + simd_extract(vqmovun_s32(vdupq_n_s32(a)), 0) +} + +/// Signed saturating extract unsigned narrow +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqxtun))] +pub unsafe fn vqmovund_s64(a: i64) -> u32 { + simd_extract(vqmovun_s64(vdupq_n_s64(a)), 0) +} + +/// Signed saturating extract unsigned narrow +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqxtun2))] +pub unsafe fn vqmovun_high_s16(a: uint8x8_t, b: int16x8_t) -> uint8x16_t { + simd_shuffle16!(a, vqmovun_s16(b), [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]) +} + +/// Signed saturating extract unsigned narrow +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqxtun2))] +pub unsafe fn vqmovun_high_s32(a: uint16x4_t, b: int32x4_t) -> uint16x8_t { + simd_shuffle8!(a, vqmovun_s32(b), [0, 1, 2, 3, 4, 5, 6, 7]) +} + +/// Signed saturating extract unsigned narrow +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqxtun2))] +pub unsafe fn vqmovun_high_s64(a: uint32x2_t, b: int64x2_t) -> uint32x4_t { + simd_shuffle4!(a, vqmovun_s64(b), [0, 1, 2, 3]) +} + +/// Signed saturating rounding doubling multiply returning high half +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqrdmulh))] +pub unsafe fn vqrdmulhh_s16(a: i16, b: i16) -> i16 { + simd_extract(vqrdmulh_s16(vdup_n_s16(a), vdup_n_s16(b)), 0) +} + +/// Signed saturating rounding doubling multiply returning high half +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqrdmulh))] +pub unsafe fn vqrdmulhs_s32(a: i32, b: i32) -> i32 { + simd_extract(vqrdmulh_s32(vdup_n_s32(a), vdup_n_s32(b)), 0) +} + +/// Signed saturating rounding doubling multiply returning high half +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqrdmulh, LANE = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vqrdmulhh_lane_s16(a: i16, b: int16x4_t) -> i16 { + static_assert_imm2!(LANE); + vqrdmulhh_s16(a, simd_extract(b, LANE as u32)) +} + +/// Signed saturating rounding doubling multiply returning high half +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqrdmulh, LANE = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vqrdmulhh_laneq_s16(a: i16, b: int16x8_t) -> i16 { + static_assert_imm3!(LANE); + vqrdmulhh_s16(a, simd_extract(b, LANE as u32)) +} + +/// Signed saturating rounding doubling multiply returning high half +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqrdmulh, LANE = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vqrdmulhs_lane_s32(a: i32, b: int32x2_t) -> i32 { + static_assert_imm1!(LANE); + vqrdmulhs_s32(a, simd_extract(b, LANE as u32)) +} + +/// Signed saturating rounding doubling multiply returning high half +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqrdmulh, LANE = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vqrdmulhs_laneq_s32(a: i32, b: int32x4_t) -> i32 { + static_assert_imm2!(LANE); + vqrdmulhs_s32(a, simd_extract(b, LANE as u32)) +} + +/// Signed saturating rounding doubling multiply accumulate returning high half +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqrdmulh))] +pub unsafe fn vqrdmlahh_s16(a: i16, b: i16, c: i16) -> i16 { + vqaddh_s16(a, vqrdmulhh_s16(b, c)) +} + +/// Signed saturating rounding doubling multiply accumulate returning high half +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqrdmulh))] +pub unsafe fn vqrdmlahs_s32(a: i32, b: i32, c: i32) -> i32 { + vqadds_s32(a, vqrdmulhs_s32(b, c)) +} + +/// Signed saturating rounding doubling multiply accumulate returning high half +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqrdmulh, LANE = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vqrdmlahh_lane_s16(a: i16, b: i16, c: int16x4_t) -> i16 { + static_assert_imm2!(LANE); + vqaddh_s16(a, vqrdmulhh_lane_s16::(b, c)) +} + +/// Signed saturating rounding doubling multiply accumulate returning high half +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqrdmulh, LANE = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vqrdmlahh_laneq_s16(a: i16, b: i16, c: int16x8_t) -> i16 { + static_assert_imm3!(LANE); + vqaddh_s16(a, vqrdmulhh_laneq_s16::(b, c)) +} + +/// Signed saturating rounding doubling multiply accumulate returning high half +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqrdmulh, LANE = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vqrdmlahs_lane_s32(a: i32, b: i32, c: int32x2_t) -> i32 { + static_assert_imm1!(LANE); + vqadds_s32(a, vqrdmulhs_lane_s32::(b, c)) +} + +/// Signed saturating rounding doubling multiply accumulate returning high half +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqrdmulh, LANE = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vqrdmlahs_laneq_s32(a: i32, b: i32, c: int32x4_t) -> i32 { + static_assert_imm2!(LANE); + vqadds_s32(a, vqrdmulhs_laneq_s32::(b, c)) +} + +/// Signed saturating rounding doubling multiply subtract returning high half +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqrdmulh))] +pub unsafe fn vqrdmlshh_s16(a: i16, b: i16, c: i16) -> i16 { + vqsubh_s16(a, vqrdmulhh_s16(b, c)) +} + +/// Signed saturating rounding doubling multiply subtract returning high half +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqrdmulh))] +pub unsafe fn vqrdmlshs_s32(a: i32, b: i32, c: i32) -> i32 { + vqsubs_s32(a, vqrdmulhs_s32(b, c)) +} + +/// Signed saturating rounding doubling multiply subtract returning high half +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqrdmulh, LANE = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vqrdmlshh_lane_s16(a: i16, b: i16, c: int16x4_t) -> i16 { + static_assert_imm2!(LANE); + vqsubh_s16(a, vqrdmulhh_lane_s16::(b, c)) +} + +/// Signed saturating rounding doubling multiply subtract returning high half +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqrdmulh, LANE = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vqrdmlshh_laneq_s16(a: i16, b: i16, c: int16x8_t) -> i16 { + static_assert_imm3!(LANE); + vqsubh_s16(a, vqrdmulhh_laneq_s16::(b, c)) +} + +/// Signed saturating rounding doubling multiply subtract returning high half +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqrdmulh, LANE = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vqrdmlshs_lane_s32(a: i32, b: i32, c: int32x2_t) -> i32 { + static_assert_imm1!(LANE); + vqsubs_s32(a, vqrdmulhs_lane_s32::(b, c)) +} + +/// Signed saturating rounding doubling multiply subtract returning high half +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqrdmulh, LANE = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vqrdmlshs_laneq_s32(a: i32, b: i32, c: int32x4_t) -> i32 { + static_assert_imm2!(LANE); + vqsubs_s32(a, vqrdmulhs_laneq_s32::(b, c)) +} + +/// Signed saturating rounding shift left +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqrshl))] +pub unsafe fn vqrshls_s32(a: i32, b: i32) -> i32 { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqrshl.i32")] + fn vqrshls_s32_(a: i32, b: i32) -> i32; + } + vqrshls_s32_(a, b) +} + +/// Signed saturating rounding shift left +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqrshl))] +pub unsafe fn vqrshld_s64(a: i64, b: i64) -> i64 { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqrshl.i64")] + fn vqrshld_s64_(a: i64, b: i64) -> i64; + } + vqrshld_s64_(a, b) +} + +/// Signed saturating rounding shift left +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqrshl))] +pub unsafe fn vqrshlb_s8(a: i8, b: i8) -> i8 { + let a: int8x8_t = vdup_n_s8(a); + let b: int8x8_t = vdup_n_s8(b); + simd_extract(vqrshl_s8(a, b), 0) +} + +/// Signed saturating rounding shift left +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqrshl))] +pub unsafe fn vqrshlh_s16(a: i16, b: i16) -> i16 { + let a: int16x4_t = vdup_n_s16(a); + let b: int16x4_t = vdup_n_s16(b); + simd_extract(vqrshl_s16(a, b), 0) +} + +/// Unsigned signed saturating rounding shift left +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uqrshl))] +pub unsafe fn vqrshls_u32(a: u32, b: i32) -> u32 { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uqrshl.i32")] + fn vqrshls_u32_(a: u32, b: i32) -> u32; + } + vqrshls_u32_(a, b) +} + +/// Unsigned signed saturating rounding shift left +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uqrshl))] +pub unsafe fn vqrshld_u64(a: u64, b: i64) -> u64 { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uqrshl.i64")] + fn vqrshld_u64_(a: u64, b: i64) -> u64; + } + vqrshld_u64_(a, b) +} + +/// Unsigned signed saturating rounding shift left +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uqrshl))] +pub unsafe fn vqrshlb_u8(a: u8, b: i8) -> u8 { + let a: uint8x8_t = vdup_n_u8(a); + let b: int8x8_t = vdup_n_s8(b); + simd_extract(vqrshl_u8(a, b), 0) +} + +/// Unsigned signed saturating rounding shift left +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uqrshl))] +pub unsafe fn vqrshlh_u16(a: u16, b: i16) -> u16 { + let a: uint16x4_t = vdup_n_u16(a); + let b: int16x4_t = vdup_n_s16(b); + simd_extract(vqrshl_u16(a, b), 0) +} + +/// Signed saturating rounded shift right narrow +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqrshrn, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vqrshrnh_n_s16(a: i16) -> i8 { + static_assert!(N : i32 where N >= 1 && N <= 8); + let a: int16x8_t = vdupq_n_s16(a); + simd_extract(vqrshrn_n_s16::(a), 0) +} + +/// Signed saturating rounded shift right narrow +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqrshrn, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vqrshrns_n_s32(a: i32) -> i16 { + static_assert!(N : i32 where N >= 1 && N <= 16); + let a: int32x4_t = vdupq_n_s32(a); + simd_extract(vqrshrn_n_s32::(a), 0) +} + +/// Signed saturating rounded shift right narrow +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqrshrn, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vqrshrnd_n_s64(a: i64) -> i32 { + static_assert!(N : i32 where N >= 1 && N <= 32); + let a: int64x2_t = vdupq_n_s64(a); + simd_extract(vqrshrn_n_s64::(a), 0) +} + +/// Signed saturating rounded shift right narrow +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqrshrn2, N = 2))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vqrshrn_high_n_s16(a: int8x8_t, b: int16x8_t) -> int8x16_t { + static_assert!(N : i32 where N >= 1 && N <= 8); + simd_shuffle16!(a, vqrshrn_n_s16::(b), [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]) +} + +/// Signed saturating rounded shift right narrow +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqrshrn2, N = 2))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vqrshrn_high_n_s32(a: int16x4_t, b: int32x4_t) -> int16x8_t { + static_assert!(N : i32 where N >= 1 && N <= 16); + simd_shuffle8!(a, vqrshrn_n_s32::(b), [0, 1, 2, 3, 4, 5, 6, 7]) +} + +/// Signed saturating rounded shift right narrow +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqrshrn2, N = 2))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vqrshrn_high_n_s64(a: int32x2_t, b: int64x2_t) -> int32x4_t { + static_assert!(N : i32 where N >= 1 && N <= 32); + simd_shuffle4!(a, vqrshrn_n_s64::(b), [0, 1, 2, 3]) +} + +/// Unsigned saturating rounded shift right narrow +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uqrshrn, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vqrshrnh_n_u16(a: u16) -> u8 { + static_assert!(N : i32 where N >= 1 && N <= 8); + let a: uint16x8_t = vdupq_n_u16(a); + simd_extract(vqrshrn_n_u16::(a), 0) +} + +/// Unsigned saturating rounded shift right narrow +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uqrshrn, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vqrshrns_n_u32(a: u32) -> u16 { + static_assert!(N : i32 where N >= 1 && N <= 16); + let a: uint32x4_t = vdupq_n_u32(a); + simd_extract(vqrshrn_n_u32::(a), 0) +} + +/// Unsigned saturating rounded shift right narrow +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uqrshrn, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vqrshrnd_n_u64(a: u64) -> u32 { + static_assert!(N : i32 where N >= 1 && N <= 32); + let a: uint64x2_t = vdupq_n_u64(a); + simd_extract(vqrshrn_n_u64::(a), 0) +} + +/// Unsigned saturating rounded shift right narrow +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uqrshrn2, N = 2))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vqrshrn_high_n_u16(a: uint8x8_t, b: uint16x8_t) -> uint8x16_t { + static_assert!(N : i32 where N >= 1 && N <= 8); + simd_shuffle16!(a, vqrshrn_n_u16::(b), [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]) +} + +/// Unsigned saturating rounded shift right narrow +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uqrshrn2, N = 2))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vqrshrn_high_n_u32(a: uint16x4_t, b: uint32x4_t) -> uint16x8_t { + static_assert!(N : i32 where N >= 1 && N <= 16); + simd_shuffle8!(a, vqrshrn_n_u32::(b), [0, 1, 2, 3, 4, 5, 6, 7]) +} + +/// Unsigned saturating rounded shift right narrow +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uqrshrn2, N = 2))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vqrshrn_high_n_u64(a: uint32x2_t, b: uint64x2_t) -> uint32x4_t { + static_assert!(N : i32 where N >= 1 && N <= 32); + simd_shuffle4!(a, vqrshrn_n_u64::(b), [0, 1, 2, 3]) +} + +/// Signed saturating rounded shift right unsigned narrow +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqrshrun, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vqrshrunh_n_s16(a: i16) -> u8 { + static_assert!(N : i32 where N >= 1 && N <= 8); + let a: int16x8_t = vdupq_n_s16(a); + simd_extract(vqrshrun_n_s16::(a), 0) +} + +/// Signed saturating rounded shift right unsigned narrow +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqrshrun, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vqrshruns_n_s32(a: i32) -> u16 { + static_assert!(N : i32 where N >= 1 && N <= 16); + let a: int32x4_t = vdupq_n_s32(a); + simd_extract(vqrshrun_n_s32::(a), 0) +} + +/// Signed saturating rounded shift right unsigned narrow +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqrshrun, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vqrshrund_n_s64(a: i64) -> u32 { + static_assert!(N : i32 where N >= 1 && N <= 32); + let a: int64x2_t = vdupq_n_s64(a); + simd_extract(vqrshrun_n_s64::(a), 0) +} + +/// Signed saturating rounded shift right unsigned narrow +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqrshrun2, N = 2))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vqrshrun_high_n_s16(a: uint8x8_t, b: int16x8_t) -> uint8x16_t { + static_assert!(N : i32 where N >= 1 && N <= 8); + simd_shuffle16!(a, vqrshrun_n_s16::(b), [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]) +} + +/// Signed saturating rounded shift right unsigned narrow +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqrshrun2, N = 2))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vqrshrun_high_n_s32(a: uint16x4_t, b: int32x4_t) -> uint16x8_t { + static_assert!(N : i32 where N >= 1 && N <= 16); + simd_shuffle8!(a, vqrshrun_n_s32::(b), [0, 1, 2, 3, 4, 5, 6, 7]) +} + +/// Signed saturating rounded shift right unsigned narrow +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqrshrun2, N = 2))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vqrshrun_high_n_s64(a: uint32x2_t, b: int64x2_t) -> uint32x4_t { + static_assert!(N : i32 where N >= 1 && N <= 32); + simd_shuffle4!(a, vqrshrun_n_s64::(b), [0, 1, 2, 3]) +} + +/// Signed saturating shift left +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqshl))] +pub unsafe fn vqshld_s64(a: i64, b: i64) -> i64 { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqshl.i64")] + fn vqshld_s64_(a: i64, b: i64) -> i64; + } + vqshld_s64_(a, b) +} + +/// Signed saturating shift left +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqshl))] +pub unsafe fn vqshlb_s8(a: i8, b: i8) -> i8 { + let c: int8x8_t = vqshl_s8(vdup_n_s8(a), vdup_n_s8(b)); + simd_extract(c, 0) +} + +/// Signed saturating shift left +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqshl))] +pub unsafe fn vqshlh_s16(a: i16, b: i16) -> i16 { + let c: int16x4_t = vqshl_s16(vdup_n_s16(a), vdup_n_s16(b)); + simd_extract(c, 0) +} + +/// Signed saturating shift left +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqshl))] +pub unsafe fn vqshls_s32(a: i32, b: i32) -> i32 { + let c: int32x2_t = vqshl_s32(vdup_n_s32(a), vdup_n_s32(b)); + simd_extract(c, 0) +} + +/// Unsigned saturating shift left +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uqshl))] +pub unsafe fn vqshld_u64(a: u64, b: i64) -> u64 { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uqshl.i64")] + fn vqshld_u64_(a: u64, b: i64) -> u64; + } + vqshld_u64_(a, b) +} + +/// Unsigned saturating shift left +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uqshl))] +pub unsafe fn vqshlb_u8(a: u8, b: i8) -> u8 { + let c: uint8x8_t = vqshl_u8(vdup_n_u8(a), vdup_n_s8(b)); + simd_extract(c, 0) +} + +/// Unsigned saturating shift left +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uqshl))] +pub unsafe fn vqshlh_u16(a: u16, b: i16) -> u16 { + let c: uint16x4_t = vqshl_u16(vdup_n_u16(a), vdup_n_s16(b)); + simd_extract(c, 0) +} + +/// Unsigned saturating shift left +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uqshl))] +pub unsafe fn vqshls_u32(a: u32, b: i32) -> u32 { + let c: uint32x2_t = vqshl_u32(vdup_n_u32(a), vdup_n_s32(b)); + simd_extract(c, 0) +} + +/// Signed saturating shift left +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqshl, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vqshlb_n_s8(a: i8) -> i8 { + static_assert_imm3!(N); + simd_extract(vqshl_n_s8::(vdup_n_s8(a)), 0) +} + +/// Signed saturating shift left +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqshl, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vqshlh_n_s16(a: i16) -> i16 { + static_assert_imm4!(N); + simd_extract(vqshl_n_s16::(vdup_n_s16(a)), 0) +} + +/// Signed saturating shift left +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqshl, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vqshls_n_s32(a: i32) -> i32 { + static_assert_imm5!(N); + simd_extract(vqshl_n_s32::(vdup_n_s32(a)), 0) +} + +/// Signed saturating shift left +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqshl, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vqshld_n_s64(a: i64) -> i64 { + static_assert_imm6!(N); + simd_extract(vqshl_n_s64::(vdup_n_s64(a)), 0) +} + +/// Unsigned saturating shift left +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uqshl, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vqshlb_n_u8(a: u8) -> u8 { + static_assert_imm3!(N); + simd_extract(vqshl_n_u8::(vdup_n_u8(a)), 0) +} + +/// Unsigned saturating shift left +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uqshl, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vqshlh_n_u16(a: u16) -> u16 { + static_assert_imm4!(N); + simd_extract(vqshl_n_u16::(vdup_n_u16(a)), 0) +} + +/// Unsigned saturating shift left +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uqshl, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vqshls_n_u32(a: u32) -> u32 { + static_assert_imm5!(N); + simd_extract(vqshl_n_u32::(vdup_n_u32(a)), 0) +} + +/// Unsigned saturating shift left +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uqshl, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vqshld_n_u64(a: u64) -> u64 { + static_assert_imm6!(N); + simd_extract(vqshl_n_u64::(vdup_n_u64(a)), 0) +} + +/// Signed saturating shift right narrow +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqshrn, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vqshrnd_n_s64(a: i64) -> i32 { + static_assert!(N : i32 where N >= 1 && N <= 32); + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqshrn.i32")] + fn vqshrnd_n_s64_(a: i64, n: i32) -> i32; + } + vqshrnd_n_s64_(a, N) +} + +/// Signed saturating shift right narrow +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqshrn, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vqshrnh_n_s16(a: i16) -> i8 { + static_assert!(N : i32 where N >= 1 && N <= 8); + simd_extract(vqshrn_n_s16::(vdupq_n_s16(a)), 0) +} + +/// Signed saturating shift right narrow +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqshrn, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vqshrns_n_s32(a: i32) -> i16 { + static_assert!(N : i32 where N >= 1 && N <= 16); + simd_extract(vqshrn_n_s32::(vdupq_n_s32(a)), 0) +} + +/// Signed saturating shift right narrow +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqshrn2, N = 2))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vqshrn_high_n_s16(a: int8x8_t, b: int16x8_t) -> int8x16_t { + static_assert!(N : i32 where N >= 1 && N <= 8); + simd_shuffle16!(a, vqshrn_n_s16::(b), [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]) +} + +/// Signed saturating shift right narrow +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqshrn2, N = 2))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vqshrn_high_n_s32(a: int16x4_t, b: int32x4_t) -> int16x8_t { + static_assert!(N : i32 where N >= 1 && N <= 16); + simd_shuffle8!(a, vqshrn_n_s32::(b), [0, 1, 2, 3, 4, 5, 6, 7]) +} + +/// Signed saturating shift right narrow +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqshrn2, N = 2))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vqshrn_high_n_s64(a: int32x2_t, b: int64x2_t) -> int32x4_t { + static_assert!(N : i32 where N >= 1 && N <= 32); + simd_shuffle4!(a, vqshrn_n_s64::(b), [0, 1, 2, 3]) +} + +/// Unsigned saturating shift right narrow +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uqshrn, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vqshrnd_n_u64(a: u64) -> u32 { + static_assert!(N : i32 where N >= 1 && N <= 32); + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uqshrn.i32")] + fn vqshrnd_n_u64_(a: u64, n: i32) -> u32; + } + vqshrnd_n_u64_(a, N) +} + +/// Unsigned saturating shift right narrow +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uqshrn, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vqshrnh_n_u16(a: u16) -> u8 { + static_assert!(N : i32 where N >= 1 && N <= 8); + simd_extract(vqshrn_n_u16::(vdupq_n_u16(a)), 0) +} + +/// Unsigned saturating shift right narrow +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uqshrn, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vqshrns_n_u32(a: u32) -> u16 { + static_assert!(N : i32 where N >= 1 && N <= 16); + simd_extract(vqshrn_n_u32::(vdupq_n_u32(a)), 0) +} + +/// Unsigned saturating shift right narrow +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uqshrn2, N = 2))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vqshrn_high_n_u16(a: uint8x8_t, b: uint16x8_t) -> uint8x16_t { + static_assert!(N : i32 where N >= 1 && N <= 8); + simd_shuffle16!(a, vqshrn_n_u16::(b), [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]) +} + +/// Unsigned saturating shift right narrow +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uqshrn2, N = 2))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vqshrn_high_n_u32(a: uint16x4_t, b: uint32x4_t) -> uint16x8_t { + static_assert!(N : i32 where N >= 1 && N <= 16); + simd_shuffle8!(a, vqshrn_n_u32::(b), [0, 1, 2, 3, 4, 5, 6, 7]) +} + +/// Unsigned saturating shift right narrow +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uqshrn2, N = 2))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vqshrn_high_n_u64(a: uint32x2_t, b: uint64x2_t) -> uint32x4_t { + static_assert!(N : i32 where N >= 1 && N <= 32); + simd_shuffle4!(a, vqshrn_n_u64::(b), [0, 1, 2, 3]) +} + +/// Signed saturating shift right unsigned narrow +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqshrun, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vqshrunh_n_s16(a: i16) -> u8 { + static_assert!(N : i32 where N >= 1 && N <= 8); + simd_extract(vqshrun_n_s16::(vdupq_n_s16(a)), 0) +} + +/// Signed saturating shift right unsigned narrow +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqshrun, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vqshruns_n_s32(a: i32) -> u16 { + static_assert!(N : i32 where N >= 1 && N <= 16); + simd_extract(vqshrun_n_s32::(vdupq_n_s32(a)), 0) +} + +/// Signed saturating shift right unsigned narrow +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqshrun, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vqshrund_n_s64(a: i64) -> u32 { + static_assert!(N : i32 where N >= 1 && N <= 32); + simd_extract(vqshrun_n_s64::(vdupq_n_s64(a)), 0) +} + +/// Signed saturating shift right unsigned narrow +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqshrun2, N = 2))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vqshrun_high_n_s16(a: uint8x8_t, b: int16x8_t) -> uint8x16_t { + static_assert!(N : i32 where N >= 1 && N <= 8); + simd_shuffle16!(a, vqshrun_n_s16::(b), [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]) +} + +/// Signed saturating shift right unsigned narrow +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqshrun2, N = 2))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vqshrun_high_n_s32(a: uint16x4_t, b: int32x4_t) -> uint16x8_t { + static_assert!(N : i32 where N >= 1 && N <= 16); + simd_shuffle8!(a, vqshrun_n_s32::(b), [0, 1, 2, 3, 4, 5, 6, 7]) +} + +/// Signed saturating shift right unsigned narrow +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqshrun2, N = 2))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vqshrun_high_n_s64(a: uint32x2_t, b: int64x2_t) -> uint32x4_t { + static_assert!(N : i32 where N >= 1 && N <= 32); + simd_shuffle4!(a, vqshrun_n_s64::(b), [0, 1, 2, 3]) +} + +/// Calculates the square root of each lane. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fsqrt))] +pub unsafe fn vsqrt_f32(a: float32x2_t) -> float32x2_t { + simd_fsqrt(a) +} + +/// Calculates the square root of each lane. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fsqrt))] +pub unsafe fn vsqrtq_f32(a: float32x4_t) -> float32x4_t { + simd_fsqrt(a) +} + +/// Calculates the square root of each lane. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fsqrt))] +pub unsafe fn vsqrt_f64(a: float64x1_t) -> float64x1_t { + simd_fsqrt(a) +} + +/// Calculates the square root of each lane. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fsqrt))] +pub unsafe fn vsqrtq_f64(a: float64x2_t) -> float64x2_t { + simd_fsqrt(a) +} + +/// Reciprocal square-root estimate. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(frsqrte))] +pub unsafe fn vrsqrte_f64(a: float64x1_t) -> float64x1_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.frsqrte.v1f64")] + fn vrsqrte_f64_(a: float64x1_t) -> float64x1_t; + } + vrsqrte_f64_(a) +} + +/// Reciprocal square-root estimate. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(frsqrte))] +pub unsafe fn vrsqrteq_f64(a: float64x2_t) -> float64x2_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.frsqrte.v2f64")] + fn vrsqrteq_f64_(a: float64x2_t) -> float64x2_t; + } + vrsqrteq_f64_(a) +} + +/// Reciprocal estimate. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(frecpe))] +pub unsafe fn vrecpe_f64(a: float64x1_t) -> float64x1_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.frecpe.v1f64")] + fn vrecpe_f64_(a: float64x1_t) -> float64x1_t; + } + vrecpe_f64_(a) +} + +/// Reciprocal estimate. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(frecpe))] +pub unsafe fn vrecpeq_f64(a: float64x2_t) -> float64x2_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.frecpe.v2f64")] + fn vrecpeq_f64_(a: float64x2_t) -> float64x2_t; + } + vrecpeq_f64_(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(str))] +pub unsafe fn vreinterpret_s64_p64(a: poly64x1_t) -> int64x1_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(str))] +pub unsafe fn vreinterpret_u64_p64(a: poly64x1_t) -> uint64x1_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(str))] +pub unsafe fn vreinterpret_p64_s64(a: int64x1_t) -> poly64x1_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(str))] +pub unsafe fn vreinterpret_p64_u64(a: uint64x1_t) -> poly64x1_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(str))] +pub unsafe fn vreinterpretq_s64_p64(a: poly64x2_t) -> int64x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(str))] +pub unsafe fn vreinterpretq_u64_p64(a: poly64x2_t) -> uint64x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(str))] +pub unsafe fn vreinterpretq_p64_s64(a: int64x2_t) -> poly64x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(str))] +pub unsafe fn vreinterpretq_p64_u64(a: uint64x2_t) -> poly64x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(str))] +pub unsafe fn vreinterpret_s32_p64(a: poly64x1_t) -> int32x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(str))] +pub unsafe fn vreinterpret_u32_p64(a: poly64x1_t) -> uint32x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(str))] +pub unsafe fn vreinterpretq_s32_p64(a: poly64x2_t) -> int32x4_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(str))] +pub unsafe fn vreinterpretq_u32_p64(a: poly64x2_t) -> uint32x4_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(str))] +pub unsafe fn vreinterpret_p64_s32(a: int32x2_t) -> poly64x1_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(str))] +pub unsafe fn vreinterpret_p64_u32(a: uint32x2_t) -> poly64x1_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(str))] +pub unsafe fn vreinterpretq_p64_s32(a: int32x4_t) -> poly64x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(str))] +pub unsafe fn vreinterpretq_p64_u32(a: uint32x4_t) -> poly64x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(str))] +pub unsafe fn vreinterpret_s16_p64(a: poly64x1_t) -> int16x4_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(str))] +pub unsafe fn vreinterpret_u16_p64(a: poly64x1_t) -> uint16x4_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(str))] +pub unsafe fn vreinterpret_p16_p64(a: poly64x1_t) -> poly16x4_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(str))] +pub unsafe fn vreinterpretq_s16_p64(a: poly64x2_t) -> int16x8_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(str))] +pub unsafe fn vreinterpretq_u16_p64(a: poly64x2_t) -> uint16x8_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(str))] +pub unsafe fn vreinterpretq_p16_p64(a: poly64x2_t) -> poly16x8_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(str))] +pub unsafe fn vreinterpret_p64_p16(a: poly16x4_t) -> poly64x1_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(str))] +pub unsafe fn vreinterpret_p64_s16(a: int16x4_t) -> poly64x1_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(str))] +pub unsafe fn vreinterpret_p64_u16(a: uint16x4_t) -> poly64x1_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(str))] +pub unsafe fn vreinterpretq_p64_p16(a: poly16x8_t) -> poly64x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(str))] +pub unsafe fn vreinterpretq_p64_s16(a: int16x8_t) -> poly64x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(str))] +pub unsafe fn vreinterpretq_p64_u16(a: uint16x8_t) -> poly64x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(str))] +pub unsafe fn vreinterpret_s8_p64(a: poly64x1_t) -> int8x8_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(str))] +pub unsafe fn vreinterpret_u8_p64(a: poly64x1_t) -> uint8x8_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(str))] +pub unsafe fn vreinterpret_p8_p64(a: poly64x1_t) -> poly8x8_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(str))] +pub unsafe fn vreinterpretq_s8_p64(a: poly64x2_t) -> int8x16_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(str))] +pub unsafe fn vreinterpretq_u8_p64(a: poly64x2_t) -> uint8x16_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(str))] +pub unsafe fn vreinterpretq_p8_p64(a: poly64x2_t) -> poly8x16_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(str))] +pub unsafe fn vreinterpret_p64_p8(a: poly8x8_t) -> poly64x1_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(str))] +pub unsafe fn vreinterpret_p64_s8(a: int8x8_t) -> poly64x1_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(str))] +pub unsafe fn vreinterpret_p64_u8(a: uint8x8_t) -> poly64x1_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(str))] +pub unsafe fn vreinterpretq_p64_p8(a: poly8x16_t) -> poly64x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(str))] +pub unsafe fn vreinterpretq_p64_s8(a: int8x16_t) -> poly64x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(str))] +pub unsafe fn vreinterpretq_p64_u8(a: uint8x16_t) -> poly64x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(str))] +pub unsafe fn vreinterpret_s8_f64(a: float64x1_t) -> int8x8_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(str))] +pub unsafe fn vreinterpret_s16_f64(a: float64x1_t) -> int16x4_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(str))] +pub unsafe fn vreinterpret_s32_f64(a: float64x1_t) -> int32x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(str))] +pub unsafe fn vreinterpret_s64_f64(a: float64x1_t) -> int64x1_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(str))] +pub unsafe fn vreinterpretq_s8_f64(a: float64x2_t) -> int8x16_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(str))] +pub unsafe fn vreinterpretq_s16_f64(a: float64x2_t) -> int16x8_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(str))] +pub unsafe fn vreinterpretq_s32_f64(a: float64x2_t) -> int32x4_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(str))] +pub unsafe fn vreinterpretq_s64_f64(a: float64x2_t) -> int64x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(str))] +pub unsafe fn vreinterpret_u8_f64(a: float64x1_t) -> uint8x8_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(str))] +pub unsafe fn vreinterpret_u16_f64(a: float64x1_t) -> uint16x4_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(str))] +pub unsafe fn vreinterpret_u32_f64(a: float64x1_t) -> uint32x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(str))] +pub unsafe fn vreinterpret_u64_f64(a: float64x1_t) -> uint64x1_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(str))] +pub unsafe fn vreinterpretq_u8_f64(a: float64x2_t) -> uint8x16_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(str))] +pub unsafe fn vreinterpretq_u16_f64(a: float64x2_t) -> uint16x8_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(str))] +pub unsafe fn vreinterpretq_u32_f64(a: float64x2_t) -> uint32x4_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(str))] +pub unsafe fn vreinterpretq_u64_f64(a: float64x2_t) -> uint64x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(str))] +pub unsafe fn vreinterpret_p8_f64(a: float64x1_t) -> poly8x8_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(str))] +pub unsafe fn vreinterpret_p16_f64(a: float64x1_t) -> poly16x4_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(str))] +pub unsafe fn vreinterpret_p64_f32(a: float32x2_t) -> poly64x1_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(str))] +pub unsafe fn vreinterpret_p64_f64(a: float64x1_t) -> poly64x1_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(str))] +pub unsafe fn vreinterpretq_p8_f64(a: float64x2_t) -> poly8x16_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(str))] +pub unsafe fn vreinterpretq_p16_f64(a: float64x2_t) -> poly16x8_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(str))] +pub unsafe fn vreinterpretq_p64_f32(a: float32x4_t) -> poly64x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(str))] +pub unsafe fn vreinterpretq_p64_f64(a: float64x2_t) -> poly64x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(str))] +pub unsafe fn vreinterpret_f64_s8(a: int8x8_t) -> float64x1_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(str))] +pub unsafe fn vreinterpret_f64_s16(a: int16x4_t) -> float64x1_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(str))] +pub unsafe fn vreinterpret_f64_s32(a: int32x2_t) -> float64x1_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(str))] +pub unsafe fn vreinterpret_f64_s64(a: int64x1_t) -> float64x1_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(str))] +pub unsafe fn vreinterpretq_f64_s8(a: int8x16_t) -> float64x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(str))] +pub unsafe fn vreinterpretq_f64_s16(a: int16x8_t) -> float64x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(str))] +pub unsafe fn vreinterpretq_f64_s32(a: int32x4_t) -> float64x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(str))] +pub unsafe fn vreinterpretq_f64_s64(a: int64x2_t) -> float64x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(str))] +pub unsafe fn vreinterpret_f64_p8(a: poly8x8_t) -> float64x1_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(str))] +pub unsafe fn vreinterpret_f64_u16(a: uint16x4_t) -> float64x1_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(str))] +pub unsafe fn vreinterpret_f64_u32(a: uint32x2_t) -> float64x1_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(str))] +pub unsafe fn vreinterpret_f64_u64(a: uint64x1_t) -> float64x1_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(str))] +pub unsafe fn vreinterpretq_f64_p8(a: poly8x16_t) -> float64x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(str))] +pub unsafe fn vreinterpretq_f64_u16(a: uint16x8_t) -> float64x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(str))] +pub unsafe fn vreinterpretq_f64_u32(a: uint32x4_t) -> float64x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(str))] +pub unsafe fn vreinterpretq_f64_u64(a: uint64x2_t) -> float64x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(str))] +pub unsafe fn vreinterpret_f64_u8(a: uint8x8_t) -> float64x1_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(str))] +pub unsafe fn vreinterpret_f64_p16(a: poly16x4_t) -> float64x1_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(str))] +pub unsafe fn vreinterpret_f64_p64(a: poly64x1_t) -> float64x1_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(str))] +pub unsafe fn vreinterpret_f32_p64(a: poly64x1_t) -> float32x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(str))] +pub unsafe fn vreinterpretq_f64_u8(a: uint8x16_t) -> float64x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(str))] +pub unsafe fn vreinterpretq_f64_p16(a: poly16x8_t) -> float64x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(str))] +pub unsafe fn vreinterpretq_f64_p64(a: poly64x2_t) -> float64x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(str))] +pub unsafe fn vreinterpretq_f32_p64(a: poly64x2_t) -> float32x4_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(str))] +pub unsafe fn vreinterpret_f64_f32(a: float32x2_t) -> float64x1_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(str))] +pub unsafe fn vreinterpret_f32_f64(a: float64x1_t) -> float32x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(str))] +pub unsafe fn vreinterpretq_f64_f32(a: float32x4_t) -> float64x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(str))] +pub unsafe fn vreinterpretq_f32_f64(a: float64x2_t) -> float32x4_t { + transmute(a) +} + +/// Signed rounding shift left +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(srshl))] +pub unsafe fn vrshld_s64(a: i64, b: i64) -> i64 { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.srshl.i64")] + fn vrshld_s64_(a: i64, b: i64) -> i64; + } + vrshld_s64_(a, b) +} + +/// Unsigned rounding shift left +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(urshl))] +pub unsafe fn vrshld_u64(a: u64, b: i64) -> u64 { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.urshl.i64")] + fn vrshld_u64_(a: u64, b: i64) -> u64; + } + vrshld_u64_(a, b) +} + +/// Signed rounding shift right +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(srshr, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vrshrd_n_s64(a: i64) -> i64 { + static_assert!(N : i32 where N >= 1 && N <= 64); + vrshld_s64(a, -N as i64) +} + +/// Unsigned rounding shift right +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(urshr, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vrshrd_n_u64(a: u64) -> u64 { + static_assert!(N : i32 where N >= 1 && N <= 64); + vrshld_u64(a, -N as i64) +} + +/// Rounding shift right narrow +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(rshrn2, N = 2))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vrshrn_high_n_s16(a: int8x8_t, b: int16x8_t) -> int8x16_t { + static_assert!(N : i32 where N >= 1 && N <= 8); + simd_shuffle16!(a, vrshrn_n_s16::(b), [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]) +} + +/// Rounding shift right narrow +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(rshrn2, N = 2))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vrshrn_high_n_s32(a: int16x4_t, b: int32x4_t) -> int16x8_t { + static_assert!(N : i32 where N >= 1 && N <= 16); + simd_shuffle8!(a, vrshrn_n_s32::(b), [0, 1, 2, 3, 4, 5, 6, 7]) +} + +/// Rounding shift right narrow +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(rshrn2, N = 2))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vrshrn_high_n_s64(a: int32x2_t, b: int64x2_t) -> int32x4_t { + static_assert!(N : i32 where N >= 1 && N <= 32); + simd_shuffle4!(a, vrshrn_n_s64::(b), [0, 1, 2, 3]) +} + +/// Rounding shift right narrow +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(rshrn2, N = 2))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vrshrn_high_n_u16(a: uint8x8_t, b: uint16x8_t) -> uint8x16_t { + static_assert!(N : i32 where N >= 1 && N <= 8); + simd_shuffle16!(a, vrshrn_n_u16::(b), [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]) +} + +/// Rounding shift right narrow +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(rshrn2, N = 2))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vrshrn_high_n_u32(a: uint16x4_t, b: uint32x4_t) -> uint16x8_t { + static_assert!(N : i32 where N >= 1 && N <= 16); + simd_shuffle8!(a, vrshrn_n_u32::(b), [0, 1, 2, 3, 4, 5, 6, 7]) +} + +/// Rounding shift right narrow +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(rshrn2, N = 2))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vrshrn_high_n_u64(a: uint32x2_t, b: uint64x2_t) -> uint32x4_t { + static_assert!(N : i32 where N >= 1 && N <= 32); + simd_shuffle4!(a, vrshrn_n_u64::(b), [0, 1, 2, 3]) +} + +/// Signed rounding shift right and accumulate. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(srsra, N = 2))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vrsrad_n_s64(a: i64, b: i64) -> i64 { + static_assert!(N : i32 where N >= 1 && N <= 64); + let b: i64 = vrshrd_n_s64::(b); + a + b +} + +/// Ungisned rounding shift right and accumulate. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ursra, N = 2))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vrsrad_n_u64(a: u64, b: u64) -> u64 { + static_assert!(N : i32 where N >= 1 && N <= 64); + let b: u64 = vrshrd_n_u64::(b); + a + b +} + +/// Insert vector element from another vector element +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop, LANE = 0))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vset_lane_f64(a: f64, b: float64x1_t) -> float64x1_t { + static_assert!(LANE : i32 where LANE == 0); + simd_insert(b, LANE as u32, a) +} + +/// Insert vector element from another vector element +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop, LANE = 0))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vsetq_lane_f64(a: f64, b: float64x2_t) -> float64x2_t { + static_assert_imm1!(LANE); + simd_insert(b, LANE as u32, a) +} + +/// Signed Shift left +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sshl))] +pub unsafe fn vshld_s64(a: i64, b: i64) -> i64 { + transmute(vshl_s64(transmute(a), transmute(b))) +} + +/// Unsigned Shift left +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ushl))] +pub unsafe fn vshld_u64(a: u64, b: i64) -> u64 { + transmute(vshl_u64(transmute(a), transmute(b))) +} + +/// Signed shift left long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sshll2, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vshll_high_n_s8(a: int8x16_t) -> int16x8_t { + static_assert!(N : i32 where N >= 0 && N <= 8); + let b: int8x8_t = simd_shuffle8!(a, a, [8, 9, 10, 11, 12, 13, 14, 15]); + vshll_n_s8::(b) +} + +/// Signed shift left long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sshll2, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vshll_high_n_s16(a: int16x8_t) -> int32x4_t { + static_assert!(N : i32 where N >= 0 && N <= 16); + let b: int16x4_t = simd_shuffle4!(a, a, [4, 5, 6, 7]); + vshll_n_s16::(b) +} + +/// Signed shift left long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sshll2, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vshll_high_n_s32(a: int32x4_t) -> int64x2_t { + static_assert!(N : i32 where N >= 0 && N <= 32); + let b: int32x2_t = simd_shuffle2!(a, a, [2, 3]); + vshll_n_s32::(b) +} + +/// Signed shift left long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ushll2, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vshll_high_n_u8(a: uint8x16_t) -> uint16x8_t { + static_assert!(N : i32 where N >= 0 && N <= 8); + let b: uint8x8_t = simd_shuffle8!(a, a, [8, 9, 10, 11, 12, 13, 14, 15]); + vshll_n_u8::(b) +} + +/// Signed shift left long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ushll2, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vshll_high_n_u16(a: uint16x8_t) -> uint32x4_t { + static_assert!(N : i32 where N >= 0 && N <= 16); + let b: uint16x4_t = simd_shuffle4!(a, a, [4, 5, 6, 7]); + vshll_n_u16::(b) +} + +/// Signed shift left long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ushll2, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vshll_high_n_u32(a: uint32x4_t) -> uint64x2_t { + static_assert!(N : i32 where N >= 0 && N <= 32); + let b: uint32x2_t = simd_shuffle2!(a, a, [2, 3]); + vshll_n_u32::(b) +} + +/// Shift right narrow +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(shrn2, N = 2))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vshrn_high_n_s16(a: int8x8_t, b: int16x8_t) -> int8x16_t { + static_assert!(N : i32 where N >= 1 && N <= 8); + simd_shuffle16!(a, vshrn_n_s16::(b), [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]) +} + +/// Shift right narrow +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(shrn2, N = 2))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vshrn_high_n_s32(a: int16x4_t, b: int32x4_t) -> int16x8_t { + static_assert!(N : i32 where N >= 1 && N <= 16); + simd_shuffle8!(a, vshrn_n_s32::(b), [0, 1, 2, 3, 4, 5, 6, 7]) +} + +/// Shift right narrow +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(shrn2, N = 2))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vshrn_high_n_s64(a: int32x2_t, b: int64x2_t) -> int32x4_t { + static_assert!(N : i32 where N >= 1 && N <= 32); + simd_shuffle4!(a, vshrn_n_s64::(b), [0, 1, 2, 3]) +} + +/// Shift right narrow +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(shrn2, N = 2))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vshrn_high_n_u16(a: uint8x8_t, b: uint16x8_t) -> uint8x16_t { + static_assert!(N : i32 where N >= 1 && N <= 8); + simd_shuffle16!(a, vshrn_n_u16::(b), [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]) +} + +/// Shift right narrow +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(shrn2, N = 2))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vshrn_high_n_u32(a: uint16x4_t, b: uint32x4_t) -> uint16x8_t { + static_assert!(N : i32 where N >= 1 && N <= 16); + simd_shuffle8!(a, vshrn_n_u32::(b), [0, 1, 2, 3, 4, 5, 6, 7]) +} + +/// Shift right narrow +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(shrn2, N = 2))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vshrn_high_n_u64(a: uint32x2_t, b: uint64x2_t) -> uint32x4_t { + static_assert!(N : i32 where N >= 1 && N <= 32); + simd_shuffle4!(a, vshrn_n_u64::(b), [0, 1, 2, 3]) +} + +/// Transpose vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(trn1))] +pub unsafe fn vtrn1_s8(a: int8x8_t, b: int8x8_t) -> int8x8_t { + simd_shuffle8!(a, b, [0, 8, 2, 10, 4, 12, 6, 14]) +} + +/// Transpose vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(trn1))] +pub unsafe fn vtrn1q_s8(a: int8x16_t, b: int8x16_t) -> int8x16_t { + simd_shuffle16!(a, b, [0, 16, 2, 18, 4, 20, 6, 22, 8, 24, 10, 26, 12, 28, 14, 30]) +} + +/// Transpose vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(trn1))] +pub unsafe fn vtrn1_s16(a: int16x4_t, b: int16x4_t) -> int16x4_t { + simd_shuffle4!(a, b, [0, 4, 2, 6]) +} + +/// Transpose vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(trn1))] +pub unsafe fn vtrn1q_s16(a: int16x8_t, b: int16x8_t) -> int16x8_t { + simd_shuffle8!(a, b, [0, 8, 2, 10, 4, 12, 6, 14]) +} + +/// Transpose vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(trn1))] +pub unsafe fn vtrn1q_s32(a: int32x4_t, b: int32x4_t) -> int32x4_t { + simd_shuffle4!(a, b, [0, 4, 2, 6]) +} + +/// Transpose vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(trn1))] +pub unsafe fn vtrn1_u8(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t { + simd_shuffle8!(a, b, [0, 8, 2, 10, 4, 12, 6, 14]) +} + +/// Transpose vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(trn1))] +pub unsafe fn vtrn1q_u8(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t { + simd_shuffle16!(a, b, [0, 16, 2, 18, 4, 20, 6, 22, 8, 24, 10, 26, 12, 28, 14, 30]) +} + +/// Transpose vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(trn1))] +pub unsafe fn vtrn1_u16(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t { + simd_shuffle4!(a, b, [0, 4, 2, 6]) +} + +/// Transpose vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(trn1))] +pub unsafe fn vtrn1q_u16(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t { + simd_shuffle8!(a, b, [0, 8, 2, 10, 4, 12, 6, 14]) +} + +/// Transpose vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(trn1))] +pub unsafe fn vtrn1q_u32(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t { + simd_shuffle4!(a, b, [0, 4, 2, 6]) +} + +/// Transpose vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(trn1))] +pub unsafe fn vtrn1_p8(a: poly8x8_t, b: poly8x8_t) -> poly8x8_t { + simd_shuffle8!(a, b, [0, 8, 2, 10, 4, 12, 6, 14]) +} + +/// Transpose vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(trn1))] +pub unsafe fn vtrn1q_p8(a: poly8x16_t, b: poly8x16_t) -> poly8x16_t { + simd_shuffle16!(a, b, [0, 16, 2, 18, 4, 20, 6, 22, 8, 24, 10, 26, 12, 28, 14, 30]) +} + +/// Transpose vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(trn1))] +pub unsafe fn vtrn1_p16(a: poly16x4_t, b: poly16x4_t) -> poly16x4_t { + simd_shuffle4!(a, b, [0, 4, 2, 6]) +} + +/// Transpose vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(trn1))] +pub unsafe fn vtrn1q_p16(a: poly16x8_t, b: poly16x8_t) -> poly16x8_t { + simd_shuffle8!(a, b, [0, 8, 2, 10, 4, 12, 6, 14]) +} + +/// Transpose vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(zip1))] +pub unsafe fn vtrn1_s32(a: int32x2_t, b: int32x2_t) -> int32x2_t { + simd_shuffle2!(a, b, [0, 2]) +} + +/// Transpose vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(zip1))] +pub unsafe fn vtrn1q_s64(a: int64x2_t, b: int64x2_t) -> int64x2_t { + simd_shuffle2!(a, b, [0, 2]) +} + +/// Transpose vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(zip1))] +pub unsafe fn vtrn1_u32(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t { + simd_shuffle2!(a, b, [0, 2]) +} + +/// Transpose vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(zip1))] +pub unsafe fn vtrn1q_u64(a: uint64x2_t, b: uint64x2_t) -> uint64x2_t { + simd_shuffle2!(a, b, [0, 2]) +} + +/// Transpose vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(zip1))] +pub unsafe fn vtrn1q_p64(a: poly64x2_t, b: poly64x2_t) -> poly64x2_t { + simd_shuffle2!(a, b, [0, 2]) +} + +/// Transpose vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(trn1))] +pub unsafe fn vtrn1q_f32(a: float32x4_t, b: float32x4_t) -> float32x4_t { + simd_shuffle4!(a, b, [0, 4, 2, 6]) +} + +/// Transpose vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(zip1))] +pub unsafe fn vtrn1_f32(a: float32x2_t, b: float32x2_t) -> float32x2_t { + simd_shuffle2!(a, b, [0, 2]) +} + +/// Transpose vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(zip1))] +pub unsafe fn vtrn1q_f64(a: float64x2_t, b: float64x2_t) -> float64x2_t { + simd_shuffle2!(a, b, [0, 2]) +} + +/// Transpose vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(trn2))] +pub unsafe fn vtrn2_s8(a: int8x8_t, b: int8x8_t) -> int8x8_t { + simd_shuffle8!(a, b, [1, 9, 3, 11, 5, 13, 7, 15]) +} + +/// Transpose vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(trn2))] +pub unsafe fn vtrn2q_s8(a: int8x16_t, b: int8x16_t) -> int8x16_t { + simd_shuffle16!(a, b, [1, 17, 3, 19, 5, 21, 7, 23, 9, 25, 11, 27, 13, 29, 15, 31]) +} + +/// Transpose vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(trn2))] +pub unsafe fn vtrn2_s16(a: int16x4_t, b: int16x4_t) -> int16x4_t { + simd_shuffle4!(a, b, [1, 5, 3, 7]) +} + +/// Transpose vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(trn2))] +pub unsafe fn vtrn2q_s16(a: int16x8_t, b: int16x8_t) -> int16x8_t { + simd_shuffle8!(a, b, [1, 9, 3, 11, 5, 13, 7, 15]) +} + +/// Transpose vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(trn2))] +pub unsafe fn vtrn2q_s32(a: int32x4_t, b: int32x4_t) -> int32x4_t { + simd_shuffle4!(a, b, [1, 5, 3, 7]) +} + +/// Transpose vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(trn2))] +pub unsafe fn vtrn2_u8(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t { + simd_shuffle8!(a, b, [1, 9, 3, 11, 5, 13, 7, 15]) +} + +/// Transpose vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(trn2))] +pub unsafe fn vtrn2q_u8(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t { + simd_shuffle16!(a, b, [1, 17, 3, 19, 5, 21, 7, 23, 9, 25, 11, 27, 13, 29, 15, 31]) +} + +/// Transpose vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(trn2))] +pub unsafe fn vtrn2_u16(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t { + simd_shuffle4!(a, b, [1, 5, 3, 7]) +} + +/// Transpose vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(trn2))] +pub unsafe fn vtrn2q_u16(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t { + simd_shuffle8!(a, b, [1, 9, 3, 11, 5, 13, 7, 15]) +} + +/// Transpose vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(trn2))] +pub unsafe fn vtrn2q_u32(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t { + simd_shuffle4!(a, b, [1, 5, 3, 7]) +} + +/// Transpose vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(trn2))] +pub unsafe fn vtrn2_p8(a: poly8x8_t, b: poly8x8_t) -> poly8x8_t { + simd_shuffle8!(a, b, [1, 9, 3, 11, 5, 13, 7, 15]) +} + +/// Transpose vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(trn2))] +pub unsafe fn vtrn2q_p8(a: poly8x16_t, b: poly8x16_t) -> poly8x16_t { + simd_shuffle16!(a, b, [1, 17, 3, 19, 5, 21, 7, 23, 9, 25, 11, 27, 13, 29, 15, 31]) +} + +/// Transpose vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(trn2))] +pub unsafe fn vtrn2_p16(a: poly16x4_t, b: poly16x4_t) -> poly16x4_t { + simd_shuffle4!(a, b, [1, 5, 3, 7]) +} + +/// Transpose vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(trn2))] +pub unsafe fn vtrn2q_p16(a: poly16x8_t, b: poly16x8_t) -> poly16x8_t { + simd_shuffle8!(a, b, [1, 9, 3, 11, 5, 13, 7, 15]) +} + +/// Transpose vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(zip2))] +pub unsafe fn vtrn2_s32(a: int32x2_t, b: int32x2_t) -> int32x2_t { + simd_shuffle2!(a, b, [1, 3]) +} + +/// Transpose vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(zip2))] +pub unsafe fn vtrn2q_s64(a: int64x2_t, b: int64x2_t) -> int64x2_t { + simd_shuffle2!(a, b, [1, 3]) +} + +/// Transpose vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(zip2))] +pub unsafe fn vtrn2_u32(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t { + simd_shuffle2!(a, b, [1, 3]) +} + +/// Transpose vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(zip2))] +pub unsafe fn vtrn2q_u64(a: uint64x2_t, b: uint64x2_t) -> uint64x2_t { + simd_shuffle2!(a, b, [1, 3]) +} + +/// Transpose vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(zip2))] +pub unsafe fn vtrn2q_p64(a: poly64x2_t, b: poly64x2_t) -> poly64x2_t { + simd_shuffle2!(a, b, [1, 3]) +} + +/// Transpose vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(trn2))] +pub unsafe fn vtrn2q_f32(a: float32x4_t, b: float32x4_t) -> float32x4_t { + simd_shuffle4!(a, b, [1, 5, 3, 7]) +} + +/// Transpose vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(zip2))] +pub unsafe fn vtrn2_f32(a: float32x2_t, b: float32x2_t) -> float32x2_t { + simd_shuffle2!(a, b, [1, 3]) +} + +/// Transpose vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(zip2))] +pub unsafe fn vtrn2q_f64(a: float64x2_t, b: float64x2_t) -> float64x2_t { + simd_shuffle2!(a, b, [1, 3]) +} + +/// Zip vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(zip1))] +pub unsafe fn vzip1_s8(a: int8x8_t, b: int8x8_t) -> int8x8_t { + simd_shuffle8!(a, b, [0, 8, 1, 9, 2, 10, 3, 11]) +} + +/// Zip vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(zip1))] +pub unsafe fn vzip1q_s8(a: int8x16_t, b: int8x16_t) -> int8x16_t { + simd_shuffle16!(a, b, [0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23]) +} + +/// Zip vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(zip1))] +pub unsafe fn vzip1_s16(a: int16x4_t, b: int16x4_t) -> int16x4_t { + simd_shuffle4!(a, b, [0, 4, 1, 5]) +} + +/// Zip vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(zip1))] +pub unsafe fn vzip1q_s16(a: int16x8_t, b: int16x8_t) -> int16x8_t { + simd_shuffle8!(a, b, [0, 8, 1, 9, 2, 10, 3, 11]) +} + +/// Zip vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(zip1))] +pub unsafe fn vzip1_s32(a: int32x2_t, b: int32x2_t) -> int32x2_t { + simd_shuffle2!(a, b, [0, 2]) +} + +/// Zip vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(zip1))] +pub unsafe fn vzip1q_s32(a: int32x4_t, b: int32x4_t) -> int32x4_t { + simd_shuffle4!(a, b, [0, 4, 1, 5]) +} + +/// Zip vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(zip1))] +pub unsafe fn vzip1q_s64(a: int64x2_t, b: int64x2_t) -> int64x2_t { + simd_shuffle2!(a, b, [0, 2]) +} + +/// Zip vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(zip1))] +pub unsafe fn vzip1_u8(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t { + simd_shuffle8!(a, b, [0, 8, 1, 9, 2, 10, 3, 11]) +} + +/// Zip vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(zip1))] +pub unsafe fn vzip1q_u8(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t { + simd_shuffle16!(a, b, [0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23]) +} + +/// Zip vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(zip1))] +pub unsafe fn vzip1_u16(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t { + simd_shuffle4!(a, b, [0, 4, 1, 5]) +} + +/// Zip vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(zip1))] +pub unsafe fn vzip1q_u16(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t { + simd_shuffle8!(a, b, [0, 8, 1, 9, 2, 10, 3, 11]) +} + +/// Zip vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(zip1))] +pub unsafe fn vzip1_u32(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t { + simd_shuffle2!(a, b, [0, 2]) +} + +/// Zip vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(zip1))] +pub unsafe fn vzip1q_u32(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t { + simd_shuffle4!(a, b, [0, 4, 1, 5]) +} + +/// Zip vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(zip1))] +pub unsafe fn vzip1q_u64(a: uint64x2_t, b: uint64x2_t) -> uint64x2_t { + simd_shuffle2!(a, b, [0, 2]) +} + +/// Zip vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(zip1))] +pub unsafe fn vzip1_p8(a: poly8x8_t, b: poly8x8_t) -> poly8x8_t { + simd_shuffle8!(a, b, [0, 8, 1, 9, 2, 10, 3, 11]) +} + +/// Zip vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(zip1))] +pub unsafe fn vzip1q_p8(a: poly8x16_t, b: poly8x16_t) -> poly8x16_t { + simd_shuffle16!(a, b, [0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23]) +} + +/// Zip vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(zip1))] +pub unsafe fn vzip1_p16(a: poly16x4_t, b: poly16x4_t) -> poly16x4_t { + simd_shuffle4!(a, b, [0, 4, 1, 5]) +} + +/// Zip vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(zip1))] +pub unsafe fn vzip1q_p16(a: poly16x8_t, b: poly16x8_t) -> poly16x8_t { + simd_shuffle8!(a, b, [0, 8, 1, 9, 2, 10, 3, 11]) +} + +/// Zip vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(zip1))] +pub unsafe fn vzip1q_p64(a: poly64x2_t, b: poly64x2_t) -> poly64x2_t { + simd_shuffle2!(a, b, [0, 2]) +} + +/// Zip vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(zip1))] +pub unsafe fn vzip1_f32(a: float32x2_t, b: float32x2_t) -> float32x2_t { + simd_shuffle2!(a, b, [0, 2]) +} + +/// Zip vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(zip1))] +pub unsafe fn vzip1q_f32(a: float32x4_t, b: float32x4_t) -> float32x4_t { + simd_shuffle4!(a, b, [0, 4, 1, 5]) +} + +/// Zip vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(zip1))] +pub unsafe fn vzip1q_f64(a: float64x2_t, b: float64x2_t) -> float64x2_t { + simd_shuffle2!(a, b, [0, 2]) +} + +/// Zip vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(zip2))] +pub unsafe fn vzip2_s8(a: int8x8_t, b: int8x8_t) -> int8x8_t { + simd_shuffle8!(a, b, [4, 12, 5, 13, 6, 14, 7, 15]) +} + +/// Zip vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(zip2))] +pub unsafe fn vzip2q_s8(a: int8x16_t, b: int8x16_t) -> int8x16_t { + simd_shuffle16!(a, b, [8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31]) +} + +/// Zip vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(zip2))] +pub unsafe fn vzip2_s16(a: int16x4_t, b: int16x4_t) -> int16x4_t { + simd_shuffle4!(a, b, [2, 6, 3, 7]) +} + +/// Zip vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(zip2))] +pub unsafe fn vzip2q_s16(a: int16x8_t, b: int16x8_t) -> int16x8_t { + simd_shuffle8!(a, b, [4, 12, 5, 13, 6, 14, 7, 15]) +} + +/// Zip vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(zip2))] +pub unsafe fn vzip2_s32(a: int32x2_t, b: int32x2_t) -> int32x2_t { + simd_shuffle2!(a, b, [1, 3]) +} + +/// Zip vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(zip2))] +pub unsafe fn vzip2q_s32(a: int32x4_t, b: int32x4_t) -> int32x4_t { + simd_shuffle4!(a, b, [2, 6, 3, 7]) +} + +/// Zip vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(zip2))] +pub unsafe fn vzip2q_s64(a: int64x2_t, b: int64x2_t) -> int64x2_t { + simd_shuffle2!(a, b, [1, 3]) +} + +/// Zip vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(zip2))] +pub unsafe fn vzip2_u8(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t { + simd_shuffle8!(a, b, [4, 12, 5, 13, 6, 14, 7, 15]) +} + +/// Zip vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(zip2))] +pub unsafe fn vzip2q_u8(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t { + simd_shuffle16!(a, b, [8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31]) +} + +/// Zip vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(zip2))] +pub unsafe fn vzip2_u16(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t { + simd_shuffle4!(a, b, [2, 6, 3, 7]) +} + +/// Zip vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(zip2))] +pub unsafe fn vzip2q_u16(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t { + simd_shuffle8!(a, b, [4, 12, 5, 13, 6, 14, 7, 15]) +} + +/// Zip vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(zip2))] +pub unsafe fn vzip2_u32(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t { + simd_shuffle2!(a, b, [1, 3]) +} + +/// Zip vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(zip2))] +pub unsafe fn vzip2q_u32(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t { + simd_shuffle4!(a, b, [2, 6, 3, 7]) +} + +/// Zip vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(zip2))] +pub unsafe fn vzip2q_u64(a: uint64x2_t, b: uint64x2_t) -> uint64x2_t { + simd_shuffle2!(a, b, [1, 3]) +} + +/// Zip vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(zip2))] +pub unsafe fn vzip2_p8(a: poly8x8_t, b: poly8x8_t) -> poly8x8_t { + simd_shuffle8!(a, b, [4, 12, 5, 13, 6, 14, 7, 15]) +} + +/// Zip vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(zip2))] +pub unsafe fn vzip2q_p8(a: poly8x16_t, b: poly8x16_t) -> poly8x16_t { + simd_shuffle16!(a, b, [8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31]) +} + +/// Zip vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(zip2))] +pub unsafe fn vzip2_p16(a: poly16x4_t, b: poly16x4_t) -> poly16x4_t { + simd_shuffle4!(a, b, [2, 6, 3, 7]) +} + +/// Zip vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(zip2))] +pub unsafe fn vzip2q_p16(a: poly16x8_t, b: poly16x8_t) -> poly16x8_t { + simd_shuffle8!(a, b, [4, 12, 5, 13, 6, 14, 7, 15]) +} + +/// Zip vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(zip2))] +pub unsafe fn vzip2q_p64(a: poly64x2_t, b: poly64x2_t) -> poly64x2_t { + simd_shuffle2!(a, b, [1, 3]) +} + +/// Zip vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(zip2))] +pub unsafe fn vzip2_f32(a: float32x2_t, b: float32x2_t) -> float32x2_t { + simd_shuffle2!(a, b, [1, 3]) +} + +/// Zip vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(zip2))] +pub unsafe fn vzip2q_f32(a: float32x4_t, b: float32x4_t) -> float32x4_t { + simd_shuffle4!(a, b, [2, 6, 3, 7]) +} + +/// Zip vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(zip2))] +pub unsafe fn vzip2q_f64(a: float64x2_t, b: float64x2_t) -> float64x2_t { + simd_shuffle2!(a, b, [1, 3]) +} + +/// Unzip vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uzp1))] +pub unsafe fn vuzp1_s8(a: int8x8_t, b: int8x8_t) -> int8x8_t { + simd_shuffle8!(a, b, [0, 2, 4, 6, 8, 10, 12, 14]) +} + +/// Unzip vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uzp1))] +pub unsafe fn vuzp1q_s8(a: int8x16_t, b: int8x16_t) -> int8x16_t { + simd_shuffle16!(a, b, [0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30]) +} + +/// Unzip vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uzp1))] +pub unsafe fn vuzp1_s16(a: int16x4_t, b: int16x4_t) -> int16x4_t { + simd_shuffle4!(a, b, [0, 2, 4, 6]) +} + +/// Unzip vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uzp1))] +pub unsafe fn vuzp1q_s16(a: int16x8_t, b: int16x8_t) -> int16x8_t { + simd_shuffle8!(a, b, [0, 2, 4, 6, 8, 10, 12, 14]) +} + +/// Unzip vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uzp1))] +pub unsafe fn vuzp1q_s32(a: int32x4_t, b: int32x4_t) -> int32x4_t { + simd_shuffle4!(a, b, [0, 2, 4, 6]) +} + +/// Unzip vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uzp1))] +pub unsafe fn vuzp1_u8(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t { + simd_shuffle8!(a, b, [0, 2, 4, 6, 8, 10, 12, 14]) +} + +/// Unzip vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uzp1))] +pub unsafe fn vuzp1q_u8(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t { + simd_shuffle16!(a, b, [0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30]) +} + +/// Unzip vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uzp1))] +pub unsafe fn vuzp1_u16(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t { + simd_shuffle4!(a, b, [0, 2, 4, 6]) +} + +/// Unzip vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uzp1))] +pub unsafe fn vuzp1q_u16(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t { + simd_shuffle8!(a, b, [0, 2, 4, 6, 8, 10, 12, 14]) +} + +/// Unzip vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uzp1))] +pub unsafe fn vuzp1q_u32(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t { + simd_shuffle4!(a, b, [0, 2, 4, 6]) +} + +/// Unzip vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uzp1))] +pub unsafe fn vuzp1_p8(a: poly8x8_t, b: poly8x8_t) -> poly8x8_t { + simd_shuffle8!(a, b, [0, 2, 4, 6, 8, 10, 12, 14]) +} + +/// Unzip vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uzp1))] +pub unsafe fn vuzp1q_p8(a: poly8x16_t, b: poly8x16_t) -> poly8x16_t { + simd_shuffle16!(a, b, [0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30]) +} + +/// Unzip vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uzp1))] +pub unsafe fn vuzp1_p16(a: poly16x4_t, b: poly16x4_t) -> poly16x4_t { + simd_shuffle4!(a, b, [0, 2, 4, 6]) +} + +/// Unzip vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uzp1))] +pub unsafe fn vuzp1q_p16(a: poly16x8_t, b: poly16x8_t) -> poly16x8_t { + simd_shuffle8!(a, b, [0, 2, 4, 6, 8, 10, 12, 14]) +} + +/// Unzip vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(zip1))] +pub unsafe fn vuzp1_s32(a: int32x2_t, b: int32x2_t) -> int32x2_t { + simd_shuffle2!(a, b, [0, 2]) +} + +/// Unzip vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(zip1))] +pub unsafe fn vuzp1q_s64(a: int64x2_t, b: int64x2_t) -> int64x2_t { + simd_shuffle2!(a, b, [0, 2]) +} + +/// Unzip vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(zip1))] +pub unsafe fn vuzp1_u32(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t { + simd_shuffle2!(a, b, [0, 2]) +} + +/// Unzip vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(zip1))] +pub unsafe fn vuzp1q_u64(a: uint64x2_t, b: uint64x2_t) -> uint64x2_t { + simd_shuffle2!(a, b, [0, 2]) +} + +/// Unzip vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(zip1))] +pub unsafe fn vuzp1q_p64(a: poly64x2_t, b: poly64x2_t) -> poly64x2_t { + simd_shuffle2!(a, b, [0, 2]) +} + +/// Unzip vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uzp1))] +pub unsafe fn vuzp1q_f32(a: float32x4_t, b: float32x4_t) -> float32x4_t { + simd_shuffle4!(a, b, [0, 2, 4, 6]) +} + +/// Unzip vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(zip1))] +pub unsafe fn vuzp1_f32(a: float32x2_t, b: float32x2_t) -> float32x2_t { + simd_shuffle2!(a, b, [0, 2]) +} + +/// Unzip vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(zip1))] +pub unsafe fn vuzp1q_f64(a: float64x2_t, b: float64x2_t) -> float64x2_t { + simd_shuffle2!(a, b, [0, 2]) +} + +/// Unzip vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uzp2))] +pub unsafe fn vuzp2_s8(a: int8x8_t, b: int8x8_t) -> int8x8_t { + simd_shuffle8!(a, b, [1, 3, 5, 7, 9, 11, 13, 15]) +} + +/// Unzip vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uzp2))] +pub unsafe fn vuzp2q_s8(a: int8x16_t, b: int8x16_t) -> int8x16_t { + simd_shuffle16!(a, b, [1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31]) +} + +/// Unzip vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uzp2))] +pub unsafe fn vuzp2_s16(a: int16x4_t, b: int16x4_t) -> int16x4_t { + simd_shuffle4!(a, b, [1, 3, 5, 7]) +} + +/// Unzip vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uzp2))] +pub unsafe fn vuzp2q_s16(a: int16x8_t, b: int16x8_t) -> int16x8_t { + simd_shuffle8!(a, b, [1, 3, 5, 7, 9, 11, 13, 15]) +} + +/// Unzip vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uzp2))] +pub unsafe fn vuzp2q_s32(a: int32x4_t, b: int32x4_t) -> int32x4_t { + simd_shuffle4!(a, b, [1, 3, 5, 7]) +} + +/// Unzip vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uzp2))] +pub unsafe fn vuzp2_u8(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t { + simd_shuffle8!(a, b, [1, 3, 5, 7, 9, 11, 13, 15]) +} + +/// Unzip vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uzp2))] +pub unsafe fn vuzp2q_u8(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t { + simd_shuffle16!(a, b, [1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31]) +} + +/// Unzip vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uzp2))] +pub unsafe fn vuzp2_u16(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t { + simd_shuffle4!(a, b, [1, 3, 5, 7]) +} + +/// Unzip vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uzp2))] +pub unsafe fn vuzp2q_u16(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t { + simd_shuffle8!(a, b, [1, 3, 5, 7, 9, 11, 13, 15]) +} + +/// Unzip vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uzp2))] +pub unsafe fn vuzp2q_u32(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t { + simd_shuffle4!(a, b, [1, 3, 5, 7]) +} + +/// Unzip vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uzp2))] +pub unsafe fn vuzp2_p8(a: poly8x8_t, b: poly8x8_t) -> poly8x8_t { + simd_shuffle8!(a, b, [1, 3, 5, 7, 9, 11, 13, 15]) +} + +/// Unzip vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uzp2))] +pub unsafe fn vuzp2q_p8(a: poly8x16_t, b: poly8x16_t) -> poly8x16_t { + simd_shuffle16!(a, b, [1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31]) +} + +/// Unzip vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uzp2))] +pub unsafe fn vuzp2_p16(a: poly16x4_t, b: poly16x4_t) -> poly16x4_t { + simd_shuffle4!(a, b, [1, 3, 5, 7]) +} + +/// Unzip vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uzp2))] +pub unsafe fn vuzp2q_p16(a: poly16x8_t, b: poly16x8_t) -> poly16x8_t { + simd_shuffle8!(a, b, [1, 3, 5, 7, 9, 11, 13, 15]) +} + +/// Unzip vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(zip2))] +pub unsafe fn vuzp2_s32(a: int32x2_t, b: int32x2_t) -> int32x2_t { + simd_shuffle2!(a, b, [1, 3]) +} + +/// Unzip vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(zip2))] +pub unsafe fn vuzp2q_s64(a: int64x2_t, b: int64x2_t) -> int64x2_t { + simd_shuffle2!(a, b, [1, 3]) +} + +/// Unzip vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(zip2))] +pub unsafe fn vuzp2_u32(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t { + simd_shuffle2!(a, b, [1, 3]) +} + +/// Unzip vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(zip2))] +pub unsafe fn vuzp2q_u64(a: uint64x2_t, b: uint64x2_t) -> uint64x2_t { + simd_shuffle2!(a, b, [1, 3]) +} + +/// Unzip vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(zip2))] +pub unsafe fn vuzp2q_p64(a: poly64x2_t, b: poly64x2_t) -> poly64x2_t { + simd_shuffle2!(a, b, [1, 3]) +} + +/// Unzip vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uzp2))] +pub unsafe fn vuzp2q_f32(a: float32x4_t, b: float32x4_t) -> float32x4_t { + simd_shuffle4!(a, b, [1, 3, 5, 7]) +} + +/// Unzip vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(zip2))] +pub unsafe fn vuzp2_f32(a: float32x2_t, b: float32x2_t) -> float32x2_t { + simd_shuffle2!(a, b, [1, 3]) +} + +/// Unzip vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(zip2))] +pub unsafe fn vuzp2q_f64(a: float64x2_t, b: float64x2_t) -> float64x2_t { + simd_shuffle2!(a, b, [1, 3]) +} + +/// Unsigned Absolute difference and Accumulate Long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uabal))] +pub unsafe fn vabal_high_u8(a: uint16x8_t, b: uint8x16_t, c: uint8x16_t) -> uint16x8_t { + let d: uint8x8_t = simd_shuffle8!(b, b, [8, 9, 10, 11, 12, 13, 14, 15]); + let e: uint8x8_t = simd_shuffle8!(c, c, [8, 9, 10, 11, 12, 13, 14, 15]); + let f: uint8x8_t = vabd_u8(d, e); + simd_add(a, simd_cast(f)) +} + +/// Unsigned Absolute difference and Accumulate Long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uabal))] +pub unsafe fn vabal_high_u16(a: uint32x4_t, b: uint16x8_t, c: uint16x8_t) -> uint32x4_t { + let d: uint16x4_t = simd_shuffle4!(b, b, [4, 5, 6, 7]); + let e: uint16x4_t = simd_shuffle4!(c, c, [4, 5, 6, 7]); + let f: uint16x4_t = vabd_u16(d, e); + simd_add(a, simd_cast(f)) +} + +/// Unsigned Absolute difference and Accumulate Long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uabal))] +pub unsafe fn vabal_high_u32(a: uint64x2_t, b: uint32x4_t, c: uint32x4_t) -> uint64x2_t { + let d: uint32x2_t = simd_shuffle2!(b, b, [2, 3]); + let e: uint32x2_t = simd_shuffle2!(c, c, [2, 3]); + let f: uint32x2_t = vabd_u32(d, e); + simd_add(a, simd_cast(f)) +} + +/// Signed Absolute difference and Accumulate Long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sabal))] +pub unsafe fn vabal_high_s8(a: int16x8_t, b: int8x16_t, c: int8x16_t) -> int16x8_t { + let d: int8x8_t = simd_shuffle8!(b, b, [8, 9, 10, 11, 12, 13, 14, 15]); + let e: int8x8_t = simd_shuffle8!(c, c, [8, 9, 10, 11, 12, 13, 14, 15]); + let f: int8x8_t = vabd_s8(d, e); + let f: uint8x8_t = simd_cast(f); + simd_add(a, simd_cast(f)) +} + +/// Signed Absolute difference and Accumulate Long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sabal))] +pub unsafe fn vabal_high_s16(a: int32x4_t, b: int16x8_t, c: int16x8_t) -> int32x4_t { + let d: int16x4_t = simd_shuffle4!(b, b, [4, 5, 6, 7]); + let e: int16x4_t = simd_shuffle4!(c, c, [4, 5, 6, 7]); + let f: int16x4_t = vabd_s16(d, e); + let f: uint16x4_t = simd_cast(f); + simd_add(a, simd_cast(f)) +} + +/// Signed Absolute difference and Accumulate Long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sabal))] +pub unsafe fn vabal_high_s32(a: int64x2_t, b: int32x4_t, c: int32x4_t) -> int64x2_t { + let d: int32x2_t = simd_shuffle2!(b, b, [2, 3]); + let e: int32x2_t = simd_shuffle2!(c, c, [2, 3]); + let f: int32x2_t = vabd_s32(d, e); + let f: uint32x2_t = simd_cast(f); + simd_add(a, simd_cast(f)) +} + +/// Singned saturating Absolute value +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqabs))] +pub unsafe fn vqabs_s64(a: int64x1_t) -> int64x1_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqabs.v1i64")] + fn vqabs_s64_(a: int64x1_t) -> int64x1_t; + } + vqabs_s64_(a) +} + +/// Singned saturating Absolute value +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(sqabs))] +pub unsafe fn vqabsq_s64(a: int64x2_t) -> int64x2_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqabs.v2i64")] + fn vqabsq_s64_(a: int64x2_t) -> int64x2_t; + } + vqabsq_s64_(a) +} + +#[cfg(test)] +mod test { + use super::*; + use crate::core_arch::simd::*; + use std::mem::transmute; + use stdarch_test::simd_test; + + #[simd_test(enable = "neon")] + unsafe fn test_vabd_f64() { + let a: f64 = 1.0; + let b: f64 = 9.0; + let e: f64 = 8.0; + let r: f64 = transmute(vabd_f64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vabdq_f64() { + let a: f64x2 = f64x2::new(1.0, 2.0); + let b: f64x2 = f64x2::new(9.0, 3.0); + let e: f64x2 = f64x2::new(8.0, 1.0); + let r: f64x2 = transmute(vabdq_f64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vabdl_high_u8() { + let a: u8x16 = u8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let b: u8x16 = u8x16::new(10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10); + let e: u16x8 = u16x8::new(1, 0, 1, 2, 3, 4, 5, 6); + let r: u16x8 = transmute(vabdl_high_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vabdl_high_u16() { + let a: u16x8 = u16x8::new(1, 2, 3, 4, 8, 9, 11, 12); + let b: u16x8 = u16x8::new(10, 10, 10, 10, 10, 10, 10, 10); + let e: u32x4 = u32x4::new(2, 1, 1, 2); + let r: u32x4 = transmute(vabdl_high_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vabdl_high_u32() { + let a: u32x4 = u32x4::new(1, 2, 3, 4); + let b: u32x4 = u32x4::new(10, 10, 10, 10); + let e: u64x2 = u64x2::new(7, 6); + let r: u64x2 = transmute(vabdl_high_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vabdl_high_s8() { + let a: i8x16 = i8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let b: i8x16 = i8x16::new(10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10); + let e: i16x8 = i16x8::new(1, 0, 1, 2, 3, 4, 5, 6); + let r: i16x8 = transmute(vabdl_high_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vabdl_high_s16() { + let a: i16x8 = i16x8::new(1, 2, 3, 4, 9, 10, 11, 12); + let b: i16x8 = i16x8::new(10, 10, 10, 10, 10, 10, 10, 10); + let e: i32x4 = i32x4::new(1, 0, 1, 2); + let r: i32x4 = transmute(vabdl_high_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vabdl_high_s32() { + let a: i32x4 = i32x4::new(1, 2, 3, 4); + let b: i32x4 = i32x4::new(10, 10, 10, 10); + let e: i64x2 = i64x2::new(7, 6); + let r: i64x2 = transmute(vabdl_high_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vceq_u64() { + let a: u64x1 = u64x1::new(0); + let b: u64x1 = u64x1::new(0); + let e: u64x1 = u64x1::new(0xFF_FF_FF_FF_FF_FF_FF_FF); + let r: u64x1 = transmute(vceq_u64(transmute(a), transmute(b))); + assert_eq!(r, e); + + let a: u64x1 = u64x1::new(0); + let b: u64x1 = u64x1::new(0); + let e: u64x1 = u64x1::new(0xFF_FF_FF_FF_FF_FF_FF_FF); + let r: u64x1 = transmute(vceq_u64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vceqq_u64() { + let a: u64x2 = u64x2::new(0, 0x01); + let b: u64x2 = u64x2::new(0, 0x01); + let e: u64x2 = u64x2::new(0xFF_FF_FF_FF_FF_FF_FF_FF, 0xFF_FF_FF_FF_FF_FF_FF_FF); + let r: u64x2 = transmute(vceqq_u64(transmute(a), transmute(b))); + assert_eq!(r, e); + + let a: u64x2 = u64x2::new(0, 0); + let b: u64x2 = u64x2::new(0, 0xFF_FF_FF_FF_FF_FF_FF_FF); + let e: u64x2 = u64x2::new(0xFF_FF_FF_FF_FF_FF_FF_FF, 0); + let r: u64x2 = transmute(vceqq_u64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vceq_s64() { + let a: i64x1 = i64x1::new(-9223372036854775808); + let b: i64x1 = i64x1::new(-9223372036854775808); + let e: u64x1 = u64x1::new(0xFF_FF_FF_FF_FF_FF_FF_FF); + let r: u64x1 = transmute(vceq_s64(transmute(a), transmute(b))); + assert_eq!(r, e); + + let a: i64x1 = i64x1::new(-9223372036854775808); + let b: i64x1 = i64x1::new(-9223372036854775808); + let e: u64x1 = u64x1::new(0xFF_FF_FF_FF_FF_FF_FF_FF); + let r: u64x1 = transmute(vceq_s64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vceqq_s64() { + let a: i64x2 = i64x2::new(-9223372036854775808, 0x01); + let b: i64x2 = i64x2::new(-9223372036854775808, 0x01); + let e: u64x2 = u64x2::new(0xFF_FF_FF_FF_FF_FF_FF_FF, 0xFF_FF_FF_FF_FF_FF_FF_FF); + let r: u64x2 = transmute(vceqq_s64(transmute(a), transmute(b))); + assert_eq!(r, e); + + let a: i64x2 = i64x2::new(-9223372036854775808, -9223372036854775808); + let b: i64x2 = i64x2::new(-9223372036854775808, 0x7F_FF_FF_FF_FF_FF_FF_FF); + let e: u64x2 = u64x2::new(0xFF_FF_FF_FF_FF_FF_FF_FF, 0); + let r: u64x2 = transmute(vceqq_s64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vceq_p64() { + let a: i64x1 = i64x1::new(-9223372036854775808); + let b: i64x1 = i64x1::new(-9223372036854775808); + let e: u64x1 = u64x1::new(0xFF_FF_FF_FF_FF_FF_FF_FF); + let r: u64x1 = transmute(vceq_p64(transmute(a), transmute(b))); + assert_eq!(r, e); + + let a: i64x1 = i64x1::new(-9223372036854775808); + let b: i64x1 = i64x1::new(-9223372036854775808); + let e: u64x1 = u64x1::new(0xFF_FF_FF_FF_FF_FF_FF_FF); + let r: u64x1 = transmute(vceq_p64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vceqq_p64() { + let a: i64x2 = i64x2::new(-9223372036854775808, 0x01); + let b: i64x2 = i64x2::new(-9223372036854775808, 0x01); + let e: u64x2 = u64x2::new(0xFF_FF_FF_FF_FF_FF_FF_FF, 0xFF_FF_FF_FF_FF_FF_FF_FF); + let r: u64x2 = transmute(vceqq_p64(transmute(a), transmute(b))); + assert_eq!(r, e); + + let a: i64x2 = i64x2::new(-9223372036854775808, -9223372036854775808); + let b: i64x2 = i64x2::new(-9223372036854775808, 0x7F_FF_FF_FF_FF_FF_FF_FF); + let e: u64x2 = u64x2::new(0xFF_FF_FF_FF_FF_FF_FF_FF, 0); + let r: u64x2 = transmute(vceqq_p64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vceq_f64() { + let a: f64 = 1.2; + let b: f64 = 1.2; + let e: u64x1 = u64x1::new(0xFF_FF_FF_FF_FF_FF_FF_FF); + let r: u64x1 = transmute(vceq_f64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vceqq_f64() { + let a: f64x2 = f64x2::new(1.2, 3.4); + let b: f64x2 = f64x2::new(1.2, 3.4); + let e: u64x2 = u64x2::new(0xFF_FF_FF_FF_FF_FF_FF_FF, 0xFF_FF_FF_FF_FF_FF_FF_FF); + let r: u64x2 = transmute(vceqq_f64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vceqz_s8() { + let a: i8x8 = i8x8::new(-128, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06); + let e: u8x8 = u8x8::new(0, 0xFF, 0, 0, 0, 0, 0, 0); + let r: u8x8 = transmute(vceqz_s8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vceqzq_s8() { + let a: i8x16 = i8x16::new(-128, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x7F); + let e: u8x16 = u8x16::new(0, 0xFF, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + let r: u8x16 = transmute(vceqzq_s8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vceqz_s16() { + let a: i16x4 = i16x4::new(-32768, 0x00, 0x01, 0x02); + let e: u16x4 = u16x4::new(0, 0xFF_FF, 0, 0); + let r: u16x4 = transmute(vceqz_s16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vceqzq_s16() { + let a: i16x8 = i16x8::new(-32768, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06); + let e: u16x8 = u16x8::new(0, 0xFF_FF, 0, 0, 0, 0, 0, 0); + let r: u16x8 = transmute(vceqzq_s16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vceqz_s32() { + let a: i32x2 = i32x2::new(-2147483648, 0x00); + let e: u32x2 = u32x2::new(0, 0xFF_FF_FF_FF); + let r: u32x2 = transmute(vceqz_s32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vceqzq_s32() { + let a: i32x4 = i32x4::new(-2147483648, 0x00, 0x01, 0x02); + let e: u32x4 = u32x4::new(0, 0xFF_FF_FF_FF, 0, 0); + let r: u32x4 = transmute(vceqzq_s32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vceqz_s64() { + let a: i64x1 = i64x1::new(-9223372036854775808); + let e: u64x1 = u64x1::new(0); + let r: u64x1 = transmute(vceqz_s64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vceqzq_s64() { + let a: i64x2 = i64x2::new(-9223372036854775808, 0x00); + let e: u64x2 = u64x2::new(0, 0xFF_FF_FF_FF_FF_FF_FF_FF); + let r: u64x2 = transmute(vceqzq_s64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vceqz_p8() { + let a: i8x8 = i8x8::new(-128, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06); + let e: u8x8 = u8x8::new(0, 0xFF, 0, 0, 0, 0, 0, 0); + let r: u8x8 = transmute(vceqz_p8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vceqzq_p8() { + let a: i8x16 = i8x16::new(-128, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x7F); + let e: u8x16 = u8x16::new(0, 0xFF, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + let r: u8x16 = transmute(vceqzq_p8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vceqz_p64() { + let a: i64x1 = i64x1::new(-9223372036854775808); + let e: u64x1 = u64x1::new(0); + let r: u64x1 = transmute(vceqz_p64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vceqzq_p64() { + let a: i64x2 = i64x2::new(-9223372036854775808, 0x00); + let e: u64x2 = u64x2::new(0, 0xFF_FF_FF_FF_FF_FF_FF_FF); + let r: u64x2 = transmute(vceqzq_p64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vceqz_u8() { + let a: u8x8 = u8x8::new(0, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06); + let e: u8x8 = u8x8::new(0xFF, 0xFF, 0, 0, 0, 0, 0, 0); + let r: u8x8 = transmute(vceqz_u8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vceqzq_u8() { + let a: u8x16 = u8x16::new(0, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0xFF); + let e: u8x16 = u8x16::new(0xFF, 0xFF, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + let r: u8x16 = transmute(vceqzq_u8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vceqz_u16() { + let a: u16x4 = u16x4::new(0, 0x00, 0x01, 0x02); + let e: u16x4 = u16x4::new(0xFF_FF, 0xFF_FF, 0, 0); + let r: u16x4 = transmute(vceqz_u16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vceqzq_u16() { + let a: u16x8 = u16x8::new(0, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06); + let e: u16x8 = u16x8::new(0xFF_FF, 0xFF_FF, 0, 0, 0, 0, 0, 0); + let r: u16x8 = transmute(vceqzq_u16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vceqz_u32() { + let a: u32x2 = u32x2::new(0, 0x00); + let e: u32x2 = u32x2::new(0xFF_FF_FF_FF, 0xFF_FF_FF_FF); + let r: u32x2 = transmute(vceqz_u32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vceqzq_u32() { + let a: u32x4 = u32x4::new(0, 0x00, 0x01, 0x02); + let e: u32x4 = u32x4::new(0xFF_FF_FF_FF, 0xFF_FF_FF_FF, 0, 0); + let r: u32x4 = transmute(vceqzq_u32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vceqz_u64() { + let a: u64x1 = u64x1::new(0); + let e: u64x1 = u64x1::new(0xFF_FF_FF_FF_FF_FF_FF_FF); + let r: u64x1 = transmute(vceqz_u64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vceqzq_u64() { + let a: u64x2 = u64x2::new(0, 0x00); + let e: u64x2 = u64x2::new(0xFF_FF_FF_FF_FF_FF_FF_FF, 0xFF_FF_FF_FF_FF_FF_FF_FF); + let r: u64x2 = transmute(vceqzq_u64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vceqz_f32() { + let a: f32x2 = f32x2::new(0.0, 1.2); + let e: u32x2 = u32x2::new(0xFF_FF_FF_FF, 0); + let r: u32x2 = transmute(vceqz_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vceqzq_f32() { + let a: f32x4 = f32x4::new(0.0, 1.2, 3.4, 5.6); + let e: u32x4 = u32x4::new(0xFF_FF_FF_FF, 0, 0, 0); + let r: u32x4 = transmute(vceqzq_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vceqz_f64() { + let a: f64 = 0.0; + let e: u64x1 = u64x1::new(0xFF_FF_FF_FF_FF_FF_FF_FF); + let r: u64x1 = transmute(vceqz_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vceqzq_f64() { + let a: f64x2 = f64x2::new(0.0, 1.2); + let e: u64x2 = u64x2::new(0xFF_FF_FF_FF_FF_FF_FF_FF, 0); + let r: u64x2 = transmute(vceqzq_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vtst_s64() { + let a: i64x1 = i64x1::new(-9223372036854775808); + let b: i64x1 = i64x1::new(-9223372036854775808); + let e: u64x1 = u64x1::new(0xFF_FF_FF_FF_FF_FF_FF_FF); + let r: u64x1 = transmute(vtst_s64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vtstq_s64() { + let a: i64x2 = i64x2::new(-9223372036854775808, 0x00); + let b: i64x2 = i64x2::new(-9223372036854775808, 0x00); + let e: u64x2 = u64x2::new(0xFF_FF_FF_FF_FF_FF_FF_FF, 0); + let r: u64x2 = transmute(vtstq_s64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vtst_p64() { + let a: i64x1 = i64x1::new(-9223372036854775808); + let b: i64x1 = i64x1::new(-9223372036854775808); + let e: u64x1 = u64x1::new(0xFF_FF_FF_FF_FF_FF_FF_FF); + let r: u64x1 = transmute(vtst_p64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vtstq_p64() { + let a: i64x2 = i64x2::new(-9223372036854775808, 0x00); + let b: i64x2 = i64x2::new(-9223372036854775808, 0x00); + let e: u64x2 = u64x2::new(0xFF_FF_FF_FF_FF_FF_FF_FF, 0); + let r: u64x2 = transmute(vtstq_p64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vtst_u64() { + let a: u64x1 = u64x1::new(0); + let b: u64x1 = u64x1::new(0); + let e: u64x1 = u64x1::new(0); + let r: u64x1 = transmute(vtst_u64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vtstq_u64() { + let a: u64x2 = u64x2::new(0, 0x00); + let b: u64x2 = u64x2::new(0, 0x00); + let e: u64x2 = u64x2::new(0, 0); + let r: u64x2 = transmute(vtstq_u64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vabs_f64() { + let a: f64 = -0.1; + let e: f64 = 0.1; + let r: f64 = transmute(vabs_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vabsq_f64() { + let a: f64x2 = f64x2::new(-0.1, -2.2); + let e: f64x2 = f64x2::new(0.1, 2.2); + let r: f64x2 = transmute(vabsq_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcgt_s64() { + let a: i64x1 = i64x1::new(1); + let b: i64x1 = i64x1::new(0); + let e: u64x1 = u64x1::new(0xFF_FF_FF_FF_FF_FF_FF_FF); + let r: u64x1 = transmute(vcgt_s64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcgtq_s64() { + let a: i64x2 = i64x2::new(1, 2); + let b: i64x2 = i64x2::new(0, 1); + let e: u64x2 = u64x2::new(0xFF_FF_FF_FF_FF_FF_FF_FF, 0xFF_FF_FF_FF_FF_FF_FF_FF); + let r: u64x2 = transmute(vcgtq_s64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcgt_u64() { + let a: u64x1 = u64x1::new(1); + let b: u64x1 = u64x1::new(0); + let e: u64x1 = u64x1::new(0xFF_FF_FF_FF_FF_FF_FF_FF); + let r: u64x1 = transmute(vcgt_u64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcgtq_u64() { + let a: u64x2 = u64x2::new(1, 2); + let b: u64x2 = u64x2::new(0, 1); + let e: u64x2 = u64x2::new(0xFF_FF_FF_FF_FF_FF_FF_FF, 0xFF_FF_FF_FF_FF_FF_FF_FF); + let r: u64x2 = transmute(vcgtq_u64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcgt_f64() { + let a: f64 = 1.2; + let b: f64 = 0.1; + let e: u64x1 = u64x1::new(0xFF_FF_FF_FF_FF_FF_FF_FF); + let r: u64x1 = transmute(vcgt_f64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcgtq_f64() { + let a: f64x2 = f64x2::new(1.2, 2.3); + let b: f64x2 = f64x2::new(0.1, 1.2); + let e: u64x2 = u64x2::new(0xFF_FF_FF_FF_FF_FF_FF_FF, 0xFF_FF_FF_FF_FF_FF_FF_FF); + let r: u64x2 = transmute(vcgtq_f64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vclt_s64() { + let a: i64x1 = i64x1::new(0); + let b: i64x1 = i64x1::new(1); + let e: u64x1 = u64x1::new(0xFF_FF_FF_FF_FF_FF_FF_FF); + let r: u64x1 = transmute(vclt_s64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcltq_s64() { + let a: i64x2 = i64x2::new(0, 1); + let b: i64x2 = i64x2::new(1, 2); + let e: u64x2 = u64x2::new(0xFF_FF_FF_FF_FF_FF_FF_FF, 0xFF_FF_FF_FF_FF_FF_FF_FF); + let r: u64x2 = transmute(vcltq_s64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vclt_u64() { + let a: u64x1 = u64x1::new(0); + let b: u64x1 = u64x1::new(1); + let e: u64x1 = u64x1::new(0xFF_FF_FF_FF_FF_FF_FF_FF); + let r: u64x1 = transmute(vclt_u64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcltq_u64() { + let a: u64x2 = u64x2::new(0, 1); + let b: u64x2 = u64x2::new(1, 2); + let e: u64x2 = u64x2::new(0xFF_FF_FF_FF_FF_FF_FF_FF, 0xFF_FF_FF_FF_FF_FF_FF_FF); + let r: u64x2 = transmute(vcltq_u64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vclt_f64() { + let a: f64 = 0.1; + let b: f64 = 1.2; + let e: u64x1 = u64x1::new(0xFF_FF_FF_FF_FF_FF_FF_FF); + let r: u64x1 = transmute(vclt_f64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcltq_f64() { + let a: f64x2 = f64x2::new(0.1, 1.2); + let b: f64x2 = f64x2::new(1.2, 2.3); + let e: u64x2 = u64x2::new(0xFF_FF_FF_FF_FF_FF_FF_FF, 0xFF_FF_FF_FF_FF_FF_FF_FF); + let r: u64x2 = transmute(vcltq_f64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcle_s64() { + let a: i64x1 = i64x1::new(0); + let b: i64x1 = i64x1::new(1); + let e: u64x1 = u64x1::new(0xFF_FF_FF_FF_FF_FF_FF_FF); + let r: u64x1 = transmute(vcle_s64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcleq_s64() { + let a: i64x2 = i64x2::new(0, 1); + let b: i64x2 = i64x2::new(1, 2); + let e: u64x2 = u64x2::new(0xFF_FF_FF_FF_FF_FF_FF_FF, 0xFF_FF_FF_FF_FF_FF_FF_FF); + let r: u64x2 = transmute(vcleq_s64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcle_u64() { + let a: u64x1 = u64x1::new(0); + let b: u64x1 = u64x1::new(1); + let e: u64x1 = u64x1::new(0xFF_FF_FF_FF_FF_FF_FF_FF); + let r: u64x1 = transmute(vcle_u64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcleq_u64() { + let a: u64x2 = u64x2::new(0, 1); + let b: u64x2 = u64x2::new(1, 2); + let e: u64x2 = u64x2::new(0xFF_FF_FF_FF_FF_FF_FF_FF, 0xFF_FF_FF_FF_FF_FF_FF_FF); + let r: u64x2 = transmute(vcleq_u64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcle_f64() { + let a: f64 = 0.1; + let b: f64 = 1.2; + let e: u64x1 = u64x1::new(0xFF_FF_FF_FF_FF_FF_FF_FF); + let r: u64x1 = transmute(vcle_f64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcleq_f64() { + let a: f64x2 = f64x2::new(0.1, 1.2); + let b: f64x2 = f64x2::new(1.2, 2.3); + let e: u64x2 = u64x2::new(0xFF_FF_FF_FF_FF_FF_FF_FF, 0xFF_FF_FF_FF_FF_FF_FF_FF); + let r: u64x2 = transmute(vcleq_f64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcge_s64() { + let a: i64x1 = i64x1::new(1); + let b: i64x1 = i64x1::new(0); + let e: u64x1 = u64x1::new(0xFF_FF_FF_FF_FF_FF_FF_FF); + let r: u64x1 = transmute(vcge_s64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcgeq_s64() { + let a: i64x2 = i64x2::new(1, 2); + let b: i64x2 = i64x2::new(0, 1); + let e: u64x2 = u64x2::new(0xFF_FF_FF_FF_FF_FF_FF_FF, 0xFF_FF_FF_FF_FF_FF_FF_FF); + let r: u64x2 = transmute(vcgeq_s64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcge_u64() { + let a: u64x1 = u64x1::new(1); + let b: u64x1 = u64x1::new(0); + let e: u64x1 = u64x1::new(0xFF_FF_FF_FF_FF_FF_FF_FF); + let r: u64x1 = transmute(vcge_u64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcgeq_u64() { + let a: u64x2 = u64x2::new(1, 2); + let b: u64x2 = u64x2::new(0, 1); + let e: u64x2 = u64x2::new(0xFF_FF_FF_FF_FF_FF_FF_FF, 0xFF_FF_FF_FF_FF_FF_FF_FF); + let r: u64x2 = transmute(vcgeq_u64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcge_f64() { + let a: f64 = 1.2; + let b: f64 = 0.1; + let e: u64x1 = u64x1::new(0xFF_FF_FF_FF_FF_FF_FF_FF); + let r: u64x1 = transmute(vcge_f64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcgeq_f64() { + let a: f64x2 = f64x2::new(1.2, 2.3); + let b: f64x2 = f64x2::new(0.1, 1.2); + let e: u64x2 = u64x2::new(0xFF_FF_FF_FF_FF_FF_FF_FF, 0xFF_FF_FF_FF_FF_FF_FF_FF); + let r: u64x2 = transmute(vcgeq_f64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcgez_s8() { + let a: i8x8 = i8x8::new(-128, -1, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05); + let e: u8x8 = u8x8::new(0, 0, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF); + let r: u8x8 = transmute(vcgez_s8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcgezq_s8() { + let a: i8x16 = i8x16::new(-128, -1, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x7F); + let e: u8x16 = u8x16::new(0, 0, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF); + let r: u8x16 = transmute(vcgezq_s8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcgez_s16() { + let a: i16x4 = i16x4::new(-32768, -1, 0x00, 0x01); + let e: u16x4 = u16x4::new(0, 0, 0xFF_FF, 0xFF_FF); + let r: u16x4 = transmute(vcgez_s16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcgezq_s16() { + let a: i16x8 = i16x8::new(-32768, -1, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05); + let e: u16x8 = u16x8::new(0, 0, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF); + let r: u16x8 = transmute(vcgezq_s16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcgez_s32() { + let a: i32x2 = i32x2::new(-2147483648, -1); + let e: u32x2 = u32x2::new(0, 0); + let r: u32x2 = transmute(vcgez_s32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcgezq_s32() { + let a: i32x4 = i32x4::new(-2147483648, -1, 0x00, 0x01); + let e: u32x4 = u32x4::new(0, 0, 0xFF_FF_FF_FF, 0xFF_FF_FF_FF); + let r: u32x4 = transmute(vcgezq_s32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcgez_s64() { + let a: i64x1 = i64x1::new(-9223372036854775808); + let e: u64x1 = u64x1::new(0); + let r: u64x1 = transmute(vcgez_s64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcgezq_s64() { + let a: i64x2 = i64x2::new(-9223372036854775808, -1); + let e: u64x2 = u64x2::new(0, 0); + let r: u64x2 = transmute(vcgezq_s64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcgez_f32() { + let a: f32x2 = f32x2::new(-1.2, 0.0); + let e: u32x2 = u32x2::new(0, 0xFF_FF_FF_FF); + let r: u32x2 = transmute(vcgez_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcgezq_f32() { + let a: f32x4 = f32x4::new(-1.2, 0.0, 1.2, 2.3); + let e: u32x4 = u32x4::new(0, 0xFF_FF_FF_FF, 0xFF_FF_FF_FF, 0xFF_FF_FF_FF); + let r: u32x4 = transmute(vcgezq_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcgez_f64() { + let a: f64 = -1.2; + let e: u64x1 = u64x1::new(0); + let r: u64x1 = transmute(vcgez_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcgezq_f64() { + let a: f64x2 = f64x2::new(-1.2, 0.0); + let e: u64x2 = u64x2::new(0, 0xFF_FF_FF_FF_FF_FF_FF_FF); + let r: u64x2 = transmute(vcgezq_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcgtz_s8() { + let a: i8x8 = i8x8::new(-128, -1, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05); + let e: u8x8 = u8x8::new(0, 0, 0, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF); + let r: u8x8 = transmute(vcgtz_s8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcgtzq_s8() { + let a: i8x16 = i8x16::new(-128, -1, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x7F); + let e: u8x16 = u8x16::new(0, 0, 0, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF); + let r: u8x16 = transmute(vcgtzq_s8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcgtz_s16() { + let a: i16x4 = i16x4::new(-32768, -1, 0x00, 0x01); + let e: u16x4 = u16x4::new(0, 0, 0, 0xFF_FF); + let r: u16x4 = transmute(vcgtz_s16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcgtzq_s16() { + let a: i16x8 = i16x8::new(-32768, -1, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05); + let e: u16x8 = u16x8::new(0, 0, 0, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF); + let r: u16x8 = transmute(vcgtzq_s16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcgtz_s32() { + let a: i32x2 = i32x2::new(-2147483648, -1); + let e: u32x2 = u32x2::new(0, 0); + let r: u32x2 = transmute(vcgtz_s32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcgtzq_s32() { + let a: i32x4 = i32x4::new(-2147483648, -1, 0x00, 0x01); + let e: u32x4 = u32x4::new(0, 0, 0, 0xFF_FF_FF_FF); + let r: u32x4 = transmute(vcgtzq_s32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcgtz_s64() { + let a: i64x1 = i64x1::new(-9223372036854775808); + let e: u64x1 = u64x1::new(0); + let r: u64x1 = transmute(vcgtz_s64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcgtzq_s64() { + let a: i64x2 = i64x2::new(-9223372036854775808, -1); + let e: u64x2 = u64x2::new(0, 0); + let r: u64x2 = transmute(vcgtzq_s64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcgtz_f32() { + let a: f32x2 = f32x2::new(-1.2, 0.0); + let e: u32x2 = u32x2::new(0, 0); + let r: u32x2 = transmute(vcgtz_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcgtzq_f32() { + let a: f32x4 = f32x4::new(-1.2, 0.0, 1.2, 2.3); + let e: u32x4 = u32x4::new(0, 0, 0xFF_FF_FF_FF, 0xFF_FF_FF_FF); + let r: u32x4 = transmute(vcgtzq_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcgtz_f64() { + let a: f64 = -1.2; + let e: u64x1 = u64x1::new(0); + let r: u64x1 = transmute(vcgtz_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcgtzq_f64() { + let a: f64x2 = f64x2::new(-1.2, 0.0); + let e: u64x2 = u64x2::new(0, 0); + let r: u64x2 = transmute(vcgtzq_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vclez_s8() { + let a: i8x8 = i8x8::new(-128, -1, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05); + let e: u8x8 = u8x8::new(0xFF, 0xFF, 0xFF, 0, 0, 0, 0, 0); + let r: u8x8 = transmute(vclez_s8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vclezq_s8() { + let a: i8x16 = i8x16::new(-128, -1, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x7F); + let e: u8x16 = u8x16::new(0xFF, 0xFF, 0xFF, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + let r: u8x16 = transmute(vclezq_s8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vclez_s16() { + let a: i16x4 = i16x4::new(-32768, -1, 0x00, 0x01); + let e: u16x4 = u16x4::new(0xFF_FF, 0xFF_FF, 0xFF_FF, 0); + let r: u16x4 = transmute(vclez_s16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vclezq_s16() { + let a: i16x8 = i16x8::new(-32768, -1, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05); + let e: u16x8 = u16x8::new(0xFF_FF, 0xFF_FF, 0xFF_FF, 0, 0, 0, 0, 0); + let r: u16x8 = transmute(vclezq_s16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vclez_s32() { + let a: i32x2 = i32x2::new(-2147483648, -1); + let e: u32x2 = u32x2::new(0xFF_FF_FF_FF, 0xFF_FF_FF_FF); + let r: u32x2 = transmute(vclez_s32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vclezq_s32() { + let a: i32x4 = i32x4::new(-2147483648, -1, 0x00, 0x01); + let e: u32x4 = u32x4::new(0xFF_FF_FF_FF, 0xFF_FF_FF_FF, 0xFF_FF_FF_FF, 0); + let r: u32x4 = transmute(vclezq_s32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vclez_s64() { + let a: i64x1 = i64x1::new(-9223372036854775808); + let e: u64x1 = u64x1::new(0xFF_FF_FF_FF_FF_FF_FF_FF); + let r: u64x1 = transmute(vclez_s64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vclezq_s64() { + let a: i64x2 = i64x2::new(-9223372036854775808, -1); + let e: u64x2 = u64x2::new(0xFF_FF_FF_FF_FF_FF_FF_FF, 0xFF_FF_FF_FF_FF_FF_FF_FF); + let r: u64x2 = transmute(vclezq_s64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vclez_f32() { + let a: f32x2 = f32x2::new(-1.2, 0.0); + let e: u32x2 = u32x2::new(0xFF_FF_FF_FF, 0xFF_FF_FF_FF); + let r: u32x2 = transmute(vclez_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vclezq_f32() { + let a: f32x4 = f32x4::new(-1.2, 0.0, 1.2, 2.3); + let e: u32x4 = u32x4::new(0xFF_FF_FF_FF, 0xFF_FF_FF_FF, 0, 0); + let r: u32x4 = transmute(vclezq_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vclez_f64() { + let a: f64 = -1.2; + let e: u64x1 = u64x1::new(0xFF_FF_FF_FF_FF_FF_FF_FF); + let r: u64x1 = transmute(vclez_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vclezq_f64() { + let a: f64x2 = f64x2::new(-1.2, 0.0); + let e: u64x2 = u64x2::new(0xFF_FF_FF_FF_FF_FF_FF_FF, 0xFF_FF_FF_FF_FF_FF_FF_FF); + let r: u64x2 = transmute(vclezq_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcltz_s8() { + let a: i8x8 = i8x8::new(-128, -1, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05); + let e: u8x8 = u8x8::new(0xFF, 0xFF, 0, 0, 0, 0, 0, 0); + let r: u8x8 = transmute(vcltz_s8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcltzq_s8() { + let a: i8x16 = i8x16::new(-128, -1, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x7F); + let e: u8x16 = u8x16::new(0xFF, 0xFF, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + let r: u8x16 = transmute(vcltzq_s8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcltz_s16() { + let a: i16x4 = i16x4::new(-32768, -1, 0x00, 0x01); + let e: u16x4 = u16x4::new(0xFF_FF, 0xFF_FF, 0, 0); + let r: u16x4 = transmute(vcltz_s16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcltzq_s16() { + let a: i16x8 = i16x8::new(-32768, -1, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05); + let e: u16x8 = u16x8::new(0xFF_FF, 0xFF_FF, 0, 0, 0, 0, 0, 0); + let r: u16x8 = transmute(vcltzq_s16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcltz_s32() { + let a: i32x2 = i32x2::new(-2147483648, -1); + let e: u32x2 = u32x2::new(0xFF_FF_FF_FF, 0xFF_FF_FF_FF); + let r: u32x2 = transmute(vcltz_s32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcltzq_s32() { + let a: i32x4 = i32x4::new(-2147483648, -1, 0x00, 0x01); + let e: u32x4 = u32x4::new(0xFF_FF_FF_FF, 0xFF_FF_FF_FF, 0, 0); + let r: u32x4 = transmute(vcltzq_s32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcltz_s64() { + let a: i64x1 = i64x1::new(-9223372036854775808); + let e: u64x1 = u64x1::new(0xFF_FF_FF_FF_FF_FF_FF_FF); + let r: u64x1 = transmute(vcltz_s64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcltzq_s64() { + let a: i64x2 = i64x2::new(-9223372036854775808, -1); + let e: u64x2 = u64x2::new(0xFF_FF_FF_FF_FF_FF_FF_FF, 0xFF_FF_FF_FF_FF_FF_FF_FF); + let r: u64x2 = transmute(vcltzq_s64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcltz_f32() { + let a: f32x2 = f32x2::new(-1.2, 0.0); + let e: u32x2 = u32x2::new(0xFF_FF_FF_FF, 0); + let r: u32x2 = transmute(vcltz_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcltzq_f32() { + let a: f32x4 = f32x4::new(-1.2, 0.0, 1.2, 2.3); + let e: u32x4 = u32x4::new(0xFF_FF_FF_FF, 0, 0, 0); + let r: u32x4 = transmute(vcltzq_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcltz_f64() { + let a: f64 = -1.2; + let e: u64x1 = u64x1::new(0xFF_FF_FF_FF_FF_FF_FF_FF); + let r: u64x1 = transmute(vcltz_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcltzq_f64() { + let a: f64x2 = f64x2::new(-1.2, 0.0); + let e: u64x2 = u64x2::new(0xFF_FF_FF_FF_FF_FF_FF_FF, 0); + let r: u64x2 = transmute(vcltzq_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcagt_f64() { + let a: f64 = -1.2; + let b: f64 = -1.1; + let e: u64x1 = u64x1::new(0xFF_FF_FF_FF_FF_FF_FF_FF); + let r: u64x1 = transmute(vcagt_f64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcagtq_f64() { + let a: f64x2 = f64x2::new(-1.2, 0.0); + let b: f64x2 = f64x2::new(-1.1, 0.0); + let e: u64x2 = u64x2::new(0xFF_FF_FF_FF_FF_FF_FF_FF, 0); + let r: u64x2 = transmute(vcagtq_f64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcage_f64() { + let a: f64 = -1.2; + let b: f64 = -1.1; + let e: u64x1 = u64x1::new(0xFF_FF_FF_FF_FF_FF_FF_FF); + let r: u64x1 = transmute(vcage_f64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcageq_f64() { + let a: f64x2 = f64x2::new(-1.2, 0.0); + let b: f64x2 = f64x2::new(-1.1, 0.0); + let e: u64x2 = u64x2::new(0xFF_FF_FF_FF_FF_FF_FF_FF, 0xFF_FF_FF_FF_FF_FF_FF_FF); + let r: u64x2 = transmute(vcageq_f64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcalt_f64() { + let a: f64 = -1.2; + let b: f64 = -1.1; + let e: u64x1 = u64x1::new(0); + let r: u64x1 = transmute(vcalt_f64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcaltq_f64() { + let a: f64x2 = f64x2::new(-1.2, 0.0); + let b: f64x2 = f64x2::new(-1.1, 0.0); + let e: u64x2 = u64x2::new(0, 0); + let r: u64x2 = transmute(vcaltq_f64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcale_f64() { + let a: f64 = -1.2; + let b: f64 = -1.1; + let e: u64x1 = u64x1::new(0); + let r: u64x1 = transmute(vcale_f64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcaleq_f64() { + let a: f64x2 = f64x2::new(-1.2, 0.0); + let b: f64x2 = f64x2::new(-1.1, 0.0); + let e: u64x2 = u64x2::new(0, 0xFF_FF_FF_FF_FF_FF_FF_FF); + let r: u64x2 = transmute(vcaleq_f64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcopy_lane_s8() { + let a: i8x8 = i8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let b: i8x8 = i8x8::new(0, 0x7F, 0, 0, 0, 0, 0, 0); + let e: i8x8 = i8x8::new(0x7F, 2, 3, 4, 5, 6, 7, 8); + let r: i8x8 = transmute(vcopy_lane_s8::<0, 1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcopyq_laneq_s8() { + let a: i8x16 = i8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let b: i8x16 = i8x16::new(0, 0x7F, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + let e: i8x16 = i8x16::new(0x7F, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let r: i8x16 = transmute(vcopyq_laneq_s8::<0, 1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcopy_lane_s16() { + let a: i16x4 = i16x4::new(1, 2, 3, 4); + let b: i16x4 = i16x4::new(0, 0x7F_FF, 0, 0); + let e: i16x4 = i16x4::new(0x7F_FF, 2, 3, 4); + let r: i16x4 = transmute(vcopy_lane_s16::<0, 1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcopyq_laneq_s16() { + let a: i16x8 = i16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let b: i16x8 = i16x8::new(0, 0x7F_FF, 0, 0, 0, 0, 0, 0); + let e: i16x8 = i16x8::new(0x7F_FF, 2, 3, 4, 5, 6, 7, 8); + let r: i16x8 = transmute(vcopyq_laneq_s16::<0, 1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcopy_lane_s32() { + let a: i32x2 = i32x2::new(1, 2); + let b: i32x2 = i32x2::new(0, 0x7F_FF_FF_FF); + let e: i32x2 = i32x2::new(0x7F_FF_FF_FF, 2); + let r: i32x2 = transmute(vcopy_lane_s32::<0, 1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcopyq_laneq_s32() { + let a: i32x4 = i32x4::new(1, 2, 3, 4); + let b: i32x4 = i32x4::new(0, 0x7F_FF_FF_FF, 0, 0); + let e: i32x4 = i32x4::new(0x7F_FF_FF_FF, 2, 3, 4); + let r: i32x4 = transmute(vcopyq_laneq_s32::<0, 1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcopyq_laneq_s64() { + let a: i64x2 = i64x2::new(1, 2); + let b: i64x2 = i64x2::new(0, 0x7F_FF_FF_FF_FF_FF_FF_FF); + let e: i64x2 = i64x2::new(0x7F_FF_FF_FF_FF_FF_FF_FF, 2); + let r: i64x2 = transmute(vcopyq_laneq_s64::<0, 1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcopy_lane_u8() { + let a: u8x8 = u8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let b: u8x8 = u8x8::new(0, 0xFF, 0, 0, 0, 0, 0, 0); + let e: u8x8 = u8x8::new(0xFF, 2, 3, 4, 5, 6, 7, 8); + let r: u8x8 = transmute(vcopy_lane_u8::<0, 1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcopyq_laneq_u8() { + let a: u8x16 = u8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let b: u8x16 = u8x16::new(0, 0xFF, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + let e: u8x16 = u8x16::new(0xFF, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let r: u8x16 = transmute(vcopyq_laneq_u8::<0, 1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcopy_lane_u16() { + let a: u16x4 = u16x4::new(1, 2, 3, 4); + let b: u16x4 = u16x4::new(0, 0xFF_FF, 0, 0); + let e: u16x4 = u16x4::new(0xFF_FF, 2, 3, 4); + let r: u16x4 = transmute(vcopy_lane_u16::<0, 1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcopyq_laneq_u16() { + let a: u16x8 = u16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let b: u16x8 = u16x8::new(0, 0xFF_FF, 0, 0, 0, 0, 0, 0); + let e: u16x8 = u16x8::new(0xFF_FF, 2, 3, 4, 5, 6, 7, 8); + let r: u16x8 = transmute(vcopyq_laneq_u16::<0, 1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcopy_lane_u32() { + let a: u32x2 = u32x2::new(1, 2); + let b: u32x2 = u32x2::new(0, 0xFF_FF_FF_FF); + let e: u32x2 = u32x2::new(0xFF_FF_FF_FF, 2); + let r: u32x2 = transmute(vcopy_lane_u32::<0, 1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcopyq_laneq_u32() { + let a: u32x4 = u32x4::new(1, 2, 3, 4); + let b: u32x4 = u32x4::new(0, 0xFF_FF_FF_FF, 0, 0); + let e: u32x4 = u32x4::new(0xFF_FF_FF_FF, 2, 3, 4); + let r: u32x4 = transmute(vcopyq_laneq_u32::<0, 1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcopyq_laneq_u64() { + let a: u64x2 = u64x2::new(1, 2); + let b: u64x2 = u64x2::new(0, 0xFF_FF_FF_FF_FF_FF_FF_FF); + let e: u64x2 = u64x2::new(0xFF_FF_FF_FF_FF_FF_FF_FF, 2); + let r: u64x2 = transmute(vcopyq_laneq_u64::<0, 1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcopy_lane_p8() { + let a: i8x8 = i8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let b: i8x8 = i8x8::new(0, 0x7F, 0, 0, 0, 0, 0, 0); + let e: i8x8 = i8x8::new(0x7F, 2, 3, 4, 5, 6, 7, 8); + let r: i8x8 = transmute(vcopy_lane_p8::<0, 1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcopyq_laneq_p8() { + let a: i8x16 = i8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let b: i8x16 = i8x16::new(0, 0x7F, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + let e: i8x16 = i8x16::new(0x7F, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let r: i8x16 = transmute(vcopyq_laneq_p8::<0, 1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcopy_lane_p16() { + let a: i16x4 = i16x4::new(1, 2, 3, 4); + let b: i16x4 = i16x4::new(0, 0x7F_FF, 0, 0); + let e: i16x4 = i16x4::new(0x7F_FF, 2, 3, 4); + let r: i16x4 = transmute(vcopy_lane_p16::<0, 1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcopyq_laneq_p16() { + let a: i16x8 = i16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let b: i16x8 = i16x8::new(0, 0x7F_FF, 0, 0, 0, 0, 0, 0); + let e: i16x8 = i16x8::new(0x7F_FF, 2, 3, 4, 5, 6, 7, 8); + let r: i16x8 = transmute(vcopyq_laneq_p16::<0, 1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcopyq_laneq_p64() { + let a: i64x2 = i64x2::new(1, 2); + let b: i64x2 = i64x2::new(0, 0x7F_FF_FF_FF_FF_FF_FF_FF); + let e: i64x2 = i64x2::new(0x7F_FF_FF_FF_FF_FF_FF_FF, 2); + let r: i64x2 = transmute(vcopyq_laneq_p64::<0, 1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcopy_lane_f32() { + let a: f32x2 = f32x2::new(1., 2.); + let b: f32x2 = f32x2::new(0., 0.5); + let e: f32x2 = f32x2::new(0.5, 2.); + let r: f32x2 = transmute(vcopy_lane_f32::<0, 1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcopyq_laneq_f32() { + let a: f32x4 = f32x4::new(1., 2., 3., 4.); + let b: f32x4 = f32x4::new(0., 0.5, 0., 0.); + let e: f32x4 = f32x4::new(0.5, 2., 3., 4.); + let r: f32x4 = transmute(vcopyq_laneq_f32::<0, 1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcopyq_laneq_f64() { + let a: f64x2 = f64x2::new(1., 2.); + let b: f64x2 = f64x2::new(0., 0.5); + let e: f64x2 = f64x2::new(0.5, 2.); + let r: f64x2 = transmute(vcopyq_laneq_f64::<0, 1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcopy_laneq_s8() { + let a: i8x8 = i8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let b: i8x16 = i8x16::new(0, 0x7F, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + let e: i8x8 = i8x8::new(0x7F, 2, 3, 4, 5, 6, 7, 8); + let r: i8x8 = transmute(vcopy_laneq_s8::<0, 1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcopy_laneq_s16() { + let a: i16x4 = i16x4::new(1, 2, 3, 4); + let b: i16x8 = i16x8::new(0, 0x7F_FF, 0, 0, 0, 0, 0, 0); + let e: i16x4 = i16x4::new(0x7F_FF, 2, 3, 4); + let r: i16x4 = transmute(vcopy_laneq_s16::<0, 1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcopy_laneq_s32() { + let a: i32x2 = i32x2::new(1, 2); + let b: i32x4 = i32x4::new(0, 0x7F_FF_FF_FF, 0, 0); + let e: i32x2 = i32x2::new(0x7F_FF_FF_FF, 2); + let r: i32x2 = transmute(vcopy_laneq_s32::<0, 1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcopy_laneq_u8() { + let a: u8x8 = u8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let b: u8x16 = u8x16::new(0, 0xFF, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + let e: u8x8 = u8x8::new(0xFF, 2, 3, 4, 5, 6, 7, 8); + let r: u8x8 = transmute(vcopy_laneq_u8::<0, 1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcopy_laneq_u16() { + let a: u16x4 = u16x4::new(1, 2, 3, 4); + let b: u16x8 = u16x8::new(0, 0xFF_FF, 0, 0, 0, 0, 0, 0); + let e: u16x4 = u16x4::new(0xFF_FF, 2, 3, 4); + let r: u16x4 = transmute(vcopy_laneq_u16::<0, 1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcopy_laneq_u32() { + let a: u32x2 = u32x2::new(1, 2); + let b: u32x4 = u32x4::new(0, 0xFF_FF_FF_FF, 0, 0); + let e: u32x2 = u32x2::new(0xFF_FF_FF_FF, 2); + let r: u32x2 = transmute(vcopy_laneq_u32::<0, 1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcopy_laneq_p8() { + let a: i8x8 = i8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let b: i8x16 = i8x16::new(0, 0x7F, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + let e: i8x8 = i8x8::new(0x7F, 2, 3, 4, 5, 6, 7, 8); + let r: i8x8 = transmute(vcopy_laneq_p8::<0, 1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcopy_laneq_p16() { + let a: i16x4 = i16x4::new(1, 2, 3, 4); + let b: i16x8 = i16x8::new(0, 0x7F_FF, 0, 0, 0, 0, 0, 0); + let e: i16x4 = i16x4::new(0x7F_FF, 2, 3, 4); + let r: i16x4 = transmute(vcopy_laneq_p16::<0, 1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcopy_laneq_f32() { + let a: f32x2 = f32x2::new(1., 2.); + let b: f32x4 = f32x4::new(0., 0.5, 0., 0.); + let e: f32x2 = f32x2::new(0.5, 2.); + let r: f32x2 = transmute(vcopy_laneq_f32::<0, 1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcopyq_lane_s8() { + let a: i8x16 = i8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let b: i8x8 = i8x8::new(0, 0x7F, 0, 0, 0, 0, 0, 0); + let e: i8x16 = i8x16::new(0x7F, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let r: i8x16 = transmute(vcopyq_lane_s8::<0, 1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcopyq_lane_s16() { + let a: i16x8 = i16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let b: i16x4 = i16x4::new(0, 0x7F_FF, 0, 0); + let e: i16x8 = i16x8::new(0x7F_FF, 2, 3, 4, 5, 6, 7, 8); + let r: i16x8 = transmute(vcopyq_lane_s16::<0, 1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcopyq_lane_s32() { + let a: i32x4 = i32x4::new(1, 2, 3, 4); + let b: i32x2 = i32x2::new(0, 0x7F_FF_FF_FF); + let e: i32x4 = i32x4::new(0x7F_FF_FF_FF, 2, 3, 4); + let r: i32x4 = transmute(vcopyq_lane_s32::<0, 1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcopyq_lane_u8() { + let a: u8x16 = u8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let b: u8x8 = u8x8::new(0, 0xFF, 0, 0, 0, 0, 0, 0); + let e: u8x16 = u8x16::new(0xFF, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let r: u8x16 = transmute(vcopyq_lane_u8::<0, 1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcopyq_lane_u16() { + let a: u16x8 = u16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let b: u16x4 = u16x4::new(0, 0xFF_FF, 0, 0); + let e: u16x8 = u16x8::new(0xFF_FF, 2, 3, 4, 5, 6, 7, 8); + let r: u16x8 = transmute(vcopyq_lane_u16::<0, 1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcopyq_lane_u32() { + let a: u32x4 = u32x4::new(1, 2, 3, 4); + let b: u32x2 = u32x2::new(0, 0xFF_FF_FF_FF); + let e: u32x4 = u32x4::new(0xFF_FF_FF_FF, 2, 3, 4); + let r: u32x4 = transmute(vcopyq_lane_u32::<0, 1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcopyq_lane_p8() { + let a: i8x16 = i8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let b: i8x8 = i8x8::new(0, 0x7F, 0, 0, 0, 0, 0, 0); + let e: i8x16 = i8x16::new(0x7F, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let r: i8x16 = transmute(vcopyq_lane_p8::<0, 1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcopyq_lane_p16() { + let a: i16x8 = i16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let b: i16x4 = i16x4::new(0, 0x7F_FF, 0, 0); + let e: i16x8 = i16x8::new(0x7F_FF, 2, 3, 4, 5, 6, 7, 8); + let r: i16x8 = transmute(vcopyq_lane_p16::<0, 1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcopyq_lane_s64() { + let a: i64x2 = i64x2::new(1, 2); + let b: i64x1 = i64x1::new(0x7F_FF_FF_FF_FF_FF_FF_FF); + let e: i64x2 = i64x2::new(1, 0x7F_FF_FF_FF_FF_FF_FF_FF); + let r: i64x2 = transmute(vcopyq_lane_s64::<1, 0>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcopyq_lane_u64() { + let a: u64x2 = u64x2::new(1, 2); + let b: u64x1 = u64x1::new(0xFF_FF_FF_FF_FF_FF_FF_FF); + let e: u64x2 = u64x2::new(1, 0xFF_FF_FF_FF_FF_FF_FF_FF); + let r: u64x2 = transmute(vcopyq_lane_u64::<1, 0>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcopyq_lane_p64() { + let a: i64x2 = i64x2::new(1, 2); + let b: i64x1 = i64x1::new(0x7F_FF_FF_FF_FF_FF_FF_FF); + let e: i64x2 = i64x2::new(1, 0x7F_FF_FF_FF_FF_FF_FF_FF); + let r: i64x2 = transmute(vcopyq_lane_p64::<1, 0>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcopyq_lane_f32() { + let a: f32x4 = f32x4::new(1., 2., 3., 4.); + let b: f32x2 = f32x2::new(0.5, 0.); + let e: f32x4 = f32x4::new(1., 0.5, 3., 4.); + let r: f32x4 = transmute(vcopyq_lane_f32::<1, 0>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcopyq_lane_f64() { + let a: f64x2 = f64x2::new(1., 2.); + let b: f64 = 0.5; + let e: f64x2 = f64x2::new(1., 0.5); + let r: f64x2 = transmute(vcopyq_lane_f64::<1, 0>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcreate_f64() { + let a: u64 = 0; + let e: f64 = 0.; + let r: f64 = transmute(vcreate_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvt_f64_s64() { + let a: i64x1 = i64x1::new(1); + let e: f64 = 1.; + let r: f64 = transmute(vcvt_f64_s64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvtq_f64_s64() { + let a: i64x2 = i64x2::new(1, 2); + let e: f64x2 = f64x2::new(1., 2.); + let r: f64x2 = transmute(vcvtq_f64_s64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvt_f64_u64() { + let a: u64x1 = u64x1::new(1); + let e: f64 = 1.; + let r: f64 = transmute(vcvt_f64_u64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvtq_f64_u64() { + let a: u64x2 = u64x2::new(1, 2); + let e: f64x2 = f64x2::new(1., 2.); + let r: f64x2 = transmute(vcvtq_f64_u64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvt_f64_f32() { + let a: f32x2 = f32x2::new(-1.2, 1.2); + let e: f64x2 = f64x2::new(-1.2f32 as f64, 1.2f32 as f64); + let r: f64x2 = transmute(vcvt_f64_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvt_high_f64_f32() { + let a: f32x4 = f32x4::new(-1.2, 1.2, 2.3, 3.4); + let e: f64x2 = f64x2::new(2.3f32 as f64, 3.4f32 as f64); + let r: f64x2 = transmute(vcvt_high_f64_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvt_f32_f64() { + let a: f64x2 = f64x2::new(-1.2, 1.2); + let e: f32x2 = f32x2::new(-1.2f64 as f32, 1.2f64 as f32); + let r: f32x2 = transmute(vcvt_f32_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvt_high_f32_f64() { + let a: f32x2 = f32x2::new(-1.2, 1.2); + let b: f64x2 = f64x2::new(-2.3, 3.4); + let e: f32x4 = f32x4::new(-1.2, 1.2, -2.3f64 as f32, 3.4f64 as f32); + let r: f32x4 = transmute(vcvt_high_f32_f64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvtx_f32_f64() { + let a: f64x2 = f64x2::new(-1.0, 2.0); + let e: f32x2 = f32x2::new(-1.0, 2.0); + let r: f32x2 = transmute(vcvtx_f32_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvtx_high_f32_f64() { + let a: f32x2 = f32x2::new(-1.0, 2.0); + let b: f64x2 = f64x2::new(-3.0, 4.0); + let e: f32x4 = f32x4::new(-1.0, 2.0, -3.0, 4.0); + let r: f32x4 = transmute(vcvtx_high_f32_f64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvt_n_f64_s64() { + let a: i64x1 = i64x1::new(1); + let e: f64 = 0.25; + let r: f64 = transmute(vcvt_n_f64_s64::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvtq_n_f64_s64() { + let a: i64x2 = i64x2::new(1, 2); + let e: f64x2 = f64x2::new(0.25, 0.5); + let r: f64x2 = transmute(vcvtq_n_f64_s64::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvts_n_f32_s32() { + let a: i32 = 1; + let e: f32 = 0.25; + let r: f32 = transmute(vcvts_n_f32_s32::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvtd_n_f64_s64() { + let a: i64 = 1; + let e: f64 = 0.25; + let r: f64 = transmute(vcvtd_n_f64_s64::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvt_n_f64_u64() { + let a: u64x1 = u64x1::new(1); + let e: f64 = 0.25; + let r: f64 = transmute(vcvt_n_f64_u64::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvtq_n_f64_u64() { + let a: u64x2 = u64x2::new(1, 2); + let e: f64x2 = f64x2::new(0.25, 0.5); + let r: f64x2 = transmute(vcvtq_n_f64_u64::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvts_n_f32_u32() { + let a: u32 = 1; + let e: f32 = 0.25; + let r: f32 = transmute(vcvts_n_f32_u32::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvtd_n_f64_u64() { + let a: u64 = 1; + let e: f64 = 0.25; + let r: f64 = transmute(vcvtd_n_f64_u64::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvt_n_s64_f64() { + let a: f64 = 0.25; + let e: i64x1 = i64x1::new(1); + let r: i64x1 = transmute(vcvt_n_s64_f64::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvtq_n_s64_f64() { + let a: f64x2 = f64x2::new(0.25, 0.5); + let e: i64x2 = i64x2::new(1, 2); + let r: i64x2 = transmute(vcvtq_n_s64_f64::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvts_n_s32_f32() { + let a: f32 = 0.25; + let e: i32 = 1; + let r: i32 = transmute(vcvts_n_s32_f32::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvtd_n_s64_f64() { + let a: f64 = 0.25; + let e: i64 = 1; + let r: i64 = transmute(vcvtd_n_s64_f64::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvt_n_u64_f64() { + let a: f64 = 0.25; + let e: u64x1 = u64x1::new(1); + let r: u64x1 = transmute(vcvt_n_u64_f64::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvtq_n_u64_f64() { + let a: f64x2 = f64x2::new(0.25, 0.5); + let e: u64x2 = u64x2::new(1, 2); + let r: u64x2 = transmute(vcvtq_n_u64_f64::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvts_n_u32_f32() { + let a: f32 = 0.25; + let e: u32 = 1; + let r: u32 = transmute(vcvts_n_u32_f32::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvtd_n_u64_f64() { + let a: f64 = 0.25; + let e: u64 = 1; + let r: u64 = transmute(vcvtd_n_u64_f64::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvts_f32_s32() { + let a: i32 = 1; + let e: f32 = 1.; + let r: f32 = transmute(vcvts_f32_s32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvtd_f64_s64() { + let a: i64 = 1; + let e: f64 = 1.; + let r: f64 = transmute(vcvtd_f64_s64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvts_f32_u32() { + let a: u32 = 1; + let e: f32 = 1.; + let r: f32 = transmute(vcvts_f32_u32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvtd_f64_u64() { + let a: u64 = 1; + let e: f64 = 1.; + let r: f64 = transmute(vcvtd_f64_u64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvts_s32_f32() { + let a: f32 = 1.; + let e: i32 = 1; + let r: i32 = transmute(vcvts_s32_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvtd_s64_f64() { + let a: f64 = 1.; + let e: i64 = 1; + let r: i64 = transmute(vcvtd_s64_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvts_u32_f32() { + let a: f32 = 1.; + let e: u32 = 1; + let r: u32 = transmute(vcvts_u32_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvtd_u64_f64() { + let a: f64 = 1.; + let e: u64 = 1; + let r: u64 = transmute(vcvtd_u64_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvt_s64_f64() { + let a: f64 = -1.1; + let e: i64x1 = i64x1::new(-1); + let r: i64x1 = transmute(vcvt_s64_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvtq_s64_f64() { + let a: f64x2 = f64x2::new(-1.1, 2.1); + let e: i64x2 = i64x2::new(-1, 2); + let r: i64x2 = transmute(vcvtq_s64_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvt_u64_f64() { + let a: f64 = 1.1; + let e: u64x1 = u64x1::new(1); + let r: u64x1 = transmute(vcvt_u64_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvtq_u64_f64() { + let a: f64x2 = f64x2::new(1.1, 2.1); + let e: u64x2 = u64x2::new(1, 2); + let r: u64x2 = transmute(vcvtq_u64_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvta_s32_f32() { + let a: f32x2 = f32x2::new(-1.1, 2.1); + let e: i32x2 = i32x2::new(-1, 2); + let r: i32x2 = transmute(vcvta_s32_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvtaq_s32_f32() { + let a: f32x4 = f32x4::new(-1.1, 2.1, -2.9, 3.9); + let e: i32x4 = i32x4::new(-1, 2, -3, 4); + let r: i32x4 = transmute(vcvtaq_s32_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvta_s64_f64() { + let a: f64 = -1.1; + let e: i64x1 = i64x1::new(-1); + let r: i64x1 = transmute(vcvta_s64_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvtaq_s64_f64() { + let a: f64x2 = f64x2::new(-1.1, 2.1); + let e: i64x2 = i64x2::new(-1, 2); + let r: i64x2 = transmute(vcvtaq_s64_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvtas_s32_f32() { + let a: f32 = 2.9; + let e: i32 = 3; + let r: i32 = transmute(vcvtas_s32_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvtad_s64_f64() { + let a: f64 = 2.9; + let e: i64 = 3; + let r: i64 = transmute(vcvtad_s64_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvtas_u32_f32() { + let a: f32 = 2.9; + let e: u32 = 3; + let r: u32 = transmute(vcvtas_u32_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvtad_u64_f64() { + let a: f64 = 2.9; + let e: u64 = 3; + let r: u64 = transmute(vcvtad_u64_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvtn_s32_f32() { + let a: f32x2 = f32x2::new(-1.5, 2.1); + let e: i32x2 = i32x2::new(-2, 2); + let r: i32x2 = transmute(vcvtn_s32_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvtnq_s32_f32() { + let a: f32x4 = f32x4::new(-1.5, 2.1, -2.9, 3.9); + let e: i32x4 = i32x4::new(-2, 2, -3, 4); + let r: i32x4 = transmute(vcvtnq_s32_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvtn_s64_f64() { + let a: f64 = -1.5; + let e: i64x1 = i64x1::new(-2); + let r: i64x1 = transmute(vcvtn_s64_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvtnq_s64_f64() { + let a: f64x2 = f64x2::new(-1.5, 2.1); + let e: i64x2 = i64x2::new(-2, 2); + let r: i64x2 = transmute(vcvtnq_s64_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvtns_s32_f32() { + let a: f32 = -1.5; + let e: i32 = -2; + let r: i32 = transmute(vcvtns_s32_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvtnd_s64_f64() { + let a: f64 = -1.5; + let e: i64 = -2; + let r: i64 = transmute(vcvtnd_s64_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvtm_s32_f32() { + let a: f32x2 = f32x2::new(-1.1, 2.1); + let e: i32x2 = i32x2::new(-2, 2); + let r: i32x2 = transmute(vcvtm_s32_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvtmq_s32_f32() { + let a: f32x4 = f32x4::new(-1.1, 2.1, -2.9, 3.9); + let e: i32x4 = i32x4::new(-2, 2, -3, 3); + let r: i32x4 = transmute(vcvtmq_s32_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvtm_s64_f64() { + let a: f64 = -1.1; + let e: i64x1 = i64x1::new(-2); + let r: i64x1 = transmute(vcvtm_s64_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvtmq_s64_f64() { + let a: f64x2 = f64x2::new(-1.1, 2.1); + let e: i64x2 = i64x2::new(-2, 2); + let r: i64x2 = transmute(vcvtmq_s64_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvtms_s32_f32() { + let a: f32 = -1.1; + let e: i32 = -2; + let r: i32 = transmute(vcvtms_s32_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvtmd_s64_f64() { + let a: f64 = -1.1; + let e: i64 = -2; + let r: i64 = transmute(vcvtmd_s64_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvtp_s32_f32() { + let a: f32x2 = f32x2::new(-1.1, 2.1); + let e: i32x2 = i32x2::new(-1, 3); + let r: i32x2 = transmute(vcvtp_s32_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvtpq_s32_f32() { + let a: f32x4 = f32x4::new(-1.1, 2.1, -2.9, 3.9); + let e: i32x4 = i32x4::new(-1, 3, -2, 4); + let r: i32x4 = transmute(vcvtpq_s32_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvtp_s64_f64() { + let a: f64 = -1.1; + let e: i64x1 = i64x1::new(-1); + let r: i64x1 = transmute(vcvtp_s64_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvtpq_s64_f64() { + let a: f64x2 = f64x2::new(-1.1, 2.1); + let e: i64x2 = i64x2::new(-1, 3); + let r: i64x2 = transmute(vcvtpq_s64_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvtps_s32_f32() { + let a: f32 = -1.1; + let e: i32 = -1; + let r: i32 = transmute(vcvtps_s32_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvtpd_s64_f64() { + let a: f64 = -1.1; + let e: i64 = -1; + let r: i64 = transmute(vcvtpd_s64_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvta_u32_f32() { + let a: f32x2 = f32x2::new(1.1, 2.1); + let e: u32x2 = u32x2::new(1, 2); + let r: u32x2 = transmute(vcvta_u32_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvtaq_u32_f32() { + let a: f32x4 = f32x4::new(1.1, 2.1, 2.9, 3.9); + let e: u32x4 = u32x4::new(1, 2, 3, 4); + let r: u32x4 = transmute(vcvtaq_u32_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvta_u64_f64() { + let a: f64 = 1.1; + let e: u64x1 = u64x1::new(1); + let r: u64x1 = transmute(vcvta_u64_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvtaq_u64_f64() { + let a: f64x2 = f64x2::new(1.1, 2.1); + let e: u64x2 = u64x2::new(1, 2); + let r: u64x2 = transmute(vcvtaq_u64_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvtn_u32_f32() { + let a: f32x2 = f32x2::new(1.5, 2.1); + let e: u32x2 = u32x2::new(2, 2); + let r: u32x2 = transmute(vcvtn_u32_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvtnq_u32_f32() { + let a: f32x4 = f32x4::new(1.5, 2.1, 2.9, 3.9); + let e: u32x4 = u32x4::new(2, 2, 3, 4); + let r: u32x4 = transmute(vcvtnq_u32_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvtn_u64_f64() { + let a: f64 = 1.5; + let e: u64x1 = u64x1::new(2); + let r: u64x1 = transmute(vcvtn_u64_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvtnq_u64_f64() { + let a: f64x2 = f64x2::new(1.5, 2.1); + let e: u64x2 = u64x2::new(2, 2); + let r: u64x2 = transmute(vcvtnq_u64_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvtns_u32_f32() { + let a: f32 = 1.5; + let e: u32 = 2; + let r: u32 = transmute(vcvtns_u32_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvtnd_u64_f64() { + let a: f64 = 1.5; + let e: u64 = 2; + let r: u64 = transmute(vcvtnd_u64_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvtm_u32_f32() { + let a: f32x2 = f32x2::new(1.1, 2.1); + let e: u32x2 = u32x2::new(1, 2); + let r: u32x2 = transmute(vcvtm_u32_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvtmq_u32_f32() { + let a: f32x4 = f32x4::new(1.1, 2.1, 2.9, 3.9); + let e: u32x4 = u32x4::new(1, 2, 2, 3); + let r: u32x4 = transmute(vcvtmq_u32_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvtm_u64_f64() { + let a: f64 = 1.1; + let e: u64x1 = u64x1::new(1); + let r: u64x1 = transmute(vcvtm_u64_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvtmq_u64_f64() { + let a: f64x2 = f64x2::new(1.1, 2.1); + let e: u64x2 = u64x2::new(1, 2); + let r: u64x2 = transmute(vcvtmq_u64_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvtms_u32_f32() { + let a: f32 = 1.1; + let e: u32 = 1; + let r: u32 = transmute(vcvtms_u32_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvtmd_u64_f64() { + let a: f64 = 1.1; + let e: u64 = 1; + let r: u64 = transmute(vcvtmd_u64_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvtp_u32_f32() { + let a: f32x2 = f32x2::new(1.1, 2.1); + let e: u32x2 = u32x2::new(2, 3); + let r: u32x2 = transmute(vcvtp_u32_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvtpq_u32_f32() { + let a: f32x4 = f32x4::new(1.1, 2.1, 2.9, 3.9); + let e: u32x4 = u32x4::new(2, 3, 3, 4); + let r: u32x4 = transmute(vcvtpq_u32_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvtp_u64_f64() { + let a: f64 = 1.1; + let e: u64x1 = u64x1::new(2); + let r: u64x1 = transmute(vcvtp_u64_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvtpq_u64_f64() { + let a: f64x2 = f64x2::new(1.1, 2.1); + let e: u64x2 = u64x2::new(2, 3); + let r: u64x2 = transmute(vcvtpq_u64_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvtps_u32_f32() { + let a: f32 = 1.1; + let e: u32 = 2; + let r: u32 = transmute(vcvtps_u32_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvtpd_u64_f64() { + let a: f64 = 1.1; + let e: u64 = 2; + let r: u64 = transmute(vcvtpd_u64_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdupq_laneq_p64() { + let a: i64x2 = i64x2::new(1, 1); + let e: i64x2 = i64x2::new(1, 1); + let r: i64x2 = transmute(vdupq_laneq_p64::<1>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdupq_lane_p64() { + let a: i64x1 = i64x1::new(1); + let e: i64x2 = i64x2::new(1, 1); + let r: i64x2 = transmute(vdupq_lane_p64::<0>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdupq_laneq_f64() { + let a: f64x2 = f64x2::new(1., 1.); + let e: f64x2 = f64x2::new(1., 1.); + let r: f64x2 = transmute(vdupq_laneq_f64::<1>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdupq_lane_f64() { + let a: f64 = 1.; + let e: f64x2 = f64x2::new(1., 1.); + let r: f64x2 = transmute(vdupq_lane_f64::<0>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdup_lane_p64() { + let a: i64x1 = i64x1::new(0); + let e: i64x1 = i64x1::new(0); + let r: i64x1 = transmute(vdup_lane_p64::<0>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdup_lane_f64() { + let a: f64 = 0.; + let e: f64 = 0.; + let r: f64 = transmute(vdup_lane_f64::<0>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdup_laneq_p64() { + let a: i64x2 = i64x2::new(0, 1); + let e: i64x1 = i64x1::new(1); + let r: i64x1 = transmute(vdup_laneq_p64::<1>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdup_laneq_f64() { + let a: f64x2 = f64x2::new(0., 1.); + let e: f64 = 1.; + let r: f64 = transmute(vdup_laneq_f64::<1>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdupb_lane_s8() { + let a: i8x8 = i8x8::new(1, 1, 1, 4, 1, 6, 7, 8); + let e: i8 = 1; + let r: i8 = transmute(vdupb_lane_s8::<4>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdupb_laneq_s8() { + let a: i8x16 = i8x16::new(1, 1, 1, 4, 1, 6, 7, 8, 1, 10, 11, 12, 13, 14, 15, 16); + let e: i8 = 1; + let r: i8 = transmute(vdupb_laneq_s8::<8>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vduph_lane_s16() { + let a: i16x4 = i16x4::new(1, 1, 1, 4); + let e: i16 = 1; + let r: i16 = transmute(vduph_lane_s16::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vduph_laneq_s16() { + let a: i16x8 = i16x8::new(1, 1, 1, 4, 1, 6, 7, 8); + let e: i16 = 1; + let r: i16 = transmute(vduph_laneq_s16::<4>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdups_lane_s32() { + let a: i32x2 = i32x2::new(1, 1); + let e: i32 = 1; + let r: i32 = transmute(vdups_lane_s32::<1>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdups_laneq_s32() { + let a: i32x4 = i32x4::new(1, 1, 1, 4); + let e: i32 = 1; + let r: i32 = transmute(vdups_laneq_s32::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdupd_lane_s64() { + let a: i64x1 = i64x1::new(1); + let e: i64 = 1; + let r: i64 = transmute(vdupd_lane_s64::<0>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdupd_laneq_s64() { + let a: i64x2 = i64x2::new(1, 1); + let e: i64 = 1; + let r: i64 = transmute(vdupd_laneq_s64::<1>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdupb_lane_u8() { + let a: u8x8 = u8x8::new(1, 1, 1, 4, 1, 6, 7, 8); + let e: u8 = 1; + let r: u8 = transmute(vdupb_lane_u8::<4>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdupb_laneq_u8() { + let a: u8x16 = u8x16::new(1, 1, 1, 4, 1, 6, 7, 8, 1, 10, 11, 12, 13, 14, 15, 16); + let e: u8 = 1; + let r: u8 = transmute(vdupb_laneq_u8::<8>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vduph_lane_u16() { + let a: u16x4 = u16x4::new(1, 1, 1, 4); + let e: u16 = 1; + let r: u16 = transmute(vduph_lane_u16::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vduph_laneq_u16() { + let a: u16x8 = u16x8::new(1, 1, 1, 4, 1, 6, 7, 8); + let e: u16 = 1; + let r: u16 = transmute(vduph_laneq_u16::<4>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdups_lane_u32() { + let a: u32x2 = u32x2::new(1, 1); + let e: u32 = 1; + let r: u32 = transmute(vdups_lane_u32::<1>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdups_laneq_u32() { + let a: u32x4 = u32x4::new(1, 1, 1, 4); + let e: u32 = 1; + let r: u32 = transmute(vdups_laneq_u32::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdupd_lane_u64() { + let a: u64x1 = u64x1::new(1); + let e: u64 = 1; + let r: u64 = transmute(vdupd_lane_u64::<0>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdupd_laneq_u64() { + let a: u64x2 = u64x2::new(1, 1); + let e: u64 = 1; + let r: u64 = transmute(vdupd_laneq_u64::<1>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdupb_lane_p8() { + let a: i8x8 = i8x8::new(1, 1, 1, 4, 1, 6, 7, 8); + let e: p8 = 1; + let r: p8 = transmute(vdupb_lane_p8::<4>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdupb_laneq_p8() { + let a: i8x16 = i8x16::new(1, 1, 1, 4, 1, 6, 7, 8, 1, 10, 11, 12, 13, 14, 15, 16); + let e: p8 = 1; + let r: p8 = transmute(vdupb_laneq_p8::<8>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vduph_lane_p16() { + let a: i16x4 = i16x4::new(1, 1, 1, 4); + let e: p16 = 1; + let r: p16 = transmute(vduph_lane_p16::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vduph_laneq_p16() { + let a: i16x8 = i16x8::new(1, 1, 1, 4, 1, 6, 7, 8); + let e: p16 = 1; + let r: p16 = transmute(vduph_laneq_p16::<4>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdups_lane_f32() { + let a: f32x2 = f32x2::new(1., 1.); + let e: f32 = 1.; + let r: f32 = transmute(vdups_lane_f32::<1>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdups_laneq_f32() { + let a: f32x4 = f32x4::new(1., 1., 1., 4.); + let e: f32 = 1.; + let r: f32 = transmute(vdups_laneq_f32::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdupd_lane_f64() { + let a: f64 = 1.; + let e: f64 = 1.; + let r: f64 = transmute(vdupd_lane_f64::<0>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdupd_laneq_f64() { + let a: f64x2 = f64x2::new(1., 1.); + let e: f64 = 1.; + let r: f64 = transmute(vdupd_laneq_f64::<1>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vextq_p64() { + let a: i64x2 = i64x2::new(0, 8); + let b: i64x2 = i64x2::new(9, 11); + let e: i64x2 = i64x2::new(8, 9); + let r: i64x2 = transmute(vextq_p64::<1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vextq_f64() { + let a: f64x2 = f64x2::new(0., 2.); + let b: f64x2 = f64x2::new(3., 4.); + let e: f64x2 = f64x2::new(2., 3.); + let r: f64x2 = transmute(vextq_f64::<1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmla_f64() { + let a: f64 = 0.; + let b: f64 = 2.; + let c: f64 = 3.; + let e: f64 = 6.; + let r: f64 = transmute(vmla_f64(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlaq_f64() { + let a: f64x2 = f64x2::new(0., 1.); + let b: f64x2 = f64x2::new(2., 2.); + let c: f64x2 = f64x2::new(3., 3.); + let e: f64x2 = f64x2::new(6., 7.); + let r: f64x2 = transmute(vmlaq_f64(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlal_high_s8() { + let a: i16x8 = i16x8::new(8, 7, 6, 5, 4, 3, 2, 1); + let b: i8x16 = i8x16::new(2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2); + let c: i8x16 = i8x16::new(3, 3, 0, 1, 0, 1, 2, 3, 0, 1, 2, 3, 4, 5, 6, 7); + let e: i16x8 = i16x8::new(8, 9, 10, 11, 12, 13, 14, 15); + let r: i16x8 = transmute(vmlal_high_s8(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlal_high_s16() { + let a: i32x4 = i32x4::new(8, 7, 6, 5); + let b: i16x8 = i16x8::new(2, 2, 2, 2, 2, 2, 2, 2); + let c: i16x8 = i16x8::new(3, 3, 0, 1, 0, 1, 2, 3); + let e: i32x4 = i32x4::new(8, 9, 10, 11); + let r: i32x4 = transmute(vmlal_high_s16(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlal_high_s32() { + let a: i64x2 = i64x2::new(8, 7); + let b: i32x4 = i32x4::new(2, 2, 2, 2); + let c: i32x4 = i32x4::new(3, 3, 0, 1); + let e: i64x2 = i64x2::new(8, 9); + let r: i64x2 = transmute(vmlal_high_s32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlal_high_u8() { + let a: u16x8 = u16x8::new(8, 7, 6, 5, 4, 3, 2, 1); + let b: u8x16 = u8x16::new(2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2); + let c: u8x16 = u8x16::new(3, 3, 0, 1, 0, 1, 2, 3, 0, 1, 2, 3, 4, 5, 6, 7); + let e: u16x8 = u16x8::new(8, 9, 10, 11, 12, 13, 14, 15); + let r: u16x8 = transmute(vmlal_high_u8(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlal_high_u16() { + let a: u32x4 = u32x4::new(8, 7, 6, 5); + let b: u16x8 = u16x8::new(2, 2, 2, 2, 2, 2, 2, 2); + let c: u16x8 = u16x8::new(3, 3, 0, 1, 0, 1, 2, 3); + let e: u32x4 = u32x4::new(8, 9, 10, 11); + let r: u32x4 = transmute(vmlal_high_u16(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlal_high_u32() { + let a: u64x2 = u64x2::new(8, 7); + let b: u32x4 = u32x4::new(2, 2, 2, 2); + let c: u32x4 = u32x4::new(3, 3, 0, 1); + let e: u64x2 = u64x2::new(8, 9); + let r: u64x2 = transmute(vmlal_high_u32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlal_high_n_s16() { + let a: i32x4 = i32x4::new(8, 7, 6, 5); + let b: i16x8 = i16x8::new(3, 3, 0, 1, 0, 1, 2, 3); + let c: i16 = 2; + let e: i32x4 = i32x4::new(8, 9, 10, 11); + let r: i32x4 = transmute(vmlal_high_n_s16(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlal_high_n_s32() { + let a: i64x2 = i64x2::new(8, 7); + let b: i32x4 = i32x4::new(3, 3, 0, 1); + let c: i32 = 2; + let e: i64x2 = i64x2::new(8, 9); + let r: i64x2 = transmute(vmlal_high_n_s32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlal_high_n_u16() { + let a: u32x4 = u32x4::new(8, 7, 6, 5); + let b: u16x8 = u16x8::new(3, 3, 0, 1, 0, 1, 2, 3); + let c: u16 = 2; + let e: u32x4 = u32x4::new(8, 9, 10, 11); + let r: u32x4 = transmute(vmlal_high_n_u16(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlal_high_n_u32() { + let a: u64x2 = u64x2::new(8, 7); + let b: u32x4 = u32x4::new(3, 3, 0, 1); + let c: u32 = 2; + let e: u64x2 = u64x2::new(8, 9); + let r: u64x2 = transmute(vmlal_high_n_u32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlal_high_lane_s16() { + let a: i32x4 = i32x4::new(8, 7, 6, 5); + let b: i16x8 = i16x8::new(3, 3, 0, 1, 0, 1, 2, 3); + let c: i16x4 = i16x4::new(0, 2, 0, 0); + let e: i32x4 = i32x4::new(8, 9, 10, 11); + let r: i32x4 = transmute(vmlal_high_lane_s16::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlal_high_laneq_s16() { + let a: i32x4 = i32x4::new(8, 7, 6, 5); + let b: i16x8 = i16x8::new(3, 3, 0, 1, 0, 1, 2, 3); + let c: i16x8 = i16x8::new(0, 2, 0, 0, 0, 0, 0, 0); + let e: i32x4 = i32x4::new(8, 9, 10, 11); + let r: i32x4 = transmute(vmlal_high_laneq_s16::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlal_high_lane_s32() { + let a: i64x2 = i64x2::new(8, 7); + let b: i32x4 = i32x4::new(3, 3, 0, 1); + let c: i32x2 = i32x2::new(0, 2); + let e: i64x2 = i64x2::new(8, 9); + let r: i64x2 = transmute(vmlal_high_lane_s32::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlal_high_laneq_s32() { + let a: i64x2 = i64x2::new(8, 7); + let b: i32x4 = i32x4::new(3, 3, 0, 1); + let c: i32x4 = i32x4::new(0, 2, 0, 0); + let e: i64x2 = i64x2::new(8, 9); + let r: i64x2 = transmute(vmlal_high_laneq_s32::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlal_high_lane_u16() { + let a: u32x4 = u32x4::new(8, 7, 6, 5); + let b: u16x8 = u16x8::new(3, 3, 0, 1, 0, 1, 2, 3); + let c: u16x4 = u16x4::new(0, 2, 0, 0); + let e: u32x4 = u32x4::new(8, 9, 10, 11); + let r: u32x4 = transmute(vmlal_high_lane_u16::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlal_high_laneq_u16() { + let a: u32x4 = u32x4::new(8, 7, 6, 5); + let b: u16x8 = u16x8::new(3, 3, 0, 1, 0, 1, 2, 3); + let c: u16x8 = u16x8::new(0, 2, 0, 0, 0, 0, 0, 0); + let e: u32x4 = u32x4::new(8, 9, 10, 11); + let r: u32x4 = transmute(vmlal_high_laneq_u16::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlal_high_lane_u32() { + let a: u64x2 = u64x2::new(8, 7); + let b: u32x4 = u32x4::new(3, 3, 0, 1); + let c: u32x2 = u32x2::new(0, 2); + let e: u64x2 = u64x2::new(8, 9); + let r: u64x2 = transmute(vmlal_high_lane_u32::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlal_high_laneq_u32() { + let a: u64x2 = u64x2::new(8, 7); + let b: u32x4 = u32x4::new(3, 3, 0, 1); + let c: u32x4 = u32x4::new(0, 2, 0, 0); + let e: u64x2 = u64x2::new(8, 9); + let r: u64x2 = transmute(vmlal_high_laneq_u32::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmls_f64() { + let a: f64 = 6.; + let b: f64 = 2.; + let c: f64 = 3.; + let e: f64 = 0.; + let r: f64 = transmute(vmls_f64(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlsq_f64() { + let a: f64x2 = f64x2::new(6., 7.); + let b: f64x2 = f64x2::new(2., 2.); + let c: f64x2 = f64x2::new(3., 3.); + let e: f64x2 = f64x2::new(0., 1.); + let r: f64x2 = transmute(vmlsq_f64(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlsl_high_s8() { + let a: i16x8 = i16x8::new(14, 15, 16, 17, 18, 19, 20, 21); + let b: i8x16 = i8x16::new(2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2); + let c: i8x16 = i8x16::new(3, 3, 0, 1, 0, 1, 2, 3, 0, 1, 2, 3, 4, 5, 6, 7); + let e: i16x8 = i16x8::new(14, 13, 12, 11, 10, 9, 8, 7); + let r: i16x8 = transmute(vmlsl_high_s8(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlsl_high_s16() { + let a: i32x4 = i32x4::new(14, 15, 16, 17); + let b: i16x8 = i16x8::new(2, 2, 2, 2, 2, 2, 2, 2); + let c: i16x8 = i16x8::new(3, 3, 0, 1, 0, 1, 2, 3); + let e: i32x4 = i32x4::new(14, 13, 12, 11); + let r: i32x4 = transmute(vmlsl_high_s16(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlsl_high_s32() { + let a: i64x2 = i64x2::new(14, 15); + let b: i32x4 = i32x4::new(2, 2, 2, 2); + let c: i32x4 = i32x4::new(3, 3, 0, 1); + let e: i64x2 = i64x2::new(14, 13); + let r: i64x2 = transmute(vmlsl_high_s32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlsl_high_u8() { + let a: u16x8 = u16x8::new(14, 15, 16, 17, 18, 19, 20, 21); + let b: u8x16 = u8x16::new(2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2); + let c: u8x16 = u8x16::new(3, 3, 0, 1, 0, 1, 2, 3, 0, 1, 2, 3, 4, 5, 6, 7); + let e: u16x8 = u16x8::new(14, 13, 12, 11, 10, 9, 8, 7); + let r: u16x8 = transmute(vmlsl_high_u8(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlsl_high_u16() { + let a: u32x4 = u32x4::new(14, 15, 16, 17); + let b: u16x8 = u16x8::new(2, 2, 2, 2, 2, 2, 2, 2); + let c: u16x8 = u16x8::new(3, 3, 0, 1, 0, 1, 2, 3); + let e: u32x4 = u32x4::new(14, 13, 12, 11); + let r: u32x4 = transmute(vmlsl_high_u16(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlsl_high_u32() { + let a: u64x2 = u64x2::new(14, 15); + let b: u32x4 = u32x4::new(2, 2, 2, 2); + let c: u32x4 = u32x4::new(3, 3, 0, 1); + let e: u64x2 = u64x2::new(14, 13); + let r: u64x2 = transmute(vmlsl_high_u32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlsl_high_n_s16() { + let a: i32x4 = i32x4::new(14, 15, 16, 17); + let b: i16x8 = i16x8::new(3, 3, 0, 1, 0, 1, 2, 3); + let c: i16 = 2; + let e: i32x4 = i32x4::new(14, 13, 12, 11); + let r: i32x4 = transmute(vmlsl_high_n_s16(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlsl_high_n_s32() { + let a: i64x2 = i64x2::new(14, 15); + let b: i32x4 = i32x4::new(3, 3, 0, 1); + let c: i32 = 2; + let e: i64x2 = i64x2::new(14, 13); + let r: i64x2 = transmute(vmlsl_high_n_s32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlsl_high_n_u16() { + let a: u32x4 = u32x4::new(14, 15, 16, 17); + let b: u16x8 = u16x8::new(3, 3, 0, 1, 0, 1, 2, 3); + let c: u16 = 2; + let e: u32x4 = u32x4::new(14, 13, 12, 11); + let r: u32x4 = transmute(vmlsl_high_n_u16(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlsl_high_n_u32() { + let a: u64x2 = u64x2::new(14, 15); + let b: u32x4 = u32x4::new(3, 3, 0, 1); + let c: u32 = 2; + let e: u64x2 = u64x2::new(14, 13); + let r: u64x2 = transmute(vmlsl_high_n_u32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlsl_high_lane_s16() { + let a: i32x4 = i32x4::new(14, 15, 16, 17); + let b: i16x8 = i16x8::new(3, 3, 0, 1, 0, 1, 2, 3); + let c: i16x4 = i16x4::new(0, 2, 0, 0); + let e: i32x4 = i32x4::new(14, 13, 12, 11); + let r: i32x4 = transmute(vmlsl_high_lane_s16::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlsl_high_laneq_s16() { + let a: i32x4 = i32x4::new(14, 15, 16, 17); + let b: i16x8 = i16x8::new(3, 3, 0, 1, 0, 1, 2, 3); + let c: i16x8 = i16x8::new(0, 2, 0, 0, 0, 0, 0, 0); + let e: i32x4 = i32x4::new(14, 13, 12, 11); + let r: i32x4 = transmute(vmlsl_high_laneq_s16::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlsl_high_lane_s32() { + let a: i64x2 = i64x2::new(14, 15); + let b: i32x4 = i32x4::new(3, 3, 0, 1); + let c: i32x2 = i32x2::new(0, 2); + let e: i64x2 = i64x2::new(14, 13); + let r: i64x2 = transmute(vmlsl_high_lane_s32::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlsl_high_laneq_s32() { + let a: i64x2 = i64x2::new(14, 15); + let b: i32x4 = i32x4::new(3, 3, 0, 1); + let c: i32x4 = i32x4::new(0, 2, 0, 0); + let e: i64x2 = i64x2::new(14, 13); + let r: i64x2 = transmute(vmlsl_high_laneq_s32::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlsl_high_lane_u16() { + let a: u32x4 = u32x4::new(14, 15, 16, 17); + let b: u16x8 = u16x8::new(3, 3, 0, 1, 0, 1, 2, 3); + let c: u16x4 = u16x4::new(0, 2, 0, 0); + let e: u32x4 = u32x4::new(14, 13, 12, 11); + let r: u32x4 = transmute(vmlsl_high_lane_u16::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlsl_high_laneq_u16() { + let a: u32x4 = u32x4::new(14, 15, 16, 17); + let b: u16x8 = u16x8::new(3, 3, 0, 1, 0, 1, 2, 3); + let c: u16x8 = u16x8::new(0, 2, 0, 0, 0, 0, 0, 0); + let e: u32x4 = u32x4::new(14, 13, 12, 11); + let r: u32x4 = transmute(vmlsl_high_laneq_u16::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlsl_high_lane_u32() { + let a: u64x2 = u64x2::new(14, 15); + let b: u32x4 = u32x4::new(3, 3, 0, 1); + let c: u32x2 = u32x2::new(0, 2); + let e: u64x2 = u64x2::new(14, 13); + let r: u64x2 = transmute(vmlsl_high_lane_u32::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlsl_high_laneq_u32() { + let a: u64x2 = u64x2::new(14, 15); + let b: u32x4 = u32x4::new(3, 3, 0, 1); + let c: u32x4 = u32x4::new(0, 2, 0, 0); + let e: u64x2 = u64x2::new(14, 13); + let r: u64x2 = transmute(vmlsl_high_laneq_u32::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmovn_high_s16() { + let a: i8x8 = i8x8::new(0, 1, 2, 3, 2, 3, 4, 5); + let b: i16x8 = i16x8::new(2, 3, 4, 5, 12, 13, 14, 15); + let e: i8x16 = i8x16::new(0, 1, 2, 3, 2, 3, 4, 5, 2, 3, 4, 5, 12, 13, 14, 15); + let r: i8x16 = transmute(vmovn_high_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmovn_high_s32() { + let a: i16x4 = i16x4::new(0, 1, 2, 3); + let b: i32x4 = i32x4::new(2, 3, 4, 5); + let e: i16x8 = i16x8::new(0, 1, 2, 3, 2, 3, 4, 5); + let r: i16x8 = transmute(vmovn_high_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmovn_high_s64() { + let a: i32x2 = i32x2::new(0, 1); + let b: i64x2 = i64x2::new(2, 3); + let e: i32x4 = i32x4::new(0, 1, 2, 3); + let r: i32x4 = transmute(vmovn_high_s64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmovn_high_u16() { + let a: u8x8 = u8x8::new(0, 1, 2, 3, 2, 3, 4, 5); + let b: u16x8 = u16x8::new(2, 3, 4, 5, 12, 13, 14, 15); + let e: u8x16 = u8x16::new(0, 1, 2, 3, 2, 3, 4, 5, 2, 3, 4, 5, 12, 13, 14, 15); + let r: u8x16 = transmute(vmovn_high_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmovn_high_u32() { + let a: u16x4 = u16x4::new(0, 1, 2, 3); + let b: u32x4 = u32x4::new(2, 3, 4, 5); + let e: u16x8 = u16x8::new(0, 1, 2, 3, 2, 3, 4, 5); + let r: u16x8 = transmute(vmovn_high_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmovn_high_u64() { + let a: u32x2 = u32x2::new(0, 1); + let b: u64x2 = u64x2::new(2, 3); + let e: u32x4 = u32x4::new(0, 1, 2, 3); + let r: u32x4 = transmute(vmovn_high_u64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vneg_s64() { + let a: i64x1 = i64x1::new(0); + let e: i64x1 = i64x1::new(0); + let r: i64x1 = transmute(vneg_s64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vnegq_s64() { + let a: i64x2 = i64x2::new(0, 1); + let e: i64x2 = i64x2::new(0, -1); + let r: i64x2 = transmute(vnegq_s64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vneg_f64() { + let a: f64 = 0.; + let e: f64 = 0.; + let r: f64 = transmute(vneg_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vnegq_f64() { + let a: f64x2 = f64x2::new(0., 1.); + let e: f64x2 = f64x2::new(0., -1.); + let r: f64x2 = transmute(vnegq_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqneg_s64() { + let a: i64x1 = i64x1::new(-9223372036854775808); + let e: i64x1 = i64x1::new(0x7F_FF_FF_FF_FF_FF_FF_FF); + let r: i64x1 = transmute(vqneg_s64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqnegq_s64() { + let a: i64x2 = i64x2::new(-9223372036854775808, 0); + let e: i64x2 = i64x2::new(0x7F_FF_FF_FF_FF_FF_FF_FF, 0); + let r: i64x2 = transmute(vqnegq_s64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqsubb_s8() { + let a: i8 = 42; + let b: i8 = 1; + let e: i8 = 41; + let r: i8 = transmute(vqsubb_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqsubh_s16() { + let a: i16 = 42; + let b: i16 = 1; + let e: i16 = 41; + let r: i16 = transmute(vqsubh_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqsubb_u8() { + let a: u8 = 42; + let b: u8 = 1; + let e: u8 = 41; + let r: u8 = transmute(vqsubb_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqsubh_u16() { + let a: u16 = 42; + let b: u16 = 1; + let e: u16 = 41; + let r: u16 = transmute(vqsubh_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqsubs_u32() { + let a: u32 = 42; + let b: u32 = 1; + let e: u32 = 41; + let r: u32 = transmute(vqsubs_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqsubd_u64() { + let a: u64 = 42; + let b: u64 = 1; + let e: u64 = 41; + let r: u64 = transmute(vqsubd_u64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqsubs_s32() { + let a: i32 = 42; + let b: i32 = 1; + let e: i32 = 41; + let r: i32 = transmute(vqsubs_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqsubd_s64() { + let a: i64 = 42; + let b: i64 = 1; + let e: i64 = 41; + let r: i64 = transmute(vqsubd_s64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrbit_s8() { + let a: i8x8 = i8x8::new(0, 2, 4, 6, 8, 10, 12, 14); + let e: i8x8 = i8x8::new(0, 64, 32, 96, 16, 80, 48, 112); + let r: i8x8 = transmute(vrbit_s8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrbitq_s8() { + let a: i8x16 = i8x16::new(0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30); + let e: i8x16 = i8x16::new(0, 64, 32, 96, 16, 80, 48, 112, 8, 72, 40, 104, 24, 88, 56, 120); + let r: i8x16 = transmute(vrbitq_s8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrbit_u8() { + let a: u8x8 = u8x8::new(0, 2, 4, 6, 8, 10, 12, 14); + let e: u8x8 = u8x8::new(0, 64, 32, 96, 16, 80, 48, 112); + let r: u8x8 = transmute(vrbit_u8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrbitq_u8() { + let a: u8x16 = u8x16::new(0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30); + let e: u8x16 = u8x16::new(0, 64, 32, 96, 16, 80, 48, 112, 8, 72, 40, 104, 24, 88, 56, 120); + let r: u8x16 = transmute(vrbitq_u8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrbit_p8() { + let a: i8x8 = i8x8::new(0, 2, 4, 6, 8, 10, 12, 14); + let e: i8x8 = i8x8::new(0, 64, 32, 96, 16, 80, 48, 112); + let r: i8x8 = transmute(vrbit_p8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrbitq_p8() { + let a: i8x16 = i8x16::new(0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30); + let e: i8x16 = i8x16::new(0, 64, 32, 96, 16, 80, 48, 112, 8, 72, 40, 104, 24, 88, 56, 120); + let r: i8x16 = transmute(vrbitq_p8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrndx_f32() { + let a: f32x2 = f32x2::new(-1.5, 0.5); + let e: f32x2 = f32x2::new(-2.0, 0.0); + let r: f32x2 = transmute(vrndx_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrndxq_f32() { + let a: f32x4 = f32x4::new(-1.5, 0.5, 1.5, 2.5); + let e: f32x4 = f32x4::new(-2.0, 0.0, 2.0, 2.0); + let r: f32x4 = transmute(vrndxq_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrndx_f64() { + let a: f64 = -1.5; + let e: f64 = -2.0; + let r: f64 = transmute(vrndx_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrndxq_f64() { + let a: f64x2 = f64x2::new(-1.5, 0.5); + let e: f64x2 = f64x2::new(-2.0, 0.0); + let r: f64x2 = transmute(vrndxq_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrnda_f32() { + let a: f32x2 = f32x2::new(-1.5, 0.5); + let e: f32x2 = f32x2::new(-2.0, 1.0); + let r: f32x2 = transmute(vrnda_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrndaq_f32() { + let a: f32x4 = f32x4::new(-1.5, 0.5, 1.5, 2.5); + let e: f32x4 = f32x4::new(-2.0, 1.0, 2.0, 3.0); + let r: f32x4 = transmute(vrndaq_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrnda_f64() { + let a: f64 = -1.5; + let e: f64 = -2.0; + let r: f64 = transmute(vrnda_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrndaq_f64() { + let a: f64x2 = f64x2::new(-1.5, 0.5); + let e: f64x2 = f64x2::new(-2.0, 1.0); + let r: f64x2 = transmute(vrndaq_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrndn_f64() { + let a: f64 = -1.5; + let e: f64 = -2.0; + let r: f64 = transmute(vrndn_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrndnq_f64() { + let a: f64x2 = f64x2::new(-1.5, 0.5); + let e: f64x2 = f64x2::new(-2.0, 0.0); + let r: f64x2 = transmute(vrndnq_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrndm_f32() { + let a: f32x2 = f32x2::new(-1.5, 0.5); + let e: f32x2 = f32x2::new(-2.0, 0.0); + let r: f32x2 = transmute(vrndm_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrndmq_f32() { + let a: f32x4 = f32x4::new(-1.5, 0.5, 1.5, 2.5); + let e: f32x4 = f32x4::new(-2.0, 0.0, 1.0, 2.0); + let r: f32x4 = transmute(vrndmq_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrndm_f64() { + let a: f64 = -1.5; + let e: f64 = -2.0; + let r: f64 = transmute(vrndm_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrndmq_f64() { + let a: f64x2 = f64x2::new(-1.5, 0.5); + let e: f64x2 = f64x2::new(-2.0, 0.0); + let r: f64x2 = transmute(vrndmq_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrndp_f32() { + let a: f32x2 = f32x2::new(-1.5, 0.5); + let e: f32x2 = f32x2::new(-1.0, 1.0); + let r: f32x2 = transmute(vrndp_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrndpq_f32() { + let a: f32x4 = f32x4::new(-1.5, 0.5, 1.5, 2.5); + let e: f32x4 = f32x4::new(-1.0, 1.0, 2.0, 3.0); + let r: f32x4 = transmute(vrndpq_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrndp_f64() { + let a: f64 = -1.5; + let e: f64 = -1.0; + let r: f64 = transmute(vrndp_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrndpq_f64() { + let a: f64x2 = f64x2::new(-1.5, 0.5); + let e: f64x2 = f64x2::new(-1.0, 1.0); + let r: f64x2 = transmute(vrndpq_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrnd_f32() { + let a: f32x2 = f32x2::new(-1.5, 0.5); + let e: f32x2 = f32x2::new(-1.0, 0.0); + let r: f32x2 = transmute(vrnd_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrndq_f32() { + let a: f32x4 = f32x4::new(-1.5, 0.5, 1.5, 2.5); + let e: f32x4 = f32x4::new(-1.0, 0.0, 1.0, 2.0); + let r: f32x4 = transmute(vrndq_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrnd_f64() { + let a: f64 = -1.5; + let e: f64 = -1.0; + let r: f64 = transmute(vrnd_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrndq_f64() { + let a: f64x2 = f64x2::new(-1.5, 0.5); + let e: f64x2 = f64x2::new(-1.0, 0.0); + let r: f64x2 = transmute(vrndq_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrndi_f32() { + let a: f32x2 = f32x2::new(-1.5, 0.5); + let e: f32x2 = f32x2::new(-2.0, 0.0); + let r: f32x2 = transmute(vrndi_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrndiq_f32() { + let a: f32x4 = f32x4::new(-1.5, 0.5, 1.5, 2.5); + let e: f32x4 = f32x4::new(-2.0, 0.0, 2.0, 2.0); + let r: f32x4 = transmute(vrndiq_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrndi_f64() { + let a: f64 = -1.5; + let e: f64 = -2.0; + let r: f64 = transmute(vrndi_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrndiq_f64() { + let a: f64x2 = f64x2::new(-1.5, 0.5); + let e: f64x2 = f64x2::new(-2.0, 0.0); + let r: f64x2 = transmute(vrndiq_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqaddb_s8() { + let a: i8 = 42; + let b: i8 = 1; + let e: i8 = 43; + let r: i8 = transmute(vqaddb_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqaddh_s16() { + let a: i16 = 42; + let b: i16 = 1; + let e: i16 = 43; + let r: i16 = transmute(vqaddh_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqaddb_u8() { + let a: u8 = 42; + let b: u8 = 1; + let e: u8 = 43; + let r: u8 = transmute(vqaddb_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqaddh_u16() { + let a: u16 = 42; + let b: u16 = 1; + let e: u16 = 43; + let r: u16 = transmute(vqaddh_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqadds_u32() { + let a: u32 = 42; + let b: u32 = 1; + let e: u32 = 43; + let r: u32 = transmute(vqadds_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqaddd_u64() { + let a: u64 = 42; + let b: u64 = 1; + let e: u64 = 43; + let r: u64 = transmute(vqaddd_u64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqadds_s32() { + let a: i32 = 42; + let b: i32 = 1; + let e: i32 = 43; + let r: i32 = transmute(vqadds_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqaddd_s64() { + let a: i64 = 42; + let b: i64 = 1; + let e: i64 = 43; + let r: i64 = transmute(vqaddd_s64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmul_f64() { + let a: f64 = 1.0; + let b: f64 = 2.0; + let e: f64 = 2.0; + let r: f64 = transmute(vmul_f64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmulq_f64() { + let a: f64x2 = f64x2::new(1.0, 2.0); + let b: f64x2 = f64x2::new(2.0, 3.0); + let e: f64x2 = f64x2::new(2.0, 6.0); + let r: f64x2 = transmute(vmulq_f64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmul_n_f64() { + let a: f64 = 1.; + let b: f64 = 2.; + let e: f64 = 2.; + let r: f64 = transmute(vmul_n_f64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmulq_n_f64() { + let a: f64x2 = f64x2::new(1., 2.); + let b: f64 = 2.; + let e: f64x2 = f64x2::new(2., 4.); + let r: f64x2 = transmute(vmulq_n_f64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmul_lane_f64() { + let a: f64 = 1.; + let b: f64 = 2.; + let e: f64 = 2.; + let r: f64 = transmute(vmul_lane_f64::<0>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmul_laneq_f64() { + let a: f64 = 1.; + let b: f64x2 = f64x2::new(2., 0.); + let e: f64 = 2.; + let r: f64 = transmute(vmul_laneq_f64::<0>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmulq_lane_f64() { + let a: f64x2 = f64x2::new(1., 2.); + let b: f64 = 2.; + let e: f64x2 = f64x2::new(2., 4.); + let r: f64x2 = transmute(vmulq_lane_f64::<0>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmulq_laneq_f64() { + let a: f64x2 = f64x2::new(1., 2.); + let b: f64x2 = f64x2::new(2., 0.); + let e: f64x2 = f64x2::new(2., 4.); + let r: f64x2 = transmute(vmulq_laneq_f64::<0>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmuls_lane_f32() { + let a: f32 = 1.; + let b: f32x2 = f32x2::new(2., 0.); + let e: f32 = 2.; + let r: f32 = transmute(vmuls_lane_f32::<0>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmuls_laneq_f32() { + let a: f32 = 1.; + let b: f32x4 = f32x4::new(2., 0., 0., 0.); + let e: f32 = 2.; + let r: f32 = transmute(vmuls_laneq_f32::<0>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmuld_lane_f64() { + let a: f64 = 1.; + let b: f64 = 2.; + let e: f64 = 2.; + let r: f64 = transmute(vmuld_lane_f64::<0>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmuld_laneq_f64() { + let a: f64 = 1.; + let b: f64x2 = f64x2::new(2., 0.); + let e: f64 = 2.; + let r: f64 = transmute(vmuld_laneq_f64::<0>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmull_high_s8() { + let a: i8x16 = i8x16::new(1, 2, 9, 10, 9, 10, 11, 12, 9, 10, 11, 12, 13, 14, 15, 16); + let b: i8x16 = i8x16::new(1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2); + let e: i16x8 = i16x8::new(9, 20, 11, 24, 13, 28, 15, 32); + let r: i16x8 = transmute(vmull_high_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmull_high_s16() { + let a: i16x8 = i16x8::new(1, 2, 9, 10, 9, 10, 11, 12); + let b: i16x8 = i16x8::new(1, 2, 1, 2, 1, 2, 1, 2); + let e: i32x4 = i32x4::new(9, 20, 11, 24); + let r: i32x4 = transmute(vmull_high_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmull_high_s32() { + let a: i32x4 = i32x4::new(1, 2, 9, 10); + let b: i32x4 = i32x4::new(1, 2, 1, 2); + let e: i64x2 = i64x2::new(9, 20); + let r: i64x2 = transmute(vmull_high_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmull_high_u8() { + let a: u8x16 = u8x16::new(1, 2, 9, 10, 9, 10, 11, 12, 9, 10, 11, 12, 13, 14, 15, 16); + let b: u8x16 = u8x16::new(1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2); + let e: u16x8 = u16x8::new(9, 20, 11, 24, 13, 28, 15, 32); + let r: u16x8 = transmute(vmull_high_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmull_high_u16() { + let a: u16x8 = u16x8::new(1, 2, 9, 10, 9, 10, 11, 12); + let b: u16x8 = u16x8::new(1, 2, 1, 2, 1, 2, 1, 2); + let e: u32x4 = u32x4::new(9, 20, 11, 24); + let r: u32x4 = transmute(vmull_high_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmull_high_u32() { + let a: u32x4 = u32x4::new(1, 2, 9, 10); + let b: u32x4 = u32x4::new(1, 2, 1, 2); + let e: u64x2 = u64x2::new(9, 20); + let r: u64x2 = transmute(vmull_high_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmull_p64() { + let a: p64 = 15; + let b: p64 = 3; + let e: p128 = 17; + let r: p128 = transmute(vmull_p64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmull_high_p8() { + let a: i8x16 = i8x16::new(1, 2, 9, 10, 9, 10, 11, 12, 9, 10, 11, 12, 13, 14, 15, 16); + let b: i8x16 = i8x16::new(1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3); + let e: i16x8 = i16x8::new(9, 30, 11, 20, 13, 18, 15, 48); + let r: i16x8 = transmute(vmull_high_p8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmull_high_p64() { + let a: i64x2 = i64x2::new(1, 15); + let b: i64x2 = i64x2::new(1, 3); + let e: p128 = 17; + let r: p128 = transmute(vmull_high_p64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmull_high_n_s16() { + let a: i16x8 = i16x8::new(1, 2, 9, 10, 9, 10, 11, 12); + let b: i16 = 2; + let e: i32x4 = i32x4::new(18, 20, 22, 24); + let r: i32x4 = transmute(vmull_high_n_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmull_high_n_s32() { + let a: i32x4 = i32x4::new(1, 2, 9, 10); + let b: i32 = 2; + let e: i64x2 = i64x2::new(18, 20); + let r: i64x2 = transmute(vmull_high_n_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmull_high_n_u16() { + let a: u16x8 = u16x8::new(1, 2, 9, 10, 9, 10, 11, 12); + let b: u16 = 2; + let e: u32x4 = u32x4::new(18, 20, 22, 24); + let r: u32x4 = transmute(vmull_high_n_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmull_high_n_u32() { + let a: u32x4 = u32x4::new(1, 2, 9, 10); + let b: u32 = 2; + let e: u64x2 = u64x2::new(18, 20); + let r: u64x2 = transmute(vmull_high_n_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmull_high_lane_s16() { + let a: i16x8 = i16x8::new(1, 2, 9, 10, 9, 10, 11, 12); + let b: i16x4 = i16x4::new(0, 2, 0, 0); + let e: i32x4 = i32x4::new(18, 20, 22, 24); + let r: i32x4 = transmute(vmull_high_lane_s16::<1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmull_high_laneq_s16() { + let a: i16x8 = i16x8::new(1, 2, 9, 10, 9, 10, 11, 12); + let b: i16x8 = i16x8::new(0, 2, 0, 0, 0, 0, 0, 0); + let e: i32x4 = i32x4::new(18, 20, 22, 24); + let r: i32x4 = transmute(vmull_high_laneq_s16::<1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmull_high_lane_s32() { + let a: i32x4 = i32x4::new(1, 2, 9, 10); + let b: i32x2 = i32x2::new(0, 2); + let e: i64x2 = i64x2::new(18, 20); + let r: i64x2 = transmute(vmull_high_lane_s32::<1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmull_high_laneq_s32() { + let a: i32x4 = i32x4::new(1, 2, 9, 10); + let b: i32x4 = i32x4::new(0, 2, 0, 0); + let e: i64x2 = i64x2::new(18, 20); + let r: i64x2 = transmute(vmull_high_laneq_s32::<1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmull_high_lane_u16() { + let a: u16x8 = u16x8::new(1, 2, 9, 10, 9, 10, 11, 12); + let b: u16x4 = u16x4::new(0, 2, 0, 0); + let e: u32x4 = u32x4::new(18, 20, 22, 24); + let r: u32x4 = transmute(vmull_high_lane_u16::<1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmull_high_laneq_u16() { + let a: u16x8 = u16x8::new(1, 2, 9, 10, 9, 10, 11, 12); + let b: u16x8 = u16x8::new(0, 2, 0, 0, 0, 0, 0, 0); + let e: u32x4 = u32x4::new(18, 20, 22, 24); + let r: u32x4 = transmute(vmull_high_laneq_u16::<1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmull_high_lane_u32() { + let a: u32x4 = u32x4::new(1, 2, 9, 10); + let b: u32x2 = u32x2::new(0, 2); + let e: u64x2 = u64x2::new(18, 20); + let r: u64x2 = transmute(vmull_high_lane_u32::<1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmull_high_laneq_u32() { + let a: u32x4 = u32x4::new(1, 2, 9, 10); + let b: u32x4 = u32x4::new(0, 2, 0, 0); + let e: u64x2 = u64x2::new(18, 20); + let r: u64x2 = transmute(vmull_high_laneq_u32::<1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmulx_f32() { + let a: f32x2 = f32x2::new(1., 2.); + let b: f32x2 = f32x2::new(2., 2.); + let e: f32x2 = f32x2::new(2., 4.); + let r: f32x2 = transmute(vmulx_f32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmulxq_f32() { + let a: f32x4 = f32x4::new(1., 2., 3., 4.); + let b: f32x4 = f32x4::new(2., 2., 2., 2.); + let e: f32x4 = f32x4::new(2., 4., 6., 8.); + let r: f32x4 = transmute(vmulxq_f32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmulx_f64() { + let a: f64 = 1.; + let b: f64 = 2.; + let e: f64 = 2.; + let r: f64 = transmute(vmulx_f64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmulxq_f64() { + let a: f64x2 = f64x2::new(1., 2.); + let b: f64x2 = f64x2::new(2., 2.); + let e: f64x2 = f64x2::new(2., 4.); + let r: f64x2 = transmute(vmulxq_f64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmulx_lane_f64() { + let a: f64 = 1.; + let b: f64 = 2.; + let e: f64 = 2.; + let r: f64 = transmute(vmulx_lane_f64::<0>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmulx_laneq_f64() { + let a: f64 = 1.; + let b: f64x2 = f64x2::new(2., 0.); + let e: f64 = 2.; + let r: f64 = transmute(vmulx_laneq_f64::<0>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmulx_lane_f32() { + let a: f32x2 = f32x2::new(1., 2.); + let b: f32x2 = f32x2::new(2., 0.); + let e: f32x2 = f32x2::new(2., 4.); + let r: f32x2 = transmute(vmulx_lane_f32::<0>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmulx_laneq_f32() { + let a: f32x2 = f32x2::new(1., 2.); + let b: f32x4 = f32x4::new(2., 0., 0., 0.); + let e: f32x2 = f32x2::new(2., 4.); + let r: f32x2 = transmute(vmulx_laneq_f32::<0>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmulxq_lane_f32() { + let a: f32x4 = f32x4::new(1., 2., 3., 4.); + let b: f32x2 = f32x2::new(2., 0.); + let e: f32x4 = f32x4::new(2., 4., 6., 8.); + let r: f32x4 = transmute(vmulxq_lane_f32::<0>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmulxq_laneq_f32() { + let a: f32x4 = f32x4::new(1., 2., 3., 4.); + let b: f32x4 = f32x4::new(2., 0., 0., 0.); + let e: f32x4 = f32x4::new(2., 4., 6., 8.); + let r: f32x4 = transmute(vmulxq_laneq_f32::<0>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmulxq_lane_f64() { + let a: f64x2 = f64x2::new(1., 2.); + let b: f64 = 2.; + let e: f64x2 = f64x2::new(2., 4.); + let r: f64x2 = transmute(vmulxq_lane_f64::<0>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmulxq_laneq_f64() { + let a: f64x2 = f64x2::new(1., 2.); + let b: f64x2 = f64x2::new(2., 0.); + let e: f64x2 = f64x2::new(2., 4.); + let r: f64x2 = transmute(vmulxq_laneq_f64::<0>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmulxs_f32() { + let a: f32 = 2.; + let b: f32 = 3.; + let e: f32 = 6.; + let r: f32 = transmute(vmulxs_f32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmulxd_f64() { + let a: f64 = 2.; + let b: f64 = 3.; + let e: f64 = 6.; + let r: f64 = transmute(vmulxd_f64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmulxs_lane_f32() { + let a: f32 = 2.; + let b: f32x2 = f32x2::new(3., 0.); + let e: f32 = 6.; + let r: f32 = transmute(vmulxs_lane_f32::<0>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmulxs_laneq_f32() { + let a: f32 = 2.; + let b: f32x4 = f32x4::new(3., 0., 0., 0.); + let e: f32 = 6.; + let r: f32 = transmute(vmulxs_laneq_f32::<0>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmulxd_lane_f64() { + let a: f64 = 2.; + let b: f64 = 3.; + let e: f64 = 6.; + let r: f64 = transmute(vmulxd_lane_f64::<0>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmulxd_laneq_f64() { + let a: f64 = 2.; + let b: f64x2 = f64x2::new(3., 0.); + let e: f64 = 6.; + let r: f64 = transmute(vmulxd_laneq_f64::<0>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vfma_f64() { + let a: f64 = 8.0; + let b: f64 = 6.0; + let c: f64 = 2.0; + let e: f64 = 20.0; + let r: f64 = transmute(vfma_f64(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vfmaq_f64() { + let a: f64x2 = f64x2::new(8.0, 18.0); + let b: f64x2 = f64x2::new(6.0, 4.0); + let c: f64x2 = f64x2::new(2.0, 3.0); + let e: f64x2 = f64x2::new(20.0, 30.0); + let r: f64x2 = transmute(vfmaq_f64(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vfma_n_f64() { + let a: f64 = 2.0; + let b: f64 = 6.0; + let c: f64 = 8.0; + let e: f64 = 50.0; + let r: f64 = transmute(vfma_n_f64(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vfmaq_n_f64() { + let a: f64x2 = f64x2::new(2.0, 3.0); + let b: f64x2 = f64x2::new(6.0, 4.0); + let c: f64 = 8.0; + let e: f64x2 = f64x2::new(50.0, 35.0); + let r: f64x2 = transmute(vfmaq_n_f64(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vfma_lane_f32() { + let a: f32x2 = f32x2::new(2., 3.); + let b: f32x2 = f32x2::new(6., 4.); + let c: f32x2 = f32x2::new(2., 0.); + let e: f32x2 = f32x2::new(14., 11.); + let r: f32x2 = transmute(vfma_lane_f32::<0>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vfma_laneq_f32() { + let a: f32x2 = f32x2::new(2., 3.); + let b: f32x2 = f32x2::new(6., 4.); + let c: f32x4 = f32x4::new(2., 0., 0., 0.); + let e: f32x2 = f32x2::new(14., 11.); + let r: f32x2 = transmute(vfma_laneq_f32::<0>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vfmaq_lane_f32() { + let a: f32x4 = f32x4::new(2., 3., 4., 5.); + let b: f32x4 = f32x4::new(6., 4., 7., 8.); + let c: f32x2 = f32x2::new(2., 0.); + let e: f32x4 = f32x4::new(14., 11., 18., 21.); + let r: f32x4 = transmute(vfmaq_lane_f32::<0>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vfmaq_laneq_f32() { + let a: f32x4 = f32x4::new(2., 3., 4., 5.); + let b: f32x4 = f32x4::new(6., 4., 7., 8.); + let c: f32x4 = f32x4::new(2., 0., 0., 0.); + let e: f32x4 = f32x4::new(14., 11., 18., 21.); + let r: f32x4 = transmute(vfmaq_laneq_f32::<0>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vfma_lane_f64() { + let a: f64 = 2.; + let b: f64 = 6.; + let c: f64 = 2.; + let e: f64 = 14.; + let r: f64 = transmute(vfma_lane_f64::<0>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vfma_laneq_f64() { + let a: f64 = 2.; + let b: f64 = 6.; + let c: f64x2 = f64x2::new(2., 0.); + let e: f64 = 14.; + let r: f64 = transmute(vfma_laneq_f64::<0>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vfmaq_lane_f64() { + let a: f64x2 = f64x2::new(2., 3.); + let b: f64x2 = f64x2::new(6., 4.); + let c: f64 = 2.; + let e: f64x2 = f64x2::new(14., 11.); + let r: f64x2 = transmute(vfmaq_lane_f64::<0>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vfmaq_laneq_f64() { + let a: f64x2 = f64x2::new(2., 3.); + let b: f64x2 = f64x2::new(6., 4.); + let c: f64x2 = f64x2::new(2., 0.); + let e: f64x2 = f64x2::new(14., 11.); + let r: f64x2 = transmute(vfmaq_laneq_f64::<0>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vfmas_lane_f32() { + let a: f32 = 2.; + let b: f32 = 6.; + let c: f32x2 = f32x2::new(3., 0.); + let e: f32 = 20.; + let r: f32 = transmute(vfmas_lane_f32::<0>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vfmas_laneq_f32() { + let a: f32 = 2.; + let b: f32 = 6.; + let c: f32x4 = f32x4::new(3., 0., 0., 0.); + let e: f32 = 20.; + let r: f32 = transmute(vfmas_laneq_f32::<0>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vfmad_lane_f64() { + let a: f64 = 2.; + let b: f64 = 6.; + let c: f64 = 3.; + let e: f64 = 20.; + let r: f64 = transmute(vfmad_lane_f64::<0>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vfmad_laneq_f64() { + let a: f64 = 2.; + let b: f64 = 6.; + let c: f64x2 = f64x2::new(3., 0.); + let e: f64 = 20.; + let r: f64 = transmute(vfmad_laneq_f64::<0>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vfms_f64() { + let a: f64 = 20.0; + let b: f64 = 6.0; + let c: f64 = 2.0; + let e: f64 = 8.0; + let r: f64 = transmute(vfms_f64(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vfmsq_f64() { + let a: f64x2 = f64x2::new(20.0, 30.0); + let b: f64x2 = f64x2::new(6.0, 4.0); + let c: f64x2 = f64x2::new(2.0, 3.0); + let e: f64x2 = f64x2::new(8.0, 18.0); + let r: f64x2 = transmute(vfmsq_f64(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vfms_n_f64() { + let a: f64 = 50.0; + let b: f64 = 6.0; + let c: f64 = 8.0; + let e: f64 = 2.0; + let r: f64 = transmute(vfms_n_f64(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vfmsq_n_f64() { + let a: f64x2 = f64x2::new(50.0, 35.0); + let b: f64x2 = f64x2::new(6.0, 4.0); + let c: f64 = 8.0; + let e: f64x2 = f64x2::new(2.0, 3.0); + let r: f64x2 = transmute(vfmsq_n_f64(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vfms_lane_f32() { + let a: f32x2 = f32x2::new(14., 11.); + let b: f32x2 = f32x2::new(6., 4.); + let c: f32x2 = f32x2::new(2., 0.); + let e: f32x2 = f32x2::new(2., 3.); + let r: f32x2 = transmute(vfms_lane_f32::<0>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vfms_laneq_f32() { + let a: f32x2 = f32x2::new(14., 11.); + let b: f32x2 = f32x2::new(6., 4.); + let c: f32x4 = f32x4::new(2., 0., 0., 0.); + let e: f32x2 = f32x2::new(2., 3.); + let r: f32x2 = transmute(vfms_laneq_f32::<0>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vfmsq_lane_f32() { + let a: f32x4 = f32x4::new(14., 11., 18., 21.); + let b: f32x4 = f32x4::new(6., 4., 7., 8.); + let c: f32x2 = f32x2::new(2., 0.); + let e: f32x4 = f32x4::new(2., 3., 4., 5.); + let r: f32x4 = transmute(vfmsq_lane_f32::<0>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vfmsq_laneq_f32() { + let a: f32x4 = f32x4::new(14., 11., 18., 21.); + let b: f32x4 = f32x4::new(6., 4., 7., 8.); + let c: f32x4 = f32x4::new(2., 0., 0., 0.); + let e: f32x4 = f32x4::new(2., 3., 4., 5.); + let r: f32x4 = transmute(vfmsq_laneq_f32::<0>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vfms_lane_f64() { + let a: f64 = 14.; + let b: f64 = 6.; + let c: f64 = 2.; + let e: f64 = 2.; + let r: f64 = transmute(vfms_lane_f64::<0>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vfms_laneq_f64() { + let a: f64 = 14.; + let b: f64 = 6.; + let c: f64x2 = f64x2::new(2., 0.); + let e: f64 = 2.; + let r: f64 = transmute(vfms_laneq_f64::<0>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vfmsq_lane_f64() { + let a: f64x2 = f64x2::new(14., 11.); + let b: f64x2 = f64x2::new(6., 4.); + let c: f64 = 2.; + let e: f64x2 = f64x2::new(2., 3.); + let r: f64x2 = transmute(vfmsq_lane_f64::<0>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vfmsq_laneq_f64() { + let a: f64x2 = f64x2::new(14., 11.); + let b: f64x2 = f64x2::new(6., 4.); + let c: f64x2 = f64x2::new(2., 0.); + let e: f64x2 = f64x2::new(2., 3.); + let r: f64x2 = transmute(vfmsq_laneq_f64::<0>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vfmss_lane_f32() { + let a: f32 = 14.; + let b: f32 = 6.; + let c: f32x2 = f32x2::new(2., 0.); + let e: f32 = 2.; + let r: f32 = transmute(vfmss_lane_f32::<0>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vfmss_laneq_f32() { + let a: f32 = 14.; + let b: f32 = 6.; + let c: f32x4 = f32x4::new(2., 0., 0., 0.); + let e: f32 = 2.; + let r: f32 = transmute(vfmss_laneq_f32::<0>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vfmsd_lane_f64() { + let a: f64 = 14.; + let b: f64 = 6.; + let c: f64 = 2.; + let e: f64 = 2.; + let r: f64 = transmute(vfmsd_lane_f64::<0>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vfmsd_laneq_f64() { + let a: f64 = 14.; + let b: f64 = 6.; + let c: f64x2 = f64x2::new(2., 0.); + let e: f64 = 2.; + let r: f64 = transmute(vfmsd_laneq_f64::<0>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdiv_f32() { + let a: f32x2 = f32x2::new(2.0, 6.0); + let b: f32x2 = f32x2::new(1.0, 2.0); + let e: f32x2 = f32x2::new(2.0, 3.0); + let r: f32x2 = transmute(vdiv_f32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdivq_f32() { + let a: f32x4 = f32x4::new(2.0, 6.0, 4.0, 10.0); + let b: f32x4 = f32x4::new(1.0, 2.0, 1.0, 2.0); + let e: f32x4 = f32x4::new(2.0, 3.0, 4.0, 5.0); + let r: f32x4 = transmute(vdivq_f32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdiv_f64() { + let a: f64 = 2.0; + let b: f64 = 1.0; + let e: f64 = 2.0; + let r: f64 = transmute(vdiv_f64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdivq_f64() { + let a: f64x2 = f64x2::new(2.0, 6.0); + let b: f64x2 = f64x2::new(1.0, 2.0); + let e: f64x2 = f64x2::new(2.0, 3.0); + let r: f64x2 = transmute(vdivq_f64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsub_f64() { + let a: f64 = 1.0; + let b: f64 = 1.0; + let e: f64 = 0.0; + let r: f64 = transmute(vsub_f64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsubq_f64() { + let a: f64x2 = f64x2::new(1.0, 4.0); + let b: f64x2 = f64x2::new(1.0, 2.0); + let e: f64x2 = f64x2::new(0.0, 2.0); + let r: f64x2 = transmute(vsubq_f64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vaddlv_s16() { + let a: i16x4 = i16x4::new(1, 2, 3, 4); + let e: i32 = 10; + let r: i32 = transmute(vaddlv_s16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vaddlvq_s16() { + let a: i16x8 = i16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let e: i32 = 36; + let r: i32 = transmute(vaddlvq_s16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vaddlv_s32() { + let a: i32x2 = i32x2::new(1, 2); + let e: i64 = 3; + let r: i64 = transmute(vaddlv_s32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vaddlvq_s32() { + let a: i32x4 = i32x4::new(1, 2, 3, 4); + let e: i64 = 10; + let r: i64 = transmute(vaddlvq_s32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vaddlv_u16() { + let a: u16x4 = u16x4::new(1, 2, 3, 4); + let e: u32 = 10; + let r: u32 = transmute(vaddlv_u16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vaddlvq_u16() { + let a: u16x8 = u16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let e: u32 = 36; + let r: u32 = transmute(vaddlvq_u16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vaddlv_u32() { + let a: u32x2 = u32x2::new(1, 2); + let e: u64 = 3; + let r: u64 = transmute(vaddlv_u32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vaddlvq_u32() { + let a: u32x4 = u32x4::new(1, 2, 3, 4); + let e: u64 = 10; + let r: u64 = transmute(vaddlvq_u32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsubw_high_s8() { + let a: i16x8 = i16x8::new(8, 9, 10, 12, 13, 14, 15, 16); + let b: i8x16 = i8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12, 13, 14, 15, 16); + let e: i16x8 = i16x8::new(0, 0, 0, 0, 0, 0, 0, 0); + let r: i16x8 = transmute(vsubw_high_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsubw_high_s16() { + let a: i32x4 = i32x4::new(8, 9, 10, 11); + let b: i16x8 = i16x8::new(0, 1, 2, 3, 8, 9, 10, 11); + let e: i32x4 = i32x4::new(0, 0, 0, 0); + let r: i32x4 = transmute(vsubw_high_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsubw_high_s32() { + let a: i64x2 = i64x2::new(8, 9); + let b: i32x4 = i32x4::new(6, 7, 8, 9); + let e: i64x2 = i64x2::new(0, 0); + let r: i64x2 = transmute(vsubw_high_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsubw_high_u8() { + let a: u16x8 = u16x8::new(8, 9, 10, 11, 12, 13, 14, 15); + let b: u8x16 = u8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let e: u16x8 = u16x8::new(0, 0, 0, 0, 0, 0, 0, 0); + let r: u16x8 = transmute(vsubw_high_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsubw_high_u16() { + let a: u32x4 = u32x4::new(8, 9, 10, 11); + let b: u16x8 = u16x8::new(0, 1, 2, 3, 8, 9, 10, 11); + let e: u32x4 = u32x4::new(0, 0, 0, 0); + let r: u32x4 = transmute(vsubw_high_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsubw_high_u32() { + let a: u64x2 = u64x2::new(8, 9); + let b: u32x4 = u32x4::new(6, 7, 8, 9); + let e: u64x2 = u64x2::new(0, 0); + let r: u64x2 = transmute(vsubw_high_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsubl_high_s8() { + let a: i8x16 = i8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let b: i8x16 = i8x16::new(1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2); + let e: i16x8 = i16x8::new(6, 7, 8, 9, 10, 11, 12, 13); + let r: i16x8 = transmute(vsubl_high_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsubl_high_s16() { + let a: i16x8 = i16x8::new(8, 9, 10, 11, 12, 13, 14, 15); + let b: i16x8 = i16x8::new(6, 6, 6, 6, 8, 8, 8, 8); + let e: i32x4 = i32x4::new(4, 5, 6, 7); + let r: i32x4 = transmute(vsubl_high_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsubl_high_s32() { + let a: i32x4 = i32x4::new(12, 13, 14, 15); + let b: i32x4 = i32x4::new(6, 6, 8, 8); + let e: i64x2 = i64x2::new(6, 7); + let r: i64x2 = transmute(vsubl_high_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsubl_high_u8() { + let a: u8x16 = u8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let b: u8x16 = u8x16::new(1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2); + let e: u16x8 = u16x8::new(6, 7, 8, 9, 10, 11, 12, 13); + let r: u16x8 = transmute(vsubl_high_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsubl_high_u16() { + let a: u16x8 = u16x8::new(8, 9, 10, 11, 12, 13, 14, 15); + let b: u16x8 = u16x8::new(6, 6, 6, 6, 8, 8, 8, 8); + let e: u32x4 = u32x4::new(4, 5, 6, 7); + let r: u32x4 = transmute(vsubl_high_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsubl_high_u32() { + let a: u32x4 = u32x4::new(12, 13, 14, 15); + let b: u32x4 = u32x4::new(6, 6, 8, 8); + let e: u64x2 = u64x2::new(6, 7); + let r: u64x2 = transmute(vsubl_high_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmax_f64() { + let a: f64 = 1.0; + let b: f64 = 0.0; + let e: f64 = 1.0; + let r: f64 = transmute(vmax_f64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmaxq_f64() { + let a: f64x2 = f64x2::new(1.0, -2.0); + let b: f64x2 = f64x2::new(0.0, 3.0); + let e: f64x2 = f64x2::new(1.0, 3.0); + let r: f64x2 = transmute(vmaxq_f64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmaxnm_f64() { + let a: f64 = 1.0; + let b: f64 = 8.0; + let e: f64 = 8.0; + let r: f64 = transmute(vmaxnm_f64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmaxnmq_f64() { + let a: f64x2 = f64x2::new(1.0, 2.0); + let b: f64x2 = f64x2::new(8.0, 16.0); + let e: f64x2 = f64x2::new(8.0, 16.0); + let r: f64x2 = transmute(vmaxnmq_f64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vpmaxnm_f32() { + let a: f32x2 = f32x2::new(1.0, 2.0); + let b: f32x2 = f32x2::new(6.0, -3.0); + let e: f32x2 = f32x2::new(2.0, 6.0); + let r: f32x2 = transmute(vpmaxnm_f32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vpmaxnmq_f64() { + let a: f64x2 = f64x2::new(1.0, 2.0); + let b: f64x2 = f64x2::new(6.0, -3.0); + let e: f64x2 = f64x2::new(2.0, 6.0); + let r: f64x2 = transmute(vpmaxnmq_f64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vpmaxnmq_f32() { + let a: f32x4 = f32x4::new(1.0, 2.0, 3.0, -4.0); + let b: f32x4 = f32x4::new(8.0, 16.0, -1.0, 6.0); + let e: f32x4 = f32x4::new(2.0, 3.0, 16.0, 6.0); + let r: f32x4 = transmute(vpmaxnmq_f32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmin_f64() { + let a: f64 = 1.0; + let b: f64 = 0.0; + let e: f64 = 0.0; + let r: f64 = transmute(vmin_f64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vminq_f64() { + let a: f64x2 = f64x2::new(1.0, -2.0); + let b: f64x2 = f64x2::new(0.0, 3.0); + let e: f64x2 = f64x2::new(0.0, -2.0); + let r: f64x2 = transmute(vminq_f64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vminnm_f64() { + let a: f64 = 1.0; + let b: f64 = 8.0; + let e: f64 = 1.0; + let r: f64 = transmute(vminnm_f64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vminnmq_f64() { + let a: f64x2 = f64x2::new(1.0, 2.0); + let b: f64x2 = f64x2::new(8.0, 16.0); + let e: f64x2 = f64x2::new(1.0, 2.0); + let r: f64x2 = transmute(vminnmq_f64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vpminnm_f32() { + let a: f32x2 = f32x2::new(1.0, 2.0); + let b: f32x2 = f32x2::new(6.0, -3.0); + let e: f32x2 = f32x2::new(1.0, -3.0); + let r: f32x2 = transmute(vpminnm_f32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vpminnmq_f64() { + let a: f64x2 = f64x2::new(1.0, 2.0); + let b: f64x2 = f64x2::new(6.0, -3.0); + let e: f64x2 = f64x2::new(1.0, -3.0); + let r: f64x2 = transmute(vpminnmq_f64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vpminnmq_f32() { + let a: f32x4 = f32x4::new(1.0, 2.0, 3.0, -4.0); + let b: f32x4 = f32x4::new(8.0, 16.0, -1.0, 6.0); + let e: f32x4 = f32x4::new(1.0, -4.0, 8.0, -1.0); + let r: f32x4 = transmute(vpminnmq_f32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqdmullh_s16() { + let a: i16 = 2; + let b: i16 = 3; + let e: i32 = 12; + let r: i32 = transmute(vqdmullh_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqdmulls_s32() { + let a: i32 = 2; + let b: i32 = 3; + let e: i64 = 12; + let r: i64 = transmute(vqdmulls_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqdmull_high_s16() { + let a: i16x8 = i16x8::new(0, 1, 4, 5, 4, 5, 6, 7); + let b: i16x8 = i16x8::new(1, 2, 5, 6, 5, 6, 7, 8); + let e: i32x4 = i32x4::new(40, 60, 84, 112); + let r: i32x4 = transmute(vqdmull_high_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqdmull_high_s32() { + let a: i32x4 = i32x4::new(0, 1, 4, 5); + let b: i32x4 = i32x4::new(1, 2, 5, 6); + let e: i64x2 = i64x2::new(40, 60); + let r: i64x2 = transmute(vqdmull_high_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqdmull_high_n_s16() { + let a: i16x8 = i16x8::new(0, 2, 8, 10, 8, 10, 12, 14); + let b: i16 = 2; + let e: i32x4 = i32x4::new(32, 40, 48, 56); + let r: i32x4 = transmute(vqdmull_high_n_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqdmull_high_n_s32() { + let a: i32x4 = i32x4::new(0, 2, 8, 10); + let b: i32 = 2; + let e: i64x2 = i64x2::new(32, 40); + let r: i64x2 = transmute(vqdmull_high_n_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqdmull_laneq_s16() { + let a: i16x4 = i16x4::new(1, 2, 3, 4); + let b: i16x8 = i16x8::new(0, 2, 2, 0, 2, 0, 0, 0); + let e: i32x4 = i32x4::new(4, 8, 12, 16); + let r: i32x4 = transmute(vqdmull_laneq_s16::<4>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqdmull_laneq_s32() { + let a: i32x2 = i32x2::new(1, 2); + let b: i32x4 = i32x4::new(0, 2, 2, 0); + let e: i64x2 = i64x2::new(4, 8); + let r: i64x2 = transmute(vqdmull_laneq_s32::<2>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqdmullh_lane_s16() { + let a: i16 = 2; + let b: i16x4 = i16x4::new(0, 2, 2, 0); + let e: i32 = 8; + let r: i32 = transmute(vqdmullh_lane_s16::<2>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqdmullh_laneq_s16() { + let a: i16 = 2; + let b: i16x8 = i16x8::new(0, 2, 2, 0, 2, 0, 0, 0); + let e: i32 = 8; + let r: i32 = transmute(vqdmullh_laneq_s16::<4>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqdmulls_lane_s32() { + let a: i32 = 2; + let b: i32x2 = i32x2::new(0, 2); + let e: i64 = 8; + let r: i64 = transmute(vqdmulls_lane_s32::<1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqdmulls_laneq_s32() { + let a: i32 = 2; + let b: i32x4 = i32x4::new(0, 2, 2, 0); + let e: i64 = 8; + let r: i64 = transmute(vqdmulls_laneq_s32::<2>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqdmull_high_lane_s16() { + let a: i16x8 = i16x8::new(0, 1, 4, 5, 4, 5, 6, 7); + let b: i16x4 = i16x4::new(0, 2, 2, 0); + let e: i32x4 = i32x4::new(16, 20, 24, 28); + let r: i32x4 = transmute(vqdmull_high_lane_s16::<2>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqdmull_high_lane_s32() { + let a: i32x4 = i32x4::new(0, 1, 4, 5); + let b: i32x2 = i32x2::new(0, 2); + let e: i64x2 = i64x2::new(16, 20); + let r: i64x2 = transmute(vqdmull_high_lane_s32::<1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqdmull_high_laneq_s16() { + let a: i16x8 = i16x8::new(0, 1, 4, 5, 4, 5, 6, 7); + let b: i16x8 = i16x8::new(0, 2, 2, 0, 2, 0, 0, 0); + let e: i32x4 = i32x4::new(16, 20, 24, 28); + let r: i32x4 = transmute(vqdmull_high_laneq_s16::<4>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqdmull_high_laneq_s32() { + let a: i32x4 = i32x4::new(0, 1, 4, 5); + let b: i32x4 = i32x4::new(0, 2, 2, 0); + let e: i64x2 = i64x2::new(16, 20); + let r: i64x2 = transmute(vqdmull_high_laneq_s32::<2>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqdmlal_high_s16() { + let a: i32x4 = i32x4::new(1, 2, 3, 4); + let b: i16x8 = i16x8::new(0, 1, 4, 5, 4, 5, 6, 7); + let c: i16x8 = i16x8::new(1, 2, 5, 6, 5, 6, 7, 8); + let e: i32x4 = i32x4::new(41, 62, 87, 116); + let r: i32x4 = transmute(vqdmlal_high_s16(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqdmlal_high_s32() { + let a: i64x2 = i64x2::new(1, 2); + let b: i32x4 = i32x4::new(0, 1, 4, 5); + let c: i32x4 = i32x4::new(1, 2, 5, 6); + let e: i64x2 = i64x2::new(41, 62); + let r: i64x2 = transmute(vqdmlal_high_s32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqdmlal_high_n_s16() { + let a: i32x4 = i32x4::new(1, 2, 3, 4); + let b: i16x8 = i16x8::new(0, 2, 8, 10, 8, 10, 12, 14); + let c: i16 = 2; + let e: i32x4 = i32x4::new(33, 42, 51, 60); + let r: i32x4 = transmute(vqdmlal_high_n_s16(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqdmlal_high_n_s32() { + let a: i64x2 = i64x2::new(1, 2); + let b: i32x4 = i32x4::new(0, 2, 8, 10); + let c: i32 = 2; + let e: i64x2 = i64x2::new(33, 42); + let r: i64x2 = transmute(vqdmlal_high_n_s32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqdmlal_laneq_s16() { + let a: i32x4 = i32x4::new(1, 2, 3, 4); + let b: i16x4 = i16x4::new(1, 2, 3, 4); + let c: i16x8 = i16x8::new(0, 2, 2, 0, 2, 0, 0, 0); + let e: i32x4 = i32x4::new(5, 10, 15, 20); + let r: i32x4 = transmute(vqdmlal_laneq_s16::<2>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqdmlal_laneq_s32() { + let a: i64x2 = i64x2::new(1, 2); + let b: i32x2 = i32x2::new(1, 2); + let c: i32x4 = i32x4::new(0, 2, 2, 0); + let e: i64x2 = i64x2::new(5, 10); + let r: i64x2 = transmute(vqdmlal_laneq_s32::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqdmlal_high_lane_s16() { + let a: i32x4 = i32x4::new(1, 2, 3, 4); + let b: i16x8 = i16x8::new(0, 1, 4, 5, 4, 5, 6, 7); + let c: i16x4 = i16x4::new(0, 2, 0, 0); + let e: i32x4 = i32x4::new(17, 22, 27, 32); + let r: i32x4 = transmute(vqdmlal_high_lane_s16::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqdmlal_high_laneq_s16() { + let a: i32x4 = i32x4::new(1, 2, 3, 4); + let b: i16x8 = i16x8::new(0, 1, 4, 5, 4, 5, 6, 7); + let c: i16x8 = i16x8::new(0, 2, 0, 0, 0, 0, 0, 0); + let e: i32x4 = i32x4::new(17, 22, 27, 32); + let r: i32x4 = transmute(vqdmlal_high_laneq_s16::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqdmlal_high_lane_s32() { + let a: i64x2 = i64x2::new(1, 2); + let b: i32x4 = i32x4::new(0, 1, 4, 5); + let c: i32x2 = i32x2::new(0, 2); + let e: i64x2 = i64x2::new(17, 22); + let r: i64x2 = transmute(vqdmlal_high_lane_s32::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqdmlal_high_laneq_s32() { + let a: i64x2 = i64x2::new(1, 2); + let b: i32x4 = i32x4::new(0, 1, 4, 5); + let c: i32x4 = i32x4::new(0, 2, 0, 0); + let e: i64x2 = i64x2::new(17, 22); + let r: i64x2 = transmute(vqdmlal_high_laneq_s32::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqdmlsl_high_s16() { + let a: i32x4 = i32x4::new(39, 58, 81, 108); + let b: i16x8 = i16x8::new(0, 1, 4, 5, 4, 5, 6, 7); + let c: i16x8 = i16x8::new(1, 2, 5, 6, 5, 6, 7, 8); + let e: i32x4 = i32x4::new(-1, -2, -3, -4); + let r: i32x4 = transmute(vqdmlsl_high_s16(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqdmlsl_high_s32() { + let a: i64x2 = i64x2::new(39, 58); + let b: i32x4 = i32x4::new(0, 1, 4, 5); + let c: i32x4 = i32x4::new(1, 2, 5, 6); + let e: i64x2 = i64x2::new(-1, -2); + let r: i64x2 = transmute(vqdmlsl_high_s32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqdmlsl_high_n_s16() { + let a: i32x4 = i32x4::new(31, 38, 45, 52); + let b: i16x8 = i16x8::new(0, 2, 8, 10, 8, 10, 12, 14); + let c: i16 = 2; + let e: i32x4 = i32x4::new(-1, -2, -3, -4); + let r: i32x4 = transmute(vqdmlsl_high_n_s16(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqdmlsl_high_n_s32() { + let a: i64x2 = i64x2::new(31, 38); + let b: i32x4 = i32x4::new(0, 2, 8, 10); + let c: i32 = 2; + let e: i64x2 = i64x2::new(-1, -2); + let r: i64x2 = transmute(vqdmlsl_high_n_s32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqdmlsl_laneq_s16() { + let a: i32x4 = i32x4::new(3, 6, 9, 12); + let b: i16x4 = i16x4::new(1, 2, 3, 4); + let c: i16x8 = i16x8::new(0, 2, 2, 0, 2, 0, 0, 0); + let e: i32x4 = i32x4::new(-1, -2, -3, -4); + let r: i32x4 = transmute(vqdmlsl_laneq_s16::<2>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqdmlsl_laneq_s32() { + let a: i64x2 = i64x2::new(3, 6); + let b: i32x2 = i32x2::new(1, 2); + let c: i32x4 = i32x4::new(0, 2, 2, 0); + let e: i64x2 = i64x2::new(-1, -2); + let r: i64x2 = transmute(vqdmlsl_laneq_s32::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqdmlsl_high_lane_s16() { + let a: i32x4 = i32x4::new(15, 18, 21, 24); + let b: i16x8 = i16x8::new(0, 1, 4, 5, 4, 5, 6, 7); + let c: i16x4 = i16x4::new(0, 2, 0, 0); + let e: i32x4 = i32x4::new(-1, -2, -3, -4); + let r: i32x4 = transmute(vqdmlsl_high_lane_s16::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqdmlsl_high_laneq_s16() { + let a: i32x4 = i32x4::new(15, 18, 21, 24); + let b: i16x8 = i16x8::new(0, 1, 4, 5, 4, 5, 6, 7); + let c: i16x8 = i16x8::new(0, 2, 0, 0, 0, 0, 0, 0); + let e: i32x4 = i32x4::new(-1, -2, -3, -4); + let r: i32x4 = transmute(vqdmlsl_high_laneq_s16::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqdmlsl_high_lane_s32() { + let a: i64x2 = i64x2::new(15, 18); + let b: i32x4 = i32x4::new(0, 1, 4, 5); + let c: i32x2 = i32x2::new(0, 2); + let e: i64x2 = i64x2::new(-1, -2); + let r: i64x2 = transmute(vqdmlsl_high_lane_s32::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqdmlsl_high_laneq_s32() { + let a: i64x2 = i64x2::new(15, 18); + let b: i32x4 = i32x4::new(0, 1, 4, 5); + let c: i32x4 = i32x4::new(0, 2, 0, 0); + let e: i64x2 = i64x2::new(-1, -2); + let r: i64x2 = transmute(vqdmlsl_high_laneq_s32::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqdmulhh_s16() { + let a: i16 = 1; + let b: i16 = 2; + let e: i16 = 0; + let r: i16 = transmute(vqdmulhh_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqdmulhs_s32() { + let a: i32 = 1; + let b: i32 = 2; + let e: i32 = 0; + let r: i32 = transmute(vqdmulhs_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqdmulhh_lane_s16() { + let a: i16 = 2; + let b: i16x4 = i16x4::new(0, 0, 0x7F_FF, 0); + let e: i16 = 1; + let r: i16 = transmute(vqdmulhh_lane_s16::<2>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqdmulhh_laneq_s16() { + let a: i16 = 2; + let b: i16x8 = i16x8::new(0, 0, 0x7F_FF, 0, 0, 0, 0, 0); + let e: i16 = 1; + let r: i16 = transmute(vqdmulhh_laneq_s16::<2>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqdmulhs_lane_s32() { + let a: i32 = 2; + let b: i32x2 = i32x2::new(0, 0x7F_FF_FF_FF); + let e: i32 = 1; + let r: i32 = transmute(vqdmulhs_lane_s32::<1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqdmulhs_laneq_s32() { + let a: i32 = 2; + let b: i32x4 = i32x4::new(0, 0x7F_FF_FF_FF, 0, 0); + let e: i32 = 1; + let r: i32 = transmute(vqdmulhs_laneq_s32::<1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqmovnh_s16() { + let a: i16 = 1; + let e: i8 = 1; + let r: i8 = transmute(vqmovnh_s16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqmovns_s32() { + let a: i32 = 1; + let e: i16 = 1; + let r: i16 = transmute(vqmovns_s32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqmovnh_u16() { + let a: u16 = 1; + let e: u8 = 1; + let r: u8 = transmute(vqmovnh_u16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqmovns_u32() { + let a: u32 = 1; + let e: u16 = 1; + let r: u16 = transmute(vqmovns_u32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqmovnd_s64() { + let a: i64 = 1; + let e: i32 = 1; + let r: i32 = transmute(vqmovnd_s64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqmovnd_u64() { + let a: u64 = 1; + let e: u32 = 1; + let r: u32 = transmute(vqmovnd_u64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqmovn_high_s16() { + let a: i8x8 = i8x8::new(0x7F, 0x7F, 0x7F, 0x7F, 0x7F, 0x7F, 0x7F, 0x7F); + let b: i16x8 = i16x8::new(0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF); + let e: i8x16 = i8x16::new(0x7F, 0x7F, 0x7F, 0x7F, 0x7F, 0x7F, 0x7F, 0x7F, 0x7F, 0x7F, 0x7F, 0x7F, 0x7F, 0x7F, 0x7F, 0x7F); + let r: i8x16 = transmute(vqmovn_high_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqmovn_high_s32() { + let a: i16x4 = i16x4::new(0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF); + let b: i32x4 = i32x4::new(0x7F_FF_FF_FF, 0x7F_FF_FF_FF, 0x7F_FF_FF_FF, 0x7F_FF_FF_FF); + let e: i16x8 = i16x8::new(0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF); + let r: i16x8 = transmute(vqmovn_high_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqmovn_high_s64() { + let a: i32x2 = i32x2::new(0x7F_FF_FF_FF, 0x7F_FF_FF_FF); + let b: i64x2 = i64x2::new(0x7F_FF_FF_FF_FF_FF_FF_FF, 0x7F_FF_FF_FF_FF_FF_FF_FF); + let e: i32x4 = i32x4::new(0x7F_FF_FF_FF, 0x7F_FF_FF_FF, 0x7F_FF_FF_FF, 0x7F_FF_FF_FF); + let r: i32x4 = transmute(vqmovn_high_s64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqmovn_high_u16() { + let a: u8x8 = u8x8::new(0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF); + let b: u16x8 = u16x8::new(0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF); + let e: u8x16 = u8x16::new(0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF); + let r: u8x16 = transmute(vqmovn_high_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqmovn_high_u32() { + let a: u16x4 = u16x4::new(0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF); + let b: u32x4 = u32x4::new(0xFF_FF_FF_FF, 0xFF_FF_FF_FF, 0xFF_FF_FF_FF, 0xFF_FF_FF_FF); + let e: u16x8 = u16x8::new(0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF); + let r: u16x8 = transmute(vqmovn_high_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqmovn_high_u64() { + let a: u32x2 = u32x2::new(0xFF_FF_FF_FF, 0xFF_FF_FF_FF); + let b: u64x2 = u64x2::new(0xFF_FF_FF_FF_FF_FF_FF_FF, 0xFF_FF_FF_FF_FF_FF_FF_FF); + let e: u32x4 = u32x4::new(0xFF_FF_FF_FF, 0xFF_FF_FF_FF, 0xFF_FF_FF_FF, 0xFF_FF_FF_FF); + let r: u32x4 = transmute(vqmovn_high_u64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqmovunh_s16() { + let a: i16 = 1; + let e: u8 = 1; + let r: u8 = transmute(vqmovunh_s16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqmovuns_s32() { + let a: i32 = 1; + let e: u16 = 1; + let r: u16 = transmute(vqmovuns_s32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqmovund_s64() { + let a: i64 = 1; + let e: u32 = 1; + let r: u32 = transmute(vqmovund_s64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqmovun_high_s16() { + let a: u8x8 = u8x8::new(0, 0, 0, 0, 0, 0, 0, 0); + let b: i16x8 = i16x8::new(-1, -1, -1, -1, -1, -1, -1, -1); + let e: u8x16 = u8x16::new(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + let r: u8x16 = transmute(vqmovun_high_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqmovun_high_s32() { + let a: u16x4 = u16x4::new(0, 0, 0, 0); + let b: i32x4 = i32x4::new(-1, -1, -1, -1); + let e: u16x8 = u16x8::new(0, 0, 0, 0, 0, 0, 0, 0); + let r: u16x8 = transmute(vqmovun_high_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqmovun_high_s64() { + let a: u32x2 = u32x2::new(0, 0); + let b: i64x2 = i64x2::new(-1, -1); + let e: u32x4 = u32x4::new(0, 0, 0, 0); + let r: u32x4 = transmute(vqmovun_high_s64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrdmulhh_s16() { + let a: i16 = 1; + let b: i16 = 2; + let e: i16 = 0; + let r: i16 = transmute(vqrdmulhh_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrdmulhs_s32() { + let a: i32 = 1; + let b: i32 = 2; + let e: i32 = 0; + let r: i32 = transmute(vqrdmulhs_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrdmulhh_lane_s16() { + let a: i16 = 1; + let b: i16x4 = i16x4::new(0, 2, 0, 0); + let e: i16 = 0; + let r: i16 = transmute(vqrdmulhh_lane_s16::<1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrdmulhh_laneq_s16() { + let a: i16 = 1; + let b: i16x8 = i16x8::new(0, 2, 0, 0, 0, 0, 0, 0); + let e: i16 = 0; + let r: i16 = transmute(vqrdmulhh_laneq_s16::<1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrdmulhs_lane_s32() { + let a: i32 = 1; + let b: i32x2 = i32x2::new(0, 2); + let e: i32 = 0; + let r: i32 = transmute(vqrdmulhs_lane_s32::<1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrdmulhs_laneq_s32() { + let a: i32 = 1; + let b: i32x4 = i32x4::new(0, 2, 0, 0); + let e: i32 = 0; + let r: i32 = transmute(vqrdmulhs_laneq_s32::<1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrdmlahh_s16() { + let a: i16 = 1; + let b: i16 = 1; + let c: i16 = 2; + let e: i16 = 1; + let r: i16 = transmute(vqrdmlahh_s16(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrdmlahs_s32() { + let a: i32 = 1; + let b: i32 = 1; + let c: i32 = 2; + let e: i32 = 1; + let r: i32 = transmute(vqrdmlahs_s32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrdmlahh_lane_s16() { + let a: i16 = 1; + let b: i16 = 1; + let c: i16x4 = i16x4::new(0, 2, 0, 0); + let e: i16 = 1; + let r: i16 = transmute(vqrdmlahh_lane_s16::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrdmlahh_laneq_s16() { + let a: i16 = 1; + let b: i16 = 1; + let c: i16x8 = i16x8::new(0, 2, 0, 0, 0, 0, 0, 0); + let e: i16 = 1; + let r: i16 = transmute(vqrdmlahh_laneq_s16::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrdmlahs_lane_s32() { + let a: i32 = 1; + let b: i32 = 1; + let c: i32x2 = i32x2::new(0, 2); + let e: i32 = 1; + let r: i32 = transmute(vqrdmlahs_lane_s32::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrdmlahs_laneq_s32() { + let a: i32 = 1; + let b: i32 = 1; + let c: i32x4 = i32x4::new(0, 2, 0, 0); + let e: i32 = 1; + let r: i32 = transmute(vqrdmlahs_laneq_s32::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrdmlshh_s16() { + let a: i16 = 1; + let b: i16 = 1; + let c: i16 = 2; + let e: i16 = 1; + let r: i16 = transmute(vqrdmlshh_s16(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrdmlshs_s32() { + let a: i32 = 1; + let b: i32 = 1; + let c: i32 = 2; + let e: i32 = 1; + let r: i32 = transmute(vqrdmlshs_s32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrdmlshh_lane_s16() { + let a: i16 = 1; + let b: i16 = 1; + let c: i16x4 = i16x4::new(0, 2, 0, 0); + let e: i16 = 1; + let r: i16 = transmute(vqrdmlshh_lane_s16::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrdmlshh_laneq_s16() { + let a: i16 = 1; + let b: i16 = 1; + let c: i16x8 = i16x8::new(0, 2, 0, 0, 0, 0, 0, 0); + let e: i16 = 1; + let r: i16 = transmute(vqrdmlshh_laneq_s16::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrdmlshs_lane_s32() { + let a: i32 = 1; + let b: i32 = 1; + let c: i32x2 = i32x2::new(0, 2); + let e: i32 = 1; + let r: i32 = transmute(vqrdmlshs_lane_s32::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrdmlshs_laneq_s32() { + let a: i32 = 1; + let b: i32 = 1; + let c: i32x4 = i32x4::new(0, 2, 0, 0); + let e: i32 = 1; + let r: i32 = transmute(vqrdmlshs_laneq_s32::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrshls_s32() { + let a: i32 = 2; + let b: i32 = 2; + let e: i32 = 8; + let r: i32 = transmute(vqrshls_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrshld_s64() { + let a: i64 = 2; + let b: i64 = 2; + let e: i64 = 8; + let r: i64 = transmute(vqrshld_s64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrshlb_s8() { + let a: i8 = 1; + let b: i8 = 2; + let e: i8 = 4; + let r: i8 = transmute(vqrshlb_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrshlh_s16() { + let a: i16 = 1; + let b: i16 = 2; + let e: i16 = 4; + let r: i16 = transmute(vqrshlh_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrshls_u32() { + let a: u32 = 2; + let b: i32 = 2; + let e: u32 = 8; + let r: u32 = transmute(vqrshls_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrshld_u64() { + let a: u64 = 2; + let b: i64 = 2; + let e: u64 = 8; + let r: u64 = transmute(vqrshld_u64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrshlb_u8() { + let a: u8 = 1; + let b: i8 = 2; + let e: u8 = 4; + let r: u8 = transmute(vqrshlb_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrshlh_u16() { + let a: u16 = 1; + let b: i16 = 2; + let e: u16 = 4; + let r: u16 = transmute(vqrshlh_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrshrnh_n_s16() { + let a: i16 = 4; + let e: i8 = 1; + let r: i8 = transmute(vqrshrnh_n_s16::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrshrns_n_s32() { + let a: i32 = 4; + let e: i16 = 1; + let r: i16 = transmute(vqrshrns_n_s32::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrshrnd_n_s64() { + let a: i64 = 4; + let e: i32 = 1; + let r: i32 = transmute(vqrshrnd_n_s64::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrshrn_high_n_s16() { + let a: i8x8 = i8x8::new(0, 1, 2, 3, 2, 3, 6, 7); + let b: i16x8 = i16x8::new(8, 12, 24, 28, 48, 52, 56, 60); + let e: i8x16 = i8x16::new(0, 1, 2, 3, 2, 3, 6, 7, 2, 3, 6, 7, 12, 13, 14, 15); + let r: i8x16 = transmute(vqrshrn_high_n_s16::<2>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrshrn_high_n_s32() { + let a: i16x4 = i16x4::new(0, 1, 2, 3); + let b: i32x4 = i32x4::new(8, 12, 24, 28); + let e: i16x8 = i16x8::new(0, 1, 2, 3, 2, 3, 6, 7); + let r: i16x8 = transmute(vqrshrn_high_n_s32::<2>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrshrn_high_n_s64() { + let a: i32x2 = i32x2::new(0, 1); + let b: i64x2 = i64x2::new(8, 12); + let e: i32x4 = i32x4::new(0, 1, 2, 3); + let r: i32x4 = transmute(vqrshrn_high_n_s64::<2>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrshrnh_n_u16() { + let a: u16 = 4; + let e: u8 = 1; + let r: u8 = transmute(vqrshrnh_n_u16::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrshrns_n_u32() { + let a: u32 = 4; + let e: u16 = 1; + let r: u16 = transmute(vqrshrns_n_u32::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrshrnd_n_u64() { + let a: u64 = 4; + let e: u32 = 1; + let r: u32 = transmute(vqrshrnd_n_u64::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrshrn_high_n_u16() { + let a: u8x8 = u8x8::new(0, 1, 2, 3, 2, 3, 6, 7); + let b: u16x8 = u16x8::new(8, 12, 24, 28, 48, 52, 56, 60); + let e: u8x16 = u8x16::new(0, 1, 2, 3, 2, 3, 6, 7, 2, 3, 6, 7, 12, 13, 14, 15); + let r: u8x16 = transmute(vqrshrn_high_n_u16::<2>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrshrn_high_n_u32() { + let a: u16x4 = u16x4::new(0, 1, 2, 3); + let b: u32x4 = u32x4::new(8, 12, 24, 28); + let e: u16x8 = u16x8::new(0, 1, 2, 3, 2, 3, 6, 7); + let r: u16x8 = transmute(vqrshrn_high_n_u32::<2>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrshrn_high_n_u64() { + let a: u32x2 = u32x2::new(0, 1); + let b: u64x2 = u64x2::new(8, 12); + let e: u32x4 = u32x4::new(0, 1, 2, 3); + let r: u32x4 = transmute(vqrshrn_high_n_u64::<2>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrshrunh_n_s16() { + let a: i16 = 4; + let e: u8 = 1; + let r: u8 = transmute(vqrshrunh_n_s16::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrshruns_n_s32() { + let a: i32 = 4; + let e: u16 = 1; + let r: u16 = transmute(vqrshruns_n_s32::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrshrund_n_s64() { + let a: i64 = 4; + let e: u32 = 1; + let r: u32 = transmute(vqrshrund_n_s64::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrshrun_high_n_s16() { + let a: u8x8 = u8x8::new(0, 1, 2, 3, 2, 3, 6, 7); + let b: i16x8 = i16x8::new(8, 12, 24, 28, 48, 52, 56, 60); + let e: u8x16 = u8x16::new(0, 1, 2, 3, 2, 3, 6, 7, 2, 3, 6, 7, 12, 13, 14, 15); + let r: u8x16 = transmute(vqrshrun_high_n_s16::<2>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrshrun_high_n_s32() { + let a: u16x4 = u16x4::new(0, 1, 2, 3); + let b: i32x4 = i32x4::new(8, 12, 24, 28); + let e: u16x8 = u16x8::new(0, 1, 2, 3, 2, 3, 6, 7); + let r: u16x8 = transmute(vqrshrun_high_n_s32::<2>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrshrun_high_n_s64() { + let a: u32x2 = u32x2::new(0, 1); + let b: i64x2 = i64x2::new(8, 12); + let e: u32x4 = u32x4::new(0, 1, 2, 3); + let r: u32x4 = transmute(vqrshrun_high_n_s64::<2>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshld_s64() { + let a: i64 = 0; + let b: i64 = 2; + let e: i64 = 0; + let r: i64 = transmute(vqshld_s64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshlb_s8() { + let a: i8 = 1; + let b: i8 = 2; + let e: i8 = 4; + let r: i8 = transmute(vqshlb_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshlh_s16() { + let a: i16 = 1; + let b: i16 = 2; + let e: i16 = 4; + let r: i16 = transmute(vqshlh_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshls_s32() { + let a: i32 = 1; + let b: i32 = 2; + let e: i32 = 4; + let r: i32 = transmute(vqshls_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshld_u64() { + let a: u64 = 0; + let b: i64 = 2; + let e: u64 = 0; + let r: u64 = transmute(vqshld_u64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshlb_u8() { + let a: u8 = 1; + let b: i8 = 2; + let e: u8 = 4; + let r: u8 = transmute(vqshlb_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshlh_u16() { + let a: u16 = 1; + let b: i16 = 2; + let e: u16 = 4; + let r: u16 = transmute(vqshlh_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshls_u32() { + let a: u32 = 1; + let b: i32 = 2; + let e: u32 = 4; + let r: u32 = transmute(vqshls_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshlb_n_s8() { + let a: i8 = 1; + let e: i8 = 4; + let r: i8 = transmute(vqshlb_n_s8::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshlh_n_s16() { + let a: i16 = 1; + let e: i16 = 4; + let r: i16 = transmute(vqshlh_n_s16::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshls_n_s32() { + let a: i32 = 1; + let e: i32 = 4; + let r: i32 = transmute(vqshls_n_s32::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshld_n_s64() { + let a: i64 = 1; + let e: i64 = 4; + let r: i64 = transmute(vqshld_n_s64::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshlb_n_u8() { + let a: u8 = 1; + let e: u8 = 4; + let r: u8 = transmute(vqshlb_n_u8::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshlh_n_u16() { + let a: u16 = 1; + let e: u16 = 4; + let r: u16 = transmute(vqshlh_n_u16::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshls_n_u32() { + let a: u32 = 1; + let e: u32 = 4; + let r: u32 = transmute(vqshls_n_u32::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshld_n_u64() { + let a: u64 = 1; + let e: u64 = 4; + let r: u64 = transmute(vqshld_n_u64::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshrnd_n_s64() { + let a: i64 = 0; + let e: i32 = 0; + let r: i32 = transmute(vqshrnd_n_s64::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshrnh_n_s16() { + let a: i16 = 4; + let e: i8 = 1; + let r: i8 = transmute(vqshrnh_n_s16::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshrns_n_s32() { + let a: i32 = 4; + let e: i16 = 1; + let r: i16 = transmute(vqshrns_n_s32::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshrn_high_n_s16() { + let a: i8x8 = i8x8::new(0, 1, 8, 9, 8, 9, 10, 11); + let b: i16x8 = i16x8::new(32, 36, 40, 44, 48, 52, 56, 60); + let e: i8x16 = i8x16::new(0, 1, 8, 9, 8, 9, 10, 11, 8, 9, 10, 11, 12, 13, 14, 15); + let r: i8x16 = transmute(vqshrn_high_n_s16::<2>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshrn_high_n_s32() { + let a: i16x4 = i16x4::new(0, 1, 8, 9); + let b: i32x4 = i32x4::new(32, 36, 40, 44); + let e: i16x8 = i16x8::new(0, 1, 8, 9, 8, 9, 10, 11); + let r: i16x8 = transmute(vqshrn_high_n_s32::<2>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshrn_high_n_s64() { + let a: i32x2 = i32x2::new(0, 1); + let b: i64x2 = i64x2::new(32, 36); + let e: i32x4 = i32x4::new(0, 1, 8, 9); + let r: i32x4 = transmute(vqshrn_high_n_s64::<2>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshrnd_n_u64() { + let a: u64 = 0; + let e: u32 = 0; + let r: u32 = transmute(vqshrnd_n_u64::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshrnh_n_u16() { + let a: u16 = 4; + let e: u8 = 1; + let r: u8 = transmute(vqshrnh_n_u16::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshrns_n_u32() { + let a: u32 = 4; + let e: u16 = 1; + let r: u16 = transmute(vqshrns_n_u32::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshrn_high_n_u16() { + let a: u8x8 = u8x8::new(0, 1, 8, 9, 8, 9, 10, 11); + let b: u16x8 = u16x8::new(32, 36, 40, 44, 48, 52, 56, 60); + let e: u8x16 = u8x16::new(0, 1, 8, 9, 8, 9, 10, 11, 8, 9, 10, 11, 12, 13, 14, 15); + let r: u8x16 = transmute(vqshrn_high_n_u16::<2>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshrn_high_n_u32() { + let a: u16x4 = u16x4::new(0, 1, 8, 9); + let b: u32x4 = u32x4::new(32, 36, 40, 44); + let e: u16x8 = u16x8::new(0, 1, 8, 9, 8, 9, 10, 11); + let r: u16x8 = transmute(vqshrn_high_n_u32::<2>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshrn_high_n_u64() { + let a: u32x2 = u32x2::new(0, 1); + let b: u64x2 = u64x2::new(32, 36); + let e: u32x4 = u32x4::new(0, 1, 8, 9); + let r: u32x4 = transmute(vqshrn_high_n_u64::<2>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshrunh_n_s16() { + let a: i16 = 4; + let e: u8 = 1; + let r: u8 = transmute(vqshrunh_n_s16::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshruns_n_s32() { + let a: i32 = 4; + let e: u16 = 1; + let r: u16 = transmute(vqshruns_n_s32::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshrund_n_s64() { + let a: i64 = 4; + let e: u32 = 1; + let r: u32 = transmute(vqshrund_n_s64::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshrun_high_n_s16() { + let a: u8x8 = u8x8::new(0, 1, 8, 9, 8, 9, 10, 11); + let b: i16x8 = i16x8::new(32, 36, 40, 44, 48, 52, 56, 60); + let e: u8x16 = u8x16::new(0, 1, 8, 9, 8, 9, 10, 11, 8, 9, 10, 11, 12, 13, 14, 15); + let r: u8x16 = transmute(vqshrun_high_n_s16::<2>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshrun_high_n_s32() { + let a: u16x4 = u16x4::new(0, 1, 8, 9); + let b: i32x4 = i32x4::new(32, 36, 40, 44); + let e: u16x8 = u16x8::new(0, 1, 8, 9, 8, 9, 10, 11); + let r: u16x8 = transmute(vqshrun_high_n_s32::<2>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshrun_high_n_s64() { + let a: u32x2 = u32x2::new(0, 1); + let b: i64x2 = i64x2::new(32, 36); + let e: u32x4 = u32x4::new(0, 1, 8, 9); + let r: u32x4 = transmute(vqshrun_high_n_s64::<2>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsqrt_f32() { + let a: f32x2 = f32x2::new(4.0, 9.0); + let e: f32x2 = f32x2::new(2.0, 3.0); + let r: f32x2 = transmute(vsqrt_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsqrtq_f32() { + let a: f32x4 = f32x4::new(4.0, 9.0, 16.0, 25.0); + let e: f32x4 = f32x4::new(2.0, 3.0, 4.0, 5.0); + let r: f32x4 = transmute(vsqrtq_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsqrt_f64() { + let a: f64 = 4.0; + let e: f64 = 2.0; + let r: f64 = transmute(vsqrt_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsqrtq_f64() { + let a: f64x2 = f64x2::new(4.0, 9.0); + let e: f64x2 = f64x2::new(2.0, 3.0); + let r: f64x2 = transmute(vsqrtq_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrsqrte_f64() { + let a: f64 = 1.0; + let e: f64 = 0.998046875; + let r: f64 = transmute(vrsqrte_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrsqrteq_f64() { + let a: f64x2 = f64x2::new(1.0, 2.0); + let e: f64x2 = f64x2::new(0.998046875, 0.705078125); + let r: f64x2 = transmute(vrsqrteq_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrecpe_f64() { + let a: f64 = 4.0; + let e: f64 = 0.24951171875; + let r: f64 = transmute(vrecpe_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrecpeq_f64() { + let a: f64x2 = f64x2::new(4.0, 3.0); + let e: f64x2 = f64x2::new(0.24951171875, 0.3330078125); + let r: f64x2 = transmute(vrecpeq_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_s64_p64() { + let a: i64x1 = i64x1::new(0); + let e: i64x1 = i64x1::new(0); + let r: i64x1 = transmute(vreinterpret_s64_p64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_u64_p64() { + let a: i64x1 = i64x1::new(0); + let e: u64x1 = u64x1::new(0); + let r: u64x1 = transmute(vreinterpret_u64_p64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_p64_s64() { + let a: i64x1 = i64x1::new(0); + let e: i64x1 = i64x1::new(0); + let r: i64x1 = transmute(vreinterpret_p64_s64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_p64_u64() { + let a: u64x1 = u64x1::new(0); + let e: i64x1 = i64x1::new(0); + let r: i64x1 = transmute(vreinterpret_p64_u64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_s64_p64() { + let a: i64x2 = i64x2::new(0, 1); + let e: i64x2 = i64x2::new(0, 1); + let r: i64x2 = transmute(vreinterpretq_s64_p64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_u64_p64() { + let a: i64x2 = i64x2::new(0, 1); + let e: u64x2 = u64x2::new(0, 1); + let r: u64x2 = transmute(vreinterpretq_u64_p64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_p64_s64() { + let a: i64x2 = i64x2::new(0, 1); + let e: i64x2 = i64x2::new(0, 1); + let r: i64x2 = transmute(vreinterpretq_p64_s64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_p64_u64() { + let a: u64x2 = u64x2::new(0, 1); + let e: i64x2 = i64x2::new(0, 1); + let r: i64x2 = transmute(vreinterpretq_p64_u64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_s32_p64() { + let a: i64x1 = i64x1::new(0); + let e: i32x2 = i32x2::new(0, 0); + let r: i32x2 = transmute(vreinterpret_s32_p64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_u32_p64() { + let a: i64x1 = i64x1::new(0); + let e: u32x2 = u32x2::new(0, 0); + let r: u32x2 = transmute(vreinterpret_u32_p64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_s32_p64() { + let a: i64x2 = i64x2::new(0, 1); + let e: i32x4 = i32x4::new(0, 0, 1, 0); + let r: i32x4 = transmute(vreinterpretq_s32_p64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_u32_p64() { + let a: i64x2 = i64x2::new(0, 1); + let e: u32x4 = u32x4::new(0, 0, 1, 0); + let r: u32x4 = transmute(vreinterpretq_u32_p64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_p64_s32() { + let a: i32x2 = i32x2::new(0, 0); + let e: i64x1 = i64x1::new(0); + let r: i64x1 = transmute(vreinterpret_p64_s32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_p64_u32() { + let a: u32x2 = u32x2::new(0, 0); + let e: i64x1 = i64x1::new(0); + let r: i64x1 = transmute(vreinterpret_p64_u32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_p64_s32() { + let a: i32x4 = i32x4::new(0, 0, 1, 0); + let e: i64x2 = i64x2::new(0, 1); + let r: i64x2 = transmute(vreinterpretq_p64_s32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_p64_u32() { + let a: u32x4 = u32x4::new(0, 0, 1, 0); + let e: i64x2 = i64x2::new(0, 1); + let r: i64x2 = transmute(vreinterpretq_p64_u32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_s16_p64() { + let a: i64x1 = i64x1::new(0); + let e: i16x4 = i16x4::new(0, 0, 0, 0); + let r: i16x4 = transmute(vreinterpret_s16_p64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_u16_p64() { + let a: i64x1 = i64x1::new(0); + let e: u16x4 = u16x4::new(0, 0, 0, 0); + let r: u16x4 = transmute(vreinterpret_u16_p64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_p16_p64() { + let a: i64x1 = i64x1::new(0); + let e: i16x4 = i16x4::new(0, 0, 0, 0); + let r: i16x4 = transmute(vreinterpret_p16_p64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_s16_p64() { + let a: i64x2 = i64x2::new(0, 1); + let e: i16x8 = i16x8::new(0, 0, 0, 0, 1, 0, 0, 0); + let r: i16x8 = transmute(vreinterpretq_s16_p64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_u16_p64() { + let a: i64x2 = i64x2::new(0, 1); + let e: u16x8 = u16x8::new(0, 0, 0, 0, 1, 0, 0, 0); + let r: u16x8 = transmute(vreinterpretq_u16_p64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_p16_p64() { + let a: i64x2 = i64x2::new(0, 1); + let e: i16x8 = i16x8::new(0, 0, 0, 0, 1, 0, 0, 0); + let r: i16x8 = transmute(vreinterpretq_p16_p64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_p64_p16() { + let a: i16x4 = i16x4::new(0, 0, 0, 0); + let e: i64x1 = i64x1::new(0); + let r: i64x1 = transmute(vreinterpret_p64_p16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_p64_s16() { + let a: i16x4 = i16x4::new(0, 0, 0, 0); + let e: i64x1 = i64x1::new(0); + let r: i64x1 = transmute(vreinterpret_p64_s16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_p64_u16() { + let a: u16x4 = u16x4::new(0, 0, 0, 0); + let e: i64x1 = i64x1::new(0); + let r: i64x1 = transmute(vreinterpret_p64_u16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_p64_p16() { + let a: i16x8 = i16x8::new(0, 0, 0, 0, 1, 0, 0, 0); + let e: i64x2 = i64x2::new(0, 1); + let r: i64x2 = transmute(vreinterpretq_p64_p16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_p64_s16() { + let a: i16x8 = i16x8::new(0, 0, 0, 0, 1, 0, 0, 0); + let e: i64x2 = i64x2::new(0, 1); + let r: i64x2 = transmute(vreinterpretq_p64_s16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_p64_u16() { + let a: u16x8 = u16x8::new(0, 0, 0, 0, 1, 0, 0, 0); + let e: i64x2 = i64x2::new(0, 1); + let r: i64x2 = transmute(vreinterpretq_p64_u16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_s8_p64() { + let a: i64x1 = i64x1::new(0); + let e: i8x8 = i8x8::new(0, 0, 0, 0, 0, 0, 0, 0); + let r: i8x8 = transmute(vreinterpret_s8_p64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_u8_p64() { + let a: i64x1 = i64x1::new(0); + let e: u8x8 = u8x8::new(0, 0, 0, 0, 0, 0, 0, 0); + let r: u8x8 = transmute(vreinterpret_u8_p64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_p8_p64() { + let a: i64x1 = i64x1::new(0); + let e: i8x8 = i8x8::new(0, 0, 0, 0, 0, 0, 0, 0); + let r: i8x8 = transmute(vreinterpret_p8_p64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_s8_p64() { + let a: i64x2 = i64x2::new(0, 1); + let e: i8x16 = i8x16::new(0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0); + let r: i8x16 = transmute(vreinterpretq_s8_p64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_u8_p64() { + let a: i64x2 = i64x2::new(0, 1); + let e: u8x16 = u8x16::new(0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0); + let r: u8x16 = transmute(vreinterpretq_u8_p64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_p8_p64() { + let a: i64x2 = i64x2::new(0, 1); + let e: i8x16 = i8x16::new(0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0); + let r: i8x16 = transmute(vreinterpretq_p8_p64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_p64_p8() { + let a: i8x8 = i8x8::new(0, 0, 0, 0, 0, 0, 0, 0); + let e: i64x1 = i64x1::new(0); + let r: i64x1 = transmute(vreinterpret_p64_p8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_p64_s8() { + let a: i8x8 = i8x8::new(0, 0, 0, 0, 0, 0, 0, 0); + let e: i64x1 = i64x1::new(0); + let r: i64x1 = transmute(vreinterpret_p64_s8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_p64_u8() { + let a: u8x8 = u8x8::new(0, 0, 0, 0, 0, 0, 0, 0); + let e: i64x1 = i64x1::new(0); + let r: i64x1 = transmute(vreinterpret_p64_u8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_p64_p8() { + let a: i8x16 = i8x16::new(0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0); + let e: i64x2 = i64x2::new(0, 1); + let r: i64x2 = transmute(vreinterpretq_p64_p8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_p64_s8() { + let a: i8x16 = i8x16::new(0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0); + let e: i64x2 = i64x2::new(0, 1); + let r: i64x2 = transmute(vreinterpretq_p64_s8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_p64_u8() { + let a: u8x16 = u8x16::new(0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0); + let e: i64x2 = i64x2::new(0, 1); + let r: i64x2 = transmute(vreinterpretq_p64_u8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_s8_f64() { + let a: f64 = 0.; + let e: i8x8 = i8x8::new(0, 0, 0, 0, 0, 0, 0, 0); + let r: i8x8 = transmute(vreinterpret_s8_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_s16_f64() { + let a: f64 = 0.; + let e: i16x4 = i16x4::new(0, 0, 0, 0); + let r: i16x4 = transmute(vreinterpret_s16_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_s32_f64() { + let a: f64 = 0.; + let e: i32x2 = i32x2::new(0, 0); + let r: i32x2 = transmute(vreinterpret_s32_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_s64_f64() { + let a: f64 = 0.; + let e: i64x1 = i64x1::new(0); + let r: i64x1 = transmute(vreinterpret_s64_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_s8_f64() { + let a: f64x2 = f64x2::new(0., 0.); + let e: i8x16 = i8x16::new(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + let r: i8x16 = transmute(vreinterpretq_s8_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_s16_f64() { + let a: f64x2 = f64x2::new(0., 0.); + let e: i16x8 = i16x8::new(0, 0, 0, 0, 0, 0, 0, 0); + let r: i16x8 = transmute(vreinterpretq_s16_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_s32_f64() { + let a: f64x2 = f64x2::new(0., 0.); + let e: i32x4 = i32x4::new(0, 0, 0, 0); + let r: i32x4 = transmute(vreinterpretq_s32_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_s64_f64() { + let a: f64x2 = f64x2::new(0., 0.); + let e: i64x2 = i64x2::new(0, 0); + let r: i64x2 = transmute(vreinterpretq_s64_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_u8_f64() { + let a: f64 = 0.; + let e: u8x8 = u8x8::new(0, 0, 0, 0, 0, 0, 0, 0); + let r: u8x8 = transmute(vreinterpret_u8_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_u16_f64() { + let a: f64 = 0.; + let e: u16x4 = u16x4::new(0, 0, 0, 0); + let r: u16x4 = transmute(vreinterpret_u16_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_u32_f64() { + let a: f64 = 0.; + let e: u32x2 = u32x2::new(0, 0); + let r: u32x2 = transmute(vreinterpret_u32_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_u64_f64() { + let a: f64 = 0.; + let e: u64x1 = u64x1::new(0); + let r: u64x1 = transmute(vreinterpret_u64_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_u8_f64() { + let a: f64x2 = f64x2::new(0., 0.); + let e: u8x16 = u8x16::new(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + let r: u8x16 = transmute(vreinterpretq_u8_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_u16_f64() { + let a: f64x2 = f64x2::new(0., 0.); + let e: u16x8 = u16x8::new(0, 0, 0, 0, 0, 0, 0, 0); + let r: u16x8 = transmute(vreinterpretq_u16_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_u32_f64() { + let a: f64x2 = f64x2::new(0., 0.); + let e: u32x4 = u32x4::new(0, 0, 0, 0); + let r: u32x4 = transmute(vreinterpretq_u32_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_u64_f64() { + let a: f64x2 = f64x2::new(0., 0.); + let e: u64x2 = u64x2::new(0, 0); + let r: u64x2 = transmute(vreinterpretq_u64_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_p8_f64() { + let a: f64 = 0.; + let e: i8x8 = i8x8::new(0, 0, 0, 0, 0, 0, 0, 0); + let r: i8x8 = transmute(vreinterpret_p8_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_p16_f64() { + let a: f64 = 0.; + let e: i16x4 = i16x4::new(0, 0, 0, 0); + let r: i16x4 = transmute(vreinterpret_p16_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_p64_f32() { + let a: f32x2 = f32x2::new(0., 0.); + let e: i64x1 = i64x1::new(0); + let r: i64x1 = transmute(vreinterpret_p64_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_p64_f64() { + let a: f64 = 0.; + let e: i64x1 = i64x1::new(0); + let r: i64x1 = transmute(vreinterpret_p64_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_p8_f64() { + let a: f64x2 = f64x2::new(0., 0.); + let e: i8x16 = i8x16::new(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + let r: i8x16 = transmute(vreinterpretq_p8_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_p16_f64() { + let a: f64x2 = f64x2::new(0., 0.); + let e: i16x8 = i16x8::new(0, 0, 0, 0, 0, 0, 0, 0); + let r: i16x8 = transmute(vreinterpretq_p16_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_p64_f32() { + let a: f32x4 = f32x4::new(0., 0., 0., 0.); + let e: i64x2 = i64x2::new(0, 0); + let r: i64x2 = transmute(vreinterpretq_p64_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_p64_f64() { + let a: f64x2 = f64x2::new(0., 0.); + let e: i64x2 = i64x2::new(0, 0); + let r: i64x2 = transmute(vreinterpretq_p64_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_f64_s8() { + let a: i8x8 = i8x8::new(0, 0, 0, 0, 0, 0, 0, 0); + let e: f64 = 0.; + let r: f64 = transmute(vreinterpret_f64_s8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_f64_s16() { + let a: i16x4 = i16x4::new(0, 0, 0, 0); + let e: f64 = 0.; + let r: f64 = transmute(vreinterpret_f64_s16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_f64_s32() { + let a: i32x2 = i32x2::new(0, 0); + let e: f64 = 0.; + let r: f64 = transmute(vreinterpret_f64_s32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_f64_s64() { + let a: i64x1 = i64x1::new(0); + let e: f64 = 0.; + let r: f64 = transmute(vreinterpret_f64_s64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_f64_s8() { + let a: i8x16 = i8x16::new(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + let e: f64x2 = f64x2::new(0., 0.); + let r: f64x2 = transmute(vreinterpretq_f64_s8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_f64_s16() { + let a: i16x8 = i16x8::new(0, 0, 0, 0, 0, 0, 0, 0); + let e: f64x2 = f64x2::new(0., 0.); + let r: f64x2 = transmute(vreinterpretq_f64_s16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_f64_s32() { + let a: i32x4 = i32x4::new(0, 0, 0, 0); + let e: f64x2 = f64x2::new(0., 0.); + let r: f64x2 = transmute(vreinterpretq_f64_s32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_f64_s64() { + let a: i64x2 = i64x2::new(0, 0); + let e: f64x2 = f64x2::new(0., 0.); + let r: f64x2 = transmute(vreinterpretq_f64_s64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_f64_p8() { + let a: i8x8 = i8x8::new(0, 0, 0, 0, 0, 0, 0, 0); + let e: f64 = 0.; + let r: f64 = transmute(vreinterpret_f64_p8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_f64_u16() { + let a: u16x4 = u16x4::new(0, 0, 0, 0); + let e: f64 = 0.; + let r: f64 = transmute(vreinterpret_f64_u16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_f64_u32() { + let a: u32x2 = u32x2::new(0, 0); + let e: f64 = 0.; + let r: f64 = transmute(vreinterpret_f64_u32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_f64_u64() { + let a: u64x1 = u64x1::new(0); + let e: f64 = 0.; + let r: f64 = transmute(vreinterpret_f64_u64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_f64_p8() { + let a: i8x16 = i8x16::new(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + let e: f64x2 = f64x2::new(0., 0.); + let r: f64x2 = transmute(vreinterpretq_f64_p8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_f64_u16() { + let a: u16x8 = u16x8::new(0, 0, 0, 0, 0, 0, 0, 0); + let e: f64x2 = f64x2::new(0., 0.); + let r: f64x2 = transmute(vreinterpretq_f64_u16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_f64_u32() { + let a: u32x4 = u32x4::new(0, 0, 0, 0); + let e: f64x2 = f64x2::new(0., 0.); + let r: f64x2 = transmute(vreinterpretq_f64_u32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_f64_u64() { + let a: u64x2 = u64x2::new(0, 0); + let e: f64x2 = f64x2::new(0., 0.); + let r: f64x2 = transmute(vreinterpretq_f64_u64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_f64_u8() { + let a: u8x8 = u8x8::new(0, 0, 0, 0, 0, 0, 0, 0); + let e: f64 = 0.; + let r: f64 = transmute(vreinterpret_f64_u8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_f64_p16() { + let a: i16x4 = i16x4::new(0, 0, 0, 0); + let e: f64 = 0.; + let r: f64 = transmute(vreinterpret_f64_p16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_f64_p64() { + let a: i64x1 = i64x1::new(0); + let e: f64 = 0.; + let r: f64 = transmute(vreinterpret_f64_p64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_f32_p64() { + let a: i64x1 = i64x1::new(0); + let e: f32x2 = f32x2::new(0., 0.); + let r: f32x2 = transmute(vreinterpret_f32_p64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_f64_u8() { + let a: u8x16 = u8x16::new(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + let e: f64x2 = f64x2::new(0., 0.); + let r: f64x2 = transmute(vreinterpretq_f64_u8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_f64_p16() { + let a: i16x8 = i16x8::new(0, 0, 0, 0, 0, 0, 0, 0); + let e: f64x2 = f64x2::new(0., 0.); + let r: f64x2 = transmute(vreinterpretq_f64_p16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_f64_p64() { + let a: i64x2 = i64x2::new(0, 0); + let e: f64x2 = f64x2::new(0., 0.); + let r: f64x2 = transmute(vreinterpretq_f64_p64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_f32_p64() { + let a: i64x2 = i64x2::new(0, 0); + let e: f32x4 = f32x4::new(0., 0., 0., 0.); + let r: f32x4 = transmute(vreinterpretq_f32_p64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_f64_f32() { + let a: f32x2 = f32x2::new(0., 0.); + let e: f64 = 0.; + let r: f64 = transmute(vreinterpret_f64_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_f32_f64() { + let a: f64 = 0.; + let e: f32x2 = f32x2::new(0., 0.); + let r: f32x2 = transmute(vreinterpret_f32_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_f64_f32() { + let a: f32x4 = f32x4::new(0., 0., 0., 0.); + let e: f64x2 = f64x2::new(0., 0.); + let r: f64x2 = transmute(vreinterpretq_f64_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_f32_f64() { + let a: f64x2 = f64x2::new(0., 0.); + let e: f32x4 = f32x4::new(0., 0., 0., 0.); + let r: f32x4 = transmute(vreinterpretq_f32_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrshld_s64() { + let a: i64 = 1; + let b: i64 = 2; + let e: i64 = 4; + let r: i64 = transmute(vrshld_s64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrshld_u64() { + let a: u64 = 1; + let b: i64 = 2; + let e: u64 = 4; + let r: u64 = transmute(vrshld_u64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrshrd_n_s64() { + let a: i64 = 4; + let e: i64 = 1; + let r: i64 = transmute(vrshrd_n_s64::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrshrd_n_u64() { + let a: u64 = 4; + let e: u64 = 1; + let r: u64 = transmute(vrshrd_n_u64::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrshrn_high_n_s16() { + let a: i8x8 = i8x8::new(0, 1, 8, 9, 8, 9, 10, 11); + let b: i16x8 = i16x8::new(32, 36, 40, 44, 48, 52, 56, 60); + let e: i8x16 = i8x16::new(0, 1, 8, 9, 8, 9, 10, 11, 8, 9, 10, 11, 12, 13, 14, 15); + let r: i8x16 = transmute(vrshrn_high_n_s16::<2>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrshrn_high_n_s32() { + let a: i16x4 = i16x4::new(0, 1, 8, 9); + let b: i32x4 = i32x4::new(32, 36, 40, 44); + let e: i16x8 = i16x8::new(0, 1, 8, 9, 8, 9, 10, 11); + let r: i16x8 = transmute(vrshrn_high_n_s32::<2>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrshrn_high_n_s64() { + let a: i32x2 = i32x2::new(0, 1); + let b: i64x2 = i64x2::new(32, 36); + let e: i32x4 = i32x4::new(0, 1, 8, 9); + let r: i32x4 = transmute(vrshrn_high_n_s64::<2>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrshrn_high_n_u16() { + let a: u8x8 = u8x8::new(0, 1, 8, 9, 8, 9, 10, 11); + let b: u16x8 = u16x8::new(32, 36, 40, 44, 48, 52, 56, 60); + let e: u8x16 = u8x16::new(0, 1, 8, 9, 8, 9, 10, 11, 8, 9, 10, 11, 12, 13, 14, 15); + let r: u8x16 = transmute(vrshrn_high_n_u16::<2>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrshrn_high_n_u32() { + let a: u16x4 = u16x4::new(0, 1, 8, 9); + let b: u32x4 = u32x4::new(32, 36, 40, 44); + let e: u16x8 = u16x8::new(0, 1, 8, 9, 8, 9, 10, 11); + let r: u16x8 = transmute(vrshrn_high_n_u32::<2>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrshrn_high_n_u64() { + let a: u32x2 = u32x2::new(0, 1); + let b: u64x2 = u64x2::new(32, 36); + let e: u32x4 = u32x4::new(0, 1, 8, 9); + let r: u32x4 = transmute(vrshrn_high_n_u64::<2>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrsrad_n_s64() { + let a: i64 = 1; + let b: i64 = 4; + let e: i64 = 2; + let r: i64 = transmute(vrsrad_n_s64::<2>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrsrad_n_u64() { + let a: u64 = 1; + let b: u64 = 4; + let e: u64 = 2; + let r: u64 = transmute(vrsrad_n_u64::<2>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vset_lane_f64() { + let a: f64 = 1.; + let b: f64 = 0.; + let e: f64 = 1.; + let r: f64 = transmute(vset_lane_f64::<0>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsetq_lane_f64() { + let a: f64 = 1.; + let b: f64x2 = f64x2::new(0., 2.); + let e: f64x2 = f64x2::new(1., 2.); + let r: f64x2 = transmute(vsetq_lane_f64::<0>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vshld_s64() { + let a: i64 = 1; + let b: i64 = 2; + let e: i64 = 4; + let r: i64 = transmute(vshld_s64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vshld_u64() { + let a: u64 = 1; + let b: i64 = 2; + let e: u64 = 4; + let r: u64 = transmute(vshld_u64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vshll_high_n_s8() { + let a: i8x16 = i8x16::new(0, 0, 1, 2, 1, 2, 3, 4, 1, 2, 3, 4, 5, 6, 7, 8); + let e: i16x8 = i16x8::new(4, 8, 12, 16, 20, 24, 28, 32); + let r: i16x8 = transmute(vshll_high_n_s8::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vshll_high_n_s16() { + let a: i16x8 = i16x8::new(0, 0, 1, 2, 1, 2, 3, 4); + let e: i32x4 = i32x4::new(4, 8, 12, 16); + let r: i32x4 = transmute(vshll_high_n_s16::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vshll_high_n_s32() { + let a: i32x4 = i32x4::new(0, 0, 1, 2); + let e: i64x2 = i64x2::new(4, 8); + let r: i64x2 = transmute(vshll_high_n_s32::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vshll_high_n_u8() { + let a: u8x16 = u8x16::new(0, 0, 1, 2, 1, 2, 3, 4, 1, 2, 3, 4, 5, 6, 7, 8); + let e: u16x8 = u16x8::new(4, 8, 12, 16, 20, 24, 28, 32); + let r: u16x8 = transmute(vshll_high_n_u8::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vshll_high_n_u16() { + let a: u16x8 = u16x8::new(0, 0, 1, 2, 1, 2, 3, 4); + let e: u32x4 = u32x4::new(4, 8, 12, 16); + let r: u32x4 = transmute(vshll_high_n_u16::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vshll_high_n_u32() { + let a: u32x4 = u32x4::new(0, 0, 1, 2); + let e: u64x2 = u64x2::new(4, 8); + let r: u64x2 = transmute(vshll_high_n_u32::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vshrn_high_n_s16() { + let a: i8x8 = i8x8::new(1, 2, 5, 6, 5, 6, 7, 8); + let b: i16x8 = i16x8::new(20, 24, 28, 32, 52, 56, 60, 64); + let e: i8x16 = i8x16::new(1, 2, 5, 6, 5, 6, 7, 8, 5, 6, 7, 8, 13, 14, 15, 16); + let r: i8x16 = transmute(vshrn_high_n_s16::<2>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vshrn_high_n_s32() { + let a: i16x4 = i16x4::new(1, 2, 5, 6); + let b: i32x4 = i32x4::new(20, 24, 28, 32); + let e: i16x8 = i16x8::new(1, 2, 5, 6, 5, 6, 7, 8); + let r: i16x8 = transmute(vshrn_high_n_s32::<2>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vshrn_high_n_s64() { + let a: i32x2 = i32x2::new(1, 2); + let b: i64x2 = i64x2::new(20, 24); + let e: i32x4 = i32x4::new(1, 2, 5, 6); + let r: i32x4 = transmute(vshrn_high_n_s64::<2>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vshrn_high_n_u16() { + let a: u8x8 = u8x8::new(1, 2, 5, 6, 5, 6, 7, 8); + let b: u16x8 = u16x8::new(20, 24, 28, 32, 52, 56, 60, 64); + let e: u8x16 = u8x16::new(1, 2, 5, 6, 5, 6, 7, 8, 5, 6, 7, 8, 13, 14, 15, 16); + let r: u8x16 = transmute(vshrn_high_n_u16::<2>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vshrn_high_n_u32() { + let a: u16x4 = u16x4::new(1, 2, 5, 6); + let b: u32x4 = u32x4::new(20, 24, 28, 32); + let e: u16x8 = u16x8::new(1, 2, 5, 6, 5, 6, 7, 8); + let r: u16x8 = transmute(vshrn_high_n_u32::<2>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vshrn_high_n_u64() { + let a: u32x2 = u32x2::new(1, 2); + let b: u64x2 = u64x2::new(20, 24); + let e: u32x4 = u32x4::new(1, 2, 5, 6); + let r: u32x4 = transmute(vshrn_high_n_u64::<2>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vtrn1_s8() { + let a: i8x8 = i8x8::new(0, 2, 4, 6, 8, 10, 12, 14); + let b: i8x8 = i8x8::new(1, 3, 5, 7, 9, 11, 13, 15); + let e: i8x8 = i8x8::new(0, 1, 4, 5, 8, 9, 12, 13); + let r: i8x8 = transmute(vtrn1_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vtrn1q_s8() { + let a: i8x16 = i8x16::new(0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30); + let b: i8x16 = i8x16::new(1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31); + let e: i8x16 = i8x16::new(0, 1, 4, 5, 8, 9, 12, 13, 16, 17, 20, 21, 24, 25, 28, 29); + let r: i8x16 = transmute(vtrn1q_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vtrn1_s16() { + let a: i16x4 = i16x4::new(0, 2, 4, 6); + let b: i16x4 = i16x4::new(1, 3, 5, 7); + let e: i16x4 = i16x4::new(0, 1, 4, 5); + let r: i16x4 = transmute(vtrn1_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vtrn1q_s16() { + let a: i16x8 = i16x8::new(0, 2, 4, 6, 8, 10, 12, 14); + let b: i16x8 = i16x8::new(1, 3, 5, 7, 9, 11, 13, 15); + let e: i16x8 = i16x8::new(0, 1, 4, 5, 8, 9, 12, 13); + let r: i16x8 = transmute(vtrn1q_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vtrn1q_s32() { + let a: i32x4 = i32x4::new(0, 2, 4, 6); + let b: i32x4 = i32x4::new(1, 3, 5, 7); + let e: i32x4 = i32x4::new(0, 1, 4, 5); + let r: i32x4 = transmute(vtrn1q_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vtrn1_u8() { + let a: u8x8 = u8x8::new(0, 2, 4, 6, 8, 10, 12, 14); + let b: u8x8 = u8x8::new(1, 3, 5, 7, 9, 11, 13, 15); + let e: u8x8 = u8x8::new(0, 1, 4, 5, 8, 9, 12, 13); + let r: u8x8 = transmute(vtrn1_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vtrn1q_u8() { + let a: u8x16 = u8x16::new(0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30); + let b: u8x16 = u8x16::new(1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31); + let e: u8x16 = u8x16::new(0, 1, 4, 5, 8, 9, 12, 13, 16, 17, 20, 21, 24, 25, 28, 29); + let r: u8x16 = transmute(vtrn1q_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vtrn1_u16() { + let a: u16x4 = u16x4::new(0, 2, 4, 6); + let b: u16x4 = u16x4::new(1, 3, 5, 7); + let e: u16x4 = u16x4::new(0, 1, 4, 5); + let r: u16x4 = transmute(vtrn1_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vtrn1q_u16() { + let a: u16x8 = u16x8::new(0, 2, 4, 6, 8, 10, 12, 14); + let b: u16x8 = u16x8::new(1, 3, 5, 7, 9, 11, 13, 15); + let e: u16x8 = u16x8::new(0, 1, 4, 5, 8, 9, 12, 13); + let r: u16x8 = transmute(vtrn1q_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vtrn1q_u32() { + let a: u32x4 = u32x4::new(0, 2, 4, 6); + let b: u32x4 = u32x4::new(1, 3, 5, 7); + let e: u32x4 = u32x4::new(0, 1, 4, 5); + let r: u32x4 = transmute(vtrn1q_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vtrn1_p8() { + let a: i8x8 = i8x8::new(0, 2, 4, 6, 8, 10, 12, 14); + let b: i8x8 = i8x8::new(1, 3, 5, 7, 9, 11, 13, 15); + let e: i8x8 = i8x8::new(0, 1, 4, 5, 8, 9, 12, 13); + let r: i8x8 = transmute(vtrn1_p8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vtrn1q_p8() { + let a: i8x16 = i8x16::new(0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30); + let b: i8x16 = i8x16::new(1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31); + let e: i8x16 = i8x16::new(0, 1, 4, 5, 8, 9, 12, 13, 16, 17, 20, 21, 24, 25, 28, 29); + let r: i8x16 = transmute(vtrn1q_p8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vtrn1_p16() { + let a: i16x4 = i16x4::new(0, 2, 4, 6); + let b: i16x4 = i16x4::new(1, 3, 5, 7); + let e: i16x4 = i16x4::new(0, 1, 4, 5); + let r: i16x4 = transmute(vtrn1_p16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vtrn1q_p16() { + let a: i16x8 = i16x8::new(0, 2, 4, 6, 8, 10, 12, 14); + let b: i16x8 = i16x8::new(1, 3, 5, 7, 9, 11, 13, 15); + let e: i16x8 = i16x8::new(0, 1, 4, 5, 8, 9, 12, 13); + let r: i16x8 = transmute(vtrn1q_p16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vtrn1_s32() { + let a: i32x2 = i32x2::new(0, 2); + let b: i32x2 = i32x2::new(1, 3); + let e: i32x2 = i32x2::new(0, 1); + let r: i32x2 = transmute(vtrn1_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vtrn1q_s64() { + let a: i64x2 = i64x2::new(0, 2); + let b: i64x2 = i64x2::new(1, 3); + let e: i64x2 = i64x2::new(0, 1); + let r: i64x2 = transmute(vtrn1q_s64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vtrn1_u32() { + let a: u32x2 = u32x2::new(0, 2); + let b: u32x2 = u32x2::new(1, 3); + let e: u32x2 = u32x2::new(0, 1); + let r: u32x2 = transmute(vtrn1_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vtrn1q_u64() { + let a: u64x2 = u64x2::new(0, 2); + let b: u64x2 = u64x2::new(1, 3); + let e: u64x2 = u64x2::new(0, 1); + let r: u64x2 = transmute(vtrn1q_u64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vtrn1q_p64() { + let a: i64x2 = i64x2::new(0, 2); + let b: i64x2 = i64x2::new(1, 3); + let e: i64x2 = i64x2::new(0, 1); + let r: i64x2 = transmute(vtrn1q_p64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vtrn1q_f32() { + let a: f32x4 = f32x4::new(0., 2., 4., 6.); + let b: f32x4 = f32x4::new(1., 3., 5., 7.); + let e: f32x4 = f32x4::new(0., 1., 4., 5.); + let r: f32x4 = transmute(vtrn1q_f32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vtrn1_f32() { + let a: f32x2 = f32x2::new(0., 2.); + let b: f32x2 = f32x2::new(1., 3.); + let e: f32x2 = f32x2::new(0., 1.); + let r: f32x2 = transmute(vtrn1_f32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vtrn1q_f64() { + let a: f64x2 = f64x2::new(0., 2.); + let b: f64x2 = f64x2::new(1., 3.); + let e: f64x2 = f64x2::new(0., 1.); + let r: f64x2 = transmute(vtrn1q_f64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vtrn2_s8() { + let a: i8x8 = i8x8::new(0, 2, 4, 6, 8, 10, 12, 14); + let b: i8x8 = i8x8::new(1, 3, 5, 7, 9, 11, 13, 15); + let e: i8x8 = i8x8::new(2, 3, 6, 7, 10, 11, 14, 15); + let r: i8x8 = transmute(vtrn2_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vtrn2q_s8() { + let a: i8x16 = i8x16::new(0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30); + let b: i8x16 = i8x16::new(1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31); + let e: i8x16 = i8x16::new(2, 3, 6, 7, 10, 11, 14, 15, 18, 19, 22, 23, 26, 27, 30, 31); + let r: i8x16 = transmute(vtrn2q_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vtrn2_s16() { + let a: i16x4 = i16x4::new(0, 2, 4, 6); + let b: i16x4 = i16x4::new(1, 3, 5, 7); + let e: i16x4 = i16x4::new(2, 3, 6, 7); + let r: i16x4 = transmute(vtrn2_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vtrn2q_s16() { + let a: i16x8 = i16x8::new(0, 2, 4, 6, 8, 10, 12, 14); + let b: i16x8 = i16x8::new(1, 3, 5, 7, 9, 11, 13, 15); + let e: i16x8 = i16x8::new(2, 3, 6, 7, 10, 11, 14, 15); + let r: i16x8 = transmute(vtrn2q_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vtrn2q_s32() { + let a: i32x4 = i32x4::new(0, 2, 4, 6); + let b: i32x4 = i32x4::new(1, 3, 5, 7); + let e: i32x4 = i32x4::new(2, 3, 6, 7); + let r: i32x4 = transmute(vtrn2q_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vtrn2_u8() { + let a: u8x8 = u8x8::new(0, 2, 4, 6, 8, 10, 12, 14); + let b: u8x8 = u8x8::new(1, 3, 5, 7, 9, 11, 13, 15); + let e: u8x8 = u8x8::new(2, 3, 6, 7, 10, 11, 14, 15); + let r: u8x8 = transmute(vtrn2_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vtrn2q_u8() { + let a: u8x16 = u8x16::new(0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30); + let b: u8x16 = u8x16::new(1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31); + let e: u8x16 = u8x16::new(2, 3, 6, 7, 10, 11, 14, 15, 18, 19, 22, 23, 26, 27, 30, 31); + let r: u8x16 = transmute(vtrn2q_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vtrn2_u16() { + let a: u16x4 = u16x4::new(0, 2, 4, 6); + let b: u16x4 = u16x4::new(1, 3, 5, 7); + let e: u16x4 = u16x4::new(2, 3, 6, 7); + let r: u16x4 = transmute(vtrn2_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vtrn2q_u16() { + let a: u16x8 = u16x8::new(0, 2, 4, 6, 8, 10, 12, 14); + let b: u16x8 = u16x8::new(1, 3, 5, 7, 9, 11, 13, 15); + let e: u16x8 = u16x8::new(2, 3, 6, 7, 10, 11, 14, 15); + let r: u16x8 = transmute(vtrn2q_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vtrn2q_u32() { + let a: u32x4 = u32x4::new(0, 2, 4, 6); + let b: u32x4 = u32x4::new(1, 3, 5, 7); + let e: u32x4 = u32x4::new(2, 3, 6, 7); + let r: u32x4 = transmute(vtrn2q_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vtrn2_p8() { + let a: i8x8 = i8x8::new(0, 2, 4, 6, 8, 10, 12, 14); + let b: i8x8 = i8x8::new(1, 3, 5, 7, 9, 11, 13, 15); + let e: i8x8 = i8x8::new(2, 3, 6, 7, 10, 11, 14, 15); + let r: i8x8 = transmute(vtrn2_p8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vtrn2q_p8() { + let a: i8x16 = i8x16::new(0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30); + let b: i8x16 = i8x16::new(1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31); + let e: i8x16 = i8x16::new(2, 3, 6, 7, 10, 11, 14, 15, 18, 19, 22, 23, 26, 27, 30, 31); + let r: i8x16 = transmute(vtrn2q_p8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vtrn2_p16() { + let a: i16x4 = i16x4::new(0, 2, 4, 6); + let b: i16x4 = i16x4::new(1, 3, 5, 7); + let e: i16x4 = i16x4::new(2, 3, 6, 7); + let r: i16x4 = transmute(vtrn2_p16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vtrn2q_p16() { + let a: i16x8 = i16x8::new(0, 2, 4, 6, 8, 10, 12, 14); + let b: i16x8 = i16x8::new(1, 3, 5, 7, 9, 11, 13, 15); + let e: i16x8 = i16x8::new(2, 3, 6, 7, 10, 11, 14, 15); + let r: i16x8 = transmute(vtrn2q_p16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vtrn2_s32() { + let a: i32x2 = i32x2::new(0, 2); + let b: i32x2 = i32x2::new(1, 3); + let e: i32x2 = i32x2::new(2, 3); + let r: i32x2 = transmute(vtrn2_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vtrn2q_s64() { + let a: i64x2 = i64x2::new(0, 2); + let b: i64x2 = i64x2::new(1, 3); + let e: i64x2 = i64x2::new(2, 3); + let r: i64x2 = transmute(vtrn2q_s64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vtrn2_u32() { + let a: u32x2 = u32x2::new(0, 2); + let b: u32x2 = u32x2::new(1, 3); + let e: u32x2 = u32x2::new(2, 3); + let r: u32x2 = transmute(vtrn2_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vtrn2q_u64() { + let a: u64x2 = u64x2::new(0, 2); + let b: u64x2 = u64x2::new(1, 3); + let e: u64x2 = u64x2::new(2, 3); + let r: u64x2 = transmute(vtrn2q_u64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vtrn2q_p64() { + let a: i64x2 = i64x2::new(0, 2); + let b: i64x2 = i64x2::new(1, 3); + let e: i64x2 = i64x2::new(2, 3); + let r: i64x2 = transmute(vtrn2q_p64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vtrn2q_f32() { + let a: f32x4 = f32x4::new(0., 2., 4., 6.); + let b: f32x4 = f32x4::new(1., 3., 5., 7.); + let e: f32x4 = f32x4::new(2., 3., 6., 7.); + let r: f32x4 = transmute(vtrn2q_f32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vtrn2_f32() { + let a: f32x2 = f32x2::new(0., 2.); + let b: f32x2 = f32x2::new(1., 3.); + let e: f32x2 = f32x2::new(2., 3.); + let r: f32x2 = transmute(vtrn2_f32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vtrn2q_f64() { + let a: f64x2 = f64x2::new(0., 2.); + let b: f64x2 = f64x2::new(1., 3.); + let e: f64x2 = f64x2::new(2., 3.); + let r: f64x2 = transmute(vtrn2q_f64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vzip1_s8() { + let a: i8x8 = i8x8::new(0, 2, 4, 6, 8, 10, 12, 14); + let b: i8x8 = i8x8::new(1, 3, 5, 7, 9, 11, 13, 15); + let e: i8x8 = i8x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let r: i8x8 = transmute(vzip1_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vzip1q_s8() { + let a: i8x16 = i8x16::new(0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30); + let b: i8x16 = i8x16::new(1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31); + let e: i8x16 = i8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let r: i8x16 = transmute(vzip1q_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vzip1_s16() { + let a: i16x4 = i16x4::new(0, 2, 4, 6); + let b: i16x4 = i16x4::new(1, 3, 5, 7); + let e: i16x4 = i16x4::new(0, 1, 2, 3); + let r: i16x4 = transmute(vzip1_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vzip1q_s16() { + let a: i16x8 = i16x8::new(0, 2, 4, 6, 8, 10, 12, 14); + let b: i16x8 = i16x8::new(1, 3, 5, 7, 9, 11, 13, 15); + let e: i16x8 = i16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let r: i16x8 = transmute(vzip1q_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vzip1_s32() { + let a: i32x2 = i32x2::new(0, 2); + let b: i32x2 = i32x2::new(1, 3); + let e: i32x2 = i32x2::new(0, 1); + let r: i32x2 = transmute(vzip1_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vzip1q_s32() { + let a: i32x4 = i32x4::new(0, 2, 4, 6); + let b: i32x4 = i32x4::new(1, 3, 5, 7); + let e: i32x4 = i32x4::new(0, 1, 2, 3); + let r: i32x4 = transmute(vzip1q_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vzip1q_s64() { + let a: i64x2 = i64x2::new(0, 2); + let b: i64x2 = i64x2::new(1, 3); + let e: i64x2 = i64x2::new(0, 1); + let r: i64x2 = transmute(vzip1q_s64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vzip1_u8() { + let a: u8x8 = u8x8::new(0, 2, 4, 6, 8, 10, 12, 14); + let b: u8x8 = u8x8::new(1, 3, 5, 7, 9, 11, 13, 15); + let e: u8x8 = u8x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let r: u8x8 = transmute(vzip1_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vzip1q_u8() { + let a: u8x16 = u8x16::new(0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30); + let b: u8x16 = u8x16::new(1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31); + let e: u8x16 = u8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let r: u8x16 = transmute(vzip1q_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vzip1_u16() { + let a: u16x4 = u16x4::new(0, 2, 4, 6); + let b: u16x4 = u16x4::new(1, 3, 5, 7); + let e: u16x4 = u16x4::new(0, 1, 2, 3); + let r: u16x4 = transmute(vzip1_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vzip1q_u16() { + let a: u16x8 = u16x8::new(0, 2, 4, 6, 8, 10, 12, 14); + let b: u16x8 = u16x8::new(1, 3, 5, 7, 9, 11, 13, 15); + let e: u16x8 = u16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let r: u16x8 = transmute(vzip1q_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vzip1_u32() { + let a: u32x2 = u32x2::new(0, 2); + let b: u32x2 = u32x2::new(1, 3); + let e: u32x2 = u32x2::new(0, 1); + let r: u32x2 = transmute(vzip1_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vzip1q_u32() { + let a: u32x4 = u32x4::new(0, 2, 4, 6); + let b: u32x4 = u32x4::new(1, 3, 5, 7); + let e: u32x4 = u32x4::new(0, 1, 2, 3); + let r: u32x4 = transmute(vzip1q_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vzip1q_u64() { + let a: u64x2 = u64x2::new(0, 2); + let b: u64x2 = u64x2::new(1, 3); + let e: u64x2 = u64x2::new(0, 1); + let r: u64x2 = transmute(vzip1q_u64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vzip1_p8() { + let a: i8x8 = i8x8::new(0, 2, 4, 6, 8, 10, 12, 14); + let b: i8x8 = i8x8::new(1, 3, 5, 7, 9, 11, 13, 15); + let e: i8x8 = i8x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let r: i8x8 = transmute(vzip1_p8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vzip1q_p8() { + let a: i8x16 = i8x16::new(0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30); + let b: i8x16 = i8x16::new(1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31); + let e: i8x16 = i8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let r: i8x16 = transmute(vzip1q_p8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vzip1_p16() { + let a: i16x4 = i16x4::new(0, 2, 4, 6); + let b: i16x4 = i16x4::new(1, 3, 5, 7); + let e: i16x4 = i16x4::new(0, 1, 2, 3); + let r: i16x4 = transmute(vzip1_p16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vzip1q_p16() { + let a: i16x8 = i16x8::new(0, 2, 4, 6, 8, 10, 12, 14); + let b: i16x8 = i16x8::new(1, 3, 5, 7, 9, 11, 13, 15); + let e: i16x8 = i16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let r: i16x8 = transmute(vzip1q_p16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vzip1q_p64() { + let a: i64x2 = i64x2::new(0, 2); + let b: i64x2 = i64x2::new(1, 3); + let e: i64x2 = i64x2::new(0, 1); + let r: i64x2 = transmute(vzip1q_p64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vzip1_f32() { + let a: f32x2 = f32x2::new(0., 2.); + let b: f32x2 = f32x2::new(1., 3.); + let e: f32x2 = f32x2::new(0., 1.); + let r: f32x2 = transmute(vzip1_f32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vzip1q_f32() { + let a: f32x4 = f32x4::new(0., 2., 4., 6.); + let b: f32x4 = f32x4::new(1., 3., 5., 7.); + let e: f32x4 = f32x4::new(0., 1., 2., 3.); + let r: f32x4 = transmute(vzip1q_f32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vzip1q_f64() { + let a: f64x2 = f64x2::new(0., 2.); + let b: f64x2 = f64x2::new(1., 3.); + let e: f64x2 = f64x2::new(0., 1.); + let r: f64x2 = transmute(vzip1q_f64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vzip2_s8() { + let a: i8x8 = i8x8::new(0, 16, 16, 18, 16, 18, 20, 22); + let b: i8x8 = i8x8::new(1, 17, 17, 19, 17, 19, 21, 23); + let e: i8x8 = i8x8::new(16, 17, 18, 19, 20, 21, 22, 23); + let r: i8x8 = transmute(vzip2_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vzip2q_s8() { + let a: i8x16 = i8x16::new(0, 16, 16, 18, 16, 18, 20, 22, 16, 18, 20, 22, 24, 26, 28, 30); + let b: i8x16 = i8x16::new(1, 17, 17, 19, 17, 19, 21, 23, 17, 19, 21, 23, 25, 27, 29, 31); + let e: i8x16 = i8x16::new(16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31); + let r: i8x16 = transmute(vzip2q_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vzip2_s16() { + let a: i16x4 = i16x4::new(0, 16, 16, 18); + let b: i16x4 = i16x4::new(1, 17, 17, 19); + let e: i16x4 = i16x4::new(16, 17, 18, 19); + let r: i16x4 = transmute(vzip2_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vzip2q_s16() { + let a: i16x8 = i16x8::new(0, 16, 16, 18, 16, 18, 20, 22); + let b: i16x8 = i16x8::new(1, 17, 17, 19, 17, 19, 21, 23); + let e: i16x8 = i16x8::new(16, 17, 18, 19, 20, 21, 22, 23); + let r: i16x8 = transmute(vzip2q_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vzip2_s32() { + let a: i32x2 = i32x2::new(0, 16); + let b: i32x2 = i32x2::new(1, 17); + let e: i32x2 = i32x2::new(16, 17); + let r: i32x2 = transmute(vzip2_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vzip2q_s32() { + let a: i32x4 = i32x4::new(0, 16, 16, 18); + let b: i32x4 = i32x4::new(1, 17, 17, 19); + let e: i32x4 = i32x4::new(16, 17, 18, 19); + let r: i32x4 = transmute(vzip2q_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vzip2q_s64() { + let a: i64x2 = i64x2::new(0, 16); + let b: i64x2 = i64x2::new(1, 17); + let e: i64x2 = i64x2::new(16, 17); + let r: i64x2 = transmute(vzip2q_s64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vzip2_u8() { + let a: u8x8 = u8x8::new(0, 16, 16, 18, 16, 18, 20, 22); + let b: u8x8 = u8x8::new(1, 17, 17, 19, 17, 19, 21, 23); + let e: u8x8 = u8x8::new(16, 17, 18, 19, 20, 21, 22, 23); + let r: u8x8 = transmute(vzip2_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vzip2q_u8() { + let a: u8x16 = u8x16::new(0, 16, 16, 18, 16, 18, 20, 22, 16, 18, 20, 22, 24, 26, 28, 30); + let b: u8x16 = u8x16::new(1, 17, 17, 19, 17, 19, 21, 23, 17, 19, 21, 23, 25, 27, 29, 31); + let e: u8x16 = u8x16::new(16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31); + let r: u8x16 = transmute(vzip2q_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vzip2_u16() { + let a: u16x4 = u16x4::new(0, 16, 16, 18); + let b: u16x4 = u16x4::new(1, 17, 17, 19); + let e: u16x4 = u16x4::new(16, 17, 18, 19); + let r: u16x4 = transmute(vzip2_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vzip2q_u16() { + let a: u16x8 = u16x8::new(0, 16, 16, 18, 16, 18, 20, 22); + let b: u16x8 = u16x8::new(1, 17, 17, 19, 17, 19, 21, 23); + let e: u16x8 = u16x8::new(16, 17, 18, 19, 20, 21, 22, 23); + let r: u16x8 = transmute(vzip2q_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vzip2_u32() { + let a: u32x2 = u32x2::new(0, 16); + let b: u32x2 = u32x2::new(1, 17); + let e: u32x2 = u32x2::new(16, 17); + let r: u32x2 = transmute(vzip2_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vzip2q_u32() { + let a: u32x4 = u32x4::new(0, 16, 16, 18); + let b: u32x4 = u32x4::new(1, 17, 17, 19); + let e: u32x4 = u32x4::new(16, 17, 18, 19); + let r: u32x4 = transmute(vzip2q_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vzip2q_u64() { + let a: u64x2 = u64x2::new(0, 16); + let b: u64x2 = u64x2::new(1, 17); + let e: u64x2 = u64x2::new(16, 17); + let r: u64x2 = transmute(vzip2q_u64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vzip2_p8() { + let a: i8x8 = i8x8::new(0, 16, 16, 18, 16, 18, 20, 22); + let b: i8x8 = i8x8::new(1, 17, 17, 19, 17, 19, 21, 23); + let e: i8x8 = i8x8::new(16, 17, 18, 19, 20, 21, 22, 23); + let r: i8x8 = transmute(vzip2_p8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vzip2q_p8() { + let a: i8x16 = i8x16::new(0, 16, 16, 18, 16, 18, 20, 22, 16, 18, 20, 22, 24, 26, 28, 30); + let b: i8x16 = i8x16::new(1, 17, 17, 19, 17, 19, 21, 23, 17, 19, 21, 23, 25, 27, 29, 31); + let e: i8x16 = i8x16::new(16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31); + let r: i8x16 = transmute(vzip2q_p8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vzip2_p16() { + let a: i16x4 = i16x4::new(0, 16, 16, 18); + let b: i16x4 = i16x4::new(1, 17, 17, 19); + let e: i16x4 = i16x4::new(16, 17, 18, 19); + let r: i16x4 = transmute(vzip2_p16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vzip2q_p16() { + let a: i16x8 = i16x8::new(0, 16, 16, 18, 16, 18, 20, 22); + let b: i16x8 = i16x8::new(1, 17, 17, 19, 17, 19, 21, 23); + let e: i16x8 = i16x8::new(16, 17, 18, 19, 20, 21, 22, 23); + let r: i16x8 = transmute(vzip2q_p16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vzip2q_p64() { + let a: i64x2 = i64x2::new(0, 16); + let b: i64x2 = i64x2::new(1, 17); + let e: i64x2 = i64x2::new(16, 17); + let r: i64x2 = transmute(vzip2q_p64(transmute(a), transmute(b))); + assert_eq!(r, e); + } #[simd_test(enable = "neon")] - unsafe fn test_vceq_u64() { - let a: u64x1 = u64x1::new(0xFF_FF_FF_FF_FF_FF_FF_FF); - let b: u64x1 = u64x1::new(0xFF_FF_FF_FF_FF_FF_FF_FF); - let e: u64x1 = u64x1::new(0xFF_FF_FF_FF_FF_FF_FF_FF); - let r: u64x1 = transmute(vceq_u64(transmute(a), transmute(b))); + unsafe fn test_vzip2_f32() { + let a: f32x2 = f32x2::new(0., 8.); + let b: f32x2 = f32x2::new(1., 9.); + let e: f32x2 = f32x2::new(8., 9.); + let r: f32x2 = transmute(vzip2_f32(transmute(a), transmute(b))); assert_eq!(r, e); + } - let a: u64x1 = u64x1::new(0xFF_FF_FF_FF_FF_FF_FF_FF); - let b: u64x1 = u64x1::new(0xFF_FF_FF_FF_FF_FF_FF_FF); - let e: u64x1 = u64x1::new(0xFF_FF_FF_FF_FF_FF_FF_FF); - let r: u64x1 = transmute(vceq_u64(transmute(a), transmute(b))); + #[simd_test(enable = "neon")] + unsafe fn test_vzip2q_f32() { + let a: f32x4 = f32x4::new(0., 8., 8., 10.); + let b: f32x4 = f32x4::new(1., 9., 9., 11.); + let e: f32x4 = f32x4::new(8., 9., 10., 11.); + let r: f32x4 = transmute(vzip2q_f32(transmute(a), transmute(b))); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vceqq_u64() { - let a: u64x2 = u64x2::new(0xFF_FF_FF_FF_FF_FF_FF_FF, 0x01); - let b: u64x2 = u64x2::new(0xFF_FF_FF_FF_FF_FF_FF_FF, 0x01); - let e: u64x2 = u64x2::new(0xFF_FF_FF_FF_FF_FF_FF_FF, 0xFF_FF_FF_FF_FF_FF_FF_FF); - let r: u64x2 = transmute(vceqq_u64(transmute(a), transmute(b))); + unsafe fn test_vzip2q_f64() { + let a: f64x2 = f64x2::new(0., 8.); + let b: f64x2 = f64x2::new(1., 9.); + let e: f64x2 = f64x2::new(8., 9.); + let r: f64x2 = transmute(vzip2q_f64(transmute(a), transmute(b))); assert_eq!(r, e); + } - let a: u64x2 = u64x2::new(0xFF_FF_FF_FF_FF_FF_FF_FF, 0xFF_FF_FF_FF_FF_FF_FF_FF); - let b: u64x2 = u64x2::new(0xFF_FF_FF_FF_FF_FF_FF_FF, 0); - let e: u64x2 = u64x2::new(0xFF_FF_FF_FF_FF_FF_FF_FF, 0); - let r: u64x2 = transmute(vceqq_u64(transmute(a), transmute(b))); + #[simd_test(enable = "neon")] + unsafe fn test_vuzp1_s8() { + let a: i8x8 = i8x8::new(1, 0, 2, 0, 2, 0, 3, 0); + let b: i8x8 = i8x8::new(2, 0, 3, 0, 7, 0, 8, 0); + let e: i8x8 = i8x8::new(1, 2, 2, 3, 2, 3, 7, 8); + let r: i8x8 = transmute(vuzp1_s8(transmute(a), transmute(b))); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vceq_s64() { - let a: i64x1 = i64x1::new(0x7F_FF_FF_FF_FF_FF_FF_FF); - let b: i64x1 = i64x1::new(0x7F_FF_FF_FF_FF_FF_FF_FF); - let e: u64x1 = u64x1::new(0xFF_FF_FF_FF_FF_FF_FF_FF); - let r: u64x1 = transmute(vceq_s64(transmute(a), transmute(b))); + unsafe fn test_vuzp1q_s8() { + let a: i8x16 = i8x16::new(1, 0, 2, 0, 2, 0, 3, 0, 2, 0, 3, 0, 7, 0, 8, 0); + let b: i8x16 = i8x16::new(2, 0, 3, 0, 7, 0, 8, 0, 13, 0, 14, 0, 15, 0, 16, 0); + let e: i8x16 = i8x16::new(1, 2, 2, 3, 2, 3, 7, 8, 2, 3, 7, 8, 13, 14, 15, 16); + let r: i8x16 = transmute(vuzp1q_s8(transmute(a), transmute(b))); assert_eq!(r, e); + } - let a: i64x1 = i64x1::new(0x7F_FF_FF_FF_FF_FF_FF_FF); - let b: i64x1 = i64x1::new(0x7F_FF_FF_FF_FF_FF_FF_FF); - let e: u64x1 = u64x1::new(0xFF_FF_FF_FF_FF_FF_FF_FF); - let r: u64x1 = transmute(vceq_s64(transmute(a), transmute(b))); + #[simd_test(enable = "neon")] + unsafe fn test_vuzp1_s16() { + let a: i16x4 = i16x4::new(1, 0, 2, 0); + let b: i16x4 = i16x4::new(2, 0, 3, 0); + let e: i16x4 = i16x4::new(1, 2, 2, 3); + let r: i16x4 = transmute(vuzp1_s16(transmute(a), transmute(b))); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vceqq_s64() { - let a: i64x2 = i64x2::new(0x7F_FF_FF_FF_FF_FF_FF_FF, 0x01); - let b: i64x2 = i64x2::new(0x7F_FF_FF_FF_FF_FF_FF_FF, 0x01); - let e: u64x2 = u64x2::new(0xFF_FF_FF_FF_FF_FF_FF_FF, 0xFF_FF_FF_FF_FF_FF_FF_FF); - let r: u64x2 = transmute(vceqq_s64(transmute(a), transmute(b))); + unsafe fn test_vuzp1q_s16() { + let a: i16x8 = i16x8::new(1, 0, 2, 0, 2, 0, 3, 0); + let b: i16x8 = i16x8::new(2, 0, 3, 0, 7, 0, 8, 0); + let e: i16x8 = i16x8::new(1, 2, 2, 3, 2, 3, 7, 8); + let r: i16x8 = transmute(vuzp1q_s16(transmute(a), transmute(b))); assert_eq!(r, e); + } - let a: i64x2 = i64x2::new(0x7F_FF_FF_FF_FF_FF_FF_FF, 0x7F_FF_FF_FF_FF_FF_FF_FF); - let b: i64x2 = i64x2::new(0x7F_FF_FF_FF_FF_FF_FF_FF, -9223372036854775808); - let e: u64x2 = u64x2::new(0xFF_FF_FF_FF_FF_FF_FF_FF, 0); - let r: u64x2 = transmute(vceqq_s64(transmute(a), transmute(b))); + #[simd_test(enable = "neon")] + unsafe fn test_vuzp1q_s32() { + let a: i32x4 = i32x4::new(1, 0, 2, 0); + let b: i32x4 = i32x4::new(2, 0, 3, 0); + let e: i32x4 = i32x4::new(1, 2, 2, 3); + let r: i32x4 = transmute(vuzp1q_s32(transmute(a), transmute(b))); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vceq_p64() { - let a: i64x1 = i64x1::new(0x7F_FF_FF_FF_FF_FF_FF_FF); - let b: i64x1 = i64x1::new(0x7F_FF_FF_FF_FF_FF_FF_FF); - let e: u64x1 = u64x1::new(0xFF_FF_FF_FF_FF_FF_FF_FF); - let r: u64x1 = transmute(vceq_p64(transmute(a), transmute(b))); + unsafe fn test_vuzp1_u8() { + let a: u8x8 = u8x8::new(1, 0, 2, 0, 2, 0, 3, 0); + let b: u8x8 = u8x8::new(2, 0, 3, 0, 7, 0, 8, 0); + let e: u8x8 = u8x8::new(1, 2, 2, 3, 2, 3, 7, 8); + let r: u8x8 = transmute(vuzp1_u8(transmute(a), transmute(b))); assert_eq!(r, e); + } - let a: i64x1 = i64x1::new(0x7F_FF_FF_FF_FF_FF_FF_FF); - let b: i64x1 = i64x1::new(0x7F_FF_FF_FF_FF_FF_FF_FF); - let e: u64x1 = u64x1::new(0xFF_FF_FF_FF_FF_FF_FF_FF); - let r: u64x1 = transmute(vceq_p64(transmute(a), transmute(b))); + #[simd_test(enable = "neon")] + unsafe fn test_vuzp1q_u8() { + let a: u8x16 = u8x16::new(1, 0, 2, 0, 2, 0, 3, 0, 2, 0, 3, 0, 7, 0, 8, 0); + let b: u8x16 = u8x16::new(2, 0, 3, 0, 7, 0, 8, 0, 13, 0, 14, 0, 15, 0, 16, 0); + let e: u8x16 = u8x16::new(1, 2, 2, 3, 2, 3, 7, 8, 2, 3, 7, 8, 13, 14, 15, 16); + let r: u8x16 = transmute(vuzp1q_u8(transmute(a), transmute(b))); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vceqq_p64() { - let a: i64x2 = i64x2::new(0x7F_FF_FF_FF_FF_FF_FF_FF, 0x01); - let b: i64x2 = i64x2::new(0x7F_FF_FF_FF_FF_FF_FF_FF, 0x01); - let e: u64x2 = u64x2::new(0xFF_FF_FF_FF_FF_FF_FF_FF, 0xFF_FF_FF_FF_FF_FF_FF_FF); - let r: u64x2 = transmute(vceqq_p64(transmute(a), transmute(b))); + unsafe fn test_vuzp1_u16() { + let a: u16x4 = u16x4::new(1, 0, 2, 0); + let b: u16x4 = u16x4::new(2, 0, 3, 0); + let e: u16x4 = u16x4::new(1, 2, 2, 3); + let r: u16x4 = transmute(vuzp1_u16(transmute(a), transmute(b))); assert_eq!(r, e); + } - let a: i64x2 = i64x2::new(0x7F_FF_FF_FF_FF_FF_FF_FF, 0x7F_FF_FF_FF_FF_FF_FF_FF); - let b: i64x2 = i64x2::new(0x7F_FF_FF_FF_FF_FF_FF_FF, -9223372036854775808); - let e: u64x2 = u64x2::new(0xFF_FF_FF_FF_FF_FF_FF_FF, 0); - let r: u64x2 = transmute(vceqq_p64(transmute(a), transmute(b))); + #[simd_test(enable = "neon")] + unsafe fn test_vuzp1q_u16() { + let a: u16x8 = u16x8::new(1, 0, 2, 0, 2, 0, 3, 0); + let b: u16x8 = u16x8::new(2, 0, 3, 0, 7, 0, 8, 0); + let e: u16x8 = u16x8::new(1, 2, 2, 3, 2, 3, 7, 8); + let r: u16x8 = transmute(vuzp1q_u16(transmute(a), transmute(b))); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vceq_f64() { - let a: f64 = 1.2; - let b: f64 = 1.2; - let e: u64x1 = u64x1::new(0xFF_FF_FF_FF_FF_FF_FF_FF); - let r: u64x1 = transmute(vceq_f64(transmute(a), transmute(b))); + unsafe fn test_vuzp1q_u32() { + let a: u32x4 = u32x4::new(1, 0, 2, 0); + let b: u32x4 = u32x4::new(2, 0, 3, 0); + let e: u32x4 = u32x4::new(1, 2, 2, 3); + let r: u32x4 = transmute(vuzp1q_u32(transmute(a), transmute(b))); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vceqq_f64() { - let a: f64x2 = f64x2::new(1.2, 3.4); - let b: f64x2 = f64x2::new(1.2, 3.4); - let e: u64x2 = u64x2::new(0xFF_FF_FF_FF_FF_FF_FF_FF, 0xFF_FF_FF_FF_FF_FF_FF_FF); - let r: u64x2 = transmute(vceqq_f64(transmute(a), transmute(b))); + unsafe fn test_vuzp1_p8() { + let a: i8x8 = i8x8::new(1, 0, 2, 0, 2, 0, 3, 0); + let b: i8x8 = i8x8::new(2, 0, 3, 0, 7, 0, 8, 0); + let e: i8x8 = i8x8::new(1, 2, 2, 3, 2, 3, 7, 8); + let r: i8x8 = transmute(vuzp1_p8(transmute(a), transmute(b))); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vcgt_s64() { - let a: i64x1 = i64x1::new(1); - let b: i64x1 = i64x1::new(0); - let e: u64x1 = u64x1::new(0xFF_FF_FF_FF_FF_FF_FF_FF); - let r: u64x1 = transmute(vcgt_s64(transmute(a), transmute(b))); + unsafe fn test_vuzp1q_p8() { + let a: i8x16 = i8x16::new(1, 0, 2, 0, 2, 0, 3, 0, 2, 0, 3, 0, 7, 0, 8, 0); + let b: i8x16 = i8x16::new(2, 0, 3, 0, 7, 0, 8, 0, 13, 0, 14, 0, 15, 0, 16, 0); + let e: i8x16 = i8x16::new(1, 2, 2, 3, 2, 3, 7, 8, 2, 3, 7, 8, 13, 14, 15, 16); + let r: i8x16 = transmute(vuzp1q_p8(transmute(a), transmute(b))); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vcgtq_s64() { - let a: i64x2 = i64x2::new(1, 2); - let b: i64x2 = i64x2::new(0, 1); - let e: u64x2 = u64x2::new(0xFF_FF_FF_FF_FF_FF_FF_FF, 0xFF_FF_FF_FF_FF_FF_FF_FF); - let r: u64x2 = transmute(vcgtq_s64(transmute(a), transmute(b))); + unsafe fn test_vuzp1_p16() { + let a: i16x4 = i16x4::new(1, 0, 2, 0); + let b: i16x4 = i16x4::new(2, 0, 3, 0); + let e: i16x4 = i16x4::new(1, 2, 2, 3); + let r: i16x4 = transmute(vuzp1_p16(transmute(a), transmute(b))); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vcgt_u64() { - let a: u64x1 = u64x1::new(1); - let b: u64x1 = u64x1::new(0); - let e: u64x1 = u64x1::new(0xFF_FF_FF_FF_FF_FF_FF_FF); - let r: u64x1 = transmute(vcgt_u64(transmute(a), transmute(b))); + unsafe fn test_vuzp1q_p16() { + let a: i16x8 = i16x8::new(1, 0, 2, 0, 2, 0, 3, 0); + let b: i16x8 = i16x8::new(2, 0, 3, 0, 7, 0, 8, 0); + let e: i16x8 = i16x8::new(1, 2, 2, 3, 2, 3, 7, 8); + let r: i16x8 = transmute(vuzp1q_p16(transmute(a), transmute(b))); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vcgtq_u64() { - let a: u64x2 = u64x2::new(1, 2); - let b: u64x2 = u64x2::new(0, 1); - let e: u64x2 = u64x2::new(0xFF_FF_FF_FF_FF_FF_FF_FF, 0xFF_FF_FF_FF_FF_FF_FF_FF); - let r: u64x2 = transmute(vcgtq_u64(transmute(a), transmute(b))); + unsafe fn test_vuzp1_s32() { + let a: i32x2 = i32x2::new(1, 0); + let b: i32x2 = i32x2::new(2, 0); + let e: i32x2 = i32x2::new(1, 2); + let r: i32x2 = transmute(vuzp1_s32(transmute(a), transmute(b))); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vcgt_f64() { - let a: f64 = 1.2; - let b: f64 = 0.1; - let e: u64x1 = u64x1::new(0xFF_FF_FF_FF_FF_FF_FF_FF); - let r: u64x1 = transmute(vcgt_f64(transmute(a), transmute(b))); + unsafe fn test_vuzp1q_s64() { + let a: i64x2 = i64x2::new(1, 0); + let b: i64x2 = i64x2::new(2, 0); + let e: i64x2 = i64x2::new(1, 2); + let r: i64x2 = transmute(vuzp1q_s64(transmute(a), transmute(b))); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vcgtq_f64() { - let a: f64x2 = f64x2::new(1.2, 2.3); - let b: f64x2 = f64x2::new(0.1, 1.2); - let e: u64x2 = u64x2::new(0xFF_FF_FF_FF_FF_FF_FF_FF, 0xFF_FF_FF_FF_FF_FF_FF_FF); - let r: u64x2 = transmute(vcgtq_f64(transmute(a), transmute(b))); + unsafe fn test_vuzp1_u32() { + let a: u32x2 = u32x2::new(1, 0); + let b: u32x2 = u32x2::new(2, 0); + let e: u32x2 = u32x2::new(1, 2); + let r: u32x2 = transmute(vuzp1_u32(transmute(a), transmute(b))); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vclt_s64() { - let a: i64x1 = i64x1::new(0); - let b: i64x1 = i64x1::new(1); - let e: u64x1 = u64x1::new(0xFF_FF_FF_FF_FF_FF_FF_FF); - let r: u64x1 = transmute(vclt_s64(transmute(a), transmute(b))); + unsafe fn test_vuzp1q_u64() { + let a: u64x2 = u64x2::new(1, 0); + let b: u64x2 = u64x2::new(2, 0); + let e: u64x2 = u64x2::new(1, 2); + let r: u64x2 = transmute(vuzp1q_u64(transmute(a), transmute(b))); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vcltq_s64() { - let a: i64x2 = i64x2::new(0, 1); - let b: i64x2 = i64x2::new(1, 2); - let e: u64x2 = u64x2::new(0xFF_FF_FF_FF_FF_FF_FF_FF, 0xFF_FF_FF_FF_FF_FF_FF_FF); - let r: u64x2 = transmute(vcltq_s64(transmute(a), transmute(b))); + unsafe fn test_vuzp1q_p64() { + let a: i64x2 = i64x2::new(1, 0); + let b: i64x2 = i64x2::new(2, 0); + let e: i64x2 = i64x2::new(1, 2); + let r: i64x2 = transmute(vuzp1q_p64(transmute(a), transmute(b))); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vclt_u64() { - let a: u64x1 = u64x1::new(0); - let b: u64x1 = u64x1::new(1); - let e: u64x1 = u64x1::new(0xFF_FF_FF_FF_FF_FF_FF_FF); - let r: u64x1 = transmute(vclt_u64(transmute(a), transmute(b))); + unsafe fn test_vuzp1q_f32() { + let a: f32x4 = f32x4::new(0., 8., 1., 9.); + let b: f32x4 = f32x4::new(1., 10., 3., 11.); + let e: f32x4 = f32x4::new(0., 1., 1., 3.); + let r: f32x4 = transmute(vuzp1q_f32(transmute(a), transmute(b))); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vcltq_u64() { - let a: u64x2 = u64x2::new(0, 1); - let b: u64x2 = u64x2::new(1, 2); - let e: u64x2 = u64x2::new(0xFF_FF_FF_FF_FF_FF_FF_FF, 0xFF_FF_FF_FF_FF_FF_FF_FF); - let r: u64x2 = transmute(vcltq_u64(transmute(a), transmute(b))); + unsafe fn test_vuzp1_f32() { + let a: f32x2 = f32x2::new(0., 8.); + let b: f32x2 = f32x2::new(1., 10.); + let e: f32x2 = f32x2::new(0., 1.); + let r: f32x2 = transmute(vuzp1_f32(transmute(a), transmute(b))); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vclt_f64() { - let a: f64 = 0.1; - let b: f64 = 1.2; - let e: u64x1 = u64x1::new(0xFF_FF_FF_FF_FF_FF_FF_FF); - let r: u64x1 = transmute(vclt_f64(transmute(a), transmute(b))); + unsafe fn test_vuzp1q_f64() { + let a: f64x2 = f64x2::new(0., 8.); + let b: f64x2 = f64x2::new(1., 10.); + let e: f64x2 = f64x2::new(0., 1.); + let r: f64x2 = transmute(vuzp1q_f64(transmute(a), transmute(b))); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vcltq_f64() { - let a: f64x2 = f64x2::new(0.1, 1.2); - let b: f64x2 = f64x2::new(1.2, 2.3); - let e: u64x2 = u64x2::new(0xFF_FF_FF_FF_FF_FF_FF_FF, 0xFF_FF_FF_FF_FF_FF_FF_FF); - let r: u64x2 = transmute(vcltq_f64(transmute(a), transmute(b))); + unsafe fn test_vuzp2_s8() { + let a: i8x8 = i8x8::new(0, 17, 0, 18, 0, 18, 0, 19); + let b: i8x8 = i8x8::new(0, 18, 0, 19, 0, 23, 0, 24); + let e: i8x8 = i8x8::new(17, 18, 18, 19, 18, 19, 23, 24); + let r: i8x8 = transmute(vuzp2_s8(transmute(a), transmute(b))); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vcle_s64() { - let a: i64x1 = i64x1::new(0); - let b: i64x1 = i64x1::new(1); - let e: u64x1 = u64x1::new(0xFF_FF_FF_FF_FF_FF_FF_FF); - let r: u64x1 = transmute(vcle_s64(transmute(a), transmute(b))); + unsafe fn test_vuzp2q_s8() { + let a: i8x16 = i8x16::new(0, 17, 0, 18, 0, 18, 0, 19, 0, 18, 0, 19, 0, 23, 0, 24); + let b: i8x16 = i8x16::new(0, 18, 0, 19, 0, 23, 0, 24, 0, 29, 0, 30, 0, 31, 0, 32); + let e: i8x16 = i8x16::new(17, 18, 18, 19, 18, 19, 23, 24, 18, 19, 23, 24, 29, 30, 31, 32); + let r: i8x16 = transmute(vuzp2q_s8(transmute(a), transmute(b))); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vcleq_s64() { - let a: i64x2 = i64x2::new(0, 1); - let b: i64x2 = i64x2::new(1, 2); - let e: u64x2 = u64x2::new(0xFF_FF_FF_FF_FF_FF_FF_FF, 0xFF_FF_FF_FF_FF_FF_FF_FF); - let r: u64x2 = transmute(vcleq_s64(transmute(a), transmute(b))); + unsafe fn test_vuzp2_s16() { + let a: i16x4 = i16x4::new(0, 17, 0, 18); + let b: i16x4 = i16x4::new(0, 18, 0, 19); + let e: i16x4 = i16x4::new(17, 18, 18, 19); + let r: i16x4 = transmute(vuzp2_s16(transmute(a), transmute(b))); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vcle_u64() { - let a: u64x1 = u64x1::new(0); - let b: u64x1 = u64x1::new(1); - let e: u64x1 = u64x1::new(0xFF_FF_FF_FF_FF_FF_FF_FF); - let r: u64x1 = transmute(vcle_u64(transmute(a), transmute(b))); + unsafe fn test_vuzp2q_s16() { + let a: i16x8 = i16x8::new(0, 17, 0, 18, 0, 18, 0, 19); + let b: i16x8 = i16x8::new(0, 18, 0, 19, 0, 23, 0, 24); + let e: i16x8 = i16x8::new(17, 18, 18, 19, 18, 19, 23, 24); + let r: i16x8 = transmute(vuzp2q_s16(transmute(a), transmute(b))); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vcleq_u64() { - let a: u64x2 = u64x2::new(0, 1); - let b: u64x2 = u64x2::new(1, 2); - let e: u64x2 = u64x2::new(0xFF_FF_FF_FF_FF_FF_FF_FF, 0xFF_FF_FF_FF_FF_FF_FF_FF); - let r: u64x2 = transmute(vcleq_u64(transmute(a), transmute(b))); + unsafe fn test_vuzp2q_s32() { + let a: i32x4 = i32x4::new(0, 17, 0, 18); + let b: i32x4 = i32x4::new(0, 18, 0, 19); + let e: i32x4 = i32x4::new(17, 18, 18, 19); + let r: i32x4 = transmute(vuzp2q_s32(transmute(a), transmute(b))); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vcle_f64() { - let a: f64 = 0.1; - let b: f64 = 1.2; - let e: u64x1 = u64x1::new(0xFF_FF_FF_FF_FF_FF_FF_FF); - let r: u64x1 = transmute(vcle_f64(transmute(a), transmute(b))); + unsafe fn test_vuzp2_u8() { + let a: u8x8 = u8x8::new(0, 17, 0, 18, 0, 18, 0, 19); + let b: u8x8 = u8x8::new(0, 18, 0, 19, 0, 23, 0, 24); + let e: u8x8 = u8x8::new(17, 18, 18, 19, 18, 19, 23, 24); + let r: u8x8 = transmute(vuzp2_u8(transmute(a), transmute(b))); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vcleq_f64() { - let a: f64x2 = f64x2::new(0.1, 1.2); - let b: f64x2 = f64x2::new(1.2, 2.3); - let e: u64x2 = u64x2::new(0xFF_FF_FF_FF_FF_FF_FF_FF, 0xFF_FF_FF_FF_FF_FF_FF_FF); - let r: u64x2 = transmute(vcleq_f64(transmute(a), transmute(b))); + unsafe fn test_vuzp2q_u8() { + let a: u8x16 = u8x16::new(0, 17, 0, 18, 0, 18, 0, 19, 0, 18, 0, 19, 0, 23, 0, 24); + let b: u8x16 = u8x16::new(0, 18, 0, 19, 0, 23, 0, 24, 0, 29, 0, 30, 0, 31, 0, 32); + let e: u8x16 = u8x16::new(17, 18, 18, 19, 18, 19, 23, 24, 18, 19, 23, 24, 29, 30, 31, 32); + let r: u8x16 = transmute(vuzp2q_u8(transmute(a), transmute(b))); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vcge_s64() { - let a: i64x1 = i64x1::new(1); - let b: i64x1 = i64x1::new(0); - let e: u64x1 = u64x1::new(0xFF_FF_FF_FF_FF_FF_FF_FF); - let r: u64x1 = transmute(vcge_s64(transmute(a), transmute(b))); + unsafe fn test_vuzp2_u16() { + let a: u16x4 = u16x4::new(0, 17, 0, 18); + let b: u16x4 = u16x4::new(0, 18, 0, 19); + let e: u16x4 = u16x4::new(17, 18, 18, 19); + let r: u16x4 = transmute(vuzp2_u16(transmute(a), transmute(b))); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vcgeq_s64() { - let a: i64x2 = i64x2::new(1, 2); - let b: i64x2 = i64x2::new(0, 1); - let e: u64x2 = u64x2::new(0xFF_FF_FF_FF_FF_FF_FF_FF, 0xFF_FF_FF_FF_FF_FF_FF_FF); - let r: u64x2 = transmute(vcgeq_s64(transmute(a), transmute(b))); + unsafe fn test_vuzp2q_u16() { + let a: u16x8 = u16x8::new(0, 17, 0, 18, 0, 18, 0, 19); + let b: u16x8 = u16x8::new(0, 18, 0, 19, 0, 23, 0, 24); + let e: u16x8 = u16x8::new(17, 18, 18, 19, 18, 19, 23, 24); + let r: u16x8 = transmute(vuzp2q_u16(transmute(a), transmute(b))); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vcge_u64() { - let a: u64x1 = u64x1::new(1); - let b: u64x1 = u64x1::new(0); - let e: u64x1 = u64x1::new(0xFF_FF_FF_FF_FF_FF_FF_FF); - let r: u64x1 = transmute(vcge_u64(transmute(a), transmute(b))); + unsafe fn test_vuzp2q_u32() { + let a: u32x4 = u32x4::new(0, 17, 0, 18); + let b: u32x4 = u32x4::new(0, 18, 0, 19); + let e: u32x4 = u32x4::new(17, 18, 18, 19); + let r: u32x4 = transmute(vuzp2q_u32(transmute(a), transmute(b))); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vcgeq_u64() { - let a: u64x2 = u64x2::new(1, 2); - let b: u64x2 = u64x2::new(0, 1); - let e: u64x2 = u64x2::new(0xFF_FF_FF_FF_FF_FF_FF_FF, 0xFF_FF_FF_FF_FF_FF_FF_FF); - let r: u64x2 = transmute(vcgeq_u64(transmute(a), transmute(b))); + unsafe fn test_vuzp2_p8() { + let a: i8x8 = i8x8::new(0, 17, 0, 18, 0, 18, 0, 19); + let b: i8x8 = i8x8::new(0, 18, 0, 19, 0, 23, 0, 24); + let e: i8x8 = i8x8::new(17, 18, 18, 19, 18, 19, 23, 24); + let r: i8x8 = transmute(vuzp2_p8(transmute(a), transmute(b))); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vcge_f64() { - let a: f64 = 1.2; - let b: f64 = 0.1; - let e: u64x1 = u64x1::new(0xFF_FF_FF_FF_FF_FF_FF_FF); - let r: u64x1 = transmute(vcge_f64(transmute(a), transmute(b))); + unsafe fn test_vuzp2q_p8() { + let a: i8x16 = i8x16::new(0, 17, 0, 18, 0, 18, 0, 19, 0, 18, 0, 19, 0, 23, 0, 24); + let b: i8x16 = i8x16::new(0, 18, 0, 19, 0, 23, 0, 24, 0, 29, 0, 30, 0, 31, 0, 32); + let e: i8x16 = i8x16::new(17, 18, 18, 19, 18, 19, 23, 24, 18, 19, 23, 24, 29, 30, 31, 32); + let r: i8x16 = transmute(vuzp2q_p8(transmute(a), transmute(b))); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vcgeq_f64() { - let a: f64x2 = f64x2::new(1.2, 2.3); - let b: f64x2 = f64x2::new(0.1, 1.2); - let e: u64x2 = u64x2::new(0xFF_FF_FF_FF_FF_FF_FF_FF, 0xFF_FF_FF_FF_FF_FF_FF_FF); - let r: u64x2 = transmute(vcgeq_f64(transmute(a), transmute(b))); + unsafe fn test_vuzp2_p16() { + let a: i16x4 = i16x4::new(0, 17, 0, 18); + let b: i16x4 = i16x4::new(0, 18, 0, 19); + let e: i16x4 = i16x4::new(17, 18, 18, 19); + let r: i16x4 = transmute(vuzp2_p16(transmute(a), transmute(b))); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vmul_f64() { - let a: f64 = 1.0; - let b: f64 = 2.0; - let e: f64 = 2.0; - let r: f64 = transmute(vmul_f64(transmute(a), transmute(b))); + unsafe fn test_vuzp2q_p16() { + let a: i16x8 = i16x8::new(0, 17, 0, 18, 0, 18, 0, 19); + let b: i16x8 = i16x8::new(0, 18, 0, 19, 0, 23, 0, 24); + let e: i16x8 = i16x8::new(17, 18, 18, 19, 18, 19, 23, 24); + let r: i16x8 = transmute(vuzp2q_p16(transmute(a), transmute(b))); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vmulq_f64() { - let a: f64x2 = f64x2::new(1.0, 2.0); - let b: f64x2 = f64x2::new(2.0, 3.0); - let e: f64x2 = f64x2::new(2.0, 6.0); - let r: f64x2 = transmute(vmulq_f64(transmute(a), transmute(b))); + unsafe fn test_vuzp2_s32() { + let a: i32x2 = i32x2::new(0, 17); + let b: i32x2 = i32x2::new(0, 18); + let e: i32x2 = i32x2::new(17, 18); + let r: i32x2 = transmute(vuzp2_s32(transmute(a), transmute(b))); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vsub_f64() { - let a: f64 = 1.0; - let b: f64 = 1.0; - let e: f64 = 0.0; - let r: f64 = transmute(vsub_f64(transmute(a), transmute(b))); + unsafe fn test_vuzp2q_s64() { + let a: i64x2 = i64x2::new(0, 17); + let b: i64x2 = i64x2::new(0, 18); + let e: i64x2 = i64x2::new(17, 18); + let r: i64x2 = transmute(vuzp2q_s64(transmute(a), transmute(b))); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vsubq_f64() { - let a: f64x2 = f64x2::new(1.0, 4.0); - let b: f64x2 = f64x2::new(1.0, 2.0); - let e: f64x2 = f64x2::new(0.0, 2.0); - let r: f64x2 = transmute(vsubq_f64(transmute(a), transmute(b))); + unsafe fn test_vuzp2_u32() { + let a: u32x2 = u32x2::new(0, 17); + let b: u32x2 = u32x2::new(0, 18); + let e: u32x2 = u32x2::new(17, 18); + let r: u32x2 = transmute(vuzp2_u32(transmute(a), transmute(b))); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vmax_f64() { - let a: f64 = 1.0; - let b: f64 = 0.0; - let e: f64 = 1.0; - let r: f64 = transmute(vmax_f64(transmute(a), transmute(b))); + unsafe fn test_vuzp2q_u64() { + let a: u64x2 = u64x2::new(0, 17); + let b: u64x2 = u64x2::new(0, 18); + let e: u64x2 = u64x2::new(17, 18); + let r: u64x2 = transmute(vuzp2q_u64(transmute(a), transmute(b))); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vmaxq_f64() { - let a: f64x2 = f64x2::new(1.0, -2.0); - let b: f64x2 = f64x2::new(0.0, 3.0); - let e: f64x2 = f64x2::new(1.0, 3.0); - let r: f64x2 = transmute(vmaxq_f64(transmute(a), transmute(b))); + unsafe fn test_vuzp2q_p64() { + let a: i64x2 = i64x2::new(0, 17); + let b: i64x2 = i64x2::new(0, 18); + let e: i64x2 = i64x2::new(17, 18); + let r: i64x2 = transmute(vuzp2q_p64(transmute(a), transmute(b))); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vmin_f64() { - let a: f64 = 1.0; - let b: f64 = 0.0; - let e: f64 = 0.0; - let r: f64 = transmute(vmin_f64(transmute(a), transmute(b))); + unsafe fn test_vuzp2q_f32() { + let a: f32x4 = f32x4::new(0., 8., 1., 9.); + let b: f32x4 = f32x4::new(2., 9., 3., 11.); + let e: f32x4 = f32x4::new(8., 9., 9., 11.); + let r: f32x4 = transmute(vuzp2q_f32(transmute(a), transmute(b))); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vminq_f64() { - let a: f64x2 = f64x2::new(1.0, -2.0); - let b: f64x2 = f64x2::new(0.0, 3.0); - let e: f64x2 = f64x2::new(0.0, -2.0); - let r: f64x2 = transmute(vminq_f64(transmute(a), transmute(b))); + unsafe fn test_vuzp2_f32() { + let a: f32x2 = f32x2::new(0., 8.); + let b: f32x2 = f32x2::new(2., 9.); + let e: f32x2 = f32x2::new(8., 9.); + let r: f32x2 = transmute(vuzp2_f32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vuzp2q_f64() { + let a: f64x2 = f64x2::new(0., 8.); + let b: f64x2 = f64x2::new(2., 9.); + let e: f64x2 = f64x2::new(8., 9.); + let r: f64x2 = transmute(vuzp2q_f64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vabal_high_u8() { + let a: u16x8 = u16x8::new(9, 10, 11, 12, 13, 14, 15, 16); + let b: u8x16 = u8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let c: u8x16 = u8x16::new(10, 10, 10, 10, 10, 10, 10, 10, 20, 0, 2, 4, 6, 8, 10, 12); + let e: u16x8 = u16x8::new(20, 20, 20, 20, 20, 20, 20, 20); + let r: u16x8 = transmute(vabal_high_u8(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vabal_high_u16() { + let a: u32x4 = u32x4::new(9, 10, 11, 12); + let b: u16x8 = u16x8::new(1, 2, 3, 4, 9, 10, 11, 12); + let c: u16x8 = u16x8::new(10, 10, 10, 10, 20, 0, 2, 4); + let e: u32x4 = u32x4::new(20, 20, 20, 20); + let r: u32x4 = transmute(vabal_high_u16(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vabal_high_u32() { + let a: u64x2 = u64x2::new(15, 16); + let b: u32x4 = u32x4::new(1, 2, 15, 16); + let c: u32x4 = u32x4::new(10, 10, 10, 12); + let e: u64x2 = u64x2::new(20, 20); + let r: u64x2 = transmute(vabal_high_u32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vabal_high_s8() { + let a: i16x8 = i16x8::new(9, 10, 11, 12, 13, 14, 15, 16); + let b: i8x16 = i8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let c: i8x16 = i8x16::new(10, 10, 10, 10, 10, 10, 10, 10, 20, 0, 2, 4, 6, 8, 10, 12); + let e: i16x8 = i16x8::new(20, 20, 20, 20, 20, 20, 20, 20); + let r: i16x8 = transmute(vabal_high_s8(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vabal_high_s16() { + let a: i32x4 = i32x4::new(9, 10, 11, 12); + let b: i16x8 = i16x8::new(1, 2, 3, 4, 9, 10, 11, 12); + let c: i16x8 = i16x8::new(10, 10, 10, 10, 20, 0, 2, 4); + let e: i32x4 = i32x4::new(20, 20, 20, 20); + let r: i32x4 = transmute(vabal_high_s16(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vabal_high_s32() { + let a: i64x2 = i64x2::new(15, 16); + let b: i32x4 = i32x4::new(1, 2, 15, 16); + let c: i32x4 = i32x4::new(10, 10, 10, 12); + let e: i64x2 = i64x2::new(20, 20); + let r: i64x2 = transmute(vabal_high_s32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqabs_s64() { + let a: i64x1 = i64x1::new(-9223372036854775808); + let e: i64x1 = i64x1::new(0x7F_FF_FF_FF_FF_FF_FF_FF); + let r: i64x1 = transmute(vqabs_s64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqabsq_s64() { + let a: i64x2 = i64x2::new(-9223372036854775808, -7); + let e: i64x2 = i64x2::new(0x7F_FF_FF_FF_FF_FF_FF_FF, 7); + let r: i64x2 = transmute(vqabsq_s64(transmute(a))); assert_eq!(r, e); } } diff --git a/library/stdarch/crates/core_arch/src/aarch64/neon/mod.rs b/library/stdarch/crates/core_arch/src/aarch64/neon/mod.rs index 227d227f4e..95aea69ef7 100644 --- a/library/stdarch/crates/core_arch/src/aarch64/neon/mod.rs +++ b/library/stdarch/crates/core_arch/src/aarch64/neon/mod.rs @@ -2,14 +2,18 @@ #![allow(non_camel_case_types)] +#[rustfmt::skip] mod generated; +#[rustfmt::skip] pub use self::generated::*; // FIXME: replace neon with asimd use crate::{ - core_arch::{arm::*, simd::*, simd_llvm::*}, - mem::{transmute, zeroed}, + core_arch::{arm_shared::*, simd::*, simd_llvm::*}, + hint::unreachable_unchecked, + mem::{size_of, transmute, zeroed}, + ptr::copy_nonoverlapping, }; #[cfg(test)] use stdarch_test::assert_instr; @@ -105,9 +109,6 @@ extern "C" { #[link_name = "llvm.aarch64.neon.usqadd.v2i64"] fn vsqaddq_u64_(a: uint64x2_t, b: int64x2_t) -> uint64x2_t; - #[link_name = "llvm.aarch64.neon.pmull64"] - fn vmull_p64_(a: i64, b: i64) -> int8x16_t; - #[link_name = "llvm.aarch64.neon.addp.v8i16"] fn vpaddq_s16_(a: int16x8_t, b: int16x8_t) -> int16x8_t; #[link_name = "llvm.aarch64.neon.addp.v4i32"] @@ -144,6 +145,15 @@ extern "C" { #[link_name = "llvm.aarch64.neon.uaddv.i64.v2i64"] fn vaddvq_u64_(a: uint64x2_t) -> u64; + #[link_name = "llvm.aarch64.neon.saddlv.i32.v8i8"] + fn vaddlv_s8_(a: int8x8_t) -> i32; + #[link_name = "llvm.aarch64.neon.uaddlv.i32.v8i8"] + fn vaddlv_u8_(a: uint8x8_t) -> u32; + #[link_name = "llvm.aarch64.neon.saddlv.i32.v16i8"] + fn vaddlvq_s8_(a: int8x16_t) -> i32; + #[link_name = "llvm.aarch64.neon.uaddlv.i32.v16i8"] + fn vaddlvq_u8_(a: uint8x16_t) -> u32; + #[link_name = "llvm.aarch64.neon.smaxv.i8.v8i8"] fn vmaxv_s8_(a: int8x8_t) -> i8; #[link_name = "llvm.aarch64.neon.smaxv.i8.6i8"] @@ -312,11 +322,6 @@ extern "C" { c: uint8x16_t, ) -> int8x16_t; - #[link_name = "llvm.aarch64.neon.fcvtzu.v4i32.v4f32"] - fn vcvtq_u32_f32_(a: float32x4_t) -> uint32x4_t; - #[link_name = "llvm.aarch64.neon.fcvtzs.v4i32.v4f32"] - fn vcvtq_s32_f32_(a: float32x4_t) -> int32x4_t; - #[link_name = "llvm.aarch64.neon.vsli.v8i8"] fn vsli_n_s8_(a: int8x8_t, b: int8x8_t, n: i32) -> int8x8_t; #[link_name = "llvm.aarch64.neon.vsli.v16i8"] @@ -352,6 +357,118 @@ extern "C" { fn vsriq_n_s64_(a: int64x2_t, b: int64x2_t, n: i32) -> int64x2_t; } +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop, N1 = 0, N2 = 0))] +#[rustc_legacy_const_generics(1, 3)] +pub unsafe fn vcopy_lane_s64( + _a: int64x1_t, + b: int64x1_t, +) -> int64x1_t { + static_assert!(N1 : i32 where N1 == 0); + static_assert!(N2 : i32 where N2 == 0); + b +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop, N1 = 0, N2 = 0))] +#[rustc_legacy_const_generics(1, 3)] +pub unsafe fn vcopy_lane_u64( + _a: uint64x1_t, + b: uint64x1_t, +) -> uint64x1_t { + static_assert!(N1 : i32 where N1 == 0); + static_assert!(N2 : i32 where N2 == 0); + b +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop, N1 = 0, N2 = 0))] +#[rustc_legacy_const_generics(1, 3)] +pub unsafe fn vcopy_lane_p64( + _a: poly64x1_t, + b: poly64x1_t, +) -> poly64x1_t { + static_assert!(N1 : i32 where N1 == 0); + static_assert!(N2 : i32 where N2 == 0); + b +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop, N1 = 0, N2 = 0))] +#[rustc_legacy_const_generics(1, 3)] +pub unsafe fn vcopy_lane_f64( + _a: float64x1_t, + b: float64x1_t, +) -> float64x1_t { + static_assert!(N1 : i32 where N1 == 0); + static_assert!(N2 : i32 where N2 == 0); + b +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop, LANE1 = 0, LANE2 = 1))] +#[rustc_legacy_const_generics(1, 3)] +pub unsafe fn vcopy_laneq_s64( + _a: int64x1_t, + b: int64x2_t, +) -> int64x1_t { + static_assert!(LANE1 : i32 where LANE1 == 0); + static_assert_imm1!(LANE2); + transmute::(simd_extract(b, LANE2 as u32)) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop, LANE1 = 0, LANE2 = 1))] +#[rustc_legacy_const_generics(1, 3)] +pub unsafe fn vcopy_laneq_u64( + _a: uint64x1_t, + b: uint64x2_t, +) -> uint64x1_t { + static_assert!(LANE1 : i32 where LANE1 == 0); + static_assert_imm1!(LANE2); + transmute::(simd_extract(b, LANE2 as u32)) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop, LANE1 = 0, LANE2 = 1))] +#[rustc_legacy_const_generics(1, 3)] +pub unsafe fn vcopy_laneq_p64( + _a: poly64x1_t, + b: poly64x2_t, +) -> poly64x1_t { + static_assert!(LANE1 : i32 where LANE1 == 0); + static_assert_imm1!(LANE2); + transmute::(simd_extract(b, LANE2 as u32)) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop, LANE1 = 0, LANE2 = 1))] +#[rustc_legacy_const_generics(1, 3)] +pub unsafe fn vcopy_laneq_f64( + _a: float64x1_t, + b: float64x2_t, +) -> float64x1_t { + static_assert!(LANE1 : i32 where LANE1 == 0); + static_assert_imm1!(LANE2); + transmute::(simd_extract(b, LANE2 as u32)) +} + /// Load multiple single-element structures to one, two, three, or four registers. #[inline] #[target_feature(enable = "neon")] @@ -679,6 +796,344 @@ pub unsafe fn vld1q_f64(ptr: *const f64) -> float64x2_t { transmute(f64x2::new(*ptr, *ptr.offset(1))) } +/// Store multiple single-element structures from one, two, three, or four registers. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(str))] +#[allow(clippy::cast_ptr_alignment)] +pub unsafe fn vst1_s8(ptr: *mut i8, a: int8x8_t) { + copy_nonoverlapping( + &a as *const int8x8_t as *const i8, + ptr as *mut i8, + size_of::(), + ) +} + +/// Store multiple single-element structures from one, two, three, or four registers. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(str))] +#[allow(clippy::cast_ptr_alignment)] +pub unsafe fn vst1q_s8(ptr: *mut i8, a: int8x16_t) { + copy_nonoverlapping( + &a as *const int8x16_t as *const i8, + ptr as *mut i8, + size_of::(), + ) +} + +/// Store multiple single-element structures from one, two, three, or four registers. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(str))] +#[allow(clippy::cast_ptr_alignment)] +pub unsafe fn vst1_s16(ptr: *mut i16, a: int16x4_t) { + copy_nonoverlapping( + &a as *const int16x4_t as *const i16, + ptr as *mut i16, + size_of::(), + ) +} + +/// Store multiple single-element structures from one, two, three, or four registers. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(str))] +#[allow(clippy::cast_ptr_alignment)] +pub unsafe fn vst1q_s16(ptr: *mut i16, a: int16x8_t) { + copy_nonoverlapping( + &a as *const int16x8_t as *const i16, + ptr as *mut i16, + size_of::(), + ) +} + +/// Store multiple single-element structures from one, two, three, or four registers. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(str))] +#[allow(clippy::cast_ptr_alignment)] +pub unsafe fn vst1_s32(ptr: *mut i32, a: int32x2_t) { + copy_nonoverlapping( + &a as *const int32x2_t as *const i32, + ptr as *mut i32, + size_of::(), + ) +} + +/// Store multiple single-element structures from one, two, three, or four registers. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(str))] +#[allow(clippy::cast_ptr_alignment)] +pub unsafe fn vst1q_s32(ptr: *mut i32, a: int32x4_t) { + copy_nonoverlapping( + &a as *const int32x4_t as *const i32, + ptr as *mut i32, + size_of::(), + ) +} + +/// Store multiple single-element structures from one, two, three, or four registers. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(str))] +#[allow(clippy::cast_ptr_alignment)] +pub unsafe fn vst1_s64(ptr: *mut i64, a: int64x1_t) { + copy_nonoverlapping( + &a as *const int64x1_t as *const i64, + ptr as *mut i64, + size_of::(), + ) +} + +/// Store multiple single-element structures from one, two, three, or four registers. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(str))] +#[allow(clippy::cast_ptr_alignment)] +pub unsafe fn vst1q_s64(ptr: *mut i64, a: int64x2_t) { + copy_nonoverlapping( + &a as *const int64x2_t as *const i64, + ptr as *mut i64, + size_of::(), + ) +} + +/// Store multiple single-element structures from one, two, three, or four registers. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(str))] +#[allow(clippy::cast_ptr_alignment)] +pub unsafe fn vst1_u8(ptr: *mut u8, a: uint8x8_t) { + copy_nonoverlapping( + &a as *const uint8x8_t as *const u8, + ptr as *mut u8, + size_of::(), + ) +} + +/// Store multiple single-element structures from one, two, three, or four registers. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(str))] +#[allow(clippy::cast_ptr_alignment)] +pub unsafe fn vst1q_u8(ptr: *mut u8, a: uint8x16_t) { + copy_nonoverlapping( + &a as *const uint8x16_t as *const u8, + ptr as *mut u8, + size_of::(), + ) +} + +/// Store multiple single-element structures from one, two, three, or four registers. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(str))] +#[allow(clippy::cast_ptr_alignment)] +pub unsafe fn vst1_u16(ptr: *mut u16, a: uint16x4_t) { + copy_nonoverlapping( + &a as *const uint16x4_t as *const u16, + ptr as *mut u16, + size_of::(), + ) +} + +/// Store multiple single-element structures from one, two, three, or four registers. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(str))] +#[allow(clippy::cast_ptr_alignment)] +pub unsafe fn vst1q_u16(ptr: *mut u16, a: uint16x8_t) { + copy_nonoverlapping( + &a as *const uint16x8_t as *const u16, + ptr as *mut u16, + size_of::(), + ) +} + +/// Store multiple single-element structures from one, two, three, or four registers. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(str))] +#[allow(clippy::cast_ptr_alignment)] +pub unsafe fn vst1_u32(ptr: *mut u32, a: uint32x2_t) { + copy_nonoverlapping( + &a as *const uint32x2_t as *const u32, + ptr as *mut u32, + size_of::(), + ) +} + +/// Store multiple single-element structures from one, two, three, or four registers. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(str))] +#[allow(clippy::cast_ptr_alignment)] +pub unsafe fn vst1q_u32(ptr: *mut u32, a: uint32x4_t) { + copy_nonoverlapping( + &a as *const uint32x4_t as *const u32, + ptr as *mut u32, + size_of::(), + ) +} + +/// Store multiple single-element structures from one, two, three, or four registers. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(str))] +#[allow(clippy::cast_ptr_alignment)] +pub unsafe fn vst1_u64(ptr: *mut u64, a: uint64x1_t) { + copy_nonoverlapping( + &a as *const uint64x1_t as *const u64, + ptr as *mut u64, + size_of::(), + ) +} + +/// Store multiple single-element structures from one, two, three, or four registers. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(str))] +#[allow(clippy::cast_ptr_alignment)] +pub unsafe fn vst1q_u64(ptr: *mut u64, a: uint64x2_t) { + copy_nonoverlapping( + &a as *const uint64x2_t as *const u64, + ptr as *mut u64, + size_of::(), + ) +} + +/// Store multiple single-element structures from one, two, three, or four registers. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(str))] +#[allow(clippy::cast_ptr_alignment)] +pub unsafe fn vst1_p8(ptr: *mut p8, a: poly8x8_t) { + copy_nonoverlapping( + &a as *const poly8x8_t as *const p8, + ptr as *mut p8, + size_of::(), + ) +} + +/// Store multiple single-element structures from one, two, three, or four registers. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(str))] +#[allow(clippy::cast_ptr_alignment)] +pub unsafe fn vst1q_p8(ptr: *mut p8, a: poly8x16_t) { + copy_nonoverlapping( + &a as *const poly8x16_t as *const p8, + ptr as *mut p8, + size_of::(), + ) +} + +/// Store multiple single-element structures from one, two, three, or four registers. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(str))] +#[allow(clippy::cast_ptr_alignment)] +pub unsafe fn vst1_p16(ptr: *mut p16, a: poly16x4_t) { + copy_nonoverlapping( + &a as *const poly16x4_t as *const p16, + ptr as *mut p16, + size_of::(), + ) +} + +/// Store multiple single-element structures from one, two, three, or four registers. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(str))] +#[allow(clippy::cast_ptr_alignment)] +pub unsafe fn vst1q_p16(ptr: *mut p16, a: poly16x8_t) { + copy_nonoverlapping( + &a as *const poly16x8_t as *const p16, + ptr as *mut p16, + size_of::(), + ) +} + +// Store multiple single-element structures from one, two, three, or four registers. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(str))] +#[allow(clippy::cast_ptr_alignment)] +pub unsafe fn vst1_p64(ptr: *mut p64, a: poly64x1_t) { + copy_nonoverlapping( + &a as *const poly64x1_t as *const p64, + ptr as *mut p64, + size_of::(), + ) +} + +// Store multiple single-element structures from one, two, three, or four registers. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(str))] +#[allow(clippy::cast_ptr_alignment)] +pub unsafe fn vst1q_p64(ptr: *mut p64, a: poly64x2_t) { + copy_nonoverlapping( + &a as *const poly64x2_t as *const p64, + ptr as *mut p64, + size_of::(), + ) +} + +// Store multiple single-element structures from one, two, three, or four registers. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(str))] +#[allow(clippy::cast_ptr_alignment)] +pub unsafe fn vst1_f32(ptr: *mut f32, a: float32x2_t) { + copy_nonoverlapping( + &a as *const float32x2_t as *const f32, + ptr as *mut f32, + size_of::(), + ) +} + +// Store multiple single-element structures from one, two, three, or four registers. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(str))] +#[allow(clippy::cast_ptr_alignment)] +pub unsafe fn vst1q_f32(ptr: *mut f32, a: float32x4_t) { + copy_nonoverlapping( + &a as *const float32x4_t as *const f32, + ptr as *mut f32, + size_of::(), + ) +} + +// Store multiple single-element structures from one, two, three, or four registers. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(str))] +#[allow(clippy::cast_ptr_alignment)] +pub unsafe fn vst1_f64(ptr: *mut f64, a: float64x1_t) { + copy_nonoverlapping( + &a as *const float64x1_t as *const f64, + ptr as *mut f64, + size_of::(), + ) +} + +// Store multiple single-element structures from one, two, three, or four registers. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(str))] +#[allow(clippy::cast_ptr_alignment)] +pub unsafe fn vst1q_f64(ptr: *mut f64, a: float64x2_t) { + copy_nonoverlapping( + &a as *const float64x2_t as *const f64, + ptr as *mut f64, + size_of::(), + ) +} + /// Absolute Value (wrapping). #[inline] #[target_feature(enable = "neon")] @@ -701,6 +1156,37 @@ pub unsafe fn vabsq_s64(a: int64x2_t) -> int64x2_t { vabsq_s64_(a) } +/// Bitwise Select instructions. This instruction sets each bit in the destination SIMD&FP register +/// to the corresponding bit from the first source SIMD&FP register when the original +/// destination bit was 1, otherwise from the second source SIMD&FP register. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(bsl))] +pub unsafe fn vbsl_f64(a: uint64x1_t, b: float64x1_t, c: float64x1_t) -> float64x1_t { + simd_select(transmute::<_, int64x1_t>(a), b, c) +} +/// Bitwise Select. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(bsl))] +pub unsafe fn vbsl_p64(a: poly64x1_t, b: poly64x1_t, c: poly64x1_t) -> poly64x1_t { + simd_select(transmute::<_, int64x1_t>(a), b, c) +} +/// Bitwise Select. (128-bit) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(bsl))] +pub unsafe fn vbslq_f64(a: uint64x2_t, b: float64x2_t, c: float64x2_t) -> float64x2_t { + simd_select(transmute::<_, int64x2_t>(a), b, c) +} +/// Bitwise Select. (128-bit) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(bsl))] +pub unsafe fn vbslq_p64(a: poly64x2_t, b: poly64x2_t, c: poly64x2_t) -> poly64x2_t { + simd_select(transmute::<_, int64x2_t>(a), b, c) +} + /// Signed saturating Accumulate of Unsigned value. #[inline] #[target_feature(enable = "neon")] @@ -971,12 +1457,33 @@ pub unsafe fn vaddvq_u64(a: uint64x2_t) -> u64 { vaddvq_u64_(a) } -/// Polynomial multiply long +/// Signed Add Long across Vector #[inline] #[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(pmull))] -pub unsafe fn vmull_p64(a: p64, b: p64) -> p128 { - transmute(vmull_p64_(transmute(a), transmute(b))) +#[cfg_attr(test, assert_instr(saddlv))] +pub unsafe fn vaddlv_s8(a: int8x8_t) -> i16 { + vaddlv_s8_(a) as i16 +} +/// Signed Add Long across Vector +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(saddlv))] +pub unsafe fn vaddlvq_s8(a: int8x16_t) -> i16 { + vaddlvq_s8_(a) as i16 +} +/// Unsigned Add Long across Vector +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uaddlv))] +pub unsafe fn vaddlv_u8(a: uint8x8_t) -> u16 { + vaddlv_u8_(a) as u16 +} +/// Unsigned Add Long across Vector +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(uaddlv))] +pub unsafe fn vaddlvq_u8(a: uint8x16_t) -> u16 { + vaddlvq_u8_(a) as u16 } /// Vector add. @@ -1016,9 +1523,7 @@ pub unsafe fn vadd_u64(a: uint64x1_t, b: uint64x1_t) -> uint64x1_t { #[target_feature(enable = "neon")] #[cfg_attr(test, assert_instr(add))] pub unsafe fn vaddd_s64(a: i64, b: i64) -> i64 { - let a: int64x1_t = transmute(a); - let b: int64x1_t = transmute(b); - simd_extract(simd_add(a, b), 0) + a.wrapping_add(b) } /// Vector add. @@ -1026,9 +1531,7 @@ pub unsafe fn vaddd_s64(a: i64, b: i64) -> i64 { #[target_feature(enable = "neon")] #[cfg_attr(test, assert_instr(add))] pub unsafe fn vaddd_u64(a: u64, b: u64) -> u64 { - let a: uint64x1_t = transmute(a); - let b: uint64x1_t = transmute(b); - simd_extract(simd_add(a, b), 0) + a.wrapping_add(b) } /// Horizontal vector max. @@ -1399,12 +1902,35 @@ pub unsafe fn vpmaxq_f64(a: float64x2_t, b: float64x2_t) -> float64x2_t { vpmaxq_f64_(a, b) } +/// Extract vector from pair of vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(str, N = 0))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vext_p64(a: poly64x1_t, _b: poly64x1_t) -> poly64x1_t { + if N != 0 { + unreachable_unchecked() + } + a +} + +/// Extract vector from pair of vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(str, N = 0))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vext_f64(a: float64x1_t, _b: float64x1_t) -> float64x1_t { + if N != 0 { + unreachable_unchecked() + } + a +} /// Vector combine #[inline] #[target_feature(enable = "neon")] #[cfg_attr(test, assert_instr(mov))] pub unsafe fn vcombine_s8(low: int8x8_t, high: int8x8_t) -> int8x16_t { - simd_shuffle16( + simd_shuffle16!( low, high, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15], @@ -1416,7 +1942,7 @@ pub unsafe fn vcombine_s8(low: int8x8_t, high: int8x8_t) -> int8x16_t { #[target_feature(enable = "neon")] #[cfg_attr(test, assert_instr(mov))] pub unsafe fn vcombine_s16(low: int16x4_t, high: int16x4_t) -> int16x8_t { - simd_shuffle8(low, high, [0, 1, 2, 3, 4, 5, 6, 7]) + simd_shuffle8!(low, high, [0, 1, 2, 3, 4, 5, 6, 7]) } /// Vector combine @@ -1424,7 +1950,7 @@ pub unsafe fn vcombine_s16(low: int16x4_t, high: int16x4_t) -> int16x8_t { #[target_feature(enable = "neon")] #[cfg_attr(test, assert_instr(mov))] pub unsafe fn vcombine_s32(low: int32x2_t, high: int32x2_t) -> int32x4_t { - simd_shuffle4(low, high, [0, 1, 2, 3]) + simd_shuffle4!(low, high, [0, 1, 2, 3]) } /// Vector combine @@ -1432,7 +1958,7 @@ pub unsafe fn vcombine_s32(low: int32x2_t, high: int32x2_t) -> int32x4_t { #[target_feature(enable = "neon")] #[cfg_attr(test, assert_instr(mov))] pub unsafe fn vcombine_s64(low: int64x1_t, high: int64x1_t) -> int64x2_t { - simd_shuffle2(low, high, [0, 1]) + simd_shuffle2!(low, high, [0, 1]) } /// Vector combine @@ -1440,7 +1966,7 @@ pub unsafe fn vcombine_s64(low: int64x1_t, high: int64x1_t) -> int64x2_t { #[target_feature(enable = "neon")] #[cfg_attr(test, assert_instr(mov))] pub unsafe fn vcombine_u8(low: uint8x8_t, high: uint8x8_t) -> uint8x16_t { - simd_shuffle16( + simd_shuffle16!( low, high, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15], @@ -1452,7 +1978,7 @@ pub unsafe fn vcombine_u8(low: uint8x8_t, high: uint8x8_t) -> uint8x16_t { #[target_feature(enable = "neon")] #[cfg_attr(test, assert_instr(mov))] pub unsafe fn vcombine_u16(low: uint16x4_t, high: uint16x4_t) -> uint16x8_t { - simd_shuffle8(low, high, [0, 1, 2, 3, 4, 5, 6, 7]) + simd_shuffle8!(low, high, [0, 1, 2, 3, 4, 5, 6, 7]) } /// Vector combine @@ -1460,7 +1986,7 @@ pub unsafe fn vcombine_u16(low: uint16x4_t, high: uint16x4_t) -> uint16x8_t { #[target_feature(enable = "neon")] #[cfg_attr(test, assert_instr(mov))] pub unsafe fn vcombine_u32(low: uint32x2_t, high: uint32x2_t) -> uint32x4_t { - simd_shuffle4(low, high, [0, 1, 2, 3]) + simd_shuffle4!(low, high, [0, 1, 2, 3]) } /// Vector combine @@ -1468,7 +1994,7 @@ pub unsafe fn vcombine_u32(low: uint32x2_t, high: uint32x2_t) -> uint32x4_t { #[target_feature(enable = "neon")] #[cfg_attr(test, assert_instr(mov))] pub unsafe fn vcombine_u64(low: uint64x1_t, high: uint64x1_t) -> uint64x2_t { - simd_shuffle2(low, high, [0, 1]) + simd_shuffle2!(low, high, [0, 1]) } /// Vector combine @@ -1476,7 +2002,123 @@ pub unsafe fn vcombine_u64(low: uint64x1_t, high: uint64x1_t) -> uint64x2_t { #[target_feature(enable = "neon")] #[cfg_attr(test, assert_instr(mov))] pub unsafe fn vcombine_p64(low: poly64x1_t, high: poly64x1_t) -> poly64x2_t { - simd_shuffle2(low, high, [0, 1]) + simd_shuffle2!(low, high, [0, 1]) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmov))] +pub unsafe fn vdup_n_p64(value: p64) -> poly64x1_t { + transmute(u64x1::new(value)) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ldr))] +pub unsafe fn vdup_n_f64(value: f64) -> float64x1_t { + float64x1_t(value) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(dup))] +pub unsafe fn vdupq_n_p64(value: p64) -> poly64x2_t { + transmute(u64x2::new(value, value)) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(dup))] +pub unsafe fn vdupq_n_f64(value: f64) -> float64x2_t { + float64x2_t(value, value) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(fmov))] +pub unsafe fn vmov_n_p64(value: p64) -> poly64x1_t { + vdup_n_p64(value) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ldr))] +pub unsafe fn vmov_n_f64(value: f64) -> float64x1_t { + vdup_n_f64(value) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(dup))] +pub unsafe fn vmovq_n_p64(value: p64) -> poly64x2_t { + vdupq_n_p64(value) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(dup))] +pub unsafe fn vmovq_n_f64(value: f64) -> float64x2_t { + vdupq_n_f64(value) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(mov))] +pub unsafe fn vget_high_f64(a: float64x2_t) -> float64x1_t { + float64x1_t(simd_extract(a, 1)) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ldr))] +pub unsafe fn vget_high_p64(a: poly64x2_t) -> poly64x1_t { + transmute(u64x1::new(simd_extract(a, 1))) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ldr))] +pub unsafe fn vget_low_f64(a: float64x2_t) -> float64x1_t { + float64x1_t(simd_extract(a, 0)) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(ldr))] +pub unsafe fn vget_low_p64(a: poly64x2_t) -> poly64x1_t { + transmute(u64x1::new(simd_extract(a, 0))) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[rustc_legacy_const_generics(1)] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop, IMM5 = 0))] +pub unsafe fn vget_lane_f64(v: float64x1_t) -> f64 { + static_assert!(IMM5 : i32 where IMM5 == 0); + simd_extract(v, IMM5 as u32) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[rustc_legacy_const_generics(1)] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop, IMM5 = 0))] +pub unsafe fn vgetq_lane_f64(v: float64x2_t) -> f64 { + static_assert_imm1!(IMM5); + simd_extract(v, IMM5 as u32) } /* FIXME: 16-bit float @@ -1485,7 +2127,7 @@ pub unsafe fn vcombine_p64(low: poly64x1_t, high: poly64x1_t) -> poly64x2_t { #[target_feature(enable = "neon")] #[cfg_attr(test, assert_instr(mov))] pub unsafe fn vcombine_f16 ( low: float16x4_t, high: float16x4_t) -> float16x8_t { - simd_shuffle8(low, high, [0, 1, 2, 3, 4, 5, 6, 7]) + simd_shuffle8!(low, high, [0, 1, 2, 3, 4, 5, 6, 7]) } */ @@ -1494,7 +2136,7 @@ pub unsafe fn vcombine_f16 ( low: float16x4_t, high: float16x4_t) -> float16x8_ #[target_feature(enable = "neon")] #[cfg_attr(test, assert_instr(mov))] pub unsafe fn vcombine_f32(low: float32x2_t, high: float32x2_t) -> float32x4_t { - simd_shuffle4(low, high, [0, 1, 2, 3]) + simd_shuffle4!(low, high, [0, 1, 2, 3]) } /// Vector combine @@ -1502,7 +2144,7 @@ pub unsafe fn vcombine_f32(low: float32x2_t, high: float32x2_t) -> float32x4_t { #[target_feature(enable = "neon")] #[cfg_attr(test, assert_instr(mov))] pub unsafe fn vcombine_p8(low: poly8x8_t, high: poly8x8_t) -> poly8x16_t { - simd_shuffle16( + simd_shuffle16!( low, high, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15], @@ -1514,7 +2156,7 @@ pub unsafe fn vcombine_p8(low: poly8x8_t, high: poly8x8_t) -> poly8x16_t { #[target_feature(enable = "neon")] #[cfg_attr(test, assert_instr(mov))] pub unsafe fn vcombine_p16(low: poly16x4_t, high: poly16x4_t) -> poly16x8_t { - simd_shuffle8(low, high, [0, 1, 2, 3, 4, 5, 6, 7]) + simd_shuffle8!(low, high, [0, 1, 2, 3, 4, 5, 6, 7]) } /// Vector combine @@ -1522,7 +2164,7 @@ pub unsafe fn vcombine_p16(low: poly16x4_t, high: poly16x4_t) -> poly16x8_t { #[target_feature(enable = "neon")] #[cfg_attr(test, assert_instr(mov))] pub unsafe fn vcombine_f64(low: float64x1_t, high: float64x1_t) -> float64x2_t { - simd_shuffle2(low, high, [0, 1]) + simd_shuffle2!(low, high, [0, 1]) } /// Table look-up @@ -2299,427 +2941,436 @@ pub unsafe fn vqtbx4q_p8(a: poly8x16_t, t: poly8x16x4_t, idx: uint8x16_t) -> pol )) } +/// Shift left +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vshld_n_s64(a: i64) -> i64 { + static_assert_imm6!(N); + a << N +} + +/// Shift left +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vshld_n_u64(a: u64) -> u64 { + static_assert_imm6!(N); + a << N +} + +/// Signed shift right +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vshrd_n_s64(a: i64) -> i64 { + static_assert!(N : i32 where N >= 1 && N <= 64); + a >> N +} + +/// Unsigned shift right +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(nop, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vshrd_n_u64(a: u64) -> u64 { + static_assert!(N : i32 where N >= 1 && N <= 64); + a >> N +} + +/// Signed shift right and accumulate #[inline] #[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(fcvtzs))] -pub unsafe fn vcvtq_s32_f32(a: float32x4_t) -> int32x4_t { - vcvtq_s32_f32_(a) +#[cfg_attr(test, assert_instr(nop, N = 2))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vsrad_n_s64(a: i64, b: i64) -> i64 { + static_assert!(N : i32 where N >= 1 && N <= 64); + a + (b >> N) } -/// Floating-point Convert to Unsigned fixed-point, rounding toward Zero (vector) +/// Unsigned shift right and accumulate #[inline] #[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(fcvtzu))] -pub unsafe fn vcvtq_u32_f32(a: float32x4_t) -> uint32x4_t { - vcvtq_u32_f32_(a) +#[cfg_attr(test, assert_instr(nop, N = 2))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vsrad_n_u64(a: u64, b: u64) -> u64 { + static_assert!(N : i32 where N >= 1 && N <= 64); + a + (b >> N) } /// Shift Left and Insert (immediate) #[inline] #[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(sli, n = 1))] -#[rustc_args_required_const(2)] -pub unsafe fn vsli_n_s8(a: int8x8_t, b: int8x8_t, n: i32) -> int8x8_t { - assert!(0 <= n && n <= 7, "must have 0 ≤ n ≤ 7, but n = {}", n); - vsli_n_s8_(a, b, n) +#[cfg_attr(test, assert_instr(sli, N = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vsli_n_s8(a: int8x8_t, b: int8x8_t) -> int8x8_t { + static_assert_imm3!(N); + vsli_n_s8_(a, b, N) } /// Shift Left and Insert (immediate) #[inline] #[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(sli, n = 1))] -#[rustc_args_required_const(2)] -pub unsafe fn vsliq_n_s8(a: int8x16_t, b: int8x16_t, n: i32) -> int8x16_t { - assert!(0 <= n && n <= 7, "must have 0 ≤ n ≤ 7, but n = {}", n); - vsliq_n_s8_(a, b, n) +#[cfg_attr(test, assert_instr(sli, N = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vsliq_n_s8(a: int8x16_t, b: int8x16_t) -> int8x16_t { + static_assert_imm3!(N); + vsliq_n_s8_(a, b, N) } /// Shift Left and Insert (immediate) #[inline] #[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(sli, n = 1))] -#[rustc_args_required_const(2)] -pub unsafe fn vsli_n_s16(a: int16x4_t, b: int16x4_t, n: i32) -> int16x4_t { - assert!(0 <= n && n <= 15, "must have 0 ≤ n ≤ 15, but n = {}", n); - vsli_n_s16_(a, b, n) +#[cfg_attr(test, assert_instr(sli, N = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vsli_n_s16(a: int16x4_t, b: int16x4_t) -> int16x4_t { + static_assert_imm4!(N); + vsli_n_s16_(a, b, N) } /// Shift Left and Insert (immediate) #[inline] #[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(sli, n = 1))] -#[rustc_args_required_const(2)] -pub unsafe fn vsliq_n_s16(a: int16x8_t, b: int16x8_t, n: i32) -> int16x8_t { - assert!(0 <= n && n <= 15, "must have 0 ≤ n ≤ 15, but n = {}", n); - vsliq_n_s16_(a, b, n) +#[cfg_attr(test, assert_instr(sli, N = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vsliq_n_s16(a: int16x8_t, b: int16x8_t) -> int16x8_t { + static_assert_imm4!(N); + vsliq_n_s16_(a, b, N) } /// Shift Left and Insert (immediate) #[inline] #[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(sli, n = 1))] -#[rustc_args_required_const(2)] -pub unsafe fn vsli_n_s32(a: int32x2_t, b: int32x2_t, n: i32) -> int32x2_t { - assert!(0 <= n && n <= 31, "must have 0 ≤ n ≤ 31, but n = {}", n); - vsli_n_s32_(a, b, n) +#[cfg_attr(test, assert_instr(sli, N = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vsli_n_s32(a: int32x2_t, b: int32x2_t) -> int32x2_t { + static_assert!(N: i32 where N >= 0 && N <= 31); + vsli_n_s32_(a, b, N) } /// Shift Left and Insert (immediate) #[inline] #[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(sli, n = 1))] -#[rustc_args_required_const(2)] -pub unsafe fn vsliq_n_s32(a: int32x4_t, b: int32x4_t, n: i32) -> int32x4_t { - assert!(0 <= n && n <= 31, "must have 0 ≤ n ≤ 31, but n = {}", n); - vsliq_n_s32_(a, b, n) +#[cfg_attr(test, assert_instr(sli, N = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vsliq_n_s32(a: int32x4_t, b: int32x4_t) -> int32x4_t { + static_assert!(N: i32 where N >= 0 && N <= 31); + vsliq_n_s32_(a, b, N) } /// Shift Left and Insert (immediate) #[inline] #[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(sli, n = 1))] -#[rustc_args_required_const(2)] -pub unsafe fn vsli_n_s64(a: int64x1_t, b: int64x1_t, n: i32) -> int64x1_t { - assert!(0 <= n && n <= 63, "must have 0 ≤ n ≤ 63, but n = {}", n); - vsli_n_s64_(a, b, n) +#[cfg_attr(test, assert_instr(sli, N = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vsli_n_s64(a: int64x1_t, b: int64x1_t) -> int64x1_t { + static_assert!(N: i32 where N >= 0 && N <= 63); + vsli_n_s64_(a, b, N) } /// Shift Left and Insert (immediate) #[inline] #[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(sli, n = 1))] -#[rustc_args_required_const(2)] -pub unsafe fn vsliq_n_s64(a: int64x2_t, b: int64x2_t, n: i32) -> int64x2_t { - assert!(0 <= n && n <= 63, "must have 0 ≤ n ≤ 63, but n = {}", n); - vsliq_n_s64_(a, b, n) +#[cfg_attr(test, assert_instr(sli, N = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vsliq_n_s64(a: int64x2_t, b: int64x2_t) -> int64x2_t { + static_assert!(N: i32 where N >= 0 && N <= 63); + vsliq_n_s64_(a, b, N) } /// Shift Left and Insert (immediate) #[inline] #[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(sli, n = 1))] -#[rustc_args_required_const(2)] -pub unsafe fn vsli_n_u8(a: uint8x8_t, b: uint8x8_t, n: i32) -> uint8x8_t { - assert!(0 <= n && n <= 7, "must have 0 ≤ n ≤ 7, but n = {}", n); - transmute(vsli_n_s8_(transmute(a), transmute(b), n)) +#[cfg_attr(test, assert_instr(sli, N = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vsli_n_u8(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t { + static_assert_imm3!(N); + transmute(vsli_n_s8_(transmute(a), transmute(b), N)) } /// Shift Left and Insert (immediate) #[inline] #[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(sli, n = 1))] -#[rustc_args_required_const(2)] -pub unsafe fn vsliq_n_u8(a: uint8x16_t, b: uint8x16_t, n: i32) -> uint8x16_t { - assert!(0 <= n && n <= 7, "must have 0 ≤ n ≤ 7, but n = {}", n); - transmute(vsliq_n_s8_(transmute(a), transmute(b), n)) +#[cfg_attr(test, assert_instr(sli, N = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vsliq_n_u8(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t { + static_assert_imm3!(N); + transmute(vsliq_n_s8_(transmute(a), transmute(b), N)) } /// Shift Left and Insert (immediate) #[inline] #[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(sli, n = 1))] -#[rustc_args_required_const(2)] -pub unsafe fn vsli_n_u16(a: uint16x4_t, b: uint16x4_t, n: i32) -> uint16x4_t { - assert!(0 <= n && n <= 15, "must have 0 ≤ n ≤ 15, but n = {}", n); - transmute(vsli_n_s16_(transmute(a), transmute(b), n)) +#[cfg_attr(test, assert_instr(sli, N = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vsli_n_u16(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t { + static_assert_imm4!(N); + transmute(vsli_n_s16_(transmute(a), transmute(b), N)) } /// Shift Left and Insert (immediate) #[inline] #[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(sli, n = 1))] -#[rustc_args_required_const(2)] -pub unsafe fn vsliq_n_u16(a: uint16x8_t, b: uint16x8_t, n: i32) -> uint16x8_t { - assert!(0 <= n && n <= 15, "must have 0 ≤ n ≤ 15, but n = {}", n); - transmute(vsliq_n_s16_(transmute(a), transmute(b), n)) +#[cfg_attr(test, assert_instr(sli, N = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vsliq_n_u16(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t { + static_assert_imm4!(N); + transmute(vsliq_n_s16_(transmute(a), transmute(b), N)) } /// Shift Left and Insert (immediate) #[inline] #[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(sli, n = 1))] -#[rustc_args_required_const(2)] -pub unsafe fn vsli_n_u32(a: uint32x2_t, b: uint32x2_t, n: i32) -> uint32x2_t { - assert!(0 <= n && n <= 31, "must have 0 ≤ n ≤ 31, but n = {}", n); - transmute(vsli_n_s32_(transmute(a), transmute(b), n)) +#[cfg_attr(test, assert_instr(sli, N = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vsli_n_u32(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t { + static_assert!(N: i32 where N >= 0 && N <= 31); + transmute(vsli_n_s32_(transmute(a), transmute(b), N)) } /// Shift Left and Insert (immediate) #[inline] #[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(sli, n = 1))] -#[rustc_args_required_const(2)] -pub unsafe fn vsliq_n_u32(a: uint32x4_t, b: uint32x4_t, n: i32) -> uint32x4_t { - assert!(0 <= n && n <= 31, "must have 0 ≤ n ≤ 31, but n = {}", n); - transmute(vsliq_n_s32_(transmute(a), transmute(b), n)) +#[cfg_attr(test, assert_instr(sli, N = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vsliq_n_u32(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t { + static_assert!(N: i32 where N >= 0 && N <= 31); + transmute(vsliq_n_s32_(transmute(a), transmute(b), N)) } /// Shift Left and Insert (immediate) #[inline] #[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(sli, n = 1))] -#[rustc_args_required_const(2)] -pub unsafe fn vsli_n_u64(a: uint64x1_t, b: uint64x1_t, n: i32) -> uint64x1_t { - assert!(0 <= n && n <= 63, "must have 0 ≤ n ≤ 63, but n = {}", n); - transmute(vsli_n_s64_(transmute(a), transmute(b), n)) +#[cfg_attr(test, assert_instr(sli, N = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vsli_n_u64(a: uint64x1_t, b: uint64x1_t) -> uint64x1_t { + static_assert!(N: i32 where N >= 0 && N <= 63); + transmute(vsli_n_s64_(transmute(a), transmute(b), N)) } /// Shift Left and Insert (immediate) #[inline] #[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(sli, n = 1))] -#[rustc_args_required_const(2)] -pub unsafe fn vsliq_n_u64(a: uint64x2_t, b: uint64x2_t, n: i32) -> uint64x2_t { - assert!(0 <= n && n <= 63, "must have 0 ≤ n ≤ 63, but n = {}", n); - transmute(vsliq_n_s64_(transmute(a), transmute(b), n)) +#[cfg_attr(test, assert_instr(sli, N = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vsliq_n_u64(a: uint64x2_t, b: uint64x2_t) -> uint64x2_t { + static_assert!(N: i32 where N >= 0 && N <= 63); + transmute(vsliq_n_s64_(transmute(a), transmute(b), N)) } /// Shift Left and Insert (immediate) #[inline] #[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(sli, n = 1))] -#[rustc_args_required_const(2)] -pub unsafe fn vsli_n_p8(a: poly8x8_t, b: poly8x8_t, n: i32) -> poly8x8_t { - assert!(0 <= n && n <= 7, "must have 0 ≤ n ≤ 7, but n = {}", n); - transmute(vsli_n_s8_(transmute(a), transmute(b), n)) +#[cfg_attr(test, assert_instr(sli, N = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vsli_n_p8(a: poly8x8_t, b: poly8x8_t) -> poly8x8_t { + static_assert_imm3!(N); + transmute(vsli_n_s8_(transmute(a), transmute(b), N)) } /// Shift Left and Insert (immediate) #[inline] #[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(sli, n = 1))] -#[rustc_args_required_const(2)] -pub unsafe fn vsliq_n_p8(a: poly8x16_t, b: poly8x16_t, n: i32) -> poly8x16_t { - assert!(0 <= n && n <= 7, "must have 0 ≤ n ≤ 7, but n = {}", n); - transmute(vsliq_n_s8_(transmute(a), transmute(b), n)) +#[cfg_attr(test, assert_instr(sli, N = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vsliq_n_p8(a: poly8x16_t, b: poly8x16_t) -> poly8x16_t { + static_assert_imm3!(N); + transmute(vsliq_n_s8_(transmute(a), transmute(b), N)) } /// Shift Left and Insert (immediate) #[inline] #[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(sli, n = 1))] -#[rustc_args_required_const(2)] -pub unsafe fn vsli_n_p16(a: poly16x4_t, b: poly16x4_t, n: i32) -> poly16x4_t { - assert!(0 <= n && n <= 15, "must have 0 ≤ n ≤ 15, but n = {}", n); - transmute(vsli_n_s16_(transmute(a), transmute(b), n)) +#[cfg_attr(test, assert_instr(sli, N = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vsli_n_p16(a: poly16x4_t, b: poly16x4_t) -> poly16x4_t { + static_assert_imm4!(N); + transmute(vsli_n_s16_(transmute(a), transmute(b), N)) } /// Shift Left and Insert (immediate) #[inline] #[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(sli, n = 1))] -#[rustc_args_required_const(2)] -pub unsafe fn vsliq_n_p16(a: poly16x8_t, b: poly16x8_t, n: i32) -> poly16x8_t { - assert!(0 <= n && n <= 15, "must have 0 ≤ n ≤ 15, but n = {}", n); - transmute(vsliq_n_s16_(transmute(a), transmute(b), n)) +#[cfg_attr(test, assert_instr(sli, N = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vsliq_n_p16(a: poly16x8_t, b: poly16x8_t) -> poly16x8_t { + static_assert_imm4!(N); + transmute(vsliq_n_s16_(transmute(a), transmute(b), N)) } /// Shift Right and Insert (immediate) #[inline] #[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(sri, n = 1))] -#[rustc_args_required_const(2)] -pub unsafe fn vsri_n_s8(a: int8x8_t, b: int8x8_t, n: i32) -> int8x8_t { - assert!(1 <= n && n <= 8, "must have 1 ≤ n ≤ 8, but n = {}", n); - vsri_n_s8_(a, b, n) +#[cfg_attr(test, assert_instr(sri, N = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vsri_n_s8(a: int8x8_t, b: int8x8_t) -> int8x8_t { + static_assert!(N: i32 where N >= 1 && N <= 8); + vsri_n_s8_(a, b, N) } /// Shift Right and Insert (immediate) #[inline] #[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(sri, n = 1))] -#[rustc_args_required_const(2)] -pub unsafe fn vsriq_n_s8(a: int8x16_t, b: int8x16_t, n: i32) -> int8x16_t { - assert!(1 <= n && n <= 8, "must have 1 ≤ n ≤ 8, but n = {}", n); - vsriq_n_s8_(a, b, n) +#[cfg_attr(test, assert_instr(sri, N = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vsriq_n_s8(a: int8x16_t, b: int8x16_t) -> int8x16_t { + static_assert!(N: i32 where N >= 1 && N <= 8); + vsriq_n_s8_(a, b, N) } /// Shift Right and Insert (immediate) #[inline] #[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(sri, n = 1))] -#[rustc_args_required_const(2)] -pub unsafe fn vsri_n_s16(a: int16x4_t, b: int16x4_t, n: i32) -> int16x4_t { - assert!(1 <= n && n <= 16, "must have 1 ≤ n ≤ 16, but n = {}", n); - vsri_n_s16_(a, b, n) +#[cfg_attr(test, assert_instr(sri, N = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vsri_n_s16(a: int16x4_t, b: int16x4_t) -> int16x4_t { + static_assert!(N: i32 where N >= 1 && N <= 16); + vsri_n_s16_(a, b, N) } /// Shift Right and Insert (immediate) #[inline] #[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(sri, n = 1))] -#[rustc_args_required_const(2)] -pub unsafe fn vsriq_n_s16(a: int16x8_t, b: int16x8_t, n: i32) -> int16x8_t { - assert!(1 <= n && n <= 16, "must have 1 ≤ n ≤ 16, but n = {}", n); - vsriq_n_s16_(a, b, n) +#[cfg_attr(test, assert_instr(sri, N = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vsriq_n_s16(a: int16x8_t, b: int16x8_t) -> int16x8_t { + static_assert!(N: i32 where N >= 1 && N <= 16); + vsriq_n_s16_(a, b, N) } /// Shift Right and Insert (immediate) #[inline] #[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(sri, n = 1))] -#[rustc_args_required_const(2)] -pub unsafe fn vsri_n_s32(a: int32x2_t, b: int32x2_t, n: i32) -> int32x2_t { - assert!(1 <= n && n <= 32, "must have 1 ≤ n ≤ 32, but n = {}", n); - vsri_n_s32_(a, b, n) +#[cfg_attr(test, assert_instr(sri, N = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vsri_n_s32(a: int32x2_t, b: int32x2_t) -> int32x2_t { + static_assert!(N: i32 where N >= 1 && N <= 32); + vsri_n_s32_(a, b, N) } /// Shift Right and Insert (immediate) #[inline] #[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(sri, n = 1))] -#[rustc_args_required_const(2)] -pub unsafe fn vsriq_n_s32(a: int32x4_t, b: int32x4_t, n: i32) -> int32x4_t { - assert!(1 <= n && n <= 32, "must have 1 ≤ n ≤ 32, but n = {}", n); - vsriq_n_s32_(a, b, n) +#[cfg_attr(test, assert_instr(sri, N = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vsriq_n_s32(a: int32x4_t, b: int32x4_t) -> int32x4_t { + static_assert!(N: i32 where N >= 1 && N <= 32); + vsriq_n_s32_(a, b, N) } /// Shift Right and Insert (immediate) #[inline] #[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(sri, n = 1))] -#[rustc_args_required_const(2)] -pub unsafe fn vsri_n_s64(a: int64x1_t, b: int64x1_t, n: i32) -> int64x1_t { - assert!(1 <= n && n <= 64, "must have 1 ≤ n ≤ 64, but n = {}", n); - vsri_n_s64_(a, b, n) +#[cfg_attr(test, assert_instr(sri, N = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vsri_n_s64(a: int64x1_t, b: int64x1_t) -> int64x1_t { + static_assert!(N: i32 where N >= 1 && N <= 64); + vsri_n_s64_(a, b, N) } /// Shift Right and Insert (immediate) #[inline] #[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(sri, n = 1))] -#[rustc_args_required_const(2)] -pub unsafe fn vsriq_n_s64(a: int64x2_t, b: int64x2_t, n: i32) -> int64x2_t { - assert!(1 <= n && n <= 64, "must have 1 ≤ n ≤ 64, but n = {}", n); - vsriq_n_s64_(a, b, n) +#[cfg_attr(test, assert_instr(sri, N = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vsriq_n_s64(a: int64x2_t, b: int64x2_t) -> int64x2_t { + static_assert!(N: i32 where N >= 1 && N <= 64); + vsriq_n_s64_(a, b, N) } /// Shift Right and Insert (immediate) #[inline] #[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(sri, n = 1))] -#[rustc_args_required_const(2)] -pub unsafe fn vsri_n_u8(a: uint8x8_t, b: uint8x8_t, n: i32) -> uint8x8_t { - assert!(1 <= n && n <= 8, "must have 1 ≤ n ≤ 8, but n = {}", n); - transmute(vsri_n_s8_(transmute(a), transmute(b), n)) +#[cfg_attr(test, assert_instr(sri, N = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vsri_n_u8(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t { + static_assert!(N: i32 where N >= 1 && N <= 8); + transmute(vsri_n_s8_(transmute(a), transmute(b), N)) } /// Shift Right and Insert (immediate) #[inline] #[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(sri, n = 1))] -#[rustc_args_required_const(2)] -pub unsafe fn vsriq_n_u8(a: uint8x16_t, b: uint8x16_t, n: i32) -> uint8x16_t { - assert!(1 <= n && n <= 8, "must have 1 ≤ n ≤ 8, but n = {}", n); - transmute(vsriq_n_s8_(transmute(a), transmute(b), n)) +#[cfg_attr(test, assert_instr(sri, N = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vsriq_n_u8(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t { + static_assert!(N: i32 where N >= 1 && N <= 8); + transmute(vsriq_n_s8_(transmute(a), transmute(b), N)) } /// Shift Right and Insert (immediate) #[inline] #[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(sri, n = 1))] -#[rustc_args_required_const(2)] -pub unsafe fn vsri_n_u16(a: uint16x4_t, b: uint16x4_t, n: i32) -> uint16x4_t { - assert!(1 <= n && n <= 16, "must have 1 ≤ n ≤ 16, but n = {}", n); - transmute(vsri_n_s16_(transmute(a), transmute(b), n)) +#[cfg_attr(test, assert_instr(sri, N = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vsri_n_u16(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t { + static_assert!(N: i32 where N >= 1 && N <= 16); + transmute(vsri_n_s16_(transmute(a), transmute(b), N)) } /// Shift Right and Insert (immediate) #[inline] #[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(sri, n = 1))] -#[rustc_args_required_const(2)] -pub unsafe fn vsriq_n_u16(a: uint16x8_t, b: uint16x8_t, n: i32) -> uint16x8_t { - assert!(1 <= n && n <= 16, "must have 1 ≤ n ≤ 16, but n = {}", n); - transmute(vsriq_n_s16_(transmute(a), transmute(b), n)) +#[cfg_attr(test, assert_instr(sri, N = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vsriq_n_u16(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t { + static_assert!(N: i32 where N >= 1 && N <= 16); + transmute(vsriq_n_s16_(transmute(a), transmute(b), N)) } /// Shift Right and Insert (immediate) #[inline] #[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(sri, n = 1))] -#[rustc_args_required_const(2)] -pub unsafe fn vsri_n_u32(a: uint32x2_t, b: uint32x2_t, n: i32) -> uint32x2_t { - assert!(1 <= n && n <= 32, "must have 1 ≤ n ≤ 32, but n = {}", n); - transmute(vsri_n_s32_(transmute(a), transmute(b), n)) +#[cfg_attr(test, assert_instr(sri, N = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vsri_n_u32(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t { + static_assert!(N: i32 where N >= 1 && N <= 32); + transmute(vsri_n_s32_(transmute(a), transmute(b), N)) } /// Shift Right and Insert (immediate) #[inline] #[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(sri, n = 1))] -#[rustc_args_required_const(2)] -pub unsafe fn vsriq_n_u32(a: uint32x4_t, b: uint32x4_t, n: i32) -> uint32x4_t { - assert!(1 <= n && n <= 32, "must have 1 ≤ n ≤ 32, but n = {}", n); - transmute(vsriq_n_s32_(transmute(a), transmute(b), n)) +#[cfg_attr(test, assert_instr(sri, N = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vsriq_n_u32(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t { + static_assert!(N: i32 where N >= 1 && N <= 32); + transmute(vsriq_n_s32_(transmute(a), transmute(b), N)) } /// Shift Right and Insert (immediate) #[inline] #[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(sri, n = 1))] -#[rustc_args_required_const(2)] -pub unsafe fn vsri_n_u64(a: uint64x1_t, b: uint64x1_t, n: i32) -> uint64x1_t { - assert!(1 <= n && n <= 64, "must have 1 ≤ n ≤ 64, but n = {}", n); - transmute(vsri_n_s64_(transmute(a), transmute(b), n)) +#[cfg_attr(test, assert_instr(sri, N = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vsri_n_u64(a: uint64x1_t, b: uint64x1_t) -> uint64x1_t { + static_assert!(N: i32 where N >= 1 && N <= 64); + transmute(vsri_n_s64_(transmute(a), transmute(b), N)) } /// Shift Right and Insert (immediate) #[inline] #[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(sri, n = 1))] -#[rustc_args_required_const(2)] -pub unsafe fn vsriq_n_u64(a: uint64x2_t, b: uint64x2_t, n: i32) -> uint64x2_t { - assert!(1 <= n && n <= 64, "must have 1 ≤ n ≤ 64, but n = {}", n); - transmute(vsriq_n_s64_(transmute(a), transmute(b), n)) +#[cfg_attr(test, assert_instr(sri, N = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vsriq_n_u64(a: uint64x2_t, b: uint64x2_t) -> uint64x2_t { + static_assert!(N: i32 where N >= 1 && N <= 64); + transmute(vsriq_n_s64_(transmute(a), transmute(b), N)) } /// Shift Right and Insert (immediate) #[inline] #[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(sri, n = 1))] -#[rustc_args_required_const(2)] -pub unsafe fn vsri_n_p8(a: poly8x8_t, b: poly8x8_t, n: i32) -> poly8x8_t { - assert!(1 <= n && n <= 8, "must have 1 ≤ n ≤ 8, but n = {}", n); - transmute(vsri_n_s8_(transmute(a), transmute(b), n)) +#[cfg_attr(test, assert_instr(sri, N = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vsri_n_p8(a: poly8x8_t, b: poly8x8_t) -> poly8x8_t { + static_assert!(N: i32 where N >= 1 && N <= 8); + transmute(vsri_n_s8_(transmute(a), transmute(b), N)) } /// Shift Right and Insert (immediate) #[inline] #[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(sri, n = 1))] -#[rustc_args_required_const(2)] -pub unsafe fn vsriq_n_p8(a: poly8x16_t, b: poly8x16_t, n: i32) -> poly8x16_t { - assert!(1 <= n && n <= 8, "must have 1 ≤ n ≤ 8, but n = {}", n); - transmute(vsriq_n_s8_(transmute(a), transmute(b), n)) +#[cfg_attr(test, assert_instr(sri, N = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vsriq_n_p8(a: poly8x16_t, b: poly8x16_t) -> poly8x16_t { + static_assert!(N: i32 where N >= 1 && N <= 8); + transmute(vsriq_n_s8_(transmute(a), transmute(b), N)) } /// Shift Right and Insert (immediate) #[inline] #[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(sri, n = 1))] -#[rustc_args_required_const(2)] -pub unsafe fn vsri_n_p16(a: poly16x4_t, b: poly16x4_t, n: i32) -> poly16x4_t { - assert!(1 <= n && n <= 16, "must have 1 ≤ n ≤ 16, but n = {}", n); - transmute(vsri_n_s16_(transmute(a), transmute(b), n)) +#[cfg_attr(test, assert_instr(sri, N = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vsri_n_p16(a: poly16x4_t, b: poly16x4_t) -> poly16x4_t { + static_assert!(N: i32 where N >= 1 && N <= 16); + transmute(vsri_n_s16_(transmute(a), transmute(b), N)) } /// Shift Right and Insert (immediate) #[inline] #[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr(sri, n = 1))] -#[rustc_args_required_const(2)] -pub unsafe fn vsriq_n_p16(a: poly16x8_t, b: poly16x8_t, n: i32) -> poly16x8_t { - assert!(1 <= n && n <= 16, "must have 1 ≤ n ≤ 16, but n = {}", n); - transmute(vsriq_n_s16_(transmute(a), transmute(b), n)) +#[cfg_attr(test, assert_instr(sri, N = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vsriq_n_p16(a: poly16x8_t, b: poly16x8_t) -> poly16x8_t { + static_assert!(N: i32 where N >= 1 && N <= 16); + transmute(vsriq_n_s16_(transmute(a), transmute(b), N)) } #[cfg(test)] mod tests { use crate::core_arch::aarch64::test_support::*; - use crate::core_arch::arm::test_support::*; + use crate::core_arch::arm_shared::test_support::*; use crate::core_arch::{aarch64::neon::*, aarch64::*, simd::*}; use std::mem::transmute; use stdarch_test::simd_test; - #[simd_test(enable = "neon")] - unsafe fn test_vcvtq_s32_f32() { - let f = f32x4::new(-1., 2., 3., 4.); - let e = i32x4::new(-1, 2, 3, 4); - let r: i32x4 = transmute(vcvtq_s32_f32(transmute(f))); - assert_eq!(r, e); - - let f = f32x4::new(10e37, 2., 3., 4.); - let e = i32x4::new(0x7fffffff, 2, 3, 4); - let r: i32x4 = transmute(vcvtq_s32_f32(transmute(f))); - assert_eq!(r, e); - - let f = f32x4::new(-10e37, 2., 3., 4.); - let e = i32x4::new(-0x80000000, 2, 3, 4); - let r: i32x4 = transmute(vcvtq_s32_f32(transmute(f))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vcvtq_u32_f32() { - let f = f32x4::new(1., 2., 3., 4.); - let e = u32x4::new(1, 2, 3, 4); - let r: u32x4 = transmute(vcvtq_u32_f32(transmute(f))); - assert_eq!(r, e); - - let f = f32x4::new(-1., 2., 3., 4.); - let e = u32x4::new(0, 2, 3, 4); - let r: u32x4 = transmute(vcvtq_u32_f32(transmute(f))); - assert_eq!(r, e); - - let f = f32x4::new(10e37, 2., 3., 4.); - let e = u32x4::new(0xffffffff, 2, 3, 4); - let r: u32x4 = transmute(vcvtq_u32_f32(transmute(f))); - assert_eq!(r, e); - } - #[simd_test(enable = "neon")] unsafe fn test_vuqadd_s8() { let a = i8x8::new(i8::MIN, -3, -2, -1, 0, 1, 2, i8::MAX); @@ -2953,36 +3604,6 @@ mod tests { assert_eq!(r, e); } - #[simd_test(enable = "neon")] - unsafe fn test_vmull_p64() { - // FIXME: I've a hard time writing a test for this as the documentation - // from arm is a bit thin as to waht exactly it does - let a: i64 = 8; - let b: i64 = 7; - let e: i128 = 56; - let r: i128 = transmute(vmull_p64(transmute(a), transmute(b))); - assert_eq!(r, e); - - /* - let a: i64 = 5; - let b: i64 = 5; - let e: i128 = 25; - let r: i128 = transmute(vmull_p64(a, b)); - - assert_eq!(r, e); - let a: i64 = 6; - let b: i64 = 6; - let e: i128 = 36; - let r: i128 = transmute(vmull_p64(a, b)); - assert_eq!(r, e); - - let a: i64 = 7; - let b: i64 = 6; - let e: i128 = 42; - let r: i128 = transmute(vmull_p64(a, b)); - assert_eq!(r, e); - */ - } #[simd_test(enable = "neon")] unsafe fn test_vadd_f64() { let a = 1.; @@ -3397,6 +4018,74 @@ mod tests { assert_eq!(r, e); } + #[simd_test(enable = "neon")] + unsafe fn test_vext_p64() { + let a: i64x1 = i64x1::new(0); + let b: i64x1 = i64x1::new(1); + let e: i64x1 = i64x1::new(0); + let r: i64x1 = transmute(vext_p64::<0>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vext_f64() { + let a: f64x1 = f64x1::new(0.); + let b: f64x1 = f64x1::new(1.); + let e: f64x1 = f64x1::new(0.); + let r: f64x1 = transmute(vext_f64::<0>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vshld_n_s64() { + let a: i64 = 1; + let e: i64 = 4; + let r: i64 = vshld_n_s64::<2>(a); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vshld_n_u64() { + let a: u64 = 1; + let e: u64 = 4; + let r: u64 = vshld_n_u64::<2>(a); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vshrd_n_s64() { + let a: i64 = 4; + let e: i64 = 1; + let r: i64 = vshrd_n_s64::<2>(a); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vshrd_n_u64() { + let a: u64 = 4; + let e: u64 = 1; + let r: u64 = vshrd_n_u64::<2>(a); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsrad_n_s64() { + let a: i64 = 1; + let b: i64 = 4; + let e: i64 = 2; + let r: i64 = vsrad_n_s64::<2>(a, b); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsrad_n_u64() { + let a: u64 = 1; + let b: u64 = 4; + let e: u64 = 2; + let r: u64 = vsrad_n_u64::<2>(a, b); + assert_eq!(r, e); + } + macro_rules! test_vcombine { ($test_id:ident => $fn_id:ident ([$($a:expr),*], [$($b:expr),*])) => { #[allow(unused_assignments)] @@ -3434,6 +4123,190 @@ mod tests { test_vcombine!(test_vcombine_p64 => vcombine_p64([3_u64], [13_u64])); test_vcombine!(test_vcombine_f64 => vcombine_f64([-3_f64], [13_f64])); + #[simd_test(enable = "neon")] + unsafe fn test_vdup_n_f64() { + let a: f64 = 3.3; + let e = f64x1::new(3.3); + let r: f64x1 = transmute(vdup_n_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdup_n_p64() { + let a: u64 = 3; + let e = u64x1::new(3); + let r: u64x1 = transmute(vdup_n_p64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdupq_n_f64() { + let a: f64 = 3.3; + let e = f64x2::new(3.3, 3.3); + let r: f64x2 = transmute(vdupq_n_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdupq_n_p64() { + let a: u64 = 3; + let e = u64x2::new(3, 3); + let r: u64x2 = transmute(vdupq_n_p64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmov_n_p64() { + let a: u64 = 3; + let e = u64x1::new(3); + let r: u64x1 = transmute(vmov_n_p64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmov_n_f64() { + let a: f64 = 3.3; + let e = f64x1::new(3.3); + let r: f64x1 = transmute(vmov_n_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmovq_n_p64() { + let a: u64 = 3; + let e = u64x2::new(3, 3); + let r: u64x2 = transmute(vmovq_n_p64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmovq_n_f64() { + let a: f64 = 3.3; + let e = f64x2::new(3.3, 3.3); + let r: f64x2 = transmute(vmovq_n_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vget_high_f64() { + let a = f64x2::new(1.0, 2.0); + let e = f64x1::new(2.0); + let r: f64x1 = transmute(vget_high_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vget_high_p64() { + let a = u64x2::new(1, 2); + let e = u64x1::new(2); + let r: u64x1 = transmute(vget_high_p64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vget_low_f64() { + let a = f64x2::new(1.0, 2.0); + let e = f64x1::new(1.0); + let r: f64x1 = transmute(vget_low_f64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vget_low_p64() { + let a = u64x2::new(1, 2); + let e = u64x1::new(1); + let r: u64x1 = transmute(vget_low_p64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vget_lane_f64() { + let v = f64x1::new(1.0); + let r = vget_lane_f64::<0>(transmute(v)); + assert_eq!(r, 1.0); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vgetq_lane_f64() { + let v = f64x2::new(0.0, 1.0); + let r = vgetq_lane_f64::<1>(transmute(v)); + assert_eq!(r, 1.0); + let r = vgetq_lane_f64::<0>(transmute(v)); + assert_eq!(r, 0.0); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcopy_lane_s64() { + let a: i64x1 = i64x1::new(1); + let b: i64x1 = i64x1::new(0x7F_FF_FF_FF_FF_FF_FF_FF); + let e: i64x1 = i64x1::new(0x7F_FF_FF_FF_FF_FF_FF_FF); + let r: i64x1 = transmute(vcopy_lane_s64::<0, 0>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcopy_lane_u64() { + let a: u64x1 = u64x1::new(1); + let b: u64x1 = u64x1::new(0xFF_FF_FF_FF_FF_FF_FF_FF); + let e: u64x1 = u64x1::new(0xFF_FF_FF_FF_FF_FF_FF_FF); + let r: u64x1 = transmute(vcopy_lane_u64::<0, 0>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcopy_lane_p64() { + let a: i64x1 = i64x1::new(1); + let b: i64x1 = i64x1::new(0x7F_FF_FF_FF_FF_FF_FF_FF); + let e: i64x1 = i64x1::new(0x7F_FF_FF_FF_FF_FF_FF_FF); + let r: i64x1 = transmute(vcopy_lane_p64::<0, 0>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcopy_lane_f64() { + let a: f64 = 1.; + let b: f64 = 0.; + let e: f64 = 0.; + let r: f64 = transmute(vcopy_lane_f64::<0, 0>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcopy_laneq_s64() { + let a: i64x1 = i64x1::new(1); + let b: i64x2 = i64x2::new(0, 0x7F_FF_FF_FF_FF_FF_FF_FF); + let e: i64x1 = i64x1::new(0x7F_FF_FF_FF_FF_FF_FF_FF); + let r: i64x1 = transmute(vcopy_laneq_s64::<0, 1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcopy_laneq_u64() { + let a: u64x1 = u64x1::new(1); + let b: u64x2 = u64x2::new(0, 0xFF_FF_FF_FF_FF_FF_FF_FF); + let e: u64x1 = u64x1::new(0xFF_FF_FF_FF_FF_FF_FF_FF); + let r: u64x1 = transmute(vcopy_laneq_u64::<0, 1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcopy_laneq_p64() { + let a: i64x1 = i64x1::new(1); + let b: i64x2 = i64x2::new(0, 0x7F_FF_FF_FF_FF_FF_FF_FF); + let e: i64x1 = i64x1::new(0x7F_FF_FF_FF_FF_FF_FF_FF); + let r: i64x1 = transmute(vcopy_laneq_p64::<0, 1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcopy_laneq_f64() { + let a: f64 = 1.; + let b: f64x2 = f64x2::new(0., 0.5); + let e: f64 = 0.5; + let r: f64 = transmute(vcopy_laneq_f64::<0, 1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] unsafe fn test_vceq_u64() { test_cmp_u64( @@ -3911,6 +4784,43 @@ mod tests { assert_eq!(r, e); } + #[simd_test(enable = "neon")] + unsafe fn test_vbsl_f64() { + let a = u64x1::new(u64::MAX); + let b = f64x1::new(f64::MAX); + let c = f64x1::new(f64::MIN); + let e = f64x1::new(f64::MAX); + let r: f64x1 = transmute(vbsl_f64(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vbsl_p64() { + let a = u64x1::new(u64::MAX); + let b = u64x1::new(u64::MAX); + let c = u64x1::new(u64::MIN); + let e = u64x1::new(u64::MAX); + let r: u64x1 = transmute(vbsl_p64(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vbslq_f64() { + let a = u64x2::new(u64::MAX, 0); + let b = f64x2::new(f64::MAX, f64::MAX); + let c = f64x2::new(f64::MIN, f64::MIN); + let e = f64x2::new(f64::MAX, f64::MIN); + let r: f64x2 = transmute(vbslq_f64(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vbslq_p64() { + let a = u64x2::new(u64::MAX, 0); + let b = u64x2::new(u64::MAX, u64::MAX); + let c = u64x2::new(u64::MIN, u64::MIN); + let e = u64x2::new(u64::MAX, u64::MIN); + let r: u64x2 = transmute(vbslq_p64(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] unsafe fn test_vaddv_s16() { let a = i16x4::new(1, 2, 3, -4); @@ -4009,17 +4919,96 @@ mod tests { let e = 3_u64; assert_eq!(r, e); } + + #[simd_test(enable = "neon")] + unsafe fn test_vaddlv_s8() { + let a = i8x8::new(1, 2, 3, 4, 5, 6, 7, -8); + let r: i16 = vaddlv_s8(transmute(a)); + let e = 20_i16; + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vaddlv_u8() { + let a = u8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let r: u16 = vaddlv_u8(transmute(a)); + let e = 36_u16; + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vaddlvq_s8() { + let a = i8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, -16); + let r: i16 = vaddlvq_s8(transmute(a)); + let e = 104_i16; + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vaddlvq_u8() { + let a = u8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let r: u16 = vaddlvq_u8(transmute(a)); + let e = 136_u16; + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst1_p64() { + let mut vals = [0_u64; 2]; + let a = u64x1::new(1); + + vst1_p64(vals[1..].as_mut_ptr(), transmute(a)); + + assert_eq!(vals[0], 0); + assert_eq!(vals[1], 1); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst1q_p64() { + let mut vals = [0_u64; 3]; + let a = u64x2::new(1, 2); + + vst1q_p64(vals[1..].as_mut_ptr(), transmute(a)); + + assert_eq!(vals[0], 0); + assert_eq!(vals[1], 1); + assert_eq!(vals[2], 2); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst1_f64() { + let mut vals = [0_f64; 2]; + let a = f64x1::new(1.); + + vst1_f64(vals[1..].as_mut_ptr(), transmute(a)); + + assert_eq!(vals[0], 0.); + assert_eq!(vals[1], 1.); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vst1q_f64() { + let mut vals = [0_f64; 3]; + let a = f64x2::new(1., 2.); + + vst1q_f64(vals[1..].as_mut_ptr(), transmute(a)); + + assert_eq!(vals[0], 0.); + assert_eq!(vals[1], 1.); + assert_eq!(vals[2], 2.); + } } #[cfg(test)] #[cfg(target_endian = "little")] -#[path = "../../arm/neon/table_lookup_tests.rs"] +#[path = "../../arm_shared/neon/table_lookup_tests.rs"] mod table_lookup_tests; #[cfg(test)] -#[path = "../../arm/neon/shift_and_insert_tests.rs"] +#[path = "../../arm_shared/neon/shift_and_insert_tests.rs"] mod shift_and_insert_tests; #[cfg(test)] -#[path = "../../arm/neon/load_tests.rs"] +#[path = "../../arm_shared/neon/load_tests.rs"] mod load_tests; + +#[cfg(test)] +#[path = "../../arm_shared/neon/store_tests.rs"] +mod store_tests; diff --git a/library/stdarch/crates/core_arch/src/aarch64/prefetch.rs b/library/stdarch/crates/core_arch/src/aarch64/prefetch.rs index 0c50be5ec0..02b3971aa4 100644 --- a/library/stdarch/crates/core_arch/src/aarch64/prefetch.rs +++ b/library/stdarch/crates/core_arch/src/aarch64/prefetch.rs @@ -24,9 +24,9 @@ pub const _PREFETCH_LOCALITY2: i32 = 2; /// See [`prefetch`](fn._prefetch.html). pub const _PREFETCH_LOCALITY3: i32 = 3; -/// Fetch the cache line that contains address `p` using the given `rw` and `locality`. +/// Fetch the cache line that contains address `p` using the given `RW` and `LOCALITY`. /// -/// The `rw` must be one of: +/// The `RW` must be one of: /// /// * [`_PREFETCH_READ`](constant._PREFETCH_READ.html): the prefetch is preparing /// for a read. @@ -34,7 +34,7 @@ pub const _PREFETCH_LOCALITY3: i32 = 3; /// * [`_PREFETCH_WRITE`](constant._PREFETCH_WRITE.html): the prefetch is preparing /// for a write. /// -/// The `locality` must be one of: +/// The `LOCALITY` must be one of: /// /// * [`_PREFETCH_LOCALITY0`](constant._PREFETCH_LOCALITY0.html): Streaming or /// non-temporal prefetch, for data that is used only once. @@ -55,35 +55,19 @@ pub const _PREFETCH_LOCALITY3: i32 = 3; /// /// [Arm's documentation](https://developer.arm.com/documentation/den0024/a/the-a64-instruction-set/memory-access-instructions/prefetching-memory?lang=en) #[inline(always)] -#[cfg_attr(test, assert_instr("prfm pldl1strm", rw = _PREFETCH_READ, locality = _PREFETCH_LOCALITY0))] -#[cfg_attr(test, assert_instr("prfm pldl3keep", rw = _PREFETCH_READ, locality = _PREFETCH_LOCALITY1))] -#[cfg_attr(test, assert_instr("prfm pldl2keep", rw = _PREFETCH_READ, locality = _PREFETCH_LOCALITY2))] -#[cfg_attr(test, assert_instr("prfm pldl1keep", rw = _PREFETCH_READ, locality = _PREFETCH_LOCALITY3))] -#[cfg_attr(test, assert_instr("prfm pstl1strm", rw = _PREFETCH_WRITE, locality = _PREFETCH_LOCALITY0))] -#[cfg_attr(test, assert_instr("prfm pstl3keep", rw = _PREFETCH_WRITE, locality = _PREFETCH_LOCALITY1))] -#[cfg_attr(test, assert_instr("prfm pstl2keep", rw = _PREFETCH_WRITE, locality = _PREFETCH_LOCALITY2))] -#[cfg_attr(test, assert_instr("prfm pstl1keep", rw = _PREFETCH_WRITE, locality = _PREFETCH_LOCALITY3))] -#[rustc_args_required_const(1, 2)] -pub unsafe fn _prefetch(p: *const i8, rw: i32, locality: i32) { +#[cfg_attr(test, assert_instr("prfm pldl1strm", RW = _PREFETCH_READ, LOCALITY = _PREFETCH_LOCALITY0))] +#[cfg_attr(test, assert_instr("prfm pldl3keep", RW = _PREFETCH_READ, LOCALITY = _PREFETCH_LOCALITY1))] +#[cfg_attr(test, assert_instr("prfm pldl2keep", RW = _PREFETCH_READ, LOCALITY = _PREFETCH_LOCALITY2))] +#[cfg_attr(test, assert_instr("prfm pldl1keep", RW = _PREFETCH_READ, LOCALITY = _PREFETCH_LOCALITY3))] +#[cfg_attr(test, assert_instr("prfm pstl1strm", RW = _PREFETCH_WRITE, LOCALITY = _PREFETCH_LOCALITY0))] +#[cfg_attr(test, assert_instr("prfm pstl3keep", RW = _PREFETCH_WRITE, LOCALITY = _PREFETCH_LOCALITY1))] +#[cfg_attr(test, assert_instr("prfm pstl2keep", RW = _PREFETCH_WRITE, LOCALITY = _PREFETCH_LOCALITY2))] +#[cfg_attr(test, assert_instr("prfm pstl1keep", RW = _PREFETCH_WRITE, LOCALITY = _PREFETCH_LOCALITY3))] +#[rustc_legacy_const_generics(1, 2)] +// FIXME: Replace this with the standard ACLE __pld/__pldx/__pli/__plix intrinsics +pub unsafe fn _prefetch(p: *const i8) { // We use the `llvm.prefetch` instrinsic with `cache type` = 1 (data cache). - // `rw` and `strategy` are based on the function parameters. - macro_rules! pref { - ($rdwr:expr, $local:expr) => { - match ($rdwr, $local) { - (0, 0) => prefetch(p, 0, 0, 1), - (0, 1) => prefetch(p, 0, 1, 1), - (0, 2) => prefetch(p, 0, 2, 1), - (0, 3) => prefetch(p, 0, 3, 1), - (1, 0) => prefetch(p, 1, 0, 1), - (1, 1) => prefetch(p, 1, 1, 1), - (1, 2) => prefetch(p, 1, 2, 1), - (1, 3) => prefetch(p, 1, 3, 1), - (_, _) => panic!( - "Illegal (rw, locality) pair in prefetch, value ({}, {}).", - $rdwr, $local - ), - } - }; - } - pref!(rw, locality); + static_assert_imm1!(RW); + static_assert_imm2!(LOCALITY); + prefetch(p, RW, LOCALITY, 1); } diff --git a/library/stdarch/crates/core_arch/src/aarch64/test_support.rs b/library/stdarch/crates/core_arch/src/aarch64/test_support.rs index e08c39a545..9c5994b150 100644 --- a/library/stdarch/crates/core_arch/src/aarch64/test_support.rs +++ b/library/stdarch/crates/core_arch/src/aarch64/test_support.rs @@ -1,4 +1,4 @@ -use crate::core_arch::{aarch64::neon::*, arm::*, simd::*}; +use crate::core_arch::{aarch64::neon::*, arm_shared::*, simd::*}; use std::{i16, i32, i8, mem::transmute, u16, u32, u8, vec::Vec}; macro_rules! V_u64 { diff --git a/library/stdarch/crates/core_arch/src/aarch64/tme.rs b/library/stdarch/crates/core_arch/src/aarch64/tme.rs index f0ff867813..edf87b8034 100644 --- a/library/stdarch/crates/core_arch/src/aarch64/tme.rs +++ b/library/stdarch/crates/core_arch/src/aarch64/tme.rs @@ -93,15 +93,11 @@ pub unsafe fn __tcommit() { /// [ARM TME Intrinsics](https://developer.arm.com/docs/101028/0010/transactional-memory-extension-tme-intrinsics). #[inline] #[target_feature(enable = "tme")] -#[cfg_attr(test, assert_instr(tcancel, imm0 = 0x0))] -#[rustc_args_required_const(0)] -pub unsafe fn __tcancel(imm0: u64) { - macro_rules! call { - ($imm0:expr) => { - aarch64_tcancel($imm0) - }; - } - constify_imm8!(imm0, call) +#[cfg_attr(test, assert_instr(tcancel, IMM16 = 0x0))] +#[rustc_legacy_const_generics(0)] +pub unsafe fn __tcancel() { + static_assert!(IMM16: u64 where IMM16 <= 65535); + aarch64_tcancel(IMM16); } /// Tests if executing inside a transaction. If no transaction is currently executing, @@ -160,7 +156,7 @@ mod tests { if code == _TMSTART_SUCCESS { x += 1; assert_eq!(x, i + 1); - tme::__tcancel(CANCEL_CODE); + tme::__tcancel::(); break; } } @@ -174,7 +170,7 @@ mod tests { let code = tme::__tstart(); if code == _TMSTART_SUCCESS { if tme::__ttest() == 2 { - tme::__tcancel(CANCEL_CODE); + tme::__tcancel::(); break; } } diff --git a/library/stdarch/crates/core_arch/src/arm/armclang.rs b/library/stdarch/crates/core_arch/src/arm/armclang.rs index 2604727984..6b332e82c0 100644 --- a/library/stdarch/crates/core_arch/src/arm/armclang.rs +++ b/library/stdarch/crates/core_arch/src/arm/armclang.rs @@ -11,58 +11,25 @@ use stdarch_test::assert_instr; /// Inserts a breakpoint instruction. /// -/// `val` is a compile-time constant integer in range `[0, 255]`. +/// `VAL` is a compile-time constant integer in range `[0, 255]`. /// -/// The breakpoint instruction inserted is: -/// -/// * `BKPT` when compiling as T32, -/// * `BRK` when compiling as A32 or A64. -/// -/// # Safety -/// -/// If `val` is out-of-range the behavior is **undefined**. +/// The breakpoint instruction inserted is `BKPT` on A32/T32. /// /// # Note /// /// [ARM's documentation][arm_docs] defines that `__breakpoint` accepts the -/// following values for `val`: +/// following values for `VAL`: /// -/// - `0...65535` when compiling as A32 or A64, +/// - `0...65535` when compiling as A32, /// - `0...255` when compiling as T32. /// -/// The current implementation only accepts values in range `[0, 255]` - if the -/// value is out-of-range the behavior is **undefined**. +/// The current implementation only accepts values in range `[0, 255]`. /// /// [arm_docs]: https://developer.arm.com/docs/100067/latest/compiler-specific-intrinsics/__breakpoint-intrinsic -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(bkpt, val = 0))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(brk, val = 0))] +#[cfg_attr(test, assert_instr(bkpt, VAL = 0))] #[inline(always)] -#[rustc_args_required_const(0)] -pub unsafe fn __breakpoint(val: i32) { - // Ensure that this compiles correctly on non-arm architectures, so libstd - // doc builds work. The proper macro will shadow this definition below. - #[allow(unused_macros)] - macro_rules! call { - ($e:expr) => { - () - }; - } - - #[cfg(target_arch = "arm")] - macro_rules! call { - ($imm8:expr) => { - llvm_asm!(concat!("BKPT ", stringify!($imm8)) : : : : "volatile") - } - } - - #[cfg(target_arch = "aarch64")] - macro_rules! call { - ($imm8:expr) => { - llvm_asm!(concat!("BRK ", stringify!($imm8)) : : : : "volatile") - } - } - - // We can't `panic!` inside this intrinsic, so we can't really validate the - // arguments here. If `val` is out-of-range this macro uses `val == 255`: - constify_imm8!(val, call); +#[rustc_legacy_const_generics(0)] +pub unsafe fn __breakpoint() { + static_assert_imm8!(VAL); + asm!("bkpt #{}", const VAL); } diff --git a/library/stdarch/crates/core_arch/src/acle/dsp.rs b/library/stdarch/crates/core_arch/src/arm/dsp.rs similarity index 100% rename from library/stdarch/crates/core_arch/src/acle/dsp.rs rename to library/stdarch/crates/core_arch/src/arm/dsp.rs diff --git a/library/stdarch/crates/core_arch/src/acle/ex.rs b/library/stdarch/crates/core_arch/src/arm/ex.rs similarity index 95% rename from library/stdarch/crates/core_arch/src/acle/ex.rs rename to library/stdarch/crates/core_arch/src/arm/ex.rs index 0426c65186..b9d5047a05 100644 --- a/library/stdarch/crates/core_arch/src/acle/ex.rs +++ b/library/stdarch/crates/core_arch/src/arm/ex.rs @@ -8,6 +8,7 @@ #[cfg(any( all(target_feature = "v6k", not(target_feature = "mclass")), // excludes v6-M all(target_feature = "v7", target_feature = "mclass"), // v7-M + doc ))] pub unsafe fn __clrex() { extern "C" { @@ -21,9 +22,10 @@ pub unsafe fn __clrex() { /// Executes a exclusive LDR instruction for 8 bit value. // Supported: v6K, v7-M, v7-A, v7-R // Not supported: v5, v6, v6-M -#[cfg( +#[cfg(any( target_feature = "v6k", // includes v7-M but excludes v6-M -)] + doc +))] pub unsafe fn __ldrexb(p: *const u8) -> u8 { extern "C" { #[link_name = "llvm.arm.ldrex.p0i8"] @@ -36,9 +38,10 @@ pub unsafe fn __ldrexb(p: *const u8) -> u8 { /// Executes a exclusive LDR instruction for 16 bit value. // Supported: v6K, v7-M, v7-A, v7-R, v8 // Not supported: v5, v6, v6-M -#[cfg( +#[cfg(any( target_feature = "v6k", // includes v7-M but excludes v6-M -)] + doc +))] pub unsafe fn __ldrexh(p: *const u16) -> u16 { extern "C" { #[link_name = "llvm.arm.ldrex.p0i16"] @@ -54,6 +57,7 @@ pub unsafe fn __ldrexh(p: *const u16) -> u16 { #[cfg(any( all(target_feature = "v6", not(target_feature = "mclass")), // excludes v6-M all(target_feature = "v7", target_feature = "mclass"), // v7-M + doc ))] pub unsafe fn __ldrex(p: *const u32) -> u32 { extern "C" { @@ -69,9 +73,10 @@ pub unsafe fn __ldrex(p: *const u32) -> u32 { /// Returns `0` if the operation succeeded, or `1` if it failed // supported: v6K, v7-M, v7-A, v7-R // Not supported: v5, v6, v6-M -#[cfg( +#[cfg(any( target_feature = "v6k", // includes v7-M but excludes v6-M -)] + doc +))] pub unsafe fn __strexb(value: u32, addr: *mut u8) -> u32 { extern "C" { #[link_name = "llvm.arm.strex.p0i8"] @@ -86,9 +91,11 @@ pub unsafe fn __strexb(value: u32, addr: *mut u8) -> u32 { /// Returns `0` if the operation succeeded, or `1` if it failed // Supported: v6K, v7-M, v7-A, v7-R, v8 // Not supported: v5, v6, v6-M -#[cfg( +#[cfg(target_feature = "aarch64")] +#[cfg(any( target_feature = "v6k", // includes v7-M but excludes v6-M -)] + doc +))] pub unsafe fn __strexh(value: u16, addr: *mut u16) -> u32 { extern "C" { #[link_name = "llvm.arm.strex.p0i16"] @@ -106,6 +113,7 @@ pub unsafe fn __strexh(value: u16, addr: *mut u16) -> u32 { #[cfg(any( all(target_feature = "v6", not(target_feature = "mclass")), // excludes v6-M all(target_feature = "v7", target_feature = "mclass"), // v7-M + doc ))] pub unsafe fn __strex(value: u32, addr: *mut u32) -> u32 { extern "C" { diff --git a/library/stdarch/crates/core_arch/src/arm/mod.rs b/library/stdarch/crates/core_arch/src/arm/mod.rs index fd0cb2cf8d..d6b12b8292 100644 --- a/library/stdarch/crates/core_arch/src/arm/mod.rs +++ b/library/stdarch/crates/core_arch/src/arm/mod.rs @@ -5,40 +5,81 @@ //! //! [arm_ref]: http://infocenter.arm.com/help/topic/com.arm.doc.ihi0073a/IHI0073A_arm_neon_intrinsics_ref.pdf //! [arm_dat]: https://developer.arm.com/technologies/neon/intrinsics -#![allow(non_camel_case_types)] mod armclang; - pub use self::armclang::*; mod v6; pub use self::v6::*; -#[cfg(any(target_arch = "aarch64", target_feature = "v7"))] -mod v7; -#[cfg(any(target_arch = "aarch64", target_feature = "v7"))] -pub use self::v7::*; +// Supported arches: 6, 7-M. See Section 10.1 of ACLE (e.g. SSAT) +#[cfg(any(target_feature = "v6", doc))] +mod sat; + +#[cfg(any(target_feature = "v6", doc))] +pub use self::sat::*; + +// Supported arches: 5TE, 7E-M. See Section 10.1 of ACLE (e.g. QADD) +// We also include the A profile even though DSP is deprecated on that profile as of ACLE 2.0 (see +// section 5.4.7) +// Here we workaround the difference between LLVM's +dsp and ACLE's __ARM_FEATURE_DSP by gating on +// '+v5te' rather than on '+dsp' +#[cfg(any( + // >= v5TE but excludes v7-M + all(target_feature = "v5te", not(target_feature = "mclass")), + // v7E-M + all(target_feature = "mclass", target_feature = "dsp"), + doc, +))] +pub mod dsp; + +#[cfg(any( + // >= v5TE but excludes v7-M + all(target_feature = "v5te", not(target_feature = "mclass")), + // v7E-M + all(target_feature = "mclass", target_feature = "dsp"), + doc, +))] +pub use self::dsp::*; + +// Deprecated in ACLE 2.0 for the A profile but fully supported on the M and R profiles, says +// Section 5.4.9 of ACLE. We'll expose these for the A profile even if deprecated +#[cfg(any( + // v7-A, v7-R + all(target_feature = "v6", not(target_feature = "mclass")), + // v7E-M + all(target_feature = "mclass", target_feature = "dsp"), + doc, +))] +mod simd32; -#[cfg(any(target_arch = "aarch64", target_feature = "v7", doc))] -mod neon; -#[cfg(any(target_arch = "aarch64", target_feature = "v7", doc))] -pub use self::neon::*; +#[cfg(any( + // v7-A, v7-R + all(target_feature = "v6", not(target_feature = "mclass")), + // v7E-M + all(target_feature = "mclass", target_feature = "dsp"), + doc, +))] +pub use self::simd32::*; -#[cfg(any(target_arch = "aarch64", target_feature = "v7"))] -mod crc; -#[cfg(any(target_arch = "aarch64", target_feature = "v7"))] -pub use self::crc::*; +#[cfg(any(target_feature = "v7", doc))] +mod v7; +#[cfg(any(target_feature = "v7", doc))] +pub use self::v7::*; -#[cfg(any(target_arch = "aarch64", target_feature = "v7"))] -mod crypto; -#[cfg(any(target_arch = "aarch64", target_feature = "v7"))] -pub use self::crypto::*; +mod ex; +pub use self::ex::*; -pub use crate::core_arch::acle::*; +pub use crate::core_arch::arm_shared::*; #[cfg(test)] use stdarch_test::assert_instr; +#[cfg(any(target_feature = "v7", doc))] +pub(crate) mod neon; +#[cfg(any(target_feature = "v7", doc))] +pub use neon::*; + /// Generates the trap instruction `UDF` #[cfg(target_arch = "arm")] #[cfg_attr(test, assert_instr(udf))] @@ -47,6 +88,26 @@ pub unsafe fn udf() -> ! { crate::intrinsics::abort() } -#[cfg(test)] -#[cfg(any(target_arch = "aarch64", target_feature = "v7"))] -pub(crate) mod test_support; +/// Generates a DBG instruction. +/// +/// This provides a hint to debugging and related systems. The argument must be +/// a constant integer from 0 to 15 inclusive. See implementation documentation +/// for the effect (if any) of this instruction and the meaning of the +/// argument. This is available only when compliling for AArch32. +// Section 10.1 of ACLE says that the supported arches are: 7, 7-M +// "The DBG hint instruction is added in ARMv7. It is UNDEFINED in the ARMv6 base architecture, and +// executes as a NOP instruction in ARMv6K and ARMv6T2." - ARM Architecture Reference Manual ARMv7-A +// and ARMv7-R edition (ARM DDI 0406C.c) sections D12.4.1 "ARM instruction set support" and D12.4.2 +// "Thumb instruction set support" +#[cfg(any(target_feature = "v7", doc))] +#[inline(always)] +#[rustc_legacy_const_generics(0)] +pub unsafe fn __dbg() { + static_assert_imm4!(IMM4); + dbg(IMM4); +} + +extern "C" { + #[link_name = "llvm.arm.dbg"] + fn dbg(_: i32); +} diff --git a/library/stdarch/crates/core_arch/src/arm/neon.rs b/library/stdarch/crates/core_arch/src/arm/neon.rs new file mode 100644 index 0000000000..473c753fd6 --- /dev/null +++ b/library/stdarch/crates/core_arch/src/arm/neon.rs @@ -0,0 +1,1286 @@ +use crate::core_arch::arm_shared::neon::*; +use crate::core_arch::simd::{f32x4, i32x4, u32x4}; +use crate::core_arch::simd_llvm::*; +use crate::mem::{align_of, transmute}; + +#[cfg(test)] +use stdarch_test::assert_instr; + +#[allow(non_camel_case_types)] +pub(crate) type p8 = u8; +#[allow(non_camel_case_types)] +pub(crate) type p16 = u16; + +#[allow(improper_ctypes)] +extern "C" { + #[link_name = "llvm.arm.neon.vbsl.v8i8"] + fn vbsl_s8_(a: int8x8_t, b: int8x8_t, c: int8x8_t) -> int8x8_t; + #[link_name = "llvm.arm.neon.vbsl.v16i8"] + fn vbslq_s8_(a: int8x16_t, b: int8x16_t, c: int8x16_t) -> int8x16_t; + #[link_name = "llvm.arm.neon.vpadals.v4i16.v8i8"] + pub(crate) fn vpadal_s8_(a: int16x4_t, b: int8x8_t) -> int16x4_t; + #[link_name = "llvm.arm.neon.vpadals.v2i32.v4i16"] + pub(crate) fn vpadal_s16_(a: int32x2_t, b: int16x4_t) -> int32x2_t; + #[link_name = "llvm.arm.neon.vpadals.v1i64.v2i32"] + pub(crate) fn vpadal_s32_(a: int64x1_t, b: int32x2_t) -> int64x1_t; + #[link_name = "llvm.arm.neon.vpadals.v8i16.v16i8"] + pub(crate) fn vpadalq_s8_(a: int16x8_t, b: int8x16_t) -> int16x8_t; + #[link_name = "llvm.arm.neon.vpadals.v4i32.v8i16"] + pub(crate) fn vpadalq_s16_(a: int32x4_t, b: int16x8_t) -> int32x4_t; + #[link_name = "llvm.arm.neon.vpadals.v2i64.v4i32"] + pub(crate) fn vpadalq_s32_(a: int64x2_t, b: int32x4_t) -> int64x2_t; + + #[link_name = "llvm.arm.neon.vpadalu.v4i16.v8i8"] + pub(crate) fn vpadal_u8_(a: uint16x4_t, b: uint8x8_t) -> uint16x4_t; + #[link_name = "llvm.arm.neon.vpadalu.v2i32.v4i16"] + pub(crate) fn vpadal_u16_(a: uint32x2_t, b: uint16x4_t) -> uint32x2_t; + #[link_name = "llvm.arm.neon.vpadalu.v1i64.v2i32"] + pub(crate) fn vpadal_u32_(a: uint64x1_t, b: uint32x2_t) -> uint64x1_t; + #[link_name = "llvm.arm.neon.vpadalu.v8i16.v16i8"] + pub(crate) fn vpadalq_u8_(a: uint16x8_t, b: uint8x16_t) -> uint16x8_t; + #[link_name = "llvm.arm.neon.vpadalu.v4i32.v8i16"] + pub(crate) fn vpadalq_u16_(a: uint32x4_t, b: uint16x8_t) -> uint32x4_t; + #[link_name = "llvm.arm.neon.vpadalu.v2i64.v4i32"] + pub(crate) fn vpadalq_u32_(a: uint64x2_t, b: uint32x4_t) -> uint64x2_t; + + #[link_name = "llvm.arm.neon.vtbl1"] + fn vtbl1(a: int8x8_t, b: int8x8_t) -> int8x8_t; + #[link_name = "llvm.arm.neon.vtbl2"] + fn vtbl2(a: int8x8_t, b: int8x8_t, b: int8x8_t) -> int8x8_t; + #[link_name = "llvm.arm.neon.vtbl3"] + fn vtbl3(a: int8x8_t, b: int8x8_t, b: int8x8_t, c: int8x8_t) -> int8x8_t; + #[link_name = "llvm.arm.neon.vtbl4"] + fn vtbl4(a: int8x8_t, b: int8x8_t, b: int8x8_t, c: int8x8_t, d: int8x8_t) -> int8x8_t; + + #[link_name = "llvm.arm.neon.vtbx1"] + fn vtbx1(a: int8x8_t, b: int8x8_t, b: int8x8_t) -> int8x8_t; + #[link_name = "llvm.arm.neon.vtbx2"] + fn vtbx2(a: int8x8_t, b: int8x8_t, b: int8x8_t, c: int8x8_t) -> int8x8_t; + #[link_name = "llvm.arm.neon.vtbx3"] + fn vtbx3(a: int8x8_t, b: int8x8_t, b: int8x8_t, c: int8x8_t, d: int8x8_t) -> int8x8_t; + #[link_name = "llvm.arm.neon.vtbx4"] + fn vtbx4( + a: int8x8_t, + b: int8x8_t, + b: int8x8_t, + c: int8x8_t, + d: int8x8_t, + e: int8x8_t, + ) -> int8x8_t; + + #[link_name = "llvm.arm.neon.vshiftins.v8i8"] + fn vshiftins_v8i8(a: int8x8_t, b: int8x8_t, shift: int8x8_t) -> int8x8_t; + #[link_name = "llvm.arm.neon.vshiftins.v16i8"] + fn vshiftins_v16i8(a: int8x16_t, b: int8x16_t, shift: int8x16_t) -> int8x16_t; + #[link_name = "llvm.arm.neon.vshiftins.v4i16"] + fn vshiftins_v4i16(a: int16x4_t, b: int16x4_t, shift: int16x4_t) -> int16x4_t; + #[link_name = "llvm.arm.neon.vshiftins.v8i16"] + fn vshiftins_v8i16(a: int16x8_t, b: int16x8_t, shift: int16x8_t) -> int16x8_t; + #[link_name = "llvm.arm.neon.vshiftins.v2i32"] + fn vshiftins_v2i32(a: int32x2_t, b: int32x2_t, shift: int32x2_t) -> int32x2_t; + #[link_name = "llvm.arm.neon.vshiftins.v4i32"] + fn vshiftins_v4i32(a: int32x4_t, b: int32x4_t, shift: int32x4_t) -> int32x4_t; + #[link_name = "llvm.arm.neon.vshiftins.v1i64"] + fn vshiftins_v1i64(a: int64x1_t, b: int64x1_t, shift: int64x1_t) -> int64x1_t; + #[link_name = "llvm.arm.neon.vshiftins.v2i64"] + fn vshiftins_v2i64(a: int64x2_t, b: int64x2_t, shift: int64x2_t) -> int64x2_t; + + #[link_name = "llvm.arm.neon.vld1.v8i8.p0i8"] + fn vld1_v8i8(addr: *const i8, align: i32) -> int8x8_t; + #[link_name = "llvm.arm.neon.vld1.v16i8.p0i8"] + fn vld1q_v16i8(addr: *const i8, align: i32) -> int8x16_t; + #[link_name = "llvm.arm.neon.vld1.v4i16.p0i8"] + fn vld1_v4i16(addr: *const i8, align: i32) -> int16x4_t; + #[link_name = "llvm.arm.neon.vld1.v8i16.p0i8"] + fn vld1q_v8i16(addr: *const i8, align: i32) -> int16x8_t; + #[link_name = "llvm.arm.neon.vld1.v2i32.p0i8"] + fn vld1_v2i32(addr: *const i8, align: i32) -> int32x2_t; + #[link_name = "llvm.arm.neon.vld1.v4i32.p0i8"] + fn vld1q_v4i32(addr: *const i8, align: i32) -> int32x4_t; + #[link_name = "llvm.arm.neon.vld1.v1i64.p0i8"] + fn vld1_v1i64(addr: *const i8, align: i32) -> int64x1_t; + #[link_name = "llvm.arm.neon.vld1.v2i64.p0i8"] + fn vld1q_v2i64(addr: *const i8, align: i32) -> int64x2_t; + #[link_name = "llvm.arm.neon.vld1.v2f32.p0i8"] + fn vld1_v2f32(addr: *const i8, align: i32) -> float32x2_t; + #[link_name = "llvm.arm.neon.vld1.v4f32.p0i8"] + fn vld1q_v4f32(addr: *const i8, align: i32) -> float32x4_t; + + #[link_name = "llvm.arm.neon.vst1.p0i8.v8i8"] + fn vst1_v8i8(addr: *const i8, val: int8x8_t, align: i32); + #[link_name = "llvm.arm.neon.vst1.p0i8.v16i8"] + fn vst1q_v16i8(addr: *const i8, val: int8x16_t, align: i32); + #[link_name = "llvm.arm.neon.vst1.p0i8.v4i16"] + fn vst1_v4i16(addr: *const i8, val: int16x4_t, align: i32); + #[link_name = "llvm.arm.neon.vst1.p0i8.v8i16"] + fn vst1q_v8i16(addr: *const i8, val: int16x8_t, align: i32); + #[link_name = "llvm.arm.neon.vst1.p0i8.v2i32"] + fn vst1_v2i32(addr: *const i8, val: int32x2_t, align: i32); + #[link_name = "llvm.arm.neon.vst1.p0i8.v4i32"] + fn vst1q_v4i32(addr: *const i8, val: int32x4_t, align: i32); + #[link_name = "llvm.arm.neon.vst1.p0i8.v1i64"] + fn vst1_v1i64(addr: *const i8, val: int64x1_t, align: i32); + #[link_name = "llvm.arm.neon.vst1.p0i8.v2i64"] + fn vst1q_v2i64(addr: *const i8, val: int64x2_t, align: i32); + #[link_name = "llvm.arm.neon.vst1.p0i8.v2f32"] + fn vst1_v2f32(addr: *const i8, val: float32x2_t, align: i32); + #[link_name = "llvm.arm.neon.vst1.p0i8.v4f32"] + fn vst1q_v4f32(addr: *const i8, val: float32x4_t, align: i32); +} + +/// Load multiple single-element structures to one, two, three, or four registers. +#[inline] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr("vld1.8"))] +pub unsafe fn vld1_s8(ptr: *const i8) -> int8x8_t { + vld1_v8i8(ptr as *const i8, align_of::() as i32) +} + +/// Load multiple single-element structures to one, two, three, or four registers. +#[inline] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr("vld1.8"))] +pub unsafe fn vld1q_s8(ptr: *const i8) -> int8x16_t { + vld1q_v16i8(ptr as *const i8, align_of::() as i32) +} + +/// Load multiple single-element structures to one, two, three, or four registers. +#[inline] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr("vld1.16"))] +pub unsafe fn vld1_s16(ptr: *const i16) -> int16x4_t { + vld1_v4i16(ptr as *const i8, align_of::() as i32) +} + +/// Load multiple single-element structures to one, two, three, or four registers. +#[inline] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr("vld1.16"))] +pub unsafe fn vld1q_s16(ptr: *const i16) -> int16x8_t { + vld1q_v8i16(ptr as *const i8, align_of::() as i32) +} + +/// Load multiple single-element structures to one, two, three, or four registers. +#[inline] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vldr))] +pub unsafe fn vld1_s32(ptr: *const i32) -> int32x2_t { + vld1_v2i32(ptr as *const i8, align_of::() as i32) +} + +/// Load multiple single-element structures to one, two, three, or four registers. +#[inline] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr("vld1.32"))] +pub unsafe fn vld1q_s32(ptr: *const i32) -> int32x4_t { + vld1q_v4i32(ptr as *const i8, align_of::() as i32) +} + +/// Load multiple single-element structures to one, two, three, or four registers. +#[inline] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vldr))] +pub unsafe fn vld1_s64(ptr: *const i64) -> int64x1_t { + vld1_v1i64(ptr as *const i8, align_of::() as i32) +} + +/// Load multiple single-element structures to one, two, three, or four registers. +#[inline] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr("vld1.64"))] +pub unsafe fn vld1q_s64(ptr: *const i64) -> int64x2_t { + vld1q_v2i64(ptr as *const i8, align_of::() as i32) +} + +/// Load multiple single-element structures to one, two, three, or four registers. +#[inline] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr("vld1.8"))] +pub unsafe fn vld1_u8(ptr: *const u8) -> uint8x8_t { + transmute(vld1_v8i8(ptr as *const i8, align_of::() as i32)) +} + +/// Load multiple single-element structures to one, two, three, or four registers. +#[inline] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr("vld1.8"))] +pub unsafe fn vld1q_u8(ptr: *const u8) -> uint8x16_t { + transmute(vld1q_v16i8(ptr as *const i8, align_of::() as i32)) +} + +/// Load multiple single-element structures to one, two, three, or four registers. +#[inline] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr("vld1.16"))] +pub unsafe fn vld1_u16(ptr: *const u16) -> uint16x4_t { + transmute(vld1_v4i16(ptr as *const i8, align_of::() as i32)) +} + +/// Load multiple single-element structures to one, two, three, or four registers. +#[inline] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr("vld1.16"))] +pub unsafe fn vld1q_u16(ptr: *const u16) -> uint16x8_t { + transmute(vld1q_v8i16(ptr as *const i8, align_of::() as i32)) +} + +/// Load multiple single-element structures to one, two, three, or four registers. +#[inline] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vldr))] +pub unsafe fn vld1_u32(ptr: *const u32) -> uint32x2_t { + transmute(vld1_v2i32(ptr as *const i8, align_of::() as i32)) +} + +/// Load multiple single-element structures to one, two, three, or four registers. +#[inline] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr("vld1.32"))] +pub unsafe fn vld1q_u32(ptr: *const u32) -> uint32x4_t { + transmute(vld1q_v4i32(ptr as *const i8, align_of::() as i32)) +} + +/// Load multiple single-element structures to one, two, three, or four registers. +#[inline] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vldr))] +pub unsafe fn vld1_u64(ptr: *const u64) -> uint64x1_t { + transmute(vld1_v1i64(ptr as *const i8, align_of::() as i32)) +} + +/// Load multiple single-element structures to one, two, three, or four registers. +#[inline] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr("vld1.64"))] +pub unsafe fn vld1q_u64(ptr: *const u64) -> uint64x2_t { + transmute(vld1q_v2i64(ptr as *const i8, align_of::() as i32)) +} + +/// Load multiple single-element structures to one, two, three, or four registers. +#[inline] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr("vld1.8"))] +pub unsafe fn vld1_p8(ptr: *const p8) -> poly8x8_t { + transmute(vld1_v8i8(ptr as *const i8, align_of::() as i32)) +} + +/// Load multiple single-element structures to one, two, three, or four registers. +#[inline] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr("vld1.8"))] +pub unsafe fn vld1q_p8(ptr: *const p8) -> poly8x16_t { + transmute(vld1q_v16i8(ptr as *const i8, align_of::() as i32)) +} + +/// Load multiple single-element structures to one, two, three, or four registers. +#[inline] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr("vld1.16"))] +pub unsafe fn vld1_p16(ptr: *const p16) -> poly16x4_t { + transmute(vld1_v4i16(ptr as *const i8, align_of::() as i32)) +} + +/// Load multiple single-element structures to one, two, three, or four registers. +#[inline] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr("vld1.16"))] +pub unsafe fn vld1q_p16(ptr: *const p16) -> poly16x8_t { + transmute(vld1q_v8i16(ptr as *const i8, align_of::() as i32)) +} + +/// Load multiple single-element structures to one, two, three, or four registers. +#[inline] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vldr))] +pub unsafe fn vld1_f32(ptr: *const f32) -> float32x2_t { + vld1_v2f32(ptr as *const i8, align_of::() as i32) +} + +/// Load multiple single-element structures to one, two, three, or four registers. +#[inline] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr("vld1.32"))] +pub unsafe fn vld1q_f32(ptr: *const f32) -> float32x4_t { + vld1q_v4f32(ptr as *const i8, align_of::() as i32) +} + +/// Store multiple single-element structures from one, two, three, or four registers. +#[inline] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(str))] +pub unsafe fn vst1_s8(ptr: *mut i8, a: int8x8_t) { + vst1_v8i8(ptr as *const i8, a, align_of::() as i32) +} + +/// Store multiple single-element structures from one, two, three, or four registers. +#[inline] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(str))] +pub unsafe fn vst1q_s8(ptr: *mut i8, a: int8x16_t) { + vst1q_v16i8(ptr as *const i8, a, align_of::() as i32) +} + +/// Store multiple single-element structures from one, two, three, or four registers. +#[inline] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(str))] +pub unsafe fn vst1_s16(ptr: *mut i16, a: int16x4_t) { + vst1_v4i16(ptr as *const i8, a, align_of::() as i32) +} + +/// Store multiple single-element structures from one, two, three, or four registers. +#[inline] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(str))] +pub unsafe fn vst1q_s16(ptr: *mut i16, a: int16x8_t) { + vst1q_v8i16(ptr as *const i8, a, align_of::() as i32) +} + +/// Store multiple single-element structures from one, two, three, or four registers. +#[inline] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(str))] +pub unsafe fn vst1_s32(ptr: *mut i32, a: int32x2_t) { + vst1_v2i32(ptr as *const i8, a, align_of::() as i32) +} + +/// Store multiple single-element structures from one, two, three, or four registers. +#[inline] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(str))] +pub unsafe fn vst1q_s32(ptr: *mut i32, a: int32x4_t) { + vst1q_v4i32(ptr as *const i8, a, align_of::() as i32) +} + +/// Store multiple single-element structures from one, two, three, or four registers. +#[inline] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(str))] +pub unsafe fn vst1_s64(ptr: *mut i64, a: int64x1_t) { + vst1_v1i64(ptr as *const i8, a, align_of::() as i32) +} + +/// Store multiple single-element structures from one, two, three, or four registers. +#[inline] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(str))] +pub unsafe fn vst1q_s64(ptr: *mut i64, a: int64x2_t) { + vst1q_v2i64(ptr as *const i8, a, align_of::() as i32) +} + +/// Store multiple single-element structures from one, two, three, or four registers. +#[inline] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(str))] +pub unsafe fn vst1_u8(ptr: *mut u8, a: uint8x8_t) { + vst1_v8i8(ptr as *const i8, transmute(a), align_of::() as i32) +} + +/// Store multiple single-element structures from one, two, three, or four registers. +#[inline] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(str))] +pub unsafe fn vst1q_u8(ptr: *mut u8, a: uint8x16_t) { + vst1q_v16i8(ptr as *const i8, transmute(a), align_of::() as i32) +} + +/// Store multiple single-element structures from one, two, three, or four registers. +#[inline] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(str))] +pub unsafe fn vst1_u16(ptr: *mut u16, a: uint16x4_t) { + vst1_v4i16(ptr as *const i8, transmute(a), align_of::() as i32) +} + +/// Store multiple single-element structures from one, two, three, or four registers. +#[inline] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(str))] +pub unsafe fn vst1q_u16(ptr: *mut u16, a: uint16x8_t) { + vst1q_v8i16(ptr as *const i8, transmute(a), align_of::() as i32) +} + +/// Store multiple single-element structures from one, two, three, or four registers. +#[inline] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(str))] +pub unsafe fn vst1_u32(ptr: *mut u32, a: uint32x2_t) { + vst1_v2i32(ptr as *const i8, transmute(a), align_of::() as i32) +} + +/// Store multiple single-element structures from one, two, three, or four registers. +#[inline] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(str))] +pub unsafe fn vst1q_u32(ptr: *mut u32, a: uint32x4_t) { + vst1q_v4i32(ptr as *const i8, transmute(a), align_of::() as i32) +} + +/// Store multiple single-element structures from one, two, three, or four registers. +#[inline] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(str))] +pub unsafe fn vst1_u64(ptr: *mut u64, a: uint64x1_t) { + vst1_v1i64(ptr as *const i8, transmute(a), align_of::() as i32) +} + +/// Store multiple single-element structures from one, two, three, or four registers. +#[inline] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(str))] +pub unsafe fn vst1q_u64(ptr: *mut u64, a: uint64x2_t) { + vst1q_v2i64(ptr as *const i8, transmute(a), align_of::() as i32) +} + +/// Store multiple single-element structures from one, two, three, or four registers. +#[inline] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(str))] +pub unsafe fn vst1_p8(ptr: *mut p8, a: poly8x8_t) { + vst1_v8i8(ptr as *const i8, transmute(a), align_of::() as i32) +} + +/// Store multiple single-element structures from one, two, three, or four registers. +#[inline] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(str))] +pub unsafe fn vst1q_p8(ptr: *mut p8, a: poly8x16_t) { + vst1q_v16i8(ptr as *const i8, transmute(a), align_of::() as i32) +} + +/// Store multiple single-element structures from one, two, three, or four registers. +#[inline] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(str))] +pub unsafe fn vst1_p16(ptr: *mut p16, a: poly16x4_t) { + vst1_v4i16(ptr as *const i8, transmute(a), align_of::() as i32) +} + +/// Store multiple single-element structures from one, two, three, or four registers. +#[inline] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(str))] +pub unsafe fn vst1q_p16(ptr: *mut p16, a: poly16x8_t) { + vst1q_v8i16(ptr as *const i8, transmute(a), align_of::() as i32) +} + +// Store multiple single-element structures from one, two, three, or four registers. +#[inline] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(str))] +pub unsafe fn vst1_f32(ptr: *mut f32, a: float32x2_t) { + vst1_v2f32(ptr as *const i8, a, align_of::() as i32) +} + +// Store multiple single-element structures from one, two, three, or four registers. +#[inline] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(str))] +pub unsafe fn vst1q_f32(ptr: *mut f32, a: float32x4_t) { + vst1q_v4f32(ptr as *const i8, a, align_of::() as i32) +} + +/// Table look-up +#[inline] +#[cfg(target_endian = "little")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vtbl))] +pub unsafe fn vtbl1_s8(a: int8x8_t, b: int8x8_t) -> int8x8_t { + vtbl1(a, b) +} + +/// Table look-up +#[inline] +#[cfg(target_endian = "little")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vtbl))] +pub unsafe fn vtbl1_u8(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t { + transmute(vtbl1(transmute(a), transmute(b))) +} + +/// Table look-up +#[inline] +#[cfg(target_endian = "little")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vtbl))] +pub unsafe fn vtbl1_p8(a: poly8x8_t, b: uint8x8_t) -> poly8x8_t { + transmute(vtbl1(transmute(a), transmute(b))) +} + +/// Table look-up +#[inline] +#[cfg(target_endian = "little")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vtbl))] +pub unsafe fn vtbl2_s8(a: int8x8x2_t, b: int8x8_t) -> int8x8_t { + vtbl2(a.0, a.1, b) +} + +/// Table look-up +#[inline] +#[cfg(target_endian = "little")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vtbl))] +pub unsafe fn vtbl2_u8(a: uint8x8x2_t, b: uint8x8_t) -> uint8x8_t { + transmute(vtbl2(transmute(a.0), transmute(a.1), transmute(b))) +} + +/// Table look-up +#[inline] +#[cfg(target_endian = "little")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vtbl))] +pub unsafe fn vtbl2_p8(a: poly8x8x2_t, b: uint8x8_t) -> poly8x8_t { + transmute(vtbl2(transmute(a.0), transmute(a.1), transmute(b))) +} + +/// Table look-up +#[inline] +#[cfg(target_endian = "little")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vtbl))] +pub unsafe fn vtbl3_s8(a: int8x8x3_t, b: int8x8_t) -> int8x8_t { + vtbl3(a.0, a.1, a.2, b) +} + +/// Table look-up +#[inline] +#[cfg(target_endian = "little")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vtbl))] +pub unsafe fn vtbl3_u8(a: uint8x8x3_t, b: uint8x8_t) -> uint8x8_t { + transmute(vtbl3( + transmute(a.0), + transmute(a.1), + transmute(a.2), + transmute(b), + )) +} + +/// Table look-up +#[inline] +#[cfg(target_endian = "little")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vtbl))] +pub unsafe fn vtbl3_p8(a: poly8x8x3_t, b: uint8x8_t) -> poly8x8_t { + transmute(vtbl3( + transmute(a.0), + transmute(a.1), + transmute(a.2), + transmute(b), + )) +} + +/// Table look-up +#[inline] +#[cfg(target_endian = "little")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vtbl))] +pub unsafe fn vtbl4_s8(a: int8x8x4_t, b: int8x8_t) -> int8x8_t { + vtbl4(a.0, a.1, a.2, a.3, b) +} + +/// Table look-up +#[inline] +#[cfg(target_endian = "little")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vtbl))] +pub unsafe fn vtbl4_u8(a: uint8x8x4_t, b: uint8x8_t) -> uint8x8_t { + transmute(vtbl4( + transmute(a.0), + transmute(a.1), + transmute(a.2), + transmute(a.3), + transmute(b), + )) +} + +/// Table look-up +#[inline] +#[cfg(target_endian = "little")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vtbl))] +pub unsafe fn vtbl4_p8(a: poly8x8x4_t, b: uint8x8_t) -> poly8x8_t { + transmute(vtbl4( + transmute(a.0), + transmute(a.1), + transmute(a.2), + transmute(a.3), + transmute(b), + )) +} + +/// Extended table look-up +#[inline] +#[cfg(target_endian = "little")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vtbx))] +pub unsafe fn vtbx1_s8(a: int8x8_t, b: int8x8_t, c: int8x8_t) -> int8x8_t { + vtbx1(a, b, c) +} + +/// Extended table look-up +#[inline] +#[cfg(target_endian = "little")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vtbx))] +pub unsafe fn vtbx1_u8(a: uint8x8_t, b: uint8x8_t, c: uint8x8_t) -> uint8x8_t { + transmute(vtbx1(transmute(a), transmute(b), transmute(c))) +} + +/// Extended table look-up +#[inline] +#[cfg(target_endian = "little")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vtbx))] +pub unsafe fn vtbx1_p8(a: poly8x8_t, b: poly8x8_t, c: uint8x8_t) -> poly8x8_t { + transmute(vtbx1(transmute(a), transmute(b), transmute(c))) +} + +/// Extended table look-up +#[inline] +#[cfg(target_endian = "little")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vtbx))] +pub unsafe fn vtbx2_s8(a: int8x8_t, b: int8x8x2_t, c: int8x8_t) -> int8x8_t { + vtbx2(a, b.0, b.1, c) +} + +/// Extended table look-up +#[inline] +#[cfg(target_endian = "little")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vtbx))] +pub unsafe fn vtbx2_u8(a: uint8x8_t, b: uint8x8x2_t, c: uint8x8_t) -> uint8x8_t { + transmute(vtbx2( + transmute(a), + transmute(b.0), + transmute(b.1), + transmute(c), + )) +} + +/// Extended table look-up +#[inline] +#[cfg(target_endian = "little")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vtbx))] +pub unsafe fn vtbx2_p8(a: poly8x8_t, b: poly8x8x2_t, c: uint8x8_t) -> poly8x8_t { + transmute(vtbx2( + transmute(a), + transmute(b.0), + transmute(b.1), + transmute(c), + )) +} + +/// Extended table look-up +#[inline] +#[cfg(target_endian = "little")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vtbx))] +pub unsafe fn vtbx3_s8(a: int8x8_t, b: int8x8x3_t, c: int8x8_t) -> int8x8_t { + vtbx3(a, b.0, b.1, b.2, c) +} + +/// Extended table look-up +#[inline] +#[cfg(target_endian = "little")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vtbx))] +pub unsafe fn vtbx3_u8(a: uint8x8_t, b: uint8x8x3_t, c: uint8x8_t) -> uint8x8_t { + transmute(vtbx3( + transmute(a), + transmute(b.0), + transmute(b.1), + transmute(b.2), + transmute(c), + )) +} + +/// Extended table look-up +#[inline] +#[cfg(target_endian = "little")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vtbx))] +pub unsafe fn vtbx3_p8(a: poly8x8_t, b: poly8x8x3_t, c: uint8x8_t) -> poly8x8_t { + transmute(vtbx3( + transmute(a), + transmute(b.0), + transmute(b.1), + transmute(b.2), + transmute(c), + )) +} + +/// Extended table look-up +#[inline] +#[cfg(target_endian = "little")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vtbx))] +pub unsafe fn vtbx4_s8(a: int8x8_t, b: int8x8x4_t, c: int8x8_t) -> int8x8_t { + vtbx4(a, b.0, b.1, b.2, b.3, c) +} + +/// Extended table look-up +#[inline] +#[cfg(target_endian = "little")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vtbx))] +pub unsafe fn vtbx4_u8(a: uint8x8_t, b: uint8x8x4_t, c: uint8x8_t) -> uint8x8_t { + transmute(vtbx4( + transmute(a), + transmute(b.0), + transmute(b.1), + transmute(b.2), + transmute(b.3), + transmute(c), + )) +} + +/// Extended table look-up +#[inline] +#[cfg(target_endian = "little")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr(vtbx))] +pub unsafe fn vtbx4_p8(a: poly8x8_t, b: poly8x8x4_t, c: uint8x8_t) -> poly8x8_t { + transmute(vtbx4( + transmute(a), + transmute(b.0), + transmute(b.1), + transmute(b.2), + transmute(b.3), + transmute(c), + )) +} + +// These float-to-int implementations have undefined behaviour when `a` overflows +// the destination type. Clang has the same problem: https://llvm.org/PR47510 + +/// Floating-point Convert to Signed fixed-point, rounding toward Zero (vector) +#[inline] +#[target_feature(enable = "neon")] +#[target_feature(enable = "v7")] +#[cfg_attr(test, assert_instr("vcvt.s32.f32"))] +pub unsafe fn vcvtq_s32_f32(a: float32x4_t) -> int32x4_t { + transmute(simd_cast::<_, i32x4>(transmute::<_, f32x4>(a))) +} + +/// Floating-point Convert to Unsigned fixed-point, rounding toward Zero (vector) +#[inline] +#[target_feature(enable = "neon")] +#[target_feature(enable = "v7")] +#[cfg_attr(test, assert_instr("vcvt.u32.f32"))] +pub unsafe fn vcvtq_u32_f32(a: float32x4_t) -> uint32x4_t { + transmute(simd_cast::<_, u32x4>(transmute::<_, f32x4>(a))) +} + +/// Shift Left and Insert (immediate) +#[inline] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr("vsli.8", N = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vsli_n_s8(a: int8x8_t, b: int8x8_t) -> int8x8_t { + static_assert_imm3!(N); + let n = N as i8; + vshiftins_v8i8(a, b, int8x8_t(n, n, n, n, n, n, n, n)) +} +/// Shift Left and Insert (immediate) +#[inline] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr("vsli.8", N = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vsliq_n_s8(a: int8x16_t, b: int8x16_t) -> int8x16_t { + static_assert_imm3!(N); + let n = N as i8; + vshiftins_v16i8( + a, + b, + int8x16_t(n, n, n, n, n, n, n, n, n, n, n, n, n, n, n, n), + ) +} +/// Shift Left and Insert (immediate) +#[inline] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr("vsli.16", N = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vsli_n_s16(a: int16x4_t, b: int16x4_t) -> int16x4_t { + static_assert_imm4!(N); + let n = N as i16; + vshiftins_v4i16(a, b, int16x4_t(n, n, n, n)) +} +/// Shift Left and Insert (immediate) +#[inline] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr("vsli.16", N = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vsliq_n_s16(a: int16x8_t, b: int16x8_t) -> int16x8_t { + static_assert_imm4!(N); + let n = N as i16; + vshiftins_v8i16(a, b, int16x8_t(n, n, n, n, n, n, n, n)) +} +/// Shift Left and Insert (immediate) +#[inline] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr("vsli.32", N = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vsli_n_s32(a: int32x2_t, b: int32x2_t) -> int32x2_t { + static_assert!(N: i32 where N >= 0 && N <= 31); + vshiftins_v2i32(a, b, int32x2_t(N, N)) +} +/// Shift Left and Insert (immediate) +#[inline] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr("vsli.32", N = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vsliq_n_s32(a: int32x4_t, b: int32x4_t) -> int32x4_t { + static_assert!(N: i32 where N >= 0 && N <= 31); + vshiftins_v4i32(a, b, int32x4_t(N, N, N, N)) +} +/// Shift Left and Insert (immediate) +#[inline] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr("vsli.64", N = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vsli_n_s64(a: int64x1_t, b: int64x1_t) -> int64x1_t { + static_assert!(N : i32 where 0 <= N && N <= 63); + vshiftins_v1i64(a, b, int64x1_t(N as i64)) +} +/// Shift Left and Insert (immediate) +#[inline] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr("vsli.64", N = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vsliq_n_s64(a: int64x2_t, b: int64x2_t) -> int64x2_t { + static_assert!(N : i32 where 0 <= N && N <= 63); + vshiftins_v2i64(a, b, int64x2_t(N as i64, N as i64)) +} +/// Shift Left and Insert (immediate) +#[inline] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr("vsli.8", N = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vsli_n_u8(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t { + static_assert_imm3!(N); + let n = N as i8; + transmute(vshiftins_v8i8( + transmute(a), + transmute(b), + int8x8_t(n, n, n, n, n, n, n, n), + )) +} +/// Shift Left and Insert (immediate) +#[inline] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr("vsli.8", N = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vsliq_n_u8(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t { + static_assert_imm3!(N); + let n = N as i8; + transmute(vshiftins_v16i8( + transmute(a), + transmute(b), + int8x16_t(n, n, n, n, n, n, n, n, n, n, n, n, n, n, n, n), + )) +} +/// Shift Left and Insert (immediate) +#[inline] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr("vsli.16", N = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vsli_n_u16(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t { + static_assert_imm4!(N); + let n = N as i16; + transmute(vshiftins_v4i16( + transmute(a), + transmute(b), + int16x4_t(n, n, n, n), + )) +} +/// Shift Left and Insert (immediate) +#[inline] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr("vsli.16", N = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vsliq_n_u16(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t { + static_assert_imm4!(N); + let n = N as i16; + transmute(vshiftins_v8i16( + transmute(a), + transmute(b), + int16x8_t(n, n, n, n, n, n, n, n), + )) +} +/// Shift Left and Insert (immediate) +#[inline] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr("vsli.32", N = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vsli_n_u32(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t { + static_assert!(N: i32 where N >= 0 && N <= 31); + transmute(vshiftins_v2i32(transmute(a), transmute(b), int32x2_t(N, N))) +} +/// Shift Left and Insert (immediate) +#[inline] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr("vsli.32", N = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vsliq_n_u32(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t { + static_assert!(N: i32 where N >= 0 && N <= 31); + transmute(vshiftins_v4i32( + transmute(a), + transmute(b), + int32x4_t(N, N, N, N), + )) +} +/// Shift Left and Insert (immediate) +#[inline] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr("vsli.64", N = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vsli_n_u64(a: uint64x1_t, b: uint64x1_t) -> uint64x1_t { + static_assert!(N : i32 where 0 <= N && N <= 63); + transmute(vshiftins_v1i64( + transmute(a), + transmute(b), + int64x1_t(N as i64), + )) +} +/// Shift Left and Insert (immediate) +#[inline] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr("vsli.64", N = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vsliq_n_u64(a: uint64x2_t, b: uint64x2_t) -> uint64x2_t { + static_assert!(N : i32 where 0 <= N && N <= 63); + transmute(vshiftins_v2i64( + transmute(a), + transmute(b), + int64x2_t(N as i64, N as i64), + )) +} +/// Shift Left and Insert (immediate) +#[inline] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr("vsli.8", N = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vsli_n_p8(a: poly8x8_t, b: poly8x8_t) -> poly8x8_t { + static_assert_imm3!(N); + let n = N as i8; + transmute(vshiftins_v8i8( + transmute(a), + transmute(b), + int8x8_t(n, n, n, n, n, n, n, n), + )) +} +/// Shift Left and Insert (immediate) +#[inline] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr("vsli.8", N = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vsliq_n_p8(a: poly8x16_t, b: poly8x16_t) -> poly8x16_t { + static_assert_imm3!(N); + let n = N as i8; + transmute(vshiftins_v16i8( + transmute(a), + transmute(b), + int8x16_t(n, n, n, n, n, n, n, n, n, n, n, n, n, n, n, n), + )) +} +/// Shift Left and Insert (immediate) +#[inline] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr("vsli.16", N = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vsli_n_p16(a: poly16x4_t, b: poly16x4_t) -> poly16x4_t { + static_assert_imm4!(N); + let n = N as i16; + transmute(vshiftins_v4i16( + transmute(a), + transmute(b), + int16x4_t(n, n, n, n), + )) +} +/// Shift Left and Insert (immediate) +#[inline] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr("vsli.16", N = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vsliq_n_p16(a: poly16x8_t, b: poly16x8_t) -> poly16x8_t { + static_assert_imm4!(N); + let n = N as i16; + transmute(vshiftins_v8i16( + transmute(a), + transmute(b), + int16x8_t(n, n, n, n, n, n, n, n), + )) +} + +/// Shift Right and Insert (immediate) +#[inline] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr("vsri.8", N = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vsri_n_s8(a: int8x8_t, b: int8x8_t) -> int8x8_t { + static_assert!(N : i32 where 1 <= N && N <= 8); + let n = -N as i8; + vshiftins_v8i8(a, b, int8x8_t(n, n, n, n, n, n, n, n)) +} +/// Shift Right and Insert (immediate) +#[inline] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr("vsri.8", N = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vsriq_n_s8(a: int8x16_t, b: int8x16_t) -> int8x16_t { + static_assert!(N : i32 where 1 <= N && N <= 8); + let n = -N as i8; + vshiftins_v16i8( + a, + b, + int8x16_t(n, n, n, n, n, n, n, n, n, n, n, n, n, n, n, n), + ) +} +/// Shift Right and Insert (immediate) +#[inline] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr("vsri.16", N = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vsri_n_s16(a: int16x4_t, b: int16x4_t) -> int16x4_t { + static_assert!(N : i32 where 1 <= N && N <= 16); + let n = -N as i16; + vshiftins_v4i16(a, b, int16x4_t(n, n, n, n)) +} +/// Shift Right and Insert (immediate) +#[inline] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr("vsri.16", N = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vsriq_n_s16(a: int16x8_t, b: int16x8_t) -> int16x8_t { + static_assert!(N : i32 where 1 <= N && N <= 16); + let n = -N as i16; + vshiftins_v8i16(a, b, int16x8_t(n, n, n, n, n, n, n, n)) +} +/// Shift Right and Insert (immediate) +#[inline] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr("vsri.32", N = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vsri_n_s32(a: int32x2_t, b: int32x2_t) -> int32x2_t { + static_assert!(N : i32 where 1 <= N && N <= 32); + vshiftins_v2i32(a, b, int32x2_t(-N, -N)) +} +/// Shift Right and Insert (immediate) +#[inline] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr("vsri.32", N = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vsriq_n_s32(a: int32x4_t, b: int32x4_t) -> int32x4_t { + static_assert!(N : i32 where 1 <= N && N <= 32); + vshiftins_v4i32(a, b, int32x4_t(-N, -N, -N, -N)) +} +/// Shift Right and Insert (immediate) +#[inline] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr("vsri.64", N = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vsri_n_s64(a: int64x1_t, b: int64x1_t) -> int64x1_t { + static_assert!(N : i32 where 1 <= N && N <= 64); + vshiftins_v1i64(a, b, int64x1_t(-N as i64)) +} +/// Shift Right and Insert (immediate) +#[inline] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr("vsri.64", N = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vsriq_n_s64(a: int64x2_t, b: int64x2_t) -> int64x2_t { + static_assert!(N : i32 where 1 <= N && N <= 64); + vshiftins_v2i64(a, b, int64x2_t(-N as i64, -N as i64)) +} +/// Shift Right and Insert (immediate) +#[inline] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr("vsri.8", N = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vsri_n_u8(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t { + static_assert!(N : i32 where 1 <= N && N <= 8); + let n = -N as i8; + transmute(vshiftins_v8i8( + transmute(a), + transmute(b), + int8x8_t(n, n, n, n, n, n, n, n), + )) +} +/// Shift Right and Insert (immediate) +#[inline] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr("vsri.8", N = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vsriq_n_u8(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t { + static_assert!(N : i32 where 1 <= N && N <= 8); + let n = -N as i8; + transmute(vshiftins_v16i8( + transmute(a), + transmute(b), + int8x16_t(n, n, n, n, n, n, n, n, n, n, n, n, n, n, n, n), + )) +} +/// Shift Right and Insert (immediate) +#[inline] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr("vsri.16", N = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vsri_n_u16(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t { + static_assert!(N : i32 where 1 <= N && N <= 16); + let n = -N as i16; + transmute(vshiftins_v4i16( + transmute(a), + transmute(b), + int16x4_t(n, n, n, n), + )) +} +/// Shift Right and Insert (immediate) +#[inline] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr("vsri.16", N = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vsriq_n_u16(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t { + static_assert!(N : i32 where 1 <= N && N <= 16); + let n = -N as i16; + transmute(vshiftins_v8i16( + transmute(a), + transmute(b), + int16x8_t(n, n, n, n, n, n, n, n), + )) +} +/// Shift Right and Insert (immediate) +#[inline] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr("vsri.32", N = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vsri_n_u32(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t { + static_assert!(N : i32 where 1 <= N && N <= 32); + transmute(vshiftins_v2i32( + transmute(a), + transmute(b), + int32x2_t(-N, -N), + )) +} +/// Shift Right and Insert (immediate) +#[inline] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr("vsri.32", N = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vsriq_n_u32(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t { + static_assert!(N : i32 where 1 <= N && N <= 32); + transmute(vshiftins_v4i32( + transmute(a), + transmute(b), + int32x4_t(-N, -N, -N, -N), + )) +} +/// Shift Right and Insert (immediate) +#[inline] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr("vsri.64", N = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vsri_n_u64(a: uint64x1_t, b: uint64x1_t) -> uint64x1_t { + static_assert!(N : i32 where 1 <= N && N <= 64); + transmute(vshiftins_v1i64( + transmute(a), + transmute(b), + int64x1_t(-N as i64), + )) +} +/// Shift Right and Insert (immediate) +#[inline] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr("vsri.64", N = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vsriq_n_u64(a: uint64x2_t, b: uint64x2_t) -> uint64x2_t { + static_assert!(N : i32 where 1 <= N && N <= 64); + transmute(vshiftins_v2i64( + transmute(a), + transmute(b), + int64x2_t(-N as i64, -N as i64), + )) +} +/// Shift Right and Insert (immediate) +#[inline] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr("vsri.8", N = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vsri_n_p8(a: poly8x8_t, b: poly8x8_t) -> poly8x8_t { + static_assert!(N : i32 where 1 <= N && N <= 8); + let n = -N as i8; + transmute(vshiftins_v8i8( + transmute(a), + transmute(b), + int8x8_t(n, n, n, n, n, n, n, n), + )) +} +/// Shift Right and Insert (immediate) +#[inline] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr("vsri.8", N = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vsriq_n_p8(a: poly8x16_t, b: poly8x16_t) -> poly8x16_t { + static_assert!(N : i32 where 1 <= N && N <= 8); + let n = -N as i8; + transmute(vshiftins_v16i8( + transmute(a), + transmute(b), + int8x16_t(n, n, n, n, n, n, n, n, n, n, n, n, n, n, n, n), + )) +} +/// Shift Right and Insert (immediate) +#[inline] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr("vsri.16", N = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vsri_n_p16(a: poly16x4_t, b: poly16x4_t) -> poly16x4_t { + static_assert!(N : i32 where 1 <= N && N <= 16); + let n = -N as i16; + transmute(vshiftins_v4i16( + transmute(a), + transmute(b), + int16x4_t(n, n, n, n), + )) +} +/// Shift Right and Insert (immediate) +#[inline] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(test, assert_instr("vsri.16", N = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vsriq_n_p16(a: poly16x8_t, b: poly16x8_t) -> poly16x8_t { + static_assert!(N : i32 where 1 <= N && N <= 16); + let n = -N as i16; + transmute(vshiftins_v8i16( + transmute(a), + transmute(b), + int16x8_t(n, n, n, n, n, n, n, n), + )) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::core_arch::{arm::*, simd::*}; + use crate::mem::transmute; + use stdarch_test::simd_test; + + #[simd_test(enable = "neon")] + unsafe fn test_vcvtq_s32_f32() { + let f = f32x4::new(-1., 2., 3., 4.); + let e = i32x4::new(-1, 2, 3, 4); + let r: i32x4 = transmute(vcvtq_s32_f32(transmute(f))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvtq_u32_f32() { + let f = f32x4::new(1., 2., 3., 4.); + let e = u32x4::new(1, 2, 3, 4); + let r: u32x4 = transmute(vcvtq_u32_f32(transmute(f))); + assert_eq!(r, e); + } +} diff --git a/library/stdarch/crates/core_arch/src/arm/neon/generated.rs b/library/stdarch/crates/core_arch/src/arm/neon/generated.rs deleted file mode 100644 index c60ad9cc50..0000000000 --- a/library/stdarch/crates/core_arch/src/arm/neon/generated.rs +++ /dev/null @@ -1,5237 +0,0 @@ -// This code is automatically generated. DO NOT MODIFY. -// -// Instead, modify `crates/stdarch-gen/neon.spec` and run the following command to re-generate this file: -// -// ``` -// OUT_DIR=`pwd`/crates/core_arch cargo run -p stdarch-gen -- crates/stdarch-gen/neon.spec -// ``` -use super::*; -#[cfg(test)] -use stdarch_test::assert_instr; - -/// Vector bitwise and -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vand))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(and))] -pub unsafe fn vand_s8(a: int8x8_t, b: int8x8_t) -> int8x8_t { - simd_and(a, b) -} - -/// Vector bitwise and -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vand))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(and))] -pub unsafe fn vandq_s8(a: int8x16_t, b: int8x16_t) -> int8x16_t { - simd_and(a, b) -} - -/// Vector bitwise and -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vand))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(and))] -pub unsafe fn vand_s16(a: int16x4_t, b: int16x4_t) -> int16x4_t { - simd_and(a, b) -} - -/// Vector bitwise and -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vand))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(and))] -pub unsafe fn vandq_s16(a: int16x8_t, b: int16x8_t) -> int16x8_t { - simd_and(a, b) -} - -/// Vector bitwise and -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vand))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(and))] -pub unsafe fn vand_s32(a: int32x2_t, b: int32x2_t) -> int32x2_t { - simd_and(a, b) -} - -/// Vector bitwise and -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vand))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(and))] -pub unsafe fn vandq_s32(a: int32x4_t, b: int32x4_t) -> int32x4_t { - simd_and(a, b) -} - -/// Vector bitwise and -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vand))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(and))] -pub unsafe fn vand_u8(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t { - simd_and(a, b) -} - -/// Vector bitwise and -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vand))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(and))] -pub unsafe fn vandq_u8(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t { - simd_and(a, b) -} - -/// Vector bitwise and -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vand))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(and))] -pub unsafe fn vand_u16(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t { - simd_and(a, b) -} - -/// Vector bitwise and -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vand))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(and))] -pub unsafe fn vandq_u16(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t { - simd_and(a, b) -} - -/// Vector bitwise and -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vand))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(and))] -pub unsafe fn vand_u32(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t { - simd_and(a, b) -} - -/// Vector bitwise and -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vand))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(and))] -pub unsafe fn vandq_u32(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t { - simd_and(a, b) -} - -/// Vector bitwise and -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vand))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(and))] -pub unsafe fn vand_s64(a: int64x1_t, b: int64x1_t) -> int64x1_t { - simd_and(a, b) -} - -/// Vector bitwise and -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vand))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(and))] -pub unsafe fn vandq_s64(a: int64x2_t, b: int64x2_t) -> int64x2_t { - simd_and(a, b) -} - -/// Vector bitwise and -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vand))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(and))] -pub unsafe fn vand_u64(a: uint64x1_t, b: uint64x1_t) -> uint64x1_t { - simd_and(a, b) -} - -/// Vector bitwise and -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vand))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(and))] -pub unsafe fn vandq_u64(a: uint64x2_t, b: uint64x2_t) -> uint64x2_t { - simd_and(a, b) -} - -/// Vector bitwise or (immediate, inclusive) -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vorr))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(orr))] -pub unsafe fn vorr_s8(a: int8x8_t, b: int8x8_t) -> int8x8_t { - simd_or(a, b) -} - -/// Vector bitwise or (immediate, inclusive) -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vorr))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(orr))] -pub unsafe fn vorrq_s8(a: int8x16_t, b: int8x16_t) -> int8x16_t { - simd_or(a, b) -} - -/// Vector bitwise or (immediate, inclusive) -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vorr))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(orr))] -pub unsafe fn vorr_s16(a: int16x4_t, b: int16x4_t) -> int16x4_t { - simd_or(a, b) -} - -/// Vector bitwise or (immediate, inclusive) -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vorr))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(orr))] -pub unsafe fn vorrq_s16(a: int16x8_t, b: int16x8_t) -> int16x8_t { - simd_or(a, b) -} - -/// Vector bitwise or (immediate, inclusive) -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vorr))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(orr))] -pub unsafe fn vorr_s32(a: int32x2_t, b: int32x2_t) -> int32x2_t { - simd_or(a, b) -} - -/// Vector bitwise or (immediate, inclusive) -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vorr))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(orr))] -pub unsafe fn vorrq_s32(a: int32x4_t, b: int32x4_t) -> int32x4_t { - simd_or(a, b) -} - -/// Vector bitwise or (immediate, inclusive) -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vorr))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(orr))] -pub unsafe fn vorr_u8(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t { - simd_or(a, b) -} - -/// Vector bitwise or (immediate, inclusive) -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vorr))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(orr))] -pub unsafe fn vorrq_u8(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t { - simd_or(a, b) -} - -/// Vector bitwise or (immediate, inclusive) -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vorr))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(orr))] -pub unsafe fn vorr_u16(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t { - simd_or(a, b) -} - -/// Vector bitwise or (immediate, inclusive) -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vorr))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(orr))] -pub unsafe fn vorrq_u16(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t { - simd_or(a, b) -} - -/// Vector bitwise or (immediate, inclusive) -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vorr))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(orr))] -pub unsafe fn vorr_u32(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t { - simd_or(a, b) -} - -/// Vector bitwise or (immediate, inclusive) -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vorr))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(orr))] -pub unsafe fn vorrq_u32(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t { - simd_or(a, b) -} - -/// Vector bitwise or (immediate, inclusive) -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vorr))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(orr))] -pub unsafe fn vorr_s64(a: int64x1_t, b: int64x1_t) -> int64x1_t { - simd_or(a, b) -} - -/// Vector bitwise or (immediate, inclusive) -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vorr))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(orr))] -pub unsafe fn vorrq_s64(a: int64x2_t, b: int64x2_t) -> int64x2_t { - simd_or(a, b) -} - -/// Vector bitwise or (immediate, inclusive) -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vorr))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(orr))] -pub unsafe fn vorr_u64(a: uint64x1_t, b: uint64x1_t) -> uint64x1_t { - simd_or(a, b) -} - -/// Vector bitwise or (immediate, inclusive) -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vorr))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(orr))] -pub unsafe fn vorrq_u64(a: uint64x2_t, b: uint64x2_t) -> uint64x2_t { - simd_or(a, b) -} - -/// Vector bitwise exclusive or (vector) -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(veor))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(eor))] -pub unsafe fn veor_s8(a: int8x8_t, b: int8x8_t) -> int8x8_t { - simd_xor(a, b) -} - -/// Vector bitwise exclusive or (vector) -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(veor))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(eor))] -pub unsafe fn veorq_s8(a: int8x16_t, b: int8x16_t) -> int8x16_t { - simd_xor(a, b) -} - -/// Vector bitwise exclusive or (vector) -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(veor))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(eor))] -pub unsafe fn veor_s16(a: int16x4_t, b: int16x4_t) -> int16x4_t { - simd_xor(a, b) -} - -/// Vector bitwise exclusive or (vector) -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(veor))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(eor))] -pub unsafe fn veorq_s16(a: int16x8_t, b: int16x8_t) -> int16x8_t { - simd_xor(a, b) -} - -/// Vector bitwise exclusive or (vector) -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(veor))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(eor))] -pub unsafe fn veor_s32(a: int32x2_t, b: int32x2_t) -> int32x2_t { - simd_xor(a, b) -} - -/// Vector bitwise exclusive or (vector) -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(veor))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(eor))] -pub unsafe fn veorq_s32(a: int32x4_t, b: int32x4_t) -> int32x4_t { - simd_xor(a, b) -} - -/// Vector bitwise exclusive or (vector) -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(veor))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(eor))] -pub unsafe fn veor_u8(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t { - simd_xor(a, b) -} - -/// Vector bitwise exclusive or (vector) -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(veor))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(eor))] -pub unsafe fn veorq_u8(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t { - simd_xor(a, b) -} - -/// Vector bitwise exclusive or (vector) -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(veor))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(eor))] -pub unsafe fn veor_u16(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t { - simd_xor(a, b) -} - -/// Vector bitwise exclusive or (vector) -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(veor))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(eor))] -pub unsafe fn veorq_u16(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t { - simd_xor(a, b) -} - -/// Vector bitwise exclusive or (vector) -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(veor))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(eor))] -pub unsafe fn veor_u32(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t { - simd_xor(a, b) -} - -/// Vector bitwise exclusive or (vector) -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(veor))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(eor))] -pub unsafe fn veorq_u32(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t { - simd_xor(a, b) -} - -/// Vector bitwise exclusive or (vector) -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(veor))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(eor))] -pub unsafe fn veor_s64(a: int64x1_t, b: int64x1_t) -> int64x1_t { - simd_xor(a, b) -} - -/// Vector bitwise exclusive or (vector) -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(veor))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(eor))] -pub unsafe fn veorq_s64(a: int64x2_t, b: int64x2_t) -> int64x2_t { - simd_xor(a, b) -} - -/// Vector bitwise exclusive or (vector) -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(veor))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(eor))] -pub unsafe fn veor_u64(a: uint64x1_t, b: uint64x1_t) -> uint64x1_t { - simd_xor(a, b) -} - -/// Vector bitwise exclusive or (vector) -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(veor))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(eor))] -pub unsafe fn veorq_u64(a: uint64x2_t, b: uint64x2_t) -> uint64x2_t { - simd_xor(a, b) -} - -/// Compare bitwise Equal (vector) -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vceq.i8"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmeq))] -pub unsafe fn vceq_u8(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t { - simd_eq(a, b) -} - -/// Compare bitwise Equal (vector) -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vceq.i8"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmeq))] -pub unsafe fn vceqq_u8(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t { - simd_eq(a, b) -} - -/// Compare bitwise Equal (vector) -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vceq.i16"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmeq))] -pub unsafe fn vceq_u16(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t { - simd_eq(a, b) -} - -/// Compare bitwise Equal (vector) -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vceq.i16"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmeq))] -pub unsafe fn vceqq_u16(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t { - simd_eq(a, b) -} - -/// Compare bitwise Equal (vector) -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vceq.i32"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmeq))] -pub unsafe fn vceq_u32(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t { - simd_eq(a, b) -} - -/// Compare bitwise Equal (vector) -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vceq.i32"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmeq))] -pub unsafe fn vceqq_u32(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t { - simd_eq(a, b) -} - -/// Compare bitwise Equal (vector) -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vceq.i8"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmeq))] -pub unsafe fn vceq_s8(a: int8x8_t, b: int8x8_t) -> uint8x8_t { - simd_eq(a, b) -} - -/// Compare bitwise Equal (vector) -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vceq.i8"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmeq))] -pub unsafe fn vceqq_s8(a: int8x16_t, b: int8x16_t) -> uint8x16_t { - simd_eq(a, b) -} - -/// Compare bitwise Equal (vector) -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vceq.i16"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmeq))] -pub unsafe fn vceq_s16(a: int16x4_t, b: int16x4_t) -> uint16x4_t { - simd_eq(a, b) -} - -/// Compare bitwise Equal (vector) -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vceq.i16"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmeq))] -pub unsafe fn vceqq_s16(a: int16x8_t, b: int16x8_t) -> uint16x8_t { - simd_eq(a, b) -} - -/// Compare bitwise Equal (vector) -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vceq.i32"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmeq))] -pub unsafe fn vceq_s32(a: int32x2_t, b: int32x2_t) -> uint32x2_t { - simd_eq(a, b) -} - -/// Compare bitwise Equal (vector) -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vceq.i32"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmeq))] -pub unsafe fn vceqq_s32(a: int32x4_t, b: int32x4_t) -> uint32x4_t { - simd_eq(a, b) -} - -/// Floating-point compare equal -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vceq.f32"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fcmeq))] -pub unsafe fn vceq_f32(a: float32x2_t, b: float32x2_t) -> uint32x2_t { - simd_eq(a, b) -} - -/// Floating-point compare equal -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vceq.f32"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fcmeq))] -pub unsafe fn vceqq_f32(a: float32x4_t, b: float32x4_t) -> uint32x4_t { - simd_eq(a, b) -} - -/// Compare signed greater than -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcgt.s8"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmgt))] -pub unsafe fn vcgt_s8(a: int8x8_t, b: int8x8_t) -> uint8x8_t { - simd_gt(a, b) -} - -/// Compare signed greater than -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcgt.s8"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmgt))] -pub unsafe fn vcgtq_s8(a: int8x16_t, b: int8x16_t) -> uint8x16_t { - simd_gt(a, b) -} - -/// Compare signed greater than -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcgt.s16"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmgt))] -pub unsafe fn vcgt_s16(a: int16x4_t, b: int16x4_t) -> uint16x4_t { - simd_gt(a, b) -} - -/// Compare signed greater than -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcgt.s16"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmgt))] -pub unsafe fn vcgtq_s16(a: int16x8_t, b: int16x8_t) -> uint16x8_t { - simd_gt(a, b) -} - -/// Compare signed greater than -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcgt.s32"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmgt))] -pub unsafe fn vcgt_s32(a: int32x2_t, b: int32x2_t) -> uint32x2_t { - simd_gt(a, b) -} - -/// Compare signed greater than -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcgt.s32"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmgt))] -pub unsafe fn vcgtq_s32(a: int32x4_t, b: int32x4_t) -> uint32x4_t { - simd_gt(a, b) -} - -/// Compare unsigned highe -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcgt.u8"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmhi))] -pub unsafe fn vcgt_u8(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t { - simd_gt(a, b) -} - -/// Compare unsigned highe -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcgt.u8"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmhi))] -pub unsafe fn vcgtq_u8(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t { - simd_gt(a, b) -} - -/// Compare unsigned highe -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcgt.u16"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmhi))] -pub unsafe fn vcgt_u16(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t { - simd_gt(a, b) -} - -/// Compare unsigned highe -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcgt.u16"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmhi))] -pub unsafe fn vcgtq_u16(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t { - simd_gt(a, b) -} - -/// Compare unsigned highe -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcgt.u32"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmhi))] -pub unsafe fn vcgt_u32(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t { - simd_gt(a, b) -} - -/// Compare unsigned highe -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcgt.u32"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmhi))] -pub unsafe fn vcgtq_u32(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t { - simd_gt(a, b) -} - -/// Floating-point compare greater than -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcgt.f32"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fcmgt))] -pub unsafe fn vcgt_f32(a: float32x2_t, b: float32x2_t) -> uint32x2_t { - simd_gt(a, b) -} - -/// Floating-point compare greater than -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcgt.f32"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fcmgt))] -pub unsafe fn vcgtq_f32(a: float32x4_t, b: float32x4_t) -> uint32x4_t { - simd_gt(a, b) -} - -/// Compare signed less than -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcgt.s8"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmgt))] -pub unsafe fn vclt_s8(a: int8x8_t, b: int8x8_t) -> uint8x8_t { - simd_lt(a, b) -} - -/// Compare signed less than -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcgt.s8"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmgt))] -pub unsafe fn vcltq_s8(a: int8x16_t, b: int8x16_t) -> uint8x16_t { - simd_lt(a, b) -} - -/// Compare signed less than -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcgt.s16"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmgt))] -pub unsafe fn vclt_s16(a: int16x4_t, b: int16x4_t) -> uint16x4_t { - simd_lt(a, b) -} - -/// Compare signed less than -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcgt.s16"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmgt))] -pub unsafe fn vcltq_s16(a: int16x8_t, b: int16x8_t) -> uint16x8_t { - simd_lt(a, b) -} - -/// Compare signed less than -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcgt.s32"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmgt))] -pub unsafe fn vclt_s32(a: int32x2_t, b: int32x2_t) -> uint32x2_t { - simd_lt(a, b) -} - -/// Compare signed less than -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcgt.s32"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmgt))] -pub unsafe fn vcltq_s32(a: int32x4_t, b: int32x4_t) -> uint32x4_t { - simd_lt(a, b) -} - -/// Compare unsigned less than -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcgt.u8"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmhi))] -pub unsafe fn vclt_u8(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t { - simd_lt(a, b) -} - -/// Compare unsigned less than -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcgt.u8"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmhi))] -pub unsafe fn vcltq_u8(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t { - simd_lt(a, b) -} - -/// Compare unsigned less than -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcgt.u16"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmhi))] -pub unsafe fn vclt_u16(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t { - simd_lt(a, b) -} - -/// Compare unsigned less than -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcgt.u16"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmhi))] -pub unsafe fn vcltq_u16(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t { - simd_lt(a, b) -} - -/// Compare unsigned less than -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcgt.u32"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmhi))] -pub unsafe fn vclt_u32(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t { - simd_lt(a, b) -} - -/// Compare unsigned less than -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcgt.u32"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmhi))] -pub unsafe fn vcltq_u32(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t { - simd_lt(a, b) -} - -/// Floating-point compare less than -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcgt.f32"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fcmgt))] -pub unsafe fn vclt_f32(a: float32x2_t, b: float32x2_t) -> uint32x2_t { - simd_lt(a, b) -} - -/// Floating-point compare less than -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcgt.f32"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fcmgt))] -pub unsafe fn vcltq_f32(a: float32x4_t, b: float32x4_t) -> uint32x4_t { - simd_lt(a, b) -} - -/// Compare signed less than or equal -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcge.s8"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmge))] -pub unsafe fn vcle_s8(a: int8x8_t, b: int8x8_t) -> uint8x8_t { - simd_le(a, b) -} - -/// Compare signed less than or equal -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcge.s8"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmge))] -pub unsafe fn vcleq_s8(a: int8x16_t, b: int8x16_t) -> uint8x16_t { - simd_le(a, b) -} - -/// Compare signed less than or equal -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcge.s16"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmge))] -pub unsafe fn vcle_s16(a: int16x4_t, b: int16x4_t) -> uint16x4_t { - simd_le(a, b) -} - -/// Compare signed less than or equal -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcge.s16"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmge))] -pub unsafe fn vcleq_s16(a: int16x8_t, b: int16x8_t) -> uint16x8_t { - simd_le(a, b) -} - -/// Compare signed less than or equal -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcge.s32"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmge))] -pub unsafe fn vcle_s32(a: int32x2_t, b: int32x2_t) -> uint32x2_t { - simd_le(a, b) -} - -/// Compare signed less than or equal -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcge.s32"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmge))] -pub unsafe fn vcleq_s32(a: int32x4_t, b: int32x4_t) -> uint32x4_t { - simd_le(a, b) -} - -/// Compare unsigned less than or equal -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcge.u8"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmhs))] -pub unsafe fn vcle_u8(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t { - simd_le(a, b) -} - -/// Compare unsigned less than or equal -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcge.u8"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmhs))] -pub unsafe fn vcleq_u8(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t { - simd_le(a, b) -} - -/// Compare unsigned less than or equal -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcge.u16"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmhs))] -pub unsafe fn vcle_u16(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t { - simd_le(a, b) -} - -/// Compare unsigned less than or equal -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcge.u16"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmhs))] -pub unsafe fn vcleq_u16(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t { - simd_le(a, b) -} - -/// Compare unsigned less than or equal -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcge.u32"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmhs))] -pub unsafe fn vcle_u32(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t { - simd_le(a, b) -} - -/// Compare unsigned less than or equal -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcge.u32"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmhs))] -pub unsafe fn vcleq_u32(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t { - simd_le(a, b) -} - -/// Floating-point compare less than or equal -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcge.f32"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fcmge))] -pub unsafe fn vcle_f32(a: float32x2_t, b: float32x2_t) -> uint32x2_t { - simd_le(a, b) -} - -/// Floating-point compare less than or equal -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcge.f32"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fcmge))] -pub unsafe fn vcleq_f32(a: float32x4_t, b: float32x4_t) -> uint32x4_t { - simd_le(a, b) -} - -/// Compare signed greater than or equal -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcge.s8"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmge))] -pub unsafe fn vcge_s8(a: int8x8_t, b: int8x8_t) -> uint8x8_t { - simd_ge(a, b) -} - -/// Compare signed greater than or equal -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcge.s8"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmge))] -pub unsafe fn vcgeq_s8(a: int8x16_t, b: int8x16_t) -> uint8x16_t { - simd_ge(a, b) -} - -/// Compare signed greater than or equal -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcge.s16"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmge))] -pub unsafe fn vcge_s16(a: int16x4_t, b: int16x4_t) -> uint16x4_t { - simd_ge(a, b) -} - -/// Compare signed greater than or equal -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcge.s16"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmge))] -pub unsafe fn vcgeq_s16(a: int16x8_t, b: int16x8_t) -> uint16x8_t { - simd_ge(a, b) -} - -/// Compare signed greater than or equal -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcge.s32"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmge))] -pub unsafe fn vcge_s32(a: int32x2_t, b: int32x2_t) -> uint32x2_t { - simd_ge(a, b) -} - -/// Compare signed greater than or equal -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcge.s32"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmge))] -pub unsafe fn vcgeq_s32(a: int32x4_t, b: int32x4_t) -> uint32x4_t { - simd_ge(a, b) -} - -/// Compare unsigned greater than or equal -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcge.u8"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmhs))] -pub unsafe fn vcge_u8(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t { - simd_ge(a, b) -} - -/// Compare unsigned greater than or equal -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcge.u8"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmhs))] -pub unsafe fn vcgeq_u8(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t { - simd_ge(a, b) -} - -/// Compare unsigned greater than or equal -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcge.u16"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmhs))] -pub unsafe fn vcge_u16(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t { - simd_ge(a, b) -} - -/// Compare unsigned greater than or equal -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcge.u16"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmhs))] -pub unsafe fn vcgeq_u16(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t { - simd_ge(a, b) -} - -/// Compare unsigned greater than or equal -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcge.u32"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmhs))] -pub unsafe fn vcge_u32(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t { - simd_ge(a, b) -} - -/// Compare unsigned greater than or equal -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcge.u32"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmhs))] -pub unsafe fn vcgeq_u32(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t { - simd_ge(a, b) -} - -/// Floating-point compare greater than or equal -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcge.f32"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fcmge))] -pub unsafe fn vcge_f32(a: float32x2_t, b: float32x2_t) -> uint32x2_t { - simd_ge(a, b) -} - -/// Floating-point compare greater than or equal -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcge.f32"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fcmge))] -pub unsafe fn vcgeq_f32(a: float32x4_t, b: float32x4_t) -> uint32x4_t { - simd_ge(a, b) -} - -/// Saturating subtract -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vqsub.u8"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uqsub))] -pub unsafe fn vqsub_u8(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t { - #[allow(improper_ctypes)] - extern "C" { - #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqsubu.v8i8")] - #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uqsub.v8i8")] - fn vqsub_u8_(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t; - } -vqsub_u8_(a, b) -} - -/// Saturating subtract -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vqsub.u8"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uqsub))] -pub unsafe fn vqsubq_u8(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t { - #[allow(improper_ctypes)] - extern "C" { - #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqsubu.v16i8")] - #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uqsub.v16i8")] - fn vqsubq_u8_(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t; - } -vqsubq_u8_(a, b) -} - -/// Saturating subtract -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vqsub.u16"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uqsub))] -pub unsafe fn vqsub_u16(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t { - #[allow(improper_ctypes)] - extern "C" { - #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqsubu.v4i16")] - #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uqsub.v4i16")] - fn vqsub_u16_(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t; - } -vqsub_u16_(a, b) -} - -/// Saturating subtract -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vqsub.u16"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uqsub))] -pub unsafe fn vqsubq_u16(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t { - #[allow(improper_ctypes)] - extern "C" { - #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqsubu.v8i16")] - #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uqsub.v8i16")] - fn vqsubq_u16_(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t; - } -vqsubq_u16_(a, b) -} - -/// Saturating subtract -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vqsub.u32"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uqsub))] -pub unsafe fn vqsub_u32(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t { - #[allow(improper_ctypes)] - extern "C" { - #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqsubu.v2i32")] - #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uqsub.v2i32")] - fn vqsub_u32_(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t; - } -vqsub_u32_(a, b) -} - -/// Saturating subtract -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vqsub.u32"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uqsub))] -pub unsafe fn vqsubq_u32(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t { - #[allow(improper_ctypes)] - extern "C" { - #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqsubu.v4i32")] - #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uqsub.v4i32")] - fn vqsubq_u32_(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t; - } -vqsubq_u32_(a, b) -} - -/// Saturating subtract -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vqsub.s8"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqsub))] -pub unsafe fn vqsub_s8(a: int8x8_t, b: int8x8_t) -> int8x8_t { - #[allow(improper_ctypes)] - extern "C" { - #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqsubs.v8i8")] - #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqsub.v8i8")] - fn vqsub_s8_(a: int8x8_t, b: int8x8_t) -> int8x8_t; - } -vqsub_s8_(a, b) -} - -/// Saturating subtract -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vqsub.s8"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqsub))] -pub unsafe fn vqsubq_s8(a: int8x16_t, b: int8x16_t) -> int8x16_t { - #[allow(improper_ctypes)] - extern "C" { - #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqsubs.v16i8")] - #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqsub.v16i8")] - fn vqsubq_s8_(a: int8x16_t, b: int8x16_t) -> int8x16_t; - } -vqsubq_s8_(a, b) -} - -/// Saturating subtract -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vqsub.s16"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqsub))] -pub unsafe fn vqsub_s16(a: int16x4_t, b: int16x4_t) -> int16x4_t { - #[allow(improper_ctypes)] - extern "C" { - #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqsubs.v4i16")] - #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqsub.v4i16")] - fn vqsub_s16_(a: int16x4_t, b: int16x4_t) -> int16x4_t; - } -vqsub_s16_(a, b) -} - -/// Saturating subtract -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vqsub.s16"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqsub))] -pub unsafe fn vqsubq_s16(a: int16x8_t, b: int16x8_t) -> int16x8_t { - #[allow(improper_ctypes)] - extern "C" { - #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqsubs.v8i16")] - #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqsub.v8i16")] - fn vqsubq_s16_(a: int16x8_t, b: int16x8_t) -> int16x8_t; - } -vqsubq_s16_(a, b) -} - -/// Saturating subtract -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vqsub.s32"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqsub))] -pub unsafe fn vqsub_s32(a: int32x2_t, b: int32x2_t) -> int32x2_t { - #[allow(improper_ctypes)] - extern "C" { - #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqsubs.v2i32")] - #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqsub.v2i32")] - fn vqsub_s32_(a: int32x2_t, b: int32x2_t) -> int32x2_t; - } -vqsub_s32_(a, b) -} - -/// Saturating subtract -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vqsub.s32"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqsub))] -pub unsafe fn vqsubq_s32(a: int32x4_t, b: int32x4_t) -> int32x4_t { - #[allow(improper_ctypes)] - extern "C" { - #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqsubs.v4i32")] - #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqsub.v4i32")] - fn vqsubq_s32_(a: int32x4_t, b: int32x4_t) -> int32x4_t; - } -vqsubq_s32_(a, b) -} - -/// Halving add -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vhadd.u8"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uhadd))] -pub unsafe fn vhadd_u8(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t { - #[allow(improper_ctypes)] - extern "C" { - #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vhaddu.v8i8")] - #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uhadd.v8i8")] - fn vhadd_u8_(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t; - } -vhadd_u8_(a, b) -} - -/// Halving add -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vhadd.u8"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uhadd))] -pub unsafe fn vhaddq_u8(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t { - #[allow(improper_ctypes)] - extern "C" { - #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vhaddu.v16i8")] - #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uhadd.v16i8")] - fn vhaddq_u8_(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t; - } -vhaddq_u8_(a, b) -} - -/// Halving add -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vhadd.u16"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uhadd))] -pub unsafe fn vhadd_u16(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t { - #[allow(improper_ctypes)] - extern "C" { - #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vhaddu.v4i16")] - #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uhadd.v4i16")] - fn vhadd_u16_(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t; - } -vhadd_u16_(a, b) -} - -/// Halving add -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vhadd.u16"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uhadd))] -pub unsafe fn vhaddq_u16(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t { - #[allow(improper_ctypes)] - extern "C" { - #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vhaddu.v8i16")] - #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uhadd.v8i16")] - fn vhaddq_u16_(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t; - } -vhaddq_u16_(a, b) -} - -/// Halving add -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vhadd.u32"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uhadd))] -pub unsafe fn vhadd_u32(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t { - #[allow(improper_ctypes)] - extern "C" { - #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vhaddu.v2i32")] - #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uhadd.v2i32")] - fn vhadd_u32_(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t; - } -vhadd_u32_(a, b) -} - -/// Halving add -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vhadd.u32"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uhadd))] -pub unsafe fn vhaddq_u32(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t { - #[allow(improper_ctypes)] - extern "C" { - #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vhaddu.v4i32")] - #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uhadd.v4i32")] - fn vhaddq_u32_(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t; - } -vhaddq_u32_(a, b) -} - -/// Halving add -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vhadd.s8"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(shadd))] -pub unsafe fn vhadd_s8(a: int8x8_t, b: int8x8_t) -> int8x8_t { - #[allow(improper_ctypes)] - extern "C" { - #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vhadds.v8i8")] - #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.shadd.v8i8")] - fn vhadd_s8_(a: int8x8_t, b: int8x8_t) -> int8x8_t; - } -vhadd_s8_(a, b) -} - -/// Halving add -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vhadd.s8"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(shadd))] -pub unsafe fn vhaddq_s8(a: int8x16_t, b: int8x16_t) -> int8x16_t { - #[allow(improper_ctypes)] - extern "C" { - #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vhadds.v16i8")] - #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.shadd.v16i8")] - fn vhaddq_s8_(a: int8x16_t, b: int8x16_t) -> int8x16_t; - } -vhaddq_s8_(a, b) -} - -/// Halving add -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vhadd.s16"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(shadd))] -pub unsafe fn vhadd_s16(a: int16x4_t, b: int16x4_t) -> int16x4_t { - #[allow(improper_ctypes)] - extern "C" { - #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vhadds.v4i16")] - #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.shadd.v4i16")] - fn vhadd_s16_(a: int16x4_t, b: int16x4_t) -> int16x4_t; - } -vhadd_s16_(a, b) -} - -/// Halving add -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vhadd.s16"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(shadd))] -pub unsafe fn vhaddq_s16(a: int16x8_t, b: int16x8_t) -> int16x8_t { - #[allow(improper_ctypes)] - extern "C" { - #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vhadds.v8i16")] - #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.shadd.v8i16")] - fn vhaddq_s16_(a: int16x8_t, b: int16x8_t) -> int16x8_t; - } -vhaddq_s16_(a, b) -} - -/// Halving add -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vhadd.s32"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(shadd))] -pub unsafe fn vhadd_s32(a: int32x2_t, b: int32x2_t) -> int32x2_t { - #[allow(improper_ctypes)] - extern "C" { - #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vhadds.v2i32")] - #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.shadd.v2i32")] - fn vhadd_s32_(a: int32x2_t, b: int32x2_t) -> int32x2_t; - } -vhadd_s32_(a, b) -} - -/// Halving add -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vhadd.s32"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(shadd))] -pub unsafe fn vhaddq_s32(a: int32x4_t, b: int32x4_t) -> int32x4_t { - #[allow(improper_ctypes)] - extern "C" { - #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vhadds.v4i32")] - #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.shadd.v4i32")] - fn vhaddq_s32_(a: int32x4_t, b: int32x4_t) -> int32x4_t; - } -vhaddq_s32_(a, b) -} - -/// Rounding halving add -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrhadd.u8"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(urhadd))] -pub unsafe fn vrhadd_u8(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t { - #[allow(improper_ctypes)] - extern "C" { - #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vrhaddu.v8i8")] - #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.urhadd.v8i8")] - fn vrhadd_u8_(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t; - } -vrhadd_u8_(a, b) -} - -/// Rounding halving add -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrhadd.u8"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(urhadd))] -pub unsafe fn vrhaddq_u8(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t { - #[allow(improper_ctypes)] - extern "C" { - #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vrhaddu.v16i8")] - #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.urhadd.v16i8")] - fn vrhaddq_u8_(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t; - } -vrhaddq_u8_(a, b) -} - -/// Rounding halving add -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrhadd.u16"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(urhadd))] -pub unsafe fn vrhadd_u16(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t { - #[allow(improper_ctypes)] - extern "C" { - #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vrhaddu.v4i16")] - #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.urhadd.v4i16")] - fn vrhadd_u16_(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t; - } -vrhadd_u16_(a, b) -} - -/// Rounding halving add -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrhadd.u16"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(urhadd))] -pub unsafe fn vrhaddq_u16(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t { - #[allow(improper_ctypes)] - extern "C" { - #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vrhaddu.v8i16")] - #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.urhadd.v8i16")] - fn vrhaddq_u16_(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t; - } -vrhaddq_u16_(a, b) -} - -/// Rounding halving add -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrhadd.u32"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(urhadd))] -pub unsafe fn vrhadd_u32(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t { - #[allow(improper_ctypes)] - extern "C" { - #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vrhaddu.v2i32")] - #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.urhadd.v2i32")] - fn vrhadd_u32_(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t; - } -vrhadd_u32_(a, b) -} - -/// Rounding halving add -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrhadd.u32"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(urhadd))] -pub unsafe fn vrhaddq_u32(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t { - #[allow(improper_ctypes)] - extern "C" { - #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vrhaddu.v4i32")] - #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.urhadd.v4i32")] - fn vrhaddq_u32_(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t; - } -vrhaddq_u32_(a, b) -} - -/// Rounding halving add -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrhadd.s8"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(srhadd))] -pub unsafe fn vrhadd_s8(a: int8x8_t, b: int8x8_t) -> int8x8_t { - #[allow(improper_ctypes)] - extern "C" { - #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vrhadds.v8i8")] - #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.srhadd.v8i8")] - fn vrhadd_s8_(a: int8x8_t, b: int8x8_t) -> int8x8_t; - } -vrhadd_s8_(a, b) -} - -/// Rounding halving add -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrhadd.s8"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(srhadd))] -pub unsafe fn vrhaddq_s8(a: int8x16_t, b: int8x16_t) -> int8x16_t { - #[allow(improper_ctypes)] - extern "C" { - #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vrhadds.v16i8")] - #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.srhadd.v16i8")] - fn vrhaddq_s8_(a: int8x16_t, b: int8x16_t) -> int8x16_t; - } -vrhaddq_s8_(a, b) -} - -/// Rounding halving add -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrhadd.s16"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(srhadd))] -pub unsafe fn vrhadd_s16(a: int16x4_t, b: int16x4_t) -> int16x4_t { - #[allow(improper_ctypes)] - extern "C" { - #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vrhadds.v4i16")] - #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.srhadd.v4i16")] - fn vrhadd_s16_(a: int16x4_t, b: int16x4_t) -> int16x4_t; - } -vrhadd_s16_(a, b) -} - -/// Rounding halving add -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrhadd.s16"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(srhadd))] -pub unsafe fn vrhaddq_s16(a: int16x8_t, b: int16x8_t) -> int16x8_t { - #[allow(improper_ctypes)] - extern "C" { - #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vrhadds.v8i16")] - #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.srhadd.v8i16")] - fn vrhaddq_s16_(a: int16x8_t, b: int16x8_t) -> int16x8_t; - } -vrhaddq_s16_(a, b) -} - -/// Rounding halving add -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrhadd.s32"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(srhadd))] -pub unsafe fn vrhadd_s32(a: int32x2_t, b: int32x2_t) -> int32x2_t { - #[allow(improper_ctypes)] - extern "C" { - #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vrhadds.v2i32")] - #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.srhadd.v2i32")] - fn vrhadd_s32_(a: int32x2_t, b: int32x2_t) -> int32x2_t; - } -vrhadd_s32_(a, b) -} - -/// Rounding halving add -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrhadd.s32"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(srhadd))] -pub unsafe fn vrhaddq_s32(a: int32x4_t, b: int32x4_t) -> int32x4_t { - #[allow(improper_ctypes)] - extern "C" { - #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vrhadds.v4i32")] - #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.srhadd.v4i32")] - fn vrhaddq_s32_(a: int32x4_t, b: int32x4_t) -> int32x4_t; - } -vrhaddq_s32_(a, b) -} - -/// Saturating add -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vqadd.u8"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uqadd))] -pub unsafe fn vqadd_u8(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t { - #[allow(improper_ctypes)] - extern "C" { - #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqaddu.v8i8")] - #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uqadd.v8i8")] - fn vqadd_u8_(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t; - } -vqadd_u8_(a, b) -} - -/// Saturating add -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vqadd.u8"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uqadd))] -pub unsafe fn vqaddq_u8(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t { - #[allow(improper_ctypes)] - extern "C" { - #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqaddu.v16i8")] - #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uqadd.v16i8")] - fn vqaddq_u8_(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t; - } -vqaddq_u8_(a, b) -} - -/// Saturating add -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vqadd.u16"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uqadd))] -pub unsafe fn vqadd_u16(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t { - #[allow(improper_ctypes)] - extern "C" { - #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqaddu.v4i16")] - #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uqadd.v4i16")] - fn vqadd_u16_(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t; - } -vqadd_u16_(a, b) -} - -/// Saturating add -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vqadd.u16"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uqadd))] -pub unsafe fn vqaddq_u16(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t { - #[allow(improper_ctypes)] - extern "C" { - #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqaddu.v8i16")] - #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uqadd.v8i16")] - fn vqaddq_u16_(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t; - } -vqaddq_u16_(a, b) -} - -/// Saturating add -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vqadd.u32"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uqadd))] -pub unsafe fn vqadd_u32(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t { - #[allow(improper_ctypes)] - extern "C" { - #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqaddu.v2i32")] - #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uqadd.v2i32")] - fn vqadd_u32_(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t; - } -vqadd_u32_(a, b) -} - -/// Saturating add -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vqadd.u32"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uqadd))] -pub unsafe fn vqaddq_u32(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t { - #[allow(improper_ctypes)] - extern "C" { - #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqaddu.v4i32")] - #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uqadd.v4i32")] - fn vqaddq_u32_(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t; - } -vqaddq_u32_(a, b) -} - -/// Saturating add -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vqadd.s8"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqadd))] -pub unsafe fn vqadd_s8(a: int8x8_t, b: int8x8_t) -> int8x8_t { - #[allow(improper_ctypes)] - extern "C" { - #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqadds.v8i8")] - #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqadd.v8i8")] - fn vqadd_s8_(a: int8x8_t, b: int8x8_t) -> int8x8_t; - } -vqadd_s8_(a, b) -} - -/// Saturating add -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vqadd.s8"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqadd))] -pub unsafe fn vqaddq_s8(a: int8x16_t, b: int8x16_t) -> int8x16_t { - #[allow(improper_ctypes)] - extern "C" { - #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqadds.v16i8")] - #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqadd.v16i8")] - fn vqaddq_s8_(a: int8x16_t, b: int8x16_t) -> int8x16_t; - } -vqaddq_s8_(a, b) -} - -/// Saturating add -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vqadd.s16"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqadd))] -pub unsafe fn vqadd_s16(a: int16x4_t, b: int16x4_t) -> int16x4_t { - #[allow(improper_ctypes)] - extern "C" { - #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqadds.v4i16")] - #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqadd.v4i16")] - fn vqadd_s16_(a: int16x4_t, b: int16x4_t) -> int16x4_t; - } -vqadd_s16_(a, b) -} - -/// Saturating add -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vqadd.s16"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqadd))] -pub unsafe fn vqaddq_s16(a: int16x8_t, b: int16x8_t) -> int16x8_t { - #[allow(improper_ctypes)] - extern "C" { - #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqadds.v8i16")] - #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqadd.v8i16")] - fn vqaddq_s16_(a: int16x8_t, b: int16x8_t) -> int16x8_t; - } -vqaddq_s16_(a, b) -} - -/// Saturating add -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vqadd.s32"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqadd))] -pub unsafe fn vqadd_s32(a: int32x2_t, b: int32x2_t) -> int32x2_t { - #[allow(improper_ctypes)] - extern "C" { - #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqadds.v2i32")] - #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqadd.v2i32")] - fn vqadd_s32_(a: int32x2_t, b: int32x2_t) -> int32x2_t; - } -vqadd_s32_(a, b) -} - -/// Saturating add -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vqadd.s32"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqadd))] -pub unsafe fn vqaddq_s32(a: int32x4_t, b: int32x4_t) -> int32x4_t { - #[allow(improper_ctypes)] - extern "C" { - #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqadds.v4i32")] - #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqadd.v4i32")] - fn vqaddq_s32_(a: int32x4_t, b: int32x4_t) -> int32x4_t; - } -vqaddq_s32_(a, b) -} - -/// Multiply -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmul.i8"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mul))] -pub unsafe fn vmul_s8(a: int8x8_t, b: int8x8_t) -> int8x8_t { - simd_mul(a, b) -} - -/// Multiply -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmul.i8"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mul))] -pub unsafe fn vmulq_s8(a: int8x16_t, b: int8x16_t) -> int8x16_t { - simd_mul(a, b) -} - -/// Multiply -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmul.i16"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mul))] -pub unsafe fn vmul_s16(a: int16x4_t, b: int16x4_t) -> int16x4_t { - simd_mul(a, b) -} - -/// Multiply -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmul.i16"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mul))] -pub unsafe fn vmulq_s16(a: int16x8_t, b: int16x8_t) -> int16x8_t { - simd_mul(a, b) -} - -/// Multiply -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmul.i32"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mul))] -pub unsafe fn vmul_s32(a: int32x2_t, b: int32x2_t) -> int32x2_t { - simd_mul(a, b) -} - -/// Multiply -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmul.i32"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mul))] -pub unsafe fn vmulq_s32(a: int32x4_t, b: int32x4_t) -> int32x4_t { - simd_mul(a, b) -} - -/// Multiply -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmul.i8"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mul))] -pub unsafe fn vmul_u8(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t { - simd_mul(a, b) -} - -/// Multiply -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmul.i8"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mul))] -pub unsafe fn vmulq_u8(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t { - simd_mul(a, b) -} - -/// Multiply -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmul.i16"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mul))] -pub unsafe fn vmul_u16(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t { - simd_mul(a, b) -} - -/// Multiply -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmul.i16"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mul))] -pub unsafe fn vmulq_u16(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t { - simd_mul(a, b) -} - -/// Multiply -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmul.i32"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mul))] -pub unsafe fn vmul_u32(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t { - simd_mul(a, b) -} - -/// Multiply -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmul.i32"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mul))] -pub unsafe fn vmulq_u32(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t { - simd_mul(a, b) -} - -/// Multiply -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmul.f32"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fmul))] -pub unsafe fn vmul_f32(a: float32x2_t, b: float32x2_t) -> float32x2_t { - simd_mul(a, b) -} - -/// Multiply -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmul.f32"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fmul))] -pub unsafe fn vmulq_f32(a: float32x4_t, b: float32x4_t) -> float32x4_t { - simd_mul(a, b) -} - -/// Subtract -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vsub.i8"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sub))] -pub unsafe fn vsub_s8(a: int8x8_t, b: int8x8_t) -> int8x8_t { - simd_sub(a, b) -} - -/// Subtract -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vsub.i8"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sub))] -pub unsafe fn vsubq_s8(a: int8x16_t, b: int8x16_t) -> int8x16_t { - simd_sub(a, b) -} - -/// Subtract -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vsub.i16"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sub))] -pub unsafe fn vsub_s16(a: int16x4_t, b: int16x4_t) -> int16x4_t { - simd_sub(a, b) -} - -/// Subtract -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vsub.i16"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sub))] -pub unsafe fn vsubq_s16(a: int16x8_t, b: int16x8_t) -> int16x8_t { - simd_sub(a, b) -} - -/// Subtract -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vsub.i32"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sub))] -pub unsafe fn vsub_s32(a: int32x2_t, b: int32x2_t) -> int32x2_t { - simd_sub(a, b) -} - -/// Subtract -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vsub.i32"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sub))] -pub unsafe fn vsubq_s32(a: int32x4_t, b: int32x4_t) -> int32x4_t { - simd_sub(a, b) -} - -/// Subtract -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vsub.i8"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sub))] -pub unsafe fn vsub_u8(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t { - simd_sub(a, b) -} - -/// Subtract -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vsub.i8"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sub))] -pub unsafe fn vsubq_u8(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t { - simd_sub(a, b) -} - -/// Subtract -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vsub.i16"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sub))] -pub unsafe fn vsub_u16(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t { - simd_sub(a, b) -} - -/// Subtract -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vsub.i16"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sub))] -pub unsafe fn vsubq_u16(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t { - simd_sub(a, b) -} - -/// Subtract -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vsub.i32"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sub))] -pub unsafe fn vsub_u32(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t { - simd_sub(a, b) -} - -/// Subtract -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vsub.i32"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sub))] -pub unsafe fn vsubq_u32(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t { - simd_sub(a, b) -} - -/// Subtract -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vsub.i64"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sub))] -pub unsafe fn vsub_s64(a: int64x1_t, b: int64x1_t) -> int64x1_t { - simd_sub(a, b) -} - -/// Subtract -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vsub.i64"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sub))] -pub unsafe fn vsubq_s64(a: int64x2_t, b: int64x2_t) -> int64x2_t { - simd_sub(a, b) -} - -/// Subtract -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vsub.i64"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sub))] -pub unsafe fn vsub_u64(a: uint64x1_t, b: uint64x1_t) -> uint64x1_t { - simd_sub(a, b) -} - -/// Subtract -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vsub.i64"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sub))] -pub unsafe fn vsubq_u64(a: uint64x2_t, b: uint64x2_t) -> uint64x2_t { - simd_sub(a, b) -} - -/// Subtract -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vsub.f32"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fsub))] -pub unsafe fn vsub_f32(a: float32x2_t, b: float32x2_t) -> float32x2_t { - simd_sub(a, b) -} - -/// Subtract -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vsub.f32"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fsub))] -pub unsafe fn vsubq_f32(a: float32x4_t, b: float32x4_t) -> float32x4_t { - simd_sub(a, b) -} - -/// Signed halving subtract -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vhsub.u8"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uhsub))] -pub unsafe fn vhsub_u8(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t { - #[allow(improper_ctypes)] - extern "C" { - #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vhsubu.v8i8")] - #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uhsub.v8i8")] - fn vhsub_u8_(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t; - } -vhsub_u8_(a, b) -} - -/// Signed halving subtract -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vhsub.u8"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uhsub))] -pub unsafe fn vhsubq_u8(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t { - #[allow(improper_ctypes)] - extern "C" { - #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vhsubu.v16i8")] - #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uhsub.v16i8")] - fn vhsubq_u8_(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t; - } -vhsubq_u8_(a, b) -} - -/// Signed halving subtract -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vhsub.u16"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uhsub))] -pub unsafe fn vhsub_u16(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t { - #[allow(improper_ctypes)] - extern "C" { - #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vhsubu.v4i16")] - #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uhsub.v4i16")] - fn vhsub_u16_(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t; - } -vhsub_u16_(a, b) -} - -/// Signed halving subtract -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vhsub.u16"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uhsub))] -pub unsafe fn vhsubq_u16(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t { - #[allow(improper_ctypes)] - extern "C" { - #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vhsubu.v8i16")] - #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uhsub.v8i16")] - fn vhsubq_u16_(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t; - } -vhsubq_u16_(a, b) -} - -/// Signed halving subtract -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vhsub.u32"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uhsub))] -pub unsafe fn vhsub_u32(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t { - #[allow(improper_ctypes)] - extern "C" { - #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vhsubu.v2i32")] - #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uhsub.v2i32")] - fn vhsub_u32_(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t; - } -vhsub_u32_(a, b) -} - -/// Signed halving subtract -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vhsub.u32"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uhsub))] -pub unsafe fn vhsubq_u32(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t { - #[allow(improper_ctypes)] - extern "C" { - #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vhsubu.v4i32")] - #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uhsub.v4i32")] - fn vhsubq_u32_(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t; - } -vhsubq_u32_(a, b) -} - -/// Signed halving subtract -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vhsub.s8"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(shsub))] -pub unsafe fn vhsub_s8(a: int8x8_t, b: int8x8_t) -> int8x8_t { - #[allow(improper_ctypes)] - extern "C" { - #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vhsubs.v8i8")] - #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.shsub.v8i8")] - fn vhsub_s8_(a: int8x8_t, b: int8x8_t) -> int8x8_t; - } -vhsub_s8_(a, b) -} - -/// Signed halving subtract -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vhsub.s8"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(shsub))] -pub unsafe fn vhsubq_s8(a: int8x16_t, b: int8x16_t) -> int8x16_t { - #[allow(improper_ctypes)] - extern "C" { - #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vhsubs.v16i8")] - #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.shsub.v16i8")] - fn vhsubq_s8_(a: int8x16_t, b: int8x16_t) -> int8x16_t; - } -vhsubq_s8_(a, b) -} - -/// Signed halving subtract -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vhsub.s16"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(shsub))] -pub unsafe fn vhsub_s16(a: int16x4_t, b: int16x4_t) -> int16x4_t { - #[allow(improper_ctypes)] - extern "C" { - #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vhsubs.v4i16")] - #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.shsub.v4i16")] - fn vhsub_s16_(a: int16x4_t, b: int16x4_t) -> int16x4_t; - } -vhsub_s16_(a, b) -} - -/// Signed halving subtract -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vhsub.s16"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(shsub))] -pub unsafe fn vhsubq_s16(a: int16x8_t, b: int16x8_t) -> int16x8_t { - #[allow(improper_ctypes)] - extern "C" { - #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vhsubs.v8i16")] - #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.shsub.v8i16")] - fn vhsubq_s16_(a: int16x8_t, b: int16x8_t) -> int16x8_t; - } -vhsubq_s16_(a, b) -} - -/// Signed halving subtract -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vhsub.s32"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(shsub))] -pub unsafe fn vhsub_s32(a: int32x2_t, b: int32x2_t) -> int32x2_t { - #[allow(improper_ctypes)] - extern "C" { - #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vhsubs.v2i32")] - #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.shsub.v2i32")] - fn vhsub_s32_(a: int32x2_t, b: int32x2_t) -> int32x2_t; - } -vhsub_s32_(a, b) -} - -/// Signed halving subtract -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vhsub.s32"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(shsub))] -pub unsafe fn vhsubq_s32(a: int32x4_t, b: int32x4_t) -> int32x4_t { - #[allow(improper_ctypes)] - extern "C" { - #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vhsubs.v4i32")] - #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.shsub.v4i32")] - fn vhsubq_s32_(a: int32x4_t, b: int32x4_t) -> int32x4_t; - } -vhsubq_s32_(a, b) -} - -/// Maximum (vector) -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmax))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(smax))] -pub unsafe fn vmax_s8(a: int8x8_t, b: int8x8_t) -> int8x8_t { - #[allow(improper_ctypes)] - extern "C" { - #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vmaxs.v8i8")] - #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.smax.v8i8")] - fn vmax_s8_(a: int8x8_t, b: int8x8_t) -> int8x8_t; - } -vmax_s8_(a, b) -} - -/// Maximum (vector) -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmax))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(smax))] -pub unsafe fn vmaxq_s8(a: int8x16_t, b: int8x16_t) -> int8x16_t { - #[allow(improper_ctypes)] - extern "C" { - #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vmaxs.v16i8")] - #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.smax.v16i8")] - fn vmaxq_s8_(a: int8x16_t, b: int8x16_t) -> int8x16_t; - } -vmaxq_s8_(a, b) -} - -/// Maximum (vector) -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmax))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(smax))] -pub unsafe fn vmax_s16(a: int16x4_t, b: int16x4_t) -> int16x4_t { - #[allow(improper_ctypes)] - extern "C" { - #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vmaxs.v4i16")] - #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.smax.v4i16")] - fn vmax_s16_(a: int16x4_t, b: int16x4_t) -> int16x4_t; - } -vmax_s16_(a, b) -} - -/// Maximum (vector) -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmax))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(smax))] -pub unsafe fn vmaxq_s16(a: int16x8_t, b: int16x8_t) -> int16x8_t { - #[allow(improper_ctypes)] - extern "C" { - #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vmaxs.v8i16")] - #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.smax.v8i16")] - fn vmaxq_s16_(a: int16x8_t, b: int16x8_t) -> int16x8_t; - } -vmaxq_s16_(a, b) -} - -/// Maximum (vector) -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmax))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(smax))] -pub unsafe fn vmax_s32(a: int32x2_t, b: int32x2_t) -> int32x2_t { - #[allow(improper_ctypes)] - extern "C" { - #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vmaxs.v2i32")] - #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.smax.v2i32")] - fn vmax_s32_(a: int32x2_t, b: int32x2_t) -> int32x2_t; - } -vmax_s32_(a, b) -} - -/// Maximum (vector) -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmax))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(smax))] -pub unsafe fn vmaxq_s32(a: int32x4_t, b: int32x4_t) -> int32x4_t { - #[allow(improper_ctypes)] - extern "C" { - #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vmaxs.v4i32")] - #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.smax.v4i32")] - fn vmaxq_s32_(a: int32x4_t, b: int32x4_t) -> int32x4_t; - } -vmaxq_s32_(a, b) -} - -/// Maximum (vector) -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmax))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(umax))] -pub unsafe fn vmax_u8(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t { - #[allow(improper_ctypes)] - extern "C" { - #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vmaxu.v8i8")] - #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.umax.v8i8")] - fn vmax_u8_(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t; - } -vmax_u8_(a, b) -} - -/// Maximum (vector) -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmax))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(umax))] -pub unsafe fn vmaxq_u8(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t { - #[allow(improper_ctypes)] - extern "C" { - #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vmaxu.v16i8")] - #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.umax.v16i8")] - fn vmaxq_u8_(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t; - } -vmaxq_u8_(a, b) -} - -/// Maximum (vector) -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmax))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(umax))] -pub unsafe fn vmax_u16(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t { - #[allow(improper_ctypes)] - extern "C" { - #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vmaxu.v4i16")] - #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.umax.v4i16")] - fn vmax_u16_(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t; - } -vmax_u16_(a, b) -} - -/// Maximum (vector) -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmax))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(umax))] -pub unsafe fn vmaxq_u16(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t { - #[allow(improper_ctypes)] - extern "C" { - #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vmaxu.v8i16")] - #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.umax.v8i16")] - fn vmaxq_u16_(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t; - } -vmaxq_u16_(a, b) -} - -/// Maximum (vector) -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmax))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(umax))] -pub unsafe fn vmax_u32(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t { - #[allow(improper_ctypes)] - extern "C" { - #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vmaxu.v2i32")] - #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.umax.v2i32")] - fn vmax_u32_(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t; - } -vmax_u32_(a, b) -} - -/// Maximum (vector) -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmax))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(umax))] -pub unsafe fn vmaxq_u32(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t { - #[allow(improper_ctypes)] - extern "C" { - #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vmaxu.v4i32")] - #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.umax.v4i32")] - fn vmaxq_u32_(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t; - } -vmaxq_u32_(a, b) -} - -/// Maximum (vector) -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmax))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fmax))] -pub unsafe fn vmax_f32(a: float32x2_t, b: float32x2_t) -> float32x2_t { - #[allow(improper_ctypes)] - extern "C" { - #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vmaxs.v2f32")] - #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fmax.v2f32")] - fn vmax_f32_(a: float32x2_t, b: float32x2_t) -> float32x2_t; - } -vmax_f32_(a, b) -} - -/// Maximum (vector) -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmax))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fmax))] -pub unsafe fn vmaxq_f32(a: float32x4_t, b: float32x4_t) -> float32x4_t { - #[allow(improper_ctypes)] - extern "C" { - #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vmaxs.v4f32")] - #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fmax.v4f32")] - fn vmaxq_f32_(a: float32x4_t, b: float32x4_t) -> float32x4_t; - } -vmaxq_f32_(a, b) -} - -/// Minimum (vector) -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmin))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(smin))] -pub unsafe fn vmin_s8(a: int8x8_t, b: int8x8_t) -> int8x8_t { - #[allow(improper_ctypes)] - extern "C" { - #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vmins.v8i8")] - #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.smin.v8i8")] - fn vmin_s8_(a: int8x8_t, b: int8x8_t) -> int8x8_t; - } -vmin_s8_(a, b) -} - -/// Minimum (vector) -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmin))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(smin))] -pub unsafe fn vminq_s8(a: int8x16_t, b: int8x16_t) -> int8x16_t { - #[allow(improper_ctypes)] - extern "C" { - #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vmins.v16i8")] - #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.smin.v16i8")] - fn vminq_s8_(a: int8x16_t, b: int8x16_t) -> int8x16_t; - } -vminq_s8_(a, b) -} - -/// Minimum (vector) -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmin))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(smin))] -pub unsafe fn vmin_s16(a: int16x4_t, b: int16x4_t) -> int16x4_t { - #[allow(improper_ctypes)] - extern "C" { - #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vmins.v4i16")] - #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.smin.v4i16")] - fn vmin_s16_(a: int16x4_t, b: int16x4_t) -> int16x4_t; - } -vmin_s16_(a, b) -} - -/// Minimum (vector) -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmin))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(smin))] -pub unsafe fn vminq_s16(a: int16x8_t, b: int16x8_t) -> int16x8_t { - #[allow(improper_ctypes)] - extern "C" { - #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vmins.v8i16")] - #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.smin.v8i16")] - fn vminq_s16_(a: int16x8_t, b: int16x8_t) -> int16x8_t; - } -vminq_s16_(a, b) -} - -/// Minimum (vector) -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmin))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(smin))] -pub unsafe fn vmin_s32(a: int32x2_t, b: int32x2_t) -> int32x2_t { - #[allow(improper_ctypes)] - extern "C" { - #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vmins.v2i32")] - #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.smin.v2i32")] - fn vmin_s32_(a: int32x2_t, b: int32x2_t) -> int32x2_t; - } -vmin_s32_(a, b) -} - -/// Minimum (vector) -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmin))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(smin))] -pub unsafe fn vminq_s32(a: int32x4_t, b: int32x4_t) -> int32x4_t { - #[allow(improper_ctypes)] - extern "C" { - #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vmins.v4i32")] - #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.smin.v4i32")] - fn vminq_s32_(a: int32x4_t, b: int32x4_t) -> int32x4_t; - } -vminq_s32_(a, b) -} - -/// Minimum (vector) -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmin))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(umin))] -pub unsafe fn vmin_u8(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t { - #[allow(improper_ctypes)] - extern "C" { - #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vminu.v8i8")] - #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.umin.v8i8")] - fn vmin_u8_(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t; - } -vmin_u8_(a, b) -} - -/// Minimum (vector) -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmin))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(umin))] -pub unsafe fn vminq_u8(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t { - #[allow(improper_ctypes)] - extern "C" { - #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vminu.v16i8")] - #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.umin.v16i8")] - fn vminq_u8_(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t; - } -vminq_u8_(a, b) -} - -/// Minimum (vector) -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmin))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(umin))] -pub unsafe fn vmin_u16(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t { - #[allow(improper_ctypes)] - extern "C" { - #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vminu.v4i16")] - #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.umin.v4i16")] - fn vmin_u16_(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t; - } -vmin_u16_(a, b) -} - -/// Minimum (vector) -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmin))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(umin))] -pub unsafe fn vminq_u16(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t { - #[allow(improper_ctypes)] - extern "C" { - #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vminu.v8i16")] - #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.umin.v8i16")] - fn vminq_u16_(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t; - } -vminq_u16_(a, b) -} - -/// Minimum (vector) -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmin))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(umin))] -pub unsafe fn vmin_u32(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t { - #[allow(improper_ctypes)] - extern "C" { - #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vminu.v2i32")] - #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.umin.v2i32")] - fn vmin_u32_(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t; - } -vmin_u32_(a, b) -} - -/// Minimum (vector) -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmin))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(umin))] -pub unsafe fn vminq_u32(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t { - #[allow(improper_ctypes)] - extern "C" { - #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vminu.v4i32")] - #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.umin.v4i32")] - fn vminq_u32_(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t; - } -vminq_u32_(a, b) -} - -/// Minimum (vector) -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmin))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fmin))] -pub unsafe fn vmin_f32(a: float32x2_t, b: float32x2_t) -> float32x2_t { - #[allow(improper_ctypes)] - extern "C" { - #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vmins.v2f32")] - #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fmin.v2f32")] - fn vmin_f32_(a: float32x2_t, b: float32x2_t) -> float32x2_t; - } -vmin_f32_(a, b) -} - -/// Minimum (vector) -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmin))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fmin))] -pub unsafe fn vminq_f32(a: float32x4_t, b: float32x4_t) -> float32x4_t { - #[allow(improper_ctypes)] - extern "C" { - #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vmins.v4f32")] - #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fmin.v4f32")] - fn vminq_f32_(a: float32x4_t, b: float32x4_t) -> float32x4_t; - } -vminq_f32_(a, b) -} - -#[cfg(test)] -#[allow(overflowing_literals)] -mod test { - use super::*; - use crate::core_arch::simd::*; - use std::mem::transmute; - use stdarch_test::simd_test; - - #[simd_test(enable = "neon")] - unsafe fn test_vand_s8() { - let a: i8x8 = i8x8::new(0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07); - let b: i8x8 = i8x8::new(0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F); - let e: i8x8 = i8x8::new(0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07); - let r: i8x8 = transmute(vand_s8(transmute(a), transmute(b))); - assert_eq!(r, e); - - let a: i8x8 = i8x8::new(0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07); - let b: i8x8 = i8x8::new(0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00); - let e: i8x8 = i8x8::new(0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00); - let r: i8x8 = transmute(vand_s8(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vandq_s8() { - let a: i8x16 = i8x16::new(0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x00); - let b: i8x16 = i8x16::new(0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F); - let e: i8x16 = i8x16::new(0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x00); - let r: i8x16 = transmute(vandq_s8(transmute(a), transmute(b))); - assert_eq!(r, e); - - let a: i8x16 = i8x16::new(0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x00); - let b: i8x16 = i8x16::new(0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00); - let e: i8x16 = i8x16::new(0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00); - let r: i8x16 = transmute(vandq_s8(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vand_s16() { - let a: i16x4 = i16x4::new(0x00, 0x01, 0x02, 0x03); - let b: i16x4 = i16x4::new(0x0F, 0x0F, 0x0F, 0x0F); - let e: i16x4 = i16x4::new(0x00, 0x01, 0x02, 0x03); - let r: i16x4 = transmute(vand_s16(transmute(a), transmute(b))); - assert_eq!(r, e); - - let a: i16x4 = i16x4::new(0x00, 0x01, 0x02, 0x03); - let b: i16x4 = i16x4::new(0x00, 0x00, 0x00, 0x00); - let e: i16x4 = i16x4::new(0x00, 0x00, 0x00, 0x00); - let r: i16x4 = transmute(vand_s16(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vandq_s16() { - let a: i16x8 = i16x8::new(0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07); - let b: i16x8 = i16x8::new(0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F); - let e: i16x8 = i16x8::new(0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07); - let r: i16x8 = transmute(vandq_s16(transmute(a), transmute(b))); - assert_eq!(r, e); - - let a: i16x8 = i16x8::new(0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07); - let b: i16x8 = i16x8::new(0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00); - let e: i16x8 = i16x8::new(0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00); - let r: i16x8 = transmute(vandq_s16(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vand_s32() { - let a: i32x2 = i32x2::new(0x00, 0x01); - let b: i32x2 = i32x2::new(0x0F, 0x0F); - let e: i32x2 = i32x2::new(0x00, 0x01); - let r: i32x2 = transmute(vand_s32(transmute(a), transmute(b))); - assert_eq!(r, e); - - let a: i32x2 = i32x2::new(0x00, 0x01); - let b: i32x2 = i32x2::new(0x00, 0x00); - let e: i32x2 = i32x2::new(0x00, 0x00); - let r: i32x2 = transmute(vand_s32(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vandq_s32() { - let a: i32x4 = i32x4::new(0x00, 0x01, 0x02, 0x03); - let b: i32x4 = i32x4::new(0x0F, 0x0F, 0x0F, 0x0F); - let e: i32x4 = i32x4::new(0x00, 0x01, 0x02, 0x03); - let r: i32x4 = transmute(vandq_s32(transmute(a), transmute(b))); - assert_eq!(r, e); - - let a: i32x4 = i32x4::new(0x00, 0x01, 0x02, 0x03); - let b: i32x4 = i32x4::new(0x00, 0x00, 0x00, 0x00); - let e: i32x4 = i32x4::new(0x00, 0x00, 0x00, 0x00); - let r: i32x4 = transmute(vandq_s32(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vand_u8() { - let a: u8x8 = u8x8::new(0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07); - let b: u8x8 = u8x8::new(0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F); - let e: u8x8 = u8x8::new(0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07); - let r: u8x8 = transmute(vand_u8(transmute(a), transmute(b))); - assert_eq!(r, e); - - let a: u8x8 = u8x8::new(0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07); - let b: u8x8 = u8x8::new(0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00); - let e: u8x8 = u8x8::new(0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00); - let r: u8x8 = transmute(vand_u8(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vandq_u8() { - let a: u8x16 = u8x16::new(0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x00); - let b: u8x16 = u8x16::new(0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F); - let e: u8x16 = u8x16::new(0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x00); - let r: u8x16 = transmute(vandq_u8(transmute(a), transmute(b))); - assert_eq!(r, e); - - let a: u8x16 = u8x16::new(0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x00); - let b: u8x16 = u8x16::new(0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00); - let e: u8x16 = u8x16::new(0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00); - let r: u8x16 = transmute(vandq_u8(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vand_u16() { - let a: u16x4 = u16x4::new(0x00, 0x01, 0x02, 0x03); - let b: u16x4 = u16x4::new(0x0F, 0x0F, 0x0F, 0x0F); - let e: u16x4 = u16x4::new(0x00, 0x01, 0x02, 0x03); - let r: u16x4 = transmute(vand_u16(transmute(a), transmute(b))); - assert_eq!(r, e); - - let a: u16x4 = u16x4::new(0x00, 0x01, 0x02, 0x03); - let b: u16x4 = u16x4::new(0x00, 0x00, 0x00, 0x00); - let e: u16x4 = u16x4::new(0x00, 0x00, 0x00, 0x00); - let r: u16x4 = transmute(vand_u16(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vandq_u16() { - let a: u16x8 = u16x8::new(0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07); - let b: u16x8 = u16x8::new(0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F); - let e: u16x8 = u16x8::new(0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07); - let r: u16x8 = transmute(vandq_u16(transmute(a), transmute(b))); - assert_eq!(r, e); - - let a: u16x8 = u16x8::new(0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07); - let b: u16x8 = u16x8::new(0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00); - let e: u16x8 = u16x8::new(0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00); - let r: u16x8 = transmute(vandq_u16(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vand_u32() { - let a: u32x2 = u32x2::new(0x00, 0x01); - let b: u32x2 = u32x2::new(0x0F, 0x0F); - let e: u32x2 = u32x2::new(0x00, 0x01); - let r: u32x2 = transmute(vand_u32(transmute(a), transmute(b))); - assert_eq!(r, e); - - let a: u32x2 = u32x2::new(0x00, 0x01); - let b: u32x2 = u32x2::new(0x00, 0x00); - let e: u32x2 = u32x2::new(0x00, 0x00); - let r: u32x2 = transmute(vand_u32(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vandq_u32() { - let a: u32x4 = u32x4::new(0x00, 0x01, 0x02, 0x03); - let b: u32x4 = u32x4::new(0x0F, 0x0F, 0x0F, 0x0F); - let e: u32x4 = u32x4::new(0x00, 0x01, 0x02, 0x03); - let r: u32x4 = transmute(vandq_u32(transmute(a), transmute(b))); - assert_eq!(r, e); - - let a: u32x4 = u32x4::new(0x00, 0x01, 0x02, 0x03); - let b: u32x4 = u32x4::new(0x00, 0x00, 0x00, 0x00); - let e: u32x4 = u32x4::new(0x00, 0x00, 0x00, 0x00); - let r: u32x4 = transmute(vandq_u32(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vand_s64() { - let a: i64x1 = i64x1::new(0x00); - let b: i64x1 = i64x1::new(0x0F); - let e: i64x1 = i64x1::new(0x00); - let r: i64x1 = transmute(vand_s64(transmute(a), transmute(b))); - assert_eq!(r, e); - - let a: i64x1 = i64x1::new(0x00); - let b: i64x1 = i64x1::new(0x00); - let e: i64x1 = i64x1::new(0x00); - let r: i64x1 = transmute(vand_s64(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vandq_s64() { - let a: i64x2 = i64x2::new(0x00, 0x01); - let b: i64x2 = i64x2::new(0x0F, 0x0F); - let e: i64x2 = i64x2::new(0x00, 0x01); - let r: i64x2 = transmute(vandq_s64(transmute(a), transmute(b))); - assert_eq!(r, e); - - let a: i64x2 = i64x2::new(0x00, 0x01); - let b: i64x2 = i64x2::new(0x00, 0x00); - let e: i64x2 = i64x2::new(0x00, 0x00); - let r: i64x2 = transmute(vandq_s64(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vand_u64() { - let a: u64x1 = u64x1::new(0x00); - let b: u64x1 = u64x1::new(0x0F); - let e: u64x1 = u64x1::new(0x00); - let r: u64x1 = transmute(vand_u64(transmute(a), transmute(b))); - assert_eq!(r, e); - - let a: u64x1 = u64x1::new(0x00); - let b: u64x1 = u64x1::new(0x00); - let e: u64x1 = u64x1::new(0x00); - let r: u64x1 = transmute(vand_u64(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vandq_u64() { - let a: u64x2 = u64x2::new(0x00, 0x01); - let b: u64x2 = u64x2::new(0x0F, 0x0F); - let e: u64x2 = u64x2::new(0x00, 0x01); - let r: u64x2 = transmute(vandq_u64(transmute(a), transmute(b))); - assert_eq!(r, e); - - let a: u64x2 = u64x2::new(0x00, 0x01); - let b: u64x2 = u64x2::new(0x00, 0x00); - let e: u64x2 = u64x2::new(0x00, 0x00); - let r: u64x2 = transmute(vandq_u64(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vorr_s8() { - let a: i8x8 = i8x8::new(0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07); - let b: i8x8 = i8x8::new(0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00); - let e: i8x8 = i8x8::new(0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07); - let r: i8x8 = transmute(vorr_s8(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vorrq_s8() { - let a: i8x16 = i8x16::new(0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F); - let b: i8x16 = i8x16::new(0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00); - let e: i8x16 = i8x16::new(0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F); - let r: i8x16 = transmute(vorrq_s8(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vorr_s16() { - let a: i16x4 = i16x4::new(0x00, 0x01, 0x02, 0x03); - let b: i16x4 = i16x4::new(0x00, 0x00, 0x00, 0x00); - let e: i16x4 = i16x4::new(0x00, 0x01, 0x02, 0x03); - let r: i16x4 = transmute(vorr_s16(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vorrq_s16() { - let a: i16x8 = i16x8::new(0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07); - let b: i16x8 = i16x8::new(0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00); - let e: i16x8 = i16x8::new(0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07); - let r: i16x8 = transmute(vorrq_s16(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vorr_s32() { - let a: i32x2 = i32x2::new(0x00, 0x01); - let b: i32x2 = i32x2::new(0x00, 0x00); - let e: i32x2 = i32x2::new(0x00, 0x01); - let r: i32x2 = transmute(vorr_s32(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vorrq_s32() { - let a: i32x4 = i32x4::new(0x00, 0x01, 0x02, 0x03); - let b: i32x4 = i32x4::new(0x00, 0x00, 0x00, 0x00); - let e: i32x4 = i32x4::new(0x00, 0x01, 0x02, 0x03); - let r: i32x4 = transmute(vorrq_s32(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vorr_u8() { - let a: u8x8 = u8x8::new(0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07); - let b: u8x8 = u8x8::new(0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00); - let e: u8x8 = u8x8::new(0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07); - let r: u8x8 = transmute(vorr_u8(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vorrq_u8() { - let a: u8x16 = u8x16::new(0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F); - let b: u8x16 = u8x16::new(0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00); - let e: u8x16 = u8x16::new(0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F); - let r: u8x16 = transmute(vorrq_u8(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vorr_u16() { - let a: u16x4 = u16x4::new(0x00, 0x01, 0x02, 0x03); - let b: u16x4 = u16x4::new(0x00, 0x00, 0x00, 0x00); - let e: u16x4 = u16x4::new(0x00, 0x01, 0x02, 0x03); - let r: u16x4 = transmute(vorr_u16(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vorrq_u16() { - let a: u16x8 = u16x8::new(0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07); - let b: u16x8 = u16x8::new(0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00); - let e: u16x8 = u16x8::new(0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07); - let r: u16x8 = transmute(vorrq_u16(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vorr_u32() { - let a: u32x2 = u32x2::new(0x00, 0x01); - let b: u32x2 = u32x2::new(0x00, 0x00); - let e: u32x2 = u32x2::new(0x00, 0x01); - let r: u32x2 = transmute(vorr_u32(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vorrq_u32() { - let a: u32x4 = u32x4::new(0x00, 0x01, 0x02, 0x03); - let b: u32x4 = u32x4::new(0x00, 0x00, 0x00, 0x00); - let e: u32x4 = u32x4::new(0x00, 0x01, 0x02, 0x03); - let r: u32x4 = transmute(vorrq_u32(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vorr_s64() { - let a: i64x1 = i64x1::new(0x00); - let b: i64x1 = i64x1::new(0x00); - let e: i64x1 = i64x1::new(0x00); - let r: i64x1 = transmute(vorr_s64(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vorrq_s64() { - let a: i64x2 = i64x2::new(0x00, 0x01); - let b: i64x2 = i64x2::new(0x00, 0x00); - let e: i64x2 = i64x2::new(0x00, 0x01); - let r: i64x2 = transmute(vorrq_s64(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vorr_u64() { - let a: u64x1 = u64x1::new(0x00); - let b: u64x1 = u64x1::new(0x00); - let e: u64x1 = u64x1::new(0x00); - let r: u64x1 = transmute(vorr_u64(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vorrq_u64() { - let a: u64x2 = u64x2::new(0x00, 0x01); - let b: u64x2 = u64x2::new(0x00, 0x00); - let e: u64x2 = u64x2::new(0x00, 0x01); - let r: u64x2 = transmute(vorrq_u64(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_veor_s8() { - let a: i8x8 = i8x8::new(0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07); - let b: i8x8 = i8x8::new(0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00); - let e: i8x8 = i8x8::new(0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07); - let r: i8x8 = transmute(veor_s8(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_veorq_s8() { - let a: i8x16 = i8x16::new(0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F); - let b: i8x16 = i8x16::new(0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00); - let e: i8x16 = i8x16::new(0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F); - let r: i8x16 = transmute(veorq_s8(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_veor_s16() { - let a: i16x4 = i16x4::new(0x00, 0x01, 0x02, 0x03); - let b: i16x4 = i16x4::new(0x00, 0x00, 0x00, 0x00); - let e: i16x4 = i16x4::new(0x00, 0x01, 0x02, 0x03); - let r: i16x4 = transmute(veor_s16(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_veorq_s16() { - let a: i16x8 = i16x8::new(0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07); - let b: i16x8 = i16x8::new(0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00); - let e: i16x8 = i16x8::new(0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07); - let r: i16x8 = transmute(veorq_s16(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_veor_s32() { - let a: i32x2 = i32x2::new(0x00, 0x01); - let b: i32x2 = i32x2::new(0x00, 0x00); - let e: i32x2 = i32x2::new(0x00, 0x01); - let r: i32x2 = transmute(veor_s32(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_veorq_s32() { - let a: i32x4 = i32x4::new(0x00, 0x01, 0x02, 0x03); - let b: i32x4 = i32x4::new(0x00, 0x00, 0x00, 0x00); - let e: i32x4 = i32x4::new(0x00, 0x01, 0x02, 0x03); - let r: i32x4 = transmute(veorq_s32(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_veor_u8() { - let a: u8x8 = u8x8::new(0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07); - let b: u8x8 = u8x8::new(0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00); - let e: u8x8 = u8x8::new(0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07); - let r: u8x8 = transmute(veor_u8(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_veorq_u8() { - let a: u8x16 = u8x16::new(0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F); - let b: u8x16 = u8x16::new(0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00); - let e: u8x16 = u8x16::new(0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F); - let r: u8x16 = transmute(veorq_u8(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_veor_u16() { - let a: u16x4 = u16x4::new(0x00, 0x01, 0x02, 0x03); - let b: u16x4 = u16x4::new(0x00, 0x00, 0x00, 0x00); - let e: u16x4 = u16x4::new(0x00, 0x01, 0x02, 0x03); - let r: u16x4 = transmute(veor_u16(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_veorq_u16() { - let a: u16x8 = u16x8::new(0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07); - let b: u16x8 = u16x8::new(0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00); - let e: u16x8 = u16x8::new(0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07); - let r: u16x8 = transmute(veorq_u16(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_veor_u32() { - let a: u32x2 = u32x2::new(0x00, 0x01); - let b: u32x2 = u32x2::new(0x00, 0x00); - let e: u32x2 = u32x2::new(0x00, 0x01); - let r: u32x2 = transmute(veor_u32(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_veorq_u32() { - let a: u32x4 = u32x4::new(0x00, 0x01, 0x02, 0x03); - let b: u32x4 = u32x4::new(0x00, 0x00, 0x00, 0x00); - let e: u32x4 = u32x4::new(0x00, 0x01, 0x02, 0x03); - let r: u32x4 = transmute(veorq_u32(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_veor_s64() { - let a: i64x1 = i64x1::new(0x00); - let b: i64x1 = i64x1::new(0x00); - let e: i64x1 = i64x1::new(0x00); - let r: i64x1 = transmute(veor_s64(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_veorq_s64() { - let a: i64x2 = i64x2::new(0x00, 0x01); - let b: i64x2 = i64x2::new(0x00, 0x00); - let e: i64x2 = i64x2::new(0x00, 0x01); - let r: i64x2 = transmute(veorq_s64(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_veor_u64() { - let a: u64x1 = u64x1::new(0x00); - let b: u64x1 = u64x1::new(0x00); - let e: u64x1 = u64x1::new(0x00); - let r: u64x1 = transmute(veor_u64(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_veorq_u64() { - let a: u64x2 = u64x2::new(0x00, 0x01); - let b: u64x2 = u64x2::new(0x00, 0x00); - let e: u64x2 = u64x2::new(0x00, 0x01); - let r: u64x2 = transmute(veorq_u64(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vceq_u8() { - let a: u8x8 = u8x8::new(0xFF, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07); - let b: u8x8 = u8x8::new(0xFF, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07); - let e: u8x8 = u8x8::new(0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF); - let r: u8x8 = transmute(vceq_u8(transmute(a), transmute(b))); - assert_eq!(r, e); - - let a: u8x8 = u8x8::new(0xFF, 0xFF, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07); - let b: u8x8 = u8x8::new(0xFF, 0, 0x02, 0x04, 0x04, 0x00, 0x06, 0x08); - let e: u8x8 = u8x8::new(0xFF, 0, 0xFF, 0, 0xFF, 0, 0xFF, 0); - let r: u8x8 = transmute(vceq_u8(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vceqq_u8() { - let a: u8x16 = u8x16::new(0xFF, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0); - let b: u8x16 = u8x16::new(0xFF, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0); - let e: u8x16 = u8x16::new(0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF); - let r: u8x16 = transmute(vceqq_u8(transmute(a), transmute(b))); - assert_eq!(r, e); - - let a: u8x16 = u8x16::new(0xFF, 0xFF, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0xCC, 0x0D, 0xEE, 0); - let b: u8x16 = u8x16::new(0xFF, 0, 0x02, 0x04, 0x04, 0x00, 0x06, 0x08, 0x08, 0x00, 0x0A, 0x0A, 0xCC, 0xD0, 0xEE, 0xFF); - let e: u8x16 = u8x16::new(0xFF, 0, 0xFF, 0, 0xFF, 0, 0xFF, 0, 0xFF, 0, 0xFF, 0, 0xFF, 0, 0xFF, 0); - let r: u8x16 = transmute(vceqq_u8(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vceq_u16() { - let a: u16x4 = u16x4::new(0xFF_FF, 0x01, 0x02, 0x03); - let b: u16x4 = u16x4::new(0xFF_FF, 0x01, 0x02, 0x03); - let e: u16x4 = u16x4::new(0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF); - let r: u16x4 = transmute(vceq_u16(transmute(a), transmute(b))); - assert_eq!(r, e); - - let a: u16x4 = u16x4::new(0xFF_FF, 0xFF_FF, 0x02, 0x03); - let b: u16x4 = u16x4::new(0xFF_FF, 0, 0x02, 0x04); - let e: u16x4 = u16x4::new(0xFF_FF, 0, 0xFF_FF, 0); - let r: u16x4 = transmute(vceq_u16(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vceqq_u16() { - let a: u16x8 = u16x8::new(0xFF_FF, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07); - let b: u16x8 = u16x8::new(0xFF_FF, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07); - let e: u16x8 = u16x8::new(0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF); - let r: u16x8 = transmute(vceqq_u16(transmute(a), transmute(b))); - assert_eq!(r, e); - - let a: u16x8 = u16x8::new(0xFF_FF, 0xFF_FF, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07); - let b: u16x8 = u16x8::new(0xFF_FF, 0, 0x02, 0x04, 0x04, 0x00, 0x06, 0x08); - let e: u16x8 = u16x8::new(0xFF_FF, 0, 0xFF_FF, 0, 0xFF_FF, 0, 0xFF_FF, 0); - let r: u16x8 = transmute(vceqq_u16(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vceq_u32() { - let a: u32x2 = u32x2::new(0xFF_FF_FF_FF, 0x01); - let b: u32x2 = u32x2::new(0xFF_FF_FF_FF, 0x01); - let e: u32x2 = u32x2::new(0xFF_FF_FF_FF, 0xFF_FF_FF_FF); - let r: u32x2 = transmute(vceq_u32(transmute(a), transmute(b))); - assert_eq!(r, e); - - let a: u32x2 = u32x2::new(0xFF_FF_FF_FF, 0xFF_FF_FF_FF); - let b: u32x2 = u32x2::new(0xFF_FF_FF_FF, 0); - let e: u32x2 = u32x2::new(0xFF_FF_FF_FF, 0); - let r: u32x2 = transmute(vceq_u32(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vceqq_u32() { - let a: u32x4 = u32x4::new(0xFF_FF_FF_FF, 0x01, 0x02, 0x03); - let b: u32x4 = u32x4::new(0xFF_FF_FF_FF, 0x01, 0x02, 0x03); - let e: u32x4 = u32x4::new(0xFF_FF_FF_FF, 0xFF_FF_FF_FF, 0xFF_FF_FF_FF, 0xFF_FF_FF_FF); - let r: u32x4 = transmute(vceqq_u32(transmute(a), transmute(b))); - assert_eq!(r, e); - - let a: u32x4 = u32x4::new(0xFF_FF_FF_FF, 0xFF_FF_FF_FF, 0x02, 0x03); - let b: u32x4 = u32x4::new(0xFF_FF_FF_FF, 0, 0x02, 0x04); - let e: u32x4 = u32x4::new(0xFF_FF_FF_FF, 0, 0xFF_FF_FF_FF, 0); - let r: u32x4 = transmute(vceqq_u32(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vceq_s8() { - let a: i8x8 = i8x8::new(0x7F, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07); - let b: i8x8 = i8x8::new(0x7F, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07); - let e: u8x8 = u8x8::new(0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF); - let r: u8x8 = transmute(vceq_s8(transmute(a), transmute(b))); - assert_eq!(r, e); - - let a: i8x8 = i8x8::new(0x7F, 0x7F, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07); - let b: i8x8 = i8x8::new(0x7F, -128, 0x02, 0x04, 0x04, 0x00, 0x06, 0x08); - let e: u8x8 = u8x8::new(0xFF, 0, 0xFF, 0, 0xFF, 0, 0xFF, 0); - let r: u8x8 = transmute(vceq_s8(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vceqq_s8() { - let a: i8x16 = i8x16::new(0x7F, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, -128); - let b: i8x16 = i8x16::new(0x7F, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, -128); - let e: u8x16 = u8x16::new(0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF); - let r: u8x16 = transmute(vceqq_s8(transmute(a), transmute(b))); - assert_eq!(r, e); - - let a: i8x16 = i8x16::new(0x7F, 0x7F, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0xCC, 0x0D, 0xEE, -128); - let b: i8x16 = i8x16::new(0x7F, -128, 0x02, 0x04, 0x04, 0x00, 0x06, 0x08, 0x08, 0x00, 0x0A, 0x0A, 0xCC, 0xD0, 0xEE, 0x7F); - let e: u8x16 = u8x16::new(0xFF, 0, 0xFF, 0, 0xFF, 0, 0xFF, 0, 0xFF, 0, 0xFF, 0, 0xFF, 0, 0xFF, 0); - let r: u8x16 = transmute(vceqq_s8(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vceq_s16() { - let a: i16x4 = i16x4::new(0x7F_FF, 0x01, 0x02, 0x03); - let b: i16x4 = i16x4::new(0x7F_FF, 0x01, 0x02, 0x03); - let e: u16x4 = u16x4::new(0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF); - let r: u16x4 = transmute(vceq_s16(transmute(a), transmute(b))); - assert_eq!(r, e); - - let a: i16x4 = i16x4::new(0x7F_FF, 0x7F_FF, 0x02, 0x03); - let b: i16x4 = i16x4::new(0x7F_FF, -32768, 0x02, 0x04); - let e: u16x4 = u16x4::new(0xFF_FF, 0, 0xFF_FF, 0); - let r: u16x4 = transmute(vceq_s16(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vceqq_s16() { - let a: i16x8 = i16x8::new(0x7F_FF, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07); - let b: i16x8 = i16x8::new(0x7F_FF, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07); - let e: u16x8 = u16x8::new(0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF); - let r: u16x8 = transmute(vceqq_s16(transmute(a), transmute(b))); - assert_eq!(r, e); - - let a: i16x8 = i16x8::new(0x7F_FF, 0x7F_FF, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07); - let b: i16x8 = i16x8::new(0x7F_FF, -32768, 0x02, 0x04, 0x04, 0x00, 0x06, 0x08); - let e: u16x8 = u16x8::new(0xFF_FF, 0, 0xFF_FF, 0, 0xFF_FF, 0, 0xFF_FF, 0); - let r: u16x8 = transmute(vceqq_s16(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vceq_s32() { - let a: i32x2 = i32x2::new(0x7F_FF_FF_FF, 0x01); - let b: i32x2 = i32x2::new(0x7F_FF_FF_FF, 0x01); - let e: u32x2 = u32x2::new(0xFF_FF_FF_FF, 0xFF_FF_FF_FF); - let r: u32x2 = transmute(vceq_s32(transmute(a), transmute(b))); - assert_eq!(r, e); - - let a: i32x2 = i32x2::new(0x7F_FF_FF_FF, 0x7F_FF_FF_FF); - let b: i32x2 = i32x2::new(0x7F_FF_FF_FF, -2147483648); - let e: u32x2 = u32x2::new(0xFF_FF_FF_FF, 0); - let r: u32x2 = transmute(vceq_s32(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vceqq_s32() { - let a: i32x4 = i32x4::new(0x7F_FF_FF_FF, 0x01, 0x02, 0x03); - let b: i32x4 = i32x4::new(0x7F_FF_FF_FF, 0x01, 0x02, 0x03); - let e: u32x4 = u32x4::new(0xFF_FF_FF_FF, 0xFF_FF_FF_FF, 0xFF_FF_FF_FF, 0xFF_FF_FF_FF); - let r: u32x4 = transmute(vceqq_s32(transmute(a), transmute(b))); - assert_eq!(r, e); - - let a: i32x4 = i32x4::new(0x7F_FF_FF_FF, 0x7F_FF_FF_FF, 0x02, 0x03); - let b: i32x4 = i32x4::new(0x7F_FF_FF_FF, -2147483648, 0x02, 0x04); - let e: u32x4 = u32x4::new(0xFF_FF_FF_FF, 0, 0xFF_FF_FF_FF, 0); - let r: u32x4 = transmute(vceqq_s32(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vceq_f32() { - let a: f32x2 = f32x2::new(1.2, 3.4); - let b: f32x2 = f32x2::new(1.2, 3.4); - let e: u32x2 = u32x2::new(0xFF_FF_FF_FF, 0xFF_FF_FF_FF); - let r: u32x2 = transmute(vceq_f32(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vceqq_f32() { - let a: f32x4 = f32x4::new(1.2, 3.4, 5.6, 7.8); - let b: f32x4 = f32x4::new(1.2, 3.4, 5.6, 7.8); - let e: u32x4 = u32x4::new(0xFF_FF_FF_FF, 0xFF_FF_FF_FF, 0xFF_FF_FF_FF, 0xFF_FF_FF_FF); - let r: u32x4 = transmute(vceqq_f32(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vcgt_s8() { - let a: i8x8 = i8x8::new(1, 2, 3, 4, 5, 6, 7, 8); - let b: i8x8 = i8x8::new(0, 1, 2, 3, 4, 5, 6, 7); - let e: u8x8 = u8x8::new(0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF); - let r: u8x8 = transmute(vcgt_s8(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vcgtq_s8() { - let a: i8x16 = i8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); - let b: i8x16 = i8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); - let e: u8x16 = u8x16::new(0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF); - let r: u8x16 = transmute(vcgtq_s8(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vcgt_s16() { - let a: i16x4 = i16x4::new(1, 2, 3, 4); - let b: i16x4 = i16x4::new(0, 1, 2, 3); - let e: u16x4 = u16x4::new(0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF); - let r: u16x4 = transmute(vcgt_s16(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vcgtq_s16() { - let a: i16x8 = i16x8::new(1, 2, 3, 4, 5, 6, 7, 8); - let b: i16x8 = i16x8::new(0, 1, 2, 3, 4, 5, 6, 7); - let e: u16x8 = u16x8::new(0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF); - let r: u16x8 = transmute(vcgtq_s16(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vcgt_s32() { - let a: i32x2 = i32x2::new(1, 2); - let b: i32x2 = i32x2::new(0, 1); - let e: u32x2 = u32x2::new(0xFF_FF_FF_FF, 0xFF_FF_FF_FF); - let r: u32x2 = transmute(vcgt_s32(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vcgtq_s32() { - let a: i32x4 = i32x4::new(1, 2, 3, 4); - let b: i32x4 = i32x4::new(0, 1, 2, 3); - let e: u32x4 = u32x4::new(0xFF_FF_FF_FF, 0xFF_FF_FF_FF, 0xFF_FF_FF_FF, 0xFF_FF_FF_FF); - let r: u32x4 = transmute(vcgtq_s32(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vcgt_u8() { - let a: u8x8 = u8x8::new(1, 2, 3, 4, 5, 6, 7, 8); - let b: u8x8 = u8x8::new(0, 1, 2, 3, 4, 5, 6, 7); - let e: u8x8 = u8x8::new(0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF); - let r: u8x8 = transmute(vcgt_u8(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vcgtq_u8() { - let a: u8x16 = u8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); - let b: u8x16 = u8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); - let e: u8x16 = u8x16::new(0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF); - let r: u8x16 = transmute(vcgtq_u8(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vcgt_u16() { - let a: u16x4 = u16x4::new(1, 2, 3, 4); - let b: u16x4 = u16x4::new(0, 1, 2, 3); - let e: u16x4 = u16x4::new(0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF); - let r: u16x4 = transmute(vcgt_u16(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vcgtq_u16() { - let a: u16x8 = u16x8::new(1, 2, 3, 4, 5, 6, 7, 8); - let b: u16x8 = u16x8::new(0, 1, 2, 3, 4, 5, 6, 7); - let e: u16x8 = u16x8::new(0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF); - let r: u16x8 = transmute(vcgtq_u16(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vcgt_u32() { - let a: u32x2 = u32x2::new(1, 2); - let b: u32x2 = u32x2::new(0, 1); - let e: u32x2 = u32x2::new(0xFF_FF_FF_FF, 0xFF_FF_FF_FF); - let r: u32x2 = transmute(vcgt_u32(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vcgtq_u32() { - let a: u32x4 = u32x4::new(1, 2, 3, 4); - let b: u32x4 = u32x4::new(0, 1, 2, 3); - let e: u32x4 = u32x4::new(0xFF_FF_FF_FF, 0xFF_FF_FF_FF, 0xFF_FF_FF_FF, 0xFF_FF_FF_FF); - let r: u32x4 = transmute(vcgtq_u32(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vcgt_f32() { - let a: f32x2 = f32x2::new(1.2, 2.3); - let b: f32x2 = f32x2::new(0.1, 1.2); - let e: u32x2 = u32x2::new(0xFF_FF_FF_FF, 0xFF_FF_FF_FF); - let r: u32x2 = transmute(vcgt_f32(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vcgtq_f32() { - let a: f32x4 = f32x4::new(1.2, 2.3, 3.4, 4.5); - let b: f32x4 = f32x4::new(0.1, 1.2, 2.3, 3.4); - let e: u32x4 = u32x4::new(0xFF_FF_FF_FF, 0xFF_FF_FF_FF, 0xFF_FF_FF_FF, 0xFF_FF_FF_FF); - let r: u32x4 = transmute(vcgtq_f32(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vclt_s8() { - let a: i8x8 = i8x8::new(0, 1, 2, 3, 4, 5, 6, 7); - let b: i8x8 = i8x8::new(1, 2, 3, 4, 5, 6, 7, 8); - let e: u8x8 = u8x8::new(0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF); - let r: u8x8 = transmute(vclt_s8(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vcltq_s8() { - let a: i8x16 = i8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); - let b: i8x16 = i8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); - let e: u8x16 = u8x16::new(0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF); - let r: u8x16 = transmute(vcltq_s8(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vclt_s16() { - let a: i16x4 = i16x4::new(0, 1, 2, 3); - let b: i16x4 = i16x4::new(1, 2, 3, 4); - let e: u16x4 = u16x4::new(0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF); - let r: u16x4 = transmute(vclt_s16(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vcltq_s16() { - let a: i16x8 = i16x8::new(0, 1, 2, 3, 4, 5, 6, 7); - let b: i16x8 = i16x8::new(1, 2, 3, 4, 5, 6, 7, 8); - let e: u16x8 = u16x8::new(0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF); - let r: u16x8 = transmute(vcltq_s16(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vclt_s32() { - let a: i32x2 = i32x2::new(0, 1); - let b: i32x2 = i32x2::new(1, 2); - let e: u32x2 = u32x2::new(0xFF_FF_FF_FF, 0xFF_FF_FF_FF); - let r: u32x2 = transmute(vclt_s32(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vcltq_s32() { - let a: i32x4 = i32x4::new(0, 1, 2, 3); - let b: i32x4 = i32x4::new(1, 2, 3, 4); - let e: u32x4 = u32x4::new(0xFF_FF_FF_FF, 0xFF_FF_FF_FF, 0xFF_FF_FF_FF, 0xFF_FF_FF_FF); - let r: u32x4 = transmute(vcltq_s32(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vclt_u8() { - let a: u8x8 = u8x8::new(0, 1, 2, 3, 4, 5, 6, 7); - let b: u8x8 = u8x8::new(1, 2, 3, 4, 5, 6, 7, 8); - let e: u8x8 = u8x8::new(0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF); - let r: u8x8 = transmute(vclt_u8(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vcltq_u8() { - let a: u8x16 = u8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); - let b: u8x16 = u8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); - let e: u8x16 = u8x16::new(0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF); - let r: u8x16 = transmute(vcltq_u8(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vclt_u16() { - let a: u16x4 = u16x4::new(0, 1, 2, 3); - let b: u16x4 = u16x4::new(1, 2, 3, 4); - let e: u16x4 = u16x4::new(0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF); - let r: u16x4 = transmute(vclt_u16(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vcltq_u16() { - let a: u16x8 = u16x8::new(0, 1, 2, 3, 4, 5, 6, 7); - let b: u16x8 = u16x8::new(1, 2, 3, 4, 5, 6, 7, 8); - let e: u16x8 = u16x8::new(0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF); - let r: u16x8 = transmute(vcltq_u16(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vclt_u32() { - let a: u32x2 = u32x2::new(0, 1); - let b: u32x2 = u32x2::new(1, 2); - let e: u32x2 = u32x2::new(0xFF_FF_FF_FF, 0xFF_FF_FF_FF); - let r: u32x2 = transmute(vclt_u32(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vcltq_u32() { - let a: u32x4 = u32x4::new(0, 1, 2, 3); - let b: u32x4 = u32x4::new(1, 2, 3, 4); - let e: u32x4 = u32x4::new(0xFF_FF_FF_FF, 0xFF_FF_FF_FF, 0xFF_FF_FF_FF, 0xFF_FF_FF_FF); - let r: u32x4 = transmute(vcltq_u32(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vclt_f32() { - let a: f32x2 = f32x2::new(0.1, 1.2); - let b: f32x2 = f32x2::new(1.2, 2.3); - let e: u32x2 = u32x2::new(0xFF_FF_FF_FF, 0xFF_FF_FF_FF); - let r: u32x2 = transmute(vclt_f32(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vcltq_f32() { - let a: f32x4 = f32x4::new(0.1, 1.2, 2.3, 3.4); - let b: f32x4 = f32x4::new(1.2, 2.3, 3.4, 4.5); - let e: u32x4 = u32x4::new(0xFF_FF_FF_FF, 0xFF_FF_FF_FF, 0xFF_FF_FF_FF, 0xFF_FF_FF_FF); - let r: u32x4 = transmute(vcltq_f32(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vcle_s8() { - let a: i8x8 = i8x8::new(0, 1, 2, 3, 4, 5, 6, 7); - let b: i8x8 = i8x8::new(1, 2, 3, 4, 5, 6, 7, 8); - let e: u8x8 = u8x8::new(0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF); - let r: u8x8 = transmute(vcle_s8(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vcleq_s8() { - let a: i8x16 = i8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); - let b: i8x16 = i8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); - let e: u8x16 = u8x16::new(0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF); - let r: u8x16 = transmute(vcleq_s8(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vcle_s16() { - let a: i16x4 = i16x4::new(0, 1, 2, 3); - let b: i16x4 = i16x4::new(1, 2, 3, 4); - let e: u16x4 = u16x4::new(0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF); - let r: u16x4 = transmute(vcle_s16(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vcleq_s16() { - let a: i16x8 = i16x8::new(0, 1, 2, 3, 4, 5, 6, 7); - let b: i16x8 = i16x8::new(1, 2, 3, 4, 5, 6, 7, 8); - let e: u16x8 = u16x8::new(0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF); - let r: u16x8 = transmute(vcleq_s16(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vcle_s32() { - let a: i32x2 = i32x2::new(0, 1); - let b: i32x2 = i32x2::new(1, 2); - let e: u32x2 = u32x2::new(0xFF_FF_FF_FF, 0xFF_FF_FF_FF); - let r: u32x2 = transmute(vcle_s32(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vcleq_s32() { - let a: i32x4 = i32x4::new(0, 1, 2, 3); - let b: i32x4 = i32x4::new(1, 2, 3, 4); - let e: u32x4 = u32x4::new(0xFF_FF_FF_FF, 0xFF_FF_FF_FF, 0xFF_FF_FF_FF, 0xFF_FF_FF_FF); - let r: u32x4 = transmute(vcleq_s32(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vcle_u8() { - let a: u8x8 = u8x8::new(0, 1, 2, 3, 4, 5, 6, 7); - let b: u8x8 = u8x8::new(1, 2, 3, 4, 5, 6, 7, 8); - let e: u8x8 = u8x8::new(0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF); - let r: u8x8 = transmute(vcle_u8(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vcleq_u8() { - let a: u8x16 = u8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); - let b: u8x16 = u8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); - let e: u8x16 = u8x16::new(0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF); - let r: u8x16 = transmute(vcleq_u8(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vcle_u16() { - let a: u16x4 = u16x4::new(0, 1, 2, 3); - let b: u16x4 = u16x4::new(1, 2, 3, 4); - let e: u16x4 = u16x4::new(0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF); - let r: u16x4 = transmute(vcle_u16(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vcleq_u16() { - let a: u16x8 = u16x8::new(0, 1, 2, 3, 4, 5, 6, 7); - let b: u16x8 = u16x8::new(1, 2, 3, 4, 5, 6, 7, 8); - let e: u16x8 = u16x8::new(0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF); - let r: u16x8 = transmute(vcleq_u16(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vcle_u32() { - let a: u32x2 = u32x2::new(0, 1); - let b: u32x2 = u32x2::new(1, 2); - let e: u32x2 = u32x2::new(0xFF_FF_FF_FF, 0xFF_FF_FF_FF); - let r: u32x2 = transmute(vcle_u32(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vcleq_u32() { - let a: u32x4 = u32x4::new(0, 1, 2, 3); - let b: u32x4 = u32x4::new(1, 2, 3, 4); - let e: u32x4 = u32x4::new(0xFF_FF_FF_FF, 0xFF_FF_FF_FF, 0xFF_FF_FF_FF, 0xFF_FF_FF_FF); - let r: u32x4 = transmute(vcleq_u32(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vcle_f32() { - let a: f32x2 = f32x2::new(0.1, 1.2); - let b: f32x2 = f32x2::new(1.2, 2.3); - let e: u32x2 = u32x2::new(0xFF_FF_FF_FF, 0xFF_FF_FF_FF); - let r: u32x2 = transmute(vcle_f32(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vcleq_f32() { - let a: f32x4 = f32x4::new(0.1, 1.2, 2.3, 3.4); - let b: f32x4 = f32x4::new(1.2, 2.3, 3.4, 4.5); - let e: u32x4 = u32x4::new(0xFF_FF_FF_FF, 0xFF_FF_FF_FF, 0xFF_FF_FF_FF, 0xFF_FF_FF_FF); - let r: u32x4 = transmute(vcleq_f32(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vcge_s8() { - let a: i8x8 = i8x8::new(1, 2, 3, 4, 5, 6, 7, 8); - let b: i8x8 = i8x8::new(0, 1, 2, 3, 4, 5, 6, 7); - let e: u8x8 = u8x8::new(0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF); - let r: u8x8 = transmute(vcge_s8(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vcgeq_s8() { - let a: i8x16 = i8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); - let b: i8x16 = i8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); - let e: u8x16 = u8x16::new(0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF); - let r: u8x16 = transmute(vcgeq_s8(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vcge_s16() { - let a: i16x4 = i16x4::new(1, 2, 3, 4); - let b: i16x4 = i16x4::new(0, 1, 2, 3); - let e: u16x4 = u16x4::new(0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF); - let r: u16x4 = transmute(vcge_s16(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vcgeq_s16() { - let a: i16x8 = i16x8::new(1, 2, 3, 4, 5, 6, 7, 8); - let b: i16x8 = i16x8::new(0, 1, 2, 3, 4, 5, 6, 7); - let e: u16x8 = u16x8::new(0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF); - let r: u16x8 = transmute(vcgeq_s16(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vcge_s32() { - let a: i32x2 = i32x2::new(1, 2); - let b: i32x2 = i32x2::new(0, 1); - let e: u32x2 = u32x2::new(0xFF_FF_FF_FF, 0xFF_FF_FF_FF); - let r: u32x2 = transmute(vcge_s32(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vcgeq_s32() { - let a: i32x4 = i32x4::new(1, 2, 3, 4); - let b: i32x4 = i32x4::new(0, 1, 2, 3); - let e: u32x4 = u32x4::new(0xFF_FF_FF_FF, 0xFF_FF_FF_FF, 0xFF_FF_FF_FF, 0xFF_FF_FF_FF); - let r: u32x4 = transmute(vcgeq_s32(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vcge_u8() { - let a: u8x8 = u8x8::new(1, 2, 3, 4, 5, 6, 7, 8); - let b: u8x8 = u8x8::new(0, 1, 2, 3, 4, 5, 6, 7); - let e: u8x8 = u8x8::new(0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF); - let r: u8x8 = transmute(vcge_u8(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vcgeq_u8() { - let a: u8x16 = u8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); - let b: u8x16 = u8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); - let e: u8x16 = u8x16::new(0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF); - let r: u8x16 = transmute(vcgeq_u8(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vcge_u16() { - let a: u16x4 = u16x4::new(1, 2, 3, 4); - let b: u16x4 = u16x4::new(0, 1, 2, 3); - let e: u16x4 = u16x4::new(0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF); - let r: u16x4 = transmute(vcge_u16(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vcgeq_u16() { - let a: u16x8 = u16x8::new(1, 2, 3, 4, 5, 6, 7, 8); - let b: u16x8 = u16x8::new(0, 1, 2, 3, 4, 5, 6, 7); - let e: u16x8 = u16x8::new(0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF); - let r: u16x8 = transmute(vcgeq_u16(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vcge_u32() { - let a: u32x2 = u32x2::new(1, 2); - let b: u32x2 = u32x2::new(0, 1); - let e: u32x2 = u32x2::new(0xFF_FF_FF_FF, 0xFF_FF_FF_FF); - let r: u32x2 = transmute(vcge_u32(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vcgeq_u32() { - let a: u32x4 = u32x4::new(1, 2, 3, 4); - let b: u32x4 = u32x4::new(0, 1, 2, 3); - let e: u32x4 = u32x4::new(0xFF_FF_FF_FF, 0xFF_FF_FF_FF, 0xFF_FF_FF_FF, 0xFF_FF_FF_FF); - let r: u32x4 = transmute(vcgeq_u32(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vcge_f32() { - let a: f32x2 = f32x2::new(1.2, 2.3); - let b: f32x2 = f32x2::new(0.1, 1.2); - let e: u32x2 = u32x2::new(0xFF_FF_FF_FF, 0xFF_FF_FF_FF); - let r: u32x2 = transmute(vcge_f32(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vcgeq_f32() { - let a: f32x4 = f32x4::new(1.2, 2.3, 3.4, 4.5); - let b: f32x4 = f32x4::new(0.1, 1.2, 2.3, 3.4); - let e: u32x4 = u32x4::new(0xFF_FF_FF_FF, 0xFF_FF_FF_FF, 0xFF_FF_FF_FF, 0xFF_FF_FF_FF); - let r: u32x4 = transmute(vcgeq_f32(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vqsub_u8() { - let a: u8x8 = u8x8::new(42, 42, 42, 42, 42, 42, 42, 42); - let b: u8x8 = u8x8::new(1, 2, 3, 4, 5, 6, 7, 8); - let e: u8x8 = u8x8::new(41, 40, 39, 38, 37, 36, 35, 34); - let r: u8x8 = transmute(vqsub_u8(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vqsubq_u8() { - let a: u8x16 = u8x16::new(42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42); - let b: u8x16 = u8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); - let e: u8x16 = u8x16::new(41, 40, 39, 38, 37, 36, 35, 34, 33, 32, 31, 30, 29, 28, 27, 26); - let r: u8x16 = transmute(vqsubq_u8(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vqsub_u16() { - let a: u16x4 = u16x4::new(42, 42, 42, 42); - let b: u16x4 = u16x4::new(1, 2, 3, 4); - let e: u16x4 = u16x4::new(41, 40, 39, 38); - let r: u16x4 = transmute(vqsub_u16(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vqsubq_u16() { - let a: u16x8 = u16x8::new(42, 42, 42, 42, 42, 42, 42, 42); - let b: u16x8 = u16x8::new(1, 2, 3, 4, 5, 6, 7, 8); - let e: u16x8 = u16x8::new(41, 40, 39, 38, 37, 36, 35, 34); - let r: u16x8 = transmute(vqsubq_u16(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vqsub_u32() { - let a: u32x2 = u32x2::new(42, 42); - let b: u32x2 = u32x2::new(1, 2); - let e: u32x2 = u32x2::new(41, 40); - let r: u32x2 = transmute(vqsub_u32(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vqsubq_u32() { - let a: u32x4 = u32x4::new(42, 42, 42, 42); - let b: u32x4 = u32x4::new(1, 2, 3, 4); - let e: u32x4 = u32x4::new(41, 40, 39, 38); - let r: u32x4 = transmute(vqsubq_u32(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vqsub_s8() { - let a: i8x8 = i8x8::new(42, 42, 42, 42, 42, 42, 42, 42); - let b: i8x8 = i8x8::new(1, 2, 3, 4, 5, 6, 7, 8); - let e: i8x8 = i8x8::new(41, 40, 39, 38, 37, 36, 35, 34); - let r: i8x8 = transmute(vqsub_s8(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vqsubq_s8() { - let a: i8x16 = i8x16::new(42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42); - let b: i8x16 = i8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); - let e: i8x16 = i8x16::new(41, 40, 39, 38, 37, 36, 35, 34, 33, 32, 31, 30, 29, 28, 27, 26); - let r: i8x16 = transmute(vqsubq_s8(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vqsub_s16() { - let a: i16x4 = i16x4::new(42, 42, 42, 42); - let b: i16x4 = i16x4::new(1, 2, 3, 4); - let e: i16x4 = i16x4::new(41, 40, 39, 38); - let r: i16x4 = transmute(vqsub_s16(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vqsubq_s16() { - let a: i16x8 = i16x8::new(42, 42, 42, 42, 42, 42, 42, 42); - let b: i16x8 = i16x8::new(1, 2, 3, 4, 5, 6, 7, 8); - let e: i16x8 = i16x8::new(41, 40, 39, 38, 37, 36, 35, 34); - let r: i16x8 = transmute(vqsubq_s16(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vqsub_s32() { - let a: i32x2 = i32x2::new(42, 42); - let b: i32x2 = i32x2::new(1, 2); - let e: i32x2 = i32x2::new(41, 40); - let r: i32x2 = transmute(vqsub_s32(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vqsubq_s32() { - let a: i32x4 = i32x4::new(42, 42, 42, 42); - let b: i32x4 = i32x4::new(1, 2, 3, 4); - let e: i32x4 = i32x4::new(41, 40, 39, 38); - let r: i32x4 = transmute(vqsubq_s32(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vhadd_u8() { - let a: u8x8 = u8x8::new(42, 42, 42, 42, 42, 42, 42, 42); - let b: u8x8 = u8x8::new(1, 2, 3, 4, 5, 6, 7, 8); - let e: u8x8 = u8x8::new(21, 22, 22, 23, 23, 24, 24, 25); - let r: u8x8 = transmute(vhadd_u8(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vhaddq_u8() { - let a: u8x16 = u8x16::new(42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42); - let b: u8x16 = u8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); - let e: u8x16 = u8x16::new(21, 22, 22, 23, 23, 24, 24, 25, 25, 26, 26, 27, 27, 28, 28, 29); - let r: u8x16 = transmute(vhaddq_u8(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vhadd_u16() { - let a: u16x4 = u16x4::new(42, 42, 42, 42); - let b: u16x4 = u16x4::new(1, 2, 3, 4); - let e: u16x4 = u16x4::new(21, 22, 22, 23); - let r: u16x4 = transmute(vhadd_u16(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vhaddq_u16() { - let a: u16x8 = u16x8::new(42, 42, 42, 42, 42, 42, 42, 42); - let b: u16x8 = u16x8::new(1, 2, 3, 4, 5, 6, 7, 8); - let e: u16x8 = u16x8::new(21, 22, 22, 23, 23, 24, 24, 25); - let r: u16x8 = transmute(vhaddq_u16(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vhadd_u32() { - let a: u32x2 = u32x2::new(42, 42); - let b: u32x2 = u32x2::new(1, 2); - let e: u32x2 = u32x2::new(21, 22); - let r: u32x2 = transmute(vhadd_u32(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vhaddq_u32() { - let a: u32x4 = u32x4::new(42, 42, 42, 42); - let b: u32x4 = u32x4::new(1, 2, 3, 4); - let e: u32x4 = u32x4::new(21, 22, 22, 23); - let r: u32x4 = transmute(vhaddq_u32(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vhadd_s8() { - let a: i8x8 = i8x8::new(42, 42, 42, 42, 42, 42, 42, 42); - let b: i8x8 = i8x8::new(1, 2, 3, 4, 5, 6, 7, 8); - let e: i8x8 = i8x8::new(21, 22, 22, 23, 23, 24, 24, 25); - let r: i8x8 = transmute(vhadd_s8(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vhaddq_s8() { - let a: i8x16 = i8x16::new(42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42); - let b: i8x16 = i8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); - let e: i8x16 = i8x16::new(21, 22, 22, 23, 23, 24, 24, 25, 25, 26, 26, 27, 27, 28, 28, 29); - let r: i8x16 = transmute(vhaddq_s8(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vhadd_s16() { - let a: i16x4 = i16x4::new(42, 42, 42, 42); - let b: i16x4 = i16x4::new(1, 2, 3, 4); - let e: i16x4 = i16x4::new(21, 22, 22, 23); - let r: i16x4 = transmute(vhadd_s16(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vhaddq_s16() { - let a: i16x8 = i16x8::new(42, 42, 42, 42, 42, 42, 42, 42); - let b: i16x8 = i16x8::new(1, 2, 3, 4, 5, 6, 7, 8); - let e: i16x8 = i16x8::new(21, 22, 22, 23, 23, 24, 24, 25); - let r: i16x8 = transmute(vhaddq_s16(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vhadd_s32() { - let a: i32x2 = i32x2::new(42, 42); - let b: i32x2 = i32x2::new(1, 2); - let e: i32x2 = i32x2::new(21, 22); - let r: i32x2 = transmute(vhadd_s32(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vhaddq_s32() { - let a: i32x4 = i32x4::new(42, 42, 42, 42); - let b: i32x4 = i32x4::new(1, 2, 3, 4); - let e: i32x4 = i32x4::new(21, 22, 22, 23); - let r: i32x4 = transmute(vhaddq_s32(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vrhadd_u8() { - let a: u8x8 = u8x8::new(42, 42, 42, 42, 42, 42, 42, 42); - let b: u8x8 = u8x8::new(1, 2, 3, 4, 5, 6, 7, 8); - let e: u8x8 = u8x8::new(22, 22, 23, 23, 24, 24, 25, 25); - let r: u8x8 = transmute(vrhadd_u8(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vrhaddq_u8() { - let a: u8x16 = u8x16::new(42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42); - let b: u8x16 = u8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); - let e: u8x16 = u8x16::new(22, 22, 23, 23, 24, 24, 25, 25, 26, 26, 27, 27, 28, 28, 29, 29); - let r: u8x16 = transmute(vrhaddq_u8(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vrhadd_u16() { - let a: u16x4 = u16x4::new(42, 42, 42, 42); - let b: u16x4 = u16x4::new(1, 2, 3, 4); - let e: u16x4 = u16x4::new(22, 22, 23, 23); - let r: u16x4 = transmute(vrhadd_u16(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vrhaddq_u16() { - let a: u16x8 = u16x8::new(42, 42, 42, 42, 42, 42, 42, 42); - let b: u16x8 = u16x8::new(1, 2, 3, 4, 5, 6, 7, 8); - let e: u16x8 = u16x8::new(22, 22, 23, 23, 24, 24, 25, 25); - let r: u16x8 = transmute(vrhaddq_u16(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vrhadd_u32() { - let a: u32x2 = u32x2::new(42, 42); - let b: u32x2 = u32x2::new(1, 2); - let e: u32x2 = u32x2::new(22, 22); - let r: u32x2 = transmute(vrhadd_u32(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vrhaddq_u32() { - let a: u32x4 = u32x4::new(42, 42, 42, 42); - let b: u32x4 = u32x4::new(1, 2, 3, 4); - let e: u32x4 = u32x4::new(22, 22, 23, 23); - let r: u32x4 = transmute(vrhaddq_u32(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vrhadd_s8() { - let a: i8x8 = i8x8::new(42, 42, 42, 42, 42, 42, 42, 42); - let b: i8x8 = i8x8::new(1, 2, 3, 4, 5, 6, 7, 8); - let e: i8x8 = i8x8::new(22, 22, 23, 23, 24, 24, 25, 25); - let r: i8x8 = transmute(vrhadd_s8(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vrhaddq_s8() { - let a: i8x16 = i8x16::new(42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42); - let b: i8x16 = i8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); - let e: i8x16 = i8x16::new(22, 22, 23, 23, 24, 24, 25, 25, 26, 26, 27, 27, 28, 28, 29, 29); - let r: i8x16 = transmute(vrhaddq_s8(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vrhadd_s16() { - let a: i16x4 = i16x4::new(42, 42, 42, 42); - let b: i16x4 = i16x4::new(1, 2, 3, 4); - let e: i16x4 = i16x4::new(22, 22, 23, 23); - let r: i16x4 = transmute(vrhadd_s16(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vrhaddq_s16() { - let a: i16x8 = i16x8::new(42, 42, 42, 42, 42, 42, 42, 42); - let b: i16x8 = i16x8::new(1, 2, 3, 4, 5, 6, 7, 8); - let e: i16x8 = i16x8::new(22, 22, 23, 23, 24, 24, 25, 25); - let r: i16x8 = transmute(vrhaddq_s16(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vrhadd_s32() { - let a: i32x2 = i32x2::new(42, 42); - let b: i32x2 = i32x2::new(1, 2); - let e: i32x2 = i32x2::new(22, 22); - let r: i32x2 = transmute(vrhadd_s32(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vrhaddq_s32() { - let a: i32x4 = i32x4::new(42, 42, 42, 42); - let b: i32x4 = i32x4::new(1, 2, 3, 4); - let e: i32x4 = i32x4::new(22, 22, 23, 23); - let r: i32x4 = transmute(vrhaddq_s32(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vqadd_u8() { - let a: u8x8 = u8x8::new(42, 42, 42, 42, 42, 42, 42, 42); - let b: u8x8 = u8x8::new(1, 2, 3, 4, 5, 6, 7, 8); - let e: u8x8 = u8x8::new(43, 44, 45, 46, 47, 48, 49, 50); - let r: u8x8 = transmute(vqadd_u8(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vqaddq_u8() { - let a: u8x16 = u8x16::new(42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42); - let b: u8x16 = u8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); - let e: u8x16 = u8x16::new(43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58); - let r: u8x16 = transmute(vqaddq_u8(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vqadd_u16() { - let a: u16x4 = u16x4::new(42, 42, 42, 42); - let b: u16x4 = u16x4::new(1, 2, 3, 4); - let e: u16x4 = u16x4::new(43, 44, 45, 46); - let r: u16x4 = transmute(vqadd_u16(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vqaddq_u16() { - let a: u16x8 = u16x8::new(42, 42, 42, 42, 42, 42, 42, 42); - let b: u16x8 = u16x8::new(1, 2, 3, 4, 5, 6, 7, 8); - let e: u16x8 = u16x8::new(43, 44, 45, 46, 47, 48, 49, 50); - let r: u16x8 = transmute(vqaddq_u16(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vqadd_u32() { - let a: u32x2 = u32x2::new(42, 42); - let b: u32x2 = u32x2::new(1, 2); - let e: u32x2 = u32x2::new(43, 44); - let r: u32x2 = transmute(vqadd_u32(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vqaddq_u32() { - let a: u32x4 = u32x4::new(42, 42, 42, 42); - let b: u32x4 = u32x4::new(1, 2, 3, 4); - let e: u32x4 = u32x4::new(43, 44, 45, 46); - let r: u32x4 = transmute(vqaddq_u32(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vqadd_s8() { - let a: i8x8 = i8x8::new(42, 42, 42, 42, 42, 42, 42, 42); - let b: i8x8 = i8x8::new(1, 2, 3, 4, 5, 6, 7, 8); - let e: i8x8 = i8x8::new(43, 44, 45, 46, 47, 48, 49, 50); - let r: i8x8 = transmute(vqadd_s8(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vqaddq_s8() { - let a: i8x16 = i8x16::new(42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42); - let b: i8x16 = i8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); - let e: i8x16 = i8x16::new(43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58); - let r: i8x16 = transmute(vqaddq_s8(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vqadd_s16() { - let a: i16x4 = i16x4::new(42, 42, 42, 42); - let b: i16x4 = i16x4::new(1, 2, 3, 4); - let e: i16x4 = i16x4::new(43, 44, 45, 46); - let r: i16x4 = transmute(vqadd_s16(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vqaddq_s16() { - let a: i16x8 = i16x8::new(42, 42, 42, 42, 42, 42, 42, 42); - let b: i16x8 = i16x8::new(1, 2, 3, 4, 5, 6, 7, 8); - let e: i16x8 = i16x8::new(43, 44, 45, 46, 47, 48, 49, 50); - let r: i16x8 = transmute(vqaddq_s16(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vqadd_s32() { - let a: i32x2 = i32x2::new(42, 42); - let b: i32x2 = i32x2::new(1, 2); - let e: i32x2 = i32x2::new(43, 44); - let r: i32x2 = transmute(vqadd_s32(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vqaddq_s32() { - let a: i32x4 = i32x4::new(42, 42, 42, 42); - let b: i32x4 = i32x4::new(1, 2, 3, 4); - let e: i32x4 = i32x4::new(43, 44, 45, 46); - let r: i32x4 = transmute(vqaddq_s32(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vmul_s8() { - let a: i8x8 = i8x8::new(1, 2, 1, 2, 1, 2, 1, 2); - let b: i8x8 = i8x8::new(1, 2, 3, 4, 5, 6, 7, 8); - let e: i8x8 = i8x8::new(1, 4, 3, 8, 5, 12, 7, 16); - let r: i8x8 = transmute(vmul_s8(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vmulq_s8() { - let a: i8x16 = i8x16::new(1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2); - let b: i8x16 = i8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); - let e: i8x16 = i8x16::new(1, 4, 3, 8, 5, 12, 7, 16, 9, 20, 11, 24, 13, 28, 15, 32); - let r: i8x16 = transmute(vmulq_s8(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vmul_s16() { - let a: i16x4 = i16x4::new(1, 2, 1, 2); - let b: i16x4 = i16x4::new(1, 2, 3, 4); - let e: i16x4 = i16x4::new(1, 4, 3, 8); - let r: i16x4 = transmute(vmul_s16(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vmulq_s16() { - let a: i16x8 = i16x8::new(1, 2, 1, 2, 1, 2, 1, 2); - let b: i16x8 = i16x8::new(1, 2, 3, 4, 5, 6, 7, 8); - let e: i16x8 = i16x8::new(1, 4, 3, 8, 5, 12, 7, 16); - let r: i16x8 = transmute(vmulq_s16(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vmul_s32() { - let a: i32x2 = i32x2::new(1, 2); - let b: i32x2 = i32x2::new(1, 2); - let e: i32x2 = i32x2::new(1, 4); - let r: i32x2 = transmute(vmul_s32(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vmulq_s32() { - let a: i32x4 = i32x4::new(1, 2, 1, 2); - let b: i32x4 = i32x4::new(1, 2, 3, 4); - let e: i32x4 = i32x4::new(1, 4, 3, 8); - let r: i32x4 = transmute(vmulq_s32(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vmul_u8() { - let a: u8x8 = u8x8::new(1, 2, 1, 2, 1, 2, 1, 2); - let b: u8x8 = u8x8::new(1, 2, 3, 4, 5, 6, 7, 8); - let e: u8x8 = u8x8::new(1, 4, 3, 8, 5, 12, 7, 16); - let r: u8x8 = transmute(vmul_u8(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vmulq_u8() { - let a: u8x16 = u8x16::new(1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2); - let b: u8x16 = u8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); - let e: u8x16 = u8x16::new(1, 4, 3, 8, 5, 12, 7, 16, 9, 20, 11, 24, 13, 28, 15, 32); - let r: u8x16 = transmute(vmulq_u8(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vmul_u16() { - let a: u16x4 = u16x4::new(1, 2, 1, 2); - let b: u16x4 = u16x4::new(1, 2, 3, 4); - let e: u16x4 = u16x4::new(1, 4, 3, 8); - let r: u16x4 = transmute(vmul_u16(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vmulq_u16() { - let a: u16x8 = u16x8::new(1, 2, 1, 2, 1, 2, 1, 2); - let b: u16x8 = u16x8::new(1, 2, 3, 4, 5, 6, 7, 8); - let e: u16x8 = u16x8::new(1, 4, 3, 8, 5, 12, 7, 16); - let r: u16x8 = transmute(vmulq_u16(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vmul_u32() { - let a: u32x2 = u32x2::new(1, 2); - let b: u32x2 = u32x2::new(1, 2); - let e: u32x2 = u32x2::new(1, 4); - let r: u32x2 = transmute(vmul_u32(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vmulq_u32() { - let a: u32x4 = u32x4::new(1, 2, 1, 2); - let b: u32x4 = u32x4::new(1, 2, 3, 4); - let e: u32x4 = u32x4::new(1, 4, 3, 8); - let r: u32x4 = transmute(vmulq_u32(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vmul_f32() { - let a: f32x2 = f32x2::new(1.0, 2.0); - let b: f32x2 = f32x2::new(2.0, 3.0); - let e: f32x2 = f32x2::new(2.0, 6.0); - let r: f32x2 = transmute(vmul_f32(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vmulq_f32() { - let a: f32x4 = f32x4::new(1.0, 2.0, 1.0, 2.0); - let b: f32x4 = f32x4::new(2.0, 3.0, 4.0, 5.0); - let e: f32x4 = f32x4::new(2.0, 6.0, 4.0, 10.0); - let r: f32x4 = transmute(vmulq_f32(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vsub_s8() { - let a: i8x8 = i8x8::new(1, 2, 3, 4, 5, 6, 7, 8); - let b: i8x8 = i8x8::new(1, 2, 1, 2, 1, 2, 1, 2); - let e: i8x8 = i8x8::new(0, 0, 2, 2, 4, 4, 6, 6); - let r: i8x8 = transmute(vsub_s8(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vsubq_s8() { - let a: i8x16 = i8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); - let b: i8x16 = i8x16::new(1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2); - let e: i8x16 = i8x16::new(0, 0, 2, 2, 4, 4, 6, 6, 8, 8, 10, 10, 12, 12, 14, 14); - let r: i8x16 = transmute(vsubq_s8(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vsub_s16() { - let a: i16x4 = i16x4::new(1, 2, 3, 4); - let b: i16x4 = i16x4::new(1, 2, 1, 2); - let e: i16x4 = i16x4::new(0, 0, 2, 2); - let r: i16x4 = transmute(vsub_s16(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vsubq_s16() { - let a: i16x8 = i16x8::new(1, 2, 3, 4, 5, 6, 7, 8); - let b: i16x8 = i16x8::new(1, 2, 1, 2, 1, 2, 1, 2); - let e: i16x8 = i16x8::new(0, 0, 2, 2, 4, 4, 6, 6); - let r: i16x8 = transmute(vsubq_s16(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vsub_s32() { - let a: i32x2 = i32x2::new(1, 2); - let b: i32x2 = i32x2::new(1, 2); - let e: i32x2 = i32x2::new(0, 0); - let r: i32x2 = transmute(vsub_s32(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vsubq_s32() { - let a: i32x4 = i32x4::new(1, 2, 3, 4); - let b: i32x4 = i32x4::new(1, 2, 1, 2); - let e: i32x4 = i32x4::new(0, 0, 2, 2); - let r: i32x4 = transmute(vsubq_s32(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vsub_u8() { - let a: u8x8 = u8x8::new(1, 2, 3, 4, 5, 6, 7, 8); - let b: u8x8 = u8x8::new(1, 2, 1, 2, 1, 2, 1, 2); - let e: u8x8 = u8x8::new(0, 0, 2, 2, 4, 4, 6, 6); - let r: u8x8 = transmute(vsub_u8(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vsubq_u8() { - let a: u8x16 = u8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); - let b: u8x16 = u8x16::new(1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2); - let e: u8x16 = u8x16::new(0, 0, 2, 2, 4, 4, 6, 6, 8, 8, 10, 10, 12, 12, 14, 14); - let r: u8x16 = transmute(vsubq_u8(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vsub_u16() { - let a: u16x4 = u16x4::new(1, 2, 3, 4); - let b: u16x4 = u16x4::new(1, 2, 1, 2); - let e: u16x4 = u16x4::new(0, 0, 2, 2); - let r: u16x4 = transmute(vsub_u16(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vsubq_u16() { - let a: u16x8 = u16x8::new(1, 2, 3, 4, 5, 6, 7, 8); - let b: u16x8 = u16x8::new(1, 2, 1, 2, 1, 2, 1, 2); - let e: u16x8 = u16x8::new(0, 0, 2, 2, 4, 4, 6, 6); - let r: u16x8 = transmute(vsubq_u16(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vsub_u32() { - let a: u32x2 = u32x2::new(1, 2); - let b: u32x2 = u32x2::new(1, 2); - let e: u32x2 = u32x2::new(0, 0); - let r: u32x2 = transmute(vsub_u32(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vsubq_u32() { - let a: u32x4 = u32x4::new(1, 2, 3, 4); - let b: u32x4 = u32x4::new(1, 2, 1, 2); - let e: u32x4 = u32x4::new(0, 0, 2, 2); - let r: u32x4 = transmute(vsubq_u32(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vsub_s64() { - let a: i64x1 = i64x1::new(1); - let b: i64x1 = i64x1::new(1); - let e: i64x1 = i64x1::new(0); - let r: i64x1 = transmute(vsub_s64(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vsubq_s64() { - let a: i64x2 = i64x2::new(1, 2); - let b: i64x2 = i64x2::new(1, 2); - let e: i64x2 = i64x2::new(0, 0); - let r: i64x2 = transmute(vsubq_s64(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vsub_u64() { - let a: u64x1 = u64x1::new(1); - let b: u64x1 = u64x1::new(1); - let e: u64x1 = u64x1::new(0); - let r: u64x1 = transmute(vsub_u64(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vsubq_u64() { - let a: u64x2 = u64x2::new(1, 2); - let b: u64x2 = u64x2::new(1, 2); - let e: u64x2 = u64x2::new(0, 0); - let r: u64x2 = transmute(vsubq_u64(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vsub_f32() { - let a: f32x2 = f32x2::new(1.0, 4.0); - let b: f32x2 = f32x2::new(1.0, 2.0); - let e: f32x2 = f32x2::new(0.0, 2.0); - let r: f32x2 = transmute(vsub_f32(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vsubq_f32() { - let a: f32x4 = f32x4::new(1.0, 4.0, 3.0, 8.0); - let b: f32x4 = f32x4::new(1.0, 2.0, 3.0, 4.0); - let e: f32x4 = f32x4::new(0.0, 2.0, 0.0, 4.0); - let r: f32x4 = transmute(vsubq_f32(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vhsub_u8() { - let a: u8x8 = u8x8::new(1, 2, 3, 4, 5, 6, 7, 8); - let b: u8x8 = u8x8::new(1, 2, 1, 2, 1, 2, 1, 2); - let e: u8x8 = u8x8::new(0, 0, 1, 1, 2, 2, 3, 3); - let r: u8x8 = transmute(vhsub_u8(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vhsubq_u8() { - let a: u8x16 = u8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); - let b: u8x16 = u8x16::new(1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2); - let e: u8x16 = u8x16::new(0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7); - let r: u8x16 = transmute(vhsubq_u8(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vhsub_u16() { - let a: u16x4 = u16x4::new(1, 2, 3, 4); - let b: u16x4 = u16x4::new(1, 2, 1, 2); - let e: u16x4 = u16x4::new(0, 0, 1, 1); - let r: u16x4 = transmute(vhsub_u16(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vhsubq_u16() { - let a: u16x8 = u16x8::new(1, 2, 3, 4, 5, 6, 7, 8); - let b: u16x8 = u16x8::new(1, 2, 1, 2, 1, 2, 1, 2); - let e: u16x8 = u16x8::new(0, 0, 1, 1, 2, 2, 3, 3); - let r: u16x8 = transmute(vhsubq_u16(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vhsub_u32() { - let a: u32x2 = u32x2::new(1, 2); - let b: u32x2 = u32x2::new(1, 2); - let e: u32x2 = u32x2::new(0, 0); - let r: u32x2 = transmute(vhsub_u32(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vhsubq_u32() { - let a: u32x4 = u32x4::new(1, 2, 3, 4); - let b: u32x4 = u32x4::new(1, 2, 1, 2); - let e: u32x4 = u32x4::new(0, 0, 1, 1); - let r: u32x4 = transmute(vhsubq_u32(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vhsub_s8() { - let a: i8x8 = i8x8::new(1, 2, 3, 4, 5, 6, 7, 8); - let b: i8x8 = i8x8::new(1, 2, 1, 2, 1, 2, 1, 2); - let e: i8x8 = i8x8::new(0, 0, 1, 1, 2, 2, 3, 3); - let r: i8x8 = transmute(vhsub_s8(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vhsubq_s8() { - let a: i8x16 = i8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); - let b: i8x16 = i8x16::new(1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2); - let e: i8x16 = i8x16::new(0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7); - let r: i8x16 = transmute(vhsubq_s8(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vhsub_s16() { - let a: i16x4 = i16x4::new(1, 2, 3, 4); - let b: i16x4 = i16x4::new(1, 2, 1, 2); - let e: i16x4 = i16x4::new(0, 0, 1, 1); - let r: i16x4 = transmute(vhsub_s16(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vhsubq_s16() { - let a: i16x8 = i16x8::new(1, 2, 3, 4, 5, 6, 7, 8); - let b: i16x8 = i16x8::new(1, 2, 1, 2, 1, 2, 1, 2); - let e: i16x8 = i16x8::new(0, 0, 1, 1, 2, 2, 3, 3); - let r: i16x8 = transmute(vhsubq_s16(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vhsub_s32() { - let a: i32x2 = i32x2::new(1, 2); - let b: i32x2 = i32x2::new(1, 2); - let e: i32x2 = i32x2::new(0, 0); - let r: i32x2 = transmute(vhsub_s32(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vhsubq_s32() { - let a: i32x4 = i32x4::new(1, 2, 3, 4); - let b: i32x4 = i32x4::new(1, 2, 1, 2); - let e: i32x4 = i32x4::new(0, 0, 1, 1); - let r: i32x4 = transmute(vhsubq_s32(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vmax_s8() { - let a: i8x8 = i8x8::new(1, 2, 3, 4, 5, 6, 7, 8); - let b: i8x8 = i8x8::new(16, 15, 14, 13, 12, 11, 10, 9); - let e: i8x8 = i8x8::new(16, 15, 14, 13, 12, 11, 10, 9); - let r: i8x8 = transmute(vmax_s8(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vmaxq_s8() { - let a: i8x16 = i8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); - let b: i8x16 = i8x16::new(16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1); - let e: i8x16 = i8x16::new(16, 15, 14, 13, 12, 11, 10, 9, 9, 10, 11, 12, 13, 14, 15, 16); - let r: i8x16 = transmute(vmaxq_s8(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vmax_s16() { - let a: i16x4 = i16x4::new(1, 2, 3, 4); - let b: i16x4 = i16x4::new(16, 15, 14, 13); - let e: i16x4 = i16x4::new(16, 15, 14, 13); - let r: i16x4 = transmute(vmax_s16(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vmaxq_s16() { - let a: i16x8 = i16x8::new(1, 2, 3, 4, 5, 6, 7, 8); - let b: i16x8 = i16x8::new(16, 15, 14, 13, 12, 11, 10, 9); - let e: i16x8 = i16x8::new(16, 15, 14, 13, 12, 11, 10, 9); - let r: i16x8 = transmute(vmaxq_s16(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vmax_s32() { - let a: i32x2 = i32x2::new(1, 2); - let b: i32x2 = i32x2::new(16, 15); - let e: i32x2 = i32x2::new(16, 15); - let r: i32x2 = transmute(vmax_s32(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vmaxq_s32() { - let a: i32x4 = i32x4::new(1, 2, 3, 4); - let b: i32x4 = i32x4::new(16, 15, 14, 13); - let e: i32x4 = i32x4::new(16, 15, 14, 13); - let r: i32x4 = transmute(vmaxq_s32(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vmax_u8() { - let a: u8x8 = u8x8::new(1, 2, 3, 4, 5, 6, 7, 8); - let b: u8x8 = u8x8::new(16, 15, 14, 13, 12, 11, 10, 9); - let e: u8x8 = u8x8::new(16, 15, 14, 13, 12, 11, 10, 9); - let r: u8x8 = transmute(vmax_u8(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vmaxq_u8() { - let a: u8x16 = u8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); - let b: u8x16 = u8x16::new(16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1); - let e: u8x16 = u8x16::new(16, 15, 14, 13, 12, 11, 10, 9, 9, 10, 11, 12, 13, 14, 15, 16); - let r: u8x16 = transmute(vmaxq_u8(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vmax_u16() { - let a: u16x4 = u16x4::new(1, 2, 3, 4); - let b: u16x4 = u16x4::new(16, 15, 14, 13); - let e: u16x4 = u16x4::new(16, 15, 14, 13); - let r: u16x4 = transmute(vmax_u16(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vmaxq_u16() { - let a: u16x8 = u16x8::new(1, 2, 3, 4, 5, 6, 7, 8); - let b: u16x8 = u16x8::new(16, 15, 14, 13, 12, 11, 10, 9); - let e: u16x8 = u16x8::new(16, 15, 14, 13, 12, 11, 10, 9); - let r: u16x8 = transmute(vmaxq_u16(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vmax_u32() { - let a: u32x2 = u32x2::new(1, 2); - let b: u32x2 = u32x2::new(16, 15); - let e: u32x2 = u32x2::new(16, 15); - let r: u32x2 = transmute(vmax_u32(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vmaxq_u32() { - let a: u32x4 = u32x4::new(1, 2, 3, 4); - let b: u32x4 = u32x4::new(16, 15, 14, 13); - let e: u32x4 = u32x4::new(16, 15, 14, 13); - let r: u32x4 = transmute(vmaxq_u32(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vmax_f32() { - let a: f32x2 = f32x2::new(1.0, -2.0); - let b: f32x2 = f32x2::new(0.0, 3.0); - let e: f32x2 = f32x2::new(1.0, 3.0); - let r: f32x2 = transmute(vmax_f32(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vmaxq_f32() { - let a: f32x4 = f32x4::new(1.0, -2.0, 3.0, -4.0); - let b: f32x4 = f32x4::new(0.0, 3.0, 2.0, 8.0); - let e: f32x4 = f32x4::new(1.0, 3.0, 3.0, 8.0); - let r: f32x4 = transmute(vmaxq_f32(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vmin_s8() { - let a: i8x8 = i8x8::new(1, 2, 3, 4, 5, 6, 7, 8); - let b: i8x8 = i8x8::new(16, 15, 14, 13, 12, 11, 10, 9); - let e: i8x8 = i8x8::new(1, 2, 3, 4, 5, 6, 7, 8); - let r: i8x8 = transmute(vmin_s8(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vminq_s8() { - let a: i8x16 = i8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); - let b: i8x16 = i8x16::new(16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1); - let e: i8x16 = i8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 8, 7, 6, 5, 4, 3, 2, 1); - let r: i8x16 = transmute(vminq_s8(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vmin_s16() { - let a: i16x4 = i16x4::new(1, 2, 3, 4); - let b: i16x4 = i16x4::new(16, 15, 14, 13); - let e: i16x4 = i16x4::new(1, 2, 3, 4); - let r: i16x4 = transmute(vmin_s16(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vminq_s16() { - let a: i16x8 = i16x8::new(1, 2, 3, 4, 5, 6, 7, 8); - let b: i16x8 = i16x8::new(16, 15, 14, 13, 12, 11, 10, 9); - let e: i16x8 = i16x8::new(1, 2, 3, 4, 5, 6, 7, 8); - let r: i16x8 = transmute(vminq_s16(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vmin_s32() { - let a: i32x2 = i32x2::new(1, 2); - let b: i32x2 = i32x2::new(16, 15); - let e: i32x2 = i32x2::new(1, 2); - let r: i32x2 = transmute(vmin_s32(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vminq_s32() { - let a: i32x4 = i32x4::new(1, 2, 3, 4); - let b: i32x4 = i32x4::new(16, 15, 14, 13); - let e: i32x4 = i32x4::new(1, 2, 3, 4); - let r: i32x4 = transmute(vminq_s32(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vmin_u8() { - let a: u8x8 = u8x8::new(1, 2, 3, 4, 5, 6, 7, 8); - let b: u8x8 = u8x8::new(16, 15, 14, 13, 12, 11, 10, 9); - let e: u8x8 = u8x8::new(1, 2, 3, 4, 5, 6, 7, 8); - let r: u8x8 = transmute(vmin_u8(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vminq_u8() { - let a: u8x16 = u8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); - let b: u8x16 = u8x16::new(16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1); - let e: u8x16 = u8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 8, 7, 6, 5, 4, 3, 2, 1); - let r: u8x16 = transmute(vminq_u8(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vmin_u16() { - let a: u16x4 = u16x4::new(1, 2, 3, 4); - let b: u16x4 = u16x4::new(16, 15, 14, 13); - let e: u16x4 = u16x4::new(1, 2, 3, 4); - let r: u16x4 = transmute(vmin_u16(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vminq_u16() { - let a: u16x8 = u16x8::new(1, 2, 3, 4, 5, 6, 7, 8); - let b: u16x8 = u16x8::new(16, 15, 14, 13, 12, 11, 10, 9); - let e: u16x8 = u16x8::new(1, 2, 3, 4, 5, 6, 7, 8); - let r: u16x8 = transmute(vminq_u16(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vmin_u32() { - let a: u32x2 = u32x2::new(1, 2); - let b: u32x2 = u32x2::new(16, 15); - let e: u32x2 = u32x2::new(1, 2); - let r: u32x2 = transmute(vmin_u32(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vminq_u32() { - let a: u32x4 = u32x4::new(1, 2, 3, 4); - let b: u32x4 = u32x4::new(16, 15, 14, 13); - let e: u32x4 = u32x4::new(1, 2, 3, 4); - let r: u32x4 = transmute(vminq_u32(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vmin_f32() { - let a: f32x2 = f32x2::new(1.0, -2.0); - let b: f32x2 = f32x2::new(0.0, 3.0); - let e: f32x2 = f32x2::new(0.0, -2.0); - let r: f32x2 = transmute(vmin_f32(transmute(a), transmute(b))); - assert_eq!(r, e); - } - - #[simd_test(enable = "neon")] - unsafe fn test_vminq_f32() { - let a: f32x4 = f32x4::new(1.0, -2.0, 3.0, -4.0); - let b: f32x4 = f32x4::new(0.0, 3.0, 2.0, 8.0); - let e: f32x4 = f32x4::new(0.0, -2.0, 2.0, -4.0); - let r: f32x4 = transmute(vminq_f32(transmute(a), transmute(b))); - assert_eq!(r, e); - } -} diff --git a/library/stdarch/crates/core_arch/src/acle/sat.rs b/library/stdarch/crates/core_arch/src/arm/sat.rs similarity index 100% rename from library/stdarch/crates/core_arch/src/acle/sat.rs rename to library/stdarch/crates/core_arch/src/arm/sat.rs diff --git a/library/stdarch/crates/core_arch/src/acle/simd32.rs b/library/stdarch/crates/core_arch/src/arm/simd32.rs similarity index 99% rename from library/stdarch/crates/core_arch/src/acle/simd32.rs rename to library/stdarch/crates/core_arch/src/arm/simd32.rs index 04183d4b88..5cae2fc2aa 100644 --- a/library/stdarch/crates/core_arch/src/acle/simd32.rs +++ b/library/stdarch/crates/core_arch/src/arm/simd32.rs @@ -65,7 +65,7 @@ #[cfg(test)] use stdarch_test::assert_instr; -use crate::{core_arch::acle::dsp::int16x2_t, mem::transmute}; +use crate::{core_arch::arm::dsp::int16x2_t, mem::transmute}; types! { /// ARM-specific 32-bit wide vector of four packed `i8`. diff --git a/library/stdarch/crates/core_arch/src/acle/barrier/common.rs b/library/stdarch/crates/core_arch/src/arm_shared/barrier/common.rs similarity index 100% rename from library/stdarch/crates/core_arch/src/acle/barrier/common.rs rename to library/stdarch/crates/core_arch/src/arm_shared/barrier/common.rs diff --git a/library/stdarch/crates/core_arch/src/acle/barrier/cp15.rs b/library/stdarch/crates/core_arch/src/arm_shared/barrier/cp15.rs similarity index 61% rename from library/stdarch/crates/core_arch/src/acle/barrier/cp15.rs rename to library/stdarch/crates/core_arch/src/arm_shared/barrier/cp15.rs index a4d101ad82..25d4c3c8cb 100644 --- a/library/stdarch/crates/core_arch/src/acle/barrier/cp15.rs +++ b/library/stdarch/crates/core_arch/src/arm_shared/barrier/cp15.rs @@ -8,20 +8,29 @@ pub struct SY; impl super::super::sealed::Dmb for SY { #[inline(always)] unsafe fn __dmb(&self) { - llvm_asm!("mcr p15, 0, r0, c7, c10, 5" : : : "memory" : "volatile") + asm!( + "mcr p15, 0, r0, c7, c10, 5", + options(preserves_flags, nostack) + ) } } impl super::super::sealed::Dsb for SY { #[inline(always)] unsafe fn __dsb(&self) { - llvm_asm!("mcr p15, 0, r0, c7, c10, 4" : : : "memory" : "volatile") + asm!( + "mcr p15, 0, r0, c7, c10, 4", + options(preserves_flags, nostack) + ) } } impl super::super::sealed::Isb for SY { #[inline(always)] unsafe fn __isb(&self) { - llvm_asm!("mcr p15, 0, r0, c7, c5, 4" : : : "memory" : "volatile") + asm!( + "mcr p15, 0, r0, c7, c5, 4", + options(preserves_flags, nostack) + ) } } diff --git a/library/stdarch/crates/core_arch/src/acle/barrier/mod.rs b/library/stdarch/crates/core_arch/src/arm_shared/barrier/mod.rs similarity index 100% rename from library/stdarch/crates/core_arch/src/acle/barrier/mod.rs rename to library/stdarch/crates/core_arch/src/arm_shared/barrier/mod.rs diff --git a/library/stdarch/crates/core_arch/src/acle/barrier/not_mclass.rs b/library/stdarch/crates/core_arch/src/arm_shared/barrier/not_mclass.rs similarity index 100% rename from library/stdarch/crates/core_arch/src/acle/barrier/not_mclass.rs rename to library/stdarch/crates/core_arch/src/arm_shared/barrier/not_mclass.rs diff --git a/library/stdarch/crates/core_arch/src/acle/barrier/v8.rs b/library/stdarch/crates/core_arch/src/arm_shared/barrier/v8.rs similarity index 100% rename from library/stdarch/crates/core_arch/src/acle/barrier/v8.rs rename to library/stdarch/crates/core_arch/src/arm_shared/barrier/v8.rs diff --git a/library/stdarch/crates/core_arch/src/arm/crc.rs b/library/stdarch/crates/core_arch/src/arm_shared/crc.rs similarity index 98% rename from library/stdarch/crates/core_arch/src/arm/crc.rs rename to library/stdarch/crates/core_arch/src/arm_shared/crc.rs index ffce20fe22..b1cfbb381b 100644 --- a/library/stdarch/crates/core_arch/src/arm/crc.rs +++ b/library/stdarch/crates/core_arch/src/arm_shared/crc.rs @@ -79,7 +79,7 @@ pub unsafe fn __crc32cw(crc: u32, data: u32) -> u32 { #[cfg(test)] mod tests { - use crate::core_arch::{arm::*, simd::*}; + use crate::core_arch::{arm_shared::*, simd::*}; use std::mem; use stdarch_test::simd_test; diff --git a/library/stdarch/crates/core_arch/src/arm/crypto.rs b/library/stdarch/crates/core_arch/src/arm_shared/crypto.rs similarity index 70% rename from library/stdarch/crates/core_arch/src/arm/crypto.rs rename to library/stdarch/crates/core_arch/src/arm_shared/crypto.rs index 8361e39646..19d9190917 100644 --- a/library/stdarch/crates/core_arch/src/arm/crypto.rs +++ b/library/stdarch/crates/core_arch/src/arm_shared/crypto.rs @@ -1,4 +1,4 @@ -use crate::core_arch::arm::{uint32x4_t, uint8x16_t}; +use crate::core_arch::arm_shared::{uint32x4_t, uint8x16_t}; #[allow(improper_ctypes)] extern "C" { @@ -51,9 +51,16 @@ extern "C" { #[cfg(test)] use stdarch_test::assert_instr; +// Rust compilers without 8a57820bca64a252489790a57cb5ea23db6f9198 need crypto (hence the bootstrap check) +// LLVM builds without b8baa2a9132498ea286dbb0d03f005760ecc6fdb need crypto for arm (hence the target_arch check) + /// AES single round encryption. #[inline] -#[target_feature(enable = "crypto")] +#[cfg_attr(any(bootstrap, target_arch = "arm"), target_feature(enable = "crypto"))] +#[cfg_attr( + not(any(bootstrap, target_arch = "arm")), + target_feature(enable = "aes") +)] #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr(test, assert_instr(aese))] pub unsafe fn vaeseq_u8(data: uint8x16_t, key: uint8x16_t) -> uint8x16_t { @@ -62,7 +69,11 @@ pub unsafe fn vaeseq_u8(data: uint8x16_t, key: uint8x16_t) -> uint8x16_t { /// AES single round decryption. #[inline] -#[target_feature(enable = "crypto")] +#[cfg_attr(any(bootstrap, target_arch = "arm"), target_feature(enable = "crypto"))] +#[cfg_attr( + not(any(bootstrap, target_arch = "arm")), + target_feature(enable = "aes") +)] #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr(test, assert_instr(aesd))] pub unsafe fn vaesdq_u8(data: uint8x16_t, key: uint8x16_t) -> uint8x16_t { @@ -71,7 +82,11 @@ pub unsafe fn vaesdq_u8(data: uint8x16_t, key: uint8x16_t) -> uint8x16_t { /// AES mix columns. #[inline] -#[target_feature(enable = "crypto")] +#[cfg_attr(any(bootstrap, target_arch = "arm"), target_feature(enable = "crypto"))] +#[cfg_attr( + not(any(bootstrap, target_arch = "arm")), + target_feature(enable = "aes") +)] #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr(test, assert_instr(aesmc))] pub unsafe fn vaesmcq_u8(data: uint8x16_t) -> uint8x16_t { @@ -80,7 +95,11 @@ pub unsafe fn vaesmcq_u8(data: uint8x16_t) -> uint8x16_t { /// AES inverse mix columns. #[inline] -#[target_feature(enable = "crypto")] +#[cfg_attr(any(bootstrap, target_arch = "arm"), target_feature(enable = "crypto"))] +#[cfg_attr( + not(any(bootstrap, target_arch = "arm")), + target_feature(enable = "aes") +)] #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr(test, assert_instr(aesimc))] pub unsafe fn vaesimcq_u8(data: uint8x16_t) -> uint8x16_t { @@ -89,7 +108,11 @@ pub unsafe fn vaesimcq_u8(data: uint8x16_t) -> uint8x16_t { /// SHA1 fixed rotate. #[inline] -#[target_feature(enable = "crypto")] +#[cfg_attr(any(bootstrap, target_arch = "arm"), target_feature(enable = "crypto"))] +#[cfg_attr( + not(any(bootstrap, target_arch = "arm")), + target_feature(enable = "sha2") +)] #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr(test, assert_instr(sha1h))] pub unsafe fn vsha1h_u32(hash_e: u32) -> u32 { @@ -98,7 +121,11 @@ pub unsafe fn vsha1h_u32(hash_e: u32) -> u32 { /// SHA1 hash update accelerator, choose. #[inline] -#[target_feature(enable = "crypto")] +#[cfg_attr(any(bootstrap, target_arch = "arm"), target_feature(enable = "crypto"))] +#[cfg_attr( + not(any(bootstrap, target_arch = "arm")), + target_feature(enable = "sha2") +)] #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr(test, assert_instr(sha1c))] pub unsafe fn vsha1cq_u32(hash_abcd: uint32x4_t, hash_e: u32, wk: uint32x4_t) -> uint32x4_t { @@ -107,7 +134,11 @@ pub unsafe fn vsha1cq_u32(hash_abcd: uint32x4_t, hash_e: u32, wk: uint32x4_t) -> /// SHA1 hash update accelerator, majority. #[inline] -#[target_feature(enable = "crypto")] +#[cfg_attr(any(bootstrap, target_arch = "arm"), target_feature(enable = "crypto"))] +#[cfg_attr( + not(any(bootstrap, target_arch = "arm")), + target_feature(enable = "sha2") +)] #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr(test, assert_instr(sha1m))] pub unsafe fn vsha1mq_u32(hash_abcd: uint32x4_t, hash_e: u32, wk: uint32x4_t) -> uint32x4_t { @@ -116,7 +147,11 @@ pub unsafe fn vsha1mq_u32(hash_abcd: uint32x4_t, hash_e: u32, wk: uint32x4_t) -> /// SHA1 hash update accelerator, parity. #[inline] -#[target_feature(enable = "crypto")] +#[cfg_attr(any(bootstrap, target_arch = "arm"), target_feature(enable = "crypto"))] +#[cfg_attr( + not(any(bootstrap, target_arch = "arm")), + target_feature(enable = "sha2") +)] #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr(test, assert_instr(sha1p))] pub unsafe fn vsha1pq_u32(hash_abcd: uint32x4_t, hash_e: u32, wk: uint32x4_t) -> uint32x4_t { @@ -125,7 +160,11 @@ pub unsafe fn vsha1pq_u32(hash_abcd: uint32x4_t, hash_e: u32, wk: uint32x4_t) -> /// SHA1 schedule update accelerator, first part. #[inline] -#[target_feature(enable = "crypto")] +#[cfg_attr(any(bootstrap, target_arch = "arm"), target_feature(enable = "crypto"))] +#[cfg_attr( + not(any(bootstrap, target_arch = "arm")), + target_feature(enable = "sha2") +)] #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr(test, assert_instr(sha1su0))] pub unsafe fn vsha1su0q_u32(w0_3: uint32x4_t, w4_7: uint32x4_t, w8_11: uint32x4_t) -> uint32x4_t { @@ -134,7 +173,11 @@ pub unsafe fn vsha1su0q_u32(w0_3: uint32x4_t, w4_7: uint32x4_t, w8_11: uint32x4_ /// SHA1 schedule update accelerator, second part. #[inline] -#[target_feature(enable = "crypto")] +#[cfg_attr(any(bootstrap, target_arch = "arm"), target_feature(enable = "crypto"))] +#[cfg_attr( + not(any(bootstrap, target_arch = "arm")), + target_feature(enable = "sha2") +)] #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr(test, assert_instr(sha1su1))] pub unsafe fn vsha1su1q_u32(tw0_3: uint32x4_t, w12_15: uint32x4_t) -> uint32x4_t { @@ -143,7 +186,11 @@ pub unsafe fn vsha1su1q_u32(tw0_3: uint32x4_t, w12_15: uint32x4_t) -> uint32x4_t /// SHA256 hash update accelerator. #[inline] -#[target_feature(enable = "crypto")] +#[cfg_attr(any(bootstrap, target_arch = "arm"), target_feature(enable = "crypto"))] +#[cfg_attr( + not(any(bootstrap, target_arch = "arm")), + target_feature(enable = "sha2") +)] #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr(test, assert_instr(sha256h))] pub unsafe fn vsha256hq_u32( @@ -156,7 +203,11 @@ pub unsafe fn vsha256hq_u32( /// SHA256 hash update accelerator, upper part. #[inline] -#[target_feature(enable = "crypto")] +#[cfg_attr(any(bootstrap, target_arch = "arm"), target_feature(enable = "crypto"))] +#[cfg_attr( + not(any(bootstrap, target_arch = "arm")), + target_feature(enable = "sha2") +)] #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr(test, assert_instr(sha256h2))] pub unsafe fn vsha256h2q_u32( @@ -169,7 +220,11 @@ pub unsafe fn vsha256h2q_u32( /// SHA256 schedule update accelerator, first part. #[inline] -#[target_feature(enable = "crypto")] +#[cfg_attr(any(bootstrap, target_arch = "arm"), target_feature(enable = "crypto"))] +#[cfg_attr( + not(any(bootstrap, target_arch = "arm")), + target_feature(enable = "sha2") +)] #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr(test, assert_instr(sha256su0))] pub unsafe fn vsha256su0q_u32(w0_3: uint32x4_t, w4_7: uint32x4_t) -> uint32x4_t { @@ -178,7 +233,11 @@ pub unsafe fn vsha256su0q_u32(w0_3: uint32x4_t, w4_7: uint32x4_t) -> uint32x4_t /// SHA256 schedule update accelerator, second part. #[inline] -#[target_feature(enable = "crypto")] +#[cfg_attr(any(bootstrap, target_arch = "arm"), target_feature(enable = "crypto"))] +#[cfg_attr( + not(any(bootstrap, target_arch = "arm")), + target_feature(enable = "sha2") +)] #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] #[cfg_attr(test, assert_instr(sha256su1))] pub unsafe fn vsha256su1q_u32( @@ -191,11 +250,16 @@ pub unsafe fn vsha256su1q_u32( #[cfg(test)] mod tests { - use crate::core_arch::{arm::*, simd::*}; + use super::*; + use crate::core_arch::{arm_shared::*, simd::*}; use std::mem; use stdarch_test::simd_test; - #[simd_test(enable = "crypto")] + #[cfg_attr(any(bootstrap, target_arch = "arm"), simd_test(enable = "crypto"))] + #[cfg_attr( + all(not(bootstrap), target_arch = "aarch64"), + simd_test(enable = "aes") + )] unsafe fn test_vaeseq_u8() { let data = mem::transmute(u8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8)); let key = mem::transmute(u8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, 4, 5, 6, 7)); @@ -208,7 +272,11 @@ mod tests { ); } - #[simd_test(enable = "crypto")] + #[cfg_attr(any(bootstrap, target_arch = "arm"), simd_test(enable = "crypto"))] + #[cfg_attr( + all(not(bootstrap), target_arch = "aarch64"), + simd_test(enable = "aes") + )] unsafe fn test_vaesdq_u8() { let data = mem::transmute(u8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8)); let key = mem::transmute(u8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, 4, 5, 6, 7)); @@ -219,7 +287,11 @@ mod tests { ); } - #[simd_test(enable = "crypto")] + #[cfg_attr(any(bootstrap, target_arch = "arm"), simd_test(enable = "crypto"))] + #[cfg_attr( + all(not(bootstrap), target_arch = "aarch64"), + simd_test(enable = "aes") + )] unsafe fn test_vaesmcq_u8() { let data = mem::transmute(u8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8)); let r: u8x16 = mem::transmute(vaesmcq_u8(data)); @@ -229,7 +301,11 @@ mod tests { ); } - #[simd_test(enable = "crypto")] + #[cfg_attr(any(bootstrap, target_arch = "arm"), simd_test(enable = "crypto"))] + #[cfg_attr( + all(not(bootstrap), target_arch = "aarch64"), + simd_test(enable = "aes") + )] unsafe fn test_vaesimcq_u8() { let data = mem::transmute(u8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8)); let r: u8x16 = mem::transmute(vaesimcq_u8(data)); @@ -239,13 +315,15 @@ mod tests { ); } - #[simd_test(enable = "crypto")] + #[cfg_attr(any(bootstrap, target_arch = "arm"), simd_test(enable = "crypto"))] + #[cfg_attr(not(any(bootstrap, target_arch = "arm")), simd_test(enable = "sha2"))] unsafe fn test_vsha1h_u32() { assert_eq!(vsha1h_u32(0x1234), 0x048d); assert_eq!(vsha1h_u32(0x5678), 0x159e); } - #[simd_test(enable = "crypto")] + #[cfg_attr(any(bootstrap, target_arch = "arm"), simd_test(enable = "crypto"))] + #[cfg_attr(not(any(bootstrap, target_arch = "arm")), simd_test(enable = "sha2"))] unsafe fn test_vsha1su0q_u32() { let r: u32x4 = mem::transmute(vsha1su0q_u32( mem::transmute(u32x4::new(0x1234_u32, 0x5678_u32, 0x9abc_u32, 0xdef0_u32)), @@ -255,7 +333,8 @@ mod tests { assert_eq!(r, u32x4::new(0x9abc, 0xdef0, 0x1234, 0x5678)); } - #[simd_test(enable = "crypto")] + #[cfg_attr(any(bootstrap, target_arch = "arm"), simd_test(enable = "crypto"))] + #[cfg_attr(not(any(bootstrap, target_arch = "arm")), simd_test(enable = "sha2"))] unsafe fn test_vsha1su1q_u32() { let r: u32x4 = mem::transmute(vsha1su1q_u32( mem::transmute(u32x4::new(0x1234, 0x5678, 0x9abc, 0xdef0)), @@ -267,7 +346,8 @@ mod tests { ); } - #[simd_test(enable = "crypto")] + #[cfg_attr(any(bootstrap, target_arch = "arm"), simd_test(enable = "crypto"))] + #[cfg_attr(not(any(bootstrap, target_arch = "arm")), simd_test(enable = "sha2"))] unsafe fn test_vsha1cq_u32() { let r: u32x4 = mem::transmute(vsha1cq_u32( mem::transmute(u32x4::new(0x1234, 0x5678, 0x9abc, 0xdef0)), @@ -280,7 +360,8 @@ mod tests { ); } - #[simd_test(enable = "crypto")] + #[cfg_attr(any(bootstrap, target_arch = "arm"), simd_test(enable = "crypto"))] + #[cfg_attr(not(any(bootstrap, target_arch = "arm")), simd_test(enable = "sha2"))] unsafe fn test_vsha1pq_u32() { let r: u32x4 = mem::transmute(vsha1pq_u32( mem::transmute(u32x4::new(0x1234, 0x5678, 0x9abc, 0xdef0)), @@ -293,7 +374,8 @@ mod tests { ); } - #[simd_test(enable = "crypto")] + #[cfg_attr(any(bootstrap, target_arch = "arm"), simd_test(enable = "crypto"))] + #[cfg_attr(not(any(bootstrap, target_arch = "arm")), simd_test(enable = "sha2"))] unsafe fn test_vsha1mq_u32() { let r: u32x4 = mem::transmute(vsha1mq_u32( mem::transmute(u32x4::new(0x1234, 0x5678, 0x9abc, 0xdef0)), @@ -306,7 +388,8 @@ mod tests { ); } - #[simd_test(enable = "crypto")] + #[cfg_attr(any(bootstrap, target_arch = "arm"), simd_test(enable = "crypto"))] + #[cfg_attr(not(any(bootstrap, target_arch = "arm")), simd_test(enable = "sha2"))] unsafe fn test_vsha256hq_u32() { let r: u32x4 = mem::transmute(vsha256hq_u32( mem::transmute(u32x4::new(0x1234, 0x5678, 0x9abc, 0xdef0)), @@ -319,7 +402,8 @@ mod tests { ); } - #[simd_test(enable = "crypto")] + #[cfg_attr(any(bootstrap, target_arch = "arm"), simd_test(enable = "crypto"))] + #[cfg_attr(not(any(bootstrap, target_arch = "arm")), simd_test(enable = "sha2"))] unsafe fn test_vsha256h2q_u32() { let r: u32x4 = mem::transmute(vsha256h2q_u32( mem::transmute(u32x4::new(0x1234, 0x5678, 0x9abc, 0xdef0)), @@ -332,7 +416,8 @@ mod tests { ); } - #[simd_test(enable = "crypto")] + #[cfg_attr(any(bootstrap, target_arch = "arm"), simd_test(enable = "crypto"))] + #[cfg_attr(not(any(bootstrap, target_arch = "arm")), simd_test(enable = "sha2"))] unsafe fn test_vsha256su0q_u32() { let r: u32x4 = mem::transmute(vsha256su0q_u32( mem::transmute(u32x4::new(0x1234, 0x5678, 0x9abc, 0xdef0)), @@ -344,7 +429,8 @@ mod tests { ); } - #[simd_test(enable = "crypto")] + #[cfg_attr(any(bootstrap, target_arch = "arm"), simd_test(enable = "crypto"))] + #[cfg_attr(not(any(bootstrap, target_arch = "arm")), simd_test(enable = "sha2"))] unsafe fn test_vsha256su1q_u32() { let r: u32x4 = mem::transmute(vsha256su1q_u32( mem::transmute(u32x4::new(0x1234, 0x5678, 0x9abc, 0xdef0)), diff --git a/library/stdarch/crates/core_arch/src/acle/hints.rs b/library/stdarch/crates/core_arch/src/arm_shared/hints.rs similarity index 63% rename from library/stdarch/crates/core_arch/src/acle/hints.rs rename to library/stdarch/crates/core_arch/src/arm_shared/hints.rs index 892d4f517f..3145cde8d5 100644 --- a/library/stdarch/crates/core_arch/src/acle/hints.rs +++ b/library/stdarch/crates/core_arch/src/arm_shared/hints.rs @@ -9,7 +9,7 @@ /// low-power state until one of a number of asynchronous events occurs. // Section 10.1 of ACLE says that the supported arches are: 8, 6K, 6-M // LLVM says "instruction requires: armv6k" -#[cfg(any(target_feature = "v6", target_arch = "aarch64"))] +#[cfg(any(target_feature = "v6", target_arch = "aarch64", doc))] #[inline(always)] pub unsafe fn __wfi() { hint(HINT_WFI); @@ -22,7 +22,7 @@ pub unsafe fn __wfi() { /// another processor. // Section 10.1 of ACLE says that the supported arches are: 8, 6K, 6-M // LLVM says "instruction requires: armv6k" -#[cfg(any(target_feature = "v6", target_arch = "aarch64"))] +#[cfg(any(target_feature = "v6", target_arch = "aarch64", doc))] #[inline(always)] pub unsafe fn __wfe() { hint(HINT_WFE); @@ -34,7 +34,7 @@ pub unsafe fn __wfe() { /// system. It is a NOP on a uniprocessor system. // Section 10.1 of ACLE says that the supported arches are: 8, 6K, 6-M, 7-M // LLVM says "instruction requires: armv6k" -#[cfg(any(target_feature = "v6", target_arch = "aarch64"))] +#[cfg(any(target_feature = "v6", target_arch = "aarch64", doc))] #[inline(always)] pub unsafe fn __sev() { hint(HINT_SEV); @@ -49,6 +49,7 @@ pub unsafe fn __sev() { #[cfg(any( target_feature = "v8", // 32-bit ARMv8 target_arch = "aarch64", // AArch64 + doc, ))] #[inline(always)] pub unsafe fn __sevl() { @@ -62,53 +63,12 @@ pub unsafe fn __sevl() { /// improve overall system performance. // Section 10.1 of ACLE says that the supported arches are: 8, 6K, 6-M // LLVM says "instruction requires: armv6k" -#[cfg(any(target_feature = "v6", target_arch = "aarch64"))] +#[cfg(any(target_feature = "v6", target_arch = "aarch64", doc))] #[inline(always)] pub unsafe fn __yield() { hint(HINT_YIELD); } -/// Generates a DBG instruction. -/// -/// This provides a hint to debugging and related systems. The argument must be -/// a constant integer from 0 to 15 inclusive. See implementation documentation -/// for the effect (if any) of this instruction and the meaning of the -/// argument. This is available only when compliling for AArch32. -// Section 10.1 of ACLE says that the supported arches are: 7, 7-M -// "The DBG hint instruction is added in ARMv7. It is UNDEFINED in the ARMv6 base architecture, and -// executes as a NOP instruction in ARMv6K and ARMv6T2." - ARM Architecture Reference Manual ARMv7-A -// and ARMv7-R edition (ARM DDI 0406C.c) sections D12.4.1 "ARM instruction set support" and D12.4.2 -// "Thumb instruction set support" -#[cfg(target_feature = "v7")] -#[inline(always)] -#[rustc_args_required_const(0)] -pub unsafe fn __dbg(imm4: u32) { - macro_rules! call { - ($imm4:expr) => { - llvm_asm!(concat!("DBG ", stringify!($imm4)) : : : : "volatile") - } - } - - match imm4 & 0b1111 { - 0 => call!(0), - 1 => call!(1), - 2 => call!(2), - 3 => call!(3), - 4 => call!(4), - 5 => call!(5), - 6 => call!(6), - 7 => call!(7), - 8 => call!(8), - 9 => call!(9), - 10 => call!(10), - 11 => call!(11), - 12 => call!(12), - 13 => call!(13), - 14 => call!(14), - _ => call!(15), - } -} - /// Generates an unspecified no-op instruction. /// /// Note that not all architectures provide a distinguished NOP instruction. On @@ -117,7 +77,7 @@ pub unsafe fn __dbg(imm4: u32) { /// will increase execution time. #[inline(always)] pub unsafe fn __nop() { - llvm_asm!("NOP" : : : : "volatile") + asm!("nop", options(nomem, nostack, preserves_flags)); } extern "C" { diff --git a/library/stdarch/crates/core_arch/src/acle/mod.rs b/library/stdarch/crates/core_arch/src/arm_shared/mod.rs similarity index 65% rename from library/stdarch/crates/core_arch/src/acle/mod.rs rename to library/stdarch/crates/core_arch/src/arm_shared/mod.rs index 5f29decf5a..4c8d19854e 100644 --- a/library/stdarch/crates/core_arch/src/acle/mod.rs +++ b/library/stdarch/crates/core_arch/src/arm_shared/mod.rs @@ -47,6 +47,9 @@ //! //! - [ACLE Q2 2018](https://developer.arm.com/docs/101028/latest) +// Only for 'neon' submodule +#![allow(non_camel_case_types)] + // 8, 7 and 6-M are supported via dedicated instructions like DMB. All other arches are supported // via CP15 instructions. See Section 10.1 of ACLE mod barrier; @@ -54,70 +57,29 @@ mod barrier; pub use self::barrier::*; mod hints; - pub use self::hints::*; mod registers; - pub use self::registers::*; -mod ex; - -pub use self::ex::*; - -// Supported arches: 5TE, 7E-M. See Section 10.1 of ACLE (e.g. QADD) -// We also include the A profile even though DSP is deprecated on that profile as of ACLE 2.0 (see -// section 5.4.7) -// Here we workaround the difference between LLVM's +dsp and ACLE's __ARM_FEATURE_DSP by gating on -// '+v5te' rather than on '+dsp' -#[cfg(all( - not(target_arch = "aarch64"), - any( - // >= v5TE but excludes v7-M - all(target_feature = "v5te", not(target_feature = "mclass")), - // v7E-M - all(target_feature = "mclass", target_feature = "dsp"), - ) -))] -mod dsp; - -#[cfg(all( - not(target_arch = "aarch64"), - any( - all(target_feature = "v5te", not(target_feature = "mclass")), - all(target_feature = "mclass", target_feature = "dsp"), - ) -))] -pub use self::dsp::*; - -// Supported arches: 6, 7-M. See Section 10.1 of ACLE (e.g. SSAT) -#[cfg(all(not(target_arch = "aarch64"), target_feature = "v6",))] -mod sat; - -#[cfg(all(not(target_arch = "aarch64"), target_feature = "v6",))] -pub use self::sat::*; - -// Deprecated in ACLE 2.0 for the A profile but fully supported on the M and R profiles, says -// Section 5.4.9 of ACLE. We'll expose these for the A profile even if deprecated -#[cfg(all( - not(target_arch = "aarch64"), - any( - // v7-A, v7-R - all(target_feature = "v6", not(target_feature = "mclass")), - // v7E-M - all(target_feature = "mclass", target_feature = "dsp") - ) -))] -mod simd32; - -#[cfg(all( - not(target_arch = "aarch64"), - any( - all(target_feature = "v6", not(target_feature = "mclass")), - all(target_feature = "mclass", target_feature = "dsp") - ) -))] -pub use self::simd32::*; +#[cfg(any(target_arch = "aarch64", target_feature = "v7", doc))] +mod crc; +#[cfg(any(target_arch = "aarch64", target_feature = "v7", doc))] +pub use crc::*; + +#[cfg(any(target_arch = "aarch64", target_feature = "v7", doc))] +mod crypto; +#[cfg(any(target_arch = "aarch64", target_feature = "v7", doc))] +pub use self::crypto::*; + +#[cfg(any(target_arch = "aarch64", target_feature = "v7", doc))] +pub(crate) mod neon; +#[cfg(any(target_arch = "aarch64", target_feature = "v7", doc))] +pub use self::neon::*; + +#[cfg(test)] +#[cfg(any(target_arch = "aarch64", target_feature = "v7", doc))] +pub(crate) mod test_support; mod sealed { pub trait Dmb { diff --git a/library/stdarch/crates/core_arch/src/arm_shared/neon/generated.rs b/library/stdarch/crates/core_arch/src/arm_shared/neon/generated.rs new file mode 100644 index 0000000000..296b86469a --- /dev/null +++ b/library/stdarch/crates/core_arch/src/arm_shared/neon/generated.rs @@ -0,0 +1,24866 @@ +// This code is automatically generated. DO NOT MODIFY. +// +// Instead, modify `crates/stdarch-gen/neon.spec` and run the following command to re-generate this file: +// +// ``` +// OUT_DIR=`pwd`/crates/core_arch cargo run -p stdarch-gen -- crates/stdarch-gen/neon.spec +// ``` +use super::*; +#[cfg(test)] +use stdarch_test::assert_instr; + +/// Vector bitwise and +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vand))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(and))] +pub unsafe fn vand_s8(a: int8x8_t, b: int8x8_t) -> int8x8_t { + simd_and(a, b) +} + +/// Vector bitwise and +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vand))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(and))] +pub unsafe fn vandq_s8(a: int8x16_t, b: int8x16_t) -> int8x16_t { + simd_and(a, b) +} + +/// Vector bitwise and +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vand))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(and))] +pub unsafe fn vand_s16(a: int16x4_t, b: int16x4_t) -> int16x4_t { + simd_and(a, b) +} + +/// Vector bitwise and +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vand))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(and))] +pub unsafe fn vandq_s16(a: int16x8_t, b: int16x8_t) -> int16x8_t { + simd_and(a, b) +} + +/// Vector bitwise and +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vand))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(and))] +pub unsafe fn vand_s32(a: int32x2_t, b: int32x2_t) -> int32x2_t { + simd_and(a, b) +} + +/// Vector bitwise and +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vand))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(and))] +pub unsafe fn vandq_s32(a: int32x4_t, b: int32x4_t) -> int32x4_t { + simd_and(a, b) +} + +/// Vector bitwise and +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vand))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(and))] +pub unsafe fn vand_u8(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t { + simd_and(a, b) +} + +/// Vector bitwise and +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vand))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(and))] +pub unsafe fn vandq_u8(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t { + simd_and(a, b) +} + +/// Vector bitwise and +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vand))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(and))] +pub unsafe fn vand_u16(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t { + simd_and(a, b) +} + +/// Vector bitwise and +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vand))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(and))] +pub unsafe fn vandq_u16(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t { + simd_and(a, b) +} + +/// Vector bitwise and +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vand))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(and))] +pub unsafe fn vand_u32(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t { + simd_and(a, b) +} + +/// Vector bitwise and +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vand))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(and))] +pub unsafe fn vandq_u32(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t { + simd_and(a, b) +} + +/// Vector bitwise and +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vand))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(and))] +pub unsafe fn vand_s64(a: int64x1_t, b: int64x1_t) -> int64x1_t { + simd_and(a, b) +} + +/// Vector bitwise and +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vand))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(and))] +pub unsafe fn vandq_s64(a: int64x2_t, b: int64x2_t) -> int64x2_t { + simd_and(a, b) +} + +/// Vector bitwise and +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vand))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(and))] +pub unsafe fn vand_u64(a: uint64x1_t, b: uint64x1_t) -> uint64x1_t { + simd_and(a, b) +} + +/// Vector bitwise and +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vand))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(and))] +pub unsafe fn vandq_u64(a: uint64x2_t, b: uint64x2_t) -> uint64x2_t { + simd_and(a, b) +} + +/// Vector bitwise or (immediate, inclusive) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vorr))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(orr))] +pub unsafe fn vorr_s8(a: int8x8_t, b: int8x8_t) -> int8x8_t { + simd_or(a, b) +} + +/// Vector bitwise or (immediate, inclusive) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vorr))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(orr))] +pub unsafe fn vorrq_s8(a: int8x16_t, b: int8x16_t) -> int8x16_t { + simd_or(a, b) +} + +/// Vector bitwise or (immediate, inclusive) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vorr))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(orr))] +pub unsafe fn vorr_s16(a: int16x4_t, b: int16x4_t) -> int16x4_t { + simd_or(a, b) +} + +/// Vector bitwise or (immediate, inclusive) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vorr))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(orr))] +pub unsafe fn vorrq_s16(a: int16x8_t, b: int16x8_t) -> int16x8_t { + simd_or(a, b) +} + +/// Vector bitwise or (immediate, inclusive) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vorr))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(orr))] +pub unsafe fn vorr_s32(a: int32x2_t, b: int32x2_t) -> int32x2_t { + simd_or(a, b) +} + +/// Vector bitwise or (immediate, inclusive) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vorr))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(orr))] +pub unsafe fn vorrq_s32(a: int32x4_t, b: int32x4_t) -> int32x4_t { + simd_or(a, b) +} + +/// Vector bitwise or (immediate, inclusive) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vorr))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(orr))] +pub unsafe fn vorr_u8(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t { + simd_or(a, b) +} + +/// Vector bitwise or (immediate, inclusive) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vorr))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(orr))] +pub unsafe fn vorrq_u8(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t { + simd_or(a, b) +} + +/// Vector bitwise or (immediate, inclusive) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vorr))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(orr))] +pub unsafe fn vorr_u16(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t { + simd_or(a, b) +} + +/// Vector bitwise or (immediate, inclusive) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vorr))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(orr))] +pub unsafe fn vorrq_u16(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t { + simd_or(a, b) +} + +/// Vector bitwise or (immediate, inclusive) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vorr))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(orr))] +pub unsafe fn vorr_u32(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t { + simd_or(a, b) +} + +/// Vector bitwise or (immediate, inclusive) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vorr))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(orr))] +pub unsafe fn vorrq_u32(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t { + simd_or(a, b) +} + +/// Vector bitwise or (immediate, inclusive) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vorr))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(orr))] +pub unsafe fn vorr_s64(a: int64x1_t, b: int64x1_t) -> int64x1_t { + simd_or(a, b) +} + +/// Vector bitwise or (immediate, inclusive) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vorr))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(orr))] +pub unsafe fn vorrq_s64(a: int64x2_t, b: int64x2_t) -> int64x2_t { + simd_or(a, b) +} + +/// Vector bitwise or (immediate, inclusive) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vorr))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(orr))] +pub unsafe fn vorr_u64(a: uint64x1_t, b: uint64x1_t) -> uint64x1_t { + simd_or(a, b) +} + +/// Vector bitwise or (immediate, inclusive) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vorr))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(orr))] +pub unsafe fn vorrq_u64(a: uint64x2_t, b: uint64x2_t) -> uint64x2_t { + simd_or(a, b) +} + +/// Vector bitwise exclusive or (vector) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(veor))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(eor))] +pub unsafe fn veor_s8(a: int8x8_t, b: int8x8_t) -> int8x8_t { + simd_xor(a, b) +} + +/// Vector bitwise exclusive or (vector) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(veor))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(eor))] +pub unsafe fn veorq_s8(a: int8x16_t, b: int8x16_t) -> int8x16_t { + simd_xor(a, b) +} + +/// Vector bitwise exclusive or (vector) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(veor))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(eor))] +pub unsafe fn veor_s16(a: int16x4_t, b: int16x4_t) -> int16x4_t { + simd_xor(a, b) +} + +/// Vector bitwise exclusive or (vector) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(veor))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(eor))] +pub unsafe fn veorq_s16(a: int16x8_t, b: int16x8_t) -> int16x8_t { + simd_xor(a, b) +} + +/// Vector bitwise exclusive or (vector) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(veor))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(eor))] +pub unsafe fn veor_s32(a: int32x2_t, b: int32x2_t) -> int32x2_t { + simd_xor(a, b) +} + +/// Vector bitwise exclusive or (vector) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(veor))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(eor))] +pub unsafe fn veorq_s32(a: int32x4_t, b: int32x4_t) -> int32x4_t { + simd_xor(a, b) +} + +/// Vector bitwise exclusive or (vector) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(veor))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(eor))] +pub unsafe fn veor_u8(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t { + simd_xor(a, b) +} + +/// Vector bitwise exclusive or (vector) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(veor))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(eor))] +pub unsafe fn veorq_u8(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t { + simd_xor(a, b) +} + +/// Vector bitwise exclusive or (vector) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(veor))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(eor))] +pub unsafe fn veor_u16(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t { + simd_xor(a, b) +} + +/// Vector bitwise exclusive or (vector) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(veor))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(eor))] +pub unsafe fn veorq_u16(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t { + simd_xor(a, b) +} + +/// Vector bitwise exclusive or (vector) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(veor))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(eor))] +pub unsafe fn veor_u32(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t { + simd_xor(a, b) +} + +/// Vector bitwise exclusive or (vector) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(veor))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(eor))] +pub unsafe fn veorq_u32(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t { + simd_xor(a, b) +} + +/// Vector bitwise exclusive or (vector) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(veor))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(eor))] +pub unsafe fn veor_s64(a: int64x1_t, b: int64x1_t) -> int64x1_t { + simd_xor(a, b) +} + +/// Vector bitwise exclusive or (vector) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(veor))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(eor))] +pub unsafe fn veorq_s64(a: int64x2_t, b: int64x2_t) -> int64x2_t { + simd_xor(a, b) +} + +/// Vector bitwise exclusive or (vector) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(veor))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(eor))] +pub unsafe fn veor_u64(a: uint64x1_t, b: uint64x1_t) -> uint64x1_t { + simd_xor(a, b) +} + +/// Vector bitwise exclusive or (vector) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(veor))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(eor))] +pub unsafe fn veorq_u64(a: uint64x2_t, b: uint64x2_t) -> uint64x2_t { + simd_xor(a, b) +} + +/// Absolute difference between the arguments +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vabd.s8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sabd))] +pub unsafe fn vabd_s8(a: int8x8_t, b: int8x8_t) -> int8x8_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vabds.v8i8")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sabd.v8i8")] + fn vabd_s8_(a: int8x8_t, b: int8x8_t) -> int8x8_t; + } +vabd_s8_(a, b) +} + +/// Absolute difference between the arguments +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vabd.s8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sabd))] +pub unsafe fn vabdq_s8(a: int8x16_t, b: int8x16_t) -> int8x16_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vabds.v16i8")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sabd.v16i8")] + fn vabdq_s8_(a: int8x16_t, b: int8x16_t) -> int8x16_t; + } +vabdq_s8_(a, b) +} + +/// Absolute difference between the arguments +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vabd.s16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sabd))] +pub unsafe fn vabd_s16(a: int16x4_t, b: int16x4_t) -> int16x4_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vabds.v4i16")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sabd.v4i16")] + fn vabd_s16_(a: int16x4_t, b: int16x4_t) -> int16x4_t; + } +vabd_s16_(a, b) +} + +/// Absolute difference between the arguments +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vabd.s16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sabd))] +pub unsafe fn vabdq_s16(a: int16x8_t, b: int16x8_t) -> int16x8_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vabds.v8i16")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sabd.v8i16")] + fn vabdq_s16_(a: int16x8_t, b: int16x8_t) -> int16x8_t; + } +vabdq_s16_(a, b) +} + +/// Absolute difference between the arguments +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vabd.s32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sabd))] +pub unsafe fn vabd_s32(a: int32x2_t, b: int32x2_t) -> int32x2_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vabds.v2i32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sabd.v2i32")] + fn vabd_s32_(a: int32x2_t, b: int32x2_t) -> int32x2_t; + } +vabd_s32_(a, b) +} + +/// Absolute difference between the arguments +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vabd.s32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sabd))] +pub unsafe fn vabdq_s32(a: int32x4_t, b: int32x4_t) -> int32x4_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vabds.v4i32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sabd.v4i32")] + fn vabdq_s32_(a: int32x4_t, b: int32x4_t) -> int32x4_t; + } +vabdq_s32_(a, b) +} + +/// Absolute difference between the arguments +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vabd.u8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uabd))] +pub unsafe fn vabd_u8(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vabdu.v8i8")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uabd.v8i8")] + fn vabd_u8_(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t; + } +vabd_u8_(a, b) +} + +/// Absolute difference between the arguments +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vabd.u8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uabd))] +pub unsafe fn vabdq_u8(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vabdu.v16i8")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uabd.v16i8")] + fn vabdq_u8_(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t; + } +vabdq_u8_(a, b) +} + +/// Absolute difference between the arguments +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vabd.u16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uabd))] +pub unsafe fn vabd_u16(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vabdu.v4i16")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uabd.v4i16")] + fn vabd_u16_(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t; + } +vabd_u16_(a, b) +} + +/// Absolute difference between the arguments +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vabd.u16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uabd))] +pub unsafe fn vabdq_u16(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vabdu.v8i16")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uabd.v8i16")] + fn vabdq_u16_(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t; + } +vabdq_u16_(a, b) +} + +/// Absolute difference between the arguments +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vabd.u32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uabd))] +pub unsafe fn vabd_u32(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vabdu.v2i32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uabd.v2i32")] + fn vabd_u32_(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t; + } +vabd_u32_(a, b) +} + +/// Absolute difference between the arguments +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vabd.u32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uabd))] +pub unsafe fn vabdq_u32(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vabdu.v4i32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uabd.v4i32")] + fn vabdq_u32_(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t; + } +vabdq_u32_(a, b) +} + +/// Absolute difference between the arguments of Floating +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vabd.f32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fabd))] +pub unsafe fn vabd_f32(a: float32x2_t, b: float32x2_t) -> float32x2_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vabds.v2f32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fabd.v2f32")] + fn vabd_f32_(a: float32x2_t, b: float32x2_t) -> float32x2_t; + } +vabd_f32_(a, b) +} + +/// Absolute difference between the arguments of Floating +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vabd.f32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fabd))] +pub unsafe fn vabdq_f32(a: float32x4_t, b: float32x4_t) -> float32x4_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vabds.v4f32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fabd.v4f32")] + fn vabdq_f32_(a: float32x4_t, b: float32x4_t) -> float32x4_t; + } +vabdq_f32_(a, b) +} + +/// Unsigned Absolute difference Long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vabdl.u8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uabdl))] +pub unsafe fn vabdl_u8(a: uint8x8_t, b: uint8x8_t) -> uint16x8_t { + simd_cast(vabd_u8(a, b)) +} + +/// Unsigned Absolute difference Long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vabdl.u16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uabdl))] +pub unsafe fn vabdl_u16(a: uint16x4_t, b: uint16x4_t) -> uint32x4_t { + simd_cast(vabd_u16(a, b)) +} + +/// Unsigned Absolute difference Long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vabdl.u32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uabdl))] +pub unsafe fn vabdl_u32(a: uint32x2_t, b: uint32x2_t) -> uint64x2_t { + simd_cast(vabd_u32(a, b)) +} + +/// Signed Absolute difference Long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vabdl.s8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sabdl))] +pub unsafe fn vabdl_s8(a: int8x8_t, b: int8x8_t) -> int16x8_t { + let c: uint8x8_t = simd_cast(vabd_s8(a, b)); + simd_cast(c) +} + +/// Signed Absolute difference Long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vabdl.s16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sabdl))] +pub unsafe fn vabdl_s16(a: int16x4_t, b: int16x4_t) -> int32x4_t { + let c: uint16x4_t = simd_cast(vabd_s16(a, b)); + simd_cast(c) +} + +/// Signed Absolute difference Long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vabdl.s32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sabdl))] +pub unsafe fn vabdl_s32(a: int32x2_t, b: int32x2_t) -> int64x2_t { + let c: uint32x2_t = simd_cast(vabd_s32(a, b)); + simd_cast(c) +} + +/// Compare bitwise Equal (vector) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vceq.i8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmeq))] +pub unsafe fn vceq_u8(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t { + simd_eq(a, b) +} + +/// Compare bitwise Equal (vector) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vceq.i8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmeq))] +pub unsafe fn vceqq_u8(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t { + simd_eq(a, b) +} + +/// Compare bitwise Equal (vector) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vceq.i16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmeq))] +pub unsafe fn vceq_u16(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t { + simd_eq(a, b) +} + +/// Compare bitwise Equal (vector) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vceq.i16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmeq))] +pub unsafe fn vceqq_u16(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t { + simd_eq(a, b) +} + +/// Compare bitwise Equal (vector) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vceq.i32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmeq))] +pub unsafe fn vceq_u32(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t { + simd_eq(a, b) +} + +/// Compare bitwise Equal (vector) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vceq.i32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmeq))] +pub unsafe fn vceqq_u32(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t { + simd_eq(a, b) +} + +/// Compare bitwise Equal (vector) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vceq.i8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmeq))] +pub unsafe fn vceq_s8(a: int8x8_t, b: int8x8_t) -> uint8x8_t { + simd_eq(a, b) +} + +/// Compare bitwise Equal (vector) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vceq.i8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmeq))] +pub unsafe fn vceqq_s8(a: int8x16_t, b: int8x16_t) -> uint8x16_t { + simd_eq(a, b) +} + +/// Compare bitwise Equal (vector) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vceq.i16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmeq))] +pub unsafe fn vceq_s16(a: int16x4_t, b: int16x4_t) -> uint16x4_t { + simd_eq(a, b) +} + +/// Compare bitwise Equal (vector) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vceq.i16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmeq))] +pub unsafe fn vceqq_s16(a: int16x8_t, b: int16x8_t) -> uint16x8_t { + simd_eq(a, b) +} + +/// Compare bitwise Equal (vector) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vceq.i32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmeq))] +pub unsafe fn vceq_s32(a: int32x2_t, b: int32x2_t) -> uint32x2_t { + simd_eq(a, b) +} + +/// Compare bitwise Equal (vector) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vceq.i32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmeq))] +pub unsafe fn vceqq_s32(a: int32x4_t, b: int32x4_t) -> uint32x4_t { + simd_eq(a, b) +} + +/// Compare bitwise Equal (vector) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vceq.i8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmeq))] +pub unsafe fn vceq_p8(a: poly8x8_t, b: poly8x8_t) -> uint8x8_t { + simd_eq(a, b) +} + +/// Compare bitwise Equal (vector) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vceq.i8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmeq))] +pub unsafe fn vceqq_p8(a: poly8x16_t, b: poly8x16_t) -> uint8x16_t { + simd_eq(a, b) +} + +/// Floating-point compare equal +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vceq.f32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fcmeq))] +pub unsafe fn vceq_f32(a: float32x2_t, b: float32x2_t) -> uint32x2_t { + simd_eq(a, b) +} + +/// Floating-point compare equal +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vceq.f32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fcmeq))] +pub unsafe fn vceqq_f32(a: float32x4_t, b: float32x4_t) -> uint32x4_t { + simd_eq(a, b) +} + +/// Signed compare bitwise Test bits nonzero +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vtst))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmtst))] +pub unsafe fn vtst_s8(a: int8x8_t, b: int8x8_t) -> uint8x8_t { + let c: int8x8_t = simd_and(a, b); + let d: i8x8 = i8x8::new(0, 0, 0, 0, 0, 0, 0, 0); + simd_ne(c, transmute(d)) +} + +/// Signed compare bitwise Test bits nonzero +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vtst))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmtst))] +pub unsafe fn vtstq_s8(a: int8x16_t, b: int8x16_t) -> uint8x16_t { + let c: int8x16_t = simd_and(a, b); + let d: i8x16 = i8x16::new(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + simd_ne(c, transmute(d)) +} + +/// Signed compare bitwise Test bits nonzero +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vtst))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmtst))] +pub unsafe fn vtst_s16(a: int16x4_t, b: int16x4_t) -> uint16x4_t { + let c: int16x4_t = simd_and(a, b); + let d: i16x4 = i16x4::new(0, 0, 0, 0); + simd_ne(c, transmute(d)) +} + +/// Signed compare bitwise Test bits nonzero +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vtst))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmtst))] +pub unsafe fn vtstq_s16(a: int16x8_t, b: int16x8_t) -> uint16x8_t { + let c: int16x8_t = simd_and(a, b); + let d: i16x8 = i16x8::new(0, 0, 0, 0, 0, 0, 0, 0); + simd_ne(c, transmute(d)) +} + +/// Signed compare bitwise Test bits nonzero +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vtst))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmtst))] +pub unsafe fn vtst_s32(a: int32x2_t, b: int32x2_t) -> uint32x2_t { + let c: int32x2_t = simd_and(a, b); + let d: i32x2 = i32x2::new(0, 0); + simd_ne(c, transmute(d)) +} + +/// Signed compare bitwise Test bits nonzero +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vtst))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmtst))] +pub unsafe fn vtstq_s32(a: int32x4_t, b: int32x4_t) -> uint32x4_t { + let c: int32x4_t = simd_and(a, b); + let d: i32x4 = i32x4::new(0, 0, 0, 0); + simd_ne(c, transmute(d)) +} + +/// Signed compare bitwise Test bits nonzero +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vtst))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmtst))] +pub unsafe fn vtst_p8(a: poly8x8_t, b: poly8x8_t) -> uint8x8_t { + let c: poly8x8_t = simd_and(a, b); + let d: i8x8 = i8x8::new(0, 0, 0, 0, 0, 0, 0, 0); + simd_ne(c, transmute(d)) +} + +/// Signed compare bitwise Test bits nonzero +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vtst))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmtst))] +pub unsafe fn vtstq_p8(a: poly8x16_t, b: poly8x16_t) -> uint8x16_t { + let c: poly8x16_t = simd_and(a, b); + let d: i8x16 = i8x16::new(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + simd_ne(c, transmute(d)) +} + +/// Unsigned compare bitwise Test bits nonzero +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vtst))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmtst))] +pub unsafe fn vtst_u8(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t { + let c: uint8x8_t = simd_and(a, b); + let d: u8x8 = u8x8::new(0, 0, 0, 0, 0, 0, 0, 0); + simd_ne(c, transmute(d)) +} + +/// Unsigned compare bitwise Test bits nonzero +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vtst))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmtst))] +pub unsafe fn vtstq_u8(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t { + let c: uint8x16_t = simd_and(a, b); + let d: u8x16 = u8x16::new(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + simd_ne(c, transmute(d)) +} + +/// Unsigned compare bitwise Test bits nonzero +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vtst))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmtst))] +pub unsafe fn vtst_u16(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t { + let c: uint16x4_t = simd_and(a, b); + let d: u16x4 = u16x4::new(0, 0, 0, 0); + simd_ne(c, transmute(d)) +} + +/// Unsigned compare bitwise Test bits nonzero +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vtst))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmtst))] +pub unsafe fn vtstq_u16(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t { + let c: uint16x8_t = simd_and(a, b); + let d: u16x8 = u16x8::new(0, 0, 0, 0, 0, 0, 0, 0); + simd_ne(c, transmute(d)) +} + +/// Unsigned compare bitwise Test bits nonzero +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vtst))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmtst))] +pub unsafe fn vtst_u32(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t { + let c: uint32x2_t = simd_and(a, b); + let d: u32x2 = u32x2::new(0, 0); + simd_ne(c, transmute(d)) +} + +/// Unsigned compare bitwise Test bits nonzero +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vtst))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmtst))] +pub unsafe fn vtstq_u32(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t { + let c: uint32x4_t = simd_and(a, b); + let d: u32x4 = u32x4::new(0, 0, 0, 0); + simd_ne(c, transmute(d)) +} + +/// Floating-point absolute value +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vabs))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fabs))] +pub unsafe fn vabs_f32(a: float32x2_t) -> float32x2_t { + simd_fabs(a) +} + +/// Floating-point absolute value +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vabs))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fabs))] +pub unsafe fn vabsq_f32(a: float32x4_t) -> float32x4_t { + simd_fabs(a) +} + +/// Compare signed greater than +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcgt.s8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmgt))] +pub unsafe fn vcgt_s8(a: int8x8_t, b: int8x8_t) -> uint8x8_t { + simd_gt(a, b) +} + +/// Compare signed greater than +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcgt.s8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmgt))] +pub unsafe fn vcgtq_s8(a: int8x16_t, b: int8x16_t) -> uint8x16_t { + simd_gt(a, b) +} + +/// Compare signed greater than +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcgt.s16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmgt))] +pub unsafe fn vcgt_s16(a: int16x4_t, b: int16x4_t) -> uint16x4_t { + simd_gt(a, b) +} + +/// Compare signed greater than +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcgt.s16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmgt))] +pub unsafe fn vcgtq_s16(a: int16x8_t, b: int16x8_t) -> uint16x8_t { + simd_gt(a, b) +} + +/// Compare signed greater than +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcgt.s32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmgt))] +pub unsafe fn vcgt_s32(a: int32x2_t, b: int32x2_t) -> uint32x2_t { + simd_gt(a, b) +} + +/// Compare signed greater than +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcgt.s32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmgt))] +pub unsafe fn vcgtq_s32(a: int32x4_t, b: int32x4_t) -> uint32x4_t { + simd_gt(a, b) +} + +/// Compare unsigned highe +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcgt.u8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmhi))] +pub unsafe fn vcgt_u8(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t { + simd_gt(a, b) +} + +/// Compare unsigned highe +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcgt.u8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmhi))] +pub unsafe fn vcgtq_u8(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t { + simd_gt(a, b) +} + +/// Compare unsigned highe +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcgt.u16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmhi))] +pub unsafe fn vcgt_u16(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t { + simd_gt(a, b) +} + +/// Compare unsigned highe +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcgt.u16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmhi))] +pub unsafe fn vcgtq_u16(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t { + simd_gt(a, b) +} + +/// Compare unsigned highe +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcgt.u32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmhi))] +pub unsafe fn vcgt_u32(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t { + simd_gt(a, b) +} + +/// Compare unsigned highe +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcgt.u32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmhi))] +pub unsafe fn vcgtq_u32(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t { + simd_gt(a, b) +} + +/// Floating-point compare greater than +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcgt.f32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fcmgt))] +pub unsafe fn vcgt_f32(a: float32x2_t, b: float32x2_t) -> uint32x2_t { + simd_gt(a, b) +} + +/// Floating-point compare greater than +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcgt.f32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fcmgt))] +pub unsafe fn vcgtq_f32(a: float32x4_t, b: float32x4_t) -> uint32x4_t { + simd_gt(a, b) +} + +/// Compare signed less than +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcgt.s8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmgt))] +pub unsafe fn vclt_s8(a: int8x8_t, b: int8x8_t) -> uint8x8_t { + simd_lt(a, b) +} + +/// Compare signed less than +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcgt.s8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmgt))] +pub unsafe fn vcltq_s8(a: int8x16_t, b: int8x16_t) -> uint8x16_t { + simd_lt(a, b) +} + +/// Compare signed less than +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcgt.s16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmgt))] +pub unsafe fn vclt_s16(a: int16x4_t, b: int16x4_t) -> uint16x4_t { + simd_lt(a, b) +} + +/// Compare signed less than +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcgt.s16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmgt))] +pub unsafe fn vcltq_s16(a: int16x8_t, b: int16x8_t) -> uint16x8_t { + simd_lt(a, b) +} + +/// Compare signed less than +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcgt.s32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmgt))] +pub unsafe fn vclt_s32(a: int32x2_t, b: int32x2_t) -> uint32x2_t { + simd_lt(a, b) +} + +/// Compare signed less than +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcgt.s32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmgt))] +pub unsafe fn vcltq_s32(a: int32x4_t, b: int32x4_t) -> uint32x4_t { + simd_lt(a, b) +} + +/// Compare unsigned less than +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcgt.u8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmhi))] +pub unsafe fn vclt_u8(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t { + simd_lt(a, b) +} + +/// Compare unsigned less than +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcgt.u8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmhi))] +pub unsafe fn vcltq_u8(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t { + simd_lt(a, b) +} + +/// Compare unsigned less than +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcgt.u16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmhi))] +pub unsafe fn vclt_u16(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t { + simd_lt(a, b) +} + +/// Compare unsigned less than +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcgt.u16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmhi))] +pub unsafe fn vcltq_u16(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t { + simd_lt(a, b) +} + +/// Compare unsigned less than +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcgt.u32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmhi))] +pub unsafe fn vclt_u32(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t { + simd_lt(a, b) +} + +/// Compare unsigned less than +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcgt.u32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmhi))] +pub unsafe fn vcltq_u32(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t { + simd_lt(a, b) +} + +/// Floating-point compare less than +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcgt.f32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fcmgt))] +pub unsafe fn vclt_f32(a: float32x2_t, b: float32x2_t) -> uint32x2_t { + simd_lt(a, b) +} + +/// Floating-point compare less than +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcgt.f32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fcmgt))] +pub unsafe fn vcltq_f32(a: float32x4_t, b: float32x4_t) -> uint32x4_t { + simd_lt(a, b) +} + +/// Compare signed less than or equal +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcge.s8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmge))] +pub unsafe fn vcle_s8(a: int8x8_t, b: int8x8_t) -> uint8x8_t { + simd_le(a, b) +} + +/// Compare signed less than or equal +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcge.s8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmge))] +pub unsafe fn vcleq_s8(a: int8x16_t, b: int8x16_t) -> uint8x16_t { + simd_le(a, b) +} + +/// Compare signed less than or equal +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcge.s16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmge))] +pub unsafe fn vcle_s16(a: int16x4_t, b: int16x4_t) -> uint16x4_t { + simd_le(a, b) +} + +/// Compare signed less than or equal +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcge.s16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmge))] +pub unsafe fn vcleq_s16(a: int16x8_t, b: int16x8_t) -> uint16x8_t { + simd_le(a, b) +} + +/// Compare signed less than or equal +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcge.s32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmge))] +pub unsafe fn vcle_s32(a: int32x2_t, b: int32x2_t) -> uint32x2_t { + simd_le(a, b) +} + +/// Compare signed less than or equal +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcge.s32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmge))] +pub unsafe fn vcleq_s32(a: int32x4_t, b: int32x4_t) -> uint32x4_t { + simd_le(a, b) +} + +/// Compare unsigned less than or equal +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcge.u8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmhs))] +pub unsafe fn vcle_u8(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t { + simd_le(a, b) +} + +/// Compare unsigned less than or equal +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcge.u8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmhs))] +pub unsafe fn vcleq_u8(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t { + simd_le(a, b) +} + +/// Compare unsigned less than or equal +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcge.u16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmhs))] +pub unsafe fn vcle_u16(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t { + simd_le(a, b) +} + +/// Compare unsigned less than or equal +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcge.u16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmhs))] +pub unsafe fn vcleq_u16(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t { + simd_le(a, b) +} + +/// Compare unsigned less than or equal +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcge.u32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmhs))] +pub unsafe fn vcle_u32(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t { + simd_le(a, b) +} + +/// Compare unsigned less than or equal +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcge.u32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmhs))] +pub unsafe fn vcleq_u32(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t { + simd_le(a, b) +} + +/// Floating-point compare less than or equal +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcge.f32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fcmge))] +pub unsafe fn vcle_f32(a: float32x2_t, b: float32x2_t) -> uint32x2_t { + simd_le(a, b) +} + +/// Floating-point compare less than or equal +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcge.f32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fcmge))] +pub unsafe fn vcleq_f32(a: float32x4_t, b: float32x4_t) -> uint32x4_t { + simd_le(a, b) +} + +/// Compare signed greater than or equal +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcge.s8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmge))] +pub unsafe fn vcge_s8(a: int8x8_t, b: int8x8_t) -> uint8x8_t { + simd_ge(a, b) +} + +/// Compare signed greater than or equal +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcge.s8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmge))] +pub unsafe fn vcgeq_s8(a: int8x16_t, b: int8x16_t) -> uint8x16_t { + simd_ge(a, b) +} + +/// Compare signed greater than or equal +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcge.s16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmge))] +pub unsafe fn vcge_s16(a: int16x4_t, b: int16x4_t) -> uint16x4_t { + simd_ge(a, b) +} + +/// Compare signed greater than or equal +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcge.s16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmge))] +pub unsafe fn vcgeq_s16(a: int16x8_t, b: int16x8_t) -> uint16x8_t { + simd_ge(a, b) +} + +/// Compare signed greater than or equal +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcge.s32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmge))] +pub unsafe fn vcge_s32(a: int32x2_t, b: int32x2_t) -> uint32x2_t { + simd_ge(a, b) +} + +/// Compare signed greater than or equal +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcge.s32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmge))] +pub unsafe fn vcgeq_s32(a: int32x4_t, b: int32x4_t) -> uint32x4_t { + simd_ge(a, b) +} + +/// Compare unsigned greater than or equal +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcge.u8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmhs))] +pub unsafe fn vcge_u8(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t { + simd_ge(a, b) +} + +/// Compare unsigned greater than or equal +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcge.u8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmhs))] +pub unsafe fn vcgeq_u8(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t { + simd_ge(a, b) +} + +/// Compare unsigned greater than or equal +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcge.u16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmhs))] +pub unsafe fn vcge_u16(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t { + simd_ge(a, b) +} + +/// Compare unsigned greater than or equal +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcge.u16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmhs))] +pub unsafe fn vcgeq_u16(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t { + simd_ge(a, b) +} + +/// Compare unsigned greater than or equal +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcge.u32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmhs))] +pub unsafe fn vcge_u32(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t { + simd_ge(a, b) +} + +/// Compare unsigned greater than or equal +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcge.u32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cmhs))] +pub unsafe fn vcgeq_u32(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t { + simd_ge(a, b) +} + +/// Floating-point compare greater than or equal +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcge.f32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fcmge))] +pub unsafe fn vcge_f32(a: float32x2_t, b: float32x2_t) -> uint32x2_t { + simd_ge(a, b) +} + +/// Floating-point compare greater than or equal +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcge.f32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fcmge))] +pub unsafe fn vcgeq_f32(a: float32x4_t, b: float32x4_t) -> uint32x4_t { + simd_ge(a, b) +} + +/// Count leading sign bits +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcls.s8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cls))] +pub unsafe fn vcls_s8(a: int8x8_t) -> int8x8_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vcls.v8i8")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.cls.v8i8")] + fn vcls_s8_(a: int8x8_t) -> int8x8_t; + } +vcls_s8_(a) +} + +/// Count leading sign bits +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcls.s8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cls))] +pub unsafe fn vclsq_s8(a: int8x16_t) -> int8x16_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vcls.v16i8")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.cls.v16i8")] + fn vclsq_s8_(a: int8x16_t) -> int8x16_t; + } +vclsq_s8_(a) +} + +/// Count leading sign bits +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcls.s16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cls))] +pub unsafe fn vcls_s16(a: int16x4_t) -> int16x4_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vcls.v4i16")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.cls.v4i16")] + fn vcls_s16_(a: int16x4_t) -> int16x4_t; + } +vcls_s16_(a) +} + +/// Count leading sign bits +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcls.s16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cls))] +pub unsafe fn vclsq_s16(a: int16x8_t) -> int16x8_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vcls.v8i16")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.cls.v8i16")] + fn vclsq_s16_(a: int16x8_t) -> int16x8_t; + } +vclsq_s16_(a) +} + +/// Count leading sign bits +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcls.s32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cls))] +pub unsafe fn vcls_s32(a: int32x2_t) -> int32x2_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vcls.v2i32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.cls.v2i32")] + fn vcls_s32_(a: int32x2_t) -> int32x2_t; + } +vcls_s32_(a) +} + +/// Count leading sign bits +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vcls.s32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cls))] +pub unsafe fn vclsq_s32(a: int32x4_t) -> int32x4_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vcls.v4i32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.cls.v4i32")] + fn vclsq_s32_(a: int32x4_t) -> int32x4_t; + } +vclsq_s32_(a) +} + +/// Signed count leading sign bits +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vclz.i8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(clz))] +pub unsafe fn vclz_s8(a: int8x8_t) -> int8x8_t { + vclz_s8_(a) +} + +/// Signed count leading sign bits +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vclz.i8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(clz))] +pub unsafe fn vclzq_s8(a: int8x16_t) -> int8x16_t { + vclzq_s8_(a) +} + +/// Signed count leading sign bits +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vclz.i16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(clz))] +pub unsafe fn vclz_s16(a: int16x4_t) -> int16x4_t { + vclz_s16_(a) +} + +/// Signed count leading sign bits +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vclz.i16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(clz))] +pub unsafe fn vclzq_s16(a: int16x8_t) -> int16x8_t { + vclzq_s16_(a) +} + +/// Signed count leading sign bits +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vclz.i32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(clz))] +pub unsafe fn vclz_s32(a: int32x2_t) -> int32x2_t { + vclz_s32_(a) +} + +/// Signed count leading sign bits +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vclz.i32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(clz))] +pub unsafe fn vclzq_s32(a: int32x4_t) -> int32x4_t { + vclzq_s32_(a) +} + +/// Unsigned count leading sign bits +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vclz.i8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(clz))] +pub unsafe fn vclz_u8(a: uint8x8_t) -> uint8x8_t { + transmute(vclz_s8_(transmute(a))) +} + +/// Unsigned count leading sign bits +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vclz.i8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(clz))] +pub unsafe fn vclzq_u8(a: uint8x16_t) -> uint8x16_t { + transmute(vclzq_s8_(transmute(a))) +} + +/// Unsigned count leading sign bits +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vclz.i16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(clz))] +pub unsafe fn vclz_u16(a: uint16x4_t) -> uint16x4_t { + transmute(vclz_s16_(transmute(a))) +} + +/// Unsigned count leading sign bits +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vclz.i16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(clz))] +pub unsafe fn vclzq_u16(a: uint16x8_t) -> uint16x8_t { + transmute(vclzq_s16_(transmute(a))) +} + +/// Unsigned count leading sign bits +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vclz.i32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(clz))] +pub unsafe fn vclz_u32(a: uint32x2_t) -> uint32x2_t { + transmute(vclz_s32_(transmute(a))) +} + +/// Unsigned count leading sign bits +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vclz.i32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(clz))] +pub unsafe fn vclzq_u32(a: uint32x4_t) -> uint32x4_t { + transmute(vclzq_s32_(transmute(a))) +} + +/// Floating-point absolute compare greater than +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vacgt.f32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(facgt))] +pub unsafe fn vcagt_f32(a: float32x2_t, b: float32x2_t) -> uint32x2_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vacgt.v2i32.v2f32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.facgt.v2i32.v2f32")] + fn vcagt_f32_(a: float32x2_t, b: float32x2_t) -> uint32x2_t; + } +vcagt_f32_(a, b) +} + +/// Floating-point absolute compare greater than +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vacgt.f32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(facgt))] +pub unsafe fn vcagtq_f32(a: float32x4_t, b: float32x4_t) -> uint32x4_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vacgt.v4i32.v4f32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.facgt.v4i32.v4f32")] + fn vcagtq_f32_(a: float32x4_t, b: float32x4_t) -> uint32x4_t; + } +vcagtq_f32_(a, b) +} + +/// Floating-point absolute compare greater than or equal +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vacge.f32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(facge))] +pub unsafe fn vcage_f32(a: float32x2_t, b: float32x2_t) -> uint32x2_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vacge.v2i32.v2f32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.facge.v2i32.v2f32")] + fn vcage_f32_(a: float32x2_t, b: float32x2_t) -> uint32x2_t; + } +vcage_f32_(a, b) +} + +/// Floating-point absolute compare greater than or equal +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vacge.f32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(facge))] +pub unsafe fn vcageq_f32(a: float32x4_t, b: float32x4_t) -> uint32x4_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vacge.v4i32.v4f32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.facge.v4i32.v4f32")] + fn vcageq_f32_(a: float32x4_t, b: float32x4_t) -> uint32x4_t; + } +vcageq_f32_(a, b) +} + +/// Floating-point absolute compare less than +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vacgt.f32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(facgt))] +pub unsafe fn vcalt_f32(a: float32x2_t, b: float32x2_t) -> uint32x2_t { + vcagt_f32(b, a) +} + +/// Floating-point absolute compare less than +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vacgt.f32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(facgt))] +pub unsafe fn vcaltq_f32(a: float32x4_t, b: float32x4_t) -> uint32x4_t { + vcagtq_f32(b, a) +} + +/// Floating-point absolute compare less than or equal +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vacge.f32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(facge))] +pub unsafe fn vcale_f32(a: float32x2_t, b: float32x2_t) -> uint32x2_t { + vcage_f32(b, a) +} + +/// Floating-point absolute compare less than or equal +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vacge.f32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(facge))] +pub unsafe fn vcaleq_f32(a: float32x4_t, b: float32x4_t) -> uint32x4_t { + vcageq_f32(b, a) +} + +/// Insert vector element from another vector element +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +pub unsafe fn vcreate_s8(a: u64) -> int8x8_t { + transmute(a) +} + +/// Insert vector element from another vector element +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +pub unsafe fn vcreate_s32(a: u64) -> int32x2_t { + transmute(a) +} + +/// Insert vector element from another vector element +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +pub unsafe fn vcreate_s64(a: u64) -> int64x1_t { + transmute(a) +} + +/// Insert vector element from another vector element +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +pub unsafe fn vcreate_u8(a: u64) -> uint8x8_t { + transmute(a) +} + +/// Insert vector element from another vector element +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +pub unsafe fn vcreate_u32(a: u64) -> uint32x2_t { + transmute(a) +} + +/// Insert vector element from another vector element +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +pub unsafe fn vcreate_u64(a: u64) -> uint64x1_t { + transmute(a) +} + +/// Insert vector element from another vector element +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +pub unsafe fn vcreate_p8(a: u64) -> poly8x8_t { + transmute(a) +} + +/// Insert vector element from another vector element +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +pub unsafe fn vcreate_p16(a: u64) -> poly16x4_t { + transmute(a) +} + +/// Insert vector element from another vector element +#[inline] +#[target_feature(enable = "neon,crypto")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "crypto,v8"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +pub unsafe fn vcreate_p64(a: u64) -> poly64x1_t { + transmute(a) +} + +/// Insert vector element from another vector element +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop))] +pub unsafe fn vcreate_f32(a: u64) -> float32x2_t { + transmute(a) +} + +/// Fixed-point convert to floating-point +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vcvt))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(scvtf))] +pub unsafe fn vcvt_f32_s32(a: int32x2_t) -> float32x2_t { + simd_cast(a) +} + +/// Fixed-point convert to floating-point +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vcvt))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(scvtf))] +pub unsafe fn vcvtq_f32_s32(a: int32x4_t) -> float32x4_t { + simd_cast(a) +} + +/// Fixed-point convert to floating-point +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vcvt))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ucvtf))] +pub unsafe fn vcvt_f32_u32(a: uint32x2_t) -> float32x2_t { + simd_cast(a) +} + +/// Fixed-point convert to floating-point +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vcvt))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ucvtf))] +pub unsafe fn vcvtq_f32_u32(a: uint32x4_t) -> float32x4_t { + simd_cast(a) +} + +/// Fixed-point convert to floating-point +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vcvt, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vcvt_n_f32_s32(a: int32x2_t) -> float32x2_t { + static_assert!(N : i32 where N >= 1 && N <= 32); + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vcvtfxs2fp.v2f32.v2i32")] + fn vcvt_n_f32_s32_(a: int32x2_t, n: i32) -> float32x2_t; + } +vcvt_n_f32_s32_(a, N) +} + +/// Fixed-point convert to floating-point +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(scvtf, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vcvt_n_f32_s32(a: int32x2_t) -> float32x2_t { + static_assert!(N : i32 where N >= 1 && N <= 32); + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.vcvtfxs2fp.v2f32.v2i32")] + fn vcvt_n_f32_s32_(a: int32x2_t, n: i32) -> float32x2_t; + } +vcvt_n_f32_s32_(a, N) +} + +/// Fixed-point convert to floating-point +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vcvt, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vcvtq_n_f32_s32(a: int32x4_t) -> float32x4_t { + static_assert!(N : i32 where N >= 1 && N <= 32); + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vcvtfxs2fp.v4f32.v4i32")] + fn vcvtq_n_f32_s32_(a: int32x4_t, n: i32) -> float32x4_t; + } +vcvtq_n_f32_s32_(a, N) +} + +/// Fixed-point convert to floating-point +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(scvtf, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vcvtq_n_f32_s32(a: int32x4_t) -> float32x4_t { + static_assert!(N : i32 where N >= 1 && N <= 32); + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.vcvtfxs2fp.v4f32.v4i32")] + fn vcvtq_n_f32_s32_(a: int32x4_t, n: i32) -> float32x4_t; + } +vcvtq_n_f32_s32_(a, N) +} + +/// Fixed-point convert to floating-point +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vcvt, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vcvt_n_f32_u32(a: uint32x2_t) -> float32x2_t { + static_assert!(N : i32 where N >= 1 && N <= 32); + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vcvtfxu2fp.v2f32.v2i32")] + fn vcvt_n_f32_u32_(a: uint32x2_t, n: i32) -> float32x2_t; + } +vcvt_n_f32_u32_(a, N) +} + +/// Fixed-point convert to floating-point +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ucvtf, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vcvt_n_f32_u32(a: uint32x2_t) -> float32x2_t { + static_assert!(N : i32 where N >= 1 && N <= 32); + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.vcvtfxu2fp.v2f32.v2i32")] + fn vcvt_n_f32_u32_(a: uint32x2_t, n: i32) -> float32x2_t; + } +vcvt_n_f32_u32_(a, N) +} + +/// Fixed-point convert to floating-point +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vcvt, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vcvtq_n_f32_u32(a: uint32x4_t) -> float32x4_t { + static_assert!(N : i32 where N >= 1 && N <= 32); + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vcvtfxu2fp.v4f32.v4i32")] + fn vcvtq_n_f32_u32_(a: uint32x4_t, n: i32) -> float32x4_t; + } +vcvtq_n_f32_u32_(a, N) +} + +/// Fixed-point convert to floating-point +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ucvtf, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vcvtq_n_f32_u32(a: uint32x4_t) -> float32x4_t { + static_assert!(N : i32 where N >= 1 && N <= 32); + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.vcvtfxu2fp.v4f32.v4i32")] + fn vcvtq_n_f32_u32_(a: uint32x4_t, n: i32) -> float32x4_t; + } +vcvtq_n_f32_u32_(a, N) +} + +/// Floating-point convert to fixed-point, rounding toward zero +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vcvt, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vcvt_n_s32_f32(a: float32x2_t) -> int32x2_t { + static_assert!(N : i32 where N >= 1 && N <= 32); + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vcvtfp2fxs.v2i32.v2f32")] + fn vcvt_n_s32_f32_(a: float32x2_t, n: i32) -> int32x2_t; + } +vcvt_n_s32_f32_(a, N) +} + +/// Floating-point convert to fixed-point, rounding toward zero +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fcvtzs, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vcvt_n_s32_f32(a: float32x2_t) -> int32x2_t { + static_assert!(N : i32 where N >= 1 && N <= 32); + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.vcvtfp2fxs.v2i32.v2f32")] + fn vcvt_n_s32_f32_(a: float32x2_t, n: i32) -> int32x2_t; + } +vcvt_n_s32_f32_(a, N) +} + +/// Floating-point convert to fixed-point, rounding toward zero +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vcvt, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vcvtq_n_s32_f32(a: float32x4_t) -> int32x4_t { + static_assert!(N : i32 where N >= 1 && N <= 32); + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vcvtfp2fxs.v4i32.v4f32")] + fn vcvtq_n_s32_f32_(a: float32x4_t, n: i32) -> int32x4_t; + } +vcvtq_n_s32_f32_(a, N) +} + +/// Floating-point convert to fixed-point, rounding toward zero +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fcvtzs, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vcvtq_n_s32_f32(a: float32x4_t) -> int32x4_t { + static_assert!(N : i32 where N >= 1 && N <= 32); + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.vcvtfp2fxs.v4i32.v4f32")] + fn vcvtq_n_s32_f32_(a: float32x4_t, n: i32) -> int32x4_t; + } +vcvtq_n_s32_f32_(a, N) +} + +/// Floating-point convert to fixed-point, rounding toward zero +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vcvt, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vcvt_n_u32_f32(a: float32x2_t) -> uint32x2_t { + static_assert!(N : i32 where N >= 1 && N <= 32); + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vcvtfp2fxu.v2i32.v2f32")] + fn vcvt_n_u32_f32_(a: float32x2_t, n: i32) -> uint32x2_t; + } +vcvt_n_u32_f32_(a, N) +} + +/// Floating-point convert to fixed-point, rounding toward zero +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fcvtzu, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vcvt_n_u32_f32(a: float32x2_t) -> uint32x2_t { + static_assert!(N : i32 where N >= 1 && N <= 32); + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.vcvtfp2fxu.v2i32.v2f32")] + fn vcvt_n_u32_f32_(a: float32x2_t, n: i32) -> uint32x2_t; + } +vcvt_n_u32_f32_(a, N) +} + +/// Floating-point convert to fixed-point, rounding toward zero +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vcvt, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vcvtq_n_u32_f32(a: float32x4_t) -> uint32x4_t { + static_assert!(N : i32 where N >= 1 && N <= 32); + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vcvtfp2fxu.v4i32.v4f32")] + fn vcvtq_n_u32_f32_(a: float32x4_t, n: i32) -> uint32x4_t; + } +vcvtq_n_u32_f32_(a, N) +} + +/// Floating-point convert to fixed-point, rounding toward zero +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fcvtzu, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vcvtq_n_u32_f32(a: float32x4_t) -> uint32x4_t { + static_assert!(N : i32 where N >= 1 && N <= 32); + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.vcvtfp2fxu.v4i32.v4f32")] + fn vcvtq_n_u32_f32_(a: float32x4_t, n: i32) -> uint32x4_t; + } +vcvtq_n_u32_f32_(a, N) +} + +/// Floating-point convert to signed fixed-point, rounding toward zero +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vcvt))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fcvtzs))] +pub unsafe fn vcvt_s32_f32(a: float32x2_t) -> int32x2_t { + simd_cast(a) +} + +/// Floating-point convert to signed fixed-point, rounding toward zero +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vcvt))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fcvtzs))] +pub unsafe fn vcvtq_s32_f32(a: float32x4_t) -> int32x4_t { + simd_cast(a) +} + +/// Floating-point convert to unsigned fixed-point, rounding toward zero +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vcvt))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fcvtzu))] +pub unsafe fn vcvt_u32_f32(a: float32x2_t) -> uint32x2_t { + simd_cast(a) +} + +/// Floating-point convert to unsigned fixed-point, rounding toward zero +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vcvt))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fcvtzu))] +pub unsafe fn vcvtq_u32_f32(a: float32x4_t) -> uint32x4_t { + simd_cast(a) +} + +/// Set all vector lanes to the same value +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.8", N = 4))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup, N = 4))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vdup_lane_s8(a: int8x8_t) -> int8x8_t { + static_assert_imm3!(N); + simd_shuffle8!(a, a, [N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32]) +} + +/// Set all vector lanes to the same value +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.8", N = 8))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup, N = 8))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vdupq_laneq_s8(a: int8x16_t) -> int8x16_t { + static_assert_imm4!(N); + simd_shuffle16!(a, a, [N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32]) +} + +/// Set all vector lanes to the same value +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.16", N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vdup_lane_s16(a: int16x4_t) -> int16x4_t { + static_assert_imm2!(N); + simd_shuffle4!(a, a, [N as u32, N as u32, N as u32, N as u32]) +} + +/// Set all vector lanes to the same value +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.16", N = 4))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup, N = 4))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vdupq_laneq_s16(a: int16x8_t) -> int16x8_t { + static_assert_imm3!(N); + simd_shuffle8!(a, a, [N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32]) +} + +/// Set all vector lanes to the same value +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.32", N = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup, N = 1))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vdup_lane_s32(a: int32x2_t) -> int32x2_t { + static_assert_imm1!(N); + simd_shuffle2!(a, a, [N as u32, N as u32]) +} + +/// Set all vector lanes to the same value +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.32", N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vdupq_laneq_s32(a: int32x4_t) -> int32x4_t { + static_assert_imm2!(N); + simd_shuffle4!(a, a, [N as u32, N as u32, N as u32, N as u32]) +} + +/// Set all vector lanes to the same value +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.8", N = 8))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup, N = 8))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vdup_laneq_s8(a: int8x16_t) -> int8x8_t { + static_assert_imm4!(N); + simd_shuffle8!(a, a, [N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32]) +} + +/// Set all vector lanes to the same value +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.16", N = 4))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup, N = 4))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vdup_laneq_s16(a: int16x8_t) -> int16x4_t { + static_assert_imm3!(N); + simd_shuffle4!(a, a, [N as u32, N as u32, N as u32, N as u32]) +} + +/// Set all vector lanes to the same value +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.32", N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vdup_laneq_s32(a: int32x4_t) -> int32x2_t { + static_assert_imm2!(N); + simd_shuffle2!(a, a, [N as u32, N as u32]) +} + +/// Set all vector lanes to the same value +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.8", N = 4))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup, N = 4))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vdupq_lane_s8(a: int8x8_t) -> int8x16_t { + static_assert_imm3!(N); + simd_shuffle16!(a, a, [N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32]) +} + +/// Set all vector lanes to the same value +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.16", N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vdupq_lane_s16(a: int16x4_t) -> int16x8_t { + static_assert_imm2!(N); + simd_shuffle8!(a, a, [N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32]) +} + +/// Set all vector lanes to the same value +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.32", N = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup, N = 1))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vdupq_lane_s32(a: int32x2_t) -> int32x4_t { + static_assert_imm1!(N); + simd_shuffle4!(a, a, [N as u32, N as u32, N as u32, N as u32]) +} + +/// Set all vector lanes to the same value +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.8", N = 4))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup, N = 4))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vdup_lane_u8(a: uint8x8_t) -> uint8x8_t { + static_assert_imm3!(N); + simd_shuffle8!(a, a, [N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32]) +} + +/// Set all vector lanes to the same value +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.8", N = 8))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup, N = 8))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vdupq_laneq_u8(a: uint8x16_t) -> uint8x16_t { + static_assert_imm4!(N); + simd_shuffle16!(a, a, [N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32]) +} + +/// Set all vector lanes to the same value +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.16", N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vdup_lane_u16(a: uint16x4_t) -> uint16x4_t { + static_assert_imm2!(N); + simd_shuffle4!(a, a, [N as u32, N as u32, N as u32, N as u32]) +} + +/// Set all vector lanes to the same value +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.16", N = 4))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup, N = 4))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vdupq_laneq_u16(a: uint16x8_t) -> uint16x8_t { + static_assert_imm3!(N); + simd_shuffle8!(a, a, [N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32]) +} + +/// Set all vector lanes to the same value +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.32", N = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup, N = 1))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vdup_lane_u32(a: uint32x2_t) -> uint32x2_t { + static_assert_imm1!(N); + simd_shuffle2!(a, a, [N as u32, N as u32]) +} + +/// Set all vector lanes to the same value +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.32", N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vdupq_laneq_u32(a: uint32x4_t) -> uint32x4_t { + static_assert_imm2!(N); + simd_shuffle4!(a, a, [N as u32, N as u32, N as u32, N as u32]) +} + +/// Set all vector lanes to the same value +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.8", N = 8))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup, N = 8))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vdup_laneq_u8(a: uint8x16_t) -> uint8x8_t { + static_assert_imm4!(N); + simd_shuffle8!(a, a, [N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32]) +} + +/// Set all vector lanes to the same value +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.16", N = 4))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup, N = 4))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vdup_laneq_u16(a: uint16x8_t) -> uint16x4_t { + static_assert_imm3!(N); + simd_shuffle4!(a, a, [N as u32, N as u32, N as u32, N as u32]) +} + +/// Set all vector lanes to the same value +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.32", N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vdup_laneq_u32(a: uint32x4_t) -> uint32x2_t { + static_assert_imm2!(N); + simd_shuffle2!(a, a, [N as u32, N as u32]) +} + +/// Set all vector lanes to the same value +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.8", N = 4))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup, N = 4))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vdupq_lane_u8(a: uint8x8_t) -> uint8x16_t { + static_assert_imm3!(N); + simd_shuffle16!(a, a, [N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32]) +} + +/// Set all vector lanes to the same value +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.16", N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vdupq_lane_u16(a: uint16x4_t) -> uint16x8_t { + static_assert_imm2!(N); + simd_shuffle8!(a, a, [N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32]) +} + +/// Set all vector lanes to the same value +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.32", N = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup, N = 1))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vdupq_lane_u32(a: uint32x2_t) -> uint32x4_t { + static_assert_imm1!(N); + simd_shuffle4!(a, a, [N as u32, N as u32, N as u32, N as u32]) +} + +/// Set all vector lanes to the same value +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.8", N = 4))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup, N = 4))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vdup_lane_p8(a: poly8x8_t) -> poly8x8_t { + static_assert_imm3!(N); + simd_shuffle8!(a, a, [N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32]) +} + +/// Set all vector lanes to the same value +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.8", N = 8))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup, N = 8))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vdupq_laneq_p8(a: poly8x16_t) -> poly8x16_t { + static_assert_imm4!(N); + simd_shuffle16!(a, a, [N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32]) +} + +/// Set all vector lanes to the same value +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.16", N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vdup_lane_p16(a: poly16x4_t) -> poly16x4_t { + static_assert_imm2!(N); + simd_shuffle4!(a, a, [N as u32, N as u32, N as u32, N as u32]) +} + +/// Set all vector lanes to the same value +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.16", N = 4))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup, N = 4))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vdupq_laneq_p16(a: poly16x8_t) -> poly16x8_t { + static_assert_imm3!(N); + simd_shuffle8!(a, a, [N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32]) +} + +/// Set all vector lanes to the same value +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.8", N = 8))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup, N = 8))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vdup_laneq_p8(a: poly8x16_t) -> poly8x8_t { + static_assert_imm4!(N); + simd_shuffle8!(a, a, [N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32]) +} + +/// Set all vector lanes to the same value +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.16", N = 4))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup, N = 4))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vdup_laneq_p16(a: poly16x8_t) -> poly16x4_t { + static_assert_imm3!(N); + simd_shuffle4!(a, a, [N as u32, N as u32, N as u32, N as u32]) +} + +/// Set all vector lanes to the same value +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.8", N = 4))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup, N = 4))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vdupq_lane_p8(a: poly8x8_t) -> poly8x16_t { + static_assert_imm3!(N); + simd_shuffle16!(a, a, [N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32]) +} + +/// Set all vector lanes to the same value +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.16", N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vdupq_lane_p16(a: poly16x4_t) -> poly16x8_t { + static_assert_imm2!(N); + simd_shuffle8!(a, a, [N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32]) +} + +/// Set all vector lanes to the same value +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmov, N = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup, N = 1))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vdupq_laneq_s64(a: int64x2_t) -> int64x2_t { + static_assert_imm1!(N); + simd_shuffle2!(a, a, [N as u32, N as u32]) +} + +/// Set all vector lanes to the same value +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmov, N = 0))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup, N = 0))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vdupq_lane_s64(a: int64x1_t) -> int64x2_t { + static_assert!(N : i32 where N == 0); + simd_shuffle2!(a, a, [N as u32, N as u32]) +} + +/// Set all vector lanes to the same value +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmov, N = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup, N = 1))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vdupq_laneq_u64(a: uint64x2_t) -> uint64x2_t { + static_assert_imm1!(N); + simd_shuffle2!(a, a, [N as u32, N as u32]) +} + +/// Set all vector lanes to the same value +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmov, N = 0))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup, N = 0))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vdupq_lane_u64(a: uint64x1_t) -> uint64x2_t { + static_assert!(N : i32 where N == 0); + simd_shuffle2!(a, a, [N as u32, N as u32]) +} + +/// Set all vector lanes to the same value +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.32", N = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup, N = 1))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vdup_lane_f32(a: float32x2_t) -> float32x2_t { + static_assert_imm1!(N); + simd_shuffle2!(a, a, [N as u32, N as u32]) +} + +/// Set all vector lanes to the same value +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.32", N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vdupq_laneq_f32(a: float32x4_t) -> float32x4_t { + static_assert_imm2!(N); + simd_shuffle4!(a, a, [N as u32, N as u32, N as u32, N as u32]) +} + +/// Set all vector lanes to the same value +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.32", N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vdup_laneq_f32(a: float32x4_t) -> float32x2_t { + static_assert_imm2!(N); + simd_shuffle2!(a, a, [N as u32, N as u32]) +} + +/// Set all vector lanes to the same value +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.32", N = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup, N = 1))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vdupq_lane_f32(a: float32x2_t) -> float32x4_t { + static_assert_imm1!(N); + simd_shuffle4!(a, a, [N as u32, N as u32, N as u32, N as u32]) +} + +/// Set all vector lanes to the same value +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, N = 0))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop, N = 0))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vdup_lane_s64(a: int64x1_t) -> int64x1_t { + static_assert!(N : i32 where N == 0); + a +} + +/// Set all vector lanes to the same value +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, N = 0))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop, N = 0))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vdup_lane_u64(a: uint64x1_t) -> uint64x1_t { + static_assert!(N : i32 where N == 0); + a +} + +/// Set all vector lanes to the same value +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmov, N = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop, N = 1))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vdup_laneq_s64(a: int64x2_t) -> int64x1_t { + static_assert_imm1!(N); + transmute::(simd_extract(a, N as u32)) +} + +/// Set all vector lanes to the same value +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmov, N = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop, N = 1))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vdup_laneq_u64(a: uint64x2_t) -> uint64x1_t { + static_assert_imm1!(N); + transmute::(simd_extract(a, N as u32)) +} + +/// Extract vector from pair of vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vext.8", N = 4))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ext, N = 4))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vext_s8(a: int8x8_t, b: int8x8_t) -> int8x8_t { + static_assert_imm3!(N); + match N & 0b111 { + 0 => simd_shuffle8!(a, b, [0, 1, 2, 3, 4, 5, 6, 7]), + 1 => simd_shuffle8!(a, b, [1, 2, 3, 4, 5, 6, 7, 8]), + 2 => simd_shuffle8!(a, b, [2, 3, 4, 5, 6, 7, 8, 9]), + 3 => simd_shuffle8!(a, b, [3, 4, 5, 6, 7, 8, 9, 10]), + 4 => simd_shuffle8!(a, b, [4, 5, 6, 7, 8, 9, 10, 11]), + 5 => simd_shuffle8!(a, b, [5, 6, 7, 8, 9, 10, 11, 12]), + 6 => simd_shuffle8!(a, b, [6, 7, 8, 9, 10, 11, 12, 13]), + 7 => simd_shuffle8!(a, b, [7, 8, 9, 10, 11, 12, 13, 14]), + _ => unreachable_unchecked(), + } +} + +/// Extract vector from pair of vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vext.8", N = 8))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ext, N = 8))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vextq_s8(a: int8x16_t, b: int8x16_t) -> int8x16_t { + static_assert_imm4!(N); + match N & 0b1111 { + 0 => simd_shuffle16!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), + 1 => simd_shuffle16!(a, b, [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]), + 2 => simd_shuffle16!(a, b, [2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17]), + 3 => simd_shuffle16!(a, b, [3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18]), + 4 => simd_shuffle16!(a, b, [4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19]), + 5 => simd_shuffle16!(a, b, [5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20]), + 6 => simd_shuffle16!(a, b, [6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21]), + 7 => simd_shuffle16!(a, b, [7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22]), + 8 => simd_shuffle16!(a, b, [8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23]), + 9 => simd_shuffle16!(a, b, [9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24]), + 10 => simd_shuffle16!(a, b, [10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25]), + 11 => simd_shuffle16!(a, b, [11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26]), + 12 => simd_shuffle16!(a, b, [12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27]), + 13 => simd_shuffle16!(a, b, [13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28]), + 14 => simd_shuffle16!(a, b, [14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29]), + 15 => simd_shuffle16!(a, b, [15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30]), + _ => unreachable_unchecked(), + } +} + +/// Extract vector from pair of vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vext.8", N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ext, N = 2))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vext_s16(a: int16x4_t, b: int16x4_t) -> int16x4_t { + static_assert_imm2!(N); + match N & 0b11 { + 0 => simd_shuffle4!(a, b, [0, 1, 2, 3]), + 1 => simd_shuffle4!(a, b, [1, 2, 3, 4]), + 2 => simd_shuffle4!(a, b, [2, 3, 4, 5]), + 3 => simd_shuffle4!(a, b, [3, 4, 5, 6]), + _ => unreachable_unchecked(), + } +} + +/// Extract vector from pair of vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vext.8", N = 4))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ext, N = 4))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vextq_s16(a: int16x8_t, b: int16x8_t) -> int16x8_t { + static_assert_imm3!(N); + match N & 0b111 { + 0 => simd_shuffle8!(a, b, [0, 1, 2, 3, 4, 5, 6, 7]), + 1 => simd_shuffle8!(a, b, [1, 2, 3, 4, 5, 6, 7, 8]), + 2 => simd_shuffle8!(a, b, [2, 3, 4, 5, 6, 7, 8, 9]), + 3 => simd_shuffle8!(a, b, [3, 4, 5, 6, 7, 8, 9, 10]), + 4 => simd_shuffle8!(a, b, [4, 5, 6, 7, 8, 9, 10, 11]), + 5 => simd_shuffle8!(a, b, [5, 6, 7, 8, 9, 10, 11, 12]), + 6 => simd_shuffle8!(a, b, [6, 7, 8, 9, 10, 11, 12, 13]), + 7 => simd_shuffle8!(a, b, [7, 8, 9, 10, 11, 12, 13, 14]), + _ => unreachable_unchecked(), + } +} + +/// Extract vector from pair of vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vext.8", N = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ext, N = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vext_s32(a: int32x2_t, b: int32x2_t) -> int32x2_t { + static_assert_imm1!(N); + match N & 0b1 { + 0 => simd_shuffle2!(a, b, [0, 1]), + 1 => simd_shuffle2!(a, b, [1, 2]), + _ => unreachable_unchecked(), + } +} + +/// Extract vector from pair of vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vext.8", N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ext, N = 2))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vextq_s32(a: int32x4_t, b: int32x4_t) -> int32x4_t { + static_assert_imm2!(N); + match N & 0b11 { + 0 => simd_shuffle4!(a, b, [0, 1, 2, 3]), + 1 => simd_shuffle4!(a, b, [1, 2, 3, 4]), + 2 => simd_shuffle4!(a, b, [2, 3, 4, 5]), + 3 => simd_shuffle4!(a, b, [3, 4, 5, 6]), + _ => unreachable_unchecked(), + } +} + +/// Extract vector from pair of vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vext.8", N = 4))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ext, N = 4))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vext_u8(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t { + static_assert_imm3!(N); + match N & 0b111 { + 0 => simd_shuffle8!(a, b, [0, 1, 2, 3, 4, 5, 6, 7]), + 1 => simd_shuffle8!(a, b, [1, 2, 3, 4, 5, 6, 7, 8]), + 2 => simd_shuffle8!(a, b, [2, 3, 4, 5, 6, 7, 8, 9]), + 3 => simd_shuffle8!(a, b, [3, 4, 5, 6, 7, 8, 9, 10]), + 4 => simd_shuffle8!(a, b, [4, 5, 6, 7, 8, 9, 10, 11]), + 5 => simd_shuffle8!(a, b, [5, 6, 7, 8, 9, 10, 11, 12]), + 6 => simd_shuffle8!(a, b, [6, 7, 8, 9, 10, 11, 12, 13]), + 7 => simd_shuffle8!(a, b, [7, 8, 9, 10, 11, 12, 13, 14]), + _ => unreachable_unchecked(), + } +} + +/// Extract vector from pair of vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vext.8", N = 8))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ext, N = 8))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vextq_u8(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t { + static_assert_imm4!(N); + match N & 0b1111 { + 0 => simd_shuffle16!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), + 1 => simd_shuffle16!(a, b, [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]), + 2 => simd_shuffle16!(a, b, [2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17]), + 3 => simd_shuffle16!(a, b, [3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18]), + 4 => simd_shuffle16!(a, b, [4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19]), + 5 => simd_shuffle16!(a, b, [5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20]), + 6 => simd_shuffle16!(a, b, [6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21]), + 7 => simd_shuffle16!(a, b, [7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22]), + 8 => simd_shuffle16!(a, b, [8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23]), + 9 => simd_shuffle16!(a, b, [9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24]), + 10 => simd_shuffle16!(a, b, [10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25]), + 11 => simd_shuffle16!(a, b, [11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26]), + 12 => simd_shuffle16!(a, b, [12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27]), + 13 => simd_shuffle16!(a, b, [13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28]), + 14 => simd_shuffle16!(a, b, [14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29]), + 15 => simd_shuffle16!(a, b, [15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30]), + _ => unreachable_unchecked(), + } +} + +/// Extract vector from pair of vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vext.8", N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ext, N = 2))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vext_u16(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t { + static_assert_imm2!(N); + match N & 0b11 { + 0 => simd_shuffle4!(a, b, [0, 1, 2, 3]), + 1 => simd_shuffle4!(a, b, [1, 2, 3, 4]), + 2 => simd_shuffle4!(a, b, [2, 3, 4, 5]), + 3 => simd_shuffle4!(a, b, [3, 4, 5, 6]), + _ => unreachable_unchecked(), + } +} + +/// Extract vector from pair of vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vext.8", N = 4))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ext, N = 4))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vextq_u16(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t { + static_assert_imm3!(N); + match N & 0b111 { + 0 => simd_shuffle8!(a, b, [0, 1, 2, 3, 4, 5, 6, 7]), + 1 => simd_shuffle8!(a, b, [1, 2, 3, 4, 5, 6, 7, 8]), + 2 => simd_shuffle8!(a, b, [2, 3, 4, 5, 6, 7, 8, 9]), + 3 => simd_shuffle8!(a, b, [3, 4, 5, 6, 7, 8, 9, 10]), + 4 => simd_shuffle8!(a, b, [4, 5, 6, 7, 8, 9, 10, 11]), + 5 => simd_shuffle8!(a, b, [5, 6, 7, 8, 9, 10, 11, 12]), + 6 => simd_shuffle8!(a, b, [6, 7, 8, 9, 10, 11, 12, 13]), + 7 => simd_shuffle8!(a, b, [7, 8, 9, 10, 11, 12, 13, 14]), + _ => unreachable_unchecked(), + } +} + +/// Extract vector from pair of vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vext.8", N = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ext, N = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vext_u32(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t { + static_assert_imm1!(N); + match N & 0b1 { + 0 => simd_shuffle2!(a, b, [0, 1]), + 1 => simd_shuffle2!(a, b, [1, 2]), + _ => unreachable_unchecked(), + } +} + +/// Extract vector from pair of vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vext.8", N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ext, N = 2))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vextq_u32(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t { + static_assert_imm2!(N); + match N & 0b11 { + 0 => simd_shuffle4!(a, b, [0, 1, 2, 3]), + 1 => simd_shuffle4!(a, b, [1, 2, 3, 4]), + 2 => simd_shuffle4!(a, b, [2, 3, 4, 5]), + 3 => simd_shuffle4!(a, b, [3, 4, 5, 6]), + _ => unreachable_unchecked(), + } +} + +/// Extract vector from pair of vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vext.8", N = 4))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ext, N = 4))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vext_p8(a: poly8x8_t, b: poly8x8_t) -> poly8x8_t { + static_assert_imm3!(N); + match N & 0b111 { + 0 => simd_shuffle8!(a, b, [0, 1, 2, 3, 4, 5, 6, 7]), + 1 => simd_shuffle8!(a, b, [1, 2, 3, 4, 5, 6, 7, 8]), + 2 => simd_shuffle8!(a, b, [2, 3, 4, 5, 6, 7, 8, 9]), + 3 => simd_shuffle8!(a, b, [3, 4, 5, 6, 7, 8, 9, 10]), + 4 => simd_shuffle8!(a, b, [4, 5, 6, 7, 8, 9, 10, 11]), + 5 => simd_shuffle8!(a, b, [5, 6, 7, 8, 9, 10, 11, 12]), + 6 => simd_shuffle8!(a, b, [6, 7, 8, 9, 10, 11, 12, 13]), + 7 => simd_shuffle8!(a, b, [7, 8, 9, 10, 11, 12, 13, 14]), + _ => unreachable_unchecked(), + } +} + +/// Extract vector from pair of vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vext.8", N = 8))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ext, N = 8))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vextq_p8(a: poly8x16_t, b: poly8x16_t) -> poly8x16_t { + static_assert_imm4!(N); + match N & 0b1111 { + 0 => simd_shuffle16!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), + 1 => simd_shuffle16!(a, b, [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]), + 2 => simd_shuffle16!(a, b, [2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17]), + 3 => simd_shuffle16!(a, b, [3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18]), + 4 => simd_shuffle16!(a, b, [4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19]), + 5 => simd_shuffle16!(a, b, [5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20]), + 6 => simd_shuffle16!(a, b, [6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21]), + 7 => simd_shuffle16!(a, b, [7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22]), + 8 => simd_shuffle16!(a, b, [8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23]), + 9 => simd_shuffle16!(a, b, [9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24]), + 10 => simd_shuffle16!(a, b, [10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25]), + 11 => simd_shuffle16!(a, b, [11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26]), + 12 => simd_shuffle16!(a, b, [12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27]), + 13 => simd_shuffle16!(a, b, [13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28]), + 14 => simd_shuffle16!(a, b, [14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29]), + 15 => simd_shuffle16!(a, b, [15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30]), + _ => unreachable_unchecked(), + } +} + +/// Extract vector from pair of vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vext.8", N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ext, N = 2))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vext_p16(a: poly16x4_t, b: poly16x4_t) -> poly16x4_t { + static_assert_imm2!(N); + match N & 0b11 { + 0 => simd_shuffle4!(a, b, [0, 1, 2, 3]), + 1 => simd_shuffle4!(a, b, [1, 2, 3, 4]), + 2 => simd_shuffle4!(a, b, [2, 3, 4, 5]), + 3 => simd_shuffle4!(a, b, [3, 4, 5, 6]), + _ => unreachable_unchecked(), + } +} + +/// Extract vector from pair of vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vext.8", N = 4))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ext, N = 4))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vextq_p16(a: poly16x8_t, b: poly16x8_t) -> poly16x8_t { + static_assert_imm3!(N); + match N & 0b111 { + 0 => simd_shuffle8!(a, b, [0, 1, 2, 3, 4, 5, 6, 7]), + 1 => simd_shuffle8!(a, b, [1, 2, 3, 4, 5, 6, 7, 8]), + 2 => simd_shuffle8!(a, b, [2, 3, 4, 5, 6, 7, 8, 9]), + 3 => simd_shuffle8!(a, b, [3, 4, 5, 6, 7, 8, 9, 10]), + 4 => simd_shuffle8!(a, b, [4, 5, 6, 7, 8, 9, 10, 11]), + 5 => simd_shuffle8!(a, b, [5, 6, 7, 8, 9, 10, 11, 12]), + 6 => simd_shuffle8!(a, b, [6, 7, 8, 9, 10, 11, 12, 13]), + 7 => simd_shuffle8!(a, b, [7, 8, 9, 10, 11, 12, 13, 14]), + _ => unreachable_unchecked(), + } +} + +/// Extract vector from pair of vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmov, N = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ext, N = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vextq_s64(a: int64x2_t, b: int64x2_t) -> int64x2_t { + static_assert_imm1!(N); + match N & 0b1 { + 0 => simd_shuffle2!(a, b, [0, 1]), + 1 => simd_shuffle2!(a, b, [1, 2]), + _ => unreachable_unchecked(), + } +} + +/// Extract vector from pair of vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmov, N = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ext, N = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vextq_u64(a: uint64x2_t, b: uint64x2_t) -> uint64x2_t { + static_assert_imm1!(N); + match N & 0b1 { + 0 => simd_shuffle2!(a, b, [0, 1]), + 1 => simd_shuffle2!(a, b, [1, 2]), + _ => unreachable_unchecked(), + } +} + +/// Extract vector from pair of vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vext.8", N = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ext, N = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vext_f32(a: float32x2_t, b: float32x2_t) -> float32x2_t { + static_assert_imm1!(N); + match N & 0b1 { + 0 => simd_shuffle2!(a, b, [0, 1]), + 1 => simd_shuffle2!(a, b, [1, 2]), + _ => unreachable_unchecked(), + } +} + +/// Extract vector from pair of vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vext.8", N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ext, N = 2))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vextq_f32(a: float32x4_t, b: float32x4_t) -> float32x4_t { + static_assert_imm2!(N); + match N & 0b11 { + 0 => simd_shuffle4!(a, b, [0, 1, 2, 3]), + 1 => simd_shuffle4!(a, b, [1, 2, 3, 4]), + 2 => simd_shuffle4!(a, b, [2, 3, 4, 5]), + 3 => simd_shuffle4!(a, b, [3, 4, 5, 6]), + _ => unreachable_unchecked(), + } +} + +/// Multiply-add to accumulator +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmla.i8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mla))] +pub unsafe fn vmla_s8(a: int8x8_t, b: int8x8_t, c: int8x8_t) -> int8x8_t { + simd_add(a, simd_mul(b, c)) +} + +/// Multiply-add to accumulator +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmla.i8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mla))] +pub unsafe fn vmlaq_s8(a: int8x16_t, b: int8x16_t, c: int8x16_t) -> int8x16_t { + simd_add(a, simd_mul(b, c)) +} + +/// Multiply-add to accumulator +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmla.i16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mla))] +pub unsafe fn vmla_s16(a: int16x4_t, b: int16x4_t, c: int16x4_t) -> int16x4_t { + simd_add(a, simd_mul(b, c)) +} + +/// Multiply-add to accumulator +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmla.i16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mla))] +pub unsafe fn vmlaq_s16(a: int16x8_t, b: int16x8_t, c: int16x8_t) -> int16x8_t { + simd_add(a, simd_mul(b, c)) +} + +/// Multiply-add to accumulator +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmla.i32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mla))] +pub unsafe fn vmla_s32(a: int32x2_t, b: int32x2_t, c: int32x2_t) -> int32x2_t { + simd_add(a, simd_mul(b, c)) +} + +/// Multiply-add to accumulator +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmla.i32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mla))] +pub unsafe fn vmlaq_s32(a: int32x4_t, b: int32x4_t, c: int32x4_t) -> int32x4_t { + simd_add(a, simd_mul(b, c)) +} + +/// Multiply-add to accumulator +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmla.i8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mla))] +pub unsafe fn vmla_u8(a: uint8x8_t, b: uint8x8_t, c: uint8x8_t) -> uint8x8_t { + simd_add(a, simd_mul(b, c)) +} + +/// Multiply-add to accumulator +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmla.i8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mla))] +pub unsafe fn vmlaq_u8(a: uint8x16_t, b: uint8x16_t, c: uint8x16_t) -> uint8x16_t { + simd_add(a, simd_mul(b, c)) +} + +/// Multiply-add to accumulator +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmla.i16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mla))] +pub unsafe fn vmla_u16(a: uint16x4_t, b: uint16x4_t, c: uint16x4_t) -> uint16x4_t { + simd_add(a, simd_mul(b, c)) +} + +/// Multiply-add to accumulator +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmla.i16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mla))] +pub unsafe fn vmlaq_u16(a: uint16x8_t, b: uint16x8_t, c: uint16x8_t) -> uint16x8_t { + simd_add(a, simd_mul(b, c)) +} + +/// Multiply-add to accumulator +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmla.i32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mla))] +pub unsafe fn vmla_u32(a: uint32x2_t, b: uint32x2_t, c: uint32x2_t) -> uint32x2_t { + simd_add(a, simd_mul(b, c)) +} + +/// Multiply-add to accumulator +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmla.i32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mla))] +pub unsafe fn vmlaq_u32(a: uint32x4_t, b: uint32x4_t, c: uint32x4_t) -> uint32x4_t { + simd_add(a, simd_mul(b, c)) +} + +/// Floating-point multiply-add to accumulator +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmla.f32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fmul))] +pub unsafe fn vmla_f32(a: float32x2_t, b: float32x2_t, c: float32x2_t) -> float32x2_t { + simd_add(a, simd_mul(b, c)) +} + +/// Floating-point multiply-add to accumulator +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmla.f32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fmul))] +pub unsafe fn vmlaq_f32(a: float32x4_t, b: float32x4_t, c: float32x4_t) -> float32x4_t { + simd_add(a, simd_mul(b, c)) +} + +/// Vector multiply accumulate with scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmla.i16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mla))] +pub unsafe fn vmla_n_s16(a: int16x4_t, b: int16x4_t, c: i16) -> int16x4_t { + vmla_s16(a, b, vdup_n_s16(c)) +} + +/// Vector multiply accumulate with scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmla.i16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mla))] +pub unsafe fn vmlaq_n_s16(a: int16x8_t, b: int16x8_t, c: i16) -> int16x8_t { + vmlaq_s16(a, b, vdupq_n_s16(c)) +} + +/// Vector multiply accumulate with scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmla.i32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mla))] +pub unsafe fn vmla_n_s32(a: int32x2_t, b: int32x2_t, c: i32) -> int32x2_t { + vmla_s32(a, b, vdup_n_s32(c)) +} + +/// Vector multiply accumulate with scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmla.i32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mla))] +pub unsafe fn vmlaq_n_s32(a: int32x4_t, b: int32x4_t, c: i32) -> int32x4_t { + vmlaq_s32(a, b, vdupq_n_s32(c)) +} + +/// Vector multiply accumulate with scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmla.i16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mla))] +pub unsafe fn vmla_n_u16(a: uint16x4_t, b: uint16x4_t, c: u16) -> uint16x4_t { + vmla_u16(a, b, vdup_n_u16(c)) +} + +/// Vector multiply accumulate with scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmla.i16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mla))] +pub unsafe fn vmlaq_n_u16(a: uint16x8_t, b: uint16x8_t, c: u16) -> uint16x8_t { + vmlaq_u16(a, b, vdupq_n_u16(c)) +} + +/// Vector multiply accumulate with scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmla.i32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mla))] +pub unsafe fn vmla_n_u32(a: uint32x2_t, b: uint32x2_t, c: u32) -> uint32x2_t { + vmla_u32(a, b, vdup_n_u32(c)) +} + +/// Vector multiply accumulate with scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmla.i32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mla))] +pub unsafe fn vmlaq_n_u32(a: uint32x4_t, b: uint32x4_t, c: u32) -> uint32x4_t { + vmlaq_u32(a, b, vdupq_n_u32(c)) +} + +/// Vector multiply accumulate with scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmla.f32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fmul))] +pub unsafe fn vmla_n_f32(a: float32x2_t, b: float32x2_t, c: f32) -> float32x2_t { + vmla_f32(a, b, vdup_n_f32(c)) +} + +/// Vector multiply accumulate with scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmla.f32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fmul))] +pub unsafe fn vmlaq_n_f32(a: float32x4_t, b: float32x4_t, c: f32) -> float32x4_t { + vmlaq_f32(a, b, vdupq_n_f32(c)) +} + +/// Vector multiply accumulate with scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmla.i16", LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mla, LANE = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vmla_lane_s16(a: int16x4_t, b: int16x4_t, c: int16x4_t) -> int16x4_t { + static_assert_imm2!(LANE); + vmla_s16(a, b, simd_shuffle4!(c, c, [LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Vector multiply accumulate with scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmla.i16", LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mla, LANE = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vmla_laneq_s16(a: int16x4_t, b: int16x4_t, c: int16x8_t) -> int16x4_t { + static_assert_imm3!(LANE); + vmla_s16(a, b, simd_shuffle4!(c, c, [LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Vector multiply accumulate with scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmla.i16", LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mla, LANE = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vmlaq_lane_s16(a: int16x8_t, b: int16x8_t, c: int16x4_t) -> int16x8_t { + static_assert_imm2!(LANE); + vmlaq_s16(a, b, simd_shuffle8!(c, c, [LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Vector multiply accumulate with scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmla.i16", LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mla, LANE = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vmlaq_laneq_s16(a: int16x8_t, b: int16x8_t, c: int16x8_t) -> int16x8_t { + static_assert_imm3!(LANE); + vmlaq_s16(a, b, simd_shuffle8!(c, c, [LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Vector multiply accumulate with scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmla.i32", LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mla, LANE = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vmla_lane_s32(a: int32x2_t, b: int32x2_t, c: int32x2_t) -> int32x2_t { + static_assert_imm1!(LANE); + vmla_s32(a, b, simd_shuffle2!(c, c, [LANE as u32, LANE as u32])) +} + +/// Vector multiply accumulate with scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmla.i32", LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mla, LANE = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vmla_laneq_s32(a: int32x2_t, b: int32x2_t, c: int32x4_t) -> int32x2_t { + static_assert_imm2!(LANE); + vmla_s32(a, b, simd_shuffle2!(c, c, [LANE as u32, LANE as u32])) +} + +/// Vector multiply accumulate with scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmla.i32", LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mla, LANE = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vmlaq_lane_s32(a: int32x4_t, b: int32x4_t, c: int32x2_t) -> int32x4_t { + static_assert_imm1!(LANE); + vmlaq_s32(a, b, simd_shuffle4!(c, c, [LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Vector multiply accumulate with scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmla.i32", LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mla, LANE = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vmlaq_laneq_s32(a: int32x4_t, b: int32x4_t, c: int32x4_t) -> int32x4_t { + static_assert_imm2!(LANE); + vmlaq_s32(a, b, simd_shuffle4!(c, c, [LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Vector multiply accumulate with scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmla.i16", LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mla, LANE = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vmla_lane_u16(a: uint16x4_t, b: uint16x4_t, c: uint16x4_t) -> uint16x4_t { + static_assert_imm2!(LANE); + vmla_u16(a, b, simd_shuffle4!(c, c, [LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Vector multiply accumulate with scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmla.i16", LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mla, LANE = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vmla_laneq_u16(a: uint16x4_t, b: uint16x4_t, c: uint16x8_t) -> uint16x4_t { + static_assert_imm3!(LANE); + vmla_u16(a, b, simd_shuffle4!(c, c, [LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Vector multiply accumulate with scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmla.i16", LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mla, LANE = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vmlaq_lane_u16(a: uint16x8_t, b: uint16x8_t, c: uint16x4_t) -> uint16x8_t { + static_assert_imm2!(LANE); + vmlaq_u16(a, b, simd_shuffle8!(c, c, [LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Vector multiply accumulate with scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmla.i16", LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mla, LANE = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vmlaq_laneq_u16(a: uint16x8_t, b: uint16x8_t, c: uint16x8_t) -> uint16x8_t { + static_assert_imm3!(LANE); + vmlaq_u16(a, b, simd_shuffle8!(c, c, [LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Vector multiply accumulate with scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmla.i32", LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mla, LANE = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vmla_lane_u32(a: uint32x2_t, b: uint32x2_t, c: uint32x2_t) -> uint32x2_t { + static_assert_imm1!(LANE); + vmla_u32(a, b, simd_shuffle2!(c, c, [LANE as u32, LANE as u32])) +} + +/// Vector multiply accumulate with scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmla.i32", LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mla, LANE = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vmla_laneq_u32(a: uint32x2_t, b: uint32x2_t, c: uint32x4_t) -> uint32x2_t { + static_assert_imm2!(LANE); + vmla_u32(a, b, simd_shuffle2!(c, c, [LANE as u32, LANE as u32])) +} + +/// Vector multiply accumulate with scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmla.i32", LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mla, LANE = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vmlaq_lane_u32(a: uint32x4_t, b: uint32x4_t, c: uint32x2_t) -> uint32x4_t { + static_assert_imm1!(LANE); + vmlaq_u32(a, b, simd_shuffle4!(c, c, [LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Vector multiply accumulate with scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmla.i32", LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mla, LANE = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vmlaq_laneq_u32(a: uint32x4_t, b: uint32x4_t, c: uint32x4_t) -> uint32x4_t { + static_assert_imm2!(LANE); + vmlaq_u32(a, b, simd_shuffle4!(c, c, [LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Vector multiply accumulate with scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmla.f32", LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fmul, LANE = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vmla_lane_f32(a: float32x2_t, b: float32x2_t, c: float32x2_t) -> float32x2_t { + static_assert_imm1!(LANE); + vmla_f32(a, b, simd_shuffle2!(c, c, [LANE as u32, LANE as u32])) +} + +/// Vector multiply accumulate with scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmla.f32", LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fmul, LANE = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vmla_laneq_f32(a: float32x2_t, b: float32x2_t, c: float32x4_t) -> float32x2_t { + static_assert_imm2!(LANE); + vmla_f32(a, b, simd_shuffle2!(c, c, [LANE as u32, LANE as u32])) +} + +/// Vector multiply accumulate with scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmla.f32", LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fmul, LANE = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vmlaq_lane_f32(a: float32x4_t, b: float32x4_t, c: float32x2_t) -> float32x4_t { + static_assert_imm1!(LANE); + vmlaq_f32(a, b, simd_shuffle4!(c, c, [LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Vector multiply accumulate with scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmla.f32", LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fmul, LANE = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vmlaq_laneq_f32(a: float32x4_t, b: float32x4_t, c: float32x4_t) -> float32x4_t { + static_assert_imm2!(LANE); + vmlaq_f32(a, b, simd_shuffle4!(c, c, [LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Signed multiply-add long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmlal.s8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(smlal))] +pub unsafe fn vmlal_s8(a: int16x8_t, b: int8x8_t, c: int8x8_t) -> int16x8_t { + simd_add(a, vmull_s8(b, c)) +} + +/// Signed multiply-add long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmlal.s16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(smlal))] +pub unsafe fn vmlal_s16(a: int32x4_t, b: int16x4_t, c: int16x4_t) -> int32x4_t { + simd_add(a, vmull_s16(b, c)) +} + +/// Signed multiply-add long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmlal.s32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(smlal))] +pub unsafe fn vmlal_s32(a: int64x2_t, b: int32x2_t, c: int32x2_t) -> int64x2_t { + simd_add(a, vmull_s32(b, c)) +} + +/// Unsigned multiply-add long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmlal.u8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(umlal))] +pub unsafe fn vmlal_u8(a: uint16x8_t, b: uint8x8_t, c: uint8x8_t) -> uint16x8_t { + simd_add(a, vmull_u8(b, c)) +} + +/// Unsigned multiply-add long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmlal.u16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(umlal))] +pub unsafe fn vmlal_u16(a: uint32x4_t, b: uint16x4_t, c: uint16x4_t) -> uint32x4_t { + simd_add(a, vmull_u16(b, c)) +} + +/// Unsigned multiply-add long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmlal.u32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(umlal))] +pub unsafe fn vmlal_u32(a: uint64x2_t, b: uint32x2_t, c: uint32x2_t) -> uint64x2_t { + simd_add(a, vmull_u32(b, c)) +} + +/// Vector widening multiply accumulate with scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmlal.s16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(smlal))] +pub unsafe fn vmlal_n_s16(a: int32x4_t, b: int16x4_t, c: i16) -> int32x4_t { + vmlal_s16(a, b, vdup_n_s16(c)) +} + +/// Vector widening multiply accumulate with scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmlal.s32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(smlal))] +pub unsafe fn vmlal_n_s32(a: int64x2_t, b: int32x2_t, c: i32) -> int64x2_t { + vmlal_s32(a, b, vdup_n_s32(c)) +} + +/// Vector widening multiply accumulate with scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmlal.u16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(umlal))] +pub unsafe fn vmlal_n_u16(a: uint32x4_t, b: uint16x4_t, c: u16) -> uint32x4_t { + vmlal_u16(a, b, vdup_n_u16(c)) +} + +/// Vector widening multiply accumulate with scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmlal.u32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(umlal))] +pub unsafe fn vmlal_n_u32(a: uint64x2_t, b: uint32x2_t, c: u32) -> uint64x2_t { + vmlal_u32(a, b, vdup_n_u32(c)) +} + +/// Vector widening multiply accumulate with scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmlal.s16", LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(smlal, LANE = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vmlal_lane_s16(a: int32x4_t, b: int16x4_t, c: int16x4_t) -> int32x4_t { + static_assert_imm2!(LANE); + vmlal_s16(a, b, simd_shuffle4!(c, c, [LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Vector widening multiply accumulate with scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmlal.s16", LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(smlal, LANE = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vmlal_laneq_s16(a: int32x4_t, b: int16x4_t, c: int16x8_t) -> int32x4_t { + static_assert_imm3!(LANE); + vmlal_s16(a, b, simd_shuffle4!(c, c, [LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Vector widening multiply accumulate with scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmlal.s32", LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(smlal, LANE = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vmlal_lane_s32(a: int64x2_t, b: int32x2_t, c: int32x2_t) -> int64x2_t { + static_assert_imm1!(LANE); + vmlal_s32(a, b, simd_shuffle2!(c, c, [LANE as u32, LANE as u32])) +} + +/// Vector widening multiply accumulate with scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmlal.s32", LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(smlal, LANE = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vmlal_laneq_s32(a: int64x2_t, b: int32x2_t, c: int32x4_t) -> int64x2_t { + static_assert_imm2!(LANE); + vmlal_s32(a, b, simd_shuffle2!(c, c, [LANE as u32, LANE as u32])) +} + +/// Vector widening multiply accumulate with scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmlal.u16", LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(umlal, LANE = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vmlal_lane_u16(a: uint32x4_t, b: uint16x4_t, c: uint16x4_t) -> uint32x4_t { + static_assert_imm2!(LANE); + vmlal_u16(a, b, simd_shuffle4!(c, c, [LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Vector widening multiply accumulate with scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmlal.u16", LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(umlal, LANE = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vmlal_laneq_u16(a: uint32x4_t, b: uint16x4_t, c: uint16x8_t) -> uint32x4_t { + static_assert_imm3!(LANE); + vmlal_u16(a, b, simd_shuffle4!(c, c, [LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Vector widening multiply accumulate with scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmlal.u32", LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(umlal, LANE = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vmlal_lane_u32(a: uint64x2_t, b: uint32x2_t, c: uint32x2_t) -> uint64x2_t { + static_assert_imm1!(LANE); + vmlal_u32(a, b, simd_shuffle2!(c, c, [LANE as u32, LANE as u32])) +} + +/// Vector widening multiply accumulate with scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmlal.u32", LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(umlal, LANE = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vmlal_laneq_u32(a: uint64x2_t, b: uint32x2_t, c: uint32x4_t) -> uint64x2_t { + static_assert_imm2!(LANE); + vmlal_u32(a, b, simd_shuffle2!(c, c, [LANE as u32, LANE as u32])) +} + +/// Multiply-subtract from accumulator +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmls.i8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mls))] +pub unsafe fn vmls_s8(a: int8x8_t, b: int8x8_t, c: int8x8_t) -> int8x8_t { + simd_sub(a, simd_mul(b, c)) +} + +/// Multiply-subtract from accumulator +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmls.i8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mls))] +pub unsafe fn vmlsq_s8(a: int8x16_t, b: int8x16_t, c: int8x16_t) -> int8x16_t { + simd_sub(a, simd_mul(b, c)) +} + +/// Multiply-subtract from accumulator +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmls.i16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mls))] +pub unsafe fn vmls_s16(a: int16x4_t, b: int16x4_t, c: int16x4_t) -> int16x4_t { + simd_sub(a, simd_mul(b, c)) +} + +/// Multiply-subtract from accumulator +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmls.i16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mls))] +pub unsafe fn vmlsq_s16(a: int16x8_t, b: int16x8_t, c: int16x8_t) -> int16x8_t { + simd_sub(a, simd_mul(b, c)) +} + +/// Multiply-subtract from accumulator +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmls.i32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mls))] +pub unsafe fn vmls_s32(a: int32x2_t, b: int32x2_t, c: int32x2_t) -> int32x2_t { + simd_sub(a, simd_mul(b, c)) +} + +/// Multiply-subtract from accumulator +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmls.i32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mls))] +pub unsafe fn vmlsq_s32(a: int32x4_t, b: int32x4_t, c: int32x4_t) -> int32x4_t { + simd_sub(a, simd_mul(b, c)) +} + +/// Multiply-subtract from accumulator +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmls.i8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mls))] +pub unsafe fn vmls_u8(a: uint8x8_t, b: uint8x8_t, c: uint8x8_t) -> uint8x8_t { + simd_sub(a, simd_mul(b, c)) +} + +/// Multiply-subtract from accumulator +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmls.i8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mls))] +pub unsafe fn vmlsq_u8(a: uint8x16_t, b: uint8x16_t, c: uint8x16_t) -> uint8x16_t { + simd_sub(a, simd_mul(b, c)) +} + +/// Multiply-subtract from accumulator +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmls.i16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mls))] +pub unsafe fn vmls_u16(a: uint16x4_t, b: uint16x4_t, c: uint16x4_t) -> uint16x4_t { + simd_sub(a, simd_mul(b, c)) +} + +/// Multiply-subtract from accumulator +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmls.i16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mls))] +pub unsafe fn vmlsq_u16(a: uint16x8_t, b: uint16x8_t, c: uint16x8_t) -> uint16x8_t { + simd_sub(a, simd_mul(b, c)) +} + +/// Multiply-subtract from accumulator +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmls.i32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mls))] +pub unsafe fn vmls_u32(a: uint32x2_t, b: uint32x2_t, c: uint32x2_t) -> uint32x2_t { + simd_sub(a, simd_mul(b, c)) +} + +/// Multiply-subtract from accumulator +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmls.i32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mls))] +pub unsafe fn vmlsq_u32(a: uint32x4_t, b: uint32x4_t, c: uint32x4_t) -> uint32x4_t { + simd_sub(a, simd_mul(b, c)) +} + +/// Floating-point multiply-subtract from accumulator +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmls.f32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fmul))] +pub unsafe fn vmls_f32(a: float32x2_t, b: float32x2_t, c: float32x2_t) -> float32x2_t { + simd_sub(a, simd_mul(b, c)) +} + +/// Floating-point multiply-subtract from accumulator +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmls.f32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fmul))] +pub unsafe fn vmlsq_f32(a: float32x4_t, b: float32x4_t, c: float32x4_t) -> float32x4_t { + simd_sub(a, simd_mul(b, c)) +} + +/// Vector multiply subtract with scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmls.i16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mls))] +pub unsafe fn vmls_n_s16(a: int16x4_t, b: int16x4_t, c: i16) -> int16x4_t { + vmls_s16(a, b, vdup_n_s16(c)) +} + +/// Vector multiply subtract with scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmls.i16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mls))] +pub unsafe fn vmlsq_n_s16(a: int16x8_t, b: int16x8_t, c: i16) -> int16x8_t { + vmlsq_s16(a, b, vdupq_n_s16(c)) +} + +/// Vector multiply subtract with scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmls.i32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mls))] +pub unsafe fn vmls_n_s32(a: int32x2_t, b: int32x2_t, c: i32) -> int32x2_t { + vmls_s32(a, b, vdup_n_s32(c)) +} + +/// Vector multiply subtract with scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmls.i32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mls))] +pub unsafe fn vmlsq_n_s32(a: int32x4_t, b: int32x4_t, c: i32) -> int32x4_t { + vmlsq_s32(a, b, vdupq_n_s32(c)) +} + +/// Vector multiply subtract with scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmls.i16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mls))] +pub unsafe fn vmls_n_u16(a: uint16x4_t, b: uint16x4_t, c: u16) -> uint16x4_t { + vmls_u16(a, b, vdup_n_u16(c)) +} + +/// Vector multiply subtract with scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmls.i16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mls))] +pub unsafe fn vmlsq_n_u16(a: uint16x8_t, b: uint16x8_t, c: u16) -> uint16x8_t { + vmlsq_u16(a, b, vdupq_n_u16(c)) +} + +/// Vector multiply subtract with scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmls.i32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mls))] +pub unsafe fn vmls_n_u32(a: uint32x2_t, b: uint32x2_t, c: u32) -> uint32x2_t { + vmls_u32(a, b, vdup_n_u32(c)) +} + +/// Vector multiply subtract with scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmls.i32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mls))] +pub unsafe fn vmlsq_n_u32(a: uint32x4_t, b: uint32x4_t, c: u32) -> uint32x4_t { + vmlsq_u32(a, b, vdupq_n_u32(c)) +} + +/// Vector multiply subtract with scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmls.f32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fmul))] +pub unsafe fn vmls_n_f32(a: float32x2_t, b: float32x2_t, c: f32) -> float32x2_t { + vmls_f32(a, b, vdup_n_f32(c)) +} + +/// Vector multiply subtract with scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmls.f32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fmul))] +pub unsafe fn vmlsq_n_f32(a: float32x4_t, b: float32x4_t, c: f32) -> float32x4_t { + vmlsq_f32(a, b, vdupq_n_f32(c)) +} + +/// Vector multiply subtract with scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmls.i16", LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mls, LANE = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vmls_lane_s16(a: int16x4_t, b: int16x4_t, c: int16x4_t) -> int16x4_t { + static_assert_imm2!(LANE); + vmls_s16(a, b, simd_shuffle4!(c, c, [LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Vector multiply subtract with scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmls.i16", LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mls, LANE = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vmls_laneq_s16(a: int16x4_t, b: int16x4_t, c: int16x8_t) -> int16x4_t { + static_assert_imm3!(LANE); + vmls_s16(a, b, simd_shuffle4!(c, c, [LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Vector multiply subtract with scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmls.i16", LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mls, LANE = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vmlsq_lane_s16(a: int16x8_t, b: int16x8_t, c: int16x4_t) -> int16x8_t { + static_assert_imm2!(LANE); + vmlsq_s16(a, b, simd_shuffle8!(c, c, [LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Vector multiply subtract with scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmls.i16", LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mls, LANE = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vmlsq_laneq_s16(a: int16x8_t, b: int16x8_t, c: int16x8_t) -> int16x8_t { + static_assert_imm3!(LANE); + vmlsq_s16(a, b, simd_shuffle8!(c, c, [LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Vector multiply subtract with scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmls.i32", LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mls, LANE = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vmls_lane_s32(a: int32x2_t, b: int32x2_t, c: int32x2_t) -> int32x2_t { + static_assert_imm1!(LANE); + vmls_s32(a, b, simd_shuffle2!(c, c, [LANE as u32, LANE as u32])) +} + +/// Vector multiply subtract with scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmls.i32", LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mls, LANE = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vmls_laneq_s32(a: int32x2_t, b: int32x2_t, c: int32x4_t) -> int32x2_t { + static_assert_imm2!(LANE); + vmls_s32(a, b, simd_shuffle2!(c, c, [LANE as u32, LANE as u32])) +} + +/// Vector multiply subtract with scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmls.i32", LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mls, LANE = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vmlsq_lane_s32(a: int32x4_t, b: int32x4_t, c: int32x2_t) -> int32x4_t { + static_assert_imm1!(LANE); + vmlsq_s32(a, b, simd_shuffle4!(c, c, [LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Vector multiply subtract with scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmls.i32", LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mls, LANE = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vmlsq_laneq_s32(a: int32x4_t, b: int32x4_t, c: int32x4_t) -> int32x4_t { + static_assert_imm2!(LANE); + vmlsq_s32(a, b, simd_shuffle4!(c, c, [LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Vector multiply subtract with scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmls.i16", LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mls, LANE = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vmls_lane_u16(a: uint16x4_t, b: uint16x4_t, c: uint16x4_t) -> uint16x4_t { + static_assert_imm2!(LANE); + vmls_u16(a, b, simd_shuffle4!(c, c, [LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Vector multiply subtract with scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmls.i16", LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mls, LANE = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vmls_laneq_u16(a: uint16x4_t, b: uint16x4_t, c: uint16x8_t) -> uint16x4_t { + static_assert_imm3!(LANE); + vmls_u16(a, b, simd_shuffle4!(c, c, [LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Vector multiply subtract with scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmls.i16", LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mls, LANE = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vmlsq_lane_u16(a: uint16x8_t, b: uint16x8_t, c: uint16x4_t) -> uint16x8_t { + static_assert_imm2!(LANE); + vmlsq_u16(a, b, simd_shuffle8!(c, c, [LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Vector multiply subtract with scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmls.i16", LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mls, LANE = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vmlsq_laneq_u16(a: uint16x8_t, b: uint16x8_t, c: uint16x8_t) -> uint16x8_t { + static_assert_imm3!(LANE); + vmlsq_u16(a, b, simd_shuffle8!(c, c, [LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Vector multiply subtract with scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmls.i32", LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mls, LANE = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vmls_lane_u32(a: uint32x2_t, b: uint32x2_t, c: uint32x2_t) -> uint32x2_t { + static_assert_imm1!(LANE); + vmls_u32(a, b, simd_shuffle2!(c, c, [LANE as u32, LANE as u32])) +} + +/// Vector multiply subtract with scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmls.i32", LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mls, LANE = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vmls_laneq_u32(a: uint32x2_t, b: uint32x2_t, c: uint32x4_t) -> uint32x2_t { + static_assert_imm2!(LANE); + vmls_u32(a, b, simd_shuffle2!(c, c, [LANE as u32, LANE as u32])) +} + +/// Vector multiply subtract with scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmls.i32", LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mls, LANE = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vmlsq_lane_u32(a: uint32x4_t, b: uint32x4_t, c: uint32x2_t) -> uint32x4_t { + static_assert_imm1!(LANE); + vmlsq_u32(a, b, simd_shuffle4!(c, c, [LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Vector multiply subtract with scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmls.i32", LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mls, LANE = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vmlsq_laneq_u32(a: uint32x4_t, b: uint32x4_t, c: uint32x4_t) -> uint32x4_t { + static_assert_imm2!(LANE); + vmlsq_u32(a, b, simd_shuffle4!(c, c, [LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Vector multiply subtract with scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmls.f32", LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fmul, LANE = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vmls_lane_f32(a: float32x2_t, b: float32x2_t, c: float32x2_t) -> float32x2_t { + static_assert_imm1!(LANE); + vmls_f32(a, b, simd_shuffle2!(c, c, [LANE as u32, LANE as u32])) +} + +/// Vector multiply subtract with scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmls.f32", LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fmul, LANE = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vmls_laneq_f32(a: float32x2_t, b: float32x2_t, c: float32x4_t) -> float32x2_t { + static_assert_imm2!(LANE); + vmls_f32(a, b, simd_shuffle2!(c, c, [LANE as u32, LANE as u32])) +} + +/// Vector multiply subtract with scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmls.f32", LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fmul, LANE = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vmlsq_lane_f32(a: float32x4_t, b: float32x4_t, c: float32x2_t) -> float32x4_t { + static_assert_imm1!(LANE); + vmlsq_f32(a, b, simd_shuffle4!(c, c, [LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Vector multiply subtract with scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmls.f32", LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fmul, LANE = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vmlsq_laneq_f32(a: float32x4_t, b: float32x4_t, c: float32x4_t) -> float32x4_t { + static_assert_imm2!(LANE); + vmlsq_f32(a, b, simd_shuffle4!(c, c, [LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Signed multiply-subtract long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmlsl.s8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(smlsl))] +pub unsafe fn vmlsl_s8(a: int16x8_t, b: int8x8_t, c: int8x8_t) -> int16x8_t { + simd_sub(a, vmull_s8(b, c)) +} + +/// Signed multiply-subtract long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmlsl.s16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(smlsl))] +pub unsafe fn vmlsl_s16(a: int32x4_t, b: int16x4_t, c: int16x4_t) -> int32x4_t { + simd_sub(a, vmull_s16(b, c)) +} + +/// Signed multiply-subtract long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmlsl.s32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(smlsl))] +pub unsafe fn vmlsl_s32(a: int64x2_t, b: int32x2_t, c: int32x2_t) -> int64x2_t { + simd_sub(a, vmull_s32(b, c)) +} + +/// Unsigned multiply-subtract long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmlsl.u8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(umlsl))] +pub unsafe fn vmlsl_u8(a: uint16x8_t, b: uint8x8_t, c: uint8x8_t) -> uint16x8_t { + simd_sub(a, vmull_u8(b, c)) +} + +/// Unsigned multiply-subtract long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmlsl.u16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(umlsl))] +pub unsafe fn vmlsl_u16(a: uint32x4_t, b: uint16x4_t, c: uint16x4_t) -> uint32x4_t { + simd_sub(a, vmull_u16(b, c)) +} + +/// Unsigned multiply-subtract long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmlsl.u32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(umlsl))] +pub unsafe fn vmlsl_u32(a: uint64x2_t, b: uint32x2_t, c: uint32x2_t) -> uint64x2_t { + simd_sub(a, vmull_u32(b, c)) +} + +/// Vector widening multiply subtract with scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmlsl.s16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(smlsl))] +pub unsafe fn vmlsl_n_s16(a: int32x4_t, b: int16x4_t, c: i16) -> int32x4_t { + vmlsl_s16(a, b, vdup_n_s16(c)) +} + +/// Vector widening multiply subtract with scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmlsl.s32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(smlsl))] +pub unsafe fn vmlsl_n_s32(a: int64x2_t, b: int32x2_t, c: i32) -> int64x2_t { + vmlsl_s32(a, b, vdup_n_s32(c)) +} + +/// Vector widening multiply subtract with scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmlsl.u16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(umlsl))] +pub unsafe fn vmlsl_n_u16(a: uint32x4_t, b: uint16x4_t, c: u16) -> uint32x4_t { + vmlsl_u16(a, b, vdup_n_u16(c)) +} + +/// Vector widening multiply subtract with scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmlsl.u32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(umlsl))] +pub unsafe fn vmlsl_n_u32(a: uint64x2_t, b: uint32x2_t, c: u32) -> uint64x2_t { + vmlsl_u32(a, b, vdup_n_u32(c)) +} + +/// Vector widening multiply subtract with scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmlsl.s16", LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(smlsl, LANE = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vmlsl_lane_s16(a: int32x4_t, b: int16x4_t, c: int16x4_t) -> int32x4_t { + static_assert_imm2!(LANE); + vmlsl_s16(a, b, simd_shuffle4!(c, c, [LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Vector widening multiply subtract with scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmlsl.s16", LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(smlsl, LANE = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vmlsl_laneq_s16(a: int32x4_t, b: int16x4_t, c: int16x8_t) -> int32x4_t { + static_assert_imm3!(LANE); + vmlsl_s16(a, b, simd_shuffle4!(c, c, [LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Vector widening multiply subtract with scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmlsl.s32", LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(smlsl, LANE = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vmlsl_lane_s32(a: int64x2_t, b: int32x2_t, c: int32x2_t) -> int64x2_t { + static_assert_imm1!(LANE); + vmlsl_s32(a, b, simd_shuffle2!(c, c, [LANE as u32, LANE as u32])) +} + +/// Vector widening multiply subtract with scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmlsl.s32", LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(smlsl, LANE = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vmlsl_laneq_s32(a: int64x2_t, b: int32x2_t, c: int32x4_t) -> int64x2_t { + static_assert_imm2!(LANE); + vmlsl_s32(a, b, simd_shuffle2!(c, c, [LANE as u32, LANE as u32])) +} + +/// Vector widening multiply subtract with scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmlsl.u16", LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(umlsl, LANE = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vmlsl_lane_u16(a: uint32x4_t, b: uint16x4_t, c: uint16x4_t) -> uint32x4_t { + static_assert_imm2!(LANE); + vmlsl_u16(a, b, simd_shuffle4!(c, c, [LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Vector widening multiply subtract with scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmlsl.u16", LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(umlsl, LANE = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vmlsl_laneq_u16(a: uint32x4_t, b: uint16x4_t, c: uint16x8_t) -> uint32x4_t { + static_assert_imm3!(LANE); + vmlsl_u16(a, b, simd_shuffle4!(c, c, [LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Vector widening multiply subtract with scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmlsl.u32", LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(umlsl, LANE = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vmlsl_lane_u32(a: uint64x2_t, b: uint32x2_t, c: uint32x2_t) -> uint64x2_t { + static_assert_imm1!(LANE); + vmlsl_u32(a, b, simd_shuffle2!(c, c, [LANE as u32, LANE as u32])) +} + +/// Vector widening multiply subtract with scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmlsl.u32", LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(umlsl, LANE = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vmlsl_laneq_u32(a: uint64x2_t, b: uint32x2_t, c: uint32x4_t) -> uint64x2_t { + static_assert_imm2!(LANE); + vmlsl_u32(a, b, simd_shuffle2!(c, c, [LANE as u32, LANE as u32])) +} + +/// Negate +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vneg.s8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(neg))] +pub unsafe fn vneg_s8(a: int8x8_t) -> int8x8_t { + simd_neg(a) +} + +/// Negate +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vneg.s8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(neg))] +pub unsafe fn vnegq_s8(a: int8x16_t) -> int8x16_t { + simd_neg(a) +} + +/// Negate +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vneg.s16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(neg))] +pub unsafe fn vneg_s16(a: int16x4_t) -> int16x4_t { + simd_neg(a) +} + +/// Negate +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vneg.s16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(neg))] +pub unsafe fn vnegq_s16(a: int16x8_t) -> int16x8_t { + simd_neg(a) +} + +/// Negate +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vneg.s32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(neg))] +pub unsafe fn vneg_s32(a: int32x2_t) -> int32x2_t { + simd_neg(a) +} + +/// Negate +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vneg.s32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(neg))] +pub unsafe fn vnegq_s32(a: int32x4_t) -> int32x4_t { + simd_neg(a) +} + +/// Negate +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vneg.f32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fneg))] +pub unsafe fn vneg_f32(a: float32x2_t) -> float32x2_t { + simd_neg(a) +} + +/// Negate +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vneg.f32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fneg))] +pub unsafe fn vnegq_f32(a: float32x4_t) -> float32x4_t { + simd_neg(a) +} + +/// Signed saturating negate +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vqneg.s8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqneg))] +pub unsafe fn vqneg_s8(a: int8x8_t) -> int8x8_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqneg.v8i8")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqneg.v8i8")] + fn vqneg_s8_(a: int8x8_t) -> int8x8_t; + } +vqneg_s8_(a) +} + +/// Signed saturating negate +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vqneg.s8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqneg))] +pub unsafe fn vqnegq_s8(a: int8x16_t) -> int8x16_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqneg.v16i8")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqneg.v16i8")] + fn vqnegq_s8_(a: int8x16_t) -> int8x16_t; + } +vqnegq_s8_(a) +} + +/// Signed saturating negate +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vqneg.s16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqneg))] +pub unsafe fn vqneg_s16(a: int16x4_t) -> int16x4_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqneg.v4i16")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqneg.v4i16")] + fn vqneg_s16_(a: int16x4_t) -> int16x4_t; + } +vqneg_s16_(a) +} + +/// Signed saturating negate +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vqneg.s16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqneg))] +pub unsafe fn vqnegq_s16(a: int16x8_t) -> int16x8_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqneg.v8i16")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqneg.v8i16")] + fn vqnegq_s16_(a: int16x8_t) -> int16x8_t; + } +vqnegq_s16_(a) +} + +/// Signed saturating negate +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vqneg.s32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqneg))] +pub unsafe fn vqneg_s32(a: int32x2_t) -> int32x2_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqneg.v2i32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqneg.v2i32")] + fn vqneg_s32_(a: int32x2_t) -> int32x2_t; + } +vqneg_s32_(a) +} + +/// Signed saturating negate +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vqneg.s32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqneg))] +pub unsafe fn vqnegq_s32(a: int32x4_t) -> int32x4_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqneg.v4i32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqneg.v4i32")] + fn vqnegq_s32_(a: int32x4_t) -> int32x4_t; + } +vqnegq_s32_(a) +} + +/// Saturating subtract +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vqsub.u8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uqsub))] +pub unsafe fn vqsub_u8(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.usub.sat.v8i8")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uqsub.v8i8")] + fn vqsub_u8_(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t; + } +vqsub_u8_(a, b) +} + +/// Saturating subtract +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vqsub.u8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uqsub))] +pub unsafe fn vqsubq_u8(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.usub.sat.v16i8")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uqsub.v16i8")] + fn vqsubq_u8_(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t; + } +vqsubq_u8_(a, b) +} + +/// Saturating subtract +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vqsub.u16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uqsub))] +pub unsafe fn vqsub_u16(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.usub.sat.v4i16")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uqsub.v4i16")] + fn vqsub_u16_(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t; + } +vqsub_u16_(a, b) +} + +/// Saturating subtract +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vqsub.u16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uqsub))] +pub unsafe fn vqsubq_u16(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.usub.sat.v8i16")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uqsub.v8i16")] + fn vqsubq_u16_(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t; + } +vqsubq_u16_(a, b) +} + +/// Saturating subtract +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vqsub.u32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uqsub))] +pub unsafe fn vqsub_u32(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.usub.sat.v2i32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uqsub.v2i32")] + fn vqsub_u32_(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t; + } +vqsub_u32_(a, b) +} + +/// Saturating subtract +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vqsub.u32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uqsub))] +pub unsafe fn vqsubq_u32(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.usub.sat.v4i32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uqsub.v4i32")] + fn vqsubq_u32_(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t; + } +vqsubq_u32_(a, b) +} + +/// Saturating subtract +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vqsub.u64"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uqsub))] +pub unsafe fn vqsub_u64(a: uint64x1_t, b: uint64x1_t) -> uint64x1_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.usub.sat.v1i64")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uqsub.v1i64")] + fn vqsub_u64_(a: uint64x1_t, b: uint64x1_t) -> uint64x1_t; + } +vqsub_u64_(a, b) +} + +/// Saturating subtract +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vqsub.u64"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uqsub))] +pub unsafe fn vqsubq_u64(a: uint64x2_t, b: uint64x2_t) -> uint64x2_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.usub.sat.v2i64")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uqsub.v2i64")] + fn vqsubq_u64_(a: uint64x2_t, b: uint64x2_t) -> uint64x2_t; + } +vqsubq_u64_(a, b) +} + +/// Saturating subtract +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vqsub.s8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqsub))] +pub unsafe fn vqsub_s8(a: int8x8_t, b: int8x8_t) -> int8x8_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.ssub.sat.v8i8")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqsub.v8i8")] + fn vqsub_s8_(a: int8x8_t, b: int8x8_t) -> int8x8_t; + } +vqsub_s8_(a, b) +} + +/// Saturating subtract +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vqsub.s8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqsub))] +pub unsafe fn vqsubq_s8(a: int8x16_t, b: int8x16_t) -> int8x16_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.ssub.sat.v16i8")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqsub.v16i8")] + fn vqsubq_s8_(a: int8x16_t, b: int8x16_t) -> int8x16_t; + } +vqsubq_s8_(a, b) +} + +/// Saturating subtract +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vqsub.s16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqsub))] +pub unsafe fn vqsub_s16(a: int16x4_t, b: int16x4_t) -> int16x4_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.ssub.sat.v4i16")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqsub.v4i16")] + fn vqsub_s16_(a: int16x4_t, b: int16x4_t) -> int16x4_t; + } +vqsub_s16_(a, b) +} + +/// Saturating subtract +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vqsub.s16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqsub))] +pub unsafe fn vqsubq_s16(a: int16x8_t, b: int16x8_t) -> int16x8_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.ssub.sat.v8i16")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqsub.v8i16")] + fn vqsubq_s16_(a: int16x8_t, b: int16x8_t) -> int16x8_t; + } +vqsubq_s16_(a, b) +} + +/// Saturating subtract +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vqsub.s32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqsub))] +pub unsafe fn vqsub_s32(a: int32x2_t, b: int32x2_t) -> int32x2_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.ssub.sat.v2i32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqsub.v2i32")] + fn vqsub_s32_(a: int32x2_t, b: int32x2_t) -> int32x2_t; + } +vqsub_s32_(a, b) +} + +/// Saturating subtract +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vqsub.s32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqsub))] +pub unsafe fn vqsubq_s32(a: int32x4_t, b: int32x4_t) -> int32x4_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.ssub.sat.v4i32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqsub.v4i32")] + fn vqsubq_s32_(a: int32x4_t, b: int32x4_t) -> int32x4_t; + } +vqsubq_s32_(a, b) +} + +/// Saturating subtract +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vqsub.s64"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqsub))] +pub unsafe fn vqsub_s64(a: int64x1_t, b: int64x1_t) -> int64x1_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.ssub.sat.v1i64")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqsub.v1i64")] + fn vqsub_s64_(a: int64x1_t, b: int64x1_t) -> int64x1_t; + } +vqsub_s64_(a, b) +} + +/// Saturating subtract +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vqsub.s64"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqsub))] +pub unsafe fn vqsubq_s64(a: int64x2_t, b: int64x2_t) -> int64x2_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.ssub.sat.v2i64")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqsub.v2i64")] + fn vqsubq_s64_(a: int64x2_t, b: int64x2_t) -> int64x2_t; + } +vqsubq_s64_(a, b) +} + +/// Halving add +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vhadd.u8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uhadd))] +pub unsafe fn vhadd_u8(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vhaddu.v8i8")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uhadd.v8i8")] + fn vhadd_u8_(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t; + } +vhadd_u8_(a, b) +} + +/// Halving add +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vhadd.u8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uhadd))] +pub unsafe fn vhaddq_u8(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vhaddu.v16i8")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uhadd.v16i8")] + fn vhaddq_u8_(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t; + } +vhaddq_u8_(a, b) +} + +/// Halving add +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vhadd.u16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uhadd))] +pub unsafe fn vhadd_u16(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vhaddu.v4i16")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uhadd.v4i16")] + fn vhadd_u16_(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t; + } +vhadd_u16_(a, b) +} + +/// Halving add +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vhadd.u16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uhadd))] +pub unsafe fn vhaddq_u16(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vhaddu.v8i16")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uhadd.v8i16")] + fn vhaddq_u16_(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t; + } +vhaddq_u16_(a, b) +} + +/// Halving add +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vhadd.u32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uhadd))] +pub unsafe fn vhadd_u32(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vhaddu.v2i32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uhadd.v2i32")] + fn vhadd_u32_(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t; + } +vhadd_u32_(a, b) +} + +/// Halving add +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vhadd.u32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uhadd))] +pub unsafe fn vhaddq_u32(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vhaddu.v4i32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uhadd.v4i32")] + fn vhaddq_u32_(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t; + } +vhaddq_u32_(a, b) +} + +/// Halving add +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vhadd.s8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(shadd))] +pub unsafe fn vhadd_s8(a: int8x8_t, b: int8x8_t) -> int8x8_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vhadds.v8i8")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.shadd.v8i8")] + fn vhadd_s8_(a: int8x8_t, b: int8x8_t) -> int8x8_t; + } +vhadd_s8_(a, b) +} + +/// Halving add +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vhadd.s8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(shadd))] +pub unsafe fn vhaddq_s8(a: int8x16_t, b: int8x16_t) -> int8x16_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vhadds.v16i8")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.shadd.v16i8")] + fn vhaddq_s8_(a: int8x16_t, b: int8x16_t) -> int8x16_t; + } +vhaddq_s8_(a, b) +} + +/// Halving add +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vhadd.s16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(shadd))] +pub unsafe fn vhadd_s16(a: int16x4_t, b: int16x4_t) -> int16x4_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vhadds.v4i16")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.shadd.v4i16")] + fn vhadd_s16_(a: int16x4_t, b: int16x4_t) -> int16x4_t; + } +vhadd_s16_(a, b) +} + +/// Halving add +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vhadd.s16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(shadd))] +pub unsafe fn vhaddq_s16(a: int16x8_t, b: int16x8_t) -> int16x8_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vhadds.v8i16")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.shadd.v8i16")] + fn vhaddq_s16_(a: int16x8_t, b: int16x8_t) -> int16x8_t; + } +vhaddq_s16_(a, b) +} + +/// Halving add +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vhadd.s32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(shadd))] +pub unsafe fn vhadd_s32(a: int32x2_t, b: int32x2_t) -> int32x2_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vhadds.v2i32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.shadd.v2i32")] + fn vhadd_s32_(a: int32x2_t, b: int32x2_t) -> int32x2_t; + } +vhadd_s32_(a, b) +} + +/// Halving add +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vhadd.s32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(shadd))] +pub unsafe fn vhaddq_s32(a: int32x4_t, b: int32x4_t) -> int32x4_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vhadds.v4i32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.shadd.v4i32")] + fn vhaddq_s32_(a: int32x4_t, b: int32x4_t) -> int32x4_t; + } +vhaddq_s32_(a, b) +} + +/// Rounding halving add +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrhadd.u8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(urhadd))] +pub unsafe fn vrhadd_u8(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vrhaddu.v8i8")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.urhadd.v8i8")] + fn vrhadd_u8_(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t; + } +vrhadd_u8_(a, b) +} + +/// Rounding halving add +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrhadd.u8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(urhadd))] +pub unsafe fn vrhaddq_u8(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vrhaddu.v16i8")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.urhadd.v16i8")] + fn vrhaddq_u8_(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t; + } +vrhaddq_u8_(a, b) +} + +/// Rounding halving add +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrhadd.u16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(urhadd))] +pub unsafe fn vrhadd_u16(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vrhaddu.v4i16")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.urhadd.v4i16")] + fn vrhadd_u16_(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t; + } +vrhadd_u16_(a, b) +} + +/// Rounding halving add +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrhadd.u16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(urhadd))] +pub unsafe fn vrhaddq_u16(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vrhaddu.v8i16")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.urhadd.v8i16")] + fn vrhaddq_u16_(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t; + } +vrhaddq_u16_(a, b) +} + +/// Rounding halving add +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrhadd.u32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(urhadd))] +pub unsafe fn vrhadd_u32(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vrhaddu.v2i32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.urhadd.v2i32")] + fn vrhadd_u32_(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t; + } +vrhadd_u32_(a, b) +} + +/// Rounding halving add +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrhadd.u32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(urhadd))] +pub unsafe fn vrhaddq_u32(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vrhaddu.v4i32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.urhadd.v4i32")] + fn vrhaddq_u32_(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t; + } +vrhaddq_u32_(a, b) +} + +/// Rounding halving add +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrhadd.s8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(srhadd))] +pub unsafe fn vrhadd_s8(a: int8x8_t, b: int8x8_t) -> int8x8_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vrhadds.v8i8")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.srhadd.v8i8")] + fn vrhadd_s8_(a: int8x8_t, b: int8x8_t) -> int8x8_t; + } +vrhadd_s8_(a, b) +} + +/// Rounding halving add +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrhadd.s8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(srhadd))] +pub unsafe fn vrhaddq_s8(a: int8x16_t, b: int8x16_t) -> int8x16_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vrhadds.v16i8")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.srhadd.v16i8")] + fn vrhaddq_s8_(a: int8x16_t, b: int8x16_t) -> int8x16_t; + } +vrhaddq_s8_(a, b) +} + +/// Rounding halving add +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrhadd.s16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(srhadd))] +pub unsafe fn vrhadd_s16(a: int16x4_t, b: int16x4_t) -> int16x4_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vrhadds.v4i16")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.srhadd.v4i16")] + fn vrhadd_s16_(a: int16x4_t, b: int16x4_t) -> int16x4_t; + } +vrhadd_s16_(a, b) +} + +/// Rounding halving add +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrhadd.s16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(srhadd))] +pub unsafe fn vrhaddq_s16(a: int16x8_t, b: int16x8_t) -> int16x8_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vrhadds.v8i16")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.srhadd.v8i16")] + fn vrhaddq_s16_(a: int16x8_t, b: int16x8_t) -> int16x8_t; + } +vrhaddq_s16_(a, b) +} + +/// Rounding halving add +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrhadd.s32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(srhadd))] +pub unsafe fn vrhadd_s32(a: int32x2_t, b: int32x2_t) -> int32x2_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vrhadds.v2i32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.srhadd.v2i32")] + fn vrhadd_s32_(a: int32x2_t, b: int32x2_t) -> int32x2_t; + } +vrhadd_s32_(a, b) +} + +/// Rounding halving add +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrhadd.s32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(srhadd))] +pub unsafe fn vrhaddq_s32(a: int32x4_t, b: int32x4_t) -> int32x4_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vrhadds.v4i32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.srhadd.v4i32")] + fn vrhaddq_s32_(a: int32x4_t, b: int32x4_t) -> int32x4_t; + } +vrhaddq_s32_(a, b) +} + +/// Floating-point round to integral, to nearest with ties to even +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "fp-armv8,v8"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrintn))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(frintn))] +pub unsafe fn vrndn_f32(a: float32x2_t) -> float32x2_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vrintn.v2f32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.frintn.v2f32")] + fn vrndn_f32_(a: float32x2_t) -> float32x2_t; + } +vrndn_f32_(a) +} + +/// Floating-point round to integral, to nearest with ties to even +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "fp-armv8,v8"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrintn))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(frintn))] +pub unsafe fn vrndnq_f32(a: float32x4_t) -> float32x4_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vrintn.v4f32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.frintn.v4f32")] + fn vrndnq_f32_(a: float32x4_t) -> float32x4_t; + } +vrndnq_f32_(a) +} + +/// Saturating add +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vqadd.u8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uqadd))] +pub unsafe fn vqadd_u8(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.uadd.sat.v8i8")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uqadd.v8i8")] + fn vqadd_u8_(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t; + } +vqadd_u8_(a, b) +} + +/// Saturating add +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vqadd.u8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uqadd))] +pub unsafe fn vqaddq_u8(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.uadd.sat.v16i8")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uqadd.v16i8")] + fn vqaddq_u8_(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t; + } +vqaddq_u8_(a, b) +} + +/// Saturating add +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vqadd.u16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uqadd))] +pub unsafe fn vqadd_u16(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.uadd.sat.v4i16")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uqadd.v4i16")] + fn vqadd_u16_(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t; + } +vqadd_u16_(a, b) +} + +/// Saturating add +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vqadd.u16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uqadd))] +pub unsafe fn vqaddq_u16(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.uadd.sat.v8i16")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uqadd.v8i16")] + fn vqaddq_u16_(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t; + } +vqaddq_u16_(a, b) +} + +/// Saturating add +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vqadd.u32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uqadd))] +pub unsafe fn vqadd_u32(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.uadd.sat.v2i32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uqadd.v2i32")] + fn vqadd_u32_(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t; + } +vqadd_u32_(a, b) +} + +/// Saturating add +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vqadd.u32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uqadd))] +pub unsafe fn vqaddq_u32(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.uadd.sat.v4i32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uqadd.v4i32")] + fn vqaddq_u32_(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t; + } +vqaddq_u32_(a, b) +} + +/// Saturating add +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vqadd.u64"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uqadd))] +pub unsafe fn vqadd_u64(a: uint64x1_t, b: uint64x1_t) -> uint64x1_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.uadd.sat.v1i64")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uqadd.v1i64")] + fn vqadd_u64_(a: uint64x1_t, b: uint64x1_t) -> uint64x1_t; + } +vqadd_u64_(a, b) +} + +/// Saturating add +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vqadd.u64"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uqadd))] +pub unsafe fn vqaddq_u64(a: uint64x2_t, b: uint64x2_t) -> uint64x2_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.uadd.sat.v2i64")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uqadd.v2i64")] + fn vqaddq_u64_(a: uint64x2_t, b: uint64x2_t) -> uint64x2_t; + } +vqaddq_u64_(a, b) +} + +/// Saturating add +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vqadd.s8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqadd))] +pub unsafe fn vqadd_s8(a: int8x8_t, b: int8x8_t) -> int8x8_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.sadd.sat.v8i8")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqadd.v8i8")] + fn vqadd_s8_(a: int8x8_t, b: int8x8_t) -> int8x8_t; + } +vqadd_s8_(a, b) +} + +/// Saturating add +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vqadd.s8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqadd))] +pub unsafe fn vqaddq_s8(a: int8x16_t, b: int8x16_t) -> int8x16_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.sadd.sat.v16i8")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqadd.v16i8")] + fn vqaddq_s8_(a: int8x16_t, b: int8x16_t) -> int8x16_t; + } +vqaddq_s8_(a, b) +} + +/// Saturating add +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vqadd.s16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqadd))] +pub unsafe fn vqadd_s16(a: int16x4_t, b: int16x4_t) -> int16x4_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.sadd.sat.v4i16")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqadd.v4i16")] + fn vqadd_s16_(a: int16x4_t, b: int16x4_t) -> int16x4_t; + } +vqadd_s16_(a, b) +} + +/// Saturating add +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vqadd.s16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqadd))] +pub unsafe fn vqaddq_s16(a: int16x8_t, b: int16x8_t) -> int16x8_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.sadd.sat.v8i16")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqadd.v8i16")] + fn vqaddq_s16_(a: int16x8_t, b: int16x8_t) -> int16x8_t; + } +vqaddq_s16_(a, b) +} + +/// Saturating add +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vqadd.s32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqadd))] +pub unsafe fn vqadd_s32(a: int32x2_t, b: int32x2_t) -> int32x2_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.sadd.sat.v2i32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqadd.v2i32")] + fn vqadd_s32_(a: int32x2_t, b: int32x2_t) -> int32x2_t; + } +vqadd_s32_(a, b) +} + +/// Saturating add +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vqadd.s32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqadd))] +pub unsafe fn vqaddq_s32(a: int32x4_t, b: int32x4_t) -> int32x4_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.sadd.sat.v4i32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqadd.v4i32")] + fn vqaddq_s32_(a: int32x4_t, b: int32x4_t) -> int32x4_t; + } +vqaddq_s32_(a, b) +} + +/// Saturating add +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vqadd.s64"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqadd))] +pub unsafe fn vqadd_s64(a: int64x1_t, b: int64x1_t) -> int64x1_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.sadd.sat.v1i64")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqadd.v1i64")] + fn vqadd_s64_(a: int64x1_t, b: int64x1_t) -> int64x1_t; + } +vqadd_s64_(a, b) +} + +/// Saturating add +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vqadd.s64"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqadd))] +pub unsafe fn vqaddq_s64(a: int64x2_t, b: int64x2_t) -> int64x2_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.sadd.sat.v2i64")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqadd.v2i64")] + fn vqaddq_s64_(a: int64x2_t, b: int64x2_t) -> int64x2_t; + } +vqaddq_s64_(a, b) +} + +/// Multiply +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmul.i8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mul))] +pub unsafe fn vmul_s8(a: int8x8_t, b: int8x8_t) -> int8x8_t { + simd_mul(a, b) +} + +/// Multiply +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmul.i8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mul))] +pub unsafe fn vmulq_s8(a: int8x16_t, b: int8x16_t) -> int8x16_t { + simd_mul(a, b) +} + +/// Multiply +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmul.i16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mul))] +pub unsafe fn vmul_s16(a: int16x4_t, b: int16x4_t) -> int16x4_t { + simd_mul(a, b) +} + +/// Multiply +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmul.i16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mul))] +pub unsafe fn vmulq_s16(a: int16x8_t, b: int16x8_t) -> int16x8_t { + simd_mul(a, b) +} + +/// Multiply +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmul.i32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mul))] +pub unsafe fn vmul_s32(a: int32x2_t, b: int32x2_t) -> int32x2_t { + simd_mul(a, b) +} + +/// Multiply +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmul.i32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mul))] +pub unsafe fn vmulq_s32(a: int32x4_t, b: int32x4_t) -> int32x4_t { + simd_mul(a, b) +} + +/// Multiply +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmul.i8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mul))] +pub unsafe fn vmul_u8(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t { + simd_mul(a, b) +} + +/// Multiply +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmul.i8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mul))] +pub unsafe fn vmulq_u8(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t { + simd_mul(a, b) +} + +/// Multiply +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmul.i16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mul))] +pub unsafe fn vmul_u16(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t { + simd_mul(a, b) +} + +/// Multiply +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmul.i16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mul))] +pub unsafe fn vmulq_u16(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t { + simd_mul(a, b) +} + +/// Multiply +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmul.i32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mul))] +pub unsafe fn vmul_u32(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t { + simd_mul(a, b) +} + +/// Multiply +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmul.i32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mul))] +pub unsafe fn vmulq_u32(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t { + simd_mul(a, b) +} + +/// Polynomial multiply +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmul))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(pmul))] +pub unsafe fn vmul_p8(a: poly8x8_t, b: poly8x8_t) -> poly8x8_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vmulp.v8i8")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.pmul.v8i8")] + fn vmul_p8_(a: poly8x8_t, b: poly8x8_t) -> poly8x8_t; + } +vmul_p8_(a, b) +} + +/// Polynomial multiply +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmul))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(pmul))] +pub unsafe fn vmulq_p8(a: poly8x16_t, b: poly8x16_t) -> poly8x16_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vmulp.v16i8")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.pmul.v16i8")] + fn vmulq_p8_(a: poly8x16_t, b: poly8x16_t) -> poly8x16_t; + } +vmulq_p8_(a, b) +} + +/// Multiply +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmul.f32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fmul))] +pub unsafe fn vmul_f32(a: float32x2_t, b: float32x2_t) -> float32x2_t { + simd_mul(a, b) +} + +/// Multiply +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmul.f32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fmul))] +pub unsafe fn vmulq_f32(a: float32x4_t, b: float32x4_t) -> float32x4_t { + simd_mul(a, b) +} + +/// Vector multiply by scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmul))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mul))] +pub unsafe fn vmul_n_s16(a: int16x4_t, b: i16) -> int16x4_t { + simd_mul(a, vdup_n_s16(b)) +} + +/// Vector multiply by scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmul))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mul))] +pub unsafe fn vmulq_n_s16(a: int16x8_t, b: i16) -> int16x8_t { + simd_mul(a, vdupq_n_s16(b)) +} + +/// Vector multiply by scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmul))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mul))] +pub unsafe fn vmul_n_s32(a: int32x2_t, b: i32) -> int32x2_t { + simd_mul(a, vdup_n_s32(b)) +} + +/// Vector multiply by scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmul))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mul))] +pub unsafe fn vmulq_n_s32(a: int32x4_t, b: i32) -> int32x4_t { + simd_mul(a, vdupq_n_s32(b)) +} + +/// Vector multiply by scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmul))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mul))] +pub unsafe fn vmul_n_u16(a: uint16x4_t, b: u16) -> uint16x4_t { + simd_mul(a, vdup_n_u16(b)) +} + +/// Vector multiply by scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmul))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mul))] +pub unsafe fn vmulq_n_u16(a: uint16x8_t, b: u16) -> uint16x8_t { + simd_mul(a, vdupq_n_u16(b)) +} + +/// Vector multiply by scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmul))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mul))] +pub unsafe fn vmul_n_u32(a: uint32x2_t, b: u32) -> uint32x2_t { + simd_mul(a, vdup_n_u32(b)) +} + +/// Vector multiply by scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmul))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mul))] +pub unsafe fn vmulq_n_u32(a: uint32x4_t, b: u32) -> uint32x4_t { + simd_mul(a, vdupq_n_u32(b)) +} + +/// Vector multiply by scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmul))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fmul))] +pub unsafe fn vmul_n_f32(a: float32x2_t, b: f32) -> float32x2_t { + simd_mul(a, vdup_n_f32(b)) +} + +/// Vector multiply by scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmul))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fmul))] +pub unsafe fn vmulq_n_f32(a: float32x4_t, b: f32) -> float32x4_t { + simd_mul(a, vdupq_n_f32(b)) +} + +/// Multiply +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmul, LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mul, LANE = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vmul_lane_s16(a: int16x4_t, b: int16x4_t) -> int16x4_t { + static_assert_imm2!(LANE); + simd_mul(a, simd_shuffle4!(b, b, [LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Multiply +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmul, LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mul, LANE = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vmul_laneq_s16(a: int16x4_t, b: int16x8_t) -> int16x4_t { + static_assert_imm3!(LANE); + simd_mul(a, simd_shuffle4!(b, b, [LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Multiply +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmul, LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mul, LANE = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vmulq_lane_s16(a: int16x8_t, b: int16x4_t) -> int16x8_t { + static_assert_imm2!(LANE); + simd_mul(a, simd_shuffle8!(b, b, [LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Multiply +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmul, LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mul, LANE = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vmulq_laneq_s16(a: int16x8_t, b: int16x8_t) -> int16x8_t { + static_assert_imm3!(LANE); + simd_mul(a, simd_shuffle8!(b, b, [LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Multiply +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmul, LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mul, LANE = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vmul_lane_s32(a: int32x2_t, b: int32x2_t) -> int32x2_t { + static_assert_imm1!(LANE); + simd_mul(a, simd_shuffle2!(b, b, [LANE as u32, LANE as u32])) +} + +/// Multiply +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmul, LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mul, LANE = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vmul_laneq_s32(a: int32x2_t, b: int32x4_t) -> int32x2_t { + static_assert_imm2!(LANE); + simd_mul(a, simd_shuffle2!(b, b, [LANE as u32, LANE as u32])) +} + +/// Multiply +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmul, LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mul, LANE = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vmulq_lane_s32(a: int32x4_t, b: int32x2_t) -> int32x4_t { + static_assert_imm1!(LANE); + simd_mul(a, simd_shuffle4!(b, b, [LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Multiply +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmul, LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mul, LANE = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vmulq_laneq_s32(a: int32x4_t, b: int32x4_t) -> int32x4_t { + static_assert_imm2!(LANE); + simd_mul(a, simd_shuffle4!(b, b, [LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Multiply +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmul, LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mul, LANE = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vmul_lane_u16(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t { + static_assert_imm2!(LANE); + simd_mul(a, simd_shuffle4!(b, b, [LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Multiply +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmul, LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mul, LANE = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vmul_laneq_u16(a: uint16x4_t, b: uint16x8_t) -> uint16x4_t { + static_assert_imm3!(LANE); + simd_mul(a, simd_shuffle4!(b, b, [LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Multiply +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmul, LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mul, LANE = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vmulq_lane_u16(a: uint16x8_t, b: uint16x4_t) -> uint16x8_t { + static_assert_imm2!(LANE); + simd_mul(a, simd_shuffle8!(b, b, [LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Multiply +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmul, LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mul, LANE = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vmulq_laneq_u16(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t { + static_assert_imm3!(LANE); + simd_mul(a, simd_shuffle8!(b, b, [LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Multiply +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmul, LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mul, LANE = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vmul_lane_u32(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t { + static_assert_imm1!(LANE); + simd_mul(a, simd_shuffle2!(b, b, [LANE as u32, LANE as u32])) +} + +/// Multiply +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmul, LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mul, LANE = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vmul_laneq_u32(a: uint32x2_t, b: uint32x4_t) -> uint32x2_t { + static_assert_imm2!(LANE); + simd_mul(a, simd_shuffle2!(b, b, [LANE as u32, LANE as u32])) +} + +/// Multiply +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmul, LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mul, LANE = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vmulq_lane_u32(a: uint32x4_t, b: uint32x2_t) -> uint32x4_t { + static_assert_imm1!(LANE); + simd_mul(a, simd_shuffle4!(b, b, [LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Multiply +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmul, LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mul, LANE = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vmulq_laneq_u32(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t { + static_assert_imm2!(LANE); + simd_mul(a, simd_shuffle4!(b, b, [LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Floating-point multiply +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmul, LANE = 0))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fmul, LANE = 0))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vmul_lane_f32(a: float32x2_t, b: float32x2_t) -> float32x2_t { + static_assert_imm1!(LANE); + simd_mul(a, simd_shuffle2!(b, b, [LANE as u32, LANE as u32])) +} + +/// Floating-point multiply +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmul, LANE = 0))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fmul, LANE = 0))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vmul_laneq_f32(a: float32x2_t, b: float32x4_t) -> float32x2_t { + static_assert_imm2!(LANE); + simd_mul(a, simd_shuffle2!(b, b, [LANE as u32, LANE as u32])) +} + +/// Floating-point multiply +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmul, LANE = 0))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fmul, LANE = 0))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vmulq_lane_f32(a: float32x4_t, b: float32x2_t) -> float32x4_t { + static_assert_imm1!(LANE); + simd_mul(a, simd_shuffle4!(b, b, [LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Floating-point multiply +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmul, LANE = 0))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fmul, LANE = 0))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vmulq_laneq_f32(a: float32x4_t, b: float32x4_t) -> float32x4_t { + static_assert_imm2!(LANE); + simd_mul(a, simd_shuffle4!(b, b, [LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Signed multiply long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmull.s8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(smull))] +pub unsafe fn vmull_s8(a: int8x8_t, b: int8x8_t) -> int16x8_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vmulls.v8i8")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.smull.v8i8")] + fn vmull_s8_(a: int8x8_t, b: int8x8_t) -> int16x8_t; + } +vmull_s8_(a, b) +} + +/// Signed multiply long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmull.s16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(smull))] +pub unsafe fn vmull_s16(a: int16x4_t, b: int16x4_t) -> int32x4_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vmulls.v4i16")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.smull.v4i16")] + fn vmull_s16_(a: int16x4_t, b: int16x4_t) -> int32x4_t; + } +vmull_s16_(a, b) +} + +/// Signed multiply long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmull.s32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(smull))] +pub unsafe fn vmull_s32(a: int32x2_t, b: int32x2_t) -> int64x2_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vmulls.v2i32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.smull.v2i32")] + fn vmull_s32_(a: int32x2_t, b: int32x2_t) -> int64x2_t; + } +vmull_s32_(a, b) +} + +/// Unsigned multiply long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmull.u8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(umull))] +pub unsafe fn vmull_u8(a: uint8x8_t, b: uint8x8_t) -> uint16x8_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vmullu.v8i8")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.umull.v8i8")] + fn vmull_u8_(a: uint8x8_t, b: uint8x8_t) -> uint16x8_t; + } +vmull_u8_(a, b) +} + +/// Unsigned multiply long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmull.u16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(umull))] +pub unsafe fn vmull_u16(a: uint16x4_t, b: uint16x4_t) -> uint32x4_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vmullu.v4i16")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.umull.v4i16")] + fn vmull_u16_(a: uint16x4_t, b: uint16x4_t) -> uint32x4_t; + } +vmull_u16_(a, b) +} + +/// Unsigned multiply long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmull.u32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(umull))] +pub unsafe fn vmull_u32(a: uint32x2_t, b: uint32x2_t) -> uint64x2_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vmullu.v2i32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.umull.v2i32")] + fn vmull_u32_(a: uint32x2_t, b: uint32x2_t) -> uint64x2_t; + } +vmull_u32_(a, b) +} + +/// Polynomial multiply long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmull.p8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(pmull))] +pub unsafe fn vmull_p8(a: poly8x8_t, b: poly8x8_t) -> poly16x8_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vmullp.v8i8")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.pmull.v8i8")] + fn vmull_p8_(a: poly8x8_t, b: poly8x8_t) -> poly16x8_t; + } +vmull_p8_(a, b) +} + +/// Vector long multiply with scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmull))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(smull))] +pub unsafe fn vmullh_n_s16(a: int16x4_t, b: i16) -> int32x4_t { + vmull_s16(a, vdup_n_s16(b)) +} + +/// Vector long multiply with scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmull))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(smull))] +pub unsafe fn vmulls_n_s32(a: int32x2_t, b: i32) -> int64x2_t { + vmull_s32(a, vdup_n_s32(b)) +} + +/// Vector long multiply with scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmull))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(umull))] +pub unsafe fn vmullh_n_u16(a: uint16x4_t, b: u16) -> uint32x4_t { + vmull_u16(a, vdup_n_u16(b)) +} + +/// Vector long multiply with scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmull))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(umull))] +pub unsafe fn vmulls_n_u32(a: uint32x2_t, b: u32) -> uint64x2_t { + vmull_u32(a, vdup_n_u32(b)) +} + +/// Vector long multiply by scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmull, LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(smull, LANE = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vmull_lane_s16(a: int16x4_t, b: int16x4_t) -> int32x4_t { + static_assert_imm2!(LANE); + vmull_s16(a, simd_shuffle4!(b, b, [LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Vector long multiply by scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmull, LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(smull, LANE = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vmull_laneq_s16(a: int16x4_t, b: int16x8_t) -> int32x4_t { + static_assert_imm3!(LANE); + vmull_s16(a, simd_shuffle4!(b, b, [LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Vector long multiply by scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmull, LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(smull, LANE = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vmull_lane_s32(a: int32x2_t, b: int32x2_t) -> int64x2_t { + static_assert_imm1!(LANE); + vmull_s32(a, simd_shuffle2!(b, b, [LANE as u32, LANE as u32])) +} + +/// Vector long multiply by scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmull, LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(smull, LANE = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vmull_laneq_s32(a: int32x2_t, b: int32x4_t) -> int64x2_t { + static_assert_imm2!(LANE); + vmull_s32(a, simd_shuffle2!(b, b, [LANE as u32, LANE as u32])) +} + +/// Vector long multiply by scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmull, LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(umull, LANE = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vmull_lane_u16(a: uint16x4_t, b: uint16x4_t) -> uint32x4_t { + static_assert_imm2!(LANE); + vmull_u16(a, simd_shuffle4!(b, b, [LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Vector long multiply by scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmull, LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(umull, LANE = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vmull_laneq_u16(a: uint16x4_t, b: uint16x8_t) -> uint32x4_t { + static_assert_imm3!(LANE); + vmull_u16(a, simd_shuffle4!(b, b, [LANE as u32, LANE as u32, LANE as u32, LANE as u32])) +} + +/// Vector long multiply by scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmull, LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(umull, LANE = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vmull_lane_u32(a: uint32x2_t, b: uint32x2_t) -> uint64x2_t { + static_assert_imm1!(LANE); + vmull_u32(a, simd_shuffle2!(b, b, [LANE as u32, LANE as u32])) +} + +/// Vector long multiply by scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmull, LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(umull, LANE = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vmull_laneq_u32(a: uint32x2_t, b: uint32x4_t) -> uint64x2_t { + static_assert_imm2!(LANE); + vmull_u32(a, simd_shuffle2!(b, b, [LANE as u32, LANE as u32])) +} + +/// Floating-point fused Multiply-Add to accumulator(vector) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "fp-armv8,v8"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vfma))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fmla))] +pub unsafe fn vfma_f32(a: float32x2_t, b: float32x2_t, c: float32x2_t) -> float32x2_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.fma.v2f32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.fma.v2f32")] + fn vfma_f32_(a: float32x2_t, b: float32x2_t, c: float32x2_t) -> float32x2_t; + } +vfma_f32_(b, c, a) +} + +/// Floating-point fused Multiply-Add to accumulator(vector) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "fp-armv8,v8"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vfma))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fmla))] +pub unsafe fn vfmaq_f32(a: float32x4_t, b: float32x4_t, c: float32x4_t) -> float32x4_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.fma.v4f32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.fma.v4f32")] + fn vfmaq_f32_(a: float32x4_t, b: float32x4_t, c: float32x4_t) -> float32x4_t; + } +vfmaq_f32_(b, c, a) +} + +/// Floating-point fused Multiply-Add to accumulator(vector) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "fp-armv8,v8"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vfma))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fmla))] +pub unsafe fn vfma_n_f32(a: float32x2_t, b: float32x2_t, c: f32) -> float32x2_t { + vfma_f32(a, b, vdup_n_f32(c)) +} + +/// Floating-point fused Multiply-Add to accumulator(vector) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "fp-armv8,v8"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vfma))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fmla))] +pub unsafe fn vfmaq_n_f32(a: float32x4_t, b: float32x4_t, c: f32) -> float32x4_t { + vfmaq_f32(a, b, vdupq_n_f32(c)) +} + +/// Floating-point fused multiply-subtract from accumulator +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "fp-armv8,v8"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vfms))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fmls))] +pub unsafe fn vfms_f32(a: float32x2_t, b: float32x2_t, c: float32x2_t) -> float32x2_t { + let b: float32x2_t = simd_neg(b); + vfma_f32(a, b, c) +} + +/// Floating-point fused multiply-subtract from accumulator +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "fp-armv8,v8"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vfms))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fmls))] +pub unsafe fn vfmsq_f32(a: float32x4_t, b: float32x4_t, c: float32x4_t) -> float32x4_t { + let b: float32x4_t = simd_neg(b); + vfmaq_f32(a, b, c) +} + +/// Floating-point fused Multiply-subtract to accumulator(vector) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "fp-armv8,v8"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vfms))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fmls))] +pub unsafe fn vfms_n_f32(a: float32x2_t, b: float32x2_t, c: f32) -> float32x2_t { + vfms_f32(a, b, vdup_n_f32(c)) +} + +/// Floating-point fused Multiply-subtract to accumulator(vector) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "fp-armv8,v8"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vfms))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fmls))] +pub unsafe fn vfmsq_n_f32(a: float32x4_t, b: float32x4_t, c: f32) -> float32x4_t { + vfmsq_f32(a, b, vdupq_n_f32(c)) +} + +/// Subtract +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vsub.i8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sub))] +pub unsafe fn vsub_s8(a: int8x8_t, b: int8x8_t) -> int8x8_t { + simd_sub(a, b) +} + +/// Subtract +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vsub.i8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sub))] +pub unsafe fn vsubq_s8(a: int8x16_t, b: int8x16_t) -> int8x16_t { + simd_sub(a, b) +} + +/// Subtract +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vsub.i16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sub))] +pub unsafe fn vsub_s16(a: int16x4_t, b: int16x4_t) -> int16x4_t { + simd_sub(a, b) +} + +/// Subtract +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vsub.i16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sub))] +pub unsafe fn vsubq_s16(a: int16x8_t, b: int16x8_t) -> int16x8_t { + simd_sub(a, b) +} + +/// Subtract +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vsub.i32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sub))] +pub unsafe fn vsub_s32(a: int32x2_t, b: int32x2_t) -> int32x2_t { + simd_sub(a, b) +} + +/// Subtract +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vsub.i32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sub))] +pub unsafe fn vsubq_s32(a: int32x4_t, b: int32x4_t) -> int32x4_t { + simd_sub(a, b) +} + +/// Subtract +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vsub.i8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sub))] +pub unsafe fn vsub_u8(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t { + simd_sub(a, b) +} + +/// Subtract +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vsub.i8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sub))] +pub unsafe fn vsubq_u8(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t { + simd_sub(a, b) +} + +/// Subtract +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vsub.i16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sub))] +pub unsafe fn vsub_u16(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t { + simd_sub(a, b) +} + +/// Subtract +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vsub.i16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sub))] +pub unsafe fn vsubq_u16(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t { + simd_sub(a, b) +} + +/// Subtract +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vsub.i32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sub))] +pub unsafe fn vsub_u32(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t { + simd_sub(a, b) +} + +/// Subtract +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vsub.i32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sub))] +pub unsafe fn vsubq_u32(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t { + simd_sub(a, b) +} + +/// Subtract +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vsub.i64"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sub))] +pub unsafe fn vsub_s64(a: int64x1_t, b: int64x1_t) -> int64x1_t { + simd_sub(a, b) +} + +/// Subtract +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vsub.i64"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sub))] +pub unsafe fn vsubq_s64(a: int64x2_t, b: int64x2_t) -> int64x2_t { + simd_sub(a, b) +} + +/// Subtract +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vsub.i64"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sub))] +pub unsafe fn vsub_u64(a: uint64x1_t, b: uint64x1_t) -> uint64x1_t { + simd_sub(a, b) +} + +/// Subtract +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vsub.i64"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sub))] +pub unsafe fn vsubq_u64(a: uint64x2_t, b: uint64x2_t) -> uint64x2_t { + simd_sub(a, b) +} + +/// Subtract +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vsub.f32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fsub))] +pub unsafe fn vsub_f32(a: float32x2_t, b: float32x2_t) -> float32x2_t { + simd_sub(a, b) +} + +/// Subtract +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vsub.f32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fsub))] +pub unsafe fn vsubq_f32(a: float32x4_t, b: float32x4_t) -> float32x4_t { + simd_sub(a, b) +} + +/// Subtract returning high narrow +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vsubhn))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(subhn))] +pub unsafe fn vsubhn_s16(a: int16x8_t, b: int16x8_t) -> int8x8_t { + let c: i16x8 = i16x8::new(8, 8, 8, 8, 8, 8, 8, 8); + simd_cast(simd_shr(simd_sub(a, b), transmute(c))) +} + +/// Subtract returning high narrow +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vsubhn))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(subhn))] +pub unsafe fn vsubhn_s32(a: int32x4_t, b: int32x4_t) -> int16x4_t { + let c: i32x4 = i32x4::new(16, 16, 16, 16); + simd_cast(simd_shr(simd_sub(a, b), transmute(c))) +} + +/// Subtract returning high narrow +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vsubhn))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(subhn))] +pub unsafe fn vsubhn_s64(a: int64x2_t, b: int64x2_t) -> int32x2_t { + let c: i64x2 = i64x2::new(32, 32); + simd_cast(simd_shr(simd_sub(a, b), transmute(c))) +} + +/// Subtract returning high narrow +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vsubhn))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(subhn))] +pub unsafe fn vsubhn_u16(a: uint16x8_t, b: uint16x8_t) -> uint8x8_t { + let c: u16x8 = u16x8::new(8, 8, 8, 8, 8, 8, 8, 8); + simd_cast(simd_shr(simd_sub(a, b), transmute(c))) +} + +/// Subtract returning high narrow +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vsubhn))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(subhn))] +pub unsafe fn vsubhn_u32(a: uint32x4_t, b: uint32x4_t) -> uint16x4_t { + let c: u32x4 = u32x4::new(16, 16, 16, 16); + simd_cast(simd_shr(simd_sub(a, b), transmute(c))) +} + +/// Subtract returning high narrow +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vsubhn))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(subhn))] +pub unsafe fn vsubhn_u64(a: uint64x2_t, b: uint64x2_t) -> uint32x2_t { + let c: u64x2 = u64x2::new(32, 32); + simd_cast(simd_shr(simd_sub(a, b), transmute(c))) +} + +/// Subtract returning high narrow +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vsubhn))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(subhn2))] +pub unsafe fn vsubhn_high_s16(a: int8x8_t, b: int16x8_t, c: int16x8_t) -> int8x16_t { + let d: int8x8_t = vsubhn_s16(b, c); + simd_shuffle16!(a, d, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]) +} + +/// Subtract returning high narrow +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vsubhn))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(subhn2))] +pub unsafe fn vsubhn_high_s32(a: int16x4_t, b: int32x4_t, c: int32x4_t) -> int16x8_t { + let d: int16x4_t = vsubhn_s32(b, c); + simd_shuffle8!(a, d, [0, 1, 2, 3, 4, 5, 6, 7]) +} + +/// Subtract returning high narrow +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vsubhn))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(subhn2))] +pub unsafe fn vsubhn_high_s64(a: int32x2_t, b: int64x2_t, c: int64x2_t) -> int32x4_t { + let d: int32x2_t = vsubhn_s64(b, c); + simd_shuffle4!(a, d, [0, 1, 2, 3]) +} + +/// Subtract returning high narrow +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vsubhn))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(subhn2))] +pub unsafe fn vsubhn_high_u16(a: uint8x8_t, b: uint16x8_t, c: uint16x8_t) -> uint8x16_t { + let d: uint8x8_t = vsubhn_u16(b, c); + simd_shuffle16!(a, d, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]) +} + +/// Subtract returning high narrow +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vsubhn))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(subhn2))] +pub unsafe fn vsubhn_high_u32(a: uint16x4_t, b: uint32x4_t, c: uint32x4_t) -> uint16x8_t { + let d: uint16x4_t = vsubhn_u32(b, c); + simd_shuffle8!(a, d, [0, 1, 2, 3, 4, 5, 6, 7]) +} + +/// Subtract returning high narrow +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vsubhn))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(subhn2))] +pub unsafe fn vsubhn_high_u64(a: uint32x2_t, b: uint64x2_t, c: uint64x2_t) -> uint32x4_t { + let d: uint32x2_t = vsubhn_u64(b, c); + simd_shuffle4!(a, d, [0, 1, 2, 3]) +} + +/// Signed halving subtract +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vhsub.u8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uhsub))] +pub unsafe fn vhsub_u8(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vhsubu.v8i8")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uhsub.v8i8")] + fn vhsub_u8_(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t; + } +vhsub_u8_(a, b) +} + +/// Signed halving subtract +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vhsub.u8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uhsub))] +pub unsafe fn vhsubq_u8(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vhsubu.v16i8")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uhsub.v16i8")] + fn vhsubq_u8_(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t; + } +vhsubq_u8_(a, b) +} + +/// Signed halving subtract +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vhsub.u16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uhsub))] +pub unsafe fn vhsub_u16(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vhsubu.v4i16")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uhsub.v4i16")] + fn vhsub_u16_(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t; + } +vhsub_u16_(a, b) +} + +/// Signed halving subtract +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vhsub.u16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uhsub))] +pub unsafe fn vhsubq_u16(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vhsubu.v8i16")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uhsub.v8i16")] + fn vhsubq_u16_(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t; + } +vhsubq_u16_(a, b) +} + +/// Signed halving subtract +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vhsub.u32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uhsub))] +pub unsafe fn vhsub_u32(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vhsubu.v2i32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uhsub.v2i32")] + fn vhsub_u32_(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t; + } +vhsub_u32_(a, b) +} + +/// Signed halving subtract +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vhsub.u32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uhsub))] +pub unsafe fn vhsubq_u32(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vhsubu.v4i32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uhsub.v4i32")] + fn vhsubq_u32_(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t; + } +vhsubq_u32_(a, b) +} + +/// Signed halving subtract +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vhsub.s8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(shsub))] +pub unsafe fn vhsub_s8(a: int8x8_t, b: int8x8_t) -> int8x8_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vhsubs.v8i8")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.shsub.v8i8")] + fn vhsub_s8_(a: int8x8_t, b: int8x8_t) -> int8x8_t; + } +vhsub_s8_(a, b) +} + +/// Signed halving subtract +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vhsub.s8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(shsub))] +pub unsafe fn vhsubq_s8(a: int8x16_t, b: int8x16_t) -> int8x16_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vhsubs.v16i8")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.shsub.v16i8")] + fn vhsubq_s8_(a: int8x16_t, b: int8x16_t) -> int8x16_t; + } +vhsubq_s8_(a, b) +} + +/// Signed halving subtract +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vhsub.s16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(shsub))] +pub unsafe fn vhsub_s16(a: int16x4_t, b: int16x4_t) -> int16x4_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vhsubs.v4i16")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.shsub.v4i16")] + fn vhsub_s16_(a: int16x4_t, b: int16x4_t) -> int16x4_t; + } +vhsub_s16_(a, b) +} + +/// Signed halving subtract +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vhsub.s16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(shsub))] +pub unsafe fn vhsubq_s16(a: int16x8_t, b: int16x8_t) -> int16x8_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vhsubs.v8i16")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.shsub.v8i16")] + fn vhsubq_s16_(a: int16x8_t, b: int16x8_t) -> int16x8_t; + } +vhsubq_s16_(a, b) +} + +/// Signed halving subtract +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vhsub.s32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(shsub))] +pub unsafe fn vhsub_s32(a: int32x2_t, b: int32x2_t) -> int32x2_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vhsubs.v2i32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.shsub.v2i32")] + fn vhsub_s32_(a: int32x2_t, b: int32x2_t) -> int32x2_t; + } +vhsub_s32_(a, b) +} + +/// Signed halving subtract +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vhsub.s32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(shsub))] +pub unsafe fn vhsubq_s32(a: int32x4_t, b: int32x4_t) -> int32x4_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vhsubs.v4i32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.shsub.v4i32")] + fn vhsubq_s32_(a: int32x4_t, b: int32x4_t) -> int32x4_t; + } +vhsubq_s32_(a, b) +} + +/// Signed Subtract Wide +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vsubw))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ssubw))] +pub unsafe fn vsubw_s8(a: int16x8_t, b: int8x8_t) -> int16x8_t { + simd_sub(a, simd_cast(b)) +} + +/// Signed Subtract Wide +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vsubw))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ssubw))] +pub unsafe fn vsubw_s16(a: int32x4_t, b: int16x4_t) -> int32x4_t { + simd_sub(a, simd_cast(b)) +} + +/// Signed Subtract Wide +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vsubw))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ssubw))] +pub unsafe fn vsubw_s32(a: int64x2_t, b: int32x2_t) -> int64x2_t { + simd_sub(a, simd_cast(b)) +} + +/// Unsigned Subtract Wide +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vsubw))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(usubw))] +pub unsafe fn vsubw_u8(a: uint16x8_t, b: uint8x8_t) -> uint16x8_t { + simd_sub(a, simd_cast(b)) +} + +/// Unsigned Subtract Wide +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vsubw))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(usubw))] +pub unsafe fn vsubw_u16(a: uint32x4_t, b: uint16x4_t) -> uint32x4_t { + simd_sub(a, simd_cast(b)) +} + +/// Unsigned Subtract Wide +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vsubw))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(usubw))] +pub unsafe fn vsubw_u32(a: uint64x2_t, b: uint32x2_t) -> uint64x2_t { + simd_sub(a, simd_cast(b)) +} + +/// Signed Subtract Long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vsubl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ssubl))] +pub unsafe fn vsubl_s8(a: int8x8_t, b: int8x8_t) -> int16x8_t { + let c: int16x8_t = simd_cast(a); + let d: int16x8_t = simd_cast(b); + simd_sub(c, d) +} + +/// Signed Subtract Long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vsubl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ssubl))] +pub unsafe fn vsubl_s16(a: int16x4_t, b: int16x4_t) -> int32x4_t { + let c: int32x4_t = simd_cast(a); + let d: int32x4_t = simd_cast(b); + simd_sub(c, d) +} + +/// Signed Subtract Long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vsubl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ssubl))] +pub unsafe fn vsubl_s32(a: int32x2_t, b: int32x2_t) -> int64x2_t { + let c: int64x2_t = simd_cast(a); + let d: int64x2_t = simd_cast(b); + simd_sub(c, d) +} + +/// Unsigned Subtract Long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vsubl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(usubl))] +pub unsafe fn vsubl_u8(a: uint8x8_t, b: uint8x8_t) -> uint16x8_t { + let c: uint16x8_t = simd_cast(a); + let d: uint16x8_t = simd_cast(b); + simd_sub(c, d) +} + +/// Unsigned Subtract Long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vsubl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(usubl))] +pub unsafe fn vsubl_u16(a: uint16x4_t, b: uint16x4_t) -> uint32x4_t { + let c: uint32x4_t = simd_cast(a); + let d: uint32x4_t = simd_cast(b); + simd_sub(c, d) +} + +/// Unsigned Subtract Long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vsubl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(usubl))] +pub unsafe fn vsubl_u32(a: uint32x2_t, b: uint32x2_t) -> uint64x2_t { + let c: uint64x2_t = simd_cast(a); + let d: uint64x2_t = simd_cast(b); + simd_sub(c, d) +} + +/// Maximum (vector) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmax))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(smax))] +pub unsafe fn vmax_s8(a: int8x8_t, b: int8x8_t) -> int8x8_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vmaxs.v8i8")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.smax.v8i8")] + fn vmax_s8_(a: int8x8_t, b: int8x8_t) -> int8x8_t; + } +vmax_s8_(a, b) +} + +/// Maximum (vector) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmax))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(smax))] +pub unsafe fn vmaxq_s8(a: int8x16_t, b: int8x16_t) -> int8x16_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vmaxs.v16i8")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.smax.v16i8")] + fn vmaxq_s8_(a: int8x16_t, b: int8x16_t) -> int8x16_t; + } +vmaxq_s8_(a, b) +} + +/// Maximum (vector) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmax))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(smax))] +pub unsafe fn vmax_s16(a: int16x4_t, b: int16x4_t) -> int16x4_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vmaxs.v4i16")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.smax.v4i16")] + fn vmax_s16_(a: int16x4_t, b: int16x4_t) -> int16x4_t; + } +vmax_s16_(a, b) +} + +/// Maximum (vector) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmax))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(smax))] +pub unsafe fn vmaxq_s16(a: int16x8_t, b: int16x8_t) -> int16x8_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vmaxs.v8i16")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.smax.v8i16")] + fn vmaxq_s16_(a: int16x8_t, b: int16x8_t) -> int16x8_t; + } +vmaxq_s16_(a, b) +} + +/// Maximum (vector) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmax))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(smax))] +pub unsafe fn vmax_s32(a: int32x2_t, b: int32x2_t) -> int32x2_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vmaxs.v2i32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.smax.v2i32")] + fn vmax_s32_(a: int32x2_t, b: int32x2_t) -> int32x2_t; + } +vmax_s32_(a, b) +} + +/// Maximum (vector) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmax))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(smax))] +pub unsafe fn vmaxq_s32(a: int32x4_t, b: int32x4_t) -> int32x4_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vmaxs.v4i32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.smax.v4i32")] + fn vmaxq_s32_(a: int32x4_t, b: int32x4_t) -> int32x4_t; + } +vmaxq_s32_(a, b) +} + +/// Maximum (vector) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmax))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(umax))] +pub unsafe fn vmax_u8(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vmaxu.v8i8")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.umax.v8i8")] + fn vmax_u8_(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t; + } +vmax_u8_(a, b) +} + +/// Maximum (vector) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmax))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(umax))] +pub unsafe fn vmaxq_u8(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vmaxu.v16i8")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.umax.v16i8")] + fn vmaxq_u8_(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t; + } +vmaxq_u8_(a, b) +} + +/// Maximum (vector) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmax))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(umax))] +pub unsafe fn vmax_u16(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vmaxu.v4i16")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.umax.v4i16")] + fn vmax_u16_(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t; + } +vmax_u16_(a, b) +} + +/// Maximum (vector) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmax))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(umax))] +pub unsafe fn vmaxq_u16(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vmaxu.v8i16")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.umax.v8i16")] + fn vmaxq_u16_(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t; + } +vmaxq_u16_(a, b) +} + +/// Maximum (vector) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmax))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(umax))] +pub unsafe fn vmax_u32(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vmaxu.v2i32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.umax.v2i32")] + fn vmax_u32_(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t; + } +vmax_u32_(a, b) +} + +/// Maximum (vector) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmax))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(umax))] +pub unsafe fn vmaxq_u32(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vmaxu.v4i32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.umax.v4i32")] + fn vmaxq_u32_(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t; + } +vmaxq_u32_(a, b) +} + +/// Maximum (vector) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmax))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fmax))] +pub unsafe fn vmax_f32(a: float32x2_t, b: float32x2_t) -> float32x2_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vmaxs.v2f32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fmax.v2f32")] + fn vmax_f32_(a: float32x2_t, b: float32x2_t) -> float32x2_t; + } +vmax_f32_(a, b) +} + +/// Maximum (vector) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmax))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fmax))] +pub unsafe fn vmaxq_f32(a: float32x4_t, b: float32x4_t) -> float32x4_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vmaxs.v4f32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fmax.v4f32")] + fn vmaxq_f32_(a: float32x4_t, b: float32x4_t) -> float32x4_t; + } +vmaxq_f32_(a, b) +} + +/// Floating-point Maximun Number (vector) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "fp-armv8,v8"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmaxnm))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fmaxnm))] +pub unsafe fn vmaxnm_f32(a: float32x2_t, b: float32x2_t) -> float32x2_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vmaxnm.v2f32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fmaxnm.v2f32")] + fn vmaxnm_f32_(a: float32x2_t, b: float32x2_t) -> float32x2_t; + } +vmaxnm_f32_(a, b) +} + +/// Floating-point Maximun Number (vector) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "fp-armv8,v8"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmaxnm))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fmaxnm))] +pub unsafe fn vmaxnmq_f32(a: float32x4_t, b: float32x4_t) -> float32x4_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vmaxnm.v4f32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fmaxnm.v4f32")] + fn vmaxnmq_f32_(a: float32x4_t, b: float32x4_t) -> float32x4_t; + } +vmaxnmq_f32_(a, b) +} + +/// Minimum (vector) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmin))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(smin))] +pub unsafe fn vmin_s8(a: int8x8_t, b: int8x8_t) -> int8x8_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vmins.v8i8")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.smin.v8i8")] + fn vmin_s8_(a: int8x8_t, b: int8x8_t) -> int8x8_t; + } +vmin_s8_(a, b) +} + +/// Minimum (vector) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmin))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(smin))] +pub unsafe fn vminq_s8(a: int8x16_t, b: int8x16_t) -> int8x16_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vmins.v16i8")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.smin.v16i8")] + fn vminq_s8_(a: int8x16_t, b: int8x16_t) -> int8x16_t; + } +vminq_s8_(a, b) +} + +/// Minimum (vector) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmin))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(smin))] +pub unsafe fn vmin_s16(a: int16x4_t, b: int16x4_t) -> int16x4_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vmins.v4i16")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.smin.v4i16")] + fn vmin_s16_(a: int16x4_t, b: int16x4_t) -> int16x4_t; + } +vmin_s16_(a, b) +} + +/// Minimum (vector) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmin))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(smin))] +pub unsafe fn vminq_s16(a: int16x8_t, b: int16x8_t) -> int16x8_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vmins.v8i16")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.smin.v8i16")] + fn vminq_s16_(a: int16x8_t, b: int16x8_t) -> int16x8_t; + } +vminq_s16_(a, b) +} + +/// Minimum (vector) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmin))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(smin))] +pub unsafe fn vmin_s32(a: int32x2_t, b: int32x2_t) -> int32x2_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vmins.v2i32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.smin.v2i32")] + fn vmin_s32_(a: int32x2_t, b: int32x2_t) -> int32x2_t; + } +vmin_s32_(a, b) +} + +/// Minimum (vector) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmin))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(smin))] +pub unsafe fn vminq_s32(a: int32x4_t, b: int32x4_t) -> int32x4_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vmins.v4i32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.smin.v4i32")] + fn vminq_s32_(a: int32x4_t, b: int32x4_t) -> int32x4_t; + } +vminq_s32_(a, b) +} + +/// Minimum (vector) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmin))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(umin))] +pub unsafe fn vmin_u8(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vminu.v8i8")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.umin.v8i8")] + fn vmin_u8_(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t; + } +vmin_u8_(a, b) +} + +/// Minimum (vector) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmin))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(umin))] +pub unsafe fn vminq_u8(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vminu.v16i8")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.umin.v16i8")] + fn vminq_u8_(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t; + } +vminq_u8_(a, b) +} + +/// Minimum (vector) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmin))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(umin))] +pub unsafe fn vmin_u16(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vminu.v4i16")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.umin.v4i16")] + fn vmin_u16_(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t; + } +vmin_u16_(a, b) +} + +/// Minimum (vector) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmin))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(umin))] +pub unsafe fn vminq_u16(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vminu.v8i16")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.umin.v8i16")] + fn vminq_u16_(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t; + } +vminq_u16_(a, b) +} + +/// Minimum (vector) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmin))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(umin))] +pub unsafe fn vmin_u32(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vminu.v2i32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.umin.v2i32")] + fn vmin_u32_(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t; + } +vmin_u32_(a, b) +} + +/// Minimum (vector) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmin))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(umin))] +pub unsafe fn vminq_u32(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vminu.v4i32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.umin.v4i32")] + fn vminq_u32_(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t; + } +vminq_u32_(a, b) +} + +/// Minimum (vector) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmin))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fmin))] +pub unsafe fn vmin_f32(a: float32x2_t, b: float32x2_t) -> float32x2_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vmins.v2f32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fmin.v2f32")] + fn vmin_f32_(a: float32x2_t, b: float32x2_t) -> float32x2_t; + } +vmin_f32_(a, b) +} + +/// Minimum (vector) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmin))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fmin))] +pub unsafe fn vminq_f32(a: float32x4_t, b: float32x4_t) -> float32x4_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vmins.v4f32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fmin.v4f32")] + fn vminq_f32_(a: float32x4_t, b: float32x4_t) -> float32x4_t; + } +vminq_f32_(a, b) +} + +/// Floating-point Minimun Number (vector) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "fp-armv8,v8"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vminnm))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fminnm))] +pub unsafe fn vminnm_f32(a: float32x2_t, b: float32x2_t) -> float32x2_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vminnm.v2f32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fminnm.v2f32")] + fn vminnm_f32_(a: float32x2_t, b: float32x2_t) -> float32x2_t; + } +vminnm_f32_(a, b) +} + +/// Floating-point Minimun Number (vector) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "fp-armv8,v8"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vminnm))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fminnm))] +pub unsafe fn vminnmq_f32(a: float32x4_t, b: float32x4_t) -> float32x4_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vminnm.v4f32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.fminnm.v4f32")] + fn vminnmq_f32_(a: float32x4_t, b: float32x4_t) -> float32x4_t; + } +vminnmq_f32_(a, b) +} + +/// Signed saturating doubling multiply long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqdmull))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqdmull))] +pub unsafe fn vqdmull_s16(a: int16x4_t, b: int16x4_t) -> int32x4_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqdmull.v4i32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqdmull.v4i32")] + fn vqdmull_s16_(a: int16x4_t, b: int16x4_t) -> int32x4_t; + } +vqdmull_s16_(a, b) +} + +/// Signed saturating doubling multiply long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqdmull))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqdmull))] +pub unsafe fn vqdmull_s32(a: int32x2_t, b: int32x2_t) -> int64x2_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqdmull.v2i64")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqdmull.v2i64")] + fn vqdmull_s32_(a: int32x2_t, b: int32x2_t) -> int64x2_t; + } +vqdmull_s32_(a, b) +} + +/// Vector saturating doubling long multiply with scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqdmull))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqdmull))] +pub unsafe fn vqdmull_n_s16(a: int16x4_t, b: i16) -> int32x4_t { + vqdmull_s16(a, vdup_n_s16(b)) +} + +/// Vector saturating doubling long multiply with scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqdmull))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqdmull))] +pub unsafe fn vqdmull_n_s32(a: int32x2_t, b: i32) -> int64x2_t { + vqdmull_s32(a, vdup_n_s32(b)) +} + +/// Vector saturating doubling long multiply by scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqdmull, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqdmull, N = 2))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vqdmull_lane_s16(a: int16x4_t, b: int16x4_t) -> int32x4_t { + static_assert_imm2!(N); + let b: int16x4_t = simd_shuffle4!(b, b, [N as u32, N as u32, N as u32, N as u32]); + vqdmull_s16(a, b) +} + +/// Vector saturating doubling long multiply by scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqdmull, N = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqdmull, N = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vqdmull_lane_s32(a: int32x2_t, b: int32x2_t) -> int64x2_t { + static_assert_imm1!(N); + let b: int32x2_t = simd_shuffle2!(b, b, [N as u32, N as u32]); + vqdmull_s32(a, b) +} + +/// Signed saturating doubling multiply-add long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqdmlal))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqdmlal))] +pub unsafe fn vqdmlal_s16(a: int32x4_t, b: int16x4_t, c: int16x4_t) -> int32x4_t { + vqaddq_s32(a, vqdmull_s16(b, c)) +} + +/// Signed saturating doubling multiply-add long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqdmlal))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqdmlal))] +pub unsafe fn vqdmlal_s32(a: int64x2_t, b: int32x2_t, c: int32x2_t) -> int64x2_t { + vqaddq_s64(a, vqdmull_s32(b, c)) +} + +/// Vector widening saturating doubling multiply accumulate with scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqdmlal))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqdmlal))] +pub unsafe fn vqdmlal_n_s16(a: int32x4_t, b: int16x4_t, c: i16) -> int32x4_t { + vqaddq_s32(a, vqdmull_n_s16(b, c)) +} + +/// Vector widening saturating doubling multiply accumulate with scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqdmlal))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqdmlal))] +pub unsafe fn vqdmlal_n_s32(a: int64x2_t, b: int32x2_t, c: i32) -> int64x2_t { + vqaddq_s64(a, vqdmull_n_s32(b, c)) +} + +/// Vector widening saturating doubling multiply accumulate with scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqdmlal, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqdmlal, N = 2))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vqdmlal_lane_s16(a: int32x4_t, b: int16x4_t, c: int16x4_t) -> int32x4_t { + static_assert_imm2!(N); + vqaddq_s32(a, vqdmull_lane_s16::(b, c)) +} + +/// Vector widening saturating doubling multiply accumulate with scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqdmlal, N = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqdmlal, N = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vqdmlal_lane_s32(a: int64x2_t, b: int32x2_t, c: int32x2_t) -> int64x2_t { + static_assert_imm1!(N); + vqaddq_s64(a, vqdmull_lane_s32::(b, c)) +} + +/// Signed saturating doubling multiply-subtract long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqdmlsl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqdmlsl))] +pub unsafe fn vqdmlsl_s16(a: int32x4_t, b: int16x4_t, c: int16x4_t) -> int32x4_t { + vqsubq_s32(a, vqdmull_s16(b, c)) +} + +/// Signed saturating doubling multiply-subtract long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqdmlsl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqdmlsl))] +pub unsafe fn vqdmlsl_s32(a: int64x2_t, b: int32x2_t, c: int32x2_t) -> int64x2_t { + vqsubq_s64(a, vqdmull_s32(b, c)) +} + +/// Vector widening saturating doubling multiply subtract with scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqdmlsl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqdmlsl))] +pub unsafe fn vqdmlsl_n_s16(a: int32x4_t, b: int16x4_t, c: i16) -> int32x4_t { + vqsubq_s32(a, vqdmull_n_s16(b, c)) +} + +/// Vector widening saturating doubling multiply subtract with scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqdmlsl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqdmlsl))] +pub unsafe fn vqdmlsl_n_s32(a: int64x2_t, b: int32x2_t, c: i32) -> int64x2_t { + vqsubq_s64(a, vqdmull_n_s32(b, c)) +} + +/// Vector widening saturating doubling multiply subtract with scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqdmlsl, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqdmlsl, N = 2))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vqdmlsl_lane_s16(a: int32x4_t, b: int16x4_t, c: int16x4_t) -> int32x4_t { + static_assert_imm2!(N); + vqsubq_s32(a, vqdmull_lane_s16::(b, c)) +} + +/// Vector widening saturating doubling multiply subtract with scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqdmlsl, N = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqdmlsl, N = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vqdmlsl_lane_s32(a: int64x2_t, b: int32x2_t, c: int32x2_t) -> int64x2_t { + static_assert_imm1!(N); + vqsubq_s64(a, vqdmull_lane_s32::(b, c)) +} + +/// Signed saturating doubling multiply returning high half +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqdmulh))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqdmulh))] +pub unsafe fn vqdmulh_s16(a: int16x4_t, b: int16x4_t) -> int16x4_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqdmulh.v4i16")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqdmulh.v4i16")] + fn vqdmulh_s16_(a: int16x4_t, b: int16x4_t) -> int16x4_t; + } +vqdmulh_s16_(a, b) +} + +/// Signed saturating doubling multiply returning high half +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqdmulh))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqdmulh))] +pub unsafe fn vqdmulhq_s16(a: int16x8_t, b: int16x8_t) -> int16x8_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqdmulh.v8i16")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqdmulh.v8i16")] + fn vqdmulhq_s16_(a: int16x8_t, b: int16x8_t) -> int16x8_t; + } +vqdmulhq_s16_(a, b) +} + +/// Signed saturating doubling multiply returning high half +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqdmulh))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqdmulh))] +pub unsafe fn vqdmulh_s32(a: int32x2_t, b: int32x2_t) -> int32x2_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqdmulh.v2i32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqdmulh.v2i32")] + fn vqdmulh_s32_(a: int32x2_t, b: int32x2_t) -> int32x2_t; + } +vqdmulh_s32_(a, b) +} + +/// Signed saturating doubling multiply returning high half +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqdmulh))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqdmulh))] +pub unsafe fn vqdmulhq_s32(a: int32x4_t, b: int32x4_t) -> int32x4_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqdmulh.v4i32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqdmulh.v4i32")] + fn vqdmulhq_s32_(a: int32x4_t, b: int32x4_t) -> int32x4_t; + } +vqdmulhq_s32_(a, b) +} + +/// Vector saturating doubling multiply high with scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqdmulh))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqdmulh))] +pub unsafe fn vqdmulh_n_s16(a: int16x4_t, b: i16) -> int16x4_t { + let b: int16x4_t = vdup_n_s16(b); + vqdmulh_s16(a, b) +} + +/// Vector saturating doubling multiply high with scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqdmulh))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqdmulh))] +pub unsafe fn vqdmulh_n_s32(a: int32x2_t, b: i32) -> int32x2_t { + let b: int32x2_t = vdup_n_s32(b); + vqdmulh_s32(a, b) +} + +/// Vector saturating doubling multiply high with scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqdmulh))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqdmulh))] +pub unsafe fn vqdmulhq_nq_s16(a: int16x8_t, b: i16) -> int16x8_t { + let b: int16x8_t = vdupq_n_s16(b); + vqdmulhq_s16(a, b) +} + +/// Vector saturating doubling multiply high with scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqdmulh))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqdmulh))] +pub unsafe fn vqdmulhq_nq_s32(a: int32x4_t, b: i32) -> int32x4_t { + let b: int32x4_t = vdupq_n_s32(b); + vqdmulhq_s32(a, b) +} + +/// Signed saturating extract narrow +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqmovn))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqxtn))] +pub unsafe fn vqmovn_s16(a: int16x8_t) -> int8x8_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqmovns.v8i8")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqxtn.v8i8")] + fn vqmovn_s16_(a: int16x8_t) -> int8x8_t; + } +vqmovn_s16_(a) +} + +/// Signed saturating extract narrow +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqmovn))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqxtn))] +pub unsafe fn vqmovn_s32(a: int32x4_t) -> int16x4_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqmovns.v4i16")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqxtn.v4i16")] + fn vqmovn_s32_(a: int32x4_t) -> int16x4_t; + } +vqmovn_s32_(a) +} + +/// Signed saturating extract narrow +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqmovn))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqxtn))] +pub unsafe fn vqmovn_s64(a: int64x2_t) -> int32x2_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqmovns.v2i32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqxtn.v2i32")] + fn vqmovn_s64_(a: int64x2_t) -> int32x2_t; + } +vqmovn_s64_(a) +} + +/// Unsigned saturating extract narrow +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqmovn))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uqxtn))] +pub unsafe fn vqmovn_u16(a: uint16x8_t) -> uint8x8_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqmovnu.v8i8")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uqxtn.v8i8")] + fn vqmovn_u16_(a: uint16x8_t) -> uint8x8_t; + } +vqmovn_u16_(a) +} + +/// Unsigned saturating extract narrow +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqmovn))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uqxtn))] +pub unsafe fn vqmovn_u32(a: uint32x4_t) -> uint16x4_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqmovnu.v4i16")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uqxtn.v4i16")] + fn vqmovn_u32_(a: uint32x4_t) -> uint16x4_t; + } +vqmovn_u32_(a) +} + +/// Unsigned saturating extract narrow +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqmovn))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uqxtn))] +pub unsafe fn vqmovn_u64(a: uint64x2_t) -> uint32x2_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqmovnu.v2i32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uqxtn.v2i32")] + fn vqmovn_u64_(a: uint64x2_t) -> uint32x2_t; + } +vqmovn_u64_(a) +} + +/// Signed saturating extract unsigned narrow +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqmovun))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqxtun))] +pub unsafe fn vqmovun_s16(a: int16x8_t) -> uint8x8_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqmovnsu.v8i8")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqxtun.v8i8")] + fn vqmovun_s16_(a: int16x8_t) -> uint8x8_t; + } +vqmovun_s16_(a) +} + +/// Signed saturating extract unsigned narrow +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqmovun))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqxtun))] +pub unsafe fn vqmovun_s32(a: int32x4_t) -> uint16x4_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqmovnsu.v4i16")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqxtun.v4i16")] + fn vqmovun_s32_(a: int32x4_t) -> uint16x4_t; + } +vqmovun_s32_(a) +} + +/// Signed saturating extract unsigned narrow +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqmovun))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqxtun))] +pub unsafe fn vqmovun_s64(a: int64x2_t) -> uint32x2_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqmovnsu.v2i32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqxtun.v2i32")] + fn vqmovun_s64_(a: int64x2_t) -> uint32x2_t; + } +vqmovun_s64_(a) +} + +/// Signed saturating rounding doubling multiply returning high half +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqrdmulh))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqrdmulh))] +pub unsafe fn vqrdmulh_s16(a: int16x4_t, b: int16x4_t) -> int16x4_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqrdmulh.v4i16")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqrdmulh.v4i16")] + fn vqrdmulh_s16_(a: int16x4_t, b: int16x4_t) -> int16x4_t; + } +vqrdmulh_s16_(a, b) +} + +/// Signed saturating rounding doubling multiply returning high half +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqrdmulh))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqrdmulh))] +pub unsafe fn vqrdmulhq_s16(a: int16x8_t, b: int16x8_t) -> int16x8_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqrdmulh.v8i16")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqrdmulh.v8i16")] + fn vqrdmulhq_s16_(a: int16x8_t, b: int16x8_t) -> int16x8_t; + } +vqrdmulhq_s16_(a, b) +} + +/// Signed saturating rounding doubling multiply returning high half +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqrdmulh))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqrdmulh))] +pub unsafe fn vqrdmulh_s32(a: int32x2_t, b: int32x2_t) -> int32x2_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqrdmulh.v2i32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqrdmulh.v2i32")] + fn vqrdmulh_s32_(a: int32x2_t, b: int32x2_t) -> int32x2_t; + } +vqrdmulh_s32_(a, b) +} + +/// Signed saturating rounding doubling multiply returning high half +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqrdmulh))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqrdmulh))] +pub unsafe fn vqrdmulhq_s32(a: int32x4_t, b: int32x4_t) -> int32x4_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqrdmulh.v4i32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqrdmulh.v4i32")] + fn vqrdmulhq_s32_(a: int32x4_t, b: int32x4_t) -> int32x4_t; + } +vqrdmulhq_s32_(a, b) +} + +/// Vector saturating rounding doubling multiply high with scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqrdmulh))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqrdmulh))] +pub unsafe fn vqrdmulh_n_s16(a: int16x4_t, b: i16) -> int16x4_t { + vqrdmulh_s16(a, vdup_n_s16(b)) +} + +/// Vector saturating rounding doubling multiply high with scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqrdmulh))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqrdmulh))] +pub unsafe fn vqrdmulhq_n_s16(a: int16x8_t, b: i16) -> int16x8_t { + vqrdmulhq_s16(a, vdupq_n_s16(b)) +} + +/// Vector saturating rounding doubling multiply high with scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqrdmulh))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqrdmulh))] +pub unsafe fn vqrdmulh_n_s32(a: int32x2_t, b: i32) -> int32x2_t { + vqrdmulh_s32(a, vdup_n_s32(b)) +} + +/// Vector saturating rounding doubling multiply high with scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqrdmulh))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqrdmulh))] +pub unsafe fn vqrdmulhq_n_s32(a: int32x4_t, b: i32) -> int32x4_t { + vqrdmulhq_s32(a, vdupq_n_s32(b)) +} + +/// Vector rounding saturating doubling multiply high by scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqrdmulh, LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqrdmulh, LANE = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vqrdmulh_lane_s16(a: int16x4_t, b: int16x4_t) -> int16x4_t { + static_assert_imm2!(LANE); + let b: int16x4_t = simd_shuffle4!(b, b, [LANE as u32, LANE as u32, LANE as u32, LANE as u32]); + vqrdmulh_s16(a, b) +} + +/// Vector rounding saturating doubling multiply high by scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqrdmulh, LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqrdmulh, LANE = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vqrdmulh_laneq_s16(a: int16x4_t, b: int16x8_t) -> int16x4_t { + static_assert_imm3!(LANE); + let b: int16x4_t = simd_shuffle4!(b, b, [LANE as u32, LANE as u32, LANE as u32, LANE as u32]); + vqrdmulh_s16(a, b) +} + +/// Vector rounding saturating doubling multiply high by scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqrdmulh, LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqrdmulh, LANE = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vqrdmulhq_lane_s16(a: int16x8_t, b: int16x4_t) -> int16x8_t { + static_assert_imm2!(LANE); + let b: int16x8_t = simd_shuffle8!(b, b, [LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32]); + vqrdmulhq_s16(a, b) +} + +/// Vector rounding saturating doubling multiply high by scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqrdmulh, LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqrdmulh, LANE = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vqrdmulhq_laneq_s16(a: int16x8_t, b: int16x8_t) -> int16x8_t { + static_assert_imm3!(LANE); + let b: int16x8_t = simd_shuffle8!(b, b, [LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32]); + vqrdmulhq_s16(a, b) +} + +/// Vector rounding saturating doubling multiply high by scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqrdmulh, LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqrdmulh, LANE = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vqrdmulh_lane_s32(a: int32x2_t, b: int32x2_t) -> int32x2_t { + static_assert_imm1!(LANE); + let b: int32x2_t = simd_shuffle2!(b, b, [LANE as u32, LANE as u32]); + vqrdmulh_s32(a, b) +} + +/// Vector rounding saturating doubling multiply high by scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqrdmulh, LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqrdmulh, LANE = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vqrdmulh_laneq_s32(a: int32x2_t, b: int32x4_t) -> int32x2_t { + static_assert_imm2!(LANE); + let b: int32x2_t = simd_shuffle2!(b, b, [LANE as u32, LANE as u32]); + vqrdmulh_s32(a, b) +} + +/// Vector rounding saturating doubling multiply high by scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqrdmulh, LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqrdmulh, LANE = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vqrdmulhq_lane_s32(a: int32x4_t, b: int32x2_t) -> int32x4_t { + static_assert_imm1!(LANE); + let b: int32x4_t = simd_shuffle4!(b, b, [LANE as u32, LANE as u32, LANE as u32, LANE as u32]); + vqrdmulhq_s32(a, b) +} + +/// Vector rounding saturating doubling multiply high by scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqrdmulh, LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqrdmulh, LANE = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vqrdmulhq_laneq_s32(a: int32x4_t, b: int32x4_t) -> int32x4_t { + static_assert_imm2!(LANE); + let b: int32x4_t = simd_shuffle4!(b, b, [LANE as u32, LANE as u32, LANE as u32, LANE as u32]); + vqrdmulhq_s32(a, b) +} + +/// Signed saturating rounding doubling multiply accumulate returning high half +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqrdmulh))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqrdmulh))] +pub unsafe fn vqrdmlah_s16(a: int16x4_t, b: int16x4_t, c: int16x4_t) -> int16x4_t { + vqadd_s16(a, vqrdmulh_s16(b, c)) +} + +/// Signed saturating rounding doubling multiply accumulate returning high half +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqrdmulh))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqrdmulh))] +pub unsafe fn vqrdmlahq_s16(a: int16x8_t, b: int16x8_t, c: int16x8_t) -> int16x8_t { + vqaddq_s16(a, vqrdmulhq_s16(b, c)) +} + +/// Signed saturating rounding doubling multiply accumulate returning high half +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqrdmulh))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqrdmulh))] +pub unsafe fn vqrdmlah_s32(a: int32x2_t, b: int32x2_t, c: int32x2_t) -> int32x2_t { + vqadd_s32(a, vqrdmulh_s32(b, c)) +} + +/// Signed saturating rounding doubling multiply accumulate returning high half +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqrdmulh))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqrdmulh))] +pub unsafe fn vqrdmlahq_s32(a: int32x4_t, b: int32x4_t, c: int32x4_t) -> int32x4_t { + vqaddq_s32(a, vqrdmulhq_s32(b, c)) +} + +/// Signed saturating rounding doubling multiply accumulate returning high half +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqrdmulh, LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqrdmulh, LANE = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vqrdmlah_lane_s16(a: int16x4_t, b: int16x4_t, c: int16x4_t) -> int16x4_t { + static_assert_imm2!(LANE); + vqadd_s16(a, vqrdmulh_lane_s16::(b, c)) +} + +/// Signed saturating rounding doubling multiply accumulate returning high half +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqrdmulh, LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqrdmulh, LANE = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vqrdmlah_laneq_s16(a: int16x4_t, b: int16x4_t, c: int16x8_t) -> int16x4_t { + static_assert_imm3!(LANE); + vqadd_s16(a, vqrdmulh_laneq_s16::(b, c)) +} + +/// Signed saturating rounding doubling multiply accumulate returning high half +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqrdmulh, LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqrdmulh, LANE = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vqrdmlahq_lane_s16(a: int16x8_t, b: int16x8_t, c: int16x4_t) -> int16x8_t { + static_assert_imm2!(LANE); + vqaddq_s16(a, vqrdmulhq_lane_s16::(b, c)) +} + +/// Signed saturating rounding doubling multiply accumulate returning high half +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqrdmulh, LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqrdmulh, LANE = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vqrdmlahq_laneq_s16(a: int16x8_t, b: int16x8_t, c: int16x8_t) -> int16x8_t { + static_assert_imm3!(LANE); + vqaddq_s16(a, vqrdmulhq_laneq_s16::(b, c)) +} + +/// Signed saturating rounding doubling multiply accumulate returning high half +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqrdmulh, LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqrdmulh, LANE = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vqrdmlah_lane_s32(a: int32x2_t, b: int32x2_t, c: int32x2_t) -> int32x2_t { + static_assert_imm1!(LANE); + vqadd_s32(a, vqrdmulh_lane_s32::(b, c)) +} + +/// Signed saturating rounding doubling multiply accumulate returning high half +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqrdmulh, LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqrdmulh, LANE = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vqrdmlah_laneq_s32(a: int32x2_t, b: int32x2_t, c: int32x4_t) -> int32x2_t { + static_assert_imm2!(LANE); + vqadd_s32(a, vqrdmulh_laneq_s32::(b, c)) +} + +/// Signed saturating rounding doubling multiply accumulate returning high half +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqrdmulh, LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqrdmulh, LANE = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vqrdmlahq_lane_s32(a: int32x4_t, b: int32x4_t, c: int32x2_t) -> int32x4_t { + static_assert_imm1!(LANE); + vqaddq_s32(a, vqrdmulhq_lane_s32::(b, c)) +} + +/// Signed saturating rounding doubling multiply accumulate returning high half +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqrdmulh, LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqrdmulh, LANE = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vqrdmlahq_laneq_s32(a: int32x4_t, b: int32x4_t, c: int32x4_t) -> int32x4_t { + static_assert_imm2!(LANE); + vqaddq_s32(a, vqrdmulhq_laneq_s32::(b, c)) +} + +/// Signed saturating rounding doubling multiply subtract returning high half +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqrdmulh))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqrdmulh))] +pub unsafe fn vqrdmlsh_s16(a: int16x4_t, b: int16x4_t, c: int16x4_t) -> int16x4_t { + vqsub_s16(a, vqrdmulh_s16(b, c)) +} + +/// Signed saturating rounding doubling multiply subtract returning high half +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqrdmulh))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqrdmulh))] +pub unsafe fn vqrdmlshq_s16(a: int16x8_t, b: int16x8_t, c: int16x8_t) -> int16x8_t { + vqsubq_s16(a, vqrdmulhq_s16(b, c)) +} + +/// Signed saturating rounding doubling multiply subtract returning high half +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqrdmulh))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqrdmulh))] +pub unsafe fn vqrdmlsh_s32(a: int32x2_t, b: int32x2_t, c: int32x2_t) -> int32x2_t { + vqsub_s32(a, vqrdmulh_s32(b, c)) +} + +/// Signed saturating rounding doubling multiply subtract returning high half +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqrdmulh))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqrdmulh))] +pub unsafe fn vqrdmlshq_s32(a: int32x4_t, b: int32x4_t, c: int32x4_t) -> int32x4_t { + vqsubq_s32(a, vqrdmulhq_s32(b, c)) +} + +/// Signed saturating rounding doubling multiply subtract returning high half +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqrdmulh, LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqrdmulh, LANE = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vqrdmlsh_lane_s16(a: int16x4_t, b: int16x4_t, c: int16x4_t) -> int16x4_t { + static_assert_imm2!(LANE); + vqsub_s16(a, vqrdmulh_lane_s16::(b, c)) +} + +/// Signed saturating rounding doubling multiply subtract returning high half +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqrdmulh, LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqrdmulh, LANE = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vqrdmlsh_laneq_s16(a: int16x4_t, b: int16x4_t, c: int16x8_t) -> int16x4_t { + static_assert_imm3!(LANE); + vqsub_s16(a, vqrdmulh_laneq_s16::(b, c)) +} + +/// Signed saturating rounding doubling multiply subtract returning high half +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqrdmulh, LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqrdmulh, LANE = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vqrdmlshq_lane_s16(a: int16x8_t, b: int16x8_t, c: int16x4_t) -> int16x8_t { + static_assert_imm2!(LANE); + vqsubq_s16(a, vqrdmulhq_lane_s16::(b, c)) +} + +/// Signed saturating rounding doubling multiply subtract returning high half +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqrdmulh, LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqrdmulh, LANE = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vqrdmlshq_laneq_s16(a: int16x8_t, b: int16x8_t, c: int16x8_t) -> int16x8_t { + static_assert_imm3!(LANE); + vqsubq_s16(a, vqrdmulhq_laneq_s16::(b, c)) +} + +/// Signed saturating rounding doubling multiply subtract returning high half +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqrdmulh, LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqrdmulh, LANE = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vqrdmlsh_lane_s32(a: int32x2_t, b: int32x2_t, c: int32x2_t) -> int32x2_t { + static_assert_imm1!(LANE); + vqsub_s32(a, vqrdmulh_lane_s32::(b, c)) +} + +/// Signed saturating rounding doubling multiply subtract returning high half +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqrdmulh, LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqrdmulh, LANE = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vqrdmlsh_laneq_s32(a: int32x2_t, b: int32x2_t, c: int32x4_t) -> int32x2_t { + static_assert_imm2!(LANE); + vqsub_s32(a, vqrdmulh_laneq_s32::(b, c)) +} + +/// Signed saturating rounding doubling multiply subtract returning high half +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqrdmulh, LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqrdmulh, LANE = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vqrdmlshq_lane_s32(a: int32x4_t, b: int32x4_t, c: int32x2_t) -> int32x4_t { + static_assert_imm1!(LANE); + vqsubq_s32(a, vqrdmulhq_lane_s32::(b, c)) +} + +/// Signed saturating rounding doubling multiply subtract returning high half +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqrdmulh, LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqrdmulh, LANE = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn vqrdmlshq_laneq_s32(a: int32x4_t, b: int32x4_t, c: int32x4_t) -> int32x4_t { + static_assert_imm2!(LANE); + vqsubq_s32(a, vqrdmulhq_laneq_s32::(b, c)) +} + +/// Signed saturating rounding shift left +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqrshl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqrshl))] +pub unsafe fn vqrshl_s8(a: int8x8_t, b: int8x8_t) -> int8x8_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqrshifts.v8i8")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqrshl.v8i8")] + fn vqrshl_s8_(a: int8x8_t, b: int8x8_t) -> int8x8_t; + } +vqrshl_s8_(a, b) +} + +/// Signed saturating rounding shift left +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqrshl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqrshl))] +pub unsafe fn vqrshlq_s8(a: int8x16_t, b: int8x16_t) -> int8x16_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqrshifts.v16i8")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqrshl.v16i8")] + fn vqrshlq_s8_(a: int8x16_t, b: int8x16_t) -> int8x16_t; + } +vqrshlq_s8_(a, b) +} + +/// Signed saturating rounding shift left +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqrshl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqrshl))] +pub unsafe fn vqrshl_s16(a: int16x4_t, b: int16x4_t) -> int16x4_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqrshifts.v4i16")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqrshl.v4i16")] + fn vqrshl_s16_(a: int16x4_t, b: int16x4_t) -> int16x4_t; + } +vqrshl_s16_(a, b) +} + +/// Signed saturating rounding shift left +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqrshl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqrshl))] +pub unsafe fn vqrshlq_s16(a: int16x8_t, b: int16x8_t) -> int16x8_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqrshifts.v8i16")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqrshl.v8i16")] + fn vqrshlq_s16_(a: int16x8_t, b: int16x8_t) -> int16x8_t; + } +vqrshlq_s16_(a, b) +} + +/// Signed saturating rounding shift left +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqrshl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqrshl))] +pub unsafe fn vqrshl_s32(a: int32x2_t, b: int32x2_t) -> int32x2_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqrshifts.v2i32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqrshl.v2i32")] + fn vqrshl_s32_(a: int32x2_t, b: int32x2_t) -> int32x2_t; + } +vqrshl_s32_(a, b) +} + +/// Signed saturating rounding shift left +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqrshl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqrshl))] +pub unsafe fn vqrshlq_s32(a: int32x4_t, b: int32x4_t) -> int32x4_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqrshifts.v4i32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqrshl.v4i32")] + fn vqrshlq_s32_(a: int32x4_t, b: int32x4_t) -> int32x4_t; + } +vqrshlq_s32_(a, b) +} + +/// Signed saturating rounding shift left +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqrshl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqrshl))] +pub unsafe fn vqrshl_s64(a: int64x1_t, b: int64x1_t) -> int64x1_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqrshifts.v1i64")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqrshl.v1i64")] + fn vqrshl_s64_(a: int64x1_t, b: int64x1_t) -> int64x1_t; + } +vqrshl_s64_(a, b) +} + +/// Signed saturating rounding shift left +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqrshl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqrshl))] +pub unsafe fn vqrshlq_s64(a: int64x2_t, b: int64x2_t) -> int64x2_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqrshifts.v2i64")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqrshl.v2i64")] + fn vqrshlq_s64_(a: int64x2_t, b: int64x2_t) -> int64x2_t; + } +vqrshlq_s64_(a, b) +} + +/// Unsigned signed saturating rounding shift left +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqrshl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uqrshl))] +pub unsafe fn vqrshl_u8(a: uint8x8_t, b: int8x8_t) -> uint8x8_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqrshiftu.v8i8")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uqrshl.v8i8")] + fn vqrshl_u8_(a: uint8x8_t, b: int8x8_t) -> uint8x8_t; + } +vqrshl_u8_(a, b) +} + +/// Unsigned signed saturating rounding shift left +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqrshl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uqrshl))] +pub unsafe fn vqrshlq_u8(a: uint8x16_t, b: int8x16_t) -> uint8x16_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqrshiftu.v16i8")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uqrshl.v16i8")] + fn vqrshlq_u8_(a: uint8x16_t, b: int8x16_t) -> uint8x16_t; + } +vqrshlq_u8_(a, b) +} + +/// Unsigned signed saturating rounding shift left +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqrshl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uqrshl))] +pub unsafe fn vqrshl_u16(a: uint16x4_t, b: int16x4_t) -> uint16x4_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqrshiftu.v4i16")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uqrshl.v4i16")] + fn vqrshl_u16_(a: uint16x4_t, b: int16x4_t) -> uint16x4_t; + } +vqrshl_u16_(a, b) +} + +/// Unsigned signed saturating rounding shift left +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqrshl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uqrshl))] +pub unsafe fn vqrshlq_u16(a: uint16x8_t, b: int16x8_t) -> uint16x8_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqrshiftu.v8i16")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uqrshl.v8i16")] + fn vqrshlq_u16_(a: uint16x8_t, b: int16x8_t) -> uint16x8_t; + } +vqrshlq_u16_(a, b) +} + +/// Unsigned signed saturating rounding shift left +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqrshl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uqrshl))] +pub unsafe fn vqrshl_u32(a: uint32x2_t, b: int32x2_t) -> uint32x2_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqrshiftu.v2i32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uqrshl.v2i32")] + fn vqrshl_u32_(a: uint32x2_t, b: int32x2_t) -> uint32x2_t; + } +vqrshl_u32_(a, b) +} + +/// Unsigned signed saturating rounding shift left +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqrshl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uqrshl))] +pub unsafe fn vqrshlq_u32(a: uint32x4_t, b: int32x4_t) -> uint32x4_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqrshiftu.v4i32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uqrshl.v4i32")] + fn vqrshlq_u32_(a: uint32x4_t, b: int32x4_t) -> uint32x4_t; + } +vqrshlq_u32_(a, b) +} + +/// Unsigned signed saturating rounding shift left +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqrshl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uqrshl))] +pub unsafe fn vqrshl_u64(a: uint64x1_t, b: int64x1_t) -> uint64x1_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqrshiftu.v1i64")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uqrshl.v1i64")] + fn vqrshl_u64_(a: uint64x1_t, b: int64x1_t) -> uint64x1_t; + } +vqrshl_u64_(a, b) +} + +/// Unsigned signed saturating rounding shift left +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqrshl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uqrshl))] +pub unsafe fn vqrshlq_u64(a: uint64x2_t, b: int64x2_t) -> uint64x2_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqrshiftu.v2i64")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uqrshl.v2i64")] + fn vqrshlq_u64_(a: uint64x2_t, b: int64x2_t) -> uint64x2_t; + } +vqrshlq_u64_(a, b) +} + +/// Signed saturating rounded shift right narrow +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqrshrn, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vqrshrn_n_s16(a: int16x8_t) -> int8x8_t { + static_assert!(N : i32 where N >= 1 && N <= 8); + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqrshiftns.v8i8")] + fn vqrshrn_n_s16_(a: int16x8_t, n: int16x8_t) -> int8x8_t; + } +vqrshrn_n_s16_(a, int16x8_t(-N as i16, -N as i16, -N as i16, -N as i16, -N as i16, -N as i16, -N as i16, -N as i16)) +} + +/// Signed saturating rounded shift right narrow +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqrshrn, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vqrshrn_n_s16(a: int16x8_t) -> int8x8_t { + static_assert!(N : i32 where N >= 1 && N <= 8); + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqrshrn.v8i8")] + fn vqrshrn_n_s16_(a: int16x8_t, n: i32) -> int8x8_t; + } +vqrshrn_n_s16_(a, N) +} + +/// Signed saturating rounded shift right narrow +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqrshrn, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vqrshrn_n_s32(a: int32x4_t) -> int16x4_t { + static_assert!(N : i32 where N >= 1 && N <= 16); + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqrshiftns.v4i16")] + fn vqrshrn_n_s32_(a: int32x4_t, n: int32x4_t) -> int16x4_t; + } +vqrshrn_n_s32_(a, int32x4_t(-N as i32, -N as i32, -N as i32, -N as i32)) +} + +/// Signed saturating rounded shift right narrow +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqrshrn, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vqrshrn_n_s32(a: int32x4_t) -> int16x4_t { + static_assert!(N : i32 where N >= 1 && N <= 16); + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqrshrn.v4i16")] + fn vqrshrn_n_s32_(a: int32x4_t, n: i32) -> int16x4_t; + } +vqrshrn_n_s32_(a, N) +} + +/// Signed saturating rounded shift right narrow +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqrshrn, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vqrshrn_n_s64(a: int64x2_t) -> int32x2_t { + static_assert!(N : i32 where N >= 1 && N <= 32); + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqrshiftns.v2i32")] + fn vqrshrn_n_s64_(a: int64x2_t, n: int64x2_t) -> int32x2_t; + } +vqrshrn_n_s64_(a, int64x2_t(-N as i64, -N as i64)) +} + +/// Signed saturating rounded shift right narrow +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqrshrn, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vqrshrn_n_s64(a: int64x2_t) -> int32x2_t { + static_assert!(N : i32 where N >= 1 && N <= 32); + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqrshrn.v2i32")] + fn vqrshrn_n_s64_(a: int64x2_t, n: i32) -> int32x2_t; + } +vqrshrn_n_s64_(a, N) +} + +/// Unsigned signed saturating rounded shift right narrow +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqrshrn, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vqrshrn_n_u16(a: uint16x8_t) -> uint8x8_t { + static_assert!(N : i32 where N >= 1 && N <= 8); + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqrshiftnu.v8i8")] + fn vqrshrn_n_u16_(a: uint16x8_t, n: uint16x8_t) -> uint8x8_t; + } +vqrshrn_n_u16_(a, uint16x8_t(-N as u16, -N as u16, -N as u16, -N as u16, -N as u16, -N as u16, -N as u16, -N as u16)) +} + +/// Unsigned signed saturating rounded shift right narrow +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uqrshrn, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vqrshrn_n_u16(a: uint16x8_t) -> uint8x8_t { + static_assert!(N : i32 where N >= 1 && N <= 8); + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uqrshrn.v8i8")] + fn vqrshrn_n_u16_(a: uint16x8_t, n: i32) -> uint8x8_t; + } +vqrshrn_n_u16_(a, N) +} + +/// Unsigned signed saturating rounded shift right narrow +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqrshrn, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vqrshrn_n_u32(a: uint32x4_t) -> uint16x4_t { + static_assert!(N : i32 where N >= 1 && N <= 16); + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqrshiftnu.v4i16")] + fn vqrshrn_n_u32_(a: uint32x4_t, n: uint32x4_t) -> uint16x4_t; + } +vqrshrn_n_u32_(a, uint32x4_t(-N as u32, -N as u32, -N as u32, -N as u32)) +} + +/// Unsigned signed saturating rounded shift right narrow +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uqrshrn, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vqrshrn_n_u32(a: uint32x4_t) -> uint16x4_t { + static_assert!(N : i32 where N >= 1 && N <= 16); + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uqrshrn.v4i16")] + fn vqrshrn_n_u32_(a: uint32x4_t, n: i32) -> uint16x4_t; + } +vqrshrn_n_u32_(a, N) +} + +/// Unsigned signed saturating rounded shift right narrow +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqrshrn, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vqrshrn_n_u64(a: uint64x2_t) -> uint32x2_t { + static_assert!(N : i32 where N >= 1 && N <= 32); + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqrshiftnu.v2i32")] + fn vqrshrn_n_u64_(a: uint64x2_t, n: uint64x2_t) -> uint32x2_t; + } +vqrshrn_n_u64_(a, uint64x2_t(-N as u64, -N as u64)) +} + +/// Unsigned signed saturating rounded shift right narrow +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uqrshrn, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vqrshrn_n_u64(a: uint64x2_t) -> uint32x2_t { + static_assert!(N : i32 where N >= 1 && N <= 32); + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uqrshrn.v2i32")] + fn vqrshrn_n_u64_(a: uint64x2_t, n: i32) -> uint32x2_t; + } +vqrshrn_n_u64_(a, N) +} + +/// Signed saturating rounded shift right unsigned narrow +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqrshrun, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vqrshrun_n_s16(a: int16x8_t) -> uint8x8_t { + static_assert!(N : i32 where N >= 1 && N <= 8); + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqrshiftnsu.v8i8")] + fn vqrshrun_n_s16_(a: int16x8_t, n: int16x8_t) -> uint8x8_t; + } +vqrshrun_n_s16_(a, int16x8_t(-N as i16, -N as i16, -N as i16, -N as i16, -N as i16, -N as i16, -N as i16, -N as i16)) +} + +/// Signed saturating rounded shift right unsigned narrow +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqrshrun, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vqrshrun_n_s16(a: int16x8_t) -> uint8x8_t { + static_assert!(N : i32 where N >= 1 && N <= 8); + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqrshrun.v8i8")] + fn vqrshrun_n_s16_(a: int16x8_t, n: i32) -> uint8x8_t; + } +vqrshrun_n_s16_(a, N) +} + +/// Signed saturating rounded shift right unsigned narrow +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqrshrun, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vqrshrun_n_s32(a: int32x4_t) -> uint16x4_t { + static_assert!(N : i32 where N >= 1 && N <= 16); + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqrshiftnsu.v4i16")] + fn vqrshrun_n_s32_(a: int32x4_t, n: int32x4_t) -> uint16x4_t; + } +vqrshrun_n_s32_(a, int32x4_t(-N as i32, -N as i32, -N as i32, -N as i32)) +} + +/// Signed saturating rounded shift right unsigned narrow +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqrshrun, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vqrshrun_n_s32(a: int32x4_t) -> uint16x4_t { + static_assert!(N : i32 where N >= 1 && N <= 16); + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqrshrun.v4i16")] + fn vqrshrun_n_s32_(a: int32x4_t, n: i32) -> uint16x4_t; + } +vqrshrun_n_s32_(a, N) +} + +/// Signed saturating rounded shift right unsigned narrow +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqrshrun, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vqrshrun_n_s64(a: int64x2_t) -> uint32x2_t { + static_assert!(N : i32 where N >= 1 && N <= 32); + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqrshiftnsu.v2i32")] + fn vqrshrun_n_s64_(a: int64x2_t, n: int64x2_t) -> uint32x2_t; + } +vqrshrun_n_s64_(a, int64x2_t(-N as i64, -N as i64)) +} + +/// Signed saturating rounded shift right unsigned narrow +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqrshrun, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vqrshrun_n_s64(a: int64x2_t) -> uint32x2_t { + static_assert!(N : i32 where N >= 1 && N <= 32); + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqrshrun.v2i32")] + fn vqrshrun_n_s64_(a: int64x2_t, n: i32) -> uint32x2_t; + } +vqrshrun_n_s64_(a, N) +} + +/// Signed saturating shift left +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqshl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqshl))] +pub unsafe fn vqshl_s8(a: int8x8_t, b: int8x8_t) -> int8x8_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqshifts.v8i8")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqshl.v8i8")] + fn vqshl_s8_(a: int8x8_t, b: int8x8_t) -> int8x8_t; + } +vqshl_s8_(a, b) +} + +/// Signed saturating shift left +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqshl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqshl))] +pub unsafe fn vqshlq_s8(a: int8x16_t, b: int8x16_t) -> int8x16_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqshifts.v16i8")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqshl.v16i8")] + fn vqshlq_s8_(a: int8x16_t, b: int8x16_t) -> int8x16_t; + } +vqshlq_s8_(a, b) +} + +/// Signed saturating shift left +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqshl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqshl))] +pub unsafe fn vqshl_s16(a: int16x4_t, b: int16x4_t) -> int16x4_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqshifts.v4i16")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqshl.v4i16")] + fn vqshl_s16_(a: int16x4_t, b: int16x4_t) -> int16x4_t; + } +vqshl_s16_(a, b) +} + +/// Signed saturating shift left +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqshl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqshl))] +pub unsafe fn vqshlq_s16(a: int16x8_t, b: int16x8_t) -> int16x8_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqshifts.v8i16")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqshl.v8i16")] + fn vqshlq_s16_(a: int16x8_t, b: int16x8_t) -> int16x8_t; + } +vqshlq_s16_(a, b) +} + +/// Signed saturating shift left +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqshl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqshl))] +pub unsafe fn vqshl_s32(a: int32x2_t, b: int32x2_t) -> int32x2_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqshifts.v2i32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqshl.v2i32")] + fn vqshl_s32_(a: int32x2_t, b: int32x2_t) -> int32x2_t; + } +vqshl_s32_(a, b) +} + +/// Signed saturating shift left +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqshl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqshl))] +pub unsafe fn vqshlq_s32(a: int32x4_t, b: int32x4_t) -> int32x4_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqshifts.v4i32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqshl.v4i32")] + fn vqshlq_s32_(a: int32x4_t, b: int32x4_t) -> int32x4_t; + } +vqshlq_s32_(a, b) +} + +/// Signed saturating shift left +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqshl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqshl))] +pub unsafe fn vqshl_s64(a: int64x1_t, b: int64x1_t) -> int64x1_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqshifts.v1i64")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqshl.v1i64")] + fn vqshl_s64_(a: int64x1_t, b: int64x1_t) -> int64x1_t; + } +vqshl_s64_(a, b) +} + +/// Signed saturating shift left +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqshl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqshl))] +pub unsafe fn vqshlq_s64(a: int64x2_t, b: int64x2_t) -> int64x2_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqshifts.v2i64")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqshl.v2i64")] + fn vqshlq_s64_(a: int64x2_t, b: int64x2_t) -> int64x2_t; + } +vqshlq_s64_(a, b) +} + +/// Unsigned saturating shift left +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqshl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uqshl))] +pub unsafe fn vqshl_u8(a: uint8x8_t, b: int8x8_t) -> uint8x8_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqshiftu.v8i8")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uqshl.v8i8")] + fn vqshl_u8_(a: uint8x8_t, b: int8x8_t) -> uint8x8_t; + } +vqshl_u8_(a, b) +} + +/// Unsigned saturating shift left +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqshl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uqshl))] +pub unsafe fn vqshlq_u8(a: uint8x16_t, b: int8x16_t) -> uint8x16_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqshiftu.v16i8")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uqshl.v16i8")] + fn vqshlq_u8_(a: uint8x16_t, b: int8x16_t) -> uint8x16_t; + } +vqshlq_u8_(a, b) +} + +/// Unsigned saturating shift left +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqshl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uqshl))] +pub unsafe fn vqshl_u16(a: uint16x4_t, b: int16x4_t) -> uint16x4_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqshiftu.v4i16")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uqshl.v4i16")] + fn vqshl_u16_(a: uint16x4_t, b: int16x4_t) -> uint16x4_t; + } +vqshl_u16_(a, b) +} + +/// Unsigned saturating shift left +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqshl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uqshl))] +pub unsafe fn vqshlq_u16(a: uint16x8_t, b: int16x8_t) -> uint16x8_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqshiftu.v8i16")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uqshl.v8i16")] + fn vqshlq_u16_(a: uint16x8_t, b: int16x8_t) -> uint16x8_t; + } +vqshlq_u16_(a, b) +} + +/// Unsigned saturating shift left +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqshl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uqshl))] +pub unsafe fn vqshl_u32(a: uint32x2_t, b: int32x2_t) -> uint32x2_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqshiftu.v2i32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uqshl.v2i32")] + fn vqshl_u32_(a: uint32x2_t, b: int32x2_t) -> uint32x2_t; + } +vqshl_u32_(a, b) +} + +/// Unsigned saturating shift left +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqshl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uqshl))] +pub unsafe fn vqshlq_u32(a: uint32x4_t, b: int32x4_t) -> uint32x4_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqshiftu.v4i32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uqshl.v4i32")] + fn vqshlq_u32_(a: uint32x4_t, b: int32x4_t) -> uint32x4_t; + } +vqshlq_u32_(a, b) +} + +/// Unsigned saturating shift left +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqshl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uqshl))] +pub unsafe fn vqshl_u64(a: uint64x1_t, b: int64x1_t) -> uint64x1_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqshiftu.v1i64")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uqshl.v1i64")] + fn vqshl_u64_(a: uint64x1_t, b: int64x1_t) -> uint64x1_t; + } +vqshl_u64_(a, b) +} + +/// Unsigned saturating shift left +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqshl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uqshl))] +pub unsafe fn vqshlq_u64(a: uint64x2_t, b: int64x2_t) -> uint64x2_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqshiftu.v2i64")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uqshl.v2i64")] + fn vqshlq_u64_(a: uint64x2_t, b: int64x2_t) -> uint64x2_t; + } +vqshlq_u64_(a, b) +} + +/// Signed saturating shift left +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqshl, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqshl, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vqshl_n_s8(a: int8x8_t) -> int8x8_t { + static_assert_imm3!(N); + vqshl_s8(a, vdup_n_s8(N.try_into().unwrap())) +} + +/// Signed saturating shift left +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqshl, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqshl, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vqshlq_n_s8(a: int8x16_t) -> int8x16_t { + static_assert_imm3!(N); + vqshlq_s8(a, vdupq_n_s8(N.try_into().unwrap())) +} + +/// Signed saturating shift left +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqshl, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqshl, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vqshl_n_s16(a: int16x4_t) -> int16x4_t { + static_assert_imm4!(N); + vqshl_s16(a, vdup_n_s16(N.try_into().unwrap())) +} + +/// Signed saturating shift left +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqshl, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqshl, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vqshlq_n_s16(a: int16x8_t) -> int16x8_t { + static_assert_imm4!(N); + vqshlq_s16(a, vdupq_n_s16(N.try_into().unwrap())) +} + +/// Signed saturating shift left +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqshl, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqshl, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vqshl_n_s32(a: int32x2_t) -> int32x2_t { + static_assert_imm5!(N); + vqshl_s32(a, vdup_n_s32(N.try_into().unwrap())) +} + +/// Signed saturating shift left +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqshl, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqshl, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vqshlq_n_s32(a: int32x4_t) -> int32x4_t { + static_assert_imm5!(N); + vqshlq_s32(a, vdupq_n_s32(N.try_into().unwrap())) +} + +/// Signed saturating shift left +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqshl, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqshl, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vqshl_n_s64(a: int64x1_t) -> int64x1_t { + static_assert_imm6!(N); + vqshl_s64(a, vdup_n_s64(N.try_into().unwrap())) +} + +/// Signed saturating shift left +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqshl, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqshl, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vqshlq_n_s64(a: int64x2_t) -> int64x2_t { + static_assert_imm6!(N); + vqshlq_s64(a, vdupq_n_s64(N.try_into().unwrap())) +} + +/// Unsigned saturating shift left +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqshl, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uqshl, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vqshl_n_u8(a: uint8x8_t) -> uint8x8_t { + static_assert_imm3!(N); + vqshl_u8(a, vdup_n_s8(N.try_into().unwrap())) +} + +/// Unsigned saturating shift left +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqshl, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uqshl, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vqshlq_n_u8(a: uint8x16_t) -> uint8x16_t { + static_assert_imm3!(N); + vqshlq_u8(a, vdupq_n_s8(N.try_into().unwrap())) +} + +/// Unsigned saturating shift left +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqshl, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uqshl, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vqshl_n_u16(a: uint16x4_t) -> uint16x4_t { + static_assert_imm4!(N); + vqshl_u16(a, vdup_n_s16(N.try_into().unwrap())) +} + +/// Unsigned saturating shift left +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqshl, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uqshl, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vqshlq_n_u16(a: uint16x8_t) -> uint16x8_t { + static_assert_imm4!(N); + vqshlq_u16(a, vdupq_n_s16(N.try_into().unwrap())) +} + +/// Unsigned saturating shift left +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqshl, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uqshl, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vqshl_n_u32(a: uint32x2_t) -> uint32x2_t { + static_assert_imm5!(N); + vqshl_u32(a, vdup_n_s32(N.try_into().unwrap())) +} + +/// Unsigned saturating shift left +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqshl, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uqshl, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vqshlq_n_u32(a: uint32x4_t) -> uint32x4_t { + static_assert_imm5!(N); + vqshlq_u32(a, vdupq_n_s32(N.try_into().unwrap())) +} + +/// Unsigned saturating shift left +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqshl, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uqshl, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vqshl_n_u64(a: uint64x1_t) -> uint64x1_t { + static_assert_imm6!(N); + vqshl_u64(a, vdup_n_s64(N.try_into().unwrap())) +} + +/// Unsigned saturating shift left +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqshl, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uqshl, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vqshlq_n_u64(a: uint64x2_t) -> uint64x2_t { + static_assert_imm6!(N); + vqshlq_u64(a, vdupq_n_s64(N.try_into().unwrap())) +} + +/// Signed saturating shift right narrow +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqshrn, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vqshrn_n_s16(a: int16x8_t) -> int8x8_t { + static_assert!(N : i32 where N >= 1 && N <= 8); + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqshiftns.v8i8")] + fn vqshrn_n_s16_(a: int16x8_t, n: int16x8_t) -> int8x8_t; + } +vqshrn_n_s16_(a, int16x8_t(-N as i16, -N as i16, -N as i16, -N as i16, -N as i16, -N as i16, -N as i16, -N as i16)) +} + +/// Signed saturating shift right narrow +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqshrn, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vqshrn_n_s16(a: int16x8_t) -> int8x8_t { + static_assert!(N : i32 where N >= 1 && N <= 8); + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqshrn.v8i8")] + fn vqshrn_n_s16_(a: int16x8_t, n: i32) -> int8x8_t; + } +vqshrn_n_s16_(a, N) +} + +/// Signed saturating shift right narrow +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqshrn, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vqshrn_n_s32(a: int32x4_t) -> int16x4_t { + static_assert!(N : i32 where N >= 1 && N <= 16); + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqshiftns.v4i16")] + fn vqshrn_n_s32_(a: int32x4_t, n: int32x4_t) -> int16x4_t; + } +vqshrn_n_s32_(a, int32x4_t(-N as i32, -N as i32, -N as i32, -N as i32)) +} + +/// Signed saturating shift right narrow +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqshrn, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vqshrn_n_s32(a: int32x4_t) -> int16x4_t { + static_assert!(N : i32 where N >= 1 && N <= 16); + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqshrn.v4i16")] + fn vqshrn_n_s32_(a: int32x4_t, n: i32) -> int16x4_t; + } +vqshrn_n_s32_(a, N) +} + +/// Signed saturating shift right narrow +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqshrn, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vqshrn_n_s64(a: int64x2_t) -> int32x2_t { + static_assert!(N : i32 where N >= 1 && N <= 32); + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqshiftns.v2i32")] + fn vqshrn_n_s64_(a: int64x2_t, n: int64x2_t) -> int32x2_t; + } +vqshrn_n_s64_(a, int64x2_t(-N as i64, -N as i64)) +} + +/// Signed saturating shift right narrow +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqshrn, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vqshrn_n_s64(a: int64x2_t) -> int32x2_t { + static_assert!(N : i32 where N >= 1 && N <= 32); + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqshrn.v2i32")] + fn vqshrn_n_s64_(a: int64x2_t, n: i32) -> int32x2_t; + } +vqshrn_n_s64_(a, N) +} + +/// Unsigned saturating shift right narrow +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqshrn, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vqshrn_n_u16(a: uint16x8_t) -> uint8x8_t { + static_assert!(N : i32 where N >= 1 && N <= 8); + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqshiftnu.v8i8")] + fn vqshrn_n_u16_(a: uint16x8_t, n: uint16x8_t) -> uint8x8_t; + } +vqshrn_n_u16_(a, uint16x8_t(-N as u16, -N as u16, -N as u16, -N as u16, -N as u16, -N as u16, -N as u16, -N as u16)) +} + +/// Unsigned saturating shift right narrow +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uqshrn, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vqshrn_n_u16(a: uint16x8_t) -> uint8x8_t { + static_assert!(N : i32 where N >= 1 && N <= 8); + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uqshrn.v8i8")] + fn vqshrn_n_u16_(a: uint16x8_t, n: i32) -> uint8x8_t; + } +vqshrn_n_u16_(a, N) +} + +/// Unsigned saturating shift right narrow +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqshrn, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vqshrn_n_u32(a: uint32x4_t) -> uint16x4_t { + static_assert!(N : i32 where N >= 1 && N <= 16); + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqshiftnu.v4i16")] + fn vqshrn_n_u32_(a: uint32x4_t, n: uint32x4_t) -> uint16x4_t; + } +vqshrn_n_u32_(a, uint32x4_t(-N as u32, -N as u32, -N as u32, -N as u32)) +} + +/// Unsigned saturating shift right narrow +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uqshrn, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vqshrn_n_u32(a: uint32x4_t) -> uint16x4_t { + static_assert!(N : i32 where N >= 1 && N <= 16); + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uqshrn.v4i16")] + fn vqshrn_n_u32_(a: uint32x4_t, n: i32) -> uint16x4_t; + } +vqshrn_n_u32_(a, N) +} + +/// Unsigned saturating shift right narrow +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqshrn, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vqshrn_n_u64(a: uint64x2_t) -> uint32x2_t { + static_assert!(N : i32 where N >= 1 && N <= 32); + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqshiftnu.v2i32")] + fn vqshrn_n_u64_(a: uint64x2_t, n: uint64x2_t) -> uint32x2_t; + } +vqshrn_n_u64_(a, uint64x2_t(-N as u64, -N as u64)) +} + +/// Unsigned saturating shift right narrow +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uqshrn, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vqshrn_n_u64(a: uint64x2_t) -> uint32x2_t { + static_assert!(N : i32 where N >= 1 && N <= 32); + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uqshrn.v2i32")] + fn vqshrn_n_u64_(a: uint64x2_t, n: i32) -> uint32x2_t; + } +vqshrn_n_u64_(a, N) +} + +/// Signed saturating shift right unsigned narrow +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqshrun, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vqshrun_n_s16(a: int16x8_t) -> uint8x8_t { + static_assert!(N : i32 where N >= 1 && N <= 8); + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqshiftnsu.v8i8")] + fn vqshrun_n_s16_(a: int16x8_t, n: int16x8_t) -> uint8x8_t; + } +vqshrun_n_s16_(a, int16x8_t(-N as i16, -N as i16, -N as i16, -N as i16, -N as i16, -N as i16, -N as i16, -N as i16)) +} + +/// Signed saturating shift right unsigned narrow +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqshrun, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vqshrun_n_s16(a: int16x8_t) -> uint8x8_t { + static_assert!(N : i32 where N >= 1 && N <= 8); + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqshrun.v8i8")] + fn vqshrun_n_s16_(a: int16x8_t, n: i32) -> uint8x8_t; + } +vqshrun_n_s16_(a, N) +} + +/// Signed saturating shift right unsigned narrow +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqshrun, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vqshrun_n_s32(a: int32x4_t) -> uint16x4_t { + static_assert!(N : i32 where N >= 1 && N <= 16); + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqshiftnsu.v4i16")] + fn vqshrun_n_s32_(a: int32x4_t, n: int32x4_t) -> uint16x4_t; + } +vqshrun_n_s32_(a, int32x4_t(-N as i32, -N as i32, -N as i32, -N as i32)) +} + +/// Signed saturating shift right unsigned narrow +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqshrun, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vqshrun_n_s32(a: int32x4_t) -> uint16x4_t { + static_assert!(N : i32 where N >= 1 && N <= 16); + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqshrun.v4i16")] + fn vqshrun_n_s32_(a: int32x4_t, n: i32) -> uint16x4_t; + } +vqshrun_n_s32_(a, N) +} + +/// Signed saturating shift right unsigned narrow +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqshrun, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vqshrun_n_s64(a: int64x2_t) -> uint32x2_t { + static_assert!(N : i32 where N >= 1 && N <= 32); + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqshiftnsu.v2i32")] + fn vqshrun_n_s64_(a: int64x2_t, n: int64x2_t) -> uint32x2_t; + } +vqshrun_n_s64_(a, int64x2_t(-N as i64, -N as i64)) +} + +/// Signed saturating shift right unsigned narrow +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqshrun, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vqshrun_n_s64(a: int64x2_t) -> uint32x2_t { + static_assert!(N : i32 where N >= 1 && N <= 32); + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqshrun.v2i32")] + fn vqshrun_n_s64_(a: int64x2_t, n: i32) -> uint32x2_t; + } +vqshrun_n_s64_(a, N) +} + +/// Reciprocal square-root estimate. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrsqrte))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(frsqrte))] +pub unsafe fn vrsqrte_f32(a: float32x2_t) -> float32x2_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vrsqrte.v2f32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.frsqrte.v2f32")] + fn vrsqrte_f32_(a: float32x2_t) -> float32x2_t; + } +vrsqrte_f32_(a) +} + +/// Reciprocal square-root estimate. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrsqrte))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(frsqrte))] +pub unsafe fn vrsqrteq_f32(a: float32x4_t) -> float32x4_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vrsqrte.v4f32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.frsqrte.v4f32")] + fn vrsqrteq_f32_(a: float32x4_t) -> float32x4_t; + } +vrsqrteq_f32_(a) +} + +/// Reciprocal estimate. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrecpe))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(frecpe))] +pub unsafe fn vrecpe_f32(a: float32x2_t) -> float32x2_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vrecpe.v2f32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.frecpe.v2f32")] + fn vrecpe_f32_(a: float32x2_t) -> float32x2_t; + } +vrecpe_f32_(a) +} + +/// Reciprocal estimate. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrecpe))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(frecpe))] +pub unsafe fn vrecpeq_f32(a: float32x4_t) -> float32x4_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vrecpe.v4f32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.frecpe.v4f32")] + fn vrecpeq_f32_(a: float32x4_t) -> float32x4_t; + } +vrecpeq_f32_(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpret_s8_u8(a: uint8x8_t) -> int8x8_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpret_s8_p8(a: poly8x8_t) -> int8x8_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpret_s16_p16(a: poly16x4_t) -> int16x4_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpret_s16_u16(a: uint16x4_t) -> int16x4_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpret_s32_u32(a: uint32x2_t) -> int32x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpret_s64_u64(a: uint64x1_t) -> int64x1_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpretq_s8_u8(a: uint8x16_t) -> int8x16_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpretq_s8_p8(a: poly8x16_t) -> int8x16_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpretq_s16_p16(a: poly16x8_t) -> int16x8_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpretq_s16_u16(a: uint16x8_t) -> int16x8_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpretq_s32_u32(a: uint32x4_t) -> int32x4_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpretq_s64_u64(a: uint64x2_t) -> int64x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpret_u8_p8(a: poly8x8_t) -> uint8x8_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpret_u8_s8(a: int8x8_t) -> uint8x8_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpret_u16_p16(a: poly16x4_t) -> uint16x4_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpret_u16_s16(a: int16x4_t) -> uint16x4_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpret_u32_s32(a: int32x2_t) -> uint32x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpret_u64_s64(a: int64x1_t) -> uint64x1_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpretq_u8_p8(a: poly8x16_t) -> uint8x16_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpretq_u8_s8(a: int8x16_t) -> uint8x16_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpretq_u16_p16(a: poly16x8_t) -> uint16x8_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpretq_u16_s16(a: int16x8_t) -> uint16x8_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpretq_u32_s32(a: int32x4_t) -> uint32x4_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpretq_u64_s64(a: int64x2_t) -> uint64x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpret_p8_s8(a: int8x8_t) -> poly8x8_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpret_p8_u8(a: uint8x8_t) -> poly8x8_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpret_p16_s16(a: int16x4_t) -> poly16x4_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpret_p16_u16(a: uint16x4_t) -> poly16x4_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpretq_p8_s8(a: int8x16_t) -> poly8x16_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpretq_p8_u8(a: uint8x16_t) -> poly8x16_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpretq_p16_s16(a: int16x8_t) -> poly16x8_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpretq_p16_u16(a: uint16x8_t) -> poly16x8_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpret_s8_s16(a: int16x4_t) -> int8x8_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpret_s8_u16(a: uint16x4_t) -> int8x8_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpret_s8_p16(a: poly16x4_t) -> int8x8_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpret_s16_s32(a: int32x2_t) -> int16x4_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpret_s16_u32(a: uint32x2_t) -> int16x4_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpret_s32_s64(a: int64x1_t) -> int32x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpret_s32_u64(a: uint64x1_t) -> int32x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpretq_s8_s16(a: int16x8_t) -> int8x16_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpretq_s8_u16(a: uint16x8_t) -> int8x16_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpretq_s8_p16(a: poly16x8_t) -> int8x16_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpretq_s16_s32(a: int32x4_t) -> int16x8_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpretq_s16_u32(a: uint32x4_t) -> int16x8_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpretq_s32_s64(a: int64x2_t) -> int32x4_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpretq_s32_u64(a: uint64x2_t) -> int32x4_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpret_u8_p16(a: poly16x4_t) -> uint8x8_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpret_u8_s16(a: int16x4_t) -> uint8x8_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpret_u8_u16(a: uint16x4_t) -> uint8x8_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpret_u16_s32(a: int32x2_t) -> uint16x4_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpret_u16_u32(a: uint32x2_t) -> uint16x4_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpret_u32_s64(a: int64x1_t) -> uint32x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpret_u32_u64(a: uint64x1_t) -> uint32x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpretq_u8_p16(a: poly16x8_t) -> uint8x16_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpretq_u8_s16(a: int16x8_t) -> uint8x16_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpretq_u8_u16(a: uint16x8_t) -> uint8x16_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpretq_u16_s32(a: int32x4_t) -> uint16x8_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpretq_u16_u32(a: uint32x4_t) -> uint16x8_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpretq_u32_s64(a: int64x2_t) -> uint32x4_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpretq_u32_u64(a: uint64x2_t) -> uint32x4_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpret_p8_p16(a: poly16x4_t) -> poly8x8_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpret_p8_s16(a: int16x4_t) -> poly8x8_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpret_p8_u16(a: uint16x4_t) -> poly8x8_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpret_p16_s32(a: int32x2_t) -> poly16x4_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpret_p16_u32(a: uint32x2_t) -> poly16x4_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpretq_p8_p16(a: poly16x8_t) -> poly8x16_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpretq_p8_s16(a: int16x8_t) -> poly8x16_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpretq_p8_u16(a: uint16x8_t) -> poly8x16_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpretq_p16_s32(a: int32x4_t) -> poly16x8_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpretq_p16_u32(a: uint32x4_t) -> poly16x8_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpret_s16_p8(a: poly8x8_t) -> int16x4_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpret_s16_s8(a: int8x8_t) -> int16x4_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpret_s16_u8(a: uint8x8_t) -> int16x4_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpret_s32_p16(a: poly16x4_t) -> int32x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpret_s32_s16(a: int16x4_t) -> int32x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpret_s32_u16(a: uint16x4_t) -> int32x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpret_s64_s32(a: int32x2_t) -> int64x1_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpret_s64_u32(a: uint32x2_t) -> int64x1_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpretq_s16_p8(a: poly8x16_t) -> int16x8_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpretq_s16_s8(a: int8x16_t) -> int16x8_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpretq_s16_u8(a: uint8x16_t) -> int16x8_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpretq_s32_p16(a: poly16x8_t) -> int32x4_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpretq_s32_s16(a: int16x8_t) -> int32x4_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpretq_s32_u16(a: uint16x8_t) -> int32x4_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpretq_s64_s32(a: int32x4_t) -> int64x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpretq_s64_u32(a: uint32x4_t) -> int64x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpret_u16_p8(a: poly8x8_t) -> uint16x4_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpret_u16_s8(a: int8x8_t) -> uint16x4_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpret_u16_u8(a: uint8x8_t) -> uint16x4_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpret_u32_p16(a: poly16x4_t) -> uint32x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpret_u32_s16(a: int16x4_t) -> uint32x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpret_u32_u16(a: uint16x4_t) -> uint32x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpret_u64_s32(a: int32x2_t) -> uint64x1_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpret_u64_u32(a: uint32x2_t) -> uint64x1_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpretq_u16_p8(a: poly8x16_t) -> uint16x8_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpretq_u16_s8(a: int8x16_t) -> uint16x8_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpretq_u16_u8(a: uint8x16_t) -> uint16x8_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpretq_u32_p16(a: poly16x8_t) -> uint32x4_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpretq_u32_s16(a: int16x8_t) -> uint32x4_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpretq_u32_u16(a: uint16x8_t) -> uint32x4_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpretq_u64_s32(a: int32x4_t) -> uint64x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpretq_u64_u32(a: uint32x4_t) -> uint64x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpret_p16_p8(a: poly8x8_t) -> poly16x4_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpret_p16_s8(a: int8x8_t) -> poly16x4_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpret_p16_u8(a: uint8x8_t) -> poly16x4_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpretq_p16_p8(a: poly8x16_t) -> poly16x8_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpretq_p16_s8(a: int8x16_t) -> poly16x8_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpretq_p16_u8(a: uint8x16_t) -> poly16x8_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpret_s8_s32(a: int32x2_t) -> int8x8_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpret_s8_u32(a: uint32x2_t) -> int8x8_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpret_s16_s64(a: int64x1_t) -> int16x4_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpret_s16_u64(a: uint64x1_t) -> int16x4_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpretq_s8_s32(a: int32x4_t) -> int8x16_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpretq_s8_u32(a: uint32x4_t) -> int8x16_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpretq_s16_s64(a: int64x2_t) -> int16x8_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpretq_s16_u64(a: uint64x2_t) -> int16x8_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpret_u8_s32(a: int32x2_t) -> uint8x8_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpret_u8_u32(a: uint32x2_t) -> uint8x8_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpret_u16_s64(a: int64x1_t) -> uint16x4_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpret_u16_u64(a: uint64x1_t) -> uint16x4_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpretq_u8_s32(a: int32x4_t) -> uint8x16_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpretq_u8_u32(a: uint32x4_t) -> uint8x16_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpretq_u16_s64(a: int64x2_t) -> uint16x8_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpretq_u16_u64(a: uint64x2_t) -> uint16x8_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpret_p8_s32(a: int32x2_t) -> poly8x8_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpret_p8_u32(a: uint32x2_t) -> poly8x8_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpret_p16_s64(a: int64x1_t) -> poly16x4_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpret_p16_u64(a: uint64x1_t) -> poly16x4_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpretq_p8_s32(a: int32x4_t) -> poly8x16_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpretq_p8_u32(a: uint32x4_t) -> poly8x16_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpretq_p16_s64(a: int64x2_t) -> poly16x8_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpretq_p16_u64(a: uint64x2_t) -> poly16x8_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpret_s32_p8(a: poly8x8_t) -> int32x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpret_s32_s8(a: int8x8_t) -> int32x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpret_s32_u8(a: uint8x8_t) -> int32x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpret_s64_p16(a: poly16x4_t) -> int64x1_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpret_s64_s16(a: int16x4_t) -> int64x1_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpret_s64_u16(a: uint16x4_t) -> int64x1_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpretq_s32_p8(a: poly8x16_t) -> int32x4_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpretq_s32_s8(a: int8x16_t) -> int32x4_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpretq_s32_u8(a: uint8x16_t) -> int32x4_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpretq_s64_p16(a: poly16x8_t) -> int64x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpretq_s64_s16(a: int16x8_t) -> int64x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpretq_s64_u16(a: uint16x8_t) -> int64x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpret_u32_p8(a: poly8x8_t) -> uint32x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpret_u32_s8(a: int8x8_t) -> uint32x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpret_u32_u8(a: uint8x8_t) -> uint32x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpret_u64_p16(a: poly16x4_t) -> uint64x1_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpret_u64_s16(a: int16x4_t) -> uint64x1_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpret_u64_u16(a: uint16x4_t) -> uint64x1_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpretq_u32_p8(a: poly8x16_t) -> uint32x4_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpretq_u32_s8(a: int8x16_t) -> uint32x4_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpretq_u32_u8(a: uint8x16_t) -> uint32x4_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpretq_u64_p16(a: poly16x8_t) -> uint64x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpretq_u64_s16(a: int16x8_t) -> uint64x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpretq_u64_u16(a: uint16x8_t) -> uint64x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpret_s8_s64(a: int64x1_t) -> int8x8_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpret_s8_u64(a: uint64x1_t) -> int8x8_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpret_u8_s64(a: int64x1_t) -> uint8x8_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpret_u8_u64(a: uint64x1_t) -> uint8x8_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpret_p8_s64(a: int64x1_t) -> poly8x8_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpret_p8_u64(a: uint64x1_t) -> poly8x8_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpretq_s8_s64(a: int64x2_t) -> int8x16_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpretq_s8_u64(a: uint64x2_t) -> int8x16_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpretq_u8_s64(a: int64x2_t) -> uint8x16_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpretq_u8_u64(a: uint64x2_t) -> uint8x16_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpretq_p8_s64(a: int64x2_t) -> poly8x16_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpretq_p8_u64(a: uint64x2_t) -> poly8x16_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpret_s64_p8(a: poly8x8_t) -> int64x1_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpret_s64_s8(a: int8x8_t) -> int64x1_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpret_s64_u8(a: uint8x8_t) -> int64x1_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpret_u64_p8(a: poly8x8_t) -> uint64x1_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpret_u64_s8(a: int8x8_t) -> uint64x1_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpret_u64_u8(a: uint8x8_t) -> uint64x1_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpretq_s64_p8(a: poly8x16_t) -> int64x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpretq_s64_s8(a: int8x16_t) -> int64x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpretq_s64_u8(a: uint8x16_t) -> int64x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpretq_u64_p8(a: poly8x16_t) -> uint64x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpretq_u64_s8(a: int8x16_t) -> uint64x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpretq_u64_u8(a: uint8x16_t) -> uint64x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpret_s8_f32(a: float32x2_t) -> int8x8_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpret_s16_f32(a: float32x2_t) -> int16x4_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpret_s32_f32(a: float32x2_t) -> int32x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpret_s64_f32(a: float32x2_t) -> int64x1_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpretq_s8_f32(a: float32x4_t) -> int8x16_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpretq_s16_f32(a: float32x4_t) -> int16x8_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpretq_s32_f32(a: float32x4_t) -> int32x4_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpretq_s64_f32(a: float32x4_t) -> int64x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpret_u8_f32(a: float32x2_t) -> uint8x8_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpret_u16_f32(a: float32x2_t) -> uint16x4_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpret_u32_f32(a: float32x2_t) -> uint32x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpret_u64_f32(a: float32x2_t) -> uint64x1_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpretq_u8_f32(a: float32x4_t) -> uint8x16_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpretq_u16_f32(a: float32x4_t) -> uint16x8_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpretq_u32_f32(a: float32x4_t) -> uint32x4_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpretq_u64_f32(a: float32x4_t) -> uint64x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpret_p8_f32(a: float32x2_t) -> poly8x8_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpret_p16_f32(a: float32x2_t) -> poly16x4_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpretq_p8_f32(a: float32x4_t) -> poly8x16_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpretq_p16_f32(a: float32x4_t) -> poly16x8_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpret_f32_s8(a: int8x8_t) -> float32x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpret_f32_s16(a: int16x4_t) -> float32x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpret_f32_s32(a: int32x2_t) -> float32x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpret_f32_s64(a: int64x1_t) -> float32x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpretq_f32_s8(a: int8x16_t) -> float32x4_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpretq_f32_s16(a: int16x8_t) -> float32x4_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpretq_f32_s32(a: int32x4_t) -> float32x4_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpretq_f32_s64(a: int64x2_t) -> float32x4_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpret_f32_u8(a: uint8x8_t) -> float32x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpret_f32_u16(a: uint16x4_t) -> float32x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpret_f32_u32(a: uint32x2_t) -> float32x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpret_f32_u64(a: uint64x1_t) -> float32x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpretq_f32_u8(a: uint8x16_t) -> float32x4_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpretq_f32_u16(a: uint16x8_t) -> float32x4_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpretq_f32_u32(a: uint32x4_t) -> float32x4_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpretq_f32_u64(a: uint64x2_t) -> float32x4_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpret_f32_p8(a: poly8x8_t) -> float32x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpret_f32_p16(a: poly16x4_t) -> float32x2_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpretq_f32_p8(a: poly8x16_t) -> float32x4_t { + transmute(a) +} + +/// Vector reinterpret cast operation +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(str))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(str))] +pub unsafe fn vreinterpretq_f32_p16(a: poly16x8_t) -> float32x4_t { + transmute(a) +} + +/// Signed rounding shift left +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrshl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(srshl))] +pub unsafe fn vrshl_s8(a: int8x8_t, b: int8x8_t) -> int8x8_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vrshifts.v8i8")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.srshl.v8i8")] + fn vrshl_s8_(a: int8x8_t, b: int8x8_t) -> int8x8_t; + } +vrshl_s8_(a, b) +} + +/// Signed rounding shift left +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrshl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(srshl))] +pub unsafe fn vrshlq_s8(a: int8x16_t, b: int8x16_t) -> int8x16_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vrshifts.v16i8")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.srshl.v16i8")] + fn vrshlq_s8_(a: int8x16_t, b: int8x16_t) -> int8x16_t; + } +vrshlq_s8_(a, b) +} + +/// Signed rounding shift left +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrshl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(srshl))] +pub unsafe fn vrshl_s16(a: int16x4_t, b: int16x4_t) -> int16x4_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vrshifts.v4i16")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.srshl.v4i16")] + fn vrshl_s16_(a: int16x4_t, b: int16x4_t) -> int16x4_t; + } +vrshl_s16_(a, b) +} + +/// Signed rounding shift left +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrshl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(srshl))] +pub unsafe fn vrshlq_s16(a: int16x8_t, b: int16x8_t) -> int16x8_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vrshifts.v8i16")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.srshl.v8i16")] + fn vrshlq_s16_(a: int16x8_t, b: int16x8_t) -> int16x8_t; + } +vrshlq_s16_(a, b) +} + +/// Signed rounding shift left +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrshl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(srshl))] +pub unsafe fn vrshl_s32(a: int32x2_t, b: int32x2_t) -> int32x2_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vrshifts.v2i32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.srshl.v2i32")] + fn vrshl_s32_(a: int32x2_t, b: int32x2_t) -> int32x2_t; + } +vrshl_s32_(a, b) +} + +/// Signed rounding shift left +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrshl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(srshl))] +pub unsafe fn vrshlq_s32(a: int32x4_t, b: int32x4_t) -> int32x4_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vrshifts.v4i32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.srshl.v4i32")] + fn vrshlq_s32_(a: int32x4_t, b: int32x4_t) -> int32x4_t; + } +vrshlq_s32_(a, b) +} + +/// Signed rounding shift left +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrshl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(srshl))] +pub unsafe fn vrshl_s64(a: int64x1_t, b: int64x1_t) -> int64x1_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vrshifts.v1i64")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.srshl.v1i64")] + fn vrshl_s64_(a: int64x1_t, b: int64x1_t) -> int64x1_t; + } +vrshl_s64_(a, b) +} + +/// Signed rounding shift left +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrshl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(srshl))] +pub unsafe fn vrshlq_s64(a: int64x2_t, b: int64x2_t) -> int64x2_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vrshifts.v2i64")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.srshl.v2i64")] + fn vrshlq_s64_(a: int64x2_t, b: int64x2_t) -> int64x2_t; + } +vrshlq_s64_(a, b) +} + +/// Unsigned rounding shift left +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrshl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(urshl))] +pub unsafe fn vrshl_u8(a: uint8x8_t, b: int8x8_t) -> uint8x8_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vrshiftu.v8i8")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.urshl.v8i8")] + fn vrshl_u8_(a: uint8x8_t, b: int8x8_t) -> uint8x8_t; + } +vrshl_u8_(a, b) +} + +/// Unsigned rounding shift left +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrshl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(urshl))] +pub unsafe fn vrshlq_u8(a: uint8x16_t, b: int8x16_t) -> uint8x16_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vrshiftu.v16i8")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.urshl.v16i8")] + fn vrshlq_u8_(a: uint8x16_t, b: int8x16_t) -> uint8x16_t; + } +vrshlq_u8_(a, b) +} + +/// Unsigned rounding shift left +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrshl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(urshl))] +pub unsafe fn vrshl_u16(a: uint16x4_t, b: int16x4_t) -> uint16x4_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vrshiftu.v4i16")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.urshl.v4i16")] + fn vrshl_u16_(a: uint16x4_t, b: int16x4_t) -> uint16x4_t; + } +vrshl_u16_(a, b) +} + +/// Unsigned rounding shift left +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrshl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(urshl))] +pub unsafe fn vrshlq_u16(a: uint16x8_t, b: int16x8_t) -> uint16x8_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vrshiftu.v8i16")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.urshl.v8i16")] + fn vrshlq_u16_(a: uint16x8_t, b: int16x8_t) -> uint16x8_t; + } +vrshlq_u16_(a, b) +} + +/// Unsigned rounding shift left +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrshl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(urshl))] +pub unsafe fn vrshl_u32(a: uint32x2_t, b: int32x2_t) -> uint32x2_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vrshiftu.v2i32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.urshl.v2i32")] + fn vrshl_u32_(a: uint32x2_t, b: int32x2_t) -> uint32x2_t; + } +vrshl_u32_(a, b) +} + +/// Unsigned rounding shift left +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrshl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(urshl))] +pub unsafe fn vrshlq_u32(a: uint32x4_t, b: int32x4_t) -> uint32x4_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vrshiftu.v4i32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.urshl.v4i32")] + fn vrshlq_u32_(a: uint32x4_t, b: int32x4_t) -> uint32x4_t; + } +vrshlq_u32_(a, b) +} + +/// Unsigned rounding shift left +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrshl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(urshl))] +pub unsafe fn vrshl_u64(a: uint64x1_t, b: int64x1_t) -> uint64x1_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vrshiftu.v1i64")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.urshl.v1i64")] + fn vrshl_u64_(a: uint64x1_t, b: int64x1_t) -> uint64x1_t; + } +vrshl_u64_(a, b) +} + +/// Unsigned rounding shift left +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrshl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(urshl))] +pub unsafe fn vrshlq_u64(a: uint64x2_t, b: int64x2_t) -> uint64x2_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vrshiftu.v2i64")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.urshl.v2i64")] + fn vrshlq_u64_(a: uint64x2_t, b: int64x2_t) -> uint64x2_t; + } +vrshlq_u64_(a, b) +} + +/// Signed rounding shift right +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrshr, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(srshr, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vrshr_n_s8(a: int8x8_t) -> int8x8_t { + static_assert!(N : i32 where N >= 1 && N <= 8); + vrshl_s8(a, vdup_n_s8((-N).try_into().unwrap())) +} + +/// Signed rounding shift right +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrshr, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(srshr, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vrshrq_n_s8(a: int8x16_t) -> int8x16_t { + static_assert!(N : i32 where N >= 1 && N <= 8); + vrshlq_s8(a, vdupq_n_s8((-N).try_into().unwrap())) +} + +/// Signed rounding shift right +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrshr, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(srshr, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vrshr_n_s16(a: int16x4_t) -> int16x4_t { + static_assert!(N : i32 where N >= 1 && N <= 16); + vrshl_s16(a, vdup_n_s16((-N).try_into().unwrap())) +} + +/// Signed rounding shift right +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrshr, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(srshr, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vrshrq_n_s16(a: int16x8_t) -> int16x8_t { + static_assert!(N : i32 where N >= 1 && N <= 16); + vrshlq_s16(a, vdupq_n_s16((-N).try_into().unwrap())) +} + +/// Signed rounding shift right +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrshr, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(srshr, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vrshr_n_s32(a: int32x2_t) -> int32x2_t { + static_assert!(N : i32 where N >= 1 && N <= 32); + vrshl_s32(a, vdup_n_s32((-N).try_into().unwrap())) +} + +/// Signed rounding shift right +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrshr, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(srshr, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vrshrq_n_s32(a: int32x4_t) -> int32x4_t { + static_assert!(N : i32 where N >= 1 && N <= 32); + vrshlq_s32(a, vdupq_n_s32((-N).try_into().unwrap())) +} + +/// Signed rounding shift right +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrshr, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(srshr, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vrshr_n_s64(a: int64x1_t) -> int64x1_t { + static_assert!(N : i32 where N >= 1 && N <= 64); + vrshl_s64(a, vdup_n_s64((-N).try_into().unwrap())) +} + +/// Signed rounding shift right +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrshr, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(srshr, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vrshrq_n_s64(a: int64x2_t) -> int64x2_t { + static_assert!(N : i32 where N >= 1 && N <= 64); + vrshlq_s64(a, vdupq_n_s64((-N).try_into().unwrap())) +} + +/// Unsigned rounding shift right +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrshr, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(urshr, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vrshr_n_u8(a: uint8x8_t) -> uint8x8_t { + static_assert!(N : i32 where N >= 1 && N <= 8); + vrshl_u8(a, vdup_n_s8((-N).try_into().unwrap())) +} + +/// Unsigned rounding shift right +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrshr, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(urshr, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vrshrq_n_u8(a: uint8x16_t) -> uint8x16_t { + static_assert!(N : i32 where N >= 1 && N <= 8); + vrshlq_u8(a, vdupq_n_s8((-N).try_into().unwrap())) +} + +/// Unsigned rounding shift right +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrshr, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(urshr, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vrshr_n_u16(a: uint16x4_t) -> uint16x4_t { + static_assert!(N : i32 where N >= 1 && N <= 16); + vrshl_u16(a, vdup_n_s16((-N).try_into().unwrap())) +} + +/// Unsigned rounding shift right +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrshr, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(urshr, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vrshrq_n_u16(a: uint16x8_t) -> uint16x8_t { + static_assert!(N : i32 where N >= 1 && N <= 16); + vrshlq_u16(a, vdupq_n_s16((-N).try_into().unwrap())) +} + +/// Unsigned rounding shift right +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrshr, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(urshr, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vrshr_n_u32(a: uint32x2_t) -> uint32x2_t { + static_assert!(N : i32 where N >= 1 && N <= 32); + vrshl_u32(a, vdup_n_s32((-N).try_into().unwrap())) +} + +/// Unsigned rounding shift right +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrshr, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(urshr, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vrshrq_n_u32(a: uint32x4_t) -> uint32x4_t { + static_assert!(N : i32 where N >= 1 && N <= 32); + vrshlq_u32(a, vdupq_n_s32((-N).try_into().unwrap())) +} + +/// Unsigned rounding shift right +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrshr, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(urshr, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vrshr_n_u64(a: uint64x1_t) -> uint64x1_t { + static_assert!(N : i32 where N >= 1 && N <= 64); + vrshl_u64(a, vdup_n_s64((-N).try_into().unwrap())) +} + +/// Unsigned rounding shift right +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrshr, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(urshr, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vrshrq_n_u64(a: uint64x2_t) -> uint64x2_t { + static_assert!(N : i32 where N >= 1 && N <= 64); + vrshlq_u64(a, vdupq_n_s64((-N).try_into().unwrap())) +} + +/// Rounding shift right narrow +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrshrn, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vrshrn_n_s16(a: int16x8_t) -> int8x8_t { + static_assert!(N : i32 where N >= 1 && N <= 8); + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vrshiftn.v8i8")] + fn vrshrn_n_s16_(a: int16x8_t, n: int16x8_t) -> int8x8_t; + } +vrshrn_n_s16_(a, int16x8_t(-N as i16, -N as i16, -N as i16, -N as i16, -N as i16, -N as i16, -N as i16, -N as i16)) +} + +/// Rounding shift right narrow +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(rshrn, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vrshrn_n_s16(a: int16x8_t) -> int8x8_t { + static_assert!(N : i32 where N >= 1 && N <= 8); + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.rshrn.v8i8")] + fn vrshrn_n_s16_(a: int16x8_t, n: i32) -> int8x8_t; + } +vrshrn_n_s16_(a, N) +} + +/// Rounding shift right narrow +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrshrn, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vrshrn_n_s32(a: int32x4_t) -> int16x4_t { + static_assert!(N : i32 where N >= 1 && N <= 16); + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vrshiftn.v4i16")] + fn vrshrn_n_s32_(a: int32x4_t, n: int32x4_t) -> int16x4_t; + } +vrshrn_n_s32_(a, int32x4_t(-N as i32, -N as i32, -N as i32, -N as i32)) +} + +/// Rounding shift right narrow +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(rshrn, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vrshrn_n_s32(a: int32x4_t) -> int16x4_t { + static_assert!(N : i32 where N >= 1 && N <= 16); + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.rshrn.v4i16")] + fn vrshrn_n_s32_(a: int32x4_t, n: i32) -> int16x4_t; + } +vrshrn_n_s32_(a, N) +} + +/// Rounding shift right narrow +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,v7")] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrshrn, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vrshrn_n_s64(a: int64x2_t) -> int32x2_t { + static_assert!(N : i32 where N >= 1 && N <= 32); + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vrshiftn.v2i32")] + fn vrshrn_n_s64_(a: int64x2_t, n: int64x2_t) -> int32x2_t; + } +vrshrn_n_s64_(a, int64x2_t(-N as i64, -N as i64)) +} + +/// Rounding shift right narrow +#[inline] +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(rshrn, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vrshrn_n_s64(a: int64x2_t) -> int32x2_t { + static_assert!(N : i32 where N >= 1 && N <= 32); + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.rshrn.v2i32")] + fn vrshrn_n_s64_(a: int64x2_t, n: i32) -> int32x2_t; + } +vrshrn_n_s64_(a, N) +} + +/// Rounding shift right narrow +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrshrn, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(rshrn, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vrshrn_n_u16(a: uint16x8_t) -> uint8x8_t { + static_assert!(N : i32 where N >= 1 && N <= 8); + transmute(vrshrn_n_s16::(transmute(a))) +} + +/// Rounding shift right narrow +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrshrn, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(rshrn, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vrshrn_n_u32(a: uint32x4_t) -> uint16x4_t { + static_assert!(N : i32 where N >= 1 && N <= 16); + transmute(vrshrn_n_s32::(transmute(a))) +} + +/// Rounding shift right narrow +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrshrn, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(rshrn, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vrshrn_n_u64(a: uint64x2_t) -> uint32x2_t { + static_assert!(N : i32 where N >= 1 && N <= 32); + transmute(vrshrn_n_s64::(transmute(a))) +} + +/// Signed rounding shift right and accumulate +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrsra, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(srsra, N = 2))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vrsra_n_s8(a: int8x8_t, b: int8x8_t) -> int8x8_t { + static_assert!(N : i32 where N >= 1 && N <= 8); + simd_add(a, vrshr_n_s8::(b)) +} + +/// Signed rounding shift right and accumulate +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrsra, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(srsra, N = 2))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vrsraq_n_s8(a: int8x16_t, b: int8x16_t) -> int8x16_t { + static_assert!(N : i32 where N >= 1 && N <= 8); + simd_add(a, vrshrq_n_s8::(b)) +} + +/// Signed rounding shift right and accumulate +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrsra, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(srsra, N = 2))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vrsra_n_s16(a: int16x4_t, b: int16x4_t) -> int16x4_t { + static_assert!(N : i32 where N >= 1 && N <= 16); + simd_add(a, vrshr_n_s16::(b)) +} + +/// Signed rounding shift right and accumulate +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrsra, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(srsra, N = 2))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vrsraq_n_s16(a: int16x8_t, b: int16x8_t) -> int16x8_t { + static_assert!(N : i32 where N >= 1 && N <= 16); + simd_add(a, vrshrq_n_s16::(b)) +} + +/// Signed rounding shift right and accumulate +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrsra, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(srsra, N = 2))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vrsra_n_s32(a: int32x2_t, b: int32x2_t) -> int32x2_t { + static_assert!(N : i32 where N >= 1 && N <= 32); + simd_add(a, vrshr_n_s32::(b)) +} + +/// Signed rounding shift right and accumulate +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrsra, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(srsra, N = 2))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vrsraq_n_s32(a: int32x4_t, b: int32x4_t) -> int32x4_t { + static_assert!(N : i32 where N >= 1 && N <= 32); + simd_add(a, vrshrq_n_s32::(b)) +} + +/// Signed rounding shift right and accumulate +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrsra, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(srsra, N = 2))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vrsra_n_s64(a: int64x1_t, b: int64x1_t) -> int64x1_t { + static_assert!(N : i32 where N >= 1 && N <= 64); + simd_add(a, vrshr_n_s64::(b)) +} + +/// Signed rounding shift right and accumulate +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrsra, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(srsra, N = 2))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vrsraq_n_s64(a: int64x2_t, b: int64x2_t) -> int64x2_t { + static_assert!(N : i32 where N >= 1 && N <= 64); + simd_add(a, vrshrq_n_s64::(b)) +} + +/// Unsigned rounding shift right and accumulate +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrsra, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ursra, N = 2))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vrsra_n_u8(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t { + static_assert!(N : i32 where N >= 1 && N <= 8); + simd_add(a, vrshr_n_u8::(b)) +} + +/// Unsigned rounding shift right and accumulate +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrsra, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ursra, N = 2))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vrsraq_n_u8(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t { + static_assert!(N : i32 where N >= 1 && N <= 8); + simd_add(a, vrshrq_n_u8::(b)) +} + +/// Unsigned rounding shift right and accumulate +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrsra, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ursra, N = 2))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vrsra_n_u16(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t { + static_assert!(N : i32 where N >= 1 && N <= 16); + simd_add(a, vrshr_n_u16::(b)) +} + +/// Unsigned rounding shift right and accumulate +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrsra, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ursra, N = 2))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vrsraq_n_u16(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t { + static_assert!(N : i32 where N >= 1 && N <= 16); + simd_add(a, vrshrq_n_u16::(b)) +} + +/// Unsigned rounding shift right and accumulate +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrsra, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ursra, N = 2))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vrsra_n_u32(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t { + static_assert!(N : i32 where N >= 1 && N <= 32); + simd_add(a, vrshr_n_u32::(b)) +} + +/// Unsigned rounding shift right and accumulate +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrsra, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ursra, N = 2))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vrsraq_n_u32(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t { + static_assert!(N : i32 where N >= 1 && N <= 32); + simd_add(a, vrshrq_n_u32::(b)) +} + +/// Unsigned rounding shift right and accumulate +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrsra, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ursra, N = 2))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vrsra_n_u64(a: uint64x1_t, b: uint64x1_t) -> uint64x1_t { + static_assert!(N : i32 where N >= 1 && N <= 64); + simd_add(a, vrshr_n_u64::(b)) +} + +/// Unsigned rounding shift right and accumulate +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrsra, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ursra, N = 2))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vrsraq_n_u64(a: uint64x2_t, b: uint64x2_t) -> uint64x2_t { + static_assert!(N : i32 where N >= 1 && N <= 64); + simd_add(a, vrshrq_n_u64::(b)) +} + +/// Insert vector element from another vector element +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, LANE = 0))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop, LANE = 0))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vset_lane_s8(a: i8, b: int8x8_t) -> int8x8_t { + static_assert_imm3!(LANE); + simd_insert(b, LANE as u32, a) +} + +/// Insert vector element from another vector element +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, LANE = 0))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop, LANE = 0))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vset_lane_s16(a: i16, b: int16x4_t) -> int16x4_t { + static_assert_imm2!(LANE); + simd_insert(b, LANE as u32, a) +} + +/// Insert vector element from another vector element +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, LANE = 0))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop, LANE = 0))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vset_lane_s32(a: i32, b: int32x2_t) -> int32x2_t { + static_assert_imm1!(LANE); + simd_insert(b, LANE as u32, a) +} + +/// Insert vector element from another vector element +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, LANE = 0))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop, LANE = 0))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vset_lane_s64(a: i64, b: int64x1_t) -> int64x1_t { + static_assert!(LANE : i32 where LANE == 0); + simd_insert(b, LANE as u32, a) +} + +/// Insert vector element from another vector element +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, LANE = 0))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop, LANE = 0))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vset_lane_u8(a: u8, b: uint8x8_t) -> uint8x8_t { + static_assert_imm3!(LANE); + simd_insert(b, LANE as u32, a) +} + +/// Insert vector element from another vector element +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, LANE = 0))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop, LANE = 0))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vset_lane_u16(a: u16, b: uint16x4_t) -> uint16x4_t { + static_assert_imm2!(LANE); + simd_insert(b, LANE as u32, a) +} + +/// Insert vector element from another vector element +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, LANE = 0))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop, LANE = 0))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vset_lane_u32(a: u32, b: uint32x2_t) -> uint32x2_t { + static_assert_imm1!(LANE); + simd_insert(b, LANE as u32, a) +} + +/// Insert vector element from another vector element +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, LANE = 0))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop, LANE = 0))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vset_lane_u64(a: u64, b: uint64x1_t) -> uint64x1_t { + static_assert!(LANE : i32 where LANE == 0); + simd_insert(b, LANE as u32, a) +} + +/// Insert vector element from another vector element +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, LANE = 0))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop, LANE = 0))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vset_lane_p8(a: p8, b: poly8x8_t) -> poly8x8_t { + static_assert_imm3!(LANE); + simd_insert(b, LANE as u32, a) +} + +/// Insert vector element from another vector element +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, LANE = 0))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop, LANE = 0))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vset_lane_p16(a: p16, b: poly16x4_t) -> poly16x4_t { + static_assert_imm2!(LANE); + simd_insert(b, LANE as u32, a) +} + +/// Insert vector element from another vector element +#[inline] +#[target_feature(enable = "neon,crypto")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "crypto,v8"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, LANE = 0))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop, LANE = 0))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vset_lane_p64(a: p64, b: poly64x1_t) -> poly64x1_t { + static_assert!(LANE : i32 where LANE == 0); + simd_insert(b, LANE as u32, a) +} + +/// Insert vector element from another vector element +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, LANE = 0))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop, LANE = 0))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vsetq_lane_s8(a: i8, b: int8x16_t) -> int8x16_t { + static_assert_imm4!(LANE); + simd_insert(b, LANE as u32, a) +} + +/// Insert vector element from another vector element +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, LANE = 0))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop, LANE = 0))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vsetq_lane_s16(a: i16, b: int16x8_t) -> int16x8_t { + static_assert_imm3!(LANE); + simd_insert(b, LANE as u32, a) +} + +/// Insert vector element from another vector element +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, LANE = 0))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop, LANE = 0))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vsetq_lane_s32(a: i32, b: int32x4_t) -> int32x4_t { + static_assert_imm2!(LANE); + simd_insert(b, LANE as u32, a) +} + +/// Insert vector element from another vector element +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, LANE = 0))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop, LANE = 0))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vsetq_lane_s64(a: i64, b: int64x2_t) -> int64x2_t { + static_assert_imm1!(LANE); + simd_insert(b, LANE as u32, a) +} + +/// Insert vector element from another vector element +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, LANE = 0))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop, LANE = 0))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vsetq_lane_u8(a: u8, b: uint8x16_t) -> uint8x16_t { + static_assert_imm4!(LANE); + simd_insert(b, LANE as u32, a) +} + +/// Insert vector element from another vector element +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, LANE = 0))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop, LANE = 0))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vsetq_lane_u16(a: u16, b: uint16x8_t) -> uint16x8_t { + static_assert_imm3!(LANE); + simd_insert(b, LANE as u32, a) +} + +/// Insert vector element from another vector element +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, LANE = 0))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop, LANE = 0))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vsetq_lane_u32(a: u32, b: uint32x4_t) -> uint32x4_t { + static_assert_imm2!(LANE); + simd_insert(b, LANE as u32, a) +} + +/// Insert vector element from another vector element +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, LANE = 0))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop, LANE = 0))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vsetq_lane_u64(a: u64, b: uint64x2_t) -> uint64x2_t { + static_assert_imm1!(LANE); + simd_insert(b, LANE as u32, a) +} + +/// Insert vector element from another vector element +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, LANE = 0))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop, LANE = 0))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vsetq_lane_p8(a: p8, b: poly8x16_t) -> poly8x16_t { + static_assert_imm4!(LANE); + simd_insert(b, LANE as u32, a) +} + +/// Insert vector element from another vector element +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, LANE = 0))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop, LANE = 0))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vsetq_lane_p16(a: p16, b: poly16x8_t) -> poly16x8_t { + static_assert_imm3!(LANE); + simd_insert(b, LANE as u32, a) +} + +/// Insert vector element from another vector element +#[inline] +#[target_feature(enable = "neon,crypto")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "crypto,v8"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, LANE = 0))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop, LANE = 0))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vsetq_lane_p64(a: p64, b: poly64x2_t) -> poly64x2_t { + static_assert_imm1!(LANE); + simd_insert(b, LANE as u32, a) +} + +/// Insert vector element from another vector element +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, LANE = 0))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop, LANE = 0))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vset_lane_f32(a: f32, b: float32x2_t) -> float32x2_t { + static_assert_imm1!(LANE); + simd_insert(b, LANE as u32, a) +} + +/// Insert vector element from another vector element +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(nop, LANE = 0))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(nop, LANE = 0))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vsetq_lane_f32(a: f32, b: float32x4_t) -> float32x4_t { + static_assert_imm2!(LANE); + simd_insert(b, LANE as u32, a) +} + +/// Signed Shift left +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vshl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sshl))] +pub unsafe fn vshl_s8(a: int8x8_t, b: int8x8_t) -> int8x8_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vshifts.v8i8")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sshl.v8i8")] + fn vshl_s8_(a: int8x8_t, b: int8x8_t) -> int8x8_t; + } +vshl_s8_(a, b) +} + +/// Signed Shift left +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vshl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sshl))] +pub unsafe fn vshlq_s8(a: int8x16_t, b: int8x16_t) -> int8x16_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vshifts.v16i8")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sshl.v16i8")] + fn vshlq_s8_(a: int8x16_t, b: int8x16_t) -> int8x16_t; + } +vshlq_s8_(a, b) +} + +/// Signed Shift left +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vshl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sshl))] +pub unsafe fn vshl_s16(a: int16x4_t, b: int16x4_t) -> int16x4_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vshifts.v4i16")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sshl.v4i16")] + fn vshl_s16_(a: int16x4_t, b: int16x4_t) -> int16x4_t; + } +vshl_s16_(a, b) +} + +/// Signed Shift left +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vshl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sshl))] +pub unsafe fn vshlq_s16(a: int16x8_t, b: int16x8_t) -> int16x8_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vshifts.v8i16")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sshl.v8i16")] + fn vshlq_s16_(a: int16x8_t, b: int16x8_t) -> int16x8_t; + } +vshlq_s16_(a, b) +} + +/// Signed Shift left +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vshl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sshl))] +pub unsafe fn vshl_s32(a: int32x2_t, b: int32x2_t) -> int32x2_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vshifts.v2i32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sshl.v2i32")] + fn vshl_s32_(a: int32x2_t, b: int32x2_t) -> int32x2_t; + } +vshl_s32_(a, b) +} + +/// Signed Shift left +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vshl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sshl))] +pub unsafe fn vshlq_s32(a: int32x4_t, b: int32x4_t) -> int32x4_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vshifts.v4i32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sshl.v4i32")] + fn vshlq_s32_(a: int32x4_t, b: int32x4_t) -> int32x4_t; + } +vshlq_s32_(a, b) +} + +/// Signed Shift left +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vshl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sshl))] +pub unsafe fn vshl_s64(a: int64x1_t, b: int64x1_t) -> int64x1_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vshifts.v1i64")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sshl.v1i64")] + fn vshl_s64_(a: int64x1_t, b: int64x1_t) -> int64x1_t; + } +vshl_s64_(a, b) +} + +/// Signed Shift left +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vshl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sshl))] +pub unsafe fn vshlq_s64(a: int64x2_t, b: int64x2_t) -> int64x2_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vshifts.v2i64")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sshl.v2i64")] + fn vshlq_s64_(a: int64x2_t, b: int64x2_t) -> int64x2_t; + } +vshlq_s64_(a, b) +} + +/// Unsigned Shift left +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vshl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ushl))] +pub unsafe fn vshl_u8(a: uint8x8_t, b: int8x8_t) -> uint8x8_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vshiftu.v8i8")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ushl.v8i8")] + fn vshl_u8_(a: uint8x8_t, b: int8x8_t) -> uint8x8_t; + } +vshl_u8_(a, b) +} + +/// Unsigned Shift left +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vshl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ushl))] +pub unsafe fn vshlq_u8(a: uint8x16_t, b: int8x16_t) -> uint8x16_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vshiftu.v16i8")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ushl.v16i8")] + fn vshlq_u8_(a: uint8x16_t, b: int8x16_t) -> uint8x16_t; + } +vshlq_u8_(a, b) +} + +/// Unsigned Shift left +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vshl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ushl))] +pub unsafe fn vshl_u16(a: uint16x4_t, b: int16x4_t) -> uint16x4_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vshiftu.v4i16")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ushl.v4i16")] + fn vshl_u16_(a: uint16x4_t, b: int16x4_t) -> uint16x4_t; + } +vshl_u16_(a, b) +} + +/// Unsigned Shift left +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vshl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ushl))] +pub unsafe fn vshlq_u16(a: uint16x8_t, b: int16x8_t) -> uint16x8_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vshiftu.v8i16")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ushl.v8i16")] + fn vshlq_u16_(a: uint16x8_t, b: int16x8_t) -> uint16x8_t; + } +vshlq_u16_(a, b) +} + +/// Unsigned Shift left +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vshl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ushl))] +pub unsafe fn vshl_u32(a: uint32x2_t, b: int32x2_t) -> uint32x2_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vshiftu.v2i32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ushl.v2i32")] + fn vshl_u32_(a: uint32x2_t, b: int32x2_t) -> uint32x2_t; + } +vshl_u32_(a, b) +} + +/// Unsigned Shift left +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vshl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ushl))] +pub unsafe fn vshlq_u32(a: uint32x4_t, b: int32x4_t) -> uint32x4_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vshiftu.v4i32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ushl.v4i32")] + fn vshlq_u32_(a: uint32x4_t, b: int32x4_t) -> uint32x4_t; + } +vshlq_u32_(a, b) +} + +/// Unsigned Shift left +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vshl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ushl))] +pub unsafe fn vshl_u64(a: uint64x1_t, b: int64x1_t) -> uint64x1_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vshiftu.v1i64")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ushl.v1i64")] + fn vshl_u64_(a: uint64x1_t, b: int64x1_t) -> uint64x1_t; + } +vshl_u64_(a, b) +} + +/// Unsigned Shift left +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vshl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ushl))] +pub unsafe fn vshlq_u64(a: uint64x2_t, b: int64x2_t) -> uint64x2_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vshiftu.v2i64")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.ushl.v2i64")] + fn vshlq_u64_(a: uint64x2_t, b: int64x2_t) -> uint64x2_t; + } +vshlq_u64_(a, b) +} + +/// Shift left +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vshl, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(shl, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vshl_n_s8(a: int8x8_t) -> int8x8_t { + static_assert_imm3!(N); + simd_shl(a, vdup_n_s8(N.try_into().unwrap())) +} + +/// Shift left +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vshl, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(shl, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vshlq_n_s8(a: int8x16_t) -> int8x16_t { + static_assert_imm3!(N); + simd_shl(a, vdupq_n_s8(N.try_into().unwrap())) +} + +/// Shift left +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vshl, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(shl, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vshl_n_s16(a: int16x4_t) -> int16x4_t { + static_assert_imm4!(N); + simd_shl(a, vdup_n_s16(N.try_into().unwrap())) +} + +/// Shift left +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vshl, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(shl, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vshlq_n_s16(a: int16x8_t) -> int16x8_t { + static_assert_imm4!(N); + simd_shl(a, vdupq_n_s16(N.try_into().unwrap())) +} + +/// Shift left +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vshl, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(shl, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vshl_n_s32(a: int32x2_t) -> int32x2_t { + static_assert_imm5!(N); + simd_shl(a, vdup_n_s32(N.try_into().unwrap())) +} + +/// Shift left +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vshl, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(shl, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vshlq_n_s32(a: int32x4_t) -> int32x4_t { + static_assert_imm5!(N); + simd_shl(a, vdupq_n_s32(N.try_into().unwrap())) +} + +/// Shift left +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vshl, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(shl, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vshl_n_u8(a: uint8x8_t) -> uint8x8_t { + static_assert_imm3!(N); + simd_shl(a, vdup_n_u8(N.try_into().unwrap())) +} + +/// Shift left +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vshl, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(shl, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vshlq_n_u8(a: uint8x16_t) -> uint8x16_t { + static_assert_imm3!(N); + simd_shl(a, vdupq_n_u8(N.try_into().unwrap())) +} + +/// Shift left +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vshl, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(shl, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vshl_n_u16(a: uint16x4_t) -> uint16x4_t { + static_assert_imm4!(N); + simd_shl(a, vdup_n_u16(N.try_into().unwrap())) +} + +/// Shift left +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vshl, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(shl, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vshlq_n_u16(a: uint16x8_t) -> uint16x8_t { + static_assert_imm4!(N); + simd_shl(a, vdupq_n_u16(N.try_into().unwrap())) +} + +/// Shift left +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vshl, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(shl, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vshl_n_u32(a: uint32x2_t) -> uint32x2_t { + static_assert_imm5!(N); + simd_shl(a, vdup_n_u32(N.try_into().unwrap())) +} + +/// Shift left +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vshl, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(shl, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vshlq_n_u32(a: uint32x4_t) -> uint32x4_t { + static_assert_imm5!(N); + simd_shl(a, vdupq_n_u32(N.try_into().unwrap())) +} + +/// Shift left +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vshl, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(shl, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vshl_n_s64(a: int64x1_t) -> int64x1_t { + static_assert_imm6!(N); + simd_shl(a, vdup_n_s64(N.try_into().unwrap())) +} + +/// Shift left +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vshl, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(shl, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vshlq_n_s64(a: int64x2_t) -> int64x2_t { + static_assert_imm6!(N); + simd_shl(a, vdupq_n_s64(N.try_into().unwrap())) +} + +/// Shift left +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vshl, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(shl, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vshl_n_u64(a: uint64x1_t) -> uint64x1_t { + static_assert_imm6!(N); + simd_shl(a, vdup_n_u64(N.try_into().unwrap())) +} + +/// Shift left +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vshl, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(shl, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vshlq_n_u64(a: uint64x2_t) -> uint64x2_t { + static_assert_imm6!(N); + simd_shl(a, vdupq_n_u64(N.try_into().unwrap())) +} + +/// Signed shift left long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vshll.s8", N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sshll, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vshll_n_s8(a: int8x8_t) -> int16x8_t { + static_assert!(N : i32 where N >= 0 && N <= 8); + simd_shl(simd_cast(a), vdupq_n_s16(N.try_into().unwrap())) +} + +/// Signed shift left long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vshll.s16", N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sshll, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vshll_n_s16(a: int16x4_t) -> int32x4_t { + static_assert!(N : i32 where N >= 0 && N <= 16); + simd_shl(simd_cast(a), vdupq_n_s32(N.try_into().unwrap())) +} + +/// Signed shift left long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vshll.s32", N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sshll, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vshll_n_s32(a: int32x2_t) -> int64x2_t { + static_assert!(N : i32 where N >= 0 && N <= 32); + simd_shl(simd_cast(a), vdupq_n_s64(N.try_into().unwrap())) +} + +/// Signed shift left long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vshll.u8", N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ushll, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vshll_n_u8(a: uint8x8_t) -> uint16x8_t { + static_assert!(N : i32 where N >= 0 && N <= 8); + simd_shl(simd_cast(a), vdupq_n_u16(N.try_into().unwrap())) +} + +/// Signed shift left long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vshll.u16", N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ushll, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vshll_n_u16(a: uint16x4_t) -> uint32x4_t { + static_assert!(N : i32 where N >= 0 && N <= 16); + simd_shl(simd_cast(a), vdupq_n_u32(N.try_into().unwrap())) +} + +/// Signed shift left long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vshll.u32", N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ushll, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vshll_n_u32(a: uint32x2_t) -> uint64x2_t { + static_assert!(N : i32 where N >= 0 && N <= 32); + simd_shl(simd_cast(a), vdupq_n_u64(N.try_into().unwrap())) +} + +/// Shift right +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vshr.s8", N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sshr, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vshr_n_s8(a: int8x8_t) -> int8x8_t { + static_assert!(N : i32 where N >= 1 && N <= 8); + simd_shr(a, vdup_n_s8(N.try_into().unwrap())) +} + +/// Shift right +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vshr.s8", N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sshr, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vshrq_n_s8(a: int8x16_t) -> int8x16_t { + static_assert!(N : i32 where N >= 1 && N <= 8); + simd_shr(a, vdupq_n_s8(N.try_into().unwrap())) +} + +/// Shift right +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vshr.s16", N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sshr, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vshr_n_s16(a: int16x4_t) -> int16x4_t { + static_assert!(N : i32 where N >= 1 && N <= 16); + simd_shr(a, vdup_n_s16(N.try_into().unwrap())) +} + +/// Shift right +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vshr.s16", N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sshr, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vshrq_n_s16(a: int16x8_t) -> int16x8_t { + static_assert!(N : i32 where N >= 1 && N <= 16); + simd_shr(a, vdupq_n_s16(N.try_into().unwrap())) +} + +/// Shift right +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vshr.s32", N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sshr, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vshr_n_s32(a: int32x2_t) -> int32x2_t { + static_assert!(N : i32 where N >= 1 && N <= 32); + simd_shr(a, vdup_n_s32(N.try_into().unwrap())) +} + +/// Shift right +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vshr.s32", N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sshr, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vshrq_n_s32(a: int32x4_t) -> int32x4_t { + static_assert!(N : i32 where N >= 1 && N <= 32); + simd_shr(a, vdupq_n_s32(N.try_into().unwrap())) +} + +/// Shift right +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vshr.s64", N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sshr, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vshr_n_s64(a: int64x1_t) -> int64x1_t { + static_assert!(N : i32 where N >= 1 && N <= 64); + simd_shr(a, vdup_n_s64(N.try_into().unwrap())) +} + +/// Shift right +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vshr.s64", N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sshr, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vshrq_n_s64(a: int64x2_t) -> int64x2_t { + static_assert!(N : i32 where N >= 1 && N <= 64); + simd_shr(a, vdupq_n_s64(N.try_into().unwrap())) +} + +/// Shift right +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vshr.u8", N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ushr, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vshr_n_u8(a: uint8x8_t) -> uint8x8_t { + static_assert!(N : i32 where N >= 1 && N <= 8); + simd_shr(a, vdup_n_u8(N.try_into().unwrap())) +} + +/// Shift right +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vshr.u8", N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ushr, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vshrq_n_u8(a: uint8x16_t) -> uint8x16_t { + static_assert!(N : i32 where N >= 1 && N <= 8); + simd_shr(a, vdupq_n_u8(N.try_into().unwrap())) +} + +/// Shift right +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vshr.u16", N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ushr, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vshr_n_u16(a: uint16x4_t) -> uint16x4_t { + static_assert!(N : i32 where N >= 1 && N <= 16); + simd_shr(a, vdup_n_u16(N.try_into().unwrap())) +} + +/// Shift right +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vshr.u16", N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ushr, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vshrq_n_u16(a: uint16x8_t) -> uint16x8_t { + static_assert!(N : i32 where N >= 1 && N <= 16); + simd_shr(a, vdupq_n_u16(N.try_into().unwrap())) +} + +/// Shift right +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vshr.u32", N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ushr, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vshr_n_u32(a: uint32x2_t) -> uint32x2_t { + static_assert!(N : i32 where N >= 1 && N <= 32); + simd_shr(a, vdup_n_u32(N.try_into().unwrap())) +} + +/// Shift right +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vshr.u32", N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ushr, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vshrq_n_u32(a: uint32x4_t) -> uint32x4_t { + static_assert!(N : i32 where N >= 1 && N <= 32); + simd_shr(a, vdupq_n_u32(N.try_into().unwrap())) +} + +/// Shift right +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vshr.u64", N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ushr, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vshr_n_u64(a: uint64x1_t) -> uint64x1_t { + static_assert!(N : i32 where N >= 1 && N <= 64); + simd_shr(a, vdup_n_u64(N.try_into().unwrap())) +} + +/// Shift right +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vshr.u64", N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ushr, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vshrq_n_u64(a: uint64x2_t) -> uint64x2_t { + static_assert!(N : i32 where N >= 1 && N <= 64); + simd_shr(a, vdupq_n_u64(N.try_into().unwrap())) +} + +/// Shift right narrow +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vshrn.i16", N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(shrn, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vshrn_n_s16(a: int16x8_t) -> int8x8_t { + static_assert!(N : i32 where N >= 1 && N <= 8); + simd_cast(simd_shr(a, vdupq_n_s16(N.try_into().unwrap()))) +} + +/// Shift right narrow +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vshrn.i32", N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(shrn, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vshrn_n_s32(a: int32x4_t) -> int16x4_t { + static_assert!(N : i32 where N >= 1 && N <= 16); + simd_cast(simd_shr(a, vdupq_n_s32(N.try_into().unwrap()))) +} + +/// Shift right narrow +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vshrn.i64", N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(shrn, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vshrn_n_s64(a: int64x2_t) -> int32x2_t { + static_assert!(N : i32 where N >= 1 && N <= 32); + simd_cast(simd_shr(a, vdupq_n_s64(N.try_into().unwrap()))) +} + +/// Shift right narrow +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vshrn.i16", N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(shrn, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vshrn_n_u16(a: uint16x8_t) -> uint8x8_t { + static_assert!(N : i32 where N >= 1 && N <= 8); + simd_cast(simd_shr(a, vdupq_n_u16(N.try_into().unwrap()))) +} + +/// Shift right narrow +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vshrn.i32", N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(shrn, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vshrn_n_u32(a: uint32x4_t) -> uint16x4_t { + static_assert!(N : i32 where N >= 1 && N <= 16); + simd_cast(simd_shr(a, vdupq_n_u32(N.try_into().unwrap()))) +} + +/// Shift right narrow +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vshrn.i64", N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(shrn, N = 2))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn vshrn_n_u64(a: uint64x2_t) -> uint32x2_t { + static_assert!(N : i32 where N >= 1 && N <= 32); + simd_cast(simd_shr(a, vdupq_n_u64(N.try_into().unwrap()))) +} + +/// Signed shift right and accumulate +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vsra, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ssra, N = 2))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vsra_n_s8(a: int8x8_t, b: int8x8_t) -> int8x8_t { + static_assert!(N : i32 where N >= 1 && N <= 8); + simd_add(a, vshr_n_s8::(b)) +} + +/// Signed shift right and accumulate +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vsra, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ssra, N = 2))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vsraq_n_s8(a: int8x16_t, b: int8x16_t) -> int8x16_t { + static_assert!(N : i32 where N >= 1 && N <= 8); + simd_add(a, vshrq_n_s8::(b)) +} + +/// Signed shift right and accumulate +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vsra, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ssra, N = 2))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vsra_n_s16(a: int16x4_t, b: int16x4_t) -> int16x4_t { + static_assert!(N : i32 where N >= 1 && N <= 16); + simd_add(a, vshr_n_s16::(b)) +} + +/// Signed shift right and accumulate +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vsra, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ssra, N = 2))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vsraq_n_s16(a: int16x8_t, b: int16x8_t) -> int16x8_t { + static_assert!(N : i32 where N >= 1 && N <= 16); + simd_add(a, vshrq_n_s16::(b)) +} + +/// Signed shift right and accumulate +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vsra, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ssra, N = 2))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vsra_n_s32(a: int32x2_t, b: int32x2_t) -> int32x2_t { + static_assert!(N : i32 where N >= 1 && N <= 32); + simd_add(a, vshr_n_s32::(b)) +} + +/// Signed shift right and accumulate +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vsra, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ssra, N = 2))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vsraq_n_s32(a: int32x4_t, b: int32x4_t) -> int32x4_t { + static_assert!(N : i32 where N >= 1 && N <= 32); + simd_add(a, vshrq_n_s32::(b)) +} + +/// Signed shift right and accumulate +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vsra, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ssra, N = 2))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vsra_n_s64(a: int64x1_t, b: int64x1_t) -> int64x1_t { + static_assert!(N : i32 where N >= 1 && N <= 64); + simd_add(a, vshr_n_s64::(b)) +} + +/// Signed shift right and accumulate +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vsra, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ssra, N = 2))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vsraq_n_s64(a: int64x2_t, b: int64x2_t) -> int64x2_t { + static_assert!(N : i32 where N >= 1 && N <= 64); + simd_add(a, vshrq_n_s64::(b)) +} + +/// Unsigned shift right and accumulate +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vsra, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(usra, N = 2))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vsra_n_u8(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t { + static_assert!(N : i32 where N >= 1 && N <= 8); + simd_add(a, vshr_n_u8::(b)) +} + +/// Unsigned shift right and accumulate +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vsra, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(usra, N = 2))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vsraq_n_u8(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t { + static_assert!(N : i32 where N >= 1 && N <= 8); + simd_add(a, vshrq_n_u8::(b)) +} + +/// Unsigned shift right and accumulate +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vsra, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(usra, N = 2))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vsra_n_u16(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t { + static_assert!(N : i32 where N >= 1 && N <= 16); + simd_add(a, vshr_n_u16::(b)) +} + +/// Unsigned shift right and accumulate +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vsra, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(usra, N = 2))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vsraq_n_u16(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t { + static_assert!(N : i32 where N >= 1 && N <= 16); + simd_add(a, vshrq_n_u16::(b)) +} + +/// Unsigned shift right and accumulate +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vsra, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(usra, N = 2))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vsra_n_u32(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t { + static_assert!(N : i32 where N >= 1 && N <= 32); + simd_add(a, vshr_n_u32::(b)) +} + +/// Unsigned shift right and accumulate +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vsra, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(usra, N = 2))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vsraq_n_u32(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t { + static_assert!(N : i32 where N >= 1 && N <= 32); + simd_add(a, vshrq_n_u32::(b)) +} + +/// Unsigned shift right and accumulate +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vsra, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(usra, N = 2))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vsra_n_u64(a: uint64x1_t, b: uint64x1_t) -> uint64x1_t { + static_assert!(N : i32 where N >= 1 && N <= 64); + simd_add(a, vshr_n_u64::(b)) +} + +/// Unsigned shift right and accumulate +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vsra, N = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(usra, N = 2))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vsraq_n_u64(a: uint64x2_t, b: uint64x2_t) -> uint64x2_t { + static_assert!(N : i32 where N >= 1 && N <= 64); + simd_add(a, vshrq_n_u64::(b)) +} + +/// Unsigned Absolute difference and Accumulate Long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vabal.u8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uabal))] +pub unsafe fn vabal_u8(a: uint16x8_t, b: uint8x8_t, c: uint8x8_t) -> uint16x8_t { + let d: uint8x8_t = vabd_u8(b, c); + simd_add(a, simd_cast(d)) +} + +/// Unsigned Absolute difference and Accumulate Long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vabal.u16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uabal))] +pub unsafe fn vabal_u16(a: uint32x4_t, b: uint16x4_t, c: uint16x4_t) -> uint32x4_t { + let d: uint16x4_t = vabd_u16(b, c); + simd_add(a, simd_cast(d)) +} + +/// Unsigned Absolute difference and Accumulate Long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vabal.u32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uabal))] +pub unsafe fn vabal_u32(a: uint64x2_t, b: uint32x2_t, c: uint32x2_t) -> uint64x2_t { + let d: uint32x2_t = vabd_u32(b, c); + simd_add(a, simd_cast(d)) +} + +/// Signed Absolute difference and Accumulate Long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vabal.s8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sabal))] +pub unsafe fn vabal_s8(a: int16x8_t, b: int8x8_t, c: int8x8_t) -> int16x8_t { + let d: int8x8_t = vabd_s8(b, c); + let e: uint8x8_t = simd_cast(d); + simd_add(a, simd_cast(e)) +} + +/// Signed Absolute difference and Accumulate Long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vabal.s16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sabal))] +pub unsafe fn vabal_s16(a: int32x4_t, b: int16x4_t, c: int16x4_t) -> int32x4_t { + let d: int16x4_t = vabd_s16(b, c); + let e: uint16x4_t = simd_cast(d); + simd_add(a, simd_cast(e)) +} + +/// Signed Absolute difference and Accumulate Long +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vabal.s32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sabal))] +pub unsafe fn vabal_s32(a: int64x2_t, b: int32x2_t, c: int32x2_t) -> int64x2_t { + let d: int32x2_t = vabd_s32(b, c); + let e: uint32x2_t = simd_cast(d); + simd_add(a, simd_cast(e)) +} + +/// Singned saturating Absolute value +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vqabs.s8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqabs))] +pub unsafe fn vqabs_s8(a: int8x8_t) -> int8x8_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqabs.v8i8")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqabs.v8i8")] + fn vqabs_s8_(a: int8x8_t) -> int8x8_t; + } +vqabs_s8_(a) +} + +/// Singned saturating Absolute value +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vqabs.s8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqabs))] +pub unsafe fn vqabsq_s8(a: int8x16_t) -> int8x16_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqabs.v16i8")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqabs.v16i8")] + fn vqabsq_s8_(a: int8x16_t) -> int8x16_t; + } +vqabsq_s8_(a) +} + +/// Singned saturating Absolute value +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vqabs.s16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqabs))] +pub unsafe fn vqabs_s16(a: int16x4_t) -> int16x4_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqabs.v4i16")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqabs.v4i16")] + fn vqabs_s16_(a: int16x4_t) -> int16x4_t; + } +vqabs_s16_(a) +} + +/// Singned saturating Absolute value +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vqabs.s16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqabs))] +pub unsafe fn vqabsq_s16(a: int16x8_t) -> int16x8_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqabs.v8i16")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqabs.v8i16")] + fn vqabsq_s16_(a: int16x8_t) -> int16x8_t; + } +vqabsq_s16_(a) +} + +/// Singned saturating Absolute value +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vqabs.s32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqabs))] +pub unsafe fn vqabs_s32(a: int32x2_t) -> int32x2_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqabs.v2i32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqabs.v2i32")] + fn vqabs_s32_(a: int32x2_t) -> int32x2_t; + } +vqabs_s32_(a) +} + +/// Singned saturating Absolute value +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vqabs.s32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sqabs))] +pub unsafe fn vqabsq_s32(a: int32x4_t) -> int32x4_t { + #[allow(improper_ctypes)] + extern "C" { + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqabs.v4i32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sqabs.v4i32")] + fn vqabsq_s32_(a: int32x4_t) -> int32x4_t; + } +vqabsq_s32_(a) +} + +#[cfg(test)] +#[allow(overflowing_literals)] +mod test { + use super::*; + use crate::core_arch::simd::*; + use std::mem::transmute; + use stdarch_test::simd_test; + + #[simd_test(enable = "neon")] + unsafe fn test_vand_s8() { + let a: i8x8 = i8x8::new(0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07); + let b: i8x8 = i8x8::new(0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F); + let e: i8x8 = i8x8::new(0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07); + let r: i8x8 = transmute(vand_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + + let a: i8x8 = i8x8::new(0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07); + let b: i8x8 = i8x8::new(0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00); + let e: i8x8 = i8x8::new(0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00); + let r: i8x8 = transmute(vand_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vandq_s8() { + let a: i8x16 = i8x16::new(0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x00); + let b: i8x16 = i8x16::new(0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F); + let e: i8x16 = i8x16::new(0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x00); + let r: i8x16 = transmute(vandq_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + + let a: i8x16 = i8x16::new(0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x00); + let b: i8x16 = i8x16::new(0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00); + let e: i8x16 = i8x16::new(0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00); + let r: i8x16 = transmute(vandq_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vand_s16() { + let a: i16x4 = i16x4::new(0x00, 0x01, 0x02, 0x03); + let b: i16x4 = i16x4::new(0x0F, 0x0F, 0x0F, 0x0F); + let e: i16x4 = i16x4::new(0x00, 0x01, 0x02, 0x03); + let r: i16x4 = transmute(vand_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + + let a: i16x4 = i16x4::new(0x00, 0x01, 0x02, 0x03); + let b: i16x4 = i16x4::new(0x00, 0x00, 0x00, 0x00); + let e: i16x4 = i16x4::new(0x00, 0x00, 0x00, 0x00); + let r: i16x4 = transmute(vand_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vandq_s16() { + let a: i16x8 = i16x8::new(0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07); + let b: i16x8 = i16x8::new(0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F); + let e: i16x8 = i16x8::new(0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07); + let r: i16x8 = transmute(vandq_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + + let a: i16x8 = i16x8::new(0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07); + let b: i16x8 = i16x8::new(0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00); + let e: i16x8 = i16x8::new(0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00); + let r: i16x8 = transmute(vandq_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vand_s32() { + let a: i32x2 = i32x2::new(0x00, 0x01); + let b: i32x2 = i32x2::new(0x0F, 0x0F); + let e: i32x2 = i32x2::new(0x00, 0x01); + let r: i32x2 = transmute(vand_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + + let a: i32x2 = i32x2::new(0x00, 0x01); + let b: i32x2 = i32x2::new(0x00, 0x00); + let e: i32x2 = i32x2::new(0x00, 0x00); + let r: i32x2 = transmute(vand_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vandq_s32() { + let a: i32x4 = i32x4::new(0x00, 0x01, 0x02, 0x03); + let b: i32x4 = i32x4::new(0x0F, 0x0F, 0x0F, 0x0F); + let e: i32x4 = i32x4::new(0x00, 0x01, 0x02, 0x03); + let r: i32x4 = transmute(vandq_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + + let a: i32x4 = i32x4::new(0x00, 0x01, 0x02, 0x03); + let b: i32x4 = i32x4::new(0x00, 0x00, 0x00, 0x00); + let e: i32x4 = i32x4::new(0x00, 0x00, 0x00, 0x00); + let r: i32x4 = transmute(vandq_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vand_u8() { + let a: u8x8 = u8x8::new(0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07); + let b: u8x8 = u8x8::new(0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F); + let e: u8x8 = u8x8::new(0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07); + let r: u8x8 = transmute(vand_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + + let a: u8x8 = u8x8::new(0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07); + let b: u8x8 = u8x8::new(0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00); + let e: u8x8 = u8x8::new(0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00); + let r: u8x8 = transmute(vand_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vandq_u8() { + let a: u8x16 = u8x16::new(0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x00); + let b: u8x16 = u8x16::new(0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F); + let e: u8x16 = u8x16::new(0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x00); + let r: u8x16 = transmute(vandq_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + + let a: u8x16 = u8x16::new(0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x00); + let b: u8x16 = u8x16::new(0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00); + let e: u8x16 = u8x16::new(0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00); + let r: u8x16 = transmute(vandq_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vand_u16() { + let a: u16x4 = u16x4::new(0x00, 0x01, 0x02, 0x03); + let b: u16x4 = u16x4::new(0x0F, 0x0F, 0x0F, 0x0F); + let e: u16x4 = u16x4::new(0x00, 0x01, 0x02, 0x03); + let r: u16x4 = transmute(vand_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + + let a: u16x4 = u16x4::new(0x00, 0x01, 0x02, 0x03); + let b: u16x4 = u16x4::new(0x00, 0x00, 0x00, 0x00); + let e: u16x4 = u16x4::new(0x00, 0x00, 0x00, 0x00); + let r: u16x4 = transmute(vand_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vandq_u16() { + let a: u16x8 = u16x8::new(0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07); + let b: u16x8 = u16x8::new(0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F, 0x0F); + let e: u16x8 = u16x8::new(0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07); + let r: u16x8 = transmute(vandq_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + + let a: u16x8 = u16x8::new(0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07); + let b: u16x8 = u16x8::new(0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00); + let e: u16x8 = u16x8::new(0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00); + let r: u16x8 = transmute(vandq_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vand_u32() { + let a: u32x2 = u32x2::new(0x00, 0x01); + let b: u32x2 = u32x2::new(0x0F, 0x0F); + let e: u32x2 = u32x2::new(0x00, 0x01); + let r: u32x2 = transmute(vand_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + + let a: u32x2 = u32x2::new(0x00, 0x01); + let b: u32x2 = u32x2::new(0x00, 0x00); + let e: u32x2 = u32x2::new(0x00, 0x00); + let r: u32x2 = transmute(vand_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vandq_u32() { + let a: u32x4 = u32x4::new(0x00, 0x01, 0x02, 0x03); + let b: u32x4 = u32x4::new(0x0F, 0x0F, 0x0F, 0x0F); + let e: u32x4 = u32x4::new(0x00, 0x01, 0x02, 0x03); + let r: u32x4 = transmute(vandq_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + + let a: u32x4 = u32x4::new(0x00, 0x01, 0x02, 0x03); + let b: u32x4 = u32x4::new(0x00, 0x00, 0x00, 0x00); + let e: u32x4 = u32x4::new(0x00, 0x00, 0x00, 0x00); + let r: u32x4 = transmute(vandq_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vand_s64() { + let a: i64x1 = i64x1::new(0x00); + let b: i64x1 = i64x1::new(0x0F); + let e: i64x1 = i64x1::new(0x00); + let r: i64x1 = transmute(vand_s64(transmute(a), transmute(b))); + assert_eq!(r, e); + + let a: i64x1 = i64x1::new(0x00); + let b: i64x1 = i64x1::new(0x00); + let e: i64x1 = i64x1::new(0x00); + let r: i64x1 = transmute(vand_s64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vandq_s64() { + let a: i64x2 = i64x2::new(0x00, 0x01); + let b: i64x2 = i64x2::new(0x0F, 0x0F); + let e: i64x2 = i64x2::new(0x00, 0x01); + let r: i64x2 = transmute(vandq_s64(transmute(a), transmute(b))); + assert_eq!(r, e); + + let a: i64x2 = i64x2::new(0x00, 0x01); + let b: i64x2 = i64x2::new(0x00, 0x00); + let e: i64x2 = i64x2::new(0x00, 0x00); + let r: i64x2 = transmute(vandq_s64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vand_u64() { + let a: u64x1 = u64x1::new(0x00); + let b: u64x1 = u64x1::new(0x0F); + let e: u64x1 = u64x1::new(0x00); + let r: u64x1 = transmute(vand_u64(transmute(a), transmute(b))); + assert_eq!(r, e); + + let a: u64x1 = u64x1::new(0x00); + let b: u64x1 = u64x1::new(0x00); + let e: u64x1 = u64x1::new(0x00); + let r: u64x1 = transmute(vand_u64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vandq_u64() { + let a: u64x2 = u64x2::new(0x00, 0x01); + let b: u64x2 = u64x2::new(0x0F, 0x0F); + let e: u64x2 = u64x2::new(0x00, 0x01); + let r: u64x2 = transmute(vandq_u64(transmute(a), transmute(b))); + assert_eq!(r, e); + + let a: u64x2 = u64x2::new(0x00, 0x01); + let b: u64x2 = u64x2::new(0x00, 0x00); + let e: u64x2 = u64x2::new(0x00, 0x00); + let r: u64x2 = transmute(vandq_u64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vorr_s8() { + let a: i8x8 = i8x8::new(0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07); + let b: i8x8 = i8x8::new(0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00); + let e: i8x8 = i8x8::new(0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07); + let r: i8x8 = transmute(vorr_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vorrq_s8() { + let a: i8x16 = i8x16::new(0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F); + let b: i8x16 = i8x16::new(0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00); + let e: i8x16 = i8x16::new(0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F); + let r: i8x16 = transmute(vorrq_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vorr_s16() { + let a: i16x4 = i16x4::new(0x00, 0x01, 0x02, 0x03); + let b: i16x4 = i16x4::new(0x00, 0x00, 0x00, 0x00); + let e: i16x4 = i16x4::new(0x00, 0x01, 0x02, 0x03); + let r: i16x4 = transmute(vorr_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vorrq_s16() { + let a: i16x8 = i16x8::new(0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07); + let b: i16x8 = i16x8::new(0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00); + let e: i16x8 = i16x8::new(0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07); + let r: i16x8 = transmute(vorrq_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vorr_s32() { + let a: i32x2 = i32x2::new(0x00, 0x01); + let b: i32x2 = i32x2::new(0x00, 0x00); + let e: i32x2 = i32x2::new(0x00, 0x01); + let r: i32x2 = transmute(vorr_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vorrq_s32() { + let a: i32x4 = i32x4::new(0x00, 0x01, 0x02, 0x03); + let b: i32x4 = i32x4::new(0x00, 0x00, 0x00, 0x00); + let e: i32x4 = i32x4::new(0x00, 0x01, 0x02, 0x03); + let r: i32x4 = transmute(vorrq_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vorr_u8() { + let a: u8x8 = u8x8::new(0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07); + let b: u8x8 = u8x8::new(0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00); + let e: u8x8 = u8x8::new(0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07); + let r: u8x8 = transmute(vorr_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vorrq_u8() { + let a: u8x16 = u8x16::new(0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F); + let b: u8x16 = u8x16::new(0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00); + let e: u8x16 = u8x16::new(0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F); + let r: u8x16 = transmute(vorrq_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vorr_u16() { + let a: u16x4 = u16x4::new(0x00, 0x01, 0x02, 0x03); + let b: u16x4 = u16x4::new(0x00, 0x00, 0x00, 0x00); + let e: u16x4 = u16x4::new(0x00, 0x01, 0x02, 0x03); + let r: u16x4 = transmute(vorr_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vorrq_u16() { + let a: u16x8 = u16x8::new(0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07); + let b: u16x8 = u16x8::new(0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00); + let e: u16x8 = u16x8::new(0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07); + let r: u16x8 = transmute(vorrq_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vorr_u32() { + let a: u32x2 = u32x2::new(0x00, 0x01); + let b: u32x2 = u32x2::new(0x00, 0x00); + let e: u32x2 = u32x2::new(0x00, 0x01); + let r: u32x2 = transmute(vorr_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vorrq_u32() { + let a: u32x4 = u32x4::new(0x00, 0x01, 0x02, 0x03); + let b: u32x4 = u32x4::new(0x00, 0x00, 0x00, 0x00); + let e: u32x4 = u32x4::new(0x00, 0x01, 0x02, 0x03); + let r: u32x4 = transmute(vorrq_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vorr_s64() { + let a: i64x1 = i64x1::new(0x00); + let b: i64x1 = i64x1::new(0x00); + let e: i64x1 = i64x1::new(0x00); + let r: i64x1 = transmute(vorr_s64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vorrq_s64() { + let a: i64x2 = i64x2::new(0x00, 0x01); + let b: i64x2 = i64x2::new(0x00, 0x00); + let e: i64x2 = i64x2::new(0x00, 0x01); + let r: i64x2 = transmute(vorrq_s64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vorr_u64() { + let a: u64x1 = u64x1::new(0x00); + let b: u64x1 = u64x1::new(0x00); + let e: u64x1 = u64x1::new(0x00); + let r: u64x1 = transmute(vorr_u64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vorrq_u64() { + let a: u64x2 = u64x2::new(0x00, 0x01); + let b: u64x2 = u64x2::new(0x00, 0x00); + let e: u64x2 = u64x2::new(0x00, 0x01); + let r: u64x2 = transmute(vorrq_u64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_veor_s8() { + let a: i8x8 = i8x8::new(0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07); + let b: i8x8 = i8x8::new(0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00); + let e: i8x8 = i8x8::new(0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07); + let r: i8x8 = transmute(veor_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_veorq_s8() { + let a: i8x16 = i8x16::new(0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F); + let b: i8x16 = i8x16::new(0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00); + let e: i8x16 = i8x16::new(0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F); + let r: i8x16 = transmute(veorq_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_veor_s16() { + let a: i16x4 = i16x4::new(0x00, 0x01, 0x02, 0x03); + let b: i16x4 = i16x4::new(0x00, 0x00, 0x00, 0x00); + let e: i16x4 = i16x4::new(0x00, 0x01, 0x02, 0x03); + let r: i16x4 = transmute(veor_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_veorq_s16() { + let a: i16x8 = i16x8::new(0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07); + let b: i16x8 = i16x8::new(0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00); + let e: i16x8 = i16x8::new(0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07); + let r: i16x8 = transmute(veorq_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_veor_s32() { + let a: i32x2 = i32x2::new(0x00, 0x01); + let b: i32x2 = i32x2::new(0x00, 0x00); + let e: i32x2 = i32x2::new(0x00, 0x01); + let r: i32x2 = transmute(veor_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_veorq_s32() { + let a: i32x4 = i32x4::new(0x00, 0x01, 0x02, 0x03); + let b: i32x4 = i32x4::new(0x00, 0x00, 0x00, 0x00); + let e: i32x4 = i32x4::new(0x00, 0x01, 0x02, 0x03); + let r: i32x4 = transmute(veorq_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_veor_u8() { + let a: u8x8 = u8x8::new(0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07); + let b: u8x8 = u8x8::new(0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00); + let e: u8x8 = u8x8::new(0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07); + let r: u8x8 = transmute(veor_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_veorq_u8() { + let a: u8x16 = u8x16::new(0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F); + let b: u8x16 = u8x16::new(0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00); + let e: u8x16 = u8x16::new(0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F); + let r: u8x16 = transmute(veorq_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_veor_u16() { + let a: u16x4 = u16x4::new(0x00, 0x01, 0x02, 0x03); + let b: u16x4 = u16x4::new(0x00, 0x00, 0x00, 0x00); + let e: u16x4 = u16x4::new(0x00, 0x01, 0x02, 0x03); + let r: u16x4 = transmute(veor_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_veorq_u16() { + let a: u16x8 = u16x8::new(0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07); + let b: u16x8 = u16x8::new(0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00); + let e: u16x8 = u16x8::new(0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07); + let r: u16x8 = transmute(veorq_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_veor_u32() { + let a: u32x2 = u32x2::new(0x00, 0x01); + let b: u32x2 = u32x2::new(0x00, 0x00); + let e: u32x2 = u32x2::new(0x00, 0x01); + let r: u32x2 = transmute(veor_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_veorq_u32() { + let a: u32x4 = u32x4::new(0x00, 0x01, 0x02, 0x03); + let b: u32x4 = u32x4::new(0x00, 0x00, 0x00, 0x00); + let e: u32x4 = u32x4::new(0x00, 0x01, 0x02, 0x03); + let r: u32x4 = transmute(veorq_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_veor_s64() { + let a: i64x1 = i64x1::new(0x00); + let b: i64x1 = i64x1::new(0x00); + let e: i64x1 = i64x1::new(0x00); + let r: i64x1 = transmute(veor_s64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_veorq_s64() { + let a: i64x2 = i64x2::new(0x00, 0x01); + let b: i64x2 = i64x2::new(0x00, 0x00); + let e: i64x2 = i64x2::new(0x00, 0x01); + let r: i64x2 = transmute(veorq_s64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_veor_u64() { + let a: u64x1 = u64x1::new(0x00); + let b: u64x1 = u64x1::new(0x00); + let e: u64x1 = u64x1::new(0x00); + let r: u64x1 = transmute(veor_u64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_veorq_u64() { + let a: u64x2 = u64x2::new(0x00, 0x01); + let b: u64x2 = u64x2::new(0x00, 0x00); + let e: u64x2 = u64x2::new(0x00, 0x01); + let r: u64x2 = transmute(veorq_u64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vabd_s8() { + let a: i8x8 = i8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let b: i8x8 = i8x8::new(16, 15, 14, 13, 12, 11, 10, 9); + let e: i8x8 = i8x8::new(15, 13, 11, 9, 7, 5, 3, 1); + let r: i8x8 = transmute(vabd_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vabdq_s8() { + let a: i8x16 = i8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let b: i8x16 = i8x16::new(16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1); + let e: i8x16 = i8x16::new(15, 13, 11, 9, 7, 5, 3, 1, 1, 3, 5, 7, 9, 11, 13, 15); + let r: i8x16 = transmute(vabdq_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vabd_s16() { + let a: i16x4 = i16x4::new(1, 2, 3, 4); + let b: i16x4 = i16x4::new(16, 15, 14, 13); + let e: i16x4 = i16x4::new(15, 13, 11, 9); + let r: i16x4 = transmute(vabd_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vabdq_s16() { + let a: i16x8 = i16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let b: i16x8 = i16x8::new(16, 15, 14, 13, 12, 11, 10, 9); + let e: i16x8 = i16x8::new(15, 13, 11, 9, 7, 5, 3, 1); + let r: i16x8 = transmute(vabdq_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vabd_s32() { + let a: i32x2 = i32x2::new(1, 2); + let b: i32x2 = i32x2::new(16, 15); + let e: i32x2 = i32x2::new(15, 13); + let r: i32x2 = transmute(vabd_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vabdq_s32() { + let a: i32x4 = i32x4::new(1, 2, 3, 4); + let b: i32x4 = i32x4::new(16, 15, 14, 13); + let e: i32x4 = i32x4::new(15, 13, 11, 9); + let r: i32x4 = transmute(vabdq_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vabd_u8() { + let a: u8x8 = u8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let b: u8x8 = u8x8::new(16, 15, 14, 13, 12, 11, 10, 9); + let e: u8x8 = u8x8::new(15, 13, 11, 9, 7, 5, 3, 1); + let r: u8x8 = transmute(vabd_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vabdq_u8() { + let a: u8x16 = u8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let b: u8x16 = u8x16::new(16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1); + let e: u8x16 = u8x16::new(15, 13, 11, 9, 7, 5, 3, 1, 1, 3, 5, 7, 9, 11, 13, 15); + let r: u8x16 = transmute(vabdq_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vabd_u16() { + let a: u16x4 = u16x4::new(1, 2, 3, 4); + let b: u16x4 = u16x4::new(16, 15, 14, 13); + let e: u16x4 = u16x4::new(15, 13, 11, 9); + let r: u16x4 = transmute(vabd_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vabdq_u16() { + let a: u16x8 = u16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let b: u16x8 = u16x8::new(16, 15, 14, 13, 12, 11, 10, 9); + let e: u16x8 = u16x8::new(15, 13, 11, 9, 7, 5, 3, 1); + let r: u16x8 = transmute(vabdq_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vabd_u32() { + let a: u32x2 = u32x2::new(1, 2); + let b: u32x2 = u32x2::new(16, 15); + let e: u32x2 = u32x2::new(15, 13); + let r: u32x2 = transmute(vabd_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vabdq_u32() { + let a: u32x4 = u32x4::new(1, 2, 3, 4); + let b: u32x4 = u32x4::new(16, 15, 14, 13); + let e: u32x4 = u32x4::new(15, 13, 11, 9); + let r: u32x4 = transmute(vabdq_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vabd_f32() { + let a: f32x2 = f32x2::new(1.0, 2.0); + let b: f32x2 = f32x2::new(9.0, 3.0); + let e: f32x2 = f32x2::new(8.0, 1.0); + let r: f32x2 = transmute(vabd_f32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vabdq_f32() { + let a: f32x4 = f32x4::new(1.0, 2.0, 5.0, -4.0); + let b: f32x4 = f32x4::new(9.0, 3.0, 2.0, 8.0); + let e: f32x4 = f32x4::new(8.0, 1.0, 3.0, 12.0); + let r: f32x4 = transmute(vabdq_f32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vabdl_u8() { + let a: u8x8 = u8x8::new(1, 2, 3, 4, 4, 3, 2, 1); + let b: u8x8 = u8x8::new(10, 10, 10, 10, 10, 10, 10, 10); + let e: u16x8 = u16x8::new(9, 8, 7, 6, 6, 7, 8, 9); + let r: u16x8 = transmute(vabdl_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vabdl_u16() { + let a: u16x4 = u16x4::new(1, 2, 3, 4); + let b: u16x4 = u16x4::new(10, 10, 10, 10); + let e: u32x4 = u32x4::new(9, 8, 7, 6); + let r: u32x4 = transmute(vabdl_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vabdl_u32() { + let a: u32x2 = u32x2::new(1, 2); + let b: u32x2 = u32x2::new(10, 10); + let e: u64x2 = u64x2::new(9, 8); + let r: u64x2 = transmute(vabdl_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vabdl_s8() { + let a: i8x8 = i8x8::new(1, 2, 3, 4, 4, 3, 2, 1); + let b: i8x8 = i8x8::new(10, 10, 10, 10, 10, 10, 10, 10); + let e: i16x8 = i16x8::new(9, 8, 7, 6, 6, 7, 8, 9); + let r: i16x8 = transmute(vabdl_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vabdl_s16() { + let a: i16x4 = i16x4::new(1, 2, 11, 12); + let b: i16x4 = i16x4::new(10, 10, 10, 10); + let e: i32x4 = i32x4::new(9, 8, 1, 2); + let r: i32x4 = transmute(vabdl_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vabdl_s32() { + let a: i32x2 = i32x2::new(1, 11); + let b: i32x2 = i32x2::new(10, 10); + let e: i64x2 = i64x2::new(9, 1); + let r: i64x2 = transmute(vabdl_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vceq_u8() { + let a: u8x8 = u8x8::new(0, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07); + let b: u8x8 = u8x8::new(0, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07); + let e: u8x8 = u8x8::new(0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF); + let r: u8x8 = transmute(vceq_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + + let a: u8x8 = u8x8::new(0, 0, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07); + let b: u8x8 = u8x8::new(0, 0xFF, 0x02, 0x04, 0x04, 0x00, 0x06, 0x08); + let e: u8x8 = u8x8::new(0xFF, 0, 0xFF, 0, 0xFF, 0, 0xFF, 0); + let r: u8x8 = transmute(vceq_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vceqq_u8() { + let a: u8x16 = u8x16::new(0, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0xFF); + let b: u8x16 = u8x16::new(0, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0xFF); + let e: u8x16 = u8x16::new(0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF); + let r: u8x16 = transmute(vceqq_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + + let a: u8x16 = u8x16::new(0, 0, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0xCC, 0x0D, 0xEE, 0xFF); + let b: u8x16 = u8x16::new(0, 0xFF, 0x02, 0x04, 0x04, 0x00, 0x06, 0x08, 0x08, 0x00, 0x0A, 0x0A, 0xCC, 0xD0, 0xEE, 0); + let e: u8x16 = u8x16::new(0xFF, 0, 0xFF, 0, 0xFF, 0, 0xFF, 0, 0xFF, 0, 0xFF, 0, 0xFF, 0, 0xFF, 0); + let r: u8x16 = transmute(vceqq_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vceq_u16() { + let a: u16x4 = u16x4::new(0, 0x01, 0x02, 0x03); + let b: u16x4 = u16x4::new(0, 0x01, 0x02, 0x03); + let e: u16x4 = u16x4::new(0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF); + let r: u16x4 = transmute(vceq_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + + let a: u16x4 = u16x4::new(0, 0, 0x02, 0x03); + let b: u16x4 = u16x4::new(0, 0xFF_FF, 0x02, 0x04); + let e: u16x4 = u16x4::new(0xFF_FF, 0, 0xFF_FF, 0); + let r: u16x4 = transmute(vceq_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vceqq_u16() { + let a: u16x8 = u16x8::new(0, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07); + let b: u16x8 = u16x8::new(0, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07); + let e: u16x8 = u16x8::new(0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF); + let r: u16x8 = transmute(vceqq_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + + let a: u16x8 = u16x8::new(0, 0, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07); + let b: u16x8 = u16x8::new(0, 0xFF_FF, 0x02, 0x04, 0x04, 0x00, 0x06, 0x08); + let e: u16x8 = u16x8::new(0xFF_FF, 0, 0xFF_FF, 0, 0xFF_FF, 0, 0xFF_FF, 0); + let r: u16x8 = transmute(vceqq_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vceq_u32() { + let a: u32x2 = u32x2::new(0, 0x01); + let b: u32x2 = u32x2::new(0, 0x01); + let e: u32x2 = u32x2::new(0xFF_FF_FF_FF, 0xFF_FF_FF_FF); + let r: u32x2 = transmute(vceq_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + + let a: u32x2 = u32x2::new(0, 0); + let b: u32x2 = u32x2::new(0, 0xFF_FF_FF_FF); + let e: u32x2 = u32x2::new(0xFF_FF_FF_FF, 0); + let r: u32x2 = transmute(vceq_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vceqq_u32() { + let a: u32x4 = u32x4::new(0, 0x01, 0x02, 0x03); + let b: u32x4 = u32x4::new(0, 0x01, 0x02, 0x03); + let e: u32x4 = u32x4::new(0xFF_FF_FF_FF, 0xFF_FF_FF_FF, 0xFF_FF_FF_FF, 0xFF_FF_FF_FF); + let r: u32x4 = transmute(vceqq_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + + let a: u32x4 = u32x4::new(0, 0, 0x02, 0x03); + let b: u32x4 = u32x4::new(0, 0xFF_FF_FF_FF, 0x02, 0x04); + let e: u32x4 = u32x4::new(0xFF_FF_FF_FF, 0, 0xFF_FF_FF_FF, 0); + let r: u32x4 = transmute(vceqq_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vceq_s8() { + let a: i8x8 = i8x8::new(-128, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07); + let b: i8x8 = i8x8::new(-128, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07); + let e: u8x8 = u8x8::new(0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF); + let r: u8x8 = transmute(vceq_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + + let a: i8x8 = i8x8::new(-128, -128, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07); + let b: i8x8 = i8x8::new(-128, 0x7F, 0x02, 0x04, 0x04, 0x00, 0x06, 0x08); + let e: u8x8 = u8x8::new(0xFF, 0, 0xFF, 0, 0xFF, 0, 0xFF, 0); + let r: u8x8 = transmute(vceq_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vceqq_s8() { + let a: i8x16 = i8x16::new(-128, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x7F); + let b: i8x16 = i8x16::new(-128, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x7F); + let e: u8x16 = u8x16::new(0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF); + let r: u8x16 = transmute(vceqq_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + + let a: i8x16 = i8x16::new(-128, -128, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0xCC, 0x0D, 0xEE, 0x7F); + let b: i8x16 = i8x16::new(-128, 0x7F, 0x02, 0x04, 0x04, 0x00, 0x06, 0x08, 0x08, 0x00, 0x0A, 0x0A, 0xCC, 0xD0, 0xEE, -128); + let e: u8x16 = u8x16::new(0xFF, 0, 0xFF, 0, 0xFF, 0, 0xFF, 0, 0xFF, 0, 0xFF, 0, 0xFF, 0, 0xFF, 0); + let r: u8x16 = transmute(vceqq_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vceq_s16() { + let a: i16x4 = i16x4::new(-32768, 0x01, 0x02, 0x03); + let b: i16x4 = i16x4::new(-32768, 0x01, 0x02, 0x03); + let e: u16x4 = u16x4::new(0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF); + let r: u16x4 = transmute(vceq_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + + let a: i16x4 = i16x4::new(-32768, -32768, 0x02, 0x03); + let b: i16x4 = i16x4::new(-32768, 0x7F_FF, 0x02, 0x04); + let e: u16x4 = u16x4::new(0xFF_FF, 0, 0xFF_FF, 0); + let r: u16x4 = transmute(vceq_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vceqq_s16() { + let a: i16x8 = i16x8::new(-32768, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07); + let b: i16x8 = i16x8::new(-32768, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07); + let e: u16x8 = u16x8::new(0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF); + let r: u16x8 = transmute(vceqq_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + + let a: i16x8 = i16x8::new(-32768, -32768, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07); + let b: i16x8 = i16x8::new(-32768, 0x7F_FF, 0x02, 0x04, 0x04, 0x00, 0x06, 0x08); + let e: u16x8 = u16x8::new(0xFF_FF, 0, 0xFF_FF, 0, 0xFF_FF, 0, 0xFF_FF, 0); + let r: u16x8 = transmute(vceqq_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vceq_s32() { + let a: i32x2 = i32x2::new(-2147483648, 0x01); + let b: i32x2 = i32x2::new(-2147483648, 0x01); + let e: u32x2 = u32x2::new(0xFF_FF_FF_FF, 0xFF_FF_FF_FF); + let r: u32x2 = transmute(vceq_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + + let a: i32x2 = i32x2::new(-2147483648, -2147483648); + let b: i32x2 = i32x2::new(-2147483648, 0x7F_FF_FF_FF); + let e: u32x2 = u32x2::new(0xFF_FF_FF_FF, 0); + let r: u32x2 = transmute(vceq_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vceqq_s32() { + let a: i32x4 = i32x4::new(-2147483648, 0x01, 0x02, 0x03); + let b: i32x4 = i32x4::new(-2147483648, 0x01, 0x02, 0x03); + let e: u32x4 = u32x4::new(0xFF_FF_FF_FF, 0xFF_FF_FF_FF, 0xFF_FF_FF_FF, 0xFF_FF_FF_FF); + let r: u32x4 = transmute(vceqq_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + + let a: i32x4 = i32x4::new(-2147483648, -2147483648, 0x02, 0x03); + let b: i32x4 = i32x4::new(-2147483648, 0x7F_FF_FF_FF, 0x02, 0x04); + let e: u32x4 = u32x4::new(0xFF_FF_FF_FF, 0, 0xFF_FF_FF_FF, 0); + let r: u32x4 = transmute(vceqq_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vceq_p8() { + let a: i8x8 = i8x8::new(-128, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07); + let b: i8x8 = i8x8::new(-128, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07); + let e: u8x8 = u8x8::new(0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF); + let r: u8x8 = transmute(vceq_p8(transmute(a), transmute(b))); + assert_eq!(r, e); + + let a: i8x8 = i8x8::new(-128, -128, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07); + let b: i8x8 = i8x8::new(-128, 0x7F, 0x02, 0x04, 0x04, 0x00, 0x06, 0x08); + let e: u8x8 = u8x8::new(0xFF, 0, 0xFF, 0, 0xFF, 0, 0xFF, 0); + let r: u8x8 = transmute(vceq_p8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vceqq_p8() { + let a: i8x16 = i8x16::new(-128, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x7F); + let b: i8x16 = i8x16::new(-128, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x7F); + let e: u8x16 = u8x16::new(0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF); + let r: u8x16 = transmute(vceqq_p8(transmute(a), transmute(b))); + assert_eq!(r, e); + + let a: i8x16 = i8x16::new(-128, -128, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0xCC, 0x0D, 0xEE, 0x7F); + let b: i8x16 = i8x16::new(-128, 0x7F, 0x02, 0x04, 0x04, 0x00, 0x06, 0x08, 0x08, 0x00, 0x0A, 0x0A, 0xCC, 0xD0, 0xEE, -128); + let e: u8x16 = u8x16::new(0xFF, 0, 0xFF, 0, 0xFF, 0, 0xFF, 0, 0xFF, 0, 0xFF, 0, 0xFF, 0, 0xFF, 0); + let r: u8x16 = transmute(vceqq_p8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vceq_f32() { + let a: f32x2 = f32x2::new(1.2, 3.4); + let b: f32x2 = f32x2::new(1.2, 3.4); + let e: u32x2 = u32x2::new(0xFF_FF_FF_FF, 0xFF_FF_FF_FF); + let r: u32x2 = transmute(vceq_f32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vceqq_f32() { + let a: f32x4 = f32x4::new(1.2, 3.4, 5.6, 7.8); + let b: f32x4 = f32x4::new(1.2, 3.4, 5.6, 7.8); + let e: u32x4 = u32x4::new(0xFF_FF_FF_FF, 0xFF_FF_FF_FF, 0xFF_FF_FF_FF, 0xFF_FF_FF_FF); + let r: u32x4 = transmute(vceqq_f32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vtst_s8() { + let a: i8x8 = i8x8::new(-128, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06); + let b: i8x8 = i8x8::new(-128, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06); + let e: u8x8 = u8x8::new(0xFF, 0, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF); + let r: u8x8 = transmute(vtst_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vtstq_s8() { + let a: i8x16 = i8x16::new(-128, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x7F); + let b: i8x16 = i8x16::new(-128, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x7F); + let e: u8x16 = u8x16::new(0xFF, 0, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF); + let r: u8x16 = transmute(vtstq_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vtst_s16() { + let a: i16x4 = i16x4::new(-32768, 0x00, 0x01, 0x02); + let b: i16x4 = i16x4::new(-32768, 0x00, 0x01, 0x02); + let e: u16x4 = u16x4::new(0xFF_FF, 0, 0xFF_FF, 0xFF_FF); + let r: u16x4 = transmute(vtst_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vtstq_s16() { + let a: i16x8 = i16x8::new(-32768, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06); + let b: i16x8 = i16x8::new(-32768, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06); + let e: u16x8 = u16x8::new(0xFF_FF, 0, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF); + let r: u16x8 = transmute(vtstq_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vtst_s32() { + let a: i32x2 = i32x2::new(-2147483648, 0x00); + let b: i32x2 = i32x2::new(-2147483648, 0x00); + let e: u32x2 = u32x2::new(0xFF_FF_FF_FF, 0); + let r: u32x2 = transmute(vtst_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vtstq_s32() { + let a: i32x4 = i32x4::new(-2147483648, 0x00, 0x01, 0x02); + let b: i32x4 = i32x4::new(-2147483648, 0x00, 0x01, 0x02); + let e: u32x4 = u32x4::new(0xFF_FF_FF_FF, 0, 0xFF_FF_FF_FF, 0xFF_FF_FF_FF); + let r: u32x4 = transmute(vtstq_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vtst_p8() { + let a: i8x8 = i8x8::new(-128, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06); + let b: i8x8 = i8x8::new(-128, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06); + let e: u8x8 = u8x8::new(0xFF, 0, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF); + let r: u8x8 = transmute(vtst_p8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vtstq_p8() { + let a: i8x16 = i8x16::new(-128, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x7F); + let b: i8x16 = i8x16::new(-128, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x7F); + let e: u8x16 = u8x16::new(0xFF, 0, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF); + let r: u8x16 = transmute(vtstq_p8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vtst_u8() { + let a: u8x8 = u8x8::new(0, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06); + let b: u8x8 = u8x8::new(0, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06); + let e: u8x8 = u8x8::new(0, 0, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF); + let r: u8x8 = transmute(vtst_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vtstq_u8() { + let a: u8x16 = u8x16::new(0, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0xFF); + let b: u8x16 = u8x16::new(0, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0xFF); + let e: u8x16 = u8x16::new(0, 0, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF); + let r: u8x16 = transmute(vtstq_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vtst_u16() { + let a: u16x4 = u16x4::new(0, 0x00, 0x01, 0x02); + let b: u16x4 = u16x4::new(0, 0x00, 0x01, 0x02); + let e: u16x4 = u16x4::new(0, 0, 0xFF_FF, 0xFF_FF); + let r: u16x4 = transmute(vtst_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vtstq_u16() { + let a: u16x8 = u16x8::new(0, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06); + let b: u16x8 = u16x8::new(0, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06); + let e: u16x8 = u16x8::new(0, 0, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF); + let r: u16x8 = transmute(vtstq_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vtst_u32() { + let a: u32x2 = u32x2::new(0, 0x00); + let b: u32x2 = u32x2::new(0, 0x00); + let e: u32x2 = u32x2::new(0, 0); + let r: u32x2 = transmute(vtst_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vtstq_u32() { + let a: u32x4 = u32x4::new(0, 0x00, 0x01, 0x02); + let b: u32x4 = u32x4::new(0, 0x00, 0x01, 0x02); + let e: u32x4 = u32x4::new(0, 0, 0xFF_FF_FF_FF, 0xFF_FF_FF_FF); + let r: u32x4 = transmute(vtstq_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vabs_f32() { + let a: f32x2 = f32x2::new(-0.1, -2.2); + let e: f32x2 = f32x2::new(0.1, 2.2); + let r: f32x2 = transmute(vabs_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vabsq_f32() { + let a: f32x4 = f32x4::new(-0.1, -2.2, -3.3, -6.6); + let e: f32x4 = f32x4::new(0.1, 2.2, 3.3, 6.6); + let r: f32x4 = transmute(vabsq_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcgt_s8() { + let a: i8x8 = i8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let b: i8x8 = i8x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let e: u8x8 = u8x8::new(0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF); + let r: u8x8 = transmute(vcgt_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcgtq_s8() { + let a: i8x16 = i8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let b: i8x16 = i8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let e: u8x16 = u8x16::new(0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF); + let r: u8x16 = transmute(vcgtq_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcgt_s16() { + let a: i16x4 = i16x4::new(1, 2, 3, 4); + let b: i16x4 = i16x4::new(0, 1, 2, 3); + let e: u16x4 = u16x4::new(0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF); + let r: u16x4 = transmute(vcgt_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcgtq_s16() { + let a: i16x8 = i16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let b: i16x8 = i16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let e: u16x8 = u16x8::new(0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF); + let r: u16x8 = transmute(vcgtq_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcgt_s32() { + let a: i32x2 = i32x2::new(1, 2); + let b: i32x2 = i32x2::new(0, 1); + let e: u32x2 = u32x2::new(0xFF_FF_FF_FF, 0xFF_FF_FF_FF); + let r: u32x2 = transmute(vcgt_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcgtq_s32() { + let a: i32x4 = i32x4::new(1, 2, 3, 4); + let b: i32x4 = i32x4::new(0, 1, 2, 3); + let e: u32x4 = u32x4::new(0xFF_FF_FF_FF, 0xFF_FF_FF_FF, 0xFF_FF_FF_FF, 0xFF_FF_FF_FF); + let r: u32x4 = transmute(vcgtq_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcgt_u8() { + let a: u8x8 = u8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let b: u8x8 = u8x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let e: u8x8 = u8x8::new(0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF); + let r: u8x8 = transmute(vcgt_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcgtq_u8() { + let a: u8x16 = u8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let b: u8x16 = u8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let e: u8x16 = u8x16::new(0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF); + let r: u8x16 = transmute(vcgtq_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcgt_u16() { + let a: u16x4 = u16x4::new(1, 2, 3, 4); + let b: u16x4 = u16x4::new(0, 1, 2, 3); + let e: u16x4 = u16x4::new(0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF); + let r: u16x4 = transmute(vcgt_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcgtq_u16() { + let a: u16x8 = u16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let b: u16x8 = u16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let e: u16x8 = u16x8::new(0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF); + let r: u16x8 = transmute(vcgtq_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcgt_u32() { + let a: u32x2 = u32x2::new(1, 2); + let b: u32x2 = u32x2::new(0, 1); + let e: u32x2 = u32x2::new(0xFF_FF_FF_FF, 0xFF_FF_FF_FF); + let r: u32x2 = transmute(vcgt_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcgtq_u32() { + let a: u32x4 = u32x4::new(1, 2, 3, 4); + let b: u32x4 = u32x4::new(0, 1, 2, 3); + let e: u32x4 = u32x4::new(0xFF_FF_FF_FF, 0xFF_FF_FF_FF, 0xFF_FF_FF_FF, 0xFF_FF_FF_FF); + let r: u32x4 = transmute(vcgtq_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcgt_f32() { + let a: f32x2 = f32x2::new(1.2, 2.3); + let b: f32x2 = f32x2::new(0.1, 1.2); + let e: u32x2 = u32x2::new(0xFF_FF_FF_FF, 0xFF_FF_FF_FF); + let r: u32x2 = transmute(vcgt_f32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcgtq_f32() { + let a: f32x4 = f32x4::new(1.2, 2.3, 3.4, 4.5); + let b: f32x4 = f32x4::new(0.1, 1.2, 2.3, 3.4); + let e: u32x4 = u32x4::new(0xFF_FF_FF_FF, 0xFF_FF_FF_FF, 0xFF_FF_FF_FF, 0xFF_FF_FF_FF); + let r: u32x4 = transmute(vcgtq_f32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vclt_s8() { + let a: i8x8 = i8x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let b: i8x8 = i8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let e: u8x8 = u8x8::new(0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF); + let r: u8x8 = transmute(vclt_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcltq_s8() { + let a: i8x16 = i8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let b: i8x16 = i8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let e: u8x16 = u8x16::new(0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF); + let r: u8x16 = transmute(vcltq_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vclt_s16() { + let a: i16x4 = i16x4::new(0, 1, 2, 3); + let b: i16x4 = i16x4::new(1, 2, 3, 4); + let e: u16x4 = u16x4::new(0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF); + let r: u16x4 = transmute(vclt_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcltq_s16() { + let a: i16x8 = i16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let b: i16x8 = i16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let e: u16x8 = u16x8::new(0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF); + let r: u16x8 = transmute(vcltq_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vclt_s32() { + let a: i32x2 = i32x2::new(0, 1); + let b: i32x2 = i32x2::new(1, 2); + let e: u32x2 = u32x2::new(0xFF_FF_FF_FF, 0xFF_FF_FF_FF); + let r: u32x2 = transmute(vclt_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcltq_s32() { + let a: i32x4 = i32x4::new(0, 1, 2, 3); + let b: i32x4 = i32x4::new(1, 2, 3, 4); + let e: u32x4 = u32x4::new(0xFF_FF_FF_FF, 0xFF_FF_FF_FF, 0xFF_FF_FF_FF, 0xFF_FF_FF_FF); + let r: u32x4 = transmute(vcltq_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vclt_u8() { + let a: u8x8 = u8x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let b: u8x8 = u8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let e: u8x8 = u8x8::new(0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF); + let r: u8x8 = transmute(vclt_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcltq_u8() { + let a: u8x16 = u8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let b: u8x16 = u8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let e: u8x16 = u8x16::new(0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF); + let r: u8x16 = transmute(vcltq_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vclt_u16() { + let a: u16x4 = u16x4::new(0, 1, 2, 3); + let b: u16x4 = u16x4::new(1, 2, 3, 4); + let e: u16x4 = u16x4::new(0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF); + let r: u16x4 = transmute(vclt_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcltq_u16() { + let a: u16x8 = u16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let b: u16x8 = u16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let e: u16x8 = u16x8::new(0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF); + let r: u16x8 = transmute(vcltq_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vclt_u32() { + let a: u32x2 = u32x2::new(0, 1); + let b: u32x2 = u32x2::new(1, 2); + let e: u32x2 = u32x2::new(0xFF_FF_FF_FF, 0xFF_FF_FF_FF); + let r: u32x2 = transmute(vclt_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcltq_u32() { + let a: u32x4 = u32x4::new(0, 1, 2, 3); + let b: u32x4 = u32x4::new(1, 2, 3, 4); + let e: u32x4 = u32x4::new(0xFF_FF_FF_FF, 0xFF_FF_FF_FF, 0xFF_FF_FF_FF, 0xFF_FF_FF_FF); + let r: u32x4 = transmute(vcltq_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vclt_f32() { + let a: f32x2 = f32x2::new(0.1, 1.2); + let b: f32x2 = f32x2::new(1.2, 2.3); + let e: u32x2 = u32x2::new(0xFF_FF_FF_FF, 0xFF_FF_FF_FF); + let r: u32x2 = transmute(vclt_f32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcltq_f32() { + let a: f32x4 = f32x4::new(0.1, 1.2, 2.3, 3.4); + let b: f32x4 = f32x4::new(1.2, 2.3, 3.4, 4.5); + let e: u32x4 = u32x4::new(0xFF_FF_FF_FF, 0xFF_FF_FF_FF, 0xFF_FF_FF_FF, 0xFF_FF_FF_FF); + let r: u32x4 = transmute(vcltq_f32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcle_s8() { + let a: i8x8 = i8x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let b: i8x8 = i8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let e: u8x8 = u8x8::new(0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF); + let r: u8x8 = transmute(vcle_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcleq_s8() { + let a: i8x16 = i8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let b: i8x16 = i8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let e: u8x16 = u8x16::new(0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF); + let r: u8x16 = transmute(vcleq_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcle_s16() { + let a: i16x4 = i16x4::new(0, 1, 2, 3); + let b: i16x4 = i16x4::new(1, 2, 3, 4); + let e: u16x4 = u16x4::new(0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF); + let r: u16x4 = transmute(vcle_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcleq_s16() { + let a: i16x8 = i16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let b: i16x8 = i16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let e: u16x8 = u16x8::new(0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF); + let r: u16x8 = transmute(vcleq_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcle_s32() { + let a: i32x2 = i32x2::new(0, 1); + let b: i32x2 = i32x2::new(1, 2); + let e: u32x2 = u32x2::new(0xFF_FF_FF_FF, 0xFF_FF_FF_FF); + let r: u32x2 = transmute(vcle_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcleq_s32() { + let a: i32x4 = i32x4::new(0, 1, 2, 3); + let b: i32x4 = i32x4::new(1, 2, 3, 4); + let e: u32x4 = u32x4::new(0xFF_FF_FF_FF, 0xFF_FF_FF_FF, 0xFF_FF_FF_FF, 0xFF_FF_FF_FF); + let r: u32x4 = transmute(vcleq_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcle_u8() { + let a: u8x8 = u8x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let b: u8x8 = u8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let e: u8x8 = u8x8::new(0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF); + let r: u8x8 = transmute(vcle_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcleq_u8() { + let a: u8x16 = u8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let b: u8x16 = u8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let e: u8x16 = u8x16::new(0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF); + let r: u8x16 = transmute(vcleq_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcle_u16() { + let a: u16x4 = u16x4::new(0, 1, 2, 3); + let b: u16x4 = u16x4::new(1, 2, 3, 4); + let e: u16x4 = u16x4::new(0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF); + let r: u16x4 = transmute(vcle_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcleq_u16() { + let a: u16x8 = u16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let b: u16x8 = u16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let e: u16x8 = u16x8::new(0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF); + let r: u16x8 = transmute(vcleq_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcle_u32() { + let a: u32x2 = u32x2::new(0, 1); + let b: u32x2 = u32x2::new(1, 2); + let e: u32x2 = u32x2::new(0xFF_FF_FF_FF, 0xFF_FF_FF_FF); + let r: u32x2 = transmute(vcle_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcleq_u32() { + let a: u32x4 = u32x4::new(0, 1, 2, 3); + let b: u32x4 = u32x4::new(1, 2, 3, 4); + let e: u32x4 = u32x4::new(0xFF_FF_FF_FF, 0xFF_FF_FF_FF, 0xFF_FF_FF_FF, 0xFF_FF_FF_FF); + let r: u32x4 = transmute(vcleq_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcle_f32() { + let a: f32x2 = f32x2::new(0.1, 1.2); + let b: f32x2 = f32x2::new(1.2, 2.3); + let e: u32x2 = u32x2::new(0xFF_FF_FF_FF, 0xFF_FF_FF_FF); + let r: u32x2 = transmute(vcle_f32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcleq_f32() { + let a: f32x4 = f32x4::new(0.1, 1.2, 2.3, 3.4); + let b: f32x4 = f32x4::new(1.2, 2.3, 3.4, 4.5); + let e: u32x4 = u32x4::new(0xFF_FF_FF_FF, 0xFF_FF_FF_FF, 0xFF_FF_FF_FF, 0xFF_FF_FF_FF); + let r: u32x4 = transmute(vcleq_f32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcge_s8() { + let a: i8x8 = i8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let b: i8x8 = i8x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let e: u8x8 = u8x8::new(0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF); + let r: u8x8 = transmute(vcge_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcgeq_s8() { + let a: i8x16 = i8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let b: i8x16 = i8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let e: u8x16 = u8x16::new(0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF); + let r: u8x16 = transmute(vcgeq_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcge_s16() { + let a: i16x4 = i16x4::new(1, 2, 3, 4); + let b: i16x4 = i16x4::new(0, 1, 2, 3); + let e: u16x4 = u16x4::new(0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF); + let r: u16x4 = transmute(vcge_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcgeq_s16() { + let a: i16x8 = i16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let b: i16x8 = i16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let e: u16x8 = u16x8::new(0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF); + let r: u16x8 = transmute(vcgeq_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcge_s32() { + let a: i32x2 = i32x2::new(1, 2); + let b: i32x2 = i32x2::new(0, 1); + let e: u32x2 = u32x2::new(0xFF_FF_FF_FF, 0xFF_FF_FF_FF); + let r: u32x2 = transmute(vcge_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcgeq_s32() { + let a: i32x4 = i32x4::new(1, 2, 3, 4); + let b: i32x4 = i32x4::new(0, 1, 2, 3); + let e: u32x4 = u32x4::new(0xFF_FF_FF_FF, 0xFF_FF_FF_FF, 0xFF_FF_FF_FF, 0xFF_FF_FF_FF); + let r: u32x4 = transmute(vcgeq_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcge_u8() { + let a: u8x8 = u8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let b: u8x8 = u8x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let e: u8x8 = u8x8::new(0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF); + let r: u8x8 = transmute(vcge_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcgeq_u8() { + let a: u8x16 = u8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let b: u8x16 = u8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let e: u8x16 = u8x16::new(0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF); + let r: u8x16 = transmute(vcgeq_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcge_u16() { + let a: u16x4 = u16x4::new(1, 2, 3, 4); + let b: u16x4 = u16x4::new(0, 1, 2, 3); + let e: u16x4 = u16x4::new(0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF); + let r: u16x4 = transmute(vcge_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcgeq_u16() { + let a: u16x8 = u16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let b: u16x8 = u16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let e: u16x8 = u16x8::new(0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF); + let r: u16x8 = transmute(vcgeq_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcge_u32() { + let a: u32x2 = u32x2::new(1, 2); + let b: u32x2 = u32x2::new(0, 1); + let e: u32x2 = u32x2::new(0xFF_FF_FF_FF, 0xFF_FF_FF_FF); + let r: u32x2 = transmute(vcge_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcgeq_u32() { + let a: u32x4 = u32x4::new(1, 2, 3, 4); + let b: u32x4 = u32x4::new(0, 1, 2, 3); + let e: u32x4 = u32x4::new(0xFF_FF_FF_FF, 0xFF_FF_FF_FF, 0xFF_FF_FF_FF, 0xFF_FF_FF_FF); + let r: u32x4 = transmute(vcgeq_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcge_f32() { + let a: f32x2 = f32x2::new(1.2, 2.3); + let b: f32x2 = f32x2::new(0.1, 1.2); + let e: u32x2 = u32x2::new(0xFF_FF_FF_FF, 0xFF_FF_FF_FF); + let r: u32x2 = transmute(vcge_f32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcgeq_f32() { + let a: f32x4 = f32x4::new(1.2, 2.3, 3.4, 4.5); + let b: f32x4 = f32x4::new(0.1, 1.2, 2.3, 3.4); + let e: u32x4 = u32x4::new(0xFF_FF_FF_FF, 0xFF_FF_FF_FF, 0xFF_FF_FF_FF, 0xFF_FF_FF_FF); + let r: u32x4 = transmute(vcgeq_f32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcls_s8() { + let a: i8x8 = i8x8::new(-128, -1, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00); + let e: i8x8 = i8x8::new(0, 7, 7, 7, 7, 7, 7, 7); + let r: i8x8 = transmute(vcls_s8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vclsq_s8() { + let a: i8x16 = i8x16::new(-128, -1, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7F); + let e: i8x16 = i8x16::new(0, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 0); + let r: i8x16 = transmute(vclsq_s8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcls_s16() { + let a: i16x4 = i16x4::new(-32768, -1, 0x00, 0x00); + let e: i16x4 = i16x4::new(0, 15, 15, 15); + let r: i16x4 = transmute(vcls_s16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vclsq_s16() { + let a: i16x8 = i16x8::new(-32768, -1, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00); + let e: i16x8 = i16x8::new(0, 15, 15, 15, 15, 15, 15, 15); + let r: i16x8 = transmute(vclsq_s16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcls_s32() { + let a: i32x2 = i32x2::new(-2147483648, -1); + let e: i32x2 = i32x2::new(0, 31); + let r: i32x2 = transmute(vcls_s32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vclsq_s32() { + let a: i32x4 = i32x4::new(-2147483648, -1, 0x00, 0x00); + let e: i32x4 = i32x4::new(0, 31, 31, 31); + let r: i32x4 = transmute(vclsq_s32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vclz_s8() { + let a: i8x8 = i8x8::new(-128, -1, 0x00, 0x01, 0x01, 0x01, 0x01, 0x01); + let e: i8x8 = i8x8::new(0, 0, 8, 7, 7, 7, 7, 7); + let r: i8x8 = transmute(vclz_s8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vclzq_s8() { + let a: i8x16 = i8x16::new(-128, -1, 0x00, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x7F); + let e: i8x16 = i8x16::new(0, 0, 8, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 1); + let r: i8x16 = transmute(vclzq_s8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vclz_s16() { + let a: i16x4 = i16x4::new(-32768, -1, 0x00, 0x01); + let e: i16x4 = i16x4::new(0, 0, 16, 15); + let r: i16x4 = transmute(vclz_s16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vclzq_s16() { + let a: i16x8 = i16x8::new(-32768, -1, 0x00, 0x01, 0x01, 0x01, 0x01, 0x01); + let e: i16x8 = i16x8::new(0, 0, 16, 15, 15, 15, 15, 15); + let r: i16x8 = transmute(vclzq_s16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vclz_s32() { + let a: i32x2 = i32x2::new(-2147483648, -1); + let e: i32x2 = i32x2::new(0, 0); + let r: i32x2 = transmute(vclz_s32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vclzq_s32() { + let a: i32x4 = i32x4::new(-2147483648, -1, 0x00, 0x01); + let e: i32x4 = i32x4::new(0, 0, 32, 31); + let r: i32x4 = transmute(vclzq_s32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vclz_u8() { + let a: u8x8 = u8x8::new(0, 0x00, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01); + let e: u8x8 = u8x8::new(8, 8, 7, 7, 7, 7, 7, 7); + let r: u8x8 = transmute(vclz_u8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vclzq_u8() { + let a: u8x16 = u8x16::new(0, 0x00, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0xFF); + let e: u8x16 = u8x16::new(8, 8, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 0); + let r: u8x16 = transmute(vclzq_u8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vclz_u16() { + let a: u16x4 = u16x4::new(0, 0x00, 0x01, 0x01); + let e: u16x4 = u16x4::new(16, 16, 15, 15); + let r: u16x4 = transmute(vclz_u16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vclzq_u16() { + let a: u16x8 = u16x8::new(0, 0x00, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01); + let e: u16x8 = u16x8::new(16, 16, 15, 15, 15, 15, 15, 15); + let r: u16x8 = transmute(vclzq_u16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vclz_u32() { + let a: u32x2 = u32x2::new(0, 0x00); + let e: u32x2 = u32x2::new(32, 32); + let r: u32x2 = transmute(vclz_u32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vclzq_u32() { + let a: u32x4 = u32x4::new(0, 0x00, 0x01, 0x01); + let e: u32x4 = u32x4::new(32, 32, 31, 31); + let r: u32x4 = transmute(vclzq_u32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcagt_f32() { + let a: f32x2 = f32x2::new(-1.2, 0.0); + let b: f32x2 = f32x2::new(-1.1, 0.0); + let e: u32x2 = u32x2::new(0xFF_FF_FF_FF, 0); + let r: u32x2 = transmute(vcagt_f32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcagtq_f32() { + let a: f32x4 = f32x4::new(-1.2, 0.0, 1.2, 2.3); + let b: f32x4 = f32x4::new(-1.1, 0.0, 1.1, 2.4); + let e: u32x4 = u32x4::new(0xFF_FF_FF_FF, 0, 0xFF_FF_FF_FF, 0); + let r: u32x4 = transmute(vcagtq_f32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcage_f32() { + let a: f32x2 = f32x2::new(-1.2, 0.0); + let b: f32x2 = f32x2::new(-1.1, 0.0); + let e: u32x2 = u32x2::new(0xFF_FF_FF_FF, 0xFF_FF_FF_FF); + let r: u32x2 = transmute(vcage_f32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcageq_f32() { + let a: f32x4 = f32x4::new(-1.2, 0.0, 1.2, 2.3); + let b: f32x4 = f32x4::new(-1.1, 0.0, 1.1, 2.4); + let e: u32x4 = u32x4::new(0xFF_FF_FF_FF, 0xFF_FF_FF_FF, 0xFF_FF_FF_FF, 0); + let r: u32x4 = transmute(vcageq_f32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcalt_f32() { + let a: f32x2 = f32x2::new(-1.2, 0.0); + let b: f32x2 = f32x2::new(-1.1, 0.0); + let e: u32x2 = u32x2::new(0, 0); + let r: u32x2 = transmute(vcalt_f32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcaltq_f32() { + let a: f32x4 = f32x4::new(-1.2, 0.0, 1.2, 2.3); + let b: f32x4 = f32x4::new(-1.1, 0.0, 1.1, 2.4); + let e: u32x4 = u32x4::new(0, 0, 0, 0xFF_FF_FF_FF); + let r: u32x4 = transmute(vcaltq_f32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcale_f32() { + let a: f32x2 = f32x2::new(-1.2, 0.0); + let b: f32x2 = f32x2::new(-1.1, 0.0); + let e: u32x2 = u32x2::new(0, 0xFF_FF_FF_FF); + let r: u32x2 = transmute(vcale_f32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcaleq_f32() { + let a: f32x4 = f32x4::new(-1.2, 0.0, 1.2, 2.3); + let b: f32x4 = f32x4::new(-1.1, 0.0, 1.1, 2.4); + let e: u32x4 = u32x4::new(0, 0xFF_FF_FF_FF, 0, 0xFF_FF_FF_FF); + let r: u32x4 = transmute(vcaleq_f32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcreate_s8() { + let a: u64 = 1; + let e: i8x8 = i8x8::new(1, 0, 0, 0, 0, 0, 0, 0); + let r: i8x8 = transmute(vcreate_s8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcreate_s32() { + let a: u64 = 1; + let e: i32x2 = i32x2::new(1, 0); + let r: i32x2 = transmute(vcreate_s32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcreate_s64() { + let a: u64 = 1; + let e: i64x1 = i64x1::new(1); + let r: i64x1 = transmute(vcreate_s64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcreate_u8() { + let a: u64 = 1; + let e: u8x8 = u8x8::new(1, 0, 0, 0, 0, 0, 0, 0); + let r: u8x8 = transmute(vcreate_u8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcreate_u32() { + let a: u64 = 1; + let e: u32x2 = u32x2::new(1, 0); + let r: u32x2 = transmute(vcreate_u32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcreate_u64() { + let a: u64 = 1; + let e: u64x1 = u64x1::new(1); + let r: u64x1 = transmute(vcreate_u64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcreate_p8() { + let a: u64 = 1; + let e: i8x8 = i8x8::new(1, 0, 0, 0, 0, 0, 0, 0); + let r: i8x8 = transmute(vcreate_p8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcreate_p16() { + let a: u64 = 1; + let e: i16x4 = i16x4::new(1, 0, 0, 0); + let r: i16x4 = transmute(vcreate_p16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcreate_p64() { + let a: u64 = 1; + let e: i64x1 = i64x1::new(1); + let r: i64x1 = transmute(vcreate_p64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcreate_f32() { + let a: u64 = 0; + let e: f32x2 = f32x2::new(0., 0.); + let r: f32x2 = transmute(vcreate_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvt_f32_s32() { + let a: i32x2 = i32x2::new(1, 2); + let e: f32x2 = f32x2::new(1., 2.); + let r: f32x2 = transmute(vcvt_f32_s32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvtq_f32_s32() { + let a: i32x4 = i32x4::new(1, 2, 3, 4); + let e: f32x4 = f32x4::new(1., 2., 3., 4.); + let r: f32x4 = transmute(vcvtq_f32_s32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvt_f32_u32() { + let a: u32x2 = u32x2::new(1, 2); + let e: f32x2 = f32x2::new(1., 2.); + let r: f32x2 = transmute(vcvt_f32_u32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvtq_f32_u32() { + let a: u32x4 = u32x4::new(1, 2, 3, 4); + let e: f32x4 = f32x4::new(1., 2., 3., 4.); + let r: f32x4 = transmute(vcvtq_f32_u32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvt_n_f32_s32() { + let a: i32x2 = i32x2::new(1, 2); + let e: f32x2 = f32x2::new(0.25, 0.5); + let r: f32x2 = transmute(vcvt_n_f32_s32::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvtq_n_f32_s32() { + let a: i32x4 = i32x4::new(1, 2, 3, 4); + let e: f32x4 = f32x4::new(0.25, 0.5, 0.75, 1.); + let r: f32x4 = transmute(vcvtq_n_f32_s32::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvt_n_f32_u32() { + let a: u32x2 = u32x2::new(1, 2); + let e: f32x2 = f32x2::new(0.25, 0.5); + let r: f32x2 = transmute(vcvt_n_f32_u32::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvtq_n_f32_u32() { + let a: u32x4 = u32x4::new(1, 2, 3, 4); + let e: f32x4 = f32x4::new(0.25, 0.5, 0.75, 1.); + let r: f32x4 = transmute(vcvtq_n_f32_u32::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvt_n_s32_f32() { + let a: f32x2 = f32x2::new(0.25, 0.5); + let e: i32x2 = i32x2::new(1, 2); + let r: i32x2 = transmute(vcvt_n_s32_f32::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvtq_n_s32_f32() { + let a: f32x4 = f32x4::new(0.25, 0.5, 0.75, 1.); + let e: i32x4 = i32x4::new(1, 2, 3, 4); + let r: i32x4 = transmute(vcvtq_n_s32_f32::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvt_n_u32_f32() { + let a: f32x2 = f32x2::new(0.25, 0.5); + let e: u32x2 = u32x2::new(1, 2); + let r: u32x2 = transmute(vcvt_n_u32_f32::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvtq_n_u32_f32() { + let a: f32x4 = f32x4::new(0.25, 0.5, 0.75, 1.); + let e: u32x4 = u32x4::new(1, 2, 3, 4); + let r: u32x4 = transmute(vcvtq_n_u32_f32::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvt_s32_f32() { + let a: f32x2 = f32x2::new(-1.1, 2.1); + let e: i32x2 = i32x2::new(-1, 2); + let r: i32x2 = transmute(vcvt_s32_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvtq_s32_f32() { + let a: f32x4 = f32x4::new(-1.1, 2.1, -2.9, 3.9); + let e: i32x4 = i32x4::new(-1, 2, -2, 3); + let r: i32x4 = transmute(vcvtq_s32_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvt_u32_f32() { + let a: f32x2 = f32x2::new(1.1, 2.1); + let e: u32x2 = u32x2::new(1, 2); + let r: u32x2 = transmute(vcvt_u32_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vcvtq_u32_f32() { + let a: f32x4 = f32x4::new(1.1, 2.1, 2.9, 3.9); + let e: u32x4 = u32x4::new(1, 2, 2, 3); + let r: u32x4 = transmute(vcvtq_u32_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdup_lane_s8() { + let a: i8x8 = i8x8::new(1, 1, 1, 4, 1, 6, 7, 8); + let e: i8x8 = i8x8::new(1, 1, 1, 1, 1, 1, 1, 1); + let r: i8x8 = transmute(vdup_lane_s8::<4>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdupq_laneq_s8() { + let a: i8x16 = i8x16::new(1, 1, 1, 4, 1, 6, 7, 8, 1, 10, 11, 12, 13, 14, 15, 16); + let e: i8x16 = i8x16::new(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1); + let r: i8x16 = transmute(vdupq_laneq_s8::<8>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdup_lane_s16() { + let a: i16x4 = i16x4::new(1, 1, 1, 4); + let e: i16x4 = i16x4::new(1, 1, 1, 1); + let r: i16x4 = transmute(vdup_lane_s16::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdupq_laneq_s16() { + let a: i16x8 = i16x8::new(1, 1, 1, 4, 1, 6, 7, 8); + let e: i16x8 = i16x8::new(1, 1, 1, 1, 1, 1, 1, 1); + let r: i16x8 = transmute(vdupq_laneq_s16::<4>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdup_lane_s32() { + let a: i32x2 = i32x2::new(1, 1); + let e: i32x2 = i32x2::new(1, 1); + let r: i32x2 = transmute(vdup_lane_s32::<1>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdupq_laneq_s32() { + let a: i32x4 = i32x4::new(1, 1, 1, 4); + let e: i32x4 = i32x4::new(1, 1, 1, 1); + let r: i32x4 = transmute(vdupq_laneq_s32::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdup_laneq_s8() { + let a: i8x16 = i8x16::new(1, 1, 1, 4, 1, 6, 7, 8, 1, 10, 11, 12, 13, 14, 15, 16); + let e: i8x8 = i8x8::new(1, 1, 1, 1, 1, 1, 1, 1); + let r: i8x8 = transmute(vdup_laneq_s8::<8>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdup_laneq_s16() { + let a: i16x8 = i16x8::new(1, 1, 1, 4, 1, 6, 7, 8); + let e: i16x4 = i16x4::new(1, 1, 1, 1); + let r: i16x4 = transmute(vdup_laneq_s16::<4>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdup_laneq_s32() { + let a: i32x4 = i32x4::new(1, 1, 1, 4); + let e: i32x2 = i32x2::new(1, 1); + let r: i32x2 = transmute(vdup_laneq_s32::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdupq_lane_s8() { + let a: i8x8 = i8x8::new(1, 1, 1, 4, 1, 6, 7, 8); + let e: i8x16 = i8x16::new(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1); + let r: i8x16 = transmute(vdupq_lane_s8::<4>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdupq_lane_s16() { + let a: i16x4 = i16x4::new(1, 1, 1, 4); + let e: i16x8 = i16x8::new(1, 1, 1, 1, 1, 1, 1, 1); + let r: i16x8 = transmute(vdupq_lane_s16::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdupq_lane_s32() { + let a: i32x2 = i32x2::new(1, 1); + let e: i32x4 = i32x4::new(1, 1, 1, 1); + let r: i32x4 = transmute(vdupq_lane_s32::<1>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdup_lane_u8() { + let a: u8x8 = u8x8::new(1, 1, 1, 4, 1, 6, 7, 8); + let e: u8x8 = u8x8::new(1, 1, 1, 1, 1, 1, 1, 1); + let r: u8x8 = transmute(vdup_lane_u8::<4>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdupq_laneq_u8() { + let a: u8x16 = u8x16::new(1, 1, 1, 4, 1, 6, 7, 8, 1, 10, 11, 12, 13, 14, 15, 16); + let e: u8x16 = u8x16::new(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1); + let r: u8x16 = transmute(vdupq_laneq_u8::<8>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdup_lane_u16() { + let a: u16x4 = u16x4::new(1, 1, 1, 4); + let e: u16x4 = u16x4::new(1, 1, 1, 1); + let r: u16x4 = transmute(vdup_lane_u16::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdupq_laneq_u16() { + let a: u16x8 = u16x8::new(1, 1, 1, 4, 1, 6, 7, 8); + let e: u16x8 = u16x8::new(1, 1, 1, 1, 1, 1, 1, 1); + let r: u16x8 = transmute(vdupq_laneq_u16::<4>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdup_lane_u32() { + let a: u32x2 = u32x2::new(1, 1); + let e: u32x2 = u32x2::new(1, 1); + let r: u32x2 = transmute(vdup_lane_u32::<1>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdupq_laneq_u32() { + let a: u32x4 = u32x4::new(1, 1, 1, 4); + let e: u32x4 = u32x4::new(1, 1, 1, 1); + let r: u32x4 = transmute(vdupq_laneq_u32::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdup_laneq_u8() { + let a: u8x16 = u8x16::new(1, 1, 1, 4, 1, 6, 7, 8, 1, 10, 11, 12, 13, 14, 15, 16); + let e: u8x8 = u8x8::new(1, 1, 1, 1, 1, 1, 1, 1); + let r: u8x8 = transmute(vdup_laneq_u8::<8>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdup_laneq_u16() { + let a: u16x8 = u16x8::new(1, 1, 1, 4, 1, 6, 7, 8); + let e: u16x4 = u16x4::new(1, 1, 1, 1); + let r: u16x4 = transmute(vdup_laneq_u16::<4>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdup_laneq_u32() { + let a: u32x4 = u32x4::new(1, 1, 1, 4); + let e: u32x2 = u32x2::new(1, 1); + let r: u32x2 = transmute(vdup_laneq_u32::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdupq_lane_u8() { + let a: u8x8 = u8x8::new(1, 1, 1, 4, 1, 6, 7, 8); + let e: u8x16 = u8x16::new(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1); + let r: u8x16 = transmute(vdupq_lane_u8::<4>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdupq_lane_u16() { + let a: u16x4 = u16x4::new(1, 1, 1, 4); + let e: u16x8 = u16x8::new(1, 1, 1, 1, 1, 1, 1, 1); + let r: u16x8 = transmute(vdupq_lane_u16::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdupq_lane_u32() { + let a: u32x2 = u32x2::new(1, 1); + let e: u32x4 = u32x4::new(1, 1, 1, 1); + let r: u32x4 = transmute(vdupq_lane_u32::<1>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdup_lane_p8() { + let a: i8x8 = i8x8::new(1, 1, 1, 4, 1, 6, 7, 8); + let e: i8x8 = i8x8::new(1, 1, 1, 1, 1, 1, 1, 1); + let r: i8x8 = transmute(vdup_lane_p8::<4>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdupq_laneq_p8() { + let a: i8x16 = i8x16::new(1, 1, 1, 4, 1, 6, 7, 8, 1, 10, 11, 12, 13, 14, 15, 16); + let e: i8x16 = i8x16::new(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1); + let r: i8x16 = transmute(vdupq_laneq_p8::<8>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdup_lane_p16() { + let a: i16x4 = i16x4::new(1, 1, 1, 4); + let e: i16x4 = i16x4::new(1, 1, 1, 1); + let r: i16x4 = transmute(vdup_lane_p16::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdupq_laneq_p16() { + let a: i16x8 = i16x8::new(1, 1, 1, 4, 1, 6, 7, 8); + let e: i16x8 = i16x8::new(1, 1, 1, 1, 1, 1, 1, 1); + let r: i16x8 = transmute(vdupq_laneq_p16::<4>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdup_laneq_p8() { + let a: i8x16 = i8x16::new(1, 1, 1, 4, 1, 6, 7, 8, 1, 10, 11, 12, 13, 14, 15, 16); + let e: i8x8 = i8x8::new(1, 1, 1, 1, 1, 1, 1, 1); + let r: i8x8 = transmute(vdup_laneq_p8::<8>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdup_laneq_p16() { + let a: i16x8 = i16x8::new(1, 1, 1, 4, 1, 6, 7, 8); + let e: i16x4 = i16x4::new(1, 1, 1, 1); + let r: i16x4 = transmute(vdup_laneq_p16::<4>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdupq_lane_p8() { + let a: i8x8 = i8x8::new(1, 1, 1, 4, 1, 6, 7, 8); + let e: i8x16 = i8x16::new(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1); + let r: i8x16 = transmute(vdupq_lane_p8::<4>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdupq_lane_p16() { + let a: i16x4 = i16x4::new(1, 1, 1, 4); + let e: i16x8 = i16x8::new(1, 1, 1, 1, 1, 1, 1, 1); + let r: i16x8 = transmute(vdupq_lane_p16::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdupq_laneq_s64() { + let a: i64x2 = i64x2::new(1, 1); + let e: i64x2 = i64x2::new(1, 1); + let r: i64x2 = transmute(vdupq_laneq_s64::<1>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdupq_lane_s64() { + let a: i64x1 = i64x1::new(1); + let e: i64x2 = i64x2::new(1, 1); + let r: i64x2 = transmute(vdupq_lane_s64::<0>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdupq_laneq_u64() { + let a: u64x2 = u64x2::new(1, 1); + let e: u64x2 = u64x2::new(1, 1); + let r: u64x2 = transmute(vdupq_laneq_u64::<1>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdupq_lane_u64() { + let a: u64x1 = u64x1::new(1); + let e: u64x2 = u64x2::new(1, 1); + let r: u64x2 = transmute(vdupq_lane_u64::<0>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdup_lane_f32() { + let a: f32x2 = f32x2::new(1., 1.); + let e: f32x2 = f32x2::new(1., 1.); + let r: f32x2 = transmute(vdup_lane_f32::<1>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdupq_laneq_f32() { + let a: f32x4 = f32x4::new(1., 1., 1., 4.); + let e: f32x4 = f32x4::new(1., 1., 1., 1.); + let r: f32x4 = transmute(vdupq_laneq_f32::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdup_laneq_f32() { + let a: f32x4 = f32x4::new(1., 1., 1., 4.); + let e: f32x2 = f32x2::new(1., 1.); + let r: f32x2 = transmute(vdup_laneq_f32::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdupq_lane_f32() { + let a: f32x2 = f32x2::new(1., 1.); + let e: f32x4 = f32x4::new(1., 1., 1., 1.); + let r: f32x4 = transmute(vdupq_lane_f32::<1>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdup_lane_s64() { + let a: i64x1 = i64x1::new(0); + let e: i64x1 = i64x1::new(0); + let r: i64x1 = transmute(vdup_lane_s64::<0>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdup_lane_u64() { + let a: u64x1 = u64x1::new(0); + let e: u64x1 = u64x1::new(0); + let r: u64x1 = transmute(vdup_lane_u64::<0>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdup_laneq_s64() { + let a: i64x2 = i64x2::new(0, 1); + let e: i64x1 = i64x1::new(1); + let r: i64x1 = transmute(vdup_laneq_s64::<1>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdup_laneq_u64() { + let a: u64x2 = u64x2::new(0, 1); + let e: u64x1 = u64x1::new(1); + let r: u64x1 = transmute(vdup_laneq_u64::<1>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vext_s8() { + let a: i8x8 = i8x8::new(0, 8, 8, 9, 8, 9, 9, 11); + let b: i8x8 = i8x8::new(9, 11, 14, 15, 16, 17, 18, 19); + let e: i8x8 = i8x8::new(8, 9, 9, 11, 9, 11, 14, 15); + let r: i8x8 = transmute(vext_s8::<4>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vextq_s8() { + let a: i8x16 = i8x16::new(0, 8, 8, 9, 8, 9, 9, 11, 8, 9, 9, 11, 9, 11, 14, 15); + let b: i8x16 = i8x16::new(9, 11, 14, 15, 16, 17, 18, 19, 0, 8, 8, 9, 8, 9, 9, 11); + let e: i8x16 = i8x16::new(8, 9, 9, 11, 9, 11, 14, 15, 9, 11, 14, 15, 16, 17, 18, 19); + let r: i8x16 = transmute(vextq_s8::<8>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vext_s16() { + let a: i16x4 = i16x4::new(0, 8, 8, 9); + let b: i16x4 = i16x4::new(9, 11, 14, 15); + let e: i16x4 = i16x4::new(8, 9, 9, 11); + let r: i16x4 = transmute(vext_s16::<2>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vextq_s16() { + let a: i16x8 = i16x8::new(0, 8, 8, 9, 8, 9, 9, 11); + let b: i16x8 = i16x8::new(9, 11, 14, 15, 16, 17, 18, 19); + let e: i16x8 = i16x8::new(8, 9, 9, 11, 9, 11, 14, 15); + let r: i16x8 = transmute(vextq_s16::<4>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vext_s32() { + let a: i32x2 = i32x2::new(0, 8); + let b: i32x2 = i32x2::new(9, 11); + let e: i32x2 = i32x2::new(8, 9); + let r: i32x2 = transmute(vext_s32::<1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vextq_s32() { + let a: i32x4 = i32x4::new(0, 8, 8, 9); + let b: i32x4 = i32x4::new(9, 11, 14, 15); + let e: i32x4 = i32x4::new(8, 9, 9, 11); + let r: i32x4 = transmute(vextq_s32::<2>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vext_u8() { + let a: u8x8 = u8x8::new(0, 8, 8, 9, 8, 9, 9, 11); + let b: u8x8 = u8x8::new(9, 11, 14, 15, 16, 17, 18, 19); + let e: u8x8 = u8x8::new(8, 9, 9, 11, 9, 11, 14, 15); + let r: u8x8 = transmute(vext_u8::<4>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vextq_u8() { + let a: u8x16 = u8x16::new(0, 8, 8, 9, 8, 9, 9, 11, 8, 9, 9, 11, 9, 11, 14, 15); + let b: u8x16 = u8x16::new(9, 11, 14, 15, 16, 17, 18, 19, 0, 8, 8, 9, 8, 9, 9, 11); + let e: u8x16 = u8x16::new(8, 9, 9, 11, 9, 11, 14, 15, 9, 11, 14, 15, 16, 17, 18, 19); + let r: u8x16 = transmute(vextq_u8::<8>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vext_u16() { + let a: u16x4 = u16x4::new(0, 8, 8, 9); + let b: u16x4 = u16x4::new(9, 11, 14, 15); + let e: u16x4 = u16x4::new(8, 9, 9, 11); + let r: u16x4 = transmute(vext_u16::<2>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vextq_u16() { + let a: u16x8 = u16x8::new(0, 8, 8, 9, 8, 9, 9, 11); + let b: u16x8 = u16x8::new(9, 11, 14, 15, 16, 17, 18, 19); + let e: u16x8 = u16x8::new(8, 9, 9, 11, 9, 11, 14, 15); + let r: u16x8 = transmute(vextq_u16::<4>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vext_u32() { + let a: u32x2 = u32x2::new(0, 8); + let b: u32x2 = u32x2::new(9, 11); + let e: u32x2 = u32x2::new(8, 9); + let r: u32x2 = transmute(vext_u32::<1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vextq_u32() { + let a: u32x4 = u32x4::new(0, 8, 8, 9); + let b: u32x4 = u32x4::new(9, 11, 14, 15); + let e: u32x4 = u32x4::new(8, 9, 9, 11); + let r: u32x4 = transmute(vextq_u32::<2>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vext_p8() { + let a: i8x8 = i8x8::new(0, 8, 8, 9, 8, 9, 9, 11); + let b: i8x8 = i8x8::new(9, 11, 14, 15, 16, 17, 18, 19); + let e: i8x8 = i8x8::new(8, 9, 9, 11, 9, 11, 14, 15); + let r: i8x8 = transmute(vext_p8::<4>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vextq_p8() { + let a: i8x16 = i8x16::new(0, 8, 8, 9, 8, 9, 9, 11, 8, 9, 9, 11, 9, 11, 14, 15); + let b: i8x16 = i8x16::new(9, 11, 14, 15, 16, 17, 18, 19, 0, 8, 8, 9, 8, 9, 9, 11); + let e: i8x16 = i8x16::new(8, 9, 9, 11, 9, 11, 14, 15, 9, 11, 14, 15, 16, 17, 18, 19); + let r: i8x16 = transmute(vextq_p8::<8>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vext_p16() { + let a: i16x4 = i16x4::new(0, 8, 8, 9); + let b: i16x4 = i16x4::new(9, 11, 14, 15); + let e: i16x4 = i16x4::new(8, 9, 9, 11); + let r: i16x4 = transmute(vext_p16::<2>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vextq_p16() { + let a: i16x8 = i16x8::new(0, 8, 8, 9, 8, 9, 9, 11); + let b: i16x8 = i16x8::new(9, 11, 14, 15, 16, 17, 18, 19); + let e: i16x8 = i16x8::new(8, 9, 9, 11, 9, 11, 14, 15); + let r: i16x8 = transmute(vextq_p16::<4>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vextq_s64() { + let a: i64x2 = i64x2::new(0, 8); + let b: i64x2 = i64x2::new(9, 11); + let e: i64x2 = i64x2::new(8, 9); + let r: i64x2 = transmute(vextq_s64::<1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vextq_u64() { + let a: u64x2 = u64x2::new(0, 8); + let b: u64x2 = u64x2::new(9, 11); + let e: u64x2 = u64x2::new(8, 9); + let r: u64x2 = transmute(vextq_u64::<1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vext_f32() { + let a: f32x2 = f32x2::new(0., 2.); + let b: f32x2 = f32x2::new(3., 4.); + let e: f32x2 = f32x2::new(2., 3.); + let r: f32x2 = transmute(vext_f32::<1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vextq_f32() { + let a: f32x4 = f32x4::new(0., 2., 2., 3.); + let b: f32x4 = f32x4::new(3., 4., 5., 6.); + let e: f32x4 = f32x4::new(2., 3., 3., 4.); + let r: f32x4 = transmute(vextq_f32::<2>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmla_s8() { + let a: i8x8 = i8x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let b: i8x8 = i8x8::new(2, 2, 2, 2, 2, 2, 2, 2); + let c: i8x8 = i8x8::new(3, 3, 3, 3, 3, 3, 3, 3); + let e: i8x8 = i8x8::new(6, 7, 8, 9, 10, 11, 12, 13); + let r: i8x8 = transmute(vmla_s8(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlaq_s8() { + let a: i8x16 = i8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let b: i8x16 = i8x16::new(2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2); + let c: i8x16 = i8x16::new(3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3); + let e: i8x16 = i8x16::new(6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21); + let r: i8x16 = transmute(vmlaq_s8(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmla_s16() { + let a: i16x4 = i16x4::new(0, 1, 2, 3); + let b: i16x4 = i16x4::new(2, 2, 2, 2); + let c: i16x4 = i16x4::new(3, 3, 3, 3); + let e: i16x4 = i16x4::new(6, 7, 8, 9); + let r: i16x4 = transmute(vmla_s16(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlaq_s16() { + let a: i16x8 = i16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let b: i16x8 = i16x8::new(2, 2, 2, 2, 2, 2, 2, 2); + let c: i16x8 = i16x8::new(3, 3, 3, 3, 3, 3, 3, 3); + let e: i16x8 = i16x8::new(6, 7, 8, 9, 10, 11, 12, 13); + let r: i16x8 = transmute(vmlaq_s16(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmla_s32() { + let a: i32x2 = i32x2::new(0, 1); + let b: i32x2 = i32x2::new(2, 2); + let c: i32x2 = i32x2::new(3, 3); + let e: i32x2 = i32x2::new(6, 7); + let r: i32x2 = transmute(vmla_s32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlaq_s32() { + let a: i32x4 = i32x4::new(0, 1, 2, 3); + let b: i32x4 = i32x4::new(2, 2, 2, 2); + let c: i32x4 = i32x4::new(3, 3, 3, 3); + let e: i32x4 = i32x4::new(6, 7, 8, 9); + let r: i32x4 = transmute(vmlaq_s32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmla_u8() { + let a: u8x8 = u8x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let b: u8x8 = u8x8::new(2, 2, 2, 2, 2, 2, 2, 2); + let c: u8x8 = u8x8::new(3, 3, 3, 3, 3, 3, 3, 3); + let e: u8x8 = u8x8::new(6, 7, 8, 9, 10, 11, 12, 13); + let r: u8x8 = transmute(vmla_u8(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlaq_u8() { + let a: u8x16 = u8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let b: u8x16 = u8x16::new(2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2); + let c: u8x16 = u8x16::new(3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3); + let e: u8x16 = u8x16::new(6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21); + let r: u8x16 = transmute(vmlaq_u8(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmla_u16() { + let a: u16x4 = u16x4::new(0, 1, 2, 3); + let b: u16x4 = u16x4::new(2, 2, 2, 2); + let c: u16x4 = u16x4::new(3, 3, 3, 3); + let e: u16x4 = u16x4::new(6, 7, 8, 9); + let r: u16x4 = transmute(vmla_u16(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlaq_u16() { + let a: u16x8 = u16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let b: u16x8 = u16x8::new(2, 2, 2, 2, 2, 2, 2, 2); + let c: u16x8 = u16x8::new(3, 3, 3, 3, 3, 3, 3, 3); + let e: u16x8 = u16x8::new(6, 7, 8, 9, 10, 11, 12, 13); + let r: u16x8 = transmute(vmlaq_u16(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmla_u32() { + let a: u32x2 = u32x2::new(0, 1); + let b: u32x2 = u32x2::new(2, 2); + let c: u32x2 = u32x2::new(3, 3); + let e: u32x2 = u32x2::new(6, 7); + let r: u32x2 = transmute(vmla_u32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlaq_u32() { + let a: u32x4 = u32x4::new(0, 1, 2, 3); + let b: u32x4 = u32x4::new(2, 2, 2, 2); + let c: u32x4 = u32x4::new(3, 3, 3, 3); + let e: u32x4 = u32x4::new(6, 7, 8, 9); + let r: u32x4 = transmute(vmlaq_u32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmla_f32() { + let a: f32x2 = f32x2::new(0., 1.); + let b: f32x2 = f32x2::new(2., 2.); + let c: f32x2 = f32x2::new(3., 3.); + let e: f32x2 = f32x2::new(6., 7.); + let r: f32x2 = transmute(vmla_f32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlaq_f32() { + let a: f32x4 = f32x4::new(0., 1., 2., 3.); + let b: f32x4 = f32x4::new(2., 2., 2., 2.); + let c: f32x4 = f32x4::new(3., 3., 3., 3.); + let e: f32x4 = f32x4::new(6., 7., 8., 9.); + let r: f32x4 = transmute(vmlaq_f32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmla_n_s16() { + let a: i16x4 = i16x4::new(0, 1, 2, 3); + let b: i16x4 = i16x4::new(2, 2, 2, 2); + let c: i16 = 3; + let e: i16x4 = i16x4::new(6, 7, 8, 9); + let r: i16x4 = transmute(vmla_n_s16(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlaq_n_s16() { + let a: i16x8 = i16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let b: i16x8 = i16x8::new(2, 2, 2, 2, 2, 2, 2, 2); + let c: i16 = 3; + let e: i16x8 = i16x8::new(6, 7, 8, 9, 10, 11, 12, 13); + let r: i16x8 = transmute(vmlaq_n_s16(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmla_n_s32() { + let a: i32x2 = i32x2::new(0, 1); + let b: i32x2 = i32x2::new(2, 2); + let c: i32 = 3; + let e: i32x2 = i32x2::new(6, 7); + let r: i32x2 = transmute(vmla_n_s32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlaq_n_s32() { + let a: i32x4 = i32x4::new(0, 1, 2, 3); + let b: i32x4 = i32x4::new(2, 2, 2, 2); + let c: i32 = 3; + let e: i32x4 = i32x4::new(6, 7, 8, 9); + let r: i32x4 = transmute(vmlaq_n_s32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmla_n_u16() { + let a: u16x4 = u16x4::new(0, 1, 2, 3); + let b: u16x4 = u16x4::new(2, 2, 2, 2); + let c: u16 = 3; + let e: u16x4 = u16x4::new(6, 7, 8, 9); + let r: u16x4 = transmute(vmla_n_u16(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlaq_n_u16() { + let a: u16x8 = u16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let b: u16x8 = u16x8::new(2, 2, 2, 2, 2, 2, 2, 2); + let c: u16 = 3; + let e: u16x8 = u16x8::new(6, 7, 8, 9, 10, 11, 12, 13); + let r: u16x8 = transmute(vmlaq_n_u16(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmla_n_u32() { + let a: u32x2 = u32x2::new(0, 1); + let b: u32x2 = u32x2::new(2, 2); + let c: u32 = 3; + let e: u32x2 = u32x2::new(6, 7); + let r: u32x2 = transmute(vmla_n_u32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlaq_n_u32() { + let a: u32x4 = u32x4::new(0, 1, 2, 3); + let b: u32x4 = u32x4::new(2, 2, 2, 2); + let c: u32 = 3; + let e: u32x4 = u32x4::new(6, 7, 8, 9); + let r: u32x4 = transmute(vmlaq_n_u32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmla_n_f32() { + let a: f32x2 = f32x2::new(0., 1.); + let b: f32x2 = f32x2::new(2., 2.); + let c: f32 = 3.; + let e: f32x2 = f32x2::new(6., 7.); + let r: f32x2 = transmute(vmla_n_f32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlaq_n_f32() { + let a: f32x4 = f32x4::new(0., 1., 2., 3.); + let b: f32x4 = f32x4::new(2., 2., 2., 2.); + let c: f32 = 3.; + let e: f32x4 = f32x4::new(6., 7., 8., 9.); + let r: f32x4 = transmute(vmlaq_n_f32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmla_lane_s16() { + let a: i16x4 = i16x4::new(0, 1, 2, 3); + let b: i16x4 = i16x4::new(2, 2, 2, 2); + let c: i16x4 = i16x4::new(0, 3, 0, 0); + let e: i16x4 = i16x4::new(6, 7, 8, 9); + let r: i16x4 = transmute(vmla_lane_s16::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmla_laneq_s16() { + let a: i16x4 = i16x4::new(0, 1, 2, 3); + let b: i16x4 = i16x4::new(2, 2, 2, 2); + let c: i16x8 = i16x8::new(0, 3, 0, 0, 0, 0, 0, 0); + let e: i16x4 = i16x4::new(6, 7, 8, 9); + let r: i16x4 = transmute(vmla_laneq_s16::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlaq_lane_s16() { + let a: i16x8 = i16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let b: i16x8 = i16x8::new(2, 2, 2, 2, 2, 2, 2, 2); + let c: i16x4 = i16x4::new(0, 3, 0, 0); + let e: i16x8 = i16x8::new(6, 7, 8, 9, 10, 11, 12, 13); + let r: i16x8 = transmute(vmlaq_lane_s16::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlaq_laneq_s16() { + let a: i16x8 = i16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let b: i16x8 = i16x8::new(2, 2, 2, 2, 2, 2, 2, 2); + let c: i16x8 = i16x8::new(0, 3, 0, 0, 0, 0, 0, 0); + let e: i16x8 = i16x8::new(6, 7, 8, 9, 10, 11, 12, 13); + let r: i16x8 = transmute(vmlaq_laneq_s16::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmla_lane_s32() { + let a: i32x2 = i32x2::new(0, 1); + let b: i32x2 = i32x2::new(2, 2); + let c: i32x2 = i32x2::new(0, 3); + let e: i32x2 = i32x2::new(6, 7); + let r: i32x2 = transmute(vmla_lane_s32::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmla_laneq_s32() { + let a: i32x2 = i32x2::new(0, 1); + let b: i32x2 = i32x2::new(2, 2); + let c: i32x4 = i32x4::new(0, 3, 0, 0); + let e: i32x2 = i32x2::new(6, 7); + let r: i32x2 = transmute(vmla_laneq_s32::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlaq_lane_s32() { + let a: i32x4 = i32x4::new(0, 1, 2, 3); + let b: i32x4 = i32x4::new(2, 2, 2, 2); + let c: i32x2 = i32x2::new(0, 3); + let e: i32x4 = i32x4::new(6, 7, 8, 9); + let r: i32x4 = transmute(vmlaq_lane_s32::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlaq_laneq_s32() { + let a: i32x4 = i32x4::new(0, 1, 2, 3); + let b: i32x4 = i32x4::new(2, 2, 2, 2); + let c: i32x4 = i32x4::new(0, 3, 0, 0); + let e: i32x4 = i32x4::new(6, 7, 8, 9); + let r: i32x4 = transmute(vmlaq_laneq_s32::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmla_lane_u16() { + let a: u16x4 = u16x4::new(0, 1, 2, 3); + let b: u16x4 = u16x4::new(2, 2, 2, 2); + let c: u16x4 = u16x4::new(0, 3, 0, 0); + let e: u16x4 = u16x4::new(6, 7, 8, 9); + let r: u16x4 = transmute(vmla_lane_u16::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmla_laneq_u16() { + let a: u16x4 = u16x4::new(0, 1, 2, 3); + let b: u16x4 = u16x4::new(2, 2, 2, 2); + let c: u16x8 = u16x8::new(0, 3, 0, 0, 0, 0, 0, 0); + let e: u16x4 = u16x4::new(6, 7, 8, 9); + let r: u16x4 = transmute(vmla_laneq_u16::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlaq_lane_u16() { + let a: u16x8 = u16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let b: u16x8 = u16x8::new(2, 2, 2, 2, 2, 2, 2, 2); + let c: u16x4 = u16x4::new(0, 3, 0, 0); + let e: u16x8 = u16x8::new(6, 7, 8, 9, 10, 11, 12, 13); + let r: u16x8 = transmute(vmlaq_lane_u16::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlaq_laneq_u16() { + let a: u16x8 = u16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let b: u16x8 = u16x8::new(2, 2, 2, 2, 2, 2, 2, 2); + let c: u16x8 = u16x8::new(0, 3, 0, 0, 0, 0, 0, 0); + let e: u16x8 = u16x8::new(6, 7, 8, 9, 10, 11, 12, 13); + let r: u16x8 = transmute(vmlaq_laneq_u16::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmla_lane_u32() { + let a: u32x2 = u32x2::new(0, 1); + let b: u32x2 = u32x2::new(2, 2); + let c: u32x2 = u32x2::new(0, 3); + let e: u32x2 = u32x2::new(6, 7); + let r: u32x2 = transmute(vmla_lane_u32::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmla_laneq_u32() { + let a: u32x2 = u32x2::new(0, 1); + let b: u32x2 = u32x2::new(2, 2); + let c: u32x4 = u32x4::new(0, 3, 0, 0); + let e: u32x2 = u32x2::new(6, 7); + let r: u32x2 = transmute(vmla_laneq_u32::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlaq_lane_u32() { + let a: u32x4 = u32x4::new(0, 1, 2, 3); + let b: u32x4 = u32x4::new(2, 2, 2, 2); + let c: u32x2 = u32x2::new(0, 3); + let e: u32x4 = u32x4::new(6, 7, 8, 9); + let r: u32x4 = transmute(vmlaq_lane_u32::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlaq_laneq_u32() { + let a: u32x4 = u32x4::new(0, 1, 2, 3); + let b: u32x4 = u32x4::new(2, 2, 2, 2); + let c: u32x4 = u32x4::new(0, 3, 0, 0); + let e: u32x4 = u32x4::new(6, 7, 8, 9); + let r: u32x4 = transmute(vmlaq_laneq_u32::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmla_lane_f32() { + let a: f32x2 = f32x2::new(0., 1.); + let b: f32x2 = f32x2::new(2., 2.); + let c: f32x2 = f32x2::new(0., 3.); + let e: f32x2 = f32x2::new(6., 7.); + let r: f32x2 = transmute(vmla_lane_f32::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmla_laneq_f32() { + let a: f32x2 = f32x2::new(0., 1.); + let b: f32x2 = f32x2::new(2., 2.); + let c: f32x4 = f32x4::new(0., 3., 0., 0.); + let e: f32x2 = f32x2::new(6., 7.); + let r: f32x2 = transmute(vmla_laneq_f32::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlaq_lane_f32() { + let a: f32x4 = f32x4::new(0., 1., 2., 3.); + let b: f32x4 = f32x4::new(2., 2., 2., 2.); + let c: f32x2 = f32x2::new(0., 3.); + let e: f32x4 = f32x4::new(6., 7., 8., 9.); + let r: f32x4 = transmute(vmlaq_lane_f32::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlaq_laneq_f32() { + let a: f32x4 = f32x4::new(0., 1., 2., 3.); + let b: f32x4 = f32x4::new(2., 2., 2., 2.); + let c: f32x4 = f32x4::new(0., 3., 0., 0.); + let e: f32x4 = f32x4::new(6., 7., 8., 9.); + let r: f32x4 = transmute(vmlaq_laneq_f32::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlal_s8() { + let a: i16x8 = i16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let b: i8x8 = i8x8::new(2, 2, 2, 2, 2, 2, 2, 2); + let c: i8x8 = i8x8::new(3, 3, 3, 3, 3, 3, 3, 3); + let e: i16x8 = i16x8::new(6, 7, 8, 9, 10, 11, 12, 13); + let r: i16x8 = transmute(vmlal_s8(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlal_s16() { + let a: i32x4 = i32x4::new(0, 1, 2, 3); + let b: i16x4 = i16x4::new(2, 2, 2, 2); + let c: i16x4 = i16x4::new(3, 3, 3, 3); + let e: i32x4 = i32x4::new(6, 7, 8, 9); + let r: i32x4 = transmute(vmlal_s16(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlal_s32() { + let a: i64x2 = i64x2::new(0, 1); + let b: i32x2 = i32x2::new(2, 2); + let c: i32x2 = i32x2::new(3, 3); + let e: i64x2 = i64x2::new(6, 7); + let r: i64x2 = transmute(vmlal_s32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlal_u8() { + let a: u16x8 = u16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let b: u8x8 = u8x8::new(2, 2, 2, 2, 2, 2, 2, 2); + let c: u8x8 = u8x8::new(3, 3, 3, 3, 3, 3, 3, 3); + let e: u16x8 = u16x8::new(6, 7, 8, 9, 10, 11, 12, 13); + let r: u16x8 = transmute(vmlal_u8(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlal_u16() { + let a: u32x4 = u32x4::new(0, 1, 2, 3); + let b: u16x4 = u16x4::new(2, 2, 2, 2); + let c: u16x4 = u16x4::new(3, 3, 3, 3); + let e: u32x4 = u32x4::new(6, 7, 8, 9); + let r: u32x4 = transmute(vmlal_u16(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlal_u32() { + let a: u64x2 = u64x2::new(0, 1); + let b: u32x2 = u32x2::new(2, 2); + let c: u32x2 = u32x2::new(3, 3); + let e: u64x2 = u64x2::new(6, 7); + let r: u64x2 = transmute(vmlal_u32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlal_n_s16() { + let a: i32x4 = i32x4::new(0, 1, 2, 3); + let b: i16x4 = i16x4::new(2, 2, 2, 2); + let c: i16 = 3; + let e: i32x4 = i32x4::new(6, 7, 8, 9); + let r: i32x4 = transmute(vmlal_n_s16(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlal_n_s32() { + let a: i64x2 = i64x2::new(0, 1); + let b: i32x2 = i32x2::new(2, 2); + let c: i32 = 3; + let e: i64x2 = i64x2::new(6, 7); + let r: i64x2 = transmute(vmlal_n_s32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlal_n_u16() { + let a: u32x4 = u32x4::new(0, 1, 2, 3); + let b: u16x4 = u16x4::new(2, 2, 2, 2); + let c: u16 = 3; + let e: u32x4 = u32x4::new(6, 7, 8, 9); + let r: u32x4 = transmute(vmlal_n_u16(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlal_n_u32() { + let a: u64x2 = u64x2::new(0, 1); + let b: u32x2 = u32x2::new(2, 2); + let c: u32 = 3; + let e: u64x2 = u64x2::new(6, 7); + let r: u64x2 = transmute(vmlal_n_u32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlal_lane_s16() { + let a: i32x4 = i32x4::new(0, 1, 2, 3); + let b: i16x4 = i16x4::new(2, 2, 2, 2); + let c: i16x4 = i16x4::new(0, 3, 0, 0); + let e: i32x4 = i32x4::new(6, 7, 8, 9); + let r: i32x4 = transmute(vmlal_lane_s16::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlal_laneq_s16() { + let a: i32x4 = i32x4::new(0, 1, 2, 3); + let b: i16x4 = i16x4::new(2, 2, 2, 2); + let c: i16x8 = i16x8::new(0, 3, 0, 0, 0, 0, 0, 0); + let e: i32x4 = i32x4::new(6, 7, 8, 9); + let r: i32x4 = transmute(vmlal_laneq_s16::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlal_lane_s32() { + let a: i64x2 = i64x2::new(0, 1); + let b: i32x2 = i32x2::new(2, 2); + let c: i32x2 = i32x2::new(0, 3); + let e: i64x2 = i64x2::new(6, 7); + let r: i64x2 = transmute(vmlal_lane_s32::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlal_laneq_s32() { + let a: i64x2 = i64x2::new(0, 1); + let b: i32x2 = i32x2::new(2, 2); + let c: i32x4 = i32x4::new(0, 3, 0, 0); + let e: i64x2 = i64x2::new(6, 7); + let r: i64x2 = transmute(vmlal_laneq_s32::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlal_lane_u16() { + let a: u32x4 = u32x4::new(0, 1, 2, 3); + let b: u16x4 = u16x4::new(2, 2, 2, 2); + let c: u16x4 = u16x4::new(0, 3, 0, 0); + let e: u32x4 = u32x4::new(6, 7, 8, 9); + let r: u32x4 = transmute(vmlal_lane_u16::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlal_laneq_u16() { + let a: u32x4 = u32x4::new(0, 1, 2, 3); + let b: u16x4 = u16x4::new(2, 2, 2, 2); + let c: u16x8 = u16x8::new(0, 3, 0, 0, 0, 0, 0, 0); + let e: u32x4 = u32x4::new(6, 7, 8, 9); + let r: u32x4 = transmute(vmlal_laneq_u16::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlal_lane_u32() { + let a: u64x2 = u64x2::new(0, 1); + let b: u32x2 = u32x2::new(2, 2); + let c: u32x2 = u32x2::new(0, 3); + let e: u64x2 = u64x2::new(6, 7); + let r: u64x2 = transmute(vmlal_lane_u32::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlal_laneq_u32() { + let a: u64x2 = u64x2::new(0, 1); + let b: u32x2 = u32x2::new(2, 2); + let c: u32x4 = u32x4::new(0, 3, 0, 0); + let e: u64x2 = u64x2::new(6, 7); + let r: u64x2 = transmute(vmlal_laneq_u32::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmls_s8() { + let a: i8x8 = i8x8::new(6, 7, 8, 9, 10, 11, 12, 13); + let b: i8x8 = i8x8::new(2, 2, 2, 2, 2, 2, 2, 2); + let c: i8x8 = i8x8::new(3, 3, 3, 3, 3, 3, 3, 3); + let e: i8x8 = i8x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let r: i8x8 = transmute(vmls_s8(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlsq_s8() { + let a: i8x16 = i8x16::new(6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21); + let b: i8x16 = i8x16::new(2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2); + let c: i8x16 = i8x16::new(3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3); + let e: i8x16 = i8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let r: i8x16 = transmute(vmlsq_s8(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmls_s16() { + let a: i16x4 = i16x4::new(6, 7, 8, 9); + let b: i16x4 = i16x4::new(2, 2, 2, 2); + let c: i16x4 = i16x4::new(3, 3, 3, 3); + let e: i16x4 = i16x4::new(0, 1, 2, 3); + let r: i16x4 = transmute(vmls_s16(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlsq_s16() { + let a: i16x8 = i16x8::new(6, 7, 8, 9, 10, 11, 12, 13); + let b: i16x8 = i16x8::new(2, 2, 2, 2, 2, 2, 2, 2); + let c: i16x8 = i16x8::new(3, 3, 3, 3, 3, 3, 3, 3); + let e: i16x8 = i16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let r: i16x8 = transmute(vmlsq_s16(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmls_s32() { + let a: i32x2 = i32x2::new(6, 7); + let b: i32x2 = i32x2::new(2, 2); + let c: i32x2 = i32x2::new(3, 3); + let e: i32x2 = i32x2::new(0, 1); + let r: i32x2 = transmute(vmls_s32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlsq_s32() { + let a: i32x4 = i32x4::new(6, 7, 8, 9); + let b: i32x4 = i32x4::new(2, 2, 2, 2); + let c: i32x4 = i32x4::new(3, 3, 3, 3); + let e: i32x4 = i32x4::new(0, 1, 2, 3); + let r: i32x4 = transmute(vmlsq_s32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmls_u8() { + let a: u8x8 = u8x8::new(6, 7, 8, 9, 10, 11, 12, 13); + let b: u8x8 = u8x8::new(2, 2, 2, 2, 2, 2, 2, 2); + let c: u8x8 = u8x8::new(3, 3, 3, 3, 3, 3, 3, 3); + let e: u8x8 = u8x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let r: u8x8 = transmute(vmls_u8(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlsq_u8() { + let a: u8x16 = u8x16::new(6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21); + let b: u8x16 = u8x16::new(2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2); + let c: u8x16 = u8x16::new(3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3); + let e: u8x16 = u8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let r: u8x16 = transmute(vmlsq_u8(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmls_u16() { + let a: u16x4 = u16x4::new(6, 7, 8, 9); + let b: u16x4 = u16x4::new(2, 2, 2, 2); + let c: u16x4 = u16x4::new(3, 3, 3, 3); + let e: u16x4 = u16x4::new(0, 1, 2, 3); + let r: u16x4 = transmute(vmls_u16(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlsq_u16() { + let a: u16x8 = u16x8::new(6, 7, 8, 9, 10, 11, 12, 13); + let b: u16x8 = u16x8::new(2, 2, 2, 2, 2, 2, 2, 2); + let c: u16x8 = u16x8::new(3, 3, 3, 3, 3, 3, 3, 3); + let e: u16x8 = u16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let r: u16x8 = transmute(vmlsq_u16(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmls_u32() { + let a: u32x2 = u32x2::new(6, 7); + let b: u32x2 = u32x2::new(2, 2); + let c: u32x2 = u32x2::new(3, 3); + let e: u32x2 = u32x2::new(0, 1); + let r: u32x2 = transmute(vmls_u32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlsq_u32() { + let a: u32x4 = u32x4::new(6, 7, 8, 9); + let b: u32x4 = u32x4::new(2, 2, 2, 2); + let c: u32x4 = u32x4::new(3, 3, 3, 3); + let e: u32x4 = u32x4::new(0, 1, 2, 3); + let r: u32x4 = transmute(vmlsq_u32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmls_f32() { + let a: f32x2 = f32x2::new(6., 7.); + let b: f32x2 = f32x2::new(2., 2.); + let c: f32x2 = f32x2::new(3., 3.); + let e: f32x2 = f32x2::new(0., 1.); + let r: f32x2 = transmute(vmls_f32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlsq_f32() { + let a: f32x4 = f32x4::new(6., 7., 8., 9.); + let b: f32x4 = f32x4::new(2., 2., 2., 2.); + let c: f32x4 = f32x4::new(3., 3., 3., 3.); + let e: f32x4 = f32x4::new(0., 1., 2., 3.); + let r: f32x4 = transmute(vmlsq_f32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmls_n_s16() { + let a: i16x4 = i16x4::new(6, 7, 8, 9); + let b: i16x4 = i16x4::new(2, 2, 2, 2); + let c: i16 = 3; + let e: i16x4 = i16x4::new(0, 1, 2, 3); + let r: i16x4 = transmute(vmls_n_s16(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlsq_n_s16() { + let a: i16x8 = i16x8::new(6, 7, 8, 9, 10, 11, 12, 13); + let b: i16x8 = i16x8::new(2, 2, 2, 2, 2, 2, 2, 2); + let c: i16 = 3; + let e: i16x8 = i16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let r: i16x8 = transmute(vmlsq_n_s16(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmls_n_s32() { + let a: i32x2 = i32x2::new(6, 7); + let b: i32x2 = i32x2::new(2, 2); + let c: i32 = 3; + let e: i32x2 = i32x2::new(0, 1); + let r: i32x2 = transmute(vmls_n_s32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlsq_n_s32() { + let a: i32x4 = i32x4::new(6, 7, 8, 9); + let b: i32x4 = i32x4::new(2, 2, 2, 2); + let c: i32 = 3; + let e: i32x4 = i32x4::new(0, 1, 2, 3); + let r: i32x4 = transmute(vmlsq_n_s32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmls_n_u16() { + let a: u16x4 = u16x4::new(6, 7, 8, 9); + let b: u16x4 = u16x4::new(2, 2, 2, 2); + let c: u16 = 3; + let e: u16x4 = u16x4::new(0, 1, 2, 3); + let r: u16x4 = transmute(vmls_n_u16(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlsq_n_u16() { + let a: u16x8 = u16x8::new(6, 7, 8, 9, 10, 11, 12, 13); + let b: u16x8 = u16x8::new(2, 2, 2, 2, 2, 2, 2, 2); + let c: u16 = 3; + let e: u16x8 = u16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let r: u16x8 = transmute(vmlsq_n_u16(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmls_n_u32() { + let a: u32x2 = u32x2::new(6, 7); + let b: u32x2 = u32x2::new(2, 2); + let c: u32 = 3; + let e: u32x2 = u32x2::new(0, 1); + let r: u32x2 = transmute(vmls_n_u32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlsq_n_u32() { + let a: u32x4 = u32x4::new(6, 7, 8, 9); + let b: u32x4 = u32x4::new(2, 2, 2, 2); + let c: u32 = 3; + let e: u32x4 = u32x4::new(0, 1, 2, 3); + let r: u32x4 = transmute(vmlsq_n_u32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmls_n_f32() { + let a: f32x2 = f32x2::new(6., 7.); + let b: f32x2 = f32x2::new(2., 2.); + let c: f32 = 3.; + let e: f32x2 = f32x2::new(0., 1.); + let r: f32x2 = transmute(vmls_n_f32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlsq_n_f32() { + let a: f32x4 = f32x4::new(6., 7., 8., 9.); + let b: f32x4 = f32x4::new(2., 2., 2., 2.); + let c: f32 = 3.; + let e: f32x4 = f32x4::new(0., 1., 2., 3.); + let r: f32x4 = transmute(vmlsq_n_f32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmls_lane_s16() { + let a: i16x4 = i16x4::new(6, 7, 8, 9); + let b: i16x4 = i16x4::new(2, 2, 2, 2); + let c: i16x4 = i16x4::new(0, 3, 0, 0); + let e: i16x4 = i16x4::new(0, 1, 2, 3); + let r: i16x4 = transmute(vmls_lane_s16::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmls_laneq_s16() { + let a: i16x4 = i16x4::new(6, 7, 8, 9); + let b: i16x4 = i16x4::new(2, 2, 2, 2); + let c: i16x8 = i16x8::new(0, 3, 0, 0, 0, 0, 0, 0); + let e: i16x4 = i16x4::new(0, 1, 2, 3); + let r: i16x4 = transmute(vmls_laneq_s16::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlsq_lane_s16() { + let a: i16x8 = i16x8::new(6, 7, 8, 9, 10, 11, 12, 13); + let b: i16x8 = i16x8::new(2, 2, 2, 2, 2, 2, 2, 2); + let c: i16x4 = i16x4::new(0, 3, 0, 0); + let e: i16x8 = i16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let r: i16x8 = transmute(vmlsq_lane_s16::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlsq_laneq_s16() { + let a: i16x8 = i16x8::new(6, 7, 8, 9, 10, 11, 12, 13); + let b: i16x8 = i16x8::new(2, 2, 2, 2, 2, 2, 2, 2); + let c: i16x8 = i16x8::new(0, 3, 0, 0, 0, 0, 0, 0); + let e: i16x8 = i16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let r: i16x8 = transmute(vmlsq_laneq_s16::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmls_lane_s32() { + let a: i32x2 = i32x2::new(6, 7); + let b: i32x2 = i32x2::new(2, 2); + let c: i32x2 = i32x2::new(0, 3); + let e: i32x2 = i32x2::new(0, 1); + let r: i32x2 = transmute(vmls_lane_s32::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmls_laneq_s32() { + let a: i32x2 = i32x2::new(6, 7); + let b: i32x2 = i32x2::new(2, 2); + let c: i32x4 = i32x4::new(0, 3, 0, 0); + let e: i32x2 = i32x2::new(0, 1); + let r: i32x2 = transmute(vmls_laneq_s32::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlsq_lane_s32() { + let a: i32x4 = i32x4::new(6, 7, 8, 9); + let b: i32x4 = i32x4::new(2, 2, 2, 2); + let c: i32x2 = i32x2::new(0, 3); + let e: i32x4 = i32x4::new(0, 1, 2, 3); + let r: i32x4 = transmute(vmlsq_lane_s32::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlsq_laneq_s32() { + let a: i32x4 = i32x4::new(6, 7, 8, 9); + let b: i32x4 = i32x4::new(2, 2, 2, 2); + let c: i32x4 = i32x4::new(0, 3, 0, 0); + let e: i32x4 = i32x4::new(0, 1, 2, 3); + let r: i32x4 = transmute(vmlsq_laneq_s32::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmls_lane_u16() { + let a: u16x4 = u16x4::new(6, 7, 8, 9); + let b: u16x4 = u16x4::new(2, 2, 2, 2); + let c: u16x4 = u16x4::new(0, 3, 0, 0); + let e: u16x4 = u16x4::new(0, 1, 2, 3); + let r: u16x4 = transmute(vmls_lane_u16::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmls_laneq_u16() { + let a: u16x4 = u16x4::new(6, 7, 8, 9); + let b: u16x4 = u16x4::new(2, 2, 2, 2); + let c: u16x8 = u16x8::new(0, 3, 0, 0, 0, 0, 0, 0); + let e: u16x4 = u16x4::new(0, 1, 2, 3); + let r: u16x4 = transmute(vmls_laneq_u16::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlsq_lane_u16() { + let a: u16x8 = u16x8::new(6, 7, 8, 9, 10, 11, 12, 13); + let b: u16x8 = u16x8::new(2, 2, 2, 2, 2, 2, 2, 2); + let c: u16x4 = u16x4::new(0, 3, 0, 0); + let e: u16x8 = u16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let r: u16x8 = transmute(vmlsq_lane_u16::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlsq_laneq_u16() { + let a: u16x8 = u16x8::new(6, 7, 8, 9, 10, 11, 12, 13); + let b: u16x8 = u16x8::new(2, 2, 2, 2, 2, 2, 2, 2); + let c: u16x8 = u16x8::new(0, 3, 0, 0, 0, 0, 0, 0); + let e: u16x8 = u16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let r: u16x8 = transmute(vmlsq_laneq_u16::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmls_lane_u32() { + let a: u32x2 = u32x2::new(6, 7); + let b: u32x2 = u32x2::new(2, 2); + let c: u32x2 = u32x2::new(0, 3); + let e: u32x2 = u32x2::new(0, 1); + let r: u32x2 = transmute(vmls_lane_u32::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmls_laneq_u32() { + let a: u32x2 = u32x2::new(6, 7); + let b: u32x2 = u32x2::new(2, 2); + let c: u32x4 = u32x4::new(0, 3, 0, 0); + let e: u32x2 = u32x2::new(0, 1); + let r: u32x2 = transmute(vmls_laneq_u32::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlsq_lane_u32() { + let a: u32x4 = u32x4::new(6, 7, 8, 9); + let b: u32x4 = u32x4::new(2, 2, 2, 2); + let c: u32x2 = u32x2::new(0, 3); + let e: u32x4 = u32x4::new(0, 1, 2, 3); + let r: u32x4 = transmute(vmlsq_lane_u32::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlsq_laneq_u32() { + let a: u32x4 = u32x4::new(6, 7, 8, 9); + let b: u32x4 = u32x4::new(2, 2, 2, 2); + let c: u32x4 = u32x4::new(0, 3, 0, 0); + let e: u32x4 = u32x4::new(0, 1, 2, 3); + let r: u32x4 = transmute(vmlsq_laneq_u32::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmls_lane_f32() { + let a: f32x2 = f32x2::new(6., 7.); + let b: f32x2 = f32x2::new(2., 2.); + let c: f32x2 = f32x2::new(0., 3.); + let e: f32x2 = f32x2::new(0., 1.); + let r: f32x2 = transmute(vmls_lane_f32::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmls_laneq_f32() { + let a: f32x2 = f32x2::new(6., 7.); + let b: f32x2 = f32x2::new(2., 2.); + let c: f32x4 = f32x4::new(0., 3., 0., 0.); + let e: f32x2 = f32x2::new(0., 1.); + let r: f32x2 = transmute(vmls_laneq_f32::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlsq_lane_f32() { + let a: f32x4 = f32x4::new(6., 7., 8., 9.); + let b: f32x4 = f32x4::new(2., 2., 2., 2.); + let c: f32x2 = f32x2::new(0., 3.); + let e: f32x4 = f32x4::new(0., 1., 2., 3.); + let r: f32x4 = transmute(vmlsq_lane_f32::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlsq_laneq_f32() { + let a: f32x4 = f32x4::new(6., 7., 8., 9.); + let b: f32x4 = f32x4::new(2., 2., 2., 2.); + let c: f32x4 = f32x4::new(0., 3., 0., 0.); + let e: f32x4 = f32x4::new(0., 1., 2., 3.); + let r: f32x4 = transmute(vmlsq_laneq_f32::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlsl_s8() { + let a: i16x8 = i16x8::new(6, 7, 8, 9, 10, 11, 12, 13); + let b: i8x8 = i8x8::new(2, 2, 2, 2, 2, 2, 2, 2); + let c: i8x8 = i8x8::new(3, 3, 3, 3, 3, 3, 3, 3); + let e: i16x8 = i16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let r: i16x8 = transmute(vmlsl_s8(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlsl_s16() { + let a: i32x4 = i32x4::new(6, 7, 8, 9); + let b: i16x4 = i16x4::new(2, 2, 2, 2); + let c: i16x4 = i16x4::new(3, 3, 3, 3); + let e: i32x4 = i32x4::new(0, 1, 2, 3); + let r: i32x4 = transmute(vmlsl_s16(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlsl_s32() { + let a: i64x2 = i64x2::new(6, 7); + let b: i32x2 = i32x2::new(2, 2); + let c: i32x2 = i32x2::new(3, 3); + let e: i64x2 = i64x2::new(0, 1); + let r: i64x2 = transmute(vmlsl_s32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlsl_u8() { + let a: u16x8 = u16x8::new(6, 7, 8, 9, 10, 11, 12, 13); + let b: u8x8 = u8x8::new(2, 2, 2, 2, 2, 2, 2, 2); + let c: u8x8 = u8x8::new(3, 3, 3, 3, 3, 3, 3, 3); + let e: u16x8 = u16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let r: u16x8 = transmute(vmlsl_u8(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlsl_u16() { + let a: u32x4 = u32x4::new(6, 7, 8, 9); + let b: u16x4 = u16x4::new(2, 2, 2, 2); + let c: u16x4 = u16x4::new(3, 3, 3, 3); + let e: u32x4 = u32x4::new(0, 1, 2, 3); + let r: u32x4 = transmute(vmlsl_u16(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlsl_u32() { + let a: u64x2 = u64x2::new(6, 7); + let b: u32x2 = u32x2::new(2, 2); + let c: u32x2 = u32x2::new(3, 3); + let e: u64x2 = u64x2::new(0, 1); + let r: u64x2 = transmute(vmlsl_u32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlsl_n_s16() { + let a: i32x4 = i32x4::new(6, 7, 8, 9); + let b: i16x4 = i16x4::new(2, 2, 2, 2); + let c: i16 = 3; + let e: i32x4 = i32x4::new(0, 1, 2, 3); + let r: i32x4 = transmute(vmlsl_n_s16(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlsl_n_s32() { + let a: i64x2 = i64x2::new(6, 7); + let b: i32x2 = i32x2::new(2, 2); + let c: i32 = 3; + let e: i64x2 = i64x2::new(0, 1); + let r: i64x2 = transmute(vmlsl_n_s32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlsl_n_u16() { + let a: u32x4 = u32x4::new(6, 7, 8, 9); + let b: u16x4 = u16x4::new(2, 2, 2, 2); + let c: u16 = 3; + let e: u32x4 = u32x4::new(0, 1, 2, 3); + let r: u32x4 = transmute(vmlsl_n_u16(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlsl_n_u32() { + let a: u64x2 = u64x2::new(6, 7); + let b: u32x2 = u32x2::new(2, 2); + let c: u32 = 3; + let e: u64x2 = u64x2::new(0, 1); + let r: u64x2 = transmute(vmlsl_n_u32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlsl_lane_s16() { + let a: i32x4 = i32x4::new(6, 7, 8, 9); + let b: i16x4 = i16x4::new(2, 2, 2, 2); + let c: i16x4 = i16x4::new(0, 3, 0, 0); + let e: i32x4 = i32x4::new(0, 1, 2, 3); + let r: i32x4 = transmute(vmlsl_lane_s16::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlsl_laneq_s16() { + let a: i32x4 = i32x4::new(6, 7, 8, 9); + let b: i16x4 = i16x4::new(2, 2, 2, 2); + let c: i16x8 = i16x8::new(0, 3, 0, 0, 0, 0, 0, 0); + let e: i32x4 = i32x4::new(0, 1, 2, 3); + let r: i32x4 = transmute(vmlsl_laneq_s16::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlsl_lane_s32() { + let a: i64x2 = i64x2::new(6, 7); + let b: i32x2 = i32x2::new(2, 2); + let c: i32x2 = i32x2::new(0, 3); + let e: i64x2 = i64x2::new(0, 1); + let r: i64x2 = transmute(vmlsl_lane_s32::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlsl_laneq_s32() { + let a: i64x2 = i64x2::new(6, 7); + let b: i32x2 = i32x2::new(2, 2); + let c: i32x4 = i32x4::new(0, 3, 0, 0); + let e: i64x2 = i64x2::new(0, 1); + let r: i64x2 = transmute(vmlsl_laneq_s32::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlsl_lane_u16() { + let a: u32x4 = u32x4::new(6, 7, 8, 9); + let b: u16x4 = u16x4::new(2, 2, 2, 2); + let c: u16x4 = u16x4::new(0, 3, 0, 0); + let e: u32x4 = u32x4::new(0, 1, 2, 3); + let r: u32x4 = transmute(vmlsl_lane_u16::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlsl_laneq_u16() { + let a: u32x4 = u32x4::new(6, 7, 8, 9); + let b: u16x4 = u16x4::new(2, 2, 2, 2); + let c: u16x8 = u16x8::new(0, 3, 0, 0, 0, 0, 0, 0); + let e: u32x4 = u32x4::new(0, 1, 2, 3); + let r: u32x4 = transmute(vmlsl_laneq_u16::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlsl_lane_u32() { + let a: u64x2 = u64x2::new(6, 7); + let b: u32x2 = u32x2::new(2, 2); + let c: u32x2 = u32x2::new(0, 3); + let e: u64x2 = u64x2::new(0, 1); + let r: u64x2 = transmute(vmlsl_lane_u32::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmlsl_laneq_u32() { + let a: u64x2 = u64x2::new(6, 7); + let b: u32x2 = u32x2::new(2, 2); + let c: u32x4 = u32x4::new(0, 3, 0, 0); + let e: u64x2 = u64x2::new(0, 1); + let r: u64x2 = transmute(vmlsl_laneq_u32::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vneg_s8() { + let a: i8x8 = i8x8::new(0, 1, -1, 2, -2, 3, -3, 4); + let e: i8x8 = i8x8::new(0, -1, 1, -2, 2, -3, 3, -4); + let r: i8x8 = transmute(vneg_s8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vnegq_s8() { + let a: i8x16 = i8x16::new(0, 1, -1, 2, -2, 3, -3, 4, -4, 5, -5, 6, -6, 7, -7, 8); + let e: i8x16 = i8x16::new(0, -1, 1, -2, 2, -3, 3, -4, 4, -5, 5, -6, 6, -7, 7, -8); + let r: i8x16 = transmute(vnegq_s8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vneg_s16() { + let a: i16x4 = i16x4::new(0, 1, -1, 2); + let e: i16x4 = i16x4::new(0, -1, 1, -2); + let r: i16x4 = transmute(vneg_s16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vnegq_s16() { + let a: i16x8 = i16x8::new(0, 1, -1, 2, -2, 3, -3, 4); + let e: i16x8 = i16x8::new(0, -1, 1, -2, 2, -3, 3, -4); + let r: i16x8 = transmute(vnegq_s16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vneg_s32() { + let a: i32x2 = i32x2::new(0, 1); + let e: i32x2 = i32x2::new(0, -1); + let r: i32x2 = transmute(vneg_s32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vnegq_s32() { + let a: i32x4 = i32x4::new(0, 1, -1, 2); + let e: i32x4 = i32x4::new(0, -1, 1, -2); + let r: i32x4 = transmute(vnegq_s32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vneg_f32() { + let a: f32x2 = f32x2::new(0., 1.); + let e: f32x2 = f32x2::new(0., -1.); + let r: f32x2 = transmute(vneg_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vnegq_f32() { + let a: f32x4 = f32x4::new(0., 1., -1., 2.); + let e: f32x4 = f32x4::new(0., -1., 1., -2.); + let r: f32x4 = transmute(vnegq_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqneg_s8() { + let a: i8x8 = i8x8::new(-128, 0, 1, -1, 2, -2, 3, -3); + let e: i8x8 = i8x8::new(0x7F, 0, -1, 1, -2, 2, -3, 3); + let r: i8x8 = transmute(vqneg_s8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqnegq_s8() { + let a: i8x16 = i8x16::new(-128, 0, 1, -1, 2, -2, 3, -3, 4, -4, 5, -5, 6, -6, 7, -7); + let e: i8x16 = i8x16::new(0x7F, 0, -1, 1, -2, 2, -3, 3, -4, 4, -5, 5, -6, 6, -7, 7); + let r: i8x16 = transmute(vqnegq_s8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqneg_s16() { + let a: i16x4 = i16x4::new(-32768, 0, 1, -1); + let e: i16x4 = i16x4::new(0x7F_FF, 0, -1, 1); + let r: i16x4 = transmute(vqneg_s16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqnegq_s16() { + let a: i16x8 = i16x8::new(-32768, 0, 1, -1, 2, -2, 3, -3); + let e: i16x8 = i16x8::new(0x7F_FF, 0, -1, 1, -2, 2, -3, 3); + let r: i16x8 = transmute(vqnegq_s16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqneg_s32() { + let a: i32x2 = i32x2::new(-2147483648, 0); + let e: i32x2 = i32x2::new(0x7F_FF_FF_FF, 0); + let r: i32x2 = transmute(vqneg_s32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqnegq_s32() { + let a: i32x4 = i32x4::new(-2147483648, 0, 1, -1); + let e: i32x4 = i32x4::new(0x7F_FF_FF_FF, 0, -1, 1); + let r: i32x4 = transmute(vqnegq_s32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqsub_u8() { + let a: u8x8 = u8x8::new(42, 42, 42, 42, 42, 42, 42, 42); + let b: u8x8 = u8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let e: u8x8 = u8x8::new(41, 40, 39, 38, 37, 36, 35, 34); + let r: u8x8 = transmute(vqsub_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqsubq_u8() { + let a: u8x16 = u8x16::new(42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42); + let b: u8x16 = u8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let e: u8x16 = u8x16::new(41, 40, 39, 38, 37, 36, 35, 34, 33, 32, 31, 30, 29, 28, 27, 26); + let r: u8x16 = transmute(vqsubq_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqsub_u16() { + let a: u16x4 = u16x4::new(42, 42, 42, 42); + let b: u16x4 = u16x4::new(1, 2, 3, 4); + let e: u16x4 = u16x4::new(41, 40, 39, 38); + let r: u16x4 = transmute(vqsub_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqsubq_u16() { + let a: u16x8 = u16x8::new(42, 42, 42, 42, 42, 42, 42, 42); + let b: u16x8 = u16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let e: u16x8 = u16x8::new(41, 40, 39, 38, 37, 36, 35, 34); + let r: u16x8 = transmute(vqsubq_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqsub_u32() { + let a: u32x2 = u32x2::new(42, 42); + let b: u32x2 = u32x2::new(1, 2); + let e: u32x2 = u32x2::new(41, 40); + let r: u32x2 = transmute(vqsub_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqsubq_u32() { + let a: u32x4 = u32x4::new(42, 42, 42, 42); + let b: u32x4 = u32x4::new(1, 2, 3, 4); + let e: u32x4 = u32x4::new(41, 40, 39, 38); + let r: u32x4 = transmute(vqsubq_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqsub_u64() { + let a: u64x1 = u64x1::new(42); + let b: u64x1 = u64x1::new(1); + let e: u64x1 = u64x1::new(41); + let r: u64x1 = transmute(vqsub_u64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqsubq_u64() { + let a: u64x2 = u64x2::new(42, 42); + let b: u64x2 = u64x2::new(1, 2); + let e: u64x2 = u64x2::new(41, 40); + let r: u64x2 = transmute(vqsubq_u64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqsub_s8() { + let a: i8x8 = i8x8::new(42, 42, 42, 42, 42, 42, 42, 42); + let b: i8x8 = i8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let e: i8x8 = i8x8::new(41, 40, 39, 38, 37, 36, 35, 34); + let r: i8x8 = transmute(vqsub_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqsubq_s8() { + let a: i8x16 = i8x16::new(42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42); + let b: i8x16 = i8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let e: i8x16 = i8x16::new(41, 40, 39, 38, 37, 36, 35, 34, 33, 32, 31, 30, 29, 28, 27, 26); + let r: i8x16 = transmute(vqsubq_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqsub_s16() { + let a: i16x4 = i16x4::new(42, 42, 42, 42); + let b: i16x4 = i16x4::new(1, 2, 3, 4); + let e: i16x4 = i16x4::new(41, 40, 39, 38); + let r: i16x4 = transmute(vqsub_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqsubq_s16() { + let a: i16x8 = i16x8::new(42, 42, 42, 42, 42, 42, 42, 42); + let b: i16x8 = i16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let e: i16x8 = i16x8::new(41, 40, 39, 38, 37, 36, 35, 34); + let r: i16x8 = transmute(vqsubq_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqsub_s32() { + let a: i32x2 = i32x2::new(42, 42); + let b: i32x2 = i32x2::new(1, 2); + let e: i32x2 = i32x2::new(41, 40); + let r: i32x2 = transmute(vqsub_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqsubq_s32() { + let a: i32x4 = i32x4::new(42, 42, 42, 42); + let b: i32x4 = i32x4::new(1, 2, 3, 4); + let e: i32x4 = i32x4::new(41, 40, 39, 38); + let r: i32x4 = transmute(vqsubq_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqsub_s64() { + let a: i64x1 = i64x1::new(42); + let b: i64x1 = i64x1::new(1); + let e: i64x1 = i64x1::new(41); + let r: i64x1 = transmute(vqsub_s64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqsubq_s64() { + let a: i64x2 = i64x2::new(42, 42); + let b: i64x2 = i64x2::new(1, 2); + let e: i64x2 = i64x2::new(41, 40); + let r: i64x2 = transmute(vqsubq_s64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vhadd_u8() { + let a: u8x8 = u8x8::new(42, 42, 42, 42, 42, 42, 42, 42); + let b: u8x8 = u8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let e: u8x8 = u8x8::new(21, 22, 22, 23, 23, 24, 24, 25); + let r: u8x8 = transmute(vhadd_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vhaddq_u8() { + let a: u8x16 = u8x16::new(42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42); + let b: u8x16 = u8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let e: u8x16 = u8x16::new(21, 22, 22, 23, 23, 24, 24, 25, 25, 26, 26, 27, 27, 28, 28, 29); + let r: u8x16 = transmute(vhaddq_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vhadd_u16() { + let a: u16x4 = u16x4::new(42, 42, 42, 42); + let b: u16x4 = u16x4::new(1, 2, 3, 4); + let e: u16x4 = u16x4::new(21, 22, 22, 23); + let r: u16x4 = transmute(vhadd_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vhaddq_u16() { + let a: u16x8 = u16x8::new(42, 42, 42, 42, 42, 42, 42, 42); + let b: u16x8 = u16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let e: u16x8 = u16x8::new(21, 22, 22, 23, 23, 24, 24, 25); + let r: u16x8 = transmute(vhaddq_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vhadd_u32() { + let a: u32x2 = u32x2::new(42, 42); + let b: u32x2 = u32x2::new(1, 2); + let e: u32x2 = u32x2::new(21, 22); + let r: u32x2 = transmute(vhadd_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vhaddq_u32() { + let a: u32x4 = u32x4::new(42, 42, 42, 42); + let b: u32x4 = u32x4::new(1, 2, 3, 4); + let e: u32x4 = u32x4::new(21, 22, 22, 23); + let r: u32x4 = transmute(vhaddq_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vhadd_s8() { + let a: i8x8 = i8x8::new(42, 42, 42, 42, 42, 42, 42, 42); + let b: i8x8 = i8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let e: i8x8 = i8x8::new(21, 22, 22, 23, 23, 24, 24, 25); + let r: i8x8 = transmute(vhadd_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vhaddq_s8() { + let a: i8x16 = i8x16::new(42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42); + let b: i8x16 = i8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let e: i8x16 = i8x16::new(21, 22, 22, 23, 23, 24, 24, 25, 25, 26, 26, 27, 27, 28, 28, 29); + let r: i8x16 = transmute(vhaddq_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vhadd_s16() { + let a: i16x4 = i16x4::new(42, 42, 42, 42); + let b: i16x4 = i16x4::new(1, 2, 3, 4); + let e: i16x4 = i16x4::new(21, 22, 22, 23); + let r: i16x4 = transmute(vhadd_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vhaddq_s16() { + let a: i16x8 = i16x8::new(42, 42, 42, 42, 42, 42, 42, 42); + let b: i16x8 = i16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let e: i16x8 = i16x8::new(21, 22, 22, 23, 23, 24, 24, 25); + let r: i16x8 = transmute(vhaddq_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vhadd_s32() { + let a: i32x2 = i32x2::new(42, 42); + let b: i32x2 = i32x2::new(1, 2); + let e: i32x2 = i32x2::new(21, 22); + let r: i32x2 = transmute(vhadd_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vhaddq_s32() { + let a: i32x4 = i32x4::new(42, 42, 42, 42); + let b: i32x4 = i32x4::new(1, 2, 3, 4); + let e: i32x4 = i32x4::new(21, 22, 22, 23); + let r: i32x4 = transmute(vhaddq_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrhadd_u8() { + let a: u8x8 = u8x8::new(42, 42, 42, 42, 42, 42, 42, 42); + let b: u8x8 = u8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let e: u8x8 = u8x8::new(22, 22, 23, 23, 24, 24, 25, 25); + let r: u8x8 = transmute(vrhadd_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrhaddq_u8() { + let a: u8x16 = u8x16::new(42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42); + let b: u8x16 = u8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let e: u8x16 = u8x16::new(22, 22, 23, 23, 24, 24, 25, 25, 26, 26, 27, 27, 28, 28, 29, 29); + let r: u8x16 = transmute(vrhaddq_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrhadd_u16() { + let a: u16x4 = u16x4::new(42, 42, 42, 42); + let b: u16x4 = u16x4::new(1, 2, 3, 4); + let e: u16x4 = u16x4::new(22, 22, 23, 23); + let r: u16x4 = transmute(vrhadd_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrhaddq_u16() { + let a: u16x8 = u16x8::new(42, 42, 42, 42, 42, 42, 42, 42); + let b: u16x8 = u16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let e: u16x8 = u16x8::new(22, 22, 23, 23, 24, 24, 25, 25); + let r: u16x8 = transmute(vrhaddq_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrhadd_u32() { + let a: u32x2 = u32x2::new(42, 42); + let b: u32x2 = u32x2::new(1, 2); + let e: u32x2 = u32x2::new(22, 22); + let r: u32x2 = transmute(vrhadd_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrhaddq_u32() { + let a: u32x4 = u32x4::new(42, 42, 42, 42); + let b: u32x4 = u32x4::new(1, 2, 3, 4); + let e: u32x4 = u32x4::new(22, 22, 23, 23); + let r: u32x4 = transmute(vrhaddq_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrhadd_s8() { + let a: i8x8 = i8x8::new(42, 42, 42, 42, 42, 42, 42, 42); + let b: i8x8 = i8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let e: i8x8 = i8x8::new(22, 22, 23, 23, 24, 24, 25, 25); + let r: i8x8 = transmute(vrhadd_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrhaddq_s8() { + let a: i8x16 = i8x16::new(42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42); + let b: i8x16 = i8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let e: i8x16 = i8x16::new(22, 22, 23, 23, 24, 24, 25, 25, 26, 26, 27, 27, 28, 28, 29, 29); + let r: i8x16 = transmute(vrhaddq_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrhadd_s16() { + let a: i16x4 = i16x4::new(42, 42, 42, 42); + let b: i16x4 = i16x4::new(1, 2, 3, 4); + let e: i16x4 = i16x4::new(22, 22, 23, 23); + let r: i16x4 = transmute(vrhadd_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrhaddq_s16() { + let a: i16x8 = i16x8::new(42, 42, 42, 42, 42, 42, 42, 42); + let b: i16x8 = i16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let e: i16x8 = i16x8::new(22, 22, 23, 23, 24, 24, 25, 25); + let r: i16x8 = transmute(vrhaddq_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrhadd_s32() { + let a: i32x2 = i32x2::new(42, 42); + let b: i32x2 = i32x2::new(1, 2); + let e: i32x2 = i32x2::new(22, 22); + let r: i32x2 = transmute(vrhadd_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrhaddq_s32() { + let a: i32x4 = i32x4::new(42, 42, 42, 42); + let b: i32x4 = i32x4::new(1, 2, 3, 4); + let e: i32x4 = i32x4::new(22, 22, 23, 23); + let r: i32x4 = transmute(vrhaddq_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrndn_f32() { + let a: f32x2 = f32x2::new(-1.5, 0.5); + let e: f32x2 = f32x2::new(-2.0, 0.0); + let r: f32x2 = transmute(vrndn_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrndnq_f32() { + let a: f32x4 = f32x4::new(-1.5, 0.5, 1.5, 2.5); + let e: f32x4 = f32x4::new(-2.0, 0.0, 2.0, 2.0); + let r: f32x4 = transmute(vrndnq_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqadd_u8() { + let a: u8x8 = u8x8::new(42, 42, 42, 42, 42, 42, 42, 42); + let b: u8x8 = u8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let e: u8x8 = u8x8::new(43, 44, 45, 46, 47, 48, 49, 50); + let r: u8x8 = transmute(vqadd_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqaddq_u8() { + let a: u8x16 = u8x16::new(42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42); + let b: u8x16 = u8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let e: u8x16 = u8x16::new(43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58); + let r: u8x16 = transmute(vqaddq_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqadd_u16() { + let a: u16x4 = u16x4::new(42, 42, 42, 42); + let b: u16x4 = u16x4::new(1, 2, 3, 4); + let e: u16x4 = u16x4::new(43, 44, 45, 46); + let r: u16x4 = transmute(vqadd_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqaddq_u16() { + let a: u16x8 = u16x8::new(42, 42, 42, 42, 42, 42, 42, 42); + let b: u16x8 = u16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let e: u16x8 = u16x8::new(43, 44, 45, 46, 47, 48, 49, 50); + let r: u16x8 = transmute(vqaddq_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqadd_u32() { + let a: u32x2 = u32x2::new(42, 42); + let b: u32x2 = u32x2::new(1, 2); + let e: u32x2 = u32x2::new(43, 44); + let r: u32x2 = transmute(vqadd_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqaddq_u32() { + let a: u32x4 = u32x4::new(42, 42, 42, 42); + let b: u32x4 = u32x4::new(1, 2, 3, 4); + let e: u32x4 = u32x4::new(43, 44, 45, 46); + let r: u32x4 = transmute(vqaddq_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqadd_u64() { + let a: u64x1 = u64x1::new(42); + let b: u64x1 = u64x1::new(1); + let e: u64x1 = u64x1::new(43); + let r: u64x1 = transmute(vqadd_u64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqaddq_u64() { + let a: u64x2 = u64x2::new(42, 42); + let b: u64x2 = u64x2::new(1, 2); + let e: u64x2 = u64x2::new(43, 44); + let r: u64x2 = transmute(vqaddq_u64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqadd_s8() { + let a: i8x8 = i8x8::new(42, 42, 42, 42, 42, 42, 42, 42); + let b: i8x8 = i8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let e: i8x8 = i8x8::new(43, 44, 45, 46, 47, 48, 49, 50); + let r: i8x8 = transmute(vqadd_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqaddq_s8() { + let a: i8x16 = i8x16::new(42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42); + let b: i8x16 = i8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let e: i8x16 = i8x16::new(43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58); + let r: i8x16 = transmute(vqaddq_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqadd_s16() { + let a: i16x4 = i16x4::new(42, 42, 42, 42); + let b: i16x4 = i16x4::new(1, 2, 3, 4); + let e: i16x4 = i16x4::new(43, 44, 45, 46); + let r: i16x4 = transmute(vqadd_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqaddq_s16() { + let a: i16x8 = i16x8::new(42, 42, 42, 42, 42, 42, 42, 42); + let b: i16x8 = i16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let e: i16x8 = i16x8::new(43, 44, 45, 46, 47, 48, 49, 50); + let r: i16x8 = transmute(vqaddq_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqadd_s32() { + let a: i32x2 = i32x2::new(42, 42); + let b: i32x2 = i32x2::new(1, 2); + let e: i32x2 = i32x2::new(43, 44); + let r: i32x2 = transmute(vqadd_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqaddq_s32() { + let a: i32x4 = i32x4::new(42, 42, 42, 42); + let b: i32x4 = i32x4::new(1, 2, 3, 4); + let e: i32x4 = i32x4::new(43, 44, 45, 46); + let r: i32x4 = transmute(vqaddq_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqadd_s64() { + let a: i64x1 = i64x1::new(42); + let b: i64x1 = i64x1::new(1); + let e: i64x1 = i64x1::new(43); + let r: i64x1 = transmute(vqadd_s64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqaddq_s64() { + let a: i64x2 = i64x2::new(42, 42); + let b: i64x2 = i64x2::new(1, 2); + let e: i64x2 = i64x2::new(43, 44); + let r: i64x2 = transmute(vqaddq_s64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmul_s8() { + let a: i8x8 = i8x8::new(1, 2, 1, 2, 1, 2, 1, 2); + let b: i8x8 = i8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let e: i8x8 = i8x8::new(1, 4, 3, 8, 5, 12, 7, 16); + let r: i8x8 = transmute(vmul_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmulq_s8() { + let a: i8x16 = i8x16::new(1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2); + let b: i8x16 = i8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let e: i8x16 = i8x16::new(1, 4, 3, 8, 5, 12, 7, 16, 9, 20, 11, 24, 13, 28, 15, 32); + let r: i8x16 = transmute(vmulq_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmul_s16() { + let a: i16x4 = i16x4::new(1, 2, 1, 2); + let b: i16x4 = i16x4::new(1, 2, 3, 4); + let e: i16x4 = i16x4::new(1, 4, 3, 8); + let r: i16x4 = transmute(vmul_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmulq_s16() { + let a: i16x8 = i16x8::new(1, 2, 1, 2, 1, 2, 1, 2); + let b: i16x8 = i16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let e: i16x8 = i16x8::new(1, 4, 3, 8, 5, 12, 7, 16); + let r: i16x8 = transmute(vmulq_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmul_s32() { + let a: i32x2 = i32x2::new(1, 2); + let b: i32x2 = i32x2::new(1, 2); + let e: i32x2 = i32x2::new(1, 4); + let r: i32x2 = transmute(vmul_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmulq_s32() { + let a: i32x4 = i32x4::new(1, 2, 1, 2); + let b: i32x4 = i32x4::new(1, 2, 3, 4); + let e: i32x4 = i32x4::new(1, 4, 3, 8); + let r: i32x4 = transmute(vmulq_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmul_u8() { + let a: u8x8 = u8x8::new(1, 2, 1, 2, 1, 2, 1, 2); + let b: u8x8 = u8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let e: u8x8 = u8x8::new(1, 4, 3, 8, 5, 12, 7, 16); + let r: u8x8 = transmute(vmul_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmulq_u8() { + let a: u8x16 = u8x16::new(1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2); + let b: u8x16 = u8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let e: u8x16 = u8x16::new(1, 4, 3, 8, 5, 12, 7, 16, 9, 20, 11, 24, 13, 28, 15, 32); + let r: u8x16 = transmute(vmulq_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmul_u16() { + let a: u16x4 = u16x4::new(1, 2, 1, 2); + let b: u16x4 = u16x4::new(1, 2, 3, 4); + let e: u16x4 = u16x4::new(1, 4, 3, 8); + let r: u16x4 = transmute(vmul_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmulq_u16() { + let a: u16x8 = u16x8::new(1, 2, 1, 2, 1, 2, 1, 2); + let b: u16x8 = u16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let e: u16x8 = u16x8::new(1, 4, 3, 8, 5, 12, 7, 16); + let r: u16x8 = transmute(vmulq_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmul_u32() { + let a: u32x2 = u32x2::new(1, 2); + let b: u32x2 = u32x2::new(1, 2); + let e: u32x2 = u32x2::new(1, 4); + let r: u32x2 = transmute(vmul_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmulq_u32() { + let a: u32x4 = u32x4::new(1, 2, 1, 2); + let b: u32x4 = u32x4::new(1, 2, 3, 4); + let e: u32x4 = u32x4::new(1, 4, 3, 8); + let r: u32x4 = transmute(vmulq_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmul_p8() { + let a: i8x8 = i8x8::new(1, 3, 1, 3, 1, 3, 1, 3); + let b: i8x8 = i8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let e: i8x8 = i8x8::new(1, 6, 3, 12, 5, 10, 7, 24); + let r: i8x8 = transmute(vmul_p8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmulq_p8() { + let a: i8x16 = i8x16::new(1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3); + let b: i8x16 = i8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let e: i8x16 = i8x16::new(1, 6, 3, 12, 5, 10, 7, 24, 9, 30, 11, 20, 13, 18, 15, 48); + let r: i8x16 = transmute(vmulq_p8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmul_f32() { + let a: f32x2 = f32x2::new(1.0, 2.0); + let b: f32x2 = f32x2::new(2.0, 3.0); + let e: f32x2 = f32x2::new(2.0, 6.0); + let r: f32x2 = transmute(vmul_f32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmulq_f32() { + let a: f32x4 = f32x4::new(1.0, 2.0, 1.0, 2.0); + let b: f32x4 = f32x4::new(2.0, 3.0, 4.0, 5.0); + let e: f32x4 = f32x4::new(2.0, 6.0, 4.0, 10.0); + let r: f32x4 = transmute(vmulq_f32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmul_n_s16() { + let a: i16x4 = i16x4::new(1, 2, 3, 4); + let b: i16 = 2; + let e: i16x4 = i16x4::new(2, 4, 6, 8); + let r: i16x4 = transmute(vmul_n_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmulq_n_s16() { + let a: i16x8 = i16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let b: i16 = 2; + let e: i16x8 = i16x8::new(2, 4, 6, 8, 10, 12, 14, 16); + let r: i16x8 = transmute(vmulq_n_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmul_n_s32() { + let a: i32x2 = i32x2::new(1, 2); + let b: i32 = 2; + let e: i32x2 = i32x2::new(2, 4); + let r: i32x2 = transmute(vmul_n_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmulq_n_s32() { + let a: i32x4 = i32x4::new(1, 2, 3, 4); + let b: i32 = 2; + let e: i32x4 = i32x4::new(2, 4, 6, 8); + let r: i32x4 = transmute(vmulq_n_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmul_n_u16() { + let a: u16x4 = u16x4::new(1, 2, 3, 4); + let b: u16 = 2; + let e: u16x4 = u16x4::new(2, 4, 6, 8); + let r: u16x4 = transmute(vmul_n_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmulq_n_u16() { + let a: u16x8 = u16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let b: u16 = 2; + let e: u16x8 = u16x8::new(2, 4, 6, 8, 10, 12, 14, 16); + let r: u16x8 = transmute(vmulq_n_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmul_n_u32() { + let a: u32x2 = u32x2::new(1, 2); + let b: u32 = 2; + let e: u32x2 = u32x2::new(2, 4); + let r: u32x2 = transmute(vmul_n_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmulq_n_u32() { + let a: u32x4 = u32x4::new(1, 2, 3, 4); + let b: u32 = 2; + let e: u32x4 = u32x4::new(2, 4, 6, 8); + let r: u32x4 = transmute(vmulq_n_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmul_n_f32() { + let a: f32x2 = f32x2::new(1., 2.); + let b: f32 = 2.; + let e: f32x2 = f32x2::new(2., 4.); + let r: f32x2 = transmute(vmul_n_f32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmulq_n_f32() { + let a: f32x4 = f32x4::new(1., 2., 3., 4.); + let b: f32 = 2.; + let e: f32x4 = f32x4::new(2., 4., 6., 8.); + let r: f32x4 = transmute(vmulq_n_f32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmul_lane_s16() { + let a: i16x4 = i16x4::new(1, 2, 3, 4); + let b: i16x4 = i16x4::new(0, 2, 0, 0); + let e: i16x4 = i16x4::new(2, 4, 6, 8); + let r: i16x4 = transmute(vmul_lane_s16::<1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmul_laneq_s16() { + let a: i16x4 = i16x4::new(1, 2, 3, 4); + let b: i16x8 = i16x8::new(0, 2, 0, 0, 0, 0, 0, 0); + let e: i16x4 = i16x4::new(2, 4, 6, 8); + let r: i16x4 = transmute(vmul_laneq_s16::<1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmulq_lane_s16() { + let a: i16x8 = i16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let b: i16x4 = i16x4::new(0, 2, 0, 0); + let e: i16x8 = i16x8::new(2, 4, 6, 8, 10, 12, 14, 16); + let r: i16x8 = transmute(vmulq_lane_s16::<1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmulq_laneq_s16() { + let a: i16x8 = i16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let b: i16x8 = i16x8::new(0, 2, 0, 0, 0, 0, 0, 0); + let e: i16x8 = i16x8::new(2, 4, 6, 8, 10, 12, 14, 16); + let r: i16x8 = transmute(vmulq_laneq_s16::<1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmul_lane_s32() { + let a: i32x2 = i32x2::new(1, 2); + let b: i32x2 = i32x2::new(0, 2); + let e: i32x2 = i32x2::new(2, 4); + let r: i32x2 = transmute(vmul_lane_s32::<1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmul_laneq_s32() { + let a: i32x2 = i32x2::new(1, 2); + let b: i32x4 = i32x4::new(0, 2, 0, 0); + let e: i32x2 = i32x2::new(2, 4); + let r: i32x2 = transmute(vmul_laneq_s32::<1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmulq_lane_s32() { + let a: i32x4 = i32x4::new(1, 2, 3, 4); + let b: i32x2 = i32x2::new(0, 2); + let e: i32x4 = i32x4::new(2, 4, 6, 8); + let r: i32x4 = transmute(vmulq_lane_s32::<1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmulq_laneq_s32() { + let a: i32x4 = i32x4::new(1, 2, 3, 4); + let b: i32x4 = i32x4::new(0, 2, 0, 0); + let e: i32x4 = i32x4::new(2, 4, 6, 8); + let r: i32x4 = transmute(vmulq_laneq_s32::<1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmul_lane_u16() { + let a: u16x4 = u16x4::new(1, 2, 3, 4); + let b: u16x4 = u16x4::new(0, 2, 0, 0); + let e: u16x4 = u16x4::new(2, 4, 6, 8); + let r: u16x4 = transmute(vmul_lane_u16::<1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmul_laneq_u16() { + let a: u16x4 = u16x4::new(1, 2, 3, 4); + let b: u16x8 = u16x8::new(0, 2, 0, 0, 0, 0, 0, 0); + let e: u16x4 = u16x4::new(2, 4, 6, 8); + let r: u16x4 = transmute(vmul_laneq_u16::<1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmulq_lane_u16() { + let a: u16x8 = u16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let b: u16x4 = u16x4::new(0, 2, 0, 0); + let e: u16x8 = u16x8::new(2, 4, 6, 8, 10, 12, 14, 16); + let r: u16x8 = transmute(vmulq_lane_u16::<1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmulq_laneq_u16() { + let a: u16x8 = u16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let b: u16x8 = u16x8::new(0, 2, 0, 0, 0, 0, 0, 0); + let e: u16x8 = u16x8::new(2, 4, 6, 8, 10, 12, 14, 16); + let r: u16x8 = transmute(vmulq_laneq_u16::<1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmul_lane_u32() { + let a: u32x2 = u32x2::new(1, 2); + let b: u32x2 = u32x2::new(0, 2); + let e: u32x2 = u32x2::new(2, 4); + let r: u32x2 = transmute(vmul_lane_u32::<1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmul_laneq_u32() { + let a: u32x2 = u32x2::new(1, 2); + let b: u32x4 = u32x4::new(0, 2, 0, 0); + let e: u32x2 = u32x2::new(2, 4); + let r: u32x2 = transmute(vmul_laneq_u32::<1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmulq_lane_u32() { + let a: u32x4 = u32x4::new(1, 2, 3, 4); + let b: u32x2 = u32x2::new(0, 2); + let e: u32x4 = u32x4::new(2, 4, 6, 8); + let r: u32x4 = transmute(vmulq_lane_u32::<1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmulq_laneq_u32() { + let a: u32x4 = u32x4::new(1, 2, 3, 4); + let b: u32x4 = u32x4::new(0, 2, 0, 0); + let e: u32x4 = u32x4::new(2, 4, 6, 8); + let r: u32x4 = transmute(vmulq_laneq_u32::<1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmul_lane_f32() { + let a: f32x2 = f32x2::new(1., 2.); + let b: f32x2 = f32x2::new(2., 0.); + let e: f32x2 = f32x2::new(2., 4.); + let r: f32x2 = transmute(vmul_lane_f32::<0>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmul_laneq_f32() { + let a: f32x2 = f32x2::new(1., 2.); + let b: f32x4 = f32x4::new(2., 0., 0., 0.); + let e: f32x2 = f32x2::new(2., 4.); + let r: f32x2 = transmute(vmul_laneq_f32::<0>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmulq_lane_f32() { + let a: f32x4 = f32x4::new(1., 2., 3., 4.); + let b: f32x2 = f32x2::new(2., 0.); + let e: f32x4 = f32x4::new(2., 4., 6., 8.); + let r: f32x4 = transmute(vmulq_lane_f32::<0>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmulq_laneq_f32() { + let a: f32x4 = f32x4::new(1., 2., 3., 4.); + let b: f32x4 = f32x4::new(2., 0., 0., 0.); + let e: f32x4 = f32x4::new(2., 4., 6., 8.); + let r: f32x4 = transmute(vmulq_laneq_f32::<0>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmull_s8() { + let a: i8x8 = i8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let b: i8x8 = i8x8::new(1, 2, 1, 2, 1, 2, 1, 2); + let e: i16x8 = i16x8::new(1, 4, 3, 8, 5, 12, 7, 16); + let r: i16x8 = transmute(vmull_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmull_s16() { + let a: i16x4 = i16x4::new(1, 2, 3, 4); + let b: i16x4 = i16x4::new(1, 2, 1, 2); + let e: i32x4 = i32x4::new(1, 4, 3, 8); + let r: i32x4 = transmute(vmull_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmull_s32() { + let a: i32x2 = i32x2::new(1, 2); + let b: i32x2 = i32x2::new(1, 2); + let e: i64x2 = i64x2::new(1, 4); + let r: i64x2 = transmute(vmull_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmull_u8() { + let a: u8x8 = u8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let b: u8x8 = u8x8::new(1, 2, 1, 2, 1, 2, 1, 2); + let e: u16x8 = u16x8::new(1, 4, 3, 8, 5, 12, 7, 16); + let r: u16x8 = transmute(vmull_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmull_u16() { + let a: u16x4 = u16x4::new(1, 2, 3, 4); + let b: u16x4 = u16x4::new(1, 2, 1, 2); + let e: u32x4 = u32x4::new(1, 4, 3, 8); + let r: u32x4 = transmute(vmull_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmull_u32() { + let a: u32x2 = u32x2::new(1, 2); + let b: u32x2 = u32x2::new(1, 2); + let e: u64x2 = u64x2::new(1, 4); + let r: u64x2 = transmute(vmull_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmull_p8() { + let a: i8x8 = i8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let b: i8x8 = i8x8::new(1, 3, 1, 3, 1, 3, 1, 3); + let e: i16x8 = i16x8::new(1, 6, 3, 12, 5, 10, 7, 24); + let r: i16x8 = transmute(vmull_p8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmullh_n_s16() { + let a: i16x4 = i16x4::new(1, 2, 3, 4); + let b: i16 = 2; + let e: i32x4 = i32x4::new(2, 4, 6, 8); + let r: i32x4 = transmute(vmullh_n_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmulls_n_s32() { + let a: i32x2 = i32x2::new(1, 2); + let b: i32 = 2; + let e: i64x2 = i64x2::new(2, 4); + let r: i64x2 = transmute(vmulls_n_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmullh_n_u16() { + let a: u16x4 = u16x4::new(1, 2, 3, 4); + let b: u16 = 2; + let e: u32x4 = u32x4::new(2, 4, 6, 8); + let r: u32x4 = transmute(vmullh_n_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmulls_n_u32() { + let a: u32x2 = u32x2::new(1, 2); + let b: u32 = 2; + let e: u64x2 = u64x2::new(2, 4); + let r: u64x2 = transmute(vmulls_n_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmull_lane_s16() { + let a: i16x4 = i16x4::new(1, 2, 3, 4); + let b: i16x4 = i16x4::new(0, 2, 0, 0); + let e: i32x4 = i32x4::new(2, 4, 6, 8); + let r: i32x4 = transmute(vmull_lane_s16::<1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmull_laneq_s16() { + let a: i16x4 = i16x4::new(1, 2, 3, 4); + let b: i16x8 = i16x8::new(0, 2, 0, 0, 0, 0, 0, 0); + let e: i32x4 = i32x4::new(2, 4, 6, 8); + let r: i32x4 = transmute(vmull_laneq_s16::<1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmull_lane_s32() { + let a: i32x2 = i32x2::new(1, 2); + let b: i32x2 = i32x2::new(0, 2); + let e: i64x2 = i64x2::new(2, 4); + let r: i64x2 = transmute(vmull_lane_s32::<1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmull_laneq_s32() { + let a: i32x2 = i32x2::new(1, 2); + let b: i32x4 = i32x4::new(0, 2, 0, 0); + let e: i64x2 = i64x2::new(2, 4); + let r: i64x2 = transmute(vmull_laneq_s32::<1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmull_lane_u16() { + let a: u16x4 = u16x4::new(1, 2, 3, 4); + let b: u16x4 = u16x4::new(0, 2, 0, 0); + let e: u32x4 = u32x4::new(2, 4, 6, 8); + let r: u32x4 = transmute(vmull_lane_u16::<1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmull_laneq_u16() { + let a: u16x4 = u16x4::new(1, 2, 3, 4); + let b: u16x8 = u16x8::new(0, 2, 0, 0, 0, 0, 0, 0); + let e: u32x4 = u32x4::new(2, 4, 6, 8); + let r: u32x4 = transmute(vmull_laneq_u16::<1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmull_lane_u32() { + let a: u32x2 = u32x2::new(1, 2); + let b: u32x2 = u32x2::new(0, 2); + let e: u64x2 = u64x2::new(2, 4); + let r: u64x2 = transmute(vmull_lane_u32::<1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmull_laneq_u32() { + let a: u32x2 = u32x2::new(1, 2); + let b: u32x4 = u32x4::new(0, 2, 0, 0); + let e: u64x2 = u64x2::new(2, 4); + let r: u64x2 = transmute(vmull_laneq_u32::<1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vfma_f32() { + let a: f32x2 = f32x2::new(8.0, 18.0); + let b: f32x2 = f32x2::new(6.0, 4.0); + let c: f32x2 = f32x2::new(2.0, 3.0); + let e: f32x2 = f32x2::new(20.0, 30.0); + let r: f32x2 = transmute(vfma_f32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vfmaq_f32() { + let a: f32x4 = f32x4::new(8.0, 18.0, 12.0, 10.0); + let b: f32x4 = f32x4::new(6.0, 4.0, 7.0, 8.0); + let c: f32x4 = f32x4::new(2.0, 3.0, 4.0, 5.0); + let e: f32x4 = f32x4::new(20.0, 30.0, 40.0, 50.0); + let r: f32x4 = transmute(vfmaq_f32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vfma_n_f32() { + let a: f32x2 = f32x2::new(2.0, 3.0); + let b: f32x2 = f32x2::new(6.0, 4.0); + let c: f32 = 8.0; + let e: f32x2 = f32x2::new(50.0, 35.0); + let r: f32x2 = transmute(vfma_n_f32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vfmaq_n_f32() { + let a: f32x4 = f32x4::new(2.0, 3.0, 4.0, 5.0); + let b: f32x4 = f32x4::new(6.0, 4.0, 7.0, 8.0); + let c: f32 = 8.0; + let e: f32x4 = f32x4::new(50.0, 35.0, 60.0, 69.0); + let r: f32x4 = transmute(vfmaq_n_f32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vfms_f32() { + let a: f32x2 = f32x2::new(20.0, 30.0); + let b: f32x2 = f32x2::new(6.0, 4.0); + let c: f32x2 = f32x2::new(2.0, 3.0); + let e: f32x2 = f32x2::new(8.0, 18.0); + let r: f32x2 = transmute(vfms_f32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vfmsq_f32() { + let a: f32x4 = f32x4::new(20.0, 30.0, 40.0, 50.0); + let b: f32x4 = f32x4::new(6.0, 4.0, 7.0, 8.0); + let c: f32x4 = f32x4::new(2.0, 3.0, 4.0, 5.0); + let e: f32x4 = f32x4::new(8.0, 18.0, 12.0, 10.0); + let r: f32x4 = transmute(vfmsq_f32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vfms_n_f32() { + let a: f32x2 = f32x2::new(50.0, 35.0); + let b: f32x2 = f32x2::new(6.0, 4.0); + let c: f32 = 8.0; + let e: f32x2 = f32x2::new(2.0, 3.0); + let r: f32x2 = transmute(vfms_n_f32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vfmsq_n_f32() { + let a: f32x4 = f32x4::new(50.0, 35.0, 60.0, 69.0); + let b: f32x4 = f32x4::new(6.0, 4.0, 7.0, 8.0); + let c: f32 = 8.0; + let e: f32x4 = f32x4::new(2.0, 3.0, 4.0, 5.0); + let r: f32x4 = transmute(vfmsq_n_f32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsub_s8() { + let a: i8x8 = i8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let b: i8x8 = i8x8::new(1, 2, 1, 2, 1, 2, 1, 2); + let e: i8x8 = i8x8::new(0, 0, 2, 2, 4, 4, 6, 6); + let r: i8x8 = transmute(vsub_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsubq_s8() { + let a: i8x16 = i8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let b: i8x16 = i8x16::new(1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2); + let e: i8x16 = i8x16::new(0, 0, 2, 2, 4, 4, 6, 6, 8, 8, 10, 10, 12, 12, 14, 14); + let r: i8x16 = transmute(vsubq_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsub_s16() { + let a: i16x4 = i16x4::new(1, 2, 3, 4); + let b: i16x4 = i16x4::new(1, 2, 1, 2); + let e: i16x4 = i16x4::new(0, 0, 2, 2); + let r: i16x4 = transmute(vsub_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsubq_s16() { + let a: i16x8 = i16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let b: i16x8 = i16x8::new(1, 2, 1, 2, 1, 2, 1, 2); + let e: i16x8 = i16x8::new(0, 0, 2, 2, 4, 4, 6, 6); + let r: i16x8 = transmute(vsubq_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsub_s32() { + let a: i32x2 = i32x2::new(1, 2); + let b: i32x2 = i32x2::new(1, 2); + let e: i32x2 = i32x2::new(0, 0); + let r: i32x2 = transmute(vsub_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsubq_s32() { + let a: i32x4 = i32x4::new(1, 2, 3, 4); + let b: i32x4 = i32x4::new(1, 2, 1, 2); + let e: i32x4 = i32x4::new(0, 0, 2, 2); + let r: i32x4 = transmute(vsubq_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsub_u8() { + let a: u8x8 = u8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let b: u8x8 = u8x8::new(1, 2, 1, 2, 1, 2, 1, 2); + let e: u8x8 = u8x8::new(0, 0, 2, 2, 4, 4, 6, 6); + let r: u8x8 = transmute(vsub_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsubq_u8() { + let a: u8x16 = u8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let b: u8x16 = u8x16::new(1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2); + let e: u8x16 = u8x16::new(0, 0, 2, 2, 4, 4, 6, 6, 8, 8, 10, 10, 12, 12, 14, 14); + let r: u8x16 = transmute(vsubq_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsub_u16() { + let a: u16x4 = u16x4::new(1, 2, 3, 4); + let b: u16x4 = u16x4::new(1, 2, 1, 2); + let e: u16x4 = u16x4::new(0, 0, 2, 2); + let r: u16x4 = transmute(vsub_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsubq_u16() { + let a: u16x8 = u16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let b: u16x8 = u16x8::new(1, 2, 1, 2, 1, 2, 1, 2); + let e: u16x8 = u16x8::new(0, 0, 2, 2, 4, 4, 6, 6); + let r: u16x8 = transmute(vsubq_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsub_u32() { + let a: u32x2 = u32x2::new(1, 2); + let b: u32x2 = u32x2::new(1, 2); + let e: u32x2 = u32x2::new(0, 0); + let r: u32x2 = transmute(vsub_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsubq_u32() { + let a: u32x4 = u32x4::new(1, 2, 3, 4); + let b: u32x4 = u32x4::new(1, 2, 1, 2); + let e: u32x4 = u32x4::new(0, 0, 2, 2); + let r: u32x4 = transmute(vsubq_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsub_s64() { + let a: i64x1 = i64x1::new(1); + let b: i64x1 = i64x1::new(1); + let e: i64x1 = i64x1::new(0); + let r: i64x1 = transmute(vsub_s64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsubq_s64() { + let a: i64x2 = i64x2::new(1, 2); + let b: i64x2 = i64x2::new(1, 2); + let e: i64x2 = i64x2::new(0, 0); + let r: i64x2 = transmute(vsubq_s64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsub_u64() { + let a: u64x1 = u64x1::new(1); + let b: u64x1 = u64x1::new(1); + let e: u64x1 = u64x1::new(0); + let r: u64x1 = transmute(vsub_u64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsubq_u64() { + let a: u64x2 = u64x2::new(1, 2); + let b: u64x2 = u64x2::new(1, 2); + let e: u64x2 = u64x2::new(0, 0); + let r: u64x2 = transmute(vsubq_u64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsub_f32() { + let a: f32x2 = f32x2::new(1.0, 4.0); + let b: f32x2 = f32x2::new(1.0, 2.0); + let e: f32x2 = f32x2::new(0.0, 2.0); + let r: f32x2 = transmute(vsub_f32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsubq_f32() { + let a: f32x4 = f32x4::new(1.0, 4.0, 3.0, 8.0); + let b: f32x4 = f32x4::new(1.0, 2.0, 3.0, 4.0); + let e: f32x4 = f32x4::new(0.0, 2.0, 0.0, 4.0); + let r: f32x4 = transmute(vsubq_f32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsubhn_s16() { + let a: i16x8 = i16x8::new(0x7F_FF, -32768, 1, 1, 0x7F_FF, -32768, 1, 1); + let b: i16x8 = i16x8::new(1, 0, 0, 0, 1, 0, 0, 0); + let e: i8x8 = i8x8::new(0x7F, -128, 0, 0, 0x7F, -128, 0, 0); + let r: i8x8 = transmute(vsubhn_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsubhn_s32() { + let a: i32x4 = i32x4::new(0x7F_FF_FF_FF, -2147483648, 1, 1); + let b: i32x4 = i32x4::new(1, 0, 0, 0); + let e: i16x4 = i16x4::new(0x7F_FF, -32768, 0, 0); + let r: i16x4 = transmute(vsubhn_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsubhn_s64() { + let a: i64x2 = i64x2::new(0x7F_FF_FF_FF_FF_FF_FF_FF, -9223372036854775808); + let b: i64x2 = i64x2::new(1, 0); + let e: i32x2 = i32x2::new(0x7F_FF_FF_FF, -2147483648); + let r: i32x2 = transmute(vsubhn_s64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsubhn_u16() { + let a: u16x8 = u16x8::new(0xFF_FF, 0, 1, 1, 0xFF_FF, 0, 1, 1); + let b: u16x8 = u16x8::new(1, 0, 0, 0, 1, 0, 0, 0); + let e: u8x8 = u8x8::new(0xFF, 0, 0, 0, 0xFF, 0, 0, 0); + let r: u8x8 = transmute(vsubhn_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsubhn_u32() { + let a: u32x4 = u32x4::new(0xFF_FF_FF_FF, 0, 1, 1); + let b: u32x4 = u32x4::new(1, 0, 0, 0); + let e: u16x4 = u16x4::new(0xFF_FF, 0, 0, 0); + let r: u16x4 = transmute(vsubhn_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsubhn_u64() { + let a: u64x2 = u64x2::new(0xFF_FF_FF_FF_FF_FF_FF_FF, 0); + let b: u64x2 = u64x2::new(1, 0); + let e: u32x2 = u32x2::new(0xFF_FF_FF_FF, 0); + let r: u32x2 = transmute(vsubhn_u64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsubhn_high_s16() { + let a: i8x8 = i8x8::new(0x7F, 0, 0x7F, 0, 0x7F, 0, 0x7F, 0); + let b: i16x8 = i16x8::new(0x7F_FF, 1, 0x7F_FF, 1, 0x7F_FF, 1, 0x7F_FF, 1); + let c: i16x8 = i16x8::new(1, 0, 1, 0, 1, 0, 1, 0); + let e: i8x16 = i8x16::new(0x7F, 0, 0x7F, 0, 0x7F, 0, 0x7F, 0, 0x7F, 0, 0x7F, 0, 0x7F, 0, 0x7F, 0); + let r: i8x16 = transmute(vsubhn_high_s16(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsubhn_high_s32() { + let a: i16x4 = i16x4::new(0x7F_FF, 0, 0x7F_FF, 0); + let b: i32x4 = i32x4::new(0x7F_FF_FF_FF, 1, 0x7F_FF_FF_FF, 1); + let c: i32x4 = i32x4::new(1, 0, 1, 0); + let e: i16x8 = i16x8::new(0x7F_FF, 0, 0x7F_FF, 0, 0x7F_FF, 0, 0x7F_FF, 0); + let r: i16x8 = transmute(vsubhn_high_s32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsubhn_high_s64() { + let a: i32x2 = i32x2::new(0x7F_FF_FF_FF, 0); + let b: i64x2 = i64x2::new(0x7F_FF_FF_FF_FF_FF_FF_FF, 1); + let c: i64x2 = i64x2::new(1, 0); + let e: i32x4 = i32x4::new(0x7F_FF_FF_FF, 0, 0x7F_FF_FF_FF, 0); + let r: i32x4 = transmute(vsubhn_high_s64(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsubhn_high_u16() { + let a: u8x8 = u8x8::new(0xFF, 0, 0xFF, 0, 0xFF, 0, 0xFF, 0); + let b: u16x8 = u16x8::new(0xFF_FF, 1, 0xFF_FF, 1, 0xFF_FF, 1, 0xFF_FF, 1); + let c: u16x8 = u16x8::new(1, 0, 1, 0, 1, 0, 1, 0); + let e: u8x16 = u8x16::new(0xFF, 0, 0xFF, 0, 0xFF, 0, 0xFF, 0, 0xFF, 0, 0xFF, 0, 0xFF, 0, 0xFF, 0); + let r: u8x16 = transmute(vsubhn_high_u16(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsubhn_high_u32() { + let a: u16x4 = u16x4::new(0xFF_FF, 0, 0xFF_FF, 0); + let b: u32x4 = u32x4::new(0xFF_FF_FF_FF, 1, 0xFF_FF_FF_FF, 1); + let c: u32x4 = u32x4::new(1, 0, 1, 0); + let e: u16x8 = u16x8::new(0xFF_FF, 0, 0xFF_FF, 0, 0xFF_FF, 0, 0xFF_FF, 0); + let r: u16x8 = transmute(vsubhn_high_u32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsubhn_high_u64() { + let a: u32x2 = u32x2::new(0xFF_FF_FF_FF, 0); + let b: u64x2 = u64x2::new(0xFF_FF_FF_FF_FF_FF_FF_FF, 1); + let c: u64x2 = u64x2::new(1, 0); + let e: u32x4 = u32x4::new(0xFF_FF_FF_FF, 0, 0xFF_FF_FF_FF, 0); + let r: u32x4 = transmute(vsubhn_high_u64(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vhsub_u8() { + let a: u8x8 = u8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let b: u8x8 = u8x8::new(1, 2, 1, 2, 1, 2, 1, 2); + let e: u8x8 = u8x8::new(0, 0, 1, 1, 2, 2, 3, 3); + let r: u8x8 = transmute(vhsub_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vhsubq_u8() { + let a: u8x16 = u8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let b: u8x16 = u8x16::new(1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2); + let e: u8x16 = u8x16::new(0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7); + let r: u8x16 = transmute(vhsubq_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vhsub_u16() { + let a: u16x4 = u16x4::new(1, 2, 3, 4); + let b: u16x4 = u16x4::new(1, 2, 1, 2); + let e: u16x4 = u16x4::new(0, 0, 1, 1); + let r: u16x4 = transmute(vhsub_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vhsubq_u16() { + let a: u16x8 = u16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let b: u16x8 = u16x8::new(1, 2, 1, 2, 1, 2, 1, 2); + let e: u16x8 = u16x8::new(0, 0, 1, 1, 2, 2, 3, 3); + let r: u16x8 = transmute(vhsubq_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vhsub_u32() { + let a: u32x2 = u32x2::new(1, 2); + let b: u32x2 = u32x2::new(1, 2); + let e: u32x2 = u32x2::new(0, 0); + let r: u32x2 = transmute(vhsub_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vhsubq_u32() { + let a: u32x4 = u32x4::new(1, 2, 3, 4); + let b: u32x4 = u32x4::new(1, 2, 1, 2); + let e: u32x4 = u32x4::new(0, 0, 1, 1); + let r: u32x4 = transmute(vhsubq_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vhsub_s8() { + let a: i8x8 = i8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let b: i8x8 = i8x8::new(1, 2, 1, 2, 1, 2, 1, 2); + let e: i8x8 = i8x8::new(0, 0, 1, 1, 2, 2, 3, 3); + let r: i8x8 = transmute(vhsub_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vhsubq_s8() { + let a: i8x16 = i8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let b: i8x16 = i8x16::new(1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2); + let e: i8x16 = i8x16::new(0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7); + let r: i8x16 = transmute(vhsubq_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vhsub_s16() { + let a: i16x4 = i16x4::new(1, 2, 3, 4); + let b: i16x4 = i16x4::new(1, 2, 1, 2); + let e: i16x4 = i16x4::new(0, 0, 1, 1); + let r: i16x4 = transmute(vhsub_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vhsubq_s16() { + let a: i16x8 = i16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let b: i16x8 = i16x8::new(1, 2, 1, 2, 1, 2, 1, 2); + let e: i16x8 = i16x8::new(0, 0, 1, 1, 2, 2, 3, 3); + let r: i16x8 = transmute(vhsubq_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vhsub_s32() { + let a: i32x2 = i32x2::new(1, 2); + let b: i32x2 = i32x2::new(1, 2); + let e: i32x2 = i32x2::new(0, 0); + let r: i32x2 = transmute(vhsub_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vhsubq_s32() { + let a: i32x4 = i32x4::new(1, 2, 3, 4); + let b: i32x4 = i32x4::new(1, 2, 1, 2); + let e: i32x4 = i32x4::new(0, 0, 1, 1); + let r: i32x4 = transmute(vhsubq_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsubw_s8() { + let a: i16x8 = i16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let b: i8x8 = i8x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let e: i16x8 = i16x8::new(0, 0, 0, 0, 0, 0, 0, 0); + let r: i16x8 = transmute(vsubw_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsubw_s16() { + let a: i32x4 = i32x4::new(0, 1, 2, 3); + let b: i16x4 = i16x4::new(0, 1, 2, 3); + let e: i32x4 = i32x4::new(0, 0, 0, 0); + let r: i32x4 = transmute(vsubw_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsubw_s32() { + let a: i64x2 = i64x2::new(0, 1); + let b: i32x2 = i32x2::new(0, 1); + let e: i64x2 = i64x2::new(0, 0); + let r: i64x2 = transmute(vsubw_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsubw_u8() { + let a: u16x8 = u16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let b: u8x8 = u8x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let e: u16x8 = u16x8::new(0, 0, 0, 0, 0, 0, 0, 0); + let r: u16x8 = transmute(vsubw_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsubw_u16() { + let a: u32x4 = u32x4::new(0, 1, 2, 3); + let b: u16x4 = u16x4::new(0, 1, 2, 3); + let e: u32x4 = u32x4::new(0, 0, 0, 0); + let r: u32x4 = transmute(vsubw_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsubw_u32() { + let a: u64x2 = u64x2::new(0, 1); + let b: u32x2 = u32x2::new(0, 1); + let e: u64x2 = u64x2::new(0, 0); + let r: u64x2 = transmute(vsubw_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsubl_s8() { + let a: i8x8 = i8x8::new(0x7F, -128, 2, 3, 4, 5, 6, 7); + let b: i8x8 = i8x8::new(0x7F, -128, 2, 3, 4, 5, 6, 7); + let e: i16x8 = i16x8::new(0, 0, 0, 0, 0, 0, 0, 0); + let r: i16x8 = transmute(vsubl_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsubl_s16() { + let a: i16x4 = i16x4::new(0x7F_FF, -32768, 2, 3); + let b: i16x4 = i16x4::new(0x7F_FF, -32768, 2, 3); + let e: i32x4 = i32x4::new(0, 0, 0, 0); + let r: i32x4 = transmute(vsubl_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsubl_s32() { + let a: i32x2 = i32x2::new(0x7F_FF_FF_FF, -2147483648); + let b: i32x2 = i32x2::new(0x7F_FF_FF_FF, -2147483648); + let e: i64x2 = i64x2::new(0, 0); + let r: i64x2 = transmute(vsubl_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsubl_u8() { + let a: u8x8 = u8x8::new(0xFF, 0, 2, 3, 4, 5, 6, 7); + let b: u8x8 = u8x8::new(0xFF, 0, 2, 3, 4, 5, 6, 7); + let e: u16x8 = u16x8::new(0, 0, 0, 0, 0, 0, 0, 0); + let r: u16x8 = transmute(vsubl_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsubl_u16() { + let a: u16x4 = u16x4::new(0xFF_FF, 0, 2, 3); + let b: u16x4 = u16x4::new(0xFF_FF, 0, 2, 3); + let e: u32x4 = u32x4::new(0, 0, 0, 0); + let r: u32x4 = transmute(vsubl_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsubl_u32() { + let a: u32x2 = u32x2::new(0xFF_FF_FF_FF, 0); + let b: u32x2 = u32x2::new(0xFF_FF_FF_FF, 0); + let e: u64x2 = u64x2::new(0, 0); + let r: u64x2 = transmute(vsubl_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmax_s8() { + let a: i8x8 = i8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let b: i8x8 = i8x8::new(16, 15, 14, 13, 12, 11, 10, 9); + let e: i8x8 = i8x8::new(16, 15, 14, 13, 12, 11, 10, 9); + let r: i8x8 = transmute(vmax_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmaxq_s8() { + let a: i8x16 = i8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let b: i8x16 = i8x16::new(16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1); + let e: i8x16 = i8x16::new(16, 15, 14, 13, 12, 11, 10, 9, 9, 10, 11, 12, 13, 14, 15, 16); + let r: i8x16 = transmute(vmaxq_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmax_s16() { + let a: i16x4 = i16x4::new(1, 2, 3, 4); + let b: i16x4 = i16x4::new(16, 15, 14, 13); + let e: i16x4 = i16x4::new(16, 15, 14, 13); + let r: i16x4 = transmute(vmax_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmaxq_s16() { + let a: i16x8 = i16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let b: i16x8 = i16x8::new(16, 15, 14, 13, 12, 11, 10, 9); + let e: i16x8 = i16x8::new(16, 15, 14, 13, 12, 11, 10, 9); + let r: i16x8 = transmute(vmaxq_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmax_s32() { + let a: i32x2 = i32x2::new(1, 2); + let b: i32x2 = i32x2::new(16, 15); + let e: i32x2 = i32x2::new(16, 15); + let r: i32x2 = transmute(vmax_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmaxq_s32() { + let a: i32x4 = i32x4::new(1, 2, 3, 4); + let b: i32x4 = i32x4::new(16, 15, 14, 13); + let e: i32x4 = i32x4::new(16, 15, 14, 13); + let r: i32x4 = transmute(vmaxq_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmax_u8() { + let a: u8x8 = u8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let b: u8x8 = u8x8::new(16, 15, 14, 13, 12, 11, 10, 9); + let e: u8x8 = u8x8::new(16, 15, 14, 13, 12, 11, 10, 9); + let r: u8x8 = transmute(vmax_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmaxq_u8() { + let a: u8x16 = u8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let b: u8x16 = u8x16::new(16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1); + let e: u8x16 = u8x16::new(16, 15, 14, 13, 12, 11, 10, 9, 9, 10, 11, 12, 13, 14, 15, 16); + let r: u8x16 = transmute(vmaxq_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmax_u16() { + let a: u16x4 = u16x4::new(1, 2, 3, 4); + let b: u16x4 = u16x4::new(16, 15, 14, 13); + let e: u16x4 = u16x4::new(16, 15, 14, 13); + let r: u16x4 = transmute(vmax_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmaxq_u16() { + let a: u16x8 = u16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let b: u16x8 = u16x8::new(16, 15, 14, 13, 12, 11, 10, 9); + let e: u16x8 = u16x8::new(16, 15, 14, 13, 12, 11, 10, 9); + let r: u16x8 = transmute(vmaxq_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmax_u32() { + let a: u32x2 = u32x2::new(1, 2); + let b: u32x2 = u32x2::new(16, 15); + let e: u32x2 = u32x2::new(16, 15); + let r: u32x2 = transmute(vmax_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmaxq_u32() { + let a: u32x4 = u32x4::new(1, 2, 3, 4); + let b: u32x4 = u32x4::new(16, 15, 14, 13); + let e: u32x4 = u32x4::new(16, 15, 14, 13); + let r: u32x4 = transmute(vmaxq_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmax_f32() { + let a: f32x2 = f32x2::new(1.0, -2.0); + let b: f32x2 = f32x2::new(0.0, 3.0); + let e: f32x2 = f32x2::new(1.0, 3.0); + let r: f32x2 = transmute(vmax_f32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmaxq_f32() { + let a: f32x4 = f32x4::new(1.0, -2.0, 3.0, -4.0); + let b: f32x4 = f32x4::new(0.0, 3.0, 2.0, 8.0); + let e: f32x4 = f32x4::new(1.0, 3.0, 3.0, 8.0); + let r: f32x4 = transmute(vmaxq_f32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmaxnm_f32() { + let a: f32x2 = f32x2::new(1.0, 2.0); + let b: f32x2 = f32x2::new(8.0, 16.0); + let e: f32x2 = f32x2::new(8.0, 16.0); + let r: f32x2 = transmute(vmaxnm_f32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmaxnmq_f32() { + let a: f32x4 = f32x4::new(1.0, 2.0, 3.0, -4.0); + let b: f32x4 = f32x4::new(8.0, 16.0, -1.0, 6.0); + let e: f32x4 = f32x4::new(8.0, 16.0, 3.0, 6.0); + let r: f32x4 = transmute(vmaxnmq_f32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmin_s8() { + let a: i8x8 = i8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let b: i8x8 = i8x8::new(16, 15, 14, 13, 12, 11, 10, 9); + let e: i8x8 = i8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let r: i8x8 = transmute(vmin_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vminq_s8() { + let a: i8x16 = i8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let b: i8x16 = i8x16::new(16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1); + let e: i8x16 = i8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 8, 7, 6, 5, 4, 3, 2, 1); + let r: i8x16 = transmute(vminq_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmin_s16() { + let a: i16x4 = i16x4::new(1, 2, 3, 4); + let b: i16x4 = i16x4::new(16, 15, 14, 13); + let e: i16x4 = i16x4::new(1, 2, 3, 4); + let r: i16x4 = transmute(vmin_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vminq_s16() { + let a: i16x8 = i16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let b: i16x8 = i16x8::new(16, 15, 14, 13, 12, 11, 10, 9); + let e: i16x8 = i16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let r: i16x8 = transmute(vminq_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmin_s32() { + let a: i32x2 = i32x2::new(1, 2); + let b: i32x2 = i32x2::new(16, 15); + let e: i32x2 = i32x2::new(1, 2); + let r: i32x2 = transmute(vmin_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vminq_s32() { + let a: i32x4 = i32x4::new(1, 2, 3, 4); + let b: i32x4 = i32x4::new(16, 15, 14, 13); + let e: i32x4 = i32x4::new(1, 2, 3, 4); + let r: i32x4 = transmute(vminq_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmin_u8() { + let a: u8x8 = u8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let b: u8x8 = u8x8::new(16, 15, 14, 13, 12, 11, 10, 9); + let e: u8x8 = u8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let r: u8x8 = transmute(vmin_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vminq_u8() { + let a: u8x16 = u8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let b: u8x16 = u8x16::new(16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1); + let e: u8x16 = u8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 8, 7, 6, 5, 4, 3, 2, 1); + let r: u8x16 = transmute(vminq_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmin_u16() { + let a: u16x4 = u16x4::new(1, 2, 3, 4); + let b: u16x4 = u16x4::new(16, 15, 14, 13); + let e: u16x4 = u16x4::new(1, 2, 3, 4); + let r: u16x4 = transmute(vmin_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vminq_u16() { + let a: u16x8 = u16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let b: u16x8 = u16x8::new(16, 15, 14, 13, 12, 11, 10, 9); + let e: u16x8 = u16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let r: u16x8 = transmute(vminq_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmin_u32() { + let a: u32x2 = u32x2::new(1, 2); + let b: u32x2 = u32x2::new(16, 15); + let e: u32x2 = u32x2::new(1, 2); + let r: u32x2 = transmute(vmin_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vminq_u32() { + let a: u32x4 = u32x4::new(1, 2, 3, 4); + let b: u32x4 = u32x4::new(16, 15, 14, 13); + let e: u32x4 = u32x4::new(1, 2, 3, 4); + let r: u32x4 = transmute(vminq_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmin_f32() { + let a: f32x2 = f32x2::new(1.0, -2.0); + let b: f32x2 = f32x2::new(0.0, 3.0); + let e: f32x2 = f32x2::new(0.0, -2.0); + let r: f32x2 = transmute(vmin_f32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vminq_f32() { + let a: f32x4 = f32x4::new(1.0, -2.0, 3.0, -4.0); + let b: f32x4 = f32x4::new(0.0, 3.0, 2.0, 8.0); + let e: f32x4 = f32x4::new(0.0, -2.0, 2.0, -4.0); + let r: f32x4 = transmute(vminq_f32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vminnm_f32() { + let a: f32x2 = f32x2::new(1.0, 2.0); + let b: f32x2 = f32x2::new(8.0, 16.0); + let e: f32x2 = f32x2::new(1.0, 2.0); + let r: f32x2 = transmute(vminnm_f32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vminnmq_f32() { + let a: f32x4 = f32x4::new(1.0, 2.0, 3.0, -4.0); + let b: f32x4 = f32x4::new(8.0, 16.0, -1.0, 6.0); + let e: f32x4 = f32x4::new(1.0, 2.0, -1.0, -4.0); + let r: f32x4 = transmute(vminnmq_f32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqdmull_s16() { + let a: i16x4 = i16x4::new(0, 1, 2, 3); + let b: i16x4 = i16x4::new(1, 2, 3, 4); + let e: i32x4 = i32x4::new(0, 4, 12, 24); + let r: i32x4 = transmute(vqdmull_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqdmull_s32() { + let a: i32x2 = i32x2::new(0, 1); + let b: i32x2 = i32x2::new(1, 2); + let e: i64x2 = i64x2::new(0, 4); + let r: i64x2 = transmute(vqdmull_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqdmull_n_s16() { + let a: i16x4 = i16x4::new(2, 4, 6, 8); + let b: i16 = 2; + let e: i32x4 = i32x4::new(8, 16, 24, 32); + let r: i32x4 = transmute(vqdmull_n_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqdmull_n_s32() { + let a: i32x2 = i32x2::new(2, 4); + let b: i32 = 2; + let e: i64x2 = i64x2::new(8, 16); + let r: i64x2 = transmute(vqdmull_n_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqdmull_lane_s16() { + let a: i16x4 = i16x4::new(1, 2, 3, 4); + let b: i16x4 = i16x4::new(0, 2, 2, 0); + let e: i32x4 = i32x4::new(4, 8, 12, 16); + let r: i32x4 = transmute(vqdmull_lane_s16::<2>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqdmull_lane_s32() { + let a: i32x2 = i32x2::new(1, 2); + let b: i32x2 = i32x2::new(0, 2); + let e: i64x2 = i64x2::new(4, 8); + let r: i64x2 = transmute(vqdmull_lane_s32::<1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqdmlal_s16() { + let a: i32x4 = i32x4::new(1, 1, 1, 1); + let b: i16x4 = i16x4::new(1, 2, 3, 4); + let c: i16x4 = i16x4::new(2, 2, 2, 2); + let e: i32x4 = i32x4::new(5, 9, 13, 17); + let r: i32x4 = transmute(vqdmlal_s16(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqdmlal_s32() { + let a: i64x2 = i64x2::new(1, 1); + let b: i32x2 = i32x2::new(1, 2); + let c: i32x2 = i32x2::new(2, 2); + let e: i64x2 = i64x2::new(5, 9); + let r: i64x2 = transmute(vqdmlal_s32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqdmlal_n_s16() { + let a: i32x4 = i32x4::new(1, 1, 1, 1); + let b: i16x4 = i16x4::new(1, 2, 3, 4); + let c: i16 = 2; + let e: i32x4 = i32x4::new(5, 9, 13, 17); + let r: i32x4 = transmute(vqdmlal_n_s16(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqdmlal_n_s32() { + let a: i64x2 = i64x2::new(1, 1); + let b: i32x2 = i32x2::new(1, 2); + let c: i32 = 2; + let e: i64x2 = i64x2::new(5, 9); + let r: i64x2 = transmute(vqdmlal_n_s32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqdmlal_lane_s16() { + let a: i32x4 = i32x4::new(1, 2, 3, 4); + let b: i16x4 = i16x4::new(1, 2, 3, 4); + let c: i16x4 = i16x4::new(0, 2, 2, 0); + let e: i32x4 = i32x4::new(5, 10, 15, 20); + let r: i32x4 = transmute(vqdmlal_lane_s16::<2>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqdmlal_lane_s32() { + let a: i64x2 = i64x2::new(1, 2); + let b: i32x2 = i32x2::new(1, 2); + let c: i32x2 = i32x2::new(0, 2); + let e: i64x2 = i64x2::new(5, 10); + let r: i64x2 = transmute(vqdmlal_lane_s32::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqdmlsl_s16() { + let a: i32x4 = i32x4::new(3, 7, 11, 15); + let b: i16x4 = i16x4::new(1, 2, 3, 4); + let c: i16x4 = i16x4::new(2, 2, 2, 2); + let e: i32x4 = i32x4::new(-1, -1, -1, -1); + let r: i32x4 = transmute(vqdmlsl_s16(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqdmlsl_s32() { + let a: i64x2 = i64x2::new(3, 7); + let b: i32x2 = i32x2::new(1, 2); + let c: i32x2 = i32x2::new(2, 2); + let e: i64x2 = i64x2::new(-1, -1); + let r: i64x2 = transmute(vqdmlsl_s32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqdmlsl_n_s16() { + let a: i32x4 = i32x4::new(3, 7, 11, 15); + let b: i16x4 = i16x4::new(1, 2, 3, 4); + let c: i16 = 2; + let e: i32x4 = i32x4::new(-1, -1, -1, -1); + let r: i32x4 = transmute(vqdmlsl_n_s16(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqdmlsl_n_s32() { + let a: i64x2 = i64x2::new(3, 7); + let b: i32x2 = i32x2::new(1, 2); + let c: i32 = 2; + let e: i64x2 = i64x2::new(-1, -1); + let r: i64x2 = transmute(vqdmlsl_n_s32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqdmlsl_lane_s16() { + let a: i32x4 = i32x4::new(3, 6, 9, 12); + let b: i16x4 = i16x4::new(1, 2, 3, 4); + let c: i16x4 = i16x4::new(0, 2, 2, 0); + let e: i32x4 = i32x4::new(-1, -2, -3, -4); + let r: i32x4 = transmute(vqdmlsl_lane_s16::<2>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqdmlsl_lane_s32() { + let a: i64x2 = i64x2::new(3, 6); + let b: i32x2 = i32x2::new(1, 2); + let c: i32x2 = i32x2::new(0, 2); + let e: i64x2 = i64x2::new(-1, -2); + let r: i64x2 = transmute(vqdmlsl_lane_s32::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqdmulh_s16() { + let a: i16x4 = i16x4::new(0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF); + let b: i16x4 = i16x4::new(2, 2, 2, 2); + let e: i16x4 = i16x4::new(1, 1, 1, 1); + let r: i16x4 = transmute(vqdmulh_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqdmulhq_s16() { + let a: i16x8 = i16x8::new(0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF); + let b: i16x8 = i16x8::new(2, 2, 2, 2, 2, 2, 2, 2); + let e: i16x8 = i16x8::new(1, 1, 1, 1, 1, 1, 1, 1); + let r: i16x8 = transmute(vqdmulhq_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqdmulh_s32() { + let a: i32x2 = i32x2::new(0x7F_FF_FF_FF, 0x7F_FF_FF_FF); + let b: i32x2 = i32x2::new(2, 2); + let e: i32x2 = i32x2::new(1, 1); + let r: i32x2 = transmute(vqdmulh_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqdmulhq_s32() { + let a: i32x4 = i32x4::new(0x7F_FF_FF_FF, 0x7F_FF_FF_FF, 0x7F_FF_FF_FF, 0x7F_FF_FF_FF); + let b: i32x4 = i32x4::new(2, 2, 2, 2); + let e: i32x4 = i32x4::new(1, 1, 1, 1); + let r: i32x4 = transmute(vqdmulhq_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqdmulh_n_s16() { + let a: i16x4 = i16x4::new(0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF); + let b: i16 = 2; + let e: i16x4 = i16x4::new(1, 1, 1, 1); + let r: i16x4 = transmute(vqdmulh_n_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqdmulh_n_s32() { + let a: i32x2 = i32x2::new(0x7F_FF_FF_FF, 0x7F_FF_FF_FF); + let b: i32 = 2; + let e: i32x2 = i32x2::new(1, 1); + let r: i32x2 = transmute(vqdmulh_n_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqdmulhq_nq_s16() { + let a: i16x8 = i16x8::new(0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF); + let b: i16 = 2; + let e: i16x8 = i16x8::new(1, 1, 1, 1, 1, 1, 1, 1); + let r: i16x8 = transmute(vqdmulhq_nq_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqdmulhq_nq_s32() { + let a: i32x4 = i32x4::new(0x7F_FF_FF_FF, 0x7F_FF_FF_FF, 0x7F_FF_FF_FF, 0x7F_FF_FF_FF); + let b: i32 = 2; + let e: i32x4 = i32x4::new(1, 1, 1, 1); + let r: i32x4 = transmute(vqdmulhq_nq_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqmovn_s16() { + let a: i16x8 = i16x8::new(0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF); + let e: i8x8 = i8x8::new(0x7F, 0x7F, 0x7F, 0x7F, 0x7F, 0x7F, 0x7F, 0x7F); + let r: i8x8 = transmute(vqmovn_s16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqmovn_s32() { + let a: i32x4 = i32x4::new(0x7F_FF_FF_FF, 0x7F_FF_FF_FF, 0x7F_FF_FF_FF, 0x7F_FF_FF_FF); + let e: i16x4 = i16x4::new(0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF); + let r: i16x4 = transmute(vqmovn_s32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqmovn_s64() { + let a: i64x2 = i64x2::new(0x7F_FF_FF_FF_FF_FF_FF_FF, 0x7F_FF_FF_FF_FF_FF_FF_FF); + let e: i32x2 = i32x2::new(0x7F_FF_FF_FF, 0x7F_FF_FF_FF); + let r: i32x2 = transmute(vqmovn_s64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqmovn_u16() { + let a: u16x8 = u16x8::new(0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF); + let e: u8x8 = u8x8::new(0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF); + let r: u8x8 = transmute(vqmovn_u16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqmovn_u32() { + let a: u32x4 = u32x4::new(0xFF_FF_FF_FF, 0xFF_FF_FF_FF, 0xFF_FF_FF_FF, 0xFF_FF_FF_FF); + let e: u16x4 = u16x4::new(0xFF_FF, 0xFF_FF, 0xFF_FF, 0xFF_FF); + let r: u16x4 = transmute(vqmovn_u32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqmovn_u64() { + let a: u64x2 = u64x2::new(0xFF_FF_FF_FF_FF_FF_FF_FF, 0xFF_FF_FF_FF_FF_FF_FF_FF); + let e: u32x2 = u32x2::new(0xFF_FF_FF_FF, 0xFF_FF_FF_FF); + let r: u32x2 = transmute(vqmovn_u64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqmovun_s16() { + let a: i16x8 = i16x8::new(-1, -1, -1, -1, -1, -1, -1, -1); + let e: u8x8 = u8x8::new(0, 0, 0, 0, 0, 0, 0, 0); + let r: u8x8 = transmute(vqmovun_s16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqmovun_s32() { + let a: i32x4 = i32x4::new(-1, -1, -1, -1); + let e: u16x4 = u16x4::new(0, 0, 0, 0); + let r: u16x4 = transmute(vqmovun_s32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqmovun_s64() { + let a: i64x2 = i64x2::new(-1, -1); + let e: u32x2 = u32x2::new(0, 0); + let r: u32x2 = transmute(vqmovun_s64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrdmulh_s16() { + let a: i16x4 = i16x4::new(0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF); + let b: i16x4 = i16x4::new(2, 2, 2, 2); + let e: i16x4 = i16x4::new(2, 2, 2, 2); + let r: i16x4 = transmute(vqrdmulh_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrdmulhq_s16() { + let a: i16x8 = i16x8::new(0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF); + let b: i16x8 = i16x8::new(2, 2, 2, 2, 2, 2, 2, 2); + let e: i16x8 = i16x8::new(2, 2, 2, 2, 2, 2, 2, 2); + let r: i16x8 = transmute(vqrdmulhq_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrdmulh_s32() { + let a: i32x2 = i32x2::new(0x7F_FF_FF_FF, 0x7F_FF_FF_FF); + let b: i32x2 = i32x2::new(2, 2); + let e: i32x2 = i32x2::new(2, 2); + let r: i32x2 = transmute(vqrdmulh_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrdmulhq_s32() { + let a: i32x4 = i32x4::new(0x7F_FF_FF_FF, 0x7F_FF_FF_FF, 0x7F_FF_FF_FF, 0x7F_FF_FF_FF); + let b: i32x4 = i32x4::new(2, 2, 2, 2); + let e: i32x4 = i32x4::new(2, 2, 2, 2); + let r: i32x4 = transmute(vqrdmulhq_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrdmulh_n_s16() { + let a: i16x4 = i16x4::new(0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF); + let b: i16 = 2; + let e: i16x4 = i16x4::new(2, 2, 2, 2); + let r: i16x4 = transmute(vqrdmulh_n_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrdmulhq_n_s16() { + let a: i16x8 = i16x8::new(0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF); + let b: i16 = 2; + let e: i16x8 = i16x8::new(2, 2, 2, 2, 2, 2, 2, 2); + let r: i16x8 = transmute(vqrdmulhq_n_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrdmulh_n_s32() { + let a: i32x2 = i32x2::new(0x7F_FF_FF_FF, 0x7F_FF_FF_FF); + let b: i32 = 2; + let e: i32x2 = i32x2::new(2, 2); + let r: i32x2 = transmute(vqrdmulh_n_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrdmulhq_n_s32() { + let a: i32x4 = i32x4::new(0x7F_FF_FF_FF, 0x7F_FF_FF_FF, 0x7F_FF_FF_FF, 0x7F_FF_FF_FF); + let b: i32 = 2; + let e: i32x4 = i32x4::new(2, 2, 2, 2); + let r: i32x4 = transmute(vqrdmulhq_n_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrdmulh_lane_s16() { + let a: i16x4 = i16x4::new(0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF); + let b: i16x4 = i16x4::new(0, 2, 0, 0); + let e: i16x4 = i16x4::new(2, 2, 2, 2); + let r: i16x4 = transmute(vqrdmulh_lane_s16::<1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrdmulh_laneq_s16() { + let a: i16x4 = i16x4::new(0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF); + let b: i16x8 = i16x8::new(0, 2, 0, 0, 0, 0, 0, 0); + let e: i16x4 = i16x4::new(2, 2, 2, 2); + let r: i16x4 = transmute(vqrdmulh_laneq_s16::<1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrdmulhq_lane_s16() { + let a: i16x8 = i16x8::new(0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF); + let b: i16x4 = i16x4::new(0, 2, 0, 0); + let e: i16x8 = i16x8::new(2, 2, 2, 2, 2, 2, 2, 2); + let r: i16x8 = transmute(vqrdmulhq_lane_s16::<1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrdmulhq_laneq_s16() { + let a: i16x8 = i16x8::new(0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF); + let b: i16x8 = i16x8::new(0, 2, 0, 0, 0, 0, 0, 0); + let e: i16x8 = i16x8::new(2, 2, 2, 2, 2, 2, 2, 2); + let r: i16x8 = transmute(vqrdmulhq_laneq_s16::<1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrdmulh_lane_s32() { + let a: i32x2 = i32x2::new(0x7F_FF_FF_FF, 0x7F_FF_FF_FF); + let b: i32x2 = i32x2::new(0, 2); + let e: i32x2 = i32x2::new(2, 2); + let r: i32x2 = transmute(vqrdmulh_lane_s32::<1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrdmulh_laneq_s32() { + let a: i32x2 = i32x2::new(0x7F_FF_FF_FF, 0x7F_FF_FF_FF); + let b: i32x4 = i32x4::new(0, 2, 0, 0); + let e: i32x2 = i32x2::new(2, 2); + let r: i32x2 = transmute(vqrdmulh_laneq_s32::<1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrdmulhq_lane_s32() { + let a: i32x4 = i32x4::new(0x7F_FF_FF_FF, 0x7F_FF_FF_FF, 0x7F_FF_FF_FF, 0x7F_FF_FF_FF); + let b: i32x2 = i32x2::new(0, 2); + let e: i32x4 = i32x4::new(2, 2, 2, 2); + let r: i32x4 = transmute(vqrdmulhq_lane_s32::<1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrdmulhq_laneq_s32() { + let a: i32x4 = i32x4::new(0x7F_FF_FF_FF, 0x7F_FF_FF_FF, 0x7F_FF_FF_FF, 0x7F_FF_FF_FF); + let b: i32x4 = i32x4::new(0, 2, 0, 0); + let e: i32x4 = i32x4::new(2, 2, 2, 2); + let r: i32x4 = transmute(vqrdmulhq_laneq_s32::<1>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrdmlah_s16() { + let a: i16x4 = i16x4::new(1, 1, 1, 1); + let b: i16x4 = i16x4::new(0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF); + let c: i16x4 = i16x4::new(2, 2, 2, 2); + let e: i16x4 = i16x4::new(3, 3, 3, 3); + let r: i16x4 = transmute(vqrdmlah_s16(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrdmlahq_s16() { + let a: i16x8 = i16x8::new(1, 1, 1, 1, 1, 1, 1, 1); + let b: i16x8 = i16x8::new(0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF); + let c: i16x8 = i16x8::new(2, 2, 2, 2, 2, 2, 2, 2); + let e: i16x8 = i16x8::new(3, 3, 3, 3, 3, 3, 3, 3); + let r: i16x8 = transmute(vqrdmlahq_s16(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrdmlah_s32() { + let a: i32x2 = i32x2::new(1, 1); + let b: i32x2 = i32x2::new(0x7F_FF_FF_FF, 0x7F_FF_FF_FF); + let c: i32x2 = i32x2::new(2, 2); + let e: i32x2 = i32x2::new(3, 3); + let r: i32x2 = transmute(vqrdmlah_s32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrdmlahq_s32() { + let a: i32x4 = i32x4::new(1, 1, 1, 1); + let b: i32x4 = i32x4::new(0x7F_FF_FF_FF, 0x7F_FF_FF_FF, 0x7F_FF_FF_FF, 0x7F_FF_FF_FF); + let c: i32x4 = i32x4::new(2, 2, 2, 2); + let e: i32x4 = i32x4::new(3, 3, 3, 3); + let r: i32x4 = transmute(vqrdmlahq_s32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrdmlah_lane_s16() { + let a: i16x4 = i16x4::new(1, 1, 1, 1); + let b: i16x4 = i16x4::new(0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF); + let c: i16x4 = i16x4::new(0, 2, 0, 0); + let e: i16x4 = i16x4::new(3, 3, 3, 3); + let r: i16x4 = transmute(vqrdmlah_lane_s16::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrdmlah_laneq_s16() { + let a: i16x4 = i16x4::new(1, 1, 1, 1); + let b: i16x4 = i16x4::new(0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF); + let c: i16x8 = i16x8::new(0, 2, 0, 0, 0, 0, 0, 0); + let e: i16x4 = i16x4::new(3, 3, 3, 3); + let r: i16x4 = transmute(vqrdmlah_laneq_s16::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrdmlahq_lane_s16() { + let a: i16x8 = i16x8::new(1, 1, 1, 1, 1, 1, 1, 1); + let b: i16x8 = i16x8::new(0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF); + let c: i16x4 = i16x4::new(0, 2, 0, 0); + let e: i16x8 = i16x8::new(3, 3, 3, 3, 3, 3, 3, 3); + let r: i16x8 = transmute(vqrdmlahq_lane_s16::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrdmlahq_laneq_s16() { + let a: i16x8 = i16x8::new(1, 1, 1, 1, 1, 1, 1, 1); + let b: i16x8 = i16x8::new(0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF); + let c: i16x8 = i16x8::new(0, 2, 0, 0, 0, 0, 0, 0); + let e: i16x8 = i16x8::new(3, 3, 3, 3, 3, 3, 3, 3); + let r: i16x8 = transmute(vqrdmlahq_laneq_s16::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrdmlah_lane_s32() { + let a: i32x2 = i32x2::new(1, 1); + let b: i32x2 = i32x2::new(0x7F_FF_FF_FF, 0x7F_FF_FF_FF); + let c: i32x2 = i32x2::new(0, 2); + let e: i32x2 = i32x2::new(3, 3); + let r: i32x2 = transmute(vqrdmlah_lane_s32::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrdmlah_laneq_s32() { + let a: i32x2 = i32x2::new(1, 1); + let b: i32x2 = i32x2::new(0x7F_FF_FF_FF, 0x7F_FF_FF_FF); + let c: i32x4 = i32x4::new(0, 2, 0, 0); + let e: i32x2 = i32x2::new(3, 3); + let r: i32x2 = transmute(vqrdmlah_laneq_s32::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrdmlahq_lane_s32() { + let a: i32x4 = i32x4::new(1, 1, 1, 1); + let b: i32x4 = i32x4::new(0x7F_FF_FF_FF, 0x7F_FF_FF_FF, 0x7F_FF_FF_FF, 0x7F_FF_FF_FF); + let c: i32x2 = i32x2::new(0, 2); + let e: i32x4 = i32x4::new(3, 3, 3, 3); + let r: i32x4 = transmute(vqrdmlahq_lane_s32::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrdmlahq_laneq_s32() { + let a: i32x4 = i32x4::new(1, 1, 1, 1); + let b: i32x4 = i32x4::new(0x7F_FF_FF_FF, 0x7F_FF_FF_FF, 0x7F_FF_FF_FF, 0x7F_FF_FF_FF); + let c: i32x4 = i32x4::new(0, 2, 0, 0); + let e: i32x4 = i32x4::new(3, 3, 3, 3); + let r: i32x4 = transmute(vqrdmlahq_laneq_s32::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrdmlsh_s16() { + let a: i16x4 = i16x4::new(1, 1, 1, 1); + let b: i16x4 = i16x4::new(0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF); + let c: i16x4 = i16x4::new(2, 2, 2, 2); + let e: i16x4 = i16x4::new(-1, -1, -1, -1); + let r: i16x4 = transmute(vqrdmlsh_s16(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrdmlshq_s16() { + let a: i16x8 = i16x8::new(1, 1, 1, 1, 1, 1, 1, 1); + let b: i16x8 = i16x8::new(0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF); + let c: i16x8 = i16x8::new(2, 2, 2, 2, 2, 2, 2, 2); + let e: i16x8 = i16x8::new(-1, -1, -1, -1, -1, -1, -1, -1); + let r: i16x8 = transmute(vqrdmlshq_s16(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrdmlsh_s32() { + let a: i32x2 = i32x2::new(1, 1); + let b: i32x2 = i32x2::new(0x7F_FF_FF_FF, 0x7F_FF_FF_FF); + let c: i32x2 = i32x2::new(2, 2); + let e: i32x2 = i32x2::new(-1, -1); + let r: i32x2 = transmute(vqrdmlsh_s32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrdmlshq_s32() { + let a: i32x4 = i32x4::new(1, 1, 1, 1); + let b: i32x4 = i32x4::new(0x7F_FF_FF_FF, 0x7F_FF_FF_FF, 0x7F_FF_FF_FF, 0x7F_FF_FF_FF); + let c: i32x4 = i32x4::new(2, 2, 2, 2); + let e: i32x4 = i32x4::new(-1, -1, -1, -1); + let r: i32x4 = transmute(vqrdmlshq_s32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrdmlsh_lane_s16() { + let a: i16x4 = i16x4::new(1, 1, 1, 1); + let b: i16x4 = i16x4::new(0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF); + let c: i16x4 = i16x4::new(0, 2, 0, 0); + let e: i16x4 = i16x4::new(-1, -1, -1, -1); + let r: i16x4 = transmute(vqrdmlsh_lane_s16::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrdmlsh_laneq_s16() { + let a: i16x4 = i16x4::new(1, 1, 1, 1); + let b: i16x4 = i16x4::new(0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF); + let c: i16x8 = i16x8::new(0, 2, 0, 0, 0, 0, 0, 0); + let e: i16x4 = i16x4::new(-1, -1, -1, -1); + let r: i16x4 = transmute(vqrdmlsh_laneq_s16::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrdmlshq_lane_s16() { + let a: i16x8 = i16x8::new(1, 1, 1, 1, 1, 1, 1, 1); + let b: i16x8 = i16x8::new(0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF); + let c: i16x4 = i16x4::new(0, 2, 0, 0); + let e: i16x8 = i16x8::new(-1, -1, -1, -1, -1, -1, -1, -1); + let r: i16x8 = transmute(vqrdmlshq_lane_s16::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrdmlshq_laneq_s16() { + let a: i16x8 = i16x8::new(1, 1, 1, 1, 1, 1, 1, 1); + let b: i16x8 = i16x8::new(0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF, 0x7F_FF); + let c: i16x8 = i16x8::new(0, 2, 0, 0, 0, 0, 0, 0); + let e: i16x8 = i16x8::new(-1, -1, -1, -1, -1, -1, -1, -1); + let r: i16x8 = transmute(vqrdmlshq_laneq_s16::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrdmlsh_lane_s32() { + let a: i32x2 = i32x2::new(1, 1); + let b: i32x2 = i32x2::new(0x7F_FF_FF_FF, 0x7F_FF_FF_FF); + let c: i32x2 = i32x2::new(0, 2); + let e: i32x2 = i32x2::new(-1, -1); + let r: i32x2 = transmute(vqrdmlsh_lane_s32::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrdmlsh_laneq_s32() { + let a: i32x2 = i32x2::new(1, 1); + let b: i32x2 = i32x2::new(0x7F_FF_FF_FF, 0x7F_FF_FF_FF); + let c: i32x4 = i32x4::new(0, 2, 0, 0); + let e: i32x2 = i32x2::new(-1, -1); + let r: i32x2 = transmute(vqrdmlsh_laneq_s32::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrdmlshq_lane_s32() { + let a: i32x4 = i32x4::new(1, 1, 1, 1); + let b: i32x4 = i32x4::new(0x7F_FF_FF_FF, 0x7F_FF_FF_FF, 0x7F_FF_FF_FF, 0x7F_FF_FF_FF); + let c: i32x2 = i32x2::new(0, 2); + let e: i32x4 = i32x4::new(-1, -1, -1, -1); + let r: i32x4 = transmute(vqrdmlshq_lane_s32::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrdmlshq_laneq_s32() { + let a: i32x4 = i32x4::new(1, 1, 1, 1); + let b: i32x4 = i32x4::new(0x7F_FF_FF_FF, 0x7F_FF_FF_FF, 0x7F_FF_FF_FF, 0x7F_FF_FF_FF); + let c: i32x4 = i32x4::new(0, 2, 0, 0); + let e: i32x4 = i32x4::new(-1, -1, -1, -1); + let r: i32x4 = transmute(vqrdmlshq_laneq_s32::<1>(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrshl_s8() { + let a: i8x8 = i8x8::new(2, -128, 0x7F, 3, 4, 5, 6, 7); + let b: i8x8 = i8x8::new(2, 2, 2, 2, 2, 2, 2, 2); + let e: i8x8 = i8x8::new(8, -128, 0x7F, 12, 16, 20, 24, 28); + let r: i8x8 = transmute(vqrshl_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrshlq_s8() { + let a: i8x16 = i8x16::new(2, -128, 0x7F, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let b: i8x16 = i8x16::new(2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2); + let e: i8x16 = i8x16::new(8, -128, 0x7F, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60); + let r: i8x16 = transmute(vqrshlq_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrshl_s16() { + let a: i16x4 = i16x4::new(2, -32768, 0x7F_FF, 3); + let b: i16x4 = i16x4::new(2, 2, 2, 2); + let e: i16x4 = i16x4::new(8, -32768, 0x7F_FF, 12); + let r: i16x4 = transmute(vqrshl_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrshlq_s16() { + let a: i16x8 = i16x8::new(2, -32768, 0x7F_FF, 3, 4, 5, 6, 7); + let b: i16x8 = i16x8::new(2, 2, 2, 2, 2, 2, 2, 2); + let e: i16x8 = i16x8::new(8, -32768, 0x7F_FF, 12, 16, 20, 24, 28); + let r: i16x8 = transmute(vqrshlq_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrshl_s32() { + let a: i32x2 = i32x2::new(2, -2147483648); + let b: i32x2 = i32x2::new(2, 2); + let e: i32x2 = i32x2::new(8, -2147483648); + let r: i32x2 = transmute(vqrshl_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrshlq_s32() { + let a: i32x4 = i32x4::new(2, -2147483648, 0x7F_FF_FF_FF, 3); + let b: i32x4 = i32x4::new(2, 2, 2, 2); + let e: i32x4 = i32x4::new(8, -2147483648, 0x7F_FF_FF_FF, 12); + let r: i32x4 = transmute(vqrshlq_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrshl_s64() { + let a: i64x1 = i64x1::new(2); + let b: i64x1 = i64x1::new(2); + let e: i64x1 = i64x1::new(8); + let r: i64x1 = transmute(vqrshl_s64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrshlq_s64() { + let a: i64x2 = i64x2::new(2, -9223372036854775808); + let b: i64x2 = i64x2::new(2, 2); + let e: i64x2 = i64x2::new(8, -9223372036854775808); + let r: i64x2 = transmute(vqrshlq_s64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrshl_u8() { + let a: u8x8 = u8x8::new(2, 0, 0xFF, 3, 4, 5, 6, 7); + let b: i8x8 = i8x8::new(2, 2, 2, 2, 2, 2, 2, 2); + let e: u8x8 = u8x8::new(8, 0, 0xFF, 12, 16, 20, 24, 28); + let r: u8x8 = transmute(vqrshl_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrshlq_u8() { + let a: u8x16 = u8x16::new(2, 0, 0xFF, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let b: i8x16 = i8x16::new(2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2); + let e: u8x16 = u8x16::new(8, 0, 0xFF, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60); + let r: u8x16 = transmute(vqrshlq_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrshl_u16() { + let a: u16x4 = u16x4::new(2, 0, 0xFF_FF, 3); + let b: i16x4 = i16x4::new(2, 2, 2, 2); + let e: u16x4 = u16x4::new(8, 0, 0xFF_FF, 12); + let r: u16x4 = transmute(vqrshl_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrshlq_u16() { + let a: u16x8 = u16x8::new(2, 0, 0xFF_FF, 3, 4, 5, 6, 7); + let b: i16x8 = i16x8::new(2, 2, 2, 2, 2, 2, 2, 2); + let e: u16x8 = u16x8::new(8, 0, 0xFF_FF, 12, 16, 20, 24, 28); + let r: u16x8 = transmute(vqrshlq_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrshl_u32() { + let a: u32x2 = u32x2::new(2, 0); + let b: i32x2 = i32x2::new(2, 2); + let e: u32x2 = u32x2::new(8, 0); + let r: u32x2 = transmute(vqrshl_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrshlq_u32() { + let a: u32x4 = u32x4::new(2, 0, 0xFF_FF_FF_FF, 3); + let b: i32x4 = i32x4::new(2, 2, 2, 2); + let e: u32x4 = u32x4::new(8, 0, 0xFF_FF_FF_FF, 12); + let r: u32x4 = transmute(vqrshlq_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrshl_u64() { + let a: u64x1 = u64x1::new(2); + let b: i64x1 = i64x1::new(2); + let e: u64x1 = u64x1::new(8); + let r: u64x1 = transmute(vqrshl_u64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrshlq_u64() { + let a: u64x2 = u64x2::new(2, 0); + let b: i64x2 = i64x2::new(2, 2); + let e: u64x2 = u64x2::new(8, 0); + let r: u64x2 = transmute(vqrshlq_u64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrshrn_n_s16() { + let a: i16x8 = i16x8::new(-32768, 4, 8, 12, 16, 20, 24, 28); + let e: i8x8 = i8x8::new(-128, 1, 2, 3, 4, 5, 6, 7); + let r: i8x8 = transmute(vqrshrn_n_s16::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrshrn_n_s32() { + let a: i32x4 = i32x4::new(-2147483648, 4, 8, 12); + let e: i16x4 = i16x4::new(-32768, 1, 2, 3); + let r: i16x4 = transmute(vqrshrn_n_s32::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrshrn_n_s64() { + let a: i64x2 = i64x2::new(-9223372036854775808, 4); + let e: i32x2 = i32x2::new(-2147483648, 1); + let r: i32x2 = transmute(vqrshrn_n_s64::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrshrn_n_u16() { + let a: u16x8 = u16x8::new(0, 4, 8, 12, 16, 20, 24, 28); + let e: u8x8 = u8x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let r: u8x8 = transmute(vqrshrn_n_u16::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrshrn_n_u32() { + let a: u32x4 = u32x4::new(0, 4, 8, 12); + let e: u16x4 = u16x4::new(0, 1, 2, 3); + let r: u16x4 = transmute(vqrshrn_n_u32::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrshrn_n_u64() { + let a: u64x2 = u64x2::new(0, 4); + let e: u32x2 = u32x2::new(0, 1); + let r: u32x2 = transmute(vqrshrn_n_u64::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrshrun_n_s16() { + let a: i16x8 = i16x8::new(0, 4, 8, 12, 16, 20, 24, 28); + let e: u8x8 = u8x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let r: u8x8 = transmute(vqrshrun_n_s16::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrshrun_n_s32() { + let a: i32x4 = i32x4::new(0, 4, 8, 12); + let e: u16x4 = u16x4::new(0, 1, 2, 3); + let r: u16x4 = transmute(vqrshrun_n_s32::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqrshrun_n_s64() { + let a: i64x2 = i64x2::new(0, 4); + let e: u32x2 = u32x2::new(0, 1); + let r: u32x2 = transmute(vqrshrun_n_s64::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshl_s8() { + let a: i8x8 = i8x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let b: i8x8 = i8x8::new(2, 2, 2, 2, 2, 2, 2, 2); + let e: i8x8 = i8x8::new(0, 4, 8, 12, 16, 20, 24, 28); + let r: i8x8 = transmute(vqshl_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshlq_s8() { + let a: i8x16 = i8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let b: i8x16 = i8x16::new(2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2); + let e: i8x16 = i8x16::new(0, 4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60); + let r: i8x16 = transmute(vqshlq_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshl_s16() { + let a: i16x4 = i16x4::new(0, 1, 2, 3); + let b: i16x4 = i16x4::new(2, 2, 2, 2); + let e: i16x4 = i16x4::new(0, 4, 8, 12); + let r: i16x4 = transmute(vqshl_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshlq_s16() { + let a: i16x8 = i16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let b: i16x8 = i16x8::new(2, 2, 2, 2, 2, 2, 2, 2); + let e: i16x8 = i16x8::new(0, 4, 8, 12, 16, 20, 24, 28); + let r: i16x8 = transmute(vqshlq_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshl_s32() { + let a: i32x2 = i32x2::new(0, 1); + let b: i32x2 = i32x2::new(2, 2); + let e: i32x2 = i32x2::new(0, 4); + let r: i32x2 = transmute(vqshl_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshlq_s32() { + let a: i32x4 = i32x4::new(0, 1, 2, 3); + let b: i32x4 = i32x4::new(2, 2, 2, 2); + let e: i32x4 = i32x4::new(0, 4, 8, 12); + let r: i32x4 = transmute(vqshlq_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshl_s64() { + let a: i64x1 = i64x1::new(0); + let b: i64x1 = i64x1::new(2); + let e: i64x1 = i64x1::new(0); + let r: i64x1 = transmute(vqshl_s64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshlq_s64() { + let a: i64x2 = i64x2::new(0, 1); + let b: i64x2 = i64x2::new(2, 2); + let e: i64x2 = i64x2::new(0, 4); + let r: i64x2 = transmute(vqshlq_s64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshl_u8() { + let a: u8x8 = u8x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let b: i8x8 = i8x8::new(2, 2, 2, 2, 2, 2, 2, 2); + let e: u8x8 = u8x8::new(0, 4, 8, 12, 16, 20, 24, 28); + let r: u8x8 = transmute(vqshl_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshlq_u8() { + let a: u8x16 = u8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let b: i8x16 = i8x16::new(2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2); + let e: u8x16 = u8x16::new(0, 4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60); + let r: u8x16 = transmute(vqshlq_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshl_u16() { + let a: u16x4 = u16x4::new(0, 1, 2, 3); + let b: i16x4 = i16x4::new(2, 2, 2, 2); + let e: u16x4 = u16x4::new(0, 4, 8, 12); + let r: u16x4 = transmute(vqshl_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshlq_u16() { + let a: u16x8 = u16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let b: i16x8 = i16x8::new(2, 2, 2, 2, 2, 2, 2, 2); + let e: u16x8 = u16x8::new(0, 4, 8, 12, 16, 20, 24, 28); + let r: u16x8 = transmute(vqshlq_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshl_u32() { + let a: u32x2 = u32x2::new(0, 1); + let b: i32x2 = i32x2::new(2, 2); + let e: u32x2 = u32x2::new(0, 4); + let r: u32x2 = transmute(vqshl_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshlq_u32() { + let a: u32x4 = u32x4::new(0, 1, 2, 3); + let b: i32x4 = i32x4::new(2, 2, 2, 2); + let e: u32x4 = u32x4::new(0, 4, 8, 12); + let r: u32x4 = transmute(vqshlq_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshl_u64() { + let a: u64x1 = u64x1::new(0); + let b: i64x1 = i64x1::new(2); + let e: u64x1 = u64x1::new(0); + let r: u64x1 = transmute(vqshl_u64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshlq_u64() { + let a: u64x2 = u64x2::new(0, 1); + let b: i64x2 = i64x2::new(2, 2); + let e: u64x2 = u64x2::new(0, 4); + let r: u64x2 = transmute(vqshlq_u64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshl_n_s8() { + let a: i8x8 = i8x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let e: i8x8 = i8x8::new(0, 4, 8, 12, 16, 20, 24, 28); + let r: i8x8 = transmute(vqshl_n_s8::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshlq_n_s8() { + let a: i8x16 = i8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let e: i8x16 = i8x16::new(0, 4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60); + let r: i8x16 = transmute(vqshlq_n_s8::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshl_n_s16() { + let a: i16x4 = i16x4::new(0, 1, 2, 3); + let e: i16x4 = i16x4::new(0, 4, 8, 12); + let r: i16x4 = transmute(vqshl_n_s16::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshlq_n_s16() { + let a: i16x8 = i16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let e: i16x8 = i16x8::new(0, 4, 8, 12, 16, 20, 24, 28); + let r: i16x8 = transmute(vqshlq_n_s16::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshl_n_s32() { + let a: i32x2 = i32x2::new(0, 1); + let e: i32x2 = i32x2::new(0, 4); + let r: i32x2 = transmute(vqshl_n_s32::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshlq_n_s32() { + let a: i32x4 = i32x4::new(0, 1, 2, 3); + let e: i32x4 = i32x4::new(0, 4, 8, 12); + let r: i32x4 = transmute(vqshlq_n_s32::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshl_n_s64() { + let a: i64x1 = i64x1::new(0); + let e: i64x1 = i64x1::new(0); + let r: i64x1 = transmute(vqshl_n_s64::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshlq_n_s64() { + let a: i64x2 = i64x2::new(0, 1); + let e: i64x2 = i64x2::new(0, 4); + let r: i64x2 = transmute(vqshlq_n_s64::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshl_n_u8() { + let a: u8x8 = u8x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let e: u8x8 = u8x8::new(0, 4, 8, 12, 16, 20, 24, 28); + let r: u8x8 = transmute(vqshl_n_u8::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshlq_n_u8() { + let a: u8x16 = u8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let e: u8x16 = u8x16::new(0, 4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60); + let r: u8x16 = transmute(vqshlq_n_u8::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshl_n_u16() { + let a: u16x4 = u16x4::new(0, 1, 2, 3); + let e: u16x4 = u16x4::new(0, 4, 8, 12); + let r: u16x4 = transmute(vqshl_n_u16::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshlq_n_u16() { + let a: u16x8 = u16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let e: u16x8 = u16x8::new(0, 4, 8, 12, 16, 20, 24, 28); + let r: u16x8 = transmute(vqshlq_n_u16::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshl_n_u32() { + let a: u32x2 = u32x2::new(0, 1); + let e: u32x2 = u32x2::new(0, 4); + let r: u32x2 = transmute(vqshl_n_u32::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshlq_n_u32() { + let a: u32x4 = u32x4::new(0, 1, 2, 3); + let e: u32x4 = u32x4::new(0, 4, 8, 12); + let r: u32x4 = transmute(vqshlq_n_u32::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshl_n_u64() { + let a: u64x1 = u64x1::new(0); + let e: u64x1 = u64x1::new(0); + let r: u64x1 = transmute(vqshl_n_u64::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshlq_n_u64() { + let a: u64x2 = u64x2::new(0, 1); + let e: u64x2 = u64x2::new(0, 4); + let r: u64x2 = transmute(vqshlq_n_u64::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshrn_n_s16() { + let a: i16x8 = i16x8::new(0, 4, 8, 12, 16, 20, 24, 28); + let e: i8x8 = i8x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let r: i8x8 = transmute(vqshrn_n_s16::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshrn_n_s32() { + let a: i32x4 = i32x4::new(0, 4, 8, 12); + let e: i16x4 = i16x4::new(0, 1, 2, 3); + let r: i16x4 = transmute(vqshrn_n_s32::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshrn_n_s64() { + let a: i64x2 = i64x2::new(0, 4); + let e: i32x2 = i32x2::new(0, 1); + let r: i32x2 = transmute(vqshrn_n_s64::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshrn_n_u16() { + let a: u16x8 = u16x8::new(0, 4, 8, 12, 16, 20, 24, 28); + let e: u8x8 = u8x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let r: u8x8 = transmute(vqshrn_n_u16::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshrn_n_u32() { + let a: u32x4 = u32x4::new(0, 4, 8, 12); + let e: u16x4 = u16x4::new(0, 1, 2, 3); + let r: u16x4 = transmute(vqshrn_n_u32::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshrn_n_u64() { + let a: u64x2 = u64x2::new(0, 4); + let e: u32x2 = u32x2::new(0, 1); + let r: u32x2 = transmute(vqshrn_n_u64::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshrun_n_s16() { + let a: i16x8 = i16x8::new(0, 4, 8, 12, 16, 20, 24, 28); + let e: u8x8 = u8x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let r: u8x8 = transmute(vqshrun_n_s16::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshrun_n_s32() { + let a: i32x4 = i32x4::new(0, 4, 8, 12); + let e: u16x4 = u16x4::new(0, 1, 2, 3); + let r: u16x4 = transmute(vqshrun_n_s32::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqshrun_n_s64() { + let a: i64x2 = i64x2::new(0, 4); + let e: u32x2 = u32x2::new(0, 1); + let r: u32x2 = transmute(vqshrun_n_s64::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrsqrte_f32() { + let a: f32x2 = f32x2::new(1.0, 2.0); + let e: f32x2 = f32x2::new(0.998046875, 0.705078125); + let r: f32x2 = transmute(vrsqrte_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrsqrteq_f32() { + let a: f32x4 = f32x4::new(1.0, 2.0, 3.0, 4.0); + let e: f32x4 = f32x4::new(0.998046875, 0.705078125, 0.576171875, 0.4990234375); + let r: f32x4 = transmute(vrsqrteq_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrecpe_f32() { + let a: f32x2 = f32x2::new(4.0, 3.0); + let e: f32x2 = f32x2::new(0.24951171875, 0.3330078125); + let r: f32x2 = transmute(vrecpe_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrecpeq_f32() { + let a: f32x4 = f32x4::new(4.0, 3.0, 2.0, 1.0); + let e: f32x4 = f32x4::new(0.24951171875, 0.3330078125, 0.4990234375, 0.998046875); + let r: f32x4 = transmute(vrecpeq_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_s8_u8() { + let a: u8x8 = u8x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let e: i8x8 = i8x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let r: i8x8 = transmute(vreinterpret_s8_u8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_s8_p8() { + let a: i8x8 = i8x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let e: i8x8 = i8x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let r: i8x8 = transmute(vreinterpret_s8_p8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_s16_p16() { + let a: i16x4 = i16x4::new(0, 1, 2, 3); + let e: i16x4 = i16x4::new(0, 1, 2, 3); + let r: i16x4 = transmute(vreinterpret_s16_p16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_s16_u16() { + let a: u16x4 = u16x4::new(0, 1, 2, 3); + let e: i16x4 = i16x4::new(0, 1, 2, 3); + let r: i16x4 = transmute(vreinterpret_s16_u16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_s32_u32() { + let a: u32x2 = u32x2::new(0, 1); + let e: i32x2 = i32x2::new(0, 1); + let r: i32x2 = transmute(vreinterpret_s32_u32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_s64_u64() { + let a: u64x1 = u64x1::new(0); + let e: i64x1 = i64x1::new(0); + let r: i64x1 = transmute(vreinterpret_s64_u64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_s8_u8() { + let a: u8x16 = u8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let e: i8x16 = i8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let r: i8x16 = transmute(vreinterpretq_s8_u8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_s8_p8() { + let a: i8x16 = i8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let e: i8x16 = i8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let r: i8x16 = transmute(vreinterpretq_s8_p8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_s16_p16() { + let a: i16x8 = i16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let e: i16x8 = i16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let r: i16x8 = transmute(vreinterpretq_s16_p16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_s16_u16() { + let a: u16x8 = u16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let e: i16x8 = i16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let r: i16x8 = transmute(vreinterpretq_s16_u16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_s32_u32() { + let a: u32x4 = u32x4::new(0, 1, 2, 3); + let e: i32x4 = i32x4::new(0, 1, 2, 3); + let r: i32x4 = transmute(vreinterpretq_s32_u32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_s64_u64() { + let a: u64x2 = u64x2::new(0, 1); + let e: i64x2 = i64x2::new(0, 1); + let r: i64x2 = transmute(vreinterpretq_s64_u64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_u8_p8() { + let a: i8x8 = i8x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let e: u8x8 = u8x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let r: u8x8 = transmute(vreinterpret_u8_p8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_u8_s8() { + let a: i8x8 = i8x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let e: u8x8 = u8x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let r: u8x8 = transmute(vreinterpret_u8_s8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_u16_p16() { + let a: i16x4 = i16x4::new(0, 1, 2, 3); + let e: u16x4 = u16x4::new(0, 1, 2, 3); + let r: u16x4 = transmute(vreinterpret_u16_p16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_u16_s16() { + let a: i16x4 = i16x4::new(0, 1, 2, 3); + let e: u16x4 = u16x4::new(0, 1, 2, 3); + let r: u16x4 = transmute(vreinterpret_u16_s16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_u32_s32() { + let a: i32x2 = i32x2::new(0, 1); + let e: u32x2 = u32x2::new(0, 1); + let r: u32x2 = transmute(vreinterpret_u32_s32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_u64_s64() { + let a: i64x1 = i64x1::new(0); + let e: u64x1 = u64x1::new(0); + let r: u64x1 = transmute(vreinterpret_u64_s64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_u8_p8() { + let a: i8x16 = i8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let e: u8x16 = u8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let r: u8x16 = transmute(vreinterpretq_u8_p8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_u8_s8() { + let a: i8x16 = i8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let e: u8x16 = u8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let r: u8x16 = transmute(vreinterpretq_u8_s8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_u16_p16() { + let a: i16x8 = i16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let e: u16x8 = u16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let r: u16x8 = transmute(vreinterpretq_u16_p16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_u16_s16() { + let a: i16x8 = i16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let e: u16x8 = u16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let r: u16x8 = transmute(vreinterpretq_u16_s16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_u32_s32() { + let a: i32x4 = i32x4::new(0, 1, 2, 3); + let e: u32x4 = u32x4::new(0, 1, 2, 3); + let r: u32x4 = transmute(vreinterpretq_u32_s32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_u64_s64() { + let a: i64x2 = i64x2::new(0, 1); + let e: u64x2 = u64x2::new(0, 1); + let r: u64x2 = transmute(vreinterpretq_u64_s64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_p8_s8() { + let a: i8x8 = i8x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let e: i8x8 = i8x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let r: i8x8 = transmute(vreinterpret_p8_s8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_p8_u8() { + let a: u8x8 = u8x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let e: i8x8 = i8x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let r: i8x8 = transmute(vreinterpret_p8_u8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_p16_s16() { + let a: i16x4 = i16x4::new(0, 1, 2, 3); + let e: i16x4 = i16x4::new(0, 1, 2, 3); + let r: i16x4 = transmute(vreinterpret_p16_s16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_p16_u16() { + let a: u16x4 = u16x4::new(0, 1, 2, 3); + let e: i16x4 = i16x4::new(0, 1, 2, 3); + let r: i16x4 = transmute(vreinterpret_p16_u16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_p8_s8() { + let a: i8x16 = i8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let e: i8x16 = i8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let r: i8x16 = transmute(vreinterpretq_p8_s8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_p8_u8() { + let a: u8x16 = u8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let e: i8x16 = i8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let r: i8x16 = transmute(vreinterpretq_p8_u8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_p16_s16() { + let a: i16x8 = i16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let e: i16x8 = i16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let r: i16x8 = transmute(vreinterpretq_p16_s16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_p16_u16() { + let a: u16x8 = u16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let e: i16x8 = i16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let r: i16x8 = transmute(vreinterpretq_p16_u16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_s8_s16() { + let a: i16x4 = i16x4::new(0, 1, 2, 3); + let e: i8x8 = i8x8::new(0, 0, 1, 0, 2, 0, 3, 0); + let r: i8x8 = transmute(vreinterpret_s8_s16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_s8_u16() { + let a: u16x4 = u16x4::new(0, 1, 2, 3); + let e: i8x8 = i8x8::new(0, 0, 1, 0, 2, 0, 3, 0); + let r: i8x8 = transmute(vreinterpret_s8_u16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_s8_p16() { + let a: i16x4 = i16x4::new(0, 1, 2, 3); + let e: i8x8 = i8x8::new(0, 0, 1, 0, 2, 0, 3, 0); + let r: i8x8 = transmute(vreinterpret_s8_p16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_s16_s32() { + let a: i32x2 = i32x2::new(0, 1); + let e: i16x4 = i16x4::new(0, 0, 1, 0); + let r: i16x4 = transmute(vreinterpret_s16_s32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_s16_u32() { + let a: u32x2 = u32x2::new(0, 1); + let e: i16x4 = i16x4::new(0, 0, 1, 0); + let r: i16x4 = transmute(vreinterpret_s16_u32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_s32_s64() { + let a: i64x1 = i64x1::new(0); + let e: i32x2 = i32x2::new(0, 0); + let r: i32x2 = transmute(vreinterpret_s32_s64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_s32_u64() { + let a: u64x1 = u64x1::new(0); + let e: i32x2 = i32x2::new(0, 0); + let r: i32x2 = transmute(vreinterpret_s32_u64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_s8_s16() { + let a: i16x8 = i16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let e: i8x16 = i8x16::new(0, 0, 1, 0, 2, 0, 3, 0, 4, 0, 5, 0, 6, 0, 7, 0); + let r: i8x16 = transmute(vreinterpretq_s8_s16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_s8_u16() { + let a: u16x8 = u16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let e: i8x16 = i8x16::new(0, 0, 1, 0, 2, 0, 3, 0, 4, 0, 5, 0, 6, 0, 7, 0); + let r: i8x16 = transmute(vreinterpretq_s8_u16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_s8_p16() { + let a: i16x8 = i16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let e: i8x16 = i8x16::new(0, 0, 1, 0, 2, 0, 3, 0, 4, 0, 5, 0, 6, 0, 7, 0); + let r: i8x16 = transmute(vreinterpretq_s8_p16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_s16_s32() { + let a: i32x4 = i32x4::new(0, 1, 2, 3); + let e: i16x8 = i16x8::new(0, 0, 1, 0, 2, 0, 3, 0); + let r: i16x8 = transmute(vreinterpretq_s16_s32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_s16_u32() { + let a: u32x4 = u32x4::new(0, 1, 2, 3); + let e: i16x8 = i16x8::new(0, 0, 1, 0, 2, 0, 3, 0); + let r: i16x8 = transmute(vreinterpretq_s16_u32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_s32_s64() { + let a: i64x2 = i64x2::new(0, 1); + let e: i32x4 = i32x4::new(0, 0, 1, 0); + let r: i32x4 = transmute(vreinterpretq_s32_s64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_s32_u64() { + let a: u64x2 = u64x2::new(0, 1); + let e: i32x4 = i32x4::new(0, 0, 1, 0); + let r: i32x4 = transmute(vreinterpretq_s32_u64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_u8_p16() { + let a: i16x4 = i16x4::new(0, 1, 2, 3); + let e: u8x8 = u8x8::new(0, 0, 1, 0, 2, 0, 3, 0); + let r: u8x8 = transmute(vreinterpret_u8_p16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_u8_s16() { + let a: i16x4 = i16x4::new(0, 1, 2, 3); + let e: u8x8 = u8x8::new(0, 0, 1, 0, 2, 0, 3, 0); + let r: u8x8 = transmute(vreinterpret_u8_s16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_u8_u16() { + let a: u16x4 = u16x4::new(0, 1, 2, 3); + let e: u8x8 = u8x8::new(0, 0, 1, 0, 2, 0, 3, 0); + let r: u8x8 = transmute(vreinterpret_u8_u16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_u16_s32() { + let a: i32x2 = i32x2::new(0, 1); + let e: u16x4 = u16x4::new(0, 0, 1, 0); + let r: u16x4 = transmute(vreinterpret_u16_s32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_u16_u32() { + let a: u32x2 = u32x2::new(0, 1); + let e: u16x4 = u16x4::new(0, 0, 1, 0); + let r: u16x4 = transmute(vreinterpret_u16_u32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_u32_s64() { + let a: i64x1 = i64x1::new(0); + let e: u32x2 = u32x2::new(0, 0); + let r: u32x2 = transmute(vreinterpret_u32_s64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_u32_u64() { + let a: u64x1 = u64x1::new(0); + let e: u32x2 = u32x2::new(0, 0); + let r: u32x2 = transmute(vreinterpret_u32_u64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_u8_p16() { + let a: i16x8 = i16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let e: u8x16 = u8x16::new(0, 0, 1, 0, 2, 0, 3, 0, 4, 0, 5, 0, 6, 0, 7, 0); + let r: u8x16 = transmute(vreinterpretq_u8_p16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_u8_s16() { + let a: i16x8 = i16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let e: u8x16 = u8x16::new(0, 0, 1, 0, 2, 0, 3, 0, 4, 0, 5, 0, 6, 0, 7, 0); + let r: u8x16 = transmute(vreinterpretq_u8_s16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_u8_u16() { + let a: u16x8 = u16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let e: u8x16 = u8x16::new(0, 0, 1, 0, 2, 0, 3, 0, 4, 0, 5, 0, 6, 0, 7, 0); + let r: u8x16 = transmute(vreinterpretq_u8_u16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_u16_s32() { + let a: i32x4 = i32x4::new(0, 1, 2, 3); + let e: u16x8 = u16x8::new(0, 0, 1, 0, 2, 0, 3, 0); + let r: u16x8 = transmute(vreinterpretq_u16_s32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_u16_u32() { + let a: u32x4 = u32x4::new(0, 1, 2, 3); + let e: u16x8 = u16x8::new(0, 0, 1, 0, 2, 0, 3, 0); + let r: u16x8 = transmute(vreinterpretq_u16_u32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_u32_s64() { + let a: i64x2 = i64x2::new(0, 1); + let e: u32x4 = u32x4::new(0, 0, 1, 0); + let r: u32x4 = transmute(vreinterpretq_u32_s64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_u32_u64() { + let a: u64x2 = u64x2::new(0, 1); + let e: u32x4 = u32x4::new(0, 0, 1, 0); + let r: u32x4 = transmute(vreinterpretq_u32_u64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_p8_p16() { + let a: i16x4 = i16x4::new(0, 1, 2, 3); + let e: i8x8 = i8x8::new(0, 0, 1, 0, 2, 0, 3, 0); + let r: i8x8 = transmute(vreinterpret_p8_p16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_p8_s16() { + let a: i16x4 = i16x4::new(0, 1, 2, 3); + let e: i8x8 = i8x8::new(0, 0, 1, 0, 2, 0, 3, 0); + let r: i8x8 = transmute(vreinterpret_p8_s16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_p8_u16() { + let a: u16x4 = u16x4::new(0, 1, 2, 3); + let e: i8x8 = i8x8::new(0, 0, 1, 0, 2, 0, 3, 0); + let r: i8x8 = transmute(vreinterpret_p8_u16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_p16_s32() { + let a: i32x2 = i32x2::new(0, 1); + let e: i16x4 = i16x4::new(0, 0, 1, 0); + let r: i16x4 = transmute(vreinterpret_p16_s32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_p16_u32() { + let a: u32x2 = u32x2::new(0, 1); + let e: i16x4 = i16x4::new(0, 0, 1, 0); + let r: i16x4 = transmute(vreinterpret_p16_u32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_p8_p16() { + let a: i16x8 = i16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let e: i8x16 = i8x16::new(0, 0, 1, 0, 2, 0, 3, 0, 4, 0, 5, 0, 6, 0, 7, 0); + let r: i8x16 = transmute(vreinterpretq_p8_p16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_p8_s16() { + let a: i16x8 = i16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let e: i8x16 = i8x16::new(0, 0, 1, 0, 2, 0, 3, 0, 4, 0, 5, 0, 6, 0, 7, 0); + let r: i8x16 = transmute(vreinterpretq_p8_s16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_p8_u16() { + let a: u16x8 = u16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let e: i8x16 = i8x16::new(0, 0, 1, 0, 2, 0, 3, 0, 4, 0, 5, 0, 6, 0, 7, 0); + let r: i8x16 = transmute(vreinterpretq_p8_u16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_p16_s32() { + let a: i32x4 = i32x4::new(0, 1, 2, 3); + let e: i16x8 = i16x8::new(0, 0, 1, 0, 2, 0, 3, 0); + let r: i16x8 = transmute(vreinterpretq_p16_s32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_p16_u32() { + let a: u32x4 = u32x4::new(0, 1, 2, 3); + let e: i16x8 = i16x8::new(0, 0, 1, 0, 2, 0, 3, 0); + let r: i16x8 = transmute(vreinterpretq_p16_u32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_s16_p8() { + let a: i8x8 = i8x8::new(0, 0, 1, 0, 2, 0, 3, 0); + let e: i16x4 = i16x4::new(0, 1, 2, 3); + let r: i16x4 = transmute(vreinterpret_s16_p8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_s16_s8() { + let a: i8x8 = i8x8::new(0, 0, 1, 0, 2, 0, 3, 0); + let e: i16x4 = i16x4::new(0, 1, 2, 3); + let r: i16x4 = transmute(vreinterpret_s16_s8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_s16_u8() { + let a: u8x8 = u8x8::new(0, 0, 1, 0, 2, 0, 3, 0); + let e: i16x4 = i16x4::new(0, 1, 2, 3); + let r: i16x4 = transmute(vreinterpret_s16_u8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_s32_p16() { + let a: i16x4 = i16x4::new(0, 0, 1, 0); + let e: i32x2 = i32x2::new(0, 1); + let r: i32x2 = transmute(vreinterpret_s32_p16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_s32_s16() { + let a: i16x4 = i16x4::new(0, 0, 1, 0); + let e: i32x2 = i32x2::new(0, 1); + let r: i32x2 = transmute(vreinterpret_s32_s16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_s32_u16() { + let a: u16x4 = u16x4::new(0, 0, 1, 0); + let e: i32x2 = i32x2::new(0, 1); + let r: i32x2 = transmute(vreinterpret_s32_u16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_s64_s32() { + let a: i32x2 = i32x2::new(0, 0); + let e: i64x1 = i64x1::new(0); + let r: i64x1 = transmute(vreinterpret_s64_s32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_s64_u32() { + let a: u32x2 = u32x2::new(0, 0); + let e: i64x1 = i64x1::new(0); + let r: i64x1 = transmute(vreinterpret_s64_u32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_s16_p8() { + let a: i8x16 = i8x16::new(0, 0, 1, 0, 2, 0, 3, 0, 4, 0, 5, 0, 6, 0, 7, 0); + let e: i16x8 = i16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let r: i16x8 = transmute(vreinterpretq_s16_p8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_s16_s8() { + let a: i8x16 = i8x16::new(0, 0, 1, 0, 2, 0, 3, 0, 4, 0, 5, 0, 6, 0, 7, 0); + let e: i16x8 = i16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let r: i16x8 = transmute(vreinterpretq_s16_s8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_s16_u8() { + let a: u8x16 = u8x16::new(0, 0, 1, 0, 2, 0, 3, 0, 4, 0, 5, 0, 6, 0, 7, 0); + let e: i16x8 = i16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let r: i16x8 = transmute(vreinterpretq_s16_u8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_s32_p16() { + let a: i16x8 = i16x8::new(0, 0, 1, 0, 2, 0, 3, 0); + let e: i32x4 = i32x4::new(0, 1, 2, 3); + let r: i32x4 = transmute(vreinterpretq_s32_p16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_s32_s16() { + let a: i16x8 = i16x8::new(0, 0, 1, 0, 2, 0, 3, 0); + let e: i32x4 = i32x4::new(0, 1, 2, 3); + let r: i32x4 = transmute(vreinterpretq_s32_s16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_s32_u16() { + let a: u16x8 = u16x8::new(0, 0, 1, 0, 2, 0, 3, 0); + let e: i32x4 = i32x4::new(0, 1, 2, 3); + let r: i32x4 = transmute(vreinterpretq_s32_u16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_s64_s32() { + let a: i32x4 = i32x4::new(0, 0, 1, 0); + let e: i64x2 = i64x2::new(0, 1); + let r: i64x2 = transmute(vreinterpretq_s64_s32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_s64_u32() { + let a: u32x4 = u32x4::new(0, 0, 1, 0); + let e: i64x2 = i64x2::new(0, 1); + let r: i64x2 = transmute(vreinterpretq_s64_u32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_u16_p8() { + let a: i8x8 = i8x8::new(0, 0, 1, 0, 2, 0, 3, 0); + let e: u16x4 = u16x4::new(0, 1, 2, 3); + let r: u16x4 = transmute(vreinterpret_u16_p8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_u16_s8() { + let a: i8x8 = i8x8::new(0, 0, 1, 0, 2, 0, 3, 0); + let e: u16x4 = u16x4::new(0, 1, 2, 3); + let r: u16x4 = transmute(vreinterpret_u16_s8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_u16_u8() { + let a: u8x8 = u8x8::new(0, 0, 1, 0, 2, 0, 3, 0); + let e: u16x4 = u16x4::new(0, 1, 2, 3); + let r: u16x4 = transmute(vreinterpret_u16_u8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_u32_p16() { + let a: i16x4 = i16x4::new(0, 0, 1, 0); + let e: u32x2 = u32x2::new(0, 1); + let r: u32x2 = transmute(vreinterpret_u32_p16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_u32_s16() { + let a: i16x4 = i16x4::new(0, 0, 1, 0); + let e: u32x2 = u32x2::new(0, 1); + let r: u32x2 = transmute(vreinterpret_u32_s16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_u32_u16() { + let a: u16x4 = u16x4::new(0, 0, 1, 0); + let e: u32x2 = u32x2::new(0, 1); + let r: u32x2 = transmute(vreinterpret_u32_u16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_u64_s32() { + let a: i32x2 = i32x2::new(0, 0); + let e: u64x1 = u64x1::new(0); + let r: u64x1 = transmute(vreinterpret_u64_s32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_u64_u32() { + let a: u32x2 = u32x2::new(0, 0); + let e: u64x1 = u64x1::new(0); + let r: u64x1 = transmute(vreinterpret_u64_u32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_u16_p8() { + let a: i8x16 = i8x16::new(0, 0, 1, 0, 2, 0, 3, 0, 4, 0, 5, 0, 6, 0, 7, 0); + let e: u16x8 = u16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let r: u16x8 = transmute(vreinterpretq_u16_p8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_u16_s8() { + let a: i8x16 = i8x16::new(0, 0, 1, 0, 2, 0, 3, 0, 4, 0, 5, 0, 6, 0, 7, 0); + let e: u16x8 = u16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let r: u16x8 = transmute(vreinterpretq_u16_s8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_u16_u8() { + let a: u8x16 = u8x16::new(0, 0, 1, 0, 2, 0, 3, 0, 4, 0, 5, 0, 6, 0, 7, 0); + let e: u16x8 = u16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let r: u16x8 = transmute(vreinterpretq_u16_u8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_u32_p16() { + let a: i16x8 = i16x8::new(0, 0, 1, 0, 2, 0, 3, 0); + let e: u32x4 = u32x4::new(0, 1, 2, 3); + let r: u32x4 = transmute(vreinterpretq_u32_p16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_u32_s16() { + let a: i16x8 = i16x8::new(0, 0, 1, 0, 2, 0, 3, 0); + let e: u32x4 = u32x4::new(0, 1, 2, 3); + let r: u32x4 = transmute(vreinterpretq_u32_s16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_u32_u16() { + let a: u16x8 = u16x8::new(0, 0, 1, 0, 2, 0, 3, 0); + let e: u32x4 = u32x4::new(0, 1, 2, 3); + let r: u32x4 = transmute(vreinterpretq_u32_u16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_u64_s32() { + let a: i32x4 = i32x4::new(0, 0, 1, 0); + let e: u64x2 = u64x2::new(0, 1); + let r: u64x2 = transmute(vreinterpretq_u64_s32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_u64_u32() { + let a: u32x4 = u32x4::new(0, 0, 1, 0); + let e: u64x2 = u64x2::new(0, 1); + let r: u64x2 = transmute(vreinterpretq_u64_u32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_p16_p8() { + let a: i8x8 = i8x8::new(0, 0, 1, 0, 2, 0, 3, 0); + let e: i16x4 = i16x4::new(0, 1, 2, 3); + let r: i16x4 = transmute(vreinterpret_p16_p8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_p16_s8() { + let a: i8x8 = i8x8::new(0, 0, 1, 0, 2, 0, 3, 0); + let e: i16x4 = i16x4::new(0, 1, 2, 3); + let r: i16x4 = transmute(vreinterpret_p16_s8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_p16_u8() { + let a: u8x8 = u8x8::new(0, 0, 1, 0, 2, 0, 3, 0); + let e: i16x4 = i16x4::new(0, 1, 2, 3); + let r: i16x4 = transmute(vreinterpret_p16_u8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_p16_p8() { + let a: i8x16 = i8x16::new(0, 0, 1, 0, 2, 0, 3, 0, 4, 0, 5, 0, 6, 0, 7, 0); + let e: i16x8 = i16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let r: i16x8 = transmute(vreinterpretq_p16_p8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_p16_s8() { + let a: i8x16 = i8x16::new(0, 0, 1, 0, 2, 0, 3, 0, 4, 0, 5, 0, 6, 0, 7, 0); + let e: i16x8 = i16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let r: i16x8 = transmute(vreinterpretq_p16_s8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_p16_u8() { + let a: u8x16 = u8x16::new(0, 0, 1, 0, 2, 0, 3, 0, 4, 0, 5, 0, 6, 0, 7, 0); + let e: i16x8 = i16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let r: i16x8 = transmute(vreinterpretq_p16_u8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_s8_s32() { + let a: i32x2 = i32x2::new(0, 1); + let e: i8x8 = i8x8::new(0, 0, 0, 0, 1, 0, 0, 0); + let r: i8x8 = transmute(vreinterpret_s8_s32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_s8_u32() { + let a: u32x2 = u32x2::new(0, 1); + let e: i8x8 = i8x8::new(0, 0, 0, 0, 1, 0, 0, 0); + let r: i8x8 = transmute(vreinterpret_s8_u32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_s16_s64() { + let a: i64x1 = i64x1::new(0); + let e: i16x4 = i16x4::new(0, 0, 0, 0); + let r: i16x4 = transmute(vreinterpret_s16_s64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_s16_u64() { + let a: u64x1 = u64x1::new(0); + let e: i16x4 = i16x4::new(0, 0, 0, 0); + let r: i16x4 = transmute(vreinterpret_s16_u64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_s8_s32() { + let a: i32x4 = i32x4::new(0, 1, 2, 3); + let e: i8x16 = i8x16::new(0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 3, 0, 0, 0); + let r: i8x16 = transmute(vreinterpretq_s8_s32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_s8_u32() { + let a: u32x4 = u32x4::new(0, 1, 2, 3); + let e: i8x16 = i8x16::new(0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 3, 0, 0, 0); + let r: i8x16 = transmute(vreinterpretq_s8_u32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_s16_s64() { + let a: i64x2 = i64x2::new(0, 1); + let e: i16x8 = i16x8::new(0, 0, 0, 0, 1, 0, 0, 0); + let r: i16x8 = transmute(vreinterpretq_s16_s64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_s16_u64() { + let a: u64x2 = u64x2::new(0, 1); + let e: i16x8 = i16x8::new(0, 0, 0, 0, 1, 0, 0, 0); + let r: i16x8 = transmute(vreinterpretq_s16_u64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_u8_s32() { + let a: i32x2 = i32x2::new(0, 1); + let e: u8x8 = u8x8::new(0, 0, 0, 0, 1, 0, 0, 0); + let r: u8x8 = transmute(vreinterpret_u8_s32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_u8_u32() { + let a: u32x2 = u32x2::new(0, 1); + let e: u8x8 = u8x8::new(0, 0, 0, 0, 1, 0, 0, 0); + let r: u8x8 = transmute(vreinterpret_u8_u32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_u16_s64() { + let a: i64x1 = i64x1::new(0); + let e: u16x4 = u16x4::new(0, 0, 0, 0); + let r: u16x4 = transmute(vreinterpret_u16_s64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_u16_u64() { + let a: u64x1 = u64x1::new(0); + let e: u16x4 = u16x4::new(0, 0, 0, 0); + let r: u16x4 = transmute(vreinterpret_u16_u64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_u8_s32() { + let a: i32x4 = i32x4::new(0, 1, 2, 3); + let e: u8x16 = u8x16::new(0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 3, 0, 0, 0); + let r: u8x16 = transmute(vreinterpretq_u8_s32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_u8_u32() { + let a: u32x4 = u32x4::new(0, 1, 2, 3); + let e: u8x16 = u8x16::new(0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 3, 0, 0, 0); + let r: u8x16 = transmute(vreinterpretq_u8_u32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_u16_s64() { + let a: i64x2 = i64x2::new(0, 1); + let e: u16x8 = u16x8::new(0, 0, 0, 0, 1, 0, 0, 0); + let r: u16x8 = transmute(vreinterpretq_u16_s64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_u16_u64() { + let a: u64x2 = u64x2::new(0, 1); + let e: u16x8 = u16x8::new(0, 0, 0, 0, 1, 0, 0, 0); + let r: u16x8 = transmute(vreinterpretq_u16_u64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_p8_s32() { + let a: i32x2 = i32x2::new(0, 1); + let e: i8x8 = i8x8::new(0, 0, 0, 0, 1, 0, 0, 0); + let r: i8x8 = transmute(vreinterpret_p8_s32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_p8_u32() { + let a: u32x2 = u32x2::new(0, 1); + let e: i8x8 = i8x8::new(0, 0, 0, 0, 1, 0, 0, 0); + let r: i8x8 = transmute(vreinterpret_p8_u32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_p16_s64() { + let a: i64x1 = i64x1::new(0); + let e: i16x4 = i16x4::new(0, 0, 0, 0); + let r: i16x4 = transmute(vreinterpret_p16_s64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_p16_u64() { + let a: u64x1 = u64x1::new(0); + let e: i16x4 = i16x4::new(0, 0, 0, 0); + let r: i16x4 = transmute(vreinterpret_p16_u64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_p8_s32() { + let a: i32x4 = i32x4::new(0, 1, 2, 3); + let e: i8x16 = i8x16::new(0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 3, 0, 0, 0); + let r: i8x16 = transmute(vreinterpretq_p8_s32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_p8_u32() { + let a: u32x4 = u32x4::new(0, 1, 2, 3); + let e: i8x16 = i8x16::new(0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 3, 0, 0, 0); + let r: i8x16 = transmute(vreinterpretq_p8_u32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_p16_s64() { + let a: i64x2 = i64x2::new(0, 1); + let e: i16x8 = i16x8::new(0, 0, 0, 0, 1, 0, 0, 0); + let r: i16x8 = transmute(vreinterpretq_p16_s64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_p16_u64() { + let a: u64x2 = u64x2::new(0, 1); + let e: i16x8 = i16x8::new(0, 0, 0, 0, 1, 0, 0, 0); + let r: i16x8 = transmute(vreinterpretq_p16_u64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_s32_p8() { + let a: i8x8 = i8x8::new(0, 0, 0, 0, 1, 0, 0, 0); + let e: i32x2 = i32x2::new(0, 1); + let r: i32x2 = transmute(vreinterpret_s32_p8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_s32_s8() { + let a: i8x8 = i8x8::new(0, 0, 0, 0, 1, 0, 0, 0); + let e: i32x2 = i32x2::new(0, 1); + let r: i32x2 = transmute(vreinterpret_s32_s8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_s32_u8() { + let a: u8x8 = u8x8::new(0, 0, 0, 0, 1, 0, 0, 0); + let e: i32x2 = i32x2::new(0, 1); + let r: i32x2 = transmute(vreinterpret_s32_u8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_s64_p16() { + let a: i16x4 = i16x4::new(0, 0, 0, 0); + let e: i64x1 = i64x1::new(0); + let r: i64x1 = transmute(vreinterpret_s64_p16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_s64_s16() { + let a: i16x4 = i16x4::new(0, 0, 0, 0); + let e: i64x1 = i64x1::new(0); + let r: i64x1 = transmute(vreinterpret_s64_s16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_s64_u16() { + let a: u16x4 = u16x4::new(0, 0, 0, 0); + let e: i64x1 = i64x1::new(0); + let r: i64x1 = transmute(vreinterpret_s64_u16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_s32_p8() { + let a: i8x16 = i8x16::new(0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 3, 0, 0, 0); + let e: i32x4 = i32x4::new(0, 1, 2, 3); + let r: i32x4 = transmute(vreinterpretq_s32_p8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_s32_s8() { + let a: i8x16 = i8x16::new(0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 3, 0, 0, 0); + let e: i32x4 = i32x4::new(0, 1, 2, 3); + let r: i32x4 = transmute(vreinterpretq_s32_s8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_s32_u8() { + let a: u8x16 = u8x16::new(0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 3, 0, 0, 0); + let e: i32x4 = i32x4::new(0, 1, 2, 3); + let r: i32x4 = transmute(vreinterpretq_s32_u8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_s64_p16() { + let a: i16x8 = i16x8::new(0, 0, 0, 0, 1, 0, 0, 0); + let e: i64x2 = i64x2::new(0, 1); + let r: i64x2 = transmute(vreinterpretq_s64_p16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_s64_s16() { + let a: i16x8 = i16x8::new(0, 0, 0, 0, 1, 0, 0, 0); + let e: i64x2 = i64x2::new(0, 1); + let r: i64x2 = transmute(vreinterpretq_s64_s16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_s64_u16() { + let a: u16x8 = u16x8::new(0, 0, 0, 0, 1, 0, 0, 0); + let e: i64x2 = i64x2::new(0, 1); + let r: i64x2 = transmute(vreinterpretq_s64_u16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_u32_p8() { + let a: i8x8 = i8x8::new(0, 0, 0, 0, 1, 0, 0, 0); + let e: u32x2 = u32x2::new(0, 1); + let r: u32x2 = transmute(vreinterpret_u32_p8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_u32_s8() { + let a: i8x8 = i8x8::new(0, 0, 0, 0, 1, 0, 0, 0); + let e: u32x2 = u32x2::new(0, 1); + let r: u32x2 = transmute(vreinterpret_u32_s8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_u32_u8() { + let a: u8x8 = u8x8::new(0, 0, 0, 0, 1, 0, 0, 0); + let e: u32x2 = u32x2::new(0, 1); + let r: u32x2 = transmute(vreinterpret_u32_u8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_u64_p16() { + let a: i16x4 = i16x4::new(0, 0, 0, 0); + let e: u64x1 = u64x1::new(0); + let r: u64x1 = transmute(vreinterpret_u64_p16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_u64_s16() { + let a: i16x4 = i16x4::new(0, 0, 0, 0); + let e: u64x1 = u64x1::new(0); + let r: u64x1 = transmute(vreinterpret_u64_s16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_u64_u16() { + let a: u16x4 = u16x4::new(0, 0, 0, 0); + let e: u64x1 = u64x1::new(0); + let r: u64x1 = transmute(vreinterpret_u64_u16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_u32_p8() { + let a: i8x16 = i8x16::new(0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 3, 0, 0, 0); + let e: u32x4 = u32x4::new(0, 1, 2, 3); + let r: u32x4 = transmute(vreinterpretq_u32_p8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_u32_s8() { + let a: i8x16 = i8x16::new(0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 3, 0, 0, 0); + let e: u32x4 = u32x4::new(0, 1, 2, 3); + let r: u32x4 = transmute(vreinterpretq_u32_s8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_u32_u8() { + let a: u8x16 = u8x16::new(0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 3, 0, 0, 0); + let e: u32x4 = u32x4::new(0, 1, 2, 3); + let r: u32x4 = transmute(vreinterpretq_u32_u8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_u64_p16() { + let a: i16x8 = i16x8::new(0, 0, 0, 0, 1, 0, 0, 0); + let e: u64x2 = u64x2::new(0, 1); + let r: u64x2 = transmute(vreinterpretq_u64_p16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_u64_s16() { + let a: i16x8 = i16x8::new(0, 0, 0, 0, 1, 0, 0, 0); + let e: u64x2 = u64x2::new(0, 1); + let r: u64x2 = transmute(vreinterpretq_u64_s16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_u64_u16() { + let a: u16x8 = u16x8::new(0, 0, 0, 0, 1, 0, 0, 0); + let e: u64x2 = u64x2::new(0, 1); + let r: u64x2 = transmute(vreinterpretq_u64_u16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_s8_s64() { + let a: i64x1 = i64x1::new(0); + let e: i8x8 = i8x8::new(0, 0, 0, 0, 0, 0, 0, 0); + let r: i8x8 = transmute(vreinterpret_s8_s64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_s8_u64() { + let a: u64x1 = u64x1::new(0); + let e: i8x8 = i8x8::new(0, 0, 0, 0, 0, 0, 0, 0); + let r: i8x8 = transmute(vreinterpret_s8_u64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_u8_s64() { + let a: i64x1 = i64x1::new(0); + let e: u8x8 = u8x8::new(0, 0, 0, 0, 0, 0, 0, 0); + let r: u8x8 = transmute(vreinterpret_u8_s64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_u8_u64() { + let a: u64x1 = u64x1::new(0); + let e: u8x8 = u8x8::new(0, 0, 0, 0, 0, 0, 0, 0); + let r: u8x8 = transmute(vreinterpret_u8_u64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_p8_s64() { + let a: i64x1 = i64x1::new(0); + let e: i8x8 = i8x8::new(0, 0, 0, 0, 0, 0, 0, 0); + let r: i8x8 = transmute(vreinterpret_p8_s64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_p8_u64() { + let a: u64x1 = u64x1::new(0); + let e: i8x8 = i8x8::new(0, 0, 0, 0, 0, 0, 0, 0); + let r: i8x8 = transmute(vreinterpret_p8_u64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_s8_s64() { + let a: i64x2 = i64x2::new(0, 1); + let e: i8x16 = i8x16::new(0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0); + let r: i8x16 = transmute(vreinterpretq_s8_s64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_s8_u64() { + let a: u64x2 = u64x2::new(0, 1); + let e: i8x16 = i8x16::new(0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0); + let r: i8x16 = transmute(vreinterpretq_s8_u64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_u8_s64() { + let a: i64x2 = i64x2::new(0, 1); + let e: u8x16 = u8x16::new(0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0); + let r: u8x16 = transmute(vreinterpretq_u8_s64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_u8_u64() { + let a: u64x2 = u64x2::new(0, 1); + let e: u8x16 = u8x16::new(0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0); + let r: u8x16 = transmute(vreinterpretq_u8_u64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_p8_s64() { + let a: i64x2 = i64x2::new(0, 1); + let e: i8x16 = i8x16::new(0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0); + let r: i8x16 = transmute(vreinterpretq_p8_s64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_p8_u64() { + let a: u64x2 = u64x2::new(0, 1); + let e: i8x16 = i8x16::new(0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0); + let r: i8x16 = transmute(vreinterpretq_p8_u64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_s64_p8() { + let a: i8x8 = i8x8::new(0, 0, 0, 0, 0, 0, 0, 0); + let e: i64x1 = i64x1::new(0); + let r: i64x1 = transmute(vreinterpret_s64_p8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_s64_s8() { + let a: i8x8 = i8x8::new(0, 0, 0, 0, 0, 0, 0, 0); + let e: i64x1 = i64x1::new(0); + let r: i64x1 = transmute(vreinterpret_s64_s8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_s64_u8() { + let a: u8x8 = u8x8::new(0, 0, 0, 0, 0, 0, 0, 0); + let e: i64x1 = i64x1::new(0); + let r: i64x1 = transmute(vreinterpret_s64_u8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_u64_p8() { + let a: i8x8 = i8x8::new(0, 0, 0, 0, 0, 0, 0, 0); + let e: u64x1 = u64x1::new(0); + let r: u64x1 = transmute(vreinterpret_u64_p8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_u64_s8() { + let a: i8x8 = i8x8::new(0, 0, 0, 0, 0, 0, 0, 0); + let e: u64x1 = u64x1::new(0); + let r: u64x1 = transmute(vreinterpret_u64_s8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_u64_u8() { + let a: u8x8 = u8x8::new(0, 0, 0, 0, 0, 0, 0, 0); + let e: u64x1 = u64x1::new(0); + let r: u64x1 = transmute(vreinterpret_u64_u8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_s64_p8() { + let a: i8x16 = i8x16::new(0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0); + let e: i64x2 = i64x2::new(0, 1); + let r: i64x2 = transmute(vreinterpretq_s64_p8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_s64_s8() { + let a: i8x16 = i8x16::new(0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0); + let e: i64x2 = i64x2::new(0, 1); + let r: i64x2 = transmute(vreinterpretq_s64_s8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_s64_u8() { + let a: u8x16 = u8x16::new(0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0); + let e: i64x2 = i64x2::new(0, 1); + let r: i64x2 = transmute(vreinterpretq_s64_u8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_u64_p8() { + let a: i8x16 = i8x16::new(0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0); + let e: u64x2 = u64x2::new(0, 1); + let r: u64x2 = transmute(vreinterpretq_u64_p8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_u64_s8() { + let a: i8x16 = i8x16::new(0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0); + let e: u64x2 = u64x2::new(0, 1); + let r: u64x2 = transmute(vreinterpretq_u64_s8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_u64_u8() { + let a: u8x16 = u8x16::new(0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0); + let e: u64x2 = u64x2::new(0, 1); + let r: u64x2 = transmute(vreinterpretq_u64_u8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_s8_f32() { + let a: f32x2 = f32x2::new(0., 0.); + let e: i8x8 = i8x8::new(0, 0, 0, 0, 0, 0, 0, 0); + let r: i8x8 = transmute(vreinterpret_s8_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_s16_f32() { + let a: f32x2 = f32x2::new(0., 0.); + let e: i16x4 = i16x4::new(0, 0, 0, 0); + let r: i16x4 = transmute(vreinterpret_s16_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_s32_f32() { + let a: f32x2 = f32x2::new(0., 0.); + let e: i32x2 = i32x2::new(0, 0); + let r: i32x2 = transmute(vreinterpret_s32_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_s64_f32() { + let a: f32x2 = f32x2::new(0., 0.); + let e: i64x1 = i64x1::new(0); + let r: i64x1 = transmute(vreinterpret_s64_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_s8_f32() { + let a: f32x4 = f32x4::new(0., 0., 0., 0.); + let e: i8x16 = i8x16::new(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + let r: i8x16 = transmute(vreinterpretq_s8_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_s16_f32() { + let a: f32x4 = f32x4::new(0., 0., 0., 0.); + let e: i16x8 = i16x8::new(0, 0, 0, 0, 0, 0, 0, 0); + let r: i16x8 = transmute(vreinterpretq_s16_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_s32_f32() { + let a: f32x4 = f32x4::new(0., 0., 0., 0.); + let e: i32x4 = i32x4::new(0, 0, 0, 0); + let r: i32x4 = transmute(vreinterpretq_s32_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_s64_f32() { + let a: f32x4 = f32x4::new(0., 0., 0., 0.); + let e: i64x2 = i64x2::new(0, 0); + let r: i64x2 = transmute(vreinterpretq_s64_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_u8_f32() { + let a: f32x2 = f32x2::new(0., 0.); + let e: u8x8 = u8x8::new(0, 0, 0, 0, 0, 0, 0, 0); + let r: u8x8 = transmute(vreinterpret_u8_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_u16_f32() { + let a: f32x2 = f32x2::new(0., 0.); + let e: u16x4 = u16x4::new(0, 0, 0, 0); + let r: u16x4 = transmute(vreinterpret_u16_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_u32_f32() { + let a: f32x2 = f32x2::new(0., 0.); + let e: u32x2 = u32x2::new(0, 0); + let r: u32x2 = transmute(vreinterpret_u32_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_u64_f32() { + let a: f32x2 = f32x2::new(0., 0.); + let e: u64x1 = u64x1::new(0); + let r: u64x1 = transmute(vreinterpret_u64_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_u8_f32() { + let a: f32x4 = f32x4::new(0., 0., 0., 0.); + let e: u8x16 = u8x16::new(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + let r: u8x16 = transmute(vreinterpretq_u8_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_u16_f32() { + let a: f32x4 = f32x4::new(0., 0., 0., 0.); + let e: u16x8 = u16x8::new(0, 0, 0, 0, 0, 0, 0, 0); + let r: u16x8 = transmute(vreinterpretq_u16_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_u32_f32() { + let a: f32x4 = f32x4::new(0., 0., 0., 0.); + let e: u32x4 = u32x4::new(0, 0, 0, 0); + let r: u32x4 = transmute(vreinterpretq_u32_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_u64_f32() { + let a: f32x4 = f32x4::new(0., 0., 0., 0.); + let e: u64x2 = u64x2::new(0, 0); + let r: u64x2 = transmute(vreinterpretq_u64_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_p8_f32() { + let a: f32x2 = f32x2::new(0., 0.); + let e: i8x8 = i8x8::new(0, 0, 0, 0, 0, 0, 0, 0); + let r: i8x8 = transmute(vreinterpret_p8_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_p16_f32() { + let a: f32x2 = f32x2::new(0., 0.); + let e: i16x4 = i16x4::new(0, 0, 0, 0); + let r: i16x4 = transmute(vreinterpret_p16_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_p8_f32() { + let a: f32x4 = f32x4::new(0., 0., 0., 0.); + let e: i8x16 = i8x16::new(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + let r: i8x16 = transmute(vreinterpretq_p8_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_p16_f32() { + let a: f32x4 = f32x4::new(0., 0., 0., 0.); + let e: i16x8 = i16x8::new(0, 0, 0, 0, 0, 0, 0, 0); + let r: i16x8 = transmute(vreinterpretq_p16_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_f32_s8() { + let a: i8x8 = i8x8::new(0, 0, 0, 0, 0, 0, 0, 0); + let e: f32x2 = f32x2::new(0., 0.); + let r: f32x2 = transmute(vreinterpret_f32_s8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_f32_s16() { + let a: i16x4 = i16x4::new(0, 0, 0, 0); + let e: f32x2 = f32x2::new(0., 0.); + let r: f32x2 = transmute(vreinterpret_f32_s16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_f32_s32() { + let a: i32x2 = i32x2::new(0, 0); + let e: f32x2 = f32x2::new(0., 0.); + let r: f32x2 = transmute(vreinterpret_f32_s32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_f32_s64() { + let a: i64x1 = i64x1::new(0); + let e: f32x2 = f32x2::new(0., 0.); + let r: f32x2 = transmute(vreinterpret_f32_s64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_f32_s8() { + let a: i8x16 = i8x16::new(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + let e: f32x4 = f32x4::new(0., 0., 0., 0.); + let r: f32x4 = transmute(vreinterpretq_f32_s8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_f32_s16() { + let a: i16x8 = i16x8::new(0, 0, 0, 0, 0, 0, 0, 0); + let e: f32x4 = f32x4::new(0., 0., 0., 0.); + let r: f32x4 = transmute(vreinterpretq_f32_s16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_f32_s32() { + let a: i32x4 = i32x4::new(0, 0, 0, 0); + let e: f32x4 = f32x4::new(0., 0., 0., 0.); + let r: f32x4 = transmute(vreinterpretq_f32_s32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_f32_s64() { + let a: i64x2 = i64x2::new(0, 0); + let e: f32x4 = f32x4::new(0., 0., 0., 0.); + let r: f32x4 = transmute(vreinterpretq_f32_s64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_f32_u8() { + let a: u8x8 = u8x8::new(0, 0, 0, 0, 0, 0, 0, 0); + let e: f32x2 = f32x2::new(0., 0.); + let r: f32x2 = transmute(vreinterpret_f32_u8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_f32_u16() { + let a: u16x4 = u16x4::new(0, 0, 0, 0); + let e: f32x2 = f32x2::new(0., 0.); + let r: f32x2 = transmute(vreinterpret_f32_u16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_f32_u32() { + let a: u32x2 = u32x2::new(0, 0); + let e: f32x2 = f32x2::new(0., 0.); + let r: f32x2 = transmute(vreinterpret_f32_u32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_f32_u64() { + let a: u64x1 = u64x1::new(0); + let e: f32x2 = f32x2::new(0., 0.); + let r: f32x2 = transmute(vreinterpret_f32_u64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_f32_u8() { + let a: u8x16 = u8x16::new(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + let e: f32x4 = f32x4::new(0., 0., 0., 0.); + let r: f32x4 = transmute(vreinterpretq_f32_u8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_f32_u16() { + let a: u16x8 = u16x8::new(0, 0, 0, 0, 0, 0, 0, 0); + let e: f32x4 = f32x4::new(0., 0., 0., 0.); + let r: f32x4 = transmute(vreinterpretq_f32_u16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_f32_u32() { + let a: u32x4 = u32x4::new(0, 0, 0, 0); + let e: f32x4 = f32x4::new(0., 0., 0., 0.); + let r: f32x4 = transmute(vreinterpretq_f32_u32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_f32_u64() { + let a: u64x2 = u64x2::new(0, 0); + let e: f32x4 = f32x4::new(0., 0., 0., 0.); + let r: f32x4 = transmute(vreinterpretq_f32_u64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_f32_p8() { + let a: i8x8 = i8x8::new(0, 0, 0, 0, 0, 0, 0, 0); + let e: f32x2 = f32x2::new(0., 0.); + let r: f32x2 = transmute(vreinterpret_f32_p8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpret_f32_p16() { + let a: i16x4 = i16x4::new(0, 0, 0, 0); + let e: f32x2 = f32x2::new(0., 0.); + let r: f32x2 = transmute(vreinterpret_f32_p16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_f32_p8() { + let a: i8x16 = i8x16::new(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + let e: f32x4 = f32x4::new(0., 0., 0., 0.); + let r: f32x4 = transmute(vreinterpretq_f32_p8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vreinterpretq_f32_p16() { + let a: i16x8 = i16x8::new(0, 0, 0, 0, 0, 0, 0, 0); + let e: f32x4 = f32x4::new(0., 0., 0., 0.); + let r: f32x4 = transmute(vreinterpretq_f32_p16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrshl_s8() { + let a: i8x8 = i8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let b: i8x8 = i8x8::new(2, 2, 2, 2, 2, 2, 2, 2); + let e: i8x8 = i8x8::new(4, 8, 12, 16, 20, 24, 28, 32); + let r: i8x8 = transmute(vrshl_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrshlq_s8() { + let a: i8x16 = i8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let b: i8x16 = i8x16::new(2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2); + let e: i8x16 = i8x16::new(4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60, 64); + let r: i8x16 = transmute(vrshlq_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrshl_s16() { + let a: i16x4 = i16x4::new(1, 2, 3, 4); + let b: i16x4 = i16x4::new(2, 2, 2, 2); + let e: i16x4 = i16x4::new(4, 8, 12, 16); + let r: i16x4 = transmute(vrshl_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrshlq_s16() { + let a: i16x8 = i16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let b: i16x8 = i16x8::new(2, 2, 2, 2, 2, 2, 2, 2); + let e: i16x8 = i16x8::new(4, 8, 12, 16, 20, 24, 28, 32); + let r: i16x8 = transmute(vrshlq_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrshl_s32() { + let a: i32x2 = i32x2::new(1, 2); + let b: i32x2 = i32x2::new(2, 2); + let e: i32x2 = i32x2::new(4, 8); + let r: i32x2 = transmute(vrshl_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrshlq_s32() { + let a: i32x4 = i32x4::new(1, 2, 3, 4); + let b: i32x4 = i32x4::new(2, 2, 2, 2); + let e: i32x4 = i32x4::new(4, 8, 12, 16); + let r: i32x4 = transmute(vrshlq_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrshl_s64() { + let a: i64x1 = i64x1::new(1); + let b: i64x1 = i64x1::new(2); + let e: i64x1 = i64x1::new(4); + let r: i64x1 = transmute(vrshl_s64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrshlq_s64() { + let a: i64x2 = i64x2::new(1, 2); + let b: i64x2 = i64x2::new(2, 2); + let e: i64x2 = i64x2::new(4, 8); + let r: i64x2 = transmute(vrshlq_s64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrshl_u8() { + let a: u8x8 = u8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let b: i8x8 = i8x8::new(2, 2, 2, 2, 2, 2, 2, 2); + let e: u8x8 = u8x8::new(4, 8, 12, 16, 20, 24, 28, 32); + let r: u8x8 = transmute(vrshl_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrshlq_u8() { + let a: u8x16 = u8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let b: i8x16 = i8x16::new(2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2); + let e: u8x16 = u8x16::new(4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60, 64); + let r: u8x16 = transmute(vrshlq_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrshl_u16() { + let a: u16x4 = u16x4::new(1, 2, 3, 4); + let b: i16x4 = i16x4::new(2, 2, 2, 2); + let e: u16x4 = u16x4::new(4, 8, 12, 16); + let r: u16x4 = transmute(vrshl_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrshlq_u16() { + let a: u16x8 = u16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let b: i16x8 = i16x8::new(2, 2, 2, 2, 2, 2, 2, 2); + let e: u16x8 = u16x8::new(4, 8, 12, 16, 20, 24, 28, 32); + let r: u16x8 = transmute(vrshlq_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrshl_u32() { + let a: u32x2 = u32x2::new(1, 2); + let b: i32x2 = i32x2::new(2, 2); + let e: u32x2 = u32x2::new(4, 8); + let r: u32x2 = transmute(vrshl_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrshlq_u32() { + let a: u32x4 = u32x4::new(1, 2, 3, 4); + let b: i32x4 = i32x4::new(2, 2, 2, 2); + let e: u32x4 = u32x4::new(4, 8, 12, 16); + let r: u32x4 = transmute(vrshlq_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrshl_u64() { + let a: u64x1 = u64x1::new(1); + let b: i64x1 = i64x1::new(2); + let e: u64x1 = u64x1::new(4); + let r: u64x1 = transmute(vrshl_u64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrshlq_u64() { + let a: u64x2 = u64x2::new(1, 2); + let b: i64x2 = i64x2::new(2, 2); + let e: u64x2 = u64x2::new(4, 8); + let r: u64x2 = transmute(vrshlq_u64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrshr_n_s8() { + let a: i8x8 = i8x8::new(4, 8, 12, 16, 20, 24, 28, 32); + let e: i8x8 = i8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let r: i8x8 = transmute(vrshr_n_s8::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrshrq_n_s8() { + let a: i8x16 = i8x16::new(4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60, 64); + let e: i8x16 = i8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let r: i8x16 = transmute(vrshrq_n_s8::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrshr_n_s16() { + let a: i16x4 = i16x4::new(4, 8, 12, 16); + let e: i16x4 = i16x4::new(1, 2, 3, 4); + let r: i16x4 = transmute(vrshr_n_s16::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrshrq_n_s16() { + let a: i16x8 = i16x8::new(4, 8, 12, 16, 20, 24, 28, 32); + let e: i16x8 = i16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let r: i16x8 = transmute(vrshrq_n_s16::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrshr_n_s32() { + let a: i32x2 = i32x2::new(4, 8); + let e: i32x2 = i32x2::new(1, 2); + let r: i32x2 = transmute(vrshr_n_s32::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrshrq_n_s32() { + let a: i32x4 = i32x4::new(4, 8, 12, 16); + let e: i32x4 = i32x4::new(1, 2, 3, 4); + let r: i32x4 = transmute(vrshrq_n_s32::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrshr_n_s64() { + let a: i64x1 = i64x1::new(4); + let e: i64x1 = i64x1::new(1); + let r: i64x1 = transmute(vrshr_n_s64::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrshrq_n_s64() { + let a: i64x2 = i64x2::new(4, 8); + let e: i64x2 = i64x2::new(1, 2); + let r: i64x2 = transmute(vrshrq_n_s64::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrshr_n_u8() { + let a: u8x8 = u8x8::new(4, 8, 12, 16, 20, 24, 28, 32); + let e: u8x8 = u8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let r: u8x8 = transmute(vrshr_n_u8::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrshrq_n_u8() { + let a: u8x16 = u8x16::new(4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60, 64); + let e: u8x16 = u8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let r: u8x16 = transmute(vrshrq_n_u8::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrshr_n_u16() { + let a: u16x4 = u16x4::new(4, 8, 12, 16); + let e: u16x4 = u16x4::new(1, 2, 3, 4); + let r: u16x4 = transmute(vrshr_n_u16::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrshrq_n_u16() { + let a: u16x8 = u16x8::new(4, 8, 12, 16, 20, 24, 28, 32); + let e: u16x8 = u16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let r: u16x8 = transmute(vrshrq_n_u16::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrshr_n_u32() { + let a: u32x2 = u32x2::new(4, 8); + let e: u32x2 = u32x2::new(1, 2); + let r: u32x2 = transmute(vrshr_n_u32::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrshrq_n_u32() { + let a: u32x4 = u32x4::new(4, 8, 12, 16); + let e: u32x4 = u32x4::new(1, 2, 3, 4); + let r: u32x4 = transmute(vrshrq_n_u32::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrshr_n_u64() { + let a: u64x1 = u64x1::new(4); + let e: u64x1 = u64x1::new(1); + let r: u64x1 = transmute(vrshr_n_u64::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrshrq_n_u64() { + let a: u64x2 = u64x2::new(4, 8); + let e: u64x2 = u64x2::new(1, 2); + let r: u64x2 = transmute(vrshrq_n_u64::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrshrn_n_s16() { + let a: i16x8 = i16x8::new(4, 8, 12, 16, 20, 24, 28, 32); + let e: i8x8 = i8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let r: i8x8 = transmute(vrshrn_n_s16::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrshrn_n_s32() { + let a: i32x4 = i32x4::new(4, 8, 12, 16); + let e: i16x4 = i16x4::new(1, 2, 3, 4); + let r: i16x4 = transmute(vrshrn_n_s32::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrshrn_n_s64() { + let a: i64x2 = i64x2::new(4, 8); + let e: i32x2 = i32x2::new(1, 2); + let r: i32x2 = transmute(vrshrn_n_s64::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrshrn_n_u16() { + let a: u16x8 = u16x8::new(4, 8, 12, 16, 20, 24, 28, 32); + let e: u8x8 = u8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let r: u8x8 = transmute(vrshrn_n_u16::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrshrn_n_u32() { + let a: u32x4 = u32x4::new(4, 8, 12, 16); + let e: u16x4 = u16x4::new(1, 2, 3, 4); + let r: u16x4 = transmute(vrshrn_n_u32::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrshrn_n_u64() { + let a: u64x2 = u64x2::new(4, 8); + let e: u32x2 = u32x2::new(1, 2); + let r: u32x2 = transmute(vrshrn_n_u64::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrsra_n_s8() { + let a: i8x8 = i8x8::new(1, 1, 1, 1, 1, 1, 1, 1); + let b: i8x8 = i8x8::new(4, 8, 12, 16, 20, 24, 28, 32); + let e: i8x8 = i8x8::new(2, 3, 4, 5, 6, 7, 8, 9); + let r: i8x8 = transmute(vrsra_n_s8::<2>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrsraq_n_s8() { + let a: i8x16 = i8x16::new(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1); + let b: i8x16 = i8x16::new(4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60, 64); + let e: i8x16 = i8x16::new(2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17); + let r: i8x16 = transmute(vrsraq_n_s8::<2>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrsra_n_s16() { + let a: i16x4 = i16x4::new(1, 1, 1, 1); + let b: i16x4 = i16x4::new(4, 8, 12, 16); + let e: i16x4 = i16x4::new(2, 3, 4, 5); + let r: i16x4 = transmute(vrsra_n_s16::<2>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrsraq_n_s16() { + let a: i16x8 = i16x8::new(1, 1, 1, 1, 1, 1, 1, 1); + let b: i16x8 = i16x8::new(4, 8, 12, 16, 20, 24, 28, 32); + let e: i16x8 = i16x8::new(2, 3, 4, 5, 6, 7, 8, 9); + let r: i16x8 = transmute(vrsraq_n_s16::<2>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrsra_n_s32() { + let a: i32x2 = i32x2::new(1, 1); + let b: i32x2 = i32x2::new(4, 8); + let e: i32x2 = i32x2::new(2, 3); + let r: i32x2 = transmute(vrsra_n_s32::<2>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrsraq_n_s32() { + let a: i32x4 = i32x4::new(1, 1, 1, 1); + let b: i32x4 = i32x4::new(4, 8, 12, 16); + let e: i32x4 = i32x4::new(2, 3, 4, 5); + let r: i32x4 = transmute(vrsraq_n_s32::<2>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrsra_n_s64() { + let a: i64x1 = i64x1::new(1); + let b: i64x1 = i64x1::new(4); + let e: i64x1 = i64x1::new(2); + let r: i64x1 = transmute(vrsra_n_s64::<2>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrsraq_n_s64() { + let a: i64x2 = i64x2::new(1, 1); + let b: i64x2 = i64x2::new(4, 8); + let e: i64x2 = i64x2::new(2, 3); + let r: i64x2 = transmute(vrsraq_n_s64::<2>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrsra_n_u8() { + let a: u8x8 = u8x8::new(1, 1, 1, 1, 1, 1, 1, 1); + let b: u8x8 = u8x8::new(4, 8, 12, 16, 20, 24, 28, 32); + let e: u8x8 = u8x8::new(2, 3, 4, 5, 6, 7, 8, 9); + let r: u8x8 = transmute(vrsra_n_u8::<2>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrsraq_n_u8() { + let a: u8x16 = u8x16::new(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1); + let b: u8x16 = u8x16::new(4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60, 64); + let e: u8x16 = u8x16::new(2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17); + let r: u8x16 = transmute(vrsraq_n_u8::<2>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrsra_n_u16() { + let a: u16x4 = u16x4::new(1, 1, 1, 1); + let b: u16x4 = u16x4::new(4, 8, 12, 16); + let e: u16x4 = u16x4::new(2, 3, 4, 5); + let r: u16x4 = transmute(vrsra_n_u16::<2>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrsraq_n_u16() { + let a: u16x8 = u16x8::new(1, 1, 1, 1, 1, 1, 1, 1); + let b: u16x8 = u16x8::new(4, 8, 12, 16, 20, 24, 28, 32); + let e: u16x8 = u16x8::new(2, 3, 4, 5, 6, 7, 8, 9); + let r: u16x8 = transmute(vrsraq_n_u16::<2>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrsra_n_u32() { + let a: u32x2 = u32x2::new(1, 1); + let b: u32x2 = u32x2::new(4, 8); + let e: u32x2 = u32x2::new(2, 3); + let r: u32x2 = transmute(vrsra_n_u32::<2>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrsraq_n_u32() { + let a: u32x4 = u32x4::new(1, 1, 1, 1); + let b: u32x4 = u32x4::new(4, 8, 12, 16); + let e: u32x4 = u32x4::new(2, 3, 4, 5); + let r: u32x4 = transmute(vrsraq_n_u32::<2>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrsra_n_u64() { + let a: u64x1 = u64x1::new(1); + let b: u64x1 = u64x1::new(4); + let e: u64x1 = u64x1::new(2); + let r: u64x1 = transmute(vrsra_n_u64::<2>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vrsraq_n_u64() { + let a: u64x2 = u64x2::new(1, 1); + let b: u64x2 = u64x2::new(4, 8); + let e: u64x2 = u64x2::new(2, 3); + let r: u64x2 = transmute(vrsraq_n_u64::<2>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vset_lane_s8() { + let a: i8 = 1; + let b: i8x8 = i8x8::new(0, 2, 3, 4, 5, 6, 7, 8); + let e: i8x8 = i8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let r: i8x8 = transmute(vset_lane_s8::<0>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vset_lane_s16() { + let a: i16 = 1; + let b: i16x4 = i16x4::new(0, 2, 3, 4); + let e: i16x4 = i16x4::new(1, 2, 3, 4); + let r: i16x4 = transmute(vset_lane_s16::<0>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vset_lane_s32() { + let a: i32 = 1; + let b: i32x2 = i32x2::new(0, 2); + let e: i32x2 = i32x2::new(1, 2); + let r: i32x2 = transmute(vset_lane_s32::<0>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vset_lane_s64() { + let a: i64 = 1; + let b: i64x1 = i64x1::new(0); + let e: i64x1 = i64x1::new(1); + let r: i64x1 = transmute(vset_lane_s64::<0>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vset_lane_u8() { + let a: u8 = 1; + let b: u8x8 = u8x8::new(0, 2, 3, 4, 5, 6, 7, 8); + let e: u8x8 = u8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let r: u8x8 = transmute(vset_lane_u8::<0>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vset_lane_u16() { + let a: u16 = 1; + let b: u16x4 = u16x4::new(0, 2, 3, 4); + let e: u16x4 = u16x4::new(1, 2, 3, 4); + let r: u16x4 = transmute(vset_lane_u16::<0>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vset_lane_u32() { + let a: u32 = 1; + let b: u32x2 = u32x2::new(0, 2); + let e: u32x2 = u32x2::new(1, 2); + let r: u32x2 = transmute(vset_lane_u32::<0>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vset_lane_u64() { + let a: u64 = 1; + let b: u64x1 = u64x1::new(0); + let e: u64x1 = u64x1::new(1); + let r: u64x1 = transmute(vset_lane_u64::<0>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vset_lane_p8() { + let a: p8 = 1; + let b: i8x8 = i8x8::new(0, 2, 3, 4, 5, 6, 7, 8); + let e: i8x8 = i8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let r: i8x8 = transmute(vset_lane_p8::<0>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vset_lane_p16() { + let a: p16 = 1; + let b: i16x4 = i16x4::new(0, 2, 3, 4); + let e: i16x4 = i16x4::new(1, 2, 3, 4); + let r: i16x4 = transmute(vset_lane_p16::<0>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vset_lane_p64() { + let a: p64 = 1; + let b: i64x1 = i64x1::new(0); + let e: i64x1 = i64x1::new(1); + let r: i64x1 = transmute(vset_lane_p64::<0>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsetq_lane_s8() { + let a: i8 = 1; + let b: i8x16 = i8x16::new(0, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let e: i8x16 = i8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let r: i8x16 = transmute(vsetq_lane_s8::<0>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsetq_lane_s16() { + let a: i16 = 1; + let b: i16x8 = i16x8::new(0, 2, 3, 4, 5, 6, 7, 8); + let e: i16x8 = i16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let r: i16x8 = transmute(vsetq_lane_s16::<0>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsetq_lane_s32() { + let a: i32 = 1; + let b: i32x4 = i32x4::new(0, 2, 3, 4); + let e: i32x4 = i32x4::new(1, 2, 3, 4); + let r: i32x4 = transmute(vsetq_lane_s32::<0>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsetq_lane_s64() { + let a: i64 = 1; + let b: i64x2 = i64x2::new(0, 2); + let e: i64x2 = i64x2::new(1, 2); + let r: i64x2 = transmute(vsetq_lane_s64::<0>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsetq_lane_u8() { + let a: u8 = 1; + let b: u8x16 = u8x16::new(0, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let e: u8x16 = u8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let r: u8x16 = transmute(vsetq_lane_u8::<0>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsetq_lane_u16() { + let a: u16 = 1; + let b: u16x8 = u16x8::new(0, 2, 3, 4, 5, 6, 7, 8); + let e: u16x8 = u16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let r: u16x8 = transmute(vsetq_lane_u16::<0>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsetq_lane_u32() { + let a: u32 = 1; + let b: u32x4 = u32x4::new(0, 2, 3, 4); + let e: u32x4 = u32x4::new(1, 2, 3, 4); + let r: u32x4 = transmute(vsetq_lane_u32::<0>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsetq_lane_u64() { + let a: u64 = 1; + let b: u64x2 = u64x2::new(0, 2); + let e: u64x2 = u64x2::new(1, 2); + let r: u64x2 = transmute(vsetq_lane_u64::<0>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsetq_lane_p8() { + let a: p8 = 1; + let b: i8x16 = i8x16::new(0, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let e: i8x16 = i8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let r: i8x16 = transmute(vsetq_lane_p8::<0>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsetq_lane_p16() { + let a: p16 = 1; + let b: i16x8 = i16x8::new(0, 2, 3, 4, 5, 6, 7, 8); + let e: i16x8 = i16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let r: i16x8 = transmute(vsetq_lane_p16::<0>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsetq_lane_p64() { + let a: p64 = 1; + let b: i64x2 = i64x2::new(0, 2); + let e: i64x2 = i64x2::new(1, 2); + let r: i64x2 = transmute(vsetq_lane_p64::<0>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vset_lane_f32() { + let a: f32 = 1.; + let b: f32x2 = f32x2::new(0., 2.); + let e: f32x2 = f32x2::new(1., 2.); + let r: f32x2 = transmute(vset_lane_f32::<0>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsetq_lane_f32() { + let a: f32 = 1.; + let b: f32x4 = f32x4::new(0., 2., 3., 4.); + let e: f32x4 = f32x4::new(1., 2., 3., 4.); + let r: f32x4 = transmute(vsetq_lane_f32::<0>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vshl_s8() { + let a: i8x8 = i8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let b: i8x8 = i8x8::new(2, 2, 2, 2, 2, 2, 2, 2); + let e: i8x8 = i8x8::new(4, 8, 12, 16, 20, 24, 28, 32); + let r: i8x8 = transmute(vshl_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vshlq_s8() { + let a: i8x16 = i8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let b: i8x16 = i8x16::new(2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2); + let e: i8x16 = i8x16::new(4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60, 64); + let r: i8x16 = transmute(vshlq_s8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vshl_s16() { + let a: i16x4 = i16x4::new(1, 2, 3, 4); + let b: i16x4 = i16x4::new(2, 2, 2, 2); + let e: i16x4 = i16x4::new(4, 8, 12, 16); + let r: i16x4 = transmute(vshl_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vshlq_s16() { + let a: i16x8 = i16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let b: i16x8 = i16x8::new(2, 2, 2, 2, 2, 2, 2, 2); + let e: i16x8 = i16x8::new(4, 8, 12, 16, 20, 24, 28, 32); + let r: i16x8 = transmute(vshlq_s16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vshl_s32() { + let a: i32x2 = i32x2::new(1, 2); + let b: i32x2 = i32x2::new(2, 2); + let e: i32x2 = i32x2::new(4, 8); + let r: i32x2 = transmute(vshl_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vshlq_s32() { + let a: i32x4 = i32x4::new(1, 2, 3, 4); + let b: i32x4 = i32x4::new(2, 2, 2, 2); + let e: i32x4 = i32x4::new(4, 8, 12, 16); + let r: i32x4 = transmute(vshlq_s32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vshl_s64() { + let a: i64x1 = i64x1::new(1); + let b: i64x1 = i64x1::new(2); + let e: i64x1 = i64x1::new(4); + let r: i64x1 = transmute(vshl_s64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vshlq_s64() { + let a: i64x2 = i64x2::new(1, 2); + let b: i64x2 = i64x2::new(2, 2); + let e: i64x2 = i64x2::new(4, 8); + let r: i64x2 = transmute(vshlq_s64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vshl_u8() { + let a: u8x8 = u8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let b: i8x8 = i8x8::new(2, 2, 2, 2, 2, 2, 2, 2); + let e: u8x8 = u8x8::new(4, 8, 12, 16, 20, 24, 28, 32); + let r: u8x8 = transmute(vshl_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vshlq_u8() { + let a: u8x16 = u8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let b: i8x16 = i8x16::new(2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2); + let e: u8x16 = u8x16::new(4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60, 64); + let r: u8x16 = transmute(vshlq_u8(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vshl_u16() { + let a: u16x4 = u16x4::new(1, 2, 3, 4); + let b: i16x4 = i16x4::new(2, 2, 2, 2); + let e: u16x4 = u16x4::new(4, 8, 12, 16); + let r: u16x4 = transmute(vshl_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vshlq_u16() { + let a: u16x8 = u16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let b: i16x8 = i16x8::new(2, 2, 2, 2, 2, 2, 2, 2); + let e: u16x8 = u16x8::new(4, 8, 12, 16, 20, 24, 28, 32); + let r: u16x8 = transmute(vshlq_u16(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vshl_u32() { + let a: u32x2 = u32x2::new(1, 2); + let b: i32x2 = i32x2::new(2, 2); + let e: u32x2 = u32x2::new(4, 8); + let r: u32x2 = transmute(vshl_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vshlq_u32() { + let a: u32x4 = u32x4::new(1, 2, 3, 4); + let b: i32x4 = i32x4::new(2, 2, 2, 2); + let e: u32x4 = u32x4::new(4, 8, 12, 16); + let r: u32x4 = transmute(vshlq_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vshl_u64() { + let a: u64x1 = u64x1::new(1); + let b: i64x1 = i64x1::new(2); + let e: u64x1 = u64x1::new(4); + let r: u64x1 = transmute(vshl_u64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vshlq_u64() { + let a: u64x2 = u64x2::new(1, 2); + let b: i64x2 = i64x2::new(2, 2); + let e: u64x2 = u64x2::new(4, 8); + let r: u64x2 = transmute(vshlq_u64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vshl_n_s8() { + let a: i8x8 = i8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let e: i8x8 = i8x8::new(4, 8, 12, 16, 20, 24, 28, 32); + let r: i8x8 = transmute(vshl_n_s8::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vshlq_n_s8() { + let a: i8x16 = i8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let e: i8x16 = i8x16::new(4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60, 64); + let r: i8x16 = transmute(vshlq_n_s8::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vshl_n_s16() { + let a: i16x4 = i16x4::new(1, 2, 3, 4); + let e: i16x4 = i16x4::new(4, 8, 12, 16); + let r: i16x4 = transmute(vshl_n_s16::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vshlq_n_s16() { + let a: i16x8 = i16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let e: i16x8 = i16x8::new(4, 8, 12, 16, 20, 24, 28, 32); + let r: i16x8 = transmute(vshlq_n_s16::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vshl_n_s32() { + let a: i32x2 = i32x2::new(1, 2); + let e: i32x2 = i32x2::new(4, 8); + let r: i32x2 = transmute(vshl_n_s32::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vshlq_n_s32() { + let a: i32x4 = i32x4::new(1, 2, 3, 4); + let e: i32x4 = i32x4::new(4, 8, 12, 16); + let r: i32x4 = transmute(vshlq_n_s32::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vshl_n_u8() { + let a: u8x8 = u8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let e: u8x8 = u8x8::new(4, 8, 12, 16, 20, 24, 28, 32); + let r: u8x8 = transmute(vshl_n_u8::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vshlq_n_u8() { + let a: u8x16 = u8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let e: u8x16 = u8x16::new(4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60, 64); + let r: u8x16 = transmute(vshlq_n_u8::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vshl_n_u16() { + let a: u16x4 = u16x4::new(1, 2, 3, 4); + let e: u16x4 = u16x4::new(4, 8, 12, 16); + let r: u16x4 = transmute(vshl_n_u16::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vshlq_n_u16() { + let a: u16x8 = u16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let e: u16x8 = u16x8::new(4, 8, 12, 16, 20, 24, 28, 32); + let r: u16x8 = transmute(vshlq_n_u16::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vshl_n_u32() { + let a: u32x2 = u32x2::new(1, 2); + let e: u32x2 = u32x2::new(4, 8); + let r: u32x2 = transmute(vshl_n_u32::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vshlq_n_u32() { + let a: u32x4 = u32x4::new(1, 2, 3, 4); + let e: u32x4 = u32x4::new(4, 8, 12, 16); + let r: u32x4 = transmute(vshlq_n_u32::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vshl_n_s64() { + let a: i64x1 = i64x1::new(1); + let e: i64x1 = i64x1::new(4); + let r: i64x1 = transmute(vshl_n_s64::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vshlq_n_s64() { + let a: i64x2 = i64x2::new(1, 2); + let e: i64x2 = i64x2::new(4, 8); + let r: i64x2 = transmute(vshlq_n_s64::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vshl_n_u64() { + let a: u64x1 = u64x1::new(1); + let e: u64x1 = u64x1::new(4); + let r: u64x1 = transmute(vshl_n_u64::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vshlq_n_u64() { + let a: u64x2 = u64x2::new(1, 2); + let e: u64x2 = u64x2::new(4, 8); + let r: u64x2 = transmute(vshlq_n_u64::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vshll_n_s8() { + let a: i8x8 = i8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let e: i16x8 = i16x8::new(4, 8, 12, 16, 20, 24, 28, 32); + let r: i16x8 = transmute(vshll_n_s8::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vshll_n_s16() { + let a: i16x4 = i16x4::new(1, 2, 3, 4); + let e: i32x4 = i32x4::new(4, 8, 12, 16); + let r: i32x4 = transmute(vshll_n_s16::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vshll_n_s32() { + let a: i32x2 = i32x2::new(1, 2); + let e: i64x2 = i64x2::new(4, 8); + let r: i64x2 = transmute(vshll_n_s32::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vshll_n_u8() { + let a: u8x8 = u8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let e: u16x8 = u16x8::new(4, 8, 12, 16, 20, 24, 28, 32); + let r: u16x8 = transmute(vshll_n_u8::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vshll_n_u16() { + let a: u16x4 = u16x4::new(1, 2, 3, 4); + let e: u32x4 = u32x4::new(4, 8, 12, 16); + let r: u32x4 = transmute(vshll_n_u16::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vshll_n_u32() { + let a: u32x2 = u32x2::new(1, 2); + let e: u64x2 = u64x2::new(4, 8); + let r: u64x2 = transmute(vshll_n_u32::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vshr_n_s8() { + let a: i8x8 = i8x8::new(4, 8, 12, 16, 20, 24, 28, 32); + let e: i8x8 = i8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let r: i8x8 = transmute(vshr_n_s8::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vshrq_n_s8() { + let a: i8x16 = i8x16::new(4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60, 64); + let e: i8x16 = i8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let r: i8x16 = transmute(vshrq_n_s8::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vshr_n_s16() { + let a: i16x4 = i16x4::new(4, 8, 12, 16); + let e: i16x4 = i16x4::new(1, 2, 3, 4); + let r: i16x4 = transmute(vshr_n_s16::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vshrq_n_s16() { + let a: i16x8 = i16x8::new(4, 8, 12, 16, 20, 24, 28, 32); + let e: i16x8 = i16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let r: i16x8 = transmute(vshrq_n_s16::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vshr_n_s32() { + let a: i32x2 = i32x2::new(4, 8); + let e: i32x2 = i32x2::new(1, 2); + let r: i32x2 = transmute(vshr_n_s32::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vshrq_n_s32() { + let a: i32x4 = i32x4::new(4, 8, 12, 16); + let e: i32x4 = i32x4::new(1, 2, 3, 4); + let r: i32x4 = transmute(vshrq_n_s32::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vshr_n_s64() { + let a: i64x1 = i64x1::new(4); + let e: i64x1 = i64x1::new(1); + let r: i64x1 = transmute(vshr_n_s64::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vshrq_n_s64() { + let a: i64x2 = i64x2::new(4, 8); + let e: i64x2 = i64x2::new(1, 2); + let r: i64x2 = transmute(vshrq_n_s64::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vshr_n_u8() { + let a: u8x8 = u8x8::new(4, 8, 12, 16, 20, 24, 28, 32); + let e: u8x8 = u8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let r: u8x8 = transmute(vshr_n_u8::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vshrq_n_u8() { + let a: u8x16 = u8x16::new(4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60, 64); + let e: u8x16 = u8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let r: u8x16 = transmute(vshrq_n_u8::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vshr_n_u16() { + let a: u16x4 = u16x4::new(4, 8, 12, 16); + let e: u16x4 = u16x4::new(1, 2, 3, 4); + let r: u16x4 = transmute(vshr_n_u16::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vshrq_n_u16() { + let a: u16x8 = u16x8::new(4, 8, 12, 16, 20, 24, 28, 32); + let e: u16x8 = u16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let r: u16x8 = transmute(vshrq_n_u16::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vshr_n_u32() { + let a: u32x2 = u32x2::new(4, 8); + let e: u32x2 = u32x2::new(1, 2); + let r: u32x2 = transmute(vshr_n_u32::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vshrq_n_u32() { + let a: u32x4 = u32x4::new(4, 8, 12, 16); + let e: u32x4 = u32x4::new(1, 2, 3, 4); + let r: u32x4 = transmute(vshrq_n_u32::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vshr_n_u64() { + let a: u64x1 = u64x1::new(4); + let e: u64x1 = u64x1::new(1); + let r: u64x1 = transmute(vshr_n_u64::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vshrq_n_u64() { + let a: u64x2 = u64x2::new(4, 8); + let e: u64x2 = u64x2::new(1, 2); + let r: u64x2 = transmute(vshrq_n_u64::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vshrn_n_s16() { + let a: i16x8 = i16x8::new(4, 8, 12, 16, 20, 24, 28, 32); + let e: i8x8 = i8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let r: i8x8 = transmute(vshrn_n_s16::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vshrn_n_s32() { + let a: i32x4 = i32x4::new(4, 8, 12, 16); + let e: i16x4 = i16x4::new(1, 2, 3, 4); + let r: i16x4 = transmute(vshrn_n_s32::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vshrn_n_s64() { + let a: i64x2 = i64x2::new(4, 8); + let e: i32x2 = i32x2::new(1, 2); + let r: i32x2 = transmute(vshrn_n_s64::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vshrn_n_u16() { + let a: u16x8 = u16x8::new(4, 8, 12, 16, 20, 24, 28, 32); + let e: u8x8 = u8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let r: u8x8 = transmute(vshrn_n_u16::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vshrn_n_u32() { + let a: u32x4 = u32x4::new(4, 8, 12, 16); + let e: u16x4 = u16x4::new(1, 2, 3, 4); + let r: u16x4 = transmute(vshrn_n_u32::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vshrn_n_u64() { + let a: u64x2 = u64x2::new(4, 8); + let e: u32x2 = u32x2::new(1, 2); + let r: u32x2 = transmute(vshrn_n_u64::<2>(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsra_n_s8() { + let a: i8x8 = i8x8::new(1, 1, 1, 1, 1, 1, 1, 1); + let b: i8x8 = i8x8::new(4, 8, 12, 16, 20, 24, 28, 32); + let e: i8x8 = i8x8::new(2, 3, 4, 5, 6, 7, 8, 9); + let r: i8x8 = transmute(vsra_n_s8::<2>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsraq_n_s8() { + let a: i8x16 = i8x16::new(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1); + let b: i8x16 = i8x16::new(4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60, 64); + let e: i8x16 = i8x16::new(2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17); + let r: i8x16 = transmute(vsraq_n_s8::<2>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsra_n_s16() { + let a: i16x4 = i16x4::new(1, 1, 1, 1); + let b: i16x4 = i16x4::new(4, 8, 12, 16); + let e: i16x4 = i16x4::new(2, 3, 4, 5); + let r: i16x4 = transmute(vsra_n_s16::<2>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsraq_n_s16() { + let a: i16x8 = i16x8::new(1, 1, 1, 1, 1, 1, 1, 1); + let b: i16x8 = i16x8::new(4, 8, 12, 16, 20, 24, 28, 32); + let e: i16x8 = i16x8::new(2, 3, 4, 5, 6, 7, 8, 9); + let r: i16x8 = transmute(vsraq_n_s16::<2>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsra_n_s32() { + let a: i32x2 = i32x2::new(1, 1); + let b: i32x2 = i32x2::new(4, 8); + let e: i32x2 = i32x2::new(2, 3); + let r: i32x2 = transmute(vsra_n_s32::<2>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsraq_n_s32() { + let a: i32x4 = i32x4::new(1, 1, 1, 1); + let b: i32x4 = i32x4::new(4, 8, 12, 16); + let e: i32x4 = i32x4::new(2, 3, 4, 5); + let r: i32x4 = transmute(vsraq_n_s32::<2>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsra_n_s64() { + let a: i64x1 = i64x1::new(1); + let b: i64x1 = i64x1::new(4); + let e: i64x1 = i64x1::new(2); + let r: i64x1 = transmute(vsra_n_s64::<2>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsraq_n_s64() { + let a: i64x2 = i64x2::new(1, 1); + let b: i64x2 = i64x2::new(4, 8); + let e: i64x2 = i64x2::new(2, 3); + let r: i64x2 = transmute(vsraq_n_s64::<2>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsra_n_u8() { + let a: u8x8 = u8x8::new(1, 1, 1, 1, 1, 1, 1, 1); + let b: u8x8 = u8x8::new(4, 8, 12, 16, 20, 24, 28, 32); + let e: u8x8 = u8x8::new(2, 3, 4, 5, 6, 7, 8, 9); + let r: u8x8 = transmute(vsra_n_u8::<2>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsraq_n_u8() { + let a: u8x16 = u8x16::new(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1); + let b: u8x16 = u8x16::new(4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60, 64); + let e: u8x16 = u8x16::new(2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17); + let r: u8x16 = transmute(vsraq_n_u8::<2>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsra_n_u16() { + let a: u16x4 = u16x4::new(1, 1, 1, 1); + let b: u16x4 = u16x4::new(4, 8, 12, 16); + let e: u16x4 = u16x4::new(2, 3, 4, 5); + let r: u16x4 = transmute(vsra_n_u16::<2>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsraq_n_u16() { + let a: u16x8 = u16x8::new(1, 1, 1, 1, 1, 1, 1, 1); + let b: u16x8 = u16x8::new(4, 8, 12, 16, 20, 24, 28, 32); + let e: u16x8 = u16x8::new(2, 3, 4, 5, 6, 7, 8, 9); + let r: u16x8 = transmute(vsraq_n_u16::<2>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsra_n_u32() { + let a: u32x2 = u32x2::new(1, 1); + let b: u32x2 = u32x2::new(4, 8); + let e: u32x2 = u32x2::new(2, 3); + let r: u32x2 = transmute(vsra_n_u32::<2>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsraq_n_u32() { + let a: u32x4 = u32x4::new(1, 1, 1, 1); + let b: u32x4 = u32x4::new(4, 8, 12, 16); + let e: u32x4 = u32x4::new(2, 3, 4, 5); + let r: u32x4 = transmute(vsraq_n_u32::<2>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsra_n_u64() { + let a: u64x1 = u64x1::new(1); + let b: u64x1 = u64x1::new(4); + let e: u64x1 = u64x1::new(2); + let r: u64x1 = transmute(vsra_n_u64::<2>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vsraq_n_u64() { + let a: u64x2 = u64x2::new(1, 1); + let b: u64x2 = u64x2::new(4, 8); + let e: u64x2 = u64x2::new(2, 3); + let r: u64x2 = transmute(vsraq_n_u64::<2>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vabal_u8() { + let a: u16x8 = u16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let b: u8x8 = u8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let c: u8x8 = u8x8::new(10, 10, 10, 10, 10, 10, 10, 10); + let e: u16x8 = u16x8::new(10, 10, 10, 10, 10, 10, 10, 10); + let r: u16x8 = transmute(vabal_u8(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vabal_u16() { + let a: u32x4 = u32x4::new(1, 2, 3, 4); + let b: u16x4 = u16x4::new(1, 2, 3, 4); + let c: u16x4 = u16x4::new(10, 10, 10, 10); + let e: u32x4 = u32x4::new(10, 10, 10, 10); + let r: u32x4 = transmute(vabal_u16(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vabal_u32() { + let a: u64x2 = u64x2::new(1, 2); + let b: u32x2 = u32x2::new(1, 2); + let c: u32x2 = u32x2::new(10, 10); + let e: u64x2 = u64x2::new(10, 10); + let r: u64x2 = transmute(vabal_u32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vabal_s8() { + let a: i16x8 = i16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let b: i8x8 = i8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let c: i8x8 = i8x8::new(10, 10, 10, 10, 10, 10, 10, 10); + let e: i16x8 = i16x8::new(10, 10, 10, 10, 10, 10, 10, 10); + let r: i16x8 = transmute(vabal_s8(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vabal_s16() { + let a: i32x4 = i32x4::new(1, 2, 3, 4); + let b: i16x4 = i16x4::new(1, 2, 3, 4); + let c: i16x4 = i16x4::new(10, 10, 10, 10); + let e: i32x4 = i32x4::new(10, 10, 10, 10); + let r: i32x4 = transmute(vabal_s16(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vabal_s32() { + let a: i64x2 = i64x2::new(1, 2); + let b: i32x2 = i32x2::new(1, 2); + let c: i32x2 = i32x2::new(10, 10); + let e: i64x2 = i64x2::new(10, 10); + let r: i64x2 = transmute(vabal_s32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqabs_s8() { + let a: i8x8 = i8x8::new(-128, 0x7F, -6, -5, -4, -3, -2, -1); + let e: i8x8 = i8x8::new(0x7F, 0x7F, 6, 5, 4, 3, 2, 1); + let r: i8x8 = transmute(vqabs_s8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqabsq_s8() { + let a: i8x16 = i8x16::new(-128, 0x7F, -6, -5, -4, -3, -2, -1, 0, -127, 127, 1, 2, 3, 4, 5); + let e: i8x16 = i8x16::new(0x7F, 0x7F, 6, 5, 4, 3, 2, 1, 0, 127, 127, 1, 2, 3, 4, 5); + let r: i8x16 = transmute(vqabsq_s8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqabs_s16() { + let a: i16x4 = i16x4::new(-32768, 0x7F_FF, -6, -5); + let e: i16x4 = i16x4::new(0x7F_FF, 0x7F_FF, 6, 5); + let r: i16x4 = transmute(vqabs_s16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqabsq_s16() { + let a: i16x8 = i16x8::new(-32768, 0x7F_FF, -6, -5, -4, -3, -2, -1); + let e: i16x8 = i16x8::new(0x7F_FF, 0x7F_FF, 6, 5, 4, 3, 2, 1); + let r: i16x8 = transmute(vqabsq_s16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqabs_s32() { + let a: i32x2 = i32x2::new(-2147483648, 0x7F_FF_FF_FF); + let e: i32x2 = i32x2::new(0x7F_FF_FF_FF, 0x7F_FF_FF_FF); + let r: i32x2 = transmute(vqabs_s32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vqabsq_s32() { + let a: i32x4 = i32x4::new(-2147483648, 0x7F_FF_FF_FF, -6, -5); + let e: i32x4 = i32x4::new(0x7F_FF_FF_FF, 0x7F_FF_FF_FF, 6, 5); + let r: i32x4 = transmute(vqabsq_s32(transmute(a))); + assert_eq!(r, e); + } +} diff --git a/library/stdarch/crates/core_arch/src/arm/neon/load_tests.rs b/library/stdarch/crates/core_arch/src/arm_shared/neon/load_tests.rs similarity index 100% rename from library/stdarch/crates/core_arch/src/arm/neon/load_tests.rs rename to library/stdarch/crates/core_arch/src/arm_shared/neon/load_tests.rs diff --git a/library/stdarch/crates/core_arch/src/arm/neon/mod.rs b/library/stdarch/crates/core_arch/src/arm_shared/neon/mod.rs similarity index 62% rename from library/stdarch/crates/core_arch/src/arm/neon/mod.rs rename to library/stdarch/crates/core_arch/src/arm_shared/neon/mod.rs index 4f6ab5aef1..9ed9f77aa7 100644 --- a/library/stdarch/crates/core_arch/src/arm/neon/mod.rs +++ b/library/stdarch/crates/core_arch/src/arm_shared/neon/mod.rs @@ -5,10 +5,9 @@ mod generated; #[rustfmt::skip] pub use self::generated::*; -#[cfg(target_arch = "arm")] -use crate::mem::align_of; use crate::{ - core_arch::simd::*, core_arch::simd_llvm::*, hint::unreachable_unchecked, mem::transmute, + convert::TryInto, core_arch::simd::*, core_arch::simd_llvm::*, hint::unreachable_unchecked, + mem::transmute, }; #[cfg(test)] use stdarch_test::assert_instr; @@ -20,67 +19,67 @@ pub(crate) type p128 = u128; types! { /// ARM-specific 64-bit wide vector of eight packed `i8`. - pub struct int8x8_t(i8, i8, i8, i8, i8, i8, i8, i8); + pub struct int8x8_t(pub(crate) i8, pub(crate) i8, pub(crate) i8, pub(crate) i8, pub(crate) i8, pub(crate) i8, pub(crate) i8, pub(crate) i8); /// ARM-specific 64-bit wide vector of eight packed `u8`. - pub struct uint8x8_t(u8, u8, u8, u8, u8, u8, u8, u8); + pub struct uint8x8_t(pub(crate) u8, pub(crate) u8, pub(crate) u8, pub(crate) u8, pub(crate) u8, pub(crate) u8, pub(crate) u8, pub(crate) u8); /// ARM-specific 64-bit wide polynomial vector of eight packed `p8`. - pub struct poly8x8_t(p8, p8, p8, p8, p8, p8, p8, p8); + pub struct poly8x8_t(pub(crate) p8, pub(crate) p8, pub(crate) p8, pub(crate) p8, pub(crate) p8, pub(crate) p8, pub(crate) p8, pub(crate) p8); /// ARM-specific 64-bit wide vector of four packed `i16`. - pub struct int16x4_t(i16, i16, i16, i16); + pub struct int16x4_t(pub(crate) i16, pub(crate) i16, pub(crate) i16, pub(crate) i16); /// ARM-specific 64-bit wide vector of four packed `u16`. - pub struct uint16x4_t(u16, u16, u16, u16); + pub struct uint16x4_t(pub(crate) u16, pub(crate) u16, pub(crate) u16, pub(crate) u16); // FIXME: ARM-specific 64-bit wide vector of four packed `f16`. // pub struct float16x4_t(f16, f16, f16, f16); /// ARM-specific 64-bit wide vector of four packed `p16`. - pub struct poly16x4_t(p16, p16, p16, p16); + pub struct poly16x4_t(pub(crate) p16, pub(crate) p16, pub(crate) p16, pub(crate) p16); /// ARM-specific 64-bit wide vector of two packed `i32`. - pub struct int32x2_t(i32, i32); + pub struct int32x2_t(pub(crate) i32, pub(crate) i32); /// ARM-specific 64-bit wide vector of two packed `u32`. - pub struct uint32x2_t(u32, u32); + pub struct uint32x2_t(pub(crate) u32, pub(crate) u32); /// ARM-specific 64-bit wide vector of two packed `f32`. - pub struct float32x2_t(f32, f32); + pub struct float32x2_t(pub(crate) f32, pub(crate) f32); /// ARM-specific 64-bit wide vector of one packed `i64`. - pub struct int64x1_t(i64); + pub struct int64x1_t(pub(crate) i64); /// ARM-specific 64-bit wide vector of one packed `u64`. - pub struct uint64x1_t(u64); + pub struct uint64x1_t(pub(crate) u64); /// ARM-specific 64-bit wide vector of one packed `p64`. - pub struct poly64x1_t(p64); + pub struct poly64x1_t(pub(crate) p64); /// ARM-specific 128-bit wide vector of sixteen packed `i8`. pub struct int8x16_t( - i8, i8, i8, i8, i8, i8 ,i8, i8, - i8, i8, i8, i8, i8, i8 ,i8, i8, + pub(crate) i8, pub(crate) i8, pub(crate) i8, pub(crate) i8, pub(crate) i8, pub(crate) i8 , pub(crate) i8, pub(crate) i8, + pub(crate) i8, pub(crate) i8, pub(crate) i8, pub(crate) i8, pub(crate) i8, pub(crate) i8 , pub(crate) i8, pub(crate) i8, ); /// ARM-specific 128-bit wide vector of sixteen packed `u8`. pub struct uint8x16_t( - u8, u8 ,u8, u8, u8, u8 ,u8, u8, - u8, u8 ,u8, u8, u8, u8 ,u8, u8, + pub(crate) u8, pub(crate) u8 , pub(crate) u8, pub(crate) u8, pub(crate) u8, pub(crate) u8 , pub(crate) u8, pub(crate) u8, + pub(crate) u8, pub(crate) u8 , pub(crate) u8, pub(crate) u8, pub(crate) u8, pub(crate) u8 , pub(crate) u8, pub(crate) u8, ); /// ARM-specific 128-bit wide vector of sixteen packed `p8`. pub struct poly8x16_t( - p8, p8, p8, p8, p8, p8, p8, p8, - p8, p8, p8, p8, p8, p8, p8, p8, + pub(crate) p8, pub(crate) p8, pub(crate) p8, pub(crate) p8, pub(crate) p8, pub(crate) p8, pub(crate) p8, pub(crate) p8, + pub(crate) p8, pub(crate) p8, pub(crate) p8, pub(crate) p8, pub(crate) p8, pub(crate) p8, pub(crate) p8, pub(crate) p8, ); /// ARM-specific 128-bit wide vector of eight packed `i16`. - pub struct int16x8_t(i16, i16, i16, i16, i16, i16, i16, i16); + pub struct int16x8_t(pub(crate) i16, pub(crate) i16, pub(crate) i16, pub(crate) i16, pub(crate) i16, pub(crate) i16, pub(crate) i16, pub(crate) i16); /// ARM-specific 128-bit wide vector of eight packed `u16`. - pub struct uint16x8_t(u16, u16, u16, u16, u16, u16, u16, u16); + pub struct uint16x8_t(pub(crate) u16, pub(crate) u16, pub(crate) u16, pub(crate) u16, pub(crate) u16, pub(crate) u16, pub(crate) u16, pub(crate) u16); // FIXME: ARM-specific 128-bit wide vector of eight packed `f16`. // pub struct float16x8_t(f16, f16, f16, f16, f16, f16, f16); /// ARM-specific 128-bit wide vector of eight packed `p16`. - pub struct poly16x8_t(p16, p16, p16, p16, p16, p16, p16, p16); + pub struct poly16x8_t(pub(crate) p16, pub(crate) p16, pub(crate) p16, pub(crate) p16, pub(crate) p16, pub(crate) p16, pub(crate) p16, pub(crate) p16); /// ARM-specific 128-bit wide vector of four packed `i32`. - pub struct int32x4_t(i32, i32, i32, i32); + pub struct int32x4_t(pub(crate) i32, pub(crate) i32, pub(crate) i32, pub(crate) i32); /// ARM-specific 128-bit wide vector of four packed `u32`. - pub struct uint32x4_t(u32, u32, u32, u32); + pub struct uint32x4_t(pub(crate) u32, pub(crate) u32, pub(crate) u32, pub(crate) u32); /// ARM-specific 128-bit wide vector of four packed `f32`. - pub struct float32x4_t(f32, f32, f32, f32); + pub struct float32x4_t(pub(crate) f32, pub(crate) f32, pub(crate) f32, pub(crate) f32); /// ARM-specific 128-bit wide vector of two packed `i64`. - pub struct int64x2_t(i64, i64); + pub struct int64x2_t(pub(crate) i64, pub(crate) i64); /// ARM-specific 128-bit wide vector of two packed `u64`. - pub struct uint64x2_t(u64, u64); + pub struct uint64x2_t(pub(crate) u64, pub(crate) u64); /// ARM-specific 128-bit wide vector of two packed `p64`. - pub struct poly64x2_t(p64, p64); + pub struct poly64x2_t(pub(crate) p64, pub(crate) p64); } /// ARM-specific type containing two `int8x8_t` vectors. @@ -136,15 +135,6 @@ extern "C" { #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.abs.v4i32")] fn vabsq_s32_(a: int32x4_t) -> int32x4_t; - #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vrsqrte.v2f32")] - #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.frsqrte.v2f32")] - fn frsqrte_v2f32(a: float32x2_t) -> float32x2_t; - - //uint32x2_t vqmovn_u64 (uint64x2_t a) - #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqmovnu.v2i32")] - #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.uqxtn.v2i32")] - fn vqmovn_u64_(a: uint64x2_t) -> uint32x2_t; - #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vpmins.v8i8")] #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.sminp.v8i8")] fn vpmins_v8i8(a: int8x8_t, b: int8x8_t) -> int8x8_t; @@ -217,74 +207,74 @@ extern "C" { target_arch = "aarch64", link_name = "llvm.aarch64.neon.saddlp.v4i16.v8i8" )] - fn vpaddl_s8_(a: int8x8_t) -> int16x4_t; + pub(crate) fn vpaddl_s8_(a: int8x8_t) -> int16x4_t; #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vpaddls.v2i32.v4i16")] #[cfg_attr( target_arch = "aarch64", link_name = "llvm.aarch64.neon.saddlp.v2i32.v4i16" )] - fn vpaddl_s16_(a: int16x4_t) -> int32x2_t; + pub(crate) fn vpaddl_s16_(a: int16x4_t) -> int32x2_t; #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vpaddls.v1i64.v2i32")] #[cfg_attr( target_arch = "aarch64", link_name = "llvm.aarch64.neon.saddlp.v1i64.v2i32" )] - fn vpaddl_s32_(a: int32x2_t) -> int64x1_t; + pub(crate) fn vpaddl_s32_(a: int32x2_t) -> int64x1_t; #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vpaddls.v8i16.v16i8")] #[cfg_attr( target_arch = "aarch64", link_name = "llvm.aarch64.neon.saddlp.v8i16.v16i8" )] - fn vpaddlq_s8_(a: int8x16_t) -> int16x8_t; + pub(crate) fn vpaddlq_s8_(a: int8x16_t) -> int16x8_t; #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vpaddls.v4i32.v8i16")] #[cfg_attr( target_arch = "aarch64", link_name = "llvm.aarch64.neon.saddlp.v4i32.v8i16" )] - fn vpaddlq_s16_(a: int16x8_t) -> int32x4_t; + pub(crate) fn vpaddlq_s16_(a: int16x8_t) -> int32x4_t; #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vpaddls.v2i64.v4i32")] #[cfg_attr( target_arch = "aarch64", link_name = "llvm.aarch64.neon.saddlp.v2i64.v4i32" )] - fn vpaddlq_s32_(a: int32x4_t) -> int64x2_t; + pub(crate) fn vpaddlq_s32_(a: int32x4_t) -> int64x2_t; #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vpaddlu.v4i16.v8i8")] #[cfg_attr( target_arch = "aarch64", link_name = "llvm.aarch64.neon.uaddlp.v4i16.v8i8" )] - fn vpaddl_u8_(a: uint8x8_t) -> uint16x4_t; + pub(crate) fn vpaddl_u8_(a: uint8x8_t) -> uint16x4_t; #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vpaddlu.v2i32.v4i16")] #[cfg_attr( target_arch = "aarch64", link_name = "llvm.aarch64.neon.uaddlp.v2i32.v4i16" )] - fn vpaddl_u16_(a: uint16x4_t) -> uint32x2_t; + pub(crate) fn vpaddl_u16_(a: uint16x4_t) -> uint32x2_t; #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vpaddlu.v1i64.v2i32")] #[cfg_attr( target_arch = "aarch64", link_name = "llvm.aarch64.neon.uaddlp.v1i64.v2i32" )] - fn vpaddl_u32_(a: uint32x2_t) -> uint64x1_t; + pub(crate) fn vpaddl_u32_(a: uint32x2_t) -> uint64x1_t; #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vpaddlu.v8i16.v16i8")] #[cfg_attr( target_arch = "aarch64", link_name = "llvm.aarch64.neon.uaddlp.v8i16.v16i8" )] - fn vpaddlq_u8_(a: uint8x16_t) -> uint16x8_t; + pub(crate) fn vpaddlq_u8_(a: uint8x16_t) -> uint16x8_t; #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vpaddlu.v4i32.v8i16")] #[cfg_attr( target_arch = "aarch64", link_name = "llvm.aarch64.neon.uaddlp.v4i32.v8i16" )] - fn vpaddlq_u16_(a: uint16x8_t) -> uint32x4_t; + pub(crate) fn vpaddlq_u16_(a: uint16x8_t) -> uint32x4_t; #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vpaddlu.v2i64.v4i32")] #[cfg_attr( target_arch = "aarch64", link_name = "llvm.aarch64.neon.uaddlp.v2i64.v4i32" )] - fn vpaddlq_u32_(a: uint32x4_t) -> uint64x2_t; + pub(crate) fn vpaddlq_u32_(a: uint32x4_t) -> uint64x2_t; #[cfg_attr(target_arch = "arm", link_name = "llvm.ctpop.v8i8")] #[cfg_attr(target_arch = "aarch64", link_name = "llvm.ctpop.v8i8")] @@ -292,649 +282,289 @@ extern "C" { #[cfg_attr(target_arch = "arm", link_name = "llvm.ctpop.v16i8")] #[cfg_attr(target_arch = "aarch64", link_name = "llvm.ctpop.v16i8")] fn vcntq_s8_(a: int8x16_t) -> int8x16_t; -} - -#[cfg(target_arch = "arm")] -#[allow(improper_ctypes)] -extern "C" { - #[link_name = "llvm.arm.neon.vpadals.v4i16.v8i8"] - fn vpadal_s8_(a: int16x4_t, b: int8x8_t) -> int16x4_t; - #[link_name = "llvm.arm.neon.vpadals.v2i32.v4i16"] - fn vpadal_s16_(a: int32x2_t, b: int16x4_t) -> int32x2_t; - #[link_name = "llvm.arm.neon.vpadals.v1i64.v2i32"] - fn vpadal_s32_(a: int64x1_t, b: int32x2_t) -> int64x1_t; - #[link_name = "llvm.arm.neon.vpadals.v8i16.v16i8"] - fn vpadalq_s8_(a: int16x8_t, b: int8x16_t) -> int16x8_t; - #[link_name = "llvm.arm.neon.vpadals.v4i32.v8i16"] - fn vpadalq_s16_(a: int32x4_t, b: int16x8_t) -> int32x4_t; - #[link_name = "llvm.arm.neon.vpadals.v2i64.v4i32"] - fn vpadalq_s32_(a: int64x2_t, b: int32x4_t) -> int64x2_t; - - #[link_name = "llvm.arm.neon.vpadalu.v4i16.v8i8"] - fn vpadal_u8_(a: uint16x4_t, b: uint8x8_t) -> uint16x4_t; - #[link_name = "llvm.arm.neon.vpadalu.v2i32.v4i16"] - fn vpadal_u16_(a: uint32x2_t, b: uint16x4_t) -> uint32x2_t; - #[link_name = "llvm.arm.neon.vpadalu.v1i64.v2i32"] - fn vpadal_u32_(a: uint64x1_t, b: uint32x2_t) -> uint64x1_t; - #[link_name = "llvm.arm.neon.vpadalu.v8i16.v16i8"] - fn vpadalq_u8_(a: uint16x8_t, b: uint8x16_t) -> uint16x8_t; - #[link_name = "llvm.arm.neon.vpadalu.v4i32.v8i16"] - fn vpadalq_u16_(a: uint32x4_t, b: uint16x8_t) -> uint32x4_t; - #[link_name = "llvm.arm.neon.vpadalu.v2i64.v4i32"] - fn vpadalq_u32_(a: uint64x2_t, b: uint32x4_t) -> uint64x2_t; - - #[link_name = "llvm.arm.neon.vtbl1"] - fn vtbl1(a: int8x8_t, b: int8x8_t) -> int8x8_t; - #[link_name = "llvm.arm.neon.vtbl2"] - fn vtbl2(a: int8x8_t, b: int8x8_t, b: int8x8_t) -> int8x8_t; - #[link_name = "llvm.arm.neon.vtbl3"] - fn vtbl3(a: int8x8_t, b: int8x8_t, b: int8x8_t, c: int8x8_t) -> int8x8_t; - #[link_name = "llvm.arm.neon.vtbl4"] - fn vtbl4(a: int8x8_t, b: int8x8_t, b: int8x8_t, c: int8x8_t, d: int8x8_t) -> int8x8_t; - - #[link_name = "llvm.arm.neon.vtbx1"] - fn vtbx1(a: int8x8_t, b: int8x8_t, b: int8x8_t) -> int8x8_t; - #[link_name = "llvm.arm.neon.vtbx2"] - fn vtbx2(a: int8x8_t, b: int8x8_t, b: int8x8_t, c: int8x8_t) -> int8x8_t; - #[link_name = "llvm.arm.neon.vtbx3"] - fn vtbx3(a: int8x8_t, b: int8x8_t, b: int8x8_t, c: int8x8_t, d: int8x8_t) -> int8x8_t; - #[link_name = "llvm.arm.neon.vtbx4"] - fn vtbx4( - a: int8x8_t, - b: int8x8_t, - b: int8x8_t, - c: int8x8_t, - d: int8x8_t, - e: int8x8_t, - ) -> int8x8_t; - - #[link_name = "llvm.arm.neon.vshiftins.v8i8"] - fn vshiftins_v8i8(a: int8x8_t, b: int8x8_t, shift: int8x8_t) -> int8x8_t; - #[link_name = "llvm.arm.neon.vshiftins.v16i8"] - fn vshiftins_v16i8(a: int8x16_t, b: int8x16_t, shift: int8x16_t) -> int8x16_t; - #[link_name = "llvm.arm.neon.vshiftins.v4i16"] - fn vshiftins_v4i16(a: int16x4_t, b: int16x4_t, shift: int16x4_t) -> int16x4_t; - #[link_name = "llvm.arm.neon.vshiftins.v8i16"] - fn vshiftins_v8i16(a: int16x8_t, b: int16x8_t, shift: int16x8_t) -> int16x8_t; - #[link_name = "llvm.arm.neon.vshiftins.v2i32"] - fn vshiftins_v2i32(a: int32x2_t, b: int32x2_t, shift: int32x2_t) -> int32x2_t; - #[link_name = "llvm.arm.neon.vshiftins.v4i32"] - fn vshiftins_v4i32(a: int32x4_t, b: int32x4_t, shift: int32x4_t) -> int32x4_t; - #[link_name = "llvm.arm.neon.vshiftins.v1i64"] - fn vshiftins_v1i64(a: int64x1_t, b: int64x1_t, shift: int64x1_t) -> int64x1_t; - #[link_name = "llvm.arm.neon.vshiftins.v2i64"] - fn vshiftins_v2i64(a: int64x2_t, b: int64x2_t, shift: int64x2_t) -> int64x2_t; - - #[link_name = "llvm.arm.neon.vld1.v8i8.p0i8"] - fn vld1_v8i8(addr: *const i8, align: i32) -> int8x8_t; - #[link_name = "llvm.arm.neon.vld1.v16i8.p0i8"] - fn vld1q_v16i8(addr: *const i8, align: i32) -> int8x16_t; - #[link_name = "llvm.arm.neon.vld1.v4i16.p0i8"] - fn vld1_v4i16(addr: *const i8, align: i32) -> int16x4_t; - #[link_name = "llvm.arm.neon.vld1.v8i16.p0i8"] - fn vld1q_v8i16(addr: *const i8, align: i32) -> int16x8_t; - #[link_name = "llvm.arm.neon.vld1.v2i32.p0i8"] - fn vld1_v2i32(addr: *const i8, align: i32) -> int32x2_t; - #[link_name = "llvm.arm.neon.vld1.v4i32.p0i8"] - fn vld1q_v4i32(addr: *const i8, align: i32) -> int32x4_t; - #[link_name = "llvm.arm.neon.vld1.v1i64.p0i8"] - fn vld1_v1i64(addr: *const i8, align: i32) -> int64x1_t; - #[link_name = "llvm.arm.neon.vld1.v2i64.p0i8"] - fn vld1q_v2i64(addr: *const i8, align: i32) -> int64x2_t; - #[link_name = "llvm.arm.neon.vld1.v2f32.p0i8"] - fn vld1_v2f32(addr: *const i8, align: i32) -> float32x2_t; - #[link_name = "llvm.arm.neon.vld1.v4f32.p0i8"] - fn vld1q_v4f32(addr: *const i8, align: i32) -> float32x4_t; -} - -/// Load multiple single-element structures to one, two, three, or four registers. -#[inline] -#[cfg(target_arch = "arm")] -#[target_feature(enable = "neon,v7")] -#[cfg_attr(test, assert_instr("vld1.8"))] -pub unsafe fn vld1_s8(ptr: *const i8) -> int8x8_t { - vld1_v8i8(ptr as *const i8, align_of::() as i32) -} - -/// Load multiple single-element structures to one, two, three, or four registers. -#[inline] -#[cfg(target_arch = "arm")] -#[target_feature(enable = "neon,v7")] -#[cfg_attr(test, assert_instr("vld1.8"))] -pub unsafe fn vld1q_s8(ptr: *const i8) -> int8x16_t { - vld1q_v16i8(ptr as *const i8, align_of::() as i32) -} - -/// Load multiple single-element structures to one, two, three, or four registers. -#[inline] -#[cfg(target_arch = "arm")] -#[target_feature(enable = "neon,v7")] -#[cfg_attr(test, assert_instr("vld1.16"))] -pub unsafe fn vld1_s16(ptr: *const i16) -> int16x4_t { - vld1_v4i16(ptr as *const i8, align_of::() as i32) -} - -/// Load multiple single-element structures to one, two, three, or four registers. -#[inline] -#[cfg(target_arch = "arm")] -#[target_feature(enable = "neon,v7")] -#[cfg_attr(test, assert_instr("vld1.16"))] -pub unsafe fn vld1q_s16(ptr: *const i16) -> int16x8_t { - vld1q_v8i16(ptr as *const i8, align_of::() as i32) -} - -/// Load multiple single-element structures to one, two, three, or four registers. -#[inline] -#[cfg(target_arch = "arm")] -#[target_feature(enable = "neon,v7")] -#[cfg_attr(test, assert_instr(vldr))] -pub unsafe fn vld1_s32(ptr: *const i32) -> int32x2_t { - vld1_v2i32(ptr as *const i8, align_of::() as i32) -} - -/// Load multiple single-element structures to one, two, three, or four registers. -#[inline] -#[cfg(target_arch = "arm")] -#[target_feature(enable = "neon,v7")] -#[cfg_attr(test, assert_instr("vld1.32"))] -pub unsafe fn vld1q_s32(ptr: *const i32) -> int32x4_t { - vld1q_v4i32(ptr as *const i8, align_of::() as i32) -} - -/// Load multiple single-element structures to one, two, three, or four registers. -#[inline] -#[cfg(target_arch = "arm")] -#[target_feature(enable = "neon,v7")] -#[cfg_attr(test, assert_instr(vldr))] -pub unsafe fn vld1_s64(ptr: *const i64) -> int64x1_t { - vld1_v1i64(ptr as *const i8, align_of::() as i32) -} - -/// Load multiple single-element structures to one, two, three, or four registers. -#[inline] -#[cfg(target_arch = "arm")] -#[target_feature(enable = "neon,v7")] -#[cfg_attr(test, assert_instr("vld1.64"))] -pub unsafe fn vld1q_s64(ptr: *const i64) -> int64x2_t { - vld1q_v2i64(ptr as *const i8, align_of::() as i32) -} - -/// Load multiple single-element structures to one, two, three, or four registers. -#[inline] -#[cfg(target_arch = "arm")] -#[target_feature(enable = "neon,v7")] -#[cfg_attr(test, assert_instr("vld1.8"))] -pub unsafe fn vld1_u8(ptr: *const u8) -> uint8x8_t { - transmute(vld1_v8i8(ptr as *const i8, align_of::() as i32)) -} - -/// Load multiple single-element structures to one, two, three, or four registers. -#[inline] -#[cfg(target_arch = "arm")] -#[target_feature(enable = "neon,v7")] -#[cfg_attr(test, assert_instr("vld1.8"))] -pub unsafe fn vld1q_u8(ptr: *const u8) -> uint8x16_t { - transmute(vld1q_v16i8(ptr as *const i8, align_of::() as i32)) -} - -/// Load multiple single-element structures to one, two, three, or four registers. -#[inline] -#[cfg(target_arch = "arm")] -#[target_feature(enable = "neon,v7")] -#[cfg_attr(test, assert_instr("vld1.16"))] -pub unsafe fn vld1_u16(ptr: *const u16) -> uint16x4_t { - transmute(vld1_v4i16(ptr as *const i8, align_of::() as i32)) -} - -/// Load multiple single-element structures to one, two, three, or four registers. -#[inline] -#[cfg(target_arch = "arm")] -#[target_feature(enable = "neon,v7")] -#[cfg_attr(test, assert_instr("vld1.16"))] -pub unsafe fn vld1q_u16(ptr: *const u16) -> uint16x8_t { - transmute(vld1q_v8i16(ptr as *const i8, align_of::() as i32)) -} - -/// Load multiple single-element structures to one, two, three, or four registers. -#[inline] -#[cfg(target_arch = "arm")] -#[target_feature(enable = "neon,v7")] -#[cfg_attr(test, assert_instr(vldr))] -pub unsafe fn vld1_u32(ptr: *const u32) -> uint32x2_t { - transmute(vld1_v2i32(ptr as *const i8, align_of::() as i32)) -} - -/// Load multiple single-element structures to one, two, three, or four registers. -#[inline] -#[cfg(target_arch = "arm")] -#[target_feature(enable = "neon,v7")] -#[cfg_attr(test, assert_instr("vld1.32"))] -pub unsafe fn vld1q_u32(ptr: *const u32) -> uint32x4_t { - transmute(vld1q_v4i32(ptr as *const i8, align_of::() as i32)) -} - -/// Load multiple single-element structures to one, two, three, or four registers. -#[inline] -#[cfg(target_arch = "arm")] -#[target_feature(enable = "neon,v7")] -#[cfg_attr(test, assert_instr(vldr))] -pub unsafe fn vld1_u64(ptr: *const u64) -> uint64x1_t { - transmute(vld1_v1i64(ptr as *const i8, align_of::() as i32)) -} - -/// Load multiple single-element structures to one, two, three, or four registers. -#[inline] -#[cfg(target_arch = "arm")] -#[target_feature(enable = "neon,v7")] -#[cfg_attr(test, assert_instr("vld1.64"))] -pub unsafe fn vld1q_u64(ptr: *const u64) -> uint64x2_t { - transmute(vld1q_v2i64(ptr as *const i8, align_of::() as i32)) -} - -/// Load multiple single-element structures to one, two, three, or four registers. -#[inline] -#[cfg(target_arch = "arm")] -#[target_feature(enable = "neon,v7")] -#[cfg_attr(test, assert_instr("vld1.8"))] -pub unsafe fn vld1_p8(ptr: *const p8) -> poly8x8_t { - transmute(vld1_v8i8(ptr as *const i8, align_of::() as i32)) -} - -/// Load multiple single-element structures to one, two, three, or four registers. -#[inline] -#[cfg(target_arch = "arm")] -#[target_feature(enable = "neon,v7")] -#[cfg_attr(test, assert_instr("vld1.8"))] -pub unsafe fn vld1q_p8(ptr: *const p8) -> poly8x16_t { - transmute(vld1q_v16i8(ptr as *const i8, align_of::() as i32)) -} - -/// Load multiple single-element structures to one, two, three, or four registers. -#[inline] -#[cfg(target_arch = "arm")] -#[target_feature(enable = "neon,v7")] -#[cfg_attr(test, assert_instr("vld1.16"))] -pub unsafe fn vld1_p16(ptr: *const p16) -> poly16x4_t { - transmute(vld1_v4i16(ptr as *const i8, align_of::() as i32)) -} - -/// Load multiple single-element structures to one, two, three, or four registers. -#[inline] -#[cfg(target_arch = "arm")] -#[target_feature(enable = "neon,v7")] -#[cfg_attr(test, assert_instr("vld1.16"))] -pub unsafe fn vld1q_p16(ptr: *const p16) -> poly16x8_t { - transmute(vld1q_v8i16(ptr as *const i8, align_of::() as i32)) -} - -/// Load multiple single-element structures to one, two, three, or four registers. -#[inline] -#[cfg(target_arch = "arm")] -#[target_feature(enable = "neon,v7")] -#[cfg_attr(test, assert_instr(vldr))] -pub unsafe fn vld1_f32(ptr: *const f32) -> float32x2_t { - vld1_v2f32(ptr as *const i8, align_of::() as i32) -} -/// Load multiple single-element structures to one, two, three, or four registers. -#[inline] -#[cfg(target_arch = "arm")] -#[target_feature(enable = "neon,v7")] -#[cfg_attr(test, assert_instr("vld1.32"))] -pub unsafe fn vld1q_f32(ptr: *const f32) -> float32x4_t { - vld1q_v4f32(ptr as *const i8, align_of::() as i32) + #[cfg_attr(target_arch = "arm", link_name = "llvm.ctlz.v8i8")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.ctlz.v8i8")] + fn vclz_s8_(a: int8x8_t) -> int8x8_t; + #[cfg_attr(target_arch = "arm", link_name = "llvm.ctlz.v16i8")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.ctlz.v16i8")] + fn vclzq_s8_(a: int8x16_t) -> int8x16_t; + #[cfg_attr(target_arch = "arm", link_name = "llvm.ctlz.v4i16")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.ctlz.v4i16")] + fn vclz_s16_(a: int16x4_t) -> int16x4_t; + #[cfg_attr(target_arch = "arm", link_name = "llvm.ctlz.v8i16")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.ctlz.v8i16")] + fn vclzq_s16_(a: int16x8_t) -> int16x8_t; + #[cfg_attr(target_arch = "arm", link_name = "llvm.ctlz.v2i32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.ctlz.v2i32")] + fn vclz_s32_(a: int32x2_t) -> int32x2_t; + #[cfg_attr(target_arch = "arm", link_name = "llvm.ctlz.v4i32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.ctlz.v4i32")] + fn vclzq_s32_(a: int32x4_t) -> int32x4_t; } /// Load one single-element structure to one lane of one register. #[inline] #[target_feature(enable = "neon")] #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[rustc_args_required_const(2)] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld1.8", lane = 7))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1, lane = 7))] -pub unsafe fn vld1_lane_s8(ptr: *const i8, src: int8x8_t, lane: i32) -> int8x8_t { - assert!( - 0 <= lane && lane <= 7, - "must have 0 ≤ lane ≤ 7, but lane = {}", - lane - ); - simd_insert(src, lane as u32, *ptr) +#[rustc_legacy_const_generics(2)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld1.8", LANE = 7))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1, LANE = 7))] +pub unsafe fn vld1_lane_s8(ptr: *const i8, src: int8x8_t) -> int8x8_t { + static_assert_imm3!(LANE); + simd_insert(src, LANE as u32, *ptr) } /// Load one single-element structure to one lane of one register. #[inline] #[target_feature(enable = "neon")] #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[rustc_args_required_const(2)] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld1.8", lane = 15))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1, lane = 15))] -pub unsafe fn vld1q_lane_s8(ptr: *const i8, src: int8x16_t, lane: i32) -> int8x16_t { - assert!( - 0 <= lane && lane <= 15, - "must have 0 ≤ lane ≤ 15, but lane = {}", - lane - ); - simd_insert(src, lane as u32, *ptr) +#[rustc_legacy_const_generics(2)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld1.8", LANE = 15))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1, LANE = 15))] +pub unsafe fn vld1q_lane_s8(ptr: *const i8, src: int8x16_t) -> int8x16_t { + static_assert_imm4!(LANE); + simd_insert(src, LANE as u32, *ptr) } /// Load one single-element structure to one lane of one register. #[inline] #[target_feature(enable = "neon")] #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[rustc_args_required_const(2)] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld1.16", lane = 3))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1, lane = 3))] -pub unsafe fn vld1_lane_s16(ptr: *const i16, src: int16x4_t, lane: i32) -> int16x4_t { - assert!( - 0 <= lane && lane <= 3, - "must have 0 ≤ lane ≤ 3, but lane = {}", - lane - ); - simd_insert(src, lane as u32, *ptr) +#[rustc_legacy_const_generics(2)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld1.16", LANE = 3))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1, LANE = 3))] +pub unsafe fn vld1_lane_s16(ptr: *const i16, src: int16x4_t) -> int16x4_t { + static_assert_imm2!(LANE); + simd_insert(src, LANE as u32, *ptr) } /// Load one single-element structure to one lane of one register. #[inline] #[target_feature(enable = "neon")] #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[rustc_args_required_const(2)] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld1.16", lane = 7))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1, lane = 7))] -pub unsafe fn vld1q_lane_s16(ptr: *const i16, src: int16x8_t, lane: i32) -> int16x8_t { - assert!( - 0 <= lane && lane <= 7, - "must have 0 ≤ lane ≤ 7, but lane = {}", - lane - ); - simd_insert(src, lane as u32, *ptr) +#[rustc_legacy_const_generics(2)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld1.16", LANE = 7))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1, LANE = 7))] +pub unsafe fn vld1q_lane_s16(ptr: *const i16, src: int16x8_t) -> int16x8_t { + static_assert_imm3!(LANE); + simd_insert(src, LANE as u32, *ptr) } /// Load one single-element structure to one lane of one register. #[inline] #[target_feature(enable = "neon")] #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[rustc_args_required_const(2)] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld1.32", lane = 1))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1, lane = 1))] -pub unsafe fn vld1_lane_s32(ptr: *const i32, src: int32x2_t, lane: i32) -> int32x2_t { - assert!( - 0 <= lane && lane <= 1, - "must have 0 ≤ lane ≤ 1, but lane = {}", - lane - ); - simd_insert(src, lane as u32, *ptr) +#[rustc_legacy_const_generics(2)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld1.32", LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1, LANE = 1))] +pub unsafe fn vld1_lane_s32(ptr: *const i32, src: int32x2_t) -> int32x2_t { + static_assert_imm1!(LANE); + simd_insert(src, LANE as u32, *ptr) } /// Load one single-element structure to one lane of one register. #[inline] #[target_feature(enable = "neon")] #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[rustc_args_required_const(2)] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld1.32", lane = 3))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1, lane = 3))] -pub unsafe fn vld1q_lane_s32(ptr: *const i32, src: int32x4_t, lane: i32) -> int32x4_t { - assert!( - 0 <= lane && lane <= 3, - "must have 0 ≤ lane ≤ 3, but lane = {}", - lane - ); - simd_insert(src, lane as u32, *ptr) +#[rustc_legacy_const_generics(2)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld1.32", LANE = 3))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1, LANE = 3))] +pub unsafe fn vld1q_lane_s32(ptr: *const i32, src: int32x4_t) -> int32x4_t { + static_assert_imm2!(LANE); + simd_insert(src, LANE as u32, *ptr) } /// Load one single-element structure to one lane of one register. #[inline] #[target_feature(enable = "neon")] #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[rustc_args_required_const(2)] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vldr", lane = 0))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ldr, lane = 0))] -pub unsafe fn vld1_lane_s64(ptr: *const i64, src: int64x1_t, lane: i32) -> int64x1_t { - assert!( - 0 <= lane && lane <= 0, - "must have 0 ≤ lane ≤ 0, but lane = {}", - lane - ); - simd_insert(src, lane as u32, *ptr) +#[rustc_legacy_const_generics(2)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vldr", LANE = 0))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ldr, LANE = 0))] +pub unsafe fn vld1_lane_s64(ptr: *const i64, src: int64x1_t) -> int64x1_t { + static_assert!(LANE : i32 where LANE == 0); + simd_insert(src, LANE as u32, *ptr) } /// Load one single-element structure to one lane of one register. #[inline] #[target_feature(enable = "neon")] #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[rustc_args_required_const(2)] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vldr", lane = 1))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1, lane = 1))] -pub unsafe fn vld1q_lane_s64(ptr: *const i64, src: int64x2_t, lane: i32) -> int64x2_t { - assert!( - 0 <= lane && lane <= 1, - "must have 0 ≤ lane ≤ 1, but lane = {}", - lane - ); - simd_insert(src, lane as u32, *ptr) +#[rustc_legacy_const_generics(2)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vldr", LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1, LANE = 1))] +pub unsafe fn vld1q_lane_s64(ptr: *const i64, src: int64x2_t) -> int64x2_t { + static_assert_imm1!(LANE); + simd_insert(src, LANE as u32, *ptr) } /// Load one single-element structure to one lane of one register. #[inline] #[target_feature(enable = "neon")] #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[rustc_args_required_const(2)] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld1.8", lane = 7))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1, lane = 7))] -pub unsafe fn vld1_lane_u8(ptr: *const u8, src: uint8x8_t, lane: i32) -> uint8x8_t { - assert!( - 0 <= lane && lane <= 7, - "must have 0 ≤ lane ≤ 7, but lane = {}", - lane - ); - simd_insert(src, lane as u32, *ptr) +#[rustc_legacy_const_generics(2)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld1.8", LANE = 7))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1, LANE = 7))] +pub unsafe fn vld1_lane_u8(ptr: *const u8, src: uint8x8_t) -> uint8x8_t { + static_assert_imm3!(LANE); + simd_insert(src, LANE as u32, *ptr) } /// Load one single-element structure to one lane of one register. #[inline] #[target_feature(enable = "neon")] #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[rustc_args_required_const(2)] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld1.8", lane = 15))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1, lane = 15))] -pub unsafe fn vld1q_lane_u8(ptr: *const u8, src: uint8x16_t, lane: i32) -> uint8x16_t { - assert!( - 0 <= lane && lane <= 15, - "must have 0 ≤ lane ≤ 15, but lane = {}", - lane - ); - simd_insert(src, lane as u32, *ptr) +#[rustc_legacy_const_generics(2)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld1.8", LANE = 15))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1, LANE = 15))] +pub unsafe fn vld1q_lane_u8(ptr: *const u8, src: uint8x16_t) -> uint8x16_t { + static_assert_imm4!(LANE); + simd_insert(src, LANE as u32, *ptr) } /// Load one single-element structure to one lane of one register. #[inline] #[target_feature(enable = "neon")] #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[rustc_args_required_const(2)] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld1.16", lane = 3))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1, lane = 3))] -pub unsafe fn vld1_lane_u16(ptr: *const u16, src: uint16x4_t, lane: i32) -> uint16x4_t { - assert!( - 0 <= lane && lane <= 3, - "must have 0 ≤ lane ≤ 3, but lane = {}", - lane - ); - simd_insert(src, lane as u32, *ptr) +#[rustc_legacy_const_generics(2)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld1.16", LANE = 3))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1, LANE = 3))] +pub unsafe fn vld1_lane_u16(ptr: *const u16, src: uint16x4_t) -> uint16x4_t { + static_assert_imm2!(LANE); + simd_insert(src, LANE as u32, *ptr) } /// Load one single-element structure to one lane of one register. #[inline] #[target_feature(enable = "neon")] #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[rustc_args_required_const(2)] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld1.16", lane = 7))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1, lane = 7))] -pub unsafe fn vld1q_lane_u16(ptr: *const u16, src: uint16x8_t, lane: i32) -> uint16x8_t { - assert!( - 0 <= lane && lane <= 7, - "must have 0 ≤ lane ≤ 7, but lane = {}", - lane - ); - simd_insert(src, lane as u32, *ptr) +#[rustc_legacy_const_generics(2)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld1.16", LANE = 7))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1, LANE = 7))] +pub unsafe fn vld1q_lane_u16(ptr: *const u16, src: uint16x8_t) -> uint16x8_t { + static_assert_imm3!(LANE); + simd_insert(src, LANE as u32, *ptr) } /// Load one single-element structure to one lane of one register. #[inline] #[target_feature(enable = "neon")] #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[rustc_args_required_const(2)] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld1.32", lane = 1))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1, lane = 1))] -pub unsafe fn vld1_lane_u32(ptr: *const u32, src: uint32x2_t, lane: i32) -> uint32x2_t { - assert!( - 0 <= lane && lane <= 1, - "must have 0 ≤ lane ≤ 1, but lane = {}", - lane - ); - simd_insert(src, lane as u32, *ptr) +#[rustc_legacy_const_generics(2)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld1.32", LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1, LANE = 1))] +pub unsafe fn vld1_lane_u32(ptr: *const u32, src: uint32x2_t) -> uint32x2_t { + static_assert_imm1!(LANE); + simd_insert(src, LANE as u32, *ptr) } /// Load one single-element structure to one lane of one register. #[inline] #[target_feature(enable = "neon")] #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[rustc_args_required_const(2)] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld1.32", lane = 3))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1, lane = 3))] -pub unsafe fn vld1q_lane_u32(ptr: *const u32, src: uint32x4_t, lane: i32) -> uint32x4_t { - assert!( - 0 <= lane && lane <= 3, - "must have 0 ≤ lane ≤ 3, but lane = {}", - lane - ); - simd_insert(src, lane as u32, *ptr) +#[rustc_legacy_const_generics(2)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld1.32", LANE = 3))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1, LANE = 3))] +pub unsafe fn vld1q_lane_u32(ptr: *const u32, src: uint32x4_t) -> uint32x4_t { + static_assert_imm2!(LANE); + simd_insert(src, LANE as u32, *ptr) } /// Load one single-element structure to one lane of one register. #[inline] #[target_feature(enable = "neon")] #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[rustc_args_required_const(2)] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vldr", lane = 0))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ldr, lane = 0))] -pub unsafe fn vld1_lane_u64(ptr: *const u64, src: uint64x1_t, lane: i32) -> uint64x1_t { - assert!( - 0 <= lane && lane <= 0, - "must have 0 ≤ lane ≤ 0, but lane = {}", - lane - ); - simd_insert(src, lane as u32, *ptr) +#[rustc_legacy_const_generics(2)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vldr", LANE = 0))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ldr, LANE = 0))] +pub unsafe fn vld1_lane_u64(ptr: *const u64, src: uint64x1_t) -> uint64x1_t { + static_assert!(LANE : i32 where LANE == 0); + simd_insert(src, LANE as u32, *ptr) } /// Load one single-element structure to one lane of one register. #[inline] #[target_feature(enable = "neon")] #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[rustc_args_required_const(2)] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vldr", lane = 1))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1, lane = 1))] -pub unsafe fn vld1q_lane_u64(ptr: *const u64, src: uint64x2_t, lane: i32) -> uint64x2_t { - assert!( - 0 <= lane && lane <= 1, - "must have 0 ≤ lane ≤ 1, but lane = {}", - lane - ); - simd_insert(src, lane as u32, *ptr) +#[rustc_legacy_const_generics(2)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vldr", LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1, LANE = 1))] +pub unsafe fn vld1q_lane_u64(ptr: *const u64, src: uint64x2_t) -> uint64x2_t { + static_assert_imm1!(LANE); + simd_insert(src, LANE as u32, *ptr) } /// Load one single-element structure to one lane of one register. #[inline] #[target_feature(enable = "neon")] #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[rustc_args_required_const(2)] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld1.8", lane = 7))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1, lane = 7))] -pub unsafe fn vld1_lane_p8(ptr: *const p8, src: poly8x8_t, lane: i32) -> poly8x8_t { - assert!( - 0 <= lane && lane <= 7, - "must have 0 ≤ lane ≤ 7, but lane = {}", - lane - ); - simd_insert(src, lane as u32, *ptr) +#[rustc_legacy_const_generics(2)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld1.8", LANE = 7))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1, LANE = 7))] +pub unsafe fn vld1_lane_p8(ptr: *const p8, src: poly8x8_t) -> poly8x8_t { + static_assert_imm3!(LANE); + simd_insert(src, LANE as u32, *ptr) } /// Load one single-element structure to one lane of one register. #[inline] #[target_feature(enable = "neon")] #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[rustc_args_required_const(2)] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld1.8", lane = 15))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1, lane = 15))] -pub unsafe fn vld1q_lane_p8(ptr: *const p8, src: poly8x16_t, lane: i32) -> poly8x16_t { - assert!( - 0 <= lane && lane <= 15, - "must have 0 ≤ lane ≤ 15, but lane = {}", - lane - ); - simd_insert(src, lane as u32, *ptr) +#[rustc_legacy_const_generics(2)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld1.8", LANE = 15))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1, LANE = 15))] +pub unsafe fn vld1q_lane_p8(ptr: *const p8, src: poly8x16_t) -> poly8x16_t { + static_assert_imm4!(LANE); + simd_insert(src, LANE as u32, *ptr) } /// Load one single-element structure to one lane of one register. #[inline] #[target_feature(enable = "neon")] #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[rustc_args_required_const(2)] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld1.16", lane = 3))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1, lane = 3))] -pub unsafe fn vld1_lane_p16(ptr: *const p16, src: poly16x4_t, lane: i32) -> poly16x4_t { - assert!( - 0 <= lane && lane <= 3, - "must have 0 ≤ lane ≤ 3, but lane = {}", - lane - ); - simd_insert(src, lane as u32, *ptr) +#[rustc_legacy_const_generics(2)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld1.16", LANE = 3))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1, LANE = 3))] +pub unsafe fn vld1_lane_p16(ptr: *const p16, src: poly16x4_t) -> poly16x4_t { + static_assert_imm2!(LANE); + simd_insert(src, LANE as u32, *ptr) } /// Load one single-element structure to one lane of one register. #[inline] #[target_feature(enable = "neon")] #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[rustc_args_required_const(2)] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld1.16", lane = 7))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1, lane = 7))] -pub unsafe fn vld1q_lane_p16(ptr: *const p16, src: poly16x8_t, lane: i32) -> poly16x8_t { - assert!( - 0 <= lane && lane <= 7, - "must have 0 ≤ lane ≤ 7, but lane = {}", - lane - ); - simd_insert(src, lane as u32, *ptr) +#[rustc_legacy_const_generics(2)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld1.16", LANE = 7))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1, LANE = 7))] +pub unsafe fn vld1q_lane_p16(ptr: *const p16, src: poly16x8_t) -> poly16x8_t { + static_assert_imm3!(LANE); + simd_insert(src, LANE as u32, *ptr) } /// Load one single-element structure to one lane of one register. #[inline] #[target_feature(enable = "neon")] #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[rustc_args_required_const(2)] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld1.32", lane = 1))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1, lane = 1))] -pub unsafe fn vld1_lane_f32(ptr: *const f32, src: float32x2_t, lane: i32) -> float32x2_t { - assert!( - 0 <= lane && lane <= 1, - "must have 0 ≤ lane ≤ 1, but lane = {}", - lane - ); - simd_insert(src, lane as u32, *ptr) +#[rustc_legacy_const_generics(2)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld1.32", LANE = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1, LANE = 1))] +pub unsafe fn vld1_lane_f32(ptr: *const f32, src: float32x2_t) -> float32x2_t { + static_assert_imm1!(LANE); + simd_insert(src, LANE as u32, *ptr) } /// Load one single-element structure to one lane of one register. #[inline] #[target_feature(enable = "neon")] #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[rustc_args_required_const(2)] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld1.32", lane = 3))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1, lane = 3))] -pub unsafe fn vld1q_lane_f32(ptr: *const f32, src: float32x4_t, lane: i32) -> float32x4_t { - assert!( - 0 <= lane && lane <= 3, - "must have 0 ≤ lane ≤ 3, but lane = {}", - lane - ); - simd_insert(src, lane as u32, *ptr) +#[rustc_legacy_const_generics(2)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld1.32", LANE = 3))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1, LANE = 3))] +pub unsafe fn vld1q_lane_f32(ptr: *const f32, src: float32x4_t) -> float32x4_t { + static_assert_imm2!(LANE); + simd_insert(src, LANE as u32, *ptr) } /// Load one single-element structure and Replicate to all lanes (of one register). @@ -944,8 +574,8 @@ pub unsafe fn vld1q_lane_f32(ptr: *const f32, src: float32x4_t, lane: i32) -> fl #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld1.8"))] #[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1r))] pub unsafe fn vld1_dup_s8(ptr: *const i8) -> int8x8_t { - let x = vld1_lane_s8(ptr, transmute(i8x8::splat(0)), 0); - simd_shuffle8(x, x, [0, 0, 0, 0, 0, 0, 0, 0]) + let x = vld1_lane_s8::<0>(ptr, transmute(i8x8::splat(0))); + simd_shuffle8!(x, x, [0, 0, 0, 0, 0, 0, 0, 0]) } /// Load one single-element structure and Replicate to all lanes (of one register). @@ -955,8 +585,8 @@ pub unsafe fn vld1_dup_s8(ptr: *const i8) -> int8x8_t { #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld1.8"))] #[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1r))] pub unsafe fn vld1q_dup_s8(ptr: *const i8) -> int8x16_t { - let x = vld1q_lane_s8(ptr, transmute(i8x16::splat(0)), 0); - simd_shuffle16(x, x, [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]) + let x = vld1q_lane_s8::<0>(ptr, transmute(i8x16::splat(0))); + simd_shuffle16!(x, x, [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]) } /// Load one single-element structure and Replicate to all lanes (of one register). @@ -966,8 +596,8 @@ pub unsafe fn vld1q_dup_s8(ptr: *const i8) -> int8x16_t { #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld1.16"))] #[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1r))] pub unsafe fn vld1_dup_s16(ptr: *const i16) -> int16x4_t { - let x = vld1_lane_s16(ptr, transmute(i16x4::splat(0)), 0); - simd_shuffle4(x, x, [0, 0, 0, 0]) + let x = vld1_lane_s16::<0>(ptr, transmute(i16x4::splat(0))); + simd_shuffle4!(x, x, [0, 0, 0, 0]) } /// Load one single-element structure and Replicate to all lanes (of one register). @@ -977,8 +607,8 @@ pub unsafe fn vld1_dup_s16(ptr: *const i16) -> int16x4_t { #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld1.16"))] #[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1r))] pub unsafe fn vld1q_dup_s16(ptr: *const i16) -> int16x8_t { - let x = vld1q_lane_s16(ptr, transmute(i16x8::splat(0)), 0); - simd_shuffle8(x, x, [0, 0, 0, 0, 0, 0, 0, 0]) + let x = vld1q_lane_s16::<0>(ptr, transmute(i16x8::splat(0))); + simd_shuffle8!(x, x, [0, 0, 0, 0, 0, 0, 0, 0]) } /// Load one single-element structure and Replicate to all lanes (of one register). @@ -988,8 +618,8 @@ pub unsafe fn vld1q_dup_s16(ptr: *const i16) -> int16x8_t { #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld1.32"))] #[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1r))] pub unsafe fn vld1_dup_s32(ptr: *const i32) -> int32x2_t { - let x = vld1_lane_s32(ptr, transmute(i32x2::splat(0)), 0); - simd_shuffle2(x, x, [0, 0]) + let x = vld1_lane_s32::<0>(ptr, transmute(i32x2::splat(0))); + simd_shuffle2!(x, x, [0, 0]) } /// Load one single-element structure and Replicate to all lanes (of one register). @@ -999,8 +629,8 @@ pub unsafe fn vld1_dup_s32(ptr: *const i32) -> int32x2_t { #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld1.32"))] #[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1r))] pub unsafe fn vld1q_dup_s32(ptr: *const i32) -> int32x4_t { - let x = vld1q_lane_s32(ptr, transmute(i32x4::splat(0)), 0); - simd_shuffle4(x, x, [0, 0, 0, 0]) + let x = vld1q_lane_s32::<0>(ptr, transmute(i32x4::splat(0))); + simd_shuffle4!(x, x, [0, 0, 0, 0]) } /// Load one single-element structure and Replicate to all lanes (of one register). @@ -1011,8 +641,13 @@ pub unsafe fn vld1q_dup_s32(ptr: *const i32) -> int32x4_t { #[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ldr))] pub unsafe fn vld1_dup_s64(ptr: *const i64) -> int64x1_t { #[cfg(target_arch = "aarch64")] - use crate::core_arch::aarch64::vld1_s64; - vld1_s64(ptr) + { + crate::core_arch::aarch64::vld1_s64(ptr) + } + #[cfg(target_arch = "arm")] + { + crate::core_arch::arm::vld1_s64(ptr) + } } /// Load one single-element structure and Replicate to all lanes (of one register). @@ -1022,8 +657,8 @@ pub unsafe fn vld1_dup_s64(ptr: *const i64) -> int64x1_t { #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vldr"))] #[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1r))] pub unsafe fn vld1q_dup_s64(ptr: *const i64) -> int64x2_t { - let x = vld1q_lane_s64(ptr, transmute(i64x2::splat(0)), 0); - simd_shuffle2(x, x, [0, 0]) + let x = vld1q_lane_s64::<0>(ptr, transmute(i64x2::splat(0))); + simd_shuffle2!(x, x, [0, 0]) } /// Load one single-element structure and Replicate to all lanes (of one register). @@ -1033,8 +668,8 @@ pub unsafe fn vld1q_dup_s64(ptr: *const i64) -> int64x2_t { #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld1.8"))] #[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1r))] pub unsafe fn vld1_dup_u8(ptr: *const u8) -> uint8x8_t { - let x = vld1_lane_u8(ptr, transmute(u8x8::splat(0)), 0); - simd_shuffle8(x, x, [0, 0, 0, 0, 0, 0, 0, 0]) + let x = vld1_lane_u8::<0>(ptr, transmute(u8x8::splat(0))); + simd_shuffle8!(x, x, [0, 0, 0, 0, 0, 0, 0, 0]) } /// Load one single-element structure and Replicate to all lanes (of one register). @@ -1044,8 +679,8 @@ pub unsafe fn vld1_dup_u8(ptr: *const u8) -> uint8x8_t { #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld1.8"))] #[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1r))] pub unsafe fn vld1q_dup_u8(ptr: *const u8) -> uint8x16_t { - let x = vld1q_lane_u8(ptr, transmute(u8x16::splat(0)), 0); - simd_shuffle16(x, x, [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]) + let x = vld1q_lane_u8::<0>(ptr, transmute(u8x16::splat(0))); + simd_shuffle16!(x, x, [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]) } /// Load one single-element structure and Replicate to all lanes (of one register). @@ -1055,8 +690,8 @@ pub unsafe fn vld1q_dup_u8(ptr: *const u8) -> uint8x16_t { #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld1.16"))] #[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1r))] pub unsafe fn vld1_dup_u16(ptr: *const u16) -> uint16x4_t { - let x = vld1_lane_u16(ptr, transmute(u16x4::splat(0)), 0); - simd_shuffle4(x, x, [0, 0, 0, 0]) + let x = vld1_lane_u16::<0>(ptr, transmute(u16x4::splat(0))); + simd_shuffle4!(x, x, [0, 0, 0, 0]) } /// Load one single-element structure and Replicate to all lanes (of one register). @@ -1066,8 +701,8 @@ pub unsafe fn vld1_dup_u16(ptr: *const u16) -> uint16x4_t { #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld1.16"))] #[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1r))] pub unsafe fn vld1q_dup_u16(ptr: *const u16) -> uint16x8_t { - let x = vld1q_lane_u16(ptr, transmute(u16x8::splat(0)), 0); - simd_shuffle8(x, x, [0, 0, 0, 0, 0, 0, 0, 0]) + let x = vld1q_lane_u16::<0>(ptr, transmute(u16x8::splat(0))); + simd_shuffle8!(x, x, [0, 0, 0, 0, 0, 0, 0, 0]) } /// Load one single-element structure and Replicate to all lanes (of one register). @@ -1077,8 +712,8 @@ pub unsafe fn vld1q_dup_u16(ptr: *const u16) -> uint16x8_t { #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld1.32"))] #[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1r))] pub unsafe fn vld1_dup_u32(ptr: *const u32) -> uint32x2_t { - let x = vld1_lane_u32(ptr, transmute(u32x2::splat(0)), 0); - simd_shuffle2(x, x, [0, 0]) + let x = vld1_lane_u32::<0>(ptr, transmute(u32x2::splat(0))); + simd_shuffle2!(x, x, [0, 0]) } /// Load one single-element structure and Replicate to all lanes (of one register). @@ -1088,8 +723,8 @@ pub unsafe fn vld1_dup_u32(ptr: *const u32) -> uint32x2_t { #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld1.32"))] #[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1r))] pub unsafe fn vld1q_dup_u32(ptr: *const u32) -> uint32x4_t { - let x = vld1q_lane_u32(ptr, transmute(u32x4::splat(0)), 0); - simd_shuffle4(x, x, [0, 0, 0, 0]) + let x = vld1q_lane_u32::<0>(ptr, transmute(u32x4::splat(0))); + simd_shuffle4!(x, x, [0, 0, 0, 0]) } /// Load one single-element structure and Replicate to all lanes (of one register). @@ -1100,8 +735,13 @@ pub unsafe fn vld1q_dup_u32(ptr: *const u32) -> uint32x4_t { #[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ldr))] pub unsafe fn vld1_dup_u64(ptr: *const u64) -> uint64x1_t { #[cfg(target_arch = "aarch64")] - use crate::core_arch::aarch64::vld1_u64; - vld1_u64(ptr) + { + crate::core_arch::aarch64::vld1_u64(ptr) + } + #[cfg(target_arch = "arm")] + { + crate::core_arch::arm::vld1_u64(ptr) + } } /// Load one single-element structure and Replicate to all lanes (of one register). @@ -1111,8 +751,8 @@ pub unsafe fn vld1_dup_u64(ptr: *const u64) -> uint64x1_t { #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vldr"))] #[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1r))] pub unsafe fn vld1q_dup_u64(ptr: *const u64) -> uint64x2_t { - let x = vld1q_lane_u64(ptr, transmute(u64x2::splat(0)), 0); - simd_shuffle2(x, x, [0, 0]) + let x = vld1q_lane_u64::<0>(ptr, transmute(u64x2::splat(0))); + simd_shuffle2!(x, x, [0, 0]) } /// Load one single-element structure and Replicate to all lanes (of one register). @@ -1122,8 +762,8 @@ pub unsafe fn vld1q_dup_u64(ptr: *const u64) -> uint64x2_t { #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld1.8"))] #[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1r))] pub unsafe fn vld1_dup_p8(ptr: *const p8) -> poly8x8_t { - let x = vld1_lane_p8(ptr, transmute(u8x8::splat(0)), 0); - simd_shuffle8(x, x, [0, 0, 0, 0, 0, 0, 0, 0]) + let x = vld1_lane_p8::<0>(ptr, transmute(u8x8::splat(0))); + simd_shuffle8!(x, x, [0, 0, 0, 0, 0, 0, 0, 0]) } /// Load one single-element structure and Replicate to all lanes (of one register). @@ -1133,8 +773,8 @@ pub unsafe fn vld1_dup_p8(ptr: *const p8) -> poly8x8_t { #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld1.8"))] #[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1r))] pub unsafe fn vld1q_dup_p8(ptr: *const p8) -> poly8x16_t { - let x = vld1q_lane_p8(ptr, transmute(u8x16::splat(0)), 0); - simd_shuffle16(x, x, [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]) + let x = vld1q_lane_p8::<0>(ptr, transmute(u8x16::splat(0))); + simd_shuffle16!(x, x, [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]) } /// Load one single-element structure and Replicate to all lanes (of one register). @@ -1144,8 +784,8 @@ pub unsafe fn vld1q_dup_p8(ptr: *const p8) -> poly8x16_t { #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld1.16"))] #[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1r))] pub unsafe fn vld1_dup_p16(ptr: *const p16) -> poly16x4_t { - let x = vld1_lane_p16(ptr, transmute(u16x4::splat(0)), 0); - simd_shuffle4(x, x, [0, 0, 0, 0]) + let x = vld1_lane_p16::<0>(ptr, transmute(u16x4::splat(0))); + simd_shuffle4!(x, x, [0, 0, 0, 0]) } /// Load one single-element structure and Replicate to all lanes (of one register). @@ -1155,8 +795,8 @@ pub unsafe fn vld1_dup_p16(ptr: *const p16) -> poly16x4_t { #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld1.16"))] #[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1r))] pub unsafe fn vld1q_dup_p16(ptr: *const p16) -> poly16x8_t { - let x = vld1q_lane_p16(ptr, transmute(u16x8::splat(0)), 0); - simd_shuffle8(x, x, [0, 0, 0, 0, 0, 0, 0, 0]) + let x = vld1q_lane_p16::<0>(ptr, transmute(u16x8::splat(0))); + simd_shuffle8!(x, x, [0, 0, 0, 0, 0, 0, 0, 0]) } /// Load one single-element structure and Replicate to all lanes (of one register). @@ -1166,8 +806,8 @@ pub unsafe fn vld1q_dup_p16(ptr: *const p16) -> poly16x8_t { #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld1.32"))] #[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1r))] pub unsafe fn vld1_dup_f32(ptr: *const f32) -> float32x2_t { - let x = vld1_lane_f32(ptr, transmute(f32x2::splat(0.)), 0); - simd_shuffle2(x, x, [0, 0]) + let x = vld1_lane_f32::<0>(ptr, transmute(f32x2::splat(0.))); + simd_shuffle2!(x, x, [0, 0]) } /// Load one single-element structure and Replicate to all lanes (of one register). @@ -1177,8 +817,107 @@ pub unsafe fn vld1_dup_f32(ptr: *const f32) -> float32x2_t { #[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld1.32"))] #[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ld1r))] pub unsafe fn vld1q_dup_f32(ptr: *const f32) -> float32x4_t { - let x = vld1q_lane_f32(ptr, transmute(f32x4::splat(0.)), 0); - simd_shuffle4(x, x, [0, 0, 0, 0]) + let x = vld1q_lane_f32::<0>(ptr, transmute(f32x4::splat(0.))); + simd_shuffle4!(x, x, [0, 0, 0, 0]) +} + +// signed absolute difference and accumulate (64-bit) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vaba.s8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr("saba"))] +pub unsafe fn vaba_s8(a: int8x8_t, b: int8x8_t, c: int8x8_t) -> int8x8_t { + simd_add(a, vabd_s8(b, c)) +} +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vaba.s16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr("saba"))] +pub unsafe fn vaba_s16(a: int16x4_t, b: int16x4_t, c: int16x4_t) -> int16x4_t { + simd_add(a, vabd_s16(b, c)) +} +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vaba.s32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr("saba"))] +pub unsafe fn vaba_s32(a: int32x2_t, b: int32x2_t, c: int32x2_t) -> int32x2_t { + simd_add(a, vabd_s32(b, c)) +} +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vaba.u8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr("uaba"))] +pub unsafe fn vaba_u8(a: uint8x8_t, b: uint8x8_t, c: uint8x8_t) -> uint8x8_t { + simd_add(a, vabd_u8(b, c)) +} +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vaba.u16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr("uaba"))] +pub unsafe fn vaba_u16(a: uint16x4_t, b: uint16x4_t, c: uint16x4_t) -> uint16x4_t { + simd_add(a, vabd_u16(b, c)) +} +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vaba.u32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr("uaba"))] +pub unsafe fn vaba_u32(a: uint32x2_t, b: uint32x2_t, c: uint32x2_t) -> uint32x2_t { + simd_add(a, vabd_u32(b, c)) +} +// signed absolute difference and accumulate (128-bit) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vaba.s8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr("saba"))] +pub unsafe fn vabaq_s8(a: int8x16_t, b: int8x16_t, c: int8x16_t) -> int8x16_t { + simd_add(a, vabdq_s8(b, c)) +} +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vaba.s16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr("saba"))] +pub unsafe fn vabaq_s16(a: int16x8_t, b: int16x8_t, c: int16x8_t) -> int16x8_t { + simd_add(a, vabdq_s16(b, c)) +} +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vaba.s32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr("saba"))] +pub unsafe fn vabaq_s32(a: int32x4_t, b: int32x4_t, c: int32x4_t) -> int32x4_t { + simd_add(a, vabdq_s32(b, c)) +} +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vaba.u8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr("uaba"))] +pub unsafe fn vabaq_u8(a: uint8x16_t, b: uint8x16_t, c: uint8x16_t) -> uint8x16_t { + simd_add(a, vabdq_u8(b, c)) +} +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vaba.u16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr("uaba"))] +pub unsafe fn vabaq_u16(a: uint16x8_t, b: uint16x8_t, c: uint16x8_t) -> uint16x8_t { + simd_add(a, vabdq_u16(b, c)) +} +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vaba.u32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr("uaba"))] +pub unsafe fn vabaq_u32(a: uint32x4_t, b: uint32x4_t, c: uint32x4_t) -> uint32x4_t { + simd_add(a, vabdq_u32(b, c)) } /// Absolute value (wrapping). @@ -1291,16 +1030,6 @@ pub unsafe fn vpadd_u8(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t { transmute(vpadd_s8_(transmute(a), transmute(b))) } -/// Unsigned saturating extract narrow. -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vqmovn.u64))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uqxtn))] -pub unsafe fn vqmovn_u64(a: uint64x2_t) -> uint32x2_t { - vqmovn_u64_(a) -} - /// Vector add. #[inline] #[target_feature(enable = "neon")] @@ -1540,8 +1269,8 @@ pub unsafe fn vaddl_u32(a: uint32x2_t, b: uint32x2_t) -> uint64x2_t { #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vaddl))] #[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(saddl2))] pub unsafe fn vaddl_high_s8(a: int8x16_t, b: int8x16_t) -> int16x8_t { - let a: int8x8_t = simd_shuffle8(a, a, [8, 9, 10, 11, 12, 13, 14, 15]); - let b: int8x8_t = simd_shuffle8(b, b, [8, 9, 10, 11, 12, 13, 14, 15]); + let a: int8x8_t = simd_shuffle8!(a, a, [8, 9, 10, 11, 12, 13, 14, 15]); + let b: int8x8_t = simd_shuffle8!(b, b, [8, 9, 10, 11, 12, 13, 14, 15]); let a: int16x8_t = simd_cast(a); let b: int16x8_t = simd_cast(b); simd_add(a, b) @@ -1554,8 +1283,8 @@ pub unsafe fn vaddl_high_s8(a: int8x16_t, b: int8x16_t) -> int16x8_t { #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vaddl))] #[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(saddl2))] pub unsafe fn vaddl_high_s16(a: int16x8_t, b: int16x8_t) -> int32x4_t { - let a: int16x4_t = simd_shuffle4(a, a, [4, 5, 6, 7]); - let b: int16x4_t = simd_shuffle4(b, b, [4, 5, 6, 7]); + let a: int16x4_t = simd_shuffle4!(a, a, [4, 5, 6, 7]); + let b: int16x4_t = simd_shuffle4!(b, b, [4, 5, 6, 7]); let a: int32x4_t = simd_cast(a); let b: int32x4_t = simd_cast(b); simd_add(a, b) @@ -1568,8 +1297,8 @@ pub unsafe fn vaddl_high_s16(a: int16x8_t, b: int16x8_t) -> int32x4_t { #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vaddl))] #[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(saddl2))] pub unsafe fn vaddl_high_s32(a: int32x4_t, b: int32x4_t) -> int64x2_t { - let a: int32x2_t = simd_shuffle2(a, a, [2, 3]); - let b: int32x2_t = simd_shuffle2(b, b, [2, 3]); + let a: int32x2_t = simd_shuffle2!(a, a, [2, 3]); + let b: int32x2_t = simd_shuffle2!(b, b, [2, 3]); let a: int64x2_t = simd_cast(a); let b: int64x2_t = simd_cast(b); simd_add(a, b) @@ -1582,8 +1311,8 @@ pub unsafe fn vaddl_high_s32(a: int32x4_t, b: int32x4_t) -> int64x2_t { #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vaddl))] #[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uaddl2))] pub unsafe fn vaddl_high_u8(a: uint8x16_t, b: uint8x16_t) -> uint16x8_t { - let a: uint8x8_t = simd_shuffle8(a, a, [8, 9, 10, 11, 12, 13, 14, 15]); - let b: uint8x8_t = simd_shuffle8(b, b, [8, 9, 10, 11, 12, 13, 14, 15]); + let a: uint8x8_t = simd_shuffle8!(a, a, [8, 9, 10, 11, 12, 13, 14, 15]); + let b: uint8x8_t = simd_shuffle8!(b, b, [8, 9, 10, 11, 12, 13, 14, 15]); let a: uint16x8_t = simd_cast(a); let b: uint16x8_t = simd_cast(b); simd_add(a, b) @@ -1596,8 +1325,8 @@ pub unsafe fn vaddl_high_u8(a: uint8x16_t, b: uint8x16_t) -> uint16x8_t { #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vaddl))] #[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uaddl2))] pub unsafe fn vaddl_high_u16(a: uint16x8_t, b: uint16x8_t) -> uint32x4_t { - let a: uint16x4_t = simd_shuffle4(a, a, [4, 5, 6, 7]); - let b: uint16x4_t = simd_shuffle4(b, b, [4, 5, 6, 7]); + let a: uint16x4_t = simd_shuffle4!(a, a, [4, 5, 6, 7]); + let b: uint16x4_t = simd_shuffle4!(b, b, [4, 5, 6, 7]); let a: uint32x4_t = simd_cast(a); let b: uint32x4_t = simd_cast(b); simd_add(a, b) @@ -1610,8 +1339,8 @@ pub unsafe fn vaddl_high_u16(a: uint16x8_t, b: uint16x8_t) -> uint32x4_t { #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vaddl))] #[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uaddl2))] pub unsafe fn vaddl_high_u32(a: uint32x4_t, b: uint32x4_t) -> uint64x2_t { - let a: uint32x2_t = simd_shuffle2(a, a, [2, 3]); - let b: uint32x2_t = simd_shuffle2(b, b, [2, 3]); + let a: uint32x2_t = simd_shuffle2!(a, a, [2, 3]); + let b: uint32x2_t = simd_shuffle2!(b, b, [2, 3]); let a: uint64x2_t = simd_cast(a); let b: uint64x2_t = simd_cast(b); simd_add(a, b) @@ -1690,7 +1419,7 @@ pub unsafe fn vaddw_u32(a: uint64x2_t, b: uint32x2_t) -> uint64x2_t { #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vaddw))] #[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(saddw2))] pub unsafe fn vaddw_high_s8(a: int16x8_t, b: int8x16_t) -> int16x8_t { - let b: int8x8_t = simd_shuffle8(b, b, [8, 9, 10, 11, 12, 13, 14, 15]); + let b: int8x8_t = simd_shuffle8!(b, b, [8, 9, 10, 11, 12, 13, 14, 15]); let b: int16x8_t = simd_cast(b); simd_add(a, b) } @@ -1702,7 +1431,7 @@ pub unsafe fn vaddw_high_s8(a: int16x8_t, b: int8x16_t) -> int16x8_t { #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vaddw))] #[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(saddw2))] pub unsafe fn vaddw_high_s16(a: int32x4_t, b: int16x8_t) -> int32x4_t { - let b: int16x4_t = simd_shuffle4(b, b, [4, 5, 6, 7]); + let b: int16x4_t = simd_shuffle4!(b, b, [4, 5, 6, 7]); let b: int32x4_t = simd_cast(b); simd_add(a, b) } @@ -1714,7 +1443,7 @@ pub unsafe fn vaddw_high_s16(a: int32x4_t, b: int16x8_t) -> int32x4_t { #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vaddw))] #[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(saddw2))] pub unsafe fn vaddw_high_s32(a: int64x2_t, b: int32x4_t) -> int64x2_t { - let b: int32x2_t = simd_shuffle2(b, b, [2, 3]); + let b: int32x2_t = simd_shuffle2!(b, b, [2, 3]); let b: int64x2_t = simd_cast(b); simd_add(a, b) } @@ -1726,7 +1455,7 @@ pub unsafe fn vaddw_high_s32(a: int64x2_t, b: int32x4_t) -> int64x2_t { #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vaddw))] #[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uaddw2))] pub unsafe fn vaddw_high_u8(a: uint16x8_t, b: uint8x16_t) -> uint16x8_t { - let b: uint8x8_t = simd_shuffle8(b, b, [8, 9, 10, 11, 12, 13, 14, 15]); + let b: uint8x8_t = simd_shuffle8!(b, b, [8, 9, 10, 11, 12, 13, 14, 15]); let b: uint16x8_t = simd_cast(b); simd_add(a, b) } @@ -1738,7 +1467,7 @@ pub unsafe fn vaddw_high_u8(a: uint16x8_t, b: uint8x16_t) -> uint16x8_t { #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vaddw))] #[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uaddw2))] pub unsafe fn vaddw_high_u16(a: uint32x4_t, b: uint16x8_t) -> uint32x4_t { - let b: uint16x4_t = simd_shuffle4(b, b, [4, 5, 6, 7]); + let b: uint16x4_t = simd_shuffle4!(b, b, [4, 5, 6, 7]); let b: uint32x4_t = simd_cast(b); simd_add(a, b) } @@ -1750,7 +1479,7 @@ pub unsafe fn vaddw_high_u16(a: uint32x4_t, b: uint16x8_t) -> uint32x4_t { #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vaddw))] #[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uaddw2))] pub unsafe fn vaddw_high_u32(a: uint64x2_t, b: uint32x4_t) -> uint64x2_t { - let b: uint32x2_t = simd_shuffle2(b, b, [2, 3]); + let b: uint32x2_t = simd_shuffle2!(b, b, [2, 3]); let b: uint64x2_t = simd_cast(b); simd_add(a, b) } @@ -1823,7 +1552,7 @@ pub unsafe fn vaddhn_u64(a: uint64x2_t, b: uint64x2_t) -> uint32x2_t { #[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(addhn2))] pub unsafe fn vaddhn_high_s16(r: int8x8_t, a: int16x8_t, b: int16x8_t) -> int8x16_t { let x = simd_cast(simd_shr(simd_add(a, b), int16x8_t(8, 8, 8, 8, 8, 8, 8, 8))); - simd_shuffle16(r, x, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]) + simd_shuffle16!(r, x, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]) } /// Add returning High Narrow (high half). @@ -1834,7 +1563,7 @@ pub unsafe fn vaddhn_high_s16(r: int8x8_t, a: int16x8_t, b: int16x8_t) -> int8x1 #[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(addhn2))] pub unsafe fn vaddhn_high_s32(r: int16x4_t, a: int32x4_t, b: int32x4_t) -> int16x8_t { let x = simd_cast(simd_shr(simd_add(a, b), int32x4_t(16, 16, 16, 16))); - simd_shuffle8(r, x, [0, 1, 2, 3, 4, 5, 6, 7]) + simd_shuffle8!(r, x, [0, 1, 2, 3, 4, 5, 6, 7]) } /// Add returning High Narrow (high half). @@ -1845,7 +1574,7 @@ pub unsafe fn vaddhn_high_s32(r: int16x4_t, a: int32x4_t, b: int32x4_t) -> int16 #[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(addhn2))] pub unsafe fn vaddhn_high_s64(r: int32x2_t, a: int64x2_t, b: int64x2_t) -> int32x4_t { let x = simd_cast(simd_shr(simd_add(a, b), int64x2_t(32, 32))); - simd_shuffle4(r, x, [0, 1, 2, 3]) + simd_shuffle4!(r, x, [0, 1, 2, 3]) } /// Add returning High Narrow (high half). @@ -1856,7 +1585,7 @@ pub unsafe fn vaddhn_high_s64(r: int32x2_t, a: int64x2_t, b: int64x2_t) -> int32 #[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(addhn2))] pub unsafe fn vaddhn_high_u16(r: uint8x8_t, a: uint16x8_t, b: uint16x8_t) -> uint8x16_t { let x = simd_cast(simd_shr(simd_add(a, b), uint16x8_t(8, 8, 8, 8, 8, 8, 8, 8))); - simd_shuffle16(r, x, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]) + simd_shuffle16!(r, x, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]) } /// Add returning High Narrow (high half). @@ -1867,7 +1596,7 @@ pub unsafe fn vaddhn_high_u16(r: uint8x8_t, a: uint16x8_t, b: uint16x8_t) -> uin #[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(addhn2))] pub unsafe fn vaddhn_high_u32(r: uint16x4_t, a: uint32x4_t, b: uint32x4_t) -> uint16x8_t { let x = simd_cast(simd_shr(simd_add(a, b), uint32x4_t(16, 16, 16, 16))); - simd_shuffle8(r, x, [0, 1, 2, 3, 4, 5, 6, 7]) + simd_shuffle8!(r, x, [0, 1, 2, 3, 4, 5, 6, 7]) } /// Add returning High Narrow (high half). @@ -1878,7 +1607,7 @@ pub unsafe fn vaddhn_high_u32(r: uint16x4_t, a: uint32x4_t, b: uint32x4_t) -> ui #[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(addhn2))] pub unsafe fn vaddhn_high_u64(r: uint32x2_t, a: uint64x2_t, b: uint64x2_t) -> uint32x4_t { let x = simd_cast(simd_shr(simd_add(a, b), uint64x2_t(32, 32))); - simd_shuffle4(r, x, [0, 1, 2, 3]) + simd_shuffle4!(r, x, [0, 1, 2, 3]) } /// Rounding Add returning High Narrow. @@ -1949,7 +1678,7 @@ pub unsafe fn vraddhn_u64(a: uint64x2_t, b: uint64x2_t) -> uint32x2_t { #[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(raddhn2))] pub unsafe fn vraddhn_high_s16(r: int8x8_t, a: int16x8_t, b: int16x8_t) -> int8x16_t { let x = vraddhn_s16_(a, b); - simd_shuffle16(r, x, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]) + simd_shuffle16!(r, x, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]) } /// Rounding Add returning High Narrow (high half). @@ -1960,7 +1689,7 @@ pub unsafe fn vraddhn_high_s16(r: int8x8_t, a: int16x8_t, b: int16x8_t) -> int8x #[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(raddhn2))] pub unsafe fn vraddhn_high_s32(r: int16x4_t, a: int32x4_t, b: int32x4_t) -> int16x8_t { let x = vraddhn_s32_(a, b); - simd_shuffle8(r, x, [0, 1, 2, 3, 4, 5, 6, 7]) + simd_shuffle8!(r, x, [0, 1, 2, 3, 4, 5, 6, 7]) } /// Rounding Add returning High Narrow (high half). @@ -1971,7 +1700,7 @@ pub unsafe fn vraddhn_high_s32(r: int16x4_t, a: int32x4_t, b: int32x4_t) -> int1 #[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(raddhn2))] pub unsafe fn vraddhn_high_s64(r: int32x2_t, a: int64x2_t, b: int64x2_t) -> int32x4_t { let x = vraddhn_s64_(a, b); - simd_shuffle4(r, x, [0, 1, 2, 3]) + simd_shuffle4!(r, x, [0, 1, 2, 3]) } /// Rounding Add returning High Narrow (high half). @@ -1982,7 +1711,7 @@ pub unsafe fn vraddhn_high_s64(r: int32x2_t, a: int64x2_t, b: int64x2_t) -> int3 #[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(raddhn2))] pub unsafe fn vraddhn_high_u16(r: uint8x8_t, a: uint16x8_t, b: uint16x8_t) -> uint8x16_t { let x: uint8x8_t = transmute(vraddhn_s16_(transmute(a), transmute(b))); - simd_shuffle16(r, x, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]) + simd_shuffle16!(r, x, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]) } /// Rounding Add returning High Narrow (high half). @@ -1993,7 +1722,7 @@ pub unsafe fn vraddhn_high_u16(r: uint8x8_t, a: uint16x8_t, b: uint16x8_t) -> ui #[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(raddhn2))] pub unsafe fn vraddhn_high_u32(r: uint16x4_t, a: uint32x4_t, b: uint32x4_t) -> uint16x8_t { let x: uint16x4_t = transmute(vraddhn_s32_(transmute(a), transmute(b))); - simd_shuffle8(r, x, [0, 1, 2, 3, 4, 5, 6, 7]) + simd_shuffle8!(r, x, [0, 1, 2, 3, 4, 5, 6, 7]) } /// Rounding Add returning High Narrow (high half). @@ -2004,7 +1733,7 @@ pub unsafe fn vraddhn_high_u32(r: uint16x4_t, a: uint32x4_t, b: uint32x4_t) -> u #[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(raddhn2))] pub unsafe fn vraddhn_high_u64(r: uint32x2_t, a: uint64x2_t, b: uint64x2_t) -> uint32x4_t { let x: uint32x2_t = transmute(vraddhn_s64_(transmute(a), transmute(b))); - simd_shuffle4(r, x, [0, 1, 2, 3]) + simd_shuffle4!(r, x, [0, 1, 2, 3]) } /// Signed Add Long Pairwise. @@ -2127,271 +1856,67 @@ pub unsafe fn vpaddlq_u32(a: uint32x4_t) -> uint64x2_t { vpaddlq_u32_(a) } -/// Signed Add and Accumulate Long Pairwise. +/// Vector narrow integer. #[inline] #[target_feature(enable = "neon")] #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vpadal.s8))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sadalp))] -pub unsafe fn vpadal_s8(a: int16x4_t, b: int8x8_t) -> int16x4_t { - #[cfg(target_arch = "arm")] - { - vpadal_s8_(a, b) - } - #[cfg(target_arch = "aarch64")] - { - simd_add(vpaddl_s8_(b), a) - } +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmovn))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(xtn))] +pub unsafe fn vmovn_s16(a: int16x8_t) -> int8x8_t { + simd_cast(a) } -/// Signed Add and Accumulate Long Pairwise. +/// Vector narrow integer. #[inline] #[target_feature(enable = "neon")] #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vpadal.s16))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sadalp))] -pub unsafe fn vpadal_s16(a: int32x2_t, b: int16x4_t) -> int32x2_t { - #[cfg(target_arch = "arm")] - { - vpadal_s16_(a, b) - } - #[cfg(target_arch = "aarch64")] - { - simd_add(vpaddl_s16_(b), a) - } +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmovn))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(xtn))] +pub unsafe fn vmovn_s32(a: int32x4_t) -> int16x4_t { + simd_cast(a) } -/// Signed Add and Accumulate Long Pairwise. +/// Vector narrow integer. #[inline] #[target_feature(enable = "neon")] #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vpadal.s32))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sadalp))] -pub unsafe fn vpadal_s32(a: int64x1_t, b: int32x2_t) -> int64x1_t { - #[cfg(target_arch = "arm")] - { - vpadal_s32_(a, b) - } - #[cfg(target_arch = "aarch64")] - { - simd_add(vpaddl_s32_(b), a) - } +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmovn))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(xtn))] +pub unsafe fn vmovn_s64(a: int64x2_t) -> int32x2_t { + simd_cast(a) } -/// Signed Add and Accumulate Long Pairwise. +/// Vector narrow integer. #[inline] #[target_feature(enable = "neon")] #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vpadal.s8))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sadalp))] -pub unsafe fn vpadalq_s8(a: int16x8_t, b: int8x16_t) -> int16x8_t { - #[cfg(target_arch = "arm")] - { - vpadalq_s8_(a, b) - } - #[cfg(target_arch = "aarch64")] - { - simd_add(vpaddlq_s8_(b), a) - } +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmovn))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(xtn))] +pub unsafe fn vmovn_u16(a: uint16x8_t) -> uint8x8_t { + simd_cast(a) } -/// Signed Add and Accumulate Long Pairwise. +/// Vector narrow integer. #[inline] #[target_feature(enable = "neon")] #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vpadal.s16))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sadalp))] -pub unsafe fn vpadalq_s16(a: int32x4_t, b: int16x8_t) -> int32x4_t { - #[cfg(target_arch = "arm")] - { - vpadalq_s16_(a, b) - } - #[cfg(target_arch = "aarch64")] - { - simd_add(vpaddlq_s16_(b), a) - } +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmovn))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(xtn))] +pub unsafe fn vmovn_u32(a: uint32x4_t) -> uint16x4_t { + simd_cast(a) } -/// Signed Add and Accumulate Long Pairwise. +/// Vector narrow integer. #[inline] #[target_feature(enable = "neon")] #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vpadal.s32))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sadalp))] -pub unsafe fn vpadalq_s32(a: int64x2_t, b: int32x4_t) -> int64x2_t { - #[cfg(target_arch = "arm")] - { - vpadalq_s32_(a, b) - } - #[cfg(target_arch = "aarch64")] - { - simd_add(vpaddlq_s32_(b), a) - } +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmovn))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(xtn))] +pub unsafe fn vmovn_u64(a: uint64x2_t) -> uint32x2_t { + simd_cast(a) } -/// Unsigned Add and Accumulate Long Pairwise. -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vpadal.u8))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uadalp))] -pub unsafe fn vpadal_u8(a: uint16x4_t, b: uint8x8_t) -> uint16x4_t { - #[cfg(target_arch = "arm")] - { - vpadal_u8_(a, b) - } - #[cfg(target_arch = "aarch64")] - { - simd_add(vpaddl_u8_(b), a) - } -} - -/// Unsigned Add and Accumulate Long Pairwise. -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vpadal.u16))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uadalp))] -pub unsafe fn vpadal_u16(a: uint32x2_t, b: uint16x4_t) -> uint32x2_t { - #[cfg(target_arch = "arm")] - { - vpadal_u16_(a, b) - } - #[cfg(target_arch = "aarch64")] - { - simd_add(vpaddl_u16_(b), a) - } -} - -/// Unsigned Add and Accumulate Long Pairwise. -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vpadal.u32))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uadalp))] -pub unsafe fn vpadal_u32(a: uint64x1_t, b: uint32x2_t) -> uint64x1_t { - #[cfg(target_arch = "arm")] - { - vpadal_u32_(a, b) - } - #[cfg(target_arch = "aarch64")] - { - simd_add(vpaddl_u32_(b), a) - } -} - -/// Unsigned Add and Accumulate Long Pairwise. -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vpadal.u8))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uadalp))] -pub unsafe fn vpadalq_u8(a: uint16x8_t, b: uint8x16_t) -> uint16x8_t { - #[cfg(target_arch = "arm")] - { - vpadalq_u8_(a, b) - } - #[cfg(target_arch = "aarch64")] - { - simd_add(vpaddlq_u8_(b), a) - } -} - -/// Unsigned Add and Accumulate Long Pairwise. -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vpadal.u16))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uadalp))] -pub unsafe fn vpadalq_u16(a: uint32x4_t, b: uint16x8_t) -> uint32x4_t { - #[cfg(target_arch = "arm")] - { - vpadalq_u16_(a, b) - } - #[cfg(target_arch = "aarch64")] - { - simd_add(vpaddlq_u16_(b), a) - } -} - -/// Unsigned Add and Accumulate Long Pairwise. -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vpadal.u32))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uadalp))] -pub unsafe fn vpadalq_u32(a: uint64x2_t, b: uint32x4_t) -> uint64x2_t { - #[cfg(target_arch = "arm")] - { - vpadalq_u32_(a, b) - } - #[cfg(target_arch = "aarch64")] - { - simd_add(vpaddlq_u32_(b), a) - } -} - -/// Vector narrow integer. -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmovn))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(xtn))] -pub unsafe fn vmovn_s16(a: int16x8_t) -> int8x8_t { - simd_cast(a) -} - -/// Vector narrow integer. -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmovn))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(xtn))] -pub unsafe fn vmovn_s32(a: int32x4_t) -> int16x4_t { - simd_cast(a) -} - -/// Vector narrow integer. -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmovn))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(xtn))] -pub unsafe fn vmovn_s64(a: int64x2_t) -> int32x2_t { - simd_cast(a) -} - -/// Vector narrow integer. -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmovn))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(xtn))] -pub unsafe fn vmovn_u16(a: uint16x8_t) -> uint8x8_t { - simd_cast(a) -} - -/// Vector narrow integer. -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmovn))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(xtn))] -pub unsafe fn vmovn_u32(a: uint32x4_t) -> uint16x4_t { - simd_cast(a) -} - -/// Vector narrow integer. -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vmovn))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(xtn))] -pub unsafe fn vmovn_u64(a: uint64x2_t) -> uint32x2_t { - simd_cast(a) -} - -/// Vector long move. +/// Vector long move. #[inline] #[target_feature(enable = "neon")] #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] @@ -2451,15 +1976,6 @@ pub unsafe fn vmovl_u32(a: uint32x2_t) -> uint64x2_t { simd_cast(a) } -/// Reciprocal square-root estimate. -#[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(frsqrte))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vrsqrte))] -pub unsafe fn vrsqrte_f32(a: float32x2_t) -> float32x2_t { - frsqrte_v2f32(a) -} - /// Vector bitwise not. #[inline] #[target_feature(enable = "neon")] @@ -2622,3387 +2138,5471 @@ pub unsafe fn vmvnq_p8(a: poly8x16_t) -> poly8x16_t { simd_xor(a, b) } -/// Folding minimum of adjacent pairs +/// Vector bitwise bit clear #[inline] #[target_feature(enable = "neon")] #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vpmin))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sminp))] -pub unsafe fn vpmin_s8(a: int8x8_t, b: int8x8_t) -> int8x8_t { - vpmins_v8i8(a, b) +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vbic))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(bic))] +pub unsafe fn vbic_s8(a: int8x8_t, b: int8x8_t) -> int8x8_t { + let c = int8x8_t(-1, -1, -1, -1, -1, -1, -1, -1); + simd_and(simd_xor(b, c), a) } -/// Folding minimum of adjacent pairs +/// Vector bitwise bit clear #[inline] #[target_feature(enable = "neon")] #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vpmin))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sminp))] -pub unsafe fn vpmin_s16(a: int16x4_t, b: int16x4_t) -> int16x4_t { - vpmins_v4i16(a, b) +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vbic))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(bic))] +pub unsafe fn vbicq_s8(a: int8x16_t, b: int8x16_t) -> int8x16_t { + let c = int8x16_t( + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + ); + simd_and(simd_xor(b, c), a) } -/// Folding minimum of adjacent pairs +/// Vector bitwise bit clear #[inline] #[target_feature(enable = "neon")] #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vpmin))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sminp))] -pub unsafe fn vpmin_s32(a: int32x2_t, b: int32x2_t) -> int32x2_t { - vpmins_v2i32(a, b) +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vbic))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(bic))] +pub unsafe fn vbic_s16(a: int16x4_t, b: int16x4_t) -> int16x4_t { + let c = int16x4_t(-1, -1, -1, -1); + simd_and(simd_xor(b, c), a) } -/// Folding minimum of adjacent pairs +/// Vector bitwise bit clear #[inline] #[target_feature(enable = "neon")] #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vpmin))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uminp))] -pub unsafe fn vpmin_u8(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t { - vpminu_v8i8(a, b) +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vbic))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(bic))] +pub unsafe fn vbicq_s16(a: int16x8_t, b: int16x8_t) -> int16x8_t { + let c = int16x8_t(-1, -1, -1, -1, -1, -1, -1, -1); + simd_and(simd_xor(b, c), a) } -/// Folding minimum of adjacent pairs +/// Vector bitwise bit clear #[inline] #[target_feature(enable = "neon")] #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vpmin))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uminp))] -pub unsafe fn vpmin_u16(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t { - vpminu_v4i16(a, b) +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vbic))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(bic))] +pub unsafe fn vbic_s32(a: int32x2_t, b: int32x2_t) -> int32x2_t { + let c = int32x2_t(-1, -1); + simd_and(simd_xor(b, c), a) } -/// Folding minimum of adjacent pairs +/// Vector bitwise bit clear #[inline] #[target_feature(enable = "neon")] #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vpmin))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uminp))] -pub unsafe fn vpmin_u32(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t { - vpminu_v2i32(a, b) +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vbic))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(bic))] +pub unsafe fn vbicq_s32(a: int32x4_t, b: int32x4_t) -> int32x4_t { + let c = int32x4_t(-1, -1, -1, -1); + simd_and(simd_xor(b, c), a) } -/// Folding minimum of adjacent pairs +/// Vector bitwise bit clear #[inline] #[target_feature(enable = "neon")] #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vpmin))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fminp))] -pub unsafe fn vpmin_f32(a: float32x2_t, b: float32x2_t) -> float32x2_t { - vpminf_v2f32(a, b) +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vbic))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(bic))] +pub unsafe fn vbic_s64(a: int64x1_t, b: int64x1_t) -> int64x1_t { + let c = int64x1_t(-1); + simd_and(simd_xor(b, c), a) } -/// Folding maximum of adjacent pairs +/// Vector bitwise bit clear #[inline] #[target_feature(enable = "neon")] #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vpmax))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(smaxp))] -pub unsafe fn vpmax_s8(a: int8x8_t, b: int8x8_t) -> int8x8_t { - vpmaxs_v8i8(a, b) +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vbic))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(bic))] +pub unsafe fn vbicq_s64(a: int64x2_t, b: int64x2_t) -> int64x2_t { + let c = int64x2_t(-1, -1); + simd_and(simd_xor(b, c), a) } -/// Folding maximum of adjacent pairs +/// Vector bitwise bit clear #[inline] #[target_feature(enable = "neon")] #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vpmax))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(smaxp))] -pub unsafe fn vpmax_s16(a: int16x4_t, b: int16x4_t) -> int16x4_t { - vpmaxs_v4i16(a, b) +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vbic))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(bic))] +pub unsafe fn vbic_u8(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t { + let c = int8x8_t(-1, -1, -1, -1, -1, -1, -1, -1); + simd_and(simd_xor(b, transmute(c)), a) } -/// Folding maximum of adjacent pairs +/// Vector bitwise bit clear #[inline] #[target_feature(enable = "neon")] #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vpmax))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(smaxp))] -pub unsafe fn vpmax_s32(a: int32x2_t, b: int32x2_t) -> int32x2_t { - vpmaxs_v2i32(a, b) +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vbic))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(bic))] +pub unsafe fn vbicq_u8(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t { + let c = int8x16_t( + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + ); + simd_and(simd_xor(b, transmute(c)), a) } -/// Folding maximum of adjacent pairs +/// Vector bitwise bit clear #[inline] #[target_feature(enable = "neon")] #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vpmax))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(umaxp))] -pub unsafe fn vpmax_u8(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t { - vpmaxu_v8i8(a, b) +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vbic))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(bic))] +pub unsafe fn vbic_u16(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t { + let c = int16x4_t(-1, -1, -1, -1); + simd_and(simd_xor(b, transmute(c)), a) } -/// Folding maximum of adjacent pairs +/// Vector bitwise bit clear #[inline] #[target_feature(enable = "neon")] #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vpmax))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(umaxp))] -pub unsafe fn vpmax_u16(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t { - vpmaxu_v4i16(a, b) +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vbic))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(bic))] +pub unsafe fn vbicq_u16(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t { + let c = int16x8_t(-1, -1, -1, -1, -1, -1, -1, -1); + simd_and(simd_xor(b, transmute(c)), a) } -/// Folding maximum of adjacent pairs +/// Vector bitwise bit clear #[inline] #[target_feature(enable = "neon")] #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vpmax))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(umaxp))] -pub unsafe fn vpmax_u32(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t { - vpmaxu_v2i32(a, b) +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vbic))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(bic))] +pub unsafe fn vbic_u32(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t { + let c = int32x2_t(-1, -1); + simd_and(simd_xor(b, transmute(c)), a) } -/// Folding maximum of adjacent pairs +/// Vector bitwise bit clear #[inline] #[target_feature(enable = "neon")] #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vpmax))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fmaxp))] -pub unsafe fn vpmax_f32(a: float32x2_t, b: float32x2_t) -> float32x2_t { - vpmaxf_v2f32(a, b) +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vbic))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(bic))] +pub unsafe fn vbicq_u32(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t { + let c = int32x4_t(-1, -1, -1, -1); + simd_and(simd_xor(b, transmute(c)), a) } -/// Table look-up +/// Vector bitwise bit clear #[inline] -#[cfg(target_arch = "arm")] -#[cfg(target_endian = "little")] -#[target_feature(enable = "neon,v7")] -#[cfg_attr(test, assert_instr(vtbl))] -pub unsafe fn vtbl1_s8(a: int8x8_t, b: int8x8_t) -> int8x8_t { - vtbl1(a, b) +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vbic))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(bic))] +pub unsafe fn vbic_u64(a: uint64x1_t, b: uint64x1_t) -> uint64x1_t { + let c = int64x1_t(-1); + simd_and(simd_xor(b, transmute(c)), a) } -/// Table look-up +/// Vector bitwise bit clear #[inline] -#[cfg(target_arch = "arm")] -#[cfg(target_endian = "little")] -#[target_feature(enable = "neon,v7")] -#[cfg_attr(test, assert_instr(vtbl))] -pub unsafe fn vtbl1_u8(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t { - transmute(vtbl1(transmute(a), transmute(b))) +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vbic))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(bic))] +pub unsafe fn vbicq_u64(a: uint64x2_t, b: uint64x2_t) -> uint64x2_t { + let c = int64x2_t(-1, -1); + simd_and(simd_xor(b, transmute(c)), a) } -/// Table look-up -#[inline] -#[cfg(target_arch = "arm")] -#[cfg(target_endian = "little")] -#[target_feature(enable = "neon,v7")] -#[cfg_attr(test, assert_instr(vtbl))] -pub unsafe fn vtbl1_p8(a: poly8x8_t, b: uint8x8_t) -> poly8x8_t { - transmute(vtbl1(transmute(a), transmute(b))) -} +/// Bitwise Select instructions. This instruction sets each bit in the destination SIMD&FP register +/// to the corresponding bit from the first source SIMD&FP register when the original +/// destination bit was 1, otherwise from the second source SIMD&FP register. -/// Table look-up +/// Bitwise Select. #[inline] -#[cfg(target_arch = "arm")] -#[cfg(target_endian = "little")] -#[target_feature(enable = "neon,v7")] -#[cfg_attr(test, assert_instr(vtbl))] -pub unsafe fn vtbl2_s8(a: int8x8x2_t, b: int8x8_t) -> int8x8_t { - vtbl2(a.0, a.1, b) +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vbsl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(bsl))] +pub unsafe fn vbsl_s8(a: uint8x8_t, b: int8x8_t, c: int8x8_t) -> int8x8_t { + simd_select(transmute::<_, int8x8_t>(a), b, c) } -/// Table look-up +/// Bitwise Select. #[inline] -#[cfg(target_arch = "arm")] -#[cfg(target_endian = "little")] -#[target_feature(enable = "neon,v7")] -#[cfg_attr(test, assert_instr(vtbl))] -pub unsafe fn vtbl2_u8(a: uint8x8x2_t, b: uint8x8_t) -> uint8x8_t { - transmute(vtbl2(transmute(a.0), transmute(a.1), transmute(b))) +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vbsl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(bsl))] +pub unsafe fn vbsl_s16(a: uint16x4_t, b: int16x4_t, c: int16x4_t) -> int16x4_t { + simd_select(transmute::<_, int16x4_t>(a), b, c) } -/// Table look-up +/// Bitwise Select. #[inline] -#[cfg(target_arch = "arm")] -#[cfg(target_endian = "little")] -#[target_feature(enable = "neon,v7")] -#[cfg_attr(test, assert_instr(vtbl))] -pub unsafe fn vtbl2_p8(a: poly8x8x2_t, b: uint8x8_t) -> poly8x8_t { - transmute(vtbl2(transmute(a.0), transmute(a.1), transmute(b))) +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vbsl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(bsl))] +pub unsafe fn vbsl_s32(a: uint32x2_t, b: int32x2_t, c: int32x2_t) -> int32x2_t { + simd_select(transmute::<_, int32x2_t>(a), b, c) } -/// Table look-up +/// Bitwise Select. #[inline] -#[cfg(target_arch = "arm")] -#[cfg(target_endian = "little")] -#[target_feature(enable = "neon,v7")] -#[cfg_attr(test, assert_instr(vtbl))] -pub unsafe fn vtbl3_s8(a: int8x8x3_t, b: int8x8_t) -> int8x8_t { - vtbl3(a.0, a.1, a.2, b) +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vbsl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(bsl))] +pub unsafe fn vbsl_s64(a: uint64x1_t, b: int64x1_t, c: int64x1_t) -> int64x1_t { + simd_select(transmute::<_, int64x1_t>(a), b, c) } -/// Table look-up +/// Bitwise Select. #[inline] -#[cfg(target_arch = "arm")] -#[cfg(target_endian = "little")] -#[target_feature(enable = "neon,v7")] -#[cfg_attr(test, assert_instr(vtbl))] -pub unsafe fn vtbl3_u8(a: uint8x8x3_t, b: uint8x8_t) -> uint8x8_t { - transmute(vtbl3( - transmute(a.0), - transmute(a.1), - transmute(a.2), - transmute(b), - )) +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vbsl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(bsl))] +pub unsafe fn vbsl_u8(a: uint8x8_t, b: uint8x8_t, c: uint8x8_t) -> uint8x8_t { + simd_select(transmute::<_, int8x8_t>(a), b, c) } -/// Table look-up +/// Bitwise Select. #[inline] -#[cfg(target_arch = "arm")] -#[cfg(target_endian = "little")] -#[target_feature(enable = "neon,v7")] -#[cfg_attr(test, assert_instr(vtbl))] -pub unsafe fn vtbl3_p8(a: poly8x8x3_t, b: uint8x8_t) -> poly8x8_t { - transmute(vtbl3( - transmute(a.0), - transmute(a.1), - transmute(a.2), - transmute(b), - )) +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vbsl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(bsl))] +pub unsafe fn vbsl_u16(a: uint16x4_t, b: uint16x4_t, c: uint16x4_t) -> uint16x4_t { + simd_select(transmute::<_, int16x4_t>(a), b, c) } -/// Table look-up +/// Bitwise Select. #[inline] -#[cfg(target_arch = "arm")] -#[cfg(target_endian = "little")] -#[target_feature(enable = "neon,v7")] -#[cfg_attr(test, assert_instr(vtbl))] -pub unsafe fn vtbl4_s8(a: int8x8x4_t, b: int8x8_t) -> int8x8_t { - vtbl4(a.0, a.1, a.2, a.3, b) +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vbsl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(bsl))] +pub unsafe fn vbsl_u32(a: uint32x2_t, b: uint32x2_t, c: uint32x2_t) -> uint32x2_t { + simd_select(transmute::<_, int32x2_t>(a), b, c) } -/// Table look-up +/// Bitwise Select. #[inline] -#[cfg(target_arch = "arm")] -#[cfg(target_endian = "little")] -#[target_feature(enable = "neon,v7")] -#[cfg_attr(test, assert_instr(vtbl))] -pub unsafe fn vtbl4_u8(a: uint8x8x4_t, b: uint8x8_t) -> uint8x8_t { - transmute(vtbl4( - transmute(a.0), - transmute(a.1), - transmute(a.2), - transmute(a.3), - transmute(b), - )) +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vbsl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(bsl))] +pub unsafe fn vbsl_u64(a: uint64x1_t, b: uint64x1_t, c: uint64x1_t) -> uint64x1_t { + simd_select(transmute::<_, int64x1_t>(a), b, c) } -/// Table look-up +/// Bitwise Select. #[inline] -#[cfg(target_arch = "arm")] -#[cfg(target_endian = "little")] -#[target_feature(enable = "neon,v7")] -#[cfg_attr(test, assert_instr(vtbl))] -pub unsafe fn vtbl4_p8(a: poly8x8x4_t, b: uint8x8_t) -> poly8x8_t { - transmute(vtbl4( - transmute(a.0), - transmute(a.1), - transmute(a.2), - transmute(a.3), - transmute(b), - )) -} - -/// Extended table look-up -#[inline] -#[cfg(target_arch = "arm")] -#[cfg(target_endian = "little")] -#[target_feature(enable = "neon,v7")] -#[cfg_attr(test, assert_instr(vtbx))] -pub unsafe fn vtbx1_s8(a: int8x8_t, b: int8x8_t, c: int8x8_t) -> int8x8_t { - vtbx1(a, b, c) -} - -/// Extended table look-up -#[inline] -#[cfg(target_arch = "arm")] -#[cfg(target_endian = "little")] -#[target_feature(enable = "neon,v7")] -#[cfg_attr(test, assert_instr(vtbx))] -pub unsafe fn vtbx1_u8(a: uint8x8_t, b: uint8x8_t, c: uint8x8_t) -> uint8x8_t { - transmute(vtbx1(transmute(a), transmute(b), transmute(c))) -} - -/// Extended table look-up -#[inline] -#[cfg(target_arch = "arm")] -#[cfg(target_endian = "little")] -#[target_feature(enable = "neon,v7")] -#[cfg_attr(test, assert_instr(vtbx))] -pub unsafe fn vtbx1_p8(a: poly8x8_t, b: poly8x8_t, c: uint8x8_t) -> poly8x8_t { - transmute(vtbx1(transmute(a), transmute(b), transmute(c))) -} - -/// Extended table look-up -#[inline] -#[cfg(target_arch = "arm")] -#[cfg(target_endian = "little")] -#[target_feature(enable = "neon,v7")] -#[cfg_attr(test, assert_instr(vtbx))] -pub unsafe fn vtbx2_s8(a: int8x8_t, b: int8x8x2_t, c: int8x8_t) -> int8x8_t { - vtbx2(a, b.0, b.1, c) -} - -/// Extended table look-up -#[inline] -#[cfg(target_arch = "arm")] -#[cfg(target_endian = "little")] -#[target_feature(enable = "neon,v7")] -#[cfg_attr(test, assert_instr(vtbx))] -pub unsafe fn vtbx2_u8(a: uint8x8_t, b: uint8x8x2_t, c: uint8x8_t) -> uint8x8_t { - transmute(vtbx2( - transmute(a), - transmute(b.0), - transmute(b.1), - transmute(c), - )) -} - -/// Extended table look-up -#[inline] -#[cfg(target_arch = "arm")] -#[cfg(target_endian = "little")] -#[target_feature(enable = "neon,v7")] -#[cfg_attr(test, assert_instr(vtbx))] -pub unsafe fn vtbx2_p8(a: poly8x8_t, b: poly8x8x2_t, c: uint8x8_t) -> poly8x8_t { - transmute(vtbx2( - transmute(a), - transmute(b.0), - transmute(b.1), - transmute(c), - )) -} - -/// Extended table look-up -#[inline] -#[cfg(target_arch = "arm")] -#[cfg(target_endian = "little")] -#[target_feature(enable = "neon,v7")] -#[cfg_attr(test, assert_instr(vtbx))] -pub unsafe fn vtbx3_s8(a: int8x8_t, b: int8x8x3_t, c: int8x8_t) -> int8x8_t { - vtbx3(a, b.0, b.1, b.2, c) -} - -/// Extended table look-up -#[inline] -#[cfg(target_arch = "arm")] -#[cfg(target_endian = "little")] -#[target_feature(enable = "neon,v7")] -#[cfg_attr(test, assert_instr(vtbx))] -pub unsafe fn vtbx3_u8(a: uint8x8_t, b: uint8x8x3_t, c: uint8x8_t) -> uint8x8_t { - transmute(vtbx3( - transmute(a), - transmute(b.0), - transmute(b.1), - transmute(b.2), - transmute(c), - )) -} - -/// Extended table look-up -#[inline] -#[cfg(target_arch = "arm")] -#[cfg(target_endian = "little")] -#[target_feature(enable = "neon,v7")] -#[cfg_attr(test, assert_instr(vtbx))] -pub unsafe fn vtbx3_p8(a: poly8x8_t, b: poly8x8x3_t, c: uint8x8_t) -> poly8x8_t { - transmute(vtbx3( - transmute(a), - transmute(b.0), - transmute(b.1), - transmute(b.2), - transmute(c), - )) -} - -/// Extended table look-up -#[inline] -#[cfg(target_arch = "arm")] -#[cfg(target_endian = "little")] -#[target_feature(enable = "neon,v7")] -#[cfg_attr(test, assert_instr(vtbx))] -pub unsafe fn vtbx4_s8(a: int8x8_t, b: int8x8x4_t, c: int8x8_t) -> int8x8_t { - vtbx4(a, b.0, b.1, b.2, b.3, c) -} - -/// Extended table look-up -#[inline] -#[cfg(target_arch = "arm")] -#[cfg(target_endian = "little")] -#[target_feature(enable = "neon,v7")] -#[cfg_attr(test, assert_instr(vtbx))] -pub unsafe fn vtbx4_u8(a: uint8x8_t, b: uint8x8x4_t, c: uint8x8_t) -> uint8x8_t { - transmute(vtbx4( - transmute(a), - transmute(b.0), - transmute(b.1), - transmute(b.2), - transmute(b.3), - transmute(c), - )) -} - -/// Extended table look-up -#[inline] -#[cfg(target_arch = "arm")] -#[cfg(target_endian = "little")] -#[target_feature(enable = "neon,v7")] -#[cfg_attr(test, assert_instr(vtbx))] -pub unsafe fn vtbx4_p8(a: poly8x8_t, b: poly8x8x4_t, c: uint8x8_t) -> poly8x8_t { - transmute(vtbx4( - transmute(a), - transmute(b.0), - transmute(b.1), - transmute(b.2), - transmute(b.3), - transmute(c), - )) +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vbsl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(bsl))] +pub unsafe fn vbsl_f32(a: uint32x2_t, b: float32x2_t, c: float32x2_t) -> float32x2_t { + simd_select(transmute::<_, int32x2_t>(a), b, c) } -/// Move vector element to general-purpose register +/// Bitwise Select. #[inline] #[target_feature(enable = "neon")] #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[rustc_args_required_const(1)] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmov.32", imm5 = 1))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mov, imm5 = 1))] -// Based on the discussion in https://github.com/rust-lang/stdarch/pull/792 -// `mov` seems to be an acceptable intrinsic to compile to -// #[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(vmov, imm5 = 1))] -pub unsafe fn vgetq_lane_u64(v: uint64x2_t, imm5: i32) -> u64 { - assert!(imm5 >= 0 && imm5 <= 1); - simd_extract(v, imm5 as u32) +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vbsl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(bsl))] +pub unsafe fn vbsl_p8(a: uint8x8_t, b: poly8x8_t, c: poly8x8_t) -> poly8x8_t { + simd_select(transmute::<_, int8x8_t>(a), b, c) } -/// Move vector element to general-purpose register +/// Bitwise Select. #[inline] #[target_feature(enable = "neon")] #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[rustc_args_required_const(1)] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmov.32", imm5 = 0))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fmov, imm5 = 0))] -// FIXME: no 32bit this seems to be turned into two vmov.32 instructions -// validate correctness -pub unsafe fn vget_lane_u64(v: uint64x1_t, imm5: i32) -> u64 { - assert!(imm5 == 0); - simd_extract(v, 0) +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vbsl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(bsl))] +pub unsafe fn vbsl_p16(a: uint16x4_t, b: poly16x4_t, c: poly16x4_t) -> poly16x4_t { + simd_select(transmute::<_, int16x4_t>(a), b, c) } -/// Move vector element to general-purpose register +/// Bitwise Select. (128-bit) #[inline] #[target_feature(enable = "neon")] #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[rustc_args_required_const(1)] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmov.u16", imm5 = 2))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(umov, imm5 = 2))] -pub unsafe fn vgetq_lane_u16(v: uint16x8_t, imm5: i32) -> u16 { - assert!(imm5 >= 0 && imm5 <= 7); - simd_extract(v, imm5 as u32) +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vbsl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(bsl))] +pub unsafe fn vbslq_s8(a: uint8x16_t, b: int8x16_t, c: int8x16_t) -> int8x16_t { + simd_select(transmute::<_, int8x16_t>(a), b, c) } -/// Move vector element to general-purpose register +/// Bitwise Select. (128-bit) #[inline] #[target_feature(enable = "neon")] #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[rustc_args_required_const(1)] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmov.32", imm5 = 2))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mov, imm5 = 2))] -pub unsafe fn vgetq_lane_u32(v: uint32x4_t, imm5: i32) -> u32 { - assert!(imm5 >= 0 && imm5 <= 3); - simd_extract(v, imm5 as u32) +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vbsl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(bsl))] +pub unsafe fn vbslq_s16(a: uint16x8_t, b: int16x8_t, c: int16x8_t) -> int16x8_t { + simd_select(transmute::<_, int16x8_t>(a), b, c) } -/// Move vector element to general-purpose register +/// Bitwise Select. (128-bit) #[inline] #[target_feature(enable = "neon")] #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[rustc_args_required_const(1)] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmov.32", imm5 = 2))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mov, imm5 = 2))] -pub unsafe fn vgetq_lane_s32(v: int32x4_t, imm5: i32) -> i32 { - assert!(imm5 >= 0 && imm5 <= 3); - simd_extract(v, imm5 as u32) +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vbsl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(bsl))] +pub unsafe fn vbslq_s32(a: uint32x4_t, b: int32x4_t, c: int32x4_t) -> int32x4_t { + simd_select(transmute::<_, int32x4_t>(a), b, c) } -/// Move vector element to general-purpose register +/// Bitwise Select. (128-bit) #[inline] #[target_feature(enable = "neon")] #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[rustc_args_required_const(1)] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmov.u8", imm5 = 2))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(umov, imm5 = 2))] -pub unsafe fn vget_lane_u8(v: uint8x8_t, imm5: i32) -> u8 { - assert!(imm5 >= 0 && imm5 <= 7); - simd_extract(v, imm5 as u32) +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vbsl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(bsl))] +pub unsafe fn vbslq_s64(a: uint64x2_t, b: int64x2_t, c: int64x2_t) -> int64x2_t { + simd_select(transmute::<_, int64x2_t>(a), b, c) } -/// Duplicate vector element to vector or scalar +/// Bitwise Select. (128-bit) #[inline] #[target_feature(enable = "neon")] #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.8"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup))] -pub unsafe fn vdupq_n_s8(value: i8) -> int8x16_t { - int8x16_t( - value, value, value, value, value, value, value, value, value, value, value, value, value, - value, value, value, - ) +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vbsl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(bsl))] +pub unsafe fn vbslq_u8(a: uint8x16_t, b: uint8x16_t, c: uint8x16_t) -> uint8x16_t { + simd_select(transmute::<_, int8x16_t>(a), b, c) } -/// Duplicate vector element to vector or scalar +/// Bitwise Select. (128-bit) #[inline] #[target_feature(enable = "neon")] #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.8"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup))] -pub unsafe fn vdupq_n_u8(value: u8) -> uint8x16_t { - uint8x16_t( - value, value, value, value, value, value, value, value, value, value, value, value, value, - value, value, value, - ) +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vbsl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(bsl))] +pub unsafe fn vbslq_u16(a: uint16x8_t, b: uint16x8_t, c: uint16x8_t) -> uint16x8_t { + simd_select(transmute::<_, int16x8_t>(a), b, c) } -/// Duplicate vector element to vector or scalar +/// Bitwise Select. (128-bit) #[inline] #[target_feature(enable = "neon")] #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.8"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup))] -pub unsafe fn vmovq_n_u8(value: u8) -> uint8x16_t { - vdupq_n_u8(value) +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vbsl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(bsl))] +pub unsafe fn vbslq_u32(a: uint32x4_t, b: uint32x4_t, c: uint32x4_t) -> uint32x4_t { + simd_select(transmute::<_, int32x4_t>(a), b, c) } -/// Vector reinterpret cast operation +/// Bitwise Select. (128-bit) #[inline] #[target_feature(enable = "neon")] #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(test, assert_instr(nop))] -pub unsafe fn vreinterpret_u64_u32(a: uint32x2_t) -> uint64x1_t { - transmute(a) +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vbsl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(bsl))] +pub unsafe fn vbslq_u64(a: uint64x2_t, b: uint64x2_t, c: uint64x2_t) -> uint64x2_t { + simd_select(transmute::<_, int64x2_t>(a), b, c) } -/// Vector reinterpret cast operation +/// Bitwise Select. (128-bit) #[inline] #[target_feature(enable = "neon")] #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(test, assert_instr(nop))] -pub unsafe fn vreinterpretq_s8_u8(a: uint8x16_t) -> int8x16_t { - transmute(a) +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vbsl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(bsl))] +pub unsafe fn vbslq_p8(a: uint8x16_t, b: poly8x16_t, c: poly8x16_t) -> poly8x16_t { + simd_select(transmute::<_, int8x16_t>(a), b, c) } -/// Vector reinterpret cast operation +/// Bitwise Select. (128-bit) #[inline] #[target_feature(enable = "neon")] #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(test, assert_instr(nop))] -pub unsafe fn vreinterpretq_u16_u8(a: uint8x16_t) -> uint16x8_t { - transmute(a) +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vbsl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(bsl))] +pub unsafe fn vbslq_p16(a: uint16x8_t, b: poly16x8_t, c: poly16x8_t) -> poly16x8_t { + simd_select(transmute::<_, int16x8_t>(a), b, c) } -/// Vector reinterpret cast operation +/// Bitwise Select. (128-bit) #[inline] #[target_feature(enable = "neon")] #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(test, assert_instr(nop))] -pub unsafe fn vreinterpretq_u32_u8(a: uint8x16_t) -> uint32x4_t { - transmute(a) +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vbsl))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(bsl))] +pub unsafe fn vbslq_f32(a: uint32x4_t, b: float32x4_t, c: float32x4_t) -> float32x4_t { + simd_select(transmute::<_, int32x4_t>(a), b, c) } -/// Vector reinterpret cast operation +/// Vector bitwise inclusive OR NOT #[inline] #[target_feature(enable = "neon")] #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(test, assert_instr(nop))] -pub unsafe fn vreinterpretq_u64_u8(a: uint8x16_t) -> uint64x2_t { - transmute(a) +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vorn))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(orn))] +pub unsafe fn vorn_s8(a: int8x8_t, b: int8x8_t) -> int8x8_t { + let c = int8x8_t(-1, -1, -1, -1, -1, -1, -1, -1); + simd_or(simd_xor(b, c), a) } -/// Vector reinterpret cast operation +/// Vector bitwise inclusive OR NOT #[inline] #[target_feature(enable = "neon")] #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(test, assert_instr(nop))] -pub unsafe fn vreinterpretq_u8_s8(a: int8x16_t) -> uint8x16_t { - transmute(a) +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vorn))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(orn))] +pub unsafe fn vornq_s8(a: int8x16_t, b: int8x16_t) -> int8x16_t { + let c = int8x16_t( + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + ); + simd_or(simd_xor(b, c), a) } -/// Unsigned shift right +/// Vector bitwise inclusive OR NOT #[inline] #[target_feature(enable = "neon")] #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vshr.u8", imm3 = 1))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr("ushr", imm3 = 1))] -#[rustc_args_required_const(1)] -pub unsafe fn vshrq_n_u8(a: uint8x16_t, imm3: i32) -> uint8x16_t { - if imm3 < 0 || imm3 > 7 { - unreachable_unchecked(); - } else { - uint8x16_t( - a.0 >> imm3, - a.1 >> imm3, - a.2 >> imm3, - a.3 >> imm3, - a.4 >> imm3, - a.5 >> imm3, - a.6 >> imm3, - a.7 >> imm3, - a.8 >> imm3, - a.9 >> imm3, - a.10 >> imm3, - a.11 >> imm3, - a.12 >> imm3, - a.13 >> imm3, - a.14 >> imm3, - a.15 >> imm3, - ) - } +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vorn))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(orn))] +pub unsafe fn vorn_s16(a: int16x4_t, b: int16x4_t) -> int16x4_t { + let c = int16x4_t(-1, -1, -1, -1); + simd_or(simd_xor(b, c), a) } -/// Shift right +/// Vector bitwise inclusive OR NOT #[inline] #[target_feature(enable = "neon")] #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vshl.s8", imm3 = 1))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(shl, imm3 = 1))] -#[rustc_args_required_const(1)] -pub unsafe fn vshlq_n_u8(a: uint8x16_t, imm3: i32) -> uint8x16_t { - if imm3 < 0 || imm3 > 7 { - unreachable_unchecked(); - } else { - uint8x16_t( - a.0 << imm3, - a.1 << imm3, - a.2 << imm3, - a.3 << imm3, - a.4 << imm3, - a.5 << imm3, - a.6 << imm3, - a.7 << imm3, - a.8 << imm3, - a.9 << imm3, - a.10 << imm3, - a.11 << imm3, - a.12 << imm3, - a.13 << imm3, - a.14 << imm3, - a.15 << imm3, - ) - } +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vorn))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(orn))] +pub unsafe fn vornq_s16(a: int16x8_t, b: int16x8_t) -> int16x8_t { + let c = int16x8_t(-1, -1, -1, -1, -1, -1, -1, -1); + simd_or(simd_xor(b, c), a) } -/// Extract vector from pair of vectors +/// Vector bitwise inclusive OR NOT #[inline] #[target_feature(enable = "neon")] #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vext.8", n = 3))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ext, n = 3))] -#[rustc_args_required_const(2)] -pub unsafe fn vextq_s8(a: int8x16_t, b: int8x16_t, n: i32) -> int8x16_t { - if n < 0 || n > 15 { - unreachable_unchecked(); - }; - match n & 0b1111 { - 0 => simd_shuffle16(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), - 1 => simd_shuffle16( - a, - b, - [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16], - ), - 2 => simd_shuffle16( - a, - b, - [2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17], - ), - 3 => simd_shuffle16( - a, - b, - [3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18], - ), - 4 => simd_shuffle16( - a, - b, - [4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19], - ), - 5 => simd_shuffle16( - a, - b, - [5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20], - ), - 6 => simd_shuffle16( - a, - b, - [6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21], - ), - 7 => simd_shuffle16( - a, - b, - [7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22], - ), - 8 => simd_shuffle16( - a, - b, - [8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23], - ), - 9 => simd_shuffle16( - a, - b, - [ - 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, - ], - ), - 10 => simd_shuffle16( - a, - b, - [ - 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, - ], - ), - 11 => simd_shuffle16( - a, - b, - [ - 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, - ], - ), - 12 => simd_shuffle16( - a, - b, - [ - 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, - ], - ), - 13 => simd_shuffle16( - a, - b, - [ - 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, - ], - ), - 14 => simd_shuffle16( - a, - b, - [ - 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, - ], - ), - 15 => simd_shuffle16( - a, - b, - [ - 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, - ], - ), - _ => unreachable_unchecked(), - } +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vorn))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(orn))] +pub unsafe fn vorn_s32(a: int32x2_t, b: int32x2_t) -> int32x2_t { + let c = int32x2_t(-1, -1); + simd_or(simd_xor(b, c), a) } -/// Extract vector from pair of vectors +/// Vector bitwise inclusive OR NOT #[inline] #[target_feature(enable = "neon")] #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vext.8", n = 3))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ext, n = 3))] -#[rustc_args_required_const(2)] -pub unsafe fn vextq_u8(a: uint8x16_t, b: uint8x16_t, n: i32) -> uint8x16_t { - if n < 0 || n > 15 { - unreachable_unchecked(); - }; - match n & 0b1111 { - 0 => simd_shuffle16(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), - 1 => simd_shuffle16( - a, - b, - [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16], - ), - 2 => simd_shuffle16( - a, - b, - [2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17], - ), - 3 => simd_shuffle16( - a, - b, - [3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18], - ), - 4 => simd_shuffle16( - a, - b, - [4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19], - ), - 5 => simd_shuffle16( - a, - b, - [5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20], - ), - 6 => simd_shuffle16( - a, - b, - [6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21], - ), - 7 => simd_shuffle16( - a, - b, - [7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22], - ), - 8 => simd_shuffle16( - a, - b, - [8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23], - ), - 9 => simd_shuffle16( - a, - b, - [ - 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, - ], - ), - 10 => simd_shuffle16( - a, - b, - [ - 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, - ], - ), - 11 => simd_shuffle16( - a, - b, - [ - 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, - ], - ), - 12 => simd_shuffle16( - a, - b, - [ - 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, - ], - ), - 13 => simd_shuffle16( - a, - b, - [ - 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, - ], - ), - 14 => simd_shuffle16( - a, - b, - [ - 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, - ], - ), - 15 => simd_shuffle16( - a, - b, - [ - 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, - ], - ), - _ => unreachable_unchecked(), - } -} - -// These float-to-int implementations have undefined behaviour when `a` overflows -// the destination type. Clang has the same problem: https://llvm.org/PR47510 - -/// Floating-point Convert to Signed fixed-point, rounding toward Zero (vector) -#[inline] -#[cfg(target_arch = "arm")] -#[target_feature(enable = "neon")] -#[target_feature(enable = "v7")] -#[cfg_attr(test, assert_instr("vcvt.s32.f32"))] -pub unsafe fn vcvtq_s32_f32(a: float32x4_t) -> int32x4_t { - transmute(simd_cast::<_, i32x4>(transmute::<_, f32x4>(a))) -} - -/// Floating-point Convert to Unsigned fixed-point, rounding toward Zero (vector) -#[inline] -#[cfg(target_arch = "arm")] -#[target_feature(enable = "neon")] -#[target_feature(enable = "v7")] -#[cfg_attr(test, assert_instr("vcvt.u32.f32"))] -pub unsafe fn vcvtq_u32_f32(a: float32x4_t) -> uint32x4_t { - transmute(simd_cast::<_, u32x4>(transmute::<_, f32x4>(a))) +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vorn))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(orn))] +pub unsafe fn vornq_s32(a: int32x4_t, b: int32x4_t) -> int32x4_t { + let c = int32x4_t(-1, -1, -1, -1); + simd_or(simd_xor(b, c), a) } -/// Population count per byte. +/// Vector bitwise inclusive OR NOT #[inline] #[target_feature(enable = "neon")] #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vcnt))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cnt))] -pub unsafe fn vcnt_s8(a: int8x8_t) -> int8x8_t { - vcnt_s8_(a) +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vorn))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(orn))] +pub unsafe fn vorn_s64(a: int64x1_t, b: int64x1_t) -> int64x1_t { + let c = int64x1_t(-1); + simd_or(simd_xor(b, c), a) } -/// Population count per byte. + +/// Vector bitwise inclusive OR NOT #[inline] #[target_feature(enable = "neon")] #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vcnt))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cnt))] -pub unsafe fn vcntq_s8(a: int8x16_t) -> int8x16_t { - vcntq_s8_(a) +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vorn))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(orn))] +pub unsafe fn vornq_s64(a: int64x2_t, b: int64x2_t) -> int64x2_t { + let c = int64x2_t(-1, -1); + simd_or(simd_xor(b, c), a) } -/// Population count per byte. + +/// Vector bitwise inclusive OR NOT #[inline] #[target_feature(enable = "neon")] #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vcnt))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cnt))] -pub unsafe fn vcnt_u8(a: uint8x8_t) -> uint8x8_t { - transmute(vcnt_s8_(transmute(a))) +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vorn))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(orn))] +pub unsafe fn vorn_u8(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t { + let c = int8x8_t(-1, -1, -1, -1, -1, -1, -1, -1); + simd_or(simd_xor(b, transmute(c)), a) } -/// Population count per byte. + +/// Vector bitwise inclusive OR NOT #[inline] #[target_feature(enable = "neon")] #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vcnt))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cnt))] -pub unsafe fn vcntq_u8(a: uint8x16_t) -> uint8x16_t { - transmute(vcntq_s8_(transmute(a))) +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vorn))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(orn))] +pub unsafe fn vornq_u8(a: uint8x16_t, b: uint8x16_t) -> uint8x16_t { + let c = int8x16_t( + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + ); + simd_or(simd_xor(b, transmute(c)), a) } -/// Population count per byte. + +/// Vector bitwise inclusive OR NOT #[inline] #[target_feature(enable = "neon")] #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vcnt))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cnt))] -pub unsafe fn vcnt_p8(a: poly8x8_t) -> poly8x8_t { - transmute(vcnt_s8_(transmute(a))) +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vorn))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(orn))] +pub unsafe fn vorn_u16(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t { + let c = int16x4_t(-1, -1, -1, -1); + simd_or(simd_xor(b, transmute(c)), a) } -/// Population count per byte. + +/// Vector bitwise inclusive OR NOT #[inline] #[target_feature(enable = "neon")] #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vcnt))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cnt))] -pub unsafe fn vcntq_p8(a: poly8x16_t) -> poly8x16_t { - transmute(vcntq_s8_(transmute(a))) -} - -/// Shift Left and Insert (immediate) -#[inline] -#[cfg(target_arch = "arm")] -#[target_feature(enable = "neon,v7")] -#[cfg_attr(test, assert_instr("vsli.8", n = 1))] -#[rustc_args_required_const(2)] -pub unsafe fn vsli_n_s8(a: int8x8_t, b: int8x8_t, n: i32) -> int8x8_t { - assert!(0 <= n && n <= 7, "must have 0 ≤ n ≤ 7, but n = {}", n); - let n = n as i8; - vshiftins_v8i8(a, b, int8x8_t(n, n, n, n, n, n, n, n)) -} -/// Shift Left and Insert (immediate) -#[inline] -#[cfg(target_arch = "arm")] -#[target_feature(enable = "neon,v7")] -#[cfg_attr(test, assert_instr("vsli.8", n = 1))] -#[rustc_args_required_const(2)] -pub unsafe fn vsliq_n_s8(a: int8x16_t, b: int8x16_t, n: i32) -> int8x16_t { - assert!(0 <= n && n <= 7, "must have 0 ≤ n ≤ 7, but n = {}", n); - let n = n as i8; - vshiftins_v16i8( - a, - b, - int8x16_t(n, n, n, n, n, n, n, n, n, n, n, n, n, n, n, n), - ) -} -/// Shift Left and Insert (immediate) -#[inline] -#[cfg(target_arch = "arm")] -#[target_feature(enable = "neon,v7")] -#[cfg_attr(test, assert_instr("vsli.16", n = 1))] -#[rustc_args_required_const(2)] -pub unsafe fn vsli_n_s16(a: int16x4_t, b: int16x4_t, n: i32) -> int16x4_t { - assert!(0 <= n && n <= 15, "must have 0 ≤ n ≤ 15, but n = {}", n); - let n = n as i16; - vshiftins_v4i16(a, b, int16x4_t(n, n, n, n)) -} -/// Shift Left and Insert (immediate) -#[inline] -#[cfg(target_arch = "arm")] -#[target_feature(enable = "neon,v7")] -#[cfg_attr(test, assert_instr("vsli.16", n = 1))] -#[rustc_args_required_const(2)] -pub unsafe fn vsliq_n_s16(a: int16x8_t, b: int16x8_t, n: i32) -> int16x8_t { - assert!(0 <= n && n <= 15, "must have 0 ≤ n ≤ 15, but n = {}", n); - let n = n as i16; - vshiftins_v8i16(a, b, int16x8_t(n, n, n, n, n, n, n, n)) -} -/// Shift Left and Insert (immediate) -#[inline] -#[cfg(target_arch = "arm")] -#[target_feature(enable = "neon,v7")] -#[cfg_attr(test, assert_instr("vsli.32", n = 1))] -#[rustc_args_required_const(2)] -pub unsafe fn vsli_n_s32(a: int32x2_t, b: int32x2_t, n: i32) -> int32x2_t { - assert!(0 <= n && n <= 31, "must have 0 ≤ n ≤ 31, but n = {}", n); - vshiftins_v2i32(a, b, int32x2_t(n, n)) -} -/// Shift Left and Insert (immediate) -#[inline] -#[cfg(target_arch = "arm")] -#[target_feature(enable = "neon,v7")] -#[cfg_attr(test, assert_instr("vsli.32", n = 1))] -#[rustc_args_required_const(2)] -pub unsafe fn vsliq_n_s32(a: int32x4_t, b: int32x4_t, n: i32) -> int32x4_t { - assert!(0 <= n && n <= 31, "must have 0 ≤ n ≤ 31, but n = {}", n); - vshiftins_v4i32(a, b, int32x4_t(n, n, n, n)) -} -/// Shift Left and Insert (immediate) -#[inline] -#[cfg(target_arch = "arm")] -#[target_feature(enable = "neon,v7")] -#[cfg_attr(test, assert_instr("vsli.64", n = 1))] -#[rustc_args_required_const(2)] -pub unsafe fn vsli_n_s64(a: int64x1_t, b: int64x1_t, n: i32) -> int64x1_t { - assert!(0 <= n && n <= 63, "must have 0 ≤ n ≤ 63, but n = {}", n); - vshiftins_v1i64(a, b, int64x1_t(n as i64)) -} -/// Shift Left and Insert (immediate) -#[inline] -#[cfg(target_arch = "arm")] -#[target_feature(enable = "neon,v7")] -#[cfg_attr(test, assert_instr("vsli.64", n = 1))] -#[rustc_args_required_const(2)] -pub unsafe fn vsliq_n_s64(a: int64x2_t, b: int64x2_t, n: i32) -> int64x2_t { - assert!(0 <= n && n <= 63, "must have 0 ≤ n ≤ 63, but n = {}", n); - vshiftins_v2i64(a, b, int64x2_t(n as i64, n as i64)) -} -/// Shift Left and Insert (immediate) -#[inline] -#[cfg(target_arch = "arm")] -#[target_feature(enable = "neon,v7")] -#[cfg_attr(test, assert_instr("vsli.8", n = 1))] -#[rustc_args_required_const(2)] -pub unsafe fn vsli_n_u8(a: uint8x8_t, b: uint8x8_t, n: i32) -> uint8x8_t { - assert!(0 <= n && n <= 7, "must have 0 ≤ n ≤ 7, but n = {}", n); - let n = n as i8; - transmute(vshiftins_v8i8( - transmute(a), - transmute(b), - int8x8_t(n, n, n, n, n, n, n, n), - )) -} -/// Shift Left and Insert (immediate) -#[inline] -#[cfg(target_arch = "arm")] -#[target_feature(enable = "neon,v7")] -#[cfg_attr(test, assert_instr("vsli.8", n = 1))] -#[rustc_args_required_const(2)] -pub unsafe fn vsliq_n_u8(a: uint8x16_t, b: uint8x16_t, n: i32) -> uint8x16_t { - assert!(0 <= n && n <= 7, "must have 0 ≤ n ≤ 7, but n = {}", n); - let n = n as i8; - transmute(vshiftins_v16i8( - transmute(a), - transmute(b), - int8x16_t(n, n, n, n, n, n, n, n, n, n, n, n, n, n, n, n), - )) -} -/// Shift Left and Insert (immediate) -#[inline] -#[cfg(target_arch = "arm")] -#[target_feature(enable = "neon,v7")] -#[cfg_attr(test, assert_instr("vsli.16", n = 1))] -#[rustc_args_required_const(2)] -pub unsafe fn vsli_n_u16(a: uint16x4_t, b: uint16x4_t, n: i32) -> uint16x4_t { - assert!(0 <= n && n <= 15, "must have 0 ≤ n ≤ 15, but n = {}", n); - let n = n as i16; - transmute(vshiftins_v4i16( - transmute(a), - transmute(b), - int16x4_t(n, n, n, n), - )) -} -/// Shift Left and Insert (immediate) -#[inline] -#[cfg(target_arch = "arm")] -#[target_feature(enable = "neon,v7")] -#[cfg_attr(test, assert_instr("vsli.16", n = 1))] -#[rustc_args_required_const(2)] -pub unsafe fn vsliq_n_u16(a: uint16x8_t, b: uint16x8_t, n: i32) -> uint16x8_t { - assert!(0 <= n && n <= 15, "must have 0 ≤ n ≤ 15, but n = {}", n); - let n = n as i16; - transmute(vshiftins_v8i16( - transmute(a), - transmute(b), - int16x8_t(n, n, n, n, n, n, n, n), - )) -} -/// Shift Left and Insert (immediate) -#[inline] -#[cfg(target_arch = "arm")] -#[target_feature(enable = "neon,v7")] -#[cfg_attr(test, assert_instr("vsli.32", n = 1))] -#[rustc_args_required_const(2)] -pub unsafe fn vsli_n_u32(a: uint32x2_t, b: uint32x2_t, n: i32) -> uint32x2_t { - assert!(0 <= n && n <= 31, "must have 0 ≤ n ≤ 31, but n = {}", n); - transmute(vshiftins_v2i32(transmute(a), transmute(b), int32x2_t(n, n))) -} -/// Shift Left and Insert (immediate) -#[inline] -#[cfg(target_arch = "arm")] -#[target_feature(enable = "neon,v7")] -#[cfg_attr(test, assert_instr("vsli.32", n = 1))] -#[rustc_args_required_const(2)] -pub unsafe fn vsliq_n_u32(a: uint32x4_t, b: uint32x4_t, n: i32) -> uint32x4_t { - assert!(0 <= n && n <= 31, "must have 0 ≤ n ≤ 31, but n = {}", n); - transmute(vshiftins_v4i32( - transmute(a), - transmute(b), - int32x4_t(n, n, n, n), - )) -} -/// Shift Left and Insert (immediate) -#[inline] -#[cfg(target_arch = "arm")] -#[target_feature(enable = "neon,v7")] -#[cfg_attr(test, assert_instr("vsli.64", n = 1))] -#[rustc_args_required_const(2)] -pub unsafe fn vsli_n_u64(a: uint64x1_t, b: uint64x1_t, n: i32) -> uint64x1_t { - assert!(0 <= n && n <= 63, "must have 0 ≤ n ≤ 63, but n = {}", n); - transmute(vshiftins_v1i64( - transmute(a), - transmute(b), - int64x1_t(n as i64), - )) -} -/// Shift Left and Insert (immediate) -#[inline] -#[cfg(target_arch = "arm")] -#[target_feature(enable = "neon,v7")] -#[cfg_attr(test, assert_instr("vsli.64", n = 1))] -#[rustc_args_required_const(2)] -pub unsafe fn vsliq_n_u64(a: uint64x2_t, b: uint64x2_t, n: i32) -> uint64x2_t { - assert!(0 <= n && n <= 63, "must have 0 ≤ n ≤ 63, but n = {}", n); - transmute(vshiftins_v2i64( - transmute(a), - transmute(b), - int64x2_t(n as i64, n as i64), - )) -} -/// Shift Left and Insert (immediate) -#[inline] -#[cfg(target_arch = "arm")] -#[target_feature(enable = "neon,v7")] -#[cfg_attr(test, assert_instr("vsli.8", n = 1))] -#[rustc_args_required_const(2)] -pub unsafe fn vsli_n_p8(a: poly8x8_t, b: poly8x8_t, n: i32) -> poly8x8_t { - assert!(0 <= n && n <= 7, "must have 0 ≤ n ≤ 7, but n = {}", n); - let n = n as i8; - transmute(vshiftins_v8i8( - transmute(a), - transmute(b), - int8x8_t(n, n, n, n, n, n, n, n), - )) -} -/// Shift Left and Insert (immediate) -#[inline] -#[cfg(target_arch = "arm")] -#[target_feature(enable = "neon,v7")] -#[cfg_attr(test, assert_instr("vsli.8", n = 1))] -#[rustc_args_required_const(2)] -pub unsafe fn vsliq_n_p8(a: poly8x16_t, b: poly8x16_t, n: i32) -> poly8x16_t { - assert!(0 <= n && n <= 7, "must have 0 ≤ n ≤ 7, but n = {}", n); - let n = n as i8; - transmute(vshiftins_v16i8( - transmute(a), - transmute(b), - int8x16_t(n, n, n, n, n, n, n, n, n, n, n, n, n, n, n, n), - )) -} -/// Shift Left and Insert (immediate) -#[inline] -#[cfg(target_arch = "arm")] -#[target_feature(enable = "neon,v7")] -#[cfg_attr(test, assert_instr("vsli.16", n = 1))] -#[rustc_args_required_const(2)] -pub unsafe fn vsli_n_p16(a: poly16x4_t, b: poly16x4_t, n: i32) -> poly16x4_t { - assert!(0 <= n && n <= 15, "must have 0 ≤ n ≤ 15, but n = {}", n); - let n = n as i16; - transmute(vshiftins_v4i16( - transmute(a), - transmute(b), - int16x4_t(n, n, n, n), - )) -} -/// Shift Left and Insert (immediate) -#[inline] -#[cfg(target_arch = "arm")] -#[target_feature(enable = "neon,v7")] -#[cfg_attr(test, assert_instr("vsli.16", n = 1))] -#[rustc_args_required_const(2)] -pub unsafe fn vsliq_n_p16(a: poly16x8_t, b: poly16x8_t, n: i32) -> poly16x8_t { - assert!(0 <= n && n <= 15, "must have 0 ≤ n ≤ 15, but n = {}", n); - let n = n as i16; - transmute(vshiftins_v8i16( - transmute(a), - transmute(b), - int16x8_t(n, n, n, n, n, n, n, n), - )) -} - -/// Shift Right and Insert (immediate) -#[inline] -#[cfg(target_arch = "arm")] -#[target_feature(enable = "neon,v7")] -#[cfg_attr(test, assert_instr("vsri.8", n = 1))] -#[rustc_args_required_const(2)] -pub unsafe fn vsri_n_s8(a: int8x8_t, b: int8x8_t, n: i32) -> int8x8_t { - assert!(1 <= n && n <= 8, "must have 1 ≤ n ≤ 8, but n = {}", n); - let n = -n as i8; - vshiftins_v8i8(a, b, int8x8_t(n, n, n, n, n, n, n, n)) -} -/// Shift Right and Insert (immediate) -#[inline] -#[cfg(target_arch = "arm")] -#[target_feature(enable = "neon,v7")] -#[cfg_attr(test, assert_instr("vsri.8", n = 1))] -#[rustc_args_required_const(2)] -pub unsafe fn vsriq_n_s8(a: int8x16_t, b: int8x16_t, n: i32) -> int8x16_t { - assert!(1 <= n && n <= 8, "must have 1 ≤ n ≤ 8, but n = {}", n); - let n = -n as i8; - vshiftins_v16i8( - a, - b, - int8x16_t(n, n, n, n, n, n, n, n, n, n, n, n, n, n, n, n), - ) -} -/// Shift Right and Insert (immediate) -#[inline] -#[cfg(target_arch = "arm")] -#[target_feature(enable = "neon,v7")] -#[cfg_attr(test, assert_instr("vsri.16", n = 1))] -#[rustc_args_required_const(2)] -pub unsafe fn vsri_n_s16(a: int16x4_t, b: int16x4_t, n: i32) -> int16x4_t { - assert!(1 <= n && n <= 16, "must have 1 ≤ n ≤ 16, but n = {}", n); - let n = -n as i16; - vshiftins_v4i16(a, b, int16x4_t(n, n, n, n)) -} -/// Shift Right and Insert (immediate) -#[inline] -#[cfg(target_arch = "arm")] -#[target_feature(enable = "neon,v7")] -#[cfg_attr(test, assert_instr("vsri.16", n = 1))] -#[rustc_args_required_const(2)] -pub unsafe fn vsriq_n_s16(a: int16x8_t, b: int16x8_t, n: i32) -> int16x8_t { - assert!(1 <= n && n <= 16, "must have 1 ≤ n ≤ 16, but n = {}", n); - let n = -n as i16; - vshiftins_v8i16(a, b, int16x8_t(n, n, n, n, n, n, n, n)) -} -/// Shift Right and Insert (immediate) -#[inline] -#[cfg(target_arch = "arm")] -#[target_feature(enable = "neon,v7")] -#[cfg_attr(test, assert_instr("vsri.32", n = 1))] -#[rustc_args_required_const(2)] -pub unsafe fn vsri_n_s32(a: int32x2_t, b: int32x2_t, n: i32) -> int32x2_t { - assert!(1 <= n && n <= 32, "must have 1 ≤ n ≤ 32, but n = {}", n); - vshiftins_v2i32(a, b, int32x2_t(-n, -n)) -} -/// Shift Right and Insert (immediate) -#[inline] -#[cfg(target_arch = "arm")] -#[target_feature(enable = "neon,v7")] -#[cfg_attr(test, assert_instr("vsri.32", n = 1))] -#[rustc_args_required_const(2)] -pub unsafe fn vsriq_n_s32(a: int32x4_t, b: int32x4_t, n: i32) -> int32x4_t { - assert!(1 <= n && n <= 32, "must have 1 ≤ n ≤ 32, but n = {}", n); - vshiftins_v4i32(a, b, int32x4_t(-n, -n, -n, -n)) -} -/// Shift Right and Insert (immediate) -#[inline] -#[cfg(target_arch = "arm")] -#[target_feature(enable = "neon,v7")] -#[cfg_attr(test, assert_instr("vsri.64", n = 1))] -#[rustc_args_required_const(2)] -pub unsafe fn vsri_n_s64(a: int64x1_t, b: int64x1_t, n: i32) -> int64x1_t { - assert!(1 <= n && n <= 64, "must have 1 ≤ n ≤ 64, but n = {}", n); - vshiftins_v1i64(a, b, int64x1_t(-n as i64)) -} -/// Shift Right and Insert (immediate) -#[inline] -#[cfg(target_arch = "arm")] -#[target_feature(enable = "neon,v7")] -#[cfg_attr(test, assert_instr("vsri.64", n = 1))] -#[rustc_args_required_const(2)] -pub unsafe fn vsriq_n_s64(a: int64x2_t, b: int64x2_t, n: i32) -> int64x2_t { - assert!(1 <= n && n <= 64, "must have 1 ≤ n ≤ 64, but n = {}", n); - vshiftins_v2i64(a, b, int64x2_t(-n as i64, -n as i64)) -} -/// Shift Right and Insert (immediate) -#[inline] -#[cfg(target_arch = "arm")] -#[target_feature(enable = "neon,v7")] -#[cfg_attr(test, assert_instr("vsri.8", n = 1))] -#[rustc_args_required_const(2)] -pub unsafe fn vsri_n_u8(a: uint8x8_t, b: uint8x8_t, n: i32) -> uint8x8_t { - assert!(1 <= n && n <= 8, "must have 1 ≤ n ≤ 8, but n = {}", n); - let n = -n as i8; - transmute(vshiftins_v8i8( - transmute(a), - transmute(b), - int8x8_t(n, n, n, n, n, n, n, n), - )) -} -/// Shift Right and Insert (immediate) -#[inline] -#[cfg(target_arch = "arm")] -#[target_feature(enable = "neon,v7")] -#[cfg_attr(test, assert_instr("vsri.8", n = 1))] -#[rustc_args_required_const(2)] -pub unsafe fn vsriq_n_u8(a: uint8x16_t, b: uint8x16_t, n: i32) -> uint8x16_t { - assert!(1 <= n && n <= 8, "must have 1 ≤ n ≤ 8, but n = {}", n); - let n = -n as i8; - transmute(vshiftins_v16i8( - transmute(a), - transmute(b), - int8x16_t(n, n, n, n, n, n, n, n, n, n, n, n, n, n, n, n), - )) -} -/// Shift Right and Insert (immediate) -#[inline] -#[cfg(target_arch = "arm")] -#[target_feature(enable = "neon,v7")] -#[cfg_attr(test, assert_instr("vsri.16", n = 1))] -#[rustc_args_required_const(2)] -pub unsafe fn vsri_n_u16(a: uint16x4_t, b: uint16x4_t, n: i32) -> uint16x4_t { - assert!(1 <= n && n <= 16, "must have 1 ≤ n ≤ 16, but n = {}", n); - let n = -n as i16; - transmute(vshiftins_v4i16( - transmute(a), - transmute(b), - int16x4_t(n, n, n, n), - )) -} -/// Shift Right and Insert (immediate) -#[inline] -#[cfg(target_arch = "arm")] -#[target_feature(enable = "neon,v7")] -#[cfg_attr(test, assert_instr("vsri.16", n = 1))] -#[rustc_args_required_const(2)] -pub unsafe fn vsriq_n_u16(a: uint16x8_t, b: uint16x8_t, n: i32) -> uint16x8_t { - assert!(1 <= n && n <= 16, "must have 1 ≤ n ≤ 16, but n = {}", n); - let n = -n as i16; - transmute(vshiftins_v8i16( - transmute(a), - transmute(b), - int16x8_t(n, n, n, n, n, n, n, n), - )) -} -/// Shift Right and Insert (immediate) -#[inline] -#[cfg(target_arch = "arm")] -#[target_feature(enable = "neon,v7")] -#[cfg_attr(test, assert_instr("vsri.32", n = 1))] -#[rustc_args_required_const(2)] -pub unsafe fn vsri_n_u32(a: uint32x2_t, b: uint32x2_t, n: i32) -> uint32x2_t { - assert!(1 <= n && n <= 32, "must have 1 ≤ n ≤ 32, but n = {}", n); - transmute(vshiftins_v2i32( - transmute(a), - transmute(b), - int32x2_t(-n, -n), - )) -} -/// Shift Right and Insert (immediate) -#[inline] -#[cfg(target_arch = "arm")] -#[target_feature(enable = "neon,v7")] -#[cfg_attr(test, assert_instr("vsri.32", n = 1))] -#[rustc_args_required_const(2)] -pub unsafe fn vsriq_n_u32(a: uint32x4_t, b: uint32x4_t, n: i32) -> uint32x4_t { - assert!(1 <= n && n <= 32, "must have 1 ≤ n ≤ 32, but n = {}", n); - transmute(vshiftins_v4i32( - transmute(a), - transmute(b), - int32x4_t(-n, -n, -n, -n), - )) -} -/// Shift Right and Insert (immediate) -#[inline] -#[cfg(target_arch = "arm")] -#[target_feature(enable = "neon,v7")] -#[cfg_attr(test, assert_instr("vsri.64", n = 1))] -#[rustc_args_required_const(2)] -pub unsafe fn vsri_n_u64(a: uint64x1_t, b: uint64x1_t, n: i32) -> uint64x1_t { - assert!(1 <= n && n <= 64, "must have 1 ≤ n ≤ 64, but n = {}", n); - transmute(vshiftins_v1i64( - transmute(a), - transmute(b), - int64x1_t(-n as i64), - )) -} -/// Shift Right and Insert (immediate) -#[inline] -#[cfg(target_arch = "arm")] -#[target_feature(enable = "neon,v7")] -#[cfg_attr(test, assert_instr("vsri.64", n = 1))] -#[rustc_args_required_const(2)] -pub unsafe fn vsriq_n_u64(a: uint64x2_t, b: uint64x2_t, n: i32) -> uint64x2_t { - assert!(1 <= n && n <= 64, "must have 1 ≤ n ≤ 64, but n = {}", n); - transmute(vshiftins_v2i64( - transmute(a), - transmute(b), - int64x2_t(-n as i64, -n as i64), - )) -} -/// Shift Right and Insert (immediate) -#[inline] -#[cfg(target_arch = "arm")] -#[target_feature(enable = "neon,v7")] -#[cfg_attr(test, assert_instr("vsri.8", n = 1))] -#[rustc_args_required_const(2)] -pub unsafe fn vsri_n_p8(a: poly8x8_t, b: poly8x8_t, n: i32) -> poly8x8_t { - assert!(1 <= n && n <= 8, "must have 1 ≤ n ≤ 8, but n = {}", n); - let n = -n as i8; - transmute(vshiftins_v8i8( - transmute(a), - transmute(b), - int8x8_t(n, n, n, n, n, n, n, n), - )) -} -/// Shift Right and Insert (immediate) -#[inline] -#[cfg(target_arch = "arm")] -#[target_feature(enable = "neon,v7")] -#[cfg_attr(test, assert_instr("vsri.8", n = 1))] -#[rustc_args_required_const(2)] -pub unsafe fn vsriq_n_p8(a: poly8x16_t, b: poly8x16_t, n: i32) -> poly8x16_t { - assert!(1 <= n && n <= 8, "must have 1 ≤ n ≤ 8, but n = {}", n); - let n = -n as i8; - transmute(vshiftins_v16i8( - transmute(a), - transmute(b), - int8x16_t(n, n, n, n, n, n, n, n, n, n, n, n, n, n, n, n), - )) -} -/// Shift Right and Insert (immediate) -#[inline] -#[cfg(target_arch = "arm")] -#[target_feature(enable = "neon,v7")] -#[cfg_attr(test, assert_instr("vsri.16", n = 1))] -#[rustc_args_required_const(2)] -pub unsafe fn vsri_n_p16(a: poly16x4_t, b: poly16x4_t, n: i32) -> poly16x4_t { - assert!(1 <= n && n <= 16, "must have 1 ≤ n ≤ 16, but n = {}", n); - let n = -n as i16; - transmute(vshiftins_v4i16( - transmute(a), - transmute(b), - int16x4_t(n, n, n, n), - )) -} -/// Shift Right and Insert (immediate) -#[inline] -#[cfg(target_arch = "arm")] -#[target_feature(enable = "neon,v7")] -#[cfg_attr(test, assert_instr("vsri.16", n = 1))] -#[rustc_args_required_const(2)] -pub unsafe fn vsriq_n_p16(a: poly16x8_t, b: poly16x8_t, n: i32) -> poly16x8_t { - assert!(1 <= n && n <= 16, "must have 1 ≤ n ≤ 16, but n = {}", n); - let n = -n as i16; - transmute(vshiftins_v8i16( - transmute(a), - transmute(b), - int16x8_t(n, n, n, n, n, n, n, n), - )) +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vorn))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(orn))] +pub unsafe fn vornq_u16(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t { + let c = int16x8_t(-1, -1, -1, -1, -1, -1, -1, -1); + simd_or(simd_xor(b, transmute(c)), a) } -/// Reversing vector elements (swap endianness) +/// Vector bitwise inclusive OR NOT #[inline] #[target_feature(enable = "neon")] #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrev16.8"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(rev16))] -pub unsafe fn vrev16_s8(a: int8x8_t) -> int8x8_t { - simd_shuffle8(a, a, [1, 0, 3, 2, 5, 4, 7, 6]) +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vorn))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(orn))] +pub unsafe fn vorn_u32(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t { + let c = int32x2_t(-1, -1); + simd_or(simd_xor(b, transmute(c)), a) } -/// Reversing vector elements (swap endianness) +/// Vector bitwise inclusive OR NOT #[inline] #[target_feature(enable = "neon")] #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrev16.8"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(rev16))] -pub unsafe fn vrev16q_s8(a: int8x16_t) -> int8x16_t { - simd_shuffle16(a, a, [1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14]) +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vorn))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(orn))] +pub unsafe fn vornq_u32(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t { + let c = int32x4_t(-1, -1, -1, -1); + simd_or(simd_xor(b, transmute(c)), a) } -/// Reversing vector elements (swap endianness) +/// Vector bitwise inclusive OR NOT #[inline] #[target_feature(enable = "neon")] #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrev16.8"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(rev16))] -pub unsafe fn vrev16_u8(a: uint8x8_t) -> uint8x8_t { - simd_shuffle8(a, a, [1, 0, 3, 2, 5, 4, 7, 6]) +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vorn))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(orn))] +pub unsafe fn vorn_u64(a: uint64x1_t, b: uint64x1_t) -> uint64x1_t { + let c = int64x1_t(-1); + simd_or(simd_xor(b, transmute(c)), a) } -/// Reversing vector elements (swap endianness) +/// Vector bitwise inclusive OR NOT #[inline] #[target_feature(enable = "neon")] #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrev16.8"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(rev16))] -pub unsafe fn vrev16q_u8(a: uint8x16_t) -> uint8x16_t { - simd_shuffle16(a, a, [1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14]) +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vorn))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(orn))] +pub unsafe fn vornq_u64(a: uint64x2_t, b: uint64x2_t) -> uint64x2_t { + let c = int64x2_t(-1, -1); + simd_or(simd_xor(b, transmute(c)), a) } -/// Reversing vector elements (swap endianness) +/// Folding minimum of adjacent pairs #[inline] #[target_feature(enable = "neon")] #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrev16.8"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(rev16))] -pub unsafe fn vrev16_p8(a: poly8x8_t) -> poly8x8_t { - simd_shuffle8(a, a, [1, 0, 3, 2, 5, 4, 7, 6]) +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vpmin))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sminp))] +pub unsafe fn vpmin_s8(a: int8x8_t, b: int8x8_t) -> int8x8_t { + vpmins_v8i8(a, b) } -/// Reversing vector elements (swap endianness) +/// Folding minimum of adjacent pairs #[inline] #[target_feature(enable = "neon")] #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrev16.8"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(rev16))] -pub unsafe fn vrev16q_p8(a: poly8x16_t) -> poly8x16_t { - simd_shuffle16(a, a, [1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14]) +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vpmin))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sminp))] +pub unsafe fn vpmin_s16(a: int16x4_t, b: int16x4_t) -> int16x4_t { + vpmins_v4i16(a, b) } -/// Reversing vector elements (swap endianness) +/// Folding minimum of adjacent pairs #[inline] #[target_feature(enable = "neon")] #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrev32.8"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(rev32))] -pub unsafe fn vrev32_s8(a: int8x8_t) -> int8x8_t { - simd_shuffle8(a, a, [3, 2, 1, 0, 7, 6, 5, 4]) +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vpmin))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sminp))] +pub unsafe fn vpmin_s32(a: int32x2_t, b: int32x2_t) -> int32x2_t { + vpmins_v2i32(a, b) } -/// Reversing vector elements (swap endianness) +/// Folding minimum of adjacent pairs #[inline] #[target_feature(enable = "neon")] #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrev32.8"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(rev32))] -pub unsafe fn vrev32q_s8(a: int8x16_t) -> int8x16_t { - simd_shuffle16(a, a, [3, 2, 1, 0, 7, 6, 5, 4, 11, 10, 9, 8, 15, 14, 13, 12]) +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vpmin))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uminp))] +pub unsafe fn vpmin_u8(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t { + vpminu_v8i8(a, b) } -/// Reversing vector elements (swap endianness) +/// Folding minimum of adjacent pairs #[inline] #[target_feature(enable = "neon")] #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrev32.8"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(rev32))] -pub unsafe fn vrev32_u8(a: uint8x8_t) -> uint8x8_t { - simd_shuffle8(a, a, [3, 2, 1, 0, 7, 6, 5, 4]) +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vpmin))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uminp))] +pub unsafe fn vpmin_u16(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t { + vpminu_v4i16(a, b) } -/// Reversing vector elements (swap endianness) +/// Folding minimum of adjacent pairs #[inline] #[target_feature(enable = "neon")] #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrev32.8"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(rev32))] -pub unsafe fn vrev32q_u8(a: uint8x16_t) -> uint8x16_t { - simd_shuffle16(a, a, [3, 2, 1, 0, 7, 6, 5, 4, 11, 10, 9, 8, 15, 14, 13, 12]) +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vpmin))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uminp))] +pub unsafe fn vpmin_u32(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t { + vpminu_v2i32(a, b) } -/// Reversing vector elements (swap endianness) +/// Folding minimum of adjacent pairs #[inline] #[target_feature(enable = "neon")] #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrev32.16"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(rev32))] -pub unsafe fn vrev32_u16(a: uint16x4_t) -> uint16x4_t { - simd_shuffle4(a, a, [1, 0, 3, 2]) +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vpmin))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fminp))] +pub unsafe fn vpmin_f32(a: float32x2_t, b: float32x2_t) -> float32x2_t { + vpminf_v2f32(a, b) } -/// Reversing vector elements (swap endianness) +/// Folding maximum of adjacent pairs #[inline] #[target_feature(enable = "neon")] #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrev32.16"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(rev32))] -pub unsafe fn vrev32q_u16(a: uint16x8_t) -> uint16x8_t { - simd_shuffle8(a, a, [1, 0, 3, 2, 5, 4, 7, 6]) +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vpmax))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(smaxp))] +pub unsafe fn vpmax_s8(a: int8x8_t, b: int8x8_t) -> int8x8_t { + vpmaxs_v8i8(a, b) } -/// Reversing vector elements (swap endianness) +/// Folding maximum of adjacent pairs #[inline] #[target_feature(enable = "neon")] #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrev32.8"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(rev32))] -pub unsafe fn vrev32_p8(a: poly8x8_t) -> poly8x8_t { - simd_shuffle8(a, a, [3, 2, 1, 0, 7, 6, 5, 4]) +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vpmax))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(smaxp))] +pub unsafe fn vpmax_s16(a: int16x4_t, b: int16x4_t) -> int16x4_t { + vpmaxs_v4i16(a, b) } -/// Reversing vector elements (swap endianness) +/// Folding maximum of adjacent pairs #[inline] #[target_feature(enable = "neon")] #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrev32.8"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(rev32))] -pub unsafe fn vrev32q_p8(a: poly8x16_t) -> poly8x16_t { - simd_shuffle16(a, a, [3, 2, 1, 0, 7, 6, 5, 4, 11, 10, 9, 8, 15, 14, 13, 12]) +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vpmax))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(smaxp))] +pub unsafe fn vpmax_s32(a: int32x2_t, b: int32x2_t) -> int32x2_t { + vpmaxs_v2i32(a, b) } -/// Reversing vector elements (swap endianness) +/// Folding maximum of adjacent pairs #[inline] #[target_feature(enable = "neon")] #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrev64.8"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(rev64))] -pub unsafe fn vrev64_s8(a: int8x8_t) -> int8x8_t { - simd_shuffle8(a, a, [7, 6, 5, 4, 3, 2, 1, 0]) +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vpmax))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(umaxp))] +pub unsafe fn vpmax_u8(a: uint8x8_t, b: uint8x8_t) -> uint8x8_t { + vpmaxu_v8i8(a, b) } -/// Reversing vector elements (swap endianness) +/// Folding maximum of adjacent pairs #[inline] #[target_feature(enable = "neon")] #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrev64.8"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(rev64))] -pub unsafe fn vrev64q_s8(a: int8x16_t) -> int8x16_t { - simd_shuffle16(a, a, [7, 6, 5, 4, 3, 2, 1, 0, 15, 14, 13, 12, 11, 10, 9, 8]) +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vpmax))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(umaxp))] +pub unsafe fn vpmax_u16(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t { + vpmaxu_v4i16(a, b) } -/// Reversing vector elements (swap endianness) +/// Folding maximum of adjacent pairs #[inline] #[target_feature(enable = "neon")] #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrev64.16"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(rev64))] -pub unsafe fn vrev64_s16(a: int16x4_t) -> int16x4_t { - simd_shuffle4(a, a, [3, 2, 1, 0]) +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vpmax))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(umaxp))] +pub unsafe fn vpmax_u32(a: uint32x2_t, b: uint32x2_t) -> uint32x2_t { + vpmaxu_v2i32(a, b) } -/// Reversing vector elements (swap endianness) +/// Folding maximum of adjacent pairs #[inline] #[target_feature(enable = "neon")] #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrev64.16"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(rev64))] -pub unsafe fn vrev64q_s16(a: int16x8_t) -> int16x8_t { - simd_shuffle8(a, a, [3, 2, 1, 0, 7, 6, 5, 4]) +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vpmax))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fmaxp))] +pub unsafe fn vpmax_f32(a: float32x2_t, b: float32x2_t) -> float32x2_t { + vpmaxf_v2f32(a, b) } -/// Reversing vector elements (swap endianness) +/// Move vector element to general-purpose register #[inline] #[target_feature(enable = "neon")] #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrev64.32"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(rev64))] -pub unsafe fn vrev64_s32(a: int32x2_t) -> int32x2_t { - simd_shuffle2(a, a, [1, 0]) +#[rustc_legacy_const_generics(1)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmov.32", IMM5 = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mov, IMM5 = 1))] +// Based on the discussion in https://github.com/rust-lang/stdarch/pull/792 +// `mov` seems to be an acceptable intrinsic to compile to +// #[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(vmov, IMM5 = 1))] +pub unsafe fn vgetq_lane_u64(v: uint64x2_t) -> u64 { + static_assert_imm1!(IMM5); + simd_extract(v, IMM5 as u32) } -/// Reversing vector elements (swap endianness) +/// Move vector element to general-purpose register #[inline] #[target_feature(enable = "neon")] #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrev64.32"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(rev64))] -pub unsafe fn vrev64q_s32(a: int32x4_t) -> int32x4_t { - simd_shuffle4(a, a, [1, 0, 3, 2]) +#[rustc_legacy_const_generics(1)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmov.32", IMM5 = 0))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fmov, IMM5 = 0))] +// FIXME: no 32bit this seems to be turned into two vmov.32 instructions +// validate correctness +pub unsafe fn vget_lane_u64(v: uint64x1_t) -> u64 { + static_assert!(IMM5 : i32 where IMM5 == 0); + simd_extract(v, 0) } -/// Reversing vector elements (swap endianness) +/// Move vector element to general-purpose register #[inline] #[target_feature(enable = "neon")] #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrev64.8"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(rev64))] -pub unsafe fn vrev64_u8(a: uint8x8_t) -> uint8x8_t { - simd_shuffle8(a, a, [7, 6, 5, 4, 3, 2, 1, 0]) +#[rustc_legacy_const_generics(1)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmov.u16", IMM5 = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(umov, IMM5 = 2))] +pub unsafe fn vget_lane_u16(v: uint16x4_t) -> u16 { + static_assert_imm2!(IMM5); + simd_extract(v, IMM5 as u32) } -/// Reversing vector elements (swap endianness) +/// Move vector element to general-purpose register #[inline] #[target_feature(enable = "neon")] #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrev64.8"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(rev64))] -pub unsafe fn vrev64q_u8(a: uint8x16_t) -> uint8x16_t { - simd_shuffle16(a, a, [7, 6, 5, 4, 3, 2, 1, 0, 15, 14, 13, 12, 11, 10, 9, 8]) +#[rustc_legacy_const_generics(1)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmov.s16", IMM5 = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(smov, IMM5 = 2))] +pub unsafe fn vget_lane_s16(v: int16x4_t) -> i16 { + static_assert_imm2!(IMM5); + simd_extract(v, IMM5 as u32) } -/// Reversing vector elements (swap endianness) +/// Move vector element to general-purpose register #[inline] #[target_feature(enable = "neon")] #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrev64.16"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(rev64))] -pub unsafe fn vrev64_u16(a: uint16x4_t) -> uint16x4_t { - simd_shuffle4(a, a, [3, 2, 1, 0]) +#[rustc_legacy_const_generics(1)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmov.u16", IMM5 = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(umov, IMM5 = 2))] +pub unsafe fn vget_lane_p16(v: poly16x4_t) -> p16 { + static_assert_imm2!(IMM5); + simd_extract(v, IMM5 as u32) } -/// Reversing vector elements (swap endianness) +/// Move vector element to general-purpose register #[inline] #[target_feature(enable = "neon")] #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrev64.16"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(rev64))] -pub unsafe fn vrev64q_u16(a: uint16x8_t) -> uint16x8_t { - simd_shuffle8(a, a, [3, 2, 1, 0, 7, 6, 5, 4]) +#[rustc_legacy_const_generics(1)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmov.32", IMM5 = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mov, IMM5 = 1))] +pub unsafe fn vget_lane_u32(v: uint32x2_t) -> u32 { + static_assert_imm1!(IMM5); + simd_extract(v, IMM5 as u32) } -/// Reversing vector elements (swap endianness) +/// Move vector element to general-purpose register #[inline] #[target_feature(enable = "neon")] #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrev64.32"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(rev64))] -pub unsafe fn vrev64_u32(a: uint32x2_t) -> uint32x2_t { - simd_shuffle2(a, a, [1, 0]) +#[rustc_legacy_const_generics(1)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmov.32", IMM5 = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mov, IMM5 = 1))] +pub unsafe fn vget_lane_s32(v: int32x2_t) -> i32 { + static_assert_imm1!(IMM5); + simd_extract(v, IMM5 as u32) } -/// Reversing vector elements (swap endianness) +/// Duplicate vector element to vector or scalar #[inline] #[target_feature(enable = "neon")] #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrev64.32"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(rev64))] -pub unsafe fn vrev64q_u32(a: uint32x4_t) -> uint32x4_t { - simd_shuffle4(a, a, [1, 0, 3, 2]) +#[rustc_legacy_const_generics(1)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmov.f32", IMM5 = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mov, IMM5 = 1))] +pub unsafe fn vget_lane_f32(v: float32x2_t) -> f32 { + static_assert_imm1!(IMM5); + simd_extract(v, IMM5 as u32) } -/// Reversing vector elements (swap endianness) +/// Duplicate vector element to vector or scalar #[inline] #[target_feature(enable = "neon")] #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrev64.32"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(rev64))] -pub unsafe fn vrev64_f32(a: float32x2_t) -> float32x2_t { - simd_shuffle2(a, a, [1, 0]) +#[rustc_legacy_const_generics(1)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmov.f32", IMM5 = 1))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mov, IMM5 = 1))] +pub unsafe fn vgetq_lane_f32(v: float32x4_t) -> f32 { + static_assert_imm2!(IMM5); + simd_extract(v, IMM5 as u32) } -/// Reversing vector elements (swap endianness) +/// Move vector element to general-purpose register #[inline] #[target_feature(enable = "neon")] #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrev64.32"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(rev64))] -pub unsafe fn vrev64q_f32(a: float32x4_t) -> float32x4_t { - simd_shuffle4(a, a, [1, 0, 3, 2]) +#[rustc_legacy_const_generics(1)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmov.32", IMM5 = 0))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fmov, IMM5 = 0))] +pub unsafe fn vget_lane_p64(v: poly64x1_t) -> p64 { + static_assert!(IMM5 : i32 where IMM5 == 0); + simd_extract(v, IMM5 as u32) } -/// Reversing vector elements (swap endianness) +/// Move vector element to general-purpose register #[inline] #[target_feature(enable = "neon")] #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrev64.8"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(rev64))] -pub unsafe fn vrev64_p8(a: poly8x8_t) -> poly8x8_t { - simd_shuffle8(a, a, [7, 6, 5, 4, 3, 2, 1, 0]) +#[rustc_legacy_const_generics(1)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmov.32", IMM5 = 0))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fmov, IMM5 = 0))] +pub unsafe fn vgetq_lane_p64(v: poly64x2_t) -> p64 { + static_assert_imm1!(IMM5); + simd_extract(v, IMM5 as u32) } -/// Reversing vector elements (swap endianness) +/// Move vector element to general-purpose register #[inline] #[target_feature(enable = "neon")] #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrev64.8"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(rev64))] -pub unsafe fn vrev64q_p8(a: poly8x16_t) -> poly8x16_t { - simd_shuffle16(a, a, [7, 6, 5, 4, 3, 2, 1, 0, 15, 14, 13, 12, 11, 10, 9, 8]) +#[rustc_legacy_const_generics(1)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmov.32", IMM5 = 0))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fmov, IMM5 = 0))] +pub unsafe fn vget_lane_s64(v: int64x1_t) -> i64 { + static_assert!(IMM5 : i32 where IMM5 == 0); + simd_extract(v, IMM5 as u32) } -/// Reversing vector elements (swap endianness) +/// Move vector element to general-purpose register #[inline] #[target_feature(enable = "neon")] #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrev64.16"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(rev64))] -pub unsafe fn vrev64_p16(a: poly16x4_t) -> poly16x4_t { - simd_shuffle4(a, a, [3, 2, 1, 0]) +#[rustc_legacy_const_generics(1)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmov.32", IMM5 = 0))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fmov, IMM5 = 0))] +pub unsafe fn vgetq_lane_s64(v: int64x2_t) -> i64 { + static_assert_imm1!(IMM5); + simd_extract(v, IMM5 as u32) } -/// Reversing vector elements (swap endianness) +/// Move vector element to general-purpose register #[inline] #[target_feature(enable = "neon")] #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrev64.16"))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(rev64))] -pub unsafe fn vrev64q_p16(a: poly16x8_t) -> poly16x8_t { - simd_shuffle8(a, a, [3, 2, 1, 0, 7, 6, 5, 4]) +#[rustc_legacy_const_generics(1)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmov.u16", IMM5 = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(umov, IMM5 = 2))] +pub unsafe fn vgetq_lane_u16(v: uint16x8_t) -> u16 { + static_assert_imm3!(IMM5); + simd_extract(v, IMM5 as u32) } -#[cfg(test)] -mod tests { - use super::*; - use crate::core_arch::arm::test_support::*; - use crate::core_arch::{arm::*, simd::*}; - use std::{i16, i32, i8, mem::transmute, u16, u32, u8, vec::Vec}; - use stdarch_test::simd_test; +/// Move vector element to general-purpose register +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[rustc_legacy_const_generics(1)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmov.32", IMM5 = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mov, IMM5 = 2))] +pub unsafe fn vgetq_lane_u32(v: uint32x4_t) -> u32 { + static_assert_imm2!(IMM5); + simd_extract(v, IMM5 as u32) +} - #[simd_test(enable = "neon")] - unsafe fn test_vld1_lane_s8() { - let a = i8x8::new(0, 1, 2, 3, 4, 5, 6, 7); - let elem: i8 = 42; - let e = i8x8::new(0, 1, 2, 3, 4, 5, 6, 42); - let r: i8x8 = transmute(vld1_lane_s8(&elem, transmute(a), 7)); - assert_eq!(r, e) - } +/// Move vector element to general-purpose register +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[rustc_legacy_const_generics(1)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmov.s16", IMM5 = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(smov, IMM5 = 2))] +pub unsafe fn vgetq_lane_s16(v: int16x8_t) -> i16 { + static_assert_imm3!(IMM5); + simd_extract(v, IMM5 as u32) +} - #[simd_test(enable = "neon")] - unsafe fn test_vld1q_lane_s8() { - let a = i8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); - let elem: i8 = 42; - let e = i8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 42); - let r: i8x16 = transmute(vld1q_lane_s8(&elem, transmute(a), 15)); - assert_eq!(r, e) - } +/// Move vector element to general-purpose register +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[rustc_legacy_const_generics(1)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmov.u16", IMM5 = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(umov, IMM5 = 2))] +pub unsafe fn vgetq_lane_p16(v: poly16x8_t) -> p16 { + static_assert_imm3!(IMM5); + simd_extract(v, IMM5 as u32) +} - #[simd_test(enable = "neon")] - unsafe fn test_vld1_lane_s16() { - let a = i16x4::new(0, 1, 2, 3); - let elem: i16 = 42; - let e = i16x4::new(0, 1, 2, 42); - let r: i16x4 = transmute(vld1_lane_s16(&elem, transmute(a), 3)); - assert_eq!(r, e) - } +/// Move vector element to general-purpose register +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[rustc_legacy_const_generics(1)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmov.32", IMM5 = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(mov, IMM5 = 2))] +pub unsafe fn vgetq_lane_s32(v: int32x4_t) -> i32 { + static_assert_imm2!(IMM5); + simd_extract(v, IMM5 as u32) +} - #[simd_test(enable = "neon")] - unsafe fn test_vld1q_lane_s16() { - let a = i16x8::new(0, 1, 2, 3, 4, 5, 6, 7); - let elem: i16 = 42; - let e = i16x8::new(0, 1, 2, 3, 4, 5, 6, 42); - let r: i16x8 = transmute(vld1q_lane_s16(&elem, transmute(a), 7)); - assert_eq!(r, e) - } +/// Move vector element to general-purpose register +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[rustc_legacy_const_generics(1)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmov.u8", IMM5 = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(umov, IMM5 = 2))] +pub unsafe fn vget_lane_u8(v: uint8x8_t) -> u8 { + static_assert_imm3!(IMM5); + simd_extract(v, IMM5 as u32) +} - #[simd_test(enable = "neon")] - unsafe fn test_vld1_lane_s32() { - let a = i32x2::new(0, 1); - let elem: i32 = 42; - let e = i32x2::new(0, 42); - let r: i32x2 = transmute(vld1_lane_s32(&elem, transmute(a), 1)); - assert_eq!(r, e) - } +/// Move vector element to general-purpose register +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[rustc_legacy_const_generics(1)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmov.s8", IMM5 = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(smov, IMM5 = 2))] +pub unsafe fn vget_lane_s8(v: int8x8_t) -> i8 { + static_assert_imm3!(IMM5); + simd_extract(v, IMM5 as u32) +} - #[simd_test(enable = "neon")] - unsafe fn test_vld1q_lane_s32() { - let a = i32x4::new(0, 1, 2, 3); - let elem: i32 = 42; - let e = i32x4::new(0, 1, 2, 42); - let r: i32x4 = transmute(vld1q_lane_s32(&elem, transmute(a), 3)); - assert_eq!(r, e) +/// Move vector element to general-purpose register +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[rustc_legacy_const_generics(1)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmov.u8", IMM5 = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(umov, IMM5 = 2))] +pub unsafe fn vget_lane_p8(v: poly8x8_t) -> p8 { + static_assert_imm3!(IMM5); + simd_extract(v, IMM5 as u32) +} + +/// Move vector element to general-purpose register +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[rustc_legacy_const_generics(1)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmov.u8", IMM5 = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(umov, IMM5 = 2))] +pub unsafe fn vgetq_lane_u8(v: uint8x16_t) -> u8 { + static_assert_imm4!(IMM5); + simd_extract(v, IMM5 as u32) +} + +/// Move vector element to general-purpose register +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[rustc_legacy_const_generics(1)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmov.s8", IMM5 = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(smov, IMM5 = 2))] +pub unsafe fn vgetq_lane_s8(v: int8x16_t) -> i8 { + static_assert_imm4!(IMM5); + simd_extract(v, IMM5 as u32) +} + +/// Move vector element to general-purpose register +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[rustc_legacy_const_generics(1)] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmov.u8", IMM5 = 2))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(umov, IMM5 = 2))] +pub unsafe fn vgetq_lane_p8(v: poly8x16_t) -> p8 { + static_assert_imm4!(IMM5); + simd_extract(v, IMM5 as u32) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmov"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ext))] +pub unsafe fn vget_high_s8(a: int8x16_t) -> int8x8_t { + simd_shuffle8!(a, a, [8, 9, 10, 11, 12, 13, 14, 15]) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmov"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ext))] +pub unsafe fn vget_high_s16(a: int16x8_t) -> int16x4_t { + simd_shuffle4!(a, a, [4, 5, 6, 7]) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmov"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ext))] +pub unsafe fn vget_high_s32(a: int32x4_t) -> int32x2_t { + simd_shuffle2!(a, a, [2, 3]) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmov"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ext))] +pub unsafe fn vget_high_s64(a: int64x2_t) -> int64x1_t { + int64x1_t(simd_extract(a, 1)) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmov"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ext))] +pub unsafe fn vget_high_u8(a: uint8x16_t) -> uint8x8_t { + simd_shuffle8!(a, a, [8, 9, 10, 11, 12, 13, 14, 15]) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmov"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ext))] +pub unsafe fn vget_high_u16(a: uint16x8_t) -> uint16x4_t { + simd_shuffle4!(a, a, [4, 5, 6, 7]) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmov"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ext))] +pub unsafe fn vget_high_u32(a: uint32x4_t) -> uint32x2_t { + simd_shuffle2!(a, a, [2, 3]) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmov"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ext))] +pub unsafe fn vget_high_u64(a: uint64x2_t) -> uint64x1_t { + uint64x1_t(simd_extract(a, 1)) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmov"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ext))] +pub unsafe fn vget_high_p8(a: poly8x16_t) -> poly8x8_t { + simd_shuffle8!(a, a, [8, 9, 10, 11, 12, 13, 14, 15]) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmov"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ext))] +pub unsafe fn vget_high_p16(a: poly16x8_t) -> poly16x4_t { + simd_shuffle4!(a, a, [4, 5, 6, 7]) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmov"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ext))] +pub unsafe fn vget_high_f32(a: float32x4_t) -> float32x2_t { + simd_shuffle2!(a, a, [2, 3]) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("ldr"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ldr))] +pub unsafe fn vget_low_s8(a: int8x16_t) -> int8x8_t { + simd_shuffle8!(a, a, [0, 1, 2, 3, 4, 5, 6, 7]) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("ldr"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ldr))] +pub unsafe fn vget_low_s16(a: int16x8_t) -> int16x4_t { + simd_shuffle4!(a, a, [0, 1, 2, 3]) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("ldr"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ldr))] +pub unsafe fn vget_low_s32(a: int32x4_t) -> int32x2_t { + simd_shuffle2!(a, a, [0, 1]) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("ldr"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ldr))] +pub unsafe fn vget_low_s64(a: int64x2_t) -> int64x1_t { + int64x1_t(simd_extract(a, 0)) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("ldr"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ldr))] +pub unsafe fn vget_low_u8(a: uint8x16_t) -> uint8x8_t { + simd_shuffle8!(a, a, [0, 1, 2, 3, 4, 5, 6, 7]) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("ldr"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ldr))] +pub unsafe fn vget_low_u16(a: uint16x8_t) -> uint16x4_t { + simd_shuffle4!(a, a, [0, 1, 2, 3]) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("ldr"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ldr))] +pub unsafe fn vget_low_u32(a: uint32x4_t) -> uint32x2_t { + simd_shuffle2!(a, a, [0, 1]) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("ldr"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ldr))] +pub unsafe fn vget_low_u64(a: uint64x2_t) -> uint64x1_t { + uint64x1_t(simd_extract(a, 0)) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("ldr"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ldr))] +pub unsafe fn vget_low_p8(a: poly8x16_t) -> poly8x8_t { + simd_shuffle8!(a, a, [0, 1, 2, 3, 4, 5, 6, 7]) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("ldr"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ldr))] +pub unsafe fn vget_low_p16(a: poly16x8_t) -> poly16x4_t { + simd_shuffle4!(a, a, [0, 1, 2, 3]) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("ldr"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(ldr))] +pub unsafe fn vget_low_f32(a: float32x4_t) -> float32x2_t { + simd_shuffle2!(a, a, [0, 1]) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup))] +pub unsafe fn vdupq_n_s8(value: i8) -> int8x16_t { + int8x16_t( + value, value, value, value, value, value, value, value, value, value, value, value, value, + value, value, value, + ) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup))] +pub unsafe fn vdupq_n_s16(value: i16) -> int16x8_t { + int16x8_t(value, value, value, value, value, value, value, value) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup))] +pub unsafe fn vdupq_n_s32(value: i32) -> int32x4_t { + int32x4_t(value, value, value, value) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmov"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup))] +pub unsafe fn vdupq_n_s64(value: i64) -> int64x2_t { + int64x2_t(value, value) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup))] +pub unsafe fn vdupq_n_u8(value: u8) -> uint8x16_t { + uint8x16_t( + value, value, value, value, value, value, value, value, value, value, value, value, value, + value, value, value, + ) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup))] +pub unsafe fn vdupq_n_u16(value: u16) -> uint16x8_t { + uint16x8_t(value, value, value, value, value, value, value, value) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup))] +pub unsafe fn vdupq_n_u32(value: u32) -> uint32x4_t { + uint32x4_t(value, value, value, value) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmov"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup))] +pub unsafe fn vdupq_n_u64(value: u64) -> uint64x2_t { + uint64x2_t(value, value) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup))] +pub unsafe fn vdupq_n_p8(value: p8) -> poly8x16_t { + poly8x16_t( + value, value, value, value, value, value, value, value, value, value, value, value, value, + value, value, value, + ) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup))] +pub unsafe fn vdupq_n_p16(value: p16) -> poly16x8_t { + poly16x8_t(value, value, value, value, value, value, value, value) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup))] +pub unsafe fn vdupq_n_f32(value: f32) -> float32x4_t { + float32x4_t(value, value, value, value) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup))] +pub unsafe fn vdup_n_s8(value: i8) -> int8x8_t { + int8x8_t(value, value, value, value, value, value, value, value) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup))] +pub unsafe fn vdup_n_s16(value: i16) -> int16x4_t { + int16x4_t(value, value, value, value) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup))] +pub unsafe fn vdup_n_s32(value: i32) -> int32x2_t { + int32x2_t(value, value) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmov"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fmov))] +pub unsafe fn vdup_n_s64(value: i64) -> int64x1_t { + int64x1_t(value) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup))] +pub unsafe fn vdup_n_u8(value: u8) -> uint8x8_t { + uint8x8_t(value, value, value, value, value, value, value, value) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup))] +pub unsafe fn vdup_n_u16(value: u16) -> uint16x4_t { + uint16x4_t(value, value, value, value) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup))] +pub unsafe fn vdup_n_u32(value: u32) -> uint32x2_t { + uint32x2_t(value, value) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmov"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fmov))] +pub unsafe fn vdup_n_u64(value: u64) -> uint64x1_t { + uint64x1_t(value) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup))] +pub unsafe fn vdup_n_p8(value: p8) -> poly8x8_t { + poly8x8_t(value, value, value, value, value, value, value, value) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup))] +pub unsafe fn vdup_n_p16(value: p16) -> poly16x4_t { + poly16x4_t(value, value, value, value) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup))] +pub unsafe fn vdup_n_f32(value: f32) -> float32x2_t { + float32x2_t(value, value) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup))] +pub unsafe fn vmov_n_s8(value: i8) -> int8x8_t { + vdup_n_s8(value) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup))] +pub unsafe fn vmov_n_s16(value: i16) -> int16x4_t { + vdup_n_s16(value) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup))] +pub unsafe fn vmov_n_s32(value: i32) -> int32x2_t { + vdup_n_s32(value) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmov"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fmov))] +pub unsafe fn vmov_n_s64(value: i64) -> int64x1_t { + vdup_n_s64(value) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup))] +pub unsafe fn vmov_n_u8(value: u8) -> uint8x8_t { + vdup_n_u8(value) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup))] +pub unsafe fn vmov_n_u16(value: u16) -> uint16x4_t { + vdup_n_u16(value) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup))] +pub unsafe fn vmov_n_u32(value: u32) -> uint32x2_t { + vdup_n_u32(value) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmov"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(fmov))] +pub unsafe fn vmov_n_u64(value: u64) -> uint64x1_t { + vdup_n_u64(value) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup))] +pub unsafe fn vmov_n_p8(value: p8) -> poly8x8_t { + vdup_n_p8(value) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup))] +pub unsafe fn vmov_n_p16(value: p16) -> poly16x4_t { + vdup_n_p16(value) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup))] +pub unsafe fn vmov_n_f32(value: f32) -> float32x2_t { + vdup_n_f32(value) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup))] +pub unsafe fn vmovq_n_s8(value: i8) -> int8x16_t { + vdupq_n_s8(value) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup))] +pub unsafe fn vmovq_n_s16(value: i16) -> int16x8_t { + vdupq_n_s16(value) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup))] +pub unsafe fn vmovq_n_s32(value: i32) -> int32x4_t { + vdupq_n_s32(value) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmov"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup))] +pub unsafe fn vmovq_n_s64(value: i64) -> int64x2_t { + vdupq_n_s64(value) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup))] +pub unsafe fn vmovq_n_u8(value: u8) -> uint8x16_t { + vdupq_n_u8(value) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup))] +pub unsafe fn vmovq_n_u16(value: u16) -> uint16x8_t { + vdupq_n_u16(value) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup))] +pub unsafe fn vmovq_n_u32(value: u32) -> uint32x4_t { + vdupq_n_u32(value) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vmov"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup))] +pub unsafe fn vmovq_n_u64(value: u64) -> uint64x2_t { + vdupq_n_u64(value) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup))] +pub unsafe fn vmovq_n_p8(value: p8) -> poly8x16_t { + vdupq_n_p8(value) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup))] +pub unsafe fn vmovq_n_p16(value: p16) -> poly16x8_t { + vdupq_n_p16(value) +} + +/// Duplicate vector element to vector or scalar +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vdup.32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(dup))] +pub unsafe fn vmovq_n_f32(value: f32) -> float32x4_t { + vdupq_n_f32(value) +} + +/// Extract vector from pair of vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("str", N = 0))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr("str", N = 0))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vext_s64(a: int64x1_t, _b: int64x1_t) -> int64x1_t { + if N != 0 { + unreachable_unchecked() + } + a +} + +/// Extract vector from pair of vectors +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("str", N = 0))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr("str", N = 0))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn vext_u64(a: uint64x1_t, _b: uint64x1_t) -> uint64x1_t { + if N != 0 { + unreachable_unchecked() + } + a +} + +/// Population count per byte. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vcnt))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cnt))] +pub unsafe fn vcnt_s8(a: int8x8_t) -> int8x8_t { + vcnt_s8_(a) +} +/// Population count per byte. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vcnt))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cnt))] +pub unsafe fn vcntq_s8(a: int8x16_t) -> int8x16_t { + vcntq_s8_(a) +} +/// Population count per byte. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vcnt))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cnt))] +pub unsafe fn vcnt_u8(a: uint8x8_t) -> uint8x8_t { + transmute(vcnt_s8_(transmute(a))) +} +/// Population count per byte. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vcnt))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cnt))] +pub unsafe fn vcntq_u8(a: uint8x16_t) -> uint8x16_t { + transmute(vcntq_s8_(transmute(a))) +} +/// Population count per byte. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vcnt))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cnt))] +pub unsafe fn vcnt_p8(a: poly8x8_t) -> poly8x8_t { + transmute(vcnt_s8_(transmute(a))) +} +/// Population count per byte. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vcnt))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(cnt))] +pub unsafe fn vcntq_p8(a: poly8x16_t) -> poly8x16_t { + transmute(vcntq_s8_(transmute(a))) +} + +/// Reversing vector elements (swap endianness) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrev16.8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(rev16))] +pub unsafe fn vrev16_s8(a: int8x8_t) -> int8x8_t { + simd_shuffle8!(a, a, [1, 0, 3, 2, 5, 4, 7, 6]) +} + +/// Reversing vector elements (swap endianness) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrev16.8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(rev16))] +pub unsafe fn vrev16q_s8(a: int8x16_t) -> int8x16_t { + simd_shuffle16!(a, a, [1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14]) +} + +/// Reversing vector elements (swap endianness) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrev16.8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(rev16))] +pub unsafe fn vrev16_u8(a: uint8x8_t) -> uint8x8_t { + simd_shuffle8!(a, a, [1, 0, 3, 2, 5, 4, 7, 6]) +} + +/// Reversing vector elements (swap endianness) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrev16.8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(rev16))] +pub unsafe fn vrev16q_u8(a: uint8x16_t) -> uint8x16_t { + simd_shuffle16!(a, a, [1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14]) +} + +/// Reversing vector elements (swap endianness) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrev16.8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(rev16))] +pub unsafe fn vrev16_p8(a: poly8x8_t) -> poly8x8_t { + simd_shuffle8!(a, a, [1, 0, 3, 2, 5, 4, 7, 6]) +} + +/// Reversing vector elements (swap endianness) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrev16.8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(rev16))] +pub unsafe fn vrev16q_p8(a: poly8x16_t) -> poly8x16_t { + simd_shuffle16!(a, a, [1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14]) +} + +/// Reversing vector elements (swap endianness) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrev32.8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(rev32))] +pub unsafe fn vrev32_s8(a: int8x8_t) -> int8x8_t { + simd_shuffle8!(a, a, [3, 2, 1, 0, 7, 6, 5, 4]) +} + +/// Reversing vector elements (swap endianness) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrev32.8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(rev32))] +pub unsafe fn vrev32q_s8(a: int8x16_t) -> int8x16_t { + simd_shuffle16!(a, a, [3, 2, 1, 0, 7, 6, 5, 4, 11, 10, 9, 8, 15, 14, 13, 12]) +} + +/// Reversing vector elements (swap endianness) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrev32.8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(rev32))] +pub unsafe fn vrev32_u8(a: uint8x8_t) -> uint8x8_t { + simd_shuffle8!(a, a, [3, 2, 1, 0, 7, 6, 5, 4]) +} + +/// Reversing vector elements (swap endianness) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrev32.8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(rev32))] +pub unsafe fn vrev32q_u8(a: uint8x16_t) -> uint8x16_t { + simd_shuffle16!(a, a, [3, 2, 1, 0, 7, 6, 5, 4, 11, 10, 9, 8, 15, 14, 13, 12]) +} + +/// Reversing vector elements (swap endianness) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrev32.16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(rev32))] +pub unsafe fn vrev32_s16(a: int16x4_t) -> int16x4_t { + simd_shuffle4!(a, a, [1, 0, 3, 2]) +} + +/// Reversing vector elements (swap endianness) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrev32.16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(rev32))] +pub unsafe fn vrev32q_s16(a: int16x8_t) -> int16x8_t { + simd_shuffle8!(a, a, [1, 0, 3, 2, 5, 4, 7, 6]) +} + +/// Reversing vector elements (swap endianness) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrev32.16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(rev32))] +pub unsafe fn vrev32_p16(a: poly16x4_t) -> poly16x4_t { + simd_shuffle4!(a, a, [1, 0, 3, 2]) +} + +/// Reversing vector elements (swap endianness) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrev32.16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(rev32))] +pub unsafe fn vrev32q_p16(a: poly16x8_t) -> poly16x8_t { + simd_shuffle8!(a, a, [1, 0, 3, 2, 5, 4, 7, 6]) +} + +/// Reversing vector elements (swap endianness) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrev32.16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(rev32))] +pub unsafe fn vrev32_u16(a: uint16x4_t) -> uint16x4_t { + simd_shuffle4!(a, a, [1, 0, 3, 2]) +} + +/// Reversing vector elements (swap endianness) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrev32.16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(rev32))] +pub unsafe fn vrev32q_u16(a: uint16x8_t) -> uint16x8_t { + simd_shuffle8!(a, a, [1, 0, 3, 2, 5, 4, 7, 6]) +} + +/// Reversing vector elements (swap endianness) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrev32.8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(rev32))] +pub unsafe fn vrev32_p8(a: poly8x8_t) -> poly8x8_t { + simd_shuffle8!(a, a, [3, 2, 1, 0, 7, 6, 5, 4]) +} + +/// Reversing vector elements (swap endianness) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrev32.8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(rev32))] +pub unsafe fn vrev32q_p8(a: poly8x16_t) -> poly8x16_t { + simd_shuffle16!(a, a, [3, 2, 1, 0, 7, 6, 5, 4, 11, 10, 9, 8, 15, 14, 13, 12]) +} + +/// Reversing vector elements (swap endianness) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrev64.8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(rev64))] +pub unsafe fn vrev64_s8(a: int8x8_t) -> int8x8_t { + simd_shuffle8!(a, a, [7, 6, 5, 4, 3, 2, 1, 0]) +} + +/// Reversing vector elements (swap endianness) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrev64.8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(rev64))] +pub unsafe fn vrev64q_s8(a: int8x16_t) -> int8x16_t { + simd_shuffle16!(a, a, [7, 6, 5, 4, 3, 2, 1, 0, 15, 14, 13, 12, 11, 10, 9, 8]) +} + +/// Reversing vector elements (swap endianness) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrev64.16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(rev64))] +pub unsafe fn vrev64_s16(a: int16x4_t) -> int16x4_t { + simd_shuffle4!(a, a, [3, 2, 1, 0]) +} + +/// Reversing vector elements (swap endianness) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrev64.16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(rev64))] +pub unsafe fn vrev64q_s16(a: int16x8_t) -> int16x8_t { + simd_shuffle8!(a, a, [3, 2, 1, 0, 7, 6, 5, 4]) +} + +/// Reversing vector elements (swap endianness) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrev64.32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(rev64))] +pub unsafe fn vrev64_s32(a: int32x2_t) -> int32x2_t { + simd_shuffle2!(a, a, [1, 0]) +} + +/// Reversing vector elements (swap endianness) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrev64.32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(rev64))] +pub unsafe fn vrev64q_s32(a: int32x4_t) -> int32x4_t { + simd_shuffle4!(a, a, [1, 0, 3, 2]) +} + +/// Reversing vector elements (swap endianness) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrev64.8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(rev64))] +pub unsafe fn vrev64_u8(a: uint8x8_t) -> uint8x8_t { + simd_shuffle8!(a, a, [7, 6, 5, 4, 3, 2, 1, 0]) +} + +/// Reversing vector elements (swap endianness) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrev64.8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(rev64))] +pub unsafe fn vrev64q_u8(a: uint8x16_t) -> uint8x16_t { + simd_shuffle16!(a, a, [7, 6, 5, 4, 3, 2, 1, 0, 15, 14, 13, 12, 11, 10, 9, 8]) +} + +/// Reversing vector elements (swap endianness) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrev64.16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(rev64))] +pub unsafe fn vrev64_u16(a: uint16x4_t) -> uint16x4_t { + simd_shuffle4!(a, a, [3, 2, 1, 0]) +} + +/// Reversing vector elements (swap endianness) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrev64.16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(rev64))] +pub unsafe fn vrev64q_u16(a: uint16x8_t) -> uint16x8_t { + simd_shuffle8!(a, a, [3, 2, 1, 0, 7, 6, 5, 4]) +} + +/// Reversing vector elements (swap endianness) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrev64.32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(rev64))] +pub unsafe fn vrev64_u32(a: uint32x2_t) -> uint32x2_t { + simd_shuffle2!(a, a, [1, 0]) +} + +/// Reversing vector elements (swap endianness) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrev64.32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(rev64))] +pub unsafe fn vrev64q_u32(a: uint32x4_t) -> uint32x4_t { + simd_shuffle4!(a, a, [1, 0, 3, 2]) +} + +/// Reversing vector elements (swap endianness) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrev64.32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(rev64))] +pub unsafe fn vrev64_f32(a: float32x2_t) -> float32x2_t { + simd_shuffle2!(a, a, [1, 0]) +} + +/// Reversing vector elements (swap endianness) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrev64.32"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(rev64))] +pub unsafe fn vrev64q_f32(a: float32x4_t) -> float32x4_t { + simd_shuffle4!(a, a, [1, 0, 3, 2]) +} + +/// Reversing vector elements (swap endianness) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrev64.8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(rev64))] +pub unsafe fn vrev64_p8(a: poly8x8_t) -> poly8x8_t { + simd_shuffle8!(a, a, [7, 6, 5, 4, 3, 2, 1, 0]) +} + +/// Reversing vector elements (swap endianness) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrev64.8"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(rev64))] +pub unsafe fn vrev64q_p8(a: poly8x16_t) -> poly8x16_t { + simd_shuffle16!(a, a, [7, 6, 5, 4, 3, 2, 1, 0, 15, 14, 13, 12, 11, 10, 9, 8]) +} + +/// Reversing vector elements (swap endianness) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrev64.16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(rev64))] +pub unsafe fn vrev64_p16(a: poly16x4_t) -> poly16x4_t { + simd_shuffle4!(a, a, [3, 2, 1, 0]) +} + +/// Reversing vector elements (swap endianness) +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vrev64.16"))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(rev64))] +pub unsafe fn vrev64q_p16(a: poly16x8_t) -> poly16x8_t { + simd_shuffle8!(a, a, [3, 2, 1, 0, 7, 6, 5, 4]) +} + +/// Signed Add and Accumulate Long Pairwise. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vpadal.s8))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sadalp))] +pub unsafe fn vpadal_s8(a: int16x4_t, b: int8x8_t) -> int16x4_t { + #[cfg(target_arch = "arm")] + { + crate::core_arch::arm::neon::vpadal_s8_(a, b) + } + #[cfg(target_arch = "aarch64")] + { + simd_add(vpaddl_s8_(b), a) + } +} + +/// Signed Add and Accumulate Long Pairwise. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vpadal.s16))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sadalp))] +pub unsafe fn vpadal_s16(a: int32x2_t, b: int16x4_t) -> int32x2_t { + #[cfg(target_arch = "arm")] + { + crate::core_arch::arm::neon::vpadal_s16_(a, b) + } + #[cfg(target_arch = "aarch64")] + { + simd_add(vpaddl_s16_(b), a) + } +} + +/// Signed Add and Accumulate Long Pairwise. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vpadal.s32))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sadalp))] +pub unsafe fn vpadal_s32(a: int64x1_t, b: int32x2_t) -> int64x1_t { + #[cfg(target_arch = "arm")] + { + crate::core_arch::arm::neon::vpadal_s32_(a, b) + } + #[cfg(target_arch = "aarch64")] + { + simd_add(vpaddl_s32_(b), a) + } +} + +/// Signed Add and Accumulate Long Pairwise. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vpadal.s8))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sadalp))] +pub unsafe fn vpadalq_s8(a: int16x8_t, b: int8x16_t) -> int16x8_t { + #[cfg(target_arch = "arm")] + { + crate::core_arch::arm::neon::vpadalq_s8_(a, b) + } + #[cfg(target_arch = "aarch64")] + { + simd_add(vpaddlq_s8_(b), a) + } +} + +/// Signed Add and Accumulate Long Pairwise. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vpadal.s16))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sadalp))] +pub unsafe fn vpadalq_s16(a: int32x4_t, b: int16x8_t) -> int32x4_t { + #[cfg(target_arch = "arm")] + { + crate::core_arch::arm::neon::vpadalq_s16_(a, b) + } + #[cfg(target_arch = "aarch64")] + { + simd_add(vpaddlq_s16_(b), a) + } +} + +/// Signed Add and Accumulate Long Pairwise. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vpadal.s32))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(sadalp))] +pub unsafe fn vpadalq_s32(a: int64x2_t, b: int32x4_t) -> int64x2_t { + #[cfg(target_arch = "arm")] + { + crate::core_arch::arm::neon::vpadalq_s32_(a, b) + } + #[cfg(target_arch = "aarch64")] + { + simd_add(vpaddlq_s32_(b), a) + } +} + +/// Unsigned Add and Accumulate Long Pairwise. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vpadal.u8))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uadalp))] +pub unsafe fn vpadal_u8(a: uint16x4_t, b: uint8x8_t) -> uint16x4_t { + #[cfg(target_arch = "arm")] + { + crate::core_arch::arm::neon::vpadal_u8_(a, b) + } + #[cfg(target_arch = "aarch64")] + { + simd_add(vpaddl_u8_(b), a) + } +} + +/// Unsigned Add and Accumulate Long Pairwise. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vpadal.u16))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uadalp))] +pub unsafe fn vpadal_u16(a: uint32x2_t, b: uint16x4_t) -> uint32x2_t { + #[cfg(target_arch = "arm")] + { + crate::core_arch::arm::neon::vpadal_u16_(a, b) + } + #[cfg(target_arch = "aarch64")] + { + simd_add(vpaddl_u16_(b), a) + } +} + +/// Unsigned Add and Accumulate Long Pairwise. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vpadal.u32))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uadalp))] +pub unsafe fn vpadal_u32(a: uint64x1_t, b: uint32x2_t) -> uint64x1_t { + #[cfg(target_arch = "arm")] + { + crate::core_arch::arm::neon::vpadal_u32_(a, b) + } + #[cfg(target_arch = "aarch64")] + { + simd_add(vpaddl_u32_(b), a) + } +} + +/// Unsigned Add and Accumulate Long Pairwise. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vpadal.u8))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uadalp))] +pub unsafe fn vpadalq_u8(a: uint16x8_t, b: uint8x16_t) -> uint16x8_t { + #[cfg(target_arch = "arm")] + { + crate::core_arch::arm::neon::vpadalq_u8_(a, b) + } + #[cfg(target_arch = "aarch64")] + { + simd_add(vpaddlq_u8_(b), a) + } +} + +/// Unsigned Add and Accumulate Long Pairwise. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vpadal.u16))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uadalp))] +pub unsafe fn vpadalq_u16(a: uint32x4_t, b: uint16x8_t) -> uint32x4_t { + #[cfg(target_arch = "arm")] + { + crate::core_arch::arm::neon::vpadalq_u16_(a, b) + } + #[cfg(target_arch = "aarch64")] + { + simd_add(vpaddlq_u16_(b), a) + } +} + +/// Unsigned Add and Accumulate Long Pairwise. +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vpadal.u32))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(uadalp))] +pub unsafe fn vpadalq_u32(a: uint64x2_t, b: uint32x4_t) -> uint64x2_t { + #[cfg(target_arch = "arm")] + { + crate::core_arch::arm::neon::vpadalq_u32_(a, b) + } + #[cfg(target_arch = "aarch64")] + { + simd_add(vpaddlq_u32_(b), a) + } +} + +#[cfg(test)] +mod tests { + use super::*; + #[cfg(target_arch = "aarch64")] + use crate::core_arch::aarch64::*; + #[cfg(target_arch = "arm")] + use crate::core_arch::arm::*; + use crate::core_arch::arm_shared::test_support::*; + use crate::core_arch::simd::*; + use std::{i16, i32, i8, mem::transmute, u16, u32, u8, vec::Vec}; + use stdarch_test::simd_test; + + #[simd_test(enable = "neon")] + unsafe fn test_vld1_lane_s8() { + let a = i8x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let elem: i8 = 42; + let e = i8x8::new(0, 1, 2, 3, 4, 5, 6, 42); + let r: i8x8 = transmute(vld1_lane_s8::<7>(&elem, transmute(a))); + assert_eq!(r, e) + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1q_lane_s8() { + let a = i8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let elem: i8 = 42; + let e = i8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 42); + let r: i8x16 = transmute(vld1q_lane_s8::<15>(&elem, transmute(a))); + assert_eq!(r, e) + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1_lane_s16() { + let a = i16x4::new(0, 1, 2, 3); + let elem: i16 = 42; + let e = i16x4::new(0, 1, 2, 42); + let r: i16x4 = transmute(vld1_lane_s16::<3>(&elem, transmute(a))); + assert_eq!(r, e) + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1q_lane_s16() { + let a = i16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let elem: i16 = 42; + let e = i16x8::new(0, 1, 2, 3, 4, 5, 6, 42); + let r: i16x8 = transmute(vld1q_lane_s16::<7>(&elem, transmute(a))); + assert_eq!(r, e) + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1_lane_s32() { + let a = i32x2::new(0, 1); + let elem: i32 = 42; + let e = i32x2::new(0, 42); + let r: i32x2 = transmute(vld1_lane_s32::<1>(&elem, transmute(a))); + assert_eq!(r, e) + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1q_lane_s32() { + let a = i32x4::new(0, 1, 2, 3); + let elem: i32 = 42; + let e = i32x4::new(0, 1, 2, 42); + let r: i32x4 = transmute(vld1q_lane_s32::<3>(&elem, transmute(a))); + assert_eq!(r, e) + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1_lane_s64() { + let a = i64x1::new(0); + let elem: i64 = 42; + let e = i64x1::new(42); + let r: i64x1 = transmute(vld1_lane_s64::<0>(&elem, transmute(a))); + assert_eq!(r, e) + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1q_lane_s64() { + let a = i64x2::new(0, 1); + let elem: i64 = 42; + let e = i64x2::new(0, 42); + let r: i64x2 = transmute(vld1q_lane_s64::<1>(&elem, transmute(a))); + assert_eq!(r, e) + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1_lane_u8() { + let a = u8x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let elem: u8 = 42; + let e = u8x8::new(0, 1, 2, 3, 4, 5, 6, 42); + let r: u8x8 = transmute(vld1_lane_u8::<7>(&elem, transmute(a))); + assert_eq!(r, e) + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1q_lane_u8() { + let a = u8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let elem: u8 = 42; + let e = u8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 42); + let r: u8x16 = transmute(vld1q_lane_u8::<15>(&elem, transmute(a))); + assert_eq!(r, e) + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1_lane_u16() { + let a = u16x4::new(0, 1, 2, 3); + let elem: u16 = 42; + let e = u16x4::new(0, 1, 2, 42); + let r: u16x4 = transmute(vld1_lane_u16::<3>(&elem, transmute(a))); + assert_eq!(r, e) + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1q_lane_u16() { + let a = u16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let elem: u16 = 42; + let e = u16x8::new(0, 1, 2, 3, 4, 5, 6, 42); + let r: u16x8 = transmute(vld1q_lane_u16::<7>(&elem, transmute(a))); + assert_eq!(r, e) + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1_lane_u32() { + let a = u32x2::new(0, 1); + let elem: u32 = 42; + let e = u32x2::new(0, 42); + let r: u32x2 = transmute(vld1_lane_u32::<1>(&elem, transmute(a))); + assert_eq!(r, e) + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1q_lane_u32() { + let a = u32x4::new(0, 1, 2, 3); + let elem: u32 = 42; + let e = u32x4::new(0, 1, 2, 42); + let r: u32x4 = transmute(vld1q_lane_u32::<3>(&elem, transmute(a))); + assert_eq!(r, e) + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1_lane_u64() { + let a = u64x1::new(0); + let elem: u64 = 42; + let e = u64x1::new(42); + let r: u64x1 = transmute(vld1_lane_u64::<0>(&elem, transmute(a))); + assert_eq!(r, e) + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1q_lane_u64() { + let a = u64x2::new(0, 1); + let elem: u64 = 42; + let e = u64x2::new(0, 42); + let r: u64x2 = transmute(vld1q_lane_u64::<1>(&elem, transmute(a))); + assert_eq!(r, e) + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1_lane_p8() { + let a = u8x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let elem: p8 = 42; + let e = u8x8::new(0, 1, 2, 3, 4, 5, 6, 42); + let r: u8x8 = transmute(vld1_lane_p8::<7>(&elem, transmute(a))); + assert_eq!(r, e) + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1q_lane_p8() { + let a = u8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let elem: p8 = 42; + let e = u8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 42); + let r: u8x16 = transmute(vld1q_lane_p8::<15>(&elem, transmute(a))); + assert_eq!(r, e) + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1_lane_p16() { + let a = u16x4::new(0, 1, 2, 3); + let elem: p16 = 42; + let e = u16x4::new(0, 1, 2, 42); + let r: u16x4 = transmute(vld1_lane_p16::<3>(&elem, transmute(a))); + assert_eq!(r, e) + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1q_lane_p16() { + let a = u16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let elem: p16 = 42; + let e = u16x8::new(0, 1, 2, 3, 4, 5, 6, 42); + let r: u16x8 = transmute(vld1q_lane_p16::<7>(&elem, transmute(a))); + assert_eq!(r, e) + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1_lane_f32() { + let a = f32x2::new(0., 1.); + let elem: f32 = 42.; + let e = f32x2::new(0., 42.); + let r: f32x2 = transmute(vld1_lane_f32::<1>(&elem, transmute(a))); + assert_eq!(r, e) + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1q_lane_f32() { + let a = f32x4::new(0., 1., 2., 3.); + let elem: f32 = 42.; + let e = f32x4::new(0., 1., 2., 42.); + let r: f32x4 = transmute(vld1q_lane_f32::<3>(&elem, transmute(a))); + assert_eq!(r, e) + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1_dup_s8() { + let elem: i8 = 42; + let e = i8x8::new(42, 42, 42, 42, 42, 42, 42, 42); + let r: i8x8 = transmute(vld1_dup_s8(&elem)); + assert_eq!(r, e) + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1q_dup_s8() { + let elem: i8 = 42; + let e = i8x16::new( + 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, + ); + let r: i8x16 = transmute(vld1q_dup_s8(&elem)); + assert_eq!(r, e) + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1_dup_s16() { + let elem: i16 = 42; + let e = i16x4::new(42, 42, 42, 42); + let r: i16x4 = transmute(vld1_dup_s16(&elem)); + assert_eq!(r, e) + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1q_dup_s16() { + let elem: i16 = 42; + let e = i16x8::new(42, 42, 42, 42, 42, 42, 42, 42); + let r: i16x8 = transmute(vld1q_dup_s16(&elem)); + assert_eq!(r, e) + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1_dup_s32() { + let elem: i32 = 42; + let e = i32x2::new(42, 42); + let r: i32x2 = transmute(vld1_dup_s32(&elem)); + assert_eq!(r, e) + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1q_dup_s32() { + let elem: i32 = 42; + let e = i32x4::new(42, 42, 42, 42); + let r: i32x4 = transmute(vld1q_dup_s32(&elem)); + assert_eq!(r, e) + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1_dup_s64() { + let elem: i64 = 42; + let e = i64x1::new(42); + let r: i64x1 = transmute(vld1_dup_s64(&elem)); + assert_eq!(r, e) + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1q_dup_s64() { + let elem: i64 = 42; + let e = i64x2::new(42, 42); + let r: i64x2 = transmute(vld1q_dup_s64(&elem)); + assert_eq!(r, e) + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1_dup_u8() { + let elem: u8 = 42; + let e = u8x8::new(42, 42, 42, 42, 42, 42, 42, 42); + let r: u8x8 = transmute(vld1_dup_u8(&elem)); + assert_eq!(r, e) + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1q_dup_u8() { + let elem: u8 = 42; + let e = u8x16::new( + 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, + ); + let r: u8x16 = transmute(vld1q_dup_u8(&elem)); + assert_eq!(r, e) + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1_dup_u16() { + let elem: u16 = 42; + let e = u16x4::new(42, 42, 42, 42); + let r: u16x4 = transmute(vld1_dup_u16(&elem)); + assert_eq!(r, e) + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1q_dup_u16() { + let elem: u16 = 42; + let e = u16x8::new(42, 42, 42, 42, 42, 42, 42, 42); + let r: u16x8 = transmute(vld1q_dup_u16(&elem)); + assert_eq!(r, e) + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1_dup_u32() { + let elem: u32 = 42; + let e = u32x2::new(42, 42); + let r: u32x2 = transmute(vld1_dup_u32(&elem)); + assert_eq!(r, e) + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1q_dup_u32() { + let elem: u32 = 42; + let e = u32x4::new(42, 42, 42, 42); + let r: u32x4 = transmute(vld1q_dup_u32(&elem)); + assert_eq!(r, e) + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1_dup_u64() { + let elem: u64 = 42; + let e = u64x1::new(42); + let r: u64x1 = transmute(vld1_dup_u64(&elem)); + assert_eq!(r, e) + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1q_dup_u64() { + let elem: u64 = 42; + let e = u64x2::new(42, 42); + let r: u64x2 = transmute(vld1q_dup_u64(&elem)); + assert_eq!(r, e) + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1_dup_p8() { + let elem: p8 = 42; + let e = u8x8::new(42, 42, 42, 42, 42, 42, 42, 42); + let r: u8x8 = transmute(vld1_dup_p8(&elem)); + assert_eq!(r, e) + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1q_dup_p8() { + let elem: p8 = 42; + let e = u8x16::new( + 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, + ); + let r: u8x16 = transmute(vld1q_dup_p8(&elem)); + assert_eq!(r, e) + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1_dup_p16() { + let elem: p16 = 42; + let e = u16x4::new(42, 42, 42, 42); + let r: u16x4 = transmute(vld1_dup_p16(&elem)); + assert_eq!(r, e) + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1q_dup_p16() { + let elem: p16 = 42; + let e = u16x8::new(42, 42, 42, 42, 42, 42, 42, 42); + let r: u16x8 = transmute(vld1q_dup_p16(&elem)); + assert_eq!(r, e) + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1_dup_f32() { + let elem: f32 = 42.; + let e = f32x2::new(42., 42.); + let r: f32x2 = transmute(vld1_dup_f32(&elem)); + assert_eq!(r, e) + } + + #[simd_test(enable = "neon")] + unsafe fn test_vld1q_dup_f32() { + let elem: f32 = 42.; + let e = f32x4::new(42., 42., 42., 42.); + let r: f32x4 = transmute(vld1q_dup_f32(&elem)); + assert_eq!(r, e) + } + + #[simd_test(enable = "neon")] + unsafe fn test_vget_lane_u8() { + let v = i8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let r = vget_lane_u8::<1>(transmute(v)); + assert_eq!(r, 2); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vgetq_lane_u32() { + let v = i32x4::new(1, 2, 3, 4); + let r = vgetq_lane_u32::<1>(transmute(v)); + assert_eq!(r, 2); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vgetq_lane_s32() { + let v = i32x4::new(1, 2, 3, 4); + let r = vgetq_lane_s32::<1>(transmute(v)); + assert_eq!(r, 2); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vget_lane_u64() { + let v: u64 = 1; + let r = vget_lane_u64::<0>(transmute(v)); + assert_eq!(r, 1); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vgetq_lane_u16() { + let v = i16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let r = vgetq_lane_u16::<1>(transmute(v)); + assert_eq!(r, 2); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vget_lane_s8() { + let v = i8x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let r = vget_lane_s8::<2>(transmute(v)); + assert_eq!(r, 2); + let r = vget_lane_s8::<4>(transmute(v)); + assert_eq!(r, 4); + let r = vget_lane_s8::<5>(transmute(v)); + assert_eq!(r, 5); + } + #[simd_test(enable = "neon")] + unsafe fn test_vget_lane_p8() { + let v = u8x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let r = vget_lane_p8::<2>(transmute(v)); + assert_eq!(r, 2); + let r = vget_lane_p8::<3>(transmute(v)); + assert_eq!(r, 3); + let r = vget_lane_p8::<5>(transmute(v)); + assert_eq!(r, 5); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vget_lane_p16() { + let v = u16x4::new(0, 1, 2, 3); + let r = vget_lane_p16::<2>(transmute(v)); + assert_eq!(r, 2); + let r = vget_lane_p16::<3>(transmute(v)); + assert_eq!(r, 3); + let r = vget_lane_p16::<0>(transmute(v)); + assert_eq!(r, 0); + let r = vget_lane_p16::<1>(transmute(v)); + assert_eq!(r, 1); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vget_lane_s16() { + let v = i16x4::new(0, 1, 2, 3); + let r = vget_lane_s16::<2>(transmute(v)); + assert_eq!(r, 2); + let r = vget_lane_s16::<3>(transmute(v)); + assert_eq!(r, 3); + let r = vget_lane_s16::<0>(transmute(v)); + assert_eq!(r, 0); + let r = vget_lane_s16::<1>(transmute(v)); + assert_eq!(r, 1); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vget_lane_u16() { + let v = u16x4::new(0, 1, 2, 3); + let r = vget_lane_u16::<2>(transmute(v)); + assert_eq!(r, 2); + let r = vget_lane_u16::<3>(transmute(v)); + assert_eq!(r, 3); + let r = vget_lane_u16::<0>(transmute(v)); + assert_eq!(r, 0); + let r = vget_lane_u16::<1>(transmute(v)); + assert_eq!(r, 1); + } + #[simd_test(enable = "neon")] + unsafe fn test_vget_lane_f32() { + let v = f32x2::new(0.0, 1.0); + let r = vget_lane_f32::<1>(transmute(v)); + assert_eq!(r, 1.0); + let r = vget_lane_f32::<0>(transmute(v)); + assert_eq!(r, 0.0); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vget_lane_s32() { + let v = i32x2::new(0, 1); + let r = vget_lane_s32::<1>(transmute(v)); + assert_eq!(r, 1); + let r = vget_lane_s32::<0>(transmute(v)); + assert_eq!(r, 0); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vget_lane_u32() { + let v = u32x2::new(0, 1); + let r = vget_lane_u32::<1>(transmute(v)); + assert_eq!(r, 1); + let r = vget_lane_u32::<0>(transmute(v)); + assert_eq!(r, 0); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vget_lane_s64() { + let v = i64x1::new(1); + let r = vget_lane_s64::<0>(transmute(v)); + assert_eq!(r, 1); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vget_lane_p64() { + let v = u64x1::new(1); + let r = vget_lane_p64::<0>(transmute(v)); + assert_eq!(r, 1); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vgetq_lane_s8() { + let v = i8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let r = vgetq_lane_s8::<7>(transmute(v)); + assert_eq!(r, 7); + let r = vgetq_lane_s8::<13>(transmute(v)); + assert_eq!(r, 13); + let r = vgetq_lane_s8::<3>(transmute(v)); + assert_eq!(r, 3); + let r = vgetq_lane_s8::<0>(transmute(v)); + assert_eq!(r, 0); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vgetq_lane_p8() { + let v = u8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let r = vgetq_lane_p8::<7>(transmute(v)); + assert_eq!(r, 7); + let r = vgetq_lane_p8::<13>(transmute(v)); + assert_eq!(r, 13); + let r = vgetq_lane_p8::<3>(transmute(v)); + assert_eq!(r, 3); + let r = vgetq_lane_p8::<0>(transmute(v)); + assert_eq!(r, 0); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vgetq_lane_u8() { + let v = u8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let r = vgetq_lane_u8::<7>(transmute(v)); + assert_eq!(r, 7); + let r = vgetq_lane_u8::<13>(transmute(v)); + assert_eq!(r, 13); + let r = vgetq_lane_u8::<3>(transmute(v)); + assert_eq!(r, 3); + let r = vgetq_lane_u8::<0>(transmute(v)); + assert_eq!(r, 0); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vgetq_lane_s16() { + let v = i16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let r = vgetq_lane_s16::<3>(transmute(v)); + assert_eq!(r, 3); + let r = vgetq_lane_s16::<6>(transmute(v)); + assert_eq!(r, 6); + let r = vgetq_lane_s16::<0>(transmute(v)); + assert_eq!(r, 0); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vgetq_lane_p16() { + let v = u16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let r = vgetq_lane_p16::<3>(transmute(v)); + assert_eq!(r, 3); + let r = vgetq_lane_p16::<7>(transmute(v)); + assert_eq!(r, 7); + let r = vgetq_lane_p16::<1>(transmute(v)); + assert_eq!(r, 1); + } + #[simd_test(enable = "neon")] + unsafe fn test_vgetq_lane_f32() { + let v = f32x4::new(0.0, 1.0, 2.0, 3.0); + let r = vgetq_lane_f32::<3>(transmute(v)); + assert_eq!(r, 3.0); + let r = vgetq_lane_f32::<0>(transmute(v)); + assert_eq!(r, 0.0); + let r = vgetq_lane_f32::<2>(transmute(v)); + assert_eq!(r, 2.0); + let r = vgetq_lane_f32::<1>(transmute(v)); + assert_eq!(r, 1.0); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vgetq_lane_s64() { + let v = i64x2::new(0, 1); + let r = vgetq_lane_s64::<1>(transmute(v)); + assert_eq!(r, 1); + let r = vgetq_lane_s64::<0>(transmute(v)); + assert_eq!(r, 0); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vgetq_lane_p64() { + let v = u64x2::new(0, 1); + let r = vgetq_lane_p64::<1>(transmute(v)); + assert_eq!(r, 1); + let r = vgetq_lane_p64::<0>(transmute(v)); + assert_eq!(r, 0); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vext_s64() { + let a: i64x1 = i64x1::new(0); + let b: i64x1 = i64x1::new(1); + let e: i64x1 = i64x1::new(0); + let r: i64x1 = transmute(vext_s64::<0>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vext_u64() { + let a: u64x1 = u64x1::new(0); + let b: u64x1 = u64x1::new(1); + let e: u64x1 = u64x1::new(0); + let r: u64x1 = transmute(vext_u64::<0>(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vget_high_s8() { + let a = i8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let e = i8x8::new(9, 10, 11, 12, 13, 14, 15, 16); + let r: i8x8 = transmute(vget_high_s8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vget_high_s16() { + let a = i16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let e = i16x4::new(5, 6, 7, 8); + let r: i16x4 = transmute(vget_high_s16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vget_high_s32() { + let a = i32x4::new(1, 2, 3, 4); + let e = i32x2::new(3, 4); + let r: i32x2 = transmute(vget_high_s32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vget_high_s64() { + let a = i64x2::new(1, 2); + let e = i64x1::new(2); + let r: i64x1 = transmute(vget_high_s64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vget_high_u8() { + let a = u8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let e = u8x8::new(9, 10, 11, 12, 13, 14, 15, 16); + let r: u8x8 = transmute(vget_high_u8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vget_high_u16() { + let a = u16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let e = u16x4::new(5, 6, 7, 8); + let r: u16x4 = transmute(vget_high_u16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vget_high_u32() { + let a = u32x4::new(1, 2, 3, 4); + let e = u32x2::new(3, 4); + let r: u32x2 = transmute(vget_high_u32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vget_high_u64() { + let a = u64x2::new(1, 2); + let e = u64x1::new(2); + let r: u64x1 = transmute(vget_high_u64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vget_high_p8() { + let a = u8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let e = u8x8::new(9, 10, 11, 12, 13, 14, 15, 16); + let r: u8x8 = transmute(vget_high_p8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vget_high_p16() { + let a = u16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let e = u16x4::new(5, 6, 7, 8); + let r: u16x4 = transmute(vget_high_p16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vget_high_f32() { + let a = f32x4::new(1.0, 2.0, 3.0, 4.0); + let e = f32x2::new(3.0, 4.0); + let r: f32x2 = transmute(vget_high_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vget_low_s8() { + let a = i8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let e = i8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let r: i8x8 = transmute(vget_low_s8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vget_low_s16() { + let a = i16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let e = i16x4::new(1, 2, 3, 4); + let r: i16x4 = transmute(vget_low_s16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vget_low_s32() { + let a = i32x4::new(1, 2, 3, 4); + let e = i32x2::new(1, 2); + let r: i32x2 = transmute(vget_low_s32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vget_low_s64() { + let a = i64x2::new(1, 2); + let e = i64x1::new(1); + let r: i64x1 = transmute(vget_low_s64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vget_low_u8() { + let a = u8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let e = u8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let r: u8x8 = transmute(vget_low_u8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vget_low_u16() { + let a = u16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let e = u16x4::new(1, 2, 3, 4); + let r: u16x4 = transmute(vget_low_u16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vget_low_u32() { + let a = u32x4::new(1, 2, 3, 4); + let e = u32x2::new(1, 2); + let r: u32x2 = transmute(vget_low_u32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vget_low_u64() { + let a = u64x2::new(1, 2); + let e = u64x1::new(1); + let r: u64x1 = transmute(vget_low_u64(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vget_low_p8() { + let a = u8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let e = u8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let r: u8x8 = transmute(vget_low_p8(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vget_low_p16() { + let a = u16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let e = u16x4::new(1, 2, 3, 4); + let r: u16x4 = transmute(vget_low_p16(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vget_low_f32() { + let a = f32x4::new(1.0, 2.0, 3.0, 4.0); + let e = f32x2::new(1.0, 2.0); + let r: f32x2 = transmute(vget_low_f32(transmute(a))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdupq_n_s8() { + let v: i8 = 42; + let e = i8x16::new( + 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, + ); + let r: i8x16 = transmute(vdupq_n_s8(v)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdupq_n_s16() { + let v: i16 = 64; + let e = i16x8::new(64, 64, 64, 64, 64, 64, 64, 64); + let r: i16x8 = transmute(vdupq_n_s16(v)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdupq_n_s32() { + let v: i32 = 64; + let e = i32x4::new(64, 64, 64, 64); + let r: i32x4 = transmute(vdupq_n_s32(v)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdupq_n_s64() { + let v: i64 = 64; + let e = i64x2::new(64, 64); + let r: i64x2 = transmute(vdupq_n_s64(v)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdupq_n_u8() { + let v: u8 = 64; + let e = u8x16::new( + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + ); + let r: u8x16 = transmute(vdupq_n_u8(v)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdupq_n_u16() { + let v: u16 = 64; + let e = u16x8::new(64, 64, 64, 64, 64, 64, 64, 64); + let r: u16x8 = transmute(vdupq_n_u16(v)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdupq_n_u32() { + let v: u32 = 64; + let e = u32x4::new(64, 64, 64, 64); + let r: u32x4 = transmute(vdupq_n_u32(v)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdupq_n_u64() { + let v: u64 = 64; + let e = u64x2::new(64, 64); + let r: u64x2 = transmute(vdupq_n_u64(v)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdupq_n_p8() { + let v: p8 = 64; + let e = u8x16::new( + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + ); + let r: u8x16 = transmute(vdupq_n_p8(v)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdupq_n_p16() { + let v: p16 = 64; + let e = u16x8::new(64, 64, 64, 64, 64, 64, 64, 64); + let r: u16x8 = transmute(vdupq_n_p16(v)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdupq_n_f32() { + let v: f32 = 64.0; + let e = f32x4::new(64.0, 64.0, 64.0, 64.0); + let r: f32x4 = transmute(vdupq_n_f32(v)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdup_n_s8() { + let v: i8 = 64; + let e = i8x8::new(64, 64, 64, 64, 64, 64, 64, 64); + let r: i8x8 = transmute(vdup_n_s8(v)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdup_n_s16() { + let v: i16 = 64; + let e = i16x4::new(64, 64, 64, 64); + let r: i16x4 = transmute(vdup_n_s16(v)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdup_n_s32() { + let v: i32 = 64; + let e = i32x2::new(64, 64); + let r: i32x2 = transmute(vdup_n_s32(v)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdup_n_s64() { + let v: i64 = 64; + let e = i64x1::new(64); + let r: i64x1 = transmute(vdup_n_s64(v)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdup_n_u8() { + let v: u8 = 64; + let e = u8x8::new(64, 64, 64, 64, 64, 64, 64, 64); + let r: u8x8 = transmute(vdup_n_u8(v)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdup_n_u16() { + let v: u16 = 64; + let e = u16x4::new(64, 64, 64, 64); + let r: u16x4 = transmute(vdup_n_u16(v)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdup_n_u32() { + let v: u32 = 64; + let e = u32x2::new(64, 64); + let r: u32x2 = transmute(vdup_n_u32(v)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdup_n_u64() { + let v: u64 = 64; + let e = u64x1::new(64); + let r: u64x1 = transmute(vdup_n_u64(v)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdup_n_p8() { + let v: p8 = 64; + let e = u8x8::new(64, 64, 64, 64, 64, 64, 64, 64); + let r: u8x8 = transmute(vdup_n_p8(v)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdup_n_p16() { + let v: p16 = 64; + let e = u16x4::new(64, 64, 64, 64); + let r: u16x4 = transmute(vdup_n_p16(v)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vdup_n_f32() { + let v: f32 = 64.0; + let e = f32x2::new(64.0, 64.0); + let r: f32x2 = transmute(vdup_n_f32(v)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmov_n_s8() { + let v: i8 = 64; + let e = i8x8::new(64, 64, 64, 64, 64, 64, 64, 64); + let r: i8x8 = transmute(vmov_n_s8(v)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmov_n_s16() { + let v: i16 = 64; + let e = i16x4::new(64, 64, 64, 64); + let r: i16x4 = transmute(vmov_n_s16(v)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmov_n_s32() { + let v: i32 = 64; + let e = i32x2::new(64, 64); + let r: i32x2 = transmute(vmov_n_s32(v)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmov_n_s64() { + let v: i64 = 64; + let e = i64x1::new(64); + let r: i64x1 = transmute(vmov_n_s64(v)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmov_n_u8() { + let v: u8 = 64; + let e = u8x8::new(64, 64, 64, 64, 64, 64, 64, 64); + let r: u8x8 = transmute(vmov_n_u8(v)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmov_n_u16() { + let v: u16 = 64; + let e = u16x4::new(64, 64, 64, 64); + let r: u16x4 = transmute(vmov_n_u16(v)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmov_n_u32() { + let v: u32 = 64; + let e = u32x2::new(64, 64); + let r: u32x2 = transmute(vmov_n_u32(v)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmov_n_u64() { + let v: u64 = 64; + let e = u64x1::new(64); + let r: u64x1 = transmute(vmov_n_u64(v)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmov_n_p8() { + let v: p8 = 64; + let e = u8x8::new(64, 64, 64, 64, 64, 64, 64, 64); + let r: u8x8 = transmute(vmov_n_p8(v)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmov_n_p16() { + let v: p16 = 64; + let e = u16x4::new(64, 64, 64, 64); + let r: u16x4 = transmute(vmov_n_p16(v)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmov_n_f32() { + let v: f32 = 64.0; + let e = f32x2::new(64.0, 64.0); + let r: f32x2 = transmute(vmov_n_f32(v)); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vmovq_n_s8() { + let v: i8 = 64; + let e = i8x16::new( + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + ); + let r: i8x16 = transmute(vmovq_n_s8(v)); + assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vld1_lane_s64() { - let a = i64x1::new(0); - let elem: i64 = 42; - let e = i64x1::new(42); - let r: i64x1 = transmute(vld1_lane_s64(&elem, transmute(a), 0)); - assert_eq!(r, e) + unsafe fn test_vmovq_n_s16() { + let v: i16 = 64; + let e = i16x8::new(64, 64, 64, 64, 64, 64, 64, 64); + let r: i16x8 = transmute(vmovq_n_s16(v)); + assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vld1q_lane_s64() { - let a = i64x2::new(0, 1); - let elem: i64 = 42; - let e = i64x2::new(0, 42); - let r: i64x2 = transmute(vld1q_lane_s64(&elem, transmute(a), 1)); - assert_eq!(r, e) + unsafe fn test_vmovq_n_s32() { + let v: i32 = 64; + let e = i32x4::new(64, 64, 64, 64); + let r: i32x4 = transmute(vmovq_n_s32(v)); + assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vld1_lane_u8() { - let a = u8x8::new(0, 1, 2, 3, 4, 5, 6, 7); - let elem: u8 = 42; - let e = u8x8::new(0, 1, 2, 3, 4, 5, 6, 42); - let r: u8x8 = transmute(vld1_lane_u8(&elem, transmute(a), 7)); - assert_eq!(r, e) + unsafe fn test_vmovq_n_s64() { + let v: i64 = 64; + let e = i64x2::new(64, 64); + let r: i64x2 = transmute(vmovq_n_s64(v)); + assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vld1q_lane_u8() { - let a = u8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); - let elem: u8 = 42; - let e = u8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 42); - let r: u8x16 = transmute(vld1q_lane_u8(&elem, transmute(a), 15)); - assert_eq!(r, e) + unsafe fn test_vmovq_n_u8() { + let v: u8 = 64; + let e = u8x16::new( + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + ); + let r: u8x16 = transmute(vmovq_n_u8(v)); + assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vld1_lane_u16() { - let a = u16x4::new(0, 1, 2, 3); - let elem: u16 = 42; - let e = u16x4::new(0, 1, 2, 42); - let r: u16x4 = transmute(vld1_lane_u16(&elem, transmute(a), 3)); - assert_eq!(r, e) + unsafe fn test_vmovq_n_u16() { + let v: u16 = 64; + let e = u16x8::new(64, 64, 64, 64, 64, 64, 64, 64); + let r: u16x8 = transmute(vmovq_n_u16(v)); + assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vld1q_lane_u16() { - let a = u16x8::new(0, 1, 2, 3, 4, 5, 6, 7); - let elem: u16 = 42; - let e = u16x8::new(0, 1, 2, 3, 4, 5, 6, 42); - let r: u16x8 = transmute(vld1q_lane_u16(&elem, transmute(a), 7)); - assert_eq!(r, e) + unsafe fn test_vmovq_n_u32() { + let v: u32 = 64; + let e = u32x4::new(64, 64, 64, 64); + let r: u32x4 = transmute(vmovq_n_u32(v)); + assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vld1_lane_u32() { - let a = u32x2::new(0, 1); - let elem: u32 = 42; - let e = u32x2::new(0, 42); - let r: u32x2 = transmute(vld1_lane_u32(&elem, transmute(a), 1)); - assert_eq!(r, e) + unsafe fn test_vmovq_n_u64() { + let v: u64 = 64; + let e = u64x2::new(64, 64); + let r: u64x2 = transmute(vmovq_n_u64(v)); + assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vld1q_lane_u32() { - let a = u32x4::new(0, 1, 2, 3); - let elem: u32 = 42; - let e = u32x4::new(0, 1, 2, 42); - let r: u32x4 = transmute(vld1q_lane_u32(&elem, transmute(a), 3)); - assert_eq!(r, e) + unsafe fn test_vmovq_n_p8() { + let v: p8 = 64; + let e = u8x16::new( + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + ); + let r: u8x16 = transmute(vmovq_n_p8(v)); + assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vld1_lane_u64() { - let a = u64x1::new(0); - let elem: u64 = 42; - let e = u64x1::new(42); - let r: u64x1 = transmute(vld1_lane_u64(&elem, transmute(a), 0)); - assert_eq!(r, e) + unsafe fn test_vmovq_n_p16() { + let v: p16 = 64; + let e = u16x8::new(64, 64, 64, 64, 64, 64, 64, 64); + let r: u16x8 = transmute(vmovq_n_p16(v)); + assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vld1q_lane_u64() { - let a = u64x2::new(0, 1); - let elem: u64 = 42; - let e = u64x2::new(0, 42); - let r: u64x2 = transmute(vld1q_lane_u64(&elem, transmute(a), 1)); - assert_eq!(r, e) + unsafe fn test_vmovq_n_f32() { + let v: f32 = 64.0; + let e = f32x4::new(64.0, 64.0, 64.0, 64.0); + let r: f32x4 = transmute(vmovq_n_f32(v)); + assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vld1_lane_p8() { - let a = u8x8::new(0, 1, 2, 3, 4, 5, 6, 7); - let elem: p8 = 42; - let e = u8x8::new(0, 1, 2, 3, 4, 5, 6, 42); - let r: u8x8 = transmute(vld1_lane_p8(&elem, transmute(a), 7)); - assert_eq!(r, e) + unsafe fn test_vgetq_lane_u64() { + let v = i64x2::new(1, 2); + let r = vgetq_lane_u64::<1>(transmute(v)); + assert_eq!(r, 2); } #[simd_test(enable = "neon")] - unsafe fn test_vld1q_lane_p8() { - let a = u8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); - let elem: p8 = 42; - let e = u8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 42); - let r: u8x16 = transmute(vld1q_lane_p8(&elem, transmute(a), 15)); - assert_eq!(r, e) + unsafe fn test_vadd_s8() { + test_ari_s8( + |i, j| vadd_s8(i, j), + |a: i8, b: i8| -> i8 { a.overflowing_add(b).0 }, + ); } - #[simd_test(enable = "neon")] - unsafe fn test_vld1_lane_p16() { - let a = u16x4::new(0, 1, 2, 3); - let elem: p16 = 42; - let e = u16x4::new(0, 1, 2, 42); - let r: u16x4 = transmute(vld1_lane_p16(&elem, transmute(a), 3)); - assert_eq!(r, e) + unsafe fn test_vaddq_s8() { + testq_ari_s8( + |i, j| vaddq_s8(i, j), + |a: i8, b: i8| -> i8 { a.overflowing_add(b).0 }, + ); } - #[simd_test(enable = "neon")] - unsafe fn test_vld1q_lane_p16() { - let a = u16x8::new(0, 1, 2, 3, 4, 5, 6, 7); - let elem: p16 = 42; - let e = u16x8::new(0, 1, 2, 3, 4, 5, 6, 42); - let r: u16x8 = transmute(vld1q_lane_p16(&elem, transmute(a), 7)); - assert_eq!(r, e) + unsafe fn test_vadd_s16() { + test_ari_s16( + |i, j| vadd_s16(i, j), + |a: i16, b: i16| -> i16 { a.overflowing_add(b).0 }, + ); } - #[simd_test(enable = "neon")] - unsafe fn test_vld1_lane_f32() { - let a = f32x2::new(0., 1.); - let elem: f32 = 42.; - let e = f32x2::new(0., 42.); - let r: f32x2 = transmute(vld1_lane_f32(&elem, transmute(a), 1)); - assert_eq!(r, e) + unsafe fn test_vaddq_s16() { + testq_ari_s16( + |i, j| vaddq_s16(i, j), + |a: i16, b: i16| -> i16 { a.overflowing_add(b).0 }, + ); } - #[simd_test(enable = "neon")] - unsafe fn test_vld1q_lane_f32() { - let a = f32x4::new(0., 1., 2., 3.); - let elem: f32 = 42.; - let e = f32x4::new(0., 1., 2., 42.); - let r: f32x4 = transmute(vld1q_lane_f32(&elem, transmute(a), 3)); - assert_eq!(r, e) + unsafe fn test_vadd_s32() { + test_ari_s32( + |i, j| vadd_s32(i, j), + |a: i32, b: i32| -> i32 { a.overflowing_add(b).0 }, + ); } - #[simd_test(enable = "neon")] - unsafe fn test_vld1_dup_s8() { - let elem: i8 = 42; - let e = i8x8::new(42, 42, 42, 42, 42, 42, 42, 42); - let r: i8x8 = transmute(vld1_dup_s8(&elem)); - assert_eq!(r, e) + unsafe fn test_vaddq_s32() { + testq_ari_s32( + |i, j| vaddq_s32(i, j), + |a: i32, b: i32| -> i32 { a.overflowing_add(b).0 }, + ); } #[simd_test(enable = "neon")] - unsafe fn test_vld1q_dup_s8() { - let elem: i8 = 42; - let e = i8x16::new( - 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, + unsafe fn test_vadd_u8() { + test_ari_u8( + |i, j| vadd_u8(i, j), + |a: u8, b: u8| -> u8 { a.overflowing_add(b).0 }, + ); + } + #[simd_test(enable = "neon")] + unsafe fn test_vaddq_u8() { + testq_ari_u8( + |i, j| vaddq_u8(i, j), + |a: u8, b: u8| -> u8 { a.overflowing_add(b).0 }, + ); + } + #[simd_test(enable = "neon")] + unsafe fn test_vadd_u16() { + test_ari_u16( + |i, j| vadd_u16(i, j), + |a: u16, b: u16| -> u16 { a.overflowing_add(b).0 }, + ); + } + #[simd_test(enable = "neon")] + unsafe fn test_vaddq_u16() { + testq_ari_u16( + |i, j| vaddq_u16(i, j), + |a: u16, b: u16| -> u16 { a.overflowing_add(b).0 }, + ); + } + #[simd_test(enable = "neon")] + unsafe fn test_vadd_u32() { + test_ari_u32( + |i, j| vadd_u32(i, j), + |a: u32, b: u32| -> u32 { a.overflowing_add(b).0 }, + ); + } + #[simd_test(enable = "neon")] + unsafe fn test_vaddq_u32() { + testq_ari_u32( + |i, j| vaddq_u32(i, j), + |a: u32, b: u32| -> u32 { a.overflowing_add(b).0 }, ); - let r: i8x16 = transmute(vld1q_dup_s8(&elem)); - assert_eq!(r, e) } #[simd_test(enable = "neon")] - unsafe fn test_vld1_dup_s16() { - let elem: i16 = 42; - let e = i16x4::new(42, 42, 42, 42); - let r: i16x4 = transmute(vld1_dup_s16(&elem)); - assert_eq!(r, e) + unsafe fn test_vadd_f32() { + test_ari_f32(|i, j| vadd_f32(i, j), |a: f32, b: f32| -> f32 { a + b }); + } + #[simd_test(enable = "neon")] + unsafe fn test_vaddq_f32() { + testq_ari_f32(|i, j| vaddq_f32(i, j), |a: f32, b: f32| -> f32 { a + b }); } #[simd_test(enable = "neon")] - unsafe fn test_vld1q_dup_s16() { - let elem: i16 = 42; - let e = i16x8::new(42, 42, 42, 42, 42, 42, 42, 42); - let r: i16x8 = transmute(vld1q_dup_s16(&elem)); - assert_eq!(r, e) + unsafe fn test_vaddl_s8() { + let v = i8::MAX; + let a = i8x8::new(v, v, v, v, v, v, v, v); + let v = 2 * (v as i16); + let e = i16x8::new(v, v, v, v, v, v, v, v); + let r: i16x8 = transmute(vaddl_s8(transmute(a), transmute(a))); + assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vld1_dup_s32() { - let elem: i32 = 42; - let e = i32x2::new(42, 42); - let r: i32x2 = transmute(vld1_dup_s32(&elem)); - assert_eq!(r, e) + unsafe fn test_vaddl_s16() { + let v = i16::MAX; + let a = i16x4::new(v, v, v, v); + let v = 2 * (v as i32); + let e = i32x4::new(v, v, v, v); + let r: i32x4 = transmute(vaddl_s16(transmute(a), transmute(a))); + assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vld1q_dup_s32() { - let elem: i32 = 42; - let e = i32x4::new(42, 42, 42, 42); - let r: i32x4 = transmute(vld1q_dup_s32(&elem)); - assert_eq!(r, e) + unsafe fn test_vaddl_s32() { + let v = i32::MAX; + let a = i32x2::new(v, v); + let v = 2 * (v as i64); + let e = i64x2::new(v, v); + let r: i64x2 = transmute(vaddl_s32(transmute(a), transmute(a))); + assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vld1_dup_s64() { - let elem: i64 = 42; - let e = i64x1::new(42); - let r: i64x1 = transmute(vld1_dup_s64(&elem)); - assert_eq!(r, e) + unsafe fn test_vaddl_u8() { + let v = u8::MAX; + let a = u8x8::new(v, v, v, v, v, v, v, v); + let v = 2 * (v as u16); + let e = u16x8::new(v, v, v, v, v, v, v, v); + let r: u16x8 = transmute(vaddl_u8(transmute(a), transmute(a))); + assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vld1q_dup_s64() { - let elem: i64 = 42; - let e = i64x2::new(42, 42); - let r: i64x2 = transmute(vld1q_dup_s64(&elem)); - assert_eq!(r, e) + unsafe fn test_vaddl_u16() { + let v = u16::MAX; + let a = u16x4::new(v, v, v, v); + let v = 2 * (v as u32); + let e = u32x4::new(v, v, v, v); + let r: u32x4 = transmute(vaddl_u16(transmute(a), transmute(a))); + assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vld1_dup_u8() { - let elem: u8 = 42; - let e = u8x8::new(42, 42, 42, 42, 42, 42, 42, 42); - let r: u8x8 = transmute(vld1_dup_u8(&elem)); - assert_eq!(r, e) + unsafe fn test_vaddl_u32() { + let v = u32::MAX; + let a = u32x2::new(v, v); + let v = 2 * (v as u64); + let e = u64x2::new(v, v); + let r: u64x2 = transmute(vaddl_u32(transmute(a), transmute(a))); + assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vld1q_dup_u8() { - let elem: u8 = 42; - let e = u8x16::new( - 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, - ); - let r: u8x16 = transmute(vld1q_dup_u8(&elem)); - assert_eq!(r, e) + unsafe fn test_vaddl_high_s8() { + let a = i8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let x = i8::MAX; + let b = i8x16::new(x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x); + let x = x as i16; + let e = i16x8::new(x + 8, x + 9, x + 10, x + 11, x + 12, x + 13, x + 14, x + 15); + let r: i16x8 = transmute(vaddl_high_s8(transmute(a), transmute(b))); + assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vld1_dup_u16() { - let elem: u16 = 42; - let e = u16x4::new(42, 42, 42, 42); - let r: u16x4 = transmute(vld1_dup_u16(&elem)); - assert_eq!(r, e) + unsafe fn test_vaddl_high_s16() { + let a = i16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let x = i16::MAX; + let b = i16x8::new(x, x, x, x, x, x, x, x); + let x = x as i32; + let e = i32x4::new(x + 4, x + 5, x + 6, x + 7); + let r: i32x4 = transmute(vaddl_high_s16(transmute(a), transmute(b))); + assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vld1q_dup_u16() { - let elem: u16 = 42; - let e = u16x8::new(42, 42, 42, 42, 42, 42, 42, 42); - let r: u16x8 = transmute(vld1q_dup_u16(&elem)); - assert_eq!(r, e) + unsafe fn test_vaddl_high_s32() { + let a = i32x4::new(0, 1, 2, 3); + let x = i32::MAX; + let b = i32x4::new(x, x, x, x); + let x = x as i64; + let e = i64x2::new(x + 2, x + 3); + let r: i64x2 = transmute(vaddl_high_s32(transmute(a), transmute(b))); + assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vld1_dup_u32() { - let elem: u32 = 42; - let e = u32x2::new(42, 42); - let r: u32x2 = transmute(vld1_dup_u32(&elem)); - assert_eq!(r, e) + unsafe fn test_vaddl_high_u8() { + let a = u8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let x = u8::MAX; + let b = u8x16::new(x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x); + let x = x as u16; + let e = u16x8::new(x + 8, x + 9, x + 10, x + 11, x + 12, x + 13, x + 14, x + 15); + let r: u16x8 = transmute(vaddl_high_u8(transmute(a), transmute(b))); + assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vld1q_dup_u32() { - let elem: u32 = 42; - let e = u32x4::new(42, 42, 42, 42); - let r: u32x4 = transmute(vld1q_dup_u32(&elem)); - assert_eq!(r, e) + unsafe fn test_vaddl_high_u16() { + let a = u16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let x = u16::MAX; + let b = u16x8::new(x, x, x, x, x, x, x, x); + let x = x as u32; + let e = u32x4::new(x + 4, x + 5, x + 6, x + 7); + let r: u32x4 = transmute(vaddl_high_u16(transmute(a), transmute(b))); + assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vld1_dup_u64() { - let elem: u64 = 42; - let e = u64x1::new(42); - let r: u64x1 = transmute(vld1_dup_u64(&elem)); - assert_eq!(r, e) + unsafe fn test_vaddl_high_u32() { + let a = u32x4::new(0, 1, 2, 3); + let x = u32::MAX; + let b = u32x4::new(x, x, x, x); + let x = x as u64; + let e = u64x2::new(x + 2, x + 3); + let r: u64x2 = transmute(vaddl_high_u32(transmute(a), transmute(b))); + assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vld1q_dup_u64() { - let elem: u64 = 42; - let e = u64x2::new(42, 42); - let r: u64x2 = transmute(vld1q_dup_u64(&elem)); - assert_eq!(r, e) + unsafe fn test_vaddw_s8() { + let x = i16::MAX; + let a = i16x8::new(x, 1, 2, 3, 4, 5, 6, 7); + let y = i8::MAX; + let b = i8x8::new(y, y, y, y, y, y, y, y); + let y = y as i16; + let e = i16x8::new( + x.wrapping_add(y), + 1 + y, + 2 + y, + 3 + y, + 4 + y, + 5 + y, + 6 + y, + 7 + y, + ); + let r: i16x8 = transmute(vaddw_s8(transmute(a), transmute(b))); + assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vld1_dup_p8() { - let elem: p8 = 42; - let e = u8x8::new(42, 42, 42, 42, 42, 42, 42, 42); - let r: u8x8 = transmute(vld1_dup_p8(&elem)); - assert_eq!(r, e) + unsafe fn test_vaddw_s16() { + let x = i32::MAX; + let a = i32x4::new(x, 1, 2, 3); + let y = i16::MAX; + let b = i16x4::new(y, y, y, y); + let y = y as i32; + let e = i32x4::new(x.wrapping_add(y), 1 + y, 2 + y, 3 + y); + let r: i32x4 = transmute(vaddw_s16(transmute(a), transmute(b))); + assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vld1q_dup_p8() { - let elem: p8 = 42; - let e = u8x16::new( - 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, - ); - let r: u8x16 = transmute(vld1q_dup_p8(&elem)); - assert_eq!(r, e) + unsafe fn test_vaddw_s32() { + let x = i64::MAX; + let a = i64x2::new(x, 1); + let y = i32::MAX; + let b = i32x2::new(y, y); + let y = y as i64; + let e = i64x2::new(x.wrapping_add(y), 1 + y); + let r: i64x2 = transmute(vaddw_s32(transmute(a), transmute(b))); + assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vld1_dup_p16() { - let elem: p16 = 42; - let e = u16x4::new(42, 42, 42, 42); - let r: u16x4 = transmute(vld1_dup_p16(&elem)); - assert_eq!(r, e) + unsafe fn test_vaddw_u8() { + let x = u16::MAX; + let a = u16x8::new(x, 1, 2, 3, 4, 5, 6, 7); + let y = u8::MAX; + let b = u8x8::new(y, y, y, y, y, y, y, y); + let y = y as u16; + let e = u16x8::new( + x.wrapping_add(y), + 1 + y, + 2 + y, + 3 + y, + 4 + y, + 5 + y, + 6 + y, + 7 + y, + ); + let r: u16x8 = transmute(vaddw_u8(transmute(a), transmute(b))); + assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vld1q_dup_p16() { - let elem: p16 = 42; - let e = u16x8::new(42, 42, 42, 42, 42, 42, 42, 42); - let r: u16x8 = transmute(vld1q_dup_p16(&elem)); - assert_eq!(r, e) + unsafe fn test_vaddw_u16() { + let x = u32::MAX; + let a = u32x4::new(x, 1, 2, 3); + let y = u16::MAX; + let b = u16x4::new(y, y, y, y); + let y = y as u32; + let e = u32x4::new(x.wrapping_add(y), 1 + y, 2 + y, 3 + y); + let r: u32x4 = transmute(vaddw_u16(transmute(a), transmute(b))); + assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vld1_dup_f32() { - let elem: f32 = 42.; - let e = f32x2::new(42., 42.); - let r: f32x2 = transmute(vld1_dup_f32(&elem)); - assert_eq!(r, e) + unsafe fn test_vaddw_u32() { + let x = u64::MAX; + let a = u64x2::new(x, 1); + let y = u32::MAX; + let b = u32x2::new(y, y); + let y = y as u64; + let e = u64x2::new(x.wrapping_add(y), 1 + y); + let r: u64x2 = transmute(vaddw_u32(transmute(a), transmute(b))); + assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vld1q_dup_f32() { - let elem: f32 = 42.; - let e = f32x4::new(42., 42., 42., 42.); - let r: f32x4 = transmute(vld1q_dup_f32(&elem)); - assert_eq!(r, e) + unsafe fn test_vaddw_high_s8() { + let x = i16::MAX; + let a = i16x8::new(x, 1, 2, 3, 4, 5, 6, 7); + let y = i8::MAX; + let b = i8x16::new(0, 0, 0, 0, 0, 0, 0, 0, y, y, y, y, y, y, y, y); + let y = y as i16; + let e = i16x8::new( + x.wrapping_add(y), + 1 + y, + 2 + y, + 3 + y, + 4 + y, + 5 + y, + 6 + y, + 7 + y, + ); + let r: i16x8 = transmute(vaddw_high_s8(transmute(a), transmute(b))); + assert_eq!(r, e); } - #[cfg(target_arch = "arm")] #[simd_test(enable = "neon")] - unsafe fn test_vcvtq_s32_f32() { - let f = f32x4::new(-1., 2., 3., 4.); - let e = i32x4::new(-1, 2, 3, 4); - let r: i32x4 = transmute(vcvtq_s32_f32(transmute(f))); + unsafe fn test_vaddw_high_s16() { + let x = i32::MAX; + let a = i32x4::new(x, 1, 2, 3); + let y = i16::MAX; + let b = i16x8::new(0, 0, 0, 0, y, y, y, y); + let y = y as i32; + let e = i32x4::new(x.wrapping_add(y), 1 + y, 2 + y, 3 + y); + let r: i32x4 = transmute(vaddw_high_s16(transmute(a), transmute(b))); assert_eq!(r, e); } - #[cfg(target_arch = "arm")] #[simd_test(enable = "neon")] - unsafe fn test_vcvtq_u32_f32() { - let f = f32x4::new(1., 2., 3., 4.); - let e = u32x4::new(1, 2, 3, 4); - let r: u32x4 = transmute(vcvtq_u32_f32(transmute(f))); + unsafe fn test_vaddw_high_s32() { + let x = i64::MAX; + let a = i64x2::new(x, 1); + let y = i32::MAX; + let b = i32x4::new(0, 0, y, y); + let y = y as i64; + let e = i64x2::new(x.wrapping_add(y), 1 + y); + let r: i64x2 = transmute(vaddw_high_s32(transmute(a), transmute(b))); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vget_lane_u8() { - let v = i8x8::new(1, 2, 3, 4, 5, 6, 7, 8); - let r = vget_lane_u8(transmute(v), 1); - assert_eq!(r, 2); + unsafe fn test_vaddw_high_u8() { + let x = u16::MAX; + let a = u16x8::new(x, 1, 2, 3, 4, 5, 6, 7); + let y = u8::MAX; + let b = u8x16::new(0, 0, 0, 0, 0, 0, 0, 0, y, y, y, y, y, y, y, y); + let y = y as u16; + let e = u16x8::new( + x.wrapping_add(y), + 1 + y, + 2 + y, + 3 + y, + 4 + y, + 5 + y, + 6 + y, + 7 + y, + ); + let r: u16x8 = transmute(vaddw_high_u8(transmute(a), transmute(b))); + assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vgetq_lane_u32() { - let v = i32x4::new(1, 2, 3, 4); - let r = vgetq_lane_u32(transmute(v), 1); - assert_eq!(r, 2); + unsafe fn test_vaddw_high_u16() { + let x = u32::MAX; + let a = u32x4::new(x, 1, 2, 3); + let y = u16::MAX; + let b = u16x8::new(0, 0, 0, 0, y, y, y, y); + let y = y as u32; + let e = u32x4::new(x.wrapping_add(y), 1 + y, 2 + y, 3 + y); + let r: u32x4 = transmute(vaddw_high_u16(transmute(a), transmute(b))); + assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vgetq_lane_s32() { - let v = i32x4::new(1, 2, 3, 4); - let r = vgetq_lane_s32(transmute(v), 1); - assert_eq!(r, 2); + unsafe fn test_vaddw_high_u32() { + let x = u64::MAX; + let a = u64x2::new(x, 1); + let y = u32::MAX; + let b = u32x4::new(0, 0, y, y); + let y = y as u64; + let e = u64x2::new(x.wrapping_add(y), 1 + y); + let r: u64x2 = transmute(vaddw_high_u32(transmute(a), transmute(b))); + assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vget_lane_u64() { - let v: u64 = 1; - let r = vget_lane_u64(transmute(v), 0); - assert_eq!(r, 1); + unsafe fn test_vaddhn_s16() { + let a = i16x8::new( + (0 << 8) + 1, + (1 << 8) + 1, + (2 << 8) + 1, + (3 << 8) + 1, + (4 << 8) + 1, + (5 << 8) + 1, + (6 << 8) + 1, + (7 << 8) + 1, + ); + let e = i8x8::new(0, 2, 4, 6, 8, 10, 12, 14); + let r: i8x8 = transmute(vaddhn_s16(transmute(a), transmute(a))); + assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vgetq_lane_u16() { - let v = i16x8::new(1, 2, 3, 4, 5, 6, 7, 8); - let r = vgetq_lane_u16(transmute(v), 1); - assert_eq!(r, 2); + unsafe fn test_vaddhn_s32() { + let a = i32x4::new((0 << 16) + 1, (1 << 16) + 1, (2 << 16) + 1, (3 << 16) + 1); + let e = i16x4::new(0, 2, 4, 6); + let r: i16x4 = transmute(vaddhn_s32(transmute(a), transmute(a))); + assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vextq_s8() { - let a = i8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); - let b = i8x16::new( - 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 31, 31, 32, - ); - let e = i8x16::new(4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19); - let r: i8x16 = transmute(vextq_s8(transmute(a), transmute(b), 3)); + unsafe fn test_vaddhn_s64() { + let a = i64x2::new((0 << 32) + 1, (1 << 32) + 1); + let e = i32x2::new(0, 2); + let r: i32x2 = transmute(vaddhn_s64(transmute(a), transmute(a))); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vextq_u8() { - let a = u8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); - let b = u8x16::new( - 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 31, 31, 32, + unsafe fn test_vaddhn_u16() { + let a = u16x8::new( + (0 << 8) + 1, + (1 << 8) + 1, + (2 << 8) + 1, + (3 << 8) + 1, + (4 << 8) + 1, + (5 << 8) + 1, + (6 << 8) + 1, + (7 << 8) + 1, ); - let e = u8x16::new(4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19); - let r: u8x16 = transmute(vextq_u8(transmute(a), transmute(b), 3)); + let e = u8x8::new(0, 2, 4, 6, 8, 10, 12, 14); + let r: u8x8 = transmute(vaddhn_u16(transmute(a), transmute(a))); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vshrq_n_u8() { - let a = u8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); - let e = u8x16::new(0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, 4); - let r: u8x16 = transmute(vshrq_n_u8(transmute(a), 2)); + unsafe fn test_vaddhn_u32() { + let a = u32x4::new((0 << 16) + 1, (1 << 16) + 1, (2 << 16) + 1, (3 << 16) + 1); + let e = u16x4::new(0, 2, 4, 6); + let r: u16x4 = transmute(vaddhn_u32(transmute(a), transmute(a))); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vshlq_n_u8() { - let a = u8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); - let e = u8x16::new(4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60, 64); - let r: u8x16 = transmute(vshlq_n_u8(transmute(a), 2)); + unsafe fn test_vaddhn_u64() { + let a = u64x2::new((0 << 32) + 1, (1 << 32) + 1); + let e = u32x2::new(0, 2); + let r: u32x2 = transmute(vaddhn_u64(transmute(a), transmute(a))); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vqmovn_u64() { - let a = u64x2::new(1, 2); - let e = u32x2::new(1, 2); - let r: u32x2 = transmute(vqmovn_u64(transmute(a))); + unsafe fn test_vaddhn_high_s16() { + let r = i8x8::splat(42); + let a = i16x8::new( + (0 << 8) + 1, + (1 << 8) + 1, + (2 << 8) + 1, + (3 << 8) + 1, + (4 << 8) + 1, + (5 << 8) + 1, + (6 << 8) + 1, + (7 << 8) + 1, + ); + let e = i8x16::new(42, 42, 42, 42, 42, 42, 42, 42, 0, 2, 4, 6, 8, 10, 12, 14); + let r: i8x16 = transmute(vaddhn_high_s16(transmute(r), transmute(a), transmute(a))); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vreinterpret_u64_u32() { - let v: i8 = 42; - let e = i8x16::new( - 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, - ); - let r: i8x16 = transmute(vdupq_n_s8(v)); + unsafe fn test_vaddhn_high_s32() { + let r = i16x4::splat(42); + let a = i32x4::new((0 << 16) + 1, (1 << 16) + 1, (2 << 16) + 1, (3 << 16) + 1); + let e = i16x8::new(42, 42, 42, 42, 0, 2, 4, 6); + let r: i16x8 = transmute(vaddhn_high_s32(transmute(r), transmute(a), transmute(a))); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vdupq_n_s8() { - let v: i8 = 42; - let e = i8x16::new( - 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, - ); - let r: i8x16 = transmute(vdupq_n_s8(v)); + unsafe fn test_vaddhn_high_s64() { + let r = i32x2::splat(42); + let a = i64x2::new((0 << 32) + 1, (1 << 32) + 1); + let e = i32x4::new(42, 42, 0, 2); + let r: i32x4 = transmute(vaddhn_high_s64(transmute(r), transmute(a), transmute(a))); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vdupq_n_u8() { - let v: u8 = 42; - let e = u8x16::new( - 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, + unsafe fn test_vaddhn_high_u16() { + let r = u8x8::splat(42); + let a = u16x8::new( + (0 << 8) + 1, + (1 << 8) + 1, + (2 << 8) + 1, + (3 << 8) + 1, + (4 << 8) + 1, + (5 << 8) + 1, + (6 << 8) + 1, + (7 << 8) + 1, ); - let r: u8x16 = transmute(vdupq_n_u8(v)); + let e = u8x16::new(42, 42, 42, 42, 42, 42, 42, 42, 0, 2, 4, 6, 8, 10, 12, 14); + let r: u8x16 = transmute(vaddhn_high_u16(transmute(r), transmute(a), transmute(a))); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vmovq_n_u8() { - let v: u8 = 42; - let e = u8x16::new( - 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, - ); - let r: u8x16 = transmute(vmovq_n_u8(v)); + unsafe fn test_vaddhn_high_u32() { + let r = u16x4::splat(42); + let a = u32x4::new((0 << 16) + 1, (1 << 16) + 1, (2 << 16) + 1, (3 << 16) + 1); + let e = u16x8::new(42, 42, 42, 42, 0, 2, 4, 6); + let r: u16x8 = transmute(vaddhn_high_u32(transmute(r), transmute(a), transmute(a))); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vgetq_lane_u64() { - let v = i64x2::new(1, 2); - let r = vgetq_lane_u64(transmute(v), 1); - assert_eq!(r, 2); + unsafe fn test_vaddhn_high_u64() { + let r = u32x2::splat(42); + let a = u64x2::new((0 << 32) + 1, (1 << 32) + 1); + let e = u32x4::new(42, 42, 0, 2); + let r: u32x4 = transmute(vaddhn_high_u64(transmute(r), transmute(a), transmute(a))); + assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vadd_s8() { - test_ari_s8( - |i, j| vadd_s8(i, j), - |a: i8, b: i8| -> i8 { a.overflowing_add(b).0 }, - ); - } - #[simd_test(enable = "neon")] - unsafe fn test_vaddq_s8() { - testq_ari_s8( - |i, j| vaddq_s8(i, j), - |a: i8, b: i8| -> i8 { a.overflowing_add(b).0 }, - ); - } - #[simd_test(enable = "neon")] - unsafe fn test_vadd_s16() { - test_ari_s16( - |i, j| vadd_s16(i, j), - |a: i16, b: i16| -> i16 { a.overflowing_add(b).0 }, - ); - } - #[simd_test(enable = "neon")] - unsafe fn test_vaddq_s16() { - testq_ari_s16( - |i, j| vaddq_s16(i, j), - |a: i16, b: i16| -> i16 { a.overflowing_add(b).0 }, - ); - } - #[simd_test(enable = "neon")] - unsafe fn test_vadd_s32() { - test_ari_s32( - |i, j| vadd_s32(i, j), - |a: i32, b: i32| -> i32 { a.overflowing_add(b).0 }, + unsafe fn test_vraddhn_s16() { + let round_constant: i16 = (1 << 8) - 1; + let a = i16x8::new( + 0 << 8, + 1 << 8, + 2 << 8, + 3 << 8, + 4 << 8, + 5 << 8, + 6 << 8, + 7 << 8, ); - } - #[simd_test(enable = "neon")] - unsafe fn test_vaddq_s32() { - testq_ari_s32( - |i, j| vaddq_s32(i, j), - |a: i32, b: i32| -> i32 { a.overflowing_add(b).0 }, + let b = i16x8::new( + 0 << 8, + (1 << 8) + round_constant, + 2 << 8, + (3 << 8) + round_constant, + 4 << 8, + (5 << 8) + round_constant, + 6 << 8, + (7 << 8) + round_constant, ); + let e = i8x8::new(0, 3, 4, 7, 8, 11, 12, 15); + let r: i8x8 = transmute(vraddhn_s16(transmute(a), transmute(b))); + assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vadd_u8() { - test_ari_u8( - |i, j| vadd_u8(i, j), - |a: u8, b: u8| -> u8 { a.overflowing_add(b).0 }, - ); - } - #[simd_test(enable = "neon")] - unsafe fn test_vaddq_u8() { - testq_ari_u8( - |i, j| vaddq_u8(i, j), - |a: u8, b: u8| -> u8 { a.overflowing_add(b).0 }, - ); - } - #[simd_test(enable = "neon")] - unsafe fn test_vadd_u16() { - test_ari_u16( - |i, j| vadd_u16(i, j), - |a: u16, b: u16| -> u16 { a.overflowing_add(b).0 }, - ); - } - #[simd_test(enable = "neon")] - unsafe fn test_vaddq_u16() { - testq_ari_u16( - |i, j| vaddq_u16(i, j), - |a: u16, b: u16| -> u16 { a.overflowing_add(b).0 }, - ); - } - #[simd_test(enable = "neon")] - unsafe fn test_vadd_u32() { - test_ari_u32( - |i, j| vadd_u32(i, j), - |a: u32, b: u32| -> u32 { a.overflowing_add(b).0 }, - ); - } - #[simd_test(enable = "neon")] - unsafe fn test_vaddq_u32() { - testq_ari_u32( - |i, j| vaddq_u32(i, j), - |a: u32, b: u32| -> u32 { a.overflowing_add(b).0 }, + unsafe fn test_vraddhn_s32() { + let round_constant: i32 = (1 << 16) - 1; + let a = i32x4::new(0 << 16, 1 << 16, 2 << 16, 3 << 16); + let b = i32x4::new( + 0 << 16, + (1 << 16) + round_constant, + 2 << 16, + (3 << 16) + round_constant, ); + let e = i16x4::new(0, 3, 4, 7); + let r: i16x4 = transmute(vraddhn_s32(transmute(a), transmute(b))); + assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vadd_f32() { - test_ari_f32(|i, j| vadd_f32(i, j), |a: f32, b: f32| -> f32 { a + b }); - } - #[simd_test(enable = "neon")] - unsafe fn test_vaddq_f32() { - testq_ari_f32(|i, j| vaddq_f32(i, j), |a: f32, b: f32| -> f32 { a + b }); + unsafe fn test_vraddhn_s64() { + let round_constant: i64 = (1 << 32) - 1; + let a = i64x2::new(0 << 32, 1 << 32); + let b = i64x2::new(0 << 32, (1 << 32) + round_constant); + let e = i32x2::new(0, 3); + let r: i32x2 = transmute(vraddhn_s64(transmute(a), transmute(b))); + assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vaddl_s8() { - let v = i8::MAX; - let a = i8x8::new(v, v, v, v, v, v, v, v); - let v = 2 * (v as i16); - let e = i16x8::new(v, v, v, v, v, v, v, v); - let r: i16x8 = transmute(vaddl_s8(transmute(a), transmute(a))); + unsafe fn test_vraddhn_u16() { + let round_constant: u16 = (1 << 8) - 1; + let a = u16x8::new( + 0 << 8, + 1 << 8, + 2 << 8, + 3 << 8, + 4 << 8, + 5 << 8, + 6 << 8, + 7 << 8, + ); + let b = u16x8::new( + 0 << 8, + (1 << 8) + round_constant, + 2 << 8, + (3 << 8) + round_constant, + 4 << 8, + (5 << 8) + round_constant, + 6 << 8, + (7 << 8) + round_constant, + ); + let e = u8x8::new(0, 3, 4, 7, 8, 11, 12, 15); + let r: u8x8 = transmute(vraddhn_u16(transmute(a), transmute(b))); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vaddl_s16() { - let v = i16::MAX; - let a = i16x4::new(v, v, v, v); - let v = 2 * (v as i32); - let e = i32x4::new(v, v, v, v); - let r: i32x4 = transmute(vaddl_s16(transmute(a), transmute(a))); + unsafe fn test_vraddhn_u32() { + let round_constant: u32 = (1 << 16) - 1; + let a = u32x4::new(0 << 16, 1 << 16, 2 << 16, 3 << 16); + let b = u32x4::new( + 0 << 16, + (1 << 16) + round_constant, + 2 << 16, + (3 << 16) + round_constant, + ); + let e = u16x4::new(0, 3, 4, 7); + let r: u16x4 = transmute(vraddhn_u32(transmute(a), transmute(b))); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vaddl_s32() { - let v = i32::MAX; - let a = i32x2::new(v, v); - let v = 2 * (v as i64); - let e = i64x2::new(v, v); - let r: i64x2 = transmute(vaddl_s32(transmute(a), transmute(a))); + unsafe fn test_vraddhn_u64() { + let round_constant: u64 = (1 << 32) - 1; + let a = u64x2::new(0 << 32, 1 << 32); + let b = u64x2::new(0 << 32, (1 << 32) + round_constant); + let e = u32x2::new(0, 3); + let r: u32x2 = transmute(vraddhn_u64(transmute(a), transmute(b))); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vaddl_u8() { - let v = u8::MAX; - let a = u8x8::new(v, v, v, v, v, v, v, v); - let v = 2 * (v as u16); - let e = u16x8::new(v, v, v, v, v, v, v, v); - let r: u16x8 = transmute(vaddl_u8(transmute(a), transmute(a))); + unsafe fn test_vraddhn_high_s16() { + let r = i8x8::splat(42); + let round_constant: i16 = (1 << 8) - 1; + let a = i16x8::new( + 0 << 8, + 1 << 8, + 2 << 8, + 3 << 8, + 4 << 8, + 5 << 8, + 6 << 8, + 7 << 8, + ); + let b = i16x8::new( + 0 << 8, + (1 << 8) + round_constant, + 2 << 8, + (3 << 8) + round_constant, + 4 << 8, + (5 << 8) + round_constant, + 6 << 8, + (7 << 8) + round_constant, + ); + let e = i8x16::new(42, 42, 42, 42, 42, 42, 42, 42, 0, 3, 4, 7, 8, 11, 12, 15); + let r: i8x16 = transmute(vraddhn_high_s16(transmute(r), transmute(a), transmute(b))); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vaddl_u16() { - let v = u16::MAX; - let a = u16x4::new(v, v, v, v); - let v = 2 * (v as u32); - let e = u32x4::new(v, v, v, v); - let r: u32x4 = transmute(vaddl_u16(transmute(a), transmute(a))); + unsafe fn test_vraddhn_high_s32() { + let r = i16x4::splat(42); + let round_constant: i32 = (1 << 16) - 1; + let a = i32x4::new(0 << 16, 1 << 16, 2 << 16, 3 << 16); + let b = i32x4::new( + 0 << 16, + (1 << 16) + round_constant, + 2 << 16, + (3 << 16) + round_constant, + ); + let e = i16x8::new(42, 42, 42, 42, 0, 3, 4, 7); + let r: i16x8 = transmute(vraddhn_high_s32(transmute(r), transmute(a), transmute(b))); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vaddl_u32() { - let v = u32::MAX; - let a = u32x2::new(v, v); - let v = 2 * (v as u64); - let e = u64x2::new(v, v); - let r: u64x2 = transmute(vaddl_u32(transmute(a), transmute(a))); + unsafe fn test_vraddhn_high_s64() { + let r = i32x2::splat(42); + let round_constant: i64 = (1 << 32) - 1; + let a = i64x2::new(0 << 32, 1 << 32); + let b = i64x2::new(0 << 32, (1 << 32) + round_constant); + let e = i32x4::new(42, 42, 0, 3); + let r: i32x4 = transmute(vraddhn_high_s64(transmute(r), transmute(a), transmute(b))); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vaddl_high_s8() { - let a = i8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); - let x = i8::MAX; - let b = i8x16::new(x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x); - let x = x as i16; - let e = i16x8::new(x + 8, x + 9, x + 10, x + 11, x + 12, x + 13, x + 14, x + 15); - let r: i16x8 = transmute(vaddl_high_s8(transmute(a), transmute(b))); + unsafe fn test_vraddhn_high_u16() { + let r = u8x8::splat(42); + let round_constant: u16 = (1 << 8) - 1; + let a = u16x8::new( + 0 << 8, + 1 << 8, + 2 << 8, + 3 << 8, + 4 << 8, + 5 << 8, + 6 << 8, + 7 << 8, + ); + let b = u16x8::new( + 0 << 8, + (1 << 8) + round_constant, + 2 << 8, + (3 << 8) + round_constant, + 4 << 8, + (5 << 8) + round_constant, + 6 << 8, + (7 << 8) + round_constant, + ); + let e = u8x16::new(42, 42, 42, 42, 42, 42, 42, 42, 0, 3, 4, 7, 8, 11, 12, 15); + let r: u8x16 = transmute(vraddhn_high_u16(transmute(r), transmute(a), transmute(b))); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vaddl_high_s16() { - let a = i16x8::new(0, 1, 2, 3, 4, 5, 6, 7); - let x = i16::MAX; - let b = i16x8::new(x, x, x, x, x, x, x, x); - let x = x as i32; - let e = i32x4::new(x + 4, x + 5, x + 6, x + 7); - let r: i32x4 = transmute(vaddl_high_s16(transmute(a), transmute(b))); + unsafe fn test_vraddhn_high_u32() { + let r = u16x4::splat(42); + let round_constant: u32 = (1 << 16) - 1; + let a = u32x4::new(0 << 16, 1 << 16, 2 << 16, 3 << 16); + let b = u32x4::new( + 0 << 16, + (1 << 16) + round_constant, + 2 << 16, + (3 << 16) + round_constant, + ); + let e = u16x8::new(42, 42, 42, 42, 0, 3, 4, 7); + let r: u16x8 = transmute(vraddhn_high_s32(transmute(r), transmute(a), transmute(b))); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vaddl_high_s32() { - let a = i32x4::new(0, 1, 2, 3); - let x = i32::MAX; - let b = i32x4::new(x, x, x, x); - let x = x as i64; - let e = i64x2::new(x + 2, x + 3); - let r: i64x2 = transmute(vaddl_high_s32(transmute(a), transmute(b))); + unsafe fn test_vraddhn_high_u64() { + let r = u32x2::splat(42); + let round_constant: u64 = (1 << 32) - 1; + let a = u64x2::new(0 << 32, 1 << 32); + let b = u64x2::new(0 << 32, (1 << 32) + round_constant); + let e = u32x4::new(42, 42, 0, 3); + let r: u32x4 = transmute(vraddhn_high_s64(transmute(r), transmute(a), transmute(b))); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vaddl_high_u8() { - let a = u8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); - let x = u8::MAX; - let b = u8x16::new(x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x); - let x = x as u16; - let e = u16x8::new(x + 8, x + 9, x + 10, x + 11, x + 12, x + 13, x + 14, x + 15); - let r: u16x8 = transmute(vaddl_high_u8(transmute(a), transmute(b))); + unsafe fn test_vpaddl_s8() { + let a = i8x8::new(-4, -3, -2, -1, 0, 1, 2, 3); + let r: i16x4 = transmute(vpaddl_s8(transmute(a))); + let e = i16x4::new(-7, -3, 1, 5); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vaddl_high_u16() { - let a = u16x8::new(0, 1, 2, 3, 4, 5, 6, 7); - let x = u16::MAX; - let b = u16x8::new(x, x, x, x, x, x, x, x); - let x = x as u32; - let e = u32x4::new(x + 4, x + 5, x + 6, x + 7); - let r: u32x4 = transmute(vaddl_high_u16(transmute(a), transmute(b))); + unsafe fn test_vpaddl_s16() { + let a = i16x4::new(-2, -1, 0, 1); + let r: i32x2 = transmute(vpaddl_s16(transmute(a))); + let e = i32x2::new(-3, 1); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vaddl_high_u32() { - let a = u32x4::new(0, 1, 2, 3); - let x = u32::MAX; - let b = u32x4::new(x, x, x, x); - let x = x as u64; - let e = u64x2::new(x + 2, x + 3); - let r: u64x2 = transmute(vaddl_high_u32(transmute(a), transmute(b))); + unsafe fn test_vpaddl_s32() { + let a = i32x2::new(-1, 0); + let r: i64x1 = transmute(vpaddl_s32(transmute(a))); + let e = i64x1::new(-1); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vaddw_s8() { - let x = i16::MAX; - let a = i16x8::new(x, 1, 2, 3, 4, 5, 6, 7); - let y = i8::MAX; - let b = i8x8::new(y, y, y, y, y, y, y, y); - let y = y as i16; - let e = i16x8::new(x + y, 1 + y, 2 + y, 3 + y, 4 + y, 5 + y, 6 + y, 7 + y); - let r: i16x8 = transmute(vaddw_s8(transmute(a), transmute(b))); + unsafe fn test_vpaddlq_s8() { + let a = i8x16::new(-8, -7, -6, -5, -4, -3, -2, -1, 0, 1, 2, 3, 4, 5, 6, 7); + let r: i16x8 = transmute(vpaddlq_s8(transmute(a))); + let e = i16x8::new(-15, -11, -7, -3, 1, 5, 9, 13); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vaddw_s16() { - let x = i32::MAX; - let a = i32x4::new(x, 1, 2, 3); - let y = i16::MAX; - let b = i16x4::new(y, y, y, y); - let y = y as i32; - let e = i32x4::new(x + y, 1 + y, 2 + y, 3 + y); - let r: i32x4 = transmute(vaddw_s16(transmute(a), transmute(b))); + unsafe fn test_vpaddlq_s16() { + let a = i16x8::new(-4, -3, -2, -1, 0, 1, 2, 3); + let r: i32x4 = transmute(vpaddlq_s16(transmute(a))); + let e = i32x4::new(-7, -3, 1, 5); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vaddw_s32() { - let x = i64::MAX; - let a = i64x2::new(x, 1); - let y = i32::MAX; - let b = i32x2::new(y, y); - let y = y as i64; - let e = i64x2::new(x + y, 1 + y); - let r: i64x2 = transmute(vaddw_s32(transmute(a), transmute(b))); + unsafe fn test_vpaddlq_s32() { + let a = i32x4::new(-2, -1, 0, 1); + let r: i64x2 = transmute(vpaddlq_s32(transmute(a))); + let e = i64x2::new(-3, 1); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vaddw_u8() { - let x = u16::MAX; - let a = u16x8::new(x, 1, 2, 3, 4, 5, 6, 7); - let y = u8::MAX; - let b = u8x8::new(y, y, y, y, y, y, y, y); - let y = y as u16; - let e = u16x8::new(x + y, 1 + y, 2 + y, 3 + y, 4 + y, 5 + y, 6 + y, 7 + y); - let r: u16x8 = transmute(vaddw_u8(transmute(a), transmute(b))); + unsafe fn test_vpaddl_u8() { + let a = u8x8::new(0, 1, 2, 3, 4, 5, 6, u8::MAX); + let r: u16x4 = transmute(vpaddl_u8(transmute(a))); + let e = u16x4::new(1, 5, 9, u8::MAX as u16 + 6); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vaddw_u16() { - let x = u32::MAX; - let a = u32x4::new(x, 1, 2, 3); - let y = u16::MAX; - let b = u16x4::new(y, y, y, y); - let y = y as u32; - let e = u32x4::new(x + y, 1 + y, 2 + y, 3 + y); - let r: u32x4 = transmute(vaddw_u16(transmute(a), transmute(b))); + unsafe fn test_vpaddl_u16() { + let a = u16x4::new(0, 1, 2, u16::MAX); + let r: u32x2 = transmute(vpaddl_u16(transmute(a))); + let e = u32x2::new(1, u16::MAX as u32 + 2); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vaddw_u32() { - let x = u64::MAX; - let a = u64x2::new(x, 1); - let y = u32::MAX; - let b = u32x2::new(y, y); - let y = y as u64; - let e = u64x2::new(x + y, 1 + y); - let r: u64x2 = transmute(vaddw_u32(transmute(a), transmute(b))); + unsafe fn test_vpaddl_u32() { + let a = u32x2::new(1, u32::MAX); + let r: u64x1 = transmute(vpaddl_u32(transmute(a))); + let e = u64x1::new(u32::MAX as u64 + 1); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vaddw_high_s8() { - let x = i16::MAX; - let a = i16x8::new(x, 1, 2, 3, 4, 5, 6, 7); - let y = i8::MAX; - let b = i8x16::new(0, 0, 0, 0, 0, 0, 0, 0, y, y, y, y, y, y, y, y); - let y = y as i16; - let e = i16x8::new(x + y, 1 + y, 2 + y, 3 + y, 4 + y, 5 + y, 6 + y, 7 + y); - let r: i16x8 = transmute(vaddw_high_s8(transmute(a), transmute(b))); + unsafe fn test_vpaddlq_u8() { + let a = u8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, u8::MAX); + let r: u16x8 = transmute(vpaddlq_u8(transmute(a))); + let e = u16x8::new(1, 5, 9, 13, 17, 21, 25, u8::MAX as u16 + 14); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vaddw_high_s16() { - let x = i32::MAX; - let a = i32x4::new(x, 1, 2, 3); - let y = i16::MAX; - let b = i16x8::new(0, 0, 0, 0, y, y, y, y); - let y = y as i32; - let e = i32x4::new(x + y, 1 + y, 2 + y, 3 + y); - let r: i32x4 = transmute(vaddw_high_s16(transmute(a), transmute(b))); + unsafe fn test_vpaddlq_u16() { + let a = u16x8::new(0, 1, 2, 3, 4, 5, 6, u16::MAX); + let r: u32x4 = transmute(vpaddlq_u16(transmute(a))); + let e = u32x4::new(1, 5, 9, u16::MAX as u32 + 6); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vaddw_high_s32() { - let x = i64::MAX; - let a = i64x2::new(x, 1); - let y = i32::MAX; - let b = i32x4::new(0, 0, y, y); - let y = y as i64; - let e = i64x2::new(x + y, 1 + y); - let r: i64x2 = transmute(vaddw_high_s32(transmute(a), transmute(b))); + unsafe fn test_vpaddlq_u32() { + let a = u32x4::new(0, 1, 2, u32::MAX); + let r: u64x2 = transmute(vpaddlq_u32(transmute(a))); + let e = u64x2::new(1, u32::MAX as u64 + 2); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vaddw_high_u8() { - let x = u16::MAX; - let a = u16x8::new(x, 1, 2, 3, 4, 5, 6, 7); - let y = u8::MAX; - let b = u8x16::new(0, 0, 0, 0, 0, 0, 0, 0, y, y, y, y, y, y, y, y); - let y = y as u16; - let e = u16x8::new(x + y, 1 + y, 2 + y, 3 + y, 4 + y, 5 + y, 6 + y, 7 + y); - let r: u16x8 = transmute(vaddw_high_u8(transmute(a), transmute(b))); + unsafe fn test_vpadal_s8() { + let a = i16x4::new(42, 42, 42, 42); + let b = i8x8::new(-4, -3, -2, -1, 0, 1, 2, 3); + let r: i16x4 = transmute(vpadal_s8(transmute(a), transmute(b))); + let e = i16x4::new(35, 39, 43, 47); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vaddw_high_u16() { - let x = u32::MAX; - let a = u32x4::new(x, 1, 2, 3); - let y = u16::MAX; - let b = u16x8::new(0, 0, 0, 0, y, y, y, y); - let y = y as u32; - let e = u32x4::new(x + y, 1 + y, 2 + y, 3 + y); - let r: u32x4 = transmute(vaddw_high_u16(transmute(a), transmute(b))); + unsafe fn test_vpadal_s16() { + let a = i32x2::new(42, 42); + let b = i16x4::new(-2, -1, 0, 1); + let r: i32x2 = transmute(vpadal_s16(transmute(a), transmute(b))); + let e = i32x2::new(39, 43); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vaddw_high_u32() { - let x = u64::MAX; - let a = u64x2::new(x, 1); - let y = u32::MAX; - let b = u32x4::new(0, 0, y, y); - let y = y as u64; - let e = u64x2::new(x + y, 1 + y); - let r: u64x2 = transmute(vaddw_high_u32(transmute(a), transmute(b))); + unsafe fn test_vpadal_s32() { + let a = i64x1::new(42); + let b = i32x2::new(-1, 0); + let r: i64x1 = transmute(vpadal_s32(transmute(a), transmute(b))); + let e = i64x1::new(41); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vaddhn_s16() { - let a = i16x8::new( - (0 << 8) + 1, - (1 << 8) + 1, - (2 << 8) + 1, - (3 << 8) + 1, - (4 << 8) + 1, - (5 << 8) + 1, - (6 << 8) + 1, - (7 << 8) + 1, - ); - let e = i8x8::new(0, 2, 4, 6, 8, 10, 12, 14); - let r: i8x8 = transmute(vaddhn_s16(transmute(a), transmute(a))); + unsafe fn test_vpadalq_s8() { + let a = i16x8::new(42, 42, 42, 42, 42, 42, 42, 42); + let b = i8x16::new(-8, -7, -6, -5, -4, -3, -2, -1, 0, 1, 2, 3, 4, 5, 6, 7); + let r: i16x8 = transmute(vpadalq_s8(transmute(a), transmute(b))); + let e = i16x8::new(27, 31, 35, 39, 43, 47, 51, 55); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vaddhn_s32() { - let a = i32x4::new((0 << 16) + 1, (1 << 16) + 1, (2 << 16) + 1, (3 << 16) + 1); - let e = i16x4::new(0, 2, 4, 6); - let r: i16x4 = transmute(vaddhn_s32(transmute(a), transmute(a))); + unsafe fn test_vpadalq_s16() { + let a = i32x4::new(42, 42, 42, 42); + let b = i16x8::new(-4, -3, -2, -1, 0, 1, 2, 3); + let r: i32x4 = transmute(vpadalq_s16(transmute(a), transmute(b))); + let e = i32x4::new(35, 39, 43, 47); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vaddhn_s64() { - let a = i64x2::new((0 << 32) + 1, (1 << 32) + 1); - let e = i32x2::new(0, 2); - let r: i32x2 = transmute(vaddhn_s64(transmute(a), transmute(a))); + unsafe fn test_vpadalq_s32() { + let a = i64x2::new(42, 42); + let b = i32x4::new(-2, -1, 0, 1); + let r: i64x2 = transmute(vpadalq_s32(transmute(a), transmute(b))); + let e = i64x2::new(39, 43); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vaddhn_u16() { - let a = u16x8::new( - (0 << 8) + 1, - (1 << 8) + 1, - (2 << 8) + 1, - (3 << 8) + 1, - (4 << 8) + 1, - (5 << 8) + 1, - (6 << 8) + 1, - (7 << 8) + 1, - ); - let e = u8x8::new(0, 2, 4, 6, 8, 10, 12, 14); - let r: u8x8 = transmute(vaddhn_u16(transmute(a), transmute(a))); + unsafe fn test_vpadal_u8() { + let a = u16x4::new(42, 42, 42, 42); + let b = u8x8::new(0, 1, 2, 3, 4, 5, 6, u8::MAX); + let r: u16x4 = transmute(vpadal_u8(transmute(a), transmute(b))); + let e = u16x4::new(43, 47, 51, u8::MAX as u16 + 48); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vaddhn_u32() { - let a = u32x4::new((0 << 16) + 1, (1 << 16) + 1, (2 << 16) + 1, (3 << 16) + 1); - let e = u16x4::new(0, 2, 4, 6); - let r: u16x4 = transmute(vaddhn_u32(transmute(a), transmute(a))); + unsafe fn test_vpadal_u16() { + let a = u32x2::new(42, 42); + let b = u16x4::new(0, 1, 2, u16::MAX); + let r: u32x2 = transmute(vpadal_u16(transmute(a), transmute(b))); + let e = u32x2::new(43, u16::MAX as u32 + 44); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vaddhn_u64() { - let a = u64x2::new((0 << 32) + 1, (1 << 32) + 1); - let e = u32x2::new(0, 2); - let r: u32x2 = transmute(vaddhn_u64(transmute(a), transmute(a))); + unsafe fn test_vpadal_u32() { + let a = u64x1::new(42); + let b = u32x2::new(1, u32::MAX); + let r: u64x1 = transmute(vpadal_u32(transmute(a), transmute(b))); + let e = u64x1::new(u32::MAX as u64 + 43); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vaddhn_high_s16() { - let r = i8x8::splat(42); - let a = i16x8::new( - (0 << 8) + 1, - (1 << 8) + 1, - (2 << 8) + 1, - (3 << 8) + 1, - (4 << 8) + 1, - (5 << 8) + 1, - (6 << 8) + 1, - (7 << 8) + 1, - ); - let e = i8x16::new(42, 42, 42, 42, 42, 42, 42, 42, 0, 2, 4, 6, 8, 10, 12, 14); - let r: i8x16 = transmute(vaddhn_high_s16(transmute(r), transmute(a), transmute(a))); + unsafe fn test_vpadalq_u8() { + let a = u16x8::new(42, 42, 42, 42, 42, 42, 42, 42); + let b = u8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, u8::MAX); + let r: u16x8 = transmute(vpadalq_u8(transmute(a), transmute(b))); + let e = u16x8::new(43, 47, 51, 55, 59, 63, 67, u8::MAX as u16 + 56); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vaddhn_high_s32() { - let r = i16x4::splat(42); - let a = i32x4::new((0 << 16) + 1, (1 << 16) + 1, (2 << 16) + 1, (3 << 16) + 1); - let e = i16x8::new(42, 42, 42, 42, 0, 2, 4, 6); - let r: i16x8 = transmute(vaddhn_high_s32(transmute(r), transmute(a), transmute(a))); + unsafe fn test_vpadalq_u16() { + let a = u32x4::new(42, 42, 42, 42); + let b = u16x8::new(0, 1, 2, 3, 4, 5, 6, u16::MAX); + let r: u32x4 = transmute(vpadalq_u16(transmute(a), transmute(b))); + let e = u32x4::new(43, 47, 51, u16::MAX as u32 + 48); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vaddhn_high_s64() { - let r = i32x2::splat(42); - let a = i64x2::new((0 << 32) + 1, (1 << 32) + 1); - let e = i32x4::new(42, 42, 0, 2); - let r: i32x4 = transmute(vaddhn_high_s64(transmute(r), transmute(a), transmute(a))); + unsafe fn test_vpadalq_u32() { + let a = u64x2::new(42, 42); + let b = u32x4::new(0, 1, 2, u32::MAX); + let r: u64x2 = transmute(vpadalq_u32(transmute(a), transmute(b))); + let e = u64x2::new(43, u32::MAX as u64 + 44); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vaddhn_high_u16() { - let r = u8x8::splat(42); - let a = u16x8::new( - (0 << 8) + 1, - (1 << 8) + 1, - (2 << 8) + 1, - (3 << 8) + 1, - (4 << 8) + 1, - (5 << 8) + 1, - (6 << 8) + 1, - (7 << 8) + 1, - ); - let e = u8x16::new(42, 42, 42, 42, 42, 42, 42, 42, 0, 2, 4, 6, 8, 10, 12, 14); - let r: u8x16 = transmute(vaddhn_high_u16(transmute(r), transmute(a), transmute(a))); + unsafe fn test_vmvn_s8() { + let a = i8x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let e = i8x8::new(-1, -2, -3, -4, -5, -6, -7, -8); + let r: i8x8 = transmute(vmvn_s8(transmute(a))); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vaddhn_high_u32() { - let r = u16x4::splat(42); - let a = u32x4::new((0 << 16) + 1, (1 << 16) + 1, (2 << 16) + 1, (3 << 16) + 1); - let e = u16x8::new(42, 42, 42, 42, 0, 2, 4, 6); - let r: u16x8 = transmute(vaddhn_high_u32(transmute(r), transmute(a), transmute(a))); + unsafe fn test_vmvnq_s8() { + let a = i8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let e = i8x16::new( + -1, -2, -3, -4, -5, -6, -7, -8, -9, -10, -11, -12, -13, -14, -15, -16, + ); + let r: i8x16 = transmute(vmvnq_s8(transmute(a))); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vaddhn_high_u64() { - let r = u32x2::splat(42); - let a = u64x2::new((0 << 32) + 1, (1 << 32) + 1); - let e = u32x4::new(42, 42, 0, 2); - let r: u32x4 = transmute(vaddhn_high_u64(transmute(r), transmute(a), transmute(a))); + unsafe fn test_vmvn_s16() { + let a = i16x4::new(0, 1, 2, 3); + let e = i16x4::new(-1, -2, -3, -4); + let r: i16x4 = transmute(vmvn_s16(transmute(a))); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vraddhn_s16() { - let round_constant: i16 = (1 << 8) - 1; - let a = i16x8::new( - 0 << 8, - 1 << 8, - 2 << 8, - 3 << 8, - 4 << 8, - 5 << 8, - 6 << 8, - 7 << 8, - ); - let b = i16x8::new( - 0 << 8, - (1 << 8) + round_constant, - 2 << 8, - (3 << 8) + round_constant, - 4 << 8, - (5 << 8) + round_constant, - 6 << 8, - (7 << 8) + round_constant, - ); - let e = i8x8::new(0, 3, 4, 7, 8, 11, 12, 15); - let r: i8x8 = transmute(vraddhn_s16(transmute(a), transmute(b))); + unsafe fn test_vmvnq_s16() { + let a = i16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let e = i16x8::new(-1, -2, -3, -4, -5, -6, -7, -8); + let r: i16x8 = transmute(vmvnq_s16(transmute(a))); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vraddhn_s32() { - let round_constant: i32 = (1 << 16) - 1; - let a = i32x4::new(0 << 16, 1 << 16, 2 << 16, 3 << 16); - let b = i32x4::new( - 0 << 16, - (1 << 16) + round_constant, - 2 << 16, - (3 << 16) + round_constant, - ); - let e = i16x4::new(0, 3, 4, 7); - let r: i16x4 = transmute(vraddhn_s32(transmute(a), transmute(b))); + unsafe fn test_vmvn_s32() { + let a = i32x2::new(0, 1); + let e = i32x2::new(-1, -2); + let r: i32x2 = transmute(vmvn_s32(transmute(a))); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vraddhn_s64() { - let round_constant: i64 = (1 << 32) - 1; - let a = i64x2::new(0 << 32, 1 << 32); - let b = i64x2::new(0 << 32, (1 << 32) + round_constant); - let e = i32x2::new(0, 3); - let r: i32x2 = transmute(vraddhn_s64(transmute(a), transmute(b))); + unsafe fn test_vmvnq_s32() { + let a = i32x4::new(0, 1, 2, 3); + let e = i32x4::new(-1, -2, -3, -4); + let r: i32x4 = transmute(vmvnq_s32(transmute(a))); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vraddhn_u16() { - let round_constant: u16 = (1 << 8) - 1; - let a = u16x8::new( - 0 << 8, - 1 << 8, - 2 << 8, - 3 << 8, - 4 << 8, - 5 << 8, - 6 << 8, - 7 << 8, - ); - let b = u16x8::new( - 0 << 8, - (1 << 8) + round_constant, - 2 << 8, - (3 << 8) + round_constant, - 4 << 8, - (5 << 8) + round_constant, - 6 << 8, - (7 << 8) + round_constant, - ); - let e = u8x8::new(0, 3, 4, 7, 8, 11, 12, 15); - let r: u8x8 = transmute(vraddhn_u16(transmute(a), transmute(b))); + unsafe fn test_vmvn_u8() { + let a = u8x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let e = u8x8::new(255, 254, 253, 252, 251, 250, 249, 248); + let r: u8x8 = transmute(vmvn_u8(transmute(a))); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vraddhn_u32() { - let round_constant: u32 = (1 << 16) - 1; - let a = u32x4::new(0 << 16, 1 << 16, 2 << 16, 3 << 16); - let b = u32x4::new( - 0 << 16, - (1 << 16) + round_constant, - 2 << 16, - (3 << 16) + round_constant, + unsafe fn test_vmvnq_u8() { + let a = u8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let e = u8x16::new( + 255, 254, 253, 252, 251, 250, 249, 248, 247, 246, 245, 244, 243, 242, 241, 240, ); - let e = u16x4::new(0, 3, 4, 7); - let r: u16x4 = transmute(vraddhn_u32(transmute(a), transmute(b))); + let r: u8x16 = transmute(vmvnq_u8(transmute(a))); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vraddhn_u64() { - let round_constant: u64 = (1 << 32) - 1; - let a = u64x2::new(0 << 32, 1 << 32); - let b = u64x2::new(0 << 32, (1 << 32) + round_constant); - let e = u32x2::new(0, 3); - let r: u32x2 = transmute(vraddhn_u64(transmute(a), transmute(b))); + unsafe fn test_vmvn_u16() { + let a = u16x4::new(0, 1, 2, 3); + let e = u16x4::new(65_535, 65_534, 65_533, 65_532); + let r: u16x4 = transmute(vmvn_u16(transmute(a))); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vraddhn_high_s16() { - let r = i8x8::splat(42); - let round_constant: i16 = (1 << 8) - 1; - let a = i16x8::new( - 0 << 8, - 1 << 8, - 2 << 8, - 3 << 8, - 4 << 8, - 5 << 8, - 6 << 8, - 7 << 8, - ); - let b = i16x8::new( - 0 << 8, - (1 << 8) + round_constant, - 2 << 8, - (3 << 8) + round_constant, - 4 << 8, - (5 << 8) + round_constant, - 6 << 8, - (7 << 8) + round_constant, + unsafe fn test_vmvnq_u16() { + let a = u16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let e = u16x8::new( + 65_535, 65_534, 65_533, 65_532, 65_531, 65_530, 65_529, 65_528, ); - let e = i8x16::new(42, 42, 42, 42, 42, 42, 42, 42, 0, 3, 4, 7, 8, 11, 12, 15); - let r: i8x16 = transmute(vraddhn_high_s16(transmute(r), transmute(a), transmute(b))); + let r: u16x8 = transmute(vmvnq_u16(transmute(a))); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vraddhn_high_s32() { - let r = i16x4::splat(42); - let round_constant: i32 = (1 << 16) - 1; - let a = i32x4::new(0 << 16, 1 << 16, 2 << 16, 3 << 16); - let b = i32x4::new( - 0 << 16, - (1 << 16) + round_constant, - 2 << 16, - (3 << 16) + round_constant, - ); - let e = i16x8::new(42, 42, 42, 42, 0, 3, 4, 7); - let r: i16x8 = transmute(vraddhn_high_s32(transmute(r), transmute(a), transmute(b))); + unsafe fn test_vmvn_u32() { + let a = u32x2::new(0, 1); + let e = u32x2::new(4_294_967_295, 4_294_967_294); + let r: u32x2 = transmute(vmvn_u32(transmute(a))); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vraddhn_high_s64() { - let r = i32x2::splat(42); - let round_constant: i64 = (1 << 32) - 1; - let a = i64x2::new(0 << 32, 1 << 32); - let b = i64x2::new(0 << 32, (1 << 32) + round_constant); - let e = i32x4::new(42, 42, 0, 3); - let r: i32x4 = transmute(vraddhn_high_s64(transmute(r), transmute(a), transmute(b))); + unsafe fn test_vmvnq_u32() { + let a = u32x4::new(0, 1, 2, 3); + let e = u32x4::new(4_294_967_295, 4_294_967_294, 4_294_967_293, 4_294_967_292); + let r: u32x4 = transmute(vmvnq_u32(transmute(a))); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vraddhn_high_u16() { - let r = u8x8::splat(42); - let round_constant: u16 = (1 << 8) - 1; - let a = u16x8::new( - 0 << 8, - 1 << 8, - 2 << 8, - 3 << 8, - 4 << 8, - 5 << 8, - 6 << 8, - 7 << 8, - ); - let b = u16x8::new( - 0 << 8, - (1 << 8) + round_constant, - 2 << 8, - (3 << 8) + round_constant, - 4 << 8, - (5 << 8) + round_constant, - 6 << 8, - (7 << 8) + round_constant, - ); - let e = u8x16::new(42, 42, 42, 42, 42, 42, 42, 42, 0, 3, 4, 7, 8, 11, 12, 15); - let r: u8x16 = transmute(vraddhn_high_u16(transmute(r), transmute(a), transmute(b))); + unsafe fn test_vmvn_p8() { + let a = u8x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let e = u8x8::new(255, 254, 253, 252, 251, 250, 249, 248); + let r: u8x8 = transmute(vmvn_p8(transmute(a))); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vraddhn_high_u32() { - let r = u16x4::splat(42); - let round_constant: u32 = (1 << 16) - 1; - let a = u32x4::new(0 << 16, 1 << 16, 2 << 16, 3 << 16); - let b = u32x4::new( - 0 << 16, - (1 << 16) + round_constant, - 2 << 16, - (3 << 16) + round_constant, + unsafe fn test_vmvnq_p8() { + let a = u8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let e = u8x16::new( + 255, 254, 253, 252, 251, 250, 249, 248, 247, 246, 245, 244, 243, 242, 241, 240, ); - let e = u16x8::new(42, 42, 42, 42, 0, 3, 4, 7); - let r: u16x8 = transmute(vraddhn_high_s32(transmute(r), transmute(a), transmute(b))); + let r: u8x16 = transmute(vmvnq_p8(transmute(a))); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vraddhn_high_u64() { - let r = u32x2::splat(42); - let round_constant: u64 = (1 << 32) - 1; - let a = u64x2::new(0 << 32, 1 << 32); - let b = u64x2::new(0 << 32, (1 << 32) + round_constant); - let e = u32x4::new(42, 42, 0, 3); - let r: u32x4 = transmute(vraddhn_high_s64(transmute(r), transmute(a), transmute(b))); + unsafe fn test_vbic_s8() { + let a = i8x8::new(0, -1, -2, -3, -4, -5, -6, -7); + let b = i8x8::new(1, 1, 1, 1, 1, 1, 1, 1); + let e = i8x8::new(0, -2, -2, -4, -4, -6, -6, -8); + let r: i8x8 = transmute(vbic_s8(transmute(a), transmute(b))); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vpaddl_s8() { - let a = i8x8::new(-4, -3, -2, -1, 0, 1, 2, 3); - let r: i16x4 = transmute(vpaddl_s8(transmute(a))); - let e = i16x4::new(-7, -3, 1, 5); + unsafe fn test_vbicq_s8() { + let a = i8x16::new( + 0, -1, -2, -3, -4, -5, -6, -7, -8, -9, -10, -11, -12, -13, -14, -15, + ); + let b = i8x16::new(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1); + let e = i8x16::new( + 0, -2, -2, -4, -4, -6, -6, -8, -8, -10, -10, -12, -12, -14, -14, -16, + ); + let r: i8x16 = transmute(vbicq_s8(transmute(a), transmute(b))); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vpaddl_s16() { - let a = i16x4::new(-2, -1, 0, 1); - let r: i32x2 = transmute(vpaddl_s16(transmute(a))); - let e = i32x2::new(-3, 1); + unsafe fn test_vbic_s16() { + let a = i16x4::new(0, -1, -2, -3); + let b = i16x4::new(1, 1, 1, 1); + let e = i16x4::new(0, -2, -2, -4); + let r: i16x4 = transmute(vbic_s16(transmute(a), transmute(b))); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vpaddl_s32() { - let a = i32x2::new(-1, 0); - let r: i64x1 = transmute(vpaddl_s32(transmute(a))); - let e = i64x1::new(-1); + unsafe fn test_vbicq_s16() { + let a = i16x8::new(0, -1, -2, -3, -4, -5, -6, -7); + let b = i16x8::new(1, 1, 1, 1, 1, 1, 1, 1); + let e = i16x8::new(0, -2, -2, -4, -4, -6, -6, -8); + let r: i16x8 = transmute(vbicq_s16(transmute(a), transmute(b))); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vpaddlq_s8() { - let a = i8x16::new(-8, -7, -6, -5, -4, -3, -2, -1, 0, 1, 2, 3, 4, 5, 6, 7); - let r: i16x8 = transmute(vpaddlq_s8(transmute(a))); - let e = i16x8::new(-15, -11, -7, -3, 1, 5, 9, 13); + unsafe fn test_vbic_s32() { + let a = i32x2::new(0, -1); + let b = i32x2::new(1, 1); + let e = i32x2::new(0, -2); + let r: i32x2 = transmute(vbic_s32(transmute(a), transmute(b))); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vpaddlq_s16() { - let a = i16x8::new(-4, -3, -2, -1, 0, 1, 2, 3); - let r: i32x4 = transmute(vpaddlq_s16(transmute(a))); - let e = i32x4::new(-7, -3, 1, 5); + unsafe fn test_vbicq_s32() { + let a = i32x4::new(0, -1, -2, -3); + let b = i32x4::new(1, 1, 1, 1); + let e = i32x4::new(0, -2, -2, -4); + let r: i32x4 = transmute(vbicq_s32(transmute(a), transmute(b))); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vpaddlq_s32() { - let a = i32x4::new(-2, -1, 0, 1); - let r: i64x2 = transmute(vpaddlq_s32(transmute(a))); - let e = i64x2::new(-3, 1); + unsafe fn test_vbic_s64() { + let a = i64x1::new(-1); + let b = i64x1::new(1); + let e = i64x1::new(-2); + let r: i64x1 = transmute(vbic_s64(transmute(a), transmute(b))); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vpaddl_u8() { - let a = u8x8::new(0, 1, 2, 3, 4, 5, 6, u8::MAX); - let r: u16x4 = transmute(vpaddl_u8(transmute(a))); - let e = u16x4::new(1, 5, 9, u8::MAX as u16 + 6); + unsafe fn test_vbicq_s64() { + let a = i64x2::new(0, -1); + let b = i64x2::new(1, 1); + let e = i64x2::new(0, -2); + let r: i64x2 = transmute(vbicq_s64(transmute(a), transmute(b))); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vpaddl_u16() { - let a = u16x4::new(0, 1, 2, u16::MAX); - let r: u32x2 = transmute(vpaddl_u16(transmute(a))); - let e = u32x2::new(1, u16::MAX as u32 + 2); + unsafe fn test_vbic_u8() { + let a = u8x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let b = u8x8::new(1, 1, 1, 1, 1, 1, 1, 1); + let e = u8x8::new(0, 0, 2, 2, 4, 4, 6, 6); + let r: u8x8 = transmute(vbic_u8(transmute(a), transmute(b))); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vpaddl_u32() { - let a = u32x2::new(1, u32::MAX); - let r: u64x1 = transmute(vpaddl_u32(transmute(a))); - let e = u64x1::new(u32::MAX as u64 + 1); + unsafe fn test_vbicq_u8() { + let a = u8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let b = u8x16::new(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1); + let e = u8x16::new(0, 0, 2, 2, 4, 4, 6, 6, 8, 8, 10, 10, 12, 12, 14, 14); + let r: u8x16 = transmute(vbicq_u8(transmute(a), transmute(b))); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vpaddlq_u8() { - let a = u8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, u8::MAX); - let r: u16x8 = transmute(vpaddlq_u8(transmute(a))); - let e = u16x8::new(1, 5, 9, 13, 17, 21, 25, u8::MAX as u16 + 14); + unsafe fn test_vbic_u16() { + let a = u16x4::new(0, 1, 2, 3); + let b = u16x4::new(1, 1, 1, 1); + let e = u16x4::new(0, 0, 2, 2); + let r: u16x4 = transmute(vbic_u16(transmute(a), transmute(b))); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vpaddlq_u16() { - let a = u16x8::new(0, 1, 2, 3, 4, 5, 6, u16::MAX); - let r: u32x4 = transmute(vpaddlq_u16(transmute(a))); - let e = u32x4::new(1, 5, 9, u16::MAX as u32 + 6); + unsafe fn test_vbicq_u16() { + let a = u16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let b = u16x8::new(1, 1, 1, 1, 1, 1, 1, 1); + let e = u16x8::new(0, 0, 2, 2, 4, 4, 6, 6); + let r: u16x8 = transmute(vbicq_u16(transmute(a), transmute(b))); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vpaddlq_u32() { - let a = u32x4::new(0, 1, 2, u32::MAX); - let r: u64x2 = transmute(vpaddlq_u32(transmute(a))); - let e = u64x2::new(1, u32::MAX as u64 + 2); + unsafe fn test_vbic_u32() { + let a = u32x2::new(0, 1); + let b = u32x2::new(1, 1); + let e = u32x2::new(0, 0); + let r: u32x2 = transmute(vbic_u32(transmute(a), transmute(b))); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vpadal_s8() { - let a = i16x4::new(42, 42, 42, 42); - let b = i8x8::new(-4, -3, -2, -1, 0, 1, 2, 3); - let r: i16x4 = transmute(vpadal_s8(transmute(a), transmute(b))); - let e = i16x4::new(35, 39, 43, 47); + unsafe fn test_vbicq_u32() { + let a = u32x4::new(0, 1, 2, 3); + let b = u32x4::new(1, 1, 1, 1); + let e = u32x4::new(0, 0, 2, 2); + let r: u32x4 = transmute(vbicq_u32(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vbic_u64() { + let a = u64x1::new(1); + let b = u64x1::new(1); + let e = u64x1::new(0); + let r: u64x1 = transmute(vbic_u64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vbicq_u64() { + let a = u64x2::new(0, 1); + let b = u64x2::new(1, 1); + let e = u64x2::new(0, 0); + let r: u64x2 = transmute(vbicq_u64(transmute(a), transmute(b))); + assert_eq!(r, e); + } + + #[simd_test(enable = "neon")] + unsafe fn test_vbsl_s8() { + let a = u8x8::new(u8::MAX, 0, u8::MAX, 0, u8::MAX, 0, u8::MAX, 0); + let b = i8x8::new( + i8::MAX, + i8::MAX, + i8::MAX, + i8::MAX, + i8::MAX, + i8::MAX, + i8::MAX, + i8::MAX, + ); + let c = i8x8::new( + i8::MIN, + i8::MIN, + i8::MIN, + i8::MIN, + i8::MIN, + i8::MIN, + i8::MIN, + i8::MIN, + ); + let e = i8x8::new( + i8::MAX, + i8::MIN, + i8::MAX, + i8::MIN, + i8::MAX, + i8::MIN, + i8::MAX, + i8::MIN, + ); + let r: i8x8 = transmute(vbsl_s8(transmute(a), transmute(b), transmute(c))); assert_eq!(r, e); } - #[simd_test(enable = "neon")] - unsafe fn test_vpadal_s16() { - let a = i32x2::new(42, 42); - let b = i16x4::new(-2, -1, 0, 1); - let r: i32x2 = transmute(vpadal_s16(transmute(a), transmute(b))); - let e = i32x2::new(39, 43); + unsafe fn test_vbsl_s16() { + let a = u16x4::new(u16::MAX, 0, u16::MAX, 0); + let b = i16x4::new(i16::MAX, i16::MAX, i16::MAX, i16::MAX); + let c = i16x4::new(i16::MIN, i16::MIN, i16::MIN, i16::MIN); + let e = i16x4::new(i16::MAX, i16::MIN, i16::MAX, i16::MIN); + let r: i16x4 = transmute(vbsl_s16(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vbsl_s32() { + let a = u32x2::new(u32::MAX, u32::MIN); + let b = i32x2::new(i32::MAX, i32::MAX); + let c = i32x2::new(i32::MIN, i32::MIN); + let e = i32x2::new(i32::MAX, i32::MIN); + let r: i32x2 = transmute(vbsl_s32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vbsl_s64() { + let a = u64x1::new(u64::MAX); + let b = i64x1::new(i64::MAX); + let c = i64x1::new(i64::MIN); + let e = i64x1::new(i64::MAX); + let r: i64x1 = transmute(vbsl_s64(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vbsl_u8() { + let a = u8x8::new(u8::MAX, 0, u8::MAX, 0, u8::MAX, 0, u8::MAX, 0); + let b = u8x8::new( + u8::MAX, + u8::MAX, + u8::MAX, + u8::MAX, + u8::MAX, + u8::MAX, + u8::MAX, + u8::MAX, + ); + let c = u8x8::new( + u8::MIN, + u8::MIN, + u8::MIN, + u8::MIN, + u8::MIN, + u8::MIN, + u8::MIN, + u8::MIN, + ); + let e = u8x8::new( + u8::MAX, + u8::MIN, + u8::MAX, + u8::MIN, + u8::MAX, + u8::MIN, + u8::MAX, + u8::MIN, + ); + let r: u8x8 = transmute(vbsl_u8(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vbsl_u16() { + let a = u16x4::new(u16::MAX, 0, u16::MAX, 0); + let b = u16x4::new(u16::MAX, u16::MAX, u16::MAX, u16::MAX); + let c = u16x4::new(u16::MIN, u16::MIN, u16::MIN, u16::MIN); + let e = u16x4::new(u16::MAX, u16::MIN, u16::MAX, u16::MIN); + let r: u16x4 = transmute(vbsl_u16(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vbsl_u32() { + let a = u32x2::new(u32::MAX, 0); + let b = u32x2::new(u32::MAX, u32::MAX); + let c = u32x2::new(u32::MIN, u32::MIN); + let e = u32x2::new(u32::MAX, u32::MIN); + let r: u32x2 = transmute(vbsl_u32(transmute(a), transmute(b), transmute(c))); assert_eq!(r, e); } - #[simd_test(enable = "neon")] - unsafe fn test_vpadal_s32() { - let a = i64x1::new(42); - let b = i32x2::new(-1, 0); - let r: i64x1 = transmute(vpadal_s32(transmute(a), transmute(b))); - let e = i64x1::new(41); + unsafe fn test_vbsl_u64() { + let a = u64x1::new(u64::MAX); + let b = u64x1::new(u64::MAX); + let c = u64x1::new(u64::MIN); + let e = u64x1::new(u64::MAX); + let r: u64x1 = transmute(vbsl_u64(transmute(a), transmute(b), transmute(c))); assert_eq!(r, e); } - #[simd_test(enable = "neon")] - unsafe fn test_vpadalq_s8() { - let a = i16x8::new(42, 42, 42, 42, 42, 42, 42, 42); - let b = i8x16::new(-8, -7, -6, -5, -4, -3, -2, -1, 0, 1, 2, 3, 4, 5, 6, 7); - let r: i16x8 = transmute(vpadalq_s8(transmute(a), transmute(b))); - let e = i16x8::new(27, 31, 35, 39, 43, 47, 51, 55); + unsafe fn test_vbsl_f32() { + let a = u32x2::new(u32::MAX, 0); + let b = f32x2::new(f32::MAX, f32::MAX); + let c = f32x2::new(f32::MIN, f32::MIN); + let e = f32x2::new(f32::MAX, f32::MIN); + let r: f32x2 = transmute(vbsl_f32(transmute(a), transmute(b), transmute(c))); assert_eq!(r, e); } - #[simd_test(enable = "neon")] - unsafe fn test_vpadalq_s16() { - let a = i32x4::new(42, 42, 42, 42); - let b = i16x8::new(-4, -3, -2, -1, 0, 1, 2, 3); - let r: i32x4 = transmute(vpadalq_s16(transmute(a), transmute(b))); - let e = i32x4::new(35, 39, 43, 47); + unsafe fn test_vbsl_p8() { + let a = u8x8::new(u8::MAX, 0, u8::MAX, 0, u8::MAX, 0, u8::MAX, 0); + let b = u8x8::new( + u8::MAX, + u8::MAX, + u8::MAX, + u8::MAX, + u8::MAX, + u8::MAX, + u8::MAX, + u8::MAX, + ); + let c = u8x8::new( + u8::MIN, + u8::MIN, + u8::MIN, + u8::MIN, + u8::MIN, + u8::MIN, + u8::MIN, + u8::MIN, + ); + let e = u8x8::new( + u8::MAX, + u8::MIN, + u8::MAX, + u8::MIN, + u8::MAX, + u8::MIN, + u8::MAX, + u8::MIN, + ); + let r: u8x8 = transmute(vbsl_p8(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vbsl_p16() { + let a = u16x4::new(u16::MAX, 0, u16::MAX, 0); + let b = u16x4::new(u16::MAX, u16::MAX, u16::MAX, u16::MAX); + let c = u16x4::new(u16::MIN, u16::MIN, u16::MIN, u16::MIN); + let e = u16x4::new(u16::MAX, u16::MIN, u16::MAX, u16::MIN); + let r: u16x4 = transmute(vbsl_p16(transmute(a), transmute(b), transmute(c))); assert_eq!(r, e); } - #[simd_test(enable = "neon")] - unsafe fn test_vpadalq_s32() { - let a = i64x2::new(42, 42); - let b = i32x4::new(-2, -1, 0, 1); - let r: i64x2 = transmute(vpadalq_s32(transmute(a), transmute(b))); - let e = i64x2::new(39, 43); + unsafe fn test_vbslq_s8() { + let a = u8x16::new( + u8::MAX, + 0, + u8::MAX, + 0, + u8::MAX, + 0, + u8::MAX, + 0, + u8::MAX, + 0, + u8::MAX, + 0, + u8::MAX, + 0, + u8::MAX, + 0, + ); + let b = i8x16::new( + i8::MAX, + i8::MAX, + i8::MAX, + i8::MAX, + i8::MAX, + i8::MAX, + i8::MAX, + i8::MAX, + i8::MAX, + i8::MAX, + i8::MAX, + i8::MAX, + i8::MAX, + i8::MAX, + i8::MAX, + i8::MAX, + ); + let c = i8x16::new( + i8::MIN, + i8::MIN, + i8::MIN, + i8::MIN, + i8::MIN, + i8::MIN, + i8::MIN, + i8::MIN, + i8::MIN, + i8::MIN, + i8::MIN, + i8::MIN, + i8::MIN, + i8::MIN, + i8::MIN, + i8::MIN, + ); + let e = i8x16::new( + i8::MAX, + i8::MIN, + i8::MAX, + i8::MIN, + i8::MAX, + i8::MIN, + i8::MAX, + i8::MIN, + i8::MAX, + i8::MIN, + i8::MAX, + i8::MIN, + i8::MAX, + i8::MIN, + i8::MAX, + i8::MIN, + ); + let r: i8x16 = transmute(vbslq_s8(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vbslq_s16() { + let a = u16x8::new(u16::MAX, 0, u16::MAX, 0, u16::MAX, 0, u16::MAX, 0); + let b = i16x8::new( + i16::MAX, + i16::MAX, + i16::MAX, + i16::MAX, + i16::MAX, + i16::MAX, + i16::MAX, + i16::MAX, + ); + let c = i16x8::new( + i16::MIN, + i16::MIN, + i16::MIN, + i16::MIN, + i16::MIN, + i16::MIN, + i16::MIN, + i16::MIN, + ); + let e = i16x8::new( + i16::MAX, + i16::MIN, + i16::MAX, + i16::MIN, + i16::MAX, + i16::MIN, + i16::MAX, + i16::MIN, + ); + let r: i16x8 = transmute(vbslq_s16(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vbslq_s32() { + let a = u32x4::new(u32::MAX, 0, u32::MAX, 0); + let b = i32x4::new(i32::MAX, i32::MAX, i32::MAX, i32::MAX); + let c = i32x4::new(i32::MIN, i32::MIN, i32::MIN, i32::MIN); + let e = i32x4::new(i32::MAX, i32::MIN, i32::MAX, i32::MIN); + let r: i32x4 = transmute(vbslq_s32(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vbslq_s64() { + let a = u64x2::new(u64::MAX, 0); + let b = i64x2::new(i64::MAX, i64::MAX); + let c = i64x2::new(i64::MIN, i64::MIN); + let e = i64x2::new(i64::MAX, i64::MIN); + let r: i64x2 = transmute(vbslq_s64(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vbslq_u8() { + let a = u8x16::new( + u8::MAX, + 0, + u8::MAX, + 0, + u8::MAX, + 0, + u8::MAX, + 0, + u8::MAX, + 0, + u8::MAX, + 0, + u8::MAX, + 0, + u8::MAX, + 0, + ); + let b = u8x16::new( + u8::MAX, + u8::MAX, + u8::MAX, + u8::MAX, + u8::MAX, + u8::MAX, + u8::MAX, + u8::MAX, + u8::MAX, + u8::MAX, + u8::MAX, + u8::MAX, + u8::MAX, + u8::MAX, + u8::MAX, + u8::MAX, + ); + let c = u8x16::new( + u8::MIN, + u8::MIN, + u8::MIN, + u8::MIN, + u8::MIN, + u8::MIN, + u8::MIN, + u8::MIN, + u8::MIN, + u8::MIN, + u8::MIN, + u8::MIN, + u8::MIN, + u8::MIN, + u8::MIN, + u8::MIN, + ); + let e = u8x16::new( + u8::MAX, + u8::MIN, + u8::MAX, + u8::MIN, + u8::MAX, + u8::MIN, + u8::MAX, + u8::MIN, + u8::MAX, + u8::MIN, + u8::MAX, + u8::MIN, + u8::MAX, + u8::MIN, + u8::MAX, + u8::MIN, + ); + let r: u8x16 = transmute(vbslq_u8(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vbslq_u16() { + let a = u16x8::new(u16::MAX, 0, u16::MAX, 0, u16::MAX, 0, u16::MAX, 0); + let b = u16x8::new( + u16::MAX, + u16::MAX, + u16::MAX, + u16::MAX, + u16::MAX, + u16::MAX, + u16::MAX, + u16::MAX, + ); + let c = u16x8::new( + u16::MIN, + u16::MIN, + u16::MIN, + u16::MIN, + u16::MIN, + u16::MIN, + u16::MIN, + u16::MIN, + ); + let e = u16x8::new( + u16::MAX, + u16::MIN, + u16::MAX, + u16::MIN, + u16::MAX, + u16::MIN, + u16::MAX, + u16::MIN, + ); + let r: u16x8 = transmute(vbslq_u16(transmute(a), transmute(b), transmute(c))); assert_eq!(r, e); } - #[simd_test(enable = "neon")] - unsafe fn test_vpadal_u8() { - let a = u16x4::new(42, 42, 42, 42); - let b = u8x8::new(0, 1, 2, 3, 4, 5, 6, u8::MAX); - let r: u16x4 = transmute(vpadal_u8(transmute(a), transmute(b))); - let e = u16x4::new(43, 47, 51, u8::MAX as u16 + 48); + unsafe fn test_vbslq_u32() { + let a = u32x4::new(u32::MAX, 0, u32::MAX, 0); + let b = u32x4::new(u32::MAX, u32::MAX, u32::MAX, u32::MAX); + let c = u32x4::new(u32::MIN, u32::MIN, u32::MIN, u32::MIN); + let e = u32x4::new(u32::MAX, u32::MIN, u32::MAX, u32::MIN); + let r: u32x4 = transmute(vbslq_u32(transmute(a), transmute(b), transmute(c))); assert_eq!(r, e); } - #[simd_test(enable = "neon")] - unsafe fn test_vpadal_u16() { - let a = u32x2::new(42, 42); - let b = u16x4::new(0, 1, 2, u16::MAX); - let r: u32x2 = transmute(vpadal_u16(transmute(a), transmute(b))); - let e = u32x2::new(43, u16::MAX as u32 + 44); + unsafe fn test_vbslq_u64() { + let a = u64x2::new(u64::MAX, 0); + let b = u64x2::new(u64::MAX, u64::MAX); + let c = u64x2::new(u64::MIN, u64::MIN); + let e = u64x2::new(u64::MAX, u64::MIN); + let r: u64x2 = transmute(vbslq_u64(transmute(a), transmute(b), transmute(c))); assert_eq!(r, e); } - #[simd_test(enable = "neon")] - unsafe fn test_vpadal_u32() { - let a = u64x1::new(42); - let b = u32x2::new(1, u32::MAX); - let r: u64x1 = transmute(vpadal_u32(transmute(a), transmute(b))); - let e = u64x1::new(u32::MAX as u64 + 43); + unsafe fn test_vbslq_f32() { + let a = u32x4::new(u32::MAX, 0, u32::MAX, 0); + let b = f32x4::new(f32::MAX, f32::MAX, f32::MAX, f32::MAX); + let c = f32x4::new(f32::MIN, f32::MIN, f32::MIN, f32::MIN); + let e = f32x4::new(f32::MAX, f32::MIN, f32::MAX, f32::MIN); + let r: f32x4 = transmute(vbslq_f32(transmute(a), transmute(b), transmute(c))); assert_eq!(r, e); } - #[simd_test(enable = "neon")] - unsafe fn test_vpadalq_u8() { - let a = u16x8::new(42, 42, 42, 42, 42, 42, 42, 42); - let b = u8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, u8::MAX); - let r: u16x8 = transmute(vpadalq_u8(transmute(a), transmute(b))); - let e = u16x8::new(43, 47, 51, 55, 59, 63, 67, u8::MAX as u16 + 56); + unsafe fn test_vbslq_p8() { + let a = u8x16::new( + u8::MAX, + 0, + u8::MAX, + 0, + u8::MAX, + 0, + u8::MAX, + 0, + u8::MAX, + 0, + u8::MAX, + 0, + u8::MAX, + 0, + u8::MAX, + 0, + ); + let b = u8x16::new( + u8::MAX, + u8::MAX, + u8::MAX, + u8::MAX, + u8::MAX, + u8::MAX, + u8::MAX, + u8::MAX, + u8::MAX, + u8::MAX, + u8::MAX, + u8::MAX, + u8::MAX, + u8::MAX, + u8::MAX, + u8::MAX, + ); + let c = u8x16::new( + u8::MIN, + u8::MIN, + u8::MIN, + u8::MIN, + u8::MIN, + u8::MIN, + u8::MIN, + u8::MIN, + u8::MIN, + u8::MIN, + u8::MIN, + u8::MIN, + u8::MIN, + u8::MIN, + u8::MIN, + u8::MIN, + ); + let e = u8x16::new( + u8::MAX, + u8::MIN, + u8::MAX, + u8::MIN, + u8::MAX, + u8::MIN, + u8::MAX, + u8::MIN, + u8::MAX, + u8::MIN, + u8::MAX, + u8::MIN, + u8::MAX, + u8::MIN, + u8::MAX, + u8::MIN, + ); + let r: u8x16 = transmute(vbslq_p8(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vbslq_p16() { + let a = u16x8::new(u16::MAX, 0, u16::MAX, 0, u16::MAX, 0, u16::MAX, 0); + let b = u16x8::new( + u16::MAX, + u16::MAX, + u16::MAX, + u16::MAX, + u16::MAX, + u16::MAX, + u16::MAX, + u16::MAX, + ); + let c = u16x8::new( + u16::MIN, + u16::MIN, + u16::MIN, + u16::MIN, + u16::MIN, + u16::MIN, + u16::MIN, + u16::MIN, + ); + let e = u16x8::new( + u16::MAX, + u16::MIN, + u16::MAX, + u16::MIN, + u16::MAX, + u16::MIN, + u16::MAX, + u16::MIN, + ); + let r: u16x8 = transmute(vbslq_p16(transmute(a), transmute(b), transmute(c))); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vpadalq_u16() { - let a = u32x4::new(42, 42, 42, 42); - let b = u16x8::new(0, 1, 2, 3, 4, 5, 6, u16::MAX); - let r: u32x4 = transmute(vpadalq_u16(transmute(a), transmute(b))); - let e = u32x4::new(43, 47, 51, u16::MAX as u32 + 48); + unsafe fn test_vorn_s8() { + let a = i8x8::new(0, -1, -2, -3, -4, -5, -6, -7); + let b = i8x8::new(-2, -2, -2, -2, -2, -2, -2, -2); + let e = i8x8::new(1, -1, -1, -3, -3, -5, -5, -7); + let r: i8x8 = transmute(vorn_s8(transmute(a), transmute(b))); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vpadalq_u32() { - let a = u64x2::new(42, 42); - let b = u32x4::new(0, 1, 2, u32::MAX); - let r: u64x2 = transmute(vpadalq_u32(transmute(a), transmute(b))); - let e = u64x2::new(43, u32::MAX as u64 + 44); + unsafe fn test_vornq_s8() { + let a = i8x16::new( + 0, -1, -2, -3, -4, -5, -6, -7, -8, -9, -10, -11, -12, -13, -14, -15, + ); + let b = i8x16::new( + -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, + ); + let e = i8x16::new( + 1, -1, -1, -3, -3, -5, -5, -7, -7, -9, -9, -11, -11, -13, -13, -15, + ); + let r: i8x16 = transmute(vornq_s8(transmute(a), transmute(b))); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vmvn_s8() { - let a = i8x8::new(0, 1, 2, 3, 4, 5, 6, 7); - let e = i8x8::new(-1, -2, -3, -4, -5, -6, -7, -8); - let r: i8x8 = transmute(vmvn_s8(transmute(a))); + unsafe fn test_vorn_s16() { + let a = i16x4::new(0, -1, -2, -3); + let b = i16x4::new(-2, -2, -2, -2); + let e = i16x4::new(1, -1, -1, -3); + let r: i16x4 = transmute(vorn_s16(transmute(a), transmute(b))); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vmvnq_s8() { - let a = i8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); - let e = i8x16::new( - -1, -2, -3, -4, -5, -6, -7, -8, -9, -10, -11, -12, -13, -14, -15, -16, - ); - let r: i8x16 = transmute(vmvnq_s8(transmute(a))); + unsafe fn test_vornq_s16() { + let a = i16x8::new(0, -1, -2, -3, -4, -5, -6, -7); + let b = i16x8::new(-2, -2, -2, -2, -2, -2, -2, -2); + let e = i16x8::new(1, -1, -1, -3, -3, -5, -5, -7); + let r: i16x8 = transmute(vornq_s16(transmute(a), transmute(b))); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vmvn_s16() { - let a = i16x4::new(0, 1, 2, 3); - let e = i16x4::new(-1, -2, -3, -4); - let r: i16x4 = transmute(vmvn_s16(transmute(a))); + unsafe fn test_vorn_s32() { + let a = i32x2::new(0, -1); + let b = i32x2::new(-2, -2); + let e = i32x2::new(1, -1); + let r: i32x2 = transmute(vorn_s32(transmute(a), transmute(b))); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vmvnq_s16() { - let a = i16x8::new(0, 1, 2, 3, 4, 5, 6, 7); - let e = i16x8::new(-1, -2, -3, -4, -5, -6, -7, -8); - let r: i16x8 = transmute(vmvnq_s16(transmute(a))); + unsafe fn test_vornq_s32() { + let a = i32x4::new(0, -1, -2, -3); + let b = i32x4::new(-2, -2, -2, -2); + let e = i32x4::new(1, -1, -1, -3); + let r: i32x4 = transmute(vornq_s32(transmute(a), transmute(b))); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vmvn_s32() { - let a = i32x2::new(0, 1); - let e = i32x2::new(-1, -2); - let r: i32x2 = transmute(vmvn_s32(transmute(a))); + unsafe fn test_vorn_s64() { + let a = i64x1::new(0); + let b = i64x1::new(-2); + let e = i64x1::new(1); + let r: i64x1 = transmute(vorn_s64(transmute(a), transmute(b))); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vmvnq_s32() { - let a = i32x4::new(0, 1, 2, 3); - let e = i32x4::new(-1, -2, -3, -4); - let r: i32x4 = transmute(vmvnq_s32(transmute(a))); + unsafe fn test_vornq_s64() { + let a = i64x2::new(0, -1); + let b = i64x2::new(-2, -2); + let e = i64x2::new(1, -1); + let r: i64x2 = transmute(vornq_s64(transmute(a), transmute(b))); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vmvn_u8() { + unsafe fn test_vorn_u8() { let a = u8x8::new(0, 1, 2, 3, 4, 5, 6, 7); - let e = u8x8::new(255, 254, 253, 252, 251, 250, 249, 248); - let r: u8x8 = transmute(vmvn_u8(transmute(a))); + let t = u8::MAX - 1; + let b = u8x8::new(t, t, t, t, t, t, t, t); + let e = u8x8::new(1, 1, 3, 3, 5, 5, 7, 7); + let r: u8x8 = transmute(vorn_u8(transmute(a), transmute(b))); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vmvnq_u8() { + unsafe fn test_vornq_u8() { let a = u8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); - let e = u8x16::new( - 255, 254, 253, 252, 251, 250, 249, 248, 247, 246, 245, 244, 243, 242, 241, 240, - ); - let r: u8x16 = transmute(vmvnq_u8(transmute(a))); + let t = u8::MAX - 1; + let b = u8x16::new(t, t, t, t, t, t, t, t, t, t, t, t, t, t, t, t); + let e = u8x16::new(1, 1, 3, 3, 5, 5, 7, 7, 9, 9, 11, 11, 13, 13, 15, 15); + let r: u8x16 = transmute(vornq_u8(transmute(a), transmute(b))); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vmvn_u16() { + unsafe fn test_vorn_u16() { let a = u16x4::new(0, 1, 2, 3); - let e = u16x4::new(65_535, 65_534, 65_533, 65_532); - let r: u16x4 = transmute(vmvn_u16(transmute(a))); + let t = u16::MAX - 1; + let b = u16x4::new(t, t, t, t); + let e = u16x4::new(1, 1, 3, 3); + let r: u16x4 = transmute(vorn_u16(transmute(a), transmute(b))); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vmvnq_u16() { + unsafe fn test_vornq_u16() { let a = u16x8::new(0, 1, 2, 3, 4, 5, 6, 7); - let e = u16x8::new( - 65_535, 65_534, 65_533, 65_532, 65_531, 65_530, 65_529, 65_528, - ); - let r: u16x8 = transmute(vmvnq_u16(transmute(a))); + let t = u16::MAX - 1; + let b = u16x8::new(t, t, t, t, t, t, t, t); + let e = u16x8::new(1, 1, 3, 3, 5, 5, 7, 7); + let r: u16x8 = transmute(vornq_u16(transmute(a), transmute(b))); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vmvn_u32() { + unsafe fn test_vorn_u32() { let a = u32x2::new(0, 1); - let e = u32x2::new(4_294_967_295, 4_294_967_294); - let r: u32x2 = transmute(vmvn_u32(transmute(a))); + let t = u32::MAX - 1; + let b = u32x2::new(t, t); + let e = u32x2::new(1, 1); + let r: u32x2 = transmute(vorn_u32(transmute(a), transmute(b))); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vmvnq_u32() { + unsafe fn test_vornq_u32() { let a = u32x4::new(0, 1, 2, 3); - let e = u32x4::new(4_294_967_295, 4_294_967_294, 4_294_967_293, 4_294_967_292); - let r: u32x4 = transmute(vmvnq_u32(transmute(a))); + let t = u32::MAX - 1; + let b = u32x4::new(t, t, t, t); + let e = u32x4::new(1, 1, 3, 3); + let r: u32x4 = transmute(vornq_u32(transmute(a), transmute(b))); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vmvn_p8() { - let a = u8x8::new(0, 1, 2, 3, 4, 5, 6, 7); - let e = u8x8::new(255, 254, 253, 252, 251, 250, 249, 248); - let r: u8x8 = transmute(vmvn_p8(transmute(a))); + unsafe fn test_vorn_u64() { + let a = u64x1::new(0); + let t = u64::MAX - 1; + let b = u64x1::new(t); + let e = u64x1::new(1); + let r: u64x1 = transmute(vorn_u64(transmute(a), transmute(b))); assert_eq!(r, e); } #[simd_test(enable = "neon")] - unsafe fn test_vmvnq_p8() { - let a = u8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); - let e = u8x16::new( - 255, 254, 253, 252, 251, 250, 249, 248, 247, 246, 245, 244, 243, 242, 241, 240, - ); - let r: u8x16 = transmute(vmvnq_p8(transmute(a))); + unsafe fn test_vornq_u64() { + let a = u64x2::new(0, 1); + let t = u64::MAX - 1; + let b = u64x2::new(t, t); + let e = u64x2::new(1, 1); + let r: u64x2 = transmute(vornq_u64(transmute(a), transmute(b))); assert_eq!(r, e); } @@ -6102,14 +7702,6 @@ mod tests { assert_eq!(r, e); } - #[simd_test(enable = "neon")] - unsafe fn test_vrsqrt_f32() { - let a = f32x2::new(1.0, 2.0); - let e = f32x2::new(0.9980469, 0.7050781); - let r: f32x2 = transmute(vrsqrte_f32(transmute(a))); - assert_eq!(r, e); - } - #[simd_test(enable = "neon")] unsafe fn test_vpmin_s8() { let a = i8x8::new(1, -2, 3, -4, 5, 6, 7, 8); @@ -7871,43 +9463,6 @@ mod tests { ); } - #[simd_test(enable = "neon")] - unsafe fn test_vreinterpretq_s8_u8() { - let a = i8x16::new(-1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); - let r: u8x16 = transmute(vreinterpretq_s8_u8(transmute(a))); - let e = u8x16::new(0xFF, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); - assert_eq!(r, e) - } - #[simd_test(enable = "neon")] - unsafe fn test_vreinterpretq_u16_u8() { - let a = u16x8::new( - 0x01_00, 0x03_02, 0x05_04, 0x07_06, 0x09_08, 0x0B_0A, 0x0D_0C, 0x0F_0E, - ); - let r: u8x16 = transmute(vreinterpretq_u16_u8(transmute(a))); - let e = u8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); - assert_eq!(r, e) - } - #[simd_test(enable = "neon")] - unsafe fn test_vreinterpretq_u32_u8() { - let a = u32x4::new(0x03_02_01_00, 0x07_06_05_04, 0x0B_0A_09_08, 0x0F_0E_0D_0C); - let r: u8x16 = transmute(vreinterpretq_u32_u8(transmute(a))); - let e = u8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); - assert_eq!(r, e) - } - #[simd_test(enable = "neon")] - unsafe fn test_vreinterpretq_u64_u8() { - let a: u64x2 = u64x2::new(0x07_06_05_04_03_02_01_00, 0x0F_0E_0D_0C_0B_0A_09_08); - let r: u8x16 = transmute(vreinterpretq_u64_u8(transmute(a))); - let e = u8x16::new(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); - assert_eq!(r, e) - } - #[simd_test(enable = "neon")] - unsafe fn test_vreinterpretq_u8_s8() { - let a = u8x16::new(0xFF, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); - let r: i8x16 = transmute(vreinterpretq_u8_s8(transmute(a))); - let e = i8x16::new(-1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); - assert_eq!(r, e) - } #[simd_test(enable = "neon")] unsafe fn test_vabs_s8() { let a = i8x8::new(-1, 0, 1, -2, 0, 2, -128, 127); @@ -7950,6 +9505,120 @@ mod tests { let e = i32x4::new(i32::MIN, i32::MAX, 0, 1); assert_eq!(r, e); } + + #[simd_test(enable = "neon")] + unsafe fn test_vaba_s8() { + let a = i8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let b = i8x8::new(1, 1, 1, 1, 1, 1, 1, 1); + let c = i8x8::new(10, 9, 8, 7, 6, 5, 4, 3); + let r: i8x8 = transmute(vaba_s8(transmute(a), transmute(b), transmute(c))); + let e = i8x8::new(10, 10, 10, 10, 10, 10, 10, 10); + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vaba_s16() { + let a = i16x4::new(1, 2, 3, 4); + let b = i16x4::new(1, 1, 1, 1); + let c = i16x4::new(10, 9, 8, 7); + let r: i16x4 = transmute(vaba_s16(transmute(a), transmute(b), transmute(c))); + let e = i16x4::new(10, 10, 10, 10); + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vaba_s32() { + let a = i32x2::new(1, 2); + let b = i32x2::new(1, 1); + let c = i32x2::new(10, 9); + let r: i32x2 = transmute(vaba_s32(transmute(a), transmute(b), transmute(c))); + let e = i32x2::new(10, 10); + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vaba_u8() { + let a = u8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let b = u8x8::new(1, 1, 1, 1, 1, 1, 1, 1); + let c = u8x8::new(10, 9, 8, 7, 6, 5, 4, 3); + let r: u8x8 = transmute(vaba_u8(transmute(a), transmute(b), transmute(c))); + let e = u8x8::new(10, 10, 10, 10, 10, 10, 10, 10); + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vaba_u16() { + let a = u16x4::new(1, 2, 3, 4); + let b = u16x4::new(1, 1, 1, 1); + let c = u16x4::new(10, 9, 8, 7); + let r: u16x4 = transmute(vaba_u16(transmute(a), transmute(b), transmute(c))); + let e = u16x4::new(10, 10, 10, 10); + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vaba_u32() { + let a = u32x2::new(1, 2); + let b = u32x2::new(1, 1); + let c = u32x2::new(10, 9); + let r: u32x2 = transmute(vaba_u32(transmute(a), transmute(b), transmute(c))); + let e = u32x2::new(10, 10); + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vabaq_s8() { + let a = i8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 8, 7, 6, 5, 4, 3, 2); + let b = i8x16::new(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1); + let c = i8x16::new(10, 9, 8, 7, 6, 5, 4, 3, 12, 13, 14, 15, 16, 17, 18, 19); + let r: i8x16 = transmute(vabaq_s8(transmute(a), transmute(b), transmute(c))); + let e = i8x16::new( + 10, 10, 10, 10, 10, 10, 10, 10, 20, 20, 20, 20, 20, 20, 20, 20, + ); + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vabaq_s16() { + let a = i16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let b = i16x8::new(1, 1, 1, 1, 1, 1, 1, 1); + let c = i16x8::new(10, 9, 8, 7, 6, 5, 4, 3); + let r: i16x8 = transmute(vabaq_s16(transmute(a), transmute(b), transmute(c))); + let e = i16x8::new(10, 10, 10, 10, 10, 10, 10, 10); + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vabaq_s32() { + let a = i32x4::new(1, 2, 3, 4); + let b = i32x4::new(1, 1, 1, 1); + let c = i32x4::new(10, 9, 8, 7); + let r: i32x4 = transmute(vabaq_s32(transmute(a), transmute(b), transmute(c))); + let e = i32x4::new(10, 10, 10, 10); + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vabaq_u8() { + let a = u8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 8, 7, 6, 5, 4, 3, 2); + let b = u8x16::new(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1); + let c = u8x16::new(10, 9, 8, 7, 6, 5, 4, 3, 12, 13, 14, 15, 16, 17, 18, 19); + let r: u8x16 = transmute(vabaq_u8(transmute(a), transmute(b), transmute(c))); + let e = u8x16::new( + 10, 10, 10, 10, 10, 10, 10, 10, 20, 20, 20, 20, 20, 20, 20, 20, + ); + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vabaq_u16() { + let a = u16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + let b = u16x8::new(1, 1, 1, 1, 1, 1, 1, 1); + let c = u16x8::new(10, 9, 8, 7, 6, 5, 4, 3); + let r: u16x8 = transmute(vabaq_u16(transmute(a), transmute(b), transmute(c))); + let e = u16x8::new(10, 10, 10, 10, 10, 10, 10, 10); + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vabaq_u32() { + let a = u32x4::new(1, 2, 3, 4); + let b = u32x4::new(1, 1, 1, 1); + let c = u32x4::new(10, 9, 8, 7); + let r: u32x4 = transmute(vabaq_u32(transmute(a), transmute(b), transmute(c))); + let e = u32x4::new(10, 10, 10, 10); + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] unsafe fn test_vpadd_s16() { let a = i16x4::new(1, 2, 3, 4); @@ -8061,6 +9730,7 @@ mod tests { let r: u8x16 = transmute(vcntq_p8(transmute(a))); assert_eq!(r, e); } + #[simd_test(enable = "neon")] unsafe fn test_vrev16_s8() { let a = i8x8::new(0, 1, 2, 3, 4, 5, 6, 7); let r = i8x8::new(1, 0, 3, 2, 5, 4, 7, 6); @@ -8131,6 +9801,34 @@ mod tests { assert_eq!(r, e); } #[simd_test(enable = "neon")] + unsafe fn test_vrev32_s16() { + let a = i16x4::new(0, 1, 2, 3); + let r = i16x4::new(1, 0, 3, 2); + let e: i16x4 = transmute(vrev32_s16(transmute(a))); + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vrev32q_s16() { + let a = i16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let r = i16x8::new(1, 0, 3, 2, 5, 4, 7, 6); + let e: i16x8 = transmute(vrev32q_s16(transmute(a))); + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vrev32_p16() { + let a = i16x4::new(0, 1, 2, 3); + let r = i16x4::new(1, 0, 3, 2); + let e: i16x4 = transmute(vrev32_p16(transmute(a))); + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vrev32q_p16() { + let a = i16x8::new(0, 1, 2, 3, 4, 5, 6, 7); + let r = i16x8::new(1, 0, 3, 2, 5, 4, 7, 6); + let e: i16x8 = transmute(vrev32q_p16(transmute(a))); + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] unsafe fn test_vrev32_u16() { let a = u16x4::new(0, 1, 2, 3); let r = u16x4::new(1, 0, 3, 2); @@ -8294,3 +9992,6 @@ mod shift_and_insert_tests; #[cfg(all(test, target_arch = "arm"))] mod load_tests; + +#[cfg(all(test, target_arch = "arm"))] +mod store_tests; diff --git a/library/stdarch/crates/core_arch/src/arm/neon/shift_and_insert_tests.rs b/library/stdarch/crates/core_arch/src/arm_shared/neon/shift_and_insert_tests.rs similarity index 98% rename from library/stdarch/crates/core_arch/src/arm/neon/shift_and_insert_tests.rs rename to library/stdarch/crates/core_arch/src/arm_shared/neon/shift_and_insert_tests.rs index a556789245..125659c105 100644 --- a/library/stdarch/crates/core_arch/src/arm/neon/shift_and_insert_tests.rs +++ b/library/stdarch/crates/core_arch/src/arm_shared/neon/shift_and_insert_tests.rs @@ -23,7 +23,7 @@ macro_rules! test_vsli { let b = [$($b as $t),*]; let n_bit_mask: $t = (1 << $n) - 1; let e = [$(($a as $t & n_bit_mask) | ($b as $t << $n)),*]; - let r = $fn_id(transmute(a), transmute(b), $n); + let r = $fn_id::<$n>(transmute(a), transmute(b)); let mut d = e; d = transmute(r); assert_eq!(d, e); @@ -60,7 +60,7 @@ macro_rules! test_vsri { let b = [$($b as $t),*]; let n_bit_mask = ((1 as $t << $n) - 1).rotate_right($n); let e = [$(($a as $t & n_bit_mask) | (($b as $t >> $n) & !n_bit_mask)),*]; - let r = $fn_id(transmute(a), transmute(b), $n); + let r = $fn_id::<$n>(transmute(a), transmute(b)); let mut d = e; d = transmute(r); assert_eq!(d, e); diff --git a/library/stdarch/crates/core_arch/src/arm_shared/neon/store_tests.rs b/library/stdarch/crates/core_arch/src/arm_shared/neon/store_tests.rs new file mode 100644 index 0000000000..c1e355fd9a --- /dev/null +++ b/library/stdarch/crates/core_arch/src/arm_shared/neon/store_tests.rs @@ -0,0 +1,366 @@ +//! Tests for ARM+v7+neon store (vst1) intrinsics. +//! +//! These are included in `{arm, aarch64}::neon`. + +use super::*; + +#[cfg(target_arch = "arm")] +use crate::core_arch::arm::*; + +#[cfg(target_arch = "aarch64")] +use crate::core_arch::aarch64::*; + +use crate::core_arch::simd::*; +use stdarch_test::simd_test; + +#[simd_test(enable = "neon")] +unsafe fn test_vst1_s8() { + let mut vals = [0_i8; 9]; + let a = i8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + + vst1_s8(vals[1..].as_mut_ptr(), transmute(a)); + + assert_eq!(vals[0], 0); + assert_eq!(vals[1], 1); + assert_eq!(vals[2], 2); + assert_eq!(vals[3], 3); + assert_eq!(vals[4], 4); + assert_eq!(vals[5], 5); + assert_eq!(vals[6], 6); + assert_eq!(vals[7], 7); + assert_eq!(vals[8], 8); +} + +#[simd_test(enable = "neon")] +unsafe fn test_vst1q_s8() { + let mut vals = [0_i8; 17]; + let a = i8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + + vst1q_s8(vals[1..].as_mut_ptr(), transmute(a)); + + assert_eq!(vals[0], 0); + assert_eq!(vals[1], 1); + assert_eq!(vals[2], 2); + assert_eq!(vals[3], 3); + assert_eq!(vals[4], 4); + assert_eq!(vals[5], 5); + assert_eq!(vals[6], 6); + assert_eq!(vals[7], 7); + assert_eq!(vals[8], 8); + assert_eq!(vals[9], 9); + assert_eq!(vals[10], 10); + assert_eq!(vals[11], 11); + assert_eq!(vals[12], 12); + assert_eq!(vals[13], 13); + assert_eq!(vals[14], 14); + assert_eq!(vals[15], 15); + assert_eq!(vals[16], 16); +} + +#[simd_test(enable = "neon")] +unsafe fn test_vst1_s16() { + let mut vals = [0_i16; 5]; + let a = i16x4::new(1, 2, 3, 4); + + vst1_s16(vals[1..].as_mut_ptr(), transmute(a)); + + assert_eq!(vals[0], 0); + assert_eq!(vals[1], 1); + assert_eq!(vals[2], 2); + assert_eq!(vals[3], 3); + assert_eq!(vals[4], 4); +} + +#[simd_test(enable = "neon")] +unsafe fn test_vst1q_s16() { + let mut vals = [0_i16; 9]; + let a = i16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + + vst1q_s16(vals[1..].as_mut_ptr(), transmute(a)); + + assert_eq!(vals[0], 0); + assert_eq!(vals[1], 1); + assert_eq!(vals[2], 2); + assert_eq!(vals[3], 3); + assert_eq!(vals[4], 4); + assert_eq!(vals[5], 5); + assert_eq!(vals[6], 6); + assert_eq!(vals[7], 7); + assert_eq!(vals[8], 8); +} + +#[simd_test(enable = "neon")] +unsafe fn test_vst1_s32() { + let mut vals = [0_i32; 3]; + let a = i32x2::new(1, 2); + + vst1_s32(vals[1..].as_mut_ptr(), transmute(a)); + + assert_eq!(vals[0], 0); + assert_eq!(vals[1], 1); + assert_eq!(vals[2], 2); +} + +#[simd_test(enable = "neon")] +unsafe fn test_vst1q_s32() { + let mut vals = [0_i32; 5]; + let a = i32x4::new(1, 2, 3, 4); + + vst1q_s32(vals[1..].as_mut_ptr(), transmute(a)); + + assert_eq!(vals[0], 0); + assert_eq!(vals[1], 1); + assert_eq!(vals[2], 2); + assert_eq!(vals[3], 3); + assert_eq!(vals[4], 4); +} + +#[simd_test(enable = "neon")] +unsafe fn test_vst1_s64() { + let mut vals = [0_i64; 2]; + let a = i64x1::new(1); + + vst1_s64(vals[1..].as_mut_ptr(), transmute(a)); + + assert_eq!(vals[0], 0); + assert_eq!(vals[1], 1); +} + +#[simd_test(enable = "neon")] +unsafe fn test_vst1q_s64() { + let mut vals = [0_i64; 3]; + let a = i64x2::new(1, 2); + + vst1q_s64(vals[1..].as_mut_ptr(), transmute(a)); + + assert_eq!(vals[0], 0); + assert_eq!(vals[1], 1); + assert_eq!(vals[2], 2); +} + +#[simd_test(enable = "neon")] +unsafe fn test_vst1_u8() { + let mut vals = [0_u8; 9]; + let a = u8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + + vst1_u8(vals[1..].as_mut_ptr(), transmute(a)); + + assert_eq!(vals[0], 0); + assert_eq!(vals[1], 1); + assert_eq!(vals[2], 2); + assert_eq!(vals[3], 3); + assert_eq!(vals[4], 4); + assert_eq!(vals[5], 5); + assert_eq!(vals[6], 6); + assert_eq!(vals[7], 7); + assert_eq!(vals[8], 8); +} + +#[simd_test(enable = "neon")] +unsafe fn test_vst1q_u8() { + let mut vals = [0_u8; 17]; + let a = u8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + + vst1q_u8(vals[1..].as_mut_ptr(), transmute(a)); + + assert_eq!(vals[0], 0); + assert_eq!(vals[1], 1); + assert_eq!(vals[2], 2); + assert_eq!(vals[3], 3); + assert_eq!(vals[4], 4); + assert_eq!(vals[5], 5); + assert_eq!(vals[6], 6); + assert_eq!(vals[7], 7); + assert_eq!(vals[8], 8); + assert_eq!(vals[9], 9); + assert_eq!(vals[10], 10); + assert_eq!(vals[11], 11); + assert_eq!(vals[12], 12); + assert_eq!(vals[13], 13); + assert_eq!(vals[14], 14); + assert_eq!(vals[15], 15); + assert_eq!(vals[16], 16); +} + +#[simd_test(enable = "neon")] +unsafe fn test_vst1_u16() { + let mut vals = [0_u16; 5]; + let a = u16x4::new(1, 2, 3, 4); + + vst1_u16(vals[1..].as_mut_ptr(), transmute(a)); + + assert_eq!(vals[0], 0); + assert_eq!(vals[1], 1); + assert_eq!(vals[2], 2); + assert_eq!(vals[3], 3); + assert_eq!(vals[4], 4); +} + +#[simd_test(enable = "neon")] +unsafe fn test_vst1q_u16() { + let mut vals = [0_u16; 9]; + let a = u16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + + vst1q_u16(vals[1..].as_mut_ptr(), transmute(a)); + + assert_eq!(vals[0], 0); + assert_eq!(vals[1], 1); + assert_eq!(vals[2], 2); + assert_eq!(vals[3], 3); + assert_eq!(vals[4], 4); + assert_eq!(vals[5], 5); + assert_eq!(vals[6], 6); + assert_eq!(vals[7], 7); + assert_eq!(vals[8], 8); +} + +#[simd_test(enable = "neon")] +unsafe fn test_vst1_u32() { + let mut vals = [0_u32; 3]; + let a = u32x2::new(1, 2); + + vst1_u32(vals[1..].as_mut_ptr(), transmute(a)); + + assert_eq!(vals[0], 0); + assert_eq!(vals[1], 1); + assert_eq!(vals[2], 2); +} + +#[simd_test(enable = "neon")] +unsafe fn test_vst1q_u32() { + let mut vals = [0_u32; 5]; + let a = u32x4::new(1, 2, 3, 4); + + vst1q_u32(vals[1..].as_mut_ptr(), transmute(a)); + + assert_eq!(vals[0], 0); + assert_eq!(vals[1], 1); + assert_eq!(vals[2], 2); + assert_eq!(vals[3], 3); + assert_eq!(vals[4], 4); +} + +#[simd_test(enable = "neon")] +unsafe fn test_vst1_u64() { + let mut vals = [0_u64; 2]; + let a = u64x1::new(1); + + vst1_u64(vals[1..].as_mut_ptr(), transmute(a)); + + assert_eq!(vals[0], 0); + assert_eq!(vals[1], 1); +} + +#[simd_test(enable = "neon")] +unsafe fn test_vst1q_u64() { + let mut vals = [0_u64; 3]; + let a = u64x2::new(1, 2); + + vst1q_u64(vals[1..].as_mut_ptr(), transmute(a)); + + assert_eq!(vals[0], 0); + assert_eq!(vals[1], 1); + assert_eq!(vals[2], 2); +} + +#[simd_test(enable = "neon")] +unsafe fn test_vst1_p8() { + let mut vals = [0_u8; 9]; + let a = u8x8::new(1, 2, 3, 4, 5, 6, 7, 8); + + vst1_p8(vals[1..].as_mut_ptr(), transmute(a)); + + assert_eq!(vals[0], 0); + assert_eq!(vals[1], 1); + assert_eq!(vals[2], 2); + assert_eq!(vals[3], 3); + assert_eq!(vals[4], 4); + assert_eq!(vals[5], 5); + assert_eq!(vals[6], 6); + assert_eq!(vals[7], 7); + assert_eq!(vals[8], 8); +} + +#[simd_test(enable = "neon")] +unsafe fn test_vst1q_p8() { + let mut vals = [0_u8; 17]; + let a = u8x16::new(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + + vst1q_p8(vals[1..].as_mut_ptr(), transmute(a)); + + assert_eq!(vals[0], 0); + assert_eq!(vals[1], 1); + assert_eq!(vals[2], 2); + assert_eq!(vals[3], 3); + assert_eq!(vals[4], 4); + assert_eq!(vals[5], 5); + assert_eq!(vals[6], 6); + assert_eq!(vals[7], 7); + assert_eq!(vals[8], 8); + assert_eq!(vals[9], 9); + assert_eq!(vals[10], 10); + assert_eq!(vals[11], 11); + assert_eq!(vals[12], 12); + assert_eq!(vals[13], 13); + assert_eq!(vals[14], 14); + assert_eq!(vals[15], 15); + assert_eq!(vals[16], 16); +} + +#[simd_test(enable = "neon")] +unsafe fn test_vst1_p16() { + let mut vals = [0_u16; 5]; + let a = u16x4::new(1, 2, 3, 4); + + vst1_p16(vals[1..].as_mut_ptr(), transmute(a)); + + assert_eq!(vals[0], 0); + assert_eq!(vals[1], 1); + assert_eq!(vals[2], 2); + assert_eq!(vals[3], 3); + assert_eq!(vals[4], 4); +} + +#[simd_test(enable = "neon")] +unsafe fn test_vst1q_p16() { + let mut vals = [0_u16; 9]; + let a = u16x8::new(1, 2, 3, 4, 5, 6, 7, 8); + + vst1q_p16(vals[1..].as_mut_ptr(), transmute(a)); + + assert_eq!(vals[0], 0); + assert_eq!(vals[1], 1); + assert_eq!(vals[2], 2); + assert_eq!(vals[3], 3); + assert_eq!(vals[4], 4); + assert_eq!(vals[5], 5); + assert_eq!(vals[6], 6); + assert_eq!(vals[7], 7); + assert_eq!(vals[8], 8); +} + +#[simd_test(enable = "neon")] +unsafe fn test_vst1_f32() { + let mut vals = [0_f32; 3]; + let a = f32x2::new(1., 2.); + + vst1_f32(vals[1..].as_mut_ptr(), transmute(a)); + + assert_eq!(vals[0], 0.); + assert_eq!(vals[1], 1.); + assert_eq!(vals[2], 2.); +} + +#[simd_test(enable = "neon")] +unsafe fn test_vst1q_f32() { + let mut vals = [0_f32; 5]; + let a = f32x4::new(1., 2., 3., 4.); + + vst1q_f32(vals[1..].as_mut_ptr(), transmute(a)); + + assert_eq!(vals[0], 0.); + assert_eq!(vals[1], 1.); + assert_eq!(vals[2], 2.); + assert_eq!(vals[3], 3.); + assert_eq!(vals[4], 4.); +} diff --git a/library/stdarch/crates/core_arch/src/arm/neon/table_lookup_tests.rs b/library/stdarch/crates/core_arch/src/arm_shared/neon/table_lookup_tests.rs similarity index 100% rename from library/stdarch/crates/core_arch/src/arm/neon/table_lookup_tests.rs rename to library/stdarch/crates/core_arch/src/arm_shared/neon/table_lookup_tests.rs diff --git a/library/stdarch/crates/core_arch/src/acle/registers/aarch32.rs b/library/stdarch/crates/core_arch/src/arm_shared/registers/aarch32.rs similarity index 100% rename from library/stdarch/crates/core_arch/src/acle/registers/aarch32.rs rename to library/stdarch/crates/core_arch/src/arm_shared/registers/aarch32.rs diff --git a/library/stdarch/crates/core_arch/src/acle/registers/mod.rs b/library/stdarch/crates/core_arch/src/arm_shared/registers/mod.rs similarity index 85% rename from library/stdarch/crates/core_arch/src/acle/registers/mod.rs rename to library/stdarch/crates/core_arch/src/arm_shared/registers/mod.rs index 391a5f0824..71e94ee5cb 100644 --- a/library/stdarch/crates/core_arch/src/acle/registers/mod.rs +++ b/library/stdarch/crates/core_arch/src/arm_shared/registers/mod.rs @@ -4,7 +4,7 @@ macro_rules! rsr { impl super::super::sealed::Rsr for $R { unsafe fn __rsr(&self) -> u32 { let r: u32; - llvm_asm!(concat!("mrs $0,", stringify!($R)) : "=r"(r) : : : "volatile"); + asm!(concat!("mrs {},", stringify!($R)), out(reg) r, options(nomem, nostack)); r } } @@ -17,7 +17,7 @@ macro_rules! rsrp { impl super::super::sealed::Rsrp for $R { unsafe fn __rsrp(&self) -> *const u8 { let r: *const u8; - llvm_asm!(concat!("mrs $0,", stringify!($R)) : "=r"(r) : : : "volatile"); + asm!(concat!("mrs {},", stringify!($R)), out(reg) r, options(nomem, nostack)); r } } @@ -29,7 +29,7 @@ macro_rules! wsr { ($R:ident) => { impl super::super::sealed::Wsr for $R { unsafe fn __wsr(&self, value: u32) { - llvm_asm!(concat!("msr ", stringify!($R), ",$0") : : "r"(value) : : "volatile"); + asm!(concat!("msr ", stringify!($R), ", {}"), in(reg) value, options(nomem, nostack)); } } }; @@ -40,7 +40,7 @@ macro_rules! wsrp { ($R:ident) => { impl super::super::sealed::Wsrp for $R { unsafe fn __wsrp(&self, value: *const u8) { - llvm_asm!(concat!("msr ", stringify!($R), ",$0") : : "r"(value) : : "volatile"); + asm!(concat!("msr ", stringify!($R), ", {}"), in(reg) value, options(nomem, nostack)); } } }; diff --git a/library/stdarch/crates/core_arch/src/acle/registers/v6m.rs b/library/stdarch/crates/core_arch/src/arm_shared/registers/v6m.rs similarity index 100% rename from library/stdarch/crates/core_arch/src/acle/registers/v6m.rs rename to library/stdarch/crates/core_arch/src/arm_shared/registers/v6m.rs diff --git a/library/stdarch/crates/core_arch/src/acle/registers/v7m.rs b/library/stdarch/crates/core_arch/src/arm_shared/registers/v7m.rs similarity index 100% rename from library/stdarch/crates/core_arch/src/acle/registers/v7m.rs rename to library/stdarch/crates/core_arch/src/arm_shared/registers/v7m.rs diff --git a/library/stdarch/crates/core_arch/src/arm/test_support.rs b/library/stdarch/crates/core_arch/src/arm_shared/test_support.rs similarity index 98% rename from library/stdarch/crates/core_arch/src/arm/test_support.rs rename to library/stdarch/crates/core_arch/src/arm_shared/test_support.rs index 337a270e40..ff752f25b3 100644 --- a/library/stdarch/crates/core_arch/src/arm/test_support.rs +++ b/library/stdarch/crates/core_arch/src/arm_shared/test_support.rs @@ -1,4 +1,10 @@ -use crate::core_arch::{arm::*, simd::*}; +#[cfg(target_arch = "arm")] +use crate::core_arch::arm::*; + +#[cfg(target_arch = "aarch64")] +use crate::core_arch::aarch64::*; + +use crate::core_arch::simd::*; use std::{i16, i32, i8, mem::transmute, u16, u32, u8, vec::Vec}; macro_rules! V_u8 { diff --git a/library/stdarch/crates/core_arch/src/lib.rs b/library/stdarch/crates/core_arch/src/lib.rs index a8f1875781..da5b4713aa 100644 --- a/library/stdarch/crates/core_arch/src/lib.rs +++ b/library/stdarch/crates/core_arch/src/lib.rs @@ -5,15 +5,14 @@ #![deny(rust_2018_idioms)] #![feature( asm, - const_fn, const_fn_union, const_fn_transmute, + const_panic, custom_inner_attributes, link_llvm_intrinsics, platform_intrinsics, repr_simd, simd_ffi, - llvm_asm, proc_macro_hygiene, stmt_expr_attributes, core_intrinsics, @@ -35,13 +34,12 @@ adx_target_feature, rtm_target_feature, f16c_target_feature, - external_doc, allow_internal_unstable, decl_macro, - extended_key_value_attributes + bench_black_box )] #![cfg_attr(test, feature(test, abi_vectorcall))] -#![cfg_attr(all(test, target_arch = "wasm32"), feature(wasm_simd))] +#![cfg_attr(target_arch = "wasm32", feature(wasm_simd_const))] #![deny(clippy::missing_inline_in_public_items)] #![allow( clippy::inline_always, @@ -76,4 +74,4 @@ mod core_arch; pub use self::core_arch::arch; #[allow(unused_imports)] -use core::{ffi, hint, intrinsics, marker, mem, ops, ptr, sync}; +use core::{convert, ffi, hint, intrinsics, marker, mem, ops, ptr, sync}; diff --git a/library/stdarch/crates/core_arch/src/macros.rs b/library/stdarch/crates/core_arch/src/macros.rs index 7ebff27e8c..5f68ff79f8 100644 --- a/library/stdarch/crates/core_arch/src/macros.rs +++ b/library/stdarch/crates/core_arch/src/macros.rs @@ -1,396 +1,81 @@ //! Utility macros. -#[allow(unused)] -macro_rules! constify_imm8 { - ($imm8:expr, $expand:ident) => { - #[allow(overflowing_literals)] - match ($imm8) & 0b1111_1111 { - 0 => $expand!(0), - 1 => $expand!(1), - 2 => $expand!(2), - 3 => $expand!(3), - 4 => $expand!(4), - 5 => $expand!(5), - 6 => $expand!(6), - 7 => $expand!(7), - 8 => $expand!(8), - 9 => $expand!(9), - 10 => $expand!(10), - 11 => $expand!(11), - 12 => $expand!(12), - 13 => $expand!(13), - 14 => $expand!(14), - 15 => $expand!(15), - 16 => $expand!(16), - 17 => $expand!(17), - 18 => $expand!(18), - 19 => $expand!(19), - 20 => $expand!(20), - 21 => $expand!(21), - 22 => $expand!(22), - 23 => $expand!(23), - 24 => $expand!(24), - 25 => $expand!(25), - 26 => $expand!(26), - 27 => $expand!(27), - 28 => $expand!(28), - 29 => $expand!(29), - 30 => $expand!(30), - 31 => $expand!(31), - 32 => $expand!(32), - 33 => $expand!(33), - 34 => $expand!(34), - 35 => $expand!(35), - 36 => $expand!(36), - 37 => $expand!(37), - 38 => $expand!(38), - 39 => $expand!(39), - 40 => $expand!(40), - 41 => $expand!(41), - 42 => $expand!(42), - 43 => $expand!(43), - 44 => $expand!(44), - 45 => $expand!(45), - 46 => $expand!(46), - 47 => $expand!(47), - 48 => $expand!(48), - 49 => $expand!(49), - 50 => $expand!(50), - 51 => $expand!(51), - 52 => $expand!(52), - 53 => $expand!(53), - 54 => $expand!(54), - 55 => $expand!(55), - 56 => $expand!(56), - 57 => $expand!(57), - 58 => $expand!(58), - 59 => $expand!(59), - 60 => $expand!(60), - 61 => $expand!(61), - 62 => $expand!(62), - 63 => $expand!(63), - 64 => $expand!(64), - 65 => $expand!(65), - 66 => $expand!(66), - 67 => $expand!(67), - 68 => $expand!(68), - 69 => $expand!(69), - 70 => $expand!(70), - 71 => $expand!(71), - 72 => $expand!(72), - 73 => $expand!(73), - 74 => $expand!(74), - 75 => $expand!(75), - 76 => $expand!(76), - 77 => $expand!(77), - 78 => $expand!(78), - 79 => $expand!(79), - 80 => $expand!(80), - 81 => $expand!(81), - 82 => $expand!(82), - 83 => $expand!(83), - 84 => $expand!(84), - 85 => $expand!(85), - 86 => $expand!(86), - 87 => $expand!(87), - 88 => $expand!(88), - 89 => $expand!(89), - 90 => $expand!(90), - 91 => $expand!(91), - 92 => $expand!(92), - 93 => $expand!(93), - 94 => $expand!(94), - 95 => $expand!(95), - 96 => $expand!(96), - 97 => $expand!(97), - 98 => $expand!(98), - 99 => $expand!(99), - 100 => $expand!(100), - 101 => $expand!(101), - 102 => $expand!(102), - 103 => $expand!(103), - 104 => $expand!(104), - 105 => $expand!(105), - 106 => $expand!(106), - 107 => $expand!(107), - 108 => $expand!(108), - 109 => $expand!(109), - 110 => $expand!(110), - 111 => $expand!(111), - 112 => $expand!(112), - 113 => $expand!(113), - 114 => $expand!(114), - 115 => $expand!(115), - 116 => $expand!(116), - 117 => $expand!(117), - 118 => $expand!(118), - 119 => $expand!(119), - 120 => $expand!(120), - 121 => $expand!(121), - 122 => $expand!(122), - 123 => $expand!(123), - 124 => $expand!(124), - 125 => $expand!(125), - 126 => $expand!(126), - 127 => $expand!(127), - 128 => $expand!(128), - 129 => $expand!(129), - 130 => $expand!(130), - 131 => $expand!(131), - 132 => $expand!(132), - 133 => $expand!(133), - 134 => $expand!(134), - 135 => $expand!(135), - 136 => $expand!(136), - 137 => $expand!(137), - 138 => $expand!(138), - 139 => $expand!(139), - 140 => $expand!(140), - 141 => $expand!(141), - 142 => $expand!(142), - 143 => $expand!(143), - 144 => $expand!(144), - 145 => $expand!(145), - 146 => $expand!(146), - 147 => $expand!(147), - 148 => $expand!(148), - 149 => $expand!(149), - 150 => $expand!(150), - 151 => $expand!(151), - 152 => $expand!(152), - 153 => $expand!(153), - 154 => $expand!(154), - 155 => $expand!(155), - 156 => $expand!(156), - 157 => $expand!(157), - 158 => $expand!(158), - 159 => $expand!(159), - 160 => $expand!(160), - 161 => $expand!(161), - 162 => $expand!(162), - 163 => $expand!(163), - 164 => $expand!(164), - 165 => $expand!(165), - 166 => $expand!(166), - 167 => $expand!(167), - 168 => $expand!(168), - 169 => $expand!(169), - 170 => $expand!(170), - 171 => $expand!(171), - 172 => $expand!(172), - 173 => $expand!(173), - 174 => $expand!(174), - 175 => $expand!(175), - 176 => $expand!(176), - 177 => $expand!(177), - 178 => $expand!(178), - 179 => $expand!(179), - 180 => $expand!(180), - 181 => $expand!(181), - 182 => $expand!(182), - 183 => $expand!(183), - 184 => $expand!(184), - 185 => $expand!(185), - 186 => $expand!(186), - 187 => $expand!(187), - 188 => $expand!(188), - 189 => $expand!(189), - 190 => $expand!(190), - 191 => $expand!(191), - 192 => $expand!(192), - 193 => $expand!(193), - 194 => $expand!(194), - 195 => $expand!(195), - 196 => $expand!(196), - 197 => $expand!(197), - 198 => $expand!(198), - 199 => $expand!(199), - 200 => $expand!(200), - 201 => $expand!(201), - 202 => $expand!(202), - 203 => $expand!(203), - 204 => $expand!(204), - 205 => $expand!(205), - 206 => $expand!(206), - 207 => $expand!(207), - 208 => $expand!(208), - 209 => $expand!(209), - 210 => $expand!(210), - 211 => $expand!(211), - 212 => $expand!(212), - 213 => $expand!(213), - 214 => $expand!(214), - 215 => $expand!(215), - 216 => $expand!(216), - 217 => $expand!(217), - 218 => $expand!(218), - 219 => $expand!(219), - 220 => $expand!(220), - 221 => $expand!(221), - 222 => $expand!(222), - 223 => $expand!(223), - 224 => $expand!(224), - 225 => $expand!(225), - 226 => $expand!(226), - 227 => $expand!(227), - 228 => $expand!(228), - 229 => $expand!(229), - 230 => $expand!(230), - 231 => $expand!(231), - 232 => $expand!(232), - 233 => $expand!(233), - 234 => $expand!(234), - 235 => $expand!(235), - 236 => $expand!(236), - 237 => $expand!(237), - 238 => $expand!(238), - 239 => $expand!(239), - 240 => $expand!(240), - 241 => $expand!(241), - 242 => $expand!(242), - 243 => $expand!(243), - 244 => $expand!(244), - 245 => $expand!(245), - 246 => $expand!(246), - 247 => $expand!(247), - 248 => $expand!(248), - 249 => $expand!(249), - 250 => $expand!(250), - 251 => $expand!(251), - 252 => $expand!(252), - 253 => $expand!(253), - 254 => $expand!(254), - _ => $expand!(255), - } +// Helper struct used to trigger const eval errors when the const generic immediate value `IMM` is +// out of `[MIN-MAX]` range. +pub(crate) struct ValidateConstImm; +impl ValidateConstImm { + pub(crate) const VALID: () = { + assert!(IMM >= MIN && IMM <= MAX, "IMM value not in expected range"); }; } -//immediate value: 0:31 -#[allow(unused)] -macro_rules! constify_imm5 { - ($imm8:expr, $expand:ident) => { - #[allow(overflowing_literals)] - match ($imm8) & 0b1_1111 { - 0 => $expand!(0), - 1 => $expand!(1), - 2 => $expand!(2), - 3 => $expand!(3), - 4 => $expand!(4), - 5 => $expand!(5), - 6 => $expand!(6), - 7 => $expand!(7), - 8 => $expand!(8), - 9 => $expand!(9), - 10 => $expand!(10), - 11 => $expand!(11), - 12 => $expand!(12), - 13 => $expand!(13), - 14 => $expand!(14), - 15 => $expand!(15), - 16 => $expand!(16), - 17 => $expand!(17), - 18 => $expand!(18), - 19 => $expand!(19), - 20 => $expand!(20), - 21 => $expand!(21), - 22 => $expand!(22), - 23 => $expand!(23), - 24 => $expand!(24), - 25 => $expand!(25), - 26 => $expand!(26), - 27 => $expand!(27), - 28 => $expand!(28), - 29 => $expand!(29), - 30 => $expand!(30), - _ => $expand!(31), - } +#[allow(unused_macros)] +macro_rules! static_assert_imm1 { + ($imm:ident) => { + let _ = $crate::core_arch::macros::ValidateConstImm::<$imm, 0, { (1 << 1) - 1 }>::VALID; }; } -//immediate value: -16:15 -#[allow(unused)] -macro_rules! constify_imm5 { - ($imm8:expr, $expand:ident) => { - #[allow(overflowing_literals)] - match ($imm8) & 0b1_1111 { - 0 => $expand!(0), - 1 => $expand!(1), - 2 => $expand!(2), - 3 => $expand!(3), - 4 => $expand!(4), - 5 => $expand!(5), - 6 => $expand!(6), - 7 => $expand!(7), - 8 => $expand!(8), - 9 => $expand!(9), - 10 => $expand!(10), - 11 => $expand!(11), - 12 => $expand!(12), - 13 => $expand!(13), - 14 => $expand!(14), - 15 => $expand!(15), - 16 => $expand!(16), - 17 => $expand!(17), - 18 => $expand!(18), - 19 => $expand!(19), - 20 => $expand!(20), - 21 => $expand!(21), - 22 => $expand!(22), - 23 => $expand!(23), - 24 => $expand!(24), - 25 => $expand!(25), - 26 => $expand!(26), - 27 => $expand!(27), - 28 => $expand!(28), - 29 => $expand!(29), - 30 => $expand!(30), - _ => $expand!(31), - } +#[allow(unused_macros)] +macro_rules! static_assert_imm2 { + ($imm:ident) => { + let _ = $crate::core_arch::macros::ValidateConstImm::<$imm, 0, { (1 << 2) - 1 }>::VALID; }; } -//immediate value: 0:16 -#[allow(unused)] -macro_rules! constify_imm4 { - ($imm8:expr, $expand:ident) => { - #[allow(overflowing_literals)] - match ($imm8) & 0b1111 { - 0 => $expand!(0), - 1 => $expand!(1), - 2 => $expand!(2), - 3 => $expand!(3), - 4 => $expand!(4), - 5 => $expand!(5), - 6 => $expand!(6), - 7 => $expand!(7), - 8 => $expand!(8), - 9 => $expand!(9), - 10 => $expand!(10), - 11 => $expand!(11), - 12 => $expand!(12), - 13 => $expand!(13), - 14 => $expand!(14), - _ => $expand!(15), - } +#[allow(unused_macros)] +macro_rules! static_assert_imm3 { + ($imm:ident) => { + let _ = $crate::core_arch::macros::ValidateConstImm::<$imm, 0, { (1 << 3) - 1 }>::VALID; + }; +} + +#[allow(unused_macros)] +macro_rules! static_assert_imm4 { + ($imm:ident) => { + let _ = $crate::core_arch::macros::ValidateConstImm::<$imm, 0, { (1 << 4) - 1 }>::VALID; + }; +} + +#[allow(unused_macros)] +macro_rules! static_assert_imm5 { + ($imm:ident) => { + let _ = $crate::core_arch::macros::ValidateConstImm::<$imm, 0, { (1 << 5) - 1 }>::VALID; + }; +} + +#[allow(unused_macros)] +macro_rules! static_assert_imm6 { + ($imm:ident) => { + let _ = $crate::core_arch::macros::ValidateConstImm::<$imm, 0, { (1 << 6) - 1 }>::VALID; + }; +} + +#[allow(unused_macros)] +macro_rules! static_assert_imm8 { + ($imm:ident) => { + let _ = $crate::core_arch::macros::ValidateConstImm::<$imm, 0, { (1 << 8) - 1 }>::VALID; + }; +} + +#[allow(unused_macros)] +macro_rules! static_assert_imm16 { + ($imm:ident) => { + let _ = $crate::core_arch::macros::ValidateConstImm::<$imm, 0, { (1 << 16) - 1 }>::VALID; }; } -//immediate value: 0:7 #[allow(unused)] -macro_rules! constify_imm3 { - ($imm8:expr, $expand:ident) => { - #[allow(overflowing_literals)] - match ($imm8) & 0b111 { - 0 => $expand!(0), - 1 => $expand!(1), - 2 => $expand!(2), - 3 => $expand!(3), - 4 => $expand!(4), - 5 => $expand!(5), - 6 => $expand!(6), - _ => $expand!(7), +macro_rules! static_assert { + ($imm:ident : $ty:ty where $e:expr) => {{ + struct Validate(); + impl Validate<$imm> { + const VALID: () = { + assert!($e, concat!("Assertion failed: ", stringify!($e))); + }; } - }; + let _ = Validate::<$imm>::VALID; + }}; } #[allow(unused)] @@ -407,3 +92,99 @@ macro_rules! types { pub struct $name($($fields)*); )*) } + +#[allow(unused_macros)] +macro_rules! simd_shuffle2 { + ($x:expr, $y:expr, <$(const $imm:ident : $ty:ty),+ $(,)?> $idx:expr $(,)?) => {{ + struct ConstParam<$(const $imm: $ty),+>; + impl<$(const $imm: $ty),+> ConstParam<$($imm),+> { + const IDX: [u32; 2] = $idx; + } + + simd_shuffle2($x, $y, ConstParam::<$($imm),+>::IDX) + }}; + ($x:expr, $y:expr, $idx:expr $(,)?) => {{ + const IDX: [u32; 2] = $idx; + simd_shuffle2($x, $y, IDX) + }}; +} + +#[allow(unused_macros)] +macro_rules! simd_shuffle4 { + ($x:expr, $y:expr, <$(const $imm:ident : $ty:ty),+ $(,)?> $idx:expr $(,)?) => {{ + struct ConstParam<$(const $imm: $ty),+>; + impl<$(const $imm: $ty),+> ConstParam<$($imm),+> { + const IDX: [u32; 4] = $idx; + } + + simd_shuffle4($x, $y, ConstParam::<$($imm),+>::IDX) + }}; + ($x:expr, $y:expr, $idx:expr $(,)?) => {{ + const IDX: [u32; 4] = $idx; + simd_shuffle4($x, $y, IDX) + }}; +} + +#[allow(unused_macros)] +macro_rules! simd_shuffle8 { + ($x:expr, $y:expr, <$(const $imm:ident : $ty:ty),+ $(,)?> $idx:expr $(,)?) => {{ + struct ConstParam<$(const $imm: $ty),+>; + impl<$(const $imm: $ty),+> ConstParam<$($imm),+> { + const IDX: [u32; 8] = $idx; + } + + simd_shuffle8($x, $y, ConstParam::<$($imm),+>::IDX) + }}; + ($x:expr, $y:expr, $idx:expr $(,)?) => {{ + const IDX: [u32; 8] = $idx; + simd_shuffle8($x, $y, IDX) + }}; +} + +#[allow(unused_macros)] +macro_rules! simd_shuffle16 { + ($x:expr, $y:expr, <$(const $imm:ident : $ty:ty),+ $(,)?> $idx:expr $(,)?) => {{ + struct ConstParam<$(const $imm: $ty),+>; + impl<$(const $imm: $ty),+> ConstParam<$($imm),+> { + const IDX: [u32; 16] = $idx; + } + + simd_shuffle16($x, $y, ConstParam::<$($imm),+>::IDX) + }}; + ($x:expr, $y:expr, $idx:expr $(,)?) => {{ + const IDX: [u32; 16] = $idx; + simd_shuffle16($x, $y, IDX) + }}; +} + +#[allow(unused_macros)] +macro_rules! simd_shuffle32 { + ($x:expr, $y:expr, <$(const $imm:ident : $ty:ty),+> $(,)? $idx:expr $(,)?) => {{ + struct ConstParam<$(const $imm: $ty),+>; + impl<$(const $imm: $ty),+> ConstParam<$($imm),+> { + const IDX: [u32; 32] = $idx; + } + + simd_shuffle32($x, $y, ConstParam::<$($imm),+>::IDX) + }}; + ($x:expr, $y:expr, $idx:expr $(,)?) => {{ + const IDX: [u32; 32] = $idx; + simd_shuffle32($x, $y, IDX) + }}; +} + +#[allow(unused_macros)] +macro_rules! simd_shuffle64 { + ($x:expr, $y:expr, <$(const $imm:ident : $ty:ty),+ $(,)?> $idx:expr $(,)?) => {{ + struct ConstParam<$(const $imm: $ty),+>; + impl<$(const $imm: $ty),+> ConstParam<$($imm),+> { + const IDX: [u32; 64] = $idx; + } + + simd_shuffle64($x, $y, ConstParam::<$($imm),+>::IDX) + }}; + ($x:expr, $y:expr, $idx:expr $(,)?) => {{ + const IDX: [u32; 64] = $idx; + simd_shuffle64($x, $y, IDX) + }}; +} diff --git a/library/stdarch/crates/core_arch/src/mips/msa.rs b/library/stdarch/crates/core_arch/src/mips/msa.rs index 7a975925cd..0ba0774313 100644 --- a/library/stdarch/crates/core_arch/src/mips/msa.rs +++ b/library/stdarch/crates/core_arch/src/mips/msa.rs @@ -1411,14 +1411,10 @@ pub unsafe fn __msa_addv_d(a: v2i64, b: v2i64) -> v2i64 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(addvi.b, imm5 = 0b10111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_addvi_b(a: v16i8, imm5: i32) -> v16i8 { - macro_rules! call { - ($imm5:expr) => { - msa_addvi_b(a, $imm5) - }; - } - constify_imm5!(imm5, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_addvi_b(a: v16i8) -> v16i8 { + static_assert_imm5!(IMM5); + msa_addvi_b(a, IMM5) } /// Immediate Add @@ -1430,14 +1426,10 @@ pub unsafe fn __msa_addvi_b(a: v16i8, imm5: i32) -> v16i8 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(addvi.h, imm5 = 0b10111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_addvi_h(a: v8i16, imm5: i32) -> v8i16 { - macro_rules! call { - ($imm5:expr) => { - msa_addvi_h(a, $imm5) - }; - } - constify_imm5!(imm5, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_addvi_h(a: v8i16) -> v8i16 { + static_assert_imm5!(IMM5); + msa_addvi_h(a, IMM5) } /// Immediate Add @@ -1449,14 +1441,10 @@ pub unsafe fn __msa_addvi_h(a: v8i16, imm5: i32) -> v8i16 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(addvi.w, imm5 = 0b10111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_addvi_w(a: v4i32, imm5: i32) -> v4i32 { - macro_rules! call { - ($imm5:expr) => { - msa_addvi_w(a, $imm5) - }; - } - constify_imm5!(imm5, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_addvi_w(a: v4i32) -> v4i32 { + static_assert_imm5!(IMM5); + msa_addvi_w(a, IMM5) } /// Immediate Add @@ -1468,14 +1456,10 @@ pub unsafe fn __msa_addvi_w(a: v4i32, imm5: i32) -> v4i32 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(addvi.d, imm5 = 0b10111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_addvi_d(a: v2i64, imm5: i32) -> v2i64 { - macro_rules! call { - ($imm5:expr) => { - msa_addvi_d(a, $imm5) - }; - } - constify_imm5!(imm5, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_addvi_d(a: v2i64) -> v2i64 { + static_assert_imm5!(IMM5); + msa_addvi_d(a, IMM5) } /// Vector Logical And @@ -1501,14 +1485,10 @@ pub unsafe fn __msa_and_v(a: v16u8, b: v16u8) -> v16u8 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(andi.b, imm8 = 0b10010111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_andi_b(a: v16u8, imm8: i32) -> v16u8 { - macro_rules! call { - ($imm8:expr) => { - msa_andi_b(a, $imm8) - }; - } - constify_imm8!(imm8, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_andi_b(a: v16u8) -> v16u8 { + static_assert_imm8!(IMM8); + msa_andi_b(a, IMM8) } /// Vector Absolute Values of Signed Subtract @@ -1928,14 +1908,10 @@ pub unsafe fn __msa_bclr_d(a: v2u64, b: v2u64) -> v2u64 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(bclri.b, imm3 = 0b111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_bclri_b(a: v16u8, imm3: i32) -> v16u8 { - macro_rules! call { - ($imm3:expr) => { - msa_bclri_b(a, $imm3) - }; - } - constify_imm3!(imm3, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_bclri_b(a: v16u8) -> v16u8 { + static_assert_imm3!(IMM3); + msa_bclri_b(a, IMM3) } /// Immediate Bit Clear @@ -1947,14 +1923,10 @@ pub unsafe fn __msa_bclri_b(a: v16u8, imm3: i32) -> v16u8 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(bclri.h, imm4 = 0b1111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_bclri_h(a: v8u16, imm4: i32) -> v8u16 { - macro_rules! call { - ($imm4:expr) => { - msa_bclri_h(a, $imm4) - }; - } - constify_imm4!(imm4, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_bclri_h(a: v8u16) -> v8u16 { + static_assert_imm4!(IMM4); + msa_bclri_h(a, IMM4) } /// Immediate Bit Clear @@ -1966,14 +1938,10 @@ pub unsafe fn __msa_bclri_h(a: v8u16, imm4: i32) -> v8u16 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(bclri.w, imm5 = 0b11111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_bclri_w(a: v4u32, imm5: i32) -> v4u32 { - macro_rules! call { - ($imm5:expr) => { - msa_bclri_w(a, $imm5) - }; - } - constify_imm5!(imm5, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_bclri_w(a: v4u32) -> v4u32 { + static_assert_imm5!(IMM5); + msa_bclri_w(a, IMM5) } /// Immediate Bit Clear @@ -1985,14 +1953,10 @@ pub unsafe fn __msa_bclri_w(a: v4u32, imm5: i32) -> v4u32 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(bclri.d, imm6 = 0b111111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_bclri_d(a: v2u64, imm6: i32) -> v2u64 { - macro_rules! call { - ($imm6:expr) => { - msa_bclri_d(a, $imm6) - }; - } - constify_imm6!(imm6, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_bclri_d(a: v2u64) -> v2u64 { + static_assert_imm6!(IMM6); + msa_bclri_d(a, IMM6) } /// Vector Bit Insert Left @@ -2060,14 +2024,10 @@ pub unsafe fn __msa_binsl_d(a: v2u64, b: v2u64, c: v2u64) -> v2u64 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(binsli.b, imm3 = 0b111))] -#[rustc_args_required_const(2)] -pub unsafe fn __msa_binsli_b(a: v16u8, b: v16u8, imm3: i32) -> v16u8 { - macro_rules! call { - ($imm3:expr) => { - msa_binsli_b(a, mem::transmute(b), $imm3) - }; - } - constify_imm3!(imm3, call) +#[rustc_legacy_const_generics(2)] +pub unsafe fn __msa_binsli_b(a: v16u8, b: v16u8) -> v16u8 { + static_assert_imm3!(IMM3); + msa_binsli_b(a, mem::transmute(b), IMM3) } /// Immediate Bit Insert Left @@ -2079,14 +2039,10 @@ pub unsafe fn __msa_binsli_b(a: v16u8, b: v16u8, imm3: i32) -> v16u8 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(binsli.h, imm4 = 0b1111))] -#[rustc_args_required_const(2)] -pub unsafe fn __msa_binsli_h(a: v8u16, b: v8u16, imm4: i32) -> v8u16 { - macro_rules! call { - ($imm4:expr) => { - msa_binsli_h(a, mem::transmute(b), $imm4) - }; - } - constify_imm4!(imm4, call) +#[rustc_legacy_const_generics(2)] +pub unsafe fn __msa_binsli_h(a: v8u16, b: v8u16) -> v8u16 { + static_assert_imm4!(IMM4); + msa_binsli_h(a, mem::transmute(b), IMM4) } /// Immediate Bit Insert Left @@ -2098,14 +2054,10 @@ pub unsafe fn __msa_binsli_h(a: v8u16, b: v8u16, imm4: i32) -> v8u16 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(binsli.w, imm5 = 0b11111))] -#[rustc_args_required_const(2)] -pub unsafe fn __msa_binsli_w(a: v4u32, b: v4u32, imm5: i32) -> v4u32 { - macro_rules! call { - ($imm5:expr) => { - msa_binsli_w(a, mem::transmute(b), $imm5) - }; - } - constify_imm5!(imm5, call) +#[rustc_legacy_const_generics(2)] +pub unsafe fn __msa_binsli_w(a: v4u32, b: v4u32) -> v4u32 { + static_assert_imm5!(IMM5); + msa_binsli_w(a, mem::transmute(b), IMM5) } /// Immediate Bit Insert Left @@ -2117,14 +2069,10 @@ pub unsafe fn __msa_binsli_w(a: v4u32, b: v4u32, imm5: i32) -> v4u32 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(binsli.d, imm6 = 0b111111))] -#[rustc_args_required_const(2)] -pub unsafe fn __msa_binsli_d(a: v2u64, b: v2u64, imm6: i32) -> v2u64 { - macro_rules! call { - ($imm6:expr) => { - msa_binsli_d(a, mem::transmute(b), $imm6) - }; - } - constify_imm6!(imm6, call) +#[rustc_legacy_const_generics(2)] +pub unsafe fn __msa_binsli_d(a: v2u64, b: v2u64) -> v2u64 { + static_assert_imm6!(IMM6); + msa_binsli_d(a, mem::transmute(b), IMM6) } /// Vector Bit Insert Right @@ -2192,14 +2140,10 @@ pub unsafe fn __msa_binsr_d(a: v2u64, b: v2u64, c: v2u64) -> v2u64 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(binsri.b, imm3 = 0b111))] -#[rustc_args_required_const(2)] -pub unsafe fn __msa_binsri_b(a: v16u8, b: v16u8, imm3: i32) -> v16u8 { - macro_rules! call { - ($imm3:expr) => { - msa_binsri_b(a, mem::transmute(b), $imm3) - }; - } - constify_imm3!(imm3, call) +#[rustc_legacy_const_generics(2)] +pub unsafe fn __msa_binsri_b(a: v16u8, b: v16u8) -> v16u8 { + static_assert_imm3!(IMM3); + msa_binsri_b(a, mem::transmute(b), IMM3) } /// Immediate Bit Insert Right @@ -2211,14 +2155,10 @@ pub unsafe fn __msa_binsri_b(a: v16u8, b: v16u8, imm3: i32) -> v16u8 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(binsri.h, imm4 = 0b1111))] -#[rustc_args_required_const(2)] -pub unsafe fn __msa_binsri_h(a: v8u16, b: v8u16, imm4: i32) -> v8u16 { - macro_rules! call { - ($imm4:expr) => { - msa_binsri_h(a, mem::transmute(b), $imm4) - }; - } - constify_imm4!(imm4, call) +#[rustc_legacy_const_generics(2)] +pub unsafe fn __msa_binsri_h(a: v8u16, b: v8u16) -> v8u16 { + static_assert_imm4!(IMM4); + msa_binsri_h(a, mem::transmute(b), IMM4) } /// Immediate Bit Insert Right @@ -2230,14 +2170,10 @@ pub unsafe fn __msa_binsri_h(a: v8u16, b: v8u16, imm4: i32) -> v8u16 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(binsri.w, imm5 = 0b11111))] -#[rustc_args_required_const(2)] -pub unsafe fn __msa_binsri_w(a: v4u32, b: v4u32, imm5: i32) -> v4u32 { - macro_rules! call { - ($imm5:expr) => { - msa_binsri_w(a, mem::transmute(b), $imm5) - }; - } - constify_imm5!(imm5, call) +#[rustc_legacy_const_generics(2)] +pub unsafe fn __msa_binsri_w(a: v4u32, b: v4u32) -> v4u32 { + static_assert_imm5!(IMM5); + msa_binsri_w(a, mem::transmute(b), IMM5) } /// Immediate Bit Insert Right @@ -2249,14 +2185,10 @@ pub unsafe fn __msa_binsri_w(a: v4u32, b: v4u32, imm5: i32) -> v4u32 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(binsri.d, imm6 = 0b111111))] -#[rustc_args_required_const(2)] -pub unsafe fn __msa_binsri_d(a: v2u64, b: v2u64, imm6: i32) -> v2u64 { - macro_rules! call { - ($imm6:expr) => { - msa_binsri_d(a, mem::transmute(b), $imm6) - }; - } - constify_imm6!(imm6, call) +#[rustc_legacy_const_generics(2)] +pub unsafe fn __msa_binsri_d(a: v2u64, b: v2u64) -> v2u64 { + static_assert_imm6!(IMM6); + msa_binsri_d(a, mem::transmute(b), IMM6) } /// Vector Bit Move If Not Zero @@ -2282,14 +2214,10 @@ pub unsafe fn __msa_bmnz_v(a: v16u8, b: v16u8, c: v16u8) -> v16u8 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(bmnzi.b, imm8 = 0b11111111))] -#[rustc_args_required_const(2)] -pub unsafe fn __msa_bmnzi_b(a: v16u8, b: v16u8, imm8: i32) -> v16u8 { - macro_rules! call { - ($imm8:expr) => { - msa_bmnzi_b(a, mem::transmute(b), $imm8) - }; - } - constify_imm8!(imm8, call) +#[rustc_legacy_const_generics(2)] +pub unsafe fn __msa_bmnzi_b(a: v16u8, b: v16u8) -> v16u8 { + static_assert_imm8!(IMM8); + msa_bmnzi_b(a, mem::transmute(b), IMM8) } /// Vector Bit Move If Zero @@ -2315,14 +2243,10 @@ pub unsafe fn __msa_bmz_v(a: v16u8, b: v16u8, c: v16u8) -> v16u8 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(bmzi.b, imm8 = 0b11111111))] -#[rustc_args_required_const(2)] -pub unsafe fn __msa_bmzi_b(a: v16u8, b: v16u8, imm8: i32) -> v16u8 { - macro_rules! call { - ($imm8:expr) => { - msa_bmzi_b(a, mem::transmute(b), $imm8) - }; - } - constify_imm8!(imm8, call) +#[rustc_legacy_const_generics(2)] +pub unsafe fn __msa_bmzi_b(a: v16u8, b: v16u8) -> v16u8 { + static_assert_imm8!(IMM8); + msa_bmzi_b(a, mem::transmute(b), IMM8) } /// Vector Bit Negate @@ -2390,14 +2314,10 @@ pub unsafe fn __msa_bneg_d(a: v2u64, b: v2u64) -> v2u64 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(bnegi.b, imm3 = 0b111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_bnegi_b(a: v16u8, imm3: i32) -> v16u8 { - macro_rules! call { - ($imm3:expr) => { - msa_bnegi_b(a, $imm3) - }; - } - constify_imm3!(imm3, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_bnegi_b(a: v16u8) -> v16u8 { + static_assert_imm3!(IMM3); + msa_bnegi_b(a, IMM3) } /// Immediate Bit Negate @@ -2409,14 +2329,10 @@ pub unsafe fn __msa_bnegi_b(a: v16u8, imm3: i32) -> v16u8 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(bnegi.h, imm4 = 0b1111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_bnegi_h(a: v8u16, imm4: i32) -> v8u16 { - macro_rules! call { - ($imm4:expr) => { - msa_bnegi_h(a, $imm4) - }; - } - constify_imm4!(imm4, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_bnegi_h(a: v8u16) -> v8u16 { + static_assert_imm4!(IMM4); + msa_bnegi_h(a, IMM4) } /// Immediate Bit Negate @@ -2428,14 +2344,10 @@ pub unsafe fn __msa_bnegi_h(a: v8u16, imm4: i32) -> v8u16 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(bnegi.w, imm5 = 0b11111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_bnegi_w(a: v4u32, imm5: i32) -> v4u32 { - macro_rules! call { - ($imm5:expr) => { - msa_bnegi_w(a, $imm5) - }; - } - constify_imm5!(imm5, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_bnegi_w(a: v4u32) -> v4u32 { + static_assert_imm5!(IMM5); + msa_bnegi_w(a, IMM5) } /// Immediate Bit Negate @@ -2447,14 +2359,10 @@ pub unsafe fn __msa_bnegi_w(a: v4u32, imm5: i32) -> v4u32 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(bnegi.d, imm6 = 0b111111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_bnegi_d(a: v2u64, imm6: i32) -> v2u64 { - macro_rules! call { - ($imm6:expr) => { - msa_bnegi_d(a, $imm6) - }; - } - constify_imm6!(imm6, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_bnegi_d(a: v2u64) -> v2u64 { + static_assert_imm6!(IMM6); + msa_bnegi_d(a, IMM6) } /// Immediate Branch If All Elements Are Not Zero @@ -2536,14 +2444,10 @@ pub unsafe fn __msa_bsel_v(a: v16u8, b: v16u8, c: v16u8) -> v16u8 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(bseli.b, imm8 = 0b11111111))] -#[rustc_args_required_const(2)] -pub unsafe fn __msa_bseli_b(a: v16u8, b: v16u8, imm8: i32) -> v16u8 { - macro_rules! call { - ($imm8:expr) => { - msa_bseli_b(a, mem::transmute(b), $imm8) - }; - } - constify_imm8!(imm8, call) +#[rustc_legacy_const_generics(2)] +pub unsafe fn __msa_bseli_b(a: v16u8, b: v16u8) -> v16u8 { + static_assert_imm8!(IMM8); + msa_bseli_b(a, mem::transmute(b), IMM8) } /// Vector Bit Set @@ -2611,14 +2515,10 @@ pub unsafe fn __msa_bset_d(a: v2u64, b: v2u64) -> v2u64 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(bseti.b, imm3 = 0b111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_bseti_b(a: v16u8, imm3: i32) -> v16u8 { - macro_rules! call { - ($imm3:expr) => { - msa_bseti_b(a, $imm3) - }; - } - constify_imm3!(imm3, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_bseti_b(a: v16u8) -> v16u8 { + static_assert_imm3!(IMM3); + msa_bseti_b(a, IMM3) } /// Immediate Bit Set @@ -2630,14 +2530,10 @@ pub unsafe fn __msa_bseti_b(a: v16u8, imm3: i32) -> v16u8 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(bseti.h, imm4 = 0b1111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_bseti_h(a: v8u16, imm4: i32) -> v8u16 { - macro_rules! call { - ($imm4:expr) => { - msa_bseti_h(a, $imm4) - }; - } - constify_imm4!(imm4, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_bseti_h(a: v8u16) -> v8u16 { + static_assert_imm4!(IMM4); + msa_bseti_h(a, IMM4) } /// Immediate Bit Set @@ -2649,14 +2545,10 @@ pub unsafe fn __msa_bseti_h(a: v8u16, imm4: i32) -> v8u16 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(bseti.w, imm5 = 0b11111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_bseti_w(a: v4u32, imm5: i32) -> v4u32 { - macro_rules! call { - ($imm5:expr) => { - msa_bseti_w(a, $imm5) - }; - } - constify_imm5!(imm5, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_bseti_w(a: v4u32) -> v4u32 { + static_assert_imm5!(IMM5); + msa_bseti_w(a, IMM5) } /// Immediate Bit Set @@ -2668,14 +2560,10 @@ pub unsafe fn __msa_bseti_w(a: v4u32, imm5: i32) -> v4u32 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(bseti.d, imm6 = 0b111111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_bseti_d(a: v2u64, imm6: i32) -> v2u64 { - macro_rules! call { - ($imm6:expr) => { - msa_bseti_d(a, $imm6) - }; - } - constify_imm6!(imm6, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_bseti_d(a: v2u64) -> v2u64 { + static_assert_imm6!(IMM6); + msa_bseti_d(a, IMM6) } /// Immediate Branch If At Least One Element Is Zero @@ -2795,14 +2683,10 @@ pub unsafe fn __msa_ceq_d(a: v2i64, b: v2i64) -> v2i64 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(ceqi.b, imm_s5 = 0b11111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_ceqi_b(a: v16i8, imm_s5: i32) -> v16i8 { - macro_rules! call { - ($imm_s5:expr) => { - msa_ceqi_b(a, $imm_s5) - }; - } - constify_imm_s5!(imm_s5, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_ceqi_b(a: v16i8) -> v16i8 { + static_assert_imm_s5!(IMM_S5); + msa_ceqi_b(a, IMM_S5) } /// Immediate Compare Equal @@ -2814,14 +2698,10 @@ pub unsafe fn __msa_ceqi_b(a: v16i8, imm_s5: i32) -> v16i8 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(ceqi.h, imm_s5 = 0b11111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_ceqi_h(a: v8i16, imm_s5: i32) -> v8i16 { - macro_rules! call { - ($imm_s5:expr) => { - msa_ceqi_h(a, $imm_s5) - }; - } - constify_imm_s5!(imm_s5, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_ceqi_h(a: v8i16) -> v8i16 { + static_assert_imm_s5!(IMM_S5); + msa_ceqi_h(a, IMM_S5) } /// Immediate Compare Equal @@ -2833,14 +2713,10 @@ pub unsafe fn __msa_ceqi_h(a: v8i16, imm_s5: i32) -> v8i16 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(ceqi.w, imm_s5 = 0b11111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_ceqi_w(a: v4i32, imm_s5: i32) -> v4i32 { - macro_rules! call { - ($imm_s5:expr) => { - msa_ceqi_w(a, $imm_s5) - }; - } - constify_imm_s5!(imm_s5, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_ceqi_w(a: v4i32) -> v4i32 { + static_assert_imm_s5!(IMM_S5); + msa_ceqi_w(a, IMM_S5) } /// Immediate Compare Equal @@ -2852,14 +2728,10 @@ pub unsafe fn __msa_ceqi_w(a: v4i32, imm_s5: i32) -> v4i32 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(ceqi.d, imm_s5 = 0b11111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_ceqi_d(a: v2i64, imm_s5: i32) -> v2i64 { - macro_rules! call { - ($imm_s5:expr) => { - msa_ceqi_d(a, $imm_s5) - }; - } - constify_imm_s5!(imm_s5, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_ceqi_d(a: v2i64) -> v2i64 { + static_assert_imm_s5!(IMM_S5); + msa_ceqi_d(a, IMM_S5) } /// GPR Copy from MSA Control Register @@ -2870,14 +2742,10 @@ pub unsafe fn __msa_ceqi_d(a: v2i64, imm_s5: i32) -> v2i64 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(cfcmsa, imm5 = 0b11111))] -#[rustc_args_required_const(0)] -pub unsafe fn __msa_cfcmsa(imm5: i32) -> i32 { - macro_rules! call { - ($imm5:expr) => { - msa_cfcmsa($imm5) - }; - } - constify_imm5!(imm5, call) +#[rustc_legacy_const_generics(0)] +pub unsafe fn __msa_cfcmsa() -> i32 { + static_assert_imm5!(IMM5); + msa_cfcmsa(IMM5) } /// Vector Compare Signed Less Than or Equal @@ -3002,14 +2870,10 @@ pub unsafe fn __msa_cle_u_d(a: v2u64, b: v2u64) -> v2i64 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(clei_s.b, imm_s5 = 0b11111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_clei_s_b(a: v16i8, imm_s5: i32) -> v16i8 { - macro_rules! call { - ($imm_s5:expr) => { - msa_clei_s_b(a, $imm_s5) - }; - } - constify_imm_s5!(imm_s5, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_clei_s_b(a: v16i8) -> v16i8 { + static_assert_imm_s5!(IMM_S5); + msa_clei_s_b(a, IMM_S5) } /// Immediate Compare Signed Less Than or Equal @@ -3022,14 +2886,10 @@ pub unsafe fn __msa_clei_s_b(a: v16i8, imm_s5: i32) -> v16i8 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(clei_s.h, imm_s5 = 0b11111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_clei_s_h(a: v8i16, imm_s5: i32) -> v8i16 { - macro_rules! call { - ($imm_s5:expr) => { - msa_clei_s_h(a, $imm_s5) - }; - } - constify_imm_s5!(imm_s5, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_clei_s_h(a: v8i16) -> v8i16 { + static_assert_imm_s5!(IMM_S5); + msa_clei_s_h(a, IMM_S5) } /// Immediate Compare Signed Less Than or Equal @@ -3042,14 +2902,10 @@ pub unsafe fn __msa_clei_s_h(a: v8i16, imm_s5: i32) -> v8i16 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(clei_s.w, imm_s5 = 0b11111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_clei_s_w(a: v4i32, imm_s5: i32) -> v4i32 { - macro_rules! call { - ($imm_s5:expr) => { - msa_clei_s_w(a, $imm_s5) - }; - } - constify_imm_s5!(imm_s5, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_clei_s_w(a: v4i32) -> v4i32 { + static_assert_imm_s5!(IMM_S5); + msa_clei_s_w(a, IMM_S5) } /// Immediate Compare Signed Less Than or Equal @@ -3062,14 +2918,10 @@ pub unsafe fn __msa_clei_s_w(a: v4i32, imm_s5: i32) -> v4i32 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(clei_s.d, imm_s5 = 0b11111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_clei_s_d(a: v2i64, imm_s5: i32) -> v2i64 { - macro_rules! call { - ($imm_s5:expr) => { - msa_clei_s_d(a, $imm_s5) - }; - } - constify_imm_s5!(imm_s5, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_clei_s_d(a: v2i64) -> v2i64 { + static_assert_imm_s5!(IMM_S5); + msa_clei_s_d(a, IMM_S5) } /// Immediate Compare Unsigned Less Than or Equal @@ -3082,14 +2934,10 @@ pub unsafe fn __msa_clei_s_d(a: v2i64, imm_s5: i32) -> v2i64 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(clei_u.b, imm5 = 0b111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_clei_u_b(a: v16u8, imm5: i32) -> v16i8 { - macro_rules! call { - ($imm5:expr) => { - msa_clei_u_b(a, $imm5) - }; - } - constify_imm5!(imm5, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_clei_u_b(a: v16u8) -> v16i8 { + static_assert_imm5!(IMM5); + msa_clei_u_b(a, IMM5) } /// Immediate Compare Unsigned Less Than or Equal @@ -3102,14 +2950,10 @@ pub unsafe fn __msa_clei_u_b(a: v16u8, imm5: i32) -> v16i8 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(clei_u.h, imm5 = 0b11111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_clei_u_h(a: v8u16, imm5: i32) -> v8i16 { - macro_rules! call { - ($imm5:expr) => { - msa_clei_u_h(a, $imm5) - }; - } - constify_imm5!(imm5, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_clei_u_h(a: v8u16) -> v8i16 { + static_assert_imm5!(IMM5); + msa_clei_u_h(a, IMM5) } /// Immediate Compare Unsigned Less Than or Equal @@ -3122,14 +2966,10 @@ pub unsafe fn __msa_clei_u_h(a: v8u16, imm5: i32) -> v8i16 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(clei_u.w, imm5 = 0b11111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_clei_u_w(a: v4u32, imm5: i32) -> v4i32 { - macro_rules! call { - ($imm5:expr) => { - msa_clei_u_w(a, $imm5) - }; - } - constify_imm5!(imm5, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_clei_u_w(a: v4u32) -> v4i32 { + static_assert_imm5!(IMM5); + msa_clei_u_w(a, IMM5) } /// Immediate Compare Unsigned Less Than or Equal @@ -3142,14 +2982,10 @@ pub unsafe fn __msa_clei_u_w(a: v4u32, imm5: i32) -> v4i32 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(clei_u.d, imm5 = 0b11111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_clei_u_d(a: v2u64, imm5: i32) -> v2i64 { - macro_rules! call { - ($imm5:expr) => { - msa_clei_u_d(a, $imm5) - }; - } - constify_imm5!(imm5, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_clei_u_d(a: v2u64) -> v2i64 { + static_assert_imm5!(IMM5); + msa_clei_u_d(a, IMM5) } /// Vector Compare Signed Less Than @@ -3274,14 +3110,10 @@ pub unsafe fn __msa_clt_u_d(a: v2u64, b: v2u64) -> v2i64 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(clti_s.b, imm_s5 = 0b111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_clti_s_b(a: v16i8, imm_s5: i32) -> v16i8 { - macro_rules! call { - ($imm_s5:expr) => { - msa_clti_s_b(a, $imm_s5) - }; - } - constify_imm_s5!(imm_s5, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_clti_s_b(a: v16i8) -> v16i8 { + static_assert_imm_s5!(IMM_S5); + msa_clti_s_b(a, IMM_S5) } /// Immediate Compare Signed Less Than @@ -3294,14 +3126,10 @@ pub unsafe fn __msa_clti_s_b(a: v16i8, imm_s5: i32) -> v16i8 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(clti_s.h, imm_s5 = 0b11111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_clti_s_h(a: v8i16, imm_s5: i32) -> v8i16 { - macro_rules! call { - ($imm_s5:expr) => { - msa_clti_s_h(a, $imm_s5) - }; - } - constify_imm_s5!(imm_s5, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_clti_s_h(a: v8i16) -> v8i16 { + static_assert_imm_s5!(IMM_S5); + msa_clti_s_h(a, IMM_S5) } /// Immediate Compare Signed Less Than @@ -3314,14 +3142,10 @@ pub unsafe fn __msa_clti_s_h(a: v8i16, imm_s5: i32) -> v8i16 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(clti_s.w, imm_s5 = 0b11111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_clti_s_w(a: v4i32, imm_s5: i32) -> v4i32 { - macro_rules! call { - ($imm_s5:expr) => { - msa_clti_s_w(a, $imm_s5) - }; - } - constify_imm_s5!(imm_s5, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_clti_s_w(a: v4i32) -> v4i32 { + static_assert_imm_s5!(IMM_S5); + msa_clti_s_w(a, IMM_S5) } /// Immediate Compare Signed Less Than @@ -3334,14 +3158,10 @@ pub unsafe fn __msa_clti_s_w(a: v4i32, imm_s5: i32) -> v4i32 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(clti_s.d, imm_s5 = 0b11111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_clti_s_d(a: v2i64, imm_s5: i32) -> v2i64 { - macro_rules! call { - ($imm_s5:expr) => { - msa_clti_s_d(a, $imm_s5) - }; - } - constify_imm_s5!(imm_s5, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_clti_s_d(a: v2i64) -> v2i64 { + static_assert_imm_s5!(IMM_S5); + msa_clti_s_d(a, IMM_S5) } /// Immediate Compare Unsigned Less Than @@ -3354,14 +3174,10 @@ pub unsafe fn __msa_clti_s_d(a: v2i64, imm_s5: i32) -> v2i64 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(clti_u.b, imm5 = 0b111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_clti_u_b(a: v16u8, imm5: i32) -> v16i8 { - macro_rules! call { - ($imm5:expr) => { - msa_clti_u_b(a, $imm5) - }; - } - constify_imm5!(imm5, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_clti_u_b(a: v16u8) -> v16i8 { + static_assert_imm5!(IMM5); + msa_clti_u_b(a, IMM5) } /// Immediate Compare Unsigned Less Than @@ -3374,14 +3190,10 @@ pub unsafe fn __msa_clti_u_b(a: v16u8, imm5: i32) -> v16i8 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(clti_u.h, imm5 = 0b11111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_clti_u_h(a: v8u16, imm5: i32) -> v8i16 { - macro_rules! call { - ($imm5:expr) => { - msa_clti_u_h(a, $imm5) - }; - } - constify_imm5!(imm5, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_clti_u_h(a: v8u16) -> v8i16 { + static_assert_imm5!(IMM5); + msa_clti_u_h(a, IMM5) } /// Immediate Compare Unsigned Less Than @@ -3394,14 +3206,10 @@ pub unsafe fn __msa_clti_u_h(a: v8u16, imm5: i32) -> v8i16 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(clti_u.w, imm5 = 0b11111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_clti_u_w(a: v4u32, imm5: i32) -> v4i32 { - macro_rules! call { - ($imm5:expr) => { - msa_clti_u_w(a, $imm5) - }; - } - constify_imm5!(imm5, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_clti_u_w(a: v4u32) -> v4i32 { + static_assert_imm5!(IMM5); + msa_clti_u_w(a, IMM5) } /// Immediate Compare Unsigned Less Than @@ -3414,14 +3222,10 @@ pub unsafe fn __msa_clti_u_w(a: v4u32, imm5: i32) -> v4i32 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(clti_u.d, imm5 = 0b11111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_clti_u_d(a: v2u64, imm5: i32) -> v2i64 { - macro_rules! call { - ($imm5:expr) => { - msa_clti_u_d(a, $imm5) - }; - } - constify_imm5!(imm5, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_clti_u_d(a: v2u64) -> v2i64 { + static_assert_imm5!(IMM5); + msa_clti_u_d(a, IMM5) } /// Element Copy to GPR Signed @@ -3432,14 +3236,10 @@ pub unsafe fn __msa_clti_u_d(a: v2u64, imm5: i32) -> v2i64 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(copy_s.b, imm4 = 0b1111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_copy_s_b(a: v16i8, imm4: i32) -> i32 { - macro_rules! call { - ($imm4:expr) => { - msa_copy_s_b(a, $imm4) - }; - } - constify_imm4!(imm4, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_copy_s_b(a: v16i8) -> i32 { + static_assert_imm4!(IMM4); + msa_copy_s_b(a, IMM4) } /// Element Copy to GPR Signed @@ -3450,14 +3250,10 @@ pub unsafe fn __msa_copy_s_b(a: v16i8, imm4: i32) -> i32 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(copy_s.h, imm3 = 0b111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_copy_s_h(a: v8i16, imm3: i32) -> i32 { - macro_rules! call { - ($imm3:expr) => { - msa_copy_s_h(a, $imm3) - }; - } - constify_imm3!(imm3, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_copy_s_h(a: v8i16) -> i32 { + static_assert_imm3!(IMM3); + msa_copy_s_h(a, IMM3) } /// Element Copy to GPR Signed @@ -3468,14 +3264,10 @@ pub unsafe fn __msa_copy_s_h(a: v8i16, imm3: i32) -> i32 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(copy_s.w, imm2 = 0b11))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_copy_s_w(a: v4i32, imm2: i32) -> i32 { - macro_rules! call { - ($imm2:expr) => { - msa_copy_s_w(a, $imm2) - }; - } - constify_imm2!(imm2, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_copy_s_w(a: v4i32) -> i32 { + static_assert_imm2!(IMM2); + msa_copy_s_w(a, IMM2) } /// Element Copy to GPR Signed @@ -3486,14 +3278,10 @@ pub unsafe fn __msa_copy_s_w(a: v4i32, imm2: i32) -> i32 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(copy_s.d, imm1 = 0b1))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_copy_s_d(a: v2i64, imm1: i32) -> i64 { - macro_rules! call { - ($imm1:expr) => { - msa_copy_s_d(a, $imm1) - }; - } - constify_imm1!(imm1, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_copy_s_d(a: v2i64) -> i64 { + static_assert_imm1!(IMM1); + msa_copy_s_d(a, IMM1) } /// Element Copy to GPR Unsigned @@ -3504,14 +3292,10 @@ pub unsafe fn __msa_copy_s_d(a: v2i64, imm1: i32) -> i64 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(copy_u.b, imm4 = 0b1111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_copy_u_b(a: v16i8, imm4: i32) -> u32 { - macro_rules! call { - ($imm4:expr) => { - msa_copy_u_b(a, $imm4) - }; - } - constify_imm4!(imm4, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_copy_u_b(a: v16i8) -> u32 { + static_assert_imm4!(IMM4); + msa_copy_u_b(a, IMM4) } /// Element Copy to GPR Unsigned @@ -3522,14 +3306,10 @@ pub unsafe fn __msa_copy_u_b(a: v16i8, imm4: i32) -> u32 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(copy_u.h, imm3 = 0b111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_copy_u_h(a: v8i16, imm3: i32) -> u32 { - macro_rules! call { - ($imm3:expr) => { - msa_copy_u_h(a, $imm3) - }; - } - constify_imm3!(imm3, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_copy_u_h(a: v8i16) -> u32 { + static_assert_imm3!(IMM3); + msa_copy_u_h(a, IMM3) } /// Element Copy to GPR Unsigned @@ -3540,14 +3320,10 @@ pub unsafe fn __msa_copy_u_h(a: v8i16, imm3: i32) -> u32 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(copy_u.w, imm2 = 0b11))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_copy_u_w(a: v4i32, imm2: i32) -> u32 { - macro_rules! call { - ($imm2:expr) => { - msa_copy_u_w(a, $imm2) - }; - } - constify_imm2!(imm2, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_copy_u_w(a: v4i32) -> u32 { + static_assert_imm2!(IMM2); + msa_copy_u_w(a, IMM2) } /// Element Copy to GPR Unsigned @@ -3558,14 +3334,10 @@ pub unsafe fn __msa_copy_u_w(a: v4i32, imm2: i32) -> u32 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(copy_u.d, imm1 = 0b1))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_copy_u_d(a: v2i64, imm1: i32) -> u64 { - macro_rules! call { - ($imm1:expr) => { - msa_copy_u_d(a, $imm1) - }; - } - constify_imm1!(imm1, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_copy_u_d(a: v2i64) -> u64 { + static_assert_imm1!(IMM1); + msa_copy_u_d(a, IMM1) } /// GPR Copy to MSA Control Register @@ -3578,14 +3350,10 @@ pub unsafe fn __msa_copy_u_d(a: v2i64, imm1: i32) -> u64 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(ctcmsa, imm1 = 0b1))] -#[rustc_args_required_const(0)] -pub unsafe fn __msa_ctcmsa(imm5: i32, a: i32) -> () { - macro_rules! call { - ($imm5:expr) => { - msa_ctcmsa($imm5, a) - }; - } - constify_imm5!(imm5, call) +#[rustc_legacy_const_generics(0)] +pub unsafe fn __msa_ctcmsa(a: i32) -> () { + static_assert_imm5!(IMM5); + msa_ctcmsa(IMM5, a) } /// Vector Signed Divide @@ -5798,14 +5566,10 @@ pub unsafe fn __msa_ilvr_d(a: v2i64, b: v2i64) -> v2i64 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(insert.b, imm4 = 0b1111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_insert_b(a: v16i8, imm4: i32, c: i32) -> v16i8 { - macro_rules! call { - ($imm4:expr) => { - msa_insert_b(a, $imm4, c) - }; - } - constify_imm4!(imm4, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_insert_b(a: v16i8, c: i32) -> v16i8 { + static_assert_imm4!(IMM4); + msa_insert_b(a, IMM4, c) } /// GPR Insert Element @@ -5817,14 +5581,10 @@ pub unsafe fn __msa_insert_b(a: v16i8, imm4: i32, c: i32) -> v16i8 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(insert.h, imm3 = 0b111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_insert_h(a: v8i16, imm3: i32, c: i32) -> v8i16 { - macro_rules! call { - ($imm3:expr) => { - msa_insert_h(a, $imm3, c) - }; - } - constify_imm3!(imm3, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_insert_h(a: v8i16, c: i32) -> v8i16 { + static_assert_imm3!(IMM3); + msa_insert_h(a, IMM3, c) } /// GPR Insert Element @@ -5836,14 +5596,10 @@ pub unsafe fn __msa_insert_h(a: v8i16, imm3: i32, c: i32) -> v8i16 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(insert.w, imm2 = 0b11))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_insert_w(a: v4i32, imm2: i32, c: i32) -> v4i32 { - macro_rules! call { - ($imm2:expr) => { - msa_insert_w(a, $imm2, c) - }; - } - constify_imm2!(imm2, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_insert_w(a: v4i32, c: i32) -> v4i32 { + static_assert_imm2!(IMM2); + msa_insert_w(a, IMM2, c) } /// GPR Insert Element @@ -5855,14 +5611,10 @@ pub unsafe fn __msa_insert_w(a: v4i32, imm2: i32, c: i32) -> v4i32 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(insert.d, imm1 = 0b1))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_insert_d(a: v2i64, imm1: i32, c: i64) -> v2i64 { - macro_rules! call { - ($imm1:expr) => { - msa_insert_d(a, $imm1, c) - }; - } - constify_imm1!(imm1, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_insert_d(a: v2i64, c: i64) -> v2i64 { + static_assert_imm1!(IMM1); + msa_insert_d(a, IMM1, c) } /// Element Insert Element @@ -5874,14 +5626,10 @@ pub unsafe fn __msa_insert_d(a: v2i64, imm1: i32, c: i64) -> v2i64 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(insve.b, imm4 = 0b1111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_insve_b(a: v16i8, imm4: i32, c: v16i8) -> v16i8 { - macro_rules! call { - ($imm4:expr) => { - msa_insve_b(a, $imm4, c) - }; - } - constify_imm4!(imm4, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_insve_b(a: v16i8, c: v16i8) -> v16i8 { + static_assert_imm4!(IMM4); + msa_insve_b(a, IMM4, c) } /// Element Insert Element @@ -5893,14 +5641,10 @@ pub unsafe fn __msa_insve_b(a: v16i8, imm4: i32, c: v16i8) -> v16i8 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(insve.h, imm3 = 0b111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_insve_h(a: v8i16, imm3: i32, c: v8i16) -> v8i16 { - macro_rules! call { - ($imm3:expr) => { - msa_insve_h(a, $imm3, c) - }; - } - constify_imm3!(imm3, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_insve_h(a: v8i16, c: v8i16) -> v8i16 { + static_assert_imm3!(IMM3); + msa_insve_h(a, IMM3, c) } /// Element Insert Element @@ -5912,14 +5656,10 @@ pub unsafe fn __msa_insve_h(a: v8i16, imm3: i32, c: v8i16) -> v8i16 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(insve.w, imm2 = 0b11))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_insve_w(a: v4i32, imm2: i32, c: v4i32) -> v4i32 { - macro_rules! call { - ($imm2:expr) => { - msa_insve_w(a, $imm2, c) - }; - } - constify_imm2!(imm2, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_insve_w(a: v4i32, c: v4i32) -> v4i32 { + static_assert_imm2!(IMM2); + msa_insve_w(a, IMM2, c) } /// Element Insert Element @@ -5931,14 +5671,10 @@ pub unsafe fn __msa_insve_w(a: v4i32, imm2: i32, c: v4i32) -> v4i32 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(insve.d, imm1 = 0b1))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_insve_d(a: v2i64, imm1: i32, c: v2i64) -> v2i64 { - macro_rules! call { - ($imm1:expr) => { - msa_insve_d(a, $imm1, c) - }; - } - constify_imm1!(imm1, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_insve_d(a: v2i64, c: v2i64) -> v2i64 { + static_assert_imm1!(IMM1); + msa_insve_d(a, IMM1, c) } /// Vector Load @@ -5950,14 +5686,10 @@ pub unsafe fn __msa_insve_d(a: v2i64, imm1: i32, c: v2i64) -> v2i64 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(ld.b, imm_s10 = 0b1111111111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_ld_b(mem_addr: *mut u8, imm_s10: i32) -> v16i8 { - macro_rules! call { - ($imm_s10:expr) => { - msa_ld_b(mem_addr, $imm_s10) - }; - } - constify_imm_s10!(imm_s10, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_ld_b(mem_addr: *mut u8) -> v16i8 { + static_assert_imm_s10!(IMM_S10); + msa_ld_b(mem_addr, IMM_S10) } /// Vector Load @@ -5969,14 +5701,11 @@ pub unsafe fn __msa_ld_b(mem_addr: *mut u8, imm_s10: i32) -> v16i8 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(ld.h, imm_s11 = 0b11111111111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_ld_h(mem_addr: *mut u8, imm_s11: i32) -> v8i16 { - macro_rules! call { - ($imm_s11:expr) => { - msa_ld_h(mem_addr, $imm_s11) - }; - } - constify_imm_s11!(imm_s11, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_ld_h(mem_addr: *mut u8) -> v8i16 { + static_assert_imm_s11!(IMM_S11); + static_assert!(IMM_S11: i32 where IMM_S11 % 2 == 0); + msa_ld_h(mem_addr, IMM_S11) } /// Vector Load @@ -5988,14 +5717,11 @@ pub unsafe fn __msa_ld_h(mem_addr: *mut u8, imm_s11: i32) -> v8i16 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(ld.w, imm_s12 = 0b111111111111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_ld_w(mem_addr: *mut u8, imm_s12: i32) -> v4i32 { - macro_rules! call { - ($imm_s12:expr) => { - msa_ld_w(mem_addr, $imm_s12) - }; - } - constify_imm_s12!(imm_s12, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_ld_w(mem_addr: *mut u8) -> v4i32 { + static_assert_imm_s12!(IMM_S12); + static_assert!(IMM_S12: i32 where IMM_S12 % 4 == 0); + msa_ld_w(mem_addr, IMM_S12) } /// Vector Load @@ -6007,14 +5733,11 @@ pub unsafe fn __msa_ld_w(mem_addr: *mut u8, imm_s12: i32) -> v4i32 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(ld.d, imm_s13 = 0b1111111111111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_ld_d(mem_addr: *mut u8, imm_s13: i32) -> v2i64 { - macro_rules! call { - ($imm_s13:expr) => { - msa_ld_d(mem_addr, $imm_s13) - }; - } - constify_imm_s13!(imm_s13, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_ld_d(mem_addr: *mut u8) -> v2i64 { + static_assert_imm_s13!(IMM_S13); + static_assert!(IMM_S13: i32 where IMM_S13 % 8 == 0); + msa_ld_d(mem_addr, IMM_S13) } /// Immediate Load @@ -6026,14 +5749,10 @@ pub unsafe fn __msa_ld_d(mem_addr: *mut u8, imm_s13: i32) -> v2i64 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(ldi.b, imm_s10 = 0b1111111111))] -#[rustc_args_required_const(0)] -pub unsafe fn __msa_ldi_b(imm_s10: i32) -> v16i8 { - macro_rules! call { - ($imm_s10:expr) => { - msa_ldi_b($imm_s10) - }; - } - constify_imm_s10!(imm_s10, call) +#[rustc_legacy_const_generics(0)] +pub unsafe fn __msa_ldi_b() -> v16i8 { + static_assert_imm_s10!(IMM_S10); + msa_ldi_b(IMM_S10) } /// Immediate Load @@ -6045,14 +5764,10 @@ pub unsafe fn __msa_ldi_b(imm_s10: i32) -> v16i8 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(ldi.h, imm_s10 = 0b1111111111))] -#[rustc_args_required_const(0)] -pub unsafe fn __msa_ldi_h(imm_s10: i32) -> v8i16 { - macro_rules! call { - ($imm_s10:expr) => { - msa_ldi_h($imm_s10) - }; - } - constify_imm_s10!(imm_s10, call) +#[rustc_legacy_const_generics(0)] +pub unsafe fn __msa_ldi_h() -> v8i16 { + static_assert_imm_s10!(IMM_S10); + msa_ldi_h(IMM_S10) } /// Immediate Load @@ -6064,14 +5779,10 @@ pub unsafe fn __msa_ldi_h(imm_s10: i32) -> v8i16 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(ldi.w, imm_s10 = 0b1111111111))] -#[rustc_args_required_const(0)] -pub unsafe fn __msa_ldi_w(imm_s10: i32) -> v4i32 { - macro_rules! call { - ($imm_s10:expr) => { - msa_ldi_w($imm_s10) - }; - } - constify_imm_s10!(imm_s10, call) +#[rustc_legacy_const_generics(0)] +pub unsafe fn __msa_ldi_w() -> v4i32 { + static_assert_imm_s10!(IMM_S10); + msa_ldi_w(IMM_S10) } /// Immediate Load @@ -6083,14 +5794,10 @@ pub unsafe fn __msa_ldi_w(imm_s10: i32) -> v4i32 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(ldi.d, imm_s10 = 0b1111111111))] -#[rustc_args_required_const(0)] -pub unsafe fn __msa_ldi_d(imm_s10: i32) -> v2i64 { - macro_rules! call { - ($imm_s10:expr) => { - msa_ldi_d($imm_s10) - }; - } - constify_imm_s10!(imm_s10, call) +#[rustc_legacy_const_generics(0)] +pub unsafe fn __msa_ldi_d() -> v2i64 { + static_assert_imm_s10!(IMM_S10); + msa_ldi_d(IMM_S10) } /// Vector Fixed-Point Multiply and Add @@ -6378,14 +6085,10 @@ pub unsafe fn __msa_max_u_d(a: v2u64, b: v2u64) -> v2u64 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(maxi_s.b, imm5 = 0b11111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_maxi_s_b(a: v16i8, imm_s5: i32) -> v16i8 { - macro_rules! call { - ($imm_s5:expr) => { - msa_maxi_s_b(a, $imm_s5) - }; - } - constify_imm_s5!(imm_s5, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_maxi_s_b(a: v16i8) -> v16i8 { + static_assert_imm_s5!(IMM_S5); + msa_maxi_s_b(a, IMM_S5) } /// Immediate Signed Maximum @@ -6397,14 +6100,10 @@ pub unsafe fn __msa_maxi_s_b(a: v16i8, imm_s5: i32) -> v16i8 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(maxi_s.h, imm_s5 = 0b11111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_maxi_s_h(a: v8i16, imm_s5: i32) -> v8i16 { - macro_rules! call { - ($imm_s5:expr) => { - msa_maxi_s_h(a, $imm_s5) - }; - } - constify_imm_s5!(imm_s5, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_maxi_s_h(a: v8i16) -> v8i16 { + static_assert_imm_s5!(IMM_S5); + msa_maxi_s_h(a, IMM_S5) } /// Immediate Signed Maximum @@ -6416,14 +6115,10 @@ pub unsafe fn __msa_maxi_s_h(a: v8i16, imm_s5: i32) -> v8i16 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(maxi_s.w, imm_s5 = 0b11111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_maxi_s_w(a: v4i32, imm_s5: i32) -> v4i32 { - macro_rules! call { - ($imm_s5:expr) => { - msa_maxi_s_w(a, $imm_s5) - }; - } - constify_imm_s5!(imm_s5, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_maxi_s_w(a: v4i32) -> v4i32 { + static_assert_imm_s5!(IMM_S5); + msa_maxi_s_w(a, IMM_S5) } /// Immediate Signed Maximum @@ -6435,14 +6130,10 @@ pub unsafe fn __msa_maxi_s_w(a: v4i32, imm_s5: i32) -> v4i32 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(maxi_s.d, imm_s5 = 0b11111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_maxi_s_d(a: v2i64, imm_s5: i32) -> v2i64 { - macro_rules! call { - ($imm_s5:expr) => { - msa_maxi_s_d(a, $imm_s5) - }; - } - constify_imm_s5!(imm_s5, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_maxi_s_d(a: v2i64) -> v2i64 { + static_assert_imm_s5!(IMM_S5); + msa_maxi_s_d(a, IMM_S5) } /// Immediate Unsigned Maximum @@ -6454,14 +6145,10 @@ pub unsafe fn __msa_maxi_s_d(a: v2i64, imm_s5: i32) -> v2i64 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(maxi_u.b, imm5 = 0b11111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_maxi_u_b(a: v16u8, imm5: i32) -> v16u8 { - macro_rules! call { - ($imm5:expr) => { - msa_maxi_u_b(a, $imm5) - }; - } - constify_imm5!(imm5, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_maxi_u_b(a: v16u8) -> v16u8 { + static_assert_imm5!(IMM5); + msa_maxi_u_b(a, IMM5) } /// Immediate Unsigned Maximum @@ -6473,14 +6160,10 @@ pub unsafe fn __msa_maxi_u_b(a: v16u8, imm5: i32) -> v16u8 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(maxi_u.h, imm5 = 0b11111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_maxi_u_h(a: v8u16, imm5: i32) -> v8u16 { - macro_rules! call { - ($imm5:expr) => { - msa_maxi_u_h(a, $imm5) - }; - } - constify_imm5!(imm5, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_maxi_u_h(a: v8u16) -> v8u16 { + static_assert_imm5!(IMM5); + msa_maxi_u_h(a, IMM5) } /// Immediate Unsigned Maximum @@ -6492,14 +6175,10 @@ pub unsafe fn __msa_maxi_u_h(a: v8u16, imm5: i32) -> v8u16 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(maxi_u.w, imm5 = 0b11111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_maxi_u_w(a: v4u32, imm5: i32) -> v4u32 { - macro_rules! call { - ($imm5:expr) => { - msa_maxi_u_w(a, $imm5) - }; - } - constify_imm5!(imm5, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_maxi_u_w(a: v4u32) -> v4u32 { + static_assert_imm5!(IMM5); + msa_maxi_u_w(a, IMM5) } /// Immediate Unsigned Maximum @@ -6511,14 +6190,10 @@ pub unsafe fn __msa_maxi_u_w(a: v4u32, imm5: i32) -> v4u32 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(maxi_u.d, imm5 = 0b11111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_maxi_u_d(a: v2u64, imm5: i32) -> v2u64 { - macro_rules! call { - ($imm5:expr) => { - msa_maxi_u_d(a, $imm5) - }; - } - constify_imm5!(imm5, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_maxi_u_d(a: v2u64) -> v2u64 { + static_assert_imm5!(IMM5); + msa_maxi_u_d(a, IMM5) } /// Vector Minimum Based on Absolute Value @@ -6638,14 +6313,10 @@ pub unsafe fn __msa_min_s_d(a: v2i64, b: v2i64) -> v2i64 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(mini_s.b, imm_s5 = 0b11111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_mini_s_b(a: v16i8, imm_s5: i32) -> v16i8 { - macro_rules! call { - ($imm_s5:expr) => { - msa_mini_s_b(a, $imm_s5) - }; - } - constify_imm_s5!(imm_s5, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_mini_s_b(a: v16i8) -> v16i8 { + static_assert_imm_s5!(IMM_S5); + msa_mini_s_b(a, IMM_S5) } /// Immediate Signed Minimum @@ -6657,14 +6328,10 @@ pub unsafe fn __msa_mini_s_b(a: v16i8, imm_s5: i32) -> v16i8 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(mini_s.h, imm_s5 = 0b11111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_mini_s_h(a: v8i16, imm_s5: i32) -> v8i16 { - macro_rules! call { - ($imm_s5:expr) => { - msa_mini_s_h(a, $imm_s5) - }; - } - constify_imm_s5!(imm_s5, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_mini_s_h(a: v8i16) -> v8i16 { + static_assert_imm_s5!(IMM_S5); + msa_mini_s_h(a, IMM_S5) } /// Immediate Signed Minimum @@ -6676,14 +6343,10 @@ pub unsafe fn __msa_mini_s_h(a: v8i16, imm_s5: i32) -> v8i16 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(mini_s.w, imm_s5 = 0b11111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_mini_s_w(a: v4i32, imm_s5: i32) -> v4i32 { - macro_rules! call { - ($imm_s5:expr) => { - msa_mini_s_w(a, $imm_s5) - }; - } - constify_imm_s5!(imm_s5, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_mini_s_w(a: v4i32) -> v4i32 { + static_assert_imm_s5!(IMM_S5); + msa_mini_s_w(a, IMM_S5) } /// Immediate Signed Minimum @@ -6695,14 +6358,10 @@ pub unsafe fn __msa_mini_s_w(a: v4i32, imm_s5: i32) -> v4i32 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(mini_s.d, imm_s5 = 0b11111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_mini_s_d(a: v2i64, imm_s5: i32) -> v2i64 { - macro_rules! call { - ($imm_s5:expr) => { - msa_mini_s_d(a, $imm_s5) - }; - } - constify_imm_s5!(imm_s5, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_mini_s_d(a: v2i64) -> v2i64 { + static_assert_imm_s5!(IMM_S5); + msa_mini_s_d(a, IMM_S5) } /// Vector Unsigned Minimum @@ -6766,14 +6425,10 @@ pub unsafe fn __msa_min_u_d(a: v2u64, b: v2u64) -> v2u64 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(mini_u.b, imm5 = 0b11111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_mini_u_b(a: v16u8, imm5: i32) -> v16u8 { - macro_rules! call { - ($imm5:expr) => { - msa_mini_u_b(a, $imm5) - }; - } - constify_imm5!(imm5, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_mini_u_b(a: v16u8) -> v16u8 { + static_assert_imm5!(IMM5); + msa_mini_u_b(a, IMM5) } /// Immediate Unsigned Minimum @@ -6785,14 +6440,10 @@ pub unsafe fn __msa_mini_u_b(a: v16u8, imm5: i32) -> v16u8 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(mini_u.h, imm5 = 0b11111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_mini_u_h(a: v8u16, imm5: i32) -> v8u16 { - macro_rules! call { - ($imm5:expr) => { - msa_mini_u_h(a, $imm5) - }; - } - constify_imm5!(imm5, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_mini_u_h(a: v8u16) -> v8u16 { + static_assert_imm5!(IMM5); + msa_mini_u_h(a, IMM5) } /// Immediate Unsigned Minimum @@ -6804,14 +6455,10 @@ pub unsafe fn __msa_mini_u_h(a: v8u16, imm5: i32) -> v8u16 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(mini_u.w, imm5 = 0b11111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_mini_u_w(a: v4u32, imm5: i32) -> v4u32 { - macro_rules! call { - ($imm5:expr) => { - msa_mini_u_w(a, $imm5) - }; - } - constify_imm5!(imm5, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_mini_u_w(a: v4u32) -> v4u32 { + static_assert_imm5!(IMM5); + msa_mini_u_w(a, IMM5) } /// Immediate Unsigned Minimum @@ -6823,14 +6470,10 @@ pub unsafe fn __msa_mini_u_w(a: v4u32, imm5: i32) -> v4u32 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(mini_u.d, imm5 = 0b11111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_mini_u_d(a: v2u64, imm5: i32) -> v2u64 { - macro_rules! call { - ($imm5:expr) => { - msa_mini_u_d(a, $imm5) - }; - } - constify_imm5!(imm5, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_mini_u_d(a: v2u64) -> v2u64 { + static_assert_imm5!(IMM5); + msa_mini_u_d(a, IMM5) } /// Vector Signed Modulo @@ -7313,14 +6956,10 @@ pub unsafe fn __msa_nor_v(a: v16u8, b: v16u8) -> v16u8 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(nori.b, imm8 = 0b11111111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_nori_b(a: v16u8, imm8: i32) -> v16u8 { - macro_rules! call { - ($imm8:expr) => { - msa_nori_b(a, $imm8) - }; - } - constify_imm8!(imm8, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_nori_b(a: v16u8) -> v16u8 { + static_assert_imm8!(IMM8); + msa_nori_b(a, IMM8) } /// Vector Logical Or @@ -7347,14 +6986,10 @@ pub unsafe fn __msa_or_v(a: v16u8, b: v16u8) -> v16u8 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(ori.b, imm8 = 0b11111111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_ori_b(a: v16u8, imm8: i32) -> v16u8 { - macro_rules! call { - ($imm8:expr) => { - msa_ori_b(a, $imm8) - }; - } - constify_imm8!(imm8, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_ori_b(a: v16u8) -> v16u8 { + static_assert_imm8!(IMM8); + msa_ori_b(a, IMM8) } /// Vector Pack Even @@ -7518,14 +7153,10 @@ pub unsafe fn __msa_pcnt_d(a: v2i64) -> v2i64 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(sat_s.b, imm4 = 0b111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_sat_s_b(a: v16i8, imm3: i32) -> v16i8 { - macro_rules! call { - ($imm3:expr) => { - msa_sat_s_b(a, $imm3) - }; - } - constify_imm3!(imm3, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_sat_s_b(a: v16i8) -> v16i8 { + static_assert_imm3!(IMM3); + msa_sat_s_b(a, IMM3) } /// Immediate Signed Saturate @@ -7537,14 +7168,10 @@ pub unsafe fn __msa_sat_s_b(a: v16i8, imm3: i32) -> v16i8 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(sat_s.h, imm3 = 0b1111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_sat_s_h(a: v8i16, imm4: i32) -> v8i16 { - macro_rules! call { - ($imm4:expr) => { - msa_sat_s_h(a, $imm4) - }; - } - constify_imm4!(imm4, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_sat_s_h(a: v8i16) -> v8i16 { + static_assert_imm4!(IMM4); + msa_sat_s_h(a, IMM4) } /// Immediate Signed Saturate @@ -7556,14 +7183,10 @@ pub unsafe fn __msa_sat_s_h(a: v8i16, imm4: i32) -> v8i16 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(sat_s.w, imm2 = 0b11111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_sat_s_w(a: v4i32, imm5: i32) -> v4i32 { - macro_rules! call { - ($imm5:expr) => { - msa_sat_s_w(a, $imm5) - }; - } - constify_imm5!(imm5, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_sat_s_w(a: v4i32) -> v4i32 { + static_assert_imm5!(IMM5); + msa_sat_s_w(a, IMM5) } /// Immediate Signed Saturate @@ -7575,14 +7198,10 @@ pub unsafe fn __msa_sat_s_w(a: v4i32, imm5: i32) -> v4i32 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(sat_s.d, imm1 = 0b111111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_sat_s_d(a: v2i64, imm6: i32) -> v2i64 { - macro_rules! call { - ($imm6:expr) => { - msa_sat_s_d(a, $imm6) - }; - } - constify_imm6!(imm6, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_sat_s_d(a: v2i64) -> v2i64 { + static_assert_imm6!(IMM6); + msa_sat_s_d(a, IMM6) } /// Immediate Unsigned Saturate @@ -7594,14 +7213,10 @@ pub unsafe fn __msa_sat_s_d(a: v2i64, imm6: i32) -> v2i64 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(sat_u.b, imm4 = 0b111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_sat_u_b(a: v16u8, imm3: i32) -> v16u8 { - macro_rules! call { - ($imm3:expr) => { - msa_sat_u_b(a, $imm3) - }; - } - constify_imm3!(imm3, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_sat_u_b(a: v16u8) -> v16u8 { + static_assert_imm3!(IMM3); + msa_sat_u_b(a, IMM3) } /// Immediate Unsigned Saturate @@ -7613,14 +7228,10 @@ pub unsafe fn __msa_sat_u_b(a: v16u8, imm3: i32) -> v16u8 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(sat_u.h, imm3 = 0b1111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_sat_u_h(a: v8u16, imm4: i32) -> v8u16 { - macro_rules! call { - ($imm4:expr) => { - msa_sat_u_h(a, $imm4) - }; - } - constify_imm4!(imm4, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_sat_u_h(a: v8u16) -> v8u16 { + static_assert_imm4!(IMM4); + msa_sat_u_h(a, IMM4) } /// Immediate Unsigned Saturate @@ -7632,14 +7243,10 @@ pub unsafe fn __msa_sat_u_h(a: v8u16, imm4: i32) -> v8u16 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(sat_u.w, imm2 = 0b11111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_sat_u_w(a: v4u32, imm5: i32) -> v4u32 { - macro_rules! call { - ($imm5:expr) => { - msa_sat_u_w(a, $imm5) - }; - } - constify_imm5!(imm5, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_sat_u_w(a: v4u32) -> v4u32 { + static_assert_imm5!(IMM5); + msa_sat_u_w(a, IMM5) } /// Immediate Unsigned Saturate @@ -7651,14 +7258,10 @@ pub unsafe fn __msa_sat_u_w(a: v4u32, imm5: i32) -> v4u32 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(sat_u.d, imm1 = 0b111111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_sat_u_d(a: v2u64, imm6: i32) -> v2u64 { - macro_rules! call { - ($imm6:expr) => { - msa_sat_u_d(a, $imm6) - }; - } - constify_imm6!(imm6, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_sat_u_d(a: v2u64) -> v2u64 { + static_assert_imm6!(IMM6); + msa_sat_u_d(a, IMM6) } /// Immediate Set Shuffle Elements @@ -7671,14 +7274,10 @@ pub unsafe fn __msa_sat_u_d(a: v2u64, imm6: i32) -> v2u64 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(shf.b, imm8 = 0b11111111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_shf_b(a: v16i8, imm8: i32) -> v16i8 { - macro_rules! call { - ($imm8:expr) => { - msa_shf_b(a, $imm8) - }; - } - constify_imm8!(imm8, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_shf_b(a: v16i8) -> v16i8 { + static_assert_imm8!(IMM8); + msa_shf_b(a, IMM8) } /// Immediate Set Shuffle Elements @@ -7691,14 +7290,10 @@ pub unsafe fn __msa_shf_b(a: v16i8, imm8: i32) -> v16i8 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(shf.h, imm8 = 0b11111111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_shf_h(a: v8i16, imm8: i32) -> v8i16 { - macro_rules! call { - ($imm8:expr) => { - msa_shf_h(a, $imm8) - }; - } - constify_imm8!(imm8, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_shf_h(a: v8i16) -> v8i16 { + static_assert_imm8!(IMM8); + msa_shf_h(a, IMM8) } /// Immediate Set Shuffle Elements @@ -7711,14 +7306,10 @@ pub unsafe fn __msa_shf_h(a: v8i16, imm8: i32) -> v8i16 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(shf.w, imm8 = 0b11111111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_shf_w(a: v4i32, imm8: i32) -> v4i32 { - macro_rules! call { - ($imm8:expr) => { - msa_shf_w(a, $imm8) - }; - } - constify_imm8!(imm8, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_shf_w(a: v4i32) -> v4i32 { + static_assert_imm8!(IMM8); + msa_shf_w(a, IMM8) } /// GPR Columns Slide @@ -7815,14 +7406,10 @@ pub unsafe fn __msa_sld_d(a: v2i64, b: v2i64, c: i32) -> v2i64 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(sldi.b, imm4 = 0b1111))] -#[rustc_args_required_const(2)] -pub unsafe fn __msa_sldi_b(a: v16i8, b: v16i8, imm4: i32) -> v16i8 { - macro_rules! call { - ($imm4:expr) => { - msa_sldi_b(a, mem::transmute(b), $imm4) - }; - } - constify_imm4!(imm4, call) +#[rustc_legacy_const_generics(2)] +pub unsafe fn __msa_sldi_b(a: v16i8, b: v16i8) -> v16i8 { + static_assert_imm4!(IMM4); + msa_sldi_b(a, mem::transmute(b), IMM4) } /// Immediate Columns Slide @@ -7839,14 +7426,10 @@ pub unsafe fn __msa_sldi_b(a: v16i8, b: v16i8, imm4: i32) -> v16i8 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(sldi.h, imm3 = 0b111))] -#[rustc_args_required_const(2)] -pub unsafe fn __msa_sldi_h(a: v8i16, b: v8i16, imm3: i32) -> v8i16 { - macro_rules! call { - ($imm3:expr) => { - msa_sldi_h(a, mem::transmute(b), $imm3) - }; - } - constify_imm3!(imm3, call) +#[rustc_legacy_const_generics(2)] +pub unsafe fn __msa_sldi_h(a: v8i16, b: v8i16) -> v8i16 { + static_assert_imm3!(IMM3); + msa_sldi_h(a, mem::transmute(b), IMM3) } /// Immediate Columns Slide @@ -7863,14 +7446,10 @@ pub unsafe fn __msa_sldi_h(a: v8i16, b: v8i16, imm3: i32) -> v8i16 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(sldi.w, imm2 = 0b11))] -#[rustc_args_required_const(2)] -pub unsafe fn __msa_sldi_w(a: v4i32, b: v4i32, imm2: i32) -> v4i32 { - macro_rules! call { - ($imm2:expr) => { - msa_sldi_w(a, mem::transmute(b), $imm2) - }; - } - constify_imm2!(imm2, call) +#[rustc_legacy_const_generics(2)] +pub unsafe fn __msa_sldi_w(a: v4i32, b: v4i32) -> v4i32 { + static_assert_imm2!(IMM2); + msa_sldi_w(a, mem::transmute(b), IMM2) } /// Immediate Columns Slide @@ -7887,14 +7466,10 @@ pub unsafe fn __msa_sldi_w(a: v4i32, b: v4i32, imm2: i32) -> v4i32 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(sldi.d, imm1 = 0b1))] -#[rustc_args_required_const(2)] -pub unsafe fn __msa_sldi_d(a: v2i64, b: v2i64, imm1: i32) -> v2i64 { - macro_rules! call { - ($imm1:expr) => { - msa_sldi_d(a, mem::transmute(b), $imm1) - }; - } - constify_imm1!(imm1, call) +#[rustc_legacy_const_generics(2)] +pub unsafe fn __msa_sldi_d(a: v2i64, b: v2i64) -> v2i64 { + static_assert_imm1!(IMM1); + msa_sldi_d(a, mem::transmute(b), IMM1) } /// Vector Shift Left @@ -7962,14 +7537,10 @@ pub unsafe fn __msa_sll_d(a: v2i64, b: v2i64) -> v2i64 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(slli.b, imm4 = 0b1111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_slli_b(a: v16i8, imm4: i32) -> v16i8 { - macro_rules! call { - ($imm4:expr) => { - msa_slli_b(a, $imm4) - }; - } - constify_imm4!(imm4, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_slli_b(a: v16i8) -> v16i8 { + static_assert_imm4!(IMM4); + msa_slli_b(a, IMM4) } /// Immediate Shift Left @@ -7981,14 +7552,10 @@ pub unsafe fn __msa_slli_b(a: v16i8, imm4: i32) -> v16i8 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(slli.h, imm3 = 0b111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_slli_h(a: v8i16, imm3: i32) -> v8i16 { - macro_rules! call { - ($imm3:expr) => { - msa_slli_h(a, $imm3) - }; - } - constify_imm3!(imm3, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_slli_h(a: v8i16) -> v8i16 { + static_assert_imm3!(IMM3); + msa_slli_h(a, IMM3) } /// Immediate Shift Left @@ -8000,14 +7567,10 @@ pub unsafe fn __msa_slli_h(a: v8i16, imm3: i32) -> v8i16 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(slli.w, imm2 = 0b11))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_slli_w(a: v4i32, imm2: i32) -> v4i32 { - macro_rules! call { - ($imm2:expr) => { - msa_slli_w(a, $imm2) - }; - } - constify_imm2!(imm2, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_slli_w(a: v4i32) -> v4i32 { + static_assert_imm2!(IMM2); + msa_slli_w(a, IMM2) } /// Immediate Shift Left @@ -8019,14 +7582,10 @@ pub unsafe fn __msa_slli_w(a: v4i32, imm2: i32) -> v4i32 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(slli.d, imm1 = 0b1))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_slli_d(a: v2i64, imm1: i32) -> v2i64 { - macro_rules! call { - ($imm1:expr) => { - msa_slli_d(a, $imm1) - }; - } - constify_imm1!(imm1, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_slli_d(a: v2i64) -> v2i64 { + static_assert_imm1!(IMM1); + msa_slli_d(a, IMM1) } /// GPR Element Splat @@ -8093,14 +7652,10 @@ pub unsafe fn __msa_splat_d(a: v2i64, b: i32) -> v2i64 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(splati.b, imm4 = 0b1111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_splati_b(a: v16i8, imm4: i32) -> v16i8 { - macro_rules! call { - ($imm4:expr) => { - msa_splati_b(a, $imm4) - }; - } - constify_imm4!(imm4, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_splati_b(a: v16i8) -> v16i8 { + static_assert_imm4!(IMM4); + msa_splati_b(a, IMM4) } /// Immediate Element Splat @@ -8111,14 +7666,10 @@ pub unsafe fn __msa_splati_b(a: v16i8, imm4: i32) -> v16i8 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(splati.h, imm3 = 0b111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_splati_h(a: v8i16, imm3: i32) -> v8i16 { - macro_rules! call { - ($imm3:expr) => { - msa_splati_h(a, $imm3) - }; - } - constify_imm3!(imm3, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_splati_h(a: v8i16) -> v8i16 { + static_assert_imm3!(IMM3); + msa_splati_h(a, IMM3) } /// Immediate Element Splat @@ -8129,14 +7680,10 @@ pub unsafe fn __msa_splati_h(a: v8i16, imm3: i32) -> v8i16 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(splati.w, imm2 = 0b11))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_splati_w(a: v4i32, imm2: i32) -> v4i32 { - macro_rules! call { - ($imm2:expr) => { - msa_splati_w(a, $imm2) - }; - } - constify_imm2!(imm2, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_splati_w(a: v4i32) -> v4i32 { + static_assert_imm2!(IMM2); + msa_splati_w(a, IMM2) } /// Immediate Element Splat @@ -8147,14 +7694,10 @@ pub unsafe fn __msa_splati_w(a: v4i32, imm2: i32) -> v4i32 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(splati.d, imm1 = 0b1))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_splati_d(a: v2i64, imm1: i32) -> v2i64 { - macro_rules! call { - ($imm1:expr) => { - msa_splati_d(a, $imm1) - }; - } - constify_imm1!(imm1, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_splati_d(a: v2i64) -> v2i64 { + static_assert_imm1!(IMM1); + msa_splati_d(a, IMM1) } /// Vector Shift Right Arithmetic @@ -8222,14 +7765,10 @@ pub unsafe fn __msa_sra_d(a: v2i64, b: v2i64) -> v2i64 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(srai.b, imm3 = 0b111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_srai_b(a: v16i8, imm3: i32) -> v16i8 { - macro_rules! call { - ($imm3:expr) => { - msa_srai_b(a, $imm3) - }; - } - constify_imm3!(imm3, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_srai_b(a: v16i8) -> v16i8 { + static_assert_imm3!(IMM3); + msa_srai_b(a, IMM3) } /// Immediate Shift Right Arithmetic @@ -8241,14 +7780,10 @@ pub unsafe fn __msa_srai_b(a: v16i8, imm3: i32) -> v16i8 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(srai.h, imm4 = 0b1111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_srai_h(a: v8i16, imm4: i32) -> v8i16 { - macro_rules! call { - ($imm4:expr) => { - msa_srai_h(a, $imm4) - }; - } - constify_imm4!(imm4, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_srai_h(a: v8i16) -> v8i16 { + static_assert_imm4!(IMM4); + msa_srai_h(a, IMM4) } /// Immediate Shift Right Arithmetic @@ -8260,14 +7795,10 @@ pub unsafe fn __msa_srai_h(a: v8i16, imm4: i32) -> v8i16 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(srai.w, imm5 = 0b11111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_srai_w(a: v4i32, imm5: i32) -> v4i32 { - macro_rules! call { - ($imm5:expr) => { - msa_srai_w(a, $imm5) - }; - } - constify_imm5!(imm5, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_srai_w(a: v4i32) -> v4i32 { + static_assert_imm5!(IMM5); + msa_srai_w(a, IMM5) } /// Immediate Shift Right Arithmetic @@ -8279,14 +7810,10 @@ pub unsafe fn __msa_srai_w(a: v4i32, imm5: i32) -> v4i32 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(srai.d, imm6 = 0b111111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_srai_d(a: v2i64, imm6: i32) -> v2i64 { - macro_rules! call { - ($imm6:expr) => { - msa_srai_d(a, $imm6) - }; - } - constify_imm6!(imm6, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_srai_d(a: v2i64) -> v2i64 { + static_assert_imm6!(IMM6); + msa_srai_d(a, IMM6) } /// Vector Shift Right Arithmetic Rounded @@ -8359,14 +7886,10 @@ pub unsafe fn __msa_srar_d(a: v2i64, b: v2i64) -> v2i64 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(srari.b, imm3 = 0b111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_srari_b(a: v16i8, imm3: i32) -> v16i8 { - macro_rules! call { - ($imm3:expr) => { - msa_srari_b(a, $imm3) - }; - } - constify_imm3!(imm3, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_srari_b(a: v16i8) -> v16i8 { + static_assert_imm3!(IMM3); + msa_srari_b(a, IMM3) } /// Immediate Shift Right Arithmetic Rounded @@ -8379,14 +7902,10 @@ pub unsafe fn __msa_srari_b(a: v16i8, imm3: i32) -> v16i8 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(srari.h, imm4 = 0b1111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_srari_h(a: v8i16, imm4: i32) -> v8i16 { - macro_rules! call { - ($imm4:expr) => { - msa_srari_h(a, $imm4) - }; - } - constify_imm4!(imm4, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_srari_h(a: v8i16) -> v8i16 { + static_assert_imm4!(IMM4); + msa_srari_h(a, IMM4) } /// Immediate Shift Right Arithmetic Rounded @@ -8399,14 +7918,10 @@ pub unsafe fn __msa_srari_h(a: v8i16, imm4: i32) -> v8i16 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(srari.w, imm5 = 0b11111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_srari_w(a: v4i32, imm5: i32) -> v4i32 { - macro_rules! call { - ($imm5:expr) => { - msa_srari_w(a, $imm5) - }; - } - constify_imm5!(imm5, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_srari_w(a: v4i32) -> v4i32 { + static_assert_imm5!(IMM5); + msa_srari_w(a, IMM5) } /// Immediate Shift Right Arithmetic Rounded @@ -8419,14 +7934,10 @@ pub unsafe fn __msa_srari_w(a: v4i32, imm5: i32) -> v4i32 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(srari.d, imm6 = 0b111111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_srari_d(a: v2i64, imm6: i32) -> v2i64 { - macro_rules! call { - ($imm6:expr) => { - msa_srari_d(a, $imm6) - }; - } - constify_imm6!(imm6, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_srari_d(a: v2i64) -> v2i64 { + static_assert_imm6!(IMM6); + msa_srari_d(a, IMM6) } /// Vector Shift Right Logical @@ -8494,14 +8005,10 @@ pub unsafe fn __msa_srl_d(a: v2i64, b: v2i64) -> v2i64 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(srli.b, imm4 = 0b1111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_srli_b(a: v16i8, imm4: i32) -> v16i8 { - macro_rules! call { - ($imm4:expr) => { - msa_srli_b(a, $imm4) - }; - } - constify_imm4!(imm4, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_srli_b(a: v16i8) -> v16i8 { + static_assert_imm4!(IMM4); + msa_srli_b(a, IMM4) } /// Immediate Shift Right Logical @@ -8513,14 +8020,10 @@ pub unsafe fn __msa_srli_b(a: v16i8, imm4: i32) -> v16i8 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(srli.h, imm3 = 0b111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_srli_h(a: v8i16, imm3: i32) -> v8i16 { - macro_rules! call { - ($imm3:expr) => { - msa_srli_h(a, $imm3) - }; - } - constify_imm3!(imm3, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_srli_h(a: v8i16) -> v8i16 { + static_assert_imm3!(IMM3); + msa_srli_h(a, IMM3) } /// Immediate Shift Right Logical @@ -8532,14 +8035,10 @@ pub unsafe fn __msa_srli_h(a: v8i16, imm3: i32) -> v8i16 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(srli.w, imm2 = 0b11))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_srli_w(a: v4i32, imm2: i32) -> v4i32 { - macro_rules! call { - ($imm2:expr) => { - msa_srli_w(a, $imm2) - }; - } - constify_imm2!(imm2, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_srli_w(a: v4i32) -> v4i32 { + static_assert_imm2!(IMM2); + msa_srli_w(a, IMM2) } /// Immediate Shift Right Logical @@ -8551,14 +8050,10 @@ pub unsafe fn __msa_srli_w(a: v4i32, imm2: i32) -> v4i32 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(srli.d, imm1 = 0b1))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_srli_d(a: v2i64, imm1: i32) -> v2i64 { - macro_rules! call { - ($imm1:expr) => { - msa_srli_d(a, $imm1) - }; - } - constify_imm1!(imm1, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_srli_d(a: v2i64) -> v2i64 { + static_assert_imm1!(IMM1); + msa_srli_d(a, IMM1) } /// Vector Shift Right Logical Rounded @@ -8631,14 +8126,10 @@ pub unsafe fn __msa_srlr_d(a: v2i64, b: v2i64) -> v2i64 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(srlri.b, imm3 = 0b111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_srlri_b(a: v16i8, imm3: i32) -> v16i8 { - macro_rules! call { - ($imm3:expr) => { - msa_srlri_b(a, $imm3) - }; - } - constify_imm3!(imm3, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_srlri_b(a: v16i8) -> v16i8 { + static_assert_imm3!(IMM3); + msa_srlri_b(a, IMM3) } /// Immediate Shift Right Logical Rounded @@ -8651,14 +8142,10 @@ pub unsafe fn __msa_srlri_b(a: v16i8, imm3: i32) -> v16i8 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(srlri.h, imm4 = 0b1111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_srlri_h(a: v8i16, imm4: i32) -> v8i16 { - macro_rules! call { - ($imm4:expr) => { - msa_srlri_h(a, $imm4) - }; - } - constify_imm4!(imm4, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_srlri_h(a: v8i16) -> v8i16 { + static_assert_imm4!(IMM4); + msa_srlri_h(a, IMM4) } /// Immediate Shift Right Logical Rounded @@ -8671,14 +8158,10 @@ pub unsafe fn __msa_srlri_h(a: v8i16, imm4: i32) -> v8i16 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(srlri.w, imm5 = 0b11111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_srlri_w(a: v4i32, imm5: i32) -> v4i32 { - macro_rules! call { - ($imm5:expr) => { - msa_srlri_w(a, $imm5) - }; - } - constify_imm5!(imm5, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_srlri_w(a: v4i32) -> v4i32 { + static_assert_imm5!(IMM5); + msa_srlri_w(a, IMM5) } /// Immediate Shift Right Logical Rounded @@ -8691,14 +8174,10 @@ pub unsafe fn __msa_srlri_w(a: v4i32, imm5: i32) -> v4i32 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(srlri.d, imm6 = 0b111111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_srlri_d(a: v2i64, imm6: i32) -> v2i64 { - macro_rules! call { - ($imm6:expr) => { - msa_srlri_d(a, $imm6) - }; - } - constify_imm6!(imm6, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_srlri_d(a: v2i64) -> v2i64 { + static_assert_imm6!(IMM6); + msa_srlri_d(a, IMM6) } /// Vector Store @@ -8710,14 +8189,10 @@ pub unsafe fn __msa_srlri_d(a: v2i64, imm6: i32) -> v2i64 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(st.b, imm_s10 = 0b1111111111))] -#[rustc_args_required_const(2)] -pub unsafe fn __msa_st_b(a: v16i8, mem_addr: *mut u8, imm_s10: i32) -> () { - macro_rules! call { - ($imm_s10:expr) => { - msa_st_b(a, mem_addr, $imm_s10) - }; - } - constify_imm_s10!(imm_s10, call) +#[rustc_legacy_const_generics(2)] +pub unsafe fn __msa_st_b(a: v16i8, mem_addr: *mut u8) -> () { + static_assert_imm_s10!(IMM_S10); + msa_st_b(a, mem_addr, IMM_S10) } /// Vector Store @@ -8729,14 +8204,11 @@ pub unsafe fn __msa_st_b(a: v16i8, mem_addr: *mut u8, imm_s10: i32) -> () { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(st.h, imm_s11 = 0b11111111111))] -#[rustc_args_required_const(2)] -pub unsafe fn __msa_st_h(a: v8i16, mem_addr: *mut u8, imm_s11: i32) -> () { - macro_rules! call { - ($imm_s11:expr) => { - msa_st_h(a, mem_addr, $imm_s11) - }; - } - constify_imm_s11!(imm_s11, call) +#[rustc_legacy_const_generics(2)] +pub unsafe fn __msa_st_h(a: v8i16, mem_addr: *mut u8) -> () { + static_assert_imm_s11!(IMM_S11); + static_assert!(IMM_S11: i32 where IMM_S11 % 2 == 0); + msa_st_h(a, mem_addr, IMM_S11) } /// Vector Store @@ -8748,14 +8220,11 @@ pub unsafe fn __msa_st_h(a: v8i16, mem_addr: *mut u8, imm_s11: i32) -> () { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(st.w, imm_s12 = 0b111111111111))] -#[rustc_args_required_const(2)] -pub unsafe fn __msa_st_w(a: v4i32, mem_addr: *mut u8, imm_s12: i32) -> () { - macro_rules! call { - ($imm_s12:expr) => { - msa_st_w(a, mem_addr, $imm_s12) - }; - } - constify_imm_s12!(imm_s12, call) +#[rustc_legacy_const_generics(2)] +pub unsafe fn __msa_st_w(a: v4i32, mem_addr: *mut u8) -> () { + static_assert_imm_s12!(IMM_S12); + static_assert!(IMM_S12: i32 where IMM_S12 % 4 == 0); + msa_st_w(a, mem_addr, IMM_S12) } /// Vector Store @@ -8767,14 +8236,11 @@ pub unsafe fn __msa_st_w(a: v4i32, mem_addr: *mut u8, imm_s12: i32) -> () { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(st.d, imm_s13 = 0b1111111111111))] -#[rustc_args_required_const(2)] -pub unsafe fn __msa_st_d(a: v2i64, mem_addr: *mut u8, imm_s13: i32) -> () { - macro_rules! call { - ($imm_s13:expr) => { - msa_st_d(a, mem_addr, $imm_s13) - }; - } - constify_imm_s13!(imm_s13, call) +#[rustc_legacy_const_generics(2)] +pub unsafe fn __msa_st_d(a: v2i64, mem_addr: *mut u8) -> () { + static_assert_imm_s13!(IMM_S13); + static_assert!(IMM_S13: i32 where IMM_S13 % 8 == 0); + msa_st_d(a, mem_addr, IMM_S13) } /// Vector Signed Saturated Subtract of Signed Values @@ -9062,14 +8528,10 @@ pub unsafe fn __msa_subv_d(a: v2i64, b: v2i64) -> v2i64 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(subvi.b, imm5 = 0b10111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_subvi_b(a: v16i8, imm5: i32) -> v16i8 { - macro_rules! call { - ($imm5:expr) => { - msa_subvi_b(a, $imm5) - }; - } - constify_imm5!(imm5, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_subvi_b(a: v16i8) -> v16i8 { + static_assert_imm5!(IMM5); + msa_subvi_b(a, IMM5) } /// Immediate Subtract @@ -9081,14 +8543,10 @@ pub unsafe fn __msa_subvi_b(a: v16i8, imm5: i32) -> v16i8 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(subvi.h, imm5 = 0b10111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_subvi_h(a: v8i16, imm5: i32) -> v8i16 { - macro_rules! call { - ($imm5:expr) => { - msa_subvi_h(a, $imm5) - }; - } - constify_imm5!(imm5, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_subvi_h(a: v8i16) -> v8i16 { + static_assert_imm5!(IMM5); + msa_subvi_h(a, IMM5) } /// Immediate Subtract @@ -9100,14 +8558,10 @@ pub unsafe fn __msa_subvi_h(a: v8i16, imm5: i32) -> v8i16 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(subvi.w, imm5 = 0b10111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_subvi_w(a: v4i32, imm5: i32) -> v4i32 { - macro_rules! call { - ($imm5:expr) => { - msa_subvi_w(a, $imm5) - }; - } - constify_imm5!(imm5, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_subvi_w(a: v4i32) -> v4i32 { + static_assert_imm5!(IMM5); + msa_subvi_w(a, IMM5) } /// Immediate Subtract @@ -9119,14 +8573,10 @@ pub unsafe fn __msa_subvi_w(a: v4i32, imm5: i32) -> v4i32 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(subvi.d, imm5 = 0b10111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_subvi_d(a: v2i64, imm5: i32) -> v2i64 { - macro_rules! call { - ($imm5:expr) => { - msa_subvi_d(a, $imm5) - }; - } - constify_imm5!(imm5, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_subvi_d(a: v2i64) -> v2i64 { + static_assert_imm5!(IMM5); + msa_subvi_d(a, IMM5) } /// Vector Data Preserving Shuffle @@ -9221,14 +8671,10 @@ pub unsafe fn __msa_xor_v(a: v16u8, b: v16u8) -> v16u8 { #[inline] #[target_feature(enable = "msa")] #[cfg_attr(test, assert_instr(xori.b, imm8 = 0b11111111))] -#[rustc_args_required_const(1)] -pub unsafe fn __msa_xori_b(a: v16u8, imm8: i32) -> v16u8 { - macro_rules! call { - ($imm8:expr) => { - msa_xori_b(a, $imm8) - }; - } - constify_imm8!(imm8, call) +#[rustc_legacy_const_generics(1)] +pub unsafe fn __msa_xori_b(a: v16u8) -> v16u8 { + static_assert_imm8!(IMM8); + msa_xori_b(a, IMM8) } #[cfg(test)] diff --git a/library/stdarch/crates/core_arch/src/mips/msa/macros.rs b/library/stdarch/crates/core_arch/src/mips/msa/macros.rs index a16a58dc1f..de8905840d 100644 --- a/library/stdarch/crates/core_arch/src/mips/msa/macros.rs +++ b/library/stdarch/crates/core_arch/src/mips/msa/macros.rs @@ -1,4353 +1,31 @@ //! Utility macros. -//immediate value: -4096:4088 -macro_rules! constify_imm_s13 { - ($imm_s13:expr, $expand:ident) => { - #[allow(overflowing_literals)] - match ($imm_s13) & 0b1_1111_1111_1111 { - 0 => $expand!(0), - 8 => $expand!(8), - 16 => $expand!(16), - 24 => $expand!(24), - 32 => $expand!(32), - 40 => $expand!(40), - 48 => $expand!(48), - 56 => $expand!(56), - 64 => $expand!(64), - 72 => $expand!(72), - 80 => $expand!(80), - 88 => $expand!(88), - 96 => $expand!(96), - 104 => $expand!(104), - 112 => $expand!(112), - 120 => $expand!(120), - 128 => $expand!(128), - 136 => $expand!(136), - 144 => $expand!(144), - 152 => $expand!(152), - 160 => $expand!(160), - 168 => $expand!(168), - 176 => $expand!(176), - 184 => $expand!(184), - 192 => $expand!(192), - 200 => $expand!(200), - 208 => $expand!(208), - 216 => $expand!(216), - 224 => $expand!(224), - 232 => $expand!(232), - 240 => $expand!(240), - 248 => $expand!(248), - 256 => $expand!(256), - 264 => $expand!(264), - 272 => $expand!(272), - 280 => $expand!(280), - 288 => $expand!(288), - 296 => $expand!(296), - 304 => $expand!(304), - 312 => $expand!(312), - 320 => $expand!(320), - 328 => $expand!(328), - 336 => $expand!(336), - 344 => $expand!(344), - 352 => $expand!(352), - 360 => $expand!(360), - 368 => $expand!(368), - 376 => $expand!(376), - 384 => $expand!(384), - 392 => $expand!(392), - 400 => $expand!(400), - 408 => $expand!(408), - 416 => $expand!(416), - 424 => $expand!(424), - 432 => $expand!(432), - 440 => $expand!(440), - 448 => $expand!(448), - 456 => $expand!(456), - 464 => $expand!(464), - 472 => $expand!(472), - 480 => $expand!(480), - 488 => $expand!(488), - 496 => $expand!(496), - 504 => $expand!(504), - 512 => $expand!(512), - 520 => $expand!(520), - 528 => $expand!(528), - 536 => $expand!(536), - 544 => $expand!(544), - 552 => $expand!(552), - 560 => $expand!(560), - 568 => $expand!(568), - 576 => $expand!(576), - 584 => $expand!(584), - 592 => $expand!(592), - 600 => $expand!(600), - 608 => $expand!(608), - 616 => $expand!(616), - 624 => $expand!(624), - 632 => $expand!(632), - 640 => $expand!(640), - 648 => $expand!(648), - 656 => $expand!(656), - 664 => $expand!(664), - 672 => $expand!(672), - 680 => $expand!(680), - 688 => $expand!(688), - 696 => $expand!(696), - 704 => $expand!(704), - 712 => $expand!(712), - 720 => $expand!(720), - 728 => $expand!(728), - 736 => $expand!(736), - 744 => $expand!(744), - 752 => $expand!(752), - 760 => $expand!(760), - 768 => $expand!(768), - 776 => $expand!(776), - 784 => $expand!(784), - 792 => $expand!(792), - 800 => $expand!(800), - 808 => $expand!(808), - 816 => $expand!(816), - 824 => $expand!(824), - 832 => $expand!(832), - 840 => $expand!(840), - 848 => $expand!(848), - 856 => $expand!(856), - 864 => $expand!(864), - 872 => $expand!(872), - 880 => $expand!(880), - 888 => $expand!(888), - 896 => $expand!(896), - 904 => $expand!(904), - 912 => $expand!(912), - 920 => $expand!(920), - 928 => $expand!(928), - 936 => $expand!(936), - 944 => $expand!(944), - 952 => $expand!(952), - 960 => $expand!(960), - 968 => $expand!(968), - 976 => $expand!(976), - 984 => $expand!(984), - 992 => $expand!(992), - 1000 => $expand!(1000), - 1008 => $expand!(1008), - 1016 => $expand!(1016), - 1024 => $expand!(1024), - 1032 => $expand!(1032), - 1040 => $expand!(1040), - 1048 => $expand!(1048), - 1056 => $expand!(1056), - 1064 => $expand!(1064), - 1072 => $expand!(1072), - 1080 => $expand!(1080), - 1088 => $expand!(1088), - 1096 => $expand!(1096), - 1104 => $expand!(1104), - 1112 => $expand!(1112), - 1120 => $expand!(1120), - 1128 => $expand!(1128), - 1136 => $expand!(1136), - 1144 => $expand!(1144), - 1152 => $expand!(1152), - 1160 => $expand!(1160), - 1168 => $expand!(1168), - 1176 => $expand!(1176), - 1184 => $expand!(1184), - 1192 => $expand!(1192), - 1200 => $expand!(1200), - 1208 => $expand!(1208), - 1216 => $expand!(1216), - 1224 => $expand!(1224), - 1232 => $expand!(1232), - 1240 => $expand!(1240), - 1248 => $expand!(1248), - 1256 => $expand!(1256), - 1264 => $expand!(1264), - 1272 => $expand!(1272), - 1280 => $expand!(1280), - 1288 => $expand!(1288), - 1296 => $expand!(1296), - 1304 => $expand!(1304), - 1312 => $expand!(1312), - 1320 => $expand!(1320), - 1328 => $expand!(1328), - 1336 => $expand!(1336), - 1344 => $expand!(1344), - 1352 => $expand!(1352), - 1360 => $expand!(1360), - 1368 => $expand!(1368), - 1376 => $expand!(1376), - 1384 => $expand!(1384), - 1392 => $expand!(1392), - 1400 => $expand!(1400), - 1408 => $expand!(1408), - 1416 => $expand!(1416), - 1424 => $expand!(1424), - 1432 => $expand!(1432), - 1440 => $expand!(1440), - 1448 => $expand!(1448), - 1456 => $expand!(1456), - 1464 => $expand!(1464), - 1472 => $expand!(1472), - 1480 => $expand!(1480), - 1488 => $expand!(1488), - 1496 => $expand!(1496), - 1504 => $expand!(1504), - 1512 => $expand!(1512), - 1520 => $expand!(1520), - 1528 => $expand!(1528), - 1536 => $expand!(1536), - 1544 => $expand!(1544), - 1552 => $expand!(1552), - 1560 => $expand!(1560), - 1568 => $expand!(1568), - 1576 => $expand!(1576), - 1584 => $expand!(1584), - 1592 => $expand!(1592), - 1600 => $expand!(1600), - 1608 => $expand!(1608), - 1616 => $expand!(1616), - 1624 => $expand!(1624), - 1632 => $expand!(1632), - 1640 => $expand!(1640), - 1648 => $expand!(1648), - 1656 => $expand!(1656), - 1664 => $expand!(1664), - 1672 => $expand!(1672), - 1680 => $expand!(1680), - 1688 => $expand!(1688), - 1696 => $expand!(1696), - 1704 => $expand!(1704), - 1712 => $expand!(1712), - 1720 => $expand!(1720), - 1728 => $expand!(1728), - 1736 => $expand!(1736), - 1744 => $expand!(1744), - 1752 => $expand!(1752), - 1760 => $expand!(1760), - 1768 => $expand!(1768), - 1776 => $expand!(1776), - 1784 => $expand!(1784), - 1792 => $expand!(1792), - 1800 => $expand!(1800), - 1808 => $expand!(1808), - 1816 => $expand!(1816), - 1824 => $expand!(1824), - 1832 => $expand!(1832), - 1840 => $expand!(1840), - 1848 => $expand!(1848), - 1856 => $expand!(1856), - 1864 => $expand!(1864), - 1872 => $expand!(1872), - 1880 => $expand!(1880), - 1888 => $expand!(1888), - 1896 => $expand!(1896), - 1904 => $expand!(1904), - 1912 => $expand!(1912), - 1920 => $expand!(1920), - 1928 => $expand!(1928), - 1936 => $expand!(1936), - 1944 => $expand!(1944), - 1952 => $expand!(1952), - 1960 => $expand!(1960), - 1968 => $expand!(1968), - 1976 => $expand!(1976), - 1984 => $expand!(1984), - 1992 => $expand!(1992), - 2000 => $expand!(2000), - 2008 => $expand!(2008), - 2016 => $expand!(2016), - 2024 => $expand!(2024), - 2032 => $expand!(2032), - 2040 => $expand!(2040), - 2048 => $expand!(2048), - 2056 => $expand!(2056), - 2064 => $expand!(2064), - 2072 => $expand!(2072), - 2080 => $expand!(2080), - 2088 => $expand!(2088), - 2096 => $expand!(2096), - 2104 => $expand!(2104), - 2112 => $expand!(2112), - 2120 => $expand!(2120), - 2128 => $expand!(2128), - 2136 => $expand!(2136), - 2144 => $expand!(2144), - 2152 => $expand!(2152), - 2160 => $expand!(2160), - 2168 => $expand!(2168), - 2176 => $expand!(2176), - 2184 => $expand!(2184), - 2192 => $expand!(2192), - 2200 => $expand!(2200), - 2208 => $expand!(2208), - 2216 => $expand!(2216), - 2224 => $expand!(2224), - 2232 => $expand!(2232), - 2240 => $expand!(2240), - 2248 => $expand!(2248), - 2256 => $expand!(2256), - 2264 => $expand!(2264), - 2272 => $expand!(2272), - 2280 => $expand!(2280), - 2288 => $expand!(2288), - 2296 => $expand!(2296), - 2304 => $expand!(2304), - 2312 => $expand!(2312), - 2320 => $expand!(2320), - 2328 => $expand!(2328), - 2336 => $expand!(2336), - 2344 => $expand!(2344), - 2352 => $expand!(2352), - 2360 => $expand!(2360), - 2368 => $expand!(2368), - 2376 => $expand!(2376), - 2384 => $expand!(2384), - 2392 => $expand!(2392), - 2400 => $expand!(2400), - 2408 => $expand!(2408), - 2416 => $expand!(2416), - 2424 => $expand!(2424), - 2432 => $expand!(2432), - 2440 => $expand!(2440), - 2448 => $expand!(2448), - 2456 => $expand!(2456), - 2464 => $expand!(2464), - 2472 => $expand!(2472), - 2480 => $expand!(2480), - 2488 => $expand!(2488), - 2496 => $expand!(2496), - 2504 => $expand!(2504), - 2512 => $expand!(2512), - 2520 => $expand!(2520), - 2528 => $expand!(2528), - 2536 => $expand!(2536), - 2544 => $expand!(2544), - 2552 => $expand!(2552), - 2560 => $expand!(2560), - 2568 => $expand!(2568), - 2576 => $expand!(2576), - 2584 => $expand!(2584), - 2592 => $expand!(2592), - 2600 => $expand!(2600), - 2608 => $expand!(2608), - 2616 => $expand!(2616), - 2624 => $expand!(2624), - 2632 => $expand!(2632), - 2640 => $expand!(2640), - 2648 => $expand!(2648), - 2656 => $expand!(2656), - 2664 => $expand!(2664), - 2672 => $expand!(2672), - 2680 => $expand!(2680), - 2688 => $expand!(2688), - 2696 => $expand!(2696), - 2704 => $expand!(2704), - 2712 => $expand!(2712), - 2720 => $expand!(2720), - 2728 => $expand!(2728), - 2736 => $expand!(2736), - 2744 => $expand!(2744), - 2752 => $expand!(2752), - 2760 => $expand!(2760), - 2768 => $expand!(2768), - 2776 => $expand!(2776), - 2784 => $expand!(2784), - 2792 => $expand!(2792), - 2800 => $expand!(2800), - 2808 => $expand!(2808), - 2816 => $expand!(2816), - 2824 => $expand!(2824), - 2832 => $expand!(2832), - 2840 => $expand!(2840), - 2848 => $expand!(2848), - 2856 => $expand!(2856), - 2864 => $expand!(2864), - 2872 => $expand!(2872), - 2880 => $expand!(2880), - 2888 => $expand!(2888), - 2896 => $expand!(2896), - 2904 => $expand!(2904), - 2912 => $expand!(2912), - 2920 => $expand!(2920), - 2928 => $expand!(2928), - 2936 => $expand!(2936), - 2944 => $expand!(2944), - 2952 => $expand!(2952), - 2960 => $expand!(2960), - 2968 => $expand!(2968), - 2976 => $expand!(2976), - 2984 => $expand!(2984), - 2992 => $expand!(2992), - 3000 => $expand!(3000), - 3008 => $expand!(3008), - 3016 => $expand!(3016), - 3024 => $expand!(3024), - 3032 => $expand!(3032), - 3040 => $expand!(3040), - 3048 => $expand!(3048), - 3056 => $expand!(3056), - 3064 => $expand!(3064), - 3072 => $expand!(3072), - 3080 => $expand!(3080), - 3088 => $expand!(3088), - 3096 => $expand!(3096), - 3104 => $expand!(3104), - 3112 => $expand!(3112), - 3120 => $expand!(3120), - 3128 => $expand!(3128), - 3136 => $expand!(3136), - 3144 => $expand!(3144), - 3152 => $expand!(3152), - 3160 => $expand!(3160), - 3168 => $expand!(3168), - 3176 => $expand!(3176), - 3184 => $expand!(3184), - 3192 => $expand!(3192), - 3200 => $expand!(3200), - 3208 => $expand!(3208), - 3216 => $expand!(3216), - 3224 => $expand!(3224), - 3232 => $expand!(3232), - 3240 => $expand!(3240), - 3248 => $expand!(3248), - 3256 => $expand!(3256), - 3264 => $expand!(3264), - 3272 => $expand!(3272), - 3280 => $expand!(3280), - 3288 => $expand!(3288), - 3296 => $expand!(3296), - 3304 => $expand!(3304), - 3312 => $expand!(3312), - 3320 => $expand!(3320), - 3328 => $expand!(3328), - 3336 => $expand!(3336), - 3344 => $expand!(3344), - 3352 => $expand!(3352), - 3360 => $expand!(3360), - 3368 => $expand!(3368), - 3376 => $expand!(3376), - 3384 => $expand!(3384), - 3392 => $expand!(3392), - 3400 => $expand!(3400), - 3408 => $expand!(3408), - 3416 => $expand!(3416), - 3424 => $expand!(3424), - 3432 => $expand!(3432), - 3440 => $expand!(3440), - 3448 => $expand!(3448), - 3456 => $expand!(3456), - 3464 => $expand!(3464), - 3472 => $expand!(3472), - 3480 => $expand!(3480), - 3488 => $expand!(3488), - 3496 => $expand!(3496), - 3504 => $expand!(3504), - 3512 => $expand!(3512), - 3520 => $expand!(3520), - 3528 => $expand!(3528), - 3536 => $expand!(3536), - 3544 => $expand!(3544), - 3552 => $expand!(3552), - 3560 => $expand!(3560), - 3568 => $expand!(3568), - 3576 => $expand!(3576), - 3584 => $expand!(3584), - 3592 => $expand!(3592), - 3600 => $expand!(3600), - 3608 => $expand!(3608), - 3616 => $expand!(3616), - 3624 => $expand!(3624), - 3632 => $expand!(3632), - 3640 => $expand!(3640), - 3648 => $expand!(3648), - 3656 => $expand!(3656), - 3664 => $expand!(3664), - 3672 => $expand!(3672), - 3680 => $expand!(3680), - 3688 => $expand!(3688), - 3696 => $expand!(3696), - 3704 => $expand!(3704), - 3712 => $expand!(3712), - 3720 => $expand!(3720), - 3728 => $expand!(3728), - 3736 => $expand!(3736), - 3744 => $expand!(3744), - 3752 => $expand!(3752), - 3760 => $expand!(3760), - 3768 => $expand!(3768), - 3776 => $expand!(3776), - 3784 => $expand!(3784), - 3792 => $expand!(3792), - 3700 => $expand!(3700), - 3808 => $expand!(3808), - 3816 => $expand!(3816), - 3824 => $expand!(3824), - 3832 => $expand!(3832), - 3840 => $expand!(3840), - 3848 => $expand!(3848), - 3856 => $expand!(3856), - 3864 => $expand!(3864), - 3872 => $expand!(3872), - 3880 => $expand!(3880), - 3888 => $expand!(3888), - 3896 => $expand!(3896), - 3904 => $expand!(3904), - 3912 => $expand!(3912), - 3920 => $expand!(3920), - 3928 => $expand!(3928), - 3936 => $expand!(3936), - 3944 => $expand!(3944), - 3952 => $expand!(3952), - 3960 => $expand!(3960), - 3968 => $expand!(3968), - 3976 => $expand!(3976), - 3984 => $expand!(3984), - 3992 => $expand!(3992), - 4000 => $expand!(4000), - 4008 => $expand!(4008), - 4016 => $expand!(4016), - 4024 => $expand!(4024), - 4032 => $expand!(4032), - 4040 => $expand!(4040), - 4048 => $expand!(4048), - 4056 => $expand!(4056), - 4064 => $expand!(4064), - 4072 => $expand!(4072), - 4080 => $expand!(4080), - 4096 => $expand!(-4096), - 4104 => $expand!(-4088), - 4112 => $expand!(-4080), - 4120 => $expand!(-4072), - 4128 => $expand!(-4064), - 4136 => $expand!(-4056), - 4144 => $expand!(-4048), - 4152 => $expand!(-4040), - 4160 => $expand!(-4032), - 4168 => $expand!(-4024), - 4176 => $expand!(-4016), - 4184 => $expand!(-4008), - 4192 => $expand!(-4000), - 4200 => $expand!(-3992), - 4208 => $expand!(-3984), - 4216 => $expand!(-3976), - 4224 => $expand!(-3968), - 4232 => $expand!(-3960), - 4240 => $expand!(-3952), - 4248 => $expand!(-3944), - 4256 => $expand!(-3936), - 4264 => $expand!(-3928), - 4272 => $expand!(-3920), - 4280 => $expand!(-3912), - 4288 => $expand!(-3904), - 4296 => $expand!(-3896), - 4304 => $expand!(-3888), - 4312 => $expand!(-3880), - 4320 => $expand!(-3872), - 4328 => $expand!(-3864), - 4336 => $expand!(-3856), - 4344 => $expand!(-3848), - 4352 => $expand!(-3840), - 4360 => $expand!(-3832), - 4368 => $expand!(-3824), - 4376 => $expand!(-3816), - 4384 => $expand!(-3808), - 4392 => $expand!(-3800), - 4400 => $expand!(-3792), - 4408 => $expand!(-3784), - 4416 => $expand!(-3776), - 4424 => $expand!(-3768), - 4432 => $expand!(-3760), - 4440 => $expand!(-3752), - 4448 => $expand!(-3744), - 4456 => $expand!(-3736), - 4464 => $expand!(-3728), - 4472 => $expand!(-3720), - 4480 => $expand!(-3712), - 4488 => $expand!(-3704), - 4496 => $expand!(-3696), - 4504 => $expand!(-3688), - 4512 => $expand!(-3680), - 4520 => $expand!(-3672), - 4528 => $expand!(-3664), - 4536 => $expand!(-3656), - 4544 => $expand!(-3648), - 4552 => $expand!(-3640), - 4560 => $expand!(-3632), - 4568 => $expand!(-3624), - 4576 => $expand!(-3616), - 4584 => $expand!(-3608), - 4592 => $expand!(-3600), - 4600 => $expand!(-3592), - 4608 => $expand!(-3584), - 4616 => $expand!(-3576), - 4624 => $expand!(-3568), - 4632 => $expand!(-3560), - 4640 => $expand!(-3552), - 4648 => $expand!(-3544), - 4656 => $expand!(-3536), - 4664 => $expand!(-3528), - 4672 => $expand!(-3520), - 4680 => $expand!(-3512), - 4688 => $expand!(-3504), - 4696 => $expand!(-3496), - 4704 => $expand!(-3488), - 4712 => $expand!(-3480), - 4720 => $expand!(-3472), - 4728 => $expand!(-3464), - 4736 => $expand!(-3456), - 4744 => $expand!(-3448), - 4752 => $expand!(-3440), - 4760 => $expand!(-3432), - 4768 => $expand!(-3424), - 4776 => $expand!(-3416), - 4784 => $expand!(-3408), - 4792 => $expand!(-3400), - 4800 => $expand!(-3392), - 4808 => $expand!(-3384), - 4816 => $expand!(-3376), - 4824 => $expand!(-3368), - 4832 => $expand!(-3360), - 4840 => $expand!(-3352), - 4848 => $expand!(-3344), - 4856 => $expand!(-3336), - 4864 => $expand!(-3328), - 4872 => $expand!(-3320), - 4880 => $expand!(-3312), - 4888 => $expand!(-3304), - 4896 => $expand!(-3296), - 4904 => $expand!(-3288), - 4912 => $expand!(-3280), - 4920 => $expand!(-3272), - 4928 => $expand!(-3264), - 4936 => $expand!(-3256), - 4944 => $expand!(-3248), - 4952 => $expand!(-3240), - 4960 => $expand!(-3232), - 4968 => $expand!(-3224), - 4976 => $expand!(-3216), - 4984 => $expand!(-3208), - 4992 => $expand!(-3200), - 5000 => $expand!(-3192), - 5008 => $expand!(-3184), - 5016 => $expand!(-3176), - 5024 => $expand!(-3168), - 5032 => $expand!(-3160), - 5040 => $expand!(-3152), - 5048 => $expand!(-3144), - 5056 => $expand!(-3136), - 5064 => $expand!(-3128), - 5072 => $expand!(-3120), - 5080 => $expand!(-3112), - 5088 => $expand!(-3104), - 5096 => $expand!(-3096), - 5104 => $expand!(-3088), - 5112 => $expand!(-3080), - 5120 => $expand!(-3072), - 5128 => $expand!(-3064), - 5136 => $expand!(-3056), - 5144 => $expand!(-3048), - 5152 => $expand!(-3040), - 5160 => $expand!(-3032), - 5168 => $expand!(-3024), - 5176 => $expand!(-3016), - 5184 => $expand!(-3008), - 5192 => $expand!(-3000), - 5200 => $expand!(-2992), - 5208 => $expand!(-2984), - 5216 => $expand!(-2976), - 5224 => $expand!(-2968), - 5232 => $expand!(-2960), - 5240 => $expand!(-2952), - 5248 => $expand!(-2944), - 5256 => $expand!(-2936), - 5264 => $expand!(-2928), - 5272 => $expand!(-2920), - 5280 => $expand!(-2912), - 5288 => $expand!(-2904), - 5296 => $expand!(-2896), - 5304 => $expand!(-2888), - 5312 => $expand!(-2880), - 5320 => $expand!(-2872), - 5328 => $expand!(-2864), - 5336 => $expand!(-2856), - 5344 => $expand!(-2848), - 5352 => $expand!(-2840), - 5360 => $expand!(-2832), - 5368 => $expand!(-2824), - 5376 => $expand!(-2816), - 5384 => $expand!(-2808), - 5392 => $expand!(-2800), - 5400 => $expand!(-2792), - 5408 => $expand!(-2784), - 5416 => $expand!(-2776), - 5424 => $expand!(-2768), - 5432 => $expand!(-2760), - 5440 => $expand!(-2752), - 5448 => $expand!(-2744), - 5456 => $expand!(-2736), - 5464 => $expand!(-2728), - 5472 => $expand!(-2720), - 5480 => $expand!(-2712), - 5488 => $expand!(-2704), - 5496 => $expand!(-2696), - 5504 => $expand!(-2688), - 5512 => $expand!(-2680), - 5520 => $expand!(-2672), - 5528 => $expand!(-2664), - 5536 => $expand!(-2656), - 5544 => $expand!(-2648), - 5552 => $expand!(-2640), - 5560 => $expand!(-2632), - 5568 => $expand!(-2624), - 5576 => $expand!(-2616), - 5584 => $expand!(-2608), - 5592 => $expand!(-2600), - 5600 => $expand!(-2592), - 5608 => $expand!(-2584), - 5616 => $expand!(-2576), - 5624 => $expand!(-2568), - 5632 => $expand!(-2560), - 5640 => $expand!(-2552), - 5648 => $expand!(-2544), - 5656 => $expand!(-2536), - 5664 => $expand!(-2528), - 5672 => $expand!(-2520), - 5680 => $expand!(-2512), - 5688 => $expand!(-2504), - 5696 => $expand!(-2496), - 5704 => $expand!(-2488), - 5712 => $expand!(-2480), - 5720 => $expand!(-2472), - 5728 => $expand!(-2464), - 5736 => $expand!(-2456), - 5744 => $expand!(-2448), - 5752 => $expand!(-2440), - 5760 => $expand!(-2432), - 5768 => $expand!(-2424), - 5776 => $expand!(-2416), - 5784 => $expand!(-2408), - 5792 => $expand!(-2400), - 5800 => $expand!(-2392), - 5808 => $expand!(-2384), - 5816 => $expand!(-2376), - 5824 => $expand!(-2368), - 5832 => $expand!(-2360), - 5840 => $expand!(-2352), - 5848 => $expand!(-2344), - 5856 => $expand!(-2336), - 5864 => $expand!(-2328), - 5872 => $expand!(-2320), - 5880 => $expand!(-2312), - 5888 => $expand!(-2304), - 5896 => $expand!(-2296), - 5904 => $expand!(-2288), - 5912 => $expand!(-2280), - 5920 => $expand!(-2272), - 5928 => $expand!(-2264), - 5936 => $expand!(-2256), - 5944 => $expand!(-2248), - 5952 => $expand!(-2240), - 5960 => $expand!(-2232), - 5968 => $expand!(-2224), - 5976 => $expand!(-2216), - 5984 => $expand!(-2208), - 5992 => $expand!(-2200), - 6000 => $expand!(-2192), - 6008 => $expand!(-2184), - 6016 => $expand!(-2176), - 6024 => $expand!(-2168), - 6032 => $expand!(-2160), - 6040 => $expand!(-2152), - 6048 => $expand!(-2144), - 6056 => $expand!(-2136), - 6064 => $expand!(-2128), - 6072 => $expand!(-2120), - 6080 => $expand!(-2112), - 6088 => $expand!(-2104), - 6096 => $expand!(-2096), - 6104 => $expand!(-2088), - 6112 => $expand!(-2080), - 6120 => $expand!(-2072), - 6128 => $expand!(-2064), - 6136 => $expand!(-2056), - 6144 => $expand!(-2048), - 6152 => $expand!(-2040), - 6160 => $expand!(-2032), - 6168 => $expand!(-2024), - 6176 => $expand!(-2016), - 6184 => $expand!(-2008), - 6192 => $expand!(-2000), - 6200 => $expand!(-1992), - 6208 => $expand!(-1984), - 6216 => $expand!(-1976), - 6224 => $expand!(-1968), - 6232 => $expand!(-1960), - 6240 => $expand!(-1952), - 6248 => $expand!(-1944), - 6256 => $expand!(-1936), - 6264 => $expand!(-1928), - 6272 => $expand!(-1920), - 6280 => $expand!(-1912), - 6288 => $expand!(-1904), - 6296 => $expand!(-1896), - 6304 => $expand!(-1888), - 6312 => $expand!(-1880), - 6320 => $expand!(-1872), - 6328 => $expand!(-1864), - 6336 => $expand!(-1856), - 6344 => $expand!(-1848), - 6352 => $expand!(-1840), - 6360 => $expand!(-1832), - 6368 => $expand!(-1824), - 6376 => $expand!(-1816), - 6384 => $expand!(-1808), - 6392 => $expand!(-1800), - 6400 => $expand!(-1792), - 6408 => $expand!(-1784), - 6416 => $expand!(-1776), - 6424 => $expand!(-1768), - 6432 => $expand!(-1760), - 6440 => $expand!(-1752), - 6448 => $expand!(-1744), - 6456 => $expand!(-1736), - 6464 => $expand!(-1728), - 6472 => $expand!(-1720), - 6480 => $expand!(-1712), - 6488 => $expand!(-1704), - 6496 => $expand!(-1696), - 6504 => $expand!(-1688), - 6512 => $expand!(-1680), - 6520 => $expand!(-1672), - 6528 => $expand!(-1664), - 6536 => $expand!(-1656), - 6544 => $expand!(-1648), - 6552 => $expand!(-1640), - 6560 => $expand!(-1632), - 6568 => $expand!(-1624), - 6576 => $expand!(-1616), - 6584 => $expand!(-1608), - 6592 => $expand!(-1600), - 6600 => $expand!(-1592), - 6608 => $expand!(-1584), - 6616 => $expand!(-1576), - 6624 => $expand!(-1568), - 6632 => $expand!(-1560), - 6640 => $expand!(-1552), - 6648 => $expand!(-1544), - 6656 => $expand!(-1536), - 6664 => $expand!(-1528), - 6672 => $expand!(-1520), - 6680 => $expand!(-1512), - 6688 => $expand!(-1504), - 6696 => $expand!(-1496), - 6704 => $expand!(-1488), - 6712 => $expand!(-1480), - 6720 => $expand!(-1472), - 6728 => $expand!(-1464), - 6736 => $expand!(-1456), - 6744 => $expand!(-1448), - 6752 => $expand!(-1440), - 6760 => $expand!(-1432), - 6768 => $expand!(-1424), - 6776 => $expand!(-1416), - 6784 => $expand!(-1408), - 6792 => $expand!(-1400), - 6800 => $expand!(-1392), - 6808 => $expand!(-1384), - 6816 => $expand!(-1376), - 6824 => $expand!(-1368), - 6832 => $expand!(-1360), - 6840 => $expand!(-1352), - 6848 => $expand!(-1344), - 6856 => $expand!(-1336), - 6864 => $expand!(-1328), - 6872 => $expand!(-1320), - 6880 => $expand!(-1312), - 6888 => $expand!(-1304), - 6896 => $expand!(-1296), - 6904 => $expand!(-1288), - 6912 => $expand!(-1280), - 6920 => $expand!(-1272), - 6928 => $expand!(-1264), - 6936 => $expand!(-1256), - 6944 => $expand!(-1248), - 6952 => $expand!(-1240), - 6960 => $expand!(-1232), - 6968 => $expand!(-1224), - 6976 => $expand!(-1216), - 6984 => $expand!(-1208), - 6992 => $expand!(-1200), - 6900 => $expand!(-1192), - 7008 => $expand!(-1184), - 7016 => $expand!(-1176), - 7024 => $expand!(-1168), - 7032 => $expand!(-1160), - 7040 => $expand!(-1152), - 7048 => $expand!(-1144), - 7056 => $expand!(-1136), - 7064 => $expand!(-1128), - 7072 => $expand!(-1120), - 7080 => $expand!(-1112), - 7088 => $expand!(-1104), - 7096 => $expand!(-1096), - 7104 => $expand!(-1088), - 7112 => $expand!(-1080), - 7120 => $expand!(-1072), - 7128 => $expand!(-1064), - 7136 => $expand!(-1056), - 7144 => $expand!(-1048), - 7152 => $expand!(-1040), - 7160 => $expand!(-1032), - 7168 => $expand!(-1024), - 7176 => $expand!(-1016), - 7184 => $expand!(-1008), - 7192 => $expand!(-1000), - 7100 => $expand!(-992), - 7208 => $expand!(-984), - 7216 => $expand!(-976), - 7224 => $expand!(-968), - 7232 => $expand!(-960), - 7240 => $expand!(-952), - 7248 => $expand!(-944), - 7256 => $expand!(-936), - 7264 => $expand!(-928), - 7272 => $expand!(-920), - 7280 => $expand!(-912), - 7288 => $expand!(-904), - 7296 => $expand!(-896), - 7304 => $expand!(-888), - 7312 => $expand!(-880), - 7320 => $expand!(-872), - 7328 => $expand!(-864), - 7336 => $expand!(-856), - 7344 => $expand!(-848), - 7352 => $expand!(-840), - 7360 => $expand!(-832), - 7368 => $expand!(-824), - 7376 => $expand!(-816), - 7384 => $expand!(-808), - 7392 => $expand!(-800), - 7400 => $expand!(-792), - 7408 => $expand!(-784), - 7416 => $expand!(-776), - 7424 => $expand!(-768), - 7432 => $expand!(-760), - 7440 => $expand!(-752), - 7448 => $expand!(-744), - 7456 => $expand!(-736), - 7464 => $expand!(-728), - 7472 => $expand!(-720), - 7480 => $expand!(-712), - 7488 => $expand!(-704), - 7496 => $expand!(-696), - 7504 => $expand!(-688), - 7512 => $expand!(-680), - 7520 => $expand!(-672), - 7528 => $expand!(-664), - 7536 => $expand!(-656), - 7544 => $expand!(-648), - 7552 => $expand!(-640), - 7560 => $expand!(-632), - 7568 => $expand!(-624), - 7576 => $expand!(-616), - 7584 => $expand!(-608), - 7592 => $expand!(-600), - 7600 => $expand!(-592), - 7608 => $expand!(-584), - 7616 => $expand!(-576), - 7624 => $expand!(-568), - 7632 => $expand!(-560), - 7640 => $expand!(-552), - 7648 => $expand!(-544), - 7656 => $expand!(-536), - 7664 => $expand!(-528), - 7672 => $expand!(-520), - 7680 => $expand!(-512), - 7688 => $expand!(-504), - 7696 => $expand!(-496), - 7704 => $expand!(-488), - 7712 => $expand!(-480), - 7720 => $expand!(-472), - 7728 => $expand!(-464), - 7736 => $expand!(-456), - 7744 => $expand!(-448), - 7752 => $expand!(-440), - 7760 => $expand!(-432), - 7768 => $expand!(-424), - 7776 => $expand!(-416), - 7784 => $expand!(-408), - 7792 => $expand!(-400), - 7700 => $expand!(-392), - 7808 => $expand!(-384), - 7816 => $expand!(-376), - 7824 => $expand!(-368), - 7832 => $expand!(-360), - 7840 => $expand!(-352), - 7848 => $expand!(-344), - 7856 => $expand!(-336), - 7864 => $expand!(-328), - 7872 => $expand!(-320), - 7880 => $expand!(-312), - 7888 => $expand!(-304), - 7896 => $expand!(-296), - 7904 => $expand!(-288), - 7912 => $expand!(-280), - 7920 => $expand!(-272), - 7928 => $expand!(-264), - 7936 => $expand!(-256), - 7944 => $expand!(-248), - 7952 => $expand!(-240), - 7960 => $expand!(-232), - 7968 => $expand!(-224), - 7976 => $expand!(-216), - 7984 => $expand!(-208), - 7992 => $expand!(-200), - 8000 => $expand!(-192), - 8008 => $expand!(-184), - 8016 => $expand!(-176), - 8024 => $expand!(-168), - 8032 => $expand!(-160), - 8040 => $expand!(-152), - 8048 => $expand!(-144), - 8056 => $expand!(-136), - 8064 => $expand!(-128), - 8072 => $expand!(-120), - 8080 => $expand!(-112), - 8088 => $expand!(-104), - 8096 => $expand!(-96), - 8104 => $expand!(-88), - 8112 => $expand!(-80), - 8120 => $expand!(-72), - 8128 => $expand!(-64), - 8136 => $expand!(-56), - 8144 => $expand!(-48), - 8152 => $expand!(-40), - 8160 => $expand!(-32), - 8168 => $expand!(-24), - 8176 => $expand!(-16), - 8184 => $expand!(-8), - _ => $expand!(4088), - } +macro_rules! static_assert_imm_s5 { + ($imm:ident) => { + let _ = $crate::core_arch::macros::ValidateConstImm::<$imm, -16, 15>::VALID; }; } -//immediate value: -2048:2044 -macro_rules! constify_imm_s12 { - ($imm_s12:expr, $expand:ident) => { - #[allow(overflowing_literals)] - match ($imm_s12) & 0b1111_1111_1111 { - 0 => $expand!(0), - 4 => $expand!(4), - 8 => $expand!(8), - 12 => $expand!(12), - 16 => $expand!(16), - 20 => $expand!(20), - 24 => $expand!(24), - 28 => $expand!(28), - 32 => $expand!(32), - 36 => $expand!(36), - 40 => $expand!(40), - 44 => $expand!(44), - 48 => $expand!(48), - 52 => $expand!(52), - 56 => $expand!(56), - 60 => $expand!(60), - 64 => $expand!(64), - 68 => $expand!(68), - 72 => $expand!(72), - 76 => $expand!(76), - 80 => $expand!(80), - 84 => $expand!(84), - 88 => $expand!(88), - 92 => $expand!(92), - 96 => $expand!(96), - 100 => $expand!(100), - 104 => $expand!(104), - 108 => $expand!(108), - 112 => $expand!(112), - 116 => $expand!(116), - 120 => $expand!(120), - 124 => $expand!(124), - 128 => $expand!(128), - 132 => $expand!(132), - 136 => $expand!(136), - 140 => $expand!(140), - 144 => $expand!(144), - 148 => $expand!(148), - 152 => $expand!(152), - 156 => $expand!(156), - 160 => $expand!(160), - 164 => $expand!(164), - 168 => $expand!(168), - 172 => $expand!(172), - 176 => $expand!(176), - 180 => $expand!(180), - 184 => $expand!(184), - 188 => $expand!(188), - 192 => $expand!(192), - 196 => $expand!(196), - 200 => $expand!(200), - 204 => $expand!(204), - 208 => $expand!(208), - 212 => $expand!(212), - 216 => $expand!(216), - 220 => $expand!(220), - 224 => $expand!(224), - 228 => $expand!(228), - 232 => $expand!(232), - 236 => $expand!(236), - 240 => $expand!(240), - 244 => $expand!(244), - 248 => $expand!(248), - 252 => $expand!(252), - 256 => $expand!(256), - 260 => $expand!(260), - 264 => $expand!(264), - 268 => $expand!(268), - 272 => $expand!(272), - 276 => $expand!(276), - 280 => $expand!(280), - 284 => $expand!(284), - 288 => $expand!(288), - 292 => $expand!(292), - 296 => $expand!(296), - 300 => $expand!(300), - 304 => $expand!(304), - 308 => $expand!(308), - 312 => $expand!(312), - 316 => $expand!(316), - 320 => $expand!(320), - 324 => $expand!(324), - 328 => $expand!(328), - 332 => $expand!(332), - 336 => $expand!(336), - 340 => $expand!(340), - 344 => $expand!(344), - 348 => $expand!(348), - 352 => $expand!(352), - 356 => $expand!(356), - 360 => $expand!(360), - 364 => $expand!(364), - 368 => $expand!(368), - 372 => $expand!(372), - 376 => $expand!(376), - 380 => $expand!(380), - 384 => $expand!(384), - 388 => $expand!(388), - 392 => $expand!(392), - 396 => $expand!(396), - 400 => $expand!(400), - 404 => $expand!(404), - 408 => $expand!(408), - 412 => $expand!(412), - 416 => $expand!(416), - 420 => $expand!(420), - 424 => $expand!(424), - 428 => $expand!(428), - 432 => $expand!(432), - 436 => $expand!(436), - 440 => $expand!(440), - 444 => $expand!(444), - 448 => $expand!(448), - 452 => $expand!(452), - 456 => $expand!(456), - 460 => $expand!(460), - 464 => $expand!(464), - 468 => $expand!(468), - 472 => $expand!(472), - 476 => $expand!(476), - 480 => $expand!(480), - 484 => $expand!(484), - 488 => $expand!(488), - 492 => $expand!(492), - 496 => $expand!(496), - 500 => $expand!(500), - 504 => $expand!(504), - 508 => $expand!(508), - 512 => $expand!(512), - 516 => $expand!(516), - 520 => $expand!(520), - 524 => $expand!(524), - 528 => $expand!(528), - 532 => $expand!(532), - 536 => $expand!(536), - 540 => $expand!(540), - 544 => $expand!(544), - 548 => $expand!(548), - 552 => $expand!(552), - 556 => $expand!(556), - 560 => $expand!(560), - 564 => $expand!(564), - 568 => $expand!(568), - 572 => $expand!(572), - 576 => $expand!(576), - 580 => $expand!(580), - 584 => $expand!(584), - 588 => $expand!(588), - 592 => $expand!(592), - 596 => $expand!(596), - 600 => $expand!(600), - 604 => $expand!(604), - 608 => $expand!(608), - 612 => $expand!(612), - 616 => $expand!(616), - 620 => $expand!(620), - 624 => $expand!(624), - 628 => $expand!(628), - 632 => $expand!(632), - 636 => $expand!(636), - 640 => $expand!(640), - 644 => $expand!(644), - 648 => $expand!(648), - 652 => $expand!(652), - 656 => $expand!(656), - 660 => $expand!(660), - 664 => $expand!(664), - 668 => $expand!(668), - 672 => $expand!(672), - 676 => $expand!(676), - 680 => $expand!(680), - 684 => $expand!(684), - 688 => $expand!(688), - 692 => $expand!(692), - 696 => $expand!(696), - 700 => $expand!(700), - 704 => $expand!(704), - 708 => $expand!(708), - 712 => $expand!(712), - 716 => $expand!(716), - 720 => $expand!(720), - 724 => $expand!(724), - 728 => $expand!(728), - 732 => $expand!(732), - 736 => $expand!(736), - 740 => $expand!(740), - 744 => $expand!(744), - 748 => $expand!(748), - 752 => $expand!(752), - 756 => $expand!(756), - 760 => $expand!(760), - 764 => $expand!(764), - 768 => $expand!(768), - 772 => $expand!(772), - 776 => $expand!(776), - 780 => $expand!(780), - 784 => $expand!(784), - 788 => $expand!(788), - 792 => $expand!(792), - 800 => $expand!(800), - 804 => $expand!(804), - 808 => $expand!(808), - 812 => $expand!(812), - 816 => $expand!(816), - 820 => $expand!(820), - 824 => $expand!(824), - 828 => $expand!(828), - 832 => $expand!(832), - 836 => $expand!(836), - 840 => $expand!(840), - 844 => $expand!(844), - 848 => $expand!(848), - 852 => $expand!(852), - 856 => $expand!(856), - 860 => $expand!(860), - 864 => $expand!(864), - 868 => $expand!(868), - 872 => $expand!(872), - 876 => $expand!(876), - 880 => $expand!(880), - 884 => $expand!(884), - 888 => $expand!(888), - 892 => $expand!(892), - 896 => $expand!(896), - 900 => $expand!(900), - 904 => $expand!(904), - 908 => $expand!(908), - 912 => $expand!(912), - 916 => $expand!(916), - 920 => $expand!(920), - 924 => $expand!(924), - 928 => $expand!(928), - 932 => $expand!(932), - 936 => $expand!(936), - 940 => $expand!(940), - 944 => $expand!(944), - 948 => $expand!(948), - 952 => $expand!(952), - 956 => $expand!(956), - 960 => $expand!(960), - 964 => $expand!(964), - 968 => $expand!(968), - 972 => $expand!(972), - 976 => $expand!(976), - 980 => $expand!(980), - 984 => $expand!(984), - 988 => $expand!(988), - 992 => $expand!(992), - 996 => $expand!(996), - 1000 => $expand!(1000), - 1004 => $expand!(1004), - 1008 => $expand!(1008), - 1012 => $expand!(1012), - 1016 => $expand!(1016), - 1020 => $expand!(1020), - 1024 => $expand!(1024), - 1028 => $expand!(1028), - 1032 => $expand!(1032), - 1036 => $expand!(1036), - 1040 => $expand!(1040), - 1044 => $expand!(1044), - 1048 => $expand!(1048), - 1052 => $expand!(1052), - 1056 => $expand!(1056), - 1060 => $expand!(1060), - 1064 => $expand!(1064), - 1068 => $expand!(1068), - 1072 => $expand!(1072), - 1076 => $expand!(1076), - 1080 => $expand!(1080), - 1084 => $expand!(1084), - 1088 => $expand!(1088), - 1092 => $expand!(1092), - 1096 => $expand!(1096), - 1100 => $expand!(1100), - 1104 => $expand!(1104), - 1108 => $expand!(1108), - 1112 => $expand!(1112), - 1116 => $expand!(1116), - 1120 => $expand!(1120), - 1124 => $expand!(1124), - 1128 => $expand!(1128), - 1132 => $expand!(1132), - 1136 => $expand!(1136), - 1140 => $expand!(1140), - 1144 => $expand!(1144), - 1148 => $expand!(1148), - 1152 => $expand!(1152), - 1156 => $expand!(1156), - 1160 => $expand!(1160), - 1164 => $expand!(1164), - 1168 => $expand!(1168), - 1172 => $expand!(1172), - 1176 => $expand!(1176), - 1180 => $expand!(1180), - 1184 => $expand!(1184), - 1188 => $expand!(1188), - 1192 => $expand!(1192), - 1196 => $expand!(1196), - 1200 => $expand!(1200), - 1204 => $expand!(1204), - 1208 => $expand!(1208), - 1212 => $expand!(1212), - 1216 => $expand!(1216), - 1220 => $expand!(1220), - 1224 => $expand!(1224), - 1228 => $expand!(1228), - 1232 => $expand!(1232), - 1236 => $expand!(1236), - 1240 => $expand!(1240), - 1244 => $expand!(1244), - 1248 => $expand!(1248), - 1252 => $expand!(1252), - 1256 => $expand!(1256), - 1260 => $expand!(1260), - 1264 => $expand!(1264), - 1268 => $expand!(1268), - 1272 => $expand!(1272), - 1276 => $expand!(1276), - 1280 => $expand!(1280), - 1284 => $expand!(1284), - 1288 => $expand!(1288), - 1292 => $expand!(1292), - 1296 => $expand!(1296), - 1300 => $expand!(1300), - 1304 => $expand!(1304), - 1308 => $expand!(1308), - 1312 => $expand!(1312), - 1316 => $expand!(1316), - 1320 => $expand!(1320), - 1324 => $expand!(1324), - 1328 => $expand!(1328), - 1332 => $expand!(1332), - 1336 => $expand!(1336), - 1340 => $expand!(1340), - 1344 => $expand!(1344), - 1348 => $expand!(1348), - 1352 => $expand!(1352), - 1356 => $expand!(1356), - 1360 => $expand!(1360), - 1364 => $expand!(1364), - 1368 => $expand!(1368), - 1372 => $expand!(1372), - 1376 => $expand!(1376), - 1380 => $expand!(1380), - 1384 => $expand!(1384), - 1388 => $expand!(1388), - 1392 => $expand!(1392), - 1396 => $expand!(1396), - 1400 => $expand!(1400), - 1404 => $expand!(1404), - 1408 => $expand!(1408), - 1412 => $expand!(1412), - 1416 => $expand!(1416), - 1420 => $expand!(1420), - 1424 => $expand!(1424), - 1428 => $expand!(1428), - 1432 => $expand!(1432), - 1436 => $expand!(1436), - 1440 => $expand!(1440), - 1444 => $expand!(1444), - 1448 => $expand!(1448), - 1452 => $expand!(1452), - 1456 => $expand!(1456), - 1460 => $expand!(1460), - 1464 => $expand!(1464), - 1468 => $expand!(1468), - 1472 => $expand!(1472), - 1476 => $expand!(1476), - 1480 => $expand!(1480), - 1484 => $expand!(1484), - 1488 => $expand!(1488), - 1492 => $expand!(1492), - 1496 => $expand!(1496), - 1500 => $expand!(1500), - 1504 => $expand!(1504), - 1508 => $expand!(1508), - 1512 => $expand!(1512), - 1516 => $expand!(1516), - 1520 => $expand!(1520), - 1524 => $expand!(1524), - 1528 => $expand!(1528), - 1532 => $expand!(1532), - 1536 => $expand!(1536), - 1540 => $expand!(1540), - 1544 => $expand!(1544), - 1548 => $expand!(1548), - 1552 => $expand!(1552), - 1556 => $expand!(1556), - 1560 => $expand!(1560), - 1564 => $expand!(1564), - 1568 => $expand!(1568), - 1572 => $expand!(1572), - 1576 => $expand!(1576), - 1580 => $expand!(1580), - 1584 => $expand!(1584), - 1588 => $expand!(1588), - 1592 => $expand!(1592), - 1596 => $expand!(1596), - 1600 => $expand!(1600), - 1604 => $expand!(1604), - 1608 => $expand!(1608), - 1612 => $expand!(1612), - 1616 => $expand!(1616), - 1620 => $expand!(1620), - 1624 => $expand!(1624), - 1628 => $expand!(1628), - 1632 => $expand!(1632), - 1636 => $expand!(1636), - 1640 => $expand!(1640), - 1644 => $expand!(1644), - 1648 => $expand!(1648), - 1652 => $expand!(1652), - 1656 => $expand!(1656), - 1660 => $expand!(1660), - 1664 => $expand!(1664), - 1668 => $expand!(1668), - 1672 => $expand!(1672), - 1676 => $expand!(1676), - 1680 => $expand!(1680), - 1684 => $expand!(1684), - 1688 => $expand!(1688), - 1692 => $expand!(1692), - 1696 => $expand!(1696), - 1700 => $expand!(1700), - 1704 => $expand!(1704), - 1708 => $expand!(1708), - 1712 => $expand!(1712), - 1716 => $expand!(1716), - 1720 => $expand!(1720), - 1724 => $expand!(1724), - 1728 => $expand!(1728), - 1732 => $expand!(1732), - 1736 => $expand!(1736), - 1740 => $expand!(1740), - 1744 => $expand!(1744), - 1748 => $expand!(1748), - 1752 => $expand!(1752), - 1756 => $expand!(1756), - 1760 => $expand!(1760), - 1764 => $expand!(1764), - 1768 => $expand!(1768), - 1772 => $expand!(1772), - 1776 => $expand!(1776), - 1780 => $expand!(1780), - 1784 => $expand!(1784), - 1788 => $expand!(1788), - 1792 => $expand!(1792), - 1796 => $expand!(1796), - 1800 => $expand!(1800), - 1804 => $expand!(1804), - 1808 => $expand!(1808), - 1812 => $expand!(1812), - 1816 => $expand!(1816), - 1820 => $expand!(1820), - 1824 => $expand!(1824), - 1828 => $expand!(1828), - 1832 => $expand!(1832), - 1836 => $expand!(1836), - 1840 => $expand!(1840), - 1844 => $expand!(1844), - 1848 => $expand!(1848), - 1852 => $expand!(1852), - 1856 => $expand!(1856), - 1860 => $expand!(1860), - 1864 => $expand!(1864), - 1868 => $expand!(1868), - 1872 => $expand!(1872), - 1876 => $expand!(1876), - 1880 => $expand!(1880), - 1884 => $expand!(1884), - 1888 => $expand!(1888), - 1892 => $expand!(1892), - 1896 => $expand!(1896), - 1900 => $expand!(1900), - 1904 => $expand!(1904), - 1908 => $expand!(1908), - 1912 => $expand!(1912), - 1916 => $expand!(1916), - 1920 => $expand!(1920), - 1924 => $expand!(1924), - 1928 => $expand!(1928), - 1932 => $expand!(1932), - 1936 => $expand!(1936), - 1940 => $expand!(1940), - 1944 => $expand!(1944), - 1948 => $expand!(1948), - 1952 => $expand!(1952), - 1956 => $expand!(1956), - 1960 => $expand!(1960), - 1964 => $expand!(1964), - 1968 => $expand!(1968), - 1972 => $expand!(1972), - 1976 => $expand!(1976), - 1980 => $expand!(1980), - 1984 => $expand!(1984), - 1988 => $expand!(1988), - 1992 => $expand!(1992), - 1996 => $expand!(1996), - 2000 => $expand!(2000), - 2004 => $expand!(2004), - 2008 => $expand!(2008), - 2012 => $expand!(2012), - 2016 => $expand!(2016), - 2020 => $expand!(2020), - 2024 => $expand!(2024), - 2028 => $expand!(2028), - 2032 => $expand!(2032), - 2036 => $expand!(2036), - 2040 => $expand!(2040), - 2048 => $expand!(-2048), - 2052 => $expand!(-2044), - 2056 => $expand!(-2040), - 2060 => $expand!(-2036), - 2064 => $expand!(-2032), - 2068 => $expand!(-2028), - 2072 => $expand!(-2024), - 2076 => $expand!(-2020), - 2080 => $expand!(-2016), - 2084 => $expand!(-2012), - 2088 => $expand!(-2008), - 2092 => $expand!(-2004), - 2096 => $expand!(-2000), - 2100 => $expand!(-1996), - 2104 => $expand!(-1992), - 2108 => $expand!(-1988), - 2112 => $expand!(-1984), - 2116 => $expand!(-1980), - 2120 => $expand!(-1976), - 2124 => $expand!(-1972), - 2128 => $expand!(-1968), - 2132 => $expand!(-1964), - 2136 => $expand!(-1960), - 2140 => $expand!(-1956), - 2144 => $expand!(-1952), - 2148 => $expand!(-1948), - 2152 => $expand!(-1944), - 2156 => $expand!(-1940), - 2160 => $expand!(-1936), - 2164 => $expand!(-1932), - 2168 => $expand!(-1928), - 2172 => $expand!(-1924), - 2176 => $expand!(-1920), - 2180 => $expand!(-1916), - 2184 => $expand!(-1912), - 2188 => $expand!(-1908), - 2192 => $expand!(-1904), - 2196 => $expand!(-1900), - 2200 => $expand!(-1896), - 2204 => $expand!(-1892), - 2208 => $expand!(-1888), - 2212 => $expand!(-1884), - 2216 => $expand!(-1880), - 2220 => $expand!(-1876), - 2224 => $expand!(-1872), - 2228 => $expand!(-1868), - 2232 => $expand!(-1864), - 2236 => $expand!(-1860), - 2240 => $expand!(-1856), - 2244 => $expand!(-1852), - 2248 => $expand!(-1848), - 2252 => $expand!(-1844), - 2256 => $expand!(-1840), - 2260 => $expand!(-1836), - 2264 => $expand!(-1832), - 2268 => $expand!(-1828), - 2272 => $expand!(-1824), - 2276 => $expand!(-1820), - 2280 => $expand!(-1816), - 2284 => $expand!(-1812), - 2288 => $expand!(-1808), - 2292 => $expand!(-1804), - 2296 => $expand!(-1800), - 2300 => $expand!(-1796), - 2304 => $expand!(-1792), - 2308 => $expand!(-1788), - 2312 => $expand!(-1784), - 2316 => $expand!(-1780), - 2320 => $expand!(-1776), - 2324 => $expand!(-1772), - 2328 => $expand!(-1768), - 2332 => $expand!(-1764), - 2336 => $expand!(-1760), - 2340 => $expand!(-1756), - 2344 => $expand!(-1752), - 2348 => $expand!(-1748), - 2352 => $expand!(-1744), - 2356 => $expand!(-1740), - 2360 => $expand!(-1736), - 2364 => $expand!(-1732), - 2368 => $expand!(-1728), - 2372 => $expand!(-1724), - 2376 => $expand!(-1720), - 2380 => $expand!(-1716), - 2384 => $expand!(-1712), - 2388 => $expand!(-1708), - 2392 => $expand!(-1704), - 2396 => $expand!(-1700), - 2400 => $expand!(-1696), - 2404 => $expand!(-1692), - 2408 => $expand!(-1688), - 2412 => $expand!(-1684), - 2416 => $expand!(-1680), - 2420 => $expand!(-1676), - 2424 => $expand!(-1672), - 2428 => $expand!(-1668), - 2432 => $expand!(-1664), - 2436 => $expand!(-1660), - 2440 => $expand!(-1656), - 2444 => $expand!(-1652), - 2448 => $expand!(-1648), - 2452 => $expand!(-1644), - 2456 => $expand!(-1640), - 2460 => $expand!(-1636), - 2464 => $expand!(-1632), - 2468 => $expand!(-1628), - 2472 => $expand!(-1624), - 2476 => $expand!(-1620), - 2480 => $expand!(-1616), - 2484 => $expand!(-1612), - 2488 => $expand!(-1608), - 2492 => $expand!(-1604), - 2496 => $expand!(-1600), - 2500 => $expand!(-1596), - 2504 => $expand!(-1592), - 2508 => $expand!(-1588), - 2512 => $expand!(-1584), - 2516 => $expand!(-1580), - 2520 => $expand!(-1576), - 2524 => $expand!(-1572), - 2528 => $expand!(-1568), - 2532 => $expand!(-1564), - 2536 => $expand!(-1560), - 2540 => $expand!(-1556), - 2544 => $expand!(-1552), - 2548 => $expand!(-1548), - 2552 => $expand!(-1544), - 2556 => $expand!(-1540), - 2560 => $expand!(-1536), - 2564 => $expand!(-1532), - 2568 => $expand!(-1528), - 2572 => $expand!(-1524), - 2576 => $expand!(-1520), - 2580 => $expand!(-1516), - 2584 => $expand!(-1512), - 2588 => $expand!(-1508), - 2592 => $expand!(-1504), - 2596 => $expand!(-1500), - 2600 => $expand!(-1496), - 2604 => $expand!(-1492), - 2608 => $expand!(-1488), - 2612 => $expand!(-1484), - 2616 => $expand!(-1480), - 2620 => $expand!(-1476), - 2624 => $expand!(-1472), - 2628 => $expand!(-1468), - 2632 => $expand!(-1464), - 2636 => $expand!(-1460), - 2640 => $expand!(-1456), - 2644 => $expand!(-1452), - 2648 => $expand!(-1448), - 2652 => $expand!(-1444), - 2656 => $expand!(-1440), - 2660 => $expand!(-1436), - 2664 => $expand!(-1432), - 2668 => $expand!(-1428), - 2672 => $expand!(-1424), - 2676 => $expand!(-1420), - 2680 => $expand!(-1416), - 2684 => $expand!(-1412), - 2688 => $expand!(-1408), - 2692 => $expand!(-1404), - 2696 => $expand!(-1400), - 2700 => $expand!(-1396), - 2704 => $expand!(-1392), - 2708 => $expand!(-1388), - 2712 => $expand!(-1384), - 2716 => $expand!(-1380), - 2720 => $expand!(-1376), - 2724 => $expand!(-1372), - 2728 => $expand!(-1368), - 2732 => $expand!(-1364), - 2736 => $expand!(-1360), - 2740 => $expand!(-1356), - 2744 => $expand!(-1352), - 2748 => $expand!(-1348), - 2752 => $expand!(-1344), - 2756 => $expand!(-1340), - 2760 => $expand!(-1336), - 2764 => $expand!(-1332), - 2768 => $expand!(-1328), - 2772 => $expand!(-1324), - 2776 => $expand!(-1320), - 2780 => $expand!(-1316), - 2784 => $expand!(-1312), - 2788 => $expand!(-1308), - 2792 => $expand!(-1304), - 2796 => $expand!(-1300), - 2800 => $expand!(-1296), - 2804 => $expand!(-1292), - 2808 => $expand!(-1288), - 2812 => $expand!(-1284), - 2816 => $expand!(-1280), - 2820 => $expand!(-1276), - 2824 => $expand!(-1272), - 2828 => $expand!(-1268), - 2832 => $expand!(-1264), - 2836 => $expand!(-1260), - 2840 => $expand!(-1256), - 2844 => $expand!(-1252), - 2848 => $expand!(-1248), - 2852 => $expand!(-1244), - 2856 => $expand!(-1240), - 2860 => $expand!(-1236), - 2864 => $expand!(-1232), - 2868 => $expand!(-1228), - 2872 => $expand!(-1224), - 2876 => $expand!(-1220), - 2880 => $expand!(-1216), - 2884 => $expand!(-1212), - 2888 => $expand!(-1208), - 2892 => $expand!(-1204), - 2896 => $expand!(-1200), - 2900 => $expand!(-1196), - 2904 => $expand!(-1192), - 2908 => $expand!(-1188), - 2912 => $expand!(-1184), - 2916 => $expand!(-1180), - 2920 => $expand!(-1176), - 2924 => $expand!(-1172), - 2928 => $expand!(-1168), - 2932 => $expand!(-1164), - 2936 => $expand!(-1160), - 2940 => $expand!(-1156), - 2944 => $expand!(-1152), - 2948 => $expand!(-1148), - 2952 => $expand!(-1144), - 2956 => $expand!(-1140), - 2960 => $expand!(-1136), - 2964 => $expand!(-1132), - 2968 => $expand!(-1128), - 2972 => $expand!(-1124), - 2976 => $expand!(-1120), - 2980 => $expand!(-1116), - 2984 => $expand!(-1112), - 2988 => $expand!(-1108), - 2992 => $expand!(-1104), - 2996 => $expand!(-1100), - 3000 => $expand!(-1096), - 3004 => $expand!(-1092), - 3008 => $expand!(-1088), - 3012 => $expand!(-1084), - 3016 => $expand!(-1080), - 3020 => $expand!(-1076), - 3024 => $expand!(-1072), - 3028 => $expand!(-1068), - 3032 => $expand!(-1064), - 3036 => $expand!(-1060), - 3040 => $expand!(-1056), - 3044 => $expand!(-1052), - 3048 => $expand!(-1048), - 3052 => $expand!(-1044), - 3056 => $expand!(-1040), - 3060 => $expand!(-1036), - 3064 => $expand!(-1032), - 3068 => $expand!(-1028), - 3072 => $expand!(-1024), - 3076 => $expand!(-1020), - 3080 => $expand!(-1016), - 3084 => $expand!(-1012), - 3088 => $expand!(-1008), - 3092 => $expand!(-1004), - 3096 => $expand!(-1000), - 3100 => $expand!(-996), - 3104 => $expand!(-992), - 3108 => $expand!(-988), - 3112 => $expand!(-984), - 3116 => $expand!(-980), - 3120 => $expand!(-976), - 3124 => $expand!(-972), - 3128 => $expand!(-968), - 3132 => $expand!(-964), - 3136 => $expand!(-960), - 3140 => $expand!(-956), - 3144 => $expand!(-952), - 3148 => $expand!(-948), - 3152 => $expand!(-944), - 3156 => $expand!(-940), - 3160 => $expand!(-936), - 3164 => $expand!(-932), - 3168 => $expand!(-928), - 3172 => $expand!(-924), - 3176 => $expand!(-920), - 3180 => $expand!(-916), - 3184 => $expand!(-912), - 3188 => $expand!(-908), - 3192 => $expand!(-904), - 3196 => $expand!(-900), - 3200 => $expand!(-896), - 3204 => $expand!(-892), - 3208 => $expand!(-888), - 3212 => $expand!(-884), - 3216 => $expand!(-880), - 3220 => $expand!(-876), - 3224 => $expand!(-872), - 3228 => $expand!(-868), - 3232 => $expand!(-864), - 3236 => $expand!(-860), - 3240 => $expand!(-856), - 3244 => $expand!(-852), - 3248 => $expand!(-848), - 3252 => $expand!(-844), - 3256 => $expand!(-840), - 3260 => $expand!(-836), - 3264 => $expand!(-832), - 3268 => $expand!(-828), - 3272 => $expand!(-824), - 3276 => $expand!(-820), - 3280 => $expand!(-816), - 3284 => $expand!(-812), - 3288 => $expand!(-808), - 3292 => $expand!(-804), - 3296 => $expand!(-800), - 3300 => $expand!(-796), - 3304 => $expand!(-792), - 3308 => $expand!(-788), - 3312 => $expand!(-784), - 3316 => $expand!(-780), - 3320 => $expand!(-776), - 3324 => $expand!(-772), - 3328 => $expand!(-768), - 3332 => $expand!(-764), - 3336 => $expand!(-760), - 3340 => $expand!(-756), - 3344 => $expand!(-752), - 3348 => $expand!(-748), - 3352 => $expand!(-744), - 3356 => $expand!(-740), - 3360 => $expand!(-736), - 3364 => $expand!(-732), - 3368 => $expand!(-728), - 3372 => $expand!(-724), - 3376 => $expand!(-720), - 3380 => $expand!(-716), - 3384 => $expand!(-712), - 3388 => $expand!(-708), - 3392 => $expand!(-704), - 3396 => $expand!(-700), - 3400 => $expand!(-696), - 3404 => $expand!(-692), - 3408 => $expand!(-688), - 3412 => $expand!(-684), - 3416 => $expand!(-680), - 3420 => $expand!(-676), - 3424 => $expand!(-672), - 3428 => $expand!(-668), - 3432 => $expand!(-664), - 3436 => $expand!(-660), - 3440 => $expand!(-656), - 3444 => $expand!(-652), - 3448 => $expand!(-648), - 3452 => $expand!(-644), - 3456 => $expand!(-640), - 3460 => $expand!(-636), - 3464 => $expand!(-632), - 3468 => $expand!(-628), - 3472 => $expand!(-624), - 3476 => $expand!(-620), - 3480 => $expand!(-616), - 3484 => $expand!(-612), - 3488 => $expand!(-608), - 3492 => $expand!(-604), - 3496 => $expand!(-600), - 3500 => $expand!(-596), - 3504 => $expand!(-592), - 3508 => $expand!(-588), - 3512 => $expand!(-584), - 3516 => $expand!(-580), - 3520 => $expand!(-576), - 3524 => $expand!(-572), - 3528 => $expand!(-568), - 3532 => $expand!(-564), - 3536 => $expand!(-560), - 3540 => $expand!(-556), - 3544 => $expand!(-552), - 3548 => $expand!(-548), - 3552 => $expand!(-544), - 3556 => $expand!(-540), - 3560 => $expand!(-536), - 3564 => $expand!(-532), - 3568 => $expand!(-528), - 3572 => $expand!(-524), - 3576 => $expand!(-520), - 3580 => $expand!(-516), - 3584 => $expand!(-512), - 3588 => $expand!(-508), - 3592 => $expand!(-504), - 3596 => $expand!(-500), - 3600 => $expand!(-496), - 3604 => $expand!(-492), - 3608 => $expand!(-488), - 3612 => $expand!(-484), - 3616 => $expand!(-480), - 3620 => $expand!(-476), - 3624 => $expand!(-472), - 3628 => $expand!(-468), - 3632 => $expand!(-464), - 3636 => $expand!(-460), - 3640 => $expand!(-456), - 3644 => $expand!(-452), - 3648 => $expand!(-448), - 3652 => $expand!(-444), - 3656 => $expand!(-440), - 3660 => $expand!(-436), - 3664 => $expand!(-432), - 3668 => $expand!(-428), - 3672 => $expand!(-424), - 3676 => $expand!(-420), - 3680 => $expand!(-416), - 3684 => $expand!(-412), - 3688 => $expand!(-408), - 3692 => $expand!(-404), - 3696 => $expand!(-400), - 3700 => $expand!(-396), - 3704 => $expand!(-392), - 3708 => $expand!(-388), - 3712 => $expand!(-384), - 3716 => $expand!(-380), - 3720 => $expand!(-376), - 3724 => $expand!(-372), - 3728 => $expand!(-368), - 3732 => $expand!(-364), - 3736 => $expand!(-360), - 3740 => $expand!(-356), - 3744 => $expand!(-352), - 3748 => $expand!(-348), - 3752 => $expand!(-344), - 3756 => $expand!(-340), - 3760 => $expand!(-336), - 3764 => $expand!(-332), - 3768 => $expand!(-328), - 3772 => $expand!(-324), - 3776 => $expand!(-320), - 3780 => $expand!(-316), - 3784 => $expand!(-312), - 3788 => $expand!(-308), - 3792 => $expand!(-304), - 3796 => $expand!(-300), - 3800 => $expand!(-296), - 3804 => $expand!(-292), - 3808 => $expand!(-288), - 3812 => $expand!(-284), - 3816 => $expand!(-280), - 3820 => $expand!(-276), - 3824 => $expand!(-272), - 3828 => $expand!(-268), - 3832 => $expand!(-264), - 3836 => $expand!(-260), - 3840 => $expand!(-256), - 3844 => $expand!(-252), - 3848 => $expand!(-248), - 3852 => $expand!(-244), - 3856 => $expand!(-240), - 3860 => $expand!(-236), - 3864 => $expand!(-232), - 3868 => $expand!(-228), - 3872 => $expand!(-224), - 3876 => $expand!(-220), - 3880 => $expand!(-216), - 3884 => $expand!(-212), - 3888 => $expand!(-208), - 3892 => $expand!(-204), - 3896 => $expand!(-200), - 3900 => $expand!(-196), - 3904 => $expand!(-192), - 3908 => $expand!(-188), - 3912 => $expand!(-184), - 3916 => $expand!(-180), - 3920 => $expand!(-176), - 3924 => $expand!(-172), - 3928 => $expand!(-168), - 3932 => $expand!(-164), - 3936 => $expand!(-160), - 3940 => $expand!(-156), - 3944 => $expand!(-152), - 3948 => $expand!(-148), - 3952 => $expand!(-144), - 3956 => $expand!(-140), - 3960 => $expand!(-136), - 3964 => $expand!(-132), - 3968 => $expand!(-128), - 3972 => $expand!(-124), - 3976 => $expand!(-120), - 3980 => $expand!(-116), - 3984 => $expand!(-112), - 3988 => $expand!(-108), - 3992 => $expand!(-104), - 3996 => $expand!(-100), - 4000 => $expand!(-96), - 4004 => $expand!(-92), - 4008 => $expand!(-88), - 4012 => $expand!(-84), - 4016 => $expand!(-80), - 4020 => $expand!(-76), - 4024 => $expand!(-72), - 4028 => $expand!(-68), - 4032 => $expand!(-64), - 4036 => $expand!(-60), - 4040 => $expand!(-56), - 4044 => $expand!(-52), - 4048 => $expand!(-48), - 4052 => $expand!(-44), - 4056 => $expand!(-40), - 4060 => $expand!(-36), - 4064 => $expand!(-32), - 4068 => $expand!(-28), - 4072 => $expand!(-24), - 4076 => $expand!(-20), - 4080 => $expand!(-16), - 4084 => $expand!(-12), - 4088 => $expand!(-8), - 4092 => $expand!(-4), - _ => $expand!(2044), - } +macro_rules! static_assert_imm_s10 { + ($imm:ident) => { + let _ = $crate::core_arch::macros::ValidateConstImm::<$imm, -512, 511>::VALID; }; } -//immediate value: -1024:1022 -macro_rules! constify_imm_s11 { - ($imm_s11:expr, $expand:ident) => { - #[allow(overflowing_literals)] - match ($imm_s11) & 0b111_1111_1111 { - 0 => $expand!(0), - 2 => $expand!(2), - 4 => $expand!(4), - 6 => $expand!(6), - 8 => $expand!(8), - 10 => $expand!(10), - 12 => $expand!(12), - 14 => $expand!(14), - 16 => $expand!(16), - 18 => $expand!(18), - 20 => $expand!(20), - 22 => $expand!(22), - 24 => $expand!(24), - 26 => $expand!(26), - 28 => $expand!(28), - 30 => $expand!(30), - 32 => $expand!(32), - 34 => $expand!(34), - 36 => $expand!(36), - 38 => $expand!(38), - 40 => $expand!(40), - 42 => $expand!(42), - 44 => $expand!(44), - 46 => $expand!(46), - 48 => $expand!(48), - 50 => $expand!(50), - 52 => $expand!(52), - 54 => $expand!(54), - 56 => $expand!(56), - 58 => $expand!(58), - 60 => $expand!(60), - 62 => $expand!(62), - 64 => $expand!(64), - 66 => $expand!(66), - 68 => $expand!(68), - 70 => $expand!(70), - 72 => $expand!(72), - 74 => $expand!(74), - 76 => $expand!(76), - 78 => $expand!(78), - 80 => $expand!(80), - 82 => $expand!(82), - 84 => $expand!(84), - 86 => $expand!(86), - 88 => $expand!(88), - 90 => $expand!(90), - 92 => $expand!(92), - 94 => $expand!(94), - 96 => $expand!(96), - 98 => $expand!(98), - 100 => $expand!(100), - 102 => $expand!(102), - 104 => $expand!(104), - 106 => $expand!(106), - 108 => $expand!(108), - 110 => $expand!(110), - 112 => $expand!(112), - 114 => $expand!(114), - 116 => $expand!(116), - 118 => $expand!(118), - 120 => $expand!(120), - 122 => $expand!(122), - 124 => $expand!(124), - 126 => $expand!(126), - 128 => $expand!(128), - 130 => $expand!(130), - 132 => $expand!(132), - 134 => $expand!(134), - 136 => $expand!(136), - 138 => $expand!(138), - 140 => $expand!(140), - 142 => $expand!(142), - 144 => $expand!(144), - 146 => $expand!(146), - 148 => $expand!(148), - 150 => $expand!(150), - 152 => $expand!(152), - 154 => $expand!(154), - 156 => $expand!(156), - 158 => $expand!(158), - 160 => $expand!(160), - 162 => $expand!(162), - 164 => $expand!(164), - 166 => $expand!(166), - 168 => $expand!(168), - 170 => $expand!(170), - 172 => $expand!(172), - 174 => $expand!(174), - 176 => $expand!(176), - 178 => $expand!(178), - 180 => $expand!(180), - 182 => $expand!(182), - 184 => $expand!(184), - 186 => $expand!(186), - 188 => $expand!(188), - 190 => $expand!(190), - 192 => $expand!(192), - 194 => $expand!(194), - 196 => $expand!(196), - 198 => $expand!(198), - 200 => $expand!(200), - 202 => $expand!(202), - 204 => $expand!(204), - 206 => $expand!(206), - 208 => $expand!(208), - 210 => $expand!(210), - 212 => $expand!(212), - 214 => $expand!(214), - 216 => $expand!(216), - 218 => $expand!(218), - 220 => $expand!(220), - 222 => $expand!(222), - 224 => $expand!(224), - 226 => $expand!(226), - 228 => $expand!(228), - 230 => $expand!(230), - 232 => $expand!(232), - 234 => $expand!(234), - 236 => $expand!(236), - 238 => $expand!(238), - 240 => $expand!(240), - 242 => $expand!(242), - 244 => $expand!(244), - 246 => $expand!(246), - 248 => $expand!(248), - 250 => $expand!(250), - 252 => $expand!(252), - 254 => $expand!(254), - 256 => $expand!(256), - 258 => $expand!(258), - 260 => $expand!(260), - 262 => $expand!(262), - 264 => $expand!(264), - 266 => $expand!(266), - 268 => $expand!(268), - 270 => $expand!(270), - 272 => $expand!(272), - 274 => $expand!(274), - 276 => $expand!(276), - 278 => $expand!(278), - 280 => $expand!(280), - 282 => $expand!(282), - 284 => $expand!(284), - 286 => $expand!(286), - 288 => $expand!(288), - 290 => $expand!(290), - 292 => $expand!(292), - 294 => $expand!(294), - 296 => $expand!(296), - 298 => $expand!(298), - 300 => $expand!(300), - 302 => $expand!(302), - 304 => $expand!(304), - 306 => $expand!(306), - 308 => $expand!(308), - 310 => $expand!(310), - 312 => $expand!(312), - 314 => $expand!(314), - 316 => $expand!(316), - 318 => $expand!(318), - 320 => $expand!(320), - 322 => $expand!(322), - 324 => $expand!(324), - 326 => $expand!(326), - 328 => $expand!(328), - 330 => $expand!(330), - 332 => $expand!(332), - 334 => $expand!(334), - 336 => $expand!(336), - 338 => $expand!(338), - 340 => $expand!(340), - 342 => $expand!(342), - 344 => $expand!(344), - 346 => $expand!(346), - 348 => $expand!(348), - 350 => $expand!(350), - 352 => $expand!(352), - 354 => $expand!(354), - 356 => $expand!(356), - 358 => $expand!(358), - 360 => $expand!(360), - 362 => $expand!(362), - 364 => $expand!(364), - 366 => $expand!(366), - 368 => $expand!(368), - 370 => $expand!(370), - 372 => $expand!(372), - 374 => $expand!(374), - 376 => $expand!(376), - 378 => $expand!(378), - 380 => $expand!(380), - 382 => $expand!(382), - 384 => $expand!(384), - 386 => $expand!(386), - 388 => $expand!(388), - 390 => $expand!(390), - 392 => $expand!(392), - 394 => $expand!(394), - 396 => $expand!(396), - 398 => $expand!(398), - 400 => $expand!(400), - 402 => $expand!(402), - 404 => $expand!(404), - 406 => $expand!(406), - 408 => $expand!(408), - 410 => $expand!(410), - 412 => $expand!(412), - 414 => $expand!(414), - 416 => $expand!(416), - 418 => $expand!(418), - 420 => $expand!(420), - 422 => $expand!(422), - 424 => $expand!(424), - 426 => $expand!(426), - 428 => $expand!(428), - 430 => $expand!(430), - 432 => $expand!(432), - 434 => $expand!(434), - 436 => $expand!(436), - 438 => $expand!(438), - 440 => $expand!(440), - 442 => $expand!(442), - 444 => $expand!(444), - 446 => $expand!(446), - 448 => $expand!(448), - 450 => $expand!(450), - 452 => $expand!(452), - 454 => $expand!(454), - 456 => $expand!(456), - 458 => $expand!(458), - 460 => $expand!(460), - 462 => $expand!(462), - 464 => $expand!(464), - 466 => $expand!(466), - 468 => $expand!(468), - 470 => $expand!(470), - 472 => $expand!(472), - 474 => $expand!(474), - 476 => $expand!(476), - 478 => $expand!(478), - 480 => $expand!(480), - 482 => $expand!(482), - 484 => $expand!(484), - 486 => $expand!(486), - 488 => $expand!(488), - 490 => $expand!(490), - 492 => $expand!(492), - 494 => $expand!(494), - 496 => $expand!(496), - 498 => $expand!(498), - 500 => $expand!(500), - 502 => $expand!(502), - 504 => $expand!(504), - 506 => $expand!(506), - 508 => $expand!(508), - 510 => $expand!(510), - 512 => $expand!(512), - 514 => $expand!(514), - 516 => $expand!(516), - 518 => $expand!(518), - 520 => $expand!(520), - 522 => $expand!(522), - 524 => $expand!(524), - 526 => $expand!(526), - 528 => $expand!(528), - 530 => $expand!(530), - 532 => $expand!(532), - 534 => $expand!(534), - 536 => $expand!(536), - 538 => $expand!(538), - 540 => $expand!(540), - 542 => $expand!(542), - 544 => $expand!(544), - 546 => $expand!(546), - 548 => $expand!(548), - 550 => $expand!(550), - 552 => $expand!(552), - 554 => $expand!(554), - 556 => $expand!(556), - 558 => $expand!(558), - 560 => $expand!(560), - 562 => $expand!(562), - 564 => $expand!(564), - 566 => $expand!(566), - 568 => $expand!(568), - 570 => $expand!(570), - 572 => $expand!(572), - 574 => $expand!(574), - 576 => $expand!(576), - 578 => $expand!(578), - 580 => $expand!(580), - 582 => $expand!(582), - 584 => $expand!(584), - 586 => $expand!(586), - 588 => $expand!(588), - 590 => $expand!(590), - 592 => $expand!(592), - 594 => $expand!(594), - 596 => $expand!(596), - 598 => $expand!(598), - 600 => $expand!(600), - 602 => $expand!(602), - 604 => $expand!(604), - 606 => $expand!(606), - 608 => $expand!(608), - 610 => $expand!(610), - 612 => $expand!(612), - 614 => $expand!(614), - 616 => $expand!(616), - 618 => $expand!(618), - 620 => $expand!(620), - 622 => $expand!(622), - 624 => $expand!(624), - 626 => $expand!(626), - 628 => $expand!(628), - 630 => $expand!(630), - 632 => $expand!(632), - 634 => $expand!(634), - 636 => $expand!(636), - 638 => $expand!(638), - 640 => $expand!(640), - 642 => $expand!(642), - 644 => $expand!(644), - 646 => $expand!(646), - 648 => $expand!(648), - 650 => $expand!(650), - 652 => $expand!(652), - 654 => $expand!(654), - 656 => $expand!(656), - 658 => $expand!(658), - 660 => $expand!(660), - 662 => $expand!(662), - 664 => $expand!(664), - 666 => $expand!(666), - 668 => $expand!(668), - 670 => $expand!(670), - 672 => $expand!(672), - 674 => $expand!(674), - 676 => $expand!(676), - 678 => $expand!(678), - 680 => $expand!(680), - 682 => $expand!(682), - 684 => $expand!(684), - 686 => $expand!(686), - 688 => $expand!(688), - 690 => $expand!(690), - 692 => $expand!(692), - 694 => $expand!(694), - 696 => $expand!(696), - 698 => $expand!(698), - 700 => $expand!(700), - 702 => $expand!(702), - 704 => $expand!(704), - 706 => $expand!(706), - 708 => $expand!(708), - 710 => $expand!(710), - 712 => $expand!(712), - 714 => $expand!(714), - 716 => $expand!(716), - 718 => $expand!(718), - 720 => $expand!(720), - 722 => $expand!(722), - 724 => $expand!(724), - 726 => $expand!(726), - 728 => $expand!(728), - 730 => $expand!(730), - 732 => $expand!(732), - 734 => $expand!(734), - 736 => $expand!(736), - 738 => $expand!(738), - 740 => $expand!(740), - 742 => $expand!(742), - 744 => $expand!(744), - 746 => $expand!(746), - 748 => $expand!(748), - 750 => $expand!(750), - 752 => $expand!(752), - 754 => $expand!(754), - 756 => $expand!(756), - 758 => $expand!(758), - 760 => $expand!(760), - 762 => $expand!(762), - 764 => $expand!(764), - 766 => $expand!(766), - 768 => $expand!(768), - 770 => $expand!(770), - 772 => $expand!(772), - 774 => $expand!(774), - 776 => $expand!(776), - 778 => $expand!(778), - 780 => $expand!(780), - 782 => $expand!(782), - 784 => $expand!(784), - 786 => $expand!(786), - 788 => $expand!(788), - 790 => $expand!(790), - 792 => $expand!(792), - 794 => $expand!(794), - 796 => $expand!(796), - 798 => $expand!(798), - 800 => $expand!(800), - 802 => $expand!(802), - 804 => $expand!(804), - 806 => $expand!(806), - 808 => $expand!(808), - 810 => $expand!(810), - 812 => $expand!(812), - 814 => $expand!(814), - 816 => $expand!(816), - 818 => $expand!(818), - 820 => $expand!(820), - 822 => $expand!(822), - 824 => $expand!(824), - 826 => $expand!(826), - 828 => $expand!(828), - 830 => $expand!(830), - 832 => $expand!(832), - 834 => $expand!(834), - 836 => $expand!(836), - 838 => $expand!(838), - 840 => $expand!(840), - 842 => $expand!(842), - 844 => $expand!(844), - 846 => $expand!(846), - 848 => $expand!(848), - 850 => $expand!(850), - 852 => $expand!(852), - 854 => $expand!(854), - 856 => $expand!(856), - 858 => $expand!(858), - 860 => $expand!(860), - 862 => $expand!(862), - 864 => $expand!(864), - 866 => $expand!(866), - 868 => $expand!(868), - 870 => $expand!(870), - 872 => $expand!(872), - 874 => $expand!(874), - 876 => $expand!(876), - 878 => $expand!(878), - 880 => $expand!(880), - 882 => $expand!(882), - 884 => $expand!(884), - 886 => $expand!(886), - 888 => $expand!(888), - 890 => $expand!(890), - 892 => $expand!(892), - 894 => $expand!(894), - 896 => $expand!(896), - 898 => $expand!(898), - 900 => $expand!(900), - 902 => $expand!(902), - 904 => $expand!(904), - 906 => $expand!(906), - 908 => $expand!(908), - 910 => $expand!(910), - 912 => $expand!(912), - 914 => $expand!(914), - 916 => $expand!(916), - 918 => $expand!(918), - 920 => $expand!(920), - 922 => $expand!(922), - 924 => $expand!(924), - 926 => $expand!(926), - 928 => $expand!(928), - 930 => $expand!(930), - 932 => $expand!(932), - 934 => $expand!(934), - 936 => $expand!(936), - 938 => $expand!(938), - 940 => $expand!(940), - 942 => $expand!(942), - 944 => $expand!(944), - 946 => $expand!(946), - 948 => $expand!(948), - 950 => $expand!(950), - 952 => $expand!(952), - 954 => $expand!(954), - 956 => $expand!(956), - 958 => $expand!(958), - 960 => $expand!(960), - 962 => $expand!(962), - 964 => $expand!(964), - 966 => $expand!(966), - 968 => $expand!(968), - 970 => $expand!(970), - 972 => $expand!(972), - 974 => $expand!(974), - 976 => $expand!(976), - 978 => $expand!(978), - 980 => $expand!(980), - 982 => $expand!(982), - 984 => $expand!(984), - 986 => $expand!(986), - 988 => $expand!(988), - 990 => $expand!(990), - 992 => $expand!(992), - 994 => $expand!(994), - 996 => $expand!(996), - 998 => $expand!(998), - 1000 => $expand!(1000), - 1002 => $expand!(1002), - 1004 => $expand!(1004), - 1006 => $expand!(1006), - 1008 => $expand!(1008), - 1010 => $expand!(1010), - 1012 => $expand!(1012), - 1014 => $expand!(1014), - 1016 => $expand!(1016), - 1018 => $expand!(1018), - 1020 => $expand!(1020), - 1024 => $expand!(-1024), - 1026 => $expand!(-1022), - 1028 => $expand!(-1020), - 1030 => $expand!(-1018), - 1032 => $expand!(-1016), - 1034 => $expand!(-1014), - 1036 => $expand!(-1012), - 1038 => $expand!(-1010), - 1040 => $expand!(-1008), - 1042 => $expand!(-1006), - 1044 => $expand!(-1004), - 1046 => $expand!(-1002), - 1048 => $expand!(-1000), - 1050 => $expand!(-998), - 1052 => $expand!(-996), - 1054 => $expand!(-994), - 1056 => $expand!(-992), - 1058 => $expand!(-990), - 1060 => $expand!(-988), - 1062 => $expand!(-986), - 1064 => $expand!(-984), - 1066 => $expand!(-982), - 1068 => $expand!(-980), - 1070 => $expand!(-978), - 1072 => $expand!(-976), - 1074 => $expand!(-974), - 1076 => $expand!(-972), - 1078 => $expand!(-970), - 1080 => $expand!(-968), - 1082 => $expand!(-966), - 1084 => $expand!(-964), - 1086 => $expand!(-962), - 1088 => $expand!(-960), - 1090 => $expand!(-958), - 1092 => $expand!(-956), - 1094 => $expand!(-954), - 1096 => $expand!(-952), - 1098 => $expand!(-950), - 1100 => $expand!(-948), - 1102 => $expand!(-946), - 1104 => $expand!(-944), - 1106 => $expand!(-942), - 1108 => $expand!(-940), - 1110 => $expand!(-938), - 1112 => $expand!(-936), - 1114 => $expand!(-934), - 1116 => $expand!(-932), - 1118 => $expand!(-930), - 1120 => $expand!(-928), - 1122 => $expand!(-926), - 1124 => $expand!(-924), - 1126 => $expand!(-922), - 1128 => $expand!(-920), - 1130 => $expand!(-918), - 1132 => $expand!(-916), - 1134 => $expand!(-914), - 1136 => $expand!(-912), - 1138 => $expand!(-910), - 1140 => $expand!(-908), - 1142 => $expand!(-906), - 1144 => $expand!(-904), - 1146 => $expand!(-902), - 1148 => $expand!(-900), - 1150 => $expand!(-898), - 1152 => $expand!(-896), - 1154 => $expand!(-894), - 1156 => $expand!(-892), - 1158 => $expand!(-890), - 1160 => $expand!(-888), - 1162 => $expand!(-886), - 1164 => $expand!(-884), - 1166 => $expand!(-882), - 1168 => $expand!(-880), - 1170 => $expand!(-878), - 1172 => $expand!(-876), - 1174 => $expand!(-874), - 1176 => $expand!(-872), - 1178 => $expand!(-870), - 1180 => $expand!(-868), - 1182 => $expand!(-866), - 1184 => $expand!(-864), - 1186 => $expand!(-862), - 1188 => $expand!(-860), - 1190 => $expand!(-858), - 1192 => $expand!(-856), - 1194 => $expand!(-854), - 1196 => $expand!(-852), - 1198 => $expand!(-850), - 1200 => $expand!(-848), - 1202 => $expand!(-846), - 1204 => $expand!(-844), - 1206 => $expand!(-842), - 1208 => $expand!(-840), - 1210 => $expand!(-838), - 1212 => $expand!(-836), - 1214 => $expand!(-834), - 1216 => $expand!(-832), - 1218 => $expand!(-830), - 1220 => $expand!(-828), - 1222 => $expand!(-826), - 1224 => $expand!(-824), - 1226 => $expand!(-822), - 1228 => $expand!(-820), - 1230 => $expand!(-818), - 1232 => $expand!(-816), - 1234 => $expand!(-814), - 1236 => $expand!(-812), - 1238 => $expand!(-810), - 1240 => $expand!(-808), - 1242 => $expand!(-806), - 1244 => $expand!(-804), - 1246 => $expand!(-802), - 1248 => $expand!(-800), - 1250 => $expand!(-798), - 1252 => $expand!(-796), - 1254 => $expand!(-794), - 1256 => $expand!(-792), - 1258 => $expand!(-790), - 1260 => $expand!(-788), - 1262 => $expand!(-786), - 1264 => $expand!(-784), - 1266 => $expand!(-782), - 1268 => $expand!(-780), - 1270 => $expand!(-778), - 1272 => $expand!(-776), - 1274 => $expand!(-774), - 1276 => $expand!(-772), - 1278 => $expand!(-770), - 1280 => $expand!(-768), - 1282 => $expand!(-766), - 1284 => $expand!(-764), - 1286 => $expand!(-762), - 1288 => $expand!(-760), - 1290 => $expand!(-758), - 1292 => $expand!(-756), - 1294 => $expand!(-754), - 1296 => $expand!(-752), - 1298 => $expand!(-750), - 1300 => $expand!(-748), - 1302 => $expand!(-746), - 1304 => $expand!(-744), - 1306 => $expand!(-742), - 1308 => $expand!(-740), - 1310 => $expand!(-738), - 1312 => $expand!(-736), - 1314 => $expand!(-734), - 1316 => $expand!(-732), - 1318 => $expand!(-730), - 1320 => $expand!(-728), - 1322 => $expand!(-726), - 1324 => $expand!(-724), - 1326 => $expand!(-722), - 1328 => $expand!(-720), - 1330 => $expand!(-718), - 1332 => $expand!(-716), - 1334 => $expand!(-714), - 1336 => $expand!(-712), - 1338 => $expand!(-710), - 1340 => $expand!(-708), - 1342 => $expand!(-706), - 1344 => $expand!(-704), - 1346 => $expand!(-702), - 1348 => $expand!(-700), - 1350 => $expand!(-698), - 1352 => $expand!(-696), - 1354 => $expand!(-694), - 1356 => $expand!(-692), - 1358 => $expand!(-690), - 1360 => $expand!(-688), - 1362 => $expand!(-686), - 1364 => $expand!(-684), - 1366 => $expand!(-682), - 1368 => $expand!(-680), - 1370 => $expand!(-678), - 1372 => $expand!(-676), - 1374 => $expand!(-674), - 1376 => $expand!(-672), - 1378 => $expand!(-670), - 1380 => $expand!(-668), - 1382 => $expand!(-666), - 1384 => $expand!(-664), - 1386 => $expand!(-662), - 1388 => $expand!(-660), - 1390 => $expand!(-658), - 1392 => $expand!(-656), - 1394 => $expand!(-654), - 1396 => $expand!(-652), - 1398 => $expand!(-650), - 1400 => $expand!(-648), - 1402 => $expand!(-646), - 1404 => $expand!(-644), - 1406 => $expand!(-642), - 1408 => $expand!(-640), - 1410 => $expand!(-638), - 1412 => $expand!(-636), - 1414 => $expand!(-634), - 1416 => $expand!(-632), - 1418 => $expand!(-630), - 1420 => $expand!(-628), - 1422 => $expand!(-626), - 1424 => $expand!(-624), - 1426 => $expand!(-622), - 1428 => $expand!(-620), - 1430 => $expand!(-618), - 1432 => $expand!(-616), - 1434 => $expand!(-614), - 1436 => $expand!(-612), - 1438 => $expand!(-610), - 1440 => $expand!(-608), - 1442 => $expand!(-606), - 1444 => $expand!(-604), - 1446 => $expand!(-602), - 1448 => $expand!(-600), - 1450 => $expand!(-598), - 1452 => $expand!(-596), - 1454 => $expand!(-594), - 1456 => $expand!(-592), - 1458 => $expand!(-590), - 1460 => $expand!(-588), - 1462 => $expand!(-586), - 1464 => $expand!(-584), - 1466 => $expand!(-582), - 1468 => $expand!(-580), - 1470 => $expand!(-578), - 1472 => $expand!(-576), - 1474 => $expand!(-574), - 1476 => $expand!(-572), - 1478 => $expand!(-570), - 1480 => $expand!(-568), - 1482 => $expand!(-566), - 1484 => $expand!(-564), - 1486 => $expand!(-562), - 1488 => $expand!(-560), - 1490 => $expand!(-558), - 1492 => $expand!(-556), - 1494 => $expand!(-554), - 1496 => $expand!(-552), - 1498 => $expand!(-550), - 1500 => $expand!(-548), - 1502 => $expand!(-546), - 1504 => $expand!(-544), - 1506 => $expand!(-542), - 1508 => $expand!(-540), - 1510 => $expand!(-538), - 1512 => $expand!(-536), - 1514 => $expand!(-534), - 1516 => $expand!(-532), - 1518 => $expand!(-530), - 1520 => $expand!(-528), - 1522 => $expand!(-526), - 1524 => $expand!(-524), - 1526 => $expand!(-522), - 1528 => $expand!(-520), - 1530 => $expand!(-518), - 1532 => $expand!(-516), - 1534 => $expand!(-514), - 1536 => $expand!(-512), - 1538 => $expand!(-510), - 1540 => $expand!(-508), - 1542 => $expand!(-506), - 1544 => $expand!(-504), - 1546 => $expand!(-502), - 1548 => $expand!(-500), - 1550 => $expand!(-498), - 1552 => $expand!(-496), - 1554 => $expand!(-494), - 1556 => $expand!(-492), - 1558 => $expand!(-490), - 1560 => $expand!(-488), - 1562 => $expand!(-486), - 1564 => $expand!(-484), - 1566 => $expand!(-482), - 1568 => $expand!(-480), - 1570 => $expand!(-478), - 1572 => $expand!(-476), - 1574 => $expand!(-474), - 1576 => $expand!(-472), - 1578 => $expand!(-470), - 1580 => $expand!(-468), - 1582 => $expand!(-466), - 1584 => $expand!(-464), - 1586 => $expand!(-462), - 1588 => $expand!(-460), - 1590 => $expand!(-458), - 1592 => $expand!(-456), - 1594 => $expand!(-454), - 1596 => $expand!(-452), - 1598 => $expand!(-450), - 1600 => $expand!(-448), - 1602 => $expand!(-446), - 1604 => $expand!(-444), - 1606 => $expand!(-442), - 1608 => $expand!(-440), - 1610 => $expand!(-438), - 1612 => $expand!(-436), - 1614 => $expand!(-434), - 1616 => $expand!(-432), - 1618 => $expand!(-430), - 1620 => $expand!(-428), - 1622 => $expand!(-426), - 1624 => $expand!(-424), - 1626 => $expand!(-422), - 1628 => $expand!(-420), - 1630 => $expand!(-418), - 1632 => $expand!(-416), - 1634 => $expand!(-414), - 1636 => $expand!(-412), - 1638 => $expand!(-410), - 1640 => $expand!(-408), - 1642 => $expand!(-406), - 1644 => $expand!(-404), - 1646 => $expand!(-402), - 1648 => $expand!(-400), - 1650 => $expand!(-398), - 1652 => $expand!(-396), - 1654 => $expand!(-394), - 1656 => $expand!(-392), - 1658 => $expand!(-390), - 1660 => $expand!(-388), - 1662 => $expand!(-386), - 1664 => $expand!(-384), - 1666 => $expand!(-382), - 1668 => $expand!(-380), - 1670 => $expand!(-378), - 1672 => $expand!(-376), - 1674 => $expand!(-374), - 1676 => $expand!(-372), - 1678 => $expand!(-370), - 1680 => $expand!(-368), - 1682 => $expand!(-366), - 1684 => $expand!(-364), - 1686 => $expand!(-362), - 1688 => $expand!(-360), - 1690 => $expand!(-358), - 1692 => $expand!(-356), - 1694 => $expand!(-354), - 1696 => $expand!(-352), - 1698 => $expand!(-350), - 1700 => $expand!(-348), - 1702 => $expand!(-346), - 1704 => $expand!(-344), - 1706 => $expand!(-342), - 1708 => $expand!(-340), - 1710 => $expand!(-338), - 1712 => $expand!(-336), - 1714 => $expand!(-334), - 1716 => $expand!(-332), - 1718 => $expand!(-330), - 1720 => $expand!(-328), - 1722 => $expand!(-326), - 1724 => $expand!(-324), - 1726 => $expand!(-322), - 1728 => $expand!(-320), - 1730 => $expand!(-318), - 1732 => $expand!(-316), - 1734 => $expand!(-314), - 1736 => $expand!(-312), - 1738 => $expand!(-310), - 1740 => $expand!(-308), - 1742 => $expand!(-306), - 1744 => $expand!(-304), - 1746 => $expand!(-302), - 1748 => $expand!(-300), - 1750 => $expand!(-298), - 1752 => $expand!(-296), - 1754 => $expand!(-294), - 1756 => $expand!(-292), - 1758 => $expand!(-290), - 1760 => $expand!(-288), - 1762 => $expand!(-286), - 1764 => $expand!(-284), - 1766 => $expand!(-282), - 1768 => $expand!(-280), - 1770 => $expand!(-278), - 1772 => $expand!(-276), - 1774 => $expand!(-274), - 1776 => $expand!(-272), - 1778 => $expand!(-270), - 1780 => $expand!(-268), - 1782 => $expand!(-266), - 1784 => $expand!(-264), - 1786 => $expand!(-262), - 1788 => $expand!(-260), - 1790 => $expand!(-258), - 1792 => $expand!(-256), - 1794 => $expand!(-254), - 1796 => $expand!(-252), - 1798 => $expand!(-250), - 1800 => $expand!(-248), - 1802 => $expand!(-246), - 1804 => $expand!(-244), - 1806 => $expand!(-242), - 1808 => $expand!(-240), - 1810 => $expand!(-238), - 1812 => $expand!(-236), - 1814 => $expand!(-234), - 1816 => $expand!(-232), - 1818 => $expand!(-230), - 1820 => $expand!(-228), - 1822 => $expand!(-226), - 1824 => $expand!(-224), - 1826 => $expand!(-222), - 1828 => $expand!(-220), - 1830 => $expand!(-218), - 1832 => $expand!(-216), - 1834 => $expand!(-214), - 1836 => $expand!(-212), - 1838 => $expand!(-210), - 1840 => $expand!(-208), - 1842 => $expand!(-206), - 1844 => $expand!(-204), - 1846 => $expand!(-202), - 1848 => $expand!(-200), - 1850 => $expand!(-198), - 1852 => $expand!(-196), - 1854 => $expand!(-194), - 1856 => $expand!(-192), - 1858 => $expand!(-190), - 1860 => $expand!(-188), - 1862 => $expand!(-186), - 1864 => $expand!(-184), - 1866 => $expand!(-182), - 1868 => $expand!(-180), - 1870 => $expand!(-178), - 1872 => $expand!(-176), - 1874 => $expand!(-174), - 1876 => $expand!(-172), - 1878 => $expand!(-170), - 1880 => $expand!(-168), - 1882 => $expand!(-166), - 1884 => $expand!(-164), - 1886 => $expand!(-162), - 1888 => $expand!(-160), - 1890 => $expand!(-158), - 1892 => $expand!(-156), - 1894 => $expand!(-154), - 1896 => $expand!(-152), - 1898 => $expand!(-150), - 1900 => $expand!(-148), - 1902 => $expand!(-146), - 1904 => $expand!(-144), - 1906 => $expand!(-142), - 1908 => $expand!(-140), - 1910 => $expand!(-138), - 1912 => $expand!(-136), - 1914 => $expand!(-134), - 1916 => $expand!(-132), - 1918 => $expand!(-130), - 1920 => $expand!(-128), - 1922 => $expand!(-126), - 1924 => $expand!(-124), - 1926 => $expand!(-122), - 1928 => $expand!(-120), - 1930 => $expand!(-118), - 1932 => $expand!(-116), - 1934 => $expand!(-114), - 1936 => $expand!(-112), - 1938 => $expand!(-110), - 1940 => $expand!(-108), - 1942 => $expand!(-106), - 1944 => $expand!(-104), - 1946 => $expand!(-102), - 1948 => $expand!(-100), - 1950 => $expand!(-98), - 1952 => $expand!(-96), - 1954 => $expand!(-94), - 1956 => $expand!(-92), - 1958 => $expand!(-90), - 1960 => $expand!(-88), - 1962 => $expand!(-86), - 1964 => $expand!(-84), - 1966 => $expand!(-82), - 1968 => $expand!(-80), - 1970 => $expand!(-78), - 1972 => $expand!(-76), - 1974 => $expand!(-74), - 1976 => $expand!(-72), - 1978 => $expand!(-70), - 1980 => $expand!(-68), - 1982 => $expand!(-66), - 1984 => $expand!(-64), - 1986 => $expand!(-62), - 1988 => $expand!(-60), - 1990 => $expand!(-58), - 1992 => $expand!(-56), - 1994 => $expand!(-54), - 1996 => $expand!(-52), - 1998 => $expand!(-50), - 2000 => $expand!(-48), - 2002 => $expand!(-46), - 2004 => $expand!(-44), - 2006 => $expand!(-42), - 2008 => $expand!(-40), - 2010 => $expand!(-38), - 2012 => $expand!(-36), - 2014 => $expand!(-34), - 2016 => $expand!(-32), - 2018 => $expand!(-30), - 2020 => $expand!(-28), - 2022 => $expand!(-26), - 2024 => $expand!(-24), - 2026 => $expand!(-22), - 2028 => $expand!(-20), - 2030 => $expand!(-18), - 2032 => $expand!(-16), - 2034 => $expand!(-14), - 2036 => $expand!(-12), - 2038 => $expand!(-10), - 2040 => $expand!(-8), - 2042 => $expand!(-6), - 2044 => $expand!(-4), - 2046 => $expand!(-2), - _ => $expand!(1022), - } +macro_rules! static_assert_imm_s11 { + ($imm:ident) => { + let _ = $crate::core_arch::macros::ValidateConstImm::<$imm, -1024, 1023>::VALID; }; } -//immediate value: -512:511 -macro_rules! constify_imm_s10 { - ($imm_s10:expr, $expand:ident) => { - #[allow(overflowing_literals)] - match ($imm_s10) & 0b11_1111_1111 { - 0 => $expand!(0), - 1 => $expand!(1), - 2 => $expand!(2), - 3 => $expand!(3), - 4 => $expand!(4), - 5 => $expand!(5), - 6 => $expand!(6), - 7 => $expand!(7), - 8 => $expand!(8), - 9 => $expand!(9), - 10 => $expand!(10), - 11 => $expand!(11), - 12 => $expand!(12), - 13 => $expand!(13), - 14 => $expand!(14), - 15 => $expand!(15), - 16 => $expand!(16), - 17 => $expand!(17), - 18 => $expand!(18), - 19 => $expand!(19), - 20 => $expand!(20), - 21 => $expand!(21), - 22 => $expand!(22), - 23 => $expand!(23), - 24 => $expand!(24), - 25 => $expand!(25), - 26 => $expand!(26), - 27 => $expand!(27), - 28 => $expand!(28), - 29 => $expand!(29), - 30 => $expand!(30), - 31 => $expand!(31), - 32 => $expand!(32), - 33 => $expand!(33), - 34 => $expand!(34), - 35 => $expand!(35), - 36 => $expand!(36), - 37 => $expand!(37), - 38 => $expand!(38), - 39 => $expand!(39), - 40 => $expand!(40), - 41 => $expand!(41), - 42 => $expand!(42), - 43 => $expand!(43), - 44 => $expand!(44), - 45 => $expand!(45), - 46 => $expand!(46), - 47 => $expand!(47), - 48 => $expand!(48), - 49 => $expand!(49), - 50 => $expand!(50), - 51 => $expand!(51), - 52 => $expand!(52), - 53 => $expand!(53), - 54 => $expand!(54), - 55 => $expand!(55), - 56 => $expand!(56), - 57 => $expand!(57), - 58 => $expand!(58), - 59 => $expand!(59), - 60 => $expand!(60), - 61 => $expand!(61), - 62 => $expand!(62), - 63 => $expand!(63), - 64 => $expand!(64), - 65 => $expand!(65), - 66 => $expand!(66), - 67 => $expand!(67), - 68 => $expand!(68), - 69 => $expand!(69), - 70 => $expand!(70), - 71 => $expand!(71), - 72 => $expand!(72), - 73 => $expand!(73), - 74 => $expand!(74), - 75 => $expand!(75), - 76 => $expand!(76), - 77 => $expand!(77), - 78 => $expand!(78), - 79 => $expand!(79), - 80 => $expand!(80), - 81 => $expand!(81), - 82 => $expand!(82), - 83 => $expand!(83), - 84 => $expand!(84), - 85 => $expand!(85), - 86 => $expand!(86), - 87 => $expand!(87), - 88 => $expand!(88), - 89 => $expand!(89), - 90 => $expand!(90), - 91 => $expand!(91), - 92 => $expand!(92), - 93 => $expand!(93), - 94 => $expand!(94), - 95 => $expand!(95), - 96 => $expand!(96), - 97 => $expand!(97), - 98 => $expand!(98), - 99 => $expand!(99), - 100 => $expand!(100), - 101 => $expand!(101), - 102 => $expand!(102), - 103 => $expand!(103), - 104 => $expand!(104), - 105 => $expand!(105), - 106 => $expand!(106), - 107 => $expand!(107), - 108 => $expand!(108), - 109 => $expand!(109), - 110 => $expand!(110), - 111 => $expand!(111), - 112 => $expand!(112), - 113 => $expand!(113), - 114 => $expand!(114), - 115 => $expand!(115), - 116 => $expand!(116), - 117 => $expand!(117), - 118 => $expand!(118), - 119 => $expand!(119), - 120 => $expand!(120), - 121 => $expand!(121), - 122 => $expand!(122), - 123 => $expand!(123), - 124 => $expand!(124), - 125 => $expand!(125), - 126 => $expand!(126), - 127 => $expand!(127), - 128 => $expand!(128), - 129 => $expand!(129), - 130 => $expand!(130), - 131 => $expand!(131), - 132 => $expand!(132), - 133 => $expand!(133), - 134 => $expand!(134), - 135 => $expand!(135), - 136 => $expand!(136), - 137 => $expand!(137), - 138 => $expand!(138), - 139 => $expand!(139), - 140 => $expand!(140), - 141 => $expand!(141), - 142 => $expand!(142), - 143 => $expand!(143), - 144 => $expand!(144), - 145 => $expand!(145), - 146 => $expand!(146), - 147 => $expand!(147), - 148 => $expand!(148), - 149 => $expand!(149), - 150 => $expand!(150), - 151 => $expand!(151), - 152 => $expand!(152), - 153 => $expand!(153), - 154 => $expand!(154), - 155 => $expand!(155), - 156 => $expand!(156), - 157 => $expand!(157), - 158 => $expand!(158), - 159 => $expand!(159), - 160 => $expand!(160), - 161 => $expand!(161), - 162 => $expand!(162), - 163 => $expand!(163), - 164 => $expand!(164), - 165 => $expand!(165), - 166 => $expand!(166), - 167 => $expand!(167), - 168 => $expand!(168), - 169 => $expand!(169), - 170 => $expand!(170), - 171 => $expand!(171), - 172 => $expand!(172), - 173 => $expand!(173), - 174 => $expand!(174), - 175 => $expand!(175), - 176 => $expand!(176), - 177 => $expand!(177), - 178 => $expand!(178), - 179 => $expand!(179), - 180 => $expand!(180), - 181 => $expand!(181), - 182 => $expand!(182), - 183 => $expand!(183), - 184 => $expand!(184), - 185 => $expand!(185), - 186 => $expand!(186), - 187 => $expand!(187), - 188 => $expand!(188), - 189 => $expand!(189), - 190 => $expand!(190), - 191 => $expand!(191), - 192 => $expand!(192), - 193 => $expand!(193), - 194 => $expand!(194), - 195 => $expand!(195), - 196 => $expand!(196), - 197 => $expand!(197), - 198 => $expand!(198), - 199 => $expand!(199), - 200 => $expand!(200), - 201 => $expand!(201), - 202 => $expand!(202), - 203 => $expand!(203), - 204 => $expand!(204), - 205 => $expand!(205), - 206 => $expand!(206), - 207 => $expand!(207), - 208 => $expand!(208), - 209 => $expand!(209), - 210 => $expand!(210), - 211 => $expand!(211), - 212 => $expand!(212), - 213 => $expand!(213), - 214 => $expand!(214), - 215 => $expand!(215), - 216 => $expand!(216), - 217 => $expand!(217), - 218 => $expand!(218), - 219 => $expand!(219), - 220 => $expand!(220), - 221 => $expand!(221), - 222 => $expand!(222), - 223 => $expand!(223), - 224 => $expand!(224), - 225 => $expand!(225), - 226 => $expand!(226), - 227 => $expand!(227), - 228 => $expand!(228), - 229 => $expand!(229), - 230 => $expand!(230), - 231 => $expand!(231), - 232 => $expand!(232), - 233 => $expand!(233), - 234 => $expand!(234), - 235 => $expand!(235), - 236 => $expand!(236), - 237 => $expand!(237), - 238 => $expand!(238), - 239 => $expand!(239), - 240 => $expand!(240), - 241 => $expand!(241), - 242 => $expand!(242), - 243 => $expand!(243), - 244 => $expand!(244), - 245 => $expand!(245), - 246 => $expand!(246), - 247 => $expand!(247), - 248 => $expand!(248), - 249 => $expand!(249), - 250 => $expand!(250), - 251 => $expand!(251), - 252 => $expand!(252), - 253 => $expand!(253), - 254 => $expand!(254), - 255 => $expand!(255), - 256 => $expand!(256), - 257 => $expand!(257), - 258 => $expand!(258), - 259 => $expand!(259), - 260 => $expand!(260), - 261 => $expand!(261), - 262 => $expand!(262), - 263 => $expand!(263), - 264 => $expand!(264), - 265 => $expand!(265), - 266 => $expand!(266), - 267 => $expand!(267), - 268 => $expand!(268), - 269 => $expand!(269), - 270 => $expand!(270), - 271 => $expand!(271), - 272 => $expand!(272), - 273 => $expand!(273), - 274 => $expand!(274), - 275 => $expand!(275), - 276 => $expand!(276), - 277 => $expand!(277), - 278 => $expand!(278), - 279 => $expand!(279), - 280 => $expand!(280), - 281 => $expand!(281), - 282 => $expand!(282), - 283 => $expand!(283), - 284 => $expand!(284), - 285 => $expand!(285), - 286 => $expand!(286), - 287 => $expand!(287), - 288 => $expand!(288), - 289 => $expand!(289), - 290 => $expand!(290), - 291 => $expand!(291), - 292 => $expand!(292), - 293 => $expand!(293), - 294 => $expand!(294), - 295 => $expand!(295), - 296 => $expand!(296), - 297 => $expand!(297), - 298 => $expand!(298), - 299 => $expand!(299), - 300 => $expand!(300), - 301 => $expand!(301), - 302 => $expand!(302), - 303 => $expand!(303), - 304 => $expand!(304), - 305 => $expand!(305), - 306 => $expand!(306), - 307 => $expand!(307), - 308 => $expand!(308), - 309 => $expand!(309), - 310 => $expand!(310), - 311 => $expand!(311), - 312 => $expand!(312), - 313 => $expand!(313), - 314 => $expand!(314), - 315 => $expand!(315), - 316 => $expand!(316), - 317 => $expand!(317), - 318 => $expand!(318), - 319 => $expand!(319), - 320 => $expand!(320), - 321 => $expand!(321), - 322 => $expand!(322), - 323 => $expand!(323), - 324 => $expand!(324), - 325 => $expand!(325), - 326 => $expand!(326), - 327 => $expand!(327), - 328 => $expand!(328), - 329 => $expand!(329), - 330 => $expand!(330), - 331 => $expand!(331), - 332 => $expand!(332), - 333 => $expand!(333), - 334 => $expand!(334), - 335 => $expand!(335), - 336 => $expand!(336), - 337 => $expand!(337), - 338 => $expand!(338), - 339 => $expand!(339), - 340 => $expand!(340), - 341 => $expand!(341), - 342 => $expand!(342), - 343 => $expand!(343), - 344 => $expand!(344), - 345 => $expand!(345), - 346 => $expand!(346), - 347 => $expand!(347), - 348 => $expand!(348), - 349 => $expand!(349), - 350 => $expand!(350), - 351 => $expand!(351), - 352 => $expand!(352), - 353 => $expand!(353), - 354 => $expand!(354), - 355 => $expand!(355), - 356 => $expand!(356), - 357 => $expand!(357), - 358 => $expand!(358), - 359 => $expand!(359), - 360 => $expand!(360), - 361 => $expand!(361), - 362 => $expand!(362), - 363 => $expand!(363), - 364 => $expand!(364), - 365 => $expand!(365), - 366 => $expand!(366), - 367 => $expand!(367), - 368 => $expand!(368), - 369 => $expand!(369), - 370 => $expand!(370), - 371 => $expand!(371), - 372 => $expand!(372), - 373 => $expand!(373), - 374 => $expand!(374), - 375 => $expand!(375), - 376 => $expand!(376), - 377 => $expand!(377), - 378 => $expand!(378), - 379 => $expand!(379), - 380 => $expand!(380), - 381 => $expand!(381), - 382 => $expand!(382), - 383 => $expand!(383), - 384 => $expand!(384), - 385 => $expand!(385), - 386 => $expand!(386), - 387 => $expand!(387), - 388 => $expand!(388), - 389 => $expand!(389), - 390 => $expand!(390), - 391 => $expand!(391), - 392 => $expand!(392), - 393 => $expand!(393), - 394 => $expand!(394), - 395 => $expand!(395), - 396 => $expand!(396), - 397 => $expand!(397), - 398 => $expand!(398), - 399 => $expand!(399), - 400 => $expand!(400), - 401 => $expand!(401), - 402 => $expand!(402), - 403 => $expand!(403), - 404 => $expand!(404), - 405 => $expand!(405), - 406 => $expand!(406), - 407 => $expand!(407), - 408 => $expand!(408), - 409 => $expand!(409), - 410 => $expand!(410), - 411 => $expand!(411), - 412 => $expand!(412), - 413 => $expand!(413), - 414 => $expand!(414), - 415 => $expand!(415), - 416 => $expand!(416), - 417 => $expand!(417), - 418 => $expand!(418), - 419 => $expand!(419), - 420 => $expand!(420), - 421 => $expand!(421), - 422 => $expand!(422), - 423 => $expand!(423), - 424 => $expand!(424), - 425 => $expand!(425), - 426 => $expand!(426), - 427 => $expand!(427), - 428 => $expand!(428), - 429 => $expand!(429), - 430 => $expand!(430), - 431 => $expand!(431), - 432 => $expand!(432), - 433 => $expand!(433), - 434 => $expand!(434), - 435 => $expand!(435), - 436 => $expand!(436), - 437 => $expand!(437), - 438 => $expand!(438), - 439 => $expand!(439), - 440 => $expand!(440), - 441 => $expand!(441), - 442 => $expand!(442), - 443 => $expand!(443), - 444 => $expand!(444), - 445 => $expand!(445), - 446 => $expand!(446), - 447 => $expand!(447), - 448 => $expand!(448), - 449 => $expand!(449), - 450 => $expand!(450), - 451 => $expand!(451), - 452 => $expand!(452), - 453 => $expand!(453), - 454 => $expand!(454), - 455 => $expand!(455), - 456 => $expand!(456), - 457 => $expand!(457), - 458 => $expand!(458), - 459 => $expand!(459), - 460 => $expand!(460), - 461 => $expand!(461), - 462 => $expand!(462), - 463 => $expand!(463), - 464 => $expand!(464), - 465 => $expand!(465), - 466 => $expand!(466), - 467 => $expand!(467), - 468 => $expand!(468), - 469 => $expand!(469), - 470 => $expand!(470), - 471 => $expand!(471), - 472 => $expand!(472), - 473 => $expand!(473), - 474 => $expand!(474), - 475 => $expand!(475), - 476 => $expand!(476), - 477 => $expand!(477), - 478 => $expand!(478), - 479 => $expand!(479), - 480 => $expand!(480), - 481 => $expand!(481), - 482 => $expand!(482), - 483 => $expand!(483), - 484 => $expand!(484), - 485 => $expand!(485), - 486 => $expand!(486), - 487 => $expand!(487), - 488 => $expand!(488), - 489 => $expand!(489), - 490 => $expand!(490), - 491 => $expand!(491), - 492 => $expand!(492), - 493 => $expand!(493), - 494 => $expand!(494), - 495 => $expand!(495), - 496 => $expand!(496), - 497 => $expand!(497), - 498 => $expand!(498), - 499 => $expand!(499), - 500 => $expand!(500), - 501 => $expand!(501), - 502 => $expand!(502), - 503 => $expand!(503), - 504 => $expand!(504), - 505 => $expand!(505), - 506 => $expand!(506), - 507 => $expand!(507), - 508 => $expand!(508), - 509 => $expand!(509), - 510 => $expand!(510), - 512 => $expand!(-512), - 513 => $expand!(-511), - 514 => $expand!(-510), - 515 => $expand!(-509), - 516 => $expand!(-508), - 517 => $expand!(-507), - 518 => $expand!(-506), - 519 => $expand!(-505), - 520 => $expand!(-504), - 521 => $expand!(-503), - 522 => $expand!(-502), - 523 => $expand!(-501), - 524 => $expand!(-500), - 525 => $expand!(-499), - 526 => $expand!(-498), - 527 => $expand!(-497), - 528 => $expand!(-496), - 529 => $expand!(-495), - 530 => $expand!(-494), - 531 => $expand!(-493), - 532 => $expand!(-492), - 533 => $expand!(-491), - 534 => $expand!(-490), - 535 => $expand!(-489), - 536 => $expand!(-488), - 537 => $expand!(-487), - 538 => $expand!(-486), - 539 => $expand!(-485), - 540 => $expand!(-484), - 541 => $expand!(-483), - 542 => $expand!(-482), - 543 => $expand!(-481), - 544 => $expand!(-480), - 545 => $expand!(-479), - 546 => $expand!(-478), - 547 => $expand!(-477), - 548 => $expand!(-476), - 549 => $expand!(-475), - 550 => $expand!(-474), - 551 => $expand!(-473), - 552 => $expand!(-472), - 553 => $expand!(-471), - 554 => $expand!(-470), - 555 => $expand!(-469), - 556 => $expand!(-468), - 557 => $expand!(-467), - 558 => $expand!(-466), - 559 => $expand!(-465), - 560 => $expand!(-464), - 561 => $expand!(-463), - 562 => $expand!(-462), - 563 => $expand!(-461), - 564 => $expand!(-460), - 565 => $expand!(-459), - 566 => $expand!(-458), - 567 => $expand!(-457), - 568 => $expand!(-456), - 569 => $expand!(-455), - 570 => $expand!(-454), - 571 => $expand!(-453), - 572 => $expand!(-452), - 573 => $expand!(-451), - 574 => $expand!(-450), - 575 => $expand!(-449), - 576 => $expand!(-448), - 577 => $expand!(-447), - 578 => $expand!(-446), - 579 => $expand!(-445), - 580 => $expand!(-444), - 581 => $expand!(-443), - 582 => $expand!(-442), - 583 => $expand!(-441), - 584 => $expand!(-440), - 585 => $expand!(-439), - 586 => $expand!(-438), - 587 => $expand!(-437), - 588 => $expand!(-436), - 589 => $expand!(-435), - 590 => $expand!(-434), - 591 => $expand!(-433), - 592 => $expand!(-432), - 593 => $expand!(-431), - 594 => $expand!(-430), - 595 => $expand!(-429), - 596 => $expand!(-428), - 597 => $expand!(-427), - 598 => $expand!(-426), - 599 => $expand!(-425), - 600 => $expand!(-424), - 601 => $expand!(-423), - 602 => $expand!(-422), - 603 => $expand!(-421), - 604 => $expand!(-420), - 605 => $expand!(-419), - 606 => $expand!(-418), - 607 => $expand!(-417), - 608 => $expand!(-416), - 609 => $expand!(-415), - 610 => $expand!(-414), - 611 => $expand!(-413), - 612 => $expand!(-412), - 613 => $expand!(-411), - 614 => $expand!(-410), - 615 => $expand!(-409), - 616 => $expand!(-408), - 617 => $expand!(-407), - 618 => $expand!(-406), - 619 => $expand!(-405), - 620 => $expand!(-404), - 621 => $expand!(-403), - 622 => $expand!(-402), - 623 => $expand!(-401), - 624 => $expand!(-400), - 625 => $expand!(-399), - 626 => $expand!(-398), - 627 => $expand!(-397), - 628 => $expand!(-396), - 629 => $expand!(-395), - 630 => $expand!(-394), - 631 => $expand!(-393), - 632 => $expand!(-392), - 633 => $expand!(-391), - 634 => $expand!(-390), - 635 => $expand!(-389), - 636 => $expand!(-388), - 637 => $expand!(-387), - 638 => $expand!(-386), - 639 => $expand!(-385), - 640 => $expand!(-384), - 641 => $expand!(-383), - 642 => $expand!(-382), - 643 => $expand!(-381), - 644 => $expand!(-380), - 645 => $expand!(-379), - 646 => $expand!(-378), - 647 => $expand!(-377), - 648 => $expand!(-376), - 649 => $expand!(-375), - 650 => $expand!(-374), - 651 => $expand!(-373), - 652 => $expand!(-372), - 653 => $expand!(-371), - 654 => $expand!(-370), - 655 => $expand!(-369), - 656 => $expand!(-368), - 657 => $expand!(-367), - 658 => $expand!(-366), - 659 => $expand!(-365), - 660 => $expand!(-364), - 661 => $expand!(-363), - 662 => $expand!(-362), - 663 => $expand!(-361), - 664 => $expand!(-360), - 665 => $expand!(-359), - 666 => $expand!(-358), - 667 => $expand!(-357), - 668 => $expand!(-356), - 669 => $expand!(-355), - 670 => $expand!(-354), - 671 => $expand!(-353), - 672 => $expand!(-352), - 673 => $expand!(-351), - 674 => $expand!(-350), - 675 => $expand!(-349), - 676 => $expand!(-348), - 677 => $expand!(-347), - 678 => $expand!(-346), - 679 => $expand!(-345), - 680 => $expand!(-344), - 681 => $expand!(-343), - 682 => $expand!(-342), - 683 => $expand!(-341), - 684 => $expand!(-340), - 685 => $expand!(-339), - 686 => $expand!(-338), - 687 => $expand!(-337), - 688 => $expand!(-336), - 689 => $expand!(-335), - 690 => $expand!(-334), - 691 => $expand!(-333), - 692 => $expand!(-332), - 693 => $expand!(-331), - 694 => $expand!(-330), - 695 => $expand!(-329), - 696 => $expand!(-328), - 697 => $expand!(-327), - 698 => $expand!(-326), - 699 => $expand!(-325), - 700 => $expand!(-324), - 701 => $expand!(-323), - 702 => $expand!(-322), - 703 => $expand!(-321), - 704 => $expand!(-320), - 705 => $expand!(-319), - 706 => $expand!(-318), - 707 => $expand!(-317), - 708 => $expand!(-316), - 709 => $expand!(-315), - 710 => $expand!(-314), - 711 => $expand!(-313), - 712 => $expand!(-312), - 713 => $expand!(-311), - 714 => $expand!(-310), - 715 => $expand!(-309), - 716 => $expand!(-308), - 717 => $expand!(-307), - 718 => $expand!(-306), - 719 => $expand!(-305), - 720 => $expand!(-304), - 721 => $expand!(-303), - 722 => $expand!(-302), - 723 => $expand!(-301), - 724 => $expand!(-300), - 725 => $expand!(-299), - 726 => $expand!(-298), - 727 => $expand!(-297), - 728 => $expand!(-296), - 729 => $expand!(-295), - 730 => $expand!(-294), - 731 => $expand!(-293), - 732 => $expand!(-292), - 733 => $expand!(-291), - 734 => $expand!(-290), - 735 => $expand!(-289), - 736 => $expand!(-288), - 737 => $expand!(-287), - 738 => $expand!(-286), - 739 => $expand!(-285), - 740 => $expand!(-284), - 741 => $expand!(-283), - 742 => $expand!(-282), - 743 => $expand!(-281), - 744 => $expand!(-280), - 745 => $expand!(-279), - 746 => $expand!(-278), - 747 => $expand!(-277), - 748 => $expand!(-276), - 749 => $expand!(-275), - 750 => $expand!(-274), - 751 => $expand!(-273), - 752 => $expand!(-272), - 753 => $expand!(-271), - 754 => $expand!(-270), - 755 => $expand!(-269), - 756 => $expand!(-268), - 757 => $expand!(-267), - 758 => $expand!(-266), - 759 => $expand!(-265), - 760 => $expand!(-264), - 761 => $expand!(-263), - 762 => $expand!(-262), - 763 => $expand!(-261), - 764 => $expand!(-260), - 765 => $expand!(-259), - 766 => $expand!(-258), - 767 => $expand!(-257), - 768 => $expand!(-256), - 769 => $expand!(-255), - 770 => $expand!(-254), - 771 => $expand!(-253), - 772 => $expand!(-252), - 773 => $expand!(-251), - 774 => $expand!(-250), - 775 => $expand!(-249), - 776 => $expand!(-248), - 777 => $expand!(-247), - 778 => $expand!(-246), - 779 => $expand!(-245), - 780 => $expand!(-244), - 781 => $expand!(-243), - 782 => $expand!(-242), - 783 => $expand!(-241), - 784 => $expand!(-240), - 785 => $expand!(-239), - 786 => $expand!(-238), - 787 => $expand!(-237), - 788 => $expand!(-236), - 789 => $expand!(-235), - 790 => $expand!(-234), - 791 => $expand!(-233), - 792 => $expand!(-232), - 793 => $expand!(-231), - 794 => $expand!(-230), - 795 => $expand!(-229), - 796 => $expand!(-228), - 797 => $expand!(-227), - 798 => $expand!(-226), - 799 => $expand!(-225), - 800 => $expand!(-224), - 801 => $expand!(-223), - 802 => $expand!(-222), - 803 => $expand!(-221), - 804 => $expand!(-220), - 805 => $expand!(-219), - 806 => $expand!(-218), - 807 => $expand!(-217), - 808 => $expand!(-216), - 809 => $expand!(-215), - 810 => $expand!(-214), - 811 => $expand!(-213), - 812 => $expand!(-212), - 813 => $expand!(-211), - 814 => $expand!(-210), - 815 => $expand!(-209), - 816 => $expand!(-208), - 817 => $expand!(-207), - 818 => $expand!(-206), - 819 => $expand!(-205), - 820 => $expand!(-204), - 821 => $expand!(-203), - 822 => $expand!(-202), - 823 => $expand!(-201), - 824 => $expand!(-200), - 825 => $expand!(-199), - 826 => $expand!(-198), - 827 => $expand!(-197), - 828 => $expand!(-196), - 829 => $expand!(-195), - 830 => $expand!(-194), - 831 => $expand!(-193), - 832 => $expand!(-192), - 833 => $expand!(-191), - 834 => $expand!(-190), - 835 => $expand!(-189), - 836 => $expand!(-188), - 837 => $expand!(-187), - 838 => $expand!(-186), - 839 => $expand!(-185), - 840 => $expand!(-184), - 841 => $expand!(-183), - 842 => $expand!(-182), - 843 => $expand!(-181), - 844 => $expand!(-180), - 845 => $expand!(-179), - 846 => $expand!(-178), - 847 => $expand!(-177), - 848 => $expand!(-176), - 849 => $expand!(-175), - 850 => $expand!(-174), - 851 => $expand!(-173), - 852 => $expand!(-172), - 853 => $expand!(-171), - 854 => $expand!(-170), - 855 => $expand!(-169), - 856 => $expand!(-168), - 857 => $expand!(-167), - 858 => $expand!(-166), - 859 => $expand!(-165), - 860 => $expand!(-164), - 861 => $expand!(-163), - 862 => $expand!(-162), - 863 => $expand!(-161), - 864 => $expand!(-160), - 865 => $expand!(-159), - 866 => $expand!(-158), - 867 => $expand!(-157), - 868 => $expand!(-156), - 869 => $expand!(-155), - 870 => $expand!(-154), - 871 => $expand!(-153), - 872 => $expand!(-152), - 873 => $expand!(-151), - 874 => $expand!(-150), - 875 => $expand!(-149), - 876 => $expand!(-148), - 877 => $expand!(-147), - 878 => $expand!(-146), - 879 => $expand!(-145), - 880 => $expand!(-144), - 881 => $expand!(-143), - 882 => $expand!(-142), - 883 => $expand!(-141), - 884 => $expand!(-140), - 885 => $expand!(-139), - 886 => $expand!(-138), - 887 => $expand!(-137), - 888 => $expand!(-136), - 889 => $expand!(-135), - 890 => $expand!(-134), - 891 => $expand!(-133), - 892 => $expand!(-132), - 893 => $expand!(-131), - 894 => $expand!(-130), - 895 => $expand!(-129), - 896 => $expand!(-128), - 897 => $expand!(-127), - 898 => $expand!(-126), - 899 => $expand!(-125), - 900 => $expand!(-124), - 901 => $expand!(-123), - 902 => $expand!(-122), - 903 => $expand!(-121), - 904 => $expand!(-120), - 905 => $expand!(-119), - 906 => $expand!(-118), - 907 => $expand!(-117), - 908 => $expand!(-116), - 909 => $expand!(-115), - 910 => $expand!(-114), - 911 => $expand!(-113), - 912 => $expand!(-112), - 913 => $expand!(-111), - 914 => $expand!(-110), - 915 => $expand!(-109), - 916 => $expand!(-108), - 917 => $expand!(-107), - 918 => $expand!(-106), - 919 => $expand!(-105), - 920 => $expand!(-104), - 921 => $expand!(-103), - 922 => $expand!(-102), - 923 => $expand!(-101), - 924 => $expand!(-100), - 925 => $expand!(-99), - 926 => $expand!(-98), - 927 => $expand!(-97), - 928 => $expand!(-96), - 929 => $expand!(-95), - 930 => $expand!(-94), - 931 => $expand!(-93), - 932 => $expand!(-92), - 933 => $expand!(-91), - 934 => $expand!(-90), - 935 => $expand!(-89), - 936 => $expand!(-88), - 937 => $expand!(-87), - 938 => $expand!(-86), - 939 => $expand!(-85), - 940 => $expand!(-84), - 941 => $expand!(-83), - 942 => $expand!(-82), - 943 => $expand!(-81), - 944 => $expand!(-80), - 945 => $expand!(-79), - 946 => $expand!(-78), - 947 => $expand!(-77), - 948 => $expand!(-76), - 949 => $expand!(-75), - 950 => $expand!(-74), - 951 => $expand!(-73), - 952 => $expand!(-72), - 953 => $expand!(-71), - 954 => $expand!(-70), - 955 => $expand!(-69), - 956 => $expand!(-68), - 957 => $expand!(-67), - 958 => $expand!(-66), - 959 => $expand!(-65), - 960 => $expand!(-64), - 961 => $expand!(-63), - 962 => $expand!(-62), - 963 => $expand!(-61), - 964 => $expand!(-60), - 965 => $expand!(-59), - 966 => $expand!(-58), - 967 => $expand!(-57), - 968 => $expand!(-56), - 969 => $expand!(-55), - 970 => $expand!(-54), - 971 => $expand!(-53), - 972 => $expand!(-52), - 973 => $expand!(-51), - 974 => $expand!(-50), - 975 => $expand!(-49), - 976 => $expand!(-48), - 977 => $expand!(-47), - 978 => $expand!(-46), - 979 => $expand!(-45), - 980 => $expand!(-44), - 981 => $expand!(-43), - 982 => $expand!(-42), - 983 => $expand!(-41), - 984 => $expand!(-40), - 985 => $expand!(-39), - 986 => $expand!(-38), - 987 => $expand!(-37), - 988 => $expand!(-36), - 989 => $expand!(-35), - 990 => $expand!(-34), - 991 => $expand!(-33), - 992 => $expand!(-32), - 993 => $expand!(-31), - 994 => $expand!(-30), - 995 => $expand!(-29), - 996 => $expand!(-28), - 997 => $expand!(-27), - 998 => $expand!(-26), - 999 => $expand!(-25), - 1000 => $expand!(-24), - 1001 => $expand!(-23), - 1002 => $expand!(-22), - 1003 => $expand!(-21), - 1004 => $expand!(-20), - 1005 => $expand!(-19), - 1006 => $expand!(-18), - 1007 => $expand!(-17), - 1008 => $expand!(-16), - 1009 => $expand!(-15), - 1010 => $expand!(-14), - 1011 => $expand!(-13), - 1012 => $expand!(-12), - 1013 => $expand!(-11), - 1014 => $expand!(-10), - 1015 => $expand!(-9), - 1016 => $expand!(-8), - 1017 => $expand!(-7), - 1018 => $expand!(-6), - 1019 => $expand!(-5), - 1020 => $expand!(-4), - 1021 => $expand!(-3), - 1022 => $expand!(-2), - 1023 => $expand!(-1), - _ => $expand!(511), - } +macro_rules! static_assert_imm_s12 { + ($imm:ident) => { + let _ = $crate::core_arch::macros::ValidateConstImm::<$imm, -2048, 2047>::VALID; }; } -//immediate value: 0:63 -macro_rules! constify_imm6 { - ($imm8:expr, $expand:ident) => { - #[allow(overflowing_literals)] - match ($imm8) & 0b11_1111 { - 0 => $expand!(0), - 1 => $expand!(1), - 2 => $expand!(2), - 3 => $expand!(3), - 4 => $expand!(4), - 5 => $expand!(5), - 6 => $expand!(6), - 7 => $expand!(7), - 8 => $expand!(8), - 9 => $expand!(9), - 10 => $expand!(10), - 11 => $expand!(11), - 12 => $expand!(12), - 13 => $expand!(13), - 14 => $expand!(14), - 15 => $expand!(15), - 16 => $expand!(16), - 17 => $expand!(17), - 18 => $expand!(18), - 19 => $expand!(19), - 20 => $expand!(20), - 21 => $expand!(21), - 22 => $expand!(22), - 23 => $expand!(23), - 24 => $expand!(24), - 25 => $expand!(25), - 26 => $expand!(26), - 27 => $expand!(27), - 28 => $expand!(28), - 29 => $expand!(29), - 30 => $expand!(30), - 31 => $expand!(31), - 32 => $expand!(32), - 33 => $expand!(33), - 34 => $expand!(34), - 35 => $expand!(35), - 36 => $expand!(36), - 37 => $expand!(37), - 38 => $expand!(38), - 39 => $expand!(39), - 40 => $expand!(40), - 41 => $expand!(41), - 42 => $expand!(42), - 43 => $expand!(43), - 44 => $expand!(44), - 45 => $expand!(45), - 46 => $expand!(46), - 47 => $expand!(47), - 48 => $expand!(48), - 49 => $expand!(49), - 50 => $expand!(50), - 51 => $expand!(51), - 52 => $expand!(52), - 53 => $expand!(53), - 54 => $expand!(54), - 55 => $expand!(55), - 56 => $expand!(56), - 57 => $expand!(57), - 58 => $expand!(58), - 59 => $expand!(59), - 60 => $expand!(60), - 61 => $expand!(61), - 62 => $expand!(62), - _ => $expand!(63), - } - }; -} - -//immediate value: 0:31 -macro_rules! constify_imm5 { - ($imm8:expr, $expand:ident) => { - #[allow(overflowing_literals)] - match ($imm8) & 0b1_1111 { - 0 => $expand!(0), - 1 => $expand!(1), - 2 => $expand!(2), - 3 => $expand!(3), - 4 => $expand!(4), - 5 => $expand!(5), - 6 => $expand!(6), - 7 => $expand!(7), - 8 => $expand!(8), - 9 => $expand!(9), - 10 => $expand!(10), - 11 => $expand!(11), - 12 => $expand!(12), - 13 => $expand!(13), - 14 => $expand!(14), - 15 => $expand!(15), - 16 => $expand!(16), - 17 => $expand!(17), - 18 => $expand!(18), - 19 => $expand!(19), - 20 => $expand!(20), - 21 => $expand!(21), - 22 => $expand!(22), - 23 => $expand!(23), - 24 => $expand!(24), - 25 => $expand!(25), - 26 => $expand!(26), - 27 => $expand!(27), - 28 => $expand!(28), - 29 => $expand!(29), - 30 => $expand!(30), - _ => $expand!(31), - } - }; -} - -//immediate value: -16:15 -macro_rules! constify_imm_s5 { - ($imm8:expr, $expand:ident) => { - #[allow(overflowing_literals)] - match ($imm8) & 0b1_1111 { - 0 => $expand!(0), - 1 => $expand!(1), - 2 => $expand!(2), - 3 => $expand!(3), - 4 => $expand!(4), - 5 => $expand!(5), - 6 => $expand!(6), - 7 => $expand!(7), - 8 => $expand!(8), - 9 => $expand!(9), - 10 => $expand!(10), - 11 => $expand!(11), - 12 => $expand!(12), - 13 => $expand!(13), - 14 => $expand!(14), - 16 => $expand!(-16), - 17 => $expand!(-15), - 18 => $expand!(-14), - 19 => $expand!(-13), - 20 => $expand!(-12), - 21 => $expand!(-11), - 22 => $expand!(-10), - 23 => $expand!(-9), - 24 => $expand!(-8), - 25 => $expand!(-7), - 26 => $expand!(-6), - 27 => $expand!(-5), - 28 => $expand!(-4), - 29 => $expand!(-3), - 30 => $expand!(-2), - 31 => $expand!(-1), - _ => $expand!(15), - } - }; -} - -//immediate value: 0:15 -macro_rules! constify_imm4 { - ($imm8:expr, $expand:ident) => { - #[allow(overflowing_literals)] - match ($imm8) & 0b1111 { - 0 => $expand!(0), - 1 => $expand!(1), - 2 => $expand!(2), - 3 => $expand!(3), - 4 => $expand!(4), - 5 => $expand!(5), - 6 => $expand!(6), - 7 => $expand!(7), - 8 => $expand!(8), - 9 => $expand!(9), - 10 => $expand!(10), - 11 => $expand!(11), - 12 => $expand!(12), - 13 => $expand!(13), - 14 => $expand!(14), - _ => $expand!(15), - } - }; -} - -//immediate value: 0:7 -macro_rules! constify_imm3 { - ($imm8:expr, $expand:ident) => { - #[allow(overflowing_literals)] - match ($imm8) & 0b111 { - 0 => $expand!(0), - 1 => $expand!(1), - 2 => $expand!(2), - 3 => $expand!(3), - 4 => $expand!(4), - 5 => $expand!(5), - 6 => $expand!(6), - _ => $expand!(7), - } - }; -} - -//immediate value: 0:3 -macro_rules! constify_imm2 { - ($imm8:expr, $expand:ident) => { - #[allow(overflowing_literals)] - match ($imm8) & 0b11 { - 0 => $expand!(0), - 1 => $expand!(1), - 2 => $expand!(2), - _ => $expand!(3), - } - }; -} - -//immediate value: 0:1 -macro_rules! constify_imm1 { - ($imm8:expr, $expand:ident) => { - #[allow(overflowing_literals)] - match ($imm8) & 0b1 { - 0 => $expand!(0), - _ => $expand!(1), - } +macro_rules! static_assert_imm_s13 { + ($imm:ident) => { + let _ = $crate::core_arch::macros::ValidateConstImm::<$imm, -4096, 4095>::VALID; }; } diff --git a/library/stdarch/crates/core_arch/src/mod.rs b/library/stdarch/crates/core_arch/src/mod.rs index a5b0e30cad..7812dc91f9 100644 --- a/library/stdarch/crates/core_arch/src/mod.rs +++ b/library/stdarch/crates/core_arch/src/mod.rs @@ -1,11 +1,10 @@ //! `core_arch` -#![cfg_attr(not(bootstrap), allow(automatic_links))] #[macro_use] mod macros; #[cfg(any(target_arch = "arm", target_arch = "aarch64", doc))] -mod acle; +mod arm_shared; mod simd; @@ -53,17 +52,29 @@ pub mod arch { #[doc(cfg(target_arch = "aarch64"))] #[unstable(feature = "stdsimd", issue = "27731")] pub mod aarch64 { - pub use crate::core_arch::{aarch64::*, arm::*}; + pub use crate::core_arch::aarch64::*; } /// Platform-specific intrinsics for the `wasm32` platform. /// /// This module provides intrinsics specific to the WebAssembly - /// architecture. Here you'll find intrinsics necessary for leveraging - /// WebAssembly proposals such as [atomics] and [simd]. These proposals are - /// evolving over time and as such the support here is unstable and requires - /// the nightly channel. As WebAssembly proposals stabilize these functions - /// will also become stable. + /// architecture. Here you'll find intrinsics specific to WebAssembly that + /// aren't otherwise surfaced somewhere in a cross-platform abstraction of + /// `std`, and you'll also find functions for leveraging WebAssembly + /// proposals such as [atomics] and [simd]. + /// + /// Intrinsics in the `wasm32` module are modeled after the WebAssembly + /// instructions that they represent. Most functions are named after the + /// instruction they intend to correspond to, and the arguments/results + /// correspond to the type signature of the instruction itself. Stable + /// WebAssembly instructions are [documented online][instrdoc]. + /// + /// [instrdoc]: https://webassembly.github.io/spec/core/valid/instructions.html + /// + /// If a proposal is not yet stable in WebAssembly itself then the functions + /// within this function may be unstable and require the nightly channel of + /// Rust to use. As the proposal itself stabilizes the intrinsics in this + /// module should stabilize as well. /// /// [atomics]: https://github.com/webassembly/threads /// [simd]: https://github.com/webassembly/simd @@ -74,38 +85,30 @@ pub mod arch { /// ## Atomics /// /// The [threads proposal][atomics] for WebAssembly adds a number of - /// instructions for dealing with multithreaded programs. Atomic - /// instructions can all be generated through `std::sync::atomic` types, but - /// some instructions have no equivalent in Rust such as - /// `memory.atomic.notify` so this module will provide these intrinsics. - /// - /// At this time, however, these intrinsics are only available **when the - /// standard library itself is compiled with atomics**. Compiling with - /// atomics is not enabled by default and requires passing - /// `-Ctarget-feature=+atomics` to rustc. The standard library shipped via - /// `rustup` is not compiled with atomics. To get access to these intrinsics - /// you'll need to compile the standard library from source with the - /// requisite compiler flags. + /// instructions for dealing with multithreaded programs. Most instructions + /// added in the [atomics] proposal are exposed in Rust through the + /// `std::sync::atomic` module. Some instructions, however, don't have + /// direct equivalents in Rust so they're exposed here instead. + /// + /// Note that the instructions added in the [atomics] proposal can work in + /// either a context with a shared wasm memory and without. These intrinsics + /// are always available in the standard library, but you likely won't be + /// able to use them too productively unless you recompile the standard + /// library (and all your code) with `-Ctarget-feature=+atomics`. + /// + /// It's also worth pointing out that multi-threaded WebAssembly and its + /// story in Rust is still in a somewhat "early days" phase as of the time + /// of this writing. Pieces should mostly work but it generally requires a + /// good deal of manual setup. At this time it's not as simple as "just call + /// `std::thread::spawn`", but it will hopefully get there one day! /// /// ## SIMD /// - /// The [simd proposal][simd] for WebAssembly adds a new `v128` type for a - /// 128-bit SIMD register. It also adds a large array of instructions to - /// operate on the `v128` type to perform data processing. The SIMD proposal - /// has been in progress for quite some time and many instructions have come - /// and gone. This module attempts to keep up with the proposal, but if you - /// notice anything awry please feel free to [open an - /// issue](https://github.com/rust-lang/stdarch/issues/new). - /// - /// It's important to be aware that the current state of development of SIMD - /// in WebAssembly is still somewhat early days. There's lots of pieces to - /// demo and prototype with, but discussions and support are still in - /// progress. There's a number of pitfalls and gotchas in various places, - /// which will attempt to be documented here, but there may be others - /// lurking! - /// - /// Using SIMD is intended to be similar to as you would on `x86_64`, for - /// example. You'd write a function such as: + /// The [simd proposal][simd] for WebAssembly added a new `v128` type for a + /// 128-bit SIMD register. It also added a large array of instructions to + /// operate on the `v128` type to perform data processing. Using SIMD on + /// wasm is intended to be similar to as you would on `x86_64`, for example. + /// You'd write a function such as: /// /// ```rust,ignore /// #[cfg(target_arch = "wasm32")] @@ -118,15 +121,17 @@ pub mod arch { /// /// Unlike `x86_64`, however, WebAssembly does not currently have dynamic /// detection at runtime as to whether SIMD is supported (this is one of the - /// motivators for the [conditional sections proposal][condsections], but - /// that is still pretty early days). This means that your binary will - /// either have SIMD and can only run on engines which support SIMD, or it - /// will not have SIMD at all. For compatibility the standard library itself - /// does not use any SIMD internally. Determining how best to ship your - /// WebAssembly binary with SIMD is largely left up to you as it can can be - /// pretty nuanced depending on your situation. + /// motivators for the [conditional sections][condsections] and [feature + /// detection] proposals, but that is still pretty early days). This means + /// that your binary will either have SIMD and can only run on engines + /// which support SIMD, or it will not have SIMD at all. For compatibility + /// the standard library itself does not use any SIMD internally. + /// Determining how best to ship your WebAssembly binary with SIMD is + /// largely left up to you as it can can be pretty nuanced depending on + /// your situation. /// /// [condsections]: https://github.com/webassembly/conditional-sections + /// [feature detection]: https://github.com/WebAssembly/feature-detection /// /// To enable SIMD support at compile time you need to do one of two things: /// @@ -138,7 +143,9 @@ pub mod arch { /// * Second you can compile your program with `-Ctarget-feature=+simd128`. /// This compilation flag blanket enables SIMD support for your entire /// compilation. Note that this does not include the standard library - /// unless you recompile the standard library. + /// unless you [recompile the standard library][buildstd]. + /// + /// [buildstd]: https://doc.rust-lang.org/nightly/cargo/reference/unstable.html#build-std /// /// If you enable SIMD via either of these routes then you'll have a /// WebAssembly binary that uses SIMD instructions, and you'll need to ship @@ -147,21 +154,6 @@ pub mod arch { /// generated in your program. This means to generate a binary without SIMD /// you'll need to avoid both options above plus calling into any intrinsics /// in this module. - /// - /// > **Note**: Due to - /// > [rust-lang/rust#74320](https://github.com/rust-lang/rust/issues/74320) - /// > it's recommended to compile your entire program with SIMD support - /// > (using `RUSTFLAGS`) or otherwise functions may not be inlined - /// > correctly. - /// - /// > **Note**: LLVM's SIMD support is actually split into two features: - /// > `simd128` and `unimplemented-simd128`. Rust code can enable `simd128` - /// > with `#[target_feature]` (and test for it with `#[cfg(target_feature = - /// > "simd128")]`, but it cannot enable `unimplemented-simd128`. The only - /// > way to enable this feature is to compile with - /// > `-Ctarget-feature=+simd128,+unimplemented-simd128`. This second - /// > feature enables more recent instructions implemented in LLVM which - /// > haven't always had enough time to make their way to runtimes. #[cfg(any(target_arch = "wasm32", doc))] #[doc(cfg(target_arch = "wasm32"))] #[stable(feature = "simd_wasm32", since = "1.33.0")] @@ -233,8 +225,8 @@ mod x86_64; #[cfg(any(target_arch = "aarch64", doc))] #[doc(cfg(target_arch = "aarch64"))] mod aarch64; -#[cfg(any(target_arch = "arm", target_arch = "aarch64", doc))] -#[doc(cfg(any(target_arch = "arm", target_arch = "aarch64")))] +#[cfg(any(target_arch = "arm", doc))] +#[doc(cfg(any(target_arch = "arm")))] mod arm; #[cfg(any(target_arch = "wasm32", doc))] diff --git a/library/stdarch/crates/core_arch/src/powerpc/vsx.rs b/library/stdarch/crates/core_arch/src/powerpc/vsx.rs index 394c9a7043..54e067eb03 100644 --- a/library/stdarch/crates/core_arch/src/powerpc/vsx.rs +++ b/library/stdarch/crates/core_arch/src/powerpc/vsx.rs @@ -47,10 +47,10 @@ mod sealed { #[cfg_attr(all(test, target_endian = "big"), assert_instr(xxspltd, dm = 0x0))] unsafe fn xxpermdi(a: i64x2, b: i64x2, dm: u8) -> i64x2 { match dm & 0b11 { - 0 => simd_shuffle2(a, b, [0b00, 0b10]), - 1 => simd_shuffle2(a, b, [0b01, 0b10]), - 2 => simd_shuffle2(a, b, [0b00, 0b11]), - _ => simd_shuffle2(a, b, [0b01, 0b11]), + 0 => simd_shuffle2!(a, b, [0b00, 0b10]), + 1 => simd_shuffle2!(a, b, [0b01, 0b10]), + 2 => simd_shuffle2!(a, b, [0b00, 0b11]), + _ => simd_shuffle2!(a, b, [0b01, 0b11]), } } @@ -75,12 +75,13 @@ mod sealed { /// Vector permute. #[inline] #[target_feature(enable = "vsx")] -#[rustc_args_required_const(2)] -pub unsafe fn vec_xxpermdi(a: T, b: T, dm: u8) -> T +//#[rustc_legacy_const_generics(2)] +pub unsafe fn vec_xxpermdi(a: T, b: T) -> T where T: sealed::VectorPermDI, { - a.vec_xxpermdi(b, dm) + static_assert_imm2!(DM); + a.vec_xxpermdi(b, DM as u8) } #[cfg(test)] @@ -102,10 +103,10 @@ mod tests { let a: $longtype = mem::transmute($shorttype::new($($a),+, $($b),+)); let b = mem::transmute($shorttype::new($($c),+, $($d),+)); - assert_eq!($shorttype::new($($a),+, $($c),+), mem::transmute(vec_xxpermdi(a, b, 0))); - assert_eq!($shorttype::new($($b),+, $($c),+), mem::transmute(vec_xxpermdi(a, b, 1))); - assert_eq!($shorttype::new($($a),+, $($d),+), mem::transmute(vec_xxpermdi(a, b, 2))); - assert_eq!($shorttype::new($($b),+, $($d),+), mem::transmute(vec_xxpermdi(a, b, 3))); + assert_eq!($shorttype::new($($a),+, $($c),+), mem::transmute(vec_xxpermdi::<_, 0>(a, b))); + assert_eq!($shorttype::new($($b),+, $($c),+), mem::transmute(vec_xxpermdi::<_, 1>(a, b))); + assert_eq!($shorttype::new($($a),+, $($d),+), mem::transmute(vec_xxpermdi::<_, 2>(a, b))); + assert_eq!($shorttype::new($($b),+, $($d),+), mem::transmute(vec_xxpermdi::<_, 3>(a, b))); } } } diff --git a/library/stdarch/crates/core_arch/src/simd_llvm.rs b/library/stdarch/crates/core_arch/src/simd_llvm.rs index f472b86f8b..1970e5c696 100644 --- a/library/stdarch/crates/core_arch/src/simd_llvm.rs +++ b/library/stdarch/crates/core_arch/src/simd_llvm.rs @@ -9,19 +9,12 @@ extern "platform-intrinsic" { pub fn simd_gt(x: T, y: T) -> U; pub fn simd_ge(x: T, y: T) -> U; - #[rustc_args_required_const(2)] pub fn simd_shuffle2(x: T, y: T, idx: [u32; 2]) -> U; - #[rustc_args_required_const(2)] pub fn simd_shuffle4(x: T, y: T, idx: [u32; 4]) -> U; - #[rustc_args_required_const(2)] pub fn simd_shuffle8(x: T, y: T, idx: [u32; 8]) -> U; - #[rustc_args_required_const(2)] pub fn simd_shuffle16(x: T, y: T, idx: [u32; 16]) -> U; - #[rustc_args_required_const(2)] pub fn simd_shuffle32(x: T, y: T, idx: [u32; 32]) -> U; - #[rustc_args_required_const(2)] pub fn simd_shuffle64(x: T, y: T, idx: [u32; 64]) -> U; - #[rustc_args_required_const(2)] pub fn simd_shuffle128(x: T, y: T, idx: [u32; 128]) -> U; #[rustc_const_unstable(feature = "const_simd_insert", issue = "none")] @@ -43,6 +36,8 @@ extern "platform-intrinsic" { pub fn simd_or(x: T, y: T) -> T; pub fn simd_xor(x: T, y: T) -> T; + pub fn simd_neg(x: T) -> T; + pub fn simd_saturating_add(x: T, y: T) -> T; pub fn simd_saturating_sub(x: T, y: T) -> T; diff --git a/library/stdarch/crates/core_arch/src/wasm32/atomic.rs b/library/stdarch/crates/core_arch/src/wasm32/atomic.rs index 5cbb162598..4a3a5c6745 100644 --- a/library/stdarch/crates/core_arch/src/wasm32/atomic.rs +++ b/library/stdarch/crates/core_arch/src/wasm32/atomic.rs @@ -1,22 +1,12 @@ -//! Intrinsics associated with WebAssembly's upcoming threads proposal. -//! -//! These intrinsics are all unstable because they're not actually stable in -//! WebAssembly itself yet. The signatures may change as [the -//! specification][spec] is updated. -//! -//! [spec]: https://github.com/WebAssembly/threads - -#![cfg(any(target_feature = "atomics", doc))] - #[cfg(test)] use stdarch_test::assert_instr; extern "C" { - #[link_name = "llvm.wasm.atomic.wait.i32"] + #[link_name = "llvm.wasm.memory.atomic.wait32"] fn llvm_atomic_wait_i32(ptr: *mut i32, exp: i32, timeout: i64) -> i32; - #[link_name = "llvm.wasm.atomic.wait.i64"] + #[link_name = "llvm.wasm.memory.atomic.wait64"] fn llvm_atomic_wait_i64(ptr: *mut i64, exp: i64, timeout: i64) -> i32; - #[link_name = "llvm.wasm.atomic.notify"] + #[link_name = "llvm.wasm.memory.atomic.notify"] fn llvm_atomic_notify(ptr: *mut i32, cnt: i32) -> i32; } @@ -41,16 +31,11 @@ extern "C" { /// didn't block /// * 2 - the thread blocked, but the timeout expired. /// -/// # Availability -/// -/// This intrinsic is only available **when the standard library itself is -/// compiled with the `atomics` target feature**. This version of the standard -/// library is not obtainable via `rustup`, but rather will require the -/// standard library to be compiled from source. -/// /// [instr]: https://webassembly.github.io/threads/syntax/instructions.html#syntax-instr-atomic-memory #[inline] -#[cfg_attr(test, assert_instr("i32.atomic.wait"))] +#[cfg_attr(test, assert_instr(memory.atomic.wait32))] +#[target_feature(enable = "atomics")] +#[doc(alias("memory.atomic.wait32"))] pub unsafe fn memory_atomic_wait32(ptr: *mut i32, expression: i32, timeout_ns: i64) -> i32 { llvm_atomic_wait_i32(ptr, expression, timeout_ns) } @@ -76,16 +61,11 @@ pub unsafe fn memory_atomic_wait32(ptr: *mut i32, expression: i32, timeout_ns: i /// didn't block /// * 2 - the thread blocked, but the timeout expired. /// -/// # Availability -/// -/// This intrinsic is only available **when the standard library itself is -/// compiled with the `atomics` target feature**. This version of the standard -/// library is not obtainable via `rustup`, but rather will require the -/// standard library to be compiled from source. -/// /// [instr]: https://webassembly.github.io/threads/syntax/instructions.html#syntax-instr-atomic-memory #[inline] -#[cfg_attr(test, assert_instr("i64.atomic.wait"))] +#[cfg_attr(test, assert_instr(memory.atomic.wait64))] +#[target_feature(enable = "atomics")] +#[doc(alias("memory.atomic.wait64"))] pub unsafe fn memory_atomic_wait64(ptr: *mut i64, expression: i64, timeout_ns: i64) -> i32 { llvm_atomic_wait_i64(ptr, expression, timeout_ns) } @@ -103,16 +83,11 @@ pub unsafe fn memory_atomic_wait64(ptr: *mut i64, expression: i64, timeout_ns: i /// /// Returns the number of waiters which were actually notified. /// -/// # Availability -/// -/// This intrinsic is only available **when the standard library itself is -/// compiled with the `atomics` target feature**. This version of the standard -/// library is not obtainable via `rustup`, but rather will require the -/// standard library to be compiled from source. -/// /// [instr]: https://webassembly.github.io/threads/syntax/instructions.html#syntax-instr-atomic-memory #[inline] -#[cfg_attr(test, assert_instr("atomic.wake"))] +#[cfg_attr(test, assert_instr(memory.atomic.notify))] +#[target_feature(enable = "atomics")] +#[doc(alias("memory.atomic.notify"))] pub unsafe fn memory_atomic_notify(ptr: *mut i32, waiters: u32) -> u32 { llvm_atomic_notify(ptr, waiters as i32) as u32 } diff --git a/library/stdarch/crates/core_arch/src/wasm32/memory.rs b/library/stdarch/crates/core_arch/src/wasm32/memory.rs index c4e801b738..4824bc6279 100644 --- a/library/stdarch/crates/core_arch/src/wasm32/memory.rs +++ b/library/stdarch/crates/core_arch/src/wasm32/memory.rs @@ -3,9 +3,9 @@ use stdarch_test::assert_instr; extern "C" { #[link_name = "llvm.wasm.memory.grow.i32"] - fn llvm_memory_grow(mem: i32, pages: i32) -> i32; + fn llvm_memory_grow(mem: u32, pages: i32) -> i32; #[link_name = "llvm.wasm.memory.size.i32"] - fn llvm_memory_size(mem: i32) -> i32; + fn llvm_memory_size(mem: u32) -> i32; } /// Corresponding intrinsic to wasm's [`memory.size` instruction][instr] @@ -13,7 +13,7 @@ extern "C" { /// This function, when called, will return the current memory size in units of /// pages. The current WebAssembly page size is 65536 bytes (64 KB). /// -/// The argument `mem` is the numerical index of which memory to return the +/// The argument `MEM` is the numerical index of which memory to return the /// size of. Note that currently the WebAssembly specification only supports one /// memory, so it is required that zero is passed in. The argument is present to /// be forward-compatible with future WebAssembly revisions. If a nonzero @@ -21,16 +21,13 @@ extern "C" { /// /// [instr]: http://webassembly.github.io/spec/core/exec/instructions.html#exec-memory-size #[inline] -#[cfg_attr(test, assert_instr("memory.size", mem = 0))] -#[rustc_args_required_const(0)] +#[cfg_attr(test, assert_instr("memory.size", MEM = 0))] +#[rustc_legacy_const_generics(0)] #[stable(feature = "simd_wasm32", since = "1.33.0")] -pub fn memory_size(mem: u32) -> usize { - unsafe { - if mem != 0 { - crate::intrinsics::abort(); - } - llvm_memory_size(0) as usize - } +#[doc(alias("memory.size"))] +pub fn memory_size() -> usize { + static_assert!(MEM: u32 where MEM == 0); + unsafe { llvm_memory_size(MEM) as usize } } /// Corresponding intrinsic to wasm's [`memory.grow` instruction][instr] @@ -41,7 +38,7 @@ pub fn memory_size(mem: u32) -> usize { /// of memory, in pages, is returned. If memory cannot be grown then /// `usize::MAX` is returned. /// -/// The argument `mem` is the numerical index of which memory to return the +/// The argument `MEM` is the numerical index of which memory to return the /// size of. Note that currently the WebAssembly specification only supports one /// memory, so it is required that zero is passed in. The argument is present to /// be forward-compatible with future WebAssembly revisions. If a nonzero @@ -49,14 +46,13 @@ pub fn memory_size(mem: u32) -> usize { /// /// [instr]: http://webassembly.github.io/spec/core/exec/instructions.html#exec-memory-grow #[inline] -#[cfg_attr(test, assert_instr("memory.grow", mem = 0))] -#[rustc_args_required_const(0)] +#[cfg_attr(test, assert_instr("memory.grow", MEM = 0))] +#[rustc_legacy_const_generics(0)] #[stable(feature = "simd_wasm32", since = "1.33.0")] -pub fn memory_grow(mem: u32, delta: usize) -> usize { +#[doc(alias("memory.grow"))] +pub fn memory_grow(delta: usize) -> usize { unsafe { - if mem != 0 { - crate::intrinsics::abort(); - } - llvm_memory_grow(0, delta as i32) as isize as usize + static_assert!(MEM: u32 where MEM == 0); + llvm_memory_grow(MEM, delta as i32) as isize as usize } } diff --git a/library/stdarch/crates/core_arch/src/wasm32/mod.rs b/library/stdarch/crates/core_arch/src/wasm32/mod.rs index cead8c36c4..9052772894 100644 --- a/library/stdarch/crates/core_arch/src/wasm32/mod.rs +++ b/library/stdarch/crates/core_arch/src/wasm32/mod.rs @@ -3,9 +3,7 @@ #[cfg(test)] use stdarch_test::assert_instr; -#[cfg(any(target_feature = "atomics", doc))] mod atomic; -#[cfg(any(target_feature = "atomics", doc))] pub use self::atomic::*; mod simd128; diff --git a/library/stdarch/crates/core_arch/src/wasm32/simd128.rs b/library/stdarch/crates/core_arch/src/wasm32/simd128.rs index bb8e238a91..fb70ce64ad 100644 --- a/library/stdarch/crates/core_arch/src/wasm32/simd128.rs +++ b/library/stdarch/crates/core_arch/src/wasm32/simd128.rs @@ -3,15 +3,13 @@ //! [WebAssembly `SIMD128` ISA]: //! https://github.com/WebAssembly/simd/blob/master/proposals/simd/SIMD.md -#![unstable(feature = "wasm_simd", issue = "74372")] #![allow(non_camel_case_types)] #![allow(unused_imports)] use crate::{ - core_arch::{simd::*, simd_llvm::*}, + core_arch::{simd, simd_llvm::*}, marker::Sized, - mem::transmute, - ptr, + mem, ptr, }; #[cfg(test)] @@ -19,287 +17,503 @@ use stdarch_test::assert_instr; types! { /// WASM-specific 128-bit wide SIMD vector type. + /// + /// This type corresponds to the `v128` type in the [WebAssembly SIMD + /// proposal](https://github.com/webassembly/simd). This type is 128-bits + /// large and the meaning of all the bits is defined within the context of + /// how this value is used. + /// + /// This same type is used simultaneously for all 128-bit-wide SIMD types, + /// for example: + /// + /// * sixteen 8-bit integers (both `i8` and `u8`) + /// * eight 16-bit integers (both `i16` and `u16`) + /// * four 32-bit integers (both `i32` and `u32`) + /// * two 64-bit integers (both `i64` and `u64`) + /// * four 32-bit floats (`f32`) + /// * two 64-bit floats (`f64`) + /// + /// The `v128` type in Rust is intended to be quite analagous to the `v128` + /// type in WebAssembly. Operations on `v128` can only be performed with the + /// functions in this module. // N.B., internals here are arbitrary. + #[stable(feature = "wasm_simd", since = "1.54.0")] pub struct v128(i32, i32, i32, i32); } -#[allow(non_camel_case_types)] -#[unstable(feature = "stdsimd_internal", issue = "none")] -pub(crate) trait v128Ext: Sized { - unsafe fn as_v128(self) -> v128; - - #[inline] - #[target_feature(enable = "simd128")] - unsafe fn as_u8x16(self) -> u8x16 { - transmute(self.as_v128()) - } - - #[inline] - #[target_feature(enable = "simd128")] - unsafe fn as_u16x8(self) -> u16x8 { - transmute(self.as_v128()) - } - - #[inline] - #[target_feature(enable = "simd128")] - unsafe fn as_u32x4(self) -> u32x4 { - transmute(self.as_v128()) - } - - #[inline] - #[target_feature(enable = "simd128")] - unsafe fn as_u64x2(self) -> u64x2 { - transmute(self.as_v128()) - } - - #[inline] - #[target_feature(enable = "simd128")] - unsafe fn as_i8x16(self) -> i8x16 { - transmute(self.as_v128()) - } - - #[inline] - #[target_feature(enable = "simd128")] - unsafe fn as_i16x8(self) -> i16x8 { - transmute(self.as_v128()) - } - - #[inline] - #[target_feature(enable = "simd128")] - unsafe fn as_i32x4(self) -> i32x4 { - transmute(self.as_v128()) - } - - #[inline] - #[target_feature(enable = "simd128")] - unsafe fn as_i64x2(self) -> i64x2 { - transmute(self.as_v128()) - } - - #[inline] - #[target_feature(enable = "simd128")] - unsafe fn as_f32x4(self) -> f32x4 { - transmute(self.as_v128()) - } - - #[inline] - #[target_feature(enable = "simd128")] - unsafe fn as_f64x2(self) -> f64x2 { - transmute(self.as_v128()) +macro_rules! conversions { + ($(($name:ident = $ty:ty))*) => { + impl v128 { + $( + #[inline(always)] + fn $name(self) -> $ty { + unsafe { mem::transmute(self) } + } + )* + } + $( + impl $ty { + #[inline(always)] + const fn v128(self) -> v128 { + unsafe { mem::transmute(self) } + } + } + )* } } -impl v128Ext for v128 { - #[inline] - #[target_feature(enable = "simd128")] - unsafe fn as_v128(self) -> Self { - self - } +conversions! { + (as_u8x16 = simd::u8x16) + (as_u16x8 = simd::u16x8) + (as_u32x4 = simd::u32x4) + (as_u64x2 = simd::u64x2) + (as_i8x16 = simd::i8x16) + (as_i16x8 = simd::i16x8) + (as_i32x4 = simd::i32x4) + (as_i64x2 = simd::i64x2) + (as_f32x4 = simd::f32x4) + (as_f64x2 = simd::f64x2) } #[allow(improper_ctypes)] extern "C" { + #[link_name = "llvm.wasm.load32.zero"] + fn llvm_load32_zero(x: *const u32) -> simd::i32x4; + #[link_name = "llvm.wasm.load64.zero"] + fn llvm_load64_zero(x: *const u64) -> simd::i64x2; + #[link_name = "llvm.wasm.load8.lane"] + fn llvm_load8_lane(x: *const u8, v: simd::u8x16, l: usize) -> simd::u8x16; + #[link_name = "llvm.wasm.load16.lane"] + fn llvm_load16_lane(x: *const u16, v: simd::u16x8, l: usize) -> simd::u16x8; + #[link_name = "llvm.wasm.load32.lane"] + fn llvm_load32_lane(x: *const u32, v: simd::u32x4, l: usize) -> simd::u32x4; + #[link_name = "llvm.wasm.load64.lane"] + fn llvm_load64_lane(x: *const u64, v: simd::u64x2, l: usize) -> simd::u64x2; + #[link_name = "llvm.wasm.store8.lane"] + fn llvm_store8_lane(x: *mut u8, v: simd::u8x16, l: usize); + #[link_name = "llvm.wasm.store16.lane"] + fn llvm_store16_lane(x: *mut u16, v: simd::u16x8, l: usize); + #[link_name = "llvm.wasm.store32.lane"] + fn llvm_store32_lane(x: *mut u32, v: simd::u32x4, l: usize); + #[link_name = "llvm.wasm.store64.lane"] + fn llvm_store64_lane(x: *mut u64, v: simd::u64x2, l: usize); + + #[link_name = "llvm.wasm.swizzle"] + fn llvm_swizzle(a: simd::i8x16, b: simd::i8x16) -> simd::i8x16; + + #[link_name = "llvm.wasm.bitselect.v16i8"] + fn llvm_bitselect(a: simd::i8x16, b: simd::i8x16, c: simd::i8x16) -> simd::i8x16; #[link_name = "llvm.wasm.anytrue.v16i8"] - fn llvm_i8x16_any_true(x: i8x16) -> i32; + fn llvm_any_true_i8x16(x: simd::i8x16) -> i32; + #[link_name = "llvm.wasm.alltrue.v16i8"] - fn llvm_i8x16_all_true(x: i8x16) -> i32; + fn llvm_i8x16_all_true(x: simd::i8x16) -> i32; + #[link_name = "llvm.wasm.popcnt"] + fn llvm_popcnt(a: simd::i8x16) -> simd::i8x16; + #[link_name = "llvm.wasm.bitmask.v16i8"] + fn llvm_bitmask_i8x16(a: simd::i8x16) -> i32; + #[link_name = "llvm.wasm.narrow.signed.v16i8.v8i16"] + fn llvm_narrow_i8x16_s(a: simd::i16x8, b: simd::i16x8) -> simd::i8x16; + #[link_name = "llvm.wasm.narrow.unsigned.v16i8.v8i16"] + fn llvm_narrow_i8x16_u(a: simd::i16x8, b: simd::i16x8) -> simd::i8x16; #[link_name = "llvm.sadd.sat.v16i8"] - fn llvm_i8x16_add_saturate_s(a: i8x16, b: i8x16) -> i8x16; + fn llvm_i8x16_add_sat_s(a: simd::i8x16, b: simd::i8x16) -> simd::i8x16; #[link_name = "llvm.uadd.sat.v16i8"] - fn llvm_i8x16_add_saturate_u(a: i8x16, b: i8x16) -> i8x16; - #[link_name = "llvm.wasm.sub.saturate.signed.v16i8"] - fn llvm_i8x16_sub_saturate_s(a: i8x16, b: i8x16) -> i8x16; - #[link_name = "llvm.wasm.sub.saturate.unsigned.v16i8"] - fn llvm_i8x16_sub_saturate_u(a: i8x16, b: i8x16) -> i8x16; - - #[link_name = "llvm.wasm.anytrue.v8i16"] - fn llvm_i16x8_any_true(x: i16x8) -> i32; + fn llvm_i8x16_add_sat_u(a: simd::i8x16, b: simd::i8x16) -> simd::i8x16; + #[link_name = "llvm.wasm.sub.sat.signed.v16i8"] + fn llvm_i8x16_sub_sat_s(a: simd::i8x16, b: simd::i8x16) -> simd::i8x16; + #[link_name = "llvm.wasm.sub.sat.unsigned.v16i8"] + fn llvm_i8x16_sub_sat_u(a: simd::i8x16, b: simd::i8x16) -> simd::i8x16; + #[link_name = "llvm.wasm.avgr.unsigned.v16i8"] + fn llvm_avgr_u_i8x16(a: simd::i8x16, b: simd::i8x16) -> simd::i8x16; + + #[link_name = "llvm.wasm.extadd.pairwise.signed.v8i16"] + fn llvm_i16x8_extadd_pairwise_i8x16_s(x: simd::i8x16) -> simd::i16x8; + #[link_name = "llvm.wasm.extadd.pairwise.unsigned.v8i16"] + fn llvm_i16x8_extadd_pairwise_i8x16_u(x: simd::i8x16) -> simd::i16x8; + #[link_name = "llvm.wasm.q15mulr.sat.signed"] + fn llvm_q15mulr(a: simd::i16x8, b: simd::i16x8) -> simd::i16x8; #[link_name = "llvm.wasm.alltrue.v8i16"] - fn llvm_i16x8_all_true(x: i16x8) -> i32; + fn llvm_i16x8_all_true(x: simd::i16x8) -> i32; + #[link_name = "llvm.wasm.bitmask.v8i16"] + fn llvm_bitmask_i16x8(a: simd::i16x8) -> i32; + #[link_name = "llvm.wasm.narrow.signed.v8i16.v4i32"] + fn llvm_narrow_i16x8_s(a: simd::i32x4, b: simd::i32x4) -> simd::i16x8; + #[link_name = "llvm.wasm.narrow.unsigned.v8i16.v4i32"] + fn llvm_narrow_i16x8_u(a: simd::i32x4, b: simd::i32x4) -> simd::i16x8; #[link_name = "llvm.sadd.sat.v8i16"] - fn llvm_i16x8_add_saturate_s(a: i16x8, b: i16x8) -> i16x8; + fn llvm_i16x8_add_sat_s(a: simd::i16x8, b: simd::i16x8) -> simd::i16x8; #[link_name = "llvm.uadd.sat.v8i16"] - fn llvm_i16x8_add_saturate_u(a: i16x8, b: i16x8) -> i16x8; - #[link_name = "llvm.wasm.sub.saturate.signed.v8i16"] - fn llvm_i16x8_sub_saturate_s(a: i16x8, b: i16x8) -> i16x8; - #[link_name = "llvm.wasm.sub.saturate.unsigned.v8i16"] - fn llvm_i16x8_sub_saturate_u(a: i16x8, b: i16x8) -> i16x8; - - #[link_name = "llvm.wasm.anytrue.v4i32"] - fn llvm_i32x4_any_true(x: i32x4) -> i32; + fn llvm_i16x8_add_sat_u(a: simd::i16x8, b: simd::i16x8) -> simd::i16x8; + #[link_name = "llvm.wasm.sub.sat.signed.v8i16"] + fn llvm_i16x8_sub_sat_s(a: simd::i16x8, b: simd::i16x8) -> simd::i16x8; + #[link_name = "llvm.wasm.sub.sat.unsigned.v8i16"] + fn llvm_i16x8_sub_sat_u(a: simd::i16x8, b: simd::i16x8) -> simd::i16x8; + #[link_name = "llvm.wasm.avgr.unsigned.v8i16"] + fn llvm_avgr_u_i16x8(a: simd::i16x8, b: simd::i16x8) -> simd::i16x8; + #[link_name = "llvm.wasm.extmul.low.signed.v8i16"] + fn llvm_i16x8_extmul_low_i8x16_s(a: simd::i8x16, b: simd::i8x16) -> simd::i16x8; + #[link_name = "llvm.wasm.extmul.high.signed.v8i16"] + fn llvm_i16x8_extmul_high_i8x16_s(a: simd::i8x16, b: simd::i8x16) -> simd::i16x8; + #[link_name = "llvm.wasm.extmul.low.unsigned.v8i16"] + fn llvm_i16x8_extmul_low_i8x16_u(a: simd::i8x16, b: simd::i8x16) -> simd::i16x8; + #[link_name = "llvm.wasm.extmul.high.unsigned.v8i16"] + fn llvm_i16x8_extmul_high_i8x16_u(a: simd::i8x16, b: simd::i8x16) -> simd::i16x8; + + #[link_name = "llvm.wasm.extadd.pairwise.signed.v16i8"] + fn llvm_i32x4_extadd_pairwise_i16x8_s(x: simd::i16x8) -> simd::i32x4; + #[link_name = "llvm.wasm.extadd.pairwise.unsigned.v16i8"] + fn llvm_i32x4_extadd_pairwise_i16x8_u(x: simd::i16x8) -> simd::i32x4; #[link_name = "llvm.wasm.alltrue.v4i32"] - fn llvm_i32x4_all_true(x: i32x4) -> i32; - + fn llvm_i32x4_all_true(x: simd::i32x4) -> i32; + #[link_name = "llvm.wasm.bitmask.v4i32"] + fn llvm_bitmask_i32x4(a: simd::i32x4) -> i32; + #[link_name = "llvm.wasm.dot"] + fn llvm_i32x4_dot_i16x8_s(a: simd::i16x8, b: simd::i16x8) -> simd::i32x4; + #[link_name = "llvm.wasm.extmul.low.signed.v4i32"] + fn llvm_i32x4_extmul_low_i16x8_s(a: simd::i16x8, b: simd::i16x8) -> simd::i32x4; + #[link_name = "llvm.wasm.extmul.high.signed.v4i32"] + fn llvm_i32x4_extmul_high_i16x8_s(a: simd::i16x8, b: simd::i16x8) -> simd::i32x4; + #[link_name = "llvm.wasm.extmul.low.unsigned.v4i32"] + fn llvm_i32x4_extmul_low_i16x8_u(a: simd::i16x8, b: simd::i16x8) -> simd::i32x4; + #[link_name = "llvm.wasm.extmul.high.unsigned.v4i32"] + fn llvm_i32x4_extmul_high_i16x8_u(a: simd::i16x8, b: simd::i16x8) -> simd::i32x4; + + #[link_name = "llvm.wasm.alltrue.v2i64"] + fn llvm_i64x2_all_true(x: simd::i64x2) -> i32; + #[link_name = "llvm.wasm.bitmask.v2i64"] + fn llvm_bitmask_i64x2(a: simd::i64x2) -> i32; + #[link_name = "llvm.wasm.extmul.low.signed.v2i64"] + fn llvm_i64x2_extmul_low_i32x4_s(a: simd::i32x4, b: simd::i32x4) -> simd::i64x2; + #[link_name = "llvm.wasm.extmul.high.signed.v2i64"] + fn llvm_i64x2_extmul_high_i32x4_s(a: simd::i32x4, b: simd::i32x4) -> simd::i64x2; + #[link_name = "llvm.wasm.extmul.low.unsigned.v2i64"] + fn llvm_i64x2_extmul_low_i32x4_u(a: simd::i32x4, b: simd::i32x4) -> simd::i64x2; + #[link_name = "llvm.wasm.extmul.high.unsigned.v2i64"] + fn llvm_i64x2_extmul_high_i32x4_u(a: simd::i32x4, b: simd::i32x4) -> simd::i64x2; + + #[link_name = "llvm.ceil.v4f32"] + fn llvm_f32x4_ceil(x: simd::f32x4) -> simd::f32x4; + #[link_name = "llvm.floor.v4f32"] + fn llvm_f32x4_floor(x: simd::f32x4) -> simd::f32x4; + #[link_name = "llvm.trunc.v4f32"] + fn llvm_f32x4_trunc(x: simd::f32x4) -> simd::f32x4; + #[link_name = "llvm.nearbyint.v4f32"] + fn llvm_f32x4_nearest(x: simd::f32x4) -> simd::f32x4; #[link_name = "llvm.fabs.v4f32"] - fn llvm_f32x4_abs(x: f32x4) -> f32x4; + fn llvm_f32x4_abs(x: simd::f32x4) -> simd::f32x4; #[link_name = "llvm.sqrt.v4f32"] - fn llvm_f32x4_sqrt(x: f32x4) -> f32x4; + fn llvm_f32x4_sqrt(x: simd::f32x4) -> simd::f32x4; #[link_name = "llvm.minimum.v4f32"] - fn llvm_f32x4_min(x: f32x4, y: f32x4) -> f32x4; + fn llvm_f32x4_min(x: simd::f32x4, y: simd::f32x4) -> simd::f32x4; #[link_name = "llvm.maximum.v4f32"] - fn llvm_f32x4_max(x: f32x4, y: f32x4) -> f32x4; + fn llvm_f32x4_max(x: simd::f32x4, y: simd::f32x4) -> simd::f32x4; + #[link_name = "llvm.wasm.pmin.v4f32"] + fn llvm_f32x4_pmin(x: simd::f32x4, y: simd::f32x4) -> simd::f32x4; + #[link_name = "llvm.wasm.pmax.v4f32"] + fn llvm_f32x4_pmax(x: simd::f32x4, y: simd::f32x4) -> simd::f32x4; + + #[link_name = "llvm.ceil.v2f64"] + fn llvm_f64x2_ceil(x: simd::f64x2) -> simd::f64x2; + #[link_name = "llvm.floor.v2f64"] + fn llvm_f64x2_floor(x: simd::f64x2) -> simd::f64x2; + #[link_name = "llvm.trunc.v2f64"] + fn llvm_f64x2_trunc(x: simd::f64x2) -> simd::f64x2; + #[link_name = "llvm.nearbyint.v2f64"] + fn llvm_f64x2_nearest(x: simd::f64x2) -> simd::f64x2; #[link_name = "llvm.fabs.v2f64"] - fn llvm_f64x2_abs(x: f64x2) -> f64x2; + fn llvm_f64x2_abs(x: simd::f64x2) -> simd::f64x2; #[link_name = "llvm.sqrt.v2f64"] - fn llvm_f64x2_sqrt(x: f64x2) -> f64x2; + fn llvm_f64x2_sqrt(x: simd::f64x2) -> simd::f64x2; #[link_name = "llvm.minimum.v2f64"] - fn llvm_f64x2_min(x: f64x2, y: f64x2) -> f64x2; + fn llvm_f64x2_min(x: simd::f64x2, y: simd::f64x2) -> simd::f64x2; #[link_name = "llvm.maximum.v2f64"] - fn llvm_f64x2_max(x: f64x2, y: f64x2) -> f64x2; - - #[link_name = "llvm.wasm.bitselect.v16i8"] - fn llvm_bitselect(a: i8x16, b: i8x16, c: i8x16) -> i8x16; - #[link_name = "llvm.wasm.swizzle"] - fn llvm_swizzle(a: i8x16, b: i8x16) -> i8x16; - - #[link_name = "llvm.wasm.bitmask.v16i8"] - fn llvm_bitmask_i8x16(a: i8x16) -> i32; - #[link_name = "llvm.wasm.narrow.signed.v16i8.v8i16"] - fn llvm_narrow_i8x16_s(a: i16x8, b: i16x8) -> i8x16; - #[link_name = "llvm.wasm.narrow.unsigned.v16i8.v8i16"] - fn llvm_narrow_i8x16_u(a: i16x8, b: i16x8) -> i8x16; - #[link_name = "llvm.wasm.avgr.unsigned.v16i8"] - fn llvm_avgr_u_i8x16(a: i8x16, b: i8x16) -> i8x16; - - #[link_name = "llvm.wasm.bitmask.v8i16"] - fn llvm_bitmask_i16x8(a: i16x8) -> i32; - #[link_name = "llvm.wasm.narrow.signed.v8i16.v8i16"] - fn llvm_narrow_i16x8_s(a: i32x4, b: i32x4) -> i16x8; - #[link_name = "llvm.wasm.narrow.unsigned.v8i16.v8i16"] - fn llvm_narrow_i16x8_u(a: i32x4, b: i32x4) -> i16x8; - #[link_name = "llvm.wasm.avgr.unsigned.v8i16"] - fn llvm_avgr_u_i16x8(a: i16x8, b: i16x8) -> i16x8; - #[link_name = "llvm.wasm.widen.low.signed.v8i16.v16i8"] - fn llvm_widen_low_i16x8_s(a: i8x16) -> i16x8; - #[link_name = "llvm.wasm.widen.high.signed.v8i16.v16i8"] - fn llvm_widen_high_i16x8_s(a: i8x16) -> i16x8; - #[link_name = "llvm.wasm.widen.low.unsigned.v8i16.v16i8"] - fn llvm_widen_low_i16x8_u(a: i8x16) -> i16x8; - #[link_name = "llvm.wasm.widen.high.unsigned.v8i16.v16i8"] - fn llvm_widen_high_i16x8_u(a: i8x16) -> i16x8; - - #[link_name = "llvm.wasm.bitmask.v4i32"] - fn llvm_bitmask_i32x4(a: i32x4) -> i32; - #[link_name = "llvm.wasm.avgr.unsigned.v4i32"] - fn llvm_avgr_u_i32x4(a: i32x4, b: i32x4) -> i32x4; - #[link_name = "llvm.wasm.widen.low.signed.v4i32.v8i16"] - fn llvm_widen_low_i32x4_s(a: i16x8) -> i32x4; - #[link_name = "llvm.wasm.widen.high.signed.v4i32.v8i16"] - fn llvm_widen_high_i32x4_s(a: i16x8) -> i32x4; - #[link_name = "llvm.wasm.widen.low.unsigned.v4i32.v8i16"] - fn llvm_widen_low_i32x4_u(a: i16x8) -> i32x4; - #[link_name = "llvm.wasm.widen.high.unsigned.v4i32.v8i16"] - fn llvm_widen_high_i32x4_u(a: i16x8) -> i32x4; + fn llvm_f64x2_max(x: simd::f64x2, y: simd::f64x2) -> simd::f64x2; + #[link_name = "llvm.wasm.pmin.v2f64"] + fn llvm_f64x2_pmin(x: simd::f64x2, y: simd::f64x2) -> simd::f64x2; + #[link_name = "llvm.wasm.pmax.v2f64"] + fn llvm_f64x2_pmax(x: simd::f64x2, y: simd::f64x2) -> simd::f64x2; + + #[link_name = "llvm.fptosi.sat.v4i32.v4f32"] + fn llvm_i32x4_trunc_sat_f32x4_s(x: simd::f32x4) -> simd::i32x4; + #[link_name = "llvm.fptoui.sat.v4i32.v4f32"] + fn llvm_i32x4_trunc_sat_f32x4_u(x: simd::f32x4) -> simd::i32x4; + #[link_name = "llvm.fptosi.sat.v2i32.v2f64"] + fn llvm_i32x2_trunc_sat_f64x2_s(x: simd::f64x2) -> simd::i32x2; + #[link_name = "llvm.fptoui.sat.v2i32.v2f64"] + fn llvm_i32x2_trunc_sat_f64x2_u(x: simd::f64x2) -> simd::i32x2; + #[link_name = "llvm.wasm.demote.zero"] + fn llvm_f32x4_demote_f64x2_zero(x: simd::f64x2) -> simd::f32x4; + #[link_name = "llvm.wasm.promote.low"] + fn llvm_f64x2_promote_low_f32x4(x: simd::f32x4) -> simd::f64x2; +} + +#[repr(packed)] +#[derive(Copy)] +struct Unaligned(T); + +impl Clone for Unaligned { + fn clone(&self) -> Unaligned { + *self + } } /// Loads a `v128` vector from the given heap address. #[inline] #[cfg_attr(test, assert_instr(v128.load))] #[target_feature(enable = "simd128")] +#[doc(alias("v128.load"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] pub unsafe fn v128_load(m: *const v128) -> v128 { - *m + (*(m as *const Unaligned)).0 } /// Load eight 8-bit integers and sign extend each one to a 16-bit lane #[inline] -#[cfg_attr(all(test, all_simd), assert_instr(i16x8.load8x8_s))] +#[cfg_attr(test, assert_instr(v128.load8x8_s))] #[target_feature(enable = "simd128")] -pub unsafe fn i16x8_load8x8_s(m: *const i8) -> v128 { - transmute(simd_cast::<_, i16x8>(*(m as *const i8x8))) +#[doc(alias("v128.load8x8_s"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub unsafe fn i16x8_load_extend_i8x8(m: *const i8) -> v128 { + let m = *(m as *const Unaligned); + simd_cast::<_, simd::i16x8>(m.0).v128() } /// Load eight 8-bit integers and zero extend each one to a 16-bit lane #[inline] -#[cfg_attr(all(test, all_simd), assert_instr(i16x8.load8x8_u))] +#[cfg_attr(test, assert_instr(v128.load8x8_u))] #[target_feature(enable = "simd128")] -pub unsafe fn i16x8_load8x8_u(m: *const u8) -> v128 { - transmute(simd_cast::<_, u16x8>(*(m as *const u8x8))) +#[doc(alias("v128.load8x8_u"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub unsafe fn i16x8_load_extend_u8x8(m: *const u8) -> v128 { + let m = *(m as *const Unaligned); + simd_cast::<_, simd::u16x8>(m.0).v128() } +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub use i16x8_load_extend_u8x8 as u16x8_load_extend_u8x8; + /// Load four 16-bit integers and sign extend each one to a 32-bit lane #[inline] -#[cfg_attr(all(test, all_simd), assert_instr(i32x4.load16x4_s))] +#[cfg_attr(test, assert_instr(v128.load16x4_s))] #[target_feature(enable = "simd128")] -pub unsafe fn i32x4_load16x4_s(m: *const i16) -> v128 { - transmute(simd_cast::<_, i32x4>(*(m as *const i16x4))) +#[doc(alias("v128.load16x4_s"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub unsafe fn i32x4_load_extend_i16x4(m: *const i16) -> v128 { + let m = *(m as *const Unaligned); + simd_cast::<_, simd::i32x4>(m.0).v128() } /// Load four 16-bit integers and zero extend each one to a 32-bit lane #[inline] -#[cfg_attr(all(test, all_simd), assert_instr(i32x4.load16x4_u))] +#[cfg_attr(test, assert_instr(v128.load16x4_u))] #[target_feature(enable = "simd128")] -pub unsafe fn i32x4_load16x4_u(m: *const u16) -> v128 { - transmute(simd_cast::<_, u32x4>(*(m as *const u16x4))) +#[doc(alias("v128.load16x4_u"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub unsafe fn i32x4_load_extend_u16x4(m: *const u16) -> v128 { + let m = *(m as *const Unaligned); + simd_cast::<_, simd::u32x4>(m.0).v128() } +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub use i32x4_load_extend_u16x4 as u32x4_load_extend_u16x4; + /// Load two 32-bit integers and sign extend each one to a 64-bit lane #[inline] -#[cfg_attr(all(test, all_simd), assert_instr(i64x2.load32x2_s))] +#[cfg_attr(test, assert_instr(v128.load32x2_s))] #[target_feature(enable = "simd128")] -pub unsafe fn i64x2_load32x2_s(m: *const i32) -> v128 { - transmute(simd_cast::<_, i64x2>(*(m as *const i32x2))) +#[doc(alias("v128.load32x2_s"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub unsafe fn i64x2_load_extend_i32x2(m: *const i32) -> v128 { + let m = *(m as *const Unaligned); + simd_cast::<_, simd::i64x2>(m.0).v128() } /// Load two 32-bit integers and zero extend each one to a 64-bit lane #[inline] -#[cfg_attr(all(test, all_simd), assert_instr(i64x2.load32x2_u))] +#[cfg_attr(test, assert_instr(v128.load32x2_u))] #[target_feature(enable = "simd128")] -pub unsafe fn i64x2_load32x2_u(m: *const u32) -> v128 { - transmute(simd_cast::<_, u64x2>(*(m as *const u32x2))) +#[doc(alias("v128.load32x2_u"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub unsafe fn i64x2_load_extend_u32x2(m: *const u32) -> v128 { + let m = *(m as *const Unaligned); + simd_cast::<_, simd::u64x2>(m.0).v128() } +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub use i64x2_load_extend_u32x2 as u64x2_load_extend_u32x2; + /// Load a single element and splat to all lanes of a v128 vector. #[inline] -#[cfg_attr(all(test, all_simd), assert_instr(v8x16.load_splat))] +#[cfg_attr(test, assert_instr(v128.load8_splat))] #[target_feature(enable = "simd128")] -pub unsafe fn v8x16_load_splat(m: *const u8) -> v128 { - let v = *m; - transmute(u8x16(v, v, v, v, v, v, v, v, v, v, v, v, v, v, v, v)) +#[doc(alias("v128.load8_splat"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub unsafe fn v128_load8_splat(m: *const u8) -> v128 { + simd::u8x16::splat(*m).v128() } /// Load a single element and splat to all lanes of a v128 vector. #[inline] -#[cfg_attr(all(test, all_simd), assert_instr(v16x8.load_splat))] +#[cfg_attr(test, assert_instr(v128.load16_splat))] #[target_feature(enable = "simd128")] -pub unsafe fn v16x8_load_splat(m: *const u16) -> v128 { - let v = *m; - transmute(u16x8(v, v, v, v, v, v, v, v)) +#[doc(alias("v128.load16_splat"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub unsafe fn v128_load16_splat(m: *const u16) -> v128 { + let m = ptr::read_unaligned(m); + simd::u16x8::splat(m).v128() } /// Load a single element and splat to all lanes of a v128 vector. #[inline] -#[cfg_attr(all(test, all_simd), assert_instr(v32x4.load_splat))] +#[cfg_attr(test, assert_instr(v128.load32_splat))] #[target_feature(enable = "simd128")] -pub unsafe fn v32x4_load_splat(m: *const u32) -> v128 { - let v = *m; - transmute(u32x4(v, v, v, v)) +#[doc(alias("v128.load32_splat"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub unsafe fn v128_load32_splat(m: *const u32) -> v128 { + let m = ptr::read_unaligned(m); + simd::u32x4::splat(m).v128() } /// Load a single element and splat to all lanes of a v128 vector. #[inline] -#[cfg_attr(all(test, all_simd), assert_instr(v64x2.load_splat))] +#[cfg_attr(test, assert_instr(v128.load64_splat))] +#[target_feature(enable = "simd128")] +#[doc(alias("v128.load64_splat"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub unsafe fn v128_load64_splat(m: *const u64) -> v128 { + let m = ptr::read_unaligned(m); + simd::u64x2::splat(m).v128() +} + +/// Load a 32-bit element into the low bits of the vector and sets all other +/// bits to zero. +#[inline] +#[cfg_attr(test, assert_instr(v128.load32_zero))] +#[target_feature(enable = "simd128")] +#[doc(alias("v128.load32_zero"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub unsafe fn v128_load32_zero(m: *const u32) -> v128 { + llvm_load32_zero(m).v128() +} + +/// Load a 64-bit element into the low bits of the vector and sets all other +/// bits to zero. +#[inline] +#[cfg_attr(test, assert_instr(v128.load64_zero))] #[target_feature(enable = "simd128")] -pub unsafe fn v64x2_load_splat(m: *const u64) -> v128 { - let v = *m; - transmute(u64x2(v, v)) +#[doc(alias("v128.load64_zero"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub unsafe fn v128_load64_zero(m: *const u64) -> v128 { + llvm_load64_zero(m).v128() } /// Stores a `v128` vector to the given heap address. #[inline] #[cfg_attr(test, assert_instr(v128.store))] #[target_feature(enable = "simd128")] +#[doc(alias("v128.store"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] pub unsafe fn v128_store(m: *mut v128, a: v128) { - *m = a; + *(m as *mut Unaligned) = Unaligned(a); +} + +/// Loads an 8-bit value from `m` and sets lane `L` of `v` to that value. +#[inline] +#[cfg_attr(test, assert_instr(v128.load8_lane, L = 0))] +#[target_feature(enable = "simd128")] +#[doc(alias("v128.load8_lane"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub unsafe fn v128_load8_lane(v: v128, m: *const u8) -> v128 { + static_assert!(L: usize where L < 16); + llvm_load8_lane(m, v.as_u8x16(), L).v128() } -/// Materializes a constant SIMD value from the immediate operands. +/// Loads a 16-bit value from `m` and sets lane `L` of `v` to that value. +#[inline] +#[cfg_attr(test, assert_instr(v128.load16_lane, L = 0))] +#[target_feature(enable = "simd128")] +#[doc(alias("v128.load16_lane"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub unsafe fn v128_load16_lane(v: v128, m: *const u16) -> v128 { + static_assert!(L: usize where L < 8); + llvm_load16_lane(m, v.as_u16x8(), L).v128() +} + +/// Loads a 32-bit value from `m` and sets lane `L` of `v` to that value. +#[inline] +#[cfg_attr(test, assert_instr(v128.load32_lane, L = 0))] +#[target_feature(enable = "simd128")] +#[doc(alias("v128.load32_lane"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub unsafe fn v128_load32_lane(v: v128, m: *const u32) -> v128 { + static_assert!(L: usize where L < 4); + llvm_load32_lane(m, v.as_u32x4(), L).v128() +} + +/// Loads a 64-bit value from `m` and sets lane `L` of `v` to that value. +#[inline] +#[cfg_attr(test, assert_instr(v128.load64_lane, L = 0))] +#[target_feature(enable = "simd128")] +#[doc(alias("v128.load64_lane"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub unsafe fn v128_load64_lane(v: v128, m: *const u64) -> v128 { + static_assert!(L: usize where L < 2); + llvm_load64_lane(m, v.as_u64x2(), L).v128() +} + +/// Stores the 8-bit value from lane `L` of `v` into `m` +#[inline] +#[cfg_attr(test, assert_instr(v128.store8_lane, L = 0))] +#[target_feature(enable = "simd128")] +#[doc(alias("v128.store8_lane"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub unsafe fn v128_store8_lane(v: v128, m: *mut u8) { + static_assert!(L: usize where L < 16); + llvm_store8_lane(m, v.as_u8x16(), L); +} + +/// Stores the 16-bit value from lane `L` of `v` into `m` +#[inline] +#[cfg_attr(test, assert_instr(v128.store16_lane, L = 0))] +#[target_feature(enable = "simd128")] +#[doc(alias("v128.store16_lane"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub unsafe fn v128_store16_lane(v: v128, m: *mut u16) { + static_assert!(L: usize where L < 8); + llvm_store16_lane(m, v.as_u16x8(), L) +} + +/// Stores the 32-bit value from lane `L` of `v` into `m` +#[inline] +#[cfg_attr(test, assert_instr(v128.store32_lane, L = 0))] +#[target_feature(enable = "simd128")] +#[doc(alias("v128.store32_lane"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub unsafe fn v128_store32_lane(v: v128, m: *mut u32) { + static_assert!(L: usize where L < 4); + llvm_store32_lane(m, v.as_u32x4(), L) +} + +/// Stores the 64-bit value from lane `L` of `v` into `m` +#[inline] +#[cfg_attr(test, assert_instr(v128.store64_lane, L = 0))] +#[target_feature(enable = "simd128")] +#[doc(alias("v128.store64_lane"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub unsafe fn v128_store64_lane(v: v128, m: *mut u64) { + static_assert!(L: usize where L < 2); + llvm_store64_lane(m, v.as_u64x2(), L) +} + +/// Materializes a SIMD value from the provided operands. /// -/// This function generates a `v128.const` instruction as if the generated -/// vector was interpreted as sixteen 8-bit integers. +/// If possible this will generate a `v128.const` instruction, otherwise it may +/// be lowered to a sequence of instructions to materialize the vector value. #[inline] #[target_feature(enable = "simd128")] #[cfg_attr( - all(test, all_simd), + test, assert_instr( v128.const, a0 = 0, @@ -320,7 +534,10 @@ pub unsafe fn v128_store(m: *mut v128, a: v128) { a15 = 15, ) )] -pub const unsafe fn i8x16_const( +#[doc(alias("v128.const"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +#[rustc_const_stable(feature = "wasm_simd", since = "1.54.0")] +pub const fn i8x16( a0: i8, a1: i8, a2: i8, @@ -338,19 +555,67 @@ pub const unsafe fn i8x16_const( a14: i8, a15: i8, ) -> v128 { - transmute(i8x16( - a0, a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12, a13, a14, a15, - )) + v128( + (a0 as u8 as i32) + | ((a1 as u8 as i32) << 8) + | ((a2 as u8 as i32) << 16) + | ((a3 as u8 as i32) << 24), + (a4 as u8 as i32) + | ((a5 as u8 as i32) << 8) + | ((a6 as u8 as i32) << 16) + | ((a7 as u8 as i32) << 24), + (a8 as u8 as i32) + | ((a9 as u8 as i32) << 8) + | ((a10 as u8 as i32) << 16) + | ((a11 as u8 as i32) << 24), + (a12 as u8 as i32) + | ((a13 as u8 as i32) << 8) + | ((a14 as u8 as i32) << 16) + | ((a15 as u8 as i32) << 24), + ) } -/// Materializes a constant SIMD value from the immediate operands. +/// Materializes a SIMD value from the provided operands. /// -/// This function generates a `v128.const` instruction as if the generated -/// vector was interpreted as eight 16-bit integers. +/// If possible this will generate a `v128.const` instruction, otherwise it may +/// be lowered to a sequence of instructions to materialize the vector value. +#[inline] +#[target_feature(enable = "simd128")] +#[doc(alias("v128.const"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +#[rustc_const_stable(feature = "wasm_simd", since = "1.54.0")] +pub const fn u8x16( + a0: u8, + a1: u8, + a2: u8, + a3: u8, + a4: u8, + a5: u8, + a6: u8, + a7: u8, + a8: u8, + a9: u8, + a10: u8, + a11: u8, + a12: u8, + a13: u8, + a14: u8, + a15: u8, +) -> v128 { + i8x16( + a0 as i8, a1 as i8, a2 as i8, a3 as i8, a4 as i8, a5 as i8, a6 as i8, a7 as i8, a8 as i8, + a9 as i8, a10 as i8, a11 as i8, a12 as i8, a13 as i8, a14 as i8, a15 as i8, + ) +} + +/// Materializes a SIMD value from the provided operands. +/// +/// If possible this will generate a `v128.const` instruction, otherwise it may +/// be lowered to a sequence of instructions to materialize the vector value. #[inline] #[target_feature(enable = "simd128")] #[cfg_attr( - all(test, all_simd), + test, assert_instr( v128.const, a0 = 0, @@ -363,67 +628,119 @@ pub const unsafe fn i8x16_const( a7 = 7, ) )] -pub const unsafe fn i16x8_const( - a0: i16, - a1: i16, - a2: i16, - a3: i16, - a4: i16, - a5: i16, - a6: i16, - a7: i16, -) -> v128 { - transmute(i16x8(a0, a1, a2, a3, a4, a5, a6, a7)) +#[doc(alias("v128.const"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +#[rustc_const_stable(feature = "wasm_simd", since = "1.54.0")] +pub const fn i16x8(a0: i16, a1: i16, a2: i16, a3: i16, a4: i16, a5: i16, a6: i16, a7: i16) -> v128 { + v128( + (a0 as u16 as i32) | ((a1 as i32) << 16), + (a2 as u16 as i32) | ((a3 as i32) << 16), + (a4 as u16 as i32) | ((a5 as i32) << 16), + (a6 as u16 as i32) | ((a7 as i32) << 16), + ) +} + +/// Materializes a SIMD value from the provided operands. +/// +/// If possible this will generate a `v128.const` instruction, otherwise it may +/// be lowered to a sequence of instructions to materialize the vector value. +#[inline] +#[target_feature(enable = "simd128")] +#[doc(alias("v128.const"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +#[rustc_const_stable(feature = "wasm_simd", since = "1.54.0")] +pub const fn u16x8(a0: u16, a1: u16, a2: u16, a3: u16, a4: u16, a5: u16, a6: u16, a7: u16) -> v128 { + i16x8( + a0 as i16, a1 as i16, a2 as i16, a3 as i16, a4 as i16, a5 as i16, a6 as i16, a7 as i16, + ) +} + +/// Materializes a SIMD value from the provided operands. +/// +/// If possible this will generate a `v128.const` instruction, otherwise it may +/// be lowered to a sequence of instructions to materialize the vector value. +#[inline] +#[target_feature(enable = "simd128")] +#[cfg_attr(test, assert_instr(v128.const, a0 = 0, a1 = 1, a2 = 2, a3 = 3))] +#[doc(alias("v128.const"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +#[rustc_const_stable(feature = "wasm_simd", since = "1.54.0")] +pub const fn i32x4(a0: i32, a1: i32, a2: i32, a3: i32) -> v128 { + v128(a0, a1, a2, a3) } -/// Materializes a constant SIMD value from the immediate operands. +/// Materializes a SIMD value from the provided operands. /// -/// This function generates a `v128.const` instruction as if the generated -/// vector was interpreted as four 32-bit integers. +/// If possible this will generate a `v128.const` instruction, otherwise it may +/// be lowered to a sequence of instructions to materialize the vector value. #[inline] #[target_feature(enable = "simd128")] -#[cfg_attr(all(test, all_simd), assert_instr(v128.const, a0 = 0, a1 = 1, a2 = 2, a3 = 3))] -pub const unsafe fn i32x4_const(a0: i32, a1: i32, a2: i32, a3: i32) -> v128 { - transmute(i32x4(a0, a1, a2, a3)) +#[doc(alias("v128.const"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +#[rustc_const_stable(feature = "wasm_simd", since = "1.54.0")] +pub const fn u32x4(a0: u32, a1: u32, a2: u32, a3: u32) -> v128 { + i32x4(a0 as i32, a1 as i32, a2 as i32, a3 as i32) } -/// Materializes a constant SIMD value from the immediate operands. +/// Materializes a SIMD value from the provided operands. /// -/// This function generates a `v128.const` instruction as if the generated -/// vector was interpreted as two 64-bit integers. +/// If possible this will generate a `v128.const` instruction, otherwise it may +/// be lowered to a sequence of instructions to materialize the vector value. #[inline] #[target_feature(enable = "simd128")] -#[cfg_attr(all(test, all_simd), assert_instr(v128.const, a0 = 0, a1 = 1))] -pub const unsafe fn i64x2_const(a0: i64, a1: i64) -> v128 { - transmute(i64x2(a0, a1)) +#[cfg_attr(test, assert_instr(v128.const, a0 = 1, a1 = 2))] +#[doc(alias("v128.const"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +#[rustc_const_stable(feature = "wasm_simd", since = "1.54.0")] +pub const fn i64x2(a0: i64, a1: i64) -> v128 { + v128(a0 as i32, (a0 >> 32) as i32, a1 as i32, (a1 >> 32) as i32) } -/// Materializes a constant SIMD value from the immediate operands. +/// Materializes a SIMD value from the provided operands. /// -/// This function generates a `v128.const` instruction as if the generated -/// vector was interpreted as four 32-bit floats. +/// If possible this will generate a `v128.const` instruction, otherwise it may +/// be lowered to a sequence of instructions to materialize the vector value. #[inline] #[target_feature(enable = "simd128")] -#[cfg_attr(all(test, all_simd), assert_instr(v128.const, a0 = 0.0, a1 = 1.0, a2 = 2.0, a3 = 3.0))] -pub const unsafe fn f32x4_const(a0: f32, a1: f32, a2: f32, a3: f32) -> v128 { - transmute(f32x4(a0, a1, a2, a3)) +#[doc(alias("v128.const"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +#[rustc_const_stable(feature = "wasm_simd", since = "1.54.0")] +pub const fn u64x2(a0: u64, a1: u64) -> v128 { + i64x2(a0 as i64, a1 as i64) } -/// Materializes a constant SIMD value from the immediate operands. +/// Materializes a SIMD value from the provided operands. /// -/// This function generates a `v128.const` instruction as if the generated -/// vector was interpreted as two 64-bit floats. +/// If possible this will generate a `v128.const` instruction, otherwise it may +/// be lowered to a sequence of instructions to materialize the vector value. #[inline] #[target_feature(enable = "simd128")] -#[cfg_attr(all(test, all_simd), assert_instr(v128.const, a0 = 0.0, a1 = 1.0))] -pub const unsafe fn f64x2_const(a0: f64, a1: f64) -> v128 { - transmute(f64x2(a0, a1)) +#[cfg_attr(test, assert_instr(v128.const, a0 = 0.0, a1 = 1.0, a2 = 2.0, a3 = 3.0))] +#[doc(alias("v128.const"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +#[rustc_const_unstable(feature = "wasm_simd_const", issue = "72447")] +pub const fn f32x4(a0: f32, a1: f32, a2: f32, a3: f32) -> v128 { + simd::f32x4(a0, a1, a2, a3).v128() +} + +/// Materializes a SIMD value from the provided operands. +/// +/// If possible this will generate a `v128.const` instruction, otherwise it may +/// be lowered to a sequence of instructions to materialize the vector value. +#[inline] +#[target_feature(enable = "simd128")] +#[cfg_attr(test, assert_instr(v128.const, a0 = 0.0, a1 = 1.0))] +#[doc(alias("v128.const"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +#[rustc_const_unstable(feature = "wasm_simd_const", issue = "72447")] +pub const fn f64x2(a0: f64, a1: f64) -> v128 { + simd::f64x2(a0, a1).v128() } /// Returns a new vector with lanes selected from the lanes of the two input /// vectors `$a` and `$b` specified in the 16 immediate operands. /// -/// The `$a` and `$b` expressions must have type `v128`, and this macro +/// The `$a` and `$b` expressions must have type `v128`, and this function /// generates a wasm instruction that is encoded with 16 bytes providing the /// indices of the elements to return. The indices `i` in range [0, 15] select /// the `i`-th element of `a`. The indices in range [16, 31] select the `i - @@ -435,8 +752,31 @@ pub const unsafe fn f64x2_const(a0: f64, a1: f64) -> v128 { /// /// All indexes `$i*` must have the type `u32`. #[inline] +#[cfg_attr(test, + assert_instr( + i8x16.shuffle, + I0 = 0, + I1 = 2, + I2 = 4, + I3 = 6, + I4 = 8, + I5 = 10, + I6 = 12, + I7 = 14, + I8 = 16, + I9 = 18, + I10 = 20, + I11 = 22, + I12 = 24, + I13 = 26, + I14 = 28, + I15 = 30, + ) +)] #[target_feature(enable = "simd128")] -pub unsafe fn v8x16_shuffle< +#[doc(alias("i8x16.shuffle"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i8x16_shuffle< const I0: usize, const I1: usize, const I2: usize, @@ -457,35 +797,81 @@ pub unsafe fn v8x16_shuffle< a: v128, b: v128, ) -> v128 { - let shuf = simd_shuffle16::( - a.as_u8x16(), - b.as_u8x16(), - [ - I0 as u32, I1 as u32, I2 as u32, I3 as u32, I4 as u32, I5 as u32, I6 as u32, I7 as u32, - I8 as u32, I9 as u32, I10 as u32, I11 as u32, I12 as u32, I13 as u32, I14 as u32, - I15 as u32, - ], - ); - transmute(shuf) -} - -#[cfg(test)] -#[assert_instr(v8x16.shuffle)] -#[target_feature(enable = "simd128")] -unsafe fn v8x16_shuffle_test(a: v128, b: v128) -> v128 { - v8x16_shuffle::<0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30>(a, b) -} - -/// Same as [`v8x16_shuffle`], except operates as if the inputs were eight + static_assert!(I0: usize where I0 < 32); + static_assert!(I1: usize where I1 < 32); + static_assert!(I2: usize where I2 < 32); + static_assert!(I3: usize where I3 < 32); + static_assert!(I4: usize where I4 < 32); + static_assert!(I5: usize where I5 < 32); + static_assert!(I6: usize where I6 < 32); + static_assert!(I7: usize where I7 < 32); + static_assert!(I8: usize where I8 < 32); + static_assert!(I9: usize where I9 < 32); + static_assert!(I10: usize where I10 < 32); + static_assert!(I11: usize where I11 < 32); + static_assert!(I12: usize where I12 < 32); + static_assert!(I13: usize where I13 < 32); + static_assert!(I14: usize where I14 < 32); + static_assert!(I15: usize where I15 < 32); + let shuf: simd::u8x16 = unsafe { + simd_shuffle16!( + a.as_u8x16(), + b.as_u8x16(), + < + const I0: usize, + const I1: usize, + const I2: usize, + const I3: usize, + const I4: usize, + const I5: usize, + const I6: usize, + const I7: usize, + const I8: usize, + const I9: usize, + const I10: usize, + const I11: usize, + const I12: usize, + const I13: usize, + const I14: usize, + const I15: usize, + > [ + I0 as u32, I1 as u32, I2 as u32, I3 as u32, I4 as u32, I5 as u32, I6 as u32, I7 as u32, + I8 as u32, I9 as u32, I10 as u32, I11 as u32, I12 as u32, I13 as u32, I14 as u32, + I15 as u32, + ], + ) + }; + shuf.v128() +} + +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub use i8x16_shuffle as u8x16_shuffle; + +/// Same as [`i8x16_shuffle`], except operates as if the inputs were eight /// 16-bit integers, only taking 8 indices to shuffle. /// /// Indices in the range [0, 7] select from `a` while [8, 15] select from `b`. -/// Note that this will generate the `v8x16.shuffle` instruction, since there -/// is no native `v16x8.shuffle` instruction (there is no need for one since -/// `v8x16.shuffle` suffices). +/// Note that this will generate the `i8x16.shuffle` instruction, since there +/// is no native `i16x8.shuffle` instruction (there is no need for one since +/// `i8x16.shuffle` suffices). #[inline] +#[cfg_attr(test, + assert_instr( + i8x16.shuffle, + I0 = 0, + I1 = 2, + I2 = 4, + I3 = 6, + I4 = 8, + I5 = 10, + I6 = 12, + I7 = 14, + ) +)] #[target_feature(enable = "simd128")] -pub unsafe fn v16x8_shuffle< +#[doc(alias("i8x16.shuffle"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i16x8_shuffle< const I0: usize, const I1: usize, const I2: usize, @@ -498,360 +884,504 @@ pub unsafe fn v16x8_shuffle< a: v128, b: v128, ) -> v128 { - let shuf = simd_shuffle8::( - a.as_u16x8(), - b.as_u16x8(), - [ - I0 as u32, I1 as u32, I2 as u32, I3 as u32, I4 as u32, I5 as u32, I6 as u32, I7 as u32, - ], - ); - transmute(shuf) -} - -#[cfg(test)] -#[assert_instr(v8x16.shuffle)] -#[target_feature(enable = "simd128")] -unsafe fn v16x8_shuffle_test(a: v128, b: v128) -> v128 { - v16x8_shuffle::<0, 2, 4, 6, 8, 10, 12, 14>(a, b) -} - -/// Same as [`v8x16_shuffle`], except operates as if the inputs were four + static_assert!(I0: usize where I0 < 16); + static_assert!(I1: usize where I1 < 16); + static_assert!(I2: usize where I2 < 16); + static_assert!(I3: usize where I3 < 16); + static_assert!(I4: usize where I4 < 16); + static_assert!(I5: usize where I5 < 16); + static_assert!(I6: usize where I6 < 16); + static_assert!(I7: usize where I7 < 16); + let shuf: simd::u16x8 = unsafe { + simd_shuffle8!( + a.as_u16x8(), + b.as_u16x8(), + < + const I0: usize, + const I1: usize, + const I2: usize, + const I3: usize, + const I4: usize, + const I5: usize, + const I6: usize, + const I7: usize, + > [ + I0 as u32, I1 as u32, I2 as u32, I3 as u32, I4 as u32, I5 as u32, I6 as u32, I7 as u32, + ], + ) + }; + shuf.v128() +} + +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub use i16x8_shuffle as u16x8_shuffle; + +/// Same as [`i8x16_shuffle`], except operates as if the inputs were four /// 32-bit integers, only taking 4 indices to shuffle. /// /// Indices in the range [0, 3] select from `a` while [4, 7] select from `b`. -/// Note that this will generate the `v8x16.shuffle` instruction, since there -/// is no native `v32x4.shuffle` instruction (there is no need for one since -/// `v8x16.shuffle` suffices). +/// Note that this will generate the `i8x16.shuffle` instruction, since there +/// is no native `i32x4.shuffle` instruction (there is no need for one since +/// `i8x16.shuffle` suffices). #[inline] +#[cfg_attr(test, assert_instr(i8x16.shuffle, I0 = 0, I1 = 2, I2 = 4, I3 = 6))] #[target_feature(enable = "simd128")] -pub unsafe fn v32x4_shuffle( +#[doc(alias("i8x16.shuffle"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i32x4_shuffle( a: v128, b: v128, ) -> v128 { - let shuf = simd_shuffle4::( - a.as_u32x4(), - b.as_u32x4(), - [I0 as u32, I1 as u32, I2 as u32, I3 as u32], - ); - transmute(shuf) -} - -#[cfg(test)] -#[assert_instr(v8x16.shuffle)] -#[target_feature(enable = "simd128")] -unsafe fn v32x4_shuffle_test(a: v128, b: v128) -> v128 { - v32x4_shuffle::<0, 2, 4, 6>(a, b) -} - -/// Same as [`v8x16_shuffle`], except operates as if the inputs were two + static_assert!(I0: usize where I0 < 8); + static_assert!(I1: usize where I1 < 8); + static_assert!(I2: usize where I2 < 8); + static_assert!(I3: usize where I3 < 8); + let shuf: simd::u32x4 = unsafe { + simd_shuffle4!( + a.as_u32x4(), + b.as_u32x4(), + [I0 as u32, I1 as u32, I2 as u32, I3 as u32], + ) + }; + shuf.v128() +} + +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub use i32x4_shuffle as u32x4_shuffle; + +/// Same as [`i8x16_shuffle`], except operates as if the inputs were two /// 64-bit integers, only taking 2 indices to shuffle. /// /// Indices in the range [0, 1] select from `a` while [2, 3] select from `b`. /// Note that this will generate the `v8x16.shuffle` instruction, since there -/// is no native `v64x2.shuffle` instruction (there is no need for one since -/// `v8x16.shuffle` suffices). +/// is no native `i64x2.shuffle` instruction (there is no need for one since +/// `i8x16.shuffle` suffices). #[inline] +#[cfg_attr(test, assert_instr(i8x16.shuffle, I0 = 0, I1 = 2))] #[target_feature(enable = "simd128")] -pub unsafe fn v64x2_shuffle(a: v128, b: v128) -> v128 { - let shuf = simd_shuffle2::(a.as_u64x2(), b.as_u64x2(), [I0 as u32, I1 as u32]); - transmute(shuf) +#[doc(alias("i8x16.shuffle"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i64x2_shuffle(a: v128, b: v128) -> v128 { + static_assert!(I0: usize where I0 < 4); + static_assert!(I1: usize where I1 < 4); + let shuf: simd::u64x2 = unsafe { + simd_shuffle2!( + a.as_u64x2(), + b.as_u64x2(), + [I0 as u32, I1 as u32], + ) + }; + shuf.v128() } -#[cfg(test)] -#[assert_instr(v8x16.shuffle)] -#[target_feature(enable = "simd128")] -unsafe fn v64x2_shuffle_test(a: v128, b: v128) -> v128 { - v64x2_shuffle::<0, 2>(a, b) -} +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub use i64x2_shuffle as u64x2_shuffle; -/// Returns a new vector with lanes selected from the lanes of the first input -/// vector `a` specified in the second input vector `s`. +/// Extracts a lane from a 128-bit vector interpreted as 16 packed i8 numbers. /// -/// The indices `i` in range [0, 15] select the `i`-th element of `a`. For -/// indices outside of the range the resulting lane is 0. +/// Extracts the scalar value of lane specified in the immediate mode operand +/// `N` from `a`. If `N` is out of bounds then it is a compile time error. #[inline] -#[cfg_attr(test, assert_instr(v8x16.swizzle))] +#[cfg_attr(test, assert_instr(i8x16.extract_lane_s, N = 3))] #[target_feature(enable = "simd128")] -pub unsafe fn v8x16_swizzle(a: v128, s: v128) -> v128 { - transmute(llvm_swizzle(transmute(a), transmute(s))) +#[doc(alias("i8x16.extract_lane_s"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i8x16_extract_lane(a: v128) -> i8 { + static_assert!(N: usize where N < 16); + unsafe { simd_extract(a.as_i8x16(), N as u32) } } -/// Creates a vector with identical lanes. +/// Extracts a lane from a 128-bit vector interpreted as 16 packed u8 numbers. /// -/// Constructs a vector with `x` replicated to all 16 lanes. +/// Extracts the scalar value of lane specified in the immediate mode operand +/// `N` from `a`. If `N` is out of bounds then it is a compile time error. #[inline] -#[cfg_attr(test, assert_instr(i8x16.splat))] +#[cfg_attr(test, assert_instr(i8x16.extract_lane_u, N = 3))] #[target_feature(enable = "simd128")] -pub unsafe fn i8x16_splat(a: i8) -> v128 { - transmute(i8x16::splat(a)) +#[doc(alias("i8x16.extract_lane_u"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn u8x16_extract_lane(a: v128) -> u8 { + static_assert!(N: usize where N < 16); + unsafe { simd_extract(a.as_u8x16(), N as u32) } } -/// Creates a vector with identical lanes. +/// Replaces a lane from a 128-bit vector interpreted as 16 packed i8 numbers. /// -/// Construct a vector with `x` replicated to all 8 lanes. +/// Replaces the scalar value of lane specified in the immediate mode operand +/// `N` from `a`. If `N` is out of bounds then it is a compile time error. #[inline] -#[cfg_attr(test, assert_instr(i16x8.splat))] +#[cfg_attr(test, assert_instr(i8x16.replace_lane, N = 2))] #[target_feature(enable = "simd128")] -pub unsafe fn i16x8_splat(a: i16) -> v128 { - transmute(i16x8::splat(a)) +#[doc(alias("i8x16.replace_lane"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i8x16_replace_lane(a: v128, val: i8) -> v128 { + static_assert!(N: usize where N < 16); + unsafe { simd_insert(a.as_i8x16(), N as u32, val).v128() } } -/// Creates a vector with identical lanes. +/// Replaces a lane from a 128-bit vector interpreted as 16 packed u8 numbers. /// -/// Constructs a vector with `x` replicated to all 4 lanes. +/// Replaces the scalar value of lane specified in the immediate mode operand +/// `N` from `a`. If `N` is out of bounds then it is a compile time error. #[inline] -#[cfg_attr(test, assert_instr(i32x4.splat))] +#[cfg_attr(test, assert_instr(i8x16.replace_lane, N = 2))] #[target_feature(enable = "simd128")] -pub unsafe fn i32x4_splat(a: i32) -> v128 { - transmute(i32x4::splat(a)) +#[doc(alias("i8x16.replace_lane"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn u8x16_replace_lane(a: v128, val: u8) -> v128 { + static_assert!(N: usize where N < 16); + unsafe { simd_insert(a.as_u8x16(), N as u32, val).v128() } } -/// Creates a vector with identical lanes. +/// Extracts a lane from a 128-bit vector interpreted as 8 packed i16 numbers. /// -/// Construct a vector with `x` replicated to all 2 lanes. +/// Extracts a the scalar value of lane specified in the immediate mode operand +/// `N` from `a`. If `N` is out of bounds then it is a compile time error. #[inline] -#[cfg_attr(test, assert_instr(i64x2.splat))] +#[cfg_attr(test, assert_instr(i16x8.extract_lane_s, N = 2))] #[target_feature(enable = "simd128")] -pub unsafe fn i64x2_splat(a: i64) -> v128 { - transmute(i64x2::splat(a)) +#[doc(alias("i16x8.extract_lane_s"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i16x8_extract_lane(a: v128) -> i16 { + static_assert!(N: usize where N < 8); + unsafe { simd_extract(a.as_i16x8(), N as u32) } } -/// Creates a vector with identical lanes. +/// Extracts a lane from a 128-bit vector interpreted as 8 packed u16 numbers. /// -/// Constructs a vector with `x` replicated to all 4 lanes. +/// Extracts a the scalar value of lane specified in the immediate mode operand +/// `N` from `a`. If `N` is out of bounds then it is a compile time error. #[inline] -#[cfg_attr(test, assert_instr(f32x4.splat))] +#[cfg_attr(test, assert_instr(i16x8.extract_lane_u, N = 2))] #[target_feature(enable = "simd128")] -pub unsafe fn f32x4_splat(a: f32) -> v128 { - transmute(f32x4::splat(a)) +#[doc(alias("i16x8.extract_lane_u"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn u16x8_extract_lane(a: v128) -> u16 { + static_assert!(N: usize where N < 8); + unsafe { simd_extract(a.as_u16x8(), N as u32) } } -/// Creates a vector with identical lanes. +/// Replaces a lane from a 128-bit vector interpreted as 8 packed i16 numbers. /// -/// Constructs a vector with `x` replicated to all 2 lanes. +/// Replaces the scalar value of lane specified in the immediate mode operand +/// `N` from `a`. If `N` is out of bounds then it is a compile time error. #[inline] -#[cfg_attr(test, assert_instr(f64x2.splat))] +#[cfg_attr(test, assert_instr(i16x8.replace_lane, N = 2))] #[target_feature(enable = "simd128")] -pub unsafe fn f64x2_splat(a: f64) -> v128 { - transmute(f64x2::splat(a)) +#[doc(alias("i16x8.replace_lane"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i16x8_replace_lane(a: v128, val: i16) -> v128 { + static_assert!(N: usize where N < 8); + unsafe { simd_insert(a.as_i16x8(), N as u32, val).v128() } } -/// Extracts a lane from a 128-bit vector interpreted as 16 packed i8 numbers. +/// Replaces a lane from a 128-bit vector interpreted as 8 packed u16 numbers. /// -/// Extracts the scalar value of lane specified in the immediate mode operand +/// Replaces the scalar value of lane specified in the immediate mode operand /// `N` from `a`. If `N` is out of bounds then it is a compile time error. #[inline] +#[cfg_attr(test, assert_instr(i16x8.replace_lane, N = 2))] #[target_feature(enable = "simd128")] -pub unsafe fn i8x16_extract_lane(a: v128) -> i8 { - simd_extract(a.as_i8x16(), N as u32) +#[doc(alias("i16x8.replace_lane"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn u16x8_replace_lane(a: v128, val: u16) -> v128 { + static_assert!(N: usize where N < 8); + unsafe { simd_insert(a.as_u16x8(), N as u32, val).v128() } } -#[cfg(test)] -#[assert_instr(i8x16.extract_lane_s)] +/// Extracts a lane from a 128-bit vector interpreted as 4 packed i32 numbers. +/// +/// Extracts the scalar value of lane specified in the immediate mode operand +/// `N` from `a`. If `N` is out of bounds then it is a compile time error. +#[inline] +#[cfg_attr(test, assert_instr(i32x4.extract_lane, N = 2))] #[target_feature(enable = "simd128")] -unsafe fn i8x16_extract_lane_s(a: v128) -> i32 { - i8x16_extract_lane::<0>(a) as i32 +#[doc(alias("i32x4.extract_lane"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i32x4_extract_lane(a: v128) -> i32 { + static_assert!(N: usize where N < 4); + unsafe { simd_extract(a.as_i32x4(), N as u32) } } -#[cfg(test)] -#[assert_instr(i8x16.extract_lane_u)] +/// Extracts a lane from a 128-bit vector interpreted as 4 packed u32 numbers. +/// +/// Extracts the scalar value of lane specified in the immediate mode operand +/// `N` from `a`. If `N` is out of bounds then it is a compile time error. +#[inline] #[target_feature(enable = "simd128")] -unsafe fn i8x16_extract_lane_u(a: v128) -> u32 { - i8x16_extract_lane::<0>(a) as u8 as u32 +#[doc(alias("i32x4.extract_lane"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn u32x4_extract_lane(a: v128) -> u32 { + i32x4_extract_lane::(a) as u32 } -/// Replaces a lane from a 128-bit vector interpreted as 16 packed i8 numbers. +/// Replaces a lane from a 128-bit vector interpreted as 4 packed i32 numbers. /// /// Replaces the scalar value of lane specified in the immediate mode operand /// `N` from `a`. If `N` is out of bounds then it is a compile time error. #[inline] +#[cfg_attr(test, assert_instr(i32x4.replace_lane, N = 2))] #[target_feature(enable = "simd128")] -pub unsafe fn i8x16_replace_lane(a: v128, val: i8) -> v128 { - transmute(simd_insert(a.as_i8x16(), N as u32, val)) +#[doc(alias("i32x4.replace_lane"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i32x4_replace_lane(a: v128, val: i32) -> v128 { + static_assert!(N: usize where N < 4); + unsafe { simd_insert(a.as_i32x4(), N as u32, val).v128() } } -#[cfg(test)] -#[assert_instr(i8x16.replace_lane)] +/// Replaces a lane from a 128-bit vector interpreted as 4 packed u32 numbers. +/// +/// Replaces the scalar value of lane specified in the immediate mode operand +/// `N` from `a`. If `N` is out of bounds then it is a compile time error. +#[inline] #[target_feature(enable = "simd128")] -unsafe fn i8x16_replace_lane_test(a: v128, val: i8) -> v128 { - i8x16_replace_lane::<0>(a, val) +#[doc(alias("i32x4.replace_lane"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn u32x4_replace_lane(a: v128, val: u32) -> v128 { + i32x4_replace_lane::(a, val as i32) } -/// Extracts a lane from a 128-bit vector interpreted as 8 packed i16 numbers. +/// Extracts a lane from a 128-bit vector interpreted as 2 packed i64 numbers. /// -/// Extracts a the scalar value of lane specified in the immediate mode operand +/// Extracts the scalar value of lane specified in the immediate mode operand /// `N` from `a`. If `N` is out of bounds then it is a compile time error. #[inline] +#[cfg_attr(test, assert_instr(i64x2.extract_lane, N = 1))] #[target_feature(enable = "simd128")] -pub unsafe fn i16x8_extract_lane(a: v128) -> i16 { - simd_extract(a.as_i16x8(), N as u32) +#[doc(alias("i64x2.extract_lane"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i64x2_extract_lane(a: v128) -> i64 { + static_assert!(N: usize where N < 2); + unsafe { simd_extract(a.as_i64x2(), N as u32) } } -#[cfg(test)] -#[assert_instr(i16x8.extract_lane_s)] +/// Extracts a lane from a 128-bit vector interpreted as 2 packed u64 numbers. +/// +/// Extracts the scalar value of lane specified in the immediate mode operand +/// `N` from `a`. If `N` is out of bounds then it is a compile time error. +#[inline] #[target_feature(enable = "simd128")] -unsafe fn i16x8_extract_lane_s(a: v128) -> i32 { - i16x8_extract_lane::<0>(a) as i32 +#[doc(alias("i64x2.extract_lane"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn u64x2_extract_lane(a: v128) -> u64 { + i64x2_extract_lane::(a) as u64 } -#[cfg(test)] -#[assert_instr(i16x8.extract_lane_u)] +/// Replaces a lane from a 128-bit vector interpreted as 2 packed i64 numbers. +/// +/// Replaces the scalar value of lane specified in the immediate mode operand +/// `N` from `a`. If `N` is out of bounds then it is a compile time error. +#[inline] +#[cfg_attr(test, assert_instr(i64x2.replace_lane, N = 0))] #[target_feature(enable = "simd128")] -unsafe fn i16x8_extract_lane_u(a: v128) -> u32 { - i16x8_extract_lane::<0>(a) as u16 as u32 +#[doc(alias("i64x2.replace_lane"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i64x2_replace_lane(a: v128, val: i64) -> v128 { + static_assert!(N: usize where N < 2); + unsafe { simd_insert(a.as_i64x2(), N as u32, val).v128() } } -/// Replaces a lane from a 128-bit vector interpreted as 8 packed i16 numbers. +/// Replaces a lane from a 128-bit vector interpreted as 2 packed u64 numbers. /// /// Replaces the scalar value of lane specified in the immediate mode operand /// `N` from `a`. If `N` is out of bounds then it is a compile time error. #[inline] #[target_feature(enable = "simd128")] -pub unsafe fn i16x8_replace_lane(a: v128, val: i16) -> v128 { - transmute(simd_insert(a.as_i16x8(), N as u32, val)) +#[doc(alias("i64x2.replace_lane"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn u64x2_replace_lane(a: v128, val: u64) -> v128 { + i64x2_replace_lane::(a, val as i64) } -#[cfg(test)] -#[assert_instr(i16x8.replace_lane)] +/// Extracts a lane from a 128-bit vector interpreted as 4 packed f32 numbers. +/// +/// Extracts the scalar value of lane specified fn the immediate mode operand +/// `N` from `a`. If `N` is out of bounds then it is a compile time error. +#[inline] +#[cfg_attr(test, assert_instr(f32x4.extract_lane, N = 1))] #[target_feature(enable = "simd128")] -unsafe fn i16x8_replace_lane_test(a: v128, val: i16) -> v128 { - i16x8_replace_lane::<0>(a, val) +#[doc(alias("f32x4.extract_lane"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn f32x4_extract_lane(a: v128) -> f32 { + static_assert!(N: usize where N < 4); + unsafe { simd_extract(a.as_f32x4(), N as u32) } } -/// Extracts a lane from a 128-bit vector interpreted as 4 packed i32 numbers. +/// Replaces a lane from a 128-bit vector interpreted as 4 packed f32 numbers. /// -/// Extracts the scalar value of lane specified in the immediate mode operand +/// Replaces the scalar value of lane specified fn the immediate mode operand /// `N` from `a`. If `N` is out of bounds then it is a compile time error. #[inline] +#[cfg_attr(test, assert_instr(f32x4.replace_lane, N = 1))] #[target_feature(enable = "simd128")] -pub unsafe fn i32x4_extract_lane(a: v128) -> i32 { - simd_extract(a.as_i32x4(), N as u32) +#[doc(alias("f32x4.replace_lane"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn f32x4_replace_lane(a: v128, val: f32) -> v128 { + static_assert!(N: usize where N < 4); + unsafe { simd_insert(a.as_f32x4(), N as u32, val).v128() } } -#[cfg(test)] -#[assert_instr(i32x4.extract_lane)] +/// Extracts a lane from a 128-bit vector interpreted as 2 packed f64 numbers. +/// +/// Extracts the scalar value of lane specified fn the immediate mode operand +/// `N` from `a`. If `N` fs out of bounds then it is a compile time error. +#[inline] +#[cfg_attr(test, assert_instr(f64x2.extract_lane, N = 1))] #[target_feature(enable = "simd128")] -unsafe fn i32x4_extract_lane_test(a: v128) -> i32 { - i32x4_extract_lane::<0>(a) +#[doc(alias("f64x2.extract_lane"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn f64x2_extract_lane(a: v128) -> f64 { + static_assert!(N: usize where N < 2); + unsafe { simd_extract(a.as_f64x2(), N as u32) } } -/// Replaces a lane from a 128-bit vector interpreted as 4 packed i32 numbers. +/// Replaces a lane from a 128-bit vector interpreted as 2 packed f64 numbers. /// /// Replaces the scalar value of lane specified in the immediate mode operand /// `N` from `a`. If `N` is out of bounds then it is a compile time error. #[inline] +#[cfg_attr(test, assert_instr(f64x2.replace_lane, N = 1))] #[target_feature(enable = "simd128")] -pub unsafe fn i32x4_replace_lane(a: v128, val: i32) -> v128 { - transmute(simd_insert(a.as_i32x4(), N as u32, val)) -} - -#[cfg(test)] -#[assert_instr(i32x4.replace_lane)] -#[target_feature(enable = "simd128")] -unsafe fn i32x4_replace_lane_test(a: v128, val: i32) -> v128 { - i32x4_replace_lane::<0>(a, val) +#[doc(alias("f64x2.replace_lane"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn f64x2_replace_lane(a: v128, val: f64) -> v128 { + static_assert!(N: usize where N < 2); + unsafe { simd_insert(a.as_f64x2(), N as u32, val).v128() } } -/// Extracts a lane from a 128-bit vector interpreted as 2 packed i64 numbers. +/// Returns a new vector with lanes selected from the lanes of the first input +/// vector `a` specified in the second input vector `s`. /// -/// Extracts the scalar value of lane specified in the immediate mode operand -/// `N` from `a`. If `N` is out of bounds then it is a compile time error. +/// The indices `i` in range [0, 15] select the `i`-th element of `a`. For +/// indices outside of the range the resulting lane is 0. #[inline] +#[cfg_attr(test, assert_instr(i8x16.swizzle))] #[target_feature(enable = "simd128")] -pub unsafe fn i64x2_extract_lane(a: v128) -> i64 { - simd_extract(a.as_i64x2(), N as u32) +#[doc(alias("i8x16.swizzle"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i8x16_swizzle(a: v128, s: v128) -> v128 { + unsafe { llvm_swizzle(a.as_i8x16(), s.as_i8x16()).v128() } } -#[cfg(test)] -#[assert_instr(i64x2.extract_lane)] -#[target_feature(enable = "simd128")] -unsafe fn i64x2_extract_lane_test(a: v128) -> i64 { - i64x2_extract_lane::<0>(a) -} +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub use i8x16_swizzle as u8x16_swizzle; -/// Replaces a lane from a 128-bit vector interpreted as 2 packed i64 numbers. +/// Creates a vector with identical lanes. /// -/// Replaces the scalar value of lane specified in the immediate mode operand -/// `N` from `a`. If `N` is out of bounds then it is a compile time error. +/// Constructs a vector with `x` replicated to all 16 lanes. #[inline] +#[cfg_attr(test, assert_instr(i8x16.splat))] #[target_feature(enable = "simd128")] -pub unsafe fn i64x2_replace_lane(a: v128, val: i64) -> v128 { - transmute(simd_insert(a.as_i64x2(), N as u32, val)) +#[doc(alias("i8x16.splat"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i8x16_splat(a: i8) -> v128 { + simd::i8x16::splat(a).v128() } -#[cfg(test)] -#[assert_instr(i64x2.replace_lane)] +/// Creates a vector with identical lanes. +/// +/// Constructs a vector with `x` replicated to all 16 lanes. +#[inline] +#[cfg_attr(test, assert_instr(i8x16.splat))] #[target_feature(enable = "simd128")] -unsafe fn i64x2_replace_lane_test(a: v128, val: i64) -> v128 { - i64x2_replace_lane::<0>(a, val) +#[doc(alias("i8x16.splat"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn u8x16_splat(a: u8) -> v128 { + simd::u8x16::splat(a).v128() } -/// Extracts a lane from a 128-bit vector interpreted as 4 packed f32 numbers. +/// Creates a vector with identical lanes. /// -/// Extracts the scalar value of lane specified fn the immediate mode operand -/// `N` from `a`. If `N` is out of bounds then it is a compile time error. +/// Construct a vector with `x` replicated to all 8 lanes. #[inline] +#[cfg_attr(test, assert_instr(i16x8.splat))] #[target_feature(enable = "simd128")] -pub unsafe fn f32x4_extract_lane(a: v128) -> f32 { - simd_extract(a.as_f32x4(), N as u32) +#[doc(alias("i16x8.splat"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i16x8_splat(a: i16) -> v128 { + simd::i16x8::splat(a).v128() } -#[cfg(test)] -#[assert_instr(f32x4.extract_lane)] +/// Creates a vector with identical lanes. +/// +/// Construct a vector with `x` replicated to all 8 lanes. +#[inline] +#[cfg_attr(test, assert_instr(i16x8.splat))] #[target_feature(enable = "simd128")] -unsafe fn f32x4_extract_lane_test(a: v128) -> f32 { - f32x4_extract_lane::<0>(a) +#[doc(alias("i16x8.splat"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn u16x8_splat(a: u16) -> v128 { + simd::u16x8::splat(a).v128() } -/// Replaces a lane from a 128-bit vector interpreted as 4 packed f32 numbers. +/// Creates a vector with identical lanes. /// -/// Replaces the scalar value of lane specified fn the immediate mode operand -/// `N` from `a`. If `N` is out of bounds then it is a compile time error. +/// Constructs a vector with `x` replicated to all 4 lanes. #[inline] +#[cfg_attr(test, assert_instr(i32x4.splat))] #[target_feature(enable = "simd128")] -pub unsafe fn f32x4_replace_lane(a: v128, val: f32) -> v128 { - transmute(simd_insert(a.as_f32x4(), N as u32, val)) +#[doc(alias("i32x4.splat"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i32x4_splat(a: i32) -> v128 { + simd::i32x4::splat(a).v128() } -#[cfg(test)] -#[assert_instr(f32x4.replace_lane)] +/// Creates a vector with identical lanes. +/// +/// Constructs a vector with `x` replicated to all 4 lanes. +#[inline] #[target_feature(enable = "simd128")] -unsafe fn f32x4_replace_lane_test(a: v128, val: f32) -> v128 { - f32x4_replace_lane::<0>(a, val) +#[doc(alias("i32x4.splat"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn u32x4_splat(a: u32) -> v128 { + i32x4_splat(a as i32) } -/// Extracts a lane from a 128-bit vector interpreted as 2 packed f64 numbers. +/// Creates a vector with identical lanes. /// -/// Extracts the scalar value of lane specified fn the immediate mode operand -/// `N` from `a`. If `N` fs out of bounds then it is a compile time error. +/// Construct a vector with `x` replicated to all 2 lanes. #[inline] +#[cfg_attr(test, assert_instr(i64x2.splat))] #[target_feature(enable = "simd128")] -pub unsafe fn f64x2_extract_lane(a: v128) -> f64 { - simd_extract(a.as_f64x2(), N as u32) +#[doc(alias("i64x2.splat"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i64x2_splat(a: i64) -> v128 { + simd::i64x2::splat(a).v128() } -#[cfg(test)] -#[assert_instr(f64x2.extract_lane)] +/// Creates a vector with identical lanes. +/// +/// Construct a vector with `x` replicated to all 2 lanes. +#[inline] #[target_feature(enable = "simd128")] -unsafe fn f64x2_extract_lane_test(a: v128) -> f64 { - f64x2_extract_lane::<0>(a) +#[doc(alias("u64x2.splat"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn u64x2_splat(a: u64) -> v128 { + i64x2_splat(a as i64) } -/// Replaces a lane from a 128-bit vector interpreted as 2 packed f64 numbers. +/// Creates a vector with identical lanes. /// -/// Replaces the scalar value of lane specified in the immediate mode operand -/// `N` from `a`. If `N` is out of bounds then it is a compile time error. +/// Constructs a vector with `x` replicated to all 4 lanes. #[inline] +#[cfg_attr(test, assert_instr(f32x4.splat))] #[target_feature(enable = "simd128")] -pub unsafe fn f64x2_replace_lane(a: v128, val: f64) -> v128 { - transmute(simd_insert(a.as_f64x2(), N as u32, val)) +#[doc(alias("f32x4.splat"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn f32x4_splat(a: f32) -> v128 { + simd::f32x4::splat(a).v128() } -#[cfg(test)] -#[assert_instr(f64x2.replace_lane)] +/// Creates a vector with identical lanes. +/// +/// Constructs a vector with `x` replicated to all 2 lanes. +#[inline] +#[cfg_attr(test, assert_instr(f64x2.splat))] #[target_feature(enable = "simd128")] -unsafe fn f64x2_replace_lane_test(a: v128, val: f64) -> v128 { - f64x2_replace_lane::<0>(a, val) +#[doc(alias("f64x2.splat"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn f64x2_splat(a: f64) -> v128 { + simd::f64x2::splat(a).v128() } /// Compares two 128-bit vectors as if they were two vectors of 16 eight-bit @@ -862,8 +1392,10 @@ unsafe fn f64x2_replace_lane_test(a: v128, val: f64) -> v128 { #[inline] #[cfg_attr(test, assert_instr(i8x16.eq))] #[target_feature(enable = "simd128")] -pub unsafe fn i8x16_eq(a: v128, b: v128) -> v128 { - transmute(simd_eq::<_, i8x16>(a.as_i8x16(), b.as_i8x16())) +#[doc(alias("i8x16.eq"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i8x16_eq(a: v128, b: v128) -> v128 { + unsafe { simd_eq::<_, simd::i8x16>(a.as_i8x16(), b.as_i8x16()).v128() } } /// Compares two 128-bit vectors as if they were two vectors of 16 eight-bit @@ -874,10 +1406,17 @@ pub unsafe fn i8x16_eq(a: v128, b: v128) -> v128 { #[inline] #[cfg_attr(test, assert_instr(i8x16.ne))] #[target_feature(enable = "simd128")] -pub unsafe fn i8x16_ne(a: v128, b: v128) -> v128 { - transmute(simd_ne::<_, i8x16>(a.as_i8x16(), b.as_i8x16())) +#[doc(alias("i8x16.ne"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i8x16_ne(a: v128, b: v128) -> v128 { + unsafe { simd_ne::<_, simd::i8x16>(a.as_i8x16(), b.as_i8x16()).v128() } } +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub use i8x16_eq as u8x16_eq; +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub use i8x16_ne as u8x16_ne; + /// Compares two 128-bit vectors as if they were two vectors of 16 eight-bit /// signed integers. /// @@ -886,8 +1425,10 @@ pub unsafe fn i8x16_ne(a: v128, b: v128) -> v128 { #[inline] #[cfg_attr(test, assert_instr(i8x16.lt_s))] #[target_feature(enable = "simd128")] -pub unsafe fn i8x16_lt_s(a: v128, b: v128) -> v128 { - transmute(simd_lt::<_, i8x16>(a.as_i8x16(), b.as_i8x16())) +#[doc(alias("i8x16.lt_s"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i8x16_lt(a: v128, b: v128) -> v128 { + unsafe { simd_lt::<_, simd::i8x16>(a.as_i8x16(), b.as_i8x16()).v128() } } /// Compares two 128-bit vectors as if they were two vectors of 16 eight-bit @@ -898,8 +1439,10 @@ pub unsafe fn i8x16_lt_s(a: v128, b: v128) -> v128 { #[inline] #[cfg_attr(test, assert_instr(i8x16.lt_u))] #[target_feature(enable = "simd128")] -pub unsafe fn i8x16_lt_u(a: v128, b: v128) -> v128 { - transmute(simd_lt::<_, i8x16>(a.as_u8x16(), b.as_u8x16())) +#[doc(alias("i8x16.lt_u"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn u8x16_lt(a: v128, b: v128) -> v128 { + unsafe { simd_lt::<_, simd::i8x16>(a.as_u8x16(), b.as_u8x16()).v128() } } /// Compares two 128-bit vectors as if they were two vectors of 16 eight-bit @@ -910,8 +1453,10 @@ pub unsafe fn i8x16_lt_u(a: v128, b: v128) -> v128 { #[inline] #[cfg_attr(test, assert_instr(i8x16.gt_s))] #[target_feature(enable = "simd128")] -pub unsafe fn i8x16_gt_s(a: v128, b: v128) -> v128 { - transmute(simd_gt::<_, i8x16>(a.as_i8x16(), b.as_i8x16())) +#[doc(alias("i8x16.gt_s"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i8x16_gt(a: v128, b: v128) -> v128 { + unsafe { simd_gt::<_, simd::i8x16>(a.as_i8x16(), b.as_i8x16()).v128() } } /// Compares two 128-bit vectors as if they were two vectors of 16 eight-bit @@ -922,8 +1467,10 @@ pub unsafe fn i8x16_gt_s(a: v128, b: v128) -> v128 { #[inline] #[cfg_attr(test, assert_instr(i8x16.gt_u))] #[target_feature(enable = "simd128")] -pub unsafe fn i8x16_gt_u(a: v128, b: v128) -> v128 { - transmute(simd_gt::<_, i8x16>(a.as_u8x16(), b.as_u8x16())) +#[doc(alias("i8x16.gt_u"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn u8x16_gt(a: v128, b: v128) -> v128 { + unsafe { simd_gt::<_, simd::i8x16>(a.as_u8x16(), b.as_u8x16()).v128() } } /// Compares two 128-bit vectors as if they were two vectors of 16 eight-bit @@ -934,8 +1481,10 @@ pub unsafe fn i8x16_gt_u(a: v128, b: v128) -> v128 { #[inline] #[cfg_attr(test, assert_instr(i8x16.le_s))] #[target_feature(enable = "simd128")] -pub unsafe fn i8x16_le_s(a: v128, b: v128) -> v128 { - transmute(simd_le::<_, i8x16>(a.as_i8x16(), b.as_i8x16())) +#[doc(alias("i8x16.le_s"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i8x16_le(a: v128, b: v128) -> v128 { + unsafe { simd_le::<_, simd::i8x16>(a.as_i8x16(), b.as_i8x16()).v128() } } /// Compares two 128-bit vectors as if they were two vectors of 16 eight-bit @@ -946,8 +1495,10 @@ pub unsafe fn i8x16_le_s(a: v128, b: v128) -> v128 { #[inline] #[cfg_attr(test, assert_instr(i8x16.le_u))] #[target_feature(enable = "simd128")] -pub unsafe fn i8x16_le_u(a: v128, b: v128) -> v128 { - transmute(simd_le::<_, i8x16>(a.as_u8x16(), b.as_u8x16())) +#[doc(alias("i8x16.le_u"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn u8x16_le(a: v128, b: v128) -> v128 { + unsafe { simd_le::<_, simd::i8x16>(a.as_u8x16(), b.as_u8x16()).v128() } } /// Compares two 128-bit vectors as if they were two vectors of 16 eight-bit @@ -958,8 +1509,10 @@ pub unsafe fn i8x16_le_u(a: v128, b: v128) -> v128 { #[inline] #[cfg_attr(test, assert_instr(i8x16.ge_s))] #[target_feature(enable = "simd128")] -pub unsafe fn i8x16_ge_s(a: v128, b: v128) -> v128 { - transmute(simd_ge::<_, i8x16>(a.as_i8x16(), b.as_i8x16())) +#[doc(alias("i8x16.ge_s"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i8x16_ge(a: v128, b: v128) -> v128 { + unsafe { simd_ge::<_, simd::i8x16>(a.as_i8x16(), b.as_i8x16()).v128() } } /// Compares two 128-bit vectors as if they were two vectors of 16 eight-bit @@ -970,8 +1523,10 @@ pub unsafe fn i8x16_ge_s(a: v128, b: v128) -> v128 { #[inline] #[cfg_attr(test, assert_instr(i8x16.ge_u))] #[target_feature(enable = "simd128")] -pub unsafe fn i8x16_ge_u(a: v128, b: v128) -> v128 { - transmute(simd_ge::<_, i8x16>(a.as_u8x16(), b.as_u8x16())) +#[doc(alias("i8x16.ge_u"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn u8x16_ge(a: v128, b: v128) -> v128 { + unsafe { simd_ge::<_, simd::i8x16>(a.as_u8x16(), b.as_u8x16()).v128() } } /// Compares two 128-bit vectors as if they were two vectors of 8 sixteen-bit @@ -982,8 +1537,10 @@ pub unsafe fn i8x16_ge_u(a: v128, b: v128) -> v128 { #[inline] #[cfg_attr(test, assert_instr(i16x8.eq))] #[target_feature(enable = "simd128")] -pub unsafe fn i16x8_eq(a: v128, b: v128) -> v128 { - transmute(simd_eq::<_, i16x8>(a.as_i16x8(), b.as_i16x8())) +#[doc(alias("i16x8.eq"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i16x8_eq(a: v128, b: v128) -> v128 { + unsafe { simd_eq::<_, simd::i16x8>(a.as_i16x8(), b.as_i16x8()).v128() } } /// Compares two 128-bit vectors as if they were two vectors of 8 sixteen-bit @@ -994,10 +1551,17 @@ pub unsafe fn i16x8_eq(a: v128, b: v128) -> v128 { #[inline] #[cfg_attr(test, assert_instr(i16x8.ne))] #[target_feature(enable = "simd128")] -pub unsafe fn i16x8_ne(a: v128, b: v128) -> v128 { - transmute(simd_ne::<_, i16x8>(a.as_i16x8(), b.as_i16x8())) +#[doc(alias("i16x8.ne"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i16x8_ne(a: v128, b: v128) -> v128 { + unsafe { simd_ne::<_, simd::i16x8>(a.as_i16x8(), b.as_i16x8()).v128() } } +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub use i16x8_eq as u16x8_eq; +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub use i16x8_ne as u16x8_ne; + /// Compares two 128-bit vectors as if they were two vectors of 8 sixteen-bit /// signed integers. /// @@ -1006,8 +1570,10 @@ pub unsafe fn i16x8_ne(a: v128, b: v128) -> v128 { #[inline] #[cfg_attr(test, assert_instr(i16x8.lt_s))] #[target_feature(enable = "simd128")] -pub unsafe fn i16x8_lt_s(a: v128, b: v128) -> v128 { - transmute(simd_lt::<_, i16x8>(a.as_i16x8(), b.as_i16x8())) +#[doc(alias("i16x8.lt_s"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i16x8_lt(a: v128, b: v128) -> v128 { + unsafe { simd_lt::<_, simd::i16x8>(a.as_i16x8(), b.as_i16x8()).v128() } } /// Compares two 128-bit vectors as if they were two vectors of 8 sixteen-bit @@ -1018,8 +1584,10 @@ pub unsafe fn i16x8_lt_s(a: v128, b: v128) -> v128 { #[inline] #[cfg_attr(test, assert_instr(i16x8.lt_u))] #[target_feature(enable = "simd128")] -pub unsafe fn i16x8_lt_u(a: v128, b: v128) -> v128 { - transmute(simd_lt::<_, i16x8>(a.as_u16x8(), b.as_u16x8())) +#[doc(alias("i16x8.lt_u"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn u16x8_lt(a: v128, b: v128) -> v128 { + unsafe { simd_lt::<_, simd::i16x8>(a.as_u16x8(), b.as_u16x8()).v128() } } /// Compares two 128-bit vectors as if they were two vectors of 8 sixteen-bit @@ -1030,8 +1598,10 @@ pub unsafe fn i16x8_lt_u(a: v128, b: v128) -> v128 { #[inline] #[cfg_attr(test, assert_instr(i16x8.gt_s))] #[target_feature(enable = "simd128")] -pub unsafe fn i16x8_gt_s(a: v128, b: v128) -> v128 { - transmute(simd_gt::<_, i16x8>(a.as_i16x8(), b.as_i16x8())) +#[doc(alias("i16x8.gt_s"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i16x8_gt(a: v128, b: v128) -> v128 { + unsafe { simd_gt::<_, simd::i16x8>(a.as_i16x8(), b.as_i16x8()).v128() } } /// Compares two 128-bit vectors as if they were two vectors of 8 sixteen-bit @@ -1042,8 +1612,10 @@ pub unsafe fn i16x8_gt_s(a: v128, b: v128) -> v128 { #[inline] #[cfg_attr(test, assert_instr(i16x8.gt_u))] #[target_feature(enable = "simd128")] -pub unsafe fn i16x8_gt_u(a: v128, b: v128) -> v128 { - transmute(simd_gt::<_, i16x8>(a.as_u16x8(), b.as_u16x8())) +#[doc(alias("i16x8.gt_u"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn u16x8_gt(a: v128, b: v128) -> v128 { + unsafe { simd_gt::<_, simd::i16x8>(a.as_u16x8(), b.as_u16x8()).v128() } } /// Compares two 128-bit vectors as if they were two vectors of 8 sixteen-bit @@ -1054,8 +1626,10 @@ pub unsafe fn i16x8_gt_u(a: v128, b: v128) -> v128 { #[inline] #[cfg_attr(test, assert_instr(i16x8.le_s))] #[target_feature(enable = "simd128")] -pub unsafe fn i16x8_le_s(a: v128, b: v128) -> v128 { - transmute(simd_le::<_, i16x8>(a.as_i16x8(), b.as_i16x8())) +#[doc(alias("i16x8.le_s"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i16x8_le(a: v128, b: v128) -> v128 { + unsafe { simd_le::<_, simd::i16x8>(a.as_i16x8(), b.as_i16x8()).v128() } } /// Compares two 128-bit vectors as if they were two vectors of 8 sixteen-bit @@ -1066,8 +1640,10 @@ pub unsafe fn i16x8_le_s(a: v128, b: v128) -> v128 { #[inline] #[cfg_attr(test, assert_instr(i16x8.le_u))] #[target_feature(enable = "simd128")] -pub unsafe fn i16x8_le_u(a: v128, b: v128) -> v128 { - transmute(simd_le::<_, i16x8>(a.as_u16x8(), b.as_u16x8())) +#[doc(alias("i16x8.le_u"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn u16x8_le(a: v128, b: v128) -> v128 { + unsafe { simd_le::<_, simd::i16x8>(a.as_u16x8(), b.as_u16x8()).v128() } } /// Compares two 128-bit vectors as if they were two vectors of 8 sixteen-bit @@ -1078,8 +1654,10 @@ pub unsafe fn i16x8_le_u(a: v128, b: v128) -> v128 { #[inline] #[cfg_attr(test, assert_instr(i16x8.ge_s))] #[target_feature(enable = "simd128")] -pub unsafe fn i16x8_ge_s(a: v128, b: v128) -> v128 { - transmute(simd_ge::<_, i16x8>(a.as_i16x8(), b.as_i16x8())) +#[doc(alias("i16x8.ge_s"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i16x8_ge(a: v128, b: v128) -> v128 { + unsafe { simd_ge::<_, simd::i16x8>(a.as_i16x8(), b.as_i16x8()).v128() } } /// Compares two 128-bit vectors as if they were two vectors of 8 sixteen-bit @@ -1090,8 +1668,10 @@ pub unsafe fn i16x8_ge_s(a: v128, b: v128) -> v128 { #[inline] #[cfg_attr(test, assert_instr(i16x8.ge_u))] #[target_feature(enable = "simd128")] -pub unsafe fn i16x8_ge_u(a: v128, b: v128) -> v128 { - transmute(simd_ge::<_, i16x8>(a.as_u16x8(), b.as_u16x8())) +#[doc(alias("i16x8.ge_u"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn u16x8_ge(a: v128, b: v128) -> v128 { + unsafe { simd_ge::<_, simd::i16x8>(a.as_u16x8(), b.as_u16x8()).v128() } } /// Compares two 128-bit vectors as if they were two vectors of 4 thirty-two-bit @@ -1102,8 +1682,10 @@ pub unsafe fn i16x8_ge_u(a: v128, b: v128) -> v128 { #[inline] #[cfg_attr(test, assert_instr(i32x4.eq))] #[target_feature(enable = "simd128")] -pub unsafe fn i32x4_eq(a: v128, b: v128) -> v128 { - transmute(simd_eq::<_, i32x4>(a.as_i32x4(), b.as_i32x4())) +#[doc(alias("i32x4.eq"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i32x4_eq(a: v128, b: v128) -> v128 { + unsafe { simd_eq::<_, simd::i32x4>(a.as_i32x4(), b.as_i32x4()).v128() } } /// Compares two 128-bit vectors as if they were two vectors of 4 thirty-two-bit @@ -1114,10 +1696,17 @@ pub unsafe fn i32x4_eq(a: v128, b: v128) -> v128 { #[inline] #[cfg_attr(test, assert_instr(i32x4.ne))] #[target_feature(enable = "simd128")] -pub unsafe fn i32x4_ne(a: v128, b: v128) -> v128 { - transmute(simd_ne::<_, i32x4>(a.as_i32x4(), b.as_i32x4())) +#[doc(alias("i32x4.ne"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i32x4_ne(a: v128, b: v128) -> v128 { + unsafe { simd_ne::<_, simd::i32x4>(a.as_i32x4(), b.as_i32x4()).v128() } } +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub use i32x4_eq as u32x4_eq; +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub use i32x4_ne as u32x4_ne; + /// Compares two 128-bit vectors as if they were two vectors of 4 thirty-two-bit /// signed integers. /// @@ -1126,8 +1715,10 @@ pub unsafe fn i32x4_ne(a: v128, b: v128) -> v128 { #[inline] #[cfg_attr(test, assert_instr(i32x4.lt_s))] #[target_feature(enable = "simd128")] -pub unsafe fn i32x4_lt_s(a: v128, b: v128) -> v128 { - transmute(simd_lt::<_, i32x4>(a.as_i32x4(), b.as_i32x4())) +#[doc(alias("i32x4.lt_s"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i32x4_lt(a: v128, b: v128) -> v128 { + unsafe { simd_lt::<_, simd::i32x4>(a.as_i32x4(), b.as_i32x4()).v128() } } /// Compares two 128-bit vectors as if they were two vectors of 4 thirty-two-bit @@ -1138,8 +1729,10 @@ pub unsafe fn i32x4_lt_s(a: v128, b: v128) -> v128 { #[inline] #[cfg_attr(test, assert_instr(i32x4.lt_u))] #[target_feature(enable = "simd128")] -pub unsafe fn i32x4_lt_u(a: v128, b: v128) -> v128 { - transmute(simd_lt::<_, i32x4>(a.as_u32x4(), b.as_u32x4())) +#[doc(alias("i32x4.lt_u"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn u32x4_lt(a: v128, b: v128) -> v128 { + unsafe { simd_lt::<_, simd::i32x4>(a.as_u32x4(), b.as_u32x4()).v128() } } /// Compares two 128-bit vectors as if they were two vectors of 4 thirty-two-bit @@ -1150,68 +1743,169 @@ pub unsafe fn i32x4_lt_u(a: v128, b: v128) -> v128 { #[inline] #[cfg_attr(test, assert_instr(i32x4.gt_s))] #[target_feature(enable = "simd128")] -pub unsafe fn i32x4_gt_s(a: v128, b: v128) -> v128 { - transmute(simd_gt::<_, i32x4>(a.as_i32x4(), b.as_i32x4())) +#[doc(alias("i32x4.gt_s"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i32x4_gt(a: v128, b: v128) -> v128 { + unsafe { simd_gt::<_, simd::i32x4>(a.as_i32x4(), b.as_i32x4()).v128() } +} + +/// Compares two 128-bit vectors as if they were two vectors of 4 thirty-two-bit +/// unsigned integers. +/// +/// Returns a new vector where each lane is all ones if the pairwise left +/// element is greater than the pairwise right element, or all zeros otherwise. +#[inline] +#[cfg_attr(test, assert_instr(i32x4.gt_u))] +#[target_feature(enable = "simd128")] +#[doc(alias("i32x4.gt_u"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn u32x4_gt(a: v128, b: v128) -> v128 { + unsafe { simd_gt::<_, simd::i32x4>(a.as_u32x4(), b.as_u32x4()).v128() } +} + +/// Compares two 128-bit vectors as if they were two vectors of 4 thirty-two-bit +/// signed integers. +/// +/// Returns a new vector where each lane is all ones if the pairwise left +/// element is less than the pairwise right element, or all zeros otherwise. +#[inline] +#[cfg_attr(test, assert_instr(i32x4.le_s))] +#[target_feature(enable = "simd128")] +#[doc(alias("i32x4.le_s"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i32x4_le(a: v128, b: v128) -> v128 { + unsafe { simd_le::<_, simd::i32x4>(a.as_i32x4(), b.as_i32x4()).v128() } } /// Compares two 128-bit vectors as if they were two vectors of 4 thirty-two-bit /// unsigned integers. /// -/// Returns a new vector where each lane is all ones if the pairwise left -/// element is greater than the pairwise right element, or all zeros otherwise. +/// Returns a new vector where each lane is all ones if the pairwise left +/// element is less than the pairwise right element, or all zeros otherwise. +#[inline] +#[cfg_attr(test, assert_instr(i32x4.le_u))] +#[target_feature(enable = "simd128")] +#[doc(alias("i32x4.le_u"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn u32x4_le(a: v128, b: v128) -> v128 { + unsafe { simd_le::<_, simd::i32x4>(a.as_u32x4(), b.as_u32x4()).v128() } +} + +/// Compares two 128-bit vectors as if they were two vectors of 4 thirty-two-bit +/// signed integers. +/// +/// Returns a new vector where each lane is all ones if the pairwise left +/// element is greater than the pairwise right element, or all zeros otherwise. +#[inline] +#[cfg_attr(test, assert_instr(i32x4.ge_s))] +#[target_feature(enable = "simd128")] +#[doc(alias("i32x4.ge_s"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i32x4_ge(a: v128, b: v128) -> v128 { + unsafe { simd_ge::<_, simd::i32x4>(a.as_i32x4(), b.as_i32x4()).v128() } +} + +/// Compares two 128-bit vectors as if they were two vectors of 4 thirty-two-bit +/// unsigned integers. +/// +/// Returns a new vector where each lane is all ones if the pairwise left +/// element is greater than the pairwise right element, or all zeros otherwise. +#[inline] +#[cfg_attr(test, assert_instr(i32x4.ge_u))] +#[target_feature(enable = "simd128")] +#[doc(alias("i32x4.ge_u"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn u32x4_ge(a: v128, b: v128) -> v128 { + unsafe { simd_ge::<_, simd::i32x4>(a.as_u32x4(), b.as_u32x4()).v128() } +} + +/// Compares two 128-bit vectors as if they were two vectors of 2 sixty-four-bit +/// integers. +/// +/// Returns a new vector where each lane is all ones if the pairwise elements +/// were equal, or all zeros if the elements were not equal. +#[inline] +#[cfg_attr(test, assert_instr(i64x2.eq))] +#[target_feature(enable = "simd128")] +#[doc(alias("i64x2.eq"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i64x2_eq(a: v128, b: v128) -> v128 { + unsafe { simd_eq::<_, simd::i64x2>(a.as_i64x2(), b.as_i64x2()).v128() } +} + +/// Compares two 128-bit vectors as if they were two vectors of 2 sixty-four-bit +/// integers. +/// +/// Returns a new vector where each lane is all ones if the pairwise elements +/// were not equal, or all zeros if the elements were equal. #[inline] -#[cfg_attr(test, assert_instr(i32x4.gt_u))] +#[cfg_attr(test, assert_instr(i64x2.ne))] #[target_feature(enable = "simd128")] -pub unsafe fn i32x4_gt_u(a: v128, b: v128) -> v128 { - transmute(simd_gt::<_, i32x4>(a.as_u32x4(), b.as_u32x4())) +#[doc(alias("i64x2.ne"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i64x2_ne(a: v128, b: v128) -> v128 { + unsafe { simd_ne::<_, simd::i64x2>(a.as_i64x2(), b.as_i64x2()).v128() } } -/// Compares two 128-bit vectors as if they were two vectors of 4 thirty-two-bit +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub use i64x2_eq as u64x2_eq; +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub use i64x2_ne as u64x2_ne; + +/// Compares two 128-bit vectors as if they were two vectors of 2 sixty-four-bit /// signed integers. /// /// Returns a new vector where each lane is all ones if the pairwise left /// element is less than the pairwise right element, or all zeros otherwise. #[inline] -#[cfg_attr(test, assert_instr(i32x4.le_s))] +#[cfg_attr(test, assert_instr(i64x2.lt_s))] #[target_feature(enable = "simd128")] -pub unsafe fn i32x4_le_s(a: v128, b: v128) -> v128 { - transmute(simd_le::<_, i32x4>(a.as_i32x4(), b.as_i32x4())) +#[doc(alias("i64x2.lt_s"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i64x2_lt(a: v128, b: v128) -> v128 { + unsafe { simd_lt::<_, simd::i64x2>(a.as_i64x2(), b.as_i64x2()).v128() } } -/// Compares two 128-bit vectors as if they were two vectors of 4 thirty-two-bit -/// unsigned integers. +/// Compares two 128-bit vectors as if they were two vectors of 2 sixty-four-bit +/// signed integers. /// /// Returns a new vector where each lane is all ones if the pairwise left -/// element is less than the pairwise right element, or all zeros otherwise. +/// element is greater than the pairwise right element, or all zeros otherwise. #[inline] -#[cfg_attr(test, assert_instr(i32x4.le_u))] +#[cfg_attr(test, assert_instr(i64x2.gt_s))] #[target_feature(enable = "simd128")] -pub unsafe fn i32x4_le_u(a: v128, b: v128) -> v128 { - transmute(simd_le::<_, i32x4>(a.as_u32x4(), b.as_u32x4())) +#[doc(alias("i64x2.gt_s"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i64x2_gt(a: v128, b: v128) -> v128 { + unsafe { simd_gt::<_, simd::i64x2>(a.as_i64x2(), b.as_i64x2()).v128() } } -/// Compares two 128-bit vectors as if they were two vectors of 4 thirty-two-bit +/// Compares two 128-bit vectors as if they were two vectors of 2 sixty-four-bit /// signed integers. /// /// Returns a new vector where each lane is all ones if the pairwise left -/// element is greater than the pairwise right element, or all zeros otherwise. +/// element is less than the pairwise right element, or all zeros otherwise. #[inline] -#[cfg_attr(test, assert_instr(i32x4.ge_s))] +#[cfg_attr(test, assert_instr(i64x2.le_s))] #[target_feature(enable = "simd128")] -pub unsafe fn i32x4_ge_s(a: v128, b: v128) -> v128 { - transmute(simd_ge::<_, i32x4>(a.as_i32x4(), b.as_i32x4())) +#[doc(alias("i64x2.le_s"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i64x2_le(a: v128, b: v128) -> v128 { + unsafe { simd_le::<_, simd::i64x2>(a.as_i64x2(), b.as_i64x2()).v128() } } -/// Compares two 128-bit vectors as if they were two vectors of 4 thirty-two-bit -/// unsigned integers. +/// Compares two 128-bit vectors as if they were two vectors of 2 sixty-four-bit +/// signed integers. /// /// Returns a new vector where each lane is all ones if the pairwise left /// element is greater than the pairwise right element, or all zeros otherwise. #[inline] -#[cfg_attr(test, assert_instr(i32x4.ge_u))] +#[cfg_attr(test, assert_instr(i64x2.ge_s))] #[target_feature(enable = "simd128")] -pub unsafe fn i32x4_ge_u(a: v128, b: v128) -> v128 { - transmute(simd_ge::<_, i32x4>(a.as_u32x4(), b.as_u32x4())) +#[doc(alias("i64x2.ge_s"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i64x2_ge(a: v128, b: v128) -> v128 { + unsafe { simd_ge::<_, simd::i64x2>(a.as_i64x2(), b.as_i64x2()).v128() } } /// Compares two 128-bit vectors as if they were two vectors of 4 thirty-two-bit @@ -1222,8 +1916,10 @@ pub unsafe fn i32x4_ge_u(a: v128, b: v128) -> v128 { #[inline] #[cfg_attr(test, assert_instr(f32x4.eq))] #[target_feature(enable = "simd128")] -pub unsafe fn f32x4_eq(a: v128, b: v128) -> v128 { - transmute(simd_eq::<_, i32x4>(a.as_f32x4(), b.as_f32x4())) +#[doc(alias("f32x4.eq"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn f32x4_eq(a: v128, b: v128) -> v128 { + unsafe { simd_eq::<_, simd::i32x4>(a.as_f32x4(), b.as_f32x4()).v128() } } /// Compares two 128-bit vectors as if they were two vectors of 4 thirty-two-bit @@ -1234,8 +1930,10 @@ pub unsafe fn f32x4_eq(a: v128, b: v128) -> v128 { #[inline] #[cfg_attr(test, assert_instr(f32x4.ne))] #[target_feature(enable = "simd128")] -pub unsafe fn f32x4_ne(a: v128, b: v128) -> v128 { - transmute(simd_ne::<_, i32x4>(a.as_f32x4(), b.as_f32x4())) +#[doc(alias("f32x4.ne"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn f32x4_ne(a: v128, b: v128) -> v128 { + unsafe { simd_ne::<_, simd::i32x4>(a.as_f32x4(), b.as_f32x4()).v128() } } /// Compares two 128-bit vectors as if they were two vectors of 4 thirty-two-bit @@ -1246,8 +1944,10 @@ pub unsafe fn f32x4_ne(a: v128, b: v128) -> v128 { #[inline] #[cfg_attr(test, assert_instr(f32x4.lt))] #[target_feature(enable = "simd128")] -pub unsafe fn f32x4_lt(a: v128, b: v128) -> v128 { - transmute(simd_lt::<_, i32x4>(a.as_f32x4(), b.as_f32x4())) +#[doc(alias("f32x4.lt"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn f32x4_lt(a: v128, b: v128) -> v128 { + unsafe { simd_lt::<_, simd::i32x4>(a.as_f32x4(), b.as_f32x4()).v128() } } /// Compares two 128-bit vectors as if they were two vectors of 4 thirty-two-bit @@ -1258,8 +1958,10 @@ pub unsafe fn f32x4_lt(a: v128, b: v128) -> v128 { #[inline] #[cfg_attr(test, assert_instr(f32x4.gt))] #[target_feature(enable = "simd128")] -pub unsafe fn f32x4_gt(a: v128, b: v128) -> v128 { - transmute(simd_gt::<_, i32x4>(a.as_f32x4(), b.as_f32x4())) +#[doc(alias("f32x4.gt"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn f32x4_gt(a: v128, b: v128) -> v128 { + unsafe { simd_gt::<_, simd::i32x4>(a.as_f32x4(), b.as_f32x4()).v128() } } /// Compares two 128-bit vectors as if they were two vectors of 4 thirty-two-bit @@ -1270,8 +1972,10 @@ pub unsafe fn f32x4_gt(a: v128, b: v128) -> v128 { #[inline] #[cfg_attr(test, assert_instr(f32x4.le))] #[target_feature(enable = "simd128")] -pub unsafe fn f32x4_le(a: v128, b: v128) -> v128 { - transmute(simd_le::<_, i32x4>(a.as_f32x4(), b.as_f32x4())) +#[doc(alias("f32x4.le"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn f32x4_le(a: v128, b: v128) -> v128 { + unsafe { simd_le::<_, simd::i32x4>(a.as_f32x4(), b.as_f32x4()).v128() } } /// Compares two 128-bit vectors as if they were two vectors of 4 thirty-two-bit @@ -1282,8 +1986,10 @@ pub unsafe fn f32x4_le(a: v128, b: v128) -> v128 { #[inline] #[cfg_attr(test, assert_instr(f32x4.ge))] #[target_feature(enable = "simd128")] -pub unsafe fn f32x4_ge(a: v128, b: v128) -> v128 { - transmute(simd_ge::<_, i32x4>(a.as_f32x4(), b.as_f32x4())) +#[doc(alias("f32x4.ge"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn f32x4_ge(a: v128, b: v128) -> v128 { + unsafe { simd_ge::<_, simd::i32x4>(a.as_f32x4(), b.as_f32x4()).v128() } } /// Compares two 128-bit vectors as if they were two vectors of 2 sixty-four-bit @@ -1294,8 +2000,10 @@ pub unsafe fn f32x4_ge(a: v128, b: v128) -> v128 { #[inline] #[cfg_attr(test, assert_instr(f64x2.eq))] #[target_feature(enable = "simd128")] -pub unsafe fn f64x2_eq(a: v128, b: v128) -> v128 { - transmute(simd_eq::<_, i64x2>(a.as_f64x2(), b.as_f64x2())) +#[doc(alias("f64x2.eq"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn f64x2_eq(a: v128, b: v128) -> v128 { + unsafe { simd_eq::<_, simd::i64x2>(a.as_f64x2(), b.as_f64x2()).v128() } } /// Compares two 128-bit vectors as if they were two vectors of 2 sixty-four-bit @@ -1306,8 +2014,10 @@ pub unsafe fn f64x2_eq(a: v128, b: v128) -> v128 { #[inline] #[cfg_attr(test, assert_instr(f64x2.ne))] #[target_feature(enable = "simd128")] -pub unsafe fn f64x2_ne(a: v128, b: v128) -> v128 { - transmute(simd_ne::<_, i64x2>(a.as_f64x2(), b.as_f64x2())) +#[doc(alias("f64x2.ne"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn f64x2_ne(a: v128, b: v128) -> v128 { + unsafe { simd_ne::<_, simd::i64x2>(a.as_f64x2(), b.as_f64x2()).v128() } } /// Compares two 128-bit vectors as if they were two vectors of 2 sixty-four-bit @@ -1318,8 +2028,10 @@ pub unsafe fn f64x2_ne(a: v128, b: v128) -> v128 { #[inline] #[cfg_attr(test, assert_instr(f64x2.lt))] #[target_feature(enable = "simd128")] -pub unsafe fn f64x2_lt(a: v128, b: v128) -> v128 { - transmute(simd_lt::<_, i64x2>(a.as_f64x2(), b.as_f64x2())) +#[doc(alias("f64x2.lt"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn f64x2_lt(a: v128, b: v128) -> v128 { + unsafe { simd_lt::<_, simd::i64x2>(a.as_f64x2(), b.as_f64x2()).v128() } } /// Compares two 128-bit vectors as if they were two vectors of 2 sixty-four-bit @@ -1330,8 +2042,10 @@ pub unsafe fn f64x2_lt(a: v128, b: v128) -> v128 { #[inline] #[cfg_attr(test, assert_instr(f64x2.gt))] #[target_feature(enable = "simd128")] -pub unsafe fn f64x2_gt(a: v128, b: v128) -> v128 { - transmute(simd_gt::<_, i64x2>(a.as_f64x2(), b.as_f64x2())) +#[doc(alias("f64x2.gt"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn f64x2_gt(a: v128, b: v128) -> v128 { + unsafe { simd_gt::<_, simd::i64x2>(a.as_f64x2(), b.as_f64x2()).v128() } } /// Compares two 128-bit vectors as if they were two vectors of 2 sixty-four-bit @@ -1342,8 +2056,10 @@ pub unsafe fn f64x2_gt(a: v128, b: v128) -> v128 { #[inline] #[cfg_attr(test, assert_instr(f64x2.le))] #[target_feature(enable = "simd128")] -pub unsafe fn f64x2_le(a: v128, b: v128) -> v128 { - transmute(simd_le::<_, i64x2>(a.as_f64x2(), b.as_f64x2())) +#[doc(alias("f64x2.le"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn f64x2_le(a: v128, b: v128) -> v128 { + unsafe { simd_le::<_, simd::i64x2>(a.as_f64x2(), b.as_f64x2()).v128() } } /// Compares two 128-bit vectors as if they were two vectors of 2 sixty-four-bit @@ -1354,16 +2070,20 @@ pub unsafe fn f64x2_le(a: v128, b: v128) -> v128 { #[inline] #[cfg_attr(test, assert_instr(f64x2.ge))] #[target_feature(enable = "simd128")] -pub unsafe fn f64x2_ge(a: v128, b: v128) -> v128 { - transmute(simd_ge::<_, i64x2>(a.as_f64x2(), b.as_f64x2())) +#[doc(alias("f64x2.ge"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn f64x2_ge(a: v128, b: v128) -> v128 { + unsafe { simd_ge::<_, simd::i64x2>(a.as_f64x2(), b.as_f64x2()).v128() } } /// Flips each bit of the 128-bit input vector. #[inline] #[cfg_attr(test, assert_instr(v128.not))] #[target_feature(enable = "simd128")] -pub unsafe fn v128_not(a: v128) -> v128 { - transmute(simd_xor(a.as_i64x2(), i64x2(!0, !0))) +#[doc(alias("v128.not"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn v128_not(a: v128) -> v128 { + unsafe { simd_xor(a.as_i64x2(), simd::i64x2(!0, !0)).v128() } } /// Performs a bitwise and of the two input 128-bit vectors, returning the @@ -1371,21 +2091,22 @@ pub unsafe fn v128_not(a: v128) -> v128 { #[inline] #[cfg_attr(test, assert_instr(v128.and))] #[target_feature(enable = "simd128")] -pub unsafe fn v128_and(a: v128, b: v128) -> v128 { - transmute(simd_and(a.as_i64x2(), b.as_i64x2())) +#[doc(alias("v128.and"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn v128_and(a: v128, b: v128) -> v128 { + unsafe { simd_and(a.as_i64x2(), b.as_i64x2()).v128() } } /// Bitwise AND of bits of `a` and the logical inverse of bits of `b`. /// /// This operation is equivalent to `v128.and(a, v128.not(b))` #[inline] -#[cfg_attr(all(test, all_simd), assert_instr(v128.andnot))] +#[cfg_attr(test, assert_instr(v128.andnot))] #[target_feature(enable = "simd128")] -pub unsafe fn v128_andnot(a: v128, b: v128) -> v128 { - transmute(simd_and( - a.as_i64x2(), - simd_xor(b.as_i64x2(), i64x2(-1, -1)), - )) +#[doc(alias("v128.andnot"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn v128_andnot(a: v128, b: v128) -> v128 { + unsafe { simd_and(a.as_i64x2(), simd_xor(b.as_i64x2(), simd::i64x2(-1, -1))).v128() } } /// Performs a bitwise or of the two input 128-bit vectors, returning the @@ -1393,8 +2114,10 @@ pub unsafe fn v128_andnot(a: v128, b: v128) -> v128 { #[inline] #[cfg_attr(test, assert_instr(v128.or))] #[target_feature(enable = "simd128")] -pub unsafe fn v128_or(a: v128, b: v128) -> v128 { - transmute(simd_or(a.as_i64x2(), b.as_i64x2())) +#[doc(alias("v128.or"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn v128_or(a: v128, b: v128) -> v128 { + unsafe { simd_or(a.as_i64x2(), b.as_i64x2()).v128() } } /// Performs a bitwise xor of the two input 128-bit vectors, returning the @@ -1402,64 +2125,99 @@ pub unsafe fn v128_or(a: v128, b: v128) -> v128 { #[inline] #[cfg_attr(test, assert_instr(v128.xor))] #[target_feature(enable = "simd128")] -pub unsafe fn v128_xor(a: v128, b: v128) -> v128 { - transmute(simd_xor(a.as_i64x2(), b.as_i64x2())) +#[doc(alias("v128.xor"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn v128_xor(a: v128, b: v128) -> v128 { + unsafe { simd_xor(a.as_i64x2(), b.as_i64x2()).v128() } } /// Use the bitmask in `c` to select bits from `v1` when 1 and `v2` when 0. #[inline] #[cfg_attr(test, assert_instr(v128.bitselect))] #[target_feature(enable = "simd128")] -pub unsafe fn v128_bitselect(v1: v128, v2: v128, c: v128) -> v128 { - transmute(llvm_bitselect(v1.as_i8x16(), v2.as_i8x16(), c.as_i8x16())) +#[doc(alias("v128.bitselect"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn v128_bitselect(v1: v128, v2: v128, c: v128) -> v128 { + unsafe { llvm_bitselect(v1.as_i8x16(), v2.as_i8x16(), c.as_i8x16()).v128() } +} + +/// Returns `true` if any bit in `a` is set, or `false` otherwise. +#[inline] +#[cfg_attr(test, assert_instr(v128.any_true))] +#[target_feature(enable = "simd128")] +#[doc(alias("v128.any_true"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn v128_any_true(a: v128) -> bool { + unsafe { llvm_any_true_i8x16(a.as_i8x16()) != 0 } } /// Lane-wise wrapping absolute value. #[inline] -// #[cfg_attr(test, assert_instr(i8x16.abs))] // FIXME support not in our LLVM yet +#[cfg_attr(test, assert_instr(i8x16.abs))] #[target_feature(enable = "simd128")] -pub unsafe fn i8x16_abs(a: v128) -> v128 { - let a = transmute::<_, i8x16>(a); - let zero = i8x16::splat(0); - transmute(simd_select::( - simd_lt(a, zero), - simd_sub(zero, a), - a, - )) +#[doc(alias("i8x16.abs"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i8x16_abs(a: v128) -> v128 { + unsafe { + let a = a.as_i8x16(); + let zero = simd::i8x16::splat(0); + simd_select::(simd_lt(a, zero), simd_sub(zero, a), a).v128() + } } /// Negates a 128-bit vectors intepreted as sixteen 8-bit signed integers #[inline] #[cfg_attr(test, assert_instr(i8x16.neg))] #[target_feature(enable = "simd128")] -pub unsafe fn i8x16_neg(a: v128) -> v128 { - transmute(simd_mul(a.as_i8x16(), i8x16::splat(-1))) +#[doc(alias("i8x16.neg"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i8x16_neg(a: v128) -> v128 { + unsafe { simd_mul(a.as_i8x16(), simd::i8x16::splat(-1)).v128() } } -/// Returns 1 if any lane is nonzero or 0 if all lanes are zero. +/// Count the number of bits set to one within each lane. #[inline] -#[cfg_attr(test, assert_instr(i8x16.any_true))] +// #[cfg_attr(test, assert_instr(i8x16.popcnt))] // FIXME wasmtime #[target_feature(enable = "simd128")] -pub unsafe fn i8x16_any_true(a: v128) -> i32 { - llvm_i8x16_any_true(a.as_i8x16()) +#[doc(alias("i8x16.popcnt"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i8x16_popcnt(v: v128) -> v128 { + unsafe { llvm_popcnt(v.as_i8x16()).v128() } } -/// Returns 1 if all lanes are nonzero or 0 if any lane is nonzero. +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub use i8x16_popcnt as u8x16_popcnt; + +/// Returns true if all lanes are nonzero or false if any lane is nonzero. #[inline] #[cfg_attr(test, assert_instr(i8x16.all_true))] #[target_feature(enable = "simd128")] -pub unsafe fn i8x16_all_true(a: v128) -> i32 { - llvm_i8x16_all_true(a.as_i8x16()) +#[doc(alias("i8x16.all_true"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i8x16_all_true(a: v128) -> bool { + unsafe { llvm_i8x16_all_true(a.as_i8x16()) != 0 } +} + +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub use i8x16_all_true as u8x16_all_true; + +/// Extracts the high bit for each lane in `a` and produce a scalar mask with +/// all bits concatenated. +#[inline] +#[cfg_attr(test, assert_instr(i8x16.bitmask))] +#[target_feature(enable = "simd128")] +#[doc(alias("i8x16.bitmask"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i8x16_bitmask(a: v128) -> u16 { + // FIXME(https://bugs.llvm.org/show_bug.cgi?id=50507) - this produces an + // extraneous `i32.and` instruction against a mask of 65535 when converting + // from the native intrinsic's i32 return value to our desired u16. This + // shouldn't be necessary, though, but requires upstream LLVM changes. + unsafe { llvm_bitmask_i8x16(a.as_i8x16()) as u16 } } -// FIXME: not available in our LLVM yet -// /// Extracts the high bit for each lane in `a` and produce a scalar mask with -// /// all bits concatenated. -// #[inline] -// #[cfg_attr(test, assert_instr(i8x16.all_true))] -// pub unsafe fn i8x16_bitmask(a: v128) -> i32 { -// llvm_bitmask_i8x16(transmute(a)) -// } +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub use i8x16_bitmask as u8x16_bitmask; /// Converts two input vectors into a smaller lane vector by narrowing each /// lane. @@ -1469,8 +2227,10 @@ pub unsafe fn i8x16_all_true(a: v128) -> i32 { #[inline] #[cfg_attr(test, assert_instr(i8x16.narrow_i16x8_s))] #[target_feature(enable = "simd128")] -pub unsafe fn i8x16_narrow_i16x8_s(a: v128, b: v128) -> v128 { - transmute(llvm_narrow_i8x16_s(transmute(a), transmute(b))) +#[doc(alias("i8x16.narrow_i16x8_s"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i8x16_narrow_i16x8(a: v128, b: v128) -> v128 { + unsafe { llvm_narrow_i8x16_s(a.as_i16x8(), b.as_i16x8()).v128() } } /// Converts two input vectors into a smaller lane vector by narrowing each @@ -1481,8 +2241,10 @@ pub unsafe fn i8x16_narrow_i16x8_s(a: v128, b: v128) -> v128 { #[inline] #[cfg_attr(test, assert_instr(i8x16.narrow_i16x8_u))] #[target_feature(enable = "simd128")] -pub unsafe fn i8x16_narrow_i16x8_u(a: v128, b: v128) -> v128 { - transmute(llvm_narrow_i8x16_u(transmute(a), transmute(b))) +#[doc(alias("i8x16.narrow_i16x8_u"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn u8x16_narrow_i16x8(a: v128, b: v128) -> v128 { + unsafe { llvm_narrow_i8x16_u(a.as_i16x8(), b.as_i16x8()).v128() } } /// Shifts each lane to the left by the specified number of bits. @@ -1492,10 +2254,15 @@ pub unsafe fn i8x16_narrow_i16x8_u(a: v128, b: v128) -> v128 { #[inline] #[cfg_attr(test, assert_instr(i8x16.shl))] #[target_feature(enable = "simd128")] -pub unsafe fn i8x16_shl(a: v128, amt: u32) -> v128 { - transmute(simd_shl(a.as_i8x16(), i8x16::splat(amt as i8))) +#[doc(alias("i8x16.shl"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i8x16_shl(a: v128, amt: u32) -> v128 { + unsafe { simd_shl(a.as_i8x16(), simd::i8x16::splat(amt as i8)).v128() } } +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub use i8x16_shl as u8x16_shl; + /// Shifts each lane to the right by the specified number of bits, sign /// extending. /// @@ -1504,8 +2271,10 @@ pub unsafe fn i8x16_shl(a: v128, amt: u32) -> v128 { #[inline] #[cfg_attr(test, assert_instr(i8x16.shr_s))] #[target_feature(enable = "simd128")] -pub unsafe fn i8x16_shr_s(a: v128, amt: u32) -> v128 { - transmute(simd_shr(a.as_i8x16(), i8x16::splat(amt as i8))) +#[doc(alias("i8x16.shr_s"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i8x16_shr(a: v128, amt: u32) -> v128 { + unsafe { simd_shr(a.as_i8x16(), simd::i8x16::splat(amt as i8)).v128() } } /// Shifts each lane to the right by the specified number of bits, shifting in @@ -1516,60 +2285,80 @@ pub unsafe fn i8x16_shr_s(a: v128, amt: u32) -> v128 { #[inline] #[cfg_attr(test, assert_instr(i8x16.shr_u))] #[target_feature(enable = "simd128")] -pub unsafe fn i8x16_shr_u(a: v128, amt: u32) -> v128 { - transmute(simd_shr(a.as_u8x16(), u8x16::splat(amt as u8))) +#[doc(alias("i8x16.shr_u"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn u8x16_shr(a: v128, amt: u32) -> v128 { + unsafe { simd_shr(a.as_u8x16(), simd::u8x16::splat(amt as u8)).v128() } } /// Adds two 128-bit vectors as if they were two packed sixteen 8-bit integers. #[inline] #[cfg_attr(test, assert_instr(i8x16.add))] #[target_feature(enable = "simd128")] -pub unsafe fn i8x16_add(a: v128, b: v128) -> v128 { - transmute(simd_add(a.as_i8x16(), b.as_i8x16())) +#[doc(alias("i8x16.add"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i8x16_add(a: v128, b: v128) -> v128 { + unsafe { simd_add(a.as_i8x16(), b.as_i8x16()).v128() } } +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub use i8x16_add as u8x16_add; + /// Adds two 128-bit vectors as if they were two packed sixteen 8-bit signed /// integers, saturating on overflow to `i8::MAX`. #[inline] -#[cfg_attr(test, assert_instr(i8x16.add_saturate_s))] +#[cfg_attr(test, assert_instr(i8x16.add_sat_s))] #[target_feature(enable = "simd128")] -pub unsafe fn i8x16_add_saturate_s(a: v128, b: v128) -> v128 { - transmute(llvm_i8x16_add_saturate_s(a.as_i8x16(), b.as_i8x16())) +#[doc(alias("i8x16.add_sat_s"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i8x16_add_sat(a: v128, b: v128) -> v128 { + unsafe { llvm_i8x16_add_sat_s(a.as_i8x16(), b.as_i8x16()).v128() } } /// Adds two 128-bit vectors as if they were two packed sixteen 8-bit unsigned /// integers, saturating on overflow to `u8::MAX`. #[inline] -#[cfg_attr(test, assert_instr(i8x16.add_saturate_u))] +#[cfg_attr(test, assert_instr(i8x16.add_sat_u))] #[target_feature(enable = "simd128")] -pub unsafe fn i8x16_add_saturate_u(a: v128, b: v128) -> v128 { - transmute(llvm_i8x16_add_saturate_u(a.as_i8x16(), b.as_i8x16())) +#[doc(alias("i8x16.add_sat_u"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn u8x16_add_sat(a: v128, b: v128) -> v128 { + unsafe { llvm_i8x16_add_sat_u(a.as_i8x16(), b.as_i8x16()).v128() } } /// Subtracts two 128-bit vectors as if they were two packed sixteen 8-bit integers. #[inline] #[cfg_attr(test, assert_instr(i8x16.sub))] #[target_feature(enable = "simd128")] -pub unsafe fn i8x16_sub(a: v128, b: v128) -> v128 { - transmute(simd_sub(a.as_i8x16(), b.as_i8x16())) +#[doc(alias("i8x16.sub"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i8x16_sub(a: v128, b: v128) -> v128 { + unsafe { simd_sub(a.as_i8x16(), b.as_i8x16()).v128() } } +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub use i8x16_sub as u8x16_sub; + /// Subtracts two 128-bit vectors as if they were two packed sixteen 8-bit /// signed integers, saturating on overflow to `i8::MIN`. #[inline] -#[cfg_attr(test, assert_instr(i8x16.sub_saturate_s))] +#[cfg_attr(test, assert_instr(i8x16.sub_sat_s))] #[target_feature(enable = "simd128")] -pub unsafe fn i8x16_sub_saturate_s(a: v128, b: v128) -> v128 { - transmute(llvm_i8x16_sub_saturate_s(a.as_i8x16(), b.as_i8x16())) +#[doc(alias("i8x16.sub_sat_s"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i8x16_sub_sat(a: v128, b: v128) -> v128 { + unsafe { llvm_i8x16_sub_sat_s(a.as_i8x16(), b.as_i8x16()).v128() } } /// Subtracts two 128-bit vectors as if they were two packed sixteen 8-bit /// unsigned integers, saturating on overflow to 0. #[inline] -#[cfg_attr(test, assert_instr(i8x16.sub_saturate_u))] +#[cfg_attr(test, assert_instr(i8x16.sub_sat_u))] #[target_feature(enable = "simd128")] -pub unsafe fn i8x16_sub_saturate_u(a: v128, b: v128) -> v128 { - transmute(llvm_i8x16_sub_saturate_u(a.as_i8x16(), b.as_i8x16())) +#[doc(alias("i8x16.sub_sat_u"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn u8x16_sub_sat(a: v128, b: v128) -> v128 { + unsafe { llvm_i8x16_sub_sat_u(a.as_i8x16(), b.as_i8x16()).v128() } } /// Compares lane-wise signed integers, and returns the minimum of @@ -1577,10 +2366,12 @@ pub unsafe fn i8x16_sub_saturate_u(a: v128, b: v128) -> v128 { #[inline] #[cfg_attr(test, assert_instr(i8x16.min_s))] #[target_feature(enable = "simd128")] -pub unsafe fn i8x16_min_s(a: v128, b: v128) -> v128 { +#[doc(alias("i8x16.min_s"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i8x16_min(a: v128, b: v128) -> v128 { let a = a.as_i8x16(); let b = b.as_i8x16(); - transmute(simd_select::(simd_lt(a, b), a, b)) + unsafe { simd_select::(simd_lt(a, b), a, b).v128() } } /// Compares lane-wise unsigned integers, and returns the minimum of @@ -1588,10 +2379,12 @@ pub unsafe fn i8x16_min_s(a: v128, b: v128) -> v128 { #[inline] #[cfg_attr(test, assert_instr(i8x16.min_u))] #[target_feature(enable = "simd128")] -pub unsafe fn i8x16_min_u(a: v128, b: v128) -> v128 { - let a = transmute::<_, u8x16>(a); - let b = transmute::<_, u8x16>(b); - transmute(simd_select::(simd_lt(a, b), a, b)) +#[doc(alias("i8x16.min_u"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn u8x16_min(a: v128, b: v128) -> v128 { + let a = a.as_u8x16(); + let b = b.as_u8x16(); + unsafe { simd_select::(simd_lt(a, b), a, b).v128() } } /// Compares lane-wise signed integers, and returns the maximum of @@ -1599,10 +2392,12 @@ pub unsafe fn i8x16_min_u(a: v128, b: v128) -> v128 { #[inline] #[cfg_attr(test, assert_instr(i8x16.max_s))] #[target_feature(enable = "simd128")] -pub unsafe fn i8x16_max_s(a: v128, b: v128) -> v128 { - let a = transmute::<_, i8x16>(a); - let b = transmute::<_, i8x16>(b); - transmute(simd_select::(simd_gt(a, b), a, b)) +#[doc(alias("i8x16.max_s"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i8x16_max(a: v128, b: v128) -> v128 { + let a = a.as_i8x16(); + let b = b.as_i8x16(); + unsafe { simd_select::(simd_gt(a, b), a, b).v128() } } /// Compares lane-wise unsigned integers, and returns the maximum of @@ -1610,66 +2405,109 @@ pub unsafe fn i8x16_max_s(a: v128, b: v128) -> v128 { #[inline] #[cfg_attr(test, assert_instr(i8x16.max_u))] #[target_feature(enable = "simd128")] -pub unsafe fn i8x16_max_u(a: v128, b: v128) -> v128 { - let a = transmute::<_, u8x16>(a); - let b = transmute::<_, u8x16>(b); - transmute(simd_select::(simd_gt(a, b), a, b)) +#[doc(alias("i8x16.max_u"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn u8x16_max(a: v128, b: v128) -> v128 { + let a = a.as_u8x16(); + let b = b.as_u8x16(); + unsafe { simd_select::(simd_gt(a, b), a, b).v128() } } /// Lane-wise rounding average. #[inline] #[cfg_attr(test, assert_instr(i8x16.avgr_u))] #[target_feature(enable = "simd128")] -pub unsafe fn i8x16_avgr_u(a: v128, b: v128) -> v128 { - transmute(llvm_avgr_u_i8x16(transmute(a), transmute(b))) +#[doc(alias("i8x16.avgr_u"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn u8x16_avgr(a: v128, b: v128) -> v128 { + unsafe { llvm_avgr_u_i8x16(a.as_i8x16(), b.as_i8x16()).v128() } +} + +/// Lane-wise integer extended pairwise addition producing extended results +/// (twice wider results than the inputs). +#[inline] +// #[cfg_attr(test, assert_instr(i16x8.extadd_pairwise_i8x16_s))] // FIXME wasmtime +#[target_feature(enable = "simd128")] +#[doc(alias("i16x8.extadd_pairwise_i8x16_s"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i16x8_extadd_pairwise_i8x16(a: v128) -> v128 { + unsafe { llvm_i16x8_extadd_pairwise_i8x16_s(a.as_i8x16()).v128() } +} + +/// Lane-wise integer extended pairwise addition producing extended results +/// (twice wider results than the inputs). +#[inline] +// #[cfg_attr(test, assert_instr(i16x8.extadd_pairwise_i8x16_u))] // FIXME wasmtime +#[target_feature(enable = "simd128")] +#[doc(alias("i16x8.extadd_pairwise_i8x16_u"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i16x8_extadd_pairwise_u8x16(a: v128) -> v128 { + unsafe { llvm_i16x8_extadd_pairwise_i8x16_u(a.as_i8x16()).v128() } } +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub use i16x8_extadd_pairwise_u8x16 as u16x8_extadd_pairwise_u8x16; + /// Lane-wise wrapping absolute value. #[inline] -// #[cfg_attr(test, assert_instr(i16x8.abs))] // FIXME support not in our LLVM yet +#[cfg_attr(test, assert_instr(i16x8.abs))] #[target_feature(enable = "simd128")] -pub unsafe fn i16x8_abs(a: v128) -> v128 { - let a = transmute::<_, i16x8>(a); - let zero = i16x8::splat(0); - transmute(simd_select::( - simd_lt(a, zero), - simd_sub(zero, a), - a, - )) +#[doc(alias("i16x8.abs"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i16x8_abs(a: v128) -> v128 { + let a = a.as_i16x8(); + let zero = simd::i16x8::splat(0); + unsafe { + simd_select::(simd_lt(a, zero), simd_sub(zero, a), a).v128() + } } /// Negates a 128-bit vectors intepreted as eight 16-bit signed integers #[inline] #[cfg_attr(test, assert_instr(i16x8.neg))] #[target_feature(enable = "simd128")] -pub unsafe fn i16x8_neg(a: v128) -> v128 { - transmute(simd_mul(a.as_i16x8(), i16x8::splat(-1))) +#[doc(alias("i16x8.neg"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i16x8_neg(a: v128) -> v128 { + unsafe { simd_mul(a.as_i16x8(), simd::i16x8::splat(-1)).v128() } } -/// Returns 1 if any lane is nonzero or 0 if all lanes are zero. +/// Lane-wise saturating rounding multiplication in Q15 format. #[inline] -#[cfg_attr(test, assert_instr(i16x8.any_true))] +// #[cfg_attr(test, assert_instr(i16x8.qmulr_sat_s))] // FIXME wasmtime #[target_feature(enable = "simd128")] -pub unsafe fn i16x8_any_true(a: v128) -> i32 { - llvm_i16x8_any_true(a.as_i16x8()) +#[doc(alias("i16x8.q15mulr_sat_s"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i16x8_q15mulr_sat(a: v128, b: v128) -> v128 { + unsafe { llvm_q15mulr(a.as_i16x8(), b.as_i16x8()).v128() } } /// Returns 1 if all lanes are nonzero or 0 if any lane is nonzero. #[inline] #[cfg_attr(test, assert_instr(i16x8.all_true))] #[target_feature(enable = "simd128")] -pub unsafe fn i16x8_all_true(a: v128) -> i32 { - llvm_i16x8_all_true(a.as_i16x8()) +#[doc(alias("i16x8.all_true"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i16x8_all_true(a: v128) -> bool { + unsafe { llvm_i16x8_all_true(a.as_i16x8()) != 0 } +} + +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub use i16x8_all_true as u16x8_all_true; + +/// Extracts the high bit for each lane in `a` and produce a scalar mask with +/// all bits concatenated. +#[inline] +#[cfg_attr(test, assert_instr(i16x8.bitmask))] +#[target_feature(enable = "simd128")] +#[doc(alias("i16x8.bitmask"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i16x8_bitmask(a: v128) -> u8 { + unsafe { llvm_bitmask_i16x8(a.as_i16x8()) as u8 } } -// FIXME: not available in our LLVM yet -// /// Extracts the high bit for each lane in `a` and produce a scalar mask with -// /// all bits concatenated. -// #[inline] -// #[cfg_attr(test, assert_instr(i16x8.all_true))] -// pub unsafe fn i16x8_bitmask(a: v128) -> i32 { -// llvm_bitmask_i16x8(transmute(a)) -// } +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub use i16x8_bitmask as u16x8_bitmask; /// Converts two input vectors into a smaller lane vector by narrowing each /// lane. @@ -1679,8 +2517,10 @@ pub unsafe fn i16x8_all_true(a: v128) -> i32 { #[inline] #[cfg_attr(test, assert_instr(i16x8.narrow_i32x4_s))] #[target_feature(enable = "simd128")] -pub unsafe fn i16x8_narrow_i32x4_s(a: v128, b: v128) -> v128 { - transmute(llvm_narrow_i16x8_s(transmute(a), transmute(b))) +#[doc(alias("i16x8.narrow_i32x4_s"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i16x8_narrow_i32x4(a: v128, b: v128) -> v128 { + unsafe { llvm_narrow_i16x8_s(a.as_i32x4(), b.as_i32x4()).v128() } } /// Converts two input vectors into a smaller lane vector by narrowing each @@ -1691,63 +2531,520 @@ pub unsafe fn i16x8_narrow_i32x4_s(a: v128, b: v128) -> v128 { #[inline] #[cfg_attr(test, assert_instr(i16x8.narrow_i32x4_u))] #[target_feature(enable = "simd128")] -pub unsafe fn i16x8_narrow_i32x4_u(a: v128, b: v128) -> v128 { - transmute(llvm_narrow_i16x8_u(transmute(a), transmute(b))) +#[doc(alias("i16x8.narrow_i32x4_u"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn u16x8_narrow_i32x4(a: v128, b: v128) -> v128 { + unsafe { llvm_narrow_i16x8_u(a.as_i32x4(), b.as_i32x4()).v128() } +} + +/// Converts low half of the smaller lane vector to a larger lane +/// vector, sign extended. +#[inline] +#[cfg_attr(test, assert_instr(i16x8.extend_low_i8x16_s))] +#[target_feature(enable = "simd128")] +#[doc(alias("i16x8.extend_low_i8x16_s"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i16x8_extend_low_i8x16(a: v128) -> v128 { + unsafe { + simd_cast::(simd_shuffle8!( + a.as_i8x16(), + a.as_i8x16(), + [0, 1, 2, 3, 4, 5, 6, 7], + )) + .v128() + } +} + +/// Converts high half of the smaller lane vector to a larger lane +/// vector, sign extended. +#[inline] +#[cfg_attr(test, assert_instr(i16x8.extend_high_i8x16_s))] +#[target_feature(enable = "simd128")] +#[doc(alias("i16x8.extend_high_i8x16_s"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i16x8_extend_high_i8x16(a: v128) -> v128 { + unsafe { + simd_cast::(simd_shuffle8!( + a.as_i8x16(), + a.as_i8x16(), + [8, 9, 10, 11, 12, 13, 14, 15], + )) + .v128() + } +} + +/// Converts low half of the smaller lane vector to a larger lane +/// vector, zero extended. +#[inline] +#[cfg_attr(test, assert_instr(i16x8.extend_low_i8x16_u))] +#[target_feature(enable = "simd128")] +#[doc(alias("i16x8.extend_low_i8x16_u"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i16x8_extend_low_u8x16(a: v128) -> v128 { + unsafe { + simd_cast::(simd_shuffle8!( + a.as_u8x16(), + a.as_u8x16(), + [0, 1, 2, 3, 4, 5, 6, 7], + )) + .v128() + } +} + +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub use i16x8_extend_low_u8x16 as u16x8_extend_low_u8x16; + +/// Converts high half of the smaller lane vector to a larger lane +/// vector, zero extended. +#[inline] +#[cfg_attr(test, assert_instr(i16x8.extend_high_i8x16_u))] +#[target_feature(enable = "simd128")] +#[doc(alias("i16x8.extend_high_i8x16_u"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i16x8_extend_high_u8x16(a: v128) -> v128 { + unsafe { + simd_cast::(simd_shuffle8!( + a.as_u8x16(), + a.as_u8x16(), + [8, 9, 10, 11, 12, 13, 14, 15], + )) + .v128() + } +} + +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub use i16x8_extend_high_u8x16 as u16x8_extend_high_u8x16; + +/// Shifts each lane to the left by the specified number of bits. +/// +/// Only the low bits of the shift amount are used if the shift amount is +/// greater than the lane width. +#[inline] +#[cfg_attr(test, assert_instr(i16x8.shl))] +#[target_feature(enable = "simd128")] +#[doc(alias("i16x8.shl"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i16x8_shl(a: v128, amt: u32) -> v128 { + unsafe { simd_shl(a.as_i16x8(), simd::i16x8::splat(amt as i16)).v128() } +} + +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub use i16x8_shl as u16x8_shl; + +/// Shifts each lane to the right by the specified number of bits, sign +/// extending. +/// +/// Only the low bits of the shift amount are used if the shift amount is +/// greater than the lane width. +#[inline] +#[cfg_attr(test, assert_instr(i16x8.shr_s))] +#[target_feature(enable = "simd128")] +#[doc(alias("i16x8.shr_s"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i16x8_shr(a: v128, amt: u32) -> v128 { + unsafe { simd_shr(a.as_i16x8(), simd::i16x8::splat(amt as i16)).v128() } +} + +/// Shifts each lane to the right by the specified number of bits, shifting in +/// zeros. +/// +/// Only the low bits of the shift amount are used if the shift amount is +/// greater than the lane width. +#[inline] +#[cfg_attr(test, assert_instr(i16x8.shr_u))] +#[target_feature(enable = "simd128")] +#[doc(alias("i16x8.shr_u"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn u16x8_shr(a: v128, amt: u32) -> v128 { + unsafe { simd_shr(a.as_u16x8(), simd::u16x8::splat(amt as u16)).v128() } +} + +/// Adds two 128-bit vectors as if they were two packed eight 16-bit integers. +#[inline] +#[cfg_attr(test, assert_instr(i16x8.add))] +#[target_feature(enable = "simd128")] +#[doc(alias("i16x8.add"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i16x8_add(a: v128, b: v128) -> v128 { + unsafe { simd_add(a.as_i16x8(), b.as_i16x8()).v128() } +} + +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub use i16x8_add as u16x8_add; + +/// Adds two 128-bit vectors as if they were two packed eight 16-bit signed +/// integers, saturating on overflow to `i16::MAX`. +#[inline] +#[cfg_attr(test, assert_instr(i16x8.add_sat_s))] +#[target_feature(enable = "simd128")] +#[doc(alias("i16x8.add_sat_s"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i16x8_add_sat(a: v128, b: v128) -> v128 { + unsafe { llvm_i16x8_add_sat_s(a.as_i16x8(), b.as_i16x8()).v128() } +} + +/// Adds two 128-bit vectors as if they were two packed eight 16-bit unsigned +/// integers, saturating on overflow to `u16::MAX`. +#[inline] +#[cfg_attr(test, assert_instr(i16x8.add_sat_u))] +#[target_feature(enable = "simd128")] +#[doc(alias("i16x8.add_sat_u"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn u16x8_add_sat(a: v128, b: v128) -> v128 { + unsafe { llvm_i16x8_add_sat_u(a.as_i16x8(), b.as_i16x8()).v128() } +} + +/// Subtracts two 128-bit vectors as if they were two packed eight 16-bit integers. +#[inline] +#[cfg_attr(test, assert_instr(i16x8.sub))] +#[target_feature(enable = "simd128")] +#[doc(alias("i16x8.sub"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i16x8_sub(a: v128, b: v128) -> v128 { + unsafe { simd_sub(a.as_i16x8(), b.as_i16x8()).v128() } +} + +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub use i16x8_sub as u16x8_sub; + +/// Subtracts two 128-bit vectors as if they were two packed eight 16-bit +/// signed integers, saturating on overflow to `i16::MIN`. +#[inline] +#[cfg_attr(test, assert_instr(i16x8.sub_sat_s))] +#[target_feature(enable = "simd128")] +#[doc(alias("i16x8.sub_sat_s"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i16x8_sub_sat(a: v128, b: v128) -> v128 { + unsafe { llvm_i16x8_sub_sat_s(a.as_i16x8(), b.as_i16x8()).v128() } +} + +/// Subtracts two 128-bit vectors as if they were two packed eight 16-bit +/// unsigned integers, saturating on overflow to 0. +#[inline] +#[cfg_attr(test, assert_instr(i16x8.sub_sat_u))] +#[target_feature(enable = "simd128")] +#[doc(alias("i16x8.sub_sat_u"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn u16x8_sub_sat(a: v128, b: v128) -> v128 { + unsafe { llvm_i16x8_sub_sat_u(a.as_i16x8(), b.as_i16x8()).v128() } +} + +/// Multiplies two 128-bit vectors as if they were two packed eight 16-bit +/// signed integers. +#[inline] +#[cfg_attr(test, assert_instr(i16x8.mul))] +#[target_feature(enable = "simd128")] +#[doc(alias("i16x8.mul"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i16x8_mul(a: v128, b: v128) -> v128 { + unsafe { simd_mul(a.as_i16x8(), b.as_i16x8()).v128() } +} + +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub use i16x8_mul as u16x8_mul; + +/// Compares lane-wise signed integers, and returns the minimum of +/// each pair. +#[inline] +#[cfg_attr(test, assert_instr(i16x8.min_s))] +#[target_feature(enable = "simd128")] +#[doc(alias("i16x8.min_s"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i16x8_min(a: v128, b: v128) -> v128 { + let a = a.as_i16x8(); + let b = b.as_i16x8(); + unsafe { simd_select::(simd_lt(a, b), a, b).v128() } +} + +/// Compares lane-wise unsigned integers, and returns the minimum of +/// each pair. +#[inline] +#[cfg_attr(test, assert_instr(i16x8.min_u))] +#[target_feature(enable = "simd128")] +#[doc(alias("i16x8.min_u"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn u16x8_min(a: v128, b: v128) -> v128 { + let a = a.as_u16x8(); + let b = b.as_u16x8(); + unsafe { simd_select::(simd_lt(a, b), a, b).v128() } +} + +/// Compares lane-wise signed integers, and returns the maximum of +/// each pair. +#[inline] +#[cfg_attr(test, assert_instr(i16x8.max_s))] +#[target_feature(enable = "simd128")] +#[doc(alias("i16x8.max_s"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i16x8_max(a: v128, b: v128) -> v128 { + let a = a.as_i16x8(); + let b = b.as_i16x8(); + unsafe { simd_select::(simd_gt(a, b), a, b).v128() } +} + +/// Compares lane-wise unsigned integers, and returns the maximum of +/// each pair. +#[inline] +#[cfg_attr(test, assert_instr(i16x8.max_u))] +#[target_feature(enable = "simd128")] +#[doc(alias("i16x8.max_u"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn u16x8_max(a: v128, b: v128) -> v128 { + let a = a.as_u16x8(); + let b = b.as_u16x8(); + unsafe { simd_select::(simd_gt(a, b), a, b).v128() } +} + +/// Lane-wise rounding average. +#[inline] +#[cfg_attr(test, assert_instr(i16x8.avgr_u))] +#[target_feature(enable = "simd128")] +#[doc(alias("i16x8.avgr_u"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn u16x8_avgr(a: v128, b: v128) -> v128 { + unsafe { llvm_avgr_u_i16x8(a.as_i16x8(), b.as_i16x8()).v128() } +} + +/// Lane-wise integer extended multiplication producing twice wider result than +/// the inputs. +/// +/// Equivalent of `i16x8_mul(i16x8_extend_low_i8x16(a), i16x8_extend_low_i8x16(b))` +#[inline] +// #[cfg_attr(test, assert_instr(i16x8.extmul_low_i8x16_s))] // FIXME wasmtime +#[target_feature(enable = "simd128")] +#[doc(alias("i16x8.extmul_low_i8x16_s"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i16x8_extmul_low_i8x16(a: v128, b: v128) -> v128 { + unsafe { llvm_i16x8_extmul_low_i8x16_s(a.as_i8x16(), b.as_i8x16()).v128() } +} + +/// Lane-wise integer extended multiplication producing twice wider result than +/// the inputs. +/// +/// Equivalent of `i16x8_mul(i16x8_extend_high_i8x16(a), i16x8_extend_high_i8x16(b))` +#[inline] +// #[cfg_attr(test, assert_instr(i16x8.extmul_high_i8x16_s))] // FIXME wasmtime +#[target_feature(enable = "simd128")] +#[doc(alias("i16x8.extmul_high_i8x16_s"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i16x8_extmul_high_i8x16(a: v128, b: v128) -> v128 { + unsafe { llvm_i16x8_extmul_high_i8x16_s(a.as_i8x16(), b.as_i8x16()).v128() } +} + +/// Lane-wise integer extended multiplication producing twice wider result than +/// the inputs. +/// +/// Equivalent of `i16x8_mul(i16x8_extend_low_u8x16(a), i16x8_extend_low_u8x16(b))` +#[inline] +// #[cfg_attr(test, assert_instr(i16x8.extmul_low_i8x16_u))] // FIXME wasmtime +#[target_feature(enable = "simd128")] +#[doc(alias("i16x8.extmul_low_i8x16_u"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i16x8_extmul_low_u8x16(a: v128, b: v128) -> v128 { + unsafe { llvm_i16x8_extmul_low_i8x16_u(a.as_i8x16(), b.as_i8x16()).v128() } +} + +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub use i16x8_extmul_low_u8x16 as u16x8_extmul_low_u8x16; + +/// Lane-wise integer extended multiplication producing twice wider result than +/// the inputs. +/// +/// Equivalent of `i16x8_mul(i16x8_extend_high_u8x16(a), i16x8_extend_high_u8x16(b))` +#[inline] +// #[cfg_attr(test, assert_instr(i16x8.extmul_high_i8x16_u))] // FIXME wasmtime +#[target_feature(enable = "simd128")] +#[doc(alias("i16x8.extmul_high_i8x16_u"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i16x8_extmul_high_u8x16(a: v128, b: v128) -> v128 { + unsafe { llvm_i16x8_extmul_high_i8x16_u(a.as_i8x16(), b.as_i8x16()).v128() } +} + +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub use i16x8_extmul_high_u8x16 as u16x8_extmul_high_u8x16; + +/// Lane-wise integer extended pairwise addition producing extended results +/// (twice wider results than the inputs). +#[inline] +// #[cfg_attr(test, assert_instr(i32x4.extadd_pairwise_i16x8_s))] // FIXME wasmtime +#[target_feature(enable = "simd128")] +#[doc(alias("i32x4.extadd_pairwise_i16x8_s"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i32x4_extadd_pairwise_i16x8(a: v128) -> v128 { + unsafe { llvm_i32x4_extadd_pairwise_i16x8_s(a.as_i16x8()).v128() } +} + +/// Lane-wise integer extended pairwise addition producing extended results +/// (twice wider results than the inputs). +#[inline] +// #[cfg_attr(test, assert_instr(i32x4.extadd_pairwise_i16x8_u))] // FIXME wasmtime +#[doc(alias("i32x4.extadd_pairwise_i16x8_u"))] +#[target_feature(enable = "simd128")] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i32x4_extadd_pairwise_u16x8(a: v128) -> v128 { + unsafe { llvm_i32x4_extadd_pairwise_i16x8_u(a.as_i16x8()).v128() } +} + +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub use i32x4_extadd_pairwise_u16x8 as u32x4_extadd_pairwise_u16x8; + +/// Lane-wise wrapping absolute value. +#[inline] +#[cfg_attr(test, assert_instr(i32x4.abs))] +#[target_feature(enable = "simd128")] +#[doc(alias("i32x4.abs"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i32x4_abs(a: v128) -> v128 { + let a = a.as_i32x4(); + let zero = simd::i32x4::splat(0); + unsafe { + simd_select::(simd_lt(a, zero), simd_sub(zero, a), a).v128() + } +} + +/// Negates a 128-bit vectors intepreted as four 32-bit signed integers +#[inline] +#[cfg_attr(test, assert_instr(i32x4.neg))] +#[target_feature(enable = "simd128")] +#[doc(alias("i32x4.neg"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i32x4_neg(a: v128) -> v128 { + unsafe { simd_mul(a.as_i32x4(), simd::i32x4::splat(-1)).v128() } +} + +/// Returns 1 if all lanes are nonzero or 0 if any lane is nonzero. +#[inline] +#[cfg_attr(test, assert_instr(i32x4.all_true))] +#[target_feature(enable = "simd128")] +#[doc(alias("i32x4.all_true"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i32x4_all_true(a: v128) -> bool { + unsafe { llvm_i32x4_all_true(a.as_i32x4()) != 0 } +} + +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub use i32x4_all_true as u32x4_all_true; + +/// Extracts the high bit for each lane in `a` and produce a scalar mask with +/// all bits concatenated. +#[inline] +#[cfg_attr(test, assert_instr(i32x4.bitmask))] +#[target_feature(enable = "simd128")] +#[doc(alias("i32x4.bitmask"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i32x4_bitmask(a: v128) -> u8 { + unsafe { llvm_bitmask_i32x4(a.as_i32x4()) as u8 } } +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub use i32x4_bitmask as u32x4_bitmask; + /// Converts low half of the smaller lane vector to a larger lane /// vector, sign extended. #[inline] -#[cfg_attr(test, assert_instr(i16x8.widen_low_i8x16_s))] -pub unsafe fn i16x8_widen_low_i8x16_s(a: v128) -> v128 { - transmute(llvm_widen_low_i16x8_s(transmute(a))) +#[cfg_attr(test, assert_instr(i32x4.extend_low_i16x8_s))] +#[target_feature(enable = "simd128")] +#[doc(alias("i32x4.extend_low_i16x8_s"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i32x4_extend_low_i16x8(a: v128) -> v128 { + unsafe { + simd_cast::(simd_shuffle4!( + a.as_i16x8(), + a.as_i16x8(), + [0, 1, 2, 3] + )) + .v128() + } } /// Converts high half of the smaller lane vector to a larger lane /// vector, sign extended. #[inline] -#[cfg_attr(test, assert_instr(i16x8.widen_high_i8x16_s))] -pub unsafe fn i16x8_widen_high_i8x16_s(a: v128) -> v128 { - transmute(llvm_widen_high_i16x8_s(transmute(a))) +#[cfg_attr(test, assert_instr(i32x4.extend_high_i16x8_s))] +#[target_feature(enable = "simd128")] +#[doc(alias("i32x4.extend_high_i16x8_s"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i32x4_extend_high_i16x8(a: v128) -> v128 { + unsafe { + simd_cast::(simd_shuffle4!( + a.as_i16x8(), + a.as_i16x8(), + [4, 5, 6, 7] + )) + .v128() + } } /// Converts low half of the smaller lane vector to a larger lane /// vector, zero extended. #[inline] -#[cfg_attr(test, assert_instr(i16x8.widen_low_i8x16_u))] -pub unsafe fn i16x8_widen_low_i8x16_u(a: v128) -> v128 { - transmute(llvm_widen_low_i16x8_u(transmute(a))) +#[cfg_attr(test, assert_instr(i32x4.extend_low_i16x8_u))] +#[target_feature(enable = "simd128")] +#[doc(alias("i32x4.extend_low_i16x8_u"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i32x4_extend_low_u16x8(a: v128) -> v128 { + unsafe { + simd_cast::(simd_shuffle4!( + a.as_u16x8(), + a.as_u16x8(), + [0, 1, 2, 3] + )) + .v128() + } } +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub use i32x4_extend_low_u16x8 as u32x4_extend_low_u16x8; + /// Converts high half of the smaller lane vector to a larger lane /// vector, zero extended. #[inline] -#[cfg_attr(test, assert_instr(i16x8.widen_high_i8x16_u))] -pub unsafe fn i16x8_widen_high_i8x16_u(a: v128) -> v128 { - transmute(llvm_widen_high_i16x8_u(transmute(a))) +#[cfg_attr(test, assert_instr(i32x4.extend_high_i16x8_u))] +#[target_feature(enable = "simd128")] +#[doc(alias("i32x4.extend_high_i16x8_u"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i32x4_extend_high_u16x8(a: v128) -> v128 { + unsafe { + simd_cast::(simd_shuffle4!( + a.as_u16x8(), + a.as_u16x8(), + [4, 5, 6, 7] + )) + .v128() + } } +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub use i32x4_extend_high_u16x8 as u32x4_extend_high_u16x8; + /// Shifts each lane to the left by the specified number of bits. /// /// Only the low bits of the shift amount are used if the shift amount is /// greater than the lane width. #[inline] -#[cfg_attr(test, assert_instr(i16x8.shl))] +#[cfg_attr(test, assert_instr(i32x4.shl))] #[target_feature(enable = "simd128")] -pub unsafe fn i16x8_shl(a: v128, amt: u32) -> v128 { - transmute(simd_shl(a.as_i16x8(), i16x8::splat(amt as i16))) +#[doc(alias("i32x4.shl"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i32x4_shl(a: v128, amt: u32) -> v128 { + unsafe { simd_shl(a.as_i32x4(), simd::i32x4::splat(amt as i32)).v128() } } +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub use i32x4_shl as u32x4_shl; + /// Shifts each lane to the right by the specified number of bits, sign /// extending. /// /// Only the low bits of the shift amount are used if the shift amount is /// greater than the lane width. #[inline] -#[cfg_attr(test, assert_instr(i16x8.shr_s))] +#[cfg_attr(test, assert_instr(i32x4.shr_s))] #[target_feature(enable = "simd128")] -pub unsafe fn i16x8_shr_s(a: v128, amt: u32) -> v128 { - transmute(simd_shr(a.as_i16x8(), i16x8::splat(amt as i16))) +#[doc(alias("i32x4.shr_s"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i32x4_shr(a: v128, amt: u32) -> v128 { + unsafe { simd_shr(a.as_i32x4(), simd::i32x4::splat(amt as i32)).v128() } } /// Shifts each lane to the right by the specified number of bits, shifting in @@ -1756,225 +3053,316 @@ pub unsafe fn i16x8_shr_s(a: v128, amt: u32) -> v128 { /// Only the low bits of the shift amount are used if the shift amount is /// greater than the lane width. #[inline] -#[cfg_attr(test, assert_instr(i16x8.shr_u))] +#[cfg_attr(test, assert_instr(i32x4.shr_u))] #[target_feature(enable = "simd128")] -pub unsafe fn i16x8_shr_u(a: v128, amt: u32) -> v128 { - transmute(simd_shr(a.as_u16x8(), u16x8::splat(amt as u16))) +#[doc(alias("i32x4.shr_u"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn u32x4_shr(a: v128, amt: u32) -> v128 { + unsafe { simd_shr(a.as_u32x4(), simd::u32x4::splat(amt as u32)).v128() } } -/// Adds two 128-bit vectors as if they were two packed eight 16-bit integers. +/// Adds two 128-bit vectors as if they were two packed four 32-bit integers. #[inline] -#[cfg_attr(test, assert_instr(i16x8.add))] +#[cfg_attr(test, assert_instr(i32x4.add))] #[target_feature(enable = "simd128")] -pub unsafe fn i16x8_add(a: v128, b: v128) -> v128 { - transmute(simd_add(a.as_i16x8(), b.as_i16x8())) +#[doc(alias("i32x4.add"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i32x4_add(a: v128, b: v128) -> v128 { + unsafe { simd_add(a.as_i32x4(), b.as_i32x4()).v128() } } -/// Adds two 128-bit vectors as if they were two packed eight 16-bit signed -/// integers, saturating on overflow to `i16::MAX`. +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub use i32x4_add as u32x4_add; + +/// Subtracts two 128-bit vectors as if they were two packed four 32-bit integers. #[inline] -#[cfg_attr(test, assert_instr(i16x8.add_saturate_s))] +#[cfg_attr(test, assert_instr(i32x4.sub))] #[target_feature(enable = "simd128")] -pub unsafe fn i16x8_add_saturate_s(a: v128, b: v128) -> v128 { - transmute(llvm_i16x8_add_saturate_s(a.as_i16x8(), b.as_i16x8())) +#[doc(alias("i32x4.sub"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i32x4_sub(a: v128, b: v128) -> v128 { + unsafe { simd_sub(a.as_i32x4(), b.as_i32x4()).v128() } } -/// Adds two 128-bit vectors as if they were two packed eight 16-bit unsigned -/// integers, saturating on overflow to `u16::MAX`. +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub use i32x4_sub as u32x4_sub; + +/// Multiplies two 128-bit vectors as if they were two packed four 32-bit +/// signed integers. #[inline] -#[cfg_attr(test, assert_instr(i16x8.add_saturate_u))] +#[cfg_attr(test, assert_instr(i32x4.mul))] #[target_feature(enable = "simd128")] -pub unsafe fn i16x8_add_saturate_u(a: v128, b: v128) -> v128 { - transmute(llvm_i16x8_add_saturate_u(a.as_i16x8(), b.as_i16x8())) +#[doc(alias("i32x4.mul"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i32x4_mul(a: v128, b: v128) -> v128 { + unsafe { simd_mul(a.as_i32x4(), b.as_i32x4()).v128() } } -/// Subtracts two 128-bit vectors as if they were two packed eight 16-bit integers. +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub use i32x4_mul as u32x4_mul; + +/// Compares lane-wise signed integers, and returns the minimum of +/// each pair. #[inline] -#[cfg_attr(test, assert_instr(i16x8.sub))] +#[cfg_attr(test, assert_instr(i32x4.min_s))] #[target_feature(enable = "simd128")] -pub unsafe fn i16x8_sub(a: v128, b: v128) -> v128 { - transmute(simd_sub(a.as_i16x8(), b.as_i16x8())) +#[doc(alias("i32x4.min_s"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i32x4_min(a: v128, b: v128) -> v128 { + let a = a.as_i32x4(); + let b = b.as_i32x4(); + unsafe { simd_select::(simd_lt(a, b), a, b).v128() } } -/// Subtracts two 128-bit vectors as if they were two packed eight 16-bit -/// signed integers, saturating on overflow to `i16::MIN`. +/// Compares lane-wise unsigned integers, and returns the minimum of +/// each pair. #[inline] -#[cfg_attr(test, assert_instr(i16x8.sub_saturate_s))] +#[cfg_attr(test, assert_instr(i32x4.min_u))] #[target_feature(enable = "simd128")] -pub unsafe fn i16x8_sub_saturate_s(a: v128, b: v128) -> v128 { - transmute(llvm_i16x8_sub_saturate_s(a.as_i16x8(), b.as_i16x8())) +#[doc(alias("i32x4.min_u"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn u32x4_min(a: v128, b: v128) -> v128 { + let a = a.as_u32x4(); + let b = b.as_u32x4(); + unsafe { simd_select::(simd_lt(a, b), a, b).v128() } } -/// Subtracts two 128-bit vectors as if they were two packed eight 16-bit -/// unsigned integers, saturating on overflow to 0. +/// Compares lane-wise signed integers, and returns the maximum of +/// each pair. #[inline] -#[cfg_attr(test, assert_instr(i16x8.sub_saturate_u))] +#[cfg_attr(test, assert_instr(i32x4.max_s))] #[target_feature(enable = "simd128")] -pub unsafe fn i16x8_sub_saturate_u(a: v128, b: v128) -> v128 { - transmute(llvm_i16x8_sub_saturate_u(a.as_i16x8(), b.as_i16x8())) +#[doc(alias("i32x4.max_s"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i32x4_max(a: v128, b: v128) -> v128 { + let a = a.as_i32x4(); + let b = b.as_i32x4(); + unsafe { simd_select::(simd_gt(a, b), a, b).v128() } } -/// Multiplies two 128-bit vectors as if they were two packed eight 16-bit -/// signed integers. +/// Compares lane-wise unsigned integers, and returns the maximum of +/// each pair. #[inline] -#[cfg_attr(test, assert_instr(i16x8.mul))] +#[cfg_attr(test, assert_instr(i32x4.max_u))] #[target_feature(enable = "simd128")] -pub unsafe fn i16x8_mul(a: v128, b: v128) -> v128 { - transmute(simd_mul(a.as_i16x8(), b.as_i16x8())) +#[doc(alias("i32x4.max_u"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn u32x4_max(a: v128, b: v128) -> v128 { + let a = a.as_u32x4(); + let b = b.as_u32x4(); + unsafe { simd_select::(simd_gt(a, b), a, b).v128() } } -/// Compares lane-wise signed integers, and returns the minimum of -/// each pair. +/// Lane-wise multiply signed 16-bit integers in the two input vectors and add +/// adjacent pairs of the full 32-bit results. #[inline] -#[cfg_attr(test, assert_instr(i16x8.min_s))] +#[cfg_attr(test, assert_instr(i32x4.dot_i16x8_s))] #[target_feature(enable = "simd128")] -pub unsafe fn i16x8_min_s(a: v128, b: v128) -> v128 { - let a = transmute::<_, i16x8>(a); - let b = transmute::<_, i16x8>(b); - transmute(simd_select::(simd_lt(a, b), a, b)) +#[doc(alias("i32x4.dot_i16x8_s"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i32x4_dot_i16x8(a: v128, b: v128) -> v128 { + unsafe { llvm_i32x4_dot_i16x8_s(a.as_i16x8(), b.as_i16x8()).v128() } } -/// Compares lane-wise unsigned integers, and returns the minimum of -/// each pair. +/// Lane-wise integer extended multiplication producing twice wider result than +/// the inputs. +/// +/// Equivalent of `i32x4_mul(i32x4_extend_low_i16x8_s(a), i32x4_extend_low_i16x8_s(b))` #[inline] -#[cfg_attr(test, assert_instr(i16x8.min_u))] +// #[cfg_attr(test, assert_instr(i32x4.extmul_low_i16x8_s))] // FIXME wasmtime #[target_feature(enable = "simd128")] -pub unsafe fn i16x8_min_u(a: v128, b: v128) -> v128 { - let a = transmute::<_, u16x8>(a); - let b = transmute::<_, u16x8>(b); - transmute(simd_select::(simd_lt(a, b), a, b)) +#[doc(alias("i32x4.extmul_low_i16x8_s"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i32x4_extmul_low_i16x8(a: v128, b: v128) -> v128 { + unsafe { llvm_i32x4_extmul_low_i16x8_s(a.as_i16x8(), b.as_i16x8()).v128() } } -/// Compares lane-wise signed integers, and returns the maximum of -/// each pair. +/// Lane-wise integer extended multiplication producing twice wider result than +/// the inputs. +/// +/// Equivalent of `i32x4_mul(i32x4_extend_high_i16x8_s(a), i32x4_extend_high_i16x8_s(b))` #[inline] -#[cfg_attr(test, assert_instr(i16x8.max_s))] +// #[cfg_attr(test, assert_instr(i32x4.extmul_high_i16x8_s))] // FIXME wasmtime #[target_feature(enable = "simd128")] -pub unsafe fn i16x8_max_s(a: v128, b: v128) -> v128 { - let a = transmute::<_, i16x8>(a); - let b = transmute::<_, i16x8>(b); - transmute(simd_select::(simd_gt(a, b), a, b)) +#[doc(alias("i32x4.extmul_high_i16x8_s"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i32x4_extmul_high_i16x8(a: v128, b: v128) -> v128 { + unsafe { llvm_i32x4_extmul_high_i16x8_s(a.as_i16x8(), b.as_i16x8()).v128() } } -/// Compares lane-wise unsigned integers, and returns the maximum of -/// each pair. +/// Lane-wise integer extended multiplication producing twice wider result than +/// the inputs. +/// +/// Equivalent of `i32x4_mul(i32x4_extend_low_u16x8(a), i32x4_extend_low_u16x8(b))` #[inline] -#[cfg_attr(test, assert_instr(i16x8.max_u))] +// #[cfg_attr(test, assert_instr(i32x4.extmul_low_i16x8_u))] // FIXME wasmtime #[target_feature(enable = "simd128")] -pub unsafe fn i16x8_max_u(a: v128, b: v128) -> v128 { - let a = transmute::<_, u16x8>(a); - let b = transmute::<_, u16x8>(b); - transmute(simd_select::(simd_gt(a, b), a, b)) +#[doc(alias("i32x4.extmul_low_i16x8_u"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i32x4_extmul_low_u16x8(a: v128, b: v128) -> v128 { + unsafe { llvm_i32x4_extmul_low_i16x8_u(a.as_i16x8(), b.as_i16x8()).v128() } } -/// Lane-wise rounding average. +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub use i32x4_extmul_low_u16x8 as u32x4_extmul_low_u16x8; + +/// Lane-wise integer extended multiplication producing twice wider result than +/// the inputs. +/// +/// Equivalent of `i32x4_mul(i32x4_extend_high_u16x8(a), i32x4_extend_high_u16x8(b))` #[inline] -#[cfg_attr(test, assert_instr(i16x8.avgr_u))] +// #[cfg_attr(test, assert_instr(i32x4.extmul_high_i16x8_u))] // FIXME wasmtime #[target_feature(enable = "simd128")] -pub unsafe fn i16x8_avgr_u(a: v128, b: v128) -> v128 { - transmute(llvm_avgr_u_i16x8(transmute(a), transmute(b))) +#[doc(alias("i32x4.extmul_high_i16x8_u"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i32x4_extmul_high_u16x8(a: v128, b: v128) -> v128 { + unsafe { llvm_i32x4_extmul_high_i16x8_u(a.as_i16x8(), b.as_i16x8()).v128() } } +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub use i32x4_extmul_high_u16x8 as u32x4_extmul_high_u16x8; + /// Lane-wise wrapping absolute value. #[inline] -// #[cfg_attr(test, assert_instr(i32x4.abs))] // FIXME support not in our LLVM yet +// #[cfg_attr(test, assert_instr(i64x2.abs))] // FIXME llvm #[target_feature(enable = "simd128")] -pub unsafe fn i32x4_abs(a: v128) -> v128 { - let a = transmute::<_, i32x4>(a); - let zero = i32x4::splat(0); - transmute(simd_select::( - simd_lt(a, zero), - simd_sub(zero, a), - a, - )) +#[doc(alias("i64x2.abs"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i64x2_abs(a: v128) -> v128 { + let a = a.as_i64x2(); + let zero = simd::i64x2::splat(0); + unsafe { + simd_select::(simd_lt(a, zero), simd_sub(zero, a), a).v128() + } } -/// Negates a 128-bit vectors intepreted as four 32-bit signed integers +/// Negates a 128-bit vectors intepreted as two 64-bit signed integers #[inline] -#[cfg_attr(test, assert_instr(i32x4.neg))] +#[cfg_attr(test, assert_instr(i64x2.neg))] #[target_feature(enable = "simd128")] -pub unsafe fn i32x4_neg(a: v128) -> v128 { - transmute(simd_mul(a.as_i32x4(), i32x4::splat(-1))) +#[doc(alias("i64x2.neg"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i64x2_neg(a: v128) -> v128 { + unsafe { simd_mul(a.as_i64x2(), simd::i64x2::splat(-1)).v128() } } -/// Returns 1 if any lane is nonzero or 0 if all lanes are zero. +/// Returns 1 if all lanes are nonzero or 0 if any lane is nonzero. #[inline] -#[cfg_attr(test, assert_instr(i32x4.any_true))] +#[cfg_attr(test, assert_instr(i64x2.all_true))] #[target_feature(enable = "simd128")] -pub unsafe fn i32x4_any_true(a: v128) -> i32 { - llvm_i32x4_any_true(a.as_i32x4()) +#[doc(alias("i64x2.all_true"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i64x2_all_true(a: v128) -> bool { + unsafe { llvm_i64x2_all_true(a.as_i64x2()) != 0 } } -/// Returns 1 if all lanes are nonzero or 0 if any lane is nonzero. +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub use i64x2_all_true as u64x2_all_true; + +/// Extracts the high bit for each lane in `a` and produce a scalar mask with +/// all bits concatenated. #[inline] -#[cfg_attr(test, assert_instr(i32x4.all_true))] +#[cfg_attr(test, assert_instr(i64x2.bitmask))] #[target_feature(enable = "simd128")] -pub unsafe fn i32x4_all_true(a: v128) -> i32 { - llvm_i32x4_all_true(a.as_i32x4()) +#[doc(alias("i64x2.bitmask"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i64x2_bitmask(a: v128) -> u8 { + unsafe { llvm_bitmask_i64x2(a.as_i64x2()) as u8 } } -// FIXME: not available in our LLVM yet -// /// Extracts the high bit for each lane in `a` and produce a scalar mask with -// /// all bits concatenated. -// #[inline] -// #[cfg_attr(test, assert_instr(i32x4.all_true))] -// pub unsafe fn i32x4_bitmask(a: v128) -> i32 { -// llvm_bitmask_i32x4(transmute(a)) -// } +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub use i64x2_bitmask as u64x2_bitmask; /// Converts low half of the smaller lane vector to a larger lane /// vector, sign extended. #[inline] -#[cfg_attr(test, assert_instr(i32x4.widen_low_i16x8_s))] -pub unsafe fn i32x4_widen_low_i16x8_s(a: v128) -> v128 { - transmute(llvm_widen_low_i32x4_s(transmute(a))) +// #[cfg_attr(test, assert_instr(i64x2.extend_low_i32x4_s))] // FIXME wasmtime +#[target_feature(enable = "simd128")] +#[doc(alias("i64x2.extend_low_i32x4_s"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i64x2_extend_low_i32x4(a: v128) -> v128 { + unsafe { + simd_cast::(simd_shuffle2!(a.as_i32x4(), a.as_i32x4(), [0, 1])) + .v128() + } } /// Converts high half of the smaller lane vector to a larger lane /// vector, sign extended. #[inline] -#[cfg_attr(test, assert_instr(i32x4.widen_high_i16x8_s))] -pub unsafe fn i32x4_widen_high_i16x8_s(a: v128) -> v128 { - transmute(llvm_widen_high_i32x4_s(transmute(a))) +// #[cfg_attr(test, assert_instr(i64x2.extend_high_i32x4_s))] // FIXME wasmtime +#[target_feature(enable = "simd128")] +#[doc(alias("i64x2.extend_high_i32x4_s"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i64x2_extend_high_i32x4(a: v128) -> v128 { + unsafe { + simd_cast::(simd_shuffle2!(a.as_i32x4(), a.as_i32x4(), [2, 3])) + .v128() + } } /// Converts low half of the smaller lane vector to a larger lane /// vector, zero extended. #[inline] -#[cfg_attr(test, assert_instr(i32x4.widen_low_i16x8_u))] -pub unsafe fn i32x4_widen_low_i16x8_u(a: v128) -> v128 { - transmute(llvm_widen_low_i32x4_u(transmute(a))) +// #[cfg_attr(test, assert_instr(i64x2.extend_low_i32x4_u))] // FIXME wasmtime +#[target_feature(enable = "simd128")] +#[doc(alias("i64x2.extend_low_i32x4_u"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i64x2_extend_low_u32x4(a: v128) -> v128 { + unsafe { + simd_cast::(simd_shuffle2!(a.as_u32x4(), a.as_u32x4(), [0, 1])) + .v128() + } } +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub use i64x2_extend_low_u32x4 as u64x2_extend_low_u32x4; + /// Converts high half of the smaller lane vector to a larger lane /// vector, zero extended. #[inline] -#[cfg_attr(test, assert_instr(i32x4.widen_high_i16x8_u))] -pub unsafe fn i32x4_widen_high_i16x8_u(a: v128) -> v128 { - transmute(llvm_widen_high_i32x4_u(transmute(a))) +// #[cfg_attr(test, assert_instr(i64x2.extend_high_i32x4_u))] // FIXME wasmtime +#[target_feature(enable = "simd128")] +#[doc(alias("i64x2.extend_high_i32x4_u"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i64x2_extend_high_u32x4(a: v128) -> v128 { + unsafe { + simd_cast::(simd_shuffle2!(a.as_u32x4(), a.as_u32x4(), [2, 3])) + .v128() + } } +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub use i64x2_extend_high_u32x4 as u64x2_extend_high_u32x4; + /// Shifts each lane to the left by the specified number of bits. /// /// Only the low bits of the shift amount are used if the shift amount is /// greater than the lane width. #[inline] -#[cfg_attr(test, assert_instr(i32x4.shl))] +#[cfg_attr(test, assert_instr(i64x2.shl))] #[target_feature(enable = "simd128")] -pub unsafe fn i32x4_shl(a: v128, amt: u32) -> v128 { - transmute(simd_shl(a.as_i32x4(), i32x4::splat(amt as i32))) +#[doc(alias("i64x2.shl"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i64x2_shl(a: v128, amt: u32) -> v128 { + unsafe { simd_shl(a.as_i64x2(), simd::i64x2::splat(amt as i64)).v128() } } +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub use i64x2_shl as u64x2_shl; + /// Shifts each lane to the right by the specified number of bits, sign /// extending. /// /// Only the low bits of the shift amount are used if the shift amount is /// greater than the lane width. #[inline] -#[cfg_attr(test, assert_instr(i32x4.shr_s))] +#[cfg_attr(test, assert_instr(i64x2.shr_s))] #[target_feature(enable = "simd128")] -pub unsafe fn i32x4_shr_s(a: v128, amt: u32) -> v128 { - transmute(simd_shr(a.as_i32x4(), i32x4::splat(amt as i32))) +#[doc(alias("i64x2.shr_s"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i64x2_shr(a: v128, amt: u32) -> v128 { + unsafe { simd_shr(a.as_i64x2(), simd::i64x2::splat(amt as i64)).v128() } } /// Shifts each lane to the right by the specified number of bits, shifting in @@ -1983,146 +3371,151 @@ pub unsafe fn i32x4_shr_s(a: v128, amt: u32) -> v128 { /// Only the low bits of the shift amount are used if the shift amount is /// greater than the lane width. #[inline] -#[cfg_attr(test, assert_instr(i32x4.shr_u))] +#[cfg_attr(test, assert_instr(i64x2.shr_u))] #[target_feature(enable = "simd128")] -pub unsafe fn i32x4_shr_u(a: v128, amt: u32) -> v128 { - transmute(simd_shr(a.as_u32x4(), u32x4::splat(amt as u32))) +#[doc(alias("i64x2.shr_u"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn u64x2_shr(a: v128, amt: u32) -> v128 { + unsafe { simd_shr(a.as_u64x2(), simd::u64x2::splat(amt as u64)).v128() } } -/// Adds two 128-bit vectors as if they were two packed four 32-bit integers. +/// Adds two 128-bit vectors as if they were two packed two 64-bit integers. #[inline] -#[cfg_attr(test, assert_instr(i32x4.add))] +#[cfg_attr(test, assert_instr(i64x2.add))] #[target_feature(enable = "simd128")] -pub unsafe fn i32x4_add(a: v128, b: v128) -> v128 { - transmute(simd_add(a.as_i32x4(), b.as_i32x4())) +#[doc(alias("i64x2.add"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i64x2_add(a: v128, b: v128) -> v128 { + unsafe { simd_add(a.as_i64x2(), b.as_i64x2()).v128() } } -/// Subtracts two 128-bit vectors as if they were two packed four 32-bit integers. -#[inline] -#[cfg_attr(test, assert_instr(i32x4.sub))] -#[target_feature(enable = "simd128")] -pub unsafe fn i32x4_sub(a: v128, b: v128) -> v128 { - transmute(simd_sub(a.as_i32x4(), b.as_i32x4())) -} +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub use i64x2_add as u64x2_add; -/// Multiplies two 128-bit vectors as if they were two packed four 32-bit -/// signed integers. +/// Subtracts two 128-bit vectors as if they were two packed two 64-bit integers. #[inline] -#[cfg_attr(test, assert_instr(i32x4.mul))] +#[cfg_attr(test, assert_instr(i64x2.sub))] #[target_feature(enable = "simd128")] -pub unsafe fn i32x4_mul(a: v128, b: v128) -> v128 { - transmute(simd_mul(a.as_i32x4(), b.as_i32x4())) +#[doc(alias("i64x2.sub"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i64x2_sub(a: v128, b: v128) -> v128 { + unsafe { simd_sub(a.as_i64x2(), b.as_i64x2()).v128() } } -/// Compares lane-wise signed integers, and returns the minimum of -/// each pair. -#[inline] -#[cfg_attr(test, assert_instr(i32x4.min_s))] -#[target_feature(enable = "simd128")] -pub unsafe fn i32x4_min_s(a: v128, b: v128) -> v128 { - let a = transmute::<_, i32x4>(a); - let b = transmute::<_, i32x4>(b); - transmute(simd_select::(simd_lt(a, b), a, b)) -} +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub use i64x2_sub as u64x2_sub; -/// Compares lane-wise unsigned integers, and returns the minimum of -/// each pair. +/// Multiplies two 128-bit vectors as if they were two packed two 64-bit integers. #[inline] -#[cfg_attr(test, assert_instr(i32x4.min_u))] +#[cfg_attr(test, assert_instr(i64x2.mul))] #[target_feature(enable = "simd128")] -pub unsafe fn i32x4_min_u(a: v128, b: v128) -> v128 { - let a = transmute::<_, u32x4>(a); - let b = transmute::<_, u32x4>(b); - transmute(simd_select::(simd_lt(a, b), a, b)) +#[doc(alias("i64x2.mul"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i64x2_mul(a: v128, b: v128) -> v128 { + unsafe { simd_mul(a.as_i64x2(), b.as_i64x2()).v128() } } -/// Compares lane-wise signed integers, and returns the maximum of -/// each pair. -#[inline] -#[cfg_attr(test, assert_instr(i32x4.max_s))] -#[target_feature(enable = "simd128")] -pub unsafe fn i32x4_max_s(a: v128, b: v128) -> v128 { - let a = transmute::<_, i32x4>(a); - let b = transmute::<_, i32x4>(b); - transmute(simd_select::(simd_gt(a, b), a, b)) -} +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub use i64x2_mul as u64x2_mul; -/// Compares lane-wise unsigned integers, and returns the maximum of -/// each pair. +/// Lane-wise integer extended multiplication producing twice wider result than +/// the inputs. +/// +/// Equivalent of `i64x2_mul(i64x2_extend_low_i32x4_s(a), i64x2_extend_low_i32x4_s(b))` #[inline] -#[cfg_attr(test, assert_instr(i32x4.max_u))] +// #[cfg_attr(test, assert_instr(i64x2.extmul_low_i32x4_s))] // FIXME wasmtime #[target_feature(enable = "simd128")] -pub unsafe fn i32x4_max_u(a: v128, b: v128) -> v128 { - let a = transmute::<_, u32x4>(a); - let b = transmute::<_, u32x4>(b); - transmute(simd_select::(simd_gt(a, b), a, b)) +#[doc(alias("i64x2.extmul_low_i32x4_s"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i64x2_extmul_low_i32x4(a: v128, b: v128) -> v128 { + unsafe { llvm_i64x2_extmul_low_i32x4_s(a.as_i32x4(), b.as_i32x4()).v128() } } -/// Negates a 128-bit vectors intepreted as two 64-bit signed integers +/// Lane-wise integer extended multiplication producing twice wider result than +/// the inputs. +/// +/// Equivalent of `i64x2_mul(i64x2_extend_high_i32x4_s(a), i64x2_extend_high_i32x4_s(b))` #[inline] -#[cfg_attr(test, assert_instr(i64x2.neg))] +// #[cfg_attr(test, assert_instr(i64x2.extmul_high_i32x4_s))] // FIXME wasmtime #[target_feature(enable = "simd128")] -pub unsafe fn i64x2_neg(a: v128) -> v128 { - transmute(simd_mul(a.as_i64x2(), i64x2::splat(-1))) +#[doc(alias("i64x2.extmul_high_i32x4_s"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i64x2_extmul_high_i32x4(a: v128, b: v128) -> v128 { + unsafe { llvm_i64x2_extmul_high_i32x4_s(a.as_i32x4(), b.as_i32x4()).v128() } } -/// Shifts each lane to the left by the specified number of bits. +/// Lane-wise integer extended multiplication producing twice wider result than +/// the inputs. /// -/// Only the low bits of the shift amount are used if the shift amount is -/// greater than the lane width. +/// Equivalent of `i64x2_mul(i64x2_extend_low_i32x4_u(a), i64x2_extend_low_i32x4_u(b))` #[inline] -#[cfg_attr(test, assert_instr(i64x2.shl))] +// #[cfg_attr(test, assert_instr(i64x2.extmul_low_i32x4_u))] // FIXME wasmtime #[target_feature(enable = "simd128")] -pub unsafe fn i64x2_shl(a: v128, amt: u32) -> v128 { - transmute(simd_shl(a.as_i64x2(), i64x2::splat(amt as i64))) +#[doc(alias("i64x2.extmul_low_i32x4_u"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i64x2_extmul_low_u32x4(a: v128, b: v128) -> v128 { + unsafe { llvm_i64x2_extmul_low_i32x4_u(a.as_i32x4(), b.as_i32x4()).v128() } } -/// Shifts each lane to the right by the specified number of bits, sign -/// extending. +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub use i64x2_extmul_low_u32x4 as u64x2_extmul_low_u32x4; + +/// Lane-wise integer extended multiplication producing twice wider result than +/// the inputs. /// -/// Only the low bits of the shift amount are used if the shift amount is -/// greater than the lane width. +/// Equivalent of `i64x2_mul(i64x2_extend_high_i32x4_u(a), i64x2_extend_high_i32x4_u(b))` #[inline] -#[cfg_attr(test, assert_instr(i64x2.shr_s))] +// #[cfg_attr(test, assert_instr(i64x2.extmul_high_i32x4_u))] // FIXME wasmtime #[target_feature(enable = "simd128")] -pub unsafe fn i64x2_shr_s(a: v128, amt: u32) -> v128 { - transmute(simd_shr(a.as_i64x2(), i64x2::splat(amt as i64))) +#[doc(alias("i64x2.extmul_high_i32x4_u"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i64x2_extmul_high_u32x4(a: v128, b: v128) -> v128 { + unsafe { llvm_i64x2_extmul_high_i32x4_u(a.as_i32x4(), b.as_i32x4()).v128() } } -/// Shifts each lane to the right by the specified number of bits, shifting in -/// zeros. -/// -/// Only the low bits of the shift amount are used if the shift amount is -/// greater than the lane width. +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub use i64x2_extmul_high_u32x4 as u64x2_extmul_high_u32x4; + +/// Lane-wise rounding to the nearest integral value not smaller than the input. #[inline] -#[cfg_attr(test, assert_instr(i64x2.shr_u))] +#[cfg_attr(test, assert_instr(f32x4.ceil))] #[target_feature(enable = "simd128")] -pub unsafe fn i64x2_shr_u(a: v128, amt: u32) -> v128 { - transmute(simd_shr(a.as_u64x2(), u64x2::splat(amt as u64))) +#[doc(alias("f32x4.ceil"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn f32x4_ceil(a: v128) -> v128 { + unsafe { llvm_f32x4_ceil(a.as_f32x4()).v128() } } -/// Adds two 128-bit vectors as if they were two packed two 64-bit integers. +/// Lane-wise rounding to the nearest integral value not greater than the input. #[inline] -#[cfg_attr(test, assert_instr(i64x2.add))] +#[cfg_attr(test, assert_instr(f32x4.floor))] #[target_feature(enable = "simd128")] -pub unsafe fn i64x2_add(a: v128, b: v128) -> v128 { - transmute(simd_add(a.as_i64x2(), b.as_i64x2())) +#[doc(alias("f32x4.floor"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn f32x4_floor(a: v128) -> v128 { + unsafe { llvm_f32x4_floor(a.as_f32x4()).v128() } } -/// Subtracts two 128-bit vectors as if they were two packed two 64-bit integers. +/// Lane-wise rounding to the nearest integral value with the magnitude not +/// larger than the input. #[inline] -#[cfg_attr(test, assert_instr(i64x2.sub))] +#[cfg_attr(test, assert_instr(f32x4.trunc))] #[target_feature(enable = "simd128")] -pub unsafe fn i64x2_sub(a: v128, b: v128) -> v128 { - transmute(simd_sub(a.as_i64x2(), b.as_i64x2())) +#[doc(alias("f32x4.trunc"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn f32x4_trunc(a: v128) -> v128 { + unsafe { llvm_f32x4_trunc(a.as_f32x4()).v128() } } -/// Multiplies two 128-bit vectors as if they were two packed two 64-bit integers. +/// Lane-wise rounding to the nearest integral value; if two values are equally +/// near, rounds to the even one. #[inline] -// #[cfg_attr(test, assert_instr(i64x2.mul))] // FIXME: not present in our LLVM +#[cfg_attr(test, assert_instr(f32x4.nearest))] #[target_feature(enable = "simd128")] -pub unsafe fn i64x2_mul(a: v128, b: v128) -> v128 { - transmute(simd_mul(a.as_i64x2(), b.as_i64x2())) +#[doc(alias("f32x4.nearest"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn f32x4_nearest(a: v128) -> v128 { + unsafe { llvm_f32x4_nearest(a.as_f32x4()).v128() } } /// Calculates the absolute value of each lane of a 128-bit vector interpreted @@ -2130,8 +3523,10 @@ pub unsafe fn i64x2_mul(a: v128, b: v128) -> v128 { #[inline] #[cfg_attr(test, assert_instr(f32x4.abs))] #[target_feature(enable = "simd128")] -pub unsafe fn f32x4_abs(a: v128) -> v128 { - transmute(llvm_f32x4_abs(a.as_f32x4())) +#[doc(alias("f32x4.abs"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn f32x4_abs(a: v128) -> v128 { + unsafe { llvm_f32x4_abs(a.as_f32x4()).v128() } } /// Negates each lane of a 128-bit vector interpreted as four 32-bit floating @@ -2139,8 +3534,10 @@ pub unsafe fn f32x4_abs(a: v128) -> v128 { #[inline] #[cfg_attr(test, assert_instr(f32x4.neg))] #[target_feature(enable = "simd128")] -pub unsafe fn f32x4_neg(a: v128) -> v128 { - f32x4_mul(a, transmute(f32x4(-1.0, -1.0, -1.0, -1.0))) +#[doc(alias("f32x4.neg"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn f32x4_neg(a: v128) -> v128 { + f32x4_mul(a, f32x4_splat(-1.)) } /// Calculates the square root of each lane of a 128-bit vector interpreted as @@ -2148,8 +3545,10 @@ pub unsafe fn f32x4_neg(a: v128) -> v128 { #[inline] #[cfg_attr(test, assert_instr(f32x4.sqrt))] #[target_feature(enable = "simd128")] -pub unsafe fn f32x4_sqrt(a: v128) -> v128 { - transmute(llvm_f32x4_sqrt(a.as_f32x4())) +#[doc(alias("f32x4.sqrt"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn f32x4_sqrt(a: v128) -> v128 { + unsafe { llvm_f32x4_sqrt(a.as_f32x4()).v128() } } /// Adds pairwise lanes of two 128-bit vectors interpreted as four 32-bit @@ -2157,8 +3556,10 @@ pub unsafe fn f32x4_sqrt(a: v128) -> v128 { #[inline] #[cfg_attr(test, assert_instr(f32x4.add))] #[target_feature(enable = "simd128")] -pub unsafe fn f32x4_add(a: v128, b: v128) -> v128 { - transmute(simd_add(a.as_f32x4(), b.as_f32x4())) +#[doc(alias("f32x4.add"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn f32x4_add(a: v128, b: v128) -> v128 { + unsafe { simd_add(a.as_f32x4(), b.as_f32x4()).v128() } } /// Subtracts pairwise lanes of two 128-bit vectors interpreted as four 32-bit @@ -2166,8 +3567,10 @@ pub unsafe fn f32x4_add(a: v128, b: v128) -> v128 { #[inline] #[cfg_attr(test, assert_instr(f32x4.sub))] #[target_feature(enable = "simd128")] -pub unsafe fn f32x4_sub(a: v128, b: v128) -> v128 { - transmute(simd_sub(a.as_f32x4(), b.as_f32x4())) +#[doc(alias("f32x4.sub"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn f32x4_sub(a: v128, b: v128) -> v128 { + unsafe { simd_sub(a.as_f32x4(), b.as_f32x4()).v128() } } /// Multiplies pairwise lanes of two 128-bit vectors interpreted as four 32-bit @@ -2175,8 +3578,10 @@ pub unsafe fn f32x4_sub(a: v128, b: v128) -> v128 { #[inline] #[cfg_attr(test, assert_instr(f32x4.mul))] #[target_feature(enable = "simd128")] -pub unsafe fn f32x4_mul(a: v128, b: v128) -> v128 { - transmute(simd_mul(a.as_f32x4(), b.as_f32x4())) +#[doc(alias("f32x4.mul"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn f32x4_mul(a: v128, b: v128) -> v128 { + unsafe { simd_mul(a.as_f32x4(), b.as_f32x4()).v128() } } /// Divides pairwise lanes of two 128-bit vectors interpreted as four 32-bit @@ -2184,8 +3589,10 @@ pub unsafe fn f32x4_mul(a: v128, b: v128) -> v128 { #[inline] #[cfg_attr(test, assert_instr(f32x4.div))] #[target_feature(enable = "simd128")] -pub unsafe fn f32x4_div(a: v128, b: v128) -> v128 { - transmute(simd_div(a.as_f32x4(), b.as_f32x4())) +#[doc(alias("f32x4.div"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn f32x4_div(a: v128, b: v128) -> v128 { + unsafe { simd_div(a.as_f32x4(), b.as_f32x4()).v128() } } /// Calculates the minimum of pairwise lanes of two 128-bit vectors interpreted @@ -2193,8 +3600,10 @@ pub unsafe fn f32x4_div(a: v128, b: v128) -> v128 { #[inline] #[cfg_attr(test, assert_instr(f32x4.min))] #[target_feature(enable = "simd128")] -pub unsafe fn f32x4_min(a: v128, b: v128) -> v128 { - transmute(llvm_f32x4_min(a.as_f32x4(), b.as_f32x4())) +#[doc(alias("f32x4.min"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn f32x4_min(a: v128, b: v128) -> v128 { + unsafe { llvm_f32x4_min(a.as_f32x4(), b.as_f32x4()).v128() } } /// Calculates the maximum of pairwise lanes of two 128-bit vectors interpreted @@ -2202,8 +3611,72 @@ pub unsafe fn f32x4_min(a: v128, b: v128) -> v128 { #[inline] #[cfg_attr(test, assert_instr(f32x4.max))] #[target_feature(enable = "simd128")] -pub unsafe fn f32x4_max(a: v128, b: v128) -> v128 { - transmute(llvm_f32x4_max(a.as_f32x4(), b.as_f32x4())) +#[doc(alias("f32x4.max"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn f32x4_max(a: v128, b: v128) -> v128 { + unsafe { llvm_f32x4_max(a.as_f32x4(), b.as_f32x4()).v128() } +} + +/// Lane-wise minimum value, defined as `b < a ? b : a` +#[inline] +#[cfg_attr(test, assert_instr(f32x4.pmin))] +#[target_feature(enable = "simd128")] +#[doc(alias("f32x4.pmin"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn f32x4_pmin(a: v128, b: v128) -> v128 { + unsafe { llvm_f32x4_pmin(a.as_f32x4(), b.as_f32x4()).v128() } +} + +/// Lane-wise maximum value, defined as `a < b ? b : a` +#[inline] +#[cfg_attr(test, assert_instr(f32x4.pmax))] +#[target_feature(enable = "simd128")] +#[doc(alias("f32x4.pmax"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn f32x4_pmax(a: v128, b: v128) -> v128 { + unsafe { llvm_f32x4_pmax(a.as_f32x4(), b.as_f32x4()).v128() } +} + +/// Lane-wise rounding to the nearest integral value not smaller than the input. +#[inline] +#[cfg_attr(test, assert_instr(f64x2.ceil))] +#[target_feature(enable = "simd128")] +#[doc(alias("f64x2.ceil"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn f64x2_ceil(a: v128) -> v128 { + unsafe { llvm_f64x2_ceil(a.as_f64x2()).v128() } +} + +/// Lane-wise rounding to the nearest integral value not greater than the input. +#[inline] +#[cfg_attr(test, assert_instr(f64x2.floor))] +#[target_feature(enable = "simd128")] +#[doc(alias("f64x2.floor"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn f64x2_floor(a: v128) -> v128 { + unsafe { llvm_f64x2_floor(a.as_f64x2()).v128() } +} + +/// Lane-wise rounding to the nearest integral value with the magnitude not +/// larger than the input. +#[inline] +#[cfg_attr(test, assert_instr(f64x2.trunc))] +#[target_feature(enable = "simd128")] +#[doc(alias("f64x2.trunc"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn f64x2_trunc(a: v128) -> v128 { + unsafe { llvm_f64x2_trunc(a.as_f64x2()).v128() } +} + +/// Lane-wise rounding to the nearest integral value; if two values are equally +/// near, rounds to the even one. +#[inline] +#[cfg_attr(test, assert_instr(f64x2.nearest))] +#[target_feature(enable = "simd128")] +#[doc(alias("f64x2.nearest"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn f64x2_nearest(a: v128) -> v128 { + unsafe { llvm_f64x2_nearest(a.as_f64x2()).v128() } } /// Calculates the absolute value of each lane of a 128-bit vector interpreted @@ -2211,8 +3684,10 @@ pub unsafe fn f32x4_max(a: v128, b: v128) -> v128 { #[inline] #[cfg_attr(test, assert_instr(f64x2.abs))] #[target_feature(enable = "simd128")] -pub unsafe fn f64x2_abs(a: v128) -> v128 { - transmute(llvm_f64x2_abs(a.as_f64x2())) +#[doc(alias("f64x2.abs"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn f64x2_abs(a: v128) -> v128 { + unsafe { llvm_f64x2_abs(a.as_f64x2()).v128() } } /// Negates each lane of a 128-bit vector interpreted as two 64-bit floating @@ -2220,8 +3695,10 @@ pub unsafe fn f64x2_abs(a: v128) -> v128 { #[inline] #[cfg_attr(test, assert_instr(f64x2.neg))] #[target_feature(enable = "simd128")] -pub unsafe fn f64x2_neg(a: v128) -> v128 { - f64x2_mul(a, transmute(f64x2(-1.0, -1.0))) +#[doc(alias("f64x2.neg"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn f64x2_neg(a: v128) -> v128 { + f64x2_mul(a, f64x2_splat(-1.0)) } /// Calculates the square root of each lane of a 128-bit vector interpreted as @@ -2229,8 +3706,10 @@ pub unsafe fn f64x2_neg(a: v128) -> v128 { #[inline] #[cfg_attr(test, assert_instr(f64x2.sqrt))] #[target_feature(enable = "simd128")] -pub unsafe fn f64x2_sqrt(a: v128) -> v128 { - transmute(llvm_f64x2_sqrt(a.as_f64x2())) +#[doc(alias("f64x2.sqrt"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn f64x2_sqrt(a: v128) -> v128 { + unsafe { llvm_f64x2_sqrt(a.as_f64x2()).v128() } } /// Adds pairwise lanes of two 128-bit vectors interpreted as two 64-bit @@ -2238,8 +3717,10 @@ pub unsafe fn f64x2_sqrt(a: v128) -> v128 { #[inline] #[cfg_attr(test, assert_instr(f64x2.add))] #[target_feature(enable = "simd128")] -pub unsafe fn f64x2_add(a: v128, b: v128) -> v128 { - transmute(simd_add(a.as_f64x2(), b.as_f64x2())) +#[doc(alias("f64x2.add"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn f64x2_add(a: v128, b: v128) -> v128 { + unsafe { simd_add(a.as_f64x2(), b.as_f64x2()).v128() } } /// Subtracts pairwise lanes of two 128-bit vectors interpreted as two 64-bit @@ -2247,8 +3728,10 @@ pub unsafe fn f64x2_add(a: v128, b: v128) -> v128 { #[inline] #[cfg_attr(test, assert_instr(f64x2.sub))] #[target_feature(enable = "simd128")] -pub unsafe fn f64x2_sub(a: v128, b: v128) -> v128 { - transmute(simd_sub(a.as_f64x2(), b.as_f64x2())) +#[doc(alias("f64x2.sub"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn f64x2_sub(a: v128, b: v128) -> v128 { + unsafe { simd_sub(a.as_f64x2(), b.as_f64x2()).v128() } } /// Multiplies pairwise lanes of two 128-bit vectors interpreted as two 64-bit @@ -2256,8 +3739,10 @@ pub unsafe fn f64x2_sub(a: v128, b: v128) -> v128 { #[inline] #[cfg_attr(test, assert_instr(f64x2.mul))] #[target_feature(enable = "simd128")] -pub unsafe fn f64x2_mul(a: v128, b: v128) -> v128 { - transmute(simd_mul(a.as_f64x2(), b.as_f64x2())) +#[doc(alias("f64x2.mul"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn f64x2_mul(a: v128, b: v128) -> v128 { + unsafe { simd_mul(a.as_f64x2(), b.as_f64x2()).v128() } } /// Divides pairwise lanes of two 128-bit vectors interpreted as two 64-bit @@ -2265,8 +3750,10 @@ pub unsafe fn f64x2_mul(a: v128, b: v128) -> v128 { #[inline] #[cfg_attr(test, assert_instr(f64x2.div))] #[target_feature(enable = "simd128")] -pub unsafe fn f64x2_div(a: v128, b: v128) -> v128 { - transmute(simd_div(a.as_f64x2(), b.as_f64x2())) +#[doc(alias("f64x2.div"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn f64x2_div(a: v128, b: v128) -> v128 { + unsafe { simd_div(a.as_f64x2(), b.as_f64x2()).v128() } } /// Calculates the minimum of pairwise lanes of two 128-bit vectors interpreted @@ -2274,8 +3761,10 @@ pub unsafe fn f64x2_div(a: v128, b: v128) -> v128 { #[inline] #[cfg_attr(test, assert_instr(f64x2.min))] #[target_feature(enable = "simd128")] -pub unsafe fn f64x2_min(a: v128, b: v128) -> v128 { - transmute(llvm_f64x2_min(a.as_f64x2(), b.as_f64x2())) +#[doc(alias("f64x2.min"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn f64x2_min(a: v128, b: v128) -> v128 { + unsafe { llvm_f64x2_min(a.as_f64x2(), b.as_f64x2()).v128() } } /// Calculates the maximum of pairwise lanes of two 128-bit vectors interpreted @@ -2283,8 +3772,30 @@ pub unsafe fn f64x2_min(a: v128, b: v128) -> v128 { #[inline] #[cfg_attr(test, assert_instr(f64x2.max))] #[target_feature(enable = "simd128")] -pub unsafe fn f64x2_max(a: v128, b: v128) -> v128 { - transmute(llvm_f64x2_max(a.as_f64x2(), b.as_f64x2())) +#[doc(alias("f64x2.max"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn f64x2_max(a: v128, b: v128) -> v128 { + unsafe { llvm_f64x2_max(a.as_f64x2(), b.as_f64x2()).v128() } +} + +/// Lane-wise minimum value, defined as `b < a ? b : a` +#[inline] +#[cfg_attr(test, assert_instr(f64x2.pmin))] +#[target_feature(enable = "simd128")] +#[doc(alias("f64x2.pmin"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn f64x2_pmin(a: v128, b: v128) -> v128 { + unsafe { llvm_f64x2_pmin(a.as_f64x2(), b.as_f64x2()).v128() } +} + +/// Lane-wise maximum value, defined as `a < b ? b : a` +#[inline] +#[cfg_attr(test, assert_instr(f64x2.pmax))] +#[target_feature(enable = "simd128")] +#[doc(alias("f64x2.pmax"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn f64x2_pmax(a: v128, b: v128) -> v128 { + unsafe { llvm_f64x2_pmax(a.as_f64x2(), b.as_f64x2()).v128() } } /// Converts a 128-bit vector interpreted as four 32-bit floating point numbers @@ -2293,10 +3804,12 @@ pub unsafe fn f64x2_max(a: v128, b: v128) -> v128 { /// NaN is converted to 0 and if it's out of bounds it becomes the nearest /// representable intger. #[inline] -#[cfg_attr(test, assert_instr("i32x4.trunc_sat_f32x4_s"))] +#[cfg_attr(test, assert_instr(i32x4.trunc_sat_f32x4_s))] #[target_feature(enable = "simd128")] -pub unsafe fn i32x4_trunc_sat_f32x4_s(a: v128) -> v128 { - transmute(simd_cast::<_, i32x4>(a.as_f32x4())) +#[doc(alias("i32x4.trunc_sat_f32x4_s"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i32x4_trunc_sat_f32x4(a: v128) -> v128 { + unsafe { llvm_i32x4_trunc_sat_f32x4_s(a.as_f32x4()).v128() } } /// Converts a 128-bit vector interpreted as four 32-bit floating point numbers @@ -2305,35 +3818,142 @@ pub unsafe fn i32x4_trunc_sat_f32x4_s(a: v128) -> v128 { /// NaN is converted to 0 and if it's out of bounds it becomes the nearest /// representable intger. #[inline] -#[cfg_attr(test, assert_instr("i32x4.trunc_sat_f32x4_u"))] +#[cfg_attr(test, assert_instr(i32x4.trunc_sat_f32x4_u))] #[target_feature(enable = "simd128")] -pub unsafe fn i32x4_trunc_sat_f32x4_u(a: v128) -> v128 { - transmute(simd_cast::<_, u32x4>(a.as_f32x4())) +#[doc(alias("i32x4.trunc_sat_f32x4_u"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn u32x4_trunc_sat_f32x4(a: v128) -> v128 { + unsafe { llvm_i32x4_trunc_sat_f32x4_u(a.as_f32x4()).v128() } } /// Converts a 128-bit vector interpreted as four 32-bit signed integers into a /// 128-bit vector of four 32-bit floating point numbers. #[inline] -#[cfg_attr(test, assert_instr("f32x4.convert_i32x4_s"))] +#[cfg_attr(test, assert_instr(f32x4.convert_i32x4_s))] #[target_feature(enable = "simd128")] -pub unsafe fn f32x4_convert_i32x4_s(a: v128) -> v128 { - transmute(simd_cast::<_, f32x4>(a.as_i32x4())) +#[doc(alias("f32x4.convert_i32x4_s"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn f32x4_convert_i32x4(a: v128) -> v128 { + unsafe { simd_cast::<_, simd::f32x4>(a.as_i32x4()).v128() } } /// Converts a 128-bit vector interpreted as four 32-bit unsigned integers into a /// 128-bit vector of four 32-bit floating point numbers. #[inline] -#[cfg_attr(test, assert_instr("f32x4.convert_i32x4_u"))] +#[cfg_attr(test, assert_instr(f32x4.convert_i32x4_u))] +#[target_feature(enable = "simd128")] +#[doc(alias("f32x4.convert_i32x4_u"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn f32x4_convert_u32x4(a: v128) -> v128 { + unsafe { simd_cast::<_, simd::f32x4>(a.as_u32x4()).v128() } +} + +/// Saturating conversion of the two double-precision floating point lanes to +/// two lower integer lanes using the IEEE `convertToIntegerTowardZero` +/// function. +/// +/// The two higher lanes of the result are initialized to zero. If any input +/// lane is a NaN, the resulting lane is 0. If the rounded integer value of a +/// lane is outside the range of the destination type, the result is saturated +/// to the nearest representable integer value. +#[inline] +// #[cfg_attr(test, assert_instr(i32x4.trunc_sat_f64x2_s_zero))] // FIXME wasmtime +#[target_feature(enable = "simd128")] +#[doc(alias("i32x4.trunc_sat_f64x2_s_zero"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn i32x4_trunc_sat_f64x2_zero(a: v128) -> v128 { + let ret: simd::i32x4 = unsafe { + simd_shuffle4!( + llvm_i32x2_trunc_sat_f64x2_s(a.as_f64x2()), + simd::i32x2::splat(0), + [0, 1, 2, 3], + ) + }; + ret.v128() +} + +/// Saturating conversion of the two double-precision floating point lanes to +/// two lower integer lanes using the IEEE `convertToIntegerTowardZero` +/// function. +/// +/// The two higher lanes of the result are initialized to zero. If any input +/// lane is a NaN, the resulting lane is 0. If the rounded integer value of a +/// lane is outside the range of the destination type, the result is saturated +/// to the nearest representable integer value. +#[inline] +// #[cfg_attr(test, assert_instr(i32x4.trunc_sat_f64x2_u_zero))] // FIXME wasmtime +#[target_feature(enable = "simd128")] +#[doc(alias("i32x4.trunc_sat_f64x2_u_zero"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn u32x4_trunc_sat_f64x2_zero(a: v128) -> v128 { + let ret: simd::i32x4 = unsafe { + simd_shuffle4!( + llvm_i32x2_trunc_sat_f64x2_u(a.as_f64x2()), + simd::i32x2::splat(0), + [0, 1, 2, 3], + ) + }; + ret.v128() +} + +/// Lane-wise conversion from integer to floating point. +#[inline] +#[cfg_attr(test, assert_instr(f64x2.convert_low_i32x4_s))] +#[target_feature(enable = "simd128")] +#[doc(alias("f64x2.convert_low_i32x4_s"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn f64x2_convert_low_i32x4(a: v128) -> v128 { + unsafe { + simd_cast::(simd_shuffle2!(a.as_i32x4(), a.as_i32x4(), [0, 1],)) + .v128() + } +} + +/// Lane-wise conversion from integer to floating point. +#[inline] +// #[cfg_attr(test, assert_instr(f64x2.convert_low_i32x4_u))] // FIXME wasmtime +#[target_feature(enable = "simd128")] +#[doc(alias("f64x2.convert_low_i32x4_u"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn f64x2_convert_low_u32x4(a: v128) -> v128 { + unsafe { + simd_cast::(simd_shuffle2!(a.as_u32x4(), a.as_u32x4(), [0, 1],)) + .v128() + } +} + +/// Conversion of the two double-precision floating point lanes to two lower +/// single-precision lanes of the result. The two higher lanes of the result are +/// initialized to zero. If the conversion result is not representable as a +/// single-precision floating point number, it is rounded to the nearest-even +/// representable number. +#[inline] +// #[cfg_attr(test, assert_instr(f32x4.demote_f64x2_zero))] // FIXME wasmtime +#[target_feature(enable = "simd128")] +#[doc(alias("f32x4.demote_f64x2_zero"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn f32x4_demote_f64x2_zero(a: v128) -> v128 { + unsafe { llvm_f32x4_demote_f64x2_zero(a.as_f64x2()).v128() } +} + +/// Conversion of the two lower single-precision floating point lanes to the two +/// double-precision lanes of the result. +#[inline] +// #[cfg_attr(test, assert_instr(f64x2.promote_low_f32x4))] // FIXME wasmtime #[target_feature(enable = "simd128")] -pub unsafe fn f32x4_convert_i32x4_u(a: v128) -> v128 { - transmute(simd_cast::<_, f32x4>(a.as_u32x4())) +#[doc(alias("f32x4.promote_low_f32x4"))] +#[stable(feature = "wasm_simd", since = "1.54.0")] +pub fn f64x2_promote_low_f32x4(a: v128) -> v128 { + unsafe { llvm_f64x2_promote_low_f32x4(a.as_f32x4()).v128() } } #[cfg(test)] pub mod tests { use super::*; + use core::ops::{Add, Div, Mul, Neg, Sub}; use std; - use std::mem; + use std::fmt::Debug; + use std::mem::transmute; use std::num::Wrapping; use std::prelude::v1::*; @@ -2344,32 +3964,168 @@ pub mod tests { } #[test] - fn test_v128_const() { - const A: v128 = - unsafe { super::i8x16_const(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15) }; - compare_bytes(A, A); + fn test_load() { + unsafe { + let arr: [i32; 4] = [0, 1, 2, 3]; + let vec = v128_load(arr.as_ptr() as *const v128); + compare_bytes(vec, i32x4(0, 1, 2, 3)); + } } - macro_rules! test_splat { - ($test_id:ident: $val:expr => $($vals:expr),*) => { - #[test] - fn $test_id() { - unsafe { - let a = super::$test_id($val); - let b: v128 = transmute([$($vals as u8),*]); - compare_bytes(a, b); - } - } + #[test] + fn test_load_extend() { + unsafe { + let arr: [i8; 8] = [-3, -2, -1, 0, 1, 2, 3, 4]; + let vec = i16x8_load_extend_i8x8(arr.as_ptr()); + compare_bytes(vec, i16x8(-3, -2, -1, 0, 1, 2, 3, 4)); + let vec = i16x8_load_extend_u8x8(arr.as_ptr() as *const u8); + compare_bytes(vec, i16x8(253, 254, 255, 0, 1, 2, 3, 4)); + + let arr: [i16; 4] = [-1, 0, 1, 2]; + let vec = i32x4_load_extend_i16x4(arr.as_ptr()); + compare_bytes(vec, i32x4(-1, 0, 1, 2)); + let vec = i32x4_load_extend_u16x4(arr.as_ptr() as *const u16); + compare_bytes(vec, i32x4(65535, 0, 1, 2)); + + let arr: [i32; 2] = [-1, 1]; + let vec = i64x2_load_extend_i32x2(arr.as_ptr()); + compare_bytes(vec, i64x2(-1, 1)); + let vec = i64x2_load_extend_u32x2(arr.as_ptr() as *const u32); + compare_bytes(vec, i64x2(u32::max_value().into(), 1)); + } + } + + #[test] + fn test_load_splat() { + unsafe { + compare_bytes(v128_load8_splat(&8), i8x16_splat(8)); + compare_bytes(v128_load16_splat(&9), i16x8_splat(9)); + compare_bytes(v128_load32_splat(&10), i32x4_splat(10)); + compare_bytes(v128_load64_splat(&11), i64x2_splat(11)); + } + } + + #[test] + fn test_load_zero() { + unsafe { + compare_bytes(v128_load32_zero(&10), i32x4(10, 0, 0, 0)); + compare_bytes(v128_load64_zero(&11), i64x2(11, 0)); + } + } + + #[test] + fn test_store() { + unsafe { + let mut spot = i8x16_splat(0); + v128_store(&mut spot, i8x16_splat(1)); + compare_bytes(spot, i8x16_splat(1)); + } + } + + #[test] + fn test_load_lane() { + unsafe { + let zero = i8x16_splat(0); + compare_bytes( + v128_load8_lane::<2>(zero, &1), + i8x16_replace_lane::<2>(zero, 1), + ); + + compare_bytes( + v128_load16_lane::<2>(zero, &1), + i16x8_replace_lane::<2>(zero, 1), + ); + + compare_bytes( + v128_load32_lane::<2>(zero, &1), + i32x4_replace_lane::<2>(zero, 1), + ); + + compare_bytes( + v128_load64_lane::<1>(zero, &1), + i64x2_replace_lane::<1>(zero, 1), + ); + } + } + + #[test] + fn test_store_lane() { + unsafe { + let mut spot = 0; + let zero = i8x16_splat(0); + v128_store8_lane::<5>(i8x16_replace_lane::<5>(zero, 7), &mut spot); + assert_eq!(spot, 7); + + let mut spot = 0; + v128_store16_lane::<5>(i16x8_replace_lane::<5>(zero, 7), &mut spot); + assert_eq!(spot, 7); + + let mut spot = 0; + v128_store32_lane::<3>(i32x4_replace_lane::<3>(zero, 7), &mut spot); + assert_eq!(spot, 7); + + let mut spot = 0; + v128_store64_lane::<0>(i64x2_replace_lane::<0>(zero, 7), &mut spot); + assert_eq!(spot, 7); } } - test_splat!(i8x16_splat: 42 => 42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42); - test_splat!(i16x8_splat: 42 => 42, 0, 42, 0, 42, 0, 42, 0, 42, 0, 42, 0, 42, 0, 42, 0); - test_splat!(i32x4_splat: 42 => 42, 0, 0, 0, 42, 0, 0, 0, 42, 0, 0, 0, 42, 0, 0, 0); - test_splat!(i64x2_splat: 42 => 42, 0, 0, 0, 0, 0, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0); - test_splat!(f32x4_splat: 42. => 0, 0, 40, 66, 0, 0, 40, 66, 0, 0, 40, 66, 0, 0, 40, 66); - test_splat!(f64x2_splat: 42. => 0, 0, 0, 0, 0, 0, 69, 64, 0, 0, 0, 0, 0, 0, 69, 64); - + #[test] + fn test_i8x16() { + const A: v128 = super::i8x16(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + compare_bytes(A, A); + + const _: v128 = i16x8(0, 1, 2, 3, 4, 5, 6, 7); + const _: v128 = i32x4(0, 1, 2, 3); + const _: v128 = i64x2(0, 1); + const _: v128 = f32x4(0., 1., 2., 3.); + const _: v128 = f64x2(0., 1.); + + let bytes: [i16; 8] = unsafe { mem::transmute(i16x8(-1, -2, -3, -4, -5, -6, -7, -8)) }; + assert_eq!(bytes, [-1, -2, -3, -4, -5, -6, -7, -8]); + let bytes: [i8; 16] = unsafe { + mem::transmute(i8x16( + -1, -2, -3, -4, -5, -6, -7, -8, -9, -10, -11, -12, -13, -14, -15, -16, + )) + }; + assert_eq!( + bytes, + [-1, -2, -3, -4, -5, -6, -7, -8, -9, -10, -11, -12, -13, -14, -15, -16] + ); + } + + #[test] + fn test_shuffle() { + let vec_a = i8x16(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + let vec_b = i8x16( + 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, + ); + + let vec_r = i8x16_shuffle::<0, 16, 2, 18, 4, 20, 6, 22, 8, 24, 10, 26, 12, 28, 14, 30>( + vec_a, vec_b, + ); + let vec_e = i8x16(0, 16, 2, 18, 4, 20, 6, 22, 8, 24, 10, 26, 12, 28, 14, 30); + compare_bytes(vec_r, vec_e); + + let vec_a = i16x8(0, 1, 2, 3, 4, 5, 6, 7); + let vec_b = i16x8(8, 9, 10, 11, 12, 13, 14, 15); + let vec_r = i16x8_shuffle::<0, 8, 2, 10, 4, 12, 6, 14>(vec_a, vec_b); + let vec_e = i16x8(0, 8, 2, 10, 4, 12, 6, 14); + compare_bytes(vec_r, vec_e); + + let vec_a = i32x4(0, 1, 2, 3); + let vec_b = i32x4(4, 5, 6, 7); + let vec_r = i32x4_shuffle::<0, 4, 2, 6>(vec_a, vec_b); + let vec_e = i32x4(0, 4, 2, 6); + compare_bytes(vec_r, vec_e); + + let vec_a = i64x2(0, 1); + let vec_b = i64x2(2, 3); + let vec_r = i64x2_shuffle::<0, 2>(vec_a, vec_b); + let vec_e = i64x2(0, 2); + compare_bytes(vec_r, vec_e); + } + // tests extract and replace lanes macro_rules! test_extract { ( @@ -2453,10 +4209,217 @@ pub mod tests { indices: [0, 1], } + #[test] + #[rustfmt::skip] + fn test_swizzle() { + compare_bytes( + i8x16_swizzle( + i32x4(1, 2, 3, 4), + i8x16( + 32, 31, 30, 29, + 0, 1, 2, 3, + 12, 13, 14, 15, + 0, 4, 8, 12), + ), + i32x4(0, 1, 4, 0x04030201), + ); + } + + macro_rules! test_splat { + ($test_id:ident: $val:expr => $($vals:expr),*) => { + #[test] + fn $test_id() { + let a = super::$test_id($val); + let b = u8x16($($vals as u8),*); + compare_bytes(a, b); + } + } + } + + mod splats { + use super::*; + test_splat!(i8x16_splat: 42 => 42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42); + test_splat!(i16x8_splat: 42 => 42, 0, 42, 0, 42, 0, 42, 0, 42, 0, 42, 0, 42, 0, 42, 0); + test_splat!(i32x4_splat: 42 => 42, 0, 0, 0, 42, 0, 0, 0, 42, 0, 0, 0, 42, 0, 0, 0); + test_splat!(i64x2_splat: 42 => 42, 0, 0, 0, 0, 0, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0); + test_splat!(f32x4_splat: 42. => 0, 0, 40, 66, 0, 0, 40, 66, 0, 0, 40, 66, 0, 0, 40, 66); + test_splat!(f64x2_splat: 42. => 0, 0, 0, 0, 0, 0, 69, 64, 0, 0, 0, 0, 0, 0, 69, 64); + } + + #[test] + fn test_bitmasks() { + let zero = i8x16_splat(0); + let ones = i8x16_splat(!0); + + assert_eq!(i8x16_bitmask(zero), 0); + assert_eq!(i8x16_bitmask(ones), 0xffff); + assert_eq!(i8x16_bitmask(i8x16_splat(i8::MAX)), 0); + assert_eq!(i8x16_bitmask(i8x16_splat(i8::MIN)), 0xffff); + assert_eq!(i8x16_bitmask(i8x16_replace_lane::<1>(zero, -1)), 0b10); + + assert_eq!(i16x8_bitmask(zero), 0); + assert_eq!(i16x8_bitmask(ones), 0xff); + assert_eq!(i16x8_bitmask(i16x8_splat(i16::MAX)), 0); + assert_eq!(i16x8_bitmask(i16x8_splat(i16::MIN)), 0xff); + assert_eq!(i16x8_bitmask(i16x8_replace_lane::<1>(zero, -1)), 0b10); + + assert_eq!(i32x4_bitmask(zero), 0); + assert_eq!(i32x4_bitmask(ones), 0b1111); + assert_eq!(i32x4_bitmask(i32x4_splat(i32::MAX)), 0); + assert_eq!(i32x4_bitmask(i32x4_splat(i32::MIN)), 0b1111); + assert_eq!(i32x4_bitmask(i32x4_replace_lane::<1>(zero, -1)), 0b10); + + assert_eq!(i64x2_bitmask(zero), 0); + assert_eq!(i64x2_bitmask(ones), 0b11); + assert_eq!(i64x2_bitmask(i64x2_splat(i64::MAX)), 0); + assert_eq!(i64x2_bitmask(i64x2_splat(i64::MIN)), 0b11); + assert_eq!(i64x2_bitmask(i64x2_replace_lane::<1>(zero, -1)), 0b10); + } + + #[test] + fn test_narrow() { + let zero = i8x16_splat(0); + let ones = i8x16_splat(!0); + + compare_bytes(i8x16_narrow_i16x8(zero, zero), zero); + compare_bytes(u8x16_narrow_i16x8(zero, zero), zero); + compare_bytes(i8x16_narrow_i16x8(ones, ones), ones); + compare_bytes(u8x16_narrow_i16x8(ones, ones), zero); + + compare_bytes( + i8x16_narrow_i16x8( + i16x8( + 0, + 1, + 2, + -1, + i8::MIN.into(), + i8::MAX.into(), + u8::MIN.into(), + u8::MAX.into(), + ), + i16x8( + i16::MIN.into(), + i16::MAX.into(), + u16::MIN as i16, + u16::MAX as i16, + 0, + 0, + 0, + 0, + ), + ), + i8x16(0, 1, 2, -1, -128, 127, 0, 127, -128, 127, 0, -1, 0, 0, 0, 0), + ); + + compare_bytes( + u8x16_narrow_i16x8( + i16x8( + 0, + 1, + 2, + -1, + i8::MIN.into(), + i8::MAX.into(), + u8::MIN.into(), + u8::MAX.into(), + ), + i16x8( + i16::MIN.into(), + i16::MAX.into(), + u16::MIN as i16, + u16::MAX as i16, + 0, + 0, + 0, + 0, + ), + ), + i8x16(0, 1, 2, 0, 0, 127, 0, -1, 0, -1, 0, 0, 0, 0, 0, 0), + ); + + compare_bytes(i16x8_narrow_i32x4(zero, zero), zero); + compare_bytes(u16x8_narrow_i32x4(zero, zero), zero); + compare_bytes(i16x8_narrow_i32x4(ones, ones), ones); + compare_bytes(u16x8_narrow_i32x4(ones, ones), zero); + + compare_bytes( + i16x8_narrow_i32x4( + i32x4(0, -1, i16::MIN.into(), i16::MAX.into()), + i32x4( + i32::MIN.into(), + i32::MAX.into(), + u32::MIN as i32, + u32::MAX as i32, + ), + ), + i16x8(0, -1, i16::MIN, i16::MAX, i16::MIN, i16::MAX, 0, -1), + ); + + compare_bytes( + u16x8_narrow_i32x4( + i32x4(u16::MAX.into(), -1, i16::MIN.into(), i16::MAX.into()), + i32x4( + i32::MIN.into(), + i32::MAX.into(), + u32::MIN as i32, + u32::MAX as i32, + ), + ), + i16x8(-1, 0, 0, i16::MAX, 0, -1, 0, 0), + ); + } + + #[test] + fn test_extend() { + let zero = i8x16_splat(0); + let ones = i8x16_splat(!0); + + compare_bytes(i16x8_extend_low_i8x16(zero), zero); + compare_bytes(i16x8_extend_high_i8x16(zero), zero); + compare_bytes(i16x8_extend_low_u8x16(zero), zero); + compare_bytes(i16x8_extend_high_u8x16(zero), zero); + compare_bytes(i16x8_extend_low_i8x16(ones), ones); + compare_bytes(i16x8_extend_high_i8x16(ones), ones); + let halves = u16x8_splat(u8::MAX.into()); + compare_bytes(i16x8_extend_low_u8x16(ones), halves); + compare_bytes(i16x8_extend_high_u8x16(ones), halves); + + compare_bytes(i32x4_extend_low_i16x8(zero), zero); + compare_bytes(i32x4_extend_high_i16x8(zero), zero); + compare_bytes(i32x4_extend_low_u16x8(zero), zero); + compare_bytes(i32x4_extend_high_u16x8(zero), zero); + compare_bytes(i32x4_extend_low_i16x8(ones), ones); + compare_bytes(i32x4_extend_high_i16x8(ones), ones); + let halves = u32x4_splat(u16::MAX.into()); + compare_bytes(i32x4_extend_low_u16x8(ones), halves); + compare_bytes(i32x4_extend_high_u16x8(ones), halves); + + // FIXME wasmtime + // compare_bytes(i64x2_extend_low_i32x4_s(zero), zero); + // compare_bytes(i64x2_extend_high_i32x4_s(zero), zero); + // compare_bytes(i64x2_extend_low_i32x4_u(zero), zero); + // compare_bytes(i64x2_extend_high_i32x4_u(zero), zero); + // compare_bytes(i64x2_extend_low_i32x4_s(ones), ones); + // compare_bytes(i64x2_extend_high_i32x4_s(ones), ones); + // let halves = i64x2_splat(u32::MAX.into()); + // compare_bytes(i64x2_extend_low_i32x4_u(ones), halves); + // compare_bytes(i64x2_extend_high_i32x4_u(ones), halves); + } + + #[test] + fn test_dot() { + let zero = i8x16_splat(0); + let ones = i8x16_splat(!0); + let two = i32x4_splat(2); + compare_bytes(i32x4_dot_i16x8(zero, zero), zero); + compare_bytes(i32x4_dot_i16x8(ones, ones), two); + } + macro_rules! test_binop { ( $($name:ident => { - $([$($vec1:tt)*] ($op:tt | $f:ident) [$($vec2:tt)*],)* + $([$($vec1:tt)*] ($op:ident | $f:ident) [$($vec2:tt)*],)* })* ) => ($( #[test] @@ -2473,7 +4436,7 @@ pub mod tests { v3 = mem::transmute(v3_v128); for (i, actual) in v3.iter().enumerate() { - let expected = (Wrapping(v1[i]) $op Wrapping(v2[i])).0; + let expected = v1[i].$op(v2[i]); assert_eq!(*actual, expected); } )* @@ -2485,7 +4448,7 @@ pub mod tests { macro_rules! test_unop { ( $($name:ident => { - $(($op:tt | $f:ident) [$($vec1:tt)*],)* + $(($op:ident | $f:ident) [$($vec1:tt)*],)* })* ) => ($( #[test] @@ -2500,7 +4463,7 @@ pub mod tests { v2 = mem::transmute(v2_v128); for (i, actual) in v2.iter().enumerate() { - let expected = ($op Wrapping(v1[i])).0; + let expected = v1[i].$op(); assert_eq!(*actual, expected); } )* @@ -2509,132 +4472,583 @@ pub mod tests { )*) } + trait Avgr: Sized { + fn avgr(self, other: Self) -> Self; + } + + macro_rules! impl_avgr { + ($($i:ident)*) => ($(impl Avgr for $i { + fn avgr(self, other: Self) -> Self { + ((self as u64 + other as u64 + 1) / 2) as $i + } + })*) + } + + impl_avgr!(u8 u16); + test_binop! { test_i8x16_add => { [0i8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] - (+ | i8x16_add) + (wrapping_add | i8x16_add) + [1i8, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], + + [1i8, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16] + (wrapping_add | i8x16_add) + [-2, -3, -4, -5, -6, -7, -8, -9, -10, -11, -12, -13, -14, -15, -16, -18], + + [1i8, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16] + (wrapping_add | i8x16_add) + [127, -44, 43, 126, 4, 2, 9, -3, -59, -43, 39, -69, 79, -3, 9, -24], + } + + test_i8x16_add_sat_s => { + [0i8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] + (saturating_add | i8x16_add_sat) [1i8, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1i8, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16] - (+ | i8x16_add) + (saturating_add | i8x16_add_sat) [-2, -3, -4, -5, -6, -7, -8, -9, -10, -11, -12, -13, -14, -15, -16, -18], [1i8, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16] - (+ | i8x16_add) + (saturating_add | i8x16_add_sat) [127, -44, 43, 126, 4, 2, 9, -3, -59, -43, 39, -69, 79, -3, 9, -24], } + + test_i8x16_add_sat_u => { + [0u8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] + (saturating_add | u8x16_add_sat) + [1u8, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], + + [1u8, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16] + (saturating_add | u8x16_add_sat) + [255, 254, 253, 252, 251, 250, 249, 248, 247, 246, 245, 244, 243, 242, 241, 240], + + [1u8, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16] + (saturating_add | u8x16_add_sat) + [127, -44i8 as u8, 43, 126, 4, 2, 9, -3i8 as u8, -59i8 as u8, -43i8 as u8, 39, -69i8 as u8, 79, -3i8 as u8, 9, -24i8 as u8], + } + test_i8x16_sub => { [0i8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] - (- | i8x16_sub) + (wrapping_sub | i8x16_sub) + [1i8, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], + + [1i8, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16] + (wrapping_sub | i8x16_sub) + [-2, -3, -4, -5, -6, -7, -8, -9, -10, -11, -12, -13, -14, -15, -16, -18], + + [1i8, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16] + (wrapping_sub | i8x16_sub) + [-127, -44, 43, 126, 4, 2, 9, -3, -59, -43, 39, -69, 79, -3, 4, 8], + } + + test_i8x16_sub_sat_s => { + [0i8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] + (saturating_sub | i8x16_sub_sat) + [1i8, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], + + [1i8, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16] + (saturating_sub | i8x16_sub_sat) + [-2, -3, -4, -5, -6, -7, -8, -9, -10, -11, -12, -13, -14, -15, -16, -18], + + [1i8, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16] + (saturating_sub | i8x16_sub_sat) + [-127, -44, 43, 126, 4, 2, 9, -3, -59, -43, 39, -69, 79, -3, 4, 8], + } + + test_i8x16_sub_sat_u => { + [0u8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] + (saturating_sub | u8x16_sub_sat) + [1u8, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], + + [1u8, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16] + (saturating_sub | u8x16_sub_sat) + [255, 254, 253, 252, 251, 250, 249, 248, 247, 246, 245, 244, 243, 242, 241, 240], + + [1u8, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16] + (saturating_sub | u8x16_sub_sat) + [127, -44i8 as u8, 43, 126, 4, 2, 9, -3i8 as u8, -59i8 as u8, -43i8 as u8, 39, -69i8 as u8, 79, -3i8 as u8, 9, -24i8 as u8], + } + + test_i8x16_min_s => { + [0i8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] + (min | i8x16_min) + [1i8, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], + + [1i8, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16] + (min | i8x16_min) + [-2, -3, -4, -5, -6, -7, -8, -9, -10, -11, -12, -13, -14, -15, -16, -18], + + [1i8, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16] + (min | i8x16_min) + [-127, -44, 43, 126, 4, 2, 9, -3, -59, -43, 39, -69, 79, -3, 4, 8], + } + + test_i8x16_min_u => { + [0u8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] + (min | u8x16_min) + [1u8, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], + + [1u8, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16] + (min | u8x16_min) + [255, 254, 253, 252, 251, 250, 249, 248, 247, 246, 245, 244, 243, 242, 241, 240], + + [1u8, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16] + (min | u8x16_min) + [127, -44i8 as u8, 43, 126, 4, 2, 9, -3i8 as u8, -59i8 as u8, -43i8 as u8, 39, -69i8 as u8, 79, -3i8 as u8, 9, -24i8 as u8], + } + + test_i8x16_max_s => { + [0i8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] + (max | i8x16_max) [1i8, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1i8, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16] - (- | i8x16_sub) + (max | i8x16_max) [-2, -3, -4, -5, -6, -7, -8, -9, -10, -11, -12, -13, -14, -15, -16, -18], [1i8, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16] - (- | i8x16_sub) + (max | i8x16_max) [-127, -44, 43, 126, 4, 2, 9, -3, -59, -43, 39, -69, 79, -3, 4, 8], } + test_i8x16_max_u => { + [0u8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] + (max | u8x16_max) + [1u8, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], + + [1u8, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16] + (max | u8x16_max) + [255, 254, 253, 252, 251, 250, 249, 248, 247, 246, 245, 244, 243, 242, 241, 240], + + [1u8, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16] + (max | u8x16_max) + [127, -44i8 as u8, 43, 126, 4, 2, 9, -3i8 as u8, -59i8 as u8, -43i8 as u8, 39, -69i8 as u8, 79, -3i8 as u8, 9, -24i8 as u8], + } + + test_i8x16_avgr_u => { + [0u8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] + (avgr | u8x16_avgr) + [1u8, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], + + [1u8, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16] + (avgr | u8x16_avgr) + [255, 254, 253, 252, 251, 250, 249, 248, 247, 246, 245, 244, 243, 242, 241, 240], + + [1u8, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16] + (avgr | u8x16_avgr) + [127, -44i8 as u8, 43, 126, 4, 2, 9, -3i8 as u8, -59i8 as u8, -43i8 as u8, 39, -69i8 as u8, 79, -3i8 as u8, 9, -24i8 as u8], + } + test_i16x8_add => { [0i16, 0, 0, 0, 0, 0, 0, 0] - (+ | i16x8_add) + (wrapping_add | i16x8_add) + [1i16, 1, 1, 1, 1, 1, 1, 1], + + [1i16, 2, 3, 4, 5, 6, 7, 8] + (wrapping_add | i16x8_add) + [32767, 8, -2494,-4, 4882, -4, 848, 3830], + } + + test_i16x8_add_sat_s => { + [0i16, 0, 0, 0, 0, 0, 0, 0] + (saturating_add | i16x8_add_sat) [1i16, 1, 1, 1, 1, 1, 1, 1], [1i16, 2, 3, 4, 5, 6, 7, 8] - (+ | i16x8_add) + (saturating_add | i16x8_add_sat) [32767, 8, -2494,-4, 4882, -4, 848, 3830], } + test_i16x8_add_sat_u => { + [0u16, 0, 0, 0, 0, 0, 0, 0] + (saturating_add | u16x8_add_sat) + [1u16, 1, 1, 1, 1, 1, 1, 1], + + [1u16, 2, 3, 4, 5, 6, 7, 8] + (saturating_add | u16x8_add_sat) + [32767, 8, -2494i16 as u16,-4i16 as u16, 4882, -4i16 as u16, 848, 3830], + } + test_i16x8_sub => { [0i16, 0, 0, 0, 0, 0, 0, 0] - (- | i16x8_sub) + (wrapping_sub | i16x8_sub) + [1i16, 1, 1, 1, 1, 1, 1, 1], + + [1i16, 2, 3, 4, 5, 6, 7, 8] + (wrapping_sub | i16x8_sub) + [32767, 8, -2494,-4, 4882, -4, 848, 3830], + } + + test_i16x8_sub_sat_s => { + [0i16, 0, 0, 0, 0, 0, 0, 0] + (saturating_sub | i16x8_sub_sat) [1i16, 1, 1, 1, 1, 1, 1, 1], [1i16, 2, 3, 4, 5, 6, 7, 8] - (- | i16x8_sub) + (saturating_sub | i16x8_sub_sat) [32767, 8, -2494,-4, 4882, -4, 848, 3830], } + test_i16x8_sub_sat_u => { + [0u16, 0, 0, 0, 0, 0, 0, 0] + (saturating_sub | u16x8_sub_sat) + [1u16, 1, 1, 1, 1, 1, 1, 1], + + [1u16, 2, 3, 4, 5, 6, 7, 8] + (saturating_sub | u16x8_sub_sat) + [32767, 8, -2494i16 as u16,-4i16 as u16, 4882, -4i16 as u16, 848, 3830], + } + test_i16x8_mul => { [0i16, 0, 0, 0, 0, 0, 0, 0] - (* | i16x8_mul) + (wrapping_mul | i16x8_mul) + [1i16, 1, 1, 1, 1, 1, 1, 1], + + [1i16, 2, 3, 4, 5, 6, 7, 8] + (wrapping_mul | i16x8_mul) + [32767, 8, -2494,-4, 4882, -4, 848, 3830], + } + + test_i16x8_min_s => { + [0i16, 0, 0, 0, 0, 0, 0, 0] + (min | i16x8_min) + [1i16, 1, 1, 1, 1, 1, 1, 1], + + [1i16, 2, 3, 4, 5, 6, 7, 8] + (min | i16x8_min) + [32767, 8, -2494,-4, 4882, -4, 848, 3830], + } + + test_i16x8_min_u => { + [0u16, 0, 0, 0, 0, 0, 0, 0] + (min | u16x8_min) + [1u16, 1, 1, 1, 1, 1, 1, 1], + + [1u16, 2, 3, 4, 5, 6, 7, 8] + (min | u16x8_min) + [32767, 8, -2494i16 as u16,-4i16 as u16, 4882, -4i16 as u16, 848, 3830], + } + + test_i16x8_max_s => { + [0i16, 0, 0, 0, 0, 0, 0, 0] + (max | i16x8_max) [1i16, 1, 1, 1, 1, 1, 1, 1], [1i16, 2, 3, 4, 5, 6, 7, 8] - (* | i16x8_mul) + (max | i16x8_max) [32767, 8, -2494,-4, 4882, -4, 848, 3830], } + test_i16x8_max_u => { + [0u16, 0, 0, 0, 0, 0, 0, 0] + (max | u16x8_max) + [1u16, 1, 1, 1, 1, 1, 1, 1], + + [1u16, 2, 3, 4, 5, 6, 7, 8] + (max | u16x8_max) + [32767, 8, -2494i16 as u16,-4i16 as u16, 4882, -4i16 as u16, 848, 3830], + } + + test_i16x8_avgr_u => { + [0u16, 0, 0, 0, 0, 0, 0, 0] + (avgr | u16x8_avgr) + [1u16, 1, 1, 1, 1, 1, 1, 1], + + [1u16, 2, 3, 4, 5, 6, 7, 8] + (avgr | u16x8_avgr) + [32767, 8, -2494i16 as u16,-4i16 as u16, 4882, -4i16 as u16, 848, 3830], + } + test_i32x4_add => { - [0i32, 0, 0, 0] (+ | i32x4_add) [1, 2, 3, 4], + [0i32, 0, 0, 0] (wrapping_add | i32x4_add) [1, 2, 3, 4], [1i32, 1283, i32::MAX, i32::MIN] - (+ | i32x4_add) + (wrapping_add | i32x4_add) [i32::MAX; 4], } test_i32x4_sub => { - [0i32, 0, 0, 0] (- | i32x4_sub) [1, 2, 3, 4], + [0i32, 0, 0, 0] (wrapping_sub | i32x4_sub) [1, 2, 3, 4], [1i32, 1283, i32::MAX, i32::MIN] - (- | i32x4_sub) + (wrapping_sub | i32x4_sub) [i32::MAX; 4], } test_i32x4_mul => { - [0i32, 0, 0, 0] (* | i32x4_mul) [1, 2, 3, 4], + [0i32, 0, 0, 0] (wrapping_mul | i32x4_mul) [1, 2, 3, 4], + [1i32, 1283, i32::MAX, i32::MIN] + (wrapping_mul | i32x4_mul) + [i32::MAX; 4], + } + + test_i32x4_min_s => { + [0i32, 0, 0, 0] (min | i32x4_min) [1, 2, 3, 4], + [1i32, 1283, i32::MAX, i32::MIN] + (min | i32x4_min) + [i32::MAX; 4], + } + + test_i32x4_min_u => { + [0u32, 0, 0, 0] (min | u32x4_min) [1, 2, 3, 4], + [1u32, 1283, i32::MAX as u32, i32::MIN as u32] + (min | u32x4_min) + [i32::MAX as u32; 4], + } + + test_i32x4_max_s => { + [0i32, 0, 0, 0] (max | i32x4_max) [1, 2, 3, 4], [1i32, 1283, i32::MAX, i32::MIN] - (* | i32x4_mul) + (max | i32x4_max) [i32::MAX; 4], } - // TODO: test_i64x2_add - // TODO: test_i64x2_sub + test_i32x4_max_u => { + [0u32, 0, 0, 0] (max | u32x4_max) [1, 2, 3, 4], + [1u32, 1283, i32::MAX as u32, i32::MIN as u32] + (max | u32x4_max) + [i32::MAX as u32; 4], + } + + test_i64x2_add => { + [0i64, 0] (wrapping_add | i64x2_add) [1, 2], + [i64::MIN, i64::MAX] (wrapping_add | i64x2_add) [i64::MAX, i64::MIN], + [i64::MAX; 2] (wrapping_add | i64x2_add) [i64::MAX; 2], + [-4i64, -4] (wrapping_add | i64x2_add) [800, 939], + } + + test_i64x2_sub => { + [0i64, 0] (wrapping_sub | i64x2_sub) [1, 2], + [i64::MIN, i64::MAX] (wrapping_sub | i64x2_sub) [i64::MAX, i64::MIN], + [i64::MAX; 2] (wrapping_sub | i64x2_sub) [i64::MAX; 2], + [-4i64, -4] (wrapping_sub | i64x2_sub) [800, 939], + } + + test_i64x2_mul => { + [0i64, 0] (wrapping_mul | i64x2_mul) [1, 2], + [i64::MIN, i64::MAX] (wrapping_mul | i64x2_mul) [i64::MAX, i64::MIN], + [i64::MAX; 2] (wrapping_mul | i64x2_mul) [i64::MAX; 2], + [-4i64, -4] (wrapping_mul | i64x2_mul) [800, 939], + } + + test_f32x4_add => { + [-1.0f32, 2.0, 3.0, 4.0] (add | f32x4_add) [1., 2., 0., 0.], + [f32::INFINITY, -0.0, f32::NEG_INFINITY, 3.0] + (add | f32x4_add) + [1., 2., 0., 0.], + } + + test_f32x4_sub => { + [-1.0f32, 2.0, 3.0, 4.0] (sub | f32x4_sub) [1., 2., 0., 0.], + [f32::INFINITY, -0.0, f32::NEG_INFINITY, 3.0] + (sub | f32x4_sub) + [1., 2., 0., 0.], + } + + test_f32x4_mul => { + [-1.0f32, 2.0, 3.0, 4.0] (mul | f32x4_mul) [1., 2., 0., 0.], + [f32::INFINITY, -0.0, f32::NEG_INFINITY, 3.0] + (mul | f32x4_mul) + [1., 2., 1., 0.], + } + + test_f32x4_div => { + [-1.0f32, 2.0, 3.0, 4.0] (div | f32x4_div) [1., 2., 0., 0.], + [f32::INFINITY, -0.0, f32::NEG_INFINITY, 3.0] + (div | f32x4_div) + [1., 2., 0., 0.], + } + + test_f32x4_min => { + [-1.0f32, 2.0, 3.0, 4.0] (min | f32x4_min) [1., 2., 0., 0.], + [f32::INFINITY, -0.0, f32::NEG_INFINITY, 3.0] + (min | f32x4_min) + [1., 2., 0., 0.], + } + + test_f32x4_max => { + [-1.0f32, 2.0, 3.0, 4.0] (max | f32x4_max) [1., 2., 0., 0.], + [f32::INFINITY, -0.0, f32::NEG_INFINITY, 3.0] + (max | f32x4_max) + [1., 2., 0., 0.], + } + + test_f32x4_pmin => { + [-1.0f32, 2.0, 3.0, 4.0] (min | f32x4_pmin) [1., 2., 0., 0.], + [f32::INFINITY, -0.0, f32::NEG_INFINITY, 3.0] + (min | f32x4_pmin) + [1., 2., 0., 0.], + } + + test_f32x4_pmax => { + [-1.0f32, 2.0, 3.0, 4.0] (max | f32x4_pmax) [1., 2., 0., 0.], + [f32::INFINITY, -0.0, f32::NEG_INFINITY, 3.0] + (max | f32x4_pmax) + [1., 2., 0., 0.], + } + + test_f64x2_add => { + [-1.0f64, 2.0] (add | f64x2_add) [1., 2.], + [f64::INFINITY, f64::NEG_INFINITY] (add | f64x2_add) [1., 2.], + } + + test_f64x2_sub => { + [-1.0f64, 2.0] (sub | f64x2_sub) [1., 2.], + [f64::INFINITY, f64::NEG_INFINITY] (sub | f64x2_sub) [1., 2.], + } + + test_f64x2_mul => { + [-1.0f64, 2.0] (mul | f64x2_mul) [1., 2.], + [f64::INFINITY, f64::NEG_INFINITY] (mul | f64x2_mul) [1., 2.], + } + + test_f64x2_div => { + [-1.0f64, 2.0] (div | f64x2_div) [1., 2.], + [f64::INFINITY, f64::NEG_INFINITY] (div | f64x2_div) [1., 2.], + } + + test_f64x2_min => { + [-1.0f64, 2.0] (min | f64x2_min) [1., 2.], + [f64::INFINITY, f64::NEG_INFINITY] (min | f64x2_min) [1., 2.], + } + + test_f64x2_max => { + [-1.0f64, 2.0] (max | f64x2_max) [1., 2.], + [f64::INFINITY, f64::NEG_INFINITY] (max | f64x2_max) [1., 2.], + } + + test_f64x2_pmin => { + [-1.0f64, 2.0] (min | f64x2_pmin) [1., 2.], + [f64::INFINITY, f64::NEG_INFINITY] (min | f64x2_pmin) [1., 2.], + } + + test_f64x2_pmax => { + [-1.0f64, 2.0] (max | f64x2_pmax) [1., 2.], + [f64::INFINITY, f64::NEG_INFINITY] (max | f64x2_pmax) [1., 2.], + } } test_unop! { + test_i8x16_abs => { + (wrapping_abs | i8x16_abs) + [1i8, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], + + (wrapping_abs | i8x16_abs) + [-2i8, -3, -4, -5, -6, -7, -8, -9, -10, -11, -12, -13, -14, -15, -16, -18], + + (wrapping_abs | i8x16_abs) + [-127i8, -44, 43, 126, 4, -128, 127, -59, -43, 39, -69, 79, -3, 35, 83, 13], + } + test_i8x16_neg => { - (- | i8x16_neg) + (wrapping_neg | i8x16_neg) [1i8, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], - (- | i8x16_neg) + (wrapping_neg | i8x16_neg) [-2i8, -3, -4, -5, -6, -7, -8, -9, -10, -11, -12, -13, -14, -15, -16, -18], - (- | i8x16_neg) + (wrapping_neg | i8x16_neg) [-127i8, -44, 43, 126, 4, -128, 127, -59, -43, 39, -69, 79, -3, 35, 83, 13], } + test_i16x8_abs => { + (wrapping_abs | i16x8_abs) [1i16, 1, 1, 1, 1, 1, 1, 1], + (wrapping_abs | i16x8_abs) [2i16, 0x7fff, !0, 4, 42, -5, 33, -4847], + } + test_i16x8_neg => { - (- | i16x8_neg) [1i16, 1, 1, 1, 1, 1, 1, 1], - (- | i16x8_neg) [2i16, 0x7fff, !0, 4, 42, -5, 33, -4847], + (wrapping_neg | i16x8_neg) [1i16, 1, 1, 1, 1, 1, 1, 1], + (wrapping_neg | i16x8_neg) [2i16, 0x7fff, !0, 4, 42, -5, 33, -4847], + } + + test_i32x4_abs => { + (wrapping_abs | i32x4_abs) [1i32, 2, 3, 4], + (wrapping_abs | i32x4_abs) [i32::MIN, i32::MAX, 0, 4], } test_i32x4_neg => { - (- | i32x4_neg) [1i32, 2, 3, 4], - (- | i32x4_neg) [i32::MIN, i32::MAX, 0, 4], + (wrapping_neg | i32x4_neg) [1i32, 2, 3, 4], + (wrapping_neg | i32x4_neg) [i32::MIN, i32::MAX, 0, 4], } - // TODO: test_i64x2_neg - } + test_i64x2_abs => { + (wrapping_abs | i64x2_abs) [1i64, 2], + (wrapping_abs | i64x2_abs) [i64::MIN, i64::MAX], + } - #[test] - fn test_v8x16_shuffle() { - unsafe { - let a = [0_u8, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]; - let b = [ - 16_u8, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, - ]; + test_i64x2_neg => { + (wrapping_neg | i64x2_neg) [1i64, 2], + (wrapping_neg | i64x2_neg) [i64::MIN, i64::MAX], + } - let vec_a: v128 = transmute(a); - let vec_b: v128 = transmute(b); + test_f32x4_ceil => { + (ceil | f32x4_ceil) [1.0f32, 2., 2.5, 3.3], + (ceil | f32x4_ceil) [0.0, -0.3, f32::INFINITY, -0.0], + } - let vec_r = v8x16_shuffle::<0, 16, 2, 18, 4, 20, 6, 22, 8, 24, 10, 26, 12, 28, 14, 30>( - vec_a, vec_b, - ); + test_f32x4_floor => { + (floor | f32x4_floor) [1.0f32, 2., 2.5, 3.3], + (floor | f32x4_floor) [0.0, -0.3, f32::INFINITY, -0.0], + } + + test_f32x4_trunc => { + (trunc | f32x4_trunc) [1.0f32, 2., 2.5, 3.3], + (trunc | f32x4_trunc) [0.0, -0.3, f32::INFINITY, -0.0], + } + + test_f32x4_nearest => { + (round | f32x4_nearest) [1.0f32, 2., 2.6, 3.3], + (round | f32x4_nearest) [0.0, -0.3, f32::INFINITY, -0.0], + } + + test_f32x4_abs => { + (abs | f32x4_abs) [1.0f32, 2., 2.6, 3.3], + (abs | f32x4_abs) [0.0, -0.3, f32::INFINITY, -0.0], + } + + test_f32x4_neg => { + (neg | f32x4_neg) [1.0f32, 2., 2.6, 3.3], + (neg | f32x4_neg) [0.0, -0.3, f32::INFINITY, -0.0], + } + + test_f32x4_sqrt => { + (sqrt | f32x4_sqrt) [1.0f32, 2., 2.6, 3.3], + (sqrt | f32x4_sqrt) [0.0, 0.3, f32::INFINITY, 0.1], + } + + test_f64x2_ceil => { + (ceil | f64x2_ceil) [1.0f64, 2.3], + (ceil | f64x2_ceil) [f64::INFINITY, -0.1], + } + + test_f64x2_floor => { + (floor | f64x2_floor) [1.0f64, 2.3], + (floor | f64x2_floor) [f64::INFINITY, -0.1], + } + + test_f64x2_trunc => { + (trunc | f64x2_trunc) [1.0f64, 2.3], + (trunc | f64x2_trunc) [f64::INFINITY, -0.1], + } - let e = [0_u8, 16, 2, 18, 4, 20, 6, 22, 8, 24, 10, 26, 12, 28, 14, 30]; - let vec_e: v128 = transmute(e); - compare_bytes(vec_r, vec_e); + test_f64x2_nearest => { + (round | f64x2_nearest) [1.0f64, 2.3], + (round | f64x2_nearest) [f64::INFINITY, -0.1], + } + + test_f64x2_abs => { + (abs | f64x2_abs) [1.0f64, 2.3], + (abs | f64x2_abs) [f64::INFINITY, -0.1], + } + + test_f64x2_neg => { + (neg | f64x2_neg) [1.0f64, 2.3], + (neg | f64x2_neg) [f64::INFINITY, -0.1], + } + + test_f64x2_sqrt => { + (sqrt | f64x2_sqrt) [1.0f64, 2.3], + (sqrt | f64x2_sqrt) [f64::INFINITY, 0.1], } } @@ -2753,26 +5167,26 @@ pub mod tests { test_bops!(i64x2[i64; 2] | i64x2_shl[i64x2_shl_test]: ([0, -1], 1) => [0, -2]); - test_bops!(i8x16[i8; 16] | i8x16_shr_s[i8x16_shr_s_test]: + test_bops!(i8x16[i8; 16] | i8x16_shr[i8x16_shr_s_test]: ([0, -1, 2, 3, 4, 5, 6, i8::MAX, 1, 1, 1, 1, 1, 1, 1, 1], 1) => [0, -1, 1, 1, 2, 2, 3, 63, 0, 0, 0, 0, 0, 0, 0, 0]); - test_bops!(i16x8[i16; 8] | i16x8_shr_s[i16x8_shr_s_test]: + test_bops!(i16x8[i16; 8] | i16x8_shr[i16x8_shr_s_test]: ([0, -1, 2, 3, 4, 5, 6, i16::MAX], 1) => [0, -1, 1, 1, 2, 2, 3, i16::MAX / 2]); - test_bops!(i32x4[i32; 4] | i32x4_shr_s[i32x4_shr_s_test]: + test_bops!(i32x4[i32; 4] | i32x4_shr[i32x4_shr_s_test]: ([0, -1, 2, 3], 1) => [0, -1, 1, 1]); - test_bops!(i64x2[i64; 2] | i64x2_shr_s[i64x2_shr_s_test]: + test_bops!(i64x2[i64; 2] | i64x2_shr[i64x2_shr_s_test]: ([0, -1], 1) => [0, -1]); - test_bops!(i8x16[i8; 16] | i8x16_shr_u[i8x16_uhr_u_test]: + test_bops!(i8x16[i8; 16] | u8x16_shr[i8x16_uhr_u_test]: ([0, -1, 2, 3, 4, 5, 6, i8::MAX, 1, 1, 1, 1, 1, 1, 1, 1], 1) => [0, i8::MAX, 1, 1, 2, 2, 3, 63, 0, 0, 0, 0, 0, 0, 0, 0]); - test_bops!(i16x8[i16; 8] | i16x8_shr_u[i16x8_uhr_u_test]: + test_bops!(i16x8[i16; 8] | u16x8_shr[i16x8_uhr_u_test]: ([0, -1, 2, 3, 4, 5, 6, i16::MAX], 1) => [0, i16::MAX, 1, 1, 2, 2, 3, i16::MAX / 2]); - test_bops!(i32x4[i32; 4] | i32x4_shr_u[i32x4_uhr_u_test]: + test_bops!(i32x4[i32; 4] | u32x4_shr[i32x4_uhr_u_test]: ([0, -1, 2, 3], 1) => [0, i32::MAX, 1, 1]); - test_bops!(i64x2[i64; 2] | i64x2_shr_u[i64x2_uhr_u_test]: + test_bops!(i64x2[i64; 2] | u64x2_shr[i64x2_uhr_u_test]: ([0, -1], 1) => [0, i64::MAX]); #[test] @@ -2790,6 +5204,12 @@ pub mod tests { compare_bytes(r, vec_a); let r: v128 = v128_and(vec_a, vec_b); compare_bytes(r, vec_a); + let r: v128 = v128_andnot(vec_a, vec_b); + compare_bytes(r, vec_c); + let r: v128 = v128_andnot(vec_a, vec_a); + compare_bytes(r, vec_c); + let r: v128 = v128_andnot(vec_a, vec_c); + compare_bytes(r, vec_a); let r: v128 = v128_or(vec_a, vec_b); compare_bytes(r, vec_b); let r: v128 = v128_not(vec_b); @@ -2815,36 +5235,43 @@ pub mod tests { let vec_b: v128 = transmute([$($false),*]); // false let vec_c: v128 = transmute([$($alt),*]); // alternating - assert_eq!($any(vec_a), 1); - assert_eq!($any(vec_b), 0); - assert_eq!($any(vec_c), 1); + // TODO + // assert_eq!($any(vec_a), true); + // assert_eq!($any(vec_b), false); + // assert_eq!($any(vec_c), true); - assert_eq!($all(vec_a), 1); - assert_eq!($all(vec_b), 0); - assert_eq!($all(vec_c), 0); + assert_eq!($all(vec_a), true); + assert_eq!($all(vec_b), false); + assert_eq!($all(vec_c), false); } } } } test_bool_red!( - [i8x16_boolean_reductions, i8x16_any_true, i8x16_all_true] + [i8x16_boolean_reductions, v128_any_true, i8x16_all_true] | [1_i8, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] | [0_i8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] | [1_i8, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0] ); test_bool_red!( - [i16x8_boolean_reductions, i16x8_any_true, i16x8_all_true] + [i16x8_boolean_reductions, v128_any_true, i16x8_all_true] | [1_i16, 1, 1, 1, 1, 1, 1, 1] | [0_i16, 0, 0, 0, 0, 0, 0, 0] | [1_i16, 0, 1, 0, 1, 0, 1, 0] ); test_bool_red!( - [i32x4_boolean_reductions, i32x4_any_true, i32x4_all_true] + [i32x4_boolean_reductions, v128_any_true, i32x4_all_true] | [1_i32, 1, 1, 1] | [0_i32, 0, 0, 0] | [1_i32, 0, 1, 0] ); + test_bool_red!( + [i64x2_boolean_reductions, v128_any_true, i64x2_all_true] + | [1_i64, 1] + | [0_i64, 0] + | [1_i64, 0] + ); test_bop!(i8x16[i8; 16] | i8x16_eq[i8x16_eq_test]: ([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15], @@ -2855,6 +5282,8 @@ pub mod tests { [-1, 0, -1, 0 ,-1, 0, -1, -1]); test_bop!(i32x4[i32; 4] | i32x4_eq[i32x4_eq_test]: ([0, 1, 2, 3], [0, 2, 2, 4]) => [-1, 0, -1, 0]); + test_bop!(i64x2[i64; 2] | i64x2_eq[i64x2_eq_test]: + ([0, 1], [0, 2]) => [-1, 0]); test_bop!(f32x4[f32; 4] => i32 | f32x4_eq[f32x4_eq_test]: ([0., 1., 2., 3.], [0., 2., 2., 4.]) => [-1, 0, -1, 0]); test_bop!(f64x2[f64; 2] => i64 | f64x2_eq[f64x2_eq_test]: ([0., 1.], [0., 2.]) => [-1, 0]); @@ -2868,76 +5297,110 @@ pub mod tests { [0, -1, 0, -1 ,0, -1, 0, 0]); test_bop!(i32x4[i32; 4] | i32x4_ne[i32x4_ne_test]: ([0, 1, 2, 3], [0, 2, 2, 4]) => [0, -1, 0, -1]); + test_bop!(i64x2[i64; 2] | i64x2_ne[i64x2_ne_test]: + ([0, 1], [0, 2]) => [0, -1]); test_bop!(f32x4[f32; 4] => i32 | f32x4_ne[f32x4_ne_test]: ([0., 1., 2., 3.], [0., 2., 2., 4.]) => [0, -1, 0, -1]); test_bop!(f64x2[f64; 2] => i64 | f64x2_ne[f64x2_ne_test]: ([0., 1.], [0., 2.]) => [0, -1]); - test_bop!(i8x16[i8; 16] | i8x16_lt_s[i8x16_lt_test]: - ([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15], + test_bop!(i8x16[i8; 16] | i8x16_lt[i8x16_lt_s_test]: + ([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, -12, 13, 14, 15], + [0, 2, 2, 4, 4, 6, 6, 7, 8, 10, 10, 12, 12, 14, 14, 15]) => + [0, -1, 0, -1 ,0, -1, 0, 0, 0, -1, 0, -1, -1, -1, 0, 0]); + test_bop!(i8x16[i8; 16] | u8x16_lt[i8x16_lt_u_test]: + ([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, -12, 13, 14, 15], [0, 2, 2, 4, 4, 6, 6, 7, 8, 10, 10, 12, 12, 14, 14, 15]) => [0, -1, 0, -1 ,0, -1, 0, 0, 0, -1, 0, -1 ,0, -1, 0, 0]); - test_bop!(i16x8[i16; 8] | i16x8_lt_s[i16x8_lt_test]: - ([0, 1, 2, 3, 4, 5, 6, 7], [0, 2, 2, 4, 4, 6, 6, 7]) => + test_bop!(i16x8[i16; 8] | i16x8_lt[i16x8_lt_s_test]: + ([0, 1, 2, 3, 4, 5, 6, -7], [0, 2, 2, 4, 4, 6, 6, 7]) => + [0, -1, 0, -1 ,0, -1, 0, -1]); + test_bop!(i16x8[i16; 8] | u16x8_lt[i16x8_lt_u_test]: + ([0, 1, 2, 3, 4, 5, 6, -7], [0, 2, 2, 4, 4, 6, 6, 7]) => [0, -1, 0, -1 ,0, -1, 0, 0]); - test_bop!(i32x4[i32; 4] | i32x4_lt_s[i32x4_lt_test]: - ([0, 1, 2, 3], [0, 2, 2, 4]) => [0, -1, 0, -1]); + test_bop!(i32x4[i32; 4] | i32x4_lt[i32x4_lt_s_test]: + ([-1, 1, 2, 3], [0, 2, 2, 4]) => [-1, -1, 0, -1]); + test_bop!(i32x4[i32; 4] | u32x4_lt[i32x4_lt_u_test]: + ([-1, 1, 2, 3], [0, 2, 2, 4]) => [0, -1, 0, -1]); + test_bop!(i64x2[i64; 2] | i64x2_lt[i64x2_lt_s_test]: + ([-1, 3], [0, 2]) => [-1, 0]); test_bop!(f32x4[f32; 4] => i32 | f32x4_lt[f32x4_lt_test]: ([0., 1., 2., 3.], [0., 2., 2., 4.]) => [0, -1, 0, -1]); test_bop!(f64x2[f64; 2] => i64 | f64x2_lt[f64x2_lt_test]: ([0., 1.], [0., 2.]) => [0, -1]); - test_bop!(i8x16[i8; 16] | i8x16_gt_s[i8x16_gt_test]: - ([0, 2, 2, 4, 4, 6, 6, 7, 8, 10, 10, 12, 12, 14, 14, 15], + test_bop!(i8x16[i8; 16] | i8x16_gt[i8x16_gt_s_test]: + ([0, 2, 2, 4, 4, 6, 6, 7, 8, 10, 10, 12, 12, 14, 14, -15], [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]) => [0, -1, 0, -1 ,0, -1, 0, 0, 0, -1, 0, -1 ,0, -1, 0, 0]); - test_bop!(i16x8[i16; 8] | i16x8_gt_s[i16x8_gt_test]: - ([0, 2, 2, 4, 4, 6, 6, 7], [0, 1, 2, 3, 4, 5, 6, 7]) => + test_bop!(i8x16[i8; 16] | u8x16_gt[i8x16_gt_u_test]: + ([0, 2, 2, 4, 4, 6, 6, 7, 8, 10, 10, 12, 12, 14, 14, -15], + [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]) => + [0, -1, 0, -1 ,0, -1, 0, 0, 0, -1, 0, -1 ,0, -1, 0, -1]); + test_bop!(i16x8[i16; 8] | i16x8_gt[i16x8_gt_s_test]: + ([0, 2, 2, 4, 4, 6, 6, -7], [0, 1, 2, 3, 4, 5, 6, 7]) => [0, -1, 0, -1 ,0, -1, 0, 0]); - test_bop!(i32x4[i32; 4] | i32x4_gt_s[i32x4_gt_test]: - ([0, 2, 2, 4], [0, 1, 2, 3]) => [0, -1, 0, -1]); + test_bop!(i16x8[i16; 8] | u16x8_gt[i16x8_gt_u_test]: + ([0, 2, 2, 4, 4, 6, 6, -7], [0, 1, 2, 3, 4, 5, 6, 7]) => + [0, -1, 0, -1 ,0, -1, 0, -1]); + test_bop!(i32x4[i32; 4] | i32x4_gt[i32x4_gt_s_test]: + ([0, 2, 2, -4], [0, 1, 2, 3]) => [0, -1, 0, 0]); + test_bop!(i32x4[i32; 4] | u32x4_gt[i32x4_gt_u_test]: + ([0, 2, 2, -4], [0, 1, 2, 3]) => [0, -1, 0, -1]); + test_bop!(i64x2[i64; 2] | i64x2_gt[i64x2_gt_s_test]: + ([-1, 2], [0, 1]) => [0, -1]); test_bop!(f32x4[f32; 4] => i32 | f32x4_gt[f32x4_gt_test]: ([0., 2., 2., 4.], [0., 1., 2., 3.]) => [0, -1, 0, -1]); test_bop!(f64x2[f64; 2] => i64 | f64x2_gt[f64x2_gt_test]: ([0., 2.], [0., 1.]) => [0, -1]); - test_bop!(i8x16[i8; 16] | i8x16_ge_s[i8x16_ge_test]: - ([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15], + test_bop!(i8x16[i8; 16] | i8x16_ge[i8x16_ge_s_test]: + ([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, -15], + [0, 2, 2, 4, 4, 6, 6, 7, 8, 10, 10, 12, 12, 14, 14, 15]) => + [-1, 0, -1, 0 ,-1, 0, -1, -1, -1, 0, -1, 0 ,-1, 0, -1, 0]); + test_bop!(i8x16[i8; 16] | u8x16_ge[i8x16_ge_u_test]: + ([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, -15], [0, 2, 2, 4, 4, 6, 6, 7, 8, 10, 10, 12, 12, 14, 14, 15]) => [-1, 0, -1, 0 ,-1, 0, -1, -1, -1, 0, -1, 0 ,-1, 0, -1, -1]); - test_bop!(i16x8[i16; 8] | i16x8_ge_s[i16x8_ge_test]: - ([0, 1, 2, 3, 4, 5, 6, 7], [0, 2, 2, 4, 4, 6, 6, 7]) => + test_bop!(i16x8[i16; 8] | i16x8_ge[i16x8_ge_s_test]: + ([0, 1, 2, 3, 4, 5, 6, -7], [0, 2, 2, 4, 4, 6, 6, 7]) => + [-1, 0, -1, 0 ,-1, 0, -1, 0]); + test_bop!(i16x8[i16; 8] | u16x8_ge[i16x8_ge_u_test]: + ([0, 1, 2, 3, 4, 5, 6, -7], [0, 2, 2, 4, 4, 6, 6, 7]) => [-1, 0, -1, 0 ,-1, 0, -1, -1]); - test_bop!(i32x4[i32; 4] | i32x4_ge_s[i32x4_ge_test]: - ([0, 1, 2, 3], [0, 2, 2, 4]) => [-1, 0, -1, 0]); + test_bop!(i32x4[i32; 4] | i32x4_ge[i32x4_ge_s_test]: + ([0, 1, 2, -3], [0, 2, 2, 4]) => [-1, 0, -1, 0]); + test_bop!(i32x4[i32; 4] | u32x4_ge[i32x4_ge_u_test]: + ([0, 1, 2, -3], [0, 2, 2, 4]) => [-1, 0, -1, -1]); + test_bop!(i64x2[i64; 2] | i64x2_ge[i64x2_ge_s_test]: + ([0, 1], [-1, 2]) => [-1, 0]); test_bop!(f32x4[f32; 4] => i32 | f32x4_ge[f32x4_ge_test]: ([0., 1., 2., 3.], [0., 2., 2., 4.]) => [-1, 0, -1, 0]); test_bop!(f64x2[f64; 2] => i64 | f64x2_ge[f64x2_ge_test]: ([0., 1.], [0., 2.]) => [-1, 0]); - test_bop!(i8x16[i8; 16] | i8x16_le_s[i8x16_le_test]: - ([0, 2, 2, 4, 4, 6, 6, 7, 8, 10, 10, 12, 12, 14, 14, 15], + test_bop!(i8x16[i8; 16] | i8x16_le[i8x16_le_s_test]: + ([0, 2, 2, 4, 4, 6, 6, 7, 8, 10, 10, 12, 12, 14, 14, -15], [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15] ) => [-1, 0, -1, 0 ,-1, 0, -1, -1, -1, 0, -1, 0 ,-1, 0, -1, -1]); - test_bop!(i16x8[i16; 8] | i16x8_le_s[i16x8_le_test]: - ([0, 2, 2, 4, 4, 6, 6, 7], [0, 1, 2, 3, 4, 5, 6, 7]) => + test_bop!(i8x16[i8; 16] | u8x16_le[i8x16_le_u_test]: + ([0, 2, 2, 4, 4, 6, 6, 7, 8, 10, 10, 12, 12, 14, 14, -15], + [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15] + ) => + [-1, 0, -1, 0 ,-1, 0, -1, -1, -1, 0, -1, 0 ,-1, 0, -1, 0]); + test_bop!(i16x8[i16; 8] | i16x8_le[i16x8_le_s_test]: + ([0, 2, 2, 4, 4, 6, 6, -7], [0, 1, 2, 3, 4, 5, 6, 7]) => [-1, 0, -1, 0 ,-1, 0, -1, -1]); - test_bop!(i32x4[i32; 4] | i32x4_le_s[i32x4_le_test]: - ([0, 2, 2, 4], [0, 1, 2, 3]) => [-1, 0, -1, 0]); + test_bop!(i16x8[i16; 8] | u16x8_le[i16x8_le_u_test]: + ([0, 2, 2, 4, 4, 6, 6, -7], [0, 1, 2, 3, 4, 5, 6, 7]) => + [-1, 0, -1, 0 ,-1, 0, -1, 0]); + test_bop!(i32x4[i32; 4] | i32x4_le[i32x4_le_s_test]: + ([0, 2, 2, -4], [0, 1, 2, 3]) => [-1, 0, -1, -1]); + test_bop!(i32x4[i32; 4] | u32x4_le[i32x4_le_u_test]: + ([0, 2, 2, -4], [0, 1, 2, 3]) => [-1, 0, -1, 0]); + test_bop!(i64x2[i64; 2] | i64x2_le[i64x2_le_s_test]: + ([0, 2], [0, 1]) => [-1, 0]); test_bop!(f32x4[f32; 4] => i32 | f32x4_le[f32x4_le_test]: ([0., 2., 2., 4.], [0., 1., 2., 3.]) => [-1, 0, -1, -0]); test_bop!(f64x2[f64; 2] => i64 | f64x2_le[f64x2_le_test]: ([0., 2.], [0., 1.]) => [-1, 0]); - #[test] - fn v128_bitwise_load_store() { - unsafe { - let mut arr: [i32; 4] = [0, 1, 2, 3]; - - let vec = v128_load(arr.as_ptr() as *const v128); - let vec = i32x4_add(vec, vec); - v128_store(arr.as_mut_ptr() as *mut v128, vec); - - assert_eq!(arr, [0, 2, 4, 6]); - } - } - test_uop!(f32x4[f32; 4] | f32x4_neg[f32x4_neg_test]: [0., 1., 2., 3.] => [ 0., -1., -2., -3.]); test_uop!(f32x4[f32; 4] | f32x4_abs[f32x4_abs_test]: [0., -1., 2., -3.] => [ 0., 1., 2., 3.]); test_bop!(f32x4[f32; 4] | f32x4_min[f32x4_min_test]: @@ -2997,21 +5460,28 @@ pub mod tests { } test_conv!( - f32x4_convert_s_i32x4 | f32x4_convert_i32x4_s | f32x4 | [1_i32, 2, 3, 4], + f32x4_convert_s_i32x4 | f32x4_convert_i32x4 | f32x4 | [1_i32, 2, 3, 4], [1_f32, 2., 3., 4.] ); test_conv!( - f32x4_convert_u_i32x4 | f32x4_convert_i32x4_u | f32x4 | [u32::MAX, 2, 3, 4], + f32x4_convert_u_i32x4 | f32x4_convert_u32x4 | f32x4 | [u32::MAX, 2, 3, 4], [u32::MAX as f32, 2., 3., 4.] ); - // FIXME: this fails, and produces 0 instead of saturating at i32::MAX - // test_conv!( - // i32x4_trunc_s_f32x4_sat - // | i32x4_trunc_sat_f32x4_s - // | i32x4 - // | [f32::NAN, 2., (i32::MAX as f32 + 1.), 4.], - // [0, 2, i32::MAX, 4] - // ); - // FIXME: add other saturating tests + #[test] + fn test_conversions() { + compare_bytes( + i32x4_trunc_sat_f32x4(f32x4(1., f32::NEG_INFINITY, f32::INFINITY, f32::NAN)), + i32x4(1, i32::MIN, i32::MAX, 0), + ); + compare_bytes( + u32x4_trunc_sat_f32x4(f32x4(1., f32::NEG_INFINITY, f32::INFINITY, f32::NAN)), + u32x4(1, 0, u32::MAX, 0), + ); + compare_bytes(f64x2_convert_low_i32x4(i32x4(1, 2, 3, 4)), f64x2(1., 2.)); + compare_bytes( + f64x2_convert_low_i32x4(i32x4(i32::MIN, i32::MAX, 3, 4)), + f64x2(f64::from(i32::MIN), f64::from(i32::MAX)), + ); + } } diff --git a/library/stdarch/crates/core_arch/src/x86/aes.rs b/library/stdarch/crates/core_arch/src/x86/aes.rs index 603744aef6..ffded1a0dc 100644 --- a/library/stdarch/crates/core_arch/src/x86/aes.rs +++ b/library/stdarch/crates/core_arch/src/x86/aes.rs @@ -87,21 +87,17 @@ pub unsafe fn _mm_aesimc_si128(a: __m128i) -> __m128i { /// /// Assist in expanding the AES cipher key by computing steps towards /// generating a round key for encryption cipher using data from `a` and an -/// 8-bit round constant `imm8`. +/// 8-bit round constant `IMM8`. /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_aeskeygenassist_si128) #[inline] #[target_feature(enable = "aes")] -#[cfg_attr(test, assert_instr(aeskeygenassist, imm8 = 0))] -#[rustc_args_required_const(1)] +#[cfg_attr(test, assert_instr(aeskeygenassist, IMM8 = 0))] +#[rustc_legacy_const_generics(1)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm_aeskeygenassist_si128(a: __m128i, imm8: i32) -> __m128i { - macro_rules! call { - ($imm8:expr) => { - aeskeygenassist(a, $imm8) - }; - } - constify_imm8!(imm8, call) +pub unsafe fn _mm_aeskeygenassist_si128(a: __m128i) -> __m128i { + static_assert_imm8!(IMM8); + aeskeygenassist(a, IMM8 as u8) } #[cfg(test)] @@ -169,7 +165,7 @@ mod tests { // Constants taken from https://msdn.microsoft.com/en-us/library/cc714138.aspx. let a = _mm_set_epi64x(0x0123456789abcdef, 0x8899aabbccddeeff); let e = _mm_set_epi64x(0x857c266b7c266e85, 0xeac4eea9c4eeacea); - let r = _mm_aeskeygenassist_si128(a, 5); + let r = _mm_aeskeygenassist_si128::<5>(a); assert_eq_m128i(r, e); } } diff --git a/library/stdarch/crates/core_arch/src/x86/avx.rs b/library/stdarch/crates/core_arch/src/x86/avx.rs index 6c9a03322d..88d5c170df 100644 --- a/library/stdarch/crates/core_arch/src/x86/avx.rs +++ b/library/stdarch/crates/core_arch/src/x86/avx.rs @@ -113,44 +113,21 @@ pub unsafe fn _mm256_or_ps(a: __m256, b: __m256) -> __m256 { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_shuffle_pd) #[inline] #[target_feature(enable = "avx")] -#[cfg_attr(test, assert_instr(vshufpd, imm8 = 0x1))] -#[rustc_args_required_const(2)] -#[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm256_shuffle_pd(a: __m256d, b: __m256d, imm8: i32) -> __m256d { - let imm8 = (imm8 & 0xFF) as u8; - macro_rules! shuffle4 { - ($a:expr, $b:expr, $c:expr, $d:expr) => { - simd_shuffle4(a, b, [$a, $b, $c, $d]) - }; - } - macro_rules! shuffle3 { - ($a:expr, $b:expr, $c:expr) => { - match (imm8 >> 3) & 0x1 { - 0 => shuffle4!($a, $b, $c, 6), - _ => shuffle4!($a, $b, $c, 7), - } - }; - } - macro_rules! shuffle2 { - ($a:expr, $b:expr) => { - match (imm8 >> 2) & 0x1 { - 0 => shuffle3!($a, $b, 2), - _ => shuffle3!($a, $b, 3), - } - }; - } - macro_rules! shuffle1 { - ($a:expr) => { - match (imm8 >> 1) & 0x1 { - 0 => shuffle2!($a, 4), - _ => shuffle2!($a, 5), - } - }; - } - match imm8 & 0x1 { - 0 => shuffle1!(0), - _ => shuffle1!(1), - } +#[cfg_attr(test, assert_instr(vshufpd, MASK = 3))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "simd_x86", since = "1.27.0")] +pub unsafe fn _mm256_shuffle_pd(a: __m256d, b: __m256d) -> __m256d { + static_assert_imm8!(MASK); + simd_shuffle4!( + a, + b, + [ + MASK as u32 & 0b1, + ((MASK as u32 >> 1) & 0b1) + 4, + ((MASK as u32 >> 2) & 0b1) + 2, + ((MASK as u32 >> 3) & 0b1) + 6, + ], + ) } /// Shuffles single-precision (32-bit) floating-point elements in `a` within @@ -159,61 +136,25 @@ pub unsafe fn _mm256_shuffle_pd(a: __m256d, b: __m256d, imm8: i32) -> __m256d { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_shuffle_ps) #[inline] #[target_feature(enable = "avx")] -#[cfg_attr(test, assert_instr(vshufps, imm8 = 0x0))] -#[rustc_args_required_const(2)] -#[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm256_shuffle_ps(a: __m256, b: __m256, imm8: i32) -> __m256 { - let imm8 = (imm8 & 0xFF) as u8; - macro_rules! shuffle4 { - ( - $a:expr, - $b:expr, - $c:expr, - $d:expr, - $e:expr, - $f:expr, - $g:expr, - $h:expr - ) => { - simd_shuffle8(a, b, [$a, $b, $c, $d, $e, $f, $g, $h]) - }; - } - macro_rules! shuffle3 { - ($a:expr, $b:expr, $c:expr, $e:expr, $f:expr, $g:expr) => { - match (imm8 >> 6) & 0x3 { - 0 => shuffle4!($a, $b, $c, 8, $e, $f, $g, 12), - 1 => shuffle4!($a, $b, $c, 9, $e, $f, $g, 13), - 2 => shuffle4!($a, $b, $c, 10, $e, $f, $g, 14), - _ => shuffle4!($a, $b, $c, 11, $e, $f, $g, 15), - } - }; - } - macro_rules! shuffle2 { - ($a:expr, $b:expr, $e:expr, $f:expr) => { - match (imm8 >> 4) & 0x3 { - 0 => shuffle3!($a, $b, 8, $e, $f, 12), - 1 => shuffle3!($a, $b, 9, $e, $f, 13), - 2 => shuffle3!($a, $b, 10, $e, $f, 14), - _ => shuffle3!($a, $b, 11, $e, $f, 15), - } - }; - } - macro_rules! shuffle1 { - ($a:expr, $e:expr) => { - match (imm8 >> 2) & 0x3 { - 0 => shuffle2!($a, 0, $e, 4), - 1 => shuffle2!($a, 1, $e, 5), - 2 => shuffle2!($a, 2, $e, 6), - _ => shuffle2!($a, 3, $e, 7), - } - }; - } - match imm8 & 0x3 { - 0 => shuffle1!(0, 4), - 1 => shuffle1!(1, 5), - 2 => shuffle1!(2, 6), - _ => shuffle1!(3, 7), - } +#[cfg_attr(test, assert_instr(vshufps, MASK = 3))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "simd_x86", since = "1.27.0")] +pub unsafe fn _mm256_shuffle_ps(a: __m256, b: __m256) -> __m256 { + static_assert_imm8!(MASK); + simd_shuffle8!( + a, + b, + [ + MASK as u32 & 0b11, + (MASK as u32 >> 2) & 0b11, + ((MASK as u32 >> 4) & 0b11) + 8, + ((MASK as u32 >> 6) & 0b11) + 8, + (MASK as u32 & 0b11) + 4, + ((MASK as u32 >> 2) & 0b11) + 4, + ((MASK as u32 >> 4) & 0b11) + 12, + ((MASK as u32 >> 6) & 0b11) + 12, + ], + ) } /// Computes the bitwise NOT of packed double-precision (64-bit) floating-point @@ -255,7 +196,7 @@ pub unsafe fn _mm256_andnot_ps(a: __m256, b: __m256) -> __m256 { #[cfg_attr(test, assert_instr(vmaxpd))] #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm256_max_pd(a: __m256d, b: __m256d) -> __m256d { - simd_fmax(a, b) + vmaxpd(a, b) } /// Compares packed single-precision (32-bit) floating-point elements in `a` @@ -267,7 +208,7 @@ pub unsafe fn _mm256_max_pd(a: __m256d, b: __m256d) -> __m256d { #[cfg_attr(test, assert_instr(vmaxps))] #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm256_max_ps(a: __m256, b: __m256) -> __m256 { - simd_fmax(a, b) + vmaxps(a, b) } /// Compares packed double-precision (64-bit) floating-point elements @@ -279,7 +220,7 @@ pub unsafe fn _mm256_max_ps(a: __m256, b: __m256) -> __m256 { #[cfg_attr(test, assert_instr(vminpd))] #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm256_min_pd(a: __m256d, b: __m256d) -> __m256d { - simd_fmin(a, b) + vminpd(a, b) } /// Compares packed single-precision (32-bit) floating-point elements in `a` @@ -291,7 +232,7 @@ pub unsafe fn _mm256_min_pd(a: __m256d, b: __m256d) -> __m256d { #[cfg_attr(test, assert_instr(vminps))] #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm256_min_ps(a: __m256, b: __m256) -> __m256 { - simd_fmin(a, b) + vminps(a, b) } /// Multiplies packed double-precision (64-bit) floating-point elements @@ -391,7 +332,7 @@ pub unsafe fn _mm256_div_pd(a: __m256d, b: __m256d) -> __m256d { } /// Rounds packed double-precision (64-bit) floating point elements in `a` -/// according to the flag `b`. The value of `b` may be as follows: +/// according to the flag `ROUNDING`. The value of `ROUNDING` may be as follows: /// /// - `0x00`: Round to the nearest whole number. /// - `0x01`: Round down, toward negative infinity. @@ -405,16 +346,12 @@ pub unsafe fn _mm256_div_pd(a: __m256d, b: __m256d) -> __m256d { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_round_pd) #[inline] #[target_feature(enable = "avx")] -#[cfg_attr(test, assert_instr(vroundpd, b = 0x3))] -#[rustc_args_required_const(1)] +#[cfg_attr(test, assert_instr(vroundpd, ROUNDING = 0x3))] +#[rustc_legacy_const_generics(1)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm256_round_pd(a: __m256d, b: i32) -> __m256d { - macro_rules! call { - ($imm8:expr) => { - roundpd256(a, $imm8) - }; - } - constify_imm8!(b, call) +pub unsafe fn _mm256_round_pd(a: __m256d) -> __m256d { + static_assert_imm4!(ROUNDING); + roundpd256(a, ROUNDING) } /// Rounds packed double-precision (64-bit) floating point elements in `a` @@ -442,7 +379,7 @@ pub unsafe fn _mm256_floor_pd(a: __m256d) -> __m256d { } /// Rounds packed single-precision (32-bit) floating point elements in `a` -/// according to the flag `b`. The value of `b` may be as follows: +/// according to the flag `ROUNDING`. The value of `ROUNDING` may be as follows: /// /// - `0x00`: Round to the nearest whole number. /// - `0x01`: Round down, toward negative infinity. @@ -456,16 +393,12 @@ pub unsafe fn _mm256_floor_pd(a: __m256d) -> __m256d { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_round_ps) #[inline] #[target_feature(enable = "avx")] -#[cfg_attr(test, assert_instr(vroundps, b = 0x00))] -#[rustc_args_required_const(1)] +#[cfg_attr(test, assert_instr(vroundps, ROUNDING = 0x00))] +#[rustc_legacy_const_generics(1)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm256_round_ps(a: __m256, b: i32) -> __m256 { - macro_rules! call { - ($imm8:expr) => { - roundps256(a, $imm8) - }; - } - constify_imm8!(b, call) +pub unsafe fn _mm256_round_ps(a: __m256) -> __m256 { + static_assert_imm4!(ROUNDING); + roundps256(a, ROUNDING) } /// Rounds packed single-precision (32-bit) floating point elements in `a` @@ -525,44 +458,21 @@ pub unsafe fn _mm256_sqrt_pd(a: __m256d) -> __m256d { // Note: LLVM7 prefers single-precision blend instructions when // possible, see: https://bugs.llvm.org/show_bug.cgi?id=38194 // #[cfg_attr(test, assert_instr(vblendpd, imm8 = 9))] -#[cfg_attr(test, assert_instr(vblendps, imm8 = 9))] -#[rustc_args_required_const(2)] -#[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm256_blend_pd(a: __m256d, b: __m256d, imm8: i32) -> __m256d { - let imm8 = (imm8 & 0xFF) as u8; - macro_rules! blend4 { - ($a:expr, $b:expr, $c:expr, $d:expr) => { - simd_shuffle4(a, b, [$a, $b, $c, $d]) - }; - } - macro_rules! blend3 { - ($a:expr, $b:expr, $c:expr) => { - match imm8 & 0x8 { - 0 => blend4!($a, $b, $c, 3), - _ => blend4!($a, $b, $c, 7), - } - }; - } - macro_rules! blend2 { - ($a:expr, $b:expr) => { - match imm8 & 0x4 { - 0 => blend3!($a, $b, 2), - _ => blend3!($a, $b, 6), - } - }; - } - macro_rules! blend1 { - ($a:expr) => { - match imm8 & 0x2 { - 0 => blend2!($a, 1), - _ => blend2!($a, 5), - } - }; - } - match imm8 & 0x1 { - 0 => blend1!(0), - _ => blend1!(4), - } +#[cfg_attr(test, assert_instr(vblendps, IMM4 = 9))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "simd_x86", since = "1.27.0")] +pub unsafe fn _mm256_blend_pd(a: __m256d, b: __m256d) -> __m256d { + static_assert_imm4!(IMM4); + simd_shuffle4!( + a, + b, + [ + ((IMM4 as u32 >> 0) & 1) * 4 + 0, + ((IMM4 as u32 >> 1) & 1) * 4 + 1, + ((IMM4 as u32 >> 2) & 1) * 4 + 2, + ((IMM4 as u32 >> 3) & 1) * 4 + 3, + ], + ) } /// Blends packed single-precision (32-bit) floating-point elements from @@ -571,61 +481,25 @@ pub unsafe fn _mm256_blend_pd(a: __m256d, b: __m256d, imm8: i32) -> __m256d { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_blend_ps) #[inline] #[target_feature(enable = "avx")] -#[cfg_attr(test, assert_instr(vblendps, imm8 = 9))] -#[rustc_args_required_const(2)] -#[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm256_blend_ps(a: __m256, b: __m256, imm8: i32) -> __m256 { - let imm8 = (imm8 & 0xFF) as u8; - macro_rules! blend4 { - ( - $a:expr, - $b:expr, - $c:expr, - $d:expr, - $e:expr, - $f:expr, - $g:expr, - $h:expr - ) => { - simd_shuffle8(a, b, [$a, $b, $c, $d, $e, $f, $g, $h]) - }; - } - macro_rules! blend3 { - ($a:expr, $b:expr, $c:expr, $d:expr, $e:expr, $f:expr) => { - match (imm8 >> 6) & 0b11 { - 0b00 => blend4!($a, $b, $c, $d, $e, $f, 6, 7), - 0b01 => blend4!($a, $b, $c, $d, $e, $f, 14, 7), - 0b10 => blend4!($a, $b, $c, $d, $e, $f, 6, 15), - _ => blend4!($a, $b, $c, $d, $e, $f, 14, 15), - } - }; - } - macro_rules! blend2 { - ($a:expr, $b:expr, $c:expr, $d:expr) => { - match (imm8 >> 4) & 0b11 { - 0b00 => blend3!($a, $b, $c, $d, 4, 5), - 0b01 => blend3!($a, $b, $c, $d, 12, 5), - 0b10 => blend3!($a, $b, $c, $d, 4, 13), - _ => blend3!($a, $b, $c, $d, 12, 13), - } - }; - } - macro_rules! blend1 { - ($a:expr, $b:expr) => { - match (imm8 >> 2) & 0b11 { - 0b00 => blend2!($a, $b, 2, 3), - 0b01 => blend2!($a, $b, 10, 3), - 0b10 => blend2!($a, $b, 2, 11), - _ => blend2!($a, $b, 10, 11), - } - }; - } - match imm8 & 0b11 { - 0b00 => blend1!(0, 1), - 0b01 => blend1!(8, 1), - 0b10 => blend1!(0, 9), - _ => blend1!(8, 9), - } +#[cfg_attr(test, assert_instr(vblendps, IMM8 = 9))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "simd_x86", since = "1.27.0")] +pub unsafe fn _mm256_blend_ps(a: __m256, b: __m256) -> __m256 { + static_assert_imm8!(IMM8); + simd_shuffle8!( + a, + b, + [ + ((IMM8 as u32 >> 0) & 1) * 8 + 0, + ((IMM8 as u32 >> 1) & 1) * 8 + 1, + ((IMM8 as u32 >> 2) & 1) * 8 + 2, + ((IMM8 as u32 >> 3) & 1) * 8 + 3, + ((IMM8 as u32 >> 4) & 1) * 8 + 4, + ((IMM8 as u32 >> 5) & 1) * 8 + 5, + ((IMM8 as u32 >> 6) & 1) * 8 + 6, + ((IMM8 as u32 >> 7) & 1) * 8 + 7, + ], + ) } /// Blends packed double-precision (64-bit) floating-point elements from @@ -660,16 +534,12 @@ pub unsafe fn _mm256_blendv_ps(a: __m256, b: __m256, c: __m256) -> __m256 { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_dp_ps) #[inline] #[target_feature(enable = "avx")] -#[cfg_attr(test, assert_instr(vdpps, imm8 = 0x0))] -#[rustc_args_required_const(2)] +#[cfg_attr(test, assert_instr(vdpps, IMM8 = 0x0))] +#[rustc_legacy_const_generics(2)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm256_dp_ps(a: __m256, b: __m256, imm8: i32) -> __m256 { - macro_rules! call { - ($imm8:expr) => { - vdpps(a, b, $imm8) - }; - } - constify_imm8!(imm8, call) +pub unsafe fn _mm256_dp_ps(a: __m256, b: __m256) -> __m256 { + static_assert_imm8!(IMM8); + vdpps(a, b, IMM8) } /// Horizontal addition of adjacent pairs in the two packed vectors @@ -858,82 +728,66 @@ pub const _CMP_TRUE_US: i32 = 0x1f; /// Compares packed double-precision (64-bit) floating-point /// elements in `a` and `b` based on the comparison operand -/// specified by `imm8`. +/// specified by `IMM5`. /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cmp_pd) #[inline] #[target_feature(enable = "avx,sse2")] -#[cfg_attr(test, assert_instr(vcmpeqpd, imm8 = 0))] // TODO Validate vcmppd -#[rustc_args_required_const(2)] +#[cfg_attr(test, assert_instr(vcmpeqpd, IMM5 = 0))] // TODO Validate vcmppd +#[rustc_legacy_const_generics(2)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm_cmp_pd(a: __m128d, b: __m128d, imm8: i32) -> __m128d { - macro_rules! call { - ($imm8:expr) => { - vcmppd(a, b, $imm8) - }; - } - constify_imm6!(imm8, call) +pub unsafe fn _mm_cmp_pd(a: __m128d, b: __m128d) -> __m128d { + static_assert_imm5!(IMM5); + vcmppd(a, b, IMM5 as i8) } /// Compares packed double-precision (64-bit) floating-point /// elements in `a` and `b` based on the comparison operand -/// specified by `imm8`. +/// specified by `IMM5`. /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_cmp_pd) #[inline] #[target_feature(enable = "avx")] -#[cfg_attr(test, assert_instr(vcmpeqpd, imm8 = 0))] // TODO Validate vcmppd -#[rustc_args_required_const(2)] +#[cfg_attr(test, assert_instr(vcmpeqpd, IMM5 = 0))] // TODO Validate vcmppd +#[rustc_legacy_const_generics(2)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm256_cmp_pd(a: __m256d, b: __m256d, imm8: i32) -> __m256d { - macro_rules! call { - ($imm8:expr) => { - vcmppd256(a, b, $imm8) - }; - } - constify_imm6!(imm8, call) +pub unsafe fn _mm256_cmp_pd(a: __m256d, b: __m256d) -> __m256d { + static_assert_imm5!(IMM5); + vcmppd256(a, b, IMM5 as u8) } /// Compares packed single-precision (32-bit) floating-point /// elements in `a` and `b` based on the comparison operand -/// specified by `imm8`. +/// specified by `IMM5`. /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cmp_ps) #[inline] #[target_feature(enable = "avx,sse")] -#[cfg_attr(test, assert_instr(vcmpeqps, imm8 = 0))] // TODO Validate vcmpps -#[rustc_args_required_const(2)] +#[cfg_attr(test, assert_instr(vcmpeqps, IMM5 = 0))] // TODO Validate vcmpps +#[rustc_legacy_const_generics(2)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm_cmp_ps(a: __m128, b: __m128, imm8: i32) -> __m128 { - macro_rules! call { - ($imm8:expr) => { - vcmpps(a, b, $imm8) - }; - } - constify_imm6!(imm8, call) +pub unsafe fn _mm_cmp_ps(a: __m128, b: __m128) -> __m128 { + static_assert_imm5!(IMM5); + vcmpps(a, b, IMM5 as i8) } /// Compares packed single-precision (32-bit) floating-point /// elements in `a` and `b` based on the comparison operand -/// specified by `imm8`. +/// specified by `IMM5`. /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_cmp_ps) #[inline] #[target_feature(enable = "avx")] -#[cfg_attr(test, assert_instr(vcmpeqps, imm8 = 0))] // TODO Validate vcmpps -#[rustc_args_required_const(2)] +#[cfg_attr(test, assert_instr(vcmpeqps, IMM5 = 0))] // TODO Validate vcmpps +#[rustc_legacy_const_generics(2)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm256_cmp_ps(a: __m256, b: __m256, imm8: i32) -> __m256 { - macro_rules! call { - ($imm8:expr) => { - vcmpps256(a, b, $imm8) - }; - } - constify_imm6!(imm8, call) +pub unsafe fn _mm256_cmp_ps(a: __m256, b: __m256) -> __m256 { + static_assert_imm5!(IMM5); + vcmpps256(a, b, IMM5 as u8) } /// Compares the lower double-precision (64-bit) floating-point element in -/// `a` and `b` based on the comparison operand specified by `imm8`, +/// `a` and `b` based on the comparison operand specified by `IMM5`, /// store the result in the lower element of returned vector, /// and copies the upper element from `a` to the upper element of returned /// vector. @@ -941,20 +795,16 @@ pub unsafe fn _mm256_cmp_ps(a: __m256, b: __m256, imm8: i32) -> __m256 { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cmp_sd) #[inline] #[target_feature(enable = "avx,sse2")] -#[cfg_attr(test, assert_instr(vcmpeqsd, imm8 = 0))] // TODO Validate vcmpsd -#[rustc_args_required_const(2)] +#[cfg_attr(test, assert_instr(vcmpeqsd, IMM5 = 0))] // TODO Validate vcmpsd +#[rustc_legacy_const_generics(2)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm_cmp_sd(a: __m128d, b: __m128d, imm8: i32) -> __m128d { - macro_rules! call { - ($imm8:expr) => { - vcmpsd(a, b, $imm8) - }; - } - constify_imm6!(imm8, call) +pub unsafe fn _mm_cmp_sd(a: __m128d, b: __m128d) -> __m128d { + static_assert_imm5!(IMM5); + vcmpsd(a, b, IMM5 as i8) } /// Compares the lower single-precision (32-bit) floating-point element in -/// `a` and `b` based on the comparison operand specified by `imm8`, +/// `a` and `b` based on the comparison operand specified by `IMM5`, /// store the result in the lower element of returned vector, /// and copies the upper 3 packed elements from `a` to the upper elements of /// returned vector. @@ -962,16 +812,12 @@ pub unsafe fn _mm_cmp_sd(a: __m128d, b: __m128d, imm8: i32) -> __m128d { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cmp_ss) #[inline] #[target_feature(enable = "avx,sse")] -#[cfg_attr(test, assert_instr(vcmpeqss, imm8 = 0))] // TODO Validate vcmpss -#[rustc_args_required_const(2)] +#[cfg_attr(test, assert_instr(vcmpeqss, IMM5 = 0))] // TODO Validate vcmpss +#[rustc_legacy_const_generics(2)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm_cmp_ss(a: __m128, b: __m128, imm8: i32) -> __m128 { - macro_rules! call { - ($imm8:expr) => { - vcmpss(a, b, $imm8) - }; - } - constify_imm6!(imm8, call) +pub unsafe fn _mm_cmp_ss(a: __m128, b: __m128) -> __m128 { + static_assert_imm5!(IMM5); + vcmpss(a, b, IMM5 as i8) } /// Converts packed 32-bit integers in `a` to packed double-precision (64-bit) @@ -1078,15 +924,17 @@ pub unsafe fn _mm256_cvttps_epi32(a: __m256) -> __m256i { #[target_feature(enable = "avx")] #[cfg_attr( all(test, not(target_os = "windows")), - assert_instr(vextractf128, imm8 = 1) + assert_instr(vextractf128, IMM1 = 1) )] -#[rustc_args_required_const(1)] -#[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm256_extractf128_ps(a: __m256, imm8: i32) -> __m128 { - match imm8 & 1 { - 0 => simd_shuffle4(a, _mm256_undefined_ps(), [0, 1, 2, 3]), - _ => simd_shuffle4(a, _mm256_undefined_ps(), [4, 5, 6, 7]), - } +#[rustc_legacy_const_generics(1)] +#[stable(feature = "simd_x86", since = "1.27.0")] +pub unsafe fn _mm256_extractf128_ps(a: __m256) -> __m128 { + static_assert_imm1!(IMM1); + simd_shuffle4!( + a, + _mm256_undefined_ps(), + [[0, 1, 2, 3], [4, 5, 6, 7]][IMM1 as usize], + ) } /// Extracts 128 bits (composed of 2 packed double-precision (64-bit) @@ -1097,15 +945,13 @@ pub unsafe fn _mm256_extractf128_ps(a: __m256, imm8: i32) -> __m128 { #[target_feature(enable = "avx")] #[cfg_attr( all(test, not(target_os = "windows")), - assert_instr(vextractf128, imm8 = 1) + assert_instr(vextractf128, IMM1 = 1) )] -#[rustc_args_required_const(1)] +#[rustc_legacy_const_generics(1)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm256_extractf128_pd(a: __m256d, imm8: i32) -> __m128d { - match imm8 & 1 { - 0 => simd_shuffle2(a, _mm256_undefined_pd(), [0, 1]), - _ => simd_shuffle2(a, _mm256_undefined_pd(), [2, 3]), - } +pub unsafe fn _mm256_extractf128_pd(a: __m256d) -> __m128d { + static_assert_imm1!(IMM1); + simd_shuffle2!(a, _mm256_undefined_pd(), [[0, 1], [2, 3]][IMM1 as usize]) } /// Extracts 128 bits (composed of integer data) from `a`, selected with `imm8`. @@ -1115,16 +961,17 @@ pub unsafe fn _mm256_extractf128_pd(a: __m256d, imm8: i32) -> __m128d { #[target_feature(enable = "avx")] #[cfg_attr( all(test, not(target_os = "windows")), - assert_instr(vextractf128, imm8 = 1) + assert_instr(vextractf128, IMM1 = 1) )] -#[rustc_args_required_const(1)] -#[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm256_extractf128_si256(a: __m256i, imm8: i32) -> __m128i { - let b = _mm256_undefined_si256().as_i64x4(); - let dst: i64x2 = match imm8 & 1 { - 0 => simd_shuffle2(a.as_i64x4(), b, [0, 1]), - _ => simd_shuffle2(a.as_i64x4(), b, [2, 3]), - }; +#[rustc_legacy_const_generics(1)] +#[stable(feature = "simd_x86", since = "1.27.0")] +pub unsafe fn _mm256_extractf128_si256(a: __m256i) -> __m128i { + static_assert_imm1!(IMM1); + let dst: i64x2 = simd_shuffle2!( + a.as_i64x4(), + _mm256_undefined_si256().as_i64x4(), + [[0, 1], [2, 3]][IMM1 as usize], + ); transmute(dst) } @@ -1181,57 +1028,25 @@ pub unsafe fn _mm_permutevar_ps(a: __m128, b: __m128i) -> __m128 { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_permute_ps) #[inline] #[target_feature(enable = "avx")] -#[cfg_attr(test, assert_instr(vpermilps, imm8 = 9))] -#[rustc_args_required_const(1)] -#[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm256_permute_ps(a: __m256, imm8: i32) -> __m256 { - let imm8 = (imm8 & 0xFF) as u8; - let undefined = _mm256_undefined_ps(); - macro_rules! shuffle4 { - ($a:expr, $b:expr, $c:expr, $d:expr) => { - simd_shuffle8( - a, - undefined, - [$a, $b, $c, $d, $a + 4, $b + 4, $c + 4, $d + 4], - ) - }; - } - macro_rules! shuffle3 { - ($a:expr, $b:expr, $c:expr) => { - match (imm8 >> 6) & 0b11 { - 0b00 => shuffle4!($a, $b, $c, 0), - 0b01 => shuffle4!($a, $b, $c, 1), - 0b10 => shuffle4!($a, $b, $c, 2), - _ => shuffle4!($a, $b, $c, 3), - } - }; - } - macro_rules! shuffle2 { - ($a:expr, $b:expr) => { - match (imm8 >> 4) & 0b11 { - 0b00 => shuffle3!($a, $b, 0), - 0b01 => shuffle3!($a, $b, 1), - 0b10 => shuffle3!($a, $b, 2), - _ => shuffle3!($a, $b, 3), - } - }; - } - macro_rules! shuffle1 { - ($a:expr) => { - match (imm8 >> 2) & 0b11 { - 0b00 => shuffle2!($a, 0), - 0b01 => shuffle2!($a, 1), - 0b10 => shuffle2!($a, 2), - _ => shuffle2!($a, 3), - } - }; - } - match imm8 & 0b11 { - 0b00 => shuffle1!(0), - 0b01 => shuffle1!(1), - 0b10 => shuffle1!(2), - _ => shuffle1!(3), - } +#[cfg_attr(test, assert_instr(vpermilps, IMM8 = 9))] +#[rustc_legacy_const_generics(1)] +#[stable(feature = "simd_x86", since = "1.27.0")] +pub unsafe fn _mm256_permute_ps(a: __m256) -> __m256 { + static_assert_imm8!(IMM8); + simd_shuffle8!( + a, + _mm256_undefined_ps(), + [ + (IMM8 as u32 >> 0) & 0b11, + (IMM8 as u32 >> 2) & 0b11, + (IMM8 as u32 >> 4) & 0b11, + (IMM8 as u32 >> 6) & 0b11, + ((IMM8 as u32 >> 0) & 0b11) + 4, + ((IMM8 as u32 >> 2) & 0b11) + 4, + ((IMM8 as u32 >> 4) & 0b11) + 4, + ((IMM8 as u32 >> 6) & 0b11) + 4, + ], + ) } /// Shuffles single-precision (32-bit) floating-point elements in `a` @@ -1240,53 +1055,21 @@ pub unsafe fn _mm256_permute_ps(a: __m256, imm8: i32) -> __m256 { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_permute_ps) #[inline] #[target_feature(enable = "avx,sse")] -#[cfg_attr(test, assert_instr(vpermilps, imm8 = 9))] -#[rustc_args_required_const(1)] -#[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm_permute_ps(a: __m128, imm8: i32) -> __m128 { - let imm8 = (imm8 & 0xFF) as u8; - let undefined = _mm_undefined_ps(); - macro_rules! shuffle4 { - ($a:expr, $b:expr, $c:expr, $d:expr) => { - simd_shuffle4(a, undefined, [$a, $b, $c, $d]) - }; - } - macro_rules! shuffle3 { - ($a:expr, $b:expr, $c:expr) => { - match (imm8 >> 6) & 0b11 { - 0b00 => shuffle4!($a, $b, $c, 0), - 0b01 => shuffle4!($a, $b, $c, 1), - 0b10 => shuffle4!($a, $b, $c, 2), - _ => shuffle4!($a, $b, $c, 3), - } - }; - } - macro_rules! shuffle2 { - ($a:expr, $b:expr) => { - match (imm8 >> 4) & 0b11 { - 0b00 => shuffle3!($a, $b, 0), - 0b01 => shuffle3!($a, $b, 1), - 0b10 => shuffle3!($a, $b, 2), - _ => shuffle3!($a, $b, 3), - } - }; - } - macro_rules! shuffle1 { - ($a:expr) => { - match (imm8 >> 2) & 0b11 { - 0b00 => shuffle2!($a, 0), - 0b01 => shuffle2!($a, 1), - 0b10 => shuffle2!($a, 2), - _ => shuffle2!($a, 3), - } - }; - } - match imm8 & 0b11 { - 0b00 => shuffle1!(0), - 0b01 => shuffle1!(1), - 0b10 => shuffle1!(2), - _ => shuffle1!(3), - } +#[cfg_attr(test, assert_instr(vpermilps, IMM8 = 9))] +#[rustc_legacy_const_generics(1)] +#[stable(feature = "simd_x86", since = "1.27.0")] +pub unsafe fn _mm_permute_ps(a: __m128) -> __m128 { + static_assert_imm8!(IMM8); + simd_shuffle4!( + a, + _mm_undefined_ps(), + [ + (IMM8 as u32 >> 0) & 0b11, + (IMM8 as u32 >> 2) & 0b11, + (IMM8 as u32 >> 4) & 0b11, + (IMM8 as u32 >> 6) & 0b11, + ], + ) } /// Shuffles double-precision (64-bit) floating-point elements in `a` @@ -1319,45 +1102,21 @@ pub unsafe fn _mm_permutevar_pd(a: __m128d, b: __m128i) -> __m128d { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_permute_pd) #[inline] #[target_feature(enable = "avx")] -#[cfg_attr(test, assert_instr(vpermilpd, imm8 = 0x1))] -#[rustc_args_required_const(1)] -#[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm256_permute_pd(a: __m256d, imm8: i32) -> __m256d { - let imm8 = (imm8 & 0xFF) as u8; - let undefined = _mm256_undefined_pd(); - macro_rules! shuffle4 { - ($a:expr, $b:expr, $c:expr, $d:expr) => { - simd_shuffle4(a, undefined, [$a, $b, $c, $d]) - }; - } - macro_rules! shuffle3 { - ($a:expr, $b:expr, $c:expr) => { - match (imm8 >> 3) & 0x1 { - 0 => shuffle4!($a, $b, $c, 2), - _ => shuffle4!($a, $b, $c, 3), - } - }; - } - macro_rules! shuffle2 { - ($a:expr, $b:expr) => { - match (imm8 >> 2) & 0x1 { - 0 => shuffle3!($a, $b, 2), - _ => shuffle3!($a, $b, 3), - } - }; - } - macro_rules! shuffle1 { - ($a:expr) => { - match (imm8 >> 1) & 0x1 { - 0 => shuffle2!($a, 0), - _ => shuffle2!($a, 1), - } - }; - } - match imm8 & 0x1 { - 0 => shuffle1!(0), - _ => shuffle1!(1), - } +#[cfg_attr(test, assert_instr(vpermilpd, IMM4 = 0x1))] +#[rustc_legacy_const_generics(1)] +#[stable(feature = "simd_x86", since = "1.27.0")] +pub unsafe fn _mm256_permute_pd(a: __m256d) -> __m256d { + static_assert_imm4!(IMM4); + simd_shuffle4!( + a, + _mm256_undefined_pd(), + [ + ((IMM4 as u32 >> 0) & 1), + ((IMM4 as u32 >> 1) & 1), + ((IMM4 as u32 >> 2) & 1) + 2, + ((IMM4 as u32 >> 3) & 1) + 2, + ], + ) } /// Shuffles double-precision (64-bit) floating-point elements in `a` @@ -1366,29 +1125,16 @@ pub unsafe fn _mm256_permute_pd(a: __m256d, imm8: i32) -> __m256d { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_permute_pd) #[inline] #[target_feature(enable = "avx,sse2")] -#[cfg_attr(test, assert_instr(vpermilpd, imm8 = 0x1))] -#[rustc_args_required_const(1)] -#[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm_permute_pd(a: __m128d, imm8: i32) -> __m128d { - let imm8 = (imm8 & 0xFF) as u8; - let undefined = _mm_undefined_pd(); - macro_rules! shuffle2 { - ($a:expr, $b:expr) => { - simd_shuffle2(a, undefined, [$a, $b]) - }; - } - macro_rules! shuffle1 { - ($a:expr) => { - match (imm8 >> 1) & 0x1 { - 0 => shuffle2!($a, 0), - _ => shuffle2!($a, 1), - } - }; - } - match imm8 & 0x1 { - 0 => shuffle1!(0), - _ => shuffle1!(1), - } +#[cfg_attr(test, assert_instr(vpermilpd, IMM2 = 0x1))] +#[rustc_legacy_const_generics(1)] +#[stable(feature = "simd_x86", since = "1.27.0")] +pub unsafe fn _mm_permute_pd(a: __m128d) -> __m128d { + static_assert_imm2!(IMM2); + simd_shuffle2!( + a, + _mm_undefined_pd(), + [(IMM2 as u32) & 1, (IMM2 as u32 >> 1) & 1], + ) } /// Shuffles 256 bits (composed of 8 packed single-precision (32-bit) @@ -1397,16 +1143,12 @@ pub unsafe fn _mm_permute_pd(a: __m128d, imm8: i32) -> __m128d { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_permute2f128_ps) #[inline] #[target_feature(enable = "avx")] -#[cfg_attr(test, assert_instr(vperm2f128, imm8 = 0x5))] -#[rustc_args_required_const(2)] +#[cfg_attr(test, assert_instr(vperm2f128, IMM8 = 0x5))] +#[rustc_legacy_const_generics(2)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm256_permute2f128_ps(a: __m256, b: __m256, imm8: i32) -> __m256 { - macro_rules! call { - ($imm8:expr) => { - vperm2f128ps256(a, b, $imm8) - }; - } - constify_imm8!(imm8, call) +pub unsafe fn _mm256_permute2f128_ps(a: __m256, b: __m256) -> __m256 { + static_assert_imm8!(IMM8); + vperm2f128ps256(a, b, IMM8 as i8) } /// Shuffles 256 bits (composed of 4 packed double-precision (64-bit) @@ -1415,16 +1157,12 @@ pub unsafe fn _mm256_permute2f128_ps(a: __m256, b: __m256, imm8: i32) -> __m256 /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_permute2f128_pd) #[inline] #[target_feature(enable = "avx")] -#[cfg_attr(test, assert_instr(vperm2f128, imm8 = 0x31))] -#[rustc_args_required_const(2)] +#[cfg_attr(test, assert_instr(vperm2f128, IMM8 = 0x31))] +#[rustc_legacy_const_generics(2)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm256_permute2f128_pd(a: __m256d, b: __m256d, imm8: i32) -> __m256d { - macro_rules! call { - ($imm8:expr) => { - vperm2f128pd256(a, b, $imm8) - }; - } - constify_imm8!(imm8, call) +pub unsafe fn _mm256_permute2f128_pd(a: __m256d, b: __m256d) -> __m256d { + static_assert_imm8!(IMM8); + vperm2f128pd256(a, b, IMM8 as i8) } /// Shuffles 128-bits (composed of integer data) selected by `imm8` @@ -1433,19 +1171,12 @@ pub unsafe fn _mm256_permute2f128_pd(a: __m256d, b: __m256d, imm8: i32) -> __m25 /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_permute2f128_si256) #[inline] #[target_feature(enable = "avx")] -#[cfg_attr(test, assert_instr(vperm2f128, imm8 = 0x31))] -#[rustc_args_required_const(2)] +#[cfg_attr(test, assert_instr(vperm2f128, IMM8 = 0x31))] +#[rustc_legacy_const_generics(2)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm256_permute2f128_si256(a: __m256i, b: __m256i, imm8: i32) -> __m256i { - let a = a.as_i32x8(); - let b = b.as_i32x8(); - macro_rules! call { - ($imm8:expr) => { - vperm2f128si256(a, b, $imm8) - }; - } - let r = constify_imm8!(imm8, call); - transmute(r) +pub unsafe fn _mm256_permute2f128_si256(a: __m256i, b: __m256i) -> __m256i { + static_assert_imm8!(IMM8); + transmute(vperm2f128si256(a.as_i32x8(), b.as_i32x8(), IMM8 as i8)) } /// Broadcasts a single-precision (32-bit) floating-point element from memory @@ -1520,16 +1251,17 @@ pub unsafe fn _mm256_broadcast_pd(a: &__m128d) -> __m256d { #[target_feature(enable = "avx")] #[cfg_attr( all(test, not(target_os = "windows")), - assert_instr(vinsertf128, imm8 = 1) + assert_instr(vinsertf128, IMM1 = 1) )] -#[rustc_args_required_const(2)] -#[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm256_insertf128_ps(a: __m256, b: __m128, imm8: i32) -> __m256 { - let b = _mm256_castps128_ps256(b); - match imm8 & 1 { - 0 => simd_shuffle8(a, b, [8, 9, 10, 11, 4, 5, 6, 7]), - _ => simd_shuffle8(a, b, [0, 1, 2, 3, 8, 9, 10, 11]), - } +#[rustc_legacy_const_generics(2)] +#[stable(feature = "simd_x86", since = "1.27.0")] +pub unsafe fn _mm256_insertf128_ps(a: __m256, b: __m128) -> __m256 { + static_assert_imm1!(IMM1); + simd_shuffle8!( + a, + _mm256_castps128_ps256(b), + [[8, 9, 10, 11, 4, 5, 6, 7], [0, 1, 2, 3, 8, 9, 10, 11]][IMM1 as usize], + ) } /// Copies `a` to result, then inserts 128 bits (composed of 2 packed @@ -1541,15 +1273,17 @@ pub unsafe fn _mm256_insertf128_ps(a: __m256, b: __m128, imm8: i32) -> __m256 { #[target_feature(enable = "avx")] #[cfg_attr( all(test, not(target_os = "windows")), - assert_instr(vinsertf128, imm8 = 1) + assert_instr(vinsertf128, IMM1 = 1) )] -#[rustc_args_required_const(2)] -#[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm256_insertf128_pd(a: __m256d, b: __m128d, imm8: i32) -> __m256d { - match imm8 & 1 { - 0 => simd_shuffle4(a, _mm256_castpd128_pd256(b), [4, 5, 2, 3]), - _ => simd_shuffle4(a, _mm256_castpd128_pd256(b), [0, 1, 4, 5]), - } +#[rustc_legacy_const_generics(2)] +#[stable(feature = "simd_x86", since = "1.27.0")] +pub unsafe fn _mm256_insertf128_pd(a: __m256d, b: __m128d) -> __m256d { + static_assert_imm1!(IMM1); + simd_shuffle4!( + a, + _mm256_castpd128_pd256(b), + [[4, 5, 2, 3], [0, 1, 4, 5]][IMM1 as usize], + ) } /// Copies `a` to result, then inserts 128 bits from `b` into result @@ -1560,16 +1294,17 @@ pub unsafe fn _mm256_insertf128_pd(a: __m256d, b: __m128d, imm8: i32) -> __m256d #[target_feature(enable = "avx")] #[cfg_attr( all(test, not(target_os = "windows")), - assert_instr(vinsertf128, imm8 = 1) + assert_instr(vinsertf128, IMM1 = 1) )] -#[rustc_args_required_const(2)] -#[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm256_insertf128_si256(a: __m256i, b: __m128i, imm8: i32) -> __m256i { - let b = _mm256_castsi128_si256(b).as_i64x4(); - let dst: i64x4 = match imm8 & 1 { - 0 => simd_shuffle4(a.as_i64x4(), b, [4, 5, 2, 3]), - _ => simd_shuffle4(a.as_i64x4(), b, [0, 1, 4, 5]), - }; +#[rustc_legacy_const_generics(2)] +#[stable(feature = "simd_x86", since = "1.27.0")] +pub unsafe fn _mm256_insertf128_si256(a: __m256i, b: __m128i) -> __m256i { + static_assert_imm1!(IMM1); + let dst: i64x4 = simd_shuffle4!( + a.as_i64x4(), + _mm256_castsi128_si256(b).as_i64x4(), + [[4, 5, 2, 3], [0, 1, 4, 5]][IMM1 as usize], + ); transmute(dst) } @@ -1580,16 +1315,11 @@ pub unsafe fn _mm256_insertf128_si256(a: __m256i, b: __m128i, imm8: i32) -> __m2 #[inline] #[target_feature(enable = "avx")] // This intrinsic has no corresponding instruction. -#[rustc_args_required_const(2)] +#[rustc_legacy_const_generics(2)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm256_insert_epi8(a: __m256i, i: i8, index: i32) -> __m256i { - let a = a.as_i8x32(); - macro_rules! call { - ($index:expr) => { - simd_insert(a, $index, i) - }; - } - transmute(constify_imm5!(index, call)) +pub unsafe fn _mm256_insert_epi8(a: __m256i, i: i8) -> __m256i { + static_assert_imm5!(INDEX); + transmute(simd_insert(a.as_i8x32(), INDEX as u32, i)) } /// Copies `a` to result, and inserts the 16-bit integer `i` into result @@ -1599,16 +1329,11 @@ pub unsafe fn _mm256_insert_epi8(a: __m256i, i: i8, index: i32) -> __m256i { #[inline] #[target_feature(enable = "avx")] // This intrinsic has no corresponding instruction. -#[rustc_args_required_const(2)] +#[rustc_legacy_const_generics(2)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm256_insert_epi16(a: __m256i, i: i16, index: i32) -> __m256i { - let a = a.as_i16x16(); - macro_rules! call { - ($index:expr) => { - simd_insert(a, $index, i) - }; - } - transmute(constify_imm4!((index & 15), call)) +pub unsafe fn _mm256_insert_epi16(a: __m256i, i: i16) -> __m256i { + static_assert_imm4!(INDEX); + transmute(simd_insert(a.as_i16x16(), INDEX as u32, i)) } /// Copies `a` to result, and inserts the 32-bit integer `i` into result @@ -1618,16 +1343,11 @@ pub unsafe fn _mm256_insert_epi16(a: __m256i, i: i16, index: i32) -> __m256i { #[inline] #[target_feature(enable = "avx")] // This intrinsic has no corresponding instruction. -#[rustc_args_required_const(2)] +#[rustc_legacy_const_generics(2)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm256_insert_epi32(a: __m256i, i: i32, index: i32) -> __m256i { - let a = a.as_i32x8(); - macro_rules! call { - ($index:expr) => { - simd_insert(a, $index, i) - }; - } - transmute(constify_imm3!(index, call)) +pub unsafe fn _mm256_insert_epi32(a: __m256i, i: i32) -> __m256i { + static_assert_imm3!(INDEX); + transmute(simd_insert(a.as_i32x8(), INDEX as u32, i)) } /// Loads 256-bits (composed of 4 packed double-precision (64-bit) @@ -1919,7 +1639,7 @@ pub unsafe fn _mm_maskstore_ps(mem_addr: *mut f32, mask: __m128i, a: __m128) { #[cfg_attr(test, assert_instr(vmovshdup))] #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm256_movehdup_ps(a: __m256) -> __m256 { - simd_shuffle8(a, a, [1, 1, 3, 3, 5, 5, 7, 7]) + simd_shuffle8!(a, a, [1, 1, 3, 3, 5, 5, 7, 7]) } /// Duplicate even-indexed single-precision (32-bit) floating-point elements @@ -1931,7 +1651,7 @@ pub unsafe fn _mm256_movehdup_ps(a: __m256) -> __m256 { #[cfg_attr(test, assert_instr(vmovsldup))] #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm256_moveldup_ps(a: __m256) -> __m256 { - simd_shuffle8(a, a, [0, 0, 2, 2, 4, 4, 6, 6]) + simd_shuffle8!(a, a, [0, 0, 2, 2, 4, 4, 6, 6]) } /// Duplicate even-indexed double-precision (64-bit) floating-point elements @@ -1943,7 +1663,7 @@ pub unsafe fn _mm256_moveldup_ps(a: __m256) -> __m256 { #[cfg_attr(test, assert_instr(vmovddup))] #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm256_movedup_pd(a: __m256d) -> __m256d { - simd_shuffle4(a, a, [0, 0, 2, 2]) + simd_shuffle4!(a, a, [0, 0, 2, 2]) } /// Loads 256-bits of integer data from unaligned memory into result. @@ -2036,7 +1756,7 @@ pub unsafe fn _mm256_rsqrt_ps(a: __m256) -> __m256 { #[cfg_attr(test, assert_instr(vunpckhpd))] #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm256_unpackhi_pd(a: __m256d, b: __m256d) -> __m256d { - simd_shuffle4(a, b, [1, 5, 3, 7]) + simd_shuffle4!(a, b, [1, 5, 3, 7]) } /// Unpacks and interleave single-precision (32-bit) floating-point elements @@ -2048,7 +1768,7 @@ pub unsafe fn _mm256_unpackhi_pd(a: __m256d, b: __m256d) -> __m256d { #[cfg_attr(test, assert_instr(vunpckhps))] #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm256_unpackhi_ps(a: __m256, b: __m256) -> __m256 { - simd_shuffle8(a, b, [2, 10, 3, 11, 6, 14, 7, 15]) + simd_shuffle8!(a, b, [2, 10, 3, 11, 6, 14, 7, 15]) } /// Unpacks and interleave double-precision (64-bit) floating-point elements @@ -2060,7 +1780,7 @@ pub unsafe fn _mm256_unpackhi_ps(a: __m256, b: __m256) -> __m256 { #[cfg_attr(test, assert_instr(vunpcklpd))] #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm256_unpacklo_pd(a: __m256d, b: __m256d) -> __m256d { - simd_shuffle4(a, b, [0, 4, 2, 6]) + simd_shuffle4!(a, b, [0, 4, 2, 6]) } /// Unpacks and interleave single-precision (32-bit) floating-point elements @@ -2072,7 +1792,7 @@ pub unsafe fn _mm256_unpacklo_pd(a: __m256d, b: __m256d) -> __m256d { #[cfg_attr(test, assert_instr(vunpcklps))] #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm256_unpacklo_ps(a: __m256, b: __m256) -> __m256 { - simd_shuffle8(a, b, [0, 8, 1, 9, 4, 12, 5, 13]) + simd_shuffle8!(a, b, [0, 8, 1, 9, 4, 12, 5, 13]) } /// Computes the bitwise AND of 256 bits (representing integer data) in `a` and @@ -2852,7 +2572,7 @@ pub unsafe fn _mm256_castsi256_pd(a: __m256i) -> __m256d { // instructions, thus it has zero latency. #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm256_castps256_ps128(a: __m256) -> __m128 { - simd_shuffle4(a, a, [0, 1, 2, 3]) + simd_shuffle4!(a, a, [0, 1, 2, 3]) } /// Casts vector of type __m256d to type __m128d. @@ -2864,7 +2584,7 @@ pub unsafe fn _mm256_castps256_ps128(a: __m256) -> __m128 { // instructions, thus it has zero latency. #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm256_castpd256_pd128(a: __m256d) -> __m128d { - simd_shuffle2(a, a, [0, 1]) + simd_shuffle2!(a, a, [0, 1]) } /// Casts vector of type __m256i to type __m128i. @@ -2877,7 +2597,7 @@ pub unsafe fn _mm256_castpd256_pd128(a: __m256d) -> __m128d { #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm256_castsi256_si128(a: __m256i) -> __m128i { let a = a.as_i64x4(); - let dst: i64x2 = simd_shuffle2(a, a, [0, 1]); + let dst: i64x2 = simd_shuffle2!(a, a, [0, 1]); transmute(dst) } @@ -2891,8 +2611,8 @@ pub unsafe fn _mm256_castsi256_si128(a: __m256i) -> __m128i { // instructions, thus it has zero latency. #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm256_castps128_ps256(a: __m128) -> __m256 { - // FIXME simd_shuffle8(a, a, [0, 1, 2, 3, -1, -1, -1, -1]) - simd_shuffle8(a, a, [0, 1, 2, 3, 0, 0, 0, 0]) + // FIXME simd_shuffle8!(a, a, [0, 1, 2, 3, -1, -1, -1, -1]) + simd_shuffle8!(a, a, [0, 1, 2, 3, 0, 0, 0, 0]) } /// Casts vector of type __m128d to type __m256d; @@ -2905,8 +2625,8 @@ pub unsafe fn _mm256_castps128_ps256(a: __m128) -> __m256 { // instructions, thus it has zero latency. #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm256_castpd128_pd256(a: __m128d) -> __m256d { - // FIXME simd_shuffle4(a, a, [0, 1, -1, -1]) - simd_shuffle4(a, a, [0, 1, 0, 0]) + // FIXME simd_shuffle4!(a, a, [0, 1, -1, -1]) + simd_shuffle4!(a, a, [0, 1, 0, 0]) } /// Casts vector of type __m128i to type __m256i; @@ -2920,8 +2640,8 @@ pub unsafe fn _mm256_castpd128_pd256(a: __m128d) -> __m256d { #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm256_castsi128_si256(a: __m128i) -> __m256i { let a = a.as_i64x2(); - // FIXME simd_shuffle4(a, a, [0, 1, -1, -1]) - let dst: i64x4 = simd_shuffle4(a, a, [0, 1, 0, 0]); + // FIXME simd_shuffle4!(a, a, [0, 1, -1, -1]) + let dst: i64x4 = simd_shuffle4!(a, a, [0, 1, 0, 0]); transmute(dst) } @@ -2936,7 +2656,7 @@ pub unsafe fn _mm256_castsi128_si256(a: __m128i) -> __m256i { // instructions, thus it has zero latency. #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm256_zextps128_ps256(a: __m128) -> __m256 { - simd_shuffle8(a, _mm_setzero_ps(), [0, 1, 2, 3, 4, 5, 6, 7]) + simd_shuffle8!(a, _mm_setzero_ps(), [0, 1, 2, 3, 4, 5, 6, 7]) } /// Constructs a 256-bit integer vector from a 128-bit integer vector. @@ -2951,7 +2671,7 @@ pub unsafe fn _mm256_zextps128_ps256(a: __m128) -> __m256 { #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm256_zextsi128_si256(a: __m128i) -> __m256i { let b = _mm_setzero_si128().as_i64x2(); - let dst: i64x4 = simd_shuffle4(a.as_i64x2(), b, [0, 1, 2, 3]); + let dst: i64x4 = simd_shuffle4!(a.as_i64x2(), b, [0, 1, 2, 3]); transmute(dst) } @@ -2967,7 +2687,7 @@ pub unsafe fn _mm256_zextsi128_si256(a: __m128i) -> __m256i { // instructions, thus it has zero latency. #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm256_zextpd128_pd256(a: __m128d) -> __m256d { - simd_shuffle4(a, _mm_setzero_pd(), [0, 1, 2, 3]) + simd_shuffle4!(a, _mm_setzero_pd(), [0, 1, 2, 3]) } /// Returns vector of type `__m256` with undefined elements. @@ -3012,7 +2732,7 @@ pub unsafe fn _mm256_undefined_si256() -> __m256i { #[cfg_attr(test, assert_instr(vinsertf128))] #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm256_set_m128(hi: __m128, lo: __m128) -> __m256 { - simd_shuffle8(lo, hi, [0, 1, 2, 3, 4, 5, 6, 7]) + simd_shuffle8!(lo, hi, [0, 1, 2, 3, 4, 5, 6, 7]) } /// Sets packed __m256d returned vector with the supplied values. @@ -3086,7 +2806,7 @@ pub unsafe fn _mm256_setr_m128i(lo: __m128i, hi: __m128i) -> __m256i { #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm256_loadu2_m128(hiaddr: *const f32, loaddr: *const f32) -> __m256 { let a = _mm256_castps128_ps256(_mm_loadu_ps(loaddr)); - _mm256_insertf128_ps(a, _mm_loadu_ps(hiaddr), 1) + _mm256_insertf128_ps::<1>(a, _mm_loadu_ps(hiaddr)) } /// Loads two 128-bit values (composed of 2 packed double-precision (64-bit) @@ -3101,7 +2821,7 @@ pub unsafe fn _mm256_loadu2_m128(hiaddr: *const f32, loaddr: *const f32) -> __m2 #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm256_loadu2_m128d(hiaddr: *const f64, loaddr: *const f64) -> __m256d { let a = _mm256_castpd128_pd256(_mm_loadu_pd(loaddr)); - _mm256_insertf128_pd(a, _mm_loadu_pd(hiaddr), 1) + _mm256_insertf128_pd::<1>(a, _mm_loadu_pd(hiaddr)) } /// Loads two 128-bit values (composed of integer data) from memory, and combine @@ -3115,7 +2835,7 @@ pub unsafe fn _mm256_loadu2_m128d(hiaddr: *const f64, loaddr: *const f64) -> __m #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm256_loadu2_m128i(hiaddr: *const __m128i, loaddr: *const __m128i) -> __m256i { let a = _mm256_castsi128_si256(_mm_loadu_si128(loaddr)); - _mm256_insertf128_si256(a, _mm_loadu_si128(hiaddr), 1) + _mm256_insertf128_si256::<1>(a, _mm_loadu_si128(hiaddr)) } /// Stores the high and low 128-bit halves (each composed of 4 packed @@ -3131,7 +2851,7 @@ pub unsafe fn _mm256_loadu2_m128i(hiaddr: *const __m128i, loaddr: *const __m128i pub unsafe fn _mm256_storeu2_m128(hiaddr: *mut f32, loaddr: *mut f32, a: __m256) { let lo = _mm256_castps256_ps128(a); _mm_storeu_ps(loaddr, lo); - let hi = _mm256_extractf128_ps(a, 1); + let hi = _mm256_extractf128_ps::<1>(a); _mm_storeu_ps(hiaddr, hi); } @@ -3148,7 +2868,7 @@ pub unsafe fn _mm256_storeu2_m128(hiaddr: *mut f32, loaddr: *mut f32, a: __m256) pub unsafe fn _mm256_storeu2_m128d(hiaddr: *mut f64, loaddr: *mut f64, a: __m256d) { let lo = _mm256_castpd256_pd128(a); _mm_storeu_pd(loaddr, lo); - let hi = _mm256_extractf128_pd(a, 1); + let hi = _mm256_extractf128_pd::<1>(a); _mm_storeu_pd(hiaddr, hi); } @@ -3164,7 +2884,7 @@ pub unsafe fn _mm256_storeu2_m128d(hiaddr: *mut f64, loaddr: *mut f64, a: __m256 pub unsafe fn _mm256_storeu2_m128i(hiaddr: *mut __m128i, loaddr: *mut __m128i, a: __m256i) { let lo = _mm256_castsi256_si128(a); _mm_storeu_si128(loaddr, lo); - let hi = _mm256_extractf128_si256(a, 1); + let hi = _mm256_extractf128_si256::<1>(a); _mm_storeu_si128(hiaddr, hi); } @@ -3314,6 +3034,14 @@ extern "C" { fn movmskpd256(a: __m256d) -> i32; #[link_name = "llvm.x86.avx.movmsk.ps.256"] fn movmskps256(a: __m256) -> i32; + #[link_name = "llvm.x86.avx.min.ps.256"] + fn vminps(a: __m256, b: __m256) -> __m256; + #[link_name = "llvm.x86.avx.max.ps.256"] + fn vmaxps(a: __m256, b: __m256) -> __m256; + #[link_name = "llvm.x86.avx.min.pd.256"] + fn vminpd(a: __m256d, b: __m256d) -> __m256d; + #[link_name = "llvm.x86.avx.max.pd.256"] + fn vmaxpd(a: __m256d, b: __m256d) -> __m256d; } #[cfg(test)] @@ -3381,7 +3109,7 @@ mod tests { unsafe fn test_mm256_shuffle_pd() { let a = _mm256_setr_pd(1., 4., 5., 8.); let b = _mm256_setr_pd(2., 3., 6., 7.); - let r = _mm256_shuffle_pd(a, b, 0xF); + let r = _mm256_shuffle_pd::<0b11_11_11_11>(a, b); let e = _mm256_setr_pd(4., 3., 8., 7.); assert_eq_m256d(r, e); } @@ -3390,7 +3118,7 @@ mod tests { unsafe fn test_mm256_shuffle_ps() { let a = _mm256_setr_ps(1., 4., 5., 8., 9., 12., 13., 16.); let b = _mm256_setr_ps(2., 3., 6., 7., 10., 11., 14., 15.); - let r = _mm256_shuffle_ps(a, b, 0x0F); + let r = _mm256_shuffle_ps::<0b00_00_11_11>(a, b); let e = _mm256_setr_ps(8., 8., 2., 2., 16., 16., 10., 10.); assert_eq_m256(r, e); } @@ -3418,6 +3146,23 @@ mod tests { let r = _mm256_max_pd(a, b); let e = _mm256_setr_pd(2., 4., 6., 8.); assert_eq_m256d(r, e); + // > If the values being compared are both 0.0s (of either sign), the + // > value in the second operand (source operand) is returned. + let w = _mm256_max_pd(_mm256_set1_pd(0.0), _mm256_set1_pd(-0.0)); + let x = _mm256_max_pd(_mm256_set1_pd(-0.0), _mm256_set1_pd(0.0)); + let wu: [u64; 4] = transmute(w); + let xu: [u64; 4] = transmute(x); + assert_eq!(wu, [0x8000_0000_0000_0000u64; 4]); + assert_eq!(xu, [0u64; 4]); + // > If only one value is a NaN (SNaN or QNaN) for this instruction, the + // > second operand (source operand), either a NaN or a valid + // > floating-point value, is written to the result. + let y = _mm256_max_pd(_mm256_set1_pd(f64::NAN), _mm256_set1_pd(0.0)); + let z = _mm256_max_pd(_mm256_set1_pd(0.0), _mm256_set1_pd(f64::NAN)); + let yf: [f64; 4] = transmute(y); + let zf: [f64; 4] = transmute(z); + assert_eq!(yf, [0.0; 4]); + assert!(zf.iter().all(|f| f.is_nan()), "{:?}", zf); } #[simd_test(enable = "avx")] @@ -3427,6 +3172,23 @@ mod tests { let r = _mm256_max_ps(a, b); let e = _mm256_setr_ps(2., 4., 6., 8., 10., 12., 14., 16.); assert_eq_m256(r, e); + // > If the values being compared are both 0.0s (of either sign), the + // > value in the second operand (source operand) is returned. + let w = _mm256_max_ps(_mm256_set1_ps(0.0), _mm256_set1_ps(-0.0)); + let x = _mm256_max_ps(_mm256_set1_ps(-0.0), _mm256_set1_ps(0.0)); + let wu: [u32; 8] = transmute(w); + let xu: [u32; 8] = transmute(x); + assert_eq!(wu, [0x8000_0000u32; 8]); + assert_eq!(xu, [0u32; 8]); + // > If only one value is a NaN (SNaN or QNaN) for this instruction, the + // > second operand (source operand), either a NaN or a valid + // > floating-point value, is written to the result. + let y = _mm256_max_ps(_mm256_set1_ps(f32::NAN), _mm256_set1_ps(0.0)); + let z = _mm256_max_ps(_mm256_set1_ps(0.0), _mm256_set1_ps(f32::NAN)); + let yf: [f32; 8] = transmute(y); + let zf: [f32; 8] = transmute(z); + assert_eq!(yf, [0.0; 8]); + assert!(zf.iter().all(|f| f.is_nan()), "{:?}", zf); } #[simd_test(enable = "avx")] @@ -3436,6 +3198,23 @@ mod tests { let r = _mm256_min_pd(a, b); let e = _mm256_setr_pd(1., 3., 5., 7.); assert_eq_m256d(r, e); + // > If the values being compared are both 0.0s (of either sign), the + // > value in the second operand (source operand) is returned. + let w = _mm256_min_pd(_mm256_set1_pd(0.0), _mm256_set1_pd(-0.0)); + let x = _mm256_min_pd(_mm256_set1_pd(-0.0), _mm256_set1_pd(0.0)); + let wu: [u64; 4] = transmute(w); + let xu: [u64; 4] = transmute(x); + assert_eq!(wu, [0x8000_0000_0000_0000u64; 4]); + assert_eq!(xu, [0u64; 4]); + // > If only one value is a NaN (SNaN or QNaN) for this instruction, the + // > second operand (source operand), either a NaN or a valid + // > floating-point value, is written to the result. + let y = _mm256_min_pd(_mm256_set1_pd(f64::NAN), _mm256_set1_pd(0.0)); + let z = _mm256_min_pd(_mm256_set1_pd(0.0), _mm256_set1_pd(f64::NAN)); + let yf: [f64; 4] = transmute(y); + let zf: [f64; 4] = transmute(z); + assert_eq!(yf, [0.0; 4]); + assert!(zf.iter().all(|f| f.is_nan()), "{:?}", zf); } #[simd_test(enable = "avx")] @@ -3445,6 +3224,23 @@ mod tests { let r = _mm256_min_ps(a, b); let e = _mm256_setr_ps(1., 3., 5., 7., 9., 11., 13., 15.); assert_eq_m256(r, e); + // > If the values being compared are both 0.0s (of either sign), the + // > value in the second operand (source operand) is returned. + let w = _mm256_min_ps(_mm256_set1_ps(0.0), _mm256_set1_ps(-0.0)); + let x = _mm256_min_ps(_mm256_set1_ps(-0.0), _mm256_set1_ps(0.0)); + let wu: [u32; 8] = transmute(w); + let xu: [u32; 8] = transmute(x); + assert_eq!(wu, [0x8000_0000u32; 8]); + assert_eq!(xu, [0u32; 8]); + // > If only one value is a NaN (SNaN or QNaN) for this instruction, the + // > second operand (source operand), either a NaN or a valid + // > floating-point value, is written to the result. + let y = _mm256_min_ps(_mm256_set1_ps(f32::NAN), _mm256_set1_ps(0.0)); + let z = _mm256_min_ps(_mm256_set1_ps(0.0), _mm256_set1_ps(f32::NAN)); + let yf: [f32; 8] = transmute(y); + let zf: [f32; 8] = transmute(z); + assert_eq!(yf, [0.0; 8]); + assert!(zf.iter().all(|f| f.is_nan()), "{:?}", zf); } #[simd_test(enable = "avx")] @@ -3504,9 +3300,9 @@ mod tests { #[simd_test(enable = "avx")] unsafe fn test_mm256_round_pd() { let a = _mm256_setr_pd(1.55, 2.2, 3.99, -1.2); - let result_closest = _mm256_round_pd(a, 0b00000000); - let result_down = _mm256_round_pd(a, 0b00000001); - let result_up = _mm256_round_pd(a, 0b00000010); + let result_closest = _mm256_round_pd::<0b0000>(a); + let result_down = _mm256_round_pd::<0b0001>(a); + let result_up = _mm256_round_pd::<0b0010>(a); let expected_closest = _mm256_setr_pd(2., 2., 4., -1.); let expected_down = _mm256_setr_pd(1., 2., 3., -2.); let expected_up = _mm256_setr_pd(2., 3., 4., -1.); @@ -3534,9 +3330,9 @@ mod tests { #[simd_test(enable = "avx")] unsafe fn test_mm256_round_ps() { let a = _mm256_setr_ps(1.55, 2.2, 3.99, -1.2, 1.55, 2.2, 3.99, -1.2); - let result_closest = _mm256_round_ps(a, 0b00000000); - let result_down = _mm256_round_ps(a, 0b00000001); - let result_up = _mm256_round_ps(a, 0b00000010); + let result_closest = _mm256_round_ps::<0b0000>(a); + let result_down = _mm256_round_ps::<0b0001>(a); + let result_up = _mm256_round_ps::<0b0010>(a); let expected_closest = _mm256_setr_ps(2., 2., 4., -1., 2., 2., 4., -1.); let expected_down = _mm256_setr_ps(1., 2., 3., -2., 1., 2., 3., -2.); let expected_up = _mm256_setr_ps(2., 3., 4., -1., 2., 3., 4., -1.); @@ -3599,11 +3395,11 @@ mod tests { unsafe fn test_mm256_blend_pd() { let a = _mm256_setr_pd(4., 9., 16., 25.); let b = _mm256_setr_pd(4., 3., 2., 5.); - let r = _mm256_blend_pd(a, b, 0x0); + let r = _mm256_blend_pd::<0x0>(a, b); assert_eq_m256d(r, _mm256_setr_pd(4., 9., 16., 25.)); - let r = _mm256_blend_pd(a, b, 0x3); + let r = _mm256_blend_pd::<0x3>(a, b); assert_eq_m256d(r, _mm256_setr_pd(4., 3., 16., 25.)); - let r = _mm256_blend_pd(a, b, 0xF); + let r = _mm256_blend_pd::<0xF>(a, b); assert_eq_m256d(r, _mm256_setr_pd(4., 3., 2., 5.)); } @@ -3611,11 +3407,11 @@ mod tests { unsafe fn test_mm256_blend_ps() { let a = _mm256_setr_ps(1., 4., 5., 8., 9., 12., 13., 16.); let b = _mm256_setr_ps(2., 3., 6., 7., 10., 11., 14., 15.); - let r = _mm256_blend_ps(a, b, 0x0); + let r = _mm256_blend_ps::<0x0>(a, b); assert_eq_m256(r, _mm256_setr_ps(1., 4., 5., 8., 9., 12., 13., 16.)); - let r = _mm256_blend_ps(a, b, 0x3); + let r = _mm256_blend_ps::<0x3>(a, b); assert_eq_m256(r, _mm256_setr_ps(2., 3., 5., 8., 9., 12., 13., 16.)); - let r = _mm256_blend_ps(a, b, 0xF); + let r = _mm256_blend_ps::<0xF>(a, b); assert_eq_m256(r, _mm256_setr_ps(2., 3., 6., 7., 9., 12., 13., 16.)); } @@ -3646,7 +3442,7 @@ mod tests { unsafe fn test_mm256_dp_ps() { let a = _mm256_setr_ps(4., 9., 16., 25., 4., 9., 16., 25.); let b = _mm256_setr_ps(4., 3., 2., 5., 8., 9., 64., 50.); - let r = _mm256_dp_ps(a, b, 0xFF); + let r = _mm256_dp_ps::<0xFF>(a, b); let e = _mm256_setr_ps(200., 200., 200., 200., 2387., 2387., 2387., 2387.); assert_eq_m256(r, e); } @@ -3731,7 +3527,7 @@ mod tests { unsafe fn test_mm_cmp_pd() { let a = _mm_setr_pd(4., 9.); let b = _mm_setr_pd(4., 3.); - let r = _mm_cmp_pd(a, b, _CMP_GE_OS); + let r = _mm_cmp_pd::<_CMP_GE_OS>(a, b); assert!(get_m128d(r, 0).is_nan()); assert!(get_m128d(r, 1).is_nan()); } @@ -3740,7 +3536,7 @@ mod tests { unsafe fn test_mm256_cmp_pd() { let a = _mm256_setr_pd(1., 2., 3., 4.); let b = _mm256_setr_pd(5., 6., 7., 8.); - let r = _mm256_cmp_pd(a, b, _CMP_GE_OS); + let r = _mm256_cmp_pd::<_CMP_GE_OS>(a, b); let e = _mm256_set1_pd(0.); assert_eq_m256d(r, e); } @@ -3749,7 +3545,7 @@ mod tests { unsafe fn test_mm_cmp_ps() { let a = _mm_setr_ps(4., 3., 2., 5.); let b = _mm_setr_ps(4., 9., 16., 25.); - let r = _mm_cmp_ps(a, b, _CMP_GE_OS); + let r = _mm_cmp_ps::<_CMP_GE_OS>(a, b); assert!(get_m128(r, 0).is_nan()); assert_eq!(get_m128(r, 1), 0.); assert_eq!(get_m128(r, 2), 0.); @@ -3760,7 +3556,7 @@ mod tests { unsafe fn test_mm256_cmp_ps() { let a = _mm256_setr_ps(1., 2., 3., 4., 1., 2., 3., 4.); let b = _mm256_setr_ps(5., 6., 7., 8., 5., 6., 7., 8.); - let r = _mm256_cmp_ps(a, b, _CMP_GE_OS); + let r = _mm256_cmp_ps::<_CMP_GE_OS>(a, b); let e = _mm256_set1_ps(0.); assert_eq_m256(r, e); } @@ -3769,7 +3565,7 @@ mod tests { unsafe fn test_mm_cmp_sd() { let a = _mm_setr_pd(4., 9.); let b = _mm_setr_pd(4., 3.); - let r = _mm_cmp_sd(a, b, _CMP_GE_OS); + let r = _mm_cmp_sd::<_CMP_GE_OS>(a, b); assert!(get_m128d(r, 0).is_nan()); assert_eq!(get_m128d(r, 1), 9.); } @@ -3778,7 +3574,7 @@ mod tests { unsafe fn test_mm_cmp_ss() { let a = _mm_setr_ps(4., 3., 2., 5.); let b = _mm_setr_ps(4., 9., 16., 25.); - let r = _mm_cmp_ss(a, b, _CMP_GE_OS); + let r = _mm_cmp_ss::<_CMP_GE_OS>(a, b); assert!(get_m128(r, 0).is_nan()); assert_eq!(get_m128(r, 1), 3.); assert_eq!(get_m128(r, 2), 2.); @@ -3852,7 +3648,7 @@ mod tests { #[simd_test(enable = "avx")] unsafe fn test_mm256_extractf128_ps() { let a = _mm256_setr_ps(4., 3., 2., 5., 8., 9., 64., 50.); - let r = _mm256_extractf128_ps(a, 0); + let r = _mm256_extractf128_ps::<0>(a); let e = _mm_setr_ps(4., 3., 2., 5.); assert_eq_m128(r, e); } @@ -3860,7 +3656,7 @@ mod tests { #[simd_test(enable = "avx")] unsafe fn test_mm256_extractf128_pd() { let a = _mm256_setr_pd(4., 3., 2., 5.); - let r = _mm256_extractf128_pd(a, 0); + let r = _mm256_extractf128_pd::<0>(a); let e = _mm_setr_pd(4., 3.); assert_eq_m128d(r, e); } @@ -3868,7 +3664,7 @@ mod tests { #[simd_test(enable = "avx")] unsafe fn test_mm256_extractf128_si256() { let a = _mm256_setr_epi64x(4, 3, 2, 5); - let r = _mm256_extractf128_si256(a, 0); + let r = _mm256_extractf128_si256::<0>(a); let e = _mm_setr_epi64x(4, 3); assert_eq_m128i(r, e); } @@ -3904,7 +3700,7 @@ mod tests { #[simd_test(enable = "avx")] unsafe fn test_mm256_permute_ps() { let a = _mm256_setr_ps(4., 3., 2., 5., 8., 9., 64., 50.); - let r = _mm256_permute_ps(a, 0x1b); + let r = _mm256_permute_ps::<0x1b>(a); let e = _mm256_setr_ps(5., 2., 3., 4., 50., 64., 9., 8.); assert_eq_m256(r, e); } @@ -3912,7 +3708,7 @@ mod tests { #[simd_test(enable = "avx")] unsafe fn test_mm_permute_ps() { let a = _mm_setr_ps(4., 3., 2., 5.); - let r = _mm_permute_ps(a, 0x1b); + let r = _mm_permute_ps::<0x1b>(a); let e = _mm_setr_ps(5., 2., 3., 4.); assert_eq_m128(r, e); } @@ -3938,7 +3734,7 @@ mod tests { #[simd_test(enable = "avx")] unsafe fn test_mm256_permute_pd() { let a = _mm256_setr_pd(4., 3., 2., 5.); - let r = _mm256_permute_pd(a, 5); + let r = _mm256_permute_pd::<5>(a); let e = _mm256_setr_pd(3., 4., 5., 2.); assert_eq_m256d(r, e); } @@ -3946,7 +3742,7 @@ mod tests { #[simd_test(enable = "avx")] unsafe fn test_mm_permute_pd() { let a = _mm_setr_pd(4., 3.); - let r = _mm_permute_pd(a, 1); + let r = _mm_permute_pd::<1>(a); let e = _mm_setr_pd(3., 4.); assert_eq_m128d(r, e); } @@ -3955,7 +3751,7 @@ mod tests { unsafe fn test_mm256_permute2f128_ps() { let a = _mm256_setr_ps(1., 2., 3., 4., 1., 2., 3., 4.); let b = _mm256_setr_ps(5., 6., 7., 8., 5., 6., 7., 8.); - let r = _mm256_permute2f128_ps(a, b, 0x13); + let r = _mm256_permute2f128_ps::<0x13>(a, b); let e = _mm256_setr_ps(5., 6., 7., 8., 1., 2., 3., 4.); assert_eq_m256(r, e); } @@ -3964,7 +3760,7 @@ mod tests { unsafe fn test_mm256_permute2f128_pd() { let a = _mm256_setr_pd(1., 2., 3., 4.); let b = _mm256_setr_pd(5., 6., 7., 8.); - let r = _mm256_permute2f128_pd(a, b, 0x31); + let r = _mm256_permute2f128_pd::<0x31>(a, b); let e = _mm256_setr_pd(3., 4., 7., 8.); assert_eq_m256d(r, e); } @@ -3973,7 +3769,7 @@ mod tests { unsafe fn test_mm256_permute2f128_si256() { let a = _mm256_setr_epi32(1, 2, 3, 4, 1, 2, 3, 4); let b = _mm256_setr_epi32(5, 6, 7, 8, 5, 6, 7, 8); - let r = _mm256_permute2f128_si256(a, b, 0x20); + let r = _mm256_permute2f128_si256::<0x20>(a, b); let e = _mm256_setr_epi32(1, 2, 3, 4, 5, 6, 7, 8); assert_eq_m256i(r, e); } @@ -4019,7 +3815,7 @@ mod tests { unsafe fn test_mm256_insertf128_ps() { let a = _mm256_setr_ps(4., 3., 2., 5., 8., 9., 64., 50.); let b = _mm_setr_ps(4., 9., 16., 25.); - let r = _mm256_insertf128_ps(a, b, 0); + let r = _mm256_insertf128_ps::<0>(a, b); let e = _mm256_setr_ps(4., 9., 16., 25., 8., 9., 64., 50.); assert_eq_m256(r, e); } @@ -4028,7 +3824,7 @@ mod tests { unsafe fn test_mm256_insertf128_pd() { let a = _mm256_setr_pd(1., 2., 3., 4.); let b = _mm_setr_pd(5., 6.); - let r = _mm256_insertf128_pd(a, b, 0); + let r = _mm256_insertf128_pd::<0>(a, b); let e = _mm256_setr_pd(5., 6., 3., 4.); assert_eq_m256d(r, e); } @@ -4037,7 +3833,7 @@ mod tests { unsafe fn test_mm256_insertf128_si256() { let a = _mm256_setr_epi64x(1, 2, 3, 4); let b = _mm_setr_epi64x(5, 6); - let r = _mm256_insertf128_si256(a, b, 0); + let r = _mm256_insertf128_si256::<0>(a, b); let e = _mm256_setr_epi64x(5, 6, 3, 4); assert_eq_m256i(r, e); } @@ -4051,7 +3847,7 @@ mod tests { 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, ); - let r = _mm256_insert_epi8(a, 0, 31); + let r = _mm256_insert_epi8::<31>(a, 0); #[rustfmt::skip] let e = _mm256_setr_epi8( 1, 2, 3, 4, 5, 6, 7, 8, @@ -4069,7 +3865,7 @@ mod tests { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, ); - let r = _mm256_insert_epi16(a, 0, 15); + let r = _mm256_insert_epi16::<15>(a, 0); #[rustfmt::skip] let e = _mm256_setr_epi16( 0, 1, 2, 3, 4, 5, 6, 7, @@ -4081,7 +3877,7 @@ mod tests { #[simd_test(enable = "avx")] unsafe fn test_mm256_insert_epi32() { let a = _mm256_setr_epi32(1, 2, 3, 4, 5, 6, 7, 8); - let r = _mm256_insert_epi32(a, 0, 7); + let r = _mm256_insert_epi32::<7>(a, 0); let e = _mm256_setr_epi32(1, 2, 3, 4, 5, 6, 7, 0); assert_eq_m256i(r, e); } diff --git a/library/stdarch/crates/core_arch/src/x86/avx2.rs b/library/stdarch/crates/core_arch/src/x86/avx2.rs index 358ce15081..081609ecea 100644 --- a/library/stdarch/crates/core_arch/src/x86/avx2.rs +++ b/library/stdarch/crates/core_arch/src/x86/avx2.rs @@ -153,29 +153,29 @@ pub unsafe fn _mm256_adds_epu16(a: __m256i, b: __m256i) -> __m256i { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_alignr_epi8) #[inline] #[target_feature(enable = "avx2")] -#[cfg_attr(test, assert_instr(vpalignr, n = 7))] -#[rustc_args_required_const(2)] +#[cfg_attr(test, assert_instr(vpalignr, IMM8 = 7))] +#[rustc_legacy_const_generics(2)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm256_alignr_epi8(a: __m256i, b: __m256i, n: i32) -> __m256i { - let n = n as u32; - // If `palignr` is shifting the pair of vectors more than the size of two +pub unsafe fn _mm256_alignr_epi8(a: __m256i, b: __m256i) -> __m256i { + static_assert_imm8!(IMM8); + // If palignr is shifting the pair of vectors more than the size of two // lanes, emit zero. - if n > 32 { + if IMM8 > 32 { return _mm256_set1_epi8(0); } - // If `palignr` is shifting the pair of input vectors more than one lane, + // If palignr is shifting the pair of input vectors more than one lane, // but less than two lanes, convert to shifting in zeroes. - let (a, b, n) = if n > 16 { - (_mm256_set1_epi8(0), a, n - 16) + let (a, b) = if IMM8 > 16 { + (_mm256_set1_epi8(0), a) } else { - (a, b, n) + (a, b) }; let a = a.as_i8x32(); let b = b.as_i8x32(); - let r: i8x32 = match n { - 0 => simd_shuffle32( + let r: i8x32 = match IMM8 % 16 { + 0 => simd_shuffle32!( b, a, [ @@ -183,7 +183,7 @@ pub unsafe fn _mm256_alignr_epi8(a: __m256i, b: __m256i, n: i32) -> __m256i { 23, 24, 25, 26, 27, 28, 29, 30, 31, ], ), - 1 => simd_shuffle32( + 1 => simd_shuffle32!( b, a, [ @@ -191,7 +191,7 @@ pub unsafe fn _mm256_alignr_epi8(a: __m256i, b: __m256i, n: i32) -> __m256i { 24, 25, 26, 27, 28, 29, 30, 31, 48, ], ), - 2 => simd_shuffle32( + 2 => simd_shuffle32!( b, a, [ @@ -199,7 +199,7 @@ pub unsafe fn _mm256_alignr_epi8(a: __m256i, b: __m256i, n: i32) -> __m256i { 25, 26, 27, 28, 29, 30, 31, 48, 49, ], ), - 3 => simd_shuffle32( + 3 => simd_shuffle32!( b, a, [ @@ -207,7 +207,7 @@ pub unsafe fn _mm256_alignr_epi8(a: __m256i, b: __m256i, n: i32) -> __m256i { 25, 26, 27, 28, 29, 30, 31, 48, 49, 50, ], ), - 4 => simd_shuffle32( + 4 => simd_shuffle32!( b, a, [ @@ -215,7 +215,7 @@ pub unsafe fn _mm256_alignr_epi8(a: __m256i, b: __m256i, n: i32) -> __m256i { 26, 27, 28, 29, 30, 31, 48, 49, 50, 51, ], ), - 5 => simd_shuffle32( + 5 => simd_shuffle32!( b, a, [ @@ -223,7 +223,7 @@ pub unsafe fn _mm256_alignr_epi8(a: __m256i, b: __m256i, n: i32) -> __m256i { 27, 28, 29, 30, 31, 48, 49, 50, 51, 52, ], ), - 6 => simd_shuffle32( + 6 => simd_shuffle32!( b, a, [ @@ -231,7 +231,7 @@ pub unsafe fn _mm256_alignr_epi8(a: __m256i, b: __m256i, n: i32) -> __m256i { 28, 29, 30, 31, 48, 49, 50, 51, 52, 53, ], ), - 7 => simd_shuffle32( + 7 => simd_shuffle32!( b, a, [ @@ -239,7 +239,7 @@ pub unsafe fn _mm256_alignr_epi8(a: __m256i, b: __m256i, n: i32) -> __m256i { 28, 29, 30, 31, 48, 49, 50, 51, 52, 53, 54, ], ), - 8 => simd_shuffle32( + 8 => simd_shuffle32!( b, a, [ @@ -247,7 +247,7 @@ pub unsafe fn _mm256_alignr_epi8(a: __m256i, b: __m256i, n: i32) -> __m256i { 29, 30, 31, 48, 49, 50, 51, 52, 53, 54, 55, ], ), - 9 => simd_shuffle32( + 9 => simd_shuffle32!( b, a, [ @@ -255,7 +255,7 @@ pub unsafe fn _mm256_alignr_epi8(a: __m256i, b: __m256i, n: i32) -> __m256i { 30, 31, 48, 49, 50, 51, 52, 53, 54, 55, 56, ], ), - 10 => simd_shuffle32( + 10 => simd_shuffle32!( b, a, [ @@ -263,7 +263,7 @@ pub unsafe fn _mm256_alignr_epi8(a: __m256i, b: __m256i, n: i32) -> __m256i { 31, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, ], ), - 11 => simd_shuffle32( + 11 => simd_shuffle32!( b, a, [ @@ -271,7 +271,7 @@ pub unsafe fn _mm256_alignr_epi8(a: __m256i, b: __m256i, n: i32) -> __m256i { 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, ], ), - 12 => simd_shuffle32( + 12 => simd_shuffle32!( b, a, [ @@ -279,7 +279,7 @@ pub unsafe fn _mm256_alignr_epi8(a: __m256i, b: __m256i, n: i32) -> __m256i { 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, ], ), - 13 => simd_shuffle32( + 13 => simd_shuffle32!( b, a, [ @@ -287,7 +287,7 @@ pub unsafe fn _mm256_alignr_epi8(a: __m256i, b: __m256i, n: i32) -> __m256i { 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, ], ), - 14 => simd_shuffle32( + 14 => simd_shuffle32!( b, a, [ @@ -295,7 +295,7 @@ pub unsafe fn _mm256_alignr_epi8(a: __m256i, b: __m256i, n: i32) -> __m256i { 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, ], ), - 15 => simd_shuffle32( + 15 => simd_shuffle32!( b, a, [ @@ -358,209 +358,95 @@ pub unsafe fn _mm256_avg_epu8(a: __m256i, b: __m256i) -> __m256i { transmute(pavgb(a.as_u8x32(), b.as_u8x32())) } -/// Blends packed 32-bit integers from `a` and `b` using control mask `imm8`. +/// Blends packed 32-bit integers from `a` and `b` using control mask `IMM4`. /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_blend_epi32) #[inline] #[target_feature(enable = "avx2")] -#[cfg_attr(test, assert_instr(vblendps, imm8 = 9))] -#[rustc_args_required_const(2)] +#[cfg_attr(test, assert_instr(vblendps, IMM4 = 9))] +#[rustc_legacy_const_generics(2)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm_blend_epi32(a: __m128i, b: __m128i, imm8: i32) -> __m128i { - let imm8 = (imm8 & 0xFF) as u8; +pub unsafe fn _mm_blend_epi32(a: __m128i, b: __m128i) -> __m128i { + static_assert_imm4!(IMM4); let a = a.as_i32x4(); let b = b.as_i32x4(); - macro_rules! blend2 { - ($a:expr, $b:expr, $c:expr, $d:expr) => { - simd_shuffle4(a, b, [$a, $b, $c, $d]) - }; - } - macro_rules! blend1 { - ($a:expr, $b:expr) => { - match (imm8 >> 2) & 0b11 { - 0b00 => blend2!($a, $b, 2, 3), - 0b01 => blend2!($a, $b, 6, 3), - 0b10 => blend2!($a, $b, 2, 7), - _ => blend2!($a, $b, 6, 7), - } - }; - } - let r: i32x4 = match imm8 & 0b11 { - 0b00 => blend1!(0, 1), - 0b01 => blend1!(4, 1), - 0b10 => blend1!(0, 5), - _ => blend1!(4, 5), - }; + let r: i32x4 = simd_shuffle4!( + a, + b, + [ + [0, 4, 0, 4][IMM4 as usize & 0b11], + [1, 1, 5, 5][IMM4 as usize & 0b11], + [2, 6, 2, 6][(IMM4 as usize >> 2) & 0b11], + [3, 3, 7, 7][(IMM4 as usize >> 2) & 0b11], + ], + ); transmute(r) } -/// Blends packed 32-bit integers from `a` and `b` using control mask `imm8`. +/// Blends packed 32-bit integers from `a` and `b` using control mask `IMM8`. /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_blend_epi32) #[inline] #[target_feature(enable = "avx2")] -#[cfg_attr(test, assert_instr(vblendps, imm8 = 9))] -#[rustc_args_required_const(2)] +#[cfg_attr(test, assert_instr(vblendps, IMM8 = 9))] +#[rustc_legacy_const_generics(2)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm256_blend_epi32(a: __m256i, b: __m256i, imm8: i32) -> __m256i { - let imm8 = (imm8 & 0xFF) as u8; +pub unsafe fn _mm256_blend_epi32(a: __m256i, b: __m256i) -> __m256i { + static_assert_imm8!(IMM8); let a = a.as_i32x8(); let b = b.as_i32x8(); - macro_rules! blend4 { - ( - $a:expr, - $b:expr, - $c:expr, - $d:expr, - $e:expr, - $f:expr, - $g:expr, - $h:expr - ) => { - simd_shuffle8(a, b, [$a, $b, $c, $d, $e, $f, $g, $h]) - }; - } - macro_rules! blend3 { - ($a:expr, $b:expr, $c:expr, $d:expr, $e:expr, $f:expr) => { - match (imm8 >> 6) & 0b11 { - 0b00 => blend4!($a, $b, $c, $d, $e, $f, 6, 7), - 0b01 => blend4!($a, $b, $c, $d, $e, $f, 14, 7), - 0b10 => blend4!($a, $b, $c, $d, $e, $f, 6, 15), - _ => blend4!($a, $b, $c, $d, $e, $f, 14, 15), - } - }; - } - macro_rules! blend2 { - ($a:expr, $b:expr, $c:expr, $d:expr) => { - match (imm8 >> 4) & 0b11 { - 0b00 => blend3!($a, $b, $c, $d, 4, 5), - 0b01 => blend3!($a, $b, $c, $d, 12, 5), - 0b10 => blend3!($a, $b, $c, $d, 4, 13), - _ => blend3!($a, $b, $c, $d, 12, 13), - } - }; - } - macro_rules! blend1 { - ($a:expr, $b:expr) => { - match (imm8 >> 2) & 0b11 { - 0b00 => blend2!($a, $b, 2, 3), - 0b01 => blend2!($a, $b, 10, 3), - 0b10 => blend2!($a, $b, 2, 11), - _ => blend2!($a, $b, 10, 11), - } - }; - } - let r: i32x8 = match imm8 & 0b11 { - 0b00 => blend1!(0, 1), - 0b01 => blend1!(8, 1), - 0b10 => blend1!(0, 9), - _ => blend1!(8, 9), - }; + let r: i32x8 = simd_shuffle8!( + a, + b, + [ + [0, 8, 0, 8][IMM8 as usize & 0b11], + [1, 1, 9, 9][IMM8 as usize & 0b11], + [2, 10, 2, 10][(IMM8 as usize >> 2) & 0b11], + [3, 3, 11, 11][(IMM8 as usize >> 2) & 0b11], + [4, 12, 4, 12][(IMM8 as usize >> 4) & 0b11], + [5, 5, 13, 13][(IMM8 as usize >> 4) & 0b11], + [6, 14, 6, 14][(IMM8 as usize >> 6) & 0b11], + [7, 7, 15, 15][(IMM8 as usize >> 6) & 0b11], + ], + ); transmute(r) } -/// Blends packed 16-bit integers from `a` and `b` using control mask `imm8`. +/// Blends packed 16-bit integers from `a` and `b` using control mask `IMM8`. /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_blend_epi16) #[inline] #[target_feature(enable = "avx2")] -#[cfg_attr(test, assert_instr(vpblendw, imm8 = 9))] -#[rustc_args_required_const(2)] +#[cfg_attr(test, assert_instr(vpblendw, IMM8 = 9))] +#[rustc_legacy_const_generics(2)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm256_blend_epi16(a: __m256i, b: __m256i, imm8: i32) -> __m256i { - let imm8 = (imm8 & 0xFF) as u8; +pub unsafe fn _mm256_blend_epi16(a: __m256i, b: __m256i) -> __m256i { + static_assert_imm8!(IMM8); let a = a.as_i16x16(); let b = b.as_i16x16(); - macro_rules! blend4 { - ( - $a:expr, - $b:expr, - $c:expr, - $d:expr, - $e:expr, - $f:expr, - $g:expr, - $h:expr, - $i:expr, - $j:expr, - $k:expr, - $l:expr, - $m:expr, - $n:expr, - $o:expr, - $p:expr - ) => { - simd_shuffle16( - a, - b, - [ - $a, $b, $c, $d, $e, $f, $g, $h, $i, $j, $k, $l, $m, $n, $o, $p, - ], - ) - }; - } - macro_rules! blend3 { - ( - $a:expr, - $b:expr, - $c:expr, - $d:expr, - $e:expr, - $f:expr, - $a2:expr, - $b2:expr, - $c2:expr, - $d2:expr, - $e2:expr, - $f2:expr - ) => { - match (imm8 >> 6) & 0b11 { - 0b00 => blend4!($a, $b, $c, $d, $e, $f, 6, 7, $a2, $b2, $c2, $d2, $e2, $f2, 14, 15), - 0b01 => { - blend4!($a, $b, $c, $d, $e, $f, 22, 7, $a2, $b2, $c2, $d2, $e2, $f2, 30, 15) - } - 0b10 => { - blend4!($a, $b, $c, $d, $e, $f, 6, 23, $a2, $b2, $c2, $d2, $e2, $f2, 14, 31) - } - _ => blend4!($a, $b, $c, $d, $e, $f, 22, 23, $a2, $b2, $c2, $d2, $e2, $f2, 30, 31), - } - }; - } - macro_rules! blend2 { - ( - $a:expr, - $b:expr, - $c:expr, - $d:expr, - $a2:expr, - $b2:expr, - $c2:expr, - $d2:expr - ) => { - match (imm8 >> 4) & 0b11 { - 0b00 => blend3!($a, $b, $c, $d, 4, 5, $a2, $b2, $c2, $d2, 12, 13), - 0b01 => blend3!($a, $b, $c, $d, 20, 5, $a2, $b2, $c2, $d2, 28, 13), - 0b10 => blend3!($a, $b, $c, $d, 4, 21, $a2, $b2, $c2, $d2, 12, 29), - _ => blend3!($a, $b, $c, $d, 20, 21, $a2, $b2, $c2, $d2, 28, 29), - } - }; - } - macro_rules! blend1 { - ($a1:expr, $b1:expr, $a2:expr, $b2:expr) => { - match (imm8 >> 2) & 0b11 { - 0b00 => blend2!($a1, $b1, 2, 3, $a2, $b2, 10, 11), - 0b01 => blend2!($a1, $b1, 18, 3, $a2, $b2, 26, 11), - 0b10 => blend2!($a1, $b1, 2, 19, $a2, $b2, 10, 27), - _ => blend2!($a1, $b1, 18, 19, $a2, $b2, 26, 27), - } - }; - } - let r: i16x16 = match imm8 & 0b11 { - 0b00 => blend1!(0, 1, 8, 9), - 0b01 => blend1!(16, 1, 24, 9), - 0b10 => blend1!(0, 17, 8, 25), - _ => blend1!(16, 17, 24, 25), - }; + + let r: i16x16 = simd_shuffle16!( + a, + b, + [ + [0, 16, 0, 16][IMM8 as usize & 0b11], + [1, 1, 17, 17][IMM8 as usize & 0b11], + [2, 18, 2, 18][(IMM8 as usize >> 2) & 0b11], + [3, 3, 19, 19][(IMM8 as usize >> 2) & 0b11], + [4, 20, 4, 20][(IMM8 as usize >> 4) & 0b11], + [5, 5, 21, 21][(IMM8 as usize >> 4) & 0b11], + [6, 22, 6, 22][(IMM8 as usize >> 6) & 0b11], + [7, 7, 23, 23][(IMM8 as usize >> 6) & 0b11], + [8, 24, 8, 24][IMM8 as usize & 0b11], + [9, 9, 25, 25][IMM8 as usize & 0b11], + [10, 26, 10, 26][(IMM8 as usize >> 2) & 0b11], + [11, 11, 27, 27][(IMM8 as usize >> 2) & 0b11], + [12, 28, 12, 28][(IMM8 as usize >> 4) & 0b11], + [13, 13, 29, 29][(IMM8 as usize >> 4) & 0b11], + [14, 30, 14, 30][(IMM8 as usize >> 6) & 0b11], + [15, 15, 31, 31][(IMM8 as usize >> 6) & 0b11], + ], + ); transmute(r) } @@ -585,7 +471,7 @@ pub unsafe fn _mm256_blendv_epi8(a: __m256i, b: __m256i, mask: __m256i) -> __m25 #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm_broadcastb_epi8(a: __m128i) -> __m128i { let zero = _mm_setzero_si128(); - let ret = simd_shuffle16(a.as_i8x16(), zero.as_i8x16(), [0_u32; 16]); + let ret = simd_shuffle16!(a.as_i8x16(), zero.as_i8x16(), [0_u32; 16]); transmute::(ret) } @@ -599,7 +485,7 @@ pub unsafe fn _mm_broadcastb_epi8(a: __m128i) -> __m128i { #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm256_broadcastb_epi8(a: __m128i) -> __m256i { let zero = _mm_setzero_si128(); - let ret = simd_shuffle32(a.as_i8x16(), zero.as_i8x16(), [0_u32; 32]); + let ret = simd_shuffle32!(a.as_i8x16(), zero.as_i8x16(), [0_u32; 32]); transmute::(ret) } @@ -615,7 +501,7 @@ pub unsafe fn _mm256_broadcastb_epi8(a: __m128i) -> __m256i { #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm_broadcastd_epi32(a: __m128i) -> __m128i { let zero = _mm_setzero_si128(); - let ret = simd_shuffle4(a.as_i32x4(), zero.as_i32x4(), [0_u32; 4]); + let ret = simd_shuffle4!(a.as_i32x4(), zero.as_i32x4(), [0_u32; 4]); transmute::(ret) } @@ -631,7 +517,7 @@ pub unsafe fn _mm_broadcastd_epi32(a: __m128i) -> __m128i { #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm256_broadcastd_epi32(a: __m128i) -> __m256i { let zero = _mm_setzero_si128(); - let ret = simd_shuffle8(a.as_i32x4(), zero.as_i32x4(), [0_u32; 8]); + let ret = simd_shuffle8!(a.as_i32x4(), zero.as_i32x4(), [0_u32; 8]); transmute::(ret) } @@ -645,7 +531,7 @@ pub unsafe fn _mm256_broadcastd_epi32(a: __m128i) -> __m256i { #[cfg_attr(test, assert_instr(vmovddup))] #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm_broadcastq_epi64(a: __m128i) -> __m128i { - let ret = simd_shuffle2(a.as_i64x2(), a.as_i64x2(), [0_u32; 2]); + let ret = simd_shuffle2!(a.as_i64x2(), a.as_i64x2(), [0_u32; 2]); transmute::(ret) } @@ -658,7 +544,7 @@ pub unsafe fn _mm_broadcastq_epi64(a: __m128i) -> __m128i { #[cfg_attr(test, assert_instr(vbroadcastsd))] #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm256_broadcastq_epi64(a: __m128i) -> __m256i { - let ret = simd_shuffle4(a.as_i64x2(), a.as_i64x2(), [0_u32; 4]); + let ret = simd_shuffle4!(a.as_i64x2(), a.as_i64x2(), [0_u32; 4]); transmute::(ret) } @@ -671,7 +557,7 @@ pub unsafe fn _mm256_broadcastq_epi64(a: __m128i) -> __m256i { #[cfg_attr(test, assert_instr(vmovddup))] #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm_broadcastsd_pd(a: __m128d) -> __m128d { - simd_shuffle2(a, _mm_setzero_pd(), [0_u32; 2]) + simd_shuffle2!(a, _mm_setzero_pd(), [0_u32; 2]) } /// Broadcasts the low double-precision (64-bit) floating-point element @@ -683,7 +569,7 @@ pub unsafe fn _mm_broadcastsd_pd(a: __m128d) -> __m128d { #[cfg_attr(test, assert_instr(vbroadcastsd))] #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm256_broadcastsd_pd(a: __m128d) -> __m256d { - simd_shuffle4(a, _mm_setzero_pd(), [0_u32; 4]) + simd_shuffle4!(a, _mm_setzero_pd(), [0_u32; 4]) } // N.B., `broadcastsi128_si256` is often compiled to `vinsertf128` or @@ -697,7 +583,7 @@ pub unsafe fn _mm256_broadcastsd_pd(a: __m128d) -> __m256d { #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm256_broadcastsi128_si256(a: __m128i) -> __m256i { let zero = _mm_setzero_si128(); - let ret = simd_shuffle4(a.as_i64x2(), zero.as_i64x2(), [0, 1, 0, 1]); + let ret = simd_shuffle4!(a.as_i64x2(), zero.as_i64x2(), [0, 1, 0, 1]); transmute::(ret) } @@ -710,7 +596,7 @@ pub unsafe fn _mm256_broadcastsi128_si256(a: __m128i) -> __m256i { #[cfg_attr(test, assert_instr(vbroadcastss))] #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm_broadcastss_ps(a: __m128) -> __m128 { - simd_shuffle4(a, _mm_setzero_ps(), [0_u32; 4]) + simd_shuffle4!(a, _mm_setzero_ps(), [0_u32; 4]) } /// Broadcasts the low single-precision (32-bit) floating-point element @@ -722,7 +608,7 @@ pub unsafe fn _mm_broadcastss_ps(a: __m128) -> __m128 { #[cfg_attr(test, assert_instr(vbroadcastss))] #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm256_broadcastss_ps(a: __m128) -> __m256 { - simd_shuffle8(a, _mm_setzero_ps(), [0_u32; 8]) + simd_shuffle8!(a, _mm_setzero_ps(), [0_u32; 8]) } /// Broadcasts the low packed 16-bit integer from a to all elements of @@ -735,7 +621,7 @@ pub unsafe fn _mm256_broadcastss_ps(a: __m128) -> __m256 { #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm_broadcastw_epi16(a: __m128i) -> __m128i { let zero = _mm_setzero_si128(); - let ret = simd_shuffle8(a.as_i16x8(), zero.as_i16x8(), [0_u32; 8]); + let ret = simd_shuffle8!(a.as_i16x8(), zero.as_i16x8(), [0_u32; 8]); transmute::(ret) } @@ -749,7 +635,7 @@ pub unsafe fn _mm_broadcastw_epi16(a: __m128i) -> __m128i { #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm256_broadcastw_epi16(a: __m128i) -> __m256i { let zero = _mm_setzero_si128(); - let ret = simd_shuffle16(a.as_i16x8(), zero.as_i16x8(), [0_u32; 16]); + let ret = simd_shuffle16!(a.as_i16x8(), zero.as_i16x8(), [0_u32; 16]); transmute::(ret) } @@ -861,7 +747,7 @@ pub unsafe fn _mm256_cvtepi16_epi32(a: __m128i) -> __m256i { #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm256_cvtepi16_epi64(a: __m128i) -> __m256i { let a = a.as_i16x8(); - let v64: i16x4 = simd_shuffle4(a, a, [0, 1, 2, 3]); + let v64: i16x4 = simd_shuffle4!(a, a, [0, 1, 2, 3]); transmute::(simd_cast(v64)) } @@ -896,7 +782,7 @@ pub unsafe fn _mm256_cvtepi8_epi16(a: __m128i) -> __m256i { #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm256_cvtepi8_epi32(a: __m128i) -> __m256i { let a = a.as_i8x16(); - let v64: i8x8 = simd_shuffle8(a, a, [0, 1, 2, 3, 4, 5, 6, 7]); + let v64: i8x8 = simd_shuffle8!(a, a, [0, 1, 2, 3, 4, 5, 6, 7]); transmute::(simd_cast(v64)) } @@ -909,7 +795,7 @@ pub unsafe fn _mm256_cvtepi8_epi32(a: __m128i) -> __m256i { #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm256_cvtepi8_epi64(a: __m128i) -> __m256i { let a = a.as_i8x16(); - let v32: i8x4 = simd_shuffle4(a, a, [0, 1, 2, 3]); + let v32: i8x4 = simd_shuffle4!(a, a, [0, 1, 2, 3]); transmute::(simd_cast(v32)) } @@ -935,7 +821,7 @@ pub unsafe fn _mm256_cvtepu16_epi32(a: __m128i) -> __m256i { #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm256_cvtepu16_epi64(a: __m128i) -> __m256i { let a = a.as_u16x8(); - let v64: u16x4 = simd_shuffle4(a, a, [0, 1, 2, 3]); + let v64: u16x4 = simd_shuffle4!(a, a, [0, 1, 2, 3]); transmute::(simd_cast(v64)) } @@ -971,7 +857,7 @@ pub unsafe fn _mm256_cvtepu8_epi16(a: __m128i) -> __m256i { #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm256_cvtepu8_epi32(a: __m128i) -> __m256i { let a = a.as_u8x16(); - let v64: u8x8 = simd_shuffle8(a, a, [0, 1, 2, 3, 4, 5, 6, 7]); + let v64: u8x8 = simd_shuffle8!(a, a, [0, 1, 2, 3, 4, 5, 6, 7]); transmute::(simd_cast(v64)) } @@ -985,28 +871,26 @@ pub unsafe fn _mm256_cvtepu8_epi32(a: __m128i) -> __m256i { #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm256_cvtepu8_epi64(a: __m128i) -> __m256i { let a = a.as_u8x16(); - let v32: u8x4 = simd_shuffle4(a, a, [0, 1, 2, 3]); + let v32: u8x4 = simd_shuffle4!(a, a, [0, 1, 2, 3]); transmute::(simd_cast(v32)) } -/// Extracts 128 bits (of integer data) from `a` selected with `imm8`. +/// Extracts 128 bits (of integer data) from `a` selected with `IMM1`. /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_extracti128_si256) #[inline] #[target_feature(enable = "avx2")] #[cfg_attr( all(test, not(target_os = "windows")), - assert_instr(vextractf128, imm8 = 1) + assert_instr(vextractf128, IMM1 = 1) )] -#[rustc_args_required_const(1)] +#[rustc_legacy_const_generics(1)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm256_extracti128_si256(a: __m256i, imm8: i32) -> __m128i { +pub unsafe fn _mm256_extracti128_si256(a: __m256i) -> __m128i { + static_assert_imm1!(IMM1); let a = a.as_i64x4(); let b = _mm256_undefined_si256().as_i64x4(); - let dst: i64x2 = match imm8 & 0b01 { - 0 => simd_shuffle2(a, b, [0, 1]), - _ => simd_shuffle2(a, b, [2, 3]), - }; + let dst: i64x2 = simd_shuffle2!(a, b, [[0, 1], [2, 3]][IMM1 as usize]); transmute(dst) } @@ -1085,20 +969,19 @@ pub unsafe fn _mm256_hsubs_epi16(a: __m256i, b: __m256i) -> __m256i { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_i32gather_epi32) #[inline] #[target_feature(enable = "avx2")] -#[cfg_attr(test, assert_instr(vpgatherdd, scale = 1))] -#[rustc_args_required_const(2)] +#[cfg_attr(test, assert_instr(vpgatherdd, SCALE = 1))] +#[rustc_legacy_const_generics(2)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm_i32gather_epi32(slice: *const i32, offsets: __m128i, scale: i32) -> __m128i { +pub unsafe fn _mm_i32gather_epi32( + slice: *const i32, + offsets: __m128i, +) -> __m128i { + static_assert_imm8_scale!(SCALE); let zero = _mm_setzero_si128().as_i32x4(); let neg_one = _mm_set1_epi32(-1).as_i32x4(); let offsets = offsets.as_i32x4(); let slice = slice as *const i8; - macro_rules! call { - ($imm8:expr) => { - pgatherdd(zero, slice, offsets, neg_one, $imm8) - }; - } - let r = constify_imm8_gather!(scale, call); + let r = pgatherdd(zero, slice, offsets, neg_one, SCALE as i8); transmute(r) } @@ -1110,26 +993,21 @@ pub unsafe fn _mm_i32gather_epi32(slice: *const i32, offsets: __m128i, scale: i3 /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_i32gather_epi32) #[inline] #[target_feature(enable = "avx2")] -#[cfg_attr(test, assert_instr(vpgatherdd, scale = 1))] -#[rustc_args_required_const(4)] +#[cfg_attr(test, assert_instr(vpgatherdd, SCALE = 1))] +#[rustc_legacy_const_generics(4)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm_mask_i32gather_epi32( +pub unsafe fn _mm_mask_i32gather_epi32( src: __m128i, slice: *const i32, offsets: __m128i, mask: __m128i, - scale: i32, ) -> __m128i { + static_assert_imm8_scale!(SCALE); let src = src.as_i32x4(); let mask = mask.as_i32x4(); let offsets = offsets.as_i32x4(); let slice = slice as *const i8; - macro_rules! call { - ($imm8:expr) => { - pgatherdd(src, slice, offsets, mask, $imm8) - }; - } - let r = constify_imm8_gather!(scale, call); + let r = pgatherdd(src, slice, offsets, mask, SCALE as i8); transmute(r) } @@ -1140,20 +1018,19 @@ pub unsafe fn _mm_mask_i32gather_epi32( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_i32gather_epi32) #[inline] #[target_feature(enable = "avx2")] -#[cfg_attr(test, assert_instr(vpgatherdd, scale = 1))] -#[rustc_args_required_const(2)] +#[cfg_attr(test, assert_instr(vpgatherdd, SCALE = 1))] +#[rustc_legacy_const_generics(2)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm256_i32gather_epi32(slice: *const i32, offsets: __m256i, scale: i32) -> __m256i { +pub unsafe fn _mm256_i32gather_epi32( + slice: *const i32, + offsets: __m256i, +) -> __m256i { + static_assert_imm8_scale!(SCALE); let zero = _mm256_setzero_si256().as_i32x8(); let neg_one = _mm256_set1_epi32(-1).as_i32x8(); let offsets = offsets.as_i32x8(); let slice = slice as *const i8; - macro_rules! call { - ($imm8:expr) => { - vpgatherdd(zero, slice, offsets, neg_one, $imm8) - }; - } - let r = constify_imm8_gather!(scale, call); + let r = vpgatherdd(zero, slice, offsets, neg_one, SCALE as i8); transmute(r) } @@ -1165,26 +1042,21 @@ pub unsafe fn _mm256_i32gather_epi32(slice: *const i32, offsets: __m256i, scale: /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_i32gather_epi32) #[inline] #[target_feature(enable = "avx2")] -#[cfg_attr(test, assert_instr(vpgatherdd, scale = 1))] -#[rustc_args_required_const(4)] +#[cfg_attr(test, assert_instr(vpgatherdd, SCALE = 1))] +#[rustc_legacy_const_generics(4)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm256_mask_i32gather_epi32( +pub unsafe fn _mm256_mask_i32gather_epi32( src: __m256i, slice: *const i32, offsets: __m256i, mask: __m256i, - scale: i32, ) -> __m256i { + static_assert_imm8_scale!(SCALE); let src = src.as_i32x8(); let mask = mask.as_i32x8(); let offsets = offsets.as_i32x8(); let slice = slice as *const i8; - macro_rules! call { - ($imm8:expr) => { - vpgatherdd(src, slice, offsets, mask, $imm8) - }; - } - let r = constify_imm8_gather!(scale, call); + let r = vpgatherdd(src, slice, offsets, mask, SCALE as i8); transmute(r) } @@ -1195,20 +1067,16 @@ pub unsafe fn _mm256_mask_i32gather_epi32( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_i32gather_ps) #[inline] #[target_feature(enable = "avx2")] -#[cfg_attr(test, assert_instr(vgatherdps, scale = 1))] -#[rustc_args_required_const(2)] +#[cfg_attr(test, assert_instr(vgatherdps, SCALE = 1))] +#[rustc_legacy_const_generics(2)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm_i32gather_ps(slice: *const f32, offsets: __m128i, scale: i32) -> __m128 { +pub unsafe fn _mm_i32gather_ps(slice: *const f32, offsets: __m128i) -> __m128 { + static_assert_imm8_scale!(SCALE); let zero = _mm_setzero_ps(); let neg_one = _mm_set1_ps(-1.0); let offsets = offsets.as_i32x4(); let slice = slice as *const i8; - macro_rules! call { - ($imm8:expr) => { - pgatherdps(zero, slice, offsets, neg_one, $imm8) - }; - } - constify_imm8_gather!(scale, call) + pgatherdps(zero, slice, offsets, neg_one, SCALE as i8) } /// Returns values from `slice` at offsets determined by `offsets * scale`, @@ -1219,24 +1087,19 @@ pub unsafe fn _mm_i32gather_ps(slice: *const f32, offsets: __m128i, scale: i32) /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_i32gather_ps) #[inline] #[target_feature(enable = "avx2")] -#[cfg_attr(test, assert_instr(vgatherdps, scale = 1))] -#[rustc_args_required_const(4)] +#[cfg_attr(test, assert_instr(vgatherdps, SCALE = 1))] +#[rustc_legacy_const_generics(4)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm_mask_i32gather_ps( +pub unsafe fn _mm_mask_i32gather_ps( src: __m128, slice: *const f32, offsets: __m128i, mask: __m128, - scale: i32, ) -> __m128 { + static_assert_imm8_scale!(SCALE); let offsets = offsets.as_i32x4(); let slice = slice as *const i8; - macro_rules! call { - ($imm8:expr) => { - pgatherdps(src, slice, offsets, mask, $imm8) - }; - } - constify_imm8_gather!(scale, call) + pgatherdps(src, slice, offsets, mask, SCALE as i8) } /// Returns values from `slice` at offsets determined by `offsets * scale`, @@ -1246,20 +1109,16 @@ pub unsafe fn _mm_mask_i32gather_ps( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_i32gather_ps) #[inline] #[target_feature(enable = "avx2")] -#[cfg_attr(test, assert_instr(vgatherdps, scale = 1))] -#[rustc_args_required_const(2)] +#[cfg_attr(test, assert_instr(vgatherdps, SCALE = 1))] +#[rustc_legacy_const_generics(2)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm256_i32gather_ps(slice: *const f32, offsets: __m256i, scale: i32) -> __m256 { +pub unsafe fn _mm256_i32gather_ps(slice: *const f32, offsets: __m256i) -> __m256 { + static_assert_imm8_scale!(SCALE); let zero = _mm256_setzero_ps(); let neg_one = _mm256_set1_ps(-1.0); let offsets = offsets.as_i32x8(); let slice = slice as *const i8; - macro_rules! call { - ($imm8:expr) => { - vpgatherdps(zero, slice, offsets, neg_one, $imm8) - }; - } - constify_imm8_gather!(scale, call) + vpgatherdps(zero, slice, offsets, neg_one, SCALE as i8) } /// Returns values from `slice` at offsets determined by `offsets * scale`, @@ -1270,24 +1129,19 @@ pub unsafe fn _mm256_i32gather_ps(slice: *const f32, offsets: __m256i, scale: i3 /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_i32gather_ps) #[inline] #[target_feature(enable = "avx2")] -#[cfg_attr(test, assert_instr(vgatherdps, scale = 1))] -#[rustc_args_required_const(4)] +#[cfg_attr(test, assert_instr(vgatherdps, SCALE = 1))] +#[rustc_legacy_const_generics(4)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm256_mask_i32gather_ps( +pub unsafe fn _mm256_mask_i32gather_ps( src: __m256, slice: *const f32, offsets: __m256i, mask: __m256, - scale: i32, ) -> __m256 { + static_assert_imm8_scale!(SCALE); let offsets = offsets.as_i32x8(); let slice = slice as *const i8; - macro_rules! call { - ($imm8:expr) => { - vpgatherdps(src, slice, offsets, mask, $imm8) - }; - } - constify_imm8_gather!(scale, call) + vpgatherdps(src, slice, offsets, mask, SCALE as i8) } /// Returns values from `slice` at offsets determined by `offsets * scale`, @@ -1297,20 +1151,19 @@ pub unsafe fn _mm256_mask_i32gather_ps( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_i32gather_epi64) #[inline] #[target_feature(enable = "avx2")] -#[cfg_attr(test, assert_instr(vpgatherdq, scale = 1))] -#[rustc_args_required_const(2)] +#[cfg_attr(test, assert_instr(vpgatherdq, SCALE = 1))] +#[rustc_legacy_const_generics(2)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm_i32gather_epi64(slice: *const i64, offsets: __m128i, scale: i32) -> __m128i { +pub unsafe fn _mm_i32gather_epi64( + slice: *const i64, + offsets: __m128i, +) -> __m128i { + static_assert_imm8_scale!(SCALE); let zero = _mm_setzero_si128().as_i64x2(); let neg_one = _mm_set1_epi64x(-1).as_i64x2(); let offsets = offsets.as_i32x4(); let slice = slice as *const i8; - macro_rules! call { - ($imm8:expr) => { - pgatherdq(zero, slice, offsets, neg_one, $imm8) - }; - } - let r = constify_imm8_gather!(scale, call); + let r = pgatherdq(zero, slice, offsets, neg_one, SCALE as i8); transmute(r) } @@ -1322,26 +1175,21 @@ pub unsafe fn _mm_i32gather_epi64(slice: *const i64, offsets: __m128i, scale: i3 /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_i32gather_epi64) #[inline] #[target_feature(enable = "avx2")] -#[cfg_attr(test, assert_instr(vpgatherdq, scale = 1))] -#[rustc_args_required_const(4)] +#[cfg_attr(test, assert_instr(vpgatherdq, SCALE = 1))] +#[rustc_legacy_const_generics(4)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm_mask_i32gather_epi64( +pub unsafe fn _mm_mask_i32gather_epi64( src: __m128i, slice: *const i64, offsets: __m128i, mask: __m128i, - scale: i32, ) -> __m128i { + static_assert_imm8_scale!(SCALE); let src = src.as_i64x2(); let mask = mask.as_i64x2(); let offsets = offsets.as_i32x4(); let slice = slice as *const i8; - macro_rules! call { - ($imm8:expr) => { - pgatherdq(src, slice, offsets, mask, $imm8) - }; - } - let r = constify_imm8_gather!(scale, call); + let r = pgatherdq(src, slice, offsets, mask, SCALE as i8); transmute(r) } @@ -1352,20 +1200,19 @@ pub unsafe fn _mm_mask_i32gather_epi64( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_i32gather_epi64) #[inline] #[target_feature(enable = "avx2")] -#[cfg_attr(test, assert_instr(vpgatherdq, scale = 1))] -#[rustc_args_required_const(2)] +#[cfg_attr(test, assert_instr(vpgatherdq, SCALE = 1))] +#[rustc_legacy_const_generics(2)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm256_i32gather_epi64(slice: *const i64, offsets: __m128i, scale: i32) -> __m256i { +pub unsafe fn _mm256_i32gather_epi64( + slice: *const i64, + offsets: __m128i, +) -> __m256i { + static_assert_imm8_scale!(SCALE); let zero = _mm256_setzero_si256().as_i64x4(); let neg_one = _mm256_set1_epi64x(-1).as_i64x4(); let offsets = offsets.as_i32x4(); let slice = slice as *const i8; - macro_rules! call { - ($imm8:expr) => { - vpgatherdq(zero, slice, offsets, neg_one, $imm8) - }; - } - let r = constify_imm8_gather!(scale, call); + let r = vpgatherdq(zero, slice, offsets, neg_one, SCALE as i8); transmute(r) } @@ -1377,26 +1224,21 @@ pub unsafe fn _mm256_i32gather_epi64(slice: *const i64, offsets: __m128i, scale: /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_i32gather_epi64) #[inline] #[target_feature(enable = "avx2")] -#[cfg_attr(test, assert_instr(vpgatherdq, scale = 1))] -#[rustc_args_required_const(4)] +#[cfg_attr(test, assert_instr(vpgatherdq, SCALE = 1))] +#[rustc_legacy_const_generics(4)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm256_mask_i32gather_epi64( +pub unsafe fn _mm256_mask_i32gather_epi64( src: __m256i, slice: *const i64, offsets: __m128i, mask: __m256i, - scale: i32, ) -> __m256i { + static_assert_imm8_scale!(SCALE); let src = src.as_i64x4(); let mask = mask.as_i64x4(); let offsets = offsets.as_i32x4(); let slice = slice as *const i8; - macro_rules! call { - ($imm8:expr) => { - vpgatherdq(src, slice, offsets, mask, $imm8) - }; - } - let r = constify_imm8_gather!(scale, call); + let r = vpgatherdq(src, slice, offsets, mask, SCALE as i8); transmute(r) } @@ -1407,20 +1249,16 @@ pub unsafe fn _mm256_mask_i32gather_epi64( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_i32gather_pd) #[inline] #[target_feature(enable = "avx2")] -#[cfg_attr(test, assert_instr(vgatherdpd, scale = 1))] -#[rustc_args_required_const(2)] +#[cfg_attr(test, assert_instr(vgatherdpd, SCALE = 1))] +#[rustc_legacy_const_generics(2)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm_i32gather_pd(slice: *const f64, offsets: __m128i, scale: i32) -> __m128d { +pub unsafe fn _mm_i32gather_pd(slice: *const f64, offsets: __m128i) -> __m128d { + static_assert_imm8_scale!(SCALE); let zero = _mm_setzero_pd(); let neg_one = _mm_set1_pd(-1.0); let offsets = offsets.as_i32x4(); let slice = slice as *const i8; - macro_rules! call { - ($imm8:expr) => { - pgatherdpd(zero, slice, offsets, neg_one, $imm8) - }; - } - constify_imm8_gather!(scale, call) + pgatherdpd(zero, slice, offsets, neg_one, SCALE as i8) } /// Returns values from `slice` at offsets determined by `offsets * scale`, @@ -1431,24 +1269,19 @@ pub unsafe fn _mm_i32gather_pd(slice: *const f64, offsets: __m128i, scale: i32) /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_i32gather_pd) #[inline] #[target_feature(enable = "avx2")] -#[cfg_attr(test, assert_instr(vgatherdpd, scale = 1))] -#[rustc_args_required_const(4)] +#[cfg_attr(test, assert_instr(vgatherdpd, SCALE = 1))] +#[rustc_legacy_const_generics(4)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm_mask_i32gather_pd( +pub unsafe fn _mm_mask_i32gather_pd( src: __m128d, slice: *const f64, offsets: __m128i, mask: __m128d, - scale: i32, ) -> __m128d { + static_assert_imm8_scale!(SCALE); let offsets = offsets.as_i32x4(); let slice = slice as *const i8; - macro_rules! call { - ($imm8:expr) => { - pgatherdpd(src, slice, offsets, mask, $imm8) - }; - } - constify_imm8_gather!(scale, call) + pgatherdpd(src, slice, offsets, mask, SCALE as i8) } /// Returns values from `slice` at offsets determined by `offsets * scale`, @@ -1458,20 +1291,19 @@ pub unsafe fn _mm_mask_i32gather_pd( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_i32gather_pd) #[inline] #[target_feature(enable = "avx2")] -#[cfg_attr(test, assert_instr(vgatherdpd, scale = 1))] -#[rustc_args_required_const(2)] +#[cfg_attr(test, assert_instr(vgatherdpd, SCALE = 1))] +#[rustc_legacy_const_generics(2)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm256_i32gather_pd(slice: *const f64, offsets: __m128i, scale: i32) -> __m256d { +pub unsafe fn _mm256_i32gather_pd( + slice: *const f64, + offsets: __m128i, +) -> __m256d { + static_assert_imm8_scale!(SCALE); let zero = _mm256_setzero_pd(); let neg_one = _mm256_set1_pd(-1.0); let offsets = offsets.as_i32x4(); let slice = slice as *const i8; - macro_rules! call { - ($imm8:expr) => { - vpgatherdpd(zero, slice, offsets, neg_one, $imm8) - }; - } - constify_imm8_gather!(scale, call) + vpgatherdpd(zero, slice, offsets, neg_one, SCALE as i8) } /// Returns values from `slice` at offsets determined by `offsets * scale`, @@ -1482,24 +1314,19 @@ pub unsafe fn _mm256_i32gather_pd(slice: *const f64, offsets: __m128i, scale: i3 /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_i32gather_pd) #[inline] #[target_feature(enable = "avx2")] -#[cfg_attr(test, assert_instr(vgatherdpd, scale = 1))] -#[rustc_args_required_const(4)] +#[cfg_attr(test, assert_instr(vgatherdpd, SCALE = 1))] +#[rustc_legacy_const_generics(4)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm256_mask_i32gather_pd( +pub unsafe fn _mm256_mask_i32gather_pd( src: __m256d, slice: *const f64, offsets: __m128i, mask: __m256d, - scale: i32, ) -> __m256d { + static_assert_imm8_scale!(SCALE); let offsets = offsets.as_i32x4(); let slice = slice as *const i8; - macro_rules! call { - ($imm8:expr) => { - vpgatherdpd(src, slice, offsets, mask, $imm8) - }; - } - constify_imm8_gather!(scale, call) + vpgatherdpd(src, slice, offsets, mask, SCALE as i8) } /// Returns values from `slice` at offsets determined by `offsets * scale`, @@ -1509,20 +1336,19 @@ pub unsafe fn _mm256_mask_i32gather_pd( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_i64gather_epi32) #[inline] #[target_feature(enable = "avx2")] -#[cfg_attr(test, assert_instr(vpgatherqd, scale = 1))] -#[rustc_args_required_const(2)] +#[cfg_attr(test, assert_instr(vpgatherqd, SCALE = 1))] +#[rustc_legacy_const_generics(2)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm_i64gather_epi32(slice: *const i32, offsets: __m128i, scale: i32) -> __m128i { +pub unsafe fn _mm_i64gather_epi32( + slice: *const i32, + offsets: __m128i, +) -> __m128i { + static_assert_imm8_scale!(SCALE); let zero = _mm_setzero_si128().as_i32x4(); let neg_one = _mm_set1_epi64x(-1).as_i32x4(); let offsets = offsets.as_i64x2(); let slice = slice as *const i8; - macro_rules! call { - ($imm8:expr) => { - pgatherqd(zero, slice, offsets, neg_one, $imm8) - }; - } - let r = constify_imm8_gather!(scale, call); + let r = pgatherqd(zero, slice, offsets, neg_one, SCALE as i8); transmute(r) } @@ -1534,26 +1360,21 @@ pub unsafe fn _mm_i64gather_epi32(slice: *const i32, offsets: __m128i, scale: i3 /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_i64gather_epi32) #[inline] #[target_feature(enable = "avx2")] -#[cfg_attr(test, assert_instr(vpgatherqd, scale = 1))] -#[rustc_args_required_const(4)] +#[cfg_attr(test, assert_instr(vpgatherqd, SCALE = 1))] +#[rustc_legacy_const_generics(4)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm_mask_i64gather_epi32( +pub unsafe fn _mm_mask_i64gather_epi32( src: __m128i, slice: *const i32, offsets: __m128i, mask: __m128i, - scale: i32, ) -> __m128i { + static_assert_imm8_scale!(SCALE); let src = src.as_i32x4(); let mask = mask.as_i32x4(); let offsets = offsets.as_i64x2(); let slice = slice as *const i8; - macro_rules! call { - ($imm8:expr) => { - pgatherqd(src, slice, offsets, mask, $imm8) - }; - } - let r = constify_imm8_gather!(scale, call); + let r = pgatherqd(src, slice, offsets, mask, SCALE as i8); transmute(r) } @@ -1564,20 +1385,19 @@ pub unsafe fn _mm_mask_i64gather_epi32( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_i64gather_epi32) #[inline] #[target_feature(enable = "avx2")] -#[cfg_attr(test, assert_instr(vpgatherqd, scale = 1))] -#[rustc_args_required_const(2)] +#[cfg_attr(test, assert_instr(vpgatherqd, SCALE = 1))] +#[rustc_legacy_const_generics(2)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm256_i64gather_epi32(slice: *const i32, offsets: __m256i, scale: i32) -> __m128i { +pub unsafe fn _mm256_i64gather_epi32( + slice: *const i32, + offsets: __m256i, +) -> __m128i { + static_assert_imm8_scale!(SCALE); let zero = _mm_setzero_si128().as_i32x4(); let neg_one = _mm_set1_epi64x(-1).as_i32x4(); let offsets = offsets.as_i64x4(); let slice = slice as *const i8; - macro_rules! call { - ($imm8:expr) => { - vpgatherqd(zero, slice, offsets, neg_one, $imm8) - }; - } - let r = constify_imm8_gather!(scale, call); + let r = vpgatherqd(zero, slice, offsets, neg_one, SCALE as i8); transmute(r) } @@ -1589,26 +1409,21 @@ pub unsafe fn _mm256_i64gather_epi32(slice: *const i32, offsets: __m256i, scale: /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_i64gather_epi32) #[inline] #[target_feature(enable = "avx2")] -#[cfg_attr(test, assert_instr(vpgatherqd, scale = 1))] -#[rustc_args_required_const(4)] +#[cfg_attr(test, assert_instr(vpgatherqd, SCALE = 1))] +#[rustc_legacy_const_generics(4)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm256_mask_i64gather_epi32( +pub unsafe fn _mm256_mask_i64gather_epi32( src: __m128i, slice: *const i32, offsets: __m256i, mask: __m128i, - scale: i32, ) -> __m128i { + static_assert_imm8_scale!(SCALE); let src = src.as_i32x4(); let mask = mask.as_i32x4(); let offsets = offsets.as_i64x4(); let slice = slice as *const i8; - macro_rules! call { - ($imm8:expr) => { - vpgatherqd(src, slice, offsets, mask, $imm8) - }; - } - let r = constify_imm8_gather!(scale, call); + let r = vpgatherqd(src, slice, offsets, mask, SCALE as i8); transmute(r) } @@ -1619,20 +1434,16 @@ pub unsafe fn _mm256_mask_i64gather_epi32( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_i64gather_ps) #[inline] #[target_feature(enable = "avx2")] -#[cfg_attr(test, assert_instr(vgatherqps, scale = 1))] -#[rustc_args_required_const(2)] +#[cfg_attr(test, assert_instr(vgatherqps, SCALE = 1))] +#[rustc_legacy_const_generics(2)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm_i64gather_ps(slice: *const f32, offsets: __m128i, scale: i32) -> __m128 { +pub unsafe fn _mm_i64gather_ps(slice: *const f32, offsets: __m128i) -> __m128 { + static_assert_imm8_scale!(SCALE); let zero = _mm_setzero_ps(); let neg_one = _mm_set1_ps(-1.0); let offsets = offsets.as_i64x2(); let slice = slice as *const i8; - macro_rules! call { - ($imm8:expr) => { - pgatherqps(zero, slice, offsets, neg_one, $imm8) - }; - } - constify_imm8_gather!(scale, call) + pgatherqps(zero, slice, offsets, neg_one, SCALE as i8) } /// Returns values from `slice` at offsets determined by `offsets * scale`, @@ -1643,24 +1454,19 @@ pub unsafe fn _mm_i64gather_ps(slice: *const f32, offsets: __m128i, scale: i32) /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_i64gather_ps) #[inline] #[target_feature(enable = "avx2")] -#[cfg_attr(test, assert_instr(vgatherqps, scale = 1))] -#[rustc_args_required_const(4)] +#[cfg_attr(test, assert_instr(vgatherqps, SCALE = 1))] +#[rustc_legacy_const_generics(4)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm_mask_i64gather_ps( +pub unsafe fn _mm_mask_i64gather_ps( src: __m128, slice: *const f32, offsets: __m128i, mask: __m128, - scale: i32, ) -> __m128 { + static_assert_imm8_scale!(SCALE); let offsets = offsets.as_i64x2(); let slice = slice as *const i8; - macro_rules! call { - ($imm8:expr) => { - pgatherqps(src, slice, offsets, mask, $imm8) - }; - } - constify_imm8_gather!(scale, call) + pgatherqps(src, slice, offsets, mask, SCALE as i8) } /// Returns values from `slice` at offsets determined by `offsets * scale`, @@ -1670,20 +1476,16 @@ pub unsafe fn _mm_mask_i64gather_ps( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_i64gather_ps) #[inline] #[target_feature(enable = "avx2")] -#[cfg_attr(test, assert_instr(vgatherqps, scale = 1))] -#[rustc_args_required_const(2)] +#[cfg_attr(test, assert_instr(vgatherqps, SCALE = 1))] +#[rustc_legacy_const_generics(2)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm256_i64gather_ps(slice: *const f32, offsets: __m256i, scale: i32) -> __m128 { +pub unsafe fn _mm256_i64gather_ps(slice: *const f32, offsets: __m256i) -> __m128 { + static_assert_imm8_scale!(SCALE); let zero = _mm_setzero_ps(); let neg_one = _mm_set1_ps(-1.0); let offsets = offsets.as_i64x4(); let slice = slice as *const i8; - macro_rules! call { - ($imm8:expr) => { - vpgatherqps(zero, slice, offsets, neg_one, $imm8) - }; - } - constify_imm8_gather!(scale, call) + vpgatherqps(zero, slice, offsets, neg_one, SCALE as i8) } /// Returns values from `slice` at offsets determined by `offsets * scale`, @@ -1694,24 +1496,19 @@ pub unsafe fn _mm256_i64gather_ps(slice: *const f32, offsets: __m256i, scale: i3 /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_i64gather_ps) #[inline] #[target_feature(enable = "avx2")] -#[cfg_attr(test, assert_instr(vgatherqps, scale = 1))] -#[rustc_args_required_const(4)] +#[cfg_attr(test, assert_instr(vgatherqps, SCALE = 1))] +#[rustc_legacy_const_generics(4)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm256_mask_i64gather_ps( +pub unsafe fn _mm256_mask_i64gather_ps( src: __m128, slice: *const f32, offsets: __m256i, mask: __m128, - scale: i32, ) -> __m128 { + static_assert_imm8_scale!(SCALE); let offsets = offsets.as_i64x4(); let slice = slice as *const i8; - macro_rules! call { - ($imm8:expr) => { - vpgatherqps(src, slice, offsets, mask, $imm8) - }; - } - constify_imm8_gather!(scale, call) + vpgatherqps(src, slice, offsets, mask, SCALE as i8) } /// Returns values from `slice` at offsets determined by `offsets * scale`, @@ -1721,20 +1518,19 @@ pub unsafe fn _mm256_mask_i64gather_ps( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_i64gather_epi64) #[inline] #[target_feature(enable = "avx2")] -#[cfg_attr(test, assert_instr(vpgatherqq, scale = 1))] -#[rustc_args_required_const(2)] +#[cfg_attr(test, assert_instr(vpgatherqq, SCALE = 1))] +#[rustc_legacy_const_generics(2)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm_i64gather_epi64(slice: *const i64, offsets: __m128i, scale: i32) -> __m128i { +pub unsafe fn _mm_i64gather_epi64( + slice: *const i64, + offsets: __m128i, +) -> __m128i { + static_assert_imm8_scale!(SCALE); let zero = _mm_setzero_si128().as_i64x2(); let neg_one = _mm_set1_epi64x(-1).as_i64x2(); let slice = slice as *const i8; let offsets = offsets.as_i64x2(); - macro_rules! call { - ($imm8:expr) => { - pgatherqq(zero, slice, offsets, neg_one, $imm8) - }; - } - let r = constify_imm8_gather!(scale, call); + let r = pgatherqq(zero, slice, offsets, neg_one, SCALE as i8); transmute(r) } @@ -1746,26 +1542,21 @@ pub unsafe fn _mm_i64gather_epi64(slice: *const i64, offsets: __m128i, scale: i3 /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_i64gather_epi64) #[inline] #[target_feature(enable = "avx2")] -#[cfg_attr(test, assert_instr(vpgatherqq, scale = 1))] -#[rustc_args_required_const(4)] +#[cfg_attr(test, assert_instr(vpgatherqq, SCALE = 1))] +#[rustc_legacy_const_generics(4)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm_mask_i64gather_epi64( +pub unsafe fn _mm_mask_i64gather_epi64( src: __m128i, slice: *const i64, offsets: __m128i, mask: __m128i, - scale: i32, ) -> __m128i { + static_assert_imm8_scale!(SCALE); let src = src.as_i64x2(); let mask = mask.as_i64x2(); let offsets = offsets.as_i64x2(); let slice = slice as *const i8; - macro_rules! call { - ($imm8:expr) => { - pgatherqq(src, slice, offsets, mask, $imm8) - }; - } - let r = constify_imm8_gather!(scale, call); + let r = pgatherqq(src, slice, offsets, mask, SCALE as i8); transmute(r) } @@ -1776,20 +1567,19 @@ pub unsafe fn _mm_mask_i64gather_epi64( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_i64gather_epi64) #[inline] #[target_feature(enable = "avx2")] -#[cfg_attr(test, assert_instr(vpgatherqq, scale = 1))] -#[rustc_args_required_const(2)] +#[cfg_attr(test, assert_instr(vpgatherqq, SCALE = 1))] +#[rustc_legacy_const_generics(2)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm256_i64gather_epi64(slice: *const i64, offsets: __m256i, scale: i32) -> __m256i { +pub unsafe fn _mm256_i64gather_epi64( + slice: *const i64, + offsets: __m256i, +) -> __m256i { + static_assert_imm8_scale!(SCALE); let zero = _mm256_setzero_si256().as_i64x4(); let neg_one = _mm256_set1_epi64x(-1).as_i64x4(); let slice = slice as *const i8; let offsets = offsets.as_i64x4(); - macro_rules! call { - ($imm8:expr) => { - vpgatherqq(zero, slice, offsets, neg_one, $imm8) - }; - } - let r = constify_imm8_gather!(scale, call); + let r = vpgatherqq(zero, slice, offsets, neg_one, SCALE as i8); transmute(r) } @@ -1801,26 +1591,21 @@ pub unsafe fn _mm256_i64gather_epi64(slice: *const i64, offsets: __m256i, scale: /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_i64gather_epi64) #[inline] #[target_feature(enable = "avx2")] -#[cfg_attr(test, assert_instr(vpgatherqq, scale = 1))] -#[rustc_args_required_const(4)] +#[cfg_attr(test, assert_instr(vpgatherqq, SCALE = 1))] +#[rustc_legacy_const_generics(4)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm256_mask_i64gather_epi64( +pub unsafe fn _mm256_mask_i64gather_epi64( src: __m256i, slice: *const i64, offsets: __m256i, mask: __m256i, - scale: i32, ) -> __m256i { + static_assert_imm8_scale!(SCALE); let src = src.as_i64x4(); let mask = mask.as_i64x4(); let offsets = offsets.as_i64x4(); let slice = slice as *const i8; - macro_rules! call { - ($imm8:expr) => { - vpgatherqq(src, slice, offsets, mask, $imm8) - }; - } - let r = constify_imm8_gather!(scale, call); + let r = vpgatherqq(src, slice, offsets, mask, SCALE as i8); transmute(r) } @@ -1831,20 +1616,16 @@ pub unsafe fn _mm256_mask_i64gather_epi64( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_i64gather_pd) #[inline] #[target_feature(enable = "avx2")] -#[cfg_attr(test, assert_instr(vgatherqpd, scale = 1))] -#[rustc_args_required_const(2)] +#[cfg_attr(test, assert_instr(vgatherqpd, SCALE = 1))] +#[rustc_legacy_const_generics(2)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm_i64gather_pd(slice: *const f64, offsets: __m128i, scale: i32) -> __m128d { +pub unsafe fn _mm_i64gather_pd(slice: *const f64, offsets: __m128i) -> __m128d { + static_assert_imm8_scale!(SCALE); let zero = _mm_setzero_pd(); let neg_one = _mm_set1_pd(-1.0); let slice = slice as *const i8; let offsets = offsets.as_i64x2(); - macro_rules! call { - ($imm8:expr) => { - pgatherqpd(zero, slice, offsets, neg_one, $imm8) - }; - } - constify_imm8_gather!(scale, call) + pgatherqpd(zero, slice, offsets, neg_one, SCALE as i8) } /// Returns values from `slice` at offsets determined by `offsets * scale`, @@ -1855,24 +1636,19 @@ pub unsafe fn _mm_i64gather_pd(slice: *const f64, offsets: __m128i, scale: i32) /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_i64gather_pd) #[inline] #[target_feature(enable = "avx2")] -#[cfg_attr(test, assert_instr(vgatherqpd, scale = 1))] -#[rustc_args_required_const(4)] +#[cfg_attr(test, assert_instr(vgatherqpd, SCALE = 1))] +#[rustc_legacy_const_generics(4)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm_mask_i64gather_pd( +pub unsafe fn _mm_mask_i64gather_pd( src: __m128d, slice: *const f64, offsets: __m128i, mask: __m128d, - scale: i32, ) -> __m128d { + static_assert_imm8_scale!(SCALE); let slice = slice as *const i8; let offsets = offsets.as_i64x2(); - macro_rules! call { - ($imm8:expr) => { - pgatherqpd(src, slice, offsets, mask, $imm8) - }; - } - constify_imm8_gather!(scale, call) + pgatherqpd(src, slice, offsets, mask, SCALE as i8) } /// Returns values from `slice` at offsets determined by `offsets * scale`, @@ -1882,20 +1658,19 @@ pub unsafe fn _mm_mask_i64gather_pd( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_i64gather_pd) #[inline] #[target_feature(enable = "avx2")] -#[cfg_attr(test, assert_instr(vgatherqpd, scale = 1))] -#[rustc_args_required_const(2)] +#[cfg_attr(test, assert_instr(vgatherqpd, SCALE = 1))] +#[rustc_legacy_const_generics(2)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm256_i64gather_pd(slice: *const f64, offsets: __m256i, scale: i32) -> __m256d { +pub unsafe fn _mm256_i64gather_pd( + slice: *const f64, + offsets: __m256i, +) -> __m256d { + static_assert_imm8_scale!(SCALE); let zero = _mm256_setzero_pd(); let neg_one = _mm256_set1_pd(-1.0); let slice = slice as *const i8; let offsets = offsets.as_i64x4(); - macro_rules! call { - ($imm8:expr) => { - vpgatherqpd(zero, slice, offsets, neg_one, $imm8) - }; - } - constify_imm8_gather!(scale, call) + vpgatherqpd(zero, slice, offsets, neg_one, SCALE as i8) } /// Returns values from `slice` at offsets determined by `offsets * scale`, @@ -1906,45 +1681,39 @@ pub unsafe fn _mm256_i64gather_pd(slice: *const f64, offsets: __m256i, scale: i3 /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_i64gather_pd) #[inline] #[target_feature(enable = "avx2")] -#[cfg_attr(test, assert_instr(vgatherqpd, scale = 1))] -#[rustc_args_required_const(4)] +#[cfg_attr(test, assert_instr(vgatherqpd, SCALE = 1))] +#[rustc_legacy_const_generics(4)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm256_mask_i64gather_pd( +pub unsafe fn _mm256_mask_i64gather_pd( src: __m256d, slice: *const f64, offsets: __m256i, mask: __m256d, - scale: i32, ) -> __m256d { + static_assert_imm8_scale!(SCALE); let slice = slice as *const i8; let offsets = offsets.as_i64x4(); - macro_rules! call { - ($imm8:expr) => { - vpgatherqpd(src, slice, offsets, mask, $imm8) - }; - } - constify_imm8_gather!(scale, call) + vpgatherqpd(src, slice, offsets, mask, SCALE as i8) } /// Copies `a` to `dst`, then insert 128 bits (of integer data) from `b` at the -/// location specified by `imm8`. +/// location specified by `IMM1`. /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_inserti128_si256) #[inline] #[target_feature(enable = "avx2")] #[cfg_attr( all(test, not(target_os = "windows")), - assert_instr(vinsertf128, imm8 = 1) + assert_instr(vinsertf128, IMM1 = 1) )] -#[rustc_args_required_const(2)] +#[rustc_legacy_const_generics(2)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm256_inserti128_si256(a: __m256i, b: __m128i, imm8: i32) -> __m256i { +pub unsafe fn _mm256_inserti128_si256(a: __m256i, b: __m128i) -> __m256i { + static_assert_imm1!(IMM1); let a = a.as_i64x4(); let b = _mm256_castsi128_si256(b).as_i64x4(); - let dst: i64x4 = match imm8 & 0b01 { - 0 => simd_shuffle4(a, b, [4, 5, 2, 3]), - _ => simd_shuffle4(a, b, [0, 1, 4, 5]), - }; + let dst: i64x4 = + simd_shuffle4!(a, b, [[4, 5, 2, 3], [0, 1, 4, 5]][IMM1 as usize]); transmute(dst) } @@ -2246,19 +2015,12 @@ pub unsafe fn _mm256_movemask_epi8(a: __m256i) -> i32 { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mpsadbw_epu8) #[inline] #[target_feature(enable = "avx2")] -#[cfg_attr(test, assert_instr(vmpsadbw, imm8 = 0))] -#[rustc_args_required_const(2)] +#[cfg_attr(test, assert_instr(vmpsadbw, IMM8 = 0))] +#[rustc_legacy_const_generics(2)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm256_mpsadbw_epu8(a: __m256i, b: __m256i, imm8: i32) -> __m256i { - let a = a.as_u8x32(); - let b = b.as_u8x32(); - macro_rules! call { - ($imm8:expr) => { - mpsadbw(a, b, $imm8) - }; - } - let r = constify_imm8!(imm8, call); - transmute(r) +pub unsafe fn _mm256_mpsadbw_epu8(a: __m256i, b: __m256i) -> __m256i { + static_assert_imm8!(IMM8); + transmute(mpsadbw(a.as_u8x32(), b.as_u8x32(), IMM8)) } /// Multiplies the low 32-bit integers from each packed 64-bit element in @@ -2434,54 +2196,22 @@ pub unsafe fn _mm256_permutevar8x32_epi32(a: __m256i, b: __m256i) -> __m256i { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_permute4x64_epi64) #[inline] #[target_feature(enable = "avx2")] -#[cfg_attr(test, assert_instr(vpermpd, imm8 = 9))] -#[rustc_args_required_const(1)] +#[cfg_attr(test, assert_instr(vpermpd, IMM8 = 9))] +#[rustc_legacy_const_generics(1)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm256_permute4x64_epi64(a: __m256i, imm8: i32) -> __m256i { - let imm8 = (imm8 & 0xFF) as u8; +pub unsafe fn _mm256_permute4x64_epi64(a: __m256i) -> __m256i { + static_assert_imm8!(IMM8); let zero = _mm256_setzero_si256().as_i64x4(); - let a = a.as_i64x4(); - macro_rules! permute4 { - ($a:expr, $b:expr, $c:expr, $d:expr) => { - simd_shuffle4(a, zero, [$a, $b, $c, $d]) - }; - } - macro_rules! permute3 { - ($a:expr, $b:expr, $c:expr) => { - match (imm8 >> 6) & 0b11 { - 0b00 => permute4!($a, $b, $c, 0), - 0b01 => permute4!($a, $b, $c, 1), - 0b10 => permute4!($a, $b, $c, 2), - _ => permute4!($a, $b, $c, 3), - } - }; - } - macro_rules! permute2 { - ($a:expr, $b:expr) => { - match (imm8 >> 4) & 0b11 { - 0b00 => permute3!($a, $b, 0), - 0b01 => permute3!($a, $b, 1), - 0b10 => permute3!($a, $b, 2), - _ => permute3!($a, $b, 3), - } - }; - } - macro_rules! permute1 { - ($a:expr) => { - match (imm8 >> 2) & 0b11 { - 0b00 => permute2!($a, 0), - 0b01 => permute2!($a, 1), - 0b10 => permute2!($a, 2), - _ => permute2!($a, 3), - } - }; - } - let r: i64x4 = match imm8 & 0b11 { - 0b00 => permute1!(0), - 0b01 => permute1!(1), - 0b10 => permute1!(2), - _ => permute1!(3), - }; + let r: i64x4 = simd_shuffle4!( + a.as_i64x4(), + zero, + [ + IMM8 as u32 & 0b11, + (IMM8 as u32 >> 2) & 0b11, + (IMM8 as u32 >> 4) & 0b11, + (IMM8 as u32 >> 6) & 0b11, + ], + ); transmute(r) } @@ -2490,18 +2220,12 @@ pub unsafe fn _mm256_permute4x64_epi64(a: __m256i, imm8: i32) -> __m256i { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_permute2x128_si256) #[inline] #[target_feature(enable = "avx2")] -#[cfg_attr(test, assert_instr(vperm2f128, imm8 = 9))] -#[rustc_args_required_const(2)] +#[cfg_attr(test, assert_instr(vperm2f128, IMM8 = 9))] +#[rustc_legacy_const_generics(2)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm256_permute2x128_si256(a: __m256i, b: __m256i, imm8: i32) -> __m256i { - let a = a.as_i64x4(); - let b = b.as_i64x4(); - macro_rules! call { - ($imm8:expr) => { - vperm2i128(a, b, $imm8) - }; - } - transmute(constify_imm8!(imm8, call)) +pub unsafe fn _mm256_permute2x128_si256(a: __m256i, b: __m256i) -> __m256i { + static_assert_imm8!(IMM8); + transmute(vperm2i128(a.as_i64x4(), b.as_i64x4(), IMM8 as i8)) } /// Shuffles 64-bit floating-point elements in `a` across lanes using the @@ -2510,53 +2234,21 @@ pub unsafe fn _mm256_permute2x128_si256(a: __m256i, b: __m256i, imm8: i32) -> __ /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_permute4x64_pd) #[inline] #[target_feature(enable = "avx2")] -#[cfg_attr(test, assert_instr(vpermpd, imm8 = 1))] -#[rustc_args_required_const(1)] -#[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm256_permute4x64_pd(a: __m256d, imm8: i32) -> __m256d { - let imm8 = (imm8 & 0xFF) as u8; - let undef = _mm256_undefined_pd(); - macro_rules! shuffle_done { - ($x01:expr, $x23:expr, $x45:expr, $x67:expr) => { - simd_shuffle4(a, undef, [$x01, $x23, $x45, $x67]) - }; - } - macro_rules! shuffle_x67 { - ($x01:expr, $x23:expr, $x45:expr) => { - match (imm8 >> 6) & 0b11 { - 0b00 => shuffle_done!($x01, $x23, $x45, 0), - 0b01 => shuffle_done!($x01, $x23, $x45, 1), - 0b10 => shuffle_done!($x01, $x23, $x45, 2), - _ => shuffle_done!($x01, $x23, $x45, 3), - } - }; - } - macro_rules! shuffle_x45 { - ($x01:expr, $x23:expr) => { - match (imm8 >> 4) & 0b11 { - 0b00 => shuffle_x67!($x01, $x23, 0), - 0b01 => shuffle_x67!($x01, $x23, 1), - 0b10 => shuffle_x67!($x01, $x23, 2), - _ => shuffle_x67!($x01, $x23, 3), - } - }; - } - macro_rules! shuffle_x23 { - ($x01:expr) => { - match (imm8 >> 2) & 0b11 { - 0b00 => shuffle_x45!($x01, 0), - 0b01 => shuffle_x45!($x01, 1), - 0b10 => shuffle_x45!($x01, 2), - _ => shuffle_x45!($x01, 3), - } - }; - } - match imm8 & 0b11 { - 0b00 => shuffle_x23!(0), - 0b01 => shuffle_x23!(1), - 0b10 => shuffle_x23!(2), - _ => shuffle_x23!(3), - } +#[cfg_attr(test, assert_instr(vpermpd, IMM8 = 1))] +#[rustc_legacy_const_generics(1)] +#[stable(feature = "simd_x86", since = "1.27.0")] +pub unsafe fn _mm256_permute4x64_pd(a: __m256d) -> __m256d { + static_assert_imm8!(IMM8); + simd_shuffle4!( + a, + _mm256_undefined_pd(), + [ + IMM8 as u32 & 0b11, + (IMM8 as u32 >> 2) & 0b11, + (IMM8 as u32 >> 4) & 0b11, + (IMM8 as u32 >> 6) & 0b11, + ], + ) } /// Shuffles eight 32-bit foating-point elements in `a` across lanes using @@ -2655,74 +2347,25 @@ pub unsafe fn _mm256_shuffle_epi8(a: __m256i, b: __m256i) -> __m256i { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_shuffle_epi32) #[inline] #[target_feature(enable = "avx2")] -#[cfg_attr(test, assert_instr(vpermilps, imm8 = 9))] -#[rustc_args_required_const(1)] -#[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm256_shuffle_epi32(a: __m256i, imm8: i32) -> __m256i { - // simd_shuffleX requires that its selector parameter be made up of - // constant values, but we can't enforce that here. In spirit, we need - // to write a `match` on all possible values of a byte, and for each value, - // hard-code the correct `simd_shuffleX` call using only constants. We - // then hope for LLVM to do the rest. - // - // Of course, that's... awful. So we try to use macros to do it for us. - let imm8 = (imm8 & 0xFF) as u8; - - let a = a.as_i32x8(); - macro_rules! shuffle_done { - ($x01:expr, $x23:expr, $x45:expr, $x67:expr) => { - simd_shuffle8( - a, - a, - [ - $x01, - $x23, - $x45, - $x67, - 4 + $x01, - 4 + $x23, - 4 + $x45, - 4 + $x67, - ], - ) - }; - } - macro_rules! shuffle_x67 { - ($x01:expr, $x23:expr, $x45:expr) => { - match (imm8 >> 6) & 0b11 { - 0b00 => shuffle_done!($x01, $x23, $x45, 0), - 0b01 => shuffle_done!($x01, $x23, $x45, 1), - 0b10 => shuffle_done!($x01, $x23, $x45, 2), - _ => shuffle_done!($x01, $x23, $x45, 3), - } - }; - } - macro_rules! shuffle_x45 { - ($x01:expr, $x23:expr) => { - match (imm8 >> 4) & 0b11 { - 0b00 => shuffle_x67!($x01, $x23, 0), - 0b01 => shuffle_x67!($x01, $x23, 1), - 0b10 => shuffle_x67!($x01, $x23, 2), - _ => shuffle_x67!($x01, $x23, 3), - } - }; - } - macro_rules! shuffle_x23 { - ($x01:expr) => { - match (imm8 >> 2) & 0b11 { - 0b00 => shuffle_x45!($x01, 0), - 0b01 => shuffle_x45!($x01, 1), - 0b10 => shuffle_x45!($x01, 2), - _ => shuffle_x45!($x01, 3), - } - }; - } - let r: i32x8 = match imm8 & 0b11 { - 0b00 => shuffle_x23!(0), - 0b01 => shuffle_x23!(1), - 0b10 => shuffle_x23!(2), - _ => shuffle_x23!(3), - }; +#[cfg_attr(test, assert_instr(vpermilps, MASK = 9))] +#[rustc_legacy_const_generics(1)] +#[stable(feature = "simd_x86", since = "1.27.0")] +pub unsafe fn _mm256_shuffle_epi32(a: __m256i) -> __m256i { + static_assert_imm8!(MASK); + let r: i32x8 = simd_shuffle8!( + a.as_i32x8(), + a.as_i32x8(), + [ + MASK as u32 & 0b11, + (MASK as u32 >> 2) & 0b11, + (MASK as u32 >> 4) & 0b11, + (MASK as u32 >> 6) & 0b11, + (MASK as u32 & 0b11) + 4, + ((MASK as u32 >> 2) & 0b11) + 4, + ((MASK as u32 >> 4) & 0b11) + 4, + ((MASK as u32 >> 6) & 0b11) + 4, + ], + ); transmute(r) } @@ -2733,57 +2376,34 @@ pub unsafe fn _mm256_shuffle_epi32(a: __m256i, imm8: i32) -> __m256i { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_shufflehi_epi16) #[inline] #[target_feature(enable = "avx2")] -#[cfg_attr(test, assert_instr(vpshufhw, imm8 = 9))] -#[rustc_args_required_const(1)] +#[cfg_attr(test, assert_instr(vpshufhw, IMM8 = 9))] +#[rustc_legacy_const_generics(1)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm256_shufflehi_epi16(a: __m256i, imm8: i32) -> __m256i { - let imm8 = (imm8 & 0xFF) as u8; +pub unsafe fn _mm256_shufflehi_epi16(a: __m256i) -> __m256i { + static_assert_imm8!(IMM8); let a = a.as_i16x16(); - macro_rules! shuffle_done { - ($x01:expr, $x23:expr, $x45:expr, $x67:expr) => { - #[rustfmt::skip] - simd_shuffle16(a, a, [ - 0, 1, 2, 3, 4+$x01, 4+$x23, 4+$x45, 4+$x67, - 8, 9, 10, 11, 12+$x01, 12+$x23, 12+$x45, 12+$x67 - ]) - }; - } - macro_rules! shuffle_x67 { - ($x01:expr, $x23:expr, $x45:expr) => { - match (imm8 >> 6) & 0b11 { - 0b00 => shuffle_done!($x01, $x23, $x45, 0), - 0b01 => shuffle_done!($x01, $x23, $x45, 1), - 0b10 => shuffle_done!($x01, $x23, $x45, 2), - _ => shuffle_done!($x01, $x23, $x45, 3), - } - }; - } - macro_rules! shuffle_x45 { - ($x01:expr, $x23:expr) => { - match (imm8 >> 4) & 0b11 { - 0b00 => shuffle_x67!($x01, $x23, 0), - 0b01 => shuffle_x67!($x01, $x23, 1), - 0b10 => shuffle_x67!($x01, $x23, 2), - _ => shuffle_x67!($x01, $x23, 3), - } - }; - } - macro_rules! shuffle_x23 { - ($x01:expr) => { - match (imm8 >> 2) & 0b11 { - 0b00 => shuffle_x45!($x01, 0), - 0b01 => shuffle_x45!($x01, 1), - 0b10 => shuffle_x45!($x01, 2), - _ => shuffle_x45!($x01, 3), - } - }; - } - let r: i16x16 = match imm8 & 0b11 { - 0b00 => shuffle_x23!(0), - 0b01 => shuffle_x23!(1), - 0b10 => shuffle_x23!(2), - _ => shuffle_x23!(3), - }; + let r: i16x16 = simd_shuffle16!( + a, + a, + [ + 0, + 1, + 2, + 3, + 4 + (IMM8 as u32 & 0b11), + 4 + ((IMM8 as u32 >> 2) & 0b11), + 4 + ((IMM8 as u32 >> 4) & 0b11), + 4 + ((IMM8 as u32 >> 6) & 0b11), + 8, + 9, + 10, + 11, + 12 + (IMM8 as u32 & 0b11), + 12 + ((IMM8 as u32 >> 2) & 0b11), + 12 + ((IMM8 as u32 >> 4) & 0b11), + 12 + ((IMM8 as u32 >> 6) & 0b11), + ], + ); transmute(r) } @@ -2794,57 +2414,34 @@ pub unsafe fn _mm256_shufflehi_epi16(a: __m256i, imm8: i32) -> __m256i { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_shufflelo_epi16) #[inline] #[target_feature(enable = "avx2")] -#[cfg_attr(test, assert_instr(vpshuflw, imm8 = 9))] -#[rustc_args_required_const(1)] +#[cfg_attr(test, assert_instr(vpshuflw, IMM8 = 9))] +#[rustc_legacy_const_generics(1)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm256_shufflelo_epi16(a: __m256i, imm8: i32) -> __m256i { - let imm8 = (imm8 & 0xFF) as u8; +pub unsafe fn _mm256_shufflelo_epi16(a: __m256i) -> __m256i { + static_assert_imm8!(IMM8); let a = a.as_i16x16(); - macro_rules! shuffle_done { - ($x01: expr, $x23: expr, $x45: expr, $x67: expr) => { - #[rustfmt::skip] - simd_shuffle16(a, a, [ - 0+$x01, 0+$x23, 0+$x45, 0+$x67, 4, 5, 6, 7, - 8+$x01, 8+$x23, 8+$x45, 8+$x67, 12, 13, 14, 15, - ]) - }; - } - macro_rules! shuffle_x67 { - ($x01:expr, $x23:expr, $x45:expr) => { - match (imm8 >> 6) & 0b11 { - 0b00 => shuffle_done!($x01, $x23, $x45, 0), - 0b01 => shuffle_done!($x01, $x23, $x45, 1), - 0b10 => shuffle_done!($x01, $x23, $x45, 2), - _ => shuffle_done!($x01, $x23, $x45, 3), - } - }; - } - macro_rules! shuffle_x45 { - ($x01:expr, $x23:expr) => { - match (imm8 >> 4) & 0b11 { - 0b00 => shuffle_x67!($x01, $x23, 0), - 0b01 => shuffle_x67!($x01, $x23, 1), - 0b10 => shuffle_x67!($x01, $x23, 2), - _ => shuffle_x67!($x01, $x23, 3), - } - }; - } - macro_rules! shuffle_x23 { - ($x01:expr) => { - match (imm8 >> 2) & 0b11 { - 0b00 => shuffle_x45!($x01, 0), - 0b01 => shuffle_x45!($x01, 1), - 0b10 => shuffle_x45!($x01, 2), - _ => shuffle_x45!($x01, 3), - } - }; - } - let r: i16x16 = match imm8 & 0b11 { - 0b00 => shuffle_x23!(0), - 0b01 => shuffle_x23!(1), - 0b10 => shuffle_x23!(2), - _ => shuffle_x23!(3), - }; + let r: i16x16 = simd_shuffle16!( + a, + a, + [ + 0 + (IMM8 as u32 & 0b11), + 0 + ((IMM8 as u32 >> 2) & 0b11), + 0 + ((IMM8 as u32 >> 4) & 0b11), + 0 + ((IMM8 as u32 >> 6) & 0b11), + 4, + 5, + 6, + 7, + 8 + (IMM8 as u32 & 0b11), + 8 + ((IMM8 as u32 >> 2) & 0b11), + 8 + ((IMM8 as u32 >> 4) & 0b11), + 8 + ((IMM8 as u32 >> 6) & 0b11), + 12, + 13, + 14, + 15, + ], + ); transmute(r) } @@ -2923,40 +2520,46 @@ pub unsafe fn _mm256_sll_epi64(a: __m256i, count: __m128i) -> __m256i { transmute(psllq(a.as_i64x4(), count.as_i64x2())) } -/// Shifts packed 16-bit integers in `a` left by `imm8` while +/// Shifts packed 16-bit integers in `a` left by `IMM8` while /// shifting in zeros, return the results; /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_slli_epi16) #[inline] #[target_feature(enable = "avx2")] -#[cfg_attr(test, assert_instr(vpsllw))] +#[cfg_attr(test, assert_instr(vpsllw, IMM8 = 7))] +#[rustc_legacy_const_generics(1)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm256_slli_epi16(a: __m256i, imm8: i32) -> __m256i { - transmute(pslliw(a.as_i16x16(), imm8)) +pub unsafe fn _mm256_slli_epi16(a: __m256i) -> __m256i { + static_assert_imm8!(IMM8); + transmute(pslliw(a.as_i16x16(), IMM8)) } -/// Shifts packed 32-bit integers in `a` left by `imm8` while +/// Shifts packed 32-bit integers in `a` left by `IMM8` while /// shifting in zeros, return the results; /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_slli_epi32) #[inline] #[target_feature(enable = "avx2")] -#[cfg_attr(test, assert_instr(vpslld))] +#[cfg_attr(test, assert_instr(vpslld, IMM8 = 7))] +#[rustc_legacy_const_generics(1)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm256_slli_epi32(a: __m256i, imm8: i32) -> __m256i { - transmute(psllid(a.as_i32x8(), imm8)) +pub unsafe fn _mm256_slli_epi32(a: __m256i) -> __m256i { + static_assert_imm8!(IMM8); + transmute(psllid(a.as_i32x8(), IMM8)) } -/// Shifts packed 64-bit integers in `a` left by `imm8` while +/// Shifts packed 64-bit integers in `a` left by `IMM8` while /// shifting in zeros, return the results; /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_slli_epi64) #[inline] #[target_feature(enable = "avx2")] -#[cfg_attr(test, assert_instr(vpsllq))] +#[cfg_attr(test, assert_instr(vpsllq, IMM8 = 7))] +#[rustc_legacy_const_generics(1)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm256_slli_epi64(a: __m256i, imm8: i32) -> __m256i { - transmute(pslliq(a.as_i64x4(), imm8)) +pub unsafe fn _mm256_slli_epi64(a: __m256i) -> __m256i { + static_assert_imm8!(IMM8); + transmute(pslliq(a.as_i64x4(), IMM8)) } /// Shifts 128-bit lanes in `a` left by `imm8` bytes while shifting in zeros. @@ -2964,17 +2567,12 @@ pub unsafe fn _mm256_slli_epi64(a: __m256i, imm8: i32) -> __m256i { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_slli_si256) #[inline] #[target_feature(enable = "avx2")] -#[cfg_attr(test, assert_instr(vpslldq, imm8 = 3))] -#[rustc_args_required_const(1)] +#[cfg_attr(test, assert_instr(vpslldq, IMM8 = 3))] +#[rustc_legacy_const_generics(1)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm256_slli_si256(a: __m256i, imm8: i32) -> __m256i { - let a = a.as_i64x4(); - macro_rules! call { - ($imm8:expr) => { - vpslldq(a, $imm8) - }; - } - transmute(constify_imm8!(imm8 * 8, call)) +pub unsafe fn _mm256_slli_si256(a: __m256i) -> __m256i { + static_assert_imm8!(IMM8); + _mm256_bslli_epi128::(a) } /// Shifts 128-bit lanes in `a` left by `imm8` bytes while shifting in zeros. @@ -2982,17 +2580,60 @@ pub unsafe fn _mm256_slli_si256(a: __m256i, imm8: i32) -> __m256i { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_bslli_epi128) #[inline] #[target_feature(enable = "avx2")] -#[cfg_attr(test, assert_instr(vpslldq, imm8 = 3))] -#[rustc_args_required_const(1)] +#[cfg_attr(test, assert_instr(vpslldq, IMM8 = 3))] +#[rustc_legacy_const_generics(1)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm256_bslli_epi128(a: __m256i, imm8: i32) -> __m256i { - let a = a.as_i64x4(); - macro_rules! call { - ($imm8:expr) => { - vpslldq(a, $imm8) - }; +pub unsafe fn _mm256_bslli_epi128(a: __m256i) -> __m256i { + static_assert_imm8!(IMM8); + const fn mask(shift: i32, i: u32) -> u32 { + let shift = shift as u32 & 0xff; + if shift > 15 || i % 16 < shift { + 0 + } else { + 32 + (i - shift) + } } - transmute(constify_imm8!(imm8 * 8, call)) + let a = a.as_i8x32(); + let zero = _mm256_setzero_si256().as_i8x32(); + let r: i8x32 = simd_shuffle32!( + zero, + a, + [ + mask(IMM8, 0), + mask(IMM8, 1), + mask(IMM8, 2), + mask(IMM8, 3), + mask(IMM8, 4), + mask(IMM8, 5), + mask(IMM8, 6), + mask(IMM8, 7), + mask(IMM8, 8), + mask(IMM8, 9), + mask(IMM8, 10), + mask(IMM8, 11), + mask(IMM8, 12), + mask(IMM8, 13), + mask(IMM8, 14), + mask(IMM8, 15), + mask(IMM8, 16), + mask(IMM8, 17), + mask(IMM8, 18), + mask(IMM8, 19), + mask(IMM8, 20), + mask(IMM8, 21), + mask(IMM8, 22), + mask(IMM8, 23), + mask(IMM8, 24), + mask(IMM8, 25), + mask(IMM8, 26), + mask(IMM8, 27), + mask(IMM8, 28), + mask(IMM8, 29), + mask(IMM8, 30), + mask(IMM8, 31), + ], + ); + transmute(r) } /// Shifts packed 32-bit integers in `a` left by the amount @@ -3071,28 +2712,32 @@ pub unsafe fn _mm256_sra_epi32(a: __m256i, count: __m128i) -> __m256i { transmute(psrad(a.as_i32x8(), count.as_i32x4())) } -/// Shifts packed 16-bit integers in `a` right by `imm8` while +/// Shifts packed 16-bit integers in `a` right by `IMM8` while /// shifting in sign bits. /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_srai_epi16) #[inline] #[target_feature(enable = "avx2")] -#[cfg_attr(test, assert_instr(vpsraw))] +#[cfg_attr(test, assert_instr(vpsraw, IMM8 = 7))] +#[rustc_legacy_const_generics(1)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm256_srai_epi16(a: __m256i, imm8: i32) -> __m256i { - transmute(psraiw(a.as_i16x16(), imm8)) +pub unsafe fn _mm256_srai_epi16(a: __m256i) -> __m256i { + static_assert_imm8!(IMM8); + transmute(psraiw(a.as_i16x16(), IMM8)) } -/// Shifts packed 32-bit integers in `a` right by `imm8` while +/// Shifts packed 32-bit integers in `a` right by `IMM8` while /// shifting in sign bits. /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_srai_epi32) #[inline] #[target_feature(enable = "avx2")] -#[cfg_attr(test, assert_instr(vpsrad))] +#[cfg_attr(test, assert_instr(vpsrad, IMM8 = 7))] +#[rustc_legacy_const_generics(1)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm256_srai_epi32(a: __m256i, imm8: i32) -> __m256i { - transmute(psraid(a.as_i32x8(), imm8)) +pub unsafe fn _mm256_srai_epi32(a: __m256i) -> __m256i { + static_assert_imm8!(IMM8); + transmute(psraid(a.as_i32x8(), IMM8)) } /// Shifts packed 32-bit integers in `a` right by the amount specified by the @@ -3124,17 +2769,12 @@ pub unsafe fn _mm256_srav_epi32(a: __m256i, count: __m256i) -> __m256i { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_srli_si256) #[inline] #[target_feature(enable = "avx2")] -#[cfg_attr(test, assert_instr(vpsrldq, imm8 = 3))] -#[rustc_args_required_const(1)] +#[cfg_attr(test, assert_instr(vpsrldq, IMM8 = 1))] +#[rustc_legacy_const_generics(1)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm256_srli_si256(a: __m256i, imm8: i32) -> __m256i { - let a = a.as_i64x4(); - macro_rules! call { - ($imm8:expr) => { - vpsrldq(a, $imm8) - }; - } - transmute(constify_imm8!(imm8 * 8, call)) +pub unsafe fn _mm256_srli_si256(a: __m256i) -> __m256i { + static_assert_imm8!(IMM8); + _mm256_bsrli_epi128::(a) } /// Shifts 128-bit lanes in `a` right by `imm8` bytes while shifting in zeros. @@ -3142,17 +2782,145 @@ pub unsafe fn _mm256_srli_si256(a: __m256i, imm8: i32) -> __m256i { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_bsrli_epi128) #[inline] #[target_feature(enable = "avx2")] -#[cfg_attr(test, assert_instr(vpsrldq, imm8 = 3))] -#[rustc_args_required_const(1)] +#[cfg_attr(test, assert_instr(vpsrldq, IMM8 = 1))] +#[rustc_legacy_const_generics(1)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm256_bsrli_epi128(a: __m256i, imm8: i32) -> __m256i { - let a = a.as_i64x4(); - macro_rules! call { - ($imm8:expr) => { - vpsrldq(a, $imm8) - }; - } - transmute(constify_imm8!(imm8 * 8, call)) +pub unsafe fn _mm256_bsrli_epi128(a: __m256i) -> __m256i { + static_assert_imm8!(IMM8); + let a = a.as_i8x32(); + let zero = _mm256_setzero_si256().as_i8x32(); + let r: i8x32 = match IMM8 % 16 { + 0 => simd_shuffle32!( + a, + zero, + [ + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, + 23, 24, 25, 26, 27, 28, 29, 30, 31, + ], + ), + 1 => simd_shuffle32!( + a, + zero, + [ + 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 32, 17, 18, 19, 20, 21, 22, 23, + 24, 25, 26, 27, 28, 29, 30, 31, 32, + ], + ), + 2 => simd_shuffle32!( + a, + zero, + [ + 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 32, 32, 18, 19, 20, 21, 22, 23, 24, + 25, 26, 27, 28, 29, 30, 31, 32, 32, + ], + ), + 3 => simd_shuffle32!( + a, + zero, + [ + 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 32, 32, 32, 19, 20, 21, 22, 23, 24, + 25, 26, 27, 28, 29, 30, 31, 32, 32, 32, + ], + ), + 4 => simd_shuffle32!( + a, + zero, + [ + 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 32, 32, 32, 32, 20, 21, 22, 23, 24, 25, + 26, 27, 28, 29, 30, 31, 32, 32, 32, 32, + ], + ), + 5 => simd_shuffle32!( + a, + zero, + [ + 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 32, 32, 32, 32, 32, 21, 22, 23, 24, 25, 26, + 27, 28, 29, 30, 31, 32, 32, 32, 32, 32, + ], + ), + 6 => simd_shuffle32!( + a, + zero, + [ + 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 32, 32, 32, 32, 32, 32, 22, 23, 24, 25, 26, 27, + 28, 29, 30, 31, 32, 32, 32, 32, 32, 32, + ], + ), + 7 => simd_shuffle32!( + a, + zero, + [ + 7, 8, 9, 10, 11, 12, 13, 14, 15, 32, 32, 32, 32, 32, 32, 32, 23, 24, 25, 26, 27, + 28, 29, 30, 31, 32, 32, 32, 32, 32, 32, 32, + ], + ), + 8 => simd_shuffle32!( + a, + zero, + [ + 8, 9, 10, 11, 12, 13, 14, 15, 32, 32, 32, 32, 32, 32, 32, 32, 24, 25, 26, 27, 28, + 29, 30, 31, 32, 32, 32, 32, 32, 32, 32, 32, + ], + ), + 9 => simd_shuffle32!( + a, + zero, + [ + 9, 10, 11, 12, 13, 14, 15, 32, 32, 32, 32, 32, 32, 32, 32, 32, 25, 26, 27, 28, 29, + 30, 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, + ], + ), + 10 => simd_shuffle32!( + a, + zero, + [ + 10, 11, 12, 13, 14, 15, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 26, 27, 28, 29, 30, + 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, + ], + ), + 11 => simd_shuffle32!( + a, + zero, + [ + 11, 12, 13, 14, 15, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 27, 28, 29, 30, 31, + 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, + ], + ), + 12 => simd_shuffle32!( + a, + zero, + [ + 12, 13, 14, 15, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 28, 29, 30, 31, 32, + 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, + ], + ), + 13 => simd_shuffle32!( + a, + zero, + [ + 13, 14, 15, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 29, 30, 31, 32, 32, + 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, + ], + ), + 14 => simd_shuffle32!( + a, + zero, + [ + 14, 15, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 30, 31, 32, 32, 32, + 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, + ], + ), + 15 => simd_shuffle32!( + a, + zero, + [ + 14, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 31, 32, 32, 32, 32, + 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, + ], + ), + _ => zero, + }; + transmute(r) } /// Shifts packed 16-bit integers in `a` right by `count` while shifting in @@ -3191,40 +2959,46 @@ pub unsafe fn _mm256_srl_epi64(a: __m256i, count: __m128i) -> __m256i { transmute(psrlq(a.as_i64x4(), count.as_i64x2())) } -/// Shifts packed 16-bit integers in `a` right by `imm8` while shifting in +/// Shifts packed 16-bit integers in `a` right by `IMM8` while shifting in /// zeros /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_srli_epi16) #[inline] #[target_feature(enable = "avx2")] -#[cfg_attr(test, assert_instr(vpsrlw))] +#[cfg_attr(test, assert_instr(vpsrlw, IMM8 = 7))] +#[rustc_legacy_const_generics(1)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm256_srli_epi16(a: __m256i, imm8: i32) -> __m256i { - transmute(psrliw(a.as_i16x16(), imm8)) +pub unsafe fn _mm256_srli_epi16(a: __m256i) -> __m256i { + static_assert_imm8!(IMM8); + transmute(psrliw(a.as_i16x16(), IMM8)) } -/// Shifts packed 32-bit integers in `a` right by `imm8` while shifting in +/// Shifts packed 32-bit integers in `a` right by `IMM8` while shifting in /// zeros /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_srli_epi32) #[inline] #[target_feature(enable = "avx2")] -#[cfg_attr(test, assert_instr(vpsrld))] +#[cfg_attr(test, assert_instr(vpsrld, IMM8 = 7))] +#[rustc_legacy_const_generics(1)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm256_srli_epi32(a: __m256i, imm8: i32) -> __m256i { - transmute(psrlid(a.as_i32x8(), imm8)) +pub unsafe fn _mm256_srli_epi32(a: __m256i) -> __m256i { + static_assert_imm8!(IMM8); + transmute(psrlid(a.as_i32x8(), IMM8)) } -/// Shifts packed 64-bit integers in `a` right by `imm8` while shifting in +/// Shifts packed 64-bit integers in `a` right by `IMM8` while shifting in /// zeros /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_srli_epi64) #[inline] #[target_feature(enable = "avx2")] -#[cfg_attr(test, assert_instr(vpsrlq))] +#[cfg_attr(test, assert_instr(vpsrlq, IMM8 = 7))] +#[rustc_legacy_const_generics(1)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm256_srli_epi64(a: __m256i, imm8: i32) -> __m256i { - transmute(psrliq(a.as_i64x4(), imm8)) +pub unsafe fn _mm256_srli_epi64(a: __m256i) -> __m256i { + static_assert_imm8!(IMM8); + transmute(psrliq(a.as_i64x4(), IMM8)) } /// Shifts packed 32-bit integers in `a` right by the amount specified by @@ -3414,7 +3188,7 @@ pub unsafe fn _mm256_subs_epu8(a: __m256i, b: __m256i) -> __m256i { #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm256_unpackhi_epi8(a: __m256i, b: __m256i) -> __m256i { #[rustfmt::skip] - let r: i8x32 = simd_shuffle32(a.as_i8x32(), b.as_i8x32(), [ + let r: i8x32 = simd_shuffle32!(a.as_i8x32(), b.as_i8x32(), [ 8, 40, 9, 41, 10, 42, 11, 43, 12, 44, 13, 45, 14, 46, 15, 47, 24, 56, 25, 57, 26, 58, 27, 59, @@ -3467,7 +3241,7 @@ pub unsafe fn _mm256_unpackhi_epi8(a: __m256i, b: __m256i) -> __m256i { #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm256_unpacklo_epi8(a: __m256i, b: __m256i) -> __m256i { #[rustfmt::skip] - let r: i8x32 = simd_shuffle32(a.as_i8x32(), b.as_i8x32(), [ + let r: i8x32 = simd_shuffle32!(a.as_i8x32(), b.as_i8x32(), [ 0, 32, 1, 33, 2, 34, 3, 35, 4, 36, 5, 37, 6, 38, 7, 39, 16, 48, 17, 49, 18, 50, 19, 51, @@ -3515,7 +3289,7 @@ pub unsafe fn _mm256_unpacklo_epi8(a: __m256i, b: __m256i) -> __m256i { #[cfg_attr(test, assert_instr(vpunpckhwd))] #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm256_unpackhi_epi16(a: __m256i, b: __m256i) -> __m256i { - let r: i16x16 = simd_shuffle16( + let r: i16x16 = simd_shuffle16!( a.as_i16x16(), b.as_i16x16(), [4, 20, 5, 21, 6, 22, 7, 23, 12, 28, 13, 29, 14, 30, 15, 31], @@ -3563,7 +3337,7 @@ pub unsafe fn _mm256_unpackhi_epi16(a: __m256i, b: __m256i) -> __m256i { #[cfg_attr(test, assert_instr(vpunpcklwd))] #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm256_unpacklo_epi16(a: __m256i, b: __m256i) -> __m256i { - let r: i16x16 = simd_shuffle16( + let r: i16x16 = simd_shuffle16!( a.as_i16x16(), b.as_i16x16(), [0, 16, 1, 17, 2, 18, 3, 19, 8, 24, 9, 25, 10, 26, 11, 27], @@ -3604,7 +3378,7 @@ pub unsafe fn _mm256_unpacklo_epi16(a: __m256i, b: __m256i) -> __m256i { #[cfg_attr(test, assert_instr(vunpckhps))] #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm256_unpackhi_epi32(a: __m256i, b: __m256i) -> __m256i { - let r: i32x8 = simd_shuffle8(a.as_i32x8(), b.as_i32x8(), [2, 10, 3, 11, 6, 14, 7, 15]); + let r: i32x8 = simd_shuffle8!(a.as_i32x8(), b.as_i32x8(), [2, 10, 3, 11, 6, 14, 7, 15]); transmute(r) } @@ -3641,7 +3415,7 @@ pub unsafe fn _mm256_unpackhi_epi32(a: __m256i, b: __m256i) -> __m256i { #[cfg_attr(test, assert_instr(vunpcklps))] #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm256_unpacklo_epi32(a: __m256i, b: __m256i) -> __m256i { - let r: i32x8 = simd_shuffle8(a.as_i32x8(), b.as_i32x8(), [0, 8, 1, 9, 4, 12, 5, 13]); + let r: i32x8 = simd_shuffle8!(a.as_i32x8(), b.as_i32x8(), [0, 8, 1, 9, 4, 12, 5, 13]); transmute(r) } @@ -3678,7 +3452,7 @@ pub unsafe fn _mm256_unpacklo_epi32(a: __m256i, b: __m256i) -> __m256i { #[cfg_attr(test, assert_instr(vunpckhpd))] #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm256_unpackhi_epi64(a: __m256i, b: __m256i) -> __m256i { - let r: i64x4 = simd_shuffle4(a.as_i64x4(), b.as_i64x4(), [1, 5, 3, 7]); + let r: i64x4 = simd_shuffle4!(a.as_i64x4(), b.as_i64x4(), [1, 5, 3, 7]); transmute(r) } @@ -3715,7 +3489,7 @@ pub unsafe fn _mm256_unpackhi_epi64(a: __m256i, b: __m256i) -> __m256i { #[cfg_attr(test, assert_instr(vunpcklpd))] #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm256_unpacklo_epi64(a: __m256i, b: __m256i) -> __m256i { - let r: i64x4 = simd_shuffle4(a.as_i64x4(), b.as_i64x4(), [0, 4, 2, 6]); + let r: i64x4 = simd_shuffle4!(a.as_i64x4(), b.as_i64x4(), [0, 4, 2, 6]); transmute(r) } @@ -3731,7 +3505,7 @@ pub unsafe fn _mm256_xor_si256(a: __m256i, b: __m256i) -> __m256i { transmute(simd_xor(a.as_i64x4(), b.as_i64x4())) } -/// Extracts an 8-bit integer from `a`, selected with `imm8`. Returns a 32-bit +/// Extracts an 8-bit integer from `a`, selected with `INDEX`. Returns a 32-bit /// integer containing the zero-extended integer data. /// /// See [LLVM commit D20468](https://reviews.llvm.org/D20468). @@ -3740,19 +3514,14 @@ pub unsafe fn _mm256_xor_si256(a: __m256i, b: __m256i) -> __m256i { #[inline] #[target_feature(enable = "avx2")] // This intrinsic has no corresponding instruction. -#[rustc_args_required_const(1)] +#[rustc_legacy_const_generics(1)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm256_extract_epi8(a: __m256i, imm8: i32) -> i32 { - let a = a.as_u8x32(); - macro_rules! call { - ($imm5:expr) => { - simd_extract::<_, u8>(a, $imm5) as i32 - }; - } - constify_imm5!(imm8, call) +pub unsafe fn _mm256_extract_epi8(a: __m256i) -> i32 { + static_assert_imm5!(INDEX); + simd_extract::<_, u8>(a.as_u8x32(), INDEX as u32) as i32 } -/// Extracts a 16-bit integer from `a`, selected with `imm8`. Returns a 32-bit +/// Extracts a 16-bit integer from `a`, selected with `INDEX`. Returns a 32-bit /// integer containing the zero-extended integer data. /// /// See [LLVM commit D20468](https://reviews.llvm.org/D20468). @@ -3761,34 +3530,24 @@ pub unsafe fn _mm256_extract_epi8(a: __m256i, imm8: i32) -> i32 { #[inline] #[target_feature(enable = "avx2")] // This intrinsic has no corresponding instruction. -#[rustc_args_required_const(1)] +#[rustc_legacy_const_generics(1)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm256_extract_epi16(a: __m256i, imm8: i32) -> i32 { - let a = a.as_u16x16(); - macro_rules! call { - ($imm4:expr) => { - simd_extract::<_, u16>(a, $imm4) as i32 - }; - } - constify_imm4!((imm8 & 15), call) +pub unsafe fn _mm256_extract_epi16(a: __m256i) -> i32 { + static_assert_imm4!(INDEX); + simd_extract::<_, u16>(a.as_u16x16(), INDEX as u32) as i32 } -/// Extracts a 32-bit integer from `a`, selected with `imm8`. +/// Extracts a 32-bit integer from `a`, selected with `INDEX`. /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_extract_epi32) #[inline] #[target_feature(enable = "avx2")] // This intrinsic has no corresponding instruction. -#[rustc_args_required_const(1)] +#[rustc_legacy_const_generics(1)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm256_extract_epi32(a: __m256i, imm8: i32) -> i32 { - let a = a.as_i32x8(); - macro_rules! call { - ($imm3:expr) => { - simd_extract(a, $imm3) - }; - } - constify_imm3!((imm8 & 7), call) +pub unsafe fn _mm256_extract_epi32(a: __m256i) -> i32 { + static_assert_imm3!(INDEX); + simd_extract(a.as_i32x8(), INDEX as u32) } /// Returns the first element of the input vector of `[4 x double]`. @@ -4354,10 +4113,10 @@ mod tests { unsafe fn test_mm_blend_epi32() { let (a, b) = (_mm_set1_epi32(3), _mm_set1_epi32(9)); let e = _mm_setr_epi32(9, 3, 3, 3); - let r = _mm_blend_epi32(a, b, 0x01 as i32); + let r = _mm_blend_epi32::<0x01>(a, b); assert_eq_m128i(r, e); - let r = _mm_blend_epi32(b, a, 0x0E as i32); + let r = _mm_blend_epi32::<0x0E>(b, a); assert_eq_m128i(r, e); } @@ -4365,15 +4124,15 @@ mod tests { unsafe fn test_mm256_blend_epi32() { let (a, b) = (_mm256_set1_epi32(3), _mm256_set1_epi32(9)); let e = _mm256_setr_epi32(9, 3, 3, 3, 3, 3, 3, 3); - let r = _mm256_blend_epi32(a, b, 0x01 as i32); + let r = _mm256_blend_epi32::<0x01>(a, b); assert_eq_m256i(r, e); let e = _mm256_setr_epi32(3, 9, 3, 3, 3, 3, 3, 9); - let r = _mm256_blend_epi32(a, b, 0x82 as i32); + let r = _mm256_blend_epi32::<0x82>(a, b); assert_eq_m256i(r, e); let e = _mm256_setr_epi32(3, 3, 9, 9, 9, 9, 9, 3); - let r = _mm256_blend_epi32(a, b, 0x7C as i32); + let r = _mm256_blend_epi32::<0x7C>(a, b); assert_eq_m256i(r, e); } @@ -4381,32 +4140,32 @@ mod tests { unsafe fn test_mm256_blend_epi16() { let (a, b) = (_mm256_set1_epi16(3), _mm256_set1_epi16(9)); let e = _mm256_setr_epi16(9, 3, 3, 3, 3, 3, 3, 3, 9, 3, 3, 3, 3, 3, 3, 3); - let r = _mm256_blend_epi16(a, b, 0x01 as i32); + let r = _mm256_blend_epi16::<0x01>(a, b); assert_eq_m256i(r, e); - let r = _mm256_blend_epi16(b, a, 0xFE as i32); + let r = _mm256_blend_epi16::<0xFE>(b, a); assert_eq_m256i(r, e); } #[simd_test(enable = "avx2")] unsafe fn test_mm256_blendv_epi8() { let (a, b) = (_mm256_set1_epi8(4), _mm256_set1_epi8(2)); - let mask = _mm256_insert_epi8(_mm256_set1_epi8(0), -1, 2); - let e = _mm256_insert_epi8(_mm256_set1_epi8(4), 2, 2); + let mask = _mm256_insert_epi8::<2>(_mm256_set1_epi8(0), -1); + let e = _mm256_insert_epi8::<2>(_mm256_set1_epi8(4), 2); let r = _mm256_blendv_epi8(a, b, mask); assert_eq_m256i(r, e); } #[simd_test(enable = "avx2")] unsafe fn test_mm_broadcastb_epi8() { - let a = _mm_insert_epi8(_mm_set1_epi8(0x00), 0x2a, 0); + let a = _mm_insert_epi8::<0>(_mm_set1_epi8(0x00), 0x2a); let res = _mm_broadcastb_epi8(a); assert_eq_m128i(res, _mm_set1_epi8(0x2a)); } #[simd_test(enable = "avx2")] unsafe fn test_mm256_broadcastb_epi8() { - let a = _mm_insert_epi8(_mm_set1_epi8(0x00), 0x2a, 0); + let a = _mm_insert_epi8::<0>(_mm_set1_epi8(0x00), 0x2a); let res = _mm256_broadcastb_epi8(a); assert_eq_m256i(res, _mm256_set1_epi8(0x2a)); } @@ -4482,14 +4241,14 @@ mod tests { #[simd_test(enable = "avx2")] unsafe fn test_mm_broadcastw_epi16() { - let a = _mm_insert_epi16(_mm_set1_epi16(0x2a), 0x22b, 0); + let a = _mm_insert_epi16::<0>(_mm_set1_epi16(0x2a), 0x22b); let res = _mm_broadcastw_epi16(a); assert_eq_m128i(res, _mm_set1_epi16(0x22b)); } #[simd_test(enable = "avx2")] unsafe fn test_mm256_broadcastw_epi16() { - let a = _mm_insert_epi16(_mm_set1_epi16(0x2a), 0x22b, 0); + let a = _mm_insert_epi16::<0>(_mm_set1_epi16(0x2a), 0x22b); let res = _mm256_broadcastw_epi16(a); assert_eq_m256i(res, _mm256_set1_epi16(0x22b)); } @@ -4511,7 +4270,7 @@ mod tests { 7, 6, 5, 4, 3, 2, 1, 0, ); let r = _mm256_cmpeq_epi8(a, b); - assert_eq_m256i(r, _mm256_insert_epi8(_mm256_set1_epi8(0), !0, 2)); + assert_eq_m256i(r, _mm256_insert_epi8::<2>(_mm256_set1_epi8(0), !0)); } #[simd_test(enable = "avx2")] @@ -4527,7 +4286,7 @@ mod tests { 7, 6, 5, 4, 3, 2, 1, 0, ); let r = _mm256_cmpeq_epi16(a, b); - assert_eq_m256i(r, _mm256_insert_epi16(_mm256_set1_epi16(0), !0, 2)); + assert_eq_m256i(r, _mm256_insert_epi16::<2>(_mm256_set1_epi16(0), !0)); } #[simd_test(enable = "avx2")] @@ -4536,7 +4295,7 @@ mod tests { let b = _mm256_setr_epi32(7, 6, 2, 4, 3, 2, 1, 0); let r = _mm256_cmpeq_epi32(a, b); let e = _mm256_set1_epi32(0); - let e = _mm256_insert_epi32(e, !0, 2); + let e = _mm256_insert_epi32::<2>(e, !0); assert_eq_m256i(r, e); } @@ -4545,39 +4304,39 @@ mod tests { let a = _mm256_setr_epi64x(0, 1, 2, 3); let b = _mm256_setr_epi64x(3, 2, 2, 0); let r = _mm256_cmpeq_epi64(a, b); - assert_eq_m256i(r, _mm256_insert_epi64(_mm256_set1_epi64x(0), !0, 2)); + assert_eq_m256i(r, _mm256_insert_epi64::<2>(_mm256_set1_epi64x(0), !0)); } #[simd_test(enable = "avx2")] unsafe fn test_mm256_cmpgt_epi8() { - let a = _mm256_insert_epi8(_mm256_set1_epi8(0), 5, 0); + let a = _mm256_insert_epi8::<0>(_mm256_set1_epi8(0), 5); let b = _mm256_set1_epi8(0); let r = _mm256_cmpgt_epi8(a, b); - assert_eq_m256i(r, _mm256_insert_epi8(_mm256_set1_epi8(0), !0, 0)); + assert_eq_m256i(r, _mm256_insert_epi8::<0>(_mm256_set1_epi8(0), !0)); } #[simd_test(enable = "avx2")] unsafe fn test_mm256_cmpgt_epi16() { - let a = _mm256_insert_epi16(_mm256_set1_epi16(0), 5, 0); + let a = _mm256_insert_epi16::<0>(_mm256_set1_epi16(0), 5); let b = _mm256_set1_epi16(0); let r = _mm256_cmpgt_epi16(a, b); - assert_eq_m256i(r, _mm256_insert_epi16(_mm256_set1_epi16(0), !0, 0)); + assert_eq_m256i(r, _mm256_insert_epi16::<0>(_mm256_set1_epi16(0), !0)); } #[simd_test(enable = "avx2")] unsafe fn test_mm256_cmpgt_epi32() { - let a = _mm256_insert_epi32(_mm256_set1_epi32(0), 5, 0); + let a = _mm256_insert_epi32::<0>(_mm256_set1_epi32(0), 5); let b = _mm256_set1_epi32(0); let r = _mm256_cmpgt_epi32(a, b); - assert_eq_m256i(r, _mm256_insert_epi32(_mm256_set1_epi32(0), !0, 0)); + assert_eq_m256i(r, _mm256_insert_epi32::<0>(_mm256_set1_epi32(0), !0)); } #[simd_test(enable = "avx2")] unsafe fn test_mm256_cmpgt_epi64() { - let a = _mm256_insert_epi64(_mm256_set1_epi64x(0), 5, 0); + let a = _mm256_insert_epi64::<0>(_mm256_set1_epi64x(0), 5); let b = _mm256_set1_epi64x(0); let r = _mm256_cmpgt_epi64(a, b); - assert_eq_m256i(r, _mm256_insert_epi64(_mm256_set1_epi64x(0), !0, 0)); + assert_eq_m256i(r, _mm256_insert_epi64::<0>(_mm256_set1_epi64x(0), !0)); } #[simd_test(enable = "avx2")] @@ -4699,7 +4458,7 @@ mod tests { #[simd_test(enable = "avx2")] unsafe fn test_mm256_extracti128_si256() { let a = _mm256_setr_epi64x(1, 2, 3, 4); - let r = _mm256_extracti128_si256(a, 0b01); + let r = _mm256_extracti128_si256::<1>(a); let e = _mm_setr_epi64x(3, 4); assert_eq_m128i(r, e); } @@ -4725,8 +4484,8 @@ mod tests { #[simd_test(enable = "avx2")] unsafe fn test_mm256_hadds_epi16() { let a = _mm256_set1_epi16(2); - let a = _mm256_insert_epi16(a, 0x7fff, 0); - let a = _mm256_insert_epi16(a, 1, 1); + let a = _mm256_insert_epi16::<0>(a, 0x7fff); + let a = _mm256_insert_epi16::<1>(a, 1); let b = _mm256_set1_epi16(4); let r = _mm256_hadds_epi16(a, b); #[rustfmt::skip] @@ -4758,11 +4517,11 @@ mod tests { #[simd_test(enable = "avx2")] unsafe fn test_mm256_hsubs_epi16() { let a = _mm256_set1_epi16(2); - let a = _mm256_insert_epi16(a, 0x7fff, 0); - let a = _mm256_insert_epi16(a, -1, 1); + let a = _mm256_insert_epi16::<0>(a, 0x7fff); + let a = _mm256_insert_epi16::<1>(a, -1); let b = _mm256_set1_epi16(4); let r = _mm256_hsubs_epi16(a, b); - let e = _mm256_insert_epi16(_mm256_set1_epi16(0), 0x7FFF, 0); + let e = _mm256_insert_epi16::<0>(_mm256_set1_epi16(0), 0x7FFF); assert_eq_m256i(r, e); } @@ -4779,7 +4538,7 @@ mod tests { unsafe fn test_mm256_inserti128_si256() { let a = _mm256_setr_epi64x(1, 2, 3, 4); let b = _mm_setr_epi64x(7, 8); - let r = _mm256_inserti128_si256(a, b, 0b01); + let r = _mm256_inserti128_si256::<1>(a, b); let e = _mm256_setr_epi64x(1, 2, 7, 8); assert_eq_m256i(r, e); } @@ -4981,7 +4740,7 @@ mod tests { unsafe fn test_mm256_mpsadbw_epu8() { let a = _mm256_set1_epi8(2); let b = _mm256_set1_epi8(4); - let r = _mm256_mpsadbw_epu8(a, b, 0); + let r = _mm256_mpsadbw_epu8::<0>(a, b); let e = _mm256_set1_epi16(8); assert_eq_m256i(r, e); } @@ -5130,7 +4889,7 @@ mod tests { 0, 1, 2, 3, 44, 22, 22, 11, 4, 5, 6, 7, 88, 66, 66, 55, ); - let r = _mm256_shufflehi_epi16(a, 0b00_01_01_11); + let r = _mm256_shufflehi_epi16::<0b00_01_01_11>(a); assert_eq_m256i(r, e); } @@ -5146,7 +4905,7 @@ mod tests { 44, 22, 22, 11, 0, 1, 2, 3, 88, 66, 66, 55, 4, 5, 6, 7, ); - let r = _mm256_shufflelo_epi16(a, 0b00_01_01_11); + let r = _mm256_shufflelo_epi16::<0b00_01_01_11>(a); assert_eq_m256i(r, e); } @@ -5180,7 +4939,7 @@ mod tests { #[simd_test(enable = "avx2")] unsafe fn test_mm256_sll_epi16() { let a = _mm256_set1_epi16(0xFF); - let b = _mm_insert_epi16(_mm_set1_epi16(0), 4, 0); + let b = _mm_insert_epi16::<0>(_mm_set1_epi16(0), 4); let r = _mm256_sll_epi16(a, b); assert_eq_m256i(r, _mm256_set1_epi16(0xFF0)); } @@ -5188,7 +4947,7 @@ mod tests { #[simd_test(enable = "avx2")] unsafe fn test_mm256_sll_epi32() { let a = _mm256_set1_epi32(0xFFFF); - let b = _mm_insert_epi32(_mm_set1_epi32(0), 4, 0); + let b = _mm_insert_epi32::<0>(_mm_set1_epi32(0), 4); let r = _mm256_sll_epi32(a, b); assert_eq_m256i(r, _mm256_set1_epi32(0xFFFF0)); } @@ -5196,7 +4955,7 @@ mod tests { #[simd_test(enable = "avx2")] unsafe fn test_mm256_sll_epi64() { let a = _mm256_set1_epi64x(0xFFFFFFFF); - let b = _mm_insert_epi64(_mm_set1_epi64x(0), 4, 0); + let b = _mm_insert_epi64::<0>(_mm_set1_epi64x(0), 4); let r = _mm256_sll_epi64(a, b); assert_eq_m256i(r, _mm256_set1_epi64x(0xFFFFFFFF0)); } @@ -5204,7 +4963,7 @@ mod tests { #[simd_test(enable = "avx2")] unsafe fn test_mm256_slli_epi16() { assert_eq_m256i( - _mm256_slli_epi16(_mm256_set1_epi16(0xFF), 4), + _mm256_slli_epi16::<4>(_mm256_set1_epi16(0xFF)), _mm256_set1_epi16(0xFF0), ); } @@ -5212,7 +4971,7 @@ mod tests { #[simd_test(enable = "avx2")] unsafe fn test_mm256_slli_epi32() { assert_eq_m256i( - _mm256_slli_epi32(_mm256_set1_epi32(0xFFFF), 4), + _mm256_slli_epi32::<4>(_mm256_set1_epi32(0xFFFF)), _mm256_set1_epi32(0xFFFF0), ); } @@ -5220,7 +4979,7 @@ mod tests { #[simd_test(enable = "avx2")] unsafe fn test_mm256_slli_epi64() { assert_eq_m256i( - _mm256_slli_epi64(_mm256_set1_epi64x(0xFFFFFFFF), 4), + _mm256_slli_epi64::<4>(_mm256_set1_epi64x(0xFFFFFFFF)), _mm256_set1_epi64x(0xFFFFFFFF0), ); } @@ -5228,7 +4987,7 @@ mod tests { #[simd_test(enable = "avx2")] unsafe fn test_mm256_slli_si256() { let a = _mm256_set1_epi64x(0xFFFFFFFF); - let r = _mm256_slli_si256(a, 3); + let r = _mm256_slli_si256::<3>(a); assert_eq_m256i(r, _mm256_set1_epi64x(0xFFFFFFFF000000)); } @@ -5279,7 +5038,7 @@ mod tests { #[simd_test(enable = "avx2")] unsafe fn test_mm256_sra_epi32() { let a = _mm256_set1_epi32(-1); - let b = _mm_insert_epi32(_mm_set1_epi32(0), 1, 0); + let b = _mm_insert_epi32::<0>(_mm_set1_epi32(0), 1); let r = _mm256_sra_epi32(a, b); assert_eq_m256i(r, _mm256_set1_epi32(-1)); } @@ -5287,7 +5046,7 @@ mod tests { #[simd_test(enable = "avx2")] unsafe fn test_mm256_srai_epi16() { assert_eq_m256i( - _mm256_srai_epi16(_mm256_set1_epi16(-1), 1), + _mm256_srai_epi16::<1>(_mm256_set1_epi16(-1)), _mm256_set1_epi16(-1), ); } @@ -5295,7 +5054,7 @@ mod tests { #[simd_test(enable = "avx2")] unsafe fn test_mm256_srai_epi32() { assert_eq_m256i( - _mm256_srai_epi32(_mm256_set1_epi32(-1), 1), + _mm256_srai_epi32::<1>(_mm256_set1_epi32(-1)), _mm256_set1_epi32(-1), ); } @@ -5327,7 +5086,7 @@ mod tests { 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, ); - let r = _mm256_srli_si256(a, 3); + let r = _mm256_srli_si256::<3>(a); #[rustfmt::skip] let e = _mm256_setr_epi8( 4, 5, 6, 7, 8, 9, 10, 11, @@ -5341,7 +5100,7 @@ mod tests { #[simd_test(enable = "avx2")] unsafe fn test_mm256_srl_epi16() { let a = _mm256_set1_epi16(0xFF); - let b = _mm_insert_epi16(_mm_set1_epi16(0), 4, 0); + let b = _mm_insert_epi16::<0>(_mm_set1_epi16(0), 4); let r = _mm256_srl_epi16(a, b); assert_eq_m256i(r, _mm256_set1_epi16(0xF)); } @@ -5349,7 +5108,7 @@ mod tests { #[simd_test(enable = "avx2")] unsafe fn test_mm256_srl_epi32() { let a = _mm256_set1_epi32(0xFFFF); - let b = _mm_insert_epi32(_mm_set1_epi32(0), 4, 0); + let b = _mm_insert_epi32::<0>(_mm_set1_epi32(0), 4); let r = _mm256_srl_epi32(a, b); assert_eq_m256i(r, _mm256_set1_epi32(0xFFF)); } @@ -5365,7 +5124,7 @@ mod tests { #[simd_test(enable = "avx2")] unsafe fn test_mm256_srli_epi16() { assert_eq_m256i( - _mm256_srli_epi16(_mm256_set1_epi16(0xFF), 4), + _mm256_srli_epi16::<4>(_mm256_set1_epi16(0xFF)), _mm256_set1_epi16(0xF), ); } @@ -5373,7 +5132,7 @@ mod tests { #[simd_test(enable = "avx2")] unsafe fn test_mm256_srli_epi32() { assert_eq_m256i( - _mm256_srli_epi32(_mm256_set1_epi32(0xFFFF), 4), + _mm256_srli_epi32::<4>(_mm256_set1_epi32(0xFFFF)), _mm256_set1_epi32(0xFFF), ); } @@ -5381,7 +5140,7 @@ mod tests { #[simd_test(enable = "avx2")] unsafe fn test_mm256_srli_epi64() { assert_eq_m256i( - _mm256_srli_epi64(_mm256_set1_epi64x(0xFFFFFFFF), 4), + _mm256_srli_epi64::<4>(_mm256_set1_epi64x(0xFFFFFFFF)), _mm256_set1_epi64x(0xFFFFFFF), ); } @@ -5510,10 +5269,10 @@ mod tests { -17, -18, -19, -20, -21, -22, -23, -24, -25, -26, -27, -28, -29, -30, -31, -32, ); - let r = _mm256_alignr_epi8(a, b, 33); + let r = _mm256_alignr_epi8::<33>(a, b); assert_eq_m256i(r, _mm256_set1_epi8(0)); - let r = _mm256_alignr_epi8(a, b, 17); + let r = _mm256_alignr_epi8::<17>(a, b); #[rustfmt::skip] let expected = _mm256_setr_epi8( 2, 3, 4, 5, 6, 7, 8, 9, @@ -5523,7 +5282,7 @@ mod tests { ); assert_eq_m256i(r, expected); - let r = _mm256_alignr_epi8(a, b, 4); + let r = _mm256_alignr_epi8::<4>(a, b); #[rustfmt::skip] let expected = _mm256_setr_epi8( -5, -6, -7, -8, -9, -10, -11, -12, @@ -5540,10 +5299,10 @@ mod tests { -18, -19, -20, -21, -22, -23, -24, -25, -26, -27, -28, -29, -30, -31, -32, ); - let r = _mm256_alignr_epi8(a, b, 16); + let r = _mm256_alignr_epi8::<16>(a, b); assert_eq_m256i(r, expected); - let r = _mm256_alignr_epi8(a, b, 15); + let r = _mm256_alignr_epi8::<15>(a, b); #[rustfmt::skip] let expected = _mm256_setr_epi8( -16, 1, 2, 3, 4, 5, 6, 7, @@ -5553,7 +5312,7 @@ mod tests { ); assert_eq_m256i(r, expected); - let r = _mm256_alignr_epi8(a, b, 0); + let r = _mm256_alignr_epi8::<0>(a, b); assert_eq_m256i(r, b); } @@ -5597,7 +5356,7 @@ mod tests { unsafe fn test_mm256_permute4x64_epi64() { let a = _mm256_setr_epi64x(100, 200, 300, 400); let expected = _mm256_setr_epi64x(400, 100, 200, 100); - let r = _mm256_permute4x64_epi64(a, 0b00010011); + let r = _mm256_permute4x64_epi64::<0b00010011>(a); assert_eq_m256i(r, expected); } @@ -5605,7 +5364,7 @@ mod tests { unsafe fn test_mm256_permute2x128_si256() { let a = _mm256_setr_epi64x(100, 200, 500, 600); let b = _mm256_setr_epi64x(300, 400, 700, 800); - let r = _mm256_permute2x128_si256(a, b, 0b00_01_00_11); + let r = _mm256_permute2x128_si256::<0b00_01_00_11>(a, b); let e = _mm256_setr_epi64x(700, 800, 500, 600); assert_eq_m256i(r, e); } @@ -5613,7 +5372,7 @@ mod tests { #[simd_test(enable = "avx2")] unsafe fn test_mm256_permute4x64_pd() { let a = _mm256_setr_pd(1., 2., 3., 4.); - let r = _mm256_permute4x64_pd(a, 0b00_01_00_11); + let r = _mm256_permute4x64_pd::<0b00_01_00_11>(a); let e = _mm256_setr_pd(4., 1., 2., 1.); assert_eq_m256d(r, e); } @@ -5634,7 +5393,7 @@ mod tests { arr[i as usize] = i; } // A multiplier of 4 is word-addressing - let r = _mm_i32gather_epi32(arr.as_ptr(), _mm_setr_epi32(0, 16, 32, 48), 4); + let r = _mm_i32gather_epi32::<4>(arr.as_ptr(), _mm_setr_epi32(0, 16, 32, 48)); assert_eq_m128i(r, _mm_setr_epi32(0, 16, 32, 48)); } @@ -5645,12 +5404,11 @@ mod tests { arr[i as usize] = i; } // A multiplier of 4 is word-addressing - let r = _mm_mask_i32gather_epi32( + let r = _mm_mask_i32gather_epi32::<4>( _mm_set1_epi32(256), arr.as_ptr(), _mm_setr_epi32(0, 16, 64, 96), _mm_setr_epi32(-1, -1, -1, 0), - 4, ); assert_eq_m128i(r, _mm_setr_epi32(0, 16, 64, 256)); } @@ -5662,11 +5420,8 @@ mod tests { arr[i as usize] = i; } // A multiplier of 4 is word-addressing - let r = _mm256_i32gather_epi32( - arr.as_ptr(), - _mm256_setr_epi32(0, 16, 32, 48, 1, 2, 3, 4), - 4, - ); + let r = + _mm256_i32gather_epi32::<4>(arr.as_ptr(), _mm256_setr_epi32(0, 16, 32, 48, 1, 2, 3, 4)); assert_eq_m256i(r, _mm256_setr_epi32(0, 16, 32, 48, 1, 2, 3, 4)); } @@ -5677,12 +5432,11 @@ mod tests { arr[i as usize] = i; } // A multiplier of 4 is word-addressing - let r = _mm256_mask_i32gather_epi32( + let r = _mm256_mask_i32gather_epi32::<4>( _mm256_set1_epi32(256), arr.as_ptr(), _mm256_setr_epi32(0, 16, 64, 96, 0, 0, 0, 0), _mm256_setr_epi32(-1, -1, -1, 0, 0, 0, 0, 0), - 4, ); assert_eq_m256i(r, _mm256_setr_epi32(0, 16, 64, 256, 256, 256, 256, 256)); } @@ -5696,7 +5450,7 @@ mod tests { j += 1.0; } // A multiplier of 4 is word-addressing for f32s - let r = _mm_i32gather_ps(arr.as_ptr(), _mm_setr_epi32(0, 16, 32, 48), 4); + let r = _mm_i32gather_ps::<4>(arr.as_ptr(), _mm_setr_epi32(0, 16, 32, 48)); assert_eq_m128(r, _mm_setr_ps(0.0, 16.0, 32.0, 48.0)); } @@ -5709,12 +5463,11 @@ mod tests { j += 1.0; } // A multiplier of 4 is word-addressing for f32s - let r = _mm_mask_i32gather_ps( + let r = _mm_mask_i32gather_ps::<4>( _mm_set1_ps(256.0), arr.as_ptr(), _mm_setr_epi32(0, 16, 64, 96), _mm_setr_ps(-1.0, -1.0, -1.0, 0.0), - 4, ); assert_eq_m128(r, _mm_setr_ps(0.0, 16.0, 64.0, 256.0)); } @@ -5728,11 +5481,8 @@ mod tests { j += 1.0; } // A multiplier of 4 is word-addressing for f32s - let r = _mm256_i32gather_ps( - arr.as_ptr(), - _mm256_setr_epi32(0, 16, 32, 48, 1, 2, 3, 4), - 4, - ); + let r = + _mm256_i32gather_ps::<4>(arr.as_ptr(), _mm256_setr_epi32(0, 16, 32, 48, 1, 2, 3, 4)); assert_eq_m256(r, _mm256_setr_ps(0.0, 16.0, 32.0, 48.0, 1.0, 2.0, 3.0, 4.0)); } @@ -5745,12 +5495,11 @@ mod tests { j += 1.0; } // A multiplier of 4 is word-addressing for f32s - let r = _mm256_mask_i32gather_ps( + let r = _mm256_mask_i32gather_ps::<4>( _mm256_set1_ps(256.0), arr.as_ptr(), _mm256_setr_epi32(0, 16, 64, 96, 0, 0, 0, 0), _mm256_setr_ps(-1.0, -1.0, -1.0, 0.0, 0.0, 0.0, 0.0, 0.0), - 4, ); assert_eq_m256( r, @@ -5765,7 +5514,7 @@ mod tests { arr[i as usize] = i; } // A multiplier of 8 is word-addressing for i64s - let r = _mm_i32gather_epi64(arr.as_ptr(), _mm_setr_epi32(0, 16, 0, 0), 8); + let r = _mm_i32gather_epi64::<8>(arr.as_ptr(), _mm_setr_epi32(0, 16, 0, 0)); assert_eq_m128i(r, _mm_setr_epi64x(0, 16)); } @@ -5776,12 +5525,11 @@ mod tests { arr[i as usize] = i; } // A multiplier of 8 is word-addressing for i64s - let r = _mm_mask_i32gather_epi64( + let r = _mm_mask_i32gather_epi64::<8>( _mm_set1_epi64x(256), arr.as_ptr(), _mm_setr_epi32(16, 16, 16, 16), _mm_setr_epi64x(-1, 0), - 8, ); assert_eq_m128i(r, _mm_setr_epi64x(16, 256)); } @@ -5793,7 +5541,7 @@ mod tests { arr[i as usize] = i; } // A multiplier of 8 is word-addressing for i64s - let r = _mm256_i32gather_epi64(arr.as_ptr(), _mm_setr_epi32(0, 16, 32, 48), 8); + let r = _mm256_i32gather_epi64::<8>(arr.as_ptr(), _mm_setr_epi32(0, 16, 32, 48)); assert_eq_m256i(r, _mm256_setr_epi64x(0, 16, 32, 48)); } @@ -5804,12 +5552,11 @@ mod tests { arr[i as usize] = i; } // A multiplier of 8 is word-addressing for i64s - let r = _mm256_mask_i32gather_epi64( + let r = _mm256_mask_i32gather_epi64::<8>( _mm256_set1_epi64x(256), arr.as_ptr(), _mm_setr_epi32(0, 16, 64, 96), _mm256_setr_epi64x(-1, -1, -1, 0), - 8, ); assert_eq_m256i(r, _mm256_setr_epi64x(0, 16, 64, 256)); } @@ -5823,7 +5570,7 @@ mod tests { j += 1.0; } // A multiplier of 8 is word-addressing for f64s - let r = _mm_i32gather_pd(arr.as_ptr(), _mm_setr_epi32(0, 16, 0, 0), 8); + let r = _mm_i32gather_pd::<8>(arr.as_ptr(), _mm_setr_epi32(0, 16, 0, 0)); assert_eq_m128d(r, _mm_setr_pd(0.0, 16.0)); } @@ -5836,12 +5583,11 @@ mod tests { j += 1.0; } // A multiplier of 8 is word-addressing for f64s - let r = _mm_mask_i32gather_pd( + let r = _mm_mask_i32gather_pd::<8>( _mm_set1_pd(256.0), arr.as_ptr(), _mm_setr_epi32(16, 16, 16, 16), _mm_setr_pd(-1.0, 0.0), - 8, ); assert_eq_m128d(r, _mm_setr_pd(16.0, 256.0)); } @@ -5855,7 +5601,7 @@ mod tests { j += 1.0; } // A multiplier of 8 is word-addressing for f64s - let r = _mm256_i32gather_pd(arr.as_ptr(), _mm_setr_epi32(0, 16, 32, 48), 8); + let r = _mm256_i32gather_pd::<8>(arr.as_ptr(), _mm_setr_epi32(0, 16, 32, 48)); assert_eq_m256d(r, _mm256_setr_pd(0.0, 16.0, 32.0, 48.0)); } @@ -5868,12 +5614,11 @@ mod tests { j += 1.0; } // A multiplier of 8 is word-addressing for f64s - let r = _mm256_mask_i32gather_pd( + let r = _mm256_mask_i32gather_pd::<8>( _mm256_set1_pd(256.0), arr.as_ptr(), _mm_setr_epi32(0, 16, 64, 96), _mm256_setr_pd(-1.0, -1.0, -1.0, 0.0), - 8, ); assert_eq_m256d(r, _mm256_setr_pd(0.0, 16.0, 64.0, 256.0)); } @@ -5885,7 +5630,7 @@ mod tests { arr[i as usize] = i; } // A multiplier of 4 is word-addressing - let r = _mm_i64gather_epi32(arr.as_ptr(), _mm_setr_epi64x(0, 16), 4); + let r = _mm_i64gather_epi32::<4>(arr.as_ptr(), _mm_setr_epi64x(0, 16)); assert_eq_m128i(r, _mm_setr_epi32(0, 16, 0, 0)); } @@ -5896,12 +5641,11 @@ mod tests { arr[i as usize] = i; } // A multiplier of 4 is word-addressing - let r = _mm_mask_i64gather_epi32( + let r = _mm_mask_i64gather_epi32::<4>( _mm_set1_epi32(256), arr.as_ptr(), _mm_setr_epi64x(0, 16), _mm_setr_epi32(-1, 0, -1, 0), - 4, ); assert_eq_m128i(r, _mm_setr_epi32(0, 256, 0, 0)); } @@ -5913,7 +5657,7 @@ mod tests { arr[i as usize] = i; } // A multiplier of 4 is word-addressing - let r = _mm256_i64gather_epi32(arr.as_ptr(), _mm256_setr_epi64x(0, 16, 32, 48), 4); + let r = _mm256_i64gather_epi32::<4>(arr.as_ptr(), _mm256_setr_epi64x(0, 16, 32, 48)); assert_eq_m128i(r, _mm_setr_epi32(0, 16, 32, 48)); } @@ -5924,12 +5668,11 @@ mod tests { arr[i as usize] = i; } // A multiplier of 4 is word-addressing - let r = _mm256_mask_i64gather_epi32( + let r = _mm256_mask_i64gather_epi32::<4>( _mm_set1_epi32(256), arr.as_ptr(), _mm256_setr_epi64x(0, 16, 64, 96), _mm_setr_epi32(-1, -1, -1, 0), - 4, ); assert_eq_m128i(r, _mm_setr_epi32(0, 16, 64, 256)); } @@ -5943,7 +5686,7 @@ mod tests { j += 1.0; } // A multiplier of 4 is word-addressing for f32s - let r = _mm_i64gather_ps(arr.as_ptr(), _mm_setr_epi64x(0, 16), 4); + let r = _mm_i64gather_ps::<4>(arr.as_ptr(), _mm_setr_epi64x(0, 16)); assert_eq_m128(r, _mm_setr_ps(0.0, 16.0, 0.0, 0.0)); } @@ -5956,12 +5699,11 @@ mod tests { j += 1.0; } // A multiplier of 4 is word-addressing for f32s - let r = _mm_mask_i64gather_ps( + let r = _mm_mask_i64gather_ps::<4>( _mm_set1_ps(256.0), arr.as_ptr(), _mm_setr_epi64x(0, 16), _mm_setr_ps(-1.0, 0.0, -1.0, 0.0), - 4, ); assert_eq_m128(r, _mm_setr_ps(0.0, 256.0, 0.0, 0.0)); } @@ -5975,7 +5717,7 @@ mod tests { j += 1.0; } // A multiplier of 4 is word-addressing for f32s - let r = _mm256_i64gather_ps(arr.as_ptr(), _mm256_setr_epi64x(0, 16, 32, 48), 4); + let r = _mm256_i64gather_ps::<4>(arr.as_ptr(), _mm256_setr_epi64x(0, 16, 32, 48)); assert_eq_m128(r, _mm_setr_ps(0.0, 16.0, 32.0, 48.0)); } @@ -5988,12 +5730,11 @@ mod tests { j += 1.0; } // A multiplier of 4 is word-addressing for f32s - let r = _mm256_mask_i64gather_ps( + let r = _mm256_mask_i64gather_ps::<4>( _mm_set1_ps(256.0), arr.as_ptr(), _mm256_setr_epi64x(0, 16, 64, 96), _mm_setr_ps(-1.0, -1.0, -1.0, 0.0), - 4, ); assert_eq_m128(r, _mm_setr_ps(0.0, 16.0, 64.0, 256.0)); } @@ -6005,7 +5746,7 @@ mod tests { arr[i as usize] = i; } // A multiplier of 8 is word-addressing for i64s - let r = _mm_i64gather_epi64(arr.as_ptr(), _mm_setr_epi64x(0, 16), 8); + let r = _mm_i64gather_epi64::<8>(arr.as_ptr(), _mm_setr_epi64x(0, 16)); assert_eq_m128i(r, _mm_setr_epi64x(0, 16)); } @@ -6016,12 +5757,11 @@ mod tests { arr[i as usize] = i; } // A multiplier of 8 is word-addressing for i64s - let r = _mm_mask_i64gather_epi64( + let r = _mm_mask_i64gather_epi64::<8>( _mm_set1_epi64x(256), arr.as_ptr(), _mm_setr_epi64x(16, 16), _mm_setr_epi64x(-1, 0), - 8, ); assert_eq_m128i(r, _mm_setr_epi64x(16, 256)); } @@ -6033,7 +5773,7 @@ mod tests { arr[i as usize] = i; } // A multiplier of 8 is word-addressing for i64s - let r = _mm256_i64gather_epi64(arr.as_ptr(), _mm256_setr_epi64x(0, 16, 32, 48), 8); + let r = _mm256_i64gather_epi64::<8>(arr.as_ptr(), _mm256_setr_epi64x(0, 16, 32, 48)); assert_eq_m256i(r, _mm256_setr_epi64x(0, 16, 32, 48)); } @@ -6044,12 +5784,11 @@ mod tests { arr[i as usize] = i; } // A multiplier of 8 is word-addressing for i64s - let r = _mm256_mask_i64gather_epi64( + let r = _mm256_mask_i64gather_epi64::<8>( _mm256_set1_epi64x(256), arr.as_ptr(), _mm256_setr_epi64x(0, 16, 64, 96), _mm256_setr_epi64x(-1, -1, -1, 0), - 8, ); assert_eq_m256i(r, _mm256_setr_epi64x(0, 16, 64, 256)); } @@ -6063,7 +5802,7 @@ mod tests { j += 1.0; } // A multiplier of 8 is word-addressing for f64s - let r = _mm_i64gather_pd(arr.as_ptr(), _mm_setr_epi64x(0, 16), 8); + let r = _mm_i64gather_pd::<8>(arr.as_ptr(), _mm_setr_epi64x(0, 16)); assert_eq_m128d(r, _mm_setr_pd(0.0, 16.0)); } @@ -6076,12 +5815,11 @@ mod tests { j += 1.0; } // A multiplier of 8 is word-addressing for f64s - let r = _mm_mask_i64gather_pd( + let r = _mm_mask_i64gather_pd::<8>( _mm_set1_pd(256.0), arr.as_ptr(), _mm_setr_epi64x(16, 16), _mm_setr_pd(-1.0, 0.0), - 8, ); assert_eq_m128d(r, _mm_setr_pd(16.0, 256.0)); } @@ -6095,7 +5833,7 @@ mod tests { j += 1.0; } // A multiplier of 8 is word-addressing for f64s - let r = _mm256_i64gather_pd(arr.as_ptr(), _mm256_setr_epi64x(0, 16, 32, 48), 8); + let r = _mm256_i64gather_pd::<8>(arr.as_ptr(), _mm256_setr_epi64x(0, 16, 32, 48)); assert_eq_m256d(r, _mm256_setr_pd(0.0, 16.0, 32.0, 48.0)); } @@ -6108,12 +5846,11 @@ mod tests { j += 1.0; } // A multiplier of 8 is word-addressing for f64s - let r = _mm256_mask_i64gather_pd( + let r = _mm256_mask_i64gather_pd::<8>( _mm256_set1_pd(256.0), arr.as_ptr(), _mm256_setr_epi64x(0, 16, 64, 96), _mm256_setr_pd(-1.0, -1.0, -1.0, 0.0), - 8, ); assert_eq_m256d(r, _mm256_setr_pd(0.0, 16.0, 64.0, 256.0)); } @@ -6127,8 +5864,8 @@ mod tests { 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31 ); - let r1 = _mm256_extract_epi8(a, 0); - let r2 = _mm256_extract_epi8(a, 35); + let r1 = _mm256_extract_epi8::<0>(a); + let r2 = _mm256_extract_epi8::<3>(a); assert_eq!(r1, 0xFF); assert_eq!(r2, 3); } @@ -6140,8 +5877,8 @@ mod tests { -1, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, ); - let r1 = _mm256_extract_epi16(a, 0); - let r2 = _mm256_extract_epi16(a, 19); + let r1 = _mm256_extract_epi16::<0>(a); + let r2 = _mm256_extract_epi16::<3>(a); assert_eq!(r1, 0xFFFF); assert_eq!(r2, 3); } @@ -6149,8 +5886,8 @@ mod tests { #[simd_test(enable = "avx2")] unsafe fn test_mm256_extract_epi32() { let a = _mm256_setr_epi32(-1, 1, 2, 3, 4, 5, 6, 7); - let r1 = _mm256_extract_epi32(a, 0); - let r2 = _mm256_extract_epi32(a, 11); + let r1 = _mm256_extract_epi32::<0>(a); + let r2 = _mm256_extract_epi32::<3>(a); assert_eq!(r1, -1); assert_eq!(r2, 3); } diff --git a/library/stdarch/crates/core_arch/src/x86/avx512bw.rs b/library/stdarch/crates/core_arch/src/x86/avx512bw.rs index f4dc00bd87..4834f19edd 100644 --- a/library/stdarch/crates/core_arch/src/x86/avx512bw.rs +++ b/library/stdarch/crates/core_arch/src/x86/avx512bw.rs @@ -3689,22 +3689,18 @@ pub unsafe fn _mm_mask_cmpneq_epi8_mask(k1: __mmask16, a: __m128i, b: __m128i) - _mm_cmpneq_epi8_mask(a, b) & k1 } -/// Compare packed unsigned 16-bit integers in a and b based on the comparison operand specified by imm8, and store the results in mask vector k. +/// Compare packed unsigned 16-bit integers in a and b based on the comparison operand specified by `IMM8`, and store the results in mask vector k. /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cmp_epu16_mask&expand=715) #[inline] #[target_feature(enable = "avx512bw")] -#[rustc_args_required_const(2)] -#[cfg_attr(test, assert_instr(vpcmp, imm8 = 0))] -pub unsafe fn _mm512_cmp_epu16_mask(a: __m512i, b: __m512i, imm8: i32) -> __mmask32 { +#[rustc_legacy_const_generics(2)] +#[cfg_attr(test, assert_instr(vpcmp, IMM8 = 0))] +pub unsafe fn _mm512_cmp_epu16_mask(a: __m512i, b: __m512i) -> __mmask32 { + static_assert_imm3!(IMM8); let a = a.as_u16x32(); let b = b.as_u16x32(); - macro_rules! call { - ($imm3:expr) => { - vpcmpuw(a, b, $imm3, 0b11111111_11111111_11111111_11111111) - }; - } - let r = constify_imm3!(imm8, call); + let r = vpcmpuw(a, b, IMM8, 0b11111111_11111111_11111111_11111111); transmute(r) } @@ -3713,22 +3709,17 @@ pub unsafe fn _mm512_cmp_epu16_mask(a: __m512i, b: __m512i, imm8: i32) -> __mmas /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cmp_epu16_mask&expand=716) #[inline] #[target_feature(enable = "avx512bw")] -#[rustc_args_required_const(3)] -#[cfg_attr(test, assert_instr(vpcmp, imm8 = 0))] -pub unsafe fn _mm512_mask_cmp_epu16_mask( +#[rustc_legacy_const_generics(3)] +#[cfg_attr(test, assert_instr(vpcmp, IMM8 = 0))] +pub unsafe fn _mm512_mask_cmp_epu16_mask( k1: __mmask32, a: __m512i, b: __m512i, - imm8: i32, ) -> __mmask32 { + static_assert_imm3!(IMM8); let a = a.as_u16x32(); let b = b.as_u16x32(); - macro_rules! call { - ($imm3:expr) => { - vpcmpuw(a, b, $imm3, k1) - }; - } - let r = constify_imm3!(imm8, call); + let r = vpcmpuw(a, b, IMM8, k1); transmute(r) } @@ -3737,17 +3728,13 @@ pub unsafe fn _mm512_mask_cmp_epu16_mask( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_cmp_epu16_mask&expand=713) #[inline] #[target_feature(enable = "avx512bw,avx512vl")] -#[rustc_args_required_const(2)] -#[cfg_attr(test, assert_instr(vpcmp, imm8 = 0))] -pub unsafe fn _mm256_cmp_epu16_mask(a: __m256i, b: __m256i, imm8: i32) -> __mmask16 { +#[rustc_legacy_const_generics(2)] +#[cfg_attr(test, assert_instr(vpcmp, IMM8 = 0))] +pub unsafe fn _mm256_cmp_epu16_mask(a: __m256i, b: __m256i) -> __mmask16 { + static_assert_imm3!(IMM8); let a = a.as_u16x16(); let b = b.as_u16x16(); - macro_rules! call { - ($imm3:expr) => { - vpcmpuw256(a, b, $imm3, 0b11111111_11111111) - }; - } - let r = constify_imm3!(imm8, call); + let r = vpcmpuw256(a, b, IMM8, 0b11111111_11111111); transmute(r) } @@ -3756,22 +3743,17 @@ pub unsafe fn _mm256_cmp_epu16_mask(a: __m256i, b: __m256i, imm8: i32) -> __mmas /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_cmp_epu16_mask&expand=714) #[inline] #[target_feature(enable = "avx512bw,avx512vl")] -#[rustc_args_required_const(3)] -#[cfg_attr(test, assert_instr(vpcmp, imm8 = 0))] -pub unsafe fn _mm256_mask_cmp_epu16_mask( +#[rustc_legacy_const_generics(3)] +#[cfg_attr(test, assert_instr(vpcmp, IMM8 = 0))] +pub unsafe fn _mm256_mask_cmp_epu16_mask( k1: __mmask16, a: __m256i, b: __m256i, - imm8: i32, ) -> __mmask16 { + static_assert_imm3!(IMM8); let a = a.as_u16x16(); let b = b.as_u16x16(); - macro_rules! call { - ($imm3:expr) => { - vpcmpuw256(a, b, $imm3, k1) - }; - } - let r = constify_imm3!(imm8, call); + let r = vpcmpuw256(a, b, IMM8, k1); transmute(r) } @@ -3780,17 +3762,13 @@ pub unsafe fn _mm256_mask_cmp_epu16_mask( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cmp_epu16_mask&expand=711) #[inline] #[target_feature(enable = "avx512bw,avx512vl")] -#[rustc_args_required_const(2)] -#[cfg_attr(test, assert_instr(vpcmp, imm8 = 0))] -pub unsafe fn _mm_cmp_epu16_mask(a: __m128i, b: __m128i, imm8: i32) -> __mmask8 { +#[rustc_legacy_const_generics(2)] +#[cfg_attr(test, assert_instr(vpcmp, IMM8 = 0))] +pub unsafe fn _mm_cmp_epu16_mask(a: __m128i, b: __m128i) -> __mmask8 { + static_assert_imm3!(IMM8); let a = a.as_u16x8(); let b = b.as_u16x8(); - macro_rules! call { - ($imm3:expr) => { - vpcmpuw128(a, b, $imm3, 0b11111111) - }; - } - let r = constify_imm3!(imm8, call); + let r = vpcmpuw128(a, b, IMM8, 0b11111111); transmute(r) } @@ -3799,17 +3777,17 @@ pub unsafe fn _mm_cmp_epu16_mask(a: __m128i, b: __m128i, imm8: i32) -> __mmask8 /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_cmp_epu16_mask&expand=712) #[inline] #[target_feature(enable = "avx512bw,avx512vl")] -#[rustc_args_required_const(3)] -#[cfg_attr(test, assert_instr(vpcmp, imm8 = 0))] -pub unsafe fn _mm_mask_cmp_epu16_mask(k1: __mmask8, a: __m128i, b: __m128i, imm8: i32) -> __mmask8 { +#[rustc_legacy_const_generics(3)] +#[cfg_attr(test, assert_instr(vpcmp, IMM8 = 0))] +pub unsafe fn _mm_mask_cmp_epu16_mask( + k1: __mmask8, + a: __m128i, + b: __m128i, +) -> __mmask8 { + static_assert_imm3!(IMM8); let a = a.as_u16x8(); let b = b.as_u16x8(); - macro_rules! call { - ($imm3:expr) => { - vpcmpuw128(a, b, $imm3, k1) - }; - } - let r = constify_imm3!(imm8, call); + let r = vpcmpuw128(a, b, IMM8, k1); transmute(r) } @@ -3818,22 +3796,18 @@ pub unsafe fn _mm_mask_cmp_epu16_mask(k1: __mmask8, a: __m128i, b: __m128i, imm8 /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cmp_epu8_mask&expand=733) #[inline] #[target_feature(enable = "avx512bw")] -#[rustc_args_required_const(2)] -#[cfg_attr(test, assert_instr(vpcmp, imm8 = 0))] -pub unsafe fn _mm512_cmp_epu8_mask(a: __m512i, b: __m512i, imm8: i32) -> __mmask64 { +#[rustc_legacy_const_generics(2)] +#[cfg_attr(test, assert_instr(vpcmp, IMM8 = 0))] +pub unsafe fn _mm512_cmp_epu8_mask(a: __m512i, b: __m512i) -> __mmask64 { + static_assert_imm3!(IMM8); let a = a.as_u8x64(); let b = b.as_u8x64(); - macro_rules! call { - ($imm3:expr) => { - vpcmpub( - a, - b, - $imm3, - 0b11111111_11111111_11111111_11111111_11111111_11111111_11111111_11111111, - ) - }; - } - let r = constify_imm3!(imm8, call); + let r = vpcmpub( + a, + b, + IMM8, + 0b11111111_11111111_11111111_11111111_11111111_11111111_11111111_11111111, + ); transmute(r) } @@ -3842,22 +3816,17 @@ pub unsafe fn _mm512_cmp_epu8_mask(a: __m512i, b: __m512i, imm8: i32) -> __mmask /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cmp_epu8_mask&expand=734) #[inline] #[target_feature(enable = "avx512bw")] -#[rustc_args_required_const(3)] -#[cfg_attr(test, assert_instr(vpcmp, imm8 = 0))] -pub unsafe fn _mm512_mask_cmp_epu8_mask( +#[rustc_legacy_const_generics(3)] +#[cfg_attr(test, assert_instr(vpcmp, IMM8 = 0))] +pub unsafe fn _mm512_mask_cmp_epu8_mask( k1: __mmask64, a: __m512i, b: __m512i, - imm8: i32, ) -> __mmask64 { + static_assert_imm3!(IMM8); let a = a.as_u8x64(); let b = b.as_u8x64(); - macro_rules! call { - ($imm3:expr) => { - vpcmpub(a, b, $imm3, k1) - }; - } - let r = constify_imm3!(imm8, call); + let r = vpcmpub(a, b, IMM8, k1); transmute(r) } @@ -3866,17 +3835,13 @@ pub unsafe fn _mm512_mask_cmp_epu8_mask( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_cmp_epu8_mask&expand=731) #[inline] #[target_feature(enable = "avx512bw,avx512vl")] -#[rustc_args_required_const(2)] -#[cfg_attr(test, assert_instr(vpcmp, imm8 = 0))] -pub unsafe fn _mm256_cmp_epu8_mask(a: __m256i, b: __m256i, imm8: i32) -> __mmask32 { +#[rustc_legacy_const_generics(2)] +#[cfg_attr(test, assert_instr(vpcmp, IMM8 = 0))] +pub unsafe fn _mm256_cmp_epu8_mask(a: __m256i, b: __m256i) -> __mmask32 { + static_assert_imm3!(IMM8); let a = a.as_u8x32(); let b = b.as_u8x32(); - macro_rules! call { - ($imm3:expr) => { - vpcmpub256(a, b, $imm3, 0b11111111_11111111_11111111_11111111) - }; - } - let r = constify_imm3!(imm8, call); + let r = vpcmpub256(a, b, IMM8, 0b11111111_11111111_11111111_11111111); transmute(r) } @@ -3885,22 +3850,17 @@ pub unsafe fn _mm256_cmp_epu8_mask(a: __m256i, b: __m256i, imm8: i32) -> __mmask /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_cmp_epu8_mask&expand=732) #[inline] #[target_feature(enable = "avx512bw,avx512vl")] -#[rustc_args_required_const(3)] -#[cfg_attr(test, assert_instr(vpcmp, imm8 = 0))] -pub unsafe fn _mm256_mask_cmp_epu8_mask( +#[rustc_legacy_const_generics(3)] +#[cfg_attr(test, assert_instr(vpcmp, IMM8 = 0))] +pub unsafe fn _mm256_mask_cmp_epu8_mask( k1: __mmask32, a: __m256i, b: __m256i, - imm8: i32, ) -> __mmask32 { + static_assert_imm3!(IMM8); let a = a.as_u8x32(); let b = b.as_u8x32(); - macro_rules! call { - ($imm3:expr) => { - vpcmpub256(a, b, $imm3, k1) - }; - } - let r = constify_imm3!(imm8, call); + let r = vpcmpub256(a, b, IMM8, k1); transmute(r) } @@ -3909,17 +3869,13 @@ pub unsafe fn _mm256_mask_cmp_epu8_mask( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cmp_epu8_mask&expand=729) #[inline] #[target_feature(enable = "avx512bw,avx512vl")] -#[rustc_args_required_const(2)] -#[cfg_attr(test, assert_instr(vpcmp, imm8 = 0))] -pub unsafe fn _mm_cmp_epu8_mask(a: __m128i, b: __m128i, imm8: i32) -> __mmask16 { +#[rustc_legacy_const_generics(2)] +#[cfg_attr(test, assert_instr(vpcmp, IMM8 = 0))] +pub unsafe fn _mm_cmp_epu8_mask(a: __m128i, b: __m128i) -> __mmask16 { + static_assert_imm3!(IMM8); let a = a.as_u8x16(); let b = b.as_u8x16(); - macro_rules! call { - ($imm3:expr) => { - vpcmpub128(a, b, $imm3, 0b11111111_11111111) - }; - } - let r = constify_imm3!(imm8, call); + let r = vpcmpub128(a, b, IMM8, 0b11111111_11111111); transmute(r) } @@ -3928,22 +3884,17 @@ pub unsafe fn _mm_cmp_epu8_mask(a: __m128i, b: __m128i, imm8: i32) -> __mmask16 /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_cmp_epu8_mask&expand=730) #[inline] #[target_feature(enable = "avx512bw,avx512vl")] -#[rustc_args_required_const(3)] -#[cfg_attr(test, assert_instr(vpcmp, imm8 = 0))] -pub unsafe fn _mm_mask_cmp_epu8_mask( +#[rustc_legacy_const_generics(3)] +#[cfg_attr(test, assert_instr(vpcmp, IMM8 = 0))] +pub unsafe fn _mm_mask_cmp_epu8_mask( k1: __mmask16, a: __m128i, b: __m128i, - imm8: i32, ) -> __mmask16 { + static_assert_imm3!(IMM8); let a = a.as_u8x16(); let b = b.as_u8x16(); - macro_rules! call { - ($imm3:expr) => { - vpcmpub128(a, b, $imm3, k1) - }; - } - let r = constify_imm3!(imm8, call); + let r = vpcmpub128(a, b, IMM8, k1); transmute(r) } @@ -3952,17 +3903,13 @@ pub unsafe fn _mm_mask_cmp_epu8_mask( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cmp_epi16_mask&expand=691) #[inline] #[target_feature(enable = "avx512bw")] -#[rustc_args_required_const(2)] -#[cfg_attr(test, assert_instr(vpcmp, imm8 = 0))] -pub unsafe fn _mm512_cmp_epi16_mask(a: __m512i, b: __m512i, imm8: i32) -> __mmask32 { +#[rustc_legacy_const_generics(2)] +#[cfg_attr(test, assert_instr(vpcmp, IMM8 = 0))] +pub unsafe fn _mm512_cmp_epi16_mask(a: __m512i, b: __m512i) -> __mmask32 { + static_assert_imm3!(IMM8); let a = a.as_i16x32(); let b = b.as_i16x32(); - macro_rules! call { - ($imm3:expr) => { - vpcmpw(a, b, $imm3, 0b11111111_11111111_11111111_11111111) - }; - } - let r = constify_imm3!(imm8, call); + let r = vpcmpw(a, b, IMM8, 0b11111111_11111111_11111111_11111111); transmute(r) } @@ -3971,22 +3918,17 @@ pub unsafe fn _mm512_cmp_epi16_mask(a: __m512i, b: __m512i, imm8: i32) -> __mmas /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cmp_epi16_mask&expand=692) #[inline] #[target_feature(enable = "avx512bw")] -#[rustc_args_required_const(3)] -#[cfg_attr(test, assert_instr(vpcmp, imm8 = 0))] -pub unsafe fn _mm512_mask_cmp_epi16_mask( +#[rustc_legacy_const_generics(3)] +#[cfg_attr(test, assert_instr(vpcmp, IMM8 = 0))] +pub unsafe fn _mm512_mask_cmp_epi16_mask( k1: __mmask32, a: __m512i, b: __m512i, - imm8: i32, ) -> __mmask32 { + static_assert_imm3!(IMM8); let a = a.as_i16x32(); let b = b.as_i16x32(); - macro_rules! call { - ($imm3:expr) => { - vpcmpw(a, b, $imm3, k1) - }; - } - let r = constify_imm3!(imm8, call); + let r = vpcmpw(a, b, IMM8, k1); transmute(r) } @@ -3995,17 +3937,13 @@ pub unsafe fn _mm512_mask_cmp_epi16_mask( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_cmp_epi16_mask&expand=689) #[inline] #[target_feature(enable = "avx512bw,avx512vl")] -#[rustc_args_required_const(2)] -#[cfg_attr(test, assert_instr(vpcmp, imm8 = 0))] -pub unsafe fn _mm256_cmp_epi16_mask(a: __m256i, b: __m256i, imm8: i32) -> __mmask16 { +#[rustc_legacy_const_generics(2)] +#[cfg_attr(test, assert_instr(vpcmp, IMM8 = 0))] +pub unsafe fn _mm256_cmp_epi16_mask(a: __m256i, b: __m256i) -> __mmask16 { + static_assert_imm3!(IMM8); let a = a.as_i16x16(); let b = b.as_i16x16(); - macro_rules! call { - ($imm3:expr) => { - vpcmpw256(a, b, $imm3, 0b11111111_11111111) - }; - } - let r = constify_imm3!(imm8, call); + let r = vpcmpw256(a, b, IMM8, 0b11111111_11111111); transmute(r) } @@ -4014,22 +3952,17 @@ pub unsafe fn _mm256_cmp_epi16_mask(a: __m256i, b: __m256i, imm8: i32) -> __mmas /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_cmp_epi16_mask&expand=690) #[inline] #[target_feature(enable = "avx512bw,avx512vl")] -#[rustc_args_required_const(3)] -#[cfg_attr(test, assert_instr(vpcmp, imm8 = 0))] -pub unsafe fn _mm256_mask_cmp_epi16_mask( +#[rustc_legacy_const_generics(3)] +#[cfg_attr(test, assert_instr(vpcmp, IMM8 = 0))] +pub unsafe fn _mm256_mask_cmp_epi16_mask( k1: __mmask16, a: __m256i, b: __m256i, - imm8: i32, ) -> __mmask16 { + static_assert_imm3!(IMM8); let a = a.as_i16x16(); let b = b.as_i16x16(); - macro_rules! call { - ($imm3:expr) => { - vpcmpw256(a, b, $imm3, k1) - }; - } - let r = constify_imm3!(imm8, call); + let r = vpcmpw256(a, b, IMM8, k1); transmute(r) } @@ -4038,17 +3971,13 @@ pub unsafe fn _mm256_mask_cmp_epi16_mask( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cmp_epi16_mask&expand=687) #[inline] #[target_feature(enable = "avx512bw,avx512vl")] -#[rustc_args_required_const(2)] -#[cfg_attr(test, assert_instr(vpcmp, imm8 = 0))] -pub unsafe fn _mm_cmp_epi16_mask(a: __m128i, b: __m128i, imm8: i32) -> __mmask8 { +#[rustc_legacy_const_generics(2)] +#[cfg_attr(test, assert_instr(vpcmp, IMM8 = 0))] +pub unsafe fn _mm_cmp_epi16_mask(a: __m128i, b: __m128i) -> __mmask8 { + static_assert_imm3!(IMM8); let a = a.as_i16x8(); let b = b.as_i16x8(); - macro_rules! call { - ($imm3:expr) => { - vpcmpw128(a, b, $imm3, 0b11111111) - }; - } - let r = constify_imm3!(imm8, call); + let r = vpcmpw128(a, b, IMM8, 0b11111111); transmute(r) } @@ -4057,17 +3986,17 @@ pub unsafe fn _mm_cmp_epi16_mask(a: __m128i, b: __m128i, imm8: i32) -> __mmask8 /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_cmp_epi16_mask&expand=688) #[inline] #[target_feature(enable = "avx512bw,avx512vl")] -#[rustc_args_required_const(3)] -#[cfg_attr(test, assert_instr(vpcmp, imm8 = 0))] -pub unsafe fn _mm_mask_cmp_epi16_mask(k1: __mmask8, a: __m128i, b: __m128i, imm8: i32) -> __mmask8 { +#[rustc_legacy_const_generics(3)] +#[cfg_attr(test, assert_instr(vpcmp, IMM8 = 0))] +pub unsafe fn _mm_mask_cmp_epi16_mask( + k1: __mmask8, + a: __m128i, + b: __m128i, +) -> __mmask8 { + static_assert_imm3!(IMM8); let a = a.as_i16x8(); let b = b.as_i16x8(); - macro_rules! call { - ($imm3:expr) => { - vpcmpw128(a, b, $imm3, k1) - }; - } - let r = constify_imm3!(imm8, call); + let r = vpcmpw128(a, b, IMM8, k1); transmute(r) } @@ -4076,22 +4005,18 @@ pub unsafe fn _mm_mask_cmp_epi16_mask(k1: __mmask8, a: __m128i, b: __m128i, imm8 /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cmp_epi8_mask&expand=709) #[inline] #[target_feature(enable = "avx512bw")] -#[rustc_args_required_const(2)] -#[cfg_attr(test, assert_instr(vpcmp, imm8 = 0))] -pub unsafe fn _mm512_cmp_epi8_mask(a: __m512i, b: __m512i, imm8: i32) -> __mmask64 { +#[rustc_legacy_const_generics(2)] +#[cfg_attr(test, assert_instr(vpcmp, IMM8 = 0))] +pub unsafe fn _mm512_cmp_epi8_mask(a: __m512i, b: __m512i) -> __mmask64 { + static_assert_imm3!(IMM8); let a = a.as_i8x64(); let b = b.as_i8x64(); - macro_rules! call { - ($imm3:expr) => { - vpcmpb( - a, - b, - $imm3, - 0b11111111_11111111_11111111_11111111_11111111_11111111_11111111_11111111, - ) - }; - } - let r = constify_imm3!(imm8, call); + let r = vpcmpb( + a, + b, + IMM8, + 0b11111111_11111111_11111111_11111111_11111111_11111111_11111111_11111111, + ); transmute(r) } @@ -4100,22 +4025,17 @@ pub unsafe fn _mm512_cmp_epi8_mask(a: __m512i, b: __m512i, imm8: i32) -> __mmask /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cmp_epi8_mask&expand=710) #[inline] #[target_feature(enable = "avx512bw")] -#[rustc_args_required_const(3)] -#[cfg_attr(test, assert_instr(vpcmp, imm8 = 0))] -pub unsafe fn _mm512_mask_cmp_epi8_mask( +#[rustc_legacy_const_generics(3)] +#[cfg_attr(test, assert_instr(vpcmp, IMM8 = 0))] +pub unsafe fn _mm512_mask_cmp_epi8_mask( k1: __mmask64, a: __m512i, b: __m512i, - imm8: i32, ) -> __mmask64 { + static_assert_imm3!(IMM8); let a = a.as_i8x64(); let b = b.as_i8x64(); - macro_rules! call { - ($imm3:expr) => { - vpcmpb(a, b, $imm3, k1) - }; - } - let r = constify_imm3!(imm8, call); + let r = vpcmpb(a, b, IMM8, k1); transmute(r) } @@ -4124,17 +4044,13 @@ pub unsafe fn _mm512_mask_cmp_epi8_mask( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_cmp_epi8_mask&expand=707) #[inline] #[target_feature(enable = "avx512bw,avx512vl")] -#[rustc_args_required_const(2)] -#[cfg_attr(test, assert_instr(vpcmp, imm8 = 0))] -pub unsafe fn _mm256_cmp_epi8_mask(a: __m256i, b: __m256i, imm8: i32) -> __mmask32 { +#[rustc_legacy_const_generics(2)] +#[cfg_attr(test, assert_instr(vpcmp, IMM8 = 0))] +pub unsafe fn _mm256_cmp_epi8_mask(a: __m256i, b: __m256i) -> __mmask32 { + static_assert_imm3!(IMM8); let a = a.as_i8x32(); let b = b.as_i8x32(); - macro_rules! call { - ($imm3:expr) => { - vpcmpb256(a, b, $imm3, 0b11111111_11111111_11111111_11111111) - }; - } - let r = constify_imm3!(imm8, call); + let r = vpcmpb256(a, b, IMM8, 0b11111111_11111111_11111111_11111111); transmute(r) } @@ -4143,22 +4059,17 @@ pub unsafe fn _mm256_cmp_epi8_mask(a: __m256i, b: __m256i, imm8: i32) -> __mmask /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_cmp_epi8_mask&expand=708) #[inline] #[target_feature(enable = "avx512bw,avx512vl")] -#[rustc_args_required_const(3)] -#[cfg_attr(test, assert_instr(vpcmp, imm8 = 0))] -pub unsafe fn _mm256_mask_cmp_epi8_mask( +#[rustc_legacy_const_generics(3)] +#[cfg_attr(test, assert_instr(vpcmp, IMM8 = 0))] +pub unsafe fn _mm256_mask_cmp_epi8_mask( k1: __mmask32, a: __m256i, b: __m256i, - imm8: i32, ) -> __mmask32 { + static_assert_imm3!(IMM8); let a = a.as_i8x32(); let b = b.as_i8x32(); - macro_rules! call { - ($imm3:expr) => { - vpcmpb256(a, b, $imm3, k1) - }; - } - let r = constify_imm3!(imm8, call); + let r = vpcmpb256(a, b, IMM8, k1); transmute(r) } @@ -4167,17 +4078,13 @@ pub unsafe fn _mm256_mask_cmp_epi8_mask( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cmp_epi8_mask&expand=705) #[inline] #[target_feature(enable = "avx512bw,avx512vl")] -#[rustc_args_required_const(2)] -#[cfg_attr(test, assert_instr(vpcmp, imm8 = 0))] -pub unsafe fn _mm_cmp_epi8_mask(a: __m128i, b: __m128i, imm8: i32) -> __mmask16 { +#[rustc_legacy_const_generics(2)] +#[cfg_attr(test, assert_instr(vpcmp, IMM8 = 0))] +pub unsafe fn _mm_cmp_epi8_mask(a: __m128i, b: __m128i) -> __mmask16 { + static_assert_imm3!(IMM8); let a = a.as_i8x16(); let b = b.as_i8x16(); - macro_rules! call { - ($imm3:expr) => { - vpcmpb128(a, b, $imm3, 0b11111111_11111111) - }; - } - let r = constify_imm3!(imm8, call); + let r = vpcmpb128(a, b, IMM8, 0b11111111_11111111); transmute(r) } @@ -4186,22 +4093,17 @@ pub unsafe fn _mm_cmp_epi8_mask(a: __m128i, b: __m128i, imm8: i32) -> __mmask16 /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_cmp_epi8_mask&expand=706) #[inline] #[target_feature(enable = "avx512bw,avx512vl")] -#[rustc_args_required_const(3)] -#[cfg_attr(test, assert_instr(vpcmp, imm8 = 0))] -pub unsafe fn _mm_mask_cmp_epi8_mask( +#[rustc_legacy_const_generics(3)] +#[cfg_attr(test, assert_instr(vpcmp, IMM8 = 0))] +pub unsafe fn _mm_mask_cmp_epi8_mask( k1: __mmask16, a: __m128i, b: __m128i, - imm8: i32, ) -> __mmask16 { + static_assert_imm3!(IMM8); let a = a.as_i8x16(); let b = b.as_i8x16(); - macro_rules! call { - ($imm3:expr) => { - vpcmpb128(a, b, $imm3, k1) - }; - } - let r = constify_imm3!(imm8, call); + let r = vpcmpb128(a, b, IMM8, k1); transmute(r) } @@ -5106,16 +5008,12 @@ pub unsafe fn _mm_maskz_sll_epi16(k: __mmask8, a: __m128i, count: __m128i) -> __ /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_slli_epi16&expand=5301) #[inline] #[target_feature(enable = "avx512bw")] -#[cfg_attr(test, assert_instr(vpsllw, imm8 = 5))] -#[rustc_args_required_const(1)] -pub unsafe fn _mm512_slli_epi16(a: __m512i, imm8: u32) -> __m512i { +#[cfg_attr(test, assert_instr(vpsllw, IMM8 = 5))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm512_slli_epi16(a: __m512i) -> __m512i { + static_assert_imm_u8!(IMM8); let a = a.as_i16x32(); - macro_rules! call { - ($imm8:expr) => { - vpslliw(a, $imm8) - }; - } - let r = constify_imm8_sae!(imm8, call); + let r = vpslliw(a, IMM8); transmute(r) } @@ -5124,16 +5022,16 @@ pub unsafe fn _mm512_slli_epi16(a: __m512i, imm8: u32) -> __m512i { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_slli_epi16&expand=5299) #[inline] #[target_feature(enable = "avx512bw")] -#[cfg_attr(test, assert_instr(vpsllw, imm8 = 5))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm512_mask_slli_epi16(src: __m512i, k: __mmask32, a: __m512i, imm8: u32) -> __m512i { +#[cfg_attr(test, assert_instr(vpsllw, IMM8 = 5))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_mask_slli_epi16( + src: __m512i, + k: __mmask32, + a: __m512i, +) -> __m512i { + static_assert_imm_u8!(IMM8); let a = a.as_i16x32(); - macro_rules! call { - ($imm8:expr) => { - vpslliw(a, $imm8) - }; - } - let shf = constify_imm8_sae!(imm8, call); + let shf = vpslliw(a, IMM8); transmute(simd_select_bitmask(k, shf, src.as_i16x32())) } @@ -5142,16 +5040,12 @@ pub unsafe fn _mm512_mask_slli_epi16(src: __m512i, k: __mmask32, a: __m512i, imm /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_slli_epi16&expand=5300) #[inline] #[target_feature(enable = "avx512bw")] -#[cfg_attr(test, assert_instr(vpsllw, imm8 = 5))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm512_maskz_slli_epi16(k: __mmask32, a: __m512i, imm8: u32) -> __m512i { +#[cfg_attr(test, assert_instr(vpsllw, IMM8 = 5))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_maskz_slli_epi16(k: __mmask32, a: __m512i) -> __m512i { + static_assert_imm_u8!(IMM8); let a = a.as_i16x32(); - macro_rules! call { - ($imm8:expr) => { - vpslliw(a, $imm8) - }; - } - let shf = constify_imm8_sae!(imm8, call); + let shf = vpslliw(a, IMM8); let zero = _mm512_setzero_si512().as_i16x32(); transmute(simd_select_bitmask(k, shf, zero)) } @@ -5161,16 +5055,17 @@ pub unsafe fn _mm512_maskz_slli_epi16(k: __mmask32, a: __m512i, imm8: u32) -> __ /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_slli_epi16&expand=5296) #[inline] #[target_feature(enable = "avx512bw,avx512vl")] -#[cfg_attr(test, assert_instr(vpsllw, imm8 = 5))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm256_mask_slli_epi16(src: __m256i, k: __mmask16, a: __m256i, imm8: u32) -> __m256i { - macro_rules! call { - ($imm8:expr) => { - _mm256_slli_epi16(a, $imm8) - }; - } - let shf = constify_imm8_sae!(imm8, call); - transmute(simd_select_bitmask(k, shf.as_i16x16(), src.as_i16x16())) +#[cfg_attr(test, assert_instr(vpsllw, IMM8 = 5))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm256_mask_slli_epi16( + src: __m256i, + k: __mmask16, + a: __m256i, +) -> __m256i { + static_assert_imm_u8!(IMM8); + let imm8 = IMM8 as i32; + let r = pslliw256(a.as_i16x16(), imm8); + transmute(simd_select_bitmask(k, r, src.as_i16x16())) } /// Shift packed 16-bit integers in a left by imm8 while shifting in zeros, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). @@ -5178,17 +5073,14 @@ pub unsafe fn _mm256_mask_slli_epi16(src: __m256i, k: __mmask16, a: __m256i, imm /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_slli_epi16&expand=5297) #[inline] #[target_feature(enable = "avx512bw,avx512vl")] -#[cfg_attr(test, assert_instr(vpsllw, imm8 = 5))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm256_maskz_slli_epi16(k: __mmask16, a: __m256i, imm8: u32) -> __m256i { - macro_rules! call { - ($imm8:expr) => { - _mm256_slli_epi16(a, $imm8) - }; - } - let shf = constify_imm8_sae!(imm8, call); +#[cfg_attr(test, assert_instr(vpsllw, IMM8 = 5))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm256_maskz_slli_epi16(k: __mmask16, a: __m256i) -> __m256i { + static_assert_imm_u8!(IMM8); + let imm8 = IMM8 as i32; + let r = pslliw256(a.as_i16x16(), imm8); let zero = _mm256_setzero_si256().as_i16x16(); - transmute(simd_select_bitmask(k, shf.as_i16x16(), zero)) + transmute(simd_select_bitmask(k, r, zero)) } /// Shift packed 16-bit integers in a left by imm8 while shifting in zeros, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). @@ -5196,16 +5088,17 @@ pub unsafe fn _mm256_maskz_slli_epi16(k: __mmask16, a: __m256i, imm8: u32) -> __ /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_slli_epi16&expand=5293) #[inline] #[target_feature(enable = "avx512bw,avx512vl")] -#[cfg_attr(test, assert_instr(vpsllw, imm8 = 5))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm_mask_slli_epi16(src: __m128i, k: __mmask8, a: __m128i, imm8: u32) -> __m128i { - macro_rules! call { - ($imm8:expr) => { - _mm_slli_epi16(a, $imm8) - }; - } - let shf = constify_imm8_sae!(imm8, call); - transmute(simd_select_bitmask(k, shf.as_i16x8(), src.as_i16x8())) +#[cfg_attr(test, assert_instr(vpsllw, IMM8 = 5))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm_mask_slli_epi16( + src: __m128i, + k: __mmask8, + a: __m128i, +) -> __m128i { + static_assert_imm_u8!(IMM8); + let imm8 = IMM8 as i32; + let r = pslliw128(a.as_i16x8(), imm8); + transmute(simd_select_bitmask(k, r, src.as_i16x8())) } /// Shift packed 16-bit integers in a left by imm8 while shifting in zeros, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). @@ -5213,17 +5106,14 @@ pub unsafe fn _mm_mask_slli_epi16(src: __m128i, k: __mmask8, a: __m128i, imm8: u /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_slli_epi16&expand=5294) #[inline] #[target_feature(enable = "avx512bw,avx512vl")] -#[cfg_attr(test, assert_instr(vpsllw, imm8 = 5))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm_maskz_slli_epi16(k: __mmask8, a: __m128i, imm8: u32) -> __m128i { - macro_rules! call { - ($imm8:expr) => { - _mm_slli_epi16(a, $imm8) - }; - } - let shf = constify_imm8_sae!(imm8, call); +#[cfg_attr(test, assert_instr(vpsllw, IMM8 = 5))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm_maskz_slli_epi16(k: __mmask8, a: __m128i) -> __m128i { + static_assert_imm_u8!(IMM8); + let imm8 = IMM8 as i32; + let r = pslliw128(a.as_i16x8(), imm8); let zero = _mm_setzero_si128().as_i16x8(); - transmute(simd_select_bitmask(k, shf.as_i16x8(), zero)) + transmute(simd_select_bitmask(k, r, zero)) } /// Shift packed 16-bit integers in a left by the amount specified by the corresponding element in count while shifting in zeros, and store the results in dst. @@ -5434,16 +5324,12 @@ pub unsafe fn _mm_maskz_srl_epi16(k: __mmask8, a: __m128i, count: __m128i) -> __ /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_srli_epi16&expand=5513) #[inline] #[target_feature(enable = "avx512bw")] -#[cfg_attr(test, assert_instr(vpsrlw, imm8 = 5))] -#[rustc_args_required_const(1)] -pub unsafe fn _mm512_srli_epi16(a: __m512i, imm8: u32) -> __m512i { +#[cfg_attr(test, assert_instr(vpsrlw, IMM8 = 5))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm512_srli_epi16(a: __m512i) -> __m512i { + static_assert_imm_u8!(IMM8); let a = a.as_i16x32(); - macro_rules! call { - ($imm8:expr) => { - vpsrliw(a, $imm8) - }; - } - let r = constify_imm8_sae!(imm8, call); + let r = vpsrliw(a, IMM8); transmute(r) } @@ -5452,16 +5338,16 @@ pub unsafe fn _mm512_srli_epi16(a: __m512i, imm8: u32) -> __m512i { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_srli_epi16&expand=5511) #[inline] #[target_feature(enable = "avx512bw")] -#[cfg_attr(test, assert_instr(vpsrlw, imm8 = 5))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm512_mask_srli_epi16(src: __m512i, k: __mmask32, a: __m512i, imm8: u32) -> __m512i { +#[cfg_attr(test, assert_instr(vpsrlw, IMM8 = 5))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_mask_srli_epi16( + src: __m512i, + k: __mmask32, + a: __m512i, +) -> __m512i { + static_assert_imm_u8!(IMM8); let a = a.as_i16x32(); - macro_rules! call { - ($imm8:expr) => { - vpsrliw(a, $imm8) - }; - } - let shf = constify_imm8_sae!(imm8, call); + let shf = vpsrliw(a, IMM8); transmute(simd_select_bitmask(k, shf, src.as_i16x32())) } @@ -5470,17 +5356,13 @@ pub unsafe fn _mm512_mask_srli_epi16(src: __m512i, k: __mmask32, a: __m512i, imm /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_srli_epi16&expand=5512) #[inline] #[target_feature(enable = "avx512bw")] -#[cfg_attr(test, assert_instr(vpsrlw, imm8 = 5))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm512_maskz_srli_epi16(k: __mmask32, a: __m512i, imm8: i32) -> __m512i { +#[cfg_attr(test, assert_instr(vpsrlw, IMM8 = 5))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_maskz_srli_epi16(k: __mmask32, a: __m512i) -> __m512i { + static_assert_imm8!(IMM8); //imm8 should be u32, it seems the document to verify is incorrect let a = a.as_i16x32(); - macro_rules! call { - ($imm8:expr) => { - vpsrliw(a, $imm8) - }; - } - let shf = constify_imm8_sae!(imm8, call); + let shf = vpsrliw(a, IMM8 as u32); let zero = _mm512_setzero_si512().as_i16x32(); transmute(simd_select_bitmask(k, shf, zero)) } @@ -5490,15 +5372,15 @@ pub unsafe fn _mm512_maskz_srli_epi16(k: __mmask32, a: __m512i, imm8: i32) -> __ /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_srli_epi16&expand=5508) #[inline] #[target_feature(enable = "avx512bw,avx512vl")] -#[cfg_attr(test, assert_instr(vpsrlw, imm8 = 5))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm256_mask_srli_epi16(src: __m256i, k: __mmask16, a: __m256i, imm8: i32) -> __m256i { - macro_rules! call { - ($imm8:expr) => { - _mm256_srli_epi16(a, $imm8) - }; - } - let shf = constify_imm8_sae!(imm8, call); +#[cfg_attr(test, assert_instr(vpsrlw, IMM8 = 5))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm256_mask_srli_epi16( + src: __m256i, + k: __mmask16, + a: __m256i, +) -> __m256i { + static_assert_imm8!(IMM8); + let shf = _mm256_srli_epi16::(a); transmute(simd_select_bitmask(k, shf.as_i16x16(), src.as_i16x16())) } @@ -5507,15 +5389,11 @@ pub unsafe fn _mm256_mask_srli_epi16(src: __m256i, k: __mmask16, a: __m256i, imm /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_srli_epi16&expand=5509) #[inline] #[target_feature(enable = "avx512bw,avx512vl")] -#[cfg_attr(test, assert_instr(vpsrlw, imm8 = 5))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm256_maskz_srli_epi16(k: __mmask16, a: __m256i, imm8: i32) -> __m256i { - macro_rules! call { - ($imm8:expr) => { - _mm256_srli_epi16(a, $imm8) - }; - } - let shf = constify_imm8_sae!(imm8, call); +#[cfg_attr(test, assert_instr(vpsrlw, IMM8 = 5))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm256_maskz_srli_epi16(k: __mmask16, a: __m256i) -> __m256i { + static_assert_imm8!(IMM8); + let shf = _mm256_srli_epi16::(a); let zero = _mm256_setzero_si256().as_i16x16(); transmute(simd_select_bitmask(k, shf.as_i16x16(), zero)) } @@ -5525,15 +5403,15 @@ pub unsafe fn _mm256_maskz_srli_epi16(k: __mmask16, a: __m256i, imm8: i32) -> __ /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_srli_epi16&expand=5505) #[inline] #[target_feature(enable = "avx512bw,avx512vl")] -#[cfg_attr(test, assert_instr(vpsrlw, imm8 = 5))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm_mask_srli_epi16(src: __m128i, k: __mmask8, a: __m128i, imm8: i32) -> __m128i { - macro_rules! call { - ($imm8:expr) => { - _mm_srli_epi16(a, $imm8) - }; - } - let shf = constify_imm8_sae!(imm8, call); +#[cfg_attr(test, assert_instr(vpsrlw, IMM8 = 5))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm_mask_srli_epi16( + src: __m128i, + k: __mmask8, + a: __m128i, +) -> __m128i { + static_assert_imm8!(IMM8); + let shf = _mm_srli_epi16::(a); transmute(simd_select_bitmask(k, shf.as_i16x8(), src.as_i16x8())) } @@ -5542,15 +5420,11 @@ pub unsafe fn _mm_mask_srli_epi16(src: __m128i, k: __mmask8, a: __m128i, imm8: i /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_srli_epi16&expand=5506) #[inline] #[target_feature(enable = "avx512bw,avx512vl")] -#[cfg_attr(test, assert_instr(vpsrlw, imm8 = 5))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm_maskz_srli_epi16(k: __mmask8, a: __m128i, imm8: i32) -> __m128i { - macro_rules! call { - ($imm8:expr) => { - _mm_srli_epi16(a, $imm8) - }; - } - let shf = constify_imm8_sae!(imm8, call); +#[cfg_attr(test, assert_instr(vpsrlw, IMM8 = 5))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm_maskz_srli_epi16(k: __mmask8, a: __m128i) -> __m128i { + static_assert_imm8!(IMM8); + let shf = _mm_srli_epi16::(a); let zero = _mm_setzero_si128().as_i16x8(); transmute(simd_select_bitmask(k, shf.as_i16x8(), zero)) } @@ -5763,16 +5637,12 @@ pub unsafe fn _mm_maskz_sra_epi16(k: __mmask8, a: __m128i, count: __m128i) -> __ /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_srai_epi16&expand=5427) #[inline] #[target_feature(enable = "avx512bw")] -#[cfg_attr(test, assert_instr(vpsraw, imm8 = 1))] -#[rustc_args_required_const(1)] -pub unsafe fn _mm512_srai_epi16(a: __m512i, imm8: u32) -> __m512i { +#[cfg_attr(test, assert_instr(vpsraw, IMM8 = 1))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm512_srai_epi16(a: __m512i) -> __m512i { + static_assert_imm_u8!(IMM8); let a = a.as_i16x32(); - macro_rules! call { - ($imm8:expr) => { - vpsraiw(a, $imm8) - }; - } - let r = constify_imm8_sae!(imm8, call); + let r = vpsraiw(a, IMM8); transmute(r) } @@ -5781,16 +5651,16 @@ pub unsafe fn _mm512_srai_epi16(a: __m512i, imm8: u32) -> __m512i { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_srai_epi16&expand=5425) #[inline] #[target_feature(enable = "avx512bw")] -#[cfg_attr(test, assert_instr(vpsraw, imm8 = 1))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm512_mask_srai_epi16(src: __m512i, k: __mmask32, a: __m512i, imm8: u32) -> __m512i { +#[cfg_attr(test, assert_instr(vpsraw, IMM8 = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_mask_srai_epi16( + src: __m512i, + k: __mmask32, + a: __m512i, +) -> __m512i { + static_assert_imm_u8!(IMM8); let a = a.as_i16x32(); - macro_rules! call { - ($imm8:expr) => { - vpsraiw(a, $imm8) - }; - } - let shf = constify_imm8_sae!(imm8, call); + let shf = vpsraiw(a, IMM8); transmute(simd_select_bitmask(k, shf, src.as_i16x32())) } @@ -5799,16 +5669,12 @@ pub unsafe fn _mm512_mask_srai_epi16(src: __m512i, k: __mmask32, a: __m512i, imm /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_srai_epi16&expand=5426) #[inline] #[target_feature(enable = "avx512bw")] -#[cfg_attr(test, assert_instr(vpsraw, imm8 = 1))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm512_maskz_srai_epi16(k: __mmask32, a: __m512i, imm8: u32) -> __m512i { +#[cfg_attr(test, assert_instr(vpsraw, IMM8 = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_maskz_srai_epi16(k: __mmask32, a: __m512i) -> __m512i { + static_assert_imm_u8!(IMM8); let a = a.as_i16x32(); - macro_rules! call { - ($imm8:expr) => { - vpsraiw(a, $imm8) - }; - } - let shf = constify_imm8_sae!(imm8, call); + let shf = vpsraiw(a, IMM8); let zero = _mm512_setzero_si512().as_i16x32(); transmute(simd_select_bitmask(k, shf, zero)) } @@ -5818,16 +5684,17 @@ pub unsafe fn _mm512_maskz_srai_epi16(k: __mmask32, a: __m512i, imm8: u32) -> __ /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_srai_epi16&expand=5422) #[inline] #[target_feature(enable = "avx512bw,avx512vl")] -#[cfg_attr(test, assert_instr(vpsraw, imm8 = 1))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm256_mask_srai_epi16(src: __m256i, k: __mmask16, a: __m256i, imm8: u32) -> __m256i { - macro_rules! call { - ($imm8:expr) => { - _mm256_srai_epi16(a, $imm8) - }; - } - let shf = constify_imm8_sae!(imm8, call); - transmute(simd_select_bitmask(k, shf.as_i16x16(), src.as_i16x16())) +#[cfg_attr(test, assert_instr(vpsraw, IMM8 = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm256_mask_srai_epi16( + src: __m256i, + k: __mmask16, + a: __m256i, +) -> __m256i { + static_assert_imm_u8!(IMM8); + let imm8 = IMM8 as i32; + let r = psraiw256(a.as_i16x16(), imm8); + transmute(simd_select_bitmask(k, r, src.as_i16x16())) } /// Shift packed 16-bit integers in a right by imm8 while shifting in sign bits, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). @@ -5835,17 +5702,14 @@ pub unsafe fn _mm256_mask_srai_epi16(src: __m256i, k: __mmask16, a: __m256i, imm /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_srai_epi16&expand=5423) #[inline] #[target_feature(enable = "avx512bw,avx512vl")] -#[cfg_attr(test, assert_instr(vpsraw, imm8 = 1))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm256_maskz_srai_epi16(k: __mmask16, a: __m256i, imm8: u32) -> __m256i { - macro_rules! call { - ($imm8:expr) => { - _mm256_srai_epi16(a, $imm8) - }; - } - let shf = constify_imm8_sae!(imm8, call); +#[cfg_attr(test, assert_instr(vpsraw, IMM8 = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm256_maskz_srai_epi16(k: __mmask16, a: __m256i) -> __m256i { + static_assert_imm_u8!(IMM8); + let imm8 = IMM8 as i32; + let r = psraiw256(a.as_i16x16(), imm8); let zero = _mm256_setzero_si256().as_i16x16(); - transmute(simd_select_bitmask(k, shf.as_i16x16(), zero)) + transmute(simd_select_bitmask(k, r, zero)) } /// Shift packed 16-bit integers in a right by imm8 while shifting in sign bits, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). @@ -5853,16 +5717,17 @@ pub unsafe fn _mm256_maskz_srai_epi16(k: __mmask16, a: __m256i, imm8: u32) -> __ /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_srai_epi16&expand=5419) #[inline] #[target_feature(enable = "avx512bw,avx512vl")] -#[cfg_attr(test, assert_instr(vpsraw, imm8 = 1))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm_mask_srai_epi16(src: __m128i, k: __mmask8, a: __m128i, imm8: u32) -> __m128i { - macro_rules! call { - ($imm8:expr) => { - _mm_srai_epi16(a, $imm8) - }; - } - let shf = constify_imm8_sae!(imm8, call); - transmute(simd_select_bitmask(k, shf.as_i16x8(), src.as_i16x8())) +#[cfg_attr(test, assert_instr(vpsraw, IMM8 = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm_mask_srai_epi16( + src: __m128i, + k: __mmask8, + a: __m128i, +) -> __m128i { + static_assert_imm_u8!(IMM8); + let imm8 = IMM8 as i32; + let r = psraiw128(a.as_i16x8(), imm8); + transmute(simd_select_bitmask(k, r, src.as_i16x8())) } /// Shift packed 16-bit integers in a right by imm8 while shifting in sign bits, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). @@ -5870,17 +5735,14 @@ pub unsafe fn _mm_mask_srai_epi16(src: __m128i, k: __mmask8, a: __m128i, imm8: u /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_srai_epi16&expand=5420) #[inline] #[target_feature(enable = "avx512bw,avx512vl")] -#[cfg_attr(test, assert_instr(vpsraw, imm8 = 1))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm_maskz_srai_epi16(k: __mmask8, a: __m128i, imm8: u32) -> __m128i { - macro_rules! call { - ($imm8:expr) => { - _mm_srai_epi16(a, $imm8) - }; - } - let shf = constify_imm8_sae!(imm8, call); +#[cfg_attr(test, assert_instr(vpsraw, IMM8 = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm_maskz_srai_epi16(k: __mmask8, a: __m128i) -> __m128i { + static_assert_imm_u8!(IMM8); + let imm8 = IMM8 as i32; + let r = psraiw128(a.as_i16x8(), imm8); let zero = _mm_setzero_si128().as_i16x8(); - transmute(simd_select_bitmask(k, shf.as_i16x8(), zero)) + transmute(simd_select_bitmask(k, r, zero)) } /// Shift packed 16-bit integers in a right by the amount specified by the corresponding element in count while shifting in sign bits, and store the results in dst. @@ -6356,7 +6218,7 @@ pub unsafe fn _mm_mask_blend_epi8(k: __mmask16, a: __m128i, b: __m128i) -> __m12 #[cfg_attr(test, assert_instr(vpbroadcastw))] pub unsafe fn _mm512_broadcastw_epi16(a: __m128i) -> __m512i { let a = _mm512_castsi128_si512(a).as_i16x32(); - let ret: i16x32 = simd_shuffle32( + let ret: i16x32 = simd_shuffle32!( a, a, [ @@ -6444,7 +6306,7 @@ pub unsafe fn _mm_maskz_broadcastw_epi16(k: __mmask8, a: __m128i) -> __m128i { #[cfg_attr(test, assert_instr(vpbroadcastb))] pub unsafe fn _mm512_broadcastb_epi8(a: __m128i) -> __m512i { let a = _mm512_castsi128_si512(a).as_i8x64(); - let ret: i8x64 = simd_shuffle64( + let ret: i8x64 = simd_shuffle64!( a, a, [ @@ -6535,7 +6397,7 @@ pub unsafe fn _mm512_unpackhi_epi16(a: __m512i, b: __m512i) -> __m512i { let a = a.as_i16x32(); let b = b.as_i16x32(); #[rustfmt::skip] - let r: i16x32 = simd_shuffle32( + let r: i16x32 = simd_shuffle32!( a, b, [ @@ -6646,7 +6508,7 @@ pub unsafe fn _mm512_unpackhi_epi8(a: __m512i, b: __m512i) -> __m512i { let a = a.as_i8x64(); let b = b.as_i8x64(); #[rustfmt::skip] - let r: i8x64 = simd_shuffle64( + let r: i8x64 = simd_shuffle64!( a, b, [ @@ -6765,7 +6627,7 @@ pub unsafe fn _mm512_unpacklo_epi16(a: __m512i, b: __m512i) -> __m512i { let a = a.as_i16x32(); let b = b.as_i16x32(); #[rustfmt::skip] - let r: i16x32 = simd_shuffle32( + let r: i16x32 = simd_shuffle32!( a, b, [ @@ -6876,7 +6738,7 @@ pub unsafe fn _mm512_unpacklo_epi8(a: __m512i, b: __m512i) -> __m512i { let a = a.as_i8x64(); let b = b.as_i8x64(); #[rustfmt::skip] - let r: i8x64 = simd_shuffle64( + let r: i8x64 = simd_shuffle64!( a, b, [ @@ -7266,56 +7128,49 @@ pub unsafe fn _mm_maskz_set1_epi8(k: __mmask16, a: i8) -> __m128i { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_shufflelo_epi16&expand=5221) #[inline] #[target_feature(enable = "avx512bw")] -#[cfg_attr(test, assert_instr(vpshuflw, imm8 = 0))] -#[rustc_args_required_const(1)] -pub unsafe fn _mm512_shufflelo_epi16(a: __m512i, imm8: i32) -> __m512i { - let imm8 = (imm8 & 0xFF) as u8; +#[cfg_attr(test, assert_instr(vpshuflw, IMM8 = 0))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm512_shufflelo_epi16(a: __m512i) -> __m512i { + static_assert_imm8!(IMM8); let a = a.as_i16x32(); - macro_rules! shuffle_done { - ($x01: expr, $x23: expr, $x45: expr, $x67: expr) => { - #[rustfmt::skip] - simd_shuffle32(a, a, [ - 0+$x01, 0+$x23, 0+$x45, 0+$x67, 4, 5, 6, 7, 8+$x01, 8+$x23, 8+$x45, 8+$x67, 12, 13, 14, 15, - 16+$x01, 16+$x23, 16+$x45, 16+$x67, 20, 21, 22, 23, 24+$x01, 24+$x23, 24+$x45, 24+$x67, 28, 29, 30, 31, - ]) - }; - } - macro_rules! shuffle_x67 { - ($x01:expr, $x23:expr, $x45:expr) => { - match (imm8 >> 6) & 0b11 { - 0b00 => shuffle_done!($x01, $x23, $x45, 0), - 0b01 => shuffle_done!($x01, $x23, $x45, 1), - 0b10 => shuffle_done!($x01, $x23, $x45, 2), - _ => shuffle_done!($x01, $x23, $x45, 3), - } - }; - } - macro_rules! shuffle_x45 { - ($x01:expr, $x23:expr) => { - match (imm8 >> 4) & 0b11 { - 0b00 => shuffle_x67!($x01, $x23, 0), - 0b01 => shuffle_x67!($x01, $x23, 1), - 0b10 => shuffle_x67!($x01, $x23, 2), - _ => shuffle_x67!($x01, $x23, 3), - } - }; - } - macro_rules! shuffle_x23 { - ($x01:expr) => { - match (imm8 >> 2) & 0b11 { - 0b00 => shuffle_x45!($x01, 0), - 0b01 => shuffle_x45!($x01, 1), - 0b10 => shuffle_x45!($x01, 2), - _ => shuffle_x45!($x01, 3), - } - }; - } - let r: i16x32 = match imm8 & 0b11 { - 0b00 => shuffle_x23!(0), - 0b01 => shuffle_x23!(1), - 0b10 => shuffle_x23!(2), - _ => shuffle_x23!(3), - }; + let r: i16x32 = simd_shuffle32!( + a, + a, + [ + IMM8 as u32 & 0b11, + (IMM8 as u32 >> 2) & 0b11, + (IMM8 as u32 >> 4) & 0b11, + (IMM8 as u32 >> 6) & 0b11, + 4, + 5, + 6, + 7, + (IMM8 as u32 & 0b11) + 8, + ((IMM8 as u32 >> 2) & 0b11) + 8, + ((IMM8 as u32 >> 4) & 0b11) + 8, + ((IMM8 as u32 >> 6) & 0b11) + 8, + 12, + 13, + 14, + 15, + (IMM8 as u32 & 0b11) + 16, + ((IMM8 as u32 >> 2) & 0b11) + 16, + ((IMM8 as u32 >> 4) & 0b11) + 16, + ((IMM8 as u32 >> 6) & 0b11) + 16, + 20, + 21, + 22, + 23, + (IMM8 as u32 & 0b11) + 24, + ((IMM8 as u32 >> 2) & 0b11) + 24, + ((IMM8 as u32 >> 4) & 0b11) + 24, + ((IMM8 as u32 >> 6) & 0b11) + 24, + 28, + 29, + 30, + 31, + ], + ); transmute(r) } @@ -7324,20 +7179,15 @@ pub unsafe fn _mm512_shufflelo_epi16(a: __m512i, imm8: i32) -> __m512i { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_shufflelo_epi16&expand=5219) #[inline] #[target_feature(enable = "avx512bw")] -#[cfg_attr(test, assert_instr(vpshuflw, imm8 = 0))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm512_mask_shufflelo_epi16( +#[cfg_attr(test, assert_instr(vpshuflw, IMM8 = 0))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_mask_shufflelo_epi16( src: __m512i, k: __mmask32, a: __m512i, - imm8: i32, ) -> __m512i { - macro_rules! call { - ($imm8:expr) => { - _mm512_shufflelo_epi16(a, $imm8) - }; - } - let r = constify_imm8_sae!(imm8, call); + static_assert_imm8!(IMM8); + let r = _mm512_shufflelo_epi16::(a); transmute(simd_select_bitmask(k, r.as_i16x32(), src.as_i16x32())) } @@ -7346,15 +7196,11 @@ pub unsafe fn _mm512_mask_shufflelo_epi16( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_shufflelo_epi16&expand=5220) #[inline] #[target_feature(enable = "avx512bw")] -#[cfg_attr(test, assert_instr(vpshuflw, imm8 = 0))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm512_maskz_shufflelo_epi16(k: __mmask32, a: __m512i, imm8: i32) -> __m512i { - macro_rules! call { - ($imm8:expr) => { - _mm512_shufflelo_epi16(a, $imm8) - }; - } - let r = constify_imm8_sae!(imm8, call); +#[cfg_attr(test, assert_instr(vpshuflw, IMM8 = 0))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_maskz_shufflelo_epi16(k: __mmask32, a: __m512i) -> __m512i { + static_assert_imm8!(IMM8); + let r = _mm512_shufflelo_epi16::(a); let zero = _mm512_setzero_si512().as_i16x32(); transmute(simd_select_bitmask(k, r.as_i16x32(), zero)) } @@ -7364,20 +7210,15 @@ pub unsafe fn _mm512_maskz_shufflelo_epi16(k: __mmask32, a: __m512i, imm8: i32) /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_shufflelo_epi16&expand=5216) #[inline] #[target_feature(enable = "avx512bw,avx512vl")] -#[cfg_attr(test, assert_instr(vpshuflw, imm8 = 5))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm256_mask_shufflelo_epi16( +#[cfg_attr(test, assert_instr(vpshuflw, IMM8 = 5))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm256_mask_shufflelo_epi16( src: __m256i, k: __mmask16, a: __m256i, - imm8: i32, ) -> __m256i { - macro_rules! call { - ($imm8:expr) => { - _mm256_shufflelo_epi16(a, $imm8) - }; - } - let shuffle = constify_imm8_sae!(imm8, call); + static_assert_imm8!(IMM8); + let shuffle = _mm256_shufflelo_epi16::(a); transmute(simd_select_bitmask(k, shuffle.as_i16x16(), src.as_i16x16())) } @@ -7386,15 +7227,11 @@ pub unsafe fn _mm256_mask_shufflelo_epi16( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_shufflelo_epi16&expand=5217) #[inline] #[target_feature(enable = "avx512bw,avx512vl")] -#[cfg_attr(test, assert_instr(vpshuflw, imm8 = 5))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm256_maskz_shufflelo_epi16(k: __mmask16, a: __m256i, imm8: i32) -> __m256i { - macro_rules! call { - ($imm8:expr) => { - _mm256_shufflelo_epi16(a, $imm8) - }; - } - let shuffle = constify_imm8_sae!(imm8, call); +#[cfg_attr(test, assert_instr(vpshuflw, IMM8 = 5))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm256_maskz_shufflelo_epi16(k: __mmask16, a: __m256i) -> __m256i { + static_assert_imm8!(IMM8); + let shuffle = _mm256_shufflelo_epi16::(a); let zero = _mm256_setzero_si256().as_i16x16(); transmute(simd_select_bitmask(k, shuffle.as_i16x16(), zero)) } @@ -7404,20 +7241,15 @@ pub unsafe fn _mm256_maskz_shufflelo_epi16(k: __mmask16, a: __m256i, imm8: i32) /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_shufflelo_epi16&expand=5213) #[inline] #[target_feature(enable = "avx512bw,avx512vl")] -#[cfg_attr(test, assert_instr(vpshuflw, imm8 = 5))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm_mask_shufflelo_epi16( +#[cfg_attr(test, assert_instr(vpshuflw, IMM8 = 5))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm_mask_shufflelo_epi16( src: __m128i, k: __mmask8, a: __m128i, - imm8: i32, ) -> __m128i { - macro_rules! call { - ($imm8:expr) => { - _mm_shufflelo_epi16(a, $imm8) - }; - } - let shuffle = constify_imm8_sae!(imm8, call); + static_assert_imm8!(IMM8); + let shuffle = _mm_shufflelo_epi16::(a); transmute(simd_select_bitmask(k, shuffle.as_i16x8(), src.as_i16x8())) } @@ -7426,15 +7258,11 @@ pub unsafe fn _mm_mask_shufflelo_epi16( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_shufflelo_epi16&expand=5214) #[inline] #[target_feature(enable = "avx512bw,avx512vl")] -#[cfg_attr(test, assert_instr(vpshuflw, imm8 = 5))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm_maskz_shufflelo_epi16(k: __mmask8, a: __m128i, imm8: i32) -> __m128i { - macro_rules! call { - ($imm8:expr) => { - _mm_shufflelo_epi16(a, $imm8) - }; - } - let shuffle = constify_imm8_sae!(imm8, call); +#[cfg_attr(test, assert_instr(vpshuflw, IMM8 = 5))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm_maskz_shufflelo_epi16(k: __mmask8, a: __m128i) -> __m128i { + static_assert_imm8!(IMM8); + let shuffle = _mm_shufflelo_epi16::(a); let zero = _mm_setzero_si128().as_i16x8(); transmute(simd_select_bitmask(k, shuffle.as_i16x8(), zero)) } @@ -7444,56 +7272,49 @@ pub unsafe fn _mm_maskz_shufflelo_epi16(k: __mmask8, a: __m128i, imm8: i32) -> _ /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_shufflehi_epi16&expand=5212) #[inline] #[target_feature(enable = "avx512bw")] -#[cfg_attr(test, assert_instr(vpshufhw, imm8 = 0))] -#[rustc_args_required_const(1)] -pub unsafe fn _mm512_shufflehi_epi16(a: __m512i, imm8: i32) -> __m512i { - let imm8 = (imm8 & 0xFF) as u8; +#[cfg_attr(test, assert_instr(vpshufhw, IMM8 = 0))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm512_shufflehi_epi16(a: __m512i) -> __m512i { + static_assert_imm8!(IMM8); let a = a.as_i16x32(); - macro_rules! shuffle_done { - ($x01: expr, $x23: expr, $x45: expr, $x67: expr) => { - #[rustfmt::skip] - simd_shuffle32(a, a, [ - 0, 1, 2, 3, 4+$x01, 4+$x23, 4+$x45, 4+$x67, 8, 9, 10, 11, 12+$x01, 12+$x23, 12+$x45, 12+$x67, - 16, 17, 18, 19, 20+$x01, 20+$x23, 20+$x45, 20+$x67, 24, 25, 26, 27, 28+$x01, 28+$x23, 28+$x45, 28+$x67, - ]) - }; - } - macro_rules! shuffle_x67 { - ($x01:expr, $x23:expr, $x45:expr) => { - match (imm8 >> 6) & 0b11 { - 0b00 => shuffle_done!($x01, $x23, $x45, 0), - 0b01 => shuffle_done!($x01, $x23, $x45, 1), - 0b10 => shuffle_done!($x01, $x23, $x45, 2), - _ => shuffle_done!($x01, $x23, $x45, 3), - } - }; - } - macro_rules! shuffle_x45 { - ($x01:expr, $x23:expr) => { - match (imm8 >> 4) & 0b11 { - 0b00 => shuffle_x67!($x01, $x23, 0), - 0b01 => shuffle_x67!($x01, $x23, 1), - 0b10 => shuffle_x67!($x01, $x23, 2), - _ => shuffle_x67!($x01, $x23, 3), - } - }; - } - macro_rules! shuffle_x23 { - ($x01:expr) => { - match (imm8 >> 2) & 0b11 { - 0b00 => shuffle_x45!($x01, 0), - 0b01 => shuffle_x45!($x01, 1), - 0b10 => shuffle_x45!($x01, 2), - _ => shuffle_x45!($x01, 3), - } - }; - } - let r: i16x32 = match imm8 & 0b11 { - 0b00 => shuffle_x23!(0), - 0b01 => shuffle_x23!(1), - 0b10 => shuffle_x23!(2), - _ => shuffle_x23!(3), - }; + let r: i16x32 = simd_shuffle32!( + a, + a, + [ + 0, + 1, + 2, + 3, + (IMM8 as u32 & 0b11) + 4, + ((IMM8 as u32 >> 2) & 0b11) + 4, + ((IMM8 as u32 >> 4) & 0b11) + 4, + ((IMM8 as u32 >> 6) & 0b11) + 4, + 8, + 9, + 10, + 11, + (IMM8 as u32 & 0b11) + 12, + ((IMM8 as u32 >> 2) & 0b11) + 12, + ((IMM8 as u32 >> 4) & 0b11) + 12, + ((IMM8 as u32 >> 6) & 0b11) + 12, + 16, + 17, + 18, + 19, + (IMM8 as u32 & 0b11) + 20, + ((IMM8 as u32 >> 2) & 0b11) + 20, + ((IMM8 as u32 >> 4) & 0b11) + 20, + ((IMM8 as u32 >> 6) & 0b11) + 20, + 24, + 25, + 26, + 27, + (IMM8 as u32 & 0b11) + 28, + ((IMM8 as u32 >> 2) & 0b11) + 28, + ((IMM8 as u32 >> 4) & 0b11) + 28, + ((IMM8 as u32 >> 6) & 0b11) + 28, + ], + ); transmute(r) } @@ -7502,20 +7323,15 @@ pub unsafe fn _mm512_shufflehi_epi16(a: __m512i, imm8: i32) -> __m512i { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_shufflehi_epi16&expand=5210) #[inline] #[target_feature(enable = "avx512bw")] -#[cfg_attr(test, assert_instr(vpshufhw, imm8 = 0))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm512_mask_shufflehi_epi16( +#[cfg_attr(test, assert_instr(vpshufhw, IMM8 = 0))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_mask_shufflehi_epi16( src: __m512i, k: __mmask32, a: __m512i, - imm8: i32, ) -> __m512i { - macro_rules! call { - ($imm8:expr) => { - _mm512_shufflehi_epi16(a, $imm8) - }; - } - let r = constify_imm8_sae!(imm8, call); + static_assert_imm8!(IMM8); + let r = _mm512_shufflehi_epi16::(a); transmute(simd_select_bitmask(k, r.as_i16x32(), src.as_i16x32())) } @@ -7524,15 +7340,11 @@ pub unsafe fn _mm512_mask_shufflehi_epi16( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_shufflehi_epi16&expand=5211) #[inline] #[target_feature(enable = "avx512bw")] -#[cfg_attr(test, assert_instr(vpshufhw, imm8 = 0))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm512_maskz_shufflehi_epi16(k: __mmask32, a: __m512i, imm8: i32) -> __m512i { - macro_rules! call { - ($imm8:expr) => { - _mm512_shufflehi_epi16(a, $imm8) - }; - } - let r = constify_imm8_sae!(imm8, call); +#[cfg_attr(test, assert_instr(vpshufhw, IMM8 = 0))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_maskz_shufflehi_epi16(k: __mmask32, a: __m512i) -> __m512i { + static_assert_imm8!(IMM8); + let r = _mm512_shufflehi_epi16::(a); let zero = _mm512_setzero_si512().as_i16x32(); transmute(simd_select_bitmask(k, r.as_i16x32(), zero)) } @@ -7542,20 +7354,15 @@ pub unsafe fn _mm512_maskz_shufflehi_epi16(k: __mmask32, a: __m512i, imm8: i32) /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_shufflehi_epi16&expand=5207) #[inline] #[target_feature(enable = "avx512bw,avx512vl")] -#[cfg_attr(test, assert_instr(vpshufhw, imm8 = 5))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm256_mask_shufflehi_epi16( +#[cfg_attr(test, assert_instr(vpshufhw, IMM8 = 5))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm256_mask_shufflehi_epi16( src: __m256i, k: __mmask16, a: __m256i, - imm8: i32, ) -> __m256i { - macro_rules! call { - ($imm8:expr) => { - _mm256_shufflehi_epi16(a, $imm8) - }; - } - let shuffle = constify_imm8_sae!(imm8, call); + static_assert_imm8!(IMM8); + let shuffle = _mm256_shufflehi_epi16::(a); transmute(simd_select_bitmask(k, shuffle.as_i16x16(), src.as_i16x16())) } @@ -7564,15 +7371,11 @@ pub unsafe fn _mm256_mask_shufflehi_epi16( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_shufflehi_epi16&expand=5208) #[inline] #[target_feature(enable = "avx512bw,avx512vl")] -#[cfg_attr(test, assert_instr(vpshufhw, imm8 = 5))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm256_maskz_shufflehi_epi16(k: __mmask16, a: __m256i, imm8: i32) -> __m256i { - macro_rules! call { - ($imm8:expr) => { - _mm256_shufflehi_epi16(a, $imm8) - }; - } - let shuffle = constify_imm8_sae!(imm8, call); +#[cfg_attr(test, assert_instr(vpshufhw, IMM8 = 5))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm256_maskz_shufflehi_epi16(k: __mmask16, a: __m256i) -> __m256i { + static_assert_imm8!(IMM8); + let shuffle = _mm256_shufflehi_epi16::(a); let zero = _mm256_setzero_si256().as_i16x16(); transmute(simd_select_bitmask(k, shuffle.as_i16x16(), zero)) } @@ -7582,20 +7385,15 @@ pub unsafe fn _mm256_maskz_shufflehi_epi16(k: __mmask16, a: __m256i, imm8: i32) /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_shufflehi_epi16&expand=5204) #[inline] #[target_feature(enable = "avx512bw,avx512vl")] -#[cfg_attr(test, assert_instr(vpshufhw, imm8 = 5))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm_mask_shufflehi_epi16( +#[cfg_attr(test, assert_instr(vpshufhw, IMM8 = 5))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm_mask_shufflehi_epi16( src: __m128i, k: __mmask8, a: __m128i, - imm8: i32, ) -> __m128i { - macro_rules! call { - ($imm8:expr) => { - _mm_shufflehi_epi16(a, $imm8) - }; - } - let shuffle = constify_imm8_sae!(imm8, call); + static_assert_imm8!(IMM8); + let shuffle = _mm_shufflehi_epi16::(a); transmute(simd_select_bitmask(k, shuffle.as_i16x8(), src.as_i16x8())) } @@ -7604,15 +7402,11 @@ pub unsafe fn _mm_mask_shufflehi_epi16( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_shufflehi_epi16&expand=5205) #[inline] #[target_feature(enable = "avx512bw,avx512vl")] -#[cfg_attr(test, assert_instr(vpshufhw, imm8 = 5))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm_maskz_shufflehi_epi16(k: __mmask8, a: __m128i, imm8: i32) -> __m128i { - macro_rules! call { - ($imm8:expr) => { - _mm_shufflehi_epi16(a, $imm8) - }; - } - let shuffle = constify_imm8_sae!(imm8, call); +#[cfg_attr(test, assert_instr(vpshufhw, IMM8 = 5))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm_maskz_shufflehi_epi16(k: __mmask8, a: __m128i) -> __m128i { + static_assert_imm8!(IMM8); + let shuffle = _mm_shufflehi_epi16::(a); let zero = _mm_setzero_si128().as_i16x8(); transmute(simd_select_bitmask(k, shuffle.as_i16x8(), zero)) } @@ -8049,17 +7843,13 @@ pub unsafe fn _mm512_sad_epu8(a: __m512i, b: __m512i) -> __m512i { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_dbsad_epu8&expand=2114) #[inline] #[target_feature(enable = "avx512bw")] -#[rustc_args_required_const(2)] -#[cfg_attr(test, assert_instr(vdbpsadbw, imm8 = 0))] -pub unsafe fn _mm512_dbsad_epu8(a: __m512i, b: __m512i, imm8: i32) -> __m512i { +#[rustc_legacy_const_generics(2)] +#[cfg_attr(test, assert_instr(vdbpsadbw, IMM8 = 0))] +pub unsafe fn _mm512_dbsad_epu8(a: __m512i, b: __m512i) -> __m512i { + static_assert_imm8!(IMM8); let a = a.as_u8x64(); let b = b.as_u8x64(); - macro_rules! call { - ($imm8:expr) => { - vdbpsadbw(a, b, $imm8) - }; - } - let r = constify_imm8_sae!(imm8, call); + let r = vdbpsadbw(a, b, IMM8); transmute(r) } @@ -8068,23 +7858,18 @@ pub unsafe fn _mm512_dbsad_epu8(a: __m512i, b: __m512i, imm8: i32) -> __m512i { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_dbsad_epu8&expand=2115) #[inline] #[target_feature(enable = "avx512bw")] -#[rustc_args_required_const(4)] -#[cfg_attr(test, assert_instr(vdbpsadbw, imm8 = 0))] -pub unsafe fn _mm512_mask_dbsad_epu8( +#[rustc_legacy_const_generics(4)] +#[cfg_attr(test, assert_instr(vdbpsadbw, IMM8 = 0))] +pub unsafe fn _mm512_mask_dbsad_epu8( src: __m512i, k: __mmask32, a: __m512i, b: __m512i, - imm8: i32, ) -> __m512i { + static_assert_imm8!(IMM8); let a = a.as_u8x64(); let b = b.as_u8x64(); - macro_rules! call { - ($imm8:expr) => { - vdbpsadbw(a, b, $imm8) - }; - } - let r = constify_imm8_sae!(imm8, call); + let r = vdbpsadbw(a, b, IMM8); transmute(simd_select_bitmask(k, r, src.as_u16x32())) } @@ -8093,17 +7878,17 @@ pub unsafe fn _mm512_mask_dbsad_epu8( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_dbsad_epu8&expand=2116) #[inline] #[target_feature(enable = "avx512bw")] -#[rustc_args_required_const(3)] -#[cfg_attr(test, assert_instr(vdbpsadbw, imm8 = 0))] -pub unsafe fn _mm512_maskz_dbsad_epu8(k: __mmask32, a: __m512i, b: __m512i, imm8: i32) -> __m512i { +#[rustc_legacy_const_generics(3)] +#[cfg_attr(test, assert_instr(vdbpsadbw, IMM8 = 0))] +pub unsafe fn _mm512_maskz_dbsad_epu8( + k: __mmask32, + a: __m512i, + b: __m512i, +) -> __m512i { + static_assert_imm8!(IMM8); let a = a.as_u8x64(); let b = b.as_u8x64(); - macro_rules! call { - ($imm8:expr) => { - vdbpsadbw(a, b, $imm8) - }; - } - let r = constify_imm8_sae!(imm8, call); + let r = vdbpsadbw(a, b, IMM8); transmute(simd_select_bitmask( k, r, @@ -8116,17 +7901,13 @@ pub unsafe fn _mm512_maskz_dbsad_epu8(k: __mmask32, a: __m512i, b: __m512i, imm8 /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_dbsad_epu8&expand=2111) #[inline] #[target_feature(enable = "avx512bw,avx512vl")] -#[rustc_args_required_const(2)] -#[cfg_attr(test, assert_instr(vdbpsadbw, imm8 = 0))] -pub unsafe fn _mm256_dbsad_epu8(a: __m256i, b: __m256i, imm8: i32) -> __m256i { +#[rustc_legacy_const_generics(2)] +#[cfg_attr(test, assert_instr(vdbpsadbw, IMM8 = 0))] +pub unsafe fn _mm256_dbsad_epu8(a: __m256i, b: __m256i) -> __m256i { + static_assert_imm8!(IMM8); let a = a.as_u8x32(); let b = b.as_u8x32(); - macro_rules! call { - ($imm8:expr) => { - vdbpsadbw256(a, b, $imm8) - }; - } - let r = constify_imm8_sae!(imm8, call); + let r = vdbpsadbw256(a, b, IMM8); transmute(r) } @@ -8135,23 +7916,18 @@ pub unsafe fn _mm256_dbsad_epu8(a: __m256i, b: __m256i, imm8: i32) -> __m256i { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_dbsad_epu8&expand=2112) #[inline] #[target_feature(enable = "avx512bw,avx512vl")] -#[rustc_args_required_const(4)] -#[cfg_attr(test, assert_instr(vdbpsadbw, imm8 = 0))] -pub unsafe fn _mm256_mask_dbsad_epu8( +#[rustc_legacy_const_generics(4)] +#[cfg_attr(test, assert_instr(vdbpsadbw, IMM8 = 0))] +pub unsafe fn _mm256_mask_dbsad_epu8( src: __m256i, k: __mmask16, a: __m256i, b: __m256i, - imm8: i32, ) -> __m256i { + static_assert_imm8!(IMM8); let a = a.as_u8x32(); let b = b.as_u8x32(); - macro_rules! call { - ($imm8:expr) => { - vdbpsadbw256(a, b, $imm8) - }; - } - let r = constify_imm8_sae!(imm8, call); + let r = vdbpsadbw256(a, b, IMM8); transmute(simd_select_bitmask(k, r, src.as_u16x16())) } @@ -8160,17 +7936,17 @@ pub unsafe fn _mm256_mask_dbsad_epu8( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_dbsad_epu8&expand=2113) #[inline] #[target_feature(enable = "avx512bw,avx512vl")] -#[rustc_args_required_const(3)] -#[cfg_attr(test, assert_instr(vdbpsadbw, imm8 = 0))] -pub unsafe fn _mm256_maskz_dbsad_epu8(k: __mmask16, a: __m256i, b: __m256i, imm8: i32) -> __m256i { +#[rustc_legacy_const_generics(3)] +#[cfg_attr(test, assert_instr(vdbpsadbw, IMM8 = 0))] +pub unsafe fn _mm256_maskz_dbsad_epu8( + k: __mmask16, + a: __m256i, + b: __m256i, +) -> __m256i { + static_assert_imm8!(IMM8); let a = a.as_u8x32(); let b = b.as_u8x32(); - macro_rules! call { - ($imm8:expr) => { - vdbpsadbw256(a, b, $imm8) - }; - } - let r = constify_imm8_sae!(imm8, call); + let r = vdbpsadbw256(a, b, IMM8); transmute(simd_select_bitmask( k, r, @@ -8183,17 +7959,13 @@ pub unsafe fn _mm256_maskz_dbsad_epu8(k: __mmask16, a: __m256i, b: __m256i, imm8 /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_dbsad_epu8&expand=2108) #[inline] #[target_feature(enable = "avx512bw,avx512vl")] -#[rustc_args_required_const(2)] -#[cfg_attr(test, assert_instr(vdbpsadbw, imm8 = 0))] -pub unsafe fn _mm_dbsad_epu8(a: __m128i, b: __m128i, imm8: i32) -> __m128i { +#[rustc_legacy_const_generics(2)] +#[cfg_attr(test, assert_instr(vdbpsadbw, IMM8 = 0))] +pub unsafe fn _mm_dbsad_epu8(a: __m128i, b: __m128i) -> __m128i { + static_assert_imm8!(IMM8); let a = a.as_u8x16(); let b = b.as_u8x16(); - macro_rules! call { - ($imm8:expr) => { - vdbpsadbw128(a, b, $imm8) - }; - } - let r = constify_imm8_sae!(imm8, call); + let r = vdbpsadbw128(a, b, IMM8); transmute(r) } @@ -8202,23 +7974,18 @@ pub unsafe fn _mm_dbsad_epu8(a: __m128i, b: __m128i, imm8: i32) -> __m128i { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_dbsad_epu8&expand=2109) #[inline] #[target_feature(enable = "avx512bw,avx512vl")] -#[rustc_args_required_const(4)] -#[cfg_attr(test, assert_instr(vdbpsadbw, imm8 = 0))] -pub unsafe fn _mm_mask_dbsad_epu8( +#[rustc_legacy_const_generics(4)] +#[cfg_attr(test, assert_instr(vdbpsadbw, IMM8 = 0))] +pub unsafe fn _mm_mask_dbsad_epu8( src: __m128i, k: __mmask8, a: __m128i, b: __m128i, - imm8: i32, ) -> __m128i { + static_assert_imm8!(IMM8); let a = a.as_u8x16(); let b = b.as_u8x16(); - macro_rules! call { - ($imm8:expr) => { - vdbpsadbw128(a, b, $imm8) - }; - } - let r = constify_imm8_sae!(imm8, call); + let r = vdbpsadbw128(a, b, IMM8); transmute(simd_select_bitmask(k, r, src.as_u16x8())) } @@ -8227,17 +7994,17 @@ pub unsafe fn _mm_mask_dbsad_epu8( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_dbsad_epu8&expand=2110) #[inline] #[target_feature(enable = "avx512bw,avx512vl")] -#[rustc_args_required_const(3)] -#[cfg_attr(test, assert_instr(vdbpsadbw, imm8 = 0))] -pub unsafe fn _mm_maskz_dbsad_epu8(k: __mmask8, a: __m128i, b: __m128i, imm8: i32) -> __m128i { +#[rustc_legacy_const_generics(3)] +#[cfg_attr(test, assert_instr(vdbpsadbw, IMM8 = 0))] +pub unsafe fn _mm_maskz_dbsad_epu8( + k: __mmask8, + a: __m128i, + b: __m128i, +) -> __m128i { + static_assert_imm8!(IMM8); let a = a.as_u8x16(); let b = b.as_u8x16(); - macro_rules! call { - ($imm8:expr) => { - vdbpsadbw128(a, b, $imm8) - }; - } - let r = constify_imm8_sae!(imm8, call); + let r = vdbpsadbw128(a, b, IMM8); transmute(simd_select_bitmask(k, r, _mm_setzero_si128().as_u16x8())) } @@ -8666,7 +8433,7 @@ pub unsafe fn _mm256_maskz_cvtepi16_epi8(k: __mmask16, a: __m256i) -> __m128i { pub unsafe fn _mm_cvtepi16_epi8(a: __m128i) -> __m128i { let a = a.as_i16x8(); let zero = _mm_setzero_si128().as_i16x8(); - let v256: i16x16 = simd_shuffle16(a, zero, [0, 1, 2, 3, 4, 5, 6, 7, 8, 8, 8, 8, 8, 8, 8, 8]); + let v256: i16x16 = simd_shuffle16!(a, zero, [0, 1, 2, 3, 4, 5, 6, 7, 8, 8, 8, 8, 8, 8, 8, 8]); transmute::(simd_cast(v256)) } @@ -9102,49 +8869,90 @@ pub unsafe fn _mm_maskz_cvtepu8_epi16(k: __mmask8, a: __m128i) -> __m128i { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_bslli_epi128&expand=591) #[inline] #[target_feature(enable = "avx512bw")] -#[rustc_args_required_const(1)] -#[cfg_attr(test, assert_instr(vpslldq, imm8 = 3))] -pub unsafe fn _mm512_bslli_epi128(a: __m512i, imm8: i32) -> __m512i { +#[cfg_attr(test, assert_instr(vpslldq, IMM8 = 3))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm512_bslli_epi128(a: __m512i) -> __m512i { + static_assert_imm8!(IMM8); + const fn mask(shift: i32, i: u32) -> u32 { + let shift = shift as u32 & 0xff; + if shift > 15 || i % 16 < shift { + 0 + } else { + 64 + (i - shift) + } + } let a = a.as_i8x64(); let zero = _mm512_setzero_si512().as_i8x64(); - #[rustfmt::skip] - macro_rules! call { - ($imm8:expr) => { - simd_shuffle64 ( - zero, - a, - [ - 64 - $imm8, 65 - $imm8, 66 - $imm8, 67 - $imm8, 68 - $imm8, 69 - $imm8, 70 - $imm8, 71 - $imm8, - 72 - $imm8, 73 - $imm8, 74 - $imm8, 75 - $imm8, 76 - $imm8, 77 - $imm8, 78 - $imm8, 79 - $imm8, - 80 - ($imm8+16), 81 - ($imm8+16), 82 - ($imm8+16), 83 - ($imm8+16), 84 - ($imm8+16), 85 - ($imm8+16), 86 - ($imm8+16), 87 - ($imm8+16), - 88 - ($imm8+16), 89 - ($imm8+16), 90 - ($imm8+16), 91 - ($imm8+16), 92 - ($imm8+16), 93 - ($imm8+16), 94 - ($imm8+16), 95 - ($imm8+16), - 96 - ($imm8+32), 97 - ($imm8+32), 98 - ($imm8+32), 99 - ($imm8+32), 100 - ($imm8+32), 101 - ($imm8+32), 102 - ($imm8+32), 103 - ($imm8+32), - 104 - ($imm8+32), 105 - ($imm8+32), 106 - ($imm8+32), 107 - ($imm8+32), 108 - ($imm8+32), 109 - ($imm8+32), 110 - ($imm8+32), 111 - ($imm8+32), - 112 - ($imm8+48), 113 - ($imm8+48), 114 - ($imm8+48), 115 - ($imm8+48), 116 - ($imm8+48), 117 - ($imm8+48), 118 - ($imm8+48), 119 - ($imm8+48), - 120 - ($imm8+48), 121 - ($imm8+48), 122 - ($imm8+48), 123 - ($imm8+48), 124 - ($imm8+48), 125 - ($imm8+48), 126 - ($imm8+48), 127 - ($imm8+48), - ], - ) - }; - } - let r: i8x64 = match imm8 { - 0 => call!(0), - 1 => call!(1), - 2 => call!(2), - 3 => call!(3), - 4 => call!(4), - 5 => call!(5), - 6 => call!(6), - 7 => call!(7), - 8 => call!(8), - 9 => call!(9), - 10 => call!(10), - 11 => call!(11), - 12 => call!(12), - 13 => call!(13), - 14 => call!(14), - 15 => call!(15), - _ => call!(16), - }; + let r: i8x64 = simd_shuffle64!( + zero, + a, + [ + mask(IMM8, 0), + mask(IMM8, 1), + mask(IMM8, 2), + mask(IMM8, 3), + mask(IMM8, 4), + mask(IMM8, 5), + mask(IMM8, 6), + mask(IMM8, 7), + mask(IMM8, 8), + mask(IMM8, 9), + mask(IMM8, 10), + mask(IMM8, 11), + mask(IMM8, 12), + mask(IMM8, 13), + mask(IMM8, 14), + mask(IMM8, 15), + mask(IMM8, 16), + mask(IMM8, 17), + mask(IMM8, 18), + mask(IMM8, 19), + mask(IMM8, 20), + mask(IMM8, 21), + mask(IMM8, 22), + mask(IMM8, 23), + mask(IMM8, 24), + mask(IMM8, 25), + mask(IMM8, 26), + mask(IMM8, 27), + mask(IMM8, 28), + mask(IMM8, 29), + mask(IMM8, 30), + mask(IMM8, 31), + mask(IMM8, 32), + mask(IMM8, 33), + mask(IMM8, 34), + mask(IMM8, 35), + mask(IMM8, 36), + mask(IMM8, 37), + mask(IMM8, 38), + mask(IMM8, 39), + mask(IMM8, 40), + mask(IMM8, 41), + mask(IMM8, 42), + mask(IMM8, 43), + mask(IMM8, 44), + mask(IMM8, 45), + mask(IMM8, 46), + mask(IMM8, 47), + mask(IMM8, 48), + mask(IMM8, 49), + mask(IMM8, 50), + mask(IMM8, 51), + mask(IMM8, 52), + mask(IMM8, 53), + mask(IMM8, 54), + mask(IMM8, 55), + mask(IMM8, 56), + mask(IMM8, 57), + mask(IMM8, 58), + mask(IMM8, 59), + mask(IMM8, 60), + mask(IMM8, 61), + mask(IMM8, 62), + mask(IMM8, 63), + ], + ); transmute(r) } @@ -9153,48 +8961,171 @@ pub unsafe fn _mm512_bslli_epi128(a: __m512i, imm8: i32) -> __m512i { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_bsrli_epi128&expand=594) #[inline] #[target_feature(enable = "avx512bw")] -#[rustc_args_required_const(1)] -#[cfg_attr(test, assert_instr(vpsrldq, imm8 = 3))] -pub unsafe fn _mm512_bsrli_epi128(a: __m512i, imm8: i32) -> __m512i { +#[cfg_attr(test, assert_instr(vpsrldq, IMM8 = 3))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm512_bsrli_epi128(a: __m512i) -> __m512i { + static_assert_imm8!(IMM8); let a = a.as_i8x64(); let zero = _mm512_setzero_si512().as_i8x64(); - #[rustfmt::skip] - macro_rules! call { - ($imm8:expr) => { - simd_shuffle64 ( - a, - zero, - [ - 0 + ($imm8+48), 1 + ($imm8+48), 2 + ($imm8+48), 3 + ($imm8+48), 4 + ($imm8+48), 5 + ($imm8+48), 6 + ($imm8+48), 7 + ($imm8+48), - 8 + ($imm8+48), 9 + ($imm8+48), 10 + ($imm8+48), 11 + ($imm8+48), 12 + ($imm8+48), 13 + ($imm8+48), 14 + ($imm8+48), 15 + ($imm8+48), - 16 + ($imm8+32), 17 + ($imm8+32), 18 + ($imm8+32), 19 + ($imm8+32), 20 + ($imm8+32), 21 + ($imm8+32), 22 + ($imm8+32), 23 + ($imm8+32), - 24 + ($imm8+32), 25 + ($imm8+32), 26 + ($imm8+32), 27 + ($imm8+32), 28 + ($imm8+32), 29 + ($imm8+32), 30 + ($imm8+32), 31 + ($imm8+32), - 32 + ($imm8+16), 33 + ($imm8+16), 34 + ($imm8+16), 35 + ($imm8+16), 36 + ($imm8+16), 37 + ($imm8+16), 38 + ($imm8+16), 39 + ($imm8+16), - 40 + ($imm8+16), 41 + ($imm8+16), 42 + ($imm8+16), 43 + ($imm8+16), 44 + ($imm8+16), 45 + ($imm8+16), 46 + ($imm8+16), 47 + ($imm8+16), - 48 + $imm8, 49 + $imm8, 50 + $imm8, 51 + $imm8, 52 + $imm8, 53 + $imm8, 54 + $imm8, 55 + $imm8, - 56 + $imm8, 57 + $imm8, 58 + $imm8, 59 + $imm8, 60 + $imm8, 61 + $imm8, 62 + $imm8, 63 + $imm8, - ], - ) - }; - } - let r: i8x64 = match imm8 { - 0 => call!(0), - 1 => call!(1), - 2 => call!(2), - 3 => call!(3), - 4 => call!(4), - 5 => call!(5), - 6 => call!(6), - 7 => call!(7), - 8 => call!(8), - 9 => call!(9), - 10 => call!(10), - 11 => call!(11), - 12 => call!(12), - 13 => call!(13), - 14 => call!(14), - 15 => call!(15), - _ => call!(16), + let r: i8x64 = match IMM8 % 16 { + 0 => simd_shuffle64!( + a, + zero, + [ + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, + 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, + 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, + ], + ), + 1 => simd_shuffle64!( + a, + zero, + [ + 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 64, 17, 18, 19, 20, 21, 22, 23, + 24, 25, 26, 27, 28, 29, 30, 31, 80, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, + 45, 46, 47, 96, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 112, + ], + ), + 2 => simd_shuffle64!( + a, + zero, + [ + 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 64, 65, 18, 19, 20, 21, 22, 23, 24, + 25, 26, 27, 28, 29, 30, 31, 80, 81, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, + 46, 47, 96, 97, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 112, 113, + ], + ), + 3 => simd_shuffle64!( + a, + zero, + [ + 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 64, 65, 66, 19, 20, 21, 22, 23, 24, + 25, 26, 27, 28, 29, 30, 31, 80, 81, 82, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, + 46, 47, 96, 97, 98, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 112, 113, + 114, + ], + ), + 4 => simd_shuffle64!( + a, + zero, + [ + 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 64, 65, 66, 67, 20, 21, 22, 23, 24, 25, + 26, 27, 28, 29, 30, 31, 80, 81, 82, 83, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, + 47, 96, 97, 98, 99, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 112, 113, 114, + 115, + ], + ), + 5 => simd_shuffle64!( + a, + zero, + [ + 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 64, 65, 66, 67, 68, 21, 22, 23, 24, 25, 26, + 27, 28, 29, 30, 31, 80, 81, 82, 83, 84, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, + 96, 97, 98, 99, 100, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 112, 113, 114, + 115, 116, + ], + ), + 6 => simd_shuffle64!( + a, + zero, + [ + 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 64, 65, 66, 67, 68, 69, 22, 23, 24, 25, 26, 27, + 28, 29, 30, 31, 80, 81, 82, 83, 84, 85, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 96, + 97, 98, 99, 100, 101, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 112, 113, 114, 115, + 116, 117, + ], + ), + 7 => simd_shuffle64!( + a, + zero, + [ + 7, 8, 9, 10, 11, 12, 13, 14, 15, 64, 65, 66, 67, 68, 69, 70, 23, 24, 25, 26, 27, + 28, 29, 30, 31, 80, 81, 82, 83, 84, 85, 86, 39, 40, 41, 42, 43, 44, 45, 46, 47, 96, + 97, 98, 99, 100, 101, 102, 55, 56, 57, 58, 59, 60, 61, 62, 63, 112, 113, 114, 115, + 116, 117, 118, + ], + ), + 8 => simd_shuffle64!( + a, + zero, + [ + 8, 9, 10, 11, 12, 13, 14, 15, 64, 65, 66, 67, 68, 69, 70, 71, 24, 25, 26, 27, 28, + 29, 30, 31, 80, 81, 82, 83, 84, 85, 86, 87, 40, 41, 42, 43, 44, 45, 46, 47, 96, 97, + 98, 99, 100, 101, 102, 103, 56, 57, 58, 59, 60, 61, 62, 63, 112, 113, 114, 115, + 116, 117, 118, 119, + ], + ), + 9 => simd_shuffle64!( + a, + zero, + [ + 9, 10, 11, 12, 13, 14, 15, 64, 65, 66, 67, 68, 69, 70, 71, 72, 25, 26, 27, 28, 29, + 30, 31, 80, 81, 82, 83, 84, 85, 86, 87, 88, 41, 42, 43, 44, 45, 46, 47, 96, 97, 98, + 99, 100, 101, 102, 103, 104, 57, 58, 59, 60, 61, 62, 63, 112, 113, 114, 115, 116, + 117, 118, 119, 120, + ], + ), + 10 => simd_shuffle64!( + a, + zero, + [ + 10, 11, 12, 13, 14, 15, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 26, 27, 28, 29, 30, + 31, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 42, 43, 44, 45, 46, 47, 96, 97, 98, 99, + 100, 101, 102, 103, 104, 105, 58, 59, 60, 61, 62, 63, 112, 113, 114, 115, 116, 117, + 118, 119, 120, 121, + ], + ), + 11 => simd_shuffle64!( + a, + zero, + [ + 11, 12, 13, 14, 15, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 27, 28, 29, 30, 31, + 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 43, 44, 45, 46, 47, 96, 97, 98, 99, + 100, 101, 102, 103, 104, 105, 106, 59, 60, 61, 62, 63, 112, 113, 114, 115, 116, + 117, 118, 119, 120, 121, 122, + ], + ), + 12 => simd_shuffle64!( + a, + zero, + [ + 12, 13, 14, 15, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 28, 29, 30, 31, 80, + 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 44, 45, 46, 47, 96, 97, 98, 99, 100, + 101, 102, 103, 104, 105, 106, 107, 60, 61, 62, 63, 112, 113, 114, 115, 116, 117, + 118, 119, 120, 121, 122, 123, + ], + ), + 13 => simd_shuffle64!( + a, + zero, + [ + 13, 14, 15, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 29, 30, 31, 80, 81, + 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 45, 46, 47, 96, 97, 98, 99, 100, 101, + 102, 103, 104, 105, 106, 107, 108, 61, 62, 63, 112, 113, 114, 115, 116, 117, 118, + 119, 120, 121, 122, 123, 124, + ], + ), + 14 => simd_shuffle64!( + a, + zero, + [ + 14, 15, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 30, 31, 80, 81, 82, + 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 46, 47, 96, 97, 98, 99, 100, 101, 102, + 103, 104, 105, 106, 107, 108, 109, 62, 63, 112, 113, 114, 115, 116, 117, 118, 119, + 120, 121, 122, 123, 124, 125, + ], + ), + 15 => simd_shuffle64!( + a, + zero, + [ + 15, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 31, 80, 81, 82, 83, + 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 47, 96, 97, 98, 99, 100, 101, 102, 103, + 104, 105, 106, 107, 108, 109, 110, 63, 112, 113, 114, 115, 116, 117, 118, 119, 120, + 121, 122, 123, 124, 125, 126, + ], + ), + _ => zero, }; transmute(r) } @@ -9204,60 +9135,183 @@ pub unsafe fn _mm512_bsrli_epi128(a: __m512i, imm8: i32) -> __m512i { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_alignr_epi8&expand=263) #[inline] #[target_feature(enable = "avx512bw")] -#[cfg_attr(test, assert_instr(vpalignr, imm8 = 1))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm512_alignr_epi8(a: __m512i, b: __m512i, imm8: i32) -> __m512i { +#[cfg_attr(test, assert_instr(vpalignr, IMM8 = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_alignr_epi8(a: __m512i, b: __m512i) -> __m512i { // If palignr is shifting the pair of vectors more than the size of two // lanes, emit zero. - if imm8 > 32 { + if IMM8 > 32 { return _mm512_set1_epi8(0); } // If palignr is shifting the pair of input vectors more than one lane, // but less than two lanes, convert to shifting in zeroes. - let (a, b, imm8) = if imm8 > 16 { - (_mm512_set1_epi8(0), a, imm8 - 16) + let (a, b) = if IMM8 > 16 { + (_mm512_set1_epi8(0), a) } else { - (a, b, imm8) + (a, b) }; let a = a.as_i8x64(); let b = b.as_i8x64(); - #[rustfmt::skip] - macro_rules! shuffle { - ($imm8:expr) => { - simd_shuffle64( - b, - a, - [ - 0 + ($imm8+48), 1 + ($imm8+48), 2 + ($imm8+48), 3 + ($imm8+48), 4 + ($imm8+48), 5 + ($imm8+48), 6 + ($imm8+48), 7 + ($imm8+48), - 8 + ($imm8+48), 9 + ($imm8+48), 10 + ($imm8+48), 11 + ($imm8+48), 12 + ($imm8+48), 13 + ($imm8+48), 14 + ($imm8+48), 15 + ($imm8+48), - 16 + ($imm8+32), 17 + ($imm8+32), 18 + ($imm8+32), 19 + ($imm8+32), 20 + ($imm8+32), 21 + ($imm8+32), 22 + ($imm8+32), 23 + ($imm8+32), - 24 + ($imm8+32), 25 + ($imm8+32), 26 + ($imm8+32), 27 + ($imm8+32), 28 + ($imm8+32), 29 + ($imm8+32), 30 + ($imm8+32), 31 + ($imm8+32), - 32 + ($imm8+16), 33 + ($imm8+16), 34 + ($imm8+16), 35 + ($imm8+16), 36 + ($imm8+16), 37 + ($imm8+16), 38 + ($imm8+16), 39 + ($imm8+16), - 40 + ($imm8+16), 41 + ($imm8+16), 42 + ($imm8+16), 43 + ($imm8+16), 44 + ($imm8+16), 45 + ($imm8+16), 46 + ($imm8+16), 47 + ($imm8+16), - 48 + $imm8, 49 + $imm8, 50 + $imm8, 51 + $imm8, 52 + $imm8, 53 + $imm8, 54 + $imm8, 55 + $imm8, - 56 + $imm8, 57 + $imm8, 58 + $imm8, 59 + $imm8, 60 + $imm8, 61 + $imm8, 62 + $imm8, 63 + $imm8, - ], - ) - }; - } - let r: i8x64 = match imm8 { - 0 => shuffle!(0), - 1 => shuffle!(1), - 2 => shuffle!(2), - 3 => shuffle!(3), - 4 => shuffle!(4), - 5 => shuffle!(5), - 6 => shuffle!(6), - 7 => shuffle!(7), - 8 => shuffle!(8), - 9 => shuffle!(9), - 10 => shuffle!(10), - 11 => shuffle!(11), - 12 => shuffle!(12), - 13 => shuffle!(13), - 14 => shuffle!(14), - 15 => shuffle!(15), - _ => shuffle!(16), + + let r: i8x64 = match IMM8 % 16 { + 0 => simd_shuffle64!( + b, + a, + [ + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, + 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, + 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, + ], + ), + 1 => simd_shuffle64!( + b, + a, + [ + 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 64, 17, 18, 19, 20, 21, 22, 23, + 24, 25, 26, 27, 28, 29, 30, 31, 80, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, + 45, 46, 47, 96, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 112, + ], + ), + 2 => simd_shuffle64!( + b, + a, + [ + 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 64, 65, 18, 19, 20, 21, 22, 23, 24, + 25, 26, 27, 28, 29, 30, 31, 80, 81, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, + 46, 47, 96, 97, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 112, 113, + ], + ), + 3 => simd_shuffle64!( + b, + a, + [ + 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 64, 65, 66, 19, 20, 21, 22, 23, 24, + 25, 26, 27, 28, 29, 30, 31, 80, 81, 82, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, + 46, 47, 96, 97, 98, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 112, 113, + 114, + ], + ), + 4 => simd_shuffle64!( + b, + a, + [ + 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 64, 65, 66, 67, 20, 21, 22, 23, 24, 25, + 26, 27, 28, 29, 30, 31, 80, 81, 82, 83, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, + 47, 96, 97, 98, 99, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 112, 113, 114, + 115, + ], + ), + 5 => simd_shuffle64!( + b, + a, + [ + 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 64, 65, 66, 67, 68, 21, 22, 23, 24, 25, 26, + 27, 28, 29, 30, 31, 80, 81, 82, 83, 84, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, + 96, 97, 98, 99, 100, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 112, 113, 114, + 115, 116, + ], + ), + 6 => simd_shuffle64!( + b, + a, + [ + 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 64, 65, 66, 67, 68, 69, 22, 23, 24, 25, 26, 27, + 28, 29, 30, 31, 80, 81, 82, 83, 84, 85, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 96, + 97, 98, 99, 100, 101, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 112, 113, 114, 115, + 116, 117, + ], + ), + 7 => simd_shuffle64!( + b, + a, + [ + 7, 8, 9, 10, 11, 12, 13, 14, 15, 64, 65, 66, 67, 68, 69, 70, 23, 24, 25, 26, 27, + 28, 29, 30, 31, 80, 81, 82, 83, 84, 85, 86, 39, 40, 41, 42, 43, 44, 45, 46, 47, 96, + 97, 98, 99, 100, 101, 102, 55, 56, 57, 58, 59, 60, 61, 62, 63, 112, 113, 114, 115, + 116, 117, 118, + ], + ), + 8 => simd_shuffle64!( + b, + a, + [ + 8, 9, 10, 11, 12, 13, 14, 15, 64, 65, 66, 67, 68, 69, 70, 71, 24, 25, 26, 27, 28, + 29, 30, 31, 80, 81, 82, 83, 84, 85, 86, 87, 40, 41, 42, 43, 44, 45, 46, 47, 96, 97, + 98, 99, 100, 101, 102, 103, 56, 57, 58, 59, 60, 61, 62, 63, 112, 113, 114, 115, + 116, 117, 118, 119, + ], + ), + 9 => simd_shuffle64!( + b, + a, + [ + 9, 10, 11, 12, 13, 14, 15, 64, 65, 66, 67, 68, 69, 70, 71, 72, 25, 26, 27, 28, 29, + 30, 31, 80, 81, 82, 83, 84, 85, 86, 87, 88, 41, 42, 43, 44, 45, 46, 47, 96, 97, 98, + 99, 100, 101, 102, 103, 104, 57, 58, 59, 60, 61, 62, 63, 112, 113, 114, 115, 116, + 117, 118, 119, 120, + ], + ), + 10 => simd_shuffle64!( + b, + a, + [ + 10, 11, 12, 13, 14, 15, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 26, 27, 28, 29, 30, + 31, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 42, 43, 44, 45, 46, 47, 96, 97, 98, 99, + 100, 101, 102, 103, 104, 105, 58, 59, 60, 61, 62, 63, 112, 113, 114, 115, 116, 117, + 118, 119, 120, 121, + ], + ), + 11 => simd_shuffle64!( + b, + a, + [ + 11, 12, 13, 14, 15, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 27, 28, 29, 30, 31, + 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 43, 44, 45, 46, 47, 96, 97, 98, 99, + 100, 101, 102, 103, 104, 105, 106, 59, 60, 61, 62, 63, 112, 113, 114, 115, 116, + 117, 118, 119, 120, 121, 122, + ], + ), + 12 => simd_shuffle64!( + b, + a, + [ + 12, 13, 14, 15, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 28, 29, 30, 31, 80, + 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 44, 45, 46, 47, 96, 97, 98, 99, 100, + 101, 102, 103, 104, 105, 106, 107, 60, 61, 62, 63, 112, 113, 114, 115, 116, 117, + 118, 119, 120, 121, 122, 123, + ], + ), + 13 => simd_shuffle64!( + b, + a, + [ + 13, 14, 15, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 29, 30, 31, 80, 81, + 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 45, 46, 47, 96, 97, 98, 99, 100, 101, + 102, 103, 104, 105, 106, 107, 108, 61, 62, 63, 112, 113, 114, 115, 116, 117, 118, + 119, 120, 121, 122, 123, 124, + ], + ), + 14 => simd_shuffle64!( + b, + a, + [ + 14, 15, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 30, 31, 80, 81, 82, + 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 46, 47, 96, 97, 98, 99, 100, 101, 102, + 103, 104, 105, 106, 107, 108, 109, 62, 63, 112, 113, 114, 115, 116, 117, 118, 119, + 120, 121, 122, 123, 124, 125, + ], + ), + 15 => simd_shuffle64!( + b, + a, + [ + 15, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 31, 80, 81, 82, 83, + 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 47, 96, 97, 98, 99, 100, 101, 102, 103, + 104, 105, 106, 107, 108, 109, 110, 63, 112, 113, 114, 115, 116, 117, 118, 119, 120, + 121, 122, 123, 124, 125, 126, + ], + ), + _ => b, }; transmute(r) } @@ -9267,21 +9321,16 @@ pub unsafe fn _mm512_alignr_epi8(a: __m512i, b: __m512i, imm8: i32) -> __m512i { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_alignr_epi8&expand=264) #[inline] #[target_feature(enable = "avx512bw")] -#[cfg_attr(test, assert_instr(vpalignr, imm8 = 1))] -#[rustc_args_required_const(4)] -pub unsafe fn _mm512_mask_alignr_epi8( +#[cfg_attr(test, assert_instr(vpalignr, IMM8 = 1))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_mask_alignr_epi8( src: __m512i, k: __mmask64, a: __m512i, b: __m512i, - imm8: i32, ) -> __m512i { - macro_rules! call { - ($imm8:expr) => { - _mm512_alignr_epi8(a, b, $imm8) - }; - } - let r = constify_imm8_sae!(imm8, call); + static_assert_imm8!(IMM8); + let r = _mm512_alignr_epi8::(a, b); transmute(simd_select_bitmask(k, r.as_i8x64(), src.as_i8x64())) } @@ -9290,15 +9339,15 @@ pub unsafe fn _mm512_mask_alignr_epi8( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_alignr_epi8&expand=265) #[inline] #[target_feature(enable = "avx512bw")] -#[cfg_attr(test, assert_instr(vpalignr, imm8 = 1))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm512_maskz_alignr_epi8(k: __mmask64, a: __m512i, b: __m512i, imm8: i32) -> __m512i { - macro_rules! call { - ($imm8:expr) => { - _mm512_alignr_epi8(a, b, $imm8) - }; - } - let r = constify_imm8_sae!(imm8, call); +#[cfg_attr(test, assert_instr(vpalignr, IMM8 = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_maskz_alignr_epi8( + k: __mmask64, + a: __m512i, + b: __m512i, +) -> __m512i { + static_assert_imm8!(IMM8); + let r = _mm512_alignr_epi8::(a, b); let zero = _mm512_setzero_si512().as_i8x64(); transmute(simd_select_bitmask(k, r.as_i8x64(), zero)) } @@ -9308,21 +9357,16 @@ pub unsafe fn _mm512_maskz_alignr_epi8(k: __mmask64, a: __m512i, b: __m512i, imm /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_alignr_epi8&expand=261) #[inline] #[target_feature(enable = "avx512bw,avx512vl")] -#[rustc_args_required_const(4)] -#[cfg_attr(test, assert_instr(vpalignr, imm8 = 5))] -pub unsafe fn _mm256_mask_alignr_epi8( +#[rustc_legacy_const_generics(4)] +#[cfg_attr(test, assert_instr(vpalignr, IMM8 = 5))] +pub unsafe fn _mm256_mask_alignr_epi8( src: __m256i, k: __mmask32, a: __m256i, b: __m256i, - imm8: i32, ) -> __m256i { - macro_rules! call { - ($imm8:expr) => { - _mm256_alignr_epi8(a, b, $imm8) - }; - } - let r = constify_imm8_sae!(imm8, call); + static_assert_imm8!(IMM8); + let r = _mm256_alignr_epi8::(a, b); transmute(simd_select_bitmask(k, r.as_i8x32(), src.as_i8x32())) } @@ -9331,15 +9375,15 @@ pub unsafe fn _mm256_mask_alignr_epi8( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_alignr_epi8&expand=262) #[inline] #[target_feature(enable = "avx512bw,avx512vl")] -#[rustc_args_required_const(3)] -#[cfg_attr(test, assert_instr(vpalignr, imm8 = 5))] -pub unsafe fn _mm256_maskz_alignr_epi8(k: __mmask32, a: __m256i, b: __m256i, imm8: i32) -> __m256i { - macro_rules! call { - ($imm8:expr) => { - _mm256_alignr_epi8(a, b, $imm8) - }; - } - let r = constify_imm8_sae!(imm8, call); +#[rustc_legacy_const_generics(3)] +#[cfg_attr(test, assert_instr(vpalignr, IMM8 = 5))] +pub unsafe fn _mm256_maskz_alignr_epi8( + k: __mmask32, + a: __m256i, + b: __m256i, +) -> __m256i { + static_assert_imm8!(IMM8); + let r = _mm256_alignr_epi8::(a, b); transmute(simd_select_bitmask( k, r.as_i8x32(), @@ -9352,21 +9396,16 @@ pub unsafe fn _mm256_maskz_alignr_epi8(k: __mmask32, a: __m256i, b: __m256i, imm /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_alignr_epi8&expand=258) #[inline] #[target_feature(enable = "avx512bw,avx512vl")] -#[rustc_args_required_const(4)] -#[cfg_attr(test, assert_instr(vpalignr, imm8 = 5))] -pub unsafe fn _mm_mask_alignr_epi8( +#[rustc_legacy_const_generics(4)] +#[cfg_attr(test, assert_instr(vpalignr, IMM8 = 5))] +pub unsafe fn _mm_mask_alignr_epi8( src: __m128i, k: __mmask16, a: __m128i, b: __m128i, - imm8: i32, ) -> __m128i { - macro_rules! call { - ($imm8:expr) => { - _mm_alignr_epi8(a, b, $imm8) - }; - } - let r = constify_imm8_sae!(imm8, call); + static_assert_imm8!(IMM8); + let r = _mm_alignr_epi8::(a, b); transmute(simd_select_bitmask(k, r.as_i8x16(), src.as_i8x16())) } @@ -9375,15 +9414,15 @@ pub unsafe fn _mm_mask_alignr_epi8( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_alignr_epi8&expand=259) #[inline] #[target_feature(enable = "avx512bw,avx512vl")] -#[rustc_args_required_const(3)] -#[cfg_attr(test, assert_instr(vpalignr, imm8 = 5))] -pub unsafe fn _mm_maskz_alignr_epi8(k: __mmask16, a: __m128i, b: __m128i, imm8: i32) -> __m128i { - macro_rules! call { - ($imm8:expr) => { - _mm_alignr_epi8(a, b, $imm8) - }; - } - let r = constify_imm8_sae!(imm8, call); +#[rustc_legacy_const_generics(3)] +#[cfg_attr(test, assert_instr(vpalignr, IMM8 = 5))] +pub unsafe fn _mm_maskz_alignr_epi8( + k: __mmask16, + a: __m128i, + b: __m128i, +) -> __m128i { + static_assert_imm8!(IMM8); + let r = _mm_alignr_epi8::(a, b); let zero = _mm_setzero_si128().as_i8x16(); transmute(simd_select_bitmask(k, r.as_i8x16(), zero)) } @@ -9613,6 +9652,11 @@ extern "C" { #[link_name = "llvm.x86.avx512.pslli.w.512"] fn vpslliw(a: i16x32, imm8: u32) -> i16x32; + #[link_name = "llvm.x86.avx2.pslli.w"] + fn pslliw256(a: i16x16, imm8: i32) -> i16x16; + #[link_name = "llvm.x86.sse2.pslli.w"] + fn pslliw128(a: i16x8, imm8: i32) -> i16x8; + #[link_name = "llvm.x86.avx512.psllv.w.512"] fn vpsllvw(a: i16x32, b: i16x32) -> i16x32; #[link_name = "llvm.x86.avx512.psllv.w.256"] @@ -9637,6 +9681,11 @@ extern "C" { #[link_name = "llvm.x86.avx512.psrai.w.512"] fn vpsraiw(a: i16x32, imm8: u32) -> i16x32; + #[link_name = "llvm.x86.avx2.psrai.w"] + fn psraiw256(a: i16x16, imm8: i32) -> i16x16; + #[link_name = "llvm.x86.sse2.psrai.w"] + fn psraiw128(a: i16x8, imm8: i32) -> i16x8; + #[link_name = "llvm.x86.avx512.psrav.w.512"] fn vpsravw(a: i16x32, count: i16x32) -> i16x32; #[link_name = "llvm.x86.avx512.psrav.w.256"] @@ -13456,7 +13505,7 @@ mod tests { unsafe fn test_mm512_cmp_epu16_mask() { let a = _mm512_set1_epi16(0); let b = _mm512_set1_epi16(1); - let m = _mm512_cmp_epu16_mask(a, b, _MM_CMPINT_LT); + let m = _mm512_cmp_epu16_mask::<_MM_CMPINT_LT>(a, b); assert_eq!(m, 0b11111111_11111111_11111111_11111111); } @@ -13465,7 +13514,7 @@ mod tests { let a = _mm512_set1_epi16(0); let b = _mm512_set1_epi16(1); let mask = 0b01010101_01010101_01010101_01010101; - let r = _mm512_mask_cmp_epu16_mask(mask, a, b, _MM_CMPINT_LT); + let r = _mm512_mask_cmp_epu16_mask::<_MM_CMPINT_LT>(mask, a, b); assert_eq!(r, 0b01010101_01010101_01010101_01010101); } @@ -13473,7 +13522,7 @@ mod tests { unsafe fn test_mm256_cmp_epu16_mask() { let a = _mm256_set1_epi16(0); let b = _mm256_set1_epi16(1); - let m = _mm256_cmp_epu16_mask(a, b, _MM_CMPINT_LT); + let m = _mm256_cmp_epu16_mask::<_MM_CMPINT_LT>(a, b); assert_eq!(m, 0b11111111_11111111); } @@ -13482,7 +13531,7 @@ mod tests { let a = _mm256_set1_epi16(0); let b = _mm256_set1_epi16(1); let mask = 0b01010101_01010101; - let r = _mm256_mask_cmp_epu16_mask(mask, a, b, _MM_CMPINT_LT); + let r = _mm256_mask_cmp_epu16_mask::<_MM_CMPINT_LT>(mask, a, b); assert_eq!(r, 0b01010101_01010101); } @@ -13490,7 +13539,7 @@ mod tests { unsafe fn test_mm_cmp_epu16_mask() { let a = _mm_set1_epi16(0); let b = _mm_set1_epi16(1); - let m = _mm_cmp_epu16_mask(a, b, _MM_CMPINT_LT); + let m = _mm_cmp_epu16_mask::<_MM_CMPINT_LT>(a, b); assert_eq!(m, 0b11111111); } @@ -13499,7 +13548,7 @@ mod tests { let a = _mm_set1_epi16(0); let b = _mm_set1_epi16(1); let mask = 0b01010101; - let r = _mm_mask_cmp_epu16_mask(mask, a, b, _MM_CMPINT_LT); + let r = _mm_mask_cmp_epu16_mask::<_MM_CMPINT_LT>(mask, a, b); assert_eq!(r, 0b01010101); } @@ -13507,7 +13556,7 @@ mod tests { unsafe fn test_mm512_cmp_epu8_mask() { let a = _mm512_set1_epi8(0); let b = _mm512_set1_epi8(1); - let m = _mm512_cmp_epu8_mask(a, b, _MM_CMPINT_LT); + let m = _mm512_cmp_epu8_mask::<_MM_CMPINT_LT>(a, b); assert_eq!( m, 0b11111111_11111111_11111111_11111111_11111111_11111111_11111111_11111111 @@ -13519,7 +13568,7 @@ mod tests { let a = _mm512_set1_epi8(0); let b = _mm512_set1_epi8(1); let mask = 0b01010101_01010101_01010101_01010101_01010101_01010101_01010101_01010101; - let r = _mm512_mask_cmp_epu8_mask(mask, a, b, _MM_CMPINT_LT); + let r = _mm512_mask_cmp_epu8_mask::<_MM_CMPINT_LT>(mask, a, b); assert_eq!( r, 0b01010101_01010101_01010101_01010101_01010101_01010101_01010101_01010101 @@ -13530,7 +13579,7 @@ mod tests { unsafe fn test_mm256_cmp_epu8_mask() { let a = _mm256_set1_epi8(0); let b = _mm256_set1_epi8(1); - let m = _mm256_cmp_epu8_mask(a, b, _MM_CMPINT_LT); + let m = _mm256_cmp_epu8_mask::<_MM_CMPINT_LT>(a, b); assert_eq!(m, 0b11111111_11111111_11111111_11111111); } @@ -13539,7 +13588,7 @@ mod tests { let a = _mm256_set1_epi8(0); let b = _mm256_set1_epi8(1); let mask = 0b01010101_01010101_01010101_01010101; - let r = _mm256_mask_cmp_epu8_mask(mask, a, b, _MM_CMPINT_LT); + let r = _mm256_mask_cmp_epu8_mask::<_MM_CMPINT_LT>(mask, a, b); assert_eq!(r, 0b01010101_01010101_01010101_01010101); } @@ -13547,7 +13596,7 @@ mod tests { unsafe fn test_mm_cmp_epu8_mask() { let a = _mm_set1_epi8(0); let b = _mm_set1_epi8(1); - let m = _mm_cmp_epu8_mask(a, b, _MM_CMPINT_LT); + let m = _mm_cmp_epu8_mask::<_MM_CMPINT_LT>(a, b); assert_eq!(m, 0b11111111_11111111); } @@ -13556,7 +13605,7 @@ mod tests { let a = _mm_set1_epi8(0); let b = _mm_set1_epi8(1); let mask = 0b01010101_01010101; - let r = _mm_mask_cmp_epu8_mask(mask, a, b, _MM_CMPINT_LT); + let r = _mm_mask_cmp_epu8_mask::<_MM_CMPINT_LT>(mask, a, b); assert_eq!(r, 0b01010101_01010101); } @@ -13564,7 +13613,7 @@ mod tests { unsafe fn test_mm512_cmp_epi16_mask() { let a = _mm512_set1_epi16(0); let b = _mm512_set1_epi16(1); - let m = _mm512_cmp_epi16_mask(a, b, _MM_CMPINT_LT); + let m = _mm512_cmp_epi16_mask::<_MM_CMPINT_LT>(a, b); assert_eq!(m, 0b11111111_11111111_11111111_11111111); } @@ -13573,7 +13622,7 @@ mod tests { let a = _mm512_set1_epi16(0); let b = _mm512_set1_epi16(1); let mask = 0b01010101_01010101_01010101_01010101; - let r = _mm512_mask_cmp_epi16_mask(mask, a, b, _MM_CMPINT_LT); + let r = _mm512_mask_cmp_epi16_mask::<_MM_CMPINT_LT>(mask, a, b); assert_eq!(r, 0b01010101_01010101_01010101_01010101); } @@ -13581,7 +13630,7 @@ mod tests { unsafe fn test_mm256_cmp_epi16_mask() { let a = _mm256_set1_epi16(0); let b = _mm256_set1_epi16(1); - let m = _mm256_cmp_epi16_mask(a, b, _MM_CMPINT_LT); + let m = _mm256_cmp_epi16_mask::<_MM_CMPINT_LT>(a, b); assert_eq!(m, 0b11111111_11111111); } @@ -13590,7 +13639,7 @@ mod tests { let a = _mm256_set1_epi16(0); let b = _mm256_set1_epi16(1); let mask = 0b01010101_01010101; - let r = _mm256_mask_cmp_epi16_mask(mask, a, b, _MM_CMPINT_LT); + let r = _mm256_mask_cmp_epi16_mask::<_MM_CMPINT_LT>(mask, a, b); assert_eq!(r, 0b01010101_01010101); } @@ -13598,7 +13647,7 @@ mod tests { unsafe fn test_mm_cmp_epi16_mask() { let a = _mm_set1_epi16(0); let b = _mm_set1_epi16(1); - let m = _mm_cmp_epi16_mask(a, b, _MM_CMPINT_LT); + let m = _mm_cmp_epi16_mask::<_MM_CMPINT_LT>(a, b); assert_eq!(m, 0b11111111); } @@ -13607,7 +13656,7 @@ mod tests { let a = _mm_set1_epi16(0); let b = _mm_set1_epi16(1); let mask = 0b01010101; - let r = _mm_mask_cmp_epi16_mask(mask, a, b, _MM_CMPINT_LT); + let r = _mm_mask_cmp_epi16_mask::<_MM_CMPINT_LT>(mask, a, b); assert_eq!(r, 0b01010101); } @@ -13615,7 +13664,7 @@ mod tests { unsafe fn test_mm512_cmp_epi8_mask() { let a = _mm512_set1_epi8(0); let b = _mm512_set1_epi8(1); - let m = _mm512_cmp_epi8_mask(a, b, _MM_CMPINT_LT); + let m = _mm512_cmp_epi8_mask::<_MM_CMPINT_LT>(a, b); assert_eq!( m, 0b11111111_11111111_11111111_11111111_11111111_11111111_11111111_11111111 @@ -13627,7 +13676,7 @@ mod tests { let a = _mm512_set1_epi8(0); let b = _mm512_set1_epi8(1); let mask = 0b01010101_01010101_01010101_01010101_01010101_01010101_01010101_01010101; - let r = _mm512_mask_cmp_epi8_mask(mask, a, b, _MM_CMPINT_LT); + let r = _mm512_mask_cmp_epi8_mask::<_MM_CMPINT_LT>(mask, a, b); assert_eq!( r, 0b01010101_01010101_01010101_01010101_01010101_01010101_01010101_01010101 @@ -13638,7 +13687,7 @@ mod tests { unsafe fn test_mm256_cmp_epi8_mask() { let a = _mm256_set1_epi8(0); let b = _mm256_set1_epi8(1); - let m = _mm256_cmp_epi8_mask(a, b, _MM_CMPINT_LT); + let m = _mm256_cmp_epi8_mask::<_MM_CMPINT_LT>(a, b); assert_eq!(m, 0b11111111_11111111_11111111_11111111); } @@ -13647,7 +13696,7 @@ mod tests { let a = _mm256_set1_epi8(0); let b = _mm256_set1_epi8(1); let mask = 0b01010101_01010101_01010101_01010101; - let r = _mm256_mask_cmp_epi8_mask(mask, a, b, _MM_CMPINT_LT); + let r = _mm256_mask_cmp_epi8_mask::<_MM_CMPINT_LT>(mask, a, b); assert_eq!(r, 0b01010101_01010101_01010101_01010101); } @@ -13655,7 +13704,7 @@ mod tests { unsafe fn test_mm_cmp_epi8_mask() { let a = _mm_set1_epi8(0); let b = _mm_set1_epi8(1); - let m = _mm_cmp_epi8_mask(a, b, _MM_CMPINT_LT); + let m = _mm_cmp_epi8_mask::<_MM_CMPINT_LT>(a, b); assert_eq!(m, 0b11111111_11111111); } @@ -13664,7 +13713,7 @@ mod tests { let a = _mm_set1_epi8(0); let b = _mm_set1_epi8(1); let mask = 0b01010101_01010101; - let r = _mm_mask_cmp_epi8_mask(mask, a, b, _MM_CMPINT_LT); + let r = _mm_mask_cmp_epi8_mask::<_MM_CMPINT_LT>(mask, a, b); assert_eq!(r, 0b01010101_01010101); } @@ -14574,7 +14623,7 @@ mod tests { #[simd_test(enable = "avx512bw")] unsafe fn test_mm512_slli_epi16() { let a = _mm512_set1_epi16(1 << 15); - let r = _mm512_slli_epi16(a, 1); + let r = _mm512_slli_epi16::<1>(a); let e = _mm512_set1_epi16(0); assert_eq_m512i(r, e); } @@ -14582,9 +14631,9 @@ mod tests { #[simd_test(enable = "avx512bw")] unsafe fn test_mm512_mask_slli_epi16() { let a = _mm512_set1_epi16(1 << 15); - let r = _mm512_mask_slli_epi16(a, 0, a, 1); + let r = _mm512_mask_slli_epi16::<1>(a, 0, a); assert_eq_m512i(r, a); - let r = _mm512_mask_slli_epi16(a, 0b11111111_11111111_11111111_11111111, a, 1); + let r = _mm512_mask_slli_epi16::<1>(a, 0b11111111_11111111_11111111_11111111, a); let e = _mm512_set1_epi16(0); assert_eq_m512i(r, e); } @@ -14592,9 +14641,9 @@ mod tests { #[simd_test(enable = "avx512bw")] unsafe fn test_mm512_maskz_slli_epi16() { let a = _mm512_set1_epi16(1 << 15); - let r = _mm512_maskz_slli_epi16(0, a, 1); + let r = _mm512_maskz_slli_epi16::<1>(0, a); assert_eq_m512i(r, _mm512_setzero_si512()); - let r = _mm512_maskz_slli_epi16(0b11111111_11111111_11111111_11111111, a, 1); + let r = _mm512_maskz_slli_epi16::<1>(0b11111111_11111111_11111111_11111111, a); let e = _mm512_set1_epi16(0); assert_eq_m512i(r, e); } @@ -14602,9 +14651,9 @@ mod tests { #[simd_test(enable = "avx512bw,avx512vl")] unsafe fn test_mm256_mask_slli_epi16() { let a = _mm256_set1_epi16(1 << 15); - let r = _mm256_mask_slli_epi16(a, 0, a, 1); + let r = _mm256_mask_slli_epi16::<1>(a, 0, a); assert_eq_m256i(r, a); - let r = _mm256_mask_slli_epi16(a, 0b11111111_11111111, a, 1); + let r = _mm256_mask_slli_epi16::<1>(a, 0b11111111_11111111, a); let e = _mm256_set1_epi16(0); assert_eq_m256i(r, e); } @@ -14612,9 +14661,9 @@ mod tests { #[simd_test(enable = "avx512bw,avx512vl")] unsafe fn test_mm256_maskz_slli_epi16() { let a = _mm256_set1_epi16(1 << 15); - let r = _mm256_maskz_slli_epi16(0, a, 1); + let r = _mm256_maskz_slli_epi16::<1>(0, a); assert_eq_m256i(r, _mm256_setzero_si256()); - let r = _mm256_maskz_slli_epi16(0b11111111_11111111, a, 1); + let r = _mm256_maskz_slli_epi16::<1>(0b11111111_11111111, a); let e = _mm256_set1_epi16(0); assert_eq_m256i(r, e); } @@ -14622,9 +14671,9 @@ mod tests { #[simd_test(enable = "avx512bw,avx512vl")] unsafe fn test_mm_mask_slli_epi16() { let a = _mm_set1_epi16(1 << 15); - let r = _mm_mask_slli_epi16(a, 0, a, 1); + let r = _mm_mask_slli_epi16::<1>(a, 0, a); assert_eq_m128i(r, a); - let r = _mm_mask_slli_epi16(a, 0b11111111, a, 1); + let r = _mm_mask_slli_epi16::<1>(a, 0b11111111, a); let e = _mm_set1_epi16(0); assert_eq_m128i(r, e); } @@ -14632,9 +14681,9 @@ mod tests { #[simd_test(enable = "avx512bw,avx512vl")] unsafe fn test_mm_maskz_slli_epi16() { let a = _mm_set1_epi16(1 << 15); - let r = _mm_maskz_slli_epi16(0, a, 1); + let r = _mm_maskz_slli_epi16::<1>(0, a); assert_eq_m128i(r, _mm_setzero_si128()); - let r = _mm_maskz_slli_epi16(0b11111111, a, 1); + let r = _mm_maskz_slli_epi16::<1>(0b11111111, a); let e = _mm_set1_epi16(0); assert_eq_m128i(r, e); } @@ -14810,7 +14859,7 @@ mod tests { #[simd_test(enable = "avx512bw")] unsafe fn test_mm512_srli_epi16() { let a = _mm512_set1_epi16(1 << 1); - let r = _mm512_srli_epi16(a, 2); + let r = _mm512_srli_epi16::<2>(a); let e = _mm512_set1_epi16(0); assert_eq_m512i(r, e); } @@ -14818,9 +14867,9 @@ mod tests { #[simd_test(enable = "avx512bw")] unsafe fn test_mm512_mask_srli_epi16() { let a = _mm512_set1_epi16(1 << 1); - let r = _mm512_mask_srli_epi16(a, 0, a, 2); + let r = _mm512_mask_srli_epi16::<2>(a, 0, a); assert_eq_m512i(r, a); - let r = _mm512_mask_srli_epi16(a, 0b11111111_11111111_11111111_11111111, a, 2); + let r = _mm512_mask_srli_epi16::<2>(a, 0b11111111_11111111_11111111_11111111, a); let e = _mm512_set1_epi16(0); assert_eq_m512i(r, e); } @@ -14828,9 +14877,9 @@ mod tests { #[simd_test(enable = "avx512bw")] unsafe fn test_mm512_maskz_srli_epi16() { let a = _mm512_set1_epi16(1 << 1); - let r = _mm512_maskz_srli_epi16(0, a, 2); + let r = _mm512_maskz_srli_epi16::<2>(0, a); assert_eq_m512i(r, _mm512_setzero_si512()); - let r = _mm512_maskz_srli_epi16(0b11111111_11111111_11111111_11111111, a, 2); + let r = _mm512_maskz_srli_epi16::<2>(0b11111111_11111111_11111111_11111111, a); let e = _mm512_set1_epi16(0); assert_eq_m512i(r, e); } @@ -14838,9 +14887,9 @@ mod tests { #[simd_test(enable = "avx512bw,avx512vl")] unsafe fn test_mm256_mask_srli_epi16() { let a = _mm256_set1_epi16(1 << 1); - let r = _mm256_mask_srli_epi16(a, 0, a, 2); + let r = _mm256_mask_srli_epi16::<2>(a, 0, a); assert_eq_m256i(r, a); - let r = _mm256_mask_srli_epi16(a, 0b11111111_11111111, a, 2); + let r = _mm256_mask_srli_epi16::<2>(a, 0b11111111_11111111, a); let e = _mm256_set1_epi16(0); assert_eq_m256i(r, e); } @@ -14848,9 +14897,9 @@ mod tests { #[simd_test(enable = "avx512bw,avx512vl")] unsafe fn test_mm256_maskz_srli_epi16() { let a = _mm256_set1_epi16(1 << 1); - let r = _mm256_maskz_srli_epi16(0, a, 2); + let r = _mm256_maskz_srli_epi16::<2>(0, a); assert_eq_m256i(r, _mm256_setzero_si256()); - let r = _mm256_maskz_srli_epi16(0b11111111_11111111, a, 2); + let r = _mm256_maskz_srli_epi16::<2>(0b11111111_11111111, a); let e = _mm256_set1_epi16(0); assert_eq_m256i(r, e); } @@ -14858,9 +14907,9 @@ mod tests { #[simd_test(enable = "avx512bw,avx512vl")] unsafe fn test_mm_mask_srli_epi16() { let a = _mm_set1_epi16(1 << 1); - let r = _mm_mask_srli_epi16(a, 0, a, 2); + let r = _mm_mask_srli_epi16::<2>(a, 0, a); assert_eq_m128i(r, a); - let r = _mm_mask_srli_epi16(a, 0b11111111, a, 2); + let r = _mm_mask_srli_epi16::<2>(a, 0b11111111, a); let e = _mm_set1_epi16(0); assert_eq_m128i(r, e); } @@ -14868,9 +14917,9 @@ mod tests { #[simd_test(enable = "avx512bw,avx512vl")] unsafe fn test_mm_maskz_srli_epi16() { let a = _mm_set1_epi16(1 << 1); - let r = _mm_maskz_srli_epi16(0, a, 2); + let r = _mm_maskz_srli_epi16::<2>(0, a); assert_eq_m128i(r, _mm_setzero_si128()); - let r = _mm_maskz_srli_epi16(0b11111111, a, 2); + let r = _mm_maskz_srli_epi16::<2>(0b11111111, a); let e = _mm_set1_epi16(0); assert_eq_m128i(r, e); } @@ -15046,7 +15095,7 @@ mod tests { #[simd_test(enable = "avx512bw")] unsafe fn test_mm512_srai_epi16() { let a = _mm512_set1_epi16(8); - let r = _mm512_srai_epi16(a, 2); + let r = _mm512_srai_epi16::<2>(a); let e = _mm512_set1_epi16(2); assert_eq_m512i(r, e); } @@ -15054,9 +15103,9 @@ mod tests { #[simd_test(enable = "avx512bw")] unsafe fn test_mm512_mask_srai_epi16() { let a = _mm512_set1_epi16(8); - let r = _mm512_mask_srai_epi16(a, 0, a, 2); + let r = _mm512_mask_srai_epi16::<2>(a, 0, a); assert_eq_m512i(r, a); - let r = _mm512_mask_srai_epi16(a, 0b11111111_11111111_11111111_11111111, a, 2); + let r = _mm512_mask_srai_epi16::<2>(a, 0b11111111_11111111_11111111_11111111, a); let e = _mm512_set1_epi16(2); assert_eq_m512i(r, e); } @@ -15064,9 +15113,9 @@ mod tests { #[simd_test(enable = "avx512bw")] unsafe fn test_mm512_maskz_srai_epi16() { let a = _mm512_set1_epi16(8); - let r = _mm512_maskz_srai_epi16(0, a, 2); + let r = _mm512_maskz_srai_epi16::<2>(0, a); assert_eq_m512i(r, _mm512_setzero_si512()); - let r = _mm512_maskz_srai_epi16(0b11111111_11111111_11111111_11111111, a, 2); + let r = _mm512_maskz_srai_epi16::<2>(0b11111111_11111111_11111111_11111111, a); let e = _mm512_set1_epi16(2); assert_eq_m512i(r, e); } @@ -15074,9 +15123,9 @@ mod tests { #[simd_test(enable = "avx512bw,avx512vl")] unsafe fn test_mm256_mask_srai_epi16() { let a = _mm256_set1_epi16(8); - let r = _mm256_mask_srai_epi16(a, 0, a, 2); + let r = _mm256_mask_srai_epi16::<2>(a, 0, a); assert_eq_m256i(r, a); - let r = _mm256_mask_srai_epi16(a, 0b11111111_11111111, a, 2); + let r = _mm256_mask_srai_epi16::<2>(a, 0b11111111_11111111, a); let e = _mm256_set1_epi16(2); assert_eq_m256i(r, e); } @@ -15084,9 +15133,9 @@ mod tests { #[simd_test(enable = "avx512bw,avx512vl")] unsafe fn test_mm256_maskz_srai_epi16() { let a = _mm256_set1_epi16(8); - let r = _mm256_maskz_srai_epi16(0, a, 2); + let r = _mm256_maskz_srai_epi16::<2>(0, a); assert_eq_m256i(r, _mm256_setzero_si256()); - let r = _mm256_maskz_srai_epi16(0b11111111_11111111, a, 2); + let r = _mm256_maskz_srai_epi16::<2>(0b11111111_11111111, a); let e = _mm256_set1_epi16(2); assert_eq_m256i(r, e); } @@ -15094,9 +15143,9 @@ mod tests { #[simd_test(enable = "avx512bw,avx512vl")] unsafe fn test_mm_mask_srai_epi16() { let a = _mm_set1_epi16(8); - let r = _mm_mask_srai_epi16(a, 0, a, 2); + let r = _mm_mask_srai_epi16::<2>(a, 0, a); assert_eq_m128i(r, a); - let r = _mm_mask_srai_epi16(a, 0b11111111, a, 2); + let r = _mm_mask_srai_epi16::<2>(a, 0b11111111, a); let e = _mm_set1_epi16(2); assert_eq_m128i(r, e); } @@ -15104,9 +15153,9 @@ mod tests { #[simd_test(enable = "avx512bw,avx512vl")] unsafe fn test_mm_maskz_srai_epi16() { let a = _mm_set1_epi16(8); - let r = _mm_maskz_srai_epi16(0, a, 2); + let r = _mm_maskz_srai_epi16::<2>(0, a); assert_eq_m128i(r, _mm_setzero_si128()); - let r = _mm_maskz_srai_epi16(0b11111111, a, 2); + let r = _mm_maskz_srai_epi16::<2>(0b11111111, a); let e = _mm_set1_epi16(2); assert_eq_m128i(r, e); } @@ -16451,7 +16500,7 @@ mod tests { 0, 1, 2, 3, 7, 6, 6, 4, 8, 9, 10, 11, 15, 14, 14, 12, 16, 17, 18, 19, 23, 22, 22, 20, 24, 25, 26, 27, 31, 30, 30, 28, ); - let r = _mm512_shufflelo_epi16(a, 0b00_01_01_11); + let r = _mm512_shufflelo_epi16::<0b00_01_01_11>(a); assert_eq_m512i(r, e); } @@ -16462,10 +16511,13 @@ mod tests { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, ); - let r = _mm512_mask_shufflelo_epi16(a, 0, a, 0b00_01_01_11); + let r = _mm512_mask_shufflelo_epi16::<0b00_01_01_11>(a, 0, a); assert_eq_m512i(r, a); - let r = - _mm512_mask_shufflelo_epi16(a, 0b11111111_11111111_11111111_11111111, a, 0b00_01_01_11); + let r = _mm512_mask_shufflelo_epi16::<0b00_01_01_11>( + a, + 0b11111111_11111111_11111111_11111111, + a, + ); #[rustfmt::skip] let e = _mm512_set_epi16( 0, 1, 2, 3, 7, 6, 6, 4, 8, 9, 10, 11, 15, 14, 14, 12, @@ -16481,10 +16533,10 @@ mod tests { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, ); - let r = _mm512_maskz_shufflelo_epi16(0, a, 0b00_01_01_11); + let r = _mm512_maskz_shufflelo_epi16::<0b00_01_01_11>(0, a); assert_eq_m512i(r, _mm512_setzero_si512()); let r = - _mm512_maskz_shufflelo_epi16(0b11111111_11111111_11111111_11111111, a, 0b00_01_01_11); + _mm512_maskz_shufflelo_epi16::<0b00_01_01_11>(0b11111111_11111111_11111111_11111111, a); #[rustfmt::skip] let e = _mm512_set_epi16( 0, 1, 2, 3, 7, 6, 6, 4, 8, 9, 10, 11, 15, 14, 14, 12, @@ -16496,9 +16548,9 @@ mod tests { #[simd_test(enable = "avx512bw,avx512vl")] unsafe fn test_mm256_mask_shufflelo_epi16() { let a = _mm256_set_epi16(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); - let r = _mm256_mask_shufflelo_epi16(a, 0, a, 0b00_01_01_11); + let r = _mm256_mask_shufflelo_epi16::<0b00_01_01_11>(a, 0, a); assert_eq_m256i(r, a); - let r = _mm256_mask_shufflelo_epi16(a, 0b11111111_11111111, a, 0b00_01_01_11); + let r = _mm256_mask_shufflelo_epi16::<0b00_01_01_11>(a, 0b11111111_11111111, a); let e = _mm256_set_epi16(0, 1, 2, 3, 7, 6, 6, 4, 8, 9, 10, 11, 15, 14, 14, 12); assert_eq_m256i(r, e); } @@ -16506,9 +16558,9 @@ mod tests { #[simd_test(enable = "avx512bw,avx512vl")] unsafe fn test_mm256_maskz_shufflelo_epi16() { let a = _mm256_set_epi16(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); - let r = _mm256_maskz_shufflelo_epi16(0, a, 0b00_01_01_11); + let r = _mm256_maskz_shufflelo_epi16::<0b00_01_01_11>(0, a); assert_eq_m256i(r, _mm256_setzero_si256()); - let r = _mm256_maskz_shufflelo_epi16(0b11111111_11111111, a, 0b00_01_01_11); + let r = _mm256_maskz_shufflelo_epi16::<0b00_01_01_11>(0b11111111_11111111, a); let e = _mm256_set_epi16(0, 1, 2, 3, 7, 6, 6, 4, 8, 9, 10, 11, 15, 14, 14, 12); assert_eq_m256i(r, e); } @@ -16516,9 +16568,9 @@ mod tests { #[simd_test(enable = "avx512bw,avx512vl")] unsafe fn test_mm_mask_shufflelo_epi16() { let a = _mm_set_epi16(0, 1, 2, 3, 4, 5, 6, 7); - let r = _mm_mask_shufflelo_epi16(a, 0, a, 0b00_01_01_11); + let r = _mm_mask_shufflelo_epi16::<0b00_01_01_11>(a, 0, a); assert_eq_m128i(r, a); - let r = _mm_mask_shufflelo_epi16(a, 0b11111111, a, 0b00_01_01_11); + let r = _mm_mask_shufflelo_epi16::<0b00_01_01_11>(a, 0b11111111, a); let e = _mm_set_epi16(0, 1, 2, 3, 7, 6, 6, 4); assert_eq_m128i(r, e); } @@ -16526,9 +16578,9 @@ mod tests { #[simd_test(enable = "avx512bw,avx512vl")] unsafe fn test_mm_maskz_shufflelo_epi16() { let a = _mm_set_epi16(0, 1, 2, 3, 4, 5, 6, 7); - let r = _mm_maskz_shufflelo_epi16(0, a, 0b00_01_01_11); + let r = _mm_maskz_shufflelo_epi16::<0b00_01_01_11>(0, a); assert_eq_m128i(r, _mm_setzero_si128()); - let r = _mm_maskz_shufflelo_epi16(0b11111111, a, 0b00_01_01_11); + let r = _mm_maskz_shufflelo_epi16::<0b00_01_01_11>(0b11111111, a); let e = _mm_set_epi16(0, 1, 2, 3, 7, 6, 6, 4); assert_eq_m128i(r, e); } @@ -16545,7 +16597,7 @@ mod tests { 3, 2, 2, 0, 4, 5, 6, 7, 11, 10, 10, 8, 12, 13, 14, 15, 19, 18, 18, 16, 20, 21, 22, 23, 27, 26, 26, 24, 28, 29, 30, 31, ); - let r = _mm512_shufflehi_epi16(a, 0b00_01_01_11); + let r = _mm512_shufflehi_epi16::<0b00_01_01_11>(a); assert_eq_m512i(r, e); } @@ -16556,10 +16608,13 @@ mod tests { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, ); - let r = _mm512_mask_shufflehi_epi16(a, 0, a, 0b00_01_01_11); + let r = _mm512_mask_shufflehi_epi16::<0b00_01_01_11>(a, 0, a); assert_eq_m512i(r, a); - let r = - _mm512_mask_shufflehi_epi16(a, 0b11111111_11111111_11111111_11111111, a, 0b00_01_01_11); + let r = _mm512_mask_shufflehi_epi16::<0b00_01_01_11>( + a, + 0b11111111_11111111_11111111_11111111, + a, + ); #[rustfmt::skip] let e = _mm512_set_epi16( 3, 2, 2, 0, 4, 5, 6, 7, 11, 10, 10, 8, 12, 13, 14, 15, @@ -16575,10 +16630,10 @@ mod tests { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, ); - let r = _mm512_maskz_shufflehi_epi16(0, a, 0b00_01_01_11); + let r = _mm512_maskz_shufflehi_epi16::<0b00_01_01_11>(0, a); assert_eq_m512i(r, _mm512_setzero_si512()); let r = - _mm512_maskz_shufflehi_epi16(0b11111111_11111111_11111111_11111111, a, 0b00_01_01_11); + _mm512_maskz_shufflehi_epi16::<0b00_01_01_11>(0b11111111_11111111_11111111_11111111, a); #[rustfmt::skip] let e = _mm512_set_epi16( 3, 2, 2, 0, 4, 5, 6, 7, 11, 10, 10, 8, 12, 13, 14, 15, @@ -16590,9 +16645,9 @@ mod tests { #[simd_test(enable = "avx512bw,avx512vl")] unsafe fn test_mm256_mask_shufflehi_epi16() { let a = _mm256_set_epi16(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); - let r = _mm256_mask_shufflehi_epi16(a, 0, a, 0b00_01_01_11); + let r = _mm256_mask_shufflehi_epi16::<0b00_01_01_11>(a, 0, a); assert_eq_m256i(r, a); - let r = _mm256_mask_shufflehi_epi16(a, 0b11111111_11111111, a, 0b00_01_01_11); + let r = _mm256_mask_shufflehi_epi16::<0b00_01_01_11>(a, 0b11111111_11111111, a); let e = _mm256_set_epi16(3, 2, 2, 0, 4, 5, 6, 7, 11, 10, 10, 8, 12, 13, 14, 15); assert_eq_m256i(r, e); } @@ -16600,9 +16655,9 @@ mod tests { #[simd_test(enable = "avx512bw,avx512vl")] unsafe fn test_mm256_maskz_shufflehi_epi16() { let a = _mm256_set_epi16(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); - let r = _mm256_maskz_shufflehi_epi16(0, a, 0b00_01_01_11); + let r = _mm256_maskz_shufflehi_epi16::<0b00_01_01_11>(0, a); assert_eq_m256i(r, _mm256_setzero_si256()); - let r = _mm256_maskz_shufflehi_epi16(0b11111111_11111111, a, 0b00_01_01_11); + let r = _mm256_maskz_shufflehi_epi16::<0b00_01_01_11>(0b11111111_11111111, a); let e = _mm256_set_epi16(3, 2, 2, 0, 4, 5, 6, 7, 11, 10, 10, 8, 12, 13, 14, 15); assert_eq_m256i(r, e); } @@ -16610,9 +16665,9 @@ mod tests { #[simd_test(enable = "avx512bw,avx512vl")] unsafe fn test_mm_mask_shufflehi_epi16() { let a = _mm_set_epi16(0, 1, 2, 3, 4, 5, 6, 7); - let r = _mm_mask_shufflehi_epi16(a, 0, a, 0b00_01_01_11); + let r = _mm_mask_shufflehi_epi16::<0b00_01_01_11>(a, 0, a); assert_eq_m128i(r, a); - let r = _mm_mask_shufflehi_epi16(a, 0b11111111, a, 0b00_01_01_11); + let r = _mm_mask_shufflehi_epi16::<0b00_01_01_11>(a, 0b11111111, a); let e = _mm_set_epi16(3, 2, 2, 0, 4, 5, 6, 7); assert_eq_m128i(r, e); } @@ -16620,9 +16675,9 @@ mod tests { #[simd_test(enable = "avx512bw,avx512vl")] unsafe fn test_mm_maskz_shufflehi_epi16() { let a = _mm_set_epi16(0, 1, 2, 3, 4, 5, 6, 7); - let r = _mm_maskz_shufflehi_epi16(0, a, 0b00_01_01_11); + let r = _mm_maskz_shufflehi_epi16::<0b00_01_01_11>(0, a); assert_eq_m128i(r, _mm_setzero_si128()); - let r = _mm_maskz_shufflehi_epi16(0b11111111, a, 0b00_01_01_11); + let r = _mm_maskz_shufflehi_epi16::<0b00_01_01_11>(0b11111111, a); let e = _mm_set_epi16(3, 2, 2, 0, 4, 5, 6, 7); assert_eq_m128i(r, e); } @@ -17048,7 +17103,7 @@ mod tests { unsafe fn test_mm512_dbsad_epu8() { let a = _mm512_set1_epi8(2); let b = _mm512_set1_epi8(4); - let r = _mm512_dbsad_epu8(a, b, 0); + let r = _mm512_dbsad_epu8::<0>(a, b); let e = _mm512_set1_epi16(8); assert_eq_m512i(r, e); } @@ -17058,9 +17113,9 @@ mod tests { let src = _mm512_set1_epi16(1); let a = _mm512_set1_epi8(2); let b = _mm512_set1_epi8(4); - let r = _mm512_mask_dbsad_epu8(src, 0, a, b, 0); + let r = _mm512_mask_dbsad_epu8::<0>(src, 0, a, b); assert_eq_m512i(r, src); - let r = _mm512_mask_dbsad_epu8(src, 0b11111111_11111111_11111111_11111111, a, b, 0); + let r = _mm512_mask_dbsad_epu8::<0>(src, 0b11111111_11111111_11111111_11111111, a, b); let e = _mm512_set1_epi16(8); assert_eq_m512i(r, e); } @@ -17069,9 +17124,9 @@ mod tests { unsafe fn test_mm512_maskz_dbsad_epu8() { let a = _mm512_set1_epi8(2); let b = _mm512_set1_epi8(4); - let r = _mm512_maskz_dbsad_epu8(0, a, b, 0); + let r = _mm512_maskz_dbsad_epu8::<0>(0, a, b); assert_eq_m512i(r, _mm512_setzero_si512()); - let r = _mm512_maskz_dbsad_epu8(0b11111111_11111111_11111111_11111111, a, b, 0); + let r = _mm512_maskz_dbsad_epu8::<0>(0b11111111_11111111_11111111_11111111, a, b); let e = _mm512_set1_epi16(8); assert_eq_m512i(r, e); } @@ -17080,7 +17135,7 @@ mod tests { unsafe fn test_mm256_dbsad_epu8() { let a = _mm256_set1_epi8(2); let b = _mm256_set1_epi8(4); - let r = _mm256_dbsad_epu8(a, b, 0); + let r = _mm256_dbsad_epu8::<0>(a, b); let e = _mm256_set1_epi16(8); assert_eq_m256i(r, e); } @@ -17090,9 +17145,9 @@ mod tests { let src = _mm256_set1_epi16(1); let a = _mm256_set1_epi8(2); let b = _mm256_set1_epi8(4); - let r = _mm256_mask_dbsad_epu8(src, 0, a, b, 0); + let r = _mm256_mask_dbsad_epu8::<0>(src, 0, a, b); assert_eq_m256i(r, src); - let r = _mm256_mask_dbsad_epu8(src, 0b11111111_11111111, a, b, 0); + let r = _mm256_mask_dbsad_epu8::<0>(src, 0b11111111_11111111, a, b); let e = _mm256_set1_epi16(8); assert_eq_m256i(r, e); } @@ -17101,9 +17156,9 @@ mod tests { unsafe fn test_mm256_maskz_dbsad_epu8() { let a = _mm256_set1_epi8(2); let b = _mm256_set1_epi8(4); - let r = _mm256_maskz_dbsad_epu8(0, a, b, 0); + let r = _mm256_maskz_dbsad_epu8::<0>(0, a, b); assert_eq_m256i(r, _mm256_setzero_si256()); - let r = _mm256_maskz_dbsad_epu8(0b11111111_11111111, a, b, 0); + let r = _mm256_maskz_dbsad_epu8::<0>(0b11111111_11111111, a, b); let e = _mm256_set1_epi16(8); assert_eq_m256i(r, e); } @@ -17112,7 +17167,7 @@ mod tests { unsafe fn test_mm_dbsad_epu8() { let a = _mm_set1_epi8(2); let b = _mm_set1_epi8(4); - let r = _mm_dbsad_epu8(a, b, 0); + let r = _mm_dbsad_epu8::<0>(a, b); let e = _mm_set1_epi16(8); assert_eq_m128i(r, e); } @@ -17122,9 +17177,9 @@ mod tests { let src = _mm_set1_epi16(1); let a = _mm_set1_epi8(2); let b = _mm_set1_epi8(4); - let r = _mm_mask_dbsad_epu8(src, 0, a, b, 0); + let r = _mm_mask_dbsad_epu8::<0>(src, 0, a, b); assert_eq_m128i(r, src); - let r = _mm_mask_dbsad_epu8(src, 0b11111111, a, b, 0); + let r = _mm_mask_dbsad_epu8::<0>(src, 0b11111111, a, b); let e = _mm_set1_epi16(8); assert_eq_m128i(r, e); } @@ -17133,9 +17188,9 @@ mod tests { unsafe fn test_mm_maskz_dbsad_epu8() { let a = _mm_set1_epi8(2); let b = _mm_set1_epi8(4); - let r = _mm_maskz_dbsad_epu8(0, a, b, 0); + let r = _mm_maskz_dbsad_epu8::<0>(0, a, b); assert_eq_m128i(r, _mm_setzero_si128()); - let r = _mm_maskz_dbsad_epu8(0b11111111, a, b, 0); + let r = _mm_maskz_dbsad_epu8::<0>(0b11111111, a, b); let e = _mm_set1_epi16(8); assert_eq_m128i(r, e); } @@ -17848,7 +17903,7 @@ mod tests { 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, ); - let r = _mm512_bslli_epi128(a, 9); + let r = _mm512_bslli_epi128::<9>(a); #[rustfmt::skip] let e = _mm512_set_epi8( 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -17863,18 +17918,18 @@ mod tests { unsafe fn test_mm512_bsrli_epi128() { #[rustfmt::skip] let a = _mm512_set_epi8( - 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, - 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, - 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, - 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, + 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, + 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, + 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, + 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, ); - let r = _mm512_bsrli_epi128(a, 9); + let r = _mm512_bsrli_epi128::<3>(a); #[rustfmt::skip] let e = _mm512_set_epi8( - 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, + 0, 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, + 0, 0, 0, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, + 0, 0, 0, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, + 0, 0, 0, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, ); assert_eq_m512i(r, e); } @@ -17889,7 +17944,7 @@ mod tests { 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, ); let b = _mm512_set1_epi8(1); - let r = _mm512_alignr_epi8(a, b, 14); + let r = _mm512_alignr_epi8::<14>(a, b); #[rustfmt::skip] let e = _mm512_set_epi8( 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 1, @@ -17910,14 +17965,13 @@ mod tests { 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, ); let b = _mm512_set1_epi8(1); - let r = _mm512_mask_alignr_epi8(a, 0, a, b, 14); + let r = _mm512_mask_alignr_epi8::<14>(a, 0, a, b); assert_eq_m512i(r, a); - let r = _mm512_mask_alignr_epi8( + let r = _mm512_mask_alignr_epi8::<14>( a, 0b11111111_11111111_11111111_11111111_11111111_11111111_11111111_11111111, a, b, - 14, ); #[rustfmt::skip] let e = _mm512_set_epi8( @@ -17939,13 +17993,12 @@ mod tests { 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, ); let b = _mm512_set1_epi8(1); - let r = _mm512_maskz_alignr_epi8(0, a, b, 14); + let r = _mm512_maskz_alignr_epi8::<14>(0, a, b); assert_eq_m512i(r, _mm512_setzero_si512()); - let r = _mm512_maskz_alignr_epi8( + let r = _mm512_maskz_alignr_epi8::<14>( 0b11111111_11111111_11111111_11111111_11111111_11111111_11111111_11111111, a, b, - 14, ); #[rustfmt::skip] let e = _mm512_set_epi8( @@ -17965,9 +18018,9 @@ mod tests { 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, ); let b = _mm256_set1_epi8(1); - let r = _mm256_mask_alignr_epi8(a, 0, a, b, 14); + let r = _mm256_mask_alignr_epi8::<14>(a, 0, a, b); assert_eq_m256i(r, a); - let r = _mm256_mask_alignr_epi8(a, 0b11111111_11111111_11111111_11111111, a, b, 14); + let r = _mm256_mask_alignr_epi8::<14>(a, 0b11111111_11111111_11111111_11111111, a, b); #[rustfmt::skip] let e = _mm256_set_epi8( 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 1, @@ -17984,9 +18037,9 @@ mod tests { 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, ); let b = _mm256_set1_epi8(1); - let r = _mm256_maskz_alignr_epi8(0, a, b, 14); + let r = _mm256_maskz_alignr_epi8::<14>(0, a, b); assert_eq_m256i(r, _mm256_setzero_si256()); - let r = _mm256_maskz_alignr_epi8(0b11111111_11111111_11111111_11111111, a, b, 14); + let r = _mm256_maskz_alignr_epi8::<14>(0b11111111_11111111_11111111_11111111, a, b); #[rustfmt::skip] let e = _mm256_set_epi8( 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 1, @@ -17999,9 +18052,9 @@ mod tests { unsafe fn test_mm_mask_alignr_epi8() { let a = _mm_set_epi8(1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0); let b = _mm_set1_epi8(1); - let r = _mm_mask_alignr_epi8(a, 0, a, b, 14); + let r = _mm_mask_alignr_epi8::<14>(a, 0, a, b); assert_eq_m128i(r, a); - let r = _mm_mask_alignr_epi8(a, 0b11111111_11111111, a, b, 14); + let r = _mm_mask_alignr_epi8::<14>(a, 0b11111111_11111111, a, b); let e = _mm_set_epi8(0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 1); assert_eq_m128i(r, e); } @@ -18010,9 +18063,9 @@ mod tests { unsafe fn test_mm_maskz_alignr_epi8() { let a = _mm_set_epi8(1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0); let b = _mm_set1_epi8(1); - let r = _mm_maskz_alignr_epi8(0, a, b, 14); + let r = _mm_maskz_alignr_epi8::<14>(0, a, b); assert_eq_m128i(r, _mm_setzero_si128()); - let r = _mm_maskz_alignr_epi8(0b11111111_11111111, a, b, 14); + let r = _mm_maskz_alignr_epi8::<14>(0b11111111_11111111, a, b); let e = _mm_set_epi8(0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 1); assert_eq_m128i(r, e); } diff --git a/library/stdarch/crates/core_arch/src/x86/avx512f.rs b/library/stdarch/crates/core_arch/src/x86/avx512f.rs index f72f56a355..766acf46f7 100644 --- a/library/stdarch/crates/core_arch/src/x86/avx512f.rs +++ b/library/stdarch/crates/core_arch/src/x86/avx512f.rs @@ -4872,23 +4872,13 @@ pub unsafe fn _mm_maskz_getexp_pd(k: __mmask8, a: __m128d) -> __m128d { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_roundscale_ps&expand=4784) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vrndscaleps, imm8 = 0))] -#[rustc_args_required_const(1)] -pub unsafe fn _mm512_roundscale_ps(a: __m512, imm8: i32) -> __m512 { +#[cfg_attr(test, assert_instr(vrndscaleps, IMM8 = 0))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm512_roundscale_ps(a: __m512) -> __m512 { + static_assert_imm8!(IMM8); let a = a.as_f32x16(); let zero = _mm512_setzero_ps().as_f32x16(); - macro_rules! call { - ($imm8:expr) => { - vrndscaleps( - a, - $imm8, - zero, - 0b11111111_11111111, - _MM_FROUND_CUR_DIRECTION, - ) - }; - } - let r = constify_imm8_sae!(imm8, call); + let r = vrndscaleps(a, IMM8, zero, 0b11111111_11111111, _MM_FROUND_CUR_DIRECTION); transmute(r) } @@ -4903,17 +4893,17 @@ pub unsafe fn _mm512_roundscale_ps(a: __m512, imm8: i32) -> __m512 { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_roundscale_ps&expand=4782) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vrndscaleps, imm8 = 0))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm512_mask_roundscale_ps(src: __m512, k: __mmask16, a: __m512, imm8: i32) -> __m512 { +#[cfg_attr(test, assert_instr(vrndscaleps, IMM8 = 0))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_mask_roundscale_ps( + src: __m512, + k: __mmask16, + a: __m512, +) -> __m512 { + static_assert_imm8!(IMM8); let a = a.as_f32x16(); let src = src.as_f32x16(); - macro_rules! call { - ($imm8:expr) => { - vrndscaleps(a, $imm8, src, k, _MM_FROUND_CUR_DIRECTION) - }; - } - let r = constify_imm8_sae!(imm8, call); + let r = vrndscaleps(a, IMM8, src, k, _MM_FROUND_CUR_DIRECTION); transmute(r) } @@ -4928,17 +4918,13 @@ pub unsafe fn _mm512_mask_roundscale_ps(src: __m512, k: __mmask16, a: __m512, im /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_roundscale_ps&expand=4783) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vrndscaleps, imm8 = 0))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm512_maskz_roundscale_ps(k: __mmask16, a: __m512, imm8: i32) -> __m512 { +#[cfg_attr(test, assert_instr(vrndscaleps, IMM8 = 0))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_maskz_roundscale_ps(k: __mmask16, a: __m512) -> __m512 { + static_assert_imm8!(IMM8); let a = a.as_f32x16(); let zero = _mm512_setzero_ps().as_f32x16(); - macro_rules! call { - ($imm8:expr) => { - vrndscaleps(a, $imm8, zero, k, _MM_FROUND_CUR_DIRECTION) - }; - } - let r = constify_imm8_sae!(imm8, call); + let r = vrndscaleps(a, IMM8, zero, k, _MM_FROUND_CUR_DIRECTION); transmute(r) } @@ -4953,17 +4939,13 @@ pub unsafe fn _mm512_maskz_roundscale_ps(k: __mmask16, a: __m512, imm8: i32) -> /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_roundscale_ps&expand=4781) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[cfg_attr(test, assert_instr(vrndscaleps, imm8 = 250))] -#[rustc_args_required_const(1)] -pub unsafe fn _mm256_roundscale_ps(a: __m256, imm8: i32) -> __m256 { +#[cfg_attr(test, assert_instr(vrndscaleps, IMM8 = 250))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm256_roundscale_ps(a: __m256) -> __m256 { + static_assert_imm8!(IMM8); let a = a.as_f32x8(); let zero = _mm256_setzero_ps().as_f32x8(); - macro_rules! call { - ($imm8:expr) => { - vrndscaleps256(a, $imm8, zero, 0b11111111) - }; - } - let r = constify_imm8_sae!(imm8, call); + let r = vrndscaleps256(a, IMM8, zero, 0b11111111); transmute(r) } @@ -4978,17 +4960,17 @@ pub unsafe fn _mm256_roundscale_ps(a: __m256, imm8: i32) -> __m256 { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_roundscale_ps&expand=4779) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[cfg_attr(test, assert_instr(vrndscaleps, imm8 = 0))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm256_mask_roundscale_ps(src: __m256, k: __mmask8, a: __m256, imm8: i32) -> __m256 { +#[cfg_attr(test, assert_instr(vrndscaleps, IMM8 = 0))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm256_mask_roundscale_ps( + src: __m256, + k: __mmask8, + a: __m256, +) -> __m256 { + static_assert_imm8!(IMM8); let a = a.as_f32x8(); let src = src.as_f32x8(); - macro_rules! call { - ($imm8:expr) => { - vrndscaleps256(a, $imm8, src, k) - }; - } - let r = constify_imm8_sae!(imm8, call); + let r = vrndscaleps256(a, IMM8, src, k); transmute(r) } @@ -5003,17 +4985,13 @@ pub unsafe fn _mm256_mask_roundscale_ps(src: __m256, k: __mmask8, a: __m256, imm /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_roundscale_ps&expand=4780) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[cfg_attr(test, assert_instr(vrndscaleps, imm8 = 0))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm256_maskz_roundscale_ps(k: __mmask8, a: __m256, imm8: i32) -> __m256 { +#[cfg_attr(test, assert_instr(vrndscaleps, IMM8 = 0))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm256_maskz_roundscale_ps(k: __mmask8, a: __m256) -> __m256 { + static_assert_imm8!(IMM8); let a = a.as_f32x8(); let zero = _mm256_setzero_ps().as_f32x8(); - macro_rules! call { - ($imm8:expr) => { - vrndscaleps256(a, $imm8, zero, k) - }; - } - let r = constify_imm8_sae!(imm8, call); + let r = vrndscaleps256(a, IMM8, zero, k); transmute(r) } @@ -5028,17 +5006,13 @@ pub unsafe fn _mm256_maskz_roundscale_ps(k: __mmask8, a: __m256, imm8: i32) -> _ /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_roundscale_ps&expand=4778) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[cfg_attr(test, assert_instr(vrndscaleps, imm8 = 250))] -#[rustc_args_required_const(1)] -pub unsafe fn _mm_roundscale_ps(a: __m128, imm8: i32) -> __m128 { +#[cfg_attr(test, assert_instr(vrndscaleps, IMM8 = 250))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm_roundscale_ps(a: __m128) -> __m128 { + static_assert_imm8!(IMM8); let a = a.as_f32x4(); let zero = _mm_setzero_ps().as_f32x4(); - macro_rules! call { - ($imm8:expr) => { - vrndscaleps128(a, $imm8, zero, 0b00001111) - }; - } - let r = constify_imm8_sae!(imm8, call); + let r = vrndscaleps128(a, IMM8, zero, 0b00001111); transmute(r) } @@ -5053,17 +5027,17 @@ pub unsafe fn _mm_roundscale_ps(a: __m128, imm8: i32) -> __m128 { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_roundscale_ps&expand=4776) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[cfg_attr(test, assert_instr(vrndscaleps, imm8 = 0))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm_mask_roundscale_ps(src: __m128, k: __mmask8, a: __m128, imm8: i32) -> __m128 { +#[cfg_attr(test, assert_instr(vrndscaleps, IMM8 = 0))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm_mask_roundscale_ps( + src: __m128, + k: __mmask8, + a: __m128, +) -> __m128 { + static_assert_imm8!(IMM8); let a = a.as_f32x4(); let src = src.as_f32x4(); - macro_rules! call { - ($imm8:expr) => { - vrndscaleps128(a, $imm8, src, k) - }; - } - let r = constify_imm8_sae!(imm8, call); + let r = vrndscaleps128(a, IMM8, src, k); transmute(r) } @@ -5078,17 +5052,13 @@ pub unsafe fn _mm_mask_roundscale_ps(src: __m128, k: __mmask8, a: __m128, imm8: /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_roundscale_ps&expand=4777) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[cfg_attr(test, assert_instr(vrndscaleps, imm8 = 0))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm_maskz_roundscale_ps(k: __mmask8, a: __m128, imm8: i32) -> __m128 { +#[cfg_attr(test, assert_instr(vrndscaleps, IMM8 = 0))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm_maskz_roundscale_ps(k: __mmask8, a: __m128) -> __m128 { + static_assert_imm8!(IMM8); let a = a.as_f32x4(); let zero = _mm_setzero_ps().as_f32x4(); - macro_rules! call { - ($imm8:expr) => { - vrndscaleps128(a, $imm8, zero, k) - }; - } - let r = constify_imm8_sae!(imm8, call); + let r = vrndscaleps128(a, IMM8, zero, k); transmute(r) } @@ -5103,17 +5073,13 @@ pub unsafe fn _mm_maskz_roundscale_ps(k: __mmask8, a: __m128, imm8: i32) -> __m1 /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_roundscale_pd&expand=4775) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vrndscalepd, imm8 = 0))] -#[rustc_args_required_const(1)] -pub unsafe fn _mm512_roundscale_pd(a: __m512d, imm8: i32) -> __m512d { +#[cfg_attr(test, assert_instr(vrndscalepd, IMM8 = 0))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm512_roundscale_pd(a: __m512d) -> __m512d { + static_assert_imm8!(IMM8); let a = a.as_f64x8(); let zero = _mm512_setzero_pd().as_f64x8(); - macro_rules! call { - ($imm8:expr) => { - vrndscalepd(a, $imm8, zero, 0b11111111, _MM_FROUND_CUR_DIRECTION) - }; - } - let r = constify_imm8_sae!(imm8, call); + let r = vrndscalepd(a, IMM8, zero, 0b11111111, _MM_FROUND_CUR_DIRECTION); transmute(r) } @@ -5128,22 +5094,17 @@ pub unsafe fn _mm512_roundscale_pd(a: __m512d, imm8: i32) -> __m512d { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_roundscale_pd&expand=4773) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vrndscalepd, imm8 = 0))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm512_mask_roundscale_pd( +#[cfg_attr(test, assert_instr(vrndscalepd, IMM8 = 0))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_mask_roundscale_pd( src: __m512d, k: __mmask8, a: __m512d, - imm8: i32, ) -> __m512d { + static_assert_imm8!(IMM8); let a = a.as_f64x8(); let src = src.as_f64x8(); - macro_rules! call { - ($imm8:expr) => { - vrndscalepd(a, $imm8, src, k, _MM_FROUND_CUR_DIRECTION) - }; - } - let r = constify_imm8_sae!(imm8, call); + let r = vrndscalepd(a, IMM8, src, k, _MM_FROUND_CUR_DIRECTION); transmute(r) } @@ -5158,17 +5119,13 @@ pub unsafe fn _mm512_mask_roundscale_pd( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_roundscale_pd&expand=4774) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vrndscalepd, imm8 = 0))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm512_maskz_roundscale_pd(k: __mmask8, a: __m512d, imm8: i32) -> __m512d { +#[cfg_attr(test, assert_instr(vrndscalepd, IMM8 = 0))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_maskz_roundscale_pd(k: __mmask8, a: __m512d) -> __m512d { + static_assert_imm8!(IMM8); let a = a.as_f64x8(); let zero = _mm512_setzero_pd().as_f64x8(); - macro_rules! call { - ($imm8:expr) => { - vrndscalepd(a, $imm8, zero, k, _MM_FROUND_CUR_DIRECTION) - }; - } - let r = constify_imm8_sae!(imm8, call); + let r = vrndscalepd(a, IMM8, zero, k, _MM_FROUND_CUR_DIRECTION); transmute(r) } @@ -5183,17 +5140,13 @@ pub unsafe fn _mm512_maskz_roundscale_pd(k: __mmask8, a: __m512d, imm8: i32) -> /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_roundscale_pd&expand=4772) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[cfg_attr(test, assert_instr(vrndscalepd, imm8 = 0))] -#[rustc_args_required_const(1)] -pub unsafe fn _mm256_roundscale_pd(a: __m256d, imm8: i32) -> __m256d { +#[cfg_attr(test, assert_instr(vrndscalepd, IMM8 = 0))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm256_roundscale_pd(a: __m256d) -> __m256d { + static_assert_imm8!(IMM8); let a = a.as_f64x4(); let zero = _mm256_setzero_pd().as_f64x4(); - macro_rules! call { - ($imm8:expr) => { - vrndscalepd256(a, $imm8, zero, 0b00001111) - }; - } - let r = constify_imm8_sae!(imm8, call); + let r = vrndscalepd256(a, IMM8, zero, 0b00001111); transmute(r) } @@ -5208,22 +5161,17 @@ pub unsafe fn _mm256_roundscale_pd(a: __m256d, imm8: i32) -> __m256d { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_roundscale_pd&expand=4770) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[cfg_attr(test, assert_instr(vrndscalepd, imm8 = 0))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm256_mask_roundscale_pd( +#[cfg_attr(test, assert_instr(vrndscalepd, IMM8 = 0))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm256_mask_roundscale_pd( src: __m256d, k: __mmask8, a: __m256d, - imm8: i32, ) -> __m256d { + static_assert_imm8!(IMM8); let a = a.as_f64x4(); let src = src.as_f64x4(); - macro_rules! call { - ($imm8:expr) => { - vrndscalepd256(a, $imm8, src, k) - }; - } - let r = constify_imm8_sae!(imm8, call); + let r = vrndscalepd256(a, IMM8, src, k); transmute(r) } @@ -5238,17 +5186,13 @@ pub unsafe fn _mm256_mask_roundscale_pd( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_roundscale_pd&expand=4771) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[cfg_attr(test, assert_instr(vrndscalepd, imm8 = 0))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm256_maskz_roundscale_pd(k: __mmask8, a: __m256d, imm8: i32) -> __m256d { +#[cfg_attr(test, assert_instr(vrndscalepd, IMM8 = 0))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm256_maskz_roundscale_pd(k: __mmask8, a: __m256d) -> __m256d { + static_assert_imm8!(IMM8); let a = a.as_f64x4(); let zero = _mm256_setzero_pd().as_f64x4(); - macro_rules! call { - ($imm8:expr) => { - vrndscalepd256(a, $imm8, zero, k) - }; - } - let r = constify_imm8_sae!(imm8, call); + let r = vrndscalepd256(a, IMM8, zero, k); transmute(r) } @@ -5263,17 +5207,13 @@ pub unsafe fn _mm256_maskz_roundscale_pd(k: __mmask8, a: __m256d, imm8: i32) -> /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_roundscale_pd&expand=4769) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[cfg_attr(test, assert_instr(vrndscalepd, imm8 = 0))] -#[rustc_args_required_const(1)] -pub unsafe fn _mm_roundscale_pd(a: __m128d, imm8: i32) -> __m128d { +#[cfg_attr(test, assert_instr(vrndscalepd, IMM8 = 0))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm_roundscale_pd(a: __m128d) -> __m128d { + static_assert_imm8!(IMM8); let a = a.as_f64x2(); let zero = _mm_setzero_pd().as_f64x2(); - macro_rules! call { - ($imm8:expr) => { - vrndscalepd128(a, $imm8, zero, 0b00000011) - }; - } - let r = constify_imm8_sae!(imm8, call); + let r = vrndscalepd128(a, IMM8, zero, 0b00000011); transmute(r) } @@ -5288,17 +5228,17 @@ pub unsafe fn _mm_roundscale_pd(a: __m128d, imm8: i32) -> __m128d { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_roundscale_pd&expand=4767) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[cfg_attr(test, assert_instr(vrndscalepd, imm8 = 0))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm_mask_roundscale_pd(src: __m128d, k: __mmask8, a: __m128d, imm8: i32) -> __m128d { +#[cfg_attr(test, assert_instr(vrndscalepd, IMM8 = 0))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm_mask_roundscale_pd( + src: __m128d, + k: __mmask8, + a: __m128d, +) -> __m128d { + static_assert_imm8!(IMM8); let a = a.as_f64x2(); let src = src.as_f64x2(); - macro_rules! call { - ($imm8:expr) => { - vrndscalepd128(a, $imm8, src, k) - }; - } - let r = constify_imm8_sae!(imm8, call); + let r = vrndscalepd128(a, IMM8, src, k); transmute(r) } @@ -5313,17 +5253,13 @@ pub unsafe fn _mm_mask_roundscale_pd(src: __m128d, k: __mmask8, a: __m128d, imm8 /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_roundscale_pd&expand=4768) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[cfg_attr(test, assert_instr(vrndscalepd, imm8 = 0))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm_maskz_roundscale_pd(k: __mmask8, a: __m128d, imm8: i32) -> __m128d { +#[cfg_attr(test, assert_instr(vrndscalepd, IMM8 = 0))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm_maskz_roundscale_pd(k: __mmask8, a: __m128d) -> __m128d { + static_assert_imm8!(IMM8); let a = a.as_f64x2(); let zero = _mm_setzero_pd().as_f64x2(); - macro_rules! call { - ($imm8:expr) => { - vrndscalepd128(a, $imm8, zero, k) - }; - } - let r = constify_imm8_sae!(imm8, call); + let r = vrndscalepd128(a, IMM8, zero, k); transmute(r) } @@ -5588,25 +5524,14 @@ pub unsafe fn _mm_maskz_scalef_pd(k: __mmask8, a: __m128d, b: __m128d) -> __m128 /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_fixupimm_ps&expand=2499) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vfixupimmps, imm8 = 0))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm512_fixupimm_ps(a: __m512, b: __m512, c: __m512i, imm8: i32) -> __m512 { +#[cfg_attr(test, assert_instr(vfixupimmps, IMM8 = 0))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_fixupimm_ps(a: __m512, b: __m512, c: __m512i) -> __m512 { + static_assert_imm8!(IMM8); let a = a.as_f32x16(); let b = b.as_f32x16(); let c = c.as_i32x16(); - macro_rules! call { - ($imm8:expr) => { - vfixupimmps( - a, - b, - c, - $imm8, - 0b11111111_11111111, - _MM_FROUND_CUR_DIRECTION, - ) - }; - } - let r = constify_imm8_sae!(imm8, call); + let r = vfixupimmps(a, b, c, IMM8, 0b11111111_11111111, _MM_FROUND_CUR_DIRECTION); transmute(r) } @@ -5615,24 +5540,19 @@ pub unsafe fn _mm512_fixupimm_ps(a: __m512, b: __m512, c: __m512i, imm8: i32) -> /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_fixupimm_ps&expand=2500) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vfixupimmps, imm8 = 0))] -#[rustc_args_required_const(4)] -pub unsafe fn _mm512_mask_fixupimm_ps( +#[cfg_attr(test, assert_instr(vfixupimmps, IMM8 = 0))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_mask_fixupimm_ps( a: __m512, k: __mmask16, b: __m512, c: __m512i, - imm8: i32, ) -> __m512 { + static_assert_imm8!(IMM8); let a = a.as_f32x16(); let b = b.as_f32x16(); let c = c.as_i32x16(); - macro_rules! call { - ($imm8:expr) => { - vfixupimmps(a, b, c, $imm8, k, _MM_FROUND_CUR_DIRECTION) - }; - } - let r = constify_imm8_sae!(imm8, call); + let r = vfixupimmps(a, b, c, IMM8, k, _MM_FROUND_CUR_DIRECTION); transmute(r) } @@ -5641,24 +5561,19 @@ pub unsafe fn _mm512_mask_fixupimm_ps( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_fixupimm_ps&expand=2501) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vfixupimmps, imm8 = 0))] -#[rustc_args_required_const(4)] -pub unsafe fn _mm512_maskz_fixupimm_ps( +#[cfg_attr(test, assert_instr(vfixupimmps, IMM8 = 0))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_maskz_fixupimm_ps( k: __mmask16, a: __m512, b: __m512, c: __m512i, - imm8: i32, ) -> __m512 { + static_assert_imm8!(IMM8); let a = a.as_f32x16(); let b = b.as_f32x16(); let c = c.as_i32x16(); - macro_rules! call { - ($imm8:expr) => { - vfixupimmpsz(a, b, c, $imm8, k, _MM_FROUND_CUR_DIRECTION) - }; - } - let r = constify_imm8_sae!(imm8, call); + let r = vfixupimmpsz(a, b, c, IMM8, k, _MM_FROUND_CUR_DIRECTION); transmute(r) } @@ -5667,18 +5582,14 @@ pub unsafe fn _mm512_maskz_fixupimm_ps( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_fixupimm_ps&expand=2496) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[cfg_attr(test, assert_instr(vfixupimmps, imm8 = 0))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm256_fixupimm_ps(a: __m256, b: __m256, c: __m256i, imm8: i32) -> __m256 { +#[cfg_attr(test, assert_instr(vfixupimmps, IMM8 = 0))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm256_fixupimm_ps(a: __m256, b: __m256, c: __m256i) -> __m256 { + static_assert_imm8!(IMM8); let a = a.as_f32x8(); let b = b.as_f32x8(); let c = c.as_i32x8(); - macro_rules! call { - ($imm8:expr) => { - vfixupimmps256(a, b, c, $imm8, 0b11111111) - }; - } - let r = constify_imm8_sae!(imm8, call); + let r = vfixupimmps256(a, b, c, IMM8, 0b11111111); transmute(r) } @@ -5687,24 +5598,19 @@ pub unsafe fn _mm256_fixupimm_ps(a: __m256, b: __m256, c: __m256i, imm8: i32) -> /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_fixupimm_ps&expand=2497) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[cfg_attr(test, assert_instr(vfixupimmps, imm8 = 0))] -#[rustc_args_required_const(4)] -pub unsafe fn _mm256_mask_fixupimm_ps( +#[cfg_attr(test, assert_instr(vfixupimmps, IMM8 = 0))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm256_mask_fixupimm_ps( a: __m256, k: __mmask8, b: __m256, c: __m256i, - imm8: i32, ) -> __m256 { + static_assert_imm8!(IMM8); let a = a.as_f32x8(); let b = b.as_f32x8(); let c = c.as_i32x8(); - macro_rules! call { - ($imm8:expr) => { - vfixupimmps256(a, b, c, $imm8, k) - }; - } - let r = constify_imm8_sae!(imm8, call); + let r = vfixupimmps256(a, b, c, IMM8, k); transmute(r) } @@ -5713,24 +5619,19 @@ pub unsafe fn _mm256_mask_fixupimm_ps( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_fixupimm_ps&expand=2498) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[cfg_attr(test, assert_instr(vfixupimmps, imm8 = 0))] -#[rustc_args_required_const(4)] -pub unsafe fn _mm256_maskz_fixupimm_ps( +#[cfg_attr(test, assert_instr(vfixupimmps, IMM8 = 0))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm256_maskz_fixupimm_ps( k: __mmask8, a: __m256, b: __m256, c: __m256i, - imm8: i32, ) -> __m256 { + static_assert_imm8!(IMM8); let a = a.as_f32x8(); let b = b.as_f32x8(); let c = c.as_i32x8(); - macro_rules! call { - ($imm8:expr) => { - vfixupimmpsz256(a, b, c, $imm8, k) - }; - } - let r = constify_imm8_sae!(imm8, call); + let r = vfixupimmpsz256(a, b, c, IMM8, k); transmute(r) } @@ -5739,18 +5640,14 @@ pub unsafe fn _mm256_maskz_fixupimm_ps( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_fixupimm_ps&expand=2493) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[cfg_attr(test, assert_instr(vfixupimmps, imm8 = 0))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm_fixupimm_ps(a: __m128, b: __m128, c: __m128i, imm8: i32) -> __m128 { +#[cfg_attr(test, assert_instr(vfixupimmps, IMM8 = 0))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm_fixupimm_ps(a: __m128, b: __m128, c: __m128i) -> __m128 { + static_assert_imm8!(IMM8); let a = a.as_f32x4(); let b = b.as_f32x4(); let c = c.as_i32x4(); - macro_rules! call { - ($imm8:expr) => { - vfixupimmps128(a, b, c, $imm8, 0b00001111) - }; - } - let r = constify_imm8_sae!(imm8, call); + let r = vfixupimmps128(a, b, c, IMM8, 0b00001111); transmute(r) } @@ -5759,24 +5656,19 @@ pub unsafe fn _mm_fixupimm_ps(a: __m128, b: __m128, c: __m128i, imm8: i32) -> __ /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_fixupimm_ps&expand=2494) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[cfg_attr(test, assert_instr(vfixupimmps, imm8 = 0))] -#[rustc_args_required_const(4)] -pub unsafe fn _mm_mask_fixupimm_ps( +#[cfg_attr(test, assert_instr(vfixupimmps, IMM8 = 0))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm_mask_fixupimm_ps( a: __m128, k: __mmask8, b: __m128, c: __m128i, - imm8: i32, ) -> __m128 { + static_assert_imm8!(IMM8); let a = a.as_f32x4(); let b = b.as_f32x4(); let c = c.as_i32x4(); - macro_rules! call { - ($imm8:expr) => { - vfixupimmps128(a, b, c, $imm8, k) - }; - } - let r = constify_imm8_sae!(imm8, call); + let r = vfixupimmps128(a, b, c, IMM8, k); transmute(r) } @@ -5785,24 +5677,19 @@ pub unsafe fn _mm_mask_fixupimm_ps( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_fixupimm_ps&expand=2495) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[cfg_attr(test, assert_instr(vfixupimmps, imm8 = 0))] -#[rustc_args_required_const(4)] -pub unsafe fn _mm_maskz_fixupimm_ps( +#[cfg_attr(test, assert_instr(vfixupimmps, IMM8 = 0))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm_maskz_fixupimm_ps( k: __mmask8, a: __m128, b: __m128, c: __m128i, - imm8: i32, ) -> __m128 { + static_assert_imm8!(IMM8); let a = a.as_f32x4(); let b = b.as_f32x4(); let c = c.as_i32x4(); - macro_rules! call { - ($imm8:expr) => { - vfixupimmpsz128(a, b, c, $imm8, k) - }; - } - let r = constify_imm8_sae!(imm8, call); + let r = vfixupimmpsz128(a, b, c, IMM8, k); transmute(r) } @@ -5811,18 +5698,14 @@ pub unsafe fn _mm_maskz_fixupimm_ps( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_fixupimm_pd&expand=2490) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vfixupimmpd, imm8 = 0))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm512_fixupimm_pd(a: __m512d, b: __m512d, c: __m512i, imm8: i32) -> __m512d { +#[cfg_attr(test, assert_instr(vfixupimmpd, IMM8 = 0))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_fixupimm_pd(a: __m512d, b: __m512d, c: __m512i) -> __m512d { + static_assert_imm8!(IMM8); let a = a.as_f64x8(); let b = b.as_f64x8(); let c = c.as_i64x8(); - macro_rules! call { - ($imm8:expr) => { - vfixupimmpd(a, b, c, $imm8, 0b11111111, _MM_FROUND_CUR_DIRECTION) - }; - } - let r = constify_imm8_sae!(imm8, call); + let r = vfixupimmpd(a, b, c, IMM8, 0b11111111, _MM_FROUND_CUR_DIRECTION); transmute(r) } @@ -5831,24 +5714,19 @@ pub unsafe fn _mm512_fixupimm_pd(a: __m512d, b: __m512d, c: __m512i, imm8: i32) /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_fixupimm_pd&expand=2491) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vfixupimmpd, imm8 = 0))] -#[rustc_args_required_const(4)] -pub unsafe fn _mm512_mask_fixupimm_pd( +#[cfg_attr(test, assert_instr(vfixupimmpd, IMM8 = 0))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_mask_fixupimm_pd( a: __m512d, k: __mmask8, b: __m512d, c: __m512i, - imm8: i32, ) -> __m512d { + static_assert_imm8!(IMM8); let a = a.as_f64x8(); let b = b.as_f64x8(); let c = c.as_i64x8(); - macro_rules! call { - ($imm8:expr) => { - vfixupimmpd(a, b, c, $imm8, k, _MM_FROUND_CUR_DIRECTION) - }; - } - let r = constify_imm8_sae!(imm8, call); + let r = vfixupimmpd(a, b, c, IMM8, k, _MM_FROUND_CUR_DIRECTION); transmute(r) } @@ -5857,24 +5735,19 @@ pub unsafe fn _mm512_mask_fixupimm_pd( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_fixupimm_pd&expand=2492) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vfixupimmpd, imm8 = 0))] -#[rustc_args_required_const(4)] -pub unsafe fn _mm512_maskz_fixupimm_pd( +#[cfg_attr(test, assert_instr(vfixupimmpd, IMM8 = 0))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_maskz_fixupimm_pd( k: __mmask8, a: __m512d, b: __m512d, c: __m512i, - imm8: i32, ) -> __m512d { + static_assert_imm8!(IMM8); let a = a.as_f64x8(); let b = b.as_f64x8(); let c = c.as_i64x8(); - macro_rules! call { - ($imm8:expr) => { - vfixupimmpdz(a, b, c, $imm8, k, _MM_FROUND_CUR_DIRECTION) - }; - } - let r = constify_imm8_sae!(imm8, call); + let r = vfixupimmpdz(a, b, c, IMM8, k, _MM_FROUND_CUR_DIRECTION); transmute(r) } @@ -5883,18 +5756,14 @@ pub unsafe fn _mm512_maskz_fixupimm_pd( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_fixupimm_pd&expand=2487) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[cfg_attr(test, assert_instr(vfixupimmpd, imm8 = 0))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm256_fixupimm_pd(a: __m256d, b: __m256d, c: __m256i, imm8: i32) -> __m256d { +#[cfg_attr(test, assert_instr(vfixupimmpd, IMM8 = 0))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm256_fixupimm_pd(a: __m256d, b: __m256d, c: __m256i) -> __m256d { + static_assert_imm8!(IMM8); let a = a.as_f64x4(); let b = b.as_f64x4(); let c = c.as_i64x4(); - macro_rules! call { - ($imm8:expr) => { - vfixupimmpd256(a, b, c, $imm8, 0b00001111) - }; - } - let r = constify_imm8_sae!(imm8, call); + let r = vfixupimmpd256(a, b, c, IMM8, 0b00001111); transmute(r) } @@ -5903,24 +5772,19 @@ pub unsafe fn _mm256_fixupimm_pd(a: __m256d, b: __m256d, c: __m256i, imm8: i32) /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_fixupimm_pd&expand=2488) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[cfg_attr(test, assert_instr(vfixupimmpd, imm8 = 0))] -#[rustc_args_required_const(4)] -pub unsafe fn _mm256_mask_fixupimm_pd( +#[cfg_attr(test, assert_instr(vfixupimmpd, IMM8 = 0))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm256_mask_fixupimm_pd( a: __m256d, k: __mmask8, b: __m256d, c: __m256i, - imm8: i32, ) -> __m256d { + static_assert_imm8!(IMM8); let a = a.as_f64x4(); let b = b.as_f64x4(); let c = c.as_i64x4(); - macro_rules! call { - ($imm8:expr) => { - vfixupimmpd256(a, b, c, $imm8, k) - }; - } - let r = constify_imm8_sae!(imm8, call); + let r = vfixupimmpd256(a, b, c, IMM8, k); transmute(r) } @@ -5929,24 +5793,19 @@ pub unsafe fn _mm256_mask_fixupimm_pd( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_fixupimm_pd&expand=2489) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[cfg_attr(test, assert_instr(vfixupimmpd, imm8 = 0))] -#[rustc_args_required_const(4)] -pub unsafe fn _mm256_maskz_fixupimm_pd( +#[cfg_attr(test, assert_instr(vfixupimmpd, IMM8 = 0))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm256_maskz_fixupimm_pd( k: __mmask8, a: __m256d, b: __m256d, c: __m256i, - imm8: i32, ) -> __m256d { + static_assert_imm8!(IMM8); let a = a.as_f64x4(); let b = b.as_f64x4(); let c = c.as_i64x4(); - macro_rules! call { - ($imm8:expr) => { - vfixupimmpdz256(a, b, c, $imm8, k) - }; - } - let r = constify_imm8_sae!(imm8, call); + let r = vfixupimmpdz256(a, b, c, IMM8, k); transmute(r) } @@ -5955,18 +5814,14 @@ pub unsafe fn _mm256_maskz_fixupimm_pd( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_fixupimm_pd&expand=2484) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[cfg_attr(test, assert_instr(vfixupimmpd, imm8 = 0))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm_fixupimm_pd(a: __m128d, b: __m128d, c: __m128i, imm8: i32) -> __m128d { +#[cfg_attr(test, assert_instr(vfixupimmpd, IMM8 = 0))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm_fixupimm_pd(a: __m128d, b: __m128d, c: __m128i) -> __m128d { + static_assert_imm8!(IMM8); let a = a.as_f64x2(); let b = b.as_f64x2(); let c = c.as_i64x2(); - macro_rules! call { - ($imm8:expr) => { - vfixupimmpd128(a, b, c, $imm8, 0b00000011) - }; - } - let r = constify_imm8_sae!(imm8, call); + let r = vfixupimmpd128(a, b, c, IMM8, 0b00000011); transmute(r) } @@ -5975,24 +5830,19 @@ pub unsafe fn _mm_fixupimm_pd(a: __m128d, b: __m128d, c: __m128i, imm8: i32) -> /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_fixupimm_pd&expand=2485) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[cfg_attr(test, assert_instr(vfixupimmpd, imm8 = 0))] -#[rustc_args_required_const(4)] -pub unsafe fn _mm_mask_fixupimm_pd( +#[cfg_attr(test, assert_instr(vfixupimmpd, IMM8 = 0))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm_mask_fixupimm_pd( a: __m128d, k: __mmask8, b: __m128d, c: __m128i, - imm8: i32, ) -> __m128d { + static_assert_imm8!(IMM8); let a = a.as_f64x2(); let b = b.as_f64x2(); let c = c.as_i64x2(); - macro_rules! call { - ($imm8:expr) => { - vfixupimmpd128(a, b, c, $imm8, k) - }; - } - let r = constify_imm8_sae!(imm8, call); + let r = vfixupimmpd128(a, b, c, IMM8, k); transmute(r) } @@ -6001,24 +5851,19 @@ pub unsafe fn _mm_mask_fixupimm_pd( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_fixupimm_pd&expand=2486) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[cfg_attr(test, assert_instr(vfixupimmpd, imm8 = 0))] -#[rustc_args_required_const(4)] -pub unsafe fn _mm_maskz_fixupimm_pd( +#[cfg_attr(test, assert_instr(vfixupimmpd, IMM8 = 0))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm_maskz_fixupimm_pd( k: __mmask8, a: __m128d, b: __m128d, c: __m128i, - imm8: i32, ) -> __m128d { + static_assert_imm8!(IMM8); let a = a.as_f64x2(); let b = b.as_f64x2(); let c = c.as_i64x2(); - macro_rules! call { - ($imm8:expr) => { - vfixupimmpdz128(a, b, c, $imm8, k) - }; - } - let r = constify_imm8_sae!(imm8, call); + let r = vfixupimmpdz128(a, b, c, IMM8, k); transmute(r) } @@ -6027,18 +5872,18 @@ pub unsafe fn _mm_maskz_fixupimm_pd( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_ternarylogic_epi32&expand=5867) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vpternlogd, imm8 = 114))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm512_ternarylogic_epi32(a: __m512i, b: __m512i, c: __m512i, imm8: i32) -> __m512i { +#[cfg_attr(test, assert_instr(vpternlogd, IMM8 = 114))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_ternarylogic_epi32( + a: __m512i, + b: __m512i, + c: __m512i, +) -> __m512i { + static_assert_imm8!(IMM8); let a = a.as_i32x16(); let b = b.as_i32x16(); let c = c.as_i32x16(); - macro_rules! call { - ($imm8:expr) => { - vpternlogd(a, b, c, $imm8) - }; - } - let r = constify_imm8_sae!(imm8, call); + let r = vpternlogd(a, b, c, IMM8); transmute(r) } @@ -6047,25 +5892,20 @@ pub unsafe fn _mm512_ternarylogic_epi32(a: __m512i, b: __m512i, c: __m512i, imm8 /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_ternarylogic_epi32&expand=5865) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vpternlogd, imm8 = 114))] -#[rustc_args_required_const(4)] -pub unsafe fn _mm512_mask_ternarylogic_epi32( +#[cfg_attr(test, assert_instr(vpternlogd, IMM8 = 114))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_mask_ternarylogic_epi32( src: __m512i, k: __mmask16, a: __m512i, b: __m512i, - imm8: i32, ) -> __m512i { + static_assert_imm8!(IMM8); let src = src.as_i32x16(); let a = a.as_i32x16(); let b = b.as_i32x16(); - macro_rules! call { - ($imm8:expr) => { - vpternlogd(src, a, b, $imm8) - }; - } - let ternarylogic = constify_imm8_sae!(imm8, call); - transmute(simd_select_bitmask(k, ternarylogic, src)) + let r = vpternlogd(src, a, b, IMM8); + transmute(simd_select_bitmask(k, r, src)) } /// Bitwise ternary logic that provides the capability to implement any three-operand binary function; the specific binary function is specified by value in imm8. For each bit in each packed 32-bit integer, the corresponding bit from a, b, and c are used to form a 3 bit index into imm8, and the value at that bit in imm8 is written to the corresponding bit in dst using zeromask k at 32-bit granularity (32-bit elements are zeroed out when the corresponding mask bit is not set). @@ -6073,26 +5913,21 @@ pub unsafe fn _mm512_mask_ternarylogic_epi32( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_ternarylogic_epi32&expand=5866) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vpternlogd, imm8 = 114))] -#[rustc_args_required_const(4)] -pub unsafe fn _mm512_maskz_ternarylogic_epi32( +#[cfg_attr(test, assert_instr(vpternlogd, IMM8 = 114))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_maskz_ternarylogic_epi32( k: __mmask16, a: __m512i, b: __m512i, c: __m512i, - imm8: i32, ) -> __m512i { + static_assert_imm8!(IMM8); let a = a.as_i32x16(); let b = b.as_i32x16(); let c = c.as_i32x16(); - macro_rules! call { - ($imm8:expr) => { - vpternlogd(a, b, c, $imm8) - }; - } - let ternarylogic = constify_imm8_sae!(imm8, call); + let r = vpternlogd(a, b, c, IMM8); let zero = _mm512_setzero_si512().as_i32x16(); - transmute(simd_select_bitmask(k, ternarylogic, zero)) + transmute(simd_select_bitmask(k, r, zero)) } /// Bitwise ternary logic that provides the capability to implement any three-operand binary function; the specific binary function is specified by value in imm8. For each bit in each packed 32-bit integer, the corresponding bit from a, b, and c are used to form a 3 bit index into imm8, and the value at that bit in imm8 is written to the corresponding bit in dst. @@ -6100,18 +5935,18 @@ pub unsafe fn _mm512_maskz_ternarylogic_epi32( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_ternarylogic_epi32&expand=5864) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[cfg_attr(test, assert_instr(vpternlogd, imm8 = 114))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm256_ternarylogic_epi32(a: __m256i, b: __m256i, c: __m256i, imm8: i32) -> __m256i { +#[cfg_attr(test, assert_instr(vpternlogd, IMM8 = 114))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm256_ternarylogic_epi32( + a: __m256i, + b: __m256i, + c: __m256i, +) -> __m256i { + static_assert_imm8!(IMM8); let a = a.as_i32x8(); let b = b.as_i32x8(); let c = c.as_i32x8(); - macro_rules! call { - ($imm8:expr) => { - vpternlogd256(a, b, c, $imm8) - }; - } - let r = constify_imm8_sae!(imm8, call); + let r = vpternlogd256(a, b, c, IMM8); transmute(r) } @@ -6120,25 +5955,20 @@ pub unsafe fn _mm256_ternarylogic_epi32(a: __m256i, b: __m256i, c: __m256i, imm8 /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_ternarylogic_epi32&expand=5862) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[cfg_attr(test, assert_instr(vpternlogd, imm8 = 114))] -#[rustc_args_required_const(4)] -pub unsafe fn _mm256_mask_ternarylogic_epi32( +#[cfg_attr(test, assert_instr(vpternlogd, IMM8 = 114))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm256_mask_ternarylogic_epi32( src: __m256i, k: __mmask8, a: __m256i, b: __m256i, - imm8: i32, ) -> __m256i { + static_assert_imm8!(IMM8); let src = src.as_i32x8(); let a = a.as_i32x8(); let b = b.as_i32x8(); - macro_rules! call { - ($imm8:expr) => { - vpternlogd256(src, a, b, $imm8) - }; - } - let ternarylogic = constify_imm8_sae!(imm8, call); - transmute(simd_select_bitmask(k, ternarylogic, src)) + let r = vpternlogd256(src, a, b, IMM8); + transmute(simd_select_bitmask(k, r, src)) } /// Bitwise ternary logic that provides the capability to implement any three-operand binary function; the specific binary function is specified by value in imm8. For each bit in each packed 32-bit integer, the corresponding bit from a, b, and c are used to form a 3 bit index into imm8, and the value at that bit in imm8 is written to the corresponding bit in dst using zeromask k at 32-bit granularity (32-bit elements are zeroed out when the corresponding mask bit is not set). @@ -6146,26 +5976,21 @@ pub unsafe fn _mm256_mask_ternarylogic_epi32( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_ternarylogic_epi32&expand=5863) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[cfg_attr(test, assert_instr(vpternlogd, imm8 = 114))] -#[rustc_args_required_const(4)] -pub unsafe fn _mm256_maskz_ternarylogic_epi32( +#[cfg_attr(test, assert_instr(vpternlogd, IMM8 = 114))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm256_maskz_ternarylogic_epi32( k: __mmask8, a: __m256i, b: __m256i, c: __m256i, - imm8: i32, ) -> __m256i { + static_assert_imm8!(IMM8); let a = a.as_i32x8(); let b = b.as_i32x8(); let c = c.as_i32x8(); - macro_rules! call { - ($imm8:expr) => { - vpternlogd256(a, b, c, $imm8) - }; - } - let ternarylogic = constify_imm8_sae!(imm8, call); + let r = vpternlogd256(a, b, c, IMM8); let zero = _mm256_setzero_si256().as_i32x8(); - transmute(simd_select_bitmask(k, ternarylogic, zero)) + transmute(simd_select_bitmask(k, r, zero)) } /// Bitwise ternary logic that provides the capability to implement any three-operand binary function; the specific binary function is specified by value in imm8. For each bit in each packed 32-bit integer, the corresponding bit from a, b, and c are used to form a 3 bit index into imm8, and the value at that bit in imm8 is written to the corresponding bit in dst. @@ -6173,18 +5998,18 @@ pub unsafe fn _mm256_maskz_ternarylogic_epi32( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_ternarylogic_epi32&expand=5861) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[cfg_attr(test, assert_instr(vpternlogd, imm8 = 114))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm_ternarylogic_epi32(a: __m128i, b: __m128i, c: __m128i, imm8: i32) -> __m128i { +#[cfg_attr(test, assert_instr(vpternlogd, IMM8 = 114))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm_ternarylogic_epi32( + a: __m128i, + b: __m128i, + c: __m128i, +) -> __m128i { + static_assert_imm8!(IMM8); let a = a.as_i32x4(); let b = b.as_i32x4(); let c = c.as_i32x4(); - macro_rules! call { - ($imm8:expr) => { - vpternlogd128(a, b, c, $imm8) - }; - } - let r = constify_imm8_sae!(imm8, call); + let r = vpternlogd128(a, b, c, IMM8); transmute(r) } @@ -6193,25 +6018,20 @@ pub unsafe fn _mm_ternarylogic_epi32(a: __m128i, b: __m128i, c: __m128i, imm8: i /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_ternarylogic_epi32&expand=5859) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[cfg_attr(test, assert_instr(vpternlogd, imm8 = 114))] -#[rustc_args_required_const(4)] -pub unsafe fn _mm_mask_ternarylogic_epi32( +#[cfg_attr(test, assert_instr(vpternlogd, IMM8 = 114))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm_mask_ternarylogic_epi32( src: __m128i, k: __mmask8, a: __m128i, b: __m128i, - imm8: i32, ) -> __m128i { + static_assert_imm8!(IMM8); let src = src.as_i32x4(); let a = a.as_i32x4(); let b = b.as_i32x4(); - macro_rules! call { - ($imm8:expr) => { - vpternlogd128(src, a, b, $imm8) - }; - } - let ternarylogic = constify_imm8_sae!(imm8, call); - transmute(simd_select_bitmask(k, ternarylogic, src)) + let r = vpternlogd128(src, a, b, IMM8); + transmute(simd_select_bitmask(k, r, src)) } /// Bitwise ternary logic that provides the capability to implement any three-operand binary function; the specific binary function is specified by value in imm8. For each bit in each packed 32-bit integer, the corresponding bit from a, b, and c are used to form a 3 bit index into imm8, and the value at that bit in imm8 is written to the corresponding bit in dst using zeromask k at 32-bit granularity (32-bit elements are zeroed out when the corresponding mask bit is not set). @@ -6219,26 +6039,21 @@ pub unsafe fn _mm_mask_ternarylogic_epi32( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_ternarylogic_epi32&expand=5860) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[cfg_attr(test, assert_instr(vpternlogd, imm8 = 114))] -#[rustc_args_required_const(4)] -pub unsafe fn _mm_maskz_ternarylogic_epi32( +#[cfg_attr(test, assert_instr(vpternlogd, IMM8 = 114))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm_maskz_ternarylogic_epi32( k: __mmask8, a: __m128i, b: __m128i, c: __m128i, - imm8: i32, ) -> __m128i { + static_assert_imm8!(IMM8); let a = a.as_i32x4(); let b = b.as_i32x4(); let c = c.as_i32x4(); - macro_rules! call { - ($imm8:expr) => { - vpternlogd128(a, b, c, $imm8) - }; - } - let ternarylogic = constify_imm8_sae!(imm8, call); + let r = vpternlogd128(a, b, c, IMM8); let zero = _mm_setzero_si128().as_i32x4(); - transmute(simd_select_bitmask(k, ternarylogic, zero)) + transmute(simd_select_bitmask(k, r, zero)) } /// Bitwise ternary logic that provides the capability to implement any three-operand binary function; the specific binary function is specified by value in imm8. For each bit in each packed 64-bit integer, the corresponding bit from a, b, and c are used to form a 3 bit index into imm8, and the value at that bit in imm8 is written to the corresponding bit in dst. @@ -6246,18 +6061,18 @@ pub unsafe fn _mm_maskz_ternarylogic_epi32( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_ternarylogic_epi64&expand=5876) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vpternlogq, imm8 = 114))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm512_ternarylogic_epi64(a: __m512i, b: __m512i, c: __m512i, imm8: i32) -> __m512i { +#[cfg_attr(test, assert_instr(vpternlogq, IMM8 = 114))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_ternarylogic_epi64( + a: __m512i, + b: __m512i, + c: __m512i, +) -> __m512i { + static_assert_imm8!(IMM8); let a = a.as_i64x8(); let b = b.as_i64x8(); let c = c.as_i64x8(); - macro_rules! call { - ($imm8:expr) => { - vpternlogq(a, b, c, $imm8) - }; - } - let r = constify_imm8_sae!(imm8, call); + let r = vpternlogq(a, b, c, IMM8); transmute(r) } @@ -6266,25 +6081,20 @@ pub unsafe fn _mm512_ternarylogic_epi64(a: __m512i, b: __m512i, c: __m512i, imm8 /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_ternarylogic_epi64&expand=5874) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vpternlogq, imm8 = 114))] -#[rustc_args_required_const(4)] -pub unsafe fn _mm512_mask_ternarylogic_epi64( +#[cfg_attr(test, assert_instr(vpternlogq, IMM8 = 114))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_mask_ternarylogic_epi64( src: __m512i, k: __mmask8, a: __m512i, b: __m512i, - imm8: i32, ) -> __m512i { + static_assert_imm8!(IMM8); let src = src.as_i64x8(); let a = a.as_i64x8(); let b = b.as_i64x8(); - macro_rules! call { - ($imm8:expr) => { - vpternlogq(src, a, b, $imm8) - }; - } - let ternarylogic = constify_imm8_sae!(imm8, call); - transmute(simd_select_bitmask(k, ternarylogic, src)) + let r = vpternlogq(src, a, b, IMM8); + transmute(simd_select_bitmask(k, r, src)) } /// Bitwise ternary logic that provides the capability to implement any three-operand binary function; the specific binary function is specified by value in imm8. For each bit in each packed 64-bit integer, the corresponding bit from a, b, and c are used to form a 3 bit index into imm8, and the value at that bit in imm8 is written to the corresponding bit in dst using zeromask k at 64-bit granularity (64-bit elements are zeroed out when the corresponding mask bit is not set). @@ -6292,26 +6102,21 @@ pub unsafe fn _mm512_mask_ternarylogic_epi64( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_ternarylogic_epi64&expand=5875) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vpternlogq, imm8 = 114))] -#[rustc_args_required_const(4)] -pub unsafe fn _mm512_maskz_ternarylogic_epi64( +#[cfg_attr(test, assert_instr(vpternlogq, IMM8 = 114))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_maskz_ternarylogic_epi64( k: __mmask8, a: __m512i, b: __m512i, c: __m512i, - imm8: i32, ) -> __m512i { + static_assert_imm8!(IMM8); let a = a.as_i64x8(); let b = b.as_i64x8(); let c = c.as_i64x8(); - macro_rules! call { - ($imm8:expr) => { - vpternlogq(a, b, c, $imm8) - }; - } - let ternarylogic = constify_imm8_sae!(imm8, call); + let r = vpternlogq(a, b, c, IMM8); let zero = _mm512_setzero_si512().as_i64x8(); - transmute(simd_select_bitmask(k, ternarylogic, zero)) + transmute(simd_select_bitmask(k, r, zero)) } /// Bitwise ternary logic that provides the capability to implement any three-operand binary function; the specific binary function is specified by value in imm8. For each bit in each packed 64-bit integer, the corresponding bit from a, b, and c are used to form a 3 bit index into imm8, and the value at that bit in imm8 is written to the corresponding bit in dst. @@ -6319,18 +6124,18 @@ pub unsafe fn _mm512_maskz_ternarylogic_epi64( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_ternarylogic_epi64&expand=5873) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[cfg_attr(test, assert_instr(vpternlogq, imm8 = 114))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm256_ternarylogic_epi64(a: __m256i, b: __m256i, c: __m256i, imm8: i32) -> __m256i { +#[cfg_attr(test, assert_instr(vpternlogq, IMM8 = 114))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm256_ternarylogic_epi64( + a: __m256i, + b: __m256i, + c: __m256i, +) -> __m256i { + static_assert_imm8!(IMM8); let a = a.as_i64x4(); let b = b.as_i64x4(); let c = c.as_i64x4(); - macro_rules! call { - ($imm8:expr) => { - vpternlogq256(a, b, c, $imm8) - }; - } - let r = constify_imm8_sae!(imm8, call); + let r = vpternlogq256(a, b, c, IMM8); transmute(r) } @@ -6339,25 +6144,20 @@ pub unsafe fn _mm256_ternarylogic_epi64(a: __m256i, b: __m256i, c: __m256i, imm8 /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_ternarylogic_epi64&expand=5871) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[cfg_attr(test, assert_instr(vpternlogq, imm8 = 114))] -#[rustc_args_required_const(4)] -pub unsafe fn _mm256_mask_ternarylogic_epi64( +#[cfg_attr(test, assert_instr(vpternlogq, IMM8 = 114))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm256_mask_ternarylogic_epi64( src: __m256i, k: __mmask8, a: __m256i, b: __m256i, - imm8: i32, ) -> __m256i { + static_assert_imm8!(IMM8); let src = src.as_i64x4(); let a = a.as_i64x4(); let b = b.as_i64x4(); - macro_rules! call { - ($imm8:expr) => { - vpternlogq256(src, a, b, $imm8) - }; - } - let ternarylogic = constify_imm8_sae!(imm8, call); - transmute(simd_select_bitmask(k, ternarylogic, src)) + let r = vpternlogq256(src, a, b, IMM8); + transmute(simd_select_bitmask(k, r, src)) } /// Bitwise ternary logic that provides the capability to implement any three-operand binary function; the specific binary function is specified by value in imm8. For each bit in each packed 64-bit integer, the corresponding bit from a, b, and c are used to form a 3 bit index into imm8, and the value at that bit in imm8 is written to the corresponding bit in dst using zeromask k at 64-bit granularity (64-bit elements are zeroed out when the corresponding mask bit is not set). @@ -6365,26 +6165,21 @@ pub unsafe fn _mm256_mask_ternarylogic_epi64( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_ternarylogic_epi64&expand=5872) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[cfg_attr(test, assert_instr(vpternlogq, imm8 = 114))] -#[rustc_args_required_const(4)] -pub unsafe fn _mm256_maskz_ternarylogic_epi64( +#[cfg_attr(test, assert_instr(vpternlogq, IMM8 = 114))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm256_maskz_ternarylogic_epi64( k: __mmask8, a: __m256i, b: __m256i, c: __m256i, - imm8: i32, ) -> __m256i { + static_assert_imm8!(IMM8); let a = a.as_i64x4(); let b = b.as_i64x4(); let c = c.as_i64x4(); - macro_rules! call { - ($imm8:expr) => { - vpternlogq256(a, b, c, $imm8) - }; - } - let ternarylogic = constify_imm8_sae!(imm8, call); + let r = vpternlogq256(a, b, c, IMM8); let zero = _mm256_setzero_si256().as_i64x4(); - transmute(simd_select_bitmask(k, ternarylogic, zero)) + transmute(simd_select_bitmask(k, r, zero)) } /// Bitwise ternary logic that provides the capability to implement any three-operand binary function; the specific binary function is specified by value in imm8. For each bit in each packed 64-bit integer, the corresponding bit from a, b, and c are used to form a 3 bit index into imm8, and the value at that bit in imm8 is written to the corresponding bit in dst. @@ -6392,18 +6187,18 @@ pub unsafe fn _mm256_maskz_ternarylogic_epi64( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_ternarylogic_epi64&expand=5870) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[cfg_attr(test, assert_instr(vpternlogq, imm8 = 114))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm_ternarylogic_epi64(a: __m128i, b: __m128i, c: __m128i, imm8: i32) -> __m128i { +#[cfg_attr(test, assert_instr(vpternlogq, IMM8 = 114))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm_ternarylogic_epi64( + a: __m128i, + b: __m128i, + c: __m128i, +) -> __m128i { + static_assert_imm8!(IMM8); let a = a.as_i64x2(); let b = b.as_i64x2(); let c = c.as_i64x2(); - macro_rules! call { - ($imm8:expr) => { - vpternlogq128(a, b, c, $imm8) - }; - } - let r = constify_imm8_sae!(imm8, call); + let r = vpternlogq128(a, b, c, IMM8); transmute(r) } @@ -6412,25 +6207,20 @@ pub unsafe fn _mm_ternarylogic_epi64(a: __m128i, b: __m128i, c: __m128i, imm8: i /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_ternarylogic_epi64&expand=5868) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[cfg_attr(test, assert_instr(vpternlogq, imm8 = 114))] -#[rustc_args_required_const(4)] -pub unsafe fn _mm_mask_ternarylogic_epi64( +#[cfg_attr(test, assert_instr(vpternlogq, IMM8 = 114))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm_mask_ternarylogic_epi64( src: __m128i, k: __mmask8, a: __m128i, b: __m128i, - imm8: i32, ) -> __m128i { + static_assert_imm8!(IMM8); let src = src.as_i64x2(); let a = a.as_i64x2(); let b = b.as_i64x2(); - macro_rules! call { - ($imm8:expr) => { - vpternlogq128(src, a, b, $imm8) - }; - } - let ternarylogic = constify_imm8_sae!(imm8, call); - transmute(simd_select_bitmask(k, ternarylogic, src)) + let r = vpternlogq128(src, a, b, IMM8); + transmute(simd_select_bitmask(k, r, src)) } /// Bitwise ternary logic that provides the capability to implement any three-operand binary function; the specific binary function is specified by value in imm8. For each bit in each packed 64-bit integer, the corresponding bit from a, b, and c are used to form a 3 bit index into imm8, and the value at that bit in imm8 is written to the corresponding bit in dst using zeromask k at 64-bit granularity (64-bit elements are zeroed out when the corresponding mask bit is not set). @@ -6438,26 +6228,21 @@ pub unsafe fn _mm_mask_ternarylogic_epi64( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_ternarylogic_epi64&expand=5869) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[cfg_attr(test, assert_instr(vpternlogq, imm8 = 114))] -#[rustc_args_required_const(4)] -pub unsafe fn _mm_maskz_ternarylogic_epi64( +#[cfg_attr(test, assert_instr(vpternlogq, IMM8 = 114))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm_maskz_ternarylogic_epi64( k: __mmask8, a: __m128i, b: __m128i, c: __m128i, - imm8: i32, ) -> __m128i { + static_assert_imm8!(IMM8); let a = a.as_i64x2(); let b = b.as_i64x2(); let c = c.as_i64x2(); - macro_rules! call { - ($imm8:expr) => { - vpternlogq128(a, b, c, $imm8) - }; - } - let ternarylogic = constify_imm8_sae!(imm8, call); + let r = vpternlogq128(a, b, c, IMM8); let zero = _mm_setzero_si128().as_i64x2(); - transmute(simd_select_bitmask(k, ternarylogic, zero)) + transmute(simd_select_bitmask(k, r, zero)) } /// Normalize the mantissas of packed single-precision (32-bit) floating-point elements in a, and store the results in dst. This intrinsic essentially calculates ±(2^k)*|x.significand|, where k depends on the interval range defined by interv and the sign depends on sc and the source sign. @@ -6474,27 +6259,25 @@ pub unsafe fn _mm_maskz_ternarylogic_epi64( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_getmant_ps&expand=2880) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vgetmantps, norm = 0, sign = 0))] -#[rustc_args_required_const(1, 2)] -pub unsafe fn _mm512_getmant_ps( +#[cfg_attr(test, assert_instr(vgetmantps, NORM = 0, SIGN = 0))] +#[rustc_legacy_const_generics(1, 2)] +pub unsafe fn _mm512_getmant_ps< + const NORM: _MM_MANTISSA_NORM_ENUM, + const SIGN: _MM_MANTISSA_SIGN_ENUM, +>( a: __m512, - norm: _MM_MANTISSA_NORM_ENUM, - sign: _MM_MANTISSA_SIGN_ENUM, ) -> __m512 { + static_assert_imm4!(NORM); + static_assert_imm2!(SIGN); let a = a.as_f32x16(); let zero = _mm512_setzero_ps().as_f32x16(); - macro_rules! call { - ($imm4:expr, $imm2:expr) => { - vgetmantps( - a, - $imm2 << 2 | $imm4, - zero, - 0b11111111_11111111, - _MM_FROUND_CUR_DIRECTION, - ) - }; - } - let r = constify_imm4_mantissas!(norm, sign, call); + let r = vgetmantps( + a, + SIGN << 2 | NORM, + zero, + 0b11111111_11111111, + _MM_FROUND_CUR_DIRECTION, + ); transmute(r) } @@ -6512,23 +6295,21 @@ pub unsafe fn _mm512_getmant_ps( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_getmant_ps&expand=2881) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vgetmantps, norm = 0, sign = 0))] -#[rustc_args_required_const(3, 4)] -pub unsafe fn _mm512_mask_getmant_ps( +#[cfg_attr(test, assert_instr(vgetmantps, NORM = 0, SIGN = 0))] +#[rustc_legacy_const_generics(3, 4)] +pub unsafe fn _mm512_mask_getmant_ps< + const NORM: _MM_MANTISSA_NORM_ENUM, + const SIGN: _MM_MANTISSA_SIGN_ENUM, +>( src: __m512, k: __mmask16, a: __m512, - norm: _MM_MANTISSA_NORM_ENUM, - sign: _MM_MANTISSA_SIGN_ENUM, ) -> __m512 { + static_assert_imm4!(NORM); + static_assert_imm2!(SIGN); let a = a.as_f32x16(); let src = src.as_f32x16(); - macro_rules! call { - ($imm4:expr, $imm2:expr) => { - vgetmantps(a, $imm2 << 2 | $imm4, src, k, _MM_FROUND_CUR_DIRECTION) - }; - } - let r = constify_imm4_mantissas!(norm, sign, call); + let r = vgetmantps(a, SIGN << 2 | NORM, src, k, _MM_FROUND_CUR_DIRECTION); transmute(r) } @@ -6546,22 +6327,20 @@ pub unsafe fn _mm512_mask_getmant_ps( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_getmant_ps&expand=2882) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vgetmantps, norm = 0, sign = 0))] -#[rustc_args_required_const(2, 3)] -pub unsafe fn _mm512_maskz_getmant_ps( +#[cfg_attr(test, assert_instr(vgetmantps, NORM = 0, SIGN = 0))] +#[rustc_legacy_const_generics(2, 3)] +pub unsafe fn _mm512_maskz_getmant_ps< + const NORM: _MM_MANTISSA_NORM_ENUM, + const SIGN: _MM_MANTISSA_SIGN_ENUM, +>( k: __mmask16, a: __m512, - norm: _MM_MANTISSA_NORM_ENUM, - sign: _MM_MANTISSA_SIGN_ENUM, ) -> __m512 { + static_assert_imm4!(NORM); + static_assert_imm2!(SIGN); let a = a.as_f32x16(); let zero = _mm512_setzero_ps().as_f32x16(); - macro_rules! call { - ($imm4:expr, $imm2:expr) => { - vgetmantps(a, $imm2 << 2 | $imm4, zero, k, _MM_FROUND_CUR_DIRECTION) - }; - } - let r = constify_imm4_mantissas!(norm, sign, call); + let r = vgetmantps(a, SIGN << 2 | NORM, zero, k, _MM_FROUND_CUR_DIRECTION); transmute(r) } @@ -6579,21 +6358,19 @@ pub unsafe fn _mm512_maskz_getmant_ps( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_getmant_ps&expand=2877) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[cfg_attr(test, assert_instr(vgetmantps, norm = 0, sign = 0))] -#[rustc_args_required_const(1, 2)] -pub unsafe fn _mm256_getmant_ps( +#[cfg_attr(test, assert_instr(vgetmantps, NORM = 0, SIGN = 0))] +#[rustc_legacy_const_generics(1, 2)] +pub unsafe fn _mm256_getmant_ps< + const NORM: _MM_MANTISSA_NORM_ENUM, + const SIGN: _MM_MANTISSA_SIGN_ENUM, +>( a: __m256, - norm: _MM_MANTISSA_NORM_ENUM, - sign: _MM_MANTISSA_SIGN_ENUM, ) -> __m256 { + static_assert_imm4!(NORM); + static_assert_imm2!(SIGN); let a = a.as_f32x8(); let zero = _mm256_setzero_ps().as_f32x8(); - macro_rules! call { - ($imm4:expr, $imm2:expr) => { - vgetmantps256(a, $imm2 << 2 | $imm4, zero, 0b11111111) - }; - } - let r = constify_imm4_mantissas!(norm, sign, call); + let r = vgetmantps256(a, SIGN << 2 | NORM, zero, 0b11111111); transmute(r) } @@ -6611,23 +6388,21 @@ pub unsafe fn _mm256_getmant_ps( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_getmant_ps&expand=2878) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[cfg_attr(test, assert_instr(vgetmantps, norm = 0, sign = 0))] -#[rustc_args_required_const(3, 4)] -pub unsafe fn _mm256_mask_getmant_ps( +#[cfg_attr(test, assert_instr(vgetmantps, NORM = 0, SIGN = 0))] +#[rustc_legacy_const_generics(3, 4)] +pub unsafe fn _mm256_mask_getmant_ps< + const NORM: _MM_MANTISSA_NORM_ENUM, + const SIGN: _MM_MANTISSA_SIGN_ENUM, +>( src: __m256, k: __mmask8, a: __m256, - norm: _MM_MANTISSA_NORM_ENUM, - sign: _MM_MANTISSA_SIGN_ENUM, ) -> __m256 { + static_assert_imm4!(NORM); + static_assert_imm2!(SIGN); let a = a.as_f32x8(); let src = src.as_f32x8(); - macro_rules! call { - ($imm4:expr, $imm2:expr) => { - vgetmantps256(a, $imm2 << 2 | $imm4, src, k) - }; - } - let r = constify_imm4_mantissas!(norm, sign, call); + let r = vgetmantps256(a, SIGN << 2 | NORM, src, k); transmute(r) } @@ -6645,22 +6420,20 @@ pub unsafe fn _mm256_mask_getmant_ps( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_getmant_ps&expand=2879) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[cfg_attr(test, assert_instr(vgetmantps, norm = 0, sign = 0))] -#[rustc_args_required_const(2, 3)] -pub unsafe fn _mm256_maskz_getmant_ps( +#[cfg_attr(test, assert_instr(vgetmantps, NORM = 0, SIGN = 0))] +#[rustc_legacy_const_generics(2, 3)] +pub unsafe fn _mm256_maskz_getmant_ps< + const NORM: _MM_MANTISSA_NORM_ENUM, + const SIGN: _MM_MANTISSA_SIGN_ENUM, +>( k: __mmask8, a: __m256, - norm: _MM_MANTISSA_NORM_ENUM, - sign: _MM_MANTISSA_SIGN_ENUM, ) -> __m256 { + static_assert_imm4!(NORM); + static_assert_imm2!(SIGN); let a = a.as_f32x8(); let zero = _mm256_setzero_ps().as_f32x8(); - macro_rules! call { - ($imm4:expr, $imm2:expr) => { - vgetmantps256(a, $imm2 << 2 | $imm4, zero, k) - }; - } - let r = constify_imm4_mantissas!(norm, sign, call); + let r = vgetmantps256(a, SIGN << 2 | NORM, zero, k); transmute(r) } @@ -6678,21 +6451,19 @@ pub unsafe fn _mm256_maskz_getmant_ps( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_getmant_ps&expand=2874) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[cfg_attr(test, assert_instr(vgetmantps, norm = 0, sign = 0))] -#[rustc_args_required_const(1, 2)] -pub unsafe fn _mm_getmant_ps( +#[cfg_attr(test, assert_instr(vgetmantps, NORM = 0, SIGN = 0))] +#[rustc_legacy_const_generics(1, 2)] +pub unsafe fn _mm_getmant_ps< + const NORM: _MM_MANTISSA_NORM_ENUM, + const SIGN: _MM_MANTISSA_SIGN_ENUM, +>( a: __m128, - norm: _MM_MANTISSA_NORM_ENUM, - sign: _MM_MANTISSA_SIGN_ENUM, ) -> __m128 { + static_assert_imm4!(NORM); + static_assert_imm2!(SIGN); let a = a.as_f32x4(); let zero = _mm_setzero_ps().as_f32x4(); - macro_rules! call { - ($imm4:expr, $imm2:expr) => { - vgetmantps128(a, $imm2 << 2 | $imm4, zero, 0b00001111) - }; - } - let r = constify_imm4_mantissas!(norm, sign, call); + let r = vgetmantps128(a, SIGN << 2 | NORM, zero, 0b00001111); transmute(r) } @@ -6710,23 +6481,21 @@ pub unsafe fn _mm_getmant_ps( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_getmant_ps&expand=2875) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[cfg_attr(test, assert_instr(vgetmantps, norm = 0, sign = 0))] -#[rustc_args_required_const(3, 4)] -pub unsafe fn _mm_mask_getmant_ps( +#[cfg_attr(test, assert_instr(vgetmantps, NORM = 0, SIGN = 0))] +#[rustc_legacy_const_generics(3, 4)] +pub unsafe fn _mm_mask_getmant_ps< + const NORM: _MM_MANTISSA_NORM_ENUM, + const SIGN: _MM_MANTISSA_SIGN_ENUM, +>( src: __m128, k: __mmask8, a: __m128, - norm: _MM_MANTISSA_NORM_ENUM, - sign: _MM_MANTISSA_SIGN_ENUM, ) -> __m128 { + static_assert_imm4!(NORM); + static_assert_imm2!(SIGN); let a = a.as_f32x4(); let src = src.as_f32x4(); - macro_rules! call { - ($imm4:expr, $imm2:expr) => { - vgetmantps128(a, $imm2 << 2 | $imm4, src, k) - }; - } - let r = constify_imm4_mantissas!(norm, sign, call); + let r = vgetmantps128(a, SIGN << 2 | NORM, src, k); transmute(r) } @@ -6744,22 +6513,20 @@ pub unsafe fn _mm_mask_getmant_ps( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_getmant_ps&expand=2876) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[cfg_attr(test, assert_instr(vgetmantps, norm = 0, sign = 0))] -#[rustc_args_required_const(2, 3)] -pub unsafe fn _mm_maskz_getmant_ps( +#[cfg_attr(test, assert_instr(vgetmantps, NORM = 0, SIGN = 0))] +#[rustc_legacy_const_generics(2, 3)] +pub unsafe fn _mm_maskz_getmant_ps< + const NORM: _MM_MANTISSA_NORM_ENUM, + const SIGN: _MM_MANTISSA_SIGN_ENUM, +>( k: __mmask8, a: __m128, - norm: _MM_MANTISSA_NORM_ENUM, - sign: _MM_MANTISSA_SIGN_ENUM, ) -> __m128 { + static_assert_imm4!(NORM); + static_assert_imm2!(SIGN); let a = a.as_f32x4(); let zero = _mm_setzero_ps().as_f32x4(); - macro_rules! call { - ($imm4:expr, $imm2:expr) => { - vgetmantps128(a, $imm2 << 2 | $imm4, zero, k) - }; - } - let r = constify_imm4_mantissas!(norm, sign, call); + let r = vgetmantps128(a, SIGN << 2 | NORM, zero, k); transmute(r) } @@ -6777,27 +6544,25 @@ pub unsafe fn _mm_maskz_getmant_ps( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_getmant_pd&expand=2871) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vgetmantpd, norm = 0, sign = 0))] -#[rustc_args_required_const(1, 2)] -pub unsafe fn _mm512_getmant_pd( +#[cfg_attr(test, assert_instr(vgetmantpd, NORM = 0, SIGN = 0))] +#[rustc_legacy_const_generics(1, 2)] +pub unsafe fn _mm512_getmant_pd< + const NORM: _MM_MANTISSA_NORM_ENUM, + const SIGN: _MM_MANTISSA_SIGN_ENUM, +>( a: __m512d, - norm: _MM_MANTISSA_NORM_ENUM, - sign: _MM_MANTISSA_SIGN_ENUM, ) -> __m512d { + static_assert_imm4!(NORM); + static_assert_imm2!(SIGN); let a = a.as_f64x8(); let zero = _mm512_setzero_pd().as_f64x8(); - macro_rules! call { - ($imm4:expr, $imm2:expr) => { - vgetmantpd( - a, - $imm2 << 2 | $imm4, - zero, - 0b11111111, - _MM_FROUND_CUR_DIRECTION, - ) - }; - } - let r = constify_imm4_mantissas!(norm, sign, call); + let r = vgetmantpd( + a, + SIGN << 2 | NORM, + zero, + 0b11111111, + _MM_FROUND_CUR_DIRECTION, + ); transmute(r) } @@ -6815,23 +6580,21 @@ pub unsafe fn _mm512_getmant_pd( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_getmant_pd&expand=2872) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vgetmantpd, norm = 0, sign = 0))] -#[rustc_args_required_const(3, 4)] -pub unsafe fn _mm512_mask_getmant_pd( +#[cfg_attr(test, assert_instr(vgetmantpd, NORM = 0, SIGN = 0))] +#[rustc_legacy_const_generics(3, 4)] +pub unsafe fn _mm512_mask_getmant_pd< + const NORM: _MM_MANTISSA_NORM_ENUM, + const SIGN: _MM_MANTISSA_SIGN_ENUM, +>( src: __m512d, k: __mmask8, a: __m512d, - norm: _MM_MANTISSA_NORM_ENUM, - sign: _MM_MANTISSA_SIGN_ENUM, ) -> __m512d { + static_assert_imm4!(NORM); + static_assert_imm2!(SIGN); let a = a.as_f64x8(); let src = src.as_f64x8(); - macro_rules! call { - ($imm4:expr, $imm2:expr) => { - vgetmantpd(a, $imm2 << 2 | $imm4, src, k, _MM_FROUND_CUR_DIRECTION) - }; - } - let r = constify_imm4_mantissas!(norm, sign, call); + let r = vgetmantpd(a, SIGN << 2 | NORM, src, k, _MM_FROUND_CUR_DIRECTION); transmute(r) } @@ -6849,22 +6612,20 @@ pub unsafe fn _mm512_mask_getmant_pd( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_getmant_pd&expand=2873) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vgetmantpd, norm = 0, sign = 0))] -#[rustc_args_required_const(2, 3)] -pub unsafe fn _mm512_maskz_getmant_pd( +#[cfg_attr(test, assert_instr(vgetmantpd, NORM = 0, SIGN = 0))] +#[rustc_legacy_const_generics(2, 3)] +pub unsafe fn _mm512_maskz_getmant_pd< + const NORM: _MM_MANTISSA_NORM_ENUM, + const SIGN: _MM_MANTISSA_SIGN_ENUM, +>( k: __mmask8, a: __m512d, - norm: _MM_MANTISSA_NORM_ENUM, - sign: _MM_MANTISSA_SIGN_ENUM, ) -> __m512d { + static_assert_imm4!(NORM); + static_assert_imm2!(SIGN); let a = a.as_f64x8(); let zero = _mm512_setzero_pd().as_f64x8(); - macro_rules! call { - ($imm4:expr, $imm2:expr) => { - vgetmantpd(a, $imm2 << 2 | $imm4, zero, k, _MM_FROUND_CUR_DIRECTION) - }; - } - let r = constify_imm4_mantissas!(norm, sign, call); + let r = vgetmantpd(a, SIGN << 2 | NORM, zero, k, _MM_FROUND_CUR_DIRECTION); transmute(r) } @@ -6882,21 +6643,19 @@ pub unsafe fn _mm512_maskz_getmant_pd( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_getmant_pd&expand=2868) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[cfg_attr(test, assert_instr(vgetmantpd, norm = 0, sign = 0))] -#[rustc_args_required_const(1, 2)] -pub unsafe fn _mm256_getmant_pd( +#[cfg_attr(test, assert_instr(vgetmantpd, NORM = 0, SIGN = 0))] +#[rustc_legacy_const_generics(1, 2)] +pub unsafe fn _mm256_getmant_pd< + const NORM: _MM_MANTISSA_NORM_ENUM, + const SIGN: _MM_MANTISSA_SIGN_ENUM, +>( a: __m256d, - norm: _MM_MANTISSA_NORM_ENUM, - sign: _MM_MANTISSA_SIGN_ENUM, ) -> __m256d { + static_assert_imm4!(NORM); + static_assert_imm2!(SIGN); let a = a.as_f64x4(); let zero = _mm256_setzero_pd().as_f64x4(); - macro_rules! call { - ($imm4:expr, $imm2:expr) => { - vgetmantpd256(a, $imm2 << 2 | $imm4, zero, 0b00001111) - }; - } - let r = constify_imm4_mantissas!(norm, sign, call); + let r = vgetmantpd256(a, SIGN << 2 | NORM, zero, 0b00001111); transmute(r) } @@ -6914,23 +6673,21 @@ pub unsafe fn _mm256_getmant_pd( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_getmant_pd&expand=2869) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[cfg_attr(test, assert_instr(vgetmantpd, norm = 0, sign = 0))] -#[rustc_args_required_const(3, 4)] -pub unsafe fn _mm256_mask_getmant_pd( +#[cfg_attr(test, assert_instr(vgetmantpd, NORM = 0, SIGN = 0))] +#[rustc_legacy_const_generics(3, 4)] +pub unsafe fn _mm256_mask_getmant_pd< + const NORM: _MM_MANTISSA_NORM_ENUM, + const SIGN: _MM_MANTISSA_SIGN_ENUM, +>( src: __m256d, k: __mmask8, a: __m256d, - norm: _MM_MANTISSA_NORM_ENUM, - sign: _MM_MANTISSA_SIGN_ENUM, ) -> __m256d { + static_assert_imm4!(NORM); + static_assert_imm2!(SIGN); let a = a.as_f64x4(); let src = src.as_f64x4(); - macro_rules! call { - ($imm4:expr, $imm2:expr) => { - vgetmantpd256(a, $imm2 << 2 | $imm4, src, k) - }; - } - let r = constify_imm4_mantissas!(norm, sign, call); + let r = vgetmantpd256(a, SIGN << 2 | NORM, src, k); transmute(r) } @@ -6948,22 +6705,20 @@ pub unsafe fn _mm256_mask_getmant_pd( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_getmant_pd&expand=2870) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[cfg_attr(test, assert_instr(vgetmantpd, norm = 0, sign = 0))] -#[rustc_args_required_const(2, 3)] -pub unsafe fn _mm256_maskz_getmant_pd( +#[cfg_attr(test, assert_instr(vgetmantpd, NORM = 0, SIGN = 0))] +#[rustc_legacy_const_generics(2, 3)] +pub unsafe fn _mm256_maskz_getmant_pd< + const NORM: _MM_MANTISSA_NORM_ENUM, + const SIGN: _MM_MANTISSA_SIGN_ENUM, +>( k: __mmask8, a: __m256d, - norm: _MM_MANTISSA_NORM_ENUM, - sign: _MM_MANTISSA_SIGN_ENUM, ) -> __m256d { + static_assert_imm4!(NORM); + static_assert_imm2!(SIGN); let a = a.as_f64x4(); let zero = _mm256_setzero_pd().as_f64x4(); - macro_rules! call { - ($imm4:expr, $imm2:expr) => { - vgetmantpd256(a, $imm2 << 2 | $imm4, zero, k) - }; - } - let r = constify_imm4_mantissas!(norm, sign, call); + let r = vgetmantpd256(a, SIGN << 2 | NORM, zero, k); transmute(r) } @@ -6981,21 +6736,19 @@ pub unsafe fn _mm256_maskz_getmant_pd( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_getmant_pd&expand=2865) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[cfg_attr(test, assert_instr(vgetmantpd, norm = 0, sign = 0))] -#[rustc_args_required_const(1, 2)] -pub unsafe fn _mm_getmant_pd( +#[cfg_attr(test, assert_instr(vgetmantpd, NORM = 0, SIGN = 0))] +#[rustc_legacy_const_generics(1, 2)] +pub unsafe fn _mm_getmant_pd< + const NORM: _MM_MANTISSA_NORM_ENUM, + const SIGN: _MM_MANTISSA_SIGN_ENUM, +>( a: __m128d, - norm: _MM_MANTISSA_NORM_ENUM, - sign: _MM_MANTISSA_SIGN_ENUM, ) -> __m128d { + static_assert_imm4!(NORM); + static_assert_imm2!(SIGN); let a = a.as_f64x2(); let zero = _mm_setzero_pd().as_f64x2(); - macro_rules! call { - ($imm4:expr, $imm2:expr) => { - vgetmantpd128(a, $imm2 << 2 | $imm4, zero, 0b00000011) - }; - } - let r = constify_imm4_mantissas!(norm, sign, call); + let r = vgetmantpd128(a, SIGN << 2 | NORM, zero, 0b00000011); transmute(r) } @@ -7013,23 +6766,21 @@ pub unsafe fn _mm_getmant_pd( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_getmant_pd&expand=2866) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[cfg_attr(test, assert_instr(vgetmantpd, norm = 0, sign = 0))] -#[rustc_args_required_const(3, 4)] -pub unsafe fn _mm_mask_getmant_pd( +#[cfg_attr(test, assert_instr(vgetmantpd, NORM = 0, SIGN = 0))] +#[rustc_legacy_const_generics(3, 4)] +pub unsafe fn _mm_mask_getmant_pd< + const NORM: _MM_MANTISSA_NORM_ENUM, + const SIGN: _MM_MANTISSA_SIGN_ENUM, +>( src: __m128d, k: __mmask8, a: __m128d, - norm: _MM_MANTISSA_NORM_ENUM, - sign: _MM_MANTISSA_SIGN_ENUM, ) -> __m128d { + static_assert_imm4!(NORM); + static_assert_imm2!(SIGN); let a = a.as_f64x2(); let src = src.as_f64x2(); - macro_rules! call { - ($imm4:expr, $imm2:expr) => { - vgetmantpd128(a, $imm2 << 2 | $imm4, src, k) - }; - } - let r = constify_imm4_mantissas!(norm, sign, call); + let r = vgetmantpd128(a, SIGN << 2 | NORM, src, k); transmute(r) } @@ -7047,22 +6798,20 @@ pub unsafe fn _mm_mask_getmant_pd( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_getmant_pd&expand=2867) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[cfg_attr(test, assert_instr(vgetmantpd, norm = 0, sign = 0))] -#[rustc_args_required_const(2, 3)] -pub unsafe fn _mm_maskz_getmant_pd( +#[cfg_attr(test, assert_instr(vgetmantpd, NORM = 0, SIGN = 0))] +#[rustc_legacy_const_generics(2, 3)] +pub unsafe fn _mm_maskz_getmant_pd< + const NORM: _MM_MANTISSA_NORM_ENUM, + const SIGN: _MM_MANTISSA_SIGN_ENUM, +>( k: __mmask8, a: __m128d, - norm: _MM_MANTISSA_NORM_ENUM, - sign: _MM_MANTISSA_SIGN_ENUM, ) -> __m128d { + static_assert_imm4!(NORM); + static_assert_imm2!(SIGN); let a = a.as_f64x2(); let zero = _mm_setzero_pd().as_f64x2(); - macro_rules! call { - ($imm4:expr, $imm2:expr) => { - vgetmantpd128(a, $imm2 << 2 | $imm4, zero, k) - }; - } - let r = constify_imm4_mantissas!(norm, sign, call); + let r = vgetmantpd128(a, SIGN << 2 | NORM, zero, k); transmute(r) } @@ -7078,17 +6827,13 @@ pub unsafe fn _mm_maskz_getmant_pd( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_add_round_ps&expand=145) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vaddps, rounding = 8))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm512_add_round_ps(a: __m512, b: __m512, rounding: i32) -> __m512 { +#[cfg_attr(test, assert_instr(vaddps, ROUNDING = 8))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_add_round_ps(a: __m512, b: __m512) -> __m512 { + static_assert_rounding!(ROUNDING); let a = a.as_f32x16(); let b = b.as_f32x16(); - macro_rules! call { - ($imm4:expr) => { - vaddps(a, b, $imm4) - }; - } - let r = constify_imm4_round!(rounding, call); + let r = vaddps(a, b, ROUNDING); transmute(r) } @@ -7104,24 +6849,19 @@ pub unsafe fn _mm512_add_round_ps(a: __m512, b: __m512, rounding: i32) -> __m512 /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_add_round_ps&expand=146) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vaddps, rounding = 8))] -#[rustc_args_required_const(4)] -pub unsafe fn _mm512_mask_add_round_ps( +#[cfg_attr(test, assert_instr(vaddps, ROUNDING = 8))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_mask_add_round_ps( src: __m512, k: __mmask16, a: __m512, b: __m512, - rounding: i32, ) -> __m512 { + static_assert_rounding!(ROUNDING); let a = a.as_f32x16(); let b = b.as_f32x16(); - macro_rules! call { - ($imm4:expr) => { - vaddps(a, b, $imm4) - }; - } - let addround = constify_imm4_round!(rounding, call); - transmute(simd_select_bitmask(k, addround, src.as_f32x16())) + let r = vaddps(a, b, ROUNDING); + transmute(simd_select_bitmask(k, r, src.as_f32x16())) } /// Add packed single-precision (32-bit) floating-point elements in a and b, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set).\ @@ -7136,24 +6876,19 @@ pub unsafe fn _mm512_mask_add_round_ps( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_add_round_ps&expand=147) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vaddps, rounding = 8))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm512_maskz_add_round_ps( +#[cfg_attr(test, assert_instr(vaddps, ROUNDING = 8))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_maskz_add_round_ps( k: __mmask16, a: __m512, b: __m512, - rounding: i32, ) -> __m512 { + static_assert_rounding!(ROUNDING); let a = a.as_f32x16(); let b = b.as_f32x16(); - macro_rules! call { - ($imm4:expr) => { - vaddps(a, b, $imm4) - }; - } - let addround = constify_imm4_round!(rounding, call); + let r = vaddps(a, b, ROUNDING); let zero = _mm512_setzero_ps().as_f32x16(); - transmute(simd_select_bitmask(k, addround, zero)) + transmute(simd_select_bitmask(k, r, zero)) } /// Add packed double-precision (64-bit) floating-point elements in a and b, and store the results in dst.\ @@ -7168,17 +6903,13 @@ pub unsafe fn _mm512_maskz_add_round_ps( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_add_round_pd&expand=142) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vaddpd, rounding = 8))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm512_add_round_pd(a: __m512d, b: __m512d, rounding: i32) -> __m512d { +#[cfg_attr(test, assert_instr(vaddpd, ROUNDING = 8))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_add_round_pd(a: __m512d, b: __m512d) -> __m512d { + static_assert_rounding!(ROUNDING); let a = a.as_f64x8(); let b = b.as_f64x8(); - macro_rules! call { - ($imm4:expr) => { - vaddpd(a, b, $imm4) - }; - } - let r = constify_imm4_round!(rounding, call); + let r = vaddpd(a, b, ROUNDING); transmute(r) } @@ -7194,24 +6925,19 @@ pub unsafe fn _mm512_add_round_pd(a: __m512d, b: __m512d, rounding: i32) -> __m5 /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_add_round_pd&expand=143) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vaddpd, rounding = 8))] -#[rustc_args_required_const(4)] -pub unsafe fn _mm512_mask_add_round_pd( +#[cfg_attr(test, assert_instr(vaddpd, ROUNDING = 8))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_mask_add_round_pd( src: __m512d, k: __mmask8, a: __m512d, b: __m512d, - rounding: i32, ) -> __m512d { + static_assert_rounding!(ROUNDING); let a = a.as_f64x8(); let b = b.as_f64x8(); - macro_rules! call { - ($imm4:expr) => { - vaddpd(a, b, $imm4) - }; - } - let addround = constify_imm4_round!(rounding, call); - transmute(simd_select_bitmask(k, addround, src.as_f64x8())) + let r = vaddpd(a, b, ROUNDING); + transmute(simd_select_bitmask(k, r, src.as_f64x8())) } /// Add packed double-precision (64-bit) floating-point elements in a and b, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set).\ @@ -7226,24 +6952,19 @@ pub unsafe fn _mm512_mask_add_round_pd( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_add_round_pd&expand=144) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vaddpd, rounding = 8))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm512_maskz_add_round_pd( +#[cfg_attr(test, assert_instr(vaddpd, ROUNDING = 8))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_maskz_add_round_pd( k: __mmask8, a: __m512d, b: __m512d, - rounding: i32, ) -> __m512d { + static_assert_rounding!(ROUNDING); let a = a.as_f64x8(); let b = b.as_f64x8(); - macro_rules! call { - ($imm4:expr) => { - vaddpd(a, b, $imm4) - }; - } - let addround = constify_imm4_round!(rounding, call); + let r = vaddpd(a, b, ROUNDING); let zero = _mm512_setzero_pd().as_f64x8(); - transmute(simd_select_bitmask(k, addround, zero)) + transmute(simd_select_bitmask(k, r, zero)) } /// Subtract packed single-precision (32-bit) floating-point elements in b from packed single-precision (32-bit) floating-point elements in a, and store the results in dst.\ @@ -7258,17 +6979,13 @@ pub unsafe fn _mm512_maskz_add_round_pd( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_sub_round_ps&expand=5739) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vsubps, rounding = 8))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm512_sub_round_ps(a: __m512, b: __m512, rounding: i32) -> __m512 { +#[cfg_attr(test, assert_instr(vsubps, ROUNDING = 8))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_sub_round_ps(a: __m512, b: __m512) -> __m512 { + static_assert_rounding!(ROUNDING); let a = a.as_f32x16(); let b = b.as_f32x16(); - macro_rules! call { - ($imm4:expr) => { - vsubps(a, b, $imm4) - }; - } - let r = constify_imm4_round!(rounding, call); + let r = vsubps(a, b, ROUNDING); transmute(r) } @@ -7284,24 +7001,19 @@ pub unsafe fn _mm512_sub_round_ps(a: __m512, b: __m512, rounding: i32) -> __m512 /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_sub_round_ps&expand=5737) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vsubps, rounding = 8))] -#[rustc_args_required_const(4)] -pub unsafe fn _mm512_mask_sub_round_ps( +#[cfg_attr(test, assert_instr(vsubps, ROUNDING = 8))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_mask_sub_round_ps( src: __m512, k: __mmask16, a: __m512, b: __m512, - rounding: i32, ) -> __m512 { + static_assert_rounding!(ROUNDING); let a = a.as_f32x16(); let b = b.as_f32x16(); - macro_rules! call { - ($imm4:expr) => { - vsubps(a, b, $imm4) - }; - } - let subround = constify_imm4_round!(rounding, call); - transmute(simd_select_bitmask(k, subround, src.as_f32x16())) + let r = vsubps(a, b, ROUNDING); + transmute(simd_select_bitmask(k, r, src.as_f32x16())) } /// Subtract packed single-precision (32-bit) floating-point elements in b from packed single-precision (32-bit) floating-point elements in a, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set).\ @@ -7316,24 +7028,19 @@ pub unsafe fn _mm512_mask_sub_round_ps( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_sub_round_ps&expand=5738) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vsubps, rounding = 8))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm512_maskz_sub_round_ps( +#[cfg_attr(test, assert_instr(vsubps, ROUNDING = 8))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_maskz_sub_round_ps( k: __mmask16, a: __m512, b: __m512, - rounding: i32, ) -> __m512 { + static_assert_rounding!(ROUNDING); let a = a.as_f32x16(); let b = b.as_f32x16(); - macro_rules! call { - ($imm4:expr) => { - vsubps(a, b, $imm4) - }; - } - let subround = constify_imm4_round!(rounding, call); + let r = vsubps(a, b, ROUNDING); let zero = _mm512_setzero_ps().as_f32x16(); - transmute(simd_select_bitmask(k, subround, zero)) + transmute(simd_select_bitmask(k, r, zero)) } /// Subtract packed double-precision (64-bit) floating-point elements in b from packed double-precision (64-bit) floating-point elements in a, and store the results in dst.\ @@ -7348,17 +7055,13 @@ pub unsafe fn _mm512_maskz_sub_round_ps( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_sub_round_pd&expand=5736) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vsubpd, rounding = 8))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm512_sub_round_pd(a: __m512d, b: __m512d, rounding: i32) -> __m512d { +#[cfg_attr(test, assert_instr(vsubpd, ROUNDING = 8))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_sub_round_pd(a: __m512d, b: __m512d) -> __m512d { + static_assert_rounding!(ROUNDING); let a = a.as_f64x8(); let b = b.as_f64x8(); - macro_rules! call { - ($imm4:expr) => { - vsubpd(a, b, $imm4) - }; - } - let r = constify_imm4_round!(rounding, call); + let r = vsubpd(a, b, ROUNDING); transmute(r) } @@ -7374,24 +7077,19 @@ pub unsafe fn _mm512_sub_round_pd(a: __m512d, b: __m512d, rounding: i32) -> __m5 /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_sub_round_pd&expand=5734) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vsubpd, rounding = 8))] -#[rustc_args_required_const(4)] -pub unsafe fn _mm512_mask_sub_round_pd( +#[cfg_attr(test, assert_instr(vsubpd, ROUNDING = 8))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_mask_sub_round_pd( src: __m512d, k: __mmask8, a: __m512d, b: __m512d, - rounding: i32, ) -> __m512d { + static_assert_rounding!(ROUNDING); let a = a.as_f64x8(); let b = b.as_f64x8(); - macro_rules! call { - ($imm4:expr) => { - vsubpd(a, b, $imm4) - }; - } - let subround = constify_imm4_round!(rounding, call); - transmute(simd_select_bitmask(k, subround, src.as_f64x8())) + let r = vsubpd(a, b, ROUNDING); + transmute(simd_select_bitmask(k, r, src.as_f64x8())) } /// Subtract packed double-precision (64-bit) floating-point elements in b from packed double-precision (64-bit) floating-point elements in a, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set).\ @@ -7406,24 +7104,19 @@ pub unsafe fn _mm512_mask_sub_round_pd( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_sub_round_pd&expand=5735) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vsubpd, rounding = 8))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm512_maskz_sub_round_pd( +#[cfg_attr(test, assert_instr(vsubpd, ROUNDING = 8))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_maskz_sub_round_pd( k: __mmask8, a: __m512d, b: __m512d, - rounding: i32, ) -> __m512d { + static_assert_rounding!(ROUNDING); let a = a.as_f64x8(); let b = b.as_f64x8(); - macro_rules! call { - ($imm4:expr) => { - vsubpd(a, b, $imm4) - }; - } - let subround = constify_imm4_round!(rounding, call); + let r = vsubpd(a, b, ROUNDING); let zero = _mm512_setzero_pd().as_f64x8(); - transmute(simd_select_bitmask(k, subround, zero)) + transmute(simd_select_bitmask(k, r, zero)) } /// Multiply packed single-precision (32-bit) floating-point elements in a and b, and store the results in dst.\ @@ -7438,17 +7131,13 @@ pub unsafe fn _mm512_maskz_sub_round_pd( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mul_round_ps&expand=3940) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vmulps, rounding = 8))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm512_mul_round_ps(a: __m512, b: __m512, rounding: i32) -> __m512 { +#[cfg_attr(test, assert_instr(vmulps, ROUNDING = 8))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_mul_round_ps(a: __m512, b: __m512) -> __m512 { + static_assert_rounding!(ROUNDING); let a = a.as_f32x16(); let b = b.as_f32x16(); - macro_rules! call { - ($imm4:expr) => { - vmulps(a, b, $imm4) - }; - } - let r = constify_imm4_round!(rounding, call); + let r = vmulps(a, b, ROUNDING); transmute(r) } @@ -7464,24 +7153,19 @@ pub unsafe fn _mm512_mul_round_ps(a: __m512, b: __m512, rounding: i32) -> __m512 /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_mul_round_ps&expand=3938) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vmulps, rounding = 8))] -#[rustc_args_required_const(4)] -pub unsafe fn _mm512_mask_mul_round_ps( +#[cfg_attr(test, assert_instr(vmulps, ROUNDING = 8))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_mask_mul_round_ps( src: __m512, k: __mmask16, a: __m512, b: __m512, - rounding: i32, ) -> __m512 { + static_assert_rounding!(ROUNDING); let a = a.as_f32x16(); let b = b.as_f32x16(); - macro_rules! call { - ($imm4:expr) => { - vmulps(a, b, $imm4) - }; - } - let mulround = constify_imm4_round!(rounding, call); - transmute(simd_select_bitmask(k, mulround, src.as_f32x16())) + let r = vmulps(a, b, ROUNDING); + transmute(simd_select_bitmask(k, r, src.as_f32x16())) } /// Multiply packed single-precision (32-bit) floating-point elements in a and b, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set).\ @@ -7496,24 +7180,19 @@ pub unsafe fn _mm512_mask_mul_round_ps( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_mul_round_ps&expand=3939) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vmulps, rounding = 8))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm512_maskz_mul_round_ps( +#[cfg_attr(test, assert_instr(vmulps, ROUNDING = 8))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_maskz_mul_round_ps( k: __mmask16, a: __m512, b: __m512, - rounding: i32, ) -> __m512 { + static_assert_rounding!(ROUNDING); let a = a.as_f32x16(); let b = b.as_f32x16(); - macro_rules! call { - ($imm4:expr) => { - vmulps(a, b, $imm4) - }; - } - let mulround = constify_imm4_round!(rounding, call); + let r = vmulps(a, b, ROUNDING); let zero = _mm512_setzero_ps().as_f32x16(); - transmute(simd_select_bitmask(k, mulround, zero)) + transmute(simd_select_bitmask(k, r, zero)) } /// Multiply packed double-precision (64-bit) floating-point elements in a and b, and store the results in dst.\ @@ -7528,17 +7207,13 @@ pub unsafe fn _mm512_maskz_mul_round_ps( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mul_round_pd&expand=3937) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vmulpd, rounding = 8))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm512_mul_round_pd(a: __m512d, b: __m512d, rounding: i32) -> __m512d { +#[cfg_attr(test, assert_instr(vmulpd, ROUNDING = 8))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_mul_round_pd(a: __m512d, b: __m512d) -> __m512d { + static_assert_rounding!(ROUNDING); let a = a.as_f64x8(); let b = b.as_f64x8(); - macro_rules! call { - ($imm4:expr) => { - vmulpd(a, b, $imm4) - }; - } - let r = constify_imm4_round!(rounding, call); + let r = vmulpd(a, b, ROUNDING); transmute(r) } @@ -7554,24 +7229,19 @@ pub unsafe fn _mm512_mul_round_pd(a: __m512d, b: __m512d, rounding: i32) -> __m5 /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_mul_round_pd&expand=3935) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vmulpd, rounding = 8))] -#[rustc_args_required_const(4)] -pub unsafe fn _mm512_mask_mul_round_pd( +#[cfg_attr(test, assert_instr(vmulpd, ROUNDING = 8))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_mask_mul_round_pd( src: __m512d, k: __mmask8, a: __m512d, b: __m512d, - rounding: i32, ) -> __m512d { + static_assert_rounding!(ROUNDING); let a = a.as_f64x8(); let b = b.as_f64x8(); - macro_rules! call { - ($imm4:expr) => { - vmulpd(a, b, $imm4) - }; - } - let mulround = constify_imm4_round!(rounding, call); - transmute(simd_select_bitmask(k, mulround, src.as_f64x8())) + let r = vmulpd(a, b, ROUNDING); + transmute(simd_select_bitmask(k, r, src.as_f64x8())) } /// Multiply packed single-precision (32-bit) floating-point elements in a and b, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set).\ @@ -7586,24 +7256,19 @@ pub unsafe fn _mm512_mask_mul_round_pd( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_mul_round_ps&expand=3939) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vmulpd, rounding = 8))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm512_maskz_mul_round_pd( +#[cfg_attr(test, assert_instr(vmulpd, ROUNDING = 8))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_maskz_mul_round_pd( k: __mmask8, a: __m512d, b: __m512d, - rounding: i32, ) -> __m512d { + static_assert_rounding!(ROUNDING); let a = a.as_f64x8(); let b = b.as_f64x8(); - macro_rules! call { - ($imm4:expr) => { - vmulpd(a, b, $imm4) - }; - } - let mulround = constify_imm4_round!(rounding, call); + let r = vmulpd(a, b, ROUNDING); let zero = _mm512_setzero_pd().as_f64x8(); - transmute(simd_select_bitmask(k, mulround, zero)) + transmute(simd_select_bitmask(k, r, zero)) } /// Divide packed single-precision (32-bit) floating-point elements in a by packed elements in b, and store the results in dst.\ @@ -7618,17 +7283,13 @@ pub unsafe fn _mm512_maskz_mul_round_pd( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_div_round_ps&expand=2168) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vdivps, rounding = 8))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm512_div_round_ps(a: __m512, b: __m512, rounding: i32) -> __m512 { +#[cfg_attr(test, assert_instr(vdivps, ROUNDING = 8))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_div_round_ps(a: __m512, b: __m512) -> __m512 { + static_assert_rounding!(ROUNDING); let a = a.as_f32x16(); let b = b.as_f32x16(); - macro_rules! call { - ($imm4:expr) => { - vdivps(a, b, $imm4) - }; - } - let r = constify_imm4_round!(rounding, call); + let r = vdivps(a, b, ROUNDING); transmute(r) } @@ -7644,24 +7305,19 @@ pub unsafe fn _mm512_div_round_ps(a: __m512, b: __m512, rounding: i32) -> __m512 /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_div_round_ps&expand=2169) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vdivps, rounding = 8))] -#[rustc_args_required_const(4)] -pub unsafe fn _mm512_mask_div_round_ps( +#[cfg_attr(test, assert_instr(vdivps, ROUNDING = 8))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_mask_div_round_ps( src: __m512, k: __mmask16, a: __m512, b: __m512, - rounding: i32, ) -> __m512 { + static_assert_rounding!(ROUNDING); let a = a.as_f32x16(); let b = b.as_f32x16(); - macro_rules! call { - ($imm4:expr) => { - vdivps(a, b, $imm4) - }; - } - let divround = constify_imm4_round!(rounding, call); - transmute(simd_select_bitmask(k, divround, src.as_f32x16())) + let r = vdivps(a, b, ROUNDING); + transmute(simd_select_bitmask(k, r, src.as_f32x16())) } /// Divide packed single-precision (32-bit) floating-point elements in a by packed elements in b, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set).\ @@ -7676,24 +7332,19 @@ pub unsafe fn _mm512_mask_div_round_ps( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_div_round_ps&expand=2170) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vdivps, rounding = 8))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm512_maskz_div_round_ps( +#[cfg_attr(test, assert_instr(vdivps, ROUNDING = 8))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_maskz_div_round_ps( k: __mmask16, a: __m512, b: __m512, - rounding: i32, ) -> __m512 { + static_assert_rounding!(ROUNDING); let a = a.as_f32x16(); let b = b.as_f32x16(); - macro_rules! call { - ($imm4:expr) => { - vdivps(a, b, $imm4) - }; - } - let divround = constify_imm4_round!(rounding, call); + let r = vdivps(a, b, ROUNDING); let zero = _mm512_setzero_ps().as_f32x16(); - transmute(simd_select_bitmask(k, divround, zero)) + transmute(simd_select_bitmask(k, r, zero)) } /// Divide packed double-precision (64-bit) floating-point elements in a by packed elements in b, =and store the results in dst.\ @@ -7708,17 +7359,13 @@ pub unsafe fn _mm512_maskz_div_round_ps( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_div_round_pd&expand=2165) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vdivpd, rounding = 8))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm512_div_round_pd(a: __m512d, b: __m512d, rounding: i32) -> __m512d { +#[cfg_attr(test, assert_instr(vdivpd, ROUNDING = 8))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_div_round_pd(a: __m512d, b: __m512d) -> __m512d { + static_assert_rounding!(ROUNDING); let a = a.as_f64x8(); let b = b.as_f64x8(); - macro_rules! call { - ($imm4:expr) => { - vdivpd(a, b, $imm4) - }; - } - let r = constify_imm4_round!(rounding, call); + let r = vdivpd(a, b, ROUNDING); transmute(r) } @@ -7734,24 +7381,19 @@ pub unsafe fn _mm512_div_round_pd(a: __m512d, b: __m512d, rounding: i32) -> __m5 /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_div_round_pd&expand=2166) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vdivpd, rounding = 8))] -#[rustc_args_required_const(4)] -pub unsafe fn _mm512_mask_div_round_pd( +#[cfg_attr(test, assert_instr(vdivpd, ROUNDING = 8))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_mask_div_round_pd( src: __m512d, k: __mmask8, a: __m512d, b: __m512d, - rounding: i32, ) -> __m512d { + static_assert_rounding!(ROUNDING); let a = a.as_f64x8(); let b = b.as_f64x8(); - macro_rules! call { - ($imm4:expr) => { - vdivpd(a, b, $imm4) - }; - } - let divround = constify_imm4_round!(rounding, call); - transmute(simd_select_bitmask(k, divround, src.as_f64x8())) + let r = vdivpd(a, b, ROUNDING); + transmute(simd_select_bitmask(k, r, src.as_f64x8())) } /// Divide packed double-precision (64-bit) floating-point elements in a by packed elements in b, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set).\ @@ -7766,24 +7408,19 @@ pub unsafe fn _mm512_mask_div_round_pd( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_div_round_pd&expand=2167) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vdivpd, rounding = 8))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm512_maskz_div_round_pd( +#[cfg_attr(test, assert_instr(vdivpd, ROUNDING = 8))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_maskz_div_round_pd( k: __mmask8, a: __m512d, b: __m512d, - rounding: i32, ) -> __m512d { + static_assert_rounding!(ROUNDING); let a = a.as_f64x8(); let b = b.as_f64x8(); - macro_rules! call { - ($imm4:expr) => { - vdivpd(a, b, $imm4) - }; - } - let divround = constify_imm4_round!(rounding, call); + let r = vdivpd(a, b, ROUNDING); let zero = _mm512_setzero_pd().as_f64x8(); - transmute(simd_select_bitmask(k, divround, zero)) + transmute(simd_select_bitmask(k, r, zero)) } /// Compute the square root of packed single-precision (32-bit) floating-point elements in a, and store the results in dst.\ @@ -7798,16 +7435,12 @@ pub unsafe fn _mm512_maskz_div_round_pd( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_sqrt_round_ps&expand=5377) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vsqrtps, rounding = 8))] -#[rustc_args_required_const(1)] -pub unsafe fn _mm512_sqrt_round_ps(a: __m512, rounding: i32) -> __m512 { +#[cfg_attr(test, assert_instr(vsqrtps, ROUNDING = 8))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm512_sqrt_round_ps(a: __m512) -> __m512 { + static_assert_rounding!(ROUNDING); let a = a.as_f32x16(); - macro_rules! call { - ($imm4:expr) => { - vsqrtps(a, $imm4) - }; - } - let r = constify_imm4_round!(rounding, call); + let r = vsqrtps(a, ROUNDING); transmute(r) } @@ -7823,22 +7456,17 @@ pub unsafe fn _mm512_sqrt_round_ps(a: __m512, rounding: i32) -> __m512 { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_sqrt_round_ps&expand=5375) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vsqrtps, rounding = 8))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm512_mask_sqrt_round_ps( +#[cfg_attr(test, assert_instr(vsqrtps, ROUNDING = 8))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_mask_sqrt_round_ps( src: __m512, k: __mmask16, a: __m512, - rounding: i32, ) -> __m512 { + static_assert_rounding!(ROUNDING); let a = a.as_f32x16(); - macro_rules! call { - ($imm4:expr) => { - vsqrtps(a, $imm4) - }; - } - let sqrtround = constify_imm4_round!(rounding, call); - transmute(simd_select_bitmask(k, sqrtround, src.as_f32x16())) + let r = vsqrtps(a, ROUNDING); + transmute(simd_select_bitmask(k, r, src.as_f32x16())) } /// Compute the square root of packed single-precision (32-bit) floating-point elements in a, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set).\ @@ -7853,18 +7481,14 @@ pub unsafe fn _mm512_mask_sqrt_round_ps( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_sqrt_round_ps&expand=5376) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vsqrtps, rounding = 8))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm512_maskz_sqrt_round_ps(k: __mmask16, a: __m512, rounding: i32) -> __m512 { +#[cfg_attr(test, assert_instr(vsqrtps, ROUNDING = 8))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_maskz_sqrt_round_ps(k: __mmask16, a: __m512) -> __m512 { + static_assert_rounding!(ROUNDING); let a = a.as_f32x16(); - macro_rules! call { - ($imm4:expr) => { - vsqrtps(a, $imm4) - }; - } - let sqrtround = constify_imm4_round!(rounding, call); + let r = vsqrtps(a, ROUNDING); let zero = _mm512_setzero_ps().as_f32x16(); - transmute(simd_select_bitmask(k, sqrtround, zero)) + transmute(simd_select_bitmask(k, r, zero)) } /// Compute the square root of packed double-precision (64-bit) floating-point elements in a, and store the results in dst.\ @@ -7879,16 +7503,12 @@ pub unsafe fn _mm512_maskz_sqrt_round_ps(k: __mmask16, a: __m512, rounding: i32) /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_sqrt_round_pd&expand=5374) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vsqrtpd, rounding = 8))] -#[rustc_args_required_const(1)] -pub unsafe fn _mm512_sqrt_round_pd(a: __m512d, rounding: i32) -> __m512d { +#[cfg_attr(test, assert_instr(vsqrtpd, ROUNDING = 8))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm512_sqrt_round_pd(a: __m512d) -> __m512d { + static_assert_rounding!(ROUNDING); let a = a.as_f64x8(); - macro_rules! call { - ($imm4:expr) => { - vsqrtpd(a, $imm4) - }; - } - let r = constify_imm4_round!(rounding, call); + let r = vsqrtpd(a, ROUNDING); transmute(r) } @@ -7904,22 +7524,17 @@ pub unsafe fn _mm512_sqrt_round_pd(a: __m512d, rounding: i32) -> __m512d { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_sqrt_round_pd&expand=5372) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vsqrtpd, rounding = 8))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm512_mask_sqrt_round_pd( +#[cfg_attr(test, assert_instr(vsqrtpd, ROUNDING = 8))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_mask_sqrt_round_pd( src: __m512d, k: __mmask8, a: __m512d, - rounding: i32, ) -> __m512d { + static_assert_rounding!(ROUNDING); let a = a.as_f64x8(); - macro_rules! call { - ($imm4:expr) => { - vsqrtpd(a, $imm4) - }; - } - let sqrtround = constify_imm4_round!(rounding, call); - transmute(simd_select_bitmask(k, sqrtround, src.as_f64x8())) + let r = vsqrtpd(a, ROUNDING); + transmute(simd_select_bitmask(k, r, src.as_f64x8())) } /// Compute the square root of packed double-precision (64-bit) floating-point elements in a, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set).\ @@ -7934,18 +7549,14 @@ pub unsafe fn _mm512_mask_sqrt_round_pd( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_sqrt_round_pd&expand=5373) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vsqrtpd, rounding = 8))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm512_maskz_sqrt_round_pd(k: __mmask8, a: __m512d, rounding: i32) -> __m512d { +#[cfg_attr(test, assert_instr(vsqrtpd, ROUNDING = 8))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_maskz_sqrt_round_pd(k: __mmask8, a: __m512d) -> __m512d { + static_assert_rounding!(ROUNDING); let a = a.as_f64x8(); - macro_rules! call { - ($imm4:expr) => { - vsqrtpd(a, $imm4) - }; - } - let sqrtround = constify_imm4_round!(rounding, call); + let r = vsqrtpd(a, ROUNDING); let zero = _mm512_setzero_pd().as_f64x8(); - transmute(simd_select_bitmask(k, sqrtround, zero)) + transmute(simd_select_bitmask(k, r, zero)) } /// Multiply packed single-precision (32-bit) floating-point elements in a and b, add the intermediate result to packed elements in c, and store the results in dst.\ @@ -7960,18 +7571,18 @@ pub unsafe fn _mm512_maskz_sqrt_round_pd(k: __mmask8, a: __m512d, rounding: i32) /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_fmadd_round_ps&expand=2565) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vfmadd, rounding = 8))] //vfmadd132ps or vfmadd213ps or vfmadd231ps -#[rustc_args_required_const(3)] -pub unsafe fn _mm512_fmadd_round_ps(a: __m512, b: __m512, c: __m512, rounding: i32) -> __m512 { +#[cfg_attr(test, assert_instr(vfmadd, ROUNDING = 8))] //vfmadd132ps or vfmadd213ps or vfmadd231ps +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_fmadd_round_ps( + a: __m512, + b: __m512, + c: __m512, +) -> __m512 { + static_assert_rounding!(ROUNDING); let a = a.as_f32x16(); let b = b.as_f32x16(); let c = c.as_f32x16(); - macro_rules! call { - ($imm4:expr) => { - vfmadd132psround(a, b, c, $imm4) - }; - } - let r = constify_imm4_round!(rounding, call); + let r = vfmadd132psround(a, b, c, ROUNDING); transmute(r) } @@ -7987,25 +7598,20 @@ pub unsafe fn _mm512_fmadd_round_ps(a: __m512, b: __m512, c: __m512, rounding: i /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_fmadd_round_ps&expand=2566) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vfmadd, rounding = 8))] //vfmadd132ps or vfmadd213ps or vfmadd231ps -#[rustc_args_required_const(4)] -pub unsafe fn _mm512_mask_fmadd_round_ps( +#[cfg_attr(test, assert_instr(vfmadd, ROUNDING = 8))] //vfmadd132ps or vfmadd213ps or vfmadd231ps +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_mask_fmadd_round_ps( a: __m512, k: __mmask16, b: __m512, c: __m512, - rounding: i32, ) -> __m512 { + static_assert_rounding!(ROUNDING); let a = a.as_f32x16(); let b = b.as_f32x16(); let c = c.as_f32x16(); - macro_rules! call { - ($imm4:expr) => { - vfmadd132psround(a, b, c, $imm4) - }; - } - let fmadd = constify_imm4_round!(rounding, call); - transmute(simd_select_bitmask(k, fmadd, a)) + let r = vfmadd132psround(a, b, c, ROUNDING); + transmute(simd_select_bitmask(k, r, a)) } /// Multiply packed single-precision (32-bit) floating-point elements in a and b, add the intermediate result to packed elements in c, and store the results in a using zeromask k (elements are zeroed out when the corresponding mask bit is not set).\ @@ -8020,26 +7626,21 @@ pub unsafe fn _mm512_mask_fmadd_round_ps( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_fmadd_round_ps&expand=2568) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vfmadd, rounding = 8))] //vfmadd132ps or vfmadd213ps or vfmadd231ps -#[rustc_args_required_const(4)] -pub unsafe fn _mm512_maskz_fmadd_round_ps( +#[cfg_attr(test, assert_instr(vfmadd, ROUNDING = 8))] //vfmadd132ps or vfmadd213ps or vfmadd231ps +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_maskz_fmadd_round_ps( k: __mmask16, a: __m512, b: __m512, c: __m512, - rounding: i32, ) -> __m512 { + static_assert_rounding!(ROUNDING); let a = a.as_f32x16(); let b = b.as_f32x16(); let c = c.as_f32x16(); - macro_rules! call { - ($imm4:expr) => { - vfmadd132psround(a, b, c, $imm4) - }; - } - let fmadd = constify_imm4_round!(rounding, call); + let r = vfmadd132psround(a, b, c, ROUNDING); let zero = _mm512_setzero_ps().as_f32x16(); - transmute(simd_select_bitmask(k, fmadd, zero)) + transmute(simd_select_bitmask(k, r, zero)) } /// Multiply packed single-precision (32-bit) floating-point elements in a and b, add the intermediate result to packed elements in c, and store the results in dst using writemask k (elements are copied from c when the corresponding mask bit is not set).\ @@ -8054,25 +7655,20 @@ pub unsafe fn _mm512_maskz_fmadd_round_ps( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask3_fmadd_round_ps&expand=2567) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vfmadd, rounding = 8))] //vfmadd132ps or vfmadd213ps or vfmadd231ps -#[rustc_args_required_const(4)] -pub unsafe fn _mm512_mask3_fmadd_round_ps( +#[cfg_attr(test, assert_instr(vfmadd, ROUNDING = 8))] //vfmadd132ps or vfmadd213ps or vfmadd231ps +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_mask3_fmadd_round_ps( a: __m512, b: __m512, c: __m512, k: __mmask16, - rounding: i32, ) -> __m512 { + static_assert_rounding!(ROUNDING); let a = a.as_f32x16(); let b = b.as_f32x16(); let c = c.as_f32x16(); - macro_rules! call { - ($imm4:expr) => { - vfmadd132psround(a, b, c, $imm4) - }; - } - let fmadd = constify_imm4_round!(rounding, call); - transmute(simd_select_bitmask(k, fmadd, c)) + let r = vfmadd132psround(a, b, c, ROUNDING); + transmute(simd_select_bitmask(k, r, c)) } /// Multiply packed double-precision (64-bit) floating-point elements in a and b, add the intermediate result to packed elements in c, and store the results in dst.\ @@ -8087,18 +7683,18 @@ pub unsafe fn _mm512_mask3_fmadd_round_ps( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_fmadd_round_pd&expand=2561) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vfmadd, rounding = 8))] //vfmadd132pd or vfmadd213pd or vfmadd231pd -#[rustc_args_required_const(3)] -pub unsafe fn _mm512_fmadd_round_pd(a: __m512d, b: __m512d, c: __m512d, rounding: i32) -> __m512d { +#[cfg_attr(test, assert_instr(vfmadd, ROUNDING = 8))] //vfmadd132pd or vfmadd213pd or vfmadd231pd +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_fmadd_round_pd( + a: __m512d, + b: __m512d, + c: __m512d, +) -> __m512d { + static_assert_rounding!(ROUNDING); let a = a.as_f64x8(); let b = b.as_f64x8(); let c = c.as_f64x8(); - macro_rules! call { - ($imm4:expr) => { - vfmadd132pdround(a, b, c, $imm4) - }; - } - let r = constify_imm4_round!(rounding, call); + let r = vfmadd132pdround(a, b, c, ROUNDING); transmute(r) } @@ -8114,25 +7710,20 @@ pub unsafe fn _mm512_fmadd_round_pd(a: __m512d, b: __m512d, c: __m512d, rounding /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_fmadd_round_pd&expand=2562) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vfmadd, rounding = 8))] //vfmadd132pd or vfmadd213pd or vfmadd231pd -#[rustc_args_required_const(4)] -pub unsafe fn _mm512_mask_fmadd_round_pd( +#[cfg_attr(test, assert_instr(vfmadd, ROUNDING = 8))] //vfmadd132pd or vfmadd213pd or vfmadd231pd +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_mask_fmadd_round_pd( a: __m512d, k: __mmask8, b: __m512d, c: __m512d, - rounding: i32, ) -> __m512d { + static_assert_rounding!(ROUNDING); let a = a.as_f64x8(); let b = b.as_f64x8(); let c = c.as_f64x8(); - macro_rules! call { - ($imm4:expr) => { - vfmadd132pdround(a, b, c, $imm4) - }; - } - let fmadd = constify_imm4_round!(rounding, call); - transmute(simd_select_bitmask(k, fmadd, a)) + let r = vfmadd132pdround(a, b, c, ROUNDING); + transmute(simd_select_bitmask(k, r, a)) } /// Multiply packed double-precision (64-bit) floating-point elements in a and b, add the intermediate result to packed elements in c, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set).\ @@ -8147,26 +7738,21 @@ pub unsafe fn _mm512_mask_fmadd_round_pd( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_fmadd_round_pd&expand=2564) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vfmadd, rounding = 8))] //vfmadd132pd or vfmadd213pd or vfmadd231pd -#[rustc_args_required_const(4)] -pub unsafe fn _mm512_maskz_fmadd_round_pd( +#[cfg_attr(test, assert_instr(vfmadd, ROUNDING = 8))] //vfmadd132pd or vfmadd213pd or vfmadd231pd +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_maskz_fmadd_round_pd( k: __mmask8, a: __m512d, b: __m512d, c: __m512d, - rounding: i32, ) -> __m512d { + static_assert_rounding!(ROUNDING); let a = a.as_f64x8(); let b = b.as_f64x8(); let c = c.as_f64x8(); - macro_rules! call { - ($imm4:expr) => { - vfmadd132pdround(a, b, c, $imm4) - }; - } - let fmadd = constify_imm4_round!(rounding, call); + let r = vfmadd132pdround(a, b, c, ROUNDING); let zero = _mm512_setzero_pd().as_f64x8(); - transmute(simd_select_bitmask(k, fmadd, zero)) + transmute(simd_select_bitmask(k, r, zero)) } /// Multiply packed double-precision (64-bit) floating-point elements in a and b, add the intermediate result to packed elements in c, and store the results in dst using writemask k (elements are copied from c when the corresponding mask bit is not set).\ @@ -8181,25 +7767,20 @@ pub unsafe fn _mm512_maskz_fmadd_round_pd( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask3_fmadd_round_pd&expand=2563) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vfmadd, rounding = 8))] //vfmadd132pd or vfmadd213pd or vfmadd231pd -#[rustc_args_required_const(4)] -pub unsafe fn _mm512_mask3_fmadd_round_pd( +#[cfg_attr(test, assert_instr(vfmadd, ROUNDING = 8))] //vfmadd132pd or vfmadd213pd or vfmadd231pd +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_mask3_fmadd_round_pd( a: __m512d, b: __m512d, c: __m512d, k: __mmask8, - rounding: i32, ) -> __m512d { + static_assert_rounding!(ROUNDING); let a = a.as_f64x8(); let b = b.as_f64x8(); let c = c.as_f64x8(); - macro_rules! call { - ($imm4:expr) => { - vfmadd132pdround(a, b, c, $imm4) - }; - } - let fmadd = constify_imm4_round!(rounding, call); - transmute(simd_select_bitmask(k, fmadd, c)) + let r = vfmadd132pdround(a, b, c, ROUNDING); + transmute(simd_select_bitmask(k, r, c)) } /// Multiply packed single-precision (32-bit) floating-point elements in a and b, subtract packed elements in c from the intermediate result, and store the results in dst.\ @@ -8214,19 +7795,19 @@ pub unsafe fn _mm512_mask3_fmadd_round_pd( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_fmsub_round_ps&expand=2651) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vfmadd, rounding = 8))] //vfmsub132ps or vfmsub213ps or vfmsub231ps, clang generates vfmadd, gcc generates vfmsub -#[rustc_args_required_const(3)] -pub unsafe fn _mm512_fmsub_round_ps(a: __m512, b: __m512, c: __m512, rounding: i32) -> __m512 { +#[cfg_attr(test, assert_instr(vfmadd, ROUNDING = 8))] //vfmsub132ps or vfmsub213ps or vfmsub231ps, clang generates vfmadd, gcc generates vfmsub +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_fmsub_round_ps( + a: __m512, + b: __m512, + c: __m512, +) -> __m512 { + static_assert_rounding!(ROUNDING); let zero: f32x16 = mem::zeroed(); let sub = simd_sub(zero, c.as_f32x16()); let a = a.as_f32x16(); let b = b.as_f32x16(); - macro_rules! call { - ($imm4:expr) => { - vfmadd132psround(a, b, sub, $imm4) - }; - } - let r = constify_imm4_round!(rounding, call); + let r = vfmadd132psround(a, b, sub, ROUNDING); transmute(r) } @@ -8242,26 +7823,21 @@ pub unsafe fn _mm512_fmsub_round_ps(a: __m512, b: __m512, c: __m512, rounding: i /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_fmsub_round_ps&expand=2652) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vfmadd, rounding = 8))] //vfmsub132ps or vfmsub213ps or vfmsub231ps, clang generates vfmadd, gcc generates vfmsub -#[rustc_args_required_const(4)] -pub unsafe fn _mm512_mask_fmsub_round_ps( +#[cfg_attr(test, assert_instr(vfmadd, ROUNDING = 8))] //vfmsub132ps or vfmsub213ps or vfmsub231ps, clang generates vfmadd, gcc generates vfmsub +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_mask_fmsub_round_ps( a: __m512, k: __mmask16, b: __m512, c: __m512, - rounding: i32, ) -> __m512 { + static_assert_rounding!(ROUNDING); let zero: f32x16 = mem::zeroed(); let sub = simd_sub(zero, c.as_f32x16()); let a = a.as_f32x16(); let b = b.as_f32x16(); - macro_rules! call { - ($imm4:expr) => { - vfmadd132psround(a, b, sub, $imm4) - }; - } - let fmsub = constify_imm4_round!(rounding, call); - transmute(simd_select_bitmask(k, fmsub, a)) + let r = vfmadd132psround(a, b, sub, ROUNDING); + transmute(simd_select_bitmask(k, r, a)) } /// Multiply packed single-precision (32-bit) floating-point elements in a and b, subtract packed elements in c from the intermediate result, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set).\ @@ -8276,26 +7852,21 @@ pub unsafe fn _mm512_mask_fmsub_round_ps( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_fmsub_round_ps&expand=2654) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vfmadd, rounding = 8))] //vfmsub132ps or vfmsub213ps or vfmsub231ps, clang generates vfmadd, gcc generates vfmsub -#[rustc_args_required_const(4)] -pub unsafe fn _mm512_maskz_fmsub_round_ps( +#[cfg_attr(test, assert_instr(vfmadd, ROUNDING = 8))] //vfmsub132ps or vfmsub213ps or vfmsub231ps, clang generates vfmadd, gcc generates vfmsub +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_maskz_fmsub_round_ps( k: __mmask16, a: __m512, b: __m512, c: __m512, - rounding: i32, ) -> __m512 { + static_assert_rounding!(ROUNDING); let zero: f32x16 = mem::zeroed(); let sub = simd_sub(zero, c.as_f32x16()); let a = a.as_f32x16(); let b = b.as_f32x16(); - macro_rules! call { - ($imm4:expr) => { - vfmadd132psround(a, b, sub, $imm4) - }; - } - let fmsub = constify_imm4_round!(rounding, call); - transmute(simd_select_bitmask(k, fmsub, zero)) + let r = vfmadd132psround(a, b, sub, ROUNDING); + transmute(simd_select_bitmask(k, r, zero)) } /// Multiply packed single-precision (32-bit) floating-point elements in a and b, subtract packed elements in c from the intermediate result, and store the results in dst using writemask k (elements are copied from c when the corresponding mask bit is not set).\ @@ -8310,27 +7881,22 @@ pub unsafe fn _mm512_maskz_fmsub_round_ps( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask3_fmsub_round_ps&expand=2653) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vfmadd, rounding = 8))] //vfmsub132ps or vfmsub213ps or vfmsub231ps, clang generates vfmadd, gcc generates vfmsub -#[rustc_args_required_const(4)] -pub unsafe fn _mm512_mask3_fmsub_round_ps( +#[cfg_attr(test, assert_instr(vfmadd, ROUNDING = 8))] //vfmsub132ps or vfmsub213ps or vfmsub231ps, clang generates vfmadd, gcc generates vfmsub +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_mask3_fmsub_round_ps( a: __m512, b: __m512, c: __m512, k: __mmask16, - rounding: i32, ) -> __m512 { + static_assert_rounding!(ROUNDING); let zero: f32x16 = mem::zeroed(); let c = c.as_f32x16(); let sub = simd_sub(zero, c); let a = a.as_f32x16(); let b = b.as_f32x16(); - macro_rules! call { - ($imm4:expr) => { - vfmadd132psround(a, b, sub, $imm4) - }; - } - let fmsub = constify_imm4_round!(rounding, call); - transmute(simd_select_bitmask(k, fmsub, c)) + let r = vfmadd132psround(a, b, sub, ROUNDING); + transmute(simd_select_bitmask(k, r, c)) } /// Multiply packed double-precision (64-bit) floating-point elements in a and b, subtract packed elements in c from the intermediate result, and store the results in dst.\ @@ -8345,19 +7911,19 @@ pub unsafe fn _mm512_mask3_fmsub_round_ps( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_fmsub_round_pd&expand=2647) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vfmadd, rounding = 8))] //vfmsub132pd or vfmsub213pd or vfmsub231pd. clang generates fmadd, gcc generates fmsub -#[rustc_args_required_const(3)] -pub unsafe fn _mm512_fmsub_round_pd(a: __m512d, b: __m512d, c: __m512d, rounding: i32) -> __m512d { +#[cfg_attr(test, assert_instr(vfmadd, ROUNDING = 8))] //vfmsub132pd or vfmsub213pd or vfmsub231pd. clang generates fmadd, gcc generates fmsub +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_fmsub_round_pd( + a: __m512d, + b: __m512d, + c: __m512d, +) -> __m512d { + static_assert_rounding!(ROUNDING); let zero: f64x8 = mem::zeroed(); let sub = simd_sub(zero, c.as_f64x8()); let a = a.as_f64x8(); let b = b.as_f64x8(); - macro_rules! call { - ($imm4:expr) => { - vfmadd132pdround(a, b, sub, $imm4) - }; - } - let r = constify_imm4_round!(rounding, call); + let r = vfmadd132pdround(a, b, sub, ROUNDING); transmute(r) } @@ -8373,26 +7939,21 @@ pub unsafe fn _mm512_fmsub_round_pd(a: __m512d, b: __m512d, c: __m512d, rounding /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_fmsub_round_pd&expand=2648) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vfmadd, rounding = 8))] //vfmsub132pd or vfmsub213pd or vfmsub231pd. clang generates fmadd, gcc generates fmsub -#[rustc_args_required_const(4)] -pub unsafe fn _mm512_mask_fmsub_round_pd( +#[cfg_attr(test, assert_instr(vfmadd, ROUNDING = 8))] //vfmsub132pd or vfmsub213pd or vfmsub231pd. clang generates fmadd, gcc generates fmsub +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_mask_fmsub_round_pd( a: __m512d, k: __mmask8, b: __m512d, c: __m512d, - rounding: i32, ) -> __m512d { + static_assert_rounding!(ROUNDING); let zero: f64x8 = mem::zeroed(); let sub = simd_sub(zero, c.as_f64x8()); let a = a.as_f64x8(); let b = b.as_f64x8(); - macro_rules! call { - ($imm4:expr) => { - vfmadd132pdround(a, b, sub, $imm4) - }; - } - let fmsub = constify_imm4_round!(rounding, call); - transmute(simd_select_bitmask(k, fmsub, a)) + let r = vfmadd132pdround(a, b, sub, ROUNDING); + transmute(simd_select_bitmask(k, r, a)) } /// Multiply packed double-precision (64-bit) floating-point elements in a and b, subtract packed elements in c from the intermediate result, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set).\ @@ -8407,26 +7968,21 @@ pub unsafe fn _mm512_mask_fmsub_round_pd( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_fmsub_round_pd&expand=2650) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vfmadd, rounding = 8))] //vfmsub132pd or vfmsub213pd or vfmsub231pd. clang generates fmadd, gcc generates fmsub -#[rustc_args_required_const(4)] -pub unsafe fn _mm512_maskz_fmsub_round_pd( +#[cfg_attr(test, assert_instr(vfmadd, ROUNDING = 8))] //vfmsub132pd or vfmsub213pd or vfmsub231pd. clang generates fmadd, gcc generates fmsub +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_maskz_fmsub_round_pd( k: __mmask8, a: __m512d, b: __m512d, c: __m512d, - rounding: i32, ) -> __m512d { + static_assert_rounding!(ROUNDING); let zero: f64x8 = mem::zeroed(); let sub = simd_sub(zero, c.as_f64x8()); let a = a.as_f64x8(); let b = b.as_f64x8(); - macro_rules! call { - ($imm4:expr) => { - vfmadd132pdround(a, b, sub, $imm4) - }; - } - let fmsub = constify_imm4_round!(rounding, call); - transmute(simd_select_bitmask(k, fmsub, zero)) + let r = vfmadd132pdround(a, b, sub, ROUNDING); + transmute(simd_select_bitmask(k, r, zero)) } /// Multiply packed double-precision (64-bit) floating-point elements in a and b, subtract packed elements in c from the intermediate result, and store the results in dst using writemask k (elements are copied from c when the corresponding mask bit is not set).\ @@ -8441,27 +7997,22 @@ pub unsafe fn _mm512_maskz_fmsub_round_pd( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask3_fmsub_round_pd&expand=2649) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vfmadd, rounding = 8))] //vfmsub132pd or vfmsub213pd or vfmsub231pd. clang generates fmadd, gcc generates fmsub -#[rustc_args_required_const(4)] -pub unsafe fn _mm512_mask3_fmsub_round_pd( +#[cfg_attr(test, assert_instr(vfmadd, ROUNDING = 8))] //vfmsub132pd or vfmsub213pd or vfmsub231pd. clang generates fmadd, gcc generates fmsub +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_mask3_fmsub_round_pd( a: __m512d, b: __m512d, c: __m512d, k: __mmask8, - rounding: i32, ) -> __m512d { + static_assert_rounding!(ROUNDING); let zero: f64x8 = mem::zeroed(); let c = c.as_f64x8(); let sub = simd_sub(zero, c); let a = a.as_f64x8(); let b = b.as_f64x8(); - macro_rules! call { - ($imm4:expr) => { - vfmadd132pdround(a, b, sub, $imm4) - }; - } - let fmsub = constify_imm4_round!(rounding, call); - transmute(simd_select_bitmask(k, fmsub, c)) + let r = vfmadd132pdround(a, b, sub, ROUNDING); + transmute(simd_select_bitmask(k, r, c)) } /// Multiply packed single-precision (32-bit) floating-point elements in a and b, alternatively add and subtract packed elements in c to/from the intermediate result, and store the results in dst.\ @@ -8476,18 +8027,18 @@ pub unsafe fn _mm512_mask3_fmsub_round_pd( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_fmaddsub_round_ps&expand=2619) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vfmaddsub, rounding = 8))] //vfmaddsub132ps or vfmaddsub213ps or vfmaddsub231ps -#[rustc_args_required_const(3)] -pub unsafe fn _mm512_fmaddsub_round_ps(a: __m512, b: __m512, c: __m512, rounding: i32) -> __m512 { +#[cfg_attr(test, assert_instr(vfmaddsub, ROUNDING = 8))] //vfmaddsub132ps or vfmaddsub213ps or vfmaddsub231ps +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_fmaddsub_round_ps( + a: __m512, + b: __m512, + c: __m512, +) -> __m512 { + static_assert_rounding!(ROUNDING); let a = a.as_f32x16(); let b = b.as_f32x16(); let c = c.as_f32x16(); - macro_rules! call { - ($imm4:expr) => { - vfmaddsub213ps(a, b, c, $imm4) - }; - } - let r = constify_imm4_round!(rounding, call); + let r = vfmaddsub213ps(a, b, c, ROUNDING); transmute(r) } @@ -8503,25 +8054,20 @@ pub unsafe fn _mm512_fmaddsub_round_ps(a: __m512, b: __m512, c: __m512, rounding /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_fmaddsub_round_ps&expand=2620) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vfmaddsub, rounding = 8))] //vfmaddsub132ps or vfmaddsub213ps or vfmaddsub231ps -#[rustc_args_required_const(4)] -pub unsafe fn _mm512_mask_fmaddsub_round_ps( +#[cfg_attr(test, assert_instr(vfmaddsub, ROUNDING = 8))] //vfmaddsub132ps or vfmaddsub213ps or vfmaddsub231ps +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_mask_fmaddsub_round_ps( a: __m512, k: __mmask16, b: __m512, c: __m512, - rounding: i32, ) -> __m512 { + static_assert_rounding!(ROUNDING); let a = a.as_f32x16(); let b = b.as_f32x16(); let c = c.as_f32x16(); - macro_rules! call { - ($imm4:expr) => { - vfmaddsub213ps(a, b, c, $imm4) - }; - } - let fmaddsub = constify_imm4_round!(rounding, call); - transmute(simd_select_bitmask(k, fmaddsub, a)) + let r = vfmaddsub213ps(a, b, c, ROUNDING); + transmute(simd_select_bitmask(k, r, a)) } /// Multiply packed single-precision (32-bit) floating-point elements in a and b, alternatively add and subtract packed elements in c to/from the intermediate result, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set).\ @@ -8536,26 +8082,21 @@ pub unsafe fn _mm512_mask_fmaddsub_round_ps( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_fmaddsub_round_ps&expand=2622) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vfmaddsub, rounding = 8))] //vfmaddsub132ps or vfmaddsub213ps or vfmaddsub231ps -#[rustc_args_required_const(4)] -pub unsafe fn _mm512_maskz_fmaddsub_round_ps( +#[cfg_attr(test, assert_instr(vfmaddsub, ROUNDING = 8))] //vfmaddsub132ps or vfmaddsub213ps or vfmaddsub231ps +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_maskz_fmaddsub_round_ps( k: __mmask16, a: __m512, b: __m512, c: __m512, - rounding: i32, ) -> __m512 { + static_assert_rounding!(ROUNDING); let a = a.as_f32x16(); let b = b.as_f32x16(); let c = c.as_f32x16(); - macro_rules! call { - ($imm4:expr) => { - vfmaddsub213ps(a, b, c, $imm4) - }; - } - let fmaddsub = constify_imm4_round!(rounding, call); + let r = vfmaddsub213ps(a, b, c, ROUNDING); let zero = _mm512_setzero_ps().as_f32x16(); - transmute(simd_select_bitmask(k, fmaddsub, zero)) + transmute(simd_select_bitmask(k, r, zero)) } /// Multiply packed single-precision (32-bit) floating-point elements in a and b, alternatively add and subtract packed elements in c to/from the intermediate result, and store the results in dst using writemask k (elements are copied from c when the corresponding mask bit is not set).\ @@ -8570,25 +8111,20 @@ pub unsafe fn _mm512_maskz_fmaddsub_round_ps( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask3_fmaddsub_round_ps&expand=2621) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vfmaddsub, rounding = 8))] //vfmaddsub132ps or vfmaddsub213ps or vfmaddsub231ps -#[rustc_args_required_const(4)] -pub unsafe fn _mm512_mask3_fmaddsub_round_ps( +#[cfg_attr(test, assert_instr(vfmaddsub, ROUNDING = 8))] //vfmaddsub132ps or vfmaddsub213ps or vfmaddsub231ps +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_mask3_fmaddsub_round_ps( a: __m512, b: __m512, c: __m512, k: __mmask16, - rounding: i32, ) -> __m512 { + static_assert_rounding!(ROUNDING); let a = a.as_f32x16(); let b = b.as_f32x16(); let c = c.as_f32x16(); - macro_rules! call { - ($imm4:expr) => { - vfmaddsub213ps(a, b, c, $imm4) - }; - } - let fmaddsub = constify_imm4_round!(rounding, call); - transmute(simd_select_bitmask(k, fmaddsub, c)) + let r = vfmaddsub213ps(a, b, c, ROUNDING); + transmute(simd_select_bitmask(k, r, c)) } /// Multiply packed double-precision (64-bit) floating-point elements in a and b, alternatively add and subtract packed elements in c to/from the intermediate result, and store the results in dst.\ @@ -8603,23 +8139,18 @@ pub unsafe fn _mm512_mask3_fmaddsub_round_ps( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_fmaddsub_round_pd&expand=2615) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vfmaddsub, rounding = 8))] //vfmaddsub132pd or vfmaddsub213pd or vfmaddsub231pd -#[rustc_args_required_const(3)] -pub unsafe fn _mm512_fmaddsub_round_pd( +#[cfg_attr(test, assert_instr(vfmaddsub, ROUNDING = 8))] //vfmaddsub132pd or vfmaddsub213pd or vfmaddsub231pd +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_fmaddsub_round_pd( a: __m512d, b: __m512d, c: __m512d, - rounding: i32, ) -> __m512d { + static_assert_rounding!(ROUNDING); let a = a.as_f64x8(); let b = b.as_f64x8(); let c = c.as_f64x8(); - macro_rules! call { - ($imm4:expr) => { - vfmaddsub213pd(a, b, c, $imm4) - }; - } - let r = constify_imm4_round!(rounding, call); + let r = vfmaddsub213pd(a, b, c, ROUNDING); transmute(r) } @@ -8635,25 +8166,20 @@ pub unsafe fn _mm512_fmaddsub_round_pd( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_fmaddsub_round_pd&expand=2616) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vfmaddsub, rounding = 8))] //vfmaddsub132pd or vfmaddsub213pd or vfmaddsub231pd -#[rustc_args_required_const(4)] -pub unsafe fn _mm512_mask_fmaddsub_round_pd( +#[cfg_attr(test, assert_instr(vfmaddsub, ROUNDING = 8))] //vfmaddsub132pd or vfmaddsub213pd or vfmaddsub231pd +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_mask_fmaddsub_round_pd( a: __m512d, k: __mmask8, b: __m512d, c: __m512d, - rounding: i32, ) -> __m512d { + static_assert_rounding!(ROUNDING); let a = a.as_f64x8(); let b = b.as_f64x8(); let c = c.as_f64x8(); - macro_rules! call { - ($imm4:expr) => { - vfmaddsub213pd(a, b, c, $imm4) - }; - } - let fmaddsub = constify_imm4_round!(rounding, call); - transmute(simd_select_bitmask(k, fmaddsub, a)) + let r = vfmaddsub213pd(a, b, c, ROUNDING); + transmute(simd_select_bitmask(k, r, a)) } /// Multiply packed double-precision (64-bit) floating-point elements in a and b, alternatively add and subtract packed elements in c to/from the intermediate result, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set).\ @@ -8668,26 +8194,21 @@ pub unsafe fn _mm512_mask_fmaddsub_round_pd( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_fmaddsub_round_pd&expand=2618) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vfmaddsub, rounding = 8))] //vfmaddsub132pd or vfmaddsub213pd or vfmaddsub231pd -#[rustc_args_required_const(4)] -pub unsafe fn _mm512_maskz_fmaddsub_round_pd( +#[cfg_attr(test, assert_instr(vfmaddsub, ROUNDING = 8))] //vfmaddsub132pd or vfmaddsub213pd or vfmaddsub231pd +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_maskz_fmaddsub_round_pd( k: __mmask8, a: __m512d, b: __m512d, c: __m512d, - rounding: i32, ) -> __m512d { + static_assert_rounding!(ROUNDING); let a = a.as_f64x8(); let b = b.as_f64x8(); let c = c.as_f64x8(); - macro_rules! call { - ($imm4:expr) => { - vfmaddsub213pd(a, b, c, $imm4) - }; - } - let fmaddsub = constify_imm4_round!(rounding, call); + let r = vfmaddsub213pd(a, b, c, ROUNDING); let zero = _mm512_setzero_pd().as_f64x8(); - transmute(simd_select_bitmask(k, fmaddsub, zero)) + transmute(simd_select_bitmask(k, r, zero)) } /// Multiply packed single-precision (32-bit) floating-point elements in a and b, alternatively add and subtract packed elements in c to/from the intermediate result, and store the results in dst using writemask k (elements are copied from c when the corresponding mask bit is not set).\ @@ -8702,25 +8223,20 @@ pub unsafe fn _mm512_maskz_fmaddsub_round_pd( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask3_fmaddsub_round_pd&expand=2617) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vfmaddsub, rounding = 8))] //vfmaddsub132pd or vfmaddsub213pd or vfmaddsub231pd -#[rustc_args_required_const(4)] -pub unsafe fn _mm512_mask3_fmaddsub_round_pd( +#[cfg_attr(test, assert_instr(vfmaddsub, ROUNDING = 8))] //vfmaddsub132pd or vfmaddsub213pd or vfmaddsub231pd +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_mask3_fmaddsub_round_pd( a: __m512d, b: __m512d, c: __m512d, k: __mmask8, - rounding: i32, ) -> __m512d { + static_assert_rounding!(ROUNDING); let a = a.as_f64x8(); let b = b.as_f64x8(); let c = c.as_f64x8(); - macro_rules! call { - ($imm4:expr) => { - vfmaddsub213pd(a, b, c, $imm4) - }; - } - let fmaddsub = constify_imm4_round!(rounding, call); - transmute(simd_select_bitmask(k, fmaddsub, c)) + let r = vfmaddsub213pd(a, b, c, ROUNDING); + transmute(simd_select_bitmask(k, r, c)) } /// Multiply packed single-precision (32-bit) floating-point elements in a and b, alternatively subtract and add packed elements in c from/to the intermediate result, and store the results in dst.\ @@ -8735,19 +8251,19 @@ pub unsafe fn _mm512_mask3_fmaddsub_round_pd( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_fmsubadd_round_ps&expand=2699) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vfmaddsub, rounding = 8))] //vfmsubadd132ps or vfmsubadd213ps or vfmsubadd231ps -#[rustc_args_required_const(3)] -pub unsafe fn _mm512_fmsubadd_round_ps(a: __m512, b: __m512, c: __m512, rounding: i32) -> __m512 { +#[cfg_attr(test, assert_instr(vfmaddsub, ROUNDING = 8))] //vfmsubadd132ps or vfmsubadd213ps or vfmsubadd231ps +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_fmsubadd_round_ps( + a: __m512, + b: __m512, + c: __m512, +) -> __m512 { + static_assert_rounding!(ROUNDING); let zero: f32x16 = mem::zeroed(); let sub = simd_sub(zero, c.as_f32x16()); let a = a.as_f32x16(); let b = b.as_f32x16(); - macro_rules! call { - ($imm4:expr) => { - vfmaddsub213ps(a, b, sub, $imm4) - }; - } - let r = constify_imm4_round!(rounding, call); + let r = vfmaddsub213ps(a, b, sub, ROUNDING); transmute(r) } @@ -8763,26 +8279,21 @@ pub unsafe fn _mm512_fmsubadd_round_ps(a: __m512, b: __m512, c: __m512, rounding /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_fmsubadd_round_ps&expand=2700) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vfmaddsub, rounding = 8))] //vfmsubadd132ps or vfmsubadd213ps or vfmsubadd231ps -#[rustc_args_required_const(4)] -pub unsafe fn _mm512_mask_fmsubadd_round_ps( +#[cfg_attr(test, assert_instr(vfmaddsub, ROUNDING = 8))] //vfmsubadd132ps or vfmsubadd213ps or vfmsubadd231ps +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_mask_fmsubadd_round_ps( a: __m512, k: __mmask16, b: __m512, c: __m512, - rounding: i32, ) -> __m512 { + static_assert_rounding!(ROUNDING); let zero: f32x16 = mem::zeroed(); let sub = simd_sub(zero, c.as_f32x16()); let a = a.as_f32x16(); let b = b.as_f32x16(); - macro_rules! call { - ($imm4:expr) => { - vfmaddsub213ps(a, b, sub, $imm4) - }; - } - let fmsubadd = constify_imm4_round!(rounding, call); - transmute(simd_select_bitmask(k, fmsubadd, a)) + let r = vfmaddsub213ps(a, b, sub, ROUNDING); + transmute(simd_select_bitmask(k, r, a)) } /// Multiply packed single-precision (32-bit) floating-point elements in a and b, alternatively subtract and add packed elements in c from/to the intermediate result, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set).\ @@ -8797,26 +8308,21 @@ pub unsafe fn _mm512_mask_fmsubadd_round_ps( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_fmsubadd_round_ps&expand=2702) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vfmaddsub, rounding = 8))] //vfmsubadd132ps or vfmsubadd213ps or vfmsubadd231ps -#[rustc_args_required_const(4)] -pub unsafe fn _mm512_maskz_fmsubadd_round_ps( +#[cfg_attr(test, assert_instr(vfmaddsub, ROUNDING = 8))] //vfmsubadd132ps or vfmsubadd213ps or vfmsubadd231ps +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_maskz_fmsubadd_round_ps( k: __mmask16, a: __m512, b: __m512, c: __m512, - rounding: i32, ) -> __m512 { + static_assert_rounding!(ROUNDING); let zero: f32x16 = mem::zeroed(); let sub = simd_sub(zero, c.as_f32x16()); let a = a.as_f32x16(); let b = b.as_f32x16(); - macro_rules! call { - ($imm4:expr) => { - vfmaddsub213ps(a, b, sub, $imm4) - }; - } - let fmsubadd = constify_imm4_round!(rounding, call); - transmute(simd_select_bitmask(k, fmsubadd, zero)) + let r = vfmaddsub213ps(a, b, sub, ROUNDING); + transmute(simd_select_bitmask(k, r, zero)) } /// Multiply packed single-precision (32-bit) floating-point elements in a and b, alternatively subtract and add packed elements in c from/to the intermediate result, and store the results in dst using writemask k (elements are copied from c when the corresponding mask bit is not set).\ @@ -8831,27 +8337,22 @@ pub unsafe fn _mm512_maskz_fmsubadd_round_ps( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask3_fmsubadd_round_ps&expand=2701) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vfmaddsub, rounding = 8))] //vfmsubadd132ps or vfmsubadd213ps or vfmsubadd231ps -#[rustc_args_required_const(4)] -pub unsafe fn _mm512_mask3_fmsubadd_round_ps( +#[cfg_attr(test, assert_instr(vfmaddsub, ROUNDING = 8))] //vfmsubadd132ps or vfmsubadd213ps or vfmsubadd231ps +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_mask3_fmsubadd_round_ps( a: __m512, b: __m512, c: __m512, k: __mmask16, - rounding: i32, ) -> __m512 { + static_assert_rounding!(ROUNDING); let zero: f32x16 = mem::zeroed(); let c = c.as_f32x16(); let sub = simd_sub(zero, c); let a = a.as_f32x16(); let b = b.as_f32x16(); - macro_rules! call { - ($imm4:expr) => { - vfmaddsub213ps(a, b, sub, $imm4) - }; - } - let fmsubadd = constify_imm4_round!(rounding, call); - transmute(simd_select_bitmask(k, fmsubadd, c)) + let r = vfmaddsub213ps(a, b, sub, ROUNDING); + transmute(simd_select_bitmask(k, r, c)) } /// Multiply packed double-precision (64-bit) floating-point elements in a and b, alternatively subtract and add packed elements in c from/to the intermediate result, and store the results in dst.\ @@ -8866,24 +8367,19 @@ pub unsafe fn _mm512_mask3_fmsubadd_round_ps( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_fmsubadd_round_pd&expand=2695) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vfmaddsub, rounding = 8))] //vfmsubadd132pd or vfmsubadd213pd or vfmsubadd231pd -#[rustc_args_required_const(3)] -pub unsafe fn _mm512_fmsubadd_round_pd( +#[cfg_attr(test, assert_instr(vfmaddsub, ROUNDING = 8))] //vfmsubadd132pd or vfmsubadd213pd or vfmsubadd231pd +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_fmsubadd_round_pd( a: __m512d, b: __m512d, c: __m512d, - rounding: i32, ) -> __m512d { + static_assert_rounding!(ROUNDING); let zero: f64x8 = mem::zeroed(); let sub = simd_sub(zero, c.as_f64x8()); let a = a.as_f64x8(); let b = b.as_f64x8(); - macro_rules! call { - ($imm4:expr) => { - vfmaddsub213pd(a, b, sub, $imm4) - }; - } - let r = constify_imm4_round!(rounding, call); + let r = vfmaddsub213pd(a, b, sub, ROUNDING); transmute(r) } @@ -8899,26 +8395,21 @@ pub unsafe fn _mm512_fmsubadd_round_pd( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_fmsubadd_round_pd&expand=2696) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vfmaddsub, rounding = 8))] //vfmsubadd132pd or vfmsubadd213pd or vfmsubadd231pd -#[rustc_args_required_const(4)] -pub unsafe fn _mm512_mask_fmsubadd_round_pd( +#[cfg_attr(test, assert_instr(vfmaddsub, ROUNDING = 8))] //vfmsubadd132pd or vfmsubadd213pd or vfmsubadd231pd +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_mask_fmsubadd_round_pd( a: __m512d, k: __mmask8, b: __m512d, c: __m512d, - rounding: i32, ) -> __m512d { + static_assert_rounding!(ROUNDING); let zero: f64x8 = mem::zeroed(); let sub = simd_sub(zero, c.as_f64x8()); let a = a.as_f64x8(); let b = b.as_f64x8(); - macro_rules! call { - ($imm4:expr) => { - vfmaddsub213pd(a, b, sub, $imm4) - }; - } - let fmsubadd = constify_imm4_round!(rounding, call); - transmute(simd_select_bitmask(k, fmsubadd, a)) + let r = vfmaddsub213pd(a, b, sub, ROUNDING); + transmute(simd_select_bitmask(k, r, a)) } /// Multiply packed double-precision (64-bit) floating-point elements in a and b, alternatively add and subtract packed elements in c to/from the intermediate result, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set).\ @@ -8933,26 +8424,21 @@ pub unsafe fn _mm512_mask_fmsubadd_round_pd( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_fmsubadd_round_pd&expand=2698) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vfmaddsub, rounding = 8))] //vfmsubadd132pd or vfmsubadd213pd or vfmsubadd231pd -#[rustc_args_required_const(4)] -pub unsafe fn _mm512_maskz_fmsubadd_round_pd( +#[cfg_attr(test, assert_instr(vfmaddsub, ROUNDING = 8))] //vfmsubadd132pd or vfmsubadd213pd or vfmsubadd231pd +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_maskz_fmsubadd_round_pd( k: __mmask8, a: __m512d, b: __m512d, c: __m512d, - rounding: i32, ) -> __m512d { + static_assert_rounding!(ROUNDING); let zero: f64x8 = mem::zeroed(); let sub = simd_sub(zero, c.as_f64x8()); let a = a.as_f64x8(); let b = b.as_f64x8(); - macro_rules! call { - ($imm4:expr) => { - vfmaddsub213pd(a, b, sub, $imm4) - }; - } - let fmsubadd = constify_imm4_round!(rounding, call); - transmute(simd_select_bitmask(k, fmsubadd, zero)) + let r = vfmaddsub213pd(a, b, sub, ROUNDING); + transmute(simd_select_bitmask(k, r, zero)) } /// Multiply packed double-precision (64-bit) floating-point elements in a and b, alternatively subtract and add packed elements in c from/to the intermediate result, and store the results in dst using writemask k (elements are copied from c when the corresponding mask bit is not set).\ @@ -8967,27 +8453,22 @@ pub unsafe fn _mm512_maskz_fmsubadd_round_pd( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask3_fmsubadd_round_pd&expand=2697) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vfmaddsub, rounding = 8))] //vfmsubadd132pd or vfmsubadd213pd or vfmsubadd231pd -#[rustc_args_required_const(4)] -pub unsafe fn _mm512_mask3_fmsubadd_round_pd( +#[cfg_attr(test, assert_instr(vfmaddsub, ROUNDING = 8))] //vfmsubadd132pd or vfmsubadd213pd or vfmsubadd231pd +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_mask3_fmsubadd_round_pd( a: __m512d, b: __m512d, c: __m512d, k: __mmask8, - rounding: i32, ) -> __m512d { + static_assert_rounding!(ROUNDING); let zero: f64x8 = mem::zeroed(); let c = c.as_f64x8(); let sub = simd_sub(zero, c); let a = a.as_f64x8(); let b = b.as_f64x8(); - macro_rules! call { - ($imm4:expr) => { - vfmaddsub213pd(a, b, sub, $imm4) - }; - } - let fmsubadd = constify_imm4_round!(rounding, call); - transmute(simd_select_bitmask(k, fmsubadd, c)) + let r = vfmaddsub213pd(a, b, sub, ROUNDING); + transmute(simd_select_bitmask(k, r, c)) } /// Multiply packed single-precision (32-bit) floating-point elements in a and b, add the negated intermediate result to packed elements in c, and store the results in dst.\ @@ -9002,19 +8483,19 @@ pub unsafe fn _mm512_mask3_fmsubadd_round_pd( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_fnmadd_round_ps&expand=2731) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vfmadd, rounding = 8))] //vfnmadd132ps or vfnmadd213ps or vfnmadd231ps -#[rustc_args_required_const(3)] -pub unsafe fn _mm512_fnmadd_round_ps(a: __m512, b: __m512, c: __m512, rounding: i32) -> __m512 { +#[cfg_attr(test, assert_instr(vfmadd, ROUNDING = 8))] //vfnmadd132ps or vfnmadd213ps or vfnmadd231ps +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_fnmadd_round_ps( + a: __m512, + b: __m512, + c: __m512, +) -> __m512 { + static_assert_rounding!(ROUNDING); let zero: f32x16 = mem::zeroed(); let sub = simd_sub(zero, a.as_f32x16()); let b = b.as_f32x16(); let c = c.as_f32x16(); - macro_rules! call { - ($imm4:expr) => { - vfmadd132psround(sub, b, c, $imm4) - }; - } - let r = constify_imm4_round!(rounding, call); + let r = vfmadd132psround(sub, b, c, ROUNDING); transmute(r) } @@ -9030,26 +8511,21 @@ pub unsafe fn _mm512_fnmadd_round_ps(a: __m512, b: __m512, c: __m512, rounding: /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_fnmadd_round_ps&expand=2732) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vfmadd, rounding = 8))] //vfnmadd132ps or vfnmadd213ps or vfnmadd231ps -#[rustc_args_required_const(4)] -pub unsafe fn _mm512_mask_fnmadd_round_ps( +#[cfg_attr(test, assert_instr(vfmadd, ROUNDING = 8))] //vfnmadd132ps or vfnmadd213ps or vfnmadd231ps +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_mask_fnmadd_round_ps( a: __m512, k: __mmask16, b: __m512, c: __m512, - rounding: i32, ) -> __m512 { + static_assert_rounding!(ROUNDING); let zero: f32x16 = mem::zeroed(); let sub = simd_sub(zero, a.as_f32x16()); let b = b.as_f32x16(); let c = c.as_f32x16(); - macro_rules! call { - ($imm4:expr) => { - vfmadd132psround(sub, b, c, $imm4) - }; - } - let fnmadd = constify_imm4_round!(rounding, call); - transmute(simd_select_bitmask(k, fnmadd, a.as_f32x16())) + let r = vfmadd132psround(sub, b, c, ROUNDING); + transmute(simd_select_bitmask(k, r, a.as_f32x16())) } /// Multiply packed single-precision (32-bit) floating-point elements in a and b, add the negated intermediate result to packed elements in c, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set).\ @@ -9064,26 +8540,21 @@ pub unsafe fn _mm512_mask_fnmadd_round_ps( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_fnmadd_round_ps&expand=2734) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vfmadd, rounding = 8))] //vfnmadd132ps or vfnmadd213ps or vfnmadd231ps -#[rustc_args_required_const(4)] -pub unsafe fn _mm512_maskz_fnmadd_round_ps( +#[cfg_attr(test, assert_instr(vfmadd, ROUNDING = 8))] //vfnmadd132ps or vfnmadd213ps or vfnmadd231ps +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_maskz_fnmadd_round_ps( k: __mmask16, a: __m512, b: __m512, c: __m512, - rounding: i32, ) -> __m512 { + static_assert_rounding!(ROUNDING); let zero: f32x16 = mem::zeroed(); let sub = simd_sub(zero, a.as_f32x16()); let b = b.as_f32x16(); let c = c.as_f32x16(); - macro_rules! call { - ($imm4:expr) => { - vfmadd132psround(sub, b, c, $imm4) - }; - } - let fnmadd = constify_imm4_round!(rounding, call); - transmute(simd_select_bitmask(k, fnmadd, zero)) + let r = vfmadd132psround(sub, b, c, ROUNDING); + transmute(simd_select_bitmask(k, r, zero)) } /// Multiply packed single-precision (32-bit) floating-point elements in a and b, add the negated intermediate result to packed elements in c, and store the results in dst using writemask k (elements are copied from c when the corresponding mask bit is not set).\ @@ -9098,26 +8569,21 @@ pub unsafe fn _mm512_maskz_fnmadd_round_ps( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask3_fnmadd_round_ps&expand=2733) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vfmadd, rounding = 8))] //vfnmadd132ps or vfnmadd213ps or vfnmadd231ps -#[rustc_args_required_const(4)] -pub unsafe fn _mm512_mask3_fnmadd_round_ps( +#[cfg_attr(test, assert_instr(vfmadd, ROUNDING = 8))] //vfnmadd132ps or vfnmadd213ps or vfnmadd231ps +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_mask3_fnmadd_round_ps( a: __m512, b: __m512, c: __m512, k: __mmask16, - rounding: i32, ) -> __m512 { + static_assert_rounding!(ROUNDING); let zero: f32x16 = mem::zeroed(); let sub = simd_sub(zero, a.as_f32x16()); let b = b.as_f32x16(); let c = c.as_f32x16(); - macro_rules! call { - ($imm4:expr) => { - vfmadd132psround(sub, b, c, $imm4) - }; - } - let fnmadd = constify_imm4_round!(rounding, call); - transmute(simd_select_bitmask(k, fnmadd, c)) + let r = vfmadd132psround(sub, b, c, ROUNDING); + transmute(simd_select_bitmask(k, r, c)) } /// Multiply packed double-precision (64-bit) floating-point elements in a and b, add the negated intermediate result to packed elements in c, and store the results in dst.\ @@ -9132,19 +8598,19 @@ pub unsafe fn _mm512_mask3_fnmadd_round_ps( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_fnmadd_pd&expand=2711) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vfmadd, rounding = 8))] //vfnmadd132pd or vfnmadd213pd or vfnmadd231pd -#[rustc_args_required_const(3)] -pub unsafe fn _mm512_fnmadd_round_pd(a: __m512d, b: __m512d, c: __m512d, rounding: i32) -> __m512d { +#[cfg_attr(test, assert_instr(vfmadd, ROUNDING = 8))] //vfnmadd132pd or vfnmadd213pd or vfnmadd231pd +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_fnmadd_round_pd( + a: __m512d, + b: __m512d, + c: __m512d, +) -> __m512d { + static_assert_rounding!(ROUNDING); let zero: f64x8 = mem::zeroed(); let sub = simd_sub(zero, a.as_f64x8()); let b = b.as_f64x8(); let c = c.as_f64x8(); - macro_rules! call { - ($imm4:expr) => { - vfmadd132pdround(sub, b, c, $imm4) - }; - } - let r = constify_imm4_round!(rounding, call); + let r = vfmadd132pdround(sub, b, c, ROUNDING); transmute(r) } @@ -9160,27 +8626,22 @@ pub unsafe fn _mm512_fnmadd_round_pd(a: __m512d, b: __m512d, c: __m512d, roundin /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_fnmadd_round_pd&expand=2728) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vfmadd, rounding = 8))] //vfnmadd132pd or vfnmadd213pd or vfnmadd231pd -#[rustc_args_required_const(4)] -pub unsafe fn _mm512_mask_fnmadd_round_pd( +#[cfg_attr(test, assert_instr(vfmadd, ROUNDING = 8))] //vfnmadd132pd or vfnmadd213pd or vfnmadd231pd +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_mask_fnmadd_round_pd( a: __m512d, k: __mmask8, b: __m512d, c: __m512d, - rounding: i32, ) -> __m512d { + static_assert_rounding!(ROUNDING); let zero: f64x8 = mem::zeroed(); let a = a.as_f64x8(); let sub = simd_sub(zero, a); let b = b.as_f64x8(); let c = c.as_f64x8(); - macro_rules! call { - ($imm4:expr) => { - vfmadd132pdround(sub, b, c, $imm4) - }; - } - let fnmadd = constify_imm4_round!(rounding, call); - transmute(simd_select_bitmask(k, fnmadd, a)) + let r = vfmadd132pdround(sub, b, c, ROUNDING); + transmute(simd_select_bitmask(k, r, a)) } /// Multiply packed double-precision (64-bit) floating-point elements in a and b, add the negated intermediate result to packed elements in c, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set).\ @@ -9195,26 +8656,21 @@ pub unsafe fn _mm512_mask_fnmadd_round_pd( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_fnmadd_round_pd&expand=2730) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vfmadd, rounding = 8))] //vfnmadd132pd or vfnmadd213pd or vfnmadd231pd -#[rustc_args_required_const(4)] -pub unsafe fn _mm512_maskz_fnmadd_round_pd( +#[cfg_attr(test, assert_instr(vfmadd, ROUNDING = 8))] //vfnmadd132pd or vfnmadd213pd or vfnmadd231pd +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_maskz_fnmadd_round_pd( k: __mmask8, a: __m512d, b: __m512d, c: __m512d, - rounding: i32, ) -> __m512d { + static_assert_rounding!(ROUNDING); let zero: f64x8 = mem::zeroed(); let sub = simd_sub(zero, a.as_f64x8()); let b = b.as_f64x8(); let c = c.as_f64x8(); - macro_rules! call { - ($imm4:expr) => { - vfmadd132pdround(sub, b, c, $imm4) - }; - } - let fnmadd = constify_imm4_round!(rounding, call); - transmute(simd_select_bitmask(k, fnmadd, zero)) + let r = vfmadd132pdround(sub, b, c, ROUNDING); + transmute(simd_select_bitmask(k, r, zero)) } /// Multiply packed double-precision (64-bit) floating-point elements in a and b, add the negated intermediate result to packed elements in c, and store the results in dst using writemask k (elements are copied from c when the corresponding mask bit is not set).\ @@ -9229,26 +8685,21 @@ pub unsafe fn _mm512_maskz_fnmadd_round_pd( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask3_fnmadd_round_pd&expand=2729) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vfmadd, rounding = 8))] //vfnmadd132pd or vfnmadd213pd or vfnmadd231pd -#[rustc_args_required_const(4)] -pub unsafe fn _mm512_mask3_fnmadd_round_pd( +#[cfg_attr(test, assert_instr(vfmadd, ROUNDING = 8))] //vfnmadd132pd or vfnmadd213pd or vfnmadd231pd +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_mask3_fnmadd_round_pd( a: __m512d, b: __m512d, c: __m512d, k: __mmask8, - rounding: i32, ) -> __m512d { + static_assert_rounding!(ROUNDING); let zero: f64x8 = mem::zeroed(); let sub = simd_sub(zero, a.as_f64x8()); let b = b.as_f64x8(); let c = c.as_f64x8(); - macro_rules! call { - ($imm4:expr) => { - vfmadd132pdround(sub, b, c, $imm4) - }; - } - let fnmadd = constify_imm4_round!(rounding, call); - transmute(simd_select_bitmask(k, fnmadd, c)) + let r = vfmadd132pdround(sub, b, c, ROUNDING); + transmute(simd_select_bitmask(k, r, c)) } /// Multiply packed single-precision (32-bit) floating-point elements in a and b, subtract packed elements in c from the negated intermediate result, and store the results in dst.\ @@ -9263,19 +8714,19 @@ pub unsafe fn _mm512_mask3_fnmadd_round_pd( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_fnmsub_round_ps&expand=2779) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vfmadd, rounding = 8))] //vfnmsub132ps or vfnmsub213ps or vfnmsub231ps -#[rustc_args_required_const(3)] -pub unsafe fn _mm512_fnmsub_round_ps(a: __m512, b: __m512, c: __m512, rounding: i32) -> __m512 { +#[cfg_attr(test, assert_instr(vfmadd, ROUNDING = 8))] //vfnmsub132ps or vfnmsub213ps or vfnmsub231ps +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_fnmsub_round_ps( + a: __m512, + b: __m512, + c: __m512, +) -> __m512 { + static_assert_rounding!(ROUNDING); let zero: f32x16 = mem::zeroed(); let suba = simd_sub(zero, a.as_f32x16()); let subc = simd_sub(zero, c.as_f32x16()); let b = b.as_f32x16(); - macro_rules! call { - ($imm4:expr) => { - vfmadd132psround(suba, b, subc, $imm4) - }; - } - let r = constify_imm4_round!(rounding, call); + let r = vfmadd132psround(suba, b, subc, ROUNDING); transmute(r) } @@ -9291,27 +8742,22 @@ pub unsafe fn _mm512_fnmsub_round_ps(a: __m512, b: __m512, c: __m512, rounding: /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_fnmsub_round_ps&expand=2780) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vfmadd, rounding = 8))] //vfnmsub132ps or vfnmsub213ps or vfnmsub231ps -#[rustc_args_required_const(4)] -pub unsafe fn _mm512_mask_fnmsub_round_ps( +#[cfg_attr(test, assert_instr(vfmadd, ROUNDING = 8))] //vfnmsub132ps or vfnmsub213ps or vfnmsub231ps +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_mask_fnmsub_round_ps( a: __m512, k: __mmask16, b: __m512, c: __m512, - rounding: i32, ) -> __m512 { + static_assert_rounding!(ROUNDING); let zero: f32x16 = mem::zeroed(); let a = a.as_f32x16(); let suba = simd_sub(zero, a); let subc = simd_sub(zero, c.as_f32x16()); let b = b.as_f32x16(); - macro_rules! call { - ($imm4:expr) => { - vfmadd132psround(suba, b, subc, $imm4) - }; - } - let fnmsub = constify_imm4_round!(rounding, call); - transmute(simd_select_bitmask(k, fnmsub, a)) + let r = vfmadd132psround(suba, b, subc, ROUNDING); + transmute(simd_select_bitmask(k, r, a)) } /// Multiply packed single-precision (32-bit) floating-point elements in a and b, subtract packed elements in c from the negated intermediate result, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set).\ @@ -9326,26 +8772,21 @@ pub unsafe fn _mm512_mask_fnmsub_round_ps( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_fnmsub_round_ps&expand=2782) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vfmadd, rounding = 8))] //vfnmsub132ps or vfnmsub213ps or vfnmsub231ps -#[rustc_args_required_const(4)] -pub unsafe fn _mm512_maskz_fnmsub_round_ps( +#[cfg_attr(test, assert_instr(vfmadd, ROUNDING = 8))] //vfnmsub132ps or vfnmsub213ps or vfnmsub231ps +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_maskz_fnmsub_round_ps( k: __mmask16, a: __m512, b: __m512, c: __m512, - rounding: i32, ) -> __m512 { + static_assert_rounding!(ROUNDING); let zero: f32x16 = mem::zeroed(); let suba = simd_sub(zero, a.as_f32x16()); let subc = simd_sub(zero, c.as_f32x16()); let b = b.as_f32x16(); - macro_rules! call { - ($imm4:expr) => { - vfmadd132psround(suba, b, subc, $imm4) - }; - } - let fnmsub = constify_imm4_round!(rounding, call); - transmute(simd_select_bitmask(k, fnmsub, zero)) + let r = vfmadd132psround(suba, b, subc, ROUNDING); + transmute(simd_select_bitmask(k, r, zero)) } /// Multiply packed single-precision (32-bit) floating-point elements in a and b, subtract packed elements in c from the negated intermediate result, and store the results in dst using writemask k (elements are copied from c when the corresponding mask bit is not set).\ @@ -9360,27 +8801,22 @@ pub unsafe fn _mm512_maskz_fnmsub_round_ps( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask3_fnmsub_round_ps&expand=2781) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vfmadd, rounding = 8))] //vfnmsub132ps or vfnmsub213ps or vfnmsub231ps -#[rustc_args_required_const(4)] -pub unsafe fn _mm512_mask3_fnmsub_round_ps( +#[cfg_attr(test, assert_instr(vfmadd, ROUNDING = 8))] //vfnmsub132ps or vfnmsub213ps or vfnmsub231ps +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_mask3_fnmsub_round_ps( a: __m512, b: __m512, c: __m512, k: __mmask16, - rounding: i32, ) -> __m512 { + static_assert_rounding!(ROUNDING); let zero: f32x16 = mem::zeroed(); let suba = simd_sub(zero, a.as_f32x16()); let c = c.as_f32x16(); let subc = simd_sub(zero, c); let b = b.as_f32x16(); - macro_rules! call { - ($imm4:expr) => { - vfmadd132psround(suba, b, subc, $imm4) - }; - } - let fnmsub = constify_imm4_round!(rounding, call); - transmute(simd_select_bitmask(k, fnmsub, c)) + let r = vfmadd132psround(suba, b, subc, ROUNDING); + transmute(simd_select_bitmask(k, r, c)) } /// Multiply packed double-precision (64-bit) floating-point elements in a and b, subtract packed elements in c from the negated intermediate result, and store the results in dst.\ @@ -9395,19 +8831,19 @@ pub unsafe fn _mm512_mask3_fnmsub_round_ps( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_fnmsub_round_pd&expand=2775) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vfmadd, rounding = 8))] //vfnmsub132pd or vfnmsub213pd or vfnmsub231pd -#[rustc_args_required_const(3)] -pub unsafe fn _mm512_fnmsub_round_pd(a: __m512d, b: __m512d, c: __m512d, rounding: i32) -> __m512d { +#[cfg_attr(test, assert_instr(vfmadd, ROUNDING = 8))] //vfnmsub132pd or vfnmsub213pd or vfnmsub231pd +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_fnmsub_round_pd( + a: __m512d, + b: __m512d, + c: __m512d, +) -> __m512d { + static_assert_rounding!(ROUNDING); let zero: f64x8 = mem::zeroed(); let suba = simd_sub(zero, a.as_f64x8()); let subc = simd_sub(zero, c.as_f64x8()); let b = b.as_f64x8(); - macro_rules! call { - ($imm4:expr) => { - vfmadd132pdround(suba, b, subc, $imm4) - }; - } - let r = constify_imm4_round!(rounding, call); + let r = vfmadd132pdround(suba, b, subc, ROUNDING); transmute(r) } @@ -9423,27 +8859,22 @@ pub unsafe fn _mm512_fnmsub_round_pd(a: __m512d, b: __m512d, c: __m512d, roundin /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_fnmsub_round_pd&expand=2776) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vfmadd, rounding = 8))] //vfnmsub132pd or vfnmsub213pd or vfnmsub231pd -#[rustc_args_required_const(4)] -pub unsafe fn _mm512_mask_fnmsub_round_pd( +#[cfg_attr(test, assert_instr(vfmadd, ROUNDING = 8))] //vfnmsub132pd or vfnmsub213pd or vfnmsub231pd +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_mask_fnmsub_round_pd( a: __m512d, k: __mmask8, b: __m512d, c: __m512d, - rounding: i32, ) -> __m512d { + static_assert_rounding!(ROUNDING); let zero: f64x8 = mem::zeroed(); let a = a.as_f64x8(); let suba = simd_sub(zero, a); let subc = simd_sub(zero, c.as_f64x8()); let b = b.as_f64x8(); - macro_rules! call { - ($imm4:expr) => { - vfmadd132pdround(suba, b, subc, $imm4) - }; - } - let fnmsub = constify_imm4_round!(rounding, call); - transmute(simd_select_bitmask(k, fnmsub, a)) + let r = vfmadd132pdround(suba, b, subc, ROUNDING); + transmute(simd_select_bitmask(k, r, a)) } /// Multiply packed double-precision (64-bit) floating-point elements in a and b, subtract packed elements in c from the negated intermediate result, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set).\ @@ -9458,26 +8889,21 @@ pub unsafe fn _mm512_mask_fnmsub_round_pd( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_fnmsub_round_pd&expand=2778) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vfmadd, rounding = 8))] //vfnmsub132pd or vfnmsub213pd or vfnmsub231pd -#[rustc_args_required_const(4)] -pub unsafe fn _mm512_maskz_fnmsub_round_pd( +#[cfg_attr(test, assert_instr(vfmadd, ROUNDING = 8))] //vfnmsub132pd or vfnmsub213pd or vfnmsub231pd +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_maskz_fnmsub_round_pd( k: __mmask8, a: __m512d, b: __m512d, c: __m512d, - rounding: i32, ) -> __m512d { + static_assert_rounding!(ROUNDING); let zero: f64x8 = mem::zeroed(); let suba = simd_sub(zero, a.as_f64x8()); let subc = simd_sub(zero, c.as_f64x8()); let b = b.as_f64x8(); - macro_rules! call { - ($imm4:expr) => { - vfmadd132pdround(suba, b, subc, $imm4) - }; - } - let fnmsub = constify_imm4_round!(rounding, call); - transmute(simd_select_bitmask(k, fnmsub, zero)) + let r = vfmadd132pdround(suba, b, subc, ROUNDING); + transmute(simd_select_bitmask(k, r, zero)) } /// Multiply packed double-precision (64-bit) floating-point elements in a and b, subtract packed elements in c from the negated intermediate result, and store the results in dst using writemask k (elements are copied from c when the corresponding mask bit is not set).\ @@ -9492,27 +8918,22 @@ pub unsafe fn _mm512_maskz_fnmsub_round_pd( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask3_fnmsub_round_pd&expand=2777) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vfmadd, rounding = 8))] //vfnmsub132pd or vfnmsub213pd or vfnmsub231pd -#[rustc_args_required_const(4)] -pub unsafe fn _mm512_mask3_fnmsub_round_pd( +#[cfg_attr(test, assert_instr(vfmadd, ROUNDING = 8))] //vfnmsub132pd or vfnmsub213pd or vfnmsub231pd +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_mask3_fnmsub_round_pd( a: __m512d, b: __m512d, c: __m512d, k: __mmask8, - rounding: i32, ) -> __m512d { + static_assert_rounding!(ROUNDING); let zero: f64x8 = mem::zeroed(); let suba = simd_sub(zero, a.as_f64x8()); let c = c.as_f64x8(); let subc = simd_sub(zero, c); let b = b.as_f64x8(); - macro_rules! call { - ($imm4:expr) => { - vfmadd132pdround(suba, b, subc, $imm4) - }; - } - let fnmsub = constify_imm4_round!(rounding, call); - transmute(simd_select_bitmask(k, fnmsub, c)) + let r = vfmadd132pdround(suba, b, subc, ROUNDING); + transmute(simd_select_bitmask(k, r, c)) } /// Compare packed single-precision (32-bit) floating-point elements in a and b, and store packed maximum values in dst.\ @@ -9521,17 +8942,13 @@ pub unsafe fn _mm512_mask3_fnmsub_round_pd( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_max_round_ps&expand=3662) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vmaxps, sae = 8))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm512_max_round_ps(a: __m512, b: __m512, sae: i32) -> __m512 { +#[cfg_attr(test, assert_instr(vmaxps, SAE = 8))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_max_round_ps(a: __m512, b: __m512) -> __m512 { + static_assert_sae!(SAE); let a = a.as_f32x16(); let b = b.as_f32x16(); - macro_rules! call { - ($imm4:expr) => { - vmaxps(a, b, $imm4) - }; - } - let r = constify_imm4_sae!(sae, call); + let r = vmaxps(a, b, SAE); transmute(r) } @@ -9541,24 +8958,19 @@ pub unsafe fn _mm512_max_round_ps(a: __m512, b: __m512, sae: i32) -> __m512 { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_max_round_ps&expand=3660) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vmaxps, sae = 8))] -#[rustc_args_required_const(4)] -pub unsafe fn _mm512_mask_max_round_ps( +#[cfg_attr(test, assert_instr(vmaxps, SAE = 8))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_mask_max_round_ps( src: __m512, k: __mmask16, a: __m512, b: __m512, - sae: i32, ) -> __m512 { + static_assert_sae!(SAE); let a = a.as_f32x16(); let b = b.as_f32x16(); - macro_rules! call { - ($imm4:expr) => { - vmaxps(a, b, $imm4) - }; - } - let max = constify_imm4_sae!(sae, call); - transmute(simd_select_bitmask(k, max, src.as_f32x16())) + let r = vmaxps(a, b, SAE); + transmute(simd_select_bitmask(k, r, src.as_f32x16())) } /// Compare packed single-precision (32-bit) floating-point elements in a and b, and store packed maximum values in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set).\ @@ -9567,19 +8979,19 @@ pub unsafe fn _mm512_mask_max_round_ps( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_max_round_ps&expand=3661) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vmaxps, sae = 8))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm512_maskz_max_round_ps(k: __mmask16, a: __m512, b: __m512, sae: i32) -> __m512 { +#[cfg_attr(test, assert_instr(vmaxps, SAE = 8))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_maskz_max_round_ps( + k: __mmask16, + a: __m512, + b: __m512, +) -> __m512 { + static_assert_sae!(SAE); let a = a.as_f32x16(); let b = b.as_f32x16(); - macro_rules! call { - ($imm4:expr) => { - vmaxps(a, b, $imm4) - }; - } - let max = constify_imm4_sae!(sae, call); + let r = vmaxps(a, b, SAE); let zero = _mm512_setzero_ps().as_f32x16(); - transmute(simd_select_bitmask(k, max, zero)) + transmute(simd_select_bitmask(k, r, zero)) } /// Compare packed double-precision (64-bit) floating-point elements in a and b, and store packed maximum values in dst.\ @@ -9588,17 +9000,13 @@ pub unsafe fn _mm512_maskz_max_round_ps(k: __mmask16, a: __m512, b: __m512, sae: /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_max_round_pd&expand=3659) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vmaxpd, sae = 8))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm512_max_round_pd(a: __m512d, b: __m512d, sae: i32) -> __m512d { +#[cfg_attr(test, assert_instr(vmaxpd, SAE = 8))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_max_round_pd(a: __m512d, b: __m512d) -> __m512d { + static_assert_sae!(SAE); let a = a.as_f64x8(); let b = b.as_f64x8(); - macro_rules! call { - ($imm4:expr) => { - vmaxpd(a, b, $imm4) - }; - } - let r = constify_imm4_sae!(sae, call); + let r = vmaxpd(a, b, SAE); transmute(r) } @@ -9608,24 +9016,19 @@ pub unsafe fn _mm512_max_round_pd(a: __m512d, b: __m512d, sae: i32) -> __m512d { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_max_round_pd&expand=3657) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vmaxpd, sae = 8))] -#[rustc_args_required_const(4)] -pub unsafe fn _mm512_mask_max_round_pd( +#[cfg_attr(test, assert_instr(vmaxpd, SAE = 8))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_mask_max_round_pd( src: __m512d, k: __mmask8, a: __m512d, b: __m512d, - sae: i32, ) -> __m512d { + static_assert_sae!(SAE); let a = a.as_f64x8(); let b = b.as_f64x8(); - macro_rules! call { - ($imm4:expr) => { - vmaxpd(a, b, $imm4) - }; - } - let max = constify_imm4_sae!(sae, call); - transmute(simd_select_bitmask(k, max, src.as_f64x8())) + let r = vmaxpd(a, b, SAE); + transmute(simd_select_bitmask(k, r, src.as_f64x8())) } /// Compare packed double-precision (64-bit) floating-point elements in a and b, and store packed maximum values in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set).\ @@ -9634,19 +9037,19 @@ pub unsafe fn _mm512_mask_max_round_pd( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_max_round_pd&expand=3658) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vmaxpd, sae = 8))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm512_maskz_max_round_pd(k: __mmask8, a: __m512d, b: __m512d, sae: i32) -> __m512d { +#[cfg_attr(test, assert_instr(vmaxpd, SAE = 8))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_maskz_max_round_pd( + k: __mmask8, + a: __m512d, + b: __m512d, +) -> __m512d { + static_assert_sae!(SAE); let a = a.as_f64x8(); let b = b.as_f64x8(); - macro_rules! call { - ($imm4:expr) => { - vmaxpd(a, b, $imm4) - }; - } - let max = constify_imm4_sae!(sae, call); + let r = vmaxpd(a, b, SAE); let zero = _mm512_setzero_pd().as_f64x8(); - transmute(simd_select_bitmask(k, max, zero)) + transmute(simd_select_bitmask(k, r, zero)) } /// Compare packed single-precision (32-bit) floating-point elements in a and b, and store packed minimum values in dst.\ @@ -9655,17 +9058,13 @@ pub unsafe fn _mm512_maskz_max_round_pd(k: __mmask8, a: __m512d, b: __m512d, sae /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_min_round_ps&expand=3776) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vminps, sae = 8))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm512_min_round_ps(a: __m512, b: __m512, sae: i32) -> __m512 { +#[cfg_attr(test, assert_instr(vminps, SAE = 8))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_min_round_ps(a: __m512, b: __m512) -> __m512 { + static_assert_sae!(SAE); let a = a.as_f32x16(); let b = b.as_f32x16(); - macro_rules! call { - ($imm4:expr) => { - vminps(a, b, $imm4) - }; - } - let r = constify_imm4_sae!(sae, call); + let r = vminps(a, b, SAE); transmute(r) } @@ -9675,24 +9074,19 @@ pub unsafe fn _mm512_min_round_ps(a: __m512, b: __m512, sae: i32) -> __m512 { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_min_round_ps&expand=3774) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vminps, sae = 8))] -#[rustc_args_required_const(4)] -pub unsafe fn _mm512_mask_min_round_ps( +#[cfg_attr(test, assert_instr(vminps, SAE = 8))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_mask_min_round_ps( src: __m512, k: __mmask16, a: __m512, b: __m512, - sae: i32, ) -> __m512 { + static_assert_sae!(SAE); let a = a.as_f32x16(); let b = b.as_f32x16(); - macro_rules! call { - ($imm4:expr) => { - vminps(a, b, $imm4) - }; - } - let max = constify_imm4_sae!(sae, call); - transmute(simd_select_bitmask(k, max, src.as_f32x16())) + let r = vminps(a, b, SAE); + transmute(simd_select_bitmask(k, r, src.as_f32x16())) } /// Compare packed single-precision (32-bit) floating-point elements in a and b, and store packed minimum values in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set).\ @@ -9701,19 +9095,19 @@ pub unsafe fn _mm512_mask_min_round_ps( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_min_round_ps&expand=3775) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vminps, sae = 8))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm512_maskz_min_round_ps(k: __mmask16, a: __m512, b: __m512, sae: i32) -> __m512 { +#[cfg_attr(test, assert_instr(vminps, SAE = 8))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_maskz_min_round_ps( + k: __mmask16, + a: __m512, + b: __m512, +) -> __m512 { + static_assert_sae!(SAE); let a = a.as_f32x16(); let b = b.as_f32x16(); - macro_rules! call { - ($imm4:expr) => { - vminps(a, b, $imm4) - }; - } - let max = constify_imm4_sae!(sae, call); + let r = vminps(a, b, SAE); let zero = _mm512_setzero_ps().as_f32x16(); - transmute(simd_select_bitmask(k, max, zero)) + transmute(simd_select_bitmask(k, r, zero)) } /// Compare packed double-precision (64-bit) floating-point elements in a and b, and store packed minimum values in dst.\ @@ -9722,17 +9116,13 @@ pub unsafe fn _mm512_maskz_min_round_ps(k: __mmask16, a: __m512, b: __m512, sae: /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_min_round_pd&expand=3773) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vminpd, sae = 8))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm512_min_round_pd(a: __m512d, b: __m512d, sae: i32) -> __m512d { +#[cfg_attr(test, assert_instr(vminpd, SAE = 8))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_min_round_pd(a: __m512d, b: __m512d) -> __m512d { + static_assert_sae!(SAE); let a = a.as_f64x8(); let b = b.as_f64x8(); - macro_rules! call { - ($imm4:expr) => { - vminpd(a, b, $imm4) - }; - } - let r = constify_imm4_sae!(sae, call); + let r = vminpd(a, b, SAE); transmute(r) } @@ -9742,24 +9132,19 @@ pub unsafe fn _mm512_min_round_pd(a: __m512d, b: __m512d, sae: i32) -> __m512d { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_min_round_pd&expand=3771) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vminpd, sae = 8))] -#[rustc_args_required_const(4)] -pub unsafe fn _mm512_mask_min_round_pd( +#[cfg_attr(test, assert_instr(vminpd, SAE = 8))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_mask_min_round_pd( src: __m512d, k: __mmask8, a: __m512d, b: __m512d, - sae: i32, ) -> __m512d { + static_assert_sae!(SAE); let a = a.as_f64x8(); let b = b.as_f64x8(); - macro_rules! call { - ($imm4:expr) => { - vminpd(a, b, $imm4) - }; - } - let max = constify_imm4_sae!(sae, call); - transmute(simd_select_bitmask(k, max, src.as_f64x8())) + let r = vminpd(a, b, SAE); + transmute(simd_select_bitmask(k, r, src.as_f64x8())) } /// Compare packed double-precision (64-bit) floating-point elements in a and b, and store packed minimum values in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set).\ @@ -9768,19 +9153,19 @@ pub unsafe fn _mm512_mask_min_round_pd( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_min_round_pd&expand=3772) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vminpd, sae = 8))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm512_maskz_min_round_pd(k: __mmask8, a: __m512d, b: __m512d, sae: i32) -> __m512d { +#[cfg_attr(test, assert_instr(vminpd, SAE = 8))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_maskz_min_round_pd( + k: __mmask8, + a: __m512d, + b: __m512d, +) -> __m512d { + static_assert_sae!(SAE); let a = a.as_f64x8(); let b = b.as_f64x8(); - macro_rules! call { - ($imm4:expr) => { - vminpd(a, b, $imm4) - }; - } - let max = constify_imm4_sae!(sae, call); + let r = vminpd(a, b, SAE); let zero = _mm512_setzero_pd().as_f64x8(); - transmute(simd_select_bitmask(k, max, zero)) + transmute(simd_select_bitmask(k, r, zero)) } /// Convert the exponent of each packed single-precision (32-bit) floating-point element in a to a single-precision (32-bit) floating-point number representing the integer exponent, and store the results in dst. This intrinsic essentially calculates floor(log2(x)) for each element.\ @@ -9789,17 +9174,13 @@ pub unsafe fn _mm512_maskz_min_round_pd(k: __mmask8, a: __m512d, b: __m512d, sae /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_getexp_round_ps&expand=2850) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vgetexpps, sae = 8))] -#[rustc_args_required_const(1)] -pub unsafe fn _mm512_getexp_round_ps(a: __m512, sae: i32) -> __m512 { +#[cfg_attr(test, assert_instr(vgetexpps, SAE = 8))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm512_getexp_round_ps(a: __m512) -> __m512 { + static_assert_sae!(SAE); let a = a.as_f32x16(); let zero = _mm512_setzero_ps().as_f32x16(); - macro_rules! call { - ($imm4:expr) => { - vgetexpps(a, zero, 0b11111111_11111111, $imm4) - }; - } - let r = constify_imm4_sae!(sae, call); + let r = vgetexpps(a, zero, 0b11111111_11111111, SAE); transmute(r) } @@ -9809,22 +9190,17 @@ pub unsafe fn _mm512_getexp_round_ps(a: __m512, sae: i32) -> __m512 { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_getexp_round_ps&expand=2851) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vgetexpps, sae = 8))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm512_mask_getexp_round_ps( +#[cfg_attr(test, assert_instr(vgetexpps, SAE = 8))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_mask_getexp_round_ps( src: __m512, k: __mmask16, a: __m512, - sae: i32, ) -> __m512 { + static_assert_sae!(SAE); let a = a.as_f32x16(); let src = src.as_f32x16(); - macro_rules! call { - ($imm4:expr) => { - vgetexpps(a, src, k, $imm4) - }; - } - let r = constify_imm4_sae!(sae, call); + let r = vgetexpps(a, src, k, SAE); transmute(r) } @@ -9834,17 +9210,13 @@ pub unsafe fn _mm512_mask_getexp_round_ps( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_getexp_round_ps&expand=2852) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vgetexpps, sae = 8))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm512_maskz_getexp_round_ps(k: __mmask16, a: __m512, sae: i32) -> __m512 { +#[cfg_attr(test, assert_instr(vgetexpps, SAE = 8))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_maskz_getexp_round_ps(k: __mmask16, a: __m512) -> __m512 { + static_assert_sae!(SAE); let a = a.as_f32x16(); let zero = _mm512_setzero_ps().as_f32x16(); - macro_rules! call { - ($imm4:expr) => { - vgetexpps(a, zero, k, $imm4) - }; - } - let r = constify_imm4_sae!(sae, call); + let r = vgetexpps(a, zero, k, SAE); transmute(r) } @@ -9854,17 +9226,13 @@ pub unsafe fn _mm512_maskz_getexp_round_ps(k: __mmask16, a: __m512, sae: i32) -> /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_getexp_round_pd&expand=2847) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vgetexppd, sae = 8))] -#[rustc_args_required_const(1)] -pub unsafe fn _mm512_getexp_round_pd(a: __m512d, sae: i32) -> __m512d { +#[cfg_attr(test, assert_instr(vgetexppd, SAE = 8))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm512_getexp_round_pd(a: __m512d) -> __m512d { + static_assert_sae!(SAE); let a = a.as_f64x8(); let zero = _mm512_setzero_pd().as_f64x8(); - macro_rules! call { - ($imm4:expr) => { - vgetexppd(a, zero, 0b11111111, $imm4) - }; - } - let r = constify_imm4_sae!(sae, call); + let r = vgetexppd(a, zero, 0b11111111, SAE); transmute(r) } @@ -9874,22 +9242,17 @@ pub unsafe fn _mm512_getexp_round_pd(a: __m512d, sae: i32) -> __m512d { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_getexp_round_pd&expand=2848) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vgetexppd, sae = 8))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm512_mask_getexp_round_pd( +#[cfg_attr(test, assert_instr(vgetexppd, SAE = 8))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_mask_getexp_round_pd( src: __m512d, k: __mmask8, a: __m512d, - sae: i32, ) -> __m512d { + static_assert_sae!(SAE); let a = a.as_f64x8(); let src = src.as_f64x8(); - macro_rules! call { - ($imm4:expr) => { - vgetexppd(a, src, k, $imm4) - }; - } - let r = constify_imm4_sae!(sae, call); + let r = vgetexppd(a, src, k, SAE); transmute(r) } @@ -9899,17 +9262,13 @@ pub unsafe fn _mm512_mask_getexp_round_pd( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_getexp_round_pd&expand=2849) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vgetexppd, sae = 8))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm512_maskz_getexp_round_pd(k: __mmask8, a: __m512d, sae: i32) -> __m512d { +#[cfg_attr(test, assert_instr(vgetexppd, SAE = 8))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_maskz_getexp_round_pd(k: __mmask8, a: __m512d) -> __m512d { + static_assert_sae!(SAE); let a = a.as_f64x8(); let zero = _mm512_setzero_pd().as_f64x8(); - macro_rules! call { - ($imm4:expr) => { - vgetexppd(a, zero, k, $imm4) - }; - } - let r = constify_imm4_sae!(sae, call); + let r = vgetexppd(a, zero, k, SAE); transmute(r) } @@ -9925,17 +9284,14 @@ pub unsafe fn _mm512_maskz_getexp_round_pd(k: __mmask8, a: __m512d, sae: i32) -> /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_roundscale_round_ps&expand=4790) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vrndscaleps, imm8 = 0, sae = 8))] -#[rustc_args_required_const(1, 2)] -pub unsafe fn _mm512_roundscale_round_ps(a: __m512, imm8: i32, sae: i32) -> __m512 { +#[cfg_attr(test, assert_instr(vrndscaleps, IMM8 = 0, SAE = 8))] +#[rustc_legacy_const_generics(1, 2)] +pub unsafe fn _mm512_roundscale_round_ps(a: __m512) -> __m512 { + static_assert_imm8!(IMM8); + static_assert_mantissas_sae!(SAE); let a = a.as_f32x16(); let zero = _mm512_setzero_ps().as_f32x16(); - macro_rules! call { - ($imm8:expr, $imm4:expr) => { - vrndscaleps(a, $imm8, zero, 0b11111111_11111111, $imm4) - }; - } - let r = constify_imm8_roundscale!(imm8, sae, call); + let r = vrndscaleps(a, IMM8, zero, 0b11111111_11111111, SAE); transmute(r) } @@ -9951,23 +9307,18 @@ pub unsafe fn _mm512_roundscale_round_ps(a: __m512, imm8: i32, sae: i32) -> __m5 /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_roundscale_round_ps&expand=4788) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vrndscaleps, imm8 = 0, sae = 8))] -#[rustc_args_required_const(3, 4)] -pub unsafe fn _mm512_mask_roundscale_round_ps( +#[cfg_attr(test, assert_instr(vrndscaleps, IMM8 = 0, SAE = 8))] +#[rustc_legacy_const_generics(3, 4)] +pub unsafe fn _mm512_mask_roundscale_round_ps( src: __m512, k: __mmask16, a: __m512, - imm8: i32, - sae: i32, ) -> __m512 { + static_assert_imm8!(IMM8); + static_assert_mantissas_sae!(SAE); let a = a.as_f32x16(); let src = src.as_f32x16(); - macro_rules! call { - ($imm8:expr, $imm4:expr) => { - vrndscaleps(a, $imm8, src, k, $imm4) - }; - } - let r = constify_imm8_roundscale!(imm8, sae, call); + let r = vrndscaleps(a, IMM8, src, k, SAE); transmute(r) } @@ -9983,22 +9334,17 @@ pub unsafe fn _mm512_mask_roundscale_round_ps( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_roundscale_round_ps&expand=4789) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vrndscaleps, imm8 = 0, sae = 8))] -#[rustc_args_required_const(2, 3)] -pub unsafe fn _mm512_maskz_roundscale_round_ps( +#[cfg_attr(test, assert_instr(vrndscaleps, IMM8 = 0, SAE = 8))] +#[rustc_legacy_const_generics(2, 3)] +pub unsafe fn _mm512_maskz_roundscale_round_ps( k: __mmask16, a: __m512, - imm8: i32, - sae: i32, ) -> __m512 { + static_assert_imm8!(IMM8); + static_assert_mantissas_sae!(SAE); let a = a.as_f32x16(); let zero = _mm512_setzero_ps().as_f32x16(); - macro_rules! call { - ($imm8:expr, $imm4:expr) => { - vrndscaleps(a, $imm8, zero, k, $imm4) - }; - } - let r = constify_imm8_roundscale!(imm8, sae, call); + let r = vrndscaleps(a, IMM8, zero, k, SAE); transmute(r) } @@ -10014,17 +9360,14 @@ pub unsafe fn _mm512_maskz_roundscale_round_ps( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_roundscale_round_pd&expand=4787) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vrndscalepd, imm8 = 0, sae = 8))] -#[rustc_args_required_const(1, 2)] -pub unsafe fn _mm512_roundscale_round_pd(a: __m512d, imm8: i32, sae: i32) -> __m512d { +#[cfg_attr(test, assert_instr(vrndscalepd, IMM8 = 0, SAE = 8))] +#[rustc_legacy_const_generics(1, 2)] +pub unsafe fn _mm512_roundscale_round_pd(a: __m512d) -> __m512d { + static_assert_imm8!(IMM8); + static_assert_mantissas_sae!(SAE); let a = a.as_f64x8(); let zero = _mm512_setzero_pd().as_f64x8(); - macro_rules! call { - ($imm8:expr, $imm4:expr) => { - vrndscalepd(a, $imm8, zero, 0b11111111, $imm4) - }; - } - let r = constify_imm8_roundscale!(imm8, sae, call); + let r = vrndscalepd(a, IMM8, zero, 0b11111111, SAE); transmute(r) } @@ -10040,23 +9383,18 @@ pub unsafe fn _mm512_roundscale_round_pd(a: __m512d, imm8: i32, sae: i32) -> __m /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_roundscale_round_pd&expand=4785) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vrndscalepd, imm8 = 0, sae = 8))] -#[rustc_args_required_const(3, 4)] -pub unsafe fn _mm512_mask_roundscale_round_pd( +#[cfg_attr(test, assert_instr(vrndscalepd, IMM8 = 0, SAE = 8))] +#[rustc_legacy_const_generics(3, 4)] +pub unsafe fn _mm512_mask_roundscale_round_pd( src: __m512d, k: __mmask8, a: __m512d, - imm8: i32, - sae: i32, ) -> __m512d { + static_assert_imm8!(IMM8); + static_assert_mantissas_sae!(SAE); let a = a.as_f64x8(); let src = src.as_f64x8(); - macro_rules! call { - ($imm8:expr, $imm4:expr) => { - vrndscalepd(a, $imm8, src, k, $imm4) - }; - } - let r = constify_imm8_roundscale!(imm8, sae, call); + let r = vrndscalepd(a, IMM8, src, k, SAE); transmute(r) } @@ -10072,22 +9410,17 @@ pub unsafe fn _mm512_mask_roundscale_round_pd( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_roundscale_round_pd&expand=4786) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vrndscalepd, imm8 = 0, sae = 8))] -#[rustc_args_required_const(2, 3)] -pub unsafe fn _mm512_maskz_roundscale_round_pd( +#[cfg_attr(test, assert_instr(vrndscalepd, IMM8 = 0, SAE = 8))] +#[rustc_legacy_const_generics(2, 3)] +pub unsafe fn _mm512_maskz_roundscale_round_pd( k: __mmask8, a: __m512d, - imm8: i32, - sae: i32, ) -> __m512d { + static_assert_imm8!(IMM8); + static_assert_mantissas_sae!(SAE); let a = a.as_f64x8(); let zero = _mm512_setzero_pd().as_f64x8(); - macro_rules! call { - ($imm8:expr, $imm4:expr) => { - vrndscalepd(a, $imm8, zero, k, $imm4) - }; - } - let r = constify_imm8_roundscale!(imm8, sae, call); + let r = vrndscalepd(a, IMM8, zero, k, SAE); transmute(r) } @@ -10103,18 +9436,14 @@ pub unsafe fn _mm512_maskz_roundscale_round_pd( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_scalef_round_ps&expand=4889) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vscalefps, rounding = 8))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm512_scalef_round_ps(a: __m512, b: __m512, rounding: i32) -> __m512 { +#[cfg_attr(test, assert_instr(vscalefps, ROUNDING = 8))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_scalef_round_ps(a: __m512, b: __m512) -> __m512 { + static_assert_rounding!(ROUNDING); let a = a.as_f32x16(); let b = b.as_f32x16(); let zero = _mm512_setzero_ps().as_f32x16(); - macro_rules! call { - ($imm4:expr) => { - vscalefps(a, b, zero, 0b11111111_11111111, $imm4) - }; - } - let r = constify_imm4_round!(rounding, call); + let r = vscalefps(a, b, zero, 0b11111111_11111111, ROUNDING); transmute(r) } @@ -10130,24 +9459,19 @@ pub unsafe fn _mm512_scalef_round_ps(a: __m512, b: __m512, rounding: i32) -> __m /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_scalef_round_ps&expand=4887) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vscalefps, rounding = 8))] -#[rustc_args_required_const(4)] -pub unsafe fn _mm512_mask_scalef_round_ps( +#[cfg_attr(test, assert_instr(vscalefps, ROUNDING = 8))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_mask_scalef_round_ps( src: __m512, k: __mmask16, a: __m512, b: __m512, - rounding: i32, ) -> __m512 { + static_assert_rounding!(ROUNDING); let a = a.as_f32x16(); let b = b.as_f32x16(); let src = src.as_f32x16(); - macro_rules! call { - ($imm4:expr) => { - vscalefps(a, b, src, k, $imm4) - }; - } - let r = constify_imm4_round!(rounding, call); + let r = vscalefps(a, b, src, k, ROUNDING); transmute(r) } @@ -10163,23 +9487,18 @@ pub unsafe fn _mm512_mask_scalef_round_ps( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_scalef_round_ps&expand=4888) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vscalefps, rounding = 8))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm512_maskz_scalef_round_ps( +#[cfg_attr(test, assert_instr(vscalefps, ROUNDING = 8))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_maskz_scalef_round_ps( k: __mmask16, a: __m512, b: __m512, - rounding: i32, ) -> __m512 { + static_assert_rounding!(ROUNDING); let a = a.as_f32x16(); let b = b.as_f32x16(); let zero = _mm512_setzero_ps().as_f32x16(); - macro_rules! call { - ($imm4:expr) => { - vscalefps(a, b, zero, k, $imm4) - }; - } - let r = constify_imm4_round!(rounding, call); + let r = vscalefps(a, b, zero, k, ROUNDING); transmute(r) } @@ -10195,18 +9514,14 @@ pub unsafe fn _mm512_maskz_scalef_round_ps( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_scalef_round_pd&expand=4886) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vscalefpd, rounding = 8))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm512_scalef_round_pd(a: __m512d, b: __m512d, rounding: i32) -> __m512d { +#[cfg_attr(test, assert_instr(vscalefpd, ROUNDING = 8))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_scalef_round_pd(a: __m512d, b: __m512d) -> __m512d { + static_assert_rounding!(ROUNDING); let a = a.as_f64x8(); let b = b.as_f64x8(); let zero = _mm512_setzero_pd().as_f64x8(); - macro_rules! call { - ($imm4:expr) => { - vscalefpd(a, b, zero, 0b11111111, $imm4) - }; - } - let r = constify_imm4_round!(rounding, call); + let r = vscalefpd(a, b, zero, 0b11111111, ROUNDING); transmute(r) } @@ -10222,24 +9537,19 @@ pub unsafe fn _mm512_scalef_round_pd(a: __m512d, b: __m512d, rounding: i32) -> _ /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_scalef_round_pd&expand=4884) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vscalefpd, rounding = 8))] -#[rustc_args_required_const(4)] -pub unsafe fn _mm512_mask_scalef_round_pd( +#[cfg_attr(test, assert_instr(vscalefpd, ROUNDING = 8))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_mask_scalef_round_pd( src: __m512d, k: __mmask8, a: __m512d, b: __m512d, - rounding: i32, ) -> __m512d { + static_assert_rounding!(ROUNDING); let a = a.as_f64x8(); let b = b.as_f64x8(); let src = src.as_f64x8(); - macro_rules! call { - ($imm4:expr) => { - vscalefpd(a, b, src, k, $imm4) - }; - } - let r = constify_imm4_round!(rounding, call); + let r = vscalefpd(a, b, src, k, ROUNDING); transmute(r) } @@ -10255,23 +9565,18 @@ pub unsafe fn _mm512_mask_scalef_round_pd( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_scalef_round_pd&expand=4885) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vscalefpd, rounding = 8))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm512_maskz_scalef_round_pd( +#[cfg_attr(test, assert_instr(vscalefpd, ROUNDING = 8))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_maskz_scalef_round_pd( k: __mmask8, a: __m512d, b: __m512d, - rounding: i32, ) -> __m512d { + static_assert_rounding!(ROUNDING); let a = a.as_f64x8(); let b = b.as_f64x8(); let zero = _mm512_setzero_pd().as_f64x8(); - macro_rules! call { - ($imm4:expr) => { - vscalefpd(a, b, zero, k, $imm4) - }; - } - let r = constify_imm4_round!(rounding, call); + let r = vscalefpd(a, b, zero, k, ROUNDING); transmute(r) } @@ -10281,24 +9586,19 @@ pub unsafe fn _mm512_maskz_scalef_round_pd( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_fixupimm_round_ps&expand=2505) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vfixupimmps, imm8 = 0, sae = 8))] -#[rustc_args_required_const(3, 4)] -pub unsafe fn _mm512_fixupimm_round_ps( +#[cfg_attr(test, assert_instr(vfixupimmps, IMM8 = 0, SAE = 8))] +#[rustc_legacy_const_generics(3, 4)] +pub unsafe fn _mm512_fixupimm_round_ps( a: __m512, b: __m512, c: __m512i, - imm8: i32, - sae: i32, ) -> __m512 { + static_assert_imm8!(IMM8); + static_assert_mantissas_sae!(SAE); let a = a.as_f32x16(); let b = b.as_f32x16(); let c = c.as_i32x16(); - macro_rules! call { - ($imm8:expr, $imm4:expr) => { - vfixupimmps(a, b, c, $imm8, 0b11111111_11111111, $imm4) - }; - } - let r = constify_imm8_roundscale!(imm8, sae, call); + let r = vfixupimmps(a, b, c, IMM8, 0b11111111_11111111, SAE); transmute(r) } @@ -10308,25 +9608,20 @@ pub unsafe fn _mm512_fixupimm_round_ps( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_fixupimm_round_ps&expand=2506) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vfixupimmps, imm8 = 0, sae = 8))] -#[rustc_args_required_const(4, 5)] -pub unsafe fn _mm512_mask_fixupimm_round_ps( +#[cfg_attr(test, assert_instr(vfixupimmps, IMM8 = 0, SAE = 8))] +#[rustc_legacy_const_generics(4, 5)] +pub unsafe fn _mm512_mask_fixupimm_round_ps( a: __m512, k: __mmask16, b: __m512, c: __m512i, - imm8: i32, - sae: i32, ) -> __m512 { + static_assert_imm8!(IMM8); + static_assert_mantissas_sae!(SAE); let a = a.as_f32x16(); let b = b.as_f32x16(); let c = c.as_i32x16(); - macro_rules! call { - ($imm8:expr, $imm4:expr) => { - vfixupimmps(a, b, c, $imm8, k, $imm4) - }; - } - let r = constify_imm8_roundscale!(imm8, sae, call); + let r = vfixupimmps(a, b, c, IMM8, k, SAE); transmute(r) } @@ -10336,25 +9631,20 @@ pub unsafe fn _mm512_mask_fixupimm_round_ps( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_fixupimm_round_ps&expand=2507) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vfixupimmps, imm8 = 0, sae = 8))] -#[rustc_args_required_const(4, 5)] -pub unsafe fn _mm512_maskz_fixupimm_round_ps( +#[cfg_attr(test, assert_instr(vfixupimmps, IMM8 = 0, SAE = 8))] +#[rustc_legacy_const_generics(4, 5)] +pub unsafe fn _mm512_maskz_fixupimm_round_ps( k: __mmask16, a: __m512, b: __m512, c: __m512i, - imm8: i32, - sae: i32, ) -> __m512 { + static_assert_imm8!(IMM8); + static_assert_mantissas_sae!(SAE); let a = a.as_f32x16(); let b = b.as_f32x16(); let c = c.as_i32x16(); - macro_rules! call { - ($imm8:expr, $imm4:expr) => { - vfixupimmpsz(a, b, c, $imm8, k, $imm4) - }; - } - let r = constify_imm8_roundscale!(imm8, sae, call); + let r = vfixupimmpsz(a, b, c, IMM8, k, SAE); transmute(r) } @@ -10364,24 +9654,19 @@ pub unsafe fn _mm512_maskz_fixupimm_round_ps( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_fixupimm_round_pd&expand=2502) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vfixupimmpd, imm8 = 0, sae = 8))] -#[rustc_args_required_const(3, 4)] -pub unsafe fn _mm512_fixupimm_round_pd( +#[cfg_attr(test, assert_instr(vfixupimmpd, IMM8 = 0, SAE = 8))] +#[rustc_legacy_const_generics(3, 4)] +pub unsafe fn _mm512_fixupimm_round_pd( a: __m512d, b: __m512d, c: __m512i, - imm8: i32, - sae: i32, ) -> __m512d { + static_assert_imm8!(IMM8); + static_assert_mantissas_sae!(SAE); let a = a.as_f64x8(); let b = b.as_f64x8(); let c = c.as_i64x8(); - macro_rules! call { - ($imm8:expr, $imm4:expr) => { - vfixupimmpd(a, b, c, $imm8, 0b11111111, $imm4) - }; - } - let r = constify_imm8_roundscale!(imm8, sae, call); + let r = vfixupimmpd(a, b, c, IMM8, 0b11111111, SAE); transmute(r) } @@ -10391,25 +9676,20 @@ pub unsafe fn _mm512_fixupimm_round_pd( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_fixupimm_round_pd&expand=2503) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vfixupimmpd, imm8 = 0, sae = 8))] -#[rustc_args_required_const(4, 5)] -pub unsafe fn _mm512_mask_fixupimm_round_pd( +#[cfg_attr(test, assert_instr(vfixupimmpd, IMM8 = 0, SAE = 8))] +#[rustc_legacy_const_generics(4, 5)] +pub unsafe fn _mm512_mask_fixupimm_round_pd( a: __m512d, k: __mmask8, b: __m512d, c: __m512i, - imm8: i32, - sae: i32, ) -> __m512d { + static_assert_imm8!(IMM8); + static_assert_mantissas_sae!(SAE); let a = a.as_f64x8(); let b = b.as_f64x8(); let c = c.as_i64x8(); - macro_rules! call { - ($imm8:expr, $imm4:expr) => { - vfixupimmpd(a, b, c, $imm8, k, $imm4) - }; - } - let r = constify_imm8_roundscale!(imm8, sae, call); + let r = vfixupimmpd(a, b, c, IMM8, k, SAE); transmute(r) } @@ -10419,25 +9699,20 @@ pub unsafe fn _mm512_mask_fixupimm_round_pd( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_fixupimm_round_pd&expand=2504) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vfixupimmpd, imm8 = 0, sae = 8))] -#[rustc_args_required_const(4, 5)] -pub unsafe fn _mm512_maskz_fixupimm_round_pd( +#[cfg_attr(test, assert_instr(vfixupimmpd, IMM8 = 0, SAE = 8))] +#[rustc_legacy_const_generics(4, 5)] +pub unsafe fn _mm512_maskz_fixupimm_round_pd( k: __mmask8, a: __m512d, b: __m512d, c: __m512i, - imm8: i32, - sae: i32, ) -> __m512d { + static_assert_imm8!(IMM8); + static_assert_mantissas_sae!(SAE); let a = a.as_f64x8(); let b = b.as_f64x8(); let c = c.as_i64x8(); - macro_rules! call { - ($imm8:expr, $imm4:expr) => { - vfixupimmpdz(a, b, c, $imm8, k, $imm4) - }; - } - let r = constify_imm8_roundscale!(imm8, sae, call); + let r = vfixupimmpdz(a, b, c, IMM8, k, SAE); transmute(r) } @@ -10456,22 +9731,21 @@ pub unsafe fn _mm512_maskz_fixupimm_round_pd( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_getmant_round_ps&expand=2886) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vgetmantps, norm = 0, sign = 0, sae = 4))] -#[rustc_args_required_const(1, 2, 3)] -pub unsafe fn _mm512_getmant_round_ps( +#[cfg_attr(test, assert_instr(vgetmantps, NORM = 0, SIGN = 0, SAE = 4))] +#[rustc_legacy_const_generics(1, 2, 3)] +pub unsafe fn _mm512_getmant_round_ps< + const NORM: _MM_MANTISSA_NORM_ENUM, + const SIGN: _MM_MANTISSA_SIGN_ENUM, + const SAE: i32, +>( a: __m512, - norm: _MM_MANTISSA_NORM_ENUM, - sign: _MM_MANTISSA_SIGN_ENUM, - sae: i32, ) -> __m512 { + static_assert_imm4!(NORM); + static_assert_imm2!(SIGN); + static_assert_mantissas_sae!(SAE); let a = a.as_f32x16(); let zero = _mm512_setzero_ps().as_f32x16(); - macro_rules! call { - ($imm4_1:expr, $imm2:expr, $imm4_2:expr) => { - vgetmantps(a, $imm2 << 2 | $imm4_1, zero, 0b11111111_11111111, $imm4_2) - }; - } - let r = constify_imm4_mantissas_sae!(norm, sign, sae, call); + let r = vgetmantps(a, SIGN << 2 | NORM, zero, 0b11111111_11111111, SAE); transmute(r) } @@ -10490,24 +9764,23 @@ pub unsafe fn _mm512_getmant_round_ps( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_getmant_round_ps&expand=2887) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vgetmantps, norm = 0, sign = 0, sae = 4))] -#[rustc_args_required_const(3, 4, 5)] -pub unsafe fn _mm512_mask_getmant_round_ps( +#[cfg_attr(test, assert_instr(vgetmantps, NORM = 0, SIGN = 0, SAE = 4))] +#[rustc_legacy_const_generics(3, 4, 5)] +pub unsafe fn _mm512_mask_getmant_round_ps< + const NORM: _MM_MANTISSA_NORM_ENUM, + const SIGN: _MM_MANTISSA_SIGN_ENUM, + const SAE: i32, +>( src: __m512, k: __mmask16, a: __m512, - norm: _MM_MANTISSA_NORM_ENUM, - sign: _MM_MANTISSA_SIGN_ENUM, - sae: i32, ) -> __m512 { + static_assert_imm4!(NORM); + static_assert_imm2!(SIGN); + static_assert_mantissas_sae!(SAE); let a = a.as_f32x16(); let src = src.as_f32x16(); - macro_rules! call { - ($imm4_1:expr, $imm2:expr, $imm4_2:expr) => { - vgetmantps(a, $imm2 << 2 | $imm4_1, src, k, $imm4_2) - }; - } - let r = constify_imm4_mantissas_sae!(norm, sign, sae, call); + let r = vgetmantps(a, SIGN << 2 | NORM, src, k, SAE); transmute(r) } @@ -10526,23 +9799,22 @@ pub unsafe fn _mm512_mask_getmant_round_ps( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_getmant_round_ps&expand=2888) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vgetmantps, norm = 0, sign = 0, sae = 4))] -#[rustc_args_required_const(2, 3, 4)] -pub unsafe fn _mm512_maskz_getmant_round_ps( +#[cfg_attr(test, assert_instr(vgetmantps, NORM = 0, SIGN = 0, SAE = 4))] +#[rustc_legacy_const_generics(2, 3, 4)] +pub unsafe fn _mm512_maskz_getmant_round_ps< + const NORM: _MM_MANTISSA_NORM_ENUM, + const SIGN: _MM_MANTISSA_SIGN_ENUM, + const SAE: i32, +>( k: __mmask16, a: __m512, - norm: _MM_MANTISSA_NORM_ENUM, - sign: _MM_MANTISSA_SIGN_ENUM, - sae: i32, ) -> __m512 { + static_assert_imm4!(NORM); + static_assert_imm2!(SIGN); + static_assert_mantissas_sae!(SAE); let a = a.as_f32x16(); let zero = _mm512_setzero_ps().as_f32x16(); - macro_rules! call { - ($imm4_1:expr, $imm2:expr, $imm4_2:expr) => { - vgetmantps(a, $imm2 << 2 | $imm4_1, zero, k, $imm4_2) - }; - } - let r = constify_imm4_mantissas_sae!(norm, sign, sae, call); + let r = vgetmantps(a, SIGN << 2 | NORM, zero, k, SAE); transmute(r) } @@ -10561,22 +9833,21 @@ pub unsafe fn _mm512_maskz_getmant_round_ps( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_getmant_round_pd&expand=2883) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vgetmantpd, norm = 0, sign = 0, sae = 4))] -#[rustc_args_required_const(1, 2, 3)] -pub unsafe fn _mm512_getmant_round_pd( +#[cfg_attr(test, assert_instr(vgetmantpd, NORM = 0, SIGN = 0, SAE = 4))] +#[rustc_legacy_const_generics(1, 2, 3)] +pub unsafe fn _mm512_getmant_round_pd< + const NORM: _MM_MANTISSA_NORM_ENUM, + const SIGN: _MM_MANTISSA_SIGN_ENUM, + const SAE: i32, +>( a: __m512d, - norm: _MM_MANTISSA_NORM_ENUM, - sign: _MM_MANTISSA_SIGN_ENUM, - sae: i32, ) -> __m512d { + static_assert_imm4!(NORM); + static_assert_imm2!(SIGN); + static_assert_mantissas_sae!(SAE); let a = a.as_f64x8(); let zero = _mm512_setzero_pd().as_f64x8(); - macro_rules! call { - ($imm4_1:expr, $imm2:expr, $imm4_2:expr) => { - vgetmantpd(a, $imm2 << 2 | $imm4_1, zero, 0b11111111, $imm4_2) - }; - } - let r = constify_imm4_mantissas_sae!(norm, sign, sae, call); + let r = vgetmantpd(a, SIGN << 2 | NORM, zero, 0b11111111, SAE); transmute(r) } @@ -10595,24 +9866,23 @@ pub unsafe fn _mm512_getmant_round_pd( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_getmant_round_pd&expand=2884) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vgetmantpd, norm = 0, sign = 0, sae = 4))] -#[rustc_args_required_const(3, 4, 5)] -pub unsafe fn _mm512_mask_getmant_round_pd( +#[cfg_attr(test, assert_instr(vgetmantpd, NORM = 0, SIGN = 0, SAE = 4))] +#[rustc_legacy_const_generics(3, 4, 5)] +pub unsafe fn _mm512_mask_getmant_round_pd< + const NORM: _MM_MANTISSA_NORM_ENUM, + const SIGN: _MM_MANTISSA_SIGN_ENUM, + const SAE: i32, +>( src: __m512d, k: __mmask8, a: __m512d, - norm: _MM_MANTISSA_NORM_ENUM, - sign: _MM_MANTISSA_SIGN_ENUM, - sae: i32, ) -> __m512d { + static_assert_imm4!(NORM); + static_assert_imm2!(SIGN); + static_assert_mantissas_sae!(SAE); let a = a.as_f64x8(); let src = src.as_f64x8(); - macro_rules! call { - ($imm4_1:expr, $imm2:expr, $imm4_2:expr) => { - vgetmantpd(a, $imm2 << 2 | $imm4_1, src, k, $imm4_2) - }; - } - let r = constify_imm4_mantissas_sae!(norm, sign, sae, call); + let r = vgetmantpd(a, SIGN << 2 | NORM, src, k, SAE); transmute(r) } @@ -10631,23 +9901,22 @@ pub unsafe fn _mm512_mask_getmant_round_pd( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_512_maskz_getmant_round_pd&expand=2885) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vgetmantpd, norm = 0, sign = 0, sae = 4))] -#[rustc_args_required_const(2, 3, 4)] -pub unsafe fn _mm512_maskz_getmant_round_pd( +#[cfg_attr(test, assert_instr(vgetmantpd, NORM = 0, SIGN = 0, SAE = 4))] +#[rustc_legacy_const_generics(2, 3, 4)] +pub unsafe fn _mm512_maskz_getmant_round_pd< + const NORM: _MM_MANTISSA_NORM_ENUM, + const SIGN: _MM_MANTISSA_SIGN_ENUM, + const SAE: i32, +>( k: __mmask8, a: __m512d, - norm: _MM_MANTISSA_NORM_ENUM, - sign: _MM_MANTISSA_SIGN_ENUM, - sae: i32, ) -> __m512d { + static_assert_imm4!(NORM); + static_assert_imm2!(SIGN); + static_assert_mantissas_sae!(SAE); let a = a.as_f64x8(); let zero = _mm512_setzero_pd().as_f64x8(); - macro_rules! call { - ($imm4_1:expr, $imm2:expr, $imm4_2:expr) => { - vgetmantpd(a, $imm2 << 2 | $imm4_1, zero, k, $imm4_2) - }; - } - let r = constify_imm4_mantissas_sae!(norm, sign, sae, call); + let r = vgetmantpd(a, SIGN << 2 | NORM, zero, k, SAE); transmute(r) } @@ -11260,7 +10529,7 @@ pub unsafe fn _mm512_cvtpd_pslo(v2: __m512d) -> __m512 { 0b11111111, _MM_FROUND_CUR_DIRECTION, ); - simd_shuffle16( + simd_shuffle16!( r, _mm256_setzero_ps().as_f32x8(), [0, 1, 2, 3, 4, 5, 6, 7, 8, 8, 8, 8, 8, 8, 8, 8], @@ -11280,7 +10549,7 @@ pub unsafe fn _mm512_mask_cvtpd_pslo(src: __m512, k: __mmask8, v2: __m512d) -> _ k, _MM_FROUND_CUR_DIRECTION, ); - simd_shuffle16( + simd_shuffle16!( r, _mm256_setzero_ps().as_f32x8(), [0, 1, 2, 3, 4, 5, 6, 7, 8, 8, 8, 8, 8, 8, 8, 8], @@ -11375,7 +10644,7 @@ pub unsafe fn _mm_maskz_cvtepi8_epi32(k: __mmask8, a: __m128i) -> __m128i { #[cfg_attr(test, assert_instr(vpmovsxbq))] pub unsafe fn _mm512_cvtepi8_epi64(a: __m128i) -> __m512i { let a = a.as_i8x16(); - let v64: i8x8 = simd_shuffle8(a, a, [0, 1, 2, 3, 4, 5, 6, 7]); + let v64: i8x8 = simd_shuffle8!(a, a, [0, 1, 2, 3, 4, 5, 6, 7]); transmute::(simd_cast(v64)) } @@ -11536,7 +10805,7 @@ pub unsafe fn _mm_maskz_cvtepu8_epi32(k: __mmask8, a: __m128i) -> __m128i { #[cfg_attr(test, assert_instr(vpmovzxbq))] pub unsafe fn _mm512_cvtepu8_epi64(a: __m128i) -> __m512i { let a = a.as_u8x16(); - let v64: u8x8 = simd_shuffle8(a, a, [0, 1, 2, 3, 4, 5, 6, 7]); + let v64: u8x8 = simd_shuffle8!(a, a, [0, 1, 2, 3, 4, 5, 6, 7]); transmute::(simd_cast(v64)) } @@ -12359,7 +11628,7 @@ pub unsafe fn _mm256_maskz_cvtepu32_pd(k: __mmask8, a: __m128i) -> __m256d { #[cfg_attr(test, assert_instr(vcvtudq2pd))] pub unsafe fn _mm_cvtepu32_pd(a: __m128i) -> __m128d { let a = a.as_u32x4(); - let u64: u32x2 = simd_shuffle2(a, a, [0, 1]); + let u64: u32x2 = simd_shuffle2!(a, a, [0, 1]); transmute::(simd_cast(u64)) } @@ -12394,7 +11663,7 @@ pub unsafe fn _mm_maskz_cvtepu32_pd(k: __mmask8, a: __m128i) -> __m128d { #[cfg_attr(test, assert_instr(vcvtdq2pd))] pub unsafe fn _mm512_cvtepi32lo_pd(v2: __m512i) -> __m512d { let v2 = v2.as_i32x16(); - let v256: i32x8 = simd_shuffle8(v2, v2, [0, 1, 2, 3, 4, 5, 6, 7]); + let v256: i32x8 = simd_shuffle8!(v2, v2, [0, 1, 2, 3, 4, 5, 6, 7]); transmute::(simd_cast(v256)) } @@ -12417,7 +11686,7 @@ pub unsafe fn _mm512_mask_cvtepi32lo_pd(src: __m512d, k: __mmask8, v2: __m512i) #[cfg_attr(test, assert_instr(vcvtudq2pd))] pub unsafe fn _mm512_cvtepu32lo_pd(v2: __m512i) -> __m512d { let v2 = v2.as_u32x16(); - let v256: u32x8 = simd_shuffle8(v2, v2, [0, 1, 2, 3, 4, 5, 6, 7]); + let v256: u32x8 = simd_shuffle8!(v2, v2, [0, 1, 2, 3, 4, 5, 6, 7]); transmute::(simd_cast(v256)) } @@ -14026,17 +13295,13 @@ pub unsafe fn _mm_maskz_cvtusepi64_epi8(k: __mmask8, a: __m128i) -> __m128i { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cvt_roundps_epi32&expand=1335) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vcvtps2dq, rounding = 8))] -#[rustc_args_required_const(1)] -pub unsafe fn _mm512_cvt_roundps_epi32(a: __m512, rounding: i32) -> __m512i { +#[cfg_attr(test, assert_instr(vcvtps2dq, ROUNDING = 8))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm512_cvt_roundps_epi32(a: __m512) -> __m512i { + static_assert_rounding!(ROUNDING); let a = a.as_f32x16(); let zero = _mm512_setzero_si512().as_i32x16(); - macro_rules! call { - ($imm4:expr) => { - vcvtps2dq(a, zero, 0b11111111_11111111, $imm4) - }; - } - let r = constify_imm4_round!(rounding, call); + let r = vcvtps2dq(a, zero, 0b11111111_11111111, ROUNDING); transmute(r) } @@ -14052,22 +13317,17 @@ pub unsafe fn _mm512_cvt_roundps_epi32(a: __m512, rounding: i32) -> __m512i { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cvt_roundps_epi32&expand=1336) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vcvtps2dq, rounding = 8))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm512_mask_cvt_roundps_epi32( +#[cfg_attr(test, assert_instr(vcvtps2dq, ROUNDING = 8))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_mask_cvt_roundps_epi32( src: __m512i, k: __mmask16, a: __m512, - rounding: i32, ) -> __m512i { + static_assert_rounding!(ROUNDING); let a = a.as_f32x16(); let src = src.as_i32x16(); - macro_rules! call { - ($imm4:expr) => { - vcvtps2dq(a, src, k, $imm4) - }; - } - let r = constify_imm4_round!(rounding, call); + let r = vcvtps2dq(a, src, k, ROUNDING); transmute(r) } @@ -14083,17 +13343,16 @@ pub unsafe fn _mm512_mask_cvt_roundps_epi32( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_cvt_roundps_epi32&expand=1337) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vcvtps2dq, rounding = 8))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm512_maskz_cvt_roundps_epi32(k: __mmask16, a: __m512, rounding: i32) -> __m512i { +#[cfg_attr(test, assert_instr(vcvtps2dq, ROUNDING = 8))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_maskz_cvt_roundps_epi32( + k: __mmask16, + a: __m512, +) -> __m512i { + static_assert_rounding!(ROUNDING); let a = a.as_f32x16(); let zero = _mm512_setzero_si512().as_i32x16(); - macro_rules! call { - ($imm4:expr) => { - vcvtps2dq(a, zero, k, $imm4) - }; - } - let r = constify_imm4_round!(rounding, call); + let r = vcvtps2dq(a, zero, k, ROUNDING); transmute(r) } @@ -14109,17 +13368,13 @@ pub unsafe fn _mm512_maskz_cvt_roundps_epi32(k: __mmask16, a: __m512, rounding: /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cvt_roundps_epu32&expand=1341) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vcvtps2udq, rounding = 8))] -#[rustc_args_required_const(1)] -pub unsafe fn _mm512_cvt_roundps_epu32(a: __m512, rounding: i32) -> __m512i { +#[cfg_attr(test, assert_instr(vcvtps2udq, ROUNDING = 8))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm512_cvt_roundps_epu32(a: __m512) -> __m512i { + static_assert_rounding!(ROUNDING); let a = a.as_f32x16(); let zero = _mm512_setzero_si512().as_u32x16(); - macro_rules! call { - ($imm4:expr) => { - vcvtps2udq(a, zero, 0b11111111_11111111, $imm4) - }; - } - let r = constify_imm4_round!(rounding, call); + let r = vcvtps2udq(a, zero, 0b11111111_11111111, ROUNDING); transmute(r) } @@ -14135,22 +13390,17 @@ pub unsafe fn _mm512_cvt_roundps_epu32(a: __m512, rounding: i32) -> __m512i { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cvt_roundps_epu32&expand=1342) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vcvtps2udq, rounding = 8))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm512_mask_cvt_roundps_epu32( +#[cfg_attr(test, assert_instr(vcvtps2udq, ROUNDING = 8))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_mask_cvt_roundps_epu32( src: __m512i, k: __mmask16, a: __m512, - rounding: i32, ) -> __m512i { + static_assert_rounding!(ROUNDING); let a = a.as_f32x16(); let src = src.as_u32x16(); - macro_rules! call { - ($imm4:expr) => { - vcvtps2udq(a, src, k, $imm4) - }; - } - let r = constify_imm4_round!(rounding, call); + let r = vcvtps2udq(a, src, k, ROUNDING); transmute(r) } @@ -14166,17 +13416,16 @@ pub unsafe fn _mm512_mask_cvt_roundps_epu32( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_cvt_roundps_epu32&expand=1343) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vcvtps2udq, rounding = 8))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm512_maskz_cvt_roundps_epu32(k: __mmask16, a: __m512, rounding: i32) -> __m512i { +#[cfg_attr(test, assert_instr(vcvtps2udq, ROUNDING = 8))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_maskz_cvt_roundps_epu32( + k: __mmask16, + a: __m512, +) -> __m512i { + static_assert_rounding!(ROUNDING); let a = a.as_f32x16(); let zero = _mm512_setzero_si512().as_u32x16(); - macro_rules! call { - ($imm4:expr) => { - vcvtps2udq(a, zero, k, $imm4) - }; - } - let r = constify_imm4_round!(rounding, call); + let r = vcvtps2udq(a, zero, k, ROUNDING); transmute(r) } @@ -14186,17 +13435,13 @@ pub unsafe fn _mm512_maskz_cvt_roundps_epu32(k: __mmask16, a: __m512, rounding: /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cvt_roundps_pd&expand=1347) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vcvtps2pd, sae = 8))] -#[rustc_args_required_const(1)] -pub unsafe fn _mm512_cvt_roundps_pd(a: __m256, sae: i32) -> __m512d { +#[cfg_attr(test, assert_instr(vcvtps2pd, SAE = 8))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm512_cvt_roundps_pd(a: __m256) -> __m512d { + static_assert_sae!(SAE); let a = a.as_f32x8(); let zero = _mm512_setzero_pd().as_f64x8(); - macro_rules! call { - ($imm4:expr) => { - vcvtps2pd(a, zero, 0b11111111, $imm4) - }; - } - let r = constify_imm4_sae!(sae, call); + let r = vcvtps2pd(a, zero, 0b11111111, SAE); transmute(r) } @@ -14206,22 +13451,17 @@ pub unsafe fn _mm512_cvt_roundps_pd(a: __m256, sae: i32) -> __m512d { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cvt_roundps_epi32&expand=1336) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vcvtps2pd, sae = 8))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm512_mask_cvt_roundps_pd( +#[cfg_attr(test, assert_instr(vcvtps2pd, SAE = 8))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_mask_cvt_roundps_pd( src: __m512d, k: __mmask8, a: __m256, - sae: i32, ) -> __m512d { + static_assert_sae!(SAE); let a = a.as_f32x8(); let src = src.as_f64x8(); - macro_rules! call { - ($imm4:expr) => { - vcvtps2pd(a, src, k, $imm4) - }; - } - let r = constify_imm4_sae!(sae, call); + let r = vcvtps2pd(a, src, k, SAE); transmute(r) } @@ -14231,17 +13471,13 @@ pub unsafe fn _mm512_mask_cvt_roundps_pd( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_cvt_roundps_epi32&expand=1337) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vcvtps2pd, sae = 8))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm512_maskz_cvt_roundps_pd(k: __mmask8, a: __m256, sae: i32) -> __m512d { +#[cfg_attr(test, assert_instr(vcvtps2pd, SAE = 8))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_maskz_cvt_roundps_pd(k: __mmask8, a: __m256) -> __m512d { + static_assert_sae!(SAE); let a = a.as_f32x8(); let zero = _mm512_setzero_pd().as_f64x8(); - macro_rules! call { - ($imm4:expr) => { - vcvtps2pd(a, zero, k, $imm4) - }; - } - let r = constify_imm4_sae!(sae, call); + let r = vcvtps2pd(a, zero, k, SAE); transmute(r) } @@ -14257,17 +13493,13 @@ pub unsafe fn _mm512_maskz_cvt_roundps_pd(k: __mmask8, a: __m256, sae: i32) -> _ /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cvt_roundpd_epi32&expand=1315) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vcvtpd2dq, rounding = 8))] -#[rustc_args_required_const(1)] -pub unsafe fn _mm512_cvt_roundpd_epi32(a: __m512d, rounding: i32) -> __m256i { +#[cfg_attr(test, assert_instr(vcvtpd2dq, ROUNDING = 8))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm512_cvt_roundpd_epi32(a: __m512d) -> __m256i { + static_assert_rounding!(ROUNDING); let a = a.as_f64x8(); let zero = _mm256_setzero_si256().as_i32x8(); - macro_rules! call { - ($imm4:expr) => { - vcvtpd2dq(a, zero, 0b11111111, $imm4) - }; - } - let r = constify_imm4_round!(rounding, call); + let r = vcvtpd2dq(a, zero, 0b11111111, ROUNDING); transmute(r) } @@ -14283,22 +13515,17 @@ pub unsafe fn _mm512_cvt_roundpd_epi32(a: __m512d, rounding: i32) -> __m256i { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cvt_roundpd_epi32&expand=1316) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vcvtpd2dq, rounding = 8))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm512_mask_cvt_roundpd_epi32( +#[cfg_attr(test, assert_instr(vcvtpd2dq, ROUNDING = 8))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_mask_cvt_roundpd_epi32( src: __m256i, k: __mmask8, a: __m512d, - rounding: i32, ) -> __m256i { + static_assert_rounding!(ROUNDING); let a = a.as_f64x8(); let src = src.as_i32x8(); - macro_rules! call { - ($imm4:expr) => { - vcvtpd2dq(a, src, k, $imm4) - }; - } - let r = constify_imm4_round!(rounding, call); + let r = vcvtpd2dq(a, src, k, ROUNDING); transmute(r) } @@ -14314,17 +13541,16 @@ pub unsafe fn _mm512_mask_cvt_roundpd_epi32( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/IntrinsicsGuide/#text=_mm512_maskz_cvt_roundpd_epi32&expand=1317) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vcvtpd2dq, rounding = 8))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm512_maskz_cvt_roundpd_epi32(k: __mmask8, a: __m512d, rounding: i32) -> __m256i { +#[cfg_attr(test, assert_instr(vcvtpd2dq, ROUNDING = 8))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_maskz_cvt_roundpd_epi32( + k: __mmask8, + a: __m512d, +) -> __m256i { + static_assert_rounding!(ROUNDING); let a = a.as_f64x8(); let zero = _mm256_setzero_si256().as_i32x8(); - macro_rules! call { - ($imm4:expr) => { - vcvtpd2dq(a, zero, k, $imm4) - }; - } - let r = constify_imm4_round!(rounding, call); + let r = vcvtpd2dq(a, zero, k, ROUNDING); transmute(r) } @@ -14340,17 +13566,13 @@ pub unsafe fn _mm512_maskz_cvt_roundpd_epi32(k: __mmask8, a: __m512d, rounding: /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cvt_roundpd_epu32&expand=1321) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vcvtpd2udq, rounding = 8))] -#[rustc_args_required_const(1)] -pub unsafe fn _mm512_cvt_roundpd_epu32(a: __m512d, rounding: i32) -> __m256i { +#[cfg_attr(test, assert_instr(vcvtpd2udq, ROUNDING = 8))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm512_cvt_roundpd_epu32(a: __m512d) -> __m256i { + static_assert_rounding!(ROUNDING); let a = a.as_f64x8(); let zero = _mm256_setzero_si256().as_u32x8(); - macro_rules! call { - ($imm4:expr) => { - vcvtpd2udq(a, zero, 0b11111111, $imm4) - }; - } - let r = constify_imm4_round!(rounding, call); + let r = vcvtpd2udq(a, zero, 0b11111111, ROUNDING); transmute(r) } @@ -14366,22 +13588,17 @@ pub unsafe fn _mm512_cvt_roundpd_epu32(a: __m512d, rounding: i32) -> __m256i { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cvt_roundpd_epu32&expand=1322) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vcvtpd2udq, rounding = 8))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm512_mask_cvt_roundpd_epu32( +#[cfg_attr(test, assert_instr(vcvtpd2udq, ROUNDING = 8))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_mask_cvt_roundpd_epu32( src: __m256i, k: __mmask8, a: __m512d, - rounding: i32, ) -> __m256i { + static_assert_rounding!(ROUNDING); let a = a.as_f64x8(); let src = src.as_u32x8(); - macro_rules! call { - ($imm4:expr) => { - vcvtpd2udq(a, src, k, $imm4) - }; - } - let r = constify_imm4_round!(rounding, call); + let r = vcvtpd2udq(a, src, k, ROUNDING); transmute(r) } @@ -14397,17 +13614,16 @@ pub unsafe fn _mm512_mask_cvt_roundpd_epu32( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/IntrinsicsGuide/#text=_mm512_maskz_cvt_roundpd_epu32&expand=1323) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vcvtpd2udq, rounding = 8))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm512_maskz_cvt_roundpd_epu32(k: __mmask8, a: __m512d, rounding: i32) -> __m256i { +#[cfg_attr(test, assert_instr(vcvtpd2udq, ROUNDING = 8))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_maskz_cvt_roundpd_epu32( + k: __mmask8, + a: __m512d, +) -> __m256i { + static_assert_rounding!(ROUNDING); let a = a.as_f64x8(); let zero = _mm256_setzero_si256().as_u32x8(); - macro_rules! call { - ($imm4:expr) => { - vcvtpd2udq(a, zero, k, $imm4) - }; - } - let r = constify_imm4_round!(rounding, call); + let r = vcvtpd2udq(a, zero, k, ROUNDING); transmute(r) } @@ -14423,17 +13639,13 @@ pub unsafe fn _mm512_maskz_cvt_roundpd_epu32(k: __mmask8, a: __m512d, rounding: /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cvt_roundpd_ps&expand=1327) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vcvtpd2ps, rounding = 8))] -#[rustc_args_required_const(1)] -pub unsafe fn _mm512_cvt_roundpd_ps(a: __m512d, rounding: i32) -> __m256 { +#[cfg_attr(test, assert_instr(vcvtpd2ps, ROUNDING = 8))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm512_cvt_roundpd_ps(a: __m512d) -> __m256 { + static_assert_rounding!(ROUNDING); let a = a.as_f64x8(); let zero = _mm256_setzero_ps().as_f32x8(); - macro_rules! call { - ($imm4:expr) => { - vcvtpd2ps(a, zero, 0b11111111, $imm4) - }; - } - let r = constify_imm4_round!(rounding, call); + let r = vcvtpd2ps(a, zero, 0b11111111, ROUNDING); transmute(r) } @@ -14449,22 +13661,17 @@ pub unsafe fn _mm512_cvt_roundpd_ps(a: __m512d, rounding: i32) -> __m256 { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cvt_roundpd_ps&expand=1328) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vcvtpd2ps, rounding = 8))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm512_mask_cvt_roundpd_ps( +#[cfg_attr(test, assert_instr(vcvtpd2ps, ROUNDING = 8))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_mask_cvt_roundpd_ps( src: __m256, k: __mmask8, a: __m512d, - rounding: i32, ) -> __m256 { + static_assert_rounding!(ROUNDING); let a = a.as_f64x8(); let src = src.as_f32x8(); - macro_rules! call { - ($imm4:expr) => { - vcvtpd2ps(a, src, k, $imm4) - }; - } - let r = constify_imm4_round!(rounding, call); + let r = vcvtpd2ps(a, src, k, ROUNDING); transmute(r) } @@ -14480,17 +13687,13 @@ pub unsafe fn _mm512_mask_cvt_roundpd_ps( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_cvt_roundpd_ps&expand=1329) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vcvtpd2ps, rounding = 8))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm512_maskz_cvt_roundpd_ps(k: __mmask8, a: __m512d, rounding: i32) -> __m256 { +#[cfg_attr(test, assert_instr(vcvtpd2ps, ROUNDING = 8))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_maskz_cvt_roundpd_ps(k: __mmask8, a: __m512d) -> __m256 { + static_assert_rounding!(ROUNDING); let a = a.as_f64x8(); let zero = _mm256_setzero_ps().as_f32x8(); - macro_rules! call { - ($imm4:expr) => { - vcvtpd2ps(a, zero, k, $imm4) - }; - } - let r = constify_imm4_round!(rounding, call); + let r = vcvtpd2ps(a, zero, k, ROUNDING); transmute(r) } @@ -14506,16 +13709,12 @@ pub unsafe fn _mm512_maskz_cvt_roundpd_ps(k: __mmask8, a: __m512d, rounding: i32 /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cvt_roundepi32_ps&expand=1294) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vcvtdq2ps, rounding = 8))] -#[rustc_args_required_const(1)] -pub unsafe fn _mm512_cvt_roundepi32_ps(a: __m512i, rounding: i32) -> __m512 { +#[cfg_attr(test, assert_instr(vcvtdq2ps, ROUNDING = 8))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm512_cvt_roundepi32_ps(a: __m512i) -> __m512 { + static_assert_rounding!(ROUNDING); let a = a.as_i32x16(); - macro_rules! call { - ($imm4:expr) => { - vcvtdq2ps(a, $imm4) - }; - } - let r = constify_imm4_round!(rounding, call); + let r = vcvtdq2ps(a, ROUNDING); transmute(r) } @@ -14531,21 +13730,16 @@ pub unsafe fn _mm512_cvt_roundepi32_ps(a: __m512i, rounding: i32) -> __m512 { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cvt_roundepi32_ps&expand=1295) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vcvtdq2ps, rounding = 8))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm512_mask_cvt_roundepi32_ps( +#[cfg_attr(test, assert_instr(vcvtdq2ps, ROUNDING = 8))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_mask_cvt_roundepi32_ps( src: __m512, k: __mmask16, a: __m512i, - rounding: i32, ) -> __m512 { + static_assert_rounding!(ROUNDING); let a = a.as_i32x16(); - macro_rules! call { - ($imm4:expr) => { - vcvtdq2ps(a, $imm4) - }; - } - let r: f32x16 = constify_imm4_round!(rounding, call); + let r = vcvtdq2ps(a, ROUNDING); transmute(simd_select_bitmask(k, r, src.as_f32x16())) } @@ -14561,16 +13755,15 @@ pub unsafe fn _mm512_mask_cvt_roundepi32_ps( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_cvt_roundepi32_ps&expand=1296) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vcvtdq2ps, rounding = 8))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm512_maskz_cvt_roundepi32_ps(k: __mmask16, a: __m512i, rounding: i32) -> __m512 { +#[cfg_attr(test, assert_instr(vcvtdq2ps, ROUNDING = 8))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_maskz_cvt_roundepi32_ps( + k: __mmask16, + a: __m512i, +) -> __m512 { + static_assert_rounding!(ROUNDING); let a = a.as_i32x16(); - macro_rules! call { - ($imm4:expr) => { - vcvtdq2ps(a, $imm4) - }; - } - let r = constify_imm4_round!(rounding, call); + let r = vcvtdq2ps(a, ROUNDING); let zero = _mm512_setzero_ps().as_f32x16(); transmute(simd_select_bitmask(k, r, zero)) } @@ -14587,16 +13780,12 @@ pub unsafe fn _mm512_maskz_cvt_roundepi32_ps(k: __mmask16, a: __m512i, rounding: /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cvt_roundepu32_ps&expand=1303) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vcvtudq2ps, rounding = 8))] -#[rustc_args_required_const(1)] -pub unsafe fn _mm512_cvt_roundepu32_ps(a: __m512i, rounding: i32) -> __m512 { +#[cfg_attr(test, assert_instr(vcvtudq2ps, ROUNDING = 8))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm512_cvt_roundepu32_ps(a: __m512i) -> __m512 { + static_assert_rounding!(ROUNDING); let a = a.as_u32x16(); - macro_rules! call { - ($imm4:expr) => { - vcvtudq2ps(a, $imm4) - }; - } - let r = constify_imm4_round!(rounding, call); + let r = vcvtudq2ps(a, ROUNDING); transmute(r) } @@ -14612,21 +13801,16 @@ pub unsafe fn _mm512_cvt_roundepu32_ps(a: __m512i, rounding: i32) -> __m512 { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cvt_roundepu32_ps&expand=1304) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vcvtudq2ps, rounding = 8))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm512_mask_cvt_roundepu32_ps( +#[cfg_attr(test, assert_instr(vcvtudq2ps, ROUNDING = 8))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_mask_cvt_roundepu32_ps( src: __m512, k: __mmask16, a: __m512i, - rounding: i32, ) -> __m512 { + static_assert_rounding!(ROUNDING); let a = a.as_u32x16(); - macro_rules! call { - ($imm4:expr) => { - vcvtudq2ps(a, $imm4) - }; - } - let r: f32x16 = constify_imm4_round!(rounding, call); + let r = vcvtudq2ps(a, ROUNDING); transmute(simd_select_bitmask(k, r, src.as_f32x16())) } @@ -14642,16 +13826,15 @@ pub unsafe fn _mm512_mask_cvt_roundepu32_ps( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_cvt_roundepu32_ps&expand=1305) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vcvtudq2ps, rounding = 8))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm512_maskz_cvt_roundepu32_ps(k: __mmask16, a: __m512i, rounding: i32) -> __m512 { +#[cfg_attr(test, assert_instr(vcvtudq2ps, ROUNDING = 8))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_maskz_cvt_roundepu32_ps( + k: __mmask16, + a: __m512i, +) -> __m512 { + static_assert_rounding!(ROUNDING); let a = a.as_u32x16(); - macro_rules! call { - ($imm4:expr) => { - vcvtudq2ps(a, $imm4) - }; - } - let r = constify_imm4_round!(rounding, call); + let r = vcvtudq2ps(a, ROUNDING); let zero = _mm512_setzero_ps().as_f32x16(); transmute(simd_select_bitmask(k, r, zero)) } @@ -14662,17 +13845,13 @@ pub unsafe fn _mm512_maskz_cvt_roundepu32_ps(k: __mmask16, a: __m512i, rounding: /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cvt_roundps_ph&expand=1354) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vcvtps2ph, sae = 8))] -#[rustc_args_required_const(1)] -pub unsafe fn _mm512_cvt_roundps_ph(a: __m512, sae: i32) -> __m256i { +#[cfg_attr(test, assert_instr(vcvtps2ph, SAE = 8))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm512_cvt_roundps_ph(a: __m512) -> __m256i { + static_assert_sae!(SAE); let a = a.as_f32x16(); let zero = _mm256_setzero_si256().as_i16x16(); - macro_rules! call { - ($imm4:expr) => { - vcvtps2ph(a, $imm4, zero, 0b11111111_11111111) - }; - } - let r = constify_imm4_sae!(sae, call); + let r = vcvtps2ph(a, SAE, zero, 0b11111111_11111111); transmute(r) } @@ -14682,22 +13861,17 @@ pub unsafe fn _mm512_cvt_roundps_ph(a: __m512, sae: i32) -> __m256i { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cvt_roundps_ph&expand=1355) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vcvtps2ph, sae = 8))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm512_mask_cvt_roundps_ph( +#[cfg_attr(test, assert_instr(vcvtps2ph, SAE = 8))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_mask_cvt_roundps_ph( src: __m256i, k: __mmask16, a: __m512, - sae: i32, ) -> __m256i { + static_assert_sae!(SAE); let a = a.as_f32x16(); let src = src.as_i16x16(); - macro_rules! call { - ($imm4:expr) => { - vcvtps2ph(a, $imm4, src, k) - }; - } - let r = constify_imm4_sae!(sae, call); + let r = vcvtps2ph(a, SAE, src, k); transmute(r) } @@ -14707,17 +13881,13 @@ pub unsafe fn _mm512_mask_cvt_roundps_ph( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_cvt_roundps_ph&expand=1356) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vcvtps2ph, sae = 8))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm512_maskz_cvt_roundps_ph(k: __mmask16, a: __m512, sae: i32) -> __m256i { +#[cfg_attr(test, assert_instr(vcvtps2ph, SAE = 8))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_maskz_cvt_roundps_ph(k: __mmask16, a: __m512) -> __m256i { + static_assert_sae!(SAE); let a = a.as_f32x16(); let zero = _mm256_setzero_si256().as_i16x16(); - macro_rules! call { - ($imm4:expr) => { - vcvtps2ph(a, $imm4, zero, k) - }; - } - let r = constify_imm4_sae!(sae, call); + let r = vcvtps2ph(a, SAE, zero, k); transmute(r) } @@ -14732,22 +13902,17 @@ pub unsafe fn _mm512_maskz_cvt_roundps_ph(k: __mmask16, a: __m512, sae: i32) -> /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_cvt_roundps_ph&expand=1352) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[cfg_attr(test, assert_instr(vcvtps2ph, imm8 = 8))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm256_mask_cvt_roundps_ph( +#[cfg_attr(test, assert_instr(vcvtps2ph, IMM8 = 8))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm256_mask_cvt_roundps_ph( src: __m128i, k: __mmask8, a: __m256, - imm8: i32, ) -> __m128i { + static_assert_imm8!(IMM8); let a = a.as_f32x8(); let src = src.as_i16x8(); - macro_rules! call { - ($imm8:expr) => { - vcvtps2ph256(a, $imm8, src, k) - }; - } - let r = constify_imm8_sae!(imm8, call); + let r = vcvtps2ph256(a, IMM8, src, k); transmute(r) } @@ -14762,17 +13927,13 @@ pub unsafe fn _mm256_mask_cvt_roundps_ph( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_cvt_roundps_ph&expand=1353) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[cfg_attr(test, assert_instr(vcvtps2ph, imm8 = 8))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm256_maskz_cvt_roundps_ph(k: __mmask8, a: __m256, imm8: i32) -> __m128i { +#[cfg_attr(test, assert_instr(vcvtps2ph, IMM8 = 8))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm256_maskz_cvt_roundps_ph(k: __mmask8, a: __m256) -> __m128i { + static_assert_imm8!(IMM8); let a = a.as_f32x8(); let zero = _mm_setzero_si128().as_i16x8(); - macro_rules! call { - ($imm8:expr) => { - vcvtps2ph256(a, $imm8, zero, k) - }; - } - let r = constify_imm8_sae!(imm8, call); + let r = vcvtps2ph256(a, IMM8, zero, k); transmute(r) } @@ -14787,17 +13948,17 @@ pub unsafe fn _mm256_maskz_cvt_roundps_ph(k: __mmask8, a: __m256, imm8: i32) -> /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_cvt_roundps_ph&expand=1350) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[cfg_attr(test, assert_instr(vcvtps2ph, imm8 = 8))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm_mask_cvt_roundps_ph(src: __m128i, k: __mmask8, a: __m128, imm8: i32) -> __m128i { +#[cfg_attr(test, assert_instr(vcvtps2ph, IMM8 = 8))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm_mask_cvt_roundps_ph( + src: __m128i, + k: __mmask8, + a: __m128, +) -> __m128i { + static_assert_imm8!(IMM8); let a = a.as_f32x4(); let src = src.as_i16x8(); - macro_rules! call { - ($imm8:expr) => { - vcvtps2ph128(a, $imm8, src, k) - }; - } - let r = constify_imm8_sae!(imm8, call); + let r = vcvtps2ph128(a, IMM8, src, k); transmute(r) } @@ -14812,17 +13973,13 @@ pub unsafe fn _mm_mask_cvt_roundps_ph(src: __m128i, k: __mmask8, a: __m128, imm8 /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_cvt_roundps_ph&expand=1351) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[cfg_attr(test, assert_instr(vcvtps2ph, imm8 = 8))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm_maskz_cvt_roundps_ph(k: __mmask8, a: __m128, imm8: i32) -> __m128i { +#[cfg_attr(test, assert_instr(vcvtps2ph, IMM8 = 8))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm_maskz_cvt_roundps_ph(k: __mmask8, a: __m128) -> __m128i { + static_assert_imm8!(IMM8); let a = a.as_f32x4(); let zero = _mm_setzero_si128().as_i16x8(); - macro_rules! call { - ($imm8:expr) => { - vcvtps2ph128(a, $imm8, zero, k) - }; - } - let r = constify_imm8_sae!(imm8, call); + let r = vcvtps2ph128(a, IMM8, zero, k); transmute(r) } @@ -14832,17 +13989,13 @@ pub unsafe fn _mm_maskz_cvt_roundps_ph(k: __mmask8, a: __m128, imm8: i32) -> __m /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cvtps_ph&expand=1778) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vcvtps2ph, sae = 8))] -#[rustc_args_required_const(1)] -pub unsafe fn _mm512_cvtps_ph(a: __m512, sae: i32) -> __m256i { +#[cfg_attr(test, assert_instr(vcvtps2ph, SAE = 8))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm512_cvtps_ph(a: __m512) -> __m256i { + static_assert_sae!(SAE); let a = a.as_f32x16(); let zero = _mm256_setzero_si256().as_i16x16(); - macro_rules! call { - ($imm4:expr) => { - vcvtps2ph(a, $imm4, zero, 0b11111111_11111111) - }; - } - let r = constify_imm4_sae!(sae, call); + let r = vcvtps2ph(a, SAE, zero, 0b11111111_11111111); transmute(r) } @@ -14852,17 +14005,17 @@ pub unsafe fn _mm512_cvtps_ph(a: __m512, sae: i32) -> __m256i { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cvtps_ph&expand=1779) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vcvtps2ph, sae = 8))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm512_mask_cvtps_ph(src: __m256i, k: __mmask16, a: __m512, sae: i32) -> __m256i { +#[cfg_attr(test, assert_instr(vcvtps2ph, SAE = 8))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_mask_cvtps_ph( + src: __m256i, + k: __mmask16, + a: __m512, +) -> __m256i { + static_assert_sae!(SAE); let a = a.as_f32x16(); let src = src.as_i16x16(); - macro_rules! call { - ($imm4:expr) => { - vcvtps2ph(a, $imm4, src, k) - }; - } - let r = constify_imm4_sae!(sae, call); + let r = vcvtps2ph(a, SAE, src, k); transmute(r) } @@ -14872,17 +14025,13 @@ pub unsafe fn _mm512_mask_cvtps_ph(src: __m256i, k: __mmask16, a: __m512, sae: i /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_cvtps_ph&expand=1780) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vcvtps2ph, sae = 8))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm512_maskz_cvtps_ph(k: __mmask16, a: __m512, sae: i32) -> __m256i { +#[cfg_attr(test, assert_instr(vcvtps2ph, SAE = 8))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_maskz_cvtps_ph(k: __mmask16, a: __m512) -> __m256i { + static_assert_sae!(SAE); let a = a.as_f32x16(); let zero = _mm256_setzero_si256().as_i16x16(); - macro_rules! call { - ($imm4:expr) => { - vcvtps2ph(a, $imm4, zero, k) - }; - } - let r = constify_imm4_sae!(sae, call); + let r = vcvtps2ph(a, SAE, zero, k); transmute(r) } @@ -14897,17 +14046,17 @@ pub unsafe fn _mm512_maskz_cvtps_ph(k: __mmask16, a: __m512, sae: i32) -> __m256 /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_cvtps_ph&expand=1776) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[cfg_attr(test, assert_instr(vcvtps2ph, imm8 = 8))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm256_mask_cvtps_ph(src: __m128i, k: __mmask8, a: __m256, imm8: i32) -> __m128i { +#[cfg_attr(test, assert_instr(vcvtps2ph, IMM8 = 8))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm256_mask_cvtps_ph( + src: __m128i, + k: __mmask8, + a: __m256, +) -> __m128i { + static_assert_imm8!(IMM8); let a = a.as_f32x8(); let src = src.as_i16x8(); - macro_rules! call { - ($imm8:expr) => { - vcvtps2ph256(a, $imm8, src, k) - }; - } - let r = constify_imm8_sae!(imm8, call); + let r = vcvtps2ph256(a, IMM8, src, k); transmute(r) } @@ -14922,17 +14071,13 @@ pub unsafe fn _mm256_mask_cvtps_ph(src: __m128i, k: __mmask8, a: __m256, imm8: i /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_cvtps_ph&expand=1777) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[cfg_attr(test, assert_instr(vcvtps2ph, imm8 = 8))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm256_maskz_cvtps_ph(k: __mmask8, a: __m256, imm8: i32) -> __m128i { +#[cfg_attr(test, assert_instr(vcvtps2ph, IMM8 = 8))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm256_maskz_cvtps_ph(k: __mmask8, a: __m256) -> __m128i { + static_assert_imm8!(IMM8); let a = a.as_f32x8(); let zero = _mm_setzero_si128().as_i16x8(); - macro_rules! call { - ($imm8:expr) => { - vcvtps2ph256(a, $imm8, zero, k) - }; - } - let r = constify_imm8_sae!(imm8, call); + let r = vcvtps2ph256(a, IMM8, zero, k); transmute(r) } @@ -14947,17 +14092,13 @@ pub unsafe fn _mm256_maskz_cvtps_ph(k: __mmask8, a: __m256, imm8: i32) -> __m128 /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_cvtps_ph&expand=1773) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[cfg_attr(test, assert_instr(vcvtps2ph, imm8 = 8))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm_mask_cvtps_ph(src: __m128i, k: __mmask8, a: __m128, imm8: i32) -> __m128i { +#[cfg_attr(test, assert_instr(vcvtps2ph, IMM8 = 8))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm_mask_cvtps_ph(src: __m128i, k: __mmask8, a: __m128) -> __m128i { + static_assert_imm8!(IMM8); let a = a.as_f32x4(); let src = src.as_i16x8(); - macro_rules! call { - ($imm8:expr) => { - vcvtps2ph128(a, $imm8, src, k) - }; - } - let r = constify_imm8_sae!(imm8, call); + let r = vcvtps2ph128(a, IMM8, src, k); transmute(r) } @@ -14972,17 +14113,13 @@ pub unsafe fn _mm_mask_cvtps_ph(src: __m128i, k: __mmask8, a: __m128, imm8: i32) /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_cvtps_ph&expand=1774) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[cfg_attr(test, assert_instr(vcvtps2ph, imm8 = 8))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm_maskz_cvtps_ph(k: __mmask8, a: __m128, imm8: i32) -> __m128i { +#[cfg_attr(test, assert_instr(vcvtps2ph, IMM8 = 8))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm_maskz_cvtps_ph(k: __mmask8, a: __m128) -> __m128i { + static_assert_imm8!(IMM8); let a = a.as_f32x4(); let zero = _mm_setzero_si128().as_i16x8(); - macro_rules! call { - ($imm8:expr) => { - vcvtps2ph128(a, $imm8, zero, k) - }; - } - let r = constify_imm8_sae!(imm8, call); + let r = vcvtps2ph128(a, IMM8, zero, k); transmute(r) } @@ -14992,17 +14129,13 @@ pub unsafe fn _mm_maskz_cvtps_ph(k: __mmask8, a: __m128, imm8: i32) -> __m128i { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cvt_roundph_ps&expand=1332) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vcvtph2ps, sae = 8))] -#[rustc_args_required_const(1)] -pub unsafe fn _mm512_cvt_roundph_ps(a: __m256i, sae: i32) -> __m512 { +#[cfg_attr(test, assert_instr(vcvtph2ps, SAE = 8))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm512_cvt_roundph_ps(a: __m256i) -> __m512 { + static_assert_sae!(SAE); let a = a.as_i16x16(); let zero = _mm512_setzero_ps().as_f32x16(); - macro_rules! call { - ($imm4:expr) => { - vcvtph2ps(a, zero, 0b11111111_11111111, $imm4) - }; - } - let r = constify_imm4_sae!(sae, call); + let r = vcvtph2ps(a, zero, 0b11111111_11111111, SAE); transmute(r) } @@ -15012,22 +14145,17 @@ pub unsafe fn _mm512_cvt_roundph_ps(a: __m256i, sae: i32) -> __m512 { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cvt_roundph_ps&expand=1333) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vcvtph2ps, sae = 8))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm512_mask_cvt_roundph_ps( +#[cfg_attr(test, assert_instr(vcvtph2ps, SAE = 8))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_mask_cvt_roundph_ps( src: __m512, k: __mmask16, a: __m256i, - sae: i32, ) -> __m512 { + static_assert_sae!(SAE); let a = a.as_i16x16(); let src = src.as_f32x16(); - macro_rules! call { - ($imm4:expr) => { - vcvtph2ps(a, src, k, $imm4) - }; - } - let r = constify_imm4_sae!(sae, call); + let r = vcvtph2ps(a, src, k, SAE); transmute(r) } @@ -15037,17 +14165,13 @@ pub unsafe fn _mm512_mask_cvt_roundph_ps( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_cvt_roundph_ps&expand=1334) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vcvtph2ps, sae = 8))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm512_maskz_cvt_roundph_ps(k: __mmask16, a: __m256i, sae: i32) -> __m512 { +#[cfg_attr(test, assert_instr(vcvtph2ps, SAE = 8))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_maskz_cvt_roundph_ps(k: __mmask16, a: __m256i) -> __m512 { + static_assert_sae!(SAE); let a = a.as_i16x16(); let zero = _mm512_setzero_ps().as_f32x16(); - macro_rules! call { - ($imm4:expr) => { - vcvtph2ps(a, zero, k, $imm4) - }; - } - let r = constify_imm4_sae!(sae, call); + let r = vcvtph2ps(a, zero, k, SAE); transmute(r) } @@ -15148,17 +14272,13 @@ pub unsafe fn _mm_maskz_cvtph_ps(k: __mmask8, a: __m128i) -> __m128 { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cvtt_roundps_epi32&expand=1916) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vcvttps2dq, sae = 8))] -#[rustc_args_required_const(1)] -pub unsafe fn _mm512_cvtt_roundps_epi32(a: __m512, sae: i32) -> __m512i { +#[cfg_attr(test, assert_instr(vcvttps2dq, SAE = 8))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm512_cvtt_roundps_epi32(a: __m512) -> __m512i { + static_assert_sae!(SAE); let a = a.as_f32x16(); let zero = _mm512_setzero_si512().as_i32x16(); - macro_rules! call { - ($imm4:expr) => { - vcvttps2dq(a, zero, 0b11111111_11111111, $imm4) - }; - } - let r = constify_imm4_sae!(sae, call); + let r = vcvttps2dq(a, zero, 0b11111111_11111111, SAE); transmute(r) } @@ -15168,22 +14288,17 @@ pub unsafe fn _mm512_cvtt_roundps_epi32(a: __m512, sae: i32) -> __m512i { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cvtt_roundps_epi32&expand=1917) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vcvttps2dq, sae = 8))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm512_mask_cvtt_roundps_epi32( +#[cfg_attr(test, assert_instr(vcvttps2dq, SAE = 8))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_mask_cvtt_roundps_epi32( src: __m512i, k: __mmask16, a: __m512, - sae: i32, ) -> __m512i { + static_assert_sae!(SAE); let a = a.as_f32x16(); let src = src.as_i32x16(); - macro_rules! call { - ($imm4:expr) => { - vcvttps2dq(a, src, k, $imm4) - }; - } - let r = constify_imm4_sae!(sae, call); + let r = vcvttps2dq(a, src, k, SAE); transmute(r) } @@ -15193,17 +14308,13 @@ pub unsafe fn _mm512_mask_cvtt_roundps_epi32( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_cvtt_roundps_epi32&expand=1918) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vcvttps2dq, sae = 8))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm512_maskz_cvtt_roundps_epi32(k: __mmask16, a: __m512, sae: i32) -> __m512i { +#[cfg_attr(test, assert_instr(vcvttps2dq, SAE = 8))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_maskz_cvtt_roundps_epi32(k: __mmask16, a: __m512) -> __m512i { + static_assert_sae!(SAE); let a = a.as_f32x16(); let zero = _mm512_setzero_si512().as_i32x16(); - macro_rules! call { - ($imm4:expr) => { - vcvttps2dq(a, zero, k, $imm4) - }; - } - let r = constify_imm4_sae!(sae, call); + let r = vcvttps2dq(a, zero, k, SAE); transmute(r) } @@ -15213,17 +14324,13 @@ pub unsafe fn _mm512_maskz_cvtt_roundps_epi32(k: __mmask16, a: __m512, sae: i32) /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cvtt_roundps_epu32&expand=1922) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vcvttps2udq, sae = 8))] -#[rustc_args_required_const(1)] -pub unsafe fn _mm512_cvtt_roundps_epu32(a: __m512, sae: i32) -> __m512i { +#[cfg_attr(test, assert_instr(vcvttps2udq, SAE = 8))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm512_cvtt_roundps_epu32(a: __m512) -> __m512i { + static_assert_sae!(SAE); let a = a.as_f32x16(); let zero = _mm512_setzero_si512().as_u32x16(); - macro_rules! call { - ($imm4:expr) => { - vcvttps2udq(a, zero, 0b11111111_11111111, $imm4) - }; - } - let r = constify_imm4_sae!(sae, call); + let r = vcvttps2udq(a, zero, 0b11111111_11111111, SAE); transmute(r) } @@ -15233,22 +14340,17 @@ pub unsafe fn _mm512_cvtt_roundps_epu32(a: __m512, sae: i32) -> __m512i { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cvtt_roundps_epu32&expand=1923) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vcvttps2udq, sae = 8))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm512_mask_cvtt_roundps_epu32( +#[cfg_attr(test, assert_instr(vcvttps2udq, SAE = 8))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_mask_cvtt_roundps_epu32( src: __m512i, k: __mmask16, a: __m512, - sae: i32, ) -> __m512i { + static_assert_sae!(SAE); let a = a.as_f32x16(); let src = src.as_u32x16(); - macro_rules! call { - ($imm4:expr) => { - vcvttps2udq(a, src, k, $imm4) - }; - } - let r = constify_imm4_sae!(sae, call); + let r = vcvttps2udq(a, src, k, SAE); transmute(r) } @@ -15258,17 +14360,13 @@ pub unsafe fn _mm512_mask_cvtt_roundps_epu32( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_cvtt_roundps_epu32&expand=1924) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vcvttps2udq, sae = 8))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm512_maskz_cvtt_roundps_epu32(k: __mmask16, a: __m512, sae: i32) -> __m512i { +#[cfg_attr(test, assert_instr(vcvttps2udq, SAE = 8))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_maskz_cvtt_roundps_epu32(k: __mmask16, a: __m512) -> __m512i { + static_assert_sae!(SAE); let a = a.as_f32x16(); let zero = _mm512_setzero_si512().as_u32x16(); - macro_rules! call { - ($imm4:expr) => { - vcvttps2udq(a, zero, k, $imm4) - }; - } - let r = constify_imm4_sae!(sae, call); + let r = vcvttps2udq(a, zero, k, SAE); transmute(r) } @@ -15278,17 +14376,13 @@ pub unsafe fn _mm512_maskz_cvtt_roundps_epu32(k: __mmask16, a: __m512, sae: i32) /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cvtt_roundpd_epi32&expand=1904) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vcvttpd2dq, sae = 8))] -#[rustc_args_required_const(1)] -pub unsafe fn _mm512_cvtt_roundpd_epi32(a: __m512d, sae: i32) -> __m256i { +#[cfg_attr(test, assert_instr(vcvttpd2dq, SAE = 8))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm512_cvtt_roundpd_epi32(a: __m512d) -> __m256i { + static_assert_sae!(SAE); let a = a.as_f64x8(); let zero = _mm256_setzero_si256().as_i32x8(); - macro_rules! call { - ($imm4:expr) => { - vcvttpd2dq(a, zero, 0b11111111, $imm4) - }; - } - let r = constify_imm4_sae!(sae, call); + let r = vcvttpd2dq(a, zero, 0b11111111, SAE); transmute(r) } @@ -15298,22 +14392,17 @@ pub unsafe fn _mm512_cvtt_roundpd_epi32(a: __m512d, sae: i32) -> __m256i { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cvtt_roundpd_epi32&expand=1905) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vcvttpd2dq, sae = 8))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm512_mask_cvtt_roundpd_epi32( +#[cfg_attr(test, assert_instr(vcvttpd2dq, SAE = 8))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_mask_cvtt_roundpd_epi32( src: __m256i, k: __mmask8, a: __m512d, - sae: i32, ) -> __m256i { + static_assert_sae!(SAE); let a = a.as_f64x8(); let src = src.as_i32x8(); - macro_rules! call { - ($imm4:expr) => { - vcvttpd2dq(a, src, k, $imm4) - }; - } - let r = constify_imm4_sae!(sae, call); + let r = vcvttpd2dq(a, src, k, SAE); transmute(r) } @@ -15323,17 +14412,13 @@ pub unsafe fn _mm512_mask_cvtt_roundpd_epi32( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_cvtt_roundps_epi32&expand=1918) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vcvttpd2dq, sae = 8))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm512_maskz_cvtt_roundpd_epi32(k: __mmask8, a: __m512d, sae: i32) -> __m256i { +#[cfg_attr(test, assert_instr(vcvttpd2dq, SAE = 8))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_maskz_cvtt_roundpd_epi32(k: __mmask8, a: __m512d) -> __m256i { + static_assert_sae!(SAE); let a = a.as_f64x8(); let zero = _mm256_setzero_si256().as_i32x8(); - macro_rules! call { - ($imm4:expr) => { - vcvttpd2dq(a, zero, k, $imm4) - }; - } - let r = constify_imm4_sae!(sae, call); + let r = vcvttpd2dq(a, zero, k, SAE); transmute(r) } @@ -15343,17 +14428,13 @@ pub unsafe fn _mm512_maskz_cvtt_roundpd_epi32(k: __mmask8, a: __m512d, sae: i32) /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cvtt_roundpd_epu32&expand=1910) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vcvttpd2udq, sae = 8))] -#[rustc_args_required_const(1)] -pub unsafe fn _mm512_cvtt_roundpd_epu32(a: __m512d, sae: i32) -> __m256i { +#[cfg_attr(test, assert_instr(vcvttpd2udq, SAE = 8))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm512_cvtt_roundpd_epu32(a: __m512d) -> __m256i { + static_assert_sae!(SAE); let a = a.as_f64x8(); let zero = _mm256_setzero_si256().as_i32x8(); - macro_rules! call { - ($imm4:expr) => { - vcvttpd2udq(a, zero, 0b11111111, $imm4) - }; - } - let r = constify_imm4_sae!(sae, call); + let r = vcvttpd2udq(a, zero, 0b11111111, SAE); transmute(r) } @@ -15363,22 +14444,17 @@ pub unsafe fn _mm512_cvtt_roundpd_epu32(a: __m512d, sae: i32) -> __m256i { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cvtt_roundpd_epu32&expand=1911) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vcvttpd2udq, sae = 8))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm512_mask_cvtt_roundpd_epu32( +#[cfg_attr(test, assert_instr(vcvttpd2udq, SAE = 8))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_mask_cvtt_roundpd_epu32( src: __m256i, k: __mmask8, a: __m512d, - sae: i32, ) -> __m256i { + static_assert_sae!(SAE); let a = a.as_f64x8(); let src = src.as_i32x8(); - macro_rules! call { - ($imm4:expr) => { - vcvttpd2udq(a, src, k, $imm4) - }; - } - let r = constify_imm4_sae!(sae, call); + let r = vcvttpd2udq(a, src, k, SAE); transmute(r) } @@ -15602,17 +14678,13 @@ pub unsafe fn _mm_maskz_cvttps_epu32(k: __mmask8, a: __m128) -> __m128i { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_cvtt_roundpd_epu32&expand=1912) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vcvttpd2udq, sae = 8))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm512_maskz_cvtt_roundpd_epu32(k: __mmask8, a: __m512d, sae: i32) -> __m256i { +#[cfg_attr(test, assert_instr(vcvttpd2udq, SAE = 8))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_maskz_cvtt_roundpd_epu32(k: __mmask8, a: __m512d) -> __m256i { + static_assert_sae!(SAE); let a = a.as_f64x8(); let zero = _mm256_setzero_si256().as_i32x8(); - macro_rules! call { - ($imm4:expr) => { - vcvttpd2udq(a, zero, k, $imm4) - }; - } - let r = constify_imm4_sae!(sae, call); + let r = vcvttpd2udq(a, zero, k, SAE); transmute(r) } @@ -16137,19 +15209,15 @@ pub unsafe fn _mm512_setr_epi64( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_i32gather_pd&expand=3002) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vgatherdpd, scale = 1))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm512_i32gather_pd(offsets: __m256i, slice: *const u8, scale: i32) -> __m512d { +#[cfg_attr(test, assert_instr(vgatherdpd, SCALE = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_i32gather_pd(offsets: __m256i, slice: *const u8) -> __m512d { + static_assert_imm8_scale!(SCALE); let zero = _mm512_setzero_pd().as_f64x8(); let neg_one = -1; let slice = slice as *const i8; let offsets = offsets.as_i32x8(); - macro_rules! call { - ($imm8:expr) => { - vgatherdpd(zero, slice, offsets, neg_one, $imm8) - }; - } - let r = constify_imm8_gather!(scale, call); + let r = vgatherdpd(zero, slice, offsets, neg_one, SCALE); transmute(r) } @@ -16158,24 +15226,19 @@ pub unsafe fn _mm512_i32gather_pd(offsets: __m256i, slice: *const u8, scale: i32 /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_i32gather_pd&expand=3003) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vgatherdpd, scale = 1))] -#[rustc_args_required_const(4)] -pub unsafe fn _mm512_mask_i32gather_pd( +#[cfg_attr(test, assert_instr(vgatherdpd, SCALE = 1))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_mask_i32gather_pd( src: __m512d, mask: __mmask8, offsets: __m256i, slice: *const u8, - scale: i32, ) -> __m512d { + static_assert_imm8_scale!(SCALE); let src = src.as_f64x8(); let slice = slice as *const i8; let offsets = offsets.as_i32x8(); - macro_rules! call { - ($imm8:expr) => { - vgatherdpd(src, slice, offsets, mask as i8, $imm8) - }; - } - let r = constify_imm8_gather!(scale, call); + let r = vgatherdpd(src, slice, offsets, mask as i8, SCALE); transmute(r) } @@ -16184,19 +15247,15 @@ pub unsafe fn _mm512_mask_i32gather_pd( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_i64gather_pd&expand=3092) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vgatherqpd, scale = 1))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm512_i64gather_pd(offsets: __m512i, slice: *const u8, scale: i32) -> __m512d { +#[cfg_attr(test, assert_instr(vgatherqpd, SCALE = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_i64gather_pd(offsets: __m512i, slice: *const u8) -> __m512d { + static_assert_imm8_scale!(SCALE); let zero = _mm512_setzero_pd().as_f64x8(); let neg_one = -1; let slice = slice as *const i8; let offsets = offsets.as_i64x8(); - macro_rules! call { - ($imm8:expr) => { - vgatherqpd(zero, slice, offsets, neg_one, $imm8) - }; - } - let r = constify_imm8_gather!(scale, call); + let r = vgatherqpd(zero, slice, offsets, neg_one, SCALE); transmute(r) } @@ -16205,24 +15264,19 @@ pub unsafe fn _mm512_i64gather_pd(offsets: __m512i, slice: *const u8, scale: i32 /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_i64gather_pd&expand=3093) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vgatherqpd, scale = 1))] -#[rustc_args_required_const(4)] -pub unsafe fn _mm512_mask_i64gather_pd( +#[cfg_attr(test, assert_instr(vgatherqpd, SCALE = 1))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_mask_i64gather_pd( src: __m512d, mask: __mmask8, offsets: __m512i, slice: *const u8, - scale: i32, ) -> __m512d { + static_assert_imm8_scale!(SCALE); let src = src.as_f64x8(); let slice = slice as *const i8; let offsets = offsets.as_i64x8(); - macro_rules! call { - ($imm8:expr) => { - vgatherqpd(src, slice, offsets, mask as i8, $imm8) - }; - } - let r = constify_imm8_gather!(scale, call); + let r = vgatherqpd(src, slice, offsets, mask as i8, SCALE); transmute(r) } @@ -16231,19 +15285,15 @@ pub unsafe fn _mm512_mask_i64gather_pd( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_i64gather_ps&expand=3100) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vgatherqps, scale = 1))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm512_i64gather_ps(offsets: __m512i, slice: *const u8, scale: i32) -> __m256 { +#[cfg_attr(test, assert_instr(vgatherqps, SCALE = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_i64gather_ps(offsets: __m512i, slice: *const u8) -> __m256 { + static_assert_imm8_scale!(SCALE); let zero = _mm256_setzero_ps().as_f32x8(); let neg_one = -1; let slice = slice as *const i8; let offsets = offsets.as_i64x8(); - macro_rules! call { - ($imm8:expr) => { - vgatherqps(zero, slice, offsets, neg_one, $imm8) - }; - } - let r = constify_imm8_gather!(scale, call); + let r = vgatherqps(zero, slice, offsets, neg_one, SCALE); transmute(r) } @@ -16252,24 +15302,19 @@ pub unsafe fn _mm512_i64gather_ps(offsets: __m512i, slice: *const u8, scale: i32 /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_i64gather_ps&expand=3101) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vgatherqps, scale = 1))] -#[rustc_args_required_const(4)] -pub unsafe fn _mm512_mask_i64gather_ps( +#[cfg_attr(test, assert_instr(vgatherqps, SCALE = 1))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_mask_i64gather_ps( src: __m256, mask: __mmask8, offsets: __m512i, slice: *const u8, - scale: i32, ) -> __m256 { + static_assert_imm8_scale!(SCALE); let src = src.as_f32x8(); let slice = slice as *const i8; let offsets = offsets.as_i64x8(); - macro_rules! call { - ($imm8:expr) => { - vgatherqps(src, slice, offsets, mask as i8, $imm8) - }; - } - let r = constify_imm8_gather!(scale, call); + let r = vgatherqps(src, slice, offsets, mask as i8, SCALE); transmute(r) } @@ -16278,19 +15323,15 @@ pub unsafe fn _mm512_mask_i64gather_ps( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_i32gather_ps&expand=3010) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vgatherdps, scale = 1))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm512_i32gather_ps(offsets: __m512i, slice: *const u8, scale: i32) -> __m512 { +#[cfg_attr(test, assert_instr(vgatherdps, SCALE = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_i32gather_ps(offsets: __m512i, slice: *const u8) -> __m512 { + static_assert_imm8_scale!(SCALE); let zero = _mm512_setzero_ps().as_f32x16(); let neg_one = -1; let slice = slice as *const i8; let offsets = offsets.as_i32x16(); - macro_rules! call { - ($imm8:expr) => { - vgatherdps(zero, slice, offsets, neg_one, $imm8) - }; - } - let r = constify_imm8_gather!(scale, call); + let r = vgatherdps(zero, slice, offsets, neg_one, SCALE); transmute(r) } @@ -16299,24 +15340,19 @@ pub unsafe fn _mm512_i32gather_ps(offsets: __m512i, slice: *const u8, scale: i32 /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_i32gather_ps&expand=3011) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vgatherdps, scale = 1))] -#[rustc_args_required_const(4)] -pub unsafe fn _mm512_mask_i32gather_ps( +#[cfg_attr(test, assert_instr(vgatherdps, SCALE = 1))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_mask_i32gather_ps( src: __m512, mask: __mmask16, offsets: __m512i, slice: *const u8, - scale: i32, ) -> __m512 { + static_assert_imm8_scale!(SCALE); let src = src.as_f32x16(); let slice = slice as *const i8; let offsets = offsets.as_i32x16(); - macro_rules! call { - ($imm8:expr) => { - vgatherdps(src, slice, offsets, mask as i16, $imm8) - }; - } - let r = constify_imm8_gather!(scale, call); + let r = vgatherdps(src, slice, offsets, mask as i16, SCALE); transmute(r) } @@ -16325,19 +15361,18 @@ pub unsafe fn _mm512_mask_i32gather_ps( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_i32gather_epi32&expand=2986) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vpgatherdd, scale = 1))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm512_i32gather_epi32(offsets: __m512i, slice: *const u8, scale: i32) -> __m512i { +#[cfg_attr(test, assert_instr(vpgatherdd, SCALE = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_i32gather_epi32( + offsets: __m512i, + slice: *const u8, +) -> __m512i { + static_assert_imm8_scale!(SCALE); let zero = _mm512_setzero_si512().as_i32x16(); let neg_one = -1; let slice = slice as *const i8; let offsets = offsets.as_i32x16(); - macro_rules! call { - ($imm8:expr) => { - vpgatherdd(zero, slice, offsets, neg_one, $imm8) - }; - } - let r = constify_imm8_gather!(scale, call); + let r = vpgatherdd(zero, slice, offsets, neg_one, SCALE); transmute(r) } @@ -16346,25 +15381,20 @@ pub unsafe fn _mm512_i32gather_epi32(offsets: __m512i, slice: *const u8, scale: /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_i32gather_epi32&expand=2987) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vpgatherdd, scale = 1))] -#[rustc_args_required_const(4)] -pub unsafe fn _mm512_mask_i32gather_epi32( +#[cfg_attr(test, assert_instr(vpgatherdd, SCALE = 1))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_mask_i32gather_epi32( src: __m512i, mask: __mmask16, offsets: __m512i, slice: *const u8, - scale: i32, ) -> __m512i { + static_assert_imm8_scale!(SCALE); let src = src.as_i32x16(); let mask = mask as i16; let slice = slice as *const i8; let offsets = offsets.as_i32x16(); - macro_rules! call { - ($imm8:expr) => { - vpgatherdd(src, slice, offsets, mask, $imm8) - }; - } - let r = constify_imm8_gather!(scale, call); + let r = vpgatherdd(src, slice, offsets, mask, SCALE); transmute(r) } @@ -16373,19 +15403,18 @@ pub unsafe fn _mm512_mask_i32gather_epi32( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_i32gather_epi64&expand=2994) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vpgatherdq, scale = 1))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm512_i32gather_epi64(offsets: __m256i, slice: *const u8, scale: i32) -> __m512i { +#[cfg_attr(test, assert_instr(vpgatherdq, SCALE = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_i32gather_epi64( + offsets: __m256i, + slice: *const u8, +) -> __m512i { + static_assert_imm8_scale!(SCALE); let zero = _mm512_setzero_si512().as_i64x8(); let neg_one = -1; let slice = slice as *const i8; let offsets = offsets.as_i32x8(); - macro_rules! call { - ($imm8:expr) => { - vpgatherdq(zero, slice, offsets, neg_one, $imm8) - }; - } - let r = constify_imm8_gather!(scale, call); + let r = vpgatherdq(zero, slice, offsets, neg_one, SCALE); transmute(r) } @@ -16394,25 +15423,20 @@ pub unsafe fn _mm512_i32gather_epi64(offsets: __m256i, slice: *const u8, scale: /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_i32gather_epi64&expand=2995) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vpgatherdq, scale = 1))] -#[rustc_args_required_const(4)] -pub unsafe fn _mm512_mask_i32gather_epi64( +#[cfg_attr(test, assert_instr(vpgatherdq, SCALE = 1))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_mask_i32gather_epi64( src: __m512i, mask: __mmask8, offsets: __m256i, slice: *const u8, - scale: i32, ) -> __m512i { + static_assert_imm8_scale!(SCALE); let src = src.as_i64x8(); let mask = mask as i8; let slice = slice as *const i8; let offsets = offsets.as_i32x8(); - macro_rules! call { - ($imm8:expr) => { - vpgatherdq(src, slice, offsets, mask, $imm8) - }; - } - let r = constify_imm8_gather!(scale, call); + let r = vpgatherdq(src, slice, offsets, mask, SCALE); transmute(r) } @@ -16421,19 +15445,18 @@ pub unsafe fn _mm512_mask_i32gather_epi64( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_i64gather_epi64&expand=3084) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vpgatherqq, scale = 1))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm512_i64gather_epi64(offsets: __m512i, slice: *const u8, scale: i32) -> __m512i { +#[cfg_attr(test, assert_instr(vpgatherqq, SCALE = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_i64gather_epi64( + offsets: __m512i, + slice: *const u8, +) -> __m512i { + static_assert_imm8_scale!(SCALE); let zero = _mm512_setzero_si512().as_i64x8(); let neg_one = -1; let slice = slice as *const i8; let offsets = offsets.as_i64x8(); - macro_rules! call { - ($imm8:expr) => { - vpgatherqq(zero, slice, offsets, neg_one, $imm8) - }; - } - let r = constify_imm8_gather!(scale, call); + let r = vpgatherqq(zero, slice, offsets, neg_one, SCALE); transmute(r) } @@ -16442,25 +15465,20 @@ pub unsafe fn _mm512_i64gather_epi64(offsets: __m512i, slice: *const u8, scale: /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_i64gather_epi64&expand=3085) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vpgatherqq, scale = 1))] -#[rustc_args_required_const(4)] -pub unsafe fn _mm512_mask_i64gather_epi64( +#[cfg_attr(test, assert_instr(vpgatherqq, SCALE = 1))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_mask_i64gather_epi64( src: __m512i, mask: __mmask8, offsets: __m512i, slice: *const u8, - scale: i32, ) -> __m512i { + static_assert_imm8_scale!(SCALE); let src = src.as_i64x8(); let mask = mask as i8; let slice = slice as *const i8; let offsets = offsets.as_i64x8(); - macro_rules! call { - ($imm8:expr) => { - vpgatherqq(src, slice, offsets, mask, $imm8) - }; - } - let r = constify_imm8_gather!(scale, call); + let r = vpgatherqq(src, slice, offsets, mask, SCALE); transmute(r) } @@ -16469,19 +15487,18 @@ pub unsafe fn _mm512_mask_i64gather_epi64( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_i64gather_epi32&expand=3074) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vpgatherqd, scale = 1))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm512_i64gather_epi32(offsets: __m512i, slice: *const u8, scale: i32) -> __m256i { +#[cfg_attr(test, assert_instr(vpgatherqd, SCALE = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_i64gather_epi32( + offsets: __m512i, + slice: *const u8, +) -> __m256i { + static_assert_imm8_scale!(SCALE); let zeros = _mm256_setzero_si256().as_i32x8(); let neg_one = -1; let slice = slice as *const i8; let offsets = offsets.as_i64x8(); - macro_rules! call { - ($imm8:expr) => { - vpgatherqd(zeros, slice, offsets, neg_one, $imm8) - }; - } - let r = constify_imm8_gather!(scale, call); + let r = vpgatherqd(zeros, slice, offsets, neg_one, SCALE); transmute(r) } @@ -16490,25 +15507,20 @@ pub unsafe fn _mm512_i64gather_epi32(offsets: __m512i, slice: *const u8, scale: /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_i64gather_epi32&expand=3075) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vpgatherqd, scale = 1))] -#[rustc_args_required_const(4)] -pub unsafe fn _mm512_mask_i64gather_epi32( +#[cfg_attr(test, assert_instr(vpgatherqd, SCALE = 1))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_mask_i64gather_epi32( src: __m256i, mask: __mmask8, offsets: __m512i, slice: *const u8, - scale: i32, ) -> __m256i { + static_assert_imm8_scale!(SCALE); let src = src.as_i32x8(); let mask = mask as i8; let slice = slice as *const i8; let offsets = offsets.as_i64x8(); - macro_rules! call { - ($imm8:expr) => { - vpgatherqd(src, slice, offsets, mask, $imm8) - }; - } - let r = constify_imm8_gather!(scale, call); + let r = vpgatherqd(src, slice, offsets, mask, SCALE); transmute(r) } @@ -16517,19 +15529,19 @@ pub unsafe fn _mm512_mask_i64gather_epi32( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_i32scatter_pd&expand=3044) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vscatterdpd, scale = 1))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm512_i32scatter_pd(slice: *mut u8, offsets: __m256i, src: __m512d, scale: i32) { +#[cfg_attr(test, assert_instr(vscatterdpd, SCALE = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_i32scatter_pd( + slice: *mut u8, + offsets: __m256i, + src: __m512d, +) { + static_assert_imm8_scale!(SCALE); let src = src.as_f64x8(); let neg_one = -1; let slice = slice as *mut i8; let offsets = offsets.as_i32x8(); - macro_rules! call { - ($imm8:expr) => { - vscatterdpd(slice, neg_one, offsets, src, $imm8) - }; - } - constify_imm8_gather!(scale, call); + vscatterdpd(slice, neg_one, offsets, src, SCALE); } /// Scatter double-precision (64-bit) floating-point elements from a into memory using 32-bit indices. 64-bit elements are stored at addresses starting at base_addr and offset by each 32-bit element in vindex (each index is scaled by the factor in scale) subject to mask k (elements are not stored when the corresponding mask bit is not set). scale should be 1, 2, 4 or 8. @@ -16537,24 +15549,19 @@ pub unsafe fn _mm512_i32scatter_pd(slice: *mut u8, offsets: __m256i, src: __m512 /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_i32scatter_pd&expand=3045) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vscatterdpd, scale = 1))] -#[rustc_args_required_const(4)] -pub unsafe fn _mm512_mask_i32scatter_pd( +#[cfg_attr(test, assert_instr(vscatterdpd, SCALE = 1))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_mask_i32scatter_pd( slice: *mut u8, mask: __mmask8, offsets: __m256i, src: __m512d, - scale: i32, ) { + static_assert_imm8_scale!(SCALE); let src = src.as_f64x8(); let slice = slice as *mut i8; let offsets = offsets.as_i32x8(); - macro_rules! call { - ($imm8:expr) => { - vscatterdpd(slice, mask as i8, offsets, src, $imm8) - }; - } - constify_imm8_gather!(scale, call); + vscatterdpd(slice, mask as i8, offsets, src, SCALE); } /// Scatter double-precision (64-bit) floating-point elements from a into memory using 64-bit indices. 64-bit elements are stored at addresses starting at base_addr and offset by each 64-bit element in vindex (each index is scaled by the factor in scale). scale should be 1, 2, 4 or 8. @@ -16562,19 +15569,19 @@ pub unsafe fn _mm512_mask_i32scatter_pd( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_i64scatter_pd&expand=3122) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vscatterqpd, scale = 1))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm512_i64scatter_pd(slice: *mut u8, offsets: __m512i, src: __m512d, scale: i32) { +#[cfg_attr(test, assert_instr(vscatterqpd, SCALE = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_i64scatter_pd( + slice: *mut u8, + offsets: __m512i, + src: __m512d, +) { + static_assert_imm8_scale!(SCALE); let src = src.as_f64x8(); let neg_one = -1; let slice = slice as *mut i8; let offsets = offsets.as_i64x8(); - macro_rules! call { - ($imm8:expr) => { - vscatterqpd(slice, neg_one, offsets, src, $imm8) - }; - } - constify_imm8_gather!(scale, call); + vscatterqpd(slice, neg_one, offsets, src, SCALE); } /// Scatter double-precision (64-bit) floating-point elements from a into memory using 64-bit indices. 64-bit elements are stored at addresses starting at base_addr and offset by each 64-bit element in vindex (each index is scaled by the factor in scale) subject to mask k (elements are not stored when the corresponding mask bit is not set). scale should be 1, 2, 4 or 8. @@ -16582,24 +15589,19 @@ pub unsafe fn _mm512_i64scatter_pd(slice: *mut u8, offsets: __m512i, src: __m512 /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_i64scatter_pd&expand=3123) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vscatterqpd, scale = 1))] -#[rustc_args_required_const(4)] -pub unsafe fn _mm512_mask_i64scatter_pd( +#[cfg_attr(test, assert_instr(vscatterqpd, SCALE = 1))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_mask_i64scatter_pd( slice: *mut u8, mask: __mmask8, offsets: __m512i, src: __m512d, - scale: i32, ) { + static_assert_imm8_scale!(SCALE); let src = src.as_f64x8(); let slice = slice as *mut i8; let offsets = offsets.as_i64x8(); - macro_rules! call { - ($imm8:expr) => { - vscatterqpd(slice, mask as i8, offsets, src, $imm8) - }; - } - constify_imm8_gather!(scale, call); + vscatterqpd(slice, mask as i8, offsets, src, SCALE); } /// Scatter single-precision (32-bit) floating-point elements from a into memory using 32-bit indices. 32-bit elements are stored at addresses starting at base_addr and offset by each 32-bit element in vindex (each index is scaled by the factor in scale). scale should be 1, 2, 4 or 8. @@ -16607,19 +15609,19 @@ pub unsafe fn _mm512_mask_i64scatter_pd( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_i32scatter_ps&expand=3050) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vscatterdps, scale = 1))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm512_i32scatter_ps(slice: *mut u8, offsets: __m512i, src: __m512, scale: i32) { +#[cfg_attr(test, assert_instr(vscatterdps, SCALE = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_i32scatter_ps( + slice: *mut u8, + offsets: __m512i, + src: __m512, +) { + static_assert_imm8_scale!(SCALE); let src = src.as_f32x16(); let neg_one = -1; let slice = slice as *mut i8; let offsets = offsets.as_i32x16(); - macro_rules! call { - ($imm8:expr) => { - vscatterdps(slice, neg_one, offsets, src, $imm8) - }; - } - constify_imm8_gather!(scale, call); + vscatterdps(slice, neg_one, offsets, src, SCALE); } /// Scatter single-precision (32-bit) floating-point elements from a into memory using 32-bit indices. 32-bit elements are stored at addresses starting at base_addr and offset by each 32-bit element in vindex (each index is scaled by the factor in scale) subject to mask k (elements are not stored when the corresponding mask bit is not set). scale should be 1, 2, 4 or 8. @@ -16627,24 +15629,19 @@ pub unsafe fn _mm512_i32scatter_ps(slice: *mut u8, offsets: __m512i, src: __m512 /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_i32scatter_ps&expand=3051) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vscatterdps, scale = 1))] -#[rustc_args_required_const(4)] -pub unsafe fn _mm512_mask_i32scatter_ps( +#[cfg_attr(test, assert_instr(vscatterdps, SCALE = 1))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_mask_i32scatter_ps( slice: *mut u8, mask: __mmask16, offsets: __m512i, src: __m512, - scale: i32, ) { + static_assert_imm8_scale!(SCALE); let src = src.as_f32x16(); let slice = slice as *mut i8; let offsets = offsets.as_i32x16(); - macro_rules! call { - ($imm8:expr) => { - vscatterdps(slice, mask as i16, offsets, src, $imm8) - }; - } - constify_imm8_gather!(scale, call); + vscatterdps(slice, mask as i16, offsets, src, SCALE); } /// Scatter single-precision (32-bit) floating-point elements from a into memory using 64-bit indices. 32-bit elements are stored at addresses starting at base_addr and offset by each 64-bit element in vindex (each index is scaled by the factor in scale) subject to mask k (elements are not stored when the corresponding mask bit is not set). scale should be 1, 2, 4 or 8. @@ -16652,19 +15649,19 @@ pub unsafe fn _mm512_mask_i32scatter_ps( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_i64scatter_ps&expand=3128) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vscatterqps, scale = 1))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm512_i64scatter_ps(slice: *mut u8, offsets: __m512i, src: __m256, scale: i32) { +#[cfg_attr(test, assert_instr(vscatterqps, SCALE = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_i64scatter_ps( + slice: *mut u8, + offsets: __m512i, + src: __m256, +) { + static_assert_imm8_scale!(SCALE); let src = src.as_f32x8(); let neg_one = -1; let slice = slice as *mut i8; let offsets = offsets.as_i64x8(); - macro_rules! call { - ($imm8:expr) => { - vscatterqps(slice, neg_one, offsets, src, $imm8) - }; - } - constify_imm8_gather!(scale, call); + vscatterqps(slice, neg_one, offsets, src, SCALE); } /// Scatter single-precision (32-bit) floating-point elements from a into memory using 64-bit indices. 32-bit elements are stored at addresses starting at base_addr and offset by each 64-bit element in vindex (each index is scaled by the factor in scale) subject to mask k (elements are not stored when the corresponding mask bit is not set). scale should be 1, 2, 4 or 8. @@ -16672,24 +15669,19 @@ pub unsafe fn _mm512_i64scatter_ps(slice: *mut u8, offsets: __m512i, src: __m256 /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_i64scatter_ps&expand=3129) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vscatterqps, scale = 1))] -#[rustc_args_required_const(4)] -pub unsafe fn _mm512_mask_i64scatter_ps( +#[cfg_attr(test, assert_instr(vscatterqps, SCALE = 1))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_mask_i64scatter_ps( slice: *mut u8, mask: __mmask8, offsets: __m512i, src: __m256, - scale: i32, ) { + static_assert_imm8_scale!(SCALE); let src = src.as_f32x8(); let slice = slice as *mut i8; let offsets = offsets.as_i64x8(); - macro_rules! call { - ($imm8:expr) => { - vscatterqps(slice, mask as i8, offsets, src, $imm8) - }; - } - constify_imm8_gather!(scale, call); + vscatterqps(slice, mask as i8, offsets, src, SCALE); } /// Scatter 64-bit integers from a into memory using 32-bit indices. 64-bit elements are stored at addresses starting at base_addr and offset by each 32-bit element in vindex (each index is scaled by the factor in scale). scale should be 1, 2, 4 or 8. @@ -16697,19 +15689,19 @@ pub unsafe fn _mm512_mask_i64scatter_ps( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_i32scatter_epi64&expand=3038) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vpscatterdq, scale = 1))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm512_i32scatter_epi64(slice: *mut u8, offsets: __m256i, src: __m512i, scale: i32) { +#[cfg_attr(test, assert_instr(vpscatterdq, SCALE = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_i32scatter_epi64( + slice: *mut u8, + offsets: __m256i, + src: __m512i, +) { + static_assert_imm8_scale!(SCALE); let src = src.as_i64x8(); let neg_one = -1; let slice = slice as *mut i8; let offsets = offsets.as_i32x8(); - macro_rules! call { - ($imm8:expr) => { - vpscatterdq(slice, neg_one, offsets, src, $imm8) - }; - } - constify_imm8_gather!(scale, call); + vpscatterdq(slice, neg_one, offsets, src, SCALE); } /// Scatter 64-bit integers from a into memory using 32-bit indices. 64-bit elements are stored at addresses starting at base_addr and offset by each 32-bit element in vindex (each index is scaled by the factor in scale) subject to mask k (elements are not stored when the corresponding mask bit is not set). scale should be 1, 2, 4 or 8. @@ -16717,25 +15709,20 @@ pub unsafe fn _mm512_i32scatter_epi64(slice: *mut u8, offsets: __m256i, src: __m /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_i32scatter_epi64&expand=3039) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vpscatterdq, scale = 1))] -#[rustc_args_required_const(4)] -pub unsafe fn _mm512_mask_i32scatter_epi64( +#[cfg_attr(test, assert_instr(vpscatterdq, SCALE = 1))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_mask_i32scatter_epi64( slice: *mut u8, mask: __mmask8, offsets: __m256i, src: __m512i, - scale: i32, ) { + static_assert_imm8_scale!(SCALE); let src = src.as_i64x8(); let mask = mask as i8; let slice = slice as *mut i8; let offsets = offsets.as_i32x8(); - macro_rules! call { - ($imm8:expr) => { - vpscatterdq(slice, mask, offsets, src, $imm8) - }; - } - constify_imm8_gather!(scale, call); + vpscatterdq(slice, mask, offsets, src, SCALE); } /// Scatter 64-bit integers from a into memory using 64-bit indices. 64-bit elements are stored at addresses starting at base_addr and offset by each 64-bit element in vindex (each index is scaled by the factor in scale). scale should be 1, 2, 4 or 8. @@ -16743,19 +15730,19 @@ pub unsafe fn _mm512_mask_i32scatter_epi64( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_i64scatter_epi64&expand=3116) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vpscatterqq, scale = 1))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm512_i64scatter_epi64(slice: *mut u8, offsets: __m512i, src: __m512i, scale: i32) { +#[cfg_attr(test, assert_instr(vpscatterqq, SCALE = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_i64scatter_epi64( + slice: *mut u8, + offsets: __m512i, + src: __m512i, +) { + static_assert_imm8_scale!(SCALE); let src = src.as_i64x8(); let neg_one = -1; let slice = slice as *mut i8; let offsets = offsets.as_i64x8(); - macro_rules! call { - ($imm8:expr) => { - vpscatterqq(slice, neg_one, offsets, src, $imm8) - }; - } - constify_imm8_gather!(scale, call); + vpscatterqq(slice, neg_one, offsets, src, SCALE); } /// Scatter 64-bit integers from a into memory using 64-bit indices. 64-bit elements are stored at addresses starting at base_addr and offset by each 64-bit element in vindex (each index is scaled by the factor in scale) subject to mask k (elements are not stored when the corresponding mask bit is not set). scale should be 1, 2, 4 or 8. @@ -16763,25 +15750,20 @@ pub unsafe fn _mm512_i64scatter_epi64(slice: *mut u8, offsets: __m512i, src: __m /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_i64scatter_epi64&expand=3117) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vpscatterqq, scale = 1))] -#[rustc_args_required_const(4)] -pub unsafe fn _mm512_mask_i64scatter_epi64( +#[cfg_attr(test, assert_instr(vpscatterqq, SCALE = 1))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_mask_i64scatter_epi64( slice: *mut u8, mask: __mmask8, offsets: __m512i, src: __m512i, - scale: i32, ) { + static_assert_imm8_scale!(SCALE); let src = src.as_i64x8(); let mask = mask as i8; let slice = slice as *mut i8; let offsets = offsets.as_i64x8(); - macro_rules! call { - ($imm8:expr) => { - vpscatterqq(slice, mask, offsets, src, $imm8) - }; - } - constify_imm8_gather!(scale, call); + vpscatterqq(slice, mask, offsets, src, SCALE); } /// Scatter 32-bit integers from a into memory using 32-bit indices. 32-bit elements are stored at addresses starting at base_addr and offset by each 32-bit element in vindex (each index is scaled by the factor in scale). scale should be 1, 2, 4 or 8. @@ -16789,19 +15771,19 @@ pub unsafe fn _mm512_mask_i64scatter_epi64( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_i32scatter_epi32&expand=3032) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vpscatterdd, scale = 1))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm512_i32scatter_epi32(slice: *mut u8, offsets: __m512i, src: __m512i, scale: i32) { +#[cfg_attr(test, assert_instr(vpscatterdd, SCALE = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_i32scatter_epi32( + slice: *mut u8, + offsets: __m512i, + src: __m512i, +) { + static_assert_imm8_scale!(SCALE); let src = src.as_i32x16(); let neg_one = -1; let slice = slice as *mut i8; let offsets = offsets.as_i32x16(); - macro_rules! call { - ($imm8:expr) => { - vpscatterdd(slice, neg_one, offsets, src, $imm8) - }; - } - constify_imm8_gather!(scale, call); + vpscatterdd(slice, neg_one, offsets, src, SCALE); } /// Scatter 32-bit integers from a into memory using 32-bit indices. 32-bit elements are stored at addresses starting at base_addr and offset by each 32-bit element in vindex (each index is scaled by the factor in scale) subject to mask k (elements are not stored when the corresponding mask bit is not set). scale should be 1, 2, 4 or 8. @@ -16809,25 +15791,20 @@ pub unsafe fn _mm512_i32scatter_epi32(slice: *mut u8, offsets: __m512i, src: __m /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_i32scatter_epi32&expand=3033) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vpscatterdd, scale = 1))] -#[rustc_args_required_const(4)] -pub unsafe fn _mm512_mask_i32scatter_epi32( +#[cfg_attr(test, assert_instr(vpscatterdd, SCALE = 1))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_mask_i32scatter_epi32( slice: *mut u8, mask: __mmask16, offsets: __m512i, src: __m512i, - scale: i32, ) { + static_assert_imm8_scale!(SCALE); let src = src.as_i32x16(); let mask = mask as i16; let slice = slice as *mut i8; let offsets = offsets.as_i32x16(); - macro_rules! call { - ($imm8:expr) => { - vpscatterdd(slice, mask, offsets, src, $imm8) - }; - } - constify_imm8_gather!(scale, call); + vpscatterdd(slice, mask, offsets, src, SCALE); } /// Scatter 32-bit integers from a into memory using 64-bit indices. 32-bit elements are stored at addresses starting at base_addr and offset by each 64-bit element in vindex (each index is scaled by the factor in scale). scale should be 1, 2, 4 or 8. @@ -16835,19 +15812,19 @@ pub unsafe fn _mm512_mask_i32scatter_epi32( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_i64scatter_epi32&expand=3108) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vpscatterqd, scale = 1))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm512_i64scatter_epi32(slice: *mut u8, offsets: __m512i, src: __m256i, scale: i32) { +#[cfg_attr(test, assert_instr(vpscatterqd, SCALE = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_i64scatter_epi32( + slice: *mut u8, + offsets: __m512i, + src: __m256i, +) { + static_assert_imm8_scale!(SCALE); let src = src.as_i32x8(); let neg_one = -1; let slice = slice as *mut i8; let offsets = offsets.as_i64x8(); - macro_rules! call { - ($imm8:expr) => { - vpscatterqd(slice, neg_one, offsets, src, $imm8) - }; - } - constify_imm8_gather!(scale, call); + vpscatterqd(slice, neg_one, offsets, src, SCALE); } /// Scatter 32-bit integers from a into memory using 64-bit indices. 32-bit elements are stored at addresses starting at base_addr and offset by each 64-bit element in vindex (each index is scaled by the factor in scale) subject to mask k (elements are not stored when the corresponding mask bit is not set). scale should be 1, 2, 4 or 8. @@ -16855,25 +15832,20 @@ pub unsafe fn _mm512_i64scatter_epi32(slice: *mut u8, offsets: __m512i, src: __m /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_i64scatter_epi32&expand=3109) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vpscatterqd, scale = 1))] -#[rustc_args_required_const(4)] -pub unsafe fn _mm512_mask_i64scatter_epi32( +#[cfg_attr(test, assert_instr(vpscatterqd, SCALE = 1))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_mask_i64scatter_epi32( slice: *mut u8, mask: __mmask8, offsets: __m512i, src: __m256i, - scale: i32, ) { + static_assert_imm8_scale!(SCALE); let src = src.as_i32x8(); let mask = mask as i8; let slice = slice as *mut i8; let offsets = offsets.as_i64x8(); - macro_rules! call { - ($imm8:expr) => { - vpscatterqd(slice, mask, offsets, src, $imm8) - }; - } - constify_imm8_gather!(scale, call); + vpscatterqd(slice, mask, offsets, src, SCALE); } /// Contiguously store the active 32-bit integers in a (those with their respective bit set in writemask k) to dst, and pass through the remaining elements from src. @@ -17429,16 +16401,12 @@ pub unsafe fn _mm_maskz_expand_pd(k: __mmask8, a: __m128d) -> __m128d { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_rol_epi32&expand=4685) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vprold, imm8 = 1))] -#[rustc_args_required_const(1)] -pub unsafe fn _mm512_rol_epi32(a: __m512i, imm8: i32) -> __m512i { +#[cfg_attr(test, assert_instr(vprold, IMM8 = 1))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm512_rol_epi32(a: __m512i) -> __m512i { + static_assert_imm8!(IMM8); let a = a.as_i32x16(); - macro_rules! call { - ($imm8:expr) => { - vprold(a, $imm8) - }; - } - let r = constify_imm8_sae!(imm8, call); + let r = vprold(a, IMM8); transmute(r) } @@ -17447,17 +16415,17 @@ pub unsafe fn _mm512_rol_epi32(a: __m512i, imm8: i32) -> __m512i { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_rol_epi32&expand=4683) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vprold, imm8 = 1))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm512_mask_rol_epi32(src: __m512i, k: __mmask16, a: __m512i, imm8: i32) -> __m512i { +#[cfg_attr(test, assert_instr(vprold, IMM8 = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_mask_rol_epi32( + src: __m512i, + k: __mmask16, + a: __m512i, +) -> __m512i { + static_assert_imm8!(IMM8); let a = a.as_i32x16(); - macro_rules! call { - ($imm8:expr) => { - vprold(a, $imm8) - }; - } - let rol = constify_imm8_sae!(imm8, call); - transmute(simd_select_bitmask(k, rol, src.as_i32x16())) + let r = vprold(a, IMM8); + transmute(simd_select_bitmask(k, r, src.as_i32x16())) } /// Rotate the bits in each packed 32-bit integer in a to the left by the number of bits specified in imm8, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). @@ -17465,18 +16433,14 @@ pub unsafe fn _mm512_mask_rol_epi32(src: __m512i, k: __mmask16, a: __m512i, imm8 /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_rol_epi32&expand=4684) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vprold, imm8 = 1))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm512_maskz_rol_epi32(k: __mmask16, a: __m512i, imm8: i32) -> __m512i { +#[cfg_attr(test, assert_instr(vprold, IMM8 = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_maskz_rol_epi32(k: __mmask16, a: __m512i) -> __m512i { + static_assert_imm8!(IMM8); let a = a.as_i32x16(); - macro_rules! call { - ($imm8:expr) => { - vprold(a, $imm8) - }; - } - let rol = constify_imm8_sae!(imm8, call); + let r = vprold(a, IMM8); let zero = _mm512_setzero_si512().as_i32x16(); - transmute(simd_select_bitmask(k, rol, zero)) + transmute(simd_select_bitmask(k, r, zero)) } /// Rotate the bits in each packed 32-bit integer in a to the left by the number of bits specified in imm8, and store the results in dst. @@ -17484,16 +16448,12 @@ pub unsafe fn _mm512_maskz_rol_epi32(k: __mmask16, a: __m512i, imm8: i32) -> __m /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_rol_epi32&expand=4682) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[cfg_attr(test, assert_instr(vprold, imm8 = 1))] -#[rustc_args_required_const(1)] -pub unsafe fn _mm256_rol_epi32(a: __m256i, imm8: i32) -> __m256i { +#[cfg_attr(test, assert_instr(vprold, IMM8 = 1))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm256_rol_epi32(a: __m256i) -> __m256i { + static_assert_imm8!(IMM8); let a = a.as_i32x8(); - macro_rules! call { - ($imm8:expr) => { - vprold256(a, $imm8) - }; - } - let r = constify_imm8_sae!(imm8, call); + let r = vprold256(a, IMM8); transmute(r) } @@ -17502,17 +16462,17 @@ pub unsafe fn _mm256_rol_epi32(a: __m256i, imm8: i32) -> __m256i { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_rol_epi32&expand=4680) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[cfg_attr(test, assert_instr(vprold, imm8 = 1))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm256_mask_rol_epi32(src: __m256i, k: __mmask8, a: __m256i, imm8: i32) -> __m256i { +#[cfg_attr(test, assert_instr(vprold, IMM8 = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm256_mask_rol_epi32( + src: __m256i, + k: __mmask8, + a: __m256i, +) -> __m256i { + static_assert_imm8!(IMM8); let a = a.as_i32x8(); - macro_rules! call { - ($imm8:expr) => { - vprold256(a, $imm8) - }; - } - let rol = constify_imm8_sae!(imm8, call); - transmute(simd_select_bitmask(k, rol, src.as_i32x8())) + let r = vprold256(a, IMM8); + transmute(simd_select_bitmask(k, r, src.as_i32x8())) } /// Rotate the bits in each packed 32-bit integer in a to the left by the number of bits specified in imm8, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). @@ -17520,18 +16480,14 @@ pub unsafe fn _mm256_mask_rol_epi32(src: __m256i, k: __mmask8, a: __m256i, imm8: /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_rol_epi32&expand=4681) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[cfg_attr(test, assert_instr(vprold, imm8 = 1))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm256_maskz_rol_epi32(k: __mmask8, a: __m256i, imm8: i32) -> __m256i { +#[cfg_attr(test, assert_instr(vprold, IMM8 = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm256_maskz_rol_epi32(k: __mmask8, a: __m256i) -> __m256i { + static_assert_imm8!(IMM8); let a = a.as_i32x8(); - macro_rules! call { - ($imm8:expr) => { - vprold256(a, $imm8) - }; - } - let rol = constify_imm8_sae!(imm8, call); + let r = vprold256(a, IMM8); let zero = _mm256_setzero_si256().as_i32x8(); - transmute(simd_select_bitmask(k, rol, zero)) + transmute(simd_select_bitmask(k, r, zero)) } /// Rotate the bits in each packed 32-bit integer in a to the left by the number of bits specified in imm8, and store the results in dst. @@ -17539,16 +16495,12 @@ pub unsafe fn _mm256_maskz_rol_epi32(k: __mmask8, a: __m256i, imm8: i32) -> __m2 /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_rol_epi32&expand=4679) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[cfg_attr(test, assert_instr(vprold, imm8 = 1))] -#[rustc_args_required_const(1)] -pub unsafe fn _mm_rol_epi32(a: __m128i, imm8: i32) -> __m128i { +#[cfg_attr(test, assert_instr(vprold, IMM8 = 1))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm_rol_epi32(a: __m128i) -> __m128i { + static_assert_imm8!(IMM8); let a = a.as_i32x4(); - macro_rules! call { - ($imm8:expr) => { - vprold128(a, $imm8) - }; - } - let r = constify_imm8_sae!(imm8, call); + let r = vprold128(a, IMM8); transmute(r) } @@ -17557,17 +16509,17 @@ pub unsafe fn _mm_rol_epi32(a: __m128i, imm8: i32) -> __m128i { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_rol_epi32&expand=4677) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[cfg_attr(test, assert_instr(vprold, imm8 = 1))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm_mask_rol_epi32(src: __m128i, k: __mmask8, a: __m128i, imm8: i32) -> __m128i { +#[cfg_attr(test, assert_instr(vprold, IMM8 = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm_mask_rol_epi32( + src: __m128i, + k: __mmask8, + a: __m128i, +) -> __m128i { + static_assert_imm8!(IMM8); let a = a.as_i32x4(); - macro_rules! call { - ($imm8:expr) => { - vprold128(a, $imm8) - }; - } - let rol = constify_imm8_sae!(imm8, call); - transmute(simd_select_bitmask(k, rol, src.as_i32x4())) + let r = vprold128(a, IMM8); + transmute(simd_select_bitmask(k, r, src.as_i32x4())) } /// Rotate the bits in each packed 32-bit integer in a to the left by the number of bits specified in imm8, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). @@ -17575,18 +16527,14 @@ pub unsafe fn _mm_mask_rol_epi32(src: __m128i, k: __mmask8, a: __m128i, imm8: i3 /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_rol_epi32&expand=4678) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[cfg_attr(test, assert_instr(vprold, imm8 = 1))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm_maskz_rol_epi32(k: __mmask8, a: __m128i, imm8: i32) -> __m128i { +#[cfg_attr(test, assert_instr(vprold, IMM8 = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm_maskz_rol_epi32(k: __mmask8, a: __m128i) -> __m128i { + static_assert_imm8!(IMM8); let a = a.as_i32x4(); - macro_rules! call { - ($imm8:expr) => { - vprold128(a, $imm8) - }; - } - let rol = constify_imm8_sae!(imm8, call); + let r = vprold128(a, IMM8); let zero = _mm_setzero_si128().as_i32x4(); - transmute(simd_select_bitmask(k, rol, zero)) + transmute(simd_select_bitmask(k, r, zero)) } /// Rotate the bits in each packed 32-bit integer in a to the right by the number of bits specified in imm8, and store the results in dst. @@ -17594,16 +16542,12 @@ pub unsafe fn _mm_maskz_rol_epi32(k: __mmask8, a: __m128i, imm8: i32) -> __m128i /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_ror_epi32&expand=4721) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vprold, imm8 = 1))] -#[rustc_args_required_const(1)] -pub unsafe fn _mm512_ror_epi32(a: __m512i, imm8: i32) -> __m512i { +#[cfg_attr(test, assert_instr(vprold, IMM8 = 1))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm512_ror_epi32(a: __m512i) -> __m512i { + static_assert_imm8!(IMM8); let a = a.as_i32x16(); - macro_rules! call { - ($imm8:expr) => { - vprord(a, $imm8) - }; - } - let r = constify_imm8_sae!(imm8, call); + let r = vprord(a, IMM8); transmute(r) } @@ -17612,17 +16556,17 @@ pub unsafe fn _mm512_ror_epi32(a: __m512i, imm8: i32) -> __m512i { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_ror_epi32&expand=4719) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vprold, imm8 = 123))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm512_mask_ror_epi32(src: __m512i, k: __mmask16, a: __m512i, imm8: i32) -> __m512i { +#[cfg_attr(test, assert_instr(vprold, IMM8 = 123))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_mask_ror_epi32( + src: __m512i, + k: __mmask16, + a: __m512i, +) -> __m512i { + static_assert_imm8!(IMM8); let a = a.as_i32x16(); - macro_rules! call { - ($imm8:expr) => { - vprord(a, $imm8) - }; - } - let ror = constify_imm8_sae!(imm8, call); - transmute(simd_select_bitmask(k, ror, src.as_i32x16())) + let r = vprord(a, IMM8); + transmute(simd_select_bitmask(k, r, src.as_i32x16())) } /// Rotate the bits in each packed 32-bit integer in a to the right by the number of bits specified in imm8, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). @@ -17630,18 +16574,14 @@ pub unsafe fn _mm512_mask_ror_epi32(src: __m512i, k: __mmask16, a: __m512i, imm8 /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_ror_epi32&expand=4720) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vprold, imm8 = 123))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm512_maskz_ror_epi32(k: __mmask16, a: __m512i, imm8: i32) -> __m512i { +#[cfg_attr(test, assert_instr(vprold, IMM8 = 123))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_maskz_ror_epi32(k: __mmask16, a: __m512i) -> __m512i { + static_assert_imm8!(IMM8); let a = a.as_i32x16(); - macro_rules! call { - ($imm8:expr) => { - vprord(a, $imm8) - }; - } - let ror = constify_imm8_sae!(imm8, call); + let r = vprord(a, IMM8); let zero = _mm512_setzero_si512().as_i32x16(); - transmute(simd_select_bitmask(k, ror, zero)) + transmute(simd_select_bitmask(k, r, zero)) } /// Rotate the bits in each packed 32-bit integer in a to the right by the number of bits specified in imm8, and store the results in dst. @@ -17649,16 +16589,12 @@ pub unsafe fn _mm512_maskz_ror_epi32(k: __mmask16, a: __m512i, imm8: i32) -> __m /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_ror_epi32&expand=4718) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[cfg_attr(test, assert_instr(vprold, imm8 = 1))] -#[rustc_args_required_const(1)] -pub unsafe fn _mm256_ror_epi32(a: __m256i, imm8: i32) -> __m256i { +#[cfg_attr(test, assert_instr(vprold, IMM8 = 1))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm256_ror_epi32(a: __m256i) -> __m256i { + static_assert_imm8!(IMM8); let a = a.as_i32x8(); - macro_rules! call { - ($imm8:expr) => { - vprord256(a, $imm8) - }; - } - let r = constify_imm8_sae!(imm8, call); + let r = vprord256(a, IMM8); transmute(r) } @@ -17667,17 +16603,17 @@ pub unsafe fn _mm256_ror_epi32(a: __m256i, imm8: i32) -> __m256i { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_ror_epi32&expand=4716) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[cfg_attr(test, assert_instr(vprold, imm8 = 123))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm256_mask_ror_epi32(src: __m256i, k: __mmask8, a: __m256i, imm8: i32) -> __m256i { +#[cfg_attr(test, assert_instr(vprold, IMM8 = 123))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm256_mask_ror_epi32( + src: __m256i, + k: __mmask8, + a: __m256i, +) -> __m256i { + static_assert_imm8!(IMM8); let a = a.as_i32x8(); - macro_rules! call { - ($imm8:expr) => { - vprord256(a, $imm8) - }; - } - let ror = constify_imm8_sae!(imm8, call); - transmute(simd_select_bitmask(k, ror, src.as_i32x8())) + let r = vprord256(a, IMM8); + transmute(simd_select_bitmask(k, r, src.as_i32x8())) } /// Rotate the bits in each packed 32-bit integer in a to the right by the number of bits specified in imm8, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). @@ -17685,18 +16621,14 @@ pub unsafe fn _mm256_mask_ror_epi32(src: __m256i, k: __mmask8, a: __m256i, imm8: /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_ror_epi32&expand=4717) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[cfg_attr(test, assert_instr(vprold, imm8 = 123))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm256_maskz_ror_epi32(k: __mmask8, a: __m256i, imm8: i32) -> __m256i { +#[cfg_attr(test, assert_instr(vprold, IMM8 = 123))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm256_maskz_ror_epi32(k: __mmask8, a: __m256i) -> __m256i { + static_assert_imm8!(IMM8); let a = a.as_i32x8(); - macro_rules! call { - ($imm8:expr) => { - vprord256(a, $imm8) - }; - } - let ror = constify_imm8_sae!(imm8, call); + let r = vprord256(a, IMM8); let zero = _mm256_setzero_si256().as_i32x8(); - transmute(simd_select_bitmask(k, ror, zero)) + transmute(simd_select_bitmask(k, r, zero)) } /// Rotate the bits in each packed 32-bit integer in a to the right by the number of bits specified in imm8, and store the results in dst. @@ -17704,16 +16636,12 @@ pub unsafe fn _mm256_maskz_ror_epi32(k: __mmask8, a: __m256i, imm8: i32) -> __m2 /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_ror_epi32&expand=4715) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[cfg_attr(test, assert_instr(vprold, imm8 = 1))] -#[rustc_args_required_const(1)] -pub unsafe fn _mm_ror_epi32(a: __m128i, imm8: i32) -> __m128i { +#[cfg_attr(test, assert_instr(vprold, IMM8 = 1))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm_ror_epi32(a: __m128i) -> __m128i { + static_assert_imm8!(IMM8); let a = a.as_i32x4(); - macro_rules! call { - ($imm8:expr) => { - vprord128(a, $imm8) - }; - } - let r = constify_imm8_sae!(imm8, call); + let r = vprord128(a, IMM8); transmute(r) } @@ -17722,17 +16650,17 @@ pub unsafe fn _mm_ror_epi32(a: __m128i, imm8: i32) -> __m128i { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_ror_epi32&expand=4713) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[cfg_attr(test, assert_instr(vprold, imm8 = 123))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm_mask_ror_epi32(src: __m128i, k: __mmask8, a: __m128i, imm8: i32) -> __m128i { +#[cfg_attr(test, assert_instr(vprold, IMM8 = 123))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm_mask_ror_epi32( + src: __m128i, + k: __mmask8, + a: __m128i, +) -> __m128i { + static_assert_imm8!(IMM8); let a = a.as_i32x4(); - macro_rules! call { - ($imm8:expr) => { - vprord128(a, $imm8) - }; - } - let ror = constify_imm8_sae!(imm8, call); - transmute(simd_select_bitmask(k, ror, src.as_i32x4())) + let r = vprord128(a, IMM8); + transmute(simd_select_bitmask(k, r, src.as_i32x4())) } /// Rotate the bits in each packed 32-bit integer in a to the right by the number of bits specified in imm8, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). @@ -17740,18 +16668,14 @@ pub unsafe fn _mm_mask_ror_epi32(src: __m128i, k: __mmask8, a: __m128i, imm8: i3 /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_ror_epi32&expand=4714) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[cfg_attr(test, assert_instr(vprold, imm8 = 123))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm_maskz_ror_epi32(k: __mmask8, a: __m128i, imm8: i32) -> __m128i { +#[cfg_attr(test, assert_instr(vprold, IMM8 = 123))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm_maskz_ror_epi32(k: __mmask8, a: __m128i) -> __m128i { + static_assert_imm8!(IMM8); let a = a.as_i32x4(); - macro_rules! call { - ($imm8:expr) => { - vprord128(a, $imm8) - }; - } - let ror = constify_imm8_sae!(imm8, call); + let r = vprord128(a, IMM8); let zero = _mm_setzero_si128().as_i32x4(); - transmute(simd_select_bitmask(k, ror, zero)) + transmute(simd_select_bitmask(k, r, zero)) } /// Rotate the bits in each packed 64-bit integer in a to the left by the number of bits specified in imm8, and store the results in dst. @@ -17759,16 +16683,12 @@ pub unsafe fn _mm_maskz_ror_epi32(k: __mmask8, a: __m128i, imm8: i32) -> __m128i /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_rol_epi64&expand=4694) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vprolq, imm8 = 1))] -#[rustc_args_required_const(1)] -pub unsafe fn _mm512_rol_epi64(a: __m512i, imm8: i32) -> __m512i { +#[cfg_attr(test, assert_instr(vprolq, IMM8 = 1))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm512_rol_epi64(a: __m512i) -> __m512i { + static_assert_imm8!(IMM8); let a = a.as_i64x8(); - macro_rules! call { - ($imm8:expr) => { - vprolq(a, $imm8) - }; - } - let r = constify_imm8_sae!(imm8, call); + let r = vprolq(a, IMM8); transmute(r) } @@ -17777,17 +16697,17 @@ pub unsafe fn _mm512_rol_epi64(a: __m512i, imm8: i32) -> __m512i { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_rol_epi64&expand=4692) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vprolq, imm8 = 1))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm512_mask_rol_epi64(src: __m512i, k: __mmask8, a: __m512i, imm8: i32) -> __m512i { +#[cfg_attr(test, assert_instr(vprolq, IMM8 = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_mask_rol_epi64( + src: __m512i, + k: __mmask8, + a: __m512i, +) -> __m512i { + static_assert_imm8!(IMM8); let a = a.as_i64x8(); - macro_rules! call { - ($imm8:expr) => { - vprolq(a, $imm8) - }; - } - let rol = constify_imm8_sae!(imm8, call); - transmute(simd_select_bitmask(k, rol, src.as_i64x8())) + let r = vprolq(a, IMM8); + transmute(simd_select_bitmask(k, r, src.as_i64x8())) } /// Rotate the bits in each packed 64-bit integer in a to the left by the number of bits specified in imm8, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). @@ -17795,18 +16715,14 @@ pub unsafe fn _mm512_mask_rol_epi64(src: __m512i, k: __mmask8, a: __m512i, imm8: /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_rol_epi64&expand=4693) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vprolq, imm8 = 1))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm512_maskz_rol_epi64(k: __mmask8, a: __m512i, imm8: i32) -> __m512i { +#[cfg_attr(test, assert_instr(vprolq, IMM8 = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_maskz_rol_epi64(k: __mmask8, a: __m512i) -> __m512i { + static_assert_imm8!(IMM8); let a = a.as_i64x8(); - macro_rules! call { - ($imm8:expr) => { - vprolq(a, $imm8) - }; - } - let rol = constify_imm8_sae!(imm8, call); + let r = vprolq(a, IMM8); let zero = _mm512_setzero_si512().as_i64x8(); - transmute(simd_select_bitmask(k, rol, zero)) + transmute(simd_select_bitmask(k, r, zero)) } /// Rotate the bits in each packed 64-bit integer in a to the left by the number of bits specified in imm8, and store the results in dst. @@ -17814,16 +16730,12 @@ pub unsafe fn _mm512_maskz_rol_epi64(k: __mmask8, a: __m512i, imm8: i32) -> __m5 /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_rol_epi64&expand=4691) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[cfg_attr(test, assert_instr(vprolq, imm8 = 1))] -#[rustc_args_required_const(1)] -pub unsafe fn _mm256_rol_epi64(a: __m256i, imm8: i32) -> __m256i { +#[cfg_attr(test, assert_instr(vprolq, IMM8 = 1))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm256_rol_epi64(a: __m256i) -> __m256i { + static_assert_imm8!(IMM8); let a = a.as_i64x4(); - macro_rules! call { - ($imm8:expr) => { - vprolq256(a, $imm8) - }; - } - let r = constify_imm8_sae!(imm8, call); + let r = vprolq256(a, IMM8); transmute(r) } @@ -17832,17 +16744,17 @@ pub unsafe fn _mm256_rol_epi64(a: __m256i, imm8: i32) -> __m256i { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_rol_epi64&expand=4689) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[cfg_attr(test, assert_instr(vprolq, imm8 = 1))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm256_mask_rol_epi64(src: __m256i, k: __mmask8, a: __m256i, imm8: i32) -> __m256i { +#[cfg_attr(test, assert_instr(vprolq, IMM8 = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm256_mask_rol_epi64( + src: __m256i, + k: __mmask8, + a: __m256i, +) -> __m256i { + static_assert_imm8!(IMM8); let a = a.as_i64x4(); - macro_rules! call { - ($imm8:expr) => { - vprolq256(a, $imm8) - }; - } - let rol = constify_imm8_sae!(imm8, call); - transmute(simd_select_bitmask(k, rol, src.as_i64x4())) + let r = vprolq256(a, IMM8); + transmute(simd_select_bitmask(k, r, src.as_i64x4())) } /// Rotate the bits in each packed 64-bit integer in a to the left by the number of bits specified in imm8, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). @@ -17850,18 +16762,14 @@ pub unsafe fn _mm256_mask_rol_epi64(src: __m256i, k: __mmask8, a: __m256i, imm8: /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_rol_epi64&expand=4690) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[cfg_attr(test, assert_instr(vprolq, imm8 = 1))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm256_maskz_rol_epi64(k: __mmask8, a: __m256i, imm8: i32) -> __m256i { +#[cfg_attr(test, assert_instr(vprolq, IMM8 = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm256_maskz_rol_epi64(k: __mmask8, a: __m256i) -> __m256i { + static_assert_imm8!(IMM8); let a = a.as_i64x4(); - macro_rules! call { - ($imm8:expr) => { - vprolq256(a, $imm8) - }; - } - let rol = constify_imm8_sae!(imm8, call); + let r = vprolq256(a, IMM8); let zero = _mm256_setzero_si256().as_i64x4(); - transmute(simd_select_bitmask(k, rol, zero)) + transmute(simd_select_bitmask(k, r, zero)) } /// Rotate the bits in each packed 64-bit integer in a to the left by the number of bits specified in imm8, and store the results in dst. @@ -17869,16 +16777,12 @@ pub unsafe fn _mm256_maskz_rol_epi64(k: __mmask8, a: __m256i, imm8: i32) -> __m2 /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_rol_epi64&expand=4688) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[cfg_attr(test, assert_instr(vprolq, imm8 = 1))] -#[rustc_args_required_const(1)] -pub unsafe fn _mm_rol_epi64(a: __m128i, imm8: i32) -> __m128i { +#[cfg_attr(test, assert_instr(vprolq, IMM8 = 1))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm_rol_epi64(a: __m128i) -> __m128i { + static_assert_imm8!(IMM8); let a = a.as_i64x2(); - macro_rules! call { - ($imm8:expr) => { - vprolq128(a, $imm8) - }; - } - let r = constify_imm8_sae!(imm8, call); + let r = vprolq128(a, IMM8); transmute(r) } @@ -17887,17 +16791,17 @@ pub unsafe fn _mm_rol_epi64(a: __m128i, imm8: i32) -> __m128i { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_rol_epi64&expand=4686) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[cfg_attr(test, assert_instr(vprolq, imm8 = 1))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm_mask_rol_epi64(src: __m128i, k: __mmask8, a: __m128i, imm8: i32) -> __m128i { +#[cfg_attr(test, assert_instr(vprolq, IMM8 = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm_mask_rol_epi64( + src: __m128i, + k: __mmask8, + a: __m128i, +) -> __m128i { + static_assert_imm8!(IMM8); let a = a.as_i64x2(); - macro_rules! call { - ($imm8:expr) => { - vprolq128(a, $imm8) - }; - } - let rol = constify_imm8_sae!(imm8, call); - transmute(simd_select_bitmask(k, rol, src.as_i64x2())) + let r = vprolq128(a, IMM8); + transmute(simd_select_bitmask(k, r, src.as_i64x2())) } /// Rotate the bits in each packed 64-bit integer in a to the left by the number of bits specified in imm8, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). @@ -17905,18 +16809,14 @@ pub unsafe fn _mm_mask_rol_epi64(src: __m128i, k: __mmask8, a: __m128i, imm8: i3 /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_rol_epi64&expand=4687) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[cfg_attr(test, assert_instr(vprolq, imm8 = 1))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm_maskz_rol_epi64(k: __mmask8, a: __m128i, imm8: i32) -> __m128i { +#[cfg_attr(test, assert_instr(vprolq, IMM8 = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm_maskz_rol_epi64(k: __mmask8, a: __m128i) -> __m128i { + static_assert_imm8!(IMM8); let a = a.as_i64x2(); - macro_rules! call { - ($imm8:expr) => { - vprolq128(a, $imm8) - }; - } - let rol = constify_imm8_sae!(imm8, call); + let r = vprolq128(a, IMM8); let zero = _mm_setzero_si128().as_i64x2(); - transmute(simd_select_bitmask(k, rol, zero)) + transmute(simd_select_bitmask(k, r, zero)) } /// Rotate the bits in each packed 64-bit integer in a to the right by the number of bits specified in imm8, and store the results in dst. @@ -17924,16 +16824,12 @@ pub unsafe fn _mm_maskz_rol_epi64(k: __mmask8, a: __m128i, imm8: i32) -> __m128i /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_ror_epi64&expand=4730) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vprolq, imm8 = 15))] -#[rustc_args_required_const(1)] -pub unsafe fn _mm512_ror_epi64(a: __m512i, imm8: i32) -> __m512i { +#[cfg_attr(test, assert_instr(vprolq, IMM8 = 15))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm512_ror_epi64(a: __m512i) -> __m512i { + static_assert_imm8!(IMM8); let a = a.as_i64x8(); - macro_rules! call { - ($imm8:expr) => { - vprorq(a, $imm8) - }; - } - let r = constify_imm8_sae!(imm8, call); + let r = vprorq(a, IMM8); transmute(r) } @@ -17942,17 +16838,17 @@ pub unsafe fn _mm512_ror_epi64(a: __m512i, imm8: i32) -> __m512i { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_ror_epi64&expand=4728) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vprolq, imm8 = 15))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm512_mask_ror_epi64(src: __m512i, k: __mmask8, a: __m512i, imm8: i32) -> __m512i { +#[cfg_attr(test, assert_instr(vprolq, IMM8 = 15))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_mask_ror_epi64( + src: __m512i, + k: __mmask8, + a: __m512i, +) -> __m512i { + static_assert_imm8!(IMM8); let a = a.as_i64x8(); - macro_rules! call { - ($imm8:expr) => { - vprorq(a, $imm8) - }; - } - let ror = constify_imm8_sae!(imm8, call); - transmute(simd_select_bitmask(k, ror, src.as_i64x8())) + let r = vprorq(a, IMM8); + transmute(simd_select_bitmask(k, r, src.as_i64x8())) } /// Rotate the bits in each packed 64-bit integer in a to the right by the number of bits specified in imm8, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). @@ -17960,18 +16856,14 @@ pub unsafe fn _mm512_mask_ror_epi64(src: __m512i, k: __mmask8, a: __m512i, imm8: /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_ror_epi64&expand=4729) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vprolq, imm8 = 15))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm512_maskz_ror_epi64(k: __mmask8, a: __m512i, imm8: i32) -> __m512i { +#[cfg_attr(test, assert_instr(vprolq, IMM8 = 15))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_maskz_ror_epi64(k: __mmask8, a: __m512i) -> __m512i { + static_assert_imm8!(IMM8); let a = a.as_i64x8(); - macro_rules! call { - ($imm8:expr) => { - vprorq(a, $imm8) - }; - } - let ror = constify_imm8_sae!(imm8, call); + let r = vprorq(a, IMM8); let zero = _mm512_setzero_si512().as_i64x8(); - transmute(simd_select_bitmask(k, ror, zero)) + transmute(simd_select_bitmask(k, r, zero)) } /// Rotate the bits in each packed 64-bit integer in a to the right by the number of bits specified in imm8, and store the results in dst. @@ -17979,16 +16871,12 @@ pub unsafe fn _mm512_maskz_ror_epi64(k: __mmask8, a: __m512i, imm8: i32) -> __m5 /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_ror_epi64&expand=4727) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[cfg_attr(test, assert_instr(vprolq, imm8 = 15))] -#[rustc_args_required_const(1)] -pub unsafe fn _mm256_ror_epi64(a: __m256i, imm8: i32) -> __m256i { +#[cfg_attr(test, assert_instr(vprolq, IMM8 = 15))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm256_ror_epi64(a: __m256i) -> __m256i { + static_assert_imm8!(IMM8); let a = a.as_i64x4(); - macro_rules! call { - ($imm8:expr) => { - vprorq256(a, $imm8) - }; - } - let r = constify_imm8_sae!(imm8, call); + let r = vprorq256(a, IMM8); transmute(r) } @@ -17997,17 +16885,17 @@ pub unsafe fn _mm256_ror_epi64(a: __m256i, imm8: i32) -> __m256i { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_ror_epi64&expand=4725) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[cfg_attr(test, assert_instr(vprolq, imm8 = 15))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm256_mask_ror_epi64(src: __m256i, k: __mmask8, a: __m256i, imm8: i32) -> __m256i { +#[cfg_attr(test, assert_instr(vprolq, IMM8 = 15))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm256_mask_ror_epi64( + src: __m256i, + k: __mmask8, + a: __m256i, +) -> __m256i { + static_assert_imm8!(IMM8); let a = a.as_i64x4(); - macro_rules! call { - ($imm8:expr) => { - vprorq256(a, $imm8) - }; - } - let ror = constify_imm8_sae!(imm8, call); - transmute(simd_select_bitmask(k, ror, src.as_i64x4())) + let r = vprorq256(a, IMM8); + transmute(simd_select_bitmask(k, r, src.as_i64x4())) } /// Rotate the bits in each packed 64-bit integer in a to the right by the number of bits specified in imm8, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). @@ -18015,18 +16903,14 @@ pub unsafe fn _mm256_mask_ror_epi64(src: __m256i, k: __mmask8, a: __m256i, imm8: /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_ror_epi64&expand=4726) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[cfg_attr(test, assert_instr(vprolq, imm8 = 15))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm256_maskz_ror_epi64(k: __mmask8, a: __m256i, imm8: i32) -> __m256i { +#[cfg_attr(test, assert_instr(vprolq, IMM8 = 15))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm256_maskz_ror_epi64(k: __mmask8, a: __m256i) -> __m256i { + static_assert_imm8!(IMM8); let a = a.as_i64x4(); - macro_rules! call { - ($imm8:expr) => { - vprorq256(a, $imm8) - }; - } - let ror = constify_imm8_sae!(imm8, call); + let r = vprorq256(a, IMM8); let zero = _mm256_setzero_si256().as_i64x4(); - transmute(simd_select_bitmask(k, ror, zero)) + transmute(simd_select_bitmask(k, r, zero)) } /// Rotate the bits in each packed 64-bit integer in a to the right by the number of bits specified in imm8, and store the results in dst. @@ -18034,16 +16918,12 @@ pub unsafe fn _mm256_maskz_ror_epi64(k: __mmask8, a: __m256i, imm8: i32) -> __m2 /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_ror_epi64&expand=4724) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[cfg_attr(test, assert_instr(vprolq, imm8 = 15))] -#[rustc_args_required_const(1)] -pub unsafe fn _mm_ror_epi64(a: __m128i, imm8: i32) -> __m128i { +#[cfg_attr(test, assert_instr(vprolq, IMM8 = 15))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm_ror_epi64(a: __m128i) -> __m128i { + static_assert_imm8!(IMM8); let a = a.as_i64x2(); - macro_rules! call { - ($imm8:expr) => { - vprorq128(a, $imm8) - }; - } - let r = constify_imm8_sae!(imm8, call); + let r = vprorq128(a, IMM8); transmute(r) } @@ -18052,17 +16932,17 @@ pub unsafe fn _mm_ror_epi64(a: __m128i, imm8: i32) -> __m128i { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_ror_epi64&expand=4722) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[cfg_attr(test, assert_instr(vprolq, imm8 = 15))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm_mask_ror_epi64(src: __m128i, k: __mmask8, a: __m128i, imm8: i32) -> __m128i { +#[cfg_attr(test, assert_instr(vprolq, IMM8 = 15))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm_mask_ror_epi64( + src: __m128i, + k: __mmask8, + a: __m128i, +) -> __m128i { + static_assert_imm8!(IMM8); let a = a.as_i64x2(); - macro_rules! call { - ($imm8:expr) => { - vprorq128(a, $imm8) - }; - } - let ror = constify_imm8_sae!(imm8, call); - transmute(simd_select_bitmask(k, ror, src.as_i64x2())) + let r = vprorq128(a, IMM8); + transmute(simd_select_bitmask(k, r, src.as_i64x2())) } /// Rotate the bits in each packed 64-bit integer in a to the right by the number of bits specified in imm8, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). @@ -18070,18 +16950,14 @@ pub unsafe fn _mm_mask_ror_epi64(src: __m128i, k: __mmask8, a: __m128i, imm8: i3 /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_ror_epi64&expand=4723) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[cfg_attr(test, assert_instr(vprolq, imm8 = 15))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm_maskz_ror_epi64(k: __mmask8, a: __m128i, imm8: i32) -> __m128i { +#[cfg_attr(test, assert_instr(vprolq, IMM8 = 15))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm_maskz_ror_epi64(k: __mmask8, a: __m128i) -> __m128i { + static_assert_imm8!(IMM8); let a = a.as_i64x2(); - macro_rules! call { - ($imm8:expr) => { - vprorq128(a, $imm8) - }; - } - let ror = constify_imm8_sae!(imm8, call); + let r = vprorq128(a, IMM8); let zero = _mm_setzero_si128().as_i64x2(); - transmute(simd_select_bitmask(k, ror, zero)) + transmute(simd_select_bitmask(k, r, zero)) } /// Shift packed 32-bit integers in a left by imm8 while shifting in zeros, and store the results in dst. @@ -18089,16 +16965,12 @@ pub unsafe fn _mm_maskz_ror_epi64(k: __mmask8, a: __m128i, imm8: i32) -> __m128i /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_slli_epi32&expand=5310) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vpslld, imm8 = 5))] -#[rustc_args_required_const(1)] -pub unsafe fn _mm512_slli_epi32(a: __m512i, imm8: u32) -> __m512i { +#[cfg_attr(test, assert_instr(vpslld, IMM8 = 5))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm512_slli_epi32(a: __m512i) -> __m512i { + static_assert_imm_u8!(IMM8); let a = a.as_i32x16(); - macro_rules! call { - ($imm8:expr) => { - vpsllid(a, $imm8) - }; - } - let r = constify_imm8_sae!(imm8, call); + let r = vpsllid(a, IMM8); transmute(r) } @@ -18107,16 +16979,16 @@ pub unsafe fn _mm512_slli_epi32(a: __m512i, imm8: u32) -> __m512i { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_slli_epi32&expand=5308) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vpslld, imm8 = 5))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm512_mask_slli_epi32(src: __m512i, k: __mmask16, a: __m512i, imm8: u32) -> __m512i { +#[cfg_attr(test, assert_instr(vpslld, IMM8 = 5))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_mask_slli_epi32( + src: __m512i, + k: __mmask16, + a: __m512i, +) -> __m512i { + static_assert_imm_u8!(IMM8); let a = a.as_i32x16(); - macro_rules! call { - ($imm8:expr) => { - vpsllid(a, $imm8) - }; - } - let shf = constify_imm8_sae!(imm8, call); + let shf = vpsllid(a, IMM8); transmute(simd_select_bitmask(k, shf, src.as_i32x16())) } @@ -18125,16 +16997,12 @@ pub unsafe fn _mm512_mask_slli_epi32(src: __m512i, k: __mmask16, a: __m512i, imm /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_slli_epi32&expand=5309) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vpslld, imm8 = 5))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm512_maskz_slli_epi32(k: __mmask16, a: __m512i, imm8: u32) -> __m512i { +#[cfg_attr(test, assert_instr(vpslld, IMM8 = 5))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_maskz_slli_epi32(k: __mmask16, a: __m512i) -> __m512i { + static_assert_imm_u8!(IMM8); let a = a.as_i32x16(); - macro_rules! call { - ($imm8:expr) => { - vpsllid(a, $imm8) - }; - } - let shf = constify_imm8_sae!(imm8, call); + let shf = vpsllid(a, IMM8); let zero = _mm512_setzero_si512().as_i32x16(); transmute(simd_select_bitmask(k, shf, zero)) } @@ -18144,16 +17012,17 @@ pub unsafe fn _mm512_maskz_slli_epi32(k: __mmask16, a: __m512i, imm8: u32) -> __ /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_slli_epi32&expand=5305) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[cfg_attr(test, assert_instr(vpslld, imm8 = 5))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm256_mask_slli_epi32(src: __m256i, k: __mmask8, a: __m256i, imm8: u32) -> __m256i { - macro_rules! call { - ($imm8:expr) => { - _mm256_slli_epi32(a, $imm8) - }; - } - let shf = constify_imm8_sae!(imm8, call); - transmute(simd_select_bitmask(k, shf.as_i32x8(), src.as_i32x8())) +#[cfg_attr(test, assert_instr(vpslld, IMM8 = 5))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm256_mask_slli_epi32( + src: __m256i, + k: __mmask8, + a: __m256i, +) -> __m256i { + static_assert_imm_u8!(IMM8); + let imm8 = IMM8 as i32; + let r = psllid256(a.as_i32x8(), imm8); + transmute(simd_select_bitmask(k, r, src.as_i32x8())) } /// Shift packed 32-bit integers in a left by imm8 while shifting in zeros, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). @@ -18161,17 +17030,14 @@ pub unsafe fn _mm256_mask_slli_epi32(src: __m256i, k: __mmask8, a: __m256i, imm8 /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_slli_epi32&expand=5306) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[cfg_attr(test, assert_instr(vpslld, imm8 = 5))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm256_maskz_slli_epi32(k: __mmask8, a: __m256i, imm8: u32) -> __m256i { - macro_rules! call { - ($imm8:expr) => { - _mm256_slli_epi32(a, $imm8) - }; - } - let shf = constify_imm8_sae!(imm8, call); +#[cfg_attr(test, assert_instr(vpslld, IMM8 = 5))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm256_maskz_slli_epi32(k: __mmask8, a: __m256i) -> __m256i { + static_assert_imm_u8!(IMM8); + let imm8 = IMM8 as i32; + let r = psllid256(a.as_i32x8(), imm8); let zero = _mm256_setzero_si256().as_i32x8(); - transmute(simd_select_bitmask(k, shf.as_i32x8(), zero)) + transmute(simd_select_bitmask(k, r, zero)) } /// Shift packed 32-bit integers in a left by imm8 while shifting in zeros, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). @@ -18179,16 +17045,17 @@ pub unsafe fn _mm256_maskz_slli_epi32(k: __mmask8, a: __m256i, imm8: u32) -> __m /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_slli_epi32&expand=5302) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[cfg_attr(test, assert_instr(vpslld, imm8 = 5))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm_mask_slli_epi32(src: __m128i, k: __mmask8, a: __m128i, imm8: u32) -> __m128i { - macro_rules! call { - ($imm8:expr) => { - _mm_slli_epi32(a, $imm8) - }; - } - let shf = constify_imm8_sae!(imm8, call); - transmute(simd_select_bitmask(k, shf.as_i32x4(), src.as_i32x4())) +#[cfg_attr(test, assert_instr(vpslld, IMM8 = 5))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm_mask_slli_epi32( + src: __m128i, + k: __mmask8, + a: __m128i, +) -> __m128i { + static_assert_imm_u8!(IMM8); + let imm8 = IMM8 as i32; + let r = psllid128(a.as_i32x4(), imm8); + transmute(simd_select_bitmask(k, r, src.as_i32x4())) } /// Shift packed 32-bit integers in a left by imm8 while shifting in zeros, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). @@ -18196,17 +17063,14 @@ pub unsafe fn _mm_mask_slli_epi32(src: __m128i, k: __mmask8, a: __m128i, imm8: u /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_slli_epi32&expand=5303) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[cfg_attr(test, assert_instr(vpslld, imm8 = 5))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm_maskz_slli_epi32(k: __mmask8, a: __m128i, imm8: u32) -> __m128i { - macro_rules! call { - ($imm8:expr) => { - _mm_slli_epi32(a, $imm8) - }; - } - let shf = constify_imm8_sae!(imm8, call); +#[cfg_attr(test, assert_instr(vpslld, IMM8 = 5))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm_maskz_slli_epi32(k: __mmask8, a: __m128i) -> __m128i { + static_assert_imm_u8!(IMM8); + let imm8 = IMM8 as i32; + let r = psllid128(a.as_i32x4(), imm8); let zero = _mm_setzero_si128().as_i32x4(); - transmute(simd_select_bitmask(k, shf.as_i32x4(), zero)) + transmute(simd_select_bitmask(k, r, zero)) } /// Shift packed 32-bit integers in a right by imm8 while shifting in zeros, and store the results in dst. @@ -18214,16 +17078,12 @@ pub unsafe fn _mm_maskz_slli_epi32(k: __mmask8, a: __m128i, imm8: u32) -> __m128 /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_srli_epi32&expand=5522) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vpsrld, imm8 = 1))] -#[rustc_args_required_const(1)] -pub unsafe fn _mm512_srli_epi32(a: __m512i, imm8: u32) -> __m512i { +#[cfg_attr(test, assert_instr(vpsrld, IMM8 = 1))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm512_srli_epi32(a: __m512i) -> __m512i { + static_assert_imm_u8!(IMM8); let a = a.as_i32x16(); - macro_rules! call { - ($imm8:expr) => { - vpsrlid(a, $imm8) - }; - } - let r = constify_imm8_sae!(imm8, call); + let r = vpsrlid(a, IMM8); transmute(r) } @@ -18232,16 +17092,16 @@ pub unsafe fn _mm512_srli_epi32(a: __m512i, imm8: u32) -> __m512i { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_srli_epi32&expand=5520) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vpsrld, imm8 = 1))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm512_mask_srli_epi32(src: __m512i, k: __mmask16, a: __m512i, imm8: u32) -> __m512i { +#[cfg_attr(test, assert_instr(vpsrld, IMM8 = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_mask_srli_epi32( + src: __m512i, + k: __mmask16, + a: __m512i, +) -> __m512i { + static_assert_imm_u8!(IMM8); let a = a.as_i32x16(); - macro_rules! call { - ($imm8:expr) => { - vpsrlid(a, $imm8) - }; - } - let shf = constify_imm8_sae!(imm8, call); + let shf = vpsrlid(a, IMM8); transmute(simd_select_bitmask(k, shf, src.as_i32x16())) } @@ -18250,16 +17110,12 @@ pub unsafe fn _mm512_mask_srli_epi32(src: __m512i, k: __mmask16, a: __m512i, imm /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_srli_epi32&expand=5521) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vpsrld, imm8 = 1))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm512_maskz_srli_epi32(k: __mmask16, a: __m512i, imm8: u32) -> __m512i { +#[cfg_attr(test, assert_instr(vpsrld, IMM8 = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_maskz_srli_epi32(k: __mmask16, a: __m512i) -> __m512i { + static_assert_imm_u8!(IMM8); let a = a.as_i32x16(); - macro_rules! call { - ($imm8:expr) => { - vpsrlid(a, $imm8) - }; - } - let shf = constify_imm8_sae!(imm8, call); + let shf = vpsrlid(a, IMM8); let zero = _mm512_setzero_si512().as_i32x16(); transmute(simd_select_bitmask(k, shf, zero)) } @@ -18269,16 +17125,17 @@ pub unsafe fn _mm512_maskz_srli_epi32(k: __mmask16, a: __m512i, imm8: u32) -> __ /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_srli_epi32&expand=5517) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[cfg_attr(test, assert_instr(vpsrld, imm8 = 1))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm256_mask_srli_epi32(src: __m256i, k: __mmask8, a: __m256i, imm8: u32) -> __m256i { - macro_rules! call { - ($imm8:expr) => { - _mm256_srli_epi32(a, $imm8) - }; - } - let shf = constify_imm8_sae!(imm8, call); - transmute(simd_select_bitmask(k, shf.as_i32x8(), src.as_i32x8())) +#[cfg_attr(test, assert_instr(vpsrld, IMM8 = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm256_mask_srli_epi32( + src: __m256i, + k: __mmask8, + a: __m256i, +) -> __m256i { + static_assert_imm_u8!(IMM8); + let imm8 = IMM8 as i32; + let r = psrlid256(a.as_i32x8(), imm8); + transmute(simd_select_bitmask(k, r, src.as_i32x8())) } /// Shift packed 32-bit integers in a right by imm8 while shifting in zeros, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). @@ -18286,17 +17143,14 @@ pub unsafe fn _mm256_mask_srli_epi32(src: __m256i, k: __mmask8, a: __m256i, imm8 /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_srli_epi32&expand=5518) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[cfg_attr(test, assert_instr(vpsrld, imm8 = 1))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm256_maskz_srli_epi32(k: __mmask8, a: __m256i, imm8: u32) -> __m256i { - macro_rules! call { - ($imm8:expr) => { - _mm256_srli_epi32(a, $imm8) - }; - } - let shf = constify_imm8_sae!(imm8, call); +#[cfg_attr(test, assert_instr(vpsrld, IMM8 = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm256_maskz_srli_epi32(k: __mmask8, a: __m256i) -> __m256i { + static_assert_imm_u8!(IMM8); + let imm8 = IMM8 as i32; + let r = psrlid256(a.as_i32x8(), imm8); let zero = _mm256_setzero_si256().as_i32x8(); - transmute(simd_select_bitmask(k, shf.as_i32x8(), zero)) + transmute(simd_select_bitmask(k, r, zero)) } /// Shift packed 32-bit integers in a right by imm8 while shifting in zeros, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). @@ -18304,16 +17158,17 @@ pub unsafe fn _mm256_maskz_srli_epi32(k: __mmask8, a: __m256i, imm8: u32) -> __m /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_srli_epi32&expand=5514) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[cfg_attr(test, assert_instr(vpsrld, imm8 = 1))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm_mask_srli_epi32(src: __m128i, k: __mmask8, a: __m128i, imm8: u32) -> __m128i { - macro_rules! call { - ($imm8:expr) => { - _mm_srli_epi32(a, $imm8) - }; - } - let shf = constify_imm8_sae!(imm8, call); - transmute(simd_select_bitmask(k, shf.as_i32x4(), src.as_i32x4())) +#[cfg_attr(test, assert_instr(vpsrld, IMM8 = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm_mask_srli_epi32( + src: __m128i, + k: __mmask8, + a: __m128i, +) -> __m128i { + static_assert_imm_u8!(IMM8); + let imm8 = IMM8 as i32; + let r = psrlid128(a.as_i32x4(), imm8); + transmute(simd_select_bitmask(k, r, src.as_i32x4())) } /// Shift packed 32-bit integers in a right by imm8 while shifting in zeros, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). @@ -18321,17 +17176,14 @@ pub unsafe fn _mm_mask_srli_epi32(src: __m128i, k: __mmask8, a: __m128i, imm8: u /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_srli_epi32&expand=5515) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[cfg_attr(test, assert_instr(vpsrld, imm8 = 1))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm_maskz_srli_epi32(k: __mmask8, a: __m128i, imm8: u32) -> __m128i { - macro_rules! call { - ($imm8:expr) => { - _mm_srli_epi32(a, $imm8) - }; - } - let shf = constify_imm8_sae!(imm8, call); +#[cfg_attr(test, assert_instr(vpsrld, IMM8 = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm_maskz_srli_epi32(k: __mmask8, a: __m128i) -> __m128i { + static_assert_imm_u8!(IMM8); + let imm8 = IMM8 as i32; + let r = psrlid128(a.as_i32x4(), imm8); let zero = _mm_setzero_si128().as_i32x4(); - transmute(simd_select_bitmask(k, shf.as_i32x4(), zero)) + transmute(simd_select_bitmask(k, r, zero)) } /// Shift packed 64-bit integers in a left by imm8 while shifting in zeros, and store the results in dst. @@ -18339,16 +17191,12 @@ pub unsafe fn _mm_maskz_srli_epi32(k: __mmask8, a: __m128i, imm8: u32) -> __m128 /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_slli_epi64&expand=5319) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vpsllq, imm8 = 5))] -#[rustc_args_required_const(1)] -pub unsafe fn _mm512_slli_epi64(a: __m512i, imm8: u32) -> __m512i { +#[cfg_attr(test, assert_instr(vpsllq, IMM8 = 5))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm512_slli_epi64(a: __m512i) -> __m512i { + static_assert_imm_u8!(IMM8); let a = a.as_i64x8(); - macro_rules! call { - ($imm8:expr) => { - vpslliq(a, $imm8) - }; - } - let r = constify_imm8_sae!(imm8, call); + let r = vpslliq(a, IMM8); transmute(r) } @@ -18357,16 +17205,16 @@ pub unsafe fn _mm512_slli_epi64(a: __m512i, imm8: u32) -> __m512i { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_slli_epi64&expand=5317) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vpsllq, imm8 = 5))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm512_mask_slli_epi64(src: __m512i, k: __mmask8, a: __m512i, imm8: u32) -> __m512i { +#[cfg_attr(test, assert_instr(vpsllq, IMM8 = 5))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_mask_slli_epi64( + src: __m512i, + k: __mmask8, + a: __m512i, +) -> __m512i { + static_assert_imm_u8!(IMM8); let a = a.as_i64x8(); - macro_rules! call { - ($imm8:expr) => { - vpslliq(a, $imm8) - }; - } - let shf = constify_imm8_sae!(imm8, call); + let shf = vpslliq(a, IMM8); transmute(simd_select_bitmask(k, shf, src.as_i64x8())) } @@ -18375,16 +17223,12 @@ pub unsafe fn _mm512_mask_slli_epi64(src: __m512i, k: __mmask8, a: __m512i, imm8 /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_slli_epi64&expand=5318) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vpsllq, imm8 = 5))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm512_maskz_slli_epi64(k: __mmask8, a: __m512i, imm8: u32) -> __m512i { +#[cfg_attr(test, assert_instr(vpsllq, IMM8 = 5))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_maskz_slli_epi64(k: __mmask8, a: __m512i) -> __m512i { + static_assert_imm_u8!(IMM8); let a = a.as_i64x8(); - macro_rules! call { - ($imm8:expr) => { - vpslliq(a, $imm8) - }; - } - let shf = constify_imm8_sae!(imm8, call); + let shf = vpslliq(a, IMM8); let zero = _mm512_setzero_si512().as_i64x8(); transmute(simd_select_bitmask(k, shf, zero)) } @@ -18394,16 +17238,17 @@ pub unsafe fn _mm512_maskz_slli_epi64(k: __mmask8, a: __m512i, imm8: u32) -> __m /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_slli_epi64&expand=5314) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[cfg_attr(test, assert_instr(vpsllq, imm8 = 5))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm256_mask_slli_epi64(src: __m256i, k: __mmask8, a: __m256i, imm8: u32) -> __m256i { - macro_rules! call { - ($imm8:expr) => { - _mm256_slli_epi64(a, $imm8) - }; - } - let shf = constify_imm8_sae!(imm8, call); - transmute(simd_select_bitmask(k, shf.as_i64x4(), src.as_i64x4())) +#[cfg_attr(test, assert_instr(vpsllq, IMM8 = 5))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm256_mask_slli_epi64( + src: __m256i, + k: __mmask8, + a: __m256i, +) -> __m256i { + static_assert_imm_u8!(IMM8); + let imm8 = IMM8 as i32; + let r = pslliq256(a.as_i64x4(), imm8); + transmute(simd_select_bitmask(k, r, src.as_i64x4())) } /// Shift packed 64-bit integers in a left by imm8 while shifting in zeros, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). @@ -18411,17 +17256,14 @@ pub unsafe fn _mm256_mask_slli_epi64(src: __m256i, k: __mmask8, a: __m256i, imm8 /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_slli_epi64&expand=5315) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[cfg_attr(test, assert_instr(vpsllq, imm8 = 5))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm256_maskz_slli_epi64(k: __mmask8, a: __m256i, imm8: u32) -> __m256i { - macro_rules! call { - ($imm8:expr) => { - _mm256_slli_epi64(a, $imm8) - }; - } - let shf = constify_imm8_sae!(imm8, call); +#[cfg_attr(test, assert_instr(vpsllq, IMM8 = 5))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm256_maskz_slli_epi64(k: __mmask8, a: __m256i) -> __m256i { + static_assert_imm_u8!(IMM8); + let imm8 = IMM8 as i32; + let r = pslliq256(a.as_i64x4(), imm8); let zero = _mm256_setzero_si256().as_i64x4(); - transmute(simd_select_bitmask(k, shf.as_i64x4(), zero)) + transmute(simd_select_bitmask(k, r, zero)) } /// Shift packed 64-bit integers in a left by imm8 while shifting in zeros, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). @@ -18429,16 +17271,17 @@ pub unsafe fn _mm256_maskz_slli_epi64(k: __mmask8, a: __m256i, imm8: u32) -> __m /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_slli_epi64&expand=5311) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[cfg_attr(test, assert_instr(vpsllq, imm8 = 5))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm_mask_slli_epi64(src: __m128i, k: __mmask8, a: __m128i, imm8: u32) -> __m128i { - macro_rules! call { - ($imm8:expr) => { - _mm_slli_epi64(a, $imm8) - }; - } - let shf = constify_imm8_sae!(imm8, call); - transmute(simd_select_bitmask(k, shf.as_i64x2(), src.as_i64x2())) +#[cfg_attr(test, assert_instr(vpsllq, IMM8 = 5))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm_mask_slli_epi64( + src: __m128i, + k: __mmask8, + a: __m128i, +) -> __m128i { + static_assert_imm_u8!(IMM8); + let imm8 = IMM8 as i32; + let r = pslliq128(a.as_i64x2(), imm8); + transmute(simd_select_bitmask(k, r, src.as_i64x2())) } /// Shift packed 64-bit integers in a left by imm8 while shifting in zeros, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). @@ -18446,17 +17289,14 @@ pub unsafe fn _mm_mask_slli_epi64(src: __m128i, k: __mmask8, a: __m128i, imm8: u /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_slli_epi64&expand=5312) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[cfg_attr(test, assert_instr(vpsllq, imm8 = 5))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm_maskz_slli_epi64(k: __mmask8, a: __m128i, imm8: u32) -> __m128i { - macro_rules! call { - ($imm8:expr) => { - _mm_slli_epi64(a, $imm8) - }; - } - let shf = constify_imm8_sae!(imm8, call); +#[cfg_attr(test, assert_instr(vpsllq, IMM8 = 5))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm_maskz_slli_epi64(k: __mmask8, a: __m128i) -> __m128i { + static_assert_imm_u8!(IMM8); + let imm8 = IMM8 as i32; + let r = pslliq128(a.as_i64x2(), imm8); let zero = _mm_setzero_si128().as_i64x2(); - transmute(simd_select_bitmask(k, shf.as_i64x2(), zero)) + transmute(simd_select_bitmask(k, r, zero)) } /// Shift packed 64-bit integers in a right by imm8 while shifting in zeros, and store the results in dst. @@ -18464,16 +17304,12 @@ pub unsafe fn _mm_maskz_slli_epi64(k: __mmask8, a: __m128i, imm8: u32) -> __m128 /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_srli_epi64&expand=5531) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vpsrlq, imm8 = 1))] -#[rustc_args_required_const(1)] -pub unsafe fn _mm512_srli_epi64(a: __m512i, imm8: u32) -> __m512i { +#[cfg_attr(test, assert_instr(vpsrlq, IMM8 = 1))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm512_srli_epi64(a: __m512i) -> __m512i { + static_assert_imm_u8!(IMM8); let a = a.as_i64x8(); - macro_rules! call { - ($imm8:expr) => { - vpsrliq(a, $imm8) - }; - } - let r = constify_imm8_sae!(imm8, call); + let r = vpsrliq(a, IMM8); transmute(r) } @@ -18482,16 +17318,16 @@ pub unsafe fn _mm512_srli_epi64(a: __m512i, imm8: u32) -> __m512i { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_srli_epi64&expand=5529) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vpsrlq, imm8 = 1))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm512_mask_srli_epi64(src: __m512i, k: __mmask8, a: __m512i, imm8: u32) -> __m512i { +#[cfg_attr(test, assert_instr(vpsrlq, IMM8 = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_mask_srli_epi64( + src: __m512i, + k: __mmask8, + a: __m512i, +) -> __m512i { + static_assert_imm_u8!(IMM8); let a = a.as_i64x8(); - macro_rules! call { - ($imm8:expr) => { - vpsrliq(a, $imm8) - }; - } - let shf = constify_imm8_sae!(imm8, call); + let shf = vpsrliq(a, IMM8); transmute(simd_select_bitmask(k, shf, src.as_i64x8())) } @@ -18500,16 +17336,12 @@ pub unsafe fn _mm512_mask_srli_epi64(src: __m512i, k: __mmask8, a: __m512i, imm8 /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_srli_epi64&expand=5530) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vpsrlq, imm8 = 1))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm512_maskz_srli_epi64(k: __mmask8, a: __m512i, imm8: u32) -> __m512i { +#[cfg_attr(test, assert_instr(vpsrlq, IMM8 = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_maskz_srli_epi64(k: __mmask8, a: __m512i) -> __m512i { + static_assert_imm_u8!(IMM8); let a = a.as_i64x8(); - macro_rules! call { - ($imm8:expr) => { - vpsrliq(a, $imm8) - }; - } - let shf = constify_imm8_sae!(imm8, call); + let shf = vpsrliq(a, IMM8); let zero = _mm512_setzero_si512().as_i64x8(); transmute(simd_select_bitmask(k, shf, zero)) } @@ -18519,16 +17351,17 @@ pub unsafe fn _mm512_maskz_srli_epi64(k: __mmask8, a: __m512i, imm8: u32) -> __m /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_srli_epi64&expand=5526) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[cfg_attr(test, assert_instr(vpsrlq, imm8 = 1))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm256_mask_srli_epi64(src: __m256i, k: __mmask8, a: __m256i, imm8: u32) -> __m256i { - macro_rules! call { - ($imm8:expr) => { - _mm256_srli_epi64(a, $imm8) - }; - } - let shf = constify_imm8_sae!(imm8, call); - transmute(simd_select_bitmask(k, shf.as_i64x4(), src.as_i64x4())) +#[cfg_attr(test, assert_instr(vpsrlq, IMM8 = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm256_mask_srli_epi64( + src: __m256i, + k: __mmask8, + a: __m256i, +) -> __m256i { + static_assert_imm_u8!(IMM8); + let imm8 = IMM8 as i32; + let r = psrliq256(a.as_i64x4(), imm8); + transmute(simd_select_bitmask(k, r, src.as_i64x4())) } /// Shift packed 64-bit integers in a right by imm8 while shifting in zeros, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). @@ -18536,17 +17369,14 @@ pub unsafe fn _mm256_mask_srli_epi64(src: __m256i, k: __mmask8, a: __m256i, imm8 /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_srli_epi64&expand=5527) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[cfg_attr(test, assert_instr(vpsrlq, imm8 = 1))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm256_maskz_srli_epi64(k: __mmask8, a: __m256i, imm8: u32) -> __m256i { - macro_rules! call { - ($imm8:expr) => { - _mm256_srli_epi64(a, $imm8) - }; - } - let shf = constify_imm8_sae!(imm8, call); +#[cfg_attr(test, assert_instr(vpsrlq, IMM8 = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm256_maskz_srli_epi64(k: __mmask8, a: __m256i) -> __m256i { + static_assert_imm_u8!(IMM8); + let imm8 = IMM8 as i32; + let r = psrliq256(a.as_i64x4(), imm8); let zero = _mm256_setzero_si256().as_i64x4(); - transmute(simd_select_bitmask(k, shf.as_i64x4(), zero)) + transmute(simd_select_bitmask(k, r, zero)) } /// Shift packed 64-bit integers in a right by imm8 while shifting in zeros, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). @@ -18554,16 +17384,17 @@ pub unsafe fn _mm256_maskz_srli_epi64(k: __mmask8, a: __m256i, imm8: u32) -> __m /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_srli_epi64&expand=5523) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[cfg_attr(test, assert_instr(vpsrlq, imm8 = 1))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm_mask_srli_epi64(src: __m128i, k: __mmask8, a: __m128i, imm8: u32) -> __m128i { - macro_rules! call { - ($imm8:expr) => { - _mm_srli_epi64(a, $imm8) - }; - } - let shf = constify_imm8_sae!(imm8, call); - transmute(simd_select_bitmask(k, shf.as_i64x2(), src.as_i64x2())) +#[cfg_attr(test, assert_instr(vpsrlq, IMM8 = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm_mask_srli_epi64( + src: __m128i, + k: __mmask8, + a: __m128i, +) -> __m128i { + static_assert_imm_u8!(IMM8); + let imm8 = IMM8 as i32; + let r = psrliq128(a.as_i64x2(), imm8); + transmute(simd_select_bitmask(k, r, src.as_i64x2())) } /// Shift packed 64-bit integers in a right by imm8 while shifting in zeros, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). @@ -18571,17 +17402,14 @@ pub unsafe fn _mm_mask_srli_epi64(src: __m128i, k: __mmask8, a: __m128i, imm8: u /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_srli_epi64&expand=5524) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[cfg_attr(test, assert_instr(vpsrlq, imm8 = 1))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm_maskz_srli_epi64(k: __mmask8, a: __m128i, imm8: u32) -> __m128i { - macro_rules! call { - ($imm8:expr) => { - _mm_srli_epi64(a, $imm8) - }; - } - let shf = constify_imm8_sae!(imm8, call); +#[cfg_attr(test, assert_instr(vpsrlq, IMM8 = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm_maskz_srli_epi64(k: __mmask8, a: __m128i) -> __m128i { + static_assert_imm_u8!(IMM8); + let imm8 = IMM8 as i32; + let r = psrliq128(a.as_i64x2(), imm8); let zero = _mm_setzero_si128().as_i64x2(); - transmute(simd_select_bitmask(k, shf.as_i64x2(), zero)) + transmute(simd_select_bitmask(k, r, zero)) } /// Shift packed 32-bit integers in a left by count while shifting in zeros, and store the results in dst. @@ -19143,16 +17971,12 @@ pub unsafe fn _mm_maskz_sra_epi64(k: __mmask8, a: __m128i, count: __m128i) -> __ /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_srai_epi32&expand=5436) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vpsrad, imm8 = 1))] -#[rustc_args_required_const(1)] -pub unsafe fn _mm512_srai_epi32(a: __m512i, imm8: u32) -> __m512i { +#[cfg_attr(test, assert_instr(vpsrad, IMM8 = 1))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm512_srai_epi32(a: __m512i) -> __m512i { + static_assert_imm_u8!(IMM8); let a = a.as_i32x16(); - macro_rules! call { - ($imm8:expr) => { - vpsraid(a, $imm8) - }; - } - let r = constify_imm8_sae!(imm8, call); + let r = vpsraid512(a, IMM8); transmute(r) } @@ -19161,17 +17985,17 @@ pub unsafe fn _mm512_srai_epi32(a: __m512i, imm8: u32) -> __m512i { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_srai_epi32&expand=5434) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vpsrad, imm8 = 1))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm512_mask_srai_epi32(src: __m512i, k: __mmask16, a: __m512i, imm8: u32) -> __m512i { +#[cfg_attr(test, assert_instr(vpsrad, IMM8 = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_mask_srai_epi32( + src: __m512i, + k: __mmask16, + a: __m512i, +) -> __m512i { + static_assert_imm_u8!(IMM8); let a = a.as_i32x16(); - macro_rules! call { - ($imm8:expr) => { - vpsraid(a, $imm8) - }; - } - let shf = constify_imm8_sae!(imm8, call); - transmute(simd_select_bitmask(k, shf, src.as_i32x16())) + let r = vpsraid512(a, IMM8); + transmute(simd_select_bitmask(k, r, src.as_i32x16())) } /// Shift packed 32-bit integers in a right by imm8 while shifting in sign bits, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). @@ -19179,18 +18003,14 @@ pub unsafe fn _mm512_mask_srai_epi32(src: __m512i, k: __mmask16, a: __m512i, imm /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_srai_epi32&expand=5435) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vpsrad, imm8 = 1))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm512_maskz_srai_epi32(k: __mmask16, a: __m512i, imm8: u32) -> __m512i { +#[cfg_attr(test, assert_instr(vpsrad, IMM8 = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_maskz_srai_epi32(k: __mmask16, a: __m512i) -> __m512i { + static_assert_imm_u8!(IMM8); let a = a.as_i32x16(); - macro_rules! call { - ($imm8:expr) => { - vpsraid(a, $imm8) - }; - } - let shf = constify_imm8_sae!(imm8, call); + let r = vpsraid512(a, IMM8); let zero = _mm512_setzero_si512().as_i32x16(); - transmute(simd_select_bitmask(k, shf, zero)) + transmute(simd_select_bitmask(k, r, zero)) } /// Shift packed 32-bit integers in a right by imm8 while shifting in sign bits, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). @@ -19198,16 +18018,16 @@ pub unsafe fn _mm512_maskz_srai_epi32(k: __mmask16, a: __m512i, imm8: u32) -> __ /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_srai_epi32&expand=5431) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[cfg_attr(test, assert_instr(vpsrad, imm8 = 1))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm256_mask_srai_epi32(src: __m256i, k: __mmask8, a: __m256i, imm8: u32) -> __m256i { - macro_rules! call { - ($imm8:expr) => { - _mm256_srai_epi32(a, $imm8) - }; - } - let shf = constify_imm8_sae!(imm8, call); - transmute(simd_select_bitmask(k, shf.as_i32x8(), src.as_i32x8())) +#[cfg_attr(test, assert_instr(vpsrad, IMM8 = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm256_mask_srai_epi32( + src: __m256i, + k: __mmask8, + a: __m256i, +) -> __m256i { + let imm8 = IMM8 as i32; + let r = psraid256(a.as_i32x8(), imm8); + transmute(simd_select_bitmask(k, r, src.as_i32x8())) } /// Shift packed 32-bit integers in a right by imm8 while shifting in sign bits, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). @@ -19215,17 +18035,13 @@ pub unsafe fn _mm256_mask_srai_epi32(src: __m256i, k: __mmask8, a: __m256i, imm8 /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_srai_epi32&expand=5432) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[cfg_attr(test, assert_instr(vpsrad, imm8 = 1))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm256_maskz_srai_epi32(k: __mmask8, a: __m256i, imm8: u32) -> __m256i { - macro_rules! call { - ($imm8:expr) => { - _mm256_srai_epi32(a, $imm8) - }; - } - let shf = constify_imm8_sae!(imm8, call); +#[cfg_attr(test, assert_instr(vpsrad, IMM8 = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm256_maskz_srai_epi32(k: __mmask8, a: __m256i) -> __m256i { + let imm8 = IMM8 as i32; + let r = psraid256(a.as_i32x8(), imm8); let zero = _mm256_setzero_si256().as_i32x8(); - transmute(simd_select_bitmask(k, shf.as_i32x8(), zero)) + transmute(simd_select_bitmask(k, r, zero)) } /// Shift packed 32-bit integers in a right by imm8 while shifting in sign bits, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). @@ -19233,16 +18049,16 @@ pub unsafe fn _mm256_maskz_srai_epi32(k: __mmask8, a: __m256i, imm8: u32) -> __m /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_srai_epi32&expand=5428) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[cfg_attr(test, assert_instr(vpsrad, imm8 = 1))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm_mask_srai_epi32(src: __m128i, k: __mmask8, a: __m128i, imm8: u32) -> __m128i { - macro_rules! call { - ($imm8:expr) => { - _mm_srai_epi32(a, $imm8) - }; - } - let shf = constify_imm8_sae!(imm8, call); - transmute(simd_select_bitmask(k, shf.as_i32x4(), src.as_i32x4())) +#[cfg_attr(test, assert_instr(vpsrad, IMM8 = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm_mask_srai_epi32( + src: __m128i, + k: __mmask8, + a: __m128i, +) -> __m128i { + let imm8 = IMM8 as i32; + let r = psraid128(a.as_i32x4(), imm8); + transmute(simd_select_bitmask(k, r, src.as_i32x4())) } /// Shift packed 32-bit integers in a right by imm8 while shifting in sign bits, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). @@ -19250,17 +18066,13 @@ pub unsafe fn _mm_mask_srai_epi32(src: __m128i, k: __mmask8, a: __m128i, imm8: u /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_srai_epi32&expand=5429) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[cfg_attr(test, assert_instr(vpsrad, imm8 = 1))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm_maskz_srai_epi32(k: __mmask8, a: __m128i, imm8: u32) -> __m128i { - macro_rules! call { - ($imm8:expr) => { - _mm_srai_epi32(a, $imm8) - }; - } - let shf = constify_imm8_sae!(imm8, call); +#[cfg_attr(test, assert_instr(vpsrad, IMM8 = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm_maskz_srai_epi32(k: __mmask8, a: __m128i) -> __m128i { + let imm8 = IMM8 as i32; + let r = psraid128(a.as_i32x4(), imm8); let zero = _mm_setzero_si128().as_i32x4(); - transmute(simd_select_bitmask(k, shf.as_i32x4(), zero)) + transmute(simd_select_bitmask(k, r, zero)) } /// Shift packed 64-bit integers in a right by imm8 while shifting in sign bits, and store the results in dst. @@ -19268,16 +18080,12 @@ pub unsafe fn _mm_maskz_srai_epi32(k: __mmask8, a: __m128i, imm8: u32) -> __m128 /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_srai_epi64&expand=5445) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vpsraq, imm8 = 1))] -#[rustc_args_required_const(1)] -pub unsafe fn _mm512_srai_epi64(a: __m512i, imm8: u32) -> __m512i { +#[cfg_attr(test, assert_instr(vpsraq, IMM8 = 1))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm512_srai_epi64(a: __m512i) -> __m512i { + static_assert_imm_u8!(IMM8); let a = a.as_i64x8(); - macro_rules! call { - ($imm8:expr) => { - vpsraiq(a, $imm8) - }; - } - let r = constify_imm8_sae!(imm8, call); + let r = vpsraiq(a, IMM8); transmute(r) } @@ -19286,16 +18094,16 @@ pub unsafe fn _mm512_srai_epi64(a: __m512i, imm8: u32) -> __m512i { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_srai_epi64&expand=5443) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vpsraq, imm8 = 1))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm512_mask_srai_epi64(src: __m512i, k: __mmask8, a: __m512i, imm8: u32) -> __m512i { +#[cfg_attr(test, assert_instr(vpsraq, IMM8 = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_mask_srai_epi64( + src: __m512i, + k: __mmask8, + a: __m512i, +) -> __m512i { + static_assert_imm_u8!(IMM8); let a = a.as_i64x8(); - macro_rules! call { - ($imm8:expr) => { - vpsraiq(a, $imm8) - }; - } - let shf = constify_imm8_sae!(imm8, call); + let shf = vpsraiq(a, IMM8); transmute(simd_select_bitmask(k, shf, src.as_i64x8())) } @@ -19304,16 +18112,12 @@ pub unsafe fn _mm512_mask_srai_epi64(src: __m512i, k: __mmask8, a: __m512i, imm8 /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_srai_epi64&expand=5444) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vpsraq, imm8 = 1))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm512_maskz_srai_epi64(k: __mmask8, a: __m512i, imm8: u32) -> __m512i { +#[cfg_attr(test, assert_instr(vpsraq, IMM8 = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_maskz_srai_epi64(k: __mmask8, a: __m512i) -> __m512i { + static_assert_imm_u8!(IMM8); let a = a.as_i64x8(); - macro_rules! call { - ($imm8:expr) => { - vpsraiq(a, $imm8) - }; - } - let shf = constify_imm8_sae!(imm8, call); + let shf = vpsraiq(a, IMM8); let zero = _mm512_setzero_si512().as_i64x8(); transmute(simd_select_bitmask(k, shf, zero)) } @@ -19323,16 +18127,12 @@ pub unsafe fn _mm512_maskz_srai_epi64(k: __mmask8, a: __m512i, imm8: u32) -> __m /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_srai_epi64&expand=5442) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[cfg_attr(test, assert_instr(vpsraq, imm8 = 1))] -#[rustc_args_required_const(1)] -pub unsafe fn _mm256_srai_epi64(a: __m256i, imm8: u32) -> __m256i { +#[cfg_attr(test, assert_instr(vpsraq, IMM8 = 1))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm256_srai_epi64(a: __m256i) -> __m256i { + static_assert_imm_u8!(IMM8); let a = a.as_i64x4(); - macro_rules! call { - ($imm8:expr) => { - vpsraiq256(a, $imm8) - }; - } - let r = constify_imm8_sae!(imm8, call); + let r = vpsraiq256(a, IMM8); transmute(r) } @@ -19341,16 +18141,16 @@ pub unsafe fn _mm256_srai_epi64(a: __m256i, imm8: u32) -> __m256i { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_srai_epi64&expand=5440) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[cfg_attr(test, assert_instr(vpsraq, imm8 = 1))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm256_mask_srai_epi64(src: __m256i, k: __mmask8, a: __m256i, imm8: u32) -> __m256i { +#[cfg_attr(test, assert_instr(vpsraq, IMM8 = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm256_mask_srai_epi64( + src: __m256i, + k: __mmask8, + a: __m256i, +) -> __m256i { + static_assert_imm_u8!(IMM8); let a = a.as_i64x4(); - macro_rules! call { - ($imm8:expr) => { - vpsraiq256(a, $imm8) - }; - } - let shf = constify_imm8_sae!(imm8, call); + let shf = vpsraiq256(a, IMM8); transmute(simd_select_bitmask(k, shf, src.as_i64x4())) } @@ -19359,16 +18159,12 @@ pub unsafe fn _mm256_mask_srai_epi64(src: __m256i, k: __mmask8, a: __m256i, imm8 /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_srai_epi64&expand=5441) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[cfg_attr(test, assert_instr(vpsraq, imm8 = 1))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm256_maskz_srai_epi64(k: __mmask8, a: __m256i, imm8: u32) -> __m256i { +#[cfg_attr(test, assert_instr(vpsraq, IMM8 = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm256_maskz_srai_epi64(k: __mmask8, a: __m256i) -> __m256i { + static_assert_imm_u8!(IMM8); let a = a.as_i64x4(); - macro_rules! call { - ($imm8:expr) => { - vpsraiq256(a, $imm8) - }; - } - let shf = constify_imm8_sae!(imm8, call); + let shf = vpsraiq256(a, IMM8); let zero = _mm256_setzero_si256().as_i64x4(); transmute(simd_select_bitmask(k, shf, zero)) } @@ -19378,16 +18174,12 @@ pub unsafe fn _mm256_maskz_srai_epi64(k: __mmask8, a: __m256i, imm8: u32) -> __m /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_srai_epi64&expand=5439) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[cfg_attr(test, assert_instr(vpsraq, imm8 = 1))] -#[rustc_args_required_const(1)] -pub unsafe fn _mm_srai_epi64(a: __m128i, imm8: u32) -> __m128i { +#[cfg_attr(test, assert_instr(vpsraq, IMM8 = 1))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm_srai_epi64(a: __m128i) -> __m128i { + static_assert_imm_u8!(IMM8); let a = a.as_i64x2(); - macro_rules! call { - ($imm8:expr) => { - vpsraiq128(a, $imm8) - }; - } - let r = constify_imm8_sae!(imm8, call); + let r = vpsraiq128(a, IMM8); transmute(r) } @@ -19396,16 +18188,16 @@ pub unsafe fn _mm_srai_epi64(a: __m128i, imm8: u32) -> __m128i { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_srai_epi64&expand=5437) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[cfg_attr(test, assert_instr(vpsraq, imm8 = 1))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm_mask_srai_epi64(src: __m128i, k: __mmask8, a: __m128i, imm8: u32) -> __m128i { +#[cfg_attr(test, assert_instr(vpsraq, IMM8 = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm_mask_srai_epi64( + src: __m128i, + k: __mmask8, + a: __m128i, +) -> __m128i { + static_assert_imm_u8!(IMM8); let a = a.as_i64x2(); - macro_rules! call { - ($imm8:expr) => { - vpsraiq128(a, $imm8) - }; - } - let shf = constify_imm8_sae!(imm8, call); + let shf = vpsraiq128(a, IMM8); transmute(simd_select_bitmask(k, shf, src.as_i64x2())) } @@ -19414,16 +18206,12 @@ pub unsafe fn _mm_mask_srai_epi64(src: __m128i, k: __mmask8, a: __m128i, imm8: u /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_srai_epi64&expand=5438) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[cfg_attr(test, assert_instr(vpsraq, imm8 = 1))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm_maskz_srai_epi64(k: __mmask8, a: __m128i, imm8: u32) -> __m128i { +#[cfg_attr(test, assert_instr(vpsraq, IMM8 = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm_maskz_srai_epi64(k: __mmask8, a: __m128i) -> __m128i { + static_assert_imm_u8!(IMM8); let a = a.as_i64x2(); - macro_rules! call { - ($imm8:expr) => { - vpsraiq128(a, $imm8) - }; - } - let shf = constify_imm8_sae!(imm8, call); + let shf = vpsraiq128(a, IMM8); let zero = _mm_setzero_si128().as_i64x2(); transmute(simd_select_bitmask(k, shf, zero)) } @@ -20423,73 +19211,32 @@ pub unsafe fn _mm_maskz_srlv_epi64(k: __mmask8, a: __m128i, count: __m128i) -> _ /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_permute_ps&expand=4170) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vpermilps, imm8 = 1))] -#[rustc_args_required_const(1)] -pub unsafe fn _mm512_permute_ps(a: __m512, imm8: i32) -> __m512 { - let imm8 = (imm8 & 0xFF) as u8; - let undefined = _mm512_undefined_ps(); - macro_rules! shuffle4 { - ($a:expr, $b:expr, $c:expr, $d:expr) => { - simd_shuffle16( - a, - undefined, - [ - $a, - $b, - $c, - $d, - $a + 4, - $b + 4, - $c + 4, - $d + 4, - $a + 8, - $b + 8, - $c + 8, - $d + 8, - $a + 12, - $b + 12, - $c + 12, - $d + 12, - ], - ) - }; - } - macro_rules! shuffle3 { - ($a:expr, $b:expr, $c:expr) => { - match (imm8 >> 6) & 0b11 { - 0b00 => shuffle4!($a, $b, $c, 0), - 0b01 => shuffle4!($a, $b, $c, 1), - 0b10 => shuffle4!($a, $b, $c, 2), - _ => shuffle4!($a, $b, $c, 3), - } - }; - } - macro_rules! shuffle2 { - ($a:expr, $b:expr) => { - match (imm8 >> 4) & 0b11 { - 0b00 => shuffle3!($a, $b, 0), - 0b01 => shuffle3!($a, $b, 1), - 0b10 => shuffle3!($a, $b, 2), - _ => shuffle3!($a, $b, 3), - } - }; - } - macro_rules! shuffle1 { - ($a:expr) => { - match (imm8 >> 2) & 0b11 { - 0b00 => shuffle2!($a, 0), - 0b01 => shuffle2!($a, 1), - 0b10 => shuffle2!($a, 2), - _ => shuffle2!($a, 3), - } - }; - } - match imm8 & 0b11 { - 0b00 => shuffle1!(0), - 0b01 => shuffle1!(1), - 0b10 => shuffle1!(2), - _ => shuffle1!(3), - } +#[cfg_attr(test, assert_instr(vpermilps, MASK = 0b11_00_01_11))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm512_permute_ps(a: __m512) -> __m512 { + static_assert_imm8!(MASK); + simd_shuffle16!( + a, + a, + [ + MASK as u32 & 0b11, + (MASK as u32 >> 2) & 0b11, + ((MASK as u32 >> 4) & 0b11), + ((MASK as u32 >> 6) & 0b11), + (MASK as u32 & 0b11) + 4, + ((MASK as u32 >> 2) & 0b11) + 4, + ((MASK as u32 >> 4) & 0b11) + 4, + ((MASK as u32 >> 6) & 0b11) + 4, + (MASK as u32 & 0b11) + 8, + ((MASK as u32 >> 2) & 0b11) + 8, + ((MASK as u32 >> 4) & 0b11) + 8, + ((MASK as u32 >> 6) & 0b11) + 8, + (MASK as u32 & 0b11) + 12, + ((MASK as u32 >> 2) & 0b11) + 12, + ((MASK as u32 >> 4) & 0b11) + 12, + ((MASK as u32 >> 6) & 0b11) + 12, + ], + ) } /// Shuffle single-precision (32-bit) floating-point elements in a within 128-bit lanes using the control in imm8, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). @@ -20497,15 +19244,15 @@ pub unsafe fn _mm512_permute_ps(a: __m512, imm8: i32) -> __m512 { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_permute_ps&expand=4168) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vpermilps, imm8 = 1))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm512_mask_permute_ps(src: __m512, k: __mmask16, a: __m512, imm8: i32) -> __m512 { - macro_rules! call { - ($imm8:expr) => { - _mm512_permute_ps(a, $imm8) - }; - } - let r = constify_imm8_sae!(imm8, call); +#[cfg_attr(test, assert_instr(vpermilps, MASK = 0b11_00_01_11))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_mask_permute_ps( + src: __m512, + k: __mmask16, + a: __m512, +) -> __m512 { + static_assert_imm8!(MASK); + let r = _mm512_permute_ps::(a); transmute(simd_select_bitmask(k, r.as_f32x16(), src.as_f32x16())) } @@ -20514,15 +19261,11 @@ pub unsafe fn _mm512_mask_permute_ps(src: __m512, k: __mmask16, a: __m512, imm8: /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_permute_ps&expand=4169) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vpermilps, imm8 = 1))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm512_maskz_permute_ps(k: __mmask16, a: __m512, imm8: i32) -> __m512 { - macro_rules! call { - ($imm8:expr) => { - _mm512_permute_ps(a, $imm8) - }; - } - let r = constify_imm8_sae!(imm8, call); +#[cfg_attr(test, assert_instr(vpermilps, MASK = 0b11_00_01_11))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_maskz_permute_ps(k: __mmask16, a: __m512) -> __m512 { + static_assert_imm8!(MASK); + let r = _mm512_permute_ps::(a); let zero = _mm512_setzero_ps().as_f32x16(); transmute(simd_select_bitmask(k, r.as_f32x16(), zero)) } @@ -20532,15 +19275,14 @@ pub unsafe fn _mm512_maskz_permute_ps(k: __mmask16, a: __m512, imm8: i32) -> __m /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_permute_ps&expand=4165) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[cfg_attr(test, assert_instr(vpermilps, imm8 = 1))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm256_mask_permute_ps(src: __m256, k: __mmask8, a: __m256, imm8: i32) -> __m256 { - macro_rules! call { - ($imm8:expr) => { - _mm256_permute_ps(a, $imm8) - }; - } - let r = constify_imm8_sae!(imm8, call); +#[cfg_attr(test, assert_instr(vpermilps, MASK = 0b11_00_01_11))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm256_mask_permute_ps( + src: __m256, + k: __mmask8, + a: __m256, +) -> __m256 { + let r = _mm256_permute_ps::(a); transmute(simd_select_bitmask(k, r.as_f32x8(), src.as_f32x8())) } @@ -20549,15 +19291,10 @@ pub unsafe fn _mm256_mask_permute_ps(src: __m256, k: __mmask8, a: __m256, imm8: /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_permute_ps&expand=4166) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[cfg_attr(test, assert_instr(vpermilps, imm8 = 1))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm256_maskz_permute_ps(k: __mmask8, a: __m256, imm8: i32) -> __m256 { - macro_rules! call { - ($imm8:expr) => { - _mm256_permute_ps(a, $imm8) - }; - } - let r = constify_imm8_sae!(imm8, call); +#[cfg_attr(test, assert_instr(vpermilps, MASK = 0b11_00_01_11))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm256_maskz_permute_ps(k: __mmask8, a: __m256) -> __m256 { + let r = _mm256_permute_ps::(a); let zero = _mm256_setzero_ps().as_f32x8(); transmute(simd_select_bitmask(k, r.as_f32x8(), zero)) } @@ -20567,15 +19304,10 @@ pub unsafe fn _mm256_maskz_permute_ps(k: __mmask8, a: __m256, imm8: i32) -> __m2 /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_permute_ps&expand=4162) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[cfg_attr(test, assert_instr(vpermilps, imm8 = 1))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm_mask_permute_ps(src: __m128, k: __mmask8, a: __m128, imm8: i32) -> __m128 { - macro_rules! call { - ($imm8:expr) => { - _mm_permute_ps(a, $imm8) - }; - } - let r = constify_imm8_sae!(imm8, call); +#[cfg_attr(test, assert_instr(vpermilps, MASK = 0b11_00_01_11))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm_mask_permute_ps(src: __m128, k: __mmask8, a: __m128) -> __m128 { + let r = _mm_permute_ps::(a); transmute(simd_select_bitmask(k, r.as_f32x4(), src.as_f32x4())) } @@ -20584,15 +19316,10 @@ pub unsafe fn _mm_mask_permute_ps(src: __m128, k: __mmask8, a: __m128, imm8: i32 /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_permute_ps&expand=4163) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[cfg_attr(test, assert_instr(vpermilps, imm8 = 1))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm_maskz_permute_ps(k: __mmask8, a: __m128, imm8: i32) -> __m128 { - macro_rules! call { - ($imm8:expr) => { - _mm_permute_ps(a, $imm8) - }; - } - let r = constify_imm8_sae!(imm8, call); +#[cfg_attr(test, assert_instr(vpermilps, MASK = 0b11_00_01_11))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm_maskz_permute_ps(k: __mmask8, a: __m128) -> __m128 { + let r = _mm_permute_ps::(a); let zero = _mm_setzero_ps().as_f32x4(); transmute(simd_select_bitmask(k, r.as_f32x4(), zero)) } @@ -20602,48 +19329,24 @@ pub unsafe fn _mm_maskz_permute_ps(k: __mmask8, a: __m128, imm8: i32) -> __m128 /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_permute_pd&expand=4161) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vpermilpd, imm8 = 2))] -#[rustc_args_required_const(1)] -pub unsafe fn _mm512_permute_pd(a: __m512d, imm8: i32) -> __m512d { - let imm8 = (imm8 & 0xFF) as u8; - let undefined = _mm512_undefined_pd(); - macro_rules! shuffle4 { - ($a:expr, $b:expr, $c:expr, $d:expr) => { - simd_shuffle8( - a, - undefined, - [$a, $b, $c, $d, $a + 4, $b + 4, $c + 4, $d + 4], - ) - }; - } - macro_rules! shuffle3 { - ($a:expr, $b:expr, $c:expr) => { - match (imm8 >> 3) & 0x1 { - 0 => shuffle4!($a, $b, $c, 2), - _ => shuffle4!($a, $b, $c, 3), - } - }; - } - macro_rules! shuffle2 { - ($a:expr, $b:expr) => { - match (imm8 >> 2) & 0x1 { - 0 => shuffle3!($a, $b, 2), - _ => shuffle3!($a, $b, 3), - } - }; - } - macro_rules! shuffle1 { - ($a:expr) => { - match (imm8 >> 1) & 0x1 { - 0 => shuffle2!($a, 0), - _ => shuffle2!($a, 1), - } - }; - } - match imm8 & 0x1 { - 0 => shuffle1!(0), - _ => shuffle1!(1), - } +#[cfg_attr(test, assert_instr(vpermilpd, MASK = 0b11_01_10_01))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm512_permute_pd(a: __m512d) -> __m512d { + static_assert_imm8!(MASK); + simd_shuffle8!( + a, + a, + [ + MASK as u32 & 0b1, + ((MASK as u32 >> 1) & 0b1), + ((MASK as u32 >> 2) & 0b1) + 2, + ((MASK as u32 >> 3) & 0b1) + 2, + ((MASK as u32 >> 4) & 0b1) + 4, + ((MASK as u32 >> 5) & 0b1) + 4, + ((MASK as u32 >> 6) & 0b1) + 6, + ((MASK as u32 >> 7) & 0b1) + 6, + ], + ) } /// Shuffle double-precision (64-bit) floating-point elements in a within 128-bit lanes using the control in imm8, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). @@ -20651,15 +19354,15 @@ pub unsafe fn _mm512_permute_pd(a: __m512d, imm8: i32) -> __m512d { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_permute_pd&expand=4159) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vpermilpd, imm8 = 2))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm512_mask_permute_pd(src: __m512d, k: __mmask8, a: __m512d, imm8: i32) -> __m512d { - macro_rules! call { - ($imm8:expr) => { - _mm512_permute_pd(a, $imm8) - }; - } - let r = constify_imm8_sae!(imm8, call); +#[cfg_attr(test, assert_instr(vpermilpd, MASK = 0b11_01_10_01))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_mask_permute_pd( + src: __m512d, + k: __mmask8, + a: __m512d, +) -> __m512d { + static_assert_imm8!(MASK); + let r = _mm512_permute_pd::(a); transmute(simd_select_bitmask(k, r.as_f64x8(), src.as_f64x8())) } @@ -20668,15 +19371,11 @@ pub unsafe fn _mm512_mask_permute_pd(src: __m512d, k: __mmask8, a: __m512d, imm8 /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_permute_pd&expand=4160) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vpermilpd, imm8 = 2))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm512_maskz_permute_pd(k: __mmask8, a: __m512d, imm8: i32) -> __m512d { - macro_rules! call { - ($imm8:expr) => { - _mm512_permute_pd(a, $imm8) - }; - } - let r = constify_imm8_sae!(imm8, call); +#[cfg_attr(test, assert_instr(vpermilpd, MASK = 0b11_01_10_01))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_maskz_permute_pd(k: __mmask8, a: __m512d) -> __m512d { + static_assert_imm8!(MASK); + let r = _mm512_permute_pd::(a); let zero = _mm512_setzero_pd().as_f64x8(); transmute(simd_select_bitmask(k, r.as_f64x8(), zero)) } @@ -20686,15 +19385,15 @@ pub unsafe fn _mm512_maskz_permute_pd(k: __mmask8, a: __m512d, imm8: i32) -> __m /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_permute_pd&expand=4156) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[cfg_attr(test, assert_instr(vpermilpd, imm8 = 2))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm256_mask_permute_pd(src: __m256d, k: __mmask8, a: __m256d, imm8: i32) -> __m256d { - macro_rules! call { - ($imm8:expr) => { - _mm256_permute_pd(a, $imm8) - }; - } - let r = constify_imm8_sae!(imm8, call); +#[cfg_attr(test, assert_instr(vpermilpd, MASK = 0b11_01))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm256_mask_permute_pd( + src: __m256d, + k: __mmask8, + a: __m256d, +) -> __m256d { + static_assert_imm4!(MASK); + let r = _mm256_permute_pd::(a); transmute(simd_select_bitmask(k, r.as_f64x4(), src.as_f64x4())) } @@ -20703,15 +19402,11 @@ pub unsafe fn _mm256_mask_permute_pd(src: __m256d, k: __mmask8, a: __m256d, imm8 /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_permute_pd&expand=4157) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[cfg_attr(test, assert_instr(vpermilpd, imm8 = 2))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm256_maskz_permute_pd(k: __mmask8, a: __m256d, imm8: i32) -> __m256d { - macro_rules! call { - ($imm8:expr) => { - _mm256_permute_pd(a, $imm8) - }; - } - let r = constify_imm8_sae!(imm8, call); +#[cfg_attr(test, assert_instr(vpermilpd, MASK = 0b11_01))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm256_maskz_permute_pd(k: __mmask8, a: __m256d) -> __m256d { + static_assert_imm4!(MASK); + let r = _mm256_permute_pd::(a); let zero = _mm256_setzero_pd().as_f64x4(); transmute(simd_select_bitmask(k, r.as_f64x4(), zero)) } @@ -20721,15 +19416,15 @@ pub unsafe fn _mm256_maskz_permute_pd(k: __mmask8, a: __m256d, imm8: i32) -> __m /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_permute_pd&expand=4153) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[cfg_attr(test, assert_instr(vpermilpd, imm8 = 0b01))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm_mask_permute_pd(src: __m128d, k: __mmask8, a: __m128d, imm8: i32) -> __m128d { - macro_rules! call { - ($imm8:expr) => { - _mm_permute_pd(a, $imm8) - }; - } - let r = constify_imm8_sae!(imm8, call); +#[cfg_attr(test, assert_instr(vpermilpd, IMM2 = 0b01))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm_mask_permute_pd( + src: __m128d, + k: __mmask8, + a: __m128d, +) -> __m128d { + static_assert_imm2!(IMM2); + let r = _mm_permute_pd::(a); transmute(simd_select_bitmask(k, r.as_f64x2(), src.as_f64x2())) } @@ -20738,15 +19433,11 @@ pub unsafe fn _mm_mask_permute_pd(src: __m128d, k: __mmask8, a: __m128d, imm8: i /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_permute_pd&expand=4154) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[cfg_attr(test, assert_instr(vpermilpd, imm8 = 0b01))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm_maskz_permute_pd(k: __mmask8, a: __m128d, imm8: i32) -> __m128d { - macro_rules! call { - ($imm8:expr) => { - _mm_permute_pd(a, $imm8) - }; - } - let r = constify_imm8_sae!(imm8, call); +#[cfg_attr(test, assert_instr(vpermilpd, IMM2 = 0b01))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm_maskz_permute_pd(k: __mmask8, a: __m128d) -> __m128d { + static_assert_imm2!(IMM2); + let r = _mm_permute_pd::(a); let zero = _mm_setzero_pd().as_f64x2(); transmute(simd_select_bitmask(k, r.as_f64x2(), zero)) } @@ -20756,56 +19447,24 @@ pub unsafe fn _mm_maskz_permute_pd(k: __mmask8, a: __m128d, imm8: i32) -> __m128 /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_permutex_epi64&expand=4208) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vperm, imm8 = 0b10011011))] //shoud be vpermq -#[rustc_args_required_const(1)] -pub unsafe fn _mm512_permutex_epi64(a: __m512i, imm8: i32) -> __m512i { - let imm8 = (imm8 & 0xFF) as u8; - let undefined = _mm512_set1_epi64(0); - macro_rules! shuffle4 { - ($a:expr, $b:expr, $c:expr, $d:expr) => { - simd_shuffle8( - a, - undefined, - [$a, $b, $c, $d, $a + 4, $b + 4, $c + 4, $d + 4], - ) - }; - } - macro_rules! shuffle3 { - ($a:expr, $b:expr, $c:expr) => { - match (imm8 >> 6) & 0b11 { - 0b00 => shuffle4!($a, $b, $c, 0), - 0b01 => shuffle4!($a, $b, $c, 1), - 0b10 => shuffle4!($a, $b, $c, 2), - _ => shuffle4!($a, $b, $c, 3), - } - }; - } - macro_rules! shuffle2 { - ($a:expr, $b:expr) => { - match (imm8 >> 4) & 0b11 { - 0b00 => shuffle3!($a, $b, 0), - 0b01 => shuffle3!($a, $b, 1), - 0b10 => shuffle3!($a, $b, 2), - _ => shuffle3!($a, $b, 3), - } - }; - } - macro_rules! shuffle1 { - ($a:expr) => { - match (imm8 >> 2) & 0b11 { - 0b00 => shuffle2!($a, 0), - 0b01 => shuffle2!($a, 1), - 0b10 => shuffle2!($a, 2), - _ => shuffle2!($a, 3), - } - }; - } - match imm8 & 0b11 { - 0b00 => shuffle1!(0), - 0b01 => shuffle1!(1), - 0b10 => shuffle1!(2), - _ => shuffle1!(3), - } +#[cfg_attr(test, assert_instr(vperm, MASK = 0b10_01_10_11))] //shoud be vpermq +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm512_permutex_epi64(a: __m512i) -> __m512i { + static_assert_imm8!(MASK); + simd_shuffle8!( + a, + a, + [ + MASK as u32 & 0b11, + (MASK as u32 >> 2) & 0b11, + ((MASK as u32 >> 4) & 0b11), + ((MASK as u32 >> 6) & 0b11), + (MASK as u32 & 0b11) + 4, + ((MASK as u32 >> 2) & 0b11) + 4, + ((MASK as u32 >> 4) & 0b11) + 4, + ((MASK as u32 >> 6) & 0b11) + 4, + ], + ) } /// Shuffle 64-bit integers in a within 256-bit lanes using the control in imm8, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). @@ -20813,20 +19472,15 @@ pub unsafe fn _mm512_permutex_epi64(a: __m512i, imm8: i32) -> __m512i { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_permutex_epi64&expand=4206) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vperm, imm8 = 0b11111111))] //should be vpermq -#[rustc_args_required_const(3)] -pub unsafe fn _mm512_mask_permutex_epi64( +#[cfg_attr(test, assert_instr(vperm, MASK = 0b10_01_10_11))] //should be vpermq +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_mask_permutex_epi64( src: __m512i, k: __mmask8, a: __m512i, - imm8: i32, ) -> __m512i { - macro_rules! call { - ($imm8:expr) => { - _mm512_permutex_epi64(a, $imm8) - }; - } - let r = constify_imm8_sae!(imm8, call); + static_assert_imm8!(MASK); + let r = _mm512_permutex_epi64::(a); transmute(simd_select_bitmask(k, r.as_i64x8(), src.as_i64x8())) } @@ -20835,15 +19489,11 @@ pub unsafe fn _mm512_mask_permutex_epi64( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_permutex_epi64&expand=4207) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vperm, imm8 = 0b11111111))] //should be vpermq -#[rustc_args_required_const(2)] -pub unsafe fn _mm512_maskz_permutex_epi64(k: __mmask8, a: __m512i, imm8: i32) -> __m512i { - macro_rules! call { - ($imm8:expr) => { - _mm512_permutex_epi64(a, $imm8) - }; - } - let r = constify_imm8_sae!(imm8, call); +#[cfg_attr(test, assert_instr(vperm, MASK = 0b10_01_10_11))] //should be vpermq +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_maskz_permutex_epi64(k: __mmask8, a: __m512i) -> __m512i { + static_assert_imm8!(MASK); + let r = _mm512_permutex_epi64::(a); let zero = _mm512_setzero_si512().as_i64x8(); transmute(simd_select_bitmask(k, r.as_i64x8(), zero)) } @@ -20853,52 +19503,20 @@ pub unsafe fn _mm512_maskz_permutex_epi64(k: __mmask8, a: __m512i, imm8: i32) -> /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_permutex_epi64&expand=4205) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[cfg_attr(test, assert_instr(vperm, imm8 = 0b10011011))] //shoud be vpermq -#[rustc_args_required_const(1)] -pub unsafe fn _mm256_permutex_epi64(a: __m256i, imm8: i32) -> __m256i { - let imm8 = (imm8 & 0xFF) as u8; - let undefined = _mm256_set1_epi64x(0); - macro_rules! shuffle4 { - ($a:expr, $b:expr, $c:expr, $d:expr) => { - simd_shuffle4(a, undefined, [$a, $b, $c, $d]) - }; - } - macro_rules! shuffle3 { - ($a:expr, $b:expr, $c:expr) => { - match (imm8 >> 6) & 0b11 { - 0b00 => shuffle4!($a, $b, $c, 0), - 0b01 => shuffle4!($a, $b, $c, 1), - 0b10 => shuffle4!($a, $b, $c, 2), - _ => shuffle4!($a, $b, $c, 3), - } - }; - } - macro_rules! shuffle2 { - ($a:expr, $b:expr) => { - match (imm8 >> 4) & 0b11 { - 0b00 => shuffle3!($a, $b, 0), - 0b01 => shuffle3!($a, $b, 1), - 0b10 => shuffle3!($a, $b, 2), - _ => shuffle3!($a, $b, 3), - } - }; - } - macro_rules! shuffle1 { - ($a:expr) => { - match (imm8 >> 2) & 0b11 { - 0b00 => shuffle2!($a, 0), - 0b01 => shuffle2!($a, 1), - 0b10 => shuffle2!($a, 2), - _ => shuffle2!($a, 3), - } - }; - } - match imm8 & 0b11 { - 0b00 => shuffle1!(0), - 0b01 => shuffle1!(1), - 0b10 => shuffle1!(2), - _ => shuffle1!(3), - } +#[cfg_attr(test, assert_instr(vperm, MASK = 0b10_01_10_11))] //shoud be vpermq +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm256_permutex_epi64(a: __m256i) -> __m256i { + static_assert_imm8!(MASK); + simd_shuffle4!( + a, + a, + [ + MASK as u32 & 0b11, + (MASK as u32 >> 2) & 0b11, + ((MASK as u32 >> 4) & 0b11), + ((MASK as u32 >> 6) & 0b11), + ], + ) } /// Shuffle 64-bit integers in a within 256-bit lanes using the control in imm8, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). @@ -20906,20 +19524,15 @@ pub unsafe fn _mm256_permutex_epi64(a: __m256i, imm8: i32) -> __m256i { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_permutex_epi6&expand=4203) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[cfg_attr(test, assert_instr(vperm, imm8 = 0b10011011))] //should be vpermq -#[rustc_args_required_const(3)] -pub unsafe fn _mm256_mask_permutex_epi64( +#[cfg_attr(test, assert_instr(vperm, MASK = 0b10_01_10_11))] //should be vpermq +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm256_mask_permutex_epi64( src: __m256i, k: __mmask8, a: __m256i, - imm8: i32, ) -> __m256i { - macro_rules! call { - ($imm8:expr) => { - _mm256_permutex_epi64(a, $imm8) - }; - } - let r = constify_imm8_sae!(imm8, call); + static_assert_imm8!(MASK); + let r = _mm256_permutex_epi64::(a); transmute(simd_select_bitmask(k, r.as_i64x4(), src.as_i64x4())) } @@ -20928,15 +19541,11 @@ pub unsafe fn _mm256_mask_permutex_epi64( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_permutex_epi64&expand=4204) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[cfg_attr(test, assert_instr(vperm, imm8 = 0b10011011))] //should be vpermq -#[rustc_args_required_const(2)] -pub unsafe fn _mm256_maskz_permutex_epi64(k: __mmask8, a: __m256i, imm8: i32) -> __m256i { - macro_rules! call { - ($imm8:expr) => { - _mm256_permutex_epi64(a, $imm8) - }; - } - let r = constify_imm8_sae!(imm8, call); +#[cfg_attr(test, assert_instr(vperm, MASK = 0b10_01_10_11))] //should be vpermq +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm256_maskz_permutex_epi64(k: __mmask8, a: __m256i) -> __m256i { + static_assert_imm8!(MASK); + let r = _mm256_permutex_epi64::(a); let zero = _mm256_setzero_si256().as_i64x4(); transmute(simd_select_bitmask(k, r.as_i64x4(), zero)) } @@ -20946,56 +19555,24 @@ pub unsafe fn _mm256_maskz_permutex_epi64(k: __mmask8, a: __m256i, imm8: i32) -> /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_permutex_pd&expand=4214) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vperm, imm8 = 0b11111111))] //shoud be vpermpd -#[rustc_args_required_const(1)] -pub unsafe fn _mm512_permutex_pd(a: __m512d, imm8: i32) -> __m512d { - let imm8 = (imm8 & 0xFF) as u8; - let undefined = _mm512_undefined_pd(); - macro_rules! shuffle4 { - ($a:expr, $b:expr, $c:expr, $d:expr) => { - simd_shuffle8( - a, - undefined, - [$a, $b, $c, $d, $a + 4, $b + 4, $c + 4, $d + 4], - ) - }; - } - macro_rules! shuffle3 { - ($a:expr, $b:expr, $c:expr) => { - match (imm8 >> 6) & 0b11 { - 0b00 => shuffle4!($a, $b, $c, 0), - 0b01 => shuffle4!($a, $b, $c, 1), - 0b10 => shuffle4!($a, $b, $c, 2), - _ => shuffle4!($a, $b, $c, 3), - } - }; - } - macro_rules! shuffle2 { - ($a:expr, $b:expr) => { - match (imm8 >> 4) & 0b11 { - 0b00 => shuffle3!($a, $b, 0), - 0b01 => shuffle3!($a, $b, 1), - 0b10 => shuffle3!($a, $b, 2), - _ => shuffle3!($a, $b, 3), - } - }; - } - macro_rules! shuffle1 { - ($a:expr) => { - match (imm8 >> 2) & 0b11 { - 0b00 => shuffle2!($a, 0), - 0b01 => shuffle2!($a, 1), - 0b10 => shuffle2!($a, 2), - _ => shuffle2!($a, 3), - } - }; - } - match imm8 & 0b11 { - 0b00 => shuffle1!(0), - 0b01 => shuffle1!(1), - 0b10 => shuffle1!(2), - _ => shuffle1!(3), - } +#[cfg_attr(test, assert_instr(vperm, MASK = 0b10_01_10_11))] //shoud be vpermpd +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm512_permutex_pd(a: __m512d) -> __m512d { + static_assert_imm8!(MASK); + simd_shuffle8!( + a, + a, + [ + MASK as u32 & 0b11, + (MASK as u32 >> 2) & 0b11, + ((MASK as u32 >> 4) & 0b11), + ((MASK as u32 >> 6) & 0b11), + (MASK as u32 & 0b11) + 4, + ((MASK as u32 >> 2) & 0b11) + 4, + ((MASK as u32 >> 4) & 0b11) + 4, + ((MASK as u32 >> 6) & 0b11) + 4, + ], + ) } /// Shuffle double-precision (64-bit) floating-point elements in a within 256-bit lanes using the control in imm8, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). @@ -21003,15 +19580,14 @@ pub unsafe fn _mm512_permutex_pd(a: __m512d, imm8: i32) -> __m512d { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_permutex_pd&expand=4212) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vperm, imm8 = 0b11111111))] //shoud be vpermpd -#[rustc_args_required_const(3)] -pub unsafe fn _mm512_mask_permutex_pd(src: __m512d, k: __mmask8, a: __m512d, imm8: i32) -> __m512d { - macro_rules! call { - ($imm8:expr) => { - _mm512_permutex_pd(a, $imm8) - }; - } - let r = constify_imm8_sae!(imm8, call); +#[cfg_attr(test, assert_instr(vperm, MASK = 0b10_01_10_11))] //shoud be vpermpd +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_mask_permutex_pd( + src: __m512d, + k: __mmask8, + a: __m512d, +) -> __m512d { + let r = _mm512_permutex_pd::(a); transmute(simd_select_bitmask(k, r.as_f64x8(), src.as_f64x8())) } @@ -21020,15 +19596,10 @@ pub unsafe fn _mm512_mask_permutex_pd(src: __m512d, k: __mmask8, a: __m512d, imm /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_permutex_pd&expand=4213) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vperm, imm8 = 0b11111111))] //shoud be vpermpd -#[rustc_args_required_const(2)] -pub unsafe fn _mm512_maskz_permutex_pd(k: __mmask8, a: __m512d, imm8: i32) -> __m512d { - macro_rules! call { - ($imm8:expr) => { - _mm512_permutex_pd(a, $imm8) - }; - } - let r = constify_imm8_sae!(imm8, call); +#[cfg_attr(test, assert_instr(vperm, MASK = 0b10_01_10_11))] //shoud be vpermpd +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_maskz_permutex_pd(k: __mmask8, a: __m512d) -> __m512d { + let r = _mm512_permutex_pd::(a); let zero = _mm512_setzero_pd().as_f64x8(); transmute(simd_select_bitmask(k, r.as_f64x8(), zero)) } @@ -21038,52 +19609,20 @@ pub unsafe fn _mm512_maskz_permutex_pd(k: __mmask8, a: __m512d, imm8: i32) -> __ /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_permutex_pd&expand=4211) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[cfg_attr(test, assert_instr(vperm, imm8 = 0b10011011))] //shoud be vpermpd -#[rustc_args_required_const(1)] -pub unsafe fn _mm256_permutex_pd(a: __m256d, imm8: i32) -> __m256d { - let imm8 = (imm8 & 0xFF) as u8; - let undefined = _mm256_undefined_pd(); - macro_rules! shuffle4 { - ($a:expr, $b:expr, $c:expr, $d:expr) => { - simd_shuffle4(a, undefined, [$a, $b, $c, $d]) - }; - } - macro_rules! shuffle3 { - ($a:expr, $b:expr, $c:expr) => { - match (imm8 >> 6) & 0b11 { - 0b00 => shuffle4!($a, $b, $c, 0), - 0b01 => shuffle4!($a, $b, $c, 1), - 0b10 => shuffle4!($a, $b, $c, 2), - _ => shuffle4!($a, $b, $c, 3), - } - }; - } - macro_rules! shuffle2 { - ($a:expr, $b:expr) => { - match (imm8 >> 4) & 0b11 { - 0b00 => shuffle3!($a, $b, 0), - 0b01 => shuffle3!($a, $b, 1), - 0b10 => shuffle3!($a, $b, 2), - _ => shuffle3!($a, $b, 3), - } - }; - } - macro_rules! shuffle1 { - ($a:expr) => { - match (imm8 >> 2) & 0b11 { - 0b00 => shuffle2!($a, 0), - 0b01 => shuffle2!($a, 1), - 0b10 => shuffle2!($a, 2), - _ => shuffle2!($a, 3), - } - }; - } - match imm8 & 0b11 { - 0b00 => shuffle1!(0), - 0b01 => shuffle1!(1), - 0b10 => shuffle1!(2), - _ => shuffle1!(3), - } +#[cfg_attr(test, assert_instr(vperm, MASK = 0b10_01_10_11))] //shoud be vpermpd +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm256_permutex_pd(a: __m256d) -> __m256d { + static_assert_imm8!(MASK); + simd_shuffle4!( + a, + a, + [ + MASK as u32 & 0b11, + (MASK as u32 >> 2) & 0b11, + ((MASK as u32 >> 4) & 0b11), + ((MASK as u32 >> 6) & 0b11), + ], + ) } /// Shuffle double-precision (64-bit) floating-point elements in a within 256-bit lanes using the control in imm8, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). @@ -21091,15 +19630,15 @@ pub unsafe fn _mm256_permutex_pd(a: __m256d, imm8: i32) -> __m256d { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_permutex_pd&expand=4209) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[cfg_attr(test, assert_instr(vperm, imm8 = 0b10011011))] //shoud be vpermpd -#[rustc_args_required_const(3)] -pub unsafe fn _mm256_mask_permutex_pd(src: __m256d, k: __mmask8, a: __m256d, imm8: i32) -> __m256d { - macro_rules! call { - ($imm8:expr) => { - _mm256_permutex_pd(a, $imm8) - }; - } - let r = constify_imm8_sae!(imm8, call); +#[cfg_attr(test, assert_instr(vperm, MASK = 0b10_01_10_11))] //shoud be vpermpd +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm256_mask_permutex_pd( + src: __m256d, + k: __mmask8, + a: __m256d, +) -> __m256d { + static_assert_imm8!(MASK); + let r = _mm256_permutex_pd::(a); transmute(simd_select_bitmask(k, r.as_f64x4(), src.as_f64x4())) } @@ -21108,15 +19647,11 @@ pub unsafe fn _mm256_mask_permutex_pd(src: __m256d, k: __mmask8, a: __m256d, imm /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_permutex_pd&expand=4210) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[cfg_attr(test, assert_instr(vperm, imm8 = 0b10011011))] //shoud be vpermpd -#[rustc_args_required_const(2)] -pub unsafe fn _mm256_maskz_permutex_pd(k: __mmask8, a: __m256d, imm8: i32) -> __m256d { - macro_rules! call { - ($imm8:expr) => { - _mm256_permutex_pd(a, $imm8) - }; - } - let r = constify_imm8_sae!(imm8, call); +#[cfg_attr(test, assert_instr(vperm, MASK = 0b10_01_10_11))] //shoud be vpermpd +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm256_maskz_permutex_pd(k: __mmask8, a: __m256d) -> __m256d { + static_assert_imm8!(MASK); + let r = _mm256_permutex_pd::(a); let zero = _mm256_setzero_pd().as_f64x4(); transmute(simd_select_bitmask(k, r.as_f64x4(), zero)) } @@ -22328,75 +20863,32 @@ pub unsafe fn _mm_mask2_permutex2var_pd( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=512_shuffle_epi32&expand=5150) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vpermilps, imm8 = 9))] //should be vpshufd, but generate vpermilps -#[rustc_args_required_const(1)] -pub unsafe fn _mm512_shuffle_epi32(a: __m512i, imm8: _MM_PERM_ENUM) -> __m512i { - let imm8 = (imm8 & 0xFF) as u8; - let a = a.as_i32x16(); - macro_rules! shuffle4 { - ( - $a:expr, - $b:expr, - $c:expr, - $d:expr, - $e:expr, - $f:expr, - $g:expr, - $h:expr, - $i:expr, - $j:expr, - $k:expr, - $l:expr, - $m:expr, - $n:expr, - $o:expr, - $p:expr - ) => { - simd_shuffle16( - a, - a, - [ - $a, $b, $c, $d, $e, $f, $g, $h, $i, $j, $k, $l, $m, $n, $o, $p, - ], - ) - }; - } - macro_rules! shuffle3 { - ($a:expr, $b:expr, $c:expr, $e:expr, $f:expr, $g:expr, $i:expr, $j:expr, $k:expr, $m:expr, $n:expr, $o:expr) => { - match (imm8 >> 6) & 0x3 { - 0 => shuffle4!($a, $b, $c, 16, $e, $f, $g, 20, $i, $j, $k, 24, $m, $n, $o, 28), - 1 => shuffle4!($a, $b, $c, 17, $e, $f, $g, 21, $i, $j, $k, 25, $m, $n, $o, 29), - 2 => shuffle4!($a, $b, $c, 18, $e, $f, $g, 22, $i, $j, $k, 26, $m, $n, $o, 30), - _ => shuffle4!($a, $b, $c, 19, $e, $f, $g, 23, $i, $j, $k, 27, $m, $n, $o, 31), - } - }; - } - macro_rules! shuffle2 { - ($a:expr, $b:expr, $e:expr, $f:expr, $i:expr, $j:expr, $m:expr, $n:expr) => { - match (imm8 >> 4) & 0x3 { - 0 => shuffle3!($a, $b, 16, $e, $f, 20, $i, $j, 24, $m, $n, 28), - 1 => shuffle3!($a, $b, 17, $e, $f, 21, $i, $j, 25, $m, $n, 29), - 2 => shuffle3!($a, $b, 18, $e, $f, 22, $i, $j, 26, $m, $n, 30), - _ => shuffle3!($a, $b, 19, $e, $f, 23, $i, $j, 27, $m, $n, 31), - } - }; - } - macro_rules! shuffle1 { - ($a:expr, $e:expr, $i: expr, $m: expr) => { - match (imm8 >> 2) & 0x3 { - 0 => shuffle2!($a, 0, $e, 4, $i, 8, $m, 12), - 1 => shuffle2!($a, 1, $e, 5, $i, 9, $m, 13), - 2 => shuffle2!($a, 2, $e, 6, $i, 10, $m, 14), - _ => shuffle2!($a, 3, $e, 7, $i, 11, $m, 15), - } - }; - } - let r: i32x16 = match imm8 & 0x3 { - 0 => shuffle1!(0, 4, 8, 12), - 1 => shuffle1!(1, 5, 9, 13), - 2 => shuffle1!(2, 6, 10, 14), - _ => shuffle1!(3, 7, 11, 15), - }; +#[cfg_attr(test, assert_instr(vpermilps, MASK = 9))] //should be vpshufd +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm512_shuffle_epi32(a: __m512i) -> __m512i { + static_assert_imm8!(MASK); + let r: i32x16 = simd_shuffle16!( + a.as_i32x16(), + a.as_i32x16(), + [ + MASK as u32 & 0b11, + (MASK as u32 >> 2) & 0b11, + (MASK as u32 >> 4) & 0b11, + (MASK as u32 >> 6) & 0b11, + (MASK as u32 & 0b11) + 4, + ((MASK as u32 >> 2) & 0b11) + 4, + ((MASK as u32 >> 4) & 0b11) + 4, + ((MASK as u32 >> 6) & 0b11) + 4, + (MASK as u32 & 0b11) + 8, + ((MASK as u32 >> 2) & 0b11) + 8, + ((MASK as u32 >> 4) & 0b11) + 8, + ((MASK as u32 >> 6) & 0b11) + 8, + (MASK as u32 & 0b11) + 12, + ((MASK as u32 >> 2) & 0b11) + 12, + ((MASK as u32 >> 4) & 0b11) + 12, + ((MASK as u32 >> 6) & 0b11) + 12, + ], + ); transmute(r) } @@ -22405,20 +20897,15 @@ pub unsafe fn _mm512_shuffle_epi32(a: __m512i, imm8: _MM_PERM_ENUM) -> __m512i { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_shuffle_epi32&expand=5148) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vpshufd, imm8 = 9))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm512_mask_shuffle_epi32( +#[cfg_attr(test, assert_instr(vpshufd, MASK = 9))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_mask_shuffle_epi32( src: __m512i, k: __mmask16, a: __m512i, - imm8: _MM_PERM_ENUM, ) -> __m512i { - macro_rules! call { - ($imm8:expr) => { - _mm512_shuffle_epi32(a, $imm8) - }; - } - let r = constify_imm8_sae!(imm8, call); + static_assert_imm8!(MASK); + let r = _mm512_shuffle_epi32::(a); transmute(simd_select_bitmask(k, r.as_i32x16(), src.as_i32x16())) } @@ -22427,15 +20914,14 @@ pub unsafe fn _mm512_mask_shuffle_epi32( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_shuffle_epi32&expand=5149) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vpshufd, imm8 = 9))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm512_maskz_shuffle_epi32(k: __mmask16, a: __m512i, imm8: _MM_PERM_ENUM) -> __m512i { - macro_rules! call { - ($imm8:expr) => { - _mm512_shuffle_epi32(a, $imm8) - }; - } - let r = constify_imm8_sae!(imm8, call); +#[cfg_attr(test, assert_instr(vpshufd, MASK = 9))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_maskz_shuffle_epi32( + k: __mmask16, + a: __m512i, +) -> __m512i { + static_assert_imm8!(MASK); + let r = _mm512_shuffle_epi32::(a); let zero = _mm512_setzero_si512().as_i32x16(); transmute(simd_select_bitmask(k, r.as_i32x16(), zero)) } @@ -22445,20 +20931,15 @@ pub unsafe fn _mm512_maskz_shuffle_epi32(k: __mmask16, a: __m512i, imm8: _MM_PER /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_shuffle_epi32&expand=5145) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[cfg_attr(test, assert_instr(vpshufd, imm8 = 9))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm256_mask_shuffle_epi32( +#[cfg_attr(test, assert_instr(vpshufd, MASK = 9))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm256_mask_shuffle_epi32( src: __m256i, k: __mmask8, a: __m256i, - imm8: _MM_PERM_ENUM, ) -> __m256i { - macro_rules! call { - ($imm8:expr) => { - _mm256_shuffle_epi32(a, $imm8) - }; - } - let r = constify_imm8_sae!(imm8, call); + static_assert_imm8!(MASK); + let r = _mm256_shuffle_epi32::(a); transmute(simd_select_bitmask(k, r.as_i32x8(), src.as_i32x8())) } @@ -22467,15 +20948,14 @@ pub unsafe fn _mm256_mask_shuffle_epi32( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_shuffle_epi32&expand=5146) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[cfg_attr(test, assert_instr(vpshufd, imm8 = 9))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm256_maskz_shuffle_epi32(k: __mmask8, a: __m256i, imm8: _MM_PERM_ENUM) -> __m256i { - macro_rules! call { - ($imm8:expr) => { - _mm256_shuffle_epi32(a, $imm8) - }; - } - let r = constify_imm8_sae!(imm8, call); +#[cfg_attr(test, assert_instr(vpshufd, MASK = 9))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm256_maskz_shuffle_epi32( + k: __mmask8, + a: __m256i, +) -> __m256i { + static_assert_imm8!(MASK); + let r = _mm256_shuffle_epi32::(a); let zero = _mm256_setzero_si256().as_i32x8(); transmute(simd_select_bitmask(k, r.as_i32x8(), zero)) } @@ -22485,20 +20965,15 @@ pub unsafe fn _mm256_maskz_shuffle_epi32(k: __mmask8, a: __m256i, imm8: _MM_PERM /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_shuffle_epi32&expand=5142) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[cfg_attr(test, assert_instr(vpshufd, imm8 = 9))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm_mask_shuffle_epi32( +#[cfg_attr(test, assert_instr(vpshufd, MASK = 9))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm_mask_shuffle_epi32( src: __m128i, k: __mmask8, a: __m128i, - imm8: _MM_PERM_ENUM, ) -> __m128i { - macro_rules! call { - ($imm8:expr) => { - _mm_shuffle_epi32(a, $imm8) - }; - } - let r = constify_imm8_sae!(imm8, call); + static_assert_imm8!(MASK); + let r = _mm_shuffle_epi32::(a); transmute(simd_select_bitmask(k, r.as_i32x4(), src.as_i32x4())) } @@ -22507,15 +20982,14 @@ pub unsafe fn _mm_mask_shuffle_epi32( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_shuffle_epi32&expand=5143) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[cfg_attr(test, assert_instr(vpshufd, imm8 = 9))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm_maskz_shuffle_epi32(k: __mmask8, a: __m128i, imm8: _MM_PERM_ENUM) -> __m128i { - macro_rules! call { - ($imm8:expr) => { - _mm_shuffle_epi32(a, $imm8) - }; - } - let r = constify_imm8_sae!(imm8, call); +#[cfg_attr(test, assert_instr(vpshufd, MASK = 9))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm_maskz_shuffle_epi32( + k: __mmask8, + a: __m128i, +) -> __m128i { + static_assert_imm8!(MASK); + let r = _mm_shuffle_epi32::(a); let zero = _mm_setzero_si128().as_i32x4(); transmute(simd_select_bitmask(k, r.as_i32x4(), zero)) } @@ -22525,75 +20999,32 @@ pub unsafe fn _mm_maskz_shuffle_epi32(k: __mmask8, a: __m128i, imm8: _MM_PERM_EN /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_shuffle_ps&expand=5203) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vshufps, imm8 = 0))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm512_shuffle_ps(a: __m512, b: __m512, imm8: i32) -> __m512 { - assert!(imm8 >= 0 && imm8 <= 255); - let imm8 = (imm8 & 0xFF) as u8; - macro_rules! shuffle4 { - ( - $a:expr, - $b:expr, - $c:expr, - $d:expr, - $e:expr, - $f:expr, - $g:expr, - $h:expr, - $i:expr, - $j:expr, - $k:expr, - $l:expr, - $m:expr, - $n:expr, - $o:expr, - $p:expr - ) => { - simd_shuffle16( - a, - b, - [ - $a, $b, $c, $d, $e, $f, $g, $h, $i, $j, $k, $l, $m, $n, $o, $p, - ], - ) - }; - } - macro_rules! shuffle3 { - ($a:expr, $b:expr, $c:expr, $e:expr, $f:expr, $g:expr, $i:expr, $j:expr, $k:expr, $m:expr, $n:expr, $o:expr) => { - match (imm8 >> 6) & 0x3 { - 0 => shuffle4!($a, $b, $c, 16, $e, $f, $g, 20, $i, $j, $k, 24, $m, $n, $o, 28), - 1 => shuffle4!($a, $b, $c, 17, $e, $f, $g, 21, $i, $j, $k, 25, $m, $n, $o, 29), - 2 => shuffle4!($a, $b, $c, 18, $e, $f, $g, 22, $i, $j, $k, 26, $m, $n, $o, 30), - _ => shuffle4!($a, $b, $c, 19, $e, $f, $g, 23, $i, $j, $k, 27, $m, $n, $o, 31), - } - }; - } - macro_rules! shuffle2 { - ($a:expr, $b:expr, $e:expr, $f:expr, $i:expr, $j:expr, $m:expr, $n:expr) => { - match (imm8 >> 4) & 0x3 { - 0 => shuffle3!($a, $b, 16, $e, $f, 20, $i, $j, 24, $m, $n, 28), - 1 => shuffle3!($a, $b, 17, $e, $f, 21, $i, $j, 25, $m, $n, 29), - 2 => shuffle3!($a, $b, 18, $e, $f, 22, $i, $j, 26, $m, $n, 30), - _ => shuffle3!($a, $b, 19, $e, $f, 23, $i, $j, 27, $m, $n, 31), - } - }; - } - macro_rules! shuffle1 { - ($a:expr, $e:expr, $i: expr, $m: expr) => { - match (imm8 >> 2) & 0x3 { - 0 => shuffle2!($a, 0, $e, 4, $i, 8, $m, 12), - 1 => shuffle2!($a, 1, $e, 5, $i, 9, $m, 13), - 2 => shuffle2!($a, 2, $e, 6, $i, 10, $m, 14), - _ => shuffle2!($a, 3, $e, 7, $i, 11, $m, 15), - } - }; - } - match imm8 & 0x3 { - 0 => shuffle1!(0, 4, 8, 12), - 1 => shuffle1!(1, 5, 9, 13), - 2 => shuffle1!(2, 6, 10, 14), - _ => shuffle1!(3, 7, 11, 15), - } +#[cfg_attr(test, assert_instr(vshufps, MASK = 3))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_shuffle_ps(a: __m512, b: __m512) -> __m512 { + static_assert_imm8!(MASK); + simd_shuffle16!( + a, + b, + [ + MASK as u32 & 0b11, + (MASK as u32 >> 2) & 0b11, + ((MASK as u32 >> 4) & 0b11) + 16, + ((MASK as u32 >> 6) & 0b11) + 16, + (MASK as u32 & 0b11) + 4, + ((MASK as u32 >> 2) & 0b11) + 4, + ((MASK as u32 >> 4) & 0b11) + 20, + ((MASK as u32 >> 6) & 0b11) + 20, + (MASK as u32 & 0b11) + 8, + ((MASK as u32 >> 2) & 0b11) + 8, + ((MASK as u32 >> 4) & 0b11) + 24, + ((MASK as u32 >> 6) & 0b11) + 24, + (MASK as u32 & 0b11) + 12, + ((MASK as u32 >> 2) & 0b11) + 12, + ((MASK as u32 >> 4) & 0b11) + 28, + ((MASK as u32 >> 6) & 0b11) + 28, + ], + ) } /// Shuffle single-precision (32-bit) floating-point elements in a within 128-bit lanes using the control in imm8, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). @@ -22601,21 +21032,16 @@ pub unsafe fn _mm512_shuffle_ps(a: __m512, b: __m512, imm8: i32) -> __m512 { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_shuffle_ps&expand=5201) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vshufps, imm8 = 0))] -#[rustc_args_required_const(4)] -pub unsafe fn _mm512_mask_shuffle_ps( +#[cfg_attr(test, assert_instr(vshufps, MASK = 3))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_mask_shuffle_ps( src: __m512, k: __mmask16, a: __m512, b: __m512, - imm8: i32, ) -> __m512 { - macro_rules! call { - ($imm8:expr) => { - _mm512_shuffle_ps(a, b, $imm8) - }; - } - let r = constify_imm8_sae!(imm8, call); + static_assert_imm8!(MASK); + let r = _mm512_shuffle_ps::(a, b); transmute(simd_select_bitmask(k, r.as_f32x16(), src.as_f32x16())) } @@ -22624,15 +21050,15 @@ pub unsafe fn _mm512_mask_shuffle_ps( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_shuffle_ps&expand=5202) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vshufps, imm8 = 0))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm512_maskz_shuffle_ps(k: __mmask16, a: __m512, b: __m512, imm8: i32) -> __m512 { - macro_rules! call { - ($imm8:expr) => { - _mm512_shuffle_ps(a, b, $imm8) - }; - } - let r = constify_imm8_sae!(imm8, call); +#[cfg_attr(test, assert_instr(vshufps, MASK = 3))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_maskz_shuffle_ps( + k: __mmask16, + a: __m512, + b: __m512, +) -> __m512 { + static_assert_imm8!(MASK); + let r = _mm512_shuffle_ps::(a, b); let zero = _mm512_setzero_ps().as_f32x16(); transmute(simd_select_bitmask(k, r.as_f32x16(), zero)) } @@ -22642,21 +21068,16 @@ pub unsafe fn _mm512_maskz_shuffle_ps(k: __mmask16, a: __m512, b: __m512, imm8: /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_shuffle_ps&expand=5198) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[cfg_attr(test, assert_instr(vshufps, imm8 = 9))] -#[rustc_args_required_const(4)] -pub unsafe fn _mm256_mask_shuffle_ps( +#[cfg_attr(test, assert_instr(vshufps, MASK = 3))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm256_mask_shuffle_ps( src: __m256, k: __mmask8, a: __m256, b: __m256, - imm8: i32, ) -> __m256 { - macro_rules! call { - ($imm8:expr) => { - _mm256_shuffle_ps(a, b, $imm8) - }; - } - let r = constify_imm8_sae!(imm8, call); + static_assert_imm8!(MASK); + let r = _mm256_shuffle_ps::(a, b); transmute(simd_select_bitmask(k, r.as_f32x8(), src.as_f32x8())) } @@ -22665,15 +21086,15 @@ pub unsafe fn _mm256_mask_shuffle_ps( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_shuffle_ps&expand=5199) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[cfg_attr(test, assert_instr(vshufps, imm8 = 9))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm256_maskz_shuffle_ps(k: __mmask8, a: __m256, b: __m256, imm8: i32) -> __m256 { - macro_rules! call { - ($imm8:expr) => { - _mm256_shuffle_ps(a, b, $imm8) - }; - } - let r = constify_imm8_sae!(imm8, call); +#[cfg_attr(test, assert_instr(vshufps, MASK = 3))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm256_maskz_shuffle_ps( + k: __mmask8, + a: __m256, + b: __m256, +) -> __m256 { + static_assert_imm8!(MASK); + let r = _mm256_shuffle_ps::(a, b); let zero = _mm256_setzero_ps().as_f32x8(); transmute(simd_select_bitmask(k, r.as_f32x8(), zero)) } @@ -22683,21 +21104,16 @@ pub unsafe fn _mm256_maskz_shuffle_ps(k: __mmask8, a: __m256, b: __m256, imm8: i /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_shuffle_ps&expand=5195) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[cfg_attr(test, assert_instr(vshufps, imm8 = 9))] -#[rustc_args_required_const(4)] -pub unsafe fn _mm_mask_shuffle_ps( +#[cfg_attr(test, assert_instr(vshufps, MASK = 3))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm_mask_shuffle_ps( src: __m128, k: __mmask8, a: __m128, b: __m128, - imm8: i32, ) -> __m128 { - macro_rules! call { - ($imm8:expr) => { - _mm_shuffle_ps(a, b, $imm8) - }; - } - let r = constify_imm8_sae!(imm8, call); + static_assert_imm8!(MASK); + let r = _mm_shuffle_ps::(a, b); transmute(simd_select_bitmask(k, r.as_f32x4(), src.as_f32x4())) } @@ -22706,15 +21122,11 @@ pub unsafe fn _mm_mask_shuffle_ps( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_shuffle_ps&expand=5196) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[cfg_attr(test, assert_instr(vshufps, imm8 = 9))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm_maskz_shuffle_ps(k: __mmask8, a: __m128, b: __m128, imm8: i32) -> __m128 { - macro_rules! call { - ($imm8:expr) => { - _mm_shuffle_ps(a, b, $imm8) - }; - } - let r = constify_imm8_sae!(imm8, call); +#[cfg_attr(test, assert_instr(vshufps, MASK = 3))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm_maskz_shuffle_ps(k: __mmask8, a: __m128, b: __m128) -> __m128 { + static_assert_imm8!(MASK); + let r = _mm_shuffle_ps::(a, b); let zero = _mm_setzero_ps().as_f32x4(); transmute(simd_select_bitmask(k, r.as_f32x4(), zero)) } @@ -22724,76 +21136,24 @@ pub unsafe fn _mm_maskz_shuffle_ps(k: __mmask8, a: __m128, b: __m128, imm8: i32) /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_shuffle_pd&expand=5192) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vshufpd, imm8 = 3))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm512_shuffle_pd(a: __m512d, b: __m512d, imm8: i32) -> __m512d { - assert!(imm8 >= 0 && imm8 <= 255); - let imm8 = (imm8 & 0xFF) as u8; - macro_rules! shuffle8 { - ($a:expr, $b:expr, $c:expr, $d:expr, $e:expr, $f:expr, $g:expr, $h:expr) => { - simd_shuffle8(a, b, [$a, $b, $c, $d, $e, $f, $g, $h]) - }; - } - macro_rules! shuffle7 { - ($a:expr, $b:expr, $c:expr, $d:expr, $e:expr, $f:expr, $g:expr) => { - match (imm8 >> 7) & 0x1 { - 0 => shuffle8!($a, $b, $c, $d, $e, $f, $g, 14), - _ => shuffle8!($a, $b, $c, $d, $e, $f, $g, 15), - } - }; - } - macro_rules! shuffle6 { - ($a:expr, $b:expr, $c:expr, $d:expr, $e:expr, $f:expr) => { - match (imm8 >> 6) & 0x1 { - 0 => shuffle7!($a, $b, $c, $d, $e, $f, 6), - _ => shuffle7!($a, $b, $c, $d, $e, $f, 7), - } - }; - } - macro_rules! shuffle5 { - ($a:expr, $b:expr, $c:expr, $d:expr, $e:expr) => { - match (imm8 >> 5) & 0x1 { - 0 => shuffle6!($a, $b, $c, $d, $e, 12), - _ => shuffle6!($a, $b, $c, $d, $e, 13), - } - }; - } - macro_rules! shuffle4 { - ($a:expr, $b:expr, $c:expr, $d:expr) => { - match (imm8 >> 4) & 0x1 { - 0 => shuffle5!($a, $b, $c, $d, 4), - _ => shuffle5!($a, $b, $c, $d, 5), - } - }; - } - macro_rules! shuffle3 { - ($a:expr, $b:expr, $c:expr) => { - match (imm8 >> 3) & 0x1 { - 0 => shuffle4!($a, $b, $c, 10), - _ => shuffle4!($a, $b, $c, 11), - } - }; - } - macro_rules! shuffle2 { - ($a:expr, $b:expr) => { - match (imm8 >> 2) & 0x1 { - 0 => shuffle3!($a, $b, 2), - _ => shuffle3!($a, $b, 3), - } - }; - } - macro_rules! shuffle1 { - ($a:expr) => { - match (imm8 >> 1) & 0x1 { - 0 => shuffle2!($a, 8), - _ => shuffle2!($a, 9), - } - }; - } - match imm8 & 0x1 { - 0 => shuffle1!(0), - _ => shuffle1!(1), - } +#[cfg_attr(test, assert_instr(vshufpd, MASK = 3))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_shuffle_pd(a: __m512d, b: __m512d) -> __m512d { + static_assert_imm8!(MASK); + simd_shuffle8!( + a, + b, + [ + MASK as u32 & 0b1, + ((MASK as u32 >> 1) & 0b1) + 8, + ((MASK as u32 >> 2) & 0b1) + 2, + ((MASK as u32 >> 3) & 0b1) + 10, + ((MASK as u32 >> 4) & 0b1) + 4, + ((MASK as u32 >> 5) & 0b1) + 12, + ((MASK as u32 >> 6) & 0b1) + 6, + ((MASK as u32 >> 7) & 0b1) + 14, + ], + ) } /// Shuffle double-precision (64-bit) floating-point elements within 128-bit lanes using the control in imm8, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). @@ -22801,21 +21161,16 @@ pub unsafe fn _mm512_shuffle_pd(a: __m512d, b: __m512d, imm8: i32) -> __m512d { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_shuffle_pd&expand=5190) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vshufpd, imm8 = 3))] -#[rustc_args_required_const(4)] -pub unsafe fn _mm512_mask_shuffle_pd( +#[cfg_attr(test, assert_instr(vshufpd, MASK = 3))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_mask_shuffle_pd( src: __m512d, k: __mmask8, a: __m512d, b: __m512d, - imm8: i32, ) -> __m512d { - macro_rules! call { - ($imm8:expr) => { - _mm512_shuffle_pd(a, b, $imm8) - }; - } - let r = constify_imm8_sae!(imm8, call); + static_assert_imm8!(MASK); + let r = _mm512_shuffle_pd::(a, b); transmute(simd_select_bitmask(k, r.as_f64x8(), src.as_f64x8())) } @@ -22824,15 +21179,15 @@ pub unsafe fn _mm512_mask_shuffle_pd( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_shuffle_pd&expand=5191) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vshufpd, imm8 = 3))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm512_maskz_shuffle_pd(k: __mmask8, a: __m512d, b: __m512d, imm8: i32) -> __m512d { - macro_rules! call { - ($imm8:expr) => { - _mm512_shuffle_pd(a, b, $imm8) - }; - } - let r = constify_imm8_sae!(imm8, call); +#[cfg_attr(test, assert_instr(vshufpd, MASK = 3))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_maskz_shuffle_pd( + k: __mmask8, + a: __m512d, + b: __m512d, +) -> __m512d { + static_assert_imm8!(MASK); + let r = _mm512_shuffle_pd::(a, b); let zero = _mm512_setzero_pd().as_f64x8(); transmute(simd_select_bitmask(k, r.as_f64x8(), zero)) } @@ -22842,21 +21197,16 @@ pub unsafe fn _mm512_maskz_shuffle_pd(k: __mmask8, a: __m512d, b: __m512d, imm8: /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_shuffle_pd&expand=5187) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[cfg_attr(test, assert_instr(vshufpd, imm8 = 9))] -#[rustc_args_required_const(4)] -pub unsafe fn _mm256_mask_shuffle_pd( +#[cfg_attr(test, assert_instr(vshufpd, MASK = 3))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm256_mask_shuffle_pd( src: __m256d, k: __mmask8, a: __m256d, b: __m256d, - imm8: i32, ) -> __m256d { - macro_rules! call { - ($imm8:expr) => { - _mm256_shuffle_pd(a, b, $imm8) - }; - } - let r = constify_imm8_sae!(imm8, call); + static_assert_imm8!(MASK); + let r = _mm256_shuffle_pd::(a, b); transmute(simd_select_bitmask(k, r.as_f64x4(), src.as_f64x4())) } @@ -22865,15 +21215,15 @@ pub unsafe fn _mm256_mask_shuffle_pd( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_shuffle_pd&expand=5188) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[cfg_attr(test, assert_instr(vshufpd, imm8 = 9))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm256_maskz_shuffle_pd(k: __mmask8, a: __m256d, b: __m256d, imm8: i32) -> __m256d { - macro_rules! call { - ($imm8:expr) => { - _mm256_shuffle_pd(a, b, $imm8) - }; - } - let r = constify_imm8_sae!(imm8, call); +#[cfg_attr(test, assert_instr(vshufpd, MASK = 3))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm256_maskz_shuffle_pd( + k: __mmask8, + a: __m256d, + b: __m256d, +) -> __m256d { + static_assert_imm8!(MASK); + let r = _mm256_shuffle_pd::(a, b); let zero = _mm256_setzero_pd().as_f64x4(); transmute(simd_select_bitmask(k, r.as_f64x4(), zero)) } @@ -22883,21 +21233,16 @@ pub unsafe fn _mm256_maskz_shuffle_pd(k: __mmask8, a: __m256d, b: __m256d, imm8: /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_shuffle_pd&expand=5184) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[cfg_attr(test, assert_instr(vshufpd, imm8 = 9))] -#[rustc_args_required_const(4)] -pub unsafe fn _mm_mask_shuffle_pd( +#[cfg_attr(test, assert_instr(vshufpd, MASK = 1))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm_mask_shuffle_pd( src: __m128d, k: __mmask8, a: __m128d, b: __m128d, - imm8: i32, ) -> __m128d { - macro_rules! call { - ($imm8:expr) => { - _mm_shuffle_pd(a, b, $imm8) - }; - } - let r = constify_imm8_sae!(imm8, call); + static_assert_imm8!(MASK); + let r = _mm_shuffle_pd::(a, b); transmute(simd_select_bitmask(k, r.as_f64x2(), src.as_f64x2())) } @@ -22906,15 +21251,15 @@ pub unsafe fn _mm_mask_shuffle_pd( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_shuffle_pd&expand=5185) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[cfg_attr(test, assert_instr(vshufpd, imm8 = 9))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm_maskz_shuffle_pd(k: __mmask8, a: __m128d, b: __m128d, imm8: i32) -> __m128d { - macro_rules! call { - ($imm8:expr) => { - _mm_shuffle_pd(a, b, $imm8) - }; - } - let r = constify_imm8_sae!(imm8, call); +#[cfg_attr(test, assert_instr(vshufpd, MASK = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm_maskz_shuffle_pd( + k: __mmask8, + a: __m128d, + b: __m128d, +) -> __m128d { + static_assert_imm8!(MASK); + let r = _mm_shuffle_pd::(a, b); let zero = _mm_setzero_pd().as_f64x2(); transmute(simd_select_bitmask(k, r.as_f64x2(), zero)) } @@ -22924,78 +21269,34 @@ pub unsafe fn _mm_maskz_shuffle_pd(k: __mmask8, a: __m128d, b: __m128d, imm8: i3 /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_shuffle_i32&expand=5177) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vshufi64x2, imm8 = 0b10010101))] //should be vshufi32x4 -#[rustc_args_required_const(2)] -pub unsafe fn _mm512_shuffle_i32x4(a: __m512i, b: __m512i, imm8: i32) -> __m512i { - assert!(imm8 >= 0 && imm8 <= 255); - let imm8 = (imm8 & 0xFF) as u8; +#[cfg_attr(test, assert_instr(vshufi64x2, MASK = 0b10_01_01_01))] //should be vshufi32x4 +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_shuffle_i32x4(a: __m512i, b: __m512i) -> __m512i { + static_assert_imm8!(MASK); let a = a.as_i32x16(); let b = b.as_i32x16(); - macro_rules! shuffle4 { - ( - $a:expr, - $b:expr, - $c:expr, - $d:expr, - $e:expr, - $f:expr, - $g:expr, - $h:expr, - $i:expr, - $j:expr, - $k:expr, - $l:expr, - $m:expr, - $n:expr, - $o:expr, - $p:expr - ) => { - simd_shuffle16( - a, - b, - [ - $a, $b, $c, $d, $e, $f, $g, $h, $i, $j, $k, $l, $m, $n, $o, $p, - ], - ) - }; - } - macro_rules! shuffle3 { - ($a:expr, $b:expr, $c:expr, $e:expr, $f:expr, $g:expr, $i:expr, $j:expr, $k:expr, $m:expr, $n:expr, $o:expr) => { - match (imm8 >> 6) & 0x3 { - 0 => shuffle4!($a, $b, $c, $e, $f, $g, $i, $j, $k, $m, $n, $o, 16, 17, 18, 19), - 1 => shuffle4!($a, $b, $c, $e, $f, $g, $i, $j, $k, $m, $n, $o, 20, 21, 22, 23), - 2 => shuffle4!($a, $b, $c, $e, $f, $g, $i, $j, $k, $m, $n, $o, 24, 25, 26, 27), - _ => shuffle4!($a, $b, $c, $e, $f, $g, $i, $j, $k, $m, $n, $o, 28, 29, 30, 31), - } - }; - } - macro_rules! shuffle2 { - ($a:expr, $b:expr, $e:expr, $f:expr, $i:expr, $j:expr, $m:expr, $n:expr) => { - match (imm8 >> 4) & 0x3 { - 0 => shuffle3!($a, $b, $e, $f, $i, $j, $m, $n, 16, 17, 18, 19), - 1 => shuffle3!($a, $b, $e, $f, $i, $j, $m, $n, 20, 21, 22, 23), - 2 => shuffle3!($a, $b, $e, $f, $i, $j, $m, $n, 24, 25, 26, 27), - _ => shuffle3!($a, $b, $e, $f, $i, $j, $m, $n, 28, 29, 30, 31), - } - }; - } - macro_rules! shuffle1 { - ($a:expr, $e:expr, $i: expr, $m: expr) => { - match (imm8 >> 2) & 0x3 { - 0 => shuffle2!($a, $e, $i, $m, 0, 1, 2, 3), - 1 => shuffle2!($a, $e, $i, $m, 4, 5, 6, 7), - 2 => shuffle2!($a, $e, $i, $m, 8, 9, 10, 11), - _ => shuffle2!($a, $e, $i, $m, 12, 13, 14, 15), - } - }; - } - let r: i32x16 = match imm8 & 0x3 { - 0 => shuffle1!(0, 1, 2, 3), - 1 => shuffle1!(4, 5, 6, 7), - 2 => shuffle1!(8, 9, 10, 11), - _ => shuffle1!(12, 13, 14, 15), - }; - + let r: i32x16 = simd_shuffle16!( + a, + b, + [ + (MASK as u32 & 0b11) * 4 + 0, + (MASK as u32 & 0b11) * 4 + 1, + (MASK as u32 & 0b11) * 4 + 2, + (MASK as u32 & 0b11) * 4 + 3, + ((MASK as u32 >> 2) & 0b11) * 4 + 0, + ((MASK as u32 >> 2) & 0b11) * 4 + 1, + ((MASK as u32 >> 2) & 0b11) * 4 + 2, + ((MASK as u32 >> 2) & 0b11) * 4 + 3, + ((MASK as u32 >> 4) & 0b11) * 4 + 0 + 16, + ((MASK as u32 >> 4) & 0b11) * 4 + 1 + 16, + ((MASK as u32 >> 4) & 0b11) * 4 + 2 + 16, + ((MASK as u32 >> 4) & 0b11) * 4 + 3 + 16, + ((MASK as u32 >> 6) & 0b11) * 4 + 0 + 16, + ((MASK as u32 >> 6) & 0b11) * 4 + 1 + 16, + ((MASK as u32 >> 6) & 0b11) * 4 + 2 + 16, + ((MASK as u32 >> 6) & 0b11) * 4 + 3 + 16, + ], + ); transmute(r) } @@ -23004,21 +21305,16 @@ pub unsafe fn _mm512_shuffle_i32x4(a: __m512i, b: __m512i, imm8: i32) -> __m512i /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_shuffle_i32x&expand=5175) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vshufi32x4, imm8 = 0b10110101))] -#[rustc_args_required_const(4)] -pub unsafe fn _mm512_mask_shuffle_i32x4( +#[cfg_attr(test, assert_instr(vshufi32x4, MASK = 0b10_11_01_01))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_mask_shuffle_i32x4( src: __m512i, k: __mmask16, a: __m512i, b: __m512i, - imm8: i32, ) -> __m512i { - macro_rules! call { - ($imm8:expr) => { - _mm512_shuffle_i32x4(a, b, $imm8) - }; - } - let r = constify_imm8_sae!(imm8, call); + static_assert_imm8!(MASK); + let r = _mm512_shuffle_i32x4::(a, b); transmute(simd_select_bitmask(k, r.as_i32x16(), src.as_i32x16())) } @@ -23027,20 +21323,15 @@ pub unsafe fn _mm512_mask_shuffle_i32x4( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_shuffle_i32&expand=5176) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vshufi32x4, imm8 = 0b10110101))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm512_maskz_shuffle_i32x4( +#[cfg_attr(test, assert_instr(vshufi32x4, MASK = 0b10_11_01_01))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_maskz_shuffle_i32x4( k: __mmask16, a: __m512i, b: __m512i, - imm8: i32, ) -> __m512i { - macro_rules! call { - ($imm8:expr) => { - _mm512_shuffle_i32x4(a, b, $imm8) - }; - } - let r = constify_imm8_sae!(imm8, call); + static_assert_imm8!(MASK); + let r = _mm512_shuffle_i32x4::(a, b); let zero = _mm512_setzero_si512().as_i32x16(); transmute(simd_select_bitmask(k, r.as_i32x16(), zero)) } @@ -23050,39 +21341,26 @@ pub unsafe fn _mm512_maskz_shuffle_i32x4( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_shuffle_i32x4&expand=5174) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[cfg_attr(test, assert_instr(vperm, imm8 = 0b01))] //should be vshufi32x4 -#[rustc_args_required_const(2)] -pub unsafe fn _mm256_shuffle_i32x4(a: __m256i, b: __m256i, imm8: i32) -> __m256i { - assert!(imm8 >= 0 && imm8 <= 255); - let imm8 = (imm8 & 0xFF) as u8; +#[cfg_attr(test, assert_instr(vperm, MASK = 0b11))] //should be vshufi32x4 +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm256_shuffle_i32x4(a: __m256i, b: __m256i) -> __m256i { + static_assert_imm8!(MASK); let a = a.as_i32x8(); let b = b.as_i32x8(); - macro_rules! shuffle2 { - ( - $a:expr, - $b:expr, - $c:expr, - $d:expr, - $e:expr, - $f:expr, - $g:expr, - $h:expr - ) => { - simd_shuffle8(a, b, [$a, $b, $c, $d, $e, $f, $g, $h]) - }; - } - macro_rules! shuffle1 { - ($a:expr, $b:expr, $c: expr, $d: expr) => { - match (imm8 >> 1) & 0x1 { - 0 => shuffle2!($a, $b, $c, $d, 8, 9, 10, 11), - _ => shuffle2!($a, $b, $c, $d, 12, 13, 14, 15), - } - }; - } - let r: i32x8 = match imm8 & 0x1 { - 0 => shuffle1!(0, 1, 2, 3), - _ => shuffle1!(4, 5, 6, 7), - }; + let r: i32x8 = simd_shuffle8!( + a, + b, + [ + (MASK as u32 & 0b1) * 4 + 0, + (MASK as u32 & 0b1) * 4 + 1, + (MASK as u32 & 0b1) * 4 + 2, + (MASK as u32 & 0b1) * 4 + 3, + ((MASK as u32 >> 1) & 0b1) * 4 + 0 + 8, + ((MASK as u32 >> 1) & 0b1) * 4 + 1 + 8, + ((MASK as u32 >> 1) & 0b1) * 4 + 2 + 8, + ((MASK as u32 >> 1) & 0b1) * 4 + 3 + 8, + ], + ); transmute(r) } @@ -23091,21 +21369,16 @@ pub unsafe fn _mm256_shuffle_i32x4(a: __m256i, b: __m256i, imm8: i32) -> __m256i /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_shuffle_i32x4&expand=5172) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[cfg_attr(test, assert_instr(vshufi32x4, imm8 = 0b11))] -#[rustc_args_required_const(4)] -pub unsafe fn _mm256_mask_shuffle_i32x4( +#[cfg_attr(test, assert_instr(vshufi32x4, MASK = 0b11))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm256_mask_shuffle_i32x4( src: __m256i, k: __mmask8, a: __m256i, b: __m256i, - imm8: i32, ) -> __m256i { - macro_rules! call { - ($imm8:expr) => { - _mm256_shuffle_i32x4(a, b, $imm8) - }; - } - let r = constify_imm8_sae!(imm8, call); + static_assert_imm8!(MASK); + let r = _mm256_shuffle_i32x4::(a, b); transmute(simd_select_bitmask(k, r.as_i32x8(), src.as_i32x8())) } @@ -23114,20 +21387,15 @@ pub unsafe fn _mm256_mask_shuffle_i32x4( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_shuffle_i32x4&expand=5173) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[cfg_attr(test, assert_instr(vshufi32x4, imm8 = 0b11))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm256_maskz_shuffle_i32x4( +#[cfg_attr(test, assert_instr(vshufi32x4, MASK = 0b11))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm256_maskz_shuffle_i32x4( k: __mmask8, a: __m256i, b: __m256i, - imm8: i32, ) -> __m256i { - macro_rules! call { - ($imm8:expr) => { - _mm256_shuffle_i32x4(a, b, $imm8) - }; - } - let r = constify_imm8_sae!(imm8, call); + static_assert_imm8!(MASK); + let r = _mm256_shuffle_i32x4::(a, b); let zero = _mm256_setzero_si256().as_i32x8(); transmute(simd_select_bitmask(k, r.as_i32x8(), zero)) } @@ -23137,61 +21405,27 @@ pub unsafe fn _mm256_maskz_shuffle_i32x4( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_shuffle_i64x2&expand=5183) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vshufi64x2, imm8 = 0b10111111))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm512_shuffle_i64x2(a: __m512i, b: __m512i, imm8: i32) -> __m512i { - assert!(imm8 >= 0 && imm8 <= 255); - let imm8 = (imm8 & 0xFF) as u8; - macro_rules! shuffle4 { - ( - $a:expr, - $b:expr, - $c:expr, - $d:expr, - $e:expr, - $f:expr, - $g:expr, - $h:expr - ) => { - simd_shuffle8(a, b, [$a, $b, $c, $d, $e, $f, $g, $h]) - }; - } - macro_rules! shuffle3 { - ($a:expr, $b:expr, $c:expr, $e:expr, $f:expr, $g:expr) => { - match (imm8 >> 6) & 0x3 { - 0 => shuffle4!($a, $b, $c, $e, $f, $g, 8, 9), - 1 => shuffle4!($a, $b, $c, $e, $f, $g, 10, 11), - 2 => shuffle4!($a, $b, $c, $e, $f, $g, 12, 13), - _ => shuffle4!($a, $b, $c, $e, $f, $g, 14, 15), - } - }; - } - macro_rules! shuffle2 { - ($a:expr, $b:expr, $e:expr, $f:expr) => { - match (imm8 >> 4) & 0x3 { - 0 => shuffle3!($a, $b, $e, $f, 8, 9), - 1 => shuffle3!($a, $b, $e, $f, 10, 11), - 2 => shuffle3!($a, $b, $e, $f, 12, 13), - _ => shuffle3!($a, $b, $e, $f, 14, 15), - } - }; - } - macro_rules! shuffle1 { - ($a:expr, $e:expr) => { - match (imm8 >> 2) & 0x3 { - 0 => shuffle2!($a, $e, 0, 1), - 1 => shuffle2!($a, $e, 2, 3), - 2 => shuffle2!($a, $e, 4, 5), - _ => shuffle2!($a, $e, 6, 7), - } - }; - } - match imm8 & 0x3 { - 0 => shuffle1!(0, 1), - 1 => shuffle1!(2, 3), - 2 => shuffle1!(4, 5), - _ => shuffle1!(6, 7), - } +#[cfg_attr(test, assert_instr(vshufi64x2, MASK = 0b10_11_11_11))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_shuffle_i64x2(a: __m512i, b: __m512i) -> __m512i { + static_assert_imm8!(MASK); + let a = a.as_i64x8(); + let b = b.as_i64x8(); + let r: i64x8 = simd_shuffle8!( + a, + b, + [ + (MASK as u32 & 0b11) * 2 + 0, + (MASK as u32 & 0b11) * 2 + 1, + ((MASK as u32 >> 2) & 0b11) * 2 + 0, + ((MASK as u32 >> 2) & 0b11) * 2 + 1, + ((MASK as u32 >> 4) & 0b11) * 2 + 0 + 8, + ((MASK as u32 >> 4) & 0b11) * 2 + 1 + 8, + ((MASK as u32 >> 6) & 0b11) * 2 + 0 + 8, + ((MASK as u32 >> 6) & 0b11) * 2 + 1 + 8, + ], + ); + transmute(r) } /// Shuffle 128-bits (composed of 2 64-bit integers) selected by imm8 from a and b, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). @@ -23199,21 +21433,16 @@ pub unsafe fn _mm512_shuffle_i64x2(a: __m512i, b: __m512i, imm8: i32) -> __m512i /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_shuffle_i64x&expand=5181) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vshufi64x2, imm8 = 0b10111111))] -#[rustc_args_required_const(4)] -pub unsafe fn _mm512_mask_shuffle_i64x2( +#[cfg_attr(test, assert_instr(vshufi64x2, MASK = 0b10_11_11_11))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_mask_shuffle_i64x2( src: __m512i, k: __mmask8, a: __m512i, b: __m512i, - imm8: i32, ) -> __m512i { - macro_rules! call { - ($imm8:expr) => { - _mm512_shuffle_i64x2(a, b, $imm8) - }; - } - let r = constify_imm8_sae!(imm8, call); + static_assert_imm8!(MASK); + let r = _mm512_shuffle_i64x2::(a, b); transmute(simd_select_bitmask(k, r.as_i64x8(), src.as_i64x8())) } @@ -23222,20 +21451,15 @@ pub unsafe fn _mm512_mask_shuffle_i64x2( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_shuffle_i64&expand=5182) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vshufi64x2, imm8 = 0b10111111))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm512_maskz_shuffle_i64x2( +#[cfg_attr(test, assert_instr(vshufi64x2, MASK = 0b10_11_11_11))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_maskz_shuffle_i64x2( k: __mmask8, a: __m512i, b: __m512i, - imm8: i32, ) -> __m512i { - macro_rules! call { - ($imm8:expr) => { - _mm512_shuffle_i64x2(a, b, $imm8) - }; - } - let r = constify_imm8_sae!(imm8, call); + static_assert_imm8!(MASK); + let r = _mm512_shuffle_i64x2::(a, b); let zero = _mm512_setzero_si512().as_i64x8(); transmute(simd_select_bitmask(k, r.as_i64x8(), zero)) } @@ -23245,35 +21469,22 @@ pub unsafe fn _mm512_maskz_shuffle_i64x2( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_shuffle_i64x2&expand=5180) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[cfg_attr(test, assert_instr(vperm, imm8 = 0b01))] //should be vshufi64x2 -#[rustc_args_required_const(2)] -pub unsafe fn _mm256_shuffle_i64x2(a: __m256i, b: __m256i, imm8: i32) -> __m256i { - assert!(imm8 >= 0 && imm8 <= 255); - let imm8 = (imm8 & 0xFF) as u8; +#[cfg_attr(test, assert_instr(vperm, MASK = 0b01))] //should be vshufi64x2 +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm256_shuffle_i64x2(a: __m256i, b: __m256i) -> __m256i { + static_assert_imm8!(MASK); let a = a.as_i64x4(); let b = b.as_i64x4(); - macro_rules! shuffle2 { - ( - $a:expr, - $b:expr, - $c:expr, - $d:expr - ) => { - simd_shuffle4(a, b, [$a, $b, $c, $d]) - }; - } - macro_rules! shuffle1 { - ($a:expr, $b:expr) => { - match (imm8 >> 1) & 0x1 { - 0 => shuffle2!($a, $b, 4, 5), - _ => shuffle2!($a, $b, 6, 7), - } - }; - } - let r: i64x4 = match imm8 & 0x1 { - 0 => shuffle1!(0, 1), - _ => shuffle1!(2, 3), - }; + let r: i64x4 = simd_shuffle4!( + a, + b, + [ + (MASK as u32 & 0b1) * 2 + 0, + (MASK as u32 & 0b1) * 2 + 1, + ((MASK as u32 >> 1) & 0b1) * 2 + 0 + 4, + ((MASK as u32 >> 1) & 0b1) * 2 + 1 + 4, + ], + ); transmute(r) } @@ -23282,21 +21493,16 @@ pub unsafe fn _mm256_shuffle_i64x2(a: __m256i, b: __m256i, imm8: i32) -> __m256i /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_shuffle_i64x2&expand=5178) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[cfg_attr(test, assert_instr(vshufi64x2, imm8 = 0b11))] -#[rustc_args_required_const(4)] -pub unsafe fn _mm256_mask_shuffle_i64x2( +#[cfg_attr(test, assert_instr(vshufi64x2, MASK = 0b11))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm256_mask_shuffle_i64x2( src: __m256i, k: __mmask8, a: __m256i, b: __m256i, - imm8: i32, ) -> __m256i { - macro_rules! call { - ($imm8:expr) => { - _mm256_shuffle_i64x2(a, b, $imm8) - }; - } - let r = constify_imm8_sae!(imm8, call); + static_assert_imm8!(MASK); + let r = _mm256_shuffle_i64x2::(a, b); transmute(simd_select_bitmask(k, r.as_i64x4(), src.as_i64x4())) } @@ -23305,20 +21511,15 @@ pub unsafe fn _mm256_mask_shuffle_i64x2( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_shuffle_i64x2&expand=5179) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[cfg_attr(test, assert_instr(vshufi64x2, imm8 = 0b11))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm256_maskz_shuffle_i64x2( +#[cfg_attr(test, assert_instr(vshufi64x2, MASK = 0b11))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm256_maskz_shuffle_i64x2( k: __mmask8, a: __m256i, b: __m256i, - imm8: i32, ) -> __m256i { - macro_rules! call { - ($imm8:expr) => { - _mm256_shuffle_i64x2(a, b, $imm8) - }; - } - let r = constify_imm8_sae!(imm8, call); + static_assert_imm8!(MASK); + let r = _mm256_shuffle_i64x2::(a, b); let zero = _mm256_setzero_si256().as_i64x4(); transmute(simd_select_bitmask(k, r.as_i64x4(), zero)) } @@ -23328,75 +21529,35 @@ pub unsafe fn _mm256_maskz_shuffle_i64x2( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_shuffle_f32x4&expand=5165) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vshuff64x2, imm8 = 0b10111111))] //should be vshuff32x4, but generate vshuff64x2 -#[rustc_args_required_const(2)] -pub unsafe fn _mm512_shuffle_f32x4(a: __m512, b: __m512, imm8: i32) -> __m512 { - assert!(imm8 >= 0 && imm8 <= 255); - let imm8 = (imm8 & 0xFF) as u8; - macro_rules! shuffle4 { - ( - $a:expr, - $b:expr, - $c:expr, - $d:expr, - $e:expr, - $f:expr, - $g:expr, - $h:expr, - $i:expr, - $j:expr, - $k:expr, - $l:expr, - $m:expr, - $n:expr, - $o:expr, - $p:expr - ) => { - simd_shuffle16( - a, - b, - [ - $a, $b, $c, $d, $e, $f, $g, $h, $i, $j, $k, $l, $m, $n, $o, $p, - ], - ) - }; - } - macro_rules! shuffle3 { - ($a:expr, $b:expr, $c:expr, $e:expr, $f:expr, $g:expr, $i:expr, $j:expr, $k:expr, $m:expr, $n:expr, $o:expr) => { - match (imm8 >> 6) & 0x3 { - 0 => shuffle4!($a, $b, $c, $e, $f, $g, $i, $j, $k, $m, $n, $o, 16, 17, 18, 19), - 1 => shuffle4!($a, $b, $c, $e, $f, $g, $i, $j, $k, $m, $n, $o, 20, 21, 22, 23), - 2 => shuffle4!($a, $b, $c, $e, $f, $g, $i, $j, $k, $m, $n, $o, 24, 25, 26, 27), - _ => shuffle4!($a, $b, $c, $e, $f, $g, $i, $j, $k, $m, $n, $o, 28, 29, 30, 31), - } - }; - } - macro_rules! shuffle2 { - ($a:expr, $b:expr, $e:expr, $f:expr, $i:expr, $j:expr, $m:expr, $n:expr) => { - match (imm8 >> 4) & 0x3 { - 0 => shuffle3!($a, $b, $e, $f, $i, $j, $m, $n, 16, 17, 18, 19), - 1 => shuffle3!($a, $b, $e, $f, $i, $j, $m, $n, 20, 21, 22, 23), - 2 => shuffle3!($a, $b, $e, $f, $i, $j, $m, $n, 24, 25, 26, 27), - _ => shuffle3!($a, $b, $e, $f, $i, $j, $m, $n, 28, 29, 30, 31), - } - }; - } - macro_rules! shuffle1 { - ($a:expr, $e:expr, $i: expr, $m: expr) => { - match (imm8 >> 2) & 0x3 { - 0 => shuffle2!($a, $e, $i, $m, 0, 1, 2, 3), - 1 => shuffle2!($a, $e, $i, $m, 4, 5, 6, 7), - 2 => shuffle2!($a, $e, $i, $m, 8, 9, 10, 11), - _ => shuffle2!($a, $e, $i, $m, 12, 13, 14, 15), - } - }; - } - match imm8 & 0x3 { - 0 => shuffle1!(0, 1, 2, 3), - 1 => shuffle1!(4, 5, 6, 7), - 2 => shuffle1!(8, 9, 10, 11), - _ => shuffle1!(12, 13, 14, 15), - } +#[cfg_attr(test, assert_instr(vshuff64x2, MASK = 0b1011))] //should be vshuff32x4, but generate vshuff64x2 +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_shuffle_f32x4(a: __m512, b: __m512) -> __m512 { + static_assert_imm8!(MASK); + let a = a.as_f32x16(); + let b = b.as_f32x16(); + let r: f32x16 = simd_shuffle16!( + a, + b, + [ + (MASK as u32 & 0b11) * 4 + 0, + (MASK as u32 & 0b11) * 4 + 1, + (MASK as u32 & 0b11) * 4 + 2, + (MASK as u32 & 0b11) * 4 + 3, + ((MASK as u32 >> 2) & 0b11) * 4 + 0, + ((MASK as u32 >> 2) & 0b11) * 4 + 1, + ((MASK as u32 >> 2) & 0b11) * 4 + 2, + ((MASK as u32 >> 2) & 0b11) * 4 + 3, + ((MASK as u32 >> 4) & 0b11) * 4 + 0 + 16, + ((MASK as u32 >> 4) & 0b11) * 4 + 1 + 16, + ((MASK as u32 >> 4) & 0b11) * 4 + 2 + 16, + ((MASK as u32 >> 4) & 0b11) * 4 + 3 + 16, + ((MASK as u32 >> 6) & 0b11) * 4 + 0 + 16, + ((MASK as u32 >> 6) & 0b11) * 4 + 1 + 16, + ((MASK as u32 >> 6) & 0b11) * 4 + 2 + 16, + ((MASK as u32 >> 6) & 0b11) * 4 + 3 + 16, + ], + ); + transmute(r) } /// Shuffle 128-bits (composed of 4 single-precision (32-bit) floating-point elements) selected by imm8 from a and b, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). @@ -23404,21 +21565,16 @@ pub unsafe fn _mm512_shuffle_f32x4(a: __m512, b: __m512, imm8: i32) -> __m512 { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_shuffle_f32&expand=5163) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vshuff32x4, imm8 = 0b10111111))] -#[rustc_args_required_const(4)] -pub unsafe fn _mm512_mask_shuffle_f32x4( +#[cfg_attr(test, assert_instr(vshuff32x4, MASK = 0b1011))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_mask_shuffle_f32x4( src: __m512, k: __mmask16, a: __m512, b: __m512, - imm8: i32, ) -> __m512 { - macro_rules! call { - ($imm8:expr) => { - _mm512_shuffle_f32x4(a, b, $imm8) - }; - } - let r = constify_imm8_sae!(imm8, call); + static_assert_imm8!(MASK); + let r = _mm512_shuffle_f32x4::(a, b); transmute(simd_select_bitmask(k, r.as_f32x16(), src.as_f32x16())) } @@ -23427,15 +21583,15 @@ pub unsafe fn _mm512_mask_shuffle_f32x4( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_shuffle_f32&expand=5164) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vshuff32x4, imm8 = 0b10111111))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm512_maskz_shuffle_f32x4(k: __mmask16, a: __m512, b: __m512, imm8: i32) -> __m512 { - macro_rules! call { - ($imm8:expr) => { - _mm512_shuffle_f32x4(a, b, $imm8) - }; - } - let r = constify_imm8_sae!(imm8, call); +#[cfg_attr(test, assert_instr(vshuff32x4, MASK = 0b1011))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_maskz_shuffle_f32x4( + k: __mmask16, + a: __m512, + b: __m512, +) -> __m512 { + static_assert_imm8!(MASK); + let r = _mm512_shuffle_f32x4::(a, b); let zero = _mm512_setzero_ps().as_f32x16(); transmute(simd_select_bitmask(k, r.as_f32x16(), zero)) } @@ -23445,40 +21601,26 @@ pub unsafe fn _mm512_maskz_shuffle_f32x4(k: __mmask16, a: __m512, b: __m512, imm /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_shuffle_f32x4&expand=5162) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[cfg_attr(test, assert_instr(vperm, imm8 = 0b01))] //should be vshuff32x4 -#[rustc_args_required_const(2)] -pub unsafe fn _mm256_shuffle_f32x4(a: __m256, b: __m256, imm8: i32) -> __m256 { - assert!(imm8 >= 0 && imm8 <= 255); - let imm8 = (imm8 & 0xFF) as u8; +#[cfg_attr(test, assert_instr(vperm, MASK = 0b01))] //should be vshuff32x4 +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm256_shuffle_f32x4(a: __m256, b: __m256) -> __m256 { + static_assert_imm8!(MASK); let a = a.as_f32x8(); let b = b.as_f32x8(); - macro_rules! shuffle2 { - ( - $a:expr, - $b:expr, - $c:expr, - $d:expr, - $e:expr, - $f:expr, - $g:expr, - $h:expr - ) => { - simd_shuffle8(a, b, [$a, $b, $c, $d, $e, $f, $g, $h]) - }; - } - macro_rules! shuffle1 { - ($a:expr, $b:expr, $c: expr, $d: expr) => { - match (imm8 >> 1) & 0x1 { - 0 => shuffle2!($a, $b, $c, $d, 8, 9, 10, 11), - _ => shuffle2!($a, $b, $c, $d, 12, 13, 14, 15), - } - }; - } - let r: f32x8 = match imm8 & 0x1 { - 0 => shuffle1!(0, 1, 2, 3), - _ => shuffle1!(4, 5, 6, 7), - }; - + let r: f32x8 = simd_shuffle8!( + a, + b, + [ + (MASK as u32 & 0b1) * 4 + 0, + (MASK as u32 & 0b1) * 4 + 1, + (MASK as u32 & 0b1) * 4 + 2, + (MASK as u32 & 0b1) * 4 + 3, + ((MASK as u32 >> 1) & 0b1) * 4 + 0 + 8, + ((MASK as u32 >> 1) & 0b1) * 4 + 1 + 8, + ((MASK as u32 >> 1) & 0b1) * 4 + 2 + 8, + ((MASK as u32 >> 1) & 0b1) * 4 + 3 + 8, + ], + ); transmute(r) } @@ -23487,21 +21629,16 @@ pub unsafe fn _mm256_shuffle_f32x4(a: __m256, b: __m256, imm8: i32) -> __m256 { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_shuffle_f32x4&expand=5160) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[cfg_attr(test, assert_instr(vshuff32x4, imm8 = 0b11))] -#[rustc_args_required_const(4)] -pub unsafe fn _mm256_mask_shuffle_f32x4( +#[cfg_attr(test, assert_instr(vshuff32x4, MASK = 0b11))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm256_mask_shuffle_f32x4( src: __m256, k: __mmask8, a: __m256, b: __m256, - imm8: i32, ) -> __m256 { - macro_rules! call { - ($imm8:expr) => { - _mm256_shuffle_f32x4(a, b, $imm8) - }; - } - let r = constify_imm8_sae!(imm8, call); + static_assert_imm8!(MASK); + let r = _mm256_shuffle_f32x4::(a, b); transmute(simd_select_bitmask(k, r.as_f32x8(), src.as_f32x8())) } @@ -23510,15 +21647,15 @@ pub unsafe fn _mm256_mask_shuffle_f32x4( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_shuffle_f32x4&expand=5161) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[cfg_attr(test, assert_instr(vshuff32x4, imm8 = 0b11))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm256_maskz_shuffle_f32x4(k: __mmask8, a: __m256, b: __m256, imm8: i32) -> __m256 { - macro_rules! call { - ($imm8:expr) => { - _mm256_shuffle_f32x4(a, b, $imm8) - }; - } - let r = constify_imm8_sae!(imm8, call); +#[cfg_attr(test, assert_instr(vshuff32x4, MASK = 0b11))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm256_maskz_shuffle_f32x4( + k: __mmask8, + a: __m256, + b: __m256, +) -> __m256 { + static_assert_imm8!(MASK); + let r = _mm256_shuffle_f32x4::(a, b); let zero = _mm256_setzero_ps().as_f32x8(); transmute(simd_select_bitmask(k, r.as_f32x8(), zero)) } @@ -23528,61 +21665,27 @@ pub unsafe fn _mm256_maskz_shuffle_f32x4(k: __mmask8, a: __m256, b: __m256, imm8 /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_shuffle_f64x2&expand=5171) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vshuff64x2, imm8 = 0b10111111))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm512_shuffle_f64x2(a: __m512d, b: __m512d, imm8: i32) -> __m512d { - assert!(imm8 >= 0 && imm8 <= 255); - let imm8 = (imm8 & 0xFF) as u8; - macro_rules! shuffle4 { - ( - $a:expr, - $b:expr, - $c:expr, - $d:expr, - $e:expr, - $f:expr, - $g:expr, - $h:expr - ) => { - simd_shuffle8(a, b, [$a, $b, $c, $d, $e, $f, $g, $h]) - }; - } - macro_rules! shuffle3 { - ($a:expr, $b:expr, $c:expr, $e:expr, $f:expr, $g:expr) => { - match (imm8 >> 6) & 0x3 { - 0 => shuffle4!($a, $b, $c, $e, $f, $g, 8, 9), - 1 => shuffle4!($a, $b, $c, $e, $f, $g, 10, 11), - 2 => shuffle4!($a, $b, $c, $e, $f, $g, 12, 13), - _ => shuffle4!($a, $b, $c, $e, $f, $g, 14, 15), - } - }; - } - macro_rules! shuffle2 { - ($a:expr, $b:expr, $e:expr, $f:expr) => { - match (imm8 >> 4) & 0x3 { - 0 => shuffle3!($a, $b, $e, $f, 8, 9), - 1 => shuffle3!($a, $b, $e, $f, 10, 11), - 2 => shuffle3!($a, $b, $e, $f, 12, 13), - _ => shuffle3!($a, $b, $e, $f, 14, 15), - } - }; - } - macro_rules! shuffle1 { - ($a:expr, $e:expr) => { - match (imm8 >> 2) & 0x3 { - 0 => shuffle2!($a, $e, 0, 1), - 1 => shuffle2!($a, $e, 2, 3), - 2 => shuffle2!($a, $e, 4, 5), - _ => shuffle2!($a, $e, 6, 7), - } - }; - } - match imm8 & 0x3 { - 0 => shuffle1!(0, 1), - 1 => shuffle1!(2, 3), - 2 => shuffle1!(4, 5), - _ => shuffle1!(6, 7), - } +#[cfg_attr(test, assert_instr(vshuff64x2, MASK = 0b10_11_11_11))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_shuffle_f64x2(a: __m512d, b: __m512d) -> __m512d { + static_assert_imm8!(MASK); + let a = a.as_f64x8(); + let b = b.as_f64x8(); + let r: f64x8 = simd_shuffle8!( + a, + b, + [ + (MASK as u32 & 0b11) * 2 + 0, + (MASK as u32 & 0b11) * 2 + 1, + ((MASK as u32 >> 2) & 0b11) * 2 + 0, + ((MASK as u32 >> 2) & 0b11) * 2 + 1, + ((MASK as u32 >> 4) & 0b11) * 2 + 0 + 8, + ((MASK as u32 >> 4) & 0b11) * 2 + 1 + 8, + ((MASK as u32 >> 6) & 0b11) * 2 + 0 + 8, + ((MASK as u32 >> 6) & 0b11) * 2 + 1 + 8, + ], + ); + transmute(r) } /// Shuffle 128-bits (composed of 2 double-precision (64-bit) floating-point elements) selected by imm8 from a and b, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). @@ -23590,21 +21693,16 @@ pub unsafe fn _mm512_shuffle_f64x2(a: __m512d, b: __m512d, imm8: i32) -> __m512d /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_shuffle_f64x2&expand=5169) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vshuff64x2, imm8 = 0b10111111))] -#[rustc_args_required_const(4)] -pub unsafe fn _mm512_mask_shuffle_f64x2( +#[cfg_attr(test, assert_instr(vshuff64x2, MASK = 0b10_11_11_11))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_mask_shuffle_f64x2( src: __m512d, k: __mmask8, a: __m512d, b: __m512d, - imm8: i32, ) -> __m512d { - macro_rules! call { - ($imm8:expr) => { - _mm512_shuffle_f64x2(a, b, $imm8) - }; - } - let r = constify_imm8_sae!(imm8, call); + static_assert_imm8!(MASK); + let r = _mm512_shuffle_f64x2::(a, b); transmute(simd_select_bitmask(k, r.as_f64x8(), src.as_f64x8())) } @@ -23613,20 +21711,15 @@ pub unsafe fn _mm512_mask_shuffle_f64x2( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_shuffle_f64x2&expand=5170) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vshuff64x2, imm8 = 0b10111111))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm512_maskz_shuffle_f64x2( +#[cfg_attr(test, assert_instr(vshuff64x2, MASK = 0b10_11_11_11))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_maskz_shuffle_f64x2( k: __mmask8, a: __m512d, b: __m512d, - imm8: i32, ) -> __m512d { - macro_rules! call { - ($imm8:expr) => { - _mm512_shuffle_f64x2(a, b, $imm8) - }; - } - let r = constify_imm8_sae!(imm8, call); + static_assert_imm8!(MASK); + let r = _mm512_shuffle_f64x2::(a, b); let zero = _mm512_setzero_pd().as_f64x8(); transmute(simd_select_bitmask(k, r.as_f64x8(), zero)) } @@ -23636,35 +21729,22 @@ pub unsafe fn _mm512_maskz_shuffle_f64x2( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_shuffle_f64x2&expand=5168) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[cfg_attr(test, assert_instr(vperm, imm8 = 0b01))] //should be vshuff64x2 -#[rustc_args_required_const(2)] -pub unsafe fn _mm256_shuffle_f64x2(a: __m256d, b: __m256d, imm8: i32) -> __m256d { - assert!(imm8 >= 0 && imm8 <= 255); - let imm8 = (imm8 & 0xFF) as u8; +#[cfg_attr(test, assert_instr(vperm, MASK = 0b01))] //should be vshuff64x2 +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm256_shuffle_f64x2(a: __m256d, b: __m256d) -> __m256d { + static_assert_imm8!(MASK); let a = a.as_f64x4(); let b = b.as_f64x4(); - macro_rules! shuffle2 { - ( - $a:expr, - $b:expr, - $c:expr, - $d:expr - ) => { - simd_shuffle4(a, b, [$a, $b, $c, $d]) - }; - } - macro_rules! shuffle1 { - ($a:expr, $b:expr) => { - match (imm8 >> 1) & 0x1 { - 0 => shuffle2!($a, $b, 4, 5), - _ => shuffle2!($a, $b, 6, 7), - } - }; - } - let r: f64x4 = match imm8 & 0x1 { - 0 => shuffle1!(0, 1), - _ => shuffle1!(2, 3), - }; + let r: f64x4 = simd_shuffle4!( + a, + b, + [ + (MASK as u32 & 0b1) * 2 + 0, + (MASK as u32 & 0b1) * 2 + 1, + ((MASK as u32 >> 1) & 0b1) * 2 + 0 + 4, + ((MASK as u32 >> 1) & 0b1) * 2 + 1 + 4, + ], + ); transmute(r) } @@ -23673,21 +21753,16 @@ pub unsafe fn _mm256_shuffle_f64x2(a: __m256d, b: __m256d, imm8: i32) -> __m256d /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_shuffle_f64x2&expand=5166) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[cfg_attr(test, assert_instr(vshuff64x2, imm8 = 0b11))] -#[rustc_args_required_const(4)] -pub unsafe fn _mm256_mask_shuffle_f64x2( +#[cfg_attr(test, assert_instr(vshuff64x2, MASK = 0b11))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm256_mask_shuffle_f64x2( src: __m256d, k: __mmask8, a: __m256d, b: __m256d, - imm8: i32, ) -> __m256d { - macro_rules! call { - ($imm8:expr) => { - _mm256_shuffle_f64x2(a, b, $imm8) - }; - } - let r = constify_imm8_sae!(imm8, call); + static_assert_imm8!(MASK); + let r = _mm256_shuffle_f64x2::(a, b); transmute(simd_select_bitmask(k, r.as_f64x4(), src.as_f64x4())) } @@ -23696,20 +21771,15 @@ pub unsafe fn _mm256_mask_shuffle_f64x2( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_shuffle_f64x2&expand=5167) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[cfg_attr(test, assert_instr(vshuff64x2, imm8 = 0b11))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm256_maskz_shuffle_f64x2( +#[cfg_attr(test, assert_instr(vshuff64x2, MASK = 0b11))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm256_maskz_shuffle_f64x2( k: __mmask8, a: __m256d, b: __m256d, - imm8: i32, ) -> __m256d { - macro_rules! call { - ($imm8:expr) => { - _mm256_shuffle_f64x2(a, b, $imm8) - }; - } - let r = constify_imm8_sae!(imm8, call); + static_assert_imm8!(MASK); + let r = _mm256_shuffle_f64x2::(a, b); let zero = _mm256_setzero_pd().as_f64x4(); transmute(simd_select_bitmask(k, r.as_f64x4(), zero)) } @@ -23721,16 +21791,16 @@ pub unsafe fn _mm256_maskz_shuffle_f64x2( #[target_feature(enable = "avx512f")] #[cfg_attr( all(test, not(target_os = "windows")), - assert_instr(vextractf32x4, imm8 = 3) + assert_instr(vextractf32x4, IMM8 = 3) )] -#[rustc_args_required_const(1)] -pub unsafe fn _mm512_extractf32x4_ps(a: __m512, imm8: i32) -> __m128 { - assert!(imm8 >= 0 && imm8 <= 3); - match imm8 & 0x3 { - 0 => simd_shuffle4(a, _mm512_undefined_ps(), [0, 1, 2, 3]), - 1 => simd_shuffle4(a, _mm512_undefined_ps(), [4, 5, 6, 7]), - 2 => simd_shuffle4(a, _mm512_undefined_ps(), [8, 9, 10, 11]), - _ => simd_shuffle4(a, _mm512_undefined_ps(), [12, 13, 14, 15]), +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm512_extractf32x4_ps(a: __m512) -> __m128 { + static_assert_imm2!(IMM8); + match IMM8 & 0x3 { + 0 => simd_shuffle4!(a, _mm512_undefined_ps(), [0, 1, 2, 3]), + 1 => simd_shuffle4!(a, _mm512_undefined_ps(), [4, 5, 6, 7]), + 2 => simd_shuffle4!(a, _mm512_undefined_ps(), [8, 9, 10, 11]), + _ => simd_shuffle4!(a, _mm512_undefined_ps(), [12, 13, 14, 15]), } } @@ -23741,21 +21811,16 @@ pub unsafe fn _mm512_extractf32x4_ps(a: __m512, imm8: i32) -> __m128 { #[target_feature(enable = "avx512f")] #[cfg_attr( all(test, not(target_os = "windows")), - assert_instr(vextractf32x4, imm8 = 3) + assert_instr(vextractf32x4, IMM8 = 3) )] -#[rustc_args_required_const(3)] -pub unsafe fn _mm512_mask_extractf32x4_ps( +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_mask_extractf32x4_ps( src: __m128, k: __mmask8, a: __m512, - imm8: i32, ) -> __m128 { - macro_rules! call { - ($imm8:expr) => { - _mm512_extractf32x4_ps(a, $imm8) - }; - } - let r = constify_imm8_sae!(imm8, call); + static_assert_imm2!(IMM8); + let r = _mm512_extractf32x4_ps::(a); transmute(simd_select_bitmask(k, r.as_f32x4(), src.as_f32x4())) } @@ -23766,16 +21831,12 @@ pub unsafe fn _mm512_mask_extractf32x4_ps( #[target_feature(enable = "avx512f")] #[cfg_attr( all(test, not(target_os = "windows")), - assert_instr(vextractf32x4, imm8 = 3) + assert_instr(vextractf32x4, IMM8 = 3) )] -#[rustc_args_required_const(2)] -pub unsafe fn _mm512_maskz_extractf32x4_ps(k: __mmask8, a: __m512, imm8: i32) -> __m128 { - macro_rules! call { - ($imm8:expr) => { - _mm512_extractf32x4_ps(a, $imm8) - }; - } - let r = constify_imm8_sae!(imm8, call); +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_maskz_extractf32x4_ps(k: __mmask8, a: __m512) -> __m128 { + static_assert_imm2!(IMM8); + let r = _mm512_extractf32x4_ps::(a); let zero = _mm_setzero_ps().as_f32x4(); transmute(simd_select_bitmask(k, r.as_f32x4(), zero)) } @@ -23787,14 +21848,14 @@ pub unsafe fn _mm512_maskz_extractf32x4_ps(k: __mmask8, a: __m512, imm8: i32) -> #[target_feature(enable = "avx512f,avx512vl")] #[cfg_attr( all(test, not(target_os = "windows")), - assert_instr(vextract, imm8 = 1) //should be vextractf32x4 + assert_instr(vextract, IMM8 = 1) //should be vextractf32x4 )] -#[rustc_args_required_const(1)] -pub unsafe fn _mm256_extractf32x4_ps(a: __m256, imm8: i32) -> __m128 { - assert!(imm8 >= 0 && imm8 <= 1); - match imm8 & 0x1 { - 0 => simd_shuffle4(a, _mm256_undefined_ps(), [0, 1, 2, 3]), - _ => simd_shuffle4(a, _mm256_undefined_ps(), [4, 5, 6, 7]), +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm256_extractf32x4_ps(a: __m256) -> __m128 { + static_assert_imm1!(IMM8); + match IMM8 & 0x1 { + 0 => simd_shuffle4!(a, _mm256_undefined_ps(), [0, 1, 2, 3]), + _ => simd_shuffle4!(a, _mm256_undefined_ps(), [4, 5, 6, 7]), } } @@ -23805,21 +21866,16 @@ pub unsafe fn _mm256_extractf32x4_ps(a: __m256, imm8: i32) -> __m128 { #[target_feature(enable = "avx512f,avx512vl")] #[cfg_attr( all(test, not(target_os = "windows")), - assert_instr(vextractf32x4, imm8 = 1) + assert_instr(vextractf32x4, IMM8 = 1) )] -#[rustc_args_required_const(3)] -pub unsafe fn _mm256_mask_extractf32x4_ps( +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm256_mask_extractf32x4_ps( src: __m128, k: __mmask8, a: __m256, - imm8: i32, ) -> __m128 { - macro_rules! call { - ($imm8:expr) => { - _mm256_extractf32x4_ps(a, $imm8) - }; - } - let r = constify_imm8_sae!(imm8, call); + static_assert_imm1!(IMM8); + let r = _mm256_extractf32x4_ps::(a); transmute(simd_select_bitmask(k, r.as_f32x4(), src.as_f32x4())) } @@ -23830,80 +21886,67 @@ pub unsafe fn _mm256_mask_extractf32x4_ps( #[target_feature(enable = "avx512f,avx512vl")] #[cfg_attr( all(test, not(target_os = "windows")), - assert_instr(vextractf32x4, imm8 = 1) + assert_instr(vextractf32x4, IMM8 = 1) )] -#[rustc_args_required_const(2)] -pub unsafe fn _mm256_maskz_extractf32x4_ps(k: __mmask8, a: __m256, imm8: i32) -> __m128 { - macro_rules! call { - ($imm8:expr) => { - _mm256_extractf32x4_ps(a, $imm8) - }; - } - let r = constify_imm8_sae!(imm8, call); +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm256_maskz_extractf32x4_ps(k: __mmask8, a: __m256) -> __m128 { + static_assert_imm1!(IMM8); + let r = _mm256_extractf32x4_ps::(a); let zero = _mm_setzero_ps().as_f32x4(); transmute(simd_select_bitmask(k, r.as_f32x4(), zero)) } -/// Extract 256 bits (composed of 4 packed 64-bit integers) from a, selected with imm8, and store the result in dst. +/// Extract 256 bits (composed of 4 packed 64-bit integers) from a, selected with IMM1, and store the result in dst. /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_extracti64x4_epi64&expand=2473) #[inline] #[target_feature(enable = "avx512f")] #[cfg_attr( all(test, not(target_os = "windows")), - assert_instr(vextractf64x4, imm8 = 1) //should be vextracti64x4 + assert_instr(vextractf64x4, IMM1 = 1) //should be vextracti64x4 )] -#[rustc_args_required_const(1)] -pub unsafe fn _mm512_extracti64x4_epi64(a: __m512i, imm8: i32) -> __m256i { - assert!(imm8 >= 0 && imm8 <= 1); - match imm8 & 0x1 { - 0 => simd_shuffle4(a, _mm512_set1_epi64(0), [0, 1, 2, 3]), - _ => simd_shuffle4(a, _mm512_set1_epi64(0), [4, 5, 6, 7]), +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm512_extracti64x4_epi64(a: __m512i) -> __m256i { + static_assert_imm1!(IMM1); + match IMM1 { + 0 => simd_shuffle4!(a, _mm512_set1_epi64(0), [0, 1, 2, 3]), + _ => simd_shuffle4!(a, _mm512_set1_epi64(0), [4, 5, 6, 7]), } } -/// Extract 256 bits (composed of 4 packed 64-bit integers) from a, selected with imm8, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// Extract 256 bits (composed of 4 packed 64-bit integers) from a, selected with IMM1, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_extracti64x4_epi64&expand=2474) #[inline] #[target_feature(enable = "avx512f")] #[cfg_attr( all(test, not(target_os = "windows")), - assert_instr(vextracti64x4, imm8 = 1) + assert_instr(vextracti64x4, IMM1 = 1) )] -#[rustc_args_required_const(3)] -pub unsafe fn _mm512_mask_extracti64x4_epi64( +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_mask_extracti64x4_epi64( src: __m256i, k: __mmask8, a: __m512i, - imm8: i32, ) -> __m256i { - macro_rules! call { - ($imm8:expr) => { - _mm512_extracti64x4_epi64(a, $imm8) - }; - } - let r = constify_imm8_sae!(imm8, call); + static_assert_imm1!(IMM1); + let r = _mm512_extracti64x4_epi64::(a); transmute(simd_select_bitmask(k, r.as_i64x4(), src.as_i64x4())) } -/// Extract 256 bits (composed of 4 packed 64-bit integers) from a, selected with imm8, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// Extract 256 bits (composed of 4 packed 64-bit integers) from a, selected with IMM1, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_extracti64x4_epi64&expand=2475) #[inline] #[target_feature(enable = "avx512f")] #[cfg_attr( all(test, not(target_os = "windows")), - assert_instr(vextracti64x4, imm8 = 1) + assert_instr(vextracti64x4, IMM1 = 1) )] -#[rustc_args_required_const(2)] -pub unsafe fn _mm512_maskz_extracti64x4_epi64(k: __mmask8, a: __m512i, imm8: i32) -> __m256i { - macro_rules! call { - ($imm8:expr) => { - _mm512_extracti64x4_epi64(a, $imm8) - }; - } - let r = constify_imm8_sae!(imm8, call); +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_maskz_extracti64x4_epi64(k: __mmask8, a: __m512i) -> __m256i { + static_assert_imm1!(IMM1); + let r = _mm512_extracti64x4_epi64::(a); let zero = _mm256_setzero_si256().as_i64x4(); transmute(simd_select_bitmask(k, r.as_i64x4(), zero)) } @@ -23915,14 +21958,14 @@ pub unsafe fn _mm512_maskz_extracti64x4_epi64(k: __mmask8, a: __m512i, imm8: i32 #[target_feature(enable = "avx512f")] #[cfg_attr( all(test, not(target_os = "windows")), - assert_instr(vextractf64x4, imm8 = 1) + assert_instr(vextractf64x4, IMM8 = 1) )] -#[rustc_args_required_const(1)] -pub unsafe fn _mm512_extractf64x4_pd(a: __m512d, imm8: i32) -> __m256d { - assert!(imm8 >= 0 && imm8 <= 1); - match imm8 & 0x1 { - 0 => simd_shuffle4(a, _mm512_undefined_pd(), [0, 1, 2, 3]), - _ => simd_shuffle4(a, _mm512_undefined_pd(), [4, 5, 6, 7]), +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm512_extractf64x4_pd(a: __m512d) -> __m256d { + static_assert_imm1!(IMM8); + match IMM8 & 0x1 { + 0 => simd_shuffle4!(a, _mm512_undefined_pd(), [0, 1, 2, 3]), + _ => simd_shuffle4!(a, _mm512_undefined_pd(), [4, 5, 6, 7]), } } @@ -23933,21 +21976,16 @@ pub unsafe fn _mm512_extractf64x4_pd(a: __m512d, imm8: i32) -> __m256d { #[target_feature(enable = "avx512f")] #[cfg_attr( all(test, not(target_os = "windows")), - assert_instr(vextractf64x4, imm8 = 1) + assert_instr(vextractf64x4, IMM8 = 1) )] -#[rustc_args_required_const(3)] -pub unsafe fn _mm512_mask_extractf64x4_pd( +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_mask_extractf64x4_pd( src: __m256d, k: __mmask8, a: __m512d, - imm8: i32, ) -> __m256d { - macro_rules! call { - ($imm8:expr) => { - _mm512_extractf64x4_pd(a, $imm8) - }; - } - let r = constify_imm8_sae!(imm8, call); + static_assert_imm1!(IMM8); + let r = _mm512_extractf64x4_pd::(a); transmute(simd_select_bitmask(k, r.as_f64x4(), src.as_f64x4())) } @@ -23958,152 +21996,130 @@ pub unsafe fn _mm512_mask_extractf64x4_pd( #[target_feature(enable = "avx512f")] #[cfg_attr( all(test, not(target_os = "windows")), - assert_instr(vextractf64x4, imm8 = 1) + assert_instr(vextractf64x4, IMM8 = 1) )] -#[rustc_args_required_const(2)] -pub unsafe fn _mm512_maskz_extractf64x4_pd(k: __mmask8, a: __m512d, imm8: i32) -> __m256d { - macro_rules! call { - ($imm8:expr) => { - _mm512_extractf64x4_pd(a, $imm8) - }; - } - let r = constify_imm8_sae!(imm8, call); +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_maskz_extractf64x4_pd(k: __mmask8, a: __m512d) -> __m256d { + static_assert_imm1!(IMM8); + let r = _mm512_extractf64x4_pd::(a); let zero = _mm256_setzero_pd().as_f64x4(); transmute(simd_select_bitmask(k, r.as_f64x4(), zero)) } -/// Extract 128 bits (composed of 4 packed 32-bit integers) from a, selected with imm8, and store the result in dst. +/// Extract 128 bits (composed of 4 packed 32-bit integers) from a, selected with IMM2, and store the result in dst. /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_extracti32x4_epi32&expand=2461) #[inline] #[target_feature(enable = "avx512f")] #[cfg_attr( all(test, not(target_os = "windows")), - assert_instr(vextractf32x4, imm8 = 3) //should be vextracti32x4 + assert_instr(vextractf32x4, IMM2 = 3) //should be vextracti32x4 )] -#[rustc_args_required_const(1)] -pub unsafe fn _mm512_extracti32x4_epi32(a: __m512i, imm8: i32) -> __m128i { - assert!(imm8 >= 0 && imm8 <= 3); +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm512_extracti32x4_epi32(a: __m512i) -> __m128i { + static_assert_imm2!(IMM2); let a = a.as_i32x16(); let undefined = _mm512_undefined_epi32().as_i32x16(); - let extract: i32x4 = match imm8 & 0x3 { - 0 => simd_shuffle4(a, undefined, [0, 1, 2, 3]), - 1 => simd_shuffle4(a, undefined, [4, 5, 6, 7]), - 2 => simd_shuffle4(a, undefined, [8, 9, 10, 11]), - _ => simd_shuffle4(a, undefined, [12, 13, 14, 15]), + let extract: i32x4 = match IMM2 { + 0 => simd_shuffle4!(a, undefined, [0, 1, 2, 3]), + 1 => simd_shuffle4!(a, undefined, [4, 5, 6, 7]), + 2 => simd_shuffle4!(a, undefined, [8, 9, 10, 11]), + _ => simd_shuffle4!(a, undefined, [12, 13, 14, 15]), }; transmute(extract) } -/// Extract 128 bits (composed of 4 packed 32-bit integers) from a, selected with imm8, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// Extract 128 bits (composed of 4 packed 32-bit integers) from a, selected with IMM2, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_extracti32x4_epi32&expand=2462) #[inline] #[target_feature(enable = "avx512f")] #[cfg_attr( all(test, not(target_os = "windows")), - assert_instr(vextracti32x4, imm8 = 3) + assert_instr(vextracti32x4, IMM2 = 3) )] -#[rustc_args_required_const(3)] -pub unsafe fn _mm512_mask_extracti32x4_epi32( +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_mask_extracti32x4_epi32( src: __m128i, k: __mmask8, a: __m512i, - imm8: i32, ) -> __m128i { - macro_rules! call { - ($imm8:expr) => { - _mm512_extracti32x4_epi32(a, $imm8) - }; - } - let r = constify_imm8_sae!(imm8, call); + static_assert_imm2!(IMM2); + let r = _mm512_extracti32x4_epi32::(a); transmute(simd_select_bitmask(k, r.as_i32x4(), src.as_i32x4())) } -/// Extract 128 bits (composed of 4 packed 32-bit integers) from a, selected with imm8, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// Extract 128 bits (composed of 4 packed 32-bit integers) from a, selected with IMM2, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_extracti32x4_epi32&expand=2463) #[inline] #[target_feature(enable = "avx512f")] #[cfg_attr( all(test, not(target_os = "windows")), - assert_instr(vextracti32x4, imm8 = 3) + assert_instr(vextracti32x4, IMM2 = 3) )] -#[rustc_args_required_const(2)] -pub unsafe fn _mm512_maskz_extracti32x4_epi32(k: __mmask8, a: __m512i, imm8: i32) -> __m128i { - macro_rules! call { - ($imm8:expr) => { - _mm512_extracti32x4_epi32(a, $imm8) - }; - } - let r = constify_imm8_sae!(imm8, call); +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_maskz_extracti32x4_epi32(k: __mmask8, a: __m512i) -> __m128i { + static_assert_imm2!(IMM2); + let r = _mm512_extracti32x4_epi32::(a); let zero = _mm_setzero_si128().as_i32x4(); transmute(simd_select_bitmask(k, r.as_i32x4(), zero)) } -/// Extract 128 bits (composed of 4 packed 32-bit integers) from a, selected with imm8, and store the result in dst. +/// Extract 128 bits (composed of 4 packed 32-bit integers) from a, selected with IMM1, and store the result in dst. /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_extracti32x4_epi32&expand=2458) #[inline] #[target_feature(enable = "avx512f,avx512vl")] #[cfg_attr( all(test, not(target_os = "windows")), - assert_instr(vextract, imm8 = 1) //should be vextracti32x4 + assert_instr(vextract, IMM1 = 1) //should be vextracti32x4 )] -#[rustc_args_required_const(1)] -pub unsafe fn _mm256_extracti32x4_epi32(a: __m256i, imm8: i32) -> __m128i { - assert!(imm8 >= 0 && imm8 <= 1); +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm256_extracti32x4_epi32(a: __m256i) -> __m128i { + static_assert_imm1!(IMM1); let a = a.as_i32x8(); let undefined = _mm256_undefined_si256().as_i32x8(); - let extract: i32x4 = match imm8 & 0x1 { - 0 => simd_shuffle4(a, undefined, [0, 1, 2, 3]), - _ => simd_shuffle4(a, undefined, [4, 5, 6, 7]), + let extract: i32x4 = match IMM1 { + 0 => simd_shuffle4!(a, undefined, [0, 1, 2, 3]), + _ => simd_shuffle4!(a, undefined, [4, 5, 6, 7]), }; transmute(extract) } -/// Extract 128 bits (composed of 4 packed 32-bit integers) from a, selected with imm8, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// Extract 128 bits (composed of 4 packed 32-bit integers) from a, selected with IMM1, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_extracti32x4_epi32&expand=2459) #[inline] #[target_feature(enable = "avx512f,avx512vl")] #[cfg_attr( all(test, not(target_os = "windows")), - assert_instr(vextracti32x4, imm8 = 1) + assert_instr(vextracti32x4, IMM1 = 1) )] -#[rustc_args_required_const(3)] -pub unsafe fn _mm256_mask_extracti32x4_epi32( +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm256_mask_extracti32x4_epi32( src: __m128i, k: __mmask8, a: __m256i, - imm8: i32, ) -> __m128i { - macro_rules! call { - ($imm8:expr) => { - _mm256_extracti32x4_epi32(a, $imm8) - }; - } - let r = constify_imm8_sae!(imm8, call); + static_assert_imm1!(IMM1); + let r = _mm256_extracti32x4_epi32::(a); transmute(simd_select_bitmask(k, r.as_i32x4(), src.as_i32x4())) } -/// Extract 128 bits (composed of 4 packed 32-bit integers) from a, selected with imm8, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// Extract 128 bits (composed of 4 packed 32-bit integers) from a, selected with IMM1, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_extracti32x4_epi32&expand=2460) #[inline] #[target_feature(enable = "avx512f,avx512vl")] #[cfg_attr( all(test, not(target_os = "windows")), - assert_instr(vextracti32x4, imm8 = 1) + assert_instr(vextracti32x4, IMM1 = 1) )] -#[rustc_args_required_const(2)] -pub unsafe fn _mm256_maskz_extracti32x4_epi32(k: __mmask8, a: __m256i, imm8: i32) -> __m128i { - macro_rules! call { - ($imm8:expr) => { - _mm256_extracti32x4_epi32(a, $imm8) - }; - } - let r = constify_imm8_sae!(imm8, call); +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm256_maskz_extracti32x4_epi32(k: __mmask8, a: __m256i) -> __m128i { + static_assert_imm1!(IMM1); + let r = _mm256_extracti32x4_epi32::(a); let zero = _mm_setzero_si128().as_i32x4(); transmute(simd_select_bitmask(k, r.as_i32x4(), zero)) } @@ -24115,7 +22131,7 @@ pub unsafe fn _mm256_maskz_extracti32x4_epi32(k: __mmask8, a: __m256i, imm8: i32 #[target_feature(enable = "avx512f")] #[cfg_attr(test, assert_instr(vmovsldup))] pub unsafe fn _mm512_moveldup_ps(a: __m512) -> __m512 { - let r: f32x16 = simd_shuffle16(a, a, [0, 0, 2, 2, 4, 4, 6, 6, 8, 8, 10, 10, 12, 12, 14, 14]); + let r: f32x16 = simd_shuffle16!(a, a, [0, 0, 2, 2, 4, 4, 6, 6, 8, 8, 10, 10, 12, 12, 14, 14]); transmute(r) } @@ -24126,7 +22142,7 @@ pub unsafe fn _mm512_moveldup_ps(a: __m512) -> __m512 { #[target_feature(enable = "avx512f")] #[cfg_attr(test, assert_instr(vmovsldup))] pub unsafe fn _mm512_mask_moveldup_ps(src: __m512, k: __mmask16, a: __m512) -> __m512 { - let mov: f32x16 = simd_shuffle16(a, a, [0, 0, 2, 2, 4, 4, 6, 6, 8, 8, 10, 10, 12, 12, 14, 14]); + let mov: f32x16 = simd_shuffle16!(a, a, [0, 0, 2, 2, 4, 4, 6, 6, 8, 8, 10, 10, 12, 12, 14, 14]); transmute(simd_select_bitmask(k, mov, src.as_f32x16())) } @@ -24137,7 +22153,7 @@ pub unsafe fn _mm512_mask_moveldup_ps(src: __m512, k: __mmask16, a: __m512) -> _ #[target_feature(enable = "avx512f")] #[cfg_attr(test, assert_instr(vmovsldup))] pub unsafe fn _mm512_maskz_moveldup_ps(k: __mmask16, a: __m512) -> __m512 { - let mov: f32x16 = simd_shuffle16(a, a, [0, 0, 2, 2, 4, 4, 6, 6, 8, 8, 10, 10, 12, 12, 14, 14]); + let mov: f32x16 = simd_shuffle16!(a, a, [0, 0, 2, 2, 4, 4, 6, 6, 8, 8, 10, 10, 12, 12, 14, 14]); let zero = _mm512_setzero_ps().as_f32x16(); transmute(simd_select_bitmask(k, mov, zero)) } @@ -24195,7 +22211,7 @@ pub unsafe fn _mm_maskz_moveldup_ps(k: __mmask8, a: __m128) -> __m128 { #[target_feature(enable = "avx512f")] #[cfg_attr(test, assert_instr(vmovshdup))] pub unsafe fn _mm512_movehdup_ps(a: __m512) -> __m512 { - let r: f32x16 = simd_shuffle16(a, a, [1, 1, 3, 3, 5, 5, 7, 7, 9, 9, 11, 11, 13, 13, 15, 15]); + let r: f32x16 = simd_shuffle16!(a, a, [1, 1, 3, 3, 5, 5, 7, 7, 9, 9, 11, 11, 13, 13, 15, 15]); transmute(r) } @@ -24206,7 +22222,7 @@ pub unsafe fn _mm512_movehdup_ps(a: __m512) -> __m512 { #[target_feature(enable = "avx512f")] #[cfg_attr(test, assert_instr(vmovshdup))] pub unsafe fn _mm512_mask_movehdup_ps(src: __m512, k: __mmask16, a: __m512) -> __m512 { - let mov: f32x16 = simd_shuffle16(a, a, [1, 1, 3, 3, 5, 5, 7, 7, 9, 9, 11, 11, 13, 13, 15, 15]); + let mov: f32x16 = simd_shuffle16!(a, a, [1, 1, 3, 3, 5, 5, 7, 7, 9, 9, 11, 11, 13, 13, 15, 15]); transmute(simd_select_bitmask(k, mov, src.as_f32x16())) } @@ -24217,7 +22233,7 @@ pub unsafe fn _mm512_mask_movehdup_ps(src: __m512, k: __mmask16, a: __m512) -> _ #[target_feature(enable = "avx512f")] #[cfg_attr(test, assert_instr(vmovshdup))] pub unsafe fn _mm512_maskz_movehdup_ps(k: __mmask16, a: __m512) -> __m512 { - let mov: f32x16 = simd_shuffle16(a, a, [1, 1, 3, 3, 5, 5, 7, 7, 9, 9, 11, 11, 13, 13, 15, 15]); + let mov: f32x16 = simd_shuffle16!(a, a, [1, 1, 3, 3, 5, 5, 7, 7, 9, 9, 11, 11, 13, 13, 15, 15]); let zero = _mm512_setzero_ps().as_f32x16(); transmute(simd_select_bitmask(k, mov, zero)) } @@ -24275,7 +22291,7 @@ pub unsafe fn _mm_maskz_movehdup_ps(k: __mmask8, a: __m128) -> __m128 { #[target_feature(enable = "avx512f")] #[cfg_attr(test, assert_instr(vmovddup))] pub unsafe fn _mm512_movedup_pd(a: __m512d) -> __m512d { - let r: f64x8 = simd_shuffle8(a, a, [0, 0, 2, 2, 4, 4, 6, 6]); + let r: f64x8 = simd_shuffle8!(a, a, [0, 0, 2, 2, 4, 4, 6, 6]); transmute(r) } @@ -24286,7 +22302,7 @@ pub unsafe fn _mm512_movedup_pd(a: __m512d) -> __m512d { #[target_feature(enable = "avx512f")] #[cfg_attr(test, assert_instr(vmovddup))] pub unsafe fn _mm512_mask_movedup_pd(src: __m512d, k: __mmask8, a: __m512d) -> __m512d { - let mov: f64x8 = simd_shuffle8(a, a, [0, 0, 2, 2, 4, 4, 6, 6]); + let mov: f64x8 = simd_shuffle8!(a, a, [0, 0, 2, 2, 4, 4, 6, 6]); transmute(simd_select_bitmask(k, mov, src.as_f64x8())) } @@ -24297,7 +22313,7 @@ pub unsafe fn _mm512_mask_movedup_pd(src: __m512d, k: __mmask8, a: __m512d) -> _ #[target_feature(enable = "avx512f")] #[cfg_attr(test, assert_instr(vmovddup))] pub unsafe fn _mm512_maskz_movedup_pd(k: __mmask8, a: __m512d) -> __m512d { - let mov: f64x8 = simd_shuffle8(a, a, [0, 0, 2, 2, 4, 4, 6, 6]); + let mov: f64x8 = simd_shuffle8!(a, a, [0, 0, 2, 2, 4, 4, 6, 6]); let zero = _mm512_setzero_pd().as_f64x8(); transmute(simd_select_bitmask(k, mov, zero)) } @@ -24353,29 +22369,29 @@ pub unsafe fn _mm_maskz_movedup_pd(k: __mmask8, a: __m128d) -> __m128d { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_inserti32x4&expand=3174) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vinsertf32x4, imm8 = 2))] //should be vinserti32x4 -#[rustc_args_required_const(2)] -pub unsafe fn _mm512_inserti32x4(a: __m512i, b: __m128i, imm8: i32) -> __m512i { - assert!(imm8 >= 0 && imm8 <= 3); +#[cfg_attr(test, assert_instr(vinsertf32x4, IMM8 = 2))] //should be vinserti32x4 +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_inserti32x4(a: __m512i, b: __m128i) -> __m512i { + static_assert_imm2!(IMM8); let a = a.as_i32x16(); let b = _mm512_castsi128_si512(b).as_i32x16(); - let ret: i32x16 = match imm8 & 0b11 { - 0 => simd_shuffle16( + let ret: i32x16 = match IMM8 & 0b11 { + 0 => simd_shuffle16!( a, b, [16, 17, 18, 19, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15], ), - 1 => simd_shuffle16( + 1 => simd_shuffle16!( a, b, [0, 1, 2, 3, 16, 17, 18, 19, 8, 9, 10, 11, 12, 13, 14, 15], ), - 2 => simd_shuffle16( + 2 => simd_shuffle16!( a, b, [0, 1, 2, 3, 4, 5, 6, 7, 16, 17, 18, 19, 12, 13, 14, 15], ), - _ => simd_shuffle16(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 16, 17, 18, 19]), + _ => simd_shuffle16!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 16, 17, 18, 19]), }; transmute(ret) } @@ -24385,21 +22401,16 @@ pub unsafe fn _mm512_inserti32x4(a: __m512i, b: __m128i, imm8: i32) -> __m512i { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_inserti32x4&expand=3175) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vinserti32x4, imm8 = 2))] -#[rustc_args_required_const(4)] -pub unsafe fn _mm512_mask_inserti32x4( +#[cfg_attr(test, assert_instr(vinserti32x4, IMM8 = 2))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_mask_inserti32x4( src: __m512i, k: __mmask16, a: __m512i, b: __m128i, - imm8: i32, ) -> __m512i { - macro_rules! call { - ($imm8:expr) => { - _mm512_inserti32x4(a, b, $imm8) - }; - } - let r = constify_imm8_sae!(imm8, call); + static_assert_imm2!(IMM8); + let r = _mm512_inserti32x4::(a, b); transmute(simd_select_bitmask(k, r.as_i32x16(), src.as_i32x16())) } @@ -24408,15 +22419,15 @@ pub unsafe fn _mm512_mask_inserti32x4( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_inserti32x4&expand=3176) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vinserti32x4, imm8 = 2))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm512_maskz_inserti32x4(k: __mmask16, a: __m512i, b: __m128i, imm8: i32) -> __m512i { - macro_rules! call { - ($imm8:expr) => { - _mm512_inserti32x4(a, b, $imm8) - }; - } - let r = constify_imm8_sae!(imm8, call); +#[cfg_attr(test, assert_instr(vinserti32x4, IMM8 = 2))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_maskz_inserti32x4( + k: __mmask16, + a: __m512i, + b: __m128i, +) -> __m512i { + static_assert_imm2!(IMM8); + let r = _mm512_inserti32x4::(a, b); let zero = _mm512_setzero_si512().as_i32x16(); transmute(simd_select_bitmask(k, r.as_i32x16(), zero)) } @@ -24428,16 +22439,16 @@ pub unsafe fn _mm512_maskz_inserti32x4(k: __mmask16, a: __m512i, b: __m128i, imm #[target_feature(enable = "avx512f,avx512vl")] #[cfg_attr( all(test, not(target_os = "windows")), - assert_instr(vinsert, imm8 = 1) //should be vinserti32x4 + assert_instr(vinsert, IMM8 = 1) //should be vinserti32x4 )] -#[rustc_args_required_const(2)] -pub unsafe fn _mm256_inserti32x4(a: __m256i, b: __m128i, imm8: i32) -> __m256i { - assert!(imm8 >= 0 && imm8 <= 1); +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm256_inserti32x4(a: __m256i, b: __m128i) -> __m256i { + static_assert_imm1!(IMM8); let a = a.as_i32x8(); let b = _mm256_castsi128_si256(b).as_i32x8(); - let ret: i32x8 = match imm8 & 0b1 { - 0 => simd_shuffle8(a, b, [8, 9, 10, 11, 4, 5, 6, 7]), - _ => simd_shuffle8(a, b, [0, 1, 2, 3, 8, 9, 10, 11]), + let ret: i32x8 = match IMM8 & 0b1 { + 0 => simd_shuffle8!(a, b, [8, 9, 10, 11, 4, 5, 6, 7]), + _ => simd_shuffle8!(a, b, [0, 1, 2, 3, 8, 9, 10, 11]), }; transmute(ret) } @@ -24449,22 +22460,17 @@ pub unsafe fn _mm256_inserti32x4(a: __m256i, b: __m128i, imm8: i32) -> __m256i { #[target_feature(enable = "avx512f,avx512vl")] #[cfg_attr( all(test, not(target_os = "windows")), - assert_instr(vinserti32x4, imm8 = 1) + assert_instr(vinserti32x4, IMM8 = 1) )] -#[rustc_args_required_const(4)] -pub unsafe fn _mm256_mask_inserti32x4( +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm256_mask_inserti32x4( src: __m256i, k: __mmask8, a: __m256i, b: __m128i, - imm8: i32, ) -> __m256i { - macro_rules! call { - ($imm8:expr) => { - _mm256_inserti32x4(a, b, $imm8) - }; - } - let r = constify_imm8_sae!(imm8, call); + static_assert_imm1!(IMM8); + let r = _mm256_inserti32x4::(a, b); transmute(simd_select_bitmask(k, r.as_i32x8(), src.as_i32x8())) } @@ -24475,16 +22481,16 @@ pub unsafe fn _mm256_mask_inserti32x4( #[target_feature(enable = "avx512f,avx512vl")] #[cfg_attr( all(test, not(target_os = "windows")), - assert_instr(vinserti32x4, imm8 = 1) + assert_instr(vinserti32x4, IMM8 = 1) )] -#[rustc_args_required_const(3)] -pub unsafe fn _mm256_maskz_inserti32x4(k: __mmask8, a: __m256i, b: __m128i, imm8: i32) -> __m256i { - macro_rules! call { - ($imm8:expr) => { - _mm256_inserti32x4(a, b, $imm8) - }; - } - let r = constify_imm8_sae!(imm8, call); +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm256_maskz_inserti32x4( + k: __mmask8, + a: __m256i, + b: __m128i, +) -> __m256i { + static_assert_imm1!(IMM8); + let r = _mm256_inserti32x4::(a, b); let zero = _mm256_setzero_si256().as_i32x8(); transmute(simd_select_bitmask(k, r.as_i32x8(), zero)) } @@ -24494,14 +22500,14 @@ pub unsafe fn _mm256_maskz_inserti32x4(k: __mmask8, a: __m256i, b: __m128i, imm8 /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_inserti64x4&expand=3186) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vinsertf64x4, imm8 = 1))] //should be vinserti64x4 -#[rustc_args_required_const(2)] -pub unsafe fn _mm512_inserti64x4(a: __m512i, b: __m256i, imm8: i32) -> __m512i { - assert!(imm8 >= 0 && imm8 <= 1); +#[cfg_attr(test, assert_instr(vinsertf64x4, IMM8 = 1))] //should be vinserti64x4 +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_inserti64x4(a: __m512i, b: __m256i) -> __m512i { + static_assert_imm1!(IMM8); let b = _mm512_castsi256_si512(b); - match imm8 & 0b1 { - 0 => simd_shuffle8(a, b, [8, 9, 10, 11, 4, 5, 6, 7]), - _ => simd_shuffle8(a, b, [0, 1, 2, 3, 8, 9, 10, 11]), + match IMM8 & 0b1 { + 0 => simd_shuffle8!(a, b, [8, 9, 10, 11, 4, 5, 6, 7]), + _ => simd_shuffle8!(a, b, [0, 1, 2, 3, 8, 9, 10, 11]), } } @@ -24510,21 +22516,16 @@ pub unsafe fn _mm512_inserti64x4(a: __m512i, b: __m256i, imm8: i32) -> __m512i { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_inserti64x4&expand=3187) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vinserti64x4, imm8 = 1))] -#[rustc_args_required_const(4)] -pub unsafe fn _mm512_mask_inserti64x4( +#[cfg_attr(test, assert_instr(vinserti64x4, IMM8 = 1))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_mask_inserti64x4( src: __m512i, k: __mmask8, a: __m512i, b: __m256i, - imm8: i32, ) -> __m512i { - macro_rules! call { - ($imm8:expr) => { - _mm512_inserti64x4(a, b, $imm8) - }; - } - let r = constify_imm8_sae!(imm8, call); + static_assert_imm1!(IMM8); + let r = _mm512_inserti64x4::(a, b); transmute(simd_select_bitmask(k, r.as_i64x8(), src.as_i64x8())) } @@ -24533,15 +22534,15 @@ pub unsafe fn _mm512_mask_inserti64x4( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_inserti64x4&expand=3188) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vinserti64x4, imm8 = 1))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm512_maskz_inserti64x4(k: __mmask8, a: __m512i, b: __m256i, imm8: i32) -> __m512i { - macro_rules! call { - ($imm8:expr) => { - _mm512_inserti64x4(a, b, $imm8) - }; - } - let r = constify_imm8_sae!(imm8, call); +#[cfg_attr(test, assert_instr(vinserti64x4, IMM8 = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_maskz_inserti64x4( + k: __mmask8, + a: __m512i, + b: __m256i, +) -> __m512i { + static_assert_imm1!(IMM8); + let r = _mm512_inserti64x4::(a, b); let zero = _mm512_setzero_si512().as_i64x8(); transmute(simd_select_bitmask(k, r.as_i64x8(), zero)) } @@ -24551,28 +22552,28 @@ pub unsafe fn _mm512_maskz_inserti64x4(k: __mmask8, a: __m512i, b: __m256i, imm8 /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_insertf32x4&expand=3155) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vinsertf32x4, imm8 = 2))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm512_insertf32x4(a: __m512, b: __m128, imm8: i32) -> __m512 { - assert!(imm8 >= 0 && imm8 <= 3); +#[cfg_attr(test, assert_instr(vinsertf32x4, IMM8 = 2))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_insertf32x4(a: __m512, b: __m128) -> __m512 { + static_assert_imm2!(IMM8); let b = _mm512_castps128_ps512(b); - match imm8 & 0b11 { - 0 => simd_shuffle16( + match IMM8 & 0b11 { + 0 => simd_shuffle16!( a, b, [16, 17, 18, 19, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15], ), - 1 => simd_shuffle16( + 1 => simd_shuffle16!( a, b, [0, 1, 2, 3, 16, 17, 18, 19, 8, 9, 10, 11, 12, 13, 14, 15], ), - 2 => simd_shuffle16( + 2 => simd_shuffle16!( a, b, [0, 1, 2, 3, 4, 5, 6, 7, 16, 17, 18, 19, 12, 13, 14, 15], ), - _ => simd_shuffle16(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 16, 17, 18, 19]), + _ => simd_shuffle16!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 16, 17, 18, 19]), } } @@ -24581,21 +22582,16 @@ pub unsafe fn _mm512_insertf32x4(a: __m512, b: __m128, imm8: i32) -> __m512 { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_insertf32x4&expand=3156) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vinsertf32x4, imm8 = 2))] -#[rustc_args_required_const(4)] -pub unsafe fn _mm512_mask_insertf32x4( +#[cfg_attr(test, assert_instr(vinsertf32x4, IMM8 = 2))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_mask_insertf32x4( src: __m512, k: __mmask16, a: __m512, b: __m128, - imm8: i32, ) -> __m512 { - macro_rules! call { - ($imm8:expr) => { - _mm512_insertf32x4(a, b, $imm8) - }; - } - let r = constify_imm8_sae!(imm8, call); + static_assert_imm2!(IMM8); + let r = _mm512_insertf32x4::(a, b); transmute(simd_select_bitmask(k, r.as_f32x16(), src.as_f32x16())) } @@ -24604,15 +22600,15 @@ pub unsafe fn _mm512_mask_insertf32x4( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_insertf32x4&expand=3157) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vinsertf32x4, imm8 = 2))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm512_maskz_insertf32x4(k: __mmask16, a: __m512, b: __m128, imm8: i32) -> __m512 { - macro_rules! call { - ($imm8:expr) => { - _mm512_insertf32x4(a, b, $imm8) - }; - } - let r = constify_imm8_sae!(imm8, call); +#[cfg_attr(test, assert_instr(vinsertf32x4, IMM8 = 2))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_maskz_insertf32x4( + k: __mmask16, + a: __m512, + b: __m128, +) -> __m512 { + static_assert_imm2!(IMM8); + let r = _mm512_insertf32x4::(a, b); let zero = _mm512_setzero_ps().as_f32x16(); transmute(simd_select_bitmask(k, r.as_f32x16(), zero)) } @@ -24624,15 +22620,15 @@ pub unsafe fn _mm512_maskz_insertf32x4(k: __mmask16, a: __m512, b: __m128, imm8: #[target_feature(enable = "avx512f,avx512vl")] #[cfg_attr( all(test, not(target_os = "windows")), - assert_instr(vinsert, imm8 = 1) //should be vinsertf32x4 + assert_instr(vinsert, IMM8 = 1) //should be vinsertf32x4 )] -#[rustc_args_required_const(2)] -pub unsafe fn _mm256_insertf32x4(a: __m256, b: __m128, imm8: i32) -> __m256 { - assert!(imm8 >= 0 && imm8 <= 1); +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm256_insertf32x4(a: __m256, b: __m128) -> __m256 { + static_assert_imm1!(IMM8); let b = _mm256_castps128_ps256(b); - match imm8 & 0b1 { - 0 => simd_shuffle8(a, b, [8, 9, 10, 11, 4, 5, 6, 7]), - _ => simd_shuffle8(a, b, [0, 1, 2, 3, 8, 9, 10, 11]), + match IMM8 & 0b1 { + 0 => simd_shuffle8!(a, b, [8, 9, 10, 11, 4, 5, 6, 7]), + _ => simd_shuffle8!(a, b, [0, 1, 2, 3, 8, 9, 10, 11]), } } @@ -24643,22 +22639,17 @@ pub unsafe fn _mm256_insertf32x4(a: __m256, b: __m128, imm8: i32) -> __m256 { #[target_feature(enable = "avx512f,avx512vl")] #[cfg_attr( all(test, not(target_os = "windows")), - assert_instr(vinsertf32x4, imm8 = 1) + assert_instr(vinsertf32x4, IMM8 = 1) )] -#[rustc_args_required_const(4)] -pub unsafe fn _mm256_mask_insertf32x4( +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm256_mask_insertf32x4( src: __m256, k: __mmask8, a: __m256, b: __m128, - imm8: i32, ) -> __m256 { - macro_rules! call { - ($imm8:expr) => { - _mm256_insertf32x4(a, b, $imm8) - }; - } - let r = constify_imm8_sae!(imm8, call); + static_assert_imm1!(IMM8); + let r = _mm256_insertf32x4::(a, b); transmute(simd_select_bitmask(k, r.as_f32x8(), src.as_f32x8())) } @@ -24669,16 +22660,16 @@ pub unsafe fn _mm256_mask_insertf32x4( #[target_feature(enable = "avx512f,avx512vl")] #[cfg_attr( all(test, not(target_os = "windows")), - assert_instr(vinsertf32x4, imm8 = 1) + assert_instr(vinsertf32x4, IMM8 = 1) )] -#[rustc_args_required_const(3)] -pub unsafe fn _mm256_maskz_insertf32x4(k: __mmask8, a: __m256, b: __m128, imm8: i32) -> __m256 { - macro_rules! call { - ($imm8:expr) => { - _mm256_insertf32x4(a, b, $imm8) - }; - } - let r = constify_imm8_sae!(imm8, call); +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm256_maskz_insertf32x4( + k: __mmask8, + a: __m256, + b: __m128, +) -> __m256 { + static_assert_imm1!(IMM8); + let r = _mm256_insertf32x4::(a, b); let zero = _mm256_setzero_ps().as_f32x8(); transmute(simd_select_bitmask(k, r.as_f32x8(), zero)) } @@ -24688,14 +22679,14 @@ pub unsafe fn _mm256_maskz_insertf32x4(k: __mmask8, a: __m256, b: __m128, imm8: /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_insertf64x4&expand=3167) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vinsertf64x4, imm8 = 1))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm512_insertf64x4(a: __m512d, b: __m256d, imm8: i32) -> __m512d { - assert!(imm8 >= 0 && imm8 <= 1); +#[cfg_attr(test, assert_instr(vinsertf64x4, IMM8 = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_insertf64x4(a: __m512d, b: __m256d) -> __m512d { + static_assert_imm1!(IMM8); let b = _mm512_castpd256_pd512(b); - match imm8 & 0b1 { - 0 => simd_shuffle8(a, b, [8, 9, 10, 11, 4, 5, 6, 7]), - _ => simd_shuffle8(a, b, [0, 1, 2, 3, 8, 9, 10, 11]), + match IMM8 & 0b1 { + 0 => simd_shuffle8!(a, b, [8, 9, 10, 11, 4, 5, 6, 7]), + _ => simd_shuffle8!(a, b, [0, 1, 2, 3, 8, 9, 10, 11]), } } @@ -24704,21 +22695,16 @@ pub unsafe fn _mm512_insertf64x4(a: __m512d, b: __m256d, imm8: i32) -> __m512d { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_insertf64x4&expand=3168) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vinsertf64x4, imm8 = 1))] -#[rustc_args_required_const(4)] -pub unsafe fn _mm512_mask_insertf64x4( +#[cfg_attr(test, assert_instr(vinsertf64x4, IMM8 = 1))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_mask_insertf64x4( src: __m512d, k: __mmask8, a: __m512d, b: __m256d, - imm8: i32, ) -> __m512d { - macro_rules! call { - ($imm8:expr) => { - _mm512_insertf64x4(a, b, $imm8) - }; - } - let r = constify_imm8_sae!(imm8, call); + static_assert_imm1!(IMM8); + let r = _mm512_insertf64x4::(a, b); transmute(simd_select_bitmask(k, r.as_f64x8(), src.as_f64x8())) } @@ -24727,15 +22713,15 @@ pub unsafe fn _mm512_mask_insertf64x4( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_insertf64x4&expand=3169) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vinsertf64x4, imm8 = 1))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm512_maskz_insertf64x4(k: __mmask8, a: __m512d, b: __m256d, imm8: i32) -> __m512d { - macro_rules! call { - ($imm8:expr) => { - _mm512_insertf64x4(a, b, $imm8) - }; - } - let r = constify_imm8_sae!(imm8, call); +#[cfg_attr(test, assert_instr(vinsertf64x4, IMM8 = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_maskz_insertf64x4( + k: __mmask8, + a: __m512d, + b: __m256d, +) -> __m512d { + static_assert_imm1!(IMM8); + let r = _mm512_insertf64x4::(a, b); let zero = _mm512_setzero_pd().as_f64x8(); transmute(simd_select_bitmask(k, r.as_f64x8(), zero)) } @@ -24750,7 +22736,7 @@ pub unsafe fn _mm512_unpackhi_epi32(a: __m512i, b: __m512i) -> __m512i { let a = a.as_i32x16(); let b = b.as_i32x16(); #[rustfmt::skip] - let r: i32x16 = simd_shuffle16( + let r: i32x16 = simd_shuffle16!( a, b, [ 2, 18, 3, 19, 2 + 4, 18 + 4, 3 + 4, 19 + 4, @@ -24851,7 +22837,7 @@ pub unsafe fn _mm_maskz_unpackhi_epi32(k: __mmask8, a: __m128i, b: __m128i) -> _ #[target_feature(enable = "avx512f")] #[cfg_attr(test, assert_instr(vunpckhpd))] //should be vpunpckhqdq pub unsafe fn _mm512_unpackhi_epi64(a: __m512i, b: __m512i) -> __m512i { - simd_shuffle8(a, b, [1, 9, 1 + 2, 9 + 2, 1 + 4, 9 + 4, 1 + 6, 9 + 6]) + simd_shuffle8!(a, b, [1, 9, 1 + 2, 9 + 2, 1 + 4, 9 + 4, 1 + 6, 9 + 6]) } /// Unpack and interleave 64-bit integers from the high half of each 128-bit lane in a and b, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). @@ -24946,7 +22932,7 @@ pub unsafe fn _mm_maskz_unpackhi_epi64(k: __mmask8, a: __m128i, b: __m128i) -> _ #[cfg_attr(test, assert_instr(vunpckhps))] pub unsafe fn _mm512_unpackhi_ps(a: __m512, b: __m512) -> __m512 { #[rustfmt::skip] - simd_shuffle16( + simd_shuffle16!( a, b, [ 2, 18, 3, 19, 2 + 4, 18 + 4, 3 + 4, 19 + 4, @@ -25031,7 +23017,7 @@ pub unsafe fn _mm_maskz_unpackhi_ps(k: __mmask8, a: __m128, b: __m128) -> __m128 #[target_feature(enable = "avx512f")] #[cfg_attr(test, assert_instr(vunpckhpd))] pub unsafe fn _mm512_unpackhi_pd(a: __m512d, b: __m512d) -> __m512d { - simd_shuffle8(a, b, [1, 9, 1 + 2, 9 + 2, 1 + 4, 9 + 4, 1 + 6, 9 + 6]) + simd_shuffle8!(a, b, [1, 9, 1 + 2, 9 + 2, 1 + 4, 9 + 4, 1 + 6, 9 + 6]) } /// Unpack and interleave double-precision (64-bit) floating-point elements from the high half of each 128-bit lane in a and b, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). @@ -25123,7 +23109,7 @@ pub unsafe fn _mm512_unpacklo_epi32(a: __m512i, b: __m512i) -> __m512i { let a = a.as_i32x16(); let b = b.as_i32x16(); #[rustfmt::skip] - let r: i32x16 = simd_shuffle16( + let r: i32x16 = simd_shuffle16!( a, b, [ 0, 16, 1, 17, 0 + 4, 16 + 4, 1 + 4, 17 + 4, @@ -25224,7 +23210,7 @@ pub unsafe fn _mm_maskz_unpacklo_epi32(k: __mmask8, a: __m128i, b: __m128i) -> _ #[target_feature(enable = "avx512f")] #[cfg_attr(test, assert_instr(vunpcklpd))] //should be vpunpcklqdq pub unsafe fn _mm512_unpacklo_epi64(a: __m512i, b: __m512i) -> __m512i { - simd_shuffle8(a, b, [0, 8, 0 + 2, 8 + 2, 0 + 4, 8 + 4, 0 + 6, 8 + 6]) + simd_shuffle8!(a, b, [0, 8, 0 + 2, 8 + 2, 0 + 4, 8 + 4, 0 + 6, 8 + 6]) } /// Unpack and interleave 64-bit integers from the low half of each 128-bit lane in a and b, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). @@ -25319,7 +23305,7 @@ pub unsafe fn _mm_maskz_unpacklo_epi64(k: __mmask8, a: __m128i, b: __m128i) -> _ #[cfg_attr(test, assert_instr(vunpcklps))] pub unsafe fn _mm512_unpacklo_ps(a: __m512, b: __m512) -> __m512 { #[rustfmt::skip] - simd_shuffle16(a, b, + simd_shuffle16!(a, b, [ 0, 16, 1, 17, 0 + 4, 16 + 4, 1 + 4, 17 + 4, 0 + 8, 16 + 8, 1 + 8, 17 + 8, @@ -25403,7 +23389,7 @@ pub unsafe fn _mm_maskz_unpacklo_ps(k: __mmask8, a: __m128, b: __m128) -> __m128 #[target_feature(enable = "avx512f")] #[cfg_attr(test, assert_instr(vunpcklpd))] pub unsafe fn _mm512_unpacklo_pd(a: __m512d, b: __m512d) -> __m512d { - simd_shuffle8(a, b, [0, 8, 0 + 2, 8 + 2, 0 + 4, 8 + 4, 0 + 6, 8 + 6]) + simd_shuffle8!(a, b, [0, 8, 0 + 2, 8 + 2, 0 + 4, 8 + 4, 0 + 6, 8 + 6]) } /// Unpack and interleave double-precision (64-bit) floating-point elements from the low half of each 128-bit lane in a and b, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). @@ -25491,7 +23477,7 @@ pub unsafe fn _mm_maskz_unpacklo_pd(k: __mmask8, a: __m128d, b: __m128d) -> __m1 #[inline] #[target_feature(enable = "avx512f")] pub unsafe fn _mm512_castps128_ps512(a: __m128) -> __m512 { - simd_shuffle16( + simd_shuffle16!( a, _mm_set1_ps(-1.), [0, 1, 2, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4], @@ -25504,7 +23490,7 @@ pub unsafe fn _mm512_castps128_ps512(a: __m128) -> __m512 { #[inline] #[target_feature(enable = "avx512f")] pub unsafe fn _mm512_castps256_ps512(a: __m256) -> __m512 { - simd_shuffle16( + simd_shuffle16!( a, _mm256_set1_ps(-1.), [0, 1, 2, 3, 4, 5, 6, 7, 8, 8, 8, 8, 8, 8, 8, 8], @@ -25517,7 +23503,7 @@ pub unsafe fn _mm512_castps256_ps512(a: __m256) -> __m512 { #[inline] #[target_feature(enable = "avx512f")] pub unsafe fn _mm512_zextps128_ps512(a: __m128) -> __m512 { - simd_shuffle16( + simd_shuffle16!( a, _mm_set1_ps(0.), [0, 1, 2, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4], @@ -25530,7 +23516,7 @@ pub unsafe fn _mm512_zextps128_ps512(a: __m128) -> __m512 { #[inline] #[target_feature(enable = "avx512f")] pub unsafe fn _mm512_zextps256_ps512(a: __m256) -> __m512 { - simd_shuffle16( + simd_shuffle16!( a, _mm256_set1_ps(0.), [0, 1, 2, 3, 4, 5, 6, 7, 8, 8, 8, 8, 8, 8, 8, 8], @@ -25543,7 +23529,7 @@ pub unsafe fn _mm512_zextps256_ps512(a: __m256) -> __m512 { #[inline] #[target_feature(enable = "avx512f")] pub unsafe fn _mm512_castps512_ps128(a: __m512) -> __m128 { - simd_shuffle4(a, a, [0, 1, 2, 3]) + simd_shuffle4!(a, a, [0, 1, 2, 3]) } /// Cast vector of type __m512 to type __m256. This intrinsic is only used for compilation and does not generate any instructions, thus it has zero latency. @@ -25552,7 +23538,7 @@ pub unsafe fn _mm512_castps512_ps128(a: __m512) -> __m128 { #[inline] #[target_feature(enable = "avx512f")] pub unsafe fn _mm512_castps512_ps256(a: __m512) -> __m256 { - simd_shuffle8(a, a, [0, 1, 2, 3, 4, 5, 6, 7]) + simd_shuffle8!(a, a, [0, 1, 2, 3, 4, 5, 6, 7]) } /// Cast vector of type __m512 to type __m512d. This intrinsic is only used for compilation and does not generate any instructions, thus it has zero latency. @@ -25579,7 +23565,7 @@ pub unsafe fn _mm512_castps_si512(a: __m512) -> __m512i { #[inline] #[target_feature(enable = "avx512f")] pub unsafe fn _mm512_castpd128_pd512(a: __m128d) -> __m512d { - simd_shuffle8(a, _mm_set1_pd(-1.), [0, 1, 2, 2, 2, 2, 2, 2]) + simd_shuffle8!(a, _mm_set1_pd(-1.), [0, 1, 2, 2, 2, 2, 2, 2]) } /// Cast vector of type __m256d to type __m512d; the upper 256 bits of the result are undefined. This intrinsic is only used for compilation and does not generate any instructions, thus it has zero latency. @@ -25588,7 +23574,7 @@ pub unsafe fn _mm512_castpd128_pd512(a: __m128d) -> __m512d { #[inline] #[target_feature(enable = "avx512f")] pub unsafe fn _mm512_castpd256_pd512(a: __m256d) -> __m512d { - simd_shuffle8(a, _mm256_set1_pd(-1.), [0, 1, 2, 3, 4, 4, 4, 4]) + simd_shuffle8!(a, _mm256_set1_pd(-1.), [0, 1, 2, 3, 4, 4, 4, 4]) } /// Cast vector of type __m128d to type __m512d; the upper 384 bits of the result are zeroed. This intrinsic is only used for compilation and does not generate any instructions, thus it has zero latency. @@ -25597,7 +23583,7 @@ pub unsafe fn _mm512_castpd256_pd512(a: __m256d) -> __m512d { #[inline] #[target_feature(enable = "avx512f")] pub unsafe fn _mm512_zextpd128_pd512(a: __m128d) -> __m512d { - simd_shuffle8(a, _mm_set1_pd(0.), [0, 1, 2, 2, 2, 2, 2, 2]) + simd_shuffle8!(a, _mm_set1_pd(0.), [0, 1, 2, 2, 2, 2, 2, 2]) } /// Cast vector of type __m256d to type __m512d; the upper 256 bits of the result are zeroed. This intrinsic is only used for compilation and does not generate any instructions, thus it has zero latency. @@ -25606,7 +23592,7 @@ pub unsafe fn _mm512_zextpd128_pd512(a: __m128d) -> __m512d { #[inline] #[target_feature(enable = "avx512f")] pub unsafe fn _mm512_zextpd256_pd512(a: __m256d) -> __m512d { - simd_shuffle8(a, _mm256_set1_pd(0.), [0, 1, 2, 3, 4, 4, 4, 4]) + simd_shuffle8!(a, _mm256_set1_pd(0.), [0, 1, 2, 3, 4, 4, 4, 4]) } /// Cast vector of type __m512d to type __m128d. This intrinsic is only used for compilation and does not generate any instructions, thus it has zero latency. @@ -25615,7 +23601,7 @@ pub unsafe fn _mm512_zextpd256_pd512(a: __m256d) -> __m512d { #[inline] #[target_feature(enable = "avx512f")] pub unsafe fn _mm512_castpd512_pd128(a: __m512d) -> __m128d { - simd_shuffle2(a, a, [0, 1]) + simd_shuffle2!(a, a, [0, 1]) } /// Cast vector of type __m512d to type __m256d. This intrinsic is only used for compilation and does not generate any instructions, thus it has zero latency. @@ -25624,7 +23610,7 @@ pub unsafe fn _mm512_castpd512_pd128(a: __m512d) -> __m128d { #[inline] #[target_feature(enable = "avx512f")] pub unsafe fn _mm512_castpd512_pd256(a: __m512d) -> __m256d { - simd_shuffle4(a, a, [0, 1, 2, 3]) + simd_shuffle4!(a, a, [0, 1, 2, 3]) } /// Cast vector of type __m512d to type __m512. This intrinsic is only used for compilation and does not generate any instructions, thus it has zero latency. @@ -25651,7 +23637,7 @@ pub unsafe fn _mm512_castpd_si512(a: __m512d) -> __m512i { #[inline] #[target_feature(enable = "avx512f")] pub unsafe fn _mm512_castsi128_si512(a: __m128i) -> __m512i { - simd_shuffle8(a, _mm_set1_epi64x(-1), [0, 1, 2, 2, 2, 2, 2, 2]) + simd_shuffle8!(a, _mm_set1_epi64x(-1), [0, 1, 2, 2, 2, 2, 2, 2]) } /// Cast vector of type __m256i to type __m512i; the upper 256 bits of the result are undefined. This intrinsic is only used for compilation and does not generate any instructions, thus it has zero latency. @@ -25660,7 +23646,7 @@ pub unsafe fn _mm512_castsi128_si512(a: __m128i) -> __m512i { #[inline] #[target_feature(enable = "avx512f")] pub unsafe fn _mm512_castsi256_si512(a: __m256i) -> __m512i { - simd_shuffle8(a, _mm256_set1_epi64x(-1), [0, 1, 2, 3, 4, 4, 4, 4]) + simd_shuffle8!(a, _mm256_set1_epi64x(-1), [0, 1, 2, 3, 4, 4, 4, 4]) } /// Cast vector of type __m128i to type __m512i; the upper 384 bits of the result are zeroed. This intrinsic is only used for compilation and does not generate any instructions, thus it has zero latency. @@ -25669,7 +23655,7 @@ pub unsafe fn _mm512_castsi256_si512(a: __m256i) -> __m512i { #[inline] #[target_feature(enable = "avx512f")] pub unsafe fn _mm512_zextsi128_si512(a: __m128i) -> __m512i { - simd_shuffle8(a, _mm_set1_epi64x(0), [0, 1, 2, 2, 2, 2, 2, 2]) + simd_shuffle8!(a, _mm_set1_epi64x(0), [0, 1, 2, 2, 2, 2, 2, 2]) } /// Cast vector of type __m256i to type __m512i; the upper 256 bits of the result are zeroed. This intrinsic is only used for compilation and does not generate any instructions, thus it has zero latency. @@ -25678,7 +23664,7 @@ pub unsafe fn _mm512_zextsi128_si512(a: __m128i) -> __m512i { #[inline] #[target_feature(enable = "avx512f")] pub unsafe fn _mm512_zextsi256_si512(a: __m256i) -> __m512i { - simd_shuffle8(a, _mm256_set1_epi64x(0), [0, 1, 2, 3, 4, 4, 4, 4]) + simd_shuffle8!(a, _mm256_set1_epi64x(0), [0, 1, 2, 3, 4, 4, 4, 4]) } /// Cast vector of type __m512i to type __m128i. This intrinsic is only used for compilation and does not generate any instructions, thus it has zero latency. @@ -25687,7 +23673,7 @@ pub unsafe fn _mm512_zextsi256_si512(a: __m256i) -> __m512i { #[inline] #[target_feature(enable = "avx512f")] pub unsafe fn _mm512_castsi512_si128(a: __m512i) -> __m128i { - simd_shuffle2(a, a, [0, 1]) + simd_shuffle2!(a, a, [0, 1]) } /// Cast vector of type __m512i to type __m256i. This intrinsic is only used for compilation and does not generate any instructions, thus it has zero latency. @@ -25696,7 +23682,7 @@ pub unsafe fn _mm512_castsi512_si128(a: __m512i) -> __m128i { #[inline] #[target_feature(enable = "avx512f")] pub unsafe fn _mm512_castsi512_si256(a: __m512i) -> __m256i { - simd_shuffle4(a, a, [0, 1, 2, 3]) + simd_shuffle4!(a, a, [0, 1, 2, 3]) } /// Cast vector of type __m512i to type __m512. This intrinsic is only used for compilation and does not generate any instructions, thus it has zero latency. @@ -25736,7 +23722,7 @@ pub unsafe fn _mm512_cvtsi512_si32(a: __m512i) -> i32 { #[cfg_attr(test, assert_instr(vbroadcast))] //should be vpbroadcastd pub unsafe fn _mm512_broadcastd_epi32(a: __m128i) -> __m512i { let a = _mm512_castsi128_si512(a).as_i32x16(); - let ret: i32x16 = simd_shuffle16(a, a, [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]); + let ret: i32x16 = simd_shuffle16!(a, a, [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]); transmute(ret) } @@ -25816,7 +23802,7 @@ pub unsafe fn _mm_maskz_broadcastd_epi32(k: __mmask8, a: __m128i) -> __m128i { #[target_feature(enable = "avx512f")] #[cfg_attr(test, assert_instr(vbroadcas))] //should be vpbroadcastq pub unsafe fn _mm512_broadcastq_epi64(a: __m128i) -> __m512i { - simd_shuffle8(a, a, [0, 0, 0, 0, 0, 0, 0, 0]) + simd_shuffle8!(a, a, [0, 0, 0, 0, 0, 0, 0, 0]) } /// Broadcast the low packed 64-bit integer from a to all elements of dst using writemask k (elements are copied from src when the corresponding mask bit is not set). @@ -25895,7 +23881,7 @@ pub unsafe fn _mm_maskz_broadcastq_epi64(k: __mmask8, a: __m128i) -> __m128i { #[target_feature(enable = "avx512f")] #[cfg_attr(test, assert_instr(vbroadcastss))] pub unsafe fn _mm512_broadcastss_ps(a: __m128) -> __m512 { - simd_shuffle16(a, a, [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]) + simd_shuffle16!(a, a, [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]) } /// Broadcast the low single-precision (32-bit) floating-point element from a to all elements of dst using writemask k (elements are copied from src when the corresponding mask bit is not set). @@ -25974,7 +23960,7 @@ pub unsafe fn _mm_maskz_broadcastss_ps(k: __mmask8, a: __m128) -> __m128 { #[target_feature(enable = "avx512f")] #[cfg_attr(test, assert_instr(vbroadcastsd))] pub unsafe fn _mm512_broadcastsd_pd(a: __m128d) -> __m512d { - simd_shuffle8(a, a, [0, 0, 0, 0, 0, 0, 0, 0]) + simd_shuffle8!(a, a, [0, 0, 0, 0, 0, 0, 0, 0]) } /// Broadcast the low double-precision (64-bit) floating-point element from a to all elements of dst using writemask k (elements are copied from src when the corresponding mask bit is not set). @@ -26030,7 +24016,7 @@ pub unsafe fn _mm256_maskz_broadcastsd_pd(k: __mmask8, a: __m128d) -> __m256d { #[target_feature(enable = "avx512f")] //msvc: vbroadcasti32x4, linux: vshuf pub unsafe fn _mm512_broadcast_i32x4(a: __m128i) -> __m512i { let a = a.as_i32x4(); - let ret: i32x16 = simd_shuffle16(a, a, [0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3]); + let ret: i32x16 = simd_shuffle16!(a, a, [0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3]); transmute(ret) } @@ -26062,7 +24048,7 @@ pub unsafe fn _mm512_maskz_broadcast_i32x4(k: __mmask16, a: __m128i) -> __m512i #[target_feature(enable = "avx512f,avx512vl")] //msvc: vbroadcasti32x4, linux: vshuf pub unsafe fn _mm256_broadcast_i32x4(a: __m128i) -> __m256i { let a = a.as_i32x4(); - let ret: i32x8 = simd_shuffle8(a, a, [0, 1, 2, 3, 0, 1, 2, 3]); + let ret: i32x8 = simd_shuffle8!(a, a, [0, 1, 2, 3, 0, 1, 2, 3]); transmute(ret) } @@ -26093,7 +24079,7 @@ pub unsafe fn _mm256_maskz_broadcast_i32x4(k: __mmask8, a: __m128i) -> __m256i { #[inline] #[target_feature(enable = "avx512f")] //msvc: vbroadcasti64x4, linux: vperm pub unsafe fn _mm512_broadcast_i64x4(a: __m256i) -> __m512i { - simd_shuffle8(a, a, [0, 1, 2, 3, 0, 1, 2, 3]) + simd_shuffle8!(a, a, [0, 1, 2, 3, 0, 1, 2, 3]) } /// Broadcast the 4 packed 64-bit integers from a to all elements of dst using writemask k (elements are copied from src when the corresponding mask bit is not set). @@ -26123,7 +24109,7 @@ pub unsafe fn _mm512_maskz_broadcast_i64x4(k: __mmask8, a: __m256i) -> __m512i { #[inline] #[target_feature(enable = "avx512f")] //msvc: vbroadcastf32x4, linux: vshuf pub unsafe fn _mm512_broadcast_f32x4(a: __m128) -> __m512 { - simd_shuffle16(a, a, [0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3]) + simd_shuffle16!(a, a, [0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3]) } /// Broadcast the 4 packed single-precision (32-bit) floating-point elements from a to all elements of dst using writemask k (elements are copied from src when the corresponding mask bit is not set). @@ -26153,7 +24139,7 @@ pub unsafe fn _mm512_maskz_broadcast_f32x4(k: __mmask16, a: __m128) -> __m512 { #[inline] #[target_feature(enable = "avx512f,avx512vl")] //msvc: vbroadcastf32x4, linux: vshuf pub unsafe fn _mm256_broadcast_f32x4(a: __m128) -> __m256 { - simd_shuffle8(a, a, [0, 1, 2, 3, 0, 1, 2, 3]) + simd_shuffle8!(a, a, [0, 1, 2, 3, 0, 1, 2, 3]) } /// Broadcast the 4 packed single-precision (32-bit) floating-point elements from a to all elements of dst using writemask k (elements are copied from src when the corresponding mask bit is not set). @@ -26183,7 +24169,7 @@ pub unsafe fn _mm256_maskz_broadcast_f32x4(k: __mmask8, a: __m128) -> __m256 { #[inline] #[target_feature(enable = "avx512f")] //msvc: vbroadcastf64x4, linux: vperm pub unsafe fn _mm512_broadcast_f64x4(a: __m256d) -> __m512d { - simd_shuffle8(a, a, [0, 1, 2, 3, 0, 1, 2, 3]) + simd_shuffle8!(a, a, [0, 1, 2, 3, 0, 1, 2, 3]) } /// Broadcast the 4 packed double-precision (64-bit) floating-point elements from a to all elements of dst using writemask k (elements are copied from src when the corresponding mask bit is not set). @@ -26332,74 +24318,70 @@ pub unsafe fn _mm_mask_blend_pd(k: __mmask8, a: __m128d, b: __m128d) -> __m128d /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=512_alignr_epi32&expand=245) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(valignd, imm8 = 1))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm512_alignr_epi32(a: __m512i, b: __m512i, imm8: i32) -> __m512i { - assert!(imm8 >= 0 && imm8 <= 255); +#[cfg_attr(test, assert_instr(valignd, IMM8 = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_alignr_epi32(a: __m512i, b: __m512i) -> __m512i { + static_assert_imm8!(IMM8); let a = a.as_i32x16(); let b = b.as_i32x16(); - let imm8: i32 = imm8 % 16; + let imm8: i32 = IMM8 % 16; let r: i32x16 = match imm8 { - 0 => simd_shuffle16( + 0 => simd_shuffle16!( a, b, - [ - 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, - ], + [16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31,], ), - 1 => simd_shuffle16( + 1 => simd_shuffle16!( a, b, - [ - 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 0, - ], + [17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 0,], ), - 2 => simd_shuffle16( + 2 => simd_shuffle16!( a, b, [18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 0, 1], ), - 3 => simd_shuffle16( + 3 => simd_shuffle16!( a, b, [19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 0, 1, 2], ), - 4 => simd_shuffle16( + 4 => simd_shuffle16!( a, b, [20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 0, 1, 2, 3], ), - 5 => simd_shuffle16( + 5 => simd_shuffle16!( a, b, [21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 0, 1, 2, 3, 4], ), - 6 => simd_shuffle16( + 6 => simd_shuffle16!( a, b, [22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 0, 1, 2, 3, 4, 5], ), - 7 => simd_shuffle16( + 7 => simd_shuffle16!( a, b, [23, 24, 25, 26, 27, 28, 29, 30, 31, 0, 1, 2, 3, 4, 5, 6], ), - 8 => simd_shuffle16( + 8 => simd_shuffle16!( a, b, [24, 25, 26, 27, 28, 29, 30, 31, 0, 1, 2, 3, 4, 5, 6, 7], ), - 9 => simd_shuffle16( + 9 => simd_shuffle16!( a, b, [25, 26, 27, 28, 29, 30, 31, 0, 1, 2, 3, 4, 5, 6, 7, 8], ), - 10 => simd_shuffle16(a, b, [26, 27, 28, 29, 30, 31, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9]), - 11 => simd_shuffle16(a, b, [27, 28, 29, 30, 31, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]), - 12 => simd_shuffle16(a, b, [28, 29, 30, 31, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]), - 13 => simd_shuffle16(a, b, [29, 30, 31, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]), - 14 => simd_shuffle16(a, b, [30, 31, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13]), - _ => simd_shuffle16(a, b, [31, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14]), + 10 => simd_shuffle16!(a, b, [26, 27, 28, 29, 30, 31, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9]), + 11 => simd_shuffle16!(a, b, [27, 28, 29, 30, 31, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]), + 12 => simd_shuffle16!(a, b, [28, 29, 30, 31, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]), + 13 => simd_shuffle16!(a, b, [29, 30, 31, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]), + 14 => simd_shuffle16!(a, b, [30, 31, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13]), + _ => simd_shuffle16!(a, b, [31, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14]), }; transmute(r) } @@ -26409,21 +24391,16 @@ pub unsafe fn _mm512_alignr_epi32(a: __m512i, b: __m512i, imm8: i32) -> __m512i /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=512_mask_alignr_epi32&expand=246) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(valignd, imm8 = 1))] -#[rustc_args_required_const(4)] -pub unsafe fn _mm512_mask_alignr_epi32( +#[cfg_attr(test, assert_instr(valignd, IMM8 = 1))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_mask_alignr_epi32( src: __m512i, k: __mmask16, a: __m512i, b: __m512i, - imm8: i32, ) -> __m512i { - macro_rules! call { - ($imm8:expr) => { - _mm512_alignr_epi32(a, b, $imm8) - }; - } - let r = constify_imm8_sae!(imm8, call); + static_assert_imm8!(IMM8); + let r = _mm512_alignr_epi32::(a, b); transmute(simd_select_bitmask(k, r.as_i32x16(), src.as_i32x16())) } @@ -26432,20 +24409,15 @@ pub unsafe fn _mm512_mask_alignr_epi32( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=512_maskz_alignr_epi32&expand=247) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(valignd, imm8 = 1))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm512_maskz_alignr_epi32( +#[cfg_attr(test, assert_instr(valignd, IMM8 = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_maskz_alignr_epi32( k: __mmask16, a: __m512i, b: __m512i, - imm8: i32, ) -> __m512i { - macro_rules! call { - ($imm8:expr) => { - _mm512_alignr_epi32(a, b, $imm8) - }; - } - let r = constify_imm8_sae!(imm8, call); + static_assert_imm8!(IMM8); + let r = _mm512_alignr_epi32::(a, b); let zero = _mm512_setzero_si512().as_i32x16(); transmute(simd_select_bitmask(k, r.as_i32x16(), zero)) } @@ -26455,30 +24427,30 @@ pub unsafe fn _mm512_maskz_alignr_epi32( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_alignr_epi32&expand=242) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[cfg_attr(test, assert_instr(valignd, imm8 = 1))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm256_alignr_epi32(a: __m256i, b: __m256i, imm8: i32) -> __m256i { - assert!(imm8 >= 0 && imm8 <= 255); +#[cfg_attr(test, assert_instr(valignd, IMM8 = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm256_alignr_epi32(a: __m256i, b: __m256i) -> __m256i { + static_assert_imm8!(IMM8); let a = a.as_i32x8(); let b = b.as_i32x8(); - let imm8: i32 = imm8 % 16; + let imm8: i32 = IMM8 % 16; let r: i32x8 = match imm8 { - 0 => simd_shuffle8(a, b, [8, 9, 10, 11, 12, 13, 14, 15]), - 1 => simd_shuffle8(a, b, [9, 10, 11, 12, 13, 14, 15, 0]), - 2 => simd_shuffle8(a, b, [10, 11, 12, 13, 14, 15, 0, 1]), - 3 => simd_shuffle8(a, b, [11, 12, 13, 14, 15, 0, 1, 2]), - 4 => simd_shuffle8(a, b, [12, 13, 14, 15, 0, 1, 2, 3]), - 5 => simd_shuffle8(a, b, [13, 14, 15, 0, 1, 2, 3, 4]), - 6 => simd_shuffle8(a, b, [14, 15, 0, 1, 2, 3, 4, 5]), - 7 => simd_shuffle8(a, b, [15, 0, 1, 2, 3, 4, 5, 6]), - 8 => simd_shuffle8(a, b, [0, 1, 2, 3, 4, 5, 6, 7]), - 9 => simd_shuffle8(a, b, [1, 2, 3, 4, 5, 6, 7, 8]), - 10 => simd_shuffle8(a, b, [2, 3, 4, 5, 6, 7, 8, 9]), - 11 => simd_shuffle8(a, b, [3, 4, 5, 6, 7, 8, 9, 10]), - 12 => simd_shuffle8(a, b, [4, 5, 6, 7, 8, 9, 10, 11]), - 13 => simd_shuffle8(a, b, [5, 6, 7, 8, 9, 10, 11, 12]), - 14 => simd_shuffle8(a, b, [6, 7, 8, 9, 10, 11, 12, 13]), - _ => simd_shuffle8(a, b, [7, 8, 9, 10, 11, 12, 13, 14]), + 0 => simd_shuffle8!(a, b, [8, 9, 10, 11, 12, 13, 14, 15]), + 1 => simd_shuffle8!(a, b, [9, 10, 11, 12, 13, 14, 15, 0]), + 2 => simd_shuffle8!(a, b, [10, 11, 12, 13, 14, 15, 0, 1]), + 3 => simd_shuffle8!(a, b, [11, 12, 13, 14, 15, 0, 1, 2]), + 4 => simd_shuffle8!(a, b, [12, 13, 14, 15, 0, 1, 2, 3]), + 5 => simd_shuffle8!(a, b, [13, 14, 15, 0, 1, 2, 3, 4]), + 6 => simd_shuffle8!(a, b, [14, 15, 0, 1, 2, 3, 4, 5]), + 7 => simd_shuffle8!(a, b, [15, 0, 1, 2, 3, 4, 5, 6]), + 8 => simd_shuffle8!(a, b, [0, 1, 2, 3, 4, 5, 6, 7]), + 9 => simd_shuffle8!(a, b, [1, 2, 3, 4, 5, 6, 7, 8]), + 10 => simd_shuffle8!(a, b, [2, 3, 4, 5, 6, 7, 8, 9]), + 11 => simd_shuffle8!(a, b, [3, 4, 5, 6, 7, 8, 9, 10]), + 12 => simd_shuffle8!(a, b, [4, 5, 6, 7, 8, 9, 10, 11]), + 13 => simd_shuffle8!(a, b, [5, 6, 7, 8, 9, 10, 11, 12]), + 14 => simd_shuffle8!(a, b, [6, 7, 8, 9, 10, 11, 12, 13]), + _ => simd_shuffle8!(a, b, [7, 8, 9, 10, 11, 12, 13, 14]), }; transmute(r) } @@ -26488,21 +24460,16 @@ pub unsafe fn _mm256_alignr_epi32(a: __m256i, b: __m256i, imm8: i32) -> __m256i /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_alignr_epi32&expand=243) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[cfg_attr(test, assert_instr(valignd, imm8 = 1))] -#[rustc_args_required_const(4)] -pub unsafe fn _mm256_mask_alignr_epi32( +#[cfg_attr(test, assert_instr(valignd, IMM8 = 1))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm256_mask_alignr_epi32( src: __m256i, k: __mmask8, a: __m256i, b: __m256i, - imm8: i32, ) -> __m256i { - macro_rules! call { - ($imm8:expr) => { - _mm256_alignr_epi32(a, b, $imm8) - }; - } - let r = constify_imm8_sae!(imm8, call); + static_assert_imm8!(IMM8); + let r = _mm256_alignr_epi32::(a, b); transmute(simd_select_bitmask(k, r.as_i32x8(), src.as_i32x8())) } @@ -26511,15 +24478,15 @@ pub unsafe fn _mm256_mask_alignr_epi32( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_alignr_epi32&expand=244) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[cfg_attr(test, assert_instr(valignd, imm8 = 1))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm256_maskz_alignr_epi32(k: __mmask8, a: __m256i, b: __m256i, imm8: i32) -> __m256i { - macro_rules! call { - ($imm8:expr) => { - _mm256_alignr_epi32(a, b, $imm8) - }; - } - let r = constify_imm8_sae!(imm8, call); +#[cfg_attr(test, assert_instr(valignd, IMM8 = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm256_maskz_alignr_epi32( + k: __mmask8, + a: __m256i, + b: __m256i, +) -> __m256i { + static_assert_imm8!(IMM8); + let r = _mm256_alignr_epi32::(a, b); let zero = _mm256_setzero_si256().as_i32x8(); transmute(simd_select_bitmask(k, r.as_i32x8(), zero)) } @@ -26529,22 +24496,22 @@ pub unsafe fn _mm256_maskz_alignr_epi32(k: __mmask8, a: __m256i, b: __m256i, imm /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_alignr_epi32&expand=239) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[cfg_attr(test, assert_instr(vpalignr, imm8 = 1))] //should be valignd -#[rustc_args_required_const(2)] -pub unsafe fn _mm_alignr_epi32(a: __m128i, b: __m128i, imm8: i32) -> __m128i { - assert!(imm8 >= 0 && imm8 <= 255); +#[cfg_attr(test, assert_instr(vpalignr, IMM8 = 1))] //should be valignd +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm_alignr_epi32(a: __m128i, b: __m128i) -> __m128i { + static_assert_imm8!(IMM8); let a = a.as_i32x4(); let b = b.as_i32x4(); - let imm8: i32 = imm8 % 8; + let imm8: i32 = IMM8 % 8; let r: i32x4 = match imm8 { - 0 => simd_shuffle4(a, b, [4, 5, 6, 7]), - 1 => simd_shuffle4(a, b, [5, 6, 7, 0]), - 2 => simd_shuffle4(a, b, [6, 7, 0, 1]), - 3 => simd_shuffle4(a, b, [7, 0, 1, 2]), - 4 => simd_shuffle4(a, b, [0, 1, 2, 3]), - 5 => simd_shuffle4(a, b, [1, 2, 3, 0]), - 6 => simd_shuffle4(a, b, [2, 3, 0, 1]), - _ => simd_shuffle4(a, b, [3, 0, 1, 2]), + 0 => simd_shuffle4!(a, b, [4, 5, 6, 7]), + 1 => simd_shuffle4!(a, b, [5, 6, 7, 0]), + 2 => simd_shuffle4!(a, b, [6, 7, 0, 1]), + 3 => simd_shuffle4!(a, b, [7, 0, 1, 2]), + 4 => simd_shuffle4!(a, b, [0, 1, 2, 3]), + 5 => simd_shuffle4!(a, b, [1, 2, 3, 0]), + 6 => simd_shuffle4!(a, b, [2, 3, 0, 1]), + _ => simd_shuffle4!(a, b, [3, 0, 1, 2]), }; transmute(r) } @@ -26554,21 +24521,16 @@ pub unsafe fn _mm_alignr_epi32(a: __m128i, b: __m128i, imm8: i32) -> __m128i { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_alignr_epi32&expand=240) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[cfg_attr(test, assert_instr(valignd, imm8 = 1))] -#[rustc_args_required_const(4)] -pub unsafe fn _mm_mask_alignr_epi32( +#[cfg_attr(test, assert_instr(valignd, IMM8 = 1))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm_mask_alignr_epi32( src: __m128i, k: __mmask8, a: __m128i, b: __m128i, - imm8: i32, ) -> __m128i { - macro_rules! call { - ($imm8:expr) => { - _mm_alignr_epi32(a, b, $imm8) - }; - } - let r = constify_imm8_sae!(imm8, call); + static_assert_imm8!(IMM8); + let r = _mm_alignr_epi32::(a, b); transmute(simd_select_bitmask(k, r.as_i32x4(), src.as_i32x4())) } @@ -26577,15 +24539,15 @@ pub unsafe fn _mm_mask_alignr_epi32( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_alignr_epi32&expand=241) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[cfg_attr(test, assert_instr(valignd, imm8 = 1))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm_maskz_alignr_epi32(k: __mmask8, a: __m128i, b: __m128i, imm8: i32) -> __m128i { - macro_rules! call { - ($imm8:expr) => { - _mm_alignr_epi32(a, b, $imm8) - }; - } - let r = constify_imm8_sae!(imm8, call); +#[cfg_attr(test, assert_instr(valignd, IMM8 = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm_maskz_alignr_epi32( + k: __mmask8, + a: __m128i, + b: __m128i, +) -> __m128i { + static_assert_imm8!(IMM8); + let r = _mm_alignr_epi32::(a, b); let zero = _mm_setzero_si128().as_i32x4(); transmute(simd_select_bitmask(k, r.as_i32x4(), zero)) } @@ -26595,20 +24557,20 @@ pub unsafe fn _mm_maskz_alignr_epi32(k: __mmask8, a: __m128i, b: __m128i, imm8: /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=512_alignr_epi64&expand=254) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(valignq, imm8 = 1))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm512_alignr_epi64(a: __m512i, b: __m512i, imm8: i32) -> __m512i { - assert!(imm8 >= 0 && imm8 <= 255); - let imm8: i32 = imm8 % 8; +#[cfg_attr(test, assert_instr(valignq, IMM8 = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_alignr_epi64(a: __m512i, b: __m512i) -> __m512i { + static_assert_imm8!(IMM8); + let imm8: i32 = IMM8 % 8; let r: i64x8 = match imm8 { - 0 => simd_shuffle8(a, b, [8, 9, 10, 11, 12, 13, 14, 15]), - 1 => simd_shuffle8(a, b, [9, 10, 11, 12, 13, 14, 15, 0]), - 2 => simd_shuffle8(a, b, [10, 11, 12, 13, 14, 15, 0, 1]), - 3 => simd_shuffle8(a, b, [11, 12, 13, 14, 15, 0, 1, 2]), - 4 => simd_shuffle8(a, b, [12, 13, 14, 15, 0, 1, 2, 3]), - 5 => simd_shuffle8(a, b, [13, 14, 15, 0, 1, 2, 3, 4]), - 6 => simd_shuffle8(a, b, [14, 15, 0, 1, 2, 3, 4, 5]), - _ => simd_shuffle8(a, b, [15, 0, 1, 2, 3, 4, 5, 6]), + 0 => simd_shuffle8!(a, b, [8, 9, 10, 11, 12, 13, 14, 15]), + 1 => simd_shuffle8!(a, b, [9, 10, 11, 12, 13, 14, 15, 0]), + 2 => simd_shuffle8!(a, b, [10, 11, 12, 13, 14, 15, 0, 1]), + 3 => simd_shuffle8!(a, b, [11, 12, 13, 14, 15, 0, 1, 2]), + 4 => simd_shuffle8!(a, b, [12, 13, 14, 15, 0, 1, 2, 3]), + 5 => simd_shuffle8!(a, b, [13, 14, 15, 0, 1, 2, 3, 4]), + 6 => simd_shuffle8!(a, b, [14, 15, 0, 1, 2, 3, 4, 5]), + _ => simd_shuffle8!(a, b, [15, 0, 1, 2, 3, 4, 5, 6]), }; transmute(r) } @@ -26618,21 +24580,16 @@ pub unsafe fn _mm512_alignr_epi64(a: __m512i, b: __m512i, imm8: i32) -> __m512i /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=512_mask_alignr_epi64&expand=255) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(valignq, imm8 = 1))] -#[rustc_args_required_const(4)] -pub unsafe fn _mm512_mask_alignr_epi64( +#[cfg_attr(test, assert_instr(valignq, IMM8 = 1))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_mask_alignr_epi64( src: __m512i, k: __mmask8, a: __m512i, b: __m512i, - imm8: i32, ) -> __m512i { - macro_rules! call { - ($imm8:expr) => { - _mm512_alignr_epi64(a, b, $imm8) - }; - } - let r = constify_imm8_sae!(imm8, call); + static_assert_imm8!(IMM8); + let r = _mm512_alignr_epi64::(a, b); transmute(simd_select_bitmask(k, r.as_i64x8(), src.as_i64x8())) } @@ -26641,15 +24598,15 @@ pub unsafe fn _mm512_mask_alignr_epi64( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=512_maskz_alignr_epi64&expand=256) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(valignq, imm8 = 1))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm512_maskz_alignr_epi64(k: __mmask8, a: __m512i, b: __m512i, imm8: i32) -> __m512i { - macro_rules! call { - ($imm8:expr) => { - _mm512_alignr_epi64(a, b, $imm8) - }; - } - let r = constify_imm8_sae!(imm8, call); +#[cfg_attr(test, assert_instr(valignq, IMM8 = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_maskz_alignr_epi64( + k: __mmask8, + a: __m512i, + b: __m512i, +) -> __m512i { + static_assert_imm8!(IMM8); + let r = _mm512_alignr_epi64::(a, b); let zero = _mm512_setzero_si512().as_i64x8(); transmute(simd_select_bitmask(k, r.as_i64x8(), zero)) } @@ -26659,20 +24616,20 @@ pub unsafe fn _mm512_maskz_alignr_epi64(k: __mmask8, a: __m512i, b: __m512i, imm /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_alignr_epi64&expand=251) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[cfg_attr(test, assert_instr(valignq, imm8 = 1))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm256_alignr_epi64(a: __m256i, b: __m256i, imm8: i32) -> __m256i { - assert!(imm8 >= 0 && imm8 <= 255); - let imm8: i32 = imm8 % 8; +#[cfg_attr(test, assert_instr(valignq, IMM8 = 1))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm256_alignr_epi64(a: __m256i, b: __m256i) -> __m256i { + static_assert_imm8!(IMM8); + let imm8: i32 = IMM8 % 8; let r: i64x4 = match imm8 { - 0 => simd_shuffle4(a, b, [4, 5, 6, 7]), - 1 => simd_shuffle4(a, b, [5, 6, 7, 0]), - 2 => simd_shuffle4(a, b, [6, 7, 0, 1]), - 3 => simd_shuffle4(a, b, [7, 0, 1, 2]), - 4 => simd_shuffle4(a, b, [0, 1, 2, 3]), - 5 => simd_shuffle4(a, b, [1, 2, 3, 4]), - 6 => simd_shuffle4(a, b, [2, 3, 4, 5]), - _ => simd_shuffle4(a, b, [3, 4, 5, 6]), + 0 => simd_shuffle4!(a, b, [4, 5, 6, 7]), + 1 => simd_shuffle4!(a, b, [5, 6, 7, 0]), + 2 => simd_shuffle4!(a, b, [6, 7, 0, 1]), + 3 => simd_shuffle4!(a, b, [7, 0, 1, 2]), + 4 => simd_shuffle4!(a, b, [0, 1, 2, 3]), + 5 => simd_shuffle4!(a, b, [1, 2, 3, 4]), + 6 => simd_shuffle4!(a, b, [2, 3, 4, 5]), + _ => simd_shuffle4!(a, b, [3, 4, 5, 6]), }; transmute(r) } @@ -26682,21 +24639,16 @@ pub unsafe fn _mm256_alignr_epi64(a: __m256i, b: __m256i, imm8: i32) -> __m256i /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_alignr_epi64&expand=252) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[cfg_attr(test, assert_instr(valignq, imm8 = 1))] -#[rustc_args_required_const(4)] -pub unsafe fn _mm256_mask_alignr_epi64( +#[cfg_attr(test, assert_instr(valignq, IMM8 = 1))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm256_mask_alignr_epi64( src: __m256i, k: __mmask8, a: __m256i, b: __m256i, - imm8: i32, ) -> __m256i { - macro_rules! call { - ($imm8:expr) => { - _mm256_alignr_epi64(a, b, $imm8) - }; - } - let r = constify_imm8_sae!(imm8, call); + static_assert_imm8!(IMM8); + let r = _mm256_alignr_epi64::(a, b); transmute(simd_select_bitmask(k, r.as_i64x4(), src.as_i64x4())) } @@ -26705,15 +24657,15 @@ pub unsafe fn _mm256_mask_alignr_epi64( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_alignr_epi64&expand=253) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[cfg_attr(test, assert_instr(valignq, imm8 = 1))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm256_maskz_alignr_epi64(k: __mmask8, a: __m256i, b: __m256i, imm8: i32) -> __m256i { - macro_rules! call { - ($imm8:expr) => { - _mm256_alignr_epi64(a, b, $imm8) - }; - } - let r = constify_imm8_sae!(imm8, call); +#[cfg_attr(test, assert_instr(valignq, IMM8 = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm256_maskz_alignr_epi64( + k: __mmask8, + a: __m256i, + b: __m256i, +) -> __m256i { + static_assert_imm8!(IMM8); + let r = _mm256_alignr_epi64::(a, b); let zero = _mm256_setzero_si256().as_i64x4(); transmute(simd_select_bitmask(k, r.as_i64x4(), zero)) } @@ -26723,16 +24675,16 @@ pub unsafe fn _mm256_maskz_alignr_epi64(k: __mmask8, a: __m256i, b: __m256i, imm /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_alignr_epi64&expand=248) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[cfg_attr(test, assert_instr(vpalignr, imm8 = 1))] //should be valignq -#[rustc_args_required_const(2)] -pub unsafe fn _mm_alignr_epi64(a: __m128i, b: __m128i, imm8: i32) -> __m128i { - assert!(imm8 >= 0 && imm8 <= 255); - let imm8: i32 = imm8 % 4; +#[cfg_attr(test, assert_instr(vpalignr, IMM8 = 1))] //should be valignq +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm_alignr_epi64(a: __m128i, b: __m128i) -> __m128i { + static_assert_imm8!(IMM8); + let imm8: i32 = IMM8 % 4; let r: i64x2 = match imm8 { - 0 => simd_shuffle2(a, b, [2, 3]), - 1 => simd_shuffle2(a, b, [3, 0]), - 2 => simd_shuffle2(a, b, [0, 1]), - _ => simd_shuffle2(a, b, [1, 2]), + 0 => simd_shuffle2!(a, b, [2, 3]), + 1 => simd_shuffle2!(a, b, [3, 0]), + 2 => simd_shuffle2!(a, b, [0, 1]), + _ => simd_shuffle2!(a, b, [1, 2]), }; transmute(r) } @@ -26742,21 +24694,16 @@ pub unsafe fn _mm_alignr_epi64(a: __m128i, b: __m128i, imm8: i32) -> __m128i { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_alignr_epi64&expand=249) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[cfg_attr(test, assert_instr(valignq, imm8 = 1))] -#[rustc_args_required_const(4)] -pub unsafe fn _mm_mask_alignr_epi64( +#[cfg_attr(test, assert_instr(valignq, IMM8 = 1))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm_mask_alignr_epi64( src: __m128i, k: __mmask8, a: __m128i, b: __m128i, - imm8: i32, ) -> __m128i { - macro_rules! call { - ($imm8:expr) => { - _mm_alignr_epi64(a, b, $imm8) - }; - } - let r = constify_imm8_sae!(imm8, call); + static_assert_imm8!(IMM8); + let r = _mm_alignr_epi64::(a, b); transmute(simd_select_bitmask(k, r.as_i64x2(), src.as_i64x2())) } @@ -26765,15 +24712,15 @@ pub unsafe fn _mm_mask_alignr_epi64( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_alignr_epi64&expand=250) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[cfg_attr(test, assert_instr(valignq, imm8 = 1))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm_maskz_alignr_epi64(k: __mmask8, a: __m128i, b: __m128i, imm8: i32) -> __m128i { - macro_rules! call { - ($imm8:expr) => { - _mm_alignr_epi64(a, b, $imm8) - }; - } - let r = constify_imm8_sae!(imm8, call); +#[cfg_attr(test, assert_instr(valignq, IMM8 = 1))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm_maskz_alignr_epi64( + k: __mmask8, + a: __m128i, + b: __m128i, +) -> __m128i { + static_assert_imm8!(IMM8); + let r = _mm_alignr_epi64::(a, b); let zero = _mm_setzero_si128().as_i64x2(); transmute(simd_select_bitmask(k, r.as_i64x2(), zero)) } @@ -28359,7 +26306,7 @@ pub unsafe fn _mm512_setr4_epi64(d: i64, c: i64, b: i64, a: i64) -> __m512i { #[target_feature(enable = "avx512f")] #[cfg_attr(test, assert_instr(vcmp))] //should be vcmpps pub unsafe fn _mm512_cmplt_ps_mask(a: __m512, b: __m512) -> __mmask16 { - _mm512_cmp_ps_mask(a, b, _CMP_LT_OS) + _mm512_cmp_ps_mask::<_CMP_LT_OS>(a, b) } /// Compare packed single-precision (32-bit) floating-point elements in a and b for less-than, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). @@ -28369,7 +26316,7 @@ pub unsafe fn _mm512_cmplt_ps_mask(a: __m512, b: __m512) -> __mmask16 { #[target_feature(enable = "avx512f")] #[cfg_attr(test, assert_instr(vcmp))] //should be vcmpps pub unsafe fn _mm512_mask_cmplt_ps_mask(k1: __mmask16, a: __m512, b: __m512) -> __mmask16 { - _mm512_mask_cmp_ps_mask(k1, a, b, _CMP_LT_OS) + _mm512_mask_cmp_ps_mask::<_CMP_LT_OS>(k1, a, b) } /// Compare packed single-precision (32-bit) floating-point elements in a and b for not-less-than, and store the results in mask vector k. @@ -28379,7 +26326,7 @@ pub unsafe fn _mm512_mask_cmplt_ps_mask(k1: __mmask16, a: __m512, b: __m512) -> #[target_feature(enable = "avx512f")] #[cfg_attr(test, assert_instr(vcmp))] //should be vcmpps pub unsafe fn _mm512_cmpnlt_ps_mask(a: __m512, b: __m512) -> __mmask16 { - _mm512_cmp_ps_mask(a, b, _CMP_NLT_US) + _mm512_cmp_ps_mask::<_CMP_NLT_US>(a, b) } /// Compare packed single-precision (32-bit) floating-point elements in a and b for not-less-than, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). @@ -28389,7 +26336,7 @@ pub unsafe fn _mm512_cmpnlt_ps_mask(a: __m512, b: __m512) -> __mmask16 { #[target_feature(enable = "avx512f")] #[cfg_attr(test, assert_instr(vcmp))] //should be vcmpps pub unsafe fn _mm512_mask_cmpnlt_ps_mask(k1: __mmask16, a: __m512, b: __m512) -> __mmask16 { - _mm512_mask_cmp_ps_mask(k1, a, b, _CMP_NLT_US) + _mm512_mask_cmp_ps_mask::<_CMP_NLT_US>(k1, a, b) } /// Compare packed single-precision (32-bit) floating-point elements in a and b for less-than-or-equal, and store the results in mask vector k. @@ -28399,7 +26346,7 @@ pub unsafe fn _mm512_mask_cmpnlt_ps_mask(k1: __mmask16, a: __m512, b: __m512) -> #[target_feature(enable = "avx512f")] #[cfg_attr(test, assert_instr(vcmp))] //should be vcmpps pub unsafe fn _mm512_cmple_ps_mask(a: __m512, b: __m512) -> __mmask16 { - _mm512_cmp_ps_mask(a, b, _CMP_LE_OS) + _mm512_cmp_ps_mask::<_CMP_LE_OS>(a, b) } /// Compare packed single-precision (32-bit) floating-point elements in a and b for less-than-or-equal, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). @@ -28409,7 +26356,7 @@ pub unsafe fn _mm512_cmple_ps_mask(a: __m512, b: __m512) -> __mmask16 { #[target_feature(enable = "avx512f")] #[cfg_attr(test, assert_instr(vcmp))] //should be vcmpps pub unsafe fn _mm512_mask_cmple_ps_mask(k1: __mmask16, a: __m512, b: __m512) -> __mmask16 { - _mm512_mask_cmp_ps_mask(k1, a, b, _CMP_LE_OS) + _mm512_mask_cmp_ps_mask::<_CMP_LE_OS>(k1, a, b) } /// Compare packed single-precision (32-bit) floating-point elements in a and b for not-less-than-or-equal, and store the results in mask vector k. @@ -28419,7 +26366,7 @@ pub unsafe fn _mm512_mask_cmple_ps_mask(k1: __mmask16, a: __m512, b: __m512) -> #[target_feature(enable = "avx512f")] #[cfg_attr(test, assert_instr(vcmp))] //should be vcmpps pub unsafe fn _mm512_cmpnle_ps_mask(a: __m512, b: __m512) -> __mmask16 { - _mm512_cmp_ps_mask(a, b, _CMP_NLE_US) + _mm512_cmp_ps_mask::<_CMP_NLE_US>(a, b) } /// Compare packed single-precision (32-bit) floating-point elements in a and b for not-less-than-or-equal, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). @@ -28429,7 +26376,7 @@ pub unsafe fn _mm512_cmpnle_ps_mask(a: __m512, b: __m512) -> __mmask16 { #[target_feature(enable = "avx512f")] #[cfg_attr(test, assert_instr(vcmp))] //should be vcmpps pub unsafe fn _mm512_mask_cmpnle_ps_mask(k1: __mmask16, a: __m512, b: __m512) -> __mmask16 { - _mm512_mask_cmp_ps_mask(k1, a, b, _CMP_NLE_US) + _mm512_mask_cmp_ps_mask::<_CMP_NLE_US>(k1, a, b) } /// Compare packed single-precision (32-bit) floating-point elements in a and b for equality, and store the results in mask vector k. @@ -28439,7 +26386,7 @@ pub unsafe fn _mm512_mask_cmpnle_ps_mask(k1: __mmask16, a: __m512, b: __m512) -> #[target_feature(enable = "avx512f")] #[cfg_attr(test, assert_instr(vcmp))] //should be vcmpps pub unsafe fn _mm512_cmpeq_ps_mask(a: __m512, b: __m512) -> __mmask16 { - _mm512_cmp_ps_mask(a, b, _CMP_EQ_OQ) + _mm512_cmp_ps_mask::<_CMP_EQ_OQ>(a, b) } /// Compare packed single-precision (32-bit) floating-point elements in a and b for equality, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). @@ -28449,7 +26396,7 @@ pub unsafe fn _mm512_cmpeq_ps_mask(a: __m512, b: __m512) -> __mmask16 { #[target_feature(enable = "avx512f")] #[cfg_attr(test, assert_instr(vcmp))] //should be vcmpps pub unsafe fn _mm512_mask_cmpeq_ps_mask(k1: __mmask16, a: __m512, b: __m512) -> __mmask16 { - _mm512_mask_cmp_ps_mask(k1, a, b, _CMP_EQ_OQ) + _mm512_mask_cmp_ps_mask::<_CMP_EQ_OQ>(k1, a, b) } /// Compare packed single-precision (32-bit) floating-point elements in a and b for not-equal, and store the results in mask vector k. @@ -28459,7 +26406,7 @@ pub unsafe fn _mm512_mask_cmpeq_ps_mask(k1: __mmask16, a: __m512, b: __m512) -> #[target_feature(enable = "avx512f")] #[cfg_attr(test, assert_instr(vcmp))] //should be vcmpps pub unsafe fn _mm512_cmpneq_ps_mask(a: __m512, b: __m512) -> __mmask16 { - _mm512_cmp_ps_mask(a, b, _CMP_NEQ_UQ) + _mm512_cmp_ps_mask::<_CMP_NEQ_UQ>(a, b) } /// Compare packed single-precision (32-bit) floating-point elements in a and b for not-equal, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). @@ -28469,7 +26416,7 @@ pub unsafe fn _mm512_cmpneq_ps_mask(a: __m512, b: __m512) -> __mmask16 { #[target_feature(enable = "avx512f")] #[cfg_attr(test, assert_instr(vcmp))] //should be vcmpps pub unsafe fn _mm512_mask_cmpneq_ps_mask(k1: __mmask16, a: __m512, b: __m512) -> __mmask16 { - _mm512_mask_cmp_ps_mask(k1, a, b, _CMP_NEQ_UQ) + _mm512_mask_cmp_ps_mask::<_CMP_NEQ_UQ>(k1, a, b) } /// Compare packed single-precision (32-bit) floating-point elements in a and b based on the comparison operand specified by imm8, and store the results in mask vector k. @@ -28477,18 +26424,14 @@ pub unsafe fn _mm512_mask_cmpneq_ps_mask(k1: __mmask16, a: __m512, b: __m512) -> /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cmp_ps_mask&expand=749) #[inline] #[target_feature(enable = "avx512f")] -#[rustc_args_required_const(2)] -#[cfg_attr(test, assert_instr(vcmp, imm8 = 0))] -pub unsafe fn _mm512_cmp_ps_mask(a: __m512, b: __m512, imm8: i32) -> __mmask16 { +#[rustc_legacy_const_generics(2)] +#[cfg_attr(test, assert_instr(vcmp, IMM8 = 0))] +pub unsafe fn _mm512_cmp_ps_mask(a: __m512, b: __m512) -> __mmask16 { + static_assert_imm5!(IMM8); let neg_one = -1; let a = a.as_f32x16(); let b = b.as_f32x16(); - macro_rules! call { - ($imm5:expr) => { - vcmpps(a, b, $imm5, neg_one, _MM_FROUND_CUR_DIRECTION) - }; - } - let r = constify_imm5!(imm8, call); + let r = vcmpps(a, b, IMM8, neg_one, _MM_FROUND_CUR_DIRECTION); transmute(r) } @@ -28497,17 +26440,17 @@ pub unsafe fn _mm512_cmp_ps_mask(a: __m512, b: __m512, imm8: i32) -> __mmask16 { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cmp_ps_mask&expand=750) #[inline] #[target_feature(enable = "avx512f")] -#[rustc_args_required_const(3)] -#[cfg_attr(test, assert_instr(vcmp, imm8 = 0))] -pub unsafe fn _mm512_mask_cmp_ps_mask(k1: __mmask16, a: __m512, b: __m512, imm8: i32) -> __mmask16 { +#[rustc_legacy_const_generics(3)] +#[cfg_attr(test, assert_instr(vcmp, IMM8 = 0))] +pub unsafe fn _mm512_mask_cmp_ps_mask( + k1: __mmask16, + a: __m512, + b: __m512, +) -> __mmask16 { + static_assert_imm5!(IMM8); let a = a.as_f32x16(); let b = b.as_f32x16(); - macro_rules! call { - ($imm5:expr) => { - vcmpps(a, b, $imm5, k1 as i16, _MM_FROUND_CUR_DIRECTION) - }; - } - let r = constify_imm5!(imm8, call); + let r = vcmpps(a, b, IMM8, k1 as i16, _MM_FROUND_CUR_DIRECTION); transmute(r) } @@ -28516,18 +26459,14 @@ pub unsafe fn _mm512_mask_cmp_ps_mask(k1: __mmask16, a: __m512, b: __m512, imm8: /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_cmp_ps_mask&expand=747) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[rustc_args_required_const(2)] -#[cfg_attr(test, assert_instr(vcmp, imm8 = 0))] -pub unsafe fn _mm256_cmp_ps_mask(a: __m256, b: __m256, imm8: i32) -> __mmask8 { +#[rustc_legacy_const_generics(2)] +#[cfg_attr(test, assert_instr(vcmp, IMM8 = 0))] +pub unsafe fn _mm256_cmp_ps_mask(a: __m256, b: __m256) -> __mmask8 { + static_assert_imm5!(IMM8); let neg_one = -1; let a = a.as_f32x8(); let b = b.as_f32x8(); - macro_rules! call { - ($imm5:expr) => { - vcmpps256(a, b, $imm5, neg_one) - }; - } - let r = constify_imm5!(imm8, call); + let r = vcmpps256(a, b, IMM8, neg_one); transmute(r) } @@ -28536,17 +26475,17 @@ pub unsafe fn _mm256_cmp_ps_mask(a: __m256, b: __m256, imm8: i32) -> __mmask8 { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_cmp_ps_mask&expand=748) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[rustc_args_required_const(3)] -#[cfg_attr(test, assert_instr(vcmp, imm8 = 0))] -pub unsafe fn _mm256_mask_cmp_ps_mask(k1: __mmask8, a: __m256, b: __m256, imm8: i32) -> __mmask8 { +#[rustc_legacy_const_generics(3)] +#[cfg_attr(test, assert_instr(vcmp, IMM8 = 0))] +pub unsafe fn _mm256_mask_cmp_ps_mask( + k1: __mmask8, + a: __m256, + b: __m256, +) -> __mmask8 { + static_assert_imm5!(IMM8); let a = a.as_f32x8(); let b = b.as_f32x8(); - macro_rules! call { - ($imm5:expr) => { - vcmpps256(a, b, $imm5, k1 as i8) - }; - } - let r = constify_imm5!(imm8, call); + let r = vcmpps256(a, b, IMM8, k1 as i8); transmute(r) } @@ -28555,18 +26494,14 @@ pub unsafe fn _mm256_mask_cmp_ps_mask(k1: __mmask8, a: __m256, b: __m256, imm8: /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cmp_ps_mask&expand=745) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[rustc_args_required_const(2)] -#[cfg_attr(test, assert_instr(vcmp, imm8 = 0))] -pub unsafe fn _mm_cmp_ps_mask(a: __m128, b: __m128, imm8: i32) -> __mmask8 { +#[rustc_legacy_const_generics(2)] +#[cfg_attr(test, assert_instr(vcmp, IMM8 = 0))] +pub unsafe fn _mm_cmp_ps_mask(a: __m128, b: __m128) -> __mmask8 { + static_assert_imm5!(IMM8); let neg_one = -1; let a = a.as_f32x4(); let b = b.as_f32x4(); - macro_rules! call { - ($imm5:expr) => { - vcmpps128(a, b, $imm5, neg_one) - }; - } - let r = constify_imm5!(imm8, call); + let r = vcmpps128(a, b, IMM8, neg_one); transmute(r) } @@ -28575,17 +26510,17 @@ pub unsafe fn _mm_cmp_ps_mask(a: __m128, b: __m128, imm8: i32) -> __mmask8 { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_cmp_ps_mask&expand=746) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[rustc_args_required_const(3)] -#[cfg_attr(test, assert_instr(vcmp, imm8 = 0))] -pub unsafe fn _mm_mask_cmp_ps_mask(k1: __mmask8, a: __m128, b: __m128, imm8: i32) -> __mmask8 { +#[rustc_legacy_const_generics(3)] +#[cfg_attr(test, assert_instr(vcmp, IMM8 = 0))] +pub unsafe fn _mm_mask_cmp_ps_mask( + k1: __mmask8, + a: __m128, + b: __m128, +) -> __mmask8 { + static_assert_imm5!(IMM8); let a = a.as_f32x4(); let b = b.as_f32x4(); - macro_rules! call { - ($imm5:expr) => { - vcmpps128(a, b, $imm5, k1 as i8) - }; - } - let r = constify_imm5!(imm8, call); + let r = vcmpps128(a, b, IMM8, k1 as i8); transmute(r) } @@ -28595,18 +26530,18 @@ pub unsafe fn _mm_mask_cmp_ps_mask(k1: __mmask8, a: __m128, b: __m128, imm8: i32 /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cmp_round_ps_mask&expand=753) #[inline] #[target_feature(enable = "avx512f")] -#[rustc_args_required_const(2, 3)] -#[cfg_attr(test, assert_instr(vcmp, imm8 = 0, sae = 4))] -pub unsafe fn _mm512_cmp_round_ps_mask(a: __m512, b: __m512, imm8: i32, sae: i32) -> __mmask16 { +#[cfg_attr(test, assert_instr(vcmp, IMM5 = 0, SAE = 4))] +#[rustc_legacy_const_generics(2, 3)] +pub unsafe fn _mm512_cmp_round_ps_mask( + a: __m512, + b: __m512, +) -> __mmask16 { + static_assert_imm5!(IMM5); + static_assert_mantissas_sae!(SAE); let neg_one = -1; let a = a.as_f32x16(); let b = b.as_f32x16(); - macro_rules! call { - ($imm5:expr, $imm4:expr) => { - vcmpps(a, b, $imm5, neg_one, $imm4) - }; - } - let r = constify_imm5_sae!(imm8, sae, call); + let r = vcmpps(a, b, IMM5, neg_one, SAE); transmute(r) } @@ -28616,23 +26551,18 @@ pub unsafe fn _mm512_cmp_round_ps_mask(a: __m512, b: __m512, imm8: i32, sae: i32 /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cmp_round_ps_mask&expand=754) #[inline] #[target_feature(enable = "avx512f")] -#[rustc_args_required_const(3, 4)] -#[cfg_attr(test, assert_instr(vcmp, imm8 = 0, sae = 4))] -pub unsafe fn _mm512_mask_cmp_round_ps_mask( +#[cfg_attr(test, assert_instr(vcmp, IMM5 = 0, SAE = 4))] +#[rustc_legacy_const_generics(3, 4)] +pub unsafe fn _mm512_mask_cmp_round_ps_mask( m: __mmask16, a: __m512, b: __m512, - imm8: i32, - sae: i32, ) -> __mmask16 { + static_assert_imm5!(IMM5); + static_assert_mantissas_sae!(SAE); let a = a.as_f32x16(); let b = b.as_f32x16(); - macro_rules! call { - ($imm5:expr, $imm4:expr) => { - vcmpps(a, b, $imm5, m as i16, $imm4) - }; - } - let r = constify_imm5_sae!(imm8, sae, call); + let r = vcmpps(a, b, IMM5, m as i16, SAE); transmute(r) } @@ -28643,7 +26573,7 @@ pub unsafe fn _mm512_mask_cmp_round_ps_mask( #[target_feature(enable = "avx512f")] #[cfg_attr(test, assert_instr(vcmp))] //should be vcmps pub unsafe fn _mm512_cmpord_ps_mask(a: __m512, b: __m512) -> __mmask16 { - _mm512_cmp_ps_mask(a, b, _CMP_ORD_Q) + _mm512_cmp_ps_mask::<_CMP_ORD_Q>(a, b) } /// Compare packed single-precision (32-bit) floating-point elements in a and b to see if neither is NaN, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). @@ -28653,7 +26583,7 @@ pub unsafe fn _mm512_cmpord_ps_mask(a: __m512, b: __m512) -> __mmask16 { #[target_feature(enable = "avx512f")] #[cfg_attr(test, assert_instr(vcmp))] //should be vcmpps pub unsafe fn _mm512_mask_cmpord_ps_mask(k1: __mmask16, a: __m512, b: __m512) -> __mmask16 { - _mm512_mask_cmp_ps_mask(k1, a, b, _CMP_ORD_Q) + _mm512_mask_cmp_ps_mask::<_CMP_ORD_Q>(k1, a, b) } /// Compare packed single-precision (32-bit) floating-point elements in a and b to see if either is NaN, and store the results in mask vector k. @@ -28663,7 +26593,7 @@ pub unsafe fn _mm512_mask_cmpord_ps_mask(k1: __mmask16, a: __m512, b: __m512) -> #[target_feature(enable = "avx512f")] #[cfg_attr(test, assert_instr(vcmp))] //should be vcmpps pub unsafe fn _mm512_cmpunord_ps_mask(a: __m512, b: __m512) -> __mmask16 { - _mm512_cmp_ps_mask(a, b, _CMP_UNORD_Q) + _mm512_cmp_ps_mask::<_CMP_UNORD_Q>(a, b) } /// Compare packed single-precision (32-bit) floating-point elements in a and b to see if either is NaN, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). @@ -28673,7 +26603,7 @@ pub unsafe fn _mm512_cmpunord_ps_mask(a: __m512, b: __m512) -> __mmask16 { #[target_feature(enable = "avx512f")] #[cfg_attr(test, assert_instr(vcmp))] //should be vcmpps pub unsafe fn _mm512_mask_cmpunord_ps_mask(k1: __mmask16, a: __m512, b: __m512) -> __mmask16 { - _mm512_mask_cmp_ps_mask(k1, a, b, _CMP_UNORD_Q) + _mm512_mask_cmp_ps_mask::<_CMP_UNORD_Q>(k1, a, b) } /// Compare packed double-precision (64-bit) floating-point elements in a and b for less-than, and store the results in mask vector k. @@ -28683,7 +26613,7 @@ pub unsafe fn _mm512_mask_cmpunord_ps_mask(k1: __mmask16, a: __m512, b: __m512) #[target_feature(enable = "avx512f")] #[cfg_attr(test, assert_instr(vcmp))] //should be vcmppd pub unsafe fn _mm512_cmplt_pd_mask(a: __m512d, b: __m512d) -> __mmask8 { - _mm512_cmp_pd_mask(a, b, _CMP_LT_OS) + _mm512_cmp_pd_mask::<_CMP_LT_OS>(a, b) } /// Compare packed double-precision (64-bit) floating-point elements in a and b for less-than, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). @@ -28693,7 +26623,7 @@ pub unsafe fn _mm512_cmplt_pd_mask(a: __m512d, b: __m512d) -> __mmask8 { #[target_feature(enable = "avx512f")] #[cfg_attr(test, assert_instr(vcmp))] //should be vcmppd pub unsafe fn _mm512_mask_cmplt_pd_mask(k1: __mmask8, a: __m512d, b: __m512d) -> __mmask8 { - _mm512_mask_cmp_pd_mask(k1, a, b, _CMP_LT_OS) + _mm512_mask_cmp_pd_mask::<_CMP_LT_OS>(k1, a, b) } /// Compare packed double-precision (64-bit) floating-point elements in a and b for not-less-than, and store the results in mask vector k. @@ -28703,7 +26633,7 @@ pub unsafe fn _mm512_mask_cmplt_pd_mask(k1: __mmask8, a: __m512d, b: __m512d) -> #[target_feature(enable = "avx512f")] #[cfg_attr(test, assert_instr(vcmp))] //should be vcmppd pub unsafe fn _mm512_cmpnlt_pd_mask(a: __m512d, b: __m512d) -> __mmask8 { - _mm512_cmp_pd_mask(a, b, _CMP_NLT_US) + _mm512_cmp_pd_mask::<_CMP_NLT_US>(a, b) } /// Compare packed double-precision (64-bit) floating-point elements in a and b for not-less-than, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). @@ -28713,7 +26643,7 @@ pub unsafe fn _mm512_cmpnlt_pd_mask(a: __m512d, b: __m512d) -> __mmask8 { #[target_feature(enable = "avx512f")] #[cfg_attr(test, assert_instr(vcmp))] //should be vcmppd pub unsafe fn _mm512_mask_cmpnlt_pd_mask(m: __mmask8, a: __m512d, b: __m512d) -> __mmask8 { - _mm512_mask_cmp_pd_mask(m, a, b, _CMP_NLT_US) + _mm512_mask_cmp_pd_mask::<_CMP_NLT_US>(m, a, b) } /// Compare packed double-precision (64-bit) floating-point elements in a and b for less-than-or-equal, and store the results in mask vector k. @@ -28723,7 +26653,7 @@ pub unsafe fn _mm512_mask_cmpnlt_pd_mask(m: __mmask8, a: __m512d, b: __m512d) -> #[target_feature(enable = "avx512f")] #[cfg_attr(test, assert_instr(vcmp))] //should be vcmppd pub unsafe fn _mm512_cmple_pd_mask(a: __m512d, b: __m512d) -> __mmask8 { - _mm512_cmp_pd_mask(a, b, _CMP_LE_OS) + _mm512_cmp_pd_mask::<_CMP_LE_OS>(a, b) } /// Compare packed double-precision (64-bit) floating-point elements in a and b for less-than-or-equal, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). @@ -28733,7 +26663,7 @@ pub unsafe fn _mm512_cmple_pd_mask(a: __m512d, b: __m512d) -> __mmask8 { #[target_feature(enable = "avx512f")] #[cfg_attr(test, assert_instr(vcmp))] //should be vcmppd pub unsafe fn _mm512_mask_cmple_pd_mask(k1: __mmask8, a: __m512d, b: __m512d) -> __mmask8 { - _mm512_mask_cmp_pd_mask(k1, a, b, _CMP_LE_OS) + _mm512_mask_cmp_pd_mask::<_CMP_LE_OS>(k1, a, b) } /// Compare packed double-precision (64-bit) floating-point elements in a and b for not-less-than-or-equal, and store the results in mask vector k. @@ -28743,7 +26673,7 @@ pub unsafe fn _mm512_mask_cmple_pd_mask(k1: __mmask8, a: __m512d, b: __m512d) -> #[target_feature(enable = "avx512f")] #[cfg_attr(test, assert_instr(vcmp))] //should be vcmppd pub unsafe fn _mm512_cmpnle_pd_mask(a: __m512d, b: __m512d) -> __mmask8 { - _mm512_cmp_pd_mask(a, b, _CMP_NLE_US) + _mm512_cmp_pd_mask::<_CMP_NLE_US>(a, b) } /// Compare packed double-precision (64-bit) floating-point elements in a and b for not-less-than-or-equal, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). @@ -28753,7 +26683,7 @@ pub unsafe fn _mm512_cmpnle_pd_mask(a: __m512d, b: __m512d) -> __mmask8 { #[target_feature(enable = "avx512f")] #[cfg_attr(test, assert_instr(vcmp))] //should be vcmppd pub unsafe fn _mm512_mask_cmpnle_pd_mask(k1: __mmask8, a: __m512d, b: __m512d) -> __mmask8 { - _mm512_mask_cmp_pd_mask(k1, a, b, _CMP_NLE_US) + _mm512_mask_cmp_pd_mask::<_CMP_NLE_US>(k1, a, b) } /// Compare packed double-precision (64-bit) floating-point elements in a and b for equality, and store the results in mask vector k. @@ -28763,7 +26693,7 @@ pub unsafe fn _mm512_mask_cmpnle_pd_mask(k1: __mmask8, a: __m512d, b: __m512d) - #[target_feature(enable = "avx512f")] #[cfg_attr(test, assert_instr(vcmp))] //should be vcmppd pub unsafe fn _mm512_cmpeq_pd_mask(a: __m512d, b: __m512d) -> __mmask8 { - _mm512_cmp_pd_mask(a, b, _CMP_EQ_OQ) + _mm512_cmp_pd_mask::<_CMP_EQ_OQ>(a, b) } /// Compare packed double-precision (64-bit) floating-point elements in a and b for equality, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). @@ -28773,7 +26703,7 @@ pub unsafe fn _mm512_cmpeq_pd_mask(a: __m512d, b: __m512d) -> __mmask8 { #[target_feature(enable = "avx512f")] #[cfg_attr(test, assert_instr(vcmp))] //should be vcmppd pub unsafe fn _mm512_mask_cmpeq_pd_mask(k1: __mmask8, a: __m512d, b: __m512d) -> __mmask8 { - _mm512_mask_cmp_pd_mask(k1, a, b, _CMP_EQ_OQ) + _mm512_mask_cmp_pd_mask::<_CMP_EQ_OQ>(k1, a, b) } /// Compare packed double-precision (64-bit) floating-point elements in a and b for not-equal, and store the results in mask vector k. @@ -28783,7 +26713,7 @@ pub unsafe fn _mm512_mask_cmpeq_pd_mask(k1: __mmask8, a: __m512d, b: __m512d) -> #[target_feature(enable = "avx512f")] #[cfg_attr(test, assert_instr(vcmp))] //should be vcmppd pub unsafe fn _mm512_cmpneq_pd_mask(a: __m512d, b: __m512d) -> __mmask8 { - _mm512_cmp_pd_mask(a, b, _CMP_NEQ_UQ) + _mm512_cmp_pd_mask::<_CMP_NEQ_UQ>(a, b) } /// Compare packed double-precision (64-bit) floating-point elements in a and b for not-equal, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). @@ -28793,7 +26723,7 @@ pub unsafe fn _mm512_cmpneq_pd_mask(a: __m512d, b: __m512d) -> __mmask8 { #[target_feature(enable = "avx512f")] #[cfg_attr(test, assert_instr(vcmp))] //should be vcmppd pub unsafe fn _mm512_mask_cmpneq_pd_mask(k1: __mmask8, a: __m512d, b: __m512d) -> __mmask8 { - _mm512_mask_cmp_pd_mask(k1, a, b, _CMP_NEQ_UQ) + _mm512_mask_cmp_pd_mask::<_CMP_NEQ_UQ>(k1, a, b) } /// Compare packed double-precision (64-bit) floating-point elements in a and b based on the comparison operand specified by imm8, and store the results in mask vector k. @@ -28801,18 +26731,14 @@ pub unsafe fn _mm512_mask_cmpneq_pd_mask(k1: __mmask8, a: __m512d, b: __m512d) - /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cmp_pd_mask&expand=741) #[inline] #[target_feature(enable = "avx512f")] -#[rustc_args_required_const(2)] -#[cfg_attr(test, assert_instr(vcmp, imm8 = 0))] -pub unsafe fn _mm512_cmp_pd_mask(a: __m512d, b: __m512d, imm8: i32) -> __mmask8 { +#[rustc_legacy_const_generics(2)] +#[cfg_attr(test, assert_instr(vcmp, IMM8 = 0))] +pub unsafe fn _mm512_cmp_pd_mask(a: __m512d, b: __m512d) -> __mmask8 { + static_assert_imm5!(IMM8); let neg_one = -1; let a = a.as_f64x8(); let b = b.as_f64x8(); - macro_rules! call { - ($imm5:expr) => { - vcmppd(a, b, $imm5, neg_one, _MM_FROUND_CUR_DIRECTION) - }; - } - let r = constify_imm5!(imm8, call); + let r = vcmppd(a, b, IMM8, neg_one, _MM_FROUND_CUR_DIRECTION); transmute(r) } @@ -28821,17 +26747,17 @@ pub unsafe fn _mm512_cmp_pd_mask(a: __m512d, b: __m512d, imm8: i32) -> __mmask8 /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cmp_pd_mask&expand=742) #[inline] #[target_feature(enable = "avx512f")] -#[rustc_args_required_const(3)] -#[cfg_attr(test, assert_instr(vcmp, imm8 = 0))] -pub unsafe fn _mm512_mask_cmp_pd_mask(k1: __mmask8, a: __m512d, b: __m512d, imm8: i32) -> __mmask8 { +#[rustc_legacy_const_generics(3)] +#[cfg_attr(test, assert_instr(vcmp, IMM8 = 0))] +pub unsafe fn _mm512_mask_cmp_pd_mask( + k1: __mmask8, + a: __m512d, + b: __m512d, +) -> __mmask8 { + static_assert_imm5!(IMM8); let a = a.as_f64x8(); let b = b.as_f64x8(); - macro_rules! call { - ($imm5:expr) => { - vcmppd(a, b, $imm5, k1 as i8, _MM_FROUND_CUR_DIRECTION) - }; - } - let r = constify_imm5!(imm8, call); + let r = vcmppd(a, b, IMM8, k1 as i8, _MM_FROUND_CUR_DIRECTION); transmute(r) } @@ -28840,18 +26766,14 @@ pub unsafe fn _mm512_mask_cmp_pd_mask(k1: __mmask8, a: __m512d, b: __m512d, imm8 /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_cmp_pd_mask&expand=739) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[rustc_args_required_const(2)] -#[cfg_attr(test, assert_instr(vcmp, imm8 = 0))] -pub unsafe fn _mm256_cmp_pd_mask(a: __m256d, b: __m256d, imm8: i32) -> __mmask8 { +#[rustc_legacy_const_generics(2)] +#[cfg_attr(test, assert_instr(vcmp, IMM8 = 0))] +pub unsafe fn _mm256_cmp_pd_mask(a: __m256d, b: __m256d) -> __mmask8 { + static_assert_imm5!(IMM8); let neg_one = -1; let a = a.as_f64x4(); let b = b.as_f64x4(); - macro_rules! call { - ($imm5:expr) => { - vcmppd256(a, b, $imm5, neg_one) - }; - } - let r = constify_imm5!(imm8, call); + let r = vcmppd256(a, b, IMM8, neg_one); transmute(r) } @@ -28860,17 +26782,17 @@ pub unsafe fn _mm256_cmp_pd_mask(a: __m256d, b: __m256d, imm8: i32) -> __mmask8 /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_cmp_pd_mask&expand=740) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[rustc_args_required_const(3)] -#[cfg_attr(test, assert_instr(vcmp, imm8 = 0))] -pub unsafe fn _mm256_mask_cmp_pd_mask(k1: __mmask8, a: __m256d, b: __m256d, imm8: i32) -> __mmask8 { +#[rustc_legacy_const_generics(3)] +#[cfg_attr(test, assert_instr(vcmp, IMM8 = 0))] +pub unsafe fn _mm256_mask_cmp_pd_mask( + k1: __mmask8, + a: __m256d, + b: __m256d, +) -> __mmask8 { + static_assert_imm5!(IMM8); let a = a.as_f64x4(); let b = b.as_f64x4(); - macro_rules! call { - ($imm5:expr) => { - vcmppd256(a, b, $imm5, k1 as i8) - }; - } - let r = constify_imm5!(imm8, call); + let r = vcmppd256(a, b, IMM8, k1 as i8); transmute(r) } @@ -28879,18 +26801,14 @@ pub unsafe fn _mm256_mask_cmp_pd_mask(k1: __mmask8, a: __m256d, b: __m256d, imm8 /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cmp_pd_mask&expand=737) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[rustc_args_required_const(2)] -#[cfg_attr(test, assert_instr(vcmp, imm8 = 0))] -pub unsafe fn _mm_cmp_pd_mask(a: __m128d, b: __m128d, imm8: i32) -> __mmask8 { +#[rustc_legacy_const_generics(2)] +#[cfg_attr(test, assert_instr(vcmp, IMM8 = 0))] +pub unsafe fn _mm_cmp_pd_mask(a: __m128d, b: __m128d) -> __mmask8 { + static_assert_imm5!(IMM8); let neg_one = -1; let a = a.as_f64x2(); let b = b.as_f64x2(); - macro_rules! call { - ($imm5:expr) => { - vcmppd128(a, b, $imm5, neg_one) - }; - } - let r = constify_imm5!(imm8, call); + let r = vcmppd128(a, b, IMM8, neg_one); transmute(r) } @@ -28899,17 +26817,17 @@ pub unsafe fn _mm_cmp_pd_mask(a: __m128d, b: __m128d, imm8: i32) -> __mmask8 { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_cmp_pd_mask&expand=738) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[rustc_args_required_const(3)] -#[cfg_attr(test, assert_instr(vcmp, imm8 = 0))] -pub unsafe fn _mm_mask_cmp_pd_mask(k1: __mmask8, a: __m128d, b: __m128d, imm8: i32) -> __mmask8 { +#[rustc_legacy_const_generics(3)] +#[cfg_attr(test, assert_instr(vcmp, IMM8 = 0))] +pub unsafe fn _mm_mask_cmp_pd_mask( + k1: __mmask8, + a: __m128d, + b: __m128d, +) -> __mmask8 { + static_assert_imm5!(IMM8); let a = a.as_f64x2(); let b = b.as_f64x2(); - macro_rules! call { - ($imm5:expr) => { - vcmppd128(a, b, $imm5, k1 as i8) - }; - } - let r = constify_imm5!(imm8, call); + let r = vcmppd128(a, b, IMM8, k1 as i8); transmute(r) } @@ -28919,18 +26837,18 @@ pub unsafe fn _mm_mask_cmp_pd_mask(k1: __mmask8, a: __m128d, b: __m128d, imm8: i /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cmp_round_pd_mask&expand=751) #[inline] #[target_feature(enable = "avx512f")] -#[rustc_args_required_const(2, 3)] -#[cfg_attr(test, assert_instr(vcmp, imm8 = 0, sae = 4))] -pub unsafe fn _mm512_cmp_round_pd_mask(a: __m512d, b: __m512d, imm8: i32, sae: i32) -> __mmask8 { +#[cfg_attr(test, assert_instr(vcmp, IMM5 = 0, SAE = 4))] +#[rustc_legacy_const_generics(2, 3)] +pub unsafe fn _mm512_cmp_round_pd_mask( + a: __m512d, + b: __m512d, +) -> __mmask8 { + static_assert_imm5!(IMM5); + static_assert_mantissas_sae!(SAE); let neg_one = -1; let a = a.as_f64x8(); let b = b.as_f64x8(); - macro_rules! call { - ($imm5:expr, $imm4:expr) => { - vcmppd(a, b, $imm5, neg_one, $imm4) - }; - } - let r = constify_imm5_sae!(imm8, sae, call); + let r = vcmppd(a, b, IMM5, neg_one, SAE); transmute(r) } @@ -28940,23 +26858,18 @@ pub unsafe fn _mm512_cmp_round_pd_mask(a: __m512d, b: __m512d, imm8: i32, sae: i /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cmp_round_pd_mask&expand=752) #[inline] #[target_feature(enable = "avx512f")] -#[rustc_args_required_const(3, 4)] -#[cfg_attr(test, assert_instr(vcmp, imm8 = 0, sae = 4))] -pub unsafe fn _mm512_mask_cmp_round_pd_mask( +#[cfg_attr(test, assert_instr(vcmp, IMM5 = 0, SAE = 4))] +#[rustc_legacy_const_generics(3, 4)] +pub unsafe fn _mm512_mask_cmp_round_pd_mask( k1: __mmask8, a: __m512d, b: __m512d, - imm8: i32, - sae: i32, ) -> __mmask8 { + static_assert_imm5!(IMM5); + static_assert_mantissas_sae!(SAE); let a = a.as_f64x8(); let b = b.as_f64x8(); - macro_rules! call { - ($imm5:expr, $imm4:expr) => { - vcmppd(a, b, $imm5, k1 as i8, $imm4) - }; - } - let r = constify_imm5_sae!(imm8, sae, call); + let r = vcmppd(a, b, IMM5, k1 as i8, SAE); transmute(r) } @@ -28967,7 +26880,7 @@ pub unsafe fn _mm512_mask_cmp_round_pd_mask( #[target_feature(enable = "avx512f")] #[cfg_attr(test, assert_instr(vcmp))] //should be vcmppd pub unsafe fn _mm512_cmpord_pd_mask(a: __m512d, b: __m512d) -> __mmask8 { - _mm512_cmp_pd_mask(a, b, _CMP_ORD_Q) + _mm512_cmp_pd_mask::<_CMP_ORD_Q>(a, b) } /// Compare packed double-precision (64-bit) floating-point elements in a and b to see if neither is NaN, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). @@ -28977,7 +26890,7 @@ pub unsafe fn _mm512_cmpord_pd_mask(a: __m512d, b: __m512d) -> __mmask8 { #[target_feature(enable = "avx512f")] #[cfg_attr(test, assert_instr(vcmp))] //should be vcmppd pub unsafe fn _mm512_mask_cmpord_pd_mask(k1: __mmask8, a: __m512d, b: __m512d) -> __mmask8 { - _mm512_mask_cmp_pd_mask(k1, a, b, _CMP_ORD_Q) + _mm512_mask_cmp_pd_mask::<_CMP_ORD_Q>(k1, a, b) } /// Compare packed double-precision (64-bit) floating-point elements in a and b to see if either is NaN, and store the results in mask vector k. @@ -28987,7 +26900,7 @@ pub unsafe fn _mm512_mask_cmpord_pd_mask(k1: __mmask8, a: __m512d, b: __m512d) - #[target_feature(enable = "avx512f")] #[cfg_attr(test, assert_instr(vcmp))] //should be vcmppd pub unsafe fn _mm512_cmpunord_pd_mask(a: __m512d, b: __m512d) -> __mmask8 { - _mm512_cmp_pd_mask(a, b, _CMP_UNORD_Q) + _mm512_cmp_pd_mask::<_CMP_UNORD_Q>(a, b) } /// Compare packed double-precision (64-bit) floating-point elements in a and b to see if either is NaN, and store the results in mask vector k using zeromask k1 (elements are zeroed out when the corresponding mask bit is not set). @@ -28997,7 +26910,7 @@ pub unsafe fn _mm512_cmpunord_pd_mask(a: __m512d, b: __m512d) -> __mmask8 { #[target_feature(enable = "avx512f")] #[cfg_attr(test, assert_instr(vcmp))] //should be vcmppd pub unsafe fn _mm512_mask_cmpunord_pd_mask(k1: __mmask8, a: __m512d, b: __m512d) -> __mmask8 { - _mm512_mask_cmp_pd_mask(k1, a, b, _CMP_UNORD_Q) + _mm512_mask_cmp_pd_mask::<_CMP_UNORD_Q>(k1, a, b) } /// Compare the lower single-precision (32-bit) floating-point element in a and b based on the comparison operand specified by imm8, and store the result in mask vector k. @@ -29005,16 +26918,12 @@ pub unsafe fn _mm512_mask_cmpunord_pd_mask(k1: __mmask8, a: __m512d, b: __m512d) /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cmp_ss_mask&expand=763) #[inline] #[target_feature(enable = "avx512f")] -#[rustc_args_required_const(2)] -#[cfg_attr(test, assert_instr(vcmp, imm8 = 0))] -pub unsafe fn _mm_cmp_ss_mask(a: __m128, b: __m128, imm8: i32) -> __mmask8 { +#[rustc_legacy_const_generics(2)] +#[cfg_attr(test, assert_instr(vcmp, IMM8 = 0))] +pub unsafe fn _mm_cmp_ss_mask(a: __m128, b: __m128) -> __mmask8 { + static_assert_imm5!(IMM8); let neg_one = -1; - macro_rules! call { - ($imm5:expr) => { - vcmpss(a, b, $imm5, neg_one, _MM_FROUND_CUR_DIRECTION) - }; - } - let r = constify_imm5!(imm8, call); + let r = vcmpss(a, b, IMM8, neg_one, _MM_FROUND_CUR_DIRECTION); transmute(r) } @@ -29023,15 +26932,15 @@ pub unsafe fn _mm_cmp_ss_mask(a: __m128, b: __m128, imm8: i32) -> __mmask8 { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_cmp_ss_mask&expand=764) #[inline] #[target_feature(enable = "avx512f")] -#[rustc_args_required_const(3)] -#[cfg_attr(test, assert_instr(vcmp, imm8 = 0))] -pub unsafe fn _mm_mask_cmp_ss_mask(k1: __mmask8, a: __m128, b: __m128, imm8: i32) -> __mmask8 { - macro_rules! call { - ($imm5:expr) => { - vcmpss(a, b, $imm5, k1 as i8, _MM_FROUND_CUR_DIRECTION) - }; - } - let r = constify_imm5!(imm8, call); +#[rustc_legacy_const_generics(3)] +#[cfg_attr(test, assert_instr(vcmp, IMM8 = 0))] +pub unsafe fn _mm_mask_cmp_ss_mask( + k1: __mmask8, + a: __m128, + b: __m128, +) -> __mmask8 { + static_assert_imm5!(IMM8); + let r = vcmpss(a, b, IMM8, k1 as i8, _MM_FROUND_CUR_DIRECTION); transmute(r) } @@ -29041,16 +26950,16 @@ pub unsafe fn _mm_mask_cmp_ss_mask(k1: __mmask8, a: __m128, b: __m128, imm8: i32 /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cmp_round_ss_mask&expand=757) #[inline] #[target_feature(enable = "avx512f")] -#[rustc_args_required_const(2, 3)] -#[cfg_attr(test, assert_instr(vcmp, imm8 = 0, sae = 4))] -pub unsafe fn _mm_cmp_round_ss_mask(a: __m128, b: __m128, imm8: i32, sae: i32) -> __mmask8 { +#[cfg_attr(test, assert_instr(vcmp, IMM5 = 0, SAE = 4))] +#[rustc_legacy_const_generics(2, 3)] +pub unsafe fn _mm_cmp_round_ss_mask( + a: __m128, + b: __m128, +) -> __mmask8 { + static_assert_imm5!(IMM5); + static_assert_mantissas_sae!(SAE); let neg_one = -1; - macro_rules! call { - ($imm5:expr, $imm4:expr) => { - vcmpss(a, b, $imm5, neg_one, $imm4) - }; - } - let r = constify_imm5_sae!(imm8, sae, call); + let r = vcmpss(a, b, IMM5, neg_one, SAE); transmute(r) } @@ -29060,21 +26969,16 @@ pub unsafe fn _mm_cmp_round_ss_mask(a: __m128, b: __m128, imm8: i32, sae: i32) - /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_cmp_round_ss_mask&expand=758) #[inline] #[target_feature(enable = "avx512f")] -#[rustc_args_required_const(3, 4)] -#[cfg_attr(test, assert_instr(vcmp, imm8 = 0, sae = 4))] -pub unsafe fn _mm_mask_cmp_round_ss_mask( +#[cfg_attr(test, assert_instr(vcmp, IMM5 = 0, SAE = 4))] +#[rustc_legacy_const_generics(3, 4)] +pub unsafe fn _mm_mask_cmp_round_ss_mask( k1: __mmask8, a: __m128, b: __m128, - imm8: i32, - sae: i32, ) -> __mmask8 { - macro_rules! call { - ($imm5:expr, $imm4:expr) => { - vcmpss(a, b, $imm5, k1 as i8, $imm4) - }; - } - let r = constify_imm5_sae!(imm8, sae, call); + static_assert_imm5!(IMM5); + static_assert_mantissas_sae!(SAE); + let r = vcmpss(a, b, IMM5, k1 as i8, SAE); transmute(r) } @@ -29083,16 +26987,12 @@ pub unsafe fn _mm_mask_cmp_round_ss_mask( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cmp_sd_mask&expand=760) #[inline] #[target_feature(enable = "avx512f")] -#[rustc_args_required_const(2)] -#[cfg_attr(test, assert_instr(vcmp, imm8 = 0))] -pub unsafe fn _mm_cmp_sd_mask(a: __m128d, b: __m128d, imm8: i32) -> __mmask8 { +#[rustc_legacy_const_generics(2)] +#[cfg_attr(test, assert_instr(vcmp, IMM8 = 0))] +pub unsafe fn _mm_cmp_sd_mask(a: __m128d, b: __m128d) -> __mmask8 { + static_assert_imm5!(IMM8); let neg_one = -1; - macro_rules! call { - ($imm5:expr) => { - vcmpsd(a, b, $imm5, neg_one, _MM_FROUND_CUR_DIRECTION) - }; - } - let r = constify_imm5!(imm8, call); + let r = vcmpsd(a, b, IMM8, neg_one, _MM_FROUND_CUR_DIRECTION); transmute(r) } @@ -29101,15 +27001,15 @@ pub unsafe fn _mm_cmp_sd_mask(a: __m128d, b: __m128d, imm8: i32) -> __mmask8 { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_cmp_sd_mask&expand=761) #[inline] #[target_feature(enable = "avx512f")] -#[rustc_args_required_const(3)] -#[cfg_attr(test, assert_instr(vcmp, imm8 = 0))] -pub unsafe fn _mm_mask_cmp_sd_mask(k1: __mmask8, a: __m128d, b: __m128d, imm8: i32) -> __mmask8 { - macro_rules! call { - ($imm5:expr) => { - vcmpsd(a, b, $imm5, k1 as i8, _MM_FROUND_CUR_DIRECTION) - }; - } - let r = constify_imm5!(imm8, call); +#[rustc_legacy_const_generics(3)] +#[cfg_attr(test, assert_instr(vcmp, IMM8 = 0))] +pub unsafe fn _mm_mask_cmp_sd_mask( + k1: __mmask8, + a: __m128d, + b: __m128d, +) -> __mmask8 { + static_assert_imm5!(IMM8); + let r = vcmpsd(a, b, IMM8, k1 as i8, _MM_FROUND_CUR_DIRECTION); transmute(r) } @@ -29119,16 +27019,16 @@ pub unsafe fn _mm_mask_cmp_sd_mask(k1: __mmask8, a: __m128d, b: __m128d, imm8: i /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cmp_round_sd_mask&expand=755) #[inline] #[target_feature(enable = "avx512f")] -#[rustc_args_required_const(2, 3)] -#[cfg_attr(test, assert_instr(vcmp, imm8 = 0, sae = 4))] -pub unsafe fn _mm_cmp_round_sd_mask(a: __m128d, b: __m128d, imm8: i32, sae: i32) -> __mmask8 { +#[cfg_attr(test, assert_instr(vcmp, IMM5 = 0, SAE = 4))] +#[rustc_legacy_const_generics(2, 3)] +pub unsafe fn _mm_cmp_round_sd_mask( + a: __m128d, + b: __m128d, +) -> __mmask8 { + static_assert_imm5!(IMM5); + static_assert_mantissas_sae!(SAE); let neg_one = -1; - macro_rules! call { - ($imm5:expr, $imm4:expr) => { - vcmpsd(a, b, $imm5, neg_one, $imm4) - }; - } - let r = constify_imm5_sae!(imm8, sae, call); + let r = vcmpsd(a, b, IMM5, neg_one, SAE); transmute(r) } @@ -29138,21 +27038,16 @@ pub unsafe fn _mm_cmp_round_sd_mask(a: __m128d, b: __m128d, imm8: i32, sae: i32) /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_cmp_round_sd_mask&expand=756) #[inline] #[target_feature(enable = "avx512f")] -#[rustc_args_required_const(3, 4)] -#[cfg_attr(test, assert_instr(vcmp, imm8 = 0, sae = 4))] -pub unsafe fn _mm_mask_cmp_round_sd_mask( +#[cfg_attr(test, assert_instr(vcmp, IMM5 = 0, SAE = 4))] +#[rustc_legacy_const_generics(3, 4)] +pub unsafe fn _mm_mask_cmp_round_sd_mask( k1: __mmask8, a: __m128d, b: __m128d, - imm8: i32, - sae: i32, ) -> __mmask8 { - macro_rules! call { - ($imm5:expr, $imm4:expr) => { - vcmpsd(a, b, $imm5, k1 as i8, $imm4) - }; - } - let r = constify_imm5_sae!(imm8, sae, call); + static_assert_imm5!(IMM5); + static_assert_mantissas_sae!(SAE); + let r = vcmpsd(a, b, IMM5, k1 as i8, SAE); transmute(r) } @@ -29521,18 +27416,17 @@ pub unsafe fn _mm_mask_cmpneq_epu32_mask(k1: __mmask8, a: __m128i, b: __m128i) - /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cmp_epu32_mask&expand=721) #[inline] #[target_feature(enable = "avx512f")] -#[rustc_args_required_const(2)] -#[cfg_attr(test, assert_instr(vpcmp, imm8 = 0))] -pub unsafe fn _mm512_cmp_epu32_mask(a: __m512i, b: __m512i, imm8: _MM_CMPINT_ENUM) -> __mmask16 { +#[rustc_legacy_const_generics(2)] +#[cfg_attr(test, assert_instr(vpcmp, IMM3 = 0))] +pub unsafe fn _mm512_cmp_epu32_mask( + a: __m512i, + b: __m512i, +) -> __mmask16 { + static_assert_imm3!(IMM3); let neg_one = -1; let a = a.as_i32x16(); let b = b.as_i32x16(); - macro_rules! call { - ($imm3:expr) => { - vpcmpud(a, b, $imm3, neg_one) - }; - } - let r = constify_imm3!(imm8, call); + let r = vpcmpud(a, b, IMM3, neg_one); transmute(r) } @@ -29541,22 +27435,17 @@ pub unsafe fn _mm512_cmp_epu32_mask(a: __m512i, b: __m512i, imm8: _MM_CMPINT_ENU /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cmp_epu32_mask&expand=722) #[inline] #[target_feature(enable = "avx512f")] -#[rustc_args_required_const(3)] -#[cfg_attr(test, assert_instr(vpcmp, imm8 = 0))] -pub unsafe fn _mm512_mask_cmp_epu32_mask( +#[rustc_legacy_const_generics(3)] +#[cfg_attr(test, assert_instr(vpcmp, IMM3 = 0))] +pub unsafe fn _mm512_mask_cmp_epu32_mask( k1: __mmask16, a: __m512i, b: __m512i, - imm8: _MM_CMPINT_ENUM, ) -> __mmask16 { + static_assert_imm3!(IMM3); let a = a.as_i32x16(); let b = b.as_i32x16(); - macro_rules! call { - ($imm3:expr) => { - vpcmpud(a, b, $imm3, k1 as i16) - }; - } - let r = constify_imm3!(imm8, call); + let r = vpcmpud(a, b, IMM3, k1 as i16); transmute(r) } @@ -29565,18 +27454,17 @@ pub unsafe fn _mm512_mask_cmp_epu32_mask( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_cmp_epu32_mask&expand=719) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[rustc_args_required_const(2)] -#[cfg_attr(test, assert_instr(vpcmp, imm8 = 0))] -pub unsafe fn _mm256_cmp_epu32_mask(a: __m256i, b: __m256i, imm8: _MM_CMPINT_ENUM) -> __mmask8 { +#[rustc_legacy_const_generics(2)] +#[cfg_attr(test, assert_instr(vpcmp, IMM3 = 0))] +pub unsafe fn _mm256_cmp_epu32_mask( + a: __m256i, + b: __m256i, +) -> __mmask8 { + static_assert_imm3!(IMM3); let neg_one = -1; let a = a.as_i32x8(); let b = b.as_i32x8(); - macro_rules! call { - ($imm3:expr) => { - vpcmpud256(a, b, $imm3, neg_one) - }; - } - let r = constify_imm3!(imm8, call); + let r = vpcmpud256(a, b, IMM3, neg_one); transmute(r) } @@ -29585,22 +27473,17 @@ pub unsafe fn _mm256_cmp_epu32_mask(a: __m256i, b: __m256i, imm8: _MM_CMPINT_ENU /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_cmp_epu32_mask&expand=720) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[rustc_args_required_const(3)] -#[cfg_attr(test, assert_instr(vpcmp, imm8 = 0))] -pub unsafe fn _mm256_mask_cmp_epu32_mask( +#[rustc_legacy_const_generics(3)] +#[cfg_attr(test, assert_instr(vpcmp, IMM3 = 0))] +pub unsafe fn _mm256_mask_cmp_epu32_mask( k1: __mmask8, a: __m256i, b: __m256i, - imm8: _MM_CMPINT_ENUM, ) -> __mmask8 { + static_assert_imm3!(IMM3); let a = a.as_i32x8(); let b = b.as_i32x8(); - macro_rules! call { - ($imm3:expr) => { - vpcmpud256(a, b, $imm3, k1 as i8) - }; - } - let r = constify_imm3!(imm8, call); + let r = vpcmpud256(a, b, IMM3, k1 as i8); transmute(r) } @@ -29609,18 +27492,14 @@ pub unsafe fn _mm256_mask_cmp_epu32_mask( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cmp_epu32_mask&expand=717) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[rustc_args_required_const(2)] -#[cfg_attr(test, assert_instr(vpcmp, imm8 = 0))] -pub unsafe fn _mm_cmp_epu32_mask(a: __m128i, b: __m128i, imm8: _MM_CMPINT_ENUM) -> __mmask8 { +#[rustc_legacy_const_generics(2)] +#[cfg_attr(test, assert_instr(vpcmp, IMM3 = 0))] +pub unsafe fn _mm_cmp_epu32_mask(a: __m128i, b: __m128i) -> __mmask8 { + static_assert_imm3!(IMM3); let neg_one = -1; let a = a.as_i32x4(); let b = b.as_i32x4(); - macro_rules! call { - ($imm3:expr) => { - vpcmpud128(a, b, $imm3, neg_one) - }; - } - let r = constify_imm3!(imm8, call); + let r = vpcmpud128(a, b, IMM3, neg_one); transmute(r) } @@ -29629,22 +27508,17 @@ pub unsafe fn _mm_cmp_epu32_mask(a: __m128i, b: __m128i, imm8: _MM_CMPINT_ENUM) /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_cmp_epu32_mask&expand=718) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[rustc_args_required_const(3)] -#[cfg_attr(test, assert_instr(vpcmp, imm8 = 0))] -pub unsafe fn _mm_mask_cmp_epu32_mask( +#[rustc_legacy_const_generics(3)] +#[cfg_attr(test, assert_instr(vpcmp, IMM3 = 0))] +pub unsafe fn _mm_mask_cmp_epu32_mask( k1: __mmask8, a: __m128i, b: __m128i, - imm8: _MM_CMPINT_ENUM, ) -> __mmask8 { + static_assert_imm3!(IMM3); let a = a.as_i32x4(); let b = b.as_i32x4(); - macro_rules! call { - ($imm3:expr) => { - vpcmpud128(a, b, $imm3, k1 as i8) - }; - } - let r = constify_imm3!(imm8, call); + let r = vpcmpud128(a, b, IMM3, k1 as i8); transmute(r) } @@ -30013,18 +27887,17 @@ pub unsafe fn _mm_mask_cmpneq_epi32_mask(k1: __mmask8, a: __m128i, b: __m128i) - /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cmp_epi32_mask&expand=697) #[inline] #[target_feature(enable = "avx512f")] -#[rustc_args_required_const(2)] -#[cfg_attr(test, assert_instr(vpcmp, imm8 = 0))] -pub unsafe fn _mm512_cmp_epi32_mask(a: __m512i, b: __m512i, imm8: _MM_CMPINT_ENUM) -> __mmask16 { +#[rustc_legacy_const_generics(2)] +#[cfg_attr(test, assert_instr(vpcmp, IMM3 = 0))] +pub unsafe fn _mm512_cmp_epi32_mask( + a: __m512i, + b: __m512i, +) -> __mmask16 { + static_assert_imm3!(IMM3); let neg_one = -1; let a = a.as_i32x16(); let b = b.as_i32x16(); - macro_rules! call { - ($imm3:expr) => { - vpcmpd(a, b, $imm3, neg_one) - }; - } - let r = constify_imm3!(imm8, call); + let r = vpcmpd(a, b, IMM3, neg_one); transmute(r) } @@ -30033,22 +27906,17 @@ pub unsafe fn _mm512_cmp_epi32_mask(a: __m512i, b: __m512i, imm8: _MM_CMPINT_ENU /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cmp_epi32_mask&expand=698) #[inline] #[target_feature(enable = "avx512f")] -#[rustc_args_required_const(3)] -#[cfg_attr(test, assert_instr(vpcmp, imm8 = 0))] -pub unsafe fn _mm512_mask_cmp_epi32_mask( +#[rustc_legacy_const_generics(3)] +#[cfg_attr(test, assert_instr(vpcmp, IMM3 = 0))] +pub unsafe fn _mm512_mask_cmp_epi32_mask( k1: __mmask16, a: __m512i, b: __m512i, - imm8: _MM_CMPINT_ENUM, ) -> __mmask16 { + static_assert_imm3!(IMM3); let a = a.as_i32x16(); let b = b.as_i32x16(); - macro_rules! call { - ($imm3:expr) => { - vpcmpd(a, b, $imm3, k1 as i16) - }; - } - let r = constify_imm3!(imm8, call); + let r = vpcmpd(a, b, IMM3, k1 as i16); transmute(r) } @@ -30057,18 +27925,17 @@ pub unsafe fn _mm512_mask_cmp_epi32_mask( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=#text=_mm256_cmp_epi32_mask&expand=695) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[rustc_args_required_const(2)] -#[cfg_attr(test, assert_instr(vpcmp, imm8 = 0))] -pub unsafe fn _mm256_cmp_epi32_mask(a: __m256i, b: __m256i, imm8: _MM_CMPINT_ENUM) -> __mmask8 { +#[rustc_legacy_const_generics(2)] +#[cfg_attr(test, assert_instr(vpcmp, IMM3 = 0))] +pub unsafe fn _mm256_cmp_epi32_mask( + a: __m256i, + b: __m256i, +) -> __mmask8 { + static_assert_imm3!(IMM3); let neg_one = -1; let a = a.as_i32x8(); let b = b.as_i32x8(); - macro_rules! call { - ($imm3:expr) => { - vpcmpd256(a, b, $imm3, neg_one) - }; - } - let r = constify_imm3!(imm8, call); + let r = vpcmpd256(a, b, IMM3, neg_one); transmute(r) } @@ -30077,22 +27944,17 @@ pub unsafe fn _mm256_cmp_epi32_mask(a: __m256i, b: __m256i, imm8: _MM_CMPINT_ENU /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_cmp_epi32_mask&expand=696) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[rustc_args_required_const(3)] -#[cfg_attr(test, assert_instr(vpcmp, imm8 = 0))] -pub unsafe fn _mm256_mask_cmp_epi32_mask( +#[rustc_legacy_const_generics(3)] +#[cfg_attr(test, assert_instr(vpcmp, IMM3 = 0))] +pub unsafe fn _mm256_mask_cmp_epi32_mask( k1: __mmask8, a: __m256i, b: __m256i, - imm8: _MM_CMPINT_ENUM, ) -> __mmask8 { + static_assert_imm3!(IMM3); let a = a.as_i32x8(); let b = b.as_i32x8(); - macro_rules! call { - ($imm3:expr) => { - vpcmpd256(a, b, $imm3, k1 as i8) - }; - } - let r = constify_imm3!(imm8, call); + let r = vpcmpd256(a, b, IMM3, k1 as i8); transmute(r) } @@ -30101,18 +27963,14 @@ pub unsafe fn _mm256_mask_cmp_epi32_mask( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cmp_epi32_mask&expand=693) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[rustc_args_required_const(2)] -#[cfg_attr(test, assert_instr(vpcmp, imm8 = 0))] -pub unsafe fn _mm_cmp_epi32_mask(a: __m128i, b: __m128i, imm8: _MM_CMPINT_ENUM) -> __mmask8 { +#[rustc_legacy_const_generics(2)] +#[cfg_attr(test, assert_instr(vpcmp, IMM3 = 0))] +pub unsafe fn _mm_cmp_epi32_mask(a: __m128i, b: __m128i) -> __mmask8 { + static_assert_imm3!(IMM3); let neg_one = -1; let a = a.as_i32x4(); let b = b.as_i32x4(); - macro_rules! call { - ($imm3:expr) => { - vpcmpd128(a, b, $imm3, neg_one) - }; - } - let r = constify_imm3!(imm8, call); + let r = vpcmpd128(a, b, IMM3, neg_one); transmute(r) } @@ -30121,22 +27979,17 @@ pub unsafe fn _mm_cmp_epi32_mask(a: __m128i, b: __m128i, imm8: _MM_CMPINT_ENUM) /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_cmp_epi32_mask&expand=694) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[rustc_args_required_const(3)] -#[cfg_attr(test, assert_instr(vpcmp, imm8 = 0))] -pub unsafe fn _mm_mask_cmp_epi32_mask( +#[rustc_legacy_const_generics(3)] +#[cfg_attr(test, assert_instr(vpcmp, IMM3 = 0))] +pub unsafe fn _mm_mask_cmp_epi32_mask( k1: __mmask8, a: __m128i, b: __m128i, - imm8: _MM_CMPINT_ENUM, ) -> __mmask8 { + static_assert_imm3!(IMM3); let a = a.as_i32x4(); let b = b.as_i32x4(); - macro_rules! call { - ($imm3:expr) => { - vpcmpd128(a, b, $imm3, k1 as i8) - }; - } - let r = constify_imm3!(imm8, call); + let r = vpcmpd128(a, b, IMM3, k1 as i8); transmute(r) } @@ -30505,18 +28358,17 @@ pub unsafe fn _mm_mask_cmpneq_epu64_mask(k1: __mmask8, a: __m128i, b: __m128i) - /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cmp_epu64_mask&expand=727) #[inline] #[target_feature(enable = "avx512f")] -#[rustc_args_required_const(2)] -#[cfg_attr(test, assert_instr(vpcmp, imm8 = 0))] -pub unsafe fn _mm512_cmp_epu64_mask(a: __m512i, b: __m512i, imm8: _MM_CMPINT_ENUM) -> __mmask8 { +#[rustc_legacy_const_generics(2)] +#[cfg_attr(test, assert_instr(vpcmp, IMM3 = 0))] +pub unsafe fn _mm512_cmp_epu64_mask( + a: __m512i, + b: __m512i, +) -> __mmask8 { + static_assert_imm3!(IMM3); let neg_one = -1; let a = a.as_i64x8(); let b = b.as_i64x8(); - macro_rules! call { - ($imm3:expr) => { - vpcmpuq(a, b, $imm3, neg_one) - }; - } - let r = constify_imm3!(imm8, call); + let r = vpcmpuq(a, b, IMM3, neg_one); transmute(r) } @@ -30525,22 +28377,17 @@ pub unsafe fn _mm512_cmp_epu64_mask(a: __m512i, b: __m512i, imm8: _MM_CMPINT_ENU /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cmp_epu64_mask&expand=728) #[inline] #[target_feature(enable = "avx512f")] -#[rustc_args_required_const(3)] -#[cfg_attr(test, assert_instr(vpcmp, imm8 = 0))] -pub unsafe fn _mm512_mask_cmp_epu64_mask( +#[rustc_legacy_const_generics(3)] +#[cfg_attr(test, assert_instr(vpcmp, IMM3 = 0))] +pub unsafe fn _mm512_mask_cmp_epu64_mask( k1: __mmask8, a: __m512i, b: __m512i, - imm8: _MM_CMPINT_ENUM, ) -> __mmask8 { + static_assert_imm3!(IMM3); let a = a.as_i64x8(); let b = b.as_i64x8(); - macro_rules! call { - ($imm3:expr) => { - vpcmpuq(a, b, $imm3, k1 as i8) - }; - } - let r = constify_imm3!(imm8, call); + let r = vpcmpuq(a, b, IMM3, k1 as i8); transmute(r) } @@ -30549,18 +28396,17 @@ pub unsafe fn _mm512_mask_cmp_epu64_mask( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_cmp_epu64_mask&expand=725) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[rustc_args_required_const(2)] -#[cfg_attr(test, assert_instr(vpcmp, imm8 = 0))] -pub unsafe fn _mm256_cmp_epu64_mask(a: __m256i, b: __m256i, imm8: _MM_CMPINT_ENUM) -> __mmask8 { +#[rustc_legacy_const_generics(2)] +#[cfg_attr(test, assert_instr(vpcmp, IMM3 = 0))] +pub unsafe fn _mm256_cmp_epu64_mask( + a: __m256i, + b: __m256i, +) -> __mmask8 { + static_assert_imm3!(IMM3); let neg_one = -1; let a = a.as_i64x4(); let b = b.as_i64x4(); - macro_rules! call { - ($imm3:expr) => { - vpcmpuq256(a, b, $imm3, neg_one) - }; - } - let r = constify_imm3!(imm8, call); + let r = vpcmpuq256(a, b, IMM3, neg_one); transmute(r) } @@ -30569,22 +28415,17 @@ pub unsafe fn _mm256_cmp_epu64_mask(a: __m256i, b: __m256i, imm8: _MM_CMPINT_ENU /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_cmp_epu64_mask&expand=726) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[rustc_args_required_const(3)] -#[cfg_attr(test, assert_instr(vpcmp, imm8 = 0))] -pub unsafe fn _mm256_mask_cmp_epu64_mask( +#[rustc_legacy_const_generics(3)] +#[cfg_attr(test, assert_instr(vpcmp, IMM3 = 0))] +pub unsafe fn _mm256_mask_cmp_epu64_mask( k1: __mmask8, a: __m256i, b: __m256i, - imm8: _MM_CMPINT_ENUM, ) -> __mmask8 { + static_assert_imm3!(IMM3); let a = a.as_i64x4(); let b = b.as_i64x4(); - macro_rules! call { - ($imm3:expr) => { - vpcmpuq256(a, b, $imm3, k1 as i8) - }; - } - let r = constify_imm3!(imm8, call); + let r = vpcmpuq256(a, b, IMM3, k1 as i8); transmute(r) } @@ -30593,18 +28434,14 @@ pub unsafe fn _mm256_mask_cmp_epu64_mask( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cmp_epu64_mask&expand=723) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[rustc_args_required_const(2)] -#[cfg_attr(test, assert_instr(vpcmp, imm8 = 0))] -pub unsafe fn _mm_cmp_epu64_mask(a: __m128i, b: __m128i, imm8: _MM_CMPINT_ENUM) -> __mmask8 { +#[rustc_legacy_const_generics(2)] +#[cfg_attr(test, assert_instr(vpcmp, IMM3 = 0))] +pub unsafe fn _mm_cmp_epu64_mask(a: __m128i, b: __m128i) -> __mmask8 { + static_assert_imm3!(IMM3); let neg_one = -1; let a = a.as_i64x2(); let b = b.as_i64x2(); - macro_rules! call { - ($imm3:expr) => { - vpcmpuq128(a, b, $imm3, neg_one) - }; - } - let r = constify_imm3!(imm8, call); + let r = vpcmpuq128(a, b, IMM3, neg_one); transmute(r) } @@ -30613,22 +28450,17 @@ pub unsafe fn _mm_cmp_epu64_mask(a: __m128i, b: __m128i, imm8: _MM_CMPINT_ENUM) /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_cmp_epu64_mask&expand=724) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[rustc_args_required_const(3)] -#[cfg_attr(test, assert_instr(vpcmp, imm8 = 0))] -pub unsafe fn _mm_mask_cmp_epu64_mask( +#[rustc_legacy_const_generics(3)] +#[cfg_attr(test, assert_instr(vpcmp, IMM3 = 0))] +pub unsafe fn _mm_mask_cmp_epu64_mask( k1: __mmask8, a: __m128i, b: __m128i, - imm8: _MM_CMPINT_ENUM, ) -> __mmask8 { + static_assert_imm3!(IMM3); let a = a.as_i64x2(); let b = b.as_i64x2(); - macro_rules! call { - ($imm3:expr) => { - vpcmpuq128(a, b, $imm3, k1 as i8) - }; - } - let r = constify_imm3!(imm8, call); + let r = vpcmpuq128(a, b, IMM3, k1 as i8); transmute(r) } @@ -30997,18 +28829,17 @@ pub unsafe fn _mm_mask_cmpneq_epi64_mask(k1: __mmask8, a: __m128i, b: __m128i) - /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_cmp_epi64_mask&expand=703) #[inline] #[target_feature(enable = "avx512f")] -#[rustc_args_required_const(2)] -#[cfg_attr(test, assert_instr(vpcmp, imm8 = 0))] -pub unsafe fn _mm512_cmp_epi64_mask(a: __m512i, b: __m512i, imm8: _MM_CMPINT_ENUM) -> __mmask8 { +#[rustc_legacy_const_generics(2)] +#[cfg_attr(test, assert_instr(vpcmp, IMM3 = 0))] +pub unsafe fn _mm512_cmp_epi64_mask( + a: __m512i, + b: __m512i, +) -> __mmask8 { + static_assert_imm3!(IMM3); let neg_one = -1; let a = a.as_i64x8(); let b = b.as_i64x8(); - macro_rules! call { - ($imm3:expr) => { - vpcmpq(a, b, $imm3, neg_one) - }; - } - let r = constify_imm3!(imm8, call); + let r = vpcmpq(a, b, IMM3, neg_one); transmute(r) } @@ -31017,22 +28848,17 @@ pub unsafe fn _mm512_cmp_epi64_mask(a: __m512i, b: __m512i, imm8: _MM_CMPINT_ENU /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_cmp_epi64_mask&expand=704) #[inline] #[target_feature(enable = "avx512f")] -#[rustc_args_required_const(3)] -#[cfg_attr(test, assert_instr(vpcmp, imm8 = 0))] -pub unsafe fn _mm512_mask_cmp_epi64_mask( +#[rustc_legacy_const_generics(3)] +#[cfg_attr(test, assert_instr(vpcmp, IMM3 = 0))] +pub unsafe fn _mm512_mask_cmp_epi64_mask( k1: __mmask8, a: __m512i, b: __m512i, - imm8: _MM_CMPINT_ENUM, ) -> __mmask8 { + static_assert_imm3!(IMM3); let a = a.as_i64x8(); let b = b.as_i64x8(); - macro_rules! call { - ($imm3:expr) => { - vpcmpq(a, b, $imm3, k1 as i8) - }; - } - let r = constify_imm3!(imm8, call); + let r = vpcmpq(a, b, IMM3, k1 as i8); transmute(r) } @@ -31041,18 +28867,17 @@ pub unsafe fn _mm512_mask_cmp_epi64_mask( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_cmp_epi64_mask&expand=701) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[rustc_args_required_const(2)] -#[cfg_attr(test, assert_instr(vpcmp, imm8 = 0))] -pub unsafe fn _mm256_cmp_epi64_mask(a: __m256i, b: __m256i, imm8: _MM_CMPINT_ENUM) -> __mmask8 { +#[rustc_legacy_const_generics(2)] +#[cfg_attr(test, assert_instr(vpcmp, IMM3 = 0))] +pub unsafe fn _mm256_cmp_epi64_mask( + a: __m256i, + b: __m256i, +) -> __mmask8 { + static_assert_imm3!(IMM3); let neg_one = -1; let a = a.as_i64x4(); let b = b.as_i64x4(); - macro_rules! call { - ($imm3:expr) => { - vpcmpq256(a, b, $imm3, neg_one) - }; - } - let r = constify_imm3!(imm8, call); + let r = vpcmpq256(a, b, IMM3, neg_one); transmute(r) } @@ -31061,22 +28886,17 @@ pub unsafe fn _mm256_cmp_epi64_mask(a: __m256i, b: __m256i, imm8: _MM_CMPINT_ENU /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_cmp_epi64_mask&expand=702) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[rustc_args_required_const(3)] -#[cfg_attr(test, assert_instr(vpcmp, imm8 = 0))] -pub unsafe fn _mm256_mask_cmp_epi64_mask( +#[rustc_legacy_const_generics(3)] +#[cfg_attr(test, assert_instr(vpcmp, IMM3 = 0))] +pub unsafe fn _mm256_mask_cmp_epi64_mask( k1: __mmask8, a: __m256i, b: __m256i, - imm8: _MM_CMPINT_ENUM, ) -> __mmask8 { + static_assert_imm3!(IMM3); let a = a.as_i64x4(); let b = b.as_i64x4(); - macro_rules! call { - ($imm3:expr) => { - vpcmpq256(a, b, $imm3, k1 as i8) - }; - } - let r = constify_imm3!(imm8, call); + let r = vpcmpq256(a, b, IMM3, k1 as i8); transmute(r) } @@ -31085,18 +28905,14 @@ pub unsafe fn _mm256_mask_cmp_epi64_mask( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cmp_epi64_mask&expand=699) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[rustc_args_required_const(2)] -#[cfg_attr(test, assert_instr(vpcmp, imm8 = 0))] -pub unsafe fn _mm_cmp_epi64_mask(a: __m128i, b: __m128i, imm8: _MM_CMPINT_ENUM) -> __mmask8 { +#[rustc_legacy_const_generics(2)] +#[cfg_attr(test, assert_instr(vpcmp, IMM3 = 0))] +pub unsafe fn _mm_cmp_epi64_mask(a: __m128i, b: __m128i) -> __mmask8 { + static_assert_imm3!(IMM3); let neg_one = -1; let a = a.as_i64x2(); let b = b.as_i64x2(); - macro_rules! call { - ($imm3:expr) => { - vpcmpq128(a, b, $imm3, neg_one) - }; - } - let r = constify_imm3!(imm8, call); + let r = vpcmpq128(a, b, IMM3, neg_one); transmute(r) } @@ -31105,22 +28921,17 @@ pub unsafe fn _mm_cmp_epi64_mask(a: __m128i, b: __m128i, imm8: _MM_CMPINT_ENUM) /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_cmp_epi64_mask&expand=700) #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[rustc_args_required_const(3)] -#[cfg_attr(test, assert_instr(vpcmp, imm8 = 0))] -pub unsafe fn _mm_mask_cmp_epi64_mask( +#[rustc_legacy_const_generics(3)] +#[cfg_attr(test, assert_instr(vpcmp, IMM3 = 0))] +pub unsafe fn _mm_mask_cmp_epi64_mask( k1: __mmask8, a: __m128i, b: __m128i, - imm8: _MM_CMPINT_ENUM, ) -> __mmask8 { + static_assert_imm3!(IMM3); let a = a.as_i64x2(); let b = b.as_i64x2(); - macro_rules! call { - ($imm3:expr) => { - vpcmpq128(a, b, $imm3, k1 as i8) - }; - } - let r = constify_imm3!(imm8, call); + let r = vpcmpq128(a, b, IMM3, k1 as i8); transmute(r) } @@ -33354,30 +31165,21 @@ pub unsafe fn _mm_maskz_getexp_sd(k: __mmask8, a: __m128d, b: __m128d) -> __m128 /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_getmant_ss&expand=2898) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vgetmantss, norm = 0, sign = 0))] -#[rustc_args_required_const(2, 3)] -pub unsafe fn _mm_getmant_ss( +#[cfg_attr(test, assert_instr(vgetmantss, NORM = 0, SIGN = 0))] +#[rustc_legacy_const_generics(2, 3)] +pub unsafe fn _mm_getmant_ss< + const NORM: _MM_MANTISSA_NORM_ENUM, + const SIGN: _MM_MANTISSA_SIGN_ENUM, +>( a: __m128, b: __m128, - norm: _MM_MANTISSA_NORM_ENUM, - sign: _MM_MANTISSA_SIGN_ENUM, ) -> __m128 { + static_assert_imm4!(NORM); + static_assert_imm2!(SIGN); let a = a.as_f32x4(); let b = b.as_f32x4(); let zero = _mm_setzero_ps().as_f32x4(); - macro_rules! call { - ($imm4_1:expr, $imm2:expr) => { - vgetmantss( - a, - b, - $imm2 << 2 | $imm4_1, - zero, - 0b1, - _MM_FROUND_CUR_DIRECTION, - ) - }; - } - let r = constify_imm4_mantissas!(norm, sign, call); + let r = vgetmantss(a, b, SIGN << 2 | NORM, zero, 0b1, _MM_FROUND_CUR_DIRECTION); transmute(r) } @@ -33396,25 +31198,23 @@ pub unsafe fn _mm_getmant_ss( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_mask_getmant_ss&expand=2899) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vgetmantss, norm = 0, sign = 0))] -#[rustc_args_required_const(4, 5)] -pub unsafe fn _mm_mask_getmant_ss( +#[cfg_attr(test, assert_instr(vgetmantss, NORM = 0, SIGN = 0))] +#[rustc_legacy_const_generics(4, 5)] +pub unsafe fn _mm_mask_getmant_ss< + const NORM: _MM_MANTISSA_NORM_ENUM, + const SIGN: _MM_MANTISSA_SIGN_ENUM, +>( src: __m128, k: __mmask8, a: __m128, b: __m128, - norm: _MM_MANTISSA_NORM_ENUM, - sign: _MM_MANTISSA_SIGN_ENUM, ) -> __m128 { + static_assert_imm4!(NORM); + static_assert_imm2!(SIGN); let a = a.as_f32x4(); let b = b.as_f32x4(); let src = src.as_f32x4(); - macro_rules! call { - ($imm4_1:expr, $imm2:expr) => { - vgetmantss(a, b, $imm2 << 2 | $imm4_1, src, k, _MM_FROUND_CUR_DIRECTION) - }; - } - let r = constify_imm4_mantissas!(norm, sign, call); + let r = vgetmantss(a, b, SIGN << 2 | NORM, src, k, _MM_FROUND_CUR_DIRECTION); transmute(r) } @@ -33433,31 +31233,22 @@ pub unsafe fn _mm_mask_getmant_ss( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_maskz_getmant_ss&expand=2900) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vgetmantss, norm = 0, sign = 0))] -#[rustc_args_required_const(3, 4)] -pub unsafe fn _mm_maskz_getmant_ss( +#[cfg_attr(test, assert_instr(vgetmantss, NORM = 0, SIGN = 0))] +#[rustc_legacy_const_generics(3, 4)] +pub unsafe fn _mm_maskz_getmant_ss< + const NORM: _MM_MANTISSA_NORM_ENUM, + const SIGN: _MM_MANTISSA_SIGN_ENUM, +>( k: __mmask8, a: __m128, b: __m128, - norm: _MM_MANTISSA_NORM_ENUM, - sign: _MM_MANTISSA_SIGN_ENUM, ) -> __m128 { + static_assert_imm4!(NORM); + static_assert_imm2!(SIGN); let a = a.as_f32x4(); let b = b.as_f32x4(); let zero = _mm_setzero_ps().as_f32x4(); - macro_rules! call { - ($imm4_1:expr, $imm2:expr) => { - vgetmantss( - a, - b, - $imm2 << 2 | $imm4_1, - zero, - k, - _MM_FROUND_CUR_DIRECTION, - ) - }; - } - let r = constify_imm4_mantissas!(norm, sign, call); + let r = vgetmantss(a, b, SIGN << 2 | NORM, zero, k, _MM_FROUND_CUR_DIRECTION); transmute(r) } @@ -33476,30 +31267,21 @@ pub unsafe fn _mm_maskz_getmant_ss( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_getmant_sd&expand=2895) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vgetmantsd, norm = 0, sign = 0))] -#[rustc_args_required_const(2, 3)] -pub unsafe fn _mm_getmant_sd( +#[cfg_attr(test, assert_instr(vgetmantsd, NORM = 0, SIGN = 0))] +#[rustc_legacy_const_generics(2, 3)] +pub unsafe fn _mm_getmant_sd< + const NORM: _MM_MANTISSA_NORM_ENUM, + const SIGN: _MM_MANTISSA_SIGN_ENUM, +>( a: __m128d, b: __m128d, - norm: _MM_MANTISSA_NORM_ENUM, - sign: _MM_MANTISSA_SIGN_ENUM, ) -> __m128d { + static_assert_imm4!(NORM); + static_assert_imm2!(SIGN); let a = a.as_f64x2(); let b = b.as_f64x2(); let zero = _mm_setzero_pd().as_f64x2(); - macro_rules! call { - ($imm4_1:expr, $imm2:expr) => { - vgetmantsd( - a, - b, - $imm2 << 2 | $imm4_1, - zero, - 0b1, - _MM_FROUND_CUR_DIRECTION, - ) - }; - } - let r = constify_imm4_mantissas!(norm, sign, call); + let r = vgetmantsd(a, b, SIGN << 2 | NORM, zero, 0b1, _MM_FROUND_CUR_DIRECTION); transmute(r) } @@ -33518,25 +31300,23 @@ pub unsafe fn _mm_getmant_sd( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_mask_getmant_sd&expand=2896) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vgetmantsd, norm = 0, sign = 0))] -#[rustc_args_required_const(4, 5)] -pub unsafe fn _mm_mask_getmant_sd( +#[cfg_attr(test, assert_instr(vgetmantsd, NORM = 0, SIGN = 0))] +#[rustc_legacy_const_generics(4, 5)] +pub unsafe fn _mm_mask_getmant_sd< + const NORM: _MM_MANTISSA_NORM_ENUM, + const SIGN: _MM_MANTISSA_SIGN_ENUM, +>( src: __m128d, k: __mmask8, a: __m128d, b: __m128d, - norm: _MM_MANTISSA_NORM_ENUM, - sign: _MM_MANTISSA_SIGN_ENUM, ) -> __m128d { + static_assert_imm4!(NORM); + static_assert_imm2!(SIGN); let a = a.as_f64x2(); let b = b.as_f64x2(); let src = src.as_f64x2(); - macro_rules! call { - ($imm4_1:expr, $imm2:expr) => { - vgetmantsd(a, b, $imm2 << 2 | $imm4_1, src, k, _MM_FROUND_CUR_DIRECTION) - }; - } - let r = constify_imm4_mantissas!(norm, sign, call); + let r = vgetmantsd(a, b, SIGN << 2 | NORM, src, k, _MM_FROUND_CUR_DIRECTION); transmute(r) } @@ -33555,31 +31335,22 @@ pub unsafe fn _mm_mask_getmant_sd( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_maskz_getmant_sd&expand=2897) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vgetmantsd, norm = 0, sign = 0))] -#[rustc_args_required_const(3, 4)] -pub unsafe fn _mm_maskz_getmant_sd( +#[cfg_attr(test, assert_instr(vgetmantsd, NORM = 0, SIGN = 0))] +#[rustc_legacy_const_generics(3, 4)] +pub unsafe fn _mm_maskz_getmant_sd< + const NORM: _MM_MANTISSA_NORM_ENUM, + const SIGN: _MM_MANTISSA_SIGN_ENUM, +>( k: __mmask8, a: __m128d, b: __m128d, - norm: _MM_MANTISSA_NORM_ENUM, - sign: _MM_MANTISSA_SIGN_ENUM, ) -> __m128d { + static_assert_imm4!(NORM); + static_assert_imm2!(SIGN); let a = a.as_f64x2(); let b = b.as_f64x2(); let zero = _mm_setzero_pd().as_f64x2(); - macro_rules! call { - ($imm4_1:expr, $imm2:expr) => { - vgetmantsd( - a, - b, - $imm2 << 2 | $imm4_1, - zero, - k, - _MM_FROUND_CUR_DIRECTION, - ) - }; - } - let r = constify_imm4_mantissas!(norm, sign, call); + let r = vgetmantsd(a, b, SIGN << 2 | NORM, zero, k, _MM_FROUND_CUR_DIRECTION); transmute(r) } @@ -33594,18 +31365,14 @@ pub unsafe fn _mm_maskz_getmant_sd( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_roundscale_ss&expand=4802) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vrndscaless, imm8 = 255))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm_roundscale_ss(a: __m128, b: __m128, imm8: i32) -> __m128 { +#[cfg_attr(test, assert_instr(vrndscaless, IMM8 = 255))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm_roundscale_ss(a: __m128, b: __m128) -> __m128 { + static_assert_imm8!(IMM8); let a = a.as_f32x4(); let b = b.as_f32x4(); let zero = _mm_setzero_ps().as_f32x4(); - macro_rules! call { - ($imm8:expr) => { - vrndscaless(a, b, zero, 0b11111111, $imm8, _MM_FROUND_CUR_DIRECTION) - }; - } - let r = constify_imm8_sae!(imm8, call); + let r = vrndscaless(a, b, zero, 0b11111111, IMM8, _MM_FROUND_CUR_DIRECTION); transmute(r) } @@ -33620,24 +31387,19 @@ pub unsafe fn _mm_roundscale_ss(a: __m128, b: __m128, imm8: i32) -> __m128 { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_mask_roundscale_ss&expand=4800) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vrndscaless, imm8 = 0))] -#[rustc_args_required_const(4)] -pub unsafe fn _mm_mask_roundscale_ss( +#[cfg_attr(test, assert_instr(vrndscaless, IMM8 = 0))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm_mask_roundscale_ss( src: __m128, k: __mmask8, a: __m128, b: __m128, - imm8: i32, ) -> __m128 { + static_assert_imm8!(IMM8); let a = a.as_f32x4(); let b = b.as_f32x4(); let src = src.as_f32x4(); - macro_rules! call { - ($imm8:expr) => { - vrndscaless(a, b, src, k, $imm8, _MM_FROUND_CUR_DIRECTION) - }; - } - let r = constify_imm8_sae!(imm8, call); + let r = vrndscaless(a, b, src, k, IMM8, _MM_FROUND_CUR_DIRECTION); transmute(r) } @@ -33652,18 +31414,18 @@ pub unsafe fn _mm_mask_roundscale_ss( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_maskz_roundscale_ss&expand=4801) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vrndscaless, imm8 = 0))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm_maskz_roundscale_ss(k: __mmask8, a: __m128, b: __m128, imm8: i32) -> __m128 { +#[cfg_attr(test, assert_instr(vrndscaless, IMM8 = 0))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm_maskz_roundscale_ss( + k: __mmask8, + a: __m128, + b: __m128, +) -> __m128 { + static_assert_imm8!(IMM8); let a = a.as_f32x4(); let b = b.as_f32x4(); let zero = _mm_setzero_ps().as_f32x4(); - macro_rules! call { - ($imm8:expr) => { - vrndscaless(a, b, zero, k, $imm8, _MM_FROUND_CUR_DIRECTION) - }; - } - let r = constify_imm8_sae!(imm8, call); + let r = vrndscaless(a, b, zero, k, IMM8, _MM_FROUND_CUR_DIRECTION); transmute(r) } @@ -33678,18 +31440,14 @@ pub unsafe fn _mm_maskz_roundscale_ss(k: __mmask8, a: __m128, b: __m128, imm8: i /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_roundscale_sd&expand=4799) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vrndscalesd, imm8 = 255))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm_roundscale_sd(a: __m128d, b: __m128d, imm8: i32) -> __m128d { +#[cfg_attr(test, assert_instr(vrndscalesd, IMM8 = 255))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm_roundscale_sd(a: __m128d, b: __m128d) -> __m128d { + static_assert_imm8!(IMM8); let a = a.as_f64x2(); let b = b.as_f64x2(); let zero = _mm_setzero_pd().as_f64x2(); - macro_rules! call { - ($imm8:expr) => { - vrndscalesd(a, b, zero, 0b11111111, $imm8, _MM_FROUND_CUR_DIRECTION) - }; - } - let r = constify_imm8_sae!(imm8, call); + let r = vrndscalesd(a, b, zero, 0b11111111, IMM8, _MM_FROUND_CUR_DIRECTION); transmute(r) } @@ -33704,24 +31462,19 @@ pub unsafe fn _mm_roundscale_sd(a: __m128d, b: __m128d, imm8: i32) -> __m128d { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_mask_roundscale_sd&expand=4797) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vrndscalesd, imm8 = 0))] -#[rustc_args_required_const(4)] -pub unsafe fn _mm_mask_roundscale_sd( +#[cfg_attr(test, assert_instr(vrndscalesd, IMM8 = 0))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm_mask_roundscale_sd( src: __m128d, k: __mmask8, a: __m128d, b: __m128d, - imm8: i32, ) -> __m128d { + static_assert_imm8!(IMM8); let a = a.as_f64x2(); let b = b.as_f64x2(); let src = src.as_f64x2(); - macro_rules! call { - ($imm8:expr) => { - vrndscalesd(a, b, src, k, $imm8, _MM_FROUND_CUR_DIRECTION) - }; - } - let r = constify_imm8_sae!(imm8, call); + let r = vrndscalesd(a, b, src, k, IMM8, _MM_FROUND_CUR_DIRECTION); transmute(r) } @@ -33736,18 +31489,18 @@ pub unsafe fn _mm_mask_roundscale_sd( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_maskz_roundscale_sd&expand=4798) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vrndscalesd, imm8 = 0))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm_maskz_roundscale_sd(k: __mmask8, a: __m128d, b: __m128d, imm8: i32) -> __m128d { +#[cfg_attr(test, assert_instr(vrndscalesd, IMM8 = 0))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm_maskz_roundscale_sd( + k: __mmask8, + a: __m128d, + b: __m128d, +) -> __m128d { + static_assert_imm8!(IMM8); let a = a.as_f64x2(); let b = b.as_f64x2(); let zero = _mm_setzero_pd().as_f64x2(); - macro_rules! call { - ($imm8:expr) => { - vrndscalesd(a, b, zero, k, $imm8, _MM_FROUND_CUR_DIRECTION) - }; - } - let r = constify_imm8_sae!(imm8, call); + let r = vrndscalesd(a, b, zero, k, IMM8, _MM_FROUND_CUR_DIRECTION); transmute(r) } @@ -34293,18 +32046,15 @@ pub unsafe fn _mm_mask3_fnmsub_sd(a: __m128d, b: __m128d, c: __m128d, k: __mmask /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_add_round_ss&expand=151) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vaddss, rounding = 8))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm_add_round_ss(a: __m128, b: __m128, rounding: i32) -> __m128 { +#[cfg_attr(test, assert_instr(vaddss, ROUNDING = 8))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm_add_round_ss(a: __m128, b: __m128) -> __m128 { + static_assert_rounding!(ROUNDING); let a = a.as_f32x4(); let b = b.as_f32x4(); let zero = _mm_setzero_ps().as_f32x4(); - macro_rules! call { - ($imm4:expr) => { - vaddss(a, b, zero, 0b1, $imm4) - }; - } - transmute(constify_imm4_round!(rounding, call)) + let r = vaddss(a, b, zero, 0b1, ROUNDING); + transmute(r) } /// Add the lower single-precision (32-bit) floating-point element in a and b, store the result in the lower element of dst using writemask k (the element is copied from src when mask bit 0 is not set), and copy the upper 3 packed elements from a to the upper elements of dst.\ @@ -34319,24 +32069,20 @@ pub unsafe fn _mm_add_round_ss(a: __m128, b: __m128, rounding: i32) -> __m128 { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_mask_add_round_ss&expand=152) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vaddss, rounding = 8))] -#[rustc_args_required_const(4)] -pub unsafe fn _mm_mask_add_round_ss( +#[cfg_attr(test, assert_instr(vaddss, ROUNDING = 8))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm_mask_add_round_ss( src: __m128, k: __mmask8, a: __m128, b: __m128, - rounding: i32, ) -> __m128 { + static_assert_rounding!(ROUNDING); let a = a.as_f32x4(); let b = b.as_f32x4(); let src = src.as_f32x4(); - macro_rules! call { - ($imm4:expr) => { - vaddss(a, b, src, k, $imm4) - }; - } - transmute(constify_imm4_round!(rounding, call)) + let r = vaddss(a, b, src, k, ROUNDING); + transmute(r) } /// Add the lower single-precision (32-bit) floating-point element in a and b, store the result in the lower element of dst using zeromask k (the element is zeroed out when mask bit 0 is not set), and copy the upper 3 packed elements from a to the upper elements of dst.\ @@ -34351,18 +32097,19 @@ pub unsafe fn _mm_mask_add_round_ss( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_add_round_ss&expand=153) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vaddss, rounding = 8))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm_maskz_add_round_ss(k: __mmask8, a: __m128, b: __m128, rounding: i32) -> __m128 { +#[cfg_attr(test, assert_instr(vaddss, ROUNDING = 8))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm_maskz_add_round_ss( + k: __mmask8, + a: __m128, + b: __m128, +) -> __m128 { + static_assert_rounding!(ROUNDING); let a = a.as_f32x4(); let b = b.as_f32x4(); let zero = _mm_setzero_ps().as_f32x4(); - macro_rules! call { - ($imm4:expr) => { - vaddss(a, b, zero, k, $imm4) - }; - } - transmute(constify_imm4_round!(rounding, call)) + let r = vaddss(a, b, zero, k, ROUNDING); + transmute(r) } /// Add the lower double-precision (64-bit) floating-point element in a and b, store the result in the lower element of dst, and copy the upper element from a to the upper element of dst.\ @@ -34377,18 +32124,15 @@ pub unsafe fn _mm_maskz_add_round_ss(k: __mmask8, a: __m128, b: __m128, rounding /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_add_round_sd&expand=148) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vaddsd, rounding = 8))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm_add_round_sd(a: __m128d, b: __m128d, rounding: i32) -> __m128d { +#[cfg_attr(test, assert_instr(vaddsd, ROUNDING = 8))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm_add_round_sd(a: __m128d, b: __m128d) -> __m128d { + static_assert_rounding!(ROUNDING); let a = a.as_f64x2(); let b = b.as_f64x2(); let zero = _mm_setzero_pd().as_f64x2(); - macro_rules! call { - ($imm4:expr) => { - vaddsd(a, b, zero, 0b1, $imm4) - }; - } - transmute(constify_imm4_round!(rounding, call)) + let r = vaddsd(a, b, zero, 0b1, ROUNDING); + transmute(r) } /// Add the lower double-precision (64-bit) floating-point element in a and b, store the result in the lower element of dst using writemask k (the element is copied from src when mask bit 0 is not set), and copy the upper element from a to the upper element of dst.\ @@ -34403,24 +32147,20 @@ pub unsafe fn _mm_add_round_sd(a: __m128d, b: __m128d, rounding: i32) -> __m128d /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_mask_add_round_Sd&expand=149) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vaddsd, rounding = 8))] -#[rustc_args_required_const(4)] -pub unsafe fn _mm_mask_add_round_sd( +#[cfg_attr(test, assert_instr(vaddsd, ROUNDING = 8))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm_mask_add_round_sd( src: __m128d, k: __mmask8, a: __m128d, b: __m128d, - rounding: i32, ) -> __m128d { + static_assert_rounding!(ROUNDING); let a = a.as_f64x2(); let b = b.as_f64x2(); let src = src.as_f64x2(); - macro_rules! call { - ($imm4:expr) => { - vaddsd(a, b, src, k, $imm4) - }; - } - transmute(constify_imm4_round!(rounding, call)) + let r = vaddsd(a, b, src, k, ROUNDING); + transmute(r) } /// Add the lower double-precision (64-bit) floating-point element in a and b, store the result in the lower element of dst using zeromask k (the element is zeroed out when mask bit 0 is not set), and copy the upper element from a to the upper element of dst.\ @@ -34435,23 +32175,19 @@ pub unsafe fn _mm_mask_add_round_sd( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_add_round_sd&expand=150) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vaddsd, rounding = 8))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm_maskz_add_round_sd( +#[cfg_attr(test, assert_instr(vaddsd, ROUNDING = 8))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm_maskz_add_round_sd( k: __mmask8, a: __m128d, b: __m128d, - rounding: i32, ) -> __m128d { + static_assert_rounding!(ROUNDING); let a = a.as_f64x2(); let b = b.as_f64x2(); let zero = _mm_setzero_pd().as_f64x2(); - macro_rules! call { - ($imm4:expr) => { - vaddsd(a, b, zero, k, $imm4) - }; - } - transmute(constify_imm4_round!(rounding, call)) + let r = vaddsd(a, b, zero, k, ROUNDING); + transmute(r) } /// Subtract the lower single-precision (32-bit) floating-point element in b from the lower single-precision (32-bit) floating-point element in a, store the result in the lower element of dst, and copy the upper 3 packed elements from a to the upper elements of dst.\ @@ -34466,18 +32202,15 @@ pub unsafe fn _mm_maskz_add_round_sd( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_sub_round_ss&expand=5745) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vsubss, rounding = 8))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm_sub_round_ss(a: __m128, b: __m128, rounding: i32) -> __m128 { +#[cfg_attr(test, assert_instr(vsubss, ROUNDING = 8))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm_sub_round_ss(a: __m128, b: __m128) -> __m128 { + static_assert_rounding!(ROUNDING); let a = a.as_f32x4(); let b = b.as_f32x4(); let zero = _mm_setzero_ps().as_f32x4(); - macro_rules! call { - ($imm4:expr) => { - vsubss(a, b, zero, 0b1, $imm4) - }; - } - transmute(constify_imm4_round!(rounding, call)) + let r = vsubss(a, b, zero, 0b1, ROUNDING); + transmute(r) } /// Subtract the lower single-precision (32-bit) floating-point element in b from the lower single-precision (32-bit) floating-point element in a, store the result in the lower element of dst using writemask k (the element is copied from src when mask bit 0 is not set), and copy the upper 3 packed elements from a to the upper elements of dst.\ @@ -34492,24 +32225,20 @@ pub unsafe fn _mm_sub_round_ss(a: __m128, b: __m128, rounding: i32) -> __m128 { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_mask_sub_round_ss&expand=5743) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vsubss, rounding = 8))] -#[rustc_args_required_const(4)] -pub unsafe fn _mm_mask_sub_round_ss( +#[cfg_attr(test, assert_instr(vsubss, ROUNDING = 8))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm_mask_sub_round_ss( src: __m128, k: __mmask8, a: __m128, b: __m128, - rounding: i32, ) -> __m128 { + static_assert_rounding!(ROUNDING); let a = a.as_f32x4(); let b = b.as_f32x4(); let src = src.as_f32x4(); - macro_rules! call { - ($imm4:expr) => { - vsubss(a, b, src, k, $imm4) - }; - } - transmute(constify_imm4_round!(rounding, call)) + let r = vsubss(a, b, src, k, ROUNDING); + transmute(r) } /// Subtract the lower single-precision (32-bit) floating-point element in b from the lower single-precision (32-bit) floating-point element in a, store the result in the lower element of dst using zeromask k (the element is zeroed out when mask bit 0 is not set), and copy the upper 3 packed elements from a to the upper elements of dst.\ @@ -34524,18 +32253,19 @@ pub unsafe fn _mm_mask_sub_round_ss( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_maskz_sub_round_ss&expand=5744) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vsubss, rounding = 8))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm_maskz_sub_round_ss(k: __mmask8, a: __m128, b: __m128, rounding: i32) -> __m128 { +#[cfg_attr(test, assert_instr(vsubss, ROUNDING = 8))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm_maskz_sub_round_ss( + k: __mmask8, + a: __m128, + b: __m128, +) -> __m128 { + static_assert_rounding!(ROUNDING); let a = a.as_f32x4(); let b = b.as_f32x4(); let zero = _mm_setzero_ps().as_f32x4(); - macro_rules! call { - ($imm4:expr) => { - vsubss(a, b, zero, k, $imm4) - }; - } - transmute(constify_imm4_round!(rounding, call)) + let r = vsubss(a, b, zero, k, ROUNDING); + transmute(r) } /// Subtract the lower double-precision (64-bit) floating-point element in b from the lower double-precision (64-bit) floating-point element in a, store the result in the lower element of dst, and copy the upper element from a to the upper element of dst.\ @@ -34550,18 +32280,15 @@ pub unsafe fn _mm_maskz_sub_round_ss(k: __mmask8, a: __m128, b: __m128, rounding /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_sub_round_sd&expand=5742) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vsubsd, rounding = 8))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm_sub_round_sd(a: __m128d, b: __m128d, rounding: i32) -> __m128d { +#[cfg_attr(test, assert_instr(vsubsd, ROUNDING = 8))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm_sub_round_sd(a: __m128d, b: __m128d) -> __m128d { + static_assert_rounding!(ROUNDING); let a = a.as_f64x2(); let b = b.as_f64x2(); let zero = _mm_setzero_pd().as_f64x2(); - macro_rules! call { - ($imm4:expr) => { - vsubsd(a, b, zero, 0b1, $imm4) - }; - } - transmute(constify_imm4_round!(rounding, call)) + let r = vsubsd(a, b, zero, 0b1, ROUNDING); + transmute(r) } /// Subtract the lower double-precision (64-bit) floating-point element in b from the lower double-precision (64-bit) floating-point element in a, store the result in the lower element of dst using writemask k (the element is copied from src when mask bit 0 is not set), and copy the upper element from a to the upper element of dst.\ @@ -34576,24 +32303,20 @@ pub unsafe fn _mm_sub_round_sd(a: __m128d, b: __m128d, rounding: i32) -> __m128d /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_mask_sub_round_sd&expand=5740) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vsubsd, rounding = 8))] -#[rustc_args_required_const(4)] -pub unsafe fn _mm_mask_sub_round_sd( +#[cfg_attr(test, assert_instr(vsubsd, ROUNDING = 8))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm_mask_sub_round_sd( src: __m128d, k: __mmask8, a: __m128d, b: __m128d, - rounding: i32, ) -> __m128d { + static_assert_rounding!(ROUNDING); let a = a.as_f64x2(); let b = b.as_f64x2(); let src = src.as_f64x2(); - macro_rules! call { - ($imm4:expr) => { - vsubsd(a, b, src, k, $imm4) - }; - } - transmute(constify_imm4_round!(rounding, call)) + let r = vsubsd(a, b, src, k, ROUNDING); + transmute(r) } /// Subtract the lower double-precision (64-bit) floating-point element in b from the lower double-precision (64-bit) floating-point element in a, store the result in the lower element of dst using zeromask k (the element is zeroed out when mask bit 0 is not set), and copy the upper element from a to the upper element of dst.\ @@ -34608,23 +32331,19 @@ pub unsafe fn _mm_mask_sub_round_sd( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_maskz_sub_round_sd&expand=5741) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vsubsd, rounding = 8))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm_maskz_sub_round_sd( +#[cfg_attr(test, assert_instr(vsubsd, ROUNDING = 8))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm_maskz_sub_round_sd( k: __mmask8, a: __m128d, b: __m128d, - rounding: i32, ) -> __m128d { + static_assert_rounding!(ROUNDING); let a = a.as_f64x2(); let b = b.as_f64x2(); let zero = _mm_setzero_pd().as_f64x2(); - macro_rules! call { - ($imm4:expr) => { - vsubsd(a, b, zero, k, $imm4) - }; - } - transmute(constify_imm4_round!(rounding, call)) + let r = vsubsd(a, b, zero, k, ROUNDING); + transmute(r) } /// Multiply the lower single-precision (32-bit) floating-point element in a and b, store the result in the lower element of dst, and copy the upper 3 packed elements from a to the upper elements of dst.\ @@ -34639,18 +32358,15 @@ pub unsafe fn _mm_maskz_sub_round_sd( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_mul_round_ss&expand=3946) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vmulss, rounding = 8))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm_mul_round_ss(a: __m128, b: __m128, rounding: i32) -> __m128 { +#[cfg_attr(test, assert_instr(vmulss, ROUNDING = 8))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm_mul_round_ss(a: __m128, b: __m128) -> __m128 { + static_assert_rounding!(ROUNDING); let a = a.as_f32x4(); let b = b.as_f32x4(); let zero = _mm_setzero_ps().as_f32x4(); - macro_rules! call { - ($imm4:expr) => { - vmulss(a, b, zero, 0b1, $imm4) - }; - } - transmute(constify_imm4_round!(rounding, call)) + let r = vmulss(a, b, zero, 0b1, ROUNDING); + transmute(r) } /// Multiply the lower single-precision (32-bit) floating-point element in a and b, store the result in the lower element of dst using writemask k (the element is copied from src when mask bit 0 is not set), and copy the upper 3 packed elements from a to the upper elements of dst.\ @@ -34665,24 +32381,20 @@ pub unsafe fn _mm_mul_round_ss(a: __m128, b: __m128, rounding: i32) -> __m128 { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_mask_mul_round_ss&expand=3944) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vmulss, rounding = 8))] -#[rustc_args_required_const(4)] -pub unsafe fn _mm_mask_mul_round_ss( +#[cfg_attr(test, assert_instr(vmulss, ROUNDING = 8))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm_mask_mul_round_ss( src: __m128, k: __mmask8, a: __m128, b: __m128, - rounding: i32, ) -> __m128 { + static_assert_rounding!(ROUNDING); let a = a.as_f32x4(); let b = b.as_f32x4(); let src = src.as_f32x4(); - macro_rules! call { - ($imm4:expr) => { - vmulss(a, b, src, k, $imm4) - }; - } - transmute(constify_imm4_round!(rounding, call)) + let r = vmulss(a, b, src, k, ROUNDING); + transmute(r) } /// Multiply the lower single-precision (32-bit) floating-point element in a and b, store the result in the lower element of dst using zeromask k (the element is zeroed out when mask bit 0 is not set), and copy the upper 3 packed elements from a to the upper elements of dst.\ @@ -34697,18 +32409,19 @@ pub unsafe fn _mm_mask_mul_round_ss( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_maskz_mul_round_ss&expand=3945) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vmulss, rounding = 8))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm_maskz_mul_round_ss(k: __mmask8, a: __m128, b: __m128, rounding: i32) -> __m128 { +#[cfg_attr(test, assert_instr(vmulss, ROUNDING = 8))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm_maskz_mul_round_ss( + k: __mmask8, + a: __m128, + b: __m128, +) -> __m128 { + static_assert_rounding!(ROUNDING); let a = a.as_f32x4(); let b = b.as_f32x4(); let zero = _mm_setzero_ps().as_f32x4(); - macro_rules! call { - ($imm4:expr) => { - vmulss(a, b, zero, k, $imm4) - }; - } - transmute(constify_imm4_round!(rounding, call)) + let r = vmulss(a, b, zero, k, ROUNDING); + transmute(r) } /// Multiply the lower double-precision (64-bit) floating-point element in a and b, store the result in the lower element of dst, and copy the upper element from a to the upper element of dst.\ @@ -34723,18 +32436,15 @@ pub unsafe fn _mm_maskz_mul_round_ss(k: __mmask8, a: __m128, b: __m128, rounding /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_mul_round_sd&expand=3943) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vmulsd, rounding = 8))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm_mul_round_sd(a: __m128d, b: __m128d, rounding: i32) -> __m128d { +#[cfg_attr(test, assert_instr(vmulsd, ROUNDING = 8))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm_mul_round_sd(a: __m128d, b: __m128d) -> __m128d { + static_assert_rounding!(ROUNDING); let a = a.as_f64x2(); let b = b.as_f64x2(); let zero = _mm_setzero_pd().as_f64x2(); - macro_rules! call { - ($imm4:expr) => { - vmulsd(a, b, zero, 0b1, $imm4) - }; - } - transmute(constify_imm4_round!(rounding, call)) + let r = vmulsd(a, b, zero, 0b1, ROUNDING); + transmute(r) } /// Multiply the lower double-precision (64-bit) floating-point element in a and b, store the result in the lower element of dst using writemask k (the element is copied from src when mask bit 0 is not set), and copy the upper element from a to the upper element of dst.\ @@ -34749,24 +32459,20 @@ pub unsafe fn _mm_mul_round_sd(a: __m128d, b: __m128d, rounding: i32) -> __m128d /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_mask_mul_round_sd&expand=3941) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vmulsd, rounding = 8))] -#[rustc_args_required_const(4)] -pub unsafe fn _mm_mask_mul_round_sd( +#[cfg_attr(test, assert_instr(vmulsd, ROUNDING = 8))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm_mask_mul_round_sd( src: __m128d, k: __mmask8, a: __m128d, b: __m128d, - rounding: i32, ) -> __m128d { + static_assert_rounding!(ROUNDING); let a = a.as_f64x2(); let b = b.as_f64x2(); let src = src.as_f64x2(); - macro_rules! call { - ($imm4:expr) => { - vmulsd(a, b, src, k, $imm4) - }; - } - transmute(constify_imm4_round!(rounding, call)) + let r = vmulsd(a, b, src, k, ROUNDING); + transmute(r) } /// Multiply the lower double-precision (64-bit) floating-point element in a and b, store the result in the lower element of dst using zeromask k (the element is zeroed out when mask bit 0 is not set), and copy the upper element from a to the upper element of dst.\ @@ -34781,23 +32487,19 @@ pub unsafe fn _mm_mask_mul_round_sd( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_maskz_mul_round_sd&expand=3942) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vmulsd, rounding = 8))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm_maskz_mul_round_sd( +#[cfg_attr(test, assert_instr(vmulsd, ROUNDING = 8))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm_maskz_mul_round_sd( k: __mmask8, a: __m128d, b: __m128d, - rounding: i32, ) -> __m128d { + static_assert_rounding!(ROUNDING); let a = a.as_f64x2(); let b = b.as_f64x2(); let zero = _mm_setzero_pd().as_f64x2(); - macro_rules! call { - ($imm4:expr) => { - vmulsd(a, b, zero, k, $imm4) - }; - } - transmute(constify_imm4_round!(rounding, call)) + let r = vmulsd(a, b, zero, k, ROUNDING); + transmute(r) } /// Divide the lower single-precision (32-bit) floating-point element in a by the lower single-precision (32-bit) floating-point element in b, store the result in the lower element of dst, and copy the upper 3 packed elements from a to the upper elements of dst.\ @@ -34812,18 +32514,15 @@ pub unsafe fn _mm_maskz_mul_round_sd( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_div_round_ss&expand=2174) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vdivss, rounding = 8))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm_div_round_ss(a: __m128, b: __m128, rounding: i32) -> __m128 { +#[cfg_attr(test, assert_instr(vdivss, ROUNDING = 8))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm_div_round_ss(a: __m128, b: __m128) -> __m128 { + static_assert_rounding!(ROUNDING); let a = a.as_f32x4(); let b = b.as_f32x4(); let zero = _mm_setzero_ps().as_f32x4(); - macro_rules! call { - ($imm4:expr) => { - vdivss(a, b, zero, 0b1, $imm4) - }; - } - transmute(constify_imm4_round!(rounding, call)) + let r = vdivss(a, b, zero, 0b1, ROUNDING); + transmute(r) } /// Divide the lower single-precision (32-bit) floating-point element in a by the lower single-precision (32-bit) floating-point element in b, store the result in the lower element of dst using writemask k (the element is copied from src when mask bit 0 is not set), and copy the upper 3 packed elements from a to the upper elements of dst.\ @@ -34838,24 +32537,20 @@ pub unsafe fn _mm_div_round_ss(a: __m128, b: __m128, rounding: i32) -> __m128 { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_mask_div_round_ss&expand=2175) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vdivss, rounding = 8))] -#[rustc_args_required_const(4)] -pub unsafe fn _mm_mask_div_round_ss( +#[cfg_attr(test, assert_instr(vdivss, ROUNDING = 8))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm_mask_div_round_ss( src: __m128, k: __mmask8, a: __m128, b: __m128, - rounding: i32, ) -> __m128 { + static_assert_rounding!(ROUNDING); let a = a.as_f32x4(); let b = b.as_f32x4(); let src = src.as_f32x4(); - macro_rules! call { - ($imm4:expr) => { - vdivss(a, b, src, k, $imm4) - }; - } - transmute(constify_imm4_round!(rounding, call)) + let r = vdivss(a, b, src, k, ROUNDING); + transmute(r) } /// Divide the lower single-precision (32-bit) floating-point element in a by the lower single-precision (32-bit) floating-point element in b, store the result in the lower element of dst using zeromask k (the element is zeroed out when mask bit 0 is not set), and copy the upper 3 packed elements from a to the upper elements of dst.\ @@ -34870,18 +32565,19 @@ pub unsafe fn _mm_mask_div_round_ss( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_maskz_div_round_ss&expand=2176) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vdivss, rounding = 8))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm_maskz_div_round_ss(k: __mmask8, a: __m128, b: __m128, rounding: i32) -> __m128 { +#[cfg_attr(test, assert_instr(vdivss, ROUNDING = 8))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm_maskz_div_round_ss( + k: __mmask8, + a: __m128, + b: __m128, +) -> __m128 { + static_assert_rounding!(ROUNDING); let a = a.as_f32x4(); let b = b.as_f32x4(); let zero = _mm_setzero_ps().as_f32x4(); - macro_rules! call { - ($imm4:expr) => { - vdivss(a, b, zero, k, $imm4) - }; - } - transmute(constify_imm4_round!(rounding, call)) + let r = vdivss(a, b, zero, k, ROUNDING); + transmute(r) } /// Divide the lower double-precision (64-bit) floating-point element in a by the lower double-precision (64-bit) floating-point element in b, store the result in the lower element of dst, and copy the upper element from a to the upper element of dst.\ @@ -34896,18 +32592,15 @@ pub unsafe fn _mm_maskz_div_round_ss(k: __mmask8, a: __m128, b: __m128, rounding /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_div_round_sd&expand=2171) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vdivsd, rounding = 8))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm_div_round_sd(a: __m128d, b: __m128d, rounding: i32) -> __m128d { +#[cfg_attr(test, assert_instr(vdivsd, ROUNDING = 8))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm_div_round_sd(a: __m128d, b: __m128d) -> __m128d { + static_assert_rounding!(ROUNDING); let a = a.as_f64x2(); let b = b.as_f64x2(); let zero = _mm_setzero_pd().as_f64x2(); - macro_rules! call { - ($imm4:expr) => { - vdivsd(a, b, zero, 0b1, $imm4) - }; - } - transmute(constify_imm4_round!(rounding, call)) + let r = vdivsd(a, b, zero, 0b1, ROUNDING); + transmute(r) } /// Divide the lower double-precision (64-bit) floating-point element in a by the lower double-precision (64-bit) floating-point element in b, store the result in the lower element of dst using writemask k (the element is copied from src when mask bit 0 is not set), and copy the upper element from a to the upper element of dst.\ @@ -34922,24 +32615,20 @@ pub unsafe fn _mm_div_round_sd(a: __m128d, b: __m128d, rounding: i32) -> __m128d /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_mask_div_round_sd&expand=2172) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vdivsd, rounding = 8))] -#[rustc_args_required_const(4)] -pub unsafe fn _mm_mask_div_round_sd( +#[cfg_attr(test, assert_instr(vdivsd, ROUNDING = 8))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm_mask_div_round_sd( src: __m128d, k: __mmask8, a: __m128d, b: __m128d, - rounding: i32, ) -> __m128d { + static_assert_rounding!(ROUNDING); let a = a.as_f64x2(); let b = b.as_f64x2(); let src = src.as_f64x2(); - macro_rules! call { - ($imm4:expr) => { - vdivsd(a, b, src, k, $imm4) - }; - } - transmute(constify_imm4_round!(rounding, call)) + let r = vdivsd(a, b, src, k, ROUNDING); + transmute(r) } /// Divide the lower double-precision (64-bit) floating-point element in a by the lower double-precision (64-bit) floating-point element in b, store the result in the lower element of dst using zeromask k (the element is zeroed out when mask bit 0 is not set), and copy the upper element from a to the upper element of dst.\ @@ -34954,23 +32643,19 @@ pub unsafe fn _mm_mask_div_round_sd( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_maskz_div_round_sd&expand=2173) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vdivsd, rounding = 8))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm_maskz_div_round_sd( +#[cfg_attr(test, assert_instr(vdivsd, ROUNDING = 8))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm_maskz_div_round_sd( k: __mmask8, a: __m128d, b: __m128d, - rounding: i32, ) -> __m128d { + static_assert_rounding!(ROUNDING); let a = a.as_f64x2(); let b = b.as_f64x2(); let zero = _mm_setzero_pd().as_f64x2(); - macro_rules! call { - ($imm4:expr) => { - vdivsd(a, b, zero, k, $imm4) - }; - } - transmute(constify_imm4_round!(rounding, call)) + let r = vdivsd(a, b, zero, k, ROUNDING); + transmute(r) } /// Compare the lower single-precision (32-bit) floating-point elements in a and b, store the maximum value in the lower element of dst, and copy the upper 3 packed elements from a to the upper elements of dst.\ @@ -34979,18 +32664,15 @@ pub unsafe fn _mm_maskz_div_round_sd( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_max_round_ss&expand=3668) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vmaxss, sae = 8))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm_max_round_ss(a: __m128, b: __m128, sae: i32) -> __m128 { +#[cfg_attr(test, assert_instr(vmaxss, SAE = 8))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm_max_round_ss(a: __m128, b: __m128) -> __m128 { + static_assert_sae!(SAE); let a = a.as_f32x4(); let b = b.as_f32x4(); let zero = _mm_setzero_ps().as_f32x4(); - macro_rules! call { - ($imm4:expr) => { - vmaxss(a, b, zero, 0b1, $imm4) - }; - } - transmute(constify_imm4_sae!(sae, call)) + let r = vmaxss(a, b, zero, 0b1, SAE); + transmute(r) } /// Compare the lower single-precision (32-bit) floating-point elements in a and b, store the maximum value in the lower element of dst using writemask k (the element is copied from src when mask bit 0 is not set), and copy the upper 3 packed elements from a to the upper elements of dst.\ @@ -34999,24 +32681,20 @@ pub unsafe fn _mm_max_round_ss(a: __m128, b: __m128, sae: i32) -> __m128 { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_mask_max_ss&expand=3672) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vmaxss, sae = 8))] -#[rustc_args_required_const(4)] -pub unsafe fn _mm_mask_max_round_ss( +#[cfg_attr(test, assert_instr(vmaxss, SAE = 8))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm_mask_max_round_ss( src: __m128, k: __mmask8, a: __m128, b: __m128, - sae: i32, ) -> __m128 { + static_assert_sae!(SAE); let a = a.as_f32x4(); let b = b.as_f32x4(); let src = src.as_f32x4(); - macro_rules! call { - ($imm4:expr) => { - vmaxss(a, b, src, k, $imm4) - }; - } - transmute(constify_imm4_sae!(sae, call)) + let r = vmaxss(a, b, src, k, SAE); + transmute(r) } /// Compare the lower single-precision (32-bit) floating-point elements in a and b, store the maximum value in the lower element of dst using zeromask k (the element is zeroed out when mask bit 0 is not set), and copy the upper 3 packed elements from a to the upper elements of dst.\ @@ -35025,18 +32703,15 @@ pub unsafe fn _mm_mask_max_round_ss( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_maskz_max_round_ss&expand=3667) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vmaxss, sae = 8))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm_maskz_max_round_ss(k: __mmask8, a: __m128, b: __m128, sae: i32) -> __m128 { +#[cfg_attr(test, assert_instr(vmaxss, SAE = 8))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm_maskz_max_round_ss(k: __mmask8, a: __m128, b: __m128) -> __m128 { + static_assert_sae!(SAE); let a = a.as_f32x4(); let b = b.as_f32x4(); let zero = _mm_setzero_ps().as_f32x4(); - macro_rules! call { - ($imm4:expr) => { - vmaxss(a, b, zero, k, $imm4) - }; - } - transmute(constify_imm4_sae!(sae, call)) + let r = vmaxss(a, b, zero, k, SAE); + transmute(r) } /// Compare the lower double-precision (64-bit) floating-point elements in a and b, store the maximum value in the lower element of dst, and copy the upper element from a to the upper element of dst.\ @@ -35045,18 +32720,15 @@ pub unsafe fn _mm_maskz_max_round_ss(k: __mmask8, a: __m128, b: __m128, sae: i32 /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_max_round_sd&expand=3665) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vmaxsd, sae = 8))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm_max_round_sd(a: __m128d, b: __m128d, sae: i32) -> __m128d { +#[cfg_attr(test, assert_instr(vmaxsd, SAE = 8))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm_max_round_sd(a: __m128d, b: __m128d) -> __m128d { + static_assert_sae!(SAE); let a = a.as_f64x2(); let b = b.as_f64x2(); let zero = _mm_setzero_pd().as_f64x2(); - macro_rules! call { - ($imm4:expr) => { - vmaxsd(a, b, zero, 0b1, $imm4) - }; - } - transmute(constify_imm4_sae!(sae, call)) + let r = vmaxsd(a, b, zero, 0b1, SAE); + transmute(r) } /// Compare the lower double-precision (64-bit) floating-point elements in a and b, store the maximum value in the lower element of dst using writemask k (the element is copied from src when mask bit 0 is not set), and copy the upper element from a to the upper element of dst.\ @@ -35065,24 +32737,20 @@ pub unsafe fn _mm_max_round_sd(a: __m128d, b: __m128d, sae: i32) -> __m128d { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_mask_max_round_sd&expand=3663) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vmaxsd, sae = 8))] -#[rustc_args_required_const(4)] -pub unsafe fn _mm_mask_max_round_sd( +#[cfg_attr(test, assert_instr(vmaxsd, SAE = 8))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm_mask_max_round_sd( src: __m128d, k: __mmask8, a: __m128d, b: __m128d, - sae: i32, ) -> __m128d { + static_assert_sae!(SAE); let a = a.as_f64x2(); let b = b.as_f64x2(); let src = src.as_f64x2(); - macro_rules! call { - ($imm4:expr) => { - vmaxsd(a, b, src, k, $imm4) - }; - } - transmute(constify_imm4_sae!(sae, call)) + let r = vmaxsd(a, b, src, k, SAE); + transmute(r) } /// Compare the lower double-precision (64-bit) floating-point elements in a and b, store the maximum value in the lower element of dst using zeromask k (the element is zeroed out when mask bit 0 is not set), and copy the upper element from a to the upper element of dst.\ @@ -35091,18 +32759,19 @@ pub unsafe fn _mm_mask_max_round_sd( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_maskz_max_sd&expand=3670) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vmaxsd, sae = 8))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm_maskz_max_round_sd(k: __mmask8, a: __m128d, b: __m128d, sae: i32) -> __m128d { +#[cfg_attr(test, assert_instr(vmaxsd, SAE = 8))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm_maskz_max_round_sd( + k: __mmask8, + a: __m128d, + b: __m128d, +) -> __m128d { + static_assert_sae!(SAE); let a = a.as_f64x2(); let b = b.as_f64x2(); let zero = _mm_setzero_pd().as_f64x2(); - macro_rules! call { - ($imm4:expr) => { - vmaxsd(a, b, zero, k, $imm4) - }; - } - transmute(constify_imm4_sae!(sae, call)) + let r = vmaxsd(a, b, zero, k, SAE); + transmute(r) } /// Compare the lower single-precision (32-bit) floating-point elements in a and b, store the minimum value in the lower element of dst, and copy the upper 3 packed elements from a to the upper elements of dst.\ @@ -35111,18 +32780,15 @@ pub unsafe fn _mm_maskz_max_round_sd(k: __mmask8, a: __m128d, b: __m128d, sae: i /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_min_round_ss&expand=3782) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vminss, sae = 8))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm_min_round_ss(a: __m128, b: __m128, sae: i32) -> __m128 { +#[cfg_attr(test, assert_instr(vminss, SAE = 8))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm_min_round_ss(a: __m128, b: __m128) -> __m128 { + static_assert_sae!(SAE); let a = a.as_f32x4(); let b = b.as_f32x4(); let zero = _mm_setzero_ps().as_f32x4(); - macro_rules! call { - ($imm4:expr) => { - vminss(a, b, zero, 0b1, $imm4) - }; - } - transmute(constify_imm4_sae!(sae, call)) + let r = vminss(a, b, zero, 0b1, SAE); + transmute(r) } /// Compare the lower single-precision (32-bit) floating-point elements in a and b, store the minimum value in the lower element of dst using writemask k (the element is copied from src when mask bit 0 is not set), and copy the upper 3 packed elements from a to the upper elements of dst.\ @@ -35131,24 +32797,20 @@ pub unsafe fn _mm_min_round_ss(a: __m128, b: __m128, sae: i32) -> __m128 { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_mask_min_round_Ss&expand=3780) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vminss, sae = 8))] -#[rustc_args_required_const(4)] -pub unsafe fn _mm_mask_min_round_ss( +#[cfg_attr(test, assert_instr(vminss, SAE = 8))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm_mask_min_round_ss( src: __m128, k: __mmask8, a: __m128, b: __m128, - sae: i32, ) -> __m128 { + static_assert_sae!(SAE); let a = a.as_f32x4(); let b = b.as_f32x4(); let src = src.as_f32x4(); - macro_rules! call { - ($imm4:expr) => { - vminss(a, b, src, k, $imm4) - }; - } - transmute(constify_imm4_sae!(sae, call)) + let r = vminss(a, b, src, k, SAE); + transmute(r) } /// Compare the lower single-precision (32-bit) floating-point elements in a and b, store the minimum value in the lower element of dst using zeromask k (the element is zeroed out when mask bit 0 is not set), and copy the upper 3 packed elements from a to the upper elements of dst.\ @@ -35157,18 +32819,15 @@ pub unsafe fn _mm_mask_min_round_ss( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_maskz_min_round_ss&expand=3781) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vminss, sae = 8))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm_maskz_min_round_ss(k: __mmask8, a: __m128, b: __m128, sae: i32) -> __m128 { +#[cfg_attr(test, assert_instr(vminss, SAE = 8))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm_maskz_min_round_ss(k: __mmask8, a: __m128, b: __m128) -> __m128 { + static_assert_sae!(SAE); let a = a.as_f32x4(); let b = b.as_f32x4(); let zero = _mm_setzero_ps().as_f32x4(); - macro_rules! call { - ($imm4:expr) => { - vminss(a, b, zero, k, $imm4) - }; - } - transmute(constify_imm4_sae!(sae, call)) + let r = vminss(a, b, zero, k, SAE); + transmute(r) } /// Compare the lower double-precision (64-bit) floating-point elements in a and b, store the minimum value in the lower element of dst , and copy the upper element from a to the upper element of dst.\ @@ -35177,18 +32836,15 @@ pub unsafe fn _mm_maskz_min_round_ss(k: __mmask8, a: __m128, b: __m128, sae: i32 /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_min_round_sd&expand=3779) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vminsd, sae = 8))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm_min_round_sd(a: __m128d, b: __m128d, sae: i32) -> __m128d { +#[cfg_attr(test, assert_instr(vminsd, SAE = 8))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm_min_round_sd(a: __m128d, b: __m128d) -> __m128d { + static_assert_sae!(SAE); let a = a.as_f64x2(); let b = b.as_f64x2(); let zero = _mm_setzero_pd().as_f64x2(); - macro_rules! call { - ($imm4:expr) => { - vminsd(a, b, zero, 0b1, $imm4) - }; - } - transmute(constify_imm4_sae!(sae, call)) + let r = vminsd(a, b, zero, 0b1, SAE); + transmute(r) } /// Compare the lower double-precision (64-bit) floating-point elements in a and b, store the minimum value in the lower element of dst using writemask k (the element is copied from src when mask bit 0 is not set), and copy the upper element from a to the upper element of dst.\ @@ -35197,24 +32853,20 @@ pub unsafe fn _mm_min_round_sd(a: __m128d, b: __m128d, sae: i32) -> __m128d { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_mask_min_round_sd&expand=3777) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vminsd, sae = 8))] -#[rustc_args_required_const(4)] -pub unsafe fn _mm_mask_min_round_sd( +#[cfg_attr(test, assert_instr(vminsd, SAE = 8))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm_mask_min_round_sd( src: __m128d, k: __mmask8, a: __m128d, b: __m128d, - sae: i32, ) -> __m128d { + static_assert_sae!(SAE); let a = a.as_f64x2(); let b = b.as_f64x2(); let src = src.as_f64x2(); - macro_rules! call { - ($imm4:expr) => { - vminsd(a, b, src, k, $imm4) - }; - } - transmute(constify_imm4_sae!(sae, call)) + let r = vminsd(a, b, src, k, SAE); + transmute(r) } /// Compare the lower double-precision (64-bit) floating-point elements in a and b, store the minimum value in the lower element of dst using zeromask k (the element is zeroed out when mask bit 0 is not set), and copy the upper element from a to the upper element of dst.\ @@ -35223,18 +32875,19 @@ pub unsafe fn _mm_mask_min_round_sd( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_maskz_min_round_Sd&expand=3778) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vminsd, sae = 8))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm_maskz_min_round_sd(k: __mmask8, a: __m128d, b: __m128d, sae: i32) -> __m128d { +#[cfg_attr(test, assert_instr(vminsd, SAE = 8))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm_maskz_min_round_sd( + k: __mmask8, + a: __m128d, + b: __m128d, +) -> __m128d { + static_assert_sae!(SAE); let a = a.as_f64x2(); let b = b.as_f64x2(); let zero = _mm_setzero_pd().as_f64x2(); - macro_rules! call { - ($imm4:expr) => { - vminsd(a, b, zero, k, $imm4) - }; - } - transmute(constify_imm4_sae!(sae, call)) + let r = vminsd(a, b, zero, k, SAE); + transmute(r) } /// Compute the square root of the lower single-precision (32-bit) floating-point element in b, store the result in the lower element of dst, and copy the upper 3 packed elements from a to the upper elements of dst.\ @@ -35249,18 +32902,15 @@ pub unsafe fn _mm_maskz_min_round_sd(k: __mmask8, a: __m128d, b: __m128d, sae: i /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_sqrt_round_ss&expand=5383) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vsqrtss, rounding = 8))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm_sqrt_round_ss(a: __m128, b: __m128, rounding: i32) -> __m128 { +#[cfg_attr(test, assert_instr(vsqrtss, ROUNDING = 8))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm_sqrt_round_ss(a: __m128, b: __m128) -> __m128 { + static_assert_rounding!(ROUNDING); let a = a.as_f32x4(); let b = b.as_f32x4(); let zero = _mm_setzero_ps().as_f32x4(); - macro_rules! call { - ($imm4:expr) => { - vsqrtss(a, b, zero, 0b1, $imm4) - }; - } - transmute(constify_imm4_round!(rounding, call)) + let r = vsqrtss(a, b, zero, 0b1, ROUNDING); + transmute(r) } /// Compute the square root of the lower single-precision (32-bit) floating-point element in b, store the result in the lower element of dst using writemask k (the element is copied from src when mask bit 0 is not set), and copy the upper 3 packed elements from a to the upper elements of dst.\ @@ -35275,24 +32925,20 @@ pub unsafe fn _mm_sqrt_round_ss(a: __m128, b: __m128, rounding: i32) -> __m128 { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_mask_sqrt_round_ss&expand=5381) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vsqrtss, rounding = 8))] -#[rustc_args_required_const(4)] -pub unsafe fn _mm_mask_sqrt_round_ss( +#[cfg_attr(test, assert_instr(vsqrtss, ROUNDING = 8))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm_mask_sqrt_round_ss( src: __m128, k: __mmask8, a: __m128, b: __m128, - rounding: i32, ) -> __m128 { + static_assert_rounding!(ROUNDING); let a = a.as_f32x4(); let b = b.as_f32x4(); let src = src.as_f32x4(); - macro_rules! call { - ($imm4:expr) => { - vsqrtss(a, b, src, k, $imm4) - }; - } - transmute(constify_imm4_round!(rounding, call)) + let r = vsqrtss(a, b, src, k, ROUNDING); + transmute(r) } /// Compute the square root of the lower single-precision (32-bit) floating-point element in b, store the result in the lower element of dst using zeromask k (the element is zeroed out when mask bit 0 is not set), and copy the upper 3 packed elements from a to the upper elements of dst.\ @@ -35307,18 +32953,19 @@ pub unsafe fn _mm_mask_sqrt_round_ss( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_maskz_sqrt_round_ss&expand=5382) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vsqrtss, rounding = 8))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm_maskz_sqrt_round_ss(k: __mmask8, a: __m128, b: __m128, rounding: i32) -> __m128 { +#[cfg_attr(test, assert_instr(vsqrtss, ROUNDING = 8))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm_maskz_sqrt_round_ss( + k: __mmask8, + a: __m128, + b: __m128, +) -> __m128 { + static_assert_rounding!(ROUNDING); let a = a.as_f32x4(); let b = b.as_f32x4(); let zero = _mm_setzero_ps().as_f32x4(); - macro_rules! call { - ($imm4:expr) => { - vsqrtss(a, b, zero, k, $imm4) - }; - } - transmute(constify_imm4_round!(rounding, call)) + let r = vsqrtss(a, b, zero, k, ROUNDING); + transmute(r) } /// Compute the square root of the lower double-precision (64-bit) floating-point element in b, store the result in the lower element of dst, and copy the upper element from a to the upper element of dst.\ @@ -35333,18 +32980,15 @@ pub unsafe fn _mm_maskz_sqrt_round_ss(k: __mmask8, a: __m128, b: __m128, roundin /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_sqrt_round_sd&expand=5380) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vsqrtsd, rounding = 8))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm_sqrt_round_sd(a: __m128d, b: __m128d, rounding: i32) -> __m128d { +#[cfg_attr(test, assert_instr(vsqrtsd, ROUNDING = 8))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm_sqrt_round_sd(a: __m128d, b: __m128d) -> __m128d { + static_assert_rounding!(ROUNDING); let a = a.as_f64x2(); let b = b.as_f64x2(); let zero = _mm_setzero_pd().as_f64x2(); - macro_rules! call { - ($imm4:expr) => { - vsqrtsd(a, b, zero, 0b1, $imm4) - }; - } - transmute(constify_imm4_round!(rounding, call)) + let r = vsqrtsd(a, b, zero, 0b1, ROUNDING); + transmute(r) } /// Compute the square root of the lower double-precision (64-bit) floating-point element in b, store the result in the lower element of dst using writemask k (the element is copied from src when mask bit 0 is not set), and copy the upper element from a to the upper element of dst.\ @@ -35359,24 +33003,20 @@ pub unsafe fn _mm_sqrt_round_sd(a: __m128d, b: __m128d, rounding: i32) -> __m128 /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_mask_sqrt_round_sd&expand=5378) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vsqrtsd, rounding = 8))] -#[rustc_args_required_const(4)] -pub unsafe fn _mm_mask_sqrt_round_sd( +#[cfg_attr(test, assert_instr(vsqrtsd, ROUNDING = 8))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm_mask_sqrt_round_sd( src: __m128d, k: __mmask8, a: __m128d, b: __m128d, - rounding: i32, ) -> __m128d { + static_assert_rounding!(ROUNDING); let a = a.as_f64x2(); let b = b.as_f64x2(); let src = src.as_f64x2(); - macro_rules! call { - ($imm4:expr) => { - vsqrtsd(a, b, src, k, $imm4) - }; - } - transmute(constify_imm4_round!(rounding, call)) + let r = vsqrtsd(a, b, src, k, ROUNDING); + transmute(r) } /// Compute the square root of the lower double-precision (64-bit) floating-point element in b, store the result in the lower element of dst using zeromask k (the element is zeroed out when mask bit 0 is not set), and copy the upper element from a to the upper element of dst.\ @@ -35391,23 +33031,19 @@ pub unsafe fn _mm_mask_sqrt_round_sd( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_maskz_sqrt_round_sd&expand=5379) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vsqrtsd, rounding = 8))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm_maskz_sqrt_round_sd( +#[cfg_attr(test, assert_instr(vsqrtsd, ROUNDING = 8))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm_maskz_sqrt_round_sd( k: __mmask8, a: __m128d, b: __m128d, - rounding: i32, ) -> __m128d { + static_assert_rounding!(ROUNDING); let a = a.as_f64x2(); let b = b.as_f64x2(); let zero = _mm_setzero_pd().as_f64x2(); - macro_rules! call { - ($imm4:expr) => { - vsqrtsd(a, b, zero, k, $imm4) - }; - } - transmute(constify_imm4_round!(rounding, call)) + let r = vsqrtsd(a, b, zero, k, ROUNDING); + transmute(r) } /// Convert the exponent of the lower single-precision (32-bit) floating-point element in b to a single-precision (32-bit) floating-point number representing the integer exponent, store the result in the lower element of dst, and copy the upper 3 packed elements from a to the upper elements of dst. This intrinsic essentially calculates floor(log2(x)) for the lower element.\ @@ -35416,18 +33052,14 @@ pub unsafe fn _mm_maskz_sqrt_round_sd( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_getexp_round_ss&expand=2856) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vgetexpss, sae = 8))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm_getexp_round_ss(a: __m128, b: __m128, sae: i32) -> __m128 { +#[cfg_attr(test, assert_instr(vgetexpss, SAE = 8))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm_getexp_round_ss(a: __m128, b: __m128) -> __m128 { + static_assert_sae!(SAE); let a = a.as_f32x4(); let b = b.as_f32x4(); let zero = _mm_setzero_ps().as_f32x4(); - macro_rules! call { - ($imm4:expr) => { - vgetexpss(a, b, zero, 0b1, $imm4) - }; - } - let r = constify_imm4_sae!(sae, call); + let r = vgetexpss(a, b, zero, 0b1, SAE); transmute(r) } @@ -35437,24 +33069,19 @@ pub unsafe fn _mm_getexp_round_ss(a: __m128, b: __m128, sae: i32) -> __m128 { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_mask_getexp_round_ss&expand=2857) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vgetexpss, sae = 8))] -#[rustc_args_required_const(4)] -pub unsafe fn _mm_mask_getexp_round_ss( +#[cfg_attr(test, assert_instr(vgetexpss, SAE = 8))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm_mask_getexp_round_ss( src: __m128, k: __mmask8, a: __m128, b: __m128, - sae: i32, ) -> __m128 { + static_assert_sae!(SAE); let a = a.as_f32x4(); let b = b.as_f32x4(); let src = src.as_f32x4(); - macro_rules! call { - ($imm4:expr) => { - vgetexpss(a, b, src, k, $imm4) - }; - } - let r = constify_imm4_sae!(sae, call); + let r = vgetexpss(a, b, src, k, SAE); transmute(r) } @@ -35464,18 +33091,18 @@ pub unsafe fn _mm_mask_getexp_round_ss( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_maskz_getexp_round_ss&expand=2858) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vgetexpss, sae = 8))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm_maskz_getexp_round_ss(k: __mmask8, a: __m128, b: __m128, sae: i32) -> __m128 { +#[cfg_attr(test, assert_instr(vgetexpss, SAE = 8))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm_maskz_getexp_round_ss( + k: __mmask8, + a: __m128, + b: __m128, +) -> __m128 { + static_assert_sae!(SAE); let a = a.as_f32x4(); let b = b.as_f32x4(); let zero = _mm_setzero_ps().as_f32x4(); - macro_rules! call { - ($imm4:expr) => { - vgetexpss(a, b, zero, k, $imm4) - }; - } - let r = constify_imm4_sae!(sae, call); + let r = vgetexpss(a, b, zero, k, SAE); transmute(r) } @@ -35485,18 +33112,14 @@ pub unsafe fn _mm_maskz_getexp_round_ss(k: __mmask8, a: __m128, b: __m128, sae: /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_getexp_round_sd&expand=2853) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vgetexpsd, sae = 8))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm_getexp_round_sd(a: __m128d, b: __m128d, sae: i32) -> __m128d { +#[cfg_attr(test, assert_instr(vgetexpsd, SAE = 8))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm_getexp_round_sd(a: __m128d, b: __m128d) -> __m128d { + static_assert_sae!(SAE); let a = a.as_f64x2(); let b = b.as_f64x2(); let zero = _mm_setzero_pd().as_f64x2(); - macro_rules! call { - ($imm4:expr) => { - vgetexpsd(a, b, zero, 0b1, $imm4) - }; - } - let r = constify_imm4_sae!(sae, call); + let r = vgetexpsd(a, b, zero, 0b1, SAE); transmute(r) } @@ -35506,24 +33129,19 @@ pub unsafe fn _mm_getexp_round_sd(a: __m128d, b: __m128d, sae: i32) -> __m128d { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_mask_getexp_round_sd&expand=2854) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vgetexpsd, sae = 8))] -#[rustc_args_required_const(4)] -pub unsafe fn _mm_mask_getexp_round_sd( +#[cfg_attr(test, assert_instr(vgetexpsd, SAE = 8))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm_mask_getexp_round_sd( src: __m128d, k: __mmask8, a: __m128d, b: __m128d, - sae: i32, ) -> __m128d { + static_assert_sae!(SAE); let a = a.as_f64x2(); let b = b.as_f64x2(); let src = src.as_f64x2(); - macro_rules! call { - ($imm4:expr) => { - vgetexpsd(a, b, src, k, $imm4) - }; - } - let r = constify_imm4_sae!(sae, call); + let r = vgetexpsd(a, b, src, k, SAE); transmute(r) } @@ -35533,18 +33151,18 @@ pub unsafe fn _mm_mask_getexp_round_sd( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_maskz_getexp_round_sd&expand=2855) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vgetexpsd, sae = 8))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm_maskz_getexp_round_sd(k: __mmask8, a: __m128d, b: __m128d, sae: i32) -> __m128d { +#[cfg_attr(test, assert_instr(vgetexpsd, SAE = 8))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm_maskz_getexp_round_sd( + k: __mmask8, + a: __m128d, + b: __m128d, +) -> __m128d { + static_assert_sae!(SAE); let a = a.as_f64x2(); let b = b.as_f64x2(); let zero = _mm_setzero_pd().as_f64x2(); - macro_rules! call { - ($imm4:expr) => { - vgetexpsd(a, b, zero, k, $imm4) - }; - } - let r = constify_imm4_sae!(sae, call); + let r = vgetexpsd(a, b, zero, k, SAE); transmute(r) } @@ -35563,24 +33181,23 @@ pub unsafe fn _mm_maskz_getexp_round_sd(k: __mmask8, a: __m128d, b: __m128d, sae /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_getmant_round_ss&expand=2892) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vgetmantss, norm = 0, sign = 0, sae = 4))] -#[rustc_args_required_const(2, 3, 4)] -pub unsafe fn _mm_getmant_round_ss( +#[cfg_attr(test, assert_instr(vgetmantss, NORM = 0, SIGN = 0, SAE = 4))] +#[rustc_legacy_const_generics(2, 3, 4)] +pub unsafe fn _mm_getmant_round_ss< + const NORM: _MM_MANTISSA_NORM_ENUM, + const SIGN: _MM_MANTISSA_SIGN_ENUM, + const SAE: i32, +>( a: __m128, b: __m128, - norm: _MM_MANTISSA_NORM_ENUM, - sign: _MM_MANTISSA_SIGN_ENUM, - sae: i32, ) -> __m128 { + static_assert_imm4!(NORM); + static_assert_imm2!(SIGN); + static_assert_mantissas_sae!(SAE); let a = a.as_f32x4(); let b = b.as_f32x4(); let zero = _mm_setzero_ps().as_f32x4(); - macro_rules! call { - ($imm4_1:expr, $imm2:expr, $imm4_2:expr) => { - vgetmantss(a, b, $imm2 << 2 | $imm4_1, zero, 0b1, $imm4_2) - }; - } - let r = constify_imm4_mantissas_sae!(norm, sign, sae, call); + let r = vgetmantss(a, b, SIGN << 2 | NORM, zero, 0b1, SAE); transmute(r) } @@ -35599,26 +33216,25 @@ pub unsafe fn _mm_getmant_round_ss( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_mask_getmant_round_ss&expand=2893) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vgetmantss, norm = 0, sign = 0, sae = 4))] -#[rustc_args_required_const(4, 5, 6)] -pub unsafe fn _mm_mask_getmant_round_ss( +#[cfg_attr(test, assert_instr(vgetmantss, NORM = 0, SIGN = 0, SAE = 4))] +#[rustc_legacy_const_generics(4, 5, 6)] +pub unsafe fn _mm_mask_getmant_round_ss< + const NORM: _MM_MANTISSA_NORM_ENUM, + const SIGN: _MM_MANTISSA_SIGN_ENUM, + const SAE: i32, +>( src: __m128, k: __mmask8, a: __m128, b: __m128, - norm: _MM_MANTISSA_NORM_ENUM, - sign: _MM_MANTISSA_SIGN_ENUM, - sae: i32, ) -> __m128 { + static_assert_imm4!(NORM); + static_assert_imm2!(SIGN); + static_assert_mantissas_sae!(SAE); let a = a.as_f32x4(); let b = b.as_f32x4(); let src = src.as_f32x4(); - macro_rules! call { - ($imm4_1:expr, $imm2:expr, $imm4_2:expr) => { - vgetmantss(a, b, $imm2 << 2 | $imm4_1, src, k, $imm4_2) - }; - } - let r = constify_imm4_mantissas_sae!(norm, sign, sae, call); + let r = vgetmantss(a, b, SIGN << 2 | NORM, src, k, SAE); transmute(r) } @@ -35637,25 +33253,24 @@ pub unsafe fn _mm_mask_getmant_round_ss( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_maskz_getmant_round_ss&expand=2894) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vgetmantss, norm = 0, sign = 0, sae = 4))] -#[rustc_args_required_const(3, 4, 5)] -pub unsafe fn _mm_maskz_getmant_round_ss( +#[cfg_attr(test, assert_instr(vgetmantss, NORM = 0, SIGN = 0, SAE = 4))] +#[rustc_legacy_const_generics(3, 4, 5)] +pub unsafe fn _mm_maskz_getmant_round_ss< + const NORM: _MM_MANTISSA_NORM_ENUM, + const SIGN: _MM_MANTISSA_SIGN_ENUM, + const SAE: i32, +>( k: __mmask8, a: __m128, b: __m128, - norm: _MM_MANTISSA_NORM_ENUM, - sign: _MM_MANTISSA_SIGN_ENUM, - sae: i32, ) -> __m128 { + static_assert_imm4!(NORM); + static_assert_imm2!(SIGN); + static_assert_mantissas_sae!(SAE); let a = a.as_f32x4(); let b = b.as_f32x4(); let zero = _mm_setzero_ps().as_f32x4(); - macro_rules! call { - ($imm4_1:expr, $imm2:expr, $imm4_2:expr) => { - vgetmantss(a, b, $imm2 << 2 | $imm4_1, zero, k, $imm4_2) - }; - } - let r = constify_imm4_mantissas_sae!(norm, sign, sae, call); + let r = vgetmantss(a, b, SIGN << 2 | NORM, zero, k, SAE); transmute(r) } @@ -35674,24 +33289,23 @@ pub unsafe fn _mm_maskz_getmant_round_ss( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_getmant_round_sd&expand=2889) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vgetmantsd, norm = 0, sign = 0, sae = 4))] -#[rustc_args_required_const(2, 3, 4)] -pub unsafe fn _mm_getmant_round_sd( +#[cfg_attr(test, assert_instr(vgetmantsd, NORM = 0, SIGN = 0, SAE = 4))] +#[rustc_legacy_const_generics(2, 3, 4)] +pub unsafe fn _mm_getmant_round_sd< + const NORM: _MM_MANTISSA_NORM_ENUM, + const SIGN: _MM_MANTISSA_SIGN_ENUM, + const SAE: i32, +>( a: __m128d, b: __m128d, - norm: _MM_MANTISSA_NORM_ENUM, - sign: _MM_MANTISSA_SIGN_ENUM, - sae: i32, ) -> __m128d { + static_assert_imm4!(NORM); + static_assert_imm2!(SIGN); + static_assert_mantissas_sae!(SAE); let a = a.as_f64x2(); let b = b.as_f64x2(); let zero = _mm_setzero_pd().as_f64x2(); - macro_rules! call { - ($imm4_1:expr, $imm2:expr, $imm4_2:expr) => { - vgetmantsd(a, b, $imm2 << 2 | $imm4_1, zero, 0b1, $imm4_2) - }; - } - let r = constify_imm4_mantissas_sae!(norm, sign, sae, call); + let r = vgetmantsd(a, b, SIGN << 2 | NORM, zero, 0b1, SAE); transmute(r) } @@ -35710,26 +33324,25 @@ pub unsafe fn _mm_getmant_round_sd( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_mask_getmant_round_sd&expand=2890) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vgetmantsd, norm = 0, sign = 0, sae = 4))] -#[rustc_args_required_const(4, 5, 6)] -pub unsafe fn _mm_mask_getmant_round_sd( +#[cfg_attr(test, assert_instr(vgetmantsd, NORM = 0, SIGN = 0, SAE = 4))] +#[rustc_legacy_const_generics(4, 5, 6)] +pub unsafe fn _mm_mask_getmant_round_sd< + const NORM: _MM_MANTISSA_NORM_ENUM, + const SIGN: _MM_MANTISSA_SIGN_ENUM, + const SAE: i32, +>( src: __m128d, k: __mmask8, a: __m128d, b: __m128d, - norm: _MM_MANTISSA_NORM_ENUM, - sign: _MM_MANTISSA_SIGN_ENUM, - sae: i32, ) -> __m128d { + static_assert_imm4!(NORM); + static_assert_imm2!(SIGN); + static_assert_mantissas_sae!(SAE); let a = a.as_f64x2(); let b = b.as_f64x2(); let src = src.as_f64x2(); - macro_rules! call { - ($imm4_1:expr, $imm2:expr, $imm4_2:expr) => { - vgetmantsd(a, b, $imm2 << 2 | $imm4_1, src, k, $imm4_2) - }; - } - let r = constify_imm4_mantissas_sae!(norm, sign, sae, call); + let r = vgetmantsd(a, b, SIGN << 2 | NORM, src, k, SAE); transmute(r) } @@ -35748,25 +33361,24 @@ pub unsafe fn _mm_mask_getmant_round_sd( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_maskz_getmant_round_sd&expand=2891) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vgetmantsd, norm = 0, sign = 0, sae = 4))] -#[rustc_args_required_const(3, 4, 5)] -pub unsafe fn _mm_maskz_getmant_round_sd( +#[cfg_attr(test, assert_instr(vgetmantsd, NORM = 0, SIGN = 0, SAE = 4))] +#[rustc_legacy_const_generics(3, 4, 5)] +pub unsafe fn _mm_maskz_getmant_round_sd< + const NORM: _MM_MANTISSA_NORM_ENUM, + const SIGN: _MM_MANTISSA_SIGN_ENUM, + const SAE: i32, +>( k: __mmask8, a: __m128d, b: __m128d, - norm: _MM_MANTISSA_NORM_ENUM, - sign: _MM_MANTISSA_SIGN_ENUM, - sae: i32, ) -> __m128d { + static_assert_imm4!(NORM); + static_assert_imm2!(SIGN); + static_assert_mantissas_sae!(SAE); let a = a.as_f64x2(); let b = b.as_f64x2(); let zero = _mm_setzero_pd().as_f64x2(); - macro_rules! call { - ($imm4_1:expr, $imm2:expr, $imm4_2:expr) => { - vgetmantsd(a, b, $imm2 << 2 | $imm4_1, zero, k, $imm4_2) - }; - } - let r = constify_imm4_mantissas_sae!(norm, sign, sae, call); + let r = vgetmantsd(a, b, SIGN << 2 | NORM, zero, k, SAE); transmute(r) } @@ -35782,18 +33394,18 @@ pub unsafe fn _mm_maskz_getmant_round_sd( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_roundscale_round_ss&expand=4796) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vrndscaless, imm8 = 0, sae = 8))] -#[rustc_args_required_const(2, 3)] -pub unsafe fn _mm_roundscale_round_ss(a: __m128, b: __m128, imm8: i32, sae: i32) -> __m128 { +#[cfg_attr(test, assert_instr(vrndscaless, IMM8 = 0, SAE = 8))] +#[rustc_legacy_const_generics(2, 3)] +pub unsafe fn _mm_roundscale_round_ss( + a: __m128, + b: __m128, +) -> __m128 { + static_assert_imm8!(IMM8); + static_assert_mantissas_sae!(SAE); let a = a.as_f32x4(); let b = b.as_f32x4(); let zero = _mm_setzero_ps().as_f32x4(); - macro_rules! call { - ($imm8:expr, $imm4:expr) => { - vrndscaless(a, b, zero, 0b11111111, $imm8, $imm4) - }; - } - let r = constify_imm8_roundscale!(imm8, sae, call); + let r = vrndscaless(a, b, zero, 0b11111111, IMM8, SAE); transmute(r) } @@ -35809,25 +33421,20 @@ pub unsafe fn _mm_roundscale_round_ss(a: __m128, b: __m128, imm8: i32, sae: i32) /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_mask_roundscale_round_ss&expand=4794) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vrndscaless, imm8 = 0, sae = 8))] -#[rustc_args_required_const(4, 5)] -pub unsafe fn _mm_mask_roundscale_round_ss( +#[cfg_attr(test, assert_instr(vrndscaless, IMM8 = 0, SAE = 8))] +#[rustc_legacy_const_generics(4, 5)] +pub unsafe fn _mm_mask_roundscale_round_ss( src: __m128, k: __mmask8, a: __m128, b: __m128, - imm8: i32, - sae: i32, ) -> __m128 { + static_assert_imm8!(IMM8); + static_assert_mantissas_sae!(SAE); let a = a.as_f32x4(); let b = b.as_f32x4(); let src = src.as_f32x4(); - macro_rules! call { - ($imm8:expr, $imm4:expr) => { - vrndscaless(a, b, src, k, $imm8, $imm4) - }; - } - let r = constify_imm8_roundscale!(imm8, sae, call); + let r = vrndscaless(a, b, src, k, IMM8, SAE); transmute(r) } @@ -35843,24 +33450,19 @@ pub unsafe fn _mm_mask_roundscale_round_ss( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_maskz_roundscale_round_ss&expand=4795) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vrndscaless, imm8 = 0, sae = 8))] -#[rustc_args_required_const(3, 4)] -pub unsafe fn _mm_maskz_roundscale_round_ss( +#[cfg_attr(test, assert_instr(vrndscaless, IMM8 = 0, SAE = 8))] +#[rustc_legacy_const_generics(3, 4)] +pub unsafe fn _mm_maskz_roundscale_round_ss( k: __mmask8, a: __m128, b: __m128, - imm8: i32, - sae: i32, ) -> __m128 { + static_assert_imm8!(IMM8); + static_assert_mantissas_sae!(SAE); let a = a.as_f32x4(); let b = b.as_f32x4(); let zero = _mm_setzero_ps().as_f32x4(); - macro_rules! call { - ($imm8:expr, $imm4:expr) => { - vrndscaless(a, b, zero, k, $imm8, $imm4) - }; - } - let r = constify_imm8_roundscale!(imm8, sae, call); + let r = vrndscaless(a, b, zero, k, IMM8, SAE); transmute(r) } @@ -35876,18 +33478,18 @@ pub unsafe fn _mm_maskz_roundscale_round_ss( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_roundscale_round_sd&expand=4793) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vrndscalesd, imm8 = 0, sae = 8))] -#[rustc_args_required_const(2, 3)] -pub unsafe fn _mm_roundscale_round_sd(a: __m128d, b: __m128d, imm8: i32, sae: i32) -> __m128d { +#[cfg_attr(test, assert_instr(vrndscalesd, IMM8 = 0, SAE = 8))] +#[rustc_legacy_const_generics(2, 3)] +pub unsafe fn _mm_roundscale_round_sd( + a: __m128d, + b: __m128d, +) -> __m128d { + static_assert_imm8!(IMM8); + static_assert_mantissas_sae!(SAE); let a = a.as_f64x2(); let b = b.as_f64x2(); let zero = _mm_setzero_pd().as_f64x2(); - macro_rules! call { - ($imm8:expr, $imm4:expr) => { - vrndscalesd(a, b, zero, 0b11111111, $imm8, $imm4) - }; - } - let r = constify_imm8_roundscale!(imm8, sae, call); + let r = vrndscalesd(a, b, zero, 0b11111111, IMM8, SAE); transmute(r) } @@ -35903,25 +33505,20 @@ pub unsafe fn _mm_roundscale_round_sd(a: __m128d, b: __m128d, imm8: i32, sae: i3 /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_mask_roundscale_round_sd&expand=4791) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vrndscalesd, imm8 = 0, sae = 8))] -#[rustc_args_required_const(4, 5)] -pub unsafe fn _mm_mask_roundscale_round_sd( +#[cfg_attr(test, assert_instr(vrndscalesd, IMM8 = 0, SAE = 8))] +#[rustc_legacy_const_generics(4, 5)] +pub unsafe fn _mm_mask_roundscale_round_sd( src: __m128d, k: __mmask8, a: __m128d, b: __m128d, - imm8: i32, - sae: i32, ) -> __m128d { + static_assert_imm8!(IMM8); + static_assert_mantissas_sae!(SAE); let a = a.as_f64x2(); let b = b.as_f64x2(); let src = src.as_f64x2(); - macro_rules! call { - ($imm8:expr, $imm4:expr) => { - vrndscalesd(a, b, src, k, $imm8, $imm4) - }; - } - let r = constify_imm8_roundscale!(imm8, sae, call); + let r = vrndscalesd(a, b, src, k, IMM8, SAE); transmute(r) } @@ -35937,24 +33534,19 @@ pub unsafe fn _mm_mask_roundscale_round_sd( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_maskz_roundscale_round_sd&expand=4792) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vrndscalesd, imm8 = 0, sae = 8))] -#[rustc_args_required_const(3, 4)] -pub unsafe fn _mm_maskz_roundscale_round_sd( +#[cfg_attr(test, assert_instr(vrndscalesd, IMM8 = 0, SAE = 8))] +#[rustc_legacy_const_generics(3, 4)] +pub unsafe fn _mm_maskz_roundscale_round_sd( k: __mmask8, a: __m128d, b: __m128d, - imm8: i32, - sae: i32, ) -> __m128d { + static_assert_imm8!(IMM8); + static_assert_mantissas_sae!(SAE); let a = a.as_f64x2(); let b = b.as_f64x2(); let zero = _mm_setzero_pd().as_f64x2(); - macro_rules! call { - ($imm8:expr, $imm4:expr) => { - vrndscalesd(a, b, zero, k, $imm8, $imm4) - }; - } - let r = constify_imm8_roundscale!(imm8, sae, call); + let r = vrndscalesd(a, b, zero, k, IMM8, SAE); transmute(r) } @@ -35970,18 +33562,14 @@ pub unsafe fn _mm_maskz_roundscale_round_sd( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_scalef_round_ss&expand=4895) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vscalefss, rounding = 8))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm_scalef_round_ss(a: __m128, b: __m128, rounding: i32) -> __m128 { +#[cfg_attr(test, assert_instr(vscalefss, ROUNDING = 8))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm_scalef_round_ss(a: __m128, b: __m128) -> __m128 { + static_assert_rounding!(ROUNDING); let a = a.as_f32x4(); let b = b.as_f32x4(); let zero = _mm_setzero_ps().as_f32x4(); - macro_rules! call { - ($imm4:expr) => { - vscalefss(a, b, zero, 0b11111111, $imm4) - }; - } - let r = constify_imm4_round!(rounding, call); + let r = vscalefss(a, b, zero, 0b11111111, ROUNDING); transmute(r) } @@ -35997,24 +33585,19 @@ pub unsafe fn _mm_scalef_round_ss(a: __m128, b: __m128, rounding: i32) -> __m128 /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_mask_scalef_round_ss&expand=4893) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vscalefss, rounding = 8))] -#[rustc_args_required_const(4)] -pub unsafe fn _mm_mask_scalef_round_ss( +#[cfg_attr(test, assert_instr(vscalefss, ROUNDING = 8))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm_mask_scalef_round_ss( src: __m128, k: __mmask8, a: __m128, b: __m128, - rounding: i32, ) -> __m128 { + static_assert_rounding!(ROUNDING); let a = a.as_f32x4(); let b = b.as_f32x4(); let src = src.as_f32x4(); - macro_rules! call { - ($imm4:expr) => { - vscalefss(a, b, src, k, $imm4) - }; - } - let r = constify_imm4_round!(rounding, call); + let r = vscalefss(a, b, src, k, ROUNDING); transmute(r) } @@ -36030,23 +33613,18 @@ pub unsafe fn _mm_mask_scalef_round_ss( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_maskz_scalef_round_ss&expand=4894) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vscalefss, rounding = 8))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm_maskz_scalef_round_ss( +#[cfg_attr(test, assert_instr(vscalefss, ROUNDING = 8))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm_maskz_scalef_round_ss( k: __mmask8, a: __m128, b: __m128, - rounding: i32, ) -> __m128 { + static_assert_rounding!(ROUNDING); let a = a.as_f32x4(); let b = b.as_f32x4(); let zero = _mm_setzero_ps().as_f32x4(); - macro_rules! call { - ($imm4:expr) => { - vscalefss(a, b, zero, k, $imm4) - }; - } - let r = constify_imm4_round!(rounding, call); + let r = vscalefss(a, b, zero, k, ROUNDING); transmute(r) } @@ -36062,18 +33640,14 @@ pub unsafe fn _mm_maskz_scalef_round_ss( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_scalef_round_sd&expand=4892) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vscalefsd, rounding = 8))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm_scalef_round_sd(a: __m128d, b: __m128d, rounding: i32) -> __m128d { +#[cfg_attr(test, assert_instr(vscalefsd, ROUNDING = 8))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm_scalef_round_sd(a: __m128d, b: __m128d) -> __m128d { + static_assert_rounding!(ROUNDING); let a = a.as_f64x2(); let b = b.as_f64x2(); let zero = _mm_setzero_pd().as_f64x2(); - macro_rules! call { - ($imm4:expr) => { - vscalefsd(a, b, zero, 0b11111111, $imm4) - }; - } - let r = constify_imm4_round!(rounding, call); + let r = vscalefsd(a, b, zero, 0b11111111, ROUNDING); transmute(r) } @@ -36089,24 +33663,18 @@ pub unsafe fn _mm_scalef_round_sd(a: __m128d, b: __m128d, rounding: i32) -> __m1 /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_mask_scalef_round_sd&expand=4890) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vscalefsd, rounding = 8))] -#[rustc_args_required_const(4)] -pub unsafe fn _mm_mask_scalef_round_sd( +#[cfg_attr(test, assert_instr(vscalefsd, ROUNDING = 8))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm_mask_scalef_round_sd( src: __m128d, k: __mmask8, a: __m128d, b: __m128d, - rounding: i32, ) -> __m128d { let a = a.as_f64x2(); let b = b.as_f64x2(); let src = src.as_f64x2(); - macro_rules! call { - ($imm4:expr) => { - vscalefsd(a, b, src, k, $imm4) - }; - } - let r = constify_imm4_round!(rounding, call); + let r = vscalefsd(a, b, src, k, ROUNDING); transmute(r) } @@ -36122,23 +33690,18 @@ pub unsafe fn _mm_mask_scalef_round_sd( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_maskz_scalef_round_sd&expand=4891) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vscalefsd, rounding = 8))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm_maskz_scalef_round_sd( +#[cfg_attr(test, assert_instr(vscalefsd, ROUNDING = 8))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm_maskz_scalef_round_sd( k: __mmask8, a: __m128d, b: __m128d, - rounding: i32, ) -> __m128d { + static_assert_rounding!(ROUNDING); let a = a.as_f64x2(); let b = b.as_f64x2(); let zero = _mm_setzero_pd().as_f64x2(); - macro_rules! call { - ($imm4:expr) => { - vscalefsd(a, b, zero, k, $imm4) - }; - } - let r = constify_imm4_round!(rounding, call); + let r = vscalefsd(a, b, zero, k, ROUNDING); transmute(r) } @@ -36154,19 +33717,15 @@ pub unsafe fn _mm_maskz_scalef_round_sd( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_fmadd_round_ss&expand=2573) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vfmadd213ss, rounding = 8))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm_fmadd_round_ss(a: __m128, b: __m128, c: __m128, rounding: i32) -> __m128 { +#[cfg_attr(test, assert_instr(vfmadd213ss, ROUNDING = 8))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm_fmadd_round_ss(a: __m128, b: __m128, c: __m128) -> __m128 { + static_assert_rounding!(ROUNDING); let extracta: f32 = simd_extract(a, 0); let extractb: f32 = simd_extract(b, 0); let extractc: f32 = simd_extract(c, 0); - macro_rules! call { - ($imm4:expr) => { - vfmadd132ss(extracta, extractb, extractc, $imm4) - }; - } - let fmadd = constify_imm4_round!(rounding, call); - let r = simd_insert(a, 0, fmadd); + let r = vfmadd132ss(extracta, extractb, extractc, ROUNDING); + let r = simd_insert(a, 0, r); transmute(r) } @@ -36182,25 +33741,20 @@ pub unsafe fn _mm_fmadd_round_ss(a: __m128, b: __m128, c: __m128, rounding: i32) /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_mask_fmadd_round_ss&expand=2574) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vfmadd213ss, rounding = 8))] -#[rustc_args_required_const(4)] -pub unsafe fn _mm_mask_fmadd_round_ss( +#[cfg_attr(test, assert_instr(vfmadd213ss, ROUNDING = 8))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm_mask_fmadd_round_ss( a: __m128, k: __mmask8, b: __m128, c: __m128, - rounding: i32, ) -> __m128 { + static_assert_rounding!(ROUNDING); let mut fmadd: f32 = simd_extract(a, 0); if (k & 0b00000001) != 0 { let extractb: f32 = simd_extract(b, 0); let extractc: f32 = simd_extract(c, 0); - macro_rules! call { - ($imm4:expr) => { - vfmadd132ss(fmadd, extractb, extractc, $imm4) - }; - } - fmadd = constify_imm4_round!(rounding, call); + fmadd = vfmadd132ss(fmadd, extractb, extractc, ROUNDING); } let r = simd_insert(a, 0, fmadd); transmute(r) @@ -36218,26 +33772,21 @@ pub unsafe fn _mm_mask_fmadd_round_ss( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_maskz_fmadd_round_ss&expand=2576) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vfmadd213ss, rounding = 8))] -#[rustc_args_required_const(4)] -pub unsafe fn _mm_maskz_fmadd_round_ss( +#[cfg_attr(test, assert_instr(vfmadd213ss, ROUNDING = 8))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm_maskz_fmadd_round_ss( k: __mmask8, a: __m128, b: __m128, c: __m128, - rounding: i32, ) -> __m128 { + static_assert_rounding!(ROUNDING); let mut fmadd: f32 = 0.; if (k & 0b00000001) != 0 { let extracta: f32 = simd_extract(a, 0); let extractb: f32 = simd_extract(b, 0); let extractc: f32 = simd_extract(c, 0); - macro_rules! call { - ($imm4:expr) => { - vfmadd132ss(extracta, extractb, extractc, $imm4) - }; - } - fmadd = constify_imm4_round!(rounding, call); + fmadd = vfmadd132ss(extracta, extractb, extractc, ROUNDING); } let r = simd_insert(a, 0, fmadd); transmute(r) @@ -36255,25 +33804,20 @@ pub unsafe fn _mm_maskz_fmadd_round_ss( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_mask3_fmadd_round_ss&expand=2575) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vfmadd213ss, rounding = 8))] -#[rustc_args_required_const(4)] -pub unsafe fn _mm_mask3_fmadd_round_ss( +#[cfg_attr(test, assert_instr(vfmadd213ss, ROUNDING = 8))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm_mask3_fmadd_round_ss( a: __m128, b: __m128, c: __m128, k: __mmask8, - rounding: i32, ) -> __m128 { + static_assert_rounding!(ROUNDING); let mut fmadd: f32 = simd_extract(c, 0); if (k & 0b00000001) != 0 { let extracta: f32 = simd_extract(a, 0); let extractb: f32 = simd_extract(b, 0); - macro_rules! call { - ($imm4:expr) => { - vfmadd132ss(extracta, extractb, fmadd, $imm4) - }; - } - fmadd = constify_imm4_round!(rounding, call); + fmadd = vfmadd132ss(extracta, extractb, fmadd, ROUNDING); } let r = simd_insert(c, 0, fmadd); transmute(r) @@ -36291,18 +33835,18 @@ pub unsafe fn _mm_mask3_fmadd_round_ss( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_fmadd_round_sd&expand=2569) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vfmadd213sd, rounding = 8))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm_fmadd_round_sd(a: __m128d, b: __m128d, c: __m128d, rounding: i32) -> __m128d { +#[cfg_attr(test, assert_instr(vfmadd213sd, ROUNDING = 8))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm_fmadd_round_sd( + a: __m128d, + b: __m128d, + c: __m128d, +) -> __m128d { + static_assert_rounding!(ROUNDING); let extracta: f64 = simd_extract(a, 0); let extractb: f64 = simd_extract(b, 0); let extractc: f64 = simd_extract(c, 0); - macro_rules! call { - ($imm4:expr) => { - vfmadd132sd(extracta, extractb, extractc, $imm4) - }; - } - let fmadd = constify_imm4_round!(rounding, call); + let fmadd = vfmadd132sd(extracta, extractb, extractc, ROUNDING); let r = simd_insert(a, 0, fmadd); transmute(r) } @@ -36319,25 +33863,20 @@ pub unsafe fn _mm_fmadd_round_sd(a: __m128d, b: __m128d, c: __m128d, rounding: i /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_mask_fmadd_round_sd&expand=2570) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vfmadd213sd, rounding = 8))] -#[rustc_args_required_const(4)] -pub unsafe fn _mm_mask_fmadd_round_sd( +#[cfg_attr(test, assert_instr(vfmadd213sd, ROUNDING = 8))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm_mask_fmadd_round_sd( a: __m128d, k: __mmask8, b: __m128d, c: __m128d, - rounding: i32, ) -> __m128d { + static_assert_rounding!(ROUNDING); let mut fmadd: f64 = simd_extract(a, 0); if (k & 0b00000001) != 0 { let extractb: f64 = simd_extract(b, 0); let extractc: f64 = simd_extract(c, 0); - macro_rules! call { - ($imm4:expr) => { - vfmadd132sd(fmadd, extractb, extractc, $imm4) - }; - } - fmadd = constify_imm4_round!(rounding, call); + fmadd = vfmadd132sd(fmadd, extractb, extractc, ROUNDING); } let r = simd_insert(a, 0, fmadd); transmute(r) @@ -36355,26 +33894,21 @@ pub unsafe fn _mm_mask_fmadd_round_sd( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_maskz_fmadd_round_sd&expand=2572) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vfmadd213sd, rounding = 8))] -#[rustc_args_required_const(4)] -pub unsafe fn _mm_maskz_fmadd_round_sd( +#[cfg_attr(test, assert_instr(vfmadd213sd, ROUNDING = 8))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm_maskz_fmadd_round_sd( k: __mmask8, a: __m128d, b: __m128d, c: __m128d, - rounding: i32, ) -> __m128d { + static_assert_rounding!(ROUNDING); let mut fmadd: f64 = 0.; if (k & 0b00000001) != 0 { let extracta: f64 = simd_extract(a, 0); let extractb: f64 = simd_extract(b, 0); let extractc: f64 = simd_extract(c, 0); - macro_rules! call { - ($imm4:expr) => { - vfmadd132sd(extracta, extractb, extractc, $imm4) - }; - } - fmadd = constify_imm4_round!(rounding, call); + fmadd = vfmadd132sd(extracta, extractb, extractc, ROUNDING); } let r = simd_insert(a, 0, fmadd); transmute(r) @@ -36392,25 +33926,20 @@ pub unsafe fn _mm_maskz_fmadd_round_sd( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_mask3_fmadd_round_Sd&expand=2571) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vfmadd213sd, rounding = 8))] -#[rustc_args_required_const(4)] -pub unsafe fn _mm_mask3_fmadd_round_sd( +#[cfg_attr(test, assert_instr(vfmadd213sd, ROUNDING = 8))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm_mask3_fmadd_round_sd( a: __m128d, b: __m128d, c: __m128d, k: __mmask8, - rounding: i32, ) -> __m128d { + static_assert_rounding!(ROUNDING); let mut fmadd: f64 = simd_extract(c, 0); if (k & 0b00000001) != 0 { let extracta: f64 = simd_extract(a, 0); let extractb: f64 = simd_extract(b, 0); - macro_rules! call { - ($imm4:expr) => { - vfmadd132sd(extracta, extractb, fmadd, $imm4) - }; - } - fmadd = constify_imm4_round!(rounding, call); + fmadd = vfmadd132sd(extracta, extractb, fmadd, ROUNDING); } let r = simd_insert(c, 0, fmadd); transmute(r) @@ -36428,19 +33957,15 @@ pub unsafe fn _mm_mask3_fmadd_round_sd( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_fmsub_round_ss&expand=2659) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vfmsub213ss, rounding = 8))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm_fmsub_round_ss(a: __m128, b: __m128, c: __m128, rounding: i32) -> __m128 { +#[cfg_attr(test, assert_instr(vfmsub213ss, ROUNDING = 8))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm_fmsub_round_ss(a: __m128, b: __m128, c: __m128) -> __m128 { + static_assert_rounding!(ROUNDING); let extracta: f32 = simd_extract(a, 0); let extractb: f32 = simd_extract(b, 0); let extractc: f32 = simd_extract(c, 0); let extractc = -extractc; - macro_rules! call { - ($imm4:expr) => { - vfmadd132ss(extracta, extractb, extractc, $imm4) - }; - } - let fmsub = constify_imm4_round!(rounding, call); + let fmsub = vfmadd132ss(extracta, extractb, extractc, ROUNDING); let r = simd_insert(a, 0, fmsub); transmute(r) } @@ -36457,26 +33982,21 @@ pub unsafe fn _mm_fmsub_round_ss(a: __m128, b: __m128, c: __m128, rounding: i32) /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_mask_fmsub_round_ss&expand=2660) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vfmsub213ss, rounding = 8))] -#[rustc_args_required_const(4)] -pub unsafe fn _mm_mask_fmsub_round_ss( +#[cfg_attr(test, assert_instr(vfmsub213ss, ROUNDING = 8))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm_mask_fmsub_round_ss( a: __m128, k: __mmask8, b: __m128, c: __m128, - rounding: i32, ) -> __m128 { + static_assert_rounding!(ROUNDING); let mut fmsub: f32 = simd_extract(a, 0); if (k & 0b00000001) != 0 { let extractb: f32 = simd_extract(b, 0); let extractc: f32 = simd_extract(c, 0); let extractc = -extractc; - macro_rules! call { - ($imm4:expr) => { - vfmadd132ss(fmsub, extractb, extractc, $imm4) - }; - } - fmsub = constify_imm4_round!(rounding, call); + fmsub = vfmadd132ss(fmsub, extractb, extractc, ROUNDING); } let r = simd_insert(a, 0, fmsub); transmute(r) @@ -36494,27 +34014,22 @@ pub unsafe fn _mm_mask_fmsub_round_ss( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_maskz_fmsub_round_ss&expand=2662) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vfmsub213ss, rounding = 8))] -#[rustc_args_required_const(4)] -pub unsafe fn _mm_maskz_fmsub_round_ss( +#[cfg_attr(test, assert_instr(vfmsub213ss, ROUNDING = 8))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm_maskz_fmsub_round_ss( k: __mmask8, a: __m128, b: __m128, c: __m128, - rounding: i32, ) -> __m128 { + static_assert_rounding!(ROUNDING); let mut fmsub: f32 = 0.; if (k & 0b00000001) != 0 { let extracta: f32 = simd_extract(a, 0); let extractb: f32 = simd_extract(b, 0); let extractc: f32 = simd_extract(c, 0); let extractc = -extractc; - macro_rules! call { - ($imm4:expr) => { - vfmadd132ss(extracta, extractb, extractc, $imm4) - }; - } - fmsub = constify_imm4_round!(rounding, call); + fmsub = vfmadd132ss(extracta, extractb, extractc, ROUNDING); } let r = simd_insert(a, 0, fmsub); transmute(r) @@ -36532,26 +34047,21 @@ pub unsafe fn _mm_maskz_fmsub_round_ss( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_mask3_fmsub_round_ss&expand=2661) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vfmsub213ss, rounding = 8))] -#[rustc_args_required_const(4)] -pub unsafe fn _mm_mask3_fmsub_round_ss( +#[cfg_attr(test, assert_instr(vfmsub213ss, ROUNDING = 8))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm_mask3_fmsub_round_ss( a: __m128, b: __m128, c: __m128, k: __mmask8, - rounding: i32, ) -> __m128 { + static_assert_rounding!(ROUNDING); let mut fmsub: f32 = simd_extract(c, 0); if (k & 0b00000001) != 0 { let extracta: f32 = simd_extract(a, 0); let extractb: f32 = simd_extract(b, 0); let extractc = -fmsub; - macro_rules! call { - ($imm4:expr) => { - vfmadd132ss(extracta, extractb, extractc, $imm4) - }; - } - fmsub = constify_imm4_round!(rounding, call); + fmsub = vfmadd132ss(extracta, extractb, extractc, ROUNDING); } let r = simd_insert(c, 0, fmsub); transmute(r) @@ -36569,19 +34079,19 @@ pub unsafe fn _mm_mask3_fmsub_round_ss( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_fmsub_round_sd&expand=2655) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vfmsub213sd, rounding = 8))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm_fmsub_round_sd(a: __m128d, b: __m128d, c: __m128d, rounding: i32) -> __m128d { +#[cfg_attr(test, assert_instr(vfmsub213sd, ROUNDING = 8))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm_fmsub_round_sd( + a: __m128d, + b: __m128d, + c: __m128d, +) -> __m128d { + static_assert_rounding!(ROUNDING); let extracta: f64 = simd_extract(a, 0); let extractb: f64 = simd_extract(b, 0); let extractc: f64 = simd_extract(c, 0); let extractc = -extractc; - macro_rules! call { - ($imm4:expr) => { - vfmadd132sd(extracta, extractb, extractc, $imm4) - }; - } - let fmsub = constify_imm4_round!(rounding, call); + let fmsub = vfmadd132sd(extracta, extractb, extractc, ROUNDING); let r = simd_insert(a, 0, fmsub); transmute(r) } @@ -36598,26 +34108,21 @@ pub unsafe fn _mm_fmsub_round_sd(a: __m128d, b: __m128d, c: __m128d, rounding: i /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_mask_fmsub_round_sd&expand=2656) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vfmsub213sd, rounding = 8))] -#[rustc_args_required_const(4)] -pub unsafe fn _mm_mask_fmsub_round_sd( +#[cfg_attr(test, assert_instr(vfmsub213sd, ROUNDING = 8))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm_mask_fmsub_round_sd( a: __m128d, k: __mmask8, b: __m128d, c: __m128d, - rounding: i32, ) -> __m128d { + static_assert_rounding!(ROUNDING); let mut fmsub: f64 = simd_extract(a, 0); if (k & 0b00000001) != 0 { let extractb: f64 = simd_extract(b, 0); let extractc: f64 = simd_extract(c, 0); let extractc = -extractc; - macro_rules! call { - ($imm4:expr) => { - vfmadd132sd(fmsub, extractb, extractc, $imm4) - }; - } - fmsub = constify_imm4_round!(rounding, call); + fmsub = vfmadd132sd(fmsub, extractb, extractc, ROUNDING); } let r = simd_insert(a, 0, fmsub); transmute(r) @@ -36635,27 +34140,22 @@ pub unsafe fn _mm_mask_fmsub_round_sd( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_maskz_fmsub_round_sd&expand=2658) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vfmsub213sd, rounding = 8))] -#[rustc_args_required_const(4)] -pub unsafe fn _mm_maskz_fmsub_round_sd( +#[cfg_attr(test, assert_instr(vfmsub213sd, ROUNDING = 8))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm_maskz_fmsub_round_sd( k: __mmask8, a: __m128d, b: __m128d, c: __m128d, - rounding: i32, ) -> __m128d { + static_assert_rounding!(ROUNDING); let mut fmsub: f64 = 0.; if (k & 0b00000001) != 0 { let extracta: f64 = simd_extract(a, 0); let extractb: f64 = simd_extract(b, 0); let extractc: f64 = simd_extract(c, 0); let extractc = -extractc; - macro_rules! call { - ($imm4:expr) => { - vfmadd132sd(extracta, extractb, extractc, $imm4) - }; - } - fmsub = constify_imm4_round!(rounding, call); + fmsub = vfmadd132sd(extracta, extractb, extractc, ROUNDING); } let r = simd_insert(a, 0, fmsub); transmute(r) @@ -36673,26 +34173,21 @@ pub unsafe fn _mm_maskz_fmsub_round_sd( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_mask3_fmsub_round_sd&expand=2657) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vfmsub213sd, rounding = 8))] -#[rustc_args_required_const(4)] -pub unsafe fn _mm_mask3_fmsub_round_sd( +#[cfg_attr(test, assert_instr(vfmsub213sd, ROUNDING = 8))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm_mask3_fmsub_round_sd( a: __m128d, b: __m128d, c: __m128d, k: __mmask8, - rounding: i32, ) -> __m128d { + static_assert_rounding!(ROUNDING); let mut fmsub: f64 = simd_extract(c, 0); if (k & 0b00000001) != 0 { let extracta: f64 = simd_extract(a, 0); let extractb: f64 = simd_extract(b, 0); let extractc = -fmsub; - macro_rules! call { - ($imm4:expr) => { - vfmadd132sd(extracta, extractb, extractc, $imm4) - }; - } - fmsub = constify_imm4_round!(rounding, call); + fmsub = vfmadd132sd(extracta, extractb, extractc, ROUNDING); } let r = simd_insert(c, 0, fmsub); transmute(r) @@ -36710,19 +34205,15 @@ pub unsafe fn _mm_mask3_fmsub_round_sd( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_fnmadd_round_ss&expand=2739) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vfnmadd213ss, rounding = 8))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm_fnmadd_round_ss(a: __m128, b: __m128, c: __m128, rounding: i32) -> __m128 { +#[cfg_attr(test, assert_instr(vfnmadd213ss, ROUNDING = 8))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm_fnmadd_round_ss(a: __m128, b: __m128, c: __m128) -> __m128 { + static_assert_rounding!(ROUNDING); let extracta: f32 = simd_extract(a, 0); let extracta = -extracta; let extractb: f32 = simd_extract(b, 0); let extractc: f32 = simd_extract(c, 0); - macro_rules! call { - ($imm4:expr) => { - vfmadd132ss(extracta, extractb, extractc, $imm4) - }; - } - let fnmadd = constify_imm4_round!(rounding, call); + let fnmadd = vfmadd132ss(extracta, extractb, extractc, ROUNDING); let r = simd_insert(a, 0, fnmadd); transmute(r) } @@ -36739,26 +34230,21 @@ pub unsafe fn _mm_fnmadd_round_ss(a: __m128, b: __m128, c: __m128, rounding: i32 /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_mask_fnmadd_round_ss&expand=2740) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vfnmadd213ss, rounding = 8))] -#[rustc_args_required_const(4)] -pub unsafe fn _mm_mask_fnmadd_round_ss( +#[cfg_attr(test, assert_instr(vfnmadd213ss, ROUNDING = 8))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm_mask_fnmadd_round_ss( a: __m128, k: __mmask8, b: __m128, c: __m128, - rounding: i32, ) -> __m128 { + static_assert_rounding!(ROUNDING); let mut fnmadd: f32 = simd_extract(a, 0); if (k & 0b00000001) != 0 { let extracta = -fnmadd; let extractb: f32 = simd_extract(b, 0); let extractc: f32 = simd_extract(c, 0); - macro_rules! call { - ($imm4:expr) => { - vfmadd132ss(extracta, extractb, extractc, $imm4) - }; - } - fnmadd = constify_imm4_round!(rounding, call); + fnmadd = vfmadd132ss(extracta, extractb, extractc, ROUNDING); } let r = simd_insert(a, 0, fnmadd); transmute(r) @@ -36776,27 +34262,22 @@ pub unsafe fn _mm_mask_fnmadd_round_ss( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_maskz_fnmadd_round_ss&expand=2742) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vfnmadd213ss, rounding = 8))] -#[rustc_args_required_const(4)] -pub unsafe fn _mm_maskz_fnmadd_round_ss( +#[cfg_attr(test, assert_instr(vfnmadd213ss, ROUNDING = 8))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm_maskz_fnmadd_round_ss( k: __mmask8, a: __m128, b: __m128, c: __m128, - rounding: i32, ) -> __m128 { + static_assert_rounding!(ROUNDING); let mut fnmadd: f32 = 0.; if (k & 0b00000001) != 0 { let extracta: f32 = simd_extract(a, 0); let extracta = -extracta; let extractb: f32 = simd_extract(b, 0); let extractc: f32 = simd_extract(c, 0); - macro_rules! call { - ($imm4:expr) => { - vfmadd132ss(extracta, extractb, extractc, $imm4) - }; - } - fnmadd = constify_imm4_round!(rounding, call); + fnmadd = vfmadd132ss(extracta, extractb, extractc, ROUNDING); } let r = simd_insert(a, 0, fnmadd); transmute(r) @@ -36814,26 +34295,21 @@ pub unsafe fn _mm_maskz_fnmadd_round_ss( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_mask3_fnmadd_round_ss&expand=2741) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vfnmadd213ss, rounding = 8))] -#[rustc_args_required_const(4)] -pub unsafe fn _mm_mask3_fnmadd_round_ss( +#[cfg_attr(test, assert_instr(vfnmadd213ss, ROUNDING = 8))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm_mask3_fnmadd_round_ss( a: __m128, b: __m128, c: __m128, k: __mmask8, - rounding: i32, ) -> __m128 { + static_assert_rounding!(ROUNDING); let mut fnmadd: f32 = simd_extract(c, 0); if (k & 0b00000001) != 0 { let extracta: f32 = simd_extract(a, 0); let extracta = -extracta; let extractb: f32 = simd_extract(b, 0); - macro_rules! call { - ($imm4:expr) => { - vfmadd132ss(extracta, extractb, fnmadd, $imm4) - }; - } - fnmadd = constify_imm4_round!(rounding, call); + fnmadd = vfmadd132ss(extracta, extractb, fnmadd, ROUNDING); } let r = simd_insert(c, 0, fnmadd); transmute(r) @@ -36851,19 +34327,19 @@ pub unsafe fn _mm_mask3_fnmadd_round_ss( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_fnmadd_round_sd&expand=2735) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vfnmadd213sd, rounding = 8))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm_fnmadd_round_sd(a: __m128d, b: __m128d, c: __m128d, rounding: i32) -> __m128d { +#[cfg_attr(test, assert_instr(vfnmadd213sd, ROUNDING = 8))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm_fnmadd_round_sd( + a: __m128d, + b: __m128d, + c: __m128d, +) -> __m128d { + static_assert_rounding!(ROUNDING); let extracta: f64 = simd_extract(a, 0); let extracta = -extracta; let extractb: f64 = simd_extract(b, 0); let extractc: f64 = simd_extract(c, 0); - macro_rules! call { - ($imm4:expr) => { - vfmadd132sd(extracta, extractb, extractc, $imm4) - }; - } - let fnmadd = constify_imm4_round!(rounding, call); + let fnmadd = vfmadd132sd(extracta, extractb, extractc, ROUNDING); let r = simd_insert(a, 0, fnmadd); transmute(r) } @@ -36880,26 +34356,21 @@ pub unsafe fn _mm_fnmadd_round_sd(a: __m128d, b: __m128d, c: __m128d, rounding: /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_mask_fnmadd_round_sd&expand=2736) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vfnmadd213sd, rounding = 8))] -#[rustc_args_required_const(4)] -pub unsafe fn _mm_mask_fnmadd_round_sd( +#[cfg_attr(test, assert_instr(vfnmadd213sd, ROUNDING = 8))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm_mask_fnmadd_round_sd( a: __m128d, k: __mmask8, b: __m128d, c: __m128d, - rounding: i32, ) -> __m128d { + static_assert_rounding!(ROUNDING); let mut fnmadd: f64 = simd_extract(a, 0); if (k & 0b00000001) != 0 { let extracta = -fnmadd; let extractb: f64 = simd_extract(b, 0); let extractc: f64 = simd_extract(c, 0); - macro_rules! call { - ($imm4:expr) => { - vfmadd132sd(extracta, extractb, extractc, $imm4) - }; - } - fnmadd = constify_imm4_round!(rounding, call); + fnmadd = vfmadd132sd(extracta, extractb, extractc, ROUNDING); } let r = simd_insert(a, 0, fnmadd); transmute(r) @@ -36917,27 +34388,22 @@ pub unsafe fn _mm_mask_fnmadd_round_sd( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_maskz_fnmadd_round_sd&expand=2738) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vfnmadd213sd, rounding = 8))] -#[rustc_args_required_const(4)] -pub unsafe fn _mm_maskz_fnmadd_round_sd( +#[cfg_attr(test, assert_instr(vfnmadd213sd, ROUNDING = 8))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm_maskz_fnmadd_round_sd( k: __mmask8, a: __m128d, b: __m128d, c: __m128d, - rounding: i32, ) -> __m128d { + static_assert_rounding!(ROUNDING); let mut fnmadd: f64 = 0.; if (k & 0b00000001) != 0 { let extracta: f64 = simd_extract(a, 0); let extracta = -extracta; let extractb: f64 = simd_extract(b, 0); let extractc: f64 = simd_extract(c, 0); - macro_rules! call { - ($imm4:expr) => { - vfmadd132sd(extracta, extractb, extractc, $imm4) - }; - } - fnmadd = constify_imm4_round!(rounding, call); + fnmadd = vfmadd132sd(extracta, extractb, extractc, ROUNDING); } let r = simd_insert(a, 0, fnmadd); transmute(r) @@ -36955,26 +34421,21 @@ pub unsafe fn _mm_maskz_fnmadd_round_sd( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_mask3_fnmadd_round_Sd&expand=2737) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vfnmadd213sd, rounding = 8))] -#[rustc_args_required_const(4)] -pub unsafe fn _mm_mask3_fnmadd_round_sd( +#[cfg_attr(test, assert_instr(vfnmadd213sd, ROUNDING = 8))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm_mask3_fnmadd_round_sd( a: __m128d, b: __m128d, c: __m128d, k: __mmask8, - rounding: i32, ) -> __m128d { + static_assert_rounding!(ROUNDING); let mut fnmadd: f64 = simd_extract(c, 0); if (k & 0b00000001) != 0 { let extracta: f64 = simd_extract(a, 0); let extracta = -extracta; let extractb: f64 = simd_extract(b, 0); - macro_rules! call { - ($imm4:expr) => { - vfmadd132sd(extracta, extractb, fnmadd, $imm4) - }; - } - fnmadd = constify_imm4_round!(rounding, call); + fnmadd = vfmadd132sd(extracta, extractb, fnmadd, ROUNDING); } let r = simd_insert(c, 0, fnmadd); transmute(r) @@ -36992,20 +34453,16 @@ pub unsafe fn _mm_mask3_fnmadd_round_sd( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_fnmsub_round_ss&expand=2787) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vfnmsub213ss, rounding = 8))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm_fnmsub_round_ss(a: __m128, b: __m128, c: __m128, rounding: i32) -> __m128 { +#[cfg_attr(test, assert_instr(vfnmsub213ss, ROUNDING = 8))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm_fnmsub_round_ss(a: __m128, b: __m128, c: __m128) -> __m128 { + static_assert_rounding!(ROUNDING); let extracta: f32 = simd_extract(a, 0); let extracta = -extracta; let extractb: f32 = simd_extract(b, 0); let extractc: f32 = simd_extract(c, 0); let extractc = -extractc; - macro_rules! call { - ($imm4:expr) => { - vfmadd132ss(extracta, extractb, extractc, $imm4) - }; - } - let fnmsub = constify_imm4_round!(rounding, call); + let fnmsub = vfmadd132ss(extracta, extractb, extractc, ROUNDING); let r = simd_insert(a, 0, fnmsub); transmute(r) } @@ -37022,27 +34479,22 @@ pub unsafe fn _mm_fnmsub_round_ss(a: __m128, b: __m128, c: __m128, rounding: i32 /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_mask_fnmsub_round_ss&expand=2788) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vfnmsub213ss, rounding = 8))] -#[rustc_args_required_const(4)] -pub unsafe fn _mm_mask_fnmsub_round_ss( +#[cfg_attr(test, assert_instr(vfnmsub213ss, ROUNDING = 8))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm_mask_fnmsub_round_ss( a: __m128, k: __mmask8, b: __m128, c: __m128, - rounding: i32, ) -> __m128 { + static_assert_rounding!(ROUNDING); let mut fnmsub: f32 = simd_extract(a, 0); if (k & 0b00000001) != 0 { let extracta = -fnmsub; let extractb: f32 = simd_extract(b, 0); let extractc: f32 = simd_extract(c, 0); let extractc = -extractc; - macro_rules! call { - ($imm4:expr) => { - vfmadd132ss(extracta, extractb, extractc, $imm4) - }; - } - fnmsub = constify_imm4_round!(rounding, call); + fnmsub = vfmadd132ss(extracta, extractb, extractc, ROUNDING); } let r = simd_insert(a, 0, fnmsub); transmute(r) @@ -37060,15 +34512,15 @@ pub unsafe fn _mm_mask_fnmsub_round_ss( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_maskz_fnmsub_round_ss&expand=2790) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vfnmsub213ss, rounding = 8))] -#[rustc_args_required_const(4)] -pub unsafe fn _mm_maskz_fnmsub_round_ss( +#[cfg_attr(test, assert_instr(vfnmsub213ss, ROUNDING = 8))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm_maskz_fnmsub_round_ss( k: __mmask8, a: __m128, b: __m128, c: __m128, - rounding: i32, ) -> __m128 { + static_assert_rounding!(ROUNDING); let mut fnmsub: f32 = 0.; if (k & 0b00000001) != 0 { let extracta: f32 = simd_extract(a, 0); @@ -37076,12 +34528,7 @@ pub unsafe fn _mm_maskz_fnmsub_round_ss( let extractb: f32 = simd_extract(b, 0); let extractc: f32 = simd_extract(c, 0); let extractc = -extractc; - macro_rules! call { - ($imm4:expr) => { - vfmadd132ss(extracta, extractb, extractc, $imm4) - }; - } - fnmsub = constify_imm4_round!(rounding, call); + fnmsub = vfmadd132ss(extracta, extractb, extractc, ROUNDING); } let r = simd_insert(a, 0, fnmsub); transmute(r) @@ -37099,27 +34546,22 @@ pub unsafe fn _mm_maskz_fnmsub_round_ss( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_mask3_fnmsub_round_ss&expand=2789) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vfnmsub213ss, rounding = 8))] -#[rustc_args_required_const(4)] -pub unsafe fn _mm_mask3_fnmsub_round_ss( +#[cfg_attr(test, assert_instr(vfnmsub213ss, ROUNDING = 8))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm_mask3_fnmsub_round_ss( a: __m128, b: __m128, c: __m128, k: __mmask8, - rounding: i32, ) -> __m128 { + static_assert_rounding!(ROUNDING); let mut fnmsub: f32 = simd_extract(c, 0); if (k & 0b00000001) != 0 { let extracta: f32 = simd_extract(a, 0); let extracta = -extracta; let extractb: f32 = simd_extract(b, 0); let extractc = -fnmsub; - macro_rules! call { - ($imm4:expr) => { - vfmadd132ss(extracta, extractb, extractc, $imm4) - }; - } - fnmsub = constify_imm4_round!(rounding, call); + fnmsub = vfmadd132ss(extracta, extractb, extractc, ROUNDING); } let r = simd_insert(c, 0, fnmsub); transmute(r) @@ -37137,20 +34579,20 @@ pub unsafe fn _mm_mask3_fnmsub_round_ss( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_fnmsub_round_sd&expand=2783) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vfnmsub213sd, rounding = 8))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm_fnmsub_round_sd(a: __m128d, b: __m128d, c: __m128d, rounding: i32) -> __m128d { +#[cfg_attr(test, assert_instr(vfnmsub213sd, ROUNDING = 8))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm_fnmsub_round_sd( + a: __m128d, + b: __m128d, + c: __m128d, +) -> __m128d { + static_assert_rounding!(ROUNDING); let extracta: f64 = simd_extract(a, 0); let extracta = -extracta; let extractb: f64 = simd_extract(b, 0); let extractc: f64 = simd_extract(c, 0); let extractc = -extractc; - macro_rules! call { - ($imm4:expr) => { - vfmadd132sd(extracta, extractb, extractc, $imm4) - }; - } - let fnmsub = constify_imm4_round!(rounding, call); + let fnmsub = vfmadd132sd(extracta, extractb, extractc, ROUNDING); let r = simd_insert(a, 0, fnmsub); transmute(r) } @@ -37167,27 +34609,22 @@ pub unsafe fn _mm_fnmsub_round_sd(a: __m128d, b: __m128d, c: __m128d, rounding: /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_mask_fnmsub_round_sd&expand=2784) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vfnmsub213sd, rounding = 8))] -#[rustc_args_required_const(4)] -pub unsafe fn _mm_mask_fnmsub_round_sd( +#[cfg_attr(test, assert_instr(vfnmsub213sd, ROUNDING = 8))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm_mask_fnmsub_round_sd( a: __m128d, k: __mmask8, b: __m128d, c: __m128d, - rounding: i32, ) -> __m128d { + static_assert_rounding!(ROUNDING); let mut fnmsub: f64 = simd_extract(a, 0); if (k & 0b00000001) != 0 { let extracta = -fnmsub; let extractb: f64 = simd_extract(b, 0); let extractc: f64 = simd_extract(c, 0); let extractc = -extractc; - macro_rules! call { - ($imm4:expr) => { - vfmadd132sd(extracta, extractb, extractc, $imm4) - }; - } - fnmsub = constify_imm4_round!(rounding, call); + fnmsub = vfmadd132sd(extracta, extractb, extractc, ROUNDING); } let r = simd_insert(a, 0, fnmsub); transmute(r) @@ -37205,15 +34642,15 @@ pub unsafe fn _mm_mask_fnmsub_round_sd( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_maskz_fnmsub_round_sd&expand=2786) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vfnmsub213sd, rounding = 8))] -#[rustc_args_required_const(4)] -pub unsafe fn _mm_maskz_fnmsub_round_sd( +#[cfg_attr(test, assert_instr(vfnmsub213sd, ROUNDING = 8))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm_maskz_fnmsub_round_sd( k: __mmask8, a: __m128d, b: __m128d, c: __m128d, - rounding: i32, ) -> __m128d { + static_assert_rounding!(ROUNDING); let mut fnmsub: f64 = 0.; if (k & 0b00000001) != 0 { let extracta: f64 = simd_extract(a, 0); @@ -37221,12 +34658,7 @@ pub unsafe fn _mm_maskz_fnmsub_round_sd( let extractb: f64 = simd_extract(b, 0); let extractc: f64 = simd_extract(c, 0); let extractc = -extractc; - macro_rules! call { - ($imm4:expr) => { - vfmadd132sd(extracta, extractb, extractc, $imm4) - }; - } - fnmsub = constify_imm4_round!(rounding, call); + fnmsub = vfmadd132sd(extracta, extractb, extractc, ROUNDING); } let r = simd_insert(a, 0, fnmsub); transmute(r) @@ -37244,27 +34676,22 @@ pub unsafe fn _mm_maskz_fnmsub_round_sd( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_mask3_fnmsub_round_sd&expand=2785) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vfnmsub213sd, rounding = 8))] -#[rustc_args_required_const(4)] -pub unsafe fn _mm_mask3_fnmsub_round_sd( +#[cfg_attr(test, assert_instr(vfnmsub213sd, ROUNDING = 8))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm_mask3_fnmsub_round_sd( a: __m128d, b: __m128d, c: __m128d, k: __mmask8, - rounding: i32, ) -> __m128d { + static_assert_rounding!(ROUNDING); let mut fnmsub: f64 = simd_extract(c, 0); if (k & 0b00000001) != 0 { let extracta: f64 = simd_extract(a, 0); let extracta = -extracta; let extractb: f64 = simd_extract(b, 0); let extractc = -fnmsub; - macro_rules! call { - ($imm4:expr) => { - vfmadd132sd(extracta, extractb, extractc, $imm4) - }; - } - fnmsub = constify_imm4_round!(rounding, call); + fnmsub = vfmadd132sd(extracta, extractb, extractc, ROUNDING); } let r = simd_insert(c, 0, fnmsub); transmute(r) @@ -37275,19 +34702,15 @@ pub unsafe fn _mm_mask3_fnmsub_round_sd( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_fixupimm_ss&expand=2517) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vfixupimmss, imm8 = 0))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm_fixupimm_ss(a: __m128, b: __m128, c: __m128i, imm8: i32) -> __m128 { +#[cfg_attr(test, assert_instr(vfixupimmss, IMM8 = 0))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm_fixupimm_ss(a: __m128, b: __m128, c: __m128i) -> __m128 { + static_assert_imm8!(IMM8); let a = a.as_f32x4(); let b = b.as_f32x4(); let c = c.as_i32x4(); - macro_rules! call { - ($imm8:expr) => { - vfixupimmss(a, b, c, $imm8, 0b11111111, _MM_FROUND_CUR_DIRECTION) - }; - } - let fixupimm = constify_imm8_sae!(imm8, call); - let fixupimm: f32 = simd_extract(fixupimm, 0); + let r = vfixupimmss(a, b, c, IMM8, 0b11111111, _MM_FROUND_CUR_DIRECTION); + let fixupimm: f32 = simd_extract(r, 0); let r = simd_insert(a, 0, fixupimm); transmute(r) } @@ -37297,24 +34720,19 @@ pub unsafe fn _mm_fixupimm_ss(a: __m128, b: __m128, c: __m128i, imm8: i32) -> __ /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_mask_fixupimm_ss&expand=2518) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vfixupimmss, imm8 = 0))] -#[rustc_args_required_const(4)] -pub unsafe fn _mm_mask_fixupimm_ss( +#[cfg_attr(test, assert_instr(vfixupimmss, IMM8 = 0))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm_mask_fixupimm_ss( a: __m128, k: __mmask8, b: __m128, c: __m128i, - imm8: i32, ) -> __m128 { + static_assert_imm8!(IMM8); let a = a.as_f32x4(); let b = b.as_f32x4(); let c = c.as_i32x4(); - macro_rules! call { - ($imm8:expr) => { - vfixupimmss(a, b, c, $imm8, k, _MM_FROUND_CUR_DIRECTION) - }; - } - let fixupimm = constify_imm8_sae!(imm8, call); + let fixupimm = vfixupimmss(a, b, c, IMM8, k, _MM_FROUND_CUR_DIRECTION); let fixupimm: f32 = simd_extract(fixupimm, 0); let r = simd_insert(a, 0, fixupimm); transmute(r) @@ -37325,24 +34743,19 @@ pub unsafe fn _mm_mask_fixupimm_ss( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_maskz_fixupimm_ss&expand=2519) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vfixupimmss, imm8 = 0))] -#[rustc_args_required_const(4)] -pub unsafe fn _mm_maskz_fixupimm_ss( +#[cfg_attr(test, assert_instr(vfixupimmss, IMM8 = 0))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm_maskz_fixupimm_ss( k: __mmask8, a: __m128, b: __m128, c: __m128i, - imm8: i32, ) -> __m128 { + static_assert_imm8!(IMM8); let a = a.as_f32x4(); let b = b.as_f32x4(); let c = c.as_i32x4(); - macro_rules! call { - ($imm8:expr) => { - vfixupimmssz(a, b, c, $imm8, k, _MM_FROUND_CUR_DIRECTION) - }; - } - let fixupimm = constify_imm8_sae!(imm8, call); + let fixupimm = vfixupimmssz(a, b, c, IMM8, k, _MM_FROUND_CUR_DIRECTION); let fixupimm: f32 = simd_extract(fixupimm, 0); let r = simd_insert(a, 0, fixupimm); transmute(r) @@ -37353,18 +34766,14 @@ pub unsafe fn _mm_maskz_fixupimm_ss( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_fixupimm_sd&expand=2514) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vfixupimmsd, imm8 = 0))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm_fixupimm_sd(a: __m128d, b: __m128d, c: __m128i, imm8: i32) -> __m128d { +#[cfg_attr(test, assert_instr(vfixupimmsd, IMM8 = 0))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm_fixupimm_sd(a: __m128d, b: __m128d, c: __m128i) -> __m128d { + static_assert_imm8!(IMM8); let a = a.as_f64x2(); let b = b.as_f64x2(); let c = c.as_i64x2(); - macro_rules! call { - ($imm8:expr) => { - vfixupimmsd(a, b, c, $imm8, 0b11111111, _MM_FROUND_CUR_DIRECTION) - }; - } - let fixupimm = constify_imm8_sae!(imm8, call); + let fixupimm = vfixupimmsd(a, b, c, IMM8, 0b11111111, _MM_FROUND_CUR_DIRECTION); let fixupimm: f64 = simd_extract(fixupimm, 0); let r = simd_insert(a, 0, fixupimm); transmute(r) @@ -37375,24 +34784,19 @@ pub unsafe fn _mm_fixupimm_sd(a: __m128d, b: __m128d, c: __m128i, imm8: i32) -> /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_mask_fixupimm_sd&expand=2515) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vfixupimmsd, imm8 = 0))] -#[rustc_args_required_const(4)] -pub unsafe fn _mm_mask_fixupimm_sd( +#[cfg_attr(test, assert_instr(vfixupimmsd, IMM8 = 0))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm_mask_fixupimm_sd( a: __m128d, k: __mmask8, b: __m128d, c: __m128i, - imm8: i32, ) -> __m128d { + static_assert_imm8!(IMM8); let a = a.as_f64x2(); let b = b.as_f64x2(); let c = c.as_i64x2(); - macro_rules! call { - ($imm8:expr) => { - vfixupimmsd(a, b, c, $imm8, k, _MM_FROUND_CUR_DIRECTION) - }; - } - let fixupimm = constify_imm8_sae!(imm8, call); + let fixupimm = vfixupimmsd(a, b, c, IMM8, k, _MM_FROUND_CUR_DIRECTION); let fixupimm: f64 = simd_extract(fixupimm, 0); let r = simd_insert(a, 0, fixupimm); transmute(r) @@ -37403,24 +34807,19 @@ pub unsafe fn _mm_mask_fixupimm_sd( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_maskz_fixupimm_sd&expand=2516) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vfixupimmsd, imm8 = 0))] -#[rustc_args_required_const(4)] -pub unsafe fn _mm_maskz_fixupimm_sd( +#[cfg_attr(test, assert_instr(vfixupimmsd, IMM8 = 0))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm_maskz_fixupimm_sd( k: __mmask8, a: __m128d, b: __m128d, c: __m128i, - imm8: i32, ) -> __m128d { + static_assert_imm8!(IMM8); let a = a.as_f64x2(); let b = b.as_f64x2(); let c = c.as_i64x2(); - macro_rules! call { - ($imm8:expr) => { - vfixupimmsdz(a, b, c, $imm8, k, _MM_FROUND_CUR_DIRECTION) - }; - } - let fixupimm = constify_imm8_sae!(imm8, call); + let fixupimm = vfixupimmsdz(a, b, c, IMM8, k, _MM_FROUND_CUR_DIRECTION); let fixupimm: f64 = simd_extract(fixupimm, 0); let r = simd_insert(a, 0, fixupimm); transmute(r) @@ -37432,25 +34831,20 @@ pub unsafe fn _mm_maskz_fixupimm_sd( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_fixupimm_round_ss&expand=2511) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vfixupimmss, imm8 = 0, sae = 8))] -#[rustc_args_required_const(3, 4)] -pub unsafe fn _mm_fixupimm_round_ss( +#[cfg_attr(test, assert_instr(vfixupimmss, IMM8 = 0, SAE = 8))] +#[rustc_legacy_const_generics(3, 4)] +pub unsafe fn _mm_fixupimm_round_ss( a: __m128, b: __m128, c: __m128i, - imm8: i32, - sae: i32, ) -> __m128 { + static_assert_imm8!(IMM8); + static_assert_mantissas_sae!(SAE); let a = a.as_f32x4(); let b = b.as_f32x4(); let c = c.as_i32x4(); - macro_rules! call { - ($imm8:expr, $imm4:expr) => { - vfixupimmss(a, b, c, $imm8, 0b11111111, $imm4) - }; - } - let fixupimm = constify_imm8_roundscale!(imm8, sae, call); - let fixupimm: f32 = simd_extract(fixupimm, 0); + let r = vfixupimmss(a, b, c, IMM8, 0b11111111, SAE); + let fixupimm: f32 = simd_extract(r, 0); let r = simd_insert(a, 0, fixupimm); transmute(r) } @@ -37461,26 +34855,21 @@ pub unsafe fn _mm_fixupimm_round_ss( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_mask_fixupimm_round_ss&expand=2512) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vfixupimmss, imm8 = 0, sae = 8))] -#[rustc_args_required_const(4, 5)] -pub unsafe fn _mm_mask_fixupimm_round_ss( +#[cfg_attr(test, assert_instr(vfixupimmss, IMM8 = 0, SAE = 8))] +#[rustc_legacy_const_generics(4, 5)] +pub unsafe fn _mm_mask_fixupimm_round_ss( a: __m128, k: __mmask8, b: __m128, c: __m128i, - imm8: i32, - sae: i32, ) -> __m128 { + static_assert_imm8!(IMM8); + static_assert_mantissas_sae!(SAE); let a = a.as_f32x4(); let b = b.as_f32x4(); let c = c.as_i32x4(); - macro_rules! call { - ($imm8:expr, $imm4:expr) => { - vfixupimmss(a, b, c, $imm8, k, $imm4) - }; - } - let fixupimm = constify_imm8_roundscale!(imm8, sae, call); - let fixupimm: f32 = simd_extract(fixupimm, 0); + let r = vfixupimmss(a, b, c, IMM8, k, SAE); + let fixupimm: f32 = simd_extract(r, 0); let r = simd_insert(a, 0, fixupimm); transmute(r) } @@ -37491,26 +34880,21 @@ pub unsafe fn _mm_mask_fixupimm_round_ss( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_maskz_fixupimm_round_ss&expand=2513) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vfixupimmss, imm8 = 0, sae = 8))] -#[rustc_args_required_const(4, 5)] -pub unsafe fn _mm_maskz_fixupimm_round_ss( +#[cfg_attr(test, assert_instr(vfixupimmss, IMM8 = 0, SAE = 8))] +#[rustc_legacy_const_generics(4, 5)] +pub unsafe fn _mm_maskz_fixupimm_round_ss( k: __mmask8, a: __m128, b: __m128, c: __m128i, - imm8: i32, - sae: i32, ) -> __m128 { + static_assert_imm8!(IMM8); + static_assert_mantissas_sae!(SAE); let a = a.as_f32x4(); let b = b.as_f32x4(); let c = c.as_i32x4(); - macro_rules! call { - ($imm8:expr, $imm4:expr) => { - vfixupimmssz(a, b, c, $imm8, k, $imm4) - }; - } - let fixupimm = constify_imm8_roundscale!(imm8, sae, call); - let fixupimm: f32 = simd_extract(fixupimm, 0); + let r = vfixupimmssz(a, b, c, IMM8, k, SAE); + let fixupimm: f32 = simd_extract(r, 0); let r = simd_insert(a, 0, fixupimm); transmute(r) } @@ -37521,25 +34905,20 @@ pub unsafe fn _mm_maskz_fixupimm_round_ss( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_fixupimm_round_sd&expand=2508) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vfixupimmsd, imm8 = 0, sae = 8))] -#[rustc_args_required_const(3, 4)] -pub unsafe fn _mm_fixupimm_round_sd( +#[cfg_attr(test, assert_instr(vfixupimmsd, IMM8 = 0, SAE = 8))] +#[rustc_legacy_const_generics(3, 4)] +pub unsafe fn _mm_fixupimm_round_sd( a: __m128d, b: __m128d, c: __m128i, - imm8: i32, - sae: i32, ) -> __m128d { + static_assert_imm8!(IMM8); + static_assert_mantissas_sae!(SAE); let a = a.as_f64x2(); let b = b.as_f64x2(); let c = c.as_i64x2(); - macro_rules! call { - ($imm8:expr, $imm4:expr) => { - vfixupimmsd(a, b, c, $imm8, 0b11111111, $imm4) - }; - } - let fixupimm = constify_imm8_roundscale!(imm8, sae, call); - let fixupimm: f64 = simd_extract(fixupimm, 0); + let r = vfixupimmsd(a, b, c, IMM8, 0b11111111, SAE); + let fixupimm: f64 = simd_extract(r, 0); let r = simd_insert(a, 0, fixupimm); transmute(r) } @@ -37550,26 +34929,21 @@ pub unsafe fn _mm_fixupimm_round_sd( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_mask_fixupimm_round_sd&expand=2509) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vfixupimmsd, imm8 = 0, sae = 8))] -#[rustc_args_required_const(4, 5)] -pub unsafe fn _mm_mask_fixupimm_round_sd( +#[cfg_attr(test, assert_instr(vfixupimmsd, IMM8 = 0, SAE = 8))] +#[rustc_legacy_const_generics(4, 5)] +pub unsafe fn _mm_mask_fixupimm_round_sd( a: __m128d, k: __mmask8, b: __m128d, c: __m128i, - imm8: i32, - sae: i32, ) -> __m128d { + static_assert_imm8!(IMM8); + static_assert_mantissas_sae!(SAE); let a = a.as_f64x2(); let b = b.as_f64x2(); let c = c.as_i64x2(); - macro_rules! call { - ($imm8:expr, $imm4:expr) => { - vfixupimmsd(a, b, c, $imm8, k, $imm4) - }; - } - let fixupimm = constify_imm8_roundscale!(imm8, sae, call); - let fixupimm: f64 = simd_extract(fixupimm, 0); + let r = vfixupimmsd(a, b, c, IMM8, k, SAE); + let fixupimm: f64 = simd_extract(r, 0); let r = simd_insert(a, 0, fixupimm); transmute(r) } @@ -37580,26 +34954,21 @@ pub unsafe fn _mm_mask_fixupimm_round_sd( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_maskz_fixupimm_round_sd&expand=2510) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vfixupimmsd, imm8 = 0, sae = 8))] -#[rustc_args_required_const(4, 5)] -pub unsafe fn _mm_maskz_fixupimm_round_sd( +#[cfg_attr(test, assert_instr(vfixupimmsd, IMM8 = 0, SAE = 8))] +#[rustc_legacy_const_generics(4, 5)] +pub unsafe fn _mm_maskz_fixupimm_round_sd( k: __mmask8, a: __m128d, b: __m128d, c: __m128i, - imm8: i32, - sae: i32, ) -> __m128d { + static_assert_imm8!(IMM8); + static_assert_mantissas_sae!(SAE); let a = a.as_f64x2(); let b = b.as_f64x2(); let c = c.as_i64x2(); - macro_rules! call { - ($imm8:expr, $imm4:expr) => { - vfixupimmsdz(a, b, c, $imm8, k, $imm4) - }; - } - let fixupimm = constify_imm8_roundscale!(imm8, sae, call); - let fixupimm: f64 = simd_extract(fixupimm, 0); + let r = vfixupimmsdz(a, b, c, IMM8, k, SAE); + let fixupimm: f64 = simd_extract(r, 0); let r = simd_insert(a, 0, fixupimm); transmute(r) } @@ -37674,21 +35043,14 @@ pub unsafe fn _mm_maskz_cvtsd_ss(k: __mmask8, a: __m128, b: __m128d) -> __m128 { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_cvt_roundss_sd&expand=1371) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vcvtss2sd, sae = 8))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm_cvt_roundss_sd(a: __m128d, b: __m128, sae: i32) -> __m128d { - macro_rules! call { - ($imm4:expr) => { - vcvtss2sd( - a.as_f64x2(), - b.as_f32x4(), - _mm_setzero_pd().as_f64x2(), - 0b11111111, - $imm4, - ) - }; - } - let r = constify_imm4_sae!(sae, call); +#[cfg_attr(test, assert_instr(vcvtss2sd, SAE = 8))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm_cvt_roundss_sd(a: __m128d, b: __m128) -> __m128d { + static_assert_sae!(SAE); + let a = a.as_f64x2(); + let b = b.as_f32x4(); + let zero = _mm_setzero_pd().as_f64x2(); + let r = vcvtss2sd(a, b, zero, 0b11111111, SAE); transmute(r) } @@ -37698,24 +35060,19 @@ pub unsafe fn _mm_cvt_roundss_sd(a: __m128d, b: __m128, sae: i32) -> __m128d { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_mask_cvt_roundss_sd&expand=1372) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vcvtss2sd, sae = 8))] -#[rustc_args_required_const(4)] -pub unsafe fn _mm_mask_cvt_roundss_sd( +#[cfg_attr(test, assert_instr(vcvtss2sd, SAE = 8))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm_mask_cvt_roundss_sd( src: __m128d, k: __mmask8, a: __m128d, b: __m128, - sae: i32, ) -> __m128d { + static_assert_sae!(SAE); let a = a.as_f64x2(); let b = b.as_f32x4(); let src = src.as_f64x2(); - macro_rules! call { - ($imm4:expr) => { - vcvtss2sd(a, b, src, k, $imm4) - }; - } - let r = constify_imm4_sae!(sae, call); + let r = vcvtss2sd(a, b, src, k, SAE); transmute(r) } @@ -37725,18 +35082,18 @@ pub unsafe fn _mm_mask_cvt_roundss_sd( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_maskz_cvt_roundss_sd&expand=1373) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vcvtss2sd, sae = 8))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm_maskz_cvt_roundss_sd(k: __mmask8, a: __m128d, b: __m128, sae: i32) -> __m128d { +#[cfg_attr(test, assert_instr(vcvtss2sd, SAE = 8))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm_maskz_cvt_roundss_sd( + k: __mmask8, + a: __m128d, + b: __m128, +) -> __m128d { + static_assert_sae!(SAE); let a = a.as_f64x2(); let b = b.as_f32x4(); let zero = _mm_setzero_pd().as_f64x2(); - macro_rules! call { - ($imm4:expr) => { - vcvtss2sd(a, b, zero, k, $imm4) - }; - } - let r = constify_imm4_sae!(sae, call); + let r = vcvtss2sd(a, b, zero, k, SAE); transmute(r) } @@ -37751,18 +35108,14 @@ pub unsafe fn _mm_maskz_cvt_roundss_sd(k: __mmask8, a: __m128d, b: __m128, sae: /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_cvt_roundsd_ss&expand=1361) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vcvtsd2ss, rounding = 8))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm_cvt_roundsd_ss(a: __m128, b: __m128d, rounding: i32) -> __m128 { +#[cfg_attr(test, assert_instr(vcvtsd2ss, ROUNDING = 8))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm_cvt_roundsd_ss(a: __m128, b: __m128d) -> __m128 { + static_assert_rounding!(ROUNDING); let a = a.as_f32x4(); let b = b.as_f64x2(); let zero = _mm_setzero_ps().as_f32x4(); - macro_rules! call { - ($imm4:expr) => { - vcvtsd2ss(a, b, zero, 0b11111111, $imm4) - }; - } - let r = constify_imm4_round!(rounding, call); + let r = vcvtsd2ss(a, b, zero, 0b11111111, ROUNDING); transmute(r) } @@ -37777,24 +35130,19 @@ pub unsafe fn _mm_cvt_roundsd_ss(a: __m128, b: __m128d, rounding: i32) -> __m128 /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_mask_cvt_roundsd_ss&expand=1362) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vcvtsd2ss, rounding = 8))] -#[rustc_args_required_const(4)] -pub unsafe fn _mm_mask_cvt_roundsd_ss( +#[cfg_attr(test, assert_instr(vcvtsd2ss, ROUNDING = 8))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm_mask_cvt_roundsd_ss( src: __m128, k: __mmask8, a: __m128, b: __m128d, - rounding: i32, ) -> __m128 { + static_assert_rounding!(ROUNDING); let a = a.as_f32x4(); let b = b.as_f64x2(); let src = src.as_f32x4(); - macro_rules! call { - ($imm4:expr) => { - vcvtsd2ss(a, b, src, k, $imm4) - }; - } - let r = constify_imm4_round!(rounding, call); + let r = vcvtsd2ss(a, b, src, k, ROUNDING); transmute(r) } @@ -37809,23 +35157,18 @@ pub unsafe fn _mm_mask_cvt_roundsd_ss( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_maskz_cvt_roundsd_ss&expand=1363) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vcvtsd2ss, rounding = 8))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm_maskz_cvt_roundsd_ss( +#[cfg_attr(test, assert_instr(vcvtsd2ss, ROUNDING = 8))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm_maskz_cvt_roundsd_ss( k: __mmask8, a: __m128, b: __m128d, - rounding: i32, ) -> __m128 { + static_assert_rounding!(ROUNDING); let a = a.as_f32x4(); let b = b.as_f64x2(); let zero = _mm_setzero_ps().as_f32x4(); - macro_rules! call { - ($imm4:expr) => { - vcvtsd2ss(a, b, zero, k, $imm4) - }; - } - let r = constify_imm4_round!(rounding, call); + let r = vcvtsd2ss(a, b, zero, k, ROUNDING); transmute(r) } @@ -37840,16 +35183,12 @@ pub unsafe fn _mm_maskz_cvt_roundsd_ss( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_cvt_roundss_si32&expand=1374) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vcvtss2si, rounding = 8))] -#[rustc_args_required_const(1)] -pub unsafe fn _mm_cvt_roundss_si32(a: __m128, rounding: i32) -> i32 { +#[cfg_attr(test, assert_instr(vcvtss2si, ROUNDING = 8))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm_cvt_roundss_si32(a: __m128) -> i32 { + static_assert_rounding!(ROUNDING); let a = a.as_f32x4(); - macro_rules! call { - ($imm4:expr) => { - vcvtss2si(a, $imm4) - }; - } - let r = constify_imm4_round!(rounding, call); + let r = vcvtss2si(a, ROUNDING); transmute(r) } @@ -37864,16 +35203,12 @@ pub unsafe fn _mm_cvt_roundss_si32(a: __m128, rounding: i32) -> i32 { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_cvt_roundss_i32&expand=1369) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vcvtss2si, rounding = 8))] -#[rustc_args_required_const(1)] -pub unsafe fn _mm_cvt_roundss_i32(a: __m128, rounding: i32) -> i32 { +#[cfg_attr(test, assert_instr(vcvtss2si, ROUNDING = 8))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm_cvt_roundss_i32(a: __m128) -> i32 { + static_assert_rounding!(ROUNDING); let a = a.as_f32x4(); - macro_rules! call { - ($imm4:expr) => { - vcvtss2si(a, $imm4) - }; - } - let r = constify_imm4_round!(rounding, call); + let r = vcvtss2si(a, ROUNDING); transmute(r) } @@ -37888,16 +35223,12 @@ pub unsafe fn _mm_cvt_roundss_i32(a: __m128, rounding: i32) -> i32 { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_cvt_roundss_u32&expand=1376) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vcvtss2usi, rounding = 8))] -#[rustc_args_required_const(1)] -pub unsafe fn _mm_cvt_roundss_u32(a: __m128, rounding: i32) -> u32 { +#[cfg_attr(test, assert_instr(vcvtss2usi, ROUNDING = 8))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm_cvt_roundss_u32(a: __m128) -> u32 { + static_assert_rounding!(ROUNDING); let a = a.as_f32x4(); - macro_rules! call { - ($imm4:expr) => { - vcvtss2usi(a, $imm4) - }; - } - let r = constify_imm4_round!(rounding, call); + let r = vcvtss2usi(a, ROUNDING); transmute(r) } @@ -37932,16 +35263,12 @@ pub unsafe fn _mm_cvtss_u32(a: __m128) -> u32 { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_cvt_roundsd_si32&expand=1359) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vcvtsd2si, rounding = 8))] -#[rustc_args_required_const(1)] -pub unsafe fn _mm_cvt_roundsd_si32(a: __m128d, rounding: i32) -> i32 { +#[cfg_attr(test, assert_instr(vcvtsd2si, ROUNDING = 8))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm_cvt_roundsd_si32(a: __m128d) -> i32 { + static_assert_rounding!(ROUNDING); let a = a.as_f64x2(); - macro_rules! call { - ($imm4:expr) => { - vcvtsd2si(a, $imm4) - }; - } - let r = constify_imm4_round!(rounding, call); + let r = vcvtsd2si(a, ROUNDING); transmute(r) } @@ -37956,16 +35283,12 @@ pub unsafe fn _mm_cvt_roundsd_si32(a: __m128d, rounding: i32) -> i32 { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_cvt_roundsd_i32&expand=1357) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vcvtsd2si, rounding = 8))] -#[rustc_args_required_const(1)] -pub unsafe fn _mm_cvt_roundsd_i32(a: __m128d, rounding: i32) -> i32 { +#[cfg_attr(test, assert_instr(vcvtsd2si, ROUNDING = 8))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm_cvt_roundsd_i32(a: __m128d) -> i32 { + static_assert_rounding!(ROUNDING); let a = a.as_f64x2(); - macro_rules! call { - ($imm4:expr) => { - vcvtsd2si(a, $imm4) - }; - } - let r = constify_imm4_round!(rounding, call); + let r = vcvtsd2si(a, ROUNDING); transmute(r) } @@ -37980,16 +35303,12 @@ pub unsafe fn _mm_cvt_roundsd_i32(a: __m128d, rounding: i32) -> i32 { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=cvt_roundsd_u32&expand=1364) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vcvtsd2usi, rounding = 8))] -#[rustc_args_required_const(1)] -pub unsafe fn _mm_cvt_roundsd_u32(a: __m128d, rounding: i32) -> u32 { +#[cfg_attr(test, assert_instr(vcvtsd2usi, ROUNDING = 8))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm_cvt_roundsd_u32(a: __m128d) -> u32 { + static_assert_rounding!(ROUNDING); let a = a.as_f64x2(); - macro_rules! call { - ($imm4:expr) => { - vcvtsd2usi(a, $imm4) - }; - } - let r = constify_imm4_round!(rounding, call); + let r = vcvtsd2usi(a, ROUNDING); transmute(r) } @@ -38025,16 +35344,12 @@ pub unsafe fn _mm_cvtsd_u32(a: __m128d) -> u32 { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_cvt_roundi32_ss&expand=1312) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vcvtsi2ss, rounding = 8))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm_cvt_roundi32_ss(a: __m128, b: i32, rounding: i32) -> __m128 { +#[cfg_attr(test, assert_instr(vcvtsi2ss, ROUNDING = 8))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm_cvt_roundi32_ss(a: __m128, b: i32) -> __m128 { + static_assert_rounding!(ROUNDING); let a = a.as_f32x4(); - macro_rules! call { - ($imm4:expr) => { - vcvtsi2ss(a, b, $imm4) - }; - } - let r = constify_imm4_round!(rounding, call); + let r = vcvtsi2ss(a, b, ROUNDING); transmute(r) } @@ -38050,16 +35365,12 @@ pub unsafe fn _mm_cvt_roundi32_ss(a: __m128, b: i32, rounding: i32) -> __m128 { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_cvt_roundsi32_ss&expand=1366) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vcvtsi2ss, rounding = 8))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm_cvt_roundsi32_ss(a: __m128, b: i32, rounding: i32) -> __m128 { +#[cfg_attr(test, assert_instr(vcvtsi2ss, ROUNDING = 8))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm_cvt_roundsi32_ss(a: __m128, b: i32) -> __m128 { + static_assert_rounding!(ROUNDING); let a = a.as_f32x4(); - macro_rules! call { - ($imm4:expr) => { - vcvtsi2ss(a, b, $imm4) - }; - } - let r = constify_imm4_round!(rounding, call); + let r = vcvtsi2ss(a, b, ROUNDING); transmute(r) } @@ -38074,16 +35385,12 @@ pub unsafe fn _mm_cvt_roundsi32_ss(a: __m128, b: i32, rounding: i32) -> __m128 { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_cvt_roundu32_ss&expand=1378) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vcvtusi2ss, rounding = 8))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm_cvt_roundu32_ss(a: __m128, b: u32, rounding: i32) -> __m128 { +#[cfg_attr(test, assert_instr(vcvtusi2ss, ROUNDING = 8))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm_cvt_roundu32_ss(a: __m128, b: u32) -> __m128 { + static_assert_rounding!(ROUNDING); let a = a.as_f32x4(); - macro_rules! call { - ($imm4:expr) => { - vcvtusi2ss(a, b, $imm4) - }; - } - let r = constify_imm4_round!(rounding, call); + let r = vcvtusi2ss(a, b, ROUNDING); transmute(r) } @@ -38117,16 +35424,12 @@ pub unsafe fn _mm_cvti32_sd(a: __m128d, b: i32) -> __m128d { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_cvtt_roundss_Si32&expand=1936) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vcvtss2si, sae = 8))] -#[rustc_args_required_const(1)] -pub unsafe fn _mm_cvtt_roundss_si32(a: __m128, sae: i32) -> i32 { +#[cfg_attr(test, assert_instr(vcvtss2si, SAE = 8))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm_cvtt_roundss_si32(a: __m128) -> i32 { + static_assert_sae!(SAE); let a = a.as_f32x4(); - macro_rules! call { - ($imm4:expr) => { - vcvtss2si(a, $imm4) - }; - } - let r = constify_imm4_sae!(sae, call); + let r = vcvtss2si(a, SAE); transmute(r) } @@ -38136,16 +35439,12 @@ pub unsafe fn _mm_cvtt_roundss_si32(a: __m128, sae: i32) -> i32 { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_cvtt_roundss_i32&expand=1934) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vcvtss2si, sae = 8))] -#[rustc_args_required_const(1)] -pub unsafe fn _mm_cvtt_roundss_i32(a: __m128, sae: i32) -> i32 { +#[cfg_attr(test, assert_instr(vcvtss2si, SAE = 8))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm_cvtt_roundss_i32(a: __m128) -> i32 { + static_assert_sae!(SAE); let a = a.as_f32x4(); - macro_rules! call { - ($imm4:expr) => { - vcvtss2si(a, $imm4) - }; - } - let r = constify_imm4_sae!(sae, call); + let r = vcvtss2si(a, SAE); transmute(r) } @@ -38155,16 +35454,12 @@ pub unsafe fn _mm_cvtt_roundss_i32(a: __m128, sae: i32) -> i32 { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_cvtt_roundss_u32&expand=1938) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vcvtss2usi, sae = 8))] -#[rustc_args_required_const(1)] -pub unsafe fn _mm_cvtt_roundss_u32(a: __m128, sae: i32) -> u32 { +#[cfg_attr(test, assert_instr(vcvtss2usi, SAE = 8))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm_cvtt_roundss_u32(a: __m128) -> u32 { + static_assert_sae!(SAE); let a = a.as_f32x4(); - macro_rules! call { - ($imm4:expr) => { - vcvtss2usi(a, $imm4) - }; - } - let r = constify_imm4_sae!(sae, call); + let r = vcvtss2usi(a, SAE); transmute(r) } @@ -38194,16 +35489,12 @@ pub unsafe fn _mm_cvttss_u32(a: __m128) -> u32 { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cvtt_roundsd_si32&expand=1930) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vcvtsd2si, sae = 8))] -#[rustc_args_required_const(1)] -pub unsafe fn _mm_cvtt_roundsd_si32(a: __m128d, sae: i32) -> i32 { +#[cfg_attr(test, assert_instr(vcvtsd2si, SAE = 8))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm_cvtt_roundsd_si32(a: __m128d) -> i32 { + static_assert_sae!(SAE); let a = a.as_f64x2(); - macro_rules! call { - ($imm4:expr) => { - vcvtsd2si(a, $imm4) - }; - } - let r = constify_imm4_sae!(sae, call); + let r = vcvtsd2si(a, SAE); transmute(r) } @@ -38213,16 +35504,12 @@ pub unsafe fn _mm_cvtt_roundsd_si32(a: __m128d, sae: i32) -> i32 { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cvtt_roundsd_i32&expand=1928) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vcvtsd2si, sae = 8))] -#[rustc_args_required_const(1)] -pub unsafe fn _mm_cvtt_roundsd_i32(a: __m128d, sae: i32) -> i32 { +#[cfg_attr(test, assert_instr(vcvtsd2si, SAE = 8))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm_cvtt_roundsd_i32(a: __m128d) -> i32 { + static_assert_sae!(SAE); let a = a.as_f64x2(); - macro_rules! call { - ($imm4:expr) => { - vcvtsd2si(a, $imm4) - }; - } - let r = constify_imm4_sae!(sae, call); + let r = vcvtsd2si(a, SAE); transmute(r) } @@ -38232,16 +35519,12 @@ pub unsafe fn _mm_cvtt_roundsd_i32(a: __m128d, sae: i32) -> i32 { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_cvtt_roundsd_u32&expand=1932) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vcvtsd2usi, sae = 8))] -#[rustc_args_required_const(1)] -pub unsafe fn _mm_cvtt_roundsd_u32(a: __m128d, sae: i32) -> u32 { +#[cfg_attr(test, assert_instr(vcvtsd2usi, SAE = 8))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm_cvtt_roundsd_u32(a: __m128d) -> u32 { + static_assert_sae!(SAE); let a = a.as_f64x2(); - macro_rules! call { - ($imm4:expr) => { - vcvtsd2usi(a, $imm4) - }; - } - let r = constify_imm4_sae!(sae, call); + let r = vcvtsd2usi(a, SAE); transmute(r) } @@ -38295,17 +35578,14 @@ pub unsafe fn _mm_cvtu32_sd(a: __m128d, b: u32) -> __m128d { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_comi_round_ss&expand=1175) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vcmp, imm8 = 5, sae = 4))] //should be vcomiss -#[rustc_args_required_const(2, 3)] -pub unsafe fn _mm_comi_round_ss(a: __m128, b: __m128, imm8: i32, sae: i32) -> i32 { +#[cfg_attr(test, assert_instr(vcmp, IMM5 = 5, SAE = 4))] //should be vcomiss +#[rustc_legacy_const_generics(2, 3)] +pub unsafe fn _mm_comi_round_ss(a: __m128, b: __m128) -> i32 { + static_assert_imm5!(IMM5); + static_assert_mantissas_sae!(SAE); let a = a.as_f32x4(); let b = b.as_f32x4(); - macro_rules! call { - ($imm8:expr, $imm4:expr) => { - vcomiss(a, b, $imm8, $imm4) - }; - } - let r = constify_imm5_sae!(imm8, sae, call); + let r = vcomiss(a, b, IMM5, SAE); transmute(r) } @@ -38315,17 +35595,14 @@ pub unsafe fn _mm_comi_round_ss(a: __m128, b: __m128, imm8: i32, sae: i32) -> i3 /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_comi_round_sd&expand=1174) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vcmp, imm8 = 5, sae = 4))] //should be vcomisd -#[rustc_args_required_const(2, 3)] -pub unsafe fn _mm_comi_round_sd(a: __m128d, b: __m128d, imm8: i32, sae: i32) -> i32 { +#[cfg_attr(test, assert_instr(vcmp, IMM5 = 5, SAE = 4))] //should be vcomisd +#[rustc_legacy_const_generics(2, 3)] +pub unsafe fn _mm_comi_round_sd(a: __m128d, b: __m128d) -> i32 { + static_assert_imm5!(IMM5); + static_assert_mantissas_sae!(SAE); let a = a.as_f64x2(); let b = b.as_f64x2(); - macro_rules! call { - ($imm8:expr, $imm4:expr) => { - vcomisd(a, b, $imm8, $imm4) - }; - } - let r = constify_imm5_sae!(imm8, sae, call); + let r = vcomisd(a, b, IMM5, SAE); transmute(r) } @@ -39252,13 +36529,36 @@ extern "C" { #[link_name = "llvm.x86.avx512.pslli.d.512"] fn vpsllid(a: i32x16, imm8: u32) -> i32x16; + + #[link_name = "llvm.x86.avx2.pslli.d"] + fn psllid256(a: i32x8, imm8: i32) -> i32x8; + #[link_name = "llvm.x86.sse2.pslli.d"] + fn psllid128(a: i32x4, imm8: i32) -> i32x4; + #[link_name = "llvm.x86.avx512.psrli.d.512"] fn vpsrlid(a: i32x16, imm8: u32) -> i32x16; + + #[link_name = "llvm.x86.avx2.psrli.d"] + fn psrlid256(a: i32x8, imm8: i32) -> i32x8; + #[link_name = "llvm.x86.sse2.psrli.d"] + fn psrlid128(a: i32x4, imm8: i32) -> i32x4; + #[link_name = "llvm.x86.avx512.pslli.q.512"] fn vpslliq(a: i64x8, imm8: u32) -> i64x8; + + #[link_name = "llvm.x86.avx2.pslli.q"] + fn pslliq256(a: i64x4, imm8: i32) -> i64x4; + #[link_name = "llvm.x86.sse2.pslli.q"] + fn pslliq128(a: i64x2, imm8: i32) -> i64x2; + #[link_name = "llvm.x86.avx512.psrli.q.512"] fn vpsrliq(a: i64x8, imm8: u32) -> i64x8; + #[link_name = "llvm.x86.avx2.psrli.q"] + fn psrliq256(a: i64x4, imm8: i32) -> i64x4; + #[link_name = "llvm.x86.sse2.psrli.q"] + fn psrliq128(a: i64x2, imm8: i32) -> i64x2; + #[link_name = "llvm.x86.avx512.psll.d.512"] fn vpslld(a: i32x16, count: i32x4) -> i32x16; #[link_name = "llvm.x86.avx512.psrl.d.512"] @@ -39279,7 +36579,11 @@ extern "C" { fn vpsraq128(a: i64x2, count: i64x2) -> i64x2; #[link_name = "llvm.x86.avx512.psrai.d.512"] - fn vpsraid(a: i32x16, imm8: u32) -> i32x16; + fn vpsraid512(a: i32x16, imm8: u32) -> i32x16; + #[link_name = "llvm.x86.avx2.psrai.d"] + fn psraid256(a: i32x8, imm8: i32) -> i32x8; + #[link_name = "llvm.x86.sse2.psrai.d"] + fn psraid128(a: i32x4, imm8: i32) -> i32x4; #[link_name = "llvm.x86.avx512.psrai.q.512"] fn vpsraiq(a: i64x8, imm8: u32) -> i64x8; @@ -42201,7 +39505,7 @@ mod tests { #[simd_test(enable = "avx512f")] unsafe fn test_mm512_roundscale_ps() { let a = _mm512_set1_ps(1.1); - let r = _mm512_roundscale_ps(a, 0); + let r = _mm512_roundscale_ps::<0b00_00_00_00>(a); let e = _mm512_set1_ps(1.0); assert_eq_m512(r, e); } @@ -42209,10 +39513,10 @@ mod tests { #[simd_test(enable = "avx512f")] unsafe fn test_mm512_mask_roundscale_ps() { let a = _mm512_set1_ps(1.1); - let r = _mm512_mask_roundscale_ps(a, 0, a, 0); + let r = _mm512_mask_roundscale_ps::<0b00_00_00_00>(a, 0, a); let e = _mm512_set1_ps(1.1); assert_eq_m512(r, e); - let r = _mm512_mask_roundscale_ps(a, 0b11111111_11111111, a, 0); + let r = _mm512_mask_roundscale_ps::<0b00_00_00_00>(a, 0b11111111_11111111, a); let e = _mm512_set1_ps(1.0); assert_eq_m512(r, e); } @@ -42220,9 +39524,9 @@ mod tests { #[simd_test(enable = "avx512f")] unsafe fn test_mm512_maskz_roundscale_ps() { let a = _mm512_set1_ps(1.1); - let r = _mm512_maskz_roundscale_ps(0, a, 0); + let r = _mm512_maskz_roundscale_ps::<0b00_00_00_00>(0, a); assert_eq_m512(r, _mm512_setzero_ps()); - let r = _mm512_maskz_roundscale_ps(0b11111111_11111111, a, 0); + let r = _mm512_maskz_roundscale_ps::<0b00_00_00_00>(0b11111111_11111111, a); let e = _mm512_set1_ps(1.0); assert_eq_m512(r, e); } @@ -42230,7 +39534,7 @@ mod tests { #[simd_test(enable = "avx512f,avx512vl")] unsafe fn test_mm256_roundscale_ps() { let a = _mm256_set1_ps(1.1); - let r = _mm256_roundscale_ps(a, 0); + let r = _mm256_roundscale_ps::<0b00_00_00_00>(a); let e = _mm256_set1_ps(1.0); assert_eq_m256(r, e); } @@ -42238,10 +39542,10 @@ mod tests { #[simd_test(enable = "avx512f,avx512vl")] unsafe fn test_mm256_mask_roundscale_ps() { let a = _mm256_set1_ps(1.1); - let r = _mm256_mask_roundscale_ps(a, 0, a, 0); + let r = _mm256_mask_roundscale_ps::<0b00_00_00_00>(a, 0, a); let e = _mm256_set1_ps(1.1); assert_eq_m256(r, e); - let r = _mm256_mask_roundscale_ps(a, 0b11111111, a, 0); + let r = _mm256_mask_roundscale_ps::<0b00_00_00_00>(a, 0b11111111, a); let e = _mm256_set1_ps(1.0); assert_eq_m256(r, e); } @@ -42249,9 +39553,9 @@ mod tests { #[simd_test(enable = "avx512f,avx512vl")] unsafe fn test_mm256_maskz_roundscale_ps() { let a = _mm256_set1_ps(1.1); - let r = _mm256_maskz_roundscale_ps(0, a, 0); + let r = _mm256_maskz_roundscale_ps::<0b00_00_00_00>(0, a); assert_eq_m256(r, _mm256_setzero_ps()); - let r = _mm256_maskz_roundscale_ps(0b11111111, a, 0); + let r = _mm256_maskz_roundscale_ps::<0b00_00_00_00>(0b11111111, a); let e = _mm256_set1_ps(1.0); assert_eq_m256(r, e); } @@ -42259,7 +39563,7 @@ mod tests { #[simd_test(enable = "avx512f,avx512vl")] unsafe fn test_mm_roundscale_ps() { let a = _mm_set1_ps(1.1); - let r = _mm_roundscale_ps(a, 0); + let r = _mm_roundscale_ps::<0b00_00_00_00>(a); let e = _mm_set1_ps(1.0); assert_eq_m128(r, e); } @@ -42267,10 +39571,10 @@ mod tests { #[simd_test(enable = "avx512f,avx512vl")] unsafe fn test_mm_mask_roundscale_ps() { let a = _mm_set1_ps(1.1); - let r = _mm_mask_roundscale_ps(a, 0, a, 0); + let r = _mm_mask_roundscale_ps::<0b00_00_00_00>(a, 0, a); let e = _mm_set1_ps(1.1); assert_eq_m128(r, e); - let r = _mm_mask_roundscale_ps(a, 0b00001111, a, 0); + let r = _mm_mask_roundscale_ps::<0b00_00_00_00>(a, 0b00001111, a); let e = _mm_set1_ps(1.0); assert_eq_m128(r, e); } @@ -42278,9 +39582,9 @@ mod tests { #[simd_test(enable = "avx512f,avx512vl")] unsafe fn test_mm_maskz_roundscale_ps() { let a = _mm_set1_ps(1.1); - let r = _mm_maskz_roundscale_ps(0, a, 0); + let r = _mm_maskz_roundscale_ps::<0b00_00_00_00>(0, a); assert_eq_m128(r, _mm_setzero_ps()); - let r = _mm_maskz_roundscale_ps(0b00001111, a, 0); + let r = _mm_maskz_roundscale_ps::<0b00_00_00_00>(0b00001111, a); let e = _mm_set1_ps(1.0); assert_eq_m128(r, e); } @@ -42387,7 +39691,8 @@ mod tests { let a = _mm512_set1_ps(f32::NAN); let b = _mm512_set1_ps(f32::MAX); let c = _mm512_set1_epi32(i32::MAX); - let r = _mm512_fixupimm_ps(a, b, c, 5); + //let r = _mm512_fixupimm_ps(a, b, c, 5); + let r = _mm512_fixupimm_ps::<5>(a, b, c); let e = _mm512_set1_ps(0.0); assert_eq_m512(r, e); } @@ -42403,7 +39708,7 @@ mod tests { ); let b = _mm512_set1_ps(f32::MAX); let c = _mm512_set1_epi32(i32::MAX); - let r = _mm512_mask_fixupimm_ps(a, 0b11111111_00000000, b, c, 5); + let r = _mm512_mask_fixupimm_ps::<5>(a, 0b11111111_00000000, b, c); let e = _mm512_set_ps( 0., 0., 0., 0., 0., 0., 0., 0., 1., 1., 1., 1., 1., 1., 1., 1., ); @@ -42421,7 +39726,7 @@ mod tests { ); let b = _mm512_set1_ps(f32::MAX); let c = _mm512_set1_epi32(i32::MAX); - let r = _mm512_maskz_fixupimm_ps(0b11111111_00000000, a, b, c, 5); + let r = _mm512_maskz_fixupimm_ps::<5>(0b11111111_00000000, a, b, c); let e = _mm512_set_ps( 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., ); @@ -42433,7 +39738,7 @@ mod tests { let a = _mm256_set1_ps(f32::NAN); let b = _mm256_set1_ps(f32::MAX); let c = _mm256_set1_epi32(i32::MAX); - let r = _mm256_fixupimm_ps(a, b, c, 5); + let r = _mm256_fixupimm_ps::<5>(a, b, c); let e = _mm256_set1_ps(0.0); assert_eq_m256(r, e); } @@ -42443,7 +39748,7 @@ mod tests { let a = _mm256_set1_ps(f32::NAN); let b = _mm256_set1_ps(f32::MAX); let c = _mm256_set1_epi32(i32::MAX); - let r = _mm256_mask_fixupimm_ps(a, 0b11111111, b, c, 5); + let r = _mm256_mask_fixupimm_ps::<5>(a, 0b11111111, b, c); let e = _mm256_set1_ps(0.0); assert_eq_m256(r, e); } @@ -42453,7 +39758,7 @@ mod tests { let a = _mm256_set1_ps(f32::NAN); let b = _mm256_set1_ps(f32::MAX); let c = _mm256_set1_epi32(i32::MAX); - let r = _mm256_maskz_fixupimm_ps(0b11111111, a, b, c, 5); + let r = _mm256_maskz_fixupimm_ps::<5>(0b11111111, a, b, c); let e = _mm256_set1_ps(0.0); assert_eq_m256(r, e); } @@ -42463,7 +39768,7 @@ mod tests { let a = _mm_set1_ps(f32::NAN); let b = _mm_set1_ps(f32::MAX); let c = _mm_set1_epi32(i32::MAX); - let r = _mm_fixupimm_ps(a, b, c, 5); + let r = _mm_fixupimm_ps::<5>(a, b, c); let e = _mm_set1_ps(0.0); assert_eq_m128(r, e); } @@ -42473,7 +39778,7 @@ mod tests { let a = _mm_set1_ps(f32::NAN); let b = _mm_set1_ps(f32::MAX); let c = _mm_set1_epi32(i32::MAX); - let r = _mm_mask_fixupimm_ps(a, 0b00001111, b, c, 5); + let r = _mm_mask_fixupimm_ps::<5>(a, 0b00001111, b, c); let e = _mm_set1_ps(0.0); assert_eq_m128(r, e); } @@ -42483,7 +39788,7 @@ mod tests { let a = _mm_set1_ps(f32::NAN); let b = _mm_set1_ps(f32::MAX); let c = _mm_set1_epi32(i32::MAX); - let r = _mm_maskz_fixupimm_ps(0b00001111, a, b, c, 5); + let r = _mm_maskz_fixupimm_ps::<5>(0b00001111, a, b, c); let e = _mm_set1_ps(0.0); assert_eq_m128(r, e); } @@ -42493,7 +39798,7 @@ mod tests { let a = _mm512_set1_epi32(1 << 2); let b = _mm512_set1_epi32(1 << 1); let c = _mm512_set1_epi32(1 << 0); - let r = _mm512_ternarylogic_epi32(a, b, c, 8); + let r = _mm512_ternarylogic_epi32::<8>(a, b, c); let e = _mm512_set1_epi32(0); assert_eq_m512i(r, e); } @@ -42503,9 +39808,9 @@ mod tests { let src = _mm512_set1_epi32(1 << 2); let a = _mm512_set1_epi32(1 << 1); let b = _mm512_set1_epi32(1 << 0); - let r = _mm512_mask_ternarylogic_epi32(src, 0, a, b, 8); + let r = _mm512_mask_ternarylogic_epi32::<8>(src, 0, a, b); assert_eq_m512i(r, src); - let r = _mm512_mask_ternarylogic_epi32(src, 0b11111111_11111111, a, b, 8); + let r = _mm512_mask_ternarylogic_epi32::<8>(src, 0b11111111_11111111, a, b); let e = _mm512_set1_epi32(0); assert_eq_m512i(r, e); } @@ -42515,9 +39820,9 @@ mod tests { let a = _mm512_set1_epi32(1 << 2); let b = _mm512_set1_epi32(1 << 1); let c = _mm512_set1_epi32(1 << 0); - let r = _mm512_maskz_ternarylogic_epi32(0, a, b, c, 9); + let r = _mm512_maskz_ternarylogic_epi32::<9>(0, a, b, c); assert_eq_m512i(r, _mm512_setzero_si512()); - let r = _mm512_maskz_ternarylogic_epi32(0b11111111_11111111, a, b, c, 8); + let r = _mm512_maskz_ternarylogic_epi32::<8>(0b11111111_11111111, a, b, c); let e = _mm512_set1_epi32(0); assert_eq_m512i(r, e); } @@ -42527,7 +39832,7 @@ mod tests { let a = _mm256_set1_epi32(1 << 2); let b = _mm256_set1_epi32(1 << 1); let c = _mm256_set1_epi32(1 << 0); - let r = _mm256_ternarylogic_epi32(a, b, c, 8); + let r = _mm256_ternarylogic_epi32::<8>(a, b, c); let e = _mm256_set1_epi32(0); assert_eq_m256i(r, e); } @@ -42537,9 +39842,9 @@ mod tests { let src = _mm256_set1_epi32(1 << 2); let a = _mm256_set1_epi32(1 << 1); let b = _mm256_set1_epi32(1 << 0); - let r = _mm256_mask_ternarylogic_epi32(src, 0, a, b, 8); + let r = _mm256_mask_ternarylogic_epi32::<8>(src, 0, a, b); assert_eq_m256i(r, src); - let r = _mm256_mask_ternarylogic_epi32(src, 0b11111111, a, b, 8); + let r = _mm256_mask_ternarylogic_epi32::<8>(src, 0b11111111, a, b); let e = _mm256_set1_epi32(0); assert_eq_m256i(r, e); } @@ -42549,9 +39854,9 @@ mod tests { let a = _mm256_set1_epi32(1 << 2); let b = _mm256_set1_epi32(1 << 1); let c = _mm256_set1_epi32(1 << 0); - let r = _mm256_maskz_ternarylogic_epi32(0, a, b, c, 9); + let r = _mm256_maskz_ternarylogic_epi32::<9>(0, a, b, c); assert_eq_m256i(r, _mm256_setzero_si256()); - let r = _mm256_maskz_ternarylogic_epi32(0b11111111, a, b, c, 8); + let r = _mm256_maskz_ternarylogic_epi32::<8>(0b11111111, a, b, c); let e = _mm256_set1_epi32(0); assert_eq_m256i(r, e); } @@ -42561,7 +39866,7 @@ mod tests { let a = _mm_set1_epi32(1 << 2); let b = _mm_set1_epi32(1 << 1); let c = _mm_set1_epi32(1 << 0); - let r = _mm_ternarylogic_epi32(a, b, c, 8); + let r = _mm_ternarylogic_epi32::<8>(a, b, c); let e = _mm_set1_epi32(0); assert_eq_m128i(r, e); } @@ -42571,9 +39876,9 @@ mod tests { let src = _mm_set1_epi32(1 << 2); let a = _mm_set1_epi32(1 << 1); let b = _mm_set1_epi32(1 << 0); - let r = _mm_mask_ternarylogic_epi32(src, 0, a, b, 8); + let r = _mm_mask_ternarylogic_epi32::<8>(src, 0, a, b); assert_eq_m128i(r, src); - let r = _mm_mask_ternarylogic_epi32(src, 0b00001111, a, b, 8); + let r = _mm_mask_ternarylogic_epi32::<8>(src, 0b00001111, a, b); let e = _mm_set1_epi32(0); assert_eq_m128i(r, e); } @@ -42583,9 +39888,9 @@ mod tests { let a = _mm_set1_epi32(1 << 2); let b = _mm_set1_epi32(1 << 1); let c = _mm_set1_epi32(1 << 0); - let r = _mm_maskz_ternarylogic_epi32(0, a, b, c, 9); + let r = _mm_maskz_ternarylogic_epi32::<9>(0, a, b, c); assert_eq_m128i(r, _mm_setzero_si128()); - let r = _mm_maskz_ternarylogic_epi32(0b00001111, a, b, c, 8); + let r = _mm_maskz_ternarylogic_epi32::<8>(0b00001111, a, b, c); let e = _mm_set1_epi32(0); assert_eq_m128i(r, e); } @@ -42593,7 +39898,7 @@ mod tests { #[simd_test(enable = "avx512f")] unsafe fn test_mm512_getmant_ps() { let a = _mm512_set1_ps(10.); - let r = _mm512_getmant_ps(a, _MM_MANT_NORM_P75_1P5, _MM_MANT_SIGN_NAN); + let r = _mm512_getmant_ps::<_MM_MANT_NORM_P75_1P5, _MM_MANT_SIGN_NAN>(a); let e = _mm512_set1_ps(1.25); assert_eq_m512(r, e); } @@ -42601,14 +39906,12 @@ mod tests { #[simd_test(enable = "avx512f")] unsafe fn test_mm512_mask_getmant_ps() { let a = _mm512_set1_ps(10.); - let r = _mm512_mask_getmant_ps(a, 0, a, _MM_MANT_NORM_1_2, _MM_MANT_SIGN_SRC); + let r = _mm512_mask_getmant_ps::<_MM_MANT_NORM_1_2, _MM_MANT_SIGN_SRC>(a, 0, a); assert_eq_m512(r, a); - let r = _mm512_mask_getmant_ps( + let r = _mm512_mask_getmant_ps::<_MM_MANT_NORM_1_2, _MM_MANT_SIGN_SRC>( a, 0b11111111_00000000, a, - _MM_MANT_NORM_1_2, - _MM_MANT_SIGN_SRC, ); let e = _mm512_setr_ps( 10., 10., 10., 10., 10., 10., 10., 10., 1.25, 1.25, 1.25, 1.25, 1.25, 1.25, 1.25, 1.25, @@ -42619,10 +39922,10 @@ mod tests { #[simd_test(enable = "avx512f")] unsafe fn test_mm512_maskz_getmant_ps() { let a = _mm512_set1_ps(10.); - let r = _mm512_maskz_getmant_ps(0, a, _MM_MANT_NORM_1_2, _MM_MANT_SIGN_SRC); + let r = _mm512_maskz_getmant_ps::<_MM_MANT_NORM_1_2, _MM_MANT_SIGN_SRC>(0, a); assert_eq_m512(r, _mm512_setzero_ps()); let r = - _mm512_maskz_getmant_ps(0b11111111_00000000, a, _MM_MANT_NORM_1_2, _MM_MANT_SIGN_SRC); + _mm512_maskz_getmant_ps::<_MM_MANT_NORM_1_2, _MM_MANT_SIGN_SRC>(0b11111111_00000000, a); let e = _mm512_setr_ps( 0., 0., 0., 0., 0., 0., 0., 0., 1.25, 1.25, 1.25, 1.25, 1.25, 1.25, 1.25, 1.25, ); @@ -42632,7 +39935,7 @@ mod tests { #[simd_test(enable = "avx512f,avx512vl")] unsafe fn test_mm256_getmant_ps() { let a = _mm256_set1_ps(10.); - let r = _mm256_getmant_ps(a, _MM_MANT_NORM_P75_1P5, _MM_MANT_SIGN_NAN); + let r = _mm256_getmant_ps::<_MM_MANT_NORM_P75_1P5, _MM_MANT_SIGN_NAN>(a); let e = _mm256_set1_ps(1.25); assert_eq_m256(r, e); } @@ -42640,9 +39943,9 @@ mod tests { #[simd_test(enable = "avx512f,avx512vl")] unsafe fn test_mm256_mask_getmant_ps() { let a = _mm256_set1_ps(10.); - let r = _mm256_mask_getmant_ps(a, 0, a, _MM_MANT_NORM_1_2, _MM_MANT_SIGN_SRC); + let r = _mm256_mask_getmant_ps::<_MM_MANT_NORM_1_2, _MM_MANT_SIGN_SRC>(a, 0, a); assert_eq_m256(r, a); - let r = _mm256_mask_getmant_ps(a, 0b11111111, a, _MM_MANT_NORM_1_2, _MM_MANT_SIGN_SRC); + let r = _mm256_mask_getmant_ps::<_MM_MANT_NORM_1_2, _MM_MANT_SIGN_SRC>(a, 0b11111111, a); let e = _mm256_set1_ps(1.25); assert_eq_m256(r, e); } @@ -42650,9 +39953,9 @@ mod tests { #[simd_test(enable = "avx512f,avx512vl")] unsafe fn test_mm256_maskz_getmant_ps() { let a = _mm256_set1_ps(10.); - let r = _mm256_maskz_getmant_ps(0, a, _MM_MANT_NORM_1_2, _MM_MANT_SIGN_SRC); + let r = _mm256_maskz_getmant_ps::<_MM_MANT_NORM_1_2, _MM_MANT_SIGN_SRC>(0, a); assert_eq_m256(r, _mm256_setzero_ps()); - let r = _mm256_maskz_getmant_ps(0b11111111, a, _MM_MANT_NORM_1_2, _MM_MANT_SIGN_SRC); + let r = _mm256_maskz_getmant_ps::<_MM_MANT_NORM_1_2, _MM_MANT_SIGN_SRC>(0b11111111, a); let e = _mm256_set1_ps(1.25); assert_eq_m256(r, e); } @@ -42660,7 +39963,7 @@ mod tests { #[simd_test(enable = "avx512f,avx512vl")] unsafe fn test_mm_getmant_ps() { let a = _mm_set1_ps(10.); - let r = _mm_getmant_ps(a, _MM_MANT_NORM_P75_1P5, _MM_MANT_SIGN_NAN); + let r = _mm_getmant_ps::<_MM_MANT_NORM_P75_1P5, _MM_MANT_SIGN_NAN>(a); let e = _mm_set1_ps(1.25); assert_eq_m128(r, e); } @@ -42668,9 +39971,9 @@ mod tests { #[simd_test(enable = "avx512f,avx512vl")] unsafe fn test_mm_mask_getmant_ps() { let a = _mm_set1_ps(10.); - let r = _mm_mask_getmant_ps(a, 0, a, _MM_MANT_NORM_1_2, _MM_MANT_SIGN_SRC); + let r = _mm_mask_getmant_ps::<_MM_MANT_NORM_1_2, _MM_MANT_SIGN_SRC>(a, 0, a); assert_eq_m128(r, a); - let r = _mm_mask_getmant_ps(a, 0b00001111, a, _MM_MANT_NORM_1_2, _MM_MANT_SIGN_SRC); + let r = _mm_mask_getmant_ps::<_MM_MANT_NORM_1_2, _MM_MANT_SIGN_SRC>(a, 0b00001111, a); let e = _mm_set1_ps(1.25); assert_eq_m128(r, e); } @@ -42678,9 +39981,9 @@ mod tests { #[simd_test(enable = "avx512f,avx512vl")] unsafe fn test_mm_maskz_getmant_ps() { let a = _mm_set1_ps(10.); - let r = _mm_maskz_getmant_ps(0, a, _MM_MANT_NORM_1_2, _MM_MANT_SIGN_SRC); + let r = _mm_maskz_getmant_ps::<_MM_MANT_NORM_1_2, _MM_MANT_SIGN_SRC>(0, a); assert_eq_m128(r, _mm_setzero_ps()); - let r = _mm_maskz_getmant_ps(0b00001111, a, _MM_MANT_NORM_1_2, _MM_MANT_SIGN_SRC); + let r = _mm_maskz_getmant_ps::<_MM_MANT_NORM_1_2, _MM_MANT_SIGN_SRC>(0b00001111, a); let e = _mm_set1_ps(1.25); assert_eq_m128(r, e); } @@ -42691,7 +39994,7 @@ mod tests { 0., 1.5, 2., 3.5, 4., 5.5, 6., 7.5, 8., 9.5, 10., 11.5, 12., 13.5, 14., 0.00000007, ); let b = _mm512_set1_ps(-1.); - let r = _mm512_add_round_ps(a, b, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); + let r = _mm512_add_round_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>(a, b); #[rustfmt::skip] let e = _mm512_setr_ps( -1., 0.5, 1., 2.5, @@ -42700,7 +40003,7 @@ mod tests { 11., 12.5, 13., -0.99999994, ); assert_eq_m512(r, e); - let r = _mm512_add_round_ps(a, b, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); + let r = _mm512_add_round_ps::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(a, b); let e = _mm512_setr_ps( -1., 0.5, 1., 2.5, 3., 4.5, 5., 6.5, 7., 8.5, 9., 10.5, 11., 12.5, 13., -0.9999999, ); @@ -42713,14 +40016,13 @@ mod tests { 0., 1.5, 2., 3.5, 4., 5.5, 6., 7.5, 8., 9.5, 10., 11.5, 12., 13.5, 14., 0.00000007, ); let b = _mm512_set1_ps(-1.); - let r = _mm512_mask_add_round_ps(a, 0, a, b, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); + let r = _mm512_mask_add_round_ps::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(a, 0, a, b); assert_eq_m512(r, a); - let r = _mm512_mask_add_round_ps( + let r = _mm512_mask_add_round_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( a, 0b11111111_00000000, a, b, - _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC, ); #[rustfmt::skip] let e = _mm512_setr_ps( @@ -42738,13 +40040,12 @@ mod tests { 0., 1.5, 2., 3.5, 4., 5.5, 6., 7.5, 8., 9.5, 10., 11.5, 12., 13.5, 14., 0.00000007, ); let b = _mm512_set1_ps(-1.); - let r = _mm512_maskz_add_round_ps(0, a, b, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); + let r = _mm512_maskz_add_round_ps::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(0, a, b); assert_eq_m512(r, _mm512_setzero_ps()); - let r = _mm512_maskz_add_round_ps( + let r = _mm512_maskz_add_round_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( 0b11111111_00000000, a, b, - _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC, ); #[rustfmt::skip] let e = _mm512_setr_ps( @@ -42762,7 +40063,7 @@ mod tests { 0., 1.5, 2., 3.5, 4., 5.5, 6., 7.5, 8., 9.5, 10., 11.5, 12., 13.5, 14., 0.00000007, ); let b = _mm512_set1_ps(1.); - let r = _mm512_sub_round_ps(a, b, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); + let r = _mm512_sub_round_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>(a, b); #[rustfmt::skip] let e = _mm512_setr_ps( -1., 0.5, 1., 2.5, @@ -42771,7 +40072,7 @@ mod tests { 11., 12.5, 13., -0.99999994, ); assert_eq_m512(r, e); - let r = _mm512_sub_round_ps(a, b, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); + let r = _mm512_sub_round_ps::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(a, b); let e = _mm512_setr_ps( -1., 0.5, 1., 2.5, 3., 4.5, 5., 6.5, 7., 8.5, 9., 10.5, 11., 12.5, 13., -0.9999999, ); @@ -42784,14 +40085,15 @@ mod tests { 0., 1.5, 2., 3.5, 4., 5.5, 6., 7.5, 8., 9.5, 10., 11.5, 12., 13.5, 14., 0.00000007, ); let b = _mm512_set1_ps(1.); - let r = _mm512_mask_sub_round_ps(a, 0, a, b, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); + let r = _mm512_mask_sub_round_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, 0, a, b, + ); assert_eq_m512(r, a); - let r = _mm512_mask_sub_round_ps( + let r = _mm512_mask_sub_round_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( a, 0b11111111_00000000, a, b, - _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC, ); #[rustfmt::skip] let e = _mm512_setr_ps( @@ -42809,13 +40111,13 @@ mod tests { 0., 1.5, 2., 3.5, 4., 5.5, 6., 7.5, 8., 9.5, 10., 11.5, 12., 13.5, 14., 0.00000007, ); let b = _mm512_set1_ps(1.); - let r = _mm512_maskz_sub_round_ps(0, a, b, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); + let r = + _mm512_maskz_sub_round_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>(0, a, b); assert_eq_m512(r, _mm512_setzero_ps()); - let r = _mm512_maskz_sub_round_ps( + let r = _mm512_maskz_sub_round_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( 0b11111111_00000000, a, b, - _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC, ); #[rustfmt::skip] let e = _mm512_setr_ps( @@ -42837,7 +40139,7 @@ mod tests { 12., 13.5, 14., 0.00000000000000000000007, ); let b = _mm512_set1_ps(0.1); - let r = _mm512_mul_round_ps(a, b, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); + let r = _mm512_mul_round_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>(a, b); #[rustfmt::skip] let e = _mm512_setr_ps( 0., 0.15, 0.2, 0.35, @@ -42846,7 +40148,7 @@ mod tests { 1.2, 1.35, 1.4, 0.000000000000000000000007000001, ); assert_eq_m512(r, e); - let r = _mm512_mul_round_ps(a, b, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); + let r = _mm512_mul_round_ps::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(a, b); #[rustfmt::skip] let e = _mm512_setr_ps( 0., 0.14999999, 0.2, 0.35, @@ -42867,14 +40169,15 @@ mod tests { 12., 13.5, 14., 0.00000000000000000000007, ); let b = _mm512_set1_ps(0.1); - let r = _mm512_mask_mul_round_ps(a, 0, a, b, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); + let r = _mm512_mask_mul_round_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, 0, a, b, + ); assert_eq_m512(r, a); - let r = _mm512_mask_mul_round_ps( + let r = _mm512_mask_mul_round_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( a, 0b11111111_00000000, a, b, - _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC, ); #[rustfmt::skip] let e = _mm512_setr_ps( @@ -42896,13 +40199,13 @@ mod tests { 12., 13.5, 14., 0.00000000000000000000007, ); let b = _mm512_set1_ps(0.1); - let r = _mm512_maskz_mul_round_ps(0, a, b, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); + let r = + _mm512_maskz_mul_round_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>(0, a, b); assert_eq_m512(r, _mm512_setzero_ps()); - let r = _mm512_maskz_mul_round_ps( + let r = _mm512_maskz_mul_round_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( 0b11111111_00000000, a, b, - _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC, ); #[rustfmt::skip] let e = _mm512_setr_ps( @@ -42918,10 +40221,10 @@ mod tests { unsafe fn test_mm512_div_round_ps() { let a = _mm512_set1_ps(1.); let b = _mm512_set1_ps(3.); - let r = _mm512_div_round_ps(a, b, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); + let r = _mm512_div_round_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>(a, b); let e = _mm512_set1_ps(0.33333334); assert_eq_m512(r, e); - let r = _mm512_div_round_ps(a, b, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); + let r = _mm512_div_round_ps::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(a, b); let e = _mm512_set1_ps(0.3333333); assert_eq_m512(r, e); } @@ -42930,14 +40233,15 @@ mod tests { unsafe fn test_mm512_mask_div_round_ps() { let a = _mm512_set1_ps(1.); let b = _mm512_set1_ps(3.); - let r = _mm512_mask_div_round_ps(a, 0, a, b, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); + let r = _mm512_mask_div_round_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, 0, a, b, + ); assert_eq_m512(r, a); - let r = _mm512_mask_div_round_ps( + let r = _mm512_mask_div_round_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( a, 0b11111111_00000000, a, b, - _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC, ); let e = _mm512_setr_ps( 1., 1., 1., 1., 1., 1., 1., 1., 0.33333334, 0.33333334, 0.33333334, 0.33333334, @@ -42950,13 +40254,13 @@ mod tests { unsafe fn test_mm512_maskz_div_round_ps() { let a = _mm512_set1_ps(1.); let b = _mm512_set1_ps(3.); - let r = _mm512_maskz_div_round_ps(0, a, b, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); + let r = + _mm512_maskz_div_round_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>(0, a, b); assert_eq_m512(r, _mm512_setzero_ps()); - let r = _mm512_maskz_div_round_ps( + let r = _mm512_maskz_div_round_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( 0b11111111_00000000, a, b, - _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC, ); let e = _mm512_setr_ps( 0., 0., 0., 0., 0., 0., 0., 0., 0.33333334, 0.33333334, 0.33333334, 0.33333334, @@ -42968,10 +40272,10 @@ mod tests { #[simd_test(enable = "avx512f")] unsafe fn test_mm512_sqrt_round_ps() { let a = _mm512_set1_ps(3.); - let r = _mm512_sqrt_round_ps(a, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); + let r = _mm512_sqrt_round_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>(a); let e = _mm512_set1_ps(1.7320508); assert_eq_m512(r, e); - let r = _mm512_sqrt_round_ps(a, _MM_FROUND_TO_POS_INF | _MM_FROUND_NO_EXC); + let r = _mm512_sqrt_round_ps::<{ _MM_FROUND_TO_POS_INF | _MM_FROUND_NO_EXC }>(a); let e = _mm512_set1_ps(1.7320509); assert_eq_m512(r, e); } @@ -42979,13 +40283,13 @@ mod tests { #[simd_test(enable = "avx512f")] unsafe fn test_mm512_mask_sqrt_round_ps() { let a = _mm512_set1_ps(3.); - let r = _mm512_mask_sqrt_round_ps(a, 0, a, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); + let r = + _mm512_mask_sqrt_round_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>(a, 0, a); assert_eq_m512(r, a); - let r = _mm512_mask_sqrt_round_ps( + let r = _mm512_mask_sqrt_round_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( a, 0b11111111_00000000, a, - _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC, ); let e = _mm512_setr_ps( 3., 3., 3., 3., 3., 3., 3., 3., 1.7320508, 1.7320508, 1.7320508, 1.7320508, 1.7320508, @@ -42997,12 +40301,12 @@ mod tests { #[simd_test(enable = "avx512f")] unsafe fn test_mm512_maskz_sqrt_round_ps() { let a = _mm512_set1_ps(3.); - let r = _mm512_maskz_sqrt_round_ps(0, a, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); + let r = + _mm512_maskz_sqrt_round_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>(0, a); assert_eq_m512(r, _mm512_setzero_ps()); - let r = _mm512_maskz_sqrt_round_ps( + let r = _mm512_maskz_sqrt_round_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( 0b11111111_00000000, a, - _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC, ); let e = _mm512_setr_ps( 0., 0., 0., 0., 0., 0., 0., 0., 1.7320508, 1.7320508, 1.7320508, 1.7320508, 1.7320508, @@ -43016,10 +40320,10 @@ mod tests { let a = _mm512_set1_ps(0.00000007); let b = _mm512_set1_ps(1.); let c = _mm512_set1_ps(-1.); - let r = _mm512_fmadd_round_ps(a, b, c, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); + let r = _mm512_fmadd_round_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>(a, b, c); let e = _mm512_set1_ps(-0.99999994); assert_eq_m512(r, e); - let r = _mm512_fmadd_round_ps(a, b, c, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); + let r = _mm512_fmadd_round_ps::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(a, b, c); let e = _mm512_set1_ps(-0.9999999); assert_eq_m512(r, e); } @@ -43029,15 +40333,15 @@ mod tests { let a = _mm512_set1_ps(0.00000007); let b = _mm512_set1_ps(1.); let c = _mm512_set1_ps(-1.); - let r = - _mm512_mask_fmadd_round_ps(a, 0, b, c, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); + let r = _mm512_mask_fmadd_round_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, 0, b, c, + ); assert_eq_m512(r, a); - let r = _mm512_mask_fmadd_round_ps( + let r = _mm512_mask_fmadd_round_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( a, 0b00000000_11111111, b, c, - _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC, ); #[rustfmt::skip] let e = _mm512_setr_ps( @@ -43054,16 +40358,16 @@ mod tests { let a = _mm512_set1_ps(0.00000007); let b = _mm512_set1_ps(1.); let c = _mm512_set1_ps(-1.); - let r = - _mm512_maskz_fmadd_round_ps(0, a, b, c, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); + let r = _mm512_maskz_fmadd_round_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + 0, a, b, c, + ); assert_eq_m512(r, _mm512_setzero_ps()); #[rustfmt::skip] - let r = _mm512_maskz_fmadd_round_ps( + let r = _mm512_maskz_fmadd_round_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( 0b00000000_11111111, a, b, c, - _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC, ); #[rustfmt::skip] let e = _mm512_setr_ps( @@ -43080,15 +40384,15 @@ mod tests { let a = _mm512_set1_ps(0.00000007); let b = _mm512_set1_ps(1.); let c = _mm512_set1_ps(-1.); - let r = - _mm512_mask3_fmadd_round_ps(a, b, c, 0, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); + let r = _mm512_mask3_fmadd_round_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, b, c, 0, + ); assert_eq_m512(r, c); - let r = _mm512_mask3_fmadd_round_ps( + let r = _mm512_mask3_fmadd_round_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( a, b, c, 0b00000000_11111111, - _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC, ); #[rustfmt::skip] let e = _mm512_setr_ps( @@ -43105,10 +40409,10 @@ mod tests { let a = _mm512_set1_ps(0.00000007); let b = _mm512_set1_ps(1.); let c = _mm512_set1_ps(1.); - let r = _mm512_fmsub_round_ps(a, b, c, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); + let r = _mm512_fmsub_round_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>(a, b, c); let e = _mm512_set1_ps(-0.99999994); assert_eq_m512(r, e); - let r = _mm512_fmsub_round_ps(a, b, c, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); + let r = _mm512_fmsub_round_ps::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(a, b, c); let e = _mm512_set1_ps(-0.9999999); assert_eq_m512(r, e); } @@ -43118,15 +40422,15 @@ mod tests { let a = _mm512_set1_ps(0.00000007); let b = _mm512_set1_ps(1.); let c = _mm512_set1_ps(1.); - let r = - _mm512_mask_fmsub_round_ps(a, 0, b, c, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); + let r = _mm512_mask_fmsub_round_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, 0, b, c, + ); assert_eq_m512(r, a); - let r = _mm512_mask_fmsub_round_ps( + let r = _mm512_mask_fmsub_round_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( a, 0b00000000_11111111, b, c, - _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC, ); #[rustfmt::skip] let e = _mm512_setr_ps( @@ -43143,15 +40447,15 @@ mod tests { let a = _mm512_set1_ps(0.00000007); let b = _mm512_set1_ps(1.); let c = _mm512_set1_ps(1.); - let r = - _mm512_maskz_fmsub_round_ps(0, a, b, c, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); + let r = _mm512_maskz_fmsub_round_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + 0, a, b, c, + ); assert_eq_m512(r, _mm512_setzero_ps()); - let r = _mm512_maskz_fmsub_round_ps( + let r = _mm512_maskz_fmsub_round_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( 0b00000000_11111111, a, b, c, - _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC, ); #[rustfmt::skip] let e = _mm512_setr_ps( @@ -43168,15 +40472,15 @@ mod tests { let a = _mm512_set1_ps(0.00000007); let b = _mm512_set1_ps(1.); let c = _mm512_set1_ps(1.); - let r = - _mm512_mask3_fmsub_round_ps(a, b, c, 0, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); + let r = _mm512_mask3_fmsub_round_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, b, c, 0, + ); assert_eq_m512(r, c); - let r = _mm512_mask3_fmsub_round_ps( + let r = _mm512_mask3_fmsub_round_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( a, b, c, 0b00000000_11111111, - _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC, ); #[rustfmt::skip] let e = _mm512_setr_ps( @@ -43193,7 +40497,8 @@ mod tests { let a = _mm512_set1_ps(0.00000007); let b = _mm512_set1_ps(1.); let c = _mm512_set1_ps(-1.); - let r = _mm512_fmaddsub_round_ps(a, b, c, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); + let r = + _mm512_fmaddsub_round_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>(a, b, c); #[rustfmt::skip] let e = _mm512_setr_ps( 1.0000001, -0.99999994, 1.0000001, -0.99999994, @@ -43202,7 +40507,7 @@ mod tests { 1.0000001, -0.99999994, 1.0000001, -0.99999994, ); assert_eq_m512(r, e); - let r = _mm512_fmaddsub_round_ps(a, b, c, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); + let r = _mm512_fmaddsub_round_ps::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(a, b, c); let e = _mm512_setr_ps( 1., -0.9999999, 1., -0.9999999, 1., -0.9999999, 1., -0.9999999, 1., -0.9999999, 1., -0.9999999, 1., -0.9999999, 1., -0.9999999, @@ -43215,20 +40520,15 @@ mod tests { let a = _mm512_set1_ps(0.00000007); let b = _mm512_set1_ps(1.); let c = _mm512_set1_ps(-1.); - let r = _mm512_mask_fmaddsub_round_ps( - a, - 0, - b, - c, - _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC, + let r = _mm512_mask_fmaddsub_round_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, 0, b, c, ); assert_eq_m512(r, a); - let r = _mm512_mask_fmaddsub_round_ps( + let r = _mm512_mask_fmaddsub_round_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( a, 0b00000000_11111111, b, c, - _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC, ); #[rustfmt::skip] let e = _mm512_setr_ps( @@ -43245,20 +40545,15 @@ mod tests { let a = _mm512_set1_ps(0.00000007); let b = _mm512_set1_ps(1.); let c = _mm512_set1_ps(-1.); - let r = _mm512_maskz_fmaddsub_round_ps( - 0, - a, - b, - c, - _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC, + let r = _mm512_maskz_fmaddsub_round_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + 0, a, b, c, ); assert_eq_m512(r, _mm512_setzero_ps()); - let r = _mm512_maskz_fmaddsub_round_ps( + let r = _mm512_maskz_fmaddsub_round_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( 0b00000000_11111111, a, b, c, - _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC, ); #[rustfmt::skip] let e = _mm512_setr_ps( @@ -43275,20 +40570,15 @@ mod tests { let a = _mm512_set1_ps(0.00000007); let b = _mm512_set1_ps(1.); let c = _mm512_set1_ps(-1.); - let r = _mm512_mask3_fmaddsub_round_ps( - a, - b, - c, - 0, - _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC, + let r = _mm512_mask3_fmaddsub_round_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, b, c, 0, ); assert_eq_m512(r, c); - let r = _mm512_mask3_fmaddsub_round_ps( + let r = _mm512_mask3_fmaddsub_round_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( a, b, c, 0b00000000_11111111, - _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC, ); #[rustfmt::skip] let e = _mm512_setr_ps( @@ -43305,7 +40595,8 @@ mod tests { let a = _mm512_set1_ps(0.00000007); let b = _mm512_set1_ps(1.); let c = _mm512_set1_ps(-1.); - let r = _mm512_fmsubadd_round_ps(a, b, c, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); + let r = + _mm512_fmsubadd_round_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>(a, b, c); #[rustfmt::skip] let e = _mm512_setr_ps( -0.99999994, 1.0000001, -0.99999994, 1.0000001, @@ -43314,7 +40605,7 @@ mod tests { -0.99999994, 1.0000001, -0.99999994, 1.0000001, ); assert_eq_m512(r, e); - let r = _mm512_fmsubadd_round_ps(a, b, c, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); + let r = _mm512_fmsubadd_round_ps::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(a, b, c); let e = _mm512_setr_ps( -0.9999999, 1., -0.9999999, 1., -0.9999999, 1., -0.9999999, 1., -0.9999999, 1., -0.9999999, 1., -0.9999999, 1., -0.9999999, 1., @@ -43327,20 +40618,15 @@ mod tests { let a = _mm512_set1_ps(0.00000007); let b = _mm512_set1_ps(1.); let c = _mm512_set1_ps(-1.); - let r = _mm512_mask_fmsubadd_round_ps( - a, - 0, - b, - c, - _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC, + let r = _mm512_mask_fmsubadd_round_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, 0, b, c, ); assert_eq_m512(r, a); - let r = _mm512_mask_fmsubadd_round_ps( + let r = _mm512_mask_fmsubadd_round_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( a, 0b00000000_11111111, b, c, - _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC, ); #[rustfmt::skip] let e = _mm512_setr_ps( @@ -43357,20 +40643,15 @@ mod tests { let a = _mm512_set1_ps(0.00000007); let b = _mm512_set1_ps(1.); let c = _mm512_set1_ps(-1.); - let r = _mm512_maskz_fmsubadd_round_ps( - 0, - a, - b, - c, - _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC, + let r = _mm512_maskz_fmsubadd_round_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + 0, a, b, c, ); assert_eq_m512(r, _mm512_setzero_ps()); - let r = _mm512_maskz_fmsubadd_round_ps( + let r = _mm512_maskz_fmsubadd_round_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( 0b00000000_11111111, a, b, c, - _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC, ); #[rustfmt::skip] let e = _mm512_setr_ps( @@ -43387,20 +40668,15 @@ mod tests { let a = _mm512_set1_ps(0.00000007); let b = _mm512_set1_ps(1.); let c = _mm512_set1_ps(-1.); - let r = _mm512_mask3_fmsubadd_round_ps( - a, - b, - c, - 0, - _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC, + let r = _mm512_mask3_fmsubadd_round_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, b, c, 0, ); assert_eq_m512(r, c); - let r = _mm512_mask3_fmsubadd_round_ps( + let r = _mm512_mask3_fmsubadd_round_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( a, b, c, 0b00000000_11111111, - _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC, ); #[rustfmt::skip] let e = _mm512_setr_ps( @@ -43417,10 +40693,11 @@ mod tests { let a = _mm512_set1_ps(0.00000007); let b = _mm512_set1_ps(1.); let c = _mm512_set1_ps(1.); - let r = _mm512_fnmadd_round_ps(a, b, c, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); + let r = + _mm512_fnmadd_round_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>(a, b, c); let e = _mm512_set1_ps(0.99999994); assert_eq_m512(r, e); - let r = _mm512_fnmadd_round_ps(a, b, c, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); + let r = _mm512_fnmadd_round_ps::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(a, b, c); let e = _mm512_set1_ps(0.9999999); assert_eq_m512(r, e); } @@ -43430,15 +40707,15 @@ mod tests { let a = _mm512_set1_ps(0.00000007); let b = _mm512_set1_ps(1.); let c = _mm512_set1_ps(1.); - let r = - _mm512_mask_fnmadd_round_ps(a, 0, b, c, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); + let r = _mm512_mask_fnmadd_round_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, 0, b, c, + ); assert_eq_m512(r, a); - let r = _mm512_mask_fnmadd_round_ps( + let r = _mm512_mask_fnmadd_round_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( a, 0b00000000_11111111, b, c, - _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC, ); let e = _mm512_setr_ps( 0.99999994, 0.99999994, 0.99999994, 0.99999994, 0.99999994, 0.99999994, 0.99999994, @@ -43453,15 +40730,15 @@ mod tests { let a = _mm512_set1_ps(0.00000007); let b = _mm512_set1_ps(1.); let c = _mm512_set1_ps(1.); - let r = - _mm512_maskz_fnmadd_round_ps(0, a, b, c, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); + let r = _mm512_maskz_fnmadd_round_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + 0, a, b, c, + ); assert_eq_m512(r, _mm512_setzero_ps()); - let r = _mm512_maskz_fnmadd_round_ps( + let r = _mm512_maskz_fnmadd_round_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( 0b00000000_11111111, a, b, c, - _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC, ); let e = _mm512_setr_ps( 0.99999994, 0.99999994, 0.99999994, 0.99999994, 0.99999994, 0.99999994, 0.99999994, @@ -43475,15 +40752,15 @@ mod tests { let a = _mm512_set1_ps(0.00000007); let b = _mm512_set1_ps(1.); let c = _mm512_set1_ps(1.); - let r = - _mm512_mask3_fnmadd_round_ps(a, b, c, 0, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); + let r = _mm512_mask3_fnmadd_round_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, b, c, 0, + ); assert_eq_m512(r, c); - let r = _mm512_mask3_fnmadd_round_ps( + let r = _mm512_mask3_fnmadd_round_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( a, b, c, 0b00000000_11111111, - _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC, ); let e = _mm512_setr_ps( 0.99999994, 0.99999994, 0.99999994, 0.99999994, 0.99999994, 0.99999994, 0.99999994, @@ -43497,10 +40774,11 @@ mod tests { let a = _mm512_set1_ps(0.00000007); let b = _mm512_set1_ps(1.); let c = _mm512_set1_ps(-1.); - let r = _mm512_fnmsub_round_ps(a, b, c, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); + let r = + _mm512_fnmsub_round_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>(a, b, c); let e = _mm512_set1_ps(0.99999994); assert_eq_m512(r, e); - let r = _mm512_fnmsub_round_ps(a, b, c, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); + let r = _mm512_fnmsub_round_ps::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(a, b, c); let e = _mm512_set1_ps(0.9999999); assert_eq_m512(r, e); } @@ -43510,15 +40788,15 @@ mod tests { let a = _mm512_set1_ps(0.00000007); let b = _mm512_set1_ps(1.); let c = _mm512_set1_ps(-1.); - let r = - _mm512_mask_fnmsub_round_ps(a, 0, b, c, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); + let r = _mm512_mask_fnmsub_round_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, 0, b, c, + ); assert_eq_m512(r, a); - let r = _mm512_mask_fnmsub_round_ps( + let r = _mm512_mask_fnmsub_round_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( a, 0b00000000_11111111, b, c, - _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC, ); let e = _mm512_setr_ps( 0.99999994, 0.99999994, 0.99999994, 0.99999994, 0.99999994, 0.99999994, 0.99999994, @@ -43533,15 +40811,15 @@ mod tests { let a = _mm512_set1_ps(0.00000007); let b = _mm512_set1_ps(1.); let c = _mm512_set1_ps(-1.); - let r = - _mm512_maskz_fnmsub_round_ps(0, a, b, c, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); + let r = _mm512_maskz_fnmsub_round_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + 0, a, b, c, + ); assert_eq_m512(r, _mm512_setzero_ps()); - let r = _mm512_maskz_fnmsub_round_ps( + let r = _mm512_maskz_fnmsub_round_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( 0b00000000_11111111, a, b, c, - _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC, ); let e = _mm512_setr_ps( 0.99999994, 0.99999994, 0.99999994, 0.99999994, 0.99999994, 0.99999994, 0.99999994, @@ -43555,15 +40833,15 @@ mod tests { let a = _mm512_set1_ps(0.00000007); let b = _mm512_set1_ps(1.); let c = _mm512_set1_ps(-1.); - let r = - _mm512_mask3_fnmsub_round_ps(a, b, c, 0, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); + let r = _mm512_mask3_fnmsub_round_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, b, c, 0, + ); assert_eq_m512(r, c); - let r = _mm512_mask3_fnmsub_round_ps( + let r = _mm512_mask3_fnmsub_round_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( a, b, c, 0b00000000_11111111, - _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC, ); let e = _mm512_setr_ps( 0.99999994, 0.99999994, 0.99999994, 0.99999994, 0.99999994, 0.99999994, 0.99999994, @@ -43580,7 +40858,7 @@ mod tests { let b = _mm512_setr_ps( 15., 14., 13., 12., 11., 10., 9., 8., 7., 6., 5., 4., 3., 2., 1., 0., ); - let r = _mm512_max_round_ps(a, b, _MM_FROUND_CUR_DIRECTION); + let r = _mm512_max_round_ps::<_MM_FROUND_CUR_DIRECTION>(a, b); let e = _mm512_setr_ps( 15., 14., 13., 12., 11., 10., 9., 8., 8., 9., 10., 11., 12., 13., 14., 15., ); @@ -43595,9 +40873,9 @@ mod tests { let b = _mm512_setr_ps( 15., 14., 13., 12., 11., 10., 9., 8., 7., 6., 5., 4., 3., 2., 1., 0., ); - let r = _mm512_mask_max_round_ps(a, 0, a, b, _MM_FROUND_CUR_DIRECTION); + let r = _mm512_mask_max_round_ps::<_MM_FROUND_CUR_DIRECTION>(a, 0, a, b); assert_eq_m512(r, a); - let r = _mm512_mask_max_round_ps(a, 0b00000000_11111111, a, b, _MM_FROUND_CUR_DIRECTION); + let r = _mm512_mask_max_round_ps::<_MM_FROUND_CUR_DIRECTION>(a, 0b00000000_11111111, a, b); let e = _mm512_setr_ps( 15., 14., 13., 12., 11., 10., 9., 8., 8., 9., 10., 11., 12., 13., 14., 15., ); @@ -43612,9 +40890,9 @@ mod tests { let b = _mm512_setr_ps( 15., 14., 13., 12., 11., 10., 9., 8., 7., 6., 5., 4., 3., 2., 1., 0., ); - let r = _mm512_maskz_max_round_ps(0, a, b, _MM_FROUND_CUR_DIRECTION); + let r = _mm512_maskz_max_round_ps::<_MM_FROUND_CUR_DIRECTION>(0, a, b); assert_eq_m512(r, _mm512_setzero_ps()); - let r = _mm512_maskz_max_round_ps(0b00000000_11111111, a, b, _MM_FROUND_CUR_DIRECTION); + let r = _mm512_maskz_max_round_ps::<_MM_FROUND_CUR_DIRECTION>(0b00000000_11111111, a, b); let e = _mm512_setr_ps( 15., 14., 13., 12., 11., 10., 9., 8., 0., 0., 0., 0., 0., 0., 0., 0., ); @@ -43629,7 +40907,7 @@ mod tests { let b = _mm512_setr_ps( 15., 14., 13., 12., 11., 10., 9., 8., 7., 6., 5., 4., 3., 2., 1., 0., ); - let r = _mm512_min_round_ps(a, b, _MM_FROUND_CUR_DIRECTION); + let r = _mm512_min_round_ps::<_MM_FROUND_CUR_DIRECTION>(a, b); let e = _mm512_setr_ps( 0., 1., 2., 3., 4., 5., 6., 7., 7., 6., 5., 4., 3., 2., 1., 0., ); @@ -43644,9 +40922,9 @@ mod tests { let b = _mm512_setr_ps( 15., 14., 13., 12., 11., 10., 9., 8., 7., 6., 5., 4., 3., 2., 1., 0., ); - let r = _mm512_mask_min_round_ps(a, 0, a, b, _MM_FROUND_CUR_DIRECTION); + let r = _mm512_mask_min_round_ps::<_MM_FROUND_CUR_DIRECTION>(a, 0, a, b); assert_eq_m512(r, a); - let r = _mm512_mask_min_round_ps(a, 0b00000000_11111111, a, b, _MM_FROUND_CUR_DIRECTION); + let r = _mm512_mask_min_round_ps::<_MM_FROUND_CUR_DIRECTION>(a, 0b00000000_11111111, a, b); let e = _mm512_setr_ps( 0., 1., 2., 3., 4., 5., 6., 7., 8., 9., 10., 11., 12., 13., 14., 15., ); @@ -43661,9 +40939,9 @@ mod tests { let b = _mm512_setr_ps( 15., 14., 13., 12., 11., 10., 9., 8., 7., 6., 5., 4., 3., 2., 1., 0., ); - let r = _mm512_maskz_min_round_ps(0, a, b, _MM_FROUND_CUR_DIRECTION); + let r = _mm512_maskz_min_round_ps::<_MM_FROUND_CUR_DIRECTION>(0, a, b); assert_eq_m512(r, _mm512_setzero_ps()); - let r = _mm512_maskz_min_round_ps(0b00000000_11111111, a, b, _MM_FROUND_CUR_DIRECTION); + let r = _mm512_maskz_min_round_ps::<_MM_FROUND_CUR_DIRECTION>(0b00000000_11111111, a, b); let e = _mm512_setr_ps( 0., 1., 2., 3., 4., 5., 6., 7., 0., 0., 0., 0., 0., 0., 0., 0., ); @@ -43673,10 +40951,7 @@ mod tests { #[simd_test(enable = "avx512f")] unsafe fn test_mm512_getexp_round_ps() { let a = _mm512_set1_ps(3.); - let r = _mm512_getexp_round_ps(a, _MM_FROUND_CUR_DIRECTION); - let e = _mm512_set1_ps(1.); - assert_eq_m512(r, e); - let r = _mm512_getexp_round_ps(a, _MM_FROUND_NO_EXC); + let r = _mm512_getexp_round_ps::<_MM_FROUND_CUR_DIRECTION>(a); let e = _mm512_set1_ps(1.); assert_eq_m512(r, e); } @@ -43684,9 +40959,9 @@ mod tests { #[simd_test(enable = "avx512f")] unsafe fn test_mm512_mask_getexp_round_ps() { let a = _mm512_set1_ps(3.); - let r = _mm512_mask_getexp_round_ps(a, 0, a, _MM_FROUND_CUR_DIRECTION); + let r = _mm512_mask_getexp_round_ps::<_MM_FROUND_CUR_DIRECTION>(a, 0, a); assert_eq_m512(r, a); - let r = _mm512_mask_getexp_round_ps(a, 0b11111111_00000000, a, _MM_FROUND_CUR_DIRECTION); + let r = _mm512_mask_getexp_round_ps::<_MM_FROUND_CUR_DIRECTION>(a, 0b11111111_00000000, a); let e = _mm512_setr_ps( 3., 3., 3., 3., 3., 3., 3., 3., 1., 1., 1., 1., 1., 1., 1., 1., ); @@ -43696,9 +40971,9 @@ mod tests { #[simd_test(enable = "avx512f")] unsafe fn test_mm512_maskz_getexp_round_ps() { let a = _mm512_set1_ps(3.); - let r = _mm512_maskz_getexp_round_ps(0, a, _MM_FROUND_CUR_DIRECTION); + let r = _mm512_maskz_getexp_round_ps::<_MM_FROUND_CUR_DIRECTION>(0, a); assert_eq_m512(r, _mm512_setzero_ps()); - let r = _mm512_maskz_getexp_round_ps(0b11111111_00000000, a, _MM_FROUND_CUR_DIRECTION); + let r = _mm512_maskz_getexp_round_ps::<_MM_FROUND_CUR_DIRECTION>(0b11111111_00000000, a); let e = _mm512_setr_ps( 0., 0., 0., 0., 0., 0., 0., 0., 1., 1., 1., 1., 1., 1., 1., 1., ); @@ -43708,7 +40983,7 @@ mod tests { #[simd_test(enable = "avx512f")] unsafe fn test_mm512_roundscale_round_ps() { let a = _mm512_set1_ps(1.1); - let r = _mm512_roundscale_round_ps(a, 0, _MM_FROUND_CUR_DIRECTION); + let r = _mm512_roundscale_round_ps::<0, _MM_FROUND_CUR_DIRECTION>(a); let e = _mm512_set1_ps(1.0); assert_eq_m512(r, e); } @@ -43716,11 +40991,14 @@ mod tests { #[simd_test(enable = "avx512f")] unsafe fn test_mm512_mask_roundscale_round_ps() { let a = _mm512_set1_ps(1.1); - let r = _mm512_mask_roundscale_round_ps(a, 0, a, 0, _MM_FROUND_CUR_DIRECTION); + let r = _mm512_mask_roundscale_round_ps::<0, _MM_FROUND_CUR_DIRECTION>(a, 0, a); let e = _mm512_set1_ps(1.1); assert_eq_m512(r, e); - let r = - _mm512_mask_roundscale_round_ps(a, 0b11111111_11111111, a, 0, _MM_FROUND_CUR_DIRECTION); + let r = _mm512_mask_roundscale_round_ps::<0, _MM_FROUND_CUR_DIRECTION>( + a, + 0b11111111_11111111, + a, + ); let e = _mm512_set1_ps(1.0); assert_eq_m512(r, e); } @@ -43728,10 +41006,10 @@ mod tests { #[simd_test(enable = "avx512f")] unsafe fn test_mm512_maskz_roundscale_round_ps() { let a = _mm512_set1_ps(1.1); - let r = _mm512_maskz_roundscale_round_ps(0, a, 0, _MM_FROUND_CUR_DIRECTION); + let r = _mm512_maskz_roundscale_round_ps::<0, _MM_FROUND_CUR_DIRECTION>(0, a); assert_eq_m512(r, _mm512_setzero_ps()); let r = - _mm512_maskz_roundscale_round_ps(0b11111111_11111111, a, 0, _MM_FROUND_CUR_DIRECTION); + _mm512_maskz_roundscale_round_ps::<0, _MM_FROUND_CUR_DIRECTION>(0b11111111_11111111, a); let e = _mm512_set1_ps(1.0); assert_eq_m512(r, e); } @@ -43740,7 +41018,7 @@ mod tests { unsafe fn test_mm512_scalef_round_ps() { let a = _mm512_set1_ps(1.); let b = _mm512_set1_ps(3.); - let r = _mm512_scalef_round_ps(a, b, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); + let r = _mm512_scalef_round_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>(a, b); let e = _mm512_set1_ps(8.); assert_eq_m512(r, e); } @@ -43749,15 +41027,15 @@ mod tests { unsafe fn test_mm512_mask_scalef_round_ps() { let a = _mm512_set1_ps(1.); let b = _mm512_set1_ps(3.); - let r = - _mm512_mask_scalef_round_ps(a, 0, a, b, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); + let r = _mm512_mask_scalef_round_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, 0, a, b, + ); assert_eq_m512(r, a); - let r = _mm512_mask_scalef_round_ps( + let r = _mm512_mask_scalef_round_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( a, 0b11111111_00000000, a, b, - _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC, ); let e = _mm512_set_ps( 8., 8., 8., 8., 8., 8., 8., 8., 1., 1., 1., 1., 1., 1., 1., 1., @@ -43769,14 +41047,14 @@ mod tests { unsafe fn test_mm512_maskz_scalef_round_ps() { let a = _mm512_set1_ps(1.); let b = _mm512_set1_ps(3.); - let r = - _mm512_maskz_scalef_round_ps(0, a, b, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); + let r = _mm512_maskz_scalef_round_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + 0, a, b, + ); assert_eq_m512(r, _mm512_setzero_ps()); - let r = _mm512_maskz_scalef_round_ps( + let r = _mm512_maskz_scalef_round_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( 0b11111111_00000000, a, b, - _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC, ); let e = _mm512_set_ps( 8., 8., 8., 8., 8., 8., 8., 8., 0., 0., 0., 0., 0., 0., 0., 0., @@ -43789,7 +41067,7 @@ mod tests { let a = _mm512_set1_ps(f32::NAN); let b = _mm512_set1_ps(f32::MAX); let c = _mm512_set1_epi32(i32::MAX); - let r = _mm512_fixupimm_round_ps(a, b, c, 5, _MM_FROUND_CUR_DIRECTION); + let r = _mm512_fixupimm_round_ps::<5, _MM_FROUND_CUR_DIRECTION>(a, b, c); let e = _mm512_set1_ps(0.0); assert_eq_m512(r, e); } @@ -43805,13 +41083,11 @@ mod tests { ); let b = _mm512_set1_ps(f32::MAX); let c = _mm512_set1_epi32(i32::MAX); - let r = _mm512_mask_fixupimm_round_ps( + let r = _mm512_mask_fixupimm_round_ps::<5, _MM_FROUND_CUR_DIRECTION>( a, 0b11111111_00000000, b, c, - 5, - _MM_FROUND_CUR_DIRECTION, ); let e = _mm512_set_ps( 0., 0., 0., 0., 0., 0., 0., 0., 1., 1., 1., 1., 1., 1., 1., 1., @@ -43830,13 +41106,11 @@ mod tests { ); let b = _mm512_set1_ps(f32::MAX); let c = _mm512_set1_epi32(i32::MAX); - let r = _mm512_maskz_fixupimm_round_ps( + let r = _mm512_maskz_fixupimm_round_ps::<5, _MM_FROUND_CUR_DIRECTION>( 0b11111111_00000000, a, b, c, - 5, - _MM_FROUND_CUR_DIRECTION, ); let e = _mm512_set_ps( 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., @@ -43847,12 +41121,11 @@ mod tests { #[simd_test(enable = "avx512f")] unsafe fn test_mm512_getmant_round_ps() { let a = _mm512_set1_ps(10.); - let r = _mm512_getmant_round_ps( - a, + let r = _mm512_getmant_round_ps::< _MM_MANT_NORM_1_2, _MM_MANT_SIGN_SRC, _MM_FROUND_CUR_DIRECTION, - ); + >(a); let e = _mm512_set1_ps(1.25); assert_eq_m512(r, e); } @@ -43860,23 +41133,17 @@ mod tests { #[simd_test(enable = "avx512f")] unsafe fn test_mm512_mask_getmant_round_ps() { let a = _mm512_set1_ps(10.); - let r = _mm512_mask_getmant_round_ps( - a, - 0, - a, + let r = _mm512_mask_getmant_round_ps::< _MM_MANT_NORM_1_2, _MM_MANT_SIGN_SRC, _MM_FROUND_CUR_DIRECTION, - ); + >(a, 0, a); assert_eq_m512(r, a); - let r = _mm512_mask_getmant_round_ps( - a, - 0b11111111_00000000, - a, + let r = _mm512_mask_getmant_round_ps::< _MM_MANT_NORM_1_2, _MM_MANT_SIGN_SRC, _MM_FROUND_CUR_DIRECTION, - ); + >(a, 0b11111111_00000000, a); let e = _mm512_setr_ps( 10., 10., 10., 10., 10., 10., 10., 10., 1.25, 1.25, 1.25, 1.25, 1.25, 1.25, 1.25, 1.25, ); @@ -43886,21 +41153,17 @@ mod tests { #[simd_test(enable = "avx512f")] unsafe fn test_mm512_maskz_getmant_round_ps() { let a = _mm512_set1_ps(10.); - let r = _mm512_maskz_getmant_round_ps( - 0, - a, + let r = _mm512_maskz_getmant_round_ps::< _MM_MANT_NORM_1_2, _MM_MANT_SIGN_SRC, _MM_FROUND_CUR_DIRECTION, - ); + >(0, a); assert_eq_m512(r, _mm512_setzero_ps()); - let r = _mm512_maskz_getmant_round_ps( - 0b11111111_00000000, - a, + let r = _mm512_maskz_getmant_round_ps::< _MM_MANT_NORM_1_2, _MM_MANT_SIGN_SRC, _MM_FROUND_CUR_DIRECTION, - ); + >(0b11111111_00000000, a); let e = _mm512_setr_ps( 0., 0., 0., 0., 0., 0., 0., 0., 1.25, 1.25, 1.25, 1.25, 1.25, 1.25, 1.25, 1.25, ); @@ -45144,10 +42407,10 @@ mod tests { let a = _mm512_setr_ps( 0., -1.5, 2., -3.5, 4., -5.5, 6., -7.5, 8., 9.5, 10., 11.5, 12., 13.5, 14., 15.5, ); - let r = _mm512_cvt_roundps_epi32(a, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); + let r = _mm512_cvt_roundps_epi32::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>(a); let e = _mm512_setr_epi32(0, -2, 2, -4, 4, -6, 6, -8, 8, 10, 10, 12, 12, 14, 14, 16); assert_eq_m512i(r, e); - let r = _mm512_cvt_roundps_epi32(a, _MM_FROUND_TO_NEG_INF | _MM_FROUND_NO_EXC); + let r = _mm512_cvt_roundps_epi32::<{ _MM_FROUND_TO_NEG_INF | _MM_FROUND_NO_EXC }>(a); let e = _mm512_setr_epi32(0, -2, 2, -4, 4, -6, 6, -8, 8, 9, 10, 11, 12, 13, 14, 15); assert_eq_m512i(r, e); } @@ -45158,14 +42421,14 @@ mod tests { 0., -1.5, 2., -3.5, 4., -5.5, 6., -7.5, 8., 9.5, 10., 11.5, 12., 13.5, 14., 15.5, ); let src = _mm512_set1_epi32(0); - let r = - _mm512_mask_cvt_roundps_epi32(src, 0, a, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); + let r = _mm512_mask_cvt_roundps_epi32::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + src, 0, a, + ); assert_eq_m512i(r, src); - let r = _mm512_mask_cvt_roundps_epi32( + let r = _mm512_mask_cvt_roundps_epi32::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( src, 0b00000000_11111111, a, - _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC, ); let e = _mm512_setr_epi32(0, -2, 2, -4, 4, -6, 6, -8, 0, 0, 0, 0, 0, 0, 0, 0); assert_eq_m512i(r, e); @@ -45176,12 +42439,13 @@ mod tests { let a = _mm512_setr_ps( 0., -1.5, 2., -3.5, 4., -5.5, 6., -7.5, 8., 9.5, 10., 11.5, 12., 13.5, 14., 15.5, ); - let r = _mm512_maskz_cvt_roundps_epi32(0, a, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); + let r = _mm512_maskz_cvt_roundps_epi32::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + 0, a, + ); assert_eq_m512i(r, _mm512_setzero_si512()); - let r = _mm512_maskz_cvt_roundps_epi32( + let r = _mm512_maskz_cvt_roundps_epi32::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( 0b00000000_11111111, a, - _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC, ); let e = _mm512_setr_epi32(0, -2, 2, -4, 4, -6, 6, -8, 0, 0, 0, 0, 0, 0, 0, 0); assert_eq_m512i(r, e); @@ -45192,10 +42456,10 @@ mod tests { let a = _mm512_setr_ps( 0., -1.5, 2., -3.5, 4., -5.5, 6., -7.5, 8., 9.5, 10., 11.5, 12., 13.5, 14., 15.5, ); - let r = _mm512_cvt_roundps_epu32(a, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); + let r = _mm512_cvt_roundps_epu32::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>(a); let e = _mm512_setr_epi32(0, -1, 2, -1, 4, -1, 6, -1, 8, 10, 10, 12, 12, 14, 14, 16); assert_eq_m512i(r, e); - let r = _mm512_cvt_roundps_epu32(a, _MM_FROUND_TO_NEG_INF | _MM_FROUND_NO_EXC); + let r = _mm512_cvt_roundps_epu32::<{ _MM_FROUND_TO_NEG_INF | _MM_FROUND_NO_EXC }>(a); let e = _mm512_setr_epi32(0, -1, 2, -1, 4, -1, 6, -1, 8, 9, 10, 11, 12, 13, 14, 15); assert_eq_m512i(r, e); } @@ -45206,14 +42470,14 @@ mod tests { 0., -1.5, 2., -3.5, 4., -5.5, 6., -7.5, 8., 9.5, 10., 11.5, 12., 13.5, 14., 15.5, ); let src = _mm512_set1_epi32(0); - let r = - _mm512_mask_cvt_roundps_epu32(src, 0, a, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); + let r = _mm512_mask_cvt_roundps_epu32::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + src, 0, a, + ); assert_eq_m512i(r, src); - let r = _mm512_mask_cvt_roundps_epu32( + let r = _mm512_mask_cvt_roundps_epu32::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( src, 0b00000000_11111111, a, - _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC, ); let e = _mm512_setr_epi32(0, -1, 2, -1, 4, -1, 6, -1, 0, 0, 0, 0, 0, 0, 0, 0); assert_eq_m512i(r, e); @@ -45224,12 +42488,13 @@ mod tests { let a = _mm512_setr_ps( 0., -1.5, 2., -3.5, 4., -5.5, 6., -7.5, 8., 9.5, 10., 11.5, 12., 13.5, 14., 15.5, ); - let r = _mm512_maskz_cvt_roundps_epu32(0, a, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); + let r = _mm512_maskz_cvt_roundps_epu32::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + 0, a, + ); assert_eq_m512i(r, _mm512_setzero_si512()); - let r = _mm512_maskz_cvt_roundps_epu32( + let r = _mm512_maskz_cvt_roundps_epu32::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( 0b00000000_11111111, a, - _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC, ); let e = _mm512_setr_epi32(0, -1, 2, -1, 4, -1, 6, -1, 0, 0, 0, 0, 0, 0, 0, 0); assert_eq_m512i(r, e); @@ -45238,7 +42503,7 @@ mod tests { #[simd_test(enable = "avx512f")] unsafe fn test_mm512_cvt_roundepi32_ps() { let a = _mm512_setr_epi32(0, -2, 2, -4, 4, -6, 6, -8, 8, 10, 10, 12, 12, 14, 14, 16); - let r = _mm512_cvt_roundepi32_ps(a, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); + let r = _mm512_cvt_roundepi32_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>(a); let e = _mm512_setr_ps( 0., -2., 2., -4., 4., -6., 6., -8., 8., 10., 10., 12., 12., 14., 14., 16., ); @@ -45249,14 +42514,14 @@ mod tests { unsafe fn test_mm512_mask_cvt_roundepi32_ps() { let a = _mm512_setr_epi32(0, -2, 2, -4, 4, -6, 6, -8, 8, 10, 10, 12, 12, 14, 14, 16); let src = _mm512_set1_ps(0.); - let r = - _mm512_mask_cvt_roundepi32_ps(src, 0, a, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); + let r = _mm512_mask_cvt_roundepi32_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + src, 0, a, + ); assert_eq_m512(r, src); - let r = _mm512_mask_cvt_roundepi32_ps( + let r = _mm512_mask_cvt_roundepi32_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( src, 0b00000000_11111111, a, - _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC, ); let e = _mm512_setr_ps( 0., -2., 2., -4., 4., -6., 6., -8., 0., 0., 0., 0., 0., 0., 0., 0., @@ -45267,12 +42532,13 @@ mod tests { #[simd_test(enable = "avx512f")] unsafe fn test_mm512_maskz_cvt_roundepi32_ps() { let a = _mm512_setr_epi32(0, -2, 2, -4, 4, -6, 6, -8, 8, 10, 10, 12, 12, 14, 14, 16); - let r = _mm512_maskz_cvt_roundepi32_ps(0, a, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); + let r = _mm512_maskz_cvt_roundepi32_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + 0, a, + ); assert_eq_m512(r, _mm512_setzero_ps()); - let r = _mm512_maskz_cvt_roundepi32_ps( + let r = _mm512_maskz_cvt_roundepi32_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( 0b00000000_11111111, a, - _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC, ); let e = _mm512_setr_ps( 0., -2., 2., -4., 4., -6., 6., -8., 0., 0., 0., 0., 0., 0., 0., 0., @@ -45283,7 +42549,7 @@ mod tests { #[simd_test(enable = "avx512f")] unsafe fn test_mm512_cvt_roundepu32_ps() { let a = _mm512_setr_epi32(0, -2, 2, -4, 4, -6, 6, -8, 8, 10, 10, 12, 12, 14, 14, 16); - let r = _mm512_cvt_roundepu32_ps(a, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); + let r = _mm512_cvt_roundepu32_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>(a); #[rustfmt::skip] let e = _mm512_setr_ps( 0., 4294967300., 2., 4294967300., @@ -45298,14 +42564,14 @@ mod tests { unsafe fn test_mm512_mask_cvt_roundepu32_ps() { let a = _mm512_setr_epi32(0, -2, 2, -4, 4, -6, 6, -8, 8, 10, 10, 12, 12, 14, 14, 16); let src = _mm512_set1_ps(0.); - let r = - _mm512_mask_cvt_roundepu32_ps(src, 0, a, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); + let r = _mm512_mask_cvt_roundepu32_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + src, 0, a, + ); assert_eq_m512(r, src); - let r = _mm512_mask_cvt_roundepu32_ps( + let r = _mm512_mask_cvt_roundepu32_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( src, 0b00000000_11111111, a, - _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC, ); #[rustfmt::skip] let e = _mm512_setr_ps( @@ -45320,12 +42586,13 @@ mod tests { #[simd_test(enable = "avx512f")] unsafe fn test_mm512_maskz_cvt_roundepu32_ps() { let a = _mm512_setr_epi32(0, -2, 2, -4, 4, -6, 6, -8, 8, 10, 10, 12, 12, 14, 14, 16); - let r = _mm512_maskz_cvt_roundepu32_ps(0, a, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); + let r = _mm512_maskz_cvt_roundepu32_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + 0, a, + ); assert_eq_m512(r, _mm512_setzero_ps()); - let r = _mm512_maskz_cvt_roundepu32_ps( + let r = _mm512_maskz_cvt_roundepu32_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( 0b00000000_11111111, a, - _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC, ); #[rustfmt::skip] let e = _mm512_setr_ps( @@ -45340,7 +42607,7 @@ mod tests { #[simd_test(enable = "avx512f")] unsafe fn test_mm512_cvt_roundps_ph() { let a = _mm512_set1_ps(1.); - let r = _mm512_cvt_roundps_ph(a, _MM_FROUND_NO_EXC); + let r = _mm512_cvt_roundps_ph::<_MM_FROUND_NO_EXC>(a); let e = _mm256_setr_epi64x( 4323521613979991040, 4323521613979991040, @@ -45354,9 +42621,9 @@ mod tests { unsafe fn test_mm512_mask_cvt_roundps_ph() { let a = _mm512_set1_ps(1.); let src = _mm256_set1_epi16(0); - let r = _mm512_mask_cvt_roundps_ph(src, 0, a, _MM_FROUND_NO_EXC); + let r = _mm512_mask_cvt_roundps_ph::<_MM_FROUND_NO_EXC>(src, 0, a); assert_eq_m256i(r, src); - let r = _mm512_mask_cvt_roundps_ph(src, 0b00000000_11111111, a, _MM_FROUND_NO_EXC); + let r = _mm512_mask_cvt_roundps_ph::<_MM_FROUND_NO_EXC>(src, 0b00000000_11111111, a); let e = _mm256_setr_epi64x(4323521613979991040, 4323521613979991040, 0, 0); assert_eq_m256i(r, e); } @@ -45364,9 +42631,9 @@ mod tests { #[simd_test(enable = "avx512f")] unsafe fn test_mm512_maskz_cvt_roundps_ph() { let a = _mm512_set1_ps(1.); - let r = _mm512_maskz_cvt_roundps_ph(0, a, _MM_FROUND_NO_EXC); + let r = _mm512_maskz_cvt_roundps_ph::<_MM_FROUND_NO_EXC>(0, a); assert_eq_m256i(r, _mm256_setzero_si256()); - let r = _mm512_maskz_cvt_roundps_ph(0b00000000_11111111, a, _MM_FROUND_NO_EXC); + let r = _mm512_maskz_cvt_roundps_ph::<_MM_FROUND_NO_EXC>(0b00000000_11111111, a); let e = _mm256_setr_epi64x(4323521613979991040, 4323521613979991040, 0, 0); assert_eq_m256i(r, e); } @@ -45375,9 +42642,9 @@ mod tests { unsafe fn test_mm256_mask_cvt_roundps_ph() { let a = _mm256_set1_ps(1.); let src = _mm_set1_epi16(0); - let r = _mm256_mask_cvt_roundps_ph(src, 0, a, _MM_FROUND_NO_EXC); + let r = _mm256_mask_cvt_roundps_ph::<_MM_FROUND_NO_EXC>(src, 0, a); assert_eq_m128i(r, src); - let r = _mm256_mask_cvt_roundps_ph(src, 0b11111111, a, _MM_FROUND_NO_EXC); + let r = _mm256_mask_cvt_roundps_ph::<_MM_FROUND_NO_EXC>(src, 0b11111111, a); let e = _mm_setr_epi64x(4323521613979991040, 4323521613979991040); assert_eq_m128i(r, e); } @@ -45385,9 +42652,9 @@ mod tests { #[simd_test(enable = "avx512f,avx512vl")] unsafe fn test_mm256_maskz_cvt_roundps_ph() { let a = _mm256_set1_ps(1.); - let r = _mm256_maskz_cvt_roundps_ph(0, a, _MM_FROUND_NO_EXC); + let r = _mm256_maskz_cvt_roundps_ph::<_MM_FROUND_NO_EXC>(0, a); assert_eq_m128i(r, _mm_setzero_si128()); - let r = _mm256_maskz_cvt_roundps_ph(0b11111111, a, _MM_FROUND_NO_EXC); + let r = _mm256_maskz_cvt_roundps_ph::<_MM_FROUND_NO_EXC>(0b11111111, a); let e = _mm_setr_epi64x(4323521613979991040, 4323521613979991040); assert_eq_m128i(r, e); } @@ -45396,9 +42663,9 @@ mod tests { unsafe fn test_mm_mask_cvt_roundps_ph() { let a = _mm_set1_ps(1.); let src = _mm_set1_epi16(0); - let r = _mm_mask_cvt_roundps_ph(src, 0, a, _MM_FROUND_NO_EXC); + let r = _mm_mask_cvt_roundps_ph::<_MM_FROUND_NO_EXC>(src, 0, a); assert_eq_m128i(r, src); - let r = _mm_mask_cvt_roundps_ph(src, 0b00001111, a, _MM_FROUND_NO_EXC); + let r = _mm_mask_cvt_roundps_ph::<_MM_FROUND_NO_EXC>(src, 0b00001111, a); let e = _mm_setr_epi64x(4323521613979991040, 0); assert_eq_m128i(r, e); } @@ -45406,9 +42673,9 @@ mod tests { #[simd_test(enable = "avx512f,avx512vl")] unsafe fn test_mm_maskz_cvt_roundps_ph() { let a = _mm_set1_ps(1.); - let r = _mm_maskz_cvt_roundps_ph(0, a, _MM_FROUND_NO_EXC); + let r = _mm_maskz_cvt_roundps_ph::<_MM_FROUND_NO_EXC>(0, a); assert_eq_m128i(r, _mm_setzero_si128()); - let r = _mm_maskz_cvt_roundps_ph(0b00001111, a, _MM_FROUND_NO_EXC); + let r = _mm_maskz_cvt_roundps_ph::<_MM_FROUND_NO_EXC>(0b00001111, a); let e = _mm_setr_epi64x(4323521613979991040, 0); assert_eq_m128i(r, e); } @@ -45416,7 +42683,7 @@ mod tests { #[simd_test(enable = "avx512f")] unsafe fn test_mm512_cvtps_ph() { let a = _mm512_set1_ps(1.); - let r = _mm512_cvtps_ph(a, _MM_FROUND_NO_EXC); + let r = _mm512_cvtps_ph::<_MM_FROUND_NO_EXC>(a); let e = _mm256_setr_epi64x( 4323521613979991040, 4323521613979991040, @@ -45430,9 +42697,9 @@ mod tests { unsafe fn test_mm512_mask_cvtps_ph() { let a = _mm512_set1_ps(1.); let src = _mm256_set1_epi16(0); - let r = _mm512_mask_cvtps_ph(src, 0, a, _MM_FROUND_NO_EXC); + let r = _mm512_mask_cvtps_ph::<_MM_FROUND_NO_EXC>(src, 0, a); assert_eq_m256i(r, src); - let r = _mm512_mask_cvtps_ph(src, 0b00000000_11111111, a, _MM_FROUND_NO_EXC); + let r = _mm512_mask_cvtps_ph::<_MM_FROUND_NO_EXC>(src, 0b00000000_11111111, a); let e = _mm256_setr_epi64x(4323521613979991040, 4323521613979991040, 0, 0); assert_eq_m256i(r, e); } @@ -45440,9 +42707,9 @@ mod tests { #[simd_test(enable = "avx512f")] unsafe fn test_mm512_maskz_cvtps_ph() { let a = _mm512_set1_ps(1.); - let r = _mm512_maskz_cvtps_ph(0, a, _MM_FROUND_NO_EXC); + let r = _mm512_maskz_cvtps_ph::<_MM_FROUND_NO_EXC>(0, a); assert_eq_m256i(r, _mm256_setzero_si256()); - let r = _mm512_maskz_cvtps_ph(0b00000000_11111111, a, _MM_FROUND_NO_EXC); + let r = _mm512_maskz_cvtps_ph::<_MM_FROUND_NO_EXC>(0b00000000_11111111, a); let e = _mm256_setr_epi64x(4323521613979991040, 4323521613979991040, 0, 0); assert_eq_m256i(r, e); } @@ -45451,9 +42718,9 @@ mod tests { unsafe fn test_mm256_mask_cvtps_ph() { let a = _mm256_set1_ps(1.); let src = _mm_set1_epi16(0); - let r = _mm256_mask_cvtps_ph(src, 0, a, _MM_FROUND_NO_EXC); + let r = _mm256_mask_cvtps_ph::<_MM_FROUND_NO_EXC>(src, 0, a); assert_eq_m128i(r, src); - let r = _mm256_mask_cvtps_ph(src, 0b11111111, a, _MM_FROUND_NO_EXC); + let r = _mm256_mask_cvtps_ph::<_MM_FROUND_NO_EXC>(src, 0b11111111, a); let e = _mm_setr_epi64x(4323521613979991040, 4323521613979991040); assert_eq_m128i(r, e); } @@ -45461,9 +42728,9 @@ mod tests { #[simd_test(enable = "avx512f,avx512vl")] unsafe fn test_mm256_maskz_cvtps_ph() { let a = _mm256_set1_ps(1.); - let r = _mm256_maskz_cvtps_ph(0, a, _MM_FROUND_NO_EXC); + let r = _mm256_maskz_cvtps_ph::<_MM_FROUND_NO_EXC>(0, a); assert_eq_m128i(r, _mm_setzero_si128()); - let r = _mm256_maskz_cvtps_ph(0b11111111, a, _MM_FROUND_NO_EXC); + let r = _mm256_maskz_cvtps_ph::<_MM_FROUND_NO_EXC>(0b11111111, a); let e = _mm_setr_epi64x(4323521613979991040, 4323521613979991040); assert_eq_m128i(r, e); } @@ -45472,9 +42739,9 @@ mod tests { unsafe fn test_mm_mask_cvtps_ph() { let a = _mm_set1_ps(1.); let src = _mm_set1_epi16(0); - let r = _mm_mask_cvtps_ph(src, 0, a, _MM_FROUND_NO_EXC); + let r = _mm_mask_cvtps_ph::<_MM_FROUND_NO_EXC>(src, 0, a); assert_eq_m128i(r, src); - let r = _mm_mask_cvtps_ph(src, 0b00001111, a, _MM_FROUND_NO_EXC); + let r = _mm_mask_cvtps_ph::<_MM_FROUND_NO_EXC>(src, 0b00001111, a); let e = _mm_setr_epi64x(4323521613979991040, 0); assert_eq_m128i(r, e); } @@ -45482,9 +42749,9 @@ mod tests { #[simd_test(enable = "avx512f,avx512vl")] unsafe fn test_mm_maskz_cvtps_ph() { let a = _mm_set1_ps(1.); - let r = _mm_maskz_cvtps_ph(0, a, _MM_FROUND_NO_EXC); + let r = _mm_maskz_cvtps_ph::<_MM_FROUND_NO_EXC>(0, a); assert_eq_m128i(r, _mm_setzero_si128()); - let r = _mm_maskz_cvtps_ph(0b00001111, a, _MM_FROUND_NO_EXC); + let r = _mm_maskz_cvtps_ph::<_MM_FROUND_NO_EXC>(0b00001111, a); let e = _mm_setr_epi64x(4323521613979991040, 0); assert_eq_m128i(r, e); } @@ -45497,7 +42764,7 @@ mod tests { 4323521613979991040, 4323521613979991040, ); - let r = _mm512_cvt_roundph_ps(a, _MM_FROUND_NO_EXC); + let r = _mm512_cvt_roundph_ps::<_MM_FROUND_NO_EXC>(a); let e = _mm512_set1_ps(1.); assert_eq_m512(r, e); } @@ -45511,9 +42778,9 @@ mod tests { 4323521613979991040, ); let src = _mm512_set1_ps(0.); - let r = _mm512_mask_cvt_roundph_ps(src, 0, a, _MM_FROUND_NO_EXC); + let r = _mm512_mask_cvt_roundph_ps::<_MM_FROUND_NO_EXC>(src, 0, a); assert_eq_m512(r, src); - let r = _mm512_mask_cvt_roundph_ps(src, 0b00000000_11111111, a, _MM_FROUND_NO_EXC); + let r = _mm512_mask_cvt_roundph_ps::<_MM_FROUND_NO_EXC>(src, 0b00000000_11111111, a); let e = _mm512_setr_ps( 1., 1., 1., 1., 1., 1., 1., 1., 0., 0., 0., 0., 0., 0., 0., 0., ); @@ -45528,9 +42795,9 @@ mod tests { 4323521613979991040, 4323521613979991040, ); - let r = _mm512_maskz_cvt_roundph_ps(0, a, _MM_FROUND_NO_EXC); + let r = _mm512_maskz_cvt_roundph_ps::<_MM_FROUND_NO_EXC>(0, a); assert_eq_m512(r, _mm512_setzero_ps()); - let r = _mm512_maskz_cvt_roundph_ps(0b00000000_11111111, a, _MM_FROUND_NO_EXC); + let r = _mm512_maskz_cvt_roundph_ps::<_MM_FROUND_NO_EXC>(0b00000000_11111111, a); let e = _mm512_setr_ps( 1., 1., 1., 1., 1., 1., 1., 1., 0., 0., 0., 0., 0., 0., 0., 0., ); @@ -45632,7 +42899,7 @@ mod tests { let a = _mm512_setr_ps( 0., -1.5, 2., -3.5, 4., -5.5, 6., -7.5, 8., 9.5, 10., 11.5, 12., 13.5, 14., 15.5, ); - let r = _mm512_cvtt_roundps_epi32(a, _MM_FROUND_NO_EXC); + let r = _mm512_cvtt_roundps_epi32::<_MM_FROUND_NO_EXC>(a); let e = _mm512_setr_epi32(0, -1, 2, -3, 4, -5, 6, -7, 8, 9, 10, 11, 12, 13, 14, 15); assert_eq_m512i(r, e); } @@ -45643,9 +42910,9 @@ mod tests { 0., -1.5, 2., -3.5, 4., -5.5, 6., -7.5, 8., 9.5, 10., 11.5, 12., 13.5, 14., 15.5, ); let src = _mm512_set1_epi32(0); - let r = _mm512_mask_cvtt_roundps_epi32(src, 0, a, _MM_FROUND_NO_EXC); + let r = _mm512_mask_cvtt_roundps_epi32::<_MM_FROUND_NO_EXC>(src, 0, a); assert_eq_m512i(r, src); - let r = _mm512_mask_cvtt_roundps_epi32(src, 0b00000000_11111111, a, _MM_FROUND_NO_EXC); + let r = _mm512_mask_cvtt_roundps_epi32::<_MM_FROUND_NO_EXC>(src, 0b00000000_11111111, a); let e = _mm512_setr_epi32(0, -1, 2, -3, 4, -5, 6, -7, 0, 0, 0, 0, 0, 0, 0, 0); assert_eq_m512i(r, e); } @@ -45655,9 +42922,9 @@ mod tests { let a = _mm512_setr_ps( 0., -1.5, 2., -3.5, 4., -5.5, 6., -7.5, 8., 9.5, 10., 11.5, 12., 13.5, 14., 15.5, ); - let r = _mm512_maskz_cvtt_roundps_epi32(0, a, _MM_FROUND_NO_EXC); + let r = _mm512_maskz_cvtt_roundps_epi32::<_MM_FROUND_NO_EXC>(0, a); assert_eq_m512i(r, _mm512_setzero_si512()); - let r = _mm512_maskz_cvtt_roundps_epi32(0b00000000_11111111, a, _MM_FROUND_NO_EXC); + let r = _mm512_maskz_cvtt_roundps_epi32::<_MM_FROUND_NO_EXC>(0b00000000_11111111, a); let e = _mm512_setr_epi32(0, -1, 2, -3, 4, -5, 6, -7, 0, 0, 0, 0, 0, 0, 0, 0); assert_eq_m512i(r, e); } @@ -45667,7 +42934,7 @@ mod tests { let a = _mm512_setr_ps( 0., -1.5, 2., -3.5, 4., -5.5, 6., -7.5, 8., 9.5, 10., 11.5, 12., 13.5, 14., 15.5, ); - let r = _mm512_cvtt_roundps_epu32(a, _MM_FROUND_NO_EXC); + let r = _mm512_cvtt_roundps_epu32::<_MM_FROUND_NO_EXC>(a); let e = _mm512_setr_epi32(0, -1, 2, -1, 4, -1, 6, -1, 8, 9, 10, 11, 12, 13, 14, 15); assert_eq_m512i(r, e); } @@ -45678,9 +42945,9 @@ mod tests { 0., -1.5, 2., -3.5, 4., -5.5, 6., -7.5, 8., 9.5, 10., 11.5, 12., 13.5, 14., 15.5, ); let src = _mm512_set1_epi32(0); - let r = _mm512_mask_cvtt_roundps_epu32(src, 0, a, _MM_FROUND_NO_EXC); + let r = _mm512_mask_cvtt_roundps_epu32::<_MM_FROUND_NO_EXC>(src, 0, a); assert_eq_m512i(r, src); - let r = _mm512_mask_cvtt_roundps_epu32(src, 0b00000000_11111111, a, _MM_FROUND_NO_EXC); + let r = _mm512_mask_cvtt_roundps_epu32::<_MM_FROUND_NO_EXC>(src, 0b00000000_11111111, a); let e = _mm512_setr_epi32(0, -1, 2, -1, 4, -1, 6, -1, 0, 0, 0, 0, 0, 0, 0, 0); assert_eq_m512i(r, e); } @@ -45690,9 +42957,9 @@ mod tests { let a = _mm512_setr_ps( 0., -1.5, 2., -3.5, 4., -5.5, 6., -7.5, 8., 9.5, 10., 11.5, 12., 13.5, 14., 15.5, ); - let r = _mm512_maskz_cvtt_roundps_epu32(0, a, _MM_FROUND_NO_EXC); + let r = _mm512_maskz_cvtt_roundps_epu32::<_MM_FROUND_NO_EXC>(0, a); assert_eq_m512i(r, _mm512_setzero_si512()); - let r = _mm512_maskz_cvtt_roundps_epu32(0b00000000_11111111, a, _MM_FROUND_NO_EXC); + let r = _mm512_maskz_cvtt_roundps_epu32::<_MM_FROUND_NO_EXC>(0b00000000_11111111, a); let e = _mm512_setr_epi32(0, -1, 2, -1, 4, -1, 6, -1, 0, 0, 0, 0, 0, 0, 0, 0); assert_eq_m512i(r, e); } @@ -45877,7 +43144,7 @@ mod tests { #[rustfmt::skip] let index = _mm512_setr_epi32(0, 16, 32, 48, 64, 80, 96, 112, 120, 128, 136, 144, 152, 160, 168, 176); - let r = _mm512_i32gather_ps(index, arr.as_ptr() as *const u8, 4); + let r = _mm512_i32gather_ps::<4>(index, arr.as_ptr() as *const u8); #[rustfmt::skip] assert_eq_m512(r, _mm512_setr_ps(0., 16., 32., 48., 64., 80., 96., 112., 120., 128., 136., 144., 152., 160., 168., 176.)); @@ -45895,7 +43162,7 @@ mod tests { let index = _mm512_setr_epi32(0, 16, 32, 48, 64, 80, 96, 112, 120, 128, 136, 144, 152, 160, 168, 176); // A multiplier of 4 is word-addressing - let r = _mm512_mask_i32gather_ps(src, mask, index, arr.as_ptr() as *const u8, 4); + let r = _mm512_mask_i32gather_ps::<4>(src, mask, index, arr.as_ptr() as *const u8); #[rustfmt::skip] assert_eq_m512(r, _mm512_setr_ps(2., 16., 2., 48., 2., 80., 2., 112., 2., 128., 2., 144., 2., 160., 2., 176.)); @@ -45911,7 +43178,7 @@ mod tests { #[rustfmt::skip] let index = _mm512_setr_epi32(0, 16, 32, 48, 64, 80, 96, 112, 120, 128, 136, 144, 152, 160, 168, 176); - let r = _mm512_i32gather_epi32(index, arr.as_ptr() as *const u8, 4); + let r = _mm512_i32gather_epi32::<4>(index, arr.as_ptr() as *const u8); #[rustfmt::skip] assert_eq_m512i(r, _mm512_setr_epi32(0, 16, 32, 48, 64, 80, 96, 112, 120, 128, 136, 144, 152, 160, 168, 176)); @@ -45929,7 +43196,7 @@ mod tests { 0, 16, 32, 48, 64, 80, 96, 112, 128, 144, 160, 176, 192, 208, 224, 240, ); // A multiplier of 4 is word-addressing - let r = _mm512_mask_i32gather_epi32(src, mask, index, arr.as_ptr() as *const u8, 4); + let r = _mm512_mask_i32gather_epi32::<4>(src, mask, index, arr.as_ptr() as *const u8); assert_eq_m512i( r, _mm512_setr_epi32(2, 16, 2, 48, 2, 80, 2, 112, 2, 144, 2, 176, 2, 208, 2, 240), @@ -45946,7 +43213,7 @@ mod tests { 1., 2., 3., 4., 5., 6., 7., 8., 9., 10., 11., 12., 13., 14., 15., 16., ); // A multiplier of 4 is word-addressing - _mm512_i32scatter_ps(arr.as_mut_ptr() as *mut u8, index, src, 4); + _mm512_i32scatter_ps::<4>(arr.as_mut_ptr() as *mut u8, index, src); let mut expected = [0f32; 256]; for i in 0..16 { expected[i * 16] = (i + 1) as f32; @@ -45965,7 +43232,7 @@ mod tests { 1., 2., 3., 4., 5., 6., 7., 8., 9., 10., 11., 12., 13., 14., 15., 16., ); // A multiplier of 4 is word-addressing - _mm512_mask_i32scatter_ps(arr.as_mut_ptr() as *mut u8, mask, index, src, 4); + _mm512_mask_i32scatter_ps::<4>(arr.as_mut_ptr() as *mut u8, mask, index, src); let mut expected = [0f32; 256]; for i in 0..8 { expected[i * 32 + 16] = 2. * (i + 1) as f32; @@ -45982,7 +43249,7 @@ mod tests { 128, 144, 160, 176, 192, 208, 224, 240); let src = _mm512_setr_epi32(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); // A multiplier of 4 is word-addressing - _mm512_i32scatter_epi32(arr.as_mut_ptr() as *mut u8, index, src, 4); + _mm512_i32scatter_epi32::<4>(arr.as_mut_ptr() as *mut u8, index, src); let mut expected = [0i32; 256]; for i in 0..16 { expected[i * 16] = (i + 1) as i32; @@ -45999,7 +43266,7 @@ mod tests { 128, 144, 160, 176, 192, 208, 224, 240); let src = _mm512_setr_epi32(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); // A multiplier of 4 is word-addressing - _mm512_mask_i32scatter_epi32(arr.as_mut_ptr() as *mut u8, mask, index, src, 4); + _mm512_mask_i32scatter_epi32::<4>(arr.as_mut_ptr() as *mut u8, mask, index, src); let mut expected = [0i32; 256]; for i in 0..8 { expected[i * 32 + 16] = 2 * (i + 1) as i32; @@ -46143,7 +43410,7 @@ mod tests { let a = _mm512_set_ps(0., 1., -1., 13., f32::MAX, f32::MIN, 100., -100., 0., 1., -1., 13., f32::MAX, f32::MIN, 100., -100.); let b = _mm512_set1_ps(-1.); - let m = _mm512_cmp_ps_mask(a, b, _CMP_LT_OQ); + let m = _mm512_cmp_ps_mask::<_CMP_LT_OQ>(a, b); assert_eq!(m, 0b00000101_00000101); } @@ -46154,7 +43421,7 @@ mod tests { 0., 1., -1., 13., f32::MAX, f32::MIN, 100., -100.); let b = _mm512_set1_ps(-1.); let mask = 0b01100110_01100110; - let r = _mm512_mask_cmp_ps_mask(mask, a, b, _CMP_LT_OQ); + let r = _mm512_mask_cmp_ps_mask::<_CMP_LT_OQ>(mask, a, b); assert_eq!(r, 0b00000100_00000100); } @@ -46162,7 +43429,7 @@ mod tests { unsafe fn test_mm256_cmp_ps_mask() { let a = _mm256_set_ps(0., 1., -1., 13., f32::MAX, f32::MIN, 100., -100.); let b = _mm256_set1_ps(-1.); - let m = _mm256_cmp_ps_mask(a, b, _CMP_LT_OQ); + let m = _mm256_cmp_ps_mask::<_CMP_LT_OQ>(a, b); assert_eq!(m, 0b00000101); } @@ -46171,7 +43438,7 @@ mod tests { let a = _mm256_set_ps(0., 1., -1., 13., f32::MAX, f32::MIN, 100., -100.); let b = _mm256_set1_ps(-1.); let mask = 0b01100110; - let r = _mm256_mask_cmp_ps_mask(mask, a, b, _CMP_LT_OQ); + let r = _mm256_mask_cmp_ps_mask::<_CMP_LT_OQ>(mask, a, b); assert_eq!(r, 0b00000100); } @@ -46179,7 +43446,7 @@ mod tests { unsafe fn test_mm_cmp_ps_mask() { let a = _mm_set_ps(0., 1., -1., 13.); let b = _mm_set1_ps(1.); - let m = _mm_cmp_ps_mask(a, b, _CMP_LT_OQ); + let m = _mm_cmp_ps_mask::<_CMP_LT_OQ>(a, b); assert_eq!(m, 0b00001010); } @@ -46188,7 +43455,7 @@ mod tests { let a = _mm_set_ps(0., 1., -1., 13.); let b = _mm_set1_ps(1.); let mask = 0b11111111; - let r = _mm_mask_cmp_ps_mask(mask, a, b, _CMP_LT_OQ); + let r = _mm_mask_cmp_ps_mask::<_CMP_LT_OQ>(mask, a, b); assert_eq!(r, 0b00001010); } @@ -46198,7 +43465,7 @@ mod tests { let a = _mm512_set_ps(0., 1., -1., 13., f32::MAX, f32::MIN, 100., -100., 0., 1., -1., 13., f32::MAX, f32::MIN, 100., -100.); let b = _mm512_set1_ps(-1.); - let m = _mm512_cmp_round_ps_mask(a, b, _CMP_LT_OQ, _MM_FROUND_CUR_DIRECTION); + let m = _mm512_cmp_round_ps_mask::<_CMP_LT_OQ, _MM_FROUND_CUR_DIRECTION>(a, b); assert_eq!(m, 0b00000101_00000101); } @@ -46209,7 +43476,7 @@ mod tests { 0., 1., -1., 13., f32::MAX, f32::MIN, 100., -100.); let b = _mm512_set1_ps(-1.); let mask = 0b01100110_01100110; - let r = _mm512_mask_cmp_round_ps_mask(mask, a, b, _CMP_LT_OQ, _MM_FROUND_CUR_DIRECTION); + let r = _mm512_mask_cmp_round_ps_mask::<_CMP_LT_OQ, _MM_FROUND_CUR_DIRECTION>(mask, a, b); assert_eq!(r, 0b00000100_00000100); } @@ -46268,7 +43535,7 @@ mod tests { unsafe fn test_mm_cmp_ss_mask() { let a = _mm_setr_ps(2., 1., 1., 1.); let b = _mm_setr_ps(1., 2., 2., 2.); - let m = _mm_cmp_ss_mask(a, b, _CMP_GE_OS); + let m = _mm_cmp_ss_mask::<_CMP_GE_OS>(a, b); assert_eq!(m, 1); } @@ -46276,9 +43543,9 @@ mod tests { unsafe fn test_mm_mask_cmp_ss_mask() { let a = _mm_setr_ps(2., 1., 1., 1.); let b = _mm_setr_ps(1., 2., 2., 2.); - let m = _mm_mask_cmp_ss_mask(0b10, a, b, _CMP_GE_OS); + let m = _mm_mask_cmp_ss_mask::<_CMP_GE_OS>(0b10, a, b); assert_eq!(m, 0); - let m = _mm_mask_cmp_ss_mask(0b1, a, b, _CMP_GE_OS); + let m = _mm_mask_cmp_ss_mask::<_CMP_GE_OS>(0b1, a, b); assert_eq!(m, 1); } @@ -46286,7 +43553,7 @@ mod tests { unsafe fn test_mm_cmp_round_ss_mask() { let a = _mm_setr_ps(2., 1., 1., 1.); let b = _mm_setr_ps(1., 2., 2., 2.); - let m = _mm_cmp_round_ss_mask(a, b, _CMP_GE_OS, _MM_FROUND_CUR_DIRECTION); + let m = _mm_cmp_round_ss_mask::<_CMP_GE_OS, _MM_FROUND_CUR_DIRECTION>(a, b); assert_eq!(m, 1); } @@ -46294,9 +43561,9 @@ mod tests { unsafe fn test_mm_mask_cmp_round_ss_mask() { let a = _mm_setr_ps(2., 1., 1., 1.); let b = _mm_setr_ps(1., 2., 2., 2.); - let m = _mm_mask_cmp_round_ss_mask(0b10, a, b, _CMP_GE_OS, _MM_FROUND_CUR_DIRECTION); + let m = _mm_mask_cmp_round_ss_mask::<_CMP_GE_OS, _MM_FROUND_CUR_DIRECTION>(0b10, a, b); assert_eq!(m, 0); - let m = _mm_mask_cmp_round_ss_mask(0b1, a, b, _CMP_GE_OS, _MM_FROUND_CUR_DIRECTION); + let m = _mm_mask_cmp_round_ss_mask::<_CMP_GE_OS, _MM_FROUND_CUR_DIRECTION>(0b1, a, b); assert_eq!(m, 1); } @@ -46304,7 +43571,7 @@ mod tests { unsafe fn test_mm_cmp_sd_mask() { let a = _mm_setr_pd(2., 1.); let b = _mm_setr_pd(1., 2.); - let m = _mm_cmp_sd_mask(a, b, _CMP_GE_OS); + let m = _mm_cmp_sd_mask::<_CMP_GE_OS>(a, b); assert_eq!(m, 1); } @@ -46312,9 +43579,9 @@ mod tests { unsafe fn test_mm_mask_cmp_sd_mask() { let a = _mm_setr_pd(2., 1.); let b = _mm_setr_pd(1., 2.); - let m = _mm_mask_cmp_sd_mask(0b10, a, b, _CMP_GE_OS); + let m = _mm_mask_cmp_sd_mask::<_CMP_GE_OS>(0b10, a, b); assert_eq!(m, 0); - let m = _mm_mask_cmp_sd_mask(0b1, a, b, _CMP_GE_OS); + let m = _mm_mask_cmp_sd_mask::<_CMP_GE_OS>(0b1, a, b); assert_eq!(m, 1); } @@ -46322,7 +43589,7 @@ mod tests { unsafe fn test_mm_cmp_round_sd_mask() { let a = _mm_setr_pd(2., 1.); let b = _mm_setr_pd(1., 2.); - let m = _mm_cmp_round_sd_mask(a, b, _CMP_GE_OS, _MM_FROUND_CUR_DIRECTION); + let m = _mm_cmp_round_sd_mask::<_CMP_GE_OS, _MM_FROUND_CUR_DIRECTION>(a, b); assert_eq!(m, 1); } @@ -46330,9 +43597,9 @@ mod tests { unsafe fn test_mm_mask_cmp_round_sd_mask() { let a = _mm_setr_pd(2., 1.); let b = _mm_setr_pd(1., 2.); - let m = _mm_mask_cmp_round_sd_mask(0b10, a, b, _CMP_GE_OS, _MM_FROUND_CUR_DIRECTION); + let m = _mm_mask_cmp_round_sd_mask::<_CMP_GE_OS, _MM_FROUND_CUR_DIRECTION>(0b10, a, b); assert_eq!(m, 0); - let m = _mm_mask_cmp_round_sd_mask(0b1, a, b, _CMP_GE_OS, _MM_FROUND_CUR_DIRECTION); + let m = _mm_mask_cmp_round_sd_mask::<_CMP_GE_OS, _MM_FROUND_CUR_DIRECTION>(0b1, a, b); assert_eq!(m, 1); } @@ -46685,7 +43952,7 @@ mod tests { let a = _mm512_set_epi32(0, 1, -1, u32::MAX as i32, i32::MAX, i32::MIN, 100, -100, 0, 1, -1, u32::MAX as i32, i32::MAX, i32::MIN, 100, -100); let b = _mm512_set1_epi32(-1); - let m = _mm512_cmp_epu32_mask(a, b, _MM_CMPINT_LT); + let m = _mm512_cmp_epu32_mask::<_MM_CMPINT_LT>(a, b); assert_eq!(m, 0b11001111_11001111); } @@ -46696,7 +43963,7 @@ mod tests { 0, 1, -1, u32::MAX as i32, i32::MAX, i32::MIN, 100, -100); let b = _mm512_set1_epi32(-1); let mask = 0b01111010_01111010; - let r = _mm512_mask_cmp_epu32_mask(mask, a, b, _MM_CMPINT_LT); + let r = _mm512_mask_cmp_epu32_mask::<_MM_CMPINT_LT>(mask, a, b); assert_eq!(r, 0b01001010_01001010); } @@ -46704,7 +43971,7 @@ mod tests { unsafe fn test_mm256_cmp_epu32_mask() { let a = _mm256_set_epi32(0, 1, -1, u32::MAX as i32, i32::MAX, i32::MIN, 100, -100); let b = _mm256_set1_epi32(-1); - let m = _mm256_cmp_epu32_mask(a, b, _MM_CMPINT_LT); + let m = _mm256_cmp_epu32_mask::<_MM_CMPINT_LT>(a, b); assert_eq!(m, 0b11001111); } @@ -46713,7 +43980,7 @@ mod tests { let a = _mm256_set_epi32(0, 1, -1, u32::MAX as i32, i32::MAX, i32::MIN, 100, -100); let b = _mm256_set1_epi32(-1); let mask = 0b11111111; - let r = _mm256_mask_cmp_epu32_mask(mask, a, b, _MM_CMPINT_LT); + let r = _mm256_mask_cmp_epu32_mask::<_MM_CMPINT_LT>(mask, a, b); assert_eq!(r, 0b11001111); } @@ -46721,7 +43988,7 @@ mod tests { unsafe fn test_mm_cmp_epu32_mask() { let a = _mm_set_epi32(0, 1, -1, i32::MAX); let b = _mm_set1_epi32(1); - let m = _mm_cmp_epu32_mask(a, b, _MM_CMPINT_LT); + let m = _mm_cmp_epu32_mask::<_MM_CMPINT_LT>(a, b); assert_eq!(m, 0b00001000); } @@ -46730,7 +43997,7 @@ mod tests { let a = _mm_set_epi32(0, 1, -1, i32::MAX); let b = _mm_set1_epi32(1); let mask = 0b11111111; - let r = _mm_mask_cmp_epu32_mask(mask, a, b, _MM_CMPINT_LT); + let r = _mm_mask_cmp_epu32_mask::<_MM_CMPINT_LT>(mask, a, b); assert_eq!(r, 0b00001000); } @@ -47083,7 +44350,7 @@ mod tests { let a = _mm512_set_epi32(0, 1, -1, 13, i32::MAX, i32::MIN, 100, -100, 0, 1, -1, 13, i32::MAX, i32::MIN, 100, -100); let b = _mm512_set1_epi32(-1); - let m = _mm512_cmp_epi32_mask(a, b, _MM_CMPINT_LT); + let m = _mm512_cmp_epi32_mask::<_MM_CMPINT_LT>(a, b); assert_eq!(m, 0b00000101_00000101); } @@ -47094,7 +44361,7 @@ mod tests { 0, 1, -1, 13, i32::MAX, i32::MIN, 100, -100); let b = _mm512_set1_epi32(-1); let mask = 0b01100110_01100110; - let r = _mm512_mask_cmp_epi32_mask(mask, a, b, _MM_CMPINT_LT); + let r = _mm512_mask_cmp_epi32_mask::<_MM_CMPINT_LT>(mask, a, b); assert_eq!(r, 0b00000100_00000100); } @@ -47102,7 +44369,7 @@ mod tests { unsafe fn test_mm256_cmp_epi32_mask() { let a = _mm256_set_epi32(0, 1, -1, 13, i32::MAX, i32::MIN, 100, -100); let b = _mm256_set1_epi32(-1); - let m = _mm256_cmp_epi32_mask(a, b, _MM_CMPINT_LT); + let m = _mm256_cmp_epi32_mask::<_MM_CMPINT_LT>(a, b); assert_eq!(m, 0b00000101); } @@ -47111,7 +44378,7 @@ mod tests { let a = _mm256_set_epi32(0, 1, -1, 13, i32::MAX, i32::MIN, 100, -100); let b = _mm256_set1_epi32(-1); let mask = 0b01100110; - let r = _mm256_mask_cmp_epi32_mask(mask, a, b, _MM_CMPINT_LT); + let r = _mm256_mask_cmp_epi32_mask::<_MM_CMPINT_LT>(mask, a, b); assert_eq!(r, 0b00000100); } @@ -47119,7 +44386,7 @@ mod tests { unsafe fn test_mm_cmp_epi32_mask() { let a = _mm_set_epi32(0, 1, -1, 13); let b = _mm_set1_epi32(1); - let m = _mm_cmp_epi32_mask(a, b, _MM_CMPINT_LT); + let m = _mm_cmp_epi32_mask::<_MM_CMPINT_LT>(a, b); assert_eq!(m, 0b00001010); } @@ -47128,7 +44395,7 @@ mod tests { let a = _mm_set_epi32(0, 1, -1, 13); let b = _mm_set1_epi32(1); let mask = 0b11111111; - let r = _mm_mask_cmp_epi32_mask(mask, a, b, _MM_CMPINT_LT); + let r = _mm_mask_cmp_epi32_mask::<_MM_CMPINT_LT>(mask, a, b); assert_eq!(r, 0b00001010); } @@ -47335,7 +44602,7 @@ mod tests { #[simd_test(enable = "avx512f")] unsafe fn test_mm512_rol_epi32() { let a = _mm512_set_epi32(1 << 31, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1); - let r = _mm512_rol_epi32(a, 1); + let r = _mm512_rol_epi32::<1>(a); let e = _mm512_set_epi32(1 << 0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2); assert_eq_m512i(r, e); } @@ -47343,9 +44610,9 @@ mod tests { #[simd_test(enable = "avx512f")] unsafe fn test_mm512_mask_rol_epi32() { let a = _mm512_set_epi32(1 << 31, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1); - let r = _mm512_mask_rol_epi32(a, 0, a, 1); + let r = _mm512_mask_rol_epi32::<1>(a, 0, a); assert_eq_m512i(r, a); - let r = _mm512_mask_rol_epi32(a, 0b11111111_11111111, a, 1); + let r = _mm512_mask_rol_epi32::<1>(a, 0b11111111_11111111, a); let e = _mm512_set_epi32(1 << 0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2); assert_eq_m512i(r, e); } @@ -47353,9 +44620,9 @@ mod tests { #[simd_test(enable = "avx512f")] unsafe fn test_mm512_maskz_rol_epi32() { let a = _mm512_set_epi32(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 << 31); - let r = _mm512_maskz_rol_epi32(0, a, 1); + let r = _mm512_maskz_rol_epi32::<1>(0, a); assert_eq_m512i(r, _mm512_setzero_si512()); - let r = _mm512_maskz_rol_epi32(0b00000000_11111111, a, 1); + let r = _mm512_maskz_rol_epi32::<1>(0b00000000_11111111, a); let e = _mm512_set_epi32(0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 2, 2, 2, 2, 2, 1 << 0); assert_eq_m512i(r, e); } @@ -47363,7 +44630,7 @@ mod tests { #[simd_test(enable = "avx512f,avx512vl")] unsafe fn test_mm256_rol_epi32() { let a = _mm256_set_epi32(1 << 31, 1, 1, 1, 1, 1, 1, 1); - let r = _mm256_rol_epi32(a, 1); + let r = _mm256_rol_epi32::<1>(a); let e = _mm256_set_epi32(1 << 0, 2, 2, 2, 2, 2, 2, 2); assert_eq_m256i(r, e); } @@ -47371,9 +44638,9 @@ mod tests { #[simd_test(enable = "avx512f,avx512vl")] unsafe fn test_mm256_mask_rol_epi32() { let a = _mm256_set_epi32(1 << 31, 1, 1, 1, 1, 1, 1, 1); - let r = _mm256_mask_rol_epi32(a, 0, a, 1); + let r = _mm256_mask_rol_epi32::<1>(a, 0, a); assert_eq_m256i(r, a); - let r = _mm256_mask_rol_epi32(a, 0b11111111, a, 1); + let r = _mm256_mask_rol_epi32::<1>(a, 0b11111111, a); let e = _mm256_set_epi32(1 << 0, 2, 2, 2, 2, 2, 2, 2); assert_eq_m256i(r, e); } @@ -47381,9 +44648,9 @@ mod tests { #[simd_test(enable = "avx512f,avx512vl")] unsafe fn test_mm256_maskz_rol_epi32() { let a = _mm256_set_epi32(1 << 31, 1, 1, 1, 1, 1, 1, 1); - let r = _mm256_maskz_rol_epi32(0, a, 1); + let r = _mm256_maskz_rol_epi32::<1>(0, a); assert_eq_m256i(r, _mm256_setzero_si256()); - let r = _mm256_maskz_rol_epi32(0b11111111, a, 1); + let r = _mm256_maskz_rol_epi32::<1>(0b11111111, a); let e = _mm256_set_epi32(1 << 0, 2, 2, 2, 2, 2, 2, 2); assert_eq_m256i(r, e); } @@ -47391,7 +44658,7 @@ mod tests { #[simd_test(enable = "avx512f,avx512vl")] unsafe fn test_mm_rol_epi32() { let a = _mm_set_epi32(1 << 31, 1, 1, 1); - let r = _mm_rol_epi32(a, 1); + let r = _mm_rol_epi32::<1>(a); let e = _mm_set_epi32(1 << 0, 2, 2, 2); assert_eq_m128i(r, e); } @@ -47399,9 +44666,9 @@ mod tests { #[simd_test(enable = "avx512f,avx512vl")] unsafe fn test_mm_mask_rol_epi32() { let a = _mm_set_epi32(1 << 31, 1, 1, 1); - let r = _mm_mask_rol_epi32(a, 0, a, 1); + let r = _mm_mask_rol_epi32::<1>(a, 0, a); assert_eq_m128i(r, a); - let r = _mm_mask_rol_epi32(a, 0b00001111, a, 1); + let r = _mm_mask_rol_epi32::<1>(a, 0b00001111, a); let e = _mm_set_epi32(1 << 0, 2, 2, 2); assert_eq_m128i(r, e); } @@ -47409,9 +44676,9 @@ mod tests { #[simd_test(enable = "avx512f,avx512vl")] unsafe fn test_mm_maskz_rol_epi32() { let a = _mm_set_epi32(1 << 31, 1, 1, 1); - let r = _mm_maskz_rol_epi32(0, a, 1); + let r = _mm_maskz_rol_epi32::<1>(0, a); assert_eq_m128i(r, _mm_setzero_si128()); - let r = _mm_maskz_rol_epi32(0b00001111, a, 1); + let r = _mm_maskz_rol_epi32::<1>(0b00001111, a); let e = _mm_set_epi32(1 << 0, 2, 2, 2); assert_eq_m128i(r, e); } @@ -47419,7 +44686,7 @@ mod tests { #[simd_test(enable = "avx512f")] unsafe fn test_mm512_ror_epi32() { let a = _mm512_set_epi32(1 << 0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2); - let r = _mm512_ror_epi32(a, 1); + let r = _mm512_ror_epi32::<1>(a); let e = _mm512_set_epi32(1 << 31, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1); assert_eq_m512i(r, e); } @@ -47427,9 +44694,9 @@ mod tests { #[simd_test(enable = "avx512f")] unsafe fn test_mm512_mask_ror_epi32() { let a = _mm512_set_epi32(1 << 0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2); - let r = _mm512_mask_ror_epi32(a, 0, a, 1); + let r = _mm512_mask_ror_epi32::<1>(a, 0, a); assert_eq_m512i(r, a); - let r = _mm512_mask_ror_epi32(a, 0b11111111_11111111, a, 1); + let r = _mm512_mask_ror_epi32::<1>(a, 0b11111111_11111111, a); let e = _mm512_set_epi32(1 << 31, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1); assert_eq_m512i(r, e); } @@ -47437,9 +44704,9 @@ mod tests { #[simd_test(enable = "avx512f")] unsafe fn test_mm512_maskz_ror_epi32() { let a = _mm512_set_epi32(2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1 << 0); - let r = _mm512_maskz_ror_epi32(0, a, 1); + let r = _mm512_maskz_ror_epi32::<1>(0, a); assert_eq_m512i(r, _mm512_setzero_si512()); - let r = _mm512_maskz_ror_epi32(0b00000000_11111111, a, 1); + let r = _mm512_maskz_ror_epi32::<1>(0b00000000_11111111, a); let e = _mm512_set_epi32(0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1 << 31); assert_eq_m512i(r, e); } @@ -47447,7 +44714,7 @@ mod tests { #[simd_test(enable = "avx512f,avx512vl")] unsafe fn test_mm256_ror_epi32() { let a = _mm256_set_epi32(1 << 0, 2, 2, 2, 2, 2, 2, 2); - let r = _mm256_ror_epi32(a, 1); + let r = _mm256_ror_epi32::<1>(a); let e = _mm256_set_epi32(1 << 31, 1, 1, 1, 1, 1, 1, 1); assert_eq_m256i(r, e); } @@ -47455,9 +44722,9 @@ mod tests { #[simd_test(enable = "avx512f,avx512vl")] unsafe fn test_mm256_mask_ror_epi32() { let a = _mm256_set_epi32(1 << 0, 2, 2, 2, 2, 2, 2, 2); - let r = _mm256_mask_ror_epi32(a, 0, a, 1); + let r = _mm256_mask_ror_epi32::<1>(a, 0, a); assert_eq_m256i(r, a); - let r = _mm256_mask_ror_epi32(a, 0b11111111, a, 1); + let r = _mm256_mask_ror_epi32::<1>(a, 0b11111111, a); let e = _mm256_set_epi32(1 << 31, 1, 1, 1, 1, 1, 1, 1); assert_eq_m256i(r, e); } @@ -47465,9 +44732,9 @@ mod tests { #[simd_test(enable = "avx512f,avx512vl")] unsafe fn test_mm256_maskz_ror_epi32() { let a = _mm256_set_epi32(1 << 0, 2, 2, 2, 2, 2, 2, 2); - let r = _mm256_maskz_ror_epi32(0, a, 1); + let r = _mm256_maskz_ror_epi32::<1>(0, a); assert_eq_m256i(r, _mm256_setzero_si256()); - let r = _mm256_maskz_ror_epi32(0b11111111, a, 1); + let r = _mm256_maskz_ror_epi32::<1>(0b11111111, a); let e = _mm256_set_epi32(1 << 31, 1, 1, 1, 1, 1, 1, 1); assert_eq_m256i(r, e); } @@ -47475,7 +44742,7 @@ mod tests { #[simd_test(enable = "avx512f,avx512vl")] unsafe fn test_mm_ror_epi32() { let a = _mm_set_epi32(1 << 0, 2, 2, 2); - let r = _mm_ror_epi32(a, 1); + let r = _mm_ror_epi32::<1>(a); let e = _mm_set_epi32(1 << 31, 1, 1, 1); assert_eq_m128i(r, e); } @@ -47483,9 +44750,9 @@ mod tests { #[simd_test(enable = "avx512f,avx512vl")] unsafe fn test_mm_mask_ror_epi32() { let a = _mm_set_epi32(1 << 0, 2, 2, 2); - let r = _mm_mask_ror_epi32(a, 0, a, 1); + let r = _mm_mask_ror_epi32::<1>(a, 0, a); assert_eq_m128i(r, a); - let r = _mm_mask_ror_epi32(a, 0b00001111, a, 1); + let r = _mm_mask_ror_epi32::<1>(a, 0b00001111, a); let e = _mm_set_epi32(1 << 31, 1, 1, 1); assert_eq_m128i(r, e); } @@ -47493,9 +44760,9 @@ mod tests { #[simd_test(enable = "avx512f,avx512vl")] unsafe fn test_mm_maskz_ror_epi32() { let a = _mm_set_epi32(1 << 0, 2, 2, 2); - let r = _mm_maskz_ror_epi32(0, a, 1); + let r = _mm_maskz_ror_epi32::<1>(0, a); assert_eq_m128i(r, _mm_setzero_si128()); - let r = _mm_maskz_ror_epi32(0b00001111, a, 1); + let r = _mm_maskz_ror_epi32::<1>(0b00001111, a); let e = _mm_set_epi32(1 << 31, 1, 1, 1); assert_eq_m128i(r, e); } @@ -47503,7 +44770,7 @@ mod tests { #[simd_test(enable = "avx512f")] unsafe fn test_mm512_slli_epi32() { let a = _mm512_set_epi32(1 << 31, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1); - let r = _mm512_slli_epi32(a, 1); + let r = _mm512_slli_epi32::<1>(a); let e = _mm512_set_epi32(0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2); assert_eq_m512i(r, e); } @@ -47511,9 +44778,9 @@ mod tests { #[simd_test(enable = "avx512f")] unsafe fn test_mm512_mask_slli_epi32() { let a = _mm512_set_epi32(1 << 31, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1); - let r = _mm512_mask_slli_epi32(a, 0, a, 1); + let r = _mm512_mask_slli_epi32::<1>(a, 0, a); assert_eq_m512i(r, a); - let r = _mm512_mask_slli_epi32(a, 0b11111111_11111111, a, 1); + let r = _mm512_mask_slli_epi32::<1>(a, 0b11111111_11111111, a); let e = _mm512_set_epi32(0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2); assert_eq_m512i(r, e); } @@ -47521,9 +44788,9 @@ mod tests { #[simd_test(enable = "avx512f")] unsafe fn test_mm512_maskz_slli_epi32() { let a = _mm512_set_epi32(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 << 31); - let r = _mm512_maskz_slli_epi32(0, a, 1); + let r = _mm512_maskz_slli_epi32::<1>(0, a); assert_eq_m512i(r, _mm512_setzero_si512()); - let r = _mm512_maskz_slli_epi32(0b00000000_11111111, a, 1); + let r = _mm512_maskz_slli_epi32::<1>(0b00000000_11111111, a); let e = _mm512_set_epi32(0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 2, 2, 2, 2, 2, 0); assert_eq_m512i(r, e); } @@ -47531,9 +44798,9 @@ mod tests { #[simd_test(enable = "avx512f,avx512vl")] unsafe fn test_mm256_mask_slli_epi32() { let a = _mm256_set_epi32(1 << 31, 1, 1, 1, 1, 1, 1, 1); - let r = _mm256_mask_slli_epi32(a, 0, a, 1); + let r = _mm256_mask_slli_epi32::<1>(a, 0, a); assert_eq_m256i(r, a); - let r = _mm256_mask_slli_epi32(a, 0b11111111, a, 1); + let r = _mm256_mask_slli_epi32::<1>(a, 0b11111111, a); let e = _mm256_set_epi32(0, 2, 2, 2, 2, 2, 2, 2); assert_eq_m256i(r, e); } @@ -47541,9 +44808,9 @@ mod tests { #[simd_test(enable = "avx512f,avx512vl")] unsafe fn test_mm256_maskz_slli_epi32() { let a = _mm256_set_epi32(1 << 31, 1, 1, 1, 1, 1, 1, 1); - let r = _mm256_maskz_slli_epi32(0, a, 1); + let r = _mm256_maskz_slli_epi32::<1>(0, a); assert_eq_m256i(r, _mm256_setzero_si256()); - let r = _mm256_maskz_slli_epi32(0b11111111, a, 1); + let r = _mm256_maskz_slli_epi32::<1>(0b11111111, a); let e = _mm256_set_epi32(0, 2, 2, 2, 2, 2, 2, 2); assert_eq_m256i(r, e); } @@ -47551,9 +44818,9 @@ mod tests { #[simd_test(enable = "avx512f,avx512vl")] unsafe fn test_mm_mask_slli_epi32() { let a = _mm_set_epi32(1 << 31, 1, 1, 1); - let r = _mm_mask_slli_epi32(a, 0, a, 1); + let r = _mm_mask_slli_epi32::<1>(a, 0, a); assert_eq_m128i(r, a); - let r = _mm_mask_slli_epi32(a, 0b00001111, a, 1); + let r = _mm_mask_slli_epi32::<1>(a, 0b00001111, a); let e = _mm_set_epi32(0, 2, 2, 2); assert_eq_m128i(r, e); } @@ -47561,9 +44828,9 @@ mod tests { #[simd_test(enable = "avx512f,avx512vl")] unsafe fn test_mm_maskz_slli_epi32() { let a = _mm_set_epi32(1 << 31, 1, 1, 1); - let r = _mm_maskz_slli_epi32(0, a, 1); + let r = _mm_maskz_slli_epi32::<1>(0, a); assert_eq_m128i(r, _mm_setzero_si128()); - let r = _mm_maskz_slli_epi32(0b00001111, a, 1); + let r = _mm_maskz_slli_epi32::<1>(0b00001111, a); let e = _mm_set_epi32(0, 2, 2, 2); assert_eq_m128i(r, e); } @@ -47571,7 +44838,7 @@ mod tests { #[simd_test(enable = "avx512f")] unsafe fn test_mm512_srli_epi32() { let a = _mm512_set_epi32(0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2); - let r = _mm512_srli_epi32(a, 1); + let r = _mm512_srli_epi32::<1>(a); let e = _mm512_set_epi32(0 << 31, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1); assert_eq_m512i(r, e); } @@ -47579,9 +44846,9 @@ mod tests { #[simd_test(enable = "avx512f")] unsafe fn test_mm512_mask_srli_epi32() { let a = _mm512_set_epi32(0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2); - let r = _mm512_mask_srli_epi32(a, 0, a, 1); + let r = _mm512_mask_srli_epi32::<1>(a, 0, a); assert_eq_m512i(r, a); - let r = _mm512_mask_srli_epi32(a, 0b11111111_11111111, a, 1); + let r = _mm512_mask_srli_epi32::<1>(a, 0b11111111_11111111, a); let e = _mm512_set_epi32(0 << 31, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1); assert_eq_m512i(r, e); } @@ -47589,9 +44856,9 @@ mod tests { #[simd_test(enable = "avx512f")] unsafe fn test_mm512_maskz_srli_epi32() { let a = _mm512_set_epi32(2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 0); - let r = _mm512_maskz_srli_epi32(0, a, 1); + let r = _mm512_maskz_srli_epi32::<1>(0, a); assert_eq_m512i(r, _mm512_setzero_si512()); - let r = _mm512_maskz_srli_epi32(0b00000000_11111111, a, 1); + let r = _mm512_maskz_srli_epi32::<1>(0b00000000_11111111, a); let e = _mm512_set_epi32(0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 0 << 31); assert_eq_m512i(r, e); } @@ -47599,9 +44866,9 @@ mod tests { #[simd_test(enable = "avx512f,avx512vl")] unsafe fn test_mm256_mask_srli_epi32() { let a = _mm256_set_epi32(1 << 5, 0, 0, 0, 0, 0, 0, 0); - let r = _mm256_mask_srli_epi32(a, 0, a, 1); + let r = _mm256_mask_srli_epi32::<1>(a, 0, a); assert_eq_m256i(r, a); - let r = _mm256_mask_srli_epi32(a, 0b11111111, a, 1); + let r = _mm256_mask_srli_epi32::<1>(a, 0b11111111, a); let e = _mm256_set_epi32(1 << 4, 0, 0, 0, 0, 0, 0, 0); assert_eq_m256i(r, e); } @@ -47609,9 +44876,9 @@ mod tests { #[simd_test(enable = "avx512f,avx512vl")] unsafe fn test_mm256_maskz_srli_epi32() { let a = _mm256_set_epi32(1 << 5, 0, 0, 0, 0, 0, 0, 0); - let r = _mm256_maskz_srli_epi32(0, a, 1); + let r = _mm256_maskz_srli_epi32::<1>(0, a); assert_eq_m256i(r, _mm256_setzero_si256()); - let r = _mm256_maskz_srli_epi32(0b11111111, a, 1); + let r = _mm256_maskz_srli_epi32::<1>(0b11111111, a); let e = _mm256_set_epi32(1 << 4, 0, 0, 0, 0, 0, 0, 0); assert_eq_m256i(r, e); } @@ -47619,9 +44886,9 @@ mod tests { #[simd_test(enable = "avx512f,avx512vl")] unsafe fn test_mm_mask_srli_epi32() { let a = _mm_set_epi32(1 << 5, 0, 0, 0); - let r = _mm_mask_srli_epi32(a, 0, a, 1); + let r = _mm_mask_srli_epi32::<1>(a, 0, a); assert_eq_m128i(r, a); - let r = _mm_mask_srli_epi32(a, 0b00001111, a, 1); + let r = _mm_mask_srli_epi32::<1>(a, 0b00001111, a); let e = _mm_set_epi32(1 << 4, 0, 0, 0); assert_eq_m128i(r, e); } @@ -47629,9 +44896,9 @@ mod tests { #[simd_test(enable = "avx512f,avx512vl")] unsafe fn test_mm_maskz_srli_epi32() { let a = _mm_set_epi32(1 << 5, 0, 0, 0); - let r = _mm_maskz_srli_epi32(0, a, 1); + let r = _mm_maskz_srli_epi32::<1>(0, a); assert_eq_m128i(r, _mm_setzero_si128()); - let r = _mm_maskz_srli_epi32(0b00001111, a, 1); + let r = _mm_maskz_srli_epi32::<1>(0b00001111, a); let e = _mm_set_epi32(1 << 4, 0, 0, 0); assert_eq_m128i(r, e); } @@ -48323,7 +45590,7 @@ mod tests { #[simd_test(enable = "avx512f")] unsafe fn test_mm512_srai_epi32() { let a = _mm512_set_epi32(8, -8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, -15); - let r = _mm512_srai_epi32(a, 2); + let r = _mm512_srai_epi32::<2>(a); let e = _mm512_set_epi32(2, -2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, -4); assert_eq_m512i(r, e); } @@ -48331,9 +45598,9 @@ mod tests { #[simd_test(enable = "avx512f")] unsafe fn test_mm512_mask_srai_epi32() { let a = _mm512_set_epi32(8, -8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, -15); - let r = _mm512_mask_srai_epi32(a, 0, a, 2); + let r = _mm512_mask_srai_epi32::<2>(a, 0, a); assert_eq_m512i(r, a); - let r = _mm512_mask_srai_epi32(a, 0b11111111_11111111, a, 2); + let r = _mm512_mask_srai_epi32::<2>(a, 0b11111111_11111111, a); let e = _mm512_set_epi32(2, -2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, -4); assert_eq_m512i(r, e); } @@ -48341,9 +45608,9 @@ mod tests { #[simd_test(enable = "avx512f")] unsafe fn test_mm512_maskz_srai_epi32() { let a = _mm512_set_epi32(8, -8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, -15); - let r = _mm512_maskz_srai_epi32(0, a, 2); + let r = _mm512_maskz_srai_epi32::<2>(0, a); assert_eq_m512i(r, _mm512_setzero_si512()); - let r = _mm512_maskz_srai_epi32(0b00000000_11111111, a, 2); + let r = _mm512_maskz_srai_epi32::<2>(0b00000000_11111111, a); let e = _mm512_set_epi32(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, -4); assert_eq_m512i(r, e); } @@ -48351,9 +45618,9 @@ mod tests { #[simd_test(enable = "avx512f,avx512vl")] unsafe fn test_mm256_mask_srai_epi32() { let a = _mm256_set_epi32(1 << 5, 0, 0, 0, 0, 0, 0, 0); - let r = _mm256_mask_srai_epi32(a, 0, a, 1); + let r = _mm256_mask_srai_epi32::<1>(a, 0, a); assert_eq_m256i(r, a); - let r = _mm256_mask_srai_epi32(a, 0b11111111, a, 1); + let r = _mm256_mask_srai_epi32::<1>(a, 0b11111111, a); let e = _mm256_set_epi32(1 << 4, 0, 0, 0, 0, 0, 0, 0); assert_eq_m256i(r, e); } @@ -48361,9 +45628,9 @@ mod tests { #[simd_test(enable = "avx512f,avx512vl")] unsafe fn test_mm256_maskz_srai_epi32() { let a = _mm256_set_epi32(1 << 5, 0, 0, 0, 0, 0, 0, 0); - let r = _mm256_maskz_srai_epi32(0, a, 1); + let r = _mm256_maskz_srai_epi32::<1>(0, a); assert_eq_m256i(r, _mm256_setzero_si256()); - let r = _mm256_maskz_srai_epi32(0b11111111, a, 1); + let r = _mm256_maskz_srai_epi32::<1>(0b11111111, a); let e = _mm256_set_epi32(1 << 4, 0, 0, 0, 0, 0, 0, 0); assert_eq_m256i(r, e); } @@ -48371,9 +45638,9 @@ mod tests { #[simd_test(enable = "avx512f,avx512vl")] unsafe fn test_mm_mask_srai_epi32() { let a = _mm_set_epi32(1 << 5, 0, 0, 0); - let r = _mm_mask_srai_epi32(a, 0, a, 1); + let r = _mm_mask_srai_epi32::<1>(a, 0, a); assert_eq_m128i(r, a); - let r = _mm_mask_srai_epi32(a, 0b00001111, a, 1); + let r = _mm_mask_srai_epi32::<1>(a, 0b00001111, a); let e = _mm_set_epi32(1 << 4, 0, 0, 0); assert_eq_m128i(r, e); } @@ -48381,9 +45648,9 @@ mod tests { #[simd_test(enable = "avx512f,avx512vl")] unsafe fn test_mm_maskz_srai_epi32() { let a = _mm_set_epi32(1 << 5, 0, 0, 0); - let r = _mm_maskz_srai_epi32(0, a, 1); + let r = _mm_maskz_srai_epi32::<1>(0, a); assert_eq_m128i(r, _mm_setzero_si128()); - let r = _mm_maskz_srai_epi32(0b00001111, a, 1); + let r = _mm_maskz_srai_epi32::<1>(0b00001111, a); let e = _mm_set_epi32(1 << 4, 0, 0, 0); assert_eq_m128i(r, e); } @@ -48393,7 +45660,7 @@ mod tests { let a = _mm512_setr_ps( 0., 1., 2., 3., 4., 5., 6., 7., 8., 9., 10., 11., 12., 13., 14., 15., ); - let r = _mm512_permute_ps(a, 0b11111111); + let r = _mm512_permute_ps::<0b11_11_11_11>(a); let e = _mm512_setr_ps( 3., 3., 3., 3., 7., 7., 7., 7., 11., 11., 11., 11., 15., 15., 15., 15., ); @@ -48405,9 +45672,9 @@ mod tests { let a = _mm512_setr_ps( 0., 1., 2., 3., 4., 5., 6., 7., 8., 9., 10., 11., 12., 13., 14., 15., ); - let r = _mm512_mask_permute_ps(a, 0, a, 0b11111111); + let r = _mm512_mask_permute_ps::<0b11_11_11_11>(a, 0, a); assert_eq_m512(r, a); - let r = _mm512_mask_permute_ps(a, 0b11111111_11111111, a, 0b111111111); + let r = _mm512_mask_permute_ps::<0b11_11_11_11>(a, 0b11111111_11111111, a); let e = _mm512_setr_ps( 3., 3., 3., 3., 7., 7., 7., 7., 11., 11., 11., 11., 15., 15., 15., 15., ); @@ -48419,9 +45686,9 @@ mod tests { let a = _mm512_setr_ps( 0., 1., 2., 3., 4., 5., 6., 7., 8., 9., 10., 11., 12., 13., 14., 15., ); - let r = _mm512_maskz_permute_ps(0, a, 0b11111111); + let r = _mm512_maskz_permute_ps::<0b11_11_11_11>(0, a); assert_eq_m512(r, _mm512_setzero_ps()); - let r = _mm512_maskz_permute_ps(0b11111111_11111111, a, 0b11111111); + let r = _mm512_maskz_permute_ps::<0b11_11_11_11>(0b11111111_11111111, a); let e = _mm512_setr_ps( 3., 3., 3., 3., 7., 7., 7., 7., 11., 11., 11., 11., 15., 15., 15., 15., ); @@ -48431,9 +45698,9 @@ mod tests { #[simd_test(enable = "avx512f,avx512vl")] unsafe fn test_mm256_mask_permute_ps() { let a = _mm256_set_ps(0., 1., 2., 3., 4., 5., 6., 7.); - let r = _mm256_mask_permute_ps(a, 0, a, 0b11111111); + let r = _mm256_mask_permute_ps::<0b11_11_11_11>(a, 0, a); assert_eq_m256(r, a); - let r = _mm256_mask_permute_ps(a, 0b11111111, a, 0b11111111); + let r = _mm256_mask_permute_ps::<0b11_11_11_11>(a, 0b11111111, a); let e = _mm256_set_ps(0., 0., 0., 0., 4., 4., 4., 4.); assert_eq_m256(r, e); } @@ -48441,9 +45708,9 @@ mod tests { #[simd_test(enable = "avx512f,avx512vl")] unsafe fn test_mm256_maskz_permute_ps() { let a = _mm256_set_ps(0., 1., 2., 3., 4., 5., 6., 7.); - let r = _mm256_maskz_permute_ps(0, a, 0b11111111); + let r = _mm256_maskz_permute_ps::<0b11_11_11_11>(0, a); assert_eq_m256(r, _mm256_setzero_ps()); - let r = _mm256_maskz_permute_ps(0b11111111, a, 0b11111111); + let r = _mm256_maskz_permute_ps::<0b11_11_11_11>(0b11111111, a); let e = _mm256_set_ps(0., 0., 0., 0., 4., 4., 4., 4.); assert_eq_m256(r, e); } @@ -48451,9 +45718,9 @@ mod tests { #[simd_test(enable = "avx512f,avx512vl")] unsafe fn test_mm_mask_permute_ps() { let a = _mm_set_ps(0., 1., 2., 3.); - let r = _mm_mask_permute_ps(a, 0, a, 0b11111111); + let r = _mm_mask_permute_ps::<0b11_11_11_11>(a, 0, a); assert_eq_m128(r, a); - let r = _mm_mask_permute_ps(a, 0b00001111, a, 0b11111111); + let r = _mm_mask_permute_ps::<0b11_11_11_11>(a, 0b00001111, a); let e = _mm_set_ps(0., 0., 0., 0.); assert_eq_m128(r, e); } @@ -48461,9 +45728,9 @@ mod tests { #[simd_test(enable = "avx512f,avx512vl")] unsafe fn test_mm_maskz_permute_ps() { let a = _mm_set_ps(0., 1., 2., 3.); - let r = _mm_maskz_permute_ps(0, a, 0b11111111); + let r = _mm_maskz_permute_ps::<0b11_11_11_11>(0, a); assert_eq_m128(r, _mm_setzero_ps()); - let r = _mm_maskz_permute_ps(0b00001111, a, 0b11111111); + let r = _mm_maskz_permute_ps::<0b11_11_11_11>(0b00001111, a); let e = _mm_set_ps(0., 0., 0., 0.); assert_eq_m128(r, e); } @@ -49060,7 +46327,7 @@ mod tests { #[simd_test(enable = "avx512f")] unsafe fn test_mm512_shuffle_epi32() { let a = _mm512_setr_epi32(1, 4, 5, 8, 9, 12, 13, 16, 1, 4, 5, 8, 9, 12, 13, 16); - let r = _mm512_shuffle_epi32(a, _MM_PERM_AADD); + let r = _mm512_shuffle_epi32::<_MM_PERM_AADD>(a); let e = _mm512_setr_epi32(8, 8, 1, 1, 16, 16, 9, 9, 8, 8, 1, 1, 16, 16, 9, 9); assert_eq_m512i(r, e); } @@ -49068,9 +46335,9 @@ mod tests { #[simd_test(enable = "avx512f")] unsafe fn test_mm512_mask_shuffle_epi32() { let a = _mm512_setr_epi32(1, 4, 5, 8, 9, 12, 13, 16, 1, 4, 5, 8, 9, 12, 13, 16); - let r = _mm512_mask_shuffle_epi32(a, 0, a, _MM_PERM_AADD); + let r = _mm512_mask_shuffle_epi32::<_MM_PERM_AADD>(a, 0, a); assert_eq_m512i(r, a); - let r = _mm512_mask_shuffle_epi32(a, 0b11111111_11111111, a, _MM_PERM_AADD); + let r = _mm512_mask_shuffle_epi32::<_MM_PERM_AADD>(a, 0b11111111_11111111, a); let e = _mm512_setr_epi32(8, 8, 1, 1, 16, 16, 9, 9, 8, 8, 1, 1, 16, 16, 9, 9); assert_eq_m512i(r, e); } @@ -49078,9 +46345,9 @@ mod tests { #[simd_test(enable = "avx512f")] unsafe fn test_mm512_maskz_shuffle_epi32() { let a = _mm512_setr_epi32(1, 4, 5, 8, 9, 12, 13, 16, 1, 4, 5, 8, 9, 12, 13, 16); - let r = _mm512_maskz_shuffle_epi32(0, a, _MM_PERM_AADD); + let r = _mm512_maskz_shuffle_epi32::<_MM_PERM_AADD>(0, a); assert_eq_m512i(r, _mm512_setzero_si512()); - let r = _mm512_maskz_shuffle_epi32(0b00000000_11111111, a, _MM_PERM_AADD); + let r = _mm512_maskz_shuffle_epi32::<_MM_PERM_AADD>(0b00000000_11111111, a); let e = _mm512_setr_epi32(8, 8, 1, 1, 16, 16, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0); assert_eq_m512i(r, e); } @@ -49088,9 +46355,9 @@ mod tests { #[simd_test(enable = "avx512f,avx512vl")] unsafe fn test_mm256_mask_shuffle_epi32() { let a = _mm256_set_epi32(1, 4, 5, 8, 9, 12, 13, 16); - let r = _mm256_mask_shuffle_epi32(a, 0, a, _MM_PERM_AADD); + let r = _mm256_mask_shuffle_epi32::<_MM_PERM_AADD>(a, 0, a); assert_eq_m256i(r, a); - let r = _mm256_mask_shuffle_epi32(a, 0b11111111, a, _MM_PERM_AADD); + let r = _mm256_mask_shuffle_epi32::<_MM_PERM_AADD>(a, 0b11111111, a); let e = _mm256_set_epi32(8, 8, 1, 1, 16, 16, 9, 9); assert_eq_m256i(r, e); } @@ -49098,9 +46365,9 @@ mod tests { #[simd_test(enable = "avx512f,avx512vl")] unsafe fn test_mm256_maskz_shuffle_epi32() { let a = _mm256_set_epi32(1, 4, 5, 8, 9, 12, 13, 16); - let r = _mm256_maskz_shuffle_epi32(0, a, _MM_PERM_AADD); + let r = _mm256_maskz_shuffle_epi32::<_MM_PERM_AADD>(0, a); assert_eq_m256i(r, _mm256_setzero_si256()); - let r = _mm256_maskz_shuffle_epi32(0b11111111, a, _MM_PERM_AADD); + let r = _mm256_maskz_shuffle_epi32::<_MM_PERM_AADD>(0b11111111, a); let e = _mm256_set_epi32(8, 8, 1, 1, 16, 16, 9, 9); assert_eq_m256i(r, e); } @@ -49108,9 +46375,9 @@ mod tests { #[simd_test(enable = "avx512f,avx512vl")] unsafe fn test_mm_mask_shuffle_epi32() { let a = _mm_set_epi32(1, 4, 5, 8); - let r = _mm_mask_shuffle_epi32(a, 0, a, _MM_PERM_AADD); + let r = _mm_mask_shuffle_epi32::<_MM_PERM_AADD>(a, 0, a); assert_eq_m128i(r, a); - let r = _mm_mask_shuffle_epi32(a, 0b00001111, a, _MM_PERM_AADD); + let r = _mm_mask_shuffle_epi32::<_MM_PERM_AADD>(a, 0b00001111, a); let e = _mm_set_epi32(8, 8, 1, 1); assert_eq_m128i(r, e); } @@ -49118,9 +46385,9 @@ mod tests { #[simd_test(enable = "avx512f,avx512vl")] unsafe fn test_mm_maskz_shuffle_epi32() { let a = _mm_set_epi32(1, 4, 5, 8); - let r = _mm_maskz_shuffle_epi32(0, a, _MM_PERM_AADD); + let r = _mm_maskz_shuffle_epi32::<_MM_PERM_AADD>(0, a); assert_eq_m128i(r, _mm_setzero_si128()); - let r = _mm_maskz_shuffle_epi32(0b00001111, a, _MM_PERM_AADD); + let r = _mm_maskz_shuffle_epi32::<_MM_PERM_AADD>(0b00001111, a); let e = _mm_set_epi32(8, 8, 1, 1); assert_eq_m128i(r, e); } @@ -49133,7 +46400,7 @@ mod tests { let b = _mm512_setr_ps( 2., 3., 6., 7., 10., 11., 14., 15., 2., 3., 6., 7., 10., 11., 14., 15., ); - let r = _mm512_shuffle_ps(a, b, 0x0F); + let r = _mm512_shuffle_ps::<0b00_00_11_11>(a, b); let e = _mm512_setr_ps( 8., 8., 2., 2., 16., 16., 10., 10., 8., 8., 2., 2., 16., 16., 10., 10., ); @@ -49148,9 +46415,9 @@ mod tests { let b = _mm512_setr_ps( 2., 3., 6., 7., 10., 11., 14., 15., 2., 3., 6., 7., 10., 11., 14., 15., ); - let r = _mm512_mask_shuffle_ps(a, 0, a, b, 0x0F); + let r = _mm512_mask_shuffle_ps::<0b00_00_11_11>(a, 0, a, b); assert_eq_m512(r, a); - let r = _mm512_mask_shuffle_ps(a, 0b11111111_11111111, a, b, 0x0F); + let r = _mm512_mask_shuffle_ps::<0b00_00_11_11>(a, 0b11111111_11111111, a, b); let e = _mm512_setr_ps( 8., 8., 2., 2., 16., 16., 10., 10., 8., 8., 2., 2., 16., 16., 10., 10., ); @@ -49165,9 +46432,9 @@ mod tests { let b = _mm512_setr_ps( 2., 3., 6., 7., 10., 11., 14., 15., 2., 3., 6., 7., 10., 11., 14., 15., ); - let r = _mm512_maskz_shuffle_ps(0, a, b, 0x0F); + let r = _mm512_maskz_shuffle_ps::<0b00_00_11_11>(0, a, b); assert_eq_m512(r, _mm512_setzero_ps()); - let r = _mm512_maskz_shuffle_ps(0b00000000_11111111, a, b, 0x0F); + let r = _mm512_maskz_shuffle_ps::<0b00_00_11_11>(0b00000000_11111111, a, b); let e = _mm512_setr_ps( 8., 8., 2., 2., 16., 16., 10., 10., 0., 0., 0., 0., 0., 0., 0., 0., ); @@ -49178,9 +46445,9 @@ mod tests { unsafe fn test_mm256_mask_shuffle_ps() { let a = _mm256_set_ps(1., 4., 5., 8., 9., 12., 13., 16.); let b = _mm256_set_ps(2., 3., 6., 7., 10., 11., 14., 15.); - let r = _mm256_mask_shuffle_ps(a, 0, a, b, 0x0F); + let r = _mm256_mask_shuffle_ps::<0b11_11_11_11>(a, 0, a, b); assert_eq_m256(r, a); - let r = _mm256_mask_shuffle_ps(a, 0b11111111, a, b, 0x0F); + let r = _mm256_mask_shuffle_ps::<0b00_00_11_11>(a, 0b11111111, a, b); let e = _mm256_set_ps(7., 7., 1., 1., 15., 15., 9., 9.); assert_eq_m256(r, e); } @@ -49189,9 +46456,9 @@ mod tests { unsafe fn test_mm256_maskz_shuffle_ps() { let a = _mm256_set_ps(1., 4., 5., 8., 9., 12., 13., 16.); let b = _mm256_set_ps(2., 3., 6., 7., 10., 11., 14., 15.); - let r = _mm256_maskz_shuffle_ps(0, a, b, 0x0F); + let r = _mm256_maskz_shuffle_ps::<0b11_11_11_11>(0, a, b); assert_eq_m256(r, _mm256_setzero_ps()); - let r = _mm256_maskz_shuffle_ps(0b11111111, a, b, 0x0F); + let r = _mm256_maskz_shuffle_ps::<0b00_00_11_11>(0b11111111, a, b); let e = _mm256_set_ps(7., 7., 1., 1., 15., 15., 9., 9.); assert_eq_m256(r, e); } @@ -49200,9 +46467,9 @@ mod tests { unsafe fn test_mm_mask_shuffle_ps() { let a = _mm_set_ps(1., 4., 5., 8.); let b = _mm_set_ps(2., 3., 6., 7.); - let r = _mm_mask_shuffle_ps(a, 0, a, b, 0x0F); + let r = _mm_mask_shuffle_ps::<0b11_11_11_11>(a, 0, a, b); assert_eq_m128(r, a); - let r = _mm_mask_shuffle_ps(a, 0b00001111, a, b, 0x0F); + let r = _mm_mask_shuffle_ps::<0b00_00_11_11>(a, 0b00001111, a, b); let e = _mm_set_ps(7., 7., 1., 1.); assert_eq_m128(r, e); } @@ -49211,9 +46478,9 @@ mod tests { unsafe fn test_mm_maskz_shuffle_ps() { let a = _mm_set_ps(1., 4., 5., 8.); let b = _mm_set_ps(2., 3., 6., 7.); - let r = _mm_maskz_shuffle_ps(0, a, b, 0x0F); + let r = _mm_maskz_shuffle_ps::<0b11_11_11_11>(0, a, b); assert_eq_m128(r, _mm_setzero_ps()); - let r = _mm_maskz_shuffle_ps(0b00001111, a, b, 0x0F); + let r = _mm_maskz_shuffle_ps::<0b00_00_11_11>(0b00001111, a, b); let e = _mm_set_ps(7., 7., 1., 1.); assert_eq_m128(r, e); } @@ -49222,7 +46489,7 @@ mod tests { unsafe fn test_mm512_shuffle_i32x4() { let a = _mm512_setr_epi32(1, 4, 5, 8, 9, 12, 13, 16, 1, 4, 5, 8, 9, 12, 13, 16); let b = _mm512_setr_epi32(2, 3, 6, 7, 10, 11, 14, 15, 2, 3, 6, 7, 10, 11, 14, 15); - let r = _mm512_shuffle_i32x4(a, b, 0b0000); + let r = _mm512_shuffle_i32x4::<0b00_00_00_00>(a, b); let e = _mm512_setr_epi32(1, 4, 5, 8, 1, 4, 5, 8, 2, 3, 6, 7, 2, 3, 6, 7); assert_eq_m512i(r, e); } @@ -49231,9 +46498,9 @@ mod tests { unsafe fn test_mm512_mask_shuffle_i32x4() { let a = _mm512_setr_epi32(1, 4, 5, 8, 9, 12, 13, 16, 1, 4, 5, 8, 9, 12, 13, 16); let b = _mm512_setr_epi32(2, 3, 6, 7, 10, 11, 14, 15, 2, 3, 6, 7, 10, 11, 14, 15); - let r = _mm512_mask_shuffle_i32x4(a, 0, a, b, 0b0000); + let r = _mm512_mask_shuffle_i32x4::<0b00_00_00_00>(a, 0, a, b); assert_eq_m512i(r, a); - let r = _mm512_mask_shuffle_i32x4(a, 0b11111111_11111111, a, b, 0b0000); + let r = _mm512_mask_shuffle_i32x4::<0b00_00_00_00>(a, 0b11111111_11111111, a, b); let e = _mm512_setr_epi32(1, 4, 5, 8, 1, 4, 5, 8, 2, 3, 6, 7, 2, 3, 6, 7); assert_eq_m512i(r, e); } @@ -49242,9 +46509,9 @@ mod tests { unsafe fn test_mm512_maskz_shuffle_i32x4() { let a = _mm512_setr_epi32(1, 4, 5, 8, 9, 12, 13, 16, 1, 4, 5, 8, 9, 12, 13, 16); let b = _mm512_setr_epi32(2, 3, 6, 7, 10, 11, 14, 15, 2, 3, 6, 7, 10, 11, 14, 15); - let r = _mm512_maskz_shuffle_i32x4(0, a, b, 0b0000); + let r = _mm512_maskz_shuffle_i32x4::<0b00_00_00_00>(0, a, b); assert_eq_m512i(r, _mm512_setzero_si512()); - let r = _mm512_maskz_shuffle_i32x4(0b00000000_11111111, a, b, 0b0000); + let r = _mm512_maskz_shuffle_i32x4::<0b00_00_00_00>(0b00000000_11111111, a, b); let e = _mm512_setr_epi32(1, 4, 5, 8, 1, 4, 5, 8, 0, 0, 0, 0, 0, 0, 0, 0); assert_eq_m512i(r, e); } @@ -49253,7 +46520,7 @@ mod tests { unsafe fn test_mm256_shuffle_i32x4() { let a = _mm256_set_epi32(1, 4, 5, 8, 9, 12, 13, 16); let b = _mm256_set_epi32(2, 3, 6, 7, 10, 11, 14, 15); - let r = _mm256_shuffle_i32x4(a, b, 0b00); + let r = _mm256_shuffle_i32x4::<0b00>(a, b); let e = _mm256_set_epi32(10, 11, 14, 15, 9, 12, 13, 16); assert_eq_m256i(r, e); } @@ -49262,9 +46529,9 @@ mod tests { unsafe fn test_mm256_mask_shuffle_i32x4() { let a = _mm256_set_epi32(1, 4, 5, 8, 9, 12, 13, 16); let b = _mm256_set_epi32(2, 3, 6, 7, 10, 11, 14, 15); - let r = _mm256_mask_shuffle_i32x4(a, 0, a, b, 0b00); + let r = _mm256_mask_shuffle_i32x4::<0b00>(a, 0, a, b); assert_eq_m256i(r, a); - let r = _mm256_mask_shuffle_i32x4(a, 0b11111111, a, b, 0b00); + let r = _mm256_mask_shuffle_i32x4::<0b00>(a, 0b11111111, a, b); let e = _mm256_set_epi32(10, 11, 14, 15, 9, 12, 13, 16); assert_eq_m256i(r, e); } @@ -49273,9 +46540,9 @@ mod tests { unsafe fn test_mm256_maskz_shuffle_i32x4() { let a = _mm256_set_epi32(1, 4, 5, 8, 9, 12, 13, 16); let b = _mm256_set_epi32(2, 3, 6, 7, 10, 11, 14, 15); - let r = _mm256_maskz_shuffle_i32x4(0, a, b, 0b00); + let r = _mm256_maskz_shuffle_i32x4::<0b00>(0, a, b); assert_eq_m256i(r, _mm256_setzero_si256()); - let r = _mm256_maskz_shuffle_i32x4(0b11111111, a, b, 0b00); + let r = _mm256_maskz_shuffle_i32x4::<0b00>(0b11111111, a, b); let e = _mm256_set_epi32(10, 11, 14, 15, 9, 12, 13, 16); assert_eq_m256i(r, e); } @@ -49288,7 +46555,7 @@ mod tests { let b = _mm512_setr_ps( 2., 3., 6., 7., 10., 11., 14., 15., 2., 3., 6., 7., 10., 11., 14., 15., ); - let r = _mm512_shuffle_f32x4(a, b, 0b00000000); + let r = _mm512_shuffle_f32x4::<0b00_00_00_00>(a, b); let e = _mm512_setr_ps( 1., 4., 5., 8., 1., 4., 5., 8., 2., 3., 6., 7., 2., 3., 6., 7., ); @@ -49303,9 +46570,9 @@ mod tests { let b = _mm512_setr_ps( 2., 3., 6., 7., 10., 11., 14., 15., 2., 3., 6., 7., 10., 11., 14., 15., ); - let r = _mm512_mask_shuffle_f32x4(a, 0, a, b, 0b00000000); + let r = _mm512_mask_shuffle_f32x4::<0b00_00_00_00>(a, 0, a, b); assert_eq_m512(r, a); - let r = _mm512_mask_shuffle_f32x4(a, 0b11111111_11111111, a, b, 0b00000000); + let r = _mm512_mask_shuffle_f32x4::<0b00_00_00_00>(a, 0b11111111_11111111, a, b); let e = _mm512_setr_ps( 1., 4., 5., 8., 1., 4., 5., 8., 2., 3., 6., 7., 2., 3., 6., 7., ); @@ -49320,9 +46587,9 @@ mod tests { let b = _mm512_setr_ps( 2., 3., 6., 7., 10., 11., 14., 15., 2., 3., 6., 7., 10., 11., 14., 15., ); - let r = _mm512_maskz_shuffle_f32x4(0, a, b, 0b00000000); + let r = _mm512_maskz_shuffle_f32x4::<0b00_00_00_00>(0, a, b); assert_eq_m512(r, _mm512_setzero_ps()); - let r = _mm512_maskz_shuffle_f32x4(0b00000000_11111111, a, b, 0b00000000); + let r = _mm512_maskz_shuffle_f32x4::<0b00_00_00_00>(0b00000000_11111111, a, b); let e = _mm512_setr_ps( 1., 4., 5., 8., 1., 4., 5., 8., 0., 0., 0., 0., 0., 0., 0., 0., ); @@ -49333,7 +46600,7 @@ mod tests { unsafe fn test_mm256_shuffle_f32x4() { let a = _mm256_set_ps(1., 4., 5., 8., 9., 12., 13., 16.); let b = _mm256_set_ps(2., 3., 6., 7., 10., 11., 14., 15.); - let r = _mm256_shuffle_f32x4(a, b, 0b00); + let r = _mm256_shuffle_f32x4::<0b00>(a, b); let e = _mm256_set_ps(10., 11., 14., 15., 9., 12., 13., 16.); assert_eq_m256(r, e); } @@ -49342,9 +46609,9 @@ mod tests { unsafe fn test_mm256_mask_shuffle_f32x4() { let a = _mm256_set_ps(1., 4., 5., 8., 9., 12., 13., 16.); let b = _mm256_set_ps(2., 3., 6., 7., 10., 11., 14., 15.); - let r = _mm256_mask_shuffle_f32x4(a, 0, a, b, 0b00); + let r = _mm256_mask_shuffle_f32x4::<0b00>(a, 0, a, b); assert_eq_m256(r, a); - let r = _mm256_mask_shuffle_f32x4(a, 0b11111111, a, b, 0b00); + let r = _mm256_mask_shuffle_f32x4::<0b00>(a, 0b11111111, a, b); let e = _mm256_set_ps(10., 11., 14., 15., 9., 12., 13., 16.); assert_eq_m256(r, e); } @@ -49353,9 +46620,9 @@ mod tests { unsafe fn test_mm256_maskz_shuffle_f32x4() { let a = _mm256_set_ps(1., 4., 5., 8., 9., 12., 13., 16.); let b = _mm256_set_ps(2., 3., 6., 7., 10., 11., 14., 15.); - let r = _mm256_maskz_shuffle_f32x4(0, a, b, 0b00); + let r = _mm256_maskz_shuffle_f32x4::<0b00>(0, a, b); assert_eq_m256(r, _mm256_setzero_ps()); - let r = _mm256_maskz_shuffle_f32x4(0b11111111, a, b, 0b00); + let r = _mm256_maskz_shuffle_f32x4::<0b00>(0b11111111, a, b); let e = _mm256_set_ps(10., 11., 14., 15., 9., 12., 13., 16.); assert_eq_m256(r, e); } @@ -49365,7 +46632,7 @@ mod tests { let a = _mm512_setr_ps( 1., 2., 3., 4., 5., 6., 7., 8., 9., 10., 11., 12., 13., 14., 15., 16., ); - let r = _mm512_extractf32x4_ps(a, 0b1); + let r = _mm512_extractf32x4_ps::<1>(a); let e = _mm_setr_ps(5., 6., 7., 8.); assert_eq_m128(r, e); } @@ -49376,9 +46643,9 @@ mod tests { 1., 2., 3., 4., 5., 6., 7., 8., 9., 10., 11., 12., 13., 14., 15., 16., ); let src = _mm_set1_ps(100.); - let r = _mm512_mask_extractf32x4_ps(src, 0, a, 0b1); + let r = _mm512_mask_extractf32x4_ps::<1>(src, 0, a); assert_eq_m128(r, src); - let r = _mm512_mask_extractf32x4_ps(src, 0b11111111, a, 0b1); + let r = _mm512_mask_extractf32x4_ps::<1>(src, 0b11111111, a); let e = _mm_setr_ps(5., 6., 7., 8.); assert_eq_m128(r, e); } @@ -49388,9 +46655,9 @@ mod tests { let a = _mm512_setr_ps( 1., 2., 3., 4., 5., 6., 7., 8., 9., 10., 11., 12., 13., 14., 15., 16., ); - let r = _mm512_maskz_extractf32x4_ps(0, a, 0b1); + let r = _mm512_maskz_extractf32x4_ps::<1>(0, a); assert_eq_m128(r, _mm_setzero_ps()); - let r = _mm512_maskz_extractf32x4_ps(0b00000001, a, 0b1); + let r = _mm512_maskz_extractf32x4_ps::<1>(0b00000001, a); let e = _mm_setr_ps(5., 0., 0., 0.); assert_eq_m128(r, e); } @@ -49398,7 +46665,7 @@ mod tests { #[simd_test(enable = "avx512f,avx512vl")] unsafe fn test_mm256_extractf32x4_ps() { let a = _mm256_set_ps(1., 2., 3., 4., 5., 6., 7., 8.); - let r = _mm256_extractf32x4_ps(a, 0b1); + let r = _mm256_extractf32x4_ps::<1>(a); let e = _mm_set_ps(1., 2., 3., 4.); assert_eq_m128(r, e); } @@ -49407,9 +46674,9 @@ mod tests { unsafe fn test_mm256_mask_extractf32x4_ps() { let a = _mm256_set_ps(1., 2., 3., 4., 5., 6., 7., 8.); let src = _mm_set1_ps(100.); - let r = _mm256_mask_extractf32x4_ps(src, 0, a, 0b1); + let r = _mm256_mask_extractf32x4_ps::<1>(src, 0, a); assert_eq_m128(r, src); - let r = _mm256_mask_extractf32x4_ps(src, 0b00001111, a, 0b1); + let r = _mm256_mask_extractf32x4_ps::<1>(src, 0b00001111, a); let e = _mm_set_ps(1., 2., 3., 4.); assert_eq_m128(r, e); } @@ -49417,9 +46684,9 @@ mod tests { #[simd_test(enable = "avx512f,avx512vl")] unsafe fn test_mm256_maskz_extractf32x4_ps() { let a = _mm256_set_ps(1., 2., 3., 4., 5., 6., 7., 8.); - let r = _mm256_maskz_extractf32x4_ps(0, a, 0b1); + let r = _mm256_maskz_extractf32x4_ps::<1>(0, a); assert_eq_m128(r, _mm_setzero_ps()); - let r = _mm256_maskz_extractf32x4_ps(0b00001111, a, 0b1); + let r = _mm256_maskz_extractf32x4_ps::<1>(0b00001111, a); let e = _mm_set_ps(1., 2., 3., 4.); assert_eq_m128(r, e); } @@ -49427,7 +46694,7 @@ mod tests { #[simd_test(enable = "avx512f")] unsafe fn test_mm512_extracti32x4_epi32() { let a = _mm512_setr_epi32(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); - let r = _mm512_extracti32x4_epi32(a, 0b1); + let r = _mm512_extracti32x4_epi32::<1>(a); let e = _mm_setr_epi32(5, 6, 7, 8); assert_eq_m128i(r, e); } @@ -49436,9 +46703,9 @@ mod tests { unsafe fn test_mm512_mask_extracti32x4_epi32() { let a = _mm512_setr_epi32(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); let src = _mm_set1_epi32(100); - let r = _mm512_mask_extracti32x4_epi32(src, 0, a, 0x1); + let r = _mm512_mask_extracti32x4_epi32::<1>(src, 0, a); assert_eq_m128i(r, src); - let r = _mm512_mask_extracti32x4_epi32(src, 0b11111111, a, 0b1); + let r = _mm512_mask_extracti32x4_epi32::<1>(src, 0b11111111, a); let e = _mm_setr_epi32(5, 6, 7, 8); assert_eq_m128i(r, e); } @@ -49446,9 +46713,9 @@ mod tests { #[simd_test(enable = "avx512f,avx512vl")] unsafe fn test_mm512_maskz_extracti32x4_epi32() { let a = _mm512_setr_epi32(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); - let r = _mm512_maskz_extracti32x4_epi32(0, a, 0b1); + let r = _mm512_maskz_extracti32x4_epi32::<1>(0, a); assert_eq_m128i(r, _mm_setzero_si128()); - let r = _mm512_maskz_extracti32x4_epi32(0b00000001, a, 0x1); + let r = _mm512_maskz_extracti32x4_epi32::<1>(0b00000001, a); let e = _mm_setr_epi32(5, 0, 0, 0); assert_eq_m128i(r, e); } @@ -49456,7 +46723,7 @@ mod tests { #[simd_test(enable = "avx512f,avx512vl")] unsafe fn test_mm256_extracti32x4_epi32() { let a = _mm256_set_epi32(1, 2, 3, 4, 5, 6, 7, 8); - let r = _mm256_extracti32x4_epi32(a, 0b1); + let r = _mm256_extracti32x4_epi32::<1>(a); let e = _mm_set_epi32(1, 2, 3, 4); assert_eq_m128i(r, e); } @@ -49465,9 +46732,9 @@ mod tests { unsafe fn test_mm256_mask_extracti32x4_epi32() { let a = _mm256_set_epi32(1, 2, 3, 4, 5, 6, 7, 8); let src = _mm_set1_epi32(100); - let r = _mm256_mask_extracti32x4_epi32(src, 0, a, 0b1); + let r = _mm256_mask_extracti32x4_epi32::<1>(src, 0, a); assert_eq_m128i(r, src); - let r = _mm256_mask_extracti32x4_epi32(src, 0b00001111, a, 0b1); + let r = _mm256_mask_extracti32x4_epi32::<1>(src, 0b00001111, a); let e = _mm_set_epi32(1, 2, 3, 4); assert_eq_m128i(r, e); } @@ -49475,9 +46742,9 @@ mod tests { #[simd_test(enable = "avx512f,avx512vl")] unsafe fn test_mm256_maskz_extracti32x4_epi32() { let a = _mm256_set_epi32(1, 2, 3, 4, 5, 6, 7, 8); - let r = _mm256_maskz_extracti32x4_epi32(0, a, 0b1); + let r = _mm256_maskz_extracti32x4_epi32::<1>(0, a); assert_eq_m128i(r, _mm_setzero_si128()); - let r = _mm256_maskz_extracti32x4_epi32(0b00001111, a, 0b1); + let r = _mm256_maskz_extracti32x4_epi32::<1>(0b00001111, a); let e = _mm_set_epi32(1, 2, 3, 4); assert_eq_m128i(r, e); } @@ -49646,7 +46913,7 @@ mod tests { unsafe fn test_mm512_inserti32x4() { let a = _mm512_setr_epi32(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); let b = _mm_setr_epi32(17, 18, 19, 20); - let r = _mm512_inserti32x4(a, b, 0); + let r = _mm512_inserti32x4::<0>(a, b); let e = _mm512_setr_epi32(17, 18, 19, 20, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); assert_eq_m512i(r, e); } @@ -49655,9 +46922,9 @@ mod tests { unsafe fn test_mm512_mask_inserti32x4() { let a = _mm512_setr_epi32(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); let b = _mm_setr_epi32(17, 18, 19, 20); - let r = _mm512_mask_inserti32x4(a, 0, a, b, 0); + let r = _mm512_mask_inserti32x4::<0>(a, 0, a, b); assert_eq_m512i(r, a); - let r = _mm512_mask_inserti32x4(a, 0b11111111_11111111, a, b, 0); + let r = _mm512_mask_inserti32x4::<0>(a, 0b11111111_11111111, a, b); let e = _mm512_setr_epi32(17, 18, 19, 20, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); assert_eq_m512i(r, e); } @@ -49666,9 +46933,9 @@ mod tests { unsafe fn test_mm512_maskz_inserti32x4() { let a = _mm512_setr_epi32(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); let b = _mm_setr_epi32(17, 18, 19, 20); - let r = _mm512_maskz_inserti32x4(0, a, b, 0); + let r = _mm512_maskz_inserti32x4::<0>(0, a, b); assert_eq_m512i(r, _mm512_setzero_si512()); - let r = _mm512_maskz_inserti32x4(0b00000000_11111111, a, b, 0); + let r = _mm512_maskz_inserti32x4::<0>(0b00000000_11111111, a, b); let e = _mm512_setr_epi32(17, 18, 19, 20, 5, 6, 7, 8, 0, 0, 0, 0, 0, 0, 0, 0); assert_eq_m512i(r, e); } @@ -49677,7 +46944,7 @@ mod tests { unsafe fn test_mm256_inserti32x4() { let a = _mm256_set_epi32(1, 2, 3, 4, 5, 6, 7, 8); let b = _mm_set_epi32(17, 18, 19, 20); - let r = _mm256_inserti32x4(a, b, 1); + let r = _mm256_inserti32x4::<1>(a, b); let e = _mm256_set_epi32(17, 18, 19, 20, 5, 6, 7, 8); assert_eq_m256i(r, e); } @@ -49686,9 +46953,9 @@ mod tests { unsafe fn test_mm256_mask_inserti32x4() { let a = _mm256_set_epi32(1, 2, 3, 4, 5, 6, 7, 8); let b = _mm_set_epi32(17, 18, 19, 20); - let r = _mm256_mask_inserti32x4(a, 0, a, b, 0); + let r = _mm256_mask_inserti32x4::<0>(a, 0, a, b); assert_eq_m256i(r, a); - let r = _mm256_mask_inserti32x4(a, 0b11111111, a, b, 1); + let r = _mm256_mask_inserti32x4::<1>(a, 0b11111111, a, b); let e = _mm256_set_epi32(17, 18, 19, 20, 5, 6, 7, 8); assert_eq_m256i(r, e); } @@ -49697,9 +46964,9 @@ mod tests { unsafe fn test_mm256_maskz_inserti32x4() { let a = _mm256_set_epi32(1, 2, 3, 4, 5, 6, 7, 8); let b = _mm_set_epi32(17, 18, 19, 20); - let r = _mm256_maskz_inserti32x4(0, a, b, 0); + let r = _mm256_maskz_inserti32x4::<0>(0, a, b); assert_eq_m256i(r, _mm256_setzero_si256()); - let r = _mm256_maskz_inserti32x4(0b11111111, a, b, 1); + let r = _mm256_maskz_inserti32x4::<1>(0b11111111, a, b); let e = _mm256_set_epi32(17, 18, 19, 20, 5, 6, 7, 8); assert_eq_m256i(r, e); } @@ -49710,7 +46977,7 @@ mod tests { 1., 2., 3., 4., 5., 6., 7., 8., 9., 10., 11., 12., 13., 14., 15., 16., ); let b = _mm_setr_ps(17., 18., 19., 20.); - let r = _mm512_insertf32x4(a, b, 0); + let r = _mm512_insertf32x4::<0>(a, b); let e = _mm512_setr_ps( 17., 18., 19., 20., 5., 6., 7., 8., 9., 10., 11., 12., 13., 14., 15., 16., ); @@ -49723,9 +46990,9 @@ mod tests { 1., 2., 3., 4., 5., 6., 7., 8., 9., 10., 11., 12., 13., 14., 15., 16., ); let b = _mm_setr_ps(17., 18., 19., 20.); - let r = _mm512_mask_insertf32x4(a, 0, a, b, 0); + let r = _mm512_mask_insertf32x4::<0>(a, 0, a, b); assert_eq_m512(r, a); - let r = _mm512_mask_insertf32x4(a, 0b11111111_11111111, a, b, 0); + let r = _mm512_mask_insertf32x4::<0>(a, 0b11111111_11111111, a, b); let e = _mm512_setr_ps( 17., 18., 19., 20., 5., 6., 7., 8., 9., 10., 11., 12., 13., 14., 15., 16., ); @@ -49738,9 +47005,9 @@ mod tests { 1., 2., 3., 4., 5., 6., 7., 8., 9., 10., 11., 12., 13., 14., 15., 16., ); let b = _mm_setr_ps(17., 18., 19., 20.); - let r = _mm512_maskz_insertf32x4(0, a, b, 0); + let r = _mm512_maskz_insertf32x4::<0>(0, a, b); assert_eq_m512(r, _mm512_setzero_ps()); - let r = _mm512_maskz_insertf32x4(0b00000000_11111111, a, b, 0); + let r = _mm512_maskz_insertf32x4::<0>(0b00000000_11111111, a, b); let e = _mm512_setr_ps( 17., 18., 19., 20., 5., 6., 7., 8., 0., 0., 0., 0., 0., 0., 0., 0., ); @@ -49751,7 +47018,7 @@ mod tests { unsafe fn test_mm256_insertf32x4() { let a = _mm256_set_ps(1., 2., 3., 4., 5., 6., 7., 8.); let b = _mm_set_ps(17., 18., 19., 20.); - let r = _mm256_insertf32x4(a, b, 1); + let r = _mm256_insertf32x4::<1>(a, b); let e = _mm256_set_ps(17., 18., 19., 20., 5., 6., 7., 8.); assert_eq_m256(r, e); } @@ -49760,9 +47027,9 @@ mod tests { unsafe fn test_mm256_mask_insertf32x4() { let a = _mm256_set_ps(1., 2., 3., 4., 5., 6., 7., 8.); let b = _mm_set_ps(17., 18., 19., 20.); - let r = _mm256_mask_insertf32x4(a, 0, a, b, 0); + let r = _mm256_mask_insertf32x4::<0>(a, 0, a, b); assert_eq_m256(r, a); - let r = _mm256_mask_insertf32x4(a, 0b11111111, a, b, 1); + let r = _mm256_mask_insertf32x4::<1>(a, 0b11111111, a, b); let e = _mm256_set_ps(17., 18., 19., 20., 5., 6., 7., 8.); assert_eq_m256(r, e); } @@ -49771,9 +47038,9 @@ mod tests { unsafe fn test_mm256_maskz_insertf32x4() { let a = _mm256_set_ps(1., 2., 3., 4., 5., 6., 7., 8.); let b = _mm_set_ps(17., 18., 19., 20.); - let r = _mm256_maskz_insertf32x4(0, a, b, 0); + let r = _mm256_maskz_insertf32x4::<0>(0, a, b); assert_eq_m256(r, _mm256_setzero_ps()); - let r = _mm256_maskz_insertf32x4(0b11111111, a, b, 1); + let r = _mm256_maskz_insertf32x4::<1>(0b11111111, a, b); let e = _mm256_set_ps(17., 18., 19., 20., 5., 6., 7., 8.); assert_eq_m256(r, e); } @@ -50534,11 +47801,11 @@ mod tests { let b = _mm512_set_epi32( 32, 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, 17, ); - let r = _mm512_alignr_epi32(a, b, 0); + let r = _mm512_alignr_epi32::<0>(a, b); assert_eq_m512i(r, b); - let r = _mm512_alignr_epi32(a, b, 16); + let r = _mm512_alignr_epi32::<16>(a, b); assert_eq_m512i(r, b); - let r = _mm512_alignr_epi32(a, b, 1); + let r = _mm512_alignr_epi32::<1>(a, b); let e = _mm512_set_epi32( 1, 32, 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, ); @@ -50551,9 +47818,9 @@ mod tests { let b = _mm512_set_epi32( 32, 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, 17, ); - let r = _mm512_mask_alignr_epi32(a, 0, a, b, 1); + let r = _mm512_mask_alignr_epi32::<1>(a, 0, a, b); assert_eq_m512i(r, a); - let r = _mm512_mask_alignr_epi32(a, 0b11111111_11111111, a, b, 1); + let r = _mm512_mask_alignr_epi32::<1>(a, 0b11111111_11111111, a, b); let e = _mm512_set_epi32( 1, 32, 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, ); @@ -50566,9 +47833,9 @@ mod tests { let b = _mm512_set_epi32( 32, 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, 17, ); - let r = _mm512_maskz_alignr_epi32(0, a, b, 1); + let r = _mm512_maskz_alignr_epi32::<1>(0, a, b); assert_eq_m512i(r, _mm512_setzero_si512()); - let r = _mm512_maskz_alignr_epi32(0b00000000_11111111, a, b, 1); + let r = _mm512_maskz_alignr_epi32::<1>(0b00000000_11111111, a, b); let e = _mm512_set_epi32(0, 0, 0, 0, 0, 0, 0, 0, 25, 24, 23, 22, 21, 20, 19, 18); assert_eq_m512i(r, e); } @@ -50577,9 +47844,9 @@ mod tests { unsafe fn test_mm256_alignr_epi32() { let a = _mm256_set_epi32(8, 7, 6, 5, 4, 3, 2, 1); let b = _mm256_set_epi32(16, 15, 14, 13, 12, 11, 10, 9); - let r = _mm256_alignr_epi32(a, b, 0); + let r = _mm256_alignr_epi32::<0>(a, b); assert_eq_m256i(r, b); - let r = _mm256_alignr_epi32(a, b, 1); + let r = _mm256_alignr_epi32::<1>(a, b); let e = _mm256_set_epi32(1, 16, 15, 14, 13, 12, 11, 10); assert_eq_m256i(r, e); } @@ -50588,9 +47855,9 @@ mod tests { unsafe fn test_mm256_mask_alignr_epi32() { let a = _mm256_set_epi32(8, 7, 6, 5, 4, 3, 2, 1); let b = _mm256_set_epi32(16, 15, 14, 13, 12, 11, 10, 9); - let r = _mm256_mask_alignr_epi32(a, 0, a, b, 1); + let r = _mm256_mask_alignr_epi32::<1>(a, 0, a, b); assert_eq_m256i(r, a); - let r = _mm256_mask_alignr_epi32(a, 0b11111111, a, b, 1); + let r = _mm256_mask_alignr_epi32::<1>(a, 0b11111111, a, b); let e = _mm256_set_epi32(1, 16, 15, 14, 13, 12, 11, 10); assert_eq_m256i(r, e); } @@ -50599,9 +47866,9 @@ mod tests { unsafe fn test_mm256_maskz_alignr_epi32() { let a = _mm256_set_epi32(8, 7, 6, 5, 4, 3, 2, 1); let b = _mm256_set_epi32(16, 15, 14, 13, 12, 11, 10, 9); - let r = _mm256_maskz_alignr_epi32(0, a, b, 1); + let r = _mm256_maskz_alignr_epi32::<1>(0, a, b); assert_eq_m256i(r, _mm256_setzero_si256()); - let r = _mm256_maskz_alignr_epi32(0b11111111, a, b, 1); + let r = _mm256_maskz_alignr_epi32::<1>(0b11111111, a, b); let e = _mm256_set_epi32(1, 16, 15, 14, 13, 12, 11, 10); assert_eq_m256i(r, e); } @@ -50610,9 +47877,9 @@ mod tests { unsafe fn test_mm_alignr_epi32() { let a = _mm_set_epi32(4, 3, 2, 1); let b = _mm_set_epi32(8, 7, 6, 5); - let r = _mm_alignr_epi32(a, b, 0); + let r = _mm_alignr_epi32::<0>(a, b); assert_eq_m128i(r, b); - let r = _mm_alignr_epi32(a, b, 1); + let r = _mm_alignr_epi32::<1>(a, b); let e = _mm_set_epi32(1, 8, 7, 6); assert_eq_m128i(r, e); } @@ -50621,9 +47888,9 @@ mod tests { unsafe fn test_mm_mask_alignr_epi32() { let a = _mm_set_epi32(4, 3, 2, 1); let b = _mm_set_epi32(8, 7, 6, 5); - let r = _mm_mask_alignr_epi32(a, 0, a, b, 1); + let r = _mm_mask_alignr_epi32::<1>(a, 0, a, b); assert_eq_m128i(r, a); - let r = _mm_mask_alignr_epi32(a, 0b00001111, a, b, 1); + let r = _mm_mask_alignr_epi32::<1>(a, 0b00001111, a, b); let e = _mm_set_epi32(1, 8, 7, 6); assert_eq_m128i(r, e); } @@ -50632,9 +47899,9 @@ mod tests { unsafe fn test_mm_maskz_alignr_epi32() { let a = _mm_set_epi32(4, 3, 2, 1); let b = _mm_set_epi32(8, 7, 6, 5); - let r = _mm_maskz_alignr_epi32(0, a, b, 1); + let r = _mm_maskz_alignr_epi32::<1>(0, a, b); assert_eq_m128i(r, _mm_setzero_si128()); - let r = _mm_maskz_alignr_epi32(0b00001111, a, b, 1); + let r = _mm_maskz_alignr_epi32::<1>(0b00001111, a, b); let e = _mm_set_epi32(1, 8, 7, 6); assert_eq_m128i(r, e); } @@ -52978,7 +50245,7 @@ mod tests { unsafe fn test_mm_getmant_ss() { let a = _mm_set1_ps(20.); let b = _mm_set1_ps(10.); - let r = _mm_getmant_ss(a, b, _MM_MANT_NORM_1_2, _MM_MANT_SIGN_SRC); + let r = _mm_getmant_ss::<_MM_MANT_NORM_1_2, _MM_MANT_SIGN_SRC>(a, b); let e = _mm_set_ps(20., 20., 20., 1.25); assert_eq_m128(r, e); } @@ -52987,10 +50254,10 @@ mod tests { unsafe fn test_mm_mask_getmant_ss() { let a = _mm_set1_ps(20.); let b = _mm_set1_ps(10.); - let r = _mm_mask_getmant_ss(a, 0, a, b, _MM_MANT_NORM_1_2, _MM_MANT_SIGN_SRC); + let r = _mm_mask_getmant_ss::<_MM_MANT_NORM_1_2, _MM_MANT_SIGN_SRC>(a, 0, a, b); let e = _mm_set_ps(20., 20., 20., 20.); assert_eq_m128(r, e); - let r = _mm_mask_getmant_ss(a, 0b11111111, a, b, _MM_MANT_NORM_1_2, _MM_MANT_SIGN_SRC); + let r = _mm_mask_getmant_ss::<_MM_MANT_NORM_1_2, _MM_MANT_SIGN_SRC>(a, 0b11111111, a, b); let e = _mm_set_ps(20., 20., 20., 1.25); assert_eq_m128(r, e); } @@ -52999,10 +50266,10 @@ mod tests { unsafe fn test_mm_maskz_getmant_ss() { let a = _mm_set1_ps(20.); let b = _mm_set1_ps(10.); - let r = _mm_maskz_getmant_ss(0, a, b, _MM_MANT_NORM_1_2, _MM_MANT_SIGN_SRC); + let r = _mm_maskz_getmant_ss::<_MM_MANT_NORM_1_2, _MM_MANT_SIGN_SRC>(0, a, b); let e = _mm_set_ps(20., 20., 20., 0.); assert_eq_m128(r, e); - let r = _mm_maskz_getmant_ss(0b11111111, a, b, _MM_MANT_NORM_1_2, _MM_MANT_SIGN_SRC); + let r = _mm_maskz_getmant_ss::<_MM_MANT_NORM_1_2, _MM_MANT_SIGN_SRC>(0b11111111, a, b); let e = _mm_set_ps(20., 20., 20., 1.25); assert_eq_m128(r, e); } @@ -53011,7 +50278,7 @@ mod tests { unsafe fn test_mm_getmant_sd() { let a = _mm_set1_pd(20.); let b = _mm_set1_pd(10.); - let r = _mm_getmant_sd(a, b, _MM_MANT_NORM_1_2, _MM_MANT_SIGN_SRC); + let r = _mm_getmant_sd::<_MM_MANT_NORM_1_2, _MM_MANT_SIGN_SRC>(a, b); let e = _mm_set_pd(20., 1.25); assert_eq_m128d(r, e); } @@ -53020,10 +50287,10 @@ mod tests { unsafe fn test_mm_mask_getmant_sd() { let a = _mm_set1_pd(20.); let b = _mm_set1_pd(10.); - let r = _mm_mask_getmant_sd(a, 0, a, b, _MM_MANT_NORM_1_2, _MM_MANT_SIGN_SRC); + let r = _mm_mask_getmant_sd::<_MM_MANT_NORM_1_2, _MM_MANT_SIGN_SRC>(a, 0, a, b); let e = _mm_set_pd(20., 20.); assert_eq_m128d(r, e); - let r = _mm_mask_getmant_sd(a, 0b11111111, a, b, _MM_MANT_NORM_1_2, _MM_MANT_SIGN_SRC); + let r = _mm_mask_getmant_sd::<_MM_MANT_NORM_1_2, _MM_MANT_SIGN_SRC>(a, 0b11111111, a, b); let e = _mm_set_pd(20., 1.25); assert_eq_m128d(r, e); } @@ -53032,10 +50299,10 @@ mod tests { unsafe fn test_mm_maskz_getmant_sd() { let a = _mm_set1_pd(20.); let b = _mm_set1_pd(10.); - let r = _mm_maskz_getmant_sd(0, a, b, _MM_MANT_NORM_1_2, _MM_MANT_SIGN_SRC); + let r = _mm_maskz_getmant_sd::<_MM_MANT_NORM_1_2, _MM_MANT_SIGN_SRC>(0, a, b); let e = _mm_set_pd(20., 0.); assert_eq_m128d(r, e); - let r = _mm_maskz_getmant_sd(0b11111111, a, b, _MM_MANT_NORM_1_2, _MM_MANT_SIGN_SRC); + let r = _mm_maskz_getmant_sd::<_MM_MANT_NORM_1_2, _MM_MANT_SIGN_SRC>(0b11111111, a, b); let e = _mm_set_pd(20., 1.25); assert_eq_m128d(r, e); } @@ -53044,7 +50311,7 @@ mod tests { unsafe fn test_mm_roundscale_ss() { let a = _mm_set1_ps(2.2); let b = _mm_set1_ps(1.1); - let r = _mm_roundscale_ss(a, b, 0); + let r = _mm_roundscale_ss::<0>(a, b); let e = _mm_set_ps(2.2, 2.2, 2.2, 1.0); assert_eq_m128(r, e); } @@ -53053,10 +50320,10 @@ mod tests { unsafe fn test_mm_mask_roundscale_ss() { let a = _mm_set1_ps(2.2); let b = _mm_set1_ps(1.1); - let r = _mm_mask_roundscale_ss(a, 0, a, b, 0); + let r = _mm_mask_roundscale_ss::<0>(a, 0, a, b); let e = _mm_set_ps(2.2, 2.2, 2.2, 2.2); assert_eq_m128(r, e); - let r = _mm_mask_roundscale_ss(a, 0b11111111, a, b, 0); + let r = _mm_mask_roundscale_ss::<0>(a, 0b11111111, a, b); let e = _mm_set_ps(2.2, 2.2, 2.2, 1.0); assert_eq_m128(r, e); } @@ -53065,10 +50332,10 @@ mod tests { unsafe fn test_mm_maskz_roundscale_ss() { let a = _mm_set1_ps(2.2); let b = _mm_set1_ps(1.1); - let r = _mm_maskz_roundscale_ss(0, a, b, 0); + let r = _mm_maskz_roundscale_ss::<0>(0, a, b); let e = _mm_set_ps(2.2, 2.2, 2.2, 0.0); assert_eq_m128(r, e); - let r = _mm_maskz_roundscale_ss(0b11111111, a, b, 0); + let r = _mm_maskz_roundscale_ss::<0>(0b11111111, a, b); let e = _mm_set_ps(2.2, 2.2, 2.2, 1.0); assert_eq_m128(r, e); } @@ -53077,7 +50344,7 @@ mod tests { unsafe fn test_mm_roundscale_sd() { let a = _mm_set1_pd(2.2); let b = _mm_set1_pd(1.1); - let r = _mm_roundscale_sd(a, b, 0); + let r = _mm_roundscale_sd::<0>(a, b); let e = _mm_set_pd(2.2, 1.0); assert_eq_m128d(r, e); } @@ -53086,10 +50353,10 @@ mod tests { unsafe fn test_mm_mask_roundscale_sd() { let a = _mm_set1_pd(2.2); let b = _mm_set1_pd(1.1); - let r = _mm_mask_roundscale_sd(a, 0, a, b, 0); + let r = _mm_mask_roundscale_sd::<0>(a, 0, a, b); let e = _mm_set_pd(2.2, 2.2); assert_eq_m128d(r, e); - let r = _mm_mask_roundscale_sd(a, 0b11111111, a, b, 0); + let r = _mm_mask_roundscale_sd::<0>(a, 0b11111111, a, b); let e = _mm_set_pd(2.2, 1.0); assert_eq_m128d(r, e); } @@ -53098,10 +50365,10 @@ mod tests { unsafe fn test_mm_maskz_roundscale_sd() { let a = _mm_set1_pd(2.2); let b = _mm_set1_pd(1.1); - let r = _mm_maskz_roundscale_sd(0, a, b, 0); + let r = _mm_maskz_roundscale_sd::<0>(0, a, b); let e = _mm_set_pd(2.2, 0.0); assert_eq_m128d(r, e); - let r = _mm_maskz_roundscale_sd(0b11111111, a, b, 0); + let r = _mm_maskz_roundscale_sd::<0>(0b11111111, a, b); let e = _mm_set_pd(2.2, 1.0); assert_eq_m128d(r, e); } @@ -53472,7 +50739,7 @@ mod tests { unsafe fn test_mm_add_round_ss() { let a = _mm_set_ps(1., 2., 10., 20.); let b = _mm_set_ps(3., 4., 30., 40.); - let r = _mm_add_round_ss(a, b, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); + let r = _mm_add_round_ss::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(a, b); let e = _mm_set_ps(1., 2., 10., 60.); assert_eq_m128(r, e); } @@ -53482,15 +50749,11 @@ mod tests { let src = _mm_set_ps(10., 11., 100., 110.); let a = _mm_set_ps(1., 2., 10., 20.); let b = _mm_set_ps(3., 4., 30., 40.); - let r = _mm_mask_add_round_ss(src, 0, a, b, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); + let r = _mm_mask_add_round_ss::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(src, 0, a, b); let e = _mm_set_ps(1., 2., 10., 110.); assert_eq_m128(r, e); - let r = _mm_mask_add_round_ss( - src, - 0b11111111, - a, - b, - _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC, + let r = _mm_mask_add_round_ss::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>( + src, 0b11111111, a, b, ); let e = _mm_set_ps(1., 2., 10., 60.); assert_eq_m128(r, e); @@ -53500,10 +50763,11 @@ mod tests { unsafe fn test_mm_maskz_add_round_ss() { let a = _mm_set_ps(1., 2., 10., 20.); let b = _mm_set_ps(3., 4., 30., 40.); - let r = _mm_maskz_add_round_ss(0, a, b, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); + let r = _mm_maskz_add_round_ss::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(0, a, b); let e = _mm_set_ps(1., 2., 10., 0.); assert_eq_m128(r, e); - let r = _mm_maskz_add_round_ss(0b11111111, a, b, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); + let r = + _mm_maskz_add_round_ss::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(0b11111111, a, b); let e = _mm_set_ps(1., 2., 10., 60.); assert_eq_m128(r, e); } @@ -53512,7 +50776,7 @@ mod tests { unsafe fn test_mm_add_round_sd() { let a = _mm_set_pd(1., 2.); let b = _mm_set_pd(3., 4.); - let r = _mm_add_round_sd(a, b, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); + let r = _mm_add_round_sd::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(a, b); let e = _mm_set_pd(1., 6.); assert_eq_m128d(r, e); } @@ -53522,15 +50786,11 @@ mod tests { let src = _mm_set_pd(10., 11.); let a = _mm_set_pd(1., 2.); let b = _mm_set_pd(3., 4.); - let r = _mm_mask_add_round_sd(src, 0, a, b, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); + let r = _mm_mask_add_round_sd::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(src, 0, a, b); let e = _mm_set_pd(1., 11.); assert_eq_m128d(r, e); - let r = _mm_mask_add_round_sd( - src, - 0b11111111, - a, - b, - _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC, + let r = _mm_mask_add_round_sd::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>( + src, 0b11111111, a, b, ); let e = _mm_set_pd(1., 6.); assert_eq_m128d(r, e); @@ -53540,10 +50800,11 @@ mod tests { unsafe fn test_mm_maskz_add_round_sd() { let a = _mm_set_pd(1., 2.); let b = _mm_set_pd(3., 4.); - let r = _mm_maskz_add_round_sd(0, a, b, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); + let r = _mm_maskz_add_round_sd::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(0, a, b); let e = _mm_set_pd(1., 0.); assert_eq_m128d(r, e); - let r = _mm_maskz_add_round_sd(0b11111111, a, b, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); + let r = + _mm_maskz_add_round_sd::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(0b11111111, a, b); let e = _mm_set_pd(1., 6.); assert_eq_m128d(r, e); } @@ -53552,7 +50813,7 @@ mod tests { unsafe fn test_mm_sub_round_ss() { let a = _mm_set_ps(1., 2., 10., 20.); let b = _mm_set_ps(3., 4., 30., 40.); - let r = _mm_sub_round_ss(a, b, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); + let r = _mm_sub_round_ss::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(a, b); let e = _mm_set_ps(1., 2., 10., -20.); assert_eq_m128(r, e); } @@ -53562,15 +50823,11 @@ mod tests { let src = _mm_set_ps(10., 11., 100., 110.); let a = _mm_set_ps(1., 2., 10., 20.); let b = _mm_set_ps(3., 4., 30., 40.); - let r = _mm_mask_sub_round_ss(src, 0, a, b, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); + let r = _mm_mask_sub_round_ss::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(src, 0, a, b); let e = _mm_set_ps(1., 2., 10., 110.); assert_eq_m128(r, e); - let r = _mm_mask_sub_round_ss( - src, - 0b11111111, - a, - b, - _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC, + let r = _mm_mask_sub_round_ss::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>( + src, 0b11111111, a, b, ); let e = _mm_set_ps(1., 2., 10., -20.); assert_eq_m128(r, e); @@ -53580,10 +50837,11 @@ mod tests { unsafe fn test_mm_maskz_sub_round_ss() { let a = _mm_set_ps(1., 2., 10., 20.); let b = _mm_set_ps(3., 4., 30., 40.); - let r = _mm_maskz_sub_round_ss(0, a, b, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); + let r = _mm_maskz_sub_round_ss::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(0, a, b); let e = _mm_set_ps(1., 2., 10., 0.); assert_eq_m128(r, e); - let r = _mm_maskz_sub_round_ss(0b11111111, a, b, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); + let r = + _mm_maskz_sub_round_ss::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(0b11111111, a, b); let e = _mm_set_ps(1., 2., 10., -20.); assert_eq_m128(r, e); } @@ -53592,7 +50850,7 @@ mod tests { unsafe fn test_mm_sub_round_sd() { let a = _mm_set_pd(1., 2.); let b = _mm_set_pd(3., 4.); - let r = _mm_sub_round_sd(a, b, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); + let r = _mm_sub_round_sd::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(a, b); let e = _mm_set_pd(1., -2.); assert_eq_m128d(r, e); } @@ -53602,15 +50860,11 @@ mod tests { let src = _mm_set_pd(10., 11.); let a = _mm_set_pd(1., 2.); let b = _mm_set_pd(3., 4.); - let r = _mm_mask_sub_round_sd(src, 0, a, b, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); + let r = _mm_mask_sub_round_sd::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(src, 0, a, b); let e = _mm_set_pd(1., 11.); assert_eq_m128d(r, e); - let r = _mm_mask_sub_round_sd( - src, - 0b11111111, - a, - b, - _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC, + let r = _mm_mask_sub_round_sd::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>( + src, 0b11111111, a, b, ); let e = _mm_set_pd(1., -2.); assert_eq_m128d(r, e); @@ -53620,10 +50874,11 @@ mod tests { unsafe fn test_mm_maskz_sub_round_sd() { let a = _mm_set_pd(1., 2.); let b = _mm_set_pd(3., 4.); - let r = _mm_maskz_sub_round_sd(0, a, b, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); + let r = _mm_maskz_sub_round_sd::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(0, a, b); let e = _mm_set_pd(1., 0.); assert_eq_m128d(r, e); - let r = _mm_maskz_sub_round_sd(0b11111111, a, b, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); + let r = + _mm_maskz_sub_round_sd::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(0b11111111, a, b); let e = _mm_set_pd(1., -2.); assert_eq_m128d(r, e); } @@ -53632,7 +50887,7 @@ mod tests { unsafe fn test_mm_mul_round_ss() { let a = _mm_set_ps(1., 2., 10., 20.); let b = _mm_set_ps(3., 4., 30., 40.); - let r = _mm_mul_round_ss(a, b, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); + let r = _mm_mul_round_ss::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(a, b); let e = _mm_set_ps(1., 2., 10., 800.); assert_eq_m128(r, e); } @@ -53642,15 +50897,11 @@ mod tests { let src = _mm_set_ps(10., 11., 100., 110.); let a = _mm_set_ps(1., 2., 10., 20.); let b = _mm_set_ps(3., 4., 30., 40.); - let r = _mm_mask_mul_round_ss(src, 0, a, b, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); + let r = _mm_mask_mul_round_ss::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(src, 0, a, b); let e = _mm_set_ps(1., 2., 10., 110.); assert_eq_m128(r, e); - let r = _mm_mask_mul_round_ss( - src, - 0b11111111, - a, - b, - _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC, + let r = _mm_mask_mul_round_ss::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>( + src, 0b11111111, a, b, ); let e = _mm_set_ps(1., 2., 10., 800.); assert_eq_m128(r, e); @@ -53660,10 +50911,11 @@ mod tests { unsafe fn test_mm_maskz_mul_round_ss() { let a = _mm_set_ps(1., 2., 10., 20.); let b = _mm_set_ps(3., 4., 30., 40.); - let r = _mm_maskz_mul_round_ss(0, a, b, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); + let r = _mm_maskz_mul_round_ss::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(0, a, b); let e = _mm_set_ps(1., 2., 10., 0.); assert_eq_m128(r, e); - let r = _mm_maskz_mul_round_ss(0b11111111, a, b, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); + let r = + _mm_maskz_mul_round_ss::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(0b11111111, a, b); let e = _mm_set_ps(1., 2., 10., 800.); assert_eq_m128(r, e); } @@ -53672,7 +50924,7 @@ mod tests { unsafe fn test_mm_mul_round_sd() { let a = _mm_set_pd(1., 2.); let b = _mm_set_pd(3., 4.); - let r = _mm_mul_round_sd(a, b, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); + let r = _mm_mul_round_sd::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(a, b); let e = _mm_set_pd(1., 8.); assert_eq_m128d(r, e); } @@ -53682,15 +50934,11 @@ mod tests { let src = _mm_set_pd(10., 11.); let a = _mm_set_pd(1., 2.); let b = _mm_set_pd(3., 4.); - let r = _mm_mask_mul_round_sd(src, 0, a, b, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); + let r = _mm_mask_mul_round_sd::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(src, 0, a, b); let e = _mm_set_pd(1., 11.); assert_eq_m128d(r, e); - let r = _mm_mask_mul_round_sd( - src, - 0b11111111, - a, - b, - _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC, + let r = _mm_mask_mul_round_sd::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>( + src, 0b11111111, a, b, ); let e = _mm_set_pd(1., 8.); assert_eq_m128d(r, e); @@ -53700,10 +50948,11 @@ mod tests { unsafe fn test_mm_maskz_mul_round_sd() { let a = _mm_set_pd(1., 2.); let b = _mm_set_pd(3., 4.); - let r = _mm_maskz_mul_round_sd(0, a, b, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); + let r = _mm_maskz_mul_round_sd::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(0, a, b); let e = _mm_set_pd(1., 0.); assert_eq_m128d(r, e); - let r = _mm_maskz_mul_round_sd(0b11111111, a, b, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); + let r = + _mm_maskz_mul_round_sd::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(0b11111111, a, b); let e = _mm_set_pd(1., 8.); assert_eq_m128d(r, e); } @@ -53712,7 +50961,7 @@ mod tests { unsafe fn test_mm_div_round_ss() { let a = _mm_set_ps(1., 2., 10., 20.); let b = _mm_set_ps(3., 4., 30., 40.); - let r = _mm_div_round_ss(a, b, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); + let r = _mm_div_round_ss::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(a, b); let e = _mm_set_ps(1., 2., 10., 0.5); assert_eq_m128(r, e); } @@ -53722,15 +50971,11 @@ mod tests { let src = _mm_set_ps(10., 11., 100., 110.); let a = _mm_set_ps(1., 2., 10., 20.); let b = _mm_set_ps(3., 4., 30., 40.); - let r = _mm_mask_div_round_ss(src, 0, a, b, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); + let r = _mm_mask_div_round_ss::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(src, 0, a, b); let e = _mm_set_ps(1., 2., 10., 110.); assert_eq_m128(r, e); - let r = _mm_mask_div_round_ss( - src, - 0b11111111, - a, - b, - _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC, + let r = _mm_mask_div_round_ss::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>( + src, 0b11111111, a, b, ); let e = _mm_set_ps(1., 2., 10., 0.5); assert_eq_m128(r, e); @@ -53740,10 +50985,11 @@ mod tests { unsafe fn test_mm_maskz_div_round_ss() { let a = _mm_set_ps(1., 2., 10., 20.); let b = _mm_set_ps(3., 4., 30., 40.); - let r = _mm_maskz_div_round_ss(0, a, b, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); + let r = _mm_maskz_div_round_ss::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(0, a, b); let e = _mm_set_ps(1., 2., 10., 0.); assert_eq_m128(r, e); - let r = _mm_maskz_div_round_ss(0b11111111, a, b, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); + let r = + _mm_maskz_div_round_ss::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(0b11111111, a, b); let e = _mm_set_ps(1., 2., 10., 0.5); assert_eq_m128(r, e); } @@ -53752,7 +50998,7 @@ mod tests { unsafe fn test_mm_div_round_sd() { let a = _mm_set_pd(1., 2.); let b = _mm_set_pd(3., 4.); - let r = _mm_div_round_sd(a, b, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); + let r = _mm_div_round_sd::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(a, b); let e = _mm_set_pd(1., 0.5); assert_eq_m128d(r, e); } @@ -53762,15 +51008,11 @@ mod tests { let src = _mm_set_pd(10., 11.); let a = _mm_set_pd(1., 2.); let b = _mm_set_pd(3., 4.); - let r = _mm_mask_div_round_sd(src, 0, a, b, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); + let r = _mm_mask_div_round_sd::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(src, 0, a, b); let e = _mm_set_pd(1., 11.); assert_eq_m128d(r, e); - let r = _mm_mask_div_round_sd( - src, - 0b11111111, - a, - b, - _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC, + let r = _mm_mask_div_round_sd::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>( + src, 0b11111111, a, b, ); let e = _mm_set_pd(1., 0.5); assert_eq_m128d(r, e); @@ -53780,10 +51022,11 @@ mod tests { unsafe fn test_mm_maskz_div_round_sd() { let a = _mm_set_pd(1., 2.); let b = _mm_set_pd(3., 4.); - let r = _mm_maskz_div_round_sd(0, a, b, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); + let r = _mm_maskz_div_round_sd::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(0, a, b); let e = _mm_set_pd(1., 0.); assert_eq_m128d(r, e); - let r = _mm_maskz_div_round_sd(0b11111111, a, b, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); + let r = + _mm_maskz_div_round_sd::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(0b11111111, a, b); let e = _mm_set_pd(1., 0.5); assert_eq_m128d(r, e); } @@ -53792,7 +51035,7 @@ mod tests { unsafe fn test_mm_max_round_ss() { let a = _mm_set_ps(0., 1., 2., 3.); let b = _mm_set_ps(4., 5., 6., 7.); - let r = _mm_max_round_ss(a, b, _MM_FROUND_CUR_DIRECTION); + let r = _mm_max_round_ss::<_MM_FROUND_CUR_DIRECTION>(a, b); let e = _mm_set_ps(0., 1., 2., 7.); assert_eq_m128(r, e); } @@ -53801,10 +51044,10 @@ mod tests { unsafe fn test_mm_mask_max_round_ss() { let a = _mm_set_ps(0., 1., 2., 3.); let b = _mm_set_ps(4., 5., 6., 7.); - let r = _mm_mask_max_round_ss(a, 0, a, b, _MM_FROUND_CUR_DIRECTION); + let r = _mm_mask_max_round_ss::<_MM_FROUND_CUR_DIRECTION>(a, 0, a, b); let e = _mm_set_ps(0., 1., 2., 3.); assert_eq_m128(r, e); - let r = _mm_mask_max_round_ss(a, 0b11111111, a, b, _MM_FROUND_CUR_DIRECTION); + let r = _mm_mask_max_round_ss::<_MM_FROUND_CUR_DIRECTION>(a, 0b11111111, a, b); let e = _mm_set_ps(0., 1., 2., 7.); assert_eq_m128(r, e); } @@ -53813,10 +51056,10 @@ mod tests { unsafe fn test_mm_maskz_max_round_ss() { let a = _mm_set_ps(0., 1., 2., 3.); let b = _mm_set_ps(4., 5., 6., 7.); - let r = _mm_maskz_max_round_ss(0, a, b, _MM_FROUND_CUR_DIRECTION); + let r = _mm_maskz_max_round_ss::<_MM_FROUND_CUR_DIRECTION>(0, a, b); let e = _mm_set_ps(0., 1., 2., 0.); assert_eq_m128(r, e); - let r = _mm_maskz_max_round_ss(0b11111111, a, b, _MM_FROUND_CUR_DIRECTION); + let r = _mm_maskz_max_round_ss::<_MM_FROUND_CUR_DIRECTION>(0b11111111, a, b); let e = _mm_set_ps(0., 1., 2., 7.); assert_eq_m128(r, e); } @@ -53825,7 +51068,7 @@ mod tests { unsafe fn test_mm_max_round_sd() { let a = _mm_set_pd(0., 1.); let b = _mm_set_pd(2., 3.); - let r = _mm_max_round_sd(a, b, _MM_FROUND_CUR_DIRECTION); + let r = _mm_max_round_sd::<_MM_FROUND_CUR_DIRECTION>(a, b); let e = _mm_set_pd(0., 3.); assert_eq_m128d(r, e); } @@ -53834,10 +51077,10 @@ mod tests { unsafe fn test_mm_mask_max_round_sd() { let a = _mm_set_pd(0., 1.); let b = _mm_set_pd(2., 3.); - let r = _mm_mask_max_round_sd(a, 0, a, b, _MM_FROUND_CUR_DIRECTION); + let r = _mm_mask_max_round_sd::<_MM_FROUND_CUR_DIRECTION>(a, 0, a, b); let e = _mm_set_pd(0., 1.); assert_eq_m128d(r, e); - let r = _mm_mask_max_round_sd(a, 0b11111111, a, b, _MM_FROUND_CUR_DIRECTION); + let r = _mm_mask_max_round_sd::<_MM_FROUND_CUR_DIRECTION>(a, 0b11111111, a, b); let e = _mm_set_pd(0., 3.); assert_eq_m128d(r, e); } @@ -53846,10 +51089,10 @@ mod tests { unsafe fn test_mm_maskz_max_round_sd() { let a = _mm_set_pd(0., 1.); let b = _mm_set_pd(2., 3.); - let r = _mm_maskz_max_round_sd(0, a, b, _MM_FROUND_CUR_DIRECTION); + let r = _mm_maskz_max_round_sd::<_MM_FROUND_CUR_DIRECTION>(0, a, b); let e = _mm_set_pd(0., 0.); assert_eq_m128d(r, e); - let r = _mm_maskz_max_round_sd(0b11111111, a, b, _MM_FROUND_CUR_DIRECTION); + let r = _mm_maskz_max_round_sd::<_MM_FROUND_CUR_DIRECTION>(0b11111111, a, b); let e = _mm_set_pd(0., 3.); assert_eq_m128d(r, e); } @@ -53858,7 +51101,7 @@ mod tests { unsafe fn test_mm_min_round_ss() { let a = _mm_set_ps(0., 1., 2., 3.); let b = _mm_set_ps(4., 5., 6., 7.); - let r = _mm_min_round_ss(a, b, _MM_FROUND_CUR_DIRECTION); + let r = _mm_min_round_ss::<_MM_FROUND_CUR_DIRECTION>(a, b); let e = _mm_set_ps(0., 1., 2., 3.); assert_eq_m128(r, e); } @@ -53867,10 +51110,10 @@ mod tests { unsafe fn test_mm_mask_min_round_ss() { let a = _mm_set_ps(0., 1., 2., 3.); let b = _mm_set_ps(4., 5., 6., 7.); - let r = _mm_mask_min_round_ss(a, 0, a, b, _MM_FROUND_CUR_DIRECTION); + let r = _mm_mask_min_round_ss::<_MM_FROUND_CUR_DIRECTION>(a, 0, a, b); let e = _mm_set_ps(0., 1., 2., 3.); assert_eq_m128(r, e); - let r = _mm_mask_min_round_ss(a, 0b11111111, a, b, _MM_FROUND_CUR_DIRECTION); + let r = _mm_mask_min_round_ss::<_MM_FROUND_CUR_DIRECTION>(a, 0b11111111, a, b); let e = _mm_set_ps(0., 1., 2., 3.); assert_eq_m128(r, e); } @@ -53879,10 +51122,10 @@ mod tests { unsafe fn test_mm_maskz_min_round_ss() { let a = _mm_set_ps(0., 1., 2., 3.); let b = _mm_set_ps(4., 5., 6., 7.); - let r = _mm_maskz_min_round_ss(0, a, b, _MM_FROUND_CUR_DIRECTION); + let r = _mm_maskz_min_round_ss::<_MM_FROUND_CUR_DIRECTION>(0, a, b); let e = _mm_set_ps(0., 1., 2., 0.); assert_eq_m128(r, e); - let r = _mm_maskz_min_round_ss(0b11111111, a, b, _MM_FROUND_CUR_DIRECTION); + let r = _mm_maskz_min_round_ss::<_MM_FROUND_CUR_DIRECTION>(0b11111111, a, b); let e = _mm_set_ps(0., 1., 2., 3.); assert_eq_m128(r, e); } @@ -53891,7 +51134,7 @@ mod tests { unsafe fn test_mm_min_round_sd() { let a = _mm_set_pd(0., 1.); let b = _mm_set_pd(2., 3.); - let r = _mm_min_round_sd(a, b, _MM_FROUND_CUR_DIRECTION); + let r = _mm_min_round_sd::<_MM_FROUND_CUR_DIRECTION>(a, b); let e = _mm_set_pd(0., 1.); assert_eq_m128d(r, e); } @@ -53900,10 +51143,10 @@ mod tests { unsafe fn test_mm_mask_min_round_sd() { let a = _mm_set_pd(0., 1.); let b = _mm_set_pd(2., 3.); - let r = _mm_mask_min_round_sd(a, 0, a, b, _MM_FROUND_CUR_DIRECTION); + let r = _mm_mask_min_round_sd::<_MM_FROUND_CUR_DIRECTION>(a, 0, a, b); let e = _mm_set_pd(0., 1.); assert_eq_m128d(r, e); - let r = _mm_mask_min_round_sd(a, 0b11111111, a, b, _MM_FROUND_CUR_DIRECTION); + let r = _mm_mask_min_round_sd::<_MM_FROUND_CUR_DIRECTION>(a, 0b11111111, a, b); let e = _mm_set_pd(0., 1.); assert_eq_m128d(r, e); } @@ -53912,10 +51155,10 @@ mod tests { unsafe fn test_mm_maskz_min_round_sd() { let a = _mm_set_pd(0., 1.); let b = _mm_set_pd(2., 3.); - let r = _mm_maskz_min_round_sd(0, a, b, _MM_FROUND_CUR_DIRECTION); + let r = _mm_maskz_min_round_sd::<_MM_FROUND_CUR_DIRECTION>(0, a, b); let e = _mm_set_pd(0., 0.); assert_eq_m128d(r, e); - let r = _mm_maskz_min_round_sd(0b11111111, a, b, _MM_FROUND_CUR_DIRECTION); + let r = _mm_maskz_min_round_sd::<_MM_FROUND_CUR_DIRECTION>(0b11111111, a, b); let e = _mm_set_pd(0., 1.); assert_eq_m128d(r, e); } @@ -53924,7 +51167,7 @@ mod tests { unsafe fn test_mm_sqrt_round_ss() { let a = _mm_set_ps(1., 2., 10., 20.); let b = _mm_set_ps(3., 4., 30., 4.); - let r = _mm_sqrt_round_ss(a, b, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); + let r = _mm_sqrt_round_ss::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(a, b); let e = _mm_set_ps(1., 2., 10., 2.); assert_eq_m128(r, e); } @@ -53934,15 +51177,11 @@ mod tests { let src = _mm_set_ps(10., 11., 100., 110.); let a = _mm_set_ps(1., 2., 10., 20.); let b = _mm_set_ps(3., 4., 30., 4.); - let r = _mm_mask_sqrt_round_ss(src, 0, a, b, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); + let r = _mm_mask_sqrt_round_ss::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(src, 0, a, b); let e = _mm_set_ps(1., 2., 10., 110.); assert_eq_m128(r, e); - let r = _mm_mask_sqrt_round_ss( - src, - 0b11111111, - a, - b, - _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC, + let r = _mm_mask_sqrt_round_ss::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>( + src, 0b11111111, a, b, ); let e = _mm_set_ps(1., 2., 10., 2.); assert_eq_m128(r, e); @@ -53952,10 +51191,11 @@ mod tests { unsafe fn test_mm_maskz_sqrt_round_ss() { let a = _mm_set_ps(1., 2., 10., 20.); let b = _mm_set_ps(3., 4., 30., 4.); - let r = _mm_maskz_sqrt_round_ss(0, a, b, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); + let r = _mm_maskz_sqrt_round_ss::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(0, a, b); let e = _mm_set_ps(1., 2., 10., 0.); assert_eq_m128(r, e); - let r = _mm_maskz_sqrt_round_ss(0b11111111, a, b, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); + let r = + _mm_maskz_sqrt_round_ss::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(0b11111111, a, b); let e = _mm_set_ps(1., 2., 10., 2.); assert_eq_m128(r, e); } @@ -53964,7 +51204,7 @@ mod tests { unsafe fn test_mm_sqrt_round_sd() { let a = _mm_set_pd(1., 2.); let b = _mm_set_pd(3., 4.); - let r = _mm_sqrt_round_sd(a, b, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); + let r = _mm_sqrt_round_sd::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(a, b); let e = _mm_set_pd(1., 2.); assert_eq_m128d(r, e); } @@ -53974,15 +51214,11 @@ mod tests { let src = _mm_set_pd(10., 11.); let a = _mm_set_pd(1., 2.); let b = _mm_set_pd(3., 4.); - let r = _mm_mask_sqrt_round_sd(src, 0, a, b, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); + let r = _mm_mask_sqrt_round_sd::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(src, 0, a, b); let e = _mm_set_pd(1., 11.); assert_eq_m128d(r, e); - let r = _mm_mask_sqrt_round_sd( - src, - 0b11111111, - a, - b, - _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC, + let r = _mm_mask_sqrt_round_sd::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>( + src, 0b11111111, a, b, ); let e = _mm_set_pd(1., 2.); assert_eq_m128d(r, e); @@ -53992,10 +51228,11 @@ mod tests { unsafe fn test_mm_maskz_sqrt_round_sd() { let a = _mm_set_pd(1., 2.); let b = _mm_set_pd(3., 4.); - let r = _mm_maskz_sqrt_round_sd(0, a, b, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); + let r = _mm_maskz_sqrt_round_sd::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(0, a, b); let e = _mm_set_pd(1., 0.); assert_eq_m128d(r, e); - let r = _mm_maskz_sqrt_round_sd(0b11111111, a, b, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); + let r = + _mm_maskz_sqrt_round_sd::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(0b11111111, a, b); let e = _mm_set_pd(1., 2.); assert_eq_m128d(r, e); } @@ -54004,7 +51241,7 @@ mod tests { unsafe fn test_mm_getexp_round_ss() { let a = _mm_set1_ps(2.); let b = _mm_set1_ps(3.); - let r = _mm_getexp_round_ss(a, b, _MM_FROUND_CUR_DIRECTION); + let r = _mm_getexp_round_ss::<_MM_FROUND_CUR_DIRECTION>(a, b); let e = _mm_set_ps(2., 2., 2., 1.); assert_eq_m128(r, e); } @@ -54013,10 +51250,10 @@ mod tests { unsafe fn test_mm_mask_getexp_round_ss() { let a = _mm_set1_ps(2.); let b = _mm_set1_ps(3.); - let r = _mm_mask_getexp_round_ss(a, 0, a, b, _MM_FROUND_CUR_DIRECTION); + let r = _mm_mask_getexp_round_ss::<_MM_FROUND_CUR_DIRECTION>(a, 0, a, b); let e = _mm_set_ps(2., 2., 2., 2.); assert_eq_m128(r, e); - let r = _mm_mask_getexp_round_ss(a, 0b11111111, a, b, _MM_FROUND_CUR_DIRECTION); + let r = _mm_mask_getexp_round_ss::<_MM_FROUND_CUR_DIRECTION>(a, 0b11111111, a, b); let e = _mm_set_ps(2., 2., 2., 1.); assert_eq_m128(r, e); } @@ -54025,10 +51262,10 @@ mod tests { unsafe fn test_mm_maskz_getexp_round_ss() { let a = _mm_set1_ps(2.); let b = _mm_set1_ps(3.); - let r = _mm_maskz_getexp_round_ss(0, a, b, _MM_FROUND_CUR_DIRECTION); + let r = _mm_maskz_getexp_round_ss::<_MM_FROUND_CUR_DIRECTION>(0, a, b); let e = _mm_set_ps(2., 2., 2., 0.); assert_eq_m128(r, e); - let r = _mm_maskz_getexp_round_ss(0b11111111, a, b, _MM_FROUND_CUR_DIRECTION); + let r = _mm_maskz_getexp_round_ss::<_MM_FROUND_CUR_DIRECTION>(0b11111111, a, b); let e = _mm_set_ps(2., 2., 2., 1.); assert_eq_m128(r, e); } @@ -54037,7 +51274,7 @@ mod tests { unsafe fn test_mm_getexp_round_sd() { let a = _mm_set1_pd(2.); let b = _mm_set1_pd(3.); - let r = _mm_getexp_round_sd(a, b, _MM_FROUND_CUR_DIRECTION); + let r = _mm_getexp_round_sd::<_MM_FROUND_CUR_DIRECTION>(a, b); let e = _mm_set_pd(2., 1.); assert_eq_m128d(r, e); } @@ -54046,10 +51283,10 @@ mod tests { unsafe fn test_mm_mask_getexp_round_sd() { let a = _mm_set1_pd(2.); let b = _mm_set1_pd(3.); - let r = _mm_mask_getexp_round_sd(a, 0, a, b, _MM_FROUND_CUR_DIRECTION); + let r = _mm_mask_getexp_round_sd::<_MM_FROUND_CUR_DIRECTION>(a, 0, a, b); let e = _mm_set_pd(2., 2.); assert_eq_m128d(r, e); - let r = _mm_mask_getexp_round_sd(a, 0b11111111, a, b, _MM_FROUND_CUR_DIRECTION); + let r = _mm_mask_getexp_round_sd::<_MM_FROUND_CUR_DIRECTION>(a, 0b11111111, a, b); let e = _mm_set_pd(2., 1.); assert_eq_m128d(r, e); } @@ -54058,10 +51295,10 @@ mod tests { unsafe fn test_mm_maskz_getexp_round_sd() { let a = _mm_set1_pd(2.); let b = _mm_set1_pd(3.); - let r = _mm_maskz_getexp_round_sd(0, a, b, _MM_FROUND_CUR_DIRECTION); + let r = _mm_maskz_getexp_round_sd::<_MM_FROUND_CUR_DIRECTION>(0, a, b); let e = _mm_set_pd(2., 0.); assert_eq_m128d(r, e); - let r = _mm_maskz_getexp_round_sd(0b11111111, a, b, _MM_FROUND_CUR_DIRECTION); + let r = _mm_maskz_getexp_round_sd::<_MM_FROUND_CUR_DIRECTION>(0b11111111, a, b); let e = _mm_set_pd(2., 1.); assert_eq_m128d(r, e); } @@ -54070,13 +51307,10 @@ mod tests { unsafe fn test_mm_getmant_round_ss() { let a = _mm_set1_ps(20.); let b = _mm_set1_ps(10.); - let r = _mm_getmant_round_ss( - a, - b, - _MM_MANT_NORM_1_2, - _MM_MANT_SIGN_SRC, - _MM_FROUND_CUR_DIRECTION, - ); + let r = + _mm_getmant_round_ss::<_MM_MANT_NORM_1_2, _MM_MANT_SIGN_SRC, _MM_FROUND_CUR_DIRECTION>( + a, b, + ); let e = _mm_set_ps(20., 20., 20., 1.25); assert_eq_m128(r, e); } @@ -54085,26 +51319,18 @@ mod tests { unsafe fn test_mm_mask_getmant_round_ss() { let a = _mm_set1_ps(20.); let b = _mm_set1_ps(10.); - let r = _mm_mask_getmant_round_ss( - a, - 0, - a, - b, + let r = _mm_mask_getmant_round_ss::< _MM_MANT_NORM_1_2, _MM_MANT_SIGN_SRC, _MM_FROUND_CUR_DIRECTION, - ); + >(a, 0, a, b); let e = _mm_set_ps(20., 20., 20., 20.); assert_eq_m128(r, e); - let r = _mm_mask_getmant_round_ss( - a, - 0b11111111, - a, - b, + let r = _mm_mask_getmant_round_ss::< _MM_MANT_NORM_1_2, _MM_MANT_SIGN_SRC, _MM_FROUND_CUR_DIRECTION, - ); + >(a, 0b11111111, a, b); let e = _mm_set_ps(20., 20., 20., 1.25); assert_eq_m128(r, e); } @@ -54113,24 +51339,18 @@ mod tests { unsafe fn test_mm_maskz_getmant_round_ss() { let a = _mm_set1_ps(20.); let b = _mm_set1_ps(10.); - let r = _mm_maskz_getmant_round_ss( - 0, - a, - b, + let r = _mm_maskz_getmant_round_ss::< _MM_MANT_NORM_1_2, _MM_MANT_SIGN_SRC, _MM_FROUND_CUR_DIRECTION, - ); + >(0, a, b); let e = _mm_set_ps(20., 20., 20., 0.); assert_eq_m128(r, e); - let r = _mm_maskz_getmant_round_ss( - 0b11111111, - a, - b, + let r = _mm_maskz_getmant_round_ss::< _MM_MANT_NORM_1_2, _MM_MANT_SIGN_SRC, _MM_FROUND_CUR_DIRECTION, - ); + >(0b11111111, a, b); let e = _mm_set_ps(20., 20., 20., 1.25); assert_eq_m128(r, e); } @@ -54139,13 +51359,10 @@ mod tests { unsafe fn test_mm_getmant_round_sd() { let a = _mm_set1_pd(20.); let b = _mm_set1_pd(10.); - let r = _mm_getmant_round_sd( - a, - b, - _MM_MANT_NORM_1_2, - _MM_MANT_SIGN_SRC, - _MM_FROUND_CUR_DIRECTION, - ); + let r = + _mm_getmant_round_sd::<_MM_MANT_NORM_1_2, _MM_MANT_SIGN_SRC, _MM_FROUND_CUR_DIRECTION>( + a, b, + ); let e = _mm_set_pd(20., 1.25); assert_eq_m128d(r, e); } @@ -54154,26 +51371,18 @@ mod tests { unsafe fn test_mm_mask_getmant_round_sd() { let a = _mm_set1_pd(20.); let b = _mm_set1_pd(10.); - let r = _mm_mask_getmant_round_sd( - a, - 0, - a, - b, + let r = _mm_mask_getmant_round_sd::< _MM_MANT_NORM_1_2, _MM_MANT_SIGN_SRC, _MM_FROUND_CUR_DIRECTION, - ); + >(a, 0, a, b); let e = _mm_set_pd(20., 20.); assert_eq_m128d(r, e); - let r = _mm_mask_getmant_round_sd( - a, - 0b11111111, - a, - b, + let r = _mm_mask_getmant_round_sd::< _MM_MANT_NORM_1_2, _MM_MANT_SIGN_SRC, _MM_FROUND_CUR_DIRECTION, - ); + >(a, 0b11111111, a, b); let e = _mm_set_pd(20., 1.25); assert_eq_m128d(r, e); } @@ -54182,24 +51391,18 @@ mod tests { unsafe fn test_mm_maskz_getmant_round_sd() { let a = _mm_set1_pd(20.); let b = _mm_set1_pd(10.); - let r = _mm_maskz_getmant_round_sd( - 0, - a, - b, + let r = _mm_maskz_getmant_round_sd::< _MM_MANT_NORM_1_2, _MM_MANT_SIGN_SRC, _MM_FROUND_CUR_DIRECTION, - ); + >(0, a, b); let e = _mm_set_pd(20., 0.); assert_eq_m128d(r, e); - let r = _mm_maskz_getmant_round_sd( - 0b11111111, - a, - b, + let r = _mm_maskz_getmant_round_sd::< _MM_MANT_NORM_1_2, _MM_MANT_SIGN_SRC, _MM_FROUND_CUR_DIRECTION, - ); + >(0b11111111, a, b); let e = _mm_set_pd(20., 1.25); assert_eq_m128d(r, e); } @@ -54208,7 +51411,7 @@ mod tests { unsafe fn test_mm_roundscale_round_ss() { let a = _mm_set1_ps(2.2); let b = _mm_set1_ps(1.1); - let r = _mm_roundscale_round_ss(a, b, 0, _MM_FROUND_CUR_DIRECTION); + let r = _mm_roundscale_round_ss::<0, _MM_FROUND_CUR_DIRECTION>(a, b); let e = _mm_set_ps(2.2, 2.2, 2.2, 1.0); assert_eq_m128(r, e); } @@ -54217,10 +51420,10 @@ mod tests { unsafe fn test_mm_mask_roundscale_round_ss() { let a = _mm_set1_ps(2.2); let b = _mm_set1_ps(1.1); - let r = _mm_mask_roundscale_round_ss(a, 0, a, b, 0, _MM_FROUND_CUR_DIRECTION); + let r = _mm_mask_roundscale_round_ss::<0, _MM_FROUND_CUR_DIRECTION>(a, 0, a, b); let e = _mm_set_ps(2.2, 2.2, 2.2, 2.2); assert_eq_m128(r, e); - let r = _mm_mask_roundscale_round_ss(a, 0b11111111, a, b, 0, _MM_FROUND_CUR_DIRECTION); + let r = _mm_mask_roundscale_round_ss::<0, _MM_FROUND_CUR_DIRECTION>(a, 0b11111111, a, b); let e = _mm_set_ps(2.2, 2.2, 2.2, 1.0); assert_eq_m128(r, e); } @@ -54229,10 +51432,10 @@ mod tests { unsafe fn test_mm_maskz_roundscale_round_ss() { let a = _mm_set1_ps(2.2); let b = _mm_set1_ps(1.1); - let r = _mm_maskz_roundscale_round_ss(0, a, b, 0, _MM_FROUND_CUR_DIRECTION); + let r = _mm_maskz_roundscale_round_ss::<0, _MM_FROUND_CUR_DIRECTION>(0, a, b); let e = _mm_set_ps(2.2, 2.2, 2.2, 0.0); assert_eq_m128(r, e); - let r = _mm_maskz_roundscale_round_ss(0b11111111, a, b, 0, _MM_FROUND_CUR_DIRECTION); + let r = _mm_maskz_roundscale_round_ss::<0, _MM_FROUND_CUR_DIRECTION>(0b11111111, a, b); let e = _mm_set_ps(2.2, 2.2, 2.2, 1.0); assert_eq_m128(r, e); } @@ -54241,7 +51444,7 @@ mod tests { unsafe fn test_mm_roundscale_round_sd() { let a = _mm_set1_pd(2.2); let b = _mm_set1_pd(1.1); - let r = _mm_roundscale_round_sd(a, b, 0, _MM_FROUND_CUR_DIRECTION); + let r = _mm_roundscale_round_sd::<0, _MM_FROUND_CUR_DIRECTION>(a, b); let e = _mm_set_pd(2.2, 1.0); assert_eq_m128d(r, e); } @@ -54250,10 +51453,10 @@ mod tests { unsafe fn test_mm_mask_roundscale_round_sd() { let a = _mm_set1_pd(2.2); let b = _mm_set1_pd(1.1); - let r = _mm_mask_roundscale_round_sd(a, 0, a, b, 0, _MM_FROUND_CUR_DIRECTION); + let r = _mm_mask_roundscale_round_sd::<0, _MM_FROUND_CUR_DIRECTION>(a, 0, a, b); let e = _mm_set_pd(2.2, 2.2); assert_eq_m128d(r, e); - let r = _mm_mask_roundscale_round_sd(a, 0b11111111, a, b, 0, _MM_FROUND_CUR_DIRECTION); + let r = _mm_mask_roundscale_round_sd::<0, _MM_FROUND_CUR_DIRECTION>(a, 0b11111111, a, b); let e = _mm_set_pd(2.2, 1.0); assert_eq_m128d(r, e); } @@ -54262,10 +51465,10 @@ mod tests { unsafe fn test_mm_maskz_roundscale_round_sd() { let a = _mm_set1_pd(2.2); let b = _mm_set1_pd(1.1); - let r = _mm_maskz_roundscale_round_sd(0, a, b, 0, _MM_FROUND_CUR_DIRECTION); + let r = _mm_maskz_roundscale_round_sd::<0, _MM_FROUND_CUR_DIRECTION>(0, a, b); let e = _mm_set_pd(2.2, 0.0); assert_eq_m128d(r, e); - let r = _mm_maskz_roundscale_round_sd(0b11111111, a, b, 0, _MM_FROUND_CUR_DIRECTION); + let r = _mm_maskz_roundscale_round_sd::<0, _MM_FROUND_CUR_DIRECTION>(0b11111111, a, b); let e = _mm_set_pd(2.2, 1.0); assert_eq_m128d(r, e); } @@ -54274,7 +51477,7 @@ mod tests { unsafe fn test_mm_scalef_round_ss() { let a = _mm_set1_ps(1.); let b = _mm_set1_ps(3.); - let r = _mm_scalef_round_ss(a, b, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); + let r = _mm_scalef_round_ss::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>(a, b); let e = _mm_set_ps(1., 1., 1., 8.); assert_eq_m128(r, e); } @@ -54283,15 +51486,13 @@ mod tests { unsafe fn test_mm_mask_scalef_round_ss() { let a = _mm_set1_ps(1.); let b = _mm_set1_ps(3.); - let r = _mm_mask_scalef_round_ss(a, 0, a, b, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); + let r = _mm_mask_scalef_round_ss::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, 0, a, b, + ); let e = _mm_set_ps(1., 1., 1., 1.); assert_eq_m128(r, e); - let r = _mm_mask_scalef_round_ss( - a, - 0b11111111, - a, - b, - _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC, + let r = _mm_mask_scalef_round_ss::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, 0b11111111, a, b, ); let e = _mm_set_ps(1., 1., 1., 8.); assert_eq_m128(r, e); @@ -54301,14 +51502,12 @@ mod tests { unsafe fn test_mm_maskz_scalef_round_ss() { let a = _mm_set1_ps(1.); let b = _mm_set1_ps(3.); - let r = _mm_maskz_scalef_round_ss(0, a, b, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); + let r = + _mm_maskz_scalef_round_ss::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>(0, a, b); let e = _mm_set_ps(1., 1., 1., 0.); assert_eq_m128(r, e); - let r = _mm_maskz_scalef_round_ss( - 0b11111111, - a, - b, - _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC, + let r = _mm_maskz_scalef_round_ss::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + 0b11111111, a, b, ); let e = _mm_set_ps(1., 1., 1., 8.); assert_eq_m128(r, e); @@ -54318,7 +51517,7 @@ mod tests { unsafe fn test_mm_scalef_round_sd() { let a = _mm_set1_pd(1.); let b = _mm_set1_pd(3.); - let r = _mm_scalef_round_sd(a, b, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); + let r = _mm_scalef_round_sd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>(a, b); let e = _mm_set_pd(1., 8.); assert_eq_m128d(r, e); } @@ -54327,15 +51526,13 @@ mod tests { unsafe fn test_mm_mask_scalef_round_sd() { let a = _mm_set1_pd(1.); let b = _mm_set1_pd(3.); - let r = _mm_mask_scalef_round_sd(a, 0, a, b, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); + let r = _mm_mask_scalef_round_sd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, 0, a, b, + ); let e = _mm_set_pd(1., 1.); assert_eq_m128d(r, e); - let r = _mm_mask_scalef_round_sd( - a, - 0b11111111, - a, - b, - _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC, + let r = _mm_mask_scalef_round_sd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, 0b11111111, a, b, ); let e = _mm_set_pd(1., 8.); assert_eq_m128d(r, e); @@ -54345,14 +51542,12 @@ mod tests { unsafe fn test_mm_maskz_scalef_round_sd() { let a = _mm_set1_pd(1.); let b = _mm_set1_pd(3.); - let r = _mm_maskz_scalef_round_sd(0, a, b, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); + let r = + _mm_maskz_scalef_round_sd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>(0, a, b); let e = _mm_set_pd(1., 0.); assert_eq_m128d(r, e); - let r = _mm_maskz_scalef_round_sd( - 0b11111111, - a, - b, - _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC, + let r = _mm_maskz_scalef_round_sd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + 0b11111111, a, b, ); let e = _mm_set_pd(1., 8.); assert_eq_m128d(r, e); @@ -54363,7 +51558,7 @@ mod tests { let a = _mm_set1_ps(1.); let b = _mm_set1_ps(2.); let c = _mm_set1_ps(3.); - let r = _mm_fmadd_round_ss(a, b, c, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); + let r = _mm_fmadd_round_ss::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>(a, b, c); let e = _mm_set_ps(1., 1., 1., 5.); assert_eq_m128(r, e); } @@ -54373,14 +51568,12 @@ mod tests { let a = _mm_set1_ps(1.); let b = _mm_set1_ps(2.); let c = _mm_set1_ps(3.); - let r = _mm_mask_fmadd_round_ss(a, 0, b, c, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); + let r = _mm_mask_fmadd_round_ss::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, 0, b, c, + ); assert_eq_m128(r, a); - let r = _mm_mask_fmadd_round_ss( - a, - 0b11111111, - b, - c, - _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC, + let r = _mm_mask_fmadd_round_ss::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, 0b11111111, b, c, ); let e = _mm_set_ps(1., 1., 1., 5.); assert_eq_m128(r, e); @@ -54391,15 +51584,13 @@ mod tests { let a = _mm_set1_ps(1.); let b = _mm_set1_ps(2.); let c = _mm_set1_ps(3.); - let r = _mm_maskz_fmadd_round_ss(0, a, b, c, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); + let r = _mm_maskz_fmadd_round_ss::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + 0, a, b, c, + ); let e = _mm_set_ps(1., 1., 1., 0.); assert_eq_m128(r, e); - let r = _mm_maskz_fmadd_round_ss( - 0b11111111, - a, - b, - c, - _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC, + let r = _mm_maskz_fmadd_round_ss::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + 0b11111111, a, b, c, ); let e = _mm_set_ps(1., 1., 1., 5.); assert_eq_m128(r, e); @@ -54410,14 +51601,12 @@ mod tests { let a = _mm_set1_ps(1.); let b = _mm_set1_ps(2.); let c = _mm_set1_ps(3.); - let r = _mm_mask3_fmadd_round_ss(a, b, c, 0, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); + let r = _mm_mask3_fmadd_round_ss::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, b, c, 0, + ); assert_eq_m128(r, c); - let r = _mm_mask3_fmadd_round_ss( - a, - b, - c, - 0b11111111, - _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC, + let r = _mm_mask3_fmadd_round_ss::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, b, c, 0b11111111, ); let e = _mm_set_ps(3., 3., 3., 5.); assert_eq_m128(r, e); @@ -54428,7 +51617,7 @@ mod tests { let a = _mm_set1_pd(1.); let b = _mm_set1_pd(2.); let c = _mm_set1_pd(3.); - let r = _mm_fmadd_round_sd(a, b, c, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); + let r = _mm_fmadd_round_sd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>(a, b, c); let e = _mm_set_pd(1., 5.); assert_eq_m128d(r, e); } @@ -54438,14 +51627,12 @@ mod tests { let a = _mm_set1_pd(1.); let b = _mm_set1_pd(2.); let c = _mm_set1_pd(3.); - let r = _mm_mask_fmadd_round_sd(a, 0, b, c, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); + let r = _mm_mask_fmadd_round_sd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, 0, b, c, + ); assert_eq_m128d(r, a); - let r = _mm_mask_fmadd_round_sd( - a, - 0b11111111, - b, - c, - _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC, + let r = _mm_mask_fmadd_round_sd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, 0b11111111, b, c, ); let e = _mm_set_pd(1., 5.); assert_eq_m128d(r, e); @@ -54456,15 +51643,13 @@ mod tests { let a = _mm_set1_pd(1.); let b = _mm_set1_pd(2.); let c = _mm_set1_pd(3.); - let r = _mm_maskz_fmadd_round_sd(0, a, b, c, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); + let r = _mm_maskz_fmadd_round_sd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + 0, a, b, c, + ); let e = _mm_set_pd(1., 0.); assert_eq_m128d(r, e); - let r = _mm_maskz_fmadd_round_sd( - 0b11111111, - a, - b, - c, - _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC, + let r = _mm_maskz_fmadd_round_sd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + 0b11111111, a, b, c, ); let e = _mm_set_pd(1., 5.); assert_eq_m128d(r, e); @@ -54475,14 +51660,12 @@ mod tests { let a = _mm_set1_pd(1.); let b = _mm_set1_pd(2.); let c = _mm_set1_pd(3.); - let r = _mm_mask3_fmadd_round_sd(a, b, c, 0, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); + let r = _mm_mask3_fmadd_round_sd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, b, c, 0, + ); assert_eq_m128d(r, c); - let r = _mm_mask3_fmadd_round_sd( - a, - b, - c, - 0b11111111, - _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC, + let r = _mm_mask3_fmadd_round_sd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, b, c, 0b11111111, ); let e = _mm_set_pd(3., 5.); assert_eq_m128d(r, e); @@ -54493,7 +51676,7 @@ mod tests { let a = _mm_set1_ps(1.); let b = _mm_set1_ps(2.); let c = _mm_set1_ps(3.); - let r = _mm_fmsub_round_ss(a, b, c, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); + let r = _mm_fmsub_round_ss::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>(a, b, c); let e = _mm_set_ps(1., 1., 1., -1.); assert_eq_m128(r, e); } @@ -54503,14 +51686,12 @@ mod tests { let a = _mm_set1_ps(1.); let b = _mm_set1_ps(2.); let c = _mm_set1_ps(3.); - let r = _mm_mask_fmsub_round_ss(a, 0, b, c, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); + let r = _mm_mask_fmsub_round_ss::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, 0, b, c, + ); assert_eq_m128(r, a); - let r = _mm_mask_fmsub_round_ss( - a, - 0b11111111, - b, - c, - _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC, + let r = _mm_mask_fmsub_round_ss::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, 0b11111111, b, c, ); let e = _mm_set_ps(1., 1., 1., -1.); assert_eq_m128(r, e); @@ -54521,15 +51702,13 @@ mod tests { let a = _mm_set1_ps(1.); let b = _mm_set1_ps(2.); let c = _mm_set1_ps(3.); - let r = _mm_maskz_fmsub_round_ss(0, a, b, c, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); + let r = _mm_maskz_fmsub_round_ss::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + 0, a, b, c, + ); let e = _mm_set_ps(1., 1., 1., 0.); assert_eq_m128(r, e); - let r = _mm_maskz_fmsub_round_ss( - 0b11111111, - a, - b, - c, - _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC, + let r = _mm_maskz_fmsub_round_ss::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + 0b11111111, a, b, c, ); let e = _mm_set_ps(1., 1., 1., -1.); assert_eq_m128(r, e); @@ -54540,14 +51719,12 @@ mod tests { let a = _mm_set1_ps(1.); let b = _mm_set1_ps(2.); let c = _mm_set1_ps(3.); - let r = _mm_mask3_fmsub_round_ss(a, b, c, 0, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); + let r = _mm_mask3_fmsub_round_ss::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, b, c, 0, + ); assert_eq_m128(r, c); - let r = _mm_mask3_fmsub_round_ss( - a, - b, - c, - 0b11111111, - _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC, + let r = _mm_mask3_fmsub_round_ss::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, b, c, 0b11111111, ); let e = _mm_set_ps(3., 3., 3., -1.); assert_eq_m128(r, e); @@ -54558,7 +51735,7 @@ mod tests { let a = _mm_set1_pd(1.); let b = _mm_set1_pd(2.); let c = _mm_set1_pd(3.); - let r = _mm_fmsub_round_sd(a, b, c, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); + let r = _mm_fmsub_round_sd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>(a, b, c); let e = _mm_set_pd(1., -1.); assert_eq_m128d(r, e); } @@ -54568,14 +51745,12 @@ mod tests { let a = _mm_set1_pd(1.); let b = _mm_set1_pd(2.); let c = _mm_set1_pd(3.); - let r = _mm_mask_fmsub_round_sd(a, 0, b, c, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); + let r = _mm_mask_fmsub_round_sd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, 0, b, c, + ); assert_eq_m128d(r, a); - let r = _mm_mask_fmsub_round_sd( - a, - 0b11111111, - b, - c, - _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC, + let r = _mm_mask_fmsub_round_sd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, 0b11111111, b, c, ); let e = _mm_set_pd(1., -1.); assert_eq_m128d(r, e); @@ -54586,15 +51761,13 @@ mod tests { let a = _mm_set1_pd(1.); let b = _mm_set1_pd(2.); let c = _mm_set1_pd(3.); - let r = _mm_maskz_fmsub_round_sd(0, a, b, c, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); + let r = _mm_maskz_fmsub_round_sd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + 0, a, b, c, + ); let e = _mm_set_pd(1., 0.); assert_eq_m128d(r, e); - let r = _mm_maskz_fmsub_round_sd( - 0b11111111, - a, - b, - c, - _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC, + let r = _mm_maskz_fmsub_round_sd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + 0b11111111, a, b, c, ); let e = _mm_set_pd(1., -1.); assert_eq_m128d(r, e); @@ -54605,14 +51778,12 @@ mod tests { let a = _mm_set1_pd(1.); let b = _mm_set1_pd(2.); let c = _mm_set1_pd(3.); - let r = _mm_mask3_fmsub_round_sd(a, b, c, 0, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); + let r = _mm_mask3_fmsub_round_sd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, b, c, 0, + ); assert_eq_m128d(r, c); - let r = _mm_mask3_fmsub_round_sd( - a, - b, - c, - 0b11111111, - _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC, + let r = _mm_mask3_fmsub_round_sd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, b, c, 0b11111111, ); let e = _mm_set_pd(3., -1.); assert_eq_m128d(r, e); @@ -54623,7 +51794,7 @@ mod tests { let a = _mm_set1_ps(1.); let b = _mm_set1_ps(2.); let c = _mm_set1_ps(3.); - let r = _mm_fnmadd_round_ss(a, b, c, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); + let r = _mm_fnmadd_round_ss::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>(a, b, c); let e = _mm_set_ps(1., 1., 1., 1.); assert_eq_m128(r, e); } @@ -54633,14 +51804,12 @@ mod tests { let a = _mm_set1_ps(1.); let b = _mm_set1_ps(2.); let c = _mm_set1_ps(3.); - let r = _mm_mask_fnmadd_round_ss(a, 0, b, c, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); + let r = _mm_mask_fnmadd_round_ss::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, 0, b, c, + ); assert_eq_m128(r, a); - let r = _mm_mask_fnmadd_round_ss( - a, - 0b11111111, - b, - c, - _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC, + let r = _mm_mask_fnmadd_round_ss::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, 0b11111111, b, c, ); let e = _mm_set_ps(1., 1., 1., 1.); assert_eq_m128(r, e); @@ -54651,16 +51820,13 @@ mod tests { let a = _mm_set1_ps(1.); let b = _mm_set1_ps(2.); let c = _mm_set1_ps(3.); - let r = - _mm_maskz_fnmadd_round_ss(0, a, b, c, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); + let r = _mm_maskz_fnmadd_round_ss::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + 0, a, b, c, + ); let e = _mm_set_ps(1., 1., 1., 0.); assert_eq_m128(r, e); - let r = _mm_maskz_fnmadd_round_ss( - 0b11111111, - a, - b, - c, - _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC, + let r = _mm_maskz_fnmadd_round_ss::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + 0b11111111, a, b, c, ); let e = _mm_set_ps(1., 1., 1., 1.); assert_eq_m128(r, e); @@ -54671,15 +51837,12 @@ mod tests { let a = _mm_set1_ps(1.); let b = _mm_set1_ps(2.); let c = _mm_set1_ps(3.); - let r = - _mm_mask3_fnmadd_round_ss(a, b, c, 0, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); + let r = _mm_mask3_fnmadd_round_ss::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, b, c, 0, + ); assert_eq_m128(r, c); - let r = _mm_mask3_fnmadd_round_ss( - a, - b, - c, - 0b11111111, - _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC, + let r = _mm_mask3_fnmadd_round_ss::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, b, c, 0b11111111, ); let e = _mm_set_ps(3., 3., 3., 1.); assert_eq_m128(r, e); @@ -54690,7 +51853,7 @@ mod tests { let a = _mm_set1_pd(1.); let b = _mm_set1_pd(2.); let c = _mm_set1_pd(3.); - let r = _mm_fnmadd_round_sd(a, b, c, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); + let r = _mm_fnmadd_round_sd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>(a, b, c); let e = _mm_set_pd(1., 1.); assert_eq_m128d(r, e); } @@ -54700,14 +51863,12 @@ mod tests { let a = _mm_set1_pd(1.); let b = _mm_set1_pd(2.); let c = _mm_set1_pd(3.); - let r = _mm_mask_fnmadd_round_sd(a, 0, b, c, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); + let r = _mm_mask_fnmadd_round_sd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, 0, b, c, + ); assert_eq_m128d(r, a); - let r = _mm_mask_fnmadd_round_sd( - a, - 0b11111111, - b, - c, - _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC, + let r = _mm_mask_fnmadd_round_sd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, 0b11111111, b, c, ); let e = _mm_set_pd(1., 1.); assert_eq_m128d(r, e); @@ -54718,16 +51879,13 @@ mod tests { let a = _mm_set1_pd(1.); let b = _mm_set1_pd(2.); let c = _mm_set1_pd(3.); - let r = - _mm_maskz_fnmadd_round_sd(0, a, b, c, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); + let r = _mm_maskz_fnmadd_round_sd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + 0, a, b, c, + ); let e = _mm_set_pd(1., 0.); assert_eq_m128d(r, e); - let r = _mm_maskz_fnmadd_round_sd( - 0b11111111, - a, - b, - c, - _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC, + let r = _mm_maskz_fnmadd_round_sd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + 0b11111111, a, b, c, ); let e = _mm_set_pd(1., 1.); assert_eq_m128d(r, e); @@ -54738,15 +51896,12 @@ mod tests { let a = _mm_set1_pd(1.); let b = _mm_set1_pd(2.); let c = _mm_set1_pd(3.); - let r = - _mm_mask3_fnmadd_round_sd(a, b, c, 0, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); + let r = _mm_mask3_fnmadd_round_sd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, b, c, 0, + ); assert_eq_m128d(r, c); - let r = _mm_mask3_fnmadd_round_sd( - a, - b, - c, - 0b11111111, - _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC, + let r = _mm_mask3_fnmadd_round_sd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, b, c, 0b11111111, ); let e = _mm_set_pd(3., 1.); assert_eq_m128d(r, e); @@ -54757,7 +51912,7 @@ mod tests { let a = _mm_set1_ps(1.); let b = _mm_set1_ps(2.); let c = _mm_set1_ps(3.); - let r = _mm_fnmsub_round_ss(a, b, c, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); + let r = _mm_fnmsub_round_ss::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>(a, b, c); let e = _mm_set_ps(1., 1., 1., -5.); assert_eq_m128(r, e); } @@ -54767,14 +51922,12 @@ mod tests { let a = _mm_set1_ps(1.); let b = _mm_set1_ps(2.); let c = _mm_set1_ps(3.); - let r = _mm_mask_fnmsub_round_ss(a, 0, b, c, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); + let r = _mm_mask_fnmsub_round_ss::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, 0, b, c, + ); assert_eq_m128(r, a); - let r = _mm_mask_fnmsub_round_ss( - a, - 0b11111111, - b, - c, - _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC, + let r = _mm_mask_fnmsub_round_ss::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, 0b11111111, b, c, ); let e = _mm_set_ps(1., 1., 1., -5.); assert_eq_m128(r, e); @@ -54785,16 +51938,13 @@ mod tests { let a = _mm_set1_ps(1.); let b = _mm_set1_ps(2.); let c = _mm_set1_ps(3.); - let r = - _mm_maskz_fnmsub_round_ss(0, a, b, c, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); + let r = _mm_maskz_fnmsub_round_ss::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + 0, a, b, c, + ); let e = _mm_set_ps(1., 1., 1., 0.); assert_eq_m128(r, e); - let r = _mm_maskz_fnmsub_round_ss( - 0b11111111, - a, - b, - c, - _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC, + let r = _mm_maskz_fnmsub_round_ss::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + 0b11111111, a, b, c, ); let e = _mm_set_ps(1., 1., 1., -5.); assert_eq_m128(r, e); @@ -54805,15 +51955,12 @@ mod tests { let a = _mm_set1_ps(1.); let b = _mm_set1_ps(2.); let c = _mm_set1_ps(3.); - let r = - _mm_mask3_fnmsub_round_ss(a, b, c, 0, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); + let r = _mm_mask3_fnmsub_round_ss::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, b, c, 0, + ); assert_eq_m128(r, c); - let r = _mm_mask3_fnmsub_round_ss( - a, - b, - c, - 0b11111111, - _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC, + let r = _mm_mask3_fnmsub_round_ss::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, b, c, 0b11111111, ); let e = _mm_set_ps(3., 3., 3., -5.); assert_eq_m128(r, e); @@ -54824,7 +51971,7 @@ mod tests { let a = _mm_set1_pd(1.); let b = _mm_set1_pd(2.); let c = _mm_set1_pd(3.); - let r = _mm_fnmsub_round_sd(a, b, c, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); + let r = _mm_fnmsub_round_sd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>(a, b, c); let e = _mm_set_pd(1., -5.); assert_eq_m128d(r, e); } @@ -54834,14 +51981,12 @@ mod tests { let a = _mm_set1_pd(1.); let b = _mm_set1_pd(2.); let c = _mm_set1_pd(3.); - let r = _mm_mask_fnmsub_round_sd(a, 0, b, c, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); + let r = _mm_mask_fnmsub_round_sd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, 0, b, c, + ); assert_eq_m128d(r, a); - let r = _mm_mask_fnmsub_round_sd( - a, - 0b11111111, - b, - c, - _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC, + let r = _mm_mask_fnmsub_round_sd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, 0b11111111, b, c, ); let e = _mm_set_pd(1., -5.); assert_eq_m128d(r, e); @@ -54852,16 +51997,13 @@ mod tests { let a = _mm_set1_pd(1.); let b = _mm_set1_pd(2.); let c = _mm_set1_pd(3.); - let r = - _mm_maskz_fnmsub_round_sd(0, a, b, c, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); + let r = _mm_maskz_fnmsub_round_sd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + 0, a, b, c, + ); let e = _mm_set_pd(1., 0.); assert_eq_m128d(r, e); - let r = _mm_maskz_fnmsub_round_sd( - 0b11111111, - a, - b, - c, - _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC, + let r = _mm_maskz_fnmsub_round_sd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + 0b11111111, a, b, c, ); let e = _mm_set_pd(1., -5.); assert_eq_m128d(r, e); @@ -54872,15 +52014,12 @@ mod tests { let a = _mm_set1_pd(1.); let b = _mm_set1_pd(2.); let c = _mm_set1_pd(3.); - let r = - _mm_mask3_fnmsub_round_sd(a, b, c, 0, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); + let r = _mm_mask3_fnmsub_round_sd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, b, c, 0, + ); assert_eq_m128d(r, c); - let r = _mm_mask3_fnmsub_round_sd( - a, - b, - c, - 0b11111111, - _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC, + let r = _mm_mask3_fnmsub_round_sd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, b, c, 0b11111111, ); let e = _mm_set_pd(3., -5.); assert_eq_m128d(r, e); @@ -54891,7 +52030,7 @@ mod tests { let a = _mm_set_ps(0., 0., 0., f32::NAN); let b = _mm_set1_ps(f32::MAX); let c = _mm_set1_epi32(i32::MAX); - let r = _mm_fixupimm_ss(a, b, c, 5); + let r = _mm_fixupimm_ss::<5>(a, b, c); let e = _mm_set_ps(0., 0., 0., -0.0); assert_eq_m128(r, e); } @@ -54901,7 +52040,7 @@ mod tests { let a = _mm_set_ps(0., 0., 0., f32::NAN); let b = _mm_set1_ps(f32::MAX); let c = _mm_set1_epi32(i32::MAX); - let r = _mm_mask_fixupimm_ss(a, 0b11111111, b, c, 5); + let r = _mm_mask_fixupimm_ss::<5>(a, 0b11111111, b, c); let e = _mm_set_ps(0., 0., 0., -0.0); assert_eq_m128(r, e); } @@ -54911,10 +52050,10 @@ mod tests { let a = _mm_set_ps(0., 0., 0., f32::NAN); let b = _mm_set1_ps(f32::MAX); let c = _mm_set1_epi32(i32::MAX); - let r = _mm_maskz_fixupimm_ss(0b00000000, a, b, c, 5); + let r = _mm_maskz_fixupimm_ss::<5>(0b00000000, a, b, c); let e = _mm_set_ps(0., 0., 0., 0.0); assert_eq_m128(r, e); - let r = _mm_maskz_fixupimm_ss(0b11111111, a, b, c, 5); + let r = _mm_maskz_fixupimm_ss::<5>(0b11111111, a, b, c); let e = _mm_set_ps(0., 0., 0., -0.0); assert_eq_m128(r, e); } @@ -54924,7 +52063,7 @@ mod tests { let a = _mm_set_pd(0., f64::NAN); let b = _mm_set1_pd(f64::MAX); let c = _mm_set1_epi64x(i32::MAX as i64); - let r = _mm_fixupimm_sd(a, b, c, 5); + let r = _mm_fixupimm_sd::<5>(a, b, c); let e = _mm_set_pd(0., -0.0); assert_eq_m128d(r, e); } @@ -54934,7 +52073,7 @@ mod tests { let a = _mm_set_pd(0., f64::NAN); let b = _mm_set1_pd(f64::MAX); let c = _mm_set1_epi64x(i32::MAX as i64); - let r = _mm_mask_fixupimm_sd(a, 0b11111111, b, c, 5); + let r = _mm_mask_fixupimm_sd::<5>(a, 0b11111111, b, c); let e = _mm_set_pd(0., -0.0); assert_eq_m128d(r, e); } @@ -54944,21 +52083,21 @@ mod tests { let a = _mm_set_pd(0., f64::NAN); let b = _mm_set1_pd(f64::MAX); let c = _mm_set1_epi64x(i32::MAX as i64); - let r = _mm_maskz_fixupimm_sd(0b00000000, a, b, c, 5); + let r = _mm_maskz_fixupimm_sd::<5>(0b00000000, a, b, c); let e = _mm_set_pd(0., 0.0); assert_eq_m128d(r, e); - let r = _mm_maskz_fixupimm_sd(0b11111111, a, b, c, 5); + let r = _mm_maskz_fixupimm_sd::<5>(0b11111111, a, b, c); let e = _mm_set_pd(0., -0.0); assert_eq_m128d(r, e); } #[simd_test(enable = "avx512f")] unsafe fn test_mm_fixupimm_round_ss() { - let a = _mm_set_ps(0., 0., 0., f32::NAN); + let a = _mm_set_ps(1., 0., 0., f32::NAN); let b = _mm_set1_ps(f32::MAX); let c = _mm_set1_epi32(i32::MAX); - let r = _mm_fixupimm_round_ss(a, b, c, 5, _MM_FROUND_CUR_DIRECTION); - let e = _mm_set_ps(0., 0., 0., -0.0); + let r = _mm_fixupimm_round_ss::<5, _MM_FROUND_CUR_DIRECTION>(a, b, c); + let e = _mm_set_ps(1., 0., 0., -0.0); assert_eq_m128(r, e); } @@ -54967,7 +52106,7 @@ mod tests { let a = _mm_set_ps(0., 0., 0., f32::NAN); let b = _mm_set1_ps(f32::MAX); let c = _mm_set1_epi32(i32::MAX); - let r = _mm_mask_fixupimm_round_ss(a, 0b11111111, b, c, 5, _MM_FROUND_CUR_DIRECTION); + let r = _mm_mask_fixupimm_round_ss::<5, _MM_FROUND_CUR_DIRECTION>(a, 0b11111111, b, c); let e = _mm_set_ps(0., 0., 0., -0.0); assert_eq_m128(r, e); } @@ -54977,10 +52116,10 @@ mod tests { let a = _mm_set_ps(0., 0., 0., f32::NAN); let b = _mm_set1_ps(f32::MAX); let c = _mm_set1_epi32(i32::MAX); - let r = _mm_maskz_fixupimm_round_ss(0b00000000, a, b, c, 5, _MM_FROUND_CUR_DIRECTION); + let r = _mm_maskz_fixupimm_round_ss::<5, _MM_FROUND_CUR_DIRECTION>(0b00000000, a, b, c); let e = _mm_set_ps(0., 0., 0., 0.0); assert_eq_m128(r, e); - let r = _mm_maskz_fixupimm_round_ss(0b11111111, a, b, c, 5, _MM_FROUND_CUR_DIRECTION); + let r = _mm_maskz_fixupimm_round_ss::<5, _MM_FROUND_CUR_DIRECTION>(0b11111111, a, b, c); let e = _mm_set_ps(0., 0., 0., -0.0); assert_eq_m128(r, e); } @@ -54990,7 +52129,7 @@ mod tests { let a = _mm_set_pd(0., f64::NAN); let b = _mm_set1_pd(f64::MAX); let c = _mm_set1_epi64x(i32::MAX as i64); - let r = _mm_fixupimm_round_sd(a, b, c, 5, _MM_FROUND_CUR_DIRECTION); + let r = _mm_fixupimm_round_sd::<5, _MM_FROUND_CUR_DIRECTION>(a, b, c); let e = _mm_set_pd(0., -0.0); assert_eq_m128d(r, e); } @@ -55000,7 +52139,7 @@ mod tests { let a = _mm_set_pd(0., f64::NAN); let b = _mm_set1_pd(f64::MAX); let c = _mm_set1_epi64x(i32::MAX as i64); - let r = _mm_mask_fixupimm_round_sd(a, 0b11111111, b, c, 5, _MM_FROUND_CUR_DIRECTION); + let r = _mm_mask_fixupimm_round_sd::<5, _MM_FROUND_CUR_DIRECTION>(a, 0b11111111, b, c); let e = _mm_set_pd(0., -0.0); assert_eq_m128d(r, e); } @@ -55010,10 +52149,10 @@ mod tests { let a = _mm_set_pd(0., f64::NAN); let b = _mm_set1_pd(f64::MAX); let c = _mm_set1_epi64x(i32::MAX as i64); - let r = _mm_maskz_fixupimm_round_sd(0b00000000, a, b, c, 5, _MM_FROUND_CUR_DIRECTION); + let r = _mm_maskz_fixupimm_round_sd::<5, _MM_FROUND_CUR_DIRECTION>(0b00000000, a, b, c); let e = _mm_set_pd(0., 0.0); assert_eq_m128d(r, e); - let r = _mm_maskz_fixupimm_round_sd(0b11111111, a, b, c, 5, _MM_FROUND_CUR_DIRECTION); + let r = _mm_maskz_fixupimm_round_sd::<5, _MM_FROUND_CUR_DIRECTION>(0b11111111, a, b, c); let e = _mm_set_pd(0., -0.0); assert_eq_m128d(r, e); } @@ -55068,7 +52207,7 @@ mod tests { unsafe fn test_mm_cvt_roundss_sd() { let a = _mm_set_pd(6., -7.5); let b = _mm_set_ps(0., -0.5, 1., -1.5); - let r = _mm_cvt_roundss_sd(a, b, _MM_FROUND_CUR_DIRECTION); + let r = _mm_cvt_roundss_sd::<_MM_FROUND_CUR_DIRECTION>(a, b); let e = _mm_set_pd(6., -1.5); assert_eq_m128d(r, e); } @@ -55077,9 +52216,9 @@ mod tests { unsafe fn test_mm_mask_cvt_roundss_sd() { let a = _mm_set_pd(6., -7.5); let b = _mm_set_ps(0., -0.5, 1., -1.5); - let r = _mm_mask_cvt_roundss_sd(a, 0, a, b, _MM_FROUND_CUR_DIRECTION); + let r = _mm_mask_cvt_roundss_sd::<_MM_FROUND_CUR_DIRECTION>(a, 0, a, b); assert_eq_m128d(r, a); - let r = _mm_mask_cvt_roundss_sd(a, 0b11111111, a, b, _MM_FROUND_CUR_DIRECTION); + let r = _mm_mask_cvt_roundss_sd::<_MM_FROUND_CUR_DIRECTION>(a, 0b11111111, a, b); let e = _mm_set_pd(6., -1.5); assert_eq_m128d(r, e); } @@ -55088,10 +52227,10 @@ mod tests { unsafe fn test_mm_maskz_cvt_roundss_sd() { let a = _mm_set_pd(6., -7.5); let b = _mm_set_ps(0., -0.5, 1., -1.5); - let r = _mm_maskz_cvt_roundss_sd(0, a, b, _MM_FROUND_CUR_DIRECTION); + let r = _mm_maskz_cvt_roundss_sd::<_MM_FROUND_CUR_DIRECTION>(0, a, b); let e = _mm_set_pd(6., 0.); assert_eq_m128d(r, e); - let r = _mm_maskz_cvt_roundss_sd(0b11111111, a, b, _MM_FROUND_CUR_DIRECTION); + let r = _mm_maskz_cvt_roundss_sd::<_MM_FROUND_CUR_DIRECTION>(0b11111111, a, b); let e = _mm_set_pd(6., -1.5); assert_eq_m128d(r, e); } @@ -55100,7 +52239,7 @@ mod tests { unsafe fn test_mm_cvt_roundsd_ss() { let a = _mm_set_ps(0., -0.5, 1., -1.5); let b = _mm_set_pd(6., -7.5); - let r = _mm_cvt_roundsd_ss(a, b, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); + let r = _mm_cvt_roundsd_ss::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(a, b); let e = _mm_set_ps(0., -0.5, 1., -7.5); assert_eq_m128(r, e); } @@ -55109,10 +52248,11 @@ mod tests { unsafe fn test_mm_mask_cvt_roundsd_ss() { let a = _mm_set_ps(0., -0.5, 1., -1.5); let b = _mm_set_pd(6., -7.5); - let r = _mm_mask_cvt_roundsd_ss(a, 0, a, b, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); + let r = _mm_mask_cvt_roundsd_ss::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(a, 0, a, b); assert_eq_m128(r, a); - let r = - _mm_mask_cvt_roundsd_ss(a, 0b11111111, a, b, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); + let r = _mm_mask_cvt_roundsd_ss::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>( + a, 0b11111111, a, b, + ); let e = _mm_set_ps(0., -0.5, 1., -7.5); assert_eq_m128(r, e); } @@ -55121,10 +52261,12 @@ mod tests { unsafe fn test_mm_maskz_cvt_roundsd_ss() { let a = _mm_set_ps(0., -0.5, 1., -1.5); let b = _mm_set_pd(6., -7.5); - let r = _mm_maskz_cvt_roundsd_ss(0, a, b, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); + let r = _mm_maskz_cvt_roundsd_ss::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(0, a, b); let e = _mm_set_ps(0., -0.5, 1., 0.); assert_eq_m128(r, e); - let r = _mm_maskz_cvt_roundsd_ss(0b11111111, a, b, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); + let r = _mm_maskz_cvt_roundsd_ss::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>( + 0b11111111, a, b, + ); let e = _mm_set_ps(0., -0.5, 1., -7.5); assert_eq_m128(r, e); } @@ -55132,7 +52274,7 @@ mod tests { #[simd_test(enable = "avx512f")] unsafe fn test_mm_cvt_roundss_si32() { let a = _mm_set_ps(0., -0.5, 1., -1.5); - let r = _mm_cvt_roundss_si32(a, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); + let r = _mm_cvt_roundss_si32::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(a); let e: i32 = -1; assert_eq!(r, e); } @@ -55140,7 +52282,7 @@ mod tests { #[simd_test(enable = "avx512f")] unsafe fn test_mm_cvt_roundss_i32() { let a = _mm_set_ps(0., -0.5, 1., -1.5); - let r = _mm_cvt_roundss_i32(a, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); + let r = _mm_cvt_roundss_i32::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(a); let e: i32 = -1; assert_eq!(r, e); } @@ -55148,7 +52290,7 @@ mod tests { #[simd_test(enable = "avx512f")] unsafe fn test_mm_cvt_roundss_u32() { let a = _mm_set_ps(0., -0.5, 1., -1.5); - let r = _mm_cvt_roundss_u32(a, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); + let r = _mm_cvt_roundss_u32::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(a); let e: u32 = u32::MAX; assert_eq!(r, e); } @@ -55172,7 +52314,7 @@ mod tests { #[simd_test(enable = "avx512f")] unsafe fn test_mm_cvt_roundsd_si32() { let a = _mm_set_pd(1., -1.5); - let r = _mm_cvt_roundsd_si32(a, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); + let r = _mm_cvt_roundsd_si32::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(a); let e: i32 = -1; assert_eq!(r, e); } @@ -55180,7 +52322,7 @@ mod tests { #[simd_test(enable = "avx512f")] unsafe fn test_mm_cvt_roundsd_i32() { let a = _mm_set_pd(1., -1.5); - let r = _mm_cvt_roundsd_i32(a, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); + let r = _mm_cvt_roundsd_i32::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(a); let e: i32 = -1; assert_eq!(r, e); } @@ -55188,7 +52330,7 @@ mod tests { #[simd_test(enable = "avx512f")] unsafe fn test_mm_cvt_roundsd_u32() { let a = _mm_set_pd(1., -1.5); - let r = _mm_cvt_roundsd_u32(a, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); + let r = _mm_cvt_roundsd_u32::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(a); let e: u32 = u32::MAX; assert_eq!(r, e); } @@ -55213,7 +52355,7 @@ mod tests { unsafe fn test_mm_cvt_roundi32_ss() { let a = _mm_set_ps(0., -0.5, 1., -1.5); let b: i32 = 9; - let r = _mm_cvt_roundi32_ss(a, b, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); + let r = _mm_cvt_roundi32_ss::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(a, b); let e = _mm_set_ps(0., -0.5, 1., 9.); assert_eq_m128(r, e); } @@ -55222,7 +52364,7 @@ mod tests { unsafe fn test_mm_cvt_roundsi32_ss() { let a = _mm_set_ps(0., -0.5, 1., -1.5); let b: i32 = 9; - let r = _mm_cvt_roundsi32_ss(a, b, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); + let r = _mm_cvt_roundsi32_ss::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(a, b); let e = _mm_set_ps(0., -0.5, 1., 9.); assert_eq_m128(r, e); } @@ -55231,7 +52373,7 @@ mod tests { unsafe fn test_mm_cvt_roundu32_ss() { let a = _mm_set_ps(0., -0.5, 1., -1.5); let b: u32 = 9; - let r = _mm_cvt_roundu32_ss(a, b, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); + let r = _mm_cvt_roundu32_ss::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(a, b); let e = _mm_set_ps(0., -0.5, 1., 9.); assert_eq_m128(r, e); } @@ -55257,7 +52399,7 @@ mod tests { #[simd_test(enable = "avx512f")] unsafe fn test_mm_cvtt_roundss_si32() { let a = _mm_set_ps(0., -0.5, 1., -1.5); - let r = _mm_cvtt_roundss_si32(a, _MM_FROUND_CUR_DIRECTION); + let r = _mm_cvtt_roundss_si32::<_MM_FROUND_CUR_DIRECTION>(a); let e: i32 = -2; assert_eq!(r, e); } @@ -55265,7 +52407,7 @@ mod tests { #[simd_test(enable = "avx512f")] unsafe fn test_mm_cvtt_roundss_i32() { let a = _mm_set_ps(0., -0.5, 1., -1.5); - let r = _mm_cvtt_roundss_i32(a, _MM_FROUND_CUR_DIRECTION); + let r = _mm_cvtt_roundss_i32::<_MM_FROUND_CUR_DIRECTION>(a); let e: i32 = -2; assert_eq!(r, e); } @@ -55273,7 +52415,7 @@ mod tests { #[simd_test(enable = "avx512f")] unsafe fn test_mm_cvtt_roundss_u32() { let a = _mm_set_ps(0., -0.5, 1., -1.5); - let r = _mm_cvtt_roundss_u32(a, _MM_FROUND_CUR_DIRECTION); + let r = _mm_cvtt_roundss_u32::<_MM_FROUND_CUR_DIRECTION>(a); let e: u32 = u32::MAX; assert_eq!(r, e); } @@ -55297,7 +52439,7 @@ mod tests { #[simd_test(enable = "avx512f")] unsafe fn test_mm_cvtt_roundsd_si32() { let a = _mm_set_pd(1., -1.5); - let r = _mm_cvtt_roundsd_si32(a, _MM_FROUND_CUR_DIRECTION); + let r = _mm_cvtt_roundsd_si32::<_MM_FROUND_CUR_DIRECTION>(a); let e: i32 = -2; assert_eq!(r, e); } @@ -55305,7 +52447,7 @@ mod tests { #[simd_test(enable = "avx512f")] unsafe fn test_mm_cvtt_roundsd_i32() { let a = _mm_set_pd(1., -1.5); - let r = _mm_cvtt_roundsd_i32(a, _MM_FROUND_CUR_DIRECTION); + let r = _mm_cvtt_roundsd_i32::<_MM_FROUND_CUR_DIRECTION>(a); let e: i32 = -2; assert_eq!(r, e); } @@ -55313,7 +52455,7 @@ mod tests { #[simd_test(enable = "avx512f")] unsafe fn test_mm_cvtt_roundsd_u32() { let a = _mm_set_pd(1., -1.5); - let r = _mm_cvtt_roundsd_u32(a, _MM_FROUND_CUR_DIRECTION); + let r = _mm_cvtt_roundsd_u32::<_MM_FROUND_CUR_DIRECTION>(a); let e: u32 = u32::MAX; assert_eq!(r, e); } @@ -55356,7 +52498,7 @@ mod tests { unsafe fn test_mm_comi_round_ss() { let a = _mm_set1_ps(2.2); let b = _mm_set1_ps(1.1); - let r = _mm_comi_round_ss(a, b, 0, _MM_FROUND_CUR_DIRECTION); + let r = _mm_comi_round_ss::<0, _MM_FROUND_CUR_DIRECTION>(a, b); let e: i32 = 0; assert_eq!(r, e); } @@ -55365,7 +52507,7 @@ mod tests { unsafe fn test_mm_comi_round_sd() { let a = _mm_set1_pd(2.2); let b = _mm_set1_pd(1.1); - let r = _mm_comi_round_sd(a, b, 0, _MM_FROUND_CUR_DIRECTION); + let r = _mm_comi_round_sd::<0, _MM_FROUND_CUR_DIRECTION>(a, b); let e: i32 = 0; assert_eq!(r, e); } @@ -55377,4 +52519,35 @@ mod tests { let e: i32 = 1; assert_eq!(r, e); } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_shuffle_pd() { + let a = _mm512_setr_pd(1., 4., 5., 8., 1., 4., 5., 8.); + let b = _mm512_setr_pd(2., 3., 6., 7., 2., 3., 6., 7.); + let r = _mm512_shuffle_pd::<0b11_11_11_11>(a, b); + let e = _mm512_setr_pd(4., 3., 8., 7., 4., 3., 8., 7.); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_mask_shuffle_pd() { + let a = _mm512_setr_pd(1., 4., 5., 8., 1., 4., 5., 8.); + let b = _mm512_setr_pd(2., 3., 6., 7., 2., 3., 6., 7.); + let r = _mm512_mask_shuffle_pd::<0b11_11_11_11>(a, 0, a, b); + assert_eq_m512d(r, a); + let r = _mm512_mask_shuffle_pd::<0b11_11_11_11>(a, 0b11111111, a, b); + let e = _mm512_setr_pd(4., 3., 8., 7., 4., 3., 8., 7.); + assert_eq_m512d(r, e); + } + + #[simd_test(enable = "avx512f")] + unsafe fn test_mm512_maskz_shuffle_pd() { + let a = _mm512_setr_pd(1., 4., 5., 8., 1., 4., 5., 8.); + let b = _mm512_setr_pd(2., 3., 6., 7., 2., 3., 6., 7.); + let r = _mm512_maskz_shuffle_pd::<0b11_11_11_11>(0, a, b); + assert_eq_m512d(r, _mm512_setzero_pd()); + let r = _mm512_maskz_shuffle_pd::<0b11_11_11_11>(0b00001111, a, b); + let e = _mm512_setr_pd(4., 3., 8., 7., 0., 0., 0., 0.); + assert_eq_m512d(r, e); + } } diff --git a/library/stdarch/crates/core_arch/src/x86/avx512gfni.rs b/library/stdarch/crates/core_arch/src/x86/avx512gfni.rs index 6647601f09..7b6d796237 100644 --- a/library/stdarch/crates/core_arch/src/x86/avx512gfni.rs +++ b/library/stdarch/crates/core_arch/src/x86/avx512gfni.rs @@ -235,18 +235,14 @@ pub unsafe fn _mm_maskz_gf2p8mul_epi8(k: __mmask16, a: __m128i, b: __m128i) -> _ /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_gf2p8affine_epi8) #[inline] #[target_feature(enable = "avx512gfni,avx512bw,avx512f")] -#[cfg_attr(test, assert_instr(vgf2p8affineqb, b = 0))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm512_gf2p8affine_epi64_epi8(x: __m512i, a: __m512i, b: i32) -> __m512i { - assert!(0 <= b && b < 256); +#[cfg_attr(test, assert_instr(vgf2p8affineqb, B = 0))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_gf2p8affine_epi64_epi8(x: __m512i, a: __m512i) -> __m512i { + static_assert_imm8!(B); + let b = B as u8; let x = x.as_i8x64(); let a = a.as_i8x64(); - macro_rules! call { - ($imm8:expr) => { - vgf2p8affineqb_512(x, a, $imm8) - }; - } - let r = constify_imm8_sae!(b, call); + let r = vgf2p8affineqb_512(x, a, b); transmute(r) } @@ -261,24 +257,19 @@ pub unsafe fn _mm512_gf2p8affine_epi64_epi8(x: __m512i, a: __m512i, b: i32) -> _ /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_gf2p8affine_epi8) #[inline] #[target_feature(enable = "avx512gfni,avx512bw,avx512f")] -#[cfg_attr(test, assert_instr(vgf2p8affineqb, b = 0))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm512_maskz_gf2p8affine_epi64_epi8( +#[cfg_attr(test, assert_instr(vgf2p8affineqb, B = 0))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_maskz_gf2p8affine_epi64_epi8( k: __mmask64, x: __m512i, a: __m512i, - b: i32, ) -> __m512i { + static_assert_imm8!(B); + let b = B as u8; let zero = _mm512_setzero_si512().as_i8x64(); - assert!(0 <= b && b < 256); let x = x.as_i8x64(); let a = a.as_i8x64(); - macro_rules! call { - ($imm8:expr) => { - vgf2p8affineqb_512(x, a, $imm8) - }; - } - let r = constify_imm8_sae!(b, call); + let r = vgf2p8affineqb_512(x, a, b); transmute(simd_select_bitmask(k, r, zero)) } @@ -293,24 +284,19 @@ pub unsafe fn _mm512_maskz_gf2p8affine_epi64_epi8( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_gf2p8affine_epi8) #[inline] #[target_feature(enable = "avx512gfni,avx512bw,avx512f")] -#[cfg_attr(test, assert_instr(vgf2p8affineqb, b = 0))] -#[rustc_args_required_const(4)] -pub unsafe fn _mm512_mask_gf2p8affine_epi64_epi8( +#[cfg_attr(test, assert_instr(vgf2p8affineqb, B = 0))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_mask_gf2p8affine_epi64_epi8( src: __m512i, k: __mmask64, x: __m512i, a: __m512i, - b: i32, ) -> __m512i { - assert!(0 <= b && b < 256); + static_assert_imm8!(B); + let b = B as u8; let x = x.as_i8x64(); let a = a.as_i8x64(); - macro_rules! call { - ($imm8:expr) => { - vgf2p8affineqb_512(x, a, $imm8) - }; - } - let r = constify_imm8_sae!(b, call); + let r = vgf2p8affineqb_512(x, a, b); transmute(simd_select_bitmask(k, r, src.as_i8x64())) } @@ -322,18 +308,14 @@ pub unsafe fn _mm512_mask_gf2p8affine_epi64_epi8( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_gf2p8affine_epi8) #[inline] #[target_feature(enable = "avx512gfni,avx512bw,avx512vl")] -#[cfg_attr(test, assert_instr(vgf2p8affineqb, b = 0))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm256_gf2p8affine_epi64_epi8(x: __m256i, a: __m256i, b: i32) -> __m256i { - assert!(0 <= b && b < 256); +#[cfg_attr(test, assert_instr(vgf2p8affineqb, B = 0))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm256_gf2p8affine_epi64_epi8(x: __m256i, a: __m256i) -> __m256i { + static_assert_imm8!(B); + let b = B as u8; let x = x.as_i8x32(); let a = a.as_i8x32(); - macro_rules! call { - ($imm8:expr) => { - vgf2p8affineqb_256(x, a, $imm8) - }; - } - let r = constify_imm8_sae!(b, call); + let r = vgf2p8affineqb_256(x, a, b); transmute(r) } @@ -348,24 +330,19 @@ pub unsafe fn _mm256_gf2p8affine_epi64_epi8(x: __m256i, a: __m256i, b: i32) -> _ /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_gf2p8affine_epi8) #[inline] #[target_feature(enable = "avx512gfni,avx512bw,avx512vl")] -#[cfg_attr(test, assert_instr(vgf2p8affineqb, b = 0))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm256_maskz_gf2p8affine_epi64_epi8( +#[cfg_attr(test, assert_instr(vgf2p8affineqb, B = 0))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm256_maskz_gf2p8affine_epi64_epi8( k: __mmask32, x: __m256i, a: __m256i, - b: i32, ) -> __m256i { + static_assert_imm8!(B); + let b = B as u8; let zero = _mm256_setzero_si256().as_i8x32(); - assert!(0 <= b && b < 256); let x = x.as_i8x32(); let a = a.as_i8x32(); - macro_rules! call { - ($imm8:expr) => { - vgf2p8affineqb_256(x, a, $imm8) - }; - } - let r = constify_imm8_sae!(b, call); + let r = vgf2p8affineqb_256(x, a, b); transmute(simd_select_bitmask(k, r, zero)) } @@ -380,24 +357,19 @@ pub unsafe fn _mm256_maskz_gf2p8affine_epi64_epi8( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_gf2p8affine_epi8) #[inline] #[target_feature(enable = "avx512gfni,avx512bw,avx512vl")] -#[cfg_attr(test, assert_instr(vgf2p8affineqb, b = 0))] -#[rustc_args_required_const(4)] -pub unsafe fn _mm256_mask_gf2p8affine_epi64_epi8( +#[cfg_attr(test, assert_instr(vgf2p8affineqb, B = 0))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm256_mask_gf2p8affine_epi64_epi8( src: __m256i, k: __mmask32, x: __m256i, a: __m256i, - b: i32, ) -> __m256i { - assert!(0 <= b && b < 256); + static_assert_imm8!(B); + let b = B as u8; let x = x.as_i8x32(); let a = a.as_i8x32(); - macro_rules! call { - ($imm8:expr) => { - vgf2p8affineqb_256(x, a, $imm8) - }; - } - let r = constify_imm8_sae!(b, call); + let r = vgf2p8affineqb_256(x, a, b); transmute(simd_select_bitmask(k, r, src.as_i8x32())) } @@ -409,18 +381,14 @@ pub unsafe fn _mm256_mask_gf2p8affine_epi64_epi8( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_gf2p8affine_epi8) #[inline] #[target_feature(enable = "avx512gfni,avx512bw,avx512vl")] -#[cfg_attr(test, assert_instr(vgf2p8affineqb, b = 0))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm_gf2p8affine_epi64_epi8(x: __m128i, a: __m128i, b: i32) -> __m128i { - assert!(0 <= b && b < 256); +#[cfg_attr(test, assert_instr(vgf2p8affineqb, B = 0))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm_gf2p8affine_epi64_epi8(x: __m128i, a: __m128i) -> __m128i { + static_assert_imm8!(B); + let b = B as u8; let x = x.as_i8x16(); let a = a.as_i8x16(); - macro_rules! call { - ($imm8:expr) => { - vgf2p8affineqb_128(x, a, $imm8) - }; - } - let r = constify_imm8_sae!(b, call); + let r = vgf2p8affineqb_128(x, a, b); transmute(r) } @@ -435,24 +403,19 @@ pub unsafe fn _mm_gf2p8affine_epi64_epi8(x: __m128i, a: __m128i, b: i32) -> __m1 /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_gf2p8affine_epi8) #[inline] #[target_feature(enable = "avx512gfni,avx512bw,avx512vl")] -#[cfg_attr(test, assert_instr(vgf2p8affineqb, b = 0))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm_maskz_gf2p8affine_epi64_epi8( +#[cfg_attr(test, assert_instr(vgf2p8affineqb, B = 0))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm_maskz_gf2p8affine_epi64_epi8( k: __mmask16, x: __m128i, a: __m128i, - b: i32, ) -> __m128i { + static_assert_imm8!(B); + let b = B as u8; let zero = _mm_setzero_si128().as_i8x16(); - assert!(0 <= b && b < 256); let x = x.as_i8x16(); let a = a.as_i8x16(); - macro_rules! call { - ($imm8:expr) => { - vgf2p8affineqb_128(x, a, $imm8) - }; - } - let r = constify_imm8_sae!(b, call); + let r = vgf2p8affineqb_128(x, a, b); transmute(simd_select_bitmask(k, r, zero)) } @@ -467,24 +430,19 @@ pub unsafe fn _mm_maskz_gf2p8affine_epi64_epi8( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_gf2p8affine_epi8) #[inline] #[target_feature(enable = "avx512gfni,avx512bw,avx512vl")] -#[cfg_attr(test, assert_instr(vgf2p8affineqb, b = 0))] -#[rustc_args_required_const(4)] -pub unsafe fn _mm_mask_gf2p8affine_epi64_epi8( +#[cfg_attr(test, assert_instr(vgf2p8affineqb, B = 0))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm_mask_gf2p8affine_epi64_epi8( src: __m128i, k: __mmask16, x: __m128i, a: __m128i, - b: i32, ) -> __m128i { - assert!(0 <= b && b < 256); + static_assert_imm8!(B); + let b = B as u8; let x = x.as_i8x16(); let a = a.as_i8x16(); - macro_rules! call { - ($imm8:expr) => { - vgf2p8affineqb_128(x, a, $imm8) - }; - } - let r = constify_imm8_sae!(b, call); + let r = vgf2p8affineqb_128(x, a, b); transmute(simd_select_bitmask(k, r, src.as_i8x16())) } @@ -498,18 +456,14 @@ pub unsafe fn _mm_mask_gf2p8affine_epi64_epi8( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_gf2p8affineinv_epi64_epi8) #[inline] #[target_feature(enable = "avx512gfni,avx512bw,avx512f")] -#[cfg_attr(test, assert_instr(vgf2p8affineinvqb, b = 0))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm512_gf2p8affineinv_epi64_epi8(x: __m512i, a: __m512i, b: i32) -> __m512i { - assert!(0 <= b && b < 256); +#[cfg_attr(test, assert_instr(vgf2p8affineinvqb, B = 0))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_gf2p8affineinv_epi64_epi8(x: __m512i, a: __m512i) -> __m512i { + static_assert_imm8!(B); + let b = B as u8; let x = x.as_i8x64(); let a = a.as_i8x64(); - macro_rules! call { - ($imm8:expr) => { - vgf2p8affineinvqb_512(x, a, $imm8) - }; - } - let r = constify_imm8_sae!(b, call); + let r = vgf2p8affineinvqb_512(x, a, b); transmute(r) } @@ -526,24 +480,19 @@ pub unsafe fn _mm512_gf2p8affineinv_epi64_epi8(x: __m512i, a: __m512i, b: i32) - /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_gf2p8affineinv_epi64_epi8) #[inline] #[target_feature(enable = "avx512gfni,avx512bw,avx512f")] -#[cfg_attr(test, assert_instr(vgf2p8affineinvqb, b = 0))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm512_maskz_gf2p8affineinv_epi64_epi8( +#[cfg_attr(test, assert_instr(vgf2p8affineinvqb, B = 0))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_maskz_gf2p8affineinv_epi64_epi8( k: __mmask64, x: __m512i, a: __m512i, - b: i32, ) -> __m512i { - assert!(0 <= b && b < 256); + static_assert_imm8!(B); + let b = B as u8; let zero = _mm512_setzero_si512().as_i8x64(); let x = x.as_i8x64(); let a = a.as_i8x64(); - macro_rules! call { - ($imm8:expr) => { - vgf2p8affineinvqb_512(x, a, $imm8) - }; - } - let r = constify_imm8_sae!(b, call); + let r = vgf2p8affineinvqb_512(x, a, b); transmute(simd_select_bitmask(k, r, zero)) } @@ -560,24 +509,19 @@ pub unsafe fn _mm512_maskz_gf2p8affineinv_epi64_epi8( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_gf2p8affineinv_epi64_epi8) #[inline] #[target_feature(enable = "avx512gfni,avx512bw,avx512f")] -#[cfg_attr(test, assert_instr(vgf2p8affineinvqb, b = 0))] -#[rustc_args_required_const(4)] -pub unsafe fn _mm512_mask_gf2p8affineinv_epi64_epi8( +#[cfg_attr(test, assert_instr(vgf2p8affineinvqb, B = 0))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_mask_gf2p8affineinv_epi64_epi8( src: __m512i, k: __mmask64, x: __m512i, a: __m512i, - b: i32, ) -> __m512i { - assert!(0 <= b && b < 256); + static_assert_imm8!(B); + let b = B as u8; let x = x.as_i8x64(); let a = a.as_i8x64(); - macro_rules! call { - ($imm8:expr) => { - vgf2p8affineinvqb_512(x, a, $imm8) - }; - } - let r = constify_imm8_sae!(b, call); + let r = vgf2p8affineinvqb_512(x, a, b); transmute(simd_select_bitmask(k, r, src.as_i8x64())) } @@ -591,18 +535,14 @@ pub unsafe fn _mm512_mask_gf2p8affineinv_epi64_epi8( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_gf2p8affineinv_epi64_epi8) #[inline] #[target_feature(enable = "avx512gfni,avx512bw,avx512vl")] -#[cfg_attr(test, assert_instr(vgf2p8affineinvqb, b = 0))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm256_gf2p8affineinv_epi64_epi8(x: __m256i, a: __m256i, b: i32) -> __m256i { - assert!(0 <= b && b < 256); +#[cfg_attr(test, assert_instr(vgf2p8affineinvqb, B = 0))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm256_gf2p8affineinv_epi64_epi8(x: __m256i, a: __m256i) -> __m256i { + static_assert_imm8!(B); + let b = B as u8; let x = x.as_i8x32(); let a = a.as_i8x32(); - macro_rules! call { - ($imm8:expr) => { - vgf2p8affineinvqb_256(x, a, $imm8) - }; - } - let r = constify_imm8_sae!(b, call); + let r = vgf2p8affineinvqb_256(x, a, b); transmute(r) } @@ -619,24 +559,19 @@ pub unsafe fn _mm256_gf2p8affineinv_epi64_epi8(x: __m256i, a: __m256i, b: i32) - /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_gf2p8affineinv_epi64_epi8) #[inline] #[target_feature(enable = "avx512gfni,avx512bw,avx512vl")] -#[cfg_attr(test, assert_instr(vgf2p8affineinvqb, b = 0))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm256_maskz_gf2p8affineinv_epi64_epi8( +#[cfg_attr(test, assert_instr(vgf2p8affineinvqb, B = 0))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm256_maskz_gf2p8affineinv_epi64_epi8( k: __mmask32, x: __m256i, a: __m256i, - b: i32, ) -> __m256i { - assert!(0 <= b && b < 256); + static_assert_imm8!(B); + let b = B as u8; let zero = _mm256_setzero_si256().as_i8x32(); let x = x.as_i8x32(); let a = a.as_i8x32(); - macro_rules! call { - ($imm8:expr) => { - vgf2p8affineinvqb_256(x, a, $imm8) - }; - } - let r = constify_imm8_sae!(b, call); + let r = vgf2p8affineinvqb_256(x, a, b); transmute(simd_select_bitmask(k, r, zero)) } @@ -653,24 +588,19 @@ pub unsafe fn _mm256_maskz_gf2p8affineinv_epi64_epi8( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_gf2p8affineinv_epi64_epi8) #[inline] #[target_feature(enable = "avx512gfni,avx512bw,avx512vl")] -#[cfg_attr(test, assert_instr(vgf2p8affineinvqb, b = 0))] -#[rustc_args_required_const(4)] -pub unsafe fn _mm256_mask_gf2p8affineinv_epi64_epi8( +#[cfg_attr(test, assert_instr(vgf2p8affineinvqb, B = 0))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm256_mask_gf2p8affineinv_epi64_epi8( src: __m256i, k: __mmask32, x: __m256i, a: __m256i, - b: i32, ) -> __m256i { - assert!(0 <= b && b < 256); + static_assert_imm8!(B); + let b = B as u8; let x = x.as_i8x32(); let a = a.as_i8x32(); - macro_rules! call { - ($imm8:expr) => { - vgf2p8affineinvqb_256(x, a, $imm8) - }; - } - let r = constify_imm8_sae!(b, call); + let r = vgf2p8affineinvqb_256(x, a, b); transmute(simd_select_bitmask(k, r, src.as_i8x32())) } @@ -684,18 +614,14 @@ pub unsafe fn _mm256_mask_gf2p8affineinv_epi64_epi8( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_gf2p8affineinv_epi64_epi8) #[inline] #[target_feature(enable = "avx512gfni,avx512bw,avx512vl")] -#[cfg_attr(test, assert_instr(vgf2p8affineinvqb, b = 0))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm_gf2p8affineinv_epi64_epi8(x: __m128i, a: __m128i, b: i32) -> __m128i { - assert!(0 <= b && b < 256); +#[cfg_attr(test, assert_instr(vgf2p8affineinvqb, B = 0))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm_gf2p8affineinv_epi64_epi8(x: __m128i, a: __m128i) -> __m128i { + static_assert_imm8!(B); + let b = B as u8; let x = x.as_i8x16(); let a = a.as_i8x16(); - macro_rules! call { - ($imm8:expr) => { - vgf2p8affineinvqb_128(x, a, $imm8) - }; - } - let r = constify_imm8_sae!(b, call); + let r = vgf2p8affineinvqb_128(x, a, b); transmute(r) } @@ -712,24 +638,19 @@ pub unsafe fn _mm_gf2p8affineinv_epi64_epi8(x: __m128i, a: __m128i, b: i32) -> _ /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_gf2p8affineinv_epi64_epi8) #[inline] #[target_feature(enable = "avx512gfni,avx512bw,avx512vl")] -#[cfg_attr(test, assert_instr(vgf2p8affineinvqb, b = 0))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm_maskz_gf2p8affineinv_epi64_epi8( +#[cfg_attr(test, assert_instr(vgf2p8affineinvqb, B = 0))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm_maskz_gf2p8affineinv_epi64_epi8( k: __mmask16, x: __m128i, a: __m128i, - b: i32, ) -> __m128i { - assert!(0 <= b && b < 256); + static_assert_imm8!(B); + let b = B as u8; let zero = _mm_setzero_si128().as_i8x16(); let x = x.as_i8x16(); let a = a.as_i8x16(); - macro_rules! call { - ($imm8:expr) => { - vgf2p8affineinvqb_128(x, a, $imm8) - }; - } - let r = constify_imm8_sae!(b, call); + let r = vgf2p8affineinvqb_128(x, a, b); transmute(simd_select_bitmask(k, r, zero)) } @@ -746,24 +667,19 @@ pub unsafe fn _mm_maskz_gf2p8affineinv_epi64_epi8( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_gf2p8affineinv_epi64_epi8) #[inline] #[target_feature(enable = "avx512gfni,avx512bw,avx512vl")] -#[cfg_attr(test, assert_instr(vgf2p8affineinvqb, b = 0))] -#[rustc_args_required_const(4)] -pub unsafe fn _mm_mask_gf2p8affineinv_epi64_epi8( +#[cfg_attr(test, assert_instr(vgf2p8affineinvqb, B = 0))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm_mask_gf2p8affineinv_epi64_epi8( src: __m128i, k: __mmask16, x: __m128i, a: __m128i, - b: i32, ) -> __m128i { - assert!(0 <= b && b < 256); + static_assert_imm8!(B); + let b = B as u8; let x = x.as_i8x16(); let a = a.as_i8x16(); - macro_rules! call { - ($imm8:expr) => { - vgf2p8affineinvqb_128(x, a, $imm8) - }; - } - let r = constify_imm8_sae!(b, call); + let r = vgf2p8affineinvqb_128(x, a, b); transmute(simd_select_bitmask(k, r, src.as_i8x16())) } @@ -1008,7 +924,7 @@ mod tests { let expected_result = _mm256_gf2p8mul_epi8(left, right); let result_masked = _mm256_maskz_gf2p8mul_epi8(mask_bytes, left, right); let expected_masked = - _mm256_blend_epi32(_mm256_setzero_si256(), expected_result, MASK_WORDS); + _mm256_blend_epi32::(_mm256_setzero_si256(), expected_result); assert_eq_m256i(result_masked, expected_masked); } } @@ -1026,7 +942,7 @@ mod tests { const MASK_WORDS: i32 = 0b01_10_11_00; let expected_result = _mm256_gf2p8mul_epi8(left, right); let result_masked = _mm256_mask_gf2p8mul_epi8(left, mask_bytes, left, right); - let expected_masked = _mm256_blend_epi32(left, expected_result, MASK_WORDS); + let expected_masked = _mm256_blend_epi32::(left, expected_result); assert_eq_m256i(result_masked, expected_masked); } } @@ -1057,7 +973,8 @@ mod tests { const MASK_WORDS: i32 = 0b01_10; let expected_result = _mm_gf2p8mul_epi8(left, right); let result_masked = _mm_maskz_gf2p8mul_epi8(mask_bytes, left, right); - let expected_masked = _mm_blend_epi32(_mm_setzero_si128(), expected_result, MASK_WORDS); + let expected_masked = + _mm_blend_epi32::(_mm_setzero_si128(), expected_result); assert_eq_m128i(result_masked, expected_masked); } } @@ -1075,7 +992,7 @@ mod tests { const MASK_WORDS: i32 = 0b01_10; let expected_result = _mm_gf2p8mul_epi8(left, right); let result_masked = _mm_mask_gf2p8mul_epi8(left, mask_bytes, left, right); - let expected_masked = _mm_blend_epi32(left, expected_result, MASK_WORDS); + let expected_masked = _mm_blend_epi32::(left, expected_result); assert_eq_m128i(result_masked, expected_masked); } } @@ -1095,21 +1012,21 @@ mod tests { for i in 0..NUM_TEST_WORDS_512 { let data = load_m512i_word(&bytes, i); - let result = _mm512_gf2p8affine_epi64_epi8(data, identity, IDENTITY_BYTE); + let result = _mm512_gf2p8affine_epi64_epi8::(data, identity); assert_eq_m512i(result, data); - let result = _mm512_gf2p8affine_epi64_epi8(data, constant, CONSTANT_BYTE); + let result = _mm512_gf2p8affine_epi64_epi8::(data, constant); assert_eq_m512i(result, constant_reference); let data = load_m512i_word(&more_bytes, i); - let result = _mm512_gf2p8affine_epi64_epi8(data, identity, IDENTITY_BYTE); + let result = _mm512_gf2p8affine_epi64_epi8::(data, identity); assert_eq_m512i(result, data); - let result = _mm512_gf2p8affine_epi64_epi8(data, constant, CONSTANT_BYTE); + let result = _mm512_gf2p8affine_epi64_epi8::(data, constant); assert_eq_m512i(result, constant_reference); let matrix = load_m512i_word(&matrices, i); let vector = load_m512i_word(&vectors, i); let reference = load_m512i_word(&references, i); - let result = _mm512_gf2p8affine_epi64_epi8(vector, matrix, IDENTITY_BYTE); + let result = _mm512_gf2p8affine_epi64_epi8::(vector, matrix); assert_eq_m512i(result, reference); } } @@ -1122,13 +1039,14 @@ mod tests { for i in 0..NUM_TEST_WORDS_512 { let matrix = load_m512i_word(&matrices, i); let vector = load_m512i_word(&vectors, i); - let result_zero = _mm512_maskz_gf2p8affine_epi64_epi8(0, vector, matrix, CONSTANT_BYTE); + let result_zero = + _mm512_maskz_gf2p8affine_epi64_epi8::(0, vector, matrix); assert_eq_m512i(result_zero, _mm512_setzero_si512()); let mask_bytes: __mmask64 = 0x0F_0F_0F_0F_FF_FF_00_00; let mask_words: __mmask16 = 0b01_01_01_01_11_11_00_00; - let expected_result = _mm512_gf2p8affine_epi64_epi8(vector, matrix, CONSTANT_BYTE); + let expected_result = _mm512_gf2p8affine_epi64_epi8::(vector, matrix); let result_masked = - _mm512_maskz_gf2p8affine_epi64_epi8(mask_bytes, vector, matrix, CONSTANT_BYTE); + _mm512_maskz_gf2p8affine_epi64_epi8::(mask_bytes, vector, matrix); let expected_masked = _mm512_mask_blend_epi32(mask_words, _mm512_setzero_si512(), expected_result); assert_eq_m512i(result_masked, expected_masked); @@ -1144,13 +1062,13 @@ mod tests { let left = load_m512i_word(&vectors, i); let right = load_m512i_word(&matrices, i); let result_left = - _mm512_mask_gf2p8affine_epi64_epi8(left, 0, left, right, CONSTANT_BYTE); + _mm512_mask_gf2p8affine_epi64_epi8::(left, 0, left, right); assert_eq_m512i(result_left, left); let mask_bytes: __mmask64 = 0x0F_0F_0F_0F_FF_FF_00_00; let mask_words: __mmask16 = 0b01_01_01_01_11_11_00_00; - let expected_result = _mm512_gf2p8affine_epi64_epi8(left, right, CONSTANT_BYTE); + let expected_result = _mm512_gf2p8affine_epi64_epi8::(left, right); let result_masked = - _mm512_mask_gf2p8affine_epi64_epi8(left, mask_bytes, left, right, CONSTANT_BYTE); + _mm512_mask_gf2p8affine_epi64_epi8::(left, mask_bytes, left, right); let expected_masked = _mm512_mask_blend_epi32(mask_words, left, expected_result); assert_eq_m512i(result_masked, expected_masked); } @@ -1171,21 +1089,21 @@ mod tests { for i in 0..NUM_TEST_WORDS_256 { let data = load_m256i_word(&bytes, i); - let result = _mm256_gf2p8affine_epi64_epi8(data, identity, IDENTITY_BYTE); + let result = _mm256_gf2p8affine_epi64_epi8::(data, identity); assert_eq_m256i(result, data); - let result = _mm256_gf2p8affine_epi64_epi8(data, constant, CONSTANT_BYTE); + let result = _mm256_gf2p8affine_epi64_epi8::(data, constant); assert_eq_m256i(result, constant_reference); let data = load_m256i_word(&more_bytes, i); - let result = _mm256_gf2p8affine_epi64_epi8(data, identity, IDENTITY_BYTE); + let result = _mm256_gf2p8affine_epi64_epi8::(data, identity); assert_eq_m256i(result, data); - let result = _mm256_gf2p8affine_epi64_epi8(data, constant, CONSTANT_BYTE); + let result = _mm256_gf2p8affine_epi64_epi8::(data, constant); assert_eq_m256i(result, constant_reference); let matrix = load_m256i_word(&matrices, i); let vector = load_m256i_word(&vectors, i); let reference = load_m256i_word(&references, i); - let result = _mm256_gf2p8affine_epi64_epi8(vector, matrix, IDENTITY_BYTE); + let result = _mm256_gf2p8affine_epi64_epi8::(vector, matrix); assert_eq_m256i(result, reference); } } @@ -1198,15 +1116,16 @@ mod tests { for i in 0..NUM_TEST_WORDS_256 { let matrix = load_m256i_word(&matrices, i); let vector = load_m256i_word(&vectors, i); - let result_zero = _mm256_maskz_gf2p8affine_epi64_epi8(0, vector, matrix, CONSTANT_BYTE); + let result_zero = + _mm256_maskz_gf2p8affine_epi64_epi8::(0, vector, matrix); assert_eq_m256i(result_zero, _mm256_setzero_si256()); let mask_bytes: __mmask32 = 0xFF_0F_F0_00; const MASK_WORDS: i32 = 0b11_01_10_00; - let expected_result = _mm256_gf2p8affine_epi64_epi8(vector, matrix, CONSTANT_BYTE); + let expected_result = _mm256_gf2p8affine_epi64_epi8::(vector, matrix); let result_masked = - _mm256_maskz_gf2p8affine_epi64_epi8(mask_bytes, vector, matrix, CONSTANT_BYTE); + _mm256_maskz_gf2p8affine_epi64_epi8::(mask_bytes, vector, matrix); let expected_masked = - _mm256_blend_epi32(_mm256_setzero_si256(), expected_result, MASK_WORDS); + _mm256_blend_epi32::(_mm256_setzero_si256(), expected_result); assert_eq_m256i(result_masked, expected_masked); } } @@ -1220,14 +1139,14 @@ mod tests { let left = load_m256i_word(&vectors, i); let right = load_m256i_word(&matrices, i); let result_left = - _mm256_mask_gf2p8affine_epi64_epi8(left, 0, left, right, CONSTANT_BYTE); + _mm256_mask_gf2p8affine_epi64_epi8::(left, 0, left, right); assert_eq_m256i(result_left, left); let mask_bytes: __mmask32 = 0xFF_0F_F0_00; const MASK_WORDS: i32 = 0b11_01_10_00; - let expected_result = _mm256_gf2p8affine_epi64_epi8(left, right, CONSTANT_BYTE); + let expected_result = _mm256_gf2p8affine_epi64_epi8::(left, right); let result_masked = - _mm256_mask_gf2p8affine_epi64_epi8(left, mask_bytes, left, right, CONSTANT_BYTE); - let expected_masked = _mm256_blend_epi32(left, expected_result, MASK_WORDS); + _mm256_mask_gf2p8affine_epi64_epi8::(left, mask_bytes, left, right); + let expected_masked = _mm256_blend_epi32::(left, expected_result); assert_eq_m256i(result_masked, expected_masked); } } @@ -1247,21 +1166,21 @@ mod tests { for i in 0..NUM_TEST_WORDS_128 { let data = load_m128i_word(&bytes, i); - let result = _mm_gf2p8affine_epi64_epi8(data, identity, IDENTITY_BYTE); + let result = _mm_gf2p8affine_epi64_epi8::(data, identity); assert_eq_m128i(result, data); - let result = _mm_gf2p8affine_epi64_epi8(data, constant, CONSTANT_BYTE); + let result = _mm_gf2p8affine_epi64_epi8::(data, constant); assert_eq_m128i(result, constant_reference); let data = load_m128i_word(&more_bytes, i); - let result = _mm_gf2p8affine_epi64_epi8(data, identity, IDENTITY_BYTE); + let result = _mm_gf2p8affine_epi64_epi8::(data, identity); assert_eq_m128i(result, data); - let result = _mm_gf2p8affine_epi64_epi8(data, constant, CONSTANT_BYTE); + let result = _mm_gf2p8affine_epi64_epi8::(data, constant); assert_eq_m128i(result, constant_reference); let matrix = load_m128i_word(&matrices, i); let vector = load_m128i_word(&vectors, i); let reference = load_m128i_word(&references, i); - let result = _mm_gf2p8affine_epi64_epi8(vector, matrix, IDENTITY_BYTE); + let result = _mm_gf2p8affine_epi64_epi8::(vector, matrix); assert_eq_m128i(result, reference); } } @@ -1274,14 +1193,15 @@ mod tests { for i in 0..NUM_TEST_WORDS_128 { let matrix = load_m128i_word(&matrices, i); let vector = load_m128i_word(&vectors, i); - let result_zero = _mm_maskz_gf2p8affine_epi64_epi8(0, vector, matrix, CONSTANT_BYTE); + let result_zero = _mm_maskz_gf2p8affine_epi64_epi8::(0, vector, matrix); assert_eq_m128i(result_zero, _mm_setzero_si128()); let mask_bytes: __mmask16 = 0x0F_F0; const MASK_WORDS: i32 = 0b01_10; - let expected_result = _mm_gf2p8affine_epi64_epi8(vector, matrix, CONSTANT_BYTE); + let expected_result = _mm_gf2p8affine_epi64_epi8::(vector, matrix); let result_masked = - _mm_maskz_gf2p8affine_epi64_epi8(mask_bytes, vector, matrix, CONSTANT_BYTE); - let expected_masked = _mm_blend_epi32(_mm_setzero_si128(), expected_result, MASK_WORDS); + _mm_maskz_gf2p8affine_epi64_epi8::(mask_bytes, vector, matrix); + let expected_masked = + _mm_blend_epi32::(_mm_setzero_si128(), expected_result); assert_eq_m128i(result_masked, expected_masked); } } @@ -1294,14 +1214,15 @@ mod tests { for i in 0..NUM_TEST_WORDS_128 { let left = load_m128i_word(&vectors, i); let right = load_m128i_word(&matrices, i); - let result_left = _mm_mask_gf2p8affine_epi64_epi8(left, 0, left, right, CONSTANT_BYTE); + let result_left = + _mm_mask_gf2p8affine_epi64_epi8::(left, 0, left, right); assert_eq_m128i(result_left, left); let mask_bytes: __mmask16 = 0x0F_F0; const MASK_WORDS: i32 = 0b01_10; - let expected_result = _mm_gf2p8affine_epi64_epi8(left, right, CONSTANT_BYTE); + let expected_result = _mm_gf2p8affine_epi64_epi8::(left, right); let result_masked = - _mm_mask_gf2p8affine_epi64_epi8(left, mask_bytes, left, right, CONSTANT_BYTE); - let expected_masked = _mm_blend_epi32(left, expected_result, MASK_WORDS); + _mm_mask_gf2p8affine_epi64_epi8::(left, mask_bytes, left, right); + let expected_masked = _mm_blend_epi32::(left, expected_result); assert_eq_m128i(result_masked, expected_masked); } } @@ -1319,7 +1240,7 @@ mod tests { for i in 0..NUM_BYTES_WORDS_512 { let input = load_m512i_word(&inputs, i); let reference = load_m512i_word(&results, i); - let result = _mm512_gf2p8affineinv_epi64_epi8(input, identity, IDENTITY_BYTE); + let result = _mm512_gf2p8affineinv_epi64_epi8::(input, identity); let remultiplied = _mm512_gf2p8mul_epi8(result, input); assert_eq_m512i(remultiplied, reference); } @@ -1332,9 +1253,9 @@ mod tests { let vector = load_m512i_word(&vectors, i); let matrix = load_m512i_word(&matrices, i); - let inv_vec = _mm512_gf2p8affineinv_epi64_epi8(vector, identity, IDENTITY_BYTE); - let reference = _mm512_gf2p8affine_epi64_epi8(inv_vec, matrix, CONSTANT_BYTE); - let result = _mm512_gf2p8affineinv_epi64_epi8(vector, matrix, CONSTANT_BYTE); + let inv_vec = _mm512_gf2p8affineinv_epi64_epi8::(vector, identity); + let reference = _mm512_gf2p8affine_epi64_epi8::(inv_vec, matrix); + let result = _mm512_gf2p8affineinv_epi64_epi8::(vector, matrix); assert_eq_m512i(result, reference); } @@ -1345,7 +1266,7 @@ mod tests { for i in 0..NUM_BYTES_WORDS_512 { let reference = load_m512i_word(&AES_S_BOX, i); let input = load_m512i_word(&inputs, i); - let result = _mm512_gf2p8affineinv_epi64_epi8(input, sbox_matrix, CONSTANT_BYTE); + let result = _mm512_gf2p8affineinv_epi64_epi8::(input, sbox_matrix); assert_eq_m512i(result, reference); } } @@ -1359,13 +1280,13 @@ mod tests { let matrix = load_m512i_word(&matrices, i); let vector = load_m512i_word(&vectors, i); let result_zero = - _mm512_maskz_gf2p8affineinv_epi64_epi8(0, vector, matrix, CONSTANT_BYTE); + _mm512_maskz_gf2p8affineinv_epi64_epi8::(0, vector, matrix); assert_eq_m512i(result_zero, _mm512_setzero_si512()); let mask_bytes: __mmask64 = 0x0F_0F_0F_0F_FF_FF_00_00; let mask_words: __mmask16 = 0b01_01_01_01_11_11_00_00; - let expected_result = _mm512_gf2p8affineinv_epi64_epi8(vector, matrix, CONSTANT_BYTE); + let expected_result = _mm512_gf2p8affineinv_epi64_epi8::(vector, matrix); let result_masked = - _mm512_maskz_gf2p8affineinv_epi64_epi8(mask_bytes, vector, matrix, CONSTANT_BYTE); + _mm512_maskz_gf2p8affineinv_epi64_epi8::(mask_bytes, vector, matrix); let expected_masked = _mm512_mask_blend_epi32(mask_words, _mm512_setzero_si512(), expected_result); assert_eq_m512i(result_masked, expected_masked); @@ -1381,13 +1302,14 @@ mod tests { let left = load_m512i_word(&vectors, i); let right = load_m512i_word(&matrices, i); let result_left = - _mm512_mask_gf2p8affineinv_epi64_epi8(left, 0, left, right, CONSTANT_BYTE); + _mm512_mask_gf2p8affineinv_epi64_epi8::(left, 0, left, right); assert_eq_m512i(result_left, left); let mask_bytes: __mmask64 = 0x0F_0F_0F_0F_FF_FF_00_00; let mask_words: __mmask16 = 0b01_01_01_01_11_11_00_00; - let expected_result = _mm512_gf2p8affineinv_epi64_epi8(left, right, CONSTANT_BYTE); - let result_masked = - _mm512_mask_gf2p8affineinv_epi64_epi8(left, mask_bytes, left, right, CONSTANT_BYTE); + let expected_result = _mm512_gf2p8affineinv_epi64_epi8::(left, right); + let result_masked = _mm512_mask_gf2p8affineinv_epi64_epi8::( + left, mask_bytes, left, right, + ); let expected_masked = _mm512_mask_blend_epi32(mask_words, left, expected_result); assert_eq_m512i(result_masked, expected_masked); } @@ -1406,7 +1328,7 @@ mod tests { for i in 0..NUM_BYTES_WORDS_256 { let input = load_m256i_word(&inputs, i); let reference = load_m256i_word(&results, i); - let result = _mm256_gf2p8affineinv_epi64_epi8(input, identity, IDENTITY_BYTE); + let result = _mm256_gf2p8affineinv_epi64_epi8::(input, identity); let remultiplied = _mm256_gf2p8mul_epi8(result, input); assert_eq_m256i(remultiplied, reference); } @@ -1419,9 +1341,9 @@ mod tests { let vector = load_m256i_word(&vectors, i); let matrix = load_m256i_word(&matrices, i); - let inv_vec = _mm256_gf2p8affineinv_epi64_epi8(vector, identity, IDENTITY_BYTE); - let reference = _mm256_gf2p8affine_epi64_epi8(inv_vec, matrix, CONSTANT_BYTE); - let result = _mm256_gf2p8affineinv_epi64_epi8(vector, matrix, CONSTANT_BYTE); + let inv_vec = _mm256_gf2p8affineinv_epi64_epi8::(vector, identity); + let reference = _mm256_gf2p8affine_epi64_epi8::(inv_vec, matrix); + let result = _mm256_gf2p8affineinv_epi64_epi8::(vector, matrix); assert_eq_m256i(result, reference); } @@ -1432,7 +1354,7 @@ mod tests { for i in 0..NUM_BYTES_WORDS_256 { let reference = load_m256i_word(&AES_S_BOX, i); let input = load_m256i_word(&inputs, i); - let result = _mm256_gf2p8affineinv_epi64_epi8(input, sbox_matrix, CONSTANT_BYTE); + let result = _mm256_gf2p8affineinv_epi64_epi8::(input, sbox_matrix); assert_eq_m256i(result, reference); } } @@ -1446,15 +1368,15 @@ mod tests { let matrix = load_m256i_word(&matrices, i); let vector = load_m256i_word(&vectors, i); let result_zero = - _mm256_maskz_gf2p8affineinv_epi64_epi8(0, vector, matrix, CONSTANT_BYTE); + _mm256_maskz_gf2p8affineinv_epi64_epi8::(0, vector, matrix); assert_eq_m256i(result_zero, _mm256_setzero_si256()); let mask_bytes: __mmask32 = 0xFF_0F_F0_00; const MASK_WORDS: i32 = 0b11_01_10_00; - let expected_result = _mm256_gf2p8affineinv_epi64_epi8(vector, matrix, CONSTANT_BYTE); + let expected_result = _mm256_gf2p8affineinv_epi64_epi8::(vector, matrix); let result_masked = - _mm256_maskz_gf2p8affineinv_epi64_epi8(mask_bytes, vector, matrix, CONSTANT_BYTE); + _mm256_maskz_gf2p8affineinv_epi64_epi8::(mask_bytes, vector, matrix); let expected_masked = - _mm256_blend_epi32(_mm256_setzero_si256(), expected_result, MASK_WORDS); + _mm256_blend_epi32::(_mm256_setzero_si256(), expected_result); assert_eq_m256i(result_masked, expected_masked); } } @@ -1468,14 +1390,15 @@ mod tests { let left = load_m256i_word(&vectors, i); let right = load_m256i_word(&matrices, i); let result_left = - _mm256_mask_gf2p8affineinv_epi64_epi8(left, 0, left, right, CONSTANT_BYTE); + _mm256_mask_gf2p8affineinv_epi64_epi8::(left, 0, left, right); assert_eq_m256i(result_left, left); let mask_bytes: __mmask32 = 0xFF_0F_F0_00; const MASK_WORDS: i32 = 0b11_01_10_00; - let expected_result = _mm256_gf2p8affineinv_epi64_epi8(left, right, CONSTANT_BYTE); - let result_masked = - _mm256_mask_gf2p8affineinv_epi64_epi8(left, mask_bytes, left, right, CONSTANT_BYTE); - let expected_masked = _mm256_blend_epi32(left, expected_result, MASK_WORDS); + let expected_result = _mm256_gf2p8affineinv_epi64_epi8::(left, right); + let result_masked = _mm256_mask_gf2p8affineinv_epi64_epi8::( + left, mask_bytes, left, right, + ); + let expected_masked = _mm256_blend_epi32::(left, expected_result); assert_eq_m256i(result_masked, expected_masked); } } @@ -1493,7 +1416,7 @@ mod tests { for i in 0..NUM_BYTES_WORDS_128 { let input = load_m128i_word(&inputs, i); let reference = load_m128i_word(&results, i); - let result = _mm_gf2p8affineinv_epi64_epi8(input, identity, IDENTITY_BYTE); + let result = _mm_gf2p8affineinv_epi64_epi8::(input, identity); let remultiplied = _mm_gf2p8mul_epi8(result, input); assert_eq_m128i(remultiplied, reference); } @@ -1506,9 +1429,9 @@ mod tests { let vector = load_m128i_word(&vectors, i); let matrix = load_m128i_word(&matrices, i); - let inv_vec = _mm_gf2p8affineinv_epi64_epi8(vector, identity, IDENTITY_BYTE); - let reference = _mm_gf2p8affine_epi64_epi8(inv_vec, matrix, CONSTANT_BYTE); - let result = _mm_gf2p8affineinv_epi64_epi8(vector, matrix, CONSTANT_BYTE); + let inv_vec = _mm_gf2p8affineinv_epi64_epi8::(vector, identity); + let reference = _mm_gf2p8affine_epi64_epi8::(inv_vec, matrix); + let result = _mm_gf2p8affineinv_epi64_epi8::(vector, matrix); assert_eq_m128i(result, reference); } @@ -1519,7 +1442,7 @@ mod tests { for i in 0..NUM_BYTES_WORDS_128 { let reference = load_m128i_word(&AES_S_BOX, i); let input = load_m128i_word(&inputs, i); - let result = _mm_gf2p8affineinv_epi64_epi8(input, sbox_matrix, CONSTANT_BYTE); + let result = _mm_gf2p8affineinv_epi64_epi8::(input, sbox_matrix); assert_eq_m128i(result, reference); } } @@ -1532,14 +1455,16 @@ mod tests { for i in 0..NUM_TEST_WORDS_128 { let matrix = load_m128i_word(&matrices, i); let vector = load_m128i_word(&vectors, i); - let result_zero = _mm_maskz_gf2p8affineinv_epi64_epi8(0, vector, matrix, CONSTANT_BYTE); + let result_zero = + _mm_maskz_gf2p8affineinv_epi64_epi8::(0, vector, matrix); assert_eq_m128i(result_zero, _mm_setzero_si128()); let mask_bytes: __mmask16 = 0x0F_F0; const MASK_WORDS: i32 = 0b01_10; - let expected_result = _mm_gf2p8affineinv_epi64_epi8(vector, matrix, CONSTANT_BYTE); + let expected_result = _mm_gf2p8affineinv_epi64_epi8::(vector, matrix); let result_masked = - _mm_maskz_gf2p8affineinv_epi64_epi8(mask_bytes, vector, matrix, CONSTANT_BYTE); - let expected_masked = _mm_blend_epi32(_mm_setzero_si128(), expected_result, MASK_WORDS); + _mm_maskz_gf2p8affineinv_epi64_epi8::(mask_bytes, vector, matrix); + let expected_masked = + _mm_blend_epi32::(_mm_setzero_si128(), expected_result); assert_eq_m128i(result_masked, expected_masked); } } @@ -1553,14 +1478,14 @@ mod tests { let left = load_m128i_word(&vectors, i); let right = load_m128i_word(&matrices, i); let result_left = - _mm_mask_gf2p8affineinv_epi64_epi8(left, 0, left, right, CONSTANT_BYTE); + _mm_mask_gf2p8affineinv_epi64_epi8::(left, 0, left, right); assert_eq_m128i(result_left, left); let mask_bytes: __mmask16 = 0x0F_F0; const MASK_WORDS: i32 = 0b01_10; - let expected_result = _mm_gf2p8affineinv_epi64_epi8(left, right, CONSTANT_BYTE); + let expected_result = _mm_gf2p8affineinv_epi64_epi8::(left, right); let result_masked = - _mm_mask_gf2p8affineinv_epi64_epi8(left, mask_bytes, left, right, CONSTANT_BYTE); - let expected_masked = _mm_blend_epi32(left, expected_result, MASK_WORDS); + _mm_mask_gf2p8affineinv_epi64_epi8::(left, mask_bytes, left, right); + let expected_masked = _mm_blend_epi32::(left, expected_result); assert_eq_m128i(result_masked, expected_masked); } } diff --git a/library/stdarch/crates/core_arch/src/x86/avx512vaes.rs b/library/stdarch/crates/core_arch/src/x86/avx512vaes.rs index b9d8433b4b..676de312b3 100644 --- a/library/stdarch/crates/core_arch/src/x86/avx512vaes.rs +++ b/library/stdarch/crates/core_arch/src/x86/avx512vaes.rs @@ -155,18 +155,18 @@ mod tests { 0x978093862CDE7187, ); let mut a_decomp = [_mm_setzero_si128(); 2]; - a_decomp[0] = _mm256_extracti128_si256(a, 0); - a_decomp[1] = _mm256_extracti128_si256(a, 1); + a_decomp[0] = _mm256_extracti128_si256::<0>(a); + a_decomp[1] = _mm256_extracti128_si256::<1>(a); let mut k_decomp = [_mm_setzero_si128(); 2]; - k_decomp[0] = _mm256_extracti128_si256(k, 0); - k_decomp[1] = _mm256_extracti128_si256(k, 1); + k_decomp[0] = _mm256_extracti128_si256::<0>(k); + k_decomp[1] = _mm256_extracti128_si256::<1>(k); let r = vectorized(a, k); let mut e_decomp = [_mm_setzero_si128(); 2]; for i in 0..2 { e_decomp[i] = linear(a_decomp[i], k_decomp[i]); } - assert_eq_m128i(_mm256_extracti128_si256(r, 0), e_decomp[0]); - assert_eq_m128i(_mm256_extracti128_si256(r, 1), e_decomp[1]); + assert_eq_m128i(_mm256_extracti128_si256::<0>(r), e_decomp[0]); + assert_eq_m128i(_mm256_extracti128_si256::<1>(r), e_decomp[1]); } #[target_feature(enable = "sse2")] @@ -262,24 +262,24 @@ mod tests { 0x19BE9F660038BDB5, ); let mut a_decomp = [_mm_setzero_si128(); 4]; - a_decomp[0] = _mm512_extracti32x4_epi32(a, 0); - a_decomp[1] = _mm512_extracti32x4_epi32(a, 1); - a_decomp[2] = _mm512_extracti32x4_epi32(a, 2); - a_decomp[3] = _mm512_extracti32x4_epi32(a, 3); + a_decomp[0] = _mm512_extracti32x4_epi32::<0>(a); + a_decomp[1] = _mm512_extracti32x4_epi32::<1>(a); + a_decomp[2] = _mm512_extracti32x4_epi32::<2>(a); + a_decomp[3] = _mm512_extracti32x4_epi32::<3>(a); let mut k_decomp = [_mm_setzero_si128(); 4]; - k_decomp[0] = _mm512_extracti32x4_epi32(k, 0); - k_decomp[1] = _mm512_extracti32x4_epi32(k, 1); - k_decomp[2] = _mm512_extracti32x4_epi32(k, 2); - k_decomp[3] = _mm512_extracti32x4_epi32(k, 3); + k_decomp[0] = _mm512_extracti32x4_epi32::<0>(k); + k_decomp[1] = _mm512_extracti32x4_epi32::<1>(k); + k_decomp[2] = _mm512_extracti32x4_epi32::<2>(k); + k_decomp[3] = _mm512_extracti32x4_epi32::<3>(k); let r = vectorized(a, k); let mut e_decomp = [_mm_setzero_si128(); 4]; for i in 0..4 { e_decomp[i] = linear(a_decomp[i], k_decomp[i]); } - assert_eq_m128i(_mm512_extracti32x4_epi32(r, 0), e_decomp[0]); - assert_eq_m128i(_mm512_extracti32x4_epi32(r, 1), e_decomp[1]); - assert_eq_m128i(_mm512_extracti32x4_epi32(r, 2), e_decomp[2]); - assert_eq_m128i(_mm512_extracti32x4_epi32(r, 3), e_decomp[3]); + assert_eq_m128i(_mm512_extracti32x4_epi32::<0>(r), e_decomp[0]); + assert_eq_m128i(_mm512_extracti32x4_epi32::<1>(r), e_decomp[1]); + assert_eq_m128i(_mm512_extracti32x4_epi32::<2>(r), e_decomp[2]); + assert_eq_m128i(_mm512_extracti32x4_epi32::<3>(r), e_decomp[3]); } #[simd_test(enable = "avx512vaes,avx512f")] diff --git a/library/stdarch/crates/core_arch/src/x86/avx512vbmi2.rs b/library/stdarch/crates/core_arch/src/x86/avx512vbmi2.rs index 032bce9176..b7a385dd97 100644 --- a/library/stdarch/crates/core_arch/src/x86/avx512vbmi2.rs +++ b/library/stdarch/crates/core_arch/src/x86/avx512vbmi2.rs @@ -920,14 +920,15 @@ pub unsafe fn _mm_maskz_shrdv_epi16(k: __mmask8, a: __m128i, b: __m128i, c: __m1 /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_shldi_epi64&expand=5060) #[inline] #[target_feature(enable = "avx512vbmi2")] -#[cfg_attr(test, assert_instr(vpshldq, imm8 = 5))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm512_shldi_epi64(a: __m512i, b: __m512i, imm8: i32) -> __m512i { - assert!(imm8 >= 0 && imm8 <= 255); +#[cfg_attr(test, assert_instr(vpshldq, IMM8 = 5))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_shldi_epi64(a: __m512i, b: __m512i) -> __m512i { + static_assert_imm8!(IMM8); + let imm8 = IMM8 as i64; transmute(vpshldvq( a.as_i64x8(), b.as_i64x8(), - _mm512_set1_epi64(imm8 as i64).as_i64x8(), + _mm512_set1_epi64(imm8).as_i64x8(), )) } @@ -936,20 +937,20 @@ pub unsafe fn _mm512_shldi_epi64(a: __m512i, b: __m512i, imm8: i32) -> __m512i { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_shldi_epi64&expand=5058) #[inline] #[target_feature(enable = "avx512vbmi2")] -#[cfg_attr(test, assert_instr(vpshldq, imm8 = 5))] -#[rustc_args_required_const(4)] -pub unsafe fn _mm512_mask_shldi_epi64( +#[cfg_attr(test, assert_instr(vpshldq, IMM8 = 5))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_mask_shldi_epi64( src: __m512i, k: __mmask8, a: __m512i, b: __m512i, - imm8: i32, ) -> __m512i { - assert!(imm8 >= 0 && imm8 <= 255); + static_assert_imm8!(IMM8); + let imm8 = IMM8 as i64; let shf: i64x8 = vpshldvq( a.as_i64x8(), b.as_i64x8(), - _mm512_set1_epi64(imm8 as i64).as_i64x8(), + _mm512_set1_epi64(imm8).as_i64x8(), ); transmute(simd_select_bitmask(k, shf, src.as_i64x8())) } @@ -959,14 +960,19 @@ pub unsafe fn _mm512_mask_shldi_epi64( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_shldi_epi64&expand=5059) #[inline] #[target_feature(enable = "avx512vbmi2")] -#[cfg_attr(test, assert_instr(vpshldq, imm8 = 5))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm512_maskz_shldi_epi64(k: __mmask8, a: __m512i, b: __m512i, imm8: i32) -> __m512i { - assert!(imm8 >= 0 && imm8 <= 255); +#[cfg_attr(test, assert_instr(vpshldq, IMM8 = 5))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_maskz_shldi_epi64( + k: __mmask8, + a: __m512i, + b: __m512i, +) -> __m512i { + static_assert_imm8!(IMM8); + let imm8 = IMM8 as i64; let shf: i64x8 = vpshldvq( a.as_i64x8(), b.as_i64x8(), - _mm512_set1_epi64(imm8 as i64).as_i64x8(), + _mm512_set1_epi64(imm8).as_i64x8(), ); let zero = _mm512_setzero_si512().as_i64x8(); transmute(simd_select_bitmask(k, shf, zero)) @@ -977,14 +983,15 @@ pub unsafe fn _mm512_maskz_shldi_epi64(k: __mmask8, a: __m512i, b: __m512i, imm8 /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_shldi_epi64&expand=5057) #[inline] #[target_feature(enable = "avx512vbmi2,avx512vl")] -#[cfg_attr(test, assert_instr(vpshldq, imm8 = 5))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm256_shldi_epi64(a: __m256i, b: __m256i, imm8: i32) -> __m256i { - assert!(imm8 >= 0 && imm8 <= 255); +#[cfg_attr(test, assert_instr(vpshldq, IMM8 = 5))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm256_shldi_epi64(a: __m256i, b: __m256i) -> __m256i { + static_assert_imm8!(IMM8); + let imm8 = IMM8 as i64; transmute(vpshldvq256( a.as_i64x4(), b.as_i64x4(), - _mm256_set1_epi64x(imm8 as i64).as_i64x4(), + _mm256_set1_epi64x(imm8).as_i64x4(), )) } @@ -993,20 +1000,20 @@ pub unsafe fn _mm256_shldi_epi64(a: __m256i, b: __m256i, imm8: i32) -> __m256i { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_shldi_epi64&expand=5055) #[inline] #[target_feature(enable = "avx512vbmi2,avx512vl")] -#[cfg_attr(test, assert_instr(vpshldq, imm8 = 5))] -#[rustc_args_required_const(4)] -pub unsafe fn _mm256_mask_shldi_epi64( +#[cfg_attr(test, assert_instr(vpshldq, IMM8 = 5))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm256_mask_shldi_epi64( src: __m256i, k: __mmask8, a: __m256i, b: __m256i, - imm8: i32, ) -> __m256i { - assert!(imm8 >= 0 && imm8 <= 255); + static_assert_imm8!(IMM8); + let imm8 = IMM8 as i64; let shf: i64x4 = vpshldvq256( a.as_i64x4(), b.as_i64x4(), - _mm256_set1_epi64x(imm8 as i64).as_i64x4(), + _mm256_set1_epi64x(imm8).as_i64x4(), ); transmute(simd_select_bitmask(k, shf, src.as_i64x4())) } @@ -1016,14 +1023,19 @@ pub unsafe fn _mm256_mask_shldi_epi64( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_shldi_epi64&expand=5056) #[inline] #[target_feature(enable = "avx512vbmi2,avx512vl")] -#[cfg_attr(test, assert_instr(vpshldq, imm8 = 5))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm256_maskz_shldi_epi64(k: __mmask8, a: __m256i, b: __m256i, imm8: i32) -> __m256i { - assert!(imm8 >= 0 && imm8 <= 255); +#[cfg_attr(test, assert_instr(vpshldq, IMM8 = 5))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm256_maskz_shldi_epi64( + k: __mmask8, + a: __m256i, + b: __m256i, +) -> __m256i { + static_assert_imm8!(IMM8); + let imm8 = IMM8 as i64; let shf: i64x4 = vpshldvq256( a.as_i64x4(), b.as_i64x4(), - _mm256_set1_epi64x(imm8 as i64).as_i64x4(), + _mm256_set1_epi64x(imm8).as_i64x4(), ); let zero = _mm256_setzero_si256().as_i64x4(); transmute(simd_select_bitmask(k, shf, zero)) @@ -1034,14 +1046,15 @@ pub unsafe fn _mm256_maskz_shldi_epi64(k: __mmask8, a: __m256i, b: __m256i, imm8 /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_shldi_epi64&expand=5054) #[inline] #[target_feature(enable = "avx512vbmi2,avx512vl")] -#[cfg_attr(test, assert_instr(vpshldq, imm8 = 5))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm_shldi_epi64(a: __m128i, b: __m128i, imm8: i32) -> __m128i { - assert!(imm8 >= 0 && imm8 <= 255); +#[cfg_attr(test, assert_instr(vpshldq, IMM8 = 5))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm_shldi_epi64(a: __m128i, b: __m128i) -> __m128i { + static_assert_imm8!(IMM8); + let imm8 = IMM8 as i64; transmute(vpshldvq128( a.as_i64x2(), b.as_i64x2(), - _mm_set1_epi64x(imm8 as i64).as_i64x2(), + _mm_set1_epi64x(imm8).as_i64x2(), )) } @@ -1050,21 +1063,17 @@ pub unsafe fn _mm_shldi_epi64(a: __m128i, b: __m128i, imm8: i32) -> __m128i { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_shldi_epi64&expand=5052) #[inline] #[target_feature(enable = "avx512vbmi2,avx512vl")] -#[cfg_attr(test, assert_instr(vpshldq, imm8 = 5))] -#[rustc_args_required_const(4)] -pub unsafe fn _mm_mask_shldi_epi64( +#[cfg_attr(test, assert_instr(vpshldq, IMM8 = 5))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm_mask_shldi_epi64( src: __m128i, k: __mmask8, a: __m128i, b: __m128i, - imm8: i32, ) -> __m128i { - assert!(imm8 >= 0 && imm8 <= 255); - let shf: i64x2 = vpshldvq128( - a.as_i64x2(), - b.as_i64x2(), - _mm_set1_epi64x(imm8 as i64).as_i64x2(), - ); + static_assert_imm8!(IMM8); + let imm8 = IMM8 as i64; + let shf: i64x2 = vpshldvq128(a.as_i64x2(), b.as_i64x2(), _mm_set1_epi64x(imm8).as_i64x2()); transmute(simd_select_bitmask(k, shf, src.as_i64x2())) } @@ -1073,15 +1082,16 @@ pub unsafe fn _mm_mask_shldi_epi64( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_shldi_epi64&expand=5053) #[inline] #[target_feature(enable = "avx512vbmi2,avx512vl")] -#[cfg_attr(test, assert_instr(vpshldq, imm8 = 5))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm_maskz_shldi_epi64(k: __mmask8, a: __m128i, b: __m128i, imm8: i32) -> __m128i { - assert!(imm8 >= 0 && imm8 <= 255); - let shf: i64x2 = vpshldvq128( - a.as_i64x2(), - b.as_i64x2(), - _mm_set1_epi64x(imm8 as i64).as_i64x2(), - ); +#[cfg_attr(test, assert_instr(vpshldq, IMM8 = 5))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm_maskz_shldi_epi64( + k: __mmask8, + a: __m128i, + b: __m128i, +) -> __m128i { + static_assert_imm8!(IMM8); + let imm8 = IMM8 as i64; + let shf: i64x2 = vpshldvq128(a.as_i64x2(), b.as_i64x2(), _mm_set1_epi64x(imm8).as_i64x2()); let zero = _mm_setzero_si128().as_i64x2(); transmute(simd_select_bitmask(k, shf, zero)) } @@ -1091,14 +1101,14 @@ pub unsafe fn _mm_maskz_shldi_epi64(k: __mmask8, a: __m128i, b: __m128i, imm8: i /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_shldi_epi32&expand=5051) #[inline] #[target_feature(enable = "avx512vbmi2")] -#[cfg_attr(test, assert_instr(vpshldd, imm8 = 5))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm512_shldi_epi32(a: __m512i, b: __m512i, imm8: i32) -> __m512i { - assert!(imm8 >= 0 && imm8 <= 255); +#[cfg_attr(test, assert_instr(vpshldd, IMM8 = 5))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_shldi_epi32(a: __m512i, b: __m512i) -> __m512i { + static_assert_imm8!(IMM8); transmute(vpshldvd( a.as_i32x16(), b.as_i32x16(), - _mm512_set1_epi32(imm8).as_i32x16(), + _mm512_set1_epi32(IMM8).as_i32x16(), )) } @@ -1107,20 +1117,19 @@ pub unsafe fn _mm512_shldi_epi32(a: __m512i, b: __m512i, imm8: i32) -> __m512i { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_shldi_epi32&expand=5049) #[inline] #[target_feature(enable = "avx512vbmi2")] -#[cfg_attr(test, assert_instr(vpshldd, imm8 = 5))] -#[rustc_args_required_const(4)] -pub unsafe fn _mm512_mask_shldi_epi32( +#[cfg_attr(test, assert_instr(vpshldd, IMM8 = 5))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_mask_shldi_epi32( src: __m512i, k: __mmask16, a: __m512i, b: __m512i, - imm8: i32, ) -> __m512i { - assert!(imm8 >= 0 && imm8 <= 255); + static_assert_imm8!(IMM8); let shf: i32x16 = vpshldvd( a.as_i32x16(), b.as_i32x16(), - _mm512_set1_epi32(imm8).as_i32x16(), + _mm512_set1_epi32(IMM8).as_i32x16(), ); transmute(simd_select_bitmask(k, shf, src.as_i32x16())) } @@ -1130,14 +1139,18 @@ pub unsafe fn _mm512_mask_shldi_epi32( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_shldi_epi32&expand=5050) #[inline] #[target_feature(enable = "avx512vbmi2")] -#[cfg_attr(test, assert_instr(vpshldd, imm8 = 5))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm512_maskz_shldi_epi32(k: __mmask16, a: __m512i, b: __m512i, imm8: i32) -> __m512i { - assert!(imm8 >= 0 && imm8 <= 255); +#[cfg_attr(test, assert_instr(vpshldd, IMM8 = 5))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_maskz_shldi_epi32( + k: __mmask16, + a: __m512i, + b: __m512i, +) -> __m512i { + static_assert_imm8!(IMM8); let shf: i32x16 = vpshldvd( a.as_i32x16(), b.as_i32x16(), - _mm512_set1_epi32(imm8).as_i32x16(), + _mm512_set1_epi32(IMM8).as_i32x16(), ); let zero = _mm512_setzero_si512().as_i32x16(); transmute(simd_select_bitmask(k, shf, zero)) @@ -1148,14 +1161,14 @@ pub unsafe fn _mm512_maskz_shldi_epi32(k: __mmask16, a: __m512i, b: __m512i, imm /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_shldi_epi32&expand=5048) #[inline] #[target_feature(enable = "avx512vbmi2,avx512vl")] -#[cfg_attr(test, assert_instr(vpshldd, imm8 = 5))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm256_shldi_epi32(a: __m256i, b: __m256i, imm8: i32) -> __m256i { - assert!(imm8 >= 0 && imm8 <= 255); +#[cfg_attr(test, assert_instr(vpshldd, IMM8 = 5))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm256_shldi_epi32(a: __m256i, b: __m256i) -> __m256i { + static_assert_imm8!(IMM8); transmute(vpshldvd256( a.as_i32x8(), b.as_i32x8(), - _mm256_set1_epi32(imm8).as_i32x8(), + _mm256_set1_epi32(IMM8).as_i32x8(), )) } @@ -1164,20 +1177,19 @@ pub unsafe fn _mm256_shldi_epi32(a: __m256i, b: __m256i, imm8: i32) -> __m256i { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_shldi_epi32&expand=5046) #[inline] #[target_feature(enable = "avx512vbmi2,avx512vl")] -#[cfg_attr(test, assert_instr(vpshldd, imm8 = 5))] -#[rustc_args_required_const(4)] -pub unsafe fn _mm256_mask_shldi_epi32( +#[cfg_attr(test, assert_instr(vpshldd, IMM8 = 5))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm256_mask_shldi_epi32( src: __m256i, k: __mmask8, a: __m256i, b: __m256i, - imm8: i32, ) -> __m256i { - assert!(imm8 >= 0 && imm8 <= 255); + static_assert_imm8!(IMM8); let shf: i32x8 = vpshldvd256( a.as_i32x8(), b.as_i32x8(), - _mm256_set1_epi32(imm8).as_i32x8(), + _mm256_set1_epi32(IMM8).as_i32x8(), ); transmute(simd_select_bitmask(k, shf, src.as_i32x8())) } @@ -1187,14 +1199,18 @@ pub unsafe fn _mm256_mask_shldi_epi32( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_shldi_epi32&expand=5047) #[inline] #[target_feature(enable = "avx512vbmi2,avx512vl")] -#[cfg_attr(test, assert_instr(vpshldd, imm8 = 5))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm256_maskz_shldi_epi32(k: __mmask8, a: __m256i, b: __m256i, imm8: i32) -> __m256i { - assert!(imm8 >= 0 && imm8 <= 255); +#[cfg_attr(test, assert_instr(vpshldd, IMM8 = 5))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm256_maskz_shldi_epi32( + k: __mmask8, + a: __m256i, + b: __m256i, +) -> __m256i { + static_assert_imm8!(IMM8); let shf: i32x8 = vpshldvd256( a.as_i32x8(), b.as_i32x8(), - _mm256_set1_epi32(imm8).as_i32x8(), + _mm256_set1_epi32(IMM8).as_i32x8(), ); let zero = _mm256_setzero_si256().as_i32x8(); transmute(simd_select_bitmask(k, shf, zero)) @@ -1205,14 +1221,14 @@ pub unsafe fn _mm256_maskz_shldi_epi32(k: __mmask8, a: __m256i, b: __m256i, imm8 /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_shldi_epi32&expand=5045) #[inline] #[target_feature(enable = "avx512vbmi2,avx512vl")] -#[cfg_attr(test, assert_instr(vpshldd, imm8 = 5))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm_shldi_epi32(a: __m128i, b: __m128i, imm8: i32) -> __m128i { - assert!(imm8 >= 0 && imm8 <= 255); +#[cfg_attr(test, assert_instr(vpshldd, IMM8 = 5))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm_shldi_epi32(a: __m128i, b: __m128i) -> __m128i { + static_assert_imm8!(IMM8); transmute(vpshldvd128( a.as_i32x4(), b.as_i32x4(), - _mm_set1_epi32(imm8).as_i32x4(), + _mm_set1_epi32(IMM8).as_i32x4(), )) } @@ -1221,17 +1237,16 @@ pub unsafe fn _mm_shldi_epi32(a: __m128i, b: __m128i, imm8: i32) -> __m128i { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_shldi_epi32&expand=5043) #[inline] #[target_feature(enable = "avx512vbmi2,avx512vl")] -#[cfg_attr(test, assert_instr(vpshldd, imm8 = 5))] -#[rustc_args_required_const(4)] -pub unsafe fn _mm_mask_shldi_epi32( +#[cfg_attr(test, assert_instr(vpshldd, IMM8 = 5))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm_mask_shldi_epi32( src: __m128i, k: __mmask8, a: __m128i, b: __m128i, - imm8: i32, ) -> __m128i { - assert!(imm8 >= 0 && imm8 <= 255); - let shf: i32x4 = vpshldvd128(a.as_i32x4(), b.as_i32x4(), _mm_set1_epi32(imm8).as_i32x4()); + static_assert_imm8!(IMM8); + let shf: i32x4 = vpshldvd128(a.as_i32x4(), b.as_i32x4(), _mm_set1_epi32(IMM8).as_i32x4()); transmute(simd_select_bitmask(k, shf, src.as_i32x4())) } @@ -1240,11 +1255,15 @@ pub unsafe fn _mm_mask_shldi_epi32( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_shldi_epi32&expand=5044) #[inline] #[target_feature(enable = "avx512vbmi2,avx512vl")] -#[cfg_attr(test, assert_instr(vpshldd, imm8 = 5))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm_maskz_shldi_epi32(k: __mmask8, a: __m128i, b: __m128i, imm8: i32) -> __m128i { - assert!(imm8 >= 0 && imm8 <= 255); - let shf: i32x4 = vpshldvd128(a.as_i32x4(), b.as_i32x4(), _mm_set1_epi32(imm8).as_i32x4()); +#[cfg_attr(test, assert_instr(vpshldd, IMM8 = 5))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm_maskz_shldi_epi32( + k: __mmask8, + a: __m128i, + b: __m128i, +) -> __m128i { + static_assert_imm8!(IMM8); + let shf: i32x4 = vpshldvd128(a.as_i32x4(), b.as_i32x4(), _mm_set1_epi32(IMM8).as_i32x4()); let zero = _mm_setzero_si128().as_i32x4(); transmute(simd_select_bitmask(k, shf, zero)) } @@ -1254,14 +1273,15 @@ pub unsafe fn _mm_maskz_shldi_epi32(k: __mmask8, a: __m128i, b: __m128i, imm8: i /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_shldi_epi16&expand=5042) #[inline] #[target_feature(enable = "avx512vbmi2")] -#[cfg_attr(test, assert_instr(vpshldw, imm8 = 5))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm512_shldi_epi16(a: __m512i, b: __m512i, imm8: i32) -> __m512i { - assert!(imm8 >= 0 && imm8 <= 255); +#[cfg_attr(test, assert_instr(vpshldw, IMM8 = 5))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_shldi_epi16(a: __m512i, b: __m512i) -> __m512i { + static_assert_imm8!(IMM8); + let imm8 = IMM8 as i16; transmute(vpshldvw( a.as_i16x32(), b.as_i16x32(), - _mm512_set1_epi16(imm8 as i16).as_i16x32(), + _mm512_set1_epi16(imm8).as_i16x32(), )) } @@ -1270,20 +1290,20 @@ pub unsafe fn _mm512_shldi_epi16(a: __m512i, b: __m512i, imm8: i32) -> __m512i { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_shldi_epi16&expand=5040) #[inline] #[target_feature(enable = "avx512vbmi2")] -#[cfg_attr(test, assert_instr(vpshldw, imm8 = 5))] -#[rustc_args_required_const(4)] -pub unsafe fn _mm512_mask_shldi_epi16( +#[cfg_attr(test, assert_instr(vpshldw, IMM8 = 5))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_mask_shldi_epi16( src: __m512i, k: __mmask32, a: __m512i, b: __m512i, - imm8: i32, ) -> __m512i { - assert!(imm8 >= 0 && imm8 <= 255); + static_assert_imm8!(IMM8); + let imm8 = IMM8 as i16; let shf: i16x32 = vpshldvw( a.as_i16x32(), b.as_i16x32(), - _mm512_set1_epi16(imm8 as i16).as_i16x32(), + _mm512_set1_epi16(imm8).as_i16x32(), ); transmute(simd_select_bitmask(k, shf, src.as_i16x32())) } @@ -1293,14 +1313,19 @@ pub unsafe fn _mm512_mask_shldi_epi16( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_shldi_epi16&expand=5041) #[inline] #[target_feature(enable = "avx512vbmi2")] -#[cfg_attr(test, assert_instr(vpshldw, imm8 = 5))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm512_maskz_shldi_epi16(k: __mmask32, a: __m512i, b: __m512i, imm8: i32) -> __m512i { - assert!(imm8 >= 0 && imm8 <= 255); +#[cfg_attr(test, assert_instr(vpshldw, IMM8 = 5))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_maskz_shldi_epi16( + k: __mmask32, + a: __m512i, + b: __m512i, +) -> __m512i { + static_assert_imm8!(IMM8); + let imm8 = IMM8 as i16; let shf: i16x32 = vpshldvw( a.as_i16x32(), b.as_i16x32(), - _mm512_set1_epi16(imm8 as i16).as_i16x32(), + _mm512_set1_epi16(imm8).as_i16x32(), ); let zero = _mm512_setzero_si512().as_i16x32(); transmute(simd_select_bitmask(k, shf, zero)) @@ -1311,14 +1336,15 @@ pub unsafe fn _mm512_maskz_shldi_epi16(k: __mmask32, a: __m512i, b: __m512i, imm /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_shldi_epi16&expand=5039) #[inline] #[target_feature(enable = "avx512vbmi2,avx512vl")] -#[cfg_attr(test, assert_instr(vpshldw, imm8 = 5))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm256_shldi_epi16(a: __m256i, b: __m256i, imm8: i32) -> __m256i { - assert!(imm8 >= 0 && imm8 <= 255); +#[cfg_attr(test, assert_instr(vpshldw, IMM8 = 5))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm256_shldi_epi16(a: __m256i, b: __m256i) -> __m256i { + static_assert_imm8!(IMM8); + let imm8 = IMM8 as i16; transmute(vpshldvw256( a.as_i16x16(), b.as_i16x16(), - _mm256_set1_epi16(imm8 as i16).as_i16x16(), + _mm256_set1_epi16(imm8).as_i16x16(), )) } @@ -1327,20 +1353,20 @@ pub unsafe fn _mm256_shldi_epi16(a: __m256i, b: __m256i, imm8: i32) -> __m256i { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_shldi_epi16&expand=5037) #[inline] #[target_feature(enable = "avx512vbmi2,avx512vl")] -#[cfg_attr(test, assert_instr(vpshldw, imm8 = 5))] -#[rustc_args_required_const(4)] -pub unsafe fn _mm256_mask_shldi_epi16( +#[cfg_attr(test, assert_instr(vpshldw, IMM8 = 5))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm256_mask_shldi_epi16( src: __m256i, k: __mmask16, a: __m256i, b: __m256i, - imm8: i32, ) -> __m256i { - assert!(imm8 >= 0 && imm8 <= 255); + static_assert_imm8!(IMM8); + let imm8 = IMM8 as i16; let shf: i16x16 = vpshldvw256( a.as_i16x16(), b.as_i16x16(), - _mm256_set1_epi16(imm8 as i16).as_i16x16(), + _mm256_set1_epi16(imm8).as_i16x16(), ); transmute(simd_select_bitmask(k, shf, src.as_i16x16())) } @@ -1350,13 +1376,19 @@ pub unsafe fn _mm256_mask_shldi_epi16( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_shldi_epi16&expand=5038) #[inline] #[target_feature(enable = "avx512vbmi2,avx512vl")] -#[cfg_attr(test, assert_instr(vpshldw, imm8 = 5))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm256_maskz_shldi_epi16(k: __mmask16, a: __m256i, b: __m256i, imm8: i32) -> __m256i { +#[cfg_attr(test, assert_instr(vpshldw, IMM8 = 5))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm256_maskz_shldi_epi16( + k: __mmask16, + a: __m256i, + b: __m256i, +) -> __m256i { + static_assert_imm8!(IMM8); + let imm8 = IMM8 as i16; let shf: i16x16 = vpshldvw256( a.as_i16x16(), b.as_i16x16(), - _mm256_set1_epi16(imm8 as i16).as_i16x16(), + _mm256_set1_epi16(imm8).as_i16x16(), ); let zero = _mm256_setzero_si256().as_i16x16(); transmute(simd_select_bitmask(k, shf, zero)) @@ -1367,13 +1399,15 @@ pub unsafe fn _mm256_maskz_shldi_epi16(k: __mmask16, a: __m256i, b: __m256i, imm /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_shldi_epi16&expand=5036) #[inline] #[target_feature(enable = "avx512vbmi2,avx512vl")] -#[cfg_attr(test, assert_instr(vpshldw, imm8 = 5))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm_shldi_epi16(a: __m128i, b: __m128i, imm8: i32) -> __m128i { +#[cfg_attr(test, assert_instr(vpshldw, IMM8 = 5))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm_shldi_epi16(a: __m128i, b: __m128i) -> __m128i { + static_assert_imm8!(IMM8); + let imm8 = IMM8 as i16; transmute(vpshldvw128( a.as_i16x8(), b.as_i16x8(), - _mm_set1_epi16(imm8 as i16).as_i16x8(), + _mm_set1_epi16(imm8).as_i16x8(), )) } @@ -1382,20 +1416,17 @@ pub unsafe fn _mm_shldi_epi16(a: __m128i, b: __m128i, imm8: i32) -> __m128i { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_shldi_epi16&expand=5034) #[inline] #[target_feature(enable = "avx512vbmi2,avx512vl")] -#[cfg_attr(test, assert_instr(vpshldw, imm8 = 5))] -#[rustc_args_required_const(4)] -pub unsafe fn _mm_mask_shldi_epi16( +#[cfg_attr(test, assert_instr(vpshldw, IMM8 = 5))] +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm_mask_shldi_epi16( src: __m128i, k: __mmask8, a: __m128i, b: __m128i, - imm8: i32, ) -> __m128i { - let shf: i16x8 = vpshldvw128( - a.as_i16x8(), - b.as_i16x8(), - _mm_set1_epi16(imm8 as i16).as_i16x8(), - ); + static_assert_imm8!(IMM8); + let imm8 = IMM8 as i16; + let shf: i16x8 = vpshldvw128(a.as_i16x8(), b.as_i16x8(), _mm_set1_epi16(imm8).as_i16x8()); transmute(simd_select_bitmask(k, shf, src.as_i16x8())) } @@ -1404,14 +1435,16 @@ pub unsafe fn _mm_mask_shldi_epi16( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_shldi_epi16&expand=5035) #[inline] #[target_feature(enable = "avx512vbmi2,avx512vl")] -#[cfg_attr(test, assert_instr(vpshldw, imm8 = 5))] -#[rustc_args_required_const(3)] -pub unsafe fn _mm_maskz_shldi_epi16(k: __mmask8, a: __m128i, b: __m128i, imm8: i32) -> __m128i { - let shf: i16x8 = vpshldvw128( - a.as_i16x8(), - b.as_i16x8(), - _mm_set1_epi16(imm8 as i16).as_i16x8(), - ); +#[cfg_attr(test, assert_instr(vpshldw, IMM8 = 5))] +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm_maskz_shldi_epi16( + k: __mmask8, + a: __m128i, + b: __m128i, +) -> __m128i { + static_assert_imm8!(IMM8); + let imm8 = IMM8 as i16; + let shf: i16x8 = vpshldvw128(a.as_i16x8(), b.as_i16x8(), _mm_set1_epi16(imm8).as_i16x8()); let zero = _mm_setzero_si128().as_i16x8(); transmute(simd_select_bitmask(k, shf, zero)) } @@ -1421,14 +1454,15 @@ pub unsafe fn _mm_maskz_shldi_epi16(k: __mmask8, a: __m128i, b: __m128i, imm8: i /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_shrdi_epi64&expand=5114) #[inline] #[target_feature(enable = "avx512vbmi2")] -#[cfg_attr(test, assert_instr(vpshldq, imm8 = 5))] //should be vpshrdq -#[rustc_args_required_const(2)] -pub unsafe fn _mm512_shrdi_epi64(a: __m512i, b: __m512i, imm8: i32) -> __m512i { - assert!(imm8 >= 0 && imm8 <= 255); +#[cfg_attr(test, assert_instr(vpshldq, IMM8 = 5))] //should be vpshrdq +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_shrdi_epi64(a: __m512i, b: __m512i) -> __m512i { + static_assert_imm8!(IMM8); + let imm8 = IMM8 as i64; transmute(vpshrdvq( a.as_i64x8(), b.as_i64x8(), - _mm512_set1_epi64(imm8 as i64).as_i64x8(), + _mm512_set1_epi64(imm8).as_i64x8(), )) } @@ -1437,20 +1471,20 @@ pub unsafe fn _mm512_shrdi_epi64(a: __m512i, b: __m512i, imm8: i32) -> __m512i { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_shrdi_epi64&expand=5112) #[inline] #[target_feature(enable = "avx512vbmi2")] -#[cfg_attr(test, assert_instr(vpshldq, imm8 = 5))] //should be vpshrdq -#[rustc_args_required_const(4)] -pub unsafe fn _mm512_mask_shrdi_epi64( +#[cfg_attr(test, assert_instr(vpshldq, IMM8 = 5))] //should be vpshrdq +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_mask_shrdi_epi64( src: __m512i, k: __mmask8, a: __m512i, b: __m512i, - imm8: i32, ) -> __m512i { - assert!(imm8 >= 0 && imm8 <= 255); + static_assert_imm8!(IMM8); + let imm8 = IMM8 as i64; let shf: i64x8 = vpshrdvq( a.as_i64x8(), b.as_i64x8(), - _mm512_set1_epi64(imm8 as i64).as_i64x8(), + _mm512_set1_epi64(imm8).as_i64x8(), ); transmute(simd_select_bitmask(k, shf, src.as_i64x8())) } @@ -1460,14 +1494,19 @@ pub unsafe fn _mm512_mask_shrdi_epi64( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_shrdi_epi64&expand=5113) #[inline] #[target_feature(enable = "avx512vbmi2")] -#[cfg_attr(test, assert_instr(vpshldq, imm8 = 255))] //should be vpshrdq -#[rustc_args_required_const(3)] -pub unsafe fn _mm512_maskz_shrdi_epi64(k: __mmask8, a: __m512i, b: __m512i, imm8: i32) -> __m512i { - assert!(imm8 >= 0 && imm8 <= 255); +#[cfg_attr(test, assert_instr(vpshldq, IMM8 = 255))] //should be vpshrdq +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_maskz_shrdi_epi64( + k: __mmask8, + a: __m512i, + b: __m512i, +) -> __m512i { + static_assert_imm8!(IMM8); + let imm8 = IMM8 as i64; let shf: i64x8 = vpshrdvq( a.as_i64x8(), b.as_i64x8(), - _mm512_set1_epi64(imm8 as i64).as_i64x8(), + _mm512_set1_epi64(imm8).as_i64x8(), ); let zero = _mm512_setzero_si512().as_i64x8(); transmute(simd_select_bitmask(k, shf, zero)) @@ -1478,14 +1517,15 @@ pub unsafe fn _mm512_maskz_shrdi_epi64(k: __mmask8, a: __m512i, b: __m512i, imm8 /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_shrdi_epi64&expand=5111) #[inline] #[target_feature(enable = "avx512vbmi2,avx512vl")] -#[cfg_attr(test, assert_instr(vpshldq, imm8 = 5))] //should be vpshrdq -#[rustc_args_required_const(2)] -pub unsafe fn _mm256_shrdi_epi64(a: __m256i, b: __m256i, imm8: i32) -> __m256i { - assert!(imm8 >= 0 && imm8 <= 255); +#[cfg_attr(test, assert_instr(vpshldq, IMM8 = 5))] //should be vpshrdq +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm256_shrdi_epi64(a: __m256i, b: __m256i) -> __m256i { + static_assert_imm8!(IMM8); + let imm8 = IMM8 as i64; transmute(vpshrdvq256( a.as_i64x4(), b.as_i64x4(), - _mm256_set1_epi64x(imm8 as i64).as_i64x4(), + _mm256_set1_epi64x(imm8).as_i64x4(), )) } @@ -1494,20 +1534,20 @@ pub unsafe fn _mm256_shrdi_epi64(a: __m256i, b: __m256i, imm8: i32) -> __m256i { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_shrdi_epi64&expand=5109) #[inline] #[target_feature(enable = "avx512vbmi2,avx512vl")] -#[cfg_attr(test, assert_instr(vpshldq, imm8 = 5))] //should be vpshrdq -#[rustc_args_required_const(4)] -pub unsafe fn _mm256_mask_shrdi_epi64( +#[cfg_attr(test, assert_instr(vpshldq, IMM8 = 5))] //should be vpshrdq +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm256_mask_shrdi_epi64( src: __m256i, k: __mmask8, a: __m256i, b: __m256i, - imm8: i32, ) -> __m256i { - assert!(imm8 >= 0 && imm8 <= 255); + static_assert_imm8!(IMM8); + let imm8 = IMM8 as i64; let shf: i64x4 = vpshrdvq256( a.as_i64x4(), b.as_i64x4(), - _mm256_set1_epi64x(imm8 as i64).as_i64x4(), + _mm256_set1_epi64x(imm8).as_i64x4(), ); transmute(simd_select_bitmask(k, shf, src.as_i64x4())) } @@ -1517,14 +1557,19 @@ pub unsafe fn _mm256_mask_shrdi_epi64( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_shrdi_epi64&expand=5110) #[inline] #[target_feature(enable = "avx512vbmi2,avx512vl")] -#[cfg_attr(test, assert_instr(vpshldq, imm8 = 5))] //should be vpshrdq -#[rustc_args_required_const(3)] -pub unsafe fn _mm256_maskz_shrdi_epi64(k: __mmask8, a: __m256i, b: __m256i, imm8: i32) -> __m256i { - assert!(imm8 >= 0 && imm8 <= 255); +#[cfg_attr(test, assert_instr(vpshldq, IMM8 = 5))] //should be vpshrdq +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm256_maskz_shrdi_epi64( + k: __mmask8, + a: __m256i, + b: __m256i, +) -> __m256i { + static_assert_imm8!(IMM8); + let imm8 = IMM8 as i64; let shf: i64x4 = vpshrdvq256( a.as_i64x4(), b.as_i64x4(), - _mm256_set1_epi64x(imm8 as i64).as_i64x4(), + _mm256_set1_epi64x(imm8).as_i64x4(), ); let zero = _mm256_setzero_si256().as_i64x4(); transmute(simd_select_bitmask(k, shf, zero)) @@ -1535,14 +1580,15 @@ pub unsafe fn _mm256_maskz_shrdi_epi64(k: __mmask8, a: __m256i, b: __m256i, imm8 /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_shrdi_epi64&expand=5108) #[inline] #[target_feature(enable = "avx512vbmi2,avx512vl")] -#[cfg_attr(test, assert_instr(vpshldq, imm8 = 5))] //should be vpshrdq -#[rustc_args_required_const(2)] -pub unsafe fn _mm_shrdi_epi64(a: __m128i, b: __m128i, imm8: i32) -> __m128i { - assert!(imm8 >= 0 && imm8 <= 255); +#[cfg_attr(test, assert_instr(vpshldq, IMM8 = 5))] //should be vpshrdq +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm_shrdi_epi64(a: __m128i, b: __m128i) -> __m128i { + static_assert_imm8!(IMM8); + let imm8 = IMM8 as i64; transmute(vpshrdvq128( a.as_i64x2(), b.as_i64x2(), - _mm_set1_epi64x(imm8 as i64).as_i64x2(), + _mm_set1_epi64x(imm8).as_i64x2(), )) } @@ -1551,21 +1597,17 @@ pub unsafe fn _mm_shrdi_epi64(a: __m128i, b: __m128i, imm8: i32) -> __m128i { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_shrdi_epi64&expand=5106) #[inline] #[target_feature(enable = "avx512vbmi2,avx512vl")] -#[cfg_attr(test, assert_instr(vpshldq, imm8 = 5))] //should be vpshrdq -#[rustc_args_required_const(4)] -pub unsafe fn _mm_mask_shrdi_epi64( +#[cfg_attr(test, assert_instr(vpshldq, IMM8 = 5))] //should be vpshrdq +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm_mask_shrdi_epi64( src: __m128i, k: __mmask8, a: __m128i, b: __m128i, - imm8: i32, ) -> __m128i { - assert!(imm8 >= 0 && imm8 <= 255); - let shf: i64x2 = vpshrdvq128( - a.as_i64x2(), - b.as_i64x2(), - _mm_set1_epi64x(imm8 as i64).as_i64x2(), - ); + static_assert_imm8!(IMM8); + let imm8 = IMM8 as i64; + let shf: i64x2 = vpshrdvq128(a.as_i64x2(), b.as_i64x2(), _mm_set1_epi64x(imm8).as_i64x2()); transmute(simd_select_bitmask(k, shf, src.as_i64x2())) } @@ -1574,15 +1616,16 @@ pub unsafe fn _mm_mask_shrdi_epi64( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_shrdi_epi64&expand=5107) #[inline] #[target_feature(enable = "avx512vbmi2,avx512vl")] -#[cfg_attr(test, assert_instr(vpshldq, imm8 = 5))] //should be vpshrdq -#[rustc_args_required_const(3)] -pub unsafe fn _mm_maskz_shrdi_epi64(k: __mmask8, a: __m128i, b: __m128i, imm8: i32) -> __m128i { - assert!(imm8 >= 0 && imm8 <= 255); - let shf: i64x2 = vpshrdvq128( - a.as_i64x2(), - b.as_i64x2(), - _mm_set1_epi64x(imm8 as i64).as_i64x2(), - ); +#[cfg_attr(test, assert_instr(vpshldq, IMM8 = 5))] //should be vpshrdq +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm_maskz_shrdi_epi64( + k: __mmask8, + a: __m128i, + b: __m128i, +) -> __m128i { + static_assert_imm8!(IMM8); + let imm8 = IMM8 as i64; + let shf: i64x2 = vpshrdvq128(a.as_i64x2(), b.as_i64x2(), _mm_set1_epi64x(imm8).as_i64x2()); let zero = _mm_setzero_si128().as_i64x2(); transmute(simd_select_bitmask(k, shf, zero)) } @@ -1592,14 +1635,14 @@ pub unsafe fn _mm_maskz_shrdi_epi64(k: __mmask8, a: __m128i, b: __m128i, imm8: i /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_shrdi_epi32&expand=5105) #[inline] #[target_feature(enable = "avx512vbmi2")] -#[cfg_attr(test, assert_instr(vpshldd, imm8 = 5))] //should be vpshldd -#[rustc_args_required_const(2)] -pub unsafe fn _mm512_shrdi_epi32(a: __m512i, b: __m512i, imm8: i32) -> __m512i { - assert!(imm8 >= 0 && imm8 <= 255); +#[cfg_attr(test, assert_instr(vpshldd, IMM8 = 5))] //should be vpshldd +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_shrdi_epi32(a: __m512i, b: __m512i) -> __m512i { + static_assert_imm8!(IMM8); transmute(vpshrdvd( a.as_i32x16(), b.as_i32x16(), - _mm512_set1_epi32(imm8).as_i32x16(), + _mm512_set1_epi32(IMM8).as_i32x16(), )) } @@ -1608,20 +1651,19 @@ pub unsafe fn _mm512_shrdi_epi32(a: __m512i, b: __m512i, imm8: i32) -> __m512i { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_shrdi_epi32&expand=5103) #[inline] #[target_feature(enable = "avx512vbmi2")] -#[cfg_attr(test, assert_instr(vpshldd, imm8 = 5))] //should be vpshldd -#[rustc_args_required_const(4)] -pub unsafe fn _mm512_mask_shrdi_epi32( +#[cfg_attr(test, assert_instr(vpshldd, IMM8 = 5))] //should be vpshldd +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_mask_shrdi_epi32( src: __m512i, k: __mmask16, a: __m512i, b: __m512i, - imm8: i32, ) -> __m512i { - assert!(imm8 >= 0 && imm8 <= 255); + static_assert_imm8!(IMM8); let shf: i32x16 = vpshrdvd( a.as_i32x16(), b.as_i32x16(), - _mm512_set1_epi32(imm8).as_i32x16(), + _mm512_set1_epi32(IMM8).as_i32x16(), ); transmute(simd_select_bitmask(k, shf, src.as_i32x16())) } @@ -1631,14 +1673,18 @@ pub unsafe fn _mm512_mask_shrdi_epi32( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_shrdi_epi32&expand=5104) #[inline] #[target_feature(enable = "avx512vbmi2")] -#[cfg_attr(test, assert_instr(vpshldd, imm8 = 5))] //should be vpshldd -#[rustc_args_required_const(3)] -pub unsafe fn _mm512_maskz_shrdi_epi32(k: __mmask16, a: __m512i, b: __m512i, imm8: i32) -> __m512i { - assert!(imm8 >= 0 && imm8 <= 255); +#[cfg_attr(test, assert_instr(vpshldd, IMM8 = 5))] //should be vpshldd +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_maskz_shrdi_epi32( + k: __mmask16, + a: __m512i, + b: __m512i, +) -> __m512i { + static_assert_imm8!(IMM8); let shf: i32x16 = vpshrdvd( a.as_i32x16(), b.as_i32x16(), - _mm512_set1_epi32(imm8).as_i32x16(), + _mm512_set1_epi32(IMM8).as_i32x16(), ); let zero = _mm512_setzero_si512().as_i32x16(); transmute(simd_select_bitmask(k, shf, zero)) @@ -1649,14 +1695,14 @@ pub unsafe fn _mm512_maskz_shrdi_epi32(k: __mmask16, a: __m512i, b: __m512i, imm /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_shrdi_epi32&expand=5102) #[inline] #[target_feature(enable = "avx512vbmi2,avx512vl")] -#[cfg_attr(test, assert_instr(vpshldd, imm8 = 5))] //should be vpshldd -#[rustc_args_required_const(2)] -pub unsafe fn _mm256_shrdi_epi32(a: __m256i, b: __m256i, imm8: i32) -> __m256i { - assert!(imm8 >= 0 && imm8 <= 255); +#[cfg_attr(test, assert_instr(vpshldd, IMM8 = 5))] //should be vpshldd +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm256_shrdi_epi32(a: __m256i, b: __m256i) -> __m256i { + static_assert_imm8!(IMM8); transmute(vpshrdvd256( a.as_i32x8(), b.as_i32x8(), - _mm256_set1_epi32(imm8).as_i32x8(), + _mm256_set1_epi32(IMM8).as_i32x8(), )) } @@ -1665,20 +1711,19 @@ pub unsafe fn _mm256_shrdi_epi32(a: __m256i, b: __m256i, imm8: i32) -> __m256i { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_shrdi_epi32&expand=5100) #[inline] #[target_feature(enable = "avx512vbmi2,avx512vl")] -#[cfg_attr(test, assert_instr(vpshldd, imm8 = 5))] //should be vpshldd -#[rustc_args_required_const(4)] -pub unsafe fn _mm256_mask_shrdi_epi32( +#[cfg_attr(test, assert_instr(vpshldd, IMM8 = 5))] //should be vpshldd +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm256_mask_shrdi_epi32( src: __m256i, k: __mmask8, a: __m256i, b: __m256i, - imm8: i32, ) -> __m256i { - assert!(imm8 >= 0 && imm8 <= 255); + static_assert_imm8!(IMM8); let shf: i32x8 = vpshrdvd256( a.as_i32x8(), b.as_i32x8(), - _mm256_set1_epi32(imm8).as_i32x8(), + _mm256_set1_epi32(IMM8).as_i32x8(), ); transmute(simd_select_bitmask(k, shf, src.as_i32x8())) } @@ -1688,14 +1733,18 @@ pub unsafe fn _mm256_mask_shrdi_epi32( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_shrdi_epi32&expand=5101) #[inline] #[target_feature(enable = "avx512vbmi2,avx512vl")] -#[cfg_attr(test, assert_instr(vpshldd, imm8 = 5))] //should be vpshldd -#[rustc_args_required_const(3)] -pub unsafe fn _mm256_maskz_shrdi_epi32(k: __mmask8, a: __m256i, b: __m256i, imm8: i32) -> __m256i { - assert!(imm8 >= 0 && imm8 <= 255); +#[cfg_attr(test, assert_instr(vpshldd, IMM8 = 5))] //should be vpshldd +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm256_maskz_shrdi_epi32( + k: __mmask8, + a: __m256i, + b: __m256i, +) -> __m256i { + static_assert_imm8!(IMM8); let shf: i32x8 = vpshrdvd256( a.as_i32x8(), b.as_i32x8(), - _mm256_set1_epi32(imm8).as_i32x8(), + _mm256_set1_epi32(IMM8).as_i32x8(), ); let zero = _mm256_setzero_si256().as_i32x8(); transmute(simd_select_bitmask(k, shf, zero)) @@ -1706,14 +1755,14 @@ pub unsafe fn _mm256_maskz_shrdi_epi32(k: __mmask8, a: __m256i, b: __m256i, imm8 /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_shrdi_epi32&expand=5099) #[inline] #[target_feature(enable = "avx512vbmi2,avx512vl")] -#[cfg_attr(test, assert_instr(vpshldd, imm8 = 5))] //should be vpshldd -#[rustc_args_required_const(2)] -pub unsafe fn _mm_shrdi_epi32(a: __m128i, b: __m128i, imm8: i32) -> __m128i { - assert!(imm8 >= 0 && imm8 <= 255); +#[cfg_attr(test, assert_instr(vpshldd, IMM8 = 5))] //should be vpshldd +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm_shrdi_epi32(a: __m128i, b: __m128i) -> __m128i { + static_assert_imm8!(IMM8); transmute(vpshrdvd128( a.as_i32x4(), b.as_i32x4(), - _mm_set1_epi32(imm8).as_i32x4(), + _mm_set1_epi32(IMM8).as_i32x4(), )) } @@ -1722,17 +1771,16 @@ pub unsafe fn _mm_shrdi_epi32(a: __m128i, b: __m128i, imm8: i32) -> __m128i { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_shrdi_epi32&expand=5097) #[inline] #[target_feature(enable = "avx512vbmi2,avx512vl")] -#[cfg_attr(test, assert_instr(vpshldd, imm8 = 5))] //should be vpshldd -#[rustc_args_required_const(4)] -pub unsafe fn _mm_mask_shrdi_epi32( +#[cfg_attr(test, assert_instr(vpshldd, IMM8 = 5))] //should be vpshldd +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm_mask_shrdi_epi32( src: __m128i, k: __mmask8, a: __m128i, b: __m128i, - imm8: i32, ) -> __m128i { - assert!(imm8 >= 0 && imm8 <= 255); - let shf: i32x4 = vpshrdvd128(a.as_i32x4(), b.as_i32x4(), _mm_set1_epi32(imm8).as_i32x4()); + static_assert_imm8!(IMM8); + let shf: i32x4 = vpshrdvd128(a.as_i32x4(), b.as_i32x4(), _mm_set1_epi32(IMM8).as_i32x4()); transmute(simd_select_bitmask(k, shf, src.as_i32x4())) } @@ -1741,11 +1789,15 @@ pub unsafe fn _mm_mask_shrdi_epi32( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_shrdi_epi32&expand=5098) #[inline] #[target_feature(enable = "avx512vbmi2,avx512vl")] -#[cfg_attr(test, assert_instr(vpshldd, imm8 = 5))] //should be vpshldd -#[rustc_args_required_const(3)] -pub unsafe fn _mm_maskz_shrdi_epi32(k: __mmask8, a: __m128i, b: __m128i, imm8: i32) -> __m128i { - assert!(imm8 >= 0 && imm8 <= 255); - let shf: i32x4 = vpshrdvd128(a.as_i32x4(), b.as_i32x4(), _mm_set1_epi32(imm8).as_i32x4()); +#[cfg_attr(test, assert_instr(vpshldd, IMM8 = 5))] //should be vpshldd +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm_maskz_shrdi_epi32( + k: __mmask8, + a: __m128i, + b: __m128i, +) -> __m128i { + static_assert_imm8!(IMM8); + let shf: i32x4 = vpshrdvd128(a.as_i32x4(), b.as_i32x4(), _mm_set1_epi32(IMM8).as_i32x4()); let zero = _mm_setzero_si128().as_i32x4(); transmute(simd_select_bitmask(k, shf, zero)) } @@ -1755,14 +1807,16 @@ pub unsafe fn _mm_maskz_shrdi_epi32(k: __mmask8, a: __m128i, b: __m128i, imm8: i /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_shrdi_epi16&expand=5096) #[inline] #[target_feature(enable = "avx512vbmi2")] -#[cfg_attr(test, assert_instr(vpshldw, imm8 = 5))] //should be vpshrdw -#[rustc_args_required_const(2)] -pub unsafe fn _mm512_shrdi_epi16(a: __m512i, b: __m512i, imm8: i32) -> __m512i { +#[cfg_attr(test, assert_instr(vpshldw, IMM8 = 5))] //should be vpshrdw +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_shrdi_epi16(a: __m512i, b: __m512i) -> __m512i { + static_assert_imm8!(IMM8); + let imm8 = IMM8 as i16; assert!(imm8 >= 0 && imm8 <= 255); transmute(vpshrdvw( a.as_i16x32(), b.as_i16x32(), - _mm512_set1_epi16(imm8 as i16).as_i16x32(), + _mm512_set1_epi16(imm8).as_i16x32(), )) } @@ -1771,20 +1825,21 @@ pub unsafe fn _mm512_shrdi_epi16(a: __m512i, b: __m512i, imm8: i32) -> __m512i { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_mask_shrdi_epi16&expand=5094) #[inline] #[target_feature(enable = "avx512vbmi2")] -#[cfg_attr(test, assert_instr(vpshldw, imm8 = 5))] //should be vpshrdw -#[rustc_args_required_const(4)] -pub unsafe fn _mm512_mask_shrdi_epi16( +#[cfg_attr(test, assert_instr(vpshldw, IMM8 = 5))] //should be vpshrdw +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm512_mask_shrdi_epi16( src: __m512i, k: __mmask32, a: __m512i, b: __m512i, - imm8: i32, ) -> __m512i { + static_assert_imm8!(IMM8); + let imm8 = IMM8 as i16; assert!(imm8 >= 0 && imm8 <= 255); let shf: i16x32 = vpshrdvw( a.as_i16x32(), b.as_i16x32(), - _mm512_set1_epi16(imm8 as i16).as_i16x32(), + _mm512_set1_epi16(imm8).as_i16x32(), ); transmute(simd_select_bitmask(k, shf, src.as_i16x32())) } @@ -1794,14 +1849,20 @@ pub unsafe fn _mm512_mask_shrdi_epi16( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm512_maskz_shrdi_epi16&expand=5095) #[inline] #[target_feature(enable = "avx512vbmi2")] -#[cfg_attr(test, assert_instr(vpshldw, imm8 = 5))] //should be vpshrdw -#[rustc_args_required_const(3)] -pub unsafe fn _mm512_maskz_shrdi_epi16(k: __mmask32, a: __m512i, b: __m512i, imm8: i32) -> __m512i { +#[cfg_attr(test, assert_instr(vpshldw, IMM8 = 5))] //should be vpshrdw +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm512_maskz_shrdi_epi16( + k: __mmask32, + a: __m512i, + b: __m512i, +) -> __m512i { + static_assert_imm8!(IMM8); + let imm8 = IMM8 as i16; assert!(imm8 >= 0 && imm8 <= 255); let shf: i16x32 = vpshrdvw( a.as_i16x32(), b.as_i16x32(), - _mm512_set1_epi16(imm8 as i16).as_i16x32(), + _mm512_set1_epi16(imm8).as_i16x32(), ); let zero = _mm512_setzero_si512().as_i16x32(); transmute(simd_select_bitmask(k, shf, zero)) @@ -1812,14 +1873,16 @@ pub unsafe fn _mm512_maskz_shrdi_epi16(k: __mmask32, a: __m512i, b: __m512i, imm /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_shrdi_epi16&expand=5093) #[inline] #[target_feature(enable = "avx512vbmi2,avx512vl")] -#[cfg_attr(test, assert_instr(vpshldw, imm8 = 5))] //should be vpshrdw -#[rustc_args_required_const(2)] -pub unsafe fn _mm256_shrdi_epi16(a: __m256i, b: __m256i, imm8: i32) -> __m256i { +#[cfg_attr(test, assert_instr(vpshldw, IMM8 = 5))] //should be vpshrdw +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm256_shrdi_epi16(a: __m256i, b: __m256i) -> __m256i { + static_assert_imm8!(IMM8); + let imm8 = IMM8 as i16; assert!(imm8 >= 0 && imm8 <= 255); transmute(vpshrdvw256( a.as_i16x16(), b.as_i16x16(), - _mm256_set1_epi16(imm8 as i16).as_i16x16(), + _mm256_set1_epi16(imm8).as_i16x16(), )) } @@ -1828,20 +1891,21 @@ pub unsafe fn _mm256_shrdi_epi16(a: __m256i, b: __m256i, imm8: i32) -> __m256i { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_mask_shrdi_epi16&expand=5091) #[inline] #[target_feature(enable = "avx512vbmi2,avx512vl")] -#[cfg_attr(test, assert_instr(vpshldw, imm8 = 5))] //should be vpshrdw -#[rustc_args_required_const(4)] -pub unsafe fn _mm256_mask_shrdi_epi16( +#[cfg_attr(test, assert_instr(vpshldw, IMM8 = 5))] //should be vpshrdw +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm256_mask_shrdi_epi16( src: __m256i, k: __mmask16, a: __m256i, b: __m256i, - imm8: i32, ) -> __m256i { + static_assert_imm8!(IMM8); + let imm8 = IMM8 as i16; assert!(imm8 >= 0 && imm8 <= 255); let shf: i16x16 = vpshrdvw256( a.as_i16x16(), b.as_i16x16(), - _mm256_set1_epi16(imm8 as i16).as_i16x16(), + _mm256_set1_epi16(imm8).as_i16x16(), ); transmute(simd_select_bitmask(k, shf, src.as_i16x16())) } @@ -1851,13 +1915,19 @@ pub unsafe fn _mm256_mask_shrdi_epi16( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_maskz_shrdi_epi16&expand=5092) #[inline] #[target_feature(enable = "avx512vbmi2,avx512vl")] -#[cfg_attr(test, assert_instr(vpshldw, imm8 = 5))] //should be vpshrdw -#[rustc_args_required_const(3)] -pub unsafe fn _mm256_maskz_shrdi_epi16(k: __mmask16, a: __m256i, b: __m256i, imm8: i32) -> __m256i { +#[cfg_attr(test, assert_instr(vpshldw, IMM8 = 5))] //should be vpshrdw +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm256_maskz_shrdi_epi16( + k: __mmask16, + a: __m256i, + b: __m256i, +) -> __m256i { + static_assert_imm8!(IMM8); + let imm8 = IMM8 as i16; let shf: i16x16 = vpshrdvw256( a.as_i16x16(), b.as_i16x16(), - _mm256_set1_epi16(imm8 as i16).as_i16x16(), + _mm256_set1_epi16(imm8).as_i16x16(), ); let zero = _mm256_setzero_si256().as_i16x16(); transmute(simd_select_bitmask(k, shf, zero)) @@ -1868,13 +1938,15 @@ pub unsafe fn _mm256_maskz_shrdi_epi16(k: __mmask16, a: __m256i, b: __m256i, imm /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_shrdi_epi16&expand=5090) #[inline] #[target_feature(enable = "avx512vbmi2,avx512vl")] -#[cfg_attr(test, assert_instr(vpshldw, imm8 = 5))] //should be vpshrdw -#[rustc_args_required_const(2)] -pub unsafe fn _mm_shrdi_epi16(a: __m128i, b: __m128i, imm8: i32) -> __m128i { +#[cfg_attr(test, assert_instr(vpshldw, IMM8 = 5))] //should be vpshrdw +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm_shrdi_epi16(a: __m128i, b: __m128i) -> __m128i { + static_assert_imm8!(IMM8); + let imm8 = IMM8 as i16; transmute(vpshrdvw128( a.as_i16x8(), b.as_i16x8(), - _mm_set1_epi16(imm8 as i16).as_i16x8(), + _mm_set1_epi16(imm8).as_i16x8(), )) } @@ -1883,20 +1955,17 @@ pub unsafe fn _mm_shrdi_epi16(a: __m128i, b: __m128i, imm8: i32) -> __m128i { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mask_shrdi_epi16&expand=5088) #[inline] #[target_feature(enable = "avx512vbmi2,avx512vl")] -#[cfg_attr(test, assert_instr(vpshldw, imm8 = 5))] //should be vpshrdw -#[rustc_args_required_const(4)] -pub unsafe fn _mm_mask_shrdi_epi16( +#[cfg_attr(test, assert_instr(vpshldw, IMM8 = 5))] //should be vpshrdw +#[rustc_legacy_const_generics(4)] +pub unsafe fn _mm_mask_shrdi_epi16( src: __m128i, k: __mmask8, a: __m128i, b: __m128i, - imm8: i32, ) -> __m128i { - let shf: i16x8 = vpshrdvw128( - a.as_i16x8(), - b.as_i16x8(), - _mm_set1_epi16(imm8 as i16).as_i16x8(), - ); + static_assert_imm8!(IMM8); + let imm8 = IMM8 as i16; + let shf: i16x8 = vpshrdvw128(a.as_i16x8(), b.as_i16x8(), _mm_set1_epi16(imm8).as_i16x8()); transmute(simd_select_bitmask(k, shf, src.as_i16x8())) } @@ -1905,14 +1974,16 @@ pub unsafe fn _mm_mask_shrdi_epi16( /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_maskz_shrdi_epi16&expand=5089) #[inline] #[target_feature(enable = "avx512vbmi2,avx512vl")] -#[cfg_attr(test, assert_instr(vpshldw, imm8 = 5))] //should be vpshrdw -#[rustc_args_required_const(3)] -pub unsafe fn _mm_maskz_shrdi_epi16(k: __mmask8, a: __m128i, b: __m128i, imm8: i32) -> __m128i { - let shf: i16x8 = vpshrdvw128( - a.as_i16x8(), - b.as_i16x8(), - _mm_set1_epi16(imm8 as i16).as_i16x8(), - ); +#[cfg_attr(test, assert_instr(vpshldw, IMM8 = 5))] //should be vpshrdw +#[rustc_legacy_const_generics(3)] +pub unsafe fn _mm_maskz_shrdi_epi16( + k: __mmask8, + a: __m128i, + b: __m128i, +) -> __m128i { + static_assert_imm8!(IMM8); + let imm8 = IMM8 as i16; + let shf: i16x8 = vpshrdvw128(a.as_i16x8(), b.as_i16x8(), _mm_set1_epi16(imm8).as_i16x8()); let zero = _mm_setzero_si128().as_i16x8(); transmute(simd_select_bitmask(k, shf, zero)) } @@ -2921,7 +2992,7 @@ mod tests { unsafe fn test_mm512_shldi_epi64() { let a = _mm512_set1_epi64(1); let b = _mm512_set1_epi64(1 << 63); - let r = _mm512_shldi_epi64(a, b, 2); + let r = _mm512_shldi_epi64::<2>(a, b); let e = _mm512_set1_epi64(6); assert_eq_m512i(r, e); } @@ -2930,9 +3001,9 @@ mod tests { unsafe fn test_mm512_mask_shldi_epi64() { let a = _mm512_set1_epi64(1); let b = _mm512_set1_epi64(1 << 63); - let r = _mm512_mask_shldi_epi64(a, 0, a, b, 2); + let r = _mm512_mask_shldi_epi64::<2>(a, 0, a, b); assert_eq_m512i(r, a); - let r = _mm512_mask_shldi_epi64(a, 0b11111111, a, b, 2); + let r = _mm512_mask_shldi_epi64::<2>(a, 0b11111111, a, b); let e = _mm512_set1_epi64(6); assert_eq_m512i(r, e); } @@ -2941,9 +3012,9 @@ mod tests { unsafe fn test_mm512_maskz_shldi_epi64() { let a = _mm512_set1_epi64(1); let b = _mm512_set1_epi64(1 << 63); - let r = _mm512_maskz_shldi_epi64(0, a, b, 2); + let r = _mm512_maskz_shldi_epi64::<2>(0, a, b); assert_eq_m512i(r, _mm512_setzero_si512()); - let r = _mm512_maskz_shldi_epi64(0b11111111, a, b, 2); + let r = _mm512_maskz_shldi_epi64::<2>(0b11111111, a, b); let e = _mm512_set1_epi64(6); assert_eq_m512i(r, e); } @@ -2952,7 +3023,7 @@ mod tests { unsafe fn test_mm256_shldi_epi64() { let a = _mm256_set1_epi64x(1); let b = _mm256_set1_epi64x(1 << 63); - let r = _mm256_shldi_epi64(a, b, 2); + let r = _mm256_shldi_epi64::<2>(a, b); let e = _mm256_set1_epi64x(6); assert_eq_m256i(r, e); } @@ -2961,9 +3032,9 @@ mod tests { unsafe fn test_mm256_mask_shldi_epi64() { let a = _mm256_set1_epi64x(1); let b = _mm256_set1_epi64x(1 << 63); - let r = _mm256_mask_shldi_epi64(a, 0, a, b, 2); + let r = _mm256_mask_shldi_epi64::<2>(a, 0, a, b); assert_eq_m256i(r, a); - let r = _mm256_mask_shldi_epi64(a, 0b00001111, a, b, 2); + let r = _mm256_mask_shldi_epi64::<2>(a, 0b00001111, a, b); let e = _mm256_set1_epi64x(6); assert_eq_m256i(r, e); } @@ -2972,9 +3043,9 @@ mod tests { unsafe fn test_mm256_maskz_shldi_epi64() { let a = _mm256_set1_epi64x(1); let b = _mm256_set1_epi64x(1 << 63); - let r = _mm256_maskz_shldi_epi64(0, a, b, 2); + let r = _mm256_maskz_shldi_epi64::<2>(0, a, b); assert_eq_m256i(r, _mm256_setzero_si256()); - let r = _mm256_maskz_shldi_epi64(0b00001111, a, b, 2); + let r = _mm256_maskz_shldi_epi64::<2>(0b00001111, a, b); let e = _mm256_set1_epi64x(6); assert_eq_m256i(r, e); } @@ -2983,7 +3054,7 @@ mod tests { unsafe fn test_mm_shldi_epi64() { let a = _mm_set1_epi64x(1); let b = _mm_set1_epi64x(1 << 63); - let r = _mm_shldi_epi64(a, b, 2); + let r = _mm_shldi_epi64::<2>(a, b); let e = _mm_set1_epi64x(6); assert_eq_m128i(r, e); } @@ -2992,9 +3063,9 @@ mod tests { unsafe fn test_mm_mask_shldi_epi64() { let a = _mm_set1_epi64x(1); let b = _mm_set1_epi64x(1 << 63); - let r = _mm_mask_shldi_epi64(a, 0, a, b, 2); + let r = _mm_mask_shldi_epi64::<2>(a, 0, a, b); assert_eq_m128i(r, a); - let r = _mm_mask_shldi_epi64(a, 0b00000011, a, b, 2); + let r = _mm_mask_shldi_epi64::<2>(a, 0b00000011, a, b); let e = _mm_set1_epi64x(6); assert_eq_m128i(r, e); } @@ -3003,9 +3074,9 @@ mod tests { unsafe fn test_mm_maskz_shldi_epi64() { let a = _mm_set1_epi64x(1); let b = _mm_set1_epi64x(1 << 63); - let r = _mm_maskz_shldi_epi64(0, a, b, 2); + let r = _mm_maskz_shldi_epi64::<2>(0, a, b); assert_eq_m128i(r, _mm_setzero_si128()); - let r = _mm_maskz_shldi_epi64(0b00000011, a, b, 2); + let r = _mm_maskz_shldi_epi64::<2>(0b00000011, a, b); let e = _mm_set1_epi64x(6); assert_eq_m128i(r, e); } @@ -3014,7 +3085,7 @@ mod tests { unsafe fn test_mm512_shldi_epi32() { let a = _mm512_set1_epi32(1); let b = _mm512_set1_epi32(1 << 31); - let r = _mm512_shldi_epi32(a, b, 2); + let r = _mm512_shldi_epi32::<2>(a, b); let e = _mm512_set1_epi32(6); assert_eq_m512i(r, e); } @@ -3023,9 +3094,9 @@ mod tests { unsafe fn test_mm512_mask_shldi_epi32() { let a = _mm512_set1_epi32(1); let b = _mm512_set1_epi32(1 << 31); - let r = _mm512_mask_shldi_epi32(a, 0, a, b, 2); + let r = _mm512_mask_shldi_epi32::<2>(a, 0, a, b); assert_eq_m512i(r, a); - let r = _mm512_mask_shldi_epi32(a, 0b11111111_11111111, a, b, 2); + let r = _mm512_mask_shldi_epi32::<2>(a, 0b11111111_11111111, a, b); let e = _mm512_set1_epi32(6); assert_eq_m512i(r, e); } @@ -3034,9 +3105,9 @@ mod tests { unsafe fn test_mm512_maskz_shldi_epi32() { let a = _mm512_set1_epi32(1); let b = _mm512_set1_epi32(1 << 31); - let r = _mm512_maskz_shldi_epi32(0, a, b, 2); + let r = _mm512_maskz_shldi_epi32::<2>(0, a, b); assert_eq_m512i(r, _mm512_setzero_si512()); - let r = _mm512_maskz_shldi_epi32(0b11111111_11111111, a, b, 2); + let r = _mm512_maskz_shldi_epi32::<2>(0b11111111_11111111, a, b); let e = _mm512_set1_epi32(6); assert_eq_m512i(r, e); } @@ -3045,7 +3116,7 @@ mod tests { unsafe fn test_mm256_shldi_epi32() { let a = _mm256_set1_epi32(1); let b = _mm256_set1_epi32(1 << 31); - let r = _mm256_shldi_epi32(a, b, 2); + let r = _mm256_shldi_epi32::<2>(a, b); let e = _mm256_set1_epi32(6); assert_eq_m256i(r, e); } @@ -3054,9 +3125,9 @@ mod tests { unsafe fn test_mm256_mask_shldi_epi32() { let a = _mm256_set1_epi32(1); let b = _mm256_set1_epi32(1 << 31); - let r = _mm256_mask_shldi_epi32(a, 0, a, b, 2); + let r = _mm256_mask_shldi_epi32::<2>(a, 0, a, b); assert_eq_m256i(r, a); - let r = _mm256_mask_shldi_epi32(a, 0b11111111, a, b, 2); + let r = _mm256_mask_shldi_epi32::<2>(a, 0b11111111, a, b); let e = _mm256_set1_epi32(6); assert_eq_m256i(r, e); } @@ -3065,9 +3136,9 @@ mod tests { unsafe fn test_mm256_maskz_shldi_epi32() { let a = _mm256_set1_epi32(1); let b = _mm256_set1_epi32(1 << 31); - let r = _mm256_maskz_shldi_epi32(0, a, b, 2); + let r = _mm256_maskz_shldi_epi32::<2>(0, a, b); assert_eq_m256i(r, _mm256_setzero_si256()); - let r = _mm256_maskz_shldi_epi32(0b11111111, a, b, 2); + let r = _mm256_maskz_shldi_epi32::<2>(0b11111111, a, b); let e = _mm256_set1_epi32(6); assert_eq_m256i(r, e); } @@ -3076,7 +3147,7 @@ mod tests { unsafe fn test_mm_shldi_epi32() { let a = _mm_set1_epi32(1); let b = _mm_set1_epi32(1 << 31); - let r = _mm_shldi_epi32(a, b, 2); + let r = _mm_shldi_epi32::<2>(a, b); let e = _mm_set1_epi32(6); assert_eq_m128i(r, e); } @@ -3085,9 +3156,9 @@ mod tests { unsafe fn test_mm_mask_shldi_epi32() { let a = _mm_set1_epi32(1); let b = _mm_set1_epi32(1 << 31); - let r = _mm_mask_shldi_epi32(a, 0, a, b, 2); + let r = _mm_mask_shldi_epi32::<2>(a, 0, a, b); assert_eq_m128i(r, a); - let r = _mm_mask_shldi_epi32(a, 0b00001111, a, b, 2); + let r = _mm_mask_shldi_epi32::<2>(a, 0b00001111, a, b); let e = _mm_set1_epi32(6); assert_eq_m128i(r, e); } @@ -3096,9 +3167,9 @@ mod tests { unsafe fn test_mm_maskz_shldi_epi32() { let a = _mm_set1_epi32(1); let b = _mm_set1_epi32(1 << 31); - let r = _mm_maskz_shldi_epi32(0, a, b, 2); + let r = _mm_maskz_shldi_epi32::<2>(0, a, b); assert_eq_m128i(r, _mm_setzero_si128()); - let r = _mm_maskz_shldi_epi32(0b00001111, a, b, 2); + let r = _mm_maskz_shldi_epi32::<2>(0b00001111, a, b); let e = _mm_set1_epi32(6); assert_eq_m128i(r, e); } @@ -3107,7 +3178,7 @@ mod tests { unsafe fn test_mm512_shldi_epi16() { let a = _mm512_set1_epi16(1); let b = _mm512_set1_epi16(1 << 15); - let r = _mm512_shldi_epi16(a, b, 2); + let r = _mm512_shldi_epi16::<2>(a, b); let e = _mm512_set1_epi16(6); assert_eq_m512i(r, e); } @@ -3116,9 +3187,9 @@ mod tests { unsafe fn test_mm512_mask_shldi_epi16() { let a = _mm512_set1_epi16(1); let b = _mm512_set1_epi16(1 << 15); - let r = _mm512_mask_shldi_epi16(a, 0, a, b, 2); + let r = _mm512_mask_shldi_epi16::<2>(a, 0, a, b); assert_eq_m512i(r, a); - let r = _mm512_mask_shldi_epi16(a, 0b11111111_11111111_11111111_11111111, a, b, 2); + let r = _mm512_mask_shldi_epi16::<2>(a, 0b11111111_11111111_11111111_11111111, a, b); let e = _mm512_set1_epi16(6); assert_eq_m512i(r, e); } @@ -3127,9 +3198,9 @@ mod tests { unsafe fn test_mm512_maskz_shldi_epi16() { let a = _mm512_set1_epi16(1); let b = _mm512_set1_epi16(1 << 15); - let r = _mm512_maskz_shldi_epi16(0, a, b, 2); + let r = _mm512_maskz_shldi_epi16::<2>(0, a, b); assert_eq_m512i(r, _mm512_setzero_si512()); - let r = _mm512_maskz_shldi_epi16(0b11111111_11111111_11111111_11111111, a, b, 2); + let r = _mm512_maskz_shldi_epi16::<2>(0b11111111_11111111_11111111_11111111, a, b); let e = _mm512_set1_epi16(6); assert_eq_m512i(r, e); } @@ -3138,7 +3209,7 @@ mod tests { unsafe fn test_mm256_shldi_epi16() { let a = _mm256_set1_epi16(1); let b = _mm256_set1_epi16(1 << 15); - let r = _mm256_shldi_epi16(a, b, 2); + let r = _mm256_shldi_epi16::<2>(a, b); let e = _mm256_set1_epi16(6); assert_eq_m256i(r, e); } @@ -3147,9 +3218,9 @@ mod tests { unsafe fn test_mm256_mask_shldi_epi16() { let a = _mm256_set1_epi16(1); let b = _mm256_set1_epi16(1 << 15); - let r = _mm256_mask_shldi_epi16(a, 0, a, b, 2); + let r = _mm256_mask_shldi_epi16::<2>(a, 0, a, b); assert_eq_m256i(r, a); - let r = _mm256_mask_shldi_epi16(a, 0b11111111_11111111, a, b, 2); + let r = _mm256_mask_shldi_epi16::<2>(a, 0b11111111_11111111, a, b); let e = _mm256_set1_epi16(6); assert_eq_m256i(r, e); } @@ -3158,9 +3229,9 @@ mod tests { unsafe fn test_mm256_maskz_shldi_epi16() { let a = _mm256_set1_epi16(1); let b = _mm256_set1_epi16(1 << 15); - let r = _mm256_maskz_shldi_epi16(0, a, b, 2); + let r = _mm256_maskz_shldi_epi16::<2>(0, a, b); assert_eq_m256i(r, _mm256_setzero_si256()); - let r = _mm256_maskz_shldi_epi16(0b11111111_11111111, a, b, 2); + let r = _mm256_maskz_shldi_epi16::<2>(0b11111111_11111111, a, b); let e = _mm256_set1_epi16(6); assert_eq_m256i(r, e); } @@ -3169,7 +3240,7 @@ mod tests { unsafe fn test_mm_shldi_epi16() { let a = _mm_set1_epi16(1); let b = _mm_set1_epi16(1 << 15); - let r = _mm_shldi_epi16(a, b, 2); + let r = _mm_shldi_epi16::<2>(a, b); let e = _mm_set1_epi16(6); assert_eq_m128i(r, e); } @@ -3178,9 +3249,9 @@ mod tests { unsafe fn test_mm_mask_shldi_epi16() { let a = _mm_set1_epi16(1); let b = _mm_set1_epi16(1 << 15); - let r = _mm_mask_shldi_epi16(a, 0, a, b, 2); + let r = _mm_mask_shldi_epi16::<2>(a, 0, a, b); assert_eq_m128i(r, a); - let r = _mm_mask_shldi_epi16(a, 0b11111111, a, b, 2); + let r = _mm_mask_shldi_epi16::<2>(a, 0b11111111, a, b); let e = _mm_set1_epi16(6); assert_eq_m128i(r, e); } @@ -3189,9 +3260,9 @@ mod tests { unsafe fn test_mm_maskz_shldi_epi16() { let a = _mm_set1_epi16(1); let b = _mm_set1_epi16(1 << 15); - let r = _mm_maskz_shldi_epi16(0, a, b, 2); + let r = _mm_maskz_shldi_epi16::<2>(0, a, b); assert_eq_m128i(r, _mm_setzero_si128()); - let r = _mm_maskz_shldi_epi16(0b11111111, a, b, 2); + let r = _mm_maskz_shldi_epi16::<2>(0b11111111, a, b); let e = _mm_set1_epi16(6); assert_eq_m128i(r, e); } @@ -3200,7 +3271,7 @@ mod tests { unsafe fn test_mm512_shrdi_epi64() { let a = _mm512_set1_epi64(8); let b = _mm512_set1_epi64(2); - let r = _mm512_shrdi_epi64(a, b, 1); + let r = _mm512_shrdi_epi64::<1>(a, b); let e = _mm512_set1_epi64(1); assert_eq_m512i(r, e); } @@ -3209,9 +3280,9 @@ mod tests { unsafe fn test_mm512_mask_shrdi_epi64() { let a = _mm512_set1_epi64(8); let b = _mm512_set1_epi64(2); - let r = _mm512_mask_shrdi_epi64(a, 0, a, b, 1); + let r = _mm512_mask_shrdi_epi64::<1>(a, 0, a, b); assert_eq_m512i(r, a); - let r = _mm512_mask_shrdi_epi64(a, 0b11111111, a, b, 1); + let r = _mm512_mask_shrdi_epi64::<1>(a, 0b11111111, a, b); let e = _mm512_set1_epi64(1); assert_eq_m512i(r, e); } @@ -3220,9 +3291,9 @@ mod tests { unsafe fn test_mm512_maskz_shrdi_epi64() { let a = _mm512_set1_epi64(8); let b = _mm512_set1_epi64(2); - let r = _mm512_maskz_shrdi_epi64(0, a, b, 1); + let r = _mm512_maskz_shrdi_epi64::<1>(0, a, b); assert_eq_m512i(r, _mm512_setzero_si512()); - let r = _mm512_maskz_shrdi_epi64(0b11111111, a, b, 1); + let r = _mm512_maskz_shrdi_epi64::<1>(0b11111111, a, b); let e = _mm512_set1_epi64(1); assert_eq_m512i(r, e); } @@ -3231,7 +3302,7 @@ mod tests { unsafe fn test_mm256_shrdi_epi64() { let a = _mm256_set1_epi64x(8); let b = _mm256_set1_epi64x(2); - let r = _mm256_shrdi_epi64(a, b, 1); + let r = _mm256_shrdi_epi64::<1>(a, b); let e = _mm256_set1_epi64x(1); assert_eq_m256i(r, e); } @@ -3240,9 +3311,9 @@ mod tests { unsafe fn test_mm256_mask_shrdi_epi64() { let a = _mm256_set1_epi64x(8); let b = _mm256_set1_epi64x(2); - let r = _mm256_mask_shrdi_epi64(a, 0, a, b, 1); + let r = _mm256_mask_shrdi_epi64::<1>(a, 0, a, b); assert_eq_m256i(r, a); - let r = _mm256_mask_shrdi_epi64(a, 0b00001111, a, b, 1); + let r = _mm256_mask_shrdi_epi64::<1>(a, 0b00001111, a, b); let e = _mm256_set1_epi64x(1); assert_eq_m256i(r, e); } @@ -3251,9 +3322,9 @@ mod tests { unsafe fn test_mm256_maskz_shrdi_epi64() { let a = _mm256_set1_epi64x(8); let b = _mm256_set1_epi64x(2); - let r = _mm256_maskz_shrdi_epi64(0, a, b, 1); + let r = _mm256_maskz_shrdi_epi64::<1>(0, a, b); assert_eq_m256i(r, _mm256_setzero_si256()); - let r = _mm256_maskz_shrdi_epi64(0b00001111, a, b, 1); + let r = _mm256_maskz_shrdi_epi64::<1>(0b00001111, a, b); let e = _mm256_set1_epi64x(1); assert_eq_m256i(r, e); } @@ -3262,7 +3333,7 @@ mod tests { unsafe fn test_mm_shrdi_epi64() { let a = _mm_set1_epi64x(8); let b = _mm_set1_epi64x(2); - let r = _mm_shrdi_epi64(a, b, 1); + let r = _mm_shrdi_epi64::<1>(a, b); let e = _mm_set1_epi64x(1); assert_eq_m128i(r, e); } @@ -3271,9 +3342,9 @@ mod tests { unsafe fn test_mm_mask_shrdi_epi64() { let a = _mm_set1_epi64x(8); let b = _mm_set1_epi64x(2); - let r = _mm_mask_shrdi_epi64(a, 0, a, b, 1); + let r = _mm_mask_shrdi_epi64::<1>(a, 0, a, b); assert_eq_m128i(r, a); - let r = _mm_mask_shrdi_epi64(a, 0b00000011, a, b, 1); + let r = _mm_mask_shrdi_epi64::<1>(a, 0b00000011, a, b); let e = _mm_set1_epi64x(1); assert_eq_m128i(r, e); } @@ -3282,9 +3353,9 @@ mod tests { unsafe fn test_mm_maskz_shrdi_epi64() { let a = _mm_set1_epi64x(8); let b = _mm_set1_epi64x(2); - let r = _mm_maskz_shrdi_epi64(0, a, b, 1); + let r = _mm_maskz_shrdi_epi64::<1>(0, a, b); assert_eq_m128i(r, _mm_setzero_si128()); - let r = _mm_maskz_shrdi_epi64(0b00000011, a, b, 1); + let r = _mm_maskz_shrdi_epi64::<1>(0b00000011, a, b); let e = _mm_set1_epi64x(1); assert_eq_m128i(r, e); } @@ -3293,7 +3364,7 @@ mod tests { unsafe fn test_mm512_shrdi_epi32() { let a = _mm512_set1_epi32(8); let b = _mm512_set1_epi32(2); - let r = _mm512_shrdi_epi32(a, b, 1); + let r = _mm512_shrdi_epi32::<1>(a, b); let e = _mm512_set1_epi32(1); assert_eq_m512i(r, e); } @@ -3302,9 +3373,9 @@ mod tests { unsafe fn test_mm512_mask_shrdi_epi32() { let a = _mm512_set1_epi32(8); let b = _mm512_set1_epi32(2); - let r = _mm512_mask_shrdi_epi32(a, 0, a, b, 1); + let r = _mm512_mask_shrdi_epi32::<1>(a, 0, a, b); assert_eq_m512i(r, a); - let r = _mm512_mask_shrdi_epi32(a, 0b11111111_11111111, a, b, 1); + let r = _mm512_mask_shrdi_epi32::<1>(a, 0b11111111_11111111, a, b); let e = _mm512_set1_epi32(1); assert_eq_m512i(r, e); } @@ -3313,9 +3384,9 @@ mod tests { unsafe fn test_mm512_maskz_shrdi_epi32() { let a = _mm512_set1_epi32(8); let b = _mm512_set1_epi32(2); - let r = _mm512_maskz_shrdi_epi32(0, a, b, 1); + let r = _mm512_maskz_shrdi_epi32::<1>(0, a, b); assert_eq_m512i(r, _mm512_setzero_si512()); - let r = _mm512_maskz_shrdi_epi32(0b11111111_11111111, a, b, 1); + let r = _mm512_maskz_shrdi_epi32::<1>(0b11111111_11111111, a, b); let e = _mm512_set1_epi32(1); assert_eq_m512i(r, e); } @@ -3324,7 +3395,7 @@ mod tests { unsafe fn test_mm256_shrdi_epi32() { let a = _mm256_set1_epi32(8); let b = _mm256_set1_epi32(2); - let r = _mm256_shrdi_epi32(a, b, 1); + let r = _mm256_shrdi_epi32::<1>(a, b); let e = _mm256_set1_epi32(1); assert_eq_m256i(r, e); } @@ -3333,9 +3404,9 @@ mod tests { unsafe fn test_mm256_mask_shrdi_epi32() { let a = _mm256_set1_epi32(8); let b = _mm256_set1_epi32(2); - let r = _mm256_mask_shrdi_epi32(a, 0, a, b, 1); + let r = _mm256_mask_shrdi_epi32::<1>(a, 0, a, b); assert_eq_m256i(r, a); - let r = _mm256_mask_shrdi_epi32(a, 0b11111111, a, b, 1); + let r = _mm256_mask_shrdi_epi32::<1>(a, 0b11111111, a, b); let e = _mm256_set1_epi32(1); assert_eq_m256i(r, e); } @@ -3344,9 +3415,9 @@ mod tests { unsafe fn test_mm256_maskz_shrdi_epi32() { let a = _mm256_set1_epi32(8); let b = _mm256_set1_epi32(2); - let r = _mm256_maskz_shrdi_epi32(0, a, b, 1); + let r = _mm256_maskz_shrdi_epi32::<1>(0, a, b); assert_eq_m256i(r, _mm256_setzero_si256()); - let r = _mm256_maskz_shrdi_epi32(0b11111111, a, b, 1); + let r = _mm256_maskz_shrdi_epi32::<1>(0b11111111, a, b); let e = _mm256_set1_epi32(1); assert_eq_m256i(r, e); } @@ -3355,7 +3426,7 @@ mod tests { unsafe fn test_mm_shrdi_epi32() { let a = _mm_set1_epi32(8); let b = _mm_set1_epi32(2); - let r = _mm_shrdi_epi32(a, b, 1); + let r = _mm_shrdi_epi32::<1>(a, b); let e = _mm_set1_epi32(1); assert_eq_m128i(r, e); } @@ -3364,9 +3435,9 @@ mod tests { unsafe fn test_mm_mask_shrdi_epi32() { let a = _mm_set1_epi32(8); let b = _mm_set1_epi32(2); - let r = _mm_mask_shrdi_epi32(a, 0, a, b, 1); + let r = _mm_mask_shrdi_epi32::<1>(a, 0, a, b); assert_eq_m128i(r, a); - let r = _mm_mask_shrdi_epi32(a, 0b00001111, a, b, 1); + let r = _mm_mask_shrdi_epi32::<1>(a, 0b00001111, a, b); let e = _mm_set1_epi32(1); assert_eq_m128i(r, e); } @@ -3375,9 +3446,9 @@ mod tests { unsafe fn test_mm_maskz_shrdi_epi32() { let a = _mm_set1_epi32(8); let b = _mm_set1_epi32(2); - let r = _mm_maskz_shrdi_epi32(0, a, b, 1); + let r = _mm_maskz_shrdi_epi32::<1>(0, a, b); assert_eq_m128i(r, _mm_setzero_si128()); - let r = _mm_maskz_shrdi_epi32(0b00001111, a, b, 1); + let r = _mm_maskz_shrdi_epi32::<1>(0b00001111, a, b); let e = _mm_set1_epi32(1); assert_eq_m128i(r, e); } @@ -3386,7 +3457,7 @@ mod tests { unsafe fn test_mm512_shrdi_epi16() { let a = _mm512_set1_epi16(8); let b = _mm512_set1_epi16(2); - let r = _mm512_shrdi_epi16(a, b, 1); + let r = _mm512_shrdi_epi16::<1>(a, b); let e = _mm512_set1_epi16(1); assert_eq_m512i(r, e); } @@ -3395,9 +3466,9 @@ mod tests { unsafe fn test_mm512_mask_shrdi_epi16() { let a = _mm512_set1_epi16(8); let b = _mm512_set1_epi16(2); - let r = _mm512_mask_shrdi_epi16(a, 0, a, b, 1); + let r = _mm512_mask_shrdi_epi16::<1>(a, 0, a, b); assert_eq_m512i(r, a); - let r = _mm512_mask_shrdi_epi16(a, 0b11111111_11111111_11111111_11111111, a, b, 1); + let r = _mm512_mask_shrdi_epi16::<1>(a, 0b11111111_11111111_11111111_11111111, a, b); let e = _mm512_set1_epi16(1); assert_eq_m512i(r, e); } @@ -3406,9 +3477,9 @@ mod tests { unsafe fn test_mm512_maskz_shrdi_epi16() { let a = _mm512_set1_epi16(8); let b = _mm512_set1_epi16(2); - let r = _mm512_maskz_shrdi_epi16(0, a, b, 1); + let r = _mm512_maskz_shrdi_epi16::<1>(0, a, b); assert_eq_m512i(r, _mm512_setzero_si512()); - let r = _mm512_maskz_shrdi_epi16(0b11111111_11111111_11111111_11111111, a, b, 1); + let r = _mm512_maskz_shrdi_epi16::<1>(0b11111111_11111111_11111111_11111111, a, b); let e = _mm512_set1_epi16(1); assert_eq_m512i(r, e); } @@ -3417,7 +3488,7 @@ mod tests { unsafe fn test_mm256_shrdi_epi16() { let a = _mm256_set1_epi16(8); let b = _mm256_set1_epi16(2); - let r = _mm256_shrdi_epi16(a, b, 1); + let r = _mm256_shrdi_epi16::<1>(a, b); let e = _mm256_set1_epi16(1); assert_eq_m256i(r, e); } @@ -3426,9 +3497,9 @@ mod tests { unsafe fn test_mm256_mask_shrdi_epi16() { let a = _mm256_set1_epi16(8); let b = _mm256_set1_epi16(2); - let r = _mm256_mask_shrdi_epi16(a, 0, a, b, 1); + let r = _mm256_mask_shrdi_epi16::<1>(a, 0, a, b); assert_eq_m256i(r, a); - let r = _mm256_mask_shrdi_epi16(a, 0b11111111_11111111, a, b, 1); + let r = _mm256_mask_shrdi_epi16::<1>(a, 0b11111111_11111111, a, b); let e = _mm256_set1_epi16(1); assert_eq_m256i(r, e); } @@ -3437,9 +3508,9 @@ mod tests { unsafe fn test_mm256_maskz_shrdi_epi16() { let a = _mm256_set1_epi16(8); let b = _mm256_set1_epi16(2); - let r = _mm256_maskz_shrdi_epi16(0, a, b, 1); + let r = _mm256_maskz_shrdi_epi16::<1>(0, a, b); assert_eq_m256i(r, _mm256_setzero_si256()); - let r = _mm256_maskz_shrdi_epi16(0b11111111_11111111, a, b, 1); + let r = _mm256_maskz_shrdi_epi16::<1>(0b11111111_11111111, a, b); let e = _mm256_set1_epi16(1); assert_eq_m256i(r, e); } @@ -3448,7 +3519,7 @@ mod tests { unsafe fn test_mm_shrdi_epi16() { let a = _mm_set1_epi16(8); let b = _mm_set1_epi16(2); - let r = _mm_shrdi_epi16(a, b, 1); + let r = _mm_shrdi_epi16::<1>(a, b); let e = _mm_set1_epi16(1); assert_eq_m128i(r, e); } @@ -3457,9 +3528,9 @@ mod tests { unsafe fn test_mm_mask_shrdi_epi16() { let a = _mm_set1_epi16(8); let b = _mm_set1_epi16(2); - let r = _mm_mask_shrdi_epi16(a, 0, a, b, 1); + let r = _mm_mask_shrdi_epi16::<1>(a, 0, a, b); assert_eq_m128i(r, a); - let r = _mm_mask_shrdi_epi16(a, 0b11111111, a, b, 1); + let r = _mm_mask_shrdi_epi16::<1>(a, 0b11111111, a, b); let e = _mm_set1_epi16(1); assert_eq_m128i(r, e); } @@ -3468,9 +3539,9 @@ mod tests { unsafe fn test_mm_maskz_shrdi_epi16() { let a = _mm_set1_epi16(8); let b = _mm_set1_epi16(2); - let r = _mm_maskz_shrdi_epi16(0, a, b, 1); + let r = _mm_maskz_shrdi_epi16::<1>(0, a, b); assert_eq_m128i(r, _mm_setzero_si128()); - let r = _mm_maskz_shrdi_epi16(0b11111111, a, b, 1); + let r = _mm_maskz_shrdi_epi16::<1>(0b11111111, a, b); let e = _mm_set1_epi16(1); assert_eq_m128i(r, e); } diff --git a/library/stdarch/crates/core_arch/src/x86/avx512vpclmulqdq.rs b/library/stdarch/crates/core_arch/src/x86/avx512vpclmulqdq.rs index 860fea46fc..9bfeb903a7 100644 --- a/library/stdarch/crates/core_arch/src/x86/avx512vpclmulqdq.rs +++ b/library/stdarch/crates/core_arch/src/x86/avx512vpclmulqdq.rs @@ -34,15 +34,11 @@ extern "C" { #[inline] #[target_feature(enable = "avx512vpclmulqdq,avx512f")] // technically according to Intel's documentation we don't need avx512f here, however LLVM gets confused otherwise -#[cfg_attr(test, assert_instr(vpclmul, imm8 = 0))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm512_clmulepi64_epi128(a: __m512i, b: __m512i, imm8: i32) -> __m512i { - macro_rules! call { - ($imm8:expr) => { - pclmulqdq_512(a, b, $imm8) - }; - } - constify_imm8!(imm8, call) +#[cfg_attr(test, assert_instr(vpclmul, IMM8 = 0))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm512_clmulepi64_epi128(a: __m512i, b: __m512i) -> __m512i { + static_assert_imm8!(IMM8); + pclmulqdq_512(a, b, IMM8 as u8) } /// Performs a carry-less multiplication of two 64-bit polynomials over the @@ -55,15 +51,11 @@ pub unsafe fn _mm512_clmulepi64_epi128(a: __m512i, b: __m512i, imm8: i32) -> __m /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_clmulepi64_epi128) #[inline] #[target_feature(enable = "avx512vpclmulqdq,avx512vl")] -#[cfg_attr(test, assert_instr(vpclmul, imm8 = 0))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm256_clmulepi64_epi128(a: __m256i, b: __m256i, imm8: i32) -> __m256i { - macro_rules! call { - ($imm8:expr) => { - pclmulqdq_256(a, b, $imm8) - }; - } - constify_imm8!(imm8, call) +#[cfg_attr(test, assert_instr(vpclmul, IMM8 = 0))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm256_clmulepi64_epi128(a: __m256i, b: __m256i) -> __m256i { + static_assert_imm8!(IMM8); + pclmulqdq_256(a, b, IMM8 as u8) } #[cfg(test)] @@ -93,37 +85,37 @@ mod tests { let r11 = _mm_set_epi64x(0x1d1e1f2c592e7c45, 0xd66ee03e410fd4ed); let r11 = $broadcast(r11); - $assert($clmul(a, b, 0x00), r00); - $assert($clmul(a, b, 0x10), r01); - $assert($clmul(a, b, 0x01), r10); - $assert($clmul(a, b, 0x11), r11); + $assert($clmul::<0x00>(a, b), r00); + $assert($clmul::<0x10>(a, b), r01); + $assert($clmul::<0x01>(a, b), r10); + $assert($clmul::<0x11>(a, b), r11); let a0 = _mm_set_epi64x(0x0000000000000000, 0x8000000000000000); let a0 = $broadcast(a0); let r = _mm_set_epi64x(0x4000000000000000, 0x0000000000000000); let r = $broadcast(r); - $assert($clmul(a0, a0, 0x00), r); + $assert($clmul::<0x00>(a0, a0), r); } } macro_rules! unroll { - ($target:ident[4] = $op:ident($source:ident,4);) => { - $target[3] = $op($source, 3); - $target[2] = $op($source, 2); - unroll! {$target[2] = $op($source,2);} + ($target:ident[4] = $op:ident::<4>($source:ident);) => { + $target[3] = $op::<3>($source); + $target[2] = $op::<2>($source); + unroll! {$target[2] = $op::<2>($source);} }; - ($target:ident[2] = $op:ident($source:ident,2);) => { - $target[1] = $op($source, 1); - $target[0] = $op($source, 0); + ($target:ident[2] = $op:ident::<2>($source:ident);) => { + $target[1] = $op::<1>($source); + $target[0] = $op::<0>($source); }; - (assert_eq_m128i($op:ident($vec_res:ident,4),$lin_res:ident[4]);) => { - assert_eq_m128i($op($vec_res, 3), $lin_res[3]); - assert_eq_m128i($op($vec_res, 2), $lin_res[2]); - unroll! {assert_eq_m128i($op($vec_res,2),$lin_res[2]);} + (assert_eq_m128i($op:ident::<4>($vec_res:ident),$lin_res:ident[4]);) => { + assert_eq_m128i($op::<3>($vec_res), $lin_res[3]); + assert_eq_m128i($op::<2>($vec_res), $lin_res[2]); + unroll! {assert_eq_m128i($op::<2>($vec_res),$lin_res[2]);} }; - (assert_eq_m128i($op:ident($vec_res:ident,2),$lin_res:ident[2]);) => { - assert_eq_m128i($op($vec_res, 1), $lin_res[1]); - assert_eq_m128i($op($vec_res, 0), $lin_res[0]); + (assert_eq_m128i($op:ident::<2>($vec_res:ident),$lin_res:ident[2]);) => { + assert_eq_m128i($op::<1>($vec_res), $lin_res[1]); + assert_eq_m128i($op::<0>($vec_res), $lin_res[0]); }; } @@ -156,16 +148,16 @@ mod tests { ); let mut a_decomp = [_mm_setzero_si128(); 4]; - unroll! {a_decomp[4] = _mm512_extracti32x4_epi32(a,4);} + unroll! {a_decomp[4] = _mm512_extracti32x4_epi32::<4>(a);} let mut b_decomp = [_mm_setzero_si128(); 4]; - unroll! {b_decomp[4] = _mm512_extracti32x4_epi32(b,4);} + unroll! {b_decomp[4] = _mm512_extracti32x4_epi32::<4>(b);} let r = vectorized(a, b); let mut e_decomp = [_mm_setzero_si128(); 4]; for i in 0..4 { e_decomp[i] = linear(a_decomp[i], b_decomp[i]); } - unroll! {assert_eq_m128i(_mm512_extracti32x4_epi32(r,4),e_decomp[4]);} + unroll! {assert_eq_m128i(_mm512_extracti32x4_epi32::<4>(r),e_decomp[4]);} } // this function tests one of the possible 4 instances @@ -197,19 +189,19 @@ mod tests { ); let mut a_decomp = [_mm_setzero_si128(); 2]; - unroll! {a_decomp[2] = _mm512_extracti32x4_epi32(a,2);} + unroll! {a_decomp[2] = _mm512_extracti32x4_epi32::<2>(a);} let mut b_decomp = [_mm_setzero_si128(); 2]; - unroll! {b_decomp[2] = _mm512_extracti32x4_epi32(b,2);} + unroll! {b_decomp[2] = _mm512_extracti32x4_epi32::<2>(b);} let r = vectorized( - _mm512_extracti64x4_epi64(a, 0), - _mm512_extracti64x4_epi64(b, 0), + _mm512_extracti64x4_epi64::<0>(a), + _mm512_extracti64x4_epi64::<0>(b), ); let mut e_decomp = [_mm_setzero_si128(); 2]; for i in 0..2 { e_decomp[i] = linear(a_decomp[i], b_decomp[i]); } - unroll! {assert_eq_m128i(_mm256_extracti128_si256(r,2),e_decomp[2]);} + unroll! {assert_eq_m128i(_mm256_extracti128_si256::<2>(r),e_decomp[2]);} } #[simd_test(enable = "avx512vpclmulqdq,avx512f")] @@ -221,20 +213,20 @@ mod tests { ); verify_512_helper( - |a, b| _mm_clmulepi64_si128(a, b, 0x00), - |a, b| _mm512_clmulepi64_epi128(a, b, 0x00), + |a, b| _mm_clmulepi64_si128::<0x00>(a, b), + |a, b| _mm512_clmulepi64_epi128::<0x00>(a, b), ); verify_512_helper( - |a, b| _mm_clmulepi64_si128(a, b, 0x01), - |a, b| _mm512_clmulepi64_epi128(a, b, 0x01), + |a, b| _mm_clmulepi64_si128::<0x01>(a, b), + |a, b| _mm512_clmulepi64_epi128::<0x01>(a, b), ); verify_512_helper( - |a, b| _mm_clmulepi64_si128(a, b, 0x10), - |a, b| _mm512_clmulepi64_epi128(a, b, 0x10), + |a, b| _mm_clmulepi64_si128::<0x10>(a, b), + |a, b| _mm512_clmulepi64_epi128::<0x10>(a, b), ); verify_512_helper( - |a, b| _mm_clmulepi64_si128(a, b, 0x11), - |a, b| _mm512_clmulepi64_epi128(a, b, 0x11), + |a, b| _mm_clmulepi64_si128::<0x11>(a, b), + |a, b| _mm512_clmulepi64_epi128::<0x11>(a, b), ); } @@ -247,20 +239,20 @@ mod tests { ); verify_256_helper( - |a, b| _mm_clmulepi64_si128(a, b, 0x00), - |a, b| _mm256_clmulepi64_epi128(a, b, 0x00), + |a, b| _mm_clmulepi64_si128::<0x00>(a, b), + |a, b| _mm256_clmulepi64_epi128::<0x00>(a, b), ); verify_256_helper( - |a, b| _mm_clmulepi64_si128(a, b, 0x01), - |a, b| _mm256_clmulepi64_epi128(a, b, 0x01), + |a, b| _mm_clmulepi64_si128::<0x01>(a, b), + |a, b| _mm256_clmulepi64_epi128::<0x01>(a, b), ); verify_256_helper( - |a, b| _mm_clmulepi64_si128(a, b, 0x10), - |a, b| _mm256_clmulepi64_epi128(a, b, 0x10), + |a, b| _mm_clmulepi64_si128::<0x10>(a, b), + |a, b| _mm256_clmulepi64_epi128::<0x10>(a, b), ); verify_256_helper( - |a, b| _mm_clmulepi64_si128(a, b, 0x11), - |a, b| _mm256_clmulepi64_epi128(a, b, 0x11), + |a, b| _mm_clmulepi64_si128::<0x11>(a, b), + |a, b| _mm256_clmulepi64_epi128::<0x11>(a, b), ); } } diff --git a/library/stdarch/crates/core_arch/src/x86/bt.rs b/library/stdarch/crates/core_arch/src/x86/bt.rs index dc172f6f38..278ddf9543 100644 --- a/library/stdarch/crates/core_arch/src/x86/bt.rs +++ b/library/stdarch/crates/core_arch/src/x86/bt.rs @@ -1,16 +1,36 @@ #[cfg(test)] use stdarch_test::assert_instr; +// x32 wants to use a 32-bit address size, but asm! defaults to using the full +// register name (e.g. rax). We have to explicitly override the placeholder to +// use the 32-bit register name in that case. +#[cfg(target_pointer_width = "32")] +macro_rules! bt { + ($inst:expr) => { + concat!($inst, " {b:e}, ({p:e})") + }; +} +#[cfg(target_pointer_width = "64")] +macro_rules! bt { + ($inst:expr) => { + concat!($inst, " {b:e}, ({p})") + }; +} + /// Returns the bit in position `b` of the memory addressed by `p`. #[inline] #[cfg_attr(test, assert_instr(bt))] #[unstable(feature = "simd_x86_bittest", issue = "59414")] pub unsafe fn _bittest(p: *const i32, b: i32) -> u8 { let r: u8; - llvm_asm!("btl $2, $1\n\tsetc ${0:b}" - : "=r"(r) - : "*m"(p), "r"(b) - : "cc", "memory"); + asm!( + bt!("btl"), + "setc {r}", + p = in(reg) p, + b = in(reg) b, + r = out(reg_byte) r, + options(readonly, nostack, pure, att_syntax) + ); r } @@ -20,10 +40,14 @@ pub unsafe fn _bittest(p: *const i32, b: i32) -> u8 { #[unstable(feature = "simd_x86_bittest", issue = "59414")] pub unsafe fn _bittestandset(p: *mut i32, b: i32) -> u8 { let r: u8; - llvm_asm!("btsl $2, $1\n\tsetc ${0:b}" - : "=r"(r), "+*m"(p) - : "r"(b) - : "cc", "memory"); + asm!( + bt!("btsl"), + "setc {r}", + p = in(reg) p, + b = in(reg) b, + r = out(reg_byte) r, + options(nostack, att_syntax) + ); r } @@ -33,10 +57,14 @@ pub unsafe fn _bittestandset(p: *mut i32, b: i32) -> u8 { #[unstable(feature = "simd_x86_bittest", issue = "59414")] pub unsafe fn _bittestandreset(p: *mut i32, b: i32) -> u8 { let r: u8; - llvm_asm!("btrl $2, $1\n\tsetc ${0:b}" - : "=r"(r), "+*m"(p) - : "r"(b) - : "cc", "memory"); + asm!( + bt!("btrl"), + "setc {r}", + p = in(reg) p, + b = in(reg) b, + r = out(reg_byte) r, + options(nostack, att_syntax) + ); r } @@ -46,10 +74,14 @@ pub unsafe fn _bittestandreset(p: *mut i32, b: i32) -> u8 { #[unstable(feature = "simd_x86_bittest", issue = "59414")] pub unsafe fn _bittestandcomplement(p: *mut i32, b: i32) -> u8 { let r: u8; - llvm_asm!("btcl $2, $1\n\tsetc ${0:b}" - : "=r"(r), "+*m"(p) - : "r"(b) - : "cc", "memory"); + asm!( + bt!("btcl"), + "setc {r}", + p = in(reg) p, + b = in(reg) b, + r = out(reg_byte) r, + options(nostack, att_syntax) + ); r } diff --git a/library/stdarch/crates/core_arch/src/x86/cpuid.rs b/library/stdarch/crates/core_arch/src/x86/cpuid.rs index 42cc95b4d0..ed383e917d 100644 --- a/library/stdarch/crates/core_arch/src/x86/cpuid.rs +++ b/library/stdarch/crates/core_arch/src/x86/cpuid.rs @@ -56,31 +56,32 @@ pub unsafe fn __cpuid_count(leaf: u32, sub_leaf: u32) -> CpuidResult { let ecx; let edx; + // LLVM sometimes reserves `ebx` for its internal use, we so we need to use + // a scratch register for it instead. #[cfg(target_arch = "x86")] { asm!( + "movl %ebx, {0}", "cpuid", + "xchgl %ebx, {0}", + lateout(reg) ebx, inlateout("eax") leaf => eax, - lateout("ebx") ebx, inlateout("ecx") sub_leaf => ecx, lateout("edx") edx, - options(nostack, preserves_flags), + options(nostack, preserves_flags, att_syntax), ); } #[cfg(target_arch = "x86_64")] { - // x86-64 uses `rbx` as the base register, so preserve it. - // This works around a bug in LLVM with ASAN enabled: - // https://bugs.llvm.org/show_bug.cgi?id=17907 asm!( - "mov rsi, rbx", + "movq %rbx, {0:r}", "cpuid", - "xchg rsi, rbx", + "xchgq %rbx, {0:r}", + lateout(reg) ebx, inlateout("eax") leaf => eax, - lateout("esi") ebx, inlateout("ecx") sub_leaf => ecx, lateout("edx") edx, - options(nostack, preserves_flags), + options(nostack, preserves_flags, att_syntax), ); } CpuidResult { eax, ebx, ecx, edx } @@ -125,30 +126,25 @@ pub fn has_cpuid() -> bool { // the 21st bit of the EFLAGS register is modifiable or not. // If it is, then `cpuid` is available. let result: u32; - let _temp: u32; - llvm_asm!(r#" - # Read eflags into $0 and copy it into $1: - pushfd - pop $0 - mov $1, $0 - # Flip 21st bit of $0. - xor $0, 0x200000 - # Set eflags to the value of $0 - # - # Bit 21st can only be modified if cpuid is available - push $0 - popfd # A - # Read eflags into $0: - pushfd # B - pop $0 - # xor with the original eflags sets the bits that - # have been modified: - xor $0, $1 - "# - : "=r"(result), "=r"(_temp) - : - : "cc", "memory" - : "intel"); + asm!( + // Read eflags and save a copy of it + "pushfd", + "pop {result}", + "mov {result}, {saved_flags}", + // Flip 21st bit of the flags + "xor $0x200000, {result}", + // Load the modified flags and read them back. + // Bit 21 can only be modified if cpuid is available. + "push {result}", + "popfd", + "pushfd", + "pop {result}", + // Use xor to find out whether bit 21 has changed + "xor {saved_flags}, {result}", + result = out(reg) result, + saved_flags = out(reg) _, + options(nomem, att_syntax), + ); // There is a race between popfd (A) and pushfd (B) // where other bits beyond 21st may have been modified due to // interrupts, a debugger stepping through the asm, etc. diff --git a/library/stdarch/crates/core_arch/src/x86/eflags.rs b/library/stdarch/crates/core_arch/src/x86/eflags.rs index acb77bc3ec..368240378f 100644 --- a/library/stdarch/crates/core_arch/src/x86/eflags.rs +++ b/library/stdarch/crates/core_arch/src/x86/eflags.rs @@ -13,7 +13,7 @@ #[doc(hidden)] pub unsafe fn __readeflags() -> u32 { let eflags: u32; - llvm_asm!("pushfd; popl $0" : "=r"(eflags) : : : "volatile"); + asm!("pushfd", "pop {}", out(reg) eflags, options(nomem, att_syntax)); eflags } @@ -30,7 +30,7 @@ pub unsafe fn __readeflags() -> u32 { #[doc(hidden)] pub unsafe fn __readeflags() -> u64 { let eflags: u64; - llvm_asm!("pushfq; popq $0" : "=r"(eflags) : : : "volatile"); + asm!("pushfq", "pop {}", out(reg) eflags, options(nomem, att_syntax)); eflags } @@ -46,7 +46,7 @@ pub unsafe fn __readeflags() -> u64 { )] #[doc(hidden)] pub unsafe fn __writeeflags(eflags: u32) { - llvm_asm!("pushl $0; popfd" : : "r"(eflags) : "cc", "flags" : "volatile"); + asm!("push {}", "popfd", in(reg) eflags, options(nomem, att_syntax)); } /// Write EFLAGS. @@ -61,7 +61,7 @@ pub unsafe fn __writeeflags(eflags: u32) { )] #[doc(hidden)] pub unsafe fn __writeeflags(eflags: u64) { - llvm_asm!("pushq $0; popfq" : : "r"(eflags) : "cc", "flags" : "volatile"); + asm!("push {}", "popfq", in(reg) eflags, options(nomem, att_syntax)); } #[cfg(test)] diff --git a/library/stdarch/crates/core_arch/src/x86/f16c.rs b/library/stdarch/crates/core_arch/src/x86/f16c.rs index 503bd41d2f..8b25fd65e8 100644 --- a/library/stdarch/crates/core_arch/src/x86/f16c.rs +++ b/library/stdarch/crates/core_arch/src/x86/f16c.rs @@ -4,7 +4,7 @@ use crate::{ core_arch::{simd::*, x86::*}, - hint::unreachable_unchecked, + // hint::unreachable_unchecked, mem::transmute, }; @@ -42,22 +42,6 @@ pub unsafe fn _mm256_cvtph_ps(a: __m128i) -> __m256 { transmute(llvm_vcvtph2ps_256(transmute(a))) } -macro_rules! dispatch_rounding { - ($rounding:ident, $call:ident) => {{ - match $rounding { - 0 => call!(0), - 1 => call!(1), - 2 => call!(2), - 3 => call!(3), - 4 => call!(4), - 5 => call!(5), - 6 => call!(6), - 7 => call!(7), - _ => unreachable_unchecked(), - } - }}; -} - /// Converts the 4 x 32-bit float values in the 128-bit vector `a` into 4 x /// 16-bit half-precision float values stored in the lowest 64-bit of a 128-bit /// vector. @@ -71,16 +55,13 @@ macro_rules! dispatch_rounding { /// * `_MM_FROUND_CUR_DIRECTION`: use `MXCSR.RC` - see [`_MM_SET_ROUNDING_MODE`]. #[inline] #[target_feature(enable = "f16c")] -#[rustc_args_required_const(1)] -#[cfg_attr(test, assert_instr("vcvtps2ph", imm_rounding = 0))] -pub unsafe fn _mm_cvtps_ph(a: __m128, imm_rounding: i32) -> __m128i { - let a = transmute(a); - macro_rules! call { - ($rounding:expr) => { - llvm_vcvtps2ph_128(a, $rounding) - }; - } - transmute(dispatch_rounding!(imm_rounding, call)) +#[cfg_attr(test, assert_instr("vcvtps2ph", IMM_ROUNDING = 0))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm_cvtps_ph(a: __m128) -> __m128i { + static_assert_imm3!(IMM_ROUNDING); + let a = a.as_f32x4(); + let r = llvm_vcvtps2ph_128(a, IMM_ROUNDING); + transmute(r) } /// Converts the 8 x 32-bit float values in the 256-bit vector `a` into 8 x @@ -95,16 +76,13 @@ pub unsafe fn _mm_cvtps_ph(a: __m128, imm_rounding: i32) -> __m128i { /// * `_MM_FROUND_CUR_DIRECTION`: use `MXCSR.RC` - see [`_MM_SET_ROUNDING_MODE`]. #[inline] #[target_feature(enable = "f16c")] -#[rustc_args_required_const(1)] -#[cfg_attr(test, assert_instr("vcvtps2ph", imm_rounding = 0))] -pub unsafe fn _mm256_cvtps_ph(a: __m256, imm_rounding: i32) -> __m128i { - let a = transmute(a); - macro_rules! call { - ($rounding:expr) => { - llvm_vcvtps2ph_256(a, $rounding) - }; - } - transmute(dispatch_rounding!(imm_rounding, call)) +#[cfg_attr(test, assert_instr("vcvtps2ph", IMM_ROUNDING = 0))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm256_cvtps_ph(a: __m256) -> __m128i { + static_assert_imm3!(IMM_ROUNDING); + let a = a.as_f32x8(); + let r = llvm_vcvtps2ph_256(a, IMM_ROUNDING); + transmute(r) } #[cfg(test)] @@ -116,7 +94,7 @@ mod tests { unsafe fn test_mm_cvtph_ps() { let array = [1_f32, 2_f32, 3_f32, 4_f32]; let float_vec: __m128 = transmute(array); - let halfs: __m128i = _mm_cvtps_ph(float_vec, 0); + let halfs: __m128i = _mm_cvtps_ph::<0>(float_vec); let floats: __m128 = _mm_cvtph_ps(halfs); let result: [f32; 4] = transmute(floats); assert_eq!(result, array); @@ -126,7 +104,7 @@ mod tests { unsafe fn test_mm256_cvtph_ps() { let array = [1_f32, 2_f32, 3_f32, 4_f32, 5_f32, 6_f32, 7_f32, 8_f32]; let float_vec: __m256 = transmute(array); - let halfs: __m128i = _mm256_cvtps_ph(float_vec, 0); + let halfs: __m128i = _mm256_cvtps_ph::<0>(float_vec); let floats: __m256 = _mm256_cvtph_ps(halfs); let result: [f32; 8] = transmute(floats); assert_eq!(result, array); diff --git a/library/stdarch/crates/core_arch/src/x86/macros.rs b/library/stdarch/crates/core_arch/src/x86/macros.rs index 46d248bdc4..e686e65b30 100644 --- a/library/stdarch/crates/core_arch/src/x86/macros.rs +++ b/library/stdarch/crates/core_arch/src/x86/macros.rs @@ -1,1579 +1,89 @@ //! Utility macros. - -macro_rules! constify_imm6 { - ($imm8:expr, $expand:ident) => { - #[allow(overflowing_literals)] - match ($imm8) & 0b1_1111 { - 0 => $expand!(0), - 1 => $expand!(1), - 2 => $expand!(2), - 3 => $expand!(3), - 4 => $expand!(4), - 5 => $expand!(5), - 6 => $expand!(6), - 7 => $expand!(7), - 8 => $expand!(8), - 9 => $expand!(9), - 10 => $expand!(10), - 11 => $expand!(11), - 12 => $expand!(12), - 13 => $expand!(13), - 14 => $expand!(14), - 15 => $expand!(15), - 16 => $expand!(16), - 17 => $expand!(17), - 18 => $expand!(18), - 19 => $expand!(19), - 20 => $expand!(20), - 21 => $expand!(21), - 22 => $expand!(22), - 23 => $expand!(23), - 24 => $expand!(24), - 25 => $expand!(25), - 26 => $expand!(26), - 27 => $expand!(27), - 28 => $expand!(28), - 29 => $expand!(29), - 30 => $expand!(30), - _ => $expand!(31), - } - }; -} - -macro_rules! constify_imm4 { - ($imm8:expr, $expand:ident) => { - #[allow(overflowing_literals)] - match ($imm8) & 0b1111 { - 0 => $expand!(0), - 1 => $expand!(1), - 2 => $expand!(2), - 3 => $expand!(3), - 4 => $expand!(4), - 5 => $expand!(5), - 6 => $expand!(6), - 7 => $expand!(7), - 8 => $expand!(8), - 9 => $expand!(9), - 10 => $expand!(10), - 11 => $expand!(11), - 12 => $expand!(12), - 13 => $expand!(13), - 14 => $expand!(14), - _ => $expand!(15), - } - }; -} - -macro_rules! constify_imm3 { - ($imm8:expr, $expand:ident) => { - #[allow(overflowing_literals)] - match ($imm8) & 0b111 { - 0 => $expand!(0), - 1 => $expand!(1), - 2 => $expand!(2), - 3 => $expand!(3), - 4 => $expand!(4), - 5 => $expand!(5), - 6 => $expand!(6), - _ => $expand!(7), - } +//! +// Helper struct used to trigger const eval errors when the const generic immediate value `imm` is +// not a round number. +pub(crate) struct ValidateConstRound; +impl ValidateConstRound { + pub(crate) const VALID: () = { + assert!( + IMM == 4 || IMM == 8 || IMM == 9 || IMM == 10 || IMM == 11, + "Invalid IMM value" + ); }; } -macro_rules! constify_imm2 { - ($imm8:expr, $expand:ident) => { - #[allow(overflowing_literals)] - match ($imm8) & 0b11 { - 0 => $expand!(0), - 1 => $expand!(1), - 2 => $expand!(2), - _ => $expand!(3), - } +#[allow(unused)] +macro_rules! static_assert_rounding { + ($imm:ident) => { + let _ = $crate::core_arch::x86::macros::ValidateConstRound::<$imm>::VALID; }; } -// Constifies 5 bits along with an sae option without rounding control. -// See: https://github.com/llvm/llvm-project/blob/bd50cf905fa7c0c7caa134301c6ca0658c81eeb1/clang/lib/Sema/SemaChecking.cpp#L3497 -#[allow(unused)] -macro_rules! constify_imm5_sae { - ($imm5:expr, $imm4:expr, $expand:ident) => { - #[allow(overflowing_literals)] - match ($imm5 & 0b1111_1, $imm4 & 0b1111) { - (0, 4) => $expand!(0, 4), - (0, 8) => $expand!(0, 8), - (0, 12) => $expand!(0, 12), - (1, 4) => $expand!(1, 4), - (1, 8) => $expand!(1, 8), - (1, 12) => $expand!(1, 12), - (2, 4) => $expand!(2, 4), - (2, 8) => $expand!(2, 8), - (2, 12) => $expand!(2, 12), - (3, 4) => $expand!(3, 4), - (3, 8) => $expand!(3, 8), - (3, 12) => $expand!(3, 12), - (4, 4) => $expand!(4, 4), - (4, 8) => $expand!(4, 8), - (4, 12) => $expand!(4, 12), - (5, 4) => $expand!(5, 4), - (5, 8) => $expand!(5, 8), - (5, 12) => $expand!(5, 12), - (6, 4) => $expand!(6, 4), - (6, 8) => $expand!(6, 8), - (6, 12) => $expand!(6, 12), - (7, 4) => $expand!(7, 4), - (7, 8) => $expand!(7, 8), - (7, 12) => $expand!(7, 12), - (8, 4) => $expand!(8, 4), - (8, 8) => $expand!(8, 8), - (8, 12) => $expand!(8, 12), - (9, 4) => $expand!(9, 4), - (9, 8) => $expand!(9, 8), - (9, 12) => $expand!(9, 12), - (10, 4) => $expand!(10, 4), - (10, 8) => $expand!(10, 8), - (10, 12) => $expand!(10, 12), - (11, 4) => $expand!(11, 4), - (11, 8) => $expand!(11, 8), - (11, 12) => $expand!(11, 12), - (12, 4) => $expand!(12, 4), - (12, 8) => $expand!(12, 8), - (12, 12) => $expand!(12, 12), - (13, 4) => $expand!(13, 4), - (13, 8) => $expand!(13, 8), - (13, 12) => $expand!(13, 12), - (14, 4) => $expand!(14, 4), - (14, 8) => $expand!(14, 8), - (14, 12) => $expand!(14, 12), - (15, 4) => $expand!(15, 4), - (15, 8) => $expand!(15, 8), - (15, 12) => $expand!(15, 12), - (16, 4) => $expand!(16, 4), - (16, 8) => $expand!(16, 8), - (16, 12) => $expand!(16, 12), - (17, 4) => $expand!(17, 4), - (17, 8) => $expand!(17, 8), - (17, 12) => $expand!(17, 12), - (18, 4) => $expand!(18, 4), - (18, 8) => $expand!(18, 8), - (18, 12) => $expand!(18, 12), - (19, 4) => $expand!(19, 4), - (19, 8) => $expand!(19, 8), - (19, 12) => $expand!(19, 12), - (20, 4) => $expand!(20, 4), - (20, 8) => $expand!(20, 8), - (20, 12) => $expand!(20, 12), - (21, 4) => $expand!(21, 4), - (21, 8) => $expand!(21, 8), - (21, 12) => $expand!(21, 12), - (22, 4) => $expand!(22, 4), - (22, 8) => $expand!(22, 8), - (22, 12) => $expand!(22, 12), - (23, 4) => $expand!(23, 4), - (23, 8) => $expand!(23, 8), - (23, 12) => $expand!(23, 12), - (24, 4) => $expand!(24, 4), - (24, 8) => $expand!(24, 8), - (24, 12) => $expand!(24, 12), - (25, 4) => $expand!(25, 4), - (25, 8) => $expand!(25, 8), - (25, 12) => $expand!(25, 12), - (26, 4) => $expand!(26, 4), - (26, 8) => $expand!(26, 8), - (26, 12) => $expand!(26, 12), - (27, 4) => $expand!(27, 4), - (27, 8) => $expand!(27, 8), - (27, 12) => $expand!(27, 12), - (28, 4) => $expand!(28, 4), - (28, 8) => $expand!(28, 8), - (28, 12) => $expand!(28, 12), - (29, 4) => $expand!(29, 4), - (29, 8) => $expand!(29, 8), - (29, 12) => $expand!(29, 12), - (30, 4) => $expand!(30, 4), - (30, 8) => $expand!(30, 8), - (30, 12) => $expand!(30, 12), - (31, 4) => $expand!(31, 4), - (31, 8) => $expand!(31, 8), - (31, 12) => $expand!(31, 12), - (_, _) => panic!("Invalid sae value"), - } +// Helper struct used to trigger const eval errors when the const generic immediate value `imm` is +// not a sae number. +pub(crate) struct ValidateConstSae; +impl ValidateConstSae { + pub(crate) const VALID: () = { + assert!(IMM == 4 || IMM == 8, "Invalid IMM value"); }; } -// For gather instructions, the only valid values for scale are 1, 2, 4 and 8. -// This macro enforces that. #[allow(unused)] -macro_rules! constify_imm8_gather { - ($imm8:expr, $expand:ident) => { - #[allow(overflowing_literals)] - match ($imm8) { - 1 => $expand!(1), - 2 => $expand!(2), - 4 => $expand!(4), - 8 => $expand!(8), - _ => panic!("Only 1, 2, 4, and 8 are valid values"), - } +macro_rules! static_assert_sae { + ($imm:ident) => { + let _ = $crate::core_arch::x86::macros::ValidateConstSae::<$imm>::VALID; }; } -// For round instructions, the only valid values for rounding are 4, 8, 9, 10 and 11. -// This macro enforces that. -#[allow(unused)] -macro_rules! constify_imm4_round { - ($imm8:expr, $expand:ident) => { - #[allow(overflowing_literals)] - match ($imm8) & 0b1111 { - 4 => $expand!(4), - 8 => $expand!(8), - 9 => $expand!(9), - 10 => $expand!(10), - 11 => $expand!(11), - _ => panic!("Invalid round value"), - } +// Helper struct used to trigger const eval errors when the const generic immediate value `imm` is +// not a mantissas sae number. +pub(crate) struct ValidateConstMantissasSae; +impl ValidateConstMantissasSae { + pub(crate) const VALID: () = { + assert!(IMM == 4 || IMM == 8 || IMM == 12, "Invalid IMM value"); }; } -// For sae instructions, the only valid values for sae are 4 and 8. -// This macro enforces that. #[allow(unused)] -macro_rules! constify_imm4_sae { - ($imm8:expr, $expand:ident) => { - #[allow(overflowing_literals)] - match ($imm8) & 0b1111 { - 4 => $expand!(4), - 8 => $expand!(8), - _ => panic!("Invalid sae value"), - } +macro_rules! static_assert_mantissas_sae { + ($imm:ident) => { + let _ = $crate::core_arch::x86::macros::ValidateConstMantissasSae::<$imm>::VALID; }; } -// Two mantissas parameters. -// This macro enforces that. -#[allow(unused)] -macro_rules! constify_imm4_mantissas { - ($imm4:expr, $imm2:expr, $expand:ident) => { - #[allow(overflowing_literals)] - match ($imm4, $imm2) { - (0, 0) => $expand!(0, 0), - (0, 1) => $expand!(0, 1), - (0, 2) => $expand!(0, 2), - (0, 3) => $expand!(0, 3), - (1, 0) => $expand!(1, 0), - (1, 1) => $expand!(1, 1), - (1, 2) => $expand!(1, 2), - (1, 3) => $expand!(1, 3), - (2, 0) => $expand!(2, 0), - (2, 1) => $expand!(2, 1), - (2, 2) => $expand!(2, 2), - (2, 3) => $expand!(2, 3), - (3, 0) => $expand!(3, 0), - (3, 1) => $expand!(3, 1), - (3, 2) => $expand!(3, 2), - (3, 3) => $expand!(3, 3), - (4, 0) => $expand!(4, 0), - (4, 1) => $expand!(4, 1), - (4, 2) => $expand!(4, 2), - (4, 3) => $expand!(4, 3), - (5, 0) => $expand!(5, 0), - (5, 1) => $expand!(5, 1), - (5, 2) => $expand!(5, 2), - (5, 3) => $expand!(5, 3), - (6, 0) => $expand!(6, 0), - (6, 1) => $expand!(6, 1), - (6, 2) => $expand!(6, 2), - (6, 3) => $expand!(6, 3), - (7, 0) => $expand!(7, 0), - (7, 1) => $expand!(7, 1), - (7, 2) => $expand!(7, 2), - (7, 3) => $expand!(7, 3), - (8, 0) => $expand!(8, 0), - (8, 1) => $expand!(8, 1), - (8, 2) => $expand!(8, 2), - (8, 3) => $expand!(8, 3), - (9, 0) => $expand!(9, 0), - (9, 1) => $expand!(9, 1), - (9, 2) => $expand!(9, 2), - (9, 3) => $expand!(9, 3), - (10, 0) => $expand!(10, 0), - (10, 1) => $expand!(10, 1), - (10, 2) => $expand!(10, 2), - (10, 3) => $expand!(10, 3), - (11, 0) => $expand!(11, 0), - (11, 1) => $expand!(11, 1), - (11, 2) => $expand!(11, 2), - (11, 3) => $expand!(11, 3), - (12, 0) => $expand!(12, 0), - (12, 1) => $expand!(12, 1), - (12, 2) => $expand!(12, 2), - (12, 3) => $expand!(12, 3), - (13, 0) => $expand!(13, 0), - (13, 1) => $expand!(13, 1), - (13, 2) => $expand!(13, 2), - (13, 3) => $expand!(13, 3), - (14, 0) => $expand!(14, 0), - (14, 1) => $expand!(14, 1), - (14, 2) => $expand!(14, 2), - (14, 3) => $expand!(14, 3), - (15, 0) => $expand!(15, 0), - (15, 1) => $expand!(15, 1), - (15, 2) => $expand!(15, 2), - (15, 3) => $expand!(15, 3), - (_, _) => panic!("Invalid sae value"), - } +// Helper struct used to trigger const eval errors when the unsigned const generic immediate value +// `IMM` is out of `[MIN-MAX]` range. +pub(crate) struct ValidateConstImmU32; +impl ValidateConstImmU32 { + pub(crate) const VALID: () = { + assert!(IMM >= MIN && IMM <= MAX, "IMM value not in expected range"); }; } -// Include mantissas parameters. -// For sae instructions, the only valid values for sae are 4 and 8. -// This macro enforces that. -#[allow(unused)] -macro_rules! constify_imm4_mantissas_sae { - ($imm4_1:expr, $imm2:expr, $imm4_2:expr, $expand:ident) => { - #[allow(overflowing_literals)] - match ($imm4_1, $imm2, $imm4_2) { - (0, 0, 4) => $expand!(0, 0, 4), - (0, 0, 8) => $expand!(0, 0, 8), - (0, 0, 12) => $expand!(0, 0, 12), - (0, 1, 4) => $expand!(0, 1, 4), - (0, 1, 8) => $expand!(0, 1, 8), - (0, 1, 12) => $expand!(0, 1, 12), - (0, 2, 4) => $expand!(0, 2, 4), - (0, 2, 8) => $expand!(0, 2, 8), - (0, 2, 12) => $expand!(0, 2, 12), - (0, 3, 4) => $expand!(0, 3, 4), - (0, 3, 8) => $expand!(0, 3, 8), - (0, 3, 12) => $expand!(0, 3, 12), - (1, 0, 4) => $expand!(1, 0, 4), - (1, 0, 8) => $expand!(1, 0, 8), - (1, 0, 12) => $expand!(1, 0, 12), - (1, 1, 4) => $expand!(1, 1, 4), - (1, 1, 8) => $expand!(1, 1, 8), - (1, 1, 12) => $expand!(1, 1, 12), - (1, 2, 4) => $expand!(1, 2, 4), - (1, 2, 8) => $expand!(1, 2, 8), - (1, 2, 12) => $expand!(1, 2, 12), - (1, 3, 4) => $expand!(1, 3, 4), - (1, 3, 8) => $expand!(1, 3, 8), - (1, 3, 12) => $expand!(1, 3, 12), - (2, 0, 4) => $expand!(2, 0, 4), - (2, 0, 8) => $expand!(2, 0, 8), - (2, 0, 12) => $expand!(2, 0, 12), - (2, 1, 4) => $expand!(2, 1, 4), - (2, 1, 8) => $expand!(2, 1, 8), - (2, 1, 12) => $expand!(2, 1, 12), - (2, 2, 4) => $expand!(2, 2, 4), - (2, 2, 8) => $expand!(2, 2, 8), - (2, 2, 12) => $expand!(2, 2, 12), - (2, 3, 4) => $expand!(2, 3, 4), - (2, 3, 8) => $expand!(2, 3, 8), - (2, 3, 12) => $expand!(2, 3, 12), - (3, 0, 4) => $expand!(3, 0, 4), - (3, 0, 8) => $expand!(3, 0, 8), - (3, 0, 12) => $expand!(3, 0, 12), - (3, 1, 4) => $expand!(3, 1, 4), - (3, 1, 8) => $expand!(3, 1, 8), - (3, 1, 12) => $expand!(3, 1, 12), - (3, 2, 4) => $expand!(3, 2, 4), - (3, 2, 8) => $expand!(3, 2, 8), - (3, 2, 12) => $expand!(3, 2, 12), - (3, 3, 4) => $expand!(3, 3, 4), - (3, 3, 8) => $expand!(3, 3, 8), - (3, 3, 12) => $expand!(3, 3, 12), - (4, 0, 4) => $expand!(4, 0, 4), - (4, 0, 8) => $expand!(4, 0, 8), - (4, 0, 12) => $expand!(4, 0, 12), - (4, 1, 4) => $expand!(4, 1, 4), - (4, 1, 8) => $expand!(4, 1, 8), - (4, 1, 12) => $expand!(4, 1, 12), - (4, 2, 4) => $expand!(4, 2, 4), - (4, 2, 8) => $expand!(4, 2, 8), - (4, 2, 12) => $expand!(4, 2, 12), - (4, 3, 4) => $expand!(4, 3, 4), - (4, 3, 8) => $expand!(4, 3, 8), - (4, 3, 12) => $expand!(4, 3, 12), - (5, 0, 4) => $expand!(5, 0, 4), - (5, 0, 8) => $expand!(5, 0, 8), - (5, 0, 12) => $expand!(5, 0, 12), - (5, 1, 4) => $expand!(5, 1, 4), - (5, 1, 8) => $expand!(5, 1, 8), - (5, 1, 12) => $expand!(5, 1, 12), - (5, 2, 4) => $expand!(5, 2, 4), - (5, 2, 8) => $expand!(5, 2, 8), - (5, 2, 12) => $expand!(5, 2, 12), - (5, 3, 4) => $expand!(5, 3, 4), - (5, 3, 8) => $expand!(5, 3, 8), - (5, 3, 12) => $expand!(5, 3, 12), - (6, 0, 4) => $expand!(6, 0, 4), - (6, 0, 8) => $expand!(6, 0, 8), - (6, 0, 12) => $expand!(6, 0, 12), - (6, 1, 4) => $expand!(6, 1, 4), - (6, 1, 8) => $expand!(6, 1, 8), - (6, 1, 12) => $expand!(6, 1, 12), - (6, 2, 4) => $expand!(6, 2, 4), - (6, 2, 8) => $expand!(6, 2, 8), - (6, 2, 12) => $expand!(6, 2, 12), - (6, 3, 4) => $expand!(6, 3, 4), - (6, 3, 8) => $expand!(6, 3, 8), - (6, 3, 12) => $expand!(6, 3, 12), - (7, 0, 4) => $expand!(7, 0, 4), - (7, 0, 8) => $expand!(7, 0, 8), - (7, 0, 12) => $expand!(7, 0, 12), - (7, 1, 4) => $expand!(7, 1, 4), - (7, 1, 8) => $expand!(7, 1, 8), - (7, 1, 12) => $expand!(7, 1, 12), - (7, 2, 4) => $expand!(7, 2, 4), - (7, 2, 8) => $expand!(7, 2, 8), - (7, 2, 12) => $expand!(7, 2, 12), - (7, 3, 4) => $expand!(7, 3, 4), - (7, 3, 8) => $expand!(7, 3, 8), - (7, 3, 12) => $expand!(7, 3, 12), - (8, 0, 4) => $expand!(8, 0, 4), - (8, 0, 8) => $expand!(8, 0, 8), - (8, 0, 12) => $expand!(8, 0, 12), - (8, 1, 4) => $expand!(8, 1, 4), - (8, 1, 8) => $expand!(8, 1, 8), - (8, 1, 12) => $expand!(8, 1, 12), - (8, 2, 4) => $expand!(8, 2, 4), - (8, 2, 8) => $expand!(8, 2, 8), - (8, 2, 12) => $expand!(8, 2, 12), - (8, 3, 4) => $expand!(8, 3, 4), - (8, 3, 8) => $expand!(8, 3, 8), - (8, 3, 12) => $expand!(8, 3, 12), - (9, 0, 4) => $expand!(9, 0, 4), - (9, 0, 8) => $expand!(9, 0, 8), - (9, 0, 12) => $expand!(9, 0, 12), - (9, 1, 4) => $expand!(9, 1, 4), - (9, 1, 8) => $expand!(9, 1, 8), - (9, 1, 12) => $expand!(9, 1, 12), - (9, 2, 4) => $expand!(9, 2, 4), - (9, 2, 8) => $expand!(9, 2, 8), - (9, 2, 12) => $expand!(9, 2, 12), - (9, 3, 4) => $expand!(9, 3, 4), - (9, 3, 8) => $expand!(9, 3, 8), - (9, 3, 12) => $expand!(9, 3, 12), - (10, 0, 4) => $expand!(10, 0, 4), - (10, 0, 8) => $expand!(10, 0, 8), - (10, 0, 12) => $expand!(10, 0, 12), - (10, 1, 4) => $expand!(10, 1, 4), - (10, 1, 8) => $expand!(10, 1, 8), - (10, 1, 12) => $expand!(10, 1, 12), - (10, 2, 4) => $expand!(10, 2, 4), - (10, 2, 8) => $expand!(10, 2, 8), - (10, 2, 12) => $expand!(10, 2, 12), - (10, 3, 4) => $expand!(10, 3, 4), - (10, 3, 8) => $expand!(10, 3, 8), - (10, 3, 12) => $expand!(10, 3, 12), - (11, 0, 4) => $expand!(11, 0, 4), - (11, 0, 8) => $expand!(11, 0, 8), - (11, 0, 12) => $expand!(11, 0, 12), - (11, 1, 4) => $expand!(11, 1, 4), - (11, 1, 8) => $expand!(11, 1, 8), - (11, 1, 12) => $expand!(11, 1, 12), - (11, 2, 4) => $expand!(11, 2, 4), - (11, 2, 8) => $expand!(11, 2, 8), - (11, 2, 12) => $expand!(11, 2, 12), - (11, 3, 4) => $expand!(11, 3, 4), - (11, 3, 8) => $expand!(11, 3, 8), - (11, 3, 12) => $expand!(11, 3, 12), - (12, 0, 4) => $expand!(12, 0, 4), - (12, 0, 8) => $expand!(12, 0, 8), - (12, 0, 12) => $expand!(12, 0, 12), - (12, 1, 4) => $expand!(12, 1, 4), - (12, 1, 8) => $expand!(12, 1, 8), - (12, 1, 12) => $expand!(12, 1, 12), - (12, 2, 4) => $expand!(12, 2, 4), - (12, 2, 8) => $expand!(12, 2, 8), - (12, 2, 12) => $expand!(12, 2, 12), - (12, 3, 4) => $expand!(12, 3, 4), - (12, 3, 8) => $expand!(12, 3, 8), - (12, 3, 12) => $expand!(12, 3, 12), - (13, 0, 4) => $expand!(13, 0, 4), - (13, 0, 8) => $expand!(13, 0, 8), - (13, 0, 12) => $expand!(13, 0, 12), - (13, 1, 4) => $expand!(13, 1, 4), - (13, 1, 8) => $expand!(13, 1, 8), - (13, 1, 12) => $expand!(13, 1, 12), - (13, 2, 4) => $expand!(13, 2, 4), - (13, 2, 8) => $expand!(13, 2, 8), - (13, 2, 12) => $expand!(13, 2, 12), - (13, 3, 4) => $expand!(13, 3, 4), - (13, 3, 8) => $expand!(13, 3, 8), - (13, 3, 12) => $expand!(13, 3, 12), - (14, 0, 4) => $expand!(14, 0, 4), - (14, 0, 8) => $expand!(14, 0, 8), - (14, 0, 12) => $expand!(14, 0, 12), - (14, 1, 4) => $expand!(14, 1, 4), - (14, 1, 8) => $expand!(14, 1, 8), - (14, 1, 12) => $expand!(14, 1, 12), - (14, 2, 4) => $expand!(14, 2, 4), - (14, 2, 8) => $expand!(14, 2, 8), - (14, 2, 12) => $expand!(14, 2, 12), - (14, 3, 4) => $expand!(14, 3, 4), - (14, 3, 8) => $expand!(14, 3, 8), - (14, 3, 12) => $expand!(14, 3, 12), - (15, 0, 4) => $expand!(15, 0, 4), - (15, 0, 8) => $expand!(15, 0, 8), - (15, 0, 12) => $expand!(15, 0, 12), - (15, 1, 4) => $expand!(15, 1, 4), - (15, 1, 8) => $expand!(15, 1, 8), - (15, 1, 12) => $expand!(15, 1, 12), - (15, 2, 4) => $expand!(15, 2, 4), - (15, 2, 8) => $expand!(15, 2, 8), - (15, 2, 12) => $expand!(15, 2, 12), - (15, 3, 4) => $expand!(15, 3, 4), - (15, 3, 8) => $expand!(15, 3, 8), - (15, 3, 12) => $expand!(15, 3, 12), - (_, _, _) => panic!("Invalid sae value"), - } +#[allow(unused_macros)] +macro_rules! static_assert_imm_u8 { + ($imm:ident) => { + let _ = + $crate::core_arch::x86::macros::ValidateConstImmU32::<$imm, 0, { (1 << 8) - 1 }>::VALID; }; } -// Constifies 8 bits along with an sae option without rounding control. -// The only valid values are 0 to 255. -// This macro enforces that. -#[allow(unused)] -macro_rules! constify_imm8_sae { - ($imm8:expr, $expand:ident) => { - #[allow(overflowing_literals)] - match ($imm8) & 0b1111_1111 { - 0 => $expand!(0), - 1 => $expand!(1), - 2 => $expand!(2), - 3 => $expand!(3), - 4 => $expand!(4), - 5 => $expand!(5), - 6 => $expand!(6), - 7 => $expand!(7), - 8 => $expand!(8), - 9 => $expand!(9), - 10 => $expand!(10), - 11 => $expand!(11), - 12 => $expand!(12), - 13 => $expand!(13), - 14 => $expand!(14), - 15 => $expand!(15), - 16 => $expand!(16), - 17 => $expand!(17), - 18 => $expand!(18), - 19 => $expand!(19), - 20 => $expand!(20), - 21 => $expand!(21), - 22 => $expand!(22), - 23 => $expand!(23), - 24 => $expand!(24), - 25 => $expand!(25), - 26 => $expand!(26), - 27 => $expand!(27), - 28 => $expand!(28), - 29 => $expand!(29), - 30 => $expand!(30), - 31 => $expand!(31), - 32 => $expand!(32), - 33 => $expand!(33), - 34 => $expand!(34), - 35 => $expand!(35), - 36 => $expand!(36), - 37 => $expand!(37), - 38 => $expand!(38), - 39 => $expand!(39), - 40 => $expand!(40), - 41 => $expand!(41), - 42 => $expand!(42), - 43 => $expand!(43), - 44 => $expand!(44), - 45 => $expand!(45), - 46 => $expand!(46), - 47 => $expand!(47), - 48 => $expand!(48), - 49 => $expand!(49), - 50 => $expand!(50), - 51 => $expand!(51), - 52 => $expand!(52), - 53 => $expand!(53), - 54 => $expand!(54), - 55 => $expand!(55), - 56 => $expand!(56), - 57 => $expand!(57), - 58 => $expand!(58), - 59 => $expand!(59), - 60 => $expand!(60), - 61 => $expand!(61), - 62 => $expand!(62), - 63 => $expand!(63), - 64 => $expand!(64), - 65 => $expand!(65), - 66 => $expand!(66), - 67 => $expand!(67), - 68 => $expand!(68), - 69 => $expand!(69), - 70 => $expand!(70), - 71 => $expand!(71), - 72 => $expand!(72), - 73 => $expand!(73), - 74 => $expand!(74), - 75 => $expand!(75), - 76 => $expand!(76), - 77 => $expand!(77), - 78 => $expand!(78), - 79 => $expand!(79), - 80 => $expand!(80), - 81 => $expand!(81), - 82 => $expand!(82), - 83 => $expand!(83), - 84 => $expand!(84), - 85 => $expand!(85), - 86 => $expand!(86), - 87 => $expand!(87), - 88 => $expand!(88), - 89 => $expand!(89), - 90 => $expand!(90), - 91 => $expand!(91), - 92 => $expand!(92), - 93 => $expand!(93), - 94 => $expand!(94), - 95 => $expand!(95), - 96 => $expand!(96), - 97 => $expand!(97), - 98 => $expand!(98), - 99 => $expand!(99), - 100 => $expand!(100), - 101 => $expand!(101), - 102 => $expand!(102), - 103 => $expand!(103), - 104 => $expand!(104), - 105 => $expand!(105), - 106 => $expand!(106), - 107 => $expand!(107), - 108 => $expand!(108), - 109 => $expand!(109), - 110 => $expand!(110), - 111 => $expand!(111), - 112 => $expand!(112), - 113 => $expand!(113), - 114 => $expand!(114), - 115 => $expand!(115), - 116 => $expand!(116), - 117 => $expand!(117), - 118 => $expand!(118), - 119 => $expand!(119), - 120 => $expand!(120), - 121 => $expand!(121), - 122 => $expand!(122), - 123 => $expand!(123), - 124 => $expand!(124), - 125 => $expand!(125), - 126 => $expand!(126), - 127 => $expand!(127), - 128 => $expand!(128), - 129 => $expand!(129), - 130 => $expand!(130), - 131 => $expand!(131), - 132 => $expand!(132), - 133 => $expand!(133), - 134 => $expand!(134), - 135 => $expand!(135), - 136 => $expand!(136), - 137 => $expand!(137), - 138 => $expand!(138), - 139 => $expand!(139), - 140 => $expand!(140), - 141 => $expand!(141), - 142 => $expand!(142), - 143 => $expand!(143), - 144 => $expand!(144), - 145 => $expand!(145), - 146 => $expand!(146), - 147 => $expand!(147), - 148 => $expand!(148), - 149 => $expand!(149), - 150 => $expand!(150), - 151 => $expand!(151), - 152 => $expand!(152), - 153 => $expand!(153), - 154 => $expand!(154), - 155 => $expand!(155), - 156 => $expand!(156), - 157 => $expand!(157), - 158 => $expand!(158), - 159 => $expand!(159), - 160 => $expand!(160), - 161 => $expand!(161), - 162 => $expand!(162), - 163 => $expand!(163), - 164 => $expand!(164), - 165 => $expand!(165), - 166 => $expand!(166), - 167 => $expand!(167), - 168 => $expand!(168), - 169 => $expand!(169), - 170 => $expand!(170), - 171 => $expand!(171), - 172 => $expand!(172), - 173 => $expand!(173), - 174 => $expand!(174), - 175 => $expand!(175), - 176 => $expand!(176), - 177 => $expand!(177), - 178 => $expand!(178), - 179 => $expand!(179), - 180 => $expand!(180), - 181 => $expand!(181), - 182 => $expand!(182), - 183 => $expand!(183), - 184 => $expand!(184), - 185 => $expand!(185), - 186 => $expand!(186), - 187 => $expand!(187), - 188 => $expand!(188), - 189 => $expand!(189), - 190 => $expand!(190), - 191 => $expand!(191), - 192 => $expand!(192), - 193 => $expand!(193), - 194 => $expand!(194), - 195 => $expand!(195), - 196 => $expand!(196), - 197 => $expand!(197), - 198 => $expand!(198), - 199 => $expand!(199), - 200 => $expand!(200), - 201 => $expand!(201), - 202 => $expand!(202), - 203 => $expand!(203), - 204 => $expand!(204), - 205 => $expand!(205), - 206 => $expand!(206), - 207 => $expand!(207), - 208 => $expand!(208), - 209 => $expand!(209), - 210 => $expand!(210), - 211 => $expand!(211), - 212 => $expand!(212), - 213 => $expand!(213), - 214 => $expand!(214), - 215 => $expand!(215), - 216 => $expand!(216), - 217 => $expand!(217), - 218 => $expand!(218), - 219 => $expand!(219), - 220 => $expand!(220), - 221 => $expand!(221), - 222 => $expand!(222), - 223 => $expand!(223), - 224 => $expand!(224), - 225 => $expand!(225), - 226 => $expand!(226), - 227 => $expand!(227), - 228 => $expand!(228), - 229 => $expand!(229), - 230 => $expand!(230), - 231 => $expand!(231), - 232 => $expand!(232), - 233 => $expand!(233), - 234 => $expand!(234), - 235 => $expand!(235), - 236 => $expand!(236), - 237 => $expand!(237), - 238 => $expand!(238), - 239 => $expand!(239), - 240 => $expand!(240), - 241 => $expand!(241), - 242 => $expand!(242), - 243 => $expand!(243), - 244 => $expand!(244), - 245 => $expand!(245), - 246 => $expand!(246), - 247 => $expand!(247), - 248 => $expand!(248), - 249 => $expand!(249), - 250 => $expand!(250), - 251 => $expand!(251), - 252 => $expand!(252), - 253 => $expand!(253), - 254 => $expand!(254), - 255 => $expand!(255), - _ => panic!("Invalid sae value"), - } +// Helper struct used to trigger const eval errors when the const generic immediate value `SCALE` is +// not valid for gather instructions: the only valid scale values are 1, 2, 4 and 8. +pub(crate) struct ValidateConstGatherScale; +impl ValidateConstGatherScale { + pub(crate) const VALID: () = { + assert!( + SCALE == 1 || SCALE == 2 || SCALE == 4 || SCALE == 8, + "Invalid SCALE value" + ); }; } -// Two sae parameters. -// This macro enforces that. #[allow(unused)] -macro_rules! constify_imm8_roundscale { - ($imm8:expr, $imm4:expr, $expand:ident) => { - #[allow(overflowing_literals)] - match ($imm8 & 0b11111111, $imm4) { - (0, 4) => $expand!(0, 4), - (0, 8) => $expand!(0, 8), - (0, 12) => $expand!(0, 12), - (1, 4) => $expand!(1, 4), - (1, 8) => $expand!(1, 8), - (1, 12) => $expand!(1, 12), - (2, 4) => $expand!(2, 4), - (2, 8) => $expand!(2, 8), - (2, 12) => $expand!(2, 12), - (3, 4) => $expand!(3, 4), - (3, 8) => $expand!(3, 8), - (3, 12) => $expand!(3, 12), - (4, 4) => $expand!(4, 4), - (4, 8) => $expand!(4, 8), - (4, 12) => $expand!(4, 12), - (5, 4) => $expand!(5, 4), - (5, 8) => $expand!(5, 8), - (5, 12) => $expand!(5, 12), - (6, 4) => $expand!(6, 4), - (6, 8) => $expand!(6, 8), - (6, 12) => $expand!(6, 12), - (7, 4) => $expand!(7, 4), - (7, 8) => $expand!(7, 8), - (7, 12) => $expand!(7, 12), - (8, 4) => $expand!(8, 4), - (8, 8) => $expand!(8, 8), - (8, 12) => $expand!(8, 12), - (9, 4) => $expand!(9, 4), - (9, 8) => $expand!(9, 8), - (9, 12) => $expand!(9, 12), - (10, 4) => $expand!(10, 4), - (10, 8) => $expand!(10, 8), - (10, 12) => $expand!(10, 12), - (11, 4) => $expand!(11, 4), - (11, 8) => $expand!(11, 8), - (11, 12) => $expand!(11, 12), - (12, 4) => $expand!(12, 4), - (12, 8) => $expand!(12, 8), - (12, 12) => $expand!(12, 12), - (13, 4) => $expand!(13, 4), - (13, 8) => $expand!(13, 8), - (13, 12) => $expand!(13, 12), - (14, 4) => $expand!(14, 4), - (14, 8) => $expand!(14, 8), - (14, 12) => $expand!(14, 12), - (15, 4) => $expand!(15, 4), - (15, 8) => $expand!(15, 8), - (15, 12) => $expand!(15, 12), - (16, 4) => $expand!(16, 4), - (16, 8) => $expand!(16, 8), - (16, 12) => $expand!(16, 12), - (17, 4) => $expand!(17, 4), - (17, 8) => $expand!(17, 8), - (17, 12) => $expand!(17, 12), - (18, 4) => $expand!(18, 4), - (18, 8) => $expand!(18, 8), - (18, 12) => $expand!(18, 12), - (19, 4) => $expand!(19, 4), - (19, 8) => $expand!(19, 8), - (19, 12) => $expand!(19, 12), - (20, 4) => $expand!(20, 4), - (20, 8) => $expand!(20, 8), - (20, 12) => $expand!(20, 12), - (21, 4) => $expand!(21, 4), - (21, 8) => $expand!(21, 8), - (21, 12) => $expand!(21, 12), - (22, 4) => $expand!(22, 4), - (22, 8) => $expand!(22, 8), - (22, 12) => $expand!(22, 12), - (23, 4) => $expand!(23, 4), - (23, 8) => $expand!(23, 8), - (23, 12) => $expand!(23, 12), - (24, 4) => $expand!(24, 4), - (24, 8) => $expand!(24, 8), - (24, 12) => $expand!(24, 12), - (25, 4) => $expand!(25, 4), - (25, 8) => $expand!(25, 8), - (25, 12) => $expand!(25, 12), - (26, 4) => $expand!(26, 4), - (26, 8) => $expand!(26, 8), - (26, 12) => $expand!(26, 12), - (27, 4) => $expand!(27, 4), - (27, 8) => $expand!(27, 8), - (27, 12) => $expand!(27, 12), - (28, 4) => $expand!(28, 4), - (28, 8) => $expand!(28, 8), - (28, 12) => $expand!(28, 12), - (29, 4) => $expand!(29, 4), - (29, 8) => $expand!(29, 8), - (29, 12) => $expand!(29, 12), - (30, 4) => $expand!(30, 4), - (30, 8) => $expand!(30, 8), - (30, 12) => $expand!(30, 12), - (31, 4) => $expand!(31, 4), - (31, 8) => $expand!(31, 8), - (31, 12) => $expand!(31, 12), - (32, 4) => $expand!(32, 4), - (32, 8) => $expand!(32, 8), - (32, 12) => $expand!(32, 12), - (33, 4) => $expand!(33, 4), - (33, 8) => $expand!(33, 8), - (33, 12) => $expand!(33, 12), - (34, 4) => $expand!(34, 4), - (34, 8) => $expand!(34, 8), - (34, 12) => $expand!(34, 12), - (35, 4) => $expand!(35, 4), - (35, 8) => $expand!(35, 8), - (35, 12) => $expand!(35, 12), - (36, 4) => $expand!(36, 4), - (36, 8) => $expand!(36, 8), - (36, 12) => $expand!(36, 12), - (37, 4) => $expand!(37, 4), - (37, 8) => $expand!(37, 8), - (37, 12) => $expand!(37, 12), - (38, 4) => $expand!(38, 4), - (38, 8) => $expand!(38, 8), - (38, 12) => $expand!(38, 12), - (39, 4) => $expand!(39, 4), - (39, 8) => $expand!(39, 8), - (39, 12) => $expand!(39, 12), - (40, 4) => $expand!(40, 4), - (40, 8) => $expand!(40, 8), - (40, 12) => $expand!(40, 12), - (41, 4) => $expand!(41, 4), - (41, 8) => $expand!(41, 8), - (41, 12) => $expand!(41, 12), - (42, 4) => $expand!(42, 4), - (42, 8) => $expand!(42, 8), - (42, 12) => $expand!(42, 12), - (43, 4) => $expand!(43, 4), - (43, 8) => $expand!(43, 8), - (43, 12) => $expand!(43, 12), - (44, 4) => $expand!(44, 4), - (44, 8) => $expand!(44, 8), - (44, 12) => $expand!(44, 12), - (45, 4) => $expand!(45, 4), - (45, 8) => $expand!(45, 8), - (45, 12) => $expand!(45, 12), - (46, 4) => $expand!(46, 4), - (46, 8) => $expand!(46, 8), - (46, 12) => $expand!(46, 12), - (47, 4) => $expand!(47, 4), - (47, 8) => $expand!(47, 8), - (47, 12) => $expand!(47, 12), - (48, 4) => $expand!(48, 4), - (48, 8) => $expand!(48, 8), - (48, 12) => $expand!(48, 12), - (49, 4) => $expand!(49, 4), - (49, 8) => $expand!(49, 8), - (49, 12) => $expand!(49, 12), - (50, 4) => $expand!(50, 4), - (50, 8) => $expand!(50, 8), - (50, 12) => $expand!(50, 12), - (51, 4) => $expand!(51, 4), - (51, 8) => $expand!(51, 8), - (51, 12) => $expand!(51, 12), - (52, 4) => $expand!(52, 4), - (52, 8) => $expand!(52, 8), - (52, 12) => $expand!(52, 12), - (53, 4) => $expand!(53, 4), - (53, 8) => $expand!(53, 8), - (53, 12) => $expand!(53, 12), - (54, 4) => $expand!(54, 4), - (54, 8) => $expand!(54, 8), - (54, 12) => $expand!(54, 12), - (55, 4) => $expand!(55, 4), - (55, 8) => $expand!(55, 8), - (55, 12) => $expand!(55, 12), - (56, 4) => $expand!(56, 4), - (56, 8) => $expand!(56, 8), - (56, 12) => $expand!(56, 12), - (57, 4) => $expand!(57, 4), - (57, 8) => $expand!(57, 8), - (57, 12) => $expand!(57, 12), - (58, 4) => $expand!(58, 4), - (58, 8) => $expand!(58, 8), - (58, 12) => $expand!(58, 12), - (59, 4) => $expand!(59, 4), - (59, 8) => $expand!(59, 8), - (59, 12) => $expand!(59, 12), - (60, 4) => $expand!(60, 4), - (60, 8) => $expand!(60, 8), - (60, 12) => $expand!(60, 12), - (61, 4) => $expand!(61, 4), - (61, 8) => $expand!(61, 8), - (61, 12) => $expand!(61, 12), - (62, 4) => $expand!(62, 4), - (62, 8) => $expand!(62, 8), - (62, 12) => $expand!(62, 12), - (63, 4) => $expand!(63, 4), - (63, 8) => $expand!(63, 8), - (63, 12) => $expand!(63, 12), - (64, 4) => $expand!(64, 4), - (64, 8) => $expand!(64, 8), - (64, 12) => $expand!(64, 12), - (65, 4) => $expand!(65, 4), - (65, 8) => $expand!(65, 8), - (65, 12) => $expand!(65, 12), - (66, 4) => $expand!(66, 4), - (66, 8) => $expand!(66, 8), - (66, 12) => $expand!(66, 12), - (67, 4) => $expand!(67, 4), - (67, 8) => $expand!(67, 8), - (67, 12) => $expand!(67, 12), - (68, 4) => $expand!(68, 4), - (68, 8) => $expand!(68, 8), - (68, 12) => $expand!(68, 12), - (69, 4) => $expand!(69, 4), - (69, 8) => $expand!(69, 8), - (69, 12) => $expand!(69, 12), - (70, 4) => $expand!(70, 4), - (70, 8) => $expand!(70, 8), - (70, 12) => $expand!(70, 12), - (71, 4) => $expand!(71, 4), - (71, 8) => $expand!(71, 8), - (71, 12) => $expand!(71, 12), - (72, 4) => $expand!(72, 4), - (72, 8) => $expand!(72, 8), - (72, 12) => $expand!(72, 12), - (73, 4) => $expand!(73, 4), - (73, 8) => $expand!(73, 8), - (73, 12) => $expand!(73, 12), - (74, 4) => $expand!(74, 4), - (74, 8) => $expand!(74, 8), - (74, 12) => $expand!(74, 12), - (75, 4) => $expand!(75, 4), - (75, 8) => $expand!(75, 8), - (75, 12) => $expand!(75, 12), - (76, 4) => $expand!(76, 4), - (76, 8) => $expand!(76, 8), - (76, 12) => $expand!(76, 12), - (77, 4) => $expand!(77, 4), - (77, 8) => $expand!(77, 8), - (77, 12) => $expand!(77, 12), - (78, 4) => $expand!(78, 4), - (78, 8) => $expand!(78, 8), - (78, 12) => $expand!(78, 12), - (79, 4) => $expand!(79, 4), - (79, 8) => $expand!(79, 8), - (79, 12) => $expand!(79, 12), - (80, 4) => $expand!(80, 4), - (80, 8) => $expand!(80, 8), - (80, 12) => $expand!(80, 12), - (81, 4) => $expand!(81, 4), - (81, 8) => $expand!(81, 8), - (81, 12) => $expand!(81, 12), - (82, 4) => $expand!(82, 4), - (82, 8) => $expand!(82, 8), - (82, 12) => $expand!(82, 12), - (83, 4) => $expand!(83, 4), - (83, 8) => $expand!(83, 8), - (83, 12) => $expand!(83, 12), - (84, 4) => $expand!(84, 4), - (84, 8) => $expand!(84, 8), - (84, 12) => $expand!(84, 12), - (85, 4) => $expand!(85, 4), - (85, 8) => $expand!(85, 8), - (85, 12) => $expand!(85, 12), - (86, 4) => $expand!(86, 4), - (86, 8) => $expand!(86, 8), - (86, 12) => $expand!(86, 12), - (87, 4) => $expand!(87, 4), - (87, 8) => $expand!(87, 8), - (87, 12) => $expand!(87, 12), - (88, 4) => $expand!(88, 4), - (88, 8) => $expand!(88, 8), - (88, 12) => $expand!(88, 12), - (89, 4) => $expand!(89, 4), - (89, 8) => $expand!(89, 8), - (89, 12) => $expand!(89, 12), - (90, 4) => $expand!(90, 4), - (90, 8) => $expand!(90, 8), - (90, 12) => $expand!(90, 12), - (91, 4) => $expand!(91, 4), - (91, 8) => $expand!(91, 8), - (91, 12) => $expand!(91, 12), - (92, 4) => $expand!(92, 4), - (92, 8) => $expand!(92, 8), - (92, 12) => $expand!(92, 12), - (93, 4) => $expand!(93, 4), - (93, 8) => $expand!(93, 8), - (93, 12) => $expand!(93, 12), - (94, 4) => $expand!(94, 4), - (94, 8) => $expand!(94, 8), - (94, 12) => $expand!(94, 12), - (95, 4) => $expand!(95, 4), - (95, 8) => $expand!(95, 8), - (95, 12) => $expand!(95, 12), - (96, 4) => $expand!(96, 4), - (96, 8) => $expand!(96, 8), - (96, 12) => $expand!(96, 12), - (97, 4) => $expand!(97, 4), - (97, 8) => $expand!(97, 8), - (97, 12) => $expand!(97, 12), - (98, 4) => $expand!(98, 4), - (98, 8) => $expand!(98, 8), - (98, 12) => $expand!(98, 12), - (99, 4) => $expand!(99, 4), - (99, 8) => $expand!(99, 8), - (99, 12) => $expand!(99, 12), - (100, 4) => $expand!(100, 4), - (100, 8) => $expand!(100, 8), - (100, 12) => $expand!(100, 12), - (101, 4) => $expand!(101, 4), - (101, 8) => $expand!(101, 8), - (101, 12) => $expand!(101, 12), - (102, 4) => $expand!(102, 4), - (102, 8) => $expand!(102, 8), - (102, 12) => $expand!(102, 12), - (103, 4) => $expand!(103, 4), - (103, 8) => $expand!(103, 8), - (103, 12) => $expand!(103, 12), - (104, 4) => $expand!(104, 4), - (104, 8) => $expand!(104, 8), - (104, 12) => $expand!(104, 12), - (105, 4) => $expand!(105, 4), - (105, 8) => $expand!(105, 8), - (105, 12) => $expand!(105, 12), - (106, 4) => $expand!(106, 4), - (106, 8) => $expand!(106, 8), - (106, 12) => $expand!(106, 12), - (107, 4) => $expand!(107, 4), - (107, 8) => $expand!(107, 8), - (107, 12) => $expand!(107, 12), - (108, 4) => $expand!(108, 4), - (108, 8) => $expand!(108, 8), - (108, 12) => $expand!(108, 12), - (109, 4) => $expand!(109, 4), - (109, 8) => $expand!(109, 8), - (109, 12) => $expand!(109, 12), - (110, 4) => $expand!(110, 4), - (110, 8) => $expand!(110, 8), - (110, 12) => $expand!(110, 12), - (111, 4) => $expand!(111, 4), - (111, 8) => $expand!(111, 8), - (111, 12) => $expand!(111, 12), - (112, 4) => $expand!(112, 4), - (112, 8) => $expand!(112, 8), - (112, 12) => $expand!(112, 12), - (113, 4) => $expand!(113, 4), - (113, 8) => $expand!(113, 8), - (113, 12) => $expand!(113, 12), - (114, 4) => $expand!(114, 4), - (114, 8) => $expand!(114, 8), - (114, 12) => $expand!(114, 12), - (115, 4) => $expand!(115, 4), - (115, 8) => $expand!(115, 8), - (115, 12) => $expand!(115, 12), - (116, 4) => $expand!(116, 4), - (116, 8) => $expand!(116, 8), - (116, 12) => $expand!(116, 12), - (117, 4) => $expand!(117, 4), - (117, 8) => $expand!(117, 8), - (117, 12) => $expand!(117, 12), - (118, 4) => $expand!(118, 4), - (118, 8) => $expand!(118, 8), - (118, 12) => $expand!(118, 12), - (119, 4) => $expand!(119, 4), - (119, 8) => $expand!(119, 8), - (119, 12) => $expand!(119, 12), - (120, 4) => $expand!(120, 4), - (120, 8) => $expand!(120, 8), - (120, 12) => $expand!(120, 12), - (121, 4) => $expand!(121, 4), - (121, 8) => $expand!(121, 8), - (121, 12) => $expand!(121, 12), - (122, 4) => $expand!(122, 4), - (122, 8) => $expand!(122, 8), - (122, 12) => $expand!(122, 12), - (123, 4) => $expand!(123, 4), - (123, 8) => $expand!(123, 8), - (123, 12) => $expand!(123, 12), - (124, 4) => $expand!(124, 4), - (124, 8) => $expand!(124, 8), - (124, 12) => $expand!(124, 12), - (125, 4) => $expand!(125, 4), - (125, 8) => $expand!(125, 8), - (125, 12) => $expand!(125, 12), - (126, 4) => $expand!(126, 4), - (126, 8) => $expand!(126, 8), - (126, 12) => $expand!(126, 12), - (127, 4) => $expand!(127, 4), - (127, 8) => $expand!(127, 8), - (127, 12) => $expand!(127, 12), - (128, 4) => $expand!(128, 4), - (128, 8) => $expand!(128, 8), - (128, 12) => $expand!(128, 12), - (129, 4) => $expand!(129, 4), - (129, 8) => $expand!(129, 8), - (129, 12) => $expand!(129, 12), - (130, 4) => $expand!(130, 4), - (130, 8) => $expand!(130, 8), - (130, 12) => $expand!(130, 12), - (131, 4) => $expand!(131, 4), - (131, 8) => $expand!(131, 8), - (131, 12) => $expand!(131, 12), - (132, 4) => $expand!(132, 4), - (132, 8) => $expand!(132, 8), - (132, 12) => $expand!(132, 12), - (133, 4) => $expand!(133, 4), - (133, 8) => $expand!(133, 8), - (133, 12) => $expand!(133, 12), - (134, 4) => $expand!(134, 4), - (134, 8) => $expand!(134, 8), - (134, 12) => $expand!(134, 12), - (135, 4) => $expand!(135, 4), - (135, 8) => $expand!(135, 8), - (135, 12) => $expand!(135, 12), - (136, 4) => $expand!(136, 4), - (136, 8) => $expand!(136, 8), - (136, 12) => $expand!(136, 12), - (137, 4) => $expand!(137, 4), - (137, 8) => $expand!(137, 8), - (137, 12) => $expand!(137, 12), - (138, 4) => $expand!(138, 4), - (138, 8) => $expand!(138, 8), - (138, 12) => $expand!(138, 12), - (139, 4) => $expand!(139, 4), - (139, 8) => $expand!(139, 8), - (139, 12) => $expand!(139, 12), - (140, 4) => $expand!(140, 4), - (140, 8) => $expand!(140, 8), - (140, 12) => $expand!(140, 12), - (141, 4) => $expand!(141, 4), - (141, 8) => $expand!(141, 8), - (141, 12) => $expand!(141, 12), - (142, 4) => $expand!(142, 4), - (142, 8) => $expand!(142, 8), - (142, 12) => $expand!(142, 12), - (143, 4) => $expand!(143, 4), - (143, 8) => $expand!(143, 8), - (143, 12) => $expand!(143, 12), - (144, 4) => $expand!(144, 4), - (144, 8) => $expand!(144, 8), - (144, 12) => $expand!(144, 12), - (145, 4) => $expand!(145, 4), - (145, 8) => $expand!(145, 8), - (145, 12) => $expand!(145, 12), - (146, 4) => $expand!(146, 4), - (146, 8) => $expand!(146, 8), - (146, 12) => $expand!(146, 12), - (147, 4) => $expand!(147, 4), - (147, 8) => $expand!(147, 8), - (147, 12) => $expand!(147, 12), - (148, 4) => $expand!(148, 4), - (148, 8) => $expand!(148, 8), - (148, 12) => $expand!(148, 12), - (149, 4) => $expand!(149, 4), - (149, 8) => $expand!(149, 8), - (149, 12) => $expand!(149, 12), - (150, 4) => $expand!(150, 4), - (150, 8) => $expand!(150, 8), - (150, 12) => $expand!(150, 12), - (151, 4) => $expand!(151, 4), - (151, 8) => $expand!(151, 8), - (151, 12) => $expand!(151, 12), - (152, 4) => $expand!(152, 4), - (152, 8) => $expand!(152, 8), - (152, 12) => $expand!(152, 12), - (153, 4) => $expand!(153, 4), - (153, 8) => $expand!(153, 8), - (153, 12) => $expand!(153, 12), - (154, 4) => $expand!(154, 4), - (154, 8) => $expand!(154, 8), - (154, 12) => $expand!(154, 12), - (155, 4) => $expand!(155, 4), - (155, 8) => $expand!(155, 8), - (155, 12) => $expand!(155, 12), - (156, 4) => $expand!(156, 4), - (156, 8) => $expand!(156, 8), - (156, 12) => $expand!(156, 12), - (157, 4) => $expand!(157, 4), - (157, 8) => $expand!(157, 8), - (157, 12) => $expand!(157, 12), - (158, 4) => $expand!(158, 4), - (158, 8) => $expand!(158, 8), - (158, 12) => $expand!(158, 12), - (159, 4) => $expand!(159, 4), - (159, 8) => $expand!(159, 8), - (159, 12) => $expand!(159, 12), - (160, 4) => $expand!(160, 4), - (160, 8) => $expand!(160, 8), - (160, 12) => $expand!(160, 12), - (161, 4) => $expand!(161, 4), - (161, 8) => $expand!(161, 8), - (161, 12) => $expand!(161, 12), - (162, 4) => $expand!(162, 4), - (162, 8) => $expand!(162, 8), - (162, 12) => $expand!(162, 12), - (163, 4) => $expand!(163, 4), - (163, 8) => $expand!(163, 8), - (163, 12) => $expand!(163, 12), - (164, 4) => $expand!(164, 4), - (164, 8) => $expand!(164, 8), - (164, 12) => $expand!(164, 12), - (165, 4) => $expand!(165, 4), - (165, 8) => $expand!(165, 8), - (165, 12) => $expand!(165, 12), - (166, 4) => $expand!(166, 4), - (166, 8) => $expand!(166, 8), - (166, 12) => $expand!(166, 12), - (167, 4) => $expand!(167, 4), - (167, 8) => $expand!(167, 8), - (167, 12) => $expand!(167, 12), - (168, 4) => $expand!(168, 4), - (168, 8) => $expand!(168, 8), - (168, 12) => $expand!(168, 12), - (169, 4) => $expand!(169, 4), - (169, 8) => $expand!(169, 8), - (169, 12) => $expand!(169, 12), - (170, 4) => $expand!(170, 4), - (170, 8) => $expand!(170, 8), - (170, 12) => $expand!(170, 12), - (171, 4) => $expand!(171, 4), - (171, 8) => $expand!(171, 8), - (171, 12) => $expand!(171, 12), - (172, 4) => $expand!(172, 4), - (172, 8) => $expand!(172, 8), - (172, 12) => $expand!(172, 12), - (173, 4) => $expand!(173, 4), - (173, 8) => $expand!(173, 8), - (173, 12) => $expand!(173, 12), - (174, 4) => $expand!(174, 4), - (174, 8) => $expand!(174, 8), - (174, 12) => $expand!(174, 12), - (175, 4) => $expand!(175, 4), - (175, 8) => $expand!(175, 8), - (175, 12) => $expand!(175, 12), - (176, 4) => $expand!(176, 4), - (176, 8) => $expand!(176, 8), - (176, 12) => $expand!(176, 12), - (177, 4) => $expand!(177, 4), - (177, 8) => $expand!(177, 8), - (177, 12) => $expand!(177, 12), - (178, 4) => $expand!(178, 4), - (178, 8) => $expand!(178, 8), - (178, 12) => $expand!(178, 12), - (179, 4) => $expand!(179, 4), - (179, 8) => $expand!(179, 8), - (179, 12) => $expand!(179, 12), - (180, 4) => $expand!(180, 4), - (180, 8) => $expand!(180, 8), - (180, 12) => $expand!(180, 12), - (181, 4) => $expand!(181, 4), - (181, 8) => $expand!(181, 8), - (181, 12) => $expand!(181, 12), - (182, 4) => $expand!(182, 4), - (182, 8) => $expand!(182, 8), - (182, 12) => $expand!(182, 12), - (183, 4) => $expand!(183, 4), - (183, 8) => $expand!(183, 8), - (183, 12) => $expand!(183, 12), - (184, 4) => $expand!(184, 4), - (184, 8) => $expand!(184, 8), - (184, 12) => $expand!(184, 12), - (185, 4) => $expand!(185, 4), - (185, 8) => $expand!(185, 8), - (185, 12) => $expand!(185, 12), - (186, 4) => $expand!(186, 4), - (186, 8) => $expand!(186, 8), - (186, 12) => $expand!(186, 12), - (187, 4) => $expand!(187, 4), - (187, 8) => $expand!(187, 8), - (187, 12) => $expand!(187, 12), - (188, 4) => $expand!(188, 4), - (188, 8) => $expand!(188, 8), - (188, 12) => $expand!(188, 12), - (189, 4) => $expand!(189, 4), - (189, 8) => $expand!(189, 8), - (189, 12) => $expand!(189, 12), - (190, 4) => $expand!(190, 4), - (190, 8) => $expand!(190, 8), - (190, 12) => $expand!(190, 12), - (191, 4) => $expand!(191, 4), - (191, 8) => $expand!(191, 8), - (191, 12) => $expand!(191, 12), - (192, 4) => $expand!(192, 4), - (192, 8) => $expand!(192, 8), - (192, 12) => $expand!(192, 12), - (193, 4) => $expand!(193, 4), - (193, 8) => $expand!(193, 8), - (193, 12) => $expand!(193, 12), - (194, 4) => $expand!(194, 4), - (194, 8) => $expand!(194, 8), - (194, 12) => $expand!(194, 12), - (195, 4) => $expand!(195, 4), - (195, 8) => $expand!(195, 8), - (195, 12) => $expand!(195, 12), - (196, 4) => $expand!(196, 4), - (196, 8) => $expand!(196, 8), - (196, 12) => $expand!(196, 12), - (197, 4) => $expand!(197, 4), - (197, 8) => $expand!(197, 8), - (197, 12) => $expand!(197, 12), - (198, 4) => $expand!(198, 4), - (198, 8) => $expand!(198, 8), - (198, 12) => $expand!(198, 12), - (199, 4) => $expand!(199, 4), - (199, 8) => $expand!(199, 8), - (199, 12) => $expand!(199, 12), - (200, 4) => $expand!(200, 4), - (200, 8) => $expand!(200, 8), - (200, 12) => $expand!(200, 12), - (201, 4) => $expand!(201, 4), - (201, 8) => $expand!(201, 8), - (201, 12) => $expand!(201, 12), - (202, 4) => $expand!(202, 4), - (202, 8) => $expand!(202, 8), - (202, 12) => $expand!(202, 12), - (203, 4) => $expand!(203, 4), - (203, 8) => $expand!(203, 8), - (203, 12) => $expand!(203, 12), - (204, 4) => $expand!(204, 4), - (204, 8) => $expand!(204, 8), - (204, 12) => $expand!(204, 12), - (205, 4) => $expand!(205, 4), - (205, 8) => $expand!(205, 8), - (205, 12) => $expand!(205, 12), - (206, 4) => $expand!(206, 4), - (206, 8) => $expand!(206, 8), - (206, 12) => $expand!(206, 12), - (207, 4) => $expand!(207, 4), - (207, 8) => $expand!(207, 8), - (207, 12) => $expand!(207, 12), - (208, 4) => $expand!(208, 4), - (208, 8) => $expand!(208, 8), - (208, 12) => $expand!(208, 12), - (209, 4) => $expand!(209, 4), - (209, 8) => $expand!(209, 8), - (209, 12) => $expand!(209, 12), - (210, 4) => $expand!(210, 4), - (210, 8) => $expand!(210, 8), - (210, 12) => $expand!(210, 12), - (211, 4) => $expand!(211, 4), - (211, 8) => $expand!(211, 8), - (211, 12) => $expand!(211, 12), - (212, 4) => $expand!(212, 4), - (212, 8) => $expand!(212, 8), - (212, 12) => $expand!(212, 12), - (213, 4) => $expand!(213, 4), - (213, 8) => $expand!(213, 8), - (213, 12) => $expand!(213, 12), - (214, 4) => $expand!(214, 4), - (214, 8) => $expand!(214, 8), - (214, 12) => $expand!(214, 12), - (215, 4) => $expand!(215, 4), - (215, 8) => $expand!(215, 8), - (215, 12) => $expand!(215, 12), - (216, 4) => $expand!(216, 4), - (216, 8) => $expand!(216, 8), - (216, 12) => $expand!(216, 12), - (217, 4) => $expand!(217, 4), - (217, 8) => $expand!(217, 8), - (217, 12) => $expand!(217, 12), - (218, 4) => $expand!(218, 4), - (218, 8) => $expand!(218, 8), - (218, 12) => $expand!(218, 12), - (219, 4) => $expand!(219, 4), - (219, 8) => $expand!(219, 8), - (219, 12) => $expand!(219, 12), - (220, 4) => $expand!(220, 4), - (220, 8) => $expand!(220, 8), - (220, 12) => $expand!(220, 12), - (221, 4) => $expand!(221, 4), - (221, 8) => $expand!(221, 8), - (221, 12) => $expand!(221, 12), - (222, 4) => $expand!(222, 4), - (222, 8) => $expand!(222, 8), - (222, 12) => $expand!(222, 12), - (223, 4) => $expand!(223, 4), - (223, 8) => $expand!(223, 8), - (223, 12) => $expand!(223, 12), - (224, 4) => $expand!(224, 4), - (224, 8) => $expand!(224, 8), - (224, 12) => $expand!(224, 12), - (225, 4) => $expand!(225, 4), - (225, 8) => $expand!(225, 8), - (225, 12) => $expand!(225, 12), - (226, 4) => $expand!(226, 4), - (226, 8) => $expand!(226, 8), - (226, 12) => $expand!(226, 12), - (227, 4) => $expand!(227, 4), - (227, 8) => $expand!(227, 8), - (227, 12) => $expand!(227, 12), - (228, 4) => $expand!(228, 4), - (228, 8) => $expand!(228, 8), - (228, 12) => $expand!(228, 12), - (229, 4) => $expand!(229, 4), - (229, 8) => $expand!(229, 8), - (229, 12) => $expand!(229, 12), - (230, 4) => $expand!(230, 4), - (230, 8) => $expand!(230, 8), - (230, 12) => $expand!(230, 12), - (231, 4) => $expand!(231, 4), - (231, 8) => $expand!(231, 8), - (231, 12) => $expand!(231, 12), - (232, 4) => $expand!(232, 4), - (232, 8) => $expand!(232, 8), - (232, 12) => $expand!(232, 12), - (233, 4) => $expand!(233, 4), - (233, 8) => $expand!(233, 8), - (233, 12) => $expand!(233, 12), - (234, 4) => $expand!(234, 4), - (234, 8) => $expand!(234, 8), - (234, 12) => $expand!(234, 12), - (235, 4) => $expand!(235, 4), - (235, 8) => $expand!(235, 8), - (235, 12) => $expand!(235, 12), - (236, 4) => $expand!(236, 4), - (236, 8) => $expand!(236, 8), - (236, 12) => $expand!(236, 12), - (237, 4) => $expand!(237, 4), - (237, 8) => $expand!(237, 8), - (237, 12) => $expand!(237, 12), - (238, 4) => $expand!(238, 4), - (238, 8) => $expand!(238, 8), - (238, 12) => $expand!(238, 12), - (239, 4) => $expand!(239, 4), - (239, 8) => $expand!(239, 8), - (239, 12) => $expand!(239, 12), - (240, 4) => $expand!(240, 4), - (240, 8) => $expand!(240, 8), - (240, 12) => $expand!(240, 12), - (241, 4) => $expand!(241, 4), - (241, 8) => $expand!(241, 8), - (241, 12) => $expand!(241, 12), - (242, 4) => $expand!(242, 4), - (242, 8) => $expand!(242, 8), - (242, 12) => $expand!(242, 12), - (243, 4) => $expand!(243, 4), - (243, 8) => $expand!(243, 8), - (243, 12) => $expand!(243, 12), - (244, 4) => $expand!(244, 4), - (244, 8) => $expand!(244, 8), - (244, 12) => $expand!(244, 12), - (245, 4) => $expand!(245, 4), - (245, 8) => $expand!(245, 8), - (245, 12) => $expand!(245, 12), - (246, 4) => $expand!(246, 4), - (246, 8) => $expand!(246, 8), - (246, 12) => $expand!(246, 12), - (247, 4) => $expand!(247, 4), - (247, 8) => $expand!(247, 8), - (247, 12) => $expand!(247, 12), - (248, 4) => $expand!(248, 4), - (248, 8) => $expand!(248, 8), - (248, 12) => $expand!(248, 12), - (249, 4) => $expand!(249, 4), - (249, 8) => $expand!(249, 8), - (249, 12) => $expand!(249, 12), - (250, 4) => $expand!(250, 4), - (250, 8) => $expand!(250, 8), - (250, 12) => $expand!(250, 12), - (251, 4) => $expand!(251, 4), - (251, 8) => $expand!(251, 8), - (251, 12) => $expand!(251, 12), - (252, 4) => $expand!(252, 4), - (252, 8) => $expand!(252, 8), - (252, 12) => $expand!(252, 12), - (253, 4) => $expand!(253, 4), - (253, 8) => $expand!(253, 8), - (253, 12) => $expand!(253, 12), - (254, 4) => $expand!(254, 4), - (254, 8) => $expand!(254, 8), - (254, 12) => $expand!(254, 12), - (255, 4) => $expand!(255, 4), - (255, 8) => $expand!(255, 8), - (255, 12) => $expand!(255, 12), - (_, _) => panic!("Invalid sae value"), - } +macro_rules! static_assert_imm8_scale { + ($imm:ident) => { + let _ = $crate::core_arch::x86::macros::ValidateConstGatherScale::<$imm>::VALID; }; } diff --git a/library/stdarch/crates/core_arch/src/x86/pclmulqdq.rs b/library/stdarch/crates/core_arch/src/x86/pclmulqdq.rs index a9d5167b91..a2ebdf9c85 100644 --- a/library/stdarch/crates/core_arch/src/x86/pclmulqdq.rs +++ b/library/stdarch/crates/core_arch/src/x86/pclmulqdq.rs @@ -25,20 +25,16 @@ extern "C" { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_clmulepi64_si128) #[inline] #[target_feature(enable = "pclmulqdq")] -#[cfg_attr(all(test, not(target_os = "linux")), assert_instr(pclmulqdq, imm8 = 0))] -#[cfg_attr(all(test, target_os = "linux"), assert_instr(pclmullqlqdq, imm8 = 0))] -#[cfg_attr(all(test, target_os = "linux"), assert_instr(pclmulhqlqdq, imm8 = 1))] -#[cfg_attr(all(test, target_os = "linux"), assert_instr(pclmullqhqdq, imm8 = 16))] -#[cfg_attr(all(test, target_os = "linux"), assert_instr(pclmulhqhqdq, imm8 = 17))] -#[rustc_args_required_const(2)] +#[cfg_attr(all(test, not(target_os = "linux")), assert_instr(pclmulqdq, IMM8 = 0))] +#[cfg_attr(all(test, target_os = "linux"), assert_instr(pclmullqlqdq, IMM8 = 0))] +#[cfg_attr(all(test, target_os = "linux"), assert_instr(pclmulhqlqdq, IMM8 = 1))] +#[cfg_attr(all(test, target_os = "linux"), assert_instr(pclmullqhqdq, IMM8 = 16))] +#[cfg_attr(all(test, target_os = "linux"), assert_instr(pclmulhqhqdq, IMM8 = 17))] +#[rustc_legacy_const_generics(2)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm_clmulepi64_si128(a: __m128i, b: __m128i, imm8: i32) -> __m128i { - macro_rules! call { - ($imm8:expr) => { - pclmulqdq(a, b, $imm8) - }; - } - constify_imm8!(imm8, call) +pub unsafe fn _mm_clmulepi64_si128(a: __m128i, b: __m128i) -> __m128i { + static_assert_imm8!(IMM8); + pclmulqdq(a, b, IMM8 as u8) } #[cfg(test)] @@ -62,13 +58,13 @@ mod tests { let r10 = _mm_set_epi64x(0x1a2bf6db3a30862f, 0xbabf262df4b7d5c9); let r11 = _mm_set_epi64x(0x1d1e1f2c592e7c45, 0xd66ee03e410fd4ed); - assert_eq_m128i(_mm_clmulepi64_si128(a, b, 0x00), r00); - assert_eq_m128i(_mm_clmulepi64_si128(a, b, 0x10), r01); - assert_eq_m128i(_mm_clmulepi64_si128(a, b, 0x01), r10); - assert_eq_m128i(_mm_clmulepi64_si128(a, b, 0x11), r11); + assert_eq_m128i(_mm_clmulepi64_si128::<0x00>(a, b), r00); + assert_eq_m128i(_mm_clmulepi64_si128::<0x10>(a, b), r01); + assert_eq_m128i(_mm_clmulepi64_si128::<0x01>(a, b), r10); + assert_eq_m128i(_mm_clmulepi64_si128::<0x11>(a, b), r11); let a0 = _mm_set_epi64x(0x0000000000000000, 0x8000000000000000); let r = _mm_set_epi64x(0x4000000000000000, 0x0000000000000000); - assert_eq_m128i(_mm_clmulepi64_si128(a0, a0, 0x00), r); + assert_eq_m128i(_mm_clmulepi64_si128::<0x00>(a0, a0), r); } } diff --git a/library/stdarch/crates/core_arch/src/x86/rtm.rs b/library/stdarch/crates/core_arch/src/x86/rtm.rs index 7cb1cc09bd..dab73cde9e 100644 --- a/library/stdarch/crates/core_arch/src/x86/rtm.rs +++ b/library/stdarch/crates/core_arch/src/x86/rtm.rs @@ -76,15 +76,11 @@ pub unsafe fn _xend() { /// [Intel's documentation](https://software.intel.com/en-us/cpp-compiler-developer-guide-and-reference-xabort). #[inline] #[target_feature(enable = "rtm")] -#[cfg_attr(test, assert_instr(xabort, imm8 = 0x0))] -#[rustc_args_required_const(0)] -pub unsafe fn _xabort(imm8: u32) { - macro_rules! call { - ($imm8:expr) => { - x86_xabort($imm8) - }; - } - constify_imm8!(imm8, call) +#[cfg_attr(test, assert_instr(xabort, IMM8 = 0x0))] +#[rustc_legacy_const_generics(0)] +pub unsafe fn _xabort() { + static_assert_imm_u8!(IMM8); + x86_xabort(IMM8 as i8) } /// Queries whether the processor is executing in a transactional region identified by restricted @@ -130,14 +126,14 @@ mod tests { unsafe fn test_xabort() { const ABORT_CODE: u32 = 42; // aborting outside a transactional region does nothing - _xabort(ABORT_CODE); + _xabort::(); for _ in 0..10 { let mut x = 0; let code = rtm::_xbegin(); if code == _XBEGIN_STARTED { x += 1; - rtm::_xabort(ABORT_CODE); + rtm::_xabort::(); } else if code & _XABORT_EXPLICIT != 0 { let test_abort_code = rtm::_xabort_code(code); assert_eq!(test_abort_code, ABORT_CODE); diff --git a/library/stdarch/crates/core_arch/src/x86/sha.rs b/library/stdarch/crates/core_arch/src/x86/sha.rs index 362a97ccd3..cfb330cfbb 100644 --- a/library/stdarch/crates/core_arch/src/x86/sha.rs +++ b/library/stdarch/crates/core_arch/src/x86/sha.rs @@ -66,25 +66,18 @@ pub unsafe fn _mm_sha1nexte_epu32(a: __m128i, b: __m128i) -> __m128i { /// Performs four rounds of SHA1 operation using an initial SHA1 state (A,B,C,D) /// from `a` and some pre-computed sum of the next 4 round message values /// (unsigned 32-bit integers), and state variable E from `b`, and return the -/// updated SHA1 state (A,B,C,D). `func` contains the logic functions and round +/// updated SHA1 state (A,B,C,D). `FUNC` contains the logic functions and round /// constants. /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_sha1rnds4_epu32) #[inline] #[target_feature(enable = "sha")] -#[cfg_attr(test, assert_instr(sha1rnds4, func = 0))] -#[rustc_args_required_const(2)] +#[cfg_attr(test, assert_instr(sha1rnds4, FUNC = 0))] +#[rustc_legacy_const_generics(2)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm_sha1rnds4_epu32(a: __m128i, b: __m128i, func: i32) -> __m128i { - let a = a.as_i32x4(); - let b = b.as_i32x4(); - macro_rules! call { - ($imm2:expr) => { - sha1rnds4(a, b, $imm2) - }; - } - let ret = constify_imm2!(func, call); - transmute(ret) +pub unsafe fn _mm_sha1rnds4_epu32(a: __m128i, b: __m128i) -> __m128i { + static_assert_imm2!(FUNC); + transmute(sha1rnds4(a.as_i32x4(), b.as_i32x4(), FUNC as i8)) } /// Performs an intermediate calculation for the next four SHA256 message values @@ -179,19 +172,19 @@ mod tests { let a = _mm_set_epi64x(0xe9b5dba5b5c0fbcf, 0x71374491428a2f98); let b = _mm_set_epi64x(0xab1c5ed5923f82a4, 0x59f111f13956c25b); let expected = _mm_set_epi64x(0x32b13cd8322f5268, 0xc54420862bd9246f); - let r = _mm_sha1rnds4_epu32(a, b, 0); + let r = _mm_sha1rnds4_epu32::<0>(a, b); assert_eq_m128i(r, expected); let expected = _mm_set_epi64x(0x6d4c43e56a3c25d9, 0xa7e00fb775cbd3fe); - let r = _mm_sha1rnds4_epu32(a, b, 1); + let r = _mm_sha1rnds4_epu32::<1>(a, b); assert_eq_m128i(r, expected); let expected = _mm_set_epi64x(0xb304e383c01222f4, 0x66f6b3b1f89d8001); - let r = _mm_sha1rnds4_epu32(a, b, 2); + let r = _mm_sha1rnds4_epu32::<2>(a, b); assert_eq_m128i(r, expected); let expected = _mm_set_epi64x(0x8189b758bfabfa79, 0xdb08f6e78cae098b); - let r = _mm_sha1rnds4_epu32(a, b, 3); + let r = _mm_sha1rnds4_epu32::<3>(a, b); assert_eq_m128i(r, expected); } diff --git a/library/stdarch/crates/core_arch/src/x86/sse.rs b/library/stdarch/crates/core_arch/src/x86/sse.rs index 7835cb461a..a75a31ffe4 100644 --- a/library/stdarch/crates/core_arch/src/x86/sse.rs +++ b/library/stdarch/crates/core_arch/src/x86/sse.rs @@ -148,8 +148,8 @@ pub unsafe fn _mm_rcp_ps(a: __m128) -> __m128 { rcpps(a) } -/// Returns the approximate reciprocal square root of the fist single-precision -/// (32-bit) floating-point elements in `a`, the other elements are unchanged. +/// Returns the approximate reciprocal square root of the first single-precision +/// (32-bit) floating-point element in `a`, the other elements are unchanged. /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_rsqrt_ss) #[inline] @@ -350,7 +350,7 @@ pub unsafe fn _mm_cmple_ss(a: __m128, b: __m128) -> __m128 { #[cfg_attr(test, assert_instr(cmpltss))] #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm_cmpgt_ss(a: __m128, b: __m128) -> __m128 { - simd_shuffle4(a, cmpss(b, a, 1), [4, 1, 2, 3]) + simd_shuffle4!(a, cmpss(b, a, 1), [4, 1, 2, 3]) } /// Compares the lowest `f32` of both inputs for greater than or equal. The @@ -364,7 +364,7 @@ pub unsafe fn _mm_cmpgt_ss(a: __m128, b: __m128) -> __m128 { #[cfg_attr(test, assert_instr(cmpless))] #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm_cmpge_ss(a: __m128, b: __m128) -> __m128 { - simd_shuffle4(a, cmpss(b, a, 2), [4, 1, 2, 3]) + simd_shuffle4!(a, cmpss(b, a, 2), [4, 1, 2, 3]) } /// Compares the lowest `f32` of both inputs for inequality. The lowest 32 bits @@ -420,7 +420,7 @@ pub unsafe fn _mm_cmpnle_ss(a: __m128, b: __m128) -> __m128 { #[cfg_attr(test, assert_instr(cmpnltss))] #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm_cmpngt_ss(a: __m128, b: __m128) -> __m128 { - simd_shuffle4(a, cmpss(b, a, 5), [4, 1, 2, 3]) + simd_shuffle4!(a, cmpss(b, a, 5), [4, 1, 2, 3]) } /// Compares the lowest `f32` of both inputs for not-greater-than-or-equal. The @@ -434,7 +434,7 @@ pub unsafe fn _mm_cmpngt_ss(a: __m128, b: __m128) -> __m128 { #[cfg_attr(test, assert_instr(cmpnless))] #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm_cmpnge_ss(a: __m128, b: __m128) -> __m128 { - simd_shuffle4(a, cmpss(b, a, 6), [4, 1, 2, 3]) + simd_shuffle4!(a, cmpss(b, a, 6), [4, 1, 2, 3]) } /// Checks if the lowest `f32` of both inputs are ordered. The lowest 32 bits of @@ -992,7 +992,7 @@ pub const fn _MM_SHUFFLE(z: u32, y: u32, x: u32, w: u32) -> i32 { } /// Shuffles packed single-precision (32-bit) floating-point elements in `a` and -/// `b` using `mask`. +/// `b` using `MASK`. /// /// The lower half of result takes values from `a` and the higher half from /// `b`. Mask is split to 2 control bits each to index the element from inputs. @@ -1006,53 +1006,21 @@ pub const fn _MM_SHUFFLE(z: u32, y: u32, x: u32, w: u32) -> i32 { /// does not cause a problem in C, however Rust's commitment to strong typing does not allow this. #[inline] #[target_feature(enable = "sse")] -#[cfg_attr(test, assert_instr(shufps, mask = 3))] -#[rustc_args_required_const(2)] -#[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm_shuffle_ps(a: __m128, b: __m128, mask: i32) -> __m128 { - let mask = (mask & 0xFF) as u8; - - macro_rules! shuffle_done { - ($x01:expr, $x23:expr, $x45:expr, $x67:expr) => { - simd_shuffle4(a, b, [$x01, $x23, $x45, $x67]) - }; - } - macro_rules! shuffle_x67 { - ($x01:expr, $x23:expr, $x45:expr) => { - match (mask >> 6) & 0b11 { - 0b00 => shuffle_done!($x01, $x23, $x45, 4), - 0b01 => shuffle_done!($x01, $x23, $x45, 5), - 0b10 => shuffle_done!($x01, $x23, $x45, 6), - _ => shuffle_done!($x01, $x23, $x45, 7), - } - }; - } - macro_rules! shuffle_x45 { - ($x01:expr, $x23:expr) => { - match (mask >> 4) & 0b11 { - 0b00 => shuffle_x67!($x01, $x23, 4), - 0b01 => shuffle_x67!($x01, $x23, 5), - 0b10 => shuffle_x67!($x01, $x23, 6), - _ => shuffle_x67!($x01, $x23, 7), - } - }; - } - macro_rules! shuffle_x23 { - ($x01:expr) => { - match (mask >> 2) & 0b11 { - 0b00 => shuffle_x45!($x01, 0), - 0b01 => shuffle_x45!($x01, 1), - 0b10 => shuffle_x45!($x01, 2), - _ => shuffle_x45!($x01, 3), - } - }; - } - match mask & 0b11 { - 0b00 => shuffle_x23!(0), - 0b01 => shuffle_x23!(1), - 0b10 => shuffle_x23!(2), - _ => shuffle_x23!(3), - } +#[cfg_attr(test, assert_instr(shufps, MASK = 3))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "simd_x86", since = "1.27.0")] +pub unsafe fn _mm_shuffle_ps(a: __m128, b: __m128) -> __m128 { + static_assert_imm8!(MASK); + simd_shuffle4!( + a, + b, + [ + MASK as u32 & 0b11, + (MASK as u32 >> 2) & 0b11, + ((MASK as u32 >> 4) & 0b11) + 4, + ((MASK as u32 >> 6) & 0b11) + 4, + ], + ) } /// Unpacks and interleave single-precision (32-bit) floating-point elements @@ -1064,7 +1032,7 @@ pub unsafe fn _mm_shuffle_ps(a: __m128, b: __m128, mask: i32) -> __m128 { #[cfg_attr(test, assert_instr(unpckhps))] #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm_unpackhi_ps(a: __m128, b: __m128) -> __m128 { - simd_shuffle4(a, b, [2, 6, 3, 7]) + simd_shuffle4!(a, b, [2, 6, 3, 7]) } /// Unpacks and interleave single-precision (32-bit) floating-point elements @@ -1076,7 +1044,7 @@ pub unsafe fn _mm_unpackhi_ps(a: __m128, b: __m128) -> __m128 { #[cfg_attr(test, assert_instr(unpcklps))] #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm_unpacklo_ps(a: __m128, b: __m128) -> __m128 { - simd_shuffle4(a, b, [0, 4, 1, 5]) + simd_shuffle4!(a, b, [0, 4, 1, 5]) } /// Combine higher half of `a` and `b`. The highwe half of `b` occupies the @@ -1089,7 +1057,7 @@ pub unsafe fn _mm_unpacklo_ps(a: __m128, b: __m128) -> __m128 { #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm_movehl_ps(a: __m128, b: __m128) -> __m128 { // TODO; figure why this is a different instruction on Windows? - simd_shuffle4(a, b, [6, 7, 2, 3]) + simd_shuffle4!(a, b, [6, 7, 2, 3]) } /// Combine lower half of `a` and `b`. The lower half of `b` occupies the @@ -1101,7 +1069,7 @@ pub unsafe fn _mm_movehl_ps(a: __m128, b: __m128) -> __m128 { #[cfg_attr(all(test, not(target_os = "windows")), assert_instr(movlhps))] #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm_movelh_ps(a: __m128, b: __m128) -> __m128 { - simd_shuffle4(a, b, [0, 1, 4, 5]) + simd_shuffle4!(a, b, [0, 1, 4, 5]) } /// Returns a mask of the most significant bit of each element in `a`. @@ -1233,7 +1201,7 @@ pub unsafe fn _mm_loadu_ps(p: *const f32) -> __m128 { #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm_loadr_ps(p: *const f32) -> __m128 { let a = _mm_load_ps(p); - simd_shuffle4(a, a, [3, 2, 1, 0]) + simd_shuffle4!(a, a, [3, 2, 1, 0]) } /// Loads unaligned 64-bits of integer data from memory into new vector. @@ -1285,7 +1253,7 @@ pub unsafe fn _mm_store_ss(p: *mut f32, a: __m128) { #[stable(feature = "simd_x86", since = "1.27.0")] #[allow(clippy::cast_ptr_alignment)] pub unsafe fn _mm_store1_ps(p: *mut f32, a: __m128) { - let b: __m128 = simd_shuffle4(a, a, [0, 0, 0, 0]); + let b: __m128 = simd_shuffle4!(a, a, [0, 0, 0, 0]); *(p as *mut __m128) = b; } @@ -1361,7 +1329,7 @@ pub unsafe fn _mm_storeu_ps(p: *mut f32, a: __m128) { #[stable(feature = "simd_x86", since = "1.27.0")] #[allow(clippy::cast_ptr_alignment)] pub unsafe fn _mm_storer_ps(p: *mut f32, a: __m128) { - let b: __m128 = simd_shuffle4(a, a, [3, 2, 1, 0]); + let b: __m128 = simd_shuffle4!(a, a, [3, 2, 1, 0]); *(p as *mut __m128) = b; } @@ -1379,7 +1347,7 @@ pub unsafe fn _mm_storer_ps(p: *mut f32, a: __m128) { #[cfg_attr(test, assert_instr(movss))] #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm_move_ss(a: __m128, b: __m128) -> __m128 { - simd_shuffle4(a, b, [4, 1, 2, 3]) + simd_shuffle4!(a, b, [4, 1, 2, 3]) } /// Performs a serializing operation on all store-to-memory instructions that @@ -1725,9 +1693,17 @@ pub const _MM_HINT_T2: i32 = 1; #[stable(feature = "simd_x86", since = "1.27.0")] pub const _MM_HINT_NTA: i32 = 0; -/// Fetch the cache line that contains address `p` using the given `strategy`. +/// See [`_mm_prefetch`](fn._mm_prefetch.html). +#[stable(feature = "simd_x86", since = "1.27.0")] +pub const _MM_HINT_ET0: i32 = 7; + +/// See [`_mm_prefetch`](fn._mm_prefetch.html). +#[stable(feature = "simd_x86", since = "1.27.0")] +pub const _MM_HINT_ET1: i32 = 6; + +/// Fetch the cache line that contains address `p` using the given `STRATEGY`. /// -/// The `strategy` must be one of: +/// The `STRATEGY` must be one of: /// /// * [`_MM_HINT_T0`](constant._MM_HINT_T0.html): Fetch into all levels of the /// cache hierarchy. @@ -1742,6 +1718,10 @@ pub const _MM_HINT_NTA: i32 = 0; /// but outside of the cache hierarchy. This is used to reduce access latency /// without polluting the cache. /// +/// * [`_MM_HINT_ET0`](constant._MM_HINT_ET0.html) and +/// [`_MM_HINT_ET1`](constant._MM_HINT_ET1.html) are similar to `_MM_HINT_T0` +/// and `_MM_HINT_T1` but indicate an anticipation to write to the address. +/// /// The actual implementation depends on the particular CPU. This instruction /// is considered a hint, so the CPU is also free to simply ignore the request. /// @@ -1765,28 +1745,16 @@ pub const _MM_HINT_NTA: i32 = 0; /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_prefetch) #[inline] #[target_feature(enable = "sse")] -#[cfg_attr(test, assert_instr(prefetcht0, strategy = _MM_HINT_T0))] -#[cfg_attr(test, assert_instr(prefetcht1, strategy = _MM_HINT_T1))] -#[cfg_attr(test, assert_instr(prefetcht2, strategy = _MM_HINT_T2))] -#[cfg_attr(test, assert_instr(prefetchnta, strategy = _MM_HINT_NTA))] -#[rustc_args_required_const(1)] +#[cfg_attr(test, assert_instr(prefetcht0, STRATEGY = _MM_HINT_T0))] +#[cfg_attr(test, assert_instr(prefetcht1, STRATEGY = _MM_HINT_T1))] +#[cfg_attr(test, assert_instr(prefetcht2, STRATEGY = _MM_HINT_T2))] +#[cfg_attr(test, assert_instr(prefetchnta, STRATEGY = _MM_HINT_NTA))] +#[rustc_legacy_const_generics(1)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm_prefetch(p: *const i8, strategy: i32) { - // The `strategy` must be a compile-time constant, so we use a short form - // of `constify_imm8!` for now. - // We use the `llvm.prefetch` instrinsic with `rw` = 0 (read), and - // `cache type` = 1 (data cache). `locality` is based on our `strategy`. - macro_rules! pref { - ($imm8:expr) => { - match $imm8 { - 0 => prefetch(p, 0, 0, 1), - 1 => prefetch(p, 0, 1, 1), - 2 => prefetch(p, 0, 2, 1), - _ => prefetch(p, 0, 3, 1), - } - }; - } - pref!(strategy) +pub unsafe fn _mm_prefetch(p: *const i8) { + // We use the `llvm.prefetch` instrinsic with `cache type` = 1 (data cache). + // `locality` and `rw` are based on our `STRATEGY`. + prefetch(p, (STRATEGY >> 2) & 1, STRATEGY & 3, 1); } /// Returns vector of type __m128 with undefined elements. @@ -2976,7 +2944,7 @@ mod tests { unsafe fn test_mm_shuffle_ps() { let a = _mm_setr_ps(1.0, 2.0, 3.0, 4.0); let b = _mm_setr_ps(5.0, 6.0, 7.0, 8.0); - let r = _mm_shuffle_ps(a, b, 0b00_01_01_11); + let r = _mm_shuffle_ps::<0b00_01_01_11>(a, b); assert_eq_m128(r, _mm_setr_ps(4.0, 2.0, 6.0, 5.0)); } diff --git a/library/stdarch/crates/core_arch/src/x86/sse2.rs b/library/stdarch/crates/core_arch/src/x86/sse2.rs index 12e7506f02..307dae77d2 100644 --- a/library/stdarch/crates/core_arch/src/x86/sse2.rs +++ b/library/stdarch/crates/core_arch/src/x86/sse2.rs @@ -406,113 +406,94 @@ pub unsafe fn _mm_subs_epu16(a: __m128i, b: __m128i) -> __m128i { transmute(simd_saturating_sub(a.as_u16x8(), b.as_u16x8())) } -/// Shifts `a` left by `imm8` bytes while shifting in zeros. +/// Shifts `a` left by `IMM8` bytes while shifting in zeros. /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_slli_si128) #[inline] #[target_feature(enable = "sse2")] -#[cfg_attr(test, assert_instr(pslldq, imm8 = 1))] -#[rustc_args_required_const(1)] +#[cfg_attr(test, assert_instr(pslldq, IMM8 = 1))] +#[rustc_legacy_const_generics(1)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm_slli_si128(a: __m128i, imm8: i32) -> __m128i { - _mm_slli_si128_impl(a, imm8) +pub unsafe fn _mm_slli_si128(a: __m128i) -> __m128i { + static_assert_imm8!(IMM8); + _mm_slli_si128_impl::(a) } /// Implementation detail: converts the immediate argument of the /// `_mm_slli_si128` intrinsic into a compile-time constant. #[inline] #[target_feature(enable = "sse2")] -unsafe fn _mm_slli_si128_impl(a: __m128i, imm8: i32) -> __m128i { - let (zero, imm8) = (_mm_set1_epi8(0).as_i8x16(), imm8 as u32); - let a = a.as_i8x16(); - macro_rules! shuffle { - ($shift:expr) => { - simd_shuffle16::( - zero, - a, - [ - 16 - $shift, - 17 - $shift, - 18 - $shift, - 19 - $shift, - 20 - $shift, - 21 - $shift, - 22 - $shift, - 23 - $shift, - 24 - $shift, - 25 - $shift, - 26 - $shift, - 27 - $shift, - 28 - $shift, - 29 - $shift, - 30 - $shift, - 31 - $shift, - ], - ) - }; +unsafe fn _mm_slli_si128_impl(a: __m128i) -> __m128i { + const fn mask(shift: i32, i: u32) -> u32 { + let shift = shift as u32 & 0xff; + if shift > 15 { + i + } else { + 16 - shift + i + } } - let x = match imm8 { - 0 => shuffle!(0), - 1 => shuffle!(1), - 2 => shuffle!(2), - 3 => shuffle!(3), - 4 => shuffle!(4), - 5 => shuffle!(5), - 6 => shuffle!(6), - 7 => shuffle!(7), - 8 => shuffle!(8), - 9 => shuffle!(9), - 10 => shuffle!(10), - 11 => shuffle!(11), - 12 => shuffle!(12), - 13 => shuffle!(13), - 14 => shuffle!(14), - 15 => shuffle!(15), - _ => shuffle!(16), - }; - transmute(x) + let zero = _mm_set1_epi8(0).as_i8x16(); + transmute::(simd_shuffle16!( + zero, + a.as_i8x16(), + [ + mask(IMM8, 0), + mask(IMM8, 1), + mask(IMM8, 2), + mask(IMM8, 3), + mask(IMM8, 4), + mask(IMM8, 5), + mask(IMM8, 6), + mask(IMM8, 7), + mask(IMM8, 8), + mask(IMM8, 9), + mask(IMM8, 10), + mask(IMM8, 11), + mask(IMM8, 12), + mask(IMM8, 13), + mask(IMM8, 14), + mask(IMM8, 15), + ], + )) } -/// Shifts `a` left by `imm8` bytes while shifting in zeros. +/// Shifts `a` left by `IMM8` bytes while shifting in zeros. /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_bslli_si128) #[inline] #[target_feature(enable = "sse2")] -#[cfg_attr(test, assert_instr(pslldq, imm8 = 1))] -#[rustc_args_required_const(1)] +#[cfg_attr(test, assert_instr(pslldq, IMM8 = 1))] +#[rustc_legacy_const_generics(1)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm_bslli_si128(a: __m128i, imm8: i32) -> __m128i { - _mm_slli_si128_impl(a, imm8) +pub unsafe fn _mm_bslli_si128(a: __m128i) -> __m128i { + static_assert_imm8!(IMM8); + _mm_slli_si128_impl::(a) } -/// Shifts `a` right by `imm8` bytes while shifting in zeros. +/// Shifts `a` right by `IMM8` bytes while shifting in zeros. /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_bsrli_si128) #[inline] #[target_feature(enable = "sse2")] -#[cfg_attr(test, assert_instr(psrldq, imm8 = 1))] -#[rustc_args_required_const(1)] +#[cfg_attr(test, assert_instr(psrldq, IMM8 = 1))] +#[rustc_legacy_const_generics(1)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm_bsrli_si128(a: __m128i, imm8: i32) -> __m128i { - _mm_srli_si128_impl(a, imm8) +pub unsafe fn _mm_bsrli_si128(a: __m128i) -> __m128i { + static_assert_imm8!(IMM8); + _mm_srli_si128_impl::(a) } -/// Shifts packed 16-bit integers in `a` left by `imm8` while shifting in zeros. +/// Shifts packed 16-bit integers in `a` left by `IMM8` while shifting in zeros. /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_slli_epi16) #[inline] #[target_feature(enable = "sse2")] -#[cfg_attr(test, assert_instr(psllw, imm8 = 7))] -#[rustc_args_required_const(1)] +#[cfg_attr(test, assert_instr(psllw, IMM8 = 7))] +#[rustc_legacy_const_generics(1)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm_slli_epi16(a: __m128i, imm8: i32) -> __m128i { - let a = a.as_i16x8(); - macro_rules! call { - ($imm8:expr) => { - transmute(pslliw(a, $imm8)) - }; - } - constify_imm8!(imm8, call) +pub unsafe fn _mm_slli_epi16(a: __m128i) -> __m128i { + static_assert_imm8!(IMM8); + transmute(pslliw(a.as_i16x8(), IMM8)) } /// Shifts packed 16-bit integers in `a` left by `count` while shifting in @@ -527,22 +508,17 @@ pub unsafe fn _mm_sll_epi16(a: __m128i, count: __m128i) -> __m128i { transmute(psllw(a.as_i16x8(), count.as_i16x8())) } -/// Shifts packed 32-bit integers in `a` left by `imm8` while shifting in zeros. +/// Shifts packed 32-bit integers in `a` left by `IMM8` while shifting in zeros. /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_slli_epi32) #[inline] #[target_feature(enable = "sse2")] -#[cfg_attr(test, assert_instr(pslld, imm8 = 7))] -#[rustc_args_required_const(1)] +#[cfg_attr(test, assert_instr(pslld, IMM8 = 7))] +#[rustc_legacy_const_generics(1)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm_slli_epi32(a: __m128i, imm8: i32) -> __m128i { - let a = a.as_i32x4(); - macro_rules! call { - ($imm8:expr) => { - transmute(psllid(a, $imm8)) - }; - } - constify_imm8!(imm8, call) +pub unsafe fn _mm_slli_epi32(a: __m128i) -> __m128i { + static_assert_imm8!(IMM8); + transmute(psllid(a.as_i32x4(), IMM8)) } /// Shifts packed 32-bit integers in `a` left by `count` while shifting in @@ -557,22 +533,17 @@ pub unsafe fn _mm_sll_epi32(a: __m128i, count: __m128i) -> __m128i { transmute(pslld(a.as_i32x4(), count.as_i32x4())) } -/// Shifts packed 64-bit integers in `a` left by `imm8` while shifting in zeros. +/// Shifts packed 64-bit integers in `a` left by `IMM8` while shifting in zeros. /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_slli_epi64) #[inline] #[target_feature(enable = "sse2")] -#[cfg_attr(test, assert_instr(psllq, imm8 = 7))] -#[rustc_args_required_const(1)] +#[cfg_attr(test, assert_instr(psllq, IMM8 = 7))] +#[rustc_legacy_const_generics(1)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm_slli_epi64(a: __m128i, imm8: i32) -> __m128i { - let a = a.as_i64x2(); - macro_rules! call { - ($imm8:expr) => { - transmute(pslliq(a, $imm8)) - }; - } - constify_imm8!(imm8, call) +pub unsafe fn _mm_slli_epi64(a: __m128i) -> __m128i { + static_assert_imm8!(IMM8); + transmute(pslliq(a.as_i64x2(), IMM8)) } /// Shifts packed 64-bit integers in `a` left by `count` while shifting in @@ -587,23 +558,18 @@ pub unsafe fn _mm_sll_epi64(a: __m128i, count: __m128i) -> __m128i { transmute(psllq(a.as_i64x2(), count.as_i64x2())) } -/// Shifts packed 16-bit integers in `a` right by `imm8` while shifting in sign +/// Shifts packed 16-bit integers in `a` right by `IMM8` while shifting in sign /// bits. /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_srai_epi16) #[inline] #[target_feature(enable = "sse2")] -#[cfg_attr(test, assert_instr(psraw, imm8 = 1))] -#[rustc_args_required_const(1)] +#[cfg_attr(test, assert_instr(psraw, IMM8 = 1))] +#[rustc_legacy_const_generics(1)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm_srai_epi16(a: __m128i, imm8: i32) -> __m128i { - let a = a.as_i16x8(); - macro_rules! call { - ($imm8:expr) => { - transmute(psraiw(a, $imm8)) - }; - } - constify_imm8!(imm8, call) +pub unsafe fn _mm_srai_epi16(a: __m128i) -> __m128i { + static_assert_imm8!(IMM8); + transmute(psraiw(a.as_i16x8(), IMM8)) } /// Shifts packed 16-bit integers in `a` right by `count` while shifting in sign @@ -618,23 +584,18 @@ pub unsafe fn _mm_sra_epi16(a: __m128i, count: __m128i) -> __m128i { transmute(psraw(a.as_i16x8(), count.as_i16x8())) } -/// Shifts packed 32-bit integers in `a` right by `imm8` while shifting in sign +/// Shifts packed 32-bit integers in `a` right by `IMM8` while shifting in sign /// bits. /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_srai_epi32) #[inline] #[target_feature(enable = "sse2")] -#[cfg_attr(test, assert_instr(psrad, imm8 = 1))] -#[rustc_args_required_const(1)] +#[cfg_attr(test, assert_instr(psrad, IMM8 = 1))] +#[rustc_legacy_const_generics(1)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm_srai_epi32(a: __m128i, imm8: i32) -> __m128i { - let a = a.as_i32x4(); - macro_rules! call { - ($imm8:expr) => { - transmute(psraid(a, $imm8)) - }; - } - constify_imm8!(imm8, call) +pub unsafe fn _mm_srai_epi32(a: __m128i) -> __m128i { + static_assert_imm8!(IMM8); + transmute(psraid(a.as_i32x4(), IMM8)) } /// Shifts packed 32-bit integers in `a` right by `count` while shifting in sign @@ -649,90 +610,69 @@ pub unsafe fn _mm_sra_epi32(a: __m128i, count: __m128i) -> __m128i { transmute(psrad(a.as_i32x4(), count.as_i32x4())) } -/// Shifts `a` right by `imm8` bytes while shifting in zeros. +/// Shifts `a` right by `IMM8` bytes while shifting in zeros. /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_srli_si128) #[inline] #[target_feature(enable = "sse2")] -#[cfg_attr(test, assert_instr(psrldq, imm8 = 1))] -#[rustc_args_required_const(1)] +#[cfg_attr(test, assert_instr(psrldq, IMM8 = 1))] +#[rustc_legacy_const_generics(1)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm_srli_si128(a: __m128i, imm8: i32) -> __m128i { - _mm_srli_si128_impl(a, imm8) +pub unsafe fn _mm_srli_si128(a: __m128i) -> __m128i { + static_assert_imm8!(IMM8); + _mm_srli_si128_impl::(a) } /// Implementation detail: converts the immediate argument of the /// `_mm_srli_si128` intrinsic into a compile-time constant. #[inline] #[target_feature(enable = "sse2")] -unsafe fn _mm_srli_si128_impl(a: __m128i, imm8: i32) -> __m128i { - let (zero, imm8) = (_mm_set1_epi8(0).as_i8x16(), imm8 as u32); - let a = a.as_i8x16(); - macro_rules! shuffle { - ($shift:expr) => { - simd_shuffle16( - a, - zero, - [ - 0 + $shift, - 1 + $shift, - 2 + $shift, - 3 + $shift, - 4 + $shift, - 5 + $shift, - 6 + $shift, - 7 + $shift, - 8 + $shift, - 9 + $shift, - 10 + $shift, - 11 + $shift, - 12 + $shift, - 13 + $shift, - 14 + $shift, - 15 + $shift, - ], - ) - }; +unsafe fn _mm_srli_si128_impl(a: __m128i) -> __m128i { + const fn mask(shift: i32, i: u32) -> u32 { + if (shift as u32) > 15 { + i + 16 + } else { + i + (shift as u32) + } } - let x: i8x16 = match imm8 { - 0 => shuffle!(0), - 1 => shuffle!(1), - 2 => shuffle!(2), - 3 => shuffle!(3), - 4 => shuffle!(4), - 5 => shuffle!(5), - 6 => shuffle!(6), - 7 => shuffle!(7), - 8 => shuffle!(8), - 9 => shuffle!(9), - 10 => shuffle!(10), - 11 => shuffle!(11), - 12 => shuffle!(12), - 13 => shuffle!(13), - 14 => shuffle!(14), - 15 => shuffle!(15), - _ => shuffle!(16), - }; + let zero = _mm_set1_epi8(0).as_i8x16(); + let x: i8x16 = simd_shuffle16!( + a.as_i8x16(), + zero, + [ + mask(IMM8, 0), + mask(IMM8, 1), + mask(IMM8, 2), + mask(IMM8, 3), + mask(IMM8, 4), + mask(IMM8, 5), + mask(IMM8, 6), + mask(IMM8, 7), + mask(IMM8, 8), + mask(IMM8, 9), + mask(IMM8, 10), + mask(IMM8, 11), + mask(IMM8, 12), + mask(IMM8, 13), + mask(IMM8, 14), + mask(IMM8, 15), + ], + ); transmute(x) } -/// Shifts packed 16-bit integers in `a` right by `imm8` while shifting in +/// Shifts packed 16-bit integers in `a` right by `IMM8` while shifting in /// zeros. /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_srli_epi16) #[inline] #[target_feature(enable = "sse2")] -#[cfg_attr(test, assert_instr(psrlw, imm8 = 1))] -#[rustc_args_required_const(1)] +#[cfg_attr(test, assert_instr(psrlw, IMM8 = 1))] +#[rustc_legacy_const_generics(1)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm_srli_epi16(a: __m128i, imm8: i32) -> __m128i { - let a = a.as_i16x8(); - macro_rules! call { - ($imm8:expr) => { - transmute(psrliw(a, $imm8)) - }; - } - constify_imm8!(imm8, call) +pub unsafe fn _mm_srli_epi16(a: __m128i) -> __m128i { + static_assert_imm8!(IMM8); + transmute(psrliw(a.as_i16x8(), IMM8)) } /// Shifts packed 16-bit integers in `a` right by `count` while shifting in @@ -747,23 +687,18 @@ pub unsafe fn _mm_srl_epi16(a: __m128i, count: __m128i) -> __m128i { transmute(psrlw(a.as_i16x8(), count.as_i16x8())) } -/// Shifts packed 32-bit integers in `a` right by `imm8` while shifting in +/// Shifts packed 32-bit integers in `a` right by `IMM8` while shifting in /// zeros. /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_srli_epi32) #[inline] #[target_feature(enable = "sse2")] -#[cfg_attr(test, assert_instr(psrld, imm8 = 8))] -#[rustc_args_required_const(1)] +#[cfg_attr(test, assert_instr(psrld, IMM8 = 8))] +#[rustc_legacy_const_generics(1)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm_srli_epi32(a: __m128i, imm8: i32) -> __m128i { - let a = a.as_i32x4(); - macro_rules! call { - ($imm8:expr) => { - transmute(psrlid(a, $imm8)) - }; - } - constify_imm8!(imm8, call) +pub unsafe fn _mm_srli_epi32(a: __m128i) -> __m128i { + static_assert_imm8!(IMM8); + transmute(psrlid(a.as_i32x4(), IMM8)) } /// Shifts packed 32-bit integers in `a` right by `count` while shifting in @@ -778,17 +713,18 @@ pub unsafe fn _mm_srl_epi32(a: __m128i, count: __m128i) -> __m128i { transmute(psrld(a.as_i32x4(), count.as_i32x4())) } -/// Shifts packed 64-bit integers in `a` right by `imm8` while shifting in +/// Shifts packed 64-bit integers in `a` right by `IMM8` while shifting in /// zeros. /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_srli_epi64) #[inline] #[target_feature(enable = "sse2")] -#[cfg_attr(test, assert_instr(psrlq, imm8 = 1))] -#[rustc_args_required_const(1)] +#[cfg_attr(test, assert_instr(psrlq, IMM8 = 1))] +#[rustc_legacy_const_generics(1)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm_srli_epi64(a: __m128i, imm8: i32) -> __m128i { - transmute(psrliq(a.as_i64x2(), imm8)) +pub unsafe fn _mm_srli_epi64(a: __m128i) -> __m128i { + static_assert_imm8!(IMM8); + transmute(psrliq(a.as_i64x2(), IMM8)) } /// Shifts packed 64-bit integers in `a` right by `count` while shifting in @@ -960,7 +896,7 @@ pub unsafe fn _mm_cmplt_epi32(a: __m128i, b: __m128i) -> __m128i { #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm_cvtepi32_pd(a: __m128i) -> __m128d { let a = a.as_i32x4(); - simd_cast::(simd_shuffle2(a, a, [0, 1])) + simd_cast::(simd_shuffle2!(a, a, [0, 1])) } /// Returns `a` with its lower element replaced by `b` after converting it to @@ -1368,7 +1304,7 @@ pub unsafe fn _mm_stream_si32(mem_addr: *mut i32, a: i32) { #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm_move_epi64(a: __m128i) -> __m128i { let zero = _mm_setzero_si128(); - let r: i64x2 = simd_shuffle2(a.as_i64x2(), zero.as_i64x2(), [0, 2]); + let r: i64x2 = simd_shuffle2!(a.as_i64x2(), zero.as_i64x2(), [0, 2]); transmute(r) } @@ -1413,17 +1349,12 @@ pub unsafe fn _mm_packus_epi16(a: __m128i, b: __m128i) -> __m128i { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_extract_epi16) #[inline] #[target_feature(enable = "sse2")] -#[cfg_attr(test, assert_instr(pextrw, imm8 = 9))] -#[rustc_args_required_const(1)] +#[cfg_attr(test, assert_instr(pextrw, IMM8 = 7))] +#[rustc_legacy_const_generics(1)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm_extract_epi16(a: __m128i, imm8: i32) -> i32 { - let a = a.as_u16x8(); - macro_rules! call { - ($imm3:expr) => { - simd_extract::<_, u16>(a, $imm3) as i32 - }; - } - constify_imm3!(imm8, call) +pub unsafe fn _mm_extract_epi16(a: __m128i) -> i32 { + static_assert_imm3!(IMM8); + simd_extract::<_, u16>(a.as_u16x8(), IMM8 as u32) as i32 } /// Returns a new vector where the `imm8` element of `a` is replaced with `i`. @@ -1431,17 +1362,12 @@ pub unsafe fn _mm_extract_epi16(a: __m128i, imm8: i32) -> i32 { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_insert_epi16) #[inline] #[target_feature(enable = "sse2")] -#[cfg_attr(test, assert_instr(pinsrw, imm8 = 9))] -#[rustc_args_required_const(2)] +#[cfg_attr(test, assert_instr(pinsrw, IMM8 = 7))] +#[rustc_legacy_const_generics(2)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm_insert_epi16(a: __m128i, i: i32, imm8: i32) -> __m128i { - let a = a.as_i16x8(); - macro_rules! call { - ($imm3:expr) => { - transmute(simd_insert(a, $imm3, i as i16)) - }; - } - constify_imm3!(imm8, call) +pub unsafe fn _mm_insert_epi16(a: __m128i, i: i32) -> __m128i { + static_assert_imm3!(IMM8); + transmute(simd_insert(a.as_i16x8(), IMM8 as u32, i as i16)) } /// Returns a mask of the most significant bit of each element in `a`. @@ -1455,71 +1381,32 @@ pub unsafe fn _mm_movemask_epi8(a: __m128i) -> i32 { pmovmskb(a.as_i8x16()) } -/// Shuffles 32-bit integers in `a` using the control in `imm8`. +/// Shuffles 32-bit integers in `a` using the control in `IMM8`. /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_shuffle_epi32) #[inline] #[target_feature(enable = "sse2")] -#[cfg_attr(test, assert_instr(pshufd, imm8 = 9))] -#[rustc_args_required_const(1)] +#[cfg_attr(test, assert_instr(pshufd, IMM8 = 9))] +#[rustc_legacy_const_generics(1)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm_shuffle_epi32(a: __m128i, imm8: i32) -> __m128i { - // simd_shuffleX requires that its selector parameter be made up of - // constant values, but we can't enforce that here. In spirit, we need - // to write a `match` on all possible values of a byte, and for each value, - // hard-code the correct `simd_shuffleX` call using only constants. We - // then hope for LLVM to do the rest. - // - // Of course, that's... awful. So we try to use macros to do it for us. - let imm8 = (imm8 & 0xFF) as u8; +pub unsafe fn _mm_shuffle_epi32(a: __m128i) -> __m128i { + static_assert_imm8!(IMM8); let a = a.as_i32x4(); - - macro_rules! shuffle_done { - ($x01:expr, $x23:expr, $x45:expr, $x67:expr) => { - simd_shuffle4(a, a, [$x01, $x23, $x45, $x67]) - }; - } - macro_rules! shuffle_x67 { - ($x01:expr, $x23:expr, $x45:expr) => { - match (imm8 >> 6) & 0b11 { - 0b00 => shuffle_done!($x01, $x23, $x45, 0), - 0b01 => shuffle_done!($x01, $x23, $x45, 1), - 0b10 => shuffle_done!($x01, $x23, $x45, 2), - _ => shuffle_done!($x01, $x23, $x45, 3), - } - }; - } - macro_rules! shuffle_x45 { - ($x01:expr, $x23:expr) => { - match (imm8 >> 4) & 0b11 { - 0b00 => shuffle_x67!($x01, $x23, 0), - 0b01 => shuffle_x67!($x01, $x23, 1), - 0b10 => shuffle_x67!($x01, $x23, 2), - _ => shuffle_x67!($x01, $x23, 3), - } - }; - } - macro_rules! shuffle_x23 { - ($x01:expr) => { - match (imm8 >> 2) & 0b11 { - 0b00 => shuffle_x45!($x01, 0), - 0b01 => shuffle_x45!($x01, 1), - 0b10 => shuffle_x45!($x01, 2), - _ => shuffle_x45!($x01, 3), - } - }; - } - let x: i32x4 = match imm8 & 0b11 { - 0b00 => shuffle_x23!(0), - 0b01 => shuffle_x23!(1), - 0b10 => shuffle_x23!(2), - _ => shuffle_x23!(3), - }; + let x: i32x4 = simd_shuffle4!( + a, + a, + [ + IMM8 as u32 & 0b11, + (IMM8 as u32 >> 2) & 0b11, + (IMM8 as u32 >> 4) & 0b11, + (IMM8 as u32 >> 6) & 0b11, + ], + ); transmute(x) } /// Shuffles 16-bit integers in the high 64 bits of `a` using the control in -/// `imm8`. +/// `IMM8`. /// /// Put the results in the high 64 bits of the returned vector, with the low 64 /// bits being copied from from `a`. @@ -1527,59 +1414,31 @@ pub unsafe fn _mm_shuffle_epi32(a: __m128i, imm8: i32) -> __m128i { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_shufflehi_epi16) #[inline] #[target_feature(enable = "sse2")] -#[cfg_attr(test, assert_instr(pshufhw, imm8 = 9))] -#[rustc_args_required_const(1)] +#[cfg_attr(test, assert_instr(pshufhw, IMM8 = 9))] +#[rustc_legacy_const_generics(1)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm_shufflehi_epi16(a: __m128i, imm8: i32) -> __m128i { - // See _mm_shuffle_epi32. - let imm8 = (imm8 & 0xFF) as u8; +pub unsafe fn _mm_shufflehi_epi16(a: __m128i) -> __m128i { + static_assert_imm8!(IMM8); let a = a.as_i16x8(); - macro_rules! shuffle_done { - ($x01:expr, $x23:expr, $x45:expr, $x67:expr) => { - simd_shuffle8(a, a, [0, 1, 2, 3, $x01 + 4, $x23 + 4, $x45 + 4, $x67 + 4]) - }; - } - macro_rules! shuffle_x67 { - ($x01:expr, $x23:expr, $x45:expr) => { - match (imm8 >> 6) & 0b11 { - 0b00 => shuffle_done!($x01, $x23, $x45, 0), - 0b01 => shuffle_done!($x01, $x23, $x45, 1), - 0b10 => shuffle_done!($x01, $x23, $x45, 2), - _ => shuffle_done!($x01, $x23, $x45, 3), - } - }; - } - macro_rules! shuffle_x45 { - ($x01:expr, $x23:expr) => { - match (imm8 >> 4) & 0b11 { - 0b00 => shuffle_x67!($x01, $x23, 0), - 0b01 => shuffle_x67!($x01, $x23, 1), - 0b10 => shuffle_x67!($x01, $x23, 2), - _ => shuffle_x67!($x01, $x23, 3), - } - }; - } - macro_rules! shuffle_x23 { - ($x01:expr) => { - match (imm8 >> 2) & 0b11 { - 0b00 => shuffle_x45!($x01, 0), - 0b01 => shuffle_x45!($x01, 1), - 0b10 => shuffle_x45!($x01, 2), - _ => shuffle_x45!($x01, 3), - } - }; - } - let x: i16x8 = match imm8 & 0b11 { - 0b00 => shuffle_x23!(0), - 0b01 => shuffle_x23!(1), - 0b10 => shuffle_x23!(2), - _ => shuffle_x23!(3), - }; + let x: i16x8 = simd_shuffle8!( + a, + a, + [ + 0, + 1, + 2, + 3, + (IMM8 as u32 & 0b11) + 4, + ((IMM8 as u32 >> 2) & 0b11) + 4, + ((IMM8 as u32 >> 4) & 0b11) + 4, + ((IMM8 as u32 >> 6) & 0b11) + 4, + ], + ); transmute(x) } /// Shuffles 16-bit integers in the low 64 bits of `a` using the control in -/// `imm8`. +/// `IMM8`. /// /// Put the results in the low 64 bits of the returned vector, with the high 64 /// bits being copied from from `a`. @@ -1587,55 +1446,26 @@ pub unsafe fn _mm_shufflehi_epi16(a: __m128i, imm8: i32) -> __m128i { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_shufflelo_epi16) #[inline] #[target_feature(enable = "sse2")] -#[cfg_attr(test, assert_instr(pshuflw, imm8 = 9))] -#[rustc_args_required_const(1)] +#[cfg_attr(test, assert_instr(pshuflw, IMM8 = 9))] +#[rustc_legacy_const_generics(1)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm_shufflelo_epi16(a: __m128i, imm8: i32) -> __m128i { - // See _mm_shuffle_epi32. - let imm8 = (imm8 & 0xFF) as u8; +pub unsafe fn _mm_shufflelo_epi16(a: __m128i) -> __m128i { + static_assert_imm8!(IMM8); let a = a.as_i16x8(); - - macro_rules! shuffle_done { - ($x01:expr, $x23:expr, $x45:expr, $x67:expr) => { - simd_shuffle8(a, a, [$x01, $x23, $x45, $x67, 4, 5, 6, 7]) - }; - } - macro_rules! shuffle_x67 { - ($x01:expr, $x23:expr, $x45:expr) => { - match (imm8 >> 6) & 0b11 { - 0b00 => shuffle_done!($x01, $x23, $x45, 0), - 0b01 => shuffle_done!($x01, $x23, $x45, 1), - 0b10 => shuffle_done!($x01, $x23, $x45, 2), - _ => shuffle_done!($x01, $x23, $x45, 3), - } - }; - } - macro_rules! shuffle_x45 { - ($x01:expr, $x23:expr) => { - match (imm8 >> 4) & 0b11 { - 0b00 => shuffle_x67!($x01, $x23, 0), - 0b01 => shuffle_x67!($x01, $x23, 1), - 0b10 => shuffle_x67!($x01, $x23, 2), - _ => shuffle_x67!($x01, $x23, 3), - } - }; - } - macro_rules! shuffle_x23 { - ($x01:expr) => { - match (imm8 >> 2) & 0b11 { - 0b00 => shuffle_x45!($x01, 0), - 0b01 => shuffle_x45!($x01, 1), - 0b10 => shuffle_x45!($x01, 2), - _ => shuffle_x45!($x01, 3), - } - }; - } - let x: i16x8 = match imm8 & 0b11 { - 0b00 => shuffle_x23!(0), - 0b01 => shuffle_x23!(1), - 0b10 => shuffle_x23!(2), - _ => shuffle_x23!(3), - }; + let x: i16x8 = simd_shuffle8!( + a, + a, + [ + IMM8 as u32 & 0b11, + (IMM8 as u32 >> 2) & 0b11, + (IMM8 as u32 >> 4) & 0b11, + (IMM8 as u32 >> 6) & 0b11, + 4, + 5, + 6, + 7, + ], + ); transmute(x) } @@ -1647,7 +1477,7 @@ pub unsafe fn _mm_shufflelo_epi16(a: __m128i, imm8: i32) -> __m128i { #[cfg_attr(test, assert_instr(punpckhbw))] #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm_unpackhi_epi8(a: __m128i, b: __m128i) -> __m128i { - transmute::(simd_shuffle16( + transmute::(simd_shuffle16!( a.as_i8x16(), b.as_i8x16(), [8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31], @@ -1662,7 +1492,7 @@ pub unsafe fn _mm_unpackhi_epi8(a: __m128i, b: __m128i) -> __m128i { #[cfg_attr(test, assert_instr(punpckhwd))] #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm_unpackhi_epi16(a: __m128i, b: __m128i) -> __m128i { - let x = simd_shuffle8(a.as_i16x8(), b.as_i16x8(), [4, 12, 5, 13, 6, 14, 7, 15]); + let x = simd_shuffle8!(a.as_i16x8(), b.as_i16x8(), [4, 12, 5, 13, 6, 14, 7, 15]); transmute::(x) } @@ -1674,7 +1504,7 @@ pub unsafe fn _mm_unpackhi_epi16(a: __m128i, b: __m128i) -> __m128i { #[cfg_attr(test, assert_instr(unpckhps))] #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm_unpackhi_epi32(a: __m128i, b: __m128i) -> __m128i { - transmute::(simd_shuffle4(a.as_i32x4(), b.as_i32x4(), [2, 6, 3, 7])) + transmute::(simd_shuffle4!(a.as_i32x4(), b.as_i32x4(), [2, 6, 3, 7])) } /// Unpacks and interleave 64-bit integers from the high half of `a` and `b`. @@ -1685,7 +1515,7 @@ pub unsafe fn _mm_unpackhi_epi32(a: __m128i, b: __m128i) -> __m128i { #[cfg_attr(test, assert_instr(unpckhpd))] #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm_unpackhi_epi64(a: __m128i, b: __m128i) -> __m128i { - transmute::(simd_shuffle2(a.as_i64x2(), b.as_i64x2(), [1, 3])) + transmute::(simd_shuffle2!(a.as_i64x2(), b.as_i64x2(), [1, 3])) } /// Unpacks and interleave 8-bit integers from the low half of `a` and `b`. @@ -1696,7 +1526,7 @@ pub unsafe fn _mm_unpackhi_epi64(a: __m128i, b: __m128i) -> __m128i { #[cfg_attr(test, assert_instr(punpcklbw))] #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm_unpacklo_epi8(a: __m128i, b: __m128i) -> __m128i { - transmute::(simd_shuffle16( + transmute::(simd_shuffle16!( a.as_i8x16(), b.as_i8x16(), [0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23], @@ -1711,7 +1541,7 @@ pub unsafe fn _mm_unpacklo_epi8(a: __m128i, b: __m128i) -> __m128i { #[cfg_attr(test, assert_instr(punpcklwd))] #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm_unpacklo_epi16(a: __m128i, b: __m128i) -> __m128i { - let x = simd_shuffle8(a.as_i16x8(), b.as_i16x8(), [0, 8, 1, 9, 2, 10, 3, 11]); + let x = simd_shuffle8!(a.as_i16x8(), b.as_i16x8(), [0, 8, 1, 9, 2, 10, 3, 11]); transmute::(x) } @@ -1723,7 +1553,7 @@ pub unsafe fn _mm_unpacklo_epi16(a: __m128i, b: __m128i) -> __m128i { #[cfg_attr(test, assert_instr(unpcklps))] #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm_unpacklo_epi32(a: __m128i, b: __m128i) -> __m128i { - transmute::(simd_shuffle4(a.as_i32x4(), b.as_i32x4(), [0, 4, 1, 5])) + transmute::(simd_shuffle4!(a.as_i32x4(), b.as_i32x4(), [0, 4, 1, 5])) } /// Unpacks and interleave 64-bit integers from the low half of `a` and `b`. @@ -1734,7 +1564,7 @@ pub unsafe fn _mm_unpacklo_epi32(a: __m128i, b: __m128i) -> __m128i { #[cfg_attr(all(test, not(target_os = "windows")), assert_instr(movlhps))] #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm_unpacklo_epi64(a: __m128i, b: __m128i) -> __m128i { - transmute::(simd_shuffle2(a.as_i64x2(), b.as_i64x2(), [0, 2])) + transmute::(simd_shuffle2!(a.as_i64x2(), b.as_i64x2(), [0, 2])) } /// Returns a new vector with the low element of `a` replaced by the sum of the @@ -2690,7 +2520,7 @@ pub unsafe fn _mm_storeu_pd(mem_addr: *mut f64, a: __m128d) { #[stable(feature = "simd_x86", since = "1.27.0")] #[allow(clippy::cast_ptr_alignment)] pub unsafe fn _mm_store1_pd(mem_addr: *mut f64, a: __m128d) { - let b: __m128d = simd_shuffle2(a, a, [0, 0]); + let b: __m128d = simd_shuffle2!(a, a, [0, 0]); *(mem_addr as *mut __m128d) = b; } @@ -2704,7 +2534,7 @@ pub unsafe fn _mm_store1_pd(mem_addr: *mut f64, a: __m128d) { #[stable(feature = "simd_x86", since = "1.27.0")] #[allow(clippy::cast_ptr_alignment)] pub unsafe fn _mm_store_pd1(mem_addr: *mut f64, a: __m128d) { - let b: __m128d = simd_shuffle2(a, a, [0, 0]); + let b: __m128d = simd_shuffle2!(a, a, [0, 0]); *(mem_addr as *mut __m128d) = b; } @@ -2719,7 +2549,7 @@ pub unsafe fn _mm_store_pd1(mem_addr: *mut f64, a: __m128d) { #[stable(feature = "simd_x86", since = "1.27.0")] #[allow(clippy::cast_ptr_alignment)] pub unsafe fn _mm_storer_pd(mem_addr: *mut f64, a: __m128d) { - let b: __m128d = simd_shuffle2(a, a, [1, 0]); + let b: __m128d = simd_shuffle2!(a, a, [1, 0]); *(mem_addr as *mut __m128d) = b; } @@ -2783,7 +2613,7 @@ pub unsafe fn _mm_load_pd1(mem_addr: *const f64) -> __m128d { #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm_loadr_pd(mem_addr: *const f64) -> __m128d { let a = _mm_load_pd(mem_addr); - simd_shuffle2(a, a, [1, 0]) + simd_shuffle2!(a, a, [1, 0]) } /// Loads 128-bits (composed of 2 packed double-precision (64-bit) @@ -2814,21 +2644,17 @@ pub unsafe fn _mm_loadu_pd(mem_addr: *const f64) -> __m128d { #[target_feature(enable = "sse2")] #[cfg_attr( all(test, any(not(target_os = "windows"), target_arch = "x86")), - assert_instr(shufps, imm8 = 1) + cfg_attr(test, assert_instr(shufps, MASK = 2)) // FIXME shufpd expected )] #[cfg_attr( all(test, all(target_os = "windows", target_arch = "x86_64")), - assert_instr(shufpd, imm8 = 1) + cfg_attr(test, assert_instr(shufpd, MASK = 1)) )] -#[rustc_args_required_const(2)] +#[rustc_legacy_const_generics(2)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm_shuffle_pd(a: __m128d, b: __m128d, imm8: i32) -> __m128d { - match imm8 & 0b11 { - 0b00 => simd_shuffle2(a, b, [0, 2]), - 0b01 => simd_shuffle2(a, b, [1, 2]), - 0b10 => simd_shuffle2(a, b, [0, 3]), - _ => simd_shuffle2(a, b, [1, 3]), - } +pub unsafe fn _mm_shuffle_pd(a: __m128d, b: __m128d) -> __m128d { + static_assert_imm8!(MASK); + simd_shuffle2!(a, b, [MASK as u32 & 0b1, ((MASK as u32 >> 1) & 0b1) + 2]) } /// Constructs a 128-bit floating-point vector of `[2 x double]`. The lower @@ -2952,7 +2778,7 @@ pub unsafe fn _mm_undefined_si128() -> __m128i { #[cfg_attr(test, assert_instr(unpckhpd))] #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm_unpackhi_pd(a: __m128d, b: __m128d) -> __m128d { - simd_shuffle2(a, b, [1, 3]) + simd_shuffle2!(a, b, [1, 3]) } /// The resulting `__m128d` element is composed by the high-order values of @@ -2967,7 +2793,7 @@ pub unsafe fn _mm_unpackhi_pd(a: __m128d, b: __m128d) -> __m128d { #[cfg_attr(all(test, not(target_os = "windows")), assert_instr(movlhps))] #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm_unpacklo_pd(a: __m128d, b: __m128d) -> __m128d { - simd_shuffle2(a, b, [0, 2]) + simd_shuffle2!(a, b, [0, 2]) } #[allow(improper_ctypes)] @@ -3505,7 +3331,7 @@ mod tests { let a = _mm_setr_epi8( 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, ); - let r = _mm_slli_si128(a, 1); + let r = _mm_slli_si128::<1>(a); let e = _mm_setr_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); assert_eq_m128i(r, e); @@ -3513,7 +3339,7 @@ mod tests { let a = _mm_setr_epi8( 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, ); - let r = _mm_slli_si128(a, 15); + let r = _mm_slli_si128::<15>(a); let e = _mm_setr_epi8(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1); assert_eq_m128i(r, e); @@ -3521,21 +3347,7 @@ mod tests { let a = _mm_setr_epi8( 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, ); - let r = _mm_slli_si128(a, 16); - assert_eq_m128i(r, _mm_set1_epi8(0)); - - #[rustfmt::skip] - let a = _mm_setr_epi8( - 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, - ); - let r = _mm_slli_si128(a, -1); - assert_eq_m128i(_mm_set1_epi8(0), r); - - #[rustfmt::skip] - let a = _mm_setr_epi8( - 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, - ); - let r = _mm_slli_si128(a, -0x80000000); + let r = _mm_slli_si128::<16>(a); assert_eq_m128i(r, _mm_set1_epi8(0)); } @@ -3545,7 +3357,7 @@ mod tests { let a = _mm_setr_epi16( 0xFFFF as u16 as i16, 0x0FFF, 0x00FF, 0x000F, 0, 0, 0, 0, ); - let r = _mm_slli_epi16(a, 4); + let r = _mm_slli_epi16::<4>(a); #[rustfmt::skip] let e = _mm_setr_epi16( @@ -3566,7 +3378,7 @@ mod tests { #[simd_test(enable = "sse2")] unsafe fn test_mm_slli_epi32() { - let r = _mm_slli_epi32(_mm_set1_epi32(0xFFFF), 4); + let r = _mm_slli_epi32::<4>(_mm_set1_epi32(0xFFFF)); assert_eq_m128i(r, _mm_set1_epi32(0xFFFF0)); } @@ -3580,7 +3392,7 @@ mod tests { #[simd_test(enable = "sse2")] unsafe fn test_mm_slli_epi64() { - let r = _mm_slli_epi64(_mm_set1_epi64x(0xFFFFFFFF), 4); + let r = _mm_slli_epi64::<4>(_mm_set1_epi64x(0xFFFFFFFF)); assert_eq_m128i(r, _mm_set1_epi64x(0xFFFFFFFF0)); } @@ -3594,7 +3406,7 @@ mod tests { #[simd_test(enable = "sse2")] unsafe fn test_mm_srai_epi16() { - let r = _mm_srai_epi16(_mm_set1_epi16(-1), 1); + let r = _mm_srai_epi16::<1>(_mm_set1_epi16(-1)); assert_eq_m128i(r, _mm_set1_epi16(-1)); } @@ -3608,7 +3420,7 @@ mod tests { #[simd_test(enable = "sse2")] unsafe fn test_mm_srai_epi32() { - let r = _mm_srai_epi32(_mm_set1_epi32(-1), 1); + let r = _mm_srai_epi32::<1>(_mm_set1_epi32(-1)); assert_eq_m128i(r, _mm_set1_epi32(-1)); } @@ -3626,7 +3438,7 @@ mod tests { let a = _mm_setr_epi8( 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, ); - let r = _mm_srli_si128(a, 1); + let r = _mm_srli_si128::<1>(a); #[rustfmt::skip] let e = _mm_setr_epi8( 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 0, @@ -3637,7 +3449,7 @@ mod tests { let a = _mm_setr_epi8( 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, ); - let r = _mm_srli_si128(a, 15); + let r = _mm_srli_si128::<15>(a); let e = _mm_setr_epi8(16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); assert_eq_m128i(r, e); @@ -3645,21 +3457,7 @@ mod tests { let a = _mm_setr_epi8( 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, ); - let r = _mm_srli_si128(a, 16); - assert_eq_m128i(r, _mm_set1_epi8(0)); - - #[rustfmt::skip] - let a = _mm_setr_epi8( - 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, - ); - let r = _mm_srli_si128(a, -1); - assert_eq_m128i(r, _mm_set1_epi8(0)); - - #[rustfmt::skip] - let a = _mm_setr_epi8( - 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, - ); - let r = _mm_srli_si128(a, -0x80000000); + let r = _mm_srli_si128::<16>(a); assert_eq_m128i(r, _mm_set1_epi8(0)); } @@ -3669,7 +3467,7 @@ mod tests { let a = _mm_setr_epi16( 0xFFFF as u16 as i16, 0x0FFF, 0x00FF, 0x000F, 0, 0, 0, 0, ); - let r = _mm_srli_epi16(a, 4); + let r = _mm_srli_epi16::<4>(a); #[rustfmt::skip] let e = _mm_setr_epi16( 0xFFF as u16 as i16, 0xFF as u16 as i16, 0xF, 0, 0, 0, 0, 0, @@ -3688,7 +3486,7 @@ mod tests { #[simd_test(enable = "sse2")] unsafe fn test_mm_srli_epi32() { - let r = _mm_srli_epi32(_mm_set1_epi32(0xFFFF), 4); + let r = _mm_srli_epi32::<4>(_mm_set1_epi32(0xFFFF)); assert_eq_m128i(r, _mm_set1_epi32(0xFFF)); } @@ -3702,7 +3500,7 @@ mod tests { #[simd_test(enable = "sse2")] unsafe fn test_mm_srli_epi64() { - let r = _mm_srli_epi64(_mm_set1_epi64x(0xFFFFFFFF), 4); + let r = _mm_srli_epi64::<4>(_mm_set1_epi64x(0xFFFFFFFF)); assert_eq_m128i(r, _mm_set1_epi64x(0xFFFFFFF)); } @@ -4077,8 +3875,8 @@ mod tests { #[simd_test(enable = "sse2")] unsafe fn test_mm_extract_epi16() { let a = _mm_setr_epi16(-1, 1, 2, 3, 4, 5, 6, 7); - let r1 = _mm_extract_epi16(a, 0); - let r2 = _mm_extract_epi16(a, 11); + let r1 = _mm_extract_epi16::<0>(a); + let r2 = _mm_extract_epi16::<3>(a); assert_eq!(r1, 0xFFFF); assert_eq!(r2, 3); } @@ -4086,7 +3884,7 @@ mod tests { #[simd_test(enable = "sse2")] unsafe fn test_mm_insert_epi16() { let a = _mm_setr_epi16(0, 1, 2, 3, 4, 5, 6, 7); - let r = _mm_insert_epi16(a, 9, 0); + let r = _mm_insert_epi16::<0>(a, 9); let e = _mm_setr_epi16(9, 1, 2, 3, 4, 5, 6, 7); assert_eq_m128i(r, e); } @@ -4107,7 +3905,7 @@ mod tests { #[simd_test(enable = "sse2")] unsafe fn test_mm_shuffle_epi32() { let a = _mm_setr_epi32(5, 10, 15, 20); - let r = _mm_shuffle_epi32(a, 0b00_01_01_11); + let r = _mm_shuffle_epi32::<0b00_01_01_11>(a); let e = _mm_setr_epi32(20, 10, 10, 5); assert_eq_m128i(r, e); } @@ -4115,7 +3913,7 @@ mod tests { #[simd_test(enable = "sse2")] unsafe fn test_mm_shufflehi_epi16() { let a = _mm_setr_epi16(1, 2, 3, 4, 5, 10, 15, 20); - let r = _mm_shufflehi_epi16(a, 0b00_01_01_11); + let r = _mm_shufflehi_epi16::<0b00_01_01_11>(a); let e = _mm_setr_epi16(1, 2, 3, 4, 20, 10, 10, 5); assert_eq_m128i(r, e); } @@ -4123,7 +3921,7 @@ mod tests { #[simd_test(enable = "sse2")] unsafe fn test_mm_shufflelo_epi16() { let a = _mm_setr_epi16(5, 10, 15, 20, 1, 2, 3, 4); - let r = _mm_shufflelo_epi16(a, 0b00_01_01_11); + let r = _mm_shufflelo_epi16::<0b00_01_01_11>(a); let e = _mm_setr_epi16(20, 10, 10, 5, 1, 2, 3, 4); assert_eq_m128i(r, e); } @@ -5041,7 +4839,7 @@ mod tests { let a = _mm_setr_pd(1., 2.); let b = _mm_setr_pd(3., 4.); let expected = _mm_setr_pd(1., 3.); - let r = _mm_shuffle_pd(a, b, 0); + let r = _mm_shuffle_pd::<0b00_00_00_00>(a, b); assert_eq_m128d(r, expected); } diff --git a/library/stdarch/crates/core_arch/src/x86/sse3.rs b/library/stdarch/crates/core_arch/src/x86/sse3.rs index 977de1dc17..ab0dd38fed 100644 --- a/library/stdarch/crates/core_arch/src/x86/sse3.rs +++ b/library/stdarch/crates/core_arch/src/x86/sse3.rs @@ -106,7 +106,7 @@ pub unsafe fn _mm_lddqu_si128(mem_addr: *const __m128i) -> __m128i { #[cfg_attr(test, assert_instr(movddup))] #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm_movedup_pd(a: __m128d) -> __m128d { - simd_shuffle2(a, a, [0, 0]) + simd_shuffle2!(a, a, [0, 0]) } /// Loads a double-precision (64-bit) floating-point element from memory @@ -130,7 +130,7 @@ pub unsafe fn _mm_loaddup_pd(mem_addr: *const f64) -> __m128d { #[cfg_attr(test, assert_instr(movshdup))] #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm_movehdup_ps(a: __m128) -> __m128 { - simd_shuffle4(a, a, [1, 1, 3, 3]) + simd_shuffle4!(a, a, [1, 1, 3, 3]) } /// Duplicate even-indexed single-precision (32-bit) floating-point elements @@ -142,7 +142,7 @@ pub unsafe fn _mm_movehdup_ps(a: __m128) -> __m128 { #[cfg_attr(test, assert_instr(movsldup))] #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm_moveldup_ps(a: __m128) -> __m128 { - simd_shuffle4(a, a, [0, 0, 2, 2]) + simd_shuffle4!(a, a, [0, 0, 2, 2]) } #[allow(improper_ctypes)] diff --git a/library/stdarch/crates/core_arch/src/x86/sse41.rs b/library/stdarch/crates/core_arch/src/x86/sse41.rs index 70c22dfeb6..6e607d4832 100644 --- a/library/stdarch/crates/core_arch/src/x86/sse41.rs +++ b/library/stdarch/crates/core_arch/src/x86/sse41.rs @@ -65,7 +65,7 @@ pub unsafe fn _mm_blendv_epi8(a: __m128i, b: __m128i, mask: __m128i) -> __m128i transmute(pblendvb(a.as_i8x16(), b.as_i8x16(), mask.as_i8x16())) } -/// Blend packed 16-bit integers from `a` and `b` using the mask `imm8`. +/// Blend packed 16-bit integers from `a` and `b` using the mask `IMM8`. /// /// The mask bits determine the selection. A clear bit selects the /// corresponding element of `a`, and a set bit the corresponding @@ -76,19 +76,13 @@ pub unsafe fn _mm_blendv_epi8(a: __m128i, b: __m128i, mask: __m128i) -> __m128i #[target_feature(enable = "sse4.1")] // Note: LLVM7 prefers the single-precision floating-point domain when possible // see https://bugs.llvm.org/show_bug.cgi?id=38195 -// #[cfg_attr(test, assert_instr(pblendw, imm8 = 0xF0))] -#[cfg_attr(test, assert_instr(blendps, imm8 = 0xF0))] -#[rustc_args_required_const(2)] +// #[cfg_attr(test, assert_instr(pblendw, IMM8 = 0xF0))] +#[cfg_attr(test, assert_instr(blendps, IMM8 = 0xF0))] +#[rustc_legacy_const_generics(2)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm_blend_epi16(a: __m128i, b: __m128i, imm8: i32) -> __m128i { - let a = a.as_i16x8(); - let b = b.as_i16x8(); - macro_rules! call { - ($imm8:expr) => { - pblendw(a, b, $imm8) - }; - } - transmute(constify_imm8!(imm8, call)) +pub unsafe fn _mm_blend_epi16(a: __m128i, b: __m128i) -> __m128i { + static_assert_imm8!(IMM8); + transmute(pblendw(a.as_i16x8(), b.as_i16x8(), IMM8 as u8)) } /// Blend packed double-precision (64-bit) floating-point elements from `a` @@ -116,66 +110,54 @@ pub unsafe fn _mm_blendv_ps(a: __m128, b: __m128, mask: __m128) -> __m128 { } /// Blend packed double-precision (64-bit) floating-point elements from `a` -/// and `b` using control mask `imm2` +/// and `b` using control mask `IMM2` /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_blend_pd) #[inline] #[target_feature(enable = "sse4.1")] // Note: LLVM7 prefers the single-precision floating-point domain when possible // see https://bugs.llvm.org/show_bug.cgi?id=38195 -// #[cfg_attr(test, assert_instr(blendpd, imm2 = 0b10))] -#[cfg_attr(test, assert_instr(blendps, imm2 = 0b10))] -#[rustc_args_required_const(2)] -#[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm_blend_pd(a: __m128d, b: __m128d, imm2: i32) -> __m128d { - macro_rules! call { - ($imm2:expr) => { - blendpd(a, b, $imm2) - }; - } - constify_imm2!(imm2, call) +// #[cfg_attr(test, assert_instr(blendpd, IMM2 = 0b10))] +#[cfg_attr(test, assert_instr(blendps, IMM2 = 0b10))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "simd_x86", since = "1.27.0")] +pub unsafe fn _mm_blend_pd(a: __m128d, b: __m128d) -> __m128d { + static_assert_imm2!(IMM2); + blendpd(a, b, IMM2 as u8) } /// Blend packed single-precision (32-bit) floating-point elements from `a` -/// and `b` using mask `imm4` +/// and `b` using mask `IMM4` /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_blend_ps) #[inline] #[target_feature(enable = "sse4.1")] -#[cfg_attr(test, assert_instr(blendps, imm4 = 0b0101))] -#[rustc_args_required_const(2)] -#[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm_blend_ps(a: __m128, b: __m128, imm4: i32) -> __m128 { - macro_rules! call { - ($imm4:expr) => { - blendps(a, b, $imm4) - }; - } - constify_imm4!(imm4, call) +#[cfg_attr(test, assert_instr(blendps, IMM4 = 0b0101))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "simd_x86", since = "1.27.0")] +pub unsafe fn _mm_blend_ps(a: __m128, b: __m128) -> __m128 { + static_assert_imm4!(IMM4); + blendps(a, b, IMM4 as u8) } /// Extracts a single-precision (32-bit) floating-point element from `a`, -/// selected with `imm8` +/// selected with `IMM8` /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_extract_ps) #[inline] #[target_feature(enable = "sse4.1")] #[cfg_attr( all(test, not(target_os = "windows")), - assert_instr(extractps, imm8 = 0) + assert_instr(extractps, IMM8 = 0) )] -#[rustc_args_required_const(1)] +#[rustc_legacy_const_generics(1)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm_extract_ps(a: __m128, imm8: i32) -> i32 { - macro_rules! call { - ($imm2:expr) => { - transmute(simd_extract::<_, f32>(a, $imm2)) - }; - } - constify_imm2!(imm8, call) +pub unsafe fn _mm_extract_ps(a: __m128) -> i32 { + static_assert_imm2!(IMM8); + transmute(simd_extract::<_, f32>(a, IMM8 as u32)) } -/// Extracts an 8-bit integer from `a`, selected with `imm8`. Returns a 32-bit +/// Extracts an 8-bit integer from `a`, selected with `IMM8`. Returns a 32-bit /// integer containing the zero-extended integer data. /// /// See [LLVM commit D20468](https://reviews.llvm.org/D20468). @@ -183,44 +165,34 @@ pub unsafe fn _mm_extract_ps(a: __m128, imm8: i32) -> i32 { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_extract_epi8) #[inline] #[target_feature(enable = "sse4.1")] -#[cfg_attr(test, assert_instr(pextrb, imm8 = 0))] -#[rustc_args_required_const(1)] +#[cfg_attr(test, assert_instr(pextrb, IMM8 = 0))] +#[rustc_legacy_const_generics(1)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm_extract_epi8(a: __m128i, imm8: i32) -> i32 { - let a = a.as_u8x16(); - macro_rules! call { - ($imm4:expr) => { - simd_extract::<_, u8>(a, $imm4) as i32 - }; - } - constify_imm4!(imm8, call) +pub unsafe fn _mm_extract_epi8(a: __m128i) -> i32 { + static_assert_imm4!(IMM8); + simd_extract::<_, u8>(a.as_u8x16(), IMM8 as u32) as i32 } -/// Extracts an 32-bit integer from `a` selected with `imm8` +/// Extracts an 32-bit integer from `a` selected with `IMM8` /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_extract_epi32) #[inline] #[target_feature(enable = "sse4.1")] #[cfg_attr( all(test, not(target_os = "windows")), - assert_instr(extractps, imm8 = 1) + assert_instr(extractps, IMM8 = 1) )] -#[rustc_args_required_const(1)] +#[rustc_legacy_const_generics(1)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm_extract_epi32(a: __m128i, imm8: i32) -> i32 { - let a = a.as_i32x4(); - macro_rules! call { - ($imm2:expr) => { - simd_extract::<_, i32>(a, $imm2) - }; - } - constify_imm2!(imm8, call) +pub unsafe fn _mm_extract_epi32(a: __m128i) -> i32 { + static_assert_imm2!(IMM8); + simd_extract::<_, i32>(a.as_i32x4(), IMM8 as u32) } /// Select a single value in `a` to store at some position in `b`, -/// Then zero elements according to `imm8`. +/// Then zero elements according to `IMM8`. /// -/// `imm8` specifies which bits from operand `a` will be copied, which bits in +/// `IMM8` specifies which bits from operand `a` will be copied, which bits in /// the result they will be copied to, and which bits in the result will be /// cleared. The following assignments are made: /// @@ -243,54 +215,40 @@ pub unsafe fn _mm_extract_epi32(a: __m128i, imm8: i32) -> i32 { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_insert_ps) #[inline] #[target_feature(enable = "sse4.1")] -#[cfg_attr(test, assert_instr(insertps, imm8 = 0b1010))] -#[rustc_args_required_const(2)] -#[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm_insert_ps(a: __m128, b: __m128, imm8: i32) -> __m128 { - macro_rules! call { - ($imm8:expr) => { - insertps(a, b, $imm8) - }; - } - constify_imm8!(imm8, call) +#[cfg_attr(test, assert_instr(insertps, IMM8 = 0b1010))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "simd_x86", since = "1.27.0")] +pub unsafe fn _mm_insert_ps(a: __m128, b: __m128) -> __m128 { + static_assert_imm8!(IMM8); + insertps(a, b, IMM8 as u8) } /// Returns a copy of `a` with the 8-bit integer from `i` inserted at a -/// location specified by `imm8`. +/// location specified by `IMM8`. /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_insert_epi8) #[inline] #[target_feature(enable = "sse4.1")] -#[cfg_attr(test, assert_instr(pinsrb, imm8 = 0))] -#[rustc_args_required_const(2)] +#[cfg_attr(test, assert_instr(pinsrb, IMM8 = 0))] +#[rustc_legacy_const_generics(2)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm_insert_epi8(a: __m128i, i: i32, imm8: i32) -> __m128i { - let a = a.as_i8x16(); - macro_rules! call { - ($imm4:expr) => { - transmute(simd_insert(a, $imm4, i as i8)) - }; - } - constify_imm4!(imm8, call) +pub unsafe fn _mm_insert_epi8(a: __m128i, i: i32) -> __m128i { + static_assert_imm4!(IMM8); + transmute(simd_insert(a.as_i8x16(), IMM8 as u32, i as i8)) } /// Returns a copy of `a` with the 32-bit integer from `i` inserted at a -/// location specified by `imm8`. +/// location specified by `IMM8`. /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_insert_epi32) #[inline] #[target_feature(enable = "sse4.1")] -#[cfg_attr(test, assert_instr(pinsrd, imm8 = 0))] -#[rustc_args_required_const(2)] +#[cfg_attr(test, assert_instr(pinsrd, IMM8 = 0))] +#[rustc_legacy_const_generics(2)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm_insert_epi32(a: __m128i, i: i32, imm8: i32) -> __m128i { - let a = a.as_i32x4(); - macro_rules! call { - ($imm2:expr) => { - transmute(simd_insert(a, $imm2, i)) - }; - } - constify_imm2!(imm8, call) +pub unsafe fn _mm_insert_epi32(a: __m128i, i: i32) -> __m128i { + static_assert_imm2!(IMM8); + transmute(simd_insert(a.as_i32x4(), IMM8 as u32, i)) } /// Compares packed 8-bit integers in `a` and `b` and returns packed maximum @@ -421,7 +379,7 @@ pub unsafe fn _mm_cmpeq_epi64(a: __m128i, b: __m128i) -> __m128i { #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm_cvtepi8_epi16(a: __m128i) -> __m128i { let a = a.as_i8x16(); - let a = simd_shuffle8::<_, i8x8>(a, a, [0, 1, 2, 3, 4, 5, 6, 7]); + let a: i8x8 = simd_shuffle8!(a, a, [0, 1, 2, 3, 4, 5, 6, 7]); transmute(simd_cast::<_, i16x8>(a)) } @@ -434,7 +392,7 @@ pub unsafe fn _mm_cvtepi8_epi16(a: __m128i) -> __m128i { #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm_cvtepi8_epi32(a: __m128i) -> __m128i { let a = a.as_i8x16(); - let a = simd_shuffle4::<_, i8x4>(a, a, [0, 1, 2, 3]); + let a: i8x4 = simd_shuffle4!(a, a, [0, 1, 2, 3]); transmute(simd_cast::<_, i32x4>(a)) } @@ -448,7 +406,7 @@ pub unsafe fn _mm_cvtepi8_epi32(a: __m128i) -> __m128i { #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm_cvtepi8_epi64(a: __m128i) -> __m128i { let a = a.as_i8x16(); - let a = simd_shuffle2::<_, i8x2>(a, a, [0, 1]); + let a: i8x2 = simd_shuffle2!(a, a, [0, 1]); transmute(simd_cast::<_, i64x2>(a)) } @@ -461,7 +419,7 @@ pub unsafe fn _mm_cvtepi8_epi64(a: __m128i) -> __m128i { #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm_cvtepi16_epi32(a: __m128i) -> __m128i { let a = a.as_i16x8(); - let a = simd_shuffle4::<_, i16x4>(a, a, [0, 1, 2, 3]); + let a: i16x4 = simd_shuffle4!(a, a, [0, 1, 2, 3]); transmute(simd_cast::<_, i32x4>(a)) } @@ -474,7 +432,7 @@ pub unsafe fn _mm_cvtepi16_epi32(a: __m128i) -> __m128i { #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm_cvtepi16_epi64(a: __m128i) -> __m128i { let a = a.as_i16x8(); - let a = simd_shuffle2::<_, i16x2>(a, a, [0, 1]); + let a: i16x2 = simd_shuffle2!(a, a, [0, 1]); transmute(simd_cast::<_, i64x2>(a)) } @@ -487,7 +445,7 @@ pub unsafe fn _mm_cvtepi16_epi64(a: __m128i) -> __m128i { #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm_cvtepi32_epi64(a: __m128i) -> __m128i { let a = a.as_i32x4(); - let a = simd_shuffle2::<_, i32x2>(a, a, [0, 1]); + let a: i32x2 = simd_shuffle2!(a, a, [0, 1]); transmute(simd_cast::<_, i64x2>(a)) } @@ -500,7 +458,7 @@ pub unsafe fn _mm_cvtepi32_epi64(a: __m128i) -> __m128i { #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm_cvtepu8_epi16(a: __m128i) -> __m128i { let a = a.as_u8x16(); - let a = simd_shuffle8::<_, u8x8>(a, a, [0, 1, 2, 3, 4, 5, 6, 7]); + let a: u8x8 = simd_shuffle8!(a, a, [0, 1, 2, 3, 4, 5, 6, 7]); transmute(simd_cast::<_, i16x8>(a)) } @@ -513,7 +471,7 @@ pub unsafe fn _mm_cvtepu8_epi16(a: __m128i) -> __m128i { #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm_cvtepu8_epi32(a: __m128i) -> __m128i { let a = a.as_u8x16(); - let a = simd_shuffle4::<_, u8x4>(a, a, [0, 1, 2, 3]); + let a: u8x4 = simd_shuffle4!(a, a, [0, 1, 2, 3]); transmute(simd_cast::<_, i32x4>(a)) } @@ -526,7 +484,7 @@ pub unsafe fn _mm_cvtepu8_epi32(a: __m128i) -> __m128i { #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm_cvtepu8_epi64(a: __m128i) -> __m128i { let a = a.as_u8x16(); - let a = simd_shuffle2::<_, u8x2>(a, a, [0, 1]); + let a: u8x2 = simd_shuffle2!(a, a, [0, 1]); transmute(simd_cast::<_, i64x2>(a)) } @@ -540,7 +498,7 @@ pub unsafe fn _mm_cvtepu8_epi64(a: __m128i) -> __m128i { #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm_cvtepu16_epi32(a: __m128i) -> __m128i { let a = a.as_u16x8(); - let a = simd_shuffle4::<_, u16x4>(a, a, [0, 1, 2, 3]); + let a: u16x4 = simd_shuffle4!(a, a, [0, 1, 2, 3]); transmute(simd_cast::<_, i32x4>(a)) } @@ -554,7 +512,7 @@ pub unsafe fn _mm_cvtepu16_epi32(a: __m128i) -> __m128i { #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm_cvtepu16_epi64(a: __m128i) -> __m128i { let a = a.as_u16x8(); - let a = simd_shuffle2::<_, u16x2>(a, a, [0, 1]); + let a: u16x2 = simd_shuffle2!(a, a, [0, 1]); transmute(simd_cast::<_, i64x2>(a)) } @@ -568,13 +526,13 @@ pub unsafe fn _mm_cvtepu16_epi64(a: __m128i) -> __m128i { #[stable(feature = "simd_x86", since = "1.27.0")] pub unsafe fn _mm_cvtepu32_epi64(a: __m128i) -> __m128i { let a = a.as_u32x4(); - let a = simd_shuffle2::<_, u32x2>(a, a, [0, 1]); + let a: u32x2 = simd_shuffle2!(a, a, [0, 1]); transmute(simd_cast::<_, i64x2>(a)) } /// Returns the dot product of two __m128d vectors. /// -/// `imm8[1:0]` is the broadcast mask, and `imm8[5:4]` is the condition mask. +/// `IMM8[1:0]` is the broadcast mask, and `IMM8[5:4]` is the condition mask. /// If a condition mask bit is zero, the corresponding multiplication is /// replaced by a value of `0.0`. If a broadcast mask bit is one, the result of /// the dot product will be stored in the return value component. Otherwise if @@ -583,21 +541,17 @@ pub unsafe fn _mm_cvtepu32_epi64(a: __m128i) -> __m128i { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_dp_pd) #[inline] #[target_feature(enable = "sse4.1")] -#[cfg_attr(test, assert_instr(dppd, imm8 = 0))] -#[rustc_args_required_const(2)] -#[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm_dp_pd(a: __m128d, b: __m128d, imm8: i32) -> __m128d { - macro_rules! call { - ($imm8:expr) => { - dppd(a, b, $imm8) - }; - } - constify_imm8!(imm8, call) +#[cfg_attr(test, assert_instr(dppd, IMM8 = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "simd_x86", since = "1.27.0")] +pub unsafe fn _mm_dp_pd(a: __m128d, b: __m128d) -> __m128d { + static_assert_imm8!(IMM8); + dppd(a, b, IMM8 as u8) } /// Returns the dot product of two __m128 vectors. /// -/// `imm8[3:0]` is the broadcast mask, and `imm8[7:4]` is the condition mask. +/// `IMM8[3:0]` is the broadcast mask, and `IMM8[7:4]` is the condition mask. /// If a condition mask bit is zero, the corresponding multiplication is /// replaced by a value of `0.0`. If a broadcast mask bit is one, the result of /// the dot product will be stored in the return value component. Otherwise if @@ -606,16 +560,12 @@ pub unsafe fn _mm_dp_pd(a: __m128d, b: __m128d, imm8: i32) -> __m128d { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_dp_ps) #[inline] #[target_feature(enable = "sse4.1")] -#[cfg_attr(test, assert_instr(dpps, imm8 = 0))] -#[rustc_args_required_const(2)] -#[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm_dp_ps(a: __m128, b: __m128, imm8: i32) -> __m128 { - macro_rules! call { - ($imm8:expr) => { - dpps(a, b, $imm8) - }; - } - constify_imm8!(imm8, call) +#[cfg_attr(test, assert_instr(dpps, IMM8 = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "simd_x86", since = "1.27.0")] +pub unsafe fn _mm_dp_ps(a: __m128, b: __m128) -> __m128 { + static_assert_imm8!(IMM8); + dpps(a, b, IMM8 as u8) } /// Round the packed double-precision (64-bit) floating-point elements in `a` @@ -731,7 +681,7 @@ pub unsafe fn _mm_ceil_ss(a: __m128, b: __m128) -> __m128 { } /// Round the packed double-precision (64-bit) floating-point elements in `a` -/// using the `rounding` parameter, and stores the results as packed +/// using the `ROUNDING` parameter, and stores the results as packed /// double-precision floating-point elements. /// Rounding is done according to the rounding parameter, which can be one of: /// @@ -763,20 +713,16 @@ pub unsafe fn _mm_ceil_ss(a: __m128, b: __m128) -> __m128 { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_round_pd) #[inline] #[target_feature(enable = "sse4.1")] -#[cfg_attr(test, assert_instr(roundpd, rounding = 0))] -#[rustc_args_required_const(1)] -#[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm_round_pd(a: __m128d, rounding: i32) -> __m128d { - macro_rules! call { - ($imm4:expr) => { - roundpd(a, $imm4) - }; - } - constify_imm4!(rounding, call) +#[cfg_attr(test, assert_instr(roundpd, ROUNDING = 0))] +#[rustc_legacy_const_generics(1)] +#[stable(feature = "simd_x86", since = "1.27.0")] +pub unsafe fn _mm_round_pd(a: __m128d) -> __m128d { + static_assert_imm4!(ROUNDING); + roundpd(a, ROUNDING) } /// Round the packed single-precision (32-bit) floating-point elements in `a` -/// using the `rounding` parameter, and stores the results as packed +/// using the `ROUNDING` parameter, and stores the results as packed /// single-precision floating-point elements. /// Rounding is done according to the rounding parameter, which can be one of: /// @@ -808,20 +754,16 @@ pub unsafe fn _mm_round_pd(a: __m128d, rounding: i32) -> __m128d { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_round_ps) #[inline] #[target_feature(enable = "sse4.1")] -#[cfg_attr(test, assert_instr(roundps, rounding = 0))] -#[rustc_args_required_const(1)] -#[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm_round_ps(a: __m128, rounding: i32) -> __m128 { - macro_rules! call { - ($imm4:expr) => { - roundps(a, $imm4) - }; - } - constify_imm4!(rounding, call) +#[cfg_attr(test, assert_instr(roundps, ROUNDING = 0))] +#[rustc_legacy_const_generics(1)] +#[stable(feature = "simd_x86", since = "1.27.0")] +pub unsafe fn _mm_round_ps(a: __m128) -> __m128 { + static_assert_imm4!(ROUNDING); + roundps(a, ROUNDING) } /// Round the lower double-precision (64-bit) floating-point element in `b` -/// using the `rounding` parameter, store the result as a double-precision +/// using the `ROUNDING` parameter, store the result as a double-precision /// floating-point element in the lower element of the intrinsic result, /// and copies the upper element from `a` to the upper element of the intrinsic /// result. @@ -855,20 +797,16 @@ pub unsafe fn _mm_round_ps(a: __m128, rounding: i32) -> __m128 { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_round_sd) #[inline] #[target_feature(enable = "sse4.1")] -#[cfg_attr(test, assert_instr(roundsd, rounding = 0))] -#[rustc_args_required_const(2)] -#[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm_round_sd(a: __m128d, b: __m128d, rounding: i32) -> __m128d { - macro_rules! call { - ($imm4:expr) => { - roundsd(a, b, $imm4) - }; - } - constify_imm4!(rounding, call) +#[cfg_attr(test, assert_instr(roundsd, ROUNDING = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "simd_x86", since = "1.27.0")] +pub unsafe fn _mm_round_sd(a: __m128d, b: __m128d) -> __m128d { + static_assert_imm4!(ROUNDING); + roundsd(a, b, ROUNDING) } /// Round the lower single-precision (32-bit) floating-point element in `b` -/// using the `rounding` parameter, store the result as a single-precision +/// using the `ROUNDING` parameter, store the result as a single-precision /// floating-point element in the lower element of the intrinsic result, /// and copies the upper 3 packed elements from `a` to the upper elements /// of the instrinsic result. @@ -902,16 +840,12 @@ pub unsafe fn _mm_round_sd(a: __m128d, b: __m128d, rounding: i32) -> __m128d { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_round_ss) #[inline] #[target_feature(enable = "sse4.1")] -#[cfg_attr(test, assert_instr(roundss, rounding = 0))] -#[rustc_args_required_const(2)] -#[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm_round_ss(a: __m128, b: __m128, rounding: i32) -> __m128 { - macro_rules! call { - ($imm4:expr) => { - roundss(a, b, $imm4) - }; - } - constify_imm4!(rounding, call) +#[cfg_attr(test, assert_instr(roundss, ROUNDING = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "simd_x86", since = "1.27.0")] +pub unsafe fn _mm_round_ss(a: __m128, b: __m128) -> __m128 { + static_assert_imm4!(ROUNDING); + roundss(a, b, ROUNDING) } /// Finds the minimum unsigned 16-bit element in the 128-bit __m128i vector, @@ -979,8 +913,8 @@ pub unsafe fn _mm_mullo_epi32(a: __m128i, b: __m128i) -> __m128i { /// The following algorithm is performed: /// /// ```ignore -/// i = imm8[2] * 4 -/// j = imm8[1:0] * 4 +/// i = IMM8[2] * 4 +/// j = IMM8[1:0] * 4 /// for k := 0 to 7 /// d0 = abs(a[i + k + 0] - b[j + 0]) /// d1 = abs(a[i + k + 1] - b[j + 1]) @@ -993,7 +927,7 @@ pub unsafe fn _mm_mullo_epi32(a: __m128i, b: __m128i) -> __m128i { /// /// * `a` - A 128-bit vector of type `__m128i`. /// * `b` - A 128-bit vector of type `__m128i`. -/// * `imm8` - An 8-bit immediate operand specifying how the absolute +/// * `IMM8` - An 8-bit immediate operand specifying how the absolute /// differences are to be calculated /// * Bit `[2]` specify the offset for operand `a` /// * Bits `[1:0]` specify the offset for operand `b` @@ -1006,18 +940,12 @@ pub unsafe fn _mm_mullo_epi32(a: __m128i, b: __m128i) -> __m128i { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mpsadbw_epu8) #[inline] #[target_feature(enable = "sse4.1")] -#[cfg_attr(test, assert_instr(mpsadbw, imm8 = 0))] -#[rustc_args_required_const(2)] +#[cfg_attr(test, assert_instr(mpsadbw, IMM8 = 0))] +#[rustc_legacy_const_generics(2)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm_mpsadbw_epu8(a: __m128i, b: __m128i, imm8: i32) -> __m128i { - let a = a.as_u8x16(); - let b = b.as_u8x16(); - macro_rules! call { - ($imm8:expr) => { - mpsadbw(a, b, $imm8) - }; - } - transmute(constify_imm3!(imm8, call)) +pub unsafe fn _mm_mpsadbw_epu8(a: __m128i, b: __m128i) -> __m128i { + static_assert_imm3!(IMM8); + transmute(mpsadbw(a.as_u8x16(), b.as_u8x16(), IMM8 as u8)) } /// Tests whether the specified bits in a 128-bit integer vector are all @@ -1270,7 +1198,7 @@ mod tests { unsafe fn test_mm_blend_pd() { let a = _mm_set1_pd(0.0); let b = _mm_set1_pd(1.0); - let r = _mm_blend_pd(a, b, 0b10); + let r = _mm_blend_pd::<0b10>(a, b); let e = _mm_setr_pd(0.0, 1.0); assert_eq_m128d(r, e); } @@ -1279,7 +1207,7 @@ mod tests { unsafe fn test_mm_blend_ps() { let a = _mm_set1_ps(0.0); let b = _mm_set1_ps(1.0); - let r = _mm_blend_ps(a, b, 0b1010); + let r = _mm_blend_ps::<0b1010>(a, b); let e = _mm_setr_ps(0.0, 1.0, 0.0, 1.0); assert_eq_m128(r, e); } @@ -1288,7 +1216,7 @@ mod tests { unsafe fn test_mm_blend_epi16() { let a = _mm_set1_epi16(0); let b = _mm_set1_epi16(1); - let r = _mm_blend_epi16(a, b, 0b1010_1100); + let r = _mm_blend_epi16::<0b1010_1100>(a, b); let e = _mm_setr_epi16(0, 0, 1, 1, 0, 1, 0, 1); assert_eq_m128i(r, e); } @@ -1296,10 +1224,10 @@ mod tests { #[simd_test(enable = "sse4.1")] unsafe fn test_mm_extract_ps() { let a = _mm_setr_ps(0.0, 1.0, 2.0, 3.0); - let r: f32 = transmute(_mm_extract_ps(a, 1)); - assert_eq!(r, 1.0); - let r: f32 = transmute(_mm_extract_ps(a, 5)); + let r: f32 = transmute(_mm_extract_ps::<1>(a)); assert_eq!(r, 1.0); + let r: f32 = transmute(_mm_extract_ps::<3>(a)); + assert_eq!(r, 3.0); } #[simd_test(enable = "sse4.1")] @@ -1309,8 +1237,8 @@ mod tests { -1, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 ); - let r1 = _mm_extract_epi8(a, 0); - let r2 = _mm_extract_epi8(a, 19); + let r1 = _mm_extract_epi8::<0>(a); + let r2 = _mm_extract_epi8::<3>(a); assert_eq!(r1, 0xFF); assert_eq!(r2, 3); } @@ -1318,17 +1246,17 @@ mod tests { #[simd_test(enable = "sse4.1")] unsafe fn test_mm_extract_epi32() { let a = _mm_setr_epi32(0, 1, 2, 3); - let r = _mm_extract_epi32(a, 1); - assert_eq!(r, 1); - let r = _mm_extract_epi32(a, 5); + let r = _mm_extract_epi32::<1>(a); assert_eq!(r, 1); + let r = _mm_extract_epi32::<3>(a); + assert_eq!(r, 3); } #[simd_test(enable = "sse4.1")] unsafe fn test_mm_insert_ps() { let a = _mm_set1_ps(1.0); let b = _mm_setr_ps(1.0, 2.0, 3.0, 4.0); - let r = _mm_insert_ps(a, b, 0b11_00_1100); + let r = _mm_insert_ps::<0b11_00_1100>(a, b); let e = _mm_setr_ps(4.0, 1.0, 0.0, 0.0); assert_eq_m128(r, e); } @@ -1337,9 +1265,10 @@ mod tests { unsafe fn test_mm_insert_epi8() { let a = _mm_set1_epi8(0); let e = _mm_setr_epi8(0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); - let r = _mm_insert_epi8(a, 32, 1); + let r = _mm_insert_epi8::<1>(a, 32); assert_eq_m128i(r, e); - let r = _mm_insert_epi8(a, 32, 17); + let e = _mm_setr_epi8(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0); + let r = _mm_insert_epi8::<14>(a, 32); assert_eq_m128i(r, e); } @@ -1347,9 +1276,10 @@ mod tests { unsafe fn test_mm_insert_epi32() { let a = _mm_set1_epi32(0); let e = _mm_setr_epi32(0, 32, 0, 0); - let r = _mm_insert_epi32(a, 32, 1); + let r = _mm_insert_epi32::<1>(a, 32); assert_eq_m128i(r, e); - let r = _mm_insert_epi32(a, 32, 5); + let e = _mm_setr_epi32(0, 0, 0, 32); + let r = _mm_insert_epi32::<3>(a, 32); assert_eq_m128i(r, e); } @@ -1622,7 +1552,7 @@ mod tests { let a = _mm_setr_pd(2.0, 3.0); let b = _mm_setr_pd(1.0, 4.0); let e = _mm_setr_pd(14.0, 0.0); - assert_eq_m128d(_mm_dp_pd(a, b, 0b00110001), e); + assert_eq_m128d(_mm_dp_pd::<0b00110001>(a, b), e); } #[simd_test(enable = "sse4.1")] @@ -1630,7 +1560,7 @@ mod tests { let a = _mm_setr_ps(2.0, 3.0, 1.0, 10.0); let b = _mm_setr_ps(1.0, 4.0, 0.5, 10.0); let e = _mm_setr_ps(14.5, 0.0, 14.5, 0.0); - assert_eq_m128(_mm_dp_ps(a, b, 0b01110101), e); + assert_eq_m128(_mm_dp_ps::<0b01110101>(a, b), e); } #[simd_test(enable = "sse4.1")] @@ -1704,7 +1634,7 @@ mod tests { #[simd_test(enable = "sse4.1")] unsafe fn test_mm_round_pd() { let a = _mm_setr_pd(1.25, 3.75); - let r = _mm_round_pd(a, _MM_FROUND_TO_NEAREST_INT); + let r = _mm_round_pd::<_MM_FROUND_TO_NEAREST_INT>(a); let e = _mm_setr_pd(1.0, 4.0); assert_eq_m128d(r, e); } @@ -1712,7 +1642,7 @@ mod tests { #[simd_test(enable = "sse4.1")] unsafe fn test_mm_round_ps() { let a = _mm_setr_ps(2.25, 4.75, -1.75, -4.25); - let r = _mm_round_ps(a, _MM_FROUND_TO_ZERO); + let r = _mm_round_ps::<_MM_FROUND_TO_ZERO>(a); let e = _mm_setr_ps(2.0, 4.0, -1.0, -4.0); assert_eq_m128(r, e); } @@ -1723,7 +1653,7 @@ mod tests { let b = _mm_setr_pd(-2.5, -4.5); let old_mode = _MM_GET_ROUNDING_MODE(); _MM_SET_ROUNDING_MODE(_MM_ROUND_TOWARD_ZERO); - let r = _mm_round_sd(a, b, _MM_FROUND_CUR_DIRECTION); + let r = _mm_round_sd::<_MM_FROUND_CUR_DIRECTION>(a, b); _MM_SET_ROUNDING_MODE(old_mode); let e = _mm_setr_pd(-2.0, 3.5); assert_eq_m128d(r, e); @@ -1735,7 +1665,7 @@ mod tests { let b = _mm_setr_ps(-1.75, -4.5, -8.5, -16.5); let old_mode = _MM_GET_ROUNDING_MODE(); _MM_SET_ROUNDING_MODE(_MM_ROUND_NEAREST); - let r = _mm_round_ss(a, b, _MM_FROUND_CUR_DIRECTION); + let r = _mm_round_ss::<_MM_FROUND_CUR_DIRECTION>(a, b); _MM_SET_ROUNDING_MODE(old_mode); let e = _mm_setr_ps(-2.0, 3.5, 7.5, 15.5); assert_eq_m128(r, e); @@ -1815,23 +1745,23 @@ mod tests { 8, 9, 10, 11, 12, 13, 14, 15, ); - let r = _mm_mpsadbw_epu8(a, a, 0b000); + let r = _mm_mpsadbw_epu8::<0b000>(a, a); let e = _mm_setr_epi16(0, 4, 8, 12, 16, 20, 24, 28); assert_eq_m128i(r, e); - let r = _mm_mpsadbw_epu8(a, a, 0b001); + let r = _mm_mpsadbw_epu8::<0b001>(a, a); let e = _mm_setr_epi16(16, 12, 8, 4, 0, 4, 8, 12); assert_eq_m128i(r, e); - let r = _mm_mpsadbw_epu8(a, a, 0b100); + let r = _mm_mpsadbw_epu8::<0b100>(a, a); let e = _mm_setr_epi16(16, 20, 24, 28, 32, 36, 40, 44); assert_eq_m128i(r, e); - let r = _mm_mpsadbw_epu8(a, a, 0b101); + let r = _mm_mpsadbw_epu8::<0b101>(a, a); let e = _mm_setr_epi16(0, 4, 8, 12, 16, 20, 24, 28); assert_eq_m128i(r, e); - let r = _mm_mpsadbw_epu8(a, a, 0b111); + let r = _mm_mpsadbw_epu8::<0b111>(a, a); let e = _mm_setr_epi16(32, 28, 24, 20, 16, 12, 8, 4); assert_eq_m128i(r, e); } diff --git a/library/stdarch/crates/core_arch/src/x86/sse42.rs b/library/stdarch/crates/core_arch/src/x86/sse42.rs index cce05e864c..3aed5e7dde 100644 --- a/library/stdarch/crates/core_arch/src/x86/sse42.rs +++ b/library/stdarch/crates/core_arch/src/x86/sse42.rs @@ -65,33 +65,27 @@ pub const _SIDD_BIT_MASK: i32 = 0b0000_0000; pub const _SIDD_UNIT_MASK: i32 = 0b0100_0000; /// Compares packed strings with implicit lengths in `a` and `b` using the -/// control in `imm8`, and return the generated mask. +/// control in `IMM8`, and return the generated mask. /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cmpistrm) #[inline] #[target_feature(enable = "sse4.2")] -#[cfg_attr(test, assert_instr(pcmpistrm, imm8 = 0))] -#[rustc_args_required_const(2)] -#[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm_cmpistrm(a: __m128i, b: __m128i, imm8: i32) -> __m128i { - let a = a.as_i8x16(); - let b = b.as_i8x16(); - macro_rules! call { - ($imm8:expr) => { - pcmpistrm128(a, b, $imm8) - }; - } - transmute(constify_imm8!(imm8, call)) +#[cfg_attr(test, assert_instr(pcmpistrm, IMM8 = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "simd_x86", since = "1.27.0")] +pub unsafe fn _mm_cmpistrm(a: __m128i, b: __m128i) -> __m128i { + static_assert_imm8!(IMM8); + transmute(pcmpistrm128(a.as_i8x16(), b.as_i8x16(), IMM8 as i8)) } /// Compares packed strings with implicit lengths in `a` and `b` using the -/// control in `imm8` and return the generated index. Similar to +/// control in `IMM8` and return the generated index. Similar to /// [`_mm_cmpestri`] with the exception that [`_mm_cmpestri`] requires the /// lengths of `a` and `b` to be explicitly specified. /// /// # Control modes /// -/// The control specified by `imm8` may be one or more of the following. +/// The control specified by `IMM8` may be one or more of the following. /// /// ## Data size and signedness /// @@ -264,152 +258,110 @@ pub unsafe fn _mm_cmpistrm(a: __m128i, b: __m128i, imm8: i32) -> __m128i { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cmpistri) #[inline] #[target_feature(enable = "sse4.2")] -#[cfg_attr(test, assert_instr(pcmpistri, imm8 = 0))] -#[rustc_args_required_const(2)] -#[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm_cmpistri(a: __m128i, b: __m128i, imm8: i32) -> i32 { - let a = a.as_i8x16(); - let b = b.as_i8x16(); - macro_rules! call { - ($imm8:expr) => { - pcmpistri128(a, b, $imm8) - }; - } - constify_imm8!(imm8, call) +#[cfg_attr(test, assert_instr(pcmpistri, IMM8 = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "simd_x86", since = "1.27.0")] +pub unsafe fn _mm_cmpistri(a: __m128i, b: __m128i) -> i32 { + static_assert_imm8!(IMM8); + pcmpistri128(a.as_i8x16(), b.as_i8x16(), IMM8 as i8) } /// Compares packed strings with implicit lengths in `a` and `b` using the -/// control in `imm8`, and return `1` if any character in `b` was null. +/// control in `IMM8`, and return `1` if any character in `b` was null. /// and `0` otherwise. /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cmpistrz) #[inline] #[target_feature(enable = "sse4.2")] -#[cfg_attr(test, assert_instr(pcmpistri, imm8 = 0))] -#[rustc_args_required_const(2)] -#[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm_cmpistrz(a: __m128i, b: __m128i, imm8: i32) -> i32 { - let a = a.as_i8x16(); - let b = b.as_i8x16(); - macro_rules! call { - ($imm8:expr) => { - pcmpistriz128(a, b, $imm8) - }; - } - constify_imm8!(imm8, call) +#[cfg_attr(test, assert_instr(pcmpistri, IMM8 = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "simd_x86", since = "1.27.0")] +pub unsafe fn _mm_cmpistrz(a: __m128i, b: __m128i) -> i32 { + static_assert_imm8!(IMM8); + pcmpistriz128(a.as_i8x16(), b.as_i8x16(), IMM8 as i8) } /// Compares packed strings with implicit lengths in `a` and `b` using the -/// control in `imm8`, and return `1` if the resulting mask was non-zero, +/// control in `IMM8`, and return `1` if the resulting mask was non-zero, /// and `0` otherwise. /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cmpistrc) #[inline] #[target_feature(enable = "sse4.2")] -#[cfg_attr(test, assert_instr(pcmpistri, imm8 = 0))] -#[rustc_args_required_const(2)] -#[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm_cmpistrc(a: __m128i, b: __m128i, imm8: i32) -> i32 { - let a = a.as_i8x16(); - let b = b.as_i8x16(); - macro_rules! call { - ($imm8:expr) => { - pcmpistric128(a, b, $imm8) - }; - } - constify_imm8!(imm8, call) +#[cfg_attr(test, assert_instr(pcmpistri, IMM8 = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "simd_x86", since = "1.27.0")] +pub unsafe fn _mm_cmpistrc(a: __m128i, b: __m128i) -> i32 { + static_assert_imm8!(IMM8); + pcmpistric128(a.as_i8x16(), b.as_i8x16(), IMM8 as i8) } /// Compares packed strings with implicit lengths in `a` and `b` using the -/// control in `imm8`, and returns `1` if any character in `a` was null, +/// control in `IMM8`, and returns `1` if any character in `a` was null, /// and `0` otherwise. /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cmpistrs) #[inline] #[target_feature(enable = "sse4.2")] -#[cfg_attr(test, assert_instr(pcmpistri, imm8 = 0))] -#[rustc_args_required_const(2)] -#[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm_cmpistrs(a: __m128i, b: __m128i, imm8: i32) -> i32 { - let a = a.as_i8x16(); - let b = b.as_i8x16(); - macro_rules! call { - ($imm8:expr) => { - pcmpistris128(a, b, $imm8) - }; - } - constify_imm8!(imm8, call) +#[cfg_attr(test, assert_instr(pcmpistri, IMM8 = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "simd_x86", since = "1.27.0")] +pub unsafe fn _mm_cmpistrs(a: __m128i, b: __m128i) -> i32 { + static_assert_imm8!(IMM8); + pcmpistris128(a.as_i8x16(), b.as_i8x16(), IMM8 as i8) } /// Compares packed strings with implicit lengths in `a` and `b` using the -/// control in `imm8`, and return bit `0` of the resulting bit mask. +/// control in `IMM8`, and return bit `0` of the resulting bit mask. /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cmpistro) #[inline] #[target_feature(enable = "sse4.2")] -#[cfg_attr(test, assert_instr(pcmpistri, imm8 = 0))] -#[rustc_args_required_const(2)] -#[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm_cmpistro(a: __m128i, b: __m128i, imm8: i32) -> i32 { - let a = a.as_i8x16(); - let b = b.as_i8x16(); - macro_rules! call { - ($imm8:expr) => { - pcmpistrio128(a, b, $imm8) - }; - } - constify_imm8!(imm8, call) +#[cfg_attr(test, assert_instr(pcmpistri, IMM8 = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "simd_x86", since = "1.27.0")] +pub unsafe fn _mm_cmpistro(a: __m128i, b: __m128i) -> i32 { + static_assert_imm8!(IMM8); + pcmpistrio128(a.as_i8x16(), b.as_i8x16(), IMM8 as i8) } /// Compares packed strings with implicit lengths in `a` and `b` using the -/// control in `imm8`, and return `1` if `b` did not contain a null +/// control in `IMM8`, and return `1` if `b` did not contain a null /// character and the resulting mask was zero, and `0` otherwise. /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cmpistra) #[inline] #[target_feature(enable = "sse4.2")] -#[cfg_attr(test, assert_instr(pcmpistri, imm8 = 0))] -#[rustc_args_required_const(2)] -#[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm_cmpistra(a: __m128i, b: __m128i, imm8: i32) -> i32 { - let a = a.as_i8x16(); - let b = b.as_i8x16(); - macro_rules! call { - ($imm8:expr) => { - pcmpistria128(a, b, $imm8) - }; - } - constify_imm8!(imm8, call) +#[cfg_attr(test, assert_instr(pcmpistri, IMM8 = 0))] +#[rustc_legacy_const_generics(2)] +#[stable(feature = "simd_x86", since = "1.27.0")] +pub unsafe fn _mm_cmpistra(a: __m128i, b: __m128i) -> i32 { + static_assert_imm8!(IMM8); + pcmpistria128(a.as_i8x16(), b.as_i8x16(), IMM8 as i8) } /// Compares packed strings in `a` and `b` with lengths `la` and `lb` -/// using the control in `imm8`, and return the generated mask. +/// using the control in `IMM8`, and return the generated mask. /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cmpestrm) #[inline] #[target_feature(enable = "sse4.2")] -#[cfg_attr(test, assert_instr(pcmpestrm, imm8 = 0))] -#[rustc_args_required_const(4)] -#[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm_cmpestrm(a: __m128i, la: i32, b: __m128i, lb: i32, imm8: i32) -> __m128i { - let a = a.as_i8x16(); - let b = b.as_i8x16(); - macro_rules! call { - ($imm8:expr) => { - pcmpestrm128(a, la, b, lb, $imm8) - }; - } - transmute(constify_imm8!(imm8, call)) +#[cfg_attr(test, assert_instr(pcmpestrm, IMM8 = 0))] +#[rustc_legacy_const_generics(4)] +#[stable(feature = "simd_x86", since = "1.27.0")] +pub unsafe fn _mm_cmpestrm(a: __m128i, la: i32, b: __m128i, lb: i32) -> __m128i { + static_assert_imm8!(IMM8); + transmute(pcmpestrm128(a.as_i8x16(), la, b.as_i8x16(), lb, IMM8 as i8)) } /// Compares packed strings `a` and `b` with lengths `la` and `lb` using the -/// control in `imm8` and return the generated index. Similar to +/// control in `IMM8` and return the generated index. Similar to /// [`_mm_cmpistri`] with the exception that [`_mm_cmpistri`] implicitly /// determines the length of `a` and `b`. /// /// # Control modes /// -/// The control specified by `imm8` may be one or more of the following. +/// The control specified by `IMM8` may be one or more of the following. /// /// ## Data size and signedness /// @@ -483,124 +435,88 @@ pub unsafe fn _mm_cmpestrm(a: __m128i, la: i32, b: __m128i, lb: i32, imm8: i32) /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cmpestri) #[inline] #[target_feature(enable = "sse4.2")] -#[cfg_attr(test, assert_instr(pcmpestri, imm8 = 0))] -#[rustc_args_required_const(4)] -#[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm_cmpestri(a: __m128i, la: i32, b: __m128i, lb: i32, imm8: i32) -> i32 { - let a = a.as_i8x16(); - let b = b.as_i8x16(); - macro_rules! call { - ($imm8:expr) => { - pcmpestri128(a, la, b, lb, $imm8) - }; - } - constify_imm8!(imm8, call) +#[cfg_attr(test, assert_instr(pcmpestri, IMM8 = 0))] +#[rustc_legacy_const_generics(4)] +#[stable(feature = "simd_x86", since = "1.27.0")] +pub unsafe fn _mm_cmpestri(a: __m128i, la: i32, b: __m128i, lb: i32) -> i32 { + static_assert_imm8!(IMM8); + pcmpestri128(a.as_i8x16(), la, b.as_i8x16(), lb, IMM8 as i8) } /// Compares packed strings in `a` and `b` with lengths `la` and `lb` -/// using the control in `imm8`, and return `1` if any character in +/// using the control in `IMM8`, and return `1` if any character in /// `b` was null, and `0` otherwise. /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cmpestrz) #[inline] #[target_feature(enable = "sse4.2")] -#[cfg_attr(test, assert_instr(pcmpestri, imm8 = 0))] -#[rustc_args_required_const(4)] -#[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm_cmpestrz(a: __m128i, la: i32, b: __m128i, lb: i32, imm8: i32) -> i32 { - let a = a.as_i8x16(); - let b = b.as_i8x16(); - macro_rules! call { - ($imm8:expr) => { - pcmpestriz128(a, la, b, lb, $imm8) - }; - } - constify_imm8!(imm8, call) +#[cfg_attr(test, assert_instr(pcmpestri, IMM8 = 0))] +#[rustc_legacy_const_generics(4)] +#[stable(feature = "simd_x86", since = "1.27.0")] +pub unsafe fn _mm_cmpestrz(a: __m128i, la: i32, b: __m128i, lb: i32) -> i32 { + static_assert_imm8!(IMM8); + pcmpestriz128(a.as_i8x16(), la, b.as_i8x16(), lb, IMM8 as i8) } /// Compares packed strings in `a` and `b` with lengths `la` and `lb` -/// using the control in `imm8`, and return `1` if the resulting mask +/// using the control in `IMM8`, and return `1` if the resulting mask /// was non-zero, and `0` otherwise. /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cmpestrc) #[inline] #[target_feature(enable = "sse4.2")] -#[cfg_attr(test, assert_instr(pcmpestri, imm8 = 0))] -#[rustc_args_required_const(4)] -#[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm_cmpestrc(a: __m128i, la: i32, b: __m128i, lb: i32, imm8: i32) -> i32 { - let a = a.as_i8x16(); - let b = b.as_i8x16(); - macro_rules! call { - ($imm8:expr) => { - pcmpestric128(a, la, b, lb, $imm8) - }; - } - constify_imm8!(imm8, call) +#[cfg_attr(test, assert_instr(pcmpestri, IMM8 = 0))] +#[rustc_legacy_const_generics(4)] +#[stable(feature = "simd_x86", since = "1.27.0")] +pub unsafe fn _mm_cmpestrc(a: __m128i, la: i32, b: __m128i, lb: i32) -> i32 { + static_assert_imm8!(IMM8); + pcmpestric128(a.as_i8x16(), la, b.as_i8x16(), lb, IMM8 as i8) } /// Compares packed strings in `a` and `b` with lengths `la` and `lb` -/// using the control in `imm8`, and return `1` if any character in +/// using the control in `IMM8`, and return `1` if any character in /// a was null, and `0` otherwise. /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cmpestrs) #[inline] #[target_feature(enable = "sse4.2")] -#[cfg_attr(test, assert_instr(pcmpestri, imm8 = 0))] -#[rustc_args_required_const(4)] -#[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm_cmpestrs(a: __m128i, la: i32, b: __m128i, lb: i32, imm8: i32) -> i32 { - let a = a.as_i8x16(); - let b = b.as_i8x16(); - macro_rules! call { - ($imm8:expr) => { - pcmpestris128(a, la, b, lb, $imm8) - }; - } - constify_imm8!(imm8, call) +#[cfg_attr(test, assert_instr(pcmpestri, IMM8 = 0))] +#[rustc_legacy_const_generics(4)] +#[stable(feature = "simd_x86", since = "1.27.0")] +pub unsafe fn _mm_cmpestrs(a: __m128i, la: i32, b: __m128i, lb: i32) -> i32 { + static_assert_imm8!(IMM8); + pcmpestris128(a.as_i8x16(), la, b.as_i8x16(), lb, IMM8 as i8) } /// Compares packed strings in `a` and `b` with lengths `la` and `lb` -/// using the control in `imm8`, and return bit `0` of the resulting +/// using the control in `IMM8`, and return bit `0` of the resulting /// bit mask. /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cmpestro) #[inline] #[target_feature(enable = "sse4.2")] -#[cfg_attr(test, assert_instr(pcmpestri, imm8 = 0))] -#[rustc_args_required_const(4)] -#[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm_cmpestro(a: __m128i, la: i32, b: __m128i, lb: i32, imm8: i32) -> i32 { - let a = a.as_i8x16(); - let b = b.as_i8x16(); - macro_rules! call { - ($imm8:expr) => { - pcmpestrio128(a, la, b, lb, $imm8) - }; - } - constify_imm8!(imm8, call) +#[cfg_attr(test, assert_instr(pcmpestri, IMM8 = 0))] +#[rustc_legacy_const_generics(4)] +#[stable(feature = "simd_x86", since = "1.27.0")] +pub unsafe fn _mm_cmpestro(a: __m128i, la: i32, b: __m128i, lb: i32) -> i32 { + static_assert_imm8!(IMM8); + pcmpestrio128(a.as_i8x16(), la, b.as_i8x16(), lb, IMM8 as i8) } /// Compares packed strings in `a` and `b` with lengths `la` and `lb` -/// using the control in `imm8`, and return `1` if `b` did not +/// using the control in `IMM8`, and return `1` if `b` did not /// contain a null character and the resulting mask was zero, and `0` /// otherwise. /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cmpestra) #[inline] #[target_feature(enable = "sse4.2")] -#[cfg_attr(test, assert_instr(pcmpestri, imm8 = 0))] -#[rustc_args_required_const(4)] -#[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm_cmpestra(a: __m128i, la: i32, b: __m128i, lb: i32, imm8: i32) -> i32 { - let a = a.as_i8x16(); - let b = b.as_i8x16(); - macro_rules! call { - ($imm8:expr) => { - pcmpestria128(a, la, b, lb, $imm8) - }; - } - constify_imm8!(imm8, call) +#[cfg_attr(test, assert_instr(pcmpestri, IMM8 = 0))] +#[rustc_legacy_const_generics(4)] +#[stable(feature = "simd_x86", since = "1.27.0")] +pub unsafe fn _mm_cmpestra(a: __m128i, la: i32, b: __m128i, lb: i32) -> i32 { + static_assert_imm8!(IMM8); + pcmpestria128(a.as_i8x16(), la, b.as_i8x16(), lb, IMM8 as i8) } /// Starting with the initial value in `crc`, return the accumulated @@ -719,7 +635,7 @@ mod tests { unsafe fn test_mm_cmpistrm() { let a = str_to_m128i(b"Hello! Good-Bye!"); let b = str_to_m128i(b"hello! good-bye!"); - let i = _mm_cmpistrm(a, b, _SIDD_UNIT_MASK); + let i = _mm_cmpistrm::<_SIDD_UNIT_MASK>(a, b); #[rustfmt::skip] let res = _mm_setr_epi8( 0x00, !0, !0, !0, !0, !0, !0, 0x00, @@ -732,7 +648,7 @@ mod tests { unsafe fn test_mm_cmpistri() { let a = str_to_m128i(b"Hello"); let b = str_to_m128i(b" Hello "); - let i = _mm_cmpistri(a, b, _SIDD_CMP_EQUAL_ORDERED); + let i = _mm_cmpistri::<_SIDD_CMP_EQUAL_ORDERED>(a, b); assert_eq!(3, i); } @@ -740,7 +656,7 @@ mod tests { unsafe fn test_mm_cmpistrz() { let a = str_to_m128i(b""); let b = str_to_m128i(b"Hello"); - let i = _mm_cmpistrz(a, b, _SIDD_CMP_EQUAL_ORDERED); + let i = _mm_cmpistrz::<_SIDD_CMP_EQUAL_ORDERED>(a, b); assert_eq!(1, i); } @@ -748,7 +664,7 @@ mod tests { unsafe fn test_mm_cmpistrc() { let a = str_to_m128i(b" "); let b = str_to_m128i(b" ! "); - let i = _mm_cmpistrc(a, b, _SIDD_UNIT_MASK); + let i = _mm_cmpistrc::<_SIDD_UNIT_MASK>(a, b); assert_eq!(1, i); } @@ -756,7 +672,7 @@ mod tests { unsafe fn test_mm_cmpistrs() { let a = str_to_m128i(b"Hello"); let b = str_to_m128i(b""); - let i = _mm_cmpistrs(a, b, _SIDD_CMP_EQUAL_ORDERED); + let i = _mm_cmpistrs::<_SIDD_CMP_EQUAL_ORDERED>(a, b); assert_eq!(1, i); } @@ -774,7 +690,7 @@ mod tests { ); let a = a_bytes; let b = b_bytes; - let i = _mm_cmpistro(a, b, _SIDD_UWORD_OPS | _SIDD_UNIT_MASK); + let i = _mm_cmpistro::<{ _SIDD_UWORD_OPS | _SIDD_UNIT_MASK }>(a, b); assert_eq!(0, i); } @@ -782,7 +698,7 @@ mod tests { unsafe fn test_mm_cmpistra() { let a = str_to_m128i(b""); let b = str_to_m128i(b"Hello!!!!!!!!!!!"); - let i = _mm_cmpistra(a, b, _SIDD_UNIT_MASK); + let i = _mm_cmpistra::<_SIDD_UNIT_MASK>(a, b); assert_eq!(1, i); } @@ -790,7 +706,7 @@ mod tests { unsafe fn test_mm_cmpestrm() { let a = str_to_m128i(b"Hello!"); let b = str_to_m128i(b"Hello."); - let i = _mm_cmpestrm(a, 5, b, 5, _SIDD_UNIT_MASK); + let i = _mm_cmpestrm::<_SIDD_UNIT_MASK>(a, 5, b, 5); #[rustfmt::skip] let r = _mm_setr_epi8( !0, !0, !0, !0, !0, 0x00, 0x00, 0x00, @@ -803,7 +719,7 @@ mod tests { unsafe fn test_mm_cmpestri() { let a = str_to_m128i(b"bar - garbage"); let b = str_to_m128i(b"foobar"); - let i = _mm_cmpestri(a, 3, b, 6, _SIDD_CMP_EQUAL_ORDERED); + let i = _mm_cmpestri::<_SIDD_CMP_EQUAL_ORDERED>(a, 3, b, 6); assert_eq!(3, i); } @@ -811,7 +727,7 @@ mod tests { unsafe fn test_mm_cmpestrz() { let a = str_to_m128i(b""); let b = str_to_m128i(b"Hello"); - let i = _mm_cmpestrz(a, 16, b, 6, _SIDD_CMP_EQUAL_ORDERED); + let i = _mm_cmpestrz::<_SIDD_CMP_EQUAL_ORDERED>(a, 16, b, 6); assert_eq!(1, i); } @@ -819,7 +735,7 @@ mod tests { unsafe fn test_mm_cmpestrc() { let va = str_to_m128i(b"!!!!!!!!"); let vb = str_to_m128i(b" "); - let i = _mm_cmpestrc(va, 7, vb, 7, _SIDD_UNIT_MASK); + let i = _mm_cmpestrc::<_SIDD_UNIT_MASK>(va, 7, vb, 7); assert_eq!(0, i); } @@ -832,7 +748,7 @@ mod tests { ); let a = a_bytes; let b = _mm_set1_epi8(0x00); - let i = _mm_cmpestrs(a, 8, b, 0, _SIDD_UWORD_OPS); + let i = _mm_cmpestrs::<_SIDD_UWORD_OPS>(a, 8, b, 0); assert_eq!(0, i); } @@ -840,7 +756,7 @@ mod tests { unsafe fn test_mm_cmpestro() { let a = str_to_m128i(b"Hello"); let b = str_to_m128i(b"World"); - let i = _mm_cmpestro(a, 5, b, 5, _SIDD_UBYTE_OPS); + let i = _mm_cmpestro::<_SIDD_UBYTE_OPS>(a, 5, b, 5); assert_eq!(0, i); } @@ -848,7 +764,7 @@ mod tests { unsafe fn test_mm_cmpestra() { let a = str_to_m128i(b"Cannot match a"); let b = str_to_m128i(b"Null after 14"); - let i = _mm_cmpestra(a, 14, b, 16, _SIDD_CMP_EQUAL_EACH | _SIDD_UNIT_MASK); + let i = _mm_cmpestra::<{ _SIDD_CMP_EQUAL_EACH | _SIDD_UNIT_MASK }>(a, 14, b, 16); assert_eq!(1, i); } diff --git a/library/stdarch/crates/core_arch/src/x86/ssse3.rs b/library/stdarch/crates/core_arch/src/x86/ssse3.rs index 669f51ce7b..4beb496b6b 100644 --- a/library/stdarch/crates/core_arch/src/x86/ssse3.rs +++ b/library/stdarch/crates/core_arch/src/x86/ssse3.rs @@ -86,71 +86,55 @@ pub unsafe fn _mm_shuffle_epi8(a: __m128i, b: __m128i) -> __m128i { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_alignr_epi8) #[inline] #[target_feature(enable = "ssse3")] -#[cfg_attr(test, assert_instr(palignr, n = 15))] -#[rustc_args_required_const(2)] +#[cfg_attr(test, assert_instr(palignr, IMM8 = 15))] +#[rustc_legacy_const_generics(2)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm_alignr_epi8(a: __m128i, b: __m128i, n: i32) -> __m128i { - let n = n as u32; +pub unsafe fn _mm_alignr_epi8(a: __m128i, b: __m128i) -> __m128i { + static_assert_imm8!(IMM8); // If palignr is shifting the pair of vectors more than the size of two // lanes, emit zero. - if n > 32 { + if IMM8 > 32 { return _mm_set1_epi8(0); } // If palignr is shifting the pair of input vectors more than one lane, // but less than two lanes, convert to shifting in zeroes. - let (a, b, n) = if n > 16 { - (_mm_set1_epi8(0), a, n - 16) + let (a, b) = if IMM8 > 16 { + (_mm_set1_epi8(0), a) } else { - (a, b, n) + (a, b) }; - let a = a.as_i8x16(); - let b = b.as_i8x16(); - - macro_rules! shuffle { - ($shift:expr) => { - simd_shuffle16( - b, - a, - [ - 0 + $shift, - 1 + $shift, - 2 + $shift, - 3 + $shift, - 4 + $shift, - 5 + $shift, - 6 + $shift, - 7 + $shift, - 8 + $shift, - 9 + $shift, - 10 + $shift, - 11 + $shift, - 12 + $shift, - 13 + $shift, - 14 + $shift, - 15 + $shift, - ], - ) - }; + const fn mask(shift: u32, i: u32) -> u32 { + if shift > 32 { + // Unused, but needs to be a valid index. + i + } else if shift > 16 { + shift - 16 + i + } else { + shift + i + } } - let r: i8x16 = match n { - 0 => shuffle!(0), - 1 => shuffle!(1), - 2 => shuffle!(2), - 3 => shuffle!(3), - 4 => shuffle!(4), - 5 => shuffle!(5), - 6 => shuffle!(6), - 7 => shuffle!(7), - 8 => shuffle!(8), - 9 => shuffle!(9), - 10 => shuffle!(10), - 11 => shuffle!(11), - 12 => shuffle!(12), - 13 => shuffle!(13), - 14 => shuffle!(14), - 15 => shuffle!(15), - _ => shuffle!(16), - }; + let r: i8x16 = simd_shuffle16!( + b.as_i8x16(), + a.as_i8x16(), + [ + mask(IMM8 as u32, 0), + mask(IMM8 as u32, 1), + mask(IMM8 as u32, 2), + mask(IMM8 as u32, 3), + mask(IMM8 as u32, 4), + mask(IMM8 as u32, 5), + mask(IMM8 as u32, 6), + mask(IMM8 as u32, 7), + mask(IMM8 as u32, 8), + mask(IMM8 as u32, 9), + mask(IMM8 as u32, 10), + mask(IMM8 as u32, 11), + mask(IMM8 as u32, 12), + mask(IMM8 as u32, 13), + mask(IMM8 as u32, 14), + mask(IMM8 as u32, 15), + ], + ); transmute(r) } @@ -404,10 +388,10 @@ mod tests { 12, 5, 5, 10, 4, 1, 8, 0, ); - let r = _mm_alignr_epi8(a, b, 33); + let r = _mm_alignr_epi8::<33>(a, b); assert_eq_m128i(r, _mm_set1_epi8(0)); - let r = _mm_alignr_epi8(a, b, 17); + let r = _mm_alignr_epi8::<17>(a, b); #[rustfmt::skip] let expected = _mm_setr_epi8( 2, 3, 4, 5, 6, 7, 8, 9, @@ -415,10 +399,10 @@ mod tests { ); assert_eq_m128i(r, expected); - let r = _mm_alignr_epi8(a, b, 16); + let r = _mm_alignr_epi8::<16>(a, b); assert_eq_m128i(r, a); - let r = _mm_alignr_epi8(a, b, 15); + let r = _mm_alignr_epi8::<15>(a, b); #[rustfmt::skip] let expected = _mm_setr_epi8( 0, 1, 2, 3, 4, 5, 6, 7, @@ -426,7 +410,7 @@ mod tests { ); assert_eq_m128i(r, expected); - let r = _mm_alignr_epi8(a, b, 0); + let r = _mm_alignr_epi8::<0>(a, b); assert_eq_m128i(r, b); } diff --git a/library/stdarch/crates/core_arch/src/x86/test.rs b/library/stdarch/crates/core_arch/src/x86/test.rs index c1f974133c..bab89e61ac 100644 --- a/library/stdarch/crates/core_arch/src/x86/test.rs +++ b/library/stdarch/crates/core_arch/src/x86/test.rs @@ -47,7 +47,7 @@ pub unsafe fn assert_eq_m256i(a: __m256i, b: __m256i) { #[target_feature(enable = "avx")] pub unsafe fn assert_eq_m256d(a: __m256d, b: __m256d) { - let cmp = _mm256_cmp_pd(a, b, _CMP_EQ_OQ); + let cmp = _mm256_cmp_pd::<_CMP_EQ_OQ>(a, b); if _mm256_movemask_pd(cmp) != 0b1111 { panic!("{:?} != {:?}", a, b); } @@ -60,7 +60,7 @@ pub unsafe fn get_m256d(a: __m256d, idx: usize) -> f64 { #[target_feature(enable = "avx")] pub unsafe fn assert_eq_m256(a: __m256, b: __m256) { - let cmp = _mm256_cmp_ps(a, b, _CMP_EQ_OQ); + let cmp = _mm256_cmp_ps::<_CMP_EQ_OQ>(a, b); if _mm256_movemask_ps(cmp) != 0b11111111 { panic!("{:?} != {:?}", a, b); } @@ -92,26 +92,30 @@ pub unsafe fn get_m512i(a: __m512i, idx: usize) -> i64 { mod x86_polyfill { use crate::core_arch::x86::*; - pub unsafe fn _mm_insert_epi64(a: __m128i, val: i64, idx: i32) -> __m128i { + #[rustc_legacy_const_generics(2)] + pub unsafe fn _mm_insert_epi64(a: __m128i, val: i64) -> __m128i { + static_assert_imm1!(INDEX); #[repr(C)] union A { a: __m128i, b: [i64; 2], } let mut a = A { a }; - a.b[idx as usize] = val; + a.b[INDEX as usize] = val; a.a } #[target_feature(enable = "avx2")] - pub unsafe fn _mm256_insert_epi64(a: __m256i, val: i64, idx: i32) -> __m256i { + #[rustc_legacy_const_generics(2)] + pub unsafe fn _mm256_insert_epi64(a: __m256i, val: i64) -> __m256i { + static_assert_imm2!(INDEX); #[repr(C)] union A { a: __m256i, b: [i64; 4], } let mut a = A { a }; - a.b[idx as usize] = val; + a.b[INDEX as usize] = val; a.a } } @@ -126,14 +130,14 @@ pub unsafe fn assert_eq_m512i(a: __m512i, b: __m512i) { } pub unsafe fn assert_eq_m512(a: __m512, b: __m512) { - let cmp = _mm512_cmp_ps_mask(a, b, _CMP_EQ_OQ); + let cmp = _mm512_cmp_ps_mask::<_CMP_EQ_OQ>(a, b); if cmp != 0b11111111_11111111 { panic!("{:?} != {:?}", a, b); } } pub unsafe fn assert_eq_m512d(a: __m512d, b: __m512d) { - let cmp = _mm512_cmp_pd_mask(a, b, _CMP_EQ_OQ); + let cmp = _mm512_cmp_pd_mask::<_CMP_EQ_OQ>(a, b); if cmp != 0b11111111 { panic!("{:?} != {:?}", a, b); } diff --git a/library/stdarch/crates/core_arch/src/x86_64/avx.rs b/library/stdarch/crates/core_arch/src/x86_64/avx.rs index fd82367714..7ba26371c6 100644 --- a/library/stdarch/crates/core_arch/src/x86_64/avx.rs +++ b/library/stdarch/crates/core_arch/src/x86_64/avx.rs @@ -23,18 +23,13 @@ use crate::{ /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_insert_epi64) #[inline] -#[rustc_args_required_const(2)] +#[rustc_legacy_const_generics(2)] #[target_feature(enable = "avx")] // This intrinsic has no corresponding instruction. #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm256_insert_epi64(a: __m256i, i: i64, index: i32) -> __m256i { - let a = a.as_i64x4(); - match index & 3 { - 0 => transmute(simd_insert(a, 0, i)), - 1 => transmute(simd_insert(a, 1, i)), - 2 => transmute(simd_insert(a, 2, i)), - _ => transmute(simd_insert(a, 3, i)), - } +pub unsafe fn _mm256_insert_epi64(a: __m256i, i: i64) -> __m256i { + static_assert_imm2!(INDEX); + transmute(simd_insert(a.as_i64x4(), INDEX as u32, i)) } #[cfg(test)] @@ -46,7 +41,7 @@ mod tests { #[simd_test(enable = "avx")] unsafe fn test_mm256_insert_epi64() { let a = _mm256_setr_epi64x(1, 2, 3, 4); - let r = _mm256_insert_epi64(a, 0, 3); + let r = _mm256_insert_epi64::<3>(a, 0); let e = _mm256_setr_epi64x(1, 2, 3, 0); assert_eq_m256i(r, e); } diff --git a/library/stdarch/crates/core_arch/src/x86_64/avx2.rs b/library/stdarch/crates/core_arch/src/x86_64/avx2.rs index 0f81cd221f..14447a1371 100644 --- a/library/stdarch/crates/core_arch/src/x86_64/avx2.rs +++ b/library/stdarch/crates/core_arch/src/x86_64/avx2.rs @@ -20,22 +20,17 @@ use crate::core_arch::{simd_llvm::*, x86::*}; -/// Extracts a 64-bit integer from `a`, selected with `imm8`. +/// Extracts a 64-bit integer from `a`, selected with `INDEX`. /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_extract_epi64) #[inline] #[target_feature(enable = "avx2")] -#[rustc_args_required_const(1)] +#[rustc_legacy_const_generics(1)] // This intrinsic has no corresponding instruction. #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm256_extract_epi64(a: __m256i, imm8: i32) -> i64 { - let a = a.as_i64x4(); - match imm8 & 3 { - 0 => simd_extract(a, 0), - 1 => simd_extract(a, 1), - 2 => simd_extract(a, 2), - _ => simd_extract(a, 3), - } +pub unsafe fn _mm256_extract_epi64(a: __m256i) -> i64 { + static_assert_imm2!(INDEX); + simd_extract(a.as_i64x4(), INDEX as u32) } #[cfg(test)] @@ -46,7 +41,7 @@ mod tests { #[simd_test(enable = "avx2")] unsafe fn test_mm256_extract_epi64() { let a = _mm256_setr_epi64x(0, 1, 2, 3); - let r = _mm256_extract_epi64(a, 3); + let r = _mm256_extract_epi64::<3>(a); assert_eq!(r, 3); } } diff --git a/library/stdarch/crates/core_arch/src/x86_64/avx512f.rs b/library/stdarch/crates/core_arch/src/x86_64/avx512f.rs index cf1b4b6220..471977821c 100644 --- a/library/stdarch/crates/core_arch/src/x86_64/avx512f.rs +++ b/library/stdarch/crates/core_arch/src/x86_64/avx512f.rs @@ -145,16 +145,12 @@ pub unsafe fn _mm_cvttss_u64(a: __m128) -> u64 { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cvt_roundi64_sd&expand=1313) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vcvtsi2sd, rounding = 8))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm_cvt_roundi64_sd(a: __m128d, b: i64, rounding: i32) -> __m128d { +#[cfg_attr(test, assert_instr(vcvtsi2sd, ROUNDING = 8))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm_cvt_roundi64_sd(a: __m128d, b: i64) -> __m128d { + static_assert_rounding!(ROUNDING); let a = a.as_f64x2(); - macro_rules! call { - ($imm4:expr) => { - vcvtsi2sd64(a, b, $imm4) - }; - } - let r = constify_imm4_round!(rounding, call); + let r = vcvtsi2sd64(a, b, ROUNDING); transmute(r) } @@ -169,16 +165,12 @@ pub unsafe fn _mm_cvt_roundi64_sd(a: __m128d, b: i64, rounding: i32) -> __m128d /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cvt_roundsi64_sd&expand=1367) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vcvtsi2sd, rounding = 8))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm_cvt_roundsi64_sd(a: __m128d, b: i64, rounding: i32) -> __m128d { +#[cfg_attr(test, assert_instr(vcvtsi2sd, ROUNDING = 8))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm_cvt_roundsi64_sd(a: __m128d, b: i64) -> __m128d { + static_assert_rounding!(ROUNDING); let a = a.as_f64x2(); - macro_rules! call { - ($imm4:expr) => { - vcvtsi2sd64(a, b, $imm4) - }; - } - let r = constify_imm4_round!(rounding, call); + let r = vcvtsi2sd64(a, b, ROUNDING); transmute(r) } @@ -193,16 +185,12 @@ pub unsafe fn _mm_cvt_roundsi64_sd(a: __m128d, b: i64, rounding: i32) -> __m128d /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cvt_roundi64_ss&expand=1314) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vcvtsi2ss, rounding = 8))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm_cvt_roundi64_ss(a: __m128, b: i64, rounding: i32) -> __m128 { +#[cfg_attr(test, assert_instr(vcvtsi2ss, ROUNDING = 8))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm_cvt_roundi64_ss(a: __m128, b: i64) -> __m128 { + static_assert_rounding!(ROUNDING); let a = a.as_f32x4(); - macro_rules! call { - ($imm4:expr) => { - vcvtsi2ss64(a, b, $imm4) - }; - } - let r = constify_imm4_round!(rounding, call); + let r = vcvtsi2ss64(a, b, ROUNDING); transmute(r) } @@ -217,16 +205,12 @@ pub unsafe fn _mm_cvt_roundi64_ss(a: __m128, b: i64, rounding: i32) -> __m128 { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cvt_roundu64_sd&expand=1379) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vcvtusi2sd, rounding = 8))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm_cvt_roundu64_sd(a: __m128d, b: u64, rounding: i32) -> __m128d { +#[cfg_attr(test, assert_instr(vcvtusi2sd, ROUNDING = 8))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm_cvt_roundu64_sd(a: __m128d, b: u64) -> __m128d { + static_assert_rounding!(ROUNDING); let a = a.as_f64x2(); - macro_rules! call { - ($imm4:expr) => { - vcvtusi2sd64(a, b, $imm4) - }; - } - let r = constify_imm4_round!(rounding, call); + let r = vcvtusi2sd64(a, b, ROUNDING); transmute(r) } @@ -241,16 +225,12 @@ pub unsafe fn _mm_cvt_roundu64_sd(a: __m128d, b: u64, rounding: i32) -> __m128d /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cvt_roundsi64_ss&expand=1368) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vcvtsi2ss, rounding = 8))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm_cvt_roundsi64_ss(a: __m128, b: i64, rounding: i32) -> __m128 { +#[cfg_attr(test, assert_instr(vcvtsi2ss, ROUNDING = 8))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm_cvt_roundsi64_ss(a: __m128, b: i64) -> __m128 { + static_assert_rounding!(ROUNDING); let a = a.as_f32x4(); - macro_rules! call { - ($imm4:expr) => { - vcvtsi2ss64(a, b, $imm4) - }; - } - let r = constify_imm4_round!(rounding, call); + let r = vcvtsi2ss64(a, b, ROUNDING); transmute(r) } @@ -265,16 +245,12 @@ pub unsafe fn _mm_cvt_roundsi64_ss(a: __m128, b: i64, rounding: i32) -> __m128 { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cvt_roundu64_ss&expand=1380) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vcvtusi2ss, rounding = 8))] -#[rustc_args_required_const(2)] -pub unsafe fn _mm_cvt_roundu64_ss(a: __m128, b: u64, rounding: i32) -> __m128 { +#[cfg_attr(test, assert_instr(vcvtusi2ss, ROUNDING = 8))] +#[rustc_legacy_const_generics(2)] +pub unsafe fn _mm_cvt_roundu64_ss(a: __m128, b: u64) -> __m128 { + static_assert_rounding!(ROUNDING); let a = a.as_f32x4(); - macro_rules! call { - ($imm4:expr) => { - vcvtusi2ss64(a, b, $imm4) - }; - } - let r = constify_imm4_round!(rounding, call); + let r = vcvtusi2ss64(a, b, ROUNDING); transmute(r) } @@ -289,16 +265,12 @@ pub unsafe fn _mm_cvt_roundu64_ss(a: __m128, b: u64, rounding: i32) -> __m128 { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cvt_roundsd_si64&expand=1360) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vcvtsd2si, rounding = 8))] -#[rustc_args_required_const(1)] -pub unsafe fn _mm_cvt_roundsd_si64(a: __m128d, rounding: i32) -> i64 { +#[cfg_attr(test, assert_instr(vcvtsd2si, ROUNDING = 8))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm_cvt_roundsd_si64(a: __m128d) -> i64 { + static_assert_rounding!(ROUNDING); let a = a.as_f64x2(); - macro_rules! call { - ($imm4:expr) => { - vcvtsd2si64(a, $imm4) - }; - } - let r = constify_imm4_round!(rounding, call); + let r = vcvtsd2si64(a, ROUNDING); transmute(r) } @@ -313,16 +285,12 @@ pub unsafe fn _mm_cvt_roundsd_si64(a: __m128d, rounding: i32) -> i64 { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cvt_roundsd_i64&expand=1358) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vcvtsd2si, rounding = 8))] -#[rustc_args_required_const(1)] -pub unsafe fn _mm_cvt_roundsd_i64(a: __m128d, rounding: i32) -> i64 { +#[cfg_attr(test, assert_instr(vcvtsd2si, ROUNDING = 8))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm_cvt_roundsd_i64(a: __m128d) -> i64 { + static_assert_rounding!(ROUNDING); let a = a.as_f64x2(); - macro_rules! call { - ($imm4:expr) => { - vcvtsd2si64(a, $imm4) - }; - } - let r = constify_imm4_round!(rounding, call); + let r = vcvtsd2si64(a, ROUNDING); transmute(r) } @@ -337,16 +305,12 @@ pub unsafe fn _mm_cvt_roundsd_i64(a: __m128d, rounding: i32) -> i64 { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cvt_roundsd_u64&expand=1365) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vcvtsd2usi, rounding = 8))] -#[rustc_args_required_const(1)] -pub unsafe fn _mm_cvt_roundsd_u64(a: __m128d, rounding: i32) -> u64 { +#[cfg_attr(test, assert_instr(vcvtsd2usi, ROUNDING = 8))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm_cvt_roundsd_u64(a: __m128d) -> u64 { + static_assert_rounding!(ROUNDING); let a = a.as_f64x2(); - macro_rules! call { - ($imm4:expr) => { - vcvtsd2usi64(a, $imm4) - }; - } - let r = constify_imm4_round!(rounding, call); + let r = vcvtsd2usi64(a, ROUNDING); transmute(r) } @@ -361,16 +325,12 @@ pub unsafe fn _mm_cvt_roundsd_u64(a: __m128d, rounding: i32) -> u64 { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cvt_roundss_si64&expand=1375) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vcvtss2si, rounding = 8))] -#[rustc_args_required_const(1)] -pub unsafe fn _mm_cvt_roundss_si64(a: __m128, rounding: i32) -> i64 { +#[cfg_attr(test, assert_instr(vcvtss2si, ROUNDING = 8))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm_cvt_roundss_si64(a: __m128) -> i64 { + static_assert_rounding!(ROUNDING); let a = a.as_f32x4(); - macro_rules! call { - ($imm4:expr) => { - vcvtss2si64(a, $imm4) - }; - } - let r = constify_imm4_round!(rounding, call); + let r = vcvtss2si64(a, ROUNDING); transmute(r) } @@ -385,16 +345,12 @@ pub unsafe fn _mm_cvt_roundss_si64(a: __m128, rounding: i32) -> i64 { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cvt_roundss_i64&expand=1370) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vcvtss2si, rounding = 8))] -#[rustc_args_required_const(1)] -pub unsafe fn _mm_cvt_roundss_i64(a: __m128, rounding: i32) -> i64 { +#[cfg_attr(test, assert_instr(vcvtss2si, ROUNDING = 8))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm_cvt_roundss_i64(a: __m128) -> i64 { + static_assert_rounding!(ROUNDING); let a = a.as_f32x4(); - macro_rules! call { - ($imm4:expr) => { - vcvtss2si64(a, $imm4) - }; - } - let r = constify_imm4_round!(rounding, call); + let r = vcvtss2si64(a, ROUNDING); transmute(r) } @@ -409,16 +365,12 @@ pub unsafe fn _mm_cvt_roundss_i64(a: __m128, rounding: i32) -> i64 { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cvt_roundss_u64&expand=1377) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vcvtss2usi, rounding = 8))] -#[rustc_args_required_const(1)] -pub unsafe fn _mm_cvt_roundss_u64(a: __m128, rounding: i32) -> u64 { +#[cfg_attr(test, assert_instr(vcvtss2usi, ROUNDING = 8))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm_cvt_roundss_u64(a: __m128) -> u64 { + static_assert_rounding!(ROUNDING); let a = a.as_f32x4(); - macro_rules! call { - ($imm4:expr) => { - vcvtss2usi64(a, $imm4) - }; - } - let r = constify_imm4_round!(rounding, call); + let r = vcvtss2usi64(a, ROUNDING); transmute(r) } @@ -428,16 +380,12 @@ pub unsafe fn _mm_cvt_roundss_u64(a: __m128, rounding: i32) -> u64 { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cvtt_roundsd_si64&expand=1931) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vcvtsd2si, sae = 8))] -#[rustc_args_required_const(1)] -pub unsafe fn _mm_cvtt_roundsd_si64(a: __m128d, sae: i32) -> i64 { +#[cfg_attr(test, assert_instr(vcvtsd2si, SAE = 8))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm_cvtt_roundsd_si64(a: __m128d) -> i64 { + static_assert_sae!(SAE); let a = a.as_f64x2(); - macro_rules! call { - ($imm4:expr) => { - vcvtsd2si64(a, $imm4) - }; - } - let r = constify_imm4_sae!(sae, call); + let r = vcvtsd2si64(a, SAE); transmute(r) } @@ -447,16 +395,12 @@ pub unsafe fn _mm_cvtt_roundsd_si64(a: __m128d, sae: i32) -> i64 { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cvtt_roundsd_i64&expand=1929) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vcvtsd2si, sae = 8))] -#[rustc_args_required_const(1)] -pub unsafe fn _mm_cvtt_roundsd_i64(a: __m128d, sae: i32) -> i64 { +#[cfg_attr(test, assert_instr(vcvtsd2si, SAE = 8))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm_cvtt_roundsd_i64(a: __m128d) -> i64 { + static_assert_sae!(SAE); let a = a.as_f64x2(); - macro_rules! call { - ($imm4:expr) => { - vcvtsd2si64(a, $imm4) - }; - } - let r = constify_imm4_sae!(sae, call); + let r = vcvtsd2si64(a, SAE); transmute(r) } @@ -466,16 +410,12 @@ pub unsafe fn _mm_cvtt_roundsd_i64(a: __m128d, sae: i32) -> i64 { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cvtt_roundsd_u64&expand=1933) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vcvtsd2usi, sae = 8))] -#[rustc_args_required_const(1)] -pub unsafe fn _mm_cvtt_roundsd_u64(a: __m128d, sae: i32) -> u64 { +#[cfg_attr(test, assert_instr(vcvtsd2usi, SAE = 8))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm_cvtt_roundsd_u64(a: __m128d) -> u64 { + static_assert_sae!(SAE); let a = a.as_f64x2(); - macro_rules! call { - ($imm4:expr) => { - vcvtsd2usi64(a, $imm4) - }; - } - let r = constify_imm4_sae!(sae, call); + let r = vcvtsd2usi64(a, SAE); transmute(r) } @@ -485,16 +425,12 @@ pub unsafe fn _mm_cvtt_roundsd_u64(a: __m128d, sae: i32) -> u64 { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cvtt_roundss_i64&expand=1935) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vcvtss2si, sae = 8))] -#[rustc_args_required_const(1)] -pub unsafe fn _mm_cvtt_roundss_i64(a: __m128, sae: i32) -> i64 { +#[cfg_attr(test, assert_instr(vcvtss2si, SAE = 8))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm_cvtt_roundss_i64(a: __m128) -> i64 { + static_assert_sae!(SAE); let a = a.as_f32x4(); - macro_rules! call { - ($imm4:expr) => { - vcvtss2si64(a, $imm4) - }; - } - let r = constify_imm4_sae!(sae, call); + let r = vcvtss2si64(a, SAE); transmute(r) } @@ -504,16 +440,12 @@ pub unsafe fn _mm_cvtt_roundss_i64(a: __m128, sae: i32) -> i64 { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cvtt_roundss_si64&expand=1937) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vcvtss2si, sae = 8))] -#[rustc_args_required_const(1)] -pub unsafe fn _mm_cvtt_roundss_si64(a: __m128, sae: i32) -> i64 { +#[cfg_attr(test, assert_instr(vcvtss2si, SAE = 8))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm_cvtt_roundss_si64(a: __m128) -> i64 { + static_assert_sae!(SAE); let a = a.as_f32x4(); - macro_rules! call { - ($imm4:expr) => { - vcvtss2si64(a, $imm4) - }; - } - let r = constify_imm4_sae!(sae, call); + let r = vcvtss2si64(a, SAE); transmute(r) } @@ -523,16 +455,12 @@ pub unsafe fn _mm_cvtt_roundss_si64(a: __m128, sae: i32) -> i64 { /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cvtt_roundss_u64&expand=1939) #[inline] #[target_feature(enable = "avx512f")] -#[cfg_attr(test, assert_instr(vcvtss2usi, sae = 8))] -#[rustc_args_required_const(1)] -pub unsafe fn _mm_cvtt_roundss_u64(a: __m128, sae: i32) -> u64 { +#[cfg_attr(test, assert_instr(vcvtss2usi, SAE = 8))] +#[rustc_legacy_const_generics(1)] +pub unsafe fn _mm_cvtt_roundss_u64(a: __m128) -> u64 { + static_assert_sae!(SAE); let a = a.as_f32x4(); - macro_rules! call { - ($imm4:expr) => { - vcvtss2usi64(a, $imm4) - }; - } - let r = constify_imm4_sae!(sae, call); + let r = vcvtss2usi64(a, SAE); transmute(r) } @@ -2920,7 +2848,7 @@ mod tests { #[simd_test(enable = "avx512f")] unsafe fn test_mm512_roundscale_pd() { let a = _mm512_set1_pd(1.1); - let r = _mm512_roundscale_pd(a, 0); + let r = _mm512_roundscale_pd::<0b00_00_00_00>(a); let e = _mm512_set1_pd(1.0); assert_eq_m512d(r, e); } @@ -2928,10 +2856,10 @@ mod tests { #[simd_test(enable = "avx512f")] unsafe fn test_mm512_mask_roundscale_pd() { let a = _mm512_set1_pd(1.1); - let r = _mm512_mask_roundscale_pd(a, 0, a, 0); + let r = _mm512_mask_roundscale_pd::<0b00_00_00_00>(a, 0, a); let e = _mm512_set1_pd(1.1); assert_eq_m512d(r, e); - let r = _mm512_mask_roundscale_pd(a, 0b11111111, a, 0); + let r = _mm512_mask_roundscale_pd::<0b00_00_00_00>(a, 0b11111111, a); let e = _mm512_set1_pd(1.0); assert_eq_m512d(r, e); } @@ -2939,9 +2867,9 @@ mod tests { #[simd_test(enable = "avx512f")] unsafe fn test_mm512_maskz_roundscale_pd() { let a = _mm512_set1_pd(1.1); - let r = _mm512_maskz_roundscale_pd(0, a, 0); + let r = _mm512_maskz_roundscale_pd::<0b00_00_00_00>(0, a); assert_eq_m512d(r, _mm512_setzero_pd()); - let r = _mm512_maskz_roundscale_pd(0b11111111, a, 0); + let r = _mm512_maskz_roundscale_pd::<0b00_00_00_00>(0b11111111, a); let e = _mm512_set1_pd(1.0); assert_eq_m512d(r, e); } @@ -2949,7 +2877,7 @@ mod tests { #[simd_test(enable = "avx512f,avx512vl")] unsafe fn test_mm256_roundscale_pd() { let a = _mm256_set1_pd(1.1); - let r = _mm256_roundscale_pd(a, 0); + let r = _mm256_roundscale_pd::<0b00_00_00_00>(a); let e = _mm256_set1_pd(1.0); assert_eq_m256d(r, e); } @@ -2957,10 +2885,9 @@ mod tests { #[simd_test(enable = "avx512f,avx512vl")] unsafe fn test_mm256_mask_roundscale_pd() { let a = _mm256_set1_pd(1.1); - let r = _mm256_mask_roundscale_pd(a, 0, a, 0); - let e = _mm256_set1_pd(1.1); - assert_eq_m256d(r, e); - let r = _mm256_mask_roundscale_pd(a, 0b00001111, a, 0); + let r = _mm256_mask_roundscale_pd::<0b00_00_00_00>(a, 0, a); + assert_eq_m256d(r, a); + let r = _mm256_mask_roundscale_pd::<0b00_00_00_00>(a, 0b00001111, a); let e = _mm256_set1_pd(1.0); assert_eq_m256d(r, e); } @@ -2968,9 +2895,9 @@ mod tests { #[simd_test(enable = "avx512f,avx512vl")] unsafe fn test_mm256_maskz_roundscale_pd() { let a = _mm256_set1_pd(1.1); - let r = _mm256_maskz_roundscale_pd(0, a, 0); + let r = _mm256_maskz_roundscale_pd::<0b00_00_00_00>(0, a); assert_eq_m256d(r, _mm256_setzero_pd()); - let r = _mm256_maskz_roundscale_pd(0b00001111, a, 0); + let r = _mm256_maskz_roundscale_pd::<0b00_00_00_00>(0b00001111, a); let e = _mm256_set1_pd(1.0); assert_eq_m256d(r, e); } @@ -2978,7 +2905,7 @@ mod tests { #[simd_test(enable = "avx512f,avx512vl")] unsafe fn test_mm_roundscale_pd() { let a = _mm_set1_pd(1.1); - let r = _mm_roundscale_pd(a, 0); + let r = _mm_roundscale_pd::<0b00_00_00_00>(a); let e = _mm_set1_pd(1.0); assert_eq_m128d(r, e); } @@ -2986,10 +2913,10 @@ mod tests { #[simd_test(enable = "avx512f,avx512vl")] unsafe fn test_mm_mask_roundscale_pd() { let a = _mm_set1_pd(1.1); - let r = _mm_mask_roundscale_pd(a, 0, a, 0); + let r = _mm_mask_roundscale_pd::<0b00_00_00_00>(a, 0, a); let e = _mm_set1_pd(1.1); assert_eq_m128d(r, e); - let r = _mm_mask_roundscale_pd(a, 0b00000011, a, 0); + let r = _mm_mask_roundscale_pd::<0b00_00_00_00>(a, 0b00000011, a); let e = _mm_set1_pd(1.0); assert_eq_m128d(r, e); } @@ -2997,9 +2924,9 @@ mod tests { #[simd_test(enable = "avx512f,avx512vl")] unsafe fn test_mm_maskz_roundscale_pd() { let a = _mm_set1_pd(1.1); - let r = _mm_maskz_roundscale_pd(0, a, 0); + let r = _mm_maskz_roundscale_pd::<0b00_00_00_00>(0, a); assert_eq_m128d(r, _mm_setzero_pd()); - let r = _mm_maskz_roundscale_pd(0b00000011, a, 0); + let r = _mm_maskz_roundscale_pd::<0b00_00_00_00>(0b00000011, a); let e = _mm_set1_pd(1.0); assert_eq_m128d(r, e); } @@ -3102,7 +3029,7 @@ mod tests { let a = _mm512_set1_pd(f64::NAN); let b = _mm512_set1_pd(f64::MAX); let c = _mm512_set1_epi64(i32::MAX as i64); - let r = _mm512_fixupimm_pd(a, b, c, 5); + let r = _mm512_fixupimm_pd::<5>(a, b, c); let e = _mm512_set1_pd(0.0); assert_eq_m512d(r, e); } @@ -3112,7 +3039,7 @@ mod tests { let a = _mm512_set_pd(f64::NAN, f64::NAN, f64::NAN, f64::NAN, 1., 1., 1., 1.); let b = _mm512_set1_pd(f64::MAX); let c = _mm512_set1_epi64(i32::MAX as i64); - let r = _mm512_mask_fixupimm_pd(a, 0b11110000, b, c, 5); + let r = _mm512_mask_fixupimm_pd::<5>(a, 0b11110000, b, c); let e = _mm512_set_pd(0., 0., 0., 0., 1., 1., 1., 1.); assert_eq_m512d(r, e); } @@ -3122,7 +3049,7 @@ mod tests { let a = _mm512_set_pd(f64::NAN, f64::NAN, f64::NAN, f64::NAN, 1., 1., 1., 1.); let b = _mm512_set1_pd(f64::MAX); let c = _mm512_set1_epi64(i32::MAX as i64); - let r = _mm512_maskz_fixupimm_pd(0b11110000, a, b, c, 5); + let r = _mm512_maskz_fixupimm_pd::<5>(0b11110000, a, b, c); let e = _mm512_set_pd(0., 0., 0., 0., 0., 0., 0., 0.); assert_eq_m512d(r, e); } @@ -3132,7 +3059,7 @@ mod tests { let a = _mm256_set1_pd(f64::NAN); let b = _mm256_set1_pd(f64::MAX); let c = _mm256_set1_epi64x(i32::MAX as i64); - let r = _mm256_fixupimm_pd(a, b, c, 5); + let r = _mm256_fixupimm_pd::<5>(a, b, c); let e = _mm256_set1_pd(0.0); assert_eq_m256d(r, e); } @@ -3142,7 +3069,7 @@ mod tests { let a = _mm256_set1_pd(f64::NAN); let b = _mm256_set1_pd(f64::MAX); let c = _mm256_set1_epi64x(i32::MAX as i64); - let r = _mm256_mask_fixupimm_pd(a, 0b00001111, b, c, 5); + let r = _mm256_mask_fixupimm_pd::<5>(a, 0b00001111, b, c); let e = _mm256_set1_pd(0.0); assert_eq_m256d(r, e); } @@ -3152,7 +3079,7 @@ mod tests { let a = _mm256_set1_pd(f64::NAN); let b = _mm256_set1_pd(f64::MAX); let c = _mm256_set1_epi64x(i32::MAX as i64); - let r = _mm256_maskz_fixupimm_pd(0b00001111, a, b, c, 5); + let r = _mm256_maskz_fixupimm_pd::<5>(0b00001111, a, b, c); let e = _mm256_set1_pd(0.0); assert_eq_m256d(r, e); } @@ -3162,7 +3089,7 @@ mod tests { let a = _mm_set1_pd(f64::NAN); let b = _mm_set1_pd(f64::MAX); let c = _mm_set1_epi64x(i32::MAX as i64); - let r = _mm_fixupimm_pd(a, b, c, 5); + let r = _mm_fixupimm_pd::<5>(a, b, c); let e = _mm_set1_pd(0.0); assert_eq_m128d(r, e); } @@ -3172,7 +3099,7 @@ mod tests { let a = _mm_set1_pd(f64::NAN); let b = _mm_set1_pd(f64::MAX); let c = _mm_set1_epi64x(i32::MAX as i64); - let r = _mm_mask_fixupimm_pd(a, 0b00000011, b, c, 5); + let r = _mm_mask_fixupimm_pd::<5>(a, 0b00000011, b, c); let e = _mm_set1_pd(0.0); assert_eq_m128d(r, e); } @@ -3182,7 +3109,7 @@ mod tests { let a = _mm_set1_pd(f64::NAN); let b = _mm_set1_pd(f64::MAX); let c = _mm_set1_epi64x(i32::MAX as i64); - let r = _mm_maskz_fixupimm_pd(0b00000011, a, b, c, 5); + let r = _mm_maskz_fixupimm_pd::<5>(0b00000011, a, b, c); let e = _mm_set1_pd(0.0); assert_eq_m128d(r, e); } @@ -3192,7 +3119,7 @@ mod tests { let a = _mm512_set1_epi64(1 << 2); let b = _mm512_set1_epi64(1 << 1); let c = _mm512_set1_epi64(1 << 0); - let r = _mm512_ternarylogic_epi64(a, b, c, 8); + let r = _mm512_ternarylogic_epi64::<8>(a, b, c); let e = _mm512_set1_epi64(0); assert_eq_m512i(r, e); } @@ -3202,9 +3129,9 @@ mod tests { let src = _mm512_set1_epi64(1 << 2); let a = _mm512_set1_epi64(1 << 1); let b = _mm512_set1_epi64(1 << 0); - let r = _mm512_mask_ternarylogic_epi64(src, 0, a, b, 8); + let r = _mm512_mask_ternarylogic_epi64::<8>(src, 0, a, b); assert_eq_m512i(r, src); - let r = _mm512_mask_ternarylogic_epi64(src, 0b11111111, a, b, 8); + let r = _mm512_mask_ternarylogic_epi64::<8>(src, 0b11111111, a, b); let e = _mm512_set1_epi64(0); assert_eq_m512i(r, e); } @@ -3214,9 +3141,9 @@ mod tests { let a = _mm512_set1_epi64(1 << 2); let b = _mm512_set1_epi64(1 << 1); let c = _mm512_set1_epi64(1 << 0); - let r = _mm512_maskz_ternarylogic_epi64(0, a, b, c, 9); + let r = _mm512_maskz_ternarylogic_epi64::<8>(0, a, b, c); assert_eq_m512i(r, _mm512_setzero_si512()); - let r = _mm512_maskz_ternarylogic_epi64(0b11111111, a, b, c, 8); + let r = _mm512_maskz_ternarylogic_epi64::<8>(0b11111111, a, b, c); let e = _mm512_set1_epi64(0); assert_eq_m512i(r, e); } @@ -3226,7 +3153,7 @@ mod tests { let a = _mm256_set1_epi64x(1 << 2); let b = _mm256_set1_epi64x(1 << 1); let c = _mm256_set1_epi64x(1 << 0); - let r = _mm256_ternarylogic_epi64(a, b, c, 8); + let r = _mm256_ternarylogic_epi64::<8>(a, b, c); let e = _mm256_set1_epi64x(0); assert_eq_m256i(r, e); } @@ -3236,9 +3163,9 @@ mod tests { let src = _mm256_set1_epi64x(1 << 2); let a = _mm256_set1_epi64x(1 << 1); let b = _mm256_set1_epi64x(1 << 0); - let r = _mm256_mask_ternarylogic_epi64(src, 0, a, b, 8); + let r = _mm256_mask_ternarylogic_epi64::<8>(src, 0, a, b); assert_eq_m256i(r, src); - let r = _mm256_mask_ternarylogic_epi64(src, 0b00001111, a, b, 8); + let r = _mm256_mask_ternarylogic_epi64::<8>(src, 0b00001111, a, b); let e = _mm256_set1_epi64x(0); assert_eq_m256i(r, e); } @@ -3248,9 +3175,9 @@ mod tests { let a = _mm256_set1_epi64x(1 << 2); let b = _mm256_set1_epi64x(1 << 1); let c = _mm256_set1_epi64x(1 << 0); - let r = _mm256_maskz_ternarylogic_epi64(0, a, b, c, 9); + let r = _mm256_maskz_ternarylogic_epi64::<9>(0, a, b, c); assert_eq_m256i(r, _mm256_setzero_si256()); - let r = _mm256_maskz_ternarylogic_epi64(0b00001111, a, b, c, 8); + let r = _mm256_maskz_ternarylogic_epi64::<8>(0b00001111, a, b, c); let e = _mm256_set1_epi64x(0); assert_eq_m256i(r, e); } @@ -3260,7 +3187,7 @@ mod tests { let a = _mm_set1_epi64x(1 << 2); let b = _mm_set1_epi64x(1 << 1); let c = _mm_set1_epi64x(1 << 0); - let r = _mm_ternarylogic_epi64(a, b, c, 8); + let r = _mm_ternarylogic_epi64::<8>(a, b, c); let e = _mm_set1_epi64x(0); assert_eq_m128i(r, e); } @@ -3270,9 +3197,9 @@ mod tests { let src = _mm_set1_epi64x(1 << 2); let a = _mm_set1_epi64x(1 << 1); let b = _mm_set1_epi64x(1 << 0); - let r = _mm_mask_ternarylogic_epi64(src, 0, a, b, 8); + let r = _mm_mask_ternarylogic_epi64::<8>(src, 0, a, b); assert_eq_m128i(r, src); - let r = _mm_mask_ternarylogic_epi64(src, 0b00000011, a, b, 8); + let r = _mm_mask_ternarylogic_epi64::<8>(src, 0b00000011, a, b); let e = _mm_set1_epi64x(0); assert_eq_m128i(r, e); } @@ -3282,9 +3209,9 @@ mod tests { let a = _mm_set1_epi64x(1 << 2); let b = _mm_set1_epi64x(1 << 1); let c = _mm_set1_epi64x(1 << 0); - let r = _mm_maskz_ternarylogic_epi64(0, a, b, c, 9); + let r = _mm_maskz_ternarylogic_epi64::<9>(0, a, b, c); assert_eq_m128i(r, _mm_setzero_si128()); - let r = _mm_maskz_ternarylogic_epi64(0b00000011, a, b, c, 8); + let r = _mm_maskz_ternarylogic_epi64::<8>(0b00000011, a, b, c); let e = _mm_set1_epi64x(0); assert_eq_m128i(r, e); } @@ -3292,7 +3219,7 @@ mod tests { #[simd_test(enable = "avx512f")] unsafe fn test_mm512_getmant_pd() { let a = _mm512_set1_pd(10.); - let r = _mm512_getmant_pd(a, _MM_MANT_NORM_1_2, _MM_MANT_SIGN_SRC); + let r = _mm512_getmant_pd::<_MM_MANT_NORM_1_2, _MM_MANT_SIGN_SRC>(a); let e = _mm512_set1_pd(1.25); assert_eq_m512d(r, e); } @@ -3300,9 +3227,9 @@ mod tests { #[simd_test(enable = "avx512f")] unsafe fn test_mm512_mask_getmant_pd() { let a = _mm512_set1_pd(10.); - let r = _mm512_mask_getmant_pd(a, 0, a, _MM_MANT_NORM_1_2, _MM_MANT_SIGN_SRC); + let r = _mm512_mask_getmant_pd::<_MM_MANT_NORM_1_2, _MM_MANT_SIGN_SRC>(a, 0, a); assert_eq_m512d(r, a); - let r = _mm512_mask_getmant_pd(a, 0b11110000, a, _MM_MANT_NORM_1_2, _MM_MANT_SIGN_SRC); + let r = _mm512_mask_getmant_pd::<_MM_MANT_NORM_1_2, _MM_MANT_SIGN_SRC>(a, 0b11110000, a); let e = _mm512_setr_pd(10., 10., 10., 10., 1.25, 1.25, 1.25, 1.25); assert_eq_m512d(r, e); } @@ -3310,9 +3237,9 @@ mod tests { #[simd_test(enable = "avx512f")] unsafe fn test_mm512_maskz_getmant_pd() { let a = _mm512_set1_pd(10.); - let r = _mm512_maskz_getmant_pd(0, a, _MM_MANT_NORM_1_2, _MM_MANT_SIGN_SRC); + let r = _mm512_maskz_getmant_pd::<_MM_MANT_NORM_1_2, _MM_MANT_SIGN_SRC>(0, a); assert_eq_m512d(r, _mm512_setzero_pd()); - let r = _mm512_maskz_getmant_pd(0b11110000, a, _MM_MANT_NORM_1_2, _MM_MANT_SIGN_SRC); + let r = _mm512_maskz_getmant_pd::<_MM_MANT_NORM_1_2, _MM_MANT_SIGN_SRC>(0b11110000, a); let e = _mm512_setr_pd(0., 0., 0., 0., 1.25, 1.25, 1.25, 1.25); assert_eq_m512d(r, e); } @@ -3320,7 +3247,7 @@ mod tests { #[simd_test(enable = "avx512f,avx512vl")] unsafe fn test_mm256_getmant_pd() { let a = _mm256_set1_pd(10.); - let r = _mm256_getmant_pd(a, _MM_MANT_NORM_1_2, _MM_MANT_SIGN_SRC); + let r = _mm256_getmant_pd::<_MM_MANT_NORM_1_2, _MM_MANT_SIGN_SRC>(a); let e = _mm256_set1_pd(1.25); assert_eq_m256d(r, e); } @@ -3328,9 +3255,9 @@ mod tests { #[simd_test(enable = "avx512f,avx512vl")] unsafe fn test_mm256_mask_getmant_pd() { let a = _mm256_set1_pd(10.); - let r = _mm256_mask_getmant_pd(a, 0, a, _MM_MANT_NORM_1_2, _MM_MANT_SIGN_SRC); + let r = _mm256_mask_getmant_pd::<_MM_MANT_NORM_1_2, _MM_MANT_SIGN_SRC>(a, 0, a); assert_eq_m256d(r, a); - let r = _mm256_mask_getmant_pd(a, 0b00001111, a, _MM_MANT_NORM_1_2, _MM_MANT_SIGN_SRC); + let r = _mm256_mask_getmant_pd::<_MM_MANT_NORM_1_2, _MM_MANT_SIGN_SRC>(a, 0b00001111, a); let e = _mm256_set1_pd(1.25); assert_eq_m256d(r, e); } @@ -3338,9 +3265,9 @@ mod tests { #[simd_test(enable = "avx512f,avx512vl")] unsafe fn test_mm256_maskz_getmant_pd() { let a = _mm256_set1_pd(10.); - let r = _mm256_maskz_getmant_pd(0, a, _MM_MANT_NORM_1_2, _MM_MANT_SIGN_SRC); + let r = _mm256_maskz_getmant_pd::<_MM_MANT_NORM_1_2, _MM_MANT_SIGN_SRC>(0, a); assert_eq_m256d(r, _mm256_setzero_pd()); - let r = _mm256_maskz_getmant_pd(0b00001111, a, _MM_MANT_NORM_1_2, _MM_MANT_SIGN_SRC); + let r = _mm256_maskz_getmant_pd::<_MM_MANT_NORM_1_2, _MM_MANT_SIGN_SRC>(0b00001111, a); let e = _mm256_set1_pd(1.25); assert_eq_m256d(r, e); } @@ -3348,7 +3275,7 @@ mod tests { #[simd_test(enable = "avx512f,avx512vl")] unsafe fn test_mm_getmant_pd() { let a = _mm_set1_pd(10.); - let r = _mm_getmant_pd(a, _MM_MANT_NORM_1_2, _MM_MANT_SIGN_SRC); + let r = _mm_getmant_pd::<_MM_MANT_NORM_1_2, _MM_MANT_SIGN_SRC>(a); let e = _mm_set1_pd(1.25); assert_eq_m128d(r, e); } @@ -3356,9 +3283,9 @@ mod tests { #[simd_test(enable = "avx512f,avx512vl")] unsafe fn test_mm_mask_getmant_pd() { let a = _mm_set1_pd(10.); - let r = _mm_mask_getmant_pd(a, 0, a, _MM_MANT_NORM_1_2, _MM_MANT_SIGN_SRC); + let r = _mm_mask_getmant_pd::<_MM_MANT_NORM_1_2, _MM_MANT_SIGN_SRC>(a, 0, a); assert_eq_m128d(r, a); - let r = _mm_mask_getmant_pd(a, 0b00000011, a, _MM_MANT_NORM_1_2, _MM_MANT_SIGN_SRC); + let r = _mm_mask_getmant_pd::<_MM_MANT_NORM_1_2, _MM_MANT_SIGN_SRC>(a, 0b00000011, a); let e = _mm_set1_pd(1.25); assert_eq_m128d(r, e); } @@ -3366,9 +3293,9 @@ mod tests { #[simd_test(enable = "avx512f,avx512vl")] unsafe fn test_mm_maskz_getmant_pd() { let a = _mm_set1_pd(10.); - let r = _mm_maskz_getmant_pd(0, a, _MM_MANT_NORM_1_2, _MM_MANT_SIGN_SRC); + let r = _mm_maskz_getmant_pd::<_MM_MANT_NORM_1_2, _MM_MANT_SIGN_SRC>(0, a); assert_eq_m128d(r, _mm_setzero_pd()); - let r = _mm_maskz_getmant_pd(0b00000011, a, _MM_MANT_NORM_1_2, _MM_MANT_SIGN_SRC); + let r = _mm_maskz_getmant_pd::<_MM_MANT_NORM_1_2, _MM_MANT_SIGN_SRC>(0b00000011, a); let e = _mm_set1_pd(1.25); assert_eq_m128d(r, e); } @@ -5091,7 +5018,7 @@ mod tests { #[simd_test(enable = "avx512f")] unsafe fn test_mm512_cvtt_roundpd_epi32() { let a = _mm512_setr_pd(0., -1.5, 2., -3.5, 4., -5.5, 6., -7.5); - let r = _mm512_cvtt_roundpd_epi32(a, _MM_FROUND_NO_EXC); + let r = _mm512_cvtt_roundpd_epi32::<_MM_FROUND_NO_EXC>(a); let e = _mm256_setr_epi32(0, -1, 2, -3, 4, -5, 6, -7); assert_eq_m256i(r, e); } @@ -5100,9 +5027,9 @@ mod tests { unsafe fn test_mm512_mask_cvtt_roundpd_epi32() { let a = _mm512_setr_pd(0., -1.5, 2., -3.5, 4., -5.5, 6., -7.5); let src = _mm256_set1_epi32(0); - let r = _mm512_mask_cvtt_roundpd_epi32(src, 0, a, _MM_FROUND_NO_EXC); + let r = _mm512_mask_cvtt_roundpd_epi32::<_MM_FROUND_NO_EXC>(src, 0, a); assert_eq_m256i(r, src); - let r = _mm512_mask_cvtt_roundpd_epi32(src, 0b00001111, a, _MM_FROUND_NO_EXC); + let r = _mm512_mask_cvtt_roundpd_epi32::<_MM_FROUND_NO_EXC>(src, 0b00001111, a); let e = _mm256_setr_epi32(0, -1, 2, -3, 0, 0, 0, 0); assert_eq_m256i(r, e); } @@ -5110,9 +5037,9 @@ mod tests { #[simd_test(enable = "avx512f")] unsafe fn test_mm512_maskz_cvtt_roundpd_epi32() { let a = _mm512_setr_pd(0., -1.5, 2., -3.5, 4., -5.5, 6., -7.5); - let r = _mm512_maskz_cvtt_roundpd_epi32(0, a, _MM_FROUND_NO_EXC); + let r = _mm512_maskz_cvtt_roundpd_epi32::<_MM_FROUND_NO_EXC>(0, a); assert_eq_m256i(r, _mm256_setzero_si256()); - let r = _mm512_maskz_cvtt_roundpd_epi32(0b00001111, a, _MM_FROUND_NO_EXC); + let r = _mm512_maskz_cvtt_roundpd_epi32::<_MM_FROUND_NO_EXC>(0b00001111, a); let e = _mm256_setr_epi32(0, -1, 2, -3, 0, 0, 0, 0); assert_eq_m256i(r, e); } @@ -5120,7 +5047,7 @@ mod tests { #[simd_test(enable = "avx512f")] unsafe fn test_mm512_cvtt_roundpd_epu32() { let a = _mm512_setr_pd(0., -1.5, 2., -3.5, 4., -5.5, 6., -7.5); - let r = _mm512_cvtt_roundpd_epu32(a, _MM_FROUND_NO_EXC); + let r = _mm512_cvtt_roundpd_epu32::<_MM_FROUND_NO_EXC>(a); let e = _mm256_setr_epi32(0, -1, 2, -1, 4, -1, 6, -1); assert_eq_m256i(r, e); } @@ -5129,9 +5056,9 @@ mod tests { unsafe fn test_mm512_mask_cvtt_roundpd_epu32() { let a = _mm512_setr_pd(0., -1.5, 2., -3.5, 4., -5.5, 6., -7.5); let src = _mm256_set1_epi32(0); - let r = _mm512_mask_cvtt_roundpd_epu32(src, 0, a, _MM_FROUND_NO_EXC); + let r = _mm512_mask_cvtt_roundpd_epu32::<_MM_FROUND_NO_EXC>(src, 0, a); assert_eq_m256i(r, src); - let r = _mm512_mask_cvtt_roundpd_epu32(src, 0b00001111, a, _MM_FROUND_NO_EXC); + let r = _mm512_mask_cvtt_roundpd_epu32::<_MM_FROUND_NO_EXC>(src, 0b00001111, a); let e = _mm256_setr_epi32(0, -1, 2, -1, 0, 0, 0, 0); assert_eq_m256i(r, e); } @@ -5139,9 +5066,9 @@ mod tests { #[simd_test(enable = "avx512f")] unsafe fn test_mm512_maskz_cvtt_roundpd_epu32() { let a = _mm512_setr_pd(0., -1.5, 2., -3.5, 4., -5.5, 6., -7.5); - let r = _mm512_maskz_cvtt_roundpd_epu32(0, a, _MM_FROUND_NO_EXC); + let r = _mm512_maskz_cvtt_roundpd_epu32::<_MM_FROUND_NO_EXC>(0, a); assert_eq_m256i(r, _mm256_setzero_si256()); - let r = _mm512_maskz_cvtt_roundpd_epu32(0b00001111, a, _MM_FROUND_NO_EXC); + let r = _mm512_maskz_cvtt_roundpd_epu32::<_MM_FROUND_NO_EXC>(0b00001111, a); let e = _mm256_setr_epi32(0, -1, 2, -1, 0, 0, 0, 0); assert_eq_m256i(r, e); } @@ -5308,10 +5235,10 @@ mod tests { unsafe fn test_mm512_add_round_pd() { let a = _mm512_setr_pd(8., 9.5, 10., 11.5, 12., 13.5, 14., 0.000000000000000007); let b = _mm512_set1_pd(-1.); - let r = _mm512_add_round_pd(a, b, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); + let r = _mm512_add_round_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>(a, b); let e = _mm512_setr_pd(7., 8.5, 9., 10.5, 11., 12.5, 13., -1.0); assert_eq_m512d(r, e); - let r = _mm512_add_round_pd(a, b, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); + let r = _mm512_add_round_pd::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(a, b); let e = _mm512_setr_pd(7., 8.5, 9., 10.5, 11., 12.5, 13., -0.9999999999999999); assert_eq_m512d(r, e); } @@ -5320,14 +5247,12 @@ mod tests { unsafe fn test_mm512_mask_add_round_pd() { let a = _mm512_setr_pd(8., 9.5, 10., 11.5, 12., 13.5, 14., 0.000000000000000007); let b = _mm512_set1_pd(-1.); - let r = _mm512_mask_add_round_pd(a, 0, a, b, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); + let r = _mm512_mask_add_round_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, 0, a, b, + ); assert_eq_m512d(r, a); - let r = _mm512_mask_add_round_pd( - a, - 0b11110000, - a, - b, - _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC, + let r = _mm512_mask_add_round_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, 0b11110000, a, b, ); let e = _mm512_setr_pd(8., 9.5, 10., 11.5, 11., 12.5, 13., -1.0); assert_eq_m512d(r, e); @@ -5337,13 +5262,11 @@ mod tests { unsafe fn test_mm512_maskz_add_round_pd() { let a = _mm512_setr_pd(8., 9.5, 10., 11.5, 12., 13.5, 14., 0.000000000000000007); let b = _mm512_set1_pd(-1.); - let r = _mm512_maskz_add_round_pd(0, a, b, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); + let r = + _mm512_maskz_add_round_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>(0, a, b); assert_eq_m512d(r, _mm512_setzero_pd()); - let r = _mm512_maskz_add_round_pd( - 0b11110000, - a, - b, - _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC, + let r = _mm512_maskz_add_round_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + 0b11110000, a, b, ); let e = _mm512_setr_pd(0., 0., 0., 0., 11., 12.5, 13., -1.0); assert_eq_m512d(r, e); @@ -5353,10 +5276,10 @@ mod tests { unsafe fn test_mm512_sub_round_pd() { let a = _mm512_setr_pd(8., 9.5, 10., 11.5, 12., 13.5, 14., 0.000000000000000007); let b = _mm512_set1_pd(1.); - let r = _mm512_sub_round_pd(a, b, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); + let r = _mm512_sub_round_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>(a, b); let e = _mm512_setr_pd(7., 8.5, 9., 10.5, 11., 12.5, 13., -1.0); assert_eq_m512d(r, e); - let r = _mm512_sub_round_pd(a, b, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); + let r = _mm512_sub_round_pd::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(a, b); let e = _mm512_setr_pd(7., 8.5, 9., 10.5, 11., 12.5, 13., -0.9999999999999999); assert_eq_m512d(r, e); } @@ -5365,14 +5288,12 @@ mod tests { unsafe fn test_mm512_mask_sub_round_pd() { let a = _mm512_setr_pd(8., 9.5, 10., 11.5, 12., 13.5, 14., 0.000000000000000007); let b = _mm512_set1_pd(1.); - let r = _mm512_mask_sub_round_pd(a, 0, a, b, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); + let r = _mm512_mask_sub_round_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, 0, a, b, + ); assert_eq_m512d(r, a); - let r = _mm512_mask_sub_round_pd( - a, - 0b11110000, - a, - b, - _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC, + let r = _mm512_mask_sub_round_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, 0b11110000, a, b, ); let e = _mm512_setr_pd(8., 9.5, 10., 11.5, 11., 12.5, 13., -1.0); assert_eq_m512d(r, e); @@ -5382,13 +5303,11 @@ mod tests { unsafe fn test_mm512_maskz_sub_round_pd() { let a = _mm512_setr_pd(8., 9.5, 10., 11.5, 12., 13.5, 14., 0.000000000000000007); let b = _mm512_set1_pd(1.); - let r = _mm512_maskz_sub_round_pd(0, a, b, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); + let r = + _mm512_maskz_sub_round_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>(0, a, b); assert_eq_m512d(r, _mm512_setzero_pd()); - let r = _mm512_maskz_sub_round_pd( - 0b11110000, - a, - b, - _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC, + let r = _mm512_maskz_sub_round_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + 0b11110000, a, b, ); let e = _mm512_setr_pd(0., 0., 0., 0., 11., 12.5, 13., -1.0); assert_eq_m512d(r, e); @@ -5398,7 +5317,7 @@ mod tests { unsafe fn test_mm512_mul_round_pd() { let a = _mm512_setr_pd(8., 9.5, 10., 11.5, 12., 13.5, 14., 0.); let b = _mm512_set1_pd(0.1); - let r = _mm512_mul_round_pd(a, b, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); + let r = _mm512_mul_round_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>(a, b); let e = _mm512_setr_pd( 0.8, 0.9500000000000001, @@ -5410,7 +5329,7 @@ mod tests { 0., ); assert_eq_m512d(r, e); - let r = _mm512_mul_round_pd(a, b, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); + let r = _mm512_mul_round_pd::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(a, b); let e = _mm512_setr_pd(0.8, 0.95, 1.0, 1.15, 1.2, 1.3499999999999999, 1.4, 0.0); assert_eq_m512d(r, e); } @@ -5419,14 +5338,12 @@ mod tests { unsafe fn test_mm512_mask_mul_round_pd() { let a = _mm512_setr_pd(8., 9.5, 10., 11.5, 12., 13.5, 14., 0.); let b = _mm512_set1_pd(0.1); - let r = _mm512_mask_mul_round_pd(a, 0, a, b, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); + let r = _mm512_mask_mul_round_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, 0, a, b, + ); assert_eq_m512d(r, a); - let r = _mm512_mask_mul_round_pd( - a, - 0b11110000, - a, - b, - _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC, + let r = _mm512_mask_mul_round_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, 0b11110000, a, b, ); let e = _mm512_setr_pd( 8., @@ -5445,13 +5362,11 @@ mod tests { unsafe fn test_mm512_maskz_mul_round_pd() { let a = _mm512_setr_pd(8., 9.5, 10., 11.5, 12., 13.5, 14., 0.); let b = _mm512_set1_pd(0.1); - let r = _mm512_maskz_mul_round_pd(0, a, b, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); + let r = + _mm512_maskz_mul_round_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>(0, a, b); assert_eq_m512d(r, _mm512_setzero_pd()); - let r = _mm512_maskz_mul_round_pd( - 0b11110000, - a, - b, - _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC, + let r = _mm512_maskz_mul_round_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + 0b11110000, a, b, ); let e = _mm512_setr_pd( 0., @@ -5470,10 +5385,10 @@ mod tests { unsafe fn test_mm512_div_round_pd() { let a = _mm512_set1_pd(1.); let b = _mm512_set1_pd(3.); - let r = _mm512_div_round_pd(a, b, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); + let r = _mm512_div_round_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>(a, b); let e = _mm512_set1_pd(0.3333333333333333); assert_eq_m512d(r, e); - let r = _mm512_div_round_pd(a, b, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); + let r = _mm512_div_round_pd::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(a, b); let e = _mm512_set1_pd(0.3333333333333333); assert_eq_m512d(r, e); } @@ -5482,14 +5397,12 @@ mod tests { unsafe fn test_mm512_mask_div_round_pd() { let a = _mm512_set1_pd(1.); let b = _mm512_set1_pd(3.); - let r = _mm512_mask_div_round_pd(a, 0, a, b, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); + let r = _mm512_mask_div_round_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, 0, a, b, + ); assert_eq_m512d(r, a); - let r = _mm512_mask_div_round_pd( - a, - 0b11110000, - a, - b, - _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC, + let r = _mm512_mask_div_round_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, 0b11110000, a, b, ); let e = _mm512_setr_pd( 1., @@ -5508,13 +5421,11 @@ mod tests { unsafe fn test_mm512_maskz_div_round_pd() { let a = _mm512_set1_pd(1.); let b = _mm512_set1_pd(3.); - let r = _mm512_maskz_div_round_pd(0, a, b, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); + let r = + _mm512_maskz_div_round_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>(0, a, b); assert_eq_m512d(r, _mm512_setzero_pd()); - let r = _mm512_maskz_div_round_pd( - 0b11110000, - a, - b, - _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC, + let r = _mm512_maskz_div_round_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + 0b11110000, a, b, ); let e = _mm512_setr_pd( 0., @@ -5532,10 +5443,10 @@ mod tests { #[simd_test(enable = "avx512f")] unsafe fn test_mm512_sqrt_round_pd() { let a = _mm512_set1_pd(3.); - let r = _mm512_sqrt_round_pd(a, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); + let r = _mm512_sqrt_round_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>(a); let e = _mm512_set1_pd(1.7320508075688772); assert_eq_m512d(r, e); - let r = _mm512_sqrt_round_pd(a, _MM_FROUND_TO_POS_INF | _MM_FROUND_NO_EXC); + let r = _mm512_sqrt_round_pd::<{ _MM_FROUND_TO_POS_INF | _MM_FROUND_NO_EXC }>(a); let e = _mm512_set1_pd(1.7320508075688774); assert_eq_m512d(r, e); } @@ -5543,13 +5454,11 @@ mod tests { #[simd_test(enable = "avx512f")] unsafe fn test_mm512_mask_sqrt_round_pd() { let a = _mm512_set1_pd(3.); - let r = _mm512_mask_sqrt_round_pd(a, 0, a, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); + let r = + _mm512_mask_sqrt_round_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>(a, 0, a); assert_eq_m512d(r, a); - let r = _mm512_mask_sqrt_round_pd( - a, - 0b11110000, - a, - _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC, + let r = _mm512_mask_sqrt_round_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, 0b11110000, a, ); let e = _mm512_setr_pd( 3., @@ -5567,12 +5476,11 @@ mod tests { #[simd_test(enable = "avx512f")] unsafe fn test_mm512_maskz_sqrt_round_pd() { let a = _mm512_set1_pd(3.); - let r = _mm512_maskz_sqrt_round_pd(0, a, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); + let r = + _mm512_maskz_sqrt_round_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>(0, a); assert_eq_m512d(r, _mm512_setzero_pd()); - let r = _mm512_maskz_sqrt_round_pd( - 0b11110000, - a, - _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC, + let r = _mm512_maskz_sqrt_round_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + 0b11110000, a, ); let e = _mm512_setr_pd( 0., @@ -5592,10 +5500,10 @@ mod tests { let a = _mm512_set1_pd(0.000000000000000007); let b = _mm512_set1_pd(1.); let c = _mm512_set1_pd(-1.); - let r = _mm512_fmadd_round_pd(a, b, c, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); + let r = _mm512_fmadd_round_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>(a, b, c); let e = _mm512_set1_pd(-1.); assert_eq_m512d(r, e); - let r = _mm512_fmadd_round_pd(a, b, c, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); + let r = _mm512_fmadd_round_pd::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(a, b, c); let e = _mm512_set1_pd(-0.9999999999999999); assert_eq_m512d(r, e); } @@ -5605,15 +5513,12 @@ mod tests { let a = _mm512_set1_pd(0.000000000000000007); let b = _mm512_set1_pd(1.); let c = _mm512_set1_pd(-1.); - let r = - _mm512_mask_fmadd_round_pd(a, 0, b, c, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); + let r = _mm512_mask_fmadd_round_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, 0, b, c, + ); assert_eq_m512d(r, a); - let r = _mm512_mask_fmadd_round_pd( - a, - 0b00001111, - b, - c, - _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC, + let r = _mm512_mask_fmadd_round_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, 0b00001111, b, c, ); let e = _mm512_setr_pd( -1., @@ -5633,15 +5538,12 @@ mod tests { let a = _mm512_set1_pd(0.000000000000000007); let b = _mm512_set1_pd(1.); let c = _mm512_set1_pd(-1.); - let r = - _mm512_maskz_fmadd_round_pd(0, a, b, c, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); + let r = _mm512_maskz_fmadd_round_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + 0, a, b, c, + ); assert_eq_m512d(r, _mm512_setzero_pd()); - let r = _mm512_maskz_fmadd_round_pd( - 0b00001111, - a, - b, - c, - _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC, + let r = _mm512_maskz_fmadd_round_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + 0b00001111, a, b, c, ); let e = _mm512_setr_pd(-1., -1., -1., -1., 0., 0., 0., 0.); assert_eq_m512d(r, e); @@ -5652,15 +5554,12 @@ mod tests { let a = _mm512_set1_pd(0.000000000000000007); let b = _mm512_set1_pd(1.); let c = _mm512_set1_pd(-1.); - let r = - _mm512_mask3_fmadd_round_pd(a, b, c, 0, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); + let r = _mm512_mask3_fmadd_round_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, b, c, 0, + ); assert_eq_m512d(r, c); - let r = _mm512_mask3_fmadd_round_pd( - a, - b, - c, - 0b00001111, - _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC, + let r = _mm512_mask3_fmadd_round_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, b, c, 0b00001111, ); let e = _mm512_setr_pd(-1., -1., -1., -1., -1., -1., -1., -1.); assert_eq_m512d(r, e); @@ -5671,10 +5570,10 @@ mod tests { let a = _mm512_set1_pd(0.000000000000000007); let b = _mm512_set1_pd(1.); let c = _mm512_set1_pd(1.); - let r = _mm512_fmsub_round_pd(a, b, c, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); + let r = _mm512_fmsub_round_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>(a, b, c); let e = _mm512_set1_pd(-1.); assert_eq_m512d(r, e); - let r = _mm512_fmsub_round_pd(a, b, c, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); + let r = _mm512_fmsub_round_pd::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(a, b, c); let e = _mm512_set1_pd(-0.9999999999999999); assert_eq_m512d(r, e); } @@ -5684,15 +5583,12 @@ mod tests { let a = _mm512_set1_pd(0.000000000000000007); let b = _mm512_set1_pd(1.); let c = _mm512_set1_pd(1.); - let r = - _mm512_mask_fmsub_round_pd(a, 0, b, c, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); + let r = _mm512_mask_fmsub_round_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, 0, b, c, + ); assert_eq_m512d(r, a); - let r = _mm512_mask_fmsub_round_pd( - a, - 0b00001111, - b, - c, - _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC, + let r = _mm512_mask_fmsub_round_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, 0b00001111, b, c, ); let e = _mm512_setr_pd( -1., @@ -5712,15 +5608,12 @@ mod tests { let a = _mm512_set1_pd(0.000000000000000007); let b = _mm512_set1_pd(1.); let c = _mm512_set1_pd(1.); - let r = - _mm512_maskz_fmsub_round_pd(0, a, b, c, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); + let r = _mm512_maskz_fmsub_round_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + 0, a, b, c, + ); assert_eq_m512d(r, _mm512_setzero_pd()); - let r = _mm512_maskz_fmsub_round_pd( - 0b00001111, - a, - b, - c, - _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC, + let r = _mm512_maskz_fmsub_round_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + 0b00001111, a, b, c, ); let e = _mm512_setr_pd(-1., -1., -1., -1., 0., 0., 0., 0.); assert_eq_m512d(r, e); @@ -5731,15 +5624,12 @@ mod tests { let a = _mm512_set1_pd(0.000000000000000007); let b = _mm512_set1_pd(1.); let c = _mm512_set1_pd(1.); - let r = - _mm512_mask3_fmsub_round_pd(a, b, c, 0, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); + let r = _mm512_mask3_fmsub_round_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, b, c, 0, + ); assert_eq_m512d(r, c); - let r = _mm512_mask3_fmsub_round_pd( - a, - b, - c, - 0b00001111, - _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC, + let r = _mm512_mask3_fmsub_round_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, b, c, 0b00001111, ); let e = _mm512_setr_pd(-1., -1., -1., -1., 1., 1., 1., 1.); assert_eq_m512d(r, e); @@ -5750,10 +5640,11 @@ mod tests { let a = _mm512_set1_pd(0.000000000000000007); let b = _mm512_set1_pd(1.); let c = _mm512_set1_pd(-1.); - let r = _mm512_fmaddsub_round_pd(a, b, c, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); + let r = + _mm512_fmaddsub_round_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>(a, b, c); let e = _mm512_setr_pd(1., -1., 1., -1., 1., -1., 1., -1.); assert_eq_m512d(r, e); - let r = _mm512_fmaddsub_round_pd(a, b, c, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); + let r = _mm512_fmaddsub_round_pd::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(a, b, c); let e = _mm512_setr_pd( 1., -0.9999999999999999, @@ -5772,20 +5663,12 @@ mod tests { let a = _mm512_set1_pd(0.000000000000000007); let b = _mm512_set1_pd(1.); let c = _mm512_set1_pd(-1.); - let r = _mm512_mask_fmaddsub_round_pd( - a, - 0, - b, - c, - _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC, + let r = _mm512_mask_fmaddsub_round_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, 0, b, c, ); assert_eq_m512d(r, a); - let r = _mm512_mask_fmaddsub_round_pd( - a, - 0b00001111, - b, - c, - _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC, + let r = _mm512_mask_fmaddsub_round_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, 0b00001111, b, c, ); let e = _mm512_setr_pd( 1., @@ -5805,20 +5688,12 @@ mod tests { let a = _mm512_set1_pd(0.000000000000000007); let b = _mm512_set1_pd(1.); let c = _mm512_set1_pd(-1.); - let r = _mm512_maskz_fmaddsub_round_pd( - 0, - a, - b, - c, - _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC, + let r = _mm512_maskz_fmaddsub_round_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + 0, a, b, c, ); assert_eq_m512d(r, _mm512_setzero_pd()); - let r = _mm512_maskz_fmaddsub_round_pd( - 0b00001111, - a, - b, - c, - _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC, + let r = _mm512_maskz_fmaddsub_round_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + 0b00001111, a, b, c, ); let e = _mm512_setr_pd(1., -1., 1., -1., 0., 0., 0., 0.); assert_eq_m512d(r, e); @@ -5829,20 +5704,12 @@ mod tests { let a = _mm512_set1_pd(0.000000000000000007); let b = _mm512_set1_pd(1.); let c = _mm512_set1_pd(-1.); - let r = _mm512_mask3_fmaddsub_round_pd( - a, - b, - c, - 0, - _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC, + let r = _mm512_mask3_fmaddsub_round_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, b, c, 0, ); assert_eq_m512d(r, c); - let r = _mm512_mask3_fmaddsub_round_pd( - a, - b, - c, - 0b00001111, - _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC, + let r = _mm512_mask3_fmaddsub_round_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, b, c, 0b00001111, ); let e = _mm512_setr_pd(1., -1., 1., -1., -1., -1., -1., -1.); assert_eq_m512d(r, e); @@ -5853,10 +5720,11 @@ mod tests { let a = _mm512_set1_pd(0.000000000000000007); let b = _mm512_set1_pd(1.); let c = _mm512_set1_pd(-1.); - let r = _mm512_fmsubadd_round_pd(a, b, c, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); + let r = + _mm512_fmsubadd_round_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>(a, b, c); let e = _mm512_setr_pd(-1., 1., -1., 1., -1., 1., -1., 1.); assert_eq_m512d(r, e); - let r = _mm512_fmsubadd_round_pd(a, b, c, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); + let r = _mm512_fmsubadd_round_pd::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(a, b, c); let e = _mm512_setr_pd( -0.9999999999999999, 1., @@ -5875,20 +5743,12 @@ mod tests { let a = _mm512_set1_pd(0.000000000000000007); let b = _mm512_set1_pd(1.); let c = _mm512_set1_pd(-1.); - let r = _mm512_mask_fmsubadd_round_pd( - a, - 0, - b, - c, - _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC, + let r = _mm512_mask_fmsubadd_round_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, 0, b, c, ); assert_eq_m512d(r, a); - let r = _mm512_mask_fmsubadd_round_pd( - a, - 0b00001111, - b, - c, - _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC, + let r = _mm512_mask_fmsubadd_round_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, 0b00001111, b, c, ); let e = _mm512_setr_pd( -1., @@ -5908,20 +5768,12 @@ mod tests { let a = _mm512_set1_pd(0.000000000000000007); let b = _mm512_set1_pd(1.); let c = _mm512_set1_pd(-1.); - let r = _mm512_maskz_fmsubadd_round_pd( - 0, - a, - b, - c, - _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC, + let r = _mm512_maskz_fmsubadd_round_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + 0, a, b, c, ); assert_eq_m512d(r, _mm512_setzero_pd()); - let r = _mm512_maskz_fmsubadd_round_pd( - 0b00001111, - a, - b, - c, - _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC, + let r = _mm512_maskz_fmsubadd_round_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + 0b00001111, a, b, c, ); let e = _mm512_setr_pd(-1., 1., -1., 1., 0., 0., 0., 0.); assert_eq_m512d(r, e); @@ -5932,20 +5784,12 @@ mod tests { let a = _mm512_set1_pd(0.000000000000000007); let b = _mm512_set1_pd(1.); let c = _mm512_set1_pd(-1.); - let r = _mm512_mask3_fmsubadd_round_pd( - a, - b, - c, - 0, - _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC, + let r = _mm512_mask3_fmsubadd_round_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, b, c, 0, ); assert_eq_m512d(r, c); - let r = _mm512_mask3_fmsubadd_round_pd( - a, - b, - c, - 0b00001111, - _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC, + let r = _mm512_mask3_fmsubadd_round_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, b, c, 0b00001111, ); let e = _mm512_setr_pd(-1., 1., -1., 1., -1., -1., -1., -1.); assert_eq_m512d(r, e); @@ -5956,10 +5800,11 @@ mod tests { let a = _mm512_set1_pd(0.000000000000000007); let b = _mm512_set1_pd(1.); let c = _mm512_set1_pd(1.); - let r = _mm512_fnmadd_round_pd(a, b, c, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); + let r = + _mm512_fnmadd_round_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>(a, b, c); let e = _mm512_set1_pd(1.); assert_eq_m512d(r, e); - let r = _mm512_fnmadd_round_pd(a, b, c, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); + let r = _mm512_fnmadd_round_pd::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(a, b, c); let e = _mm512_set1_pd(0.9999999999999999); assert_eq_m512d(r, e); } @@ -5969,15 +5814,12 @@ mod tests { let a = _mm512_set1_pd(0.000000000000000007); let b = _mm512_set1_pd(1.); let c = _mm512_set1_pd(1.); - let r = - _mm512_mask_fnmadd_round_pd(a, 0, b, c, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); + let r = _mm512_mask_fnmadd_round_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, 0, b, c, + ); assert_eq_m512d(r, a); - let r = _mm512_mask_fnmadd_round_pd( - a, - 0b00001111, - b, - c, - _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC, + let r = _mm512_mask_fnmadd_round_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, 0b00001111, b, c, ); let e = _mm512_setr_pd( 1., @@ -5997,15 +5839,12 @@ mod tests { let a = _mm512_set1_pd(0.000000000000000007); let b = _mm512_set1_pd(1.); let c = _mm512_set1_pd(1.); - let r = - _mm512_maskz_fnmadd_round_pd(0, a, b, c, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); + let r = _mm512_maskz_fnmadd_round_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + 0, a, b, c, + ); assert_eq_m512d(r, _mm512_setzero_pd()); - let r = _mm512_maskz_fnmadd_round_pd( - 0b00001111, - a, - b, - c, - _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC, + let r = _mm512_maskz_fnmadd_round_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + 0b00001111, a, b, c, ); let e = _mm512_setr_pd(1., 1., 1., 1., 0., 0., 0., 0.); assert_eq_m512d(r, e); @@ -6016,15 +5855,12 @@ mod tests { let a = _mm512_set1_pd(0.000000000000000007); let b = _mm512_set1_pd(1.); let c = _mm512_set1_pd(1.); - let r = - _mm512_mask3_fnmadd_round_pd(a, b, c, 0, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); + let r = _mm512_mask3_fnmadd_round_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, b, c, 0, + ); assert_eq_m512d(r, c); - let r = _mm512_mask3_fnmadd_round_pd( - a, - b, - c, - 0b00001111, - _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC, + let r = _mm512_mask3_fnmadd_round_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, b, c, 0b00001111, ); let e = _mm512_setr_pd(1., 1., 1., 1., 1., 1., 1., 1.); assert_eq_m512d(r, e); @@ -6035,10 +5871,11 @@ mod tests { let a = _mm512_set1_pd(0.000000000000000007); let b = _mm512_set1_pd(1.); let c = _mm512_set1_pd(-1.); - let r = _mm512_fnmsub_round_pd(a, b, c, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); + let r = + _mm512_fnmsub_round_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>(a, b, c); let e = _mm512_set1_pd(1.); assert_eq_m512d(r, e); - let r = _mm512_fnmsub_round_pd(a, b, c, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); + let r = _mm512_fnmsub_round_pd::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(a, b, c); let e = _mm512_set1_pd(0.9999999999999999); assert_eq_m512d(r, e); } @@ -6048,15 +5885,12 @@ mod tests { let a = _mm512_set1_pd(0.000000000000000007); let b = _mm512_set1_pd(1.); let c = _mm512_set1_pd(-1.); - let r = - _mm512_mask_fnmsub_round_pd(a, 0, b, c, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); + let r = _mm512_mask_fnmsub_round_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, 0, b, c, + ); assert_eq_m512d(r, a); - let r = _mm512_mask_fnmsub_round_pd( - a, - 0b00001111, - b, - c, - _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC, + let r = _mm512_mask_fnmsub_round_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, 0b00001111, b, c, ); let e = _mm512_setr_pd( 1., @@ -6076,15 +5910,12 @@ mod tests { let a = _mm512_set1_pd(0.000000000000000007); let b = _mm512_set1_pd(1.); let c = _mm512_set1_pd(-1.); - let r = - _mm512_maskz_fnmsub_round_pd(0, a, b, c, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); + let r = _mm512_maskz_fnmsub_round_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + 0, a, b, c, + ); assert_eq_m512d(r, _mm512_setzero_pd()); - let r = _mm512_maskz_fnmsub_round_pd( - 0b00001111, - a, - b, - c, - _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC, + let r = _mm512_maskz_fnmsub_round_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + 0b00001111, a, b, c, ); let e = _mm512_setr_pd(1., 1., 1., 1., 0., 0., 0., 0.); assert_eq_m512d(r, e); @@ -6095,15 +5926,12 @@ mod tests { let a = _mm512_set1_pd(0.000000000000000007); let b = _mm512_set1_pd(1.); let c = _mm512_set1_pd(-1.); - let r = - _mm512_mask3_fnmsub_round_pd(a, b, c, 0, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); + let r = _mm512_mask3_fnmsub_round_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, b, c, 0, + ); assert_eq_m512d(r, c); - let r = _mm512_mask3_fnmsub_round_pd( - a, - b, - c, - 0b00001111, - _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC, + let r = _mm512_mask3_fnmsub_round_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, b, c, 0b00001111, ); let e = _mm512_setr_pd(1., 1., 1., 1., -1., -1., -1., -1.); assert_eq_m512d(r, e); @@ -6113,7 +5941,7 @@ mod tests { unsafe fn test_mm512_max_round_pd() { let a = _mm512_setr_pd(0., 1., 2., 3., 4., 5., 6., 7.); let b = _mm512_setr_pd(7., 6., 5., 4., 3., 2., 1., 0.); - let r = _mm512_max_round_pd(a, b, _MM_FROUND_CUR_DIRECTION); + let r = _mm512_max_round_pd::<_MM_FROUND_CUR_DIRECTION>(a, b); let e = _mm512_setr_pd(7., 6., 5., 4., 4., 5., 6., 7.); assert_eq_m512d(r, e); } @@ -6122,9 +5950,9 @@ mod tests { unsafe fn test_mm512_mask_max_round_pd() { let a = _mm512_setr_pd(0., 1., 2., 3., 4., 5., 6., 7.); let b = _mm512_setr_pd(7., 6., 5., 4., 3., 2., 1., 0.); - let r = _mm512_mask_max_round_pd(a, 0, a, b, _MM_FROUND_CUR_DIRECTION); + let r = _mm512_mask_max_round_pd::<_MM_FROUND_CUR_DIRECTION>(a, 0, a, b); assert_eq_m512d(r, a); - let r = _mm512_mask_max_round_pd(a, 0b00001111, a, b, _MM_FROUND_CUR_DIRECTION); + let r = _mm512_mask_max_round_pd::<_MM_FROUND_CUR_DIRECTION>(a, 0b00001111, a, b); let e = _mm512_setr_pd(7., 6., 5., 4., 4., 5., 6., 7.); assert_eq_m512d(r, e); } @@ -6133,9 +5961,9 @@ mod tests { unsafe fn test_mm512_maskz_max_round_pd() { let a = _mm512_setr_pd(0., 1., 2., 3., 4., 5., 6., 7.); let b = _mm512_setr_pd(7., 6., 5., 4., 3., 2., 1., 0.); - let r = _mm512_maskz_max_round_pd(0, a, b, _MM_FROUND_CUR_DIRECTION); + let r = _mm512_maskz_max_round_pd::<_MM_FROUND_CUR_DIRECTION>(0, a, b); assert_eq_m512d(r, _mm512_setzero_pd()); - let r = _mm512_maskz_max_round_pd(0b00001111, a, b, _MM_FROUND_CUR_DIRECTION); + let r = _mm512_maskz_max_round_pd::<_MM_FROUND_CUR_DIRECTION>(0b00001111, a, b); let e = _mm512_setr_pd(7., 6., 5., 4., 0., 0., 0., 0.); assert_eq_m512d(r, e); } @@ -6144,7 +5972,7 @@ mod tests { unsafe fn test_mm512_min_round_pd() { let a = _mm512_setr_pd(0., 1., 2., 3., 4., 5., 6., 7.); let b = _mm512_setr_pd(7., 6., 5., 4., 3., 2., 1., 0.); - let r = _mm512_min_round_pd(a, b, _MM_FROUND_CUR_DIRECTION); + let r = _mm512_min_round_pd::<_MM_FROUND_CUR_DIRECTION>(a, b); let e = _mm512_setr_pd(0., 1., 2., 3., 3., 2., 1., 0.); assert_eq_m512d(r, e); } @@ -6153,9 +5981,9 @@ mod tests { unsafe fn test_mm512_mask_min_round_pd() { let a = _mm512_setr_pd(0., 1., 2., 3., 4., 5., 6., 7.); let b = _mm512_setr_pd(7., 6., 5., 4., 3., 2., 1., 0.); - let r = _mm512_mask_min_round_pd(a, 0, a, b, _MM_FROUND_CUR_DIRECTION); + let r = _mm512_mask_min_round_pd::<_MM_FROUND_CUR_DIRECTION>(a, 0, a, b); assert_eq_m512d(r, a); - let r = _mm512_mask_min_round_pd(a, 0b00001111, a, b, _MM_FROUND_CUR_DIRECTION); + let r = _mm512_mask_min_round_pd::<_MM_FROUND_CUR_DIRECTION>(a, 0b00001111, a, b); let e = _mm512_setr_pd(0., 1., 2., 3., 4., 5., 6., 7.); assert_eq_m512d(r, e); } @@ -6164,9 +5992,9 @@ mod tests { unsafe fn test_mm512_maskz_min_round_pd() { let a = _mm512_setr_pd(0., 1., 2., 3., 4., 5., 6., 7.); let b = _mm512_setr_pd(7., 6., 5., 4., 3., 2., 1., 0.); - let r = _mm512_maskz_min_round_pd(0, a, b, _MM_FROUND_CUR_DIRECTION); + let r = _mm512_maskz_min_round_pd::<_MM_FROUND_CUR_DIRECTION>(0, a, b); assert_eq_m512d(r, _mm512_setzero_pd()); - let r = _mm512_maskz_min_round_pd(0b00001111, a, b, _MM_FROUND_CUR_DIRECTION); + let r = _mm512_maskz_min_round_pd::<_MM_FROUND_CUR_DIRECTION>(0b00001111, a, b); let e = _mm512_setr_pd(0., 1., 2., 3., 0., 0., 0., 0.); assert_eq_m512d(r, e); } @@ -6174,7 +6002,7 @@ mod tests { #[simd_test(enable = "avx512f")] unsafe fn test_mm512_getexp_round_pd() { let a = _mm512_set1_pd(3.); - let r = _mm512_getexp_round_pd(a, _MM_FROUND_CUR_DIRECTION); + let r = _mm512_getexp_round_pd::<_MM_FROUND_CUR_DIRECTION>(a); let e = _mm512_set1_pd(1.); assert_eq_m512d(r, e); } @@ -6182,9 +6010,9 @@ mod tests { #[simd_test(enable = "avx512f")] unsafe fn test_mm512_mask_getexp_round_pd() { let a = _mm512_set1_pd(3.); - let r = _mm512_mask_getexp_round_pd(a, 0, a, _MM_FROUND_CUR_DIRECTION); + let r = _mm512_mask_getexp_round_pd::<_MM_FROUND_CUR_DIRECTION>(a, 0, a); assert_eq_m512d(r, a); - let r = _mm512_mask_getexp_round_pd(a, 0b11110000, a, _MM_FROUND_CUR_DIRECTION); + let r = _mm512_mask_getexp_round_pd::<_MM_FROUND_CUR_DIRECTION>(a, 0b11110000, a); let e = _mm512_setr_pd(3., 3., 3., 3., 1., 1., 1., 1.); assert_eq_m512d(r, e); } @@ -6192,9 +6020,9 @@ mod tests { #[simd_test(enable = "avx512f")] unsafe fn test_mm512_maskz_getexp_round_pd() { let a = _mm512_set1_pd(3.); - let r = _mm512_maskz_getexp_round_pd(0, a, _MM_FROUND_CUR_DIRECTION); + let r = _mm512_maskz_getexp_round_pd::<_MM_FROUND_CUR_DIRECTION>(0, a); assert_eq_m512d(r, _mm512_setzero_pd()); - let r = _mm512_maskz_getexp_round_pd(0b11110000, a, _MM_FROUND_CUR_DIRECTION); + let r = _mm512_maskz_getexp_round_pd::<_MM_FROUND_CUR_DIRECTION>(0b11110000, a); let e = _mm512_setr_pd(0., 0., 0., 0., 1., 1., 1., 1.); assert_eq_m512d(r, e); } @@ -6202,7 +6030,7 @@ mod tests { #[simd_test(enable = "avx512f")] unsafe fn test_mm512_roundscale_round_pd() { let a = _mm512_set1_pd(1.1); - let r = _mm512_roundscale_round_pd(a, 0, _MM_FROUND_CUR_DIRECTION); + let r = _mm512_roundscale_round_pd::<0, _MM_FROUND_CUR_DIRECTION>(a); let e = _mm512_set1_pd(1.0); assert_eq_m512d(r, e); } @@ -6210,10 +6038,10 @@ mod tests { #[simd_test(enable = "avx512f")] unsafe fn test_mm512_mask_roundscale_round_pd() { let a = _mm512_set1_pd(1.1); - let r = _mm512_mask_roundscale_round_pd(a, 0, a, 0, _MM_FROUND_CUR_DIRECTION); + let r = _mm512_mask_roundscale_round_pd::<0, _MM_FROUND_CUR_DIRECTION>(a, 0, a); let e = _mm512_set1_pd(1.1); assert_eq_m512d(r, e); - let r = _mm512_mask_roundscale_round_pd(a, 0b11111111, a, 0, _MM_FROUND_CUR_DIRECTION); + let r = _mm512_mask_roundscale_round_pd::<0, _MM_FROUND_CUR_DIRECTION>(a, 0b11111111, a); let e = _mm512_set1_pd(1.0); assert_eq_m512d(r, e); } @@ -6221,9 +6049,9 @@ mod tests { #[simd_test(enable = "avx512f")] unsafe fn test_mm512_maskz_roundscale_round_pd() { let a = _mm512_set1_pd(1.1); - let r = _mm512_maskz_roundscale_round_pd(0, a, 0, _MM_FROUND_CUR_DIRECTION); + let r = _mm512_maskz_roundscale_round_pd::<0, _MM_FROUND_CUR_DIRECTION>(0, a); assert_eq_m512d(r, _mm512_setzero_pd()); - let r = _mm512_maskz_roundscale_round_pd(0b11111111, a, 0, _MM_FROUND_CUR_DIRECTION); + let r = _mm512_maskz_roundscale_round_pd::<0, _MM_FROUND_CUR_DIRECTION>(0b11111111, a); let e = _mm512_set1_pd(1.0); assert_eq_m512d(r, e); } @@ -6232,7 +6060,7 @@ mod tests { unsafe fn test_mm512_scalef_round_pd() { let a = _mm512_set1_pd(1.); let b = _mm512_set1_pd(3.); - let r = _mm512_scalef_round_pd(a, b, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); + let r = _mm512_scalef_round_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>(a, b); let e = _mm512_set1_pd(8.); assert_eq_m512d(r, e); } @@ -6241,15 +6069,12 @@ mod tests { unsafe fn test_mm512_mask_scalef_round_pd() { let a = _mm512_set1_pd(1.); let b = _mm512_set1_pd(3.); - let r = - _mm512_mask_scalef_round_pd(a, 0, a, b, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); + let r = _mm512_mask_scalef_round_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, 0, a, b, + ); assert_eq_m512d(r, a); - let r = _mm512_mask_scalef_round_pd( - a, - 0b11110000, - a, - b, - _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC, + let r = _mm512_mask_scalef_round_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + a, 0b11110000, a, b, ); let e = _mm512_set_pd(8., 8., 8., 8., 1., 1., 1., 1.); assert_eq_m512d(r, e); @@ -6259,14 +6084,12 @@ mod tests { unsafe fn test_mm512_maskz_scalef_round_pd() { let a = _mm512_set1_pd(1.); let b = _mm512_set1_pd(3.); - let r = - _mm512_maskz_scalef_round_pd(0, a, b, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); + let r = _mm512_maskz_scalef_round_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + 0, a, b, + ); assert_eq_m512d(r, _mm512_setzero_pd()); - let r = _mm512_maskz_scalef_round_pd( - 0b11110000, - a, - b, - _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC, + let r = _mm512_maskz_scalef_round_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>( + 0b11110000, a, b, ); let e = _mm512_set_pd(8., 8., 8., 8., 0., 0., 0., 0.); assert_eq_m512d(r, e); @@ -6277,7 +6100,7 @@ mod tests { let a = _mm512_set1_pd(f64::NAN); let b = _mm512_set1_pd(f64::MAX); let c = _mm512_set1_epi64(i32::MAX as i64); - let r = _mm512_fixupimm_round_pd(a, b, c, 5, _MM_FROUND_CUR_DIRECTION); + let r = _mm512_fixupimm_round_pd::<5, _MM_FROUND_CUR_DIRECTION>(a, b, c); let e = _mm512_set1_pd(0.0); assert_eq_m512d(r, e); } @@ -6287,7 +6110,7 @@ mod tests { let a = _mm512_set_pd(f64::NAN, f64::NAN, f64::NAN, f64::NAN, 1., 1., 1., 1.); let b = _mm512_set1_pd(f64::MAX); let c = _mm512_set1_epi64(i32::MAX as i64); - let r = _mm512_mask_fixupimm_round_pd(a, 0b11110000, b, c, 5, _MM_FROUND_CUR_DIRECTION); + let r = _mm512_mask_fixupimm_round_pd::<5, _MM_FROUND_CUR_DIRECTION>(a, 0b11110000, b, c); let e = _mm512_set_pd(0., 0., 0., 0., 1., 1., 1., 1.); assert_eq_m512d(r, e); } @@ -6297,7 +6120,7 @@ mod tests { let a = _mm512_set_pd(f64::NAN, f64::NAN, f64::NAN, f64::NAN, 1., 1., 1., 1.); let b = _mm512_set1_pd(f64::MAX); let c = _mm512_set1_epi64(i32::MAX as i64); - let r = _mm512_maskz_fixupimm_round_pd(0b11110000, a, b, c, 5, _MM_FROUND_CUR_DIRECTION); + let r = _mm512_maskz_fixupimm_round_pd::<5, _MM_FROUND_CUR_DIRECTION>(0b11110000, a, b, c); let e = _mm512_set_pd(0., 0., 0., 0., 0., 0., 0., 0.); assert_eq_m512d(r, e); } @@ -6305,12 +6128,11 @@ mod tests { #[simd_test(enable = "avx512f")] unsafe fn test_mm512_getmant_round_pd() { let a = _mm512_set1_pd(10.); - let r = _mm512_getmant_round_pd( - a, + let r = _mm512_getmant_round_pd::< _MM_MANT_NORM_1_2, _MM_MANT_SIGN_SRC, _MM_FROUND_CUR_DIRECTION, - ); + >(a); let e = _mm512_set1_pd(1.25); assert_eq_m512d(r, e); } @@ -6318,23 +6140,17 @@ mod tests { #[simd_test(enable = "avx512f")] unsafe fn test_mm512_mask_getmant_round_pd() { let a = _mm512_set1_pd(10.); - let r = _mm512_mask_getmant_round_pd( - a, - 0, - a, + let r = _mm512_mask_getmant_round_pd::< _MM_MANT_NORM_1_2, _MM_MANT_SIGN_SRC, _MM_FROUND_CUR_DIRECTION, - ); + >(a, 0, a); assert_eq_m512d(r, a); - let r = _mm512_mask_getmant_round_pd( - a, - 0b11110000, - a, + let r = _mm512_mask_getmant_round_pd::< _MM_MANT_NORM_1_2, _MM_MANT_SIGN_SRC, _MM_FROUND_CUR_DIRECTION, - ); + >(a, 0b11110000, a); let e = _mm512_setr_pd(10., 10., 10., 10., 1.25, 1.25, 1.25, 1.25); assert_eq_m512d(r, e); } @@ -6342,21 +6158,17 @@ mod tests { #[simd_test(enable = "avx512f")] unsafe fn test_mm512_maskz_getmant_round_pd() { let a = _mm512_set1_pd(10.); - let r = _mm512_maskz_getmant_round_pd( - 0, - a, + let r = _mm512_maskz_getmant_round_pd::< _MM_MANT_NORM_1_2, _MM_MANT_SIGN_SRC, _MM_FROUND_CUR_DIRECTION, - ); + >(0, a); assert_eq_m512d(r, _mm512_setzero_pd()); - let r = _mm512_maskz_getmant_round_pd( - 0b11110000, - a, + let r = _mm512_maskz_getmant_round_pd::< _MM_MANT_NORM_1_2, _MM_MANT_SIGN_SRC, _MM_FROUND_CUR_DIRECTION, - ); + >(0b11110000, a); let e = _mm512_setr_pd(0., 0., 0., 0., 1.25, 1.25, 1.25, 1.25); assert_eq_m512d(r, e); } @@ -6364,7 +6176,7 @@ mod tests { #[simd_test(enable = "avx512f")] unsafe fn test_mm512_cvt_roundps_pd() { let a = _mm256_setr_ps(0., -1.5, 2., -3.5, 4., -5.5, 6., -7.5); - let r = _mm512_cvt_roundps_pd(a, _MM_FROUND_CUR_DIRECTION); + let r = _mm512_cvt_roundps_pd::<_MM_FROUND_CUR_DIRECTION>(a); let e = _mm512_setr_pd(0., -1.5, 2., -3.5, 4., -5.5, 6., -7.5); assert_eq_m512d(r, e); } @@ -6373,9 +6185,9 @@ mod tests { unsafe fn test_mm512_mask_cvt_roundps_pd() { let a = _mm256_setr_ps(0., -1.5, 2., -3.5, 4., -5.5, 6., -7.5); let src = _mm512_set1_pd(0.); - let r = _mm512_mask_cvt_roundps_pd(src, 0, a, _MM_FROUND_CUR_DIRECTION); + let r = _mm512_mask_cvt_roundps_pd::<_MM_FROUND_CUR_DIRECTION>(src, 0, a); assert_eq_m512d(r, src); - let r = _mm512_mask_cvt_roundps_pd(src, 0b00001111, a, _MM_FROUND_CUR_DIRECTION); + let r = _mm512_mask_cvt_roundps_pd::<_MM_FROUND_CUR_DIRECTION>(src, 0b00001111, a); let e = _mm512_setr_pd(0., -1.5, 2., -3.5, 0., 0., 0., 0.); assert_eq_m512d(r, e); } @@ -6383,9 +6195,9 @@ mod tests { #[simd_test(enable = "avx512f")] unsafe fn test_mm512_maskz_cvt_roundps_pd() { let a = _mm256_setr_ps(0., -1.5, 2., -3.5, 4., -5.5, 6., -7.5); - let r = _mm512_maskz_cvt_roundps_pd(0, a, _MM_FROUND_CUR_DIRECTION); + let r = _mm512_maskz_cvt_roundps_pd::<_MM_FROUND_CUR_DIRECTION>(0, a); assert_eq_m512d(r, _mm512_setzero_pd()); - let r = _mm512_maskz_cvt_roundps_pd(0b00001111, a, _MM_FROUND_CUR_DIRECTION); + let r = _mm512_maskz_cvt_roundps_pd::<_MM_FROUND_CUR_DIRECTION>(0b00001111, a); let e = _mm512_setr_pd(0., -1.5, 2., -3.5, 0., 0., 0., 0.); assert_eq_m512d(r, e); } @@ -6393,7 +6205,7 @@ mod tests { #[simd_test(enable = "avx512f")] unsafe fn test_mm512_cvt_roundpd_ps() { let a = _mm512_setr_pd(0., -1.5, 2., -3.5, 4., -5.5, 6., -7.5); - let r = _mm512_cvt_roundpd_ps(a, _MM_FROUND_CUR_DIRECTION); + let r = _mm512_cvt_roundpd_ps::<_MM_FROUND_CUR_DIRECTION>(a); let e = _mm256_setr_ps(0., -1.5, 2., -3.5, 4., -5.5, 6., -7.5); assert_eq_m256(r, e); } @@ -6402,9 +6214,9 @@ mod tests { unsafe fn test_mm512_mask_cvt_roundpd_ps() { let a = _mm512_setr_pd(0., -1.5, 2., -3.5, 4., -5.5, 6., -7.5); let src = _mm256_set1_ps(0.); - let r = _mm512_mask_cvt_roundpd_ps(src, 0, a, _MM_FROUND_CUR_DIRECTION); + let r = _mm512_mask_cvt_roundpd_ps::<_MM_FROUND_CUR_DIRECTION>(src, 0, a); assert_eq_m256(r, src); - let r = _mm512_mask_cvt_roundpd_ps(src, 0b00001111, a, _MM_FROUND_CUR_DIRECTION); + let r = _mm512_mask_cvt_roundpd_ps::<_MM_FROUND_CUR_DIRECTION>(src, 0b00001111, a); let e = _mm256_setr_ps(0., -1.5, 2., -3.5, 0., 0., 0., 0.); assert_eq_m256(r, e); } @@ -6412,9 +6224,9 @@ mod tests { #[simd_test(enable = "avx512f")] unsafe fn test_mm512_maskz_cvt_roundpd_ps() { let a = _mm512_setr_pd(0., -1.5, 2., -3.5, 4., -5.5, 6., -7.5); - let r = _mm512_maskz_cvt_roundpd_ps(0, a, _MM_FROUND_CUR_DIRECTION); + let r = _mm512_maskz_cvt_roundpd_ps::<_MM_FROUND_CUR_DIRECTION>(0, a); assert_eq_m256(r, _mm256_setzero_ps()); - let r = _mm512_maskz_cvt_roundpd_ps(0b00001111, a, _MM_FROUND_CUR_DIRECTION); + let r = _mm512_maskz_cvt_roundpd_ps::<_MM_FROUND_CUR_DIRECTION>(0b00001111, a); let e = _mm256_setr_ps(0., -1.5, 2., -3.5, 0., 0., 0., 0.); assert_eq_m256(r, e); } @@ -6422,7 +6234,7 @@ mod tests { #[simd_test(enable = "avx512f")] unsafe fn test_mm512_cvt_roundpd_epi32() { let a = _mm512_setr_pd(0., -1.5, 2., -3.5, 4., -5.5, 6., -7.5); - let r = _mm512_cvt_roundpd_epi32(a, _MM_FROUND_CUR_DIRECTION); + let r = _mm512_cvt_roundpd_epi32::<_MM_FROUND_CUR_DIRECTION>(a); let e = _mm256_setr_epi32(0, -2, 2, -4, 4, -6, 6, -8); assert_eq_m256i(r, e); } @@ -6431,9 +6243,9 @@ mod tests { unsafe fn test_mm512_mask_cvt_roundpd_epi32() { let a = _mm512_setr_pd(0., -1.5, 2., -3.5, 4., -5.5, 6., -7.5); let src = _mm256_set1_epi32(0); - let r = _mm512_mask_cvt_roundpd_epi32(src, 0, a, _MM_FROUND_CUR_DIRECTION); + let r = _mm512_mask_cvt_roundpd_epi32::<_MM_FROUND_CUR_DIRECTION>(src, 0, a); assert_eq_m256i(r, src); - let r = _mm512_mask_cvt_roundpd_epi32(src, 0b00001111, a, _MM_FROUND_CUR_DIRECTION); + let r = _mm512_mask_cvt_roundpd_epi32::<_MM_FROUND_CUR_DIRECTION>(src, 0b00001111, a); let e = _mm256_setr_epi32(0, -2, 2, -4, 0, 0, 0, 0); assert_eq_m256i(r, e); } @@ -6441,9 +6253,9 @@ mod tests { #[simd_test(enable = "avx512f")] unsafe fn test_mm512_maskz_cvt_roundpd_epi32() { let a = _mm512_setr_pd(0., -1.5, 2., -3.5, 4., -5.5, 6., -7.5); - let r = _mm512_maskz_cvt_roundpd_epi32(0, a, _MM_FROUND_CUR_DIRECTION); + let r = _mm512_maskz_cvt_roundpd_epi32::<_MM_FROUND_CUR_DIRECTION>(0, a); assert_eq_m256i(r, _mm256_setzero_si256()); - let r = _mm512_maskz_cvt_roundpd_epi32(0b00001111, a, _MM_FROUND_CUR_DIRECTION); + let r = _mm512_maskz_cvt_roundpd_epi32::<_MM_FROUND_CUR_DIRECTION>(0b00001111, a); let e = _mm256_setr_epi32(0, -2, 2, -4, 0, 0, 0, 0); assert_eq_m256i(r, e); } @@ -6451,7 +6263,7 @@ mod tests { #[simd_test(enable = "avx512f")] unsafe fn test_mm512_cvt_roundpd_epu32() { let a = _mm512_setr_pd(0., -1.5, 2., -3.5, 4., -5.5, 6., -7.5); - let r = _mm512_cvt_roundpd_epu32(a, _MM_FROUND_CUR_DIRECTION); + let r = _mm512_cvt_roundpd_epu32::<_MM_FROUND_CUR_DIRECTION>(a); let e = _mm256_setr_epi32(0, -1, 2, -1, 4, -1, 6, -1); assert_eq_m256i(r, e); } @@ -6460,9 +6272,9 @@ mod tests { unsafe fn test_mm512_mask_cvt_roundpd_epu32() { let a = _mm512_setr_pd(0., -1.5, 2., -3.5, 4., -5.5, 6., -7.5); let src = _mm256_set1_epi32(0); - let r = _mm512_mask_cvt_roundpd_epu32(src, 0, a, _MM_FROUND_CUR_DIRECTION); + let r = _mm512_mask_cvt_roundpd_epu32::<_MM_FROUND_CUR_DIRECTION>(src, 0, a); assert_eq_m256i(r, src); - let r = _mm512_mask_cvt_roundpd_epu32(src, 0b00001111, a, _MM_FROUND_CUR_DIRECTION); + let r = _mm512_mask_cvt_roundpd_epu32::<_MM_FROUND_CUR_DIRECTION>(src, 0b00001111, a); let e = _mm256_setr_epi32(0, -1, 2, -1, 0, 0, 0, 0); assert_eq_m256i(r, e); } @@ -6470,9 +6282,9 @@ mod tests { #[simd_test(enable = "avx512f")] unsafe fn test_mm512_maskz_cvt_roundpd_epu32() { let a = _mm512_setr_pd(0., -1.5, 2., -3.5, 4., -5.5, 6., -7.5); - let r = _mm512_maskz_cvt_roundpd_epu32(0, a, _MM_FROUND_CUR_DIRECTION); + let r = _mm512_maskz_cvt_roundpd_epu32::<_MM_FROUND_CUR_DIRECTION>(0, a); assert_eq_m256i(r, _mm256_setzero_si256()); - let r = _mm512_maskz_cvt_roundpd_epu32(0b00001111, a, _MM_FROUND_CUR_DIRECTION); + let r = _mm512_maskz_cvt_roundpd_epu32::<_MM_FROUND_CUR_DIRECTION>(0b00001111, a); let e = _mm256_setr_epi32(0, -1, 2, -1, 0, 0, 0, 0); assert_eq_m256i(r, e); } @@ -6625,7 +6437,7 @@ mod tests { unsafe fn test_mm512_cmp_pd_mask() { let a = _mm512_set_pd(0., 1., -1., 13., f64::MAX, f64::MIN, 100., -100.); let b = _mm512_set1_pd(-1.); - let m = _mm512_cmp_pd_mask(a, b, _CMP_LT_OQ); + let m = _mm512_cmp_pd_mask::<_CMP_LT_OQ>(a, b); assert_eq!(m, 0b00000101); } @@ -6634,7 +6446,7 @@ mod tests { let a = _mm512_set_pd(0., 1., -1., 13., f64::MAX, f64::MIN, 100., -100.); let b = _mm512_set1_pd(-1.); let mask = 0b01100110; - let r = _mm512_mask_cmp_pd_mask(mask, a, b, _CMP_LT_OQ); + let r = _mm512_mask_cmp_pd_mask::<_CMP_LT_OQ>(mask, a, b); assert_eq!(r, 0b00000100); } @@ -6642,7 +6454,7 @@ mod tests { unsafe fn test_mm256_cmp_pd_mask() { let a = _mm256_set_pd(0., 1., -1., 13.); let b = _mm256_set1_pd(1.); - let m = _mm256_cmp_pd_mask(a, b, _CMP_LT_OQ); + let m = _mm256_cmp_pd_mask::<_CMP_LT_OQ>(a, b); assert_eq!(m, 0b00001010); } @@ -6651,7 +6463,7 @@ mod tests { let a = _mm256_set_pd(0., 1., -1., 13.); let b = _mm256_set1_pd(1.); let mask = 0b11111111; - let r = _mm256_mask_cmp_pd_mask(mask, a, b, _CMP_LT_OQ); + let r = _mm256_mask_cmp_pd_mask::<_CMP_LT_OQ>(mask, a, b); assert_eq!(r, 0b00001010); } @@ -6659,7 +6471,7 @@ mod tests { unsafe fn test_mm_cmp_pd_mask() { let a = _mm_set_pd(0., 1.); let b = _mm_set1_pd(1.); - let m = _mm_cmp_pd_mask(a, b, _CMP_LT_OQ); + let m = _mm_cmp_pd_mask::<_CMP_LT_OQ>(a, b); assert_eq!(m, 0b00000010); } @@ -6668,7 +6480,7 @@ mod tests { let a = _mm_set_pd(0., 1.); let b = _mm_set1_pd(1.); let mask = 0b11111111; - let r = _mm_mask_cmp_pd_mask(mask, a, b, _CMP_LT_OQ); + let r = _mm_mask_cmp_pd_mask::<_CMP_LT_OQ>(mask, a, b); assert_eq!(r, 0b00000010); } @@ -6677,7 +6489,7 @@ mod tests { #[rustfmt::skip] let a = _mm512_set_pd(0., 1., -1., 13., f64::MAX, f64::MIN, 100., -100.); let b = _mm512_set1_pd(-1.); - let m = _mm512_cmp_round_pd_mask(a, b, _CMP_LT_OQ, _MM_FROUND_CUR_DIRECTION); + let m = _mm512_cmp_round_pd_mask::<_CMP_LT_OQ, _MM_FROUND_CUR_DIRECTION>(a, b); assert_eq!(m, 0b00000101); } @@ -6687,7 +6499,7 @@ mod tests { let a = _mm512_set_pd(0., 1., -1., 13., f64::MAX, f64::MIN, 100., -100.); let b = _mm512_set1_pd(-1.); let mask = 0b01100110; - let r = _mm512_mask_cmp_round_pd_mask(mask, a, b, _CMP_LT_OQ, _MM_FROUND_CUR_DIRECTION); + let r = _mm512_mask_cmp_round_pd_mask::<_CMP_LT_OQ, _MM_FROUND_CUR_DIRECTION>(mask, a, b); assert_eq!(r, 0b00000100); } @@ -7047,7 +6859,7 @@ mod tests { unsafe fn test_mm512_cmp_epu64_mask() { let a = _mm512_set_epi64(0, 1, -1, u64::MAX as i64, i64::MAX, i64::MIN, 100, -100); let b = _mm512_set1_epi64(-1); - let m = _mm512_cmp_epu64_mask(a, b, _MM_CMPINT_LT); + let m = _mm512_cmp_epu64_mask::<_MM_CMPINT_LT>(a, b); assert_eq!(m, 0b11001111); } @@ -7056,7 +6868,7 @@ mod tests { let a = _mm512_set_epi64(0, 1, -1, u64::MAX as i64, i64::MAX, i64::MIN, 100, -100); let b = _mm512_set1_epi64(-1); let mask = 0b01111010; - let r = _mm512_mask_cmp_epu64_mask(mask, a, b, _MM_CMPINT_LT); + let r = _mm512_mask_cmp_epu64_mask::<_MM_CMPINT_LT>(mask, a, b); assert_eq!(r, 0b01001010); } @@ -7064,7 +6876,7 @@ mod tests { unsafe fn test_mm256_cmp_epu64_mask() { let a = _mm256_set_epi64x(0, 1, -1, 100); let b = _mm256_set1_epi64x(1); - let m = _mm256_cmp_epu64_mask(a, b, _MM_CMPINT_LT); + let m = _mm256_cmp_epu64_mask::<_MM_CMPINT_LT>(a, b); assert_eq!(m, 0b00001000); } @@ -7073,7 +6885,7 @@ mod tests { let a = _mm256_set_epi64x(0, 1, -1, 100); let b = _mm256_set1_epi64x(1); let mask = 0b11111111; - let r = _mm256_mask_cmp_epu64_mask(mask, a, b, _MM_CMPINT_LT); + let r = _mm256_mask_cmp_epu64_mask::<_MM_CMPINT_LT>(mask, a, b); assert_eq!(r, 0b00001000); } @@ -7081,7 +6893,7 @@ mod tests { unsafe fn test_mm_cmp_epu64_mask() { let a = _mm_set_epi64x(0, 1); let b = _mm_set1_epi64x(1); - let m = _mm_cmp_epu64_mask(a, b, _MM_CMPINT_LT); + let m = _mm_cmp_epu64_mask::<_MM_CMPINT_LT>(a, b); assert_eq!(m, 0b00000010); } @@ -7090,7 +6902,7 @@ mod tests { let a = _mm_set_epi64x(0, 1); let b = _mm_set1_epi64x(1); let mask = 0b11111111; - let r = _mm_mask_cmp_epu64_mask(mask, a, b, _MM_CMPINT_LT); + let r = _mm_mask_cmp_epu64_mask::<_MM_CMPINT_LT>(mask, a, b); assert_eq!(r, 0b00000010); } @@ -7418,7 +7230,7 @@ mod tests { unsafe fn test_mm512_cmp_epi64_mask() { let a = _mm512_set_epi64(0, 1, -1, 13, i64::MAX, i64::MIN, 100, -100); let b = _mm512_set1_epi64(-1); - let m = _mm512_cmp_epi64_mask(a, b, _MM_CMPINT_LT); + let m = _mm512_cmp_epi64_mask::<_MM_CMPINT_LT>(a, b); assert_eq!(m, 0b00000101); } @@ -7427,7 +7239,7 @@ mod tests { let a = _mm512_set_epi64(0, 1, -1, 13, i64::MAX, i64::MIN, 100, -100); let b = _mm512_set1_epi64(-1); let mask = 0b01100110; - let r = _mm512_mask_cmp_epi64_mask(mask, a, b, _MM_CMPINT_LT); + let r = _mm512_mask_cmp_epi64_mask::<_MM_CMPINT_LT>(mask, a, b); assert_eq!(r, 0b00000100); } @@ -7435,7 +7247,7 @@ mod tests { unsafe fn test_mm256_cmp_epi64_mask() { let a = _mm256_set_epi64x(0, 1, -1, 13); let b = _mm256_set1_epi64x(1); - let m = _mm256_cmp_epi64_mask(a, b, _MM_CMPINT_LT); + let m = _mm256_cmp_epi64_mask::<_MM_CMPINT_LT>(a, b); assert_eq!(m, 0b00001010); } @@ -7444,7 +7256,7 @@ mod tests { let a = _mm256_set_epi64x(0, 1, -1, 13); let b = _mm256_set1_epi64x(1); let mask = 0b11111111; - let r = _mm256_mask_cmp_epi64_mask(mask, a, b, _MM_CMPINT_LT); + let r = _mm256_mask_cmp_epi64_mask::<_MM_CMPINT_LT>(mask, a, b); assert_eq!(r, 0b00001010); } @@ -7452,7 +7264,7 @@ mod tests { unsafe fn test_mm_cmp_epi64_mask() { let a = _mm_set_epi64x(0, 1); let b = _mm_set1_epi64x(1); - let m = _mm_cmp_epi64_mask(a, b, _MM_CMPINT_LT); + let m = _mm_cmp_epi64_mask::<_MM_CMPINT_LT>(a, b); assert_eq!(m, 0b00000010); } @@ -7461,7 +7273,7 @@ mod tests { let a = _mm_set_epi64x(0, 1); let b = _mm_set1_epi64x(1); let mask = 0b11111111; - let r = _mm_mask_cmp_epi64_mask(mask, a, b, _MM_CMPINT_LT); + let r = _mm_mask_cmp_epi64_mask::<_MM_CMPINT_LT>(mask, a, b); assert_eq!(r, 0b00000010); } @@ -7473,7 +7285,7 @@ mod tests { } // A multiplier of 8 is word-addressing let index = _mm256_setr_epi32(0, 16, 32, 48, 64, 80, 96, 112); - let r = _mm512_i32gather_pd(index, arr.as_ptr() as *const u8, 8); + let r = _mm512_i32gather_pd::<8>(index, arr.as_ptr() as *const u8); assert_eq_m512d(r, _mm512_setr_pd(0., 16., 32., 48., 64., 80., 96., 112.)); } @@ -7487,7 +7299,7 @@ mod tests { let mask = 0b10101010; let index = _mm256_setr_epi32(0, 16, 32, 48, 64, 80, 96, 112); // A multiplier of 8 is word-addressing - let r = _mm512_mask_i32gather_pd(src, mask, index, arr.as_ptr() as *const u8, 8); + let r = _mm512_mask_i32gather_pd::<8>(src, mask, index, arr.as_ptr() as *const u8); assert_eq_m512d(r, _mm512_setr_pd(2., 16., 2., 48., 2., 80., 2., 112.)); } @@ -7499,7 +7311,7 @@ mod tests { } // A multiplier of 8 is word-addressing let index = _mm512_setr_epi64(0, 16, 32, 48, 64, 80, 96, 112); - let r = _mm512_i64gather_pd(index, arr.as_ptr() as *const u8, 8); + let r = _mm512_i64gather_pd::<8>(index, arr.as_ptr() as *const u8); assert_eq_m512d(r, _mm512_setr_pd(0., 16., 32., 48., 64., 80., 96., 112.)); } @@ -7513,7 +7325,7 @@ mod tests { let mask = 0b10101010; let index = _mm512_setr_epi64(0, 16, 32, 48, 64, 80, 96, 112); // A multiplier of 8 is word-addressing - let r = _mm512_mask_i64gather_pd(src, mask, index, arr.as_ptr() as *const u8, 8); + let r = _mm512_mask_i64gather_pd::<8>(src, mask, index, arr.as_ptr() as *const u8); assert_eq_m512d(r, _mm512_setr_pd(2., 16., 2., 48., 2., 80., 2., 112.)); } @@ -7526,7 +7338,7 @@ mod tests { // A multiplier of 4 is word-addressing #[rustfmt::skip] let index = _mm512_setr_epi64(0, 16, 32, 48, 64, 80, 96, 112); - let r = _mm512_i64gather_ps(index, arr.as_ptr() as *const u8, 4); + let r = _mm512_i64gather_ps::<4>(index, arr.as_ptr() as *const u8); assert_eq_m256(r, _mm256_setr_ps(0., 16., 32., 48., 64., 80., 96., 112.)); } @@ -7541,7 +7353,7 @@ mod tests { #[rustfmt::skip] let index = _mm512_setr_epi64(0, 16, 32, 48, 64, 80, 96, 112); // A multiplier of 4 is word-addressing - let r = _mm512_mask_i64gather_ps(src, mask, index, arr.as_ptr() as *const u8, 4); + let r = _mm512_mask_i64gather_ps::<4>(src, mask, index, arr.as_ptr() as *const u8); assert_eq_m256(r, _mm256_setr_ps(2., 16., 2., 48., 2., 80., 2., 112.)); } @@ -7553,7 +7365,7 @@ mod tests { } // A multiplier of 8 is word-addressing let index = _mm256_setr_epi32(0, 16, 32, 48, 64, 80, 96, 112); - let r = _mm512_i32gather_epi64(index, arr.as_ptr() as *const u8, 8); + let r = _mm512_i32gather_epi64::<8>(index, arr.as_ptr() as *const u8); assert_eq_m512i(r, _mm512_setr_epi64(0, 16, 32, 48, 64, 80, 96, 112)); } @@ -7567,7 +7379,7 @@ mod tests { let mask = 0b10101010; let index = _mm256_setr_epi32(0, 16, 32, 48, 64, 80, 96, 112); // A multiplier of 8 is word-addressing - let r = _mm512_mask_i32gather_epi64(src, mask, index, arr.as_ptr() as *const u8, 8); + let r = _mm512_mask_i32gather_epi64::<8>(src, mask, index, arr.as_ptr() as *const u8); assert_eq_m512i(r, _mm512_setr_epi64(2, 16, 2, 48, 2, 80, 2, 112)); } @@ -7579,7 +7391,7 @@ mod tests { } // A multiplier of 8 is word-addressing let index = _mm512_setr_epi64(0, 16, 32, 48, 64, 80, 96, 112); - let r = _mm512_i64gather_epi64(index, arr.as_ptr() as *const u8, 8); + let r = _mm512_i64gather_epi64::<8>(index, arr.as_ptr() as *const u8); assert_eq_m512i(r, _mm512_setr_epi64(0, 16, 32, 48, 64, 80, 96, 112)); } @@ -7593,7 +7405,7 @@ mod tests { let mask = 0b10101010; let index = _mm512_setr_epi64(0, 16, 32, 48, 64, 80, 96, 112); // A multiplier of 8 is word-addressing - let r = _mm512_mask_i64gather_epi64(src, mask, index, arr.as_ptr() as *const u8, 8); + let r = _mm512_mask_i64gather_epi64::<8>(src, mask, index, arr.as_ptr() as *const u8); assert_eq_m512i(r, _mm512_setr_epi64(2, 16, 2, 48, 2, 80, 2, 112)); } @@ -7605,7 +7417,7 @@ mod tests { } // A multiplier of 8 is word-addressing let index = _mm512_setr_epi64(0, 16, 32, 48, 64, 80, 96, 112); - let r = _mm512_i64gather_epi32(index, arr.as_ptr() as *const u8, 8); + let r = _mm512_i64gather_epi32::<8>(index, arr.as_ptr() as *const u8); assert_eq_m256i(r, _mm256_setr_epi32(0, 16, 32, 48, 64, 80, 96, 112)); } @@ -7619,7 +7431,7 @@ mod tests { let mask = 0b10101010; let index = _mm512_setr_epi64(0, 16, 32, 48, 64, 80, 96, 112); // A multiplier of 8 is word-addressing - let r = _mm512_mask_i64gather_epi32(src, mask, index, arr.as_ptr() as *const u8, 8); + let r = _mm512_mask_i64gather_epi32::<8>(src, mask, index, arr.as_ptr() as *const u8); assert_eq_m256i(r, _mm256_setr_epi32(2, 16, 2, 48, 2, 80, 2, 112)); } @@ -7629,7 +7441,7 @@ mod tests { let index = _mm256_setr_epi32(0, 16, 32, 48, 64, 80, 96, 112); let src = _mm512_setr_pd(1., 2., 3., 4., 5., 6., 7., 8.); // A multiplier of 8 is word-addressing - _mm512_i32scatter_pd(arr.as_mut_ptr() as *mut u8, index, src, 8); + _mm512_i32scatter_pd::<8>(arr.as_mut_ptr() as *mut u8, index, src); let mut expected = [0f64; 128]; for i in 0..8 { expected[i * 16] = (i + 1) as f64; @@ -7644,7 +7456,7 @@ mod tests { let index = _mm256_setr_epi32(0, 16, 32, 48, 64, 80, 96, 112); let src = _mm512_setr_pd(1., 2., 3., 4., 5., 6., 7., 8.); // A multiplier of 8 is word-addressing - _mm512_mask_i32scatter_pd(arr.as_mut_ptr() as *mut u8, mask, index, src, 8); + _mm512_mask_i32scatter_pd::<8>(arr.as_mut_ptr() as *mut u8, mask, index, src); let mut expected = [0f64; 128]; for i in 0..4 { expected[i * 32 + 16] = 2. * (i + 1) as f64; @@ -7658,7 +7470,7 @@ mod tests { let index = _mm512_setr_epi64(0, 16, 32, 48, 64, 80, 96, 112); let src = _mm512_setr_pd(1., 2., 3., 4., 5., 6., 7., 8.); // A multiplier of 8 is word-addressing - _mm512_i64scatter_pd(arr.as_mut_ptr() as *mut u8, index, src, 8); + _mm512_i64scatter_pd::<8>(arr.as_mut_ptr() as *mut u8, index, src); let mut expected = [0f64; 128]; for i in 0..8 { expected[i * 16] = (i + 1) as f64; @@ -7673,7 +7485,7 @@ mod tests { let index = _mm512_setr_epi64(0, 16, 32, 48, 64, 80, 96, 112); let src = _mm512_setr_pd(1., 2., 3., 4., 5., 6., 7., 8.); // A multiplier of 8 is word-addressing - _mm512_mask_i64scatter_pd(arr.as_mut_ptr() as *mut u8, mask, index, src, 8); + _mm512_mask_i64scatter_pd::<8>(arr.as_mut_ptr() as *mut u8, mask, index, src); let mut expected = [0f64; 128]; for i in 0..4 { expected[i * 32 + 16] = 2. * (i + 1) as f64; @@ -7687,7 +7499,7 @@ mod tests { let index = _mm512_setr_epi64(0, 16, 32, 48, 64, 80, 96, 112); let src = _mm256_setr_ps(1., 2., 3., 4., 5., 6., 7., 8.); // A multiplier of 4 is word-addressing - _mm512_i64scatter_ps(arr.as_mut_ptr() as *mut u8, index, src, 4); + _mm512_i64scatter_ps::<4>(arr.as_mut_ptr() as *mut u8, index, src); let mut expected = [0f32; 128]; for i in 0..8 { expected[i * 16] = (i + 1) as f32; @@ -7702,7 +7514,7 @@ mod tests { let index = _mm512_setr_epi64(0, 16, 32, 48, 64, 80, 96, 112); let src = _mm256_setr_ps(1., 2., 3., 4., 5., 6., 7., 8.); // A multiplier of 4 is word-addressing - _mm512_mask_i64scatter_ps(arr.as_mut_ptr() as *mut u8, mask, index, src, 4); + _mm512_mask_i64scatter_ps::<4>(arr.as_mut_ptr() as *mut u8, mask, index, src); let mut expected = [0f32; 128]; for i in 0..4 { expected[i * 32 + 16] = 2. * (i + 1) as f32; @@ -7716,7 +7528,7 @@ mod tests { let index = _mm256_setr_epi32(0, 16, 32, 48, 64, 80, 96, 112); let src = _mm512_setr_epi64(1, 2, 3, 4, 5, 6, 7, 8); // A multiplier of 8 is word-addressing - _mm512_i32scatter_epi64(arr.as_mut_ptr() as *mut u8, index, src, 8); + _mm512_i32scatter_epi64::<8>(arr.as_mut_ptr() as *mut u8, index, src); let mut expected = [0i64; 128]; for i in 0..8 { expected[i * 16] = (i + 1) as i64; @@ -7731,7 +7543,7 @@ mod tests { let index = _mm256_setr_epi32(0, 16, 32, 48, 64, 80, 96, 112); let src = _mm512_setr_epi64(1, 2, 3, 4, 5, 6, 7, 8); // A multiplier of 8 is word-addressing - _mm512_mask_i32scatter_epi64(arr.as_mut_ptr() as *mut u8, mask, index, src, 8); + _mm512_mask_i32scatter_epi64::<8>(arr.as_mut_ptr() as *mut u8, mask, index, src); let mut expected = [0i64; 128]; for i in 0..4 { expected[i * 32 + 16] = 2 * (i + 1) as i64; @@ -7745,7 +7557,7 @@ mod tests { let index = _mm512_setr_epi64(0, 16, 32, 48, 64, 80, 96, 112); let src = _mm512_setr_epi64(1, 2, 3, 4, 5, 6, 7, 8); // A multiplier of 8 is word-addressing - _mm512_i64scatter_epi64(arr.as_mut_ptr() as *mut u8, index, src, 8); + _mm512_i64scatter_epi64::<8>(arr.as_mut_ptr() as *mut u8, index, src); let mut expected = [0i64; 128]; for i in 0..8 { expected[i * 16] = (i + 1) as i64; @@ -7760,7 +7572,7 @@ mod tests { let index = _mm512_setr_epi64(0, 16, 32, 48, 64, 80, 96, 112); let src = _mm512_setr_epi64(1, 2, 3, 4, 5, 6, 7, 8); // A multiplier of 8 is word-addressing - _mm512_mask_i64scatter_epi64(arr.as_mut_ptr() as *mut u8, mask, index, src, 8); + _mm512_mask_i64scatter_epi64::<8>(arr.as_mut_ptr() as *mut u8, mask, index, src); let mut expected = [0i64; 128]; for i in 0..4 { expected[i * 32 + 16] = 2 * (i + 1) as i64; @@ -7774,7 +7586,7 @@ mod tests { let index = _mm512_setr_epi64(0, 16, 32, 48, 64, 80, 96, 112); let src = _mm256_setr_epi32(1, 2, 3, 4, 5, 6, 7, 8); // A multiplier of 4 is word-addressing - _mm512_i64scatter_epi32(arr.as_mut_ptr() as *mut u8, index, src, 4); + _mm512_i64scatter_epi32::<4>(arr.as_mut_ptr() as *mut u8, index, src); let mut expected = [0i32; 128]; for i in 0..8 { expected[i * 16] = (i + 1) as i32; @@ -7789,7 +7601,7 @@ mod tests { let index = _mm512_setr_epi64(0, 16, 32, 48, 64, 80, 96, 112); let src = _mm256_setr_epi32(1, 2, 3, 4, 5, 6, 7, 8); // A multiplier of 4 is word-addressing - _mm512_mask_i64scatter_epi32(arr.as_mut_ptr() as *mut u8, mask, index, src, 4); + _mm512_mask_i64scatter_epi32::<4>(arr.as_mut_ptr() as *mut u8, mask, index, src); let mut expected = [0i32; 128]; for i in 0..4 { expected[i * 32 + 16] = 2 * (i + 1) as i32; @@ -7804,7 +7616,7 @@ mod tests { 1 << 63, 1 << 32, 1 << 32, 1 << 32, 1 << 32, 1 << 32, 1 << 32, 1 << 32, ); - let r = _mm512_rol_epi64(a, 1); + let r = _mm512_rol_epi64::<1>(a); #[rustfmt::skip] let e = _mm512_set_epi64( 1 << 0, 1 << 33, 1 << 33, 1 << 33, @@ -7820,9 +7632,9 @@ mod tests { 1 << 63, 1 << 32, 1 << 32, 1 << 32, 1 << 32, 1 << 32, 1 << 32, 1 << 32, ); - let r = _mm512_mask_rol_epi64(a, 0, a, 1); + let r = _mm512_mask_rol_epi64::<1>(a, 0, a); assert_eq_m512i(r, a); - let r = _mm512_mask_rol_epi64(a, 0b11111111, a, 1); + let r = _mm512_mask_rol_epi64::<1>(a, 0b11111111, a); #[rustfmt::skip] let e = _mm512_set_epi64( 1 << 0, 1 << 33, 1 << 33, 1 << 33, @@ -7838,9 +7650,9 @@ mod tests { 1 << 32, 1 << 32, 1 << 32, 1 << 32, 1 << 32, 1 << 32, 1 << 32, 1 << 63, ); - let r = _mm512_maskz_rol_epi64(0, a, 1); + let r = _mm512_maskz_rol_epi64::<1>(0, a); assert_eq_m512i(r, _mm512_setzero_si512()); - let r = _mm512_maskz_rol_epi64(0b00001111, a, 1); + let r = _mm512_maskz_rol_epi64::<1>(0b00001111, a); let e = _mm512_set_epi64(0, 0, 0, 0, 1 << 33, 1 << 33, 1 << 33, 1 << 0); assert_eq_m512i(r, e); } @@ -7848,7 +7660,7 @@ mod tests { #[simd_test(enable = "avx512f,avx512vl")] unsafe fn test_mm256_rol_epi64() { let a = _mm256_set_epi64x(1 << 63, 1 << 32, 1 << 32, 1 << 32); - let r = _mm256_rol_epi64(a, 1); + let r = _mm256_rol_epi64::<1>(a); let e = _mm256_set_epi64x(1 << 0, 1 << 33, 1 << 33, 1 << 33); assert_eq_m256i(r, e); } @@ -7856,9 +7668,9 @@ mod tests { #[simd_test(enable = "avx512f,avx512vl")] unsafe fn test_mm256_mask_rol_epi64() { let a = _mm256_set_epi64x(1 << 63, 1 << 32, 1 << 32, 1 << 32); - let r = _mm256_mask_rol_epi64(a, 0, a, 1); + let r = _mm256_mask_rol_epi64::<1>(a, 0, a); assert_eq_m256i(r, a); - let r = _mm256_mask_rol_epi64(a, 0b00001111, a, 1); + let r = _mm256_mask_rol_epi64::<1>(a, 0b00001111, a); let e = _mm256_set_epi64x(1 << 0, 1 << 33, 1 << 33, 1 << 33); assert_eq_m256i(r, e); } @@ -7866,9 +7678,9 @@ mod tests { #[simd_test(enable = "avx512f,avx512vl")] unsafe fn test_mm256_maskz_rol_epi64() { let a = _mm256_set_epi64x(1 << 63, 1 << 32, 1 << 32, 1 << 32); - let r = _mm256_maskz_rol_epi64(0, a, 1); + let r = _mm256_maskz_rol_epi64::<1>(0, a); assert_eq_m256i(r, _mm256_setzero_si256()); - let r = _mm256_maskz_rol_epi64(0b00001111, a, 1); + let r = _mm256_maskz_rol_epi64::<1>(0b00001111, a); let e = _mm256_set_epi64x(1 << 0, 1 << 33, 1 << 33, 1 << 33); assert_eq_m256i(r, e); } @@ -7876,7 +7688,7 @@ mod tests { #[simd_test(enable = "avx512f,avx512vl")] unsafe fn test_mm_rol_epi64() { let a = _mm_set_epi64x(1 << 63, 1 << 32); - let r = _mm_rol_epi64(a, 1); + let r = _mm_rol_epi64::<1>(a); let e = _mm_set_epi64x(1 << 0, 1 << 33); assert_eq_m128i(r, e); } @@ -7884,9 +7696,9 @@ mod tests { #[simd_test(enable = "avx512f,avx512vl")] unsafe fn test_mm_mask_rol_epi64() { let a = _mm_set_epi64x(1 << 63, 1 << 32); - let r = _mm_mask_rol_epi64(a, 0, a, 1); + let r = _mm_mask_rol_epi64::<1>(a, 0, a); assert_eq_m128i(r, a); - let r = _mm_mask_rol_epi64(a, 0b00000011, a, 1); + let r = _mm_mask_rol_epi64::<1>(a, 0b00000011, a); let e = _mm_set_epi64x(1 << 0, 1 << 33); assert_eq_m128i(r, e); } @@ -7894,9 +7706,9 @@ mod tests { #[simd_test(enable = "avx512f,avx512vl")] unsafe fn test_mm_maskz_rol_epi64() { let a = _mm_set_epi64x(1 << 63, 1 << 32); - let r = _mm_maskz_rol_epi64(0, a, 1); + let r = _mm_maskz_rol_epi64::<1>(0, a); assert_eq_m128i(r, _mm_setzero_si128()); - let r = _mm_maskz_rol_epi64(0b00000011, a, 1); + let r = _mm_maskz_rol_epi64::<1>(0b00000011, a); let e = _mm_set_epi64x(1 << 0, 1 << 33); assert_eq_m128i(r, e); } @@ -7908,7 +7720,7 @@ mod tests { 1 << 0, 1 << 32, 1 << 32, 1 << 32, 1 << 32, 1 << 32, 1 << 32, 1 << 32, ); - let r = _mm512_ror_epi64(a, 1); + let r = _mm512_ror_epi64::<1>(a); #[rustfmt::skip] let e = _mm512_set_epi64( 1 << 63, 1 << 31, 1 << 31, 1 << 31, @@ -7924,9 +7736,9 @@ mod tests { 1 << 0, 1 << 32, 1 << 32, 1 << 32, 1 << 32, 1 << 32, 1 << 32, 1 << 32, ); - let r = _mm512_mask_ror_epi64(a, 0, a, 1); + let r = _mm512_mask_ror_epi64::<1>(a, 0, a); assert_eq_m512i(r, a); - let r = _mm512_mask_ror_epi64(a, 0b11111111, a, 1); + let r = _mm512_mask_ror_epi64::<1>(a, 0b11111111, a); #[rustfmt::skip] let e = _mm512_set_epi64( 1 << 63, 1 << 31, 1 << 31, 1 << 31, @@ -7942,9 +7754,9 @@ mod tests { 1 << 32, 1 << 32, 1 << 32, 1 << 32, 1 << 32, 1 << 32, 1 << 32, 1 << 0, ); - let r = _mm512_maskz_ror_epi64(0, a, 1); + let r = _mm512_maskz_ror_epi64::<1>(0, a); assert_eq_m512i(r, _mm512_setzero_si512()); - let r = _mm512_maskz_ror_epi64(0b00001111, a, 1); + let r = _mm512_maskz_ror_epi64::<1>(0b00001111, a); let e = _mm512_set_epi64(0, 0, 0, 0, 1 << 31, 1 << 31, 1 << 31, 1 << 63); assert_eq_m512i(r, e); } @@ -7952,7 +7764,7 @@ mod tests { #[simd_test(enable = "avx512f,avx512vl")] unsafe fn test_mm256_ror_epi64() { let a = _mm256_set_epi64x(1 << 0, 1 << 32, 1 << 32, 1 << 32); - let r = _mm256_ror_epi64(a, 1); + let r = _mm256_ror_epi64::<1>(a); let e = _mm256_set_epi64x(1 << 63, 1 << 31, 1 << 31, 1 << 31); assert_eq_m256i(r, e); } @@ -7960,9 +7772,9 @@ mod tests { #[simd_test(enable = "avx512f,avx512vl")] unsafe fn test_mm256_mask_ror_epi64() { let a = _mm256_set_epi64x(1 << 0, 1 << 32, 1 << 32, 1 << 32); - let r = _mm256_mask_ror_epi64(a, 0, a, 1); + let r = _mm256_mask_ror_epi64::<1>(a, 0, a); assert_eq_m256i(r, a); - let r = _mm256_mask_ror_epi64(a, 0b00001111, a, 1); + let r = _mm256_mask_ror_epi64::<1>(a, 0b00001111, a); let e = _mm256_set_epi64x(1 << 63, 1 << 31, 1 << 31, 1 << 31); assert_eq_m256i(r, e); } @@ -7970,9 +7782,9 @@ mod tests { #[simd_test(enable = "avx512f,avx512vl")] unsafe fn test_mm256_maskz_ror_epi64() { let a = _mm256_set_epi64x(1 << 0, 1 << 32, 1 << 32, 1 << 32); - let r = _mm256_maskz_ror_epi64(0, a, 1); + let r = _mm256_maskz_ror_epi64::<1>(0, a); assert_eq_m256i(r, _mm256_setzero_si256()); - let r = _mm256_maskz_ror_epi64(0b00001111, a, 1); + let r = _mm256_maskz_ror_epi64::<1>(0b00001111, a); let e = _mm256_set_epi64x(1 << 63, 1 << 31, 1 << 31, 1 << 31); assert_eq_m256i(r, e); } @@ -7980,7 +7792,7 @@ mod tests { #[simd_test(enable = "avx512f,avx512vl")] unsafe fn test_mm_ror_epi64() { let a = _mm_set_epi64x(1 << 0, 1 << 32); - let r = _mm_ror_epi64(a, 1); + let r = _mm_ror_epi64::<1>(a); let e = _mm_set_epi64x(1 << 63, 1 << 31); assert_eq_m128i(r, e); } @@ -7988,9 +7800,9 @@ mod tests { #[simd_test(enable = "avx512f,avx512vl")] unsafe fn test_mm_mask_ror_epi64() { let a = _mm_set_epi64x(1 << 0, 1 << 32); - let r = _mm_mask_ror_epi64(a, 0, a, 1); + let r = _mm_mask_ror_epi64::<1>(a, 0, a); assert_eq_m128i(r, a); - let r = _mm_mask_ror_epi64(a, 0b00000011, a, 1); + let r = _mm_mask_ror_epi64::<1>(a, 0b00000011, a); let e = _mm_set_epi64x(1 << 63, 1 << 31); assert_eq_m128i(r, e); } @@ -7998,9 +7810,9 @@ mod tests { #[simd_test(enable = "avx512f,avx512vl")] unsafe fn test_mm_maskz_ror_epi64() { let a = _mm_set_epi64x(1 << 0, 1 << 32); - let r = _mm_maskz_ror_epi64(0, a, 1); + let r = _mm_maskz_ror_epi64::<1>(0, a); assert_eq_m128i(r, _mm_setzero_si128()); - let r = _mm_maskz_ror_epi64(0b00000011, a, 1); + let r = _mm_maskz_ror_epi64::<1>(0b00000011, a); let e = _mm_set_epi64x(1 << 63, 1 << 31); assert_eq_m128i(r, e); } @@ -8012,7 +7824,7 @@ mod tests { 1 << 63, 1 << 32, 1 << 32, 1 << 32, 1 << 32, 1 << 32, 1 << 32, 1 << 32, ); - let r = _mm512_slli_epi64(a, 1); + let r = _mm512_slli_epi64::<1>(a); #[rustfmt::skip] let e = _mm512_set_epi64( 0, 1 << 33, 1 << 33, 1 << 33, @@ -8028,9 +7840,9 @@ mod tests { 1 << 63, 1 << 32, 1 << 32, 1 << 32, 1 << 32, 1 << 32, 1 << 32, 1 << 32, ); - let r = _mm512_mask_slli_epi64(a, 0, a, 1); + let r = _mm512_mask_slli_epi64::<1>(a, 0, a); assert_eq_m512i(r, a); - let r = _mm512_mask_slli_epi64(a, 0b11111111, a, 1); + let r = _mm512_mask_slli_epi64::<1>(a, 0b11111111, a); #[rustfmt::skip] let e = _mm512_set_epi64( 0, 1 << 33, 1 << 33, 1 << 33, @@ -8046,9 +7858,9 @@ mod tests { 1 << 32, 1 << 32, 1 << 32, 1 << 32, 1 << 32, 1 << 32, 1 << 32, 1 << 63, ); - let r = _mm512_maskz_slli_epi64(0, a, 1); + let r = _mm512_maskz_slli_epi64::<1>(0, a); assert_eq_m512i(r, _mm512_setzero_si512()); - let r = _mm512_maskz_slli_epi64(0b00001111, a, 1); + let r = _mm512_maskz_slli_epi64::<1>(0b00001111, a); let e = _mm512_set_epi64(0, 0, 0, 0, 1 << 33, 1 << 33, 1 << 33, 0); assert_eq_m512i(r, e); } @@ -8056,9 +7868,9 @@ mod tests { #[simd_test(enable = "avx512f,avx512vl")] unsafe fn test_mm256_mask_slli_epi64() { let a = _mm256_set_epi64x(1 << 63, 1 << 32, 1 << 32, 1 << 32); - let r = _mm256_mask_slli_epi64(a, 0, a, 1); + let r = _mm256_mask_slli_epi64::<1>(a, 0, a); assert_eq_m256i(r, a); - let r = _mm256_mask_slli_epi64(a, 0b00001111, a, 1); + let r = _mm256_mask_slli_epi64::<1>(a, 0b00001111, a); let e = _mm256_set_epi64x(0, 1 << 33, 1 << 33, 1 << 33); assert_eq_m256i(r, e); } @@ -8066,9 +7878,9 @@ mod tests { #[simd_test(enable = "avx512f,avx512vl")] unsafe fn test_mm256_maskz_slli_epi64() { let a = _mm256_set_epi64x(1 << 63, 1 << 32, 1 << 32, 1 << 32); - let r = _mm256_maskz_slli_epi64(0, a, 1); + let r = _mm256_maskz_slli_epi64::<1>(0, a); assert_eq_m256i(r, _mm256_setzero_si256()); - let r = _mm256_maskz_slli_epi64(0b00001111, a, 1); + let r = _mm256_maskz_slli_epi64::<1>(0b00001111, a); let e = _mm256_set_epi64x(0, 1 << 33, 1 << 33, 1 << 33); assert_eq_m256i(r, e); } @@ -8076,9 +7888,9 @@ mod tests { #[simd_test(enable = "avx512f,avx512vl")] unsafe fn test_mm_mask_slli_epi64() { let a = _mm_set_epi64x(1 << 63, 1 << 32); - let r = _mm_mask_slli_epi64(a, 0, a, 1); + let r = _mm_mask_slli_epi64::<1>(a, 0, a); assert_eq_m128i(r, a); - let r = _mm_mask_slli_epi64(a, 0b00000011, a, 1); + let r = _mm_mask_slli_epi64::<1>(a, 0b00000011, a); let e = _mm_set_epi64x(0, 1 << 33); assert_eq_m128i(r, e); } @@ -8086,9 +7898,9 @@ mod tests { #[simd_test(enable = "avx512f,avx512vl")] unsafe fn test_mm_maskz_slli_epi64() { let a = _mm_set_epi64x(1 << 63, 1 << 32); - let r = _mm_maskz_slli_epi64(0, a, 1); + let r = _mm_maskz_slli_epi64::<1>(0, a); assert_eq_m128i(r, _mm_setzero_si128()); - let r = _mm_maskz_slli_epi64(0b00000011, a, 1); + let r = _mm_maskz_slli_epi64::<1>(0b00000011, a); let e = _mm_set_epi64x(0, 1 << 33); assert_eq_m128i(r, e); } @@ -8100,7 +7912,7 @@ mod tests { 1 << 0, 1 << 32, 1 << 32, 1 << 32, 1 << 32, 1 << 32, 1 << 32, 1 << 32, ); - let r = _mm512_srli_epi64(a, 1); + let r = _mm512_srli_epi64::<1>(a); #[rustfmt::skip] let e = _mm512_set_epi64( 0, 1 << 31, 1 << 31, 1 << 31, @@ -8116,9 +7928,9 @@ mod tests { 1 << 0, 1 << 32, 1 << 32, 1 << 32, 1 << 32, 1 << 32, 1 << 32, 1 << 32, ); - let r = _mm512_mask_srli_epi64(a, 0, a, 1); + let r = _mm512_mask_srli_epi64::<1>(a, 0, a); assert_eq_m512i(r, a); - let r = _mm512_mask_srli_epi64(a, 0b11111111, a, 1); + let r = _mm512_mask_srli_epi64::<1>(a, 0b11111111, a); #[rustfmt::skip] let e = _mm512_set_epi64( 0, 1 << 31, 1 << 31, 1 << 31, @@ -8134,9 +7946,9 @@ mod tests { 1 << 32, 1 << 32, 1 << 32, 1 << 32, 1 << 32, 1 << 32, 1 << 32, 1 << 0, ); - let r = _mm512_maskz_srli_epi64(0, a, 1); + let r = _mm512_maskz_srli_epi64::<1>(0, a); assert_eq_m512i(r, _mm512_setzero_si512()); - let r = _mm512_maskz_srli_epi64(0b00001111, a, 1); + let r = _mm512_maskz_srli_epi64::<1>(0b00001111, a); let e = _mm512_set_epi64(0, 0, 0, 0, 1 << 31, 1 << 31, 1 << 31, 0); assert_eq_m512i(r, e); } @@ -8144,9 +7956,9 @@ mod tests { #[simd_test(enable = "avx512f,avx512vl")] unsafe fn test_mm256_mask_srli_epi64() { let a = _mm256_set_epi64x(1 << 5, 0, 0, 0); - let r = _mm256_mask_srli_epi64(a, 0, a, 1); + let r = _mm256_mask_srli_epi64::<1>(a, 0, a); assert_eq_m256i(r, a); - let r = _mm256_mask_srli_epi64(a, 0b00001111, a, 1); + let r = _mm256_mask_srli_epi64::<1>(a, 0b00001111, a); let e = _mm256_set_epi64x(1 << 4, 0, 0, 0); assert_eq_m256i(r, e); } @@ -8154,9 +7966,9 @@ mod tests { #[simd_test(enable = "avx512f,avx512vl")] unsafe fn test_mm256_maskz_srli_epi64() { let a = _mm256_set_epi64x(1 << 5, 0, 0, 0); - let r = _mm256_maskz_srli_epi64(0, a, 1); + let r = _mm256_maskz_srli_epi64::<1>(0, a); assert_eq_m256i(r, _mm256_setzero_si256()); - let r = _mm256_maskz_srli_epi64(0b00001111, a, 1); + let r = _mm256_maskz_srli_epi64::<1>(0b00001111, a); let e = _mm256_set_epi64x(1 << 4, 0, 0, 0); assert_eq_m256i(r, e); } @@ -8164,9 +7976,9 @@ mod tests { #[simd_test(enable = "avx512f,avx512vl")] unsafe fn test_mm_mask_srli_epi64() { let a = _mm_set_epi64x(1 << 5, 0); - let r = _mm_mask_srli_epi64(a, 0, a, 1); + let r = _mm_mask_srli_epi64::<1>(a, 0, a); assert_eq_m128i(r, a); - let r = _mm_mask_srli_epi64(a, 0b00000011, a, 1); + let r = _mm_mask_srli_epi64::<1>(a, 0b00000011, a); let e = _mm_set_epi64x(1 << 4, 0); assert_eq_m128i(r, e); } @@ -8174,9 +7986,9 @@ mod tests { #[simd_test(enable = "avx512f,avx512vl")] unsafe fn test_mm_maskz_srli_epi64() { let a = _mm_set_epi64x(1 << 5, 0); - let r = _mm_maskz_srli_epi64(0, a, 1); + let r = _mm_maskz_srli_epi64::<1>(0, a); assert_eq_m128i(r, _mm_setzero_si128()); - let r = _mm_maskz_srli_epi64(0b00000011, a, 1); + let r = _mm_maskz_srli_epi64::<1>(0b00000011, a); let e = _mm_set_epi64x(1 << 4, 0); assert_eq_m128i(r, e); } @@ -8979,7 +8791,7 @@ mod tests { #[simd_test(enable = "avx512f")] unsafe fn test_mm512_srai_epi64() { let a = _mm512_set_epi64(1, -4, 15, 0, 0, 0, 0, -16); - let r = _mm512_srai_epi64(a, 2); + let r = _mm512_srai_epi64::<2>(a); let e = _mm512_set_epi64(0, -1, 3, 0, 0, 0, 0, -4); assert_eq_m512i(r, e); } @@ -8987,9 +8799,9 @@ mod tests { #[simd_test(enable = "avx512f")] unsafe fn test_mm512_mask_srai_epi64() { let a = _mm512_set_epi64(1, -4, 15, 0, 0, 0, 0, -16); - let r = _mm512_mask_srai_epi64(a, 0, a, 2); + let r = _mm512_mask_srai_epi64::<2>(a, 0, a); assert_eq_m512i(r, a); - let r = _mm512_mask_srai_epi64(a, 0b11111111, a, 2); + let r = _mm512_mask_srai_epi64::<2>(a, 0b11111111, a); let e = _mm512_set_epi64(0, -1, 3, 0, 0, 0, 0, -4); assert_eq_m512i(r, e); } @@ -8997,9 +8809,9 @@ mod tests { #[simd_test(enable = "avx512f")] unsafe fn test_mm512_maskz_srai_epi64() { let a = _mm512_set_epi64(1, -4, 15, 0, 0, 0, 0, -16); - let r = _mm512_maskz_srai_epi64(0, a, 2); + let r = _mm512_maskz_srai_epi64::<2>(0, a); assert_eq_m512i(r, _mm512_setzero_si512()); - let r = _mm512_maskz_srai_epi64(0b00001111, a, 2); + let r = _mm512_maskz_srai_epi64::<2>(0b00001111, a); let e = _mm512_set_epi64(0, 0, 0, 0, 0, 0, 0, -4); assert_eq_m512i(r, e); } @@ -9007,7 +8819,7 @@ mod tests { #[simd_test(enable = "avx512f,avx512vl")] unsafe fn test_mm256_srai_epi64() { let a = _mm256_set_epi64x(1 << 5, 0, 0, 0); - let r = _mm256_srai_epi64(a, 1); + let r = _mm256_srai_epi64::<1>(a); let e = _mm256_set_epi64x(1 << 4, 0, 0, 0); assert_eq_m256i(r, e); } @@ -9015,9 +8827,9 @@ mod tests { #[simd_test(enable = "avx512f,avx512vl")] unsafe fn test_mm256_mask_srai_epi64() { let a = _mm256_set_epi64x(1 << 5, 0, 0, 0); - let r = _mm256_mask_srai_epi64(a, 0, a, 1); + let r = _mm256_mask_srai_epi64::<1>(a, 0, a); assert_eq_m256i(r, a); - let r = _mm256_mask_srai_epi64(a, 0b00001111, a, 1); + let r = _mm256_mask_srai_epi64::<1>(a, 0b00001111, a); let e = _mm256_set_epi64x(1 << 4, 0, 0, 0); assert_eq_m256i(r, e); } @@ -9025,9 +8837,9 @@ mod tests { #[simd_test(enable = "avx512f,avx512vl")] unsafe fn test_mm256_maskz_srai_epi64() { let a = _mm256_set_epi64x(1 << 5, 0, 0, 0); - let r = _mm256_maskz_srai_epi64(0, a, 1); + let r = _mm256_maskz_srai_epi64::<1>(0, a); assert_eq_m256i(r, _mm256_setzero_si256()); - let r = _mm256_maskz_srai_epi64(0b00001111, a, 1); + let r = _mm256_maskz_srai_epi64::<1>(0b00001111, a); let e = _mm256_set_epi64x(1 << 4, 0, 0, 0); assert_eq_m256i(r, e); } @@ -9035,7 +8847,7 @@ mod tests { #[simd_test(enable = "avx512f,avx512vl")] unsafe fn test_mm_srai_epi64() { let a = _mm_set_epi64x(1 << 5, 0); - let r = _mm_srai_epi64(a, 1); + let r = _mm_srai_epi64::<1>(a); let e = _mm_set_epi64x(1 << 4, 0); assert_eq_m128i(r, e); } @@ -9043,9 +8855,9 @@ mod tests { #[simd_test(enable = "avx512f,avx512vl")] unsafe fn test_mm_mask_srai_epi64() { let a = _mm_set_epi64x(1 << 5, 0); - let r = _mm_mask_srai_epi64(a, 0, a, 1); + let r = _mm_mask_srai_epi64::<1>(a, 0, a); assert_eq_m128i(r, a); - let r = _mm_mask_srai_epi64(a, 0b00000011, a, 1); + let r = _mm_mask_srai_epi64::<1>(a, 0b00000011, a); let e = _mm_set_epi64x(1 << 4, 0); assert_eq_m128i(r, e); } @@ -9053,9 +8865,9 @@ mod tests { #[simd_test(enable = "avx512f,avx512vl")] unsafe fn test_mm_maskz_srai_epi64() { let a = _mm_set_epi64x(1 << 5, 0); - let r = _mm_maskz_srai_epi64(0, a, 1); + let r = _mm_maskz_srai_epi64::<1>(0, a); assert_eq_m128i(r, _mm_setzero_si128()); - let r = _mm_maskz_srai_epi64(0b00000011, a, 1); + let r = _mm_maskz_srai_epi64::<1>(0b00000011, a); let e = _mm_set_epi64x(1 << 4, 0); assert_eq_m128i(r, e); } @@ -9063,7 +8875,7 @@ mod tests { #[simd_test(enable = "avx512f")] unsafe fn test_mm512_permute_pd() { let a = _mm512_setr_pd(0., 1., 2., 3., 4., 5., 6., 7.); - let r = _mm512_permute_pd(a, 0b1111); + let r = _mm512_permute_pd::<0b11_11_11_11>(a); let e = _mm512_setr_pd(1., 1., 3., 3., 5., 5., 7., 7.); assert_eq_m512d(r, e); } @@ -9071,9 +8883,9 @@ mod tests { #[simd_test(enable = "avx512f")] unsafe fn test_mm512_mask_permute_pd() { let a = _mm512_setr_pd(0., 1., 2., 3., 4., 5., 6., 7.); - let r = _mm512_mask_permute_pd(a, 0, a, 0b1111); + let r = _mm512_mask_permute_pd::<0b11_11_11_11>(a, 0, a); assert_eq_m512d(r, a); - let r = _mm512_mask_permute_pd(a, 0b11111111, a, 0b1111); + let r = _mm512_mask_permute_pd::<0b11_11_11_11>(a, 0b11111111, a); let e = _mm512_setr_pd(1., 1., 3., 3., 5., 5., 7., 7.); assert_eq_m512d(r, e); } @@ -9081,9 +8893,9 @@ mod tests { #[simd_test(enable = "avx512f")] unsafe fn test_mm512_maskz_permute_pd() { let a = _mm512_setr_pd(0., 1., 2., 3., 4., 5., 6., 7.); - let r = _mm512_maskz_permute_pd(0, a, 0b1111); + let r = _mm512_maskz_permute_pd::<0b11_11_11_11>(0, a); assert_eq_m512d(r, _mm512_setzero_pd()); - let r = _mm512_maskz_permute_pd(0b11111111, a, 0b1111); + let r = _mm512_maskz_permute_pd::<0b11_11_11_11>(0b11111111, a); let e = _mm512_setr_pd(1., 1., 3., 3., 5., 5., 7., 7.); assert_eq_m512d(r, e); } @@ -9091,9 +8903,9 @@ mod tests { #[simd_test(enable = "avx512f,avx512vl")] unsafe fn test_mm256_mask_permute_pd() { let a = _mm256_set_pd(3., 2., 1., 0.); - let r = _mm256_mask_permute_pd(a, 0, a, 0b1111); + let r = _mm256_mask_permute_pd::<0b11_11>(a, 0, a); assert_eq_m256d(r, a); - let r = _mm256_mask_permute_pd(a, 0b00001111, a, 0b1111); + let r = _mm256_mask_permute_pd::<0b11_11>(a, 0b00001111, a); let e = _mm256_set_pd(3., 3., 1., 1.); assert_eq_m256d(r, e); } @@ -9101,9 +8913,9 @@ mod tests { #[simd_test(enable = "avx512f,avx512vl")] unsafe fn test_mm256_maskz_permute_pd() { let a = _mm256_set_pd(3., 2., 1., 0.); - let r = _mm256_maskz_permute_pd(0, a, 0b1111); + let r = _mm256_maskz_permute_pd::<0b11_11>(0, a); assert_eq_m256d(r, _mm256_setzero_pd()); - let r = _mm256_maskz_permute_pd(0b00001111, a, 0b1111); + let r = _mm256_maskz_permute_pd::<0b11_11>(0b00001111, a); let e = _mm256_set_pd(3., 3., 1., 1.); assert_eq_m256d(r, e); } @@ -9111,9 +8923,9 @@ mod tests { #[simd_test(enable = "avx512f,avx512vl")] unsafe fn test_mm_mask_permute_pd() { let a = _mm_set_pd(1., 0.); - let r = _mm_mask_permute_pd(a, 0, a, 0b1111); + let r = _mm_mask_permute_pd::<0b11>(a, 0, a); assert_eq_m128d(r, a); - let r = _mm_mask_permute_pd(a, 0b00000011, a, 0b1111); + let r = _mm_mask_permute_pd::<0b11>(a, 0b00000011, a); let e = _mm_set_pd(1., 1.); assert_eq_m128d(r, e); } @@ -9121,9 +8933,9 @@ mod tests { #[simd_test(enable = "avx512f,avx512vl")] unsafe fn test_mm_maskz_permute_pd() { let a = _mm_set_pd(1., 0.); - let r = _mm_maskz_permute_pd(0, a, 0b1111); + let r = _mm_maskz_permute_pd::<0b11>(0, a); assert_eq_m128d(r, _mm_setzero_pd()); - let r = _mm_maskz_permute_pd(0b00000011, a, 0b1111); + let r = _mm_maskz_permute_pd::<0b11>(0b00000011, a); let e = _mm_set_pd(1., 1.); assert_eq_m128d(r, e); } @@ -9131,7 +8943,7 @@ mod tests { #[simd_test(enable = "avx512f")] unsafe fn test_mm512_permutex_epi64() { let a = _mm512_setr_epi64(0, 1, 2, 3, 4, 5, 6, 7); - let r = _mm512_permutex_epi64(a, 0b11111111); + let r = _mm512_permutex_epi64::<0b11_11_11_11>(a); let e = _mm512_setr_epi64(3, 3, 3, 3, 7, 7, 7, 7); assert_eq_m512i(r, e); } @@ -9139,9 +8951,9 @@ mod tests { #[simd_test(enable = "avx512f")] unsafe fn test_mm512_mask_permutex_epi64() { let a = _mm512_setr_epi64(0, 1, 2, 3, 4, 5, 6, 7); - let r = _mm512_mask_permutex_epi64(a, 0, a, 0b11111111); + let r = _mm512_mask_permutex_epi64::<0b11_11_11_11>(a, 0, a); assert_eq_m512i(r, a); - let r = _mm512_mask_permutex_epi64(a, 0b11111111, a, 0b11111111); + let r = _mm512_mask_permutex_epi64::<0b11_11_11_11>(a, 0b11111111, a); let e = _mm512_setr_epi64(3, 3, 3, 3, 7, 7, 7, 7); assert_eq_m512i(r, e); } @@ -9149,9 +8961,9 @@ mod tests { #[simd_test(enable = "avx512f")] unsafe fn test_mm512_maskz_permutex_epi64() { let a = _mm512_setr_epi64(0, 1, 2, 3, 4, 5, 6, 7); - let r = _mm512_maskz_permutex_epi64(0, a, 0b11111111); + let r = _mm512_maskz_permutex_epi64::<0b11_11_11_11>(0, a); assert_eq_m512i(r, _mm512_setzero_si512()); - let r = _mm512_maskz_permutex_epi64(0b11111111, a, 0b11111111); + let r = _mm512_maskz_permutex_epi64::<0b11_11_11_11>(0b11111111, a); let e = _mm512_setr_epi64(3, 3, 3, 3, 7, 7, 7, 7); assert_eq_m512i(r, e); } @@ -9159,7 +8971,7 @@ mod tests { #[simd_test(enable = "avx512f,avx512vl")] unsafe fn test_mm256_permutex_epi64() { let a = _mm256_set_epi64x(3, 2, 1, 0); - let r = _mm256_permutex_epi64(a, 0b11111111); + let r = _mm256_permutex_epi64::<0b11_11_11_11>(a); let e = _mm256_set_epi64x(3, 3, 3, 3); assert_eq_m256i(r, e); } @@ -9167,9 +8979,9 @@ mod tests { #[simd_test(enable = "avx512f,avx512vl")] unsafe fn test_mm256_mask_permutex_epi64() { let a = _mm256_set_epi64x(3, 2, 1, 0); - let r = _mm256_mask_permutex_epi64(a, 0, a, 0b11111111); + let r = _mm256_mask_permutex_epi64::<0b11_11_11_11>(a, 0, a); assert_eq_m256i(r, a); - let r = _mm256_mask_permutex_epi64(a, 0b00001111, a, 0b11111111); + let r = _mm256_mask_permutex_epi64::<0b11_11_11_11>(a, 0b00001111, a); let e = _mm256_set_epi64x(3, 3, 3, 3); assert_eq_m256i(r, e); } @@ -9177,9 +8989,9 @@ mod tests { #[simd_test(enable = "avx512f")] unsafe fn test_mm256_maskz_permutex_epi64() { let a = _mm256_set_epi64x(3, 2, 1, 0); - let r = _mm256_maskz_permutex_epi64(0, a, 0b11111111); + let r = _mm256_maskz_permutex_epi64::<0b11_11_11_11>(0, a); assert_eq_m256i(r, _mm256_setzero_si256()); - let r = _mm256_maskz_permutex_epi64(0b00001111, a, 0b11111111); + let r = _mm256_maskz_permutex_epi64::<0b11_11_11_11>(0b00001111, a); let e = _mm256_set_epi64x(3, 3, 3, 3); assert_eq_m256i(r, e); } @@ -9187,7 +8999,7 @@ mod tests { #[simd_test(enable = "avx512f")] unsafe fn test_mm512_permutex_pd() { let a = _mm512_setr_pd(0., 1., 2., 3., 4., 5., 6., 7.); - let r = _mm512_permutex_pd(a, 0b11111111); + let r = _mm512_permutex_pd::<0b11_11_11_11>(a); let e = _mm512_setr_pd(3., 3., 3., 3., 7., 7., 7., 7.); assert_eq_m512d(r, e); } @@ -9195,9 +9007,9 @@ mod tests { #[simd_test(enable = "avx512f")] unsafe fn test_mm512_mask_permutex_pd() { let a = _mm512_setr_pd(0., 1., 2., 3., 4., 5., 6., 7.); - let r = _mm512_mask_permutex_pd(a, 0, a, 0b11111111); + let r = _mm512_mask_permutex_pd::<0b11_11_11_11>(a, 0, a); assert_eq_m512d(r, a); - let r = _mm512_mask_permutex_pd(a, 0b11111111, a, 0b11111111); + let r = _mm512_mask_permutex_pd::<0b11_11_11_11>(a, 0b11111111, a); let e = _mm512_setr_pd(3., 3., 3., 3., 7., 7., 7., 7.); assert_eq_m512d(r, e); } @@ -9205,9 +9017,9 @@ mod tests { #[simd_test(enable = "avx512f")] unsafe fn test_mm512_maskz_permutex_pd() { let a = _mm512_setr_pd(0., 1., 2., 3., 4., 5., 6., 7.); - let r = _mm512_maskz_permutex_pd(0, a, 0b11111111); + let r = _mm512_maskz_permutex_pd::<0b11_11_11_11>(0, a); assert_eq_m512d(r, _mm512_setzero_pd()); - let r = _mm512_maskz_permutex_pd(0b11111111, a, 0b11111111); + let r = _mm512_maskz_permutex_pd::<0b11_11_11_11>(0b11111111, a); let e = _mm512_setr_pd(3., 3., 3., 3., 7., 7., 7., 7.); assert_eq_m512d(r, e); } @@ -9215,7 +9027,7 @@ mod tests { #[simd_test(enable = "avx512f,avx512vl")] unsafe fn test_mm256_permutex_pd() { let a = _mm256_set_pd(0., 1., 2., 3.); - let r = _mm256_permutex_pd(a, 0b11111111); + let r = _mm256_permutex_pd::<0b11_11_11_11>(a); let e = _mm256_set_pd(0., 0., 0., 0.); assert_eq_m256d(r, e); } @@ -9223,9 +9035,9 @@ mod tests { #[simd_test(enable = "avx512f,avx512vl")] unsafe fn test_mm256_mask_permutex_pd() { let a = _mm256_set_pd(0., 1., 2., 3.); - let r = _mm256_mask_permutex_pd(a, 0, a, 0b11111111); + let r = _mm256_mask_permutex_pd::<0b11_11_11_11>(a, 0, a); assert_eq_m256d(r, a); - let r = _mm256_mask_permutex_pd(a, 0b00001111, a, 0b11111111); + let r = _mm256_mask_permutex_pd::<0b11_11_11_11>(a, 0b00001111, a); let e = _mm256_set_pd(0., 0., 0., 0.); assert_eq_m256d(r, e); } @@ -9233,9 +9045,9 @@ mod tests { #[simd_test(enable = "avx512f,avx512vl")] unsafe fn test_mm256_maskz_permutex_pd() { let a = _mm256_set_pd(0., 1., 2., 3.); - let r = _mm256_maskz_permutex_pd(0, a, 0b11111111); + let r = _mm256_maskz_permutex_pd::<0b11_11_11_11>(0, a); assert_eq_m256d(r, _mm256_setzero_pd()); - let r = _mm256_maskz_permutex_pd(0b00001111, a, 0b11111111); + let r = _mm256_maskz_permutex_pd::<0b11_11_11_11>(0b00001111, a); let e = _mm256_set_pd(0., 0., 0., 0.); assert_eq_m256d(r, e); } @@ -9715,70 +9527,13 @@ mod tests { assert_eq_m128d(r, e); } - #[simd_test(enable = "avx512f")] - unsafe fn test_mm512_shuffle_pd() { - let a = _mm512_setr_pd(1., 4., 5., 8., 1., 4., 5., 8.); - let b = _mm512_setr_pd(2., 3., 6., 7., 2., 3., 6., 7.); - let r = _mm512_shuffle_pd( - a, - b, - 1 << 0 | 1 << 1 | 1 << 2 | 1 << 3 | 1 << 4 | 1 << 5 | 1 << 6 | 1 << 7, - ); - let e = _mm512_setr_pd(4., 3., 8., 7., 4., 3., 8., 7.); - assert_eq_m512d(r, e); - } - - #[simd_test(enable = "avx512f")] - unsafe fn test_mm512_mask_shuffle_pd() { - let a = _mm512_setr_pd(1., 4., 5., 8., 1., 4., 5., 8.); - let b = _mm512_setr_pd(2., 3., 6., 7., 2., 3., 6., 7.); - let r = _mm512_mask_shuffle_pd( - a, - 0, - a, - b, - 1 << 0 | 1 << 1 | 1 << 2 | 1 << 3 | 1 << 4 | 1 << 5 | 1 << 6 | 1 << 7, - ); - assert_eq_m512d(r, a); - let r = _mm512_mask_shuffle_pd( - a, - 0b11111111, - a, - b, - 1 << 0 | 1 << 1 | 1 << 2 | 1 << 3 | 1 << 4 | 1 << 5 | 1 << 6 | 1 << 7, - ); - let e = _mm512_setr_pd(4., 3., 8., 7., 4., 3., 8., 7.); - assert_eq_m512d(r, e); - } - - #[simd_test(enable = "avx512f")] - unsafe fn test_mm512_maskz_shuffle_pd() { - let a = _mm512_setr_pd(1., 4., 5., 8., 1., 4., 5., 8.); - let b = _mm512_setr_pd(2., 3., 6., 7., 2., 3., 6., 7.); - let r = _mm512_maskz_shuffle_pd( - 0, - a, - b, - 1 << 0 | 1 << 1 | 1 << 2 | 1 << 3 | 1 << 4 | 1 << 5 | 1 << 6 | 1 << 7, - ); - assert_eq_m512d(r, _mm512_setzero_pd()); - let r = _mm512_maskz_shuffle_pd( - 0b00001111, - a, - b, - 1 << 0 | 1 << 1 | 1 << 2 | 1 << 3 | 1 << 4 | 1 << 5 | 1 << 6 | 1 << 7, - ); - let e = _mm512_setr_pd(4., 3., 8., 7., 0., 0., 0., 0.); - assert_eq_m512d(r, e); - } - #[simd_test(enable = "avx512f,avx512vl")] unsafe fn test_mm256_mask_shuffle_pd() { let a = _mm256_set_pd(1., 4., 5., 8.); let b = _mm256_set_pd(2., 3., 6., 7.); - let r = _mm256_mask_shuffle_pd(a, 0, a, b, 1 << 0 | 1 << 1 | 1 << 2 | 1 << 3); + let r = _mm256_mask_shuffle_pd::<0b11_11_11_11>(a, 0, a, b); assert_eq_m256d(r, a); - let r = _mm256_mask_shuffle_pd(a, 0b00001111, a, b, 1 << 0 | 1 << 1 | 1 << 2 | 1 << 3); + let r = _mm256_mask_shuffle_pd::<0b11_11_11_11>(a, 0b00001111, a, b); let e = _mm256_set_pd(2., 1., 6., 5.); assert_eq_m256d(r, e); } @@ -9787,9 +9542,9 @@ mod tests { unsafe fn test_mm256_maskz_shuffle_pd() { let a = _mm256_set_pd(1., 4., 5., 8.); let b = _mm256_set_pd(2., 3., 6., 7.); - let r = _mm256_maskz_shuffle_pd(0, a, b, 1 << 0 | 1 << 1 | 1 << 2 | 1 << 3); + let r = _mm256_maskz_shuffle_pd::<0b11_11_11_11>(0, a, b); assert_eq_m256d(r, _mm256_setzero_pd()); - let r = _mm256_maskz_shuffle_pd(0b00001111, a, b, 1 << 0 | 1 << 1 | 1 << 2 | 1 << 3); + let r = _mm256_maskz_shuffle_pd::<0b11_11_11_11>(0b00001111, a, b); let e = _mm256_set_pd(2., 1., 6., 5.); assert_eq_m256d(r, e); } @@ -9798,9 +9553,9 @@ mod tests { unsafe fn test_mm_mask_shuffle_pd() { let a = _mm_set_pd(1., 4.); let b = _mm_set_pd(2., 3.); - let r = _mm_mask_shuffle_pd(a, 0, a, b, 1 << 0 | 1 << 1); + let r = _mm_mask_shuffle_pd::<0b11_11_11_11>(a, 0, a, b); assert_eq_m128d(r, a); - let r = _mm_mask_shuffle_pd(a, 0b00000011, a, b, 1 << 0 | 1 << 1); + let r = _mm_mask_shuffle_pd::<0b11_11_11_11>(a, 0b00000011, a, b); let e = _mm_set_pd(2., 1.); assert_eq_m128d(r, e); } @@ -9809,9 +9564,9 @@ mod tests { unsafe fn test_mm_maskz_shuffle_pd() { let a = _mm_set_pd(1., 4.); let b = _mm_set_pd(2., 3.); - let r = _mm_maskz_shuffle_pd(0, a, b, 1 << 0 | 1 << 1); + let r = _mm_maskz_shuffle_pd::<0b11_11_11_11>(0, a, b); assert_eq_m128d(r, _mm_setzero_pd()); - let r = _mm_maskz_shuffle_pd(0b00000011, a, b, 1 << 0 | 1 << 1); + let r = _mm_maskz_shuffle_pd::<0b11_11_11_11>(0b00000011, a, b); let e = _mm_set_pd(2., 1.); assert_eq_m128d(r, e); } @@ -9820,7 +9575,7 @@ mod tests { unsafe fn test_mm512_shuffle_i64x2() { let a = _mm512_setr_epi64(1, 4, 5, 8, 9, 12, 13, 16); let b = _mm512_setr_epi64(2, 3, 6, 7, 10, 11, 14, 15); - let r = _mm512_shuffle_i64x2(a, b, 0b00000000); + let r = _mm512_shuffle_i64x2::<0b00_00_00_00>(a, b); let e = _mm512_setr_epi64(1, 4, 1, 4, 2, 3, 2, 3); assert_eq_m512i(r, e); } @@ -9829,9 +9584,9 @@ mod tests { unsafe fn test_mm512_mask_shuffle_i64x2() { let a = _mm512_setr_epi64(1, 4, 5, 8, 9, 12, 13, 16); let b = _mm512_setr_epi64(2, 3, 6, 7, 10, 11, 14, 15); - let r = _mm512_mask_shuffle_i64x2(a, 0, a, b, 0b00000000); + let r = _mm512_mask_shuffle_i64x2::<0b00_00_00_00>(a, 0, a, b); assert_eq_m512i(r, a); - let r = _mm512_mask_shuffle_i64x2(a, 0b11111111, a, b, 0b00000000); + let r = _mm512_mask_shuffle_i64x2::<0b00_00_00_00>(a, 0b11111111, a, b); let e = _mm512_setr_epi64(1, 4, 1, 4, 2, 3, 2, 3); assert_eq_m512i(r, e); } @@ -9840,9 +9595,9 @@ mod tests { unsafe fn test_mm512_maskz_shuffle_i64x2() { let a = _mm512_setr_epi64(1, 4, 5, 8, 9, 12, 13, 16); let b = _mm512_setr_epi64(2, 3, 6, 7, 10, 11, 14, 15); - let r = _mm512_maskz_shuffle_i64x2(0, a, b, 0b00000000); + let r = _mm512_maskz_shuffle_i64x2::<0b00_00_00_00>(0, a, b); assert_eq_m512i(r, _mm512_setzero_si512()); - let r = _mm512_maskz_shuffle_i64x2(0b00001111, a, b, 0b00000000); + let r = _mm512_maskz_shuffle_i64x2::<0b00_00_00_00>(0b00001111, a, b); let e = _mm512_setr_epi64(1, 4, 1, 4, 0, 0, 0, 0); assert_eq_m512i(r, e); } @@ -9851,7 +9606,7 @@ mod tests { unsafe fn test_mm256_shuffle_i64x2() { let a = _mm256_set_epi64x(1, 4, 5, 8); let b = _mm256_set_epi64x(2, 3, 6, 7); - let r = _mm256_shuffle_i64x2(a, b, 0b00); + let r = _mm256_shuffle_i64x2::<0b00>(a, b); let e = _mm256_set_epi64x(6, 7, 5, 8); assert_eq_m256i(r, e); } @@ -9860,9 +9615,9 @@ mod tests { unsafe fn test_mm256_mask_shuffle_i64x2() { let a = _mm256_set_epi64x(1, 4, 5, 8); let b = _mm256_set_epi64x(2, 3, 6, 7); - let r = _mm256_mask_shuffle_i64x2(a, 0, a, b, 0b00); + let r = _mm256_mask_shuffle_i64x2::<0b00>(a, 0, a, b); assert_eq_m256i(r, a); - let r = _mm256_mask_shuffle_i64x2(a, 0b00001111, a, b, 0b00); + let r = _mm256_mask_shuffle_i64x2::<0b00>(a, 0b00001111, a, b); let e = _mm256_set_epi64x(6, 7, 5, 8); assert_eq_m256i(r, e); } @@ -9871,9 +9626,9 @@ mod tests { unsafe fn test_mm256_maskz_shuffle_i64x2() { let a = _mm256_set_epi64x(1, 4, 5, 8); let b = _mm256_set_epi64x(2, 3, 6, 7); - let r = _mm256_maskz_shuffle_i64x2(0, a, b, 0b00); + let r = _mm256_maskz_shuffle_i64x2::<0b00>(0, a, b); assert_eq_m256i(r, _mm256_setzero_si256()); - let r = _mm256_maskz_shuffle_i64x2(0b00001111, a, b, 0b00); + let r = _mm256_maskz_shuffle_i64x2::<0b00>(0b00001111, a, b); let e = _mm256_set_epi64x(6, 7, 5, 8); assert_eq_m256i(r, e); } @@ -9882,7 +9637,7 @@ mod tests { unsafe fn test_mm512_shuffle_f64x2() { let a = _mm512_setr_pd(1., 4., 5., 8., 9., 12., 13., 16.); let b = _mm512_setr_pd(2., 3., 6., 7., 10., 11., 14., 15.); - let r = _mm512_shuffle_f64x2(a, b, 0b00000000); + let r = _mm512_shuffle_f64x2::<0b00_00_00_00>(a, b); let e = _mm512_setr_pd(1., 4., 1., 4., 2., 3., 2., 3.); assert_eq_m512d(r, e); } @@ -9891,9 +9646,9 @@ mod tests { unsafe fn test_mm512_mask_shuffle_f64x2() { let a = _mm512_setr_pd(1., 4., 5., 8., 9., 12., 13., 16.); let b = _mm512_setr_pd(2., 3., 6., 7., 10., 11., 14., 15.); - let r = _mm512_mask_shuffle_f64x2(a, 0, a, b, 0b00000000); + let r = _mm512_mask_shuffle_f64x2::<0b00_00_00_00>(a, 0, a, b); assert_eq_m512d(r, a); - let r = _mm512_mask_shuffle_f64x2(a, 0b11111111, a, b, 0b00000000); + let r = _mm512_mask_shuffle_f64x2::<0b00_00_00_00>(a, 0b11111111, a, b); let e = _mm512_setr_pd(1., 4., 1., 4., 2., 3., 2., 3.); assert_eq_m512d(r, e); } @@ -9902,9 +9657,9 @@ mod tests { unsafe fn test_mm512_maskz_shuffle_f64x2() { let a = _mm512_setr_pd(1., 4., 5., 8., 9., 12., 13., 16.); let b = _mm512_setr_pd(2., 3., 6., 7., 10., 11., 14., 15.); - let r = _mm512_maskz_shuffle_f64x2(0, a, b, 0b00000000); + let r = _mm512_maskz_shuffle_f64x2::<0b00_00_00_00>(0, a, b); assert_eq_m512d(r, _mm512_setzero_pd()); - let r = _mm512_maskz_shuffle_f64x2(0b00001111, a, b, 0b00000000); + let r = _mm512_maskz_shuffle_f64x2::<0b00_00_00_00>(0b00001111, a, b); let e = _mm512_setr_pd(1., 4., 1., 4., 0., 0., 0., 0.); assert_eq_m512d(r, e); } @@ -9913,7 +9668,7 @@ mod tests { unsafe fn test_mm256_shuffle_f64x2() { let a = _mm256_set_pd(1., 4., 5., 8.); let b = _mm256_set_pd(2., 3., 6., 7.); - let r = _mm256_shuffle_f64x2(a, b, 0b00); + let r = _mm256_shuffle_f64x2::<0b00>(a, b); let e = _mm256_set_pd(6., 7., 5., 8.); assert_eq_m256d(r, e); } @@ -9922,9 +9677,9 @@ mod tests { unsafe fn test_mm256_mask_shuffle_f64x2() { let a = _mm256_set_pd(1., 4., 5., 8.); let b = _mm256_set_pd(2., 3., 6., 7.); - let r = _mm256_mask_shuffle_f64x2(a, 0, a, b, 0b00); + let r = _mm256_mask_shuffle_f64x2::<0b00>(a, 0, a, b); assert_eq_m256d(r, a); - let r = _mm256_mask_shuffle_f64x2(a, 0b00001111, a, b, 0b00); + let r = _mm256_mask_shuffle_f64x2::<0b00>(a, 0b00001111, a, b); let e = _mm256_set_pd(6., 7., 5., 8.); assert_eq_m256d(r, e); } @@ -9933,9 +9688,9 @@ mod tests { unsafe fn test_mm256_maskz_shuffle_f64x2() { let a = _mm256_set_pd(1., 4., 5., 8.); let b = _mm256_set_pd(2., 3., 6., 7.); - let r = _mm256_maskz_shuffle_f64x2(0, a, b, 0b00); + let r = _mm256_maskz_shuffle_f64x2::<0b00>(0, a, b); assert_eq_m256d(r, _mm256_setzero_pd()); - let r = _mm256_maskz_shuffle_f64x2(0b00001111, a, b, 0b00); + let r = _mm256_maskz_shuffle_f64x2::<0b00>(0b00001111, a, b); let e = _mm256_set_pd(6., 7., 5., 8.); assert_eq_m256d(r, e); } @@ -10012,7 +9767,7 @@ mod tests { unsafe fn test_mm512_inserti64x4() { let a = _mm512_setr_epi64(1, 2, 3, 4, 5, 6, 7, 8); let b = _mm256_setr_epi64x(17, 18, 19, 20); - let r = _mm512_inserti64x4(a, b, 1); + let r = _mm512_inserti64x4::<1>(a, b); let e = _mm512_setr_epi64(1, 2, 3, 4, 17, 18, 19, 20); assert_eq_m512i(r, e); } @@ -10021,9 +9776,9 @@ mod tests { unsafe fn test_mm512_mask_inserti64x4() { let a = _mm512_setr_epi64(1, 2, 3, 4, 5, 6, 7, 8); let b = _mm256_setr_epi64x(17, 18, 19, 20); - let r = _mm512_mask_inserti64x4(a, 0, a, b, 1); + let r = _mm512_mask_inserti64x4::<1>(a, 0, a, b); assert_eq_m512i(r, a); - let r = _mm512_mask_inserti64x4(a, 0b11111111, a, b, 1); + let r = _mm512_mask_inserti64x4::<1>(a, 0b11111111, a, b); let e = _mm512_setr_epi64(1, 2, 3, 4, 17, 18, 19, 20); assert_eq_m512i(r, e); } @@ -10032,9 +9787,9 @@ mod tests { unsafe fn test_mm512_maskz_inserti64x4() { let a = _mm512_setr_epi64(1, 2, 3, 4, 5, 6, 7, 8); let b = _mm256_setr_epi64x(17, 18, 19, 20); - let r = _mm512_maskz_inserti64x4(0, a, b, 1); + let r = _mm512_maskz_inserti64x4::<1>(0, a, b); assert_eq_m512i(r, _mm512_setzero_si512()); - let r = _mm512_maskz_inserti64x4(0b00001111, a, b, 1); + let r = _mm512_maskz_inserti64x4::<1>(0b00001111, a, b); let e = _mm512_setr_epi64(1, 2, 3, 4, 0, 0, 0, 0); assert_eq_m512i(r, e); } @@ -10043,7 +9798,7 @@ mod tests { unsafe fn test_mm512_insertf64x4() { let a = _mm512_setr_pd(1., 2., 3., 4., 5., 6., 7., 8.); let b = _mm256_setr_pd(17., 18., 19., 20.); - let r = _mm512_insertf64x4(a, b, 1); + let r = _mm512_insertf64x4::<1>(a, b); let e = _mm512_setr_pd(1., 2., 3., 4., 17., 18., 19., 20.); assert_eq_m512d(r, e); } @@ -10052,9 +9807,9 @@ mod tests { unsafe fn test_mm512_mask_insertf64x4() { let a = _mm512_setr_pd(1., 2., 3., 4., 5., 6., 7., 8.); let b = _mm256_setr_pd(17., 18., 19., 20.); - let r = _mm512_mask_insertf64x4(a, 0, a, b, 1); + let r = _mm512_mask_insertf64x4::<1>(a, 0, a, b); assert_eq_m512d(r, a); - let r = _mm512_mask_insertf64x4(a, 0b11111111, a, b, 1); + let r = _mm512_mask_insertf64x4::<1>(a, 0b11111111, a, b); let e = _mm512_setr_pd(1., 2., 3., 4., 17., 18., 19., 20.); assert_eq_m512d(r, e); } @@ -10063,9 +9818,9 @@ mod tests { unsafe fn test_mm512_maskz_insertf64x4() { let a = _mm512_setr_pd(1., 2., 3., 4., 5., 6., 7., 8.); let b = _mm256_setr_pd(17., 18., 19., 20.); - let r = _mm512_maskz_insertf64x4(0, a, b, 1); + let r = _mm512_maskz_insertf64x4::<1>(0, a, b); assert_eq_m512d(r, _mm512_setzero_pd()); - let r = _mm512_maskz_insertf64x4(0b00001111, a, b, 1); + let r = _mm512_maskz_insertf64x4::<1>(0b00001111, a, b); let e = _mm512_setr_pd(1., 2., 3., 4., 0., 0., 0., 0.); assert_eq_m512d(r, e); } @@ -10743,11 +10498,11 @@ mod tests { unsafe fn test_mm512_alignr_epi64() { let a = _mm512_set_epi64(8, 7, 6, 5, 4, 3, 2, 1); let b = _mm512_set_epi64(16, 15, 14, 13, 12, 11, 10, 9); - let r = _mm512_alignr_epi64(a, b, 0); + let r = _mm512_alignr_epi64::<0>(a, b); assert_eq_m512i(r, b); - let r = _mm512_alignr_epi64(a, b, 8); + let r = _mm512_alignr_epi64::<8>(a, b); assert_eq_m512i(r, b); - let r = _mm512_alignr_epi64(a, b, 1); + let r = _mm512_alignr_epi64::<1>(a, b); let e = _mm512_set_epi64(1, 16, 15, 14, 13, 12, 11, 10); assert_eq_m512i(r, e); } @@ -10756,9 +10511,9 @@ mod tests { unsafe fn test_mm512_mask_alignr_epi64() { let a = _mm512_set_epi64(8, 7, 6, 5, 4, 3, 2, 1); let b = _mm512_set_epi64(16, 15, 14, 13, 12, 11, 10, 9); - let r = _mm512_mask_alignr_epi64(a, 0, a, b, 1); + let r = _mm512_mask_alignr_epi64::<1>(a, 0, a, b); assert_eq_m512i(r, a); - let r = _mm512_mask_alignr_epi64(a, 0b11111111, a, b, 1); + let r = _mm512_mask_alignr_epi64::<1>(a, 0b11111111, a, b); let e = _mm512_set_epi64(1, 16, 15, 14, 13, 12, 11, 10); assert_eq_m512i(r, e); } @@ -10767,9 +10522,9 @@ mod tests { unsafe fn test_mm512_maskz_alignr_epi64() { let a = _mm512_set_epi64(8, 7, 6, 5, 4, 3, 2, 1); let b = _mm512_set_epi64(16, 15, 14, 13, 12, 11, 10, 9); - let r = _mm512_maskz_alignr_epi64(0, a, b, 1); + let r = _mm512_maskz_alignr_epi64::<1>(0, a, b); assert_eq_m512i(r, _mm512_setzero_si512()); - let r = _mm512_maskz_alignr_epi64(0b00001111, a, b, 1); + let r = _mm512_maskz_alignr_epi64::<1>(0b00001111, a, b); let e = _mm512_set_epi64(0, 0, 0, 0, 13, 12, 11, 10); assert_eq_m512i(r, e); } @@ -10778,10 +10533,10 @@ mod tests { unsafe fn test_mm256_alignr_epi64() { let a = _mm256_set_epi64x(4, 3, 2, 1); let b = _mm256_set_epi64x(8, 7, 6, 5); - let r = _mm256_alignr_epi64(a, b, 0); + let r = _mm256_alignr_epi64::<0>(a, b); let e = _mm256_set_epi64x(8, 7, 6, 5); assert_eq_m256i(r, e); - let r = _mm256_alignr_epi64(a, b, 6); + let r = _mm256_alignr_epi64::<6>(a, b); let e = _mm256_set_epi64x(6, 5, 4, 3); assert_eq_m256i(r, e); } @@ -10790,9 +10545,9 @@ mod tests { unsafe fn test_mm256_mask_alignr_epi64() { let a = _mm256_set_epi64x(4, 3, 2, 1); let b = _mm256_set_epi64x(8, 7, 6, 5); - let r = _mm256_mask_alignr_epi64(a, 0, a, b, 1); + let r = _mm256_mask_alignr_epi64::<1>(a, 0, a, b); assert_eq_m256i(r, a); - let r = _mm256_mask_alignr_epi64(a, 0b00001111, a, b, 0); + let r = _mm256_mask_alignr_epi64::<0>(a, 0b00001111, a, b); let e = _mm256_set_epi64x(8, 7, 6, 5); assert_eq_m256i(r, e); } @@ -10801,9 +10556,9 @@ mod tests { unsafe fn test_mm256_maskz_alignr_epi64() { let a = _mm256_set_epi64x(4, 3, 2, 1); let b = _mm256_set_epi64x(8, 7, 6, 5); - let r = _mm256_maskz_alignr_epi64(0, a, b, 1); + let r = _mm256_maskz_alignr_epi64::<1>(0, a, b); assert_eq_m256i(r, _mm256_setzero_si256()); - let r = _mm256_maskz_alignr_epi64(0b00001111, a, b, 0); + let r = _mm256_maskz_alignr_epi64::<0>(0b00001111, a, b); let e = _mm256_set_epi64x(8, 7, 6, 5); assert_eq_m256i(r, e); } @@ -10812,7 +10567,7 @@ mod tests { unsafe fn test_mm_alignr_epi64() { let a = _mm_set_epi64x(2, 1); let b = _mm_set_epi64x(4, 3); - let r = _mm_alignr_epi64(a, b, 0); + let r = _mm_alignr_epi64::<0>(a, b); let e = _mm_set_epi64x(4, 3); assert_eq_m128i(r, e); } @@ -10821,9 +10576,9 @@ mod tests { unsafe fn test_mm_mask_alignr_epi64() { let a = _mm_set_epi64x(2, 1); let b = _mm_set_epi64x(4, 3); - let r = _mm_mask_alignr_epi64(a, 0, a, b, 1); + let r = _mm_mask_alignr_epi64::<1>(a, 0, a, b); assert_eq_m128i(r, a); - let r = _mm_mask_alignr_epi64(a, 0b00000011, a, b, 0); + let r = _mm_mask_alignr_epi64::<0>(a, 0b00000011, a, b); let e = _mm_set_epi64x(4, 3); assert_eq_m128i(r, e); } @@ -10832,9 +10587,9 @@ mod tests { unsafe fn test_mm_maskz_alignr_epi64() { let a = _mm_set_epi64x(2, 1); let b = _mm_set_epi64x(4, 3); - let r = _mm_maskz_alignr_epi64(0, a, b, 1); + let r = _mm_maskz_alignr_epi64::<1>(0, a, b); assert_eq_m128i(r, _mm_setzero_si128()); - let r = _mm_maskz_alignr_epi64(0b00000011, a, b, 0); + let r = _mm_maskz_alignr_epi64::<0>(0b00000011, a, b); let e = _mm_set_epi64x(4, 3); assert_eq_m128i(r, e); } @@ -11398,7 +11153,7 @@ mod tests { #[simd_test(enable = "avx512f")] unsafe fn test_mm512_extractf64x4_pd() { let a = _mm512_setr_pd(1., 2., 3., 4., 5., 6., 7., 8.); - let r = _mm512_extractf64x4_pd(a, 0x1); + let r = _mm512_extractf64x4_pd::<1>(a); let e = _mm256_setr_pd(5., 6., 7., 8.); assert_eq_m256d(r, e); } @@ -11407,9 +11162,9 @@ mod tests { unsafe fn test_mm512_mask_extractf64x4_pd() { let a = _mm512_setr_pd(1., 2., 3., 4., 5., 6., 7., 8.); let src = _mm256_set1_pd(100.); - let r = _mm512_mask_extractf64x4_pd(src, 0, a, 0x1); + let r = _mm512_mask_extractf64x4_pd::<1>(src, 0, a); assert_eq_m256d(r, src); - let r = _mm512_mask_extractf64x4_pd(src, 0b11111111, a, 0x1); + let r = _mm512_mask_extractf64x4_pd::<1>(src, 0b11111111, a); let e = _mm256_setr_pd(5., 6., 7., 8.); assert_eq_m256d(r, e); } @@ -11417,9 +11172,9 @@ mod tests { #[simd_test(enable = "avx512f")] unsafe fn test_mm512_maskz_extractf64x4_pd() { let a = _mm512_setr_pd(1., 2., 3., 4., 5., 6., 7., 8.); - let r = _mm512_maskz_extractf64x4_pd(0, a, 0x1); + let r = _mm512_maskz_extractf64x4_pd::<1>(0, a); assert_eq_m256d(r, _mm256_setzero_pd()); - let r = _mm512_maskz_extractf64x4_pd(0b00000001, a, 0x1); + let r = _mm512_maskz_extractf64x4_pd::<1>(0b00000001, a); let e = _mm256_setr_pd(5., 0., 0., 0.); assert_eq_m256d(r, e); } @@ -11427,7 +11182,7 @@ mod tests { #[simd_test(enable = "avx512f")] unsafe fn test_mm512_extracti64x4_epi64() { let a = _mm512_setr_epi64(1, 2, 3, 4, 5, 6, 7, 8); - let r = _mm512_extracti64x4_epi64(a, 0x1); + let r = _mm512_extracti64x4_epi64::<0x1>(a); let e = _mm256_setr_epi64x(5, 6, 7, 8); assert_eq_m256i(r, e); } @@ -11436,9 +11191,9 @@ mod tests { unsafe fn test_mm512_mask_extracti64x4_epi64() { let a = _mm512_setr_epi64(1, 2, 3, 4, 5, 6, 7, 8); let src = _mm256_set1_epi64x(100); - let r = _mm512_mask_extracti64x4_epi64(src, 0, a, 0x1); + let r = _mm512_mask_extracti64x4_epi64::<0x1>(src, 0, a); assert_eq_m256i(r, src); - let r = _mm512_mask_extracti64x4_epi64(src, 0b11111111, a, 0x1); + let r = _mm512_mask_extracti64x4_epi64::<0x1>(src, 0b11111111, a); let e = _mm256_setr_epi64x(5, 6, 7, 8); assert_eq_m256i(r, e); } @@ -11446,9 +11201,9 @@ mod tests { #[simd_test(enable = "avx512f")] unsafe fn test_mm512_maskz_extracti64x4_epi64() { let a = _mm512_setr_epi64(1, 2, 3, 4, 5, 6, 7, 8); - let r = _mm512_maskz_extracti64x4_epi64(0, a, 0x1); + let r = _mm512_maskz_extracti64x4_epi64::<0x1>(0, a); assert_eq_m256i(r, _mm256_setzero_si256()); - let r = _mm512_maskz_extracti64x4_epi64(0b00000001, a, 0x1); + let r = _mm512_maskz_extracti64x4_epi64::<0x1>(0b00000001, a); let e = _mm256_setr_epi64x(5, 0, 0, 0); assert_eq_m256i(r, e); } @@ -12359,7 +12114,7 @@ mod tests { unsafe fn test_mm_cvt_roundi64_ss() { let a = _mm_set_ps(0., -0.5, 1., -1.5); let b: i64 = 9; - let r = _mm_cvt_roundi64_ss(a, b, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); + let r = _mm_cvt_roundi64_ss::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(a, b); let e = _mm_set_ps(0., -0.5, 1., 9.); assert_eq_m128(r, e); } @@ -12368,7 +12123,7 @@ mod tests { unsafe fn test_mm_cvt_roundsi64_ss() { let a = _mm_set_ps(0., -0.5, 1., -1.5); let b: i64 = 9; - let r = _mm_cvt_roundsi64_ss(a, b, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); + let r = _mm_cvt_roundsi64_ss::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(a, b); let e = _mm_set_ps(0., -0.5, 1., 9.); assert_eq_m128(r, e); } @@ -12394,7 +12149,7 @@ mod tests { #[simd_test(enable = "avx512f")] unsafe fn test_mm_cvt_roundsd_si64() { let a = _mm_set_pd(1., -1.5); - let r = _mm_cvt_roundsd_si64(a, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); + let r = _mm_cvt_roundsd_si64::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(a); let e: i64 = -1; assert_eq!(r, e); } @@ -12402,7 +12157,7 @@ mod tests { #[simd_test(enable = "avx512f")] unsafe fn test_mm_cvt_roundsd_i64() { let a = _mm_set_pd(1., -1.5); - let r = _mm_cvt_roundsd_i64(a, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); + let r = _mm_cvt_roundsd_i64::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(a); let e: i64 = -1; assert_eq!(r, e); } @@ -12410,7 +12165,7 @@ mod tests { #[simd_test(enable = "avx512f")] unsafe fn test_mm_cvt_roundsd_u64() { let a = _mm_set_pd(1., f64::MAX); - let r = _mm_cvt_roundsd_u64(a, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); + let r = _mm_cvt_roundsd_u64::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(a); let e: u64 = u64::MAX; assert_eq!(r, e); } @@ -12426,7 +12181,7 @@ mod tests { #[simd_test(enable = "avx512f")] unsafe fn test_mm_cvt_roundss_i64() { let a = _mm_set_ps(0., -0.5, 1., -1.5); - let r = _mm_cvt_roundss_i64(a, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); + let r = _mm_cvt_roundss_i64::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(a); let e: i64 = -1; assert_eq!(r, e); } @@ -12434,7 +12189,7 @@ mod tests { #[simd_test(enable = "avx512f")] unsafe fn test_mm_cvt_roundss_si64() { let a = _mm_set_ps(0., -0.5, 1., -1.5); - let r = _mm_cvt_roundss_si64(a, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); + let r = _mm_cvt_roundss_si64::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(a); let e: i64 = -1; assert_eq!(r, e); } @@ -12442,7 +12197,7 @@ mod tests { #[simd_test(enable = "avx512f")] unsafe fn test_mm_cvt_roundss_u64() { let a = _mm_set_ps(0., -0.5, 1., -1.5); - let r = _mm_cvt_roundss_u64(a, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); + let r = _mm_cvt_roundss_u64::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(a); let e: u64 = u64::MAX; assert_eq!(r, e); } @@ -12466,7 +12221,7 @@ mod tests { #[simd_test(enable = "avx512f")] unsafe fn test_mm_cvtt_roundsd_i64() { let a = _mm_set_pd(1., -1.5); - let r = _mm_cvtt_roundsd_i64(a, _MM_FROUND_CUR_DIRECTION); + let r = _mm_cvtt_roundsd_i64::<_MM_FROUND_CUR_DIRECTION>(a); let e: i64 = -2; assert_eq!(r, e); } @@ -12474,7 +12229,7 @@ mod tests { #[simd_test(enable = "avx512f")] unsafe fn test_mm_cvtt_roundsd_si64() { let a = _mm_set_pd(1., -1.5); - let r = _mm_cvtt_roundsd_si64(a, _MM_FROUND_CUR_DIRECTION); + let r = _mm_cvtt_roundsd_si64::<_MM_FROUND_CUR_DIRECTION>(a); let e: i64 = -2; assert_eq!(r, e); } @@ -12482,7 +12237,7 @@ mod tests { #[simd_test(enable = "avx512f")] unsafe fn test_mm_cvtt_roundsd_u64() { let a = _mm_set_pd(1., -1.5); - let r = _mm_cvtt_roundsd_u64(a, _MM_FROUND_CUR_DIRECTION); + let r = _mm_cvtt_roundsd_u64::<_MM_FROUND_CUR_DIRECTION>(a); let e: u64 = u64::MAX; assert_eq!(r, e); } @@ -12506,7 +12261,7 @@ mod tests { #[simd_test(enable = "avx512f")] unsafe fn test_mm_cvtt_roundss_i64() { let a = _mm_set_ps(0., -0.5, 1., -1.5); - let r = _mm_cvtt_roundss_i64(a, _MM_FROUND_CUR_DIRECTION); + let r = _mm_cvtt_roundss_i64::<_MM_FROUND_CUR_DIRECTION>(a); let e: i64 = -2; assert_eq!(r, e); } @@ -12514,7 +12269,7 @@ mod tests { #[simd_test(enable = "avx512f")] unsafe fn test_mm_cvtt_roundss_si64() { let a = _mm_set_ps(0., -0.5, 1., -1.5); - let r = _mm_cvtt_roundss_si64(a, _MM_FROUND_CUR_DIRECTION); + let r = _mm_cvtt_roundss_si64::<_MM_FROUND_CUR_DIRECTION>(a); let e: i64 = -2; assert_eq!(r, e); } @@ -12522,7 +12277,7 @@ mod tests { #[simd_test(enable = "avx512f")] unsafe fn test_mm_cvtt_roundss_u64() { let a = _mm_set_ps(0., -0.5, 1., -1.5); - let r = _mm_cvtt_roundss_u64(a, _MM_FROUND_CUR_DIRECTION); + let r = _mm_cvtt_roundss_u64::<_MM_FROUND_CUR_DIRECTION>(a); let e: u64 = u64::MAX; assert_eq!(r, e); } @@ -12557,7 +12312,7 @@ mod tests { unsafe fn test_mm_cvt_roundu64_ss() { let a = _mm_set_ps(0., -0.5, 1., -1.5); let b: u64 = 9; - let r = _mm_cvt_roundu64_ss(a, b, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); + let r = _mm_cvt_roundu64_ss::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(a, b); let e = _mm_set_ps(0., -0.5, 1., 9.); assert_eq_m128(r, e); } @@ -12566,7 +12321,7 @@ mod tests { unsafe fn test_mm_cvt_roundu64_sd() { let a = _mm_set_pd(1., -1.5); let b: u64 = 9; - let r = _mm_cvt_roundu64_sd(a, b, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); + let r = _mm_cvt_roundu64_sd::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(a, b); let e = _mm_set_pd(1., 9.); assert_eq_m128d(r, e); } @@ -12575,7 +12330,7 @@ mod tests { unsafe fn test_mm_cvt_roundi64_sd() { let a = _mm_set_pd(1., -1.5); let b: i64 = 9; - let r = _mm_cvt_roundi64_sd(a, b, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); + let r = _mm_cvt_roundi64_sd::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(a, b); let e = _mm_set_pd(1., 9.); assert_eq_m128d(r, e); } @@ -12584,7 +12339,7 @@ mod tests { unsafe fn test_mm_cvt_roundsi64_sd() { let a = _mm_set_pd(1., -1.5); let b: i64 = 9; - let r = _mm_cvt_roundsi64_sd(a, b, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); + let r = _mm_cvt_roundsi64_sd::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(a, b); let e = _mm_set_pd(1., 9.); assert_eq_m128d(r, e); } diff --git a/library/stdarch/crates/core_arch/src/x86_64/bt.rs b/library/stdarch/crates/core_arch/src/x86_64/bt.rs index f0b16baab6..6e8579fa25 100644 --- a/library/stdarch/crates/core_arch/src/x86_64/bt.rs +++ b/library/stdarch/crates/core_arch/src/x86_64/bt.rs @@ -1,16 +1,36 @@ #[cfg(test)] use stdarch_test::assert_instr; +// x32 wants to use a 32-bit address size, but asm! defaults to using the full +// register name (e.g. rax). We have to explicitly override the placeholder to +// use the 32-bit register name in that case. +#[cfg(target_pointer_width = "32")] +macro_rules! bt { + ($inst:expr) => { + concat!($inst, " {b}, ({p:e})") + }; +} +#[cfg(target_pointer_width = "64")] +macro_rules! bt { + ($inst:expr) => { + concat!($inst, " {b}, ({p})") + }; +} + /// Returns the bit in position `b` of the memory addressed by `p`. #[inline] #[cfg_attr(test, assert_instr(bt))] #[unstable(feature = "simd_x86_bittest", issue = "59414")] pub unsafe fn _bittest64(p: *const i64, b: i64) -> u8 { let r: u8; - llvm_asm!("btq $2, $1\n\tsetc ${0:b}" - : "=r"(r) - : "*m"(p), "r"(b) - : "cc", "memory"); + asm!( + bt!("btq"), + "setc {r}", + p = in(reg) p, + b = in(reg) b, + r = out(reg_byte) r, + options(readonly, nostack, pure, att_syntax) + ); r } @@ -20,10 +40,14 @@ pub unsafe fn _bittest64(p: *const i64, b: i64) -> u8 { #[unstable(feature = "simd_x86_bittest", issue = "59414")] pub unsafe fn _bittestandset64(p: *mut i64, b: i64) -> u8 { let r: u8; - llvm_asm!("btsq $2, $1\n\tsetc ${0:b}" - : "=r"(r), "+*m"(p) - : "r"(b) - : "cc", "memory"); + asm!( + bt!("btsq"), + "setc {r}", + p = in(reg) p, + b = in(reg) b, + r = out(reg_byte) r, + options(nostack, att_syntax) + ); r } @@ -33,10 +57,14 @@ pub unsafe fn _bittestandset64(p: *mut i64, b: i64) -> u8 { #[unstable(feature = "simd_x86_bittest", issue = "59414")] pub unsafe fn _bittestandreset64(p: *mut i64, b: i64) -> u8 { let r: u8; - llvm_asm!("btrq $2, $1\n\tsetc ${0:b}" - : "=r"(r), "+*m"(p) - : "r"(b) - : "cc", "memory"); + asm!( + bt!("btrq"), + "setc {r}", + p = in(reg) p, + b = in(reg) b, + r = out(reg_byte) r, + options(nostack, att_syntax) + ); r } @@ -46,10 +74,14 @@ pub unsafe fn _bittestandreset64(p: *mut i64, b: i64) -> u8 { #[unstable(feature = "simd_x86_bittest", issue = "59414")] pub unsafe fn _bittestandcomplement64(p: *mut i64, b: i64) -> u8 { let r: u8; - llvm_asm!("btcq $2, $1\n\tsetc ${0:b}" - : "=r"(r), "+*m"(p) - : "r"(b) - : "cc", "memory"); + asm!( + bt!("btcq"), + "setc {r}", + p = in(reg) p, + b = in(reg) b, + r = out(reg_byte) r, + options(nostack, att_syntax) + ); r } diff --git a/library/stdarch/crates/core_arch/src/x86_64/macros.rs b/library/stdarch/crates/core_arch/src/x86_64/macros.rs index e3682d40fe..a3ea0e8216 100644 --- a/library/stdarch/crates/core_arch/src/x86_64/macros.rs +++ b/library/stdarch/crates/core_arch/src/x86_64/macros.rs @@ -1,32 +1,36 @@ //! Utility macros. -// For round instructions, the only valid values for rounding are 4, 8, 9, 10 and 11. -// This macro enforces that. +// Helper struct used to trigger const eval errors when the const generic immediate value `imm` is +// not a round number. +pub(crate) struct ValidateConstRound; +impl ValidateConstRound { + pub(crate) const VALID: () = { + assert!( + IMM == 4 || IMM == 8 || IMM == 9 || IMM == 10 || IMM == 11, + "Invalid IMM value" + ); + }; +} + #[allow(unused)] -macro_rules! constify_imm4_round { - ($imm8:expr, $expand:ident) => { - #[allow(overflowing_literals)] - match ($imm8) & 0b1111 { - 4 => $expand!(4), - 8 => $expand!(8), - 9 => $expand!(9), - 10 => $expand!(10), - 11 => $expand!(11), - _ => panic!("Invalid round value"), - } +macro_rules! static_assert_rounding { + ($imm:ident) => { + let _ = $crate::core_arch::x86_64::macros::ValidateConstRound::<$imm>::VALID; + }; +} + +// Helper struct used to trigger const eval errors when the const generic immediate value `imm` is +// not a sae number. +pub(crate) struct ValidateConstSae; +impl ValidateConstSae { + pub(crate) const VALID: () = { + assert!(IMM == 4 || IMM == 8, "Invalid IMM value"); }; } -// For sae instructions, the only valid values for sae are 4 and 8. -// This macro enforces that. #[allow(unused)] -macro_rules! constify_imm4_sae { - ($imm8:expr, $expand:ident) => { - #[allow(overflowing_literals)] - match ($imm8) & 0b1111 { - 4 => $expand!(4), - 8 => $expand!(8), - _ => panic!("Invalid sae value"), - } +macro_rules! static_assert_sae { + ($imm:ident) => { + let _ = $crate::core_arch::x86_64::macros::ValidateConstSae::<$imm>::VALID; }; } diff --git a/library/stdarch/crates/core_arch/src/x86_64/sse41.rs b/library/stdarch/crates/core_arch/src/x86_64/sse41.rs index 1b37967325..3d1ea0cf65 100644 --- a/library/stdarch/crates/core_arch/src/x86_64/sse41.rs +++ b/library/stdarch/crates/core_arch/src/x86_64/sse41.rs @@ -8,37 +8,31 @@ use crate::{ #[cfg(test)] use stdarch_test::assert_instr; -/// Extracts an 64-bit integer from `a` selected with `imm8` +/// Extracts an 64-bit integer from `a` selected with `IMM1` /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_extract_epi64) #[inline] #[target_feature(enable = "sse4.1")] -#[cfg_attr(all(test, not(target_os = "windows")), assert_instr(pextrq, imm8 = 1))] -#[rustc_args_required_const(1)] +#[cfg_attr(all(test, not(target_os = "windows")), assert_instr(pextrq, IMM1 = 1))] +#[rustc_legacy_const_generics(1)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm_extract_epi64(a: __m128i, imm8: i32) -> i64 { - let a = a.as_i64x2(); - match imm8 & 1 { - 0 => simd_extract(a, 0), - _ => simd_extract(a, 1), - } +pub unsafe fn _mm_extract_epi64(a: __m128i) -> i64 { + static_assert_imm1!(IMM1); + simd_extract(a.as_i64x2(), IMM1 as u32) } /// Returns a copy of `a` with the 64-bit integer from `i` inserted at a -/// location specified by `imm8`. +/// location specified by `IMM1`. /// /// [Intel's documentation](https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_insert_epi64) #[inline] #[target_feature(enable = "sse4.1")] -#[cfg_attr(test, assert_instr(pinsrq, imm8 = 0))] -#[rustc_args_required_const(2)] +#[cfg_attr(test, assert_instr(pinsrq, IMM1 = 0))] +#[rustc_legacy_const_generics(2)] #[stable(feature = "simd_x86", since = "1.27.0")] -pub unsafe fn _mm_insert_epi64(a: __m128i, i: i64, imm8: i32) -> __m128i { - let a = a.as_i64x2(); - match imm8 & 1 { - 0 => transmute(simd_insert(a, 0, i)), - _ => transmute(simd_insert(a, 1, i)), - } +pub unsafe fn _mm_insert_epi64(a: __m128i, i: i64) -> __m128i { + static_assert_imm1!(IMM1); + transmute(simd_insert(a.as_i64x2(), IMM1 as u32, i)) } #[cfg(test)] @@ -49,19 +43,20 @@ mod tests { #[simd_test(enable = "sse4.1")] unsafe fn test_mm_extract_epi64() { let a = _mm_setr_epi64x(0, 1); - let r = _mm_extract_epi64(a, 1); - assert_eq!(r, 1); - let r = _mm_extract_epi64(a, 3); + let r = _mm_extract_epi64::<1>(a); assert_eq!(r, 1); + let r = _mm_extract_epi64::<0>(a); + assert_eq!(r, 0); } #[simd_test(enable = "sse4.1")] unsafe fn test_mm_insert_epi64() { let a = _mm_set1_epi64x(0); let e = _mm_setr_epi64x(0, 32); - let r = _mm_insert_epi64(a, 32, 1); + let r = _mm_insert_epi64::<1>(a, 32); assert_eq_m128i(r, e); - let r = _mm_insert_epi64(a, 32, 3); + let e = _mm_setr_epi64x(32, 0); + let r = _mm_insert_epi64::<0>(a, 32); assert_eq_m128i(r, e); } } diff --git a/library/stdarch/crates/simd-test-macro/Cargo.toml b/library/stdarch/crates/simd-test-macro/Cargo.toml index 039fe8c768..c3ecf981e6 100644 --- a/library/stdarch/crates/simd-test-macro/Cargo.toml +++ b/library/stdarch/crates/simd-test-macro/Cargo.toml @@ -2,6 +2,7 @@ name = "simd-test-macro" version = "0.1.0" authors = ["Alex Crichton "] +edition = "2018" [lib] proc-macro = true diff --git a/library/stdarch/crates/simd-test-macro/src/lib.rs b/library/stdarch/crates/simd-test-macro/src/lib.rs index 4c863ae701..407c154475 100644 --- a/library/stdarch/crates/simd-test-macro/src/lib.rs +++ b/library/stdarch/crates/simd-test-macro/src/lib.rs @@ -4,8 +4,6 @@ //! for the appropriate cfg before calling the inner test function. #![deny(rust_2018_idioms)] -extern crate proc_macro; -extern crate proc_macro2; #[macro_use] extern crate quote; diff --git a/library/stdarch/crates/std_detect/src/detect/arch/aarch64.rs b/library/stdarch/crates/std_detect/src/detect/arch/aarch64.rs index 761cde9583..81979ba967 100644 --- a/library/stdarch/crates/std_detect/src/detect/arch/aarch64.rs +++ b/library/stdarch/crates/std_detect/src/detect/arch/aarch64.rs @@ -4,35 +4,145 @@ features! { @TARGET: aarch64; @MACRO_NAME: is_aarch64_feature_detected; @MACRO_ATTRS: - /// Checks if `aarch64` feature is enabled. + /// This macro tests, at runtime, whether an `aarch64` feature is enabled on aarch64 platforms. + /// Currently most features are only supported on linux-based platforms. + /// + /// This macro takes one argument which is a string literal of the feature being tested for. + /// The feature names are mostly taken from their FEAT_* definitiions in the [ARM Architecture + /// Reference Manual][docs]. + /// + /// ## Supported arguments + /// + /// * `"asimd"` or "neon" - FEAT_AdvSIMD + /// * `"pmull"` - FEAT_PMULL + /// * `"fp"` - FEAT_FP + /// * `"fp16"` - FEAT_FP16 + /// * `"sve"` - FEAT_SVE + /// * `"crc"` - FEAT_CRC + /// * `"lse"` - FEAT_LSE + /// * `"lse2"` - FEAT_LSE2 + /// * `"rdm"` - FEAT_RDM + /// * `"rcpc"` - FEAT_LRCPC + /// * `"rcpc2"` - FEAT_LRCPC2 + /// * `"dotprod"` - FEAT_DotProd + /// * `"tme"` - FEAT_TME + /// * `"fhm"` - FEAT_FHM + /// * `"dit"` - FEAT_DIT + /// * `"flagm"` - FEAT_FLAGM + /// * `"ssbs"` - FEAT_SSBS + /// * `"sb"` - FEAT_SB + /// * `"pauth"` - FEAT_PAuth + /// * `"dpb"` - FEAT_DPB + /// * `"dpb2"` - FEAT_DPB2 + /// * `"sve2"` - FEAT_SVE2 + /// * `"sve2-aes"` - FEAT_SVE2_AES + /// * `"sve2-sm4"` - FEAT_SVE2_SM4 + /// * `"sve2-sha3"` - FEAT_SVE2_SHA3 + /// * `"sve2-bitperm"` - FEAT_SVE2_BitPerm + /// * `"frintts"` - FEAT_FRINTTS + /// * `"i8mm"` - FEAT_I8MM + /// * `"f32mm"` - FEAT_F32MM + /// * `"f64mm"` - FEAT_F64MM + /// * `"bf16"` - FEAT_BF16 + /// * `"rand"` - FEAT_RNG + /// * `"bti"` - FEAT_BTI + /// * `"mte"` - FEAT_MTE + /// * `"jsconv"` - FEAT_JSCVT + /// * `"fcma"` - FEAT_FCMA + /// * `"aes"` - FEAT_AES + /// * `"sha2"` - FEAT_SHA1 & FEAT_SHA256 + /// * `"sha3"` - FEAT_SHA512 & FEAT_SHA3 + /// * `"sm4"` - FEAT_SM3 & FEAT_SM4 + /// + /// [docs]: https://developer.arm.com/documentation/ddi0487/latest #[unstable(feature = "stdsimd", issue = "27731")] @BIND_FEATURE_NAME: "asimd"; "neon"; @NO_RUNTIME_DETECTION: "ras"; @NO_RUNTIME_DETECTION: "v8.1a"; @NO_RUNTIME_DETECTION: "v8.2a"; @NO_RUNTIME_DETECTION: "v8.3a"; + @NO_RUNTIME_DETECTION: "v8.4a"; + @NO_RUNTIME_DETECTION: "v8.5a"; + @NO_RUNTIME_DETECTION: "v8.6a"; + @NO_RUNTIME_DETECTION: "v8.7a"; @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] asimd: "neon"; - /// ARM Advanced SIMD (ASIMD) + /// FEAT_AdvSIMD (Advanced SIMD/NEON) @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] pmull: "pmull"; - /// Polynomial Multiply + /// FEAT_PMULL (Polynomial Multiply) @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] fp: "fp"; - /// Floating point support + /// FEAT_FP (Floating point support) @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] fp16: "fp16"; - /// Half-float support. + /// FEAT_FP16 (Half-float support) @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] sve: "sve"; - /// Scalable Vector Extension (SVE) + /// FEAT_SVE (Scalable Vector Extension) @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] crc: "crc"; - /// CRC32 (Cyclic Redundancy Check) - @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] crypto: "crypto"; - /// Crypto: AES + PMULL + SHA1 + SHA2 + /// FEAT_CRC32 (Cyclic Redundancy Check) @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] lse: "lse"; - /// Atomics (Large System Extension) + /// FEAT_LSE (Large System Extension - atomics) + @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] lse2: "lse2"; + /// FEAT_LSE2 (unaligned and register-pair atomics) @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] rdm: "rdm"; - /// Rounding Double Multiply (ASIMDRDM) + /// FEAT_RDM (Rounding Doubling Multiply - ASIMDRDM) @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] rcpc: "rcpc"; - /// Release consistent Processor consistent (RcPc) + /// FEAT_LRCPC (Release consistent Processor consistent) + @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] rcpc2: "rcpc2"; + /// FEAT_LRCPC2 (RCPC with immediate offsets) @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] dotprod: "dotprod"; - /// Vector Dot-Product (ASIMDDP) + /// FEAT_DotProd (Vector Dot-Product - ASIMDDP) @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] tme: "tme"; - /// Transactional Memory Extensions (TME) + /// FEAT_TME (Transactional Memory Extensions) + @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] fhm: "fhm"; + /// FEAT_FHM (fp16 multiplication instructions) + @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] dit: "dit"; + /// FEAT_DIT (Data Independent Timing instructions) + @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] flagm: "flagm"; + /// FEAT_FLAGM (flag manipulation instructions) + @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] ssbs: "ssbs"; + /// FEAT_SSBS (speculative store bypass safe) + @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] sb: "sb"; + /// FEAT_SB (speculation barrier) + @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] pauth: "pauth"; + /// FEAT_PAuth (pointer authentication) + @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] dpb: "dpb"; + /// FEAT_DPB (aka dcpop - data cache clean to point of persistance) + @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] dpb2: "dpb2"; + /// FEAT_DPB2 (aka dcpodp - data cache clean to point of deep persistance) + @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] sve2: "sve2"; + /// FEAT_SVE2 (Scalable Vector Extension 2) + @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] sve2_aes: "sve2-aes"; + /// FEAT_SVE_AES (SVE2 AES crypto) + @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] sve2_sm4: "sve2-sm4"; + /// FEAT_SVE_SM4 (SVE2 SM4 crypto) + @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] sve2_sha3: "sve2-sha3"; + /// FEAT_SVE_SHA3 (SVE2 SHA3 crypto) + @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] sve2_bitperm: "sve2-bitperm"; + /// FEAT_SVE_BitPerm (SVE2 bit permutation instructions) + @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] frintts: "frintts"; + /// FEAT_FRINTTS (float to integer rounding instructions) + @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] i8mm: "i8mm"; + /// FEAT_I8MM (integer matrix multiplication, plus ASIMD support) + @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] f32mm: "f32mm"; + /// FEAT_F32MM (single-precision matrix multiplication) + @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] f64mm: "f64mm"; + /// FEAT_F64MM (double-precision matrix multiplication) + @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] bf16: "bf16"; + /// FEAT_BF16 (BFloat16 type, plus MM instructions, plus ASIMD support) + @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] rand: "rand"; + /// FEAT_RNG (Random Number Generator) + @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] bti: "bti"; + /// FEAT_BTI (Branch Target Identification) + @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] mte: "mte"; + /// FEAT_MTE (Memory Tagging Extension) + @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] jsconv: "jsconv"; + /// FEAT_JSCVT (JavaScript float conversion instructions) + @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] fcma: "fcma"; + /// FEAT_FCMA (float complex number operations) + @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] aes: "aes"; + /// FEAT_AES (AES instructions) + @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] sha2: "sha2"; + /// FEAT_SHA1 & FEAT_SHA256 (SHA1 & SHA2-256 instructions) + @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] sha3: "sha3"; + /// FEAT_SHA512 & FEAT_SHA3 (SHA2-512 & SHA3 instructions) + @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] sm4: "sm4"; + /// FEAT_SM3 & FEAT_SM4 (SM3 & SM4 instructions) } diff --git a/library/stdarch/crates/std_detect/src/detect/arch/arm.rs b/library/stdarch/crates/std_detect/src/detect/arch/arm.rs index 46fd56384b..d96514c844 100644 --- a/library/stdarch/crates/std_detect/src/detect/arch/arm.rs +++ b/library/stdarch/crates/std_detect/src/detect/arch/arm.rs @@ -17,5 +17,9 @@ features! { @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] crc: "crc"; /// CRC32 (Cyclic Redundancy Check) @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] crypto: "crypto"; - /// Crypto: AES + PMULL + SHA1 + SHA2 + /// Crypto: AES + PMULL + SHA1 + SHA256. Prefer using the individual features where possible. + @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] aes: "aes"; + /// FEAT_AES (AES instructions) + @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] sha2: "sha2"; + /// FEAT_SHA1 & FEAT_SHA256 (SHA1 & SHA2-256 instructions) } diff --git a/library/stdarch/crates/std_detect/src/detect/bit.rs b/library/stdarch/crates/std_detect/src/detect/bit.rs index 578f0b16b7..6f06c5523e 100644 --- a/library/stdarch/crates/std_detect/src/detect/bit.rs +++ b/library/stdarch/crates/std_detect/src/detect/bit.rs @@ -4,6 +4,6 @@ #[allow(dead_code)] #[inline] pub(crate) fn test(x: usize, bit: u32) -> bool { - debug_assert!(bit < 32, "bit index out-of-bounds"); + debug_assert!(bit < usize::BITS, "bit index out-of-bounds"); x & (1 << bit) != 0 } diff --git a/library/stdarch/crates/std_detect/src/detect/macros.rs b/library/stdarch/crates/std_detect/src/detect/macros.rs index b9dbf9184e..a4af3d1c02 100644 --- a/library/stdarch/crates/std_detect/src/detect/macros.rs +++ b/library/stdarch/crates/std_detect/src/detect/macros.rs @@ -14,11 +14,12 @@ macro_rules! features { macro_rules! $macro_name { $( ($feature_lit) => { - $crate::detect::__is_feature_detected::$feature() + cfg!(target_feature = $feature_lit) || + $crate::detect::__is_feature_detected::$feature() }; )* $( - ($bind_feature) => { $macro_name!($feature_impl); }; + ($bind_feature) => { $macro_name!($feature_impl) }; )* $( ($nort_feature) => { @@ -94,8 +95,7 @@ macro_rules! features { #[doc(hidden)] #[$stability_attr] pub fn $feature() -> bool { - cfg!(target_feature = $feature_lit) || - $crate::detect::check_for($crate::detect::Feature::$feature) + $crate::detect::check_for($crate::detect::Feature::$feature) } )* } diff --git a/library/stdarch/crates/std_detect/src/detect/os/aarch64.rs b/library/stdarch/crates/std_detect/src/detect/os/aarch64.rs index c95b688970..0fd2702731 100644 --- a/library/stdarch/crates/std_detect/src/detect/os/aarch64.rs +++ b/library/stdarch/crates/std_detect/src/detect/os/aarch64.rs @@ -34,16 +34,14 @@ pub(crate) fn detect_features() -> cache::Initializer { // ID_AA64ISAR0_EL1 - Instruction Set Attribute Register 0 let aa64isar0: u64; unsafe { - llvm_asm!("mrs $0, ID_AA64ISAR0_EL1" : "=r"(aa64isar0)); + asm!( + "mrs {}, ID_AA64ISAR0_EL1", + out(reg) aa64isar0, + options(pure, nomem, preserves_flags, nostack) + ); } - let aes = bits_shift(aa64isar0, 7, 4) >= 1; - let pmull = bits_shift(aa64isar0, 7, 4) >= 2; - let sha1 = bits_shift(aa64isar0, 11, 8) >= 1; - let sha2 = bits_shift(aa64isar0, 15, 12) >= 1; - enable_feature(Feature::pmull, pmull); - // Crypto is specified as AES + PMULL + SHA1 + SHA2 per LLVM/hosts.cpp - enable_feature(Feature::crypto, aes && pmull && sha1 && sha2); + enable_feature(Feature::pmull, bits_shift(aa64isar0, 7, 4) >= 2); enable_feature(Feature::tme, bits_shift(aa64isar0, 27, 24) == 1); enable_feature(Feature::lse, bits_shift(aa64isar0, 23, 20) >= 1); enable_feature(Feature::crc, bits_shift(aa64isar0, 19, 16) >= 1); @@ -51,7 +49,11 @@ pub(crate) fn detect_features() -> cache::Initializer { // ID_AA64PFR0_EL1 - Processor Feature Register 0 let aa64pfr0: u64; unsafe { - llvm_asm!("mrs $0, ID_AA64PFR0_EL1" : "=r"(aa64pfr0)); + asm!( + "mrs {}, ID_AA64PFR0_EL1", + out(reg) aa64pfr0, + options(pure, nomem, preserves_flags, nostack) + ); } let fp = bits_shift(aa64pfr0, 19, 16) < 0xF; @@ -64,6 +66,10 @@ pub(crate) fn detect_features() -> cache::Initializer { // supported, it also requires half-float support: enable_feature(Feature::asimd, fp && asimd && (!fphp | asimdhp)); // SIMD extensions require SIMD support: + enable_feature(Feature::aes, asimd && bits_shift(aa64isar0, 7, 4) >= 1); + let sha1 = bits_shift(aa64isar0, 11, 8) >= 1; + let sha2 = bits_shift(aa64isar0, 15, 12) >= 1; + enable_feature(Feature::sha2, asimd && sha1 && sha2); enable_feature(Feature::rdm, asimd && bits_shift(aa64isar0, 31, 28) >= 1); enable_feature( Feature::dotprod, @@ -74,7 +80,11 @@ pub(crate) fn detect_features() -> cache::Initializer { // ID_AA64ISAR1_EL1 - Instruction Set Attribute Register 1 let aa64isar1: u64; unsafe { - llvm_asm!("mrs $0, ID_AA64ISAR1_EL1" : "=r"(aa64isar1)); + asm!( + "mrs {}, ID_AA64ISAR1_EL1", + out(reg) aa64isar1, + options(pure, nomem, preserves_flags, nostack) + ); } enable_feature(Feature::rcpc, bits_shift(aa64isar1, 23, 20) >= 1); diff --git a/library/stdarch/crates/std_detect/src/detect/os/freebsd/aarch64.rs b/library/stdarch/crates/std_detect/src/detect/os/freebsd/aarch64.rs index 7e086ca057..7d972b373f 100644 --- a/library/stdarch/crates/std_detect/src/detect/os/freebsd/aarch64.rs +++ b/library/stdarch/crates/std_detect/src/detect/os/freebsd/aarch64.rs @@ -1,6 +1,6 @@ //! Run-time feature detection for Aarch64 on FreeBSD. -pub use super::super::aarch64::detect_features; +pub(crate) use super::super::aarch64::detect_features; #[cfg(test)] mod tests { @@ -12,7 +12,6 @@ mod tests { println!("fp16: {:?}", is_aarch64_feature_detected!("fp16")); println!("sve: {:?}", is_aarch64_feature_detected!("sve")); println!("crc: {:?}", is_aarch64_feature_detected!("crc")); - println!("crypto: {:?}", is_aarch64_feature_detected!("crypto")); println!("lse: {:?}", is_aarch64_feature_detected!("lse")); println!("rdm: {:?}", is_aarch64_feature_detected!("rdm")); println!("rcpc: {:?}", is_aarch64_feature_detected!("rcpc")); diff --git a/library/stdarch/crates/std_detect/src/detect/os/freebsd/auxvec.rs b/library/stdarch/crates/std_detect/src/detect/os/freebsd/auxvec.rs index 832ce2252e..f12476adac 100644 --- a/library/stdarch/crates/std_detect/src/detect/os/freebsd/auxvec.rs +++ b/library/stdarch/crates/std_detect/src/detect/os/freebsd/auxvec.rs @@ -1,5 +1,12 @@ //! Parses ELF auxiliary vectors. -#![cfg_attr(any(target_arch = "arm", target_arch = "powerpc64"), allow(dead_code))] +#![cfg_attr( + any( + target_arch = "aarch64", + target_arch = "arm", + target_arch = "powerpc64" + ), + allow(dead_code) +)] /// Key to access the CPU Hardware capabilities bitfield. pub(crate) const AT_HWCAP: usize = 25; diff --git a/library/stdarch/crates/std_detect/src/detect/os/linux/aarch64.rs b/library/stdarch/crates/std_detect/src/detect/os/linux/aarch64.rs index 80c36e9b99..4dc6362057 100644 --- a/library/stdarch/crates/std_detect/src/detect/os/linux/aarch64.rs +++ b/library/stdarch/crates/std_detect/src/detect/os/linux/aarch64.rs @@ -20,11 +20,13 @@ pub(crate) fn detect_features() -> cache::Initializer { /// These values are part of the platform-specific [asm/hwcap.h][hwcap] . /// +/// The names match those used for cpuinfo. +/// /// [hwcap]: https://github.com/torvalds/linux/blob/master/arch/arm64/include/uapi/asm/hwcap.h struct AtHwcap { fp: bool, // 0 asimd: bool, // 1 - // evtstrm: bool, // 2 + // evtstrm: bool, // 2 No LLVM support aes: bool, // 3 pmull: bool, // 4 sha1: bool, // 5 @@ -33,18 +35,46 @@ struct AtHwcap { atomics: bool, // 8 fphp: bool, // 9 asimdhp: bool, // 10 - // cpuid: bool, // 11 + // cpuid: bool, // 11 No LLVM support asimdrdm: bool, // 12 - // jscvt: bool, // 13 - // fcma: bool, // 14 - lrcpc: bool, // 15 - // dcpop: bool, // 16 - // sha3: bool, // 17 - // sm3: bool, // 18 - // sm4: bool, // 19 - asimddp: bool, // 20 - // sha512: bool, // 21 - sve: bool, // 22 + jscvt: bool, // 13 + fcma: bool, // 14 + lrcpc: bool, // 15 + dcpop: bool, // 16 + sha3: bool, // 17 + sm3: bool, // 18 + sm4: bool, // 19 + asimddp: bool, // 20 + sha512: bool, // 21 + sve: bool, // 22 + fhm: bool, // 23 + dit: bool, // 24 + uscat: bool, // 25 + ilrcpc: bool, // 26 + flagm: bool, // 27 + ssbs: bool, // 28 + sb: bool, // 29 + paca: bool, // 30 + pacg: bool, // 31 + dcpodp: bool, // 32 + sve2: bool, // 33 + sveaes: bool, // 34 + // svepmull: bool, // 35 No LLVM support + svebitperm: bool, // 36 + svesha3: bool, // 37 + svesm4: bool, // 38 + // flagm2: bool, // 39 No LLVM support + frint: bool, // 40 + // svei8mm: bool, // 41 See i8mm feature + svef32mm: bool, // 42 + svef64mm: bool, // 43 + // svebf16: bool, // 44 See bf16 feature + i8mm: bool, // 45 + bf16: bool, // 46 + // dgh: bool, // 47 No LLVM support + rng: bool, // 48 + bti: bool, // 49 + mte: bool, // 50 } impl From for AtHwcap { @@ -64,16 +94,44 @@ impl From for AtHwcap { asimdhp: bit::test(auxv.hwcap, 10), // cpuid: bit::test(auxv.hwcap, 11), asimdrdm: bit::test(auxv.hwcap, 12), - // jscvt: bit::test(auxv.hwcap, 13), - // fcma: bit::test(auxv.hwcap, 14), + jscvt: bit::test(auxv.hwcap, 13), + fcma: bit::test(auxv.hwcap, 14), lrcpc: bit::test(auxv.hwcap, 15), - // dcpop: bit::test(auxv.hwcap, 16), - // sha3: bit::test(auxv.hwcap, 17), - // sm3: bit::test(auxv.hwcap, 18), - // sm4: bit::test(auxv.hwcap, 19), + dcpop: bit::test(auxv.hwcap, 16), + sha3: bit::test(auxv.hwcap, 17), + sm3: bit::test(auxv.hwcap, 18), + sm4: bit::test(auxv.hwcap, 19), asimddp: bit::test(auxv.hwcap, 20), - // sha512: bit::test(auxv.hwcap, 21), + sha512: bit::test(auxv.hwcap, 21), sve: bit::test(auxv.hwcap, 22), + fhm: bit::test(auxv.hwcap, 23), + dit: bit::test(auxv.hwcap, 24), + uscat: bit::test(auxv.hwcap, 25), + ilrcpc: bit::test(auxv.hwcap, 26), + flagm: bit::test(auxv.hwcap, 27), + ssbs: bit::test(auxv.hwcap, 28), + sb: bit::test(auxv.hwcap, 29), + paca: bit::test(auxv.hwcap, 30), + pacg: bit::test(auxv.hwcap, 31), + dcpodp: bit::test(auxv.hwcap, 32), + sve2: bit::test(auxv.hwcap, 33), + sveaes: bit::test(auxv.hwcap, 34), + // svepmull: bit::test(auxv.hwcap, 35), + svebitperm: bit::test(auxv.hwcap, 36), + svesha3: bit::test(auxv.hwcap, 37), + svesm4: bit::test(auxv.hwcap, 38), + // flagm2: bit::test(auxv.hwcap, 39), + frint: bit::test(auxv.hwcap, 40), + // svei8mm: bit::test(auxv.hwcap, 41), + svef32mm: bit::test(auxv.hwcap, 42), + svef64mm: bit::test(auxv.hwcap, 43), + // svebf16: bit::test(auxv.hwcap, 44), + i8mm: bit::test(auxv.hwcap, 45), + bf16: bit::test(auxv.hwcap, 46), + // dgh: bit::test(auxv.hwcap, 47), + rng: bit::test(auxv.hwcap, 48), + bti: bit::test(auxv.hwcap, 49), + mte: bit::test(auxv.hwcap, 50), } } } @@ -100,16 +158,44 @@ impl From for AtHwcap { asimdhp: f.has("asimdhp"), // cpuid: f.has("cpuid"), asimdrdm: f.has("asimdrdm"), - // jscvt: f.has("jscvt"), - // fcma: f.has("fcma"), + jscvt: f.has("jscvt"), + fcma: f.has("fcma"), lrcpc: f.has("lrcpc"), - // dcpop: f.has("dcpop"), - // sha3: f.has("sha3"), - // sm3: f.has("sm3"), - // sm4: f.has("sm4"), + dcpop: f.has("dcpop"), + sha3: f.has("sha3"), + sm3: f.has("sm3"), + sm4: f.has("sm4"), asimddp: f.has("asimddp"), - // sha512: f.has("sha512"), + sha512: f.has("sha512"), sve: f.has("sve"), + fhm: f.has("asimdfhm"), + dit: f.has("dit"), + uscat: f.has("uscat"), + ilrcpc: f.has("ilrcpc"), + flagm: f.has("flagm"), + ssbs: f.has("ssbs"), + sb: f.has("sb"), + paca: f.has("paca"), + pacg: f.has("pacg"), + dcpodp: f.has("dcpodp"), + sve2: f.has("sve2"), + sveaes: f.has("sveaes"), + // svepmull: f.has("svepmull"), + svebitperm: f.has("svebitperm"), + svesha3: f.has("svesha3"), + svesm4: f.has("svesm4"), + // flagm2: f.has("flagm2"), + frint: f.has("frint"), + // svei8mm: f.has("svei8mm"), + svef32mm: f.has("svef32mm"), + svef64mm: f.has("svef64mm"), + // svebf16: f.has("svebf16"), + i8mm: f.has("i8mm"), + bf16: f.has("bf16"), + // dgh: f.has("dgh"), + rng: f.has("rng"), + bti: f.has("bti"), + mte: f.has("mte"), } } } @@ -117,8 +203,8 @@ impl From for AtHwcap { impl AtHwcap { /// Initializes the cache from the feature -bits. /// - /// The features are enabled approximately like in LLVM host feature detection: - /// https://github.com/llvm-mirror/llvm/blob/master/lib/Support/Host.cpp#L1273 + /// The feature dependencies here come directly from LLVM's feature definintions: + /// https://github.com/llvm/llvm-project/blob/main/llvm/lib/Target/AArch64/AArch64.td fn cache(self) -> cache::Initializer { let mut value = cache::Initializer::default(); { @@ -131,25 +217,73 @@ impl AtHwcap { enable_feature(Feature::fp, self.fp); // Half-float support requires float support enable_feature(Feature::fp16, self.fp && self.fphp); + // FHM (fp16fml in LLVM) requires half float support + enable_feature(Feature::fhm, self.fphp && self.fhm); enable_feature(Feature::pmull, self.pmull); enable_feature(Feature::crc, self.crc32); enable_feature(Feature::lse, self.atomics); + enable_feature(Feature::lse2, self.uscat); enable_feature(Feature::rcpc, self.lrcpc); + // RCPC2 (rcpc-immo in LLVM) requires RCPC support + enable_feature(Feature::rcpc2, self.ilrcpc && self.lrcpc); + enable_feature(Feature::dit, self.dit); + enable_feature(Feature::flagm, self.flagm); + enable_feature(Feature::ssbs, self.ssbs); + enable_feature(Feature::sb, self.sb); + // FEAT_PAuth provides both paca & pacg + enable_feature(Feature::pauth, self.paca && self.pacg); + enable_feature(Feature::dpb, self.dcpop); + enable_feature(Feature::dpb2, self.dcpodp); + enable_feature(Feature::rand, self.rng); + enable_feature(Feature::bti, self.bti); + enable_feature(Feature::mte, self.mte); + // jsconv requires float support + enable_feature(Feature::jsconv, self.jscvt && self.fp); + enable_feature(Feature::rdm, self.asimdrdm); + enable_feature(Feature::dotprod, self.asimddp); + enable_feature(Feature::frintts, self.frint); + + // FEAT_I8MM & FEAT_BF16 also include optional SVE components which linux exposes + // separately. We ignore that distinction here. + enable_feature(Feature::i8mm, self.i8mm); + enable_feature(Feature::bf16, self.bf16); - // SIMD support requires float support - if half-floats are + // ASIMD support requires float support - if half-floats are // supported, it also requires half-float support: let asimd = self.fp && self.asimd && (!self.fphp | self.asimdhp); enable_feature(Feature::asimd, asimd); - // SIMD extensions require SIMD support: - enable_feature(Feature::rdm, self.asimdrdm && asimd); - enable_feature(Feature::dotprod, self.asimddp && asimd); + // ASIMD extensions require ASIMD support: + enable_feature(Feature::fcma, self.fcma && asimd); enable_feature(Feature::sve, self.sve && asimd); - // Crypto is specified as AES + PMULL + SHA1 + SHA2 per LLVM/hosts.cpp + // SVE extensions require SVE & ASIMD + enable_feature(Feature::f32mm, self.svef32mm && self.sve && asimd); + enable_feature(Feature::f64mm, self.svef64mm && self.sve && asimd); + + // Cryptographic extensions require ASIMD + enable_feature(Feature::aes, self.aes && asimd); + enable_feature(Feature::sha2, self.sha1 && self.sha2 && asimd); + // SHA512/SHA3 require SHA1 & SHA256 + enable_feature( + Feature::sha3, + self.sha512 && self.sha3 && self.sha1 && self.sha2 && asimd, + ); + enable_feature(Feature::sm4, self.sm3 && self.sm4 && asimd); + + // SVE2 requires SVE + let sve2 = self.sve2 && self.sve && asimd; + enable_feature(Feature::sve2, sve2); + // SVE2 extensions require SVE2 and crypto features + enable_feature(Feature::sve2_aes, self.sveaes && sve2 && self.aes); + enable_feature( + Feature::sve2_sm4, + self.svesm4 && sve2 && self.sm3 && self.sm4, + ); enable_feature( - Feature::crypto, - self.aes && self.pmull && self.sha1 && self.sha2, + Feature::sve2_sha3, + self.svesha3 && sve2 && self.sha512 && self.sha3 && self.sha1 && self.sha2, ); + enable_feature(Feature::sve2_bitperm, self.svebitperm && self.sve2); } value } diff --git a/library/stdarch/crates/std_detect/src/detect/os/linux/arm.rs b/library/stdarch/crates/std_detect/src/detect/os/linux/arm.rs index 66cfd05e80..7383e487f1 100644 --- a/library/stdarch/crates/std_detect/src/detect/os/linux/arm.rs +++ b/library/stdarch/crates/std_detect/src/detect/os/linux/arm.rs @@ -28,6 +28,13 @@ pub(crate) fn detect_features() -> cache::Initializer { && bit::test(auxv.hwcap2, 2) && bit::test(auxv.hwcap2, 3), ); + enable_feature(&mut value, Feature::aes, bit::test(auxv.hwcap2, 0)); + // SHA2 requires SHA1 & SHA2 features + enable_feature( + &mut value, + Feature::sha2, + bit::test(auxv.hwcap2, 2) && bit::test(auxv.hwcap2, 3), + ); return value; } @@ -48,6 +55,12 @@ pub(crate) fn detect_features() -> cache::Initializer { && c.field("Features").has("sha1") && c.field("Features").has("sha2"), ); + enable_feature(&mut value, Feature::aes, c.field("Features").has("aes")); + enable_feature( + &mut value, + Feature::sha2, + c.field("Features").has("sha1") && c.field("Features").has("sha2"), + ); return value; } value diff --git a/library/stdarch/crates/std_detect/src/detect/os/windows/aarch64.rs b/library/stdarch/crates/std_detect/src/detect/os/windows/aarch64.rs index 3c5edfb44a..051ad6d1bf 100644 --- a/library/stdarch/crates/std_detect/src/detect/os/windows/aarch64.rs +++ b/library/stdarch/crates/std_detect/src/detect/os/windows/aarch64.rs @@ -42,13 +42,17 @@ pub(crate) fn detect_features() -> cache::Initializer { // PF_ARM_V8_CRYPTO_INSTRUCTIONS_AVAILABLE means aes, sha1, sha2 and // pmull support enable_feature( - Feature::crypto, + Feature::aes, IsProcessorFeaturePresent(PF_ARM_V8_CRYPTO_INSTRUCTIONS_AVAILABLE) != FALSE, ); enable_feature( Feature::pmull, IsProcessorFeaturePresent(PF_ARM_V8_CRYPTO_INSTRUCTIONS_AVAILABLE) != FALSE, ); + enable_feature( + Feature::sha2, + IsProcessorFeaturePresent(PF_ARM_V8_CRYPTO_INSTRUCTIONS_AVAILABLE) != FALSE, + ); } } value diff --git a/library/stdarch/crates/std_detect/src/lib.rs b/library/stdarch/crates/std_detect/src/lib.rs index 782833cbb6..a95af4145f 100644 --- a/library/stdarch/crates/std_detect/src/lib.rs +++ b/library/stdarch/crates/std_detect/src/lib.rs @@ -12,11 +12,11 @@ //! * `powerpc64`: [`is_powerpc64_feature_detected`] #![unstable(feature = "stdsimd", issue = "27731")] -#![feature(const_fn, staged_api, stdsimd, doc_cfg, allow_internal_unstable)] +#![feature(staged_api, stdsimd, doc_cfg, allow_internal_unstable)] #![deny(rust_2018_idioms)] #![allow(clippy::shadow_reuse)] #![deny(clippy::missing_inline_in_public_items)] -#![cfg_attr(all(target_os = "freebsd", target_arch = "aarch64"), feature(llvm_asm))] +#![cfg_attr(all(target_os = "freebsd", target_arch = "aarch64"), feature(asm))] #![cfg_attr(test, allow(unused_imports))] #![cfg_attr(feature = "std_detect_file_io", feature(vec_spare_capacity))] #![no_std] diff --git a/library/stdarch/crates/std_detect/tests/cpu-detection.rs b/library/stdarch/crates/std_detect/tests/cpu-detection.rs index 8f29ea45cd..adbe4fa9a7 100644 --- a/library/stdarch/crates/std_detect/tests/cpu-detection.rs +++ b/library/stdarch/crates/std_detect/tests/cpu-detection.rs @@ -26,6 +26,8 @@ fn arm_linux() { println!("pmull: {}", is_arm_feature_detected!("pmull")); println!("crc: {}", is_arm_feature_detected!("crc")); println!("crypto: {}", is_arm_feature_detected!("crypto")); + println!("aes: {}", is_arm_feature_detected!("aes")); + println!("sha2: {}", is_arm_feature_detected!("sha2")); } #[test] @@ -34,18 +36,50 @@ fn arm_linux() { any(target_os = "linux", target_os = "android") ))] fn aarch64_linux() { + println!("asimd: {}", is_aarch64_feature_detected!("asimd")); + println!("neon: {}", is_aarch64_feature_detected!("neon")); + println!("pmull: {}", is_aarch64_feature_detected!("pmull")); println!("fp: {}", is_aarch64_feature_detected!("fp")); println!("fp16: {}", is_aarch64_feature_detected!("fp16")); - println!("neon: {}", is_aarch64_feature_detected!("neon")); - println!("asimd: {}", is_aarch64_feature_detected!("asimd")); println!("sve: {}", is_aarch64_feature_detected!("sve")); println!("crc: {}", is_aarch64_feature_detected!("crc")); - println!("crypto: {}", is_aarch64_feature_detected!("crypto")); println!("lse: {}", is_aarch64_feature_detected!("lse")); + println!("lse2: {}", is_aarch64_feature_detected!("lse2")); println!("rdm: {}", is_aarch64_feature_detected!("rdm")); println!("rcpc: {}", is_aarch64_feature_detected!("rcpc")); + println!("rcpc2: {}", is_aarch64_feature_detected!("rcpc2")); println!("dotprod: {}", is_aarch64_feature_detected!("dotprod")); println!("tme: {}", is_aarch64_feature_detected!("tme")); + println!("fhm: {}", is_aarch64_feature_detected!("fhm")); + println!("dit: {}", is_aarch64_feature_detected!("dit")); + println!("flagm: {}", is_aarch64_feature_detected!("flagm")); + println!("ssbs: {}", is_aarch64_feature_detected!("ssbs")); + println!("sb: {}", is_aarch64_feature_detected!("sb")); + println!("pauth: {}", is_aarch64_feature_detected!("pauth")); + println!("dpb: {}", is_aarch64_feature_detected!("dpb")); + println!("dpb2: {}", is_aarch64_feature_detected!("dpb2")); + println!("sve2: {}", is_aarch64_feature_detected!("sve2")); + println!("sve2-aes: {}", is_aarch64_feature_detected!("sve2-aes")); + println!("sve2-sm4: {}", is_aarch64_feature_detected!("sve2-sm4")); + println!("sve2-sha3: {}", is_aarch64_feature_detected!("sve2-sha3")); + println!( + "sve2-bitperm: {}", + is_aarch64_feature_detected!("sve2-bitperm") + ); + println!("frintts: {}", is_aarch64_feature_detected!("frintts")); + println!("i8mm: {}", is_aarch64_feature_detected!("i8mm")); + println!("f32mm: {}", is_aarch64_feature_detected!("f32mm")); + println!("f64mm: {}", is_aarch64_feature_detected!("f64mm")); + println!("bf16: {}", is_aarch64_feature_detected!("bf16")); + println!("rand: {}", is_aarch64_feature_detected!("rand")); + println!("bti: {}", is_aarch64_feature_detected!("bti")); + println!("mte: {}", is_aarch64_feature_detected!("mte")); + println!("jsconv: {}", is_aarch64_feature_detected!("jsconv")); + println!("fcma: {}", is_aarch64_feature_detected!("fcma")); + println!("aes: {}", is_aarch64_feature_detected!("aes")); + println!("sha2: {}", is_aarch64_feature_detected!("sha2")); + println!("sha3: {}", is_aarch64_feature_detected!("sha3")); + println!("sm4: {}", is_aarch64_feature_detected!("sm4")); } #[test] diff --git a/library/stdarch/crates/stdarch-gen/neon.spec b/library/stdarch/crates/stdarch-gen/neon.spec index 5c705c15db..159419285f 100644 --- a/library/stdarch/crates/stdarch-gen/neon.spec +++ b/library/stdarch/crates/stdarch-gen/neon.spec @@ -53,12 +53,12 @@ // FALSE - 'false' all bits are set to 0 // FF - same as 'true' // MIN - minimal value (either 0 or the lowest negative number) -// MAX - maximal value propr to overflow +// MAX - maximal value proper to overflow // // # validate // Validates a and b aginst the expected result of the test. // The special values 'TRUE' and 'FALSE' can be used to -// represent the corect NEON representation of true or +// represent the correct NEON representation of true or // false values. It too gets scaled to the type. // // Validate needs to be called before generate as it sets @@ -102,6 +102,176 @@ b = 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 validate 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F generate int*_t, uint*_t, int64x*_t, uint64x*_t +//////////////////// +// Absolute difference between the arguments +//////////////////// + +/// Absolute difference between the arguments +name = vabd +a = 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 +b = 16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1 +validate 15, 13, 11, 9, 7, 5, 3, 1, 1, 3, 5, 7, 9, 11, 13, 15 + +arm = vabd.s +aarch64 = sabd +link-arm = vabds._EXT_ +link-aarch64 = sabd._EXT_ +generate int*_t + +arm = vabd.s +aarch64 = uabd +link-arm = vabdu._EXT_ +link-aarch64 = uabd._EXT_ +generate uint*_t + +/// Absolute difference between the arguments of Floating +name = vabd +a = 1.0, 2.0, 5.0, -4.0 +b = 9.0, 3.0, 2.0, 8.0 +validate 8.0, 1.0, 3.0, 12.0 + +aarch64 = fabd +link-aarch64 = fabd._EXT_ +generate float64x*_t + +arm = vabd.s +aarch64 = fabd +link-arm = vabds._EXT_ +link-aarch64 = fabd._EXT_ +generate float*_t + +//////////////////// +// Absolute difference Long +//////////////////// + +/// Unsigned Absolute difference Long +name = vabdl +multi_fn = simd_cast, {vabd-unsigned-noext, a, b} +a = 1, 2, 3, 4, 4, 3, 2, 1 +b = 10, 10, 10, 10, 10, 10, 10, 10 +validate 9, 8, 7, 6, 6, 7, 8, 9 + +arm = vabdl.s +aarch64 = uabdl +generate uint8x8_t:uint8x8_t:uint16x8_t, uint16x4_t:uint16x4_t:uint32x4_t, uint32x2_t:uint32x2_t:uint64x2_t + +/// Signed Absolute difference Long +name = vabdl +multi_fn = simd_cast, c:uint8x8_t, {vabd-signed-noext, a, b} +multi_fn = simd_cast, c +a = 1, 2, 3, 4, 4, 3, 2, 1 +b = 10, 10, 10, 10, 10, 10, 10, 10 +validate 9, 8, 7, 6, 6, 7, 8, 9 + +arm = vabdl.s +aarch64 = sabdl +generate int8x8_t:int8x8_t:int16x8_t + +/// Signed Absolute difference Long +name = vabdl +multi_fn = simd_cast, c:uint16x4_t, {vabd-signed-noext, a, b} +multi_fn = simd_cast, c +a = 1, 2, 11, 12 +b = 10, 10, 10, 10 +validate 9, 8, 1, 2 + +arm = vabdl.s +aarch64 = sabdl +generate int16x4_t:int16x4_t:int32x4_t + +/// Signed Absolute difference Long +name = vabdl +multi_fn = simd_cast, c:uint32x2_t, {vabd-signed-noext, a, b} +multi_fn = simd_cast, c +a = 1, 11 +b = 10, 10 +validate 9, 1 + +arm = vabdl.s +aarch64 = sabdl +generate int32x2_t:int32x2_t:int64x2_t + +/// Unsigned Absolute difference Long +name = vabdl_high +no-q +multi_fn = simd_shuffle8!, c:uint8x8_t, a, a, [8, 9, 10, 11, 12, 13, 14, 15] +multi_fn = simd_shuffle8!, d:uint8x8_t, b, b, [8, 9, 10, 11, 12, 13, 14, 15] +multi_fn = simd_cast, {vabd_u8, c, d} +a = 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 +b = 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10 +validate 1, 0, 1, 2, 3, 4, 5, 6 + +aarch64 = uabdl +generate uint8x16_t:uint8x16_t:uint16x8_t + +/// Unsigned Absolute difference Long +name = vabdl_high +no-q +multi_fn = simd_shuffle4!, c:uint16x4_t, a, a, [4, 5, 6, 7] +multi_fn = simd_shuffle4!, d:uint16x4_t, b, b, [4, 5, 6, 7] +multi_fn = simd_cast, {vabd_u16, c, d} +a = 1, 2, 3, 4, 8, 9, 11, 12 +b = 10, 10, 10, 10, 10, 10, 10, 10 +validate 2, 1, 1, 2 + +aarch64 = uabdl +generate uint16x8_t:uint16x8_t:uint32x4_t + +/// Unsigned Absolute difference Long +name = vabdl_high +no-q +multi_fn = simd_shuffle2!, c:uint32x2_t, a, a, [2, 3] +multi_fn = simd_shuffle2!, d:uint32x2_t, b, b, [2, 3] +multi_fn = simd_cast, {vabd_u32, c, d} +a = 1, 2, 3, 4 +b = 10, 10, 10, 10 +validate 7, 6 + +aarch64 = uabdl +generate uint32x4_t:uint32x4_t:uint64x2_t + +/// Signed Absolute difference Long +name = vabdl_high +no-q +multi_fn = simd_shuffle8!, c:int8x8_t, a, a, [8, 9, 10, 11, 12, 13, 14, 15] +multi_fn = simd_shuffle8!, d:int8x8_t, b, b, [8, 9, 10, 11, 12, 13, 14, 15] +multi_fn = simd_cast, e:uint8x8_t, {vabd_s8, c, d} +multi_fn = simd_cast, e +a = 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 +b = 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10 +validate 1, 0, 1, 2, 3, 4, 5, 6 + +aarch64 = sabdl +generate int8x16_t:int8x16_t:int16x8_t + +/// Signed Absolute difference Long +name = vabdl_high +no-q +multi_fn = simd_shuffle4!, c:int16x4_t, a, a, [4, 5, 6, 7] +multi_fn = simd_shuffle4!, d:int16x4_t, b, b, [4, 5, 6, 7] +multi_fn = simd_cast, e:uint16x4_t, {vabd_s16, c, d} +multi_fn = simd_cast, e +a = 1, 2, 3, 4, 9, 10, 11, 12 +b = 10, 10, 10, 10, 10, 10, 10, 10 +validate 1, 0, 1, 2 + +aarch64 = sabdl +generate int16x8_t:int16x8_t:int32x4_t + +/// Signed Absolute difference Long +name = vabdl_high +no-q +multi_fn = simd_shuffle2!, c:int32x2_t, a, a, [2, 3] +multi_fn = simd_shuffle2!, d:int32x2_t, b, b, [2, 3] +multi_fn = simd_cast, e:uint32x2_t, {vabd_s32, c, d} +multi_fn = simd_cast, e +a = 1, 2, 3, 4 +b = 10, 10, 10, 10 +validate 7, 6 + +aarch64 = sabdl +generate int32x4_t:int32x4_t:int64x2_t + //////////////////// // equality //////////////////// @@ -120,7 +290,7 @@ aarch64 = cmeq generate uint64x*_t, int64x1_t:uint64x1_t, int64x2_t:uint64x2_t, poly64x1_t:uint64x1_t, poly64x2_t:uint64x2_t arm = vceq. -generate uint*_t, int8x8_t:uint8x8_t, int8x16_t:uint8x16_t, int16x4_t:uint16x4_t, int16x8_t:uint16x8_t, int32x2_t:uint32x2_t, int32x4_t:uint32x4_t +generate uint*_t, int8x8_t:uint8x8_t, int8x16_t:uint8x16_t, int16x4_t:uint16x4_t, int16x8_t:uint16x8_t, int32x2_t:uint32x2_t, int32x4_t:uint32x4_t, poly8x8_t:uint8x8_t, poly8x16_t:uint8x16_t /// Floating-point compare equal name = vceq @@ -136,6 +306,83 @@ arm = vceq. // we are missing float16x4_t:uint16x4_t, float16x8_t:uint16x8_t generate float32x2_t:uint32x2_t, float32x4_t:uint32x4_t +/// Signed compare bitwise equal to zero +name = vceqz +fn = simd_eq +a = MIN, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, MAX +fixed = 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 +validate FALSE, TRUE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE + +aarch64 = cmeq +generate int8x8_t:uint8x8_t, int8x16_t:uint8x16_t, int16x4_t:uint16x4_t, int16x8_t:uint16x8_t, int32x2_t:uint32x2_t, int32x4_t:uint32x4_t, int64x1_t:uint64x1_t, int64x2_t:uint64x2_t, poly8x8_t:uint8x8_t, poly8x16_t:uint8x16_t, poly64x1_t:uint64x1_t, poly64x2_t:uint64x2_t + +/// Unsigned compare bitwise equal to zero +name = vceqz +fn = simd_eq +a = MIN, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, MAX +fixed = 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 +validate TRUE, TRUE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE + +aarch64 = cmeq +generate uint*_t, uint64x*_t + +/// Floating-point compare bitwise equal to zero +name = vceqz +fn = simd_eq +a = 0.0, 1.2, 3.4, 5.6 +fixed = 0.0, 0.0, 0.0, 0.0 +validate TRUE, FALSE, FALSE, FALSE + +aarch64 = fcmeq +generate float32x2_t:uint32x2_t, float32x4_t:uint32x4_t, float64x1_t:uint64x1_t, float64x2_t:uint64x2_t + +/// Signed compare bitwise Test bits nonzero +name = vtst +multi_fn = simd_and, c:in_t, a, b +multi_fn = fixed, d:in_t +multi_fn = simd_ne, c, transmute(d) +a = MIN, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, MAX +b = MIN, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, MAX +fixed = 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 +validate TRUE, FALSE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE + +aarch64 = cmtst +generate int64x1_t:uint64x1_t, int64x2_t:uint64x2_t, poly64x1_t:uint64x1_t, poly64x2_t:uint64x2_t + +arm = vtst +generate int8x8_t:uint8x8_t, int8x16_t:uint8x16_t, int16x4_t:uint16x4_t, int16x8_t:uint16x8_t, int32x2_t:uint32x2_t, int32x4_t:uint32x4_t, poly8x8_t:uint8x8_t, poly8x16_t:uint8x16_t + +/// Unsigned compare bitwise Test bits nonzero +name = vtst +multi_fn = simd_and, c:in_t, a, b +multi_fn = fixed, d:in_t +multi_fn = simd_ne, c, transmute(d) +a = MIN, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, MAX +b = MIN, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, MAX +fixed = 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 +validate FALSE, FALSE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE + +aarch64 = cmtst +generate uint64x*_t + +arm = vtst +generate uint*_t + +//////////////////// +// Floating-point absolute value +//////////////////// + +/// Floating-point absolute value +name = vabs +fn = simd_fabs +a = -0.1, -2.2, -3.3, -6.6 +validate 0.1, 2.2, 3.3, 6.6 +aarch64 = fabs +generate float64x1_t:float64x1_t, float64x2_t:float64x2_t + +arm = vabs +generate float32x2_t:float32x2_t, float32x4_t:float32x4_t + //////////////////// // greater then //////////////////// @@ -309,210 +556,4579 @@ arm = vcge.s // we are missing float16x4_t:uint16x4_t, float16x8_t:uint16x8_t generate float32x2_t:uint32x2_t, float32x4_t:uint32x4_t -/// Saturating subtract -name = vqsub -a = 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42 -b = 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 -validate 41, 40, 39, 38, 37, 36, 35, 34, 33, 32, 31, 30, 29, 28, 27, 26 +/// Compare signed greater than or equal to zero +name = vcgez +fn = simd_ge +a = MIN, -1, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, MAX +fixed = 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 +validate FALSE, FALSE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE -arm = vqsub.s -aarch64 = uqsub -link-arm = vqsubu._EXT_ -link-aarch64 = uqsub._EXT_ -generate uint*_t +aarch64 = cmge +generate int8x8_t:uint8x8_t, int8x16_t:uint8x16_t, int16x4_t:uint16x4_t, int16x8_t:uint16x8_t, int32x2_t:uint32x2_t, int32x4_t:uint32x4_t, int64x1_t:uint64x1_t, int64x2_t:uint64x2_t -arm = vqsub.s -aarch64 = sqsub -link-arm = vqsubs._EXT_ -link-aarch64 = sqsub._EXT_ -generate int*_t +/// Floating-point compare greater than or equal to zero +name = vcgez +fn = simd_ge +a = -1.2, 0.0, 1.2, 2.3, 3.4, 4.5, 5.6, 6.7 +fixed = 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0 +validate FALSE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE -/// Halving add -name = vhadd -a = 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42 -b = 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 -validate 21, 22, 22, 23, 23, 24, 24, 25, 25, 26, 26, 27, 27, 28, 28, 29 +aarch64 = fcmge +generate float32x2_t:uint32x2_t, float32x4_t:uint32x4_t, float64x1_t:uint64x1_t, float64x2_t:uint64x2_t +/// Compare signed greater than zero +name = vcgtz +fn = simd_gt +a = MIN, -1, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, MAX +fixed = 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 +validate FALSE, FALSE, FALSE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE -arm = vhadd.s -aarch64 = uhadd -link-aarch64 = uhadd._EXT_ -link-arm = vhaddu._EXT_ -generate uint*_t +aarch64 = cmgt +generate int8x8_t:uint8x8_t, int8x16_t:uint8x16_t, int16x4_t:uint16x4_t, int16x8_t:uint16x8_t, int32x2_t:uint32x2_t, int32x4_t:uint32x4_t, int64x1_t:uint64x1_t, int64x2_t:uint64x2_t +/// Floating-point compare greater than zero +name = vcgtz +fn = simd_gt +a = -1.2, 0.0, 1.2, 2.3, 3.4, 4.5, 5.6, 6.7 +fixed = 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0 +validate FALSE, FALSE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE -arm = vhadd.s -aarch64 = shadd -link-aarch64 = shadd._EXT_ -link-arm = vhadds._EXT_ -generate int*_t +aarch64 = fcmgt +generate float32x2_t:uint32x2_t, float32x4_t:uint32x4_t, float64x1_t:uint64x1_t, float64x2_t:uint64x2_t -/// Rounding halving add -name = vrhadd -a = 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42 -b = 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 -validate 22, 22, 23, 23, 24, 24, 25, 25, 26, 26, 27, 27, 28, 28, 29, 29 +/// Compare signed less than or equal to zero +name = vclez +fn = simd_le +a = MIN, -1, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, MAX +fixed = 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 +validate TRUE, TRUE, TRUE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE -arm = vrhadd.s -aarch64 = urhadd -link-arm = vrhaddu._EXT_ -link-aarch64 = urhadd._EXT_ -generate uint*_t +aarch64 = cmgt +generate int8x8_t:uint8x8_t, int8x16_t:uint8x16_t, int16x4_t:uint16x4_t, int16x8_t:uint16x8_t, int32x2_t:uint32x2_t, int32x4_t:uint32x4_t, int64x1_t:uint64x1_t, int64x2_t:uint64x2_t -arm = vrhadd.s -aarch64 = srhadd -link-arm = vrhadds._EXT_ -link-aarch64 = srhadd._EXT_ +/// Floating-point compare less than or equal to zero +name = vclez +fn = simd_le +a = -1.2, 0.0, 1.2, 2.3, 3.4, 4.5, 5.6, 6.7 +fixed = 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0 +validate TRUE, TRUE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE + +aarch64 = fcmle +generate float32x2_t:uint32x2_t, float32x4_t:uint32x4_t, float64x1_t:uint64x1_t, float64x2_t:uint64x2_t + +/// Compare signed less than zero +name = vcltz +fn = simd_lt +a = MIN, -1, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, MAX +fixed = 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 +validate TRUE, TRUE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE + +aarch64 = sshr +generate int8x8_t:uint8x8_t, int8x16_t:uint8x16_t, int16x4_t:uint16x4_t, int16x8_t:uint16x8_t, int32x2_t:uint32x2_t, int32x4_t:uint32x4_t, int64x1_t:uint64x1_t, int64x2_t:uint64x2_t + +/// Floating-point compare less than zero +name = vcltz +fn = simd_lt +a = -1.2, 0.0, 1.2, 2.3, 3.4, 4.5, 5.6, 6.7 +fixed = 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0 +validate TRUE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE + +aarch64 = fcmlt +generate float32x2_t:uint32x2_t, float32x4_t:uint32x4_t, float64x1_t:uint64x1_t, float64x2_t:uint64x2_t + +/// Count leading sign bits +name = vcls +a = MIN, -1, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, MAX +validate 0, BITS_M1, BITS_M1, BITS_M1, BITS_M1, BITS_M1, BITS_M1, BITS_M1, BITS_M1, BITS_M1, BITS_M1, BITS_M1, BITS_M1, BITS_M1, BITS_M1, 0 + +arm = vcls.s +aarch64 = cls +link-arm = vcls._EXT_ +link-aarch64 = cls._EXT_ generate int*_t -/// Saturating add -name = vqadd -a = 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42 -b = 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 -validate 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58 +/// Signed count leading sign bits +name = vclz +multi_fn = self-signed-ext, a +a = MIN, -1, 0x00, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, MAX +validate 0, 0, BITS, BITS_M1, BITS_M1, BITS_M1, BITS_M1, BITS_M1, BITS_M1, BITS_M1, BITS_M1, BITS_M1, BITS_M1, BITS_M1, BITS_M1, 1 -arm = vqadd.s -aarch64 = uqadd -link-arm = vqaddu._EXT_ -link-aarch64 = uqadd._EXT_ +arm = vclz. +aarch64 = clz +generate int*_t + +/// Unsigned count leading sign bits +name = vclz +multi_fn = transmute, {self-signed-ext, transmute(a)} +a = MIN, 0x00, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, MAX +validate BITS, BITS, BITS_M1, BITS_M1, BITS_M1, BITS_M1, BITS_M1, BITS_M1, BITS_M1, BITS_M1, BITS_M1, BITS_M1, BITS_M1, BITS_M1, BITS_M1, 0 + +arm = vclz. +aarch64 = clz generate uint*_t -arm = vqadd.s -aarch64 = sqadd -link-arm = vqadds._EXT_ -link-aarch64 = sqadd._EXT_ -generate int*_t +/// Floating-point absolute compare greater than +name = vcagt +a = -1.2, 0.0, 1.2, 2.3, 3.4, 4.5, 5.6, 6.7 +b = -1.1, 0.0, 1.1, 2.4, 3.3, 4.6, 5.5, 6.8 +validate TRUE, FALSE, TRUE, FALSE, TRUE, FALSE, TRUE, FALSE -/// Multiply -name = vmul -a = 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2 -b = 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 -validate 1, 4, 3, 8, 5, 12, 7, 16, 9, 20, 11, 24, 13, 28, 15, 32 -arm = vmul. -aarch64 = mul -fn = simd_mul -generate int*_t, uint*_t +aarch64 = facgt +link-aarch64 = facgt._EXT2_._EXT_ +generate float64x1_t:uint64x1_t, float64x2_t:uint64x2_t -/// Multiply -name = vmul -fn = simd_mul -a = 1.0, 2.0, 1.0, 2.0 -b = 2.0, 3.0, 4.0, 5.0 -validate 2.0, 6.0, 4.0, 10.0 +arm = vacgt.s +link-arm = vacgt._EXT2_._EXT_ +generate float32x2_t:uint32x2_t, float32x4_t:uint32x4_t -aarch64 = fmul -generate float64x*_t +/// Floating-point absolute compare greater than or equal +name = vcage +a = -1.2, 0.0, 1.2, 2.3, 3.4, 4.5, 5.6, 6.7 +b = -1.1, 0.0, 1.1, 2.4, 3.3, 4.6, 5.5, 6.8 +validate TRUE, TRUE, TRUE, FALSE, TRUE, FALSE, TRUE, FALSE -arm = vmul. -generate float*_t +aarch64 = facge +link-aarch64 = facge._EXT2_._EXT_ +generate float64x1_t:uint64x1_t, float64x2_t:uint64x2_t +arm = vacge.s +link-arm = vacge._EXT2_._EXT_ +generate float32x2_t:uint32x2_t, float32x4_t:uint32x4_t -/// Subtract -name = vsub -a = 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 -b = 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2 -validate 0, 0, 2, 2, 4, 4, 6, 6, 8, 8, 10, 10, 12, 12, 14, 14 -arm = vsub. -aarch64 = sub -fn = simd_sub -generate int*_t, uint*_t, int64x*_t, uint64x*_t +/// Floating-point absolute compare less than +name = vcalt +multi_fn = vcagt-self-noext, b, a +a = -1.2, 0.0, 1.2, 2.3, 3.4, 4.5, 5.6, 6.7 +b = -1.1, 0.0, 1.1, 2.4, 3.3, 4.6, 5.5, 6.8 +validate FALSE, FALSE, FALSE, TRUE, FALSE, TRUE, FALSE, TRUE -/// Subtract -name = vsub -fn = simd_sub -a = 1.0, 4.0, 3.0, 8.0 -b = 1.0, 2.0, 3.0, 4.0 -validate 0.0, 2.0, 0.0, 4.0 +aarch64 = facgt +generate float64x1_t:uint64x1_t, float64x2_t:uint64x2_t -aarch64 = fsub -generate float64x*_t +arm = vacgt.s +generate float32x2_t:uint32x2_t, float32x4_t:uint32x4_t -arm = vsub. -generate float*_t +/// Floating-point absolute compare less than or equal +name = vcale +multi_fn = vcage-self-noext , b, a +a = -1.2, 0.0, 1.2, 2.3, 3.4, 4.5, 5.6, 6.7 +b = -1.1, 0.0, 1.1, 2.4, 3.3, 4.6, 5.5, 6.8 +validate FALSE, TRUE, FALSE, TRUE, FALSE, TRUE, FALSE, TRUE +aarch64 = facge +generate float64x1_t:uint64x1_t, float64x2_t:uint64x2_t -/// Signed halving subtract -name = vhsub +arm = vacge.s +generate float32x2_t:uint32x2_t, float32x4_t:uint32x4_t + +/// Insert vector element from another vector element +name = vcopy +lane-suffixes +constn = LANE1:LANE2 +multi_fn = static_assert_imm-in0_exp_len-LANE1 +multi_fn = static_assert_imm-in_exp_len-LANE2 +multi_fn = matchn-in0_exp_len-LANE1, simd_shuffle-out_len-!, a, b, {ins-in0_len-in0_len-LANE2} a = 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 -b = 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2 -validate 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7 +b = 0, MAX, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 +n = 0:1 +validate MAX, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 -arm = vhsub.s -aarch64 = uhsub -link-arm = vhsubu._EXT_ -link-aarch64 = uhsub._EXT_ -generate uint*_t +aarch64 = mov +generate int8x8_t, int8x16_t, int16x4_t, int16x8_t, int32x2_t, int32x4_t, int64x2_t +generate uint8x8_t, uint8x16_t, uint16x4_t, uint16x8_t, uint32x2_t, uint32x4_t, uint64x2_t +generate poly8x8_t, poly8x16_t, poly16x4_t, poly16x8_t, poly64x2_t -arm = vhsub.s -aarch64 = shsub -link-arm = vhsubs._EXT_ -link-aarch64 = shsub._EXT_ -generate int*_t +/// Insert vector element from another vector element +name = vcopy +lane-suffixes +constn = LANE1:LANE2 +multi_fn = static_assert_imm-in0_exp_len-LANE1 +multi_fn = static_assert_imm-in_exp_len-LANE2 +multi_fn = matchn-in0_exp_len-LANE1, simd_shuffle-out_len-!, a, b, {ins-in0_len-in0_len-LANE2} +a = 1., 2., 3., 4. +b = 0., 0.5, 0., 0. +n = 0:1 +validate 0.5, 2., 3., 4. -/// Maximum (vector) -name = vmax +aarch64 = mov +generate float32x2_t, float32x4_t, float64x2_t + +/// Insert vector element from another vector element +name = vcopy +lane-suffixes +constn = LANE1:LANE2 +multi_fn = static_assert_imm-in0_exp_len-LANE1 +multi_fn = static_assert_imm-in_exp_len-LANE2 +multi_fn = simd_shuffle-in_len-!, a:in_t, a, a, {asc-0-in_len} +multi_fn = matchn-in0_exp_len-LANE1, simd_shuffle-out_len-!, a, b, {ins-in0_len-in_len-LANE2} a = 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 -b = 16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1 -validate 16, 15, 14, 13, 12, 11, 10, 9, 9, 10, 11, 12, 13, 14, 15, 16 +b = 0, MAX, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 +n = 0:1 +validate MAX, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 -arm = vmax -aarch64 = smax -link-arm = vmaxs._EXT_ -link-aarch64 = smax._EXT_ -generate int*_t +aarch64 = mov +generate int8x8_t:int8x16_t:int8x8_t, int16x4_t:int16x8_t:int16x4_t, int32x2_t:int32x4_t:int32x2_t +generate uint8x8_t:uint8x16_t:uint8x8_t, uint16x4_t:uint16x8_t:uint16x4_t, uint32x2_t:uint32x4_t:uint32x2_t +generate poly8x8_t:poly8x16_t:poly8x8_t, poly16x4_t:poly16x8_t:poly16x4_t -arm = vmax -aarch64 = umax -link-arm = vmaxu._EXT_ -link-aarch64 = umax._EXT_ -generate uint*_t +/// Insert vector element from another vector element +name = vcopy +lane-suffixes +constn = LANE1:LANE2 +multi_fn = static_assert_imm-in0_exp_len-LANE1 +multi_fn = static_assert_imm-in_exp_len-LANE2 +multi_fn = simd_shuffle-in_len-!, a:in_t, a, a, {asc-0-in_len} +multi_fn = matchn-in0_exp_len-LANE1, simd_shuffle-out_len-!, a, b, {ins-in0_len-in_len-LANE2} +a = 1., 2., 3., 4. +b = 0., 0.5, 0., 0. +n = 0:1 +validate 0.5, 2., 3., 4. -/// Maximum (vector) -name = vmax -a = 1.0, -2.0, 3.0, -4.0 -b = 0.0, 3.0, 2.0, 8.0 -validate 1.0, 3.0, 3.0, 8.0 +aarch64 = mov +generate float32x2_t:float32x4_t:float32x2_t -aarch64 = fmax -link-aarch64 = fmax._EXT_ -generate float64x*_t +/// Insert vector element from another vector element +name = vcopy +lane-suffixes +constn = LANE1:LANE2 +multi_fn = static_assert_imm-in0_exp_len-LANE1 +multi_fn = static_assert_imm-in_exp_len-LANE2 +multi_fn = simd_shuffle-in0_len-!, b:in_t0, b, b, {asc-0-in0_len} +multi_fn = matchn-in0_exp_len-LANE1, simd_shuffle-out_len-!, a, b, {ins-in0_len-in0_len-LANE2} +a = 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 +b = 0, MAX, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 +n = 0:1 +validate MAX, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 -arm = vmax -aarch64 = fmax -link-arm = vmaxs._EXT_ -link-aarch64 = fmax._EXT_ -generate float*_t +aarch64 = mov +generate int8x16_t:int8x8_t:int8x16_t, int16x8_t:int16x4_t:int16x8_t, int32x4_t:int32x2_t:int32x4_t +generate uint8x16_t:uint8x8_t:uint8x16_t, uint16x8_t:uint16x4_t:uint16x8_t, uint32x4_t:uint32x2_t:uint32x4_t +generate poly8x16_t:poly8x8_t:poly8x16_t, poly16x8_t:poly16x4_t:poly16x8_t -/// Minimum (vector) -name = vmin +/// Insert vector element from another vector element +name = vcopy +lane-suffixes +constn = LANE1:LANE2 +multi_fn = static_assert_imm-in0_exp_len-LANE1 +multi_fn = static_assert_imm-in_exp_len-LANE2 +multi_fn = simd_shuffle-in0_len-!, b:in_t0, b, b, {asc-0-in0_len} +multi_fn = matchn-in0_exp_len-LANE1, simd_shuffle-out_len-!, a, b, {ins-in0_len-in0_len-LANE2} a = 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 -b = 16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1 -validate 1, 2, 3, 4, 5, 6, 7, 8, 8, 7, 6, 5, 4, 3, 2, 1 +b = MAX, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 +n = 1:0 +validate 1, MAX -arm = vmin -aarch64 = smin -link-arm = vmins._EXT_ -link-aarch64 = smin._EXT_ -generate int*_t +aarch64 = zip1 +generate int64x2_t:int64x1_t:int64x2_t, uint64x2_t:uint64x1_t:uint64x2_t, poly64x2_t:poly64x1_t:poly64x2_t -arm = vmin -aarch64 = umin -link-arm = vminu._EXT_ -link-aarch64 = umin._EXT_ -generate uint*_t +/// Insert vector element from another vector element +name = vcopy +lane-suffixes +constn = LANE1:LANE2 +multi_fn = static_assert_imm-in0_exp_len-LANE1 +multi_fn = static_assert_imm-in_exp_len-LANE2 +multi_fn = simd_shuffle-in0_len-!, b:in_t0, b, b, {asc-0-in0_len} +multi_fn = matchn-in0_exp_len-LANE1, simd_shuffle-out_len-!, a, b, {ins-in0_len-in0_len-LANE2} +a = 1., 2., 3., 4. +b = 0.5, 0., 0., 0. +n = 1:0 +validate 1., 0.5, 3., 4. -/// Minimum (vector) -name = vmin -a = 1.0, -2.0, 3.0, -4.0 -b = 0.0, 3.0, 2.0, 8.0 -validate 0.0, -2.0, 2.0, -4.0 +aarch64 = mov +generate float32x4_t:float32x2_t:float32x4_t +aarch64 = zip1 +generate float64x2_t:float64x1_t:float64x2_t -aarch64 = fmin -link-aarch64 = fmin._EXT_ -generate float64x*_t +/// Insert vector element from another vector element +name = vcreate +out-suffix +multi_fn = transmute, a +a = 1 +validate 1, 0, 0, 0, 0, 0, 0, 0 -arm = vmin -aarch64 = fmin +aarch64 = nop +arm = nop +generate u64:int8x8_t, u64:int16x4_t: u64:int32x2_t, u64:int64x1_t +generate u64:uint8x8_t, u64:uint16x4_t: u64:uint32x2_t, u64:uint64x1_t +generate u64:poly8x8_t, u64:poly16x4_t +target = crypto +generate u64:poly64x1_t + +/// Insert vector element from another vector element +name = vcreate +out-suffix +multi_fn = transmute, a +a = 0 +validate 0., 0. + +aarch64 = nop +generate u64:float64x1_t +arm = nop +generate u64:float32x2_t + +/// Fixed-point convert to floating-point +name = vcvt +double-suffixes +fn = simd_cast +a = 1, 2, 3, 4 +validate 1., 2., 3., 4. + +aarch64 = scvtf +generate int64x1_t:float64x1_t, int64x2_t:float64x2_t +aarch64 = ucvtf +generate uint64x1_t:float64x1_t, uint64x2_t:float64x2_t + +arm = vcvt +aarch64 = scvtf +generate int32x2_t:float32x2_t, int32x4_t:float32x4_t +aarch64 = ucvtf +generate uint32x2_t:float32x2_t, uint32x4_t:float32x4_t + +/// Floating-point convert to higher precision long +name = vcvt +double-suffixes +fn = simd_cast +a = -1.2, 1.2 +validate -1.2f32 as f64, 1.2f32 as f64 + +aarch64 = fcvtl +generate float32x2_t:float64x2_t + +/// Floating-point convert to higher precision long +name = vcvt_high +noq-double-suffixes +multi_fn = simd_shuffle2!, b:float32x2_t, a, a, [2, 3] +multi_fn = simd_cast, b +a = -1.2, 1.2, 2.3, 3.4 +validate 2.3f32 as f64, 3.4f32 as f64 + +aarch64 = fcvtl +generate float32x4_t:float64x2_t + +/// Floating-point convert to lower precision narrow +name = vcvt +double-suffixes +fn = simd_cast +a = -1.2, 1.2 +validate -1.2f64 as f32, 1.2f64 as f32 + +aarch64 = fcvtn +generate float64x2_t:float32x2_t + +/// Floating-point convert to lower precision narrow +name = vcvt_high +noq-double-suffixes +multi_fn = simd_shuffle4!, a, {simd_cast, b}, [0, 1, 2, 3] +a = -1.2, 1.2 +b = -2.3, 3.4 +validate -1.2, 1.2, -2.3f64 as f32, 3.4f64 as f32 + +aarch64 = fcvtn +generate float32x2_t:float64x2_t:float32x4_t + +/// Floating-point convert to lower precision narrow, rounding to odd +name = vcvtx +double-suffixes +a = -1.0, 2.0 +validate -1.0, 2.0 + +aarch64 = fcvtxn +link-aarch64 = fcvtxn._EXT2_._EXT_ +generate float64x2_t:float32x2_t + +/// Floating-point convert to lower precision narrow, rounding to odd +name = vcvtx_high +noq-double-suffixes +multi_fn = simd_shuffle4!, a, {vcvtx-noq_doubleself-noext, b}, [0, 1, 2, 3] +a = -1.0, 2.0 +b = -3.0, 4.0 +validate -1.0, 2.0, -3.0, 4.0 + +aarch64 = fcvtxn +generate float32x2_t:float64x2_t:float32x4_t + +/// Fixed-point convert to floating-point +name = vcvt +double-n-suffixes +constn = N +multi_fn = static_assert-N-1-bits +a = 1, 2, 3, 4 +n = 2 +validate 0.25, 0.5, 0.75, 1. + +aarch64 = scvtf +link-aarch64 = vcvtfxs2fp._EXT2_._EXT_ +const-aarch64 = N +generate int64x1_t:float64x1_t, int64x2_t:float64x2_t, i32:f32, i64:f64 + +aarch64 = ucvtf +link-aarch64 = vcvtfxu2fp._EXT2_._EXT_ +const-aarch64 = N +generate uint64x1_t:float64x1_t, uint64x2_t:float64x2_t, u32:f32, u64:f64 + +aarch64 = scvtf +link-aarch64 = vcvtfxs2fp._EXT2_._EXT_ +arm = vcvt +link-arm = vcvtfxs2fp._EXT2_._EXT_ +const-arm = N:i32 +generate int32x2_t:float32x2_t, int32x4_t:float32x4_t + +aarch64 = ucvtf +link-aarch64 = vcvtfxu2fp._EXT2_._EXT_ +arm = vcvt +link-arm = vcvtfxu2fp._EXT2_._EXT_ +const-arm = N:i32 +generate uint32x2_t:float32x2_t, uint32x4_t:float32x4_t + +/// Floating-point convert to fixed-point, rounding toward zero +name = vcvt +double-n-suffixes +constn = N +multi_fn = static_assert-N-1-bits +a = 0.25, 0.5, 0.75, 1. +n = 2 +validate 1, 2, 3, 4 + +aarch64 = fcvtzs +link-aarch64 = vcvtfp2fxs._EXT2_._EXT_ +const-aarch64 = N +generate float64x1_t:int64x1_t, float64x2_t:int64x2_t, f32:i32, f64:i64 + +aarch64 = fcvtzu +link-aarch64 = vcvtfp2fxu._EXT2_._EXT_ +const-aarch64 = N +generate float64x1_t:uint64x1_t, float64x2_t:uint64x2_t, f32:u32, f64:u64 + +aarch64 = fcvtzs +link-aarch64 = vcvtfp2fxs._EXT2_._EXT_ +arm = vcvt +link-arm = vcvtfp2fxs._EXT2_._EXT_ +const-arm = N:i32 +generate float32x2_t:int32x2_t, float32x4_t:int32x4_t + +aarch64 = fcvtzu +link-aarch64 = vcvtfp2fxu._EXT2_._EXT_ +arm = vcvt +link-arm = vcvtfp2fxu._EXT2_._EXT_ +const-arm = N:i32 +generate float32x2_t:uint32x2_t, float32x4_t:uint32x4_t + +/// Fixed-point convert to floating-point +name = vcvt +double-suffixes +multi_fn = a as out_t +a = 1 +validate 1. + +aarch64 = scvtf +generate i32:f32, i64:f64 +aarch64 = ucvtf +generate u32:f32, u64:f64 + +/// Fixed-point convert to floating-point +name = vcvt +double-suffixes +multi_fn = a as out_t +a = 1. +validate 1 + +aarch64 = fcvtzs +generate f32:i32, f64:i64 +aarch64 = fcvtzu +generate f32:u32, f64:u64 + +/// Floating-point convert to signed fixed-point, rounding toward zero +name = vcvt +double-suffixes +fn = simd_cast +a = -1.1, 2.1, -2.9, 3.9 +validate -1, 2, -2, 3 + +aarch64 = fcvtzs +generate float64x1_t:int64x1_t, float64x2_t:int64x2_t + +arm = vcvt +generate float32x2_t:int32x2_t, float32x4_t:int32x4_t + +/// Floating-point convert to unsigned fixed-point, rounding toward zero +name = vcvt +double-suffixes +fn = simd_cast +a = 1.1, 2.1, 2.9, 3.9 +validate 1, 2, 2, 3 + +aarch64 = fcvtzu +generate float64x1_t:uint64x1_t, float64x2_t:uint64x2_t + +arm = vcvt +generate float32x2_t:uint32x2_t, float32x4_t:uint32x4_t + +/// Floating-point convert to signed integer, rounding to nearest with ties to away +name = vcvta +double-suffixes +a = -1.1, 2.1, -2.9, 3.9 +validate -1, 2, -3, 4 + +aarch64 = fcvtas +link-aarch64 = fcvtas._EXT2_._EXT_ +generate float32x2_t:int32x2_t, float32x4_t:int32x4_t, float64x1_t:int64x1_t, float64x2_t:int64x2_t + +/// Floating-point convert to integer, rounding to nearest with ties to away +name = vcvta +double-suffixes +a = 2.9 +validate 3 + +aarch64 = fcvtas +link-aarch64 = fcvtas._EXT2_._EXT_ +generate f32:i32, f64:i64 + +aarch64 = fcvtau +link-aarch64 = fcvtau._EXT2_._EXT_ +generate f32:u32, f64:u64 + +/// Floating-point convert to signed integer, rounding to nearest with ties to even +name = vcvtn +double-suffixes +a = -1.5, 2.1, -2.9, 3.9 +validate -2, 2, -3, 4 + +aarch64 = fcvtns +link-aarch64 = fcvtns._EXT2_._EXT_ +generate float32x2_t:int32x2_t, float32x4_t:int32x4_t, float64x1_t:int64x1_t, float64x2_t:int64x2_t, f32:i32, f64:i64 + +/// Floating-point convert to signed integer, rounding toward minus infinity +name = vcvtm +double-suffixes +a = -1.1, 2.1, -2.9, 3.9 +validate -2, 2, -3, 3 + +aarch64 = fcvtms +link-aarch64 = fcvtms._EXT2_._EXT_ +generate float32x2_t:int32x2_t, float32x4_t:int32x4_t, float64x1_t:int64x1_t, float64x2_t:int64x2_t, f32:i32, f64:i64 + +/// Floating-point convert to signed integer, rounding toward plus infinity +name = vcvtp +double-suffixes +a = -1.1, 2.1, -2.9, 3.9 +validate -1, 3, -2, 4 + +aarch64 = fcvtps +link-aarch64 = fcvtps._EXT2_._EXT_ +generate float32x2_t:int32x2_t, float32x4_t:int32x4_t, float64x1_t:int64x1_t, float64x2_t:int64x2_t, f32:i32, f64:i64 + +/// Floating-point convert to unsigned integer, rounding to nearest with ties to away +name = vcvta +double-suffixes +a = 1.1, 2.1, 2.9, 3.9 +validate 1, 2, 3, 4 + +aarch64 = fcvtau +link-aarch64 = fcvtau._EXT2_._EXT_ +generate float32x2_t:uint32x2_t, float32x4_t:uint32x4_t, float64x1_t:uint64x1_t, float64x2_t:uint64x2_t + +/// Floating-point convert to unsigned integer, rounding to nearest with ties to even +name = vcvtn +double-suffixes +a = 1.5, 2.1, 2.9, 3.9 +validate 2, 2, 3, 4 + +aarch64 = fcvtnu +link-aarch64 = fcvtnu._EXT2_._EXT_ +generate float32x2_t:uint32x2_t, float32x4_t:uint32x4_t, float64x1_t:uint64x1_t, float64x2_t:uint64x2_t, f32:u32, f64:u64 + +/// Floating-point convert to unsigned integer, rounding toward minus infinity +name = vcvtm +double-suffixes +a = 1.1, 2.1, 2.9, 3.9 +validate 1, 2, 2, 3 + +aarch64 = fcvtmu +link-aarch64 = fcvtmu._EXT2_._EXT_ +generate float32x2_t:uint32x2_t, float32x4_t:uint32x4_t, float64x1_t:uint64x1_t, float64x2_t:uint64x2_t, f32:u32, f64:u64 + +/// Floating-point convert to unsigned integer, rounding toward plus infinity +name = vcvtp +double-suffixes +a = 1.1, 2.1, 2.9, 3.9 +validate 2, 3, 3, 4 + +aarch64 = fcvtpu +link-aarch64 = fcvtpu._EXT2_._EXT_ +generate float32x2_t:uint32x2_t, float32x4_t:uint32x4_t, float64x1_t:uint64x1_t, float64x2_t:uint64x2_t, f32:u32, f64:u64 + +/// Set all vector lanes to the same value +name = vdup +lane-suffixes +constn = N +multi_fn = static_assert_imm-in_exp_len-N +multi_fn = simd_shuffle-out_len-!, a, a, {dup-out_len-N as u32} +a = 1, 1, 1, 4, 1, 6, 7, 8, 1, 10, 11, 12, 13, 14, 15, 16 +n = HFLEN +validate 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 + +aarch64 = dup +generate poly64x2_t, poly64x1_t:poly64x2_t + +arm = vdup.l +generate int*_t +generate int8x16_t:int8x8_t, int16x8_t:int16x4_t, int32x4_t:int32x2_t +generate int8x8_t:int8x16_t, int16x4_t:int16x8_t, int32x2_t:int32x4_t + +generate uint*_t +generate uint8x16_t:uint8x8_t, uint16x8_t:uint16x4_t, uint32x4_t:uint32x2_t +generate uint8x8_t:uint8x16_t, uint16x4_t:uint16x8_t, uint32x2_t:uint32x4_t + +generate poly8x8_t, poly8x16_t, poly16x4_t, poly16x8_t +generate poly8x16_t:poly8x8_t, poly16x8_t:poly16x4_t +generate poly8x8_t:poly8x16_t, poly16x4_t:poly16x8_t + +/// Set all vector lanes to the same value +name = vdup +lane-suffixes +constn = N +multi_fn = static_assert_imm-in_exp_len-N +multi_fn = simd_shuffle-out_len-!, a, a, {dup-out_len-N as u32} +a = 1, 1, 1, 4, 1, 6, 7, 8, 1, 10, 11, 12, 13, 14, 15, 16 +n = HFLEN +validate 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 + +aarch64 = dup +arm = vmov +generate int64x2_t, int64x1_t:int64x2_t, uint64x2_t, uint64x1_t:uint64x2_t + +/// Set all vector lanes to the same value +name = vdup +lane-suffixes +constn = N +multi_fn = static_assert_imm-in_exp_len-N +multi_fn = simd_shuffle-out_len-!, a, a, {dup-out_len-N as u32} +a = 1., 1., 1., 4. +n = HFLEN +validate 1., 1., 1., 1. + +aarch64 = dup +generate float64x2_t, float64x1_t:float64x2_t + +arm = vdup.l +generate float*_t, float32x4_t:float32x2_t, float32x2_t:float32x4_t + +/// Set all vector lanes to the same value +name = vdup +lane-suffixes +constn = N +multi_fn = static_assert_imm-in_exp_len-N +multi_fn = a +a = 0 +n = HFLEN +validate 0 + +aarch64 = nop +generate poly64x1_t + +arm = nop +generate int64x1_t, uint64x1_t + +/// Set all vector lanes to the same value +name = vdup +lane-suffixes +constn = N +multi_fn = static_assert_imm-in_exp_len-N +multi_fn = a +a = 0. +n = HFLEN +validate 0. + +aarch64 = nop +generate float64x1_t + +/// Set all vector lanes to the same value +name = vdup +lane-suffixes +constn = N +multi_fn = static_assert_imm-in_exp_len-N +multi_fn = transmute--, {simd_extract, a, N as u32} +a = 0, 1 +n = HFLEN +validate 1 + +aarch64 = nop +generate poly64x2_t:poly64x1_t + +arm = vmov +generate int64x2_t:int64x1_t, uint64x2_t:uint64x1_t + +/// Set all vector lanes to the same value +name = vdup +lane-suffixes +constn = N +multi_fn = static_assert_imm-in_exp_len-N +multi_fn = transmute--, {simd_extract, a, N as u32} +a = 0., 1. +n = HFLEN +validate 1. + +aarch64 = nop +generate float64x2_t:float64x1_t + +/// Set all vector lanes to the same value +name = vdup +lane-suffixes +constn = N +multi_fn = static_assert_imm-in_exp_len-N +multi_fn = simd_extract, a, N as u32 +a = 1, 1, 1, 4, 1, 6, 7, 8, 1, 10, 11, 12, 13, 14, 15, 16 +n = HFLEN +validate 1 + +aarch64 = nop +generate int8x8_t:i8, int8x16_t:i8, int16x4_t:i16, int16x8_t:i16, int32x2_t:i32, int32x4_t:i32, int64x1_t:i64, int64x2_t:i64 +generate uint8x8_t:u8, uint8x16_t:u8, uint16x4_t:u16, uint16x8_t:u16, uint32x2_t:u32, uint32x4_t:u32, uint64x1_t:u64, uint64x2_t:u64 +generate poly8x8_t:p8, poly8x16_t:p8, poly16x4_t:p16, poly16x8_t:p16 + +/// Set all vector lanes to the same value +name = vdup +lane-suffixes +constn = N +multi_fn = static_assert_imm-in_exp_len-N +multi_fn = simd_extract, a, N as u32 +a = 1., 1., 1., 4. +n = HFLEN +validate 1. + +aarch64 = nop +generate float32x2_t:f32, float32x4_t:f32, float64x1_t:f64, float64x2_t:f64 + +/// Extract vector from pair of vectors +name = vext +constn = N +multi_fn = static_assert_imm-out_exp_len-N +multi_fn = matchn-out_exp_len-N, simd_shuffle-out_len-!, a, b, {asc-n-out_len} +a = 0, 8, 8, 9, 8, 9, 9, 11, 8, 9, 9, 11, 9, 11, 14, 15 +b = 9, 11, 14, 15, 16, 17, 18, 19, 0, 8, 8, 9, 8, 9, 9, 11 +n = HFLEN +validate 8, 9, 9, 11, 9, 11, 14, 15, 9, 11, 14, 15, 16, 17, 18, 19 + +arm = "vext.8" +aarch64 = ext +generate int*_t, uint*_t, poly8x8_t, poly8x16_t, poly16x4_t, poly16x8_t + +/// Extract vector from pair of vectors +name = vext +constn = N +multi_fn = static_assert_imm-out_exp_len-N +multi_fn = matchn-out_exp_len-N, simd_shuffle-out_len-!, a, b, {asc-n-out_len} +a = 0, 8, 8, 9, 8, 9, 9, 11, 8, 9, 9, 11, 9, 11, 14, 15 +b = 9, 11, 14, 15, 16, 17, 18, 19, 0, 8, 8, 9, 8, 9, 9, 11 +n = HFLEN +validate 8, 9, 9, 11, 9, 11, 14, 15, 9, 11, 14, 15, 16, 17, 18, 19 + +aarch64 = ext +generate poly64x2_t + +arm = vmov +generate int64x2_t, uint64x2_t + +/// Extract vector from pair of vectors +name = vext +constn = N +multi_fn = static_assert_imm-out_exp_len-N +multi_fn = matchn-out_exp_len-N, simd_shuffle-out_len-!, a, b, {asc-n-out_len} +a = 0., 2., 2., 3. +b = 3., 4., 5., 6., +n = HFLEN +validate 2., 3., 3., 4. + +aarch64 = ext +generate float64x2_t + +arm = "vext.8" +generate float*_t + +/// Multiply-add to accumulator +name = vmla +multi_fn = simd_add, a, {simd_mul, b, c} +a = 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 +b = 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2 +c = 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3 +validate 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21 + +arm = vmla. +aarch64 = mla +generate int*_t, uint*_t + +/// Floating-point multiply-add to accumulator +name = vmla +multi_fn = simd_add, a, {simd_mul, b, c} +a = 0., 1., 2., 3. +b = 2., 2., 2., 2. +c = 3., 3., 3., 3. +validate 6., 7., 8., 9. + +aarch64 = fmul +generate float64x*_t + +arm = vmla. +generate float*_t + +/// Vector multiply accumulate with scalar +name = vmla +n-suffix +multi_fn = vmla-self-noext, a, b, {vdup-nself-noext, c} +a = 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 +b = 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2 +c = 3 +validate 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21 + +aarch64 = mla +arm = vmla. +generate int16x4_t:int16x4_t:i16:int16x4_t, int16x8_t:int16x8_t:i16:int16x8_t, int32x2_t:int32x2_t:i32:int32x2_t, int32x4_t:int32x4_t:i32:int32x4_t +generate uint16x4_t:uint16x4_t:u16:uint16x4_t, uint16x8_t:uint16x8_t:u16:uint16x8_t, uint32x2_t:uint32x2_t:u32:uint32x2_t, uint32x4_t:uint32x4_t:u32:uint32x4_t + +/// Vector multiply accumulate with scalar +name = vmla +n-suffix +multi_fn = vmla-self-noext, a, b, {vdup-nself-noext, c} +a = 0., 1., 2., 3. +b = 2., 2., 2., 2. +c = 3. +validate 6., 7., 8., 9. + +aarch64 = fmul +arm = vmla. +generate float32x2_t:float32x2_t:f32:float32x2_t, float32x4_t:float32x4_t:f32:float32x4_t + +/// Vector multiply accumulate with scalar +name = vmla +in2-lane-suffixes +constn = LANE +multi_fn = static_assert_imm-in2_exp_len-LANE +multi_fn = vmla-self-noext, a, b, {simd_shuffle-in_len-!, c, c, {dup-in_len-LANE as u32}} +a = 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 +b = 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2 +c = 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 +n = 1 +validate 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21 + +aarch64 = mla +arm = vmla. +generate int16x4_t, int16x4_t:int16x4_t:int16x8_t:int16x4_t, int16x8_t:int16x8_t:int16x4_t:int16x8_t, int16x8_t +generate int32x2_t, int32x2_t:int32x2_t:int32x4_t:int32x2_t, int32x4_t:int32x4_t:int32x2_t:int32x4_t, int32x4_t +generate uint16x4_t, uint16x4_t:uint16x4_t:uint16x8_t:uint16x4_t, uint16x8_t:uint16x8_t:uint16x4_t:uint16x8_t, uint16x8_t +generate uint32x2_t, uint32x2_t:uint32x2_t:uint32x4_t:uint32x2_t, uint32x4_t:uint32x4_t:uint32x2_t:uint32x4_t, uint32x4_t + +/// Vector multiply accumulate with scalar +name = vmla +in2-lane-suffixes +constn = LANE +multi_fn = static_assert_imm-in2_exp_len-LANE +multi_fn = vmla-self-noext, a, b, {simd_shuffle-in_len-!, c, c, {dup-in_len-LANE as u32}} +a = 0., 1., 2., 3. +b = 2., 2., 2., 2. +c = 0., 3., 0., 0. +n = 1 +validate 6., 7., 8., 9. + +aarch64 = fmul +arm = vmla. +generate float32x2_t, float32x2_t:float32x2_t:float32x4_t:float32x2_t, float32x4_t:float32x4_t:float32x2_t:float32x4_t, float32x4_t + +/// Signed multiply-add long +name = vmlal +multi_fn = simd_add, a, {vmull-self-noext, b, c} +a = 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 +b = 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2 +c = 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3 +validate 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21 + +arm = vmlal.s +aarch64 = smlal +generate int16x8_t:int8x8_t:int8x8_t:int16x8_t, int32x4_t:int16x4_t:int16x4_t:int32x4_t, int64x2_t:int32x2_t:int32x2_t:int64x2_t + +/// Unsigned multiply-add long +name = vmlal +multi_fn = simd_add, a, {vmull-self-noext, b, c} +a = 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 +b = 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2 +c = 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3 +validate 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21 + +arm = vmlal.s +aarch64 = umlal +generate uint16x8_t:uint8x8_t:uint8x8_t:uint16x8_t, uint32x4_t:uint16x4_t:uint16x4_t:uint32x4_t, uint64x2_t:uint32x2_t:uint32x2_t:uint64x2_t + +/// Vector widening multiply accumulate with scalar +name = vmlal +n-suffix +multi_fn = vmlal-self-noext, a, b, {vdup-nself-noext, c} +a = 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 +b = 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2 +c = 3 +validate 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21 + +arm = vmlal.s +aarch64 = smlal +generate int32x4_t:int16x4_t:i16:int32x4_t, int64x2_t:int32x2_t:i32:int64x2_t +aarch64 = umlal +generate uint32x4_t:uint16x4_t:u16:uint32x4_t, uint64x2_t:uint32x2_t:u32:uint64x2_t + +/// Vector widening multiply accumulate with scalar +name = vmlal_lane +in2-suffix +constn = LANE +multi_fn = static_assert_imm-in2_exp_len-LANE +multi_fn = vmlal-self-noext, a, b, {simd_shuffle-in_len-!, c, c, {dup-in_len-LANE as u32}} +a = 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 +b = 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2 +c = 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 +n = 1 +validate 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21 + +arm = vmlal.s +aarch64 = smlal +generate int32x4_t:int16x4_t:int16x4_t:int32x4_t, int32x4_t:int16x4_t:int16x8_t:int32x4_t +generate int64x2_t:int32x2_t:int32x2_t:int64x2_t, int64x2_t:int32x2_t:int32x4_t:int64x2_t +aarch64 = umlal +generate uint32x4_t:uint16x4_t:uint16x4_t:uint32x4_t, uint32x4_t:uint16x4_t:uint16x8_t:uint32x4_t +generate uint64x2_t:uint32x2_t:uint32x2_t:uint64x2_t, uint64x2_t:uint32x2_t:uint32x4_t:uint64x2_t + +/// Signed multiply-add long +name = vmlal_high +no-q +multi_fn = simd_shuffle-out_len-!, b:half, b, b, {fixed-half-right} +multi_fn = simd_shuffle-out_len-!, c:half, c, c, {fixed-half-right} +multi_fn = vmlal-noqself-noext, a, b, c +a = 8, 7, 6, 5, 4, 3, 2, 1 +b = 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2 +c = 3, 3, 0, 1, 0, 1, 2, 3, 0, 1, 2, 3, 4, 5, 6, 7 +fixed = 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 +validate 8, 9, 10, 11, 12, 13, 14, 15 + +aarch64 = smlal2 +generate int16x8_t:int8x16_t:int8x16_t:int16x8_t, int32x4_t:int16x8_t:int16x8_t:int32x4_t, int64x2_t:int32x4_t:int32x4_t:int64x2_t + +/// Unsigned multiply-add long +name = vmlal_high +no-q +multi_fn = simd_shuffle-out_len-!, b:half, b, b, {fixed-half-right} +multi_fn = simd_shuffle-out_len-!, c:half, c, c, {fixed-half-right} +multi_fn = vmlal-noqself-noext, a, b, c +a = 8, 7, 6, 5, 4, 3, 2, 1 +b = 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2 +c = 3, 3, 0, 1, 0, 1, 2, 3, 0, 1, 2, 3, 4, 5, 6, 7 +fixed = 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 +validate 8, 9, 10, 11, 12, 13, 14, 15 + +aarch64 = umlal2 +generate uint16x8_t:uint8x16_t:uint8x16_t:uint16x8_t, uint32x4_t:uint16x8_t:uint16x8_t:uint32x4_t, uint64x2_t:uint32x4_t:uint32x4_t:uint64x2_t + +/// Multiply-add long +name = vmlal_high_n +no-q +multi_fn = vmlal_high-noqself-noext, a, b, {vdupq_n-noqself-noext, c} +a = 8, 7, 6, 5, 4, 3, 2, 1 +b = 3, 3, 0, 1, 0, 1, 2, 3, 0, 1, 2, 3, 4, 5, 6, 7 +c = 2 +validate 8, 9, 10, 11, 12, 13, 14, 15 + +aarch64 = smlal2 +generate int32x4_t:int16x8_t:i16:int32x4_t, int64x2_t:int32x4_t:i32:int64x2_t +aarch64 = umlal2 +generate uint32x4_t:uint16x8_t:u16:uint32x4_t, uint64x2_t:uint32x4_t:u32:uint64x2_t + +/// Multiply-add long +name = vmlal_high_lane +in2-suffix +constn = LANE +multi_fn = static_assert_imm-in2_exp_len-LANE +multi_fn = vmlal_high-noqself-noext, a, b, {simd_shuffle-in_len-!, c, c, {dup-in_len-LANE as u32}} +a = 8, 7, 6, 5, 4, 3, 2, 1 +b = 3, 3, 0, 1, 0, 1, 2, 3, 0, 1, 2, 3, 4, 5, 6, 7 +c = 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 +n = 1 +validate 8, 9, 10, 11, 12, 13, 14, 15 + +aarch64 = smlal2 +generate int32x4_t:int16x8_t:int16x4_t:int32x4_t, int32x4_t:int16x8_t:int16x8_t:int32x4_t +generate int64x2_t:int32x4_t:int32x2_t:int64x2_t, int64x2_t:int32x4_t:int32x4_t:int64x2_t +aarch64 = umlal2 +generate uint32x4_t:uint16x8_t:uint16x4_t:uint32x4_t, uint32x4_t:uint16x8_t:uint16x8_t:uint32x4_t +generate uint64x2_t:uint32x4_t:uint32x2_t:uint64x2_t, uint64x2_t:uint32x4_t:uint32x4_t:uint64x2_t + +/// Multiply-subtract from accumulator +name = vmls +multi_fn = simd_sub, a, {simd_mul, b, c} +a = 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21 +b = 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2 +c = 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3 +validate 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 + +arm = vmls. +aarch64 = mls +generate int*_t, uint*_t + +/// Floating-point multiply-subtract from accumulator +name = vmls +multi_fn = simd_sub, a, {simd_mul, b, c} +a = 6., 7., 8., 9. +b = 2., 2., 2., 2. +c = 3., 3., 3., 3. +validate 0., 1., 2., 3. + +aarch64 = fmul +generate float64x*_t + +arm = vmls. +generate float*_t + +/// Vector multiply subtract with scalar +name = vmls +n-suffix +multi_fn = vmls-self-noext, a, b, {vdup-nself-noext, c} +a = 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21 +b = 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2 +c = 3 +validate 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 + +aarch64 = mls +arm = vmls. +generate int16x4_t:int16x4_t:i16:int16x4_t, int16x8_t:int16x8_t:i16:int16x8_t, int32x2_t:int32x2_t:i32:int32x2_t, int32x4_t:int32x4_t:i32:int32x4_t +generate uint16x4_t:uint16x4_t:u16:uint16x4_t, uint16x8_t:uint16x8_t:u16:uint16x8_t, uint32x2_t:uint32x2_t:u32:uint32x2_t, uint32x4_t:uint32x4_t:u32:uint32x4_t + +/// Vector multiply subtract with scalar +name = vmls +n-suffix +multi_fn = vmls-self-noext, a, b, {vdup-nself-noext, c} +a = 6., 7., 8., 9. +b = 2., 2., 2., 2. +c = 3. +validate 0., 1., 2., 3. + +aarch64 = fmul +arm = vmls. +generate float32x2_t:float32x2_t:f32:float32x2_t, float32x4_t:float32x4_t:f32:float32x4_t + +/// Vector multiply subtract with scalar +name = vmls +in2-lane-suffixes +constn = LANE +multi_fn = static_assert_imm-in2_exp_len-LANE +multi_fn = vmls-self-noext, a, b, {simd_shuffle-in_len-!, c, c, {dup-in_len-LANE as u32}} +a = 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21 +b = 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2 +c = 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 +n = 1 +validate 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 + +aarch64 = mls +arm = vmls. +generate int16x4_t, int16x4_t:int16x4_t:int16x8_t:int16x4_t, int16x8_t:int16x8_t:int16x4_t:int16x8_t, int16x8_t +generate int32x2_t, int32x2_t:int32x2_t:int32x4_t:int32x2_t, int32x4_t:int32x4_t:int32x2_t:int32x4_t, int32x4_t +generate uint16x4_t, uint16x4_t:uint16x4_t:uint16x8_t:uint16x4_t, uint16x8_t:uint16x8_t:uint16x4_t:uint16x8_t, uint16x8_t +generate uint32x2_t, uint32x2_t:uint32x2_t:uint32x4_t:uint32x2_t, uint32x4_t:uint32x4_t:uint32x2_t:uint32x4_t, uint32x4_t + +/// Vector multiply subtract with scalar +name = vmls +in2-lane-suffixes +constn = LANE +multi_fn = static_assert_imm-in2_exp_len-LANE +multi_fn = vmls-self-noext, a, b, {simd_shuffle-in_len-!, c, c, {dup-in_len-LANE as u32}} +a = 6., 7., 8., 9. +b = 2., 2., 2., 2. +c = 0., 3., 0., 0. +n = 1 +validate 0., 1., 2., 3. + +aarch64 = fmul +arm = vmls. +generate float32x2_t, float32x2_t:float32x2_t:float32x4_t:float32x2_t, float32x4_t:float32x4_t:float32x2_t:float32x4_t, float32x4_t + +/// Signed multiply-subtract long +name = vmlsl +multi_fn = simd_sub, a, {vmull-self-noext, b, c} +a = 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21 +b = 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2 +c = 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3 +validate 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 + +arm = vmlsl.s +aarch64 = smlsl +generate int16x8_t:int8x8_t:int8x8_t:int16x8_t, int32x4_t:int16x4_t:int16x4_t:int32x4_t, int64x2_t:int32x2_t:int32x2_t:int64x2_t + +/// Unsigned multiply-subtract long +name = vmlsl +multi_fn = simd_sub, a, {vmull-self-noext, b, c} +a = 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21 +b = 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2 +c = 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3 +validate 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 + +arm = vmlsl.s +aarch64 = umlsl +generate uint16x8_t:uint8x8_t:uint8x8_t:uint16x8_t, uint32x4_t:uint16x4_t:uint16x4_t:uint32x4_t, uint64x2_t:uint32x2_t:uint32x2_t:uint64x2_t + +/// Vector widening multiply subtract with scalar +name = vmlsl +n-suffix +multi_fn = vmlsl-self-noext, a, b, {vdup-nself-noext, c} +a = 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21 +b = 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2 +c = 3 +validate 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 + +arm = vmlsl.s +aarch64 = smlsl +generate int32x4_t:int16x4_t:i16:int32x4_t, int64x2_t:int32x2_t:i32:int64x2_t +aarch64 = umlsl +generate uint32x4_t:uint16x4_t:u16:uint32x4_t, uint64x2_t:uint32x2_t:u32:uint64x2_t + +/// Vector widening multiply subtract with scalar +name = vmlsl_lane +in2-suffix +constn = LANE +multi_fn = static_assert_imm-in2_exp_len-LANE +multi_fn = vmlsl-self-noext, a, b, {simd_shuffle-in_len-!, c, c, {dup-in_len-LANE as u32}} +a = 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21 +b = 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2 +c = 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 +n = 1 +validate 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 + +arm = vmlsl.s +aarch64 = smlsl +generate int32x4_t:int16x4_t:int16x4_t:int32x4_t, int32x4_t:int16x4_t:int16x8_t:int32x4_t +generate int64x2_t:int32x2_t:int32x2_t:int64x2_t, int64x2_t:int32x2_t:int32x4_t:int64x2_t +aarch64 = umlsl +generate uint32x4_t:uint16x4_t:uint16x4_t:uint32x4_t, uint32x4_t:uint16x4_t:uint16x8_t:uint32x4_t +generate uint64x2_t:uint32x2_t:uint32x2_t:uint64x2_t, uint64x2_t:uint32x2_t:uint32x4_t:uint64x2_t + +/// Signed multiply-subtract long +name = vmlsl_high +no-q +multi_fn = simd_shuffle-out_len-!, b:half, b, b, {fixed-half-right} +multi_fn = simd_shuffle-out_len-!, c:half, c, c, {fixed-half-right} +multi_fn = vmlsl-noqself-noext, a, b, c +a = 14, 15, 16, 17, 18, 19, 20, 21 +b = 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2 +c = 3, 3, 0, 1, 0, 1, 2, 3, 0, 1, 2, 3, 4, 5, 6, 7 +fixed = 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 +validate 14, 13, 12, 11, 10, 9, 8, 7 + +aarch64 = smlsl2 +generate int16x8_t:int8x16_t:int8x16_t:int16x8_t, int32x4_t:int16x8_t:int16x8_t:int32x4_t, int64x2_t:int32x4_t:int32x4_t:int64x2_t + +/// Unsigned multiply-subtract long +name = vmlsl_high +no-q +multi_fn = simd_shuffle-out_len-!, b:half, b, b, {fixed-half-right} +multi_fn = simd_shuffle-out_len-!, c:half, c, c, {fixed-half-right} +multi_fn = vmlsl-noqself-noext, a, b, c +a = 14, 15, 16, 17, 18, 19, 20, 21 +b = 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2 +c = 3, 3, 0, 1, 0, 1, 2, 3, 0, 1, 2, 3, 4, 5, 6, 7 +fixed = 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 +validate 14, 13, 12, 11, 10, 9, 8, 7 + +aarch64 = umlsl2 +generate uint16x8_t:uint8x16_t:uint8x16_t:uint16x8_t, uint32x4_t:uint16x8_t:uint16x8_t:uint32x4_t, uint64x2_t:uint32x4_t:uint32x4_t:uint64x2_t + +/// Multiply-subtract long +name = vmlsl_high_n +no-q +multi_fn = vmlsl_high-noqself-noext, a, b, {vdupq_n-noqself-noext, c} +a = 14, 15, 16, 17, 18, 19, 20, 21 +b = 3, 3, 0, 1, 0, 1, 2, 3, 0, 1, 2, 3, 4, 5, 6, 7 +c = 2 +validate 14, 13, 12, 11, 10, 9, 8, 7 + +aarch64 = smlsl2 +generate int32x4_t:int16x8_t:i16:int32x4_t, int64x2_t:int32x4_t:i32:int64x2_t +aarch64 = umlsl2 +generate uint32x4_t:uint16x8_t:u16:uint32x4_t, uint64x2_t:uint32x4_t:u32:uint64x2_t + +/// Multiply-subtract long +name = vmlsl_high_lane +in2-suffix +constn = LANE +multi_fn = static_assert_imm-in2_exp_len-LANE +multi_fn = vmlsl_high-noqself-noext, a, b, {simd_shuffle-in_len-!, c, c, {dup-in_len-LANE as u32}} +a = 14, 15, 16, 17, 18, 19, 20, 21 +b = 3, 3, 0, 1, 0, 1, 2, 3, 0, 1, 2, 3, 4, 5, 6, 7 +c = 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 +n = 1 +validate 14, 13, 12, 11, 10, 9, 8, 7 + +aarch64 = smlsl2 +generate int32x4_t:int16x8_t:int16x4_t:int32x4_t, int32x4_t:int16x8_t:int16x8_t:int32x4_t +generate int64x2_t:int32x4_t:int32x2_t:int64x2_t, int64x2_t:int32x4_t:int32x4_t:int64x2_t +aarch64 = umlsl2 +generate uint32x4_t:uint16x8_t:uint16x4_t:uint32x4_t, uint32x4_t:uint16x8_t:uint16x8_t:uint32x4_t +generate uint64x2_t:uint32x4_t:uint32x2_t:uint64x2_t, uint64x2_t:uint32x4_t:uint32x4_t:uint64x2_t + +/// Extract narrow +name = vmovn_high +no-q +multi_fn = simd_cast, c:in_t0, b +multi_fn = simd_shuffle-out_len-!, a, c, {asc-0-out_len} +a = 0, 1, 2, 3, 2, 3, 4, 5 +b = 2, 3, 4, 5, 12, 13, 14, 15 +validate 0, 1, 2, 3, 2, 3, 4, 5, 2, 3, 4, 5, 12, 13, 14, 15 + +aarch64 = xtn2 +generate int8x8_t:int16x8_t:int8x16_t, int16x4_t:int32x4_t:int16x8_t, int32x2_t:int64x2_t:int32x4_t +generate uint8x8_t:uint16x8_t:uint8x16_t, uint16x4_t:uint32x4_t:uint16x8_t, uint32x2_t:uint64x2_t:uint32x4_t + +/// Negate +name = vneg +fn = simd_neg +a = 0, 1, -1, 2, -2, 3, -3, 4, -4, 5, -5, 6, -6, 7, -7, 8 +validate 0, -1, 1, -2, 2, -3, 3, -4, 4, -5, 5, -6, 6, -7, 7, -8 + +aarch64 = neg +generate int64x*_t + +arm = vneg.s +generate int*_t + +/// Negate +name = vneg +fn = simd_neg +a = 0., 1., -1., 2., -2., 3., -3., 4. +validate 0., -1., 1., -2., 2., -3., 3., -4. + +aarch64 = fneg +generate float64x*_t + +arm = vneg.s +generate float*_t + +/// Signed saturating negate +name = vqneg +a = MIN, 0, 1, -1, 2, -2, 3, -3, 4, -4, 5, -5, 6, -6, 7, -7 +validate MAX, 0, -1, 1, -2, 2, -3, 3, -4, 4, -5, 5, -6, 6, -7, 7 +link-arm = vqneg._EXT_ +link-aarch64 = sqneg._EXT_ + +aarch64 = sqneg +generate int64x*_t + +arm = vqneg.s +generate int*_t + +/// Saturating subtract +name = vqsub +a = 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42 +b = 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 +validate 41, 40, 39, 38, 37, 36, 35, 34, 33, 32, 31, 30, 29, 28, 27, 26 + +arm = vqsub.s +aarch64 = uqsub +link-arm = llvm.usub.sat._EXT_ +link-aarch64 = uqsub._EXT_ +generate uint*_t, uint64x*_t + +arm = vqsub.s +aarch64 = sqsub +link-arm = llvm.ssub.sat._EXT_ +link-aarch64 = sqsub._EXT_ +generate int*_t, int64x*_t + +/// Saturating subtract +name = vqsub +multi_fn = vdup_n-in_ntt-noext, a:in_ntt, a +multi_fn = vdup_n-in_ntt-noext, b:in_ntt, b +multi_fn = simd_extract, {vqsub-in_ntt-noext, a, b}, 0 +a = 42 +b = 1 +validate 41 + +aarch64 = sqsub +generate i8, i16 +aarch64 = uqsub +generate u8, u16 + +/// Saturating subtract +name = vqsub +a = 42 +b = 1 +validate 41 + +aarch64 = uqsub +link-aarch64 = uqsub._EXT_ +generate u32, u64 + +aarch64 = sqsub +link-aarch64 = sqsub._EXT_ +generate i32, i64 + +/// Halving add +name = vhadd +a = 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42 +b = 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 +validate 21, 22, 22, 23, 23, 24, 24, 25, 25, 26, 26, 27, 27, 28, 28, 29 + +arm = vhadd.s +aarch64 = uhadd +link-aarch64 = uhadd._EXT_ +link-arm = vhaddu._EXT_ +generate uint*_t + +arm = vhadd.s +aarch64 = shadd +link-aarch64 = shadd._EXT_ +link-arm = vhadds._EXT_ +generate int*_t + +/// Reverse bit order +name = vrbit +a = 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30 +validate 0, 64, 32, 96, 16, 80, 48, 112, 8, 72, 40, 104, 24, 88, 56, 120 + +aarch64 = rbit +link-aarch64 = rbit._EXT_ + +generate int8x8_t, int8x16_t + +/// Reverse bit order +name = vrbit +multi_fn = transmute, {vrbit-signed-noext, transmute(a)} +a = 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30 +validate 0, 64, 32, 96, 16, 80, 48, 112, 8, 72, 40, 104, 24, 88, 56, 120 + +aarch64 = rbit + +generate uint8x8_t, uint8x16_t, poly8x8_t, poly8x16_t + +/// Rounding halving add +name = vrhadd +a = 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42 +b = 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 +validate 22, 22, 23, 23, 24, 24, 25, 25, 26, 26, 27, 27, 28, 28, 29, 29 + +arm = vrhadd.s +aarch64 = urhadd +link-arm = vrhaddu._EXT_ +link-aarch64 = urhadd._EXT_ +generate uint*_t + +arm = vrhadd.s +aarch64 = srhadd +link-arm = vrhadds._EXT_ +link-aarch64 = srhadd._EXT_ +generate int*_t + +/// Floating-point round to integral exact, using current rounding mode +name = vrndx +a = -1.5, 0.5, 1.5, 2.5 +validate -2.0, 0.0, 2.0, 2.0 + +aarch64 = frintx +link-aarch64 = llvm.rint._EXT_ +generate float*_t, float64x*_t + +/// Floating-point round to integral, to nearest with ties to away +name = vrnda +a = -1.5, 0.5, 1.5, 2.5 +validate -2.0, 1.0, 2.0, 3.0 + +aarch64 = frinta +link-aarch64 = llvm.round._EXT_ +generate float*_t, float64x*_t + +/// Floating-point round to integral, to nearest with ties to even +name = vrndn +a = -1.5, 0.5, 1.5, 2.5 +validate -2.0, 0.0, 2.0, 2.0 + +link-aarch64 = frintn._EXT_ +aarch64 = frintn +generate float64x*_t + +target = fp-armv8 +arm = vrintn +link-arm = vrintn._EXT_ +generate float*_t + +/// Floating-point round to integral, toward minus infinity +name = vrndm +a = -1.5, 0.5, 1.5, 2.5 +validate -2.0, 0.0, 1.0, 2.0 + +aarch64 = frintm +link-aarch64 = llvm.floor._EXT_ +generate float*_t, float64x*_t + +/// Floating-point round to integral, toward plus infinity +name = vrndp +a = -1.5, 0.5, 1.5, 2.5 +validate -1.0, 1.0, 2.0, 3.0 + +aarch64 = frintp +link-aarch64 = llvm.ceil._EXT_ +generate float*_t, float64x*_t + +/// Floating-point round to integral, toward zero +name = vrnd +a = -1.5, 0.5, 1.5, 2.5 +validate -1.0, 0.0, 1.0, 2.0 + +aarch64 = frintz +link-aarch64 = llvm.trunc._EXT_ +generate float*_t, float64x*_t + +/// Floating-point round to integral, using current rounding mode +name = vrndi +a = -1.5, 0.5, 1.5, 2.5 +validate -2.0, 0.0, 2.0, 2.0 + +aarch64 = frinti +link-aarch64 = llvm.nearbyint._EXT_ +generate float*_t, float64x*_t + +/// Saturating add +name = vqadd +a = 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42 +b = 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 +validate 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58 + +arm = vqadd.s +aarch64 = uqadd +link-arm = llvm.uadd.sat._EXT_ +link-aarch64 = uqadd._EXT_ +generate uint*_t, uint64x*_t + +arm = vqadd.s +aarch64 = sqadd +link-arm = llvm.sadd.sat._EXT_ +link-aarch64 = sqadd._EXT_ +generate int*_t, int64x*_t + +/// Saturating add +name = vqadd +multi_fn = vdup_n-in_ntt-noext, a:in_ntt, a +multi_fn = vdup_n-in_ntt-noext, b:in_ntt, b +multi_fn = simd_extract, {vqadd-in_ntt-noext, a, b}, 0 +a = 42 +b = 1 +validate 43 + +aarch64 = sqadd +generate i8, i16 +aarch64 = uqadd +generate u8, u16 + +/// Saturating add +name = vqadd +a = 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42 +b = 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 +validate 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58 + +aarch64 = uqadd +link-aarch64 = uqadd._EXT_ +generate u32, u64 + +aarch64 = sqadd +link-aarch64 = sqadd._EXT_ +generate i32, i64 + +/// Multiply +name = vmul +a = 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2 +b = 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 +validate 1, 4, 3, 8, 5, 12, 7, 16, 9, 20, 11, 24, 13, 28, 15, 32 +arm = vmul. +aarch64 = mul +fn = simd_mul +generate int*_t, uint*_t + +/// Polynomial multiply +name = vmul +a = 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3 +b = 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 +validate 1, 6, 3, 12, 5, 10, 7, 24, 9, 30, 11, 20, 13, 18, 15, 48 + +aarch64 = pmul +link-aarch64 = pmul._EXT_ +arm = vmul +link-arm = vmulp._EXT_ +generate poly8x8_t, poly8x16_t + +/// Multiply +name = vmul +fn = simd_mul +a = 1.0, 2.0, 1.0, 2.0 +b = 2.0, 3.0, 4.0, 5.0 +validate 2.0, 6.0, 4.0, 10.0 + +aarch64 = fmul +generate float64x*_t + +arm = vmul. +generate float*_t + +/// Vector multiply by scalar +name = vmul +out-n-suffix +multi_fn = simd_mul, a, {vdup-nout-noext, b} +a = 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 +b = 2 +validate 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30, 32 + +arm = vmul +aarch64 = mul +generate int16x4_t:i16:int16x4_t, int16x8_t:i16:int16x8_t, int32x2_t:i32:int32x2_t, int32x4_t:i32:int32x4_t +generate uint16x4_t:u16:uint16x4_t, uint16x8_t:u16:uint16x8_t, uint32x2_t:u32:uint32x2_t, uint32x4_t:u32:uint32x4_t + +/// Vector multiply by scalar +name = vmul +out-n-suffix +multi_fn = simd_mul, a, {vdup-nout-noext, b} +a = 1., 2., 3., 4. +b = 2. +validate 2., 4., 6., 8. + +aarch64 = fmul +generate float64x1_t:f64:float64x1_t, float64x2_t:f64:float64x2_t + +arm = vmul +generate float32x2_t:f32:float32x2_t, float32x4_t:f32:float32x4_t + +/// Multiply +name = vmul +lane-suffixes +constn = LANE +multi_fn = static_assert_imm-in_exp_len-LANE +multi_fn = simd_mul, a, {simd_shuffle-out_len-!, b, b, {dup-out_len-LANE as u32}} +a = 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 +b = 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 +n = 1 +validate 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30, 32 + +aarch64 = mul +arm = vmul +generate int16x4_t, int16x4_t:int16x8_t:int16x4_t, int16x8_t:int16x4_t:int16x8_t, int16x8_t +generate int32x2_t, int32x2_t:int32x4_t:int32x2_t, int32x4_t:int32x2_t:int32x4_t, int32x4_t +generate uint16x4_t, uint16x4_t:uint16x8_t:uint16x4_t, uint16x8_t:uint16x4_t:uint16x8_t, uint16x8_t +generate uint32x2_t, uint32x2_t:uint32x4_t:uint32x2_t, uint32x4_t:uint32x2_t:uint32x4_t, uint32x4_t + +/// Floating-point multiply +name = vmul +lane-suffixes +constn = LANE +multi_fn = static_assert_imm-in_exp_len-LANE +multi_fn = simd_mul, a, {transmute--, {simd_extract, b, LANE as u32}} +a = 1., 2., 3., 4. +b = 2., 0., 0., 0. +n = 0 +validate 2., 4., 6., 8. + +aarch64 = fmul +generate float64x1_t, float64x1_t:float64x2_t:float64x1_t + +/// Floating-point multiply +name = vmul +lane-suffixes +constn = LANE +multi_fn = static_assert_imm-in_exp_len-LANE +multi_fn = simd_mul, a, {simd_shuffle-out_len-!, b, b, {dup-out_len-LANE as u32}} +a = 1., 2., 3., 4. +b = 2., 0., 0., 0. +n = 0 +validate 2., 4., 6., 8. + +aarch64 = fmul +generate float64x2_t:float64x1_t:float64x2_t, float64x2_t + +arm = vmul +generate float32x2_t, float32x2_t:float32x4_t:float32x2_t, float32x4_t:float32x2_t:float32x4_t, float32x4_t + +/// Floating-point multiply +name = vmuls_lane +constn = LANE +multi_fn = static_assert_imm-in_exp_len-LANE +multi_fn = simd_extract, b:f32, b, LANE as u32 +multi_fn = a * b +a = 1. +b = 2., 0., 0., 0. +n = 0 +validate 2. +aarch64 = fmul +generate f32:float32x2_t:f32, f32:float32x4_t:f32 + +/// Floating-point multiply +name = vmuld_lane +constn = LANE +multi_fn = static_assert_imm-in_exp_len-LANE +multi_fn = simd_extract, b:f64, b, LANE as u32 +multi_fn = a * b +a = 1. +b = 2., 0. +n = 0 +validate 2. +aarch64 = fmul +generate f64:float64x1_t:f64, f64:float64x2_t:f64 + +/// Signed multiply long +name = vmull +a = 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 +b = 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2 +validate 1, 4, 3, 8, 5, 12, 7, 16, 9, 20, 11, 24, 13, 28, 15, 32 + +arm = vmull.s +aarch64 = smull +link-arm = vmulls._EXT_ +link-aarch64 = smull._EXT_ +generate int8x8_t:int8x8_t:int16x8_t, int16x4_t:int16x4_t:int32x4_t, int32x2_t:int32x2_t:int64x2_t + +/// Signed multiply long +name = vmull_high +no-q +multi_fn = simd_shuffle-out_len-!, a:half, a, a, {fixed-half-right} +multi_fn = simd_shuffle-out_len-!, b:half, b, b, {fixed-half-right} +multi_fn = vmull-noqself-noext, a, b +a = 1, 2, 9, 10, 9, 10, 11, 12, 9, 10, 11, 12, 13, 14, 15, 16 +b = 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2 +fixed = 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 +validate 9, 20, 11, 24, 13, 28, 15, 32 + +aarch64 = smull2 +generate int8x16_t:int8x16_t:int16x8_t, int16x8_t:int16x8_t:int32x4_t, int32x4_t:int32x4_t:int64x2_t + +/// Unsigned multiply long +name = vmull +a = 1, 2, 3, 4, 5, 6, 7, 8 +b = 1, 2, 1, 2, 1, 2, 1, 2 +validate 1, 4, 3, 8, 5, 12, 7, 16 + +arm = vmull.s +aarch64 = umull +link-arm = vmullu._EXT_ +link-aarch64 = umull._EXT_ +generate uint8x8_t:uint8x8_t:uint16x8_t, uint16x4_t:uint16x4_t:uint32x4_t, uint32x2_t:uint32x2_t:uint64x2_t + +/// Unsigned multiply long +name = vmull_high +no-q +multi_fn = simd_shuffle-out_len-!, a:half, a, a, {fixed-half-right} +multi_fn = simd_shuffle-out_len-!, b:half, b, b, {fixed-half-right} +multi_fn = vmull-noqself-noext, a, b +a = 1, 2, 9, 10, 9, 10, 11, 12, 9, 10, 11, 12, 13, 14, 15, 16 +b = 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2 +fixed = 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 +validate 9, 20, 11, 24, 13, 28, 15, 32 + +aarch64 = umull2 +generate uint8x16_t:uint8x16_t:uint16x8_t, uint16x8_t:uint16x8_t:uint32x4_t, uint32x4_t:uint32x4_t:uint64x2_t + +/// Polynomial multiply long +name = vmull +a = 1, 2, 3, 4, 5, 6, 7, 8 +b = 1, 3, 1, 3, 1, 3, 1, 3 +validate 1, 6, 3, 12, 5, 10, 7, 24 + +arm = vmull.s +aarch64 = pmull +link-arm = vmullp._EXT_ +link-aarch64 = pmull._EXT_ +generate poly8x8_t:poly8x8_t:poly16x8_t + +/// Polynomial multiply long +name = vmull +no-q +a = 15 +b = 3 +validate 17 +target = crypto + +aarch64 = pmull +link-aarch64 = pmull64:p64:p64:p64:int8x16_t +// Because of the support status of llvm, vmull_p64 is currently only available on aarch64 +// arm = vmull +// link-arm = vmullp.v2i64:int64x1_t:int64x1_t:int64x1_t:int64x2_t +generate p64:p64:p128 + + +/// Polynomial multiply long +name = vmull_high +no-q +multi_fn = simd_shuffle-out_len-!, a:half, a, a, {fixed-half-right} +multi_fn = simd_shuffle-out_len-!, b:half, b, b, {fixed-half-right} +multi_fn = vmull-noqself-noext, a, b +a = 1, 2, 9, 10, 9, 10, 11, 12, 9, 10, 11, 12, 13, 14, 15, 16 +b = 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3 +fixed = 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 +validate 9, 30, 11, 20, 13, 18, 15, 48 + +aarch64 = pmull +generate poly8x16_t:poly8x16_t:poly16x8_t + +/// Polynomial multiply long +name = vmull_high +no-q +multi_fn = vmull-noqself-noext, {simd_extract, a, 1}, {simd_extract, b, 1} +a = 1, 15 +b = 1, 3 +validate 17 +target = crypto + +aarch64 = pmull +generate poly64x2_t:poly64x2_t:p128 + +/// Vector long multiply with scalar +name = vmull +n-suffix +multi_fn = vmull-in0-noext, a, {vdup-nin0-noext, b} +a = 1, 2, 3, 4, 5, 6, 7, 8 +b = 2 +validate 2, 4, 6, 8, 10, 12, 14, 16 + +arm = vmull +aarch64 = smull +generate int16x4_t:i16:int32x4_t, int32x2_t:i32:int64x2_t +aarch64 = umull +generate uint16x4_t:u16:uint32x4_t, uint32x2_t:u32:uint64x2_t + +/// Vector long multiply by scalar +name = vmull_lane +constn = LANE +multi_fn = static_assert_imm-in_exp_len-LANE +multi_fn = vmull-in0-noext, a, {simd_shuffle-in0_len-!, b, b, {dup-in0_len-LANE as u32}} +a = 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 +b = 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 +n = 1 +validate 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30, 32 + +arm = vmull +aarch64 = smull +generate int16x4_t:int16x4_t:int32x4_t, int16x4_t:int16x8_t:int32x4_t +generate int32x2_t:int32x2_t:int64x2_t, int32x2_t:int32x4_t:int64x2_t +aarch64 = umull +generate uint16x4_t:uint16x4_t:uint32x4_t, uint16x4_t:uint16x8_t:uint32x4_t +generate uint32x2_t:uint32x2_t:uint64x2_t, uint32x2_t:uint32x4_t:uint64x2_t + +/// Multiply long +name = vmull_high_n +no-q +multi_fn = vmull_high-noqself-noext, a, {vdup-nin0-noext, b} +a = 1, 2, 9, 10, 9, 10, 11, 12, 9, 10, 11, 12, 13, 14, 15, 16 +b = 2 +validate 18, 20, 22, 24, 26, 28, 30, 32 + +aarch64 = smull2 +generate int16x8_t:i16:int32x4_t, int32x4_t:i32:int64x2_t +aarch64 = umull2 +generate uint16x8_t:u16:uint32x4_t, uint32x4_t:u32:uint64x2_t + +/// Multiply long +name = vmull_high_lane +constn = LANE +multi_fn = static_assert_imm-in_exp_len-LANE +multi_fn = vmull_high-noqself-noext, a, {simd_shuffle-in0_len-!, b, b, {dup-in0_len-LANE as u32}} +a = 1, 2, 9, 10, 9, 10, 11, 12, 9, 10, 11, 12, 13, 14, 15, 16 +b = 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 +n = 1 +validate 18, 20, 22, 24, 26, 28, 30, 32 + +aarch64 = smull2 +generate int16x8_t:int16x4_t:int32x4_t, int16x8_t:int16x8_t:int32x4_t +generate int32x4_t:int32x2_t:int64x2_t, int32x4_t:int32x4_t:int64x2_t +aarch64 = umull2 +generate uint16x8_t:uint16x4_t:uint32x4_t, uint16x8_t:uint16x8_t:uint32x4_t +generate uint32x4_t:uint32x2_t:uint64x2_t, uint32x4_t:uint32x4_t:uint64x2_t + +/// Floating-point multiply extended +name = vmulx +a = 1., 2., 3., 4. +b = 2., 2., 2., 2. +validate 2., 4., 6., 8. + +aarch64 = fmulx +link-aarch64 = fmulx._EXT_ +generate float*_t, float64x*_t + +/// Floating-point multiply extended +name = vmulx +lane-suffixes +constn = LANE +multi_fn = static_assert_imm-in_exp_len-LANE +multi_fn = vmulx-in0-noext, a, {transmute--, {simd_extract, b, LANE as u32}} +a = 1. +b = 2., 0. +n = 0 +validate 2. + +aarch64 = fmulx +generate float64x1_t, float64x1_t:float64x2_t:float64x1_t + +/// Floating-point multiply extended +name = vmulx +lane-suffixes +constn = LANE +multi_fn = static_assert_imm-in_exp_len-LANE +multi_fn = vmulx-in0-noext, a, {simd_shuffle-in0_len-!, b, b, {dup-in0_len-LANE as u32}} +a = 1., 2., 3., 4. +b = 2., 0., 0., 0. +n = 0 +validate 2., 4., 6., 8. + +aarch64 = fmulx +generate float32x2_t, float32x2_t:float32x4_t:float32x2_t, float32x4_t:float32x2_t:float32x4_t, float32x4_t +generate float64x2_t:float64x1_t:float64x2_t, float64x2_t + +/// Floating-point multiply extended +name = vmulx +a = 2. +b = 3. +validate 6. + +aarch64 = fmulx +link-aarch64 = fmulx._EXT_ +generate f32, f64 + +/// Floating-point multiply extended +name = vmulx +lane-suffixes +constn = LANE +multi_fn = static_assert_imm-in_exp_len-LANE +multi_fn = vmulx-out-noext, a, {simd_extract, b, LANE as u32} + +a = 2. +b = 3., 0., 0., 0. +n = 0 +validate 6. + +aarch64 = fmulx +generate f32:float32x2_t:f32, f32:float32x4_t:f32, f64:float64x1_t:f64, f64:float64x2_t:f64 + +/// Floating-point fused Multiply-Add to accumulator(vector) +name = vfma +multi_fn = vfma-self-_, b, c, a +a = 8.0, 18.0, 12.0, 10.0 +b = 6.0, 4.0, 7.0, 8.0 +c = 2.0, 3.0, 4.0, 5.0 +validate 20.0, 30.0, 40.0, 50.0 + +link-aarch64 = llvm.fma._EXT_ +aarch64 = fmadd +generate float64x1_t +aarch64 = fmla +generate float64x2_t + +target = fp-armv8 +arm = vfma +link-arm = llvm.fma._EXT_ +generate float*_t + +/// Floating-point fused Multiply-Add to accumulator(vector) +name = vfma +n-suffix +multi_fn = vfma-self-noext, a, b, {vdup-nself-noext, c} +a = 2.0, 3.0, 4.0, 5.0 +b = 6.0, 4.0, 7.0, 8.0 +c = 8.0 +validate 50.0, 35.0, 60.0, 69.0 + +aarch64 = fmadd +generate float64x1_t:float64x1_t:f64:float64x1_t +aarch64 = fmla +generate float64x2_t:float64x2_t:f64:float64x2_t + +target = fp-armv8 +arm = vfma +generate float32x2_t:float32x2_t:f32:float32x2_t, float32x4_t:float32x4_t:f32:float32x4_t + +/// Floating-point fused multiply-add to accumulator +name = vfma +in2-lane-suffixes +constn = LANE +multi_fn = static_assert_imm-in2_exp_len-LANE +multi_fn = vfma-out-noext, a, b, {vdup-nout-noext, {simd_extract, c, LANE as u32}} +a = 2., 3., 4., 5. +b = 6., 4., 7., 8. +c = 2., 0., 0., 0. +n = 0 +validate 14., 11., 18., 21. + +aarch64 = fmla +generate float32x2_t, float32x2_t:float32x2_t:float32x4_t:float32x2_t, float32x4_t:float32x4_t:float32x2_t:float32x4_t, float32x4_t +aarch64 = fmadd +generate float64x1_t +aarch64 = fmla +generate float64x1_t:float64x1_t:float64x2_t:float64x1_t, float64x2_t:float64x2_t:float64x1_t:float64x2_t, float64x2_t + +/// Floating-point fused multiply-add to accumulator +name = vfma +in2-lane-suffixes +constn = LANE +multi_fn = static_assert_imm-in2_exp_len-LANE +multi_fn = simd_extract, c:out_t, c, LANE as u32 +multi_fn = vfma-in2lane-_, b, c, a +a = 2. +b = 6. +c = 3., 0., 0., 0. +n = 0 +validate 20. + +aarch64 = fmla +link-aarch64 = llvm.fma._EXT_:f32:f32:f32:f32 +generate f32:f32:float32x2_t:f32, f32:f32:float32x4_t:f32 +link-aarch64 = llvm.fma._EXT_:f64:f64:f64:f64 +aarch64 = fmadd +generate f64:f64:float64x1_t:f64 +aarch64 = fmla +generate f64:f64:float64x2_t:f64 + +/// Floating-point fused multiply-subtract from accumulator +name = vfms +multi_fn = simd_neg, b:in_t, b +multi_fn = vfma-self-noext, a, b, c +a = 20.0, 30.0, 40.0, 50.0 +b = 6.0, 4.0, 7.0, 8.0 +c = 2.0, 3.0, 4.0, 5.0 +validate 8.0, 18.0, 12.0, 10.0 + +aarch64 = fmsub +generate float64x1_t +aarch64 = fmls +generate float64x2_t + +target = fp-armv8 +arm = vfms +generate float*_t + +/// Floating-point fused Multiply-subtract to accumulator(vector) +name = vfms +n-suffix +multi_fn = vfms-self-noext, a, b, {vdup-nself-noext, c} +a = 50.0, 35.0, 60.0, 69.0 +b = 6.0, 4.0, 7.0, 8.0 +c = 8.0 +validate 2.0, 3.0, 4.0, 5.0 + +aarch64 = fmsub +generate float64x1_t:float64x1_t:f64:float64x1_t +aarch64 = fmls +generate float64x2_t:float64x2_t:f64:float64x2_t + +target = fp-armv8 +arm = vfms +generate float32x2_t:float32x2_t:f32:float32x2_t, float32x4_t:float32x4_t:f32:float32x4_t + +/// Floating-point fused multiply-subtract to accumulator +name = vfms +in2-lane-suffixes +constn = LANE +multi_fn = static_assert_imm-in2_exp_len-LANE +multi_fn = vfms-out-noext, a, b, {vdup-nout-noext, {simd_extract, c, LANE as u32}} +a = 14., 11., 18., 21. +b = 6., 4., 7., 8. +c = 2., 0., 0., 0. +n = 0 +validate 2., 3., 4., 5. + +aarch64 = fmls +generate float32x2_t, float32x2_t:float32x2_t:float32x4_t:float32x2_t, float32x4_t:float32x4_t:float32x2_t:float32x4_t, float32x4_t +aarch64 = fmsub +generate float64x1_t +aarch64 = fmls +generate float64x1_t:float64x1_t:float64x2_t:float64x1_t, float64x2_t:float64x2_t:float64x1_t:float64x2_t, float64x2_t + +/// Floating-point fused multiply-subtract to accumulator +name = vfms +in2-lane-suffixes +constn = LANE +multi_fn = vfma-in2lane-::, a, -b, c +a = 14. +b = 6. +c = 2., 0., 0., 0. +n = 0 +validate 2. + +aarch64 = fmls +generate f32:f32:float32x2_t:f32, f32:f32:float32x4_t:f32 +aarch64 = fmsub +generate f64:f64:float64x1_t:f64 +aarch64 = fmls +generate f64:f64:float64x2_t:f64 + +/// Divide +name = vdiv +fn = simd_div +a = 2.0, 6.0, 4.0, 10.0 +b = 1.0, 2.0, 1.0, 2.0 +validate 2.0, 3.0, 4.0, 5.0 + +aarch64 = fdiv +generate float*_t, float64x*_t + +/// Subtract +name = vsub +a = 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 +b = 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2 +validate 0, 0, 2, 2, 4, 4, 6, 6, 8, 8, 10, 10, 12, 12, 14, 14 +arm = vsub. +aarch64 = sub +fn = simd_sub +generate int*_t, uint*_t, int64x*_t, uint64x*_t + +/// Subtract +name = vsub +fn = simd_sub +a = 1.0, 4.0, 3.0, 8.0 +b = 1.0, 2.0, 3.0, 4.0 +validate 0.0, 2.0, 0.0, 4.0 + +aarch64 = fsub +generate float64x*_t + +arm = vsub. +generate float*_t + +/// Signed Add Long across Vector +name = vaddlv +a = 1, 2, 3, 4 +validate 10 + +aarch64 = saddlv +link-aarch64 = llvm.aarch64.neon.saddlv.i32._EXT_ +generate int16x4_t:i32 + +/// Signed Add Long across Vector +name = vaddlv +a = 1, 2, 3, 4, 5, 6, 7, 8 +validate 36 + +aarch64 = saddlv +link-aarch64 = llvm.aarch64.neon.saddlv.i32._EXT_ +generate int16x8_t:i32 + +/// Signed Add Long across Vector +name = vaddlv +a = 1, 2 +validate 3 + +aarch64 = saddlp +link-aarch64 = llvm.aarch64.neon.saddlv.i64._EXT_ +generate int32x2_t:i64 + +/// Signed Add Long across Vector +name = vaddlv +a = 1, 2, 3, 4 +validate 10 + +aarch64 = saddlv +link-aarch64 = llvm.aarch64.neon.saddlv.i64._EXT_ +generate int32x4_t:i64 + +/// Unsigned Add Long across Vector +name = vaddlv +a = 1, 2, 3, 4 +validate 10 + +aarch64 = uaddlv +link-aarch64 = llvm.aarch64.neon.uaddlv.i32._EXT_ +generate uint16x4_t:u32 + +/// Unsigned Add Long across Vector +name = vaddlv +a = 1, 2, 3, 4, 5, 6, 7, 8 +validate 36 + +aarch64 = uaddlv +link-aarch64 = llvm.aarch64.neon.uaddlv.i32._EXT_ +generate uint16x8_t:u32 + +/// Unsigned Add Long across Vector +name = vaddlv +a = 1, 2 +validate 3 + +aarch64 = uaddlp +link-aarch64 = llvm.aarch64.neon.uaddlv.i64._EXT_ +generate uint32x2_t:u64 + +/// Unsigned Add Long across Vector +name = vaddlv +a = 1, 2, 3, 4 +validate 10 + +aarch64 = uaddlv +link-aarch64 = llvm.aarch64.neon.uaddlv.i64._EXT_ +generate uint32x4_t:u64 + +/// Subtract returning high narrow +name = vsubhn +no-q +multi_fn = fixed, c:in_t +multi_fn = simd_cast, {simd_shr, {simd_sub, a, b}, transmute(c)} +a = MAX, MIN, 1, 1, MAX, MIN, 1, 1 +b = 1, 0, 0, 0, 1, 0, 0, 0 +fixed = HFBITS, HFBITS, HFBITS, HFBITS, HFBITS, HFBITS, HFBITS, HFBITS, HFBITS, HFBITS, HFBITS, HFBITS, HFBITS, HFBITS, HFBITS, HFBITS +validate MAX, MIN, 0, 0, MAX, MIN, 0, 0 + +arm = vsubhn +aarch64 = subhn +generate int16x8_t:int8x8_t, int32x4_t:int16x4_t, int64x2_t:int32x2_t +generate uint16x8_t:uint8x8_t, uint32x4_t:uint16x4_t, uint64x2_t:uint32x2_t + +/// Subtract returning high narrow +name = vsubhn_high +no-q +multi_fn = vsubhn-noqself-noext, d:in_t0, b, c +multi_fn = simd_shuffle-out_len-!, a, d, {asc-0-out_len} +a = MAX, 0, MAX, 0, MAX, 0, MAX, 0 +b = MAX, 1, MAX, 1, MAX, 1, MAX, 1 +c = 1, 0, 1, 0, 1, 0, 1, 0 +validate MAX, 0, MAX, 0, MAX, 0, MAX, 0, MAX, 0, MAX, 0, MAX, 0, MAX, 0 + +arm = vsubhn +aarch64 = subhn2 +generate int8x8_t:int16x8_t:int16x8_t:int8x16_t, int16x4_t:int32x4_t:int32x4_t:int16x8_t, int32x2_t:int64x2_t:int64x2_t:int32x4_t +generate uint8x8_t:uint16x8_t:uint16x8_t:uint8x16_t, uint16x4_t:uint32x4_t:uint32x4_t:uint16x8_t, uint32x2_t:uint64x2_t:uint64x2_t:uint32x4_t + +/// Signed halving subtract +name = vhsub +a = 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 +b = 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2 +validate 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7 + +arm = vhsub.s +aarch64 = uhsub +link-arm = vhsubu._EXT_ +link-aarch64 = uhsub._EXT_ +generate uint*_t + +arm = vhsub.s +aarch64 = shsub +link-arm = vhsubs._EXT_ +link-aarch64 = shsub._EXT_ +generate int*_t + +/// Signed Subtract Wide +name = vsubw +no-q +multi_fn = simd_sub, a, {simd_cast, b} +a = 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12, 13, 14, 15, 16 +b = 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12, 13, 14, 15, 16 +validate 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 + +arm = vsubw +aarch64 = ssubw +generate int16x8_t:int8x8_t:int16x8_t, int32x4_t:int16x4_t:int32x4_t, int64x2_t:int32x2_t:int64x2_t + +/// Unsigned Subtract Wide +name = vsubw +no-q +multi_fn = simd_sub, a, {simd_cast, b} +a = 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12, 13, 14, 15, 16 +b = 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12, 13, 14, 15, 16 +validate 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 + +arm = vsubw +aarch64 = usubw +generate uint16x8_t:uint8x8_t:uint16x8_t, uint32x4_t:uint16x4_t:uint32x4_t, uint64x2_t:uint32x2_t:uint64x2_t + +/// Signed Subtract Wide +name = vsubw_high +no-q +multi_fn = simd_shuffle8!, c:int8x8_t, b, b, [8, 9, 10, 11, 12, 13, 14, 15] +multi_fn = simd_sub, a, {simd_cast, c} +a = 8, 9, 10, 12, 13, 14, 15, 16 +b = 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12, 13, 14, 15, 16 +validate 0, 0, 0, 0, 0, 0, 0, 0 + +aarch64 = ssubw +generate int16x8_t:int8x16_t:int16x8_t + +/// Signed Subtract Wide +name = vsubw_high +no-q +multi_fn = simd_shuffle4!, c:int16x4_t, b, b, [4, 5, 6, 7] +multi_fn = simd_sub, a, {simd_cast, c} +a = 8, 9, 10, 11 +b = 0, 1, 2, 3, 8, 9, 10, 11 +validate 0, 0, 0, 0 + +aarch64 = ssubw +generate int32x4_t:int16x8_t:int32x4_t + +/// Signed Subtract Wide +name = vsubw_high +no-q +multi_fn = simd_shuffle2!, c:int32x2_t, b, b, [2, 3] +multi_fn = simd_sub, a, {simd_cast, c} +a = 8, 9 +b = 6, 7, 8, 9 +validate 0, 0 + +aarch64 = ssubw +generate int64x2_t:int32x4_t:int64x2_t + +/// Unsigned Subtract Wide +name = vsubw_high +no-q +multi_fn = simd_shuffle8!, c:uint8x8_t, b, b, [8, 9, 10, 11, 12, 13, 14, 15] +multi_fn = simd_sub, a, {simd_cast, c} +a = 8, 9, 10, 11, 12, 13, 14, 15 +b = 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 +validate 0, 0, 0, 0, 0, 0, 0, 0 + +aarch64 = usubw +generate uint16x8_t:uint8x16_t:uint16x8_t + +/// Unsigned Subtract Wide +name = vsubw_high +no-q +multi_fn = simd_shuffle4!, c:uint16x4_t, b, b, [4, 5, 6, 7] +multi_fn = simd_sub, a, {simd_cast, c} +a = 8, 9, 10, 11 +b = 0, 1, 2, 3, 8, 9, 10, 11 +validate 0, 0, 0, 0 + +aarch64 = usubw +generate uint32x4_t:uint16x8_t:uint32x4_t + +/// Unsigned Subtract Wide +name = vsubw_high +no-q +multi_fn = simd_shuffle2!, c:uint32x2_t, b, b, [2, 3] +multi_fn = simd_sub, a, {simd_cast, c} +a = 8, 9 +b = 6, 7, 8, 9 +validate 0, 0 + +aarch64 = usubw +generate uint64x2_t:uint32x4_t:uint64x2_t + +/// Signed Subtract Long +name = vsubl +no-q +multi_fn = simd_cast, c:out_t, a +multi_fn = simd_cast, d:out_t, b +multi_fn = simd_sub, c, d + +a = MAX, MIN, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 +b = MAX, MIN, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 +validate 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 + +arm = vsubl +aarch64 = ssubl +generate int8x8_t:int8x8_t:int16x8_t, int16x4_t:int16x4_t:int32x4_t, int32x2_t:int32x2_t:int64x2_t + +/// Unsigned Subtract Long +name = vsubl +no-q +multi_fn = simd_cast, c:out_t, a +multi_fn = simd_cast, d:out_t, b +multi_fn = simd_sub, c, d + +a = MAX, MIN, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 +b = MAX, MIN, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 +validate 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 + +arm = vsubl +aarch64 = usubl +generate uint8x8_t:uint8x8_t:uint16x8_t, uint16x4_t:uint16x4_t:uint32x4_t, uint32x2_t:uint32x2_t:uint64x2_t + +/// Signed Subtract Long +name = vsubl_high +no-q +multi_fn = simd_shuffle8!, c:int8x8_t, a, a, [8, 9, 10, 11, 12, 13, 14, 15] +multi_fn = simd_cast, d:out_t, c +multi_fn = simd_shuffle8!, e:int8x8_t, b, b, [8, 9, 10, 11, 12, 13, 14, 15] +multi_fn = simd_cast, f:out_t, e +multi_fn = simd_sub, d, f + +a = 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 +b = 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2 +validate 6, 7, 8, 9, 10, 11, 12, 13 + +aarch64 = ssubl +generate int8x16_t:int8x16_t:int16x8_t + +/// Signed Subtract Long +name = vsubl_high +no-q +multi_fn = simd_shuffle4!, c:int16x4_t, a, a, [4, 5, 6, 7] +multi_fn = simd_cast, d:out_t, c +multi_fn = simd_shuffle4!, e:int16x4_t, b, b, [4, 5, 6, 7] +multi_fn = simd_cast, f:out_t, e +multi_fn = simd_sub, d, f + +a = 8, 9, 10, 11, 12, 13, 14, 15 +b = 6, 6, 6, 6, 8, 8, 8, 8 +validate 4, 5, 6, 7 + +aarch64 = ssubl +generate int16x8_t:int16x8_t:int32x4_t + +/// Signed Subtract Long +name = vsubl_high +no-q +multi_fn = simd_shuffle2!, c:int32x2_t, a, a, [2, 3] +multi_fn = simd_cast, d:out_t, c +multi_fn = simd_shuffle2!, e:int32x2_t, b, b, [2, 3] +multi_fn = simd_cast, f:out_t, e +multi_fn = simd_sub, d, f + +a = 12, 13, 14, 15 +b = 6, 6, 8, 8 +validate 6, 7 + +aarch64 = ssubl +generate int32x4_t:int32x4_t:int64x2_t + +/// Unsigned Subtract Long +name = vsubl_high +no-q +multi_fn = simd_shuffle8!, c:uint8x8_t, a, a, [8, 9, 10, 11, 12, 13, 14, 15] +multi_fn = simd_cast, d:out_t, c +multi_fn = simd_shuffle8!, e:uint8x8_t, b, b, [8, 9, 10, 11, 12, 13, 14, 15] +multi_fn = simd_cast, f:out_t, e +multi_fn = simd_sub, d, f + +a = 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 +b = 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2 +validate 6, 7, 8, 9, 10, 11, 12, 13 + +aarch64 = usubl +generate uint8x16_t:uint8x16_t:uint16x8_t + +/// Unsigned Subtract Long +name = vsubl_high +no-q +multi_fn = simd_shuffle4!, c:uint16x4_t, a, a, [4, 5, 6, 7] +multi_fn = simd_cast, d:out_t, c +multi_fn = simd_shuffle4!, e:uint16x4_t, b, b, [4, 5, 6, 7] +multi_fn = simd_cast, f:out_t, e +multi_fn = simd_sub, d, f + +a = 8, 9, 10, 11, 12, 13, 14, 15 +b = 6, 6, 6, 6, 8, 8, 8, 8 +validate 4, 5, 6, 7 + +aarch64 = usubl +generate uint16x8_t:uint16x8_t:uint32x4_t + +/// Unsigned Subtract Long +name = vsubl_high +no-q +multi_fn = simd_shuffle2!, c:uint32x2_t, a, a, [2, 3] +multi_fn = simd_cast, d:out_t, c +multi_fn = simd_shuffle2!, e:uint32x2_t, b, b, [2, 3] +multi_fn = simd_cast, f:out_t, e +multi_fn = simd_sub, d, f + +a = 12, 13, 14, 15 +b = 6, 6, 8, 8 +validate 6, 7 + +aarch64 = usubl +generate uint32x4_t:uint32x4_t:uint64x2_t + +/// Maximum (vector) +name = vmax +a = 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 +b = 16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1 +validate 16, 15, 14, 13, 12, 11, 10, 9, 9, 10, 11, 12, 13, 14, 15, 16 + +arm = vmax +aarch64 = smax +link-arm = vmaxs._EXT_ +link-aarch64 = smax._EXT_ +generate int*_t + +arm = vmax +aarch64 = umax +link-arm = vmaxu._EXT_ +link-aarch64 = umax._EXT_ +generate uint*_t + +/// Maximum (vector) +name = vmax +a = 1.0, -2.0, 3.0, -4.0 +b = 0.0, 3.0, 2.0, 8.0 +validate 1.0, 3.0, 3.0, 8.0 + +aarch64 = fmax +link-aarch64 = fmax._EXT_ +generate float64x*_t + +arm = vmax +aarch64 = fmax +link-arm = vmaxs._EXT_ +link-aarch64 = fmax._EXT_ +generate float*_t + +/// Floating-point Maximun Number (vector) +name = vmaxnm +a = 1.0, 2.0, 3.0, -4.0 +b = 8.0, 16.0, -1.0, 6.0 +validate 8.0, 16.0, 3.0, 6.0 + +aarch64 = fmaxnm +link-aarch64 = fmaxnm._EXT_ +generate float64x*_t + +target = fp-armv8 +arm = vmaxnm +aarch64 = fmaxnm +link-arm = vmaxnm._EXT_ +link-aarch64 = fmaxnm._EXT_ +generate float*_t + +/// Floating-point Maximum Number Pairwise (vector). +name = vpmaxnm +a = 1.0, 2.0 +b = 6.0, -3.0 +validate 2.0, 6.0 +aarch64 = fmaxnmp +link-aarch64 = fmaxnmp._EXT_ +generate float32x2_t:float32x2_t:float32x2_t, float64x2_t:float64x2_t:float64x2_t + +/// Floating-point Maximum Number Pairwise (vector). +name = vpmaxnm +a = 1.0, 2.0, 3.0, -4.0 +b = 8.0, 16.0, -1.0, 6.0 +validate 2.0, 3.0, 16.0, 6.0 +aarch64 = fmaxnmp +link-aarch64 = fmaxnmp._EXT_ +generate float32x4_t:float32x4_t:float32x4_t + +/// Minimum (vector) +name = vmin +a = 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 +b = 16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1 +validate 1, 2, 3, 4, 5, 6, 7, 8, 8, 7, 6, 5, 4, 3, 2, 1 + +arm = vmin +aarch64 = smin +link-arm = vmins._EXT_ +link-aarch64 = smin._EXT_ +generate int*_t + +arm = vmin +aarch64 = umin +link-arm = vminu._EXT_ +link-aarch64 = umin._EXT_ +generate uint*_t + +/// Minimum (vector) +name = vmin +a = 1.0, -2.0, 3.0, -4.0 +b = 0.0, 3.0, 2.0, 8.0 +validate 0.0, -2.0, 2.0, -4.0 + +aarch64 = fmin +link-aarch64 = fmin._EXT_ +generate float64x*_t + +arm = vmin +aarch64 = fmin link-arm = vmins._EXT_ link-aarch64 = fmin._EXT_ generate float*_t + +/// Floating-point Minimun Number (vector) +name = vminnm +a = 1.0, 2.0, 3.0, -4.0 +b = 8.0, 16.0, -1.0, 6.0 +validate 1.0, 2.0, -1.0, -4.0 + +aarch64 = fminnm +link-aarch64 = fminnm._EXT_ +generate float64x*_t + +target = fp-armv8 +arm = vminnm +aarch64 = fminnm +link-arm = vminnm._EXT_ +link-aarch64 = fminnm._EXT_ +generate float*_t + +/// Floating-point Minimum Number Pairwise (vector). +name = vpminnm +a = 1.0, 2.0 +b = 6.0, -3.0 +validate 1.0, -3.0 +aarch64 = fminnmp +link-aarch64 = fminnmp._EXT_ +generate float32x2_t:float32x2_t:float32x2_t, float64x2_t:float64x2_t:float64x2_t + +/// Floating-point Minimum Number Pairwise (vector). +name = vpminnm +a = 1.0, 2.0, 3.0, -4.0 +b = 8.0, 16.0, -1.0, 6.0 +validate 1.0, -4.0, 8.0, -1.0 +aarch64 = fminnmp +link-aarch64 = fminnmp._EXT_ +generate float32x4_t:float32x4_t:float32x4_t + +/// Signed saturating doubling multiply long +name = vqdmull +a = 0, 1, 2, 3, 4, 5, 6, 7 +b = 1, 2, 3, 4, 5, 6, 7, 8 +validate 0, 4, 12, 24, 40, 60, 84, 108 + +aarch64 = sqdmull +link-aarch64 = sqdmull._EXT2_ +arm = vqdmull +link-arm = vqdmull._EXT2_ +generate int16x4_t:int16x4_t:int32x4_t, int32x2_t:int32x2_t:int64x2_t + +/// Signed saturating doubling multiply long +name = vqdmull +multi_fn = vdup_n-in_ntt-noext, a:in_ntt, a +multi_fn = vdup_n-in_ntt-noext, b:in_ntt, b +multi_fn = simd_extract, {vqdmull-in_ntt-noext, a, b}, 0 +a = 2 +b = 3 +validate 12 + +aarch64 = sqdmull +generate i16:i16:i32 + +/// Signed saturating doubling multiply long +name = vqdmull +a = 2 +b = 3 +validate 12 + +aarch64 = sqdmull +link-aarch64 = sqdmulls.scalar +generate i32:i32:i64 + +/// Vector saturating doubling long multiply with scalar +name = vqdmull_n +no-q +multi_fn = vqdmull-in_ntt-noext, a, {vdup_n-in_ntt-noext, b} +a = 2, 4, 6, 8 +b = 2 +validate 8, 16, 24, 32 + +aarch64 = sqdmull +arm = vqdmull +generate int16x4_t:i16:int32x4_t, int32x2_t:i32:int64x2_t + +/// Signed saturating doubling multiply long +name = vqdmull_high +no-q +multi_fn = simd_shuffle-out_len-!, a:half, a, a, {asc-halflen-halflen} +multi_fn = simd_shuffle-out_len-!, b:half, b, b, {asc-halflen-halflen} +multi_fn = vqdmull-noqself-noext, a, b +a = 0, 1, 4, 5, 4, 5, 6, 7 +b = 1, 2, 5, 6, 5, 6, 7, 8 +validate 40, 60, 84, 112 + +aarch64 = sqdmull2 +generate int16x8_t:int16x8_t:int32x4_t, int32x4_t:int32x4_t:int64x2_t + +/// Signed saturating doubling multiply long +name = vqdmull_high_n +no-q +multi_fn = simd_shuffle-out_len-!, a:in_ntt, a, a, {asc-out_len-out_len} +multi_fn = vdup_n-in_ntt-noext, b:in_ntt, b +multi_fn = vqdmull-in_ntt-noext, a, b +a = 0, 2, 8, 10, 8, 10, 12, 14 +b = 2 +validate 32, 40, 48, 56 + +aarch64 = sqdmull2 +generate int16x8_t:i16:int32x4_t, int32x4_t:i32:int64x2_t + +/// Vector saturating doubling long multiply by scalar +name = vqdmull_lane +constn = N +multi_fn = static_assert_imm-in_exp_len-N +multi_fn = simd_shuffle-out_len-!, b:in_t0, b, b, {dup-out_len-N as u32} +multi_fn = vqdmull-noqself-noext, a, b +a = 1, 2, 3, 4 +b = 0, 2, 2, 0, 2, 0, 0, 0 +n = HFLEN +validate 4, 8, 12, 16 + +aarch64 = sqdmull +generate int16x4_t:int16x8_t:int32x4_t, int32x2_t:int32x4_t:int64x2_t + +arm = vqdmull +generate int16x4_t:int16x4_t:int32x4_t, int32x2_t:int32x2_t:int64x2_t + +/// Signed saturating doubling multiply long +name = vqdmullh_lane +constn = N +multi_fn = static_assert_imm-in_exp_len-N +multi_fn = simd_extract, b:in_t0, b, N as u32 +multi_fn = vqdmullh-noqself-noext, a, b +a = 2 +b = 0, 2, 2, 0, 2, 0, 0, 0 +n = HFLEN +validate 8 + +aarch64 = sqdmull +generate i16:int16x4_t:i32, i16:int16x8_t:i32 + +/// Signed saturating doubling multiply long +name = vqdmulls_lane +constn = N +multi_fn = static_assert_imm-in_exp_len-N +multi_fn = simd_extract, b:in_t0, b, N as u32 +multi_fn = vqdmulls-noqself-noext, a, b +a = 2 +b = 0, 2, 2, 0, 2, 0, 0, 0 +n = HFLEN +validate 8 + +aarch64 = sqdmull +generate i32:int32x2_t:i64, i32:int32x4_t:i64 + +/// Signed saturating doubling multiply long +name = vqdmull_high_lane +constn = N +multi_fn = static_assert_imm-in_exp_len-N +multi_fn = simd_shuffle-out_len-!, a:in_t, a, a, {asc-out_len-out_len} +multi_fn = simd_shuffle-out_len-!, b:in_t, b, b, {dup-out_len-N as u32} +multi_fn = vqdmull-self-noext, a, b +a = 0, 1, 4, 5, 4, 5, 6, 7 +b = 0, 2, 2, 0, 2, 0, 0, 0 +n = HFLEN +validate 16, 20, 24, 28 + +aarch64 = sqdmull2 +generate int16x8_t:int16x4_t:int32x4_t, int32x4_t:int32x2_t:int64x2_t + +/// Signed saturating doubling multiply long +name = vqdmull_high_lane +constn = N +multi_fn = static_assert_imm-in_exp_len-N +multi_fn = simd_shuffle-out_len-!, a:half, a, a, {asc-out_len-out_len} +multi_fn = simd_shuffle-out_len-!, b:half, b, b, {dup-out_len-N as u32} +multi_fn = vqdmull-noqself-noext, a, b +a = 0, 1, 4, 5, 4, 5, 6, 7 +b = 0, 2, 2, 0, 2, 0, 0, 0 +n = HFLEN +validate 16, 20, 24, 28 + +aarch64 = sqdmull2 +generate int16x8_t:int16x8_t:int32x4_t, int32x4_t:int32x4_t:int64x2_t + +/// Signed saturating doubling multiply-add long +name = vqdmlal +multi_fn = vqadd-out-noext, a, {vqdmull-self-noext, b, c} +a = 1, 1, 1, 1 +b = 1, 2, 3, 4 +c = 2, 2, 2, 2 +validate 5, 9, 13, 17 + +aarch64 = sqdmlal +arm = vqdmlal +generate int32x4_t:int16x4_t:int16x4_t:int32x4_t, int64x2_t:int32x2_t:int32x2_t:int64x2_t + +/// Vector widening saturating doubling multiply accumulate with scalar +name = vqdmlal +n-suffix +multi_fn = vqadd-out-noext, a, {vqdmull_n-self-noext, b, c} +a = 1, 1, 1, 1 +b = 1, 2, 3, 4 +c = 2 +validate 5, 9, 13, 17 + +aarch64 = sqdmlal +arm = vqdmlal +generate int32x4_t:int16x4_t:i16:int32x4_t, int64x2_t:int32x2_t:i32:int64x2_t + +/// Signed saturating doubling multiply-add long +name = vqdmlal_high +no-q +multi_fn = vqadd-out-noext, a, {vqdmull_high-noqself-noext, b, c} +a = 1, 2, 3, 4 +b = 0, 1, 4, 5, 4, 5, 6, 7 +c = 1, 2, 5, 6, 5, 6, 7, 8 +validate 41, 62, 87, 116 + +aarch64 = sqdmlal2 +generate int32x4_t:int16x8_t:int16x8_t:int32x4_t, int64x2_t:int32x4_t:int32x4_t:int64x2_t + +/// Signed saturating doubling multiply-add long +name = vqdmlal_high_n +no-q +multi_fn = vqadd-out-noext, a, {vqdmull_high_n-noqself-noext, b, c} +a = 1, 2, 3, 4 +b = 0, 2, 8, 10, 8, 10, 12, 14 +c = 2 +validate 33, 42, 51, 60 + +aarch64 = sqdmlal2 +generate int32x4_t:int16x8_t:i16:int32x4_t, int64x2_t:int32x4_t:i32:int64x2_t + +/// Vector widening saturating doubling multiply accumulate with scalar +name = vqdmlal_lane +in2-suffix +constn = N +multi_fn = static_assert_imm-in2_exp_len-N +multi_fn = vqadd-out-noext, a, {vqdmull_lane-in2-::, b, c} +a = 1, 2, 3, 4 +b = 1, 2, 3, 4 +c = 0, 2, 2, 0, 2, 0, 0, 0 +n = HFLEN +validate 5, 10, 15, 20 + +aarch64 = sqdmlal +generate int32x4_t:int16x4_t:int16x8_t:int32x4_t, int64x2_t:int32x2_t:int32x4_t:int64x2_t + +arm = vqdmlal +generate int32x4_t:int16x4_t:int16x4_t:int32x4_t, int64x2_t:int32x2_t:int32x2_t:int64x2_t + +/// Signed saturating doubling multiply-add long +name = vqdmlal_high_lane +in2-suffix +constn = N +multi_fn = static_assert_imm-in2_exp_len-N +multi_fn = vqadd-out-noext, a, {vqdmull_high_lane-in2-::, b, c} +a = 1, 2, 3, 4 +b = 0, 1, 4, 5, 4, 5, 6, 7 +c = 0, 2, 0, 0, 0, 0, 0, 0 +n = 1 +validate 17, 22, 27, 32 + +aarch64 = sqdmlal2 +generate int32x4_t:int16x8_t:int16x4_t:int32x4_t, int32x4_t:int16x8_t:int16x8_t:int32x4_t, int64x2_t: int32x4_t:int32x2_t:int64x2_t, int64x2_t:int32x4_t:int32x4_t:int64x2_t + +/// Signed saturating doubling multiply-subtract long +name = vqdmlsl +multi_fn = vqsub-out-noext, a, {vqdmull-self-noext, b, c} +a = 3, 7, 11, 15 +b = 1, 2, 3, 4 +c = 2, 2, 2, 2 +validate -1, -1, -1, -1 + +aarch64 = sqdmlsl +arm = vqdmlsl +generate int32x4_t:int16x4_t:int16x4_t:int32x4_t, int64x2_t:int32x2_t:int32x2_t:int64x2_t + +/// Vector widening saturating doubling multiply subtract with scalar +name = vqdmlsl +n-suffix +multi_fn = vqsub-out-noext, a, {vqdmull_n-self-noext, b, c} +a = 3, 7, 11, 15 +b = 1, 2, 3, 4 +c = 2 +validate -1, -1, -1, -1 + +aarch64 = sqdmlsl +arm = vqdmlsl +generate int32x4_t:int16x4_t:i16:int32x4_t, int64x2_t:int32x2_t:i32:int64x2_t + +/// Signed saturating doubling multiply-subtract long +name = vqdmlsl_high +no-q +multi_fn = vqsub-out-noext, a, {vqdmull_high-noqself-noext, b, c} +a = 39, 58, 81, 108 +b = 0, 1, 4, 5, 4, 5, 6, 7 +c = 1, 2, 5, 6, 5, 6, 7, 8 +validate -1, -2, -3, -4 + +aarch64 = sqdmlsl2 +generate int32x4_t:int16x8_t:int16x8_t:int32x4_t, int64x2_t:int32x4_t:int32x4_t:int64x2_t + +/// Signed saturating doubling multiply-subtract long +name = vqdmlsl_high_n +no-q +multi_fn = vqsub-out-noext, a, {vqdmull_high_n-noqself-noext, b, c} +a = 31, 38, 45, 52 +b = 0, 2, 8, 10, 8, 10, 12, 14 +c = 2 +validate -1, -2, -3, -4 + +aarch64 = sqdmlsl2 +generate int32x4_t:int16x8_t:i16:int32x4_t, int64x2_t:int32x4_t:i32:int64x2_t + +/// Vector widening saturating doubling multiply subtract with scalar +name = vqdmlsl_lane +in2-suffix +constn = N +multi_fn = static_assert_imm-in2_exp_len-N +multi_fn = vqsub-out-noext, a, {vqdmull_lane-in2-::, b, c} +a = 3, 6, 9, 12 +b = 1, 2, 3, 4 +c = 0, 2, 2, 0, 2, 0, 0, 0 +n = HFLEN +validate -1, -2, -3, -4 + +aarch64 = sqdmlsl +generate int32x4_t:int16x4_t:int16x8_t:int32x4_t, int64x2_t:int32x2_t:int32x4_t:int64x2_t + +arm = vqdmlsl +generate int32x4_t:int16x4_t:int16x4_t:int32x4_t, int64x2_t:int32x2_t:int32x2_t:int64x2_t + +/// Signed saturating doubling multiply-subtract long +name = vqdmlsl_high_lane +in2-suffix +constn = N +multi_fn = static_assert_imm-in2_exp_len-N +multi_fn = vqsub-out-noext, a, {vqdmull_high_lane-in2-::, b, c} +a = 15, 18, 21, 24 +b = 0, 1, 4, 5, 4, 5, 6, 7 +c = 0, 2, 0, 0, 0, 0, 0, 0 +n = 1 +validate -1, -2, -3, -4 + +aarch64 = sqdmlsl2 +generate int32x4_t:int16x8_t:int16x4_t:int32x4_t, int32x4_t:int16x8_t:int16x8_t:int32x4_t, int64x2_t: int32x4_t:int32x2_t:int64x2_t, int64x2_t:int32x4_t:int32x4_t:int64x2_t + +/// Signed saturating doubling multiply returning high half +name = vqdmulh +a = MAX, MAX, MAX, MAX, MAX, MAX, MAX, MAX +b = 2, 2, 2, 2, 2, 2, 2, 2 +validate 1, 1, 1, 1, 1, 1, 1, 1 + +aarch64 = sqdmulh +link-aarch64 = sqdmulh._EXT_ +arm = vqdmulh +link-arm = vqdmulh._EXT_ +generate int16x4_t, int16x8_t, int32x2_t, int32x4_t + +/// Signed saturating doubling multiply returning high half +name = vqdmulh +multi_fn = vdup_n-in_ntt-noext, a:in_ntt, a +multi_fn = vdup_n-in_ntt-noext, b:in_ntt, b +multi_fn = simd_extract, {vqdmulh-in_ntt-noext, a, b}, 0 +a = 1 +b = 2 +validate 0 + +aarch64 = sqdmulh +generate i16, i32 + +/// Vector saturating doubling multiply high with scalar +name = vqdmulh_n +out-suffix +multi_fn = vdup_n-in_ntt-noext, b:in_ntt, b +multi_fn = vqdmulh-out-noext, a, b +a = MAX, MAX, MAX, MAX +b = 2 +validate 1, 1, 1, 1 + +aarch64 = sqdmulh +arm = vqdmulh +generate int16x4_t:i16:int16x4_t, int32x2_t:i32:int32x2_t + +/// Vector saturating doubling multiply high with scalar +name = vqdmulhq_n +out-suffix +multi_fn = vdupq_n-in_ntt-noext, b:out_t, b +multi_fn = vqdmulh-out-noext, a, b +a = MAX, MAX, MAX, MAX, MAX, MAX, MAX, MAX +b = 2 +validate 1, 1, 1, 1, 1, 1, 1, 1 + +aarch64 = sqdmulh +arm = vqdmulh +generate int16x8_t:i16:int16x8_t, int32x4_t:i32:int32x4_t + +/// Signed saturating doubling multiply returning high half +name = vqdmulhh_lane +constn = N +multi_fn = static_assert_imm-in_exp_len-N +multi_fn = simd_extract, b:in_t0, b, N as u32 +multi_fn = vqdmulhh-out_ntt-noext, a, b +a = 2 +b = 0, 0, MAX, 0, 0, 0, 0, 0 +n = 2 +validate 1 + +aarch64 = sqdmulh +generate i16:int16x4_t:i16, i16:int16x8_t:i16 + +/// Signed saturating doubling multiply returning high half +name = vqdmulhs_lane +constn = N +multi_fn = static_assert_imm-in_exp_len-N +multi_fn = simd_extract, b:in_t0, b, N as u32 +multi_fn = vqdmulhs-out_ntt-noext, a, b +a = 2 +b = 0, MAX, 0, 0 +n = 1 +validate 1 + +aarch64 = sqdmulh +generate i32:int32x2_t:i32, i32:int32x4_t:i32 + +/// Signed saturating extract narrow +name = vqmovn +no-q +a = MAX, MAX, MAX, MAX, MAX, MAX, MAX, MAX +validate MAX, MAX, MAX, MAX, MAX, MAX, MAX, MAX + +aarch64 = sqxtn +link-aarch64 = sqxtn._EXT2_ +arm = vqmovn +link-arm = vqmovns._EXT2_ +generate int16x8_t:int8x8_t, int32x4_t:int16x4_t, int64x2_t:int32x2_t + +/// Unsigned saturating extract narrow +name = vqmovn +no-q +a = MAX, MAX, MAX, MAX, MAX, MAX, MAX, MAX +validate MAX, MAX, MAX, MAX, MAX, MAX, MAX, MAX + +aarch64 = uqxtn +link-aarch64 = uqxtn._EXT2_ +arm = vqmovn +link-arm = vqmovnu._EXT2_ +generate uint16x8_t:uint8x8_t, uint32x4_t:uint16x4_t, uint64x2_t:uint32x2_t + +/// Saturating extract narrow +name = vqmovn +multi_fn = simd_extract, {vqmovn-in_ntt-noext, {vdupq_n-in_ntt-noext, a}}, 0 +a = 1 +validate 1 + +aarch64 = sqxtn +generate i16:i8, i32:i16 +aarch64 = uqxtn +generate u16:u8, u32:u16 + +/// Saturating extract narrow +name = vqmovn +a = 1 +validate 1 + +aarch64 = sqxtn +link-aarch64 = scalar.sqxtn._EXT2_._EXT_ +generate i64:i32 + +aarch64 = uqxtn +link-aarch64 = scalar.uqxtn._EXT2_._EXT_ +generate u64:u32 + +/// Signed saturating extract narrow +name = vqmovn_high +no-q +multi_fn = simd_shuffle-out_len-!, a, {vqmovn-noqself-noext, b}, {asc-0-out_len} +a = MAX, MAX, MAX, MAX, MAX, MAX, MAX, MAX +b = MAX, MAX, MAX, MAX, MAX, MAX, MAX, MAX +validate MAX, MAX, MAX, MAX, MAX, MAX, MAX, MAX, MAX, MAX, MAX, MAX, MAX, MAX, MAX, MAX + +aarch64 = sqxtn2 +generate int8x8_t:int16x8_t:int8x16_t, int16x4_t:int32x4_t:int16x8_t, int32x2_t:int64x2_t:int32x4_t +aarch64 = uqxtn2 +generate uint8x8_t:uint16x8_t:uint8x16_t, uint16x4_t:uint32x4_t:uint16x8_t, uint32x2_t:uint64x2_t:uint32x4_t + +/// Signed saturating extract unsigned narrow +name = vqmovun +no-q +a = -1, -1, -1, -1, -1, -1, -1, -1 +validate 0, 0, 0, 0, 0, 0, 0, 0 + +aarch64 = sqxtun +link-aarch64 = sqxtun._EXT2_ +arm = vqmovun +link-arm = vqmovnsu._EXT2_ +generate int16x8_t:uint8x8_t, int32x4_t:uint16x4_t, int64x2_t:uint32x2_t + +/// Signed saturating extract unsigned narrow +name = vqmovun +multi_fn = simd_extract, {vqmovun-in_ntt-noext, {vdupq_n-in_ntt-noext, a}}, 0 +a = 1 +validate 1 + +aarch64 = sqxtun +generate i16:u8, i32:u16, i64:u32 + +/// Signed saturating extract unsigned narrow +name = vqmovun_high +no-q +multi_fn = simd_shuffle-out_len-!, a, {vqmovun-noqself-noext, b}, {asc-0-out_len} +a = 0, 0, 0, 0, 0, 0, 0, 0 +b = -1, -1, -1, -1, -1, -1, -1, -1 +validate 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 + +aarch64 = sqxtun2 +generate uint8x8_t:int16x8_t:uint8x16_t, uint16x4_t:int32x4_t:uint16x8_t, uint32x2_t:int64x2_t:uint32x4_t + +/// Signed saturating rounding doubling multiply returning high half +name = vqrdmulh +a = MAX, MAX, MAX, MAX, MAX, MAX, MAX, MAX +b = 2, 2, 2, 2, 2, 2, 2, 2 +validate 2, 2, 2, 2, 2, 2, 2, 2 + +aarch64 = sqrdmulh +link-aarch64 = sqrdmulh._EXT_ +arm = vqrdmulh +link-arm = vqrdmulh._EXT_ +generate int16x4_t, int16x8_t, int32x2_t, int32x4_t + +/// Signed saturating rounding doubling multiply returning high half +name = vqrdmulh +multi_fn = simd_extract, {vqrdmulh-in_ntt-noext, {vdup_n-in_ntt-noext, a}, {vdup_n-in_ntt-noext, b}}, 0 +a = 1 +b = 2 +validate 0 + +aarch64 = sqrdmulh +generate i16, i32 + +/// Vector saturating rounding doubling multiply high with scalar +name = vqrdmulh +out-n-suffix +multi_fn = vqrdmulh-out-noext, a, {vdup-nout-noext, b} +a = MAX, MAX, MAX, MAX, MAX, MAX, MAX, MAX +b = 2 +validate 2, 2, 2, 2, 2, 2, 2, 2 + +aarch64 = sqrdmulh +arm = vqrdmulh +generate int16x4_t:i16:int16x4_t, int16x8_t:i16:int16x8_t, int32x2_t:i32:int32x2_t, int32x4_t:i32:int32x4_t + +/// Vector rounding saturating doubling multiply high by scalar +name = vqrdmulh +lane-suffixes +constn = LANE +multi_fn = static_assert_imm-in_exp_len-LANE +multi_fn = simd_shuffle-out_len-!, b:out_t, b, b, {dup-out_len-LANE as u32} +multi_fn = vqrdmulh-out-noext, a, b +a = MAX, MAX, MAX, MAX, MAX, MAX, MAX, MAX +b = 0, 2, 0, 0, 0, 0, 0, 0, +n = 1 +validate 2, 2, 2, 2, 2, 2, 2, 2 + +aarch64 = sqrdmulh +arm = vqrdmulh +generate int16x4_t, int16x4_t:int16x8_t:int16x4_t, int16x8_t:int16x4_t:int16x8_t, int16x8_t +generate int32x2_t, int32x2_t:int32x4_t:int32x2_t, int32x4_t:int32x2_t:int32x4_t, int32x4_t + +/// Signed saturating rounding doubling multiply returning high half +name = vqrdmulh +lane-suffixes +constn = LANE +multi_fn = static_assert_imm-in_exp_len-LANE +multi_fn = vqrdmulh-out-noext, a, {simd_extract, b, LANE as u32} +a = 1 +b = 0, 2, 0, 0, 0, 0, 0, 0, +n = 1 +validate 0 + +aarch64 = sqrdmulh +generate i16:int16x4_t:i16, i16:int16x8_t:i16, i32:int32x2_t:i32, i32:int32x4_t:i32 + +/// Signed saturating rounding doubling multiply accumulate returning high half +name = vqrdmlah +multi_fn = vqadd-out-noext, a, {vqrdmulh-out-noext, b, c} +a = 1, 1, 1, 1, 1, 1, 1, 1 +b = MAX, MAX, MAX, MAX, MAX, MAX, MAX, MAX +c = 2, 2, 2, 2, 2, 2, 2, 2 +validate 3, 3, 3, 3, 3, 3, 3, 3 + +aarch64 = sqrdmulh +arm = vqrdmulh +generate int16x4_t, int16x8_t, int32x2_t, int32x4_t + +/// Signed saturating rounding doubling multiply accumulate returning high half +name = vqrdmlah +multi_fn = vqadd-self-noext, a, {vqrdmulh-self-noext, b, c} +a = 1 +b = 1 +c = 2 +validate 1 + +aarch64 = sqrdmulh +generate i16, i32 + +/// Signed saturating rounding doubling multiply accumulate returning high half +name = vqrdmlah +in2-lane-suffixes +constn = LANE +multi_fn = static_assert_imm-in2_exp_len-LANE +multi_fn = vqadd-self-noext, a, {vqrdmulh-in2lane-::, b, c} +a = 1, 1, 1, 1, 1, 1, 1, 1 +b = MAX, MAX, MAX, MAX, MAX, MAX, MAX, MAX +c = 0, 2, 0, 0, 0, 0, 0, 0 +n = 1 +validate 3, 3, 3, 3, 3, 3, 3, 3 + +aarch64 = sqrdmulh +arm = vqrdmulh +generate int16x4_t, int16x4_t:int16x4_t:int16x8_t:int16x4_t, int16x8_t:int16x8_t:int16x4_t:int16x8_t, int16x8_t +generate int32x2_t, int32x2_t:int32x2_t:int32x4_t:int32x2_t, int32x4_t:int32x4_t:int32x2_t:int32x4_t, int32x4_t + +/// Signed saturating rounding doubling multiply accumulate returning high half +name = vqrdmlah +in2-lane-suffixes +constn = LANE +multi_fn = static_assert_imm-in2_exp_len-LANE +multi_fn = vqadd-self-noext, a, {vqrdmulh-in2lane-::, b, c} +a = 1 +b = 1 +c = 0, 2, 0, 0, 0, 0, 0, 0 +n = 1 +validate 1 + +aarch64 = sqrdmulh +generate i16:i16:int16x4_t:i16, i16:i16:int16x8_t:i16, i32:i32:int32x2_t:i32, i32:i32:int32x4_t:i32 + +/// Signed saturating rounding doubling multiply subtract returning high half +name = vqrdmlsh +multi_fn = vqsub-out-noext, a, {vqrdmulh-out-noext, b, c} +a = 1, 1, 1, 1, 1, 1, 1, 1 +b = MAX, MAX, MAX, MAX, MAX, MAX, MAX, MAX +c = 2, 2, 2, 2, 2, 2, 2, 2 +validate -1, -1, -1, -1, -1, -1, -1, -1 + +aarch64 = sqrdmulh +arm = vqrdmulh +generate int16x4_t, int16x8_t, int32x2_t, int32x4_t + +/// Signed saturating rounding doubling multiply subtract returning high half +name = vqrdmlsh +multi_fn = vqsub-self-noext, a, {vqrdmulh-self-noext, b, c} +a = 1 +b = 1 +c = 2 +validate 1 + +aarch64 = sqrdmulh +generate i16, i32 + +/// Signed saturating rounding doubling multiply subtract returning high half +name = vqrdmlsh +in2-lane-suffixes +constn = LANE +multi_fn = static_assert_imm-in2_exp_len-LANE +multi_fn = vqsub-self-noext, a, {vqrdmulh-in2lane-::, b, c} +a = 1, 1, 1, 1, 1, 1, 1, 1 +b = MAX, MAX, MAX, MAX, MAX, MAX, MAX, MAX +c = 0, 2, 0, 0, 0, 0, 0, 0 +n = 1 +validate -1, -1, -1, -1, -1, -1, -1, -1 + +aarch64 = sqrdmulh +arm = vqrdmulh +generate int16x4_t, int16x4_t:int16x4_t:int16x8_t:int16x4_t, int16x8_t:int16x8_t:int16x4_t:int16x8_t, int16x8_t +generate int32x2_t, int32x2_t:int32x2_t:int32x4_t:int32x2_t, int32x4_t:int32x4_t:int32x2_t:int32x4_t, int32x4_t + +/// Signed saturating rounding doubling multiply subtract returning high half +name = vqrdmlsh +in2-lane-suffixes +constn = LANE +multi_fn = static_assert_imm-in2_exp_len-LANE +multi_fn = vqsub-self-noext, a, {vqrdmulh-in2lane-::, b, c} +a = 1 +b = 1 +c = 0, 2, 0, 0, 0, 0, 0, 0 +n = 1 +validate 1 + +aarch64 = sqrdmulh +generate i16:i16:int16x4_t:i16, i16:i16:int16x8_t:i16, i32:i32:int32x2_t:i32, i32:i32:int32x4_t:i32 + +/// Signed saturating rounding shift left +name = vqrshl +a = 2, MIN, MAX, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 +b = 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2 +validate 8, MIN, MAX, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60 + +aarch64 = sqrshl +link-aarch64 = sqrshl._EXT_ +generate i32, i64 + +arm = vqrshl +link-arm = vqrshifts._EXT_ +generate int*_t, int64x*_t + +/// Signed saturating rounding shift left +name = vqrshl +multi_fn = vdup_n-in_ntt-noext, a:in_ntt, a +multi_fn = vdup_n-in_ntt-noext, b:in_ntt, b +multi_fn = simd_extract, {vqrshl-in_ntt-noext, a, b}, 0 +a = 1 +b = 2 +validate 4 + +aarch64 = sqrshl +generate i8, i16 + +/// Unsigned signed saturating rounding shift left +name = vqrshl +out-suffix +a = 2, MIN, MAX, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 +b = 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2 +validate 8, 0, MAX, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60 + +aarch64 = uqrshl +link-aarch64 = uqrshl._EXT_ +generate u32:i32:u32, u64:i64:u64 + +arm = vqrshl +link-arm = vqrshiftu._EXT_ +generate uint8x8_t:int8x8_t:uint8x8_t, uint8x16_t:int8x16_t:uint8x16_t, uint16x4_t:int16x4_t:uint16x4_t, uint16x8_t:int16x8_t:uint16x8_t +generate uint32x2_t:int32x2_t:uint32x2_t, uint32x4_t:int32x4_t:uint32x4_t, uint64x1_t:int64x1_t:uint64x1_t, uint64x2_t:int64x2_t:uint64x2_t + +/// Unsigned signed saturating rounding shift left +name = vqrshl +out-suffix +multi_fn = vdup_n-out_ntt-noext, a:out_ntt, a +multi_fn = vdup_n-in_ntt-noext, b:in_ntt, b +multi_fn = simd_extract, {vqrshl-out_ntt-noext, a, b}, 0 +a = 1 +b = 2 +validate 4 + +aarch64 = uqrshl +generate u8:i8:u8, u16:i16:u16 + +/// Signed saturating rounded shift right narrow +name = vqrshrn +noq-n-suffix +constn = N +multi_fn = static_assert-N-1-halfbits +a = MIN, 4, 8, 12, 16, 20, 24, 28 +n = 2 +validate MIN, 1, 2, 3, 4, 5, 6, 7 + +aarch64 = sqrshrn +link-aarch64 = sqrshrn._EXT2_ +const-aarch64 = N + +arm = vqrshrn +link-arm = vqrshiftns._EXT2_ +const-arm = -N as ttn +generate int16x8_t:int8x8_t, int32x4_t:int16x4_t, int64x2_t:int32x2_t + +/// Signed saturating rounded shift right narrow +name = vqrshrn +noq-n-suffix +constn = N +multi_fn = static_assert-N-1-halfbits +multi_fn = vdupq_n-in_ntt-noext, a:in_long_ntt, a +multi_fn = simd_extract, {vqrshrn_n-in_ntt-::, a}, 0 +a = 4 +n = 2 +validate 1 + +aarch64 = sqrshrn +generate i16:i8, i32:i16, i64:i32 + +/// Signed saturating rounded shift right narrow +name = vqrshrn_high +noq-n-suffix +constn = N +multi_fn = static_assert-N-1-halfbits +multi_fn = simd_shuffle-out_len-!, a, {vqrshrn_n-noqself-::, b}, {asc-0-out_len} +a = 0, 1, 2, 3, 2, 3, 6, 7 +b = 8, 12, 24, 28, 48, 52, 56, 60 +n = 2 +validate 0, 1, 2, 3, 2, 3, 6, 7, 2, 3, 6, 7, 12, 13, 14, 15 + +aarch64 = sqrshrn2 +generate int8x8_t:int16x8_t:int8x16_t, int16x4_t:int32x4_t:int16x8_t, int32x2_t:int64x2_t:int32x4_t + +/// Unsigned signed saturating rounded shift right narrow +name = vqrshrn +noq-n-suffix +constn = N +multi_fn = static_assert-N-1-halfbits +a = MIN, 4, 8, 12, 16, 20, 24, 28 +n = 2 +validate 0, 1, 2, 3, 4, 5, 6, 7 + +aarch64 = uqrshrn +link-aarch64 = uqrshrn._EXT2_ +const-aarch64 = N + +arm = vqrshrn +link-arm = vqrshiftnu._EXT2_ +const-arm = -N as ttn +generate uint16x8_t:uint8x8_t, uint32x4_t:uint16x4_t, uint64x2_t:uint32x2_t + +/// Unsigned saturating rounded shift right narrow +name = vqrshrn +noq-n-suffix +constn = N +multi_fn = static_assert-N-1-halfbits +multi_fn = vdupq_n-in_ntt-noext, a:in_long_ntt, a +multi_fn = simd_extract, {vqrshrn_n-in_ntt-::, a}, 0 +a = 4 +n = 2 +validate 1 + +aarch64 = uqrshrn +generate u16:u8, u32:u16, u64:u32 + +/// Unsigned saturating rounded shift right narrow +name = vqrshrn_high +noq-n-suffix +constn = N +multi_fn = static_assert-N-1-halfbits +multi_fn = simd_shuffle-out_len-!, a, {vqrshrn_n-noqself-::, b}, {asc-0-out_len} +a = 0, 1, 2, 3, 2, 3, 6, 7 +b = 8, 12, 24, 28, 48, 52, 56, 60 +n = 2 +validate 0, 1, 2, 3, 2, 3, 6, 7, 2, 3, 6, 7, 12, 13, 14, 15 + +aarch64 = uqrshrn2 +generate uint8x8_t:uint16x8_t:uint8x16_t, uint16x4_t:uint32x4_t:uint16x8_t, uint32x2_t:uint64x2_t:uint32x4_t + +/// Signed saturating rounded shift right unsigned narrow +name = vqrshrun +noq-n-suffix +constn = N +multi_fn = static_assert-N-1-halfbits +a = 0, 4, 8, 12, 16, 20, 24, 28 +n = 2 +validate 0, 1, 2, 3, 4, 5, 6, 7 + +aarch64 = sqrshrun +link-aarch64 = sqrshrun._EXT2_ +const-aarch64 = N + +arm = vqrshrun +link-arm = vqrshiftnsu._EXT2_ +const-arm = -N as ttn +generate int16x8_t:uint8x8_t, int32x4_t:uint16x4_t, int64x2_t:uint32x2_t + +/// Signed saturating rounded shift right unsigned narrow +name = vqrshrun +noq-n-suffix +constn = N +multi_fn = static_assert-N-1-halfbits +multi_fn = vdupq_n-in_ntt-noext, a:in_long_ntt, a +multi_fn = simd_extract, {vqrshrun_n-in_ntt-::, a}, 0 +a = 4 +n = 2 +validate 1 + +aarch64 = sqrshrun +generate i16:u8, i32:u16, i64:u32 + +/// Signed saturating rounded shift right unsigned narrow +name = vqrshrun_high +noq-n-suffix +constn = N +multi_fn = static_assert-N-1-halfbits +multi_fn = simd_shuffle-out_len-!, a, {vqrshrun_n-noqself-::, b}, {asc-0-out_len} +a = 0, 1, 2, 3, 2, 3, 6, 7 +b = 8, 12, 24, 28, 48, 52, 56, 60 +n = 2 +validate 0, 1, 2, 3, 2, 3, 6, 7, 2, 3, 6, 7, 12, 13, 14, 15 + +aarch64 = sqrshrun2 +generate uint8x8_t:int16x8_t:uint8x16_t, uint16x4_t:int32x4_t:uint16x8_t, uint32x2_t:int64x2_t:uint32x4_t + +/// Signed saturating shift left +name = vqshl +a = 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 +b = 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2 +validate 0, 4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60 + +aarch64 = sqshl +link-aarch64 = sqshl._EXT_ +generate i64 + +arm = vqshl +link-arm = vqshifts._EXT_ +generate int*_t, int64x*_t + +/// Signed saturating shift left +name = vqshl +multi_fn = vqshl-in_ntt-noext, c:in_ntt, {vdup_n-in_ntt-noext, a}, {vdup_n-in_ntt-noext, b} +multi_fn = simd_extract, c, 0 +a = 1 +b = 2 +validate 4 + +aarch64 = sqshl +generate i8, i16, i32 + +/// Unsigned saturating shift left +name = vqshl +out-suffix +a = 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 +b = 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2 +validate 0, 4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60 + +aarch64 = uqshl +link-aarch64 = uqshl._EXT_ +generate u64:i64:u64 + +arm = vqshl +link-arm = vqshiftu._EXT_ +generate uint8x8_t:int8x8_t:uint8x8_t, uint8x16_t:int8x16_t:uint8x16_t, uint16x4_t:int16x4_t:uint16x4_t, uint16x8_t:int16x8_t:uint16x8_t +generate uint32x2_t:int32x2_t:uint32x2_t, uint32x4_t:int32x4_t:uint32x4_t, uint64x1_t:int64x1_t:uint64x1_t, uint64x2_t:int64x2_t:uint64x2_t + +/// Unsigned saturating shift left +name = vqshl +out-suffix +multi_fn = vqshl-out_ntt-noext, c:out_ntt, {vdup_n-out_ntt-noext, a}, {vdup_n-in_ntt-noext, b} +multi_fn = simd_extract, c, 0 +a = 1 +b = 2 +validate 4 + +aarch64 = uqshl +generate u8:i8:u8, u16:i16:u16, u32:i32:u32 + +/// Signed saturating shift left +name = vqshl +n-suffix +constn = N +multi_fn = static_assert_imm-out_bits_exp_len-N +multi_fn = vqshl-self-noext, a, {vdup-nself-noext, N.try_into().unwrap()} +a = 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 +n = 2 +validate 0, 4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60 + +aarch64 = sqshl +arm = vqshl +generate int*_t, int64x*_t + +/// Signed saturating shift left +name = vqshl +n-suffix +constn = N +multi_fn = static_assert_imm-out_bits_exp_len-N +multi_fn = simd_extract, {vqshl_n-in_ntt-::, {vdup_n-in_ntt-noext, a}}, 0 +a = 1 +n = 2 +validate 4 + +aarch64 = sqshl +generate i8, i16, i32, i64 + +/// Unsigned saturating shift left +name = vqshl +n-suffix +constn = N +multi_fn = static_assert_imm-out_bits_exp_len-N +multi_fn = vqshl-self-noext, a, {vdup-nsigned-noext, N.try_into().unwrap()} +a = 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 +n = 2 +validate 0, 4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60 + +aarch64 = uqshl +arm = vqshl +generate uint*_t, uint64x*_t + +/// Unsigned saturating shift left +name = vqshl +n-suffix +constn = N +multi_fn = static_assert_imm-out_bits_exp_len-N +multi_fn = simd_extract, {vqshl_n-in_ntt-::, {vdup_n-in_ntt-noext, a}}, 0 +a = 1 +n = 2 +validate 4 + +aarch64 = uqshl +generate u8, u16, u32, u64 + +/// Signed saturating shift right narrow +name = vqshrn +noq-n-suffix +constn = N +multi_fn = static_assert-N-1-halfbits +a = 0, 4, 8, 12, 16, 20, 24, 28 +n = 2 +validate 0, 1, 2, 3, 4, 5, 6, 7 + +aarch64 = sqshrn +link-aarch64 = sqshrn._EXT2_ +const-aarch64 = N +generate i64:i32 + +arm = vqshrn +link-arm = vqshiftns._EXT2_ +const-arm = -N as ttn +generate int16x8_t:int8x8_t, int32x4_t:int16x4_t, int64x2_t:int32x2_t + +/// Signed saturating shift right narrow +name = vqshrn +noq-n-suffix +constn = N +multi_fn = static_assert-N-1-halfbits +multi_fn = simd_extract, {vqshrn_n-in_ntt-::, {vdupq_n-in_ntt-noext, a}}, 0 +a = 4 +n = 2 +validate 1 + +aarch64 = sqshrn +generate i16:i8, i32:i16 + +/// Signed saturating shift right narrow +name = vqshrn_high +noq-n-suffix +constn = N +multi_fn = static_assert-N-1-halfbits +multi_fn = simd_shuffle-out_len-!, a, {vqshrn_n-noqself-::, b}, {asc-0-out_len} +a = 0, 1, 8, 9, 8, 9, 10, 11 +b = 32, 36, 40, 44, 48, 52, 56, 60 +n = 2 +validate 0, 1, 8, 9, 8, 9, 10, 11, 8, 9, 10, 11, 12, 13, 14, 15 + +aarch64 = sqshrn2 +generate int8x8_t:int16x8_t:int8x16_t, int16x4_t:int32x4_t:int16x8_t, int32x2_t:int64x2_t:int32x4_t + +/// Unsigned saturating shift right narrow +name = vqshrn +noq-n-suffix +constn = N +multi_fn = static_assert-N-1-halfbits +a = 0, 4, 8, 12, 16, 20, 24, 28 +n = 2 +validate 0, 1, 2, 3, 4, 5, 6, 7 + +aarch64 = uqshrn +link-aarch64 = uqshrn._EXT2_ +const-aarch64 = N +generate u64:u32 + +arm = vqshrn +link-arm = vqshiftnu._EXT2_ +const-arm = -N as ttn +generate uint16x8_t:uint8x8_t, uint32x4_t:uint16x4_t, uint64x2_t:uint32x2_t + +/// Unsigned saturating shift right narrow +name = vqshrn +noq-n-suffix +constn = N +multi_fn = static_assert-N-1-halfbits +multi_fn = simd_extract, {vqshrn_n-in_ntt-::, {vdupq_n-in_ntt-noext, a}}, 0 +a = 4 +n = 2 +validate 1 + +aarch64 = uqshrn +generate u16:u8, u32:u16 + +/// Unsigned saturating shift right narrow +name = vqshrn_high +noq-n-suffix +constn = N +multi_fn = static_assert-N-1-halfbits +multi_fn = simd_shuffle-out_len-!, a, {vqshrn_n-noqself-::, b}, {asc-0-out_len} +a = 0, 1, 8, 9, 8, 9, 10, 11 +b = 32, 36, 40, 44, 48, 52, 56, 60 +n = 2 +validate 0, 1, 8, 9, 8, 9, 10, 11, 8, 9, 10, 11, 12, 13, 14, 15 + +aarch64 = uqshrn2 +generate uint8x8_t:uint16x8_t:uint8x16_t, uint16x4_t:uint32x4_t:uint16x8_t, uint32x2_t:uint64x2_t:uint32x4_t + +/// Signed saturating shift right unsigned narrow +name = vqshrun +noq-n-suffix +constn = N +multi_fn = static_assert-N-1-halfbits +a = 0, 4, 8, 12, 16, 20, 24, 28 +n = 2 +validate 0, 1, 2, 3, 4, 5, 6, 7 + +aarch64 = sqshrun +link-aarch64 = sqshrun._EXT2_ +const-aarch64 = N + +arm = vqshrun +link-arm = vqshiftnsu._EXT2_ +const-arm = -N as ttn +generate int16x8_t:uint8x8_t, int32x4_t:uint16x4_t, int64x2_t:uint32x2_t + +/// Signed saturating shift right unsigned narrow +name = vqshrun +noq-n-suffix +constn = N +multi_fn = static_assert-N-1-halfbits +multi_fn = simd_extract, {vqshrun_n-in_ntt-::, {vdupq_n-in_ntt-noext, a}}, 0 +a = 4 +n = 2 +validate 1 + +aarch64 = sqshrun +generate i16:u8, i32:u16, i64:u32 + +/// Signed saturating shift right unsigned narrow +name = vqshrun_high +noq-n-suffix +constn = N +multi_fn = static_assert-N-1-halfbits +multi_fn = simd_shuffle-out_len-!, a, {vqshrun_n-noqself-::, b}, {asc-0-out_len} +a = 0, 1, 8, 9, 8, 9, 10, 11 +b = 32, 36, 40, 44, 48, 52, 56, 60 +n = 2 +validate 0, 1, 8, 9, 8, 9, 10, 11, 8, 9, 10, 11, 12, 13, 14, 15 + +aarch64 = sqshrun2 +generate uint8x8_t:int16x8_t:uint8x16_t, uint16x4_t:int32x4_t:uint16x8_t, uint32x2_t:int64x2_t:uint32x4_t + +/// Calculates the square root of each lane. +name = vsqrt +fn = simd_fsqrt +a = 4.0, 9.0, 16.0, 25.0 +validate 2.0, 3.0, 4.0, 5.0 + +aarch64 = fsqrt +generate float*_t, float64x*_t + +/// Reciprocal square-root estimate. +name = vrsqrte +a = 1.0, 2.0, 3.0, 4.0 +validate 0.998046875, 0.705078125, 0.576171875, 0.4990234375 + +aarch64 = frsqrte +link-aarch64 = frsqrte._EXT_ +generate float64x*_t + +arm = vrsqrte +link-arm = vrsqrte._EXT_ +generate float*_t + +/// Reciprocal estimate. +name = vrecpe +a = 4.0, 3.0, 2.0, 1.0 +validate 0.24951171875, 0.3330078125, 0.4990234375, 0.998046875 + +aarch64 = frecpe +link-aarch64 = frecpe._EXT_ +generate float64x*_t + +arm = vrecpe +link-arm = vrecpe._EXT_ +generate float*_t + +/// Vector reinterpret cast operation +name = vreinterpret +double-suffixes +fn = transmute +a = 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 +validate 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 + +aarch64 = str +generate poly64x1_t:int64x1_t, poly64x1_t:uint64x1_t, int64x1_t:poly64x1_t, uint64x1_t:poly64x1_t +generate poly64x2_t:int64x2_t, poly64x2_t:uint64x2_t, int64x2_t:poly64x2_t, uint64x2_t:poly64x2_t + +arm = str +generate uint8x8_t:int8x8_t, poly8x8_t:int8x8_t, poly16x4_t:int16x4_t, uint16x4_t:int16x4_t, uint32x2_t:int32x2_t, uint64x1_t:int64x1_t +generate uint8x16_t:int8x16_t, poly8x16_t:int8x16_t, poly16x8_t:int16x8_t, uint16x8_t:int16x8_t, uint32x4_t:int32x4_t, uint64x2_t:int64x2_t +generate poly8x8_t:uint8x8_t, int8x8_t:uint8x8_t, poly16x4_t:uint16x4_t, int16x4_t:uint16x4_t, int32x2_t:uint32x2_t, int64x1_t:uint64x1_t +generate poly8x16_t:uint8x16_t, int8x16_t:uint8x16_t, poly16x8_t:uint16x8_t, int16x8_t:uint16x8_t, int32x4_t:uint32x4_t, int64x2_t:uint64x2_t +generate int8x8_t:poly8x8_t, uint8x8_t:poly8x8_t, int16x4_t:poly16x4_t, uint16x4_t:poly16x4_t +generate int8x16_t:poly8x16_t, uint8x16_t:poly8x16_t, int16x8_t:poly16x8_t, uint16x8_t:poly16x8_t + +/// Vector reinterpret cast operation +name = vreinterpret +double-suffixes +fn = transmute +a = 0, 1, 2, 3, 4, 5, 6, 7 +validate 0, 0, 1, 0, 2, 0, 3, 0, 4, 0, 5, 0, 6, 0, 7, 0 + +aarch64 = str +generate poly64x1_t:int32x2_t, poly64x1_t:uint32x2_t +generate poly64x2_t:int32x4_t, poly64x2_t:uint32x4_t + +arm = str +generate int16x4_t:int8x8_t, uint16x4_t:int8x8_t, poly16x4_t:int8x8_t, int32x2_t:int16x4_t, uint32x2_t:int16x4_t, int64x1_t:int32x2_t, uint64x1_t:int32x2_t +generate int16x8_t:int8x16_t, uint16x8_t:int8x16_t, poly16x8_t:int8x16_t, int32x4_t:int16x8_t, uint32x4_t:int16x8_t, int64x2_t:int32x4_t, uint64x2_t:int32x4_t +generate poly16x4_t:uint8x8_t, int16x4_t:uint8x8_t, uint16x4_t:uint8x8_t, int32x2_t:uint16x4_t, uint32x2_t:uint16x4_t, int64x1_t:uint32x2_t, uint64x1_t:uint32x2_t +generate poly16x8_t:uint8x16_t, int16x8_t:uint8x16_t, uint16x8_t:uint8x16_t, int32x4_t:uint16x8_t, uint32x4_t:uint16x8_t, int64x2_t:uint32x4_t, uint64x2_t:uint32x4_t +generate poly16x4_t:poly8x8_t, int16x4_t:poly8x8_t, uint16x4_t:poly8x8_t, int32x2_t:poly16x4_t, uint32x2_t:poly16x4_t +generate poly16x8_t:poly8x16_t, int16x8_t:poly8x16_t, uint16x8_t:poly8x16_t, int32x4_t:poly16x8_t, uint32x4_t:poly16x8_t + +/// Vector reinterpret cast operation +name = vreinterpret +double-suffixes +fn = transmute +a = 0, 0, 1, 0, 2, 0, 3, 0, 4, 0, 5, 0, 6, 0, 7, 0 +validate 0, 1, 2, 3, 4, 5, 6, 7 + +aarch64 = str +generate int32x2_t:poly64x1_t, uint32x2_t:poly64x1_t +generate int32x4_t:poly64x2_t, uint32x4_t:poly64x2_t + +arm = str +generate poly8x8_t:int16x4_t, int8x8_t:int16x4_t, uint8x8_t:int16x4_t, poly16x4_t:int32x2_t, int16x4_t:int32x2_t, uint16x4_t:int32x2_t, int32x2_t:int64x1_t, uint32x2_t:int64x1_t +generate poly8x16_t:int16x8_t, int8x16_t:int16x8_t, uint8x16_t:int16x8_t, poly16x8_t:int32x4_t, int16x8_t:int32x4_t, uint16x8_t:int32x4_t, int32x4_t:int64x2_t, uint32x4_t:int64x2_t +generate poly8x8_t:uint16x4_t, int8x8_t:uint16x4_t, uint8x8_t:uint16x4_t, poly16x4_t:uint32x2_t, int16x4_t:uint32x2_t, uint16x4_t:uint32x2_t, int32x2_t:uint64x1_t, uint32x2_t:uint64x1_t +generate poly8x16_t:uint16x8_t, int8x16_t:uint16x8_t, uint8x16_t:uint16x8_t, poly16x8_t:uint32x4_t, int16x8_t:uint32x4_t, uint16x8_t:uint32x4_t, int32x4_t:uint64x2_t, uint32x4_t:uint64x2_t +generate poly8x8_t:poly16x4_t, int8x8_t:poly16x4_t, uint8x8_t:poly16x4_t +generate poly8x16_t:poly16x8_t, int8x16_t:poly16x8_t, uint8x16_t:poly16x8_t + +/// Vector reinterpret cast operation +name = vreinterpret +double-suffixes +fn = transmute +a = 0, 1, 2, 3 +validate 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 3, 0, 0, 0 + +aarch64 = str +generate poly64x1_t:int16x4_t, poly64x1_t:uint16x4_t, poly64x1_t:poly16x4_t +generate poly64x2_t:int16x8_t, poly64x2_t:uint16x8_t, poly64x2_t:poly16x8_t + +arm = str +generate int32x2_t:int8x8_t, uint32x2_t:int8x8_t, int64x1_t:int16x4_t, uint64x1_t:int16x4_t +generate int32x4_t:int8x16_t, uint32x4_t:int8x16_t, int64x2_t:int16x8_t, uint64x2_t:int16x8_t +generate int32x2_t:uint8x8_t, uint32x2_t:uint8x8_t, int64x1_t:uint16x4_t, uint64x1_t:uint16x4_t +generate int32x4_t:uint8x16_t, uint32x4_t:uint8x16_t, int64x2_t:uint16x8_t, uint64x2_t:uint16x8_t +generate int32x2_t:poly8x8_t, uint32x2_t:poly8x8_t, int64x1_t:poly16x4_t, uint64x1_t:poly16x4_t +generate int32x4_t:poly8x16_t, uint32x4_t:poly8x16_t, int64x2_t:poly16x8_t, uint64x2_t:poly16x8_t + +/// Vector reinterpret cast operation +name = vreinterpret +double-suffixes +fn = transmute +a = 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 3, 0, 0, 0 +validate 0, 1, 2, 3 + +aarch64 = str +generate poly16x4_t:poly64x1_t, int16x4_t:poly64x1_t, uint16x4_t:poly64x1_t +generate poly16x8_t:poly64x2_t, int16x8_t:poly64x2_t, uint16x8_t:poly64x2_t + +arm = str +generate poly8x8_t:int32x2_t, int8x8_t:int32x2_t, uint8x8_t:int32x2_t, poly16x4_t:int64x1_t, int16x4_t:int64x1_t, uint16x4_t:int64x1_t +generate poly8x16_t:int32x4_t, int8x16_t:int32x4_t, uint8x16_t:int32x4_t, poly16x8_t:int64x2_t, int16x8_t:int64x2_t, uint16x8_t:int64x2_t +generate poly8x8_t:uint32x2_t, int8x8_t:uint32x2_t, uint8x8_t:uint32x2_t, poly16x4_t:uint64x1_t, int16x4_t:uint64x1_t, uint16x4_t:uint64x1_t +generate poly8x16_t:uint32x4_t, int8x16_t:uint32x4_t, uint8x16_t:uint32x4_t, poly16x8_t:uint64x2_t, int16x8_t:uint64x2_t, uint16x8_t:uint64x2_t + +/// Vector reinterpret cast operation +name = vreinterpret +double-suffixes +fn = transmute +a = 0, 1 +validate 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0 + +aarch64 = str +generate poly64x1_t:int8x8_t, poly64x1_t:uint8x8_t, poly64x1_t:poly8x8_t +generate poly64x2_t:int8x16_t, poly64x2_t:uint8x16_t, poly64x2_t:poly8x16_t + +arm = str +generate int64x1_t:int8x8_t, uint64x1_t:int8x8_t, int64x1_t:uint8x8_t, uint64x1_t:uint8x8_t, int64x1_t:poly8x8_t, uint64x1_t:poly8x8_t +generate int64x2_t:int8x16_t, uint64x2_t:int8x16_t, int64x2_t:uint8x16_t, uint64x2_t:uint8x16_t, int64x2_t:poly8x16_t, uint64x2_t:poly8x16_t + +/// Vector reinterpret cast operation +name = vreinterpret +double-suffixes +fn = transmute +a = 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0 +validate 0, 1 + +aarch64 = str +generate poly8x8_t:poly64x1_t, int8x8_t:poly64x1_t, uint8x8_t:poly64x1_t +generate poly8x16_t:poly64x2_t, int8x16_t:poly64x2_t, uint8x16_t:poly64x2_t + +arm = str +generate poly8x8_t:int64x1_t, int8x8_t:int64x1_t, uint8x8_t:int64x1_t, poly8x8_t:uint64x1_t, int8x8_t:uint64x1_t, uint8x8_t:uint64x1_t +generate poly8x16_t:int64x2_t, int8x16_t:int64x2_t, uint8x16_t:int64x2_t, poly8x16_t:uint64x2_t, int8x16_t:uint64x2_t, uint8x16_t:uint64x2_t + +/// Vector reinterpret cast operation +name = vreinterpret +double-suffixes +fn = transmute +a = 0., 0., 0., 0., 0., 0., 0., 0. +validate 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 + +aarch64 = str +generate float64x1_t:int8x8_t, float64x1_t:int16x4_t, float64x1_t:int32x2_t, float64x1_t:int64x1_t +generate float64x2_t:int8x16_t, float64x2_t:int16x8_t, float64x2_t:int32x4_t, float64x2_t:int64x2_t +generate float64x1_t:uint8x8_t, float64x1_t:uint16x4_t, float64x1_t:uint32x2_t, float64x1_t:uint64x1_t +generate float64x2_t:uint8x16_t, float64x2_t:uint16x8_t, float64x2_t:uint32x4_t, float64x2_t:uint64x2_t +generate float64x1_t:poly8x8_t, float64x1_t:poly16x4_t, float32x2_t:poly64x1_t, float64x1_t:poly64x1_t +generate float64x2_t:poly8x16_t, float64x2_t:poly16x8_t, float32x4_t:poly64x2_t, float64x2_t:poly64x2_t + +arm = str +generate float32x2_t:int8x8_t, float32x2_t:int16x4_t, float32x2_t:int32x2_t, float32x2_t:int64x1_t +generate float32x4_t:int8x16_t, float32x4_t:int16x8_t, float32x4_t:int32x4_t, float32x4_t:int64x2_t +generate float32x2_t:uint8x8_t, float32x2_t:uint16x4_t, float32x2_t:uint32x2_t, float32x2_t:uint64x1_t +generate float32x4_t:uint8x16_t, float32x4_t:uint16x8_t, float32x4_t:uint32x4_t, float32x4_t:uint64x2_t +generate float32x2_t:poly8x8_t, float32x2_t:poly16x4_t +generate float32x4_t:poly8x16_t, float32x4_t:poly16x8_t + +/// Vector reinterpret cast operation +name = vreinterpret +double-suffixes +fn = transmute +a = 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 +validate 0., 0., 0., 0., 0., 0., 0., 0. + +aarch64 = str +generate int8x8_t:float64x1_t, int16x4_t:float64x1_t, int32x2_t:float64x1_t, int64x1_t:float64x1_t +generate int8x16_t:float64x2_t, int16x8_t:float64x2_t, int32x4_t:float64x2_t, int64x2_t:float64x2_t +generate poly8x8_t:float64x1_t, uint16x4_t:float64x1_t, uint32x2_t:float64x1_t, uint64x1_t:float64x1_t +generate poly8x16_t:float64x2_t, uint16x8_t:float64x2_t, uint32x4_t:float64x2_t, uint64x2_t:float64x2_t +generate uint8x8_t:float64x1_t, poly16x4_t:float64x1_t, poly64x1_t:float64x1_t, poly64x1_t:float32x2_t +generate uint8x16_t:float64x2_t, poly16x8_t:float64x2_t, poly64x2_t:float64x2_t, poly64x2_t:float32x4_t + +arm = str +generate int8x8_t:float32x2_t, int16x4_t:float32x2_t, int32x2_t:float32x2_t, int64x1_t:float32x2_t +generate int8x16_t:float32x4_t, int16x8_t:float32x4_t, int32x4_t:float32x4_t, int64x2_t:float32x4_t +generate uint8x8_t:float32x2_t, uint16x4_t:float32x2_t, uint32x2_t:float32x2_t, uint64x1_t:float32x2_t +generate uint8x16_t:float32x4_t, uint16x8_t:float32x4_t, uint32x4_t:float32x4_t, uint64x2_t:float32x4_t +generate poly8x8_t:float32x2_t, poly16x4_t:float32x2_t +generate poly8x16_t:float32x4_t, poly16x8_t:float32x4_t + +/// Vector reinterpret cast operation +name = vreinterpret +double-suffixes +fn = transmute +a = 0., 0., 0., 0., 0., 0., 0., 0. +validate 0., 0., 0., 0., 0., 0., 0., 0. + +aarch64 = str +generate float32x2_t:float64x1_t, float64x1_t:float32x2_t +generate float32x4_t:float64x2_t, float64x2_t:float32x4_t + +/// Signed rounding shift left +name = vrshl +a = 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 +b = 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2 +validate 4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60, 64 + +aarch64 = srshl +link-aarch64 = srshl._EXT_ +generate i64 + +arm = vrshl +link-arm = vrshifts._EXT_ +generate int*_t, int64x*_t + +/// Unsigned rounding shift left +name = vrshl +out-suffix +a = 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 +b = 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2 +validate 4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60, 64 + +aarch64 = urshl +link-aarch64 = urshl._EXT_ +generate u64:i64:u64 + +arm = vrshl +link-arm = vrshiftu._EXT_ +generate uint8x8_t:int8x8_t:uint8x8_t, uint8x16_t:int8x16_t:uint8x16_t, uint16x4_t:int16x4_t:uint16x4_t, uint16x8_t:int16x8_t:uint16x8_t +generate uint32x2_t:int32x2_t:uint32x2_t, uint32x4_t:int32x4_t:uint32x4_t, uint64x1_t:int64x1_t:uint64x1_t, uint64x2_t:int64x2_t:uint64x2_t + +/// Signed rounding shift right +name = vrshr +n-suffix +constn = N +multi_fn = static_assert-N-1-bits +multi_fn = vrshl-self-noext, a, {vdup-nself-noext, (-N).try_into().unwrap()} +a = 4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60, 64 +n = 2 +validate 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 + +aarch64 = srshr +arm = vrshr +generate int*_t, int64x*_t + +/// Signed rounding shift right +name = vrshr +n-suffix +constn = N +multi_fn = static_assert-N-1-bits +multi_fn = vrshl-self-noext, a, -N as i64 +a = 4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60, 64 +n = 2 +validate 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 + +aarch64 = srshr +generate i64 + +/// Unsigned rounding shift right +name = vrshr +n-suffix +constn = N +multi_fn = static_assert-N-1-bits +multi_fn = vrshl-self-noext, a, {vdup-nsigned-noext, (-N).try_into().unwrap()} +a = 4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60, 64 +n = 2 +validate 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 + +aarch64 = urshr +arm = vrshr +generate uint*_t, uint64x*_t + +/// Unsigned rounding shift right +name = vrshr +n-suffix +constn = N +multi_fn = static_assert-N-1-bits +multi_fn = vrshl-self-noext, a, -N as i64 +a = 4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60, 64 +n = 2 +validate 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 + +aarch64 = urshr +generate u64 + +/// Rounding shift right narrow +name = vrshrn +noq-n-suffix +constn = N +multi_fn = static_assert-N-1-halfbits +a = 4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60, 64 +n = 2 +validate 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 + +aarch64 = rshrn +link-aarch64 = rshrn._EXT2_ +const-aarch64 = N + +arm = vrshrn +link-arm = vrshiftn._EXT2_ +const-arm = -N as ttn +generate int16x8_t:int8x8_t, int32x4_t:int16x4_t, int64x2_t:int32x2_t + +/// Rounding shift right narrow +name = vrshrn +noq-n-suffix +constn = N +multi_fn = static_assert-N-1-halfbits +multi_fn = transmute, {vrshrn_n-noqsigned-::, transmute(a)} +a = 4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60, 64 +n = 2 +validate 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 + +aarch64 = rshrn +arm = vrshrn +generate uint16x8_t:uint8x8_t, uint32x4_t:uint16x4_t, uint64x2_t:uint32x2_t + +/// Rounding shift right narrow +name = vrshrn_high +noq-n-suffix +constn = N +multi_fn = static_assert-N-1-halfbits +multi_fn = simd_shuffle-out_len-!, a, {vrshrn_n-noqself-::, b}, {asc-0-out_len} +a = 0, 1, 8, 9, 8, 9, 10, 11 +b = 32, 36, 40, 44, 48, 52, 56, 60 +n = 2 +validate 0, 1, 8, 9, 8, 9, 10, 11, 8, 9, 10, 11, 12, 13, 14, 15 + +aarch64 = rshrn2 +generate int8x8_t:int16x8_t:int8x16_t, int16x4_t:int32x4_t:int16x8_t, int32x2_t:int64x2_t:int32x4_t +generate uint8x8_t:uint16x8_t:uint8x16_t, uint16x4_t:uint32x4_t:uint16x8_t, uint32x2_t:uint64x2_t:uint32x4_t + +/// Signed rounding shift right and accumulate +name = vrsra +n-suffix +constn = N +multi_fn = static_assert-N-1-bits +multi_fn = simd_add, a, {vrshr-nself-::, b} +a = 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 +b = 4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60, 64 +n = 2 +validate 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17 + +aarch64 = srsra +arm = vrsra +generate int*_t, int64x*_t + +/// Unsigned rounding shift right and accumulate +name = vrsra +n-suffix +constn = N +multi_fn = static_assert-N-1-bits +multi_fn = simd_add, a, {vrshr-nself-::, b} +a = 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 +b = 4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60, 64 +n = 2 +validate 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17 + +aarch64 = ursra +arm = vrsra +generate uint*_t, uint64x*_t + +/// Signed rounding shift right and accumulate. +name = vrsra +n-suffix +constn = N +multi_fn = static_assert-N-1-bits +multi_fn = vrshr-nself-::, b:in_t, b +multi_fn = a + b +a = 1 +b = 4 +n = 2 +validate 2 + +aarch64 = srsra +generate i64 + +/// Ungisned rounding shift right and accumulate. +name = vrsra +n-suffix +constn = N +multi_fn = static_assert-N-1-bits +multi_fn = vrshr-nself-::, b:in_t, b +multi_fn = a + b +a = 1 +b = 4 +n = 2 +validate 2 + +aarch64 = ursra +generate u64 + +/// Insert vector element from another vector element +name = vset_lane +constn = LANE +multi_fn = static_assert_imm-in_exp_len-LANE +multi_fn = simd_insert, b, LANE as u32, a +a = 1 +b = 0, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 +n = 0 +validate 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 + +aarch64 = nop +arm = nop +generate i8:int8x8_t:int8x8_t, i16:int16x4_t:int16x4_t +generate i32:int32x2_t:int32x2_t, i64:int64x1_t:int64x1_t +generate u8:uint8x8_t:uint8x8_t, u16:uint16x4_t:uint16x4_t +generate u32:uint32x2_t:uint32x2_t, u64:uint64x1_t:uint64x1_t +generate p8:poly8x8_t:poly8x8_t, p16:poly16x4_t:poly16x4_t + +target = crypto +generate p64:poly64x1_t:poly64x1_t + +/// Insert vector element from another vector element +name = vsetq_lane +no-q +constn = LANE +multi_fn = static_assert_imm-in_exp_len-LANE +multi_fn = simd_insert, b, LANE as u32, a +a = 1 +b = 0, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 +n = 0 +validate 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 + +aarch64 = nop +arm = nop +generate i8:int8x16_t:int8x16_t, i16:int16x8_t:int16x8_t +generate i32:int32x4_t:int32x4_t, i64:int64x2_t:int64x2_t +generate u8:uint8x16_t:uint8x16_t, u16:uint16x8_t:uint16x8_t +generate u32:uint32x4_t:uint32x4_t, u64:uint64x2_t:uint64x2_t +generate p8:poly8x16_t:poly8x16_t, p16:poly16x8_t:poly16x8_t + +target = crypto +generate p64:poly64x2_t:poly64x2_t + +/// Insert vector element from another vector element +name = vset_lane +constn = LANE +multi_fn = static_assert_imm-in_exp_len-LANE +multi_fn = simd_insert, b, LANE as u32, a +a = 1. +b = 0., 2., 3., 4. +n = 0 +validate 1., 2., 3., 4. + +aarch64 = nop +generate f64:float64x1_t:float64x1_t + +arm = nop +generate f32:float32x2_t:float32x2_t + +/// Insert vector element from another vector element +name = vsetq_lane +no-q +constn = LANE +multi_fn = static_assert_imm-in_exp_len-LANE +multi_fn = simd_insert, b, LANE as u32, a +a = 1. +b = 0., 2., 3., 4. +n = 0 +validate 1., 2., 3., 4. + +aarch64 = nop +generate f64:float64x2_t:float64x2_t + +arm = nop +generate f32:float32x4_t:float32x4_t + +/// Signed Shift left +name = vshl +a = 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 +b = 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2 +validate 4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60, 64 + +aarch64 = sshl +link-aarch64 = sshl._EXT_ +arm = vshl +link-arm = vshifts._EXT_ +generate int*_t, int64x*_t + +/// Signed Shift left +name = vshl +multi_fn = transmute, {vshl-in_ntt-noext, transmute(a), transmute(b)} +a = 1 +b = 2 +validate 4 + +aarch64 = sshl +generate i64 + +/// Unsigned Shift left +name = vshl +out-suffix +a = 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 +b = 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2 +validate 4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60, 64 + +aarch64 = ushl +link-aarch64 = ushl._EXT_ +arm = vshl +link-arm = vshiftu._EXT_ +generate uint8x8_t:int8x8_t:uint8x8_t, uint8x16_t:int8x16_t:uint8x16_t, uint16x4_t:int16x4_t:uint16x4_t, uint16x8_t:int16x8_t:uint16x8_t +generate uint32x2_t:int32x2_t:uint32x2_t, uint32x4_t:int32x4_t:uint32x4_t, uint64x1_t:int64x1_t:uint64x1_t, uint64x2_t:int64x2_t:uint64x2_t + +/// Unsigned Shift left +out-suffix +name = vshl +multi_fn = transmute, {vshl-out_ntt-noext, transmute(a), transmute(b)} +a = 1 +b = 2 +validate 4 + +aarch64 = ushl +generate u64:i64:u64 + +/// Shift left +name = vshl +n-suffix +constn = N +multi_fn = static_assert_imm-out_bits_exp_len-N +multi_fn = simd_shl, a, {vdup-nself-noext, N.try_into().unwrap()} +a = 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 +n = 2 +validate 4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60, 64 + +arm = vshl +aarch64 = shl +generate int*_t, uint*_t, int64x*_t, uint64x*_t + +/// Signed shift left long +name = vshll +n-suffix +constn = N +multi_fn = static_assert-N-0-bits +multi_fn = simd_shl, {simd_cast, a}, {vdup-nout-noext, N.try_into().unwrap()} +a = 1, 2, 3, 4, 5, 6, 7, 8 +n = 2 +validate 4, 8, 12, 16, 20, 24, 28, 32 + +arm = vshll.s +aarch64 = sshll +generate int8x8_t:int16x8_t, int16x4_t:int32x4_t, int32x2_t:int64x2_t +aarch64 = ushll +generate uint8x8_t:uint16x8_t, uint16x4_t:uint32x4_t, uint32x2_t:uint64x2_t + +/// Signed shift left long +name = vshll_high_n +no-q +constn = N +multi_fn = static_assert-N-0-bits +multi_fn = simd_shuffle-out_len-!, b:half, a, a, {asc-halflen-halflen} +multi_fn = vshll_n-noqself-::, b +a = 0, 0, 1, 2, 1, 2, 3, 4, 1, 2, 3, 4, 5, 6, 7, 8 +n = 2 +validate 4, 8, 12, 16, 20, 24, 28, 32 + +aarch64 = sshll2 +generate int8x16_t:int16x8_t, int16x8_t:int32x4_t, int32x4_t:int64x2_t +aarch64 = ushll2 +generate uint8x16_t:uint16x8_t, uint16x8_t:uint32x4_t, uint32x4_t:uint64x2_t + +/// Shift right +name = vshr +n-suffix +constn = N +multi_fn = static_assert-N-1-bits +multi_fn = simd_shr, a, {vdup-nself-noext, N.try_into().unwrap()} +a = 4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60, 64 +n = 2 +validate 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 + +arm = vshr.s +aarch64 = sshr +generate int*_t, int64x*_t +aarch64 = ushr +generate uint*_t, uint64x*_t + +/// Shift right narrow +name = vshrn_n +no-q +constn = N +multi_fn = static_assert-N-1-halfbits +multi_fn = simd_cast, {simd_shr, a, {vdup-nself-noext, N.try_into().unwrap()}} +a = 4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60, 64 +n = 2 +validate 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 + +arm = vshrn. +aarch64 = shrn +generate int16x8_t:int8x8_t, int32x4_t:int16x4_t, int64x2_t:int32x2_t +generate uint16x8_t:uint8x8_t, uint32x4_t:uint16x4_t, uint64x2_t:uint32x2_t + +/// Shift right narrow +name = vshrn_high_n +no-q +constn = N +multi_fn = static_assert-N-1-halfbits +multi_fn = simd_shuffle-out_len-!, a, {vshrn_n-noqself-::, b}, {asc-0-out_len} +a = 1, 2, 5, 6, 5, 6, 7, 8 +b = 20, 24, 28, 32, 52, 56, 60, 64 +n = 2 +validate 1, 2, 5, 6, 5, 6, 7, 8, 5, 6, 7, 8, 13, 14, 15, 16 + +aarch64 = shrn2 +generate int8x8_t:int16x8_t:int8x16_t, int16x4_t:int32x4_t:int16x8_t, int32x2_t:int64x2_t:int32x4_t +generate uint8x8_t:uint16x8_t:uint8x16_t, uint16x4_t:uint32x4_t:uint16x8_t, uint32x2_t:uint64x2_t:uint32x4_t + +/// Signed shift right and accumulate +name = vsra +n-suffix +constn = N +multi_fn = static_assert-N-1-bits +multi_fn = simd_add, a, {vshr-nself-::, b} +a = 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 +b = 4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60, 64 +n = 2 +validate 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17 + +aarch64 = ssra +arm = vsra +generate int*_t, int64x*_t + +/// Unsigned shift right and accumulate +name = vsra +n-suffix +constn = N +multi_fn = static_assert-N-1-bits +multi_fn = simd_add, a, {vshr-nself-::, b} +a = 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 +b = 4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60, 64 +n = 2 +validate 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17 + +aarch64 = usra +arm = vsra +generate uint*_t, uint64x*_t + +/// Transpose vectors +name = vtrn1 +multi_fn = simd_shuffle-in_len-!, a, b, {transpose-1-in_len} +a = 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30 +b = 1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31 +validate 0, 1, 4, 5, 8, 9, 12, 13, 16, 17, 20, 21, 24, 25, 28, 29 + +aarch64 = trn1 +generate int8x8_t, int8x16_t, int16x4_t, int16x8_t, int32x4_t, uint8x8_t, uint8x16_t, uint16x4_t, uint16x8_t, uint32x4_t, poly8x8_t, poly8x16_t, poly16x4_t, poly16x8_t + +aarch64 = zip1 +generate int32x2_t, int64x2_t, uint32x2_t, uint64x2_t, poly64x2_t + +/// Transpose vectors +name = vtrn1 +multi_fn = simd_shuffle-in_len-!, a, b, {transpose-1-in_len} +a = 0., 2., 4., 6., 8., 10., 12., 14. +b = 1., 3., 5., 7., 9., 11., 13., 15. +validate 0., 1., 4., 5., 8., 9., 12., 13. + +aarch64 = trn1 +generate float32x4_t + +aarch64 = zip1 +generate float32x2_t, float64x2_t + +/// Transpose vectors +name = vtrn2 +multi_fn = simd_shuffle-in_len-!, a, b, {transpose-2-in_len} +a = 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30 +b = 1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31 +validate 2, 3, 6, 7, 10, 11, 14, 15, 18, 19, 22, 23, 26, 27, 30, 31 + +aarch64 = trn2 +generate int8x8_t, int8x16_t, int16x4_t, int16x8_t, int32x4_t, uint8x8_t, uint8x16_t, uint16x4_t, uint16x8_t, uint32x4_t, poly8x8_t, poly8x16_t, poly16x4_t, poly16x8_t + +aarch64 = zip2 +generate int32x2_t, int64x2_t, uint32x2_t, uint64x2_t, poly64x2_t + +/// Transpose vectors +name = vtrn2 +multi_fn = simd_shuffle-in_len-!, a, b, {transpose-2-in_len} +a = 0., 2., 4., 6., 8., 10., 12., 14. +b = 1., 3., 5., 7., 9., 11., 13., 15. +validate 2., 3., 6., 7., 10., 11., 14., 15. + +aarch64 = trn2 +generate float32x4_t + +aarch64 = zip2 +generate float32x2_t, float64x2_t + +/// Zip vectors +name = vzip1 +multi_fn = simd_shuffle-in_len-!, a, b, {zip-1-in_len} +a = 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30 +b = 1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31 +validate 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 + +aarch64 = zip1 +generate int*_t, int64x2_t, uint*_t, uint64x2_t, poly8x8_t, poly8x16_t, poly16x4_t, poly16x8_t, poly64x2_t + +/// Zip vectors +name = vzip1 +multi_fn = simd_shuffle-in_len-!, a, b, {zip-1-in_len} +a = 0., 2., 4., 6., 8., 10., 12., 14. +b = 1., 3., 5., 7., 9., 11., 13., 15. +validate 0., 1., 2., 3., 4., 5., 6., 7. + +aarch64 = zip1 +generate float32x2_t, float32x4_t, float64x2_t + +/// Zip vectors +name = vzip2 +multi_fn = simd_shuffle-in_len-!, a, b, {zip-2-in_len} +a = 0, 16, 16, 18, 16, 18, 20, 22, 16, 18, 20, 22, 24, 26, 28, 30 +b = 1, 17, 17, 19, 17, 19, 21, 23, 17, 19, 21, 23, 25, 27, 29, 31 +validate 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31 + +aarch64 = zip2 +generate int*_t, int64x2_t, uint*_t, uint64x2_t, poly8x8_t, poly8x16_t, poly16x4_t, poly16x8_t, poly64x2_t + +/// Zip vectors +name = vzip2 +multi_fn = simd_shuffle-in_len-!, a, b, {zip-2-in_len} +a = 0., 8., 8., 10., 8., 10., 12., 14. +b = 1., 9., 9., 11., 9., 11., 13., 15. +validate 8., 9., 10., 11., 12., 13., 14., 15. + +aarch64 = zip2 +generate float32x2_t, float32x4_t, float64x2_t + +/// Unzip vectors +name = vuzp1 +multi_fn = simd_shuffle-in_len-!, a, b, {unzip-1-in_len} +a = 1, 0, 2, 0, 2, 0, 3, 0, 2, 0, 3, 0, 7, 0, 8, 0 +b = 2, 0, 3, 0, 7, 0, 8, 0, 13, 0, 14, 0, 15, 0, 16, 0 +validate 1, 2, 2, 3, 2, 3, 7, 8, 2, 3, 7, 8, 13, 14, 15, 16 + +aarch64 = uzp1 +generate int8x8_t, int8x16_t, int16x4_t, int16x8_t, int32x4_t, uint8x8_t, uint8x16_t, uint16x4_t, uint16x8_t, uint32x4_t, poly8x8_t, poly8x16_t, poly16x4_t, poly16x8_t + +aarch64 = zip1 +generate int32x2_t, int64x2_t, uint32x2_t, uint64x2_t, poly64x2_t + +/// Unzip vectors +name = vuzp1 +multi_fn = simd_shuffle-in_len-!, a, b, {unzip-1-in_len} +a = 0., 8., 1., 9., 4., 12., 5., 13. +b = 1., 10., 3., 11., 6., 14., 7., 15. +validate 0., 1., 1., 3., 4., 5., 6., 7. + +aarch64 = uzp1 +generate float32x4_t + +aarch64 = zip1 +generate float32x2_t, float64x2_t + +/// Unzip vectors +name = vuzp2 +multi_fn = simd_shuffle-in_len-!, a, b, {unzip-2-in_len} +a = 0, 17, 0, 18, 0, 18, 0, 19, 0, 18, 0, 19, 0, 23, 0, 24 +b = 0, 18, 0, 19, 0, 23, 0, 24, 0, 29, 0, 30, 0, 31, 0, 32 +validate 17, 18, 18, 19, 18, 19, 23, 24, 18, 19, 23, 24, 29, 30, 31, 32 + +aarch64 = uzp2 +generate int8x8_t, int8x16_t, int16x4_t, int16x8_t, int32x4_t, uint8x8_t, uint8x16_t, uint16x4_t, uint16x8_t, uint32x4_t, poly8x8_t, poly8x16_t, poly16x4_t, poly16x8_t + +aarch64 = zip2 +generate int32x2_t, int64x2_t, uint32x2_t, uint64x2_t, poly64x2_t + +/// Unzip vectors +name = vuzp2 +multi_fn = simd_shuffle-in_len-!, a, b, {unzip-2-in_len} +a = 0., 8., 1., 9., 4., 12., 5., 13. +b = 2., 9., 3., 11., 6., 14., 7., 15. +validate 8., 9., 9., 11., 12., 13., 14., 15. + +aarch64 = uzp2 +generate float32x4_t + +aarch64 = zip2 +generate float32x2_t, float64x2_t + +//////////////////// +// Unsigned Absolute difference and Accumulate Long +//////////////////// + +/// Unsigned Absolute difference and Accumulate Long +name = vabal +multi_fn = vabd-unsigned-noext, b, c, d:in_t +multi_fn = simd_add, a, {simd_cast, d} +a = 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 +b = 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 +c = 10, 10, 10, 10, 10, 10, 10, 10, 20, 0, 2, 4, 6, 8, 10, 12 +validate 10, 10, 10, 10, 10, 10, 10, 10, 20, 20, 20, 20, 20, 20, 20, 20 + +arm = vabal.s +aarch64 = uabal +generate uint16x8_t:uint8x8_t:uint8x8_t:uint16x8_t, uint32x4_t:uint16x4_t:uint16x4_t:uint32x4_t, uint64x2_t:uint32x2_t:uint32x2_t:uint64x2_t + +/// Unsigned Absolute difference and Accumulate Long +name = vabal_high +no-q +multi_fn = simd_shuffle8!, d:uint8x8_t, b, b, [8, 9, 10, 11, 12, 13, 14, 15] +multi_fn = simd_shuffle8!, e:uint8x8_t, c, c, [8, 9, 10, 11, 12, 13, 14, 15] +multi_fn = vabd_u8, d, e, f:uint8x8_t +multi_fn = simd_add, a, {simd_cast, f} +a = 9, 10, 11, 12, 13, 14, 15, 16 +b = 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 +c = 10, 10, 10, 10, 10, 10, 10, 10, 20, 0, 2, 4, 6, 8, 10, 12 +validate 20, 20, 20, 20, 20, 20, 20, 20 + +aarch64 = uabal +generate uint16x8_t:uint8x16_t:uint8x16_t:uint16x8_t + +/// Unsigned Absolute difference and Accumulate Long +name = vabal_high +no-q +multi_fn = simd_shuffle4!, d:uint16x4_t, b, b, [4, 5, 6, 7] +multi_fn = simd_shuffle4!, e:uint16x4_t, c, c, [4, 5, 6, 7] +multi_fn = vabd_u16, d, e, f:uint16x4_t +multi_fn = simd_add, a, {simd_cast, f} +a = 9, 10, 11, 12 +b = 1, 2, 3, 4, 9, 10, 11, 12 +c = 10, 10, 10, 10, 20, 0, 2, 4 +validate 20, 20, 20, 20 + +aarch64 = uabal +generate uint32x4_t:uint16x8_t:uint16x8_t:uint32x4_t + +/// Unsigned Absolute difference and Accumulate Long +name = vabal_high +no-q +multi_fn = simd_shuffle2!, d:uint32x2_t, b, b, [2, 3] +multi_fn = simd_shuffle2!, e:uint32x2_t, c, c, [2, 3] +multi_fn = vabd_u32, d, e, f:uint32x2_t +multi_fn = simd_add, a, {simd_cast, f} +a = 15, 16 +b = 1, 2, 15, 16 +c = 10, 10, 10, 12 +validate 20, 20 + +aarch64 = uabal +generate uint64x2_t:uint32x4_t:uint32x4_t:uint64x2_t + +//////////////////// +// Signed Absolute difference and Accumulate Long +//////////////////// + +/// Signed Absolute difference and Accumulate Long +name = vabal +multi_fn = vabd-signed-noext, b, c, d:int8x8_t +multi_fn = simd_cast, e:uint8x8_t, d +multi_fn = simd_add, a, {simd_cast, e} +a = 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 +b = 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 +c = 10, 10, 10, 10, 10, 10, 10, 10, 20, 0, 2, 4, 6, 8, 10, 12 +validate 10, 10, 10, 10, 10, 10, 10, 10, 20, 20, 20, 20, 20, 20, 20, 20 + +arm = vabal.s +aarch64 = sabal +generate int16x8_t:int8x8_t:int8x8_t:int16x8_t + +/// Signed Absolute difference and Accumulate Long +name = vabal +multi_fn = vabd-signed-noext, b, c, d:int16x4_t +multi_fn = simd_cast, e:uint16x4_t, d +multi_fn = simd_add, a, {simd_cast, e} +a = 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 +b = 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 +c = 10, 10, 10, 10, 10, 10, 10, 10, 20, 0, 2, 4, 6, 8, 10, 12 +validate 10, 10, 10, 10, 10, 10, 10, 10, 20, 20, 20, 20, 20, 20, 20, 20 + +arm = vabal.s +aarch64 = sabal +generate int32x4_t:int16x4_t:int16x4_t:int32x4_t + +/// Signed Absolute difference and Accumulate Long +name = vabal +multi_fn = vabd-signed-noext, b, c, d:int32x2_t +multi_fn = simd_cast, e:uint32x2_t, d +multi_fn = simd_add, a, {simd_cast, e} +a = 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 +b = 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 +c = 10, 10, 10, 10, 10, 10, 10, 10, 20, 0, 2, 4, 6, 8, 10, 12 +validate 10, 10, 10, 10, 10, 10, 10, 10, 20, 20, 20, 20, 20, 20, 20, 20 + +arm = vabal.s +aarch64 = sabal +generate int64x2_t:int32x2_t:int32x2_t:int64x2_t + +/// Signed Absolute difference and Accumulate Long +name = vabal_high +no-q +multi_fn = simd_shuffle8!, d:int8x8_t, b, b, [8, 9, 10, 11, 12, 13, 14, 15] +multi_fn = simd_shuffle8!, e:int8x8_t, c, c, [8, 9, 10, 11, 12, 13, 14, 15] +multi_fn = vabd_s8, d, e, f:int8x8_t +multi_fn = simd_cast, f:uint8x8_t, f +multi_fn = simd_add, a, {simd_cast, f} +a = 9, 10, 11, 12, 13, 14, 15, 16 +b = 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 +c = 10, 10, 10, 10, 10, 10, 10, 10, 20, 0, 2, 4, 6, 8, 10, 12 +validate 20, 20, 20, 20, 20, 20, 20, 20 + +aarch64 = sabal +generate int16x8_t:int8x16_t:int8x16_t:int16x8_t + +/// Signed Absolute difference and Accumulate Long +name = vabal_high +no-q +multi_fn = simd_shuffle4!, d:int16x4_t, b, b, [4, 5, 6, 7] +multi_fn = simd_shuffle4!, e:int16x4_t, c, c, [4, 5, 6, 7] +multi_fn = vabd_s16, d, e, f:int16x4_t +multi_fn = simd_cast, f:uint16x4_t, f +multi_fn = simd_add, a, {simd_cast, f} +a = 9, 10, 11, 12 +b = 1, 2, 3, 4, 9, 10, 11, 12 +c = 10, 10, 10, 10, 20, 0, 2, 4 +validate 20, 20, 20, 20 + +aarch64 = sabal +generate int32x4_t:int16x8_t:int16x8_t:int32x4_t + +/// Signed Absolute difference and Accumulate Long +name = vabal_high +no-q +multi_fn = simd_shuffle2!, d:int32x2_t, b, b, [2, 3] +multi_fn = simd_shuffle2!, e:int32x2_t, c, c, [2, 3] +multi_fn = vabd_s32, d, e, f:int32x2_t +multi_fn = simd_cast, f:uint32x2_t, f +multi_fn = simd_add, a, {simd_cast, f} +a = 15, 16 +b = 1, 2, 15, 16 +c = 10, 10, 10, 12 +validate 20, 20 + +aarch64 = sabal +generate int64x2_t:int32x4_t:int32x4_t:int64x2_t + +//////////////////// +// Singned saturating Absolute value +//////////////////// + +/// Singned saturating Absolute value +name = vqabs +a = MIN, MAX, -6, -5, -4, -3, -2, -1, 0, -127, 127, 1, 2, 3, 4, 5 +validate MAX, MAX, 6, 5, 4, 3, 2, 1, 0, 127, 127, 1, 2, 3, 4, 5 + +arm = vqabs.s +aarch64 = sqabs +link-arm = vqabs._EXT_ +link-aarch64 = sqabs._EXT_ +generate int*_t + +/// Singned saturating Absolute value +name = vqabs +a = MIN, -7 +validate MAX, 7 + +aarch64 = sqabs +link-aarch64 = sqabs._EXT_ +generate int64x*_t diff --git a/library/stdarch/crates/stdarch-gen/src/main.rs b/library/stdarch/crates/stdarch-gen/src/main.rs index 6e341c00dd..03e6f409ab 100644 --- a/library/stdarch/crates/stdarch-gen/src/main.rs +++ b/library/stdarch/crates/stdarch-gen/src/main.rs @@ -1,3 +1,5 @@ +use self::Suffix::*; +use self::TargetFeature::*; use std::env; use std::fs::File; use std::io::prelude::*; @@ -72,8 +74,75 @@ fn type_len(t: &str) -> usize { "float32x4_t" => 4, "float64x1_t" => 1, "float64x2_t" => 2, + "poly8x8_t" => 8, + "poly8x16_t" => 16, + "poly16x4_t" => 4, + "poly16x8_t" => 8, "poly64x1_t" => 1, "poly64x2_t" => 2, + "i8" | "i16" | "i32" | "i64" | "u8" | "u16" | "u32" | "u64" | "f32" | "f64" | "p8" + | "p16" | "p64" | "p128" => 1, + _ => panic!("unknown type: {}", t), + } +} + +fn type_bits(t: &str) -> usize { + match t { + "int8x8_t" | "int8x16_t" | "uint8x8_t" | "uint8x16_t" | "poly8x8_t" | "poly8x16_t" + | "i8" | "u8" => 8, + "int16x4_t" | "int16x8_t" | "uint16x4_t" | "uint16x8_t" | "poly16x4_t" | "poly16x8_t" + | "i16" | "u16" => 16, + "int32x2_t" | "int32x4_t" | "uint32x2_t" | "uint32x4_t" | "i32" | "u32" | "float32x2_t" + | "float32x4_t" | "f32" => 32, + "int64x1_t" | "int64x2_t" | "uint64x1_t" | "uint64x2_t" | "poly64x1_t" | "poly64x2_t" + | "i64" | "u64" | "float64x1_t" | "float64x2_t" | "f64" => 64, + _ => panic!("unknown type: {}", t), + } +} + +fn type_exp_len(t: &str) -> usize { + match t { + "int8x8_t" => 3, + "int8x16_t" => 4, + "int16x4_t" => 2, + "int16x8_t" => 3, + "int32x2_t" => 1, + "int32x4_t" => 2, + "int64x1_t" => 0, + "int64x2_t" => 1, + "uint8x8_t" => 3, + "uint8x16_t" => 4, + "uint16x4_t" => 2, + "uint16x8_t" => 3, + "uint32x2_t" => 1, + "uint32x4_t" => 2, + "uint64x1_t" => 0, + "uint64x2_t" => 1, + "float16x4_t" => 2, + "float16x8_t" => 3, + "float32x2_t" => 1, + "float32x4_t" => 2, + "float64x1_t" => 0, + "float64x2_t" => 1, + "poly8x8_t" => 3, + "poly8x16_t" => 4, + "poly16x4_t" => 2, + "poly16x8_t" => 3, + "poly64x1_t" => 0, + "poly64x2_t" => 1, + _ => panic!("unknown type: {}", t), + } +} + +fn type_bits_exp_len(t: &str) -> usize { + match t { + "int8x8_t" | "int8x16_t" | "uint8x8_t" | "uint8x16_t" | "poly8x8_t" | "poly8x16_t" + | "i8" | "u8" => 3, + "int16x4_t" | "int16x8_t" | "uint16x4_t" | "uint16x8_t" | "poly16x4_t" | "poly16x8_t" + | "i16" | "u16" => 4, + "int32x2_t" | "int32x4_t" | "uint32x2_t" | "uint32x4_t" | "i32" | "u32" => 5, + "int64x1_t" | "int64x2_t" | "uint64x1_t" | "uint64x2_t" | "poly64x1_t" | "poly64x2_t" + | "i64" | "u64" => 6, _ => panic!("unknown type: {}", t), } } @@ -102,12 +171,217 @@ fn type_to_suffix(t: &str) -> &str { "float32x4_t" => "q_f32", "float64x1_t" => "_f64", "float64x2_t" => "q_f64", + "poly8x8_t" => "_p8", + "poly8x16_t" => "q_p8", + "poly16x4_t" => "_p16", + "poly16x8_t" => "q_p16", "poly64x1_t" => "_p64", "poly64x2_t" => "q_p64", + "i8" => "b_s8", + "i16" => "h_s16", + "i32" => "s_s32", + "i64" => "d_s64", + "u8" => "b_u8", + "u16" => "h_u16", + "u32" => "s_u32", + "u64" => "d_u64", + "f32" => "s_f32", + "f64" => "d_f64", + "p8" => "b_p8", + "p16" => "h_p16", + _ => panic!("unknown type: {}", t), + } +} + +fn type_to_n_suffix(t: &str) -> &str { + match t { + "int8x8_t" => "_n_s8", + "int8x16_t" => "q_n_s8", + "int16x4_t" => "_n_s16", + "int16x8_t" => "q_n_s16", + "int32x2_t" => "_n_s32", + "int32x4_t" => "q_n_s32", + "int64x1_t" => "_n_s64", + "int64x2_t" => "q_n_s64", + "uint8x8_t" => "_n_u8", + "uint8x16_t" => "q_n_u8", + "uint16x4_t" => "_n_u16", + "uint16x8_t" => "q_n_u16", + "uint32x2_t" => "_n_u32", + "uint32x4_t" => "q_n_u32", + "uint64x1_t" => "_n_u64", + "uint64x2_t" => "q_n_u64", + "float16x4_t" => "_n_f16", + "float16x8_t" => "q_n_f16", + "float32x2_t" => "_n_f32", + "float32x4_t" => "q_n_f32", + "float64x1_t" => "_n_f64", + "float64x2_t" => "q_n_f64", + "poly8x8_t" => "_n_p8", + "poly8x16_t" => "q_n_p8", + "poly16x4_t" => "_n_p16", + "poly16x8_t" => "q_n_p16", + "poly64x1_t" => "_n_p64", + "poly64x2_t" => "q_n_p64", + "i8" => "b_n_s8", + "i16" => "h_n_s16", + "i32" => "s_n_s32", + "i64" => "d_n_s64", + "u8" => "b_n_u8", + "u16" => "h_n_u16", + "u32" => "s_n_u32", + "u64" => "d_n_u64", _ => panic!("unknown type: {}", t), } } +fn type_to_noq_n_suffix(t: &str) -> &str { + match t { + "int8x8_t" | "int8x16_t" => "_n_s8", + "int16x4_t" | "int16x8_t" => "_n_s16", + "int32x2_t" | "int32x4_t" => "_n_s32", + "int64x1_t" | "int64x2_t" => "_n_s64", + "uint8x8_t" | "uint8x16_t" => "_n_u8", + "uint16x4_t" | "uint16x8_t" => "_n_u16", + "uint32x2_t" | "uint32x4_t" => "_n_u32", + "uint64x1_t" | "uint64x2_t" => "_n_u64", + "float16x4_t" | "float16x8_t" => "_n_f16", + "float32x2_t" | "float32x4_t" => "_n_f32", + "float64x1_t" | "float64x2_t" => "_n_f64", + "poly8x8_t" | "poly8x16_t" => "_n_p8", + "poly16x4_t" | "poly16x8_t" => "_n_p16", + "poly64x1_t" | "poly64x2_t" => "_n_p64", + "i8" => "b_n_s8", + "i16" => "h_n_s16", + "i32" => "s_n_s32", + "i64" => "d_n_s64", + "u8" => "b_n_u8", + "u16" => "h_n_u16", + "u32" => "s_n_u32", + "u64" => "d_n_u64", + _ => panic!("unknown type: {}", t), + } +} + +fn type_to_lane_suffixes<'a>(out_t: &'a str, in_t: &'a str) -> String { + let mut str = String::new(); + let suf = type_to_suffix(out_t); + if !suf.starts_with("_") { + str.push_str(&suf[0..1]); + } + str.push_str("_lane"); + str.push_str(type_to_suffix(in_t)); + str +} + +fn type_to_signed(t: &str) -> &str { + match t { + "int8x8_t" | "uint8x8_t" | "poly8x8_t" => "int8x8_t", + "int8x16_t" | "uint8x16_t" | "poly8x16_t" => "int8x16_t", + "int16x4_t" | "uint16x4_t" | "poly16x4_t" => "int16x4_t", + "int16x8_t" | "uint16x8_t" | "poly16x8_t" => "int16x8_t", + "int32x2_t" | "uint32x2_t" => "int32x2_t", + "int32x4_t" | "uint32x4_t" => "int32x4_t", + "int64x1_t" | "uint64x1_t" | "poly64x1_t" => "int64x1_t", + "int64x2_t" | "uint64x2_t" | "poly64x2_t" => "int64x2_t", + _ => panic!("unknown type: {}", t), + } +} + +fn type_to_unsigned(t: &str) -> &str { + match t { + "int8x8_t" | "uint8x8_t" | "poly8x8_t" => "uint8x8_t", + "int8x16_t" | "uint8x16_t" | "poly8x16_t" => "uint8x16_t", + "int16x4_t" | "uint16x4_t" | "poly16x4_t" => "uint16x4_t", + "int16x8_t" | "uint16x8_t" | "poly16x8_t" => "uint16x8_t", + "int32x2_t" | "uint32x2_t" => "uint32x2_t", + "int32x4_t" | "uint32x4_t" => "uint32x4_t", + "int64x1_t" | "uint64x1_t" | "poly64x1_t" => "uint64x1_t", + "int64x2_t" | "uint64x2_t" | "poly64x2_t" => "uint64x2_t", + _ => panic!("unknown type: {}", t), + } +} + +fn type_to_double_suffixes<'a>(out_t: &'a str, in_t: &'a str) -> String { + let mut str = String::new(); + let suf = type_to_suffix(in_t); + if suf.starts_with("q") && type_to_suffix(out_t).starts_with("q") { + str.push_str("q"); + } + if !suf.starts_with("_") && !suf.starts_with("q") { + str.push_str(&suf[0..1]); + } + str.push_str(type_to_noq_suffix(out_t)); + str.push_str(type_to_noq_suffix(in_t)); + str +} + +fn type_to_double_n_suffixes<'a>(out_t: &'a str, in_t: &'a str) -> String { + let mut str = String::new(); + let suf = type_to_suffix(in_t); + if suf.starts_with("q") && type_to_suffix(out_t).starts_with("q") { + str.push_str("q"); + } + if !suf.starts_with("_") && !suf.starts_with("q") { + str.push_str(&suf[0..1]); + } + str.push_str("_n"); + str.push_str(type_to_noq_suffix(out_t)); + str.push_str(type_to_noq_suffix(in_t)); + str +} + +fn type_to_noq_double_suffixes<'a>(out_t: &'a str, in_t: &'a str) -> String { + let mut str = String::new(); + str.push_str(type_to_noq_suffix(out_t)); + str.push_str(type_to_noq_suffix(in_t)); + str +} + +fn type_to_noq_suffix(t: &str) -> &str { + match t { + "int8x8_t" | "int8x16_t" | "i8" => "_s8", + "int16x4_t" | "int16x8_t" | "i16" => "_s16", + "int32x2_t" | "int32x4_t" | "i32" => "_s32", + "int64x1_t" | "int64x2_t" | "i64" => "_s64", + "uint8x8_t" | "uint8x16_t" | "u8" => "_u8", + "uint16x4_t" | "uint16x8_t" | "u16" => "_u16", + "uint32x2_t" | "uint32x4_t" | "u32" => "_u32", + "uint64x1_t" | "uint64x2_t" | "u64" => "_u64", + "float16x4_t" | "float16x8_t" => "_f16", + "float32x2_t" | "float32x4_t" | "f32" => "_f32", + "float64x1_t" | "float64x2_t" | "f64" => "_f64", + "poly8x8_t" | "poly8x16_t" => "_p8", + "poly16x4_t" | "poly16x8_t" => "_p16", + "poly64x1_t" | "poly64x2_t" | "p64" => "_p64", + _ => panic!("unknown type: {}", t), + } +} + +#[derive(Clone, Copy)] +enum Suffix { + Normal, + Double, + NoQ, + NoQDouble, + NSuffix, + DoubleN, + NoQNSuffix, + OutSuffix, + OutNSuffix, + Lane, + In2, + In2Lane, +} + +#[derive(Clone, Copy)] +enum TargetFeature { + Default, + ArmV7, + FPArmV8, + Crypto, +} + fn type_to_global_type(t: &str) -> &str { match t { "int8x8_t" => "i8x8", @@ -132,41 +406,81 @@ fn type_to_global_type(t: &str) -> &str { "float32x4_t" => "f32x4", "float64x1_t" => "f64", "float64x2_t" => "f64x2", + "poly8x8_t" => "i8x8", + "poly8x16_t" => "i8x16", + "poly16x4_t" => "i16x4", + "poly16x8_t" => "i16x8", "poly64x1_t" => "i64x1", "poly64x2_t" => "i64x2", + "i8" => "i8", + "i16" => "i16", + "i32" => "i32", + "i64" => "i64", + "u8" => "u8", + "u16" => "u16", + "u32" => "u32", + "u64" => "u64", + "f32" => "f32", + "f64" => "f64", + "p8" => "p8", + "p16" => "p16", + "p64" => "p64", + "p128" => "p128", _ => panic!("unknown type: {}", t), } } -// fn type_to_native_type(t: &str) -> &str { -// match t { -// "int8x8_t" => "i8", -// "int8x16_t" => "i8", -// "int16x4_t" => "i16", -// "int16x8_t" => "i16", -// "int32x2_t" => "i32", -// "int32x4_t" => "i32", -// "int64x1_t" => "i64", -// "int64x2_t" => "i64", -// "uint8x8_t" => "u8", -// "uint8x16_t" => "u8", -// "uint16x4_t" => "u16", -// "uint16x8_t" => "u16", -// "uint32x2_t" => "u32", -// "uint32x4_t" => "u32", -// "uint64x1_t" => "u64", -// "uint64x2_t" => "u64", -// "float16x4_t" => "f16", -// "float16x8_t" => "f16", -// "float32x2_t" => "f32", -// "float32x4_t" => "f32", -// "float64x1_t" => "f64", -// "float64x2_t" => "f64", -// "poly64x1_t" => "i64", -// "poly64x2_t" => "i64", -// _ => panic!("unknown type: {}", t), -// } -// } +fn type_to_native_type(t: &str) -> &str { + match t { + "int8x8_t" | "int8x16_t" | "i8" => "i8", + "int16x4_t" | "int16x8_t" | "i16" => "i16", + "int32x2_t" | "int32x4_t" | "i32" => "i32", + "int64x1_t" | "int64x2_t" | "i64" => "i64", + "uint8x8_t" | "uint8x16_t" | "u8" => "u8", + "uint16x4_t" | "uint16x8_t" | "u16" => "u16", + "uint32x2_t" | "uint32x4_t" | "u32" => "u32", + "uint64x1_t" | "uint64x2_t" | "u64" => "u64", + "float16x4_t" | "float16x8_t" => "f16", + "float32x2_t" | "float32x4_t" => "f32", + "float64x1_t" | "float64x2_t" => "f64", + "poly64x1_t" | "poly64x2_t" => "u64", + _ => panic!("unknown type: {}", t), + } +} + +fn native_type_to_type(t: &str) -> &str { + match t { + "i8" => "int8x8_t", + "i16" => "int16x4_t", + "i32" => "int32x2_t", + "i64" => "int64x1_t", + "u8" => "uint8x8_t", + "u16" => "uint16x4_t", + "u32" => "uint32x2_t", + "u64" => "uint64x1_t", + "f16" => "float16x4_t", + "f32" => "float32x2_t", + "f64" => "float64x1_t", + _ => panic!("unknown type: {}", t), + } +} + +fn native_type_to_long_type(t: &str) -> &str { + match t { + "i8" => "int8x16_t", + "i16" => "int16x8_t", + "i32" => "int32x4_t", + "i64" => "int64x2_t", + "u8" => "uint8x16_t", + "u16" => "uint16x8_t", + "u32" => "uint32x4_t", + "u64" => "uint64x2_t", + "f16" => "float16x8_t", + "f32" => "float32x4_t", + "f64" => "float64x2_t", + _ => panic!("unknown type: {}", t), + } +} fn type_to_ext(t: &str) -> &str { match t { @@ -192,6 +506,22 @@ fn type_to_ext(t: &str) -> &str { "float32x4_t" => "v4f32", "float64x1_t" => "v1f64", "float64x2_t" => "v2f64", + "poly8x8_t" => "v8i8", + "poly8x16_t" => "v16i8", + "poly16x4_t" => "v4i16", + "poly16x8_t" => "v8i16", + "i8" => "i8", + "i16" => "i16", + "i32" => "i32", + "i64" => "i64", + "u8" => "i8", + "u16" => "i16", + "u32" => "i32", + "u64" => "i64", + "f32" => "f32", + "f64" => "f64", + "p64" => "p64", + "p128" => "p128", /* "poly64x1_t" => "i64x1", "poly64x2_t" => "i64x2", @@ -200,6 +530,97 @@ fn type_to_ext(t: &str) -> &str { } } +fn type_to_half(t: &str) -> &str { + match t { + "int8x16_t" => "int8x8_t", + "int16x8_t" => "int16x4_t", + "int32x4_t" => "int32x2_t", + "int64x2_t" => "int64x1_t", + "uint8x16_t" => "uint8x8_t", + "uint16x8_t" => "uint16x4_t", + "uint32x4_t" => "uint32x2_t", + "uint64x2_t" => "uint64x1_t", + "poly8x16_t" => "poly8x8_t", + "poly16x8_t" => "poly16x4_t", + "float32x4_t" => "float32x2_t", + "float64x2_t" => "float64x1_t", + _ => panic!("unknown half type for {}", t), + } +} + +fn asc(start: i32, len: usize) -> String { + let mut s = String::from("["); + for i in 0..len { + if i != 0 { + s.push_str(", "); + } + let n = start + i as i32; + s.push_str(&n.to_string()); + } + s.push_str("]"); + s +} + +fn transpose1(x: usize) -> &'static str { + match x { + 2 => "[0, 2]", + 4 => "[0, 4, 2, 6]", + 8 => "[0, 8, 2, 10, 4, 12, 6, 14]", + 16 => "[0, 16, 2, 18, 4, 20, 6, 22, 8, 24, 10, 26, 12, 28, 14, 30]", + _ => panic!("unknown transpose order of len {}", x), + } +} + +fn transpose2(x: usize) -> &'static str { + match x { + 2 => "[1, 3]", + 4 => "[1, 5, 3, 7]", + 8 => "[1, 9, 3, 11, 5, 13, 7, 15]", + 16 => "[1, 17, 3, 19, 5, 21, 7, 23, 9, 25, 11, 27, 13, 29, 15, 31]", + _ => panic!("unknown transpose order of len {}", x), + } +} + +fn zip1(x: usize) -> &'static str { + match x { + 2 => "[0, 2]", + 4 => "[0, 4, 1, 5]", + 8 => "[0, 8, 1, 9, 2, 10, 3, 11]", + 16 => "[0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23]", + _ => panic!("unknown zip order of len {}", x), + } +} + +fn zip2(x: usize) -> &'static str { + match x { + 2 => "[1, 3]", + 4 => "[2, 6, 3, 7]", + 8 => "[4, 12, 5, 13, 6, 14, 7, 15]", + 16 => "[8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31]", + _ => panic!("unknown zip order of len {}", x), + } +} + +fn unzip1(x: usize) -> &'static str { + match x { + 2 => "[0, 2]", + 4 => "[0, 2, 4, 6]", + 8 => "[0, 2, 4, 6, 8, 10, 12, 14]", + 16 => "[0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30]", + _ => panic!("unknown unzip order of len {}", x), + } +} + +fn unzip2(x: usize) -> &'static str { + match x { + 2 => "[1, 3]", + 4 => "[1, 3, 5, 7]", + 8 => "[1, 3, 5, 7, 9, 11, 13, 15]", + 16 => "[1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31]", + _ => panic!("unknown unzip order of len {}", x), + } +} + fn values(t: &str, vs: &[String]) -> String { if vs.len() == 1 && !t.contains('x') { format!(": {} = {}", t, vs[0]) @@ -278,13 +699,138 @@ fn ff_val(t: &str) -> &'static str { fn false_val(_t: &str) -> &'static str { "0" } + +fn bits(t: &str) -> &'static str { + match &t[..3] { + "u8x" => "8", + "u16" => "16", + "u32" => "32", + "u64" => "64", + "i8x" => "8", + "i16" => "16", + "i32" => "32", + "i64" => "64", + "p8x" => "8", + "p16" => "16", + "p64" => "64", + _ => panic!("Unknown bits for type {}", t), + } +} + +fn bits_minus_one(t: &str) -> &'static str { + match &t[..3] { + "u8x" => "7", + "u16" => "15", + "u32" => "31", + "u64" => "63", + "i8x" => "7", + "i16" => "15", + "i32" => "31", + "i64" => "63", + "p8x" => "7", + "p16" => "15", + "p64" => "63", + _ => panic!("Unknown bits for type {}", t), + } +} + +fn half_bits(t: &str) -> &'static str { + match &t[..3] { + "u8x" => "4", + "u16" => "8", + "u32" => "16", + "u64" => "32", + "i8x" => "4", + "i16" => "8", + "i32" => "16", + "i64" => "32", + "p8x" => "4", + "p16" => "8", + "p64" => "32", + _ => panic!("Unknown bits for type {}", t), + } +} + +fn type_len_str(t: &str) -> &'static str { + match t { + "int8x8_t" => "8", + "int8x16_t" => "16", + "int16x4_t" => "4", + "int16x8_t" => "8", + "int32x2_t" => "2", + "int32x4_t" => "4", + "int64x1_t" => "1", + "int64x2_t" => "2", + "uint8x8_t" => "8", + "uint8x16_t" => "16", + "uint16x4_t" => "4", + "uint16x8_t" => "8", + "uint32x2_t" => "2", + "uint32x4_t" => "4", + "uint64x1_t" => "1", + "uint64x2_t" => "2", + "float16x4_t" => "4", + "float16x8_t" => "8", + "float32x2_t" => "2", + "float32x4_t" => "4", + "float64x1_t" => "1", + "float64x2_t" => "2", + "poly8x8_t" => "8", + "poly8x16_t" => "16", + "poly16x4_t" => "4", + "poly16x8_t" => "8", + "poly64x1_t" => "1", + "poly64x2_t" => "2", + _ => panic!("unknown type: {}", t), + } +} + +fn type_half_len_str(t: &str) -> &'static str { + match t { + "int8x8_t" => "4", + "int8x16_t" => "8", + "int16x4_t" => "2", + "int16x8_t" => "4", + "int32x2_t" => "1", + "int32x4_t" => "2", + "int64x1_t" => "0", + "int64x2_t" => "1", + "uint8x8_t" => "4", + "uint8x16_t" => "8", + "uint16x4_t" => "2", + "uint16x8_t" => "4", + "uint32x2_t" => "1", + "uint32x4_t" => "2", + "uint64x1_t" => "0", + "uint64x2_t" => "1", + "float16x4_t" => "2", + "float16x8_t" => "4", + "float32x2_t" => "1", + "float32x4_t" => "2", + "float64x1_t" => "0", + "float64x2_t" => "1", + "poly8x8_t" => "4", + "poly8x16_t" => "8", + "poly16x4_t" => "2", + "poly16x8_t" => "4", + "poly64x1_t" => "0", + "poly64x2_t" => "1", + _ => panic!("unknown type: {}", t), + } +} + fn map_val<'v>(t: &str, v: &'v str) -> &'v str { match v { "FALSE" => false_val(t), "TRUE" => true_val(t), - "MAX" => min_val(t), - "MIN" => max_val(t), + "MAX" => max_val(t), + "MIN" => min_val(t), "FF" => ff_val(t), + "BITS" => bits(t), + "BITS_M1" => bits_minus_one(t), + "HFBITS" => half_bits(t), + "LEN" => type_len_str(t), + "HFLEN" => type_half_len_str(t), o => o, } } @@ -293,69 +839,355 @@ fn map_val<'v>(t: &str, v: &'v str) -> &'v str { fn gen_aarch64( current_comment: &str, current_fn: &Option, - name: &str, + current_name: &str, current_aarch64: &Option, link_aarch64: &Option, - in_t: &str, + const_aarch64: &Option, + constn: &Option, + in_t: &[&str; 3], out_t: &str, - current_tests: &[(Vec, Vec, Vec)], + current_tests: &[( + Vec, + Vec, + Vec, + Option, + Vec, + )], + suffix: Suffix, + para_num: i32, + target: TargetFeature, + fixed: &Vec, + multi_fn: &Vec, ) -> (String, String) { - let _global_t = type_to_global_type(in_t); + let _global_t = type_to_global_type(in_t[0]); let _global_ret_t = type_to_global_type(out_t); + let name = match suffix { + Normal => format!("{}{}", current_name, type_to_suffix(in_t[1])), + NoQ => format!("{}{}", current_name, type_to_noq_suffix(in_t[1])), + Double => format!( + "{}{}", + current_name, + type_to_double_suffixes(out_t, in_t[1]) + ), + NoQDouble => format!( + "{}{}", + current_name, + type_to_noq_double_suffixes(out_t, in_t[1]) + ), + NSuffix => format!("{}{}", current_name, type_to_n_suffix(in_t[1])), + DoubleN => format!( + "{}{}", + current_name, + type_to_double_n_suffixes(out_t, in_t[1]) + ), + NoQNSuffix => format!("{}{}", current_name, type_to_noq_n_suffix(in_t[1])), + OutSuffix => format!("{}{}", current_name, type_to_suffix(out_t)), + OutNSuffix => format!("{}{}", current_name, type_to_n_suffix(out_t)), + Lane => format!("{}{}", current_name, type_to_lane_suffixes(out_t, in_t[1])), + In2 => format!("{}{}", current_name, type_to_suffix(in_t[2])), + In2Lane => format!("{}{}", current_name, type_to_lane_suffixes(out_t, in_t[2])), + }; + let current_target = match target { + Default => "neon", + ArmV7 => "v7", + FPArmV8 => "fp-armv8,v8", + Crypto => "neon,crypto", + }; let current_fn = if let Some(current_fn) = current_fn.clone() { if link_aarch64.is_some() { panic!("[{}] Can't specify link and fn at the same time.", name) } current_fn + } else if link_aarch64.is_some() { + format!("{}_", name) } else { - if link_aarch64.is_none() { - panic!("[{}] Either fn or link-aarch have to be specified.", name) + if multi_fn.is_empty() { + panic!( + "[{}] Either (multi) fn or link-aarch have to be specified.", + name + ) } - format!("{}_", name) + String::new() }; let current_aarch64 = current_aarch64.clone().unwrap(); - let ext_c = if let Some(link_aarch64) = link_aarch64.clone() { - let ext = type_to_ext(in_t); - - format!( + let mut ext_c = String::new(); + let mut ext_c_const = String::new(); + let mut link_t: Vec = vec![ + in_t[0].to_string(), + in_t[1].to_string(), + in_t[2].to_string(), + out_t.to_string(), + ]; + if let Some(mut link_aarch64) = link_aarch64.clone() { + if link_aarch64.contains(":") { + let links: Vec<_> = link_aarch64.split(':').map(|v| v.to_string()).collect(); + assert_eq!(links.len(), 5); + link_aarch64 = links[0].to_string(); + link_t = vec![ + links[1].clone(), + links[2].clone(), + links[3].clone(), + links[4].clone(), + ]; + } + let ext = type_to_ext(in_t[0]); + let ext2 = type_to_ext(out_t); + let link_aarch64 = if link_aarch64.starts_with("llvm") { + link_aarch64.replace("_EXT_", ext).replace("_EXT2_", ext2) + } else { + let mut link = String::from("llvm.aarch64.neon."); + link.push_str(&link_aarch64); + link.replace("_EXT_", ext).replace("_EXT2_", ext2) + }; + ext_c = format!( r#"#[allow(improper_ctypes)] extern "C" {{ - #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.{}")] - fn {}(a: {}, a: {}) -> {}; + #[cfg_attr(target_arch = "aarch64", link_name = "{}")] + fn {}({}) -> {}; }} "#, - link_aarch64.replace("_EXT_", ext), + link_aarch64, current_fn, - in_t, - in_t, - out_t - ) + match para_num { + 1 => { + format!("a: {}", link_t[0]) + } + 2 => { + format!("a: {}, b: {}", link_t[0], link_t[1]) + } + 3 => { + format!("a: {}, b: {}, c: {}", link_t[0], link_t[1], link_t[2]) + } + _ => unimplemented!("unknown para_num"), + }, + link_t[3] + ); + if const_aarch64.is_some() { + ext_c_const = format!( + r#"#[allow(improper_ctypes)] + extern "C" {{ + #[cfg_attr(target_arch = "aarch64", link_name = "{}")] + fn {}({}) -> {}; + }} + "#, + link_aarch64, + current_fn, + match para_num { + 1 => { + format!("a: {}, n: i32", in_t[0]) + } + 2 => { + format!("a: {}, b: {}, n: i32", in_t[0], in_t[1]) + } + 3 => { + format!("a: {}, b: {}, c: {}, n: i32", in_t[0], in_t[1], in_t[2]) + } + _ => unimplemented!("unknown para_num"), + }, + out_t + ); + } + }; + let const_declare = if let Some(constn) = constn { + if constn.contains(":") { + let constns: Vec<_> = constn.split(':').map(|v| v.to_string()).collect(); + assert_eq!(constns.len(), 2); + format!(r#""#, constns[0], constns[1]) + } else { + format!(r#""#, constn) + } } else { String::new() }; + let multi_calls = if !multi_fn.is_empty() { + let mut calls = String::new(); + for i in 0..multi_fn.len() { + if i > 0 { + calls.push_str("\n "); + } + calls.push_str(&get_call( + &multi_fn[i], + current_name, + &const_declare, + in_t, + out_t, + fixed, + None, + )); + } + calls + } else { + String::new() + }; + let const_assert = if let Some(constn) = constn { + if constn.contains(":") { + let constns: Vec<_> = constn.split(':').map(|v| v.to_string()).collect(); + let const_test = current_tests[0].3.as_ref().unwrap(); + let const_tests: Vec<_> = const_test.split(':').map(|v| v.to_string()).collect(); + assert_eq!(constns.len(), 2); + assert_eq!(const_tests.len(), 2); + format!( + r#", {} = {}, {} = {}"#, + constns[0], + map_val(in_t[1], &const_tests[0]), + constns[1], + map_val(in_t[1], &const_tests[1]), + ) + } else { + format!( + r#", {} = {}"#, + constn, + map_val(in_t[1], current_tests[0].3.as_ref().unwrap()) + ) + } + } else { + String::new() + }; + let const_legacy = if let Some(constn) = constn { + if constn.contains(":") { + format!( + "\n#[rustc_legacy_const_generics({}, {})]", + para_num - 1, + para_num + 1 + ) + } else { + format!("\n#[rustc_legacy_const_generics({})]", para_num) + } + } else { + String::new() + }; + let trans: [&str; 2] = if link_t[3] != out_t { + ["transmute(", ")"] + } else { + ["", ""] + }; + let call = if let Some(const_aarch64) = const_aarch64 { + match para_num { + 1 => format!( + r#"pub unsafe fn {}{}(a: {}) -> {} {{ + {} + {}{}(a, {}) +}}"#, + name, + const_declare, + in_t[0], + out_t, + multi_calls, + ext_c_const, + current_fn, + const_aarch64 + ), + 2 => format!( + r#"pub unsafe fn {}{}(a: {}) -> {} {{ + {}{}{}(a, b, {}) +}}"#, + name, + const_declare, + in_t[0], + out_t, + multi_calls, + ext_c_const, + current_fn, + const_aarch64 + ), + _ => String::new(), + } + } else { + match (multi_calls.len(), para_num, fixed.len()) { + (0, 1, 0) => format!( + r#"pub unsafe fn {}{}(a: {}) -> {} {{ + {}{}{}(a){} +}}"#, + name, const_declare, in_t[0], out_t, ext_c, trans[0], current_fn, trans[1] + ), + (0, 1, _) => { + let fixed: Vec = fixed.iter().take(type_len(in_t[0])).cloned().collect(); + format!( + r#"pub unsafe fn {}{}(a: {}) -> {} {{ + let b{}; + {}{}{}(a, transmute(b)){} +}}"#, + name, + const_declare, + in_t[0], + out_t, + values(in_t[0], &fixed), + ext_c, + trans[0], + current_fn, + trans[1], + ) + } + (0, 2, _) => format!( + r#"pub unsafe fn {}{}(a: {}, b: {}) -> {} {{ + {}{}{}(a, b){} +}}"#, + name, const_declare, in_t[0], in_t[1], out_t, ext_c, trans[0], current_fn, trans[1], + ), + (0, 3, _) => format!( + r#"pub unsafe fn {}{}(a: {}, b: {}, c: {}) -> {} {{ + {}{}(a, b, c) +}}"#, + name, const_declare, in_t[0], in_t[1], in_t[2], out_t, ext_c, current_fn, + ), + (_, 1, _) => format!( + r#"pub unsafe fn {}{}(a: {}) -> {} {{ + {}{} +}}"#, + name, const_declare, in_t[0], out_t, ext_c, multi_calls, + ), + (_, 2, _) => format!( + r#"pub unsafe fn {}{}(a: {}, b: {}) -> {} {{ + {}{} +}}"#, + name, const_declare, in_t[0], in_t[1], out_t, ext_c, multi_calls, + ), + (_, 3, _) => format!( + r#"pub unsafe fn {}{}(a: {}, b: {}, c: {}) -> {} {{ + {}{} +}}"#, + name, const_declare, in_t[0], in_t[1], in_t[2], out_t, ext_c, multi_calls, + ), + (_, _, _) => String::new(), + } + }; let function = format!( r#" {} #[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(test, assert_instr({}))] -pub unsafe fn {}(a: {}, b: {}) -> {} {{ - {}{}(a, b) -}} +#[target_feature(enable = "{}")] +#[cfg_attr(test, assert_instr({}{}))]{} +{} "#, - current_comment, current_aarch64, name, in_t, in_t, out_t, ext_c, current_fn, + current_comment, current_target, current_aarch64, const_assert, const_legacy, call ); - let test = gen_test(name, &in_t, &out_t, current_tests, type_len(in_t)); + let test = gen_test( + &name, + in_t, + &out_t, + current_tests, + [type_len(in_t[0]), type_len(in_t[1]), type_len(in_t[2])], + type_len(out_t), + para_num, + ); (function, test) } fn gen_test( name: &str, - in_t: &str, + in_t: &[&str; 3], out_t: &str, - current_tests: &[(Vec, Vec, Vec)], - len: usize, + current_tests: &[( + Vec, + Vec, + Vec, + Option, + Vec, + )], + len_in: [usize; 3], + len_out: usize, + para_num: i32, ) -> String { let mut test = format!( r#" @@ -363,24 +1195,84 @@ fn gen_test( unsafe fn test_{}() {{"#, name, ); - for (a, b, e) in current_tests { - let a: Vec = a.iter().take(len).cloned().collect(); - let b: Vec = b.iter().take(len).cloned().collect(); - let e: Vec = e.iter().take(len).cloned().collect(); - let t = format!( - r#" + for (a, b, c, n, e) in current_tests { + let a: Vec = a.iter().take(len_in[0]).cloned().collect(); + let b: Vec = b.iter().take(len_in[1]).cloned().collect(); + let c: Vec = c.iter().take(len_in[2]).cloned().collect(); + let e: Vec = e.iter().take(len_out).cloned().collect(); + let const_value = if let Some(constn) = n { + if constn.contains(":") { + let constns: Vec<_> = constn.split(':').map(|v| v.to_string()).collect(); + format!( + r#"::<{}, {}>"#, + map_val(in_t[1], &constns[0]), + map_val(in_t[1], &constns[1]) + ) + } else { + format!(r#"::<{}>"#, map_val(in_t[1], constn)) + } + } else { + String::new() + }; + let t = { + match para_num { + 1 => { + format!( + r#" + let a{}; + let e{}; + let r: {} = transmute({}{}(transmute(a))); + assert_eq!(r, e); +"#, + values(in_t[0], &a), + values(out_t, &e), + type_to_global_type(out_t), + name, + const_value + ) + } + 2 => { + format!( + r#" let a{}; let b{}; let e{}; - let r: {} = transmute({}(transmute(a), transmute(b))); + let r: {} = transmute({}{}(transmute(a), transmute(b))); assert_eq!(r, e); "#, - values(in_t, &a), - values(in_t, &b), - values(out_t, &e), - type_to_global_type(out_t), - name - ); + values(in_t[0], &a), + values(in_t[1], &b), + values(out_t, &e), + type_to_global_type(out_t), + name, + const_value + ) + } + 3 => { + format!( + r#" + let a{}; + let b{}; + let c{}; + let e{}; + let r: {} = transmute({}{}(transmute(a), transmute(b), transmute(c))); + assert_eq!(r, e); +"#, + values(in_t[0], &a), + values(in_t[1], &b), + values(in_t[2], &c), + values(out_t, &e), + type_to_global_type(out_t), + name, + const_value + ) + } + _ => { + panic!("no support para_num:{}", para_num.to_string()) + } + } + }; + test.push_str(&t); } test.push_str(" }\n"); @@ -391,20 +1283,72 @@ fn gen_test( fn gen_arm( current_comment: &str, current_fn: &Option, - name: &str, + current_name: &str, current_arm: &str, link_arm: &Option, current_aarch64: &Option, link_aarch64: &Option, - in_t: &str, + const_arm: &Option, + const_aarch64: &Option, + constn: &Option, + in_t: &[&str; 3], out_t: &str, - current_tests: &[(Vec, Vec, Vec)], + current_tests: &[( + Vec, + Vec, + Vec, + Option, + Vec, + )], + suffix: Suffix, + para_num: i32, + target: TargetFeature, + fixed: &Vec, + multi_fn: &Vec, ) -> (String, String) { - let _global_t = type_to_global_type(in_t); + let _global_t = type_to_global_type(in_t[0]); let _global_ret_t = type_to_global_type(out_t); + let name = match suffix { + Normal => format!("{}{}", current_name, type_to_suffix(in_t[1])), + NoQ => format!("{}{}", current_name, type_to_noq_suffix(in_t[1])), + Double => format!( + "{}{}", + current_name, + type_to_double_suffixes(out_t, in_t[1]) + ), + NoQDouble => format!( + "{}{}", + current_name, + type_to_noq_double_suffixes(out_t, in_t[1]) + ), + NSuffix => format!("{}{}", current_name, type_to_n_suffix(in_t[1])), + DoubleN => format!( + "{}{}", + current_name, + type_to_double_n_suffixes(out_t, in_t[1]) + ), + NoQNSuffix => format!("{}{}", current_name, type_to_noq_n_suffix(in_t[1])), + OutSuffix => format!("{}{}", current_name, type_to_suffix(out_t)), + OutNSuffix => format!("{}{}", current_name, type_to_n_suffix(out_t)), + Lane => format!("{}{}", current_name, type_to_lane_suffixes(out_t, in_t[1])), + In2 => format!("{}{}", current_name, type_to_suffix(in_t[2])), + In2Lane => format!("{}{}", current_name, type_to_lane_suffixes(out_t, in_t[2])), + }; let current_aarch64 = current_aarch64 .clone() .unwrap_or_else(|| current_arm.to_string()); + let current_target_aarch64 = match target { + Default => "neon", + ArmV7 => "neon", + FPArmV8 => "neon", + Crypto => "neon,crypto", + }; + let current_target_arm = match target { + Default => "v7", + ArmV7 => "v7", + FPArmV8 => "fp-armv8,v8", + Crypto => "crypto,v8", + }; let current_fn = if let Some(current_fn) = current_fn.clone() { if link_aarch64.is_some() || link_arm.is_some() { @@ -414,62 +1358,537 @@ fn gen_arm( ) } current_fn + } else if link_aarch64.is_some() || link_arm.is_some() { + format!("{}_", name) } else { - if link_aarch64.is_none() || link_arm.is_none() { + if multi_fn.is_empty() { panic!( "[{}] Either fn or link-arm and link-aarch have to be specified.", name ) } - format!("{}_", name) + String::new() }; - - let ext_c = - if let (Some(link_arm), Some(link_aarch64)) = (link_arm.clone(), link_aarch64.clone()) { - let ext = type_to_ext(in_t); - - format!( + let mut ext_c = String::new(); + let mut ext_c_arm = if multi_fn.is_empty() { + String::new() + } else { + String::from( + r#" + "#, + ) + }; + let mut ext_c_aarch64 = if multi_fn.is_empty() { + String::new() + } else { + String::from( + r#" + "#, + ) + }; + let mut link_arm_t: Vec = vec![ + in_t[0].to_string(), + in_t[1].to_string(), + in_t[2].to_string(), + out_t.to_string(), + ]; + let mut link_aarch64_t: Vec = vec![ + in_t[0].to_string(), + in_t[1].to_string(), + in_t[2].to_string(), + out_t.to_string(), + ]; + if let (Some(mut link_arm), Some(mut link_aarch64)) = (link_arm.clone(), link_aarch64.clone()) { + if link_arm.contains(":") { + let links: Vec<_> = link_arm.split(':').map(|v| v.to_string()).collect(); + assert_eq!(links.len(), 5); + link_arm = links[0].to_string(); + link_arm_t = vec![ + links[1].clone(), + links[2].clone(), + links[3].clone(), + links[4].clone(), + ]; + } + if link_aarch64.contains(":") { + let links: Vec<_> = link_aarch64.split(':').map(|v| v.to_string()).collect(); + assert_eq!(links.len(), 5); + link_aarch64 = links[0].to_string(); + link_aarch64_t = vec![ + links[1].clone(), + links[2].clone(), + links[3].clone(), + links[4].clone(), + ]; + } + let ext = type_to_ext(in_t[0]); + let ext2 = type_to_ext(out_t); + let link_arm = if link_arm.starts_with("llvm") { + link_arm.replace("_EXT_", ext).replace("_EXT2_", ext2) + } else { + let mut link = String::from("llvm.arm.neon."); + link.push_str(&link_arm); + link.replace("_EXT_", ext).replace("_EXT2_", ext2) + }; + let link_aarch64 = if link_aarch64.starts_with("llvm") { + link_aarch64.replace("_EXT_", ext).replace("_EXT2_", ext2) + } else { + let mut link = String::from("llvm.aarch64.neon."); + link.push_str(&link_aarch64); + link.replace("_EXT_", ext).replace("_EXT2_", ext2) + }; + if out_t == link_arm_t[3] && out_t == link_aarch64_t[3] { + ext_c = format!( r#"#[allow(improper_ctypes)] extern "C" {{ - #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.{}")] - #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.{}")] - fn {}(a: {}, b: {}) -> {}; + #[cfg_attr(target_arch = "arm", link_name = "{}")] + #[cfg_attr(target_arch = "aarch64", link_name = "{}")] + fn {}({}) -> {}; }} "#, - link_arm.replace("_EXT_", ext), - link_aarch64.replace("_EXT_", ext), + link_arm, + link_aarch64, current_fn, - in_t, - in_t, + match para_num { + 1 => { + format!("a: {}", in_t[0]) + } + 2 => { + format!("a: {}, b: {}", in_t[0], in_t[1]) + } + 3 => { + format!("a: {}, b: {}, c: {}", in_t[0], in_t[1], in_t[2]) + } + _ => unimplemented!("unknown para_num"), + }, out_t + ); + }; + if let Some(const_arm) = const_arm { + let (_, const_type) = if const_arm.contains(":") { + let consts: Vec<_> = const_arm.split(':').map(|v| v.trim().to_string()).collect(); + (consts[0].clone(), consts[1].clone()) + } else { + ( + const_arm.to_string(), + in_t[para_num as usize - 1].to_string(), + ) + }; + ext_c_arm.push_str(&format!( + r#"#[allow(improper_ctypes)] + extern "C" {{ + #[cfg_attr(target_arch = "arm", link_name = "{}")] + fn {}({}) -> {}; + }} +"#, + link_arm, + current_fn, + match para_num { + 1 => { + format!("a: {}, n: {}", in_t[0], const_type) + } + 2 => { + format!("a: {}, b: {}, n: {}", in_t[0], in_t[1], const_type) + } + 3 => { + format!( + "a: {}, b: {}, c: {}, n: {}", + in_t[0], in_t[1], in_t[2], const_type + ) + } + _ => unimplemented!("unknown para_num"), + }, + out_t + )); + }; + if out_t != link_arm_t[3] { + ext_c_arm.push_str(&format!( + r#"#[allow(improper_ctypes)] + extern "C" {{ + #[cfg_attr(target_arch = "arm", link_name = "{}")] + fn {}({}) -> {}; + }} +"#, + link_arm, + current_fn, + match para_num { + 1 => { + format!("a: {}", link_arm_t[0]) + } + 2 => { + format!("a: {}, b: {}", link_arm_t[0], link_arm_t[1]) + } + 3 => { + format!( + "a: {}, b: {}, c: {}", + link_arm_t[0], link_arm_t[1], link_arm_t[2] + ) + } + _ => unimplemented!("unknown para_num"), + }, + link_arm_t[3] + )); + } + if const_aarch64.is_some() { + ext_c_aarch64.push_str(&format!( + r#"#[allow(improper_ctypes)] + extern "C" {{ + #[cfg_attr(target_arch = "aarch64", link_name = "{}")] + fn {}({}) -> {}; + }} +"#, + link_aarch64, + current_fn, + match para_num { + 1 => { + format!("a: {}, n: i32", in_t[0]) + } + 2 => { + format!("a: {}, b: {}, n: i32", in_t[0], in_t[1]) + } + 3 => { + format!("a: {}, b: {}, c: {}, n: i32", in_t[0], in_t[1], in_t[2]) + } + _ => unimplemented!("unknown para_num"), + }, + out_t + )); + } + if out_t != link_aarch64_t[3] { + ext_c_aarch64.push_str(&format!( + r#"#[allow(improper_ctypes)] + extern "C" {{ + #[cfg_attr(target_arch = "aarch64", link_name = "{}")] + fn {}({}) -> {}; + }} +"#, + link_aarch64, + current_fn, + match para_num { + 1 => { + format!("a: {}", link_aarch64_t[0]) + } + 2 => { + format!("a: {}, b: {}", link_aarch64_t[0], link_aarch64_t[1]) + } + 3 => { + format!( + "a: {}, b: {}, c: {}", + link_aarch64_t[0], link_aarch64_t[1], link_aarch64_t[2] + ) + } + _ => unimplemented!("unknown para_num"), + }, + link_aarch64_t[3] + )); + } + }; + let const_declare = if let Some(constn) = constn { + format!(r#""#, constn) + } else { + String::new() + }; + let multi_calls = if !multi_fn.is_empty() { + let mut calls = String::new(); + for i in 0..multi_fn.len() { + if i > 0 { + calls.push_str("\n "); + } + calls.push_str(&get_call( + &multi_fn[i], + current_name, + &const_declare, + in_t, + out_t, + fixed, + None, + )); + } + calls + } else { + String::new() + }; + let const_assert = if let Some(constn) = constn { + format!( + r#", {} = {}"#, + constn, + map_val(in_t[1], current_tests[0].3.as_ref().unwrap()) + ) + } else { + String::new() + }; + let const_legacy = if constn.is_some() { + format!("\n#[rustc_legacy_const_generics({})]", para_num) + } else { + String::new() + }; + let trans: [&str; 2] = if out_t == link_arm_t[3] && out_t == link_aarch64_t[3] { + ["", ""] + } else { + ["transmute(", ")"] + }; + let call = match (multi_calls.len(), para_num, fixed.len()) { + (0, 1, 0) => format!( + r#"pub unsafe fn {}{}(a: {}) -> {} {{ + {}{}(a) +}}"#, + name, const_declare, in_t[0], out_t, ext_c, current_fn, + ), + (0, 1, _) => { + let fixed: Vec = fixed.iter().take(type_len(in_t[0])).cloned().collect(); + format!( + r#"pub unsafe fn {}{}(a: {}) -> {} {{ + let b{}; + {}{}(a, transmute(b)) +}}"#, + name, + const_declare, + in_t[0], + out_t, + values(in_t[0], &fixed), + ext_c, + current_fn, ) + } + (0, 2, _) => format!( + r#"pub unsafe fn {}{}(a: {}, b: {}) -> {} {{ + {}{}(a, b) +}}"#, + name, const_declare, in_t[0], in_t[1], out_t, ext_c, current_fn, + ), + (0, 3, _) => format!( + r#"pub unsafe fn {}{}(a: {}, b: {}, c: {}) -> {} {{ + {}{}(a, b, c) +}}"#, + name, const_declare, in_t[0], in_t[1], in_t[2], out_t, ext_c, current_fn, + ), + (_, 1, _) => format!( + r#"pub unsafe fn {}{}(a: {}) -> {} {{ + {}{} +}}"#, + name, const_declare, in_t[0], out_t, ext_c, multi_calls, + ), + (_, 2, _) => format!( + r#"pub unsafe fn {}{}(a: {}, b: {}) -> {} {{ + {}{} +}}"#, + name, const_declare, in_t[0], in_t[1], out_t, ext_c, multi_calls, + ), + (_, 3, _) => format!( + r#"pub unsafe fn {}{}(a: {}, b: {}, c: {}) -> {} {{ + {}{} +}}"#, + name, const_declare, in_t[0], in_t[1], in_t[2], out_t, ext_c, multi_calls, + ), + (_, _, _) => String::new(), + }; + let call_arm = if let Some(const_arm) = const_arm { + let cnt = if const_arm.contains(':') { + let consts: Vec<_> = const_arm.split(':').map(|v| v.trim().to_string()).collect(); + consts[0].clone() } else { - String::new() + let const_arm = const_arm.replace("ttn", type_to_native_type(in_t[1])); + let mut cnt = String::from(in_t[1]); + cnt.push_str("("); + for i in 0..type_len(in_t[1]) { + if i != 0 { + cnt.push_str(", "); + } + cnt.push_str(&const_arm); + } + cnt.push_str(")"); + cnt }; + match para_num { + 1 => format!( + r#"pub unsafe fn {}{}(a: {}) -> {} {{ + {}{}{}(a, {}) +}}"#, + name, const_declare, in_t[0], out_t, multi_calls, ext_c_arm, current_fn, cnt + ), + 2 => format!( + r#"pub unsafe fn {}{}(a: {}, b:{}) -> {} {{ + {}{}{}(a, b, {}) +}}"#, + name, + const_declare, + in_t[0], + in_t[1], + out_t, + multi_calls, + ext_c_arm, + current_fn, + cnt + ), + _ => String::new(), + } + } else if out_t != link_arm_t[3] { + match para_num { + 1 => format!( + r#"pub unsafe fn {}{}(a: {}) -> {} {{ + {}{}{}{}(a){} +}}"#, + name, + const_declare, + in_t[0], + out_t, + multi_calls, + ext_c_arm, + trans[0], + current_fn, + trans[1] + ), + 2 => format!( + r#"pub unsafe fn {}{}(a: {}, b: {}) -> {} {{ + {}{}{}{}(transmute(a), transmute(b)){} +}}"#, + name, + const_declare, + in_t[0], + in_t[1], + out_t, + multi_calls, + ext_c_arm, + trans[0], + current_fn, + trans[1], + ), + _ => String::new(), + } + } else { + String::new() + }; + let call_aarch64 = if let Some(const_aarch64) = const_aarch64 { + match para_num { + 1 => format!( + r#"pub unsafe fn {}{}(a: {}) -> {} {{ + {}{}{}(a, {}) +}}"#, + name, + const_declare, + in_t[0], + out_t, + multi_calls, + ext_c_aarch64, + current_fn, + const_aarch64 + ), + 2 => format!( + r#"pub unsafe fn {}{}(a: {}, b: {}) -> {} {{ + {}{}{}(a, b, {}) +}}"#, + name, + const_declare, + in_t[0], + in_t[1], + out_t, + multi_calls, + ext_c_aarch64, + current_fn, + const_aarch64 + ), + _ => String::new(), + } + } else if out_t != link_aarch64_t[3] { + match para_num { + 1 => format!( + r#"pub unsafe fn {}{}(a: {}) -> {} {{ + {}{}{}{}(a){} +}}"#, + name, + const_declare, + in_t[0], + out_t, + multi_calls, + ext_c_aarch64, + trans[0], + current_fn, + trans[1], + ), + 2 => format!( + r#"pub unsafe fn {}{}(a: {}, b: {}) -> {} {{ + {}{}{}{}(a, b){} +}}"#, + name, + const_declare, + in_t[0], + in_t[1], + out_t, + multi_calls, + ext_c_aarch64, + trans[0], + current_fn, + trans[1], + ), + _ => String::new(), + } + } else { + String::new() + }; + let function = if (const_arm.is_some() && const_aarch64.is_some()) + || out_t != link_arm_t[3] + || out_t != link_aarch64_t[3] + { + format!( + r#" +{} +#[inline] +#[cfg(target_arch = "arm")] +#[target_feature(enable = "neon,{}")] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr({}{}))]{} +{} - let function = format!( - r#" {} #[inline] -#[target_feature(enable = "neon")] -#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr({}))] -#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr({}))] -pub unsafe fn {}(a: {}, b: {}) -> {} {{ - {}{}(a, b) -}} +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "{}")] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr({}{}))]{} +{} "#, - current_comment, - expand_intrinsic(¤t_arm, in_t), - expand_intrinsic(¤t_aarch64, in_t), - name, - in_t, + current_comment, + current_target_arm, + expand_intrinsic(¤t_arm, in_t[1]), + const_assert, + const_legacy, + call_arm, + current_comment, + current_target_aarch64, + expand_intrinsic(¤t_aarch64, in_t[1]), + const_assert, + const_legacy, + call_aarch64, + ) + } else { + format!( + r#" +{} +#[inline] +#[target_feature(enable = "{}")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "{}"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr({}{}))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr({}{}))]{} +{} +"#, + current_comment, + current_target_aarch64, + current_target_arm, + expand_intrinsic(¤t_arm, in_t[1]), + const_assert, + expand_intrinsic(¤t_aarch64, in_t[1]), + const_assert, + const_legacy, + call, + ) + }; + let test = gen_test( + &name, in_t, - out_t, - ext_c, - current_fn, + &out_t, + current_tests, + [type_len(in_t[0]), type_len(in_t[1]), type_len(in_t[2])], + type_len(out_t), + para_num, ); - let test = gen_test(name, &in_t, &out_t, current_tests, type_len(in_t)); (function, test) } @@ -499,6 +1918,10 @@ fn expand_intrinsic(intr: &str, t: &str) -> String { "float32x4_t" => "f32", "float64x1_t" => "f64", "float64x2_t" => "f64", + "poly8x8_t" => "i8", + "poly8x16_t" => "i8", + "poly16x4_t" => "i16", + "poly16x8_t" => "i16", /* "poly64x1_t" => "i64x1", "poly64x2_t" => "i64x2", @@ -524,6 +1947,10 @@ fn expand_intrinsic(intr: &str, t: &str) -> String { "uint32x4_t" => "u32", "uint64x1_t" => "u64", "uint64x2_t" => "u64", + "poly8x8_t" => "p8", + "poly8x16_t" => "p8", + "poly16x4_t" => "p16", + "poly16x8_t" => "p16", "float16x4_t" => "f16", "float16x8_t" => "f16", "float32x2_t" => "f32", @@ -537,11 +1964,419 @@ fn expand_intrinsic(intr: &str, t: &str) -> String { _ => panic!("unknown type for extension: {}", t), }; format!(r#""{}{}""#, &intr[..intr.len() - 1], ext) + } else if intr.ends_with(".l") { + let ext = match t { + "int8x8_t" => "8", + "int8x16_t" => "8", + "int16x4_t" => "16", + "int16x8_t" => "16", + "int32x2_t" => "32", + "int32x4_t" => "32", + "int64x1_t" => "64", + "int64x2_t" => "64", + "uint8x8_t" => "8", + "uint8x16_t" => "8", + "uint16x4_t" => "16", + "uint16x8_t" => "16", + "uint32x2_t" => "32", + "uint32x4_t" => "32", + "uint64x1_t" => "64", + "uint64x2_t" => "64", + "poly8x8_t" => "8", + "poly8x16_t" => "8", + "poly16x4_t" => "16", + "poly16x8_t" => "16", + "float16x4_t" => "16", + "float16x8_t" => "16", + "float32x2_t" => "32", + "float32x4_t" => "32", + "float64x1_t" => "64", + "float64x2_t" => "64", + "poly64x1_t" => "64", + "poly64x2_t" => "64", + _ => panic!("unknown type for extension: {}", t), + }; + format!(r#""{}{}""#, &intr[..intr.len() - 1], ext) } else { intr.to_string() } } +fn get_call( + in_str: &str, + current_name: &str, + const_declare: &str, + in_t: &[&str; 3], + out_t: &str, + fixed: &Vec, + n: Option, +) -> String { + let params: Vec<_> = in_str.split(',').map(|v| v.trim().to_string()).collect(); + assert!(params.len() > 0); + let mut fn_name = params[0].clone(); + if fn_name == "a" { + return String::from("a"); + } + if fn_name == "transpose-1-in_len" { + return transpose1(type_len(in_t[1])).to_string(); + } + if fn_name == "transpose-2-in_len" { + return transpose2(type_len(in_t[1])).to_string(); + } + if fn_name == "zip-1-in_len" { + return zip1(type_len(in_t[1])).to_string(); + } + if fn_name == "zip-2-in_len" { + return zip2(type_len(in_t[1])).to_string(); + } + if fn_name == "unzip-1-in_len" { + return unzip1(type_len(in_t[1])).to_string(); + } + if fn_name == "unzip-2-in_len" { + return unzip2(type_len(in_t[1])).to_string(); + } + if fn_name.starts_with("dup") { + let fn_format: Vec<_> = fn_name.split('-').map(|v| v.to_string()).collect(); + let len = match &*fn_format[1] { + "out_len" => type_len(out_t), + "in_len" => type_len(in_t[1]), + "in0_len" => type_len(in_t[0]), + "halflen" => type_len(in_t[1]) / 2, + _ => 0, + }; + let mut s = format!("{} [", const_declare); + for i in 0..len { + if i != 0 { + s.push_str(", "); + } + s.push_str(&fn_format[2]); + } + s.push_str("]"); + return s; + } + if fn_name.starts_with("asc") { + let fn_format: Vec<_> = fn_name.split('-').map(|v| v.to_string()).collect(); + let start = match &*fn_format[1] { + "0" => 0, + "n" => n.unwrap(), + "out_len" => type_len(out_t) as i32, + "halflen" => (type_len(in_t[1]) / 2) as i32, + s => s.parse::().unwrap(), + }; + let len = match &*fn_format[2] { + "out_len" => type_len(out_t), + "in_len" => type_len(in_t[1]), + "in0_len" => type_len(in_t[0]), + "halflen" => type_len(in_t[1]) / 2, + _ => 0, + }; + return asc(start, len); + } + if fn_name.starts_with("ins") { + let fn_format: Vec<_> = fn_name.split('-').map(|v| v.to_string()).collect(); + let n = n.unwrap(); + let len = match &*fn_format[1] { + "out_len" => type_len(out_t), + "in_len" => type_len(in_t[1]), + "in0_len" => type_len(in_t[0]), + _ => 0, + }; + let offset = match &*fn_format[2] { + "out_len" => type_len(out_t), + "in_len" => type_len(in_t[1]), + "in0_len" => type_len(in_t[0]), + _ => 0, + }; + let mut s = format!("{} [", const_declare); + for i in 0..len { + if i != 0 { + s.push_str(", "); + } + if i == n as usize { + s.push_str(&format!("{} + {} as u32", offset.to_string(), fn_format[3])); + } else { + s.push_str(&i.to_string()); + } + } + s.push_str("]"); + return s; + } + if fn_name.starts_with("static_assert_imm") { + let fn_format: Vec<_> = fn_name.split('-').map(|v| v.to_string()).collect(); + let len = match &*fn_format[1] { + "out_exp_len" => type_exp_len(out_t), + "out_bits_exp_len" => type_bits_exp_len(out_t), + "in_exp_len" => type_exp_len(in_t[1]), + "in_bits_exp_len" => type_bits_exp_len(in_t[1]), + "in0_exp_len" => type_exp_len(in_t[0]), + "in1_exp_len" => type_exp_len(in_t[1]), + "in2_exp_len" => type_exp_len(in_t[2]), + _ => 0, + }; + if len == 0 { + return format!( + r#"static_assert!({} : i32 where {} == 0);"#, + fn_format[2], fn_format[2] + ); + } else { + return format!(r#"static_assert_imm{}!({});"#, len, fn_format[2]); + } + } + if fn_name.starts_with("static_assert") { + let fn_format: Vec<_> = fn_name.split('-').map(|v| v.to_string()).collect(); + let lim1 = if fn_format[2] == "bits" { + type_bits(in_t[1]).to_string() + } else if fn_format[2] == "halfbits" { + (type_bits(in_t[1]) / 2).to_string() + } else { + fn_format[2].clone() + }; + let lim2 = if fn_format[3] == "bits" { + type_bits(in_t[1]).to_string() + } else if fn_format[3] == "halfbits" { + (type_bits(in_t[1]) / 2).to_string() + } else { + fn_format[3].clone() + }; + if lim1 == lim2 { + return format!( + r#"static_assert!({} : i32 where {} == {});"#, + fn_format[1], fn_format[1], lim1 + ); + } else { + return format!( + r#"static_assert!({} : i32 where {} >= {} && {} <= {});"#, + fn_format[1], fn_format[1], lim1, fn_format[1], lim2 + ); + } + } + if fn_name.starts_with("matchn") { + let fn_format: Vec<_> = fn_name.split('-').map(|v| v.to_string()).collect(); + let len = match &*fn_format[1] { + "out_exp_len" => type_exp_len(out_t), + "in_exp_len" => type_exp_len(in_t[1]), + "in0_exp_len" => type_exp_len(in_t[0]), + _ => 0, + }; + let mut call = format!("match {} & 0b{} {{\n", &fn_format[2], "1".repeat(len)); + let mut sub_call = String::new(); + for p in 1..params.len() { + if !sub_call.is_empty() { + sub_call.push_str(", "); + } + sub_call.push_str(¶ms[p]); + } + for i in 0..(2u32.pow(len as u32) as usize) { + let sub_match = format!( + " {} => {},\n", + i, + get_call( + &sub_call, + current_name, + const_declare, + in_t, + out_t, + fixed, + Some(i as i32) + ) + ); + call.push_str(&sub_match); + } + call.push_str(" _ => unreachable_unchecked(),\n }"); + return call; + } + let mut re: Option<(String, String)> = None; + let mut param_str = String::new(); + let mut i = 1; + while i < params.len() { + let s = ¶ms[i]; + if s.starts_with('{') { + let mut sub_fn = String::new(); + let mut paranthes = 0; + while i < params.len() { + if !sub_fn.is_empty() { + sub_fn.push_str(", "); + } + sub_fn.push_str(¶ms[i]); + let l = params[i].len(); + for j in 0..l { + if ¶ms[i][j..j + 1] == "{" { + paranthes += 1; + } else { + break; + } + } + for j in 0..l { + if ¶ms[i][l - j - 1..l - j] == "}" { + paranthes -= 1; + } else { + break; + } + } + if paranthes == 0 { + break; + } + i += 1; + } + let sub_call = get_call( + &sub_fn[1..sub_fn.len() - 1], + current_name, + const_declare, + in_t, + out_t, + fixed, + n.clone(), + ); + if !param_str.is_empty() { + param_str.push_str(", "); + } + param_str.push_str(&sub_call); + } else if s.contains(':') { + let re_params: Vec<_> = s.split(':').map(|v| v.to_string()).collect(); + if re_params[1] == "" { + re = Some((re_params[0].clone(), in_t[1].to_string())); + } else if re_params[1] == "in_t" { + re = Some((re_params[0].clone(), in_t[1].to_string())); + } else if re_params[1] == "in_t0" { + re = Some((re_params[0].clone(), in_t[0].to_string())); + } else if re_params[1] == "in_t1" { + re = Some((re_params[0].clone(), in_t[1].to_string())); + } else if re_params[1] == "out_t" { + re = Some((re_params[0].clone(), out_t.to_string())); + } else if re_params[1] == "half" { + re = Some((re_params[0].clone(), type_to_half(in_t[1]).to_string())); + } else if re_params[1] == "in_ntt" { + re = Some(( + re_params[0].clone(), + native_type_to_type(in_t[1]).to_string(), + )); + } else if re_params[1] == "in_long_ntt" { + re = Some(( + re_params[0].clone(), + native_type_to_long_type(in_t[1]).to_string(), + )); + } else if re_params[1] == "out_ntt" { + re = Some((re_params[0].clone(), native_type_to_type(out_t).to_string())); + } else { + re = Some((re_params[0].clone(), re_params[1].clone())); + } + } else { + if !param_str.is_empty() { + param_str.push_str(", "); + } + param_str.push_str(s); + } + i += 1; + } + if fn_name == "fixed" { + let (re_name, re_type) = re.unwrap(); + let fixed: Vec = fixed.iter().take(type_len(in_t[1])).cloned().collect(); + return format!(r#"let {}{};"#, re_name, values(&re_type, &fixed)); + } + if fn_name == "fixed-half-right" { + let fixed: Vec = fixed.iter().take(type_len(in_t[1])).cloned().collect(); + let half = fixed[type_len(in_t[1]) / 2..] + .iter() + .fold(String::new(), |mut s, fix| { + s.push_str(fix); + s.push_str(", "); + s + }); + return format!(r#"[{}]"#, &half[..half.len() - 2]); + } + if fn_name.contains('-') { + let fn_format: Vec<_> = fn_name.split('-').map(|v| v.to_string()).collect(); + assert_eq!(fn_format.len(), 3); + fn_name = if fn_format[0] == "self" { + current_name.to_string() + } else { + fn_format[0].clone() + }; + if fn_format[1] == "self" { + fn_name.push_str(type_to_suffix(in_t[1])); + } else if fn_format[1] == "nself" { + fn_name.push_str(type_to_n_suffix(in_t[1])); + } else if fn_format[1] == "out" { + fn_name.push_str(type_to_suffix(out_t)); + } else if fn_format[1] == "in0" { + fn_name.push_str(type_to_suffix(in_t[0])); + } else if fn_format[1] == "in2" { + fn_name.push_str(type_to_suffix(in_t[2])); + } else if fn_format[1] == "in2lane" { + fn_name.push_str(&type_to_lane_suffixes(out_t, in_t[2])); + } else if fn_format[1] == "signed" { + fn_name.push_str(type_to_suffix(type_to_signed(in_t[1]))); + } else if fn_format[1] == "unsigned" { + fn_name.push_str(type_to_suffix(type_to_unsigned(in_t[1]))); + } else if fn_format[1] == "doubleself" { + fn_name.push_str(&type_to_double_suffixes(out_t, in_t[1])); + } else if fn_format[1] == "noq_doubleself" { + fn_name.push_str(&type_to_noq_double_suffixes(out_t, in_t[1])); + } else if fn_format[1] == "noqself" { + fn_name.push_str(type_to_noq_suffix(in_t[1])); + } else if fn_format[1] == "noqsigned" { + fn_name.push_str(type_to_noq_suffix(type_to_signed(in_t[1]))); + } else if fn_format[1] == "nosuffix" { + } else if fn_format[1] == "in_len" { + fn_name.push_str(&type_len(in_t[1]).to_string()); + } else if fn_format[1] == "in0_len" { + fn_name.push_str(&type_len(in_t[0]).to_string()); + } else if fn_format[1] == "out_len" { + fn_name.push_str(&type_len(out_t).to_string()); + } else if fn_format[1] == "halflen" { + fn_name.push_str(&(type_len(in_t[1]) / 2).to_string()); + } else if fn_format[1] == "nout" { + fn_name.push_str(type_to_n_suffix(out_t)); + } else if fn_format[1] == "nin0" { + fn_name.push_str(type_to_n_suffix(in_t[0])); + } else if fn_format[1] == "nsigned" { + fn_name.push_str(type_to_n_suffix(type_to_signed(in_t[1]))); + } else if fn_format[1] == "in_ntt" { + fn_name.push_str(type_to_suffix(native_type_to_type(in_t[1]))); + } else if fn_format[1] == "out_ntt" { + fn_name.push_str(type_to_suffix(native_type_to_type(out_t))); + } else { + fn_name.push_str(&fn_format[1]); + }; + if fn_format[2] == "ext" { + fn_name.push_str("_"); + } else if fn_format[2] == "noext" { + } else if fn_format[2].starts_with("<") { + assert!(fn_format[2].ends_with(">")); + let types: Vec<_> = fn_format[2][1..fn_format[2].len() - 1] + .split(' ') + .map(|v| v.to_string()) + .collect(); + assert_eq!(types.len(), 2); + let type1 = if types[0] == "element_t" { + type_to_native_type(in_t[1]) + } else { + &types[0] + }; + let type2 = if types[1] == "element_t" { + type_to_native_type(in_t[1]) + } else { + &types[1] + }; + fn_name.push_str(&format!("::<{}, {}>", type1, type2)); + } else { + fn_name.push_str(&fn_format[2]); + } + } + if param_str.is_empty() { + return fn_name.replace("out_t", out_t); + } + let fn_str = if let Some((re_name, re_type)) = re.clone() { + format!( + r#"let {}: {} = {}({});"#, + re_name, re_type, fn_name, param_str + ) + } else { + format!(r#"{}({})"#, fn_name, param_str) + }; + return fn_str; +} + fn main() -> io::Result<()> { let args: Vec = env::args().collect(); let in_file = args.get(1).cloned().unwrap_or_else(|| IN.to_string()); @@ -556,9 +2391,25 @@ fn main() -> io::Result<()> { let mut current_aarch64: Option = None; let mut link_arm: Option = None; let mut link_aarch64: Option = None; + let mut const_arm: Option = None; + let mut const_aarch64: Option = None; + let mut constn: Option = None; + let mut para_num = 2; + let mut suffix: Suffix = Normal; let mut a: Vec = Vec::new(); let mut b: Vec = Vec::new(); - let mut current_tests: Vec<(Vec, Vec, Vec)> = Vec::new(); + let mut c: Vec = Vec::new(); + let mut n: Option = None; + let mut fixed: Vec = Vec::new(); + let mut current_tests: Vec<( + Vec, + Vec, + Vec, + Option, + Vec, + )> = Vec::new(); + let mut multi_fn: Vec = Vec::new(); + let mut target: TargetFeature = Default; // // THIS FILE IS GENERATED FORM neon.spec DO NOT CHANGE IT MANUALLY @@ -627,27 +2478,85 @@ mod test { current_aarch64 = None; link_aarch64 = None; link_arm = None; + const_aarch64 = None; + const_arm = None; current_tests = Vec::new(); + constn = None; + para_num = 2; + suffix = Normal; + a = Vec::new(); + b = Vec::new(); + c = Vec::new(); + fixed = Vec::new(); + n = None; + multi_fn = Vec::new(); + target = Default; } else if line.starts_with("//") { } else if line.starts_with("name = ") { current_name = Some(String::from(&line[7..])); } else if line.starts_with("fn = ") { current_fn = Some(String::from(&line[5..])); + } else if line.starts_with("multi_fn = ") { + multi_fn.push(String::from(&line[11..])); + } else if line.starts_with("constn = ") { + constn = Some(String::from(&line[9..])); } else if line.starts_with("arm = ") { current_arm = Some(String::from(&line[6..])); } else if line.starts_with("aarch64 = ") { current_aarch64 = Some(String::from(&line[10..])); + } else if line.starts_with("double-suffixes") { + suffix = Double; + } else if line.starts_with("no-q") { + suffix = NoQ; + } else if line.starts_with("noq-double-suffixes") { + suffix = NoQDouble; + } else if line.starts_with("n-suffix") { + suffix = NSuffix; + } else if line.starts_with("double-n-suffixes") { + suffix = DoubleN; + } else if line.starts_with("out-n-suffix") { + suffix = OutNSuffix; + } else if line.starts_with("noq-n-suffix") { + suffix = NoQNSuffix; + } else if line.starts_with("out-suffix") { + suffix = OutSuffix; + } else if line.starts_with("lane-suffixes") { + suffix = Lane; + } else if line.starts_with("in2-suffix") { + suffix = In2; + } else if line.starts_with("in2-lane-suffixes") { + suffix = In2Lane; } else if line.starts_with("a = ") { a = line[4..].split(',').map(|v| v.trim().to_string()).collect(); } else if line.starts_with("b = ") { b = line[4..].split(',').map(|v| v.trim().to_string()).collect(); + } else if line.starts_with("c = ") { + c = line[4..].split(',').map(|v| v.trim().to_string()).collect(); + } else if line.starts_with("n = ") { + n = Some(String::from(&line[4..])); + } else if line.starts_with("fixed = ") { + fixed = line[8..].split(',').map(|v| v.trim().to_string()).collect(); } else if line.starts_with("validate ") { let e = line[9..].split(',').map(|v| v.trim().to_string()).collect(); - current_tests.push((a.clone(), b.clone(), e)); + current_tests.push((a.clone(), b.clone(), c.clone(), n.clone(), e)); } else if line.starts_with("link-aarch64 = ") { link_aarch64 = Some(String::from(&line[15..])); + } else if line.starts_with("const-aarch64 = ") { + const_aarch64 = Some(String::from(&line[16..])); } else if line.starts_with("link-arm = ") { link_arm = Some(String::from(&line[11..])); + } else if line.starts_with("const-arm = ") { + const_arm = Some(String::from(&line[12..])); + } else if line.starts_with("target = ") { + target = match Some(String::from(&line[9..])) { + Some(input) => match input.as_str() { + "v7" => ArmV7, + "fp-armv8" => FPArmV8, + "crypto" => Crypto, + _ => Default, + }, + _ => Default, + } } else if line.starts_with("generate ") { let line = &line[9..]; let types: Vec = line @@ -666,32 +2575,49 @@ mod test { for line in types { let spec: Vec<&str> = line.split(':').map(|e| e.trim()).collect(); - let in_t; + let in_t: [&str; 3]; let out_t; if spec.len() == 1 { - in_t = spec[0]; + in_t = [spec[0], spec[0], spec[0]]; out_t = spec[0]; } else if spec.len() == 2 { - in_t = spec[0]; + in_t = [spec[0], spec[0], spec[0]]; out_t = spec[1]; + } else if spec.len() == 3 { + in_t = [spec[0], spec[1], spec[1]]; + out_t = spec[2]; + } else if spec.len() == 4 { + in_t = [spec[0], spec[1], spec[2]]; + out_t = spec[3]; } else { panic!("Bad spec: {}", line) } + if b.len() == 0 { + para_num = 1; + } else if c.len() != 0 { + para_num = 3; + } let current_name = current_name.clone().unwrap(); - let name = format!("{}{}", current_name, type_to_suffix(in_t),); - if let Some(current_arm) = current_arm.clone() { let (function, test) = gen_arm( ¤t_comment, ¤t_fn, - &name, + ¤t_name, ¤t_arm, &link_arm, ¤t_aarch64, &link_aarch64, + &const_arm, + &const_aarch64, + &constn, &in_t, &out_t, ¤t_tests, + suffix, + para_num, + target, + &fixed, + &multi_fn, ); out_arm.push_str(&function); tests_arm.push_str(&test); @@ -699,12 +2625,19 @@ mod test { let (function, test) = gen_aarch64( ¤t_comment, ¤t_fn, - &name, + ¤t_name, ¤t_aarch64, &link_aarch64, + &const_aarch64, + &constn, &in_t, &out_t, ¤t_tests, + suffix, + para_num, + target, + &fixed, + &multi_fn, ); out_aarch64.push_str(&function); tests_aarch64.push_str(&test); @@ -719,7 +2652,7 @@ mod test { let arm_out_path: PathBuf = PathBuf::from(env::var("OUT_DIR").unwrap()) .join("src") - .join("arm") + .join("arm_shared") .join("neon"); std::fs::create_dir_all(&arm_out_path)?; diff --git a/library/stdarch/crates/stdarch-test/Cargo.toml b/library/stdarch/crates/stdarch-test/Cargo.toml index 9eb6b64d16..9ac1057be7 100644 --- a/library/stdarch/crates/stdarch-test/Cargo.toml +++ b/library/stdarch/crates/stdarch-test/Cargo.toml @@ -2,6 +2,7 @@ name = "stdarch-test" version = "0.1.0" authors = ["Alex Crichton "] +edition = "2018" [dependencies] assert-instr-macro = { path = "../assert-instr-macro" } @@ -17,7 +18,7 @@ cfg-if = "0.1" # time, and we want to make updates to this explicit rather than automatically # picking up updates which might break CI with new instruction names. [target.'cfg(target_arch = "wasm32")'.dependencies] -wasmprinter = "=0.2.6" +wasmprinter = "=0.2.24" [features] default = [] diff --git a/library/stdarch/crates/stdarch-test/src/lib.rs b/library/stdarch/crates/stdarch-test/src/lib.rs index 25852c9479..8ea14f0f7f 100644 --- a/library/stdarch/crates/stdarch-test/src/lib.rs +++ b/library/stdarch/crates/stdarch-test/src/lib.rs @@ -3,14 +3,12 @@ //! This basically just disassembles the current executable and then parses the //! output once globally and then provides the `assert` function which makes //! assertions about the disassembly of a function. -#![feature(test)] // For black_box +#![feature(bench_black_box)] // For black_box #![deny(rust_2018_idioms)] #![allow(clippy::missing_docs_in_private_items, clippy::print_stdout)] -extern crate assert_instr_macro; #[macro_use] extern crate lazy_static; -extern crate simd_test_macro; #[macro_use] extern crate cfg_if; @@ -24,7 +22,7 @@ cfg_if! { use wasm::disassemble_myself; } else { mod disassembly; - use disassembly::disassemble_myself; + use crate::disassembly::disassemble_myself; } } @@ -78,8 +76,12 @@ pub fn assert(shim_addr: usize, fnname: &str, expected: &str) { instrs = &instrs[..instrs.len() - 1]; } - // If the expected intrinsic is a nop it is compiled away so we - // can't check for it - aka the intrinsic is not generating any code + // There are two cases when the expected instruction is nop: + // 1. The expected intrinsic is compiled away so we can't + // check for it - aka the intrinsic is not generating any code. + // 2. It is a mark, indicating that the instruction will be + // compiled into other instructions - mainly because of llvm + // optimization. if expected == "nop" { return; } @@ -118,9 +120,9 @@ pub fn assert(shim_addr: usize, fnname: &str, expected: &str) { // Intrinsics using `cvtpi2ps` are typically "composites" and // in some cases exceed the limit. "cvtpi2ps" => 25, - - // core_arch/src/acle/simd32 - "usad8" => 27, + // core_arch/src/arm_shared/simd32 + // vfmaq_n_f32_vfma : #instructions = 26 >= 22 (limit) + "usad8" | "vfma" | "vfms" => 27, "qadd8" | "qsub8" | "sadd8" | "sel" | "shadd8" | "shsub8" | "usub8" | "ssub8" => 29, // Original limit was 20 instructions, but ARM DSP Intrinsics diff --git a/library/stdarch/crates/stdarch-verify/src/lib.rs b/library/stdarch/crates/stdarch-verify/src/lib.rs index 44e6fdc1fd..e85f0489a8 100644 --- a/library/stdarch/crates/stdarch-verify/src/lib.rs +++ b/library/stdarch/crates/stdarch-verify/src/lib.rs @@ -80,6 +80,7 @@ fn functions(input: TokenStream, dirs: &[&str]) -> TokenStream { let name = &f.sig.ident; // println!("{}", name); let mut arguments = Vec::new(); + let mut const_arguments = Vec::new(); for input in f.sig.inputs.iter() { let ty = match *input { syn::FnArg::Typed(ref c) => &c.ty, @@ -87,6 +88,13 @@ fn functions(input: TokenStream, dirs: &[&str]) -> TokenStream { }; arguments.push(to_type(ty)); } + for generic in f.sig.generics.params.iter() { + let ty = match *generic { + syn::GenericParam::Const(ref c) => &c.ty, + _ => panic!("invalid generic argument on {}", name), + }; + const_arguments.push(to_type(ty)); + } let ret = match f.sig.output { syn::ReturnType::Default => quote! { None }, syn::ReturnType::Type(_, ref t) => { @@ -100,7 +108,32 @@ fn functions(input: TokenStream, dirs: &[&str]) -> TokenStream { } else { quote! { None } }; - let required_const = find_required_const(&f.attrs); + + let required_const = find_required_const("rustc_args_required_const", &f.attrs); + let mut legacy_const_generics = + find_required_const("rustc_legacy_const_generics", &f.attrs); + if !required_const.is_empty() && !legacy_const_generics.is_empty() { + panic!( + "Can't have both #[rustc_args_required_const] and \ + #[rustc_legacy_const_generics]" + ); + } + + // The list of required consts, used to verify the arguments, comes from either the + // `rustc_args_required_const` or the `rustc_legacy_const_generics` attribute. + let required_const = if required_const.is_empty() { + legacy_const_generics.clone() + } else { + required_const + }; + + legacy_const_generics.sort(); + for (idx, ty) in legacy_const_generics + .into_iter() + .zip(const_arguments.into_iter()) + { + arguments.insert(idx, ty); + } // strip leading underscore from fn name when building a test // _mm_foo -> mm_foo such that the test name is test_mm_foo. @@ -191,6 +224,7 @@ fn to_type(t: &syn::Type) -> proc_macro2::TokenStream { "int64x1_t" => quote! { &I64X1 }, "int64x2_t" => quote! { &I64X2 }, "uint8x8_t" => quote! { &U8X8 }, + "uint8x4_t" => quote! { &U8X4 }, "uint8x8x2_t" => quote! { &U8X8X2 }, "uint8x16x2_t" => quote! { &U8X16X2 }, "uint8x16x3_t" => quote! { &U8X16X3 }, @@ -389,11 +423,11 @@ fn find_target_feature(attrs: &[syn::Attribute]) -> Option { }) } -fn find_required_const(attrs: &[syn::Attribute]) -> Vec { +fn find_required_const(name: &str, attrs: &[syn::Attribute]) -> Vec { attrs .iter() .flat_map(|a| { - if a.path.segments[0].ident == "rustc_args_required_const" { + if a.path.segments[0].ident == name { syn::parse::(a.tokens.clone().into()) .unwrap() .args diff --git a/library/stdarch/crates/stdarch-verify/tests/arm.rs b/library/stdarch/crates/stdarch-verify/tests/arm.rs index 597645489c..da17eb313c 100644 --- a/library/stdarch/crates/stdarch-verify/tests/arm.rs +++ b/library/stdarch/crates/stdarch-verify/tests/arm.rs @@ -149,6 +149,7 @@ static U8X16X2: Type = Type::U(8, 16, 2); static U8X16X3: Type = Type::U(8, 16, 3); static U8X16X4: Type = Type::U(8, 16, 4); static U8X8: Type = Type::U(8, 8, 1); +static U8X4: Type = Type::U(8, 4, 1); static U8X8X2: Type = Type::U(8, 8, 2); static U8X8X3: Type = Type::U(8, 8, 3); static U8X8X4: Type = Type::U(8, 8, 4); @@ -371,6 +372,112 @@ fn verify_all_signatures() { "vsriq_n_p8", "vsri_n_p16", "vsriq_n_p16", + "__smulbb", + "__smultb", + "__smulbt", + "__smultt", + "__smulwb", + "__smulwt", + "__qadd", + "__qsub", + "__qdbl", + "__smlabb", + "__smlabt", + "__smlatb", + "__smlatt", + "__smlawb", + "__smlawt", + "__qadd8", + "__qsub8", + "__qsub16", + "__qadd16", + "__qasx", + "__qsax", + "__sadd16", + "__sadd8", + "__smlad", + "__smlsd", + "__sasx", + "__sel", + "__shadd8", + "__shadd16", + "__shsub8", + "__usub8", + "__ssub8", + "__shsub16", + "__smuad", + "__smuadx", + "__smusd", + "__smusdx", + "__usad8", + "__usada8", + "vld1_s8", + "vld1q_s8", + "vld1q_s8", + "vld1_s16", + "vld1q_s16", + "vld1_s32", + "vld1q_s32", + "vld1_s64", + "vld1q_s64", + "vld1_u8", + "vld1q_u8", + "vld1_u16", + "vld1q_u16", + "vld1_u32", + "vld1q_u32", + "vld1_u64", + "vld1q_u64", + "vld1_p8", + "vld1q_p8", + "vld1_p16", + "vld1q_p16", + "vld1_f32", + "vld1q_f32", + "vld1_f64", + "vld1q_f64", + "vst1_s8", + "vst1q_s8", + "vst1_s16", + "vst1q_s16", + "vst1_s32", + "vst1q_s32", + "vst1_s64", + "vst1q_s64", + "vst1_u8", + "vst1q_u8", + "vst1_u16", + "vst1q_u16", + "vst1_u32", + "vst1q_u32", + "vst1_u64", + "vst1q_u64", + "vst1_p8", + "vst1q_p8", + "vst1_p16", + "vst1q_p16", + "vst1_f32", + "vst1q_f32", + "vpadal_s8", + "vpadal_s16", + "vpadal_s32", + "vpadalq_s8", + "vpadalq_s16", + "vpadalq_s32", + "vpadal_u8", + "vpadal_u16", + "vpadal_u32", + "vpadalq_u8", + "vpadalq_u16", + "vpadalq_u32", + "__ldrex", + "__strex", + "__ldrexb", + "__strexb", + "__ldrexh", + "__strexh", + "__clrex", + "__dbg", ]; if !skip.contains(&rust.name) { println!( @@ -394,6 +501,28 @@ fn verify_all_signatures() { "brk" | "__breakpoint" | "udf" | "_prefetch" => continue, _ => {} } + // Skip some intrinsics that are present in GCC and Clang but + // are missing from the official documentation. + let skip_intrinsic_verify = [ + "vmov_n_p64", + "vmovq_n_p64", + "vreinterpret_p64_s64", + "vreinterpret_f32_p64", + "vreinterpretq_f32_p64", + "vqrdmlahh_s16", + "vqrdmlahs_s32", + "vqrdmlahh_lane_s16", + "vqrdmlahh_laneq_s16", + "vqrdmlahs_lane_s32", + "vqrdmlahs_laneq_s32", + "vqrdmlshh_s16", + "vqrdmlshs_s32", + "vqrdmlshh_lane_s16", + "vqrdmlshh_laneq_s16", + "vqrdmlshs_lane_s32", + "vqrdmlshs_laneq_s32", + "__dbg", + ]; let arm = match map.get(rust.name) { Some(i) => i, None => { @@ -403,11 +532,14 @@ fn verify_all_signatures() { // TODO: we still need to verify these intrinsics or find a // reference for them, need to figure out where though! if !rust.file.ends_with("dsp.rs\"") + && !rust.file.ends_with("simd32.rs\"") && !rust.file.ends_with("cmsis.rs\"") && !rust.file.ends_with("v6.rs\"") && !rust.file.ends_with("v7.rs\"") && !rust.file.ends_with("v8.rs\"") && !rust.file.ends_with("tme.rs\"") + && !rust.file.ends_with("ex.rs\"") + && !skip_intrinsic_verify.contains(&rust.name) { println!( "missing arm definition for {:?} in {}", diff --git a/library/stdarch/crates/stdarch-verify/tests/mips.rs b/library/stdarch/crates/stdarch-verify/tests/mips.rs index 283d5d74b5..1eb86dc29c 100644 --- a/library/stdarch/crates/stdarch-verify/tests/mips.rs +++ b/library/stdarch/crates/stdarch-verify/tests/mips.rs @@ -1,5 +1,4 @@ //! Verification of MIPS MSA intrinsics -#![feature(try_trait)] #![allow(bad_style, unused)] // This file is obtained from @@ -139,47 +138,53 @@ struct MsaIntrinsic { instruction: String, } +struct NoneError; + impl std::convert::TryFrom<&'static str> for MsaIntrinsic { // The intrinsics are just C function declarations of the form: // $ret_ty __builtin_${fn_id}($($arg_ty),*); - type Error = std::option::NoneError; + type Error = NoneError; fn try_from(line: &'static str) -> Result { - let first_whitespace = line.find(char::is_whitespace)?; - let ret_ty = &line[0..first_whitespace]; - let ret_ty = MsaTy::from(ret_ty); + return inner(line).ok_or(NoneError); - let first_parentheses = line.find('(')?; - assert!(first_parentheses > first_whitespace); - let id = &line[first_whitespace + 1..first_parentheses].trim(); - assert!(id.starts_with("__builtin")); - let mut id_str = "_".to_string(); - id_str += &id[9..]; - let id = id_str; + fn inner(line: &'static str) -> Option { + let first_whitespace = line.find(char::is_whitespace)?; + let ret_ty = &line[0..first_whitespace]; + let ret_ty = MsaTy::from(ret_ty); - let mut arg_tys = Vec::new(); + let first_parentheses = line.find('(')?; + assert!(first_parentheses > first_whitespace); + let id = &line[first_whitespace + 1..first_parentheses].trim(); + assert!(id.starts_with("__builtin")); + let mut id_str = "_".to_string(); + id_str += &id[9..]; + let id = id_str; - let last_parentheses = line.find(')')?; - for arg in (&line[first_parentheses + 1..last_parentheses]).split(',') { - let arg = arg.trim(); - arg_tys.push(MsaTy::from(arg)); - } + let mut arg_tys = Vec::new(); - // The instruction is the intrinsic name without the __msa_ prefix. - let instruction = &id[6..]; - let mut instruction = instruction.to_string(); - // With all underscores but the first one replaced with a `.` - if let Some(first_underscore) = instruction.find('_') { - let postfix = instruction[first_underscore + 1..].replace('_', "."); - instruction = instruction[0..=first_underscore].to_string(); - instruction += &postfix; - } + let last_parentheses = line.find(')')?; + for arg in (&line[first_parentheses + 1..last_parentheses]).split(',') { + let arg = arg.trim(); + arg_tys.push(MsaTy::from(arg)); + } - Ok(MsaIntrinsic { - id, - ret_ty, - arg_tys, - instruction, - }) + // The instruction is the intrinsic name without the __msa_ prefix. + let instruction = &id[6..]; + let mut instruction = instruction.to_string(); + // With all underscores but the first one replaced with a `.` + if let Some(first_underscore) = instruction.find('_') { + let postfix = instruction[first_underscore + 1..].replace('_', "."); + instruction = instruction[0..=first_underscore].to_string(); + instruction += &postfix; + } + + Some(MsaIntrinsic { + id, + ret_ty, + arg_tys, + instruction, + }) + } } } diff --git a/library/stdarch/crates/stdarch-verify/tests/x86-intel.rs b/library/stdarch/crates/stdarch-verify/tests/x86-intel.rs index 01a245c54c..89494bfd2a 100644 --- a/library/stdarch/crates/stdarch-verify/tests/x86-intel.rs +++ b/library/stdarch/crates/stdarch-verify/tests/x86-intel.rs @@ -128,6 +128,8 @@ struct Intrinsic { struct Parameter { #[serde(rename = "type")] type_: String, + #[serde(default)] + etype: String, } #[derive(Deserialize)] @@ -548,7 +550,7 @@ fn matches(rust: &Function, intel: &Intrinsic) -> Result<(), String> { // Make sure we've got the right return type. if let Some(t) = rust.ret { - equate(t, &intel.return_.type_, rust.name, false)?; + equate(t, &intel.return_.type_, "", rust.name, false)?; } else if intel.return_.type_ != "" && intel.return_.type_ != "void" { bail!( "{} returns `{}` with intel, void in rust", @@ -570,7 +572,7 @@ fn matches(rust: &Function, intel: &Intrinsic) -> Result<(), String> { } for (i, (a, b)) in intel.parameters.iter().zip(rust.arguments).enumerate() { let is_const = rust.required_const.contains(&i); - equate(b, &a.type_, &intel.name, is_const)?; + equate(b, &a.type_, &a.etype, &intel.name, is_const)?; } } @@ -669,7 +671,13 @@ fn matches(rust: &Function, intel: &Intrinsic) -> Result<(), String> { Ok(()) } -fn equate(t: &Type, intel: &str, intrinsic: &str, is_const: bool) -> Result<(), String> { +fn equate( + t: &Type, + intel: &str, + etype: &str, + intrinsic: &str, + is_const: bool, +) -> Result<(), String> { // Make pointer adjacent to the type: float * foo => float* foo let mut intel = intel.replace(" *", "*"); // Make mutability modifier adjacent to the pointer: @@ -681,19 +689,26 @@ fn equate(t: &Type, intel: &str, intrinsic: &str, is_const: bool) -> Result<(), intel = intel.replace("const ", ""); intel = intel.replace("*", " const*"); } - let require_const = || { + if etype == "IMM" { + // The _bittest intrinsics claim to only accept immediates but actually + // accept run-time values as well. + if !is_const && !intrinsic.starts_with("_bittest") { + return bail!("argument required to be const but isn't"); + } + } else { + // const int must be an IMM + assert_ne!(intel, "const int"); if is_const { - return Ok(()); + return bail!("argument is const but shouldn't be"); } - Err(format!("argument required to be const but isn't")) - }; + } match (t, &intel[..]) { (&Type::PrimFloat(32), "float") => {} (&Type::PrimFloat(64), "double") => {} (&Type::PrimSigned(16), "__int16") => {} (&Type::PrimSigned(16), "short") => {} (&Type::PrimSigned(32), "__int32") => {} - (&Type::PrimSigned(32), "const int") => require_const()?, + (&Type::PrimSigned(32), "const int") => {} (&Type::PrimSigned(32), "int") => {} (&Type::PrimSigned(64), "__int64") => {} (&Type::PrimSigned(64), "long long") => {} diff --git a/library/stdarch/examples/hex.rs b/library/stdarch/examples/hex.rs index 5269958a4e..812836d661 100644 --- a/library/stdarch/examples/hex.rs +++ b/library/stdarch/examples/hex.rs @@ -14,7 +14,6 @@ #![feature(stdsimd, wasm_target_feature)] #![cfg_attr(test, feature(test))] -#![cfg_attr(target_arch = "wasm32", feature(wasm_simd))] #![allow( clippy::unwrap_used, clippy::print_stdout, @@ -159,34 +158,34 @@ unsafe fn hex_encode_sse41<'a>(mut src: &[u8], dst: &'a mut [u8]) -> Result<&'a unsafe fn hex_encode_simd128<'a>(mut src: &[u8], dst: &'a mut [u8]) -> Result<&'a str, usize> { use core_arch::arch::wasm32::*; - let ascii_zero = i8x16_splat(b'0' as i8); - let nines = i8x16_splat(9); - let ascii_a = i8x16_splat((b'a' - 9 - 1) as i8); - let and4bits = i8x16_splat(0xf); + let ascii_zero = u8x16_splat(b'0'); + let nines = u8x16_splat(9); + let ascii_a = u8x16_splat(b'a' - 9 - 1); + let and4bits = u8x16_splat(0xf); let mut i = 0_isize; while src.len() >= 16 { let invec = v128_load(src.as_ptr() as *const _); let masked1 = v128_and(invec, and4bits); - let masked2 = v128_and(i8x16_shr_u(invec, 4), and4bits); + let masked2 = v128_and(u8x16_shr(invec, 4), and4bits); // return 0xff corresponding to the elements > 9, or 0x00 otherwise - let cmpmask1 = i8x16_gt_u(masked1, nines); - let cmpmask2 = i8x16_gt_u(masked2, nines); + let cmpmask1 = u8x16_gt(masked1, nines); + let cmpmask2 = u8x16_gt(masked2, nines); // add '0' or the offset depending on the masks - let masked1 = i8x16_add(masked1, v128_bitselect(ascii_a, ascii_zero, cmpmask1)); - let masked2 = i8x16_add(masked2, v128_bitselect(ascii_a, ascii_zero, cmpmask2)); + let masked1 = u8x16_add(masked1, v128_bitselect(ascii_a, ascii_zero, cmpmask1)); + let masked2 = u8x16_add(masked2, v128_bitselect(ascii_a, ascii_zero, cmpmask2)); // Next we need to shuffle around masked{1,2} to get back to the // original source text order. The first element (res1) we'll store uses // all the low bytes from the 2 masks and the second element (res2) uses // all the upper bytes. - let res1 = v8x16_shuffle::<0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23>( + let res1 = u8x16_shuffle::<0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23>( masked2, masked1, ); - let res2 = v8x16_shuffle::<8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31>( + let res2 = u8x16_shuffle::<8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31>( masked2, masked1, ); diff --git a/library/term/src/lib.rs b/library/term/src/lib.rs index 2116b433fc..943b276a22 100644 --- a/library/term/src/lib.rs +++ b/library/term/src/lib.rs @@ -30,11 +30,7 @@ //! [win]: https://docs.microsoft.com/en-us/windows/console/character-mode-applications //! [ti]: https://en.wikipedia.org/wiki/Terminfo -#![doc( - html_root_url = "https://doc.rust-lang.org/nightly/", - html_playground_url = "https://play.rust-lang.org/", - test(attr(deny(warnings))) -)] +#![doc(html_playground_url = "https://play.rust-lang.org/", test(attr(deny(warnings))))] #![deny(missing_docs)] #![cfg_attr(windows, feature(libc))] diff --git a/library/test/src/cli.rs b/library/test/src/cli.rs index b7791b1b24..84874a2d22 100644 --- a/library/test/src/cli.rs +++ b/library/test/src/cli.rs @@ -95,8 +95,9 @@ fn optgroups() -> getopts::Options { "Configure formatting of output: pretty = Print verbose output; terse = Display one character per test; - json = Output a json document", - "pretty|terse|json", + json = Output a json document; + junit = Output a JUnit document", + "pretty|terse|json|junit", ) .optflag("", "show-output", "Show captured stdout of successful tests") .optopt( @@ -336,10 +337,15 @@ fn get_format( } OutputFormat::Json } - + Some("junit") => { + if !allow_unstable { + return Err("The \"junit\" format is only accepted on the nightly compiler".into()); + } + OutputFormat::Junit + } Some(v) => { return Err(format!( - "argument for --format must be pretty, terse, or json (was \ + "argument for --format must be pretty, terse, json or junit (was \ {})", v )); diff --git a/library/test/src/console.rs b/library/test/src/console.rs index 1721c3c14f..9cfc7eaf4b 100644 --- a/library/test/src/console.rs +++ b/library/test/src/console.rs @@ -10,7 +10,7 @@ use super::{ cli::TestOpts, event::{CompletedTest, TestEvent}, filter_tests, - formatters::{JsonFormatter, OutputFormatter, PrettyFormatter, TerseFormatter}, + formatters::{JsonFormatter, JunitFormatter, OutputFormatter, PrettyFormatter, TerseFormatter}, helpers::{concurrency::get_concurrency, metrics::MetricMap}, options::{Options, OutputFormat}, run_tests, @@ -277,6 +277,7 @@ pub fn run_tests_console(opts: &TestOpts, tests: Vec) -> io::Resu Box::new(TerseFormatter::new(output, opts.use_color(), max_name_len, is_multithreaded)) } OutputFormat::Json => Box::new(JsonFormatter::new(output)), + OutputFormat::Junit => Box::new(JunitFormatter::new(output)), }; let mut st = ConsoleTestState::new(opts)?; diff --git a/library/test/src/formatters/junit.rs b/library/test/src/formatters/junit.rs new file mode 100644 index 0000000000..ec66fc1219 --- /dev/null +++ b/library/test/src/formatters/junit.rs @@ -0,0 +1,174 @@ +use std::io::{self, prelude::Write}; +use std::time::Duration; + +use super::OutputFormatter; +use crate::{ + console::{ConsoleTestState, OutputLocation}, + test_result::TestResult, + time, + types::{TestDesc, TestType}, +}; + +pub struct JunitFormatter { + out: OutputLocation, + results: Vec<(TestDesc, TestResult, Duration)>, +} + +impl JunitFormatter { + pub fn new(out: OutputLocation) -> Self { + Self { out, results: Vec::new() } + } + + fn write_message(&mut self, s: &str) -> io::Result<()> { + assert!(!s.contains('\n')); + + self.out.write_all(s.as_ref()) + } +} + +impl OutputFormatter for JunitFormatter { + fn write_run_start(&mut self, _test_count: usize) -> io::Result<()> { + // We write xml header on run start + self.write_message(&"") + } + + fn write_test_start(&mut self, _desc: &TestDesc) -> io::Result<()> { + // We do not output anything on test start. + Ok(()) + } + + fn write_timeout(&mut self, _desc: &TestDesc) -> io::Result<()> { + // We do not output anything on test timeout. + Ok(()) + } + + fn write_result( + &mut self, + desc: &TestDesc, + result: &TestResult, + exec_time: Option<&time::TestExecTime>, + _stdout: &[u8], + _state: &ConsoleTestState, + ) -> io::Result<()> { + // Because the testsuit node holds some of the information as attributes, we can't write it + // until all of the tests has ran. Instead of writting every result as they come in, we add + // them to a Vec and write them all at once when run is complete. + let duration = exec_time.map(|t| t.0.clone()).unwrap_or_default(); + self.results.push((desc.clone(), result.clone(), duration)); + Ok(()) + } + fn write_run_finish(&mut self, state: &ConsoleTestState) -> io::Result { + self.write_message("")?; + + self.write_message(&*format!( + "", + state.failed, state.total, state.ignored + ))?; + for (desc, result, duration) in std::mem::replace(&mut self.results, Vec::new()) { + let (class_name, test_name) = parse_class_name(&desc); + match result { + TestResult::TrIgnored => { /* no-op */ } + TestResult::TrFailed => { + self.write_message(&*format!( + "", + class_name, + test_name, + duration.as_secs() + ))?; + self.write_message("")?; + self.write_message("")?; + } + + TestResult::TrFailedMsg(ref m) => { + self.write_message(&*format!( + "", + class_name, + test_name, + duration.as_secs() + ))?; + self.write_message(&*format!("", m))?; + self.write_message("")?; + } + + TestResult::TrTimedFail => { + self.write_message(&*format!( + "", + class_name, + test_name, + duration.as_secs() + ))?; + self.write_message("")?; + self.write_message("")?; + } + + TestResult::TrBench(ref b) => { + self.write_message(&*format!( + "", + class_name, test_name, b.ns_iter_summ.sum + ))?; + } + + TestResult::TrOk | TestResult::TrAllowedFail => { + self.write_message(&*format!( + "", + class_name, + test_name, + duration.as_secs() + ))?; + } + } + } + self.write_message("")?; + self.write_message("")?; + self.write_message("")?; + self.write_message("")?; + + Ok(state.failed == 0) + } +} + +fn parse_class_name(desc: &TestDesc) -> (String, String) { + match desc.test_type { + TestType::UnitTest => parse_class_name_unit(desc), + TestType::DocTest => parse_class_name_doc(desc), + TestType::IntegrationTest => parse_class_name_integration(desc), + TestType::Unknown => (String::from("unknown"), String::from(desc.name.as_slice())), + } +} + +fn parse_class_name_unit(desc: &TestDesc) -> (String, String) { + // Module path => classname + // Function name => name + let module_segments: Vec<&str> = desc.name.as_slice().split("::").collect(); + let (class_name, test_name) = match module_segments[..] { + [test] => (String::from("crate"), String::from(test)), + [ref path @ .., test] => (path.join("::"), String::from(test)), + [..] => unreachable!(), + }; + (class_name, test_name) +} + +fn parse_class_name_doc(desc: &TestDesc) -> (String, String) { + // File path => classname + // Line # => test name + let segments: Vec<&str> = desc.name.as_slice().split(" - ").collect(); + let (class_name, test_name) = match segments[..] { + [file, line] => (String::from(file.trim()), String::from(line.trim())), + [..] => unreachable!(), + }; + (class_name, test_name) +} + +fn parse_class_name_integration(desc: &TestDesc) -> (String, String) { + (String::from("integration"), String::from(desc.name.as_slice())) +} diff --git a/library/test/src/formatters/mod.rs b/library/test/src/formatters/mod.rs index 1fb840520a..2e03581b3a 100644 --- a/library/test/src/formatters/mod.rs +++ b/library/test/src/formatters/mod.rs @@ -8,10 +8,12 @@ use crate::{ }; mod json; +mod junit; mod pretty; mod terse; pub(crate) use self::json::JsonFormatter; +pub(crate) use self::junit::JunitFormatter; pub(crate) use self::pretty::PrettyFormatter; pub(crate) use self::terse::TerseFormatter; diff --git a/library/test/src/formatters/pretty.rs b/library/test/src/formatters/pretty.rs index 5e41d6d969..e17fc08a9a 100644 --- a/library/test/src/formatters/pretty.rs +++ b/library/test/src/formatters/pretty.rs @@ -169,7 +169,11 @@ impl PrettyFormatter { fn write_test_name(&mut self, desc: &TestDesc) -> io::Result<()> { let name = desc.padded_name(self.max_name_len, desc.name.padding()); - self.write_plain(&format!("test {} ... ", name))?; + if let Some(test_mode) = desc.test_mode() { + self.write_plain(&format!("test {} - {} ... ", name, test_mode))?; + } else { + self.write_plain(&format!("test {} ... ", name))?; + } Ok(()) } diff --git a/library/test/src/formatters/terse.rs b/library/test/src/formatters/terse.rs index 6f46f7255a..a2c223c494 100644 --- a/library/test/src/formatters/terse.rs +++ b/library/test/src/formatters/terse.rs @@ -158,7 +158,11 @@ impl TerseFormatter { fn write_test_name(&mut self, desc: &TestDesc) -> io::Result<()> { let name = desc.padded_name(self.max_name_len, desc.name.padding()); - self.write_plain(&format!("test {} ... ", name))?; + if let Some(test_mode) = desc.test_mode() { + self.write_plain(&format!("test {} - {} ... ", name, test_mode))?; + } else { + self.write_plain(&format!("test {} ... ", name))?; + } Ok(()) } diff --git a/library/test/src/lib.rs b/library/test/src/lib.rs index 9adc099aaa..3da4d434f4 100644 --- a/library/test/src/lib.rs +++ b/library/test/src/lib.rs @@ -19,7 +19,7 @@ #![crate_name = "test"] #![unstable(feature = "test", issue = "50297")] -#![doc(html_root_url = "https://doc.rust-lang.org/nightly/", test(attr(deny(warnings))))] +#![doc(test(attr(deny(warnings))))] #![cfg_attr(unix, feature(libc))] #![feature(rustc_private)] #![feature(nll)] @@ -49,7 +49,7 @@ pub mod test { cli::{parse_opts, TestOpts}, filter_tests, helpers::metrics::{Metric, MetricMap}, - options::{Options, RunIgnored, RunStrategy, ShouldPanic}, + options::{Concurrent, Options, RunIgnored, RunStrategy, ShouldPanic}, run_test, test_main, test_main_static, test_result::{TestResult, TrFailed, TrFailedMsg, TrIgnored, TrOk}, time::{TestExecTime, TestTimeOptions}, diff --git a/library/test/src/options.rs b/library/test/src/options.rs index 8e7bd8de92..baf36b5f1d 100644 --- a/library/test/src/options.rs +++ b/library/test/src/options.rs @@ -39,6 +39,8 @@ pub enum OutputFormat { Terse, /// JSON output Json, + /// JUnit output + Junit, } /// Whether ignored test should be run or not diff --git a/library/test/src/tests.rs b/library/test/src/tests.rs index 6a3f31b74e..5a4a540b04 100644 --- a/library/test/src/tests.rs +++ b/library/test/src/tests.rs @@ -61,6 +61,10 @@ fn one_ignored_one_unignored_test() -> Vec { ignore: true, should_panic: ShouldPanic::No, allow_fail: false, + #[cfg(not(bootstrap))] + compile_fail: false, + #[cfg(not(bootstrap))] + no_run: false, test_type: TestType::Unknown, }, testfn: DynTestFn(Box::new(move || {})), @@ -71,6 +75,10 @@ fn one_ignored_one_unignored_test() -> Vec { ignore: false, should_panic: ShouldPanic::No, allow_fail: false, + #[cfg(not(bootstrap))] + compile_fail: false, + #[cfg(not(bootstrap))] + no_run: false, test_type: TestType::Unknown, }, testfn: DynTestFn(Box::new(move || {})), @@ -89,6 +97,10 @@ pub fn do_not_run_ignored_tests() { ignore: true, should_panic: ShouldPanic::No, allow_fail: false, + #[cfg(not(bootstrap))] + compile_fail: false, + #[cfg(not(bootstrap))] + no_run: false, test_type: TestType::Unknown, }, testfn: DynTestFn(Box::new(f)), @@ -108,6 +120,10 @@ pub fn ignored_tests_result_in_ignored() { ignore: true, should_panic: ShouldPanic::No, allow_fail: false, + #[cfg(not(bootstrap))] + compile_fail: false, + #[cfg(not(bootstrap))] + no_run: false, test_type: TestType::Unknown, }, testfn: DynTestFn(Box::new(f)), @@ -131,6 +147,10 @@ fn test_should_panic() { ignore: false, should_panic: ShouldPanic::Yes, allow_fail: false, + #[cfg(not(bootstrap))] + compile_fail: false, + #[cfg(not(bootstrap))] + no_run: false, test_type: TestType::Unknown, }, testfn: DynTestFn(Box::new(f)), @@ -154,6 +174,10 @@ fn test_should_panic_good_message() { ignore: false, should_panic: ShouldPanic::YesWithMessage("error message"), allow_fail: false, + #[cfg(not(bootstrap))] + compile_fail: false, + #[cfg(not(bootstrap))] + no_run: false, test_type: TestType::Unknown, }, testfn: DynTestFn(Box::new(f)), @@ -182,6 +206,10 @@ fn test_should_panic_bad_message() { ignore: false, should_panic: ShouldPanic::YesWithMessage(expected), allow_fail: false, + #[cfg(not(bootstrap))] + compile_fail: false, + #[cfg(not(bootstrap))] + no_run: false, test_type: TestType::Unknown, }, testfn: DynTestFn(Box::new(f)), @@ -214,6 +242,10 @@ fn test_should_panic_non_string_message_type() { ignore: false, should_panic: ShouldPanic::YesWithMessage(expected), allow_fail: false, + #[cfg(not(bootstrap))] + compile_fail: false, + #[cfg(not(bootstrap))] + no_run: false, test_type: TestType::Unknown, }, testfn: DynTestFn(Box::new(f)), @@ -238,6 +270,10 @@ fn test_should_panic_but_succeeds() { ignore: false, should_panic, allow_fail: false, + #[cfg(not(bootstrap))] + compile_fail: false, + #[cfg(not(bootstrap))] + no_run: false, test_type: TestType::Unknown, }, testfn: DynTestFn(Box::new(f)), @@ -270,6 +306,10 @@ fn report_time_test_template(report_time: bool) -> Option { ignore: false, should_panic: ShouldPanic::No, allow_fail: false, + #[cfg(not(bootstrap))] + compile_fail: false, + #[cfg(not(bootstrap))] + no_run: false, test_type: TestType::Unknown, }, testfn: DynTestFn(Box::new(f)), @@ -303,6 +343,10 @@ fn time_test_failure_template(test_type: TestType) -> TestResult { ignore: false, should_panic: ShouldPanic::No, allow_fail: false, + #[cfg(not(bootstrap))] + compile_fail: false, + #[cfg(not(bootstrap))] + no_run: false, test_type, }, testfn: DynTestFn(Box::new(f)), @@ -340,6 +384,10 @@ fn typed_test_desc(test_type: TestType) -> TestDesc { ignore: false, should_panic: ShouldPanic::No, allow_fail: false, + #[cfg(not(bootstrap))] + compile_fail: false, + #[cfg(not(bootstrap))] + no_run: false, test_type, } } @@ -451,6 +499,10 @@ pub fn exclude_should_panic_option() { ignore: false, should_panic: ShouldPanic::Yes, allow_fail: false, + #[cfg(not(bootstrap))] + compile_fail: false, + #[cfg(not(bootstrap))] + no_run: false, test_type: TestType::Unknown, }, testfn: DynTestFn(Box::new(move || {})), @@ -473,6 +525,10 @@ pub fn exact_filter_match() { ignore: false, should_panic: ShouldPanic::No, allow_fail: false, + #[cfg(not(bootstrap))] + compile_fail: false, + #[cfg(not(bootstrap))] + no_run: false, test_type: TestType::Unknown, }, testfn: DynTestFn(Box::new(move || {})), @@ -565,6 +621,10 @@ pub fn sort_tests() { ignore: false, should_panic: ShouldPanic::No, allow_fail: false, + #[cfg(not(bootstrap))] + compile_fail: false, + #[cfg(not(bootstrap))] + no_run: false, test_type: TestType::Unknown, }, testfn: DynTestFn(Box::new(testfn)), @@ -642,6 +702,10 @@ pub fn test_bench_no_iter() { ignore: false, should_panic: ShouldPanic::No, allow_fail: false, + #[cfg(not(bootstrap))] + compile_fail: false, + #[cfg(not(bootstrap))] + no_run: false, test_type: TestType::Unknown, }; @@ -662,6 +726,10 @@ pub fn test_bench_iter() { ignore: false, should_panic: ShouldPanic::No, allow_fail: false, + #[cfg(not(bootstrap))] + compile_fail: false, + #[cfg(not(bootstrap))] + no_run: false, test_type: TestType::Unknown, }; @@ -676,6 +744,10 @@ fn should_sort_failures_before_printing_them() { ignore: false, should_panic: ShouldPanic::No, allow_fail: false, + #[cfg(not(bootstrap))] + compile_fail: false, + #[cfg(not(bootstrap))] + no_run: false, test_type: TestType::Unknown, }; @@ -684,6 +756,10 @@ fn should_sort_failures_before_printing_them() { ignore: false, should_panic: ShouldPanic::No, allow_fail: false, + #[cfg(not(bootstrap))] + compile_fail: false, + #[cfg(not(bootstrap))] + no_run: false, test_type: TestType::Unknown, }; diff --git a/library/test/src/types.rs b/library/test/src/types.rs index c5d91f653b..63907c71ea 100644 --- a/library/test/src/types.rs +++ b/library/test/src/types.rs @@ -124,6 +124,10 @@ pub struct TestDesc { pub ignore: bool, pub should_panic: options::ShouldPanic, pub allow_fail: bool, + #[cfg(not(bootstrap))] + pub compile_fail: bool, + #[cfg(not(bootstrap))] + pub no_run: bool, pub test_type: TestType, } @@ -140,6 +144,36 @@ impl TestDesc { } } } + + /// Returns None for ignored test or that that are just run, otherwise give a description of the type of test. + /// Descriptions include "should panic", "compile fail" and "compile". + #[cfg(not(bootstrap))] + pub fn test_mode(&self) -> Option<&'static str> { + if self.ignore { + return None; + } + match self.should_panic { + options::ShouldPanic::Yes | options::ShouldPanic::YesWithMessage(_) => { + return Some("should panic"); + } + options::ShouldPanic::No => {} + } + if self.allow_fail { + return Some("allow fail"); + } + if self.compile_fail { + return Some("compile fail"); + } + if self.no_run { + return Some("compile"); + } + None + } + + #[cfg(bootstrap)] + pub fn test_mode(&self) -> Option<&'static str> { + None + } } #[derive(Debug)] diff --git a/library/unwind/Cargo.toml b/library/unwind/Cargo.toml index 69128591e0..c76ba7667d 100644 --- a/library/unwind/Cargo.toml +++ b/library/unwind/Cargo.toml @@ -21,8 +21,15 @@ compiler_builtins = "0.1.0" cfg-if = "0.1.8" [build-dependencies] -cc = { version = "1.0.1" } +cc = "1.0.68" [features] + +# Only applies for Linux and Fuchsia targets +# Static link to the in-tree build of llvm libunwind llvm-libunwind = [] + +# Only applies for Linux and Fuchsia targets +# If crt-static is enabled, static link to `libunwind.a` provided by system +# If crt-static is disabled, dynamic link to `libunwind.so` provided by system system-llvm-libunwind = [] diff --git a/library/unwind/build.rs b/library/unwind/build.rs index d8bf152e4d..0529d24a27 100644 --- a/library/unwind/build.rs +++ b/library/unwind/build.rs @@ -4,7 +4,8 @@ fn main() { println!("cargo:rerun-if-changed=build.rs"); let target = env::var("TARGET").expect("TARGET was not set"); - if cfg!(feature = "system-llvm-libunwind") { + if cfg!(target_os = "linux") && cfg!(feature = "system-llvm-libunwind") { + // linking for Linux is handled in lib.rs return; } @@ -19,6 +20,20 @@ fn main() { // linking for Linux is handled in lib.rs if target.contains("musl") { llvm_libunwind::compile(); + } else if target.contains("android") { + let build = cc::Build::new(); + + // Since ndk r23 beta 3 `libgcc` was replaced with `libunwind` thus + // check if we have `libunwind` available and if so use it. Otherwise + // fall back to `libgcc` to support older ndk versions. + let has_unwind = + build.is_flag_supported("-lunwind").expect("Unable to invoke compiler"); + + if has_unwind { + println!("cargo:rustc-link-lib=unwind"); + } else { + println!("cargo:rustc-link-lib=gcc"); + } } } else if target.contains("freebsd") { println!("cargo:rustc-link-lib=gcc_s"); @@ -57,101 +72,102 @@ mod llvm_libunwind { pub fn compile() { let target = env::var("TARGET").expect("TARGET was not set"); let target_env = env::var("CARGO_CFG_TARGET_ENV").unwrap(); - let target_vendor = env::var("CARGO_CFG_TARGET_VENDOR").unwrap(); - let target_endian_little = env::var("CARGO_CFG_TARGET_ENDIAN").unwrap() != "big"; - let cfg = &mut cc::Build::new(); - - cfg.cpp(true); - cfg.cpp_set_stdlib(None); - cfg.warnings(false); + let mut cc_cfg = cc::Build::new(); + let mut cpp_cfg = cc::Build::new(); + let root = Path::new("../../src/llvm-project/libunwind"); - // libunwind expects a __LITTLE_ENDIAN__ macro to be set for LE archs, cf. #65765 - if target_endian_little { - cfg.define("__LITTLE_ENDIAN__", Some("1")); + cpp_cfg.cpp(true); + cpp_cfg.cpp_set_stdlib(None); + cpp_cfg.flag("-nostdinc++"); + cpp_cfg.flag("-fno-exceptions"); + cpp_cfg.flag("-fno-rtti"); + cpp_cfg.flag_if_supported("-fvisibility-global-new-delete-hidden"); + + // Don't set this for clang + // By default, Clang builds C code in GNU C17 mode. + // By default, Clang builds C++ code according to the C++98 standard, + // with many C++11 features accepted as extensions. + if cpp_cfg.get_compiler().is_like_gnu() { + cpp_cfg.flag("-std=c++11"); + cc_cfg.flag("-std=c99"); } - if target_env == "msvc" { - // Don't pull in extra libraries on MSVC - cfg.flag("/Zl"); - cfg.flag("/EHsc"); - cfg.define("_CRT_SECURE_NO_WARNINGS", None); - cfg.define("_LIBUNWIND_DISABLE_VISIBILITY_ANNOTATIONS", None); - } else if target.contains("x86_64-fortanix-unknown-sgx") { - cfg.cpp(false); - - cfg.static_flag(true); - cfg.opt_level(3); - - cfg.flag("-nostdinc++"); - cfg.flag("-fno-exceptions"); - cfg.flag("-fno-rtti"); - cfg.flag("-fstrict-aliasing"); - cfg.flag("-funwind-tables"); - cfg.flag("-fvisibility=hidden"); - cfg.flag("-fno-stack-protector"); - cfg.flag("-ffreestanding"); - cfg.flag("-fexceptions"); - - // easiest way to undefine since no API available in cc::Build to undefine - cfg.flag("-U_FORTIFY_SOURCE"); - cfg.define("_FORTIFY_SOURCE", "0"); - - cfg.flag_if_supported("-fvisibility-global-new-delete-hidden"); + if target.contains("x86_64-fortanix-unknown-sgx") || target_env == "musl" { + // use the same GCC C compiler command to compile C++ code so we do not need to setup the + // C++ compiler env variables on the builders. + // Don't set this for clang++, as clang++ is able to compile this without libc++. + if cpp_cfg.get_compiler().is_like_gnu() { + cpp_cfg.cpp(false); + } + } - cfg.define("_LIBUNWIND_DISABLE_VISIBILITY_ANNOTATIONS", None); - cfg.define("RUST_SGX", "1"); - cfg.define("__NO_STRING_INLINES", None); - cfg.define("__NO_MATH_INLINES", None); - cfg.define("_LIBUNWIND_IS_BAREMETAL", None); - cfg.define("__LIBUNWIND_IS_NATIVE_ONLY", None); - cfg.define("NDEBUG", None); - } else { - cfg.flag("-std=c99"); - cfg.flag("-std=c++11"); - cfg.flag("-nostdinc++"); - cfg.flag("-fno-exceptions"); - cfg.flag("-fno-rtti"); + for cfg in [&mut cc_cfg, &mut cpp_cfg].iter_mut() { + cfg.warnings(false); cfg.flag("-fstrict-aliasing"); cfg.flag("-funwind-tables"); cfg.flag("-fvisibility=hidden"); - cfg.flag_if_supported("-fvisibility-global-new-delete-hidden"); cfg.define("_LIBUNWIND_DISABLE_VISIBILITY_ANNOTATIONS", None); + cfg.include(root.join("include")); + cfg.cargo_metadata(false); + + if target.contains("x86_64-fortanix-unknown-sgx") { + cfg.static_flag(true); + cfg.opt_level(3); + cfg.flag("-fno-stack-protector"); + cfg.flag("-ffreestanding"); + cfg.flag("-fexceptions"); + + // easiest way to undefine since no API available in cc::Build to undefine + cfg.flag("-U_FORTIFY_SOURCE"); + cfg.define("_FORTIFY_SOURCE", "0"); + cfg.define("RUST_SGX", "1"); + cfg.define("__NO_STRING_INLINES", None); + cfg.define("__NO_MATH_INLINES", None); + cfg.define("_LIBUNWIND_IS_BAREMETAL", None); + cfg.define("__LIBUNWIND_IS_NATIVE_ONLY", None); + cfg.define("NDEBUG", None); + } } - let mut unwind_sources = vec![ - "Unwind-EHABI.cpp", - "Unwind-seh.cpp", + let mut c_sources = vec![ "Unwind-sjlj.c", "UnwindLevel1-gcc-ext.c", "UnwindLevel1.c", "UnwindRegistersRestore.S", "UnwindRegistersSave.S", - "libunwind.cpp", ]; - if target_vendor == "apple" { - unwind_sources.push("Unwind_AppleExtras.cpp"); - } + let cpp_sources = vec!["Unwind-EHABI.cpp", "Unwind-seh.cpp", "libunwind.cpp"]; + let cpp_len = cpp_sources.len(); if target.contains("x86_64-fortanix-unknown-sgx") { - unwind_sources.push("UnwindRustSgx.c"); + c_sources.push("UnwindRustSgx.c"); } - let root = Path::new("../../src/llvm-project/libunwind"); - cfg.include(root.join("include")); - for src in unwind_sources { - cfg.file(root.join("src").join(src)); + for src in c_sources { + cc_cfg.file(root.join("src").join(src).canonicalize().unwrap()); } - if target_env == "musl" { - // use the same C compiler command to compile C++ code so we do not need to setup the - // C++ compiler env variables on the builders - cfg.cpp(false); - // linking for musl is handled in lib.rs - cfg.cargo_metadata(false); - println!("cargo:rustc-link-search=native={}", env::var("OUT_DIR").unwrap()); + for src in cpp_sources { + cpp_cfg.file(root.join("src").join(src).canonicalize().unwrap()); } - cfg.compile("unwind"); + let out_dir = env::var("OUT_DIR").unwrap(); + println!("cargo:rustc-link-search=native={}", &out_dir); + + cpp_cfg.compile("unwind-cpp"); + + let mut count = 0; + for entry in std::fs::read_dir(&out_dir).unwrap() { + let obj = entry.unwrap().path().canonicalize().unwrap(); + if let Some(ext) = obj.extension() { + if ext == "o" { + cc_cfg.object(&obj); + count += 1; + } + } + } + assert_eq!(cpp_len, count, "Can't get object files from {:?}", &out_dir); + cc_cfg.compile("unwind"); } } diff --git a/library/unwind/src/lib.rs b/library/unwind/src/lib.rs index be5e56c71e..eaeec72fbb 100644 --- a/library/unwind/src/lib.rs +++ b/library/unwind/src/lib.rs @@ -37,9 +37,22 @@ cfg_if::cfg_if! { } #[cfg(target_env = "musl")] -#[link(name = "unwind", kind = "static", cfg(target_feature = "crt-static"))] -#[link(name = "gcc_s", cfg(not(target_feature = "crt-static")))] -extern "C" {} +cfg_if::cfg_if! { + if #[cfg(all(feature = "llvm-libunwind", feature = "system-llvm-libunwind"))] { + compile_error!("`llvm-libunwind` and `system-llvm-libunwind` cannot be enabled at the same time"); + } else if #[cfg(feature = "llvm-libunwind")] { + #[link(name = "unwind", kind = "static")] + extern "C" {} + } else if #[cfg(feature = "system-llvm-libunwind")] { + #[link(name = "unwind", kind = "static-nobundle", cfg(target_feature = "crt-static"))] + #[link(name = "unwind", cfg(not(target_feature = "crt-static")))] + extern "C" {} + } else { + #[link(name = "unwind", kind = "static", cfg(target_feature = "crt-static"))] + #[link(name = "gcc_s", cfg(not(target_feature = "crt-static")))] + extern "C" {} + } +} // When building with crt-static, we get `gcc_eh` from the `libc` crate, since // glibc needs it, and needs it listed later on the linker command line. We @@ -68,5 +81,5 @@ extern "C" {} extern "C" {} #[cfg(all(target_vendor = "fortanix", target_env = "sgx"))] -#[link(name = "unwind", kind = "static-nobundle")] +#[link(name = "unwind", kind = "static")] extern "C" {} diff --git a/src/bootstrap/Cargo.toml b/src/bootstrap/Cargo.toml index c14ad6fa5f..8445d811e0 100644 --- a/src/bootstrap/Cargo.toml +++ b/src/bootstrap/Cargo.toml @@ -40,7 +40,7 @@ cmake = "0.1.38" filetime = "0.2" num_cpus = "1.0" getopts = "0.2.19" -cc = "1.0.35" +cc = "1.0.68" libc = "0.2" serde = { version = "1.0.8", features = ["derive"] } serde_json = "1.0.2" diff --git a/src/bootstrap/bin/rustc.rs b/src/bootstrap/bin/rustc.rs index d462dc4d11..ac8bbfe102 100644 --- a/src/bootstrap/bin/rustc.rs +++ b/src/bootstrap/bin/rustc.rs @@ -124,6 +124,13 @@ fn main() { cmd.arg("-C").arg("target-feature=-crt-static"); } } + + if stage == "0" { + // Cargo doesn't pass RUSTFLAGS to proc_macros: + // https://github.com/rust-lang/cargo/issues/4423 + // Set `--cfg=bootstrap` explicitly instead. + cmd.arg("--cfg=bootstrap"); + } } if let Ok(map) = env::var("RUSTC_DEBUGINFO_MAP") { @@ -305,7 +312,7 @@ fn format_rusage_data(_child: Child) -> Option { }; // Mac OS X reports the maxrss in bytes, not kb. let divisor = if env::consts::OS == "macos" { 1024 } else { 1 }; - let maxrss = rusage.ru_maxrss + (divisor - 1) / divisor; + let maxrss = (rusage.ru_maxrss + (divisor - 1)) / divisor; let mut init_str = format!( "user: {USER_SEC}.{USER_USEC:03} \ diff --git a/src/bootstrap/bin/rustdoc.rs b/src/bootstrap/bin/rustdoc.rs index cba17c8e60..e4396d5301 100644 --- a/src/bootstrap/bin/rustdoc.rs +++ b/src/bootstrap/bin/rustdoc.rs @@ -41,7 +41,12 @@ fn main() { cmd.arg(arg); } if env::var_os("RUSTDOC_FUSE_LD_LLD").is_some() { - cmd.arg("-Clink-args=-fuse-ld=lld"); + cmd.arg("-Clink-arg=-fuse-ld=lld"); + if cfg!(windows) { + cmd.arg("-Clink-arg=-Wl,/threads:1"); + } else { + cmd.arg("-Clink-arg=-Wl,--threads=1"); + } } // Needed to be able to run all rustdoc tests. diff --git a/src/bootstrap/bootstrap.py b/src/bootstrap/bootstrap.py index 4111420e47..7c7f162b82 100644 --- a/src/bootstrap/bootstrap.py +++ b/src/bootstrap/bootstrap.py @@ -414,10 +414,13 @@ class RustBuild(object): filename = "rustc-{}-{}{}".format(rustc_channel, self.build, tarball_suffix) self._download_component_helper(filename, "rustc", tarball_suffix, stage0) - filename = "cargo-{}-{}{}".format(rustc_channel, self.build, - tarball_suffix) - self._download_component_helper(filename, "cargo", tarball_suffix) - if not stage0: + # download-rustc doesn't need its own cargo, it can just use beta's. + if stage0: + filename = "cargo-{}-{}{}".format(rustc_channel, self.build, + tarball_suffix) + self._download_component_helper(filename, "cargo", tarball_suffix) + self.fix_bin_or_dylib("{}/bin/cargo".format(bin_root)) + else: filename = "rustc-dev-{}-{}{}".format(rustc_channel, self.build, tarball_suffix) self._download_component_helper( filename, "rustc-dev", tarball_suffix, stage0 @@ -425,7 +428,6 @@ class RustBuild(object): self.fix_bin_or_dylib("{}/bin/rustc".format(bin_root)) self.fix_bin_or_dylib("{}/bin/rustdoc".format(bin_root)) - self.fix_bin_or_dylib("{}/bin/cargo".format(bin_root)) lib_dir = "{}/lib".format(bin_root) for lib in os.listdir(lib_dir): if lib.endswith(".so"): @@ -646,18 +648,20 @@ class RustBuild(object): rev_parse = ["git", "rev-parse", "--show-toplevel"] top_level = subprocess.check_output(rev_parse, universal_newlines=True).strip() compiler = "{}/compiler/".format(top_level) + library = "{}/library/".format(top_level) # Look for a version to compare to based on the current commit. # Only commits merged by bors will have CI artifacts. merge_base = ["git", "log", "--author=bors", "--pretty=%H", "-n1"] commit = subprocess.check_output(merge_base, universal_newlines=True).strip() - # Warn if there were changes to the compiler since the ancestor commit. - status = subprocess.call(["git", "diff-index", "--quiet", commit, "--", compiler]) + # Warn if there were changes to the compiler or standard library since the ancestor commit. + status = subprocess.call(["git", "diff-index", "--quiet", commit, "--", compiler, library]) if status != 0: if download_rustc == "if-unchanged": return None - print("warning: `download-rustc` is enabled, but there are changes to compiler/") + print("warning: `download-rustc` is enabled, but there are changes to \ + compiler/ or library/") if self.verbose: print("using downloaded stage1 artifacts from CI (commit {})".format(commit)) @@ -898,8 +902,8 @@ class RustBuild(object): target_linker = self.get_toml("linker", build_section) if target_linker is not None: env["RUSTFLAGS"] += " -C linker=" + target_linker - # cfg(bootstrap): Add `-Wsemicolon_in_expressions_from_macros` after the next beta bump env["RUSTFLAGS"] += " -Wrust_2018_idioms -Wunused_lifetimes" + env["RUSTFLAGS"] += " -Wsemicolon_in_expressions_from_macros" if self.get_toml("deny-warnings", "rust") != "false": env["RUSTFLAGS"] += " -Dwarnings" diff --git a/src/bootstrap/builder.rs b/src/bootstrap/builder.rs index cff1ec843f..bc499fdba5 100644 --- a/src/bootstrap/builder.rs +++ b/src/bootstrap/builder.rs @@ -378,6 +378,9 @@ impl<'a> Builder<'a> { check::Rustdoc, check::CodegenBackend, check::Clippy, + check::Miri, + check::Rls, + check::Rustfmt, check::Bootstrap ), Kind::Test => describe!( @@ -571,6 +574,18 @@ impl<'a> Builder<'a> { self.run_step_descriptions(&Builder::get_step_descriptions(Kind::Doc), paths); } + /// NOTE: keep this in sync with `rustdoc::clean::utils::doc_rust_lang_org_channel`, or tests will fail on beta/stable. + pub fn doc_rust_lang_org_channel(&self) -> String { + let channel = match &*self.config.channel { + "stable" => &self.version, + "beta" => "beta", + "nightly" | "dev" => "nightly", + // custom build of rustdoc maybe? link to the latest stable docs just in case + _ => "stable", + }; + "https://doc.rust-lang.org/".to_owned() + channel + } + fn run_step_descriptions(&self, v: &[StepDescription], paths: &[PathBuf]) { StepDescription::run(v, self, paths); } @@ -706,7 +721,15 @@ impl<'a> Builder<'a> { return; } - add_dylib_path(vec![self.rustc_libdir(compiler)], cmd); + let mut dylib_dirs = vec![self.rustc_libdir(compiler)]; + + // Ensure that the downloaded LLVM libraries can be found. + if self.config.llvm_from_ci { + let ci_llvm_lib = self.out.join(&*compiler.host.triple).join("ci-llvm").join("lib"); + dylib_dirs.push(ci_llvm_lib); + } + + add_dylib_path(dylib_dirs, cmd); } /// Gets a path to the compiler specified. @@ -1119,6 +1142,7 @@ impl<'a> Builder<'a> { } if self.is_fuse_ld_lld(compiler.host) { cargo.env("RUSTC_HOST_FUSE_LD_LLD", "1"); + cargo.env("RUSTDOC_FUSE_LD_LLD", "1"); } if let Some(target_linker) = self.linker(target) { @@ -1128,6 +1152,9 @@ impl<'a> Builder<'a> { if self.is_fuse_ld_lld(target) { rustflags.arg("-Clink-args=-fuse-ld=lld"); } + self.lld_flags(target).for_each(|flag| { + rustdocflags.arg(&flag); + }); if !(["build", "check", "clippy", "fix", "rustc"].contains(&cmd)) && want_rustdoc { cargo.env("RUSTDOC_LIBDIR", self.rustc_libdir(compiler)); @@ -1271,12 +1298,7 @@ impl<'a> Builder<'a> { // some code doesn't go through this `rustc` wrapper. lint_flags.push("-Wrust_2018_idioms"); lint_flags.push("-Wunused_lifetimes"); - // cfg(bootstrap): unconditionally enable this warning after the next beta bump - // This is currently disabled for the stage1 libstd, since build scripts - // will end up using the bootstrap compiler (which doesn't yet support this lint) - if compiler.stage != 0 && mode != Mode::Std { - lint_flags.push("-Wsemicolon_in_expressions_from_macros"); - } + lint_flags.push("-Wsemicolon_in_expressions_from_macros"); if self.config.deny_warnings { lint_flags.push("-Dwarnings"); diff --git a/src/bootstrap/builder/tests.rs b/src/bootstrap/builder/tests.rs index a881512e98..4d7c207e3a 100644 --- a/src/bootstrap/builder/tests.rs +++ b/src/bootstrap/builder/tests.rs @@ -489,6 +489,7 @@ mod dist { compare_mode: None, rustfix_coverage: false, pass: None, + run: None, }; let build = Build::new(config); @@ -529,6 +530,7 @@ mod dist { compare_mode: None, rustfix_coverage: false, pass: None, + run: None, }; let build = Build::new(config); @@ -584,6 +586,7 @@ mod dist { compare_mode: None, rustfix_coverage: false, pass: None, + run: None, }; // Make sure rustfmt binary not being found isn't an error. config.channel = "beta".to_string(); diff --git a/src/bootstrap/check.rs b/src/bootstrap/check.rs index 9b76c8b9a2..5003d6693f 100644 --- a/src/bootstrap/check.rs +++ b/src/bootstrap/check.rs @@ -280,7 +280,7 @@ impl Step for CodegenBackend { } macro_rules! tool_check_step { - ($name:ident, $path:literal, $($alias:literal, )* $source_type:path) => { + ($name:ident, $path:literal, $($alias:literal, )* $source_type:path $(, $default:literal )?) => { #[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] pub struct $name { pub target: TargetSelection, @@ -289,7 +289,8 @@ macro_rules! tool_check_step { impl Step for $name { type Output = (); const ONLY_HOSTS: bool = true; - const DEFAULT: bool = true; + // don't ever check out-of-tree tools by default, they'll fail when toolstate is broken + const DEFAULT: bool = matches!($source_type, SourceType::InTree) $( && $default )?; fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { run.paths(&[ $path, $($alias),* ]) @@ -341,10 +342,6 @@ macro_rules! tool_check_step { true, ); - let libdir = builder.sysroot_libdir(compiler, target); - let hostdir = builder.sysroot_libdir(compiler, compiler.host); - add_to_sysroot(&builder, &libdir, &hostdir, &stamp(builder, compiler, target)); - /// Cargo's output path in a given stage, compiled by a particular /// compiler for the specified target. fn stamp( @@ -362,13 +359,16 @@ macro_rules! tool_check_step { } tool_check_step!(Rustdoc, "src/tools/rustdoc", "src/librustdoc", SourceType::InTree); -// Clippy is a hybrid. It is an external tool, but uses a git subtree instead +// Clippy and Rustfmt are hybrids. They are external tools, but use a git subtree instead // of a submodule. Since the SourceType only drives the deny-warnings // behavior, treat it as in-tree so that any new warnings in clippy will be // rejected. tool_check_step!(Clippy, "src/tools/clippy", SourceType::InTree); +tool_check_step!(Miri, "src/tools/miri", SourceType::Submodule); +tool_check_step!(Rls, "src/tools/rls", SourceType::Submodule); +tool_check_step!(Rustfmt, "src/tools/rustfmt", SourceType::InTree); -tool_check_step!(Bootstrap, "src/bootstrap", SourceType::InTree); +tool_check_step!(Bootstrap, "src/bootstrap", SourceType::InTree, false); /// Cargo's output path for the standard library in a given stage, compiled /// by a particular compiler for the specified target. diff --git a/src/bootstrap/compile.rs b/src/bootstrap/compile.rs index ee3527b6bf..112a6ea939 100644 --- a/src/bootstrap/compile.rs +++ b/src/bootstrap/compile.rs @@ -269,7 +269,9 @@ pub fn std_cargo(builder: &Builder<'_>, target: TargetSelection, stage: u32, car if builder.no_std(target) == Some(true) { let mut features = "compiler-builtins-mem".to_string(); - features.push_str(compiler_builtins_c_feature); + if !target.starts_with("bpf") { + features.push_str(compiler_builtins_c_feature); + } // for no-std targets we only compile a few no_std crates cargo @@ -326,6 +328,11 @@ pub fn std_cargo(builder: &Builder<'_>, target: TargetSelection, stage: u32, car if target.contains("riscv") { cargo.rustflag("-Cforce-unwind-tables=yes"); } + + let html_root = + format!("-Zcrate-attr=doc(html_root_url=\"{}/\")", builder.doc_rust_lang_org_channel(),); + cargo.rustflag(&html_root); + cargo.rustdocflag(&html_root); } #[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] @@ -629,8 +636,7 @@ pub fn rustc_cargo_env(builder: &Builder<'_>, cargo: &mut Cargo, target: TargetS cargo .env("CFG_RELEASE", builder.rust_release()) .env("CFG_RELEASE_CHANNEL", &builder.config.channel) - .env("CFG_VERSION", builder.rust_version()) - .env("CFG_PREFIX", builder.config.prefix.clone().unwrap_or_default()); + .env("CFG_VERSION", builder.rust_version()); let libdir_relative = builder.config.libdir_relative().unwrap_or_else(|| Path::new("lib")); cargo.env("CFG_LIBDIR_RELATIVE", libdir_relative); @@ -649,6 +655,7 @@ pub fn rustc_cargo_env(builder: &Builder<'_>, cargo: &mut Cargo, target: TargetS } if builder.config.rustc_parallel { cargo.rustflag("--cfg=parallel_compiler"); + cargo.rustdocflag("--cfg=parallel_compiler"); } if builder.config.rust_verify_llvm_ir { cargo.env("RUSTC_VERIFY_LLVM_IR", "1"); @@ -1101,6 +1108,13 @@ impl Step for Assemble { let src_exe = exe("lld", target_compiler.host); let dst_exe = exe("rust-lld", target_compiler.host); builder.copy(&lld_install.join("bin").join(&src_exe), &libdir_bin.join(&dst_exe)); + // for `-Z gcc-ld=lld` + let gcc_ld_dir = libdir_bin.join("gcc-ld"); + t!(fs::create_dir(&gcc_ld_dir)); + builder.copy( + &lld_install.join("bin").join(&src_exe), + &gcc_ld_dir.join(exe("ld", target_compiler.host)), + ); } // Similarly, copy `llvm-dwp` into libdir for Split DWARF. Only copy it when the LLVM diff --git a/src/bootstrap/dist.rs b/src/bootstrap/dist.rs index 8224eac823..16d148db61 100644 --- a/src/bootstrap/dist.rs +++ b/src/bootstrap/dist.rs @@ -402,6 +402,10 @@ impl Step for Rustc { if builder.config.lld_enabled { let exe = exe("rust-lld", compiler.host); builder.copy(&src_dir.join(&exe), &dst_dir.join(&exe)); + // for `-Z gcc-ld=lld` + let gcc_lld_dir = dst_dir.join("gcc-ld"); + t!(fs::create_dir(&gcc_lld_dir)); + builder.copy(&src_dir.join(&exe), &gcc_lld_dir.join(&exe)); } // Copy over llvm-dwp if it's there @@ -1074,6 +1078,12 @@ impl Step for RustAnalyzer { } fn run(self, builder: &Builder<'_>) -> Option { + // This prevents rust-analyzer from being built for "dist" or "install" + // on the stable/beta channels. It is a nightly-only tool and should + // not be included. + if !builder.build.unstable_features() { + return None; + } let compiler = self.compiler; let target = self.target; assert!(builder.config.extended); @@ -1173,6 +1183,12 @@ impl Step for Miri { } fn run(self, builder: &Builder<'_>) -> Option { + // This prevents miri from being built for "dist" or "install" + // on the stable/beta channels. It is a nightly-only tool and should + // not be included. + if !builder.build.unstable_features() { + return None; + } let compiler = self.compiler; let target = self.target; assert!(builder.config.extended); diff --git a/src/bootstrap/doc.rs b/src/bootstrap/doc.rs index a32b92ef1a..d2fabf9967 100644 --- a/src/bootstrap/doc.rs +++ b/src/bootstrap/doc.rs @@ -434,6 +434,7 @@ impl Step for Std { cargo .arg("-p") .arg(package) + .arg("-Zskip-rustdoc-fingerprint") .arg("--") .arg("--markdown-css") .arg("rust.css") @@ -451,6 +452,22 @@ impl Step for Std { builder.run(&mut cargo.into()); }; + + let paths = builder + .paths + .iter() + .map(components_simplified) + .filter_map(|path| { + if path.get(0) == Some(&"library") { + Some(path[1].to_owned()) + } else if !path.is_empty() { + Some(path[0].to_owned()) + } else { + None + } + }) + .collect::>(); + // Only build the following crates. While we could just iterate over the // folder structure, that would also build internal crates that we do // not want to show in documentation. These crates will later be visited @@ -464,20 +481,17 @@ impl Step for Std { let krates = ["core", "alloc", "std", "proc_macro", "test"]; for krate in &krates { run_cargo_rustdoc_for(krate); + if paths.iter().any(|p| p == krate) { + // No need to document more of the libraries if we have the one we want. + break; + } } builder.cp_r(&out_dir, &out); // Look for library/std, library/core etc in the `x.py doc` arguments and // open the corresponding rendered docs. - for path in builder.paths.iter().map(components_simplified) { - let requested_crate = if path.get(0) == Some(&"library") { - &path[1] - } else if !path.is_empty() { - &path[0] - } else { - continue; - }; - if krates.contains(&requested_crate) { + for requested_crate in paths { + if krates.iter().any(|k| *k == requested_crate.as_str()) { let index = out.join(requested_crate).join("index.html"); open(builder, &index); } @@ -549,7 +563,9 @@ impl Step for Rustc { cargo.rustdocflag("--enable-index-page"); cargo.rustdocflag("-Zunstable-options"); cargo.rustdocflag("-Znormalize-docs"); + cargo.rustdocflag("--show-type-layout"); compile::rustc_cargo(builder, &mut cargo, target); + cargo.arg("-Zskip-rustdoc-fingerprint"); // Only include compiler crates, no dependencies of those, such as `libc`. cargo.arg("--no-deps"); @@ -641,6 +657,7 @@ impl Step for Rustdoc { &[], ); + cargo.arg("-Zskip-rustdoc-fingerprint"); // Only include compiler crates, no dependencies of those, such as `libc`. cargo.arg("--no-deps"); cargo.arg("-p").arg("rustdoc"); @@ -648,6 +665,7 @@ impl Step for Rustdoc { cargo.rustdocflag("--document-private-items"); cargo.rustdocflag("--enable-index-page"); + cargo.rustdocflag("--show-type-layout"); cargo.rustdocflag("-Zunstable-options"); builder.run(&mut cargo.into()); } diff --git a/src/bootstrap/flags.rs b/src/bootstrap/flags.rs index 6044899c23..80a60c79ed 100644 --- a/src/bootstrap/flags.rs +++ b/src/bootstrap/flags.rs @@ -91,6 +91,7 @@ pub enum Subcommand { paths: Vec, }, Format { + paths: Vec, check: bool, }, Doc { @@ -103,6 +104,7 @@ pub enum Subcommand { bless: bool, compare_mode: Option, pass: Option, + run: Option, test_args: Vec, rustc_args: Vec, fail_fast: bool, @@ -222,8 +224,8 @@ To learn more about a subcommand, run `./x.py -h`", VALUE overrides the skip-rebuild option in config.toml.", "VALUE", ); - opts.optopt("", "rust-profile-generate", "rustc error format", "FORMAT"); - opts.optopt("", "rust-profile-use", "rustc error format", "FORMAT"); + opts.optopt("", "rust-profile-generate", "generate PGO profile with rustc build", "FORMAT"); + opts.optopt("", "rust-profile-use", "use PGO profile for rustc build", "FORMAT"); // We can't use getopt to parse the options until we have completed specifying which // options are valid, but under the current implementation, some options are conditional on @@ -293,6 +295,7 @@ To learn more about a subcommand, run `./x.py -h`", "force {check,build,run}-pass tests to this mode.", "check | build | run", ); + opts.optopt("", "run", "whether to execute run-* tests", "auto | always | never"); opts.optflag( "", "rustfix-coverage", @@ -556,6 +559,7 @@ Arguments: bless: matches.opt_present("bless"), compare_mode: matches.opt_str("compare-mode"), pass: matches.opt_str("pass"), + run: matches.opt_str("run"), test_args: matches.opt_strs("test-args"), rustc_args: matches.opt_strs("rustc-args"), fail_fast: !matches.opt_present("no-fail-fast"), @@ -578,7 +582,7 @@ Arguments: Subcommand::Clean { all: matches.opt_present("all") } } - "fmt" => Subcommand::Format { check: matches.opt_present("check") }, + "fmt" => Subcommand::Format { check: matches.opt_present("check"), paths }, "dist" => Subcommand::Dist { paths }, "install" => Subcommand::Install { paths }, "run" | "r" => { @@ -742,6 +746,13 @@ impl Subcommand { } } + pub fn run(&self) -> Option<&str> { + match *self { + Subcommand::Test { ref run, .. } => run.as_ref().map(|s| &s[..]), + _ => None, + } + } + pub fn open(&self) -> bool { match *self { Subcommand::Doc { open, .. } => open, diff --git a/src/bootstrap/format.rs b/src/bootstrap/format.rs index d21e340814..2408344487 100644 --- a/src/bootstrap/format.rs +++ b/src/bootstrap/format.rs @@ -42,7 +42,7 @@ struct RustfmtConfig { ignore: Vec, } -pub fn format(build: &Build, check: bool) { +pub fn format(build: &Build, check: bool, paths: &[PathBuf]) { if build.config.dry_run { return; } @@ -118,8 +118,19 @@ pub fn format(build: &Build, check: bool) { .to_path_buf(); let src = build.src.clone(); let (tx, rx): (SyncSender, _) = std::sync::mpsc::sync_channel(128); - let walker = - WalkBuilder::new(src.clone()).types(matcher).overrides(ignore_fmt).build_parallel(); + let walker = match paths.get(0) { + Some(first) => { + let mut walker = WalkBuilder::new(first); + for path in &paths[1..] { + walker.add(path); + } + walker + } + None => WalkBuilder::new(src.clone()), + } + .types(matcher) + .overrides(ignore_fmt) + .build_parallel(); // there is a lot of blocking involved in spawning a child process and reading files to format. // spawn more processes than available concurrency to keep the CPU busy diff --git a/src/bootstrap/install.rs b/src/bootstrap/install.rs index 13ee909afd..8a1b6df0da 100644 --- a/src/bootstrap/install.rs +++ b/src/bootstrap/install.rs @@ -139,11 +139,17 @@ macro_rules! install { install!((self, builder, _config), Docs, "src/doc", _config.docs, only_hosts: false, { - let tarball = builder.ensure(dist::Docs { host: self.target }).expect("missing docs"); - install_sh(builder, "docs", self.compiler.stage, Some(self.target), &tarball); + if let Some(tarball) = builder.ensure(dist::Docs { host: self.target }) { + install_sh(builder, "docs", self.compiler.stage, Some(self.target), &tarball); + } else { + panic!("docs are not available to install, \ + check that `build.docs` is true in `config.toml`"); + } }; Std, "library/std", true, only_hosts: false, { for target in &builder.targets { + // `expect` should be safe, only None when host != build, but this + // only runs when host == build let tarball = builder.ensure(dist::Std { compiler: self.compiler, target: *target @@ -165,10 +171,15 @@ install!((self, builder, _config), } }; RustAnalyzer, "rust-analyzer", Self::should_build(_config), only_hosts: true, { - let tarball = builder - .ensure(dist::RustAnalyzer { compiler: self.compiler, target: self.target }) - .expect("missing rust-analyzer"); - install_sh(builder, "rust-analyzer", self.compiler.stage, Some(self.target), &tarball); + if let Some(tarball) = + builder.ensure(dist::RustAnalyzer { compiler: self.compiler, target: self.target }) + { + install_sh(builder, "rust-analyzer", self.compiler.stage, Some(self.target), &tarball); + } else { + builder.info( + &format!("skipping Install rust-analyzer stage{} ({})", self.compiler.stage, self.target), + ); + } }; Clippy, "clippy", Self::should_build(_config), only_hosts: true, { let tarball = builder.ensure(dist::Clippy { compiler: self.compiler, target: self.target }); @@ -212,6 +223,8 @@ install!((self, builder, _config), } }; Analysis, "analysis", Self::should_build(_config), only_hosts: false, { + // `expect` should be safe, only None with host != build, but this + // only uses the `build` compiler let tarball = builder.ensure(dist::Analysis { // Find the actual compiler (handling the full bootstrap option) which // produced the save-analysis data because that data isn't copied diff --git a/src/bootstrap/lib.rs b/src/bootstrap/lib.rs index 24da44b933..347236c655 100644 --- a/src/bootstrap/lib.rs +++ b/src/bootstrap/lib.rs @@ -444,8 +444,13 @@ impl Build { build.verbose("finding compilers"); cc_detect::find(&mut build); - build.verbose("running sanity check"); - sanity::check(&mut build); + // When running `setup`, the profile is about to change, so any requirements we have now may + // be different on the next invocation. Don't check for them until the next time x.py is + // run. This is ok because `setup` never runs any build commands, so it won't fail if commands are missing. + if !matches!(build.config.cmd, Subcommand::Setup { .. }) { + build.verbose("running sanity check"); + sanity::check(&mut build); + } // If local-rust is the same major.minor as the current version, then force a // local-rebuild @@ -478,8 +483,8 @@ impl Build { job::setup(self); } - if let Subcommand::Format { check } = self.config.cmd { - return format::format(self, check); + if let Subcommand::Format { check, paths } = &self.config.cmd { + return format::format(self, *check, &paths); } if let Subcommand::Clean { all } = self.config.cmd { @@ -918,6 +923,21 @@ impl Build { self.config.use_lld && !target.contains("msvc") } + fn lld_flags(&self, target: TargetSelection) -> impl Iterator { + let mut options = [None, None]; + + if self.config.use_lld { + if self.is_fuse_ld_lld(target) { + options[0] = Some("-Clink-arg=-fuse-ld=lld".to_string()); + } + + let threads = if target.contains("windows") { "/threads:1" } else { "--threads=1" }; + options[1] = Some(format!("-Clink-arg=-Wl,{}", threads)); + } + + std::array::IntoIter::new(options).flatten() + } + /// Returns if this target should statically link the C runtime, if specified fn crt_static(&self, target: TargetSelection) -> Option { if target.contains("pc-windows-msvc") { @@ -1366,7 +1386,7 @@ impl Build { eprintln!( " Couldn't find required command: ninja -You should install ninja, or set ninja=false in config.toml +You should install ninja, or set `ninja=false` in config.toml in the `[llvm]` section. " ); std::process::exit(1); diff --git a/src/bootstrap/mk/Makefile.in b/src/bootstrap/mk/Makefile.in index 47cf1172d3..fd39944e17 100644 --- a/src/bootstrap/mk/Makefile.in +++ b/src/bootstrap/mk/Makefile.in @@ -45,10 +45,6 @@ check-aux: src/tools/cargo \ src/tools/cargotest \ $(BOOTSTRAP_ARGS) -check-aux-and-gui: check-aux - $(Q)$(BOOTSTRAP) test --stage 2 \ - src/test/rustdoc-gui \ - $(BOOTSTRAP_ARGS) check-bootstrap: $(Q)$(CFG_PYTHON) $(CFG_SRC_DIR)src/bootstrap/bootstrap_test.py dist: diff --git a/src/bootstrap/native.rs b/src/bootstrap/native.rs index 156e8dd53d..449fdb87b0 100644 --- a/src/bootstrap/native.rs +++ b/src/bootstrap/native.rs @@ -153,7 +153,7 @@ impl Step for Llvm { let llvm_targets = match &builder.config.llvm_targets { Some(s) => s, None => { - "AArch64;ARM;Hexagon;MSP430;Mips;NVPTX;PowerPC;RISCV;\ + "AArch64;ARM;BPF;Hexagon;MSP430;Mips;NVPTX;PowerPC;RISCV;\ Sparc;SystemZ;WebAssembly;X86" } }; @@ -181,7 +181,7 @@ impl Step for Llvm { .define("LLVM_TARGET_ARCH", target_native.split('-').next().unwrap()) .define("LLVM_DEFAULT_TARGET_TRIPLE", target_native); - if target != "aarch64-apple-darwin" { + if target != "aarch64-apple-darwin" && !target.contains("windows") { cfg.define("LLVM_ENABLE_ZLIB", "ON"); } else { cfg.define("LLVM_ENABLE_ZLIB", "OFF"); diff --git a/src/bootstrap/sanity.rs b/src/bootstrap/sanity.rs index ed0cbdf97b..a28762ac48 100644 --- a/src/bootstrap/sanity.rs +++ b/src/bootstrap/sanity.rs @@ -93,7 +93,8 @@ pub fn check(build: &mut Build) { .unwrap_or(true) }) .any(|build_llvm_ourselves| build_llvm_ourselves); - if building_llvm || build.config.any_sanitizers_enabled() { + let need_cmake = building_llvm || build.config.any_sanitizers_enabled(); + if need_cmake { cmd_finder.must_have("cmake"); } @@ -204,7 +205,7 @@ pub fn check(build: &mut Build) { } } - if target.contains("msvc") { + if need_cmake && target.contains("msvc") { // There are three builds of cmake on windows: MSVC, MinGW, and // Cygwin. The Cygwin build does not have generators for Visual // Studio, so detect that here and error. diff --git a/src/bootstrap/test.rs b/src/bootstrap/test.rs index 965d116214..fe4666effe 100644 --- a/src/bootstrap/test.rs +++ b/src/bootstrap/test.rs @@ -183,6 +183,7 @@ impl Step for Cargotest { builder, cmd.arg(&cargo) .arg(&out_dir) + .args(builder.config.cmd.test_args()) .env("RUSTC", builder.rustc(compiler)) .env("RUSTDOC", builder.rustdoc(compiler)), ); @@ -318,15 +319,9 @@ impl Step for Rustfmt { let host = self.host; let compiler = builder.compiler(stage, host); - let build_result = builder.ensure(tool::Rustfmt { - compiler, - target: self.host, - extra_features: Vec::new(), - }); - if build_result.is_none() { - eprintln!("failed to test rustfmt: could not build"); - return; - } + builder + .ensure(tool::Rustfmt { compiler, target: self.host, extra_features: Vec::new() }) + .expect("in-tree tool"); let mut cargo = tool::prepare_tool_cargo( builder, @@ -345,9 +340,7 @@ impl Step for Rustfmt { cargo.add_rustc_lib_path(builder, compiler); - if try_run(builder, &mut cargo.into()) { - builder.save_toolstate("rustfmt", ToolState::TestPass); - } + builder.run(&mut cargo.into()); } } @@ -456,6 +449,7 @@ impl Step for Miri { SourceType::Submodule, &[], ); + cargo.add_rustc_lib_path(builder, compiler); cargo.arg("--").arg("miri").arg("setup"); // Tell `cargo miri setup` where to find the sources. @@ -507,6 +501,7 @@ impl Step for Miri { SourceType::Submodule, &[], ); + cargo.add_rustc_lib_path(builder, compiler); // miri tests need to know about the stage sysroot cargo.env("MIRI_SYSROOT", miri_sysroot); @@ -515,8 +510,6 @@ impl Step for Miri { cargo.arg("--").args(builder.config.cmd.test_args()); - cargo.add_rustc_lib_path(builder, compiler); - let mut cargo = Command::from(cargo); if !try_run(builder, &mut cargo) { return; @@ -781,6 +774,24 @@ impl Step for RustdocJSNotStd { } } +fn check_if_browser_ui_test_is_installed_global(npm: &Path, global: bool) -> bool { + let mut command = Command::new(&npm); + command.arg("list").arg("--depth=0"); + if global { + command.arg("--global"); + } + let lines = command + .output() + .map(|output| String::from_utf8_lossy(&output.stdout).into_owned()) + .unwrap_or(String::new()); + lines.contains(&" browser-ui-test@") +} + +fn check_if_browser_ui_test_is_installed(npm: &Path) -> bool { + check_if_browser_ui_test_is_installed_global(npm, false) + || check_if_browser_ui_test_is_installed_global(npm, true) +} + #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] pub struct RustdocGUI { pub target: TargetSelection, @@ -793,7 +804,17 @@ impl Step for RustdocGUI { const ONLY_HOSTS: bool = true; fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.path("src/test/rustdoc-gui") + let builder = run.builder; + let run = run.path("src/test/rustdoc-gui"); + run.default_condition( + builder.config.nodejs.is_some() + && builder + .config + .npm + .as_ref() + .map(|p| check_if_browser_ui_test_is_installed(p)) + .unwrap_or(false), + ) } fn make_run(run: RunConfig<'_>) { @@ -802,54 +823,54 @@ impl Step for RustdocGUI { } fn run(self, builder: &Builder<'_>) { - if let (Some(nodejs), Some(npm)) = (&builder.config.nodejs, &builder.config.npm) { - builder.ensure(compile::Std { compiler: self.compiler, target: self.target }); - - // The goal here is to check if the necessary packages are installed, and if not, we - // display a warning and move on. - let mut command = Command::new(&npm); - command.arg("list").arg("--depth=0"); - let lines = command - .output() - .map(|output| String::from_utf8_lossy(&output.stdout).to_string()) - .unwrap_or(String::new()); - if !lines.contains(&" browser-ui-test@") { - println!( - "warning: rustdoc-gui test suite cannot be run because npm `browser-ui-test` \ - dependency is missing", - ); - println!( - "If you want to install the `{0}` dependency, run `npm install {0}`", - "browser-ui-test", - ); - return; - } + let nodejs = builder.config.nodejs.as_ref().expect("nodejs isn't available"); + let npm = builder.config.npm.as_ref().expect("npm isn't available"); - let out_dir = builder.test_out(self.target).join("rustdoc-gui"); - let mut command = builder.rustdoc_cmd(self.compiler); - command.arg("src/test/rustdoc-gui/lib.rs").arg("-o").arg(&out_dir); - builder.run(&mut command); + builder.ensure(compile::Std { compiler: self.compiler, target: self.target }); - for file in fs::read_dir("src/test/rustdoc-gui").unwrap() { - let file = file.unwrap(); - let file_path = file.path(); - let file_name = file.file_name(); + // The goal here is to check if the necessary packages are installed, and if not, we + // panic. + if !check_if_browser_ui_test_is_installed(&npm) { + eprintln!( + "error: rustdoc-gui test suite cannot be run because npm `browser-ui-test` \ + dependency is missing", + ); + eprintln!( + "If you want to install the `{0}` dependency, run `npm install {0}`", + "browser-ui-test", + ); + panic!("Cannot run rustdoc-gui tests"); + } - if !file_name.to_str().unwrap().ends_with(".goml") { - continue; - } - let mut command = Command::new(&nodejs); - command - .arg("src/tools/rustdoc-gui/tester.js") - .arg("--doc-folder") - .arg(out_dir.join("test_docs")) - .arg("--test-file") - .arg(file_path); + let out_dir = builder.test_out(self.target).join("rustdoc-gui"); + + // We remove existing folder to be sure there won't be artifacts remaining. + let _ = fs::remove_dir_all(&out_dir); + + let mut nb_generated = 0; + // We generate docs for the libraries present in the rustdoc-gui's src folder. + let libs_dir = builder.build.src.join("src/test/rustdoc-gui/src"); + for entry in libs_dir.read_dir().expect("read_dir call failed") { + let entry = entry.expect("invalid entry"); + let path = entry.path(); + if path.extension().map(|e| e == "rs").unwrap_or(false) { + let mut command = builder.rustdoc_cmd(self.compiler); + command.arg(path).arg("-o").arg(&out_dir); builder.run(&mut command); + nb_generated += 1; } - } else { - builder.info("No nodejs found, skipping \"src/test/rustdoc-gui\" tests"); } + assert!(nb_generated > 0, "no documentation was generated..."); + + // We now run GUI tests. + let mut command = Command::new(&nodejs); + command + .arg(builder.build.src.join("src/tools/rustdoc-gui/tester.js")) + .arg("--doc-folder") + .arg(out_dir) + .arg("--tests-folder") + .arg(builder.build.src.join("src/test/rustdoc-gui")); + builder.run(&mut command); } } @@ -897,7 +918,7 @@ help: to skip test's attempt to check tidiness, pass `--exclude src/tools/tidy` ); std::process::exit(1); } - crate::format::format(&builder.build, !builder.config.cmd.bless()); + crate::format::format(&builder.build, !builder.config.cmd.bless(), &[]); } } @@ -1227,6 +1248,11 @@ note: if you're sure you want to do this, please open an issue as to why. In the cmd.arg(pass); } + if let Some(ref run) = builder.config.cmd.run() { + cmd.arg("--run"); + cmd.arg(run); + } + if let Some(ref nodejs) = builder.config.nodejs { cmd.arg("--nodejs").arg(nodejs); } @@ -1241,7 +1267,6 @@ note: if you're sure you want to do this, please open an issue as to why. In the } } flags.push(format!("-Cdebuginfo={}", builder.config.rust_debuginfo_level_tests)); - flags.push("-Zunstable-options".to_string()); flags.push(builder.config.cmd.rustc_args().join(" ")); if let Some(linker) = builder.linker(target) { @@ -1250,18 +1275,12 @@ note: if you're sure you want to do this, please open an issue as to why. In the let mut hostflags = flags.clone(); hostflags.push(format!("-Lnative={}", builder.test_helpers_out(compiler.host).display())); - if builder.is_fuse_ld_lld(compiler.host) { - hostflags.push("-Clink-args=-fuse-ld=lld".to_string()); - hostflags.push("-Clink-arg=-Wl,--threads=1".to_string()); - } + hostflags.extend(builder.lld_flags(compiler.host)); cmd.arg("--host-rustcflags").arg(hostflags.join(" ")); let mut targetflags = flags; targetflags.push(format!("-Lnative={}", builder.test_helpers_out(target).display())); - if builder.is_fuse_ld_lld(target) { - targetflags.push("-Clink-args=-fuse-ld=lld".to_string()); - targetflags.push("-Clink-arg=-Wl,--threads=1".to_string()); - } + targetflags.extend(builder.lld_flags(target)); cmd.arg("--target-rustcflags").arg(targetflags.join(" ")); cmd.arg("--docck-python").arg(builder.python()); @@ -1453,6 +1472,7 @@ note: if you're sure you want to do this, please open an issue as to why. In the } } cmd.env("RUSTC_BOOTSTRAP", "1"); + cmd.env("DOC_RUST_LANG_ORG_CHANNEL", builder.doc_rust_lang_org_channel()); builder.add_rust_test_threads(&mut cmd); if builder.config.sanitizers_enabled(target) { @@ -1483,6 +1503,8 @@ note: if you're sure you want to do this, please open an issue as to why. In the cmd.env("BOOTSTRAP_CARGO", &builder.initial_cargo); + cmd.arg("--channel").arg(&builder.config.channel); + builder.ci_env.force_coloring_in_ci(&mut cmd); builder.info(&format!( @@ -1709,6 +1731,9 @@ fn markdown_test(builder: &Builder<'_>, compiler: Compiler, markdown: &Path) -> builder.info(&format!("doc tests for: {}", markdown.display())); let mut cmd = builder.rustdoc_cmd(compiler); builder.add_rust_test_threads(&mut cmd); + // allow for unstable options such as new editions + cmd.arg("-Z"); + cmd.arg("unstable-options"); cmd.arg("--test"); cmd.arg(markdown); cmd.env("RUSTC_BOOTSTRAP", "1"); diff --git a/src/bootstrap/tool.rs b/src/bootstrap/tool.rs index e85f4628fb..9d75ad0918 100644 --- a/src/bootstrap/tool.rs +++ b/src/bootstrap/tool.rs @@ -52,7 +52,10 @@ impl Step for ToolBuild { let is_optional_tool = self.is_optional_tool; match self.mode { - Mode::ToolRustc => builder.ensure(compile::Rustc { compiler, target }), + Mode::ToolRustc => { + builder.ensure(compile::Std { compiler, target: compiler.host }); + builder.ensure(compile::Rustc { compiler, target }); + } Mode::ToolStd => builder.ensure(compile::Std { compiler, target }), Mode::ToolBootstrap => {} // uses downloaded stage0 compiler libs _ => panic!("unexpected Mode for tool build"), @@ -214,9 +217,8 @@ impl Step for ToolBuild { if tool == "tidy" { tool = "rust-tidy"; } - let cargo_out = - builder.cargo_out(compiler, self.mode, target).join(exe(tool, compiler.host)); - let bin = builder.tools_dir(compiler).join(exe(tool, compiler.host)); + let cargo_out = builder.cargo_out(compiler, self.mode, target).join(exe(tool, target)); + let bin = builder.tools_dir(compiler).join(exe(tool, target)); builder.copy(&cargo_out, &bin); Some(bin) } @@ -261,6 +263,7 @@ pub fn prepare_tool_cargo( cargo.env("CFG_RELEASE_CHANNEL", &builder.config.channel); cargo.env("CFG_VERSION", builder.rust_version()); cargo.env("CFG_RELEASE_NUM", &builder.version); + cargo.env("DOC_RUST_LANG_ORG_CHANNEL", builder.doc_rust_lang_org_channel()); let info = GitInfo::new(builder.config.ignore_git, &dir); if let Some(sha) = info.sha() { @@ -392,7 +395,10 @@ impl ErrorIndex { let compiler = builder.compiler(builder.top_stage.saturating_sub(1), builder.config.build); let mut cmd = Command::new(builder.ensure(ErrorIndex { compiler })); add_dylib_path( - vec![PathBuf::from(&builder.sysroot_libdir(compiler, compiler.host))], + vec![ + PathBuf::from(&builder.sysroot_libdir(compiler, compiler.host)), + PathBuf::from(builder.rustc_libdir(compiler)), + ], &mut cmd, ); cmd @@ -590,7 +596,14 @@ impl Step for Cargo { fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { let builder = run.builder; - run.path("src/tools/cargo").default_condition(builder.config.extended) + run.path("src/tools/cargo").default_condition( + builder.config.extended + && builder.config.tools.as_ref().map_or( + true, + // If `tools` is set, search list for this tool. + |tools| tools.iter().any(|tool| tool == "cargo"), + ), + ) } fn make_run(run: RunConfig<'_>) { @@ -723,7 +736,7 @@ macro_rules! tool_extended { // Note: tools need to be also added to `Builder::get_step_descriptions` in `builder.rs` // to make `./x.py build ` work. tool_extended!((self, builder), - Cargofmt, rustfmt, "src/tools/rustfmt", "cargo-fmt", stable=true, {}; + Cargofmt, rustfmt, "src/tools/rustfmt", "cargo-fmt", stable=true, in_tree=true, {}; CargoClippy, clippy, "src/tools/clippy", "cargo-clippy", stable=true, in_tree=true, {}; Clippy, clippy, "src/tools/clippy", "clippy-driver", stable=true, in_tree=true, {}; Miri, miri, "src/tools/miri", "miri", stable=false, {}; @@ -737,7 +750,7 @@ tool_extended!((self, builder), self.extra_features.push("clippy".to_owned()); }; RustDemangler, rust_demangler, "src/tools/rust-demangler", "rust-demangler", stable=false, in_tree=true, {}; - Rustfmt, rustfmt, "src/tools/rustfmt", "rustfmt", stable=true, {}; + Rustfmt, rustfmt, "src/tools/rustfmt", "rustfmt", stable=true, in_tree=true, {}; RustAnalyzer, rust_analyzer, "src/tools/rust-analyzer/crates/rust-analyzer", "rust-analyzer", stable=false, {}; ); diff --git a/src/bootstrap/toolstate.rs b/src/bootstrap/toolstate.rs index 205524ad84..2394c5e020 100644 --- a/src/bootstrap/toolstate.rs +++ b/src/bootstrap/toolstate.rs @@ -77,7 +77,6 @@ static STABLE_TOOLS: &[(&str, &str)] = &[ ("rust-by-example", "src/doc/rust-by-example"), ("edition-guide", "src/doc/edition-guide"), ("rls", "src/tools/rls"), - ("rustfmt", "src/tools/rustfmt"), ]; // These tools are permitted to not build on the beta/stable channels. @@ -278,10 +277,9 @@ impl Builder<'_> { if self.config.dry_run { return; } - // Toolstate isn't tracked for clippy, but since most tools do, we avoid - // checking in all the places we could save toolstate and just do so - // here. - if tool == "clippy-driver" { + // Toolstate isn't tracked for clippy or rustfmt, but since most tools do, we avoid checking + // in all the places we could save toolstate and just do so here. + if tool == "clippy-driver" || tool == "rustfmt" { return; } if let Some(ref path) = self.config.save_toolstates { diff --git a/src/bootstrap/util.rs b/src/bootstrap/util.rs index b4421a8271..112979b0be 100644 --- a/src/bootstrap/util.rs +++ b/src/bootstrap/util.rs @@ -45,6 +45,7 @@ pub fn libdir(target: TargetSelection) -> &'static str { } /// Adds a list of lookup paths to `cmd`'s dynamic library lookup path. +/// If The dylib_path_par is already set for this cmd, the old value will be overwritten! pub fn add_dylib_path(path: Vec, cmd: &mut Command) { let mut list = dylib_path(); for path in path { @@ -306,5 +307,6 @@ pub fn use_host_linker(target: TargetSelection) -> bool { || target.contains("wasm32") || target.contains("nvptx") || target.contains("fortanix") - || target.contains("fuchsia")) + || target.contains("fuchsia") + || target.contains("bpf")) } diff --git a/src/ci/channel b/src/ci/channel new file mode 100644 index 0000000000..2bf5ad0447 --- /dev/null +++ b/src/ci/channel @@ -0,0 +1 @@ +stable diff --git a/src/ci/docker/host-x86_64/dist-riscv64-linux/riscv64-unknown-linux-gnu.config b/src/ci/docker/host-x86_64/dist-riscv64-linux/riscv64-unknown-linux-gnu.config index 7124ee3aeb..ffb2e9195b 100644 --- a/src/ci/docker/host-x86_64/dist-riscv64-linux/riscv64-unknown-linux-gnu.config +++ b/src/ci/docker/host-x86_64/dist-riscv64-linux/riscv64-unknown-linux-gnu.config @@ -644,7 +644,7 @@ CT_EXPAT_PATCH_GLOBAL=y CT_EXPAT_PATCH_ORDER="global" CT_EXPAT_V_2_2=y # CT_EXPAT_NO_VERSIONS is not set -CT_EXPAT_VERSION="2.3.0" +CT_EXPAT_VERSION="2.4.1" CT_EXPAT_MIRRORS="http://downloads.sourceforge.net/project/expat/expat/${CT_EXPAT_VERSION}" CT_EXPAT_ARCHIVE_FILENAME="@{pkg_name}-@{version}" CT_EXPAT_ARCHIVE_DIRNAME="@{pkg_name}-@{version}" diff --git a/src/ci/docker/host-x86_64/mingw-check/Dockerfile b/src/ci/docker/host-x86_64/mingw-check/Dockerfile index a9398649cf..66afe84be4 100644 --- a/src/ci/docker/host-x86_64/mingw-check/Dockerfile +++ b/src/ci/docker/host-x86_64/mingw-check/Dockerfile @@ -22,6 +22,7 @@ ENV PATH="/node-v14.4.0-linux-x64/bin:${PATH}" # Install es-check # Pin its version to prevent unrelated CI failures due to future es-check versions. RUN npm install es-check@5.2.3 -g +RUN npm install eslint@7.20.0 -g COPY scripts/sccache.sh /scripts/ RUN sh /scripts/sccache.sh @@ -34,7 +35,8 @@ ENV SCRIPT python3 ../x.py --stage 2 test src/tools/expand-yaml-anchors && \ python3 ../x.py build --stage 0 src/tools/build-manifest && \ python3 ../x.py test --stage 0 src/tools/compiletest && \ python3 ../x.py test --stage 2 src/tools/tidy && \ - python3 ../x.py doc --stage 0 library/std && \ + python3 ../x.py doc --stage 0 library/test && \ /scripts/validate-toolstate.sh && \ # Runs checks to ensure that there are no ES5 issues in our JS code. - es-check es5 ../src/librustdoc/html/static/*.js + es-check es5 ../src/librustdoc/html/static/*.js && \ + eslint ../src/librustdoc/html/static/*.js diff --git a/src/ci/docker/host-x86_64/x86_64-gnu-aux/Dockerfile b/src/ci/docker/host-x86_64/x86_64-gnu-aux/Dockerfile index 00ad7b0a71..7f1a5820e2 100644 --- a/src/ci/docker/host-x86_64/x86_64-gnu-aux/Dockerfile +++ b/src/ci/docker/host-x86_64/x86_64-gnu-aux/Dockerfile @@ -17,27 +17,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ libgl1-mesa-dev \ llvm-dev \ libfreetype6-dev \ - libexpat1-dev \ - libexpat1-dev \ - gnupg \ - apt-utils \ - wget \ - fonts-ipafont-gothic \ - fonts-wqy-zenhei \ - fonts-thai-tlwg \ - fonts-kacst \ - fonts-freefont-ttf \ - libxss1 \ - libxtst6 - -RUN curl -sL https://nodejs.org/dist/v14.4.0/node-v14.4.0-linux-x64.tar.xz | tar -xJ -ENV PATH="/node-v14.4.0-linux-x64/bin:${PATH}" - -# Install required dependencies from browser-UI-test framework -# For now, we need to use `--unsafe-perm=true` to go around an issue when npm tries -# to create a new folder. For reference: -# https://github.com/puppeteer/puppeteer/issues/375 -RUN npm install browser-ui-test -g --unsafe-perm=true + libexpat1-dev COPY scripts/sccache.sh /scripts/ RUN sh /scripts/sccache.sh @@ -46,4 +26,4 @@ COPY scripts/cmake.sh /scripts/ RUN /scripts/cmake.sh ENV RUST_CONFIGURE_ARGS --build=x86_64-unknown-linux-gnu -ENV RUST_CHECK_TARGET check-aux-and-gui +ENV RUST_CHECK_TARGET check-aux diff --git a/src/ci/docker/host-x86_64/x86_64-gnu-tools/Dockerfile b/src/ci/docker/host-x86_64/x86_64-gnu-tools/Dockerfile index d4838c0d6f..ee7c223ba5 100644 --- a/src/ci/docker/host-x86_64/x86_64-gnu-tools/Dockerfile +++ b/src/ci/docker/host-x86_64/x86_64-gnu-tools/Dockerfile @@ -12,8 +12,48 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ cmake \ libssl-dev \ sudo \ - xz-utils \ - pkg-config + xz-utils + +# Install dependencies for chromium browser +RUN apt-get install -y \ + gconf-service \ + libasound2 \ + libatk1.0-0 \ + libatk-bridge2.0-0 \ + libc6 \ + libcairo2 \ + libcups2 \ + libdbus-1-3 \ + libexpat1 \ + libfontconfig1 \ + libgcc1 \ + libgconf-2-4 \ + libgdk-pixbuf2.0-0 \ + libglib2.0-0 \ + libgtk-3-0 \ + libnspr4 \ + libpango-1.0-0 \ + libpangocairo-1.0-0 \ + libstdc++6 \ + libx11-6 \ + libx11-xcb1 \ + libxcb1 \ + libxcomposite1 \ + libxcursor1 \ + libxdamage1 \ + libxext6 \ + libxfixes3 \ + libxi6 \ + libxrandr2 \ + libxrender1 \ + libxss1 \ + libxtst6 \ + fonts-liberation \ + libappindicator1 \ + libnss3 \ + lsb-release \ + xdg-utils \ + wget COPY scripts/sccache.sh /scripts/ RUN sh /scripts/sccache.sh @@ -23,7 +63,19 @@ RUN /scripts/cmake.sh COPY host-x86_64/x86_64-gnu-tools/checktools.sh /tmp/ +RUN curl -sL https://nodejs.org/dist/v14.4.0/node-v14.4.0-linux-x64.tar.xz | tar -xJ +ENV PATH="/node-v14.4.0-linux-x64/bin:${PATH}" + +# For now, we need to use `--unsafe-perm=true` to go around an issue when npm tries +# to create a new folder. For reference: +# https://github.com/puppeteer/puppeteer/issues/375 +# +# We also specify the version in case we need to update it to go around cache limitations. +RUN npm install -g browser-ui-test@0.4.2 --unsafe-perm=true + ENV RUST_CONFIGURE_ARGS \ --build=x86_64-unknown-linux-gnu \ --save-toolstates=/tmp/toolstate/toolstates.json -ENV SCRIPT /tmp/checktools.sh ../x.py + +ENV SCRIPT /tmp/checktools.sh ../x.py && \ + NODE_PATH=`npm root -g` python3 ../x.py test src/test/rustdoc-gui --stage 2 diff --git a/src/ci/docker/host-x86_64/x86_64-gnu-tools/checktools.sh b/src/ci/docker/host-x86_64/x86_64-gnu-tools/checktools.sh index 49a8e5e88a..412efe5c44 100755 --- a/src/ci/docker/host-x86_64/x86_64-gnu-tools/checktools.sh +++ b/src/ci/docker/host-x86_64/x86_64-gnu-tools/checktools.sh @@ -15,7 +15,6 @@ python3 "$X_PY" test --stage 2 --no-fail-fast \ src/doc/embedded-book \ src/doc/edition-guide \ src/tools/rls \ - src/tools/rustfmt \ src/tools/miri \ set -e @@ -24,3 +23,4 @@ set -e cat /tmp/toolstate/toolstates.json python3 "$X_PY" test --stage 2 check-tools python3 "$X_PY" test --stage 2 src/tools/clippy +python3 "$X_PY" test --stage 2 src/tools/rustfmt diff --git a/src/ci/docker/run.sh b/src/ci/docker/run.sh index c2ff62e748..3a47076722 100755 --- a/src/ci/docker/run.sh +++ b/src/ci/docker/run.sh @@ -235,6 +235,7 @@ docker \ --env TOOLSTATE_REPO_ACCESS_TOKEN \ --env TOOLSTATE_REPO \ --env TOOLSTATE_PUBLISH \ + --env RUST_CI_OVERRIDE_RELEASE_CHANNEL \ --env CI_JOB_NAME="${CI_JOB_NAME-$IMAGE}" \ --init \ --rm \ diff --git a/src/ci/github-actions/ci.yml b/src/ci/github-actions/ci.yml index a59a90b86b..e704071e40 100644 --- a/src/ci/github-actions/ci.yml +++ b/src/ci/github-actions/ci.yml @@ -126,6 +126,10 @@ x--expand-yaml-anchors--remove: run: src/ci/scripts/should-skip-this.sh <<: *step + - name: ensure the channel matches the target branch + run: src/ci/scripts/verify-channel.sh + <<: *step + - name: configure GitHub Actions to kill the build when outdated uses: rust-lang/simpleinfra/github-actions/cancel-outdated-builds@master with: @@ -403,6 +407,17 @@ jobs: - name: x86_64-gnu <<: *job-linux-xl + # This job ensures commits landing on nightly still pass the full + # test suite on the stable channel. There are some UI tests that + # depend on the channel being built (for example if they include the + # channel name on the output), and this builder prevents landing + # changes that would result in broken builds after a promotion. + - name: x86_64-gnu-stable + env: + IMAGE: x86_64-gnu + RUST_CI_OVERRIDE_RELEASE_CHANNEL: stable + <<: *job-linux-xl + - name: x86_64-gnu-aux <<: *job-linux-xl diff --git a/src/ci/init_repo.sh b/src/ci/init_repo.sh index 060b3079da..3c61dcc9d9 100755 --- a/src/ci/init_repo.sh +++ b/src/ci/init_repo.sh @@ -31,7 +31,7 @@ mkdir "$CACHE_DIR" # On the beta channel we'll be automatically calculating the prerelease version # via the git history, so unshallow our shallow clone from CI. -if grep -q RUST_RELEASE_CHANNEL=beta src/ci/run.sh; then +if [ "$(releaseChannel)" = "beta" ]; then git fetch origin --unshallow beta master fi diff --git a/src/ci/pgo.sh b/src/ci/pgo.sh index ad2a8c771d..c3c717266d 100755 --- a/src/ci/pgo.sh +++ b/src/ci/pgo.sh @@ -12,7 +12,7 @@ RUSTC_BOOTSTRAP=1 ./build/$PGO_HOST/stage2/bin/rustc --edition=2018 \ # Download and build a single-file stress test benchmark on perf.rust-lang.org. function pgo_perf_benchmark { - local PERF=9442def56a39d742bf27ebcc3e0614cf117e1bc2 + local PERF=1e19fc4c6168d2f7596e512f42f358f245d8f09d local github_prefix=https://raw.githubusercontent.com/rust-lang/rustc-perf/$PERF local name=$1 curl -o /tmp/$name.rs $github_prefix/collector/benchmarks/$name/src/lib.rs diff --git a/src/ci/run.sh b/src/ci/run.sh index a408fa83e5..b5019d83af 100755 --- a/src/ci/run.sh +++ b/src/ci/run.sh @@ -62,17 +62,10 @@ if [ "$DIST_SRC" = "" ]; then RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --disable-dist-src" fi -# If we're deploying artifacts then we set the release channel, otherwise if -# we're not deploying then we want to be sure to enable all assertions because -# we'll be running tests -# -# FIXME: need a scheme for changing this `nightly` value to `beta` and `stable` -# either automatically or manually. -export RUST_RELEASE_CHANNEL=stable - # Always set the release channel for bootstrap; this is normally not important (i.e., only dist # builds would seem to matter) but in practice bootstrap wants to know whether we're targeting # master, beta, or stable with a build to determine whether to run some checks (notably toolstate). +export RUST_RELEASE_CHANNEL=$(releaseChannel) RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --release-channel=$RUST_RELEASE_CHANNEL" if [ "$DEPLOY$DEPLOY_ALT" = "1" ]; then diff --git a/src/ci/scripts/install-clang.sh b/src/ci/scripts/install-clang.sh index 8a30acc2e4..7b540b5c6e 100755 --- a/src/ci/scripts/install-clang.sh +++ b/src/ci/scripts/install-clang.sh @@ -18,7 +18,8 @@ if isMacOS; then bindir="$(xcode-select --print-path)/Toolchains/XcodeDefault.xctoolchain/usr/bin" else file="${MIRRORS_BASE}/clang%2Bllvm-${LLVM_VERSION}-x86_64-apple-darwin.tar.xz" - curl -f "${file}" | tar xJf - + retry curl -f "${file}" -o "clang+llvm-${LLVM_VERSION}-x86_64-apple-darwin.tar.xz" + tar xJf "clang+llvm-${LLVM_VERSION}-x86_64-apple-darwin.tar.xz" bindir="$(pwd)/clang+llvm-${LLVM_VERSION}-x86_64-apple-darwin/bin" fi @@ -48,7 +49,8 @@ elif isWindows && [[ ${CUSTOM_MINGW-0} -ne 1 ]]; then mkdir -p citools/clang-rust cd citools - curl -f "${MIRRORS_BASE}/LLVM-${LLVM_VERSION}-win64.exe" -o "LLVM-${LLVM_VERSION}-win64.exe" + retry curl -f "${MIRRORS_BASE}/LLVM-${LLVM_VERSION}-win64.exe" \ + -o "LLVM-${LLVM_VERSION}-win64.exe" 7z x -oclang-rust/ "LLVM-${LLVM_VERSION}-win64.exe" ciCommandSetEnv RUST_CONFIGURE_ARGS \ "${RUST_CONFIGURE_ARGS} --set llvm.clang-cl=$(pwd)/clang-rust/bin/clang-cl.exe" diff --git a/src/ci/scripts/should-skip-this.sh b/src/ci/scripts/should-skip-this.sh index 36bf436899..fa738fe70c 100755 --- a/src/ci/scripts/should-skip-this.sh +++ b/src/ci/scripts/should-skip-this.sh @@ -10,15 +10,29 @@ source "$(cd "$(dirname "$0")" && pwd)/../shared.sh" if [[ -z "${CI_ONLY_WHEN_SUBMODULES_CHANGED+x}" ]]; then echo "Executing the job since there is no skip rule in effect" -elif git diff HEAD^ | grep --quiet "^index .* 160000"; then + exit 0 +fi + +git fetch "https://github.com/$GITHUB_REPOSITORY" "$GITHUB_BASE_REF" +BASE_COMMIT="$(git merge-base FETCH_HEAD HEAD)" + +echo "Searching for toolstate changes between $BASE_COMMIT and $(git rev-parse HEAD)" + +if git diff "$BASE_COMMIT" | grep --quiet "^index .* 160000"; then # Submodules pseudo-files inside git have the 160000 permissions, so when # those files are present in the diff a submodule was updated. echo "Executing the job since submodules are updated" -elif git diff --name-only HEAD^ | grep --quiet src/tools/clippy; then +elif ! git diff --quiet "$BASE_COMMIT" -- src/tools/clippy src/tools/rustfmt; then # There is not an easy blanket search for subtrees. For now, manually list - # clippy. - echo "Executing the job since clippy subtree was updated" + # the subtrees. + echo "Executing the job since clippy or rustfmt subtree was updated" +elif ! (git diff --quiet "$BASE_COMMIT" -- \ + src/test/rustdoc-gui \ + src/librustdoc \ + src/tools/rustdoc-gui); then + # There was a change in either rustdoc or in its GUI tests. + echo "Executing the job since rustdoc was updated" else - echo "Not executing this job since no submodules were updated" + echo "Not executing this job since no submodules nor subtrees were updated" ciCommandSetEnv SKIP_JOB 1 fi diff --git a/src/ci/scripts/verify-channel.sh b/src/ci/scripts/verify-channel.sh new file mode 100755 index 0000000000..d02dc362c6 --- /dev/null +++ b/src/ci/scripts/verify-channel.sh @@ -0,0 +1,38 @@ +#!/bin/bash +# We want to make sure all PRs are targeting the right branch when they're +# opened, otherwise we risk (for example) to land a beta-specific change to the +# master branch. This script ensures the branch of the PR matches the channel. + +set -euo pipefail +IFS=$'\n\t' + +source "$(cd "$(dirname "$0")" && pwd)/../shared.sh" + +if isCiBranch auto || isCiBranch try; then + echo "channel verification is only executed on PR builds" + exit +fi + +channel=$(cat "$(ciCheckoutPath)/src/ci/channel") +case "${channel}" in + nightly) + channel_branch="master" + ;; + beta) + channel_branch="beta" + ;; + stable) + channel_branch="stable" + ;; + *) + echo "error: unknown channel defined in src/ci/channel: ${channel}" + exit 1 +esac + +branch="$(ciBaseBranch)" +if [[ "${branch}" != "${channel_branch}" ]]; then + echo "error: PRs changing the \`${channel}\` channel should be sent to the \ +\`${channel_branch}\` branch!" + + exit 1 +fi diff --git a/src/ci/shared.sh b/src/ci/shared.sh index 3c196c9478..b095afb542 100644 --- a/src/ci/shared.sh +++ b/src/ci/shared.sh @@ -73,6 +73,18 @@ function isCiBranch { fi } +function ciBaseBranch { + if isAzurePipelines; then + echo "unsupported on Azure Pipelines" + exit 1 + elif isGitHubActions; then + echo "${GITHUB_BASE_REF#refs/heads/}" + else + echo "ciBaseBranch only works inside CI!" + exit 1 + fi +} + function ciCommit { if isAzurePipelines; then echo "${BUILD_SOURCEVERSION}" @@ -129,3 +141,11 @@ function ciCommandSetEnv { exit 1 fi } + +function releaseChannel { + if [[ -z "${RUST_CI_OVERRIDE_RELEASE_CHANNEL+x}" ]]; then + cat "${ci_dir}/channel" + else + echo $RUST_CI_OVERRIDE_RELEASE_CHANNEL + fi +} diff --git a/src/doc/book/.github/workflows/main.yml b/src/doc/book/.github/workflows/main.yml index 48eb4e25ae..3c76a73ae0 100644 --- a/src/doc/book/.github/workflows/main.yml +++ b/src/doc/book/.github/workflows/main.yml @@ -12,8 +12,8 @@ jobs: - name: Install Rust run: | rustup set profile minimal - rustup toolchain install 1.50 -c rust-docs - rustup default 1.50 + rustup toolchain install 1.52 -c rust-docs + rustup default 1.52 - name: Install mdbook run: | mkdir bin diff --git a/src/doc/book/listings/ch02-guessing-game-tutorial/no-listing-02-without-expect/output.txt b/src/doc/book/listings/ch02-guessing-game-tutorial/no-listing-02-without-expect/output.txt index 38c695dba1..c56c88971d 100644 --- a/src/doc/book/listings/ch02-guessing-game-tutorial/no-listing-02-without-expect/output.txt +++ b/src/doc/book/listings/ch02-guessing-game-tutorial/no-listing-02-without-expect/output.txt @@ -1,6 +1,6 @@ $ cargo build Compiling guessing_game v0.1.0 (file:///projects/guessing_game) -warning: unused `std::result::Result` that must be used +warning: unused `Result` that must be used --> src/main.rs:10:5 | 10 | io::stdin().read_line(&mut guess); diff --git a/src/doc/book/listings/ch03-common-programming-concepts/no-listing-19-statements-vs-expressions/output.txt b/src/doc/book/listings/ch03-common-programming-concepts/no-listing-19-statements-vs-expressions/output.txt index d4bb2b2ea3..7409e856ed 100644 --- a/src/doc/book/listings/ch03-common-programming-concepts/no-listing-19-statements-vs-expressions/output.txt +++ b/src/doc/book/listings/ch03-common-programming-concepts/no-listing-19-statements-vs-expressions/output.txt @@ -7,6 +7,7 @@ error[E0658]: `let` expressions in this position are experimental | ^^^^^^^^^ | = note: see issue #53667 for more information + = help: you can write `matches!(, )` instead of `let = ` error: expected expression, found statement (`let`) --> src/main.rs:2:14 diff --git a/src/doc/book/listings/ch09-error-handling/no-listing-02-ask-compiler-for-type/output.txt b/src/doc/book/listings/ch09-error-handling/no-listing-02-ask-compiler-for-type/output.txt index 1b63b37cfd..6436fe6774 100644 --- a/src/doc/book/listings/ch09-error-handling/no-listing-02-ask-compiler-for-type/output.txt +++ b/src/doc/book/listings/ch09-error-handling/no-listing-02-ask-compiler-for-type/output.txt @@ -4,12 +4,12 @@ error[E0308]: mismatched types --> src/main.rs:4:18 | 4 | let f: u32 = File::open("hello.txt"); - | --- ^^^^^^^^^^^^^^^^^^^^^^^ expected `u32`, found enum `std::result::Result` + | --- ^^^^^^^^^^^^^^^^^^^^^^^ expected `u32`, found enum `Result` | | | expected due to this | = note: expected type `u32` - found enum `std::result::Result` + found enum `Result` error: aborting due to previous error diff --git a/src/doc/book/listings/ch11-writing-automated-tests/listing-11-01/output.txt b/src/doc/book/listings/ch11-writing-automated-tests/listing-11-01/output.txt index 5396eba5b8..c07ffd25d6 100644 --- a/src/doc/book/listings/ch11-writing-automated-tests/listing-11-01/output.txt +++ b/src/doc/book/listings/ch11-writing-automated-tests/listing-11-01/output.txt @@ -1,7 +1,7 @@ $ cargo test Compiling adder v0.1.0 (file:///projects/adder) Finished test [unoptimized + debuginfo] target(s) in 0.57s - Running target/debug/deps/adder-92948b65e88960b4 + Running unittests (target/debug/deps/adder-92948b65e88960b4) running 1 test test tests::it_works ... ok diff --git a/src/doc/book/listings/ch11-writing-automated-tests/listing-11-03/output.txt b/src/doc/book/listings/ch11-writing-automated-tests/listing-11-03/output.txt index 4b2a589f70..9680e7c208 100644 --- a/src/doc/book/listings/ch11-writing-automated-tests/listing-11-03/output.txt +++ b/src/doc/book/listings/ch11-writing-automated-tests/listing-11-03/output.txt @@ -1,7 +1,7 @@ $ cargo test Compiling adder v0.1.0 (file:///projects/adder) Finished test [unoptimized + debuginfo] target(s) in 0.72s - Running target/debug/deps/adder-92948b65e88960b4 + Running unittests (target/debug/deps/adder-92948b65e88960b4) running 2 tests test tests::another ... FAILED diff --git a/src/doc/book/listings/ch11-writing-automated-tests/listing-11-06/output.txt b/src/doc/book/listings/ch11-writing-automated-tests/listing-11-06/output.txt index cb32c7feac..820c75b1ab 100644 --- a/src/doc/book/listings/ch11-writing-automated-tests/listing-11-06/output.txt +++ b/src/doc/book/listings/ch11-writing-automated-tests/listing-11-06/output.txt @@ -1,7 +1,7 @@ $ cargo test Compiling rectangle v0.1.0 (file:///projects/rectangle) Finished test [unoptimized + debuginfo] target(s) in 0.66s - Running target/debug/deps/rectangle-6584c4561e48942e + Running unittests (target/debug/deps/rectangle-6584c4561e48942e) running 1 test test tests::larger_can_hold_smaller ... ok diff --git a/src/doc/book/listings/ch11-writing-automated-tests/listing-11-07/output.txt b/src/doc/book/listings/ch11-writing-automated-tests/listing-11-07/output.txt index 499ead4808..aa30cdb629 100644 --- a/src/doc/book/listings/ch11-writing-automated-tests/listing-11-07/output.txt +++ b/src/doc/book/listings/ch11-writing-automated-tests/listing-11-07/output.txt @@ -1,7 +1,7 @@ $ cargo test Compiling adder v0.1.0 (file:///projects/adder) Finished test [unoptimized + debuginfo] target(s) in 0.58s - Running target/debug/deps/adder-92948b65e88960b4 + Running unittests (target/debug/deps/adder-92948b65e88960b4) running 1 test test tests::it_adds_two ... ok diff --git a/src/doc/book/listings/ch11-writing-automated-tests/listing-11-08/output.txt b/src/doc/book/listings/ch11-writing-automated-tests/listing-11-08/output.txt index a75b3cb9d8..8372eeedf3 100644 --- a/src/doc/book/listings/ch11-writing-automated-tests/listing-11-08/output.txt +++ b/src/doc/book/listings/ch11-writing-automated-tests/listing-11-08/output.txt @@ -1,7 +1,7 @@ $ cargo test Compiling guessing_game v0.1.0 (file:///projects/guessing_game) Finished test [unoptimized + debuginfo] target(s) in 0.58s - Running target/debug/deps/guessing_game-57d70c3acb738f4d + Running unittests (target/debug/deps/guessing_game-57d70c3acb738f4d) running 1 test test tests::greater_than_100 ... ok diff --git a/src/doc/book/listings/ch11-writing-automated-tests/listing-11-10/output.txt b/src/doc/book/listings/ch11-writing-automated-tests/listing-11-10/output.txt index 69ef7da765..ce983169d2 100644 --- a/src/doc/book/listings/ch11-writing-automated-tests/listing-11-10/output.txt +++ b/src/doc/book/listings/ch11-writing-automated-tests/listing-11-10/output.txt @@ -1,7 +1,7 @@ $ cargo test Compiling silly-function v0.1.0 (file:///projects/silly-function) Finished test [unoptimized + debuginfo] target(s) in 0.58s - Running target/debug/deps/silly_function-160869f38cff9166 + Running unittests (target/debug/deps/silly_function-160869f38cff9166) running 2 tests test tests::this_test_will_fail ... FAILED diff --git a/src/doc/book/listings/ch11-writing-automated-tests/listing-11-11/output.txt b/src/doc/book/listings/ch11-writing-automated-tests/listing-11-11/output.txt index 35f6e79160..9be5abe619 100644 --- a/src/doc/book/listings/ch11-writing-automated-tests/listing-11-11/output.txt +++ b/src/doc/book/listings/ch11-writing-automated-tests/listing-11-11/output.txt @@ -1,7 +1,7 @@ $ cargo test Compiling adder v0.1.0 (file:///projects/adder) Finished test [unoptimized + debuginfo] target(s) in 0.62s - Running target/debug/deps/adder-92948b65e88960b4 + Running unittests (target/debug/deps/adder-92948b65e88960b4) running 3 tests test tests::add_three_and_two ... ok diff --git a/src/doc/book/listings/ch11-writing-automated-tests/listing-11-13/output.txt b/src/doc/book/listings/ch11-writing-automated-tests/listing-11-13/output.txt index 6266b5085a..12e231cf26 100644 --- a/src/doc/book/listings/ch11-writing-automated-tests/listing-11-13/output.txt +++ b/src/doc/book/listings/ch11-writing-automated-tests/listing-11-13/output.txt @@ -1,14 +1,14 @@ $ cargo test Compiling adder v0.1.0 (file:///projects/adder) Finished test [unoptimized + debuginfo] target(s) in 1.31s - Running target/debug/deps/adder-1082c4b063a8fbe6 + Running unittests (target/debug/deps/adder-1082c4b063a8fbe6) running 1 test test tests::internal ... ok test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s - Running target/debug/deps/integration_test-1082c4b063a8fbe6 + Running tests/integration_test.rs (target/debug/deps/integration_test-1082c4b063a8fbe6) running 1 test test it_adds_two ... ok diff --git a/src/doc/book/listings/ch11-writing-automated-tests/no-listing-01-changing-test-name/output.txt b/src/doc/book/listings/ch11-writing-automated-tests/no-listing-01-changing-test-name/output.txt index 1a8c7f77f8..5de1386739 100644 --- a/src/doc/book/listings/ch11-writing-automated-tests/no-listing-01-changing-test-name/output.txt +++ b/src/doc/book/listings/ch11-writing-automated-tests/no-listing-01-changing-test-name/output.txt @@ -1,7 +1,7 @@ $ cargo test Compiling adder v0.1.0 (file:///projects/adder) Finished test [unoptimized + debuginfo] target(s) in 0.59s - Running target/debug/deps/adder-92948b65e88960b4 + Running unittests (target/debug/deps/adder-92948b65e88960b4) running 1 test test tests::exploration ... ok diff --git a/src/doc/book/listings/ch11-writing-automated-tests/no-listing-02-adding-another-rectangle-test/output.txt b/src/doc/book/listings/ch11-writing-automated-tests/no-listing-02-adding-another-rectangle-test/output.txt index 393c09a055..29320be50d 100644 --- a/src/doc/book/listings/ch11-writing-automated-tests/no-listing-02-adding-another-rectangle-test/output.txt +++ b/src/doc/book/listings/ch11-writing-automated-tests/no-listing-02-adding-another-rectangle-test/output.txt @@ -1,7 +1,7 @@ $ cargo test Compiling rectangle v0.1.0 (file:///projects/rectangle) Finished test [unoptimized + debuginfo] target(s) in 0.66s - Running target/debug/deps/rectangle-6584c4561e48942e + Running unittests (target/debug/deps/rectangle-6584c4561e48942e) running 2 tests test tests::larger_can_hold_smaller ... ok diff --git a/src/doc/book/listings/ch11-writing-automated-tests/no-listing-03-introducing-a-bug/output.txt b/src/doc/book/listings/ch11-writing-automated-tests/no-listing-03-introducing-a-bug/output.txt index db41fca7b1..6629b53850 100644 --- a/src/doc/book/listings/ch11-writing-automated-tests/no-listing-03-introducing-a-bug/output.txt +++ b/src/doc/book/listings/ch11-writing-automated-tests/no-listing-03-introducing-a-bug/output.txt @@ -1,7 +1,7 @@ $ cargo test Compiling rectangle v0.1.0 (file:///projects/rectangle) Finished test [unoptimized + debuginfo] target(s) in 0.66s - Running target/debug/deps/rectangle-6584c4561e48942e + Running unittests (target/debug/deps/rectangle-6584c4561e48942e) running 2 tests test tests::larger_can_hold_smaller ... FAILED diff --git a/src/doc/book/listings/ch11-writing-automated-tests/no-listing-04-bug-in-add-two/output.txt b/src/doc/book/listings/ch11-writing-automated-tests/no-listing-04-bug-in-add-two/output.txt index 00165cbc43..2c0ffe83ae 100644 --- a/src/doc/book/listings/ch11-writing-automated-tests/no-listing-04-bug-in-add-two/output.txt +++ b/src/doc/book/listings/ch11-writing-automated-tests/no-listing-04-bug-in-add-two/output.txt @@ -1,7 +1,7 @@ $ cargo test Compiling adder v0.1.0 (file:///projects/adder) Finished test [unoptimized + debuginfo] target(s) in 0.61s - Running target/debug/deps/adder-92948b65e88960b4 + Running unittests (target/debug/deps/adder-92948b65e88960b4) running 1 test test tests::it_adds_two ... FAILED diff --git a/src/doc/book/listings/ch11-writing-automated-tests/no-listing-06-greeter-with-bug/output.txt b/src/doc/book/listings/ch11-writing-automated-tests/no-listing-06-greeter-with-bug/output.txt index 67b846115c..d54451fd4b 100644 --- a/src/doc/book/listings/ch11-writing-automated-tests/no-listing-06-greeter-with-bug/output.txt +++ b/src/doc/book/listings/ch11-writing-automated-tests/no-listing-06-greeter-with-bug/output.txt @@ -1,7 +1,7 @@ $ cargo test Compiling greeter v0.1.0 (file:///projects/greeter) Finished test [unoptimized + debuginfo] target(s) in 0.91s - Running target/debug/deps/greeter-170b942eb5bf5e3a + Running unittests (target/debug/deps/greeter-170b942eb5bf5e3a) running 1 test test tests::greeting_contains_name ... FAILED diff --git a/src/doc/book/listings/ch11-writing-automated-tests/no-listing-07-custom-failure-message/output.txt b/src/doc/book/listings/ch11-writing-automated-tests/no-listing-07-custom-failure-message/output.txt index e22aa81f1f..2ecf804590 100644 --- a/src/doc/book/listings/ch11-writing-automated-tests/no-listing-07-custom-failure-message/output.txt +++ b/src/doc/book/listings/ch11-writing-automated-tests/no-listing-07-custom-failure-message/output.txt @@ -1,7 +1,7 @@ $ cargo test Compiling greeter v0.1.0 (file:///projects/greeter) Finished test [unoptimized + debuginfo] target(s) in 0.93s - Running target/debug/deps/greeter-170b942eb5bf5e3a + Running unittests (target/debug/deps/greeter-170b942eb5bf5e3a) running 1 test test tests::greeting_contains_name ... FAILED diff --git a/src/doc/book/listings/ch11-writing-automated-tests/no-listing-08-guess-with-bug/output.txt b/src/doc/book/listings/ch11-writing-automated-tests/no-listing-08-guess-with-bug/output.txt index bd95d20e5e..c624c24fad 100644 --- a/src/doc/book/listings/ch11-writing-automated-tests/no-listing-08-guess-with-bug/output.txt +++ b/src/doc/book/listings/ch11-writing-automated-tests/no-listing-08-guess-with-bug/output.txt @@ -1,7 +1,7 @@ $ cargo test Compiling guessing_game v0.1.0 (file:///projects/guessing_game) Finished test [unoptimized + debuginfo] target(s) in 0.62s - Running target/debug/deps/guessing_game-57d70c3acb738f4d + Running unittests (target/debug/deps/guessing_game-57d70c3acb738f4d) running 1 test test tests::greater_than_100 ... FAILED diff --git a/src/doc/book/listings/ch11-writing-automated-tests/no-listing-09-guess-with-panic-msg-bug/output.txt b/src/doc/book/listings/ch11-writing-automated-tests/no-listing-09-guess-with-panic-msg-bug/output.txt index 3cbee93ff7..d79268c7a7 100644 --- a/src/doc/book/listings/ch11-writing-automated-tests/no-listing-09-guess-with-panic-msg-bug/output.txt +++ b/src/doc/book/listings/ch11-writing-automated-tests/no-listing-09-guess-with-panic-msg-bug/output.txt @@ -1,7 +1,7 @@ $ cargo test Compiling guessing_game v0.1.0 (file:///projects/guessing_game) Finished test [unoptimized + debuginfo] target(s) in 0.66s - Running target/debug/deps/guessing_game-57d70c3acb738f4d + Running unittests (target/debug/deps/guessing_game-57d70c3acb738f4d) running 1 test test tests::greater_than_100 ... FAILED diff --git a/src/doc/book/listings/ch11-writing-automated-tests/no-listing-11-ignore-a-test/output.txt b/src/doc/book/listings/ch11-writing-automated-tests/no-listing-11-ignore-a-test/output.txt index 727f60c1cb..fcdff2517e 100644 --- a/src/doc/book/listings/ch11-writing-automated-tests/no-listing-11-ignore-a-test/output.txt +++ b/src/doc/book/listings/ch11-writing-automated-tests/no-listing-11-ignore-a-test/output.txt @@ -1,7 +1,7 @@ $ cargo test Compiling adder v0.1.0 (file:///projects/adder) Finished test [unoptimized + debuginfo] target(s) in 0.60s - Running target/debug/deps/adder-92948b65e88960b4 + Running unittests (target/debug/deps/adder-92948b65e88960b4) running 2 tests test expensive_test ... ignored diff --git a/src/doc/book/listings/ch11-writing-automated-tests/no-listing-12-shared-test-code-problem/output.txt b/src/doc/book/listings/ch11-writing-automated-tests/no-listing-12-shared-test-code-problem/output.txt index 41056125e7..0afdb52ee6 100644 --- a/src/doc/book/listings/ch11-writing-automated-tests/no-listing-12-shared-test-code-problem/output.txt +++ b/src/doc/book/listings/ch11-writing-automated-tests/no-listing-12-shared-test-code-problem/output.txt @@ -1,20 +1,20 @@ $ cargo test Compiling adder v0.1.0 (file:///projects/adder) Finished test [unoptimized + debuginfo] target(s) in 0.89s - Running target/debug/deps/adder-92948b65e88960b4 + Running unittests (target/debug/deps/adder-92948b65e88960b4) running 1 test test tests::internal ... ok test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s - Running target/debug/deps/common-92948b65e88960b4 + Running tests/common.rs (target/debug/deps/common-92948b65e88960b4) running 0 tests test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s - Running target/debug/deps/integration_test-92948b65e88960b4 + Running tests/integration_test.rs (target/debug/deps/integration_test-92948b65e88960b4) running 1 test test it_adds_two ... ok diff --git a/src/doc/book/listings/ch11-writing-automated-tests/output-only-01-show-output/output.txt b/src/doc/book/listings/ch11-writing-automated-tests/output-only-01-show-output/output.txt index cb9f454e7a..94b0b4b906 100644 --- a/src/doc/book/listings/ch11-writing-automated-tests/output-only-01-show-output/output.txt +++ b/src/doc/book/listings/ch11-writing-automated-tests/output-only-01-show-output/output.txt @@ -1,7 +1,7 @@ $ cargo test -- --show-output Compiling silly-function v0.1.0 (file:///projects/silly-function) Finished test [unoptimized + debuginfo] target(s) in 0.60s - Running target/debug/deps/silly_function-160869f38cff9166 + Running unittests (target/debug/deps/silly_function-160869f38cff9166) running 2 tests test tests::this_test_will_fail ... FAILED diff --git a/src/doc/book/listings/ch11-writing-automated-tests/output-only-02-single-test/output.txt b/src/doc/book/listings/ch11-writing-automated-tests/output-only-02-single-test/output.txt index e5716fe45b..e5e9ffdd7a 100644 --- a/src/doc/book/listings/ch11-writing-automated-tests/output-only-02-single-test/output.txt +++ b/src/doc/book/listings/ch11-writing-automated-tests/output-only-02-single-test/output.txt @@ -1,7 +1,7 @@ $ cargo test one_hundred Compiling adder v0.1.0 (file:///projects/adder) Finished test [unoptimized + debuginfo] target(s) in 0.69s - Running target/debug/deps/adder-92948b65e88960b4 + Running unittests (target/debug/deps/adder-92948b65e88960b4) running 1 test test tests::one_hundred ... ok diff --git a/src/doc/book/listings/ch11-writing-automated-tests/output-only-03-multiple-tests/output.txt b/src/doc/book/listings/ch11-writing-automated-tests/output-only-03-multiple-tests/output.txt index 55402231d7..822cbb5ee5 100644 --- a/src/doc/book/listings/ch11-writing-automated-tests/output-only-03-multiple-tests/output.txt +++ b/src/doc/book/listings/ch11-writing-automated-tests/output-only-03-multiple-tests/output.txt @@ -1,7 +1,7 @@ $ cargo test add Compiling adder v0.1.0 (file:///projects/adder) Finished test [unoptimized + debuginfo] target(s) in 0.61s - Running target/debug/deps/adder-92948b65e88960b4 + Running unittests (target/debug/deps/adder-92948b65e88960b4) running 2 tests test tests::add_three_and_two ... ok diff --git a/src/doc/book/listings/ch11-writing-automated-tests/output-only-04-running-ignored/output.txt b/src/doc/book/listings/ch11-writing-automated-tests/output-only-04-running-ignored/output.txt index 6fe42c9dec..009ac4a5b4 100644 --- a/src/doc/book/listings/ch11-writing-automated-tests/output-only-04-running-ignored/output.txt +++ b/src/doc/book/listings/ch11-writing-automated-tests/output-only-04-running-ignored/output.txt @@ -1,7 +1,7 @@ $ cargo test -- --ignored Compiling adder v0.1.0 (file:///projects/adder) Finished test [unoptimized + debuginfo] target(s) in 0.61s - Running target/debug/deps/adder-92948b65e88960b4 + Running unittests (target/debug/deps/adder-92948b65e88960b4) running 1 test test expensive_test ... ok diff --git a/src/doc/book/listings/ch11-writing-automated-tests/output-only-05-single-integration/output.txt b/src/doc/book/listings/ch11-writing-automated-tests/output-only-05-single-integration/output.txt index b641879f24..260beaa2d6 100644 --- a/src/doc/book/listings/ch11-writing-automated-tests/output-only-05-single-integration/output.txt +++ b/src/doc/book/listings/ch11-writing-automated-tests/output-only-05-single-integration/output.txt @@ -1,7 +1,7 @@ $ cargo test --test integration_test Compiling adder v0.1.0 (file:///projects/adder) Finished test [unoptimized + debuginfo] target(s) in 0.64s - Running target/debug/deps/integration_test-82e7799c1bc62298 + Running tests/integration_test.rs (target/debug/deps/integration_test-82e7799c1bc62298) running 1 test test it_adds_two ... ok diff --git a/src/doc/book/listings/ch12-an-io-project/listing-12-12/output.txt b/src/doc/book/listings/ch12-an-io-project/listing-12-12/output.txt index 8d1c221872..91ee1df01b 100644 --- a/src/doc/book/listings/ch12-an-io-project/listing-12-12/output.txt +++ b/src/doc/book/listings/ch12-an-io-project/listing-12-12/output.txt @@ -1,6 +1,6 @@ $ cargo run the poem.txt Compiling minigrep v0.1.0 (file:///projects/minigrep) -warning: unused `std::result::Result` that must be used +warning: unused `Result` that must be used --> src/main.rs:19:5 | 19 | run(config); diff --git a/src/doc/book/listings/ch15-smart-pointers/listing-15-15/output.txt b/src/doc/book/listings/ch15-smart-pointers/listing-15-15/output.txt index b7893963ee..86fe81065e 100644 --- a/src/doc/book/listings/ch15-smart-pointers/listing-15-15/output.txt +++ b/src/doc/book/listings/ch15-smart-pointers/listing-15-15/output.txt @@ -4,10 +4,10 @@ error[E0040]: explicit use of destructor method --> src/main.rs:16:7 | 16 | c.drop(); - | ^^^^ - | | - | explicit destructor calls not allowed - | help: consider using `drop` function: `drop(c)` + | --^^^^-- + | | | + | | explicit destructor calls not allowed + | help: consider using `drop` function: `drop(c)` error: aborting due to previous error diff --git a/src/doc/book/listings/ch18-patterns-and-matching/listing-18-10/output.txt b/src/doc/book/listings/ch18-patterns-and-matching/listing-18-10/output.txt index 2fef90cec7..f30dd5fa88 100644 --- a/src/doc/book/listings/ch18-patterns-and-matching/listing-18-10/output.txt +++ b/src/doc/book/listings/ch18-patterns-and-matching/listing-18-10/output.txt @@ -1,6 +1,6 @@ $ cargo run Compiling patterns v0.1.0 (file:///projects/patterns) -warning: irrefutable if-let pattern +warning: irrefutable `if let` pattern --> src/main.rs:2:5 | 2 | / if let x = 5 { @@ -9,6 +9,8 @@ warning: irrefutable if-let pattern | |_____^ | = note: `#[warn(irrefutable_let_patterns)]` on by default + = note: this pattern will always match, so the `if let` is useless + = help: consider replacing the `if let` with a `let` warning: 1 warning emitted diff --git a/src/doc/book/rust-toolchain b/src/doc/book/rust-toolchain index ec8bede263..d96ae405eb 100644 --- a/src/doc/book/rust-toolchain +++ b/src/doc/book/rust-toolchain @@ -1 +1 @@ -1.50 +1.52 diff --git a/src/doc/book/src/ch11-01-writing-tests.md b/src/doc/book/src/ch11-01-writing-tests.md index 47b37f7404..c2d001bd0d 100644 --- a/src/doc/book/src/ch11-01-writing-tests.md +++ b/src/doc/book/src/ch11-01-writing-tests.md @@ -264,7 +264,7 @@ more conveniently. These macros compare two arguments for equality or inequality, respectively. They’ll also print the two values if the assertion fails, which makes it easier to see *why* the test failed; conversely, the `assert!` macro only indicates that it got a `false` value for the `==` -expression, not the values that lead to the `false` value. +expression, not the values that led to the `false` value. In Listing 11-7, we write a function named `add_two` that adds `2` to its parameter and returns the result. Then we test this function using the diff --git a/src/doc/book/src/title-page.md b/src/doc/book/src/title-page.md index c7b3d673f7..4760ba98fb 100644 --- a/src/doc/book/src/title-page.md +++ b/src/doc/book/src/title-page.md @@ -2,7 +2,7 @@ *by Steve Klabnik and Carol Nichols, with contributions from the Rust Community* -This version of the text assumes you’re using Rust 1.50 or later with +This version of the text assumes you’re using Rust 1.52 or later with `edition="2018"` in *Cargo.toml* of all projects to use Rust 2018 Edition idioms. See the [“Installation” section of Chapter 1][install] to install or update Rust, and see the new [Appendix E][editions] src/main.rs:6:17 - | -6 | for i in 0..256 { - | ^^^ - | - = note: #[deny(overflowing_literals)] on by default -``` - -That’s right, since `i` is a `u8`, this overflows, and the compiler produces -an error. - -We can do this with inclusive ranges, however: - -```rust -fn takes_u8(x: u8) { - // ... -} - -fn main() { - for i in 0..=255 { - println!("i: {}", i); - takes_u8(i); - } -} -``` - -This will produce those 256 lines of output you might have been expecting. diff --git a/src/doc/edition-guide/src/rust-2018/data-types/index.md b/src/doc/edition-guide/src/rust-2018/data-types/index.md deleted file mode 100644 index 3da89d676a..0000000000 --- a/src/doc/edition-guide/src/rust-2018/data-types/index.md +++ /dev/null @@ -1,6 +0,0 @@ -# Data types - -[fis]: field-init-shorthand.md - -In this chapter of the guide, we discuss a few improvements to data types. -One of these are [field-init-shorthand][fis]. \ No newline at end of file diff --git a/src/doc/edition-guide/src/rust-2018/data-types/operator-equals-are-now-implementable.md b/src/doc/edition-guide/src/rust-2018/data-types/operator-equals-are-now-implementable.md deleted file mode 100644 index 19390fa3c4..0000000000 --- a/src/doc/edition-guide/src/rust-2018/data-types/operator-equals-are-now-implementable.md +++ /dev/null @@ -1,33 +0,0 @@ -# "Operator-equals" are now implementable - -![Minimum Rust version: 1.8](https://img.shields.io/badge/Minimum%20Rust%20Version-1.8-brightgreen.svg) - -The various “operator equals” operators, such as `+=` and `-=`, are -implementable via various traits. For example, to implement `+=` on -a type of your own: - -```rust -use std::ops::AddAssign; - -#[derive(Debug)] -struct Count { - value: i32, -} - -impl AddAssign for Count { - fn add_assign(&mut self, other: Count) { - self.value += other.value; - } -} - -fn main() { - let mut c1 = Count { value: 1 }; - let c2 = Count { value: 5 }; - - c1 += c2; - - println!("{:?}", c1); -} -``` - -This will print `Count { value: 6 }`. \ No newline at end of file diff --git a/src/doc/edition-guide/src/rust-2018/data-types/union-for-an-unsafe-form-of-enum.md b/src/doc/edition-guide/src/rust-2018/data-types/union-for-an-unsafe-form-of-enum.md deleted file mode 100644 index a0bea75839..0000000000 --- a/src/doc/edition-guide/src/rust-2018/data-types/union-for-an-unsafe-form-of-enum.md +++ /dev/null @@ -1,60 +0,0 @@ -# `union` for an unsafe form of `enum` - -![Minimum Rust version: 1.19](https://img.shields.io/badge/Minimum%20Rust%20Version-1.19-brightgreen.svg) - -Rust now supports `unions`: - -```rust -union MyUnion { - f1: u32, - f2: f32, -} -``` - -Unions are kind of like enums, but they are “untagged”. Enums have a “tag” -that stores which variant is the correct one at runtime; unions don't have -this tag. - -Since we can interpret the data held in the union using the wrong variant and -Rust can’t check this for us, that means reading a union’s field is unsafe: - -```rust -# union MyUnion { -# f1: u32, -# f2: f32, -# } -let mut u = MyUnion { f1: 1 }; - -u.f1 = 5; - -let value = unsafe { u.f1 }; -``` - -Pattern matching works too: - -```rust -# union MyUnion { -# f1: u32, -# f2: f32, -# } -fn f(u: MyUnion) { - unsafe { - match u { - MyUnion { f1: 10 } => { println!("ten"); } - MyUnion { f2 } => { println!("{}", f2); } - } - } -} -``` - -When are unions useful? One major use-case is interoperability with C. C APIs -can (and depending on the area, often do) expose unions, and so this makes -writing API wrappers for those libraries significantly easier. Additionally, -unions also simplify Rust implementations of space-efficient or -cache-efficient structures relying on value representation, such as -machine-word-sized unions using the least-significant bits of aligned -pointers to distinguish cases. - -There’s still more improvements to come. For now, unions can only include -`Copy` types and may not implement `Drop`. We expect to lift these -restrictions in the future. diff --git a/src/doc/edition-guide/src/rust-2018/documentation/index.md b/src/doc/edition-guide/src/rust-2018/documentation/index.md deleted file mode 100644 index 4c56654421..0000000000 --- a/src/doc/edition-guide/src/rust-2018/documentation/index.md +++ /dev/null @@ -1,6 +0,0 @@ -# Documentation - -[sec_ed]: new-editions-of-the-book.md - -In this chapter of the guide, we discuss a few improvements to documentation. -A notable addition here is the [second edition of "the book"][sec_ed]. \ No newline at end of file diff --git a/src/doc/edition-guide/src/rust-2018/documentation/new-editions-of-the-book.md b/src/doc/edition-guide/src/rust-2018/documentation/new-editions-of-the-book.md deleted file mode 100644 index ecee4bc81a..0000000000 --- a/src/doc/edition-guide/src/rust-2018/documentation/new-editions-of-the-book.md +++ /dev/null @@ -1,31 +0,0 @@ -# New editions of the "the book" - -![Minimum Rust version: 1.26](https://img.shields.io/badge/Minimum%20Rust%20Version-1.26-brightgreen.svg) for the final version of the second edition - -![Minimum Rust version: 1.31](https://img.shields.io/badge/Minimum%20Rust%20Version-1.31-brightgreen.svg) for the 2018 edition - -We've distributed a copy of "The Rust Programming Language," affectionately -nicknamed "the book", with every version of Rust since Rust 1.0. - -However, because it was written before Rust 1.0, it started showing its age. -Many parts of the book are vague, because it was written before the true -details were nailed down for the 1.0 release. It didn't do a fantastic job of -teaching lifetimes. - -Starting with Rust 1.18, we shipped drafts of a second edition of the book. -The final version was shipped with Rust 1.26. The second edition is a complete -re-write from the ground up, using the last two years of knowledge we’ve -gained from teaching people Rust. - -You can purchase [a printed version of the second edition from No Starch -Press](https://nostarch.com/Rust). Now that the print version has shipped, the -second edition is frozen. - -As of 1.31, the book has been completely updated for the 2018 Edition release. -It's still pretty close to the second edition, but contains information about -newer features since the book's content was frozen. Additionally, instead of -publishing separate editions of the book, only the latest version of the book -is published online. You’ll find brand-new explanations for a lot of Rust’s -core concepts, new projects to build, and all kinds of other good stuff. -Please [check it out](https://doc.rust-lang.org/book/index.html) and let us -know what you think! diff --git a/src/doc/edition-guide/src/rust-2018/documentation/std-os-has-documentation-for-all-platforms.md b/src/doc/edition-guide/src/rust-2018/documentation/std-os-has-documentation-for-all-platforms.md deleted file mode 100644 index 5be8f01bc5..0000000000 --- a/src/doc/edition-guide/src/rust-2018/documentation/std-os-has-documentation-for-all-platforms.md +++ /dev/null @@ -1,11 +0,0 @@ -# `std::os` has documentation for all platforms - -![Minimum Rust version: 1.21](https://img.shields.io/badge/Minimum%20Rust%20Version-1.21-brightgreen.svg) - -The `std::os` module contains operating system specific functionality. You’ll -now see more than just linux, the platform we build the documentation on. - -We’ve long regretted that the hosted version of the documentation has been -Linux-specific; this is a first step towards rectifying that. This is -specific to the standard library and not for general use; we hope to improve -this further in the future. \ No newline at end of file diff --git a/src/doc/edition-guide/src/rust-2018/documentation/the-rust-bookshelf.md b/src/doc/edition-guide/src/rust-2018/documentation/the-rust-bookshelf.md deleted file mode 100644 index e91627ac1f..0000000000 --- a/src/doc/edition-guide/src/rust-2018/documentation/the-rust-bookshelf.md +++ /dev/null @@ -1,33 +0,0 @@ -# The Rust Bookshelf - -![Minimum Rust version: various](https://img.shields.io/badge/Minimum%20Rust%20Version-various-brightgreen.svg), each book is different. - -As Rust's documentation has grown, we've gained far more than just "The book" -and the reference. We now have a collection of various long-form docs, -nicknamed "the Rust Bookshelf." Different resources are added at various -times, and we're adding new ones as more get written. - -## The Cargo book - -![Minimum Rust version: 1.21](https://img.shields.io/badge/Minimum%20Rust%20Version-1.21-brightgreen.svg) - -Historically, Cargo’s docs were hosted on , which -doesn’t follow the release train model, even though Cargo itself does. This -led to situations where a feature would land in Cargo nightly, the docs would -be updated, and then for up to twelve weeks, users would think that it should -work, but it wouldn’t yet. is the new home -of Cargo’s docs, and now redirects there. - -## The `rustdoc` book - -![Minimum Rust version: 1.21](https://img.shields.io/badge/Minimum%20Rust%20Version-1.21-brightgreen.svg) - -Rustdoc, our documentation tool, now has a guide at . - -## Rust By Example - -![Minimum Rust version: 1.25](https://img.shields.io/badge/Minimum%20Rust%20Version-1.25-brightgreen.svg) - -Rust by Example used to live at , but now is part of the Bookshelf! -It can be found at . RBE lets you learn Rust through -short code examples and exercises, as opposed to the lengthy prose of The Book. \ No newline at end of file diff --git a/src/doc/edition-guide/src/rust-2018/documentation/the-rustonomicon.md b/src/doc/edition-guide/src/rust-2018/documentation/the-rustonomicon.md deleted file mode 100644 index 514520c02c..0000000000 --- a/src/doc/edition-guide/src/rust-2018/documentation/the-rustonomicon.md +++ /dev/null @@ -1,10 +0,0 @@ -# The Rustonomicon - -![Minimum Rust version: 1.3](https://img.shields.io/badge/Minimum%20Rust%20Version-1.3-brightgreen.svg) - -We now have a draft book, [The Rustonomicon: the Dark Arts of Advanced and -Unsafe Rust Programming](https://doc.rust-lang.org/stable/nomicon/). - -From the title, I'm sure you can guess: this book discusses some advanced -topics, including `unsafe`. It's a must-read for anyone who's working at the -lowest levels with Rust. \ No newline at end of file diff --git a/src/doc/edition-guide/src/rust-2018/edition-changes.md b/src/doc/edition-guide/src/rust-2018/edition-changes.md deleted file mode 100644 index 7c4d1762d7..0000000000 --- a/src/doc/edition-guide/src/rust-2018/edition-changes.md +++ /dev/null @@ -1,36 +0,0 @@ -# 2018-Specific Changes - -The following is a summary of changes that only apply to code compiled with -the 2018 edition compared to the 2015 edition. - -- [Path changes]: - - Paths in `use` declarations work the same as other paths. - - Paths starting with `::` must be followed with an external crate. - - Paths in `pub(in path)` visibility modifiers must start with `crate`, - `self`, or `super`. -- [Anonymous trait function parameters] are not allowed. - - [Trait function parameters] may use any irrefutable pattern when the - function has a body. -- Keyword changes: - - [`dyn`] is a [strict keyword][strict], in 2015 it is a [weak keyword]. - - `async` and `await` are [strict keywords][strict]. - - `try` is a [reserved keyword]. -- The following lints are now a hard error that you cannot silence: - - [tyvar_behind_raw_pointer] - -## Cargo -- If there is a target definition in a `Cargo.toml` manifest, it no longer - automatically disables automatic discovery of other targets. -- Target paths of the form `src/{target_name}.rs` are no longer inferred for - targets where the `path` field is not set. -- `cargo install` for the current directory is no longer allowed, you must - specify `cargo install --path .` to install the current package. - -[Anonymous trait function parameters]: trait-system/no-anon-params.md -[Path changes]: module-system/path-clarity.md -[Trait function parameters]: https://doc.rust-lang.org/stable/reference/items/traits.html#parameter-patterns -[`dyn`]: trait-system/dyn-trait-for-trait-objects.md -[reserved keyword]: https://doc.rust-lang.org/reference/keywords.html#reserved-keywords -[strict]: https://doc.rust-lang.org/reference/keywords.html#strict-keywords -[tyvar_behind_raw_pointer]: https://github.com/rust-lang/rust/issues/46906 -[weak keyword]: https://doc.rust-lang.org/reference/keywords.html#weak-keywords diff --git a/src/doc/edition-guide/src/rust-2018/error-handling-and-panics/aborting-on-panic.md b/src/doc/edition-guide/src/rust-2018/error-handling-and-panics/aborting-on-panic.md deleted file mode 100644 index 303395e285..0000000000 --- a/src/doc/edition-guide/src/rust-2018/error-handling-and-panics/aborting-on-panic.md +++ /dev/null @@ -1,18 +0,0 @@ -# Aborting on panic - -![Minimum Rust version: 1.10](https://img.shields.io/badge/Minimum%20Rust%20Version-1.10-brightgreen.svg) - -By default, Rust programs will unwind the stack when a `panic!` happens. If you'd prefer an -immediate abort instead, you can configure this in `Cargo.toml`: - -```toml -[profile.dev] -panic = "abort" - -[profile.release] -panic = "abort" -``` - -Why might you choose to do this? By removing support for unwinding, you'll -get smaller binaries. You will lose the ability to catch panics. Which choice -is right for you depends on exactly what you're doing. diff --git a/src/doc/edition-guide/src/rust-2018/error-handling-and-panics/controlling-panics-with-std-panic.md b/src/doc/edition-guide/src/rust-2018/error-handling-and-panics/controlling-panics-with-std-panic.md deleted file mode 100644 index 3f4ae23640..0000000000 --- a/src/doc/edition-guide/src/rust-2018/error-handling-and-panics/controlling-panics-with-std-panic.md +++ /dev/null @@ -1,80 +0,0 @@ -# Controlling panics with `std::panic` - -![Minimum Rust version: 1.9](https://img.shields.io/badge/Minimum%20Rust%20Version-1.9-brightgreen.svg) - -There is a `std::panic` module, which includes methods for halting the -unwinding process started by a panic: - -```rust -use std::panic; - -let result = panic::catch_unwind(|| { - println!("hello!"); -}); -assert!(result.is_ok()); - -let result = panic::catch_unwind(|| { - panic!("oh no!"); -}); -assert!(result.is_err()); -``` - -In general, Rust distinguishes between two ways that an operation can fail: - -- Due to an *expected problem*, like a file not being found. -- Due to an *unexpected problem*, like an index being out of bounds for an array. - -Expected problems usually arise from conditions that are outside of your -control; robust code should be prepared for anything its environment might throw -at it. In Rust, expected problems are handled via [the `Result` type][result], -which allows a function to return information about the problem to its caller, -which can then handle the error in a fine-grained way. - -[result]: http://doc.rust-lang.org/std/result/index.html - -Unexpected problems are *bugs*: they arise due to a contract or assertion being -violated. Since they are unexpected, it doesn't make sense to handle them in a -fine-grained way. Instead, Rust employs a "fail fast" approach by *panicking*, -which by default unwinds the stack (running destructors but no other code) of -the thread which discovered the error. Other threads continue running, but will -discover the panic any time they try to communicate with the panicked thread -(whether through channels or shared memory). Panics thus abort execution up to -some "isolation boundary", with code on the other side of the boundary still -able to run, and perhaps to "recover" from the panic in some very coarse-grained -way. A server, for example, does not necessarily need to go down just because of -an assertion failure in one of its threads. - -It's also worth noting that programs may choose to *abort* instead of unwind, -and so catching panics may not work. If your code relies on `catch_unwind`, you -should add this to your Cargo.toml: - -```toml -[profile.dev] -panic = "unwind" - -[profile.release] -panic = "unwind" -``` - -If any of your users choose to abort, they'll get a compile-time failure. - -The `catch_unwind` API offers a way to introduce new isolation boundaries -*within a thread*. There are a couple of key motivating examples: - -* Embedding Rust in other languages -* Abstractions that manage threads -* Test frameworks, because tests may panic and you don't want that to kill the test runner - -For the first case, unwinding across a language boundary is undefined behavior, -and often leads to segfaults in practice. Allowing panics to be caught means -that you can safely expose Rust code via a C API, and translate unwinding into -an error on the C side. - -For the second case, consider a threadpool library. If a thread in the pool -panics, you generally don't want to kill the thread itself, but rather catch the -panic and communicate it to the client of the pool. The `catch_unwind` API is -paired with `resume_unwind`, which can then be used to restart the panicking -process on the client of the pool, where it belongs. - -In both cases, you're introducing a new isolation boundary within a thread, and -then translating the panic into some other form of error elsewhere. diff --git a/src/doc/edition-guide/src/rust-2018/error-handling-and-panics/index.md b/src/doc/edition-guide/src/rust-2018/error-handling-and-panics/index.md deleted file mode 100644 index c89a7d5ef1..0000000000 --- a/src/doc/edition-guide/src/rust-2018/error-handling-and-panics/index.md +++ /dev/null @@ -1,6 +0,0 @@ -# Error handling and Panics - -[qop]: the-question-mark-operator-for-easier-error-handling.md - -In this chapter of the guide, we discuss a few improvements to error handling -in Rust. The most notable of these is [the introduction of the `?` operator][qop]. \ No newline at end of file diff --git a/src/doc/edition-guide/src/rust-2018/error-handling-and-panics/question-mark-in-main-and-tests.md b/src/doc/edition-guide/src/rust-2018/error-handling-and-panics/question-mark-in-main-and-tests.md deleted file mode 100644 index 4c116b564a..0000000000 --- a/src/doc/edition-guide/src/rust-2018/error-handling-and-panics/question-mark-in-main-and-tests.md +++ /dev/null @@ -1,131 +0,0 @@ -# `?` in `main` and tests - -![Minimum Rust version: 1.26](https://img.shields.io/badge/Minimum%20Rust%20Version-1.26-brightgreen.svg) - -Rust's error handling revolves around returning `Result` and using `?` -to propagate errors. For those who write many small programs and, hopefully, -many tests, one common paper cut has been mixing entry points such as `main` -and `#[test]`s with error handling. - -As an example, you might have tried to write: - -```rust,ignore -use std::fs::File; - -fn main() { - let f = File::open("bar.txt")?; -} -``` - -Since `?` works by propagating the `Result` with an early return to the -enclosing function, the snippet above does not work, and results today -in the following error: - -```rust,ignore -error[E0277]: the `?` operator can only be used in a function that returns `Result` - or `Option` (or another type that implements `std::ops::Try`) - --> src/main.rs:5:13 - | -5 | let f = File::open("bar.txt")?; - | ^^^^^^^^^^^^^^^^^^^^^^ cannot use the `?` operator in a function that returns `()` - | - = help: the trait `std::ops::Try` is not implemented for `()` - = note: required by `std::ops::Try::from_error` -``` - -To solve this problem in Rust 2015, you might have written something like: - -```rust -// Rust 2015 - -# use std::process; -# use std::error::Error; - -fn run() -> Result<(), Box> { - // real logic.. - Ok(()) -} - -fn main() { - if let Err(e) = run() { - println!("Application error: {}", e); - process::exit(1); - } -} -``` - -However, in this case, the `run` function has all the interesting logic and -`main` is just boilerplate. The problem is even worse for `#[test]`s, since -there tend to be a lot more of them. - -In Rust 2018 you can instead let your `#[test]`s and `main` functions return -a `Result`: - -```rust,no_run -// Rust 2018 - -use std::fs::File; - -fn main() -> Result<(), std::io::Error> { - let f = File::open("bar.txt")?; - - Ok(()) -} -``` - -In this case, if say the file doesn't exist and there is an `Err(err)` somewhere, -then `main` will exit with an error code (not `0`) and print out a `Debug` -representation of `err`. Note that this will always print out the `Debug` representation. -If you would like to, for example, print out the `Display` representation of `err`, -you will still have to do what you would in Rust 2015. - -## More details - -Getting `-> Result<..>` to work in the context of `main` and `#[test]`s is not -magic. It is all backed up by a `Termination` trait which all valid return -types of `main` and testing functions must implement. The trait is defined as: - -```rust -pub trait Termination { - fn report(self) -> i32; -} -``` - -When setting up the entry point for your application, the compiler will use this -trait and call `.report()` on the `Result` of the `main` function you have written. - -Two simplified example implementations of this trait for `Result` and `()` are: - -```rust -# #![feature(process_exitcode_placeholder, termination_trait_lib)] -# use std::process::ExitCode; -# use std::fmt; -# -# pub trait Termination { fn report(self) -> i32; } - -impl Termination for () { - fn report(self) -> i32 { - # use std::process::Termination; - ExitCode::SUCCESS.report() - } -} - -impl Termination for Result<(), E> { - fn report(self) -> i32 { - match self { - Ok(()) => ().report(), - Err(err) => { - eprintln!("Error: {:?}", err); - # use std::process::Termination; - ExitCode::FAILURE.report() - } - } - } -} -``` - -As you can see in the case of `()`, a success code is simply returned. -In the case of `Result`, the success case delegates to the implementation for -`()` but prints out an error message and a failure exit code on `Err(..)`. - -To learn more about the finer details, consult either [the tracking issue](https://github.com/rust-lang/rust/issues/43301) or [the RFC](https://github.com/rust-lang/rfcs/blob/master/text/1937-ques-in-main.md). diff --git a/src/doc/edition-guide/src/rust-2018/error-handling-and-panics/the-question-mark-operator-for-easier-error-handling.md b/src/doc/edition-guide/src/rust-2018/error-handling-and-panics/the-question-mark-operator-for-easier-error-handling.md deleted file mode 100644 index d521a4a15a..0000000000 --- a/src/doc/edition-guide/src/rust-2018/error-handling-and-panics/the-question-mark-operator-for-easier-error-handling.md +++ /dev/null @@ -1,120 +0,0 @@ -# The `?` operator for easier error handling - -![Minimum Rust version: 1.13](https://img.shields.io/badge/Minimum%20Rust%20Version-1.13-brightgreen.svg) for `Result` - -![Minimum Rust version: 1.22](https://img.shields.io/badge/Minimum%20Rust%20Version-1.22-brightgreen.svg) for `Option` - -Rust has gained a new operator, `?`, that makes error handling more pleasant -by reducing the visual noise involved. It does this by solving one simple -problem. To illustrate, imagine we had some code to read some data from a -file: - -```rust -# use std::{io::{self, prelude::*}, fs::File}; -fn read_username_from_file() -> Result { - let f = File::open("username.txt"); - - let mut f = match f { - Ok(file) => file, - Err(e) => return Err(e), - }; - - let mut s = String::new(); - - match f.read_to_string(&mut s) { - Ok(_) => Ok(s), - Err(e) => Err(e), - } -} -``` - -> Note: this code could be made simpler with a single call to -> [`std::fs::read_to_string`](https://doc.rust-lang.org/stable/std/fs/fn.read_to_string.html), -> but we're writing it all out manually here to have an example with multiple -> errors. - -This code has two paths that can fail, opening the file and reading the data -from it. If either of these fail to work, we'd like to return an error from -`read_username_from_file`. Doing so involves `match`ing on the result of the -I/O operations. In simple cases like this though, where we are only -propagating errors up the call stack, the matching is just boilerplate - -seeing it written out, in the same pattern every time, doesn't provide the -reader with a great deal of useful information. - -With `?`, the above code looks like this: - -```rust -# use std::{io::{self, prelude::*}, fs::File}; -fn read_username_from_file() -> Result { - let mut f = File::open("username.txt")?; - let mut s = String::new(); - - f.read_to_string(&mut s)?; - - Ok(s) -} -``` - -The `?` is shorthand for the entire match statements we wrote earlier. In -other words, `?` applies to a `Result` value, and if it was an `Ok`, it -unwraps it and gives the inner value. If it was an `Err`, it returns from the -function you're currently in. Visually, it is much more straightforward. -Instead of an entire match statement, now we are just using the single "?" -character to indicate that here we are handling errors in the standard way, -by passing them up the call stack. - -Seasoned Rustaceans may recognize that this is the same as the `try!` macro -that's been available since Rust `1.0`. And indeed, they are the same. -Previously, `read_username_from_file` could have been implemented like this: - -```rust -# use std::{io::{self, prelude::*}, fs::File}; -fn read_username_from_file() -> Result { - let mut f = try!(File::open("username.txt")); - let mut s = String::new(); - - try!(f.read_to_string(&mut s)); - - Ok(s) -} -``` - -So why extend the language when we already have a macro? There are multiple -reasons. First, `try!` has proved to be extremely useful, and is used often -in idiomatic Rust. It is used so often that we think it's worth having a -sweet syntax. This sort of evolution is one of the great advantages of a -powerful macro system: speculative extensions to the language syntax can be -prototyped and iterated on without modifying the language itself, and in -return, macros that turn out to be especially useful can indicate missing -language features. This evolution, from `try!` to `?` is a great example. - -One of the reasons `try!` needs a sweeter syntax is that it is quite -unattractive when multiple invocations of `try!` are used in succession. -Consider: - -```rust,ignore -try!(try!(try!(foo()).bar()).baz()) -``` - -as opposed to - -```rust,ignore -foo()?.bar()?.baz()? -``` - -The first is quite difficult to scan visually, and each layer of error -handling prefixes the expression with an additional call to `try!`. This -brings undue attention to the trivial error propagation, obscuring the main -code path, in this example the calls to `foo`, `bar` and `baz`. This sort of -method chaining with error handling occurs in situations like the builder -pattern. - -Finally, the dedicated syntax will make it easier in the future to produce -nicer error messages tailored specifically to `?`, whereas it is difficult to -produce nice errors for macro-expanded code generally. - -You can use `?` with `Result`s, but also with `Option`. In that -case, `?` will return a value for `Some(T)` and return `None` for `None`. One -current restriction is that you cannot use `?` for both in the same function, -as the return type needs to match the type you use `?` on. In the future, -this restriction will be lifted. diff --git a/src/doc/edition-guide/src/rust-2018/index.md b/src/doc/edition-guide/src/rust-2018/index.md index ee982334b1..77d3f48f3a 100644 --- a/src/doc/edition-guide/src/rust-2018/index.md +++ b/src/doc/edition-guide/src/rust-2018/index.md @@ -1,8 +1,8 @@ # Rust 2018 -The edition system was created for the release of Rust 2018. The theme of Rust 2018 -is *productivity*. Rust 2018 improves upon Rust 2015 through new features, -simpler syntax in some cases, a smarter borrow-checker, and a host of other things. -These are all in service of the productivity goal. Rust 2015 was a foundation; -Rust 2018 smooths off rough edges, makes writing code simpler and easier, -and removes some inconsistencies. \ No newline at end of file +| Info | | +| --- | --- | +| RFC | [#2052](https://rust-lang.github.io/rfcs/2052-epochs.html), which also proposed the Edition system | +| Release version | [1.31.0](https://blog.rust-lang.org/2018/12/06/Rust-1.31-and-rust-2018.html) | + +The edition system was created for the release of Rust 2018. The release of the Rust 2018 edition coincided with a number of other features all coordinated around the theme of *productivity*. The majority of those features were backwards compatible and are now available on all editions; however, some of those changes required the edition mechanism (most notably the [module system changes](path-changes.md)). diff --git a/src/doc/edition-guide/src/rust-2018/macros/at-most-once.md b/src/doc/edition-guide/src/rust-2018/macros/at-most-once.md deleted file mode 100644 index 56e30b9790..0000000000 --- a/src/doc/edition-guide/src/rust-2018/macros/at-most-once.md +++ /dev/null @@ -1,42 +0,0 @@ -# At most one repetition - -![Minimum Rust version: 1.32](https://img.shields.io/badge/Minimum%20Rust%20Version-1.32-brightgreen.svg) for 2018 edition - -![Minimum Rust version: 1.37](https://img.shields.io/badge/Minimum%20Rust%20Version-1.37-brightgreen.svg) for 2015 edition - -In Rust 2018, we have made a couple of changes to the macros-by-example syntax. - -1. We have added a new Kleene operator `?` which means "at most one" - repetition. This operator does not accept a separator token. -2. We have disallowed using `?` as a separator to remove ambiguity with `?`. - -For example, consider the following Rust 2015 code: - -```rust -macro_rules! foo { - ($a:ident, $b:expr) => { - println!("{}", $a); - println!("{}", $b); - }; - ($a:ident) => { - println!("{}", $a); - } -} -``` - -Macro `foo` can be called with 1 or 2 arguments; the second one is optional, -but you need a whole other matcher to represent this possibility. This is -annoying if your matchers are long. In Rust 2018, one can simply write the -following: - -```rust -macro_rules! foo { - ($a:ident $(, $b:expr)?) => { - println!("{}", $a); - - $( - println!("{}", $b); - )? - } -} -``` diff --git a/src/doc/edition-guide/src/rust-2018/macros/custom-derive.md b/src/doc/edition-guide/src/rust-2018/macros/custom-derive.md deleted file mode 100644 index 4a1cd83304..0000000000 --- a/src/doc/edition-guide/src/rust-2018/macros/custom-derive.md +++ /dev/null @@ -1,49 +0,0 @@ -# Custom Derive - -![Minimum Rust version: 1.15](https://img.shields.io/badge/Minimum%20Rust%20Version-1.15-brightgreen.svg) - -In Rust, you’ve always been able to automatically implement some traits -through the derive attribute: - -```rust -#[derive(Debug)] -struct Pet { - name: String, -} -``` - -The `Debug` trait is then implemented for `Pet`, with vastly less boilerplate. For example, without `derive`, you'd have -to write this: - -```rust -use std::fmt; - -struct Pet { - name: String, -} - -impl fmt::Debug for Pet { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - match self { - Pet { name } => { - let mut debug_trait_builder = f.debug_struct("Pet"); - - let _ = debug_trait_builder.field("name", name); - - debug_trait_builder.finish() - } - } - } -} -``` - -Whew! - -However, this only worked for traits provided as part of the standard -library; it was not customizable. But now, you can tell Rust what to do when -someone wants to derive your trait. This is used heavily in popular crates -like [serde](https://serde.rs/) and [Diesel](http://diesel.rs/). - -For more, including learning how to build your own custom derive, see [The -Rust Programming -Language](https://doc.rust-lang.org/book/ch19-06-macros.html#how-to-write-a-custom-derive-macro). \ No newline at end of file diff --git a/src/doc/edition-guide/src/rust-2018/macros/index.md b/src/doc/edition-guide/src/rust-2018/macros/index.md deleted file mode 100644 index a87105fd91..0000000000 --- a/src/doc/edition-guide/src/rust-2018/macros/index.md +++ /dev/null @@ -1,6 +0,0 @@ -# Macros - -[custom-derive]: custom-derive.md - -In this chapter of the guide, we discuss a few improvements to the macro system. -A notable addition here is the introduction of [custom derive macros][custom-derive]. \ No newline at end of file diff --git a/src/doc/edition-guide/src/rust-2018/macros/macro-changes.md b/src/doc/edition-guide/src/rust-2018/macros/macro-changes.md deleted file mode 100644 index b561c9c363..0000000000 --- a/src/doc/edition-guide/src/rust-2018/macros/macro-changes.md +++ /dev/null @@ -1,271 +0,0 @@ -# Macro changes - -![Minimum Rust version: 1.31](https://img.shields.io/badge/Minimum%20Rust%20Version-1.31-brightgreen.svg) - -## `macro_rules!` style macros - -In Rust 2018, you can import specific macros from external crates via `use` -statements, rather than the old `#[macro_use]` attribute. - -For example, consider a `bar` crate that implements a `baz!` macro. In -`src/lib.rs`: - -```rust -#[macro_export] -macro_rules! baz { - () => () -} -``` - -In your crate, you would have written - -```rust,ignore -// Rust 2015 - -#[macro_use] -extern crate bar; - -fn main() { - baz!(); -} -``` - -Now, you write: - -```rust,ignore -// Rust 2018 - -use bar::baz; - -fn main() { - baz!(); -} -``` - -This moves `macro_rules` macros to be a bit closer to other kinds of items. - -Note that you'll still need `#[macro_use]` to use macros you've defined -in your own crate; this feature only works for importing macros from -external crates. - -## Procedural macros - -When using procedural macros to derive traits, you will have to name the macro -that provides the custom derive. This generally matches the name of the trait, -but check with the documentation of the crate providing the derives to be sure. - -For example, with Serde you would have written - -```rust,ignore -// Rust 2015 -extern crate serde; -#[macro_use] extern crate serde_derive; - -#[derive(Serialize, Deserialize)] -struct Bar; -``` - -Now, you write instead: - -```rust,ignore -// Rust 2018 -use serde_derive::{Serialize, Deserialize}; - -#[derive(Serialize, Deserialize)] -struct Bar; -``` - - -## More details - -This only works for macros defined in external crates. -For macros defined locally, `#[macro_use] mod foo;` is still required, as it was in Rust 2015. - -### Local helper macros - -Sometimes it is helpful or necessary to have helper macros inside your module. This can make -supporting both versions of rust more complicated. - -For example, let's make a simplified (and slightly contrived) version of the `log` crate in 2015 -edition style: - -```rust -use std::fmt; - -/// How important/severe the log message is. -#[derive(Copy, Clone)] -pub enum LogLevel { - Warn, - Error -} - -impl fmt::Display for LogLevel { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - match self { - LogLevel::Warn => write!(f, "warning"), - LogLevel::Error => write!(f, "error"), - } - } -} - -// A helper macro to log the message. -#[doc(hidden)] -#[macro_export] -macro_rules! __impl_log { - ($level:expr, $msg:expr) => {{ - println!("{}: {}", $level, $msg) - }} -} - -/// Warn level log message -#[macro_export] -macro_rules! warn { - ($($args:tt)*) => { - __impl_log!($crate::LogLevel::Warn, format_args!($($args)*)) - } -} - -/// Error level log message -#[macro_export] -macro_rules! error { - ($($args:tt)*) => { - __impl_log!($crate::LogLevel::Error, format_args!($($args)*)) - } -} -``` - -Our `__impl_log!` macro is private to our module, but needs to be exported as it is called by other -macros, and in 2015 edition all used macros must be exported. - -Now, in 2018 this example will not compile: - -```rust,ignore -use log::error; - -fn main() { - error!("error message"); -} -``` - -will give an error message about not finding the `__impl_log!` macro. This is because unlike in -the 2015 edition, macros are namespaced and we must import them. We could do - -```rust,ignore -use log::{__impl_log, error}; -``` - -which would make our code compile, but `__impl_log` is meant to be an implementation detail! - -#### Macros with `$crate::` prefix. - -The cleanest way to handle this situation is to use the `$crate::` prefix for macros, the same as -you would for any other path. Versions of the compiler >= 1.30 will handle this in both editions: - -```rust -macro_rules! warn { - ($($args:tt)*) => { - $crate::__impl_log!($crate::LogLevel::Warn, format_args!($($args)*)) - } -} - -// ... -``` - -However, this will not work for older versions of the compiler that don't understand the -`$crate::` prefix for macros. - -#### Macros using `local_inner_macros` - -We also have the `local_inner_macros` modifier that we can add to our `#[macro_export]` attribute. -This has the advantage of working with older rustc versions (older versions just ignore the extra -modifier). The downside is that it's a bit messier: - -```rust,ignore -#[macro_export(local_inner_macros)] -macro_rules! warn { - ($($args:tt)*) => { - __impl_log!($crate::LogLevel::Warn, format_args!($($args)*)) - } -} -``` - -So the code knows to look for any macros used locally. But wait - this won't compile, because we -use the `format_args!` macro that isn't in our local crate (hence the convoluted example). The -solution is to add a level of indirection: we create a macro that wraps `format_args`, but is local -to our crate. That way everything works in both editions (sadly we have to pollute the global -namespace a bit, but that's ok). - -```rust -// I've used the pattern `___` to name this macro, hopefully avoiding -// name clashes. -#[doc(hidden)] -#[macro_export] -macro_rules! _log__format_args { - ($($inner:tt)*) => { - format_args! { $($inner)* } - } -} -``` - -Here we're using the most general macro pattern possible, a list of token trees. We just pass -whatever tokens we get to the inner macro, and rely on it to report errors. - -So the full 2015/2018 working example would be: - -```rust -use std::fmt; - -/// How important/severe the log message is. -#[derive(Debug, Copy, Clone)] -pub enum LogLevel { - Warn, - Error -} - -impl fmt::Display for LogLevel { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - match self { - LogLevel::Warn => write!(f, "warning"), - LogLevel::Error => write!(f, "error"), - } - } -} - -// A helper macro to log the message. -#[doc(hidden)] -#[macro_export] -macro_rules! __impl_log { - ($level:expr, $msg:expr) => {{ - println!("{}: {}", $level, $msg) - }} -} - -/// Warn level log message -#[macro_export(local_inner_macros)] -macro_rules! warn { - ($($args:tt)*) => { - __impl_log!($crate::LogLevel::Warn, _log__format_args!($($args)*)) - } -} - -/// Error level log message -#[macro_export(local_inner_macros)] -macro_rules! error { - ($($args:tt)*) => { - __impl_log!($crate::LogLevel::Error, _log__format_args!($($args)*)) - } -} - -#[doc(hidden)] -#[macro_export] -macro_rules! _log__format_args { - ($($inner:tt)*) => { - format_args! { $($inner)* } - } -} -``` - -Once everyone is using a rustc version >= 1.30, we can all just use the `$crate::` method (2015 -crates are guaranteed to carry on compiling fine with later versions of the compiler). We need to -wait for package managers and larger organisations to update their compilers before this happens, -so in the mean time we can use the `local_inner_macros` method to support everybody. :) diff --git a/src/doc/edition-guide/src/rust-2018/module-system/index.md b/src/doc/edition-guide/src/rust-2018/module-system/index.md deleted file mode 100644 index 42fea5fd2c..0000000000 --- a/src/doc/edition-guide/src/rust-2018/module-system/index.md +++ /dev/null @@ -1,6 +0,0 @@ -# Module system - -[path clarity changes]: path-clarity.md - -In this chapter of the guide, we discuss a few changes to the module system. -The most notable of these are the [path clarity changes]. \ No newline at end of file diff --git a/src/doc/edition-guide/src/rust-2018/module-system/more-visibility-modifiers.md b/src/doc/edition-guide/src/rust-2018/module-system/more-visibility-modifiers.md deleted file mode 100644 index cd7b7c1b65..0000000000 --- a/src/doc/edition-guide/src/rust-2018/module-system/more-visibility-modifiers.md +++ /dev/null @@ -1,16 +0,0 @@ -# More visibility modifiers - -![Minimum Rust version: 1.18](https://img.shields.io/badge/Minimum%20Rust%20Version-1.18-brightgreen.svg) - -You can use the `pub` keyword to make something a part of a module's public interface. But in -addition, there are some new forms: - -```rust,ignore -pub(crate) struct Foo; - -pub(in a::b::c) struct Bar; -``` - -The first form makes the `Foo` struct public to your entire crate, but not -externally. The second form is similar, but makes `Bar` public for one other -module, `a::b::c` in this case. \ No newline at end of file diff --git a/src/doc/edition-guide/src/rust-2018/module-system/nested-imports-with-use.md b/src/doc/edition-guide/src/rust-2018/module-system/nested-imports-with-use.md deleted file mode 100644 index b813590cde..0000000000 --- a/src/doc/edition-guide/src/rust-2018/module-system/nested-imports-with-use.md +++ /dev/null @@ -1,35 +0,0 @@ -# Nested imports with `use` - -![Minimum Rust version: 1.25](https://img.shields.io/badge/Minimum%20Rust%20Version-1.25-brightgreen.svg) - -A new way to write `use` statements has been added to Rust: nested import -groups. If you’ve ever written a set of imports like this: - -```rust -use std::fs::File; -use std::io::Read; -use std::path::{Path, PathBuf}; -``` - -You can now write this: - -```rust -# mod foo { -// on one line -use std::{fs::File, io::Read, path::{Path, PathBuf}}; -# } - -# mod bar { -// with some more breathing room -use std::{ - fs::File, - io::Read, - path::{ - Path, - PathBuf - } -}; -# } -``` - -This can reduce some repetition, and make things a bit more clear. diff --git a/src/doc/edition-guide/src/rust-2018/module-system/raw-identifiers.md b/src/doc/edition-guide/src/rust-2018/module-system/raw-identifiers.md deleted file mode 100644 index 208501dea6..0000000000 --- a/src/doc/edition-guide/src/rust-2018/module-system/raw-identifiers.md +++ /dev/null @@ -1,68 +0,0 @@ -# Raw identifiers - -![Minimum Rust version: 1.30](https://img.shields.io/badge/Minimum%20Rust%20Version-1.30-brightgreen.svg) - -Rust, like many programming languages, has the concept of "keywords". -These identifiers mean something to the language, and so you cannot use them in -places like variable names, function names, and other places. -Raw identifiers let you use keywords where they would not normally be allowed. - -For example, `match` is a keyword. If you try to compile this function: - -```rust,ignore -fn match(needle: &str, haystack: &str) -> bool { - haystack.contains(needle) -} -``` - -You'll get this error: - -```text -error: expected identifier, found keyword `match` - --> src/main.rs:4:4 - | -4 | fn match(needle: &str, haystack: &str) -> bool { - | ^^^^^ expected identifier, found keyword -``` - -You can write this with a raw identifier: - -```rust -fn r#match(needle: &str, haystack: &str) -> bool { - haystack.contains(needle) -} - -fn main() { - assert!(r#match("foo", "foobar")); -} -``` - -Note the `r#` prefix on both the function name as well as the call. - -## Motivation - -This feature is useful for a few reasons, but the primary motivation was -inter-edition situations. For example, `try` is not a keyword in the 2015 -edition, but is in the 2018 edition. So if you have a library that is written -in Rust 2015 and has a `try` function, to call it in Rust 2018, you'll need -to use the raw identifier. - -## New keywords - -The new confirmed keywords in edition 2018 are: - -### `async` and `await` - -[RFC 2394]: https://github.com/rust-lang/rfcs/blob/master/text/2394-async_await.md#final-syntax-for-the-await-expression - -Here, `async` is reserved for use in `async fn` as well as in `async ||` closures and -`async { .. }` blocks. Meanwhile, `await` is reserved to keep our options open -with respect to `await!(expr)` syntax. See [RFC 2394] for more details. - -### `try` - -[RFC 2388]: https://github.com/rust-lang/rfcs/pull/2388 - -The `do catch { .. }` blocks have been renamed to `try { .. }` and to support -that, the keyword `try` is reserved in edition 2018. -See [RFC 2388] for more details. diff --git a/src/doc/edition-guide/src/rust-2018/trait-system/dyn-trait-for-trait-objects.md b/src/doc/edition-guide/src/rust-2018/new-keywords.md similarity index 50% rename from src/doc/edition-guide/src/rust-2018/trait-system/dyn-trait-for-trait-objects.md rename to src/doc/edition-guide/src/rust-2018/new-keywords.md index d6e62a262b..92ee877087 100644 --- a/src/doc/edition-guide/src/rust-2018/trait-system/dyn-trait-for-trait-objects.md +++ b/src/doc/edition-guide/src/rust-2018/new-keywords.md @@ -1,7 +1,21 @@ -# `dyn Trait` for trait objects +# New keywords ![Minimum Rust version: 1.27](https://img.shields.io/badge/Minimum%20Rust%20Version-1.27-brightgreen.svg) +## Summary + +- `dyn` is a [strict keyword][strict], in 2015 it is a [weak keyword]. +- `async` and `await` are [strict keywords][strict]. +- `try` is a [reserved keyword]. + +[strict]: https://doc.rust-lang.org/reference/keywords.html#strict-keywords +[weak keyword]: https://doc.rust-lang.org/reference/keywords.html#weak-keywords +[reserved keyword]: https://doc.rust-lang.org/reference/keywords.html#reserved-keywords + +## Motivation + +### `dyn Trait` for trait objects + The `dyn Trait` feature is the new syntax for using trait objects. In short: * `Box` becomes `Box` @@ -27,7 +41,7 @@ fn function2() -> Box { That's it! -## More details +#### Why? Using just the trait name for trait objects turned out to be a bad decision. The current syntax is often ambiguous and confusing, even to veterans, @@ -36,7 +50,17 @@ is sometimes slower, and often cannot be used at all when its alternatives can. Furthermore, with `impl Trait` arriving, "`impl Trait` vs `dyn Trait`" is much more symmetric, and therefore a bit nicer, than "`impl Trait` vs `Trait`". -`impl Trait` is explained [here](impl-trait-for-returning-complex-types-with-ease.md). +`impl Trait` is explained [here][impl-trait]. In the new edition, you should therefore prefer `dyn Trait` to just `Trait` -where you need a trait object. \ No newline at end of file +where you need a trait object. + +[impl-trait]: ../../rust-by-example/trait/impl_trait.html + +### `async` and `await` + +These keywords are reserved to implement the async-await feature of Rust, which was ultimately [released to stable in 1.39.0](https://blog.rust-lang.org/2019/11/07/Async-await-stable.html). + +### `try` keyword + +The `try` keyword is reserved for use in `try` blocks, which have not (as of this writing) been stabilized ([tracking issue](https://github.com/rust-lang/rust/issues/31436)) diff --git a/src/doc/edition-guide/src/rust-2018/ownership-and-lifetimes/default-match-bindings.md b/src/doc/edition-guide/src/rust-2018/ownership-and-lifetimes/default-match-bindings.md deleted file mode 100644 index 05107c613e..0000000000 --- a/src/doc/edition-guide/src/rust-2018/ownership-and-lifetimes/default-match-bindings.md +++ /dev/null @@ -1,61 +0,0 @@ -# Default match bindings - -![Minimum Rust version: 1.26](https://img.shields.io/badge/Minimum%20Rust%20Version-1.26-brightgreen.svg) - -Have you ever had a borrowed `Option` and tried to match on it? You -probably wrote this: - -```rust,ignore -let s: &Option = &Some("hello".to_string()); - -match s { - Some(s) => println!("s is: {}", s), - _ => (), -}; -``` - -In Rust 2015, this would fail to compile, and you would have to write the -following instead: - -```rust,ignore -// Rust 2015 - -let s: &Option = &Some("hello".to_string()); - -match s { - &Some(ref s) => println!("s is: {}", s), - _ => (), -}; -``` - -Rust 2018, by contrast, will infer the `&`s and `ref`s, and your original -code will Just Work. - -This affects not just `match`, but patterns everywhere, such as in `let` -statements, closure arguments, and `for` loops. - -## More details - -The mental model of patterns has shifted a bit with this change, to bring it -into line with other aspects of the language. For example, when writing a -`for` loop, you can iterate over borrowed contents of a collection by -borrowing the collection itself: - -```rust,ignore -let my_vec: Vec = vec![0, 1, 2]; - -for x in &my_vec { ... } -``` - -The idea is that an `&T` can be understood as a *borrowed view of `T`*, and -so when you iterate, match, or otherwise destructure a `&T` you get a -borrowed view of its internals as well. - -More formally, patterns have a "binding mode," which is either by value -(`x`), by reference (`ref x`), or by mutable reference (`ref mut x`). In Rust -2015, `match` always started in by-value mode, and required you to explicitly -write `ref` or `ref mut` in patterns to switch to a borrowing mode. In Rust -2018, the type of the value being matched informs the binding mode, so that -if you match against an `&Option` with a `Some` variant, you are put -into `ref` mode automatically, giving you a borrowed view of the internal -data. Similarly, `&mut Option` would give you a `ref mut` view. diff --git a/src/doc/edition-guide/src/rust-2018/ownership-and-lifetimes/index.md b/src/doc/edition-guide/src/rust-2018/ownership-and-lifetimes/index.md deleted file mode 100644 index 0645610753..0000000000 --- a/src/doc/edition-guide/src/rust-2018/ownership-and-lifetimes/index.md +++ /dev/null @@ -1,6 +0,0 @@ -# Ownership and lifetimes - -[dmbm]: default-match-bindings.md - -In this chapter of the guide, we discuss a few improvements to ownership and lifetimes. -One of the most notable of these is [default match binding modes][dmbm]. diff --git a/src/doc/edition-guide/src/rust-2018/ownership-and-lifetimes/inference-in-structs.md b/src/doc/edition-guide/src/rust-2018/ownership-and-lifetimes/inference-in-structs.md deleted file mode 100644 index d4396ffd7e..0000000000 --- a/src/doc/edition-guide/src/rust-2018/ownership-and-lifetimes/inference-in-structs.md +++ /dev/null @@ -1,72 +0,0 @@ -# `T: 'a` inference in structs - -![Minimum Rust version: 1.31](https://img.shields.io/badge/Minimum%20Rust%20Version-1.31-brightgreen.svg) - -An annotation in the form of `T: 'a`, where `T` is either a type or another -lifetime, is called an *"outlives"* requirement. Note that *"outlives"* also -implies `'a: 'a`. - -One way in which edition 2018 helps you out in maintaining flow when writing -programs is by removing the need to explicitly annotate these `T: 'a` outlives -requirements in `struct` definitions. Instead, the requirements will be -inferred from the fields present in the definitions. - -Consider the following `struct` definitions in Rust 2015: - -```rust -// Rust 2015 - -struct Ref<'a, T: 'a> { - field: &'a T -} - -// or written with a `where` clause: - -struct WhereRef<'a, T> where T: 'a { - data: &'a T -} - -// with nested references: - -struct RefRef<'a, 'b: 'a, T: 'b> { - field: &'a &'b T, -} - -// using an associated type: - -struct ItemRef<'a, T: Iterator> -where - T::Item: 'a -{ - field: &'a T::Item -} -``` - -In Rust 2018, since the requirements are inferred, you can instead write: - -```rust,ignore -// Rust 2018 - -struct Ref<'a, T> { - field: &'a T -} - -struct WhereRef<'a, T> { - data: &'a T -} - -struct RefRef<'a, 'b, T> { - field: &'a &'b T, -} - -struct ItemRef<'a, T: Iterator> { - field: &'a T::Item -} -``` - -If you prefer to be more explicit in some cases, that is still possible. - -## More details - -For more details, see [the tracking issue](https://github.com/rust-lang/rust/issues/44493) -and [the RFC](https://github.com/rust-lang/rfcs/pull/2093). diff --git a/src/doc/edition-guide/src/rust-2018/ownership-and-lifetimes/lifetime-elision-in-impl.md b/src/doc/edition-guide/src/rust-2018/ownership-and-lifetimes/lifetime-elision-in-impl.md deleted file mode 100644 index a8c22c30df..0000000000 --- a/src/doc/edition-guide/src/rust-2018/ownership-and-lifetimes/lifetime-elision-in-impl.md +++ /dev/null @@ -1,75 +0,0 @@ -# Lifetime elision in impl - -![Minimum Rust version: 1.31](https://img.shields.io/badge/Minimum%20Rust%20Version-1.31-brightgreen.svg) - -When writing `impl` blocks, you can now elide lifetime annotations in some -situations. - -Consider a trait like `MyIterator`: - -```rust,ignore -trait MyIterator { - type Item; - fn next(&mut self) -> Option; -} -``` - -In Rust 2015, if we wanted to implement this iterator for mutable references -to `Iterators`, we'd need to write this: - -```rust,ignore -impl<'a, I: MyIterator> MyIterator for &'a mut I { - type Item = I::Item; - fn next(&mut self) -> Option { - (*self).next() - } -} -``` - -Note all of the `'a` annotations. In Rust 2018, we can write this: - -```rust,ignore -impl MyIterator for &mut I { - type Item = I::Item; - fn next(&mut self) -> Option { - (*self).next() - } -} -``` - -Similarly, lifetime annotations can appear due to a struct that contains -references: - -```rust,ignore -struct SetOnDrop<'a, T> { - borrow: &'a mut T, - value: Option, -} -``` - -In Rust 2015, to implement `Drop` on this struct, we'd write: - -```rust,ignore -impl<'a, T> Drop for SetOnDrop<'a, T> { - fn drop(&mut self) { - if let Some(x) = self.value.take() { - *self.borrow = x; - } - } -} -``` - -But in Rust 2018, we can combine elision with [the anonymous lifetime] and -write this instead. - -```rust,ignore -impl Drop for SetOnDrop<'_, T> { - fn drop(&mut self) { - if let Some(x) = self.value.take() { - *self.borrow = x; - } - } -} -``` - -[the anonymous lifetime]: the-anonymous-lifetime.md \ No newline at end of file diff --git a/src/doc/edition-guide/src/rust-2018/ownership-and-lifetimes/non-lexical-lifetimes.md b/src/doc/edition-guide/src/rust-2018/ownership-and-lifetimes/non-lexical-lifetimes.md deleted file mode 100644 index 8daaef9a5d..0000000000 --- a/src/doc/edition-guide/src/rust-2018/ownership-and-lifetimes/non-lexical-lifetimes.md +++ /dev/null @@ -1,85 +0,0 @@ -# Non-lexical lifetimes - -![Minimum Rust version: 1.31](https://img.shields.io/badge/Minimum%20Rust%20Version-1.31-brightgreen.svg) for 2018 edition - -![Minimum Rust version: 1.36](https://img.shields.io/badge/Minimum%20Rust%20Version-1.36-brightgreen.svg) for 2015 edition - -The borrow checker has been enhanced to accept more code, via a mechanism -called "non-lexical lifetimes." Consider this example: - -```rust,ignore -fn main() { - let mut x = 5; - - let y = &x; - - let z = &mut x; -} -``` - -In older Rust, this is a compile-time error: - -```text -error[E0502]: cannot borrow `x` as mutable because it is also borrowed as immutable - --> src/main.rs:5:18 - | -4 | let y = &x; - | - immutable borrow occurs here -5 | let z = &mut x; - | ^ mutable borrow occurs here -6 | } - | - immutable borrow ends here -``` - -This is because lifetimes follow "lexical scope"; that is, the borrow from `y` is -considered to be held until `y` goes out of scope at the end of `main`, even though -we never use `y` again. This code is fine, but the borrow checker could not handle it. - -Today, this code will compile just fine. - -## Better errors - -What if we did use `y`, like this? - -```rust,ignore -fn main() { - let mut x = 5; - let y = &x; - let z = &mut x; - - println!("y: {}", y); -} -``` - -Here's the error: - -```text -error[E0502]: cannot borrow `x` as mutable because it is also borrowed as immutable - --> src/main.rs:5:18 - | -4 | let y = &x; - | - immutable borrow occurs here -5 | let z = &mut x; - | ^ mutable borrow occurs here -... -8 | } - | - immutable borrow ends here -``` - -With non-lexical lifetimes, the error changes slightly: - -```text -error[E0502]: cannot borrow `x` as mutable because it is also borrowed as immutable - --> src/main.rs:5:13 - | -4 | let y = &x; - | -- immutable borrow occurs here -5 | let z = &mut x; - | ^^^^^^ mutable borrow occurs here -6 | -7 | println!("y: {}", y); - | - borrow later used here -``` - -Instead of pointing to where `y` goes out of scope, it shows you where -the conflicting borrow occurs. This makes these sorts of errors *far* easier to debug. diff --git a/src/doc/edition-guide/src/rust-2018/ownership-and-lifetimes/simpler-lifetimes-in-static-and-const.md b/src/doc/edition-guide/src/rust-2018/ownership-and-lifetimes/simpler-lifetimes-in-static-and-const.md deleted file mode 100644 index 113c5705aa..0000000000 --- a/src/doc/edition-guide/src/rust-2018/ownership-and-lifetimes/simpler-lifetimes-in-static-and-const.md +++ /dev/null @@ -1,41 +0,0 @@ -# Simpler lifetimes in `static` and `const` - -![Minimum Rust version: 1.17](https://img.shields.io/badge/Minimum%20Rust%20Version-1.17-brightgreen.svg) - -In older Rust, you had to explicitly write the `'static` lifetime in any -`static` or `const` that needed a lifetime: - -```rust -# mod foo { -const NAME: &'static str = "Ferris"; -# } -# mod bar { -static NAME: &'static str = "Ferris"; -# } -``` - -But `'static` is the only possible lifetime there. So Rust now assumes the `'static` lifetime, -and you don't have to write it out: - -```rust -# mod foo { -const NAME: &str = "Ferris"; -# } -# mod bar { -static NAME: &str = "Ferris"; -# } -``` - -In some situations, this can remove a *lot* of boilerplate: - -```rust -# mod foo { -// old -const NAMES: &'static [&'static str; 2] = &["Ferris", "Bors"]; -# } -# mod bar { - -// new -const NAMES: &[&str; 2] = &["Ferris", "Bors"]; -# } -``` diff --git a/src/doc/edition-guide/src/rust-2018/ownership-and-lifetimes/the-anonymous-lifetime.md b/src/doc/edition-guide/src/rust-2018/ownership-and-lifetimes/the-anonymous-lifetime.md deleted file mode 100644 index aa2da7a508..0000000000 --- a/src/doc/edition-guide/src/rust-2018/ownership-and-lifetimes/the-anonymous-lifetime.md +++ /dev/null @@ -1,96 +0,0 @@ -# `'_`, the anonymous lifetime - -![Minimum Rust version: 1.31](https://img.shields.io/badge/Minimum%20Rust%20Version-1.31-brightgreen.svg) - -Rust 2018 allows you to explicitly mark where a lifetime is elided, for types -where this elision might otherwise be unclear. To do this, you can use the -special lifetime `'_` much like you can explicitly mark that a type is inferred -with the syntax `let x: _ = ..;`. - -Let's say, for whatever reason, that we have a simple wrapper around `&'a str`: - -```rust -struct StrWrap<'a>(&'a str); -``` - -In Rust 2015, you might have written: - -```rust -# use std::fmt; -# struct StrWrap<'a>(&'a str); -// Rust 2015 - -fn make_wrapper(string: &str) -> StrWrap { - StrWrap(string) -} - -impl<'a> fmt::Debug for StrWrap<'a> { - fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result { - fmt.write_str(self.0) - } -} -``` - -In Rust 2018, you can instead write: - -```rust -# use std::fmt; -# struct StrWrap<'a>(&'a str); -// Rust 2018 - -fn make_wrapper(string: &str) -> StrWrap<'_> { - StrWrap(string) -} - -impl fmt::Debug for StrWrap<'_> { - fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { - fmt.write_str(self.0) - } -} -``` - -## More details - -In the Rust 2015 snippet above, we've used `-> StrWrap`. However, unless you take -a look at the definition of `StrWrap`, it is not clear that the returned value -is actually borrowing something. Therefore, starting with Rust 2018, it is -deprecated to leave off the lifetime parameters for non-reference-types (types -other than `&` and `&mut`). Instead, where you previously wrote `-> StrWrap`, -you should now write `-> StrWrap<'_>`, making clear that borrowing is occurring. - -What exactly does `'_` mean? It depends on the context! -In output contexts, as in the return type of `make_wrapper`, -it refers to a single lifetime for all "output" locations. -In input contexts, a fresh lifetime is generated for each "input location". -More concretely, to understand input contexts, consider the following example: - -```rust -// Rust 2015 - -struct Foo<'a, 'b: 'a> { - field: &'a &'b str, -} - -impl<'a, 'b: 'a> Foo<'a, 'b> { - // some methods... -} -``` - -We can rewrite this as: - -```rust -# struct Foo<'a, 'b: 'a> { -# field: &'a &'b str, -# } - -// Rust 2018 - -impl Foo<'_, '_> { - // some methods... -} -``` - -This is the same, because for each `'_`, a fresh lifetime is generated. -Finally, the relationship `'a: 'b` which the struct requires must be upheld. - -For more details, see the [tracking issue on In-band lifetime bindings](https://github.com/rust-lang/rust/issues/44524). diff --git a/src/doc/edition-guide/src/rust-2018/module-system/path-clarity.md b/src/doc/edition-guide/src/rust-2018/path-changes.md similarity index 91% rename from src/doc/edition-guide/src/rust-2018/module-system/path-clarity.md rename to src/doc/edition-guide/src/rust-2018/path-changes.md index 03270aa034..fc16d472ba 100644 --- a/src/doc/edition-guide/src/rust-2018/module-system/path-clarity.md +++ b/src/doc/edition-guide/src/rust-2018/path-changes.md @@ -1,7 +1,15 @@ -# Path clarity +# Path and module system changes ![Minimum Rust version: 1.31](https://img.shields.io/badge/Minimum%20Rust%20Version-1.31-brightgreen.svg) +## Summary + +- Paths in `use` declarations now work the same as other paths. +- Paths starting with `::` must now be followed with an external crate. +- Paths in `pub(in path)` visibility modifiers must now start with `crate`, `self`, or `super`. + +## Motivation + The module system is often one of the hardest things for people new to Rust. Everyone has their own things that take time to master, of course, but there's a root cause for why it's so confusing to many: while there are simple and @@ -91,19 +99,39 @@ Some examples of needing to explicitly import sysroot crates are: * [`test`]: This is only available on the [nightly channel], and is usually only used for the unstable benchmark support. -[`alloc`]: ../../../alloc/index.html -[`core`]: ../../../core/index.html -[`proc_macro`]: ../../../proc_macro/index.html -[`std`]: ../../../std/index.html -[`test`]: ../../../test/index.html -[nightly channel]: ../../../book/appendix-07-nightly-rust.html +[`alloc`]: ../../alloc/index.html +[`core`]: ../../core/index.html +[`proc_macro`]: ../../proc_macro/index.html +[`std`]: ../../std/index.html +[`test`]: ../../test/index.html +[nightly channel]: ../../book/appendix-07-nightly-rust.html [no_core]: https://github.com/rust-lang/rust/issues/29639 -[no_std]: ../../../reference/names/preludes.html#the-no_std-attribute +[no_std]: ../../reference/names/preludes.html#the-no_std-attribute #### Macros One other use for `extern crate` was to import macros; that's no longer needed. -Check [the macro section](../macros/macro-changes.md) for more. +Macros may be imported with `use` like any other item. For example, the +following use of `extern crate`: + +```rust,ignore +#[macro_use] +extern crate bar; + +fn main() { + baz!(); +} +``` + +Can be changed to something like the following: + +```rust,ignore +use bar::baz; + +fn main() { + baz!(); +} +``` #### Renaming crates diff --git a/src/doc/edition-guide/src/rust-2018/platform-and-target-support/cdylib-crates-for-c-interoperability.md b/src/doc/edition-guide/src/rust-2018/platform-and-target-support/cdylib-crates-for-c-interoperability.md deleted file mode 100644 index 5a8a394f7d..0000000000 --- a/src/doc/edition-guide/src/rust-2018/platform-and-target-support/cdylib-crates-for-c-interoperability.md +++ /dev/null @@ -1,18 +0,0 @@ -# cdylib crates for C interoperability - -![Minimum Rust version: 1.10](https://img.shields.io/badge/Minimum%20Rust%20Version-1.10-brightgreen.svg) for `rustc` - -![Minimum Rust version: 1.11](https://img.shields.io/badge/Minimum%20Rust%20Version-1.11-brightgreen.svg) for `cargo` - -If you're producing a library that you intend to be used from C (or another -language through a C FFI), there's no need for Rust to include Rust-specific -stuff in the final object code. For libraries like that, you'll want to use -the `cdylib` crate type in your `Cargo.toml`: - -```toml -[lib] -crate-type = ["cdylib"] -``` - -This will produce a smaller binary, with no Rust-specific information inside -of it. diff --git a/src/doc/edition-guide/src/rust-2018/platform-and-target-support/global-allocators.md b/src/doc/edition-guide/src/rust-2018/platform-and-target-support/global-allocators.md deleted file mode 100644 index 43f6ce8d67..0000000000 --- a/src/doc/edition-guide/src/rust-2018/platform-and-target-support/global-allocators.md +++ /dev/null @@ -1,35 +0,0 @@ -# Global allocators - -![Minimum Rust version: 1.28](https://img.shields.io/badge/Minimum%20Rust%20Version-1.28-brightgreen.svg) - -Allocators are the way that programs in Rust obtain memory from the system at -runtime. Previously, Rust did not allow changing the way memory is obtained, -which prevented some use cases. On some platforms, this meant using jemalloc, -on others, the system allocator, but there was no way for users to control -this key component. With 1.28.0, the `#[global_allocator]` attribute is now -stable, which allows Rust programs to set their allocator to the system -allocator, as well as define new allocators by implementing the `GlobalAlloc` -trait. - -The default allocator for Rust programs on some platforms is jemalloc. The -standard library now provides a handle to the system allocator, which can be -used to switch to the system allocator when desired, by declaring a static -and marking it with the `#[global_allocator]` attribute. - -```rust -use std::alloc::System; - -#[global_allocator] -static GLOBAL: System = System; - -fn main() { - let mut v = Vec::new(); - // This will allocate memory using the system allocator. - v.push(1); -} -``` - -However, sometimes you want to define a custom allocator for a given -application domain. This is also relatively easy to do by implementing the -`GlobalAlloc` trait. You can read more about how to do this in [the -documentation](https://doc.rust-lang.org/std/alloc/trait.GlobalAlloc.html). \ No newline at end of file diff --git a/src/doc/edition-guide/src/rust-2018/platform-and-target-support/index.md b/src/doc/edition-guide/src/rust-2018/platform-and-target-support/index.md deleted file mode 100644 index bf2fcdf179..0000000000 --- a/src/doc/edition-guide/src/rust-2018/platform-and-target-support/index.md +++ /dev/null @@ -1,6 +0,0 @@ -# Platform and target support - -[libcore]: libcore-for-low-level-rust.md - -In this chapter of the guide, we discuss a few improvements to platform and target support. -A notable addition to it was [that the `libcore` library now works on stable Rust][libcore]. \ No newline at end of file diff --git a/src/doc/edition-guide/src/rust-2018/platform-and-target-support/libcore-for-low-level-rust.md b/src/doc/edition-guide/src/rust-2018/platform-and-target-support/libcore-for-low-level-rust.md deleted file mode 100644 index fc64cba67d..0000000000 --- a/src/doc/edition-guide/src/rust-2018/platform-and-target-support/libcore-for-low-level-rust.md +++ /dev/null @@ -1,31 +0,0 @@ -# libcore for low-level Rust - -![Minimum Rust version: 1.6](https://img.shields.io/badge/Minimum%20Rust%20Version-1.6-brightgreen.svg) - -Rust’s standard library is two-tiered: there’s a small core library, -`libcore`, and the full standard library, `libstd`, that builds on top of it. -`libcore` is completely platform agnostic, and requires only a handful of -external symbols to be defined. Rust’s `libstd` builds on top of `libcore`, -adding support for things like memory allocation and I/O. Applications using -Rust in the embedded space, as well as those writing operating systems, often -eschew `libstd`, using only `libcore`. - -As an additional note, while building *libraries* with `libcore` is supported -today, building full applications is not yet stable. - -To use `libcore`, add this flag to your crate root: - -```rust,ignore -#![no_std] -``` - -This will remove the standard library, and bring the `core` crate into your -namespace for use: - -```rust,ignore -#![no_std] - -use core::cell::Cell; -``` - -You can find `libcore`'s documentation [here](https://doc.rust-lang.org/core/). diff --git a/src/doc/edition-guide/src/rust-2018/platform-and-target-support/msvc-toolchain-support.md b/src/doc/edition-guide/src/rust-2018/platform-and-target-support/msvc-toolchain-support.md deleted file mode 100644 index 67549e5588..0000000000 --- a/src/doc/edition-guide/src/rust-2018/platform-and-target-support/msvc-toolchain-support.md +++ /dev/null @@ -1,18 +0,0 @@ -# MSVC toolchain support - -![Minimum Rust version: 1.2](https://img.shields.io/badge/Minimum%20Rust%20Version-1.2-brightgreen.svg) - -At the release of Rust 1.0, we only supported the GNU toolchain on Windows. With the -release of Rust 1.2, we introduced initial support for the MSVC toolchain. After that, -as support matured, we eventually made it the default choice for Windows users. - -The difference between the two matters for interacting with C. If you're using a library -built with one toolchain or another, you need to match that with the appropriate Rust -toolchain. If you're not sure, go with MSVC; it's the default for good reason. - -To use this feature, simply use Rust on Windows, and the installer will default to it. -If you'd prefer to switch to the GNU toolchain, you can install it with Rustup: - -```console -$ rustup toolchain install stable-x86_64-pc-windows-gnu -``` \ No newline at end of file diff --git a/src/doc/edition-guide/src/rust-2018/platform-and-target-support/musl-support-for-fully-static-binaries.md b/src/doc/edition-guide/src/rust-2018/platform-and-target-support/musl-support-for-fully-static-binaries.md deleted file mode 100644 index 4a3752f6e9..0000000000 --- a/src/doc/edition-guide/src/rust-2018/platform-and-target-support/musl-support-for-fully-static-binaries.md +++ /dev/null @@ -1,45 +0,0 @@ -# MUSL support for fully static binaries - -![Minimum Rust version: 1.1](https://img.shields.io/badge/Minimum%20Rust%20Version-1.1-brightgreen.svg) - -By default, Rust will statically link all Rust code. However, if you use the -standard library, it will dynamically link to the system's `libc` -implementation. - -If you'd like a 100% static binary, the [`MUSL -libc`](https://musl.libc.org/) can be used on Linux. - -## Installing MUSL support - -To add support for MUSL, you need to choose the correct target. [The forge -has a full list of -targets](https://forge.rust-lang.org/release/platform-support.html) supported, -with a number of ones using `musl`. - -If you're not sure what you want, it's probably `x86_64-unknown-linux-musl`, -for 64-bit Linux. We'll be using this target in this guide, but the -instructions remain the same for other targets, just change the name wherever -we mention the target. - -To get support for this target, you use `rustup`: - -```console -$ rustup target add x86_64-unknown-linux-musl -``` - -This will install support for the default toolchain; to install for other toolchains, -add the `--toolchain` flag. For example: - -```console -$ rustup target add x86_64-unknown-linux-musl --toolchain=nightly -``` - -## Building with MUSL - -To use this new target, pass the `--target` flag to Cargo: - -```console -$ cargo build --target x86_64-unknown-linux-musl -``` - -The binary produced will now be built with MUSL! diff --git a/src/doc/edition-guide/src/rust-2018/platform-and-target-support/webassembly-support.md b/src/doc/edition-guide/src/rust-2018/platform-and-target-support/webassembly-support.md deleted file mode 100644 index 55a3a2805d..0000000000 --- a/src/doc/edition-guide/src/rust-2018/platform-and-target-support/webassembly-support.md +++ /dev/null @@ -1,28 +0,0 @@ -# WebAssembly support - -![Minimum Rust version: 1.14](https://img.shields.io/badge/Minimum%20Rust%20Version-1.14-brightgreen.svg) for `emscripten` - -![Minimum Rust version: nightly](https://img.shields.io/badge/Minimum%20Rust%20Version-nightly-red.svg) for `wasm32-unknown-unknown` - -Rust has gained support for [WebAssembly](https://webassembly.org/), meaning -that you can run Rust code in your browser, client-side. - -In Rust 1.14, we gained support through -[emscripten](http://kripken.github.io/emscripten-site/index.html). With it -installed, you can write Rust code and have it produce -[asm.js](http://asmjs.org/) (the precusor to wasm) and/or WebAssembly. - -Here's an example of using this support: - -```console -$ rustup target add wasm32-unknown-emscripten -$ echo 'fn main() { println!("Hello, Emscripten!"); }' > hello.rs -$ rustc --target=wasm32-unknown-emscripten hello.rs -$ node hello.js -``` - -However, in the meantime, Rust has also grown its own support, independent -from Emscripten. This is known as "the unknown target", because instead of -`wasm32-unknown-emscripten`, it's `wasm32-unknown-unknown`. This will be -the preferred target to use once it's ready, but for now, it's really -only well-supported in nightly. \ No newline at end of file diff --git a/src/doc/edition-guide/src/rust-2018/rustdoc/documentation-tests-can-now-compile-fail.md b/src/doc/edition-guide/src/rust-2018/rustdoc/documentation-tests-can-now-compile-fail.md deleted file mode 100644 index 7fbf4fb7e3..0000000000 --- a/src/doc/edition-guide/src/rust-2018/rustdoc/documentation-tests-can-now-compile-fail.md +++ /dev/null @@ -1,19 +0,0 @@ -# Documentation tests can now compile-fail - -![Minimum Rust version: 1.22](https://img.shields.io/badge/Minimum%20Rust%20Version-1.22-brightgreen.svg) - -You can now create `compile-fail` tests in Rustdoc, like this: - -```rust -/// ```compile_fail -/// let x = 5; -/// x += 2; // shouldn't compile! -/// ``` -# fn foo() {} -``` - -Please note that these kinds of tests can be more fragile than others, as -additions to Rust may cause code to compile when it previously would not. -Consider the first release with `?`, for example: code using `?` would fail -to compile on Rust 1.21, but compile successfully on Rust 1.22, causing your -test suite to start failing. diff --git a/src/doc/edition-guide/src/rust-2018/rustdoc/index.md b/src/doc/edition-guide/src/rust-2018/rustdoc/index.md deleted file mode 100644 index a63ee9a2e3..0000000000 --- a/src/doc/edition-guide/src/rust-2018/rustdoc/index.md +++ /dev/null @@ -1,6 +0,0 @@ -# `rustdoc` - -[cf]: documentation-tests-can-now-compile-fail.md - -In this chapter of the guide, we discuss a few improvements to `rustdoc`. -A notable addition to it was [that documentation tests can now compile-fail][cf]. \ No newline at end of file diff --git a/src/doc/edition-guide/src/rust-2018/rustdoc/rustdoc-uses-commonmark.md b/src/doc/edition-guide/src/rust-2018/rustdoc/rustdoc-uses-commonmark.md deleted file mode 100644 index c187114988..0000000000 --- a/src/doc/edition-guide/src/rust-2018/rustdoc/rustdoc-uses-commonmark.md +++ /dev/null @@ -1,16 +0,0 @@ -# Rustdoc uses CommonMark - -![Minimum Rust version: 1.25](https://img.shields.io/badge/Minimum%20Rust%20Version-1.25-brightgreen.svg) for support by default - -![Minimum Rust version: 1.23](https://img.shields.io/badge/Minimum%20Rust%20Version-1.23-red.svg) for support via a flag - -Rustdoc lets you write documentation comments in Markdown. At Rust 1.0, we -were using the `hoedown` markdown implementation, written in C. Markdown is -more of a family of implementations of an idea, and so `hoedown` had its own -dialect, like many parsers. The [CommonMark project](https://commonmark.org/) -has attempted to define a more strict version of Markdown, and so now, Rustdoc -uses it by default. - -As of Rust 1.23, we still defaulted to `hoedown`, but you could enable -Commonmark via a flag, `--enable-commonmark`. Today, we only support -CommonMark. \ No newline at end of file diff --git a/src/doc/edition-guide/src/rust-2018/rustup-for-managing-rust-versions.md b/src/doc/edition-guide/src/rust-2018/rustup-for-managing-rust-versions.md deleted file mode 100644 index 715f3fec13..0000000000 --- a/src/doc/edition-guide/src/rust-2018/rustup-for-managing-rust-versions.md +++ /dev/null @@ -1,224 +0,0 @@ -# Rustup for managing Rust versions - -![Minimum Rust version: various](https://img.shields.io/badge/Minimum%20Rust%20Version-various-brightgreen.svg) (this tool has its own versioning scheme and works with all Rust versions) - -The [Rustup](https://rustup.rs/) tool has become *the* recommended way to -install Rust, and is advertised on our website. Its powers go further than -that though, allowing you to manage various versions, components, and -platforms. - -## For installing Rust - -To install Rust through Rustup, you can go to -, which will let you know how to do -so on your platform. This will install both `rustup` itself and the `stable` -version of `rustc` and `cargo`. - -To install a specific Rust version, you can use `rustup toolchain install`: - -```console -$ rustup toolchain install 1.30.0 -``` - -This works for a specific nightly, as well: - -```console -$ rustup toolchain install nightly-2018-08-01 -``` - -As well as any of our release channels: - -```console -$ rustup toolchain install stable -$ rustup toolchain install beta -$ rustup toolchain install nightly -``` - -## For updating your installation - -To update all of the various channels you may have installed: - -```console -$ rustup update -``` - -This will look at everything you've installed, and if there are new releases, -will update anything that has one. - -## Managing versions - -To set the default toolchain to something other than `stable`: - -```console -$ rustup default nightly -``` - -To uninstall a specific Rust version, you can use `rustup toolchain uninstall`: - -```console -$ rustup toolchain uninstall 1.30.0 -``` - -To use a toolchain other than the default, use `rustup run`: - -```console -$ rustup run nightly cargo build -``` - -There's also an alias for this that's a little shorter: - -```console -$ cargo +nightly build -``` - -If you'd like to have a different default per-directory, that's easy too! -If you run this inside of a project: - -```console -$ rustup override set nightly -``` - -Or, if you'd like to target a different version of Rust: -```console -$ rustup override set 1.30.0 -``` - -Then when you're in that directory, any invocations of `rustc` or `cargo` -will use that toolchain. To share this with others, you can create a -`rust-toolchain` file with the contents of a toolchain, and check it into -source control. Now, when someone clones your project, they'll get the -right version without needing to `override set` themselves. - -## Installing other targets - -Rust supports cross-compiling to other targets, and Rustup can help you -manage them. For example, to use MUSL: - -```console -$ rustup target add x86_64-unknown-linux-musl -``` - -And then you can - -```console -$ cargo build --target=x86_64-unknown-linux-musl -``` - -To see the full list of targets you can install: - -```console -$ rustup target list -``` - -## Installing components - -Components are used to install certain kinds of tools. While `cargo-install` -has you covered for most tools, some tools need deep integration into the -compiler. Rustup knows exactly what version of the compiler you're using, and -so it's got just the information that these tools need. - -Components are per-toolchain, so if you want them to be available to more -than one toolchain, you'll need to install them multiple times. In the -following examples, add a `--toolchain` flag, set to the toolchain you -want to install for, `nightly` for example. Without this flag, it will -install the component for the default toolchain. - -To see the full list of components you can install: - -```console -$ rustup component list -``` - -Next, let's talk about some popular components and when you might want to -install them. - -### `rust-docs`, for local documentation - -This first component is installed by default when you install a toolchain. It -contains a copy of Rust's documentation, so that you can read it offline. - -This component cannot be removed for now; if that's of interest, please -comment on [this -issue](https://github.com/rust-lang/rustup.rs/issues/998). - -### `rust-src` for a copy of Rust's source code - -The `rust-src` component can give you a local copy of Rust's source code. Why -might you need this? Well, autocompletion tools like Racer use this -information to know more about the functions you're trying to call. - -```console -$ rustup component add rust-src -``` - -### `rustfmt` for automatic code formatting - -![Minimum Rust version: 1.24](https://img.shields.io/badge/Minimum%20Rust%20Version-1.24-brightgreen.svg) - -If you'd like to have your code automatically formatted, you can -install this component: - -```console -$ rustup component add rustfmt -``` - -This will install two tools, `rustfmt` and `cargo-fmt`, that will reformat your -code for you! For example: - -```console -$ cargo fmt -``` - -will reformat your entire Cargo project. - -### `rls` for IDE integration - -![Minimum Rust version: 1.21](https://img.shields.io/badge/Minimum%20Rust%20Version-1.21-brightgreen.svg) - -Many IDE features are built off of the [`langserver` -protocol](http://langserver.org/). To gain support for Rust with these IDEs, -you'll need to install the Rust language sever, aka the "RLS": - -```console -$ rustup component add rls -``` - -For more information about integrating this into your IDE, see the [RLS -documentation](https://github.com/rust-lang/rls). - -### `clippy` for more lints - -For even more lints to help you write Rust code, you can install `clippy`: - -```console -$ rustup component add clippy -``` - -This will install `cargo-clippy` for you: - -```console -$ cargo clippy -``` - -For more, check out [clippy's -documentation](https://github.com/rust-lang/rust-clippy). - -### The "preview" components - -There are several components in a "preview" stage. These components currently -have `-preview` in their name, and this indicates that they're not quite 100% -ready for general consumption yet. Please try them out and give us feedback, -but know that they do not follow Rust's stability guarantees, and are still -actively changing, possibly in backwards-incompatible ways. - -#### `llvm-tools-preview` for using extra LLVM tools - -If you'd like to use the `lld` linker, or other tools like `llvm-objdump` or -`llvm-objcopy`, you can install this component: - -```console -$ rustup component add llvm-tools-preview -``` - -This is the newest component, and so doesn't have good documentation at the -moment. diff --git a/src/doc/edition-guide/src/rust-2018/simd-for-faster-computing.md b/src/doc/edition-guide/src/rust-2018/simd-for-faster-computing.md deleted file mode 100644 index b7943eb085..0000000000 --- a/src/doc/edition-guide/src/rust-2018/simd-for-faster-computing.md +++ /dev/null @@ -1,108 +0,0 @@ -# SIMD for faster computing - -![Minimum Rust version: 1.27](https://img.shields.io/badge/Minimum%20Rust%20Version-1.27-brightgreen.svg) - -The basics of [SIMD](https://en.wikipedia.org/wiki/SIMD) are now available! -SIMD stands for “single instruction, multiple data.” Consider a function like -this: - -```rust -pub fn foo(a: &[u8], b: &[u8], c: &mut [u8]) { - for ((a, b), c) in a.iter().zip(b).zip(c) { - *c = *a + *b; - } -} -``` - -Here, we’re taking two slices, and adding the numbers together, placing the -result in a third slice. The simplest possible way to do this would be to do -exactly what the code does, and loop through each set of elements, add them -together, and store it in the result. However, compilers can often do better. -LLVM will usually “autovectorize” code like this, which is a fancy term for -“use SIMD.” Imagine that `a` and `b` were both 16 elements long. Each element -is a `u8`, and so that means that each slice would be 128 bits of data. Using -SIMD, we could put both `a` and `b` into 128 bit registers, add them together -in a *single* instruction, and then copy the resulting 128 bits into `c`. -That’d be much faster! - -While stable Rust has always been able to take advantage of -autovectorization, sometimes, the compiler just isn’t smart enough to realize -that we can do something like this. Additionally, not every CPU has these -features, and so LLVM may not use them so your program can be used on a wide -variety of hardware. The `std::arch` module allows us to use these kinds of -instructions directly, which means we don’t need to rely on a smart compiler. -Additionally, it includes some features that allow us to choose a particular -implementation based on various criteria. For example: - -```rust,ignore -#[cfg(all(any(target_arch = "x86", target_arch = "x86_64"), - target_feature = "avx2"))] -fn foo() { - #[cfg(target_arch = "x86")] - use std::arch::x86::_mm256_add_epi64; - #[cfg(target_arch = "x86_64")] - use std::arch::x86_64::_mm256_add_epi64; - - unsafe { - _mm256_add_epi64(...); - } -} -``` - -Here, we use cfg flags to choose the correct version based on the machine -we’re targeting; on x86 we use that version, and on x86_64 we use its -version. We can also choose at runtime: - -```rust,ignore -fn foo() { - #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] - { - if is_x86_feature_detected!("avx2") { - return unsafe { foo_avx2() }; - } - } - - foo_fallback(); -} -``` - -Here, we have two versions of the function: one which uses AVX2, a specific -kind of SIMD feature that lets you do 256-bit operations. The -`is_x86_feature_detected!` macro will generate code that detects if your CPU -supports AVX2, and if so, calls the foo_avx2 function. If not, then we fall -back to a non-AVX implementation, foo_fallback. This means that our code will -run super fast on CPUs that support AVX2, but still work on ones that don’t, -albeit slower. - -If all of this seems a bit low-level and fiddly, well, it is! `std::arch` is -specifically primitives for building these kinds of things. We hope to -eventually stabilize a `std::simd` module with higher-level stuff in the -future. But landing the basics now lets the ecosystem experiment with higher -level libraries starting today. For example, check out the -[faster](https://github.com/AdamNiederer/faster) crate. Here’s a code snippet -with no SIMD: - -```rust,ignore -let lots_of_3s = (&[-123.456f32; 128][..]).iter() - .map(|v| { - 9.0 * v.abs().sqrt().sqrt().recip().ceil().sqrt() - 4.0 - 2.0 - }) - .collect::>(); -``` - -To use SIMD with this code via faster, you’d change it to this: - -```rust,ignore -let lots_of_3s = (&[-123.456f32; 128][..]).simd_iter() - .simd_map(f32s(0.0), |v| { - f32s(9.0) * v.abs().sqrt().rsqrt().ceil().sqrt() - f32s(4.0) - f32s(2.0) - }) - .scalar_collect(); -``` - -It looks almost the same: `simd_iter` instead of `iter`, `simd_map` instead of `map`, -`f32s(2.0)` instead of `2.0`. But you get a SIMD-ified version generated for you. - -Beyond that, you may never write any of this yourself, but as always, the -libraries you depend on may. For example, the regex crate contains these SIMD -speedups without you needing to do anything at all! diff --git a/src/doc/edition-guide/src/rust-2018/slice-patterns.md b/src/doc/edition-guide/src/rust-2018/slice-patterns.md deleted file mode 100644 index a69dba15a6..0000000000 --- a/src/doc/edition-guide/src/rust-2018/slice-patterns.md +++ /dev/null @@ -1,91 +0,0 @@ -# Slice patterns - -![Minimum Rust version: 1.26](https://img.shields.io/badge/Minimum%20Rust%20Version-1.26-brightgreen.svg) - -Have you ever tried to pattern match on the contents and structure of a slice? -Rust 2018 will let you do just that. - -For example, say we want to accept a list of names and respond to that with a -greeting. With slice patterns, we can do that easy as pie with: - -```rust -fn main() { - greet(&[]); - // output: Bummer, there's no one here :( - greet(&["Alan"]); - // output: Hey, there Alan! You seem to be alone. - greet(&["Joan", "Hugh"]); - // output: Hello, Joan and Hugh. Nice to see you are exactly 2! - greet(&["John", "Peter", "Stewart"]); - // output: Hey everyone, we seem to be 3 here today. -} - -fn greet(people: &[&str]) { - match people { - [] => println!("Bummer, there's no one here :("), - [only_one] => println!("Hey, there {}! You seem to be alone.", only_one), - [first, second] => println!( - "Hello, {} and {}. Nice to see you are exactly 2!", - first, second - ), - _ => println!("Hey everyone, we seem to be {} here today.", people.len()), - } -} -``` - -Now, you don't have to check the length first. - -We can also match on arrays like so: - -```rust -let arr = [1, 2, 3]; - -assert_eq!("ends with 3", match arr { - [_, _, 3] => "ends with 3", - [a, b, c] => "ends with something else", -}); -``` - -## More details - -### Exhaustive patterns - -In the first example, note in particular the `_ => ...` pattern. -Since we are matching on a slice, it could be of any length, so we need a -*"catch all pattern"* to handle it. If we forgot the `_ => ...` or -`identifier => ...` pattern, we would instead get an error saying: - -```ignore -error[E0004]: non-exhaustive patterns: `&[_, _, _]` not covered -``` - -If we added a case for a slice of size `3` we would instead get: - -```ignore -error[E0004]: non-exhaustive patterns: `&[_, _, _, _]` not covered -``` - -and so on... - -### Arrays and exact lengths - -In the second example above, since arrays in Rust are of known lengths, -we have to match on exactly three elements. -If we try to match on 2 or 4 elements,we get the errors: - -```ignore -error[E0527]: pattern requires 2 elements but array has 3 -``` - -and - -```ignore -error[E0527]: pattern requires 4 elements but array has 3 -``` - -### In the pipeline - -[the tracking issue]: https://github.com/rust-lang/rust/issues/23121 - -When it comes to slice patterns, more advanced forms are planned but -have not been stabilized yet. To learn more, follow [the tracking issue]. diff --git a/src/doc/edition-guide/src/rust-2018/the-compiler/an-attribute-for-deprecation.md b/src/doc/edition-guide/src/rust-2018/the-compiler/an-attribute-for-deprecation.md deleted file mode 100644 index d414dbaa94..0000000000 --- a/src/doc/edition-guide/src/rust-2018/the-compiler/an-attribute-for-deprecation.md +++ /dev/null @@ -1,35 +0,0 @@ -# An attribute for deprecation - -![Minimum Rust version: 1.9](https://img.shields.io/badge/Minimum%20Rust%20Version-1.9-brightgreen.svg) - -If you're writing a library, and you'd like to deprecate something, you can -use the `deprecated` attribute: - -```rust -#[deprecated( - since = "0.2.1", - note = "Please use the bar function instead" -)] -pub fn foo() { - // ... -} -``` - -This will give your users a warning if they use the deprecated functionality: - -```text - Compiling playground v0.0.1 (file:///playground) -warning: use of deprecated item 'foo': Please use the bar function instead - --> src/main.rs:10:5 - | -10 | foo(); - | ^^^ - | - = note: #[warn(deprecated)] on by default - -``` - -Both `since` and `note` are optional. - -`since` can be in the future; you can put whatever you'd like, and what's put in -there isn't checked. diff --git a/src/doc/edition-guide/src/rust-2018/the-compiler/improved-error-messages.md b/src/doc/edition-guide/src/rust-2018/the-compiler/improved-error-messages.md deleted file mode 100644 index 711d00995f..0000000000 --- a/src/doc/edition-guide/src/rust-2018/the-compiler/improved-error-messages.md +++ /dev/null @@ -1,50 +0,0 @@ -# Improved error messages - -![Minimum Rust version: 1.12](https://img.shields.io/badge/Minimum%20Rust%20Version-1.12-brightgreen.svg) - -We're always working on error improvements, and there are little improvements -in almost every Rust version, but in Rust 1.12, a significant overhaul of the -error message system was created. - -For example, here's some code that produces an error: - -```rust,compile_fail -fn main() { - let mut x = 5; - let y = &x; - x += 1; - println!("{} {}", x, y); -} -``` - -Here's the error in Rust 1.11: - -```text -foo.rs:4:5: 4:11 error: cannot assign to `x` because it is borrowed [E0506] -foo.rs:4 x += 1; - ^~~~~~ -foo.rs:3:14: 3:15 note: borrow of `x` occurs here -foo.rs:3 let y = &x; - ^ -foo.rs:4:5: 4:11 help: run `rustc --explain E0506` to see a detailed explanation -error: aborting due to previous error -``` - -Here's the error in Rust 1.28: - -```text -error[E0506]: cannot assign to `x` because it is borrowed - --> foo.rs:4:5 - | -3 | let y = &x; - | - borrow of `x` occurs here -4 | x += 1; - | ^^^^^^ assignment to borrowed `x` occurs here - -error: aborting due to previous error - -For more information about this error, try `rustc --explain E0506`. -``` - -This error isn't terribly different, but shows off how the format has changed. It shows -off your code in context, rather than just showing the text of the lines themselves. diff --git a/src/doc/edition-guide/src/rust-2018/the-compiler/incremental-compilation-for-faster-compiles.md b/src/doc/edition-guide/src/rust-2018/the-compiler/incremental-compilation-for-faster-compiles.md deleted file mode 100644 index 8ce7a3c94f..0000000000 --- a/src/doc/edition-guide/src/rust-2018/the-compiler/incremental-compilation-for-faster-compiles.md +++ /dev/null @@ -1,23 +0,0 @@ -# Incremental Compilation - -![Minimum Rust version: 1.24](https://img.shields.io/badge/Minimum%20Rust%20Version-1.24-brightgreen.svg) - -Back in September of 2016, we [blogged about Incremental -Compilation](https://blog.rust-lang.org/2016/09/08/incremental.html). While -that post goes into the details, the idea is basically this: when you’re -working on a project, you often compile it, then change something small, then -compile again. Historically, the compiler has compiled your entire project, -no matter how little you’ve changed the code. The idea with incremental -compilation is that you only need to compile the code you’ve actually -changed, which means that that second build is faster. - -This is now turned on by default. This means that your builds should be -faster! Don’t forget about `cargo check` when trying to get the lowest possible -build times. - -This is still not the end story for compiler performance generally, nor -incremental compilation specifically. We have a lot more work planned in the -future. - -One small note about this change: it makes builds faster, but makes the final -binary a bit slower. For this reason, it's not turned on in release builds. \ No newline at end of file diff --git a/src/doc/edition-guide/src/rust-2018/the-compiler/index.md b/src/doc/edition-guide/src/rust-2018/the-compiler/index.md deleted file mode 100644 index c463757789..0000000000 --- a/src/doc/edition-guide/src/rust-2018/the-compiler/index.md +++ /dev/null @@ -1,6 +0,0 @@ -# The compiler - -[errors]: improved-error-messages.md - -In this chapter of the guide, we discuss a few improvements to the compiler. -A notable addition here is our new and [improved error messages][errors]. \ No newline at end of file diff --git a/src/doc/edition-guide/src/rust-2018/trait-system/no-anon-params.md b/src/doc/edition-guide/src/rust-2018/trait-fn-parameters.md similarity index 65% rename from src/doc/edition-guide/src/rust-2018/trait-system/no-anon-params.md rename to src/doc/edition-guide/src/rust-2018/trait-fn-parameters.md index c0607516e7..e82af19c07 100644 --- a/src/doc/edition-guide/src/rust-2018/trait-system/no-anon-params.md +++ b/src/doc/edition-guide/src/rust-2018/trait-fn-parameters.md @@ -1,7 +1,16 @@ -# No more anonymous trait parameters +# Anonymous trait function parameters deprecated ![Minimum Rust version: 1.31](https://img.shields.io/badge/Minimum%20Rust%20Version-1.31-brightgreen.svg) +## Summary + +- [Trait function parameters] may use any irrefutable pattern when the function has a body. + +[Trait function parameters]: https://doc.rust-lang.org/stable/reference/items/traits.html#parameter-patterns + + +## Details + In accordance with RFC [#1685](https://github.com/rust-lang/rfcs/pull/1685), parameters in trait method declarations are no longer allowed to be anonymous. diff --git a/src/doc/edition-guide/src/rust-2018/trait-system/associated-constants.md b/src/doc/edition-guide/src/rust-2018/trait-system/associated-constants.md deleted file mode 100644 index 26564943c2..0000000000 --- a/src/doc/edition-guide/src/rust-2018/trait-system/associated-constants.md +++ /dev/null @@ -1,117 +0,0 @@ -# Associated constants - -![Minimum Rust version: 1.20](https://img.shields.io/badge/Minimum%20Rust%20Version-1.20-brightgreen.svg) - -You can define traits, structs, and enums that have “associated functions”: - -```rust -struct Struct; - -impl Struct { - fn foo() { - println!("foo is an associated function of Struct"); - } -} - -fn main() { - Struct::foo(); -} -``` - -These are called “associated functions” because they are functions that are -associated with the type, that is, they’re attached to the type itself, and -not any particular instance. - -Rust 1.20 adds the ability to define “associated constants” as well: - -```rust -struct Struct; - -impl Struct { - const ID: u32 = 0; -} - -fn main() { - println!("the ID of Struct is: {}", Struct::ID); -} -``` - -That is, the constant `ID` is associated with `Struct`. Like functions, -associated constants work with traits and enums as well. - -Traits have an extra ability with associated constants that gives them some -extra power. With a trait, you can use an associated constant in the same way -you’d use an associated type: by declaring it, but not giving it a value. The -implementor of the trait then declares its value upon implementation: - -```rust -trait Trait { - const ID: u32; -} - -struct Struct; - -impl Trait for Struct { - const ID: u32 = 5; -} - -fn main() { - println!("{}", Struct::ID); -} -``` - -Before this feature, if you wanted to make a trait that represented floating -point numbers, you’d have to write this: - -```rust -trait Float { - fn nan() -> Self; - fn infinity() -> Self; - // ... -} -``` - -This is slightly unwieldy, but more importantly, because they’re functions, -they cannot be used in constant expressions, even though they only return a -constant. Because of this, a design for `Float` would also have to include -constants as well: - -```rust,ignore -mod f32 { - const NAN: f32 = 0.0f32 / 0.0f32; - const INFINITY: f32 = 1.0f32 / 0.0f32; - - impl Float for f32 { - fn nan() -> Self { - f32::NAN - } - fn infinity() -> Self { - f32::INFINITY - } - } -} -``` - -Associated constants let you do this in a much cleaner way. This trait -definition: - -```rust -trait Float { - const NAN: Self; - const INFINITY: Self; - // ... -} -``` - -Leads to this implementation: - -```rust,ignore -mod f32 { - impl Float for f32 { - const NAN: f32 = 0.0f32 / 0.0f32; - const INFINITY: f32 = 1.0f32 / 0.0f32; - } -} -``` - -much cleaner, and more versatile. diff --git a/src/doc/edition-guide/src/rust-2018/trait-system/impl-trait-for-returning-complex-types-with-ease.md b/src/doc/edition-guide/src/rust-2018/trait-system/impl-trait-for-returning-complex-types-with-ease.md deleted file mode 100644 index 87efbfe726..0000000000 --- a/src/doc/edition-guide/src/rust-2018/trait-system/impl-trait-for-returning-complex-types-with-ease.md +++ /dev/null @@ -1,168 +0,0 @@ -# `impl Trait` for returning complex types with ease - -![Minimum Rust version: 1.26](https://img.shields.io/badge/Minimum%20Rust%20Version-1.26-brightgreen.svg) - -`impl Trait` is the new way to specify unnamed but concrete types that -implement a specific trait. There are two places you can put it: argument -position, and return position. - -```rust,ignore -trait Trait {} - -// argument position -fn foo(arg: impl Trait) { -} - -// return position -fn foo() -> impl Trait { -} -``` - -## Argument Position - -In argument position, this feature is quite simple. These two forms are -almost the same: - -```rust,ignore -trait Trait {} - -fn foo(arg: T) { -} - -fn foo(arg: impl Trait) { -} -``` - -That is, it's a slightly shorter syntax for a generic type parameter. It -means, "`arg` is an argument that takes any type that implements the `Trait` -trait." - -However, there's also an important technical difference between `T: Trait` -and `impl Trait` here. When you write the former, you can specify the type of -`T` at the call site with turbo-fish syntax as with `foo::(1)`. In the -case of `impl Trait`, if it is used anywhere in the function definition, then -you can't use turbo-fish at all. Therefore, you should be mindful that -changing both from and to `impl Trait` can constitute a breaking change for -the users of your code. - -## Return Position - -In return position, this feature is more interesting. It means "I am -returning some type that implements the `Trait` trait, but I'm not going to -tell you exactly what the type is." Before `impl Trait`, you could do this -with trait objects: - -```rust -trait Trait {} - -impl Trait for i32 {} - -fn returns_a_trait_object() -> Box { - Box::new(5) -} -``` - -However, this has some overhead: the `Box` means that there's a heap -allocation here, and this will use dynamic dispatch. See the `dyn Trait` -section for an explanation of this syntax. But we only ever return one -possible thing here, the `Box`. This means that we're paying for dynamic -dispatch, even though we don't use it! - -With `impl Trait`, the code above could be written like this: - -```rust -trait Trait {} - -impl Trait for i32 {} - -fn returns_a_trait_object() -> impl Trait { - 5 -} -``` - -Here, we have no `Box`, no trait object, and no dynamic dispatch. But we -still can obscure the `i32` return type. - -With `i32`, this isn't super useful. But there's one major place in Rust -where this is much more useful: closures. - -### `impl Trait` and closures - -> If you need to catch up on closures, check out [their chapter in the -> book](https://doc.rust-lang.org/book/second-edition/ch13-01-closures.html). - -In Rust, closures have a unique, un-writable type. They do implement the `Fn` -family of traits, however. This means that previously, the only way to return -a closure from a function was to use a trait object: - -```rust -fn returns_closure() -> Box i32> { - Box::new(|x| x + 1) -} -``` - -You couldn't write the type of the closure, only use the `Fn` trait. That means -that the trait object is necessary. However, with `impl Trait`: - -```rust -fn returns_closure() -> impl Fn(i32) -> i32 { - |x| x + 1 -} -``` - -We can now return closures by value, just like any other type! - -## More details - -The above is all you need to know to get going with `impl Trait`, but for -some more nitty-gritty details: type parameters and `impl Trait` work -slightly differently when they're in argument position versus return -position. Consider this function: - -```rust,ignore -fn foo(x: T) { -``` - -When you call it, you set the type, `T`. "you" being the caller here. This -signature says "I accept any type that implements `Trait`." ("any type" == -universal in the jargon) - -This version: - -```rust,ignore -fn foo() -> T { -``` - -is similar, but also different. You, the caller, provide the type you want, -`T`, and then the function returns it. You can see this in Rust today with -things like parse or collect: - -```rust,ignore -let x: i32 = "5".parse()?; -let x: u64 = "5".parse()?; -``` - -Here, `.parse` has this signature: - -```rust,ignore -pub fn parse(&self) -> Result::Err> where - F: FromStr, -``` - -Same general idea, though with a result type and `FromStr` has an associated -type... anyway, you can see how `F` is in the return position here. So you -have the ability to choose. - -With `impl Trait`, you're saying "hey, some type exists that implements this -trait, but I'm not gonna tell you what it is." So now, the caller can't -choose, and the function itself gets to choose. If we tried to define parse -with `Result`, one of Rust's reference-counted types: - -```rust -use std::rc::Rc; - -trait Foo {} - -impl Foo for i32 { - -} - -fn main() { - let obj: Rc = Rc::new(5); -} -``` - -This code would not work with Rust 1.0, but now works. - -> If you haven't seen the `dyn` syntax before, see [the section on -> it](dyn-trait-for-trait-objects.md). For versions that do not support it, replace `Rc` -> with `Rc`. diff --git a/src/doc/edition-guide/src/rust-2018/tyvar-behind-raw-pointer.md b/src/doc/edition-guide/src/rust-2018/tyvar-behind-raw-pointer.md new file mode 100644 index 0000000000..fce7d06a43 --- /dev/null +++ b/src/doc/edition-guide/src/rust-2018/tyvar-behind-raw-pointer.md @@ -0,0 +1,11 @@ +# Method dispatch for raw pointers to inference variables + +## Summary + +- The [`tyvar_behind_raw_pointer`][#46906] lint is now a hard error. + +[#46906]: https://github.com/rust-lang/rust/issues/46906 + +## Details + +See Rust issue [#46906] for details. \ No newline at end of file diff --git a/src/doc/edition-guide/src/rust-2021/IntoIterator-for-arrays.md b/src/doc/edition-guide/src/rust-2021/IntoIterator-for-arrays.md new file mode 100644 index 0000000000..e26e7fbfeb --- /dev/null +++ b/src/doc/edition-guide/src/rust-2021/IntoIterator-for-arrays.md @@ -0,0 +1,49 @@ +# IntoIterator for arrays + +## Summary + +## Details + +Until Rust 1.53, only *references* to arrays implement `IntoIterator`. +This means you can iterate over `&[1, 2, 3]` and `&mut [1, 2, 3]`, +but not over `[1, 2, 3]` directly. + +```rust,ignore +for &e in &[1, 2, 3] {} // Ok :) + +for e in [1, 2, 3] {} // Error :( +``` + +This has been [a long-standing issue][25], but the solution is not as simple as it seems. +Just [adding the trait implementation][20] would break existing code. +`array.into_iter()` already compiles today because that implicitly calls +`(&array).into_iter()` due to [how method call syntax works][22]. +Adding the trait implementation would change the meaning. + +Usually we categorize this type of breakage +(adding a trait implementation) 'minor' and acceptable. +But in this case there is too much code that would be broken by it. + +It has been suggested many times to "only implement `IntoIterator` for arrays in Rust 2021". +However, this is simply not possible. +You can't have a trait implementation exist in one edition and not in another, +since editions can be mixed. + +Instead, we decided to add the trait implementation in *all* editions (starting in Rust 1.53.0), +but add a small hack to avoid breakage until Rust 2021. +In Rust 2015 and 2018 code, the compiler will still resolve `array.into_iter()` +to `(&array).into_iter()` like before, as if the trait implementation does not exist. +This *only* applies to the `.into_iter()` method call syntax. +It does not affect any other syntax such as `for e in [1, 2, 3]`, `iter.zip([1, 2, 3])` or +`IntoIterator::into_iter([1, 2, 3])`. +Those will start to work in *all* editions. + +While it's a shame that this required a small hack to avoid breakage, +we're very happy with how this solution keeps the difference between +the editions to an absolute minimum. +Since the hack is only present in the older editions, +there is no added complexity in the new edition. + +[25]: https://github.com/rust-lang/rust/issues/25725 +[20]: https://github.com/rust-lang/rust/pull/65819 +[22]: https://doc.rust-lang.org/book/ch05-03-method-syntax.html#wheres-the---operator \ No newline at end of file diff --git a/src/doc/edition-guide/src/rust-2021/default-cargo-resolver.md b/src/doc/edition-guide/src/rust-2021/default-cargo-resolver.md new file mode 100644 index 0000000000..a798b873fd --- /dev/null +++ b/src/doc/edition-guide/src/rust-2021/default-cargo-resolver.md @@ -0,0 +1,18 @@ +# Default Cargo feature resolver + +## Summary + +## Details + +Since Rust 1.51.0, Cargo has opt-in support for a [new feature resolver][4] +which can be activated with `resolver = "2"` in `Cargo.toml`. + +Starting in Rust 2021, this will be the default. +That is, writing `edition = "2021"` in `Cargo.toml` will imply `resolver = "2"`. + +The new feature resolver no longer merges all requested features for +crates that are depended on in multiple ways. +See [the announcement of Rust 1.51][5] for details. + +[4]: https://doc.rust-lang.org/cargo/reference/resolver.html#feature-resolver-version-2 +[5]: https://blog.rust-lang.org/2021/03/25/Rust-1.51.0.html#cargos-new-feature-resolver \ No newline at end of file diff --git a/src/doc/edition-guide/src/rust-2021/disjoint-capture-in-closures.md b/src/doc/edition-guide/src/rust-2021/disjoint-capture-in-closures.md new file mode 100644 index 0000000000..4e47fd452d --- /dev/null +++ b/src/doc/edition-guide/src/rust-2021/disjoint-capture-in-closures.md @@ -0,0 +1,37 @@ +# Disjoint capture in closures + +## Summary + +## Details + +[Closures](https://doc.rust-lang.org/book/ch13-01-closures.html) +automatically capture anything that you refer to from within their body. +For example, `|| a + 1` automatically captures a reference to `a` from the surrounding context. + +Currently, this applies to whole structs, even when only using one field. +For example, `|| a.x + 1` captures a reference to `a` and not just `a.x`. +In some situations, this is a problem. +When a field of the struct is already borrowed (mutably) or moved out of, +the other fields can no longer be used in a closure, +since that would capture the whole struct, which is no longer available. + +```rust,ignore +let a = SomeStruct::new(); + +drop(a.x); // Move out of one field of the struct + +println!("{}", a.y); // Ok: Still use another field of the struct + +let c = || println!("{}", a.y); // Error: Tries to capture all of `a` +c(); +``` + +Starting in Rust 2021, closures will only capture the fields that they use. +So, the above example will compile fine in Rust 2021. + +This new behavior is only activated in the new edition, +since it can change the order in which fields are dropped. +As for all edition changes, an automatic migration is available, +which will update your closures for which this matters. +It can insert `let _ = &a;` inside the closure to force the entire +struct to be captured as before. diff --git a/src/doc/edition-guide/src/rust-2021/index.md b/src/doc/edition-guide/src/rust-2021/index.md new file mode 100644 index 0000000000..45f09abe38 --- /dev/null +++ b/src/doc/edition-guide/src/rust-2021/index.md @@ -0,0 +1,22 @@ +# Rust 2021 + +🚧 The 2021 Edition has not yet been released and hence this section is still "under construction". +You can [read more about our plans in this blog post](https://blog.rust-lang.org/2021/05/11/edition-2021.html). + +| Info | | +| --- | --- | +| RFC | [#3085](https://github.com/rust-lang/rfcs/pull/3085) | +| Release version | 1.56.0 (anticipated) | + +The Rust 2021 Edition is currently slated for release in Rust 1.56.0. +Rust 1.56.0 will then be in beta for six weeks, +after which it is released as stable on October 21st. + +However, note that Rust is a project run by volunteers. +We prioritize the personal well-being of everyone working on Rust +over any deadlines and expectations we might have set. +This could mean delaying the edition a version if necessary, +or dropping a feature that turns out to be too difficult or stressful to finish in time. + +That said, we are on schedule and many of the difficult problems are already tackled, +thanks to all the people contributing to Rust 2021! 💛 diff --git a/src/doc/edition-guide/src/rust-2021/or-patterns-macro-rules.md b/src/doc/edition-guide/src/rust-2021/or-patterns-macro-rules.md new file mode 100644 index 0000000000..451b0e5f64 --- /dev/null +++ b/src/doc/edition-guide/src/rust-2021/or-patterns-macro-rules.md @@ -0,0 +1,28 @@ +# Or patterns in macro-rules + +## Summary + +## Details + +Starting in Rust 1.53.0, [patterns](https://doc.rust-lang.org/stable/reference/patterns.html) +are extended to support `|` nested anywhere in the pattern. +This enables you to write `Some(1 | 2)` instead of `Some(1) | Some(2)`. +Since this was simply not allowed before, this is not a breaking change. + +However, this change also affects [`macro_rules` macros](https://doc.rust-lang.org/stable/reference/macros-by-example.html). +Such macros can accept patterns using the `:pat` fragment specifier. +Currently, `:pat` does *not* match `|`, since before Rust 1.53, +not all patterns (at all nested levels) could contain a `|`. +Macros that accept patterns like `A | B`, +such as [`matches!()`](https://doc.rust-lang.org/1.51.0/std/macro.matches.html) +use something like `$($_:pat)|+`. +Because we don't want to break any existing macros, +we did *not* change the meaning of `:pat` in Rust 1.53.0 to include `|`. + +Instead, we will make that change as part of Rust 2021. +In the new edition, the `:pat` fragment specifier *will* match `A | B`. + +Since there are times that one still wishes to match a single pattern +variant without `|`, the fragment specified `:pat_param` has been added +to retain the older behavior. +The name refers to its main use case: a pattern in a closure parameter. diff --git a/src/doc/edition-guide/src/rust-2021/panic-macro-consistency.md b/src/doc/edition-guide/src/rust-2021/panic-macro-consistency.md new file mode 100644 index 0000000000..cf4265bcc5 --- /dev/null +++ b/src/doc/edition-guide/src/rust-2021/panic-macro-consistency.md @@ -0,0 +1,44 @@ +# Panic macro consistency + +## Summary + +## Details + + +The `panic!()` macro is one of Rust's most well known macros. +However, it has [some subtle surprises](https://github.com/rust-lang/rfcs/blob/master/text/3007-panic-plan.md) +that we can't just change due to backwards compatibility. + +```rust,ignore +panic!("{}", 1); // Ok, panics with the message "1" +panic!("{}"); // Ok, panics with the message "{}" +``` + +The `panic!()` macro only uses string formatting when it's invoked with more than one argument. +When invoked with a single argument, it doesn't even look at that argument. + +```rust,ignore +let a = "{"; +println!(a); // Error: First argument must be a format string literal +panic!(a); // Ok: The panic macro doesn't care +``` + +(It even accepts non-strings such as `panic!(123)`, which is uncommon and rarely useful.) + +This will especially be a problem once +[implicit format arguments](https://rust-lang.github.io/rfcs/2795-format-args-implicit-identifiers.html) +are stabilized. +That feature will make `println!("hello {name}")` a short-hand for `println!("hello {}", name)`. +However, `panic!("hello {name}")` would not work as expected, +since `panic!()` doesn't process a single argument as format string. + +To avoid that confusing situation, Rust 2021 features a more consistent `panic!()` macro. +The new `panic!()` macro will no longer accept arbitrary expressions as the only argument. +It will, just like `println!()`, always process the first argument as format string. +Since `panic!()` will no longer accept arbitrary payloads, +[`panic_any()`](https://doc.rust-lang.org/stable/std/panic/fn.panic_any.html) +will be the only way to panic with something other than a formatted string. + +In addition, `core::panic!()` and `std::panic!()` will be identical in Rust 2021. +Currently, there are some historical differences between those two, +which can be noticable when switching `#![no_std]` on or off. \ No newline at end of file diff --git a/src/doc/edition-guide/src/rust-2021/prelude.md b/src/doc/edition-guide/src/rust-2021/prelude.md new file mode 100644 index 0000000000..2d182066f0 --- /dev/null +++ b/src/doc/edition-guide/src/rust-2021/prelude.md @@ -0,0 +1,32 @@ +# Additions to the prelude + +## Summary + +## Details + +The [prelude of the standard library](https://doc.rust-lang.org/stable/std/prelude/index.html) +is the module containing everything that is automatically imported in every module. +It contains commonly used items such as `Option`, `Vec`, `drop`, and `Clone`. + +The Rust compiler prioritizes any manually imported items over those +from the prelude, to make sure additions to the prelude will not break any existing code. +For example, if you have a crate or module called `example` containing a `pub struct Option;`, +then `use example::*;` will make `Option` unambiguously refer to the one from `example`; +not the one from the standard library. + +However, adding a *trait* to the prelude can break existing code in a subtle way. +A call to `x.try_into()` using a `MyTryInto` trait might become ambiguous and +fail to compile if `std`'s `TryInto` is also imported, +since it provides a method with the same name. +This is the reason we haven't added `TryInto` to the prelude yet, +since there is a lot of code that would break this way. + +As a solution, Rust 2021 will use a new prelude. +It's identical to the current one, except for three new additions: + +- [`std::convert::TryInto`](https://doc.rust-lang.org/stable/std/convert/trait.TryInto.html) +- [`std::convert::TryFrom`](https://doc.rust-lang.org/stable/std/convert/trait.TryFrom.html) +- [`std::iter::FromIterator`](https://doc.rust-lang.org/stable/std/iter/trait.FromIterator.html) + +The library team still needs to formally approve these, which will likely happen soon. + diff --git a/src/doc/edition-guide/src/rust-2021/reserving-syntax.md b/src/doc/edition-guide/src/rust-2021/reserving-syntax.md new file mode 100644 index 0000000000..315954be53 --- /dev/null +++ b/src/doc/edition-guide/src/rust-2021/reserving-syntax.md @@ -0,0 +1,40 @@ +# Reserving syntax + +## Summary + +## Details + +To make space for some new syntax in the future, +we've decided to reserve syntax for prefixed identifiers and literals: +`prefix#identifier`, `prefix"string"`, `prefix'c'`, and `prefix#123`, +where `prefix` can be any identifier. +(Except those that already have a meaning, such as `b'…'` and `r"…"`.) + +This is a breaking change, since macros can currently accept `hello"world"`, +which they will see as two separate tokens: `hello` and `"world"`. +The (automatic) fix is simple though. Just insert a space: `hello "world"`. + + + +Other than turning these into a tokenization error, +[the RFC][10] does not attach a meaning to any prefix yet. +Assigning meaning to specific prefixes is left to future proposals, +which will—thanks to reserving these prefixes now—not be breaking changes. + +These are some new prefixes you might see in the future: + +- `f""` as a short-hand for a format string. + For example, `f"hello {name}"` as a short-hand for the equivalent `format_args!()` invocation. + +- `c""` or `z""` for null-terminated C strings. + +- `k#keyword` to allow writing keywords that don't exist yet in the current edition. + For example, while `async` is not a keyword in edition 2015, + this prefix would've allowed us to accept `k#async` in edition 2015 + without having to wait for edition 2018 to reserve `async` as a keyword. + +[10]: https://github.com/rust-lang/rfcs/pull/3101 \ No newline at end of file diff --git a/src/doc/edition-guide/src/rust-2021/warnings-promoted-to-error.md b/src/doc/edition-guide/src/rust-2021/warnings-promoted-to-error.md new file mode 100644 index 0000000000..5c20246d18 --- /dev/null +++ b/src/doc/edition-guide/src/rust-2021/warnings-promoted-to-error.md @@ -0,0 +1,17 @@ +# Warnings promoted to errors + +## Summary + +## Details + +Two existing lints are becoming hard errors in Rust 2021. +These lints will remain warnings in older editions. + +* `bare-trait-objects`: + The use of the `dyn` keyword to identify [trait objects](https://doc.rust-lang.org/book/ch17-02-trait-objects.html) + will be mandatory in Rust 2021. + +* `ellipsis-inclusive-range-patterns`: + The [deprecated `...` syntax](https://doc.rust-lang.org/stable/reference/patterns.html#range-patterns) + for inclusive range patterns is no longer accepted in Rust 2021. + It has been superseded by `..=`, which is consistent with expressions. \ No newline at end of file diff --git a/src/doc/edition-guide/src/rust-next/alloc.md b/src/doc/edition-guide/src/rust-next/alloc.md deleted file mode 100644 index 036328d847..0000000000 --- a/src/doc/edition-guide/src/rust-next/alloc.md +++ /dev/null @@ -1,28 +0,0 @@ -# The alloc crate - -Initially added: ![Minimum Rust version: 1.36](https://img.shields.io/badge/Minimum%20Rust%20Version-1.36-brightgreen.svg) - -Before 1.36.0, the standard library consisted of the crates `std`, `core`, and `proc_macro`. -The `core` crate provided core functionality such as `Iterator` and `Copy` -and could be used in `#![no_std]` environments since it did not impose any requirements. -Meanwhile, the `std` crate provided types like `Box` and OS functionality -but required a global allocator and other OS capabilities in return. - -Starting with Rust 1.36.0, the parts of `std` that depend on a global allocator, e.g. `Vec`, -are now available in the `alloc` crate. The `std` crate then re-exports these parts. -While `#![no_std]` *binaries* using `alloc` still require nightly Rust, -`#![no_std]` *library* crates can use the `alloc` crate in stable Rust. -Meanwhile, normal binaries, without `#![no_std]`, can depend on such library crates. -We hope this will facilitate the development of a `#![no_std]` compatible ecosystem of libraries -prior to stabilizing support for `#![no_std]` binaries using `alloc`. - -If you are the maintainer of a library that only relies on some allocation primitives to function, -consider making your library `#[no_std]` compatible by using the following at the top of your `lib.rs` file: - -```rust,ignore -#![no_std] - -extern crate alloc; - -use alloc::vec::Vec; -``` diff --git a/src/doc/edition-guide/src/rust-next/alternative-cargo-registries.md b/src/doc/edition-guide/src/rust-next/alternative-cargo-registries.md deleted file mode 100644 index 84e70672bc..0000000000 --- a/src/doc/edition-guide/src/rust-next/alternative-cargo-registries.md +++ /dev/null @@ -1,26 +0,0 @@ -# Alternative Cargo registries - -Initially added: ![Minimum Rust version: 1.34](https://img.shields.io/badge/Minimum%20Rust%20Version-1.34-brightgreen.svg) - -For various reasons, you may not want to publish code to crates.io, but you -may want to share it with others. For example, maybe your company writes Rust -code that's not open source, but you'd still like to use these internal -packages. - -Cargo supports alternative registries by settings in `.cargo/config`: - -```toml -[registries] -my-registry = { index = "https://my-intranet:7878/git/index" } -``` - -When you want to depend on a package from another registry, you add that -in to your `Cargo.toml`: - -```toml -[dependencies] -other-crate = { version = "1.0", registry = "my-registry" } -``` - -To learn more, check out the [registries section of the Cargo -book](https://doc.rust-lang.org/nightly/cargo/reference/registries.html). diff --git a/src/doc/edition-guide/src/rust-next/cargo-vendor.md b/src/doc/edition-guide/src/rust-next/cargo-vendor.md deleted file mode 100644 index 7ef779b289..0000000000 --- a/src/doc/edition-guide/src/rust-next/cargo-vendor.md +++ /dev/null @@ -1,9 +0,0 @@ -# cargo vendor - -Initially added: ![Minimum Rust version: 1.37](https://img.shields.io/badge/Minimum%20Rust%20Version-1.37-brightgreen.svg) - -After being available [as a separate crate][vendor-crate] for years, the `cargo vendor` command is now integrated directly into Cargo. The command fetches all your project's dependencies unpacking them into the `vendor/` directory, and shows the configuration snippet required to use the vendored code during builds. - -There are multiple cases where `cargo vendor` is already used in production: the Rust compiler `rustc` uses it to ship all its dependencies in release tarballs, and projects with monorepos use it to commit the dependencies' code in source control. - -[vendor-crate]: https://crates.io/crates/cargo-vendor diff --git a/src/doc/edition-guide/src/rust-next/const-fn.md b/src/doc/edition-guide/src/rust-next/const-fn.md deleted file mode 100644 index 13bb50bc92..0000000000 --- a/src/doc/edition-guide/src/rust-next/const-fn.md +++ /dev/null @@ -1,241 +0,0 @@ -# `const fn` - -Initially added: ![Minimum Rust version: 1.31](https://img.shields.io/badge/Minimum%20Rust%20Version-1.31-brightgreen.svg) - -Expanded in many releases, see each aspect below for more details. - -A `const fn` allows you to execute code in a "const context." For example: - -```rust -const fn five() -> i32 { - 5 -} - -const FIVE: i32 = five(); -``` - -You cannot execute arbitrary code; the reasons why boil down to "you can -destroy the type system." The details are a bit too much to put here, but the -core idea is that `const fn` started off allowing the absolutely minimal -subset of the language, and has slowly added more abilities over time. -Therefore, while you can create a `const fn` in Rust 1.31, you cannot do much -with it. This is why we didn't add `const fn` to the Rust 2018 section; it -truly didn't become useful until after the release of the 2018 edition. This -means that if you read this document top to bottom, the earlier versions may -describe restrictions that are relaxed in later versions. - -Additionally, this has allowed more and more of the standard library to be -made `const`, we won't put all of those changes here, but you should know -that it is becoming more `const` over time. - -## Arithmetic and comparison operators on integers - -![Minimum Rust version: 1.31](https://img.shields.io/badge/Minimum%20Rust%20Version-1.31-brightgreen.svg) - -You can do arithmetic on integer literals: - -```rust -const fn foo() -> i32 { - 5 + 6 -} -``` - -## Many boolean operators - -![Minimum Rust version: 1.31](https://img.shields.io/badge/Minimum%20Rust%20Version-1.31-brightgreen.svg) - -You can use boolean operators other than `&&` and `||`, because they short-circut evaluation: - -```rust -const fn mask(val: u8) -> u8 { - let mask = 0x0f; - - mask & val -} -``` - -## Constructing arrays, structs, enums, and tuples - -![Minimum Rust version: 1.31](https://img.shields.io/badge/Minimum%20Rust%20Version-1.31-brightgreen.svg) - -You can create arrays, structs, enums, and tuples: - -```rust -struct Point { - x: i32, - y: i32, -} - -enum Error { - Incorrect, - FileNotFound, -} - -const fn foo() { - let array = [1, 2, 3]; - - let point = Point { - x: 5, - y: 10, - }; - - let error = Error::FileNotFound; - - let tuple = (1, 2, 3); -} -``` - -## Calls to other const fns - -![Minimum Rust version: 1.31](https://img.shields.io/badge/Minimum%20Rust%20Version-1.31-brightgreen.svg) - -You can call `const fn` from a `const fn`: - - -```rust -const fn foo() -> i32 { - 5 -} - -const fn bar() -> i32 { - foo() -} -``` - -## Index expressions on arrays and slices - -![Minimum Rust version: 1.31](https://img.shields.io/badge/Minimum%20Rust%20Version-1.31-brightgreen.svg) - -You can index into an array or slice: - -```rust -const fn foo() -> i32 { - let array = [1, 2, 3]; - - array[1] -} -``` - -## Field accesses on structs and tuples - -![Minimum Rust version: 1.31](https://img.shields.io/badge/Minimum%20Rust%20Version-1.31-brightgreen.svg) - -You can access parts of a struct or tuple: - -```rust -struct Point { - x: i32, - y: i32, -} - -const fn foo() { - let point = Point { - x: 5, - y: 10, - }; - - let tuple = (1, 2, 3); - - point.x; - tuple.0; -} -``` - -## Reading from constants - -![Minimum Rust version: 1.31](https://img.shields.io/badge/Minimum%20Rust%20Version-1.31-brightgreen.svg) - -You can read from a constant: - -```rust -const FOO: i32 = 5; - -const fn foo() -> i32 { - FOO -} -``` - -Note that this is *only* `const`, not `static`. - -## & and * of references - -![Minimum Rust version: 1.31](https://img.shields.io/badge/Minimum%20Rust%20Version-1.31-brightgreen.svg) - -You can create and de-reference references: - -```rust -const fn foo(r: &i32) { - *r; - - &5; -} -``` - -## Casts, except for raw pointer to integer casts - -![Minimum Rust version: 1.31](https://img.shields.io/badge/Minimum%20Rust%20Version-1.31-brightgreen.svg) - -You may cast things, except for raw pointers may not be casted to an integer: - -```rust -const fn foo() { - let x: usize = 5; - - x as i32; -} -``` - -## Irrefutable destructuring patterns - -![Minimum Rust version: 1.33](https://img.shields.io/badge/Minimum%20Rust%20Version-1.33-brightgreen.svg) - -You can use irrefutable patterns that destructure values. For example: - -```rust -const fn foo((x, y): (u8, u8)) { - // ... -} -``` - -Here, `foo` destructures the tuple into `x` and `y`. `if let` is another -place that uses irrefutable patterns. - -## `let` bindings - -![Minimum Rust version: 1.33](https://img.shields.io/badge/Minimum%20Rust%20Version-1.33-brightgreen.svg) - -You can use both mutable and immutable `let` bindings: - -```rust -const fn foo() { - let x = 5; - let mut y = 10; -} -``` - -## Assignment - -![Minimum Rust version: 1.33](https://img.shields.io/badge/Minimum%20Rust%20Version-1.33-brightgreen.svg) - -You can use assignment and assignment operators: - -```rust -const fn foo() { - let mut x = 5; - x = 10; -} -``` - -## Calling `unsafe fn` - -![Minimum Rust version: 1.33](https://img.shields.io/badge/Minimum%20Rust%20Version-1.33-brightgreen.svg) - -You can call an `unsafe fn` inside a `const fn`: - -```rust -const unsafe fn foo() -> i32 { 5 } - -const fn bar() -> i32 { - unsafe { foo() } -} -``` diff --git a/src/doc/edition-guide/src/rust-next/dbg-macro.md b/src/doc/edition-guide/src/rust-next/dbg-macro.md deleted file mode 100644 index a2ac7ad8db..0000000000 --- a/src/doc/edition-guide/src/rust-next/dbg-macro.md +++ /dev/null @@ -1,104 +0,0 @@ -# The dbg! macro - -![Minimum Rust version: 1.32](https://img.shields.io/badge/Minimum%20Rust%20Version-1.32-brightgreen.svg) - -The `dbg!` macro provides a nicer experience for debugging than `println!`: - -```rust -fn main() { - let x = 5; - - dbg!(x); -} -``` - -If you run this program, you'll see: - -```text -[src/main.rs:4] x = 5 -``` - -You get the file and line number of where this was invoked, as well as the -name and value. Additionally, `println!` prints to the standard output, so you -really should be using `eprintln!` to print to standard error. `dbg!` does the -right thing and goes to stderr. - -It even works in more complex circumstances. Consider this factorial example: - -```rust -fn factorial(n: u32) -> u32 { - if n <= 1 { - n - } else { - n * factorial(n - 1) - } -} -``` - -If we wanted to debug this, we might write it like this with `eprintln!`: - -```rust -fn factorial(n: u32) -> u32 { - eprintln!("n: {}", n); - - if n <= 1 { - eprintln!("n <= 1"); - - n - } else { - let n = n * factorial(n - 1); - - eprintln!("n: {}", n); - - n - } -} -``` - -We want to log `n` on each iteration, as well as have some kind of context -for each of the branches. We see this output for `factorial(4)`: - -```text -n: 4 -n: 3 -n: 2 -n: 1 -n <= 1 -n: 2 -n: 6 -n: 24 -``` - -This is servicable, but not particularly great. Maybe we could work on how we -print out the context to make it more clear, but now we're not debugging our -code, we're figuring out how to make our debugging code better. - -Consider this version using `dbg!`: - -```rust -fn factorial(n: u32) -> u32 { - if dbg!(n <= 1) { - dbg!(1) - } else { - dbg!(n * factorial(n - 1)) - } -} -``` - -We simply wrap each of the various expressions we want to print with the macro. We get this output instead: - -```text -[src/main.rs:3] n <= 1 = false -[src/main.rs:3] n <= 1 = false -[src/main.rs:3] n <= 1 = false -[src/main.rs:3] n <= 1 = true -[src/main.rs:4] 1 = 1 -[src/main.rs:5] n * factorial(n - 1) = 2 -[src/main.rs:5] n * factorial(n - 1) = 6 -[src/main.rs:5] n * factorial(n - 1) = 24 -[src/main.rs:11] factorial(4) = 24 -``` - -Because the `dbg!` macro returns the value of what it's debugging, instead of -`eprintln!` which returns `()`, we need to make no changes to the structure of -our code. Additionally, we have vastly more useful output. diff --git a/src/doc/edition-guide/src/rust-next/edition-changes.md b/src/doc/edition-guide/src/rust-next/edition-changes.md deleted file mode 100644 index 5847529267..0000000000 --- a/src/doc/edition-guide/src/rust-next/edition-changes.md +++ /dev/null @@ -1,3 +0,0 @@ -# Next-Specific Changes - -There have been no specific changes accepted for the next edition yet. \ No newline at end of file diff --git a/src/doc/edition-guide/src/rust-next/future.md b/src/doc/edition-guide/src/rust-next/future.md deleted file mode 100644 index 4fe158c3f2..0000000000 --- a/src/doc/edition-guide/src/rust-next/future.md +++ /dev/null @@ -1,10 +0,0 @@ -# The Future trait - -Initially added: ![Minimum Rust version: 1.36](https://img.shields.io/badge/Minimum%20Rust%20Version-1.36-brightgreen.svg) - -In Rust 1.36.0 the long awaited [`Future`] trait has been stabilized! - -TODO: this will probably be folded into a larger async section once we're -closer to the next edition. - -[`Future`]: https://doc.rust-lang.org/std/future/trait.Future.html diff --git a/src/doc/edition-guide/src/rust-next/index.md b/src/doc/edition-guide/src/rust-next/index.md deleted file mode 100644 index 195118dd18..0000000000 --- a/src/doc/edition-guide/src/rust-next/index.md +++ /dev/null @@ -1,7 +0,0 @@ -# The Next Edition - -We have not decided if and when the next edition will ship; there is talk of -a 2021 edition to keep up the three-year schedule, but that has not been -formally decided. - -Until we do, this section keeps track of changes that landed after Rust 2018. \ No newline at end of file diff --git a/src/doc/edition-guide/src/rust-next/literal-macro-matcher.md b/src/doc/edition-guide/src/rust-next/literal-macro-matcher.md deleted file mode 100644 index 1b84aade65..0000000000 --- a/src/doc/edition-guide/src/rust-next/literal-macro-matcher.md +++ /dev/null @@ -1,18 +0,0 @@ -# literal macro matcher - -![Minimum Rust version: 1.32](https://img.shields.io/badge/Minimum%20Rust%20Version-1.32-brightgreen.svg) - -A new `literal` matcher was added for macros: - -```rust -macro_rules! m { - ($lt:literal) => {}; -} - -fn main() { - m!("some string literal"); -} -``` - -`literal` matches against literals of any type; string literals, numeric -literals, `char` literals. diff --git a/src/doc/edition-guide/src/rust-next/maybe-uninit.md b/src/doc/edition-guide/src/rust-next/maybe-uninit.md deleted file mode 100644 index 6c1d4ca05b..0000000000 --- a/src/doc/edition-guide/src/rust-next/maybe-uninit.md +++ /dev/null @@ -1,28 +0,0 @@ -# MaybeUninit - -Initially added: ![Minimum Rust version: 1.36](https://img.shields.io/badge/Minimum%20Rust%20Version-1.36-brightgreen.svg) - -In previous releases of Rust, the [`mem::uninitialized`] function has allowed -you to bypass Rust's initialization checks by pretending that you've -initialized a value at type `T` without doing anything. One of the main uses -of this function has been to lazily allocate arrays. - -However, [`mem::uninitialized`] is an incredibly dangerous operation that -essentially cannot be used correctly as the Rust compiler assumes that values -are properly initialized. For example, calling `mem::uninitialized::()` -causes *instantaneous __undefined behavior__* as, from Rust's point of view, -the uninitialized bits are neither `0` (for `false`) nor `1` (for `true`) - -the only two allowed bit patterns for `bool`. - -To remedy this situation, in Rust 1.36.0, the type [`MaybeUninit`] has -been stabilized. The Rust compiler will understand that it should not assume -that a [`MaybeUninit`] is a properly initialized `T`. Therefore, you can -do gradual initialization more safely and eventually use `.assume_init()` -once you are certain that `maybe_t: MaybeUninit` contains an initialized -`T`. - -As [`MaybeUninit`] is the safer alternative, starting with Rust 1.39, the -function [`mem::uninitialized`] will be deprecated. - -[`MaybeUninit`]: https://doc.rust-lang.org/std/mem/union.MaybeUninit.html -[`mem::uninitialized`]: https://doc.rust-lang.org/std/mem/fn.uninitialized.html diff --git a/src/doc/edition-guide/src/rust-next/no-jemalloc.md b/src/doc/edition-guide/src/rust-next/no-jemalloc.md deleted file mode 100644 index cd0241088d..0000000000 --- a/src/doc/edition-guide/src/rust-next/no-jemalloc.md +++ /dev/null @@ -1,43 +0,0 @@ -# No jemalloc by default - -![Minimum Rust version: 1.32](https://img.shields.io/badge/Minimum%20Rust%20Version-1.32-brightgreen.svg) - -Long, long ago, Rust had a large, Erlang-like runtime. We chose to use -jemalloc instead of the system allocator, because it often improved -performance over the default system one. Over time, we shed more and more of -this runtime, and eventually almost all of it was removed, but jemalloc was -not. We didn't have a way to choose a custom allocator, and so we couldn't -really remove it without causing a regression for people who do need -jemalloc. - -Also, saying that jemalloc was always the default is a bit UNIX-centric, as -it was only the default on some platforms. Notably, the MSVC target on -Windows has shipped the system allocator for a long time. - -While jemalloc usually has great performance, that's not always the case. -Additionally, it adds about 300kb to every Rust binary. We've also had a host -of other issues with jemalloc in the past. It has also felt a little strange -that a systems language does not default to the system's allocator. - -For all of these reasons, once Rust 1.28 shipped a way to choose a global -allocator, we started making plans to switch the default to the system -allocator, and allow you to use jemalloc via a crate. In Rust 1.32, we've -finally finished this work, and by default, you will get the system allocator -for your programs. - -If you'd like to continue to use jemalloc, use the jemallocator crate. In -your Cargo.toml: - -```toml -jemallocator = "0.1.8" -``` - -And in your crate root: - -```rust,ignore -#[global_allocator] -static ALLOC: jemallocator::Jemalloc = jemallocator::Jemalloc; -``` - -That's it! If you don't need jemalloc, it's not forced upon you, and if you -do need it, it's a few lines of code away. diff --git a/src/doc/edition-guide/src/rust-next/no-more-fnbox.md b/src/doc/edition-guide/src/rust-next/no-more-fnbox.md deleted file mode 100644 index c5e18f44dd..0000000000 --- a/src/doc/edition-guide/src/rust-next/no-more-fnbox.md +++ /dev/null @@ -1,38 +0,0 @@ -# No more FnBox - -![Minimum Rust version: 1.35](https://img.shields.io/badge/Minimum%20Rust%20Version-1.35-brightgreen.svg) - -The book used to have this code in Chapter 20, section 2: - -```rust -trait FnBox { - fn call_box(self: Box); -} - -impl FnBox for F { - fn call_box(self: Box) { - (*self)() - } -} - -type Job = Box; -``` - -Here, we define a new trait called `FnBox`, and then implement it for all -`FnOnce` closures. All the implementation does is call the closure. These -sorts of hacks were needed because a `Box` didn't implement -`FnOnce`. This was true for all three posibilities: - -* `Box` and `Fn` -* `Box` and `FnMut` -* `Box` and `FnOnce` - -However, as of Rust 1.35, these traits are implemented for these types, -and so the `FnBox` trick is no longer required. In the latest version of -the book, the `Job` type looks like this: - -```rust -type Job = Box; -``` - -No need for all that other code. diff --git a/src/doc/edition-guide/src/rust-next/pin.md b/src/doc/edition-guide/src/rust-next/pin.md deleted file mode 100644 index a52f70cf0a..0000000000 --- a/src/doc/edition-guide/src/rust-next/pin.md +++ /dev/null @@ -1,21 +0,0 @@ -# Pinning - -![Minimum Rust version: 1.33](https://img.shields.io/badge/Minimum%20Rust%20Version-1.33-brightgreen.svg) - -Rust 1.33 introduced a new concept, implemented as two types: - -* [`Pin