ArmPkg/CompilerIntrinsicsLib: provide atomics intrinsics

Gary reports the GCC 10 will emit calls to atomics intrinsics routines
unless -mno-outline-atomics is specified. This means GCC-10 introduces
new intrinsics, and even though it would be possible to work around this
by specifying the command line option, this would require a new GCC10
toolchain profile to be created, which we prefer to avoid.

So instead, add the new intrinsics to our library so they are provided
when necessary.

Signed-off-by: Ard Biesheuvel <ard.biesheuvel@arm.com>
Tested-by: Gary Lin <glin@suse.com>
Acked-by: Laszlo Ersek <lersek@redhat.com>
Reviewed-by: Leif Lindholm <leif@nuviainc.com>
Reviewed-by: Philippe Mathieu-Daude <philmd@redhat.com>

diff --git a/ArmPkg/Library/CompilerIntrinsicsLib/AArch64/Atomics.S b/ArmPkg/Library/CompilerIntrinsicsLib/AArch64/Atomics.S
new file mode 100644 (file)
index 0000000..3792020
--- /dev/null
+++ b/ArmPkg/Library/CompilerIntrinsicsLib/AArch64/Atomics.S
@@ -0,0 +1,142 @@
+#------------------------------------------------------------------------------\r
+#\r
+# Copyright (c) 2020, Arm, Limited. All rights reserved.<BR>\r
+#\r
+# SPDX-License-Identifier: BSD-2-Clause-Patent\r
+#\r
+#------------------------------------------------------------------------------\r
+\r
+        /*\r
+         * Provide the GCC intrinsics that are required when using GCC 9 or\r
+         * later with the -moutline-atomics options (which became the default\r
+         * in GCC 10)\r
+         */\r
+        .arch armv8-a\r
+\r
+        .macro          reg_alias, pfx, sz\r
+        r0_\sz          .req    \pfx\()0\r
+        r1_\sz          .req    \pfx\()1\r
+        tmp0_\sz        .req    \pfx\()16\r
+        tmp1_\sz        .req    \pfx\()17\r
+        .endm\r
+\r
+        /*\r
+         * Define register aliases of the right type for each size\r
+         * (xN for 8 bytes, wN for everything smaller)\r
+         */\r
+        reg_alias       w, 1\r
+        reg_alias       w, 2\r
+        reg_alias       w, 4\r
+        reg_alias       x, 8\r
+\r
+        .macro          fn_start, name:req\r
+        .section        .text.\name\r
+        .globl          \name\r
+        .type           \name, %function\r
+\name\():\r
+        .endm\r
+\r
+        .macro          fn_end, name:req\r
+        .size           \name, . - \name\r
+        .endm\r
+\r
+        /*\r
+         * Emit an atomic helper for \model with operands of size \sz, using\r
+         * the operation specified by \insn (which is the LSE name), and which\r
+         * can be implemented using the generic load-locked/store-conditional\r
+         * (LL/SC) sequence below, using the arithmetic operation given by\r
+         * \opc.\r
+         */\r
+        .macro          emit_ld_sz, sz:req, insn:req, opc:req, model:req, s, a, l\r
+        fn_start        __aarch64_\insn\()\sz\()\model\r
+        mov             tmp0_\sz, r0_\sz\r
+0:      ld\a\()xr\s     r0_\sz, [x1]\r
+        .ifnc           \insn, swp\r
+        \opc            tmp1_\sz, r0_\sz, tmp0_\sz\r
+        st\l\()xr\s     w15, tmp1_\sz, [x1]\r
+        .else\r
+        st\l\()xr\s     w15, tmp0_\sz, [x1]\r
+        .endif\r
+        cbnz            w15, 0b\r
+        ret\r
+        fn_end          __aarch64_\insn\()\sz\()\model\r
+        .endm\r
+\r
+        /*\r
+         * Emit atomic helpers for \model for operand sizes in the\r
+         * set {1, 2, 4, 8}, for the instruction pattern given by\r
+         * \insn. (This is the LSE name, but this implementation uses\r
+         * the generic LL/SC sequence using \opc as the arithmetic\r
+         * operation on the target.)\r
+         */\r
+        .macro          emit_ld, insn:req, opc:req, model:req, a, l\r
+        emit_ld_sz      1, \insn, \opc, \model, b, \a, \l\r
+        emit_ld_sz      2, \insn, \opc, \model, h, \a, \l\r
+        emit_ld_sz      4, \insn, \opc, \model,  , \a, \l\r
+        emit_ld_sz      8, \insn, \opc, \model,  , \a, \l\r
+        .endm\r
+\r
+        /*\r
+         * Emit the compare and swap helper for \model and size \sz\r
+         * using LL/SC instructions.\r
+         */\r
+        .macro          emit_cas_sz, sz:req, model:req, uxt:req, s, a, l\r
+        fn_start        __aarch64_cas\sz\()\model\r
+        \uxt            tmp0_\sz, r0_\sz\r
+0:      ld\a\()xr\s     r0_\sz, [x2]\r
+        cmp             r0_\sz, tmp0_\sz\r
+        bne             1f\r
+        st\l\()xr\s     w15, r1_\sz, [x2]\r
+        cbnz            w15, 0b\r
+1:      ret\r
+        fn_end          __aarch64_cas\sz\()\model\r
+        .endm\r
+\r
+        /*\r
+         * Emit compare-and-swap helpers for \model for operand sizes in the\r
+         * set {1, 2, 4, 8, 16}.\r
+         */\r
+        .macro          emit_cas, model:req, a, l\r
+        emit_cas_sz     1, \model, uxtb, b, \a, \l\r
+        emit_cas_sz     2, \model, uxth, h, \a, \l\r
+        emit_cas_sz     4, \model, mov ,  , \a, \l\r
+        emit_cas_sz     8, \model, mov ,  , \a, \l\r
+\r
+        /*\r
+         * We cannot use the parameterized sequence for 16 byte CAS, so we\r
+         * need to define it explicitly.\r
+         */\r
+        fn_start        __aarch64_cas16\model\r
+        mov             x16, x0\r
+        mov             x17, x1\r
+0:      ld\a\()xp       x0, x1, [x4]\r
+        cmp             x0, x16\r
+        ccmp            x1, x17, #0, eq\r
+        bne             1f\r
+        st\l\()xp       w15, x16, x17, [x4]\r
+        cbnz            w15, 0b\r
+1:      ret\r
+        fn_end          __aarch64_cas16\model\r
+        .endm\r
+\r
+        /*\r
+         * Emit the set of GCC outline atomic helper functions for\r
+         * the memory ordering model given by \model:\r
+         * - relax      unordered loads and stores\r
+         * - acq        load-acquire, unordered store\r
+         * - rel        unordered load, store-release\r
+         * - acq_rel    load-acquire, store-release\r
+         */\r
+        .macro          emit_model, model:req, a, l\r
+        emit_ld         ldadd, add, \model, \a, \l\r
+        emit_ld         ldclr, bic, \model, \a, \l\r
+        emit_ld         ldeor, eor, \model, \a, \l\r
+        emit_ld         ldset, orr, \model, \a, \l\r
+        emit_ld         swp,   mov, \model, \a, \l\r
+        emit_cas        \model, \a, \l\r
+        .endm\r
+\r
+        emit_model      _relax\r
+        emit_model      _acq, a\r
+        emit_model      _rel,, l\r
+        emit_model      _acq_rel, a, l\r

diff --git a/ArmPkg/Library/CompilerIntrinsicsLib/CompilerIntrinsicsLib.inf b/ArmPkg/Library/CompilerIntrinsicsLib/CompilerIntrinsicsLib.inf
index d5bad946775885c0f2110521a81ceb348049d295..fcf48c67811906d8633fefb8fafa3a0e9d3eada9 100644 (file)
--- a/ArmPkg/Library/CompilerIntrinsicsLib/CompilerIntrinsicsLib.inf
+++ b/ArmPkg/Library/CompilerIntrinsicsLib/CompilerIntrinsicsLib.inf
@@ -79,6 +79,9 @@
   Arm/ldivmod.asm      | MSFT\r
   Arm/llsr.asm         | MSFT\r
 \r
+[Sources.AARCH64]\r
+  AArch64/Atomics.S    | GCC\r
+\r
 [Packages]\r
   MdePkg/MdePkg.dec\r
   ArmPkg/ArmPkg.dec\r