projects / mirror_edk2.git / blame
| Commit | Line | Data |
|---|---|---|
| ca407c72 AB |
1 | #------------------------------------------------------------------------------\r |
| 2 | #\r | |
| 3 | # Copyright (c) 2020, Arm, Limited. All rights reserved.<BR>\r | |
| 4 | #\r | |
| 5 | # SPDX-License-Identifier: BSD-2-Clause-Patent\r | |
| 6 | #\r | |
| 7 | #------------------------------------------------------------------------------\r | |
| 8 | \r | |
| 9 | /*\r | |
| 10 | * Provide the GCC intrinsics that are required when using GCC 9 or\r | |
| 11 | * later with the -moutline-atomics options (which became the default\r | |
| 12 | * in GCC 10)\r | |
| 13 | */\r | |
| 14 | .arch armv8-a\r | |
| 15 | \r | |
| 16 | .macro reg_alias, pfx, sz\r | |
| 17 | r0_\sz .req \pfx\()0\r | |
| 18 | r1_\sz .req \pfx\()1\r | |
| 19 | tmp0_\sz .req \pfx\()16\r | |
| 20 | tmp1_\sz .req \pfx\()17\r | |
| 21 | .endm\r | |
| 22 | \r | |
| 23 | /*\r | |
| 24 | * Define register aliases of the right type for each size\r | |
| 25 | * (xN for 8 bytes, wN for everything smaller)\r | |
| 26 | */\r | |
| 27 | reg_alias w, 1\r | |
| 28 | reg_alias w, 2\r | |
| 29 | reg_alias w, 4\r | |
| 30 | reg_alias x, 8\r | |
| 31 | \r | |
| 32 | .macro fn_start, name:req\r | |
| 33 | .section .text.\name\r | |
| 34 | .globl \name\r | |
| 35 | .type \name, %function\r | |
| 36 | \name\():\r | |
| 37 | .endm\r | |
| 38 | \r | |
| 39 | .macro fn_end, name:req\r | |
| 40 | .size \name, . - \name\r | |
| 41 | .endm\r | |
| 42 | \r | |
| 43 | /*\r | |
| 44 | * Emit an atomic helper for \model with operands of size \sz, using\r | |
| 45 | * the operation specified by \insn (which is the LSE name), and which\r | |
| 46 | * can be implemented using the generic load-locked/store-conditional\r | |
| 47 | * (LL/SC) sequence below, using the arithmetic operation given by\r | |
| 48 | * \opc.\r | |
| 49 | */\r | |
| 50 | .macro emit_ld_sz, sz:req, insn:req, opc:req, model:req, s, a, l\r | |
| 51 | fn_start __aarch64_\insn\()\sz\()\model\r | |
| 52 | mov tmp0_\sz, r0_\sz\r | |
| 53 | 0: ld\a\()xr\s r0_\sz, [x1]\r | |
| 54 | .ifnc \insn, swp\r | |
| 55 | \opc tmp1_\sz, r0_\sz, tmp0_\sz\r | |
| 56 | st\l\()xr\s w15, tmp1_\sz, [x1]\r | |
| 57 | .else\r | |
| 58 | st\l\()xr\s w15, tmp0_\sz, [x1]\r | |
| 59 | .endif\r | |
| 60 | cbnz w15, 0b\r | |
| 61 | ret\r | |
| 62 | fn_end __aarch64_\insn\()\sz\()\model\r | |
| 63 | .endm\r | |
| 64 | \r | |
| 65 | /*\r | |
| 66 | * Emit atomic helpers for \model for operand sizes in the\r | |
| 67 | * set {1, 2, 4, 8}, for the instruction pattern given by\r | |
| 68 | * \insn. (This is the LSE name, but this implementation uses\r | |
| 69 | * the generic LL/SC sequence using \opc as the arithmetic\r | |
| 70 | * operation on the target.)\r | |
| 71 | */\r | |
| 72 | .macro emit_ld, insn:req, opc:req, model:req, a, l\r | |
| 73 | emit_ld_sz 1, \insn, \opc, \model, b, \a, \l\r | |
| 74 | emit_ld_sz 2, \insn, \opc, \model, h, \a, \l\r | |
| 75 | emit_ld_sz 4, \insn, \opc, \model, , \a, \l\r | |
| 76 | emit_ld_sz 8, \insn, \opc, \model, , \a, \l\r | |
| 77 | .endm\r | |
| 78 | \r | |
| 79 | /*\r | |
| 80 | * Emit the compare and swap helper for \model and size \sz\r | |
| 81 | * using LL/SC instructions.\r | |
| 82 | */\r | |
| 83 | .macro emit_cas_sz, sz:req, model:req, uxt:req, s, a, l\r | |
| 84 | fn_start __aarch64_cas\sz\()\model\r | |
| 85 | \uxt tmp0_\sz, r0_\sz\r | |
| 86 | 0: ld\a\()xr\s r0_\sz, [x2]\r | |
| 87 | cmp r0_\sz, tmp0_\sz\r | |
| 88 | bne 1f\r | |
| 89 | st\l\()xr\s w15, r1_\sz, [x2]\r | |
| 90 | cbnz w15, 0b\r | |
| 91 | 1: ret\r | |
| 92 | fn_end __aarch64_cas\sz\()\model\r | |
| 93 | .endm\r | |
| 94 | \r | |
| 95 | /*\r | |
| 96 | * Emit compare-and-swap helpers for \model for operand sizes in the\r | |
| 97 | * set {1, 2, 4, 8, 16}.\r | |
| 98 | */\r | |
| 99 | .macro emit_cas, model:req, a, l\r | |
| 100 | emit_cas_sz 1, \model, uxtb, b, \a, \l\r | |
| 101 | emit_cas_sz 2, \model, uxth, h, \a, \l\r | |
| 102 | emit_cas_sz 4, \model, mov , , \a, \l\r | |
| 103 | emit_cas_sz 8, \model, mov , , \a, \l\r | |
| 104 | \r | |
| 105 | /*\r | |
| 106 | * We cannot use the parameterized sequence for 16 byte CAS, so we\r | |
| 107 | * need to define it explicitly.\r | |
| 108 | */\r | |
| 109 | fn_start __aarch64_cas16\model\r | |
| 110 | mov x16, x0\r | |
| 111 | mov x17, x1\r | |
| 112 | 0: ld\a\()xp x0, x1, [x4]\r | |
| 113 | cmp x0, x16\r | |
| 114 | ccmp x1, x17, #0, eq\r | |
| 115 | bne 1f\r | |
| 116 | st\l\()xp w15, x16, x17, [x4]\r | |
| 117 | cbnz w15, 0b\r | |
| 118 | 1: ret\r | |
| 119 | fn_end __aarch64_cas16\model\r | |
| 120 | .endm\r | |
| 121 | \r | |
| 122 | /*\r | |
| 123 | * Emit the set of GCC outline atomic helper functions for\r | |
| 124 | * the memory ordering model given by \model:\r | |
| 125 | * - relax unordered loads and stores\r | |
| 126 | * - acq load-acquire, unordered store\r | |
| 127 | * - rel unordered load, store-release\r | |
| 128 | * - acq_rel load-acquire, store-release\r | |
| 129 | */\r | |
| 130 | .macro emit_model, model:req, a, l\r | |
| 131 | emit_ld ldadd, add, \model, \a, \l\r | |
| 132 | emit_ld ldclr, bic, \model, \a, \l\r | |
| 133 | emit_ld ldeor, eor, \model, \a, \l\r | |
| 134 | emit_ld ldset, orr, \model, \a, \l\r | |
| 135 | emit_ld swp, mov, \model, \a, \l\r | |
| 136 | emit_cas \model, \a, \l\r | |
| 137 | .endm\r | |
| 138 | \r | |
| 139 | emit_model _relax\r | |
| 140 | emit_model _acq, a\r | |
| 141 | emit_model _rel,, l\r | |
| 142 | emit_model _acq_rel, a, l\r |