2 * ARM v8.3-PAuth Operations
4 * Copyright (c) 2019 Linaro, Ltd.
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2.1 of the License, or (at your option) any later version.
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
20 #include "qemu/osdep.h"
22 #include "internals.h"
23 #include "exec/exec-all.h"
24 #include "exec/cpu_ldst.h"
25 #include "exec/helper-proto.h"
26 #include "tcg/tcg-gvec-desc.h"
27 #include "qemu/xxhash.h"
30 static uint64_t pac_cell_shuffle(uint64_t i
)
34 o
|= extract64(i
, 52, 4);
35 o
|= extract64(i
, 24, 4) << 4;
36 o
|= extract64(i
, 44, 4) << 8;
37 o
|= extract64(i
, 0, 4) << 12;
39 o
|= extract64(i
, 28, 4) << 16;
40 o
|= extract64(i
, 48, 4) << 20;
41 o
|= extract64(i
, 4, 4) << 24;
42 o
|= extract64(i
, 40, 4) << 28;
44 o
|= extract64(i
, 32, 4) << 32;
45 o
|= extract64(i
, 12, 4) << 36;
46 o
|= extract64(i
, 56, 4) << 40;
47 o
|= extract64(i
, 20, 4) << 44;
49 o
|= extract64(i
, 8, 4) << 48;
50 o
|= extract64(i
, 36, 4) << 52;
51 o
|= extract64(i
, 16, 4) << 56;
52 o
|= extract64(i
, 60, 4) << 60;
57 static uint64_t pac_cell_inv_shuffle(uint64_t i
)
61 o
|= extract64(i
, 12, 4);
62 o
|= extract64(i
, 24, 4) << 4;
63 o
|= extract64(i
, 48, 4) << 8;
64 o
|= extract64(i
, 36, 4) << 12;
66 o
|= extract64(i
, 56, 4) << 16;
67 o
|= extract64(i
, 44, 4) << 20;
68 o
|= extract64(i
, 4, 4) << 24;
69 o
|= extract64(i
, 16, 4) << 28;
71 o
|= i
& MAKE_64BIT_MASK(32, 4);
72 o
|= extract64(i
, 52, 4) << 36;
73 o
|= extract64(i
, 28, 4) << 40;
74 o
|= extract64(i
, 8, 4) << 44;
76 o
|= extract64(i
, 20, 4) << 48;
77 o
|= extract64(i
, 0, 4) << 52;
78 o
|= extract64(i
, 40, 4) << 56;
79 o
|= i
& MAKE_64BIT_MASK(60, 4);
84 static uint64_t pac_sub(uint64_t i
)
86 static const uint8_t sub
[16] = {
87 0xb, 0x6, 0x8, 0xf, 0xc, 0x0, 0x9, 0xe,
88 0x3, 0x7, 0x4, 0x5, 0xd, 0x2, 0x1, 0xa,
93 for (b
= 0; b
< 64; b
+= 4) {
94 o
|= (uint64_t)sub
[(i
>> b
) & 0xf] << b
;
99 static uint64_t pac_sub1(uint64_t i
)
101 static const uint8_t sub1
[16] = {
102 0xa, 0xd, 0xe, 0x6, 0xf, 0x7, 0x3, 0x5,
103 0x9, 0x8, 0x0, 0xc, 0xb, 0x1, 0x2, 0x4,
108 for (b
= 0; b
< 64; b
+= 4) {
109 o
|= (uint64_t)sub1
[(i
>> b
) & 0xf] << b
;
114 static uint64_t pac_inv_sub(uint64_t i
)
116 static const uint8_t inv_sub
[16] = {
117 0x5, 0xe, 0xd, 0x8, 0xa, 0xb, 0x1, 0x9,
118 0x2, 0x6, 0xf, 0x0, 0x4, 0xc, 0x7, 0x3,
123 for (b
= 0; b
< 64; b
+= 4) {
124 o
|= (uint64_t)inv_sub
[(i
>> b
) & 0xf] << b
;
129 static int rot_cell(int cell
, int n
)
131 /* 4-bit rotate left by n. */
133 return extract32(cell
, 4 - n
, 4);
136 static uint64_t pac_mult(uint64_t i
)
141 for (b
= 0; b
< 4 * 4; b
+= 4) {
142 int i0
, i4
, i8
, ic
, t0
, t1
, t2
, t3
;
144 i0
= extract64(i
, b
, 4);
145 i4
= extract64(i
, b
+ 4 * 4, 4);
146 i8
= extract64(i
, b
+ 8 * 4, 4);
147 ic
= extract64(i
, b
+ 12 * 4, 4);
149 t0
= rot_cell(i8
, 1) ^ rot_cell(i4
, 2) ^ rot_cell(i0
, 1);
150 t1
= rot_cell(ic
, 1) ^ rot_cell(i4
, 1) ^ rot_cell(i0
, 2);
151 t2
= rot_cell(ic
, 2) ^ rot_cell(i8
, 1) ^ rot_cell(i0
, 1);
152 t3
= rot_cell(ic
, 1) ^ rot_cell(i8
, 2) ^ rot_cell(i4
, 1);
154 o
|= (uint64_t)t3
<< b
;
155 o
|= (uint64_t)t2
<< (b
+ 4 * 4);
156 o
|= (uint64_t)t1
<< (b
+ 8 * 4);
157 o
|= (uint64_t)t0
<< (b
+ 12 * 4);
162 static uint64_t tweak_cell_rot(uint64_t cell
)
164 return (cell
>> 1) | (((cell
^ (cell
>> 1)) & 1) << 3);
167 static uint64_t tweak_shuffle(uint64_t i
)
171 o
|= extract64(i
, 16, 4) << 0;
172 o
|= extract64(i
, 20, 4) << 4;
173 o
|= tweak_cell_rot(extract64(i
, 24, 4)) << 8;
174 o
|= extract64(i
, 28, 4) << 12;
176 o
|= tweak_cell_rot(extract64(i
, 44, 4)) << 16;
177 o
|= extract64(i
, 8, 4) << 20;
178 o
|= extract64(i
, 12, 4) << 24;
179 o
|= tweak_cell_rot(extract64(i
, 32, 4)) << 28;
181 o
|= extract64(i
, 48, 4) << 32;
182 o
|= extract64(i
, 52, 4) << 36;
183 o
|= extract64(i
, 56, 4) << 40;
184 o
|= tweak_cell_rot(extract64(i
, 60, 4)) << 44;
186 o
|= tweak_cell_rot(extract64(i
, 0, 4)) << 48;
187 o
|= extract64(i
, 4, 4) << 52;
188 o
|= tweak_cell_rot(extract64(i
, 40, 4)) << 56;
189 o
|= tweak_cell_rot(extract64(i
, 36, 4)) << 60;
194 static uint64_t tweak_cell_inv_rot(uint64_t cell
)
196 return ((cell
<< 1) & 0xf) | ((cell
& 1) ^ (cell
>> 3));
199 static uint64_t tweak_inv_shuffle(uint64_t i
)
203 o
|= tweak_cell_inv_rot(extract64(i
, 48, 4));
204 o
|= extract64(i
, 52, 4) << 4;
205 o
|= extract64(i
, 20, 4) << 8;
206 o
|= extract64(i
, 24, 4) << 12;
208 o
|= extract64(i
, 0, 4) << 16;
209 o
|= extract64(i
, 4, 4) << 20;
210 o
|= tweak_cell_inv_rot(extract64(i
, 8, 4)) << 24;
211 o
|= extract64(i
, 12, 4) << 28;
213 o
|= tweak_cell_inv_rot(extract64(i
, 28, 4)) << 32;
214 o
|= tweak_cell_inv_rot(extract64(i
, 60, 4)) << 36;
215 o
|= tweak_cell_inv_rot(extract64(i
, 56, 4)) << 40;
216 o
|= tweak_cell_inv_rot(extract64(i
, 16, 4)) << 44;
218 o
|= extract64(i
, 32, 4) << 48;
219 o
|= extract64(i
, 36, 4) << 52;
220 o
|= extract64(i
, 40, 4) << 56;
221 o
|= tweak_cell_inv_rot(extract64(i
, 44, 4)) << 60;
226 static uint64_t pauth_computepac_architected(uint64_t data
, uint64_t modifier
,
227 ARMPACKey key
, bool isqarma3
)
229 static const uint64_t RC
[5] = {
230 0x0000000000000000ull
,
231 0x13198A2E03707344ull
,
232 0xA4093822299F31D0ull
,
233 0x082EFA98EC4E6C89ull
,
234 0x452821E638D01377ull
,
236 const uint64_t alpha
= 0xC0AC29B7C97C50DDull
;
237 int iterations
= isqarma3
? 2 : 4;
239 * Note that in the ARM pseudocode, key0 contains bits <127:64>
240 * and key1 contains bits <63:0> of the 128-bit key.
242 uint64_t key0
= key
.hi
, key1
= key
.lo
;
243 uint64_t workingval
, runningmod
, roundkey
, modk0
;
246 modk0
= (key0
<< 63) | ((key0
>> 1) ^ (key0
>> 63));
247 runningmod
= modifier
;
248 workingval
= data
^ key0
;
250 for (i
= 0; i
<= iterations
; ++i
) {
251 roundkey
= key1
^ runningmod
;
252 workingval
^= roundkey
;
255 workingval
= pac_cell_shuffle(workingval
);
256 workingval
= pac_mult(workingval
);
259 workingval
= pac_sub1(workingval
);
261 workingval
= pac_sub(workingval
);
263 runningmod
= tweak_shuffle(runningmod
);
265 roundkey
= modk0
^ runningmod
;
266 workingval
^= roundkey
;
267 workingval
= pac_cell_shuffle(workingval
);
268 workingval
= pac_mult(workingval
);
270 workingval
= pac_sub1(workingval
);
272 workingval
= pac_sub(workingval
);
274 workingval
= pac_cell_shuffle(workingval
);
275 workingval
= pac_mult(workingval
);
277 workingval
= pac_cell_inv_shuffle(workingval
);
279 workingval
= pac_sub1(workingval
);
281 workingval
= pac_inv_sub(workingval
);
283 workingval
= pac_mult(workingval
);
284 workingval
= pac_cell_inv_shuffle(workingval
);
286 workingval
^= runningmod
;
287 for (i
= 0; i
<= iterations
; ++i
) {
289 workingval
= pac_sub1(workingval
);
291 workingval
= pac_inv_sub(workingval
);
293 if (i
< iterations
) {
294 workingval
= pac_mult(workingval
);
295 workingval
= pac_cell_inv_shuffle(workingval
);
297 runningmod
= tweak_inv_shuffle(runningmod
);
298 roundkey
= key1
^ runningmod
;
299 workingval
^= RC
[iterations
- i
];
300 workingval
^= roundkey
;
308 static uint64_t pauth_computepac_impdef(uint64_t data
, uint64_t modifier
,
311 return qemu_xxhash64_4(data
, modifier
, key
.lo
, key
.hi
);
314 static uint64_t pauth_computepac(CPUARMState
*env
, uint64_t data
,
315 uint64_t modifier
, ARMPACKey key
)
317 if (cpu_isar_feature(aa64_pauth_qarma5
, env_archcpu(env
))) {
318 return pauth_computepac_architected(data
, modifier
, key
, false);
319 } else if (cpu_isar_feature(aa64_pauth_qarma3
, env_archcpu(env
))) {
320 return pauth_computepac_architected(data
, modifier
, key
, true);
322 return pauth_computepac_impdef(data
, modifier
, key
);
326 static uint64_t pauth_addpac(CPUARMState
*env
, uint64_t ptr
, uint64_t modifier
,
327 ARMPACKey
*key
, bool data
)
329 ARMCPU
*cpu
= env_archcpu(env
);
330 ARMMMUIdx mmu_idx
= arm_stage1_mmu_idx(env
);
331 ARMVAParameters param
= aa64_va_parameters(env
, ptr
, mmu_idx
, data
, false);
332 ARMPauthFeature pauth_feature
= cpu_isar_feature(pauth_feature
, cpu
);
333 uint64_t pac
, ext_ptr
, ext
, test
;
334 int bot_bit
, top_bit
;
336 /* If tagged pointers are in use, use ptr<55>, otherwise ptr<63>. */
338 ext
= sextract64(ptr
, 55, 1);
340 ext
= sextract64(ptr
, 63, 1);
343 /* Build a pointer with known good extension bits. */
344 top_bit
= 64 - 8 * param
.tbi
;
345 bot_bit
= 64 - param
.tsz
;
346 ext_ptr
= deposit64(ptr
, bot_bit
, top_bit
- bot_bit
, ext
);
348 pac
= pauth_computepac(env
, ext_ptr
, modifier
, *key
);
351 * Check if the ptr has good extension bits and corrupt the
352 * pointer authentication code if not.
354 test
= sextract64(ptr
, bot_bit
, top_bit
- bot_bit
);
355 if (test
!= 0 && test
!= -1) {
356 if (pauth_feature
>= PauthFeat_2
) {
357 /* No action required */
358 } else if (pauth_feature
== PauthFeat_EPAC
) {
362 * Note that our top_bit is one greater than the pseudocode's
363 * version, hence "- 2" here.
365 pac
^= MAKE_64BIT_MASK(top_bit
- 2, 1);
370 * Preserve the determination between upper and lower at bit 55,
371 * and insert pointer authentication code.
373 if (pauth_feature
>= PauthFeat_2
) {
377 ptr
&= ~MAKE_64BIT_MASK(bot_bit
, 55 - bot_bit
+ 1);
378 pac
&= MAKE_64BIT_MASK(bot_bit
, 54 - bot_bit
+ 1);
380 ptr
&= MAKE_64BIT_MASK(0, bot_bit
);
381 pac
&= ~(MAKE_64BIT_MASK(55, 1) | MAKE_64BIT_MASK(0, bot_bit
));
383 ext
&= MAKE_64BIT_MASK(55, 1);
384 return pac
| ext
| ptr
;
387 static uint64_t pauth_original_ptr(uint64_t ptr
, ARMVAParameters param
)
389 uint64_t mask
= pauth_ptr_mask(param
);
391 /* Note that bit 55 is used whether or not the regime has 2 ranges. */
392 if (extract64(ptr
, 55, 1)) {
400 void pauth_fail_exception(CPUARMState
*env
, bool data
,
401 int keynumber
, uintptr_t ra
)
403 raise_exception_ra(env
, EXCP_UDEF
, syn_pacfail(data
, keynumber
),
404 exception_target_el(env
), ra
);
407 static uint64_t pauth_auth(CPUARMState
*env
, uint64_t ptr
, uint64_t modifier
,
408 ARMPACKey
*key
, bool data
, int keynumber
,
409 uintptr_t ra
, bool is_combined
)
411 ARMCPU
*cpu
= env_archcpu(env
);
412 ARMMMUIdx mmu_idx
= arm_stage1_mmu_idx(env
);
413 ARMVAParameters param
= aa64_va_parameters(env
, ptr
, mmu_idx
, data
, false);
414 ARMPauthFeature pauth_feature
= cpu_isar_feature(pauth_feature
, cpu
);
415 int bot_bit
, top_bit
;
416 uint64_t pac
, orig_ptr
, cmp_mask
;
418 orig_ptr
= pauth_original_ptr(ptr
, param
);
419 pac
= pauth_computepac(env
, orig_ptr
, modifier
, *key
);
420 bot_bit
= 64 - param
.tsz
;
421 top_bit
= 64 - 8 * param
.tbi
;
423 cmp_mask
= MAKE_64BIT_MASK(bot_bit
, top_bit
- bot_bit
);
424 cmp_mask
&= ~MAKE_64BIT_MASK(55, 1);
426 if (pauth_feature
>= PauthFeat_2
) {
427 ARMPauthFeature fault_feature
=
428 is_combined
? PauthFeat_FPACCOMBINED
: PauthFeat_FPAC
;
429 uint64_t result
= ptr
^ (pac
& cmp_mask
);
431 if (pauth_feature
>= fault_feature
432 && ((result
^ sextract64(result
, 55, 1)) & cmp_mask
)) {
433 pauth_fail_exception(env
, data
, keynumber
, ra
);
438 if ((pac
^ ptr
) & cmp_mask
) {
439 int error_code
= (keynumber
<< 1) | (keynumber
^ 1);
441 return deposit64(orig_ptr
, 53, 2, error_code
);
443 return deposit64(orig_ptr
, 61, 2, error_code
);
449 static uint64_t pauth_strip(CPUARMState
*env
, uint64_t ptr
, bool data
)
451 ARMMMUIdx mmu_idx
= arm_stage1_mmu_idx(env
);
452 ARMVAParameters param
= aa64_va_parameters(env
, ptr
, mmu_idx
, data
, false);
454 return pauth_original_ptr(ptr
, param
);
458 void pauth_trap(CPUARMState
*env
, int target_el
, uintptr_t ra
)
460 raise_exception_ra(env
, EXCP_UDEF
, syn_pactrap(), target_el
, ra
);
463 static void pauth_check_trap(CPUARMState
*env
, int el
, uintptr_t ra
)
465 if (el
< 2 && arm_is_el2_enabled(env
)) {
466 uint64_t hcr
= arm_hcr_el2_eff(env
);
467 bool trap
= !(hcr
& HCR_API
);
469 /* Trap only applies to EL1&0 regime. */
470 trap
&= (hcr
& (HCR_E2H
| HCR_TGE
)) != (HCR_E2H
| HCR_TGE
);
472 /* FIXME: ARMv8.3-NV: HCR_NV trap takes precedence for ERETA[AB]. */
474 pauth_trap(env
, 2, ra
);
477 if (el
< 3 && arm_feature(env
, ARM_FEATURE_EL3
)) {
478 if (!(env
->cp15
.scr_el3
& SCR_API
)) {
479 pauth_trap(env
, 3, ra
);
484 static bool pauth_key_enabled(CPUARMState
*env
, int el
, uint32_t bit
)
486 return (arm_sctlr(env
, el
) & bit
) != 0;
489 uint64_t HELPER(pacia
)(CPUARMState
*env
, uint64_t x
, uint64_t y
)
491 int el
= arm_current_el(env
);
492 if (!pauth_key_enabled(env
, el
, SCTLR_EnIA
)) {
495 pauth_check_trap(env
, el
, GETPC());
496 return pauth_addpac(env
, x
, y
, &env
->keys
.apia
, false);
499 uint64_t HELPER(pacib
)(CPUARMState
*env
, uint64_t x
, uint64_t y
)
501 int el
= arm_current_el(env
);
502 if (!pauth_key_enabled(env
, el
, SCTLR_EnIB
)) {
505 pauth_check_trap(env
, el
, GETPC());
506 return pauth_addpac(env
, x
, y
, &env
->keys
.apib
, false);
509 uint64_t HELPER(pacda
)(CPUARMState
*env
, uint64_t x
, uint64_t y
)
511 int el
= arm_current_el(env
);
512 if (!pauth_key_enabled(env
, el
, SCTLR_EnDA
)) {
515 pauth_check_trap(env
, el
, GETPC());
516 return pauth_addpac(env
, x
, y
, &env
->keys
.apda
, true);
519 uint64_t HELPER(pacdb
)(CPUARMState
*env
, uint64_t x
, uint64_t y
)
521 int el
= arm_current_el(env
);
522 if (!pauth_key_enabled(env
, el
, SCTLR_EnDB
)) {
525 pauth_check_trap(env
, el
, GETPC());
526 return pauth_addpac(env
, x
, y
, &env
->keys
.apdb
, true);
529 uint64_t HELPER(pacga
)(CPUARMState
*env
, uint64_t x
, uint64_t y
)
533 pauth_check_trap(env
, arm_current_el(env
), GETPC());
534 pac
= pauth_computepac(env
, x
, y
, env
->keys
.apga
);
536 return pac
& 0xffffffff00000000ull
;
539 static uint64_t pauth_autia(CPUARMState
*env
, uint64_t x
, uint64_t y
,
540 uintptr_t ra
, bool is_combined
)
542 int el
= arm_current_el(env
);
543 if (!pauth_key_enabled(env
, el
, SCTLR_EnIA
)) {
546 pauth_check_trap(env
, el
, ra
);
547 return pauth_auth(env
, x
, y
, &env
->keys
.apia
, false, 0, ra
, is_combined
);
550 uint64_t HELPER(autia
)(CPUARMState
*env
, uint64_t x
, uint64_t y
)
552 return pauth_autia(env
, x
, y
, GETPC(), false);
555 uint64_t HELPER(autia_combined
)(CPUARMState
*env
, uint64_t x
, uint64_t y
)
557 return pauth_autia(env
, x
, y
, GETPC(), true);
560 static uint64_t pauth_autib(CPUARMState
*env
, uint64_t x
, uint64_t y
,
561 uintptr_t ra
, bool is_combined
)
563 int el
= arm_current_el(env
);
564 if (!pauth_key_enabled(env
, el
, SCTLR_EnIB
)) {
567 pauth_check_trap(env
, el
, ra
);
568 return pauth_auth(env
, x
, y
, &env
->keys
.apib
, false, 1, ra
, is_combined
);
571 uint64_t HELPER(autib
)(CPUARMState
*env
, uint64_t x
, uint64_t y
)
573 return pauth_autib(env
, x
, y
, GETPC(), false);
576 uint64_t HELPER(autib_combined
)(CPUARMState
*env
, uint64_t x
, uint64_t y
)
578 return pauth_autib(env
, x
, y
, GETPC(), true);
581 static uint64_t pauth_autda(CPUARMState
*env
, uint64_t x
, uint64_t y
,
582 uintptr_t ra
, bool is_combined
)
584 int el
= arm_current_el(env
);
585 if (!pauth_key_enabled(env
, el
, SCTLR_EnDA
)) {
588 pauth_check_trap(env
, el
, ra
);
589 return pauth_auth(env
, x
, y
, &env
->keys
.apda
, true, 0, ra
, is_combined
);
592 uint64_t HELPER(autda
)(CPUARMState
*env
, uint64_t x
, uint64_t y
)
594 return pauth_autda(env
, x
, y
, GETPC(), false);
597 uint64_t HELPER(autda_combined
)(CPUARMState
*env
, uint64_t x
, uint64_t y
)
599 return pauth_autda(env
, x
, y
, GETPC(), true);
602 static uint64_t pauth_autdb(CPUARMState
*env
, uint64_t x
, uint64_t y
,
603 uintptr_t ra
, bool is_combined
)
605 int el
= arm_current_el(env
);
606 if (!pauth_key_enabled(env
, el
, SCTLR_EnDB
)) {
609 pauth_check_trap(env
, el
, ra
);
610 return pauth_auth(env
, x
, y
, &env
->keys
.apdb
, true, 1, ra
, is_combined
);
613 uint64_t HELPER(autdb
)(CPUARMState
*env
, uint64_t x
, uint64_t y
)
615 return pauth_autdb(env
, x
, y
, GETPC(), false);
618 uint64_t HELPER(autdb_combined
)(CPUARMState
*env
, uint64_t x
, uint64_t y
)
620 return pauth_autdb(env
, x
, y
, GETPC(), true);
623 uint64_t HELPER(xpaci
)(CPUARMState
*env
, uint64_t a
)
625 return pauth_strip(env
, a
, false);
628 uint64_t HELPER(xpacd
)(CPUARMState
*env
, uint64_t a
)
630 return pauth_strip(env
, a
, true);