]> git.proxmox.com Git - mirror_ubuntu-hirsute-kernel.git/blame - arch/arm64/kvm/sys_regs.c
projects / mirror_ubuntu-hirsute-kernel.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
UBUNTU: Ubuntu-5.11.0-22.23
[mirror_ubuntu-hirsute-kernel.git] / arch / arm64 / kvm / sys_regs.c
CommitLineData
caab277b 1// SPDX-License-Identifier: GPL-2.0-only
7c8c5e6a
MZ		 2/*
3 * Copyright (C) 2012,2013 - ARM Ltd
4 * Author: Marc Zyngier <marc.zyngier@arm.com>
5 *
6 * Derived from arch/arm/kvm/coproc.c:
7 * Copyright (C) 2012 - Virtual Open Systems and Columbia University
8 * Authors: Rusty Russell <rusty@rustcorp.com.au>
9 * Christoffer Dall <c.dall@virtualopensystems.com>
7c8c5e6a
MZ		 10 */
11
623eefa8 12#include <linux/bsearch.h>
7c8c5e6a 13#include <linux/kvm_host.h>
c6d01a94 14#include <linux/mm.h>
07d79fe7 15#include <linux/printk.h>
7c8c5e6a 16#include <linux/uaccess.h>
c6d01a94 17
7c8c5e6a
MZ		 18#include <asm/cacheflush.h>
19#include <asm/cputype.h>
0c557ed4 20#include <asm/debug-monitors.h>
c6d01a94
MR		 21#include <asm/esr.h>
22#include <asm/kvm_arm.h>
c6d01a94 23#include <asm/kvm_emulate.h>
d47533da 24#include <asm/kvm_hyp.h>
c6d01a94 25#include <asm/kvm_mmu.h>
ab946834 26#include <asm/perf_event.h>
1f3d8699 27#include <asm/sysreg.h>
c6d01a94 28
7c8c5e6a
MZ		 29#include <trace/events/kvm.h>
30
31#include "sys_regs.h"
32
eef8c85a
AB		 33#include "trace.h"
34
7c8c5e6a 35/*
656012c7 36 * All of this file is extremely similar to the ARM coproc.c, but the
7c8c5e6a
MZ		 37 * types are different. My gut feeling is that it should be pretty
38 * easy to merge, but that would be an ABI breakage -- again. VFP
39 * would also need to be abstracted.
62a89c44
MZ		 40 *
41 * For AArch32, we only take care of what is being trapped. Anything
42 * that has to do with init and userspace access has to go via the
43 * 64bit interface.
7c8c5e6a
MZ		 44 */
45
7ba8b438
AE		 46#define reg_to_encoding(x) \
47 sys_reg((u32)(x)->Op0, (u32)(x)->Op1, \
48 (u32)(x)->CRn, (u32)(x)->CRm, (u32)(x)->Op2)
49
7b5b4df1 50static bool read_from_write_only(struct kvm_vcpu *vcpu,
e7f1d1ee
MZ		 51 struct sys_reg_params *params,
52 const struct sys_reg_desc *r)
7b5b4df1
MZ		 53{
54 WARN_ONCE(1, "Unexpected sys_reg read to write-only register\n");
55 print_sys_reg_instr(params);
56 kvm_inject_undefined(vcpu);
57 return false;
58}
59
7b1dba1f
MZ		 60static bool write_to_read_only(struct kvm_vcpu *vcpu,
61 struct sys_reg_params *params,
62 const struct sys_reg_desc *r)
63{
64 WARN_ONCE(1, "Unexpected sys_reg write to read-only register\n");
65 print_sys_reg_instr(params);
66 kvm_inject_undefined(vcpu);
67 return false;
68}
69
7ea90bdd
MZ		 70u64 vcpu_read_sys_reg(const struct kvm_vcpu *vcpu, int reg)
71{
72 u64 val = 0x8badf00d8badf00d;
73
74 if (vcpu->arch.sysregs_loaded_on_cpu &&
75 __vcpu_read_sys_reg_from_cpu(reg, &val))
76 return val;
77
78 return __vcpu_sys_reg(vcpu, reg);
79}
80
81void vcpu_write_sys_reg(struct kvm_vcpu *vcpu, u64 val, int reg)
82{
83 if (vcpu->arch.sysregs_loaded_on_cpu &&
84 __vcpu_write_sys_reg_to_cpu(val, reg))
85 return;
86
d47533da
CD		 87 __vcpu_sys_reg(vcpu, reg) = val;
88}
89
7c8c5e6a
MZ		 90/* 3 bits per cache level, as per CLIDR, but non-existent caches always 0 */
91static u32 cache_levels;
92
93/* CSSELR values; used to index KVM_REG_ARM_DEMUX_ID_CCSIDR */
c73a4416 94#define CSSELR_MAX 14
7c8c5e6a
MZ		 95
96/* Which cache CCSIDR represents depends on CSSELR value. */
97static u32 get_ccsidr(u32 csselr)
98{
99 u32 ccsidr;
100
101 /* Make sure noone else changes CSSELR during this! */
102 local_irq_disable();
1f3d8699 103 write_sysreg(csselr, csselr_el1);
7c8c5e6a 104 isb();
1f3d8699 105 ccsidr = read_sysreg(ccsidr_el1);
7c8c5e6a
MZ		 106 local_irq_enable();
107
108 return ccsidr;
109}
110
3c1e7165
MZ		 111/*
112 * See note at ARMv7 ARM B1.14.4 (TL;DR: S/W ops are not easily virtualized).
113 */
7c8c5e6a 114static bool access_dcsw(struct kvm_vcpu *vcpu,
3fec037d 115 struct sys_reg_params *p,
7c8c5e6a
MZ		 116 const struct sys_reg_desc *r)
117{
7c8c5e6a 118 if (!p->is_write)
e7f1d1ee 119 return read_from_write_only(vcpu, p, r);
7c8c5e6a 120
09605e94
MZ		 121 /*
122 * Only track S/W ops if we don't have FWB. It still indicates
123 * that the guest is a bit broken (S/W operations should only
124 * be done by firmware, knowing that there is only a single
125 * CPU left in the system, and certainly not from non-secure
126 * software).
127 */
128 if (!cpus_have_const_cap(ARM64_HAS_STAGE2_FWB))
129 kvm_set_way_flush(vcpu);
130
7c8c5e6a
MZ		 131 return true;
132}
133
b1ea1d76
MZ		 134static void get_access_mask(const struct sys_reg_desc *r, u64 *mask, u64 *shift)
135{
136 switch (r->aarch32_map) {
137 case AA32_LO:
138 *mask = GENMASK_ULL(31, 0);
139 *shift = 0;
140 break;
141 case AA32_HI:
142 *mask = GENMASK_ULL(63, 32);
143 *shift = 32;
144 break;
145 default:
146 *mask = GENMASK_ULL(63, 0);
147 *shift = 0;
148 break;
149 }
150}
151
4d44923b
MZ		 152/*
153 * Generic accessor for VM registers. Only called as long as HCR_TVM
3c1e7165
MZ		 154 * is set. If the guest enables the MMU, we stop trapping the VM
155 * sys_regs and leave it in complete control of the caches.
4d44923b
MZ		 156 */
157static bool access_vm_reg(struct kvm_vcpu *vcpu,
3fec037d 158 struct sys_reg_params *p,
4d44923b
MZ		 159 const struct sys_reg_desc *r)
160{
3c1e7165 161 bool was_enabled = vcpu_has_cache_enabled(vcpu);
b1ea1d76 162 u64 val, mask, shift;
4d44923b
MZ		 163
164 BUG_ON(!p->is_write);
165
b1ea1d76 166 get_access_mask(r, &mask, &shift);
52f6c4f0 167
b1ea1d76
MZ		 168 if (~mask) {
169 val = vcpu_read_sys_reg(vcpu, r->reg);
170 val &= ~mask;
dedf97e8 171 } else {
b1ea1d76 172 val = 0;
dedf97e8 173 }
b1ea1d76
MZ		 174
175 val |= (p->regval & (mask >> shift)) << shift;
176 vcpu_write_sys_reg(vcpu, val, r->reg);
f0a3eaff 177
3c1e7165 178 kvm_toggle_cache(vcpu, was_enabled);
4d44923b
MZ		 179 return true;
180}
181
af473829
JM		 182static bool access_actlr(struct kvm_vcpu *vcpu,
183 struct sys_reg_params *p,
184 const struct sys_reg_desc *r)
185{
b1ea1d76
MZ		 186 u64 mask, shift;
187
af473829
JM		 188 if (p->is_write)
189 return ignore_write(vcpu, p);
190
b1ea1d76
MZ		 191 get_access_mask(r, &mask, &shift);
192 p->regval = (vcpu_read_sys_reg(vcpu, r->reg) & mask) >> shift;
af473829
JM		 193
194 return true;
195}
196
6d52f35a
AP		 197/*
198 * Trap handler for the GICv3 SGI generation system register.
199 * Forward the request to the VGIC emulation.
200 * The cp15_64 code makes sure this automatically works
201 * for both AArch64 and AArch32 accesses.
202 */
203static bool access_gic_sgi(struct kvm_vcpu *vcpu,
3fec037d 204 struct sys_reg_params *p,
6d52f35a
AP		 205 const struct sys_reg_desc *r)
206{
03bd646d
MZ		 207 bool g1;
208
6d52f35a 209 if (!p->is_write)
e7f1d1ee 210 return read_from_write_only(vcpu, p, r);
6d52f35a 211
03bd646d
MZ		 212 /*
213 * In a system where GICD_CTLR.DS=1, a ICC_SGI0R_EL1 access generates
214 * Group0 SGIs only, while ICC_SGI1R_EL1 can generate either group,
215 * depending on the SGI configuration. ICC_ASGI1R_EL1 is effectively
216 * equivalent to ICC_SGI0R_EL1, as there is no "alternative" secure
217 * group.
218 */
50f30453 219 if (p->Op0 == 0) { /* AArch32 */
03bd646d
MZ		 220 switch (p->Op1) {
221 default: /* Keep GCC quiet */
222 case 0: /* ICC_SGI1R */
223 g1 = true;
224 break;
225 case 1: /* ICC_ASGI1R */
226 case 2: /* ICC_SGI0R */
227 g1 = false;
228 break;
229 }
50f30453 230 } else { /* AArch64 */
03bd646d
MZ		 231 switch (p->Op2) {
232 default: /* Keep GCC quiet */
233 case 5: /* ICC_SGI1R_EL1 */
234 g1 = true;
235 break;
236 case 6: /* ICC_ASGI1R_EL1 */
237 case 7: /* ICC_SGI0R_EL1 */
238 g1 = false;
239 break;
240 }
241 }
242
243 vgic_v3_dispatch_sgi(vcpu, p->regval, g1);
6d52f35a
AP		 244
245 return true;
246}
247
b34f2bcb
MZ		 248static bool access_gic_sre(struct kvm_vcpu *vcpu,
249 struct sys_reg_params *p,
250 const struct sys_reg_desc *r)
251{
252 if (p->is_write)
253 return ignore_write(vcpu, p);
254
255 p->regval = vcpu->arch.vgic_cpu.vgic_v3.vgic_sre;
256 return true;
257}
258
7609c125 259static bool trap_raz_wi(struct kvm_vcpu *vcpu,
3fec037d 260 struct sys_reg_params *p,
7609c125 261 const struct sys_reg_desc *r)
7c8c5e6a
MZ		 262{
263 if (p->is_write)
264 return ignore_write(vcpu, p);
265 else
266 return read_zero(vcpu, p);
267}
268
22925521
MZ		 269/*
270 * ARMv8.1 mandates at least a trivial LORegion implementation, where all the
271 * RW registers are RES0 (which we can implement as RAZ/WI). On an ARMv8.0
272 * system, these registers should UNDEF. LORID_EL1 being a RO register, we
273 * treat it separately.
274 */
275static bool trap_loregion(struct kvm_vcpu *vcpu,
276 struct sys_reg_params *p,
277 const struct sys_reg_desc *r)
cc33c4e2 278{
22925521 279 u64 val = read_sanitised_ftr_reg(SYS_ID_AA64MMFR1_EL1);
7ba8b438 280 u32 sr = reg_to_encoding(r);
22925521
MZ		 281
282 if (!(val & (0xfUL << ID_AA64MMFR1_LOR_SHIFT))) {
283 kvm_inject_undefined(vcpu);
284 return false;
285 }
286
287 if (p->is_write && sr == SYS_LORID_EL1)
288 return write_to_read_only(vcpu, p, r);
289
290 return trap_raz_wi(vcpu, p, r);
cc33c4e2
MR		 291}
292
0c557ed4 293static bool trap_oslsr_el1(struct kvm_vcpu *vcpu,
3fec037d 294 struct sys_reg_params *p,
0c557ed4
MZ		 295 const struct sys_reg_desc *r)
296{
297 if (p->is_write) {
298 return ignore_write(vcpu, p);
299 } else {
2ec5be3d 300 p->regval = (1 << 3);
0c557ed4
MZ		 301 return true;
302 }
303}
304
305static bool trap_dbgauthstatus_el1(struct kvm_vcpu *vcpu,
3fec037d 306 struct sys_reg_params *p,
0c557ed4
MZ		 307 const struct sys_reg_desc *r)
308{
309 if (p->is_write) {
310 return ignore_write(vcpu, p);
311 } else {
1f3d8699 312 p->regval = read_sysreg(dbgauthstatus_el1);
0c557ed4
MZ		 313 return true;
314 }
315}
316
317/*
318 * We want to avoid world-switching all the DBG registers all the
319 * time:
320 *
321 * - If we've touched any debug register, it is likely that we're
322 * going to touch more of them. It then makes sense to disable the
323 * traps and start doing the save/restore dance
324 * - If debug is active (DBG_MDSCR_KDE or DBG_MDSCR_MDE set), it is
325 * then mandatory to save/restore the registers, as the guest
326 * depends on them.
327 *
328 * For this, we use a DIRTY bit, indicating the guest has modified the
329 * debug registers, used as follow:
330 *
331 * On guest entry:
332 * - If the dirty bit is set (because we're coming back from trapping),
333 * disable the traps, save host registers, restore guest registers.
334 * - If debug is actively in use (DBG_MDSCR_KDE or DBG_MDSCR_MDE set),
335 * set the dirty bit, disable the traps, save host registers,
336 * restore guest registers.
337 * - Otherwise, enable the traps
338 *
339 * On guest exit:
340 * - If the dirty bit is set, save guest registers, restore host
341 * registers and clear the dirty bit. This ensure that the host can
342 * now use the debug registers.
343 */
344static bool trap_debug_regs(struct kvm_vcpu *vcpu,
3fec037d 345 struct sys_reg_params *p,
0c557ed4
MZ		 346 const struct sys_reg_desc *r)
347{
348 if (p->is_write) {
8d404c4c 349 vcpu_write_sys_reg(vcpu, p->regval, r->reg);
fa89d31c 350 vcpu->arch.flags |= KVM_ARM64_DEBUG_DIRTY;
0c557ed4 351 } else {
8d404c4c 352 p->regval = vcpu_read_sys_reg(vcpu, r->reg);
0c557ed4
MZ		 353 }
354
2ec5be3d 355 trace_trap_reg(__func__, r->reg, p->is_write, p->regval);
eef8c85a 356
0c557ed4
MZ		 357 return true;
358}
359
84e690bf
AB		 360/*
361 * reg_to_dbg/dbg_to_reg
362 *
363 * A 32 bit write to a debug register leave top bits alone
364 * A 32 bit read from a debug register only returns the bottom bits
365 *
366 * All writes will set the KVM_ARM64_DEBUG_DIRTY flag to ensure the
367 * hyp.S code switches between host and guest values in future.
368 */
281243cb
MZ		 369static void reg_to_dbg(struct kvm_vcpu *vcpu,
370 struct sys_reg_params *p,
1da42c34 371 const struct sys_reg_desc *rd,
281243cb 372 u64 *dbg_reg)
84e690bf 373{
1da42c34 374 u64 mask, shift, val;
84e690bf 375
1da42c34 376 get_access_mask(rd, &mask, &shift);
84e690bf 377
1da42c34
MZ		 378 val = *dbg_reg;
379 val &= ~mask;
380 val |= (p->regval & (mask >> shift)) << shift;
84e690bf 381 *dbg_reg = val;
1da42c34 382
fa89d31c 383 vcpu->arch.flags |= KVM_ARM64_DEBUG_DIRTY;
84e690bf
AB		 384}
385
281243cb
MZ		 386static void dbg_to_reg(struct kvm_vcpu *vcpu,
387 struct sys_reg_params *p,
1da42c34 388 const struct sys_reg_desc *rd,
281243cb 389 u64 *dbg_reg)
84e690bf 390{
1da42c34
MZ		 391 u64 mask, shift;
392
393 get_access_mask(rd, &mask, &shift);
394 p->regval = (*dbg_reg & mask) >> shift;
84e690bf
AB		 395}
396
281243cb
MZ		 397static bool trap_bvr(struct kvm_vcpu *vcpu,
398 struct sys_reg_params *p,
399 const struct sys_reg_desc *rd)
84e690bf
AB		 400{
401 u64 *dbg_reg = &vcpu->arch.vcpu_debug_state.dbg_bvr[rd->reg];
402
403 if (p->is_write)
1da42c34 404 reg_to_dbg(vcpu, p, rd, dbg_reg);
84e690bf 405 else
1da42c34 406 dbg_to_reg(vcpu, p, rd, dbg_reg);
84e690bf 407
eef8c85a
AB		 408 trace_trap_reg(__func__, rd->reg, p->is_write, *dbg_reg);
409
84e690bf
AB		 410 return true;
411}
412
413static int set_bvr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
414 const struct kvm_one_reg *reg, void __user *uaddr)
415{
416 __u64 *r = &vcpu->arch.vcpu_debug_state.dbg_bvr[rd->reg];
417
1713e5aa 418 if (copy_from_user(r, uaddr, KVM_REG_SIZE(reg->id)) != 0)
84e690bf
AB		 419 return -EFAULT;
420 return 0;
421}
422
423static int get_bvr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
424 const struct kvm_one_reg *reg, void __user *uaddr)
425{
426 __u64 *r = &vcpu->arch.vcpu_debug_state.dbg_bvr[rd->reg];
427
428 if (copy_to_user(uaddr, r, KVM_REG_SIZE(reg->id)) != 0)
429 return -EFAULT;
430 return 0;
431}
432
281243cb
MZ		 433static void reset_bvr(struct kvm_vcpu *vcpu,
434 const struct sys_reg_desc *rd)
84e690bf
AB		 435{
436 vcpu->arch.vcpu_debug_state.dbg_bvr[rd->reg] = rd->val;
437}
438
281243cb
MZ		 439static bool trap_bcr(struct kvm_vcpu *vcpu,
440 struct sys_reg_params *p,
441 const struct sys_reg_desc *rd)
84e690bf
AB		 442{
443 u64 *dbg_reg = &vcpu->arch.vcpu_debug_state.dbg_bcr[rd->reg];
444
445 if (p->is_write)
1da42c34 446 reg_to_dbg(vcpu, p, rd, dbg_reg);
84e690bf 447 else
1da42c34 448 dbg_to_reg(vcpu, p, rd, dbg_reg);
84e690bf 449
eef8c85a
AB		 450 trace_trap_reg(__func__, rd->reg, p->is_write, *dbg_reg);
451
84e690bf
AB		 452 return true;
453}
454
455static int set_bcr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
456 const struct kvm_one_reg *reg, void __user *uaddr)
457{
458 __u64 *r = &vcpu->arch.vcpu_debug_state.dbg_bcr[rd->reg];
459
1713e5aa 460 if (copy_from_user(r, uaddr, KVM_REG_SIZE(reg->id)) != 0)
84e690bf
AB		 461 return -EFAULT;
462
463 return 0;
464}
465
466static int get_bcr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
467 const struct kvm_one_reg *reg, void __user *uaddr)
468{
469 __u64 *r = &vcpu->arch.vcpu_debug_state.dbg_bcr[rd->reg];
470
471 if (copy_to_user(uaddr, r, KVM_REG_SIZE(reg->id)) != 0)
472 return -EFAULT;
473 return 0;
474}
475
281243cb
MZ		 476static void reset_bcr(struct kvm_vcpu *vcpu,
477 const struct sys_reg_desc *rd)
84e690bf
AB		 478{
479 vcpu->arch.vcpu_debug_state.dbg_bcr[rd->reg] = rd->val;
480}
481
281243cb
MZ		 482static bool trap_wvr(struct kvm_vcpu *vcpu,
483 struct sys_reg_params *p,
484 const struct sys_reg_desc *rd)
84e690bf
AB		 485{
486 u64 *dbg_reg = &vcpu->arch.vcpu_debug_state.dbg_wvr[rd->reg];
487
488 if (p->is_write)
1da42c34 489 reg_to_dbg(vcpu, p, rd, dbg_reg);
84e690bf 490 else
1da42c34 491 dbg_to_reg(vcpu, p, rd, dbg_reg);
84e690bf 492
eef8c85a
AB		 493 trace_trap_reg(__func__, rd->reg, p->is_write,
494 vcpu->arch.vcpu_debug_state.dbg_wvr[rd->reg]);
495
84e690bf
AB		 496 return true;
497}
498
499static int set_wvr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
500 const struct kvm_one_reg *reg, void __user *uaddr)
501{
502 __u64 *r = &vcpu->arch.vcpu_debug_state.dbg_wvr[rd->reg];
503
1713e5aa 504 if (copy_from_user(r, uaddr, KVM_REG_SIZE(reg->id)) != 0)
84e690bf
AB		 505 return -EFAULT;
506 return 0;
507}
508
509static int get_wvr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
510 const struct kvm_one_reg *reg, void __user *uaddr)
511{
512 __u64 *r = &vcpu->arch.vcpu_debug_state.dbg_wvr[rd->reg];
513
514 if (copy_to_user(uaddr, r, KVM_REG_SIZE(reg->id)) != 0)
515 return -EFAULT;
516 return 0;
517}
518
281243cb
MZ		 519static void reset_wvr(struct kvm_vcpu *vcpu,
520 const struct sys_reg_desc *rd)
84e690bf
AB		 521{
522 vcpu->arch.vcpu_debug_state.dbg_wvr[rd->reg] = rd->val;
523}
524
281243cb
MZ		 525static bool trap_wcr(struct kvm_vcpu *vcpu,
526 struct sys_reg_params *p,
527 const struct sys_reg_desc *rd)
84e690bf
AB		 528{
529 u64 *dbg_reg = &vcpu->arch.vcpu_debug_state.dbg_wcr[rd->reg];
530
531 if (p->is_write)
1da42c34 532 reg_to_dbg(vcpu, p, rd, dbg_reg);
84e690bf 533 else
1da42c34 534 dbg_to_reg(vcpu, p, rd, dbg_reg);
84e690bf 535
eef8c85a
AB		 536 trace_trap_reg(__func__, rd->reg, p->is_write, *dbg_reg);
537
84e690bf
AB		 538 return true;
539}
540
541static int set_wcr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
542 const struct kvm_one_reg *reg, void __user *uaddr)
543{
544 __u64 *r = &vcpu->arch.vcpu_debug_state.dbg_wcr[rd->reg];
545
1713e5aa 546 if (copy_from_user(r, uaddr, KVM_REG_SIZE(reg->id)) != 0)
84e690bf
AB		 547 return -EFAULT;
548 return 0;
549}
550
551static int get_wcr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
552 const struct kvm_one_reg *reg, void __user *uaddr)
553{
554 __u64 *r = &vcpu->arch.vcpu_debug_state.dbg_wcr[rd->reg];
555
556 if (copy_to_user(uaddr, r, KVM_REG_SIZE(reg->id)) != 0)
557 return -EFAULT;
558 return 0;
559}
560
281243cb
MZ		 561static void reset_wcr(struct kvm_vcpu *vcpu,
562 const struct sys_reg_desc *rd)
84e690bf
AB		 563{
564 vcpu->arch.vcpu_debug_state.dbg_wcr[rd->reg] = rd->val;
565}
566
7c8c5e6a
MZ		 567static void reset_amair_el1(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r)
568{
8d404c4c
CD		 569 u64 amair = read_sysreg(amair_el1);
570 vcpu_write_sys_reg(vcpu, amair, AMAIR_EL1);
7c8c5e6a
MZ		 571}
572
af473829
JM		 573static void reset_actlr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r)
574{
575 u64 actlr = read_sysreg(actlr_el1);
576 vcpu_write_sys_reg(vcpu, actlr, ACTLR_EL1);
577}
578
7c8c5e6a
MZ		 579static void reset_mpidr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r)
580{
4429fc64
AP		 581 u64 mpidr;
582
7c8c5e6a 583 /*
4429fc64
AP		 584 * Map the vcpu_id into the first three affinity level fields of
585 * the MPIDR. We limit the number of VCPUs in level 0 due to a
586 * limitation to 16 CPUs in that level in the ICC_SGIxR registers
587 * of the GICv3 to be able to address each CPU directly when
588 * sending IPIs.
7c8c5e6a 589 */
4429fc64
AP		 590 mpidr = (vcpu->vcpu_id & 0x0f) << MPIDR_LEVEL_SHIFT(0);
591 mpidr |= ((vcpu->vcpu_id >> 4) & 0xff) << MPIDR_LEVEL_SHIFT(1);
592 mpidr |= ((vcpu->vcpu_id >> 12) & 0xff) << MPIDR_LEVEL_SHIFT(2);
8d404c4c 593 vcpu_write_sys_reg(vcpu, (1ULL << 31) | mpidr, MPIDR_EL1);
7c8c5e6a
MZ		 594}
595
11663111
MZ		 596static unsigned int pmu_visibility(const struct kvm_vcpu *vcpu,
597 const struct sys_reg_desc *r)
598{
599 if (kvm_vcpu_has_pmu(vcpu))
600 return 0;
601
602 return REG_HIDDEN;
603}
604
ab946834
SZ		 605static void reset_pmcr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r)
606{
607 u64 pmcr, val;
608
2a5f1b67
MZ		 609 /* No PMU available, PMCR_EL0 may UNDEF... */
610 if (!kvm_arm_support_pmu_v3())
611 return;
612
1f3d8699
MR		 613 pmcr = read_sysreg(pmcr_el0);
614 /*
615 * Writable bits of PMCR_EL0 (ARMV8_PMU_PMCR_MASK) are reset to UNKNOWN
ab946834
SZ		 616 * except PMCR.E resetting to zero.
617 */
618 val = ((pmcr & ~ARMV8_PMU_PMCR_MASK)
619 | (ARMV8_PMU_PMCR_MASK & 0xdecafbad)) & (~ARMV8_PMU_PMCR_E);
6f163714
MZ		 620 if (!system_supports_32bit_el0())
621 val |= ARMV8_PMU_PMCR_LC;
03fdfb26 622 __vcpu_sys_reg(vcpu, r->reg) = val;
ab946834
SZ		 623}
624
6c007036 625static bool check_pmu_access_disabled(struct kvm_vcpu *vcpu, u64 flags)
d692b8ad 626{
8d404c4c 627 u64 reg = __vcpu_sys_reg(vcpu, PMUSERENR_EL0);
7ded92e2 628 bool enabled = (reg & flags) || vcpu_mode_priv(vcpu);
d692b8ad 629
24d5950f
MZ		 630 if (!enabled)
631 kvm_inject_undefined(vcpu);
d692b8ad 632
6c007036 633 return !enabled;
d692b8ad
SZ		 634}
635
6c007036 636static bool pmu_access_el0_disabled(struct kvm_vcpu *vcpu)
d692b8ad 637{
6c007036
MZ		 638 return check_pmu_access_disabled(vcpu, ARMV8_PMU_USERENR_EN);
639}
d692b8ad 640
6c007036
MZ		 641static bool pmu_write_swinc_el0_disabled(struct kvm_vcpu *vcpu)
642{
643 return check_pmu_access_disabled(vcpu, ARMV8_PMU_USERENR_SW | ARMV8_PMU_USERENR_EN);
d692b8ad
SZ		 644}
645
646static bool pmu_access_cycle_counter_el0_disabled(struct kvm_vcpu *vcpu)
647{
6c007036 648 return check_pmu_access_disabled(vcpu, ARMV8_PMU_USERENR_CR | ARMV8_PMU_USERENR_EN);
d692b8ad
SZ		 649}
650
651static bool pmu_access_event_counter_el0_disabled(struct kvm_vcpu *vcpu)
652{
6c007036 653 return check_pmu_access_disabled(vcpu, ARMV8_PMU_USERENR_ER | ARMV8_PMU_USERENR_EN);
d692b8ad
SZ		 654}
655
ab946834
SZ		 656static bool access_pmcr(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
657 const struct sys_reg_desc *r)
658{
659 u64 val;
660
d692b8ad
SZ		 661 if (pmu_access_el0_disabled(vcpu))
662 return false;
663
ab946834
SZ		 664 if (p->is_write) {
665 /* Only update writeable bits of PMCR */
8d404c4c 666 val = __vcpu_sys_reg(vcpu, PMCR_EL0);
ab946834
SZ		 667 val &= ~ARMV8_PMU_PMCR_MASK;
668 val |= p->regval & ARMV8_PMU_PMCR_MASK;
6f163714
MZ		 669 if (!system_supports_32bit_el0())
670 val |= ARMV8_PMU_PMCR_LC;
8d404c4c 671 __vcpu_sys_reg(vcpu, PMCR_EL0) = val;
76993739 672 kvm_pmu_handle_pmcr(vcpu, val);
435e53fb 673 kvm_vcpu_pmu_restore_guest(vcpu);
ab946834
SZ		 674 } else {
675 /* PMCR.P & PMCR.C are RAZ */
8d404c4c 676 val = __vcpu_sys_reg(vcpu, PMCR_EL0)
ab946834
SZ		 677 & ~(ARMV8_PMU_PMCR_P | ARMV8_PMU_PMCR_C);
678 p->regval = val;
679 }
680
681 return true;
682}
683
3965c3ce
SZ		 684static bool access_pmselr(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
685 const struct sys_reg_desc *r)
686{
d692b8ad
SZ		 687 if (pmu_access_event_counter_el0_disabled(vcpu))
688 return false;
689
3965c3ce 690 if (p->is_write)
8d404c4c 691 __vcpu_sys_reg(vcpu, PMSELR_EL0) = p->regval;
3965c3ce
SZ		 692 else
693 /* return PMSELR.SEL field */
8d404c4c 694 p->regval = __vcpu_sys_reg(vcpu, PMSELR_EL0)
3965c3ce
SZ		 695 & ARMV8_PMU_COUNTER_MASK;
696
697 return true;
698}
699
a86b5505
SZ		 700static bool access_pmceid(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
701 const struct sys_reg_desc *r)
702{
703 u64 pmceid;
704
a86b5505
SZ		 705 BUG_ON(p->is_write);
706
d692b8ad
SZ		 707 if (pmu_access_el0_disabled(vcpu))
708 return false;
709
88865bec 710 pmceid = kvm_pmu_get_pmceid(vcpu, (p->Op2 & 1));
a86b5505
SZ		 711
712 p->regval = pmceid;
713
714 return true;
715}
716
051ff581
SZ		 717static bool pmu_counter_idx_valid(struct kvm_vcpu *vcpu, u64 idx)
718{
719 u64 pmcr, val;
720
8d404c4c 721 pmcr = __vcpu_sys_reg(vcpu, PMCR_EL0);
051ff581 722 val = (pmcr >> ARMV8_PMU_PMCR_N_SHIFT) & ARMV8_PMU_PMCR_N_MASK;
24d5950f
MZ		 723 if (idx >= val && idx != ARMV8_PMU_CYCLE_IDX) {
724 kvm_inject_undefined(vcpu);
051ff581 725 return false;
24d5950f 726 }
051ff581
SZ		 727
728 return true;
729}
730
731static bool access_pmu_evcntr(struct kvm_vcpu *vcpu,
732 struct sys_reg_params *p,
733 const struct sys_reg_desc *r)
734{
a3da9358 735 u64 idx = ~0UL;
051ff581
SZ		 736
737 if (r->CRn == 9 && r->CRm == 13) {
738 if (r->Op2 == 2) {
739 /* PMXEVCNTR_EL0 */
d692b8ad
SZ		 740 if (pmu_access_event_counter_el0_disabled(vcpu))
741 return false;
742
8d404c4c 743 idx = __vcpu_sys_reg(vcpu, PMSELR_EL0)
051ff581
SZ		 744 & ARMV8_PMU_COUNTER_MASK;
745 } else if (r->Op2 == 0) {
746 /* PMCCNTR_EL0 */
d692b8ad
SZ		 747 if (pmu_access_cycle_counter_el0_disabled(vcpu))
748 return false;
749
051ff581 750 idx = ARMV8_PMU_CYCLE_IDX;
051ff581 751 }
9e3f7a29
WH		 752 } else if (r->CRn == 0 && r->CRm == 9) {
753 /* PMCCNTR */
754 if (pmu_access_event_counter_el0_disabled(vcpu))
755 return false;
756
757 idx = ARMV8_PMU_CYCLE_IDX;
051ff581
SZ		 758 } else if (r->CRn == 14 && (r->CRm & 12) == 8) {
759 /* PMEVCNTRn_EL0 */
d692b8ad
SZ		 760 if (pmu_access_event_counter_el0_disabled(vcpu))
761 return false;
762
051ff581 763 idx = ((r->CRm & 3) << 3) | (r->Op2 & 7);
051ff581
SZ		 764 }
765
a3da9358
MZ		 766 /* Catch any decoding mistake */
767 WARN_ON(idx == ~0UL);
768
051ff581
SZ		 769 if (!pmu_counter_idx_valid(vcpu, idx))
770 return false;
771
d692b8ad
SZ		 772 if (p->is_write) {
773 if (pmu_access_el0_disabled(vcpu))
774 return false;
775
051ff581 776 kvm_pmu_set_counter_value(vcpu, idx, p->regval);
d692b8ad 777 } else {
051ff581 778 p->regval = kvm_pmu_get_counter_value(vcpu, idx);
d692b8ad 779 }
051ff581
SZ		 780
781 return true;
782}
783
9feb21ac
SZ		 784static bool access_pmu_evtyper(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
785 const struct sys_reg_desc *r)
786{
787 u64 idx, reg;
788
d692b8ad
SZ		 789 if (pmu_access_el0_disabled(vcpu))
790 return false;
791
9feb21ac
SZ		 792 if (r->CRn == 9 && r->CRm == 13 && r->Op2 == 1) {
793 /* PMXEVTYPER_EL0 */
8d404c4c 794 idx = __vcpu_sys_reg(vcpu, PMSELR_EL0) & ARMV8_PMU_COUNTER_MASK;
9feb21ac
SZ		 795 reg = PMEVTYPER0_EL0 + idx;
796 } else if (r->CRn == 14 && (r->CRm & 12) == 12) {
797 idx = ((r->CRm & 3) << 3) | (r->Op2 & 7);
798 if (idx == ARMV8_PMU_CYCLE_IDX)
799 reg = PMCCFILTR_EL0;
800 else
801 /* PMEVTYPERn_EL0 */
802 reg = PMEVTYPER0_EL0 + idx;
803 } else {
804 BUG();
805 }
806
807 if (!pmu_counter_idx_valid(vcpu, idx))
808 return false;
809
810 if (p->is_write) {
811 kvm_pmu_set_counter_event_type(vcpu, p->regval, idx);
8d404c4c 812 __vcpu_sys_reg(vcpu, reg) = p->regval & ARMV8_PMU_EVTYPE_MASK;
435e53fb 813 kvm_vcpu_pmu_restore_guest(vcpu);
9feb21ac 814 } else {
8d404c4c 815 p->regval = __vcpu_sys_reg(vcpu, reg) & ARMV8_PMU_EVTYPE_MASK;
9feb21ac
SZ		 816 }
817
818 return true;
819}
820
96b0eebc
SZ		 821static bool access_pmcnten(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
822 const struct sys_reg_desc *r)
823{
824 u64 val, mask;
825
d692b8ad
SZ		 826 if (pmu_access_el0_disabled(vcpu))
827 return false;
828
96b0eebc
SZ		 829 mask = kvm_pmu_valid_counter_mask(vcpu);
830 if (p->is_write) {
831 val = p->regval & mask;
832 if (r->Op2 & 0x1) {
833 /* accessing PMCNTENSET_EL0 */
8d404c4c 834 __vcpu_sys_reg(vcpu, PMCNTENSET_EL0) |= val;
418e5ca8 835 kvm_pmu_enable_counter_mask(vcpu, val);
435e53fb 836 kvm_vcpu_pmu_restore_guest(vcpu);
96b0eebc
SZ		 837 } else {
838 /* accessing PMCNTENCLR_EL0 */
8d404c4c 839 __vcpu_sys_reg(vcpu, PMCNTENSET_EL0) &= ~val;
418e5ca8 840 kvm_pmu_disable_counter_mask(vcpu, val);
96b0eebc
SZ		 841 }
842 } else {
8d404c4c 843 p->regval = __vcpu_sys_reg(vcpu, PMCNTENSET_EL0) & mask;
96b0eebc
SZ		 844 }
845
846 return true;
847}
848
9db52c78
SZ		 849static bool access_pminten(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
850 const struct sys_reg_desc *r)
851{
852 u64 mask = kvm_pmu_valid_counter_mask(vcpu);
853
b0737e99 854 if (check_pmu_access_disabled(vcpu, 0))
d692b8ad
SZ		 855 return false;
856
9db52c78
SZ		 857 if (p->is_write) {
858 u64 val = p->regval & mask;
859
860 if (r->Op2 & 0x1)
861 /* accessing PMINTENSET_EL1 */
8d404c4c 862 __vcpu_sys_reg(vcpu, PMINTENSET_EL1) |= val;
9db52c78
SZ		 863 else
864 /* accessing PMINTENCLR_EL1 */
8d404c4c 865 __vcpu_sys_reg(vcpu, PMINTENSET_EL1) &= ~val;
9db52c78 866 } else {
8d404c4c 867 p->regval = __vcpu_sys_reg(vcpu, PMINTENSET_EL1) & mask;
9db52c78
SZ		 868 }
869
870 return true;
871}
872
76d883c4
SZ		 873static bool access_pmovs(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
874 const struct sys_reg_desc *r)
875{
876 u64 mask = kvm_pmu_valid_counter_mask(vcpu);
877
d692b8ad
SZ		 878 if (pmu_access_el0_disabled(vcpu))
879 return false;
880
76d883c4
SZ		 881 if (p->is_write) {
882 if (r->CRm & 0x2)
883 /* accessing PMOVSSET_EL0 */
8d404c4c 884 __vcpu_sys_reg(vcpu, PMOVSSET_EL0) |= (p->regval & mask);
76d883c4
SZ		 885 else
886 /* accessing PMOVSCLR_EL0 */
8d404c4c 887 __vcpu_sys_reg(vcpu, PMOVSSET_EL0) &= ~(p->regval & mask);
76d883c4 888 } else {
8d404c4c 889 p->regval = __vcpu_sys_reg(vcpu, PMOVSSET_EL0) & mask;
76d883c4
SZ		 890 }
891
892 return true;
893}
894
7a0adc70
SZ		 895static bool access_pmswinc(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
896 const struct sys_reg_desc *r)
897{
898 u64 mask;
899
e0443230 900 if (!p->is_write)
e7f1d1ee 901 return read_from_write_only(vcpu, p, r);
e0443230 902
d692b8ad
SZ		 903 if (pmu_write_swinc_el0_disabled(vcpu))
904 return false;
905
e0443230
MZ		 906 mask = kvm_pmu_valid_counter_mask(vcpu);
907 kvm_pmu_software_increment(vcpu, p->regval & mask);
908 return true;
7a0adc70
SZ		 909}
910
d692b8ad
SZ		 911static bool access_pmuserenr(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
912 const struct sys_reg_desc *r)
913{
d692b8ad 914 if (p->is_write) {
9008c235
MZ		 915 if (!vcpu_mode_priv(vcpu)) {
916 kvm_inject_undefined(vcpu);
d692b8ad 917 return false;
9008c235 918 }
d692b8ad 919
8d404c4c
CD		 920 __vcpu_sys_reg(vcpu, PMUSERENR_EL0) =
921 p->regval & ARMV8_PMU_USERENR_MASK;
d692b8ad 922 } else {
8d404c4c 923 p->regval = __vcpu_sys_reg(vcpu, PMUSERENR_EL0)
d692b8ad
SZ		 924 & ARMV8_PMU_USERENR_MASK;
925 }
926
927 return true;
928}
929
0c557ed4
MZ		 930/* Silly macro to expand the DBG{BCR,BVR,WVR,WCR}n_EL1 registers in one go */
931#define DBG_BCR_BVR_WCR_WVR_EL1(n) \
ee1b64e6 932 { SYS_DESC(SYS_DBGBVRn_EL1(n)), \
03fdfb26 933 trap_bvr, reset_bvr, 0, 0, get_bvr, set_bvr }, \
ee1b64e6 934 { SYS_DESC(SYS_DBGBCRn_EL1(n)), \
03fdfb26 935 trap_bcr, reset_bcr, 0, 0, get_bcr, set_bcr }, \
ee1b64e6 936 { SYS_DESC(SYS_DBGWVRn_EL1(n)), \
03fdfb26 937 trap_wvr, reset_wvr, 0, 0, get_wvr, set_wvr }, \
ee1b64e6 938 { SYS_DESC(SYS_DBGWCRn_EL1(n)), \
03fdfb26 939 trap_wcr, reset_wcr, 0, 0, get_wcr, set_wcr }
0c557ed4 940
11663111
MZ		 941#define PMU_SYS_REG(r) \
942 SYS_DESC(r), .reset = reset_unknown, .visibility = pmu_visibility
943
051ff581
SZ		 944/* Macro to expand the PMEVCNTRn_EL0 register */
945#define PMU_PMEVCNTR_EL0(n) \
11663111
MZ		 946 { PMU_SYS_REG(SYS_PMEVCNTRn_EL0(n)), \
947 .access = access_pmu_evcntr, .reg = (PMEVCNTR0_EL0 + n), }
051ff581 948
9feb21ac
SZ		 949/* Macro to expand the PMEVTYPERn_EL0 register */
950#define PMU_PMEVTYPER_EL0(n) \
11663111
MZ		 951 { PMU_SYS_REG(SYS_PMEVTYPERn_EL0(n)), \
952 .access = access_pmu_evtyper, .reg = (PMEVTYPER0_EL0 + n), }
9feb21ac 953
338b1793
MZ		 954static bool undef_access(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
955 const struct sys_reg_desc *r)
4fcdf106
IV		 956{
957 kvm_inject_undefined(vcpu);
958
959 return false;
960}
961
962/* Macro to expand the AMU counter and type registers*/
338b1793
MZ		 963#define AMU_AMEVCNTR0_EL0(n) { SYS_DESC(SYS_AMEVCNTR0_EL0(n)), undef_access }
964#define AMU_AMEVTYPER0_EL0(n) { SYS_DESC(SYS_AMEVTYPER0_EL0(n)), undef_access }
965#define AMU_AMEVCNTR1_EL0(n) { SYS_DESC(SYS_AMEVCNTR1_EL0(n)), undef_access }
966#define AMU_AMEVTYPER1_EL0(n) { SYS_DESC(SYS_AMEVTYPER1_EL0(n)), undef_access }
384b40ca
MR		 967
968static unsigned int ptrauth_visibility(const struct kvm_vcpu *vcpu,
969 const struct sys_reg_desc *rd)
970{
01fe5ace 971 return vcpu_has_ptrauth(vcpu) ? 0 : REG_HIDDEN;
384b40ca
MR		 972}
973
338b1793
MZ		 974/*
975 * If we land here on a PtrAuth access, that is because we didn't
976 * fixup the access on exit by allowing the PtrAuth sysregs. The only
977 * way this happens is when the guest does not have PtrAuth support
978 * enabled.
979 */
384b40ca 980#define __PTRAUTH_KEY(k) \
338b1793 981 { SYS_DESC(SYS_## k), undef_access, reset_unknown, k, \
384b40ca
MR		 982 .visibility = ptrauth_visibility}
983
984#define PTRAUTH_KEY(k) \
985 __PTRAUTH_KEY(k ## KEYLO_EL1), \
986 __PTRAUTH_KEY(k ## KEYHI_EL1)
987
84135d3d
AP		 988static bool access_arch_timer(struct kvm_vcpu *vcpu,
989 struct sys_reg_params *p,
990 const struct sys_reg_desc *r)
c9a3c58f 991{
84135d3d
AP		 992 enum kvm_arch_timers tmr;
993 enum kvm_arch_timer_regs treg;
994 u64 reg = reg_to_encoding(r);
7b6b4631 995
84135d3d
AP		 996 switch (reg) {
997 case SYS_CNTP_TVAL_EL0:
998 case SYS_AARCH32_CNTP_TVAL:
999 tmr = TIMER_PTIMER;
1000 treg = TIMER_REG_TVAL;
1001 break;
1002 case SYS_CNTP_CTL_EL0:
1003 case SYS_AARCH32_CNTP_CTL:
1004 tmr = TIMER_PTIMER;
1005 treg = TIMER_REG_CTL;
1006 break;
1007 case SYS_CNTP_CVAL_EL0:
1008 case SYS_AARCH32_CNTP_CVAL:
1009 tmr = TIMER_PTIMER;
1010 treg = TIMER_REG_CVAL;
1011 break;
1012 default:
1013 BUG();
c1b135af 1014 }
7b6b4631 1015
7b6b4631 1016 if (p->is_write)
84135d3d 1017 kvm_arm_timer_write_sysreg(vcpu, tmr, treg, p->regval);
7b6b4631 1018 else
84135d3d 1019 p->regval = kvm_arm_timer_read_sysreg(vcpu, tmr, treg);
7b6b4631 1020
c9a3c58f
JL		 1021 return true;
1022}
1023
93390c0a 1024/* Read a sanitised cpufeature ID register by sys_reg_desc */
1c199913
DM		 1025static u64 read_id_reg(const struct kvm_vcpu *vcpu,
1026 struct sys_reg_desc const *r, bool raz)
93390c0a 1027{
7ba8b438 1028 u32 id = reg_to_encoding(r);
07d79fe7 1029 u64 val = raz ? 0 : read_sanitised_ftr_reg(id);
93390c0a 1030
4fcdf106
IV		 1031 if (id == SYS_ID_AA64PFR0_EL1) {
1032 if (!vcpu_has_sve(vcpu))
1033 val &= ~(0xfUL << ID_AA64PFR0_SVE_SHIFT);
1034 val &= ~(0xfUL << ID_AA64PFR0_AMU_SHIFT);
23711a5e
MZ		 1035 val &= ~(0xfUL << ID_AA64PFR0_CSV2_SHIFT);
1036 val |= ((u64)vcpu->kvm->arch.pfr0_csv2 << ID_AA64PFR0_CSV2_SHIFT);
4f1df628
MZ		 1037 val &= ~(0xfUL << ID_AA64PFR0_CSV3_SHIFT);
1038 val |= ((u64)vcpu->kvm->arch.pfr0_csv3 << ID_AA64PFR0_CSV3_SHIFT);
2ac638fc
CM		 1039 } else if (id == SYS_ID_AA64PFR1_EL1) {
1040 val &= ~(0xfUL << ID_AA64PFR1_MTE_SHIFT);
384b40ca 1041 } else if (id == SYS_ID_AA64ISAR1_EL1 && !vcpu_has_ptrauth(vcpu)) {
9eecfc22
KM		 1042 val &= ~((0xfUL << ID_AA64ISAR1_APA_SHIFT) |
1043 (0xfUL << ID_AA64ISAR1_API_SHIFT) |
1044 (0xfUL << ID_AA64ISAR1_GPA_SHIFT) |
1045 (0xfUL << ID_AA64ISAR1_GPI_SHIFT));
c854188e 1046 } else if (id == SYS_ID_AA64DFR0_EL1) {
04355e41
MZ		 1047 u64 cap = 0;
1048
c854188e 1049 /* Limit guests to PMUv3 for ARMv8.1 */
04355e41
MZ		 1050 if (kvm_vcpu_has_pmu(vcpu))
1051 cap = ID_AA64DFR0_PMUVER_8_1;
1052
c854188e
AM		 1053 val = cpuid_feature_cap_perfmon_field(val,
1054 ID_AA64DFR0_PMUVER_SHIFT,
04355e41 1055 cap);
c854188e
AM		 1056 } else if (id == SYS_ID_DFR0_EL1) {
1057 /* Limit guests to PMUv3 for ARMv8.1 */
1058 val = cpuid_feature_cap_perfmon_field(val,
1059 ID_DFR0_PERFMON_SHIFT,
1060 ID_DFR0_PERFMON_8_1);
07d79fe7
DM		 1061 }
1062
1063 return val;
93390c0a
DM		 1064}
1065
912dee57
AJ		 1066static unsigned int id_visibility(const struct kvm_vcpu *vcpu,
1067 const struct sys_reg_desc *r)
1068{
7ba8b438 1069 u32 id = reg_to_encoding(r);
c512298e
AJ		 1070
1071 switch (id) {
1072 case SYS_ID_AA64ZFR0_EL1:
1073 if (!vcpu_has_sve(vcpu))
1074 return REG_RAZ;
1075 break;
1076 }
1077
912dee57
AJ		 1078 return 0;
1079}
1080
93390c0a
DM		 1081/* cpufeature ID register access trap handlers */
1082
1083static bool __access_id_reg(struct kvm_vcpu *vcpu,
1084 struct sys_reg_params *p,
1085 const struct sys_reg_desc *r,
1086 bool raz)
1087{
1088 if (p->is_write)
1089 return write_to_read_only(vcpu, p, r);
1090
1c199913 1091 p->regval = read_id_reg(vcpu, r, raz);
93390c0a
DM		 1092 return true;
1093}
1094
1095static bool access_id_reg(struct kvm_vcpu *vcpu,
1096 struct sys_reg_params *p,
1097 const struct sys_reg_desc *r)
1098{
912dee57
AJ		 1099 bool raz = sysreg_visible_as_raz(vcpu, r);
1100
1101 return __access_id_reg(vcpu, p, r, raz);
93390c0a
DM		 1102}
1103
1104static bool access_raz_id_reg(struct kvm_vcpu *vcpu,
1105 struct sys_reg_params *p,
1106 const struct sys_reg_desc *r)
1107{
1108 return __access_id_reg(vcpu, p, r, true);
1109}
1110
1111static int reg_from_user(u64 *val, const void __user *uaddr, u64 id);
1112static int reg_to_user(void __user *uaddr, const u64 *val, u64 id);
1113static u64 sys_reg_to_index(const struct sys_reg_desc *reg);
1114
73433762
DM		 1115/* Visibility overrides for SVE-specific control registers */
1116static unsigned int sve_visibility(const struct kvm_vcpu *vcpu,
1117 const struct sys_reg_desc *rd)
1118{
1119 if (vcpu_has_sve(vcpu))
1120 return 0;
1121
01fe5ace 1122 return REG_HIDDEN;
73433762
DM		 1123}
1124
23711a5e
MZ		 1125static int set_id_aa64pfr0_el1(struct kvm_vcpu *vcpu,
1126 const struct sys_reg_desc *rd,
1127 const struct kvm_one_reg *reg, void __user *uaddr)
1128{
1129 const u64 id = sys_reg_to_index(rd);
4f1df628 1130 u8 csv2, csv3;
23711a5e
MZ		 1131 int err;
1132 u64 val;
23711a5e
MZ		 1133
1134 err = reg_from_user(&val, uaddr, id);
1135 if (err)
1136 return err;
1137
1138 /*
1139 * Allow AA64PFR0_EL1.CSV2 to be set from userspace as long as
1140 * it doesn't promise more than what is actually provided (the
1141 * guest could otherwise be covered in ectoplasmic residue).
1142 */
1143 csv2 = cpuid_feature_extract_unsigned_field(val, ID_AA64PFR0_CSV2_SHIFT);
1144 if (csv2 > 1 ||
1145 (csv2 && arm64_get_spectre_v2_state() != SPECTRE_UNAFFECTED))
1146 return -EINVAL;
1147
4f1df628
MZ		 1148 /* Same thing for CSV3 */
1149 csv3 = cpuid_feature_extract_unsigned_field(val, ID_AA64PFR0_CSV3_SHIFT);
1150 if (csv3 > 1 ||
1151 (csv3 && arm64_get_meltdown_state() != SPECTRE_UNAFFECTED))
1152 return -EINVAL;
1153
1154 /* We can only differ with CSV[23], and anything else is an error */
23711a5e 1155 val ^= read_id_reg(vcpu, rd, false);
4f1df628
MZ		 1156 val &= ~((0xFUL << ID_AA64PFR0_CSV2_SHIFT) |
1157 (0xFUL << ID_AA64PFR0_CSV3_SHIFT));
23711a5e
MZ		 1158 if (val)
1159 return -EINVAL;
1160
1161 vcpu->kvm->arch.pfr0_csv2 = csv2;
4f1df628 1162 vcpu->kvm->arch.pfr0_csv3 = csv3 ;
23711a5e
MZ		 1163
1164 return 0;
1165}
1166
93390c0a
DM		 1167/*
1168 * cpufeature ID register user accessors
1169 *
1170 * For now, these registers are immutable for userspace, so no values
1171 * are stored, and for set_id_reg() we don't allow the effective value
1172 * to be changed.
1173 */
1c199913
DM		 1174static int __get_id_reg(const struct kvm_vcpu *vcpu,
1175 const struct sys_reg_desc *rd, void __user *uaddr,
93390c0a
DM		 1176 bool raz)
1177{
1178 const u64 id = sys_reg_to_index(rd);
1c199913 1179 const u64 val = read_id_reg(vcpu, rd, raz);
93390c0a
DM		 1180
1181 return reg_to_user(uaddr, &val, id);
1182}
1183
1c199913
DM		 1184static int __set_id_reg(const struct kvm_vcpu *vcpu,
1185 const struct sys_reg_desc *rd, void __user *uaddr,
93390c0a
DM		 1186 bool raz)
1187{
1188 const u64 id = sys_reg_to_index(rd);
1189 int err;
1190 u64 val;
1191
1192 err = reg_from_user(&val, uaddr, id);
1193 if (err)
1194 return err;
1195
1196 /* This is what we mean by invariant: you can't change it. */
1c199913 1197 if (val != read_id_reg(vcpu, rd, raz))
93390c0a
DM		 1198 return -EINVAL;
1199
1200 return 0;
1201}
1202
1203static int get_id_reg(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
1204 const struct kvm_one_reg *reg, void __user *uaddr)
1205{
912dee57
AJ		 1206 bool raz = sysreg_visible_as_raz(vcpu, rd);
1207
1208 return __get_id_reg(vcpu, rd, uaddr, raz);
93390c0a
DM		 1209}
1210
1211static int set_id_reg(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
1212 const struct kvm_one_reg *reg, void __user *uaddr)
1213{
912dee57
AJ		 1214 bool raz = sysreg_visible_as_raz(vcpu, rd);
1215
1216 return __set_id_reg(vcpu, rd, uaddr, raz);
93390c0a
DM		 1217}
1218
1219static int get_raz_id_reg(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
1220 const struct kvm_one_reg *reg, void __user *uaddr)
1221{
1c199913 1222 return __get_id_reg(vcpu, rd, uaddr, true);
93390c0a
DM		 1223}
1224
1225static int set_raz_id_reg(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
1226 const struct kvm_one_reg *reg, void __user *uaddr)
1227{
1c199913 1228 return __set_id_reg(vcpu, rd, uaddr, true);
93390c0a
DM		 1229}
1230
f7f2b15c
AB		 1231static bool access_ctr(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
1232 const struct sys_reg_desc *r)
1233{
1234 if (p->is_write)
1235 return write_to_read_only(vcpu, p, r);
1236
1237 p->regval = read_sanitised_ftr_reg(SYS_CTR_EL0);
1238 return true;
1239}
1240
1241static bool access_clidr(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
1242 const struct sys_reg_desc *r)
1243{
1244 if (p->is_write)
1245 return write_to_read_only(vcpu, p, r);
1246
1247 p->regval = read_sysreg(clidr_el1);
1248 return true;
1249}
1250
1251static bool access_csselr(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
1252 const struct sys_reg_desc *r)
1253{
7c582bf4
JM		 1254 int reg = r->reg;
1255
f7f2b15c 1256 if (p->is_write)
7c582bf4 1257 vcpu_write_sys_reg(vcpu, p->regval, reg);
f7f2b15c 1258 else
7c582bf4 1259 p->regval = vcpu_read_sys_reg(vcpu, reg);
f7f2b15c
AB		 1260 return true;
1261}
1262
1263static bool access_ccsidr(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
1264 const struct sys_reg_desc *r)
1265{
1266 u32 csselr;
1267
1268 if (p->is_write)
1269 return write_to_read_only(vcpu, p, r);
1270
1271 csselr = vcpu_read_sys_reg(vcpu, CSSELR_EL1);
1272 p->regval = get_ccsidr(csselr);
793acf87
AB		 1273
1274 /*
1275 * Guests should not be doing cache operations by set/way at all, and
1276 * for this reason, we trap them and attempt to infer the intent, so
1277 * that we can flush the entire guest's address space at the appropriate
1278 * time.
1279 * To prevent this trapping from causing performance problems, let's
1280 * expose the geometry of all data and unified caches (which are
1281 * guaranteed to be PIPT and thus non-aliasing) as 1 set and 1 way.
1282 * [If guests should attempt to infer aliasing properties from the
1283 * geometry (which is not permitted by the architecture), they would
1284 * only do so for virtually indexed caches.]
1285 */
1286 if (!(csselr & 1)) // data or unified cache
1287 p->regval &= ~GENMASK(27, 3);
f7f2b15c
AB		 1288 return true;
1289}
1290
93390c0a
DM		 1291/* sys_reg_desc initialiser for known cpufeature ID registers */
1292#define ID_SANITISED(name) { \
1293 SYS_DESC(SYS_##name), \
1294 .access = access_id_reg, \
1295 .get_user = get_id_reg, \
1296 .set_user = set_id_reg, \
912dee57 1297 .visibility = id_visibility, \
93390c0a
DM		 1298}
1299
1300/*
1301 * sys_reg_desc initialiser for architecturally unallocated cpufeature ID
1302 * register with encoding Op0=3, Op1=0, CRn=0, CRm=crm, Op2=op2
1303 * (1 <= crm < 8, 0 <= Op2 < 8).
1304 */
1305#define ID_UNALLOCATED(crm, op2) { \
1306 Op0(3), Op1(0), CRn(0), CRm(crm), Op2(op2), \
1307 .access = access_raz_id_reg, \
1308 .get_user = get_raz_id_reg, \
1309 .set_user = set_raz_id_reg, \
1310}
1311
1312/*
1313 * sys_reg_desc initialiser for known ID registers that we hide from guests.
1314 * For now, these are exposed just like unallocated ID regs: they appear
1315 * RAZ for the guest.
1316 */
1317#define ID_HIDDEN(name) { \
1318 SYS_DESC(SYS_##name), \
1319 .access = access_raz_id_reg, \
1320 .get_user = get_raz_id_reg, \
1321 .set_user = set_raz_id_reg, \
1322}
1323
7c8c5e6a
MZ		 1324/*
1325 * Architected system registers.
1326 * Important: Must be sorted ascending by Op0, Op1, CRn, CRm, Op2
7609c125 1327 *
0c557ed4
MZ		 1328 * Debug handling: We do trap most, if not all debug related system
1329 * registers. The implementation is good enough to ensure that a guest
1330 * can use these with minimal performance degradation. The drawback is
1331 * that we don't implement any of the external debug, none of the
1332 * OSlock protocol. This should be revisited if we ever encounter a
1333 * more demanding guest...
7c8c5e6a
MZ		 1334 */
1335static const struct sys_reg_desc sys_reg_descs[] = {
7606e078
MR		 1336 { SYS_DESC(SYS_DC_ISW), access_dcsw },
1337 { SYS_DESC(SYS_DC_CSW), access_dcsw },
1338 { SYS_DESC(SYS_DC_CISW), access_dcsw },
7c8c5e6a 1339
0c557ed4
MZ		 1340 DBG_BCR_BVR_WCR_WVR_EL1(0),
1341 DBG_BCR_BVR_WCR_WVR_EL1(1),
ee1b64e6
MR		 1342 { SYS_DESC(SYS_MDCCINT_EL1), trap_debug_regs, reset_val, MDCCINT_EL1, 0 },
1343 { SYS_DESC(SYS_MDSCR_EL1), trap_debug_regs, reset_val, MDSCR_EL1, 0 },
0c557ed4
MZ		 1344 DBG_BCR_BVR_WCR_WVR_EL1(2),
1345 DBG_BCR_BVR_WCR_WVR_EL1(3),
1346 DBG_BCR_BVR_WCR_WVR_EL1(4),
1347 DBG_BCR_BVR_WCR_WVR_EL1(5),
1348 DBG_BCR_BVR_WCR_WVR_EL1(6),
1349 DBG_BCR_BVR_WCR_WVR_EL1(7),
1350 DBG_BCR_BVR_WCR_WVR_EL1(8),
1351 DBG_BCR_BVR_WCR_WVR_EL1(9),
1352 DBG_BCR_BVR_WCR_WVR_EL1(10),
1353 DBG_BCR_BVR_WCR_WVR_EL1(11),
1354 DBG_BCR_BVR_WCR_WVR_EL1(12),
1355 DBG_BCR_BVR_WCR_WVR_EL1(13),
1356 DBG_BCR_BVR_WCR_WVR_EL1(14),
1357 DBG_BCR_BVR_WCR_WVR_EL1(15),
1358
ee1b64e6
MR		 1359 { SYS_DESC(SYS_MDRAR_EL1), trap_raz_wi },
1360 { SYS_DESC(SYS_OSLAR_EL1), trap_raz_wi },
1361 { SYS_DESC(SYS_OSLSR_EL1), trap_oslsr_el1 },
1362 { SYS_DESC(SYS_OSDLR_EL1), trap_raz_wi },
1363 { SYS_DESC(SYS_DBGPRCR_EL1), trap_raz_wi },
1364 { SYS_DESC(SYS_DBGCLAIMSET_EL1), trap_raz_wi },
1365 { SYS_DESC(SYS_DBGCLAIMCLR_EL1), trap_raz_wi },
1366 { SYS_DESC(SYS_DBGAUTHSTATUS_EL1), trap_dbgauthstatus_el1 },
1367
1368 { SYS_DESC(SYS_MDCCSR_EL0), trap_raz_wi },
1369 { SYS_DESC(SYS_DBGDTR_EL0), trap_raz_wi },
1370 // DBGDTR[TR]X_EL0 share the same encoding
1371 { SYS_DESC(SYS_DBGDTRTX_EL0), trap_raz_wi },
1372
1373 { SYS_DESC(SYS_DBGVCR32_EL2), NULL, reset_val, DBGVCR32_EL2, 0 },
62a89c44 1374
851050a5 1375 { SYS_DESC(SYS_MPIDR_EL1), NULL, reset_mpidr, MPIDR_EL1 },
93390c0a
DM		 1376
1377 /*
1378 * ID regs: all ID_SANITISED() entries here must have corresponding
1379 * entries in arm64_ftr_regs[].
1380 */
1381
1382 /* AArch64 mappings of the AArch32 ID registers */
1383 /* CRm=1 */
1384 ID_SANITISED(ID_PFR0_EL1),
1385 ID_SANITISED(ID_PFR1_EL1),
1386 ID_SANITISED(ID_DFR0_EL1),
1387 ID_HIDDEN(ID_AFR0_EL1),
1388 ID_SANITISED(ID_MMFR0_EL1),
1389 ID_SANITISED(ID_MMFR1_EL1),
1390 ID_SANITISED(ID_MMFR2_EL1),
1391 ID_SANITISED(ID_MMFR3_EL1),
1392
1393 /* CRm=2 */
1394 ID_SANITISED(ID_ISAR0_EL1),
1395 ID_SANITISED(ID_ISAR1_EL1),
1396 ID_SANITISED(ID_ISAR2_EL1),
1397 ID_SANITISED(ID_ISAR3_EL1),
1398 ID_SANITISED(ID_ISAR4_EL1),
1399 ID_SANITISED(ID_ISAR5_EL1),
1400 ID_SANITISED(ID_MMFR4_EL1),
8e3747be 1401 ID_SANITISED(ID_ISAR6_EL1),
93390c0a
DM		 1402
1403 /* CRm=3 */
1404 ID_SANITISED(MVFR0_EL1),
1405 ID_SANITISED(MVFR1_EL1),
1406 ID_SANITISED(MVFR2_EL1),
1407 ID_UNALLOCATED(3,3),
16824085 1408 ID_SANITISED(ID_PFR2_EL1),
dd35ec07 1409 ID_HIDDEN(ID_DFR1_EL1),
152accf8 1410 ID_SANITISED(ID_MMFR5_EL1),
93390c0a
DM		 1411 ID_UNALLOCATED(3,7),
1412
1413 /* AArch64 ID registers */
1414 /* CRm=4 */
23711a5e
MZ		 1415 { SYS_DESC(SYS_ID_AA64PFR0_EL1), .access = access_id_reg,
1416 .get_user = get_id_reg, .set_user = set_id_aa64pfr0_el1, },
93390c0a
DM		 1417 ID_SANITISED(ID_AA64PFR1_EL1),
1418 ID_UNALLOCATED(4,2),
1419 ID_UNALLOCATED(4,3),
c512298e 1420 ID_SANITISED(ID_AA64ZFR0_EL1),
93390c0a
DM		 1421 ID_UNALLOCATED(4,5),
1422 ID_UNALLOCATED(4,6),
1423 ID_UNALLOCATED(4,7),
1424
1425 /* CRm=5 */
1426 ID_SANITISED(ID_AA64DFR0_EL1),
1427 ID_SANITISED(ID_AA64DFR1_EL1),
1428 ID_UNALLOCATED(5,2),
1429 ID_UNALLOCATED(5,3),
1430 ID_HIDDEN(ID_AA64AFR0_EL1),
1431 ID_HIDDEN(ID_AA64AFR1_EL1),
1432 ID_UNALLOCATED(5,6),
1433 ID_UNALLOCATED(5,7),
1434
1435 /* CRm=6 */
1436 ID_SANITISED(ID_AA64ISAR0_EL1),
1437 ID_SANITISED(ID_AA64ISAR1_EL1),
1438 ID_UNALLOCATED(6,2),
1439 ID_UNALLOCATED(6,3),
1440 ID_UNALLOCATED(6,4),
1441 ID_UNALLOCATED(6,5),
1442 ID_UNALLOCATED(6,6),
1443 ID_UNALLOCATED(6,7),
1444
1445 /* CRm=7 */
1446 ID_SANITISED(ID_AA64MMFR0_EL1),
1447 ID_SANITISED(ID_AA64MMFR1_EL1),
1448 ID_SANITISED(ID_AA64MMFR2_EL1),
1449 ID_UNALLOCATED(7,3),
1450 ID_UNALLOCATED(7,4),
1451 ID_UNALLOCATED(7,5),
1452 ID_UNALLOCATED(7,6),
1453 ID_UNALLOCATED(7,7),
1454
851050a5 1455 { SYS_DESC(SYS_SCTLR_EL1), access_vm_reg, reset_val, SCTLR_EL1, 0x00C50078 },
af473829 1456 { SYS_DESC(SYS_ACTLR_EL1), access_actlr, reset_actlr, ACTLR_EL1 },
851050a5 1457 { SYS_DESC(SYS_CPACR_EL1), NULL, reset_val, CPACR_EL1, 0 },
2ac638fc 1458
338b1793
MZ		 1459 { SYS_DESC(SYS_RGSR_EL1), undef_access },
1460 { SYS_DESC(SYS_GCR_EL1), undef_access },
2ac638fc 1461
73433762 1462 { SYS_DESC(SYS_ZCR_EL1), NULL, reset_val, ZCR_EL1, 0, .visibility = sve_visibility },
851050a5
MR		 1463 { SYS_DESC(SYS_TTBR0_EL1), access_vm_reg, reset_unknown, TTBR0_EL1 },
1464 { SYS_DESC(SYS_TTBR1_EL1), access_vm_reg, reset_unknown, TTBR1_EL1 },
1465 { SYS_DESC(SYS_TCR_EL1), access_vm_reg, reset_val, TCR_EL1, 0 },
1466
384b40ca
MR		 1467 PTRAUTH_KEY(APIA),
1468 PTRAUTH_KEY(APIB),
1469 PTRAUTH_KEY(APDA),
1470 PTRAUTH_KEY(APDB),
1471 PTRAUTH_KEY(APGA),
1472
851050a5
MR		 1473 { SYS_DESC(SYS_AFSR0_EL1), access_vm_reg, reset_unknown, AFSR0_EL1 },
1474 { SYS_DESC(SYS_AFSR1_EL1), access_vm_reg, reset_unknown, AFSR1_EL1 },
1475 { SYS_DESC(SYS_ESR_EL1), access_vm_reg, reset_unknown, ESR_EL1 },
558daf69
DG		 1476
1477 { SYS_DESC(SYS_ERRIDR_EL1), trap_raz_wi },
1478 { SYS_DESC(SYS_ERRSELR_EL1), trap_raz_wi },
1479 { SYS_DESC(SYS_ERXFR_EL1), trap_raz_wi },
1480 { SYS_DESC(SYS_ERXCTLR_EL1), trap_raz_wi },
1481 { SYS_DESC(SYS_ERXSTATUS_EL1), trap_raz_wi },
1482 { SYS_DESC(SYS_ERXADDR_EL1), trap_raz_wi },
1483 { SYS_DESC(SYS_ERXMISC0_EL1), trap_raz_wi },
1484 { SYS_DESC(SYS_ERXMISC1_EL1), trap_raz_wi },
1485
338b1793
MZ		 1486 { SYS_DESC(SYS_TFSR_EL1), undef_access },
1487 { SYS_DESC(SYS_TFSRE0_EL1), undef_access },
2ac638fc 1488
851050a5
MR		 1489 { SYS_DESC(SYS_FAR_EL1), access_vm_reg, reset_unknown, FAR_EL1 },
1490 { SYS_DESC(SYS_PAR_EL1), NULL, reset_unknown, PAR_EL1 },
7c8c5e6a 1491
11663111
MZ		 1492 { PMU_SYS_REG(SYS_PMINTENSET_EL1),
1493 .access = access_pminten, .reg = PMINTENSET_EL1 },
1494 { PMU_SYS_REG(SYS_PMINTENCLR_EL1),
1495 .access = access_pminten, .reg = PMINTENSET_EL1 },
7c8c5e6a 1496
851050a5
MR		 1497 { SYS_DESC(SYS_MAIR_EL1), access_vm_reg, reset_unknown, MAIR_EL1 },
1498 { SYS_DESC(SYS_AMAIR_EL1), access_vm_reg, reset_amair_el1, AMAIR_EL1 },
7c8c5e6a 1499
22925521
MZ		 1500 { SYS_DESC(SYS_LORSA_EL1), trap_loregion },
1501 { SYS_DESC(SYS_LOREA_EL1), trap_loregion },
1502 { SYS_DESC(SYS_LORN_EL1), trap_loregion },
1503 { SYS_DESC(SYS_LORC_EL1), trap_loregion },
1504 { SYS_DESC(SYS_LORID_EL1), trap_loregion },
cc33c4e2 1505
851050a5 1506 { SYS_DESC(SYS_VBAR_EL1), NULL, reset_val, VBAR_EL1, 0 },
c773ae2b 1507 { SYS_DESC(SYS_DISR_EL1), NULL, reset_val, DISR_EL1, 0 },
db7dedd0 1508
7b1dba1f 1509 { SYS_DESC(SYS_ICC_IAR0_EL1), write_to_read_only },
e7f1d1ee 1510 { SYS_DESC(SYS_ICC_EOIR0_EL1), read_from_write_only },
7b1dba1f 1511 { SYS_DESC(SYS_ICC_HPPIR0_EL1), write_to_read_only },
e7f1d1ee 1512 { SYS_DESC(SYS_ICC_DIR_EL1), read_from_write_only },
7b1dba1f 1513 { SYS_DESC(SYS_ICC_RPR_EL1), write_to_read_only },
e804d208 1514 { SYS_DESC(SYS_ICC_SGI1R_EL1), access_gic_sgi },
03bd646d
MZ		 1515 { SYS_DESC(SYS_ICC_ASGI1R_EL1), access_gic_sgi },
1516 { SYS_DESC(SYS_ICC_SGI0R_EL1), access_gic_sgi },
7b1dba1f 1517 { SYS_DESC(SYS_ICC_IAR1_EL1), write_to_read_only },
e7f1d1ee 1518 { SYS_DESC(SYS_ICC_EOIR1_EL1), read_from_write_only },
7b1dba1f 1519 { SYS_DESC(SYS_ICC_HPPIR1_EL1), write_to_read_only },
e804d208 1520 { SYS_DESC(SYS_ICC_SRE_EL1), access_gic_sre },
db7dedd0 1521
851050a5
MR		 1522 { SYS_DESC(SYS_CONTEXTIDR_EL1), access_vm_reg, reset_val, CONTEXTIDR_EL1, 0 },
1523 { SYS_DESC(SYS_TPIDR_EL1), NULL, reset_unknown, TPIDR_EL1 },
7c8c5e6a 1524
ed4ffaf4
MZ		 1525 { SYS_DESC(SYS_SCXTNUM_EL1), undef_access },
1526
851050a5 1527 { SYS_DESC(SYS_CNTKCTL_EL1), NULL, reset_val, CNTKCTL_EL1, 0},
7c8c5e6a 1528
f7f2b15c
AB		 1529 { SYS_DESC(SYS_CCSIDR_EL1), access_ccsidr },
1530 { SYS_DESC(SYS_CLIDR_EL1), access_clidr },
1531 { SYS_DESC(SYS_CSSELR_EL1), access_csselr, reset_unknown, CSSELR_EL1 },
1532 { SYS_DESC(SYS_CTR_EL0), access_ctr },
7c8c5e6a 1533
11663111
MZ		 1534 { PMU_SYS_REG(SYS_PMCR_EL0), .access = access_pmcr,
1535 .reset = reset_pmcr, .reg = PMCR_EL0 },
1536 { PMU_SYS_REG(SYS_PMCNTENSET_EL0),
1537 .access = access_pmcnten, .reg = PMCNTENSET_EL0 },
1538 { PMU_SYS_REG(SYS_PMCNTENCLR_EL0),
1539 .access = access_pmcnten, .reg = PMCNTENSET_EL0 },
1540 { PMU_SYS_REG(SYS_PMOVSCLR_EL0),
1541 .access = access_pmovs, .reg = PMOVSSET_EL0 },
1542 { PMU_SYS_REG(SYS_PMSWINC_EL0),
1543 .access = access_pmswinc, .reg = PMSWINC_EL0 },
1544 { PMU_SYS_REG(SYS_PMSELR_EL0),
1545 .access = access_pmselr, .reg = PMSELR_EL0 },
1546 { PMU_SYS_REG(SYS_PMCEID0_EL0),
1547 .access = access_pmceid, .reset = NULL },
1548 { PMU_SYS_REG(SYS_PMCEID1_EL0),
1549 .access = access_pmceid, .reset = NULL },
1550 { PMU_SYS_REG(SYS_PMCCNTR_EL0),
1551 .access = access_pmu_evcntr, .reg = PMCCNTR_EL0 },
1552 { PMU_SYS_REG(SYS_PMXEVTYPER_EL0),
1553 .access = access_pmu_evtyper, .reset = NULL },
1554 { PMU_SYS_REG(SYS_PMXEVCNTR_EL0),
1555 .access = access_pmu_evcntr, .reset = NULL },
174ed3e4
MR		 1556 /*
1557 * PMUSERENR_EL0 resets as unknown in 64bit mode while it resets as zero
d692b8ad
SZ		 1558 * in 32bit mode. Here we choose to reset it as zero for consistency.
1559 */
11663111
MZ		 1560 { PMU_SYS_REG(SYS_PMUSERENR_EL0), .access = access_pmuserenr,
1561 .reset = reset_val, .reg = PMUSERENR_EL0, .val = 0 },
1562 { PMU_SYS_REG(SYS_PMOVSSET_EL0),
1563 .access = access_pmovs, .reg = PMOVSSET_EL0 },
7c8c5e6a 1564
851050a5
MR		 1565 { SYS_DESC(SYS_TPIDR_EL0), NULL, reset_unknown, TPIDR_EL0 },
1566 { SYS_DESC(SYS_TPIDRRO_EL0), NULL, reset_unknown, TPIDRRO_EL0 },
4fcdf106 1567
ed4ffaf4
MZ		 1568 { SYS_DESC(SYS_SCXTNUM_EL0), undef_access },
1569
338b1793
MZ		 1570 { SYS_DESC(SYS_AMCR_EL0), undef_access },
1571 { SYS_DESC(SYS_AMCFGR_EL0), undef_access },
1572 { SYS_DESC(SYS_AMCGCR_EL0), undef_access },
1573 { SYS_DESC(SYS_AMUSERENR_EL0), undef_access },
1574 { SYS_DESC(SYS_AMCNTENCLR0_EL0), undef_access },
1575 { SYS_DESC(SYS_AMCNTENSET0_EL0), undef_access },
1576 { SYS_DESC(SYS_AMCNTENCLR1_EL0), undef_access },
1577 { SYS_DESC(SYS_AMCNTENSET1_EL0), undef_access },
4fcdf106
IV		 1578 AMU_AMEVCNTR0_EL0(0),
1579 AMU_AMEVCNTR0_EL0(1),
1580 AMU_AMEVCNTR0_EL0(2),
1581 AMU_AMEVCNTR0_EL0(3),
1582 AMU_AMEVCNTR0_EL0(4),
1583 AMU_AMEVCNTR0_EL0(5),
1584 AMU_AMEVCNTR0_EL0(6),
1585 AMU_AMEVCNTR0_EL0(7),
1586 AMU_AMEVCNTR0_EL0(8),
1587 AMU_AMEVCNTR0_EL0(9),
1588 AMU_AMEVCNTR0_EL0(10),
1589 AMU_AMEVCNTR0_EL0(11),
1590 AMU_AMEVCNTR0_EL0(12),
1591 AMU_AMEVCNTR0_EL0(13),
1592 AMU_AMEVCNTR0_EL0(14),
1593 AMU_AMEVCNTR0_EL0(15),
493cf9b7
VM		 1594 AMU_AMEVTYPER0_EL0(0),
1595 AMU_AMEVTYPER0_EL0(1),
1596 AMU_AMEVTYPER0_EL0(2),
1597 AMU_AMEVTYPER0_EL0(3),
1598 AMU_AMEVTYPER0_EL0(4),
1599 AMU_AMEVTYPER0_EL0(5),
1600 AMU_AMEVTYPER0_EL0(6),
1601 AMU_AMEVTYPER0_EL0(7),
1602 AMU_AMEVTYPER0_EL0(8),
1603 AMU_AMEVTYPER0_EL0(9),
1604 AMU_AMEVTYPER0_EL0(10),
1605 AMU_AMEVTYPER0_EL0(11),
1606 AMU_AMEVTYPER0_EL0(12),
1607 AMU_AMEVTYPER0_EL0(13),
1608 AMU_AMEVTYPER0_EL0(14),
1609 AMU_AMEVTYPER0_EL0(15),
4fcdf106
IV		 1610 AMU_AMEVCNTR1_EL0(0),
1611 AMU_AMEVCNTR1_EL0(1),
1612 AMU_AMEVCNTR1_EL0(2),
1613 AMU_AMEVCNTR1_EL0(3),
1614 AMU_AMEVCNTR1_EL0(4),
1615 AMU_AMEVCNTR1_EL0(5),
1616 AMU_AMEVCNTR1_EL0(6),
1617 AMU_AMEVCNTR1_EL0(7),
1618 AMU_AMEVCNTR1_EL0(8),
1619 AMU_AMEVCNTR1_EL0(9),
1620 AMU_AMEVCNTR1_EL0(10),
1621 AMU_AMEVCNTR1_EL0(11),
1622 AMU_AMEVCNTR1_EL0(12),
1623 AMU_AMEVCNTR1_EL0(13),
1624 AMU_AMEVCNTR1_EL0(14),
1625 AMU_AMEVCNTR1_EL0(15),
493cf9b7
VM		 1626 AMU_AMEVTYPER1_EL0(0),
1627 AMU_AMEVTYPER1_EL0(1),
1628 AMU_AMEVTYPER1_EL0(2),
1629 AMU_AMEVTYPER1_EL0(3),
1630 AMU_AMEVTYPER1_EL0(4),
1631 AMU_AMEVTYPER1_EL0(5),
1632 AMU_AMEVTYPER1_EL0(6),
1633 AMU_AMEVTYPER1_EL0(7),
1634 AMU_AMEVTYPER1_EL0(8),
1635 AMU_AMEVTYPER1_EL0(9),
1636 AMU_AMEVTYPER1_EL0(10),
1637 AMU_AMEVTYPER1_EL0(11),
1638 AMU_AMEVTYPER1_EL0(12),
1639 AMU_AMEVTYPER1_EL0(13),
1640 AMU_AMEVTYPER1_EL0(14),
1641 AMU_AMEVTYPER1_EL0(15),
62a89c44 1642
84135d3d
AP		 1643 { SYS_DESC(SYS_CNTP_TVAL_EL0), access_arch_timer },
1644 { SYS_DESC(SYS_CNTP_CTL_EL0), access_arch_timer },
1645 { SYS_DESC(SYS_CNTP_CVAL_EL0), access_arch_timer },
c9a3c58f 1646
051ff581
SZ		 1647 /* PMEVCNTRn_EL0 */
1648 PMU_PMEVCNTR_EL0(0),
1649 PMU_PMEVCNTR_EL0(1),
1650 PMU_PMEVCNTR_EL0(2),
1651 PMU_PMEVCNTR_EL0(3),
1652 PMU_PMEVCNTR_EL0(4),
1653 PMU_PMEVCNTR_EL0(5),
1654 PMU_PMEVCNTR_EL0(6),
1655 PMU_PMEVCNTR_EL0(7),
1656 PMU_PMEVCNTR_EL0(8),
1657 PMU_PMEVCNTR_EL0(9),
1658 PMU_PMEVCNTR_EL0(10),
1659 PMU_PMEVCNTR_EL0(11),
1660 PMU_PMEVCNTR_EL0(12),
1661 PMU_PMEVCNTR_EL0(13),
1662 PMU_PMEVCNTR_EL0(14),
1663 PMU_PMEVCNTR_EL0(15),
1664 PMU_PMEVCNTR_EL0(16),
1665 PMU_PMEVCNTR_EL0(17),
1666 PMU_PMEVCNTR_EL0(18),
1667 PMU_PMEVCNTR_EL0(19),
1668 PMU_PMEVCNTR_EL0(20),
1669 PMU_PMEVCNTR_EL0(21),
1670 PMU_PMEVCNTR_EL0(22),
1671 PMU_PMEVCNTR_EL0(23),
1672 PMU_PMEVCNTR_EL0(24),
1673 PMU_PMEVCNTR_EL0(25),
1674 PMU_PMEVCNTR_EL0(26),
1675 PMU_PMEVCNTR_EL0(27),
1676 PMU_PMEVCNTR_EL0(28),
1677 PMU_PMEVCNTR_EL0(29),
1678 PMU_PMEVCNTR_EL0(30),
9feb21ac
SZ		 1679 /* PMEVTYPERn_EL0 */
1680 PMU_PMEVTYPER_EL0(0),
1681 PMU_PMEVTYPER_EL0(1),
1682 PMU_PMEVTYPER_EL0(2),
1683 PMU_PMEVTYPER_EL0(3),
1684 PMU_PMEVTYPER_EL0(4),
1685 PMU_PMEVTYPER_EL0(5),
1686 PMU_PMEVTYPER_EL0(6),
1687 PMU_PMEVTYPER_EL0(7),
1688 PMU_PMEVTYPER_EL0(8),
1689 PMU_PMEVTYPER_EL0(9),
1690 PMU_PMEVTYPER_EL0(10),
1691 PMU_PMEVTYPER_EL0(11),
1692 PMU_PMEVTYPER_EL0(12),
1693 PMU_PMEVTYPER_EL0(13),
1694 PMU_PMEVTYPER_EL0(14),
1695 PMU_PMEVTYPER_EL0(15),
1696 PMU_PMEVTYPER_EL0(16),
1697 PMU_PMEVTYPER_EL0(17),
1698 PMU_PMEVTYPER_EL0(18),
1699 PMU_PMEVTYPER_EL0(19),
1700 PMU_PMEVTYPER_EL0(20),
1701 PMU_PMEVTYPER_EL0(21),
1702 PMU_PMEVTYPER_EL0(22),
1703 PMU_PMEVTYPER_EL0(23),
1704 PMU_PMEVTYPER_EL0(24),
1705 PMU_PMEVTYPER_EL0(25),
1706 PMU_PMEVTYPER_EL0(26),
1707 PMU_PMEVTYPER_EL0(27),
1708 PMU_PMEVTYPER_EL0(28),
1709 PMU_PMEVTYPER_EL0(29),
1710 PMU_PMEVTYPER_EL0(30),
174ed3e4
MR		 1711 /*
1712 * PMCCFILTR_EL0 resets as unknown in 64bit mode while it resets as zero
9feb21ac
SZ		 1713 * in 32bit mode. Here we choose to reset it as zero for consistency.
1714 */
11663111
MZ		 1715 { PMU_SYS_REG(SYS_PMCCFILTR_EL0), .access = access_pmu_evtyper,
1716 .reset = reset_val, .reg = PMCCFILTR_EL0, .val = 0 },
051ff581 1717
851050a5
MR		 1718 { SYS_DESC(SYS_DACR32_EL2), NULL, reset_unknown, DACR32_EL2 },
1719 { SYS_DESC(SYS_IFSR32_EL2), NULL, reset_unknown, IFSR32_EL2 },
c88b0936 1720 { SYS_DESC(SYS_FPEXC32_EL2), NULL, reset_val, FPEXC32_EL2, 0x700 },
62a89c44
MZ		 1721};
1722
bdfb4b38 1723static bool trap_dbgidr(struct kvm_vcpu *vcpu,
3fec037d 1724 struct sys_reg_params *p,
bdfb4b38
MZ		 1725 const struct sys_reg_desc *r)
1726{
1727 if (p->is_write) {
1728 return ignore_write(vcpu, p);
1729 } else {
46823dd1
DM		 1730 u64 dfr = read_sanitised_ftr_reg(SYS_ID_AA64DFR0_EL1);
1731 u64 pfr = read_sanitised_ftr_reg(SYS_ID_AA64PFR0_EL1);
28c5dcb2 1732 u32 el3 = !!cpuid_feature_extract_unsigned_field(pfr, ID_AA64PFR0_EL3_SHIFT);
bdfb4b38 1733
2ec5be3d
PF		 1734 p->regval = ((((dfr >> ID_AA64DFR0_WRPS_SHIFT) & 0xf) << 28) |
1735 (((dfr >> ID_AA64DFR0_BRPS_SHIFT) & 0xf) << 24) |
1736 (((dfr >> ID_AA64DFR0_CTX_CMPS_SHIFT) & 0xf) << 20)
1737 | (6 << 16) | (el3 << 14) | (el3 << 12));
bdfb4b38
MZ		 1738 return true;
1739 }
1740}
1741
1da42c34
MZ		 1742/*
1743 * AArch32 debug register mappings
84e690bf
AB		 1744 *
1745 * AArch32 DBGBVRn is mapped to DBGBVRn_EL1[31:0]
1746 * AArch32 DBGBXVRn is mapped to DBGBVRn_EL1[63:32]
1747 *
1da42c34
MZ		 1748 * None of the other registers share their location, so treat them as
1749 * if they were 64bit.
84e690bf 1750 */
1da42c34
MZ		 1751#define DBG_BCR_BVR_WCR_WVR(n) \
1752 /* DBGBVRn */ \
1753 { AA32(LO), Op1( 0), CRn( 0), CRm((n)), Op2( 4), trap_bvr, NULL, n }, \
1754 /* DBGBCRn */ \
1755 { Op1( 0), CRn( 0), CRm((n)), Op2( 5), trap_bcr, NULL, n }, \
1756 /* DBGWVRn */ \
1757 { Op1( 0), CRn( 0), CRm((n)), Op2( 6), trap_wvr, NULL, n }, \
1758 /* DBGWCRn */ \
84e690bf
AB		 1759 { Op1( 0), CRn( 0), CRm((n)), Op2( 7), trap_wcr, NULL, n }
1760
1da42c34
MZ		 1761#define DBGBXVR(n) \
1762 { AA32(HI), Op1( 0), CRn( 1), CRm((n)), Op2( 1), trap_bvr, NULL, n }
bdfb4b38
MZ		 1763
1764/*
1765 * Trapped cp14 registers. We generally ignore most of the external
1766 * debug, on the principle that they don't really make sense to a
84e690bf 1767 * guest. Revisit this one day, would this principle change.
bdfb4b38 1768 */
72564016 1769static const struct sys_reg_desc cp14_regs[] = {
bdfb4b38
MZ		 1770 /* DBGIDR */
1771 { Op1( 0), CRn( 0), CRm( 0), Op2( 0), trap_dbgidr },
1772 /* DBGDTRRXext */
1773 { Op1( 0), CRn( 0), CRm( 0), Op2( 2), trap_raz_wi },
1774
1775 DBG_BCR_BVR_WCR_WVR(0),
1776 /* DBGDSCRint */
1777 { Op1( 0), CRn( 0), CRm( 1), Op2( 0), trap_raz_wi },
1778 DBG_BCR_BVR_WCR_WVR(1),
1779 /* DBGDCCINT */
1da42c34 1780 { Op1( 0), CRn( 0), CRm( 2), Op2( 0), trap_debug_regs, NULL, MDCCINT_EL1 },
bdfb4b38 1781 /* DBGDSCRext */
1da42c34 1782 { Op1( 0), CRn( 0), CRm( 2), Op2( 2), trap_debug_regs, NULL, MDSCR_EL1 },
bdfb4b38
MZ		 1783 DBG_BCR_BVR_WCR_WVR(2),
1784 /* DBGDTR[RT]Xint */
1785 { Op1( 0), CRn( 0), CRm( 3), Op2( 0), trap_raz_wi },
1786 /* DBGDTR[RT]Xext */
1787 { Op1( 0), CRn( 0), CRm( 3), Op2( 2), trap_raz_wi },
1788 DBG_BCR_BVR_WCR_WVR(3),
1789 DBG_BCR_BVR_WCR_WVR(4),
1790 DBG_BCR_BVR_WCR_WVR(5),
1791 /* DBGWFAR */
1792 { Op1( 0), CRn( 0), CRm( 6), Op2( 0), trap_raz_wi },
1793 /* DBGOSECCR */
1794 { Op1( 0), CRn( 0), CRm( 6), Op2( 2), trap_raz_wi },
1795 DBG_BCR_BVR_WCR_WVR(6),
1796 /* DBGVCR */
1da42c34 1797 { Op1( 0), CRn( 0), CRm( 7), Op2( 0), trap_debug_regs, NULL, DBGVCR32_EL2 },
bdfb4b38
MZ		 1798 DBG_BCR_BVR_WCR_WVR(7),
1799 DBG_BCR_BVR_WCR_WVR(8),
1800 DBG_BCR_BVR_WCR_WVR(9),
1801 DBG_BCR_BVR_WCR_WVR(10),
1802 DBG_BCR_BVR_WCR_WVR(11),
1803 DBG_BCR_BVR_WCR_WVR(12),
1804 DBG_BCR_BVR_WCR_WVR(13),
1805 DBG_BCR_BVR_WCR_WVR(14),
1806 DBG_BCR_BVR_WCR_WVR(15),
1807
1808 /* DBGDRAR (32bit) */
1809 { Op1( 0), CRn( 1), CRm( 0), Op2( 0), trap_raz_wi },
1810
1811 DBGBXVR(0),
1812 /* DBGOSLAR */
1813 { Op1( 0), CRn( 1), CRm( 0), Op2( 4), trap_raz_wi },
1814 DBGBXVR(1),
1815 /* DBGOSLSR */
1816 { Op1( 0), CRn( 1), CRm( 1), Op2( 4), trap_oslsr_el1 },
1817 DBGBXVR(2),
1818 DBGBXVR(3),
1819 /* DBGOSDLR */
1820 { Op1( 0), CRn( 1), CRm( 3), Op2( 4), trap_raz_wi },
1821 DBGBXVR(4),
1822 /* DBGPRCR */
1823 { Op1( 0), CRn( 1), CRm( 4), Op2( 4), trap_raz_wi },
1824 DBGBXVR(5),
1825 DBGBXVR(6),
1826 DBGBXVR(7),
1827 DBGBXVR(8),
1828 DBGBXVR(9),
1829 DBGBXVR(10),
1830 DBGBXVR(11),
1831 DBGBXVR(12),
1832 DBGBXVR(13),
1833 DBGBXVR(14),
1834 DBGBXVR(15),
1835
1836 /* DBGDSAR (32bit) */
1837 { Op1( 0), CRn( 2), CRm( 0), Op2( 0), trap_raz_wi },
1838
1839 /* DBGDEVID2 */
1840 { Op1( 0), CRn( 7), CRm( 0), Op2( 7), trap_raz_wi },
1841 /* DBGDEVID1 */
1842 { Op1( 0), CRn( 7), CRm( 1), Op2( 7), trap_raz_wi },
1843 /* DBGDEVID */
1844 { Op1( 0), CRn( 7), CRm( 2), Op2( 7), trap_raz_wi },
1845 /* DBGCLAIMSET */
1846 { Op1( 0), CRn( 7), CRm( 8), Op2( 6), trap_raz_wi },
1847 /* DBGCLAIMCLR */
1848 { Op1( 0), CRn( 7), CRm( 9), Op2( 6), trap_raz_wi },
1849 /* DBGAUTHSTATUS */
1850 { Op1( 0), CRn( 7), CRm(14), Op2( 6), trap_dbgauthstatus_el1 },
72564016
MZ		 1851};
1852
a9866ba0
MZ		 1853/* Trapped cp14 64bit registers */
1854static const struct sys_reg_desc cp14_64_regs[] = {
bdfb4b38
MZ		 1855 /* DBGDRAR (64bit) */
1856 { Op1( 0), CRm( 1), .access = trap_raz_wi },
1857
1858 /* DBGDSAR (64bit) */
1859 { Op1( 0), CRm( 2), .access = trap_raz_wi },
a9866ba0
MZ		 1860};
1861
051ff581
SZ		 1862/* Macro to expand the PMEVCNTRn register */
1863#define PMU_PMEVCNTR(n) \
1864 /* PMEVCNTRn */ \
1865 { Op1(0), CRn(0b1110), \
1866 CRm((0b1000 | (((n) >> 3) & 0x3))), Op2(((n) & 0x7)), \
1867 access_pmu_evcntr }
1868
9feb21ac
SZ		 1869/* Macro to expand the PMEVTYPERn register */
1870#define PMU_PMEVTYPER(n) \
1871 /* PMEVTYPERn */ \
1872 { Op1(0), CRn(0b1110), \
1873 CRm((0b1100 | (((n) >> 3) & 0x3))), Op2(((n) & 0x7)), \
1874 access_pmu_evtyper }
1875
4d44923b
MZ		 1876/*
1877 * Trapped cp15 registers. TTBR0/TTBR1 get a double encoding,
1878 * depending on the way they are accessed (as a 32bit or a 64bit
1879 * register).
1880 */
62a89c44 1881static const struct sys_reg_desc cp15_regs[] = {
f7f2b15c 1882 { Op1( 0), CRn( 0), CRm( 0), Op2( 1), access_ctr },
b1ea1d76
MZ		 1883 { Op1( 0), CRn( 1), CRm( 0), Op2( 0), access_vm_reg, NULL, SCTLR_EL1 },
1884 /* ACTLR */
1885 { AA32(LO), Op1( 0), CRn( 1), CRm( 0), Op2( 1), access_actlr, NULL, ACTLR_EL1 },
1886 /* ACTLR2 */
1887 { AA32(HI), Op1( 0), CRn( 1), CRm( 0), Op2( 3), access_actlr, NULL, ACTLR_EL1 },
1888 { Op1( 0), CRn( 2), CRm( 0), Op2( 0), access_vm_reg, NULL, TTBR0_EL1 },
1889 { Op1( 0), CRn( 2), CRm( 0), Op2( 1), access_vm_reg, NULL, TTBR1_EL1 },
1890 /* TTBCR */
1891 { AA32(LO), Op1( 0), CRn( 2), CRm( 0), Op2( 2), access_vm_reg, NULL, TCR_EL1 },
1892 /* TTBCR2 */
1893 { AA32(HI), Op1( 0), CRn( 2), CRm( 0), Op2( 3), access_vm_reg, NULL, TCR_EL1 },
1894 { Op1( 0), CRn( 3), CRm( 0), Op2( 0), access_vm_reg, NULL, DACR32_EL2 },
1895 /* DFSR */
1896 { Op1( 0), CRn( 5), CRm( 0), Op2( 0), access_vm_reg, NULL, ESR_EL1 },
1897 { Op1( 0), CRn( 5), CRm( 0), Op2( 1), access_vm_reg, NULL, IFSR32_EL2 },
1898 /* ADFSR */
1899 { Op1( 0), CRn( 5), CRm( 1), Op2( 0), access_vm_reg, NULL, AFSR0_EL1 },
1900 /* AIFSR */
1901 { Op1( 0), CRn( 5), CRm( 1), Op2( 1), access_vm_reg, NULL, AFSR1_EL1 },
1902 /* DFAR */
1903 { AA32(LO), Op1( 0), CRn( 6), CRm( 0), Op2( 0), access_vm_reg, NULL, FAR_EL1 },
1904 /* IFAR */
1905 { AA32(HI), Op1( 0), CRn( 6), CRm( 0), Op2( 2), access_vm_reg, NULL, FAR_EL1 },
4d44923b 1906
62a89c44
MZ		 1907 /*
1908 * DC{C,I,CI}SW operations:
1909 */
1910 { Op1( 0), CRn( 7), CRm( 6), Op2( 2), access_dcsw },
1911 { Op1( 0), CRn( 7), CRm(10), Op2( 2), access_dcsw },
1912 { Op1( 0), CRn( 7), CRm(14), Op2( 2), access_dcsw },
4d44923b 1913
7609c125 1914 /* PMU */
ab946834 1915 { Op1( 0), CRn( 9), CRm(12), Op2( 0), access_pmcr },
96b0eebc
SZ		 1916 { Op1( 0), CRn( 9), CRm(12), Op2( 1), access_pmcnten },
1917 { Op1( 0), CRn( 9), CRm(12), Op2( 2), access_pmcnten },
76d883c4 1918 { Op1( 0), CRn( 9), CRm(12), Op2( 3), access_pmovs },
7a0adc70 1919 { Op1( 0), CRn( 9), CRm(12), Op2( 4), access_pmswinc },
3965c3ce 1920 { Op1( 0), CRn( 9), CRm(12), Op2( 5), access_pmselr },
a86b5505
SZ		 1921 { Op1( 0), CRn( 9), CRm(12), Op2( 6), access_pmceid },
1922 { Op1( 0), CRn( 9), CRm(12), Op2( 7), access_pmceid },
051ff581 1923 { Op1( 0), CRn( 9), CRm(13), Op2( 0), access_pmu_evcntr },
9feb21ac 1924 { Op1( 0), CRn( 9), CRm(13), Op2( 1), access_pmu_evtyper },
051ff581 1925 { Op1( 0), CRn( 9), CRm(13), Op2( 2), access_pmu_evcntr },
d692b8ad 1926 { Op1( 0), CRn( 9), CRm(14), Op2( 0), access_pmuserenr },
9db52c78
SZ		 1927 { Op1( 0), CRn( 9), CRm(14), Op2( 1), access_pminten },
1928 { Op1( 0), CRn( 9), CRm(14), Op2( 2), access_pminten },
76d883c4 1929 { Op1( 0), CRn( 9), CRm(14), Op2( 3), access_pmovs },
4d44923b 1930
b1ea1d76
MZ		 1931 /* PRRR/MAIR0 */
1932 { AA32(LO), Op1( 0), CRn(10), CRm( 2), Op2( 0), access_vm_reg, NULL, MAIR_EL1 },
1933 /* NMRR/MAIR1 */
1934 { AA32(HI), Op1( 0), CRn(10), CRm( 2), Op2( 1), access_vm_reg, NULL, MAIR_EL1 },
1935 /* AMAIR0 */
1936 { AA32(LO), Op1( 0), CRn(10), CRm( 3), Op2( 0), access_vm_reg, NULL, AMAIR_EL1 },
1937 /* AMAIR1 */
1938 { AA32(HI), Op1( 0), CRn(10), CRm( 3), Op2( 1), access_vm_reg, NULL, AMAIR_EL1 },
db7dedd0
CD		 1939
1940 /* ICC_SRE */
f7f6f2d9 1941 { Op1( 0), CRn(12), CRm(12), Op2( 5), access_gic_sre },
db7dedd0 1942
b1ea1d76 1943 { Op1( 0), CRn(13), CRm( 0), Op2( 1), access_vm_reg, NULL, CONTEXTIDR_EL1 },
051ff581 1944
84135d3d
AP		 1945 /* Arch Tmers */
1946 { SYS_DESC(SYS_AARCH32_CNTP_TVAL), access_arch_timer },
1947 { SYS_DESC(SYS_AARCH32_CNTP_CTL), access_arch_timer },
eac137b4 1948
051ff581
SZ		 1949 /* PMEVCNTRn */
1950 PMU_PMEVCNTR(0),
1951 PMU_PMEVCNTR(1),
1952 PMU_PMEVCNTR(2),
1953 PMU_PMEVCNTR(3),
1954 PMU_PMEVCNTR(4),
1955 PMU_PMEVCNTR(5),
1956 PMU_PMEVCNTR(6),
1957 PMU_PMEVCNTR(7),
1958 PMU_PMEVCNTR(8),
1959 PMU_PMEVCNTR(9),
1960 PMU_PMEVCNTR(10),
1961 PMU_PMEVCNTR(11),
1962 PMU_PMEVCNTR(12),
1963 PMU_PMEVCNTR(13),
1964 PMU_PMEVCNTR(14),
1965 PMU_PMEVCNTR(15),
1966 PMU_PMEVCNTR(16),
1967 PMU_PMEVCNTR(17),
1968 PMU_PMEVCNTR(18),
1969 PMU_PMEVCNTR(19),
1970 PMU_PMEVCNTR(20),
1971 PMU_PMEVCNTR(21),
1972 PMU_PMEVCNTR(22),
1973 PMU_PMEVCNTR(23),
1974 PMU_PMEVCNTR(24),
1975 PMU_PMEVCNTR(25),
1976 PMU_PMEVCNTR(26),
1977 PMU_PMEVCNTR(27),
1978 PMU_PMEVCNTR(28),
1979 PMU_PMEVCNTR(29),
1980 PMU_PMEVCNTR(30),
9feb21ac
SZ		 1981 /* PMEVTYPERn */
1982 PMU_PMEVTYPER(0),
1983 PMU_PMEVTYPER(1),
1984 PMU_PMEVTYPER(2),
1985 PMU_PMEVTYPER(3),
1986 PMU_PMEVTYPER(4),
1987 PMU_PMEVTYPER(5),
1988 PMU_PMEVTYPER(6),
1989 PMU_PMEVTYPER(7),
1990 PMU_PMEVTYPER(8),
1991 PMU_PMEVTYPER(9),
1992 PMU_PMEVTYPER(10),
1993 PMU_PMEVTYPER(11),
1994 PMU_PMEVTYPER(12),
1995 PMU_PMEVTYPER(13),
1996 PMU_PMEVTYPER(14),
1997 PMU_PMEVTYPER(15),
1998 PMU_PMEVTYPER(16),
1999 PMU_PMEVTYPER(17),
2000 PMU_PMEVTYPER(18),
2001 PMU_PMEVTYPER(19),
2002 PMU_PMEVTYPER(20),
2003 PMU_PMEVTYPER(21),
2004 PMU_PMEVTYPER(22),
2005 PMU_PMEVTYPER(23),
2006 PMU_PMEVTYPER(24),
2007 PMU_PMEVTYPER(25),
2008 PMU_PMEVTYPER(26),
2009 PMU_PMEVTYPER(27),
2010 PMU_PMEVTYPER(28),
2011 PMU_PMEVTYPER(29),
2012 PMU_PMEVTYPER(30),
2013 /* PMCCFILTR */
2014 { Op1(0), CRn(14), CRm(15), Op2(7), access_pmu_evtyper },
f7f2b15c
AB		 2015
2016 { Op1(1), CRn( 0), CRm( 0), Op2(0), access_ccsidr },
2017 { Op1(1), CRn( 0), CRm( 0), Op2(1), access_clidr },
b1ea1d76 2018 { Op1(2), CRn( 0), CRm( 0), Op2(0), access_csselr, NULL, CSSELR_EL1 },
a9866ba0
MZ		 2019};
2020
2021static const struct sys_reg_desc cp15_64_regs[] = {
b1ea1d76 2022 { Op1( 0), CRn( 0), CRm( 2), Op2( 0), access_vm_reg, NULL, TTBR0_EL1 },
051ff581 2023 { Op1( 0), CRn( 0), CRm( 9), Op2( 0), access_pmu_evcntr },
03bd646d 2024 { Op1( 0), CRn( 0), CRm(12), Op2( 0), access_gic_sgi }, /* ICC_SGI1R */
b1ea1d76 2025 { Op1( 1), CRn( 0), CRm( 2), Op2( 0), access_vm_reg, NULL, TTBR1_EL1 },
03bd646d
MZ		 2026 { Op1( 1), CRn( 0), CRm(12), Op2( 0), access_gic_sgi }, /* ICC_ASGI1R */
2027 { Op1( 2), CRn( 0), CRm(12), Op2( 0), access_gic_sgi }, /* ICC_SGI0R */
84135d3d 2028 { SYS_DESC(SYS_AARCH32_CNTP_CVAL), access_arch_timer },
7c8c5e6a
MZ		 2029};
2030
bb44a8db
MZ		 2031static int check_sysreg_table(const struct sys_reg_desc *table, unsigned int n,
2032 bool is_32)
2033{
2034 unsigned int i;
2035
2036 for (i = 0; i < n; i++) {
2037 if (!is_32 && table[i].reg && !table[i].reset) {
2038 kvm_err("sys_reg table %p entry %d has lacks reset\n",
2039 table, i);
2040 return 1;
2041 }
2042
2043 if (i && cmp_sys_reg(&table[i-1], &table[i]) >= 0) {
2044 kvm_err("sys_reg table %p out of order (%d)\n", table, i - 1);
2045 return 1;
2046 }
2047 }
2048
2049 return 0;
2050}
2051
623eefa8
MZ		 2052static int match_sys_reg(const void *key, const void *elt)
2053{
2054 const unsigned long pval = (unsigned long)key;
2055 const struct sys_reg_desc *r = elt;
2056
09838de9 2057 return pval - reg_to_encoding(r);
623eefa8
MZ		 2058}
2059
7c8c5e6a
MZ		 2060static const struct sys_reg_desc *find_reg(const struct sys_reg_params *params,
2061 const struct sys_reg_desc table[],
2062 unsigned int num)
2063{
09838de9 2064 unsigned long pval = reg_to_encoding(params);
623eefa8
MZ		 2065
2066 return bsearch((void *)pval, table, num, sizeof(table[0]), match_sys_reg);
7c8c5e6a
MZ		 2067}
2068
74cc7e0c 2069int kvm_handle_cp14_load_store(struct kvm_vcpu *vcpu)
62a89c44
MZ		 2070{
2071 kvm_inject_undefined(vcpu);
2072 return 1;
2073}
2074
e70b9522
MZ		 2075static void perform_access(struct kvm_vcpu *vcpu,
2076 struct sys_reg_params *params,
2077 const struct sys_reg_desc *r)
2078{
599d79dc
MZ		 2079 trace_kvm_sys_access(*vcpu_pc(vcpu), params, r);
2080
7f34e409 2081 /* Check for regs disabled by runtime config */
01fe5ace 2082 if (sysreg_hidden(vcpu, r)) {
7f34e409
DM		 2083 kvm_inject_undefined(vcpu);
2084 return;
2085 }
2086
e70b9522
MZ		 2087 /*
2088 * Not having an accessor means that we have configured a trap
2089 * that we don't know how to handle. This certainly qualifies
2090 * as a gross bug that should be fixed right away.
2091 */
2092 BUG_ON(!r->access);
2093
2094 /* Skip instruction if instructed so */
2095 if (likely(r->access(vcpu, params, r)))
cdb5e02e 2096 kvm_incr_pc(vcpu);
e70b9522
MZ		 2097}
2098
72564016
MZ		 2099/*
2100 * emulate_cp -- tries to match a sys_reg access in a handling table, and
2101 * call the corresponding trap handler.
2102 *
2103 * @params: pointer to the descriptor of the access
2104 * @table: array of trap descriptors
2105 * @num: size of the trap descriptor array
2106 *
2107 * Return 0 if the access has been handled, and -1 if not.
2108 */
2109static int emulate_cp(struct kvm_vcpu *vcpu,
3fec037d 2110 struct sys_reg_params *params,
72564016
MZ		 2111 const struct sys_reg_desc *table,
2112 size_t num)
62a89c44 2113{
72564016 2114 const struct sys_reg_desc *r;
62a89c44 2115
72564016
MZ		 2116 if (!table)
2117 return -1; /* Not handled */
62a89c44 2118
62a89c44 2119 r = find_reg(params, table, num);
62a89c44 2120
72564016 2121 if (r) {
e70b9522
MZ		 2122 perform_access(vcpu, params, r);
2123 return 0;
72564016
MZ		 2124 }
2125
2126 /* Not handled */
2127 return -1;
2128}
2129
2130static void unhandled_cp_access(struct kvm_vcpu *vcpu,
2131 struct sys_reg_params *params)
2132{
3a949f4c 2133 u8 esr_ec = kvm_vcpu_trap_get_class(vcpu);
40c4f8d2 2134 int cp = -1;
72564016 2135
3a949f4c 2136 switch (esr_ec) {
c6d01a94
MR		 2137 case ESR_ELx_EC_CP15_32:
2138 case ESR_ELx_EC_CP15_64:
72564016
MZ		 2139 cp = 15;
2140 break;
c6d01a94
MR		 2141 case ESR_ELx_EC_CP14_MR:
2142 case ESR_ELx_EC_CP14_64:
72564016
MZ		 2143 cp = 14;
2144 break;
2145 default:
40c4f8d2 2146 WARN_ON(1);
62a89c44
MZ		 2147 }
2148
bf4b96bb
MR		 2149 print_sys_reg_msg(params,
2150 "Unsupported guest CP%d access at: %08lx [%08lx]\n",
2151 cp, *vcpu_pc(vcpu), *vcpu_cpsr(vcpu));
62a89c44
MZ		 2152 kvm_inject_undefined(vcpu);
2153}
2154
2155/**
7769db90 2156 * kvm_handle_cp_64 -- handles a mrrc/mcrr trap on a guest CP14/CP15 access
62a89c44
MZ		 2157 * @vcpu: The VCPU pointer
2158 * @run: The kvm_run struct
2159 */
72564016
MZ		 2160static int kvm_handle_cp_64(struct kvm_vcpu *vcpu,
2161 const struct sys_reg_desc *global,
dcaffa7b 2162 size_t nr_global)
62a89c44
MZ		 2163{
2164 struct sys_reg_params params;
3a949f4c 2165 u32 esr = kvm_vcpu_get_esr(vcpu);
c667186f 2166 int Rt = kvm_vcpu_sys_get_rt(vcpu);
3a949f4c 2167 int Rt2 = (esr >> 10) & 0x1f;
62a89c44 2168
3a949f4c
GS		 2169 params.CRm = (esr >> 1) & 0xf;
2170 params.is_write = ((esr & 1) == 0);
62a89c44
MZ		 2171
2172 params.Op0 = 0;
3a949f4c 2173 params.Op1 = (esr >> 16) & 0xf;
62a89c44
MZ		 2174 params.Op2 = 0;
2175 params.CRn = 0;
2176
2177 /*
2ec5be3d 2178 * Make a 64-bit value out of Rt and Rt2. As we use the same trap
62a89c44
MZ		 2179 * backends between AArch32 and AArch64, we get away with it.
2180 */
2181 if (params.is_write) {
2ec5be3d
PF		 2182 params.regval = vcpu_get_reg(vcpu, Rt) & 0xffffffff;
2183 params.regval |= vcpu_get_reg(vcpu, Rt2) << 32;
62a89c44
MZ		 2184 }
2185
b6b7a806 2186 /*
dcaffa7b 2187 * If the table contains a handler, handle the
b6b7a806
MZ		 2188 * potential register operation in the case of a read and return
2189 * with success.
2190 */
dcaffa7b 2191 if (!emulate_cp(vcpu, &params, global, nr_global)) {
b6b7a806
MZ		 2192 /* Split up the value between registers for the read side */
2193 if (!params.is_write) {
2194 vcpu_set_reg(vcpu, Rt, lower_32_bits(params.regval));
2195 vcpu_set_reg(vcpu, Rt2, upper_32_bits(params.regval));
2196 }
62a89c44 2197
b6b7a806 2198 return 1;
62a89c44
MZ		 2199 }
2200
b6b7a806 2201 unhandled_cp_access(vcpu, &params);
62a89c44
MZ		 2202 return 1;
2203}
2204
2205/**
7769db90 2206 * kvm_handle_cp_32 -- handles a mrc/mcr trap on a guest CP14/CP15 access
62a89c44
MZ		 2207 * @vcpu: The VCPU pointer
2208 * @run: The kvm_run struct
2209 */
72564016
MZ		 2210static int kvm_handle_cp_32(struct kvm_vcpu *vcpu,
2211 const struct sys_reg_desc *global,
dcaffa7b 2212 size_t nr_global)
62a89c44
MZ		 2213{
2214 struct sys_reg_params params;
3a949f4c 2215 u32 esr = kvm_vcpu_get_esr(vcpu);
c667186f 2216 int Rt = kvm_vcpu_sys_get_rt(vcpu);
62a89c44 2217
3a949f4c 2218 params.CRm = (esr >> 1) & 0xf;
2ec5be3d 2219 params.regval = vcpu_get_reg(vcpu, Rt);
3a949f4c
GS		 2220 params.is_write = ((esr & 1) == 0);
2221 params.CRn = (esr >> 10) & 0xf;
62a89c44 2222 params.Op0 = 0;
3a949f4c
GS		 2223 params.Op1 = (esr >> 14) & 0x7;
2224 params.Op2 = (esr >> 17) & 0x7;
62a89c44 2225
dcaffa7b 2226 if (!emulate_cp(vcpu, &params, global, nr_global)) {
2ec5be3d
PF		 2227 if (!params.is_write)
2228 vcpu_set_reg(vcpu, Rt, params.regval);
72564016 2229 return 1;
2ec5be3d 2230 }
72564016
MZ		 2231
2232 unhandled_cp_access(vcpu, &params);
62a89c44
MZ		 2233 return 1;
2234}
2235
74cc7e0c 2236int kvm_handle_cp15_64(struct kvm_vcpu *vcpu)
72564016 2237{
dcaffa7b 2238 return kvm_handle_cp_64(vcpu, cp15_64_regs, ARRAY_SIZE(cp15_64_regs));
72564016
MZ		 2239}
2240
74cc7e0c 2241int kvm_handle_cp15_32(struct kvm_vcpu *vcpu)
72564016 2242{
dcaffa7b 2243 return kvm_handle_cp_32(vcpu, cp15_regs, ARRAY_SIZE(cp15_regs));
72564016
MZ		 2244}
2245
74cc7e0c 2246int kvm_handle_cp14_64(struct kvm_vcpu *vcpu)
72564016 2247{
dcaffa7b 2248 return kvm_handle_cp_64(vcpu, cp14_64_regs, ARRAY_SIZE(cp14_64_regs));
72564016
MZ		 2249}
2250
74cc7e0c 2251int kvm_handle_cp14_32(struct kvm_vcpu *vcpu)
72564016 2252{
dcaffa7b 2253 return kvm_handle_cp_32(vcpu, cp14_regs, ARRAY_SIZE(cp14_regs));
72564016
MZ		 2254}
2255
54ad68b7
MR		 2256static bool is_imp_def_sys_reg(struct sys_reg_params *params)
2257{
2258 // See ARM DDI 0487E.a, section D12.3.2
2259 return params->Op0 == 3 && (params->CRn & 0b1011) == 0b1011;
2260}
2261
7c8c5e6a 2262static int emulate_sys_reg(struct kvm_vcpu *vcpu,
3fec037d 2263 struct sys_reg_params *params)
7c8c5e6a 2264{
dcaffa7b 2265 const struct sys_reg_desc *r;
7c8c5e6a 2266
dcaffa7b 2267 r = find_reg(params, sys_reg_descs, ARRAY_SIZE(sys_reg_descs));
7c8c5e6a
MZ		 2268
2269 if (likely(r)) {
e70b9522 2270 perform_access(vcpu, params, r);
54ad68b7
MR		 2271 } else if (is_imp_def_sys_reg(params)) {
2272 kvm_inject_undefined(vcpu);
7c8c5e6a 2273 } else {
bf4b96bb
MR		 2274 print_sys_reg_msg(params,
2275 "Unsupported guest sys_reg access at: %lx [%08lx]\n",
2276 *vcpu_pc(vcpu), *vcpu_cpsr(vcpu));
e70b9522 2277 kvm_inject_undefined(vcpu);
7c8c5e6a 2278 }
7c8c5e6a
MZ		 2279 return 1;
2280}
2281
750ed566
JM		 2282/**
2283 * kvm_reset_sys_regs - sets system registers to reset value
2284 * @vcpu: The VCPU pointer
2285 *
2286 * This function finds the right table above and sets the registers on the
2287 * virtual CPU struct to their architecturally defined reset values.
2288 */
2289void kvm_reset_sys_regs(struct kvm_vcpu *vcpu)
7c8c5e6a
MZ		 2290{
2291 unsigned long i;
2292
750ed566
JM		 2293 for (i = 0; i < ARRAY_SIZE(sys_reg_descs); i++)
2294 if (sys_reg_descs[i].reset)
2295 sys_reg_descs[i].reset(vcpu, &sys_reg_descs[i]);
7c8c5e6a
MZ		 2296}
2297
2298/**
2299 * kvm_handle_sys_reg -- handles a mrs/msr trap on a guest sys_reg access
2300 * @vcpu: The VCPU pointer
7c8c5e6a 2301 */
74cc7e0c 2302int kvm_handle_sys_reg(struct kvm_vcpu *vcpu)
7c8c5e6a
MZ		 2303{
2304 struct sys_reg_params params;
3a949f4c 2305 unsigned long esr = kvm_vcpu_get_esr(vcpu);
c667186f 2306 int Rt = kvm_vcpu_sys_get_rt(vcpu);
2ec5be3d 2307 int ret;
7c8c5e6a 2308
eef8c85a
AB		 2309 trace_kvm_handle_sys_reg(esr);
2310
7c8c5e6a
MZ		 2311 params.Op0 = (esr >> 20) & 3;
2312 params.Op1 = (esr >> 14) & 0x7;
2313 params.CRn = (esr >> 10) & 0xf;
2314 params.CRm = (esr >> 1) & 0xf;
2315 params.Op2 = (esr >> 17) & 0x7;
2ec5be3d 2316 params.regval = vcpu_get_reg(vcpu, Rt);
7c8c5e6a
MZ		 2317 params.is_write = !(esr & 1);
2318
2ec5be3d
PF		 2319 ret = emulate_sys_reg(vcpu, &params);
2320
2321 if (!params.is_write)
2322 vcpu_set_reg(vcpu, Rt, params.regval);
2323 return ret;
7c8c5e6a
MZ		 2324}
2325
2326/******************************************************************************
2327 * Userspace API
2328 *****************************************************************************/
2329
2330static bool index_to_params(u64 id, struct sys_reg_params *params)
2331{
2332 switch (id & KVM_REG_SIZE_MASK) {
2333 case KVM_REG_SIZE_U64:
2334 /* Any unused index bits means it's not valid. */
2335 if (id & ~(KVM_REG_ARCH_MASK | KVM_REG_SIZE_MASK
2336 | KVM_REG_ARM_COPROC_MASK
2337 | KVM_REG_ARM64_SYSREG_OP0_MASK
2338 | KVM_REG_ARM64_SYSREG_OP1_MASK
2339 | KVM_REG_ARM64_SYSREG_CRN_MASK
2340 | KVM_REG_ARM64_SYSREG_CRM_MASK
2341 | KVM_REG_ARM64_SYSREG_OP2_MASK))
2342 return false;
2343 params->Op0 = ((id & KVM_REG_ARM64_SYSREG_OP0_MASK)
2344 >> KVM_REG_ARM64_SYSREG_OP0_SHIFT);
2345 params->Op1 = ((id & KVM_REG_ARM64_SYSREG_OP1_MASK)
2346 >> KVM_REG_ARM64_SYSREG_OP1_SHIFT);
2347 params->CRn = ((id & KVM_REG_ARM64_SYSREG_CRN_MASK)
2348 >> KVM_REG_ARM64_SYSREG_CRN_SHIFT);
2349 params->CRm = ((id & KVM_REG_ARM64_SYSREG_CRM_MASK)
2350 >> KVM_REG_ARM64_SYSREG_CRM_SHIFT);
2351 params->Op2 = ((id & KVM_REG_ARM64_SYSREG_OP2_MASK)
2352 >> KVM_REG_ARM64_SYSREG_OP2_SHIFT);
2353 return true;
2354 default:
2355 return false;
2356 }
2357}
2358
4b927b94
VK		 2359const struct sys_reg_desc *find_reg_by_id(u64 id,
2360 struct sys_reg_params *params,
2361 const struct sys_reg_desc table[],
2362 unsigned int num)
2363{
2364 if (!index_to_params(id, params))
2365 return NULL;
2366
2367 return find_reg(params, table, num);
2368}
2369
7c8c5e6a
MZ		 2370/* Decode an index value, and find the sys_reg_desc entry. */
2371static const struct sys_reg_desc *index_to_sys_reg_desc(struct kvm_vcpu *vcpu,
2372 u64 id)
2373{
dcaffa7b 2374 const struct sys_reg_desc *r;
7c8c5e6a
MZ		 2375 struct sys_reg_params params;
2376
2377 /* We only do sys_reg for now. */
2378 if ((id & KVM_REG_ARM_COPROC_MASK) != KVM_REG_ARM64_SYSREG)
2379 return NULL;
2380
1ce74e96
WD		 2381 if (!index_to_params(id, &params))
2382 return NULL;
2383
dcaffa7b 2384 r = find_reg(&params, sys_reg_descs, ARRAY_SIZE(sys_reg_descs));
7c8c5e6a 2385
93390c0a
DM		 2386 /* Not saved in the sys_reg array and not otherwise accessible? */
2387 if (r && !(r->reg || r->get_user))
7c8c5e6a
MZ		 2388 r = NULL;
2389
2390 return r;
2391}
2392
2393/*
2394 * These are the invariant sys_reg registers: we let the guest see the
2395 * host versions of these, so they're part of the guest state.
2396 *
2397 * A future CPU may provide a mechanism to present different values to
2398 * the guest, or a future kvm may trap them.
2399 */
2400
2401#define FUNCTION_INVARIANT(reg) \
2402 static void get_##reg(struct kvm_vcpu *v, \
2403 const struct sys_reg_desc *r) \
2404 { \
1f3d8699 2405 ((struct sys_reg_desc *)r)->val = read_sysreg(reg); \
7c8c5e6a
MZ		 2406 }
2407
2408FUNCTION_INVARIANT(midr_el1)
7c8c5e6a 2409FUNCTION_INVARIANT(revidr_el1)
7c8c5e6a
MZ		 2410FUNCTION_INVARIANT(clidr_el1)
2411FUNCTION_INVARIANT(aidr_el1)
2412
f7f2b15c
AB		 2413static void get_ctr_el0(struct kvm_vcpu *v, const struct sys_reg_desc *r)
2414{
2415 ((struct sys_reg_desc *)r)->val = read_sanitised_ftr_reg(SYS_CTR_EL0);
2416}
2417
7c8c5e6a
MZ		 2418/* ->val is filled in by kvm_sys_reg_table_init() */
2419static struct sys_reg_desc invariant_sys_regs[] = {
0d449541
MR		 2420 { SYS_DESC(SYS_MIDR_EL1), NULL, get_midr_el1 },
2421 { SYS_DESC(SYS_REVIDR_EL1), NULL, get_revidr_el1 },
0d449541
MR		 2422 { SYS_DESC(SYS_CLIDR_EL1), NULL, get_clidr_el1 },
2423 { SYS_DESC(SYS_AIDR_EL1), NULL, get_aidr_el1 },
2424 { SYS_DESC(SYS_CTR_EL0), NULL, get_ctr_el0 },
7c8c5e6a
MZ		 2425};
2426
26c99af1 2427static int reg_from_user(u64 *val, const void __user *uaddr, u64 id)
7c8c5e6a 2428{
7c8c5e6a
MZ		 2429 if (copy_from_user(val, uaddr, KVM_REG_SIZE(id)) != 0)
2430 return -EFAULT;
2431 return 0;
2432}
2433
26c99af1 2434static int reg_to_user(void __user *uaddr, const u64 *val, u64 id)
7c8c5e6a 2435{
7c8c5e6a
MZ		 2436 if (copy_to_user(uaddr, val, KVM_REG_SIZE(id)) != 0)
2437 return -EFAULT;
2438 return 0;
2439}
2440
2441static int get_invariant_sys_reg(u64 id, void __user *uaddr)
2442{
2443 struct sys_reg_params params;
2444 const struct sys_reg_desc *r;
2445
4b927b94
VK		 2446 r = find_reg_by_id(id, &params, invariant_sys_regs,
2447 ARRAY_SIZE(invariant_sys_regs));
7c8c5e6a
MZ		 2448 if (!r)
2449 return -ENOENT;
2450
2451 return reg_to_user(uaddr, &r->val, id);
2452}
2453
2454static int set_invariant_sys_reg(u64 id, void __user *uaddr)
2455{
2456 struct sys_reg_params params;
2457 const struct sys_reg_desc *r;
2458 int err;
2459 u64 val = 0; /* Make sure high bits are 0 for 32-bit regs */
2460
4b927b94
VK		 2461 r = find_reg_by_id(id, &params, invariant_sys_regs,
2462 ARRAY_SIZE(invariant_sys_regs));
7c8c5e6a
MZ		 2463 if (!r)
2464 return -ENOENT;
2465
2466 err = reg_from_user(&val, uaddr, id);
2467 if (err)
2468 return err;
2469
2470 /* This is what we mean by invariant: you can't change it. */
2471 if (r->val != val)
2472 return -EINVAL;
2473
2474 return 0;
2475}
2476
2477static bool is_valid_cache(u32 val)
2478{
2479 u32 level, ctype;
2480
2481 if (val >= CSSELR_MAX)
18d45766 2482 return false;
7c8c5e6a
MZ		 2483
2484 /* Bottom bit is Instruction or Data bit. Next 3 bits are level. */
2485 level = (val >> 1);
2486 ctype = (cache_levels >> (level * 3)) & 7;
2487
2488 switch (ctype) {
2489 case 0: /* No cache */
2490 return false;
2491 case 1: /* Instruction cache only */
2492 return (val & 1);
2493 case 2: /* Data cache only */
2494 case 4: /* Unified cache */
2495 return !(val & 1);
2496 case 3: /* Separate instruction and data caches */
2497 return true;
2498 default: /* Reserved: we can't know instruction or data. */
2499 return false;
2500 }
2501}
2502
2503static int demux_c15_get(u64 id, void __user *uaddr)
2504{
2505 u32 val;
2506 u32 __user *uval = uaddr;
2507
2508 /* Fail if we have unknown bits set. */
2509 if (id & ~(KVM_REG_ARCH_MASK|KVM_REG_SIZE_MASK|KVM_REG_ARM_COPROC_MASK
2510 | ((1 << KVM_REG_ARM_COPROC_SHIFT)-1)))
2511 return -ENOENT;
2512
2513 switch (id & KVM_REG_ARM_DEMUX_ID_MASK) {
2514 case KVM_REG_ARM_DEMUX_ID_CCSIDR:
2515 if (KVM_REG_SIZE(id) != 4)
2516 return -ENOENT;
2517 val = (id & KVM_REG_ARM_DEMUX_VAL_MASK)
2518 >> KVM_REG_ARM_DEMUX_VAL_SHIFT;
2519 if (!is_valid_cache(val))
2520 return -ENOENT;
2521
2522 return put_user(get_ccsidr(val), uval);
2523 default:
2524 return -ENOENT;
2525 }
2526}
2527
2528static int demux_c15_set(u64 id, void __user *uaddr)
2529{
2530 u32 val, newval;
2531 u32 __user *uval = uaddr;
2532
2533 /* Fail if we have unknown bits set. */
2534 if (id & ~(KVM_REG_ARCH_MASK|KVM_REG_SIZE_MASK|KVM_REG_ARM_COPROC_MASK
2535 | ((1 << KVM_REG_ARM_COPROC_SHIFT)-1)))
2536 return -ENOENT;
2537
2538 switch (id & KVM_REG_ARM_DEMUX_ID_MASK) {
2539 case KVM_REG_ARM_DEMUX_ID_CCSIDR:
2540 if (KVM_REG_SIZE(id) != 4)
2541 return -ENOENT;
2542 val = (id & KVM_REG_ARM_DEMUX_VAL_MASK)
2543 >> KVM_REG_ARM_DEMUX_VAL_SHIFT;
2544 if (!is_valid_cache(val))
2545 return -ENOENT;
2546
2547 if (get_user(newval, uval))
2548 return -EFAULT;
2549
2550 /* This is also invariant: you can't change it. */
2551 if (newval != get_ccsidr(val))
2552 return -EINVAL;
2553 return 0;
2554 default:
2555 return -ENOENT;
2556 }
2557}
2558
2559int kvm_arm_sys_reg_get_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg)
2560{
2561 const struct sys_reg_desc *r;
2562 void __user *uaddr = (void __user *)(unsigned long)reg->addr;
2563
2564 if ((reg->id & KVM_REG_ARM_COPROC_MASK) == KVM_REG_ARM_DEMUX)
2565 return demux_c15_get(reg->id, uaddr);
2566
2567 if (KVM_REG_SIZE(reg->id) != sizeof(__u64))
2568 return -ENOENT;
2569
2570 r = index_to_sys_reg_desc(vcpu, reg->id);
2571 if (!r)
2572 return get_invariant_sys_reg(reg->id, uaddr);
2573
7f34e409 2574 /* Check for regs disabled by runtime config */
01fe5ace 2575 if (sysreg_hidden(vcpu, r))
7f34e409
DM		 2576 return -ENOENT;
2577
84e690bf
AB		 2578 if (r->get_user)
2579 return (r->get_user)(vcpu, r, reg, uaddr);
2580
8d404c4c 2581 return reg_to_user(uaddr, &__vcpu_sys_reg(vcpu, r->reg), reg->id);
7c8c5e6a
MZ		 2582}
2583
2584int kvm_arm_sys_reg_set_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg)
2585{
2586 const struct sys_reg_desc *r;
2587 void __user *uaddr = (void __user *)(unsigned long)reg->addr;
2588
2589 if ((reg->id & KVM_REG_ARM_COPROC_MASK) == KVM_REG_ARM_DEMUX)
2590 return demux_c15_set(reg->id, uaddr);
2591
2592 if (KVM_REG_SIZE(reg->id) != sizeof(__u64))
2593 return -ENOENT;
2594
2595 r = index_to_sys_reg_desc(vcpu, reg->id);
2596 if (!r)
2597 return set_invariant_sys_reg(reg->id, uaddr);
2598
7f34e409 2599 /* Check for regs disabled by runtime config */
01fe5ace 2600 if (sysreg_hidden(vcpu, r))
7f34e409
DM		 2601 return -ENOENT;
2602
84e690bf
AB		 2603 if (r->set_user)
2604 return (r->set_user)(vcpu, r, reg, uaddr);
2605
8d404c4c 2606 return reg_from_user(&__vcpu_sys_reg(vcpu, r->reg), uaddr, reg->id);
7c8c5e6a
MZ		 2607}
2608
2609static unsigned int num_demux_regs(void)
2610{
2611 unsigned int i, count = 0;
2612
2613 for (i = 0; i < CSSELR_MAX; i++)
2614 if (is_valid_cache(i))
2615 count++;
2616
2617 return count;
2618}
2619
2620static int write_demux_regids(u64 __user *uindices)
2621{
efd48cea 2622 u64 val = KVM_REG_ARM64 | KVM_REG_SIZE_U32 | KVM_REG_ARM_DEMUX;
7c8c5e6a
MZ		 2623 unsigned int i;
2624
2625 val |= KVM_REG_ARM_DEMUX_ID_CCSIDR;
2626 for (i = 0; i < CSSELR_MAX; i++) {
2627 if (!is_valid_cache(i))
2628 continue;
2629 if (put_user(val | i, uindices))
2630 return -EFAULT;
2631 uindices++;
2632 }
2633 return 0;
2634}
2635
2636static u64 sys_reg_to_index(const struct sys_reg_desc *reg)
2637{
2638 return (KVM_REG_ARM64 | KVM_REG_SIZE_U64 |
2639 KVM_REG_ARM64_SYSREG |
2640 (reg->Op0 << KVM_REG_ARM64_SYSREG_OP0_SHIFT) |
2641 (reg->Op1 << KVM_REG_ARM64_SYSREG_OP1_SHIFT) |
2642 (reg->CRn << KVM_REG_ARM64_SYSREG_CRN_SHIFT) |
2643 (reg->CRm << KVM_REG_ARM64_SYSREG_CRM_SHIFT) |
2644 (reg->Op2 << KVM_REG_ARM64_SYSREG_OP2_SHIFT));
2645}
2646
2647static bool copy_reg_to_user(const struct sys_reg_desc *reg, u64 __user **uind)
2648{
2649 if (!*uind)
2650 return true;
2651
2652 if (put_user(sys_reg_to_index(reg), *uind))
2653 return false;
2654
2655 (*uind)++;
2656 return true;
2657}
2658
7f34e409
DM		 2659static int walk_one_sys_reg(const struct kvm_vcpu *vcpu,
2660 const struct sys_reg_desc *rd,
93390c0a
DM		 2661 u64 __user **uind,
2662 unsigned int *total)
2663{
2664 /*
2665 * Ignore registers we trap but don't save,
2666 * and for which no custom user accessor is provided.
2667 */
2668 if (!(rd->reg || rd->get_user))
2669 return 0;
2670
01fe5ace 2671 if (sysreg_hidden(vcpu, rd))
7f34e409
DM		 2672 return 0;
2673
93390c0a
DM		 2674 if (!copy_reg_to_user(rd, uind))
2675 return -EFAULT;
2676
2677 (*total)++;
2678 return 0;
2679}
2680
7c8c5e6a
MZ		 2681/* Assumed ordered tables, see kvm_sys_reg_table_init. */
2682static int walk_sys_regs(struct kvm_vcpu *vcpu, u64 __user *uind)
2683{
dcaffa7b 2684 const struct sys_reg_desc *i2, *end2;
7c8c5e6a 2685 unsigned int total = 0;
93390c0a 2686 int err;
7c8c5e6a 2687
7c8c5e6a
MZ		 2688 i2 = sys_reg_descs;
2689 end2 = sys_reg_descs + ARRAY_SIZE(sys_reg_descs);
2690
dcaffa7b
JM		 2691 while (i2 != end2) {
2692 err = walk_one_sys_reg(vcpu, i2++, &uind, &total);
93390c0a
DM		 2693 if (err)
2694 return err;
7c8c5e6a
MZ		 2695 }
2696 return total;
2697}
2698
2699unsigned long kvm_arm_num_sys_reg_descs(struct kvm_vcpu *vcpu)
2700{
2701 return ARRAY_SIZE(invariant_sys_regs)
2702 + num_demux_regs()
2703 + walk_sys_regs(vcpu, (u64 __user *)NULL);
2704}
2705
2706int kvm_arm_copy_sys_reg_indices(struct kvm_vcpu *vcpu, u64 __user *uindices)
2707{
2708 unsigned int i;
2709 int err;
2710
2711 /* Then give them all the invariant registers' indices. */
2712 for (i = 0; i < ARRAY_SIZE(invariant_sys_regs); i++) {
2713 if (put_user(sys_reg_to_index(&invariant_sys_regs[i]), uindices))
2714 return -EFAULT;
2715 uindices++;
2716 }
2717
2718 err = walk_sys_regs(vcpu, uindices);
2719 if (err < 0)
2720 return err;
2721 uindices += err;
2722
2723 return write_demux_regids(uindices);
2724}
2725
2726void kvm_sys_reg_table_init(void)
2727{
2728 unsigned int i;
2729 struct sys_reg_desc clidr;
2730
2731 /* Make sure tables are unique and in order. */
bb44a8db
MZ		 2732 BUG_ON(check_sysreg_table(sys_reg_descs, ARRAY_SIZE(sys_reg_descs), false));
2733 BUG_ON(check_sysreg_table(cp14_regs, ARRAY_SIZE(cp14_regs), true));
2734 BUG_ON(check_sysreg_table(cp14_64_regs, ARRAY_SIZE(cp14_64_regs), true));
2735 BUG_ON(check_sysreg_table(cp15_regs, ARRAY_SIZE(cp15_regs), true));
2736 BUG_ON(check_sysreg_table(cp15_64_regs, ARRAY_SIZE(cp15_64_regs), true));
2737 BUG_ON(check_sysreg_table(invariant_sys_regs, ARRAY_SIZE(invariant_sys_regs), false));
7c8c5e6a
MZ		 2738
2739 /* We abuse the reset function to overwrite the table itself. */
2740 for (i = 0; i < ARRAY_SIZE(invariant_sys_regs); i++)
2741 invariant_sys_regs[i].reset(NULL, &invariant_sys_regs[i]);
2742
2743 /*
2744 * CLIDR format is awkward, so clean it up. See ARM B4.1.20:
2745 *
2746 * If software reads the Cache Type fields from Ctype1
2747 * upwards, once it has seen a value of 0b000, no caches
2748 * exist at further-out levels of the hierarchy. So, for
2749 * example, if Ctype3 is the first Cache Type field with a
2750 * value of 0b000, the values of Ctype4 to Ctype7 must be
2751 * ignored.
2752 */
2753 get_clidr_el1(NULL, &clidr); /* Ugly... */
2754 cache_levels = clidr.val;
2755 for (i = 0; i < 7; i++)
2756 if (((cache_levels >> (i*3)) & 7) == 0)
2757 break;
2758 /* Clear all higher bits. */
2759 cache_levels &= (1 << (i*3))-1;
2760}
Ubuntu Hirsute Linux kernel mirror