]> git.proxmox.com Git - mirror_ubuntu-hirsute-kernel.git/blob - arch/arm64/kvm/sys_regs.c
projects / mirror_ubuntu-hirsute-kernel.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
UBUNTU: Ubuntu-5.11.0-22.23
[mirror_ubuntu-hirsute-kernel.git] / arch / arm64 / kvm / sys_regs.c
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copyright (C) 2012,2013 - ARM Ltd
4 * Author: Marc Zyngier <marc.zyngier@arm.com>
5 *
6 * Derived from arch/arm/kvm/coproc.c:
7 * Copyright (C) 2012 - Virtual Open Systems and Columbia University
8 * Authors: Rusty Russell <rusty@rustcorp.com.au>
9 * Christoffer Dall <c.dall@virtualopensystems.com>
10 */
11
12 #include <linux/bsearch.h>
13 #include <linux/kvm_host.h>
14 #include <linux/mm.h>
15 #include <linux/printk.h>
16 #include <linux/uaccess.h>
17
18 #include <asm/cacheflush.h>
19 #include <asm/cputype.h>
20 #include <asm/debug-monitors.h>
21 #include <asm/esr.h>
22 #include <asm/kvm_arm.h>
23 #include <asm/kvm_emulate.h>
24 #include <asm/kvm_hyp.h>
25 #include <asm/kvm_mmu.h>
26 #include <asm/perf_event.h>
27 #include <asm/sysreg.h>
28
29 #include <trace/events/kvm.h>
30
31 #include "sys_regs.h"
32
33 #include "trace.h"
34
35 /*
36 * All of this file is extremely similar to the ARM coproc.c, but the
37 * types are different. My gut feeling is that it should be pretty
38 * easy to merge, but that would be an ABI breakage -- again. VFP
39 * would also need to be abstracted.
40 *
41 * For AArch32, we only take care of what is being trapped. Anything
42 * that has to do with init and userspace access has to go via the
43 * 64bit interface.
44 */
45
46 #define reg_to_encoding(x) \
47 sys_reg((u32)(x)->Op0, (u32)(x)->Op1, \
48 (u32)(x)->CRn, (u32)(x)->CRm, (u32)(x)->Op2)
49
50 static bool read_from_write_only(struct kvm_vcpu *vcpu,
51 struct sys_reg_params *params,
52 const struct sys_reg_desc *r)
53 {
54 WARN_ONCE(1, "Unexpected sys_reg read to write-only register\n");
55 print_sys_reg_instr(params);
56 kvm_inject_undefined(vcpu);
57 return false;
58 }
59
60 static bool write_to_read_only(struct kvm_vcpu *vcpu,
61 struct sys_reg_params *params,
62 const struct sys_reg_desc *r)
63 {
64 WARN_ONCE(1, "Unexpected sys_reg write to read-only register\n");
65 print_sys_reg_instr(params);
66 kvm_inject_undefined(vcpu);
67 return false;
68 }
69
70 u64 vcpu_read_sys_reg(const struct kvm_vcpu *vcpu, int reg)
71 {
72 u64 val = 0x8badf00d8badf00d;
73
74 if (vcpu->arch.sysregs_loaded_on_cpu &&
75 __vcpu_read_sys_reg_from_cpu(reg, &val))
76 return val;
77
78 return __vcpu_sys_reg(vcpu, reg);
79 }
80
81 void vcpu_write_sys_reg(struct kvm_vcpu *vcpu, u64 val, int reg)
82 {
83 if (vcpu->arch.sysregs_loaded_on_cpu &&
84 __vcpu_write_sys_reg_to_cpu(val, reg))
85 return;
86
87 __vcpu_sys_reg(vcpu, reg) = val;
88 }
89
90 /* 3 bits per cache level, as per CLIDR, but non-existent caches always 0 */
91 static u32 cache_levels;
92
93 /* CSSELR values; used to index KVM_REG_ARM_DEMUX_ID_CCSIDR */
94 #define CSSELR_MAX 14
95
96 /* Which cache CCSIDR represents depends on CSSELR value. */
97 static u32 get_ccsidr(u32 csselr)
98 {
99 u32 ccsidr;
100
101 /* Make sure noone else changes CSSELR during this! */
102 local_irq_disable();
103 write_sysreg(csselr, csselr_el1);
104 isb();
105 ccsidr = read_sysreg(ccsidr_el1);
106 local_irq_enable();
107
108 return ccsidr;
109 }
110
111 /*
112 * See note at ARMv7 ARM B1.14.4 (TL;DR: S/W ops are not easily virtualized).
113 */
114 static bool access_dcsw(struct kvm_vcpu *vcpu,
115 struct sys_reg_params *p,
116 const struct sys_reg_desc *r)
117 {
118 if (!p->is_write)
119 return read_from_write_only(vcpu, p, r);
120
121 /*
122 * Only track S/W ops if we don't have FWB. It still indicates
123 * that the guest is a bit broken (S/W operations should only
124 * be done by firmware, knowing that there is only a single
125 * CPU left in the system, and certainly not from non-secure
126 * software).
127 */
128 if (!cpus_have_const_cap(ARM64_HAS_STAGE2_FWB))
129 kvm_set_way_flush(vcpu);
130
131 return true;
132 }
133
134 static void get_access_mask(const struct sys_reg_desc *r, u64 *mask, u64 *shift)
135 {
136 switch (r->aarch32_map) {
137 case AA32_LO:
138 *mask = GENMASK_ULL(31, 0);
139 *shift = 0;
140 break;
141 case AA32_HI:
142 *mask = GENMASK_ULL(63, 32);
143 *shift = 32;
144 break;
145 default:
146 *mask = GENMASK_ULL(63, 0);
147 *shift = 0;
148 break;
149 }
150 }
151
152 /*
153 * Generic accessor for VM registers. Only called as long as HCR_TVM
154 * is set. If the guest enables the MMU, we stop trapping the VM
155 * sys_regs and leave it in complete control of the caches.
156 */
157 static bool access_vm_reg(struct kvm_vcpu *vcpu,
158 struct sys_reg_params *p,
159 const struct sys_reg_desc *r)
160 {
161 bool was_enabled = vcpu_has_cache_enabled(vcpu);
162 u64 val, mask, shift;
163
164 BUG_ON(!p->is_write);
165
166 get_access_mask(r, &mask, &shift);
167
168 if (~mask) {
169 val = vcpu_read_sys_reg(vcpu, r->reg);
170 val &= ~mask;
171 } else {
172 val = 0;
173 }
174
175 val |= (p->regval & (mask >> shift)) << shift;
176 vcpu_write_sys_reg(vcpu, val, r->reg);
177
178 kvm_toggle_cache(vcpu, was_enabled);
179 return true;
180 }
181
182 static bool access_actlr(struct kvm_vcpu *vcpu,
183 struct sys_reg_params *p,
184 const struct sys_reg_desc *r)
185 {
186 u64 mask, shift;
187
188 if (p->is_write)
189 return ignore_write(vcpu, p);
190
191 get_access_mask(r, &mask, &shift);
192 p->regval = (vcpu_read_sys_reg(vcpu, r->reg) & mask) >> shift;
193
194 return true;
195 }
196
197 /*
198 * Trap handler for the GICv3 SGI generation system register.
199 * Forward the request to the VGIC emulation.
200 * The cp15_64 code makes sure this automatically works
201 * for both AArch64 and AArch32 accesses.
202 */
203 static bool access_gic_sgi(struct kvm_vcpu *vcpu,
204 struct sys_reg_params *p,
205 const struct sys_reg_desc *r)
206 {
207 bool g1;
208
209 if (!p->is_write)
210 return read_from_write_only(vcpu, p, r);
211
212 /*
213 * In a system where GICD_CTLR.DS=1, a ICC_SGI0R_EL1 access generates
214 * Group0 SGIs only, while ICC_SGI1R_EL1 can generate either group,
215 * depending on the SGI configuration. ICC_ASGI1R_EL1 is effectively
216 * equivalent to ICC_SGI0R_EL1, as there is no "alternative" secure
217 * group.
218 */
219 if (p->Op0 == 0) { /* AArch32 */
220 switch (p->Op1) {
221 default: /* Keep GCC quiet */
222 case 0: /* ICC_SGI1R */
223 g1 = true;
224 break;
225 case 1: /* ICC_ASGI1R */
226 case 2: /* ICC_SGI0R */
227 g1 = false;
228 break;
229 }
230 } else { /* AArch64 */
231 switch (p->Op2) {
232 default: /* Keep GCC quiet */
233 case 5: /* ICC_SGI1R_EL1 */
234 g1 = true;
235 break;
236 case 6: /* ICC_ASGI1R_EL1 */
237 case 7: /* ICC_SGI0R_EL1 */
238 g1 = false;
239 break;
240 }
241 }
242
243 vgic_v3_dispatch_sgi(vcpu, p->regval, g1);
244
245 return true;
246 }
247
248 static bool access_gic_sre(struct kvm_vcpu *vcpu,
249 struct sys_reg_params *p,
250 const struct sys_reg_desc *r)
251 {
252 if (p->is_write)
253 return ignore_write(vcpu, p);
254
255 p->regval = vcpu->arch.vgic_cpu.vgic_v3.vgic_sre;
256 return true;
257 }
258
259 static bool trap_raz_wi(struct kvm_vcpu *vcpu,
260 struct sys_reg_params *p,
261 const struct sys_reg_desc *r)
262 {
263 if (p->is_write)
264 return ignore_write(vcpu, p);
265 else
266 return read_zero(vcpu, p);
267 }
268
269 /*
270 * ARMv8.1 mandates at least a trivial LORegion implementation, where all the
271 * RW registers are RES0 (which we can implement as RAZ/WI). On an ARMv8.0
272 * system, these registers should UNDEF. LORID_EL1 being a RO register, we
273 * treat it separately.
274 */
275 static bool trap_loregion(struct kvm_vcpu *vcpu,
276 struct sys_reg_params *p,
277 const struct sys_reg_desc *r)
278 {
279 u64 val = read_sanitised_ftr_reg(SYS_ID_AA64MMFR1_EL1);
280 u32 sr = reg_to_encoding(r);
281
282 if (!(val & (0xfUL << ID_AA64MMFR1_LOR_SHIFT))) {
283 kvm_inject_undefined(vcpu);
284 return false;
285 }
286
287 if (p->is_write && sr == SYS_LORID_EL1)
288 return write_to_read_only(vcpu, p, r);
289
290 return trap_raz_wi(vcpu, p, r);
291 }
292
293 static bool trap_oslsr_el1(struct kvm_vcpu *vcpu,
294 struct sys_reg_params *p,
295 const struct sys_reg_desc *r)
296 {
297 if (p->is_write) {
298 return ignore_write(vcpu, p);
299 } else {
300 p->regval = (1 << 3);
301 return true;
302 }
303 }
304
305 static bool trap_dbgauthstatus_el1(struct kvm_vcpu *vcpu,
306 struct sys_reg_params *p,
307 const struct sys_reg_desc *r)
308 {
309 if (p->is_write) {
310 return ignore_write(vcpu, p);
311 } else {
312 p->regval = read_sysreg(dbgauthstatus_el1);
313 return true;
314 }
315 }
316
317 /*
318 * We want to avoid world-switching all the DBG registers all the
319 * time:
320 *
321 * - If we've touched any debug register, it is likely that we're
322 * going to touch more of them. It then makes sense to disable the
323 * traps and start doing the save/restore dance
324 * - If debug is active (DBG_MDSCR_KDE or DBG_MDSCR_MDE set), it is
325 * then mandatory to save/restore the registers, as the guest
326 * depends on them.
327 *
328 * For this, we use a DIRTY bit, indicating the guest has modified the
329 * debug registers, used as follow:
330 *
331 * On guest entry:
332 * - If the dirty bit is set (because we're coming back from trapping),
333 * disable the traps, save host registers, restore guest registers.
334 * - If debug is actively in use (DBG_MDSCR_KDE or DBG_MDSCR_MDE set),
335 * set the dirty bit, disable the traps, save host registers,
336 * restore guest registers.
337 * - Otherwise, enable the traps
338 *
339 * On guest exit:
340 * - If the dirty bit is set, save guest registers, restore host
341 * registers and clear the dirty bit. This ensure that the host can
342 * now use the debug registers.
343 */
344 static bool trap_debug_regs(struct kvm_vcpu *vcpu,
345 struct sys_reg_params *p,
346 const struct sys_reg_desc *r)
347 {
348 if (p->is_write) {
349 vcpu_write_sys_reg(vcpu, p->regval, r->reg);
350 vcpu->arch.flags |= KVM_ARM64_DEBUG_DIRTY;
351 } else {
352 p->regval = vcpu_read_sys_reg(vcpu, r->reg);
353 }
354
355 trace_trap_reg(__func__, r->reg, p->is_write, p->regval);
356
357 return true;
358 }
359
360 /*
361 * reg_to_dbg/dbg_to_reg
362 *
363 * A 32 bit write to a debug register leave top bits alone
364 * A 32 bit read from a debug register only returns the bottom bits
365 *
366 * All writes will set the KVM_ARM64_DEBUG_DIRTY flag to ensure the
367 * hyp.S code switches between host and guest values in future.
368 */
369 static void reg_to_dbg(struct kvm_vcpu *vcpu,
370 struct sys_reg_params *p,
371 const struct sys_reg_desc *rd,
372 u64 *dbg_reg)
373 {
374 u64 mask, shift, val;
375
376 get_access_mask(rd, &mask, &shift);
377
378 val = *dbg_reg;
379 val &= ~mask;
380 val |= (p->regval & (mask >> shift)) << shift;
381 *dbg_reg = val;
382
383 vcpu->arch.flags |= KVM_ARM64_DEBUG_DIRTY;
384 }
385
386 static void dbg_to_reg(struct kvm_vcpu *vcpu,
387 struct sys_reg_params *p,
388 const struct sys_reg_desc *rd,
389 u64 *dbg_reg)
390 {
391 u64 mask, shift;
392
393 get_access_mask(rd, &mask, &shift);
394 p->regval = (*dbg_reg & mask) >> shift;
395 }
396
397 static bool trap_bvr(struct kvm_vcpu *vcpu,
398 struct sys_reg_params *p,
399 const struct sys_reg_desc *rd)
400 {
401 u64 *dbg_reg = &vcpu->arch.vcpu_debug_state.dbg_bvr[rd->reg];
402
403 if (p->is_write)
404 reg_to_dbg(vcpu, p, rd, dbg_reg);
405 else
406 dbg_to_reg(vcpu, p, rd, dbg_reg);
407
408 trace_trap_reg(__func__, rd->reg, p->is_write, *dbg_reg);
409
410 return true;
411 }
412
413 static int set_bvr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
414 const struct kvm_one_reg *reg, void __user *uaddr)
415 {
416 __u64 *r = &vcpu->arch.vcpu_debug_state.dbg_bvr[rd->reg];
417
418 if (copy_from_user(r, uaddr, KVM_REG_SIZE(reg->id)) != 0)
419 return -EFAULT;
420 return 0;
421 }
422
423 static int get_bvr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
424 const struct kvm_one_reg *reg, void __user *uaddr)
425 {
426 __u64 *r = &vcpu->arch.vcpu_debug_state.dbg_bvr[rd->reg];
427
428 if (copy_to_user(uaddr, r, KVM_REG_SIZE(reg->id)) != 0)
429 return -EFAULT;
430 return 0;
431 }
432
433 static void reset_bvr(struct kvm_vcpu *vcpu,
434 const struct sys_reg_desc *rd)
435 {
436 vcpu->arch.vcpu_debug_state.dbg_bvr[rd->reg] = rd->val;
437 }
438
439 static bool trap_bcr(struct kvm_vcpu *vcpu,
440 struct sys_reg_params *p,
441 const struct sys_reg_desc *rd)
442 {
443 u64 *dbg_reg = &vcpu->arch.vcpu_debug_state.dbg_bcr[rd->reg];
444
445 if (p->is_write)
446 reg_to_dbg(vcpu, p, rd, dbg_reg);
447 else
448 dbg_to_reg(vcpu, p, rd, dbg_reg);
449
450 trace_trap_reg(__func__, rd->reg, p->is_write, *dbg_reg);
451
452 return true;
453 }
454
455 static int set_bcr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
456 const struct kvm_one_reg *reg, void __user *uaddr)
457 {
458 __u64 *r = &vcpu->arch.vcpu_debug_state.dbg_bcr[rd->reg];
459
460 if (copy_from_user(r, uaddr, KVM_REG_SIZE(reg->id)) != 0)
461 return -EFAULT;
462
463 return 0;
464 }
465
466 static int get_bcr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
467 const struct kvm_one_reg *reg, void __user *uaddr)
468 {
469 __u64 *r = &vcpu->arch.vcpu_debug_state.dbg_bcr[rd->reg];
470
471 if (copy_to_user(uaddr, r, KVM_REG_SIZE(reg->id)) != 0)
472 return -EFAULT;
473 return 0;
474 }
475
476 static void reset_bcr(struct kvm_vcpu *vcpu,
477 const struct sys_reg_desc *rd)
478 {
479 vcpu->arch.vcpu_debug_state.dbg_bcr[rd->reg] = rd->val;
480 }
481
482 static bool trap_wvr(struct kvm_vcpu *vcpu,
483 struct sys_reg_params *p,
484 const struct sys_reg_desc *rd)
485 {
486 u64 *dbg_reg = &vcpu->arch.vcpu_debug_state.dbg_wvr[rd->reg];
487
488 if (p->is_write)
489 reg_to_dbg(vcpu, p, rd, dbg_reg);
490 else
491 dbg_to_reg(vcpu, p, rd, dbg_reg);
492
493 trace_trap_reg(__func__, rd->reg, p->is_write,
494 vcpu->arch.vcpu_debug_state.dbg_wvr[rd->reg]);
495
496 return true;
497 }
498
499 static int set_wvr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
500 const struct kvm_one_reg *reg, void __user *uaddr)
501 {
502 __u64 *r = &vcpu->arch.vcpu_debug_state.dbg_wvr[rd->reg];
503
504 if (copy_from_user(r, uaddr, KVM_REG_SIZE(reg->id)) != 0)
505 return -EFAULT;
506 return 0;
507 }
508
509 static int get_wvr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
510 const struct kvm_one_reg *reg, void __user *uaddr)
511 {
512 __u64 *r = &vcpu->arch.vcpu_debug_state.dbg_wvr[rd->reg];
513
514 if (copy_to_user(uaddr, r, KVM_REG_SIZE(reg->id)) != 0)
515 return -EFAULT;
516 return 0;
517 }
518
519 static void reset_wvr(struct kvm_vcpu *vcpu,
520 const struct sys_reg_desc *rd)
521 {
522 vcpu->arch.vcpu_debug_state.dbg_wvr[rd->reg] = rd->val;
523 }
524
525 static bool trap_wcr(struct kvm_vcpu *vcpu,
526 struct sys_reg_params *p,
527 const struct sys_reg_desc *rd)
528 {
529 u64 *dbg_reg = &vcpu->arch.vcpu_debug_state.dbg_wcr[rd->reg];
530
531 if (p->is_write)
532 reg_to_dbg(vcpu, p, rd, dbg_reg);
533 else
534 dbg_to_reg(vcpu, p, rd, dbg_reg);
535
536 trace_trap_reg(__func__, rd->reg, p->is_write, *dbg_reg);
537
538 return true;
539 }
540
541 static int set_wcr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
542 const struct kvm_one_reg *reg, void __user *uaddr)
543 {
544 __u64 *r = &vcpu->arch.vcpu_debug_state.dbg_wcr[rd->reg];
545
546 if (copy_from_user(r, uaddr, KVM_REG_SIZE(reg->id)) != 0)
547 return -EFAULT;
548 return 0;
549 }
550
551 static int get_wcr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
552 const struct kvm_one_reg *reg, void __user *uaddr)
553 {
554 __u64 *r = &vcpu->arch.vcpu_debug_state.dbg_wcr[rd->reg];
555
556 if (copy_to_user(uaddr, r, KVM_REG_SIZE(reg->id)) != 0)
557 return -EFAULT;
558 return 0;
559 }
560
561 static void reset_wcr(struct kvm_vcpu *vcpu,
562 const struct sys_reg_desc *rd)
563 {
564 vcpu->arch.vcpu_debug_state.dbg_wcr[rd->reg] = rd->val;
565 }
566
567 static void reset_amair_el1(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r)
568 {
569 u64 amair = read_sysreg(amair_el1);
570 vcpu_write_sys_reg(vcpu, amair, AMAIR_EL1);
571 }
572
573 static void reset_actlr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r)
574 {
575 u64 actlr = read_sysreg(actlr_el1);
576 vcpu_write_sys_reg(vcpu, actlr, ACTLR_EL1);
577 }
578
579 static void reset_mpidr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r)
580 {
581 u64 mpidr;
582
583 /*
584 * Map the vcpu_id into the first three affinity level fields of
585 * the MPIDR. We limit the number of VCPUs in level 0 due to a
586 * limitation to 16 CPUs in that level in the ICC_SGIxR registers
587 * of the GICv3 to be able to address each CPU directly when
588 * sending IPIs.
589 */
590 mpidr = (vcpu->vcpu_id & 0x0f) << MPIDR_LEVEL_SHIFT(0);
591 mpidr |= ((vcpu->vcpu_id >> 4) & 0xff) << MPIDR_LEVEL_SHIFT(1);
592 mpidr |= ((vcpu->vcpu_id >> 12) & 0xff) << MPIDR_LEVEL_SHIFT(2);
593 vcpu_write_sys_reg(vcpu, (1ULL << 31) | mpidr, MPIDR_EL1);
594 }
595
596 static unsigned int pmu_visibility(const struct kvm_vcpu *vcpu,
597 const struct sys_reg_desc *r)
598 {
599 if (kvm_vcpu_has_pmu(vcpu))
600 return 0;
601
602 return REG_HIDDEN;
603 }
604
605 static void reset_pmcr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r)
606 {
607 u64 pmcr, val;
608
609 /* No PMU available, PMCR_EL0 may UNDEF... */
610 if (!kvm_arm_support_pmu_v3())
611 return;
612
613 pmcr = read_sysreg(pmcr_el0);
614 /*
615 * Writable bits of PMCR_EL0 (ARMV8_PMU_PMCR_MASK) are reset to UNKNOWN
616 * except PMCR.E resetting to zero.
617 */
618 val = ((pmcr & ~ARMV8_PMU_PMCR_MASK)
619 | (ARMV8_PMU_PMCR_MASK & 0xdecafbad)) & (~ARMV8_PMU_PMCR_E);
620 if (!system_supports_32bit_el0())
621 val |= ARMV8_PMU_PMCR_LC;
622 __vcpu_sys_reg(vcpu, r->reg) = val;
623 }
624
625 static bool check_pmu_access_disabled(struct kvm_vcpu *vcpu, u64 flags)
626 {
627 u64 reg = __vcpu_sys_reg(vcpu, PMUSERENR_EL0);
628 bool enabled = (reg & flags) || vcpu_mode_priv(vcpu);
629
630 if (!enabled)
631 kvm_inject_undefined(vcpu);
632
633 return !enabled;
634 }
635
636 static bool pmu_access_el0_disabled(struct kvm_vcpu *vcpu)
637 {
638 return check_pmu_access_disabled(vcpu, ARMV8_PMU_USERENR_EN);
639 }
640
641 static bool pmu_write_swinc_el0_disabled(struct kvm_vcpu *vcpu)
642 {
643 return check_pmu_access_disabled(vcpu, ARMV8_PMU_USERENR_SW | ARMV8_PMU_USERENR_EN);
644 }
645
646 static bool pmu_access_cycle_counter_el0_disabled(struct kvm_vcpu *vcpu)
647 {
648 return check_pmu_access_disabled(vcpu, ARMV8_PMU_USERENR_CR | ARMV8_PMU_USERENR_EN);
649 }
650
651 static bool pmu_access_event_counter_el0_disabled(struct kvm_vcpu *vcpu)
652 {
653 return check_pmu_access_disabled(vcpu, ARMV8_PMU_USERENR_ER | ARMV8_PMU_USERENR_EN);
654 }
655
656 static bool access_pmcr(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
657 const struct sys_reg_desc *r)
658 {
659 u64 val;
660
661 if (pmu_access_el0_disabled(vcpu))
662 return false;
663
664 if (p->is_write) {
665 /* Only update writeable bits of PMCR */
666 val = __vcpu_sys_reg(vcpu, PMCR_EL0);
667 val &= ~ARMV8_PMU_PMCR_MASK;
668 val |= p->regval & ARMV8_PMU_PMCR_MASK;
669 if (!system_supports_32bit_el0())
670 val |= ARMV8_PMU_PMCR_LC;
671 __vcpu_sys_reg(vcpu, PMCR_EL0) = val;
672 kvm_pmu_handle_pmcr(vcpu, val);
673 kvm_vcpu_pmu_restore_guest(vcpu);
674 } else {
675 /* PMCR.P & PMCR.C are RAZ */
676 val = __vcpu_sys_reg(vcpu, PMCR_EL0)
677 & ~(ARMV8_PMU_PMCR_P | ARMV8_PMU_PMCR_C);
678 p->regval = val;
679 }
680
681 return true;
682 }
683
684 static bool access_pmselr(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
685 const struct sys_reg_desc *r)
686 {
687 if (pmu_access_event_counter_el0_disabled(vcpu))
688 return false;
689
690 if (p->is_write)
691 __vcpu_sys_reg(vcpu, PMSELR_EL0) = p->regval;
692 else
693 /* return PMSELR.SEL field */
694 p->regval = __vcpu_sys_reg(vcpu, PMSELR_EL0)
695 & ARMV8_PMU_COUNTER_MASK;
696
697 return true;
698 }
699
700 static bool access_pmceid(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
701 const struct sys_reg_desc *r)
702 {
703 u64 pmceid;
704
705 BUG_ON(p->is_write);
706
707 if (pmu_access_el0_disabled(vcpu))
708 return false;
709
710 pmceid = kvm_pmu_get_pmceid(vcpu, (p->Op2 & 1));
711
712 p->regval = pmceid;
713
714 return true;
715 }
716
717 static bool pmu_counter_idx_valid(struct kvm_vcpu *vcpu, u64 idx)
718 {
719 u64 pmcr, val;
720
721 pmcr = __vcpu_sys_reg(vcpu, PMCR_EL0);
722 val = (pmcr >> ARMV8_PMU_PMCR_N_SHIFT) & ARMV8_PMU_PMCR_N_MASK;
723 if (idx >= val && idx != ARMV8_PMU_CYCLE_IDX) {
724 kvm_inject_undefined(vcpu);
725 return false;
726 }
727
728 return true;
729 }
730
731 static bool access_pmu_evcntr(struct kvm_vcpu *vcpu,
732 struct sys_reg_params *p,
733 const struct sys_reg_desc *r)
734 {
735 u64 idx = ~0UL;
736
737 if (r->CRn == 9 && r->CRm == 13) {
738 if (r->Op2 == 2) {
739 /* PMXEVCNTR_EL0 */
740 if (pmu_access_event_counter_el0_disabled(vcpu))
741 return false;
742
743 idx = __vcpu_sys_reg(vcpu, PMSELR_EL0)
744 & ARMV8_PMU_COUNTER_MASK;
745 } else if (r->Op2 == 0) {
746 /* PMCCNTR_EL0 */
747 if (pmu_access_cycle_counter_el0_disabled(vcpu))
748 return false;
749
750 idx = ARMV8_PMU_CYCLE_IDX;
751 }
752 } else if (r->CRn == 0 && r->CRm == 9) {
753 /* PMCCNTR */
754 if (pmu_access_event_counter_el0_disabled(vcpu))
755 return false;
756
757 idx = ARMV8_PMU_CYCLE_IDX;
758 } else if (r->CRn == 14 && (r->CRm & 12) == 8) {
759 /* PMEVCNTRn_EL0 */
760 if (pmu_access_event_counter_el0_disabled(vcpu))
761 return false;
762
763 idx = ((r->CRm & 3) << 3) | (r->Op2 & 7);
764 }
765
766 /* Catch any decoding mistake */
767 WARN_ON(idx == ~0UL);
768
769 if (!pmu_counter_idx_valid(vcpu, idx))
770 return false;
771
772 if (p->is_write) {
773 if (pmu_access_el0_disabled(vcpu))
774 return false;
775
776 kvm_pmu_set_counter_value(vcpu, idx, p->regval);
777 } else {
778 p->regval = kvm_pmu_get_counter_value(vcpu, idx);
779 }
780
781 return true;
782 }
783
784 static bool access_pmu_evtyper(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
785 const struct sys_reg_desc *r)
786 {
787 u64 idx, reg;
788
789 if (pmu_access_el0_disabled(vcpu))
790 return false;
791
792 if (r->CRn == 9 && r->CRm == 13 && r->Op2 == 1) {
793 /* PMXEVTYPER_EL0 */
794 idx = __vcpu_sys_reg(vcpu, PMSELR_EL0) & ARMV8_PMU_COUNTER_MASK;
795 reg = PMEVTYPER0_EL0 + idx;
796 } else if (r->CRn == 14 && (r->CRm & 12) == 12) {
797 idx = ((r->CRm & 3) << 3) | (r->Op2 & 7);
798 if (idx == ARMV8_PMU_CYCLE_IDX)
799 reg = PMCCFILTR_EL0;
800 else
801 /* PMEVTYPERn_EL0 */
802 reg = PMEVTYPER0_EL0 + idx;
803 } else {
804 BUG();
805 }
806
807 if (!pmu_counter_idx_valid(vcpu, idx))
808 return false;
809
810 if (p->is_write) {
811 kvm_pmu_set_counter_event_type(vcpu, p->regval, idx);
812 __vcpu_sys_reg(vcpu, reg) = p->regval & ARMV8_PMU_EVTYPE_MASK;
813 kvm_vcpu_pmu_restore_guest(vcpu);
814 } else {
815 p->regval = __vcpu_sys_reg(vcpu, reg) & ARMV8_PMU_EVTYPE_MASK;
816 }
817
818 return true;
819 }
820
821 static bool access_pmcnten(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
822 const struct sys_reg_desc *r)
823 {
824 u64 val, mask;
825
826 if (pmu_access_el0_disabled(vcpu))
827 return false;
828
829 mask = kvm_pmu_valid_counter_mask(vcpu);
830 if (p->is_write) {
831 val = p->regval & mask;
832 if (r->Op2 & 0x1) {
833 /* accessing PMCNTENSET_EL0 */
834 __vcpu_sys_reg(vcpu, PMCNTENSET_EL0) |= val;
835 kvm_pmu_enable_counter_mask(vcpu, val);
836 kvm_vcpu_pmu_restore_guest(vcpu);
837 } else {
838 /* accessing PMCNTENCLR_EL0 */
839 __vcpu_sys_reg(vcpu, PMCNTENSET_EL0) &= ~val;
840 kvm_pmu_disable_counter_mask(vcpu, val);
841 }
842 } else {
843 p->regval = __vcpu_sys_reg(vcpu, PMCNTENSET_EL0) & mask;
844 }
845
846 return true;
847 }
848
849 static bool access_pminten(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
850 const struct sys_reg_desc *r)
851 {
852 u64 mask = kvm_pmu_valid_counter_mask(vcpu);
853
854 if (check_pmu_access_disabled(vcpu, 0))
855 return false;
856
857 if (p->is_write) {
858 u64 val = p->regval & mask;
859
860 if (r->Op2 & 0x1)
861 /* accessing PMINTENSET_EL1 */
862 __vcpu_sys_reg(vcpu, PMINTENSET_EL1) |= val;
863 else
864 /* accessing PMINTENCLR_EL1 */
865 __vcpu_sys_reg(vcpu, PMINTENSET_EL1) &= ~val;
866 } else {
867 p->regval = __vcpu_sys_reg(vcpu, PMINTENSET_EL1) & mask;
868 }
869
870 return true;
871 }
872
873 static bool access_pmovs(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
874 const struct sys_reg_desc *r)
875 {
876 u64 mask = kvm_pmu_valid_counter_mask(vcpu);
877
878 if (pmu_access_el0_disabled(vcpu))
879 return false;
880
881 if (p->is_write) {
882 if (r->CRm & 0x2)
883 /* accessing PMOVSSET_EL0 */
884 __vcpu_sys_reg(vcpu, PMOVSSET_EL0) |= (p->regval & mask);
885 else
886 /* accessing PMOVSCLR_EL0 */
887 __vcpu_sys_reg(vcpu, PMOVSSET_EL0) &= ~(p->regval & mask);
888 } else {
889 p->regval = __vcpu_sys_reg(vcpu, PMOVSSET_EL0) & mask;
890 }
891
892 return true;
893 }
894
895 static bool access_pmswinc(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
896 const struct sys_reg_desc *r)
897 {
898 u64 mask;
899
900 if (!p->is_write)
901 return read_from_write_only(vcpu, p, r);
902
903 if (pmu_write_swinc_el0_disabled(vcpu))
904 return false;
905
906 mask = kvm_pmu_valid_counter_mask(vcpu);
907 kvm_pmu_software_increment(vcpu, p->regval & mask);
908 return true;
909 }
910
911 static bool access_pmuserenr(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
912 const struct sys_reg_desc *r)
913 {
914 if (p->is_write) {
915 if (!vcpu_mode_priv(vcpu)) {
916 kvm_inject_undefined(vcpu);
917 return false;
918 }
919
920 __vcpu_sys_reg(vcpu, PMUSERENR_EL0) =
921 p->regval & ARMV8_PMU_USERENR_MASK;
922 } else {
923 p->regval = __vcpu_sys_reg(vcpu, PMUSERENR_EL0)
924 & ARMV8_PMU_USERENR_MASK;
925 }
926
927 return true;
928 }
929
930 /* Silly macro to expand the DBG{BCR,BVR,WVR,WCR}n_EL1 registers in one go */
931 #define DBG_BCR_BVR_WCR_WVR_EL1(n) \
932 { SYS_DESC(SYS_DBGBVRn_EL1(n)), \
933 trap_bvr, reset_bvr, 0, 0, get_bvr, set_bvr }, \
934 { SYS_DESC(SYS_DBGBCRn_EL1(n)), \
935 trap_bcr, reset_bcr, 0, 0, get_bcr, set_bcr }, \
936 { SYS_DESC(SYS_DBGWVRn_EL1(n)), \
937 trap_wvr, reset_wvr, 0, 0, get_wvr, set_wvr }, \
938 { SYS_DESC(SYS_DBGWCRn_EL1(n)), \
939 trap_wcr, reset_wcr, 0, 0, get_wcr, set_wcr }
940
941 #define PMU_SYS_REG(r) \
942 SYS_DESC(r), .reset = reset_unknown, .visibility = pmu_visibility
943
944 /* Macro to expand the PMEVCNTRn_EL0 register */
945 #define PMU_PMEVCNTR_EL0(n) \
946 { PMU_SYS_REG(SYS_PMEVCNTRn_EL0(n)), \
947 .access = access_pmu_evcntr, .reg = (PMEVCNTR0_EL0 + n), }
948
949 /* Macro to expand the PMEVTYPERn_EL0 register */
950 #define PMU_PMEVTYPER_EL0(n) \
951 { PMU_SYS_REG(SYS_PMEVTYPERn_EL0(n)), \
952 .access = access_pmu_evtyper, .reg = (PMEVTYPER0_EL0 + n), }
953
954 static bool undef_access(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
955 const struct sys_reg_desc *r)
956 {
957 kvm_inject_undefined(vcpu);
958
959 return false;
960 }
961
962 /* Macro to expand the AMU counter and type registers*/
963 #define AMU_AMEVCNTR0_EL0(n) { SYS_DESC(SYS_AMEVCNTR0_EL0(n)), undef_access }
964 #define AMU_AMEVTYPER0_EL0(n) { SYS_DESC(SYS_AMEVTYPER0_EL0(n)), undef_access }
965 #define AMU_AMEVCNTR1_EL0(n) { SYS_DESC(SYS_AMEVCNTR1_EL0(n)), undef_access }
966 #define AMU_AMEVTYPER1_EL0(n) { SYS_DESC(SYS_AMEVTYPER1_EL0(n)), undef_access }
967
968 static unsigned int ptrauth_visibility(const struct kvm_vcpu *vcpu,
969 const struct sys_reg_desc *rd)
970 {
971 return vcpu_has_ptrauth(vcpu) ? 0 : REG_HIDDEN;
972 }
973
974 /*
975 * If we land here on a PtrAuth access, that is because we didn't
976 * fixup the access on exit by allowing the PtrAuth sysregs. The only
977 * way this happens is when the guest does not have PtrAuth support
978 * enabled.
979 */
980 #define __PTRAUTH_KEY(k) \
981 { SYS_DESC(SYS_## k), undef_access, reset_unknown, k, \
982 .visibility = ptrauth_visibility}
983
984 #define PTRAUTH_KEY(k) \
985 __PTRAUTH_KEY(k ## KEYLO_EL1), \
986 __PTRAUTH_KEY(k ## KEYHI_EL1)
987
988 static bool access_arch_timer(struct kvm_vcpu *vcpu,
989 struct sys_reg_params *p,
990 const struct sys_reg_desc *r)
991 {
992 enum kvm_arch_timers tmr;
993 enum kvm_arch_timer_regs treg;
994 u64 reg = reg_to_encoding(r);
995
996 switch (reg) {
997 case SYS_CNTP_TVAL_EL0:
998 case SYS_AARCH32_CNTP_TVAL:
999 tmr = TIMER_PTIMER;
1000 treg = TIMER_REG_TVAL;
1001 break;
1002 case SYS_CNTP_CTL_EL0:
1003 case SYS_AARCH32_CNTP_CTL:
1004 tmr = TIMER_PTIMER;
1005 treg = TIMER_REG_CTL;
1006 break;
1007 case SYS_CNTP_CVAL_EL0:
1008 case SYS_AARCH32_CNTP_CVAL:
1009 tmr = TIMER_PTIMER;
1010 treg = TIMER_REG_CVAL;
1011 break;
1012 default:
1013 BUG();
1014 }
1015
1016 if (p->is_write)
1017 kvm_arm_timer_write_sysreg(vcpu, tmr, treg, p->regval);
1018 else
1019 p->regval = kvm_arm_timer_read_sysreg(vcpu, tmr, treg);
1020
1021 return true;
1022 }
1023
1024 /* Read a sanitised cpufeature ID register by sys_reg_desc */
1025 static u64 read_id_reg(const struct kvm_vcpu *vcpu,
1026 struct sys_reg_desc const *r, bool raz)
1027 {
1028 u32 id = reg_to_encoding(r);
1029 u64 val = raz ? 0 : read_sanitised_ftr_reg(id);
1030
1031 if (id == SYS_ID_AA64PFR0_EL1) {
1032 if (!vcpu_has_sve(vcpu))
1033 val &= ~(0xfUL << ID_AA64PFR0_SVE_SHIFT);
1034 val &= ~(0xfUL << ID_AA64PFR0_AMU_SHIFT);
1035 val &= ~(0xfUL << ID_AA64PFR0_CSV2_SHIFT);
1036 val |= ((u64)vcpu->kvm->arch.pfr0_csv2 << ID_AA64PFR0_CSV2_SHIFT);
1037 val &= ~(0xfUL << ID_AA64PFR0_CSV3_SHIFT);
1038 val |= ((u64)vcpu->kvm->arch.pfr0_csv3 << ID_AA64PFR0_CSV3_SHIFT);
1039 } else if (id == SYS_ID_AA64PFR1_EL1) {
1040 val &= ~(0xfUL << ID_AA64PFR1_MTE_SHIFT);
1041 } else if (id == SYS_ID_AA64ISAR1_EL1 && !vcpu_has_ptrauth(vcpu)) {
1042 val &= ~((0xfUL << ID_AA64ISAR1_APA_SHIFT) |
1043 (0xfUL << ID_AA64ISAR1_API_SHIFT) |
1044 (0xfUL << ID_AA64ISAR1_GPA_SHIFT) |
1045 (0xfUL << ID_AA64ISAR1_GPI_SHIFT));
1046 } else if (id == SYS_ID_AA64DFR0_EL1) {
1047 u64 cap = 0;
1048
1049 /* Limit guests to PMUv3 for ARMv8.1 */
1050 if (kvm_vcpu_has_pmu(vcpu))
1051 cap = ID_AA64DFR0_PMUVER_8_1;
1052
1053 val = cpuid_feature_cap_perfmon_field(val,
1054 ID_AA64DFR0_PMUVER_SHIFT,
1055 cap);
1056 } else if (id == SYS_ID_DFR0_EL1) {
1057 /* Limit guests to PMUv3 for ARMv8.1 */
1058 val = cpuid_feature_cap_perfmon_field(val,
1059 ID_DFR0_PERFMON_SHIFT,
1060 ID_DFR0_PERFMON_8_1);
1061 }
1062
1063 return val;
1064 }
1065
1066 static unsigned int id_visibility(const struct kvm_vcpu *vcpu,
1067 const struct sys_reg_desc *r)
1068 {
1069 u32 id = reg_to_encoding(r);
1070
1071 switch (id) {
1072 case SYS_ID_AA64ZFR0_EL1:
1073 if (!vcpu_has_sve(vcpu))
1074 return REG_RAZ;
1075 break;
1076 }
1077
1078 return 0;
1079 }
1080
1081 /* cpufeature ID register access trap handlers */
1082
1083 static bool __access_id_reg(struct kvm_vcpu *vcpu,
1084 struct sys_reg_params *p,
1085 const struct sys_reg_desc *r,
1086 bool raz)
1087 {
1088 if (p->is_write)
1089 return write_to_read_only(vcpu, p, r);
1090
1091 p->regval = read_id_reg(vcpu, r, raz);
1092 return true;
1093 }
1094
1095 static bool access_id_reg(struct kvm_vcpu *vcpu,
1096 struct sys_reg_params *p,
1097 const struct sys_reg_desc *r)
1098 {
1099 bool raz = sysreg_visible_as_raz(vcpu, r);
1100
1101 return __access_id_reg(vcpu, p, r, raz);
1102 }
1103
1104 static bool access_raz_id_reg(struct kvm_vcpu *vcpu,
1105 struct sys_reg_params *p,
1106 const struct sys_reg_desc *r)
1107 {
1108 return __access_id_reg(vcpu, p, r, true);
1109 }
1110
1111 static int reg_from_user(u64 *val, const void __user *uaddr, u64 id);
1112 static int reg_to_user(void __user *uaddr, const u64 *val, u64 id);
1113 static u64 sys_reg_to_index(const struct sys_reg_desc *reg);
1114
1115 /* Visibility overrides for SVE-specific control registers */
1116 static unsigned int sve_visibility(const struct kvm_vcpu *vcpu,
1117 const struct sys_reg_desc *rd)
1118 {
1119 if (vcpu_has_sve(vcpu))
1120 return 0;
1121
1122 return REG_HIDDEN;
1123 }
1124
1125 static int set_id_aa64pfr0_el1(struct kvm_vcpu *vcpu,
1126 const struct sys_reg_desc *rd,
1127 const struct kvm_one_reg *reg, void __user *uaddr)
1128 {
1129 const u64 id = sys_reg_to_index(rd);
1130 u8 csv2, csv3;
1131 int err;
1132 u64 val;
1133
1134 err = reg_from_user(&val, uaddr, id);
1135 if (err)
1136 return err;
1137
1138 /*
1139 * Allow AA64PFR0_EL1.CSV2 to be set from userspace as long as
1140 * it doesn't promise more than what is actually provided (the
1141 * guest could otherwise be covered in ectoplasmic residue).
1142 */
1143 csv2 = cpuid_feature_extract_unsigned_field(val, ID_AA64PFR0_CSV2_SHIFT);
1144 if (csv2 > 1 ||
1145 (csv2 && arm64_get_spectre_v2_state() != SPECTRE_UNAFFECTED))
1146 return -EINVAL;
1147
1148 /* Same thing for CSV3 */
1149 csv3 = cpuid_feature_extract_unsigned_field(val, ID_AA64PFR0_CSV3_SHIFT);
1150 if (csv3 > 1 ||
1151 (csv3 && arm64_get_meltdown_state() != SPECTRE_UNAFFECTED))
1152 return -EINVAL;
1153
1154 /* We can only differ with CSV[23], and anything else is an error */
1155 val ^= read_id_reg(vcpu, rd, false);
1156 val &= ~((0xFUL << ID_AA64PFR0_CSV2_SHIFT) |
1157 (0xFUL << ID_AA64PFR0_CSV3_SHIFT));
1158 if (val)
1159 return -EINVAL;
1160
1161 vcpu->kvm->arch.pfr0_csv2 = csv2;
1162 vcpu->kvm->arch.pfr0_csv3 = csv3 ;
1163
1164 return 0;
1165 }
1166
1167 /*
1168 * cpufeature ID register user accessors
1169 *
1170 * For now, these registers are immutable for userspace, so no values
1171 * are stored, and for set_id_reg() we don't allow the effective value
1172 * to be changed.
1173 */
1174 static int __get_id_reg(const struct kvm_vcpu *vcpu,
1175 const struct sys_reg_desc *rd, void __user *uaddr,
1176 bool raz)
1177 {
1178 const u64 id = sys_reg_to_index(rd);
1179 const u64 val = read_id_reg(vcpu, rd, raz);
1180
1181 return reg_to_user(uaddr, &val, id);
1182 }
1183
1184 static int __set_id_reg(const struct kvm_vcpu *vcpu,
1185 const struct sys_reg_desc *rd, void __user *uaddr,
1186 bool raz)
1187 {
1188 const u64 id = sys_reg_to_index(rd);
1189 int err;
1190 u64 val;
1191
1192 err = reg_from_user(&val, uaddr, id);
1193 if (err)
1194 return err;
1195
1196 /* This is what we mean by invariant: you can't change it. */
1197 if (val != read_id_reg(vcpu, rd, raz))
1198 return -EINVAL;
1199
1200 return 0;
1201 }
1202
1203 static int get_id_reg(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
1204 const struct kvm_one_reg *reg, void __user *uaddr)
1205 {
1206 bool raz = sysreg_visible_as_raz(vcpu, rd);
1207
1208 return __get_id_reg(vcpu, rd, uaddr, raz);
1209 }
1210
1211 static int set_id_reg(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
1212 const struct kvm_one_reg *reg, void __user *uaddr)
1213 {
1214 bool raz = sysreg_visible_as_raz(vcpu, rd);
1215
1216 return __set_id_reg(vcpu, rd, uaddr, raz);
1217 }
1218
1219 static int get_raz_id_reg(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
1220 const struct kvm_one_reg *reg, void __user *uaddr)
1221 {
1222 return __get_id_reg(vcpu, rd, uaddr, true);
1223 }
1224
1225 static int set_raz_id_reg(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
1226 const struct kvm_one_reg *reg, void __user *uaddr)
1227 {
1228 return __set_id_reg(vcpu, rd, uaddr, true);
1229 }
1230
1231 static bool access_ctr(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
1232 const struct sys_reg_desc *r)
1233 {
1234 if (p->is_write)
1235 return write_to_read_only(vcpu, p, r);
1236
1237 p->regval = read_sanitised_ftr_reg(SYS_CTR_EL0);
1238 return true;
1239 }
1240
1241 static bool access_clidr(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
1242 const struct sys_reg_desc *r)
1243 {
1244 if (p->is_write)
1245 return write_to_read_only(vcpu, p, r);
1246
1247 p->regval = read_sysreg(clidr_el1);
1248 return true;
1249 }
1250
1251 static bool access_csselr(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
1252 const struct sys_reg_desc *r)
1253 {
1254 int reg = r->reg;
1255
1256 if (p->is_write)
1257 vcpu_write_sys_reg(vcpu, p->regval, reg);
1258 else
1259 p->regval = vcpu_read_sys_reg(vcpu, reg);
1260 return true;
1261 }
1262
1263 static bool access_ccsidr(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
1264 const struct sys_reg_desc *r)
1265 {
1266 u32 csselr;
1267
1268 if (p->is_write)
1269 return write_to_read_only(vcpu, p, r);
1270
1271 csselr = vcpu_read_sys_reg(vcpu, CSSELR_EL1);
1272 p->regval = get_ccsidr(csselr);
1273
1274 /*
1275 * Guests should not be doing cache operations by set/way at all, and
1276 * for this reason, we trap them and attempt to infer the intent, so
1277 * that we can flush the entire guest's address space at the appropriate
1278 * time.
1279 * To prevent this trapping from causing performance problems, let's
1280 * expose the geometry of all data and unified caches (which are
1281 * guaranteed to be PIPT and thus non-aliasing) as 1 set and 1 way.
1282 * [If guests should attempt to infer aliasing properties from the
1283 * geometry (which is not permitted by the architecture), they would
1284 * only do so for virtually indexed caches.]
1285 */
1286 if (!(csselr & 1)) // data or unified cache
1287 p->regval &= ~GENMASK(27, 3);
1288 return true;
1289 }
1290
1291 /* sys_reg_desc initialiser for known cpufeature ID registers */
1292 #define ID_SANITISED(name) { \
1293 SYS_DESC(SYS_##name), \
1294 .access = access_id_reg, \
1295 .get_user = get_id_reg, \
1296 .set_user = set_id_reg, \
1297 .visibility = id_visibility, \
1298 }
1299
1300 /*
1301 * sys_reg_desc initialiser for architecturally unallocated cpufeature ID
1302 * register with encoding Op0=3, Op1=0, CRn=0, CRm=crm, Op2=op2
1303 * (1 <= crm < 8, 0 <= Op2 < 8).
1304 */
1305 #define ID_UNALLOCATED(crm, op2) { \
1306 Op0(3), Op1(0), CRn(0), CRm(crm), Op2(op2), \
1307 .access = access_raz_id_reg, \
1308 .get_user = get_raz_id_reg, \
1309 .set_user = set_raz_id_reg, \
1310 }
1311
1312 /*
1313 * sys_reg_desc initialiser for known ID registers that we hide from guests.
1314 * For now, these are exposed just like unallocated ID regs: they appear
1315 * RAZ for the guest.
1316 */
1317 #define ID_HIDDEN(name) { \
1318 SYS_DESC(SYS_##name), \
1319 .access = access_raz_id_reg, \
1320 .get_user = get_raz_id_reg, \
1321 .set_user = set_raz_id_reg, \
1322 }
1323
1324 /*
1325 * Architected system registers.
1326 * Important: Must be sorted ascending by Op0, Op1, CRn, CRm, Op2
1327 *
1328 * Debug handling: We do trap most, if not all debug related system
1329 * registers. The implementation is good enough to ensure that a guest
1330 * can use these with minimal performance degradation. The drawback is
1331 * that we don't implement any of the external debug, none of the
1332 * OSlock protocol. This should be revisited if we ever encounter a
1333 * more demanding guest...
1334 */
1335 static const struct sys_reg_desc sys_reg_descs[] = {
1336 { SYS_DESC(SYS_DC_ISW), access_dcsw },
1337 { SYS_DESC(SYS_DC_CSW), access_dcsw },
1338 { SYS_DESC(SYS_DC_CISW), access_dcsw },
1339
1340 DBG_BCR_BVR_WCR_WVR_EL1(0),
1341 DBG_BCR_BVR_WCR_WVR_EL1(1),
1342 { SYS_DESC(SYS_MDCCINT_EL1), trap_debug_regs, reset_val, MDCCINT_EL1, 0 },
1343 { SYS_DESC(SYS_MDSCR_EL1), trap_debug_regs, reset_val, MDSCR_EL1, 0 },
1344 DBG_BCR_BVR_WCR_WVR_EL1(2),
1345 DBG_BCR_BVR_WCR_WVR_EL1(3),
1346 DBG_BCR_BVR_WCR_WVR_EL1(4),
1347 DBG_BCR_BVR_WCR_WVR_EL1(5),
1348 DBG_BCR_BVR_WCR_WVR_EL1(6),
1349 DBG_BCR_BVR_WCR_WVR_EL1(7),
1350 DBG_BCR_BVR_WCR_WVR_EL1(8),
1351 DBG_BCR_BVR_WCR_WVR_EL1(9),
1352 DBG_BCR_BVR_WCR_WVR_EL1(10),
1353 DBG_BCR_BVR_WCR_WVR_EL1(11),
1354 DBG_BCR_BVR_WCR_WVR_EL1(12),
1355 DBG_BCR_BVR_WCR_WVR_EL1(13),
1356 DBG_BCR_BVR_WCR_WVR_EL1(14),
1357 DBG_BCR_BVR_WCR_WVR_EL1(15),
1358
1359 { SYS_DESC(SYS_MDRAR_EL1), trap_raz_wi },
1360 { SYS_DESC(SYS_OSLAR_EL1), trap_raz_wi },
1361 { SYS_DESC(SYS_OSLSR_EL1), trap_oslsr_el1 },
1362 { SYS_DESC(SYS_OSDLR_EL1), trap_raz_wi },
1363 { SYS_DESC(SYS_DBGPRCR_EL1), trap_raz_wi },
1364 { SYS_DESC(SYS_DBGCLAIMSET_EL1), trap_raz_wi },
1365 { SYS_DESC(SYS_DBGCLAIMCLR_EL1), trap_raz_wi },
1366 { SYS_DESC(SYS_DBGAUTHSTATUS_EL1), trap_dbgauthstatus_el1 },
1367
1368 { SYS_DESC(SYS_MDCCSR_EL0), trap_raz_wi },
1369 { SYS_DESC(SYS_DBGDTR_EL0), trap_raz_wi },
1370 // DBGDTR[TR]X_EL0 share the same encoding
1371 { SYS_DESC(SYS_DBGDTRTX_EL0), trap_raz_wi },
1372
1373 { SYS_DESC(SYS_DBGVCR32_EL2), NULL, reset_val, DBGVCR32_EL2, 0 },
1374
1375 { SYS_DESC(SYS_MPIDR_EL1), NULL, reset_mpidr, MPIDR_EL1 },
1376
1377 /*
1378 * ID regs: all ID_SANITISED() entries here must have corresponding
1379 * entries in arm64_ftr_regs[].
1380 */
1381
1382 /* AArch64 mappings of the AArch32 ID registers */
1383 /* CRm=1 */
1384 ID_SANITISED(ID_PFR0_EL1),
1385 ID_SANITISED(ID_PFR1_EL1),
1386 ID_SANITISED(ID_DFR0_EL1),
1387 ID_HIDDEN(ID_AFR0_EL1),
1388 ID_SANITISED(ID_MMFR0_EL1),
1389 ID_SANITISED(ID_MMFR1_EL1),
1390 ID_SANITISED(ID_MMFR2_EL1),
1391 ID_SANITISED(ID_MMFR3_EL1),
1392
1393 /* CRm=2 */
1394 ID_SANITISED(ID_ISAR0_EL1),
1395 ID_SANITISED(ID_ISAR1_EL1),
1396 ID_SANITISED(ID_ISAR2_EL1),
1397 ID_SANITISED(ID_ISAR3_EL1),
1398 ID_SANITISED(ID_ISAR4_EL1),
1399 ID_SANITISED(ID_ISAR5_EL1),
1400 ID_SANITISED(ID_MMFR4_EL1),
1401 ID_SANITISED(ID_ISAR6_EL1),
1402
1403 /* CRm=3 */
1404 ID_SANITISED(MVFR0_EL1),
1405 ID_SANITISED(MVFR1_EL1),
1406 ID_SANITISED(MVFR2_EL1),
1407 ID_UNALLOCATED(3,3),
1408 ID_SANITISED(ID_PFR2_EL1),
1409 ID_HIDDEN(ID_DFR1_EL1),
1410 ID_SANITISED(ID_MMFR5_EL1),
1411 ID_UNALLOCATED(3,7),
1412
1413 /* AArch64 ID registers */
1414 /* CRm=4 */
1415 { SYS_DESC(SYS_ID_AA64PFR0_EL1), .access = access_id_reg,
1416 .get_user = get_id_reg, .set_user = set_id_aa64pfr0_el1, },
1417 ID_SANITISED(ID_AA64PFR1_EL1),
1418 ID_UNALLOCATED(4,2),
1419 ID_UNALLOCATED(4,3),
1420 ID_SANITISED(ID_AA64ZFR0_EL1),
1421 ID_UNALLOCATED(4,5),
1422 ID_UNALLOCATED(4,6),
1423 ID_UNALLOCATED(4,7),
1424
1425 /* CRm=5 */
1426 ID_SANITISED(ID_AA64DFR0_EL1),
1427 ID_SANITISED(ID_AA64DFR1_EL1),
1428 ID_UNALLOCATED(5,2),
1429 ID_UNALLOCATED(5,3),
1430 ID_HIDDEN(ID_AA64AFR0_EL1),
1431 ID_HIDDEN(ID_AA64AFR1_EL1),
1432 ID_UNALLOCATED(5,6),
1433 ID_UNALLOCATED(5,7),
1434
1435 /* CRm=6 */
1436 ID_SANITISED(ID_AA64ISAR0_EL1),
1437 ID_SANITISED(ID_AA64ISAR1_EL1),
1438 ID_UNALLOCATED(6,2),
1439 ID_UNALLOCATED(6,3),
1440 ID_UNALLOCATED(6,4),
1441 ID_UNALLOCATED(6,5),
1442 ID_UNALLOCATED(6,6),
1443 ID_UNALLOCATED(6,7),
1444
1445 /* CRm=7 */
1446 ID_SANITISED(ID_AA64MMFR0_EL1),
1447 ID_SANITISED(ID_AA64MMFR1_EL1),
1448 ID_SANITISED(ID_AA64MMFR2_EL1),
1449 ID_UNALLOCATED(7,3),
1450 ID_UNALLOCATED(7,4),
1451 ID_UNALLOCATED(7,5),
1452 ID_UNALLOCATED(7,6),
1453 ID_UNALLOCATED(7,7),
1454
1455 { SYS_DESC(SYS_SCTLR_EL1), access_vm_reg, reset_val, SCTLR_EL1, 0x00C50078 },
1456 { SYS_DESC(SYS_ACTLR_EL1), access_actlr, reset_actlr, ACTLR_EL1 },
1457 { SYS_DESC(SYS_CPACR_EL1), NULL, reset_val, CPACR_EL1, 0 },
1458
1459 { SYS_DESC(SYS_RGSR_EL1), undef_access },
1460 { SYS_DESC(SYS_GCR_EL1), undef_access },
1461
1462 { SYS_DESC(SYS_ZCR_EL1), NULL, reset_val, ZCR_EL1, 0, .visibility = sve_visibility },
1463 { SYS_DESC(SYS_TTBR0_EL1), access_vm_reg, reset_unknown, TTBR0_EL1 },
1464 { SYS_DESC(SYS_TTBR1_EL1), access_vm_reg, reset_unknown, TTBR1_EL1 },
1465 { SYS_DESC(SYS_TCR_EL1), access_vm_reg, reset_val, TCR_EL1, 0 },
1466
1467 PTRAUTH_KEY(APIA),
1468 PTRAUTH_KEY(APIB),
1469 PTRAUTH_KEY(APDA),
1470 PTRAUTH_KEY(APDB),
1471 PTRAUTH_KEY(APGA),
1472
1473 { SYS_DESC(SYS_AFSR0_EL1), access_vm_reg, reset_unknown, AFSR0_EL1 },
1474 { SYS_DESC(SYS_AFSR1_EL1), access_vm_reg, reset_unknown, AFSR1_EL1 },
1475 { SYS_DESC(SYS_ESR_EL1), access_vm_reg, reset_unknown, ESR_EL1 },
1476
1477 { SYS_DESC(SYS_ERRIDR_EL1), trap_raz_wi },
1478 { SYS_DESC(SYS_ERRSELR_EL1), trap_raz_wi },
1479 { SYS_DESC(SYS_ERXFR_EL1), trap_raz_wi },
1480 { SYS_DESC(SYS_ERXCTLR_EL1), trap_raz_wi },
1481 { SYS_DESC(SYS_ERXSTATUS_EL1), trap_raz_wi },
1482 { SYS_DESC(SYS_ERXADDR_EL1), trap_raz_wi },
1483 { SYS_DESC(SYS_ERXMISC0_EL1), trap_raz_wi },
1484 { SYS_DESC(SYS_ERXMISC1_EL1), trap_raz_wi },
1485
1486 { SYS_DESC(SYS_TFSR_EL1), undef_access },
1487 { SYS_DESC(SYS_TFSRE0_EL1), undef_access },
1488
1489 { SYS_DESC(SYS_FAR_EL1), access_vm_reg, reset_unknown, FAR_EL1 },
1490 { SYS_DESC(SYS_PAR_EL1), NULL, reset_unknown, PAR_EL1 },
1491
1492 { PMU_SYS_REG(SYS_PMINTENSET_EL1),
1493 .access = access_pminten, .reg = PMINTENSET_EL1 },
1494 { PMU_SYS_REG(SYS_PMINTENCLR_EL1),
1495 .access = access_pminten, .reg = PMINTENSET_EL1 },
1496
1497 { SYS_DESC(SYS_MAIR_EL1), access_vm_reg, reset_unknown, MAIR_EL1 },
1498 { SYS_DESC(SYS_AMAIR_EL1), access_vm_reg, reset_amair_el1, AMAIR_EL1 },
1499
1500 { SYS_DESC(SYS_LORSA_EL1), trap_loregion },
1501 { SYS_DESC(SYS_LOREA_EL1), trap_loregion },
1502 { SYS_DESC(SYS_LORN_EL1), trap_loregion },
1503 { SYS_DESC(SYS_LORC_EL1), trap_loregion },
1504 { SYS_DESC(SYS_LORID_EL1), trap_loregion },
1505
1506 { SYS_DESC(SYS_VBAR_EL1), NULL, reset_val, VBAR_EL1, 0 },
1507 { SYS_DESC(SYS_DISR_EL1), NULL, reset_val, DISR_EL1, 0 },
1508
1509 { SYS_DESC(SYS_ICC_IAR0_EL1), write_to_read_only },
1510 { SYS_DESC(SYS_ICC_EOIR0_EL1), read_from_write_only },
1511 { SYS_DESC(SYS_ICC_HPPIR0_EL1), write_to_read_only },
1512 { SYS_DESC(SYS_ICC_DIR_EL1), read_from_write_only },
1513 { SYS_DESC(SYS_ICC_RPR_EL1), write_to_read_only },
1514 { SYS_DESC(SYS_ICC_SGI1R_EL1), access_gic_sgi },
1515 { SYS_DESC(SYS_ICC_ASGI1R_EL1), access_gic_sgi },
1516 { SYS_DESC(SYS_ICC_SGI0R_EL1), access_gic_sgi },
1517 { SYS_DESC(SYS_ICC_IAR1_EL1), write_to_read_only },
1518 { SYS_DESC(SYS_ICC_EOIR1_EL1), read_from_write_only },
1519 { SYS_DESC(SYS_ICC_HPPIR1_EL1), write_to_read_only },
1520 { SYS_DESC(SYS_ICC_SRE_EL1), access_gic_sre },
1521
1522 { SYS_DESC(SYS_CONTEXTIDR_EL1), access_vm_reg, reset_val, CONTEXTIDR_EL1, 0 },
1523 { SYS_DESC(SYS_TPIDR_EL1), NULL, reset_unknown, TPIDR_EL1 },
1524
1525 { SYS_DESC(SYS_SCXTNUM_EL1), undef_access },
1526
1527 { SYS_DESC(SYS_CNTKCTL_EL1), NULL, reset_val, CNTKCTL_EL1, 0},
1528
1529 { SYS_DESC(SYS_CCSIDR_EL1), access_ccsidr },
1530 { SYS_DESC(SYS_CLIDR_EL1), access_clidr },
1531 { SYS_DESC(SYS_CSSELR_EL1), access_csselr, reset_unknown, CSSELR_EL1 },
1532 { SYS_DESC(SYS_CTR_EL0), access_ctr },
1533
1534 { PMU_SYS_REG(SYS_PMCR_EL0), .access = access_pmcr,
1535 .reset = reset_pmcr, .reg = PMCR_EL0 },
1536 { PMU_SYS_REG(SYS_PMCNTENSET_EL0),
1537 .access = access_pmcnten, .reg = PMCNTENSET_EL0 },
1538 { PMU_SYS_REG(SYS_PMCNTENCLR_EL0),
1539 .access = access_pmcnten, .reg = PMCNTENSET_EL0 },
1540 { PMU_SYS_REG(SYS_PMOVSCLR_EL0),
1541 .access = access_pmovs, .reg = PMOVSSET_EL0 },
1542 { PMU_SYS_REG(SYS_PMSWINC_EL0),
1543 .access = access_pmswinc, .reg = PMSWINC_EL0 },
1544 { PMU_SYS_REG(SYS_PMSELR_EL0),
1545 .access = access_pmselr, .reg = PMSELR_EL0 },
1546 { PMU_SYS_REG(SYS_PMCEID0_EL0),
1547 .access = access_pmceid, .reset = NULL },
1548 { PMU_SYS_REG(SYS_PMCEID1_EL0),
1549 .access = access_pmceid, .reset = NULL },
1550 { PMU_SYS_REG(SYS_PMCCNTR_EL0),
1551 .access = access_pmu_evcntr, .reg = PMCCNTR_EL0 },
1552 { PMU_SYS_REG(SYS_PMXEVTYPER_EL0),
1553 .access = access_pmu_evtyper, .reset = NULL },
1554 { PMU_SYS_REG(SYS_PMXEVCNTR_EL0),
1555 .access = access_pmu_evcntr, .reset = NULL },
1556 /*
1557 * PMUSERENR_EL0 resets as unknown in 64bit mode while it resets as zero
1558 * in 32bit mode. Here we choose to reset it as zero for consistency.
1559 */
1560 { PMU_SYS_REG(SYS_PMUSERENR_EL0), .access = access_pmuserenr,
1561 .reset = reset_val, .reg = PMUSERENR_EL0, .val = 0 },
1562 { PMU_SYS_REG(SYS_PMOVSSET_EL0),
1563 .access = access_pmovs, .reg = PMOVSSET_EL0 },
1564
1565 { SYS_DESC(SYS_TPIDR_EL0), NULL, reset_unknown, TPIDR_EL0 },
1566 { SYS_DESC(SYS_TPIDRRO_EL0), NULL, reset_unknown, TPIDRRO_EL0 },
1567
1568 { SYS_DESC(SYS_SCXTNUM_EL0), undef_access },
1569
1570 { SYS_DESC(SYS_AMCR_EL0), undef_access },
1571 { SYS_DESC(SYS_AMCFGR_EL0), undef_access },
1572 { SYS_DESC(SYS_AMCGCR_EL0), undef_access },
1573 { SYS_DESC(SYS_AMUSERENR_EL0), undef_access },
1574 { SYS_DESC(SYS_AMCNTENCLR0_EL0), undef_access },
1575 { SYS_DESC(SYS_AMCNTENSET0_EL0), undef_access },
1576 { SYS_DESC(SYS_AMCNTENCLR1_EL0), undef_access },
1577 { SYS_DESC(SYS_AMCNTENSET1_EL0), undef_access },
1578 AMU_AMEVCNTR0_EL0(0),
1579 AMU_AMEVCNTR0_EL0(1),
1580 AMU_AMEVCNTR0_EL0(2),
1581 AMU_AMEVCNTR0_EL0(3),
1582 AMU_AMEVCNTR0_EL0(4),
1583 AMU_AMEVCNTR0_EL0(5),
1584 AMU_AMEVCNTR0_EL0(6),
1585 AMU_AMEVCNTR0_EL0(7),
1586 AMU_AMEVCNTR0_EL0(8),
1587 AMU_AMEVCNTR0_EL0(9),
1588 AMU_AMEVCNTR0_EL0(10),
1589 AMU_AMEVCNTR0_EL0(11),
1590 AMU_AMEVCNTR0_EL0(12),
1591 AMU_AMEVCNTR0_EL0(13),
1592 AMU_AMEVCNTR0_EL0(14),
1593 AMU_AMEVCNTR0_EL0(15),
1594 AMU_AMEVTYPER0_EL0(0),
1595 AMU_AMEVTYPER0_EL0(1),
1596 AMU_AMEVTYPER0_EL0(2),
1597 AMU_AMEVTYPER0_EL0(3),
1598 AMU_AMEVTYPER0_EL0(4),
1599 AMU_AMEVTYPER0_EL0(5),
1600 AMU_AMEVTYPER0_EL0(6),
1601 AMU_AMEVTYPER0_EL0(7),
1602 AMU_AMEVTYPER0_EL0(8),
1603 AMU_AMEVTYPER0_EL0(9),
1604 AMU_AMEVTYPER0_EL0(10),
1605 AMU_AMEVTYPER0_EL0(11),
1606 AMU_AMEVTYPER0_EL0(12),
1607 AMU_AMEVTYPER0_EL0(13),
1608 AMU_AMEVTYPER0_EL0(14),
1609 AMU_AMEVTYPER0_EL0(15),
1610 AMU_AMEVCNTR1_EL0(0),
1611 AMU_AMEVCNTR1_EL0(1),
1612 AMU_AMEVCNTR1_EL0(2),
1613 AMU_AMEVCNTR1_EL0(3),
1614 AMU_AMEVCNTR1_EL0(4),
1615 AMU_AMEVCNTR1_EL0(5),
1616 AMU_AMEVCNTR1_EL0(6),
1617 AMU_AMEVCNTR1_EL0(7),
1618 AMU_AMEVCNTR1_EL0(8),
1619 AMU_AMEVCNTR1_EL0(9),
1620 AMU_AMEVCNTR1_EL0(10),
1621 AMU_AMEVCNTR1_EL0(11),
1622 AMU_AMEVCNTR1_EL0(12),
1623 AMU_AMEVCNTR1_EL0(13),
1624 AMU_AMEVCNTR1_EL0(14),
1625 AMU_AMEVCNTR1_EL0(15),
1626 AMU_AMEVTYPER1_EL0(0),
1627 AMU_AMEVTYPER1_EL0(1),
1628 AMU_AMEVTYPER1_EL0(2),
1629 AMU_AMEVTYPER1_EL0(3),
1630 AMU_AMEVTYPER1_EL0(4),
1631 AMU_AMEVTYPER1_EL0(5),
1632 AMU_AMEVTYPER1_EL0(6),
1633 AMU_AMEVTYPER1_EL0(7),
1634 AMU_AMEVTYPER1_EL0(8),
1635 AMU_AMEVTYPER1_EL0(9),
1636 AMU_AMEVTYPER1_EL0(10),
1637 AMU_AMEVTYPER1_EL0(11),
1638 AMU_AMEVTYPER1_EL0(12),
1639 AMU_AMEVTYPER1_EL0(13),
1640 AMU_AMEVTYPER1_EL0(14),
1641 AMU_AMEVTYPER1_EL0(15),
1642
1643 { SYS_DESC(SYS_CNTP_TVAL_EL0), access_arch_timer },
1644 { SYS_DESC(SYS_CNTP_CTL_EL0), access_arch_timer },
1645 { SYS_DESC(SYS_CNTP_CVAL_EL0), access_arch_timer },
1646
1647 /* PMEVCNTRn_EL0 */
1648 PMU_PMEVCNTR_EL0(0),
1649 PMU_PMEVCNTR_EL0(1),
1650 PMU_PMEVCNTR_EL0(2),
1651 PMU_PMEVCNTR_EL0(3),
1652 PMU_PMEVCNTR_EL0(4),
1653 PMU_PMEVCNTR_EL0(5),
1654 PMU_PMEVCNTR_EL0(6),
1655 PMU_PMEVCNTR_EL0(7),
1656 PMU_PMEVCNTR_EL0(8),
1657 PMU_PMEVCNTR_EL0(9),
1658 PMU_PMEVCNTR_EL0(10),
1659 PMU_PMEVCNTR_EL0(11),
1660 PMU_PMEVCNTR_EL0(12),
1661 PMU_PMEVCNTR_EL0(13),
1662 PMU_PMEVCNTR_EL0(14),
1663 PMU_PMEVCNTR_EL0(15),
1664 PMU_PMEVCNTR_EL0(16),
1665 PMU_PMEVCNTR_EL0(17),
1666 PMU_PMEVCNTR_EL0(18),
1667 PMU_PMEVCNTR_EL0(19),
1668 PMU_PMEVCNTR_EL0(20),
1669 PMU_PMEVCNTR_EL0(21),
1670 PMU_PMEVCNTR_EL0(22),
1671 PMU_PMEVCNTR_EL0(23),
1672 PMU_PMEVCNTR_EL0(24),
1673 PMU_PMEVCNTR_EL0(25),
1674 PMU_PMEVCNTR_EL0(26),
1675 PMU_PMEVCNTR_EL0(27),
1676 PMU_PMEVCNTR_EL0(28),
1677 PMU_PMEVCNTR_EL0(29),
1678 PMU_PMEVCNTR_EL0(30),
1679 /* PMEVTYPERn_EL0 */
1680 PMU_PMEVTYPER_EL0(0),
1681 PMU_PMEVTYPER_EL0(1),
1682 PMU_PMEVTYPER_EL0(2),
1683 PMU_PMEVTYPER_EL0(3),
1684 PMU_PMEVTYPER_EL0(4),
1685 PMU_PMEVTYPER_EL0(5),
1686 PMU_PMEVTYPER_EL0(6),
1687 PMU_PMEVTYPER_EL0(7),
1688 PMU_PMEVTYPER_EL0(8),
1689 PMU_PMEVTYPER_EL0(9),
1690 PMU_PMEVTYPER_EL0(10),
1691 PMU_PMEVTYPER_EL0(11),
1692 PMU_PMEVTYPER_EL0(12),
1693 PMU_PMEVTYPER_EL0(13),
1694 PMU_PMEVTYPER_EL0(14),
1695 PMU_PMEVTYPER_EL0(15),
1696 PMU_PMEVTYPER_EL0(16),
1697 PMU_PMEVTYPER_EL0(17),
1698 PMU_PMEVTYPER_EL0(18),
1699 PMU_PMEVTYPER_EL0(19),
1700 PMU_PMEVTYPER_EL0(20),
1701 PMU_PMEVTYPER_EL0(21),
1702 PMU_PMEVTYPER_EL0(22),
1703 PMU_PMEVTYPER_EL0(23),
1704 PMU_PMEVTYPER_EL0(24),
1705 PMU_PMEVTYPER_EL0(25),
1706 PMU_PMEVTYPER_EL0(26),
1707 PMU_PMEVTYPER_EL0(27),
1708 PMU_PMEVTYPER_EL0(28),
1709 PMU_PMEVTYPER_EL0(29),
1710 PMU_PMEVTYPER_EL0(30),
1711 /*
1712 * PMCCFILTR_EL0 resets as unknown in 64bit mode while it resets as zero
1713 * in 32bit mode. Here we choose to reset it as zero for consistency.
1714 */
1715 { PMU_SYS_REG(SYS_PMCCFILTR_EL0), .access = access_pmu_evtyper,
1716 .reset = reset_val, .reg = PMCCFILTR_EL0, .val = 0 },
1717
1718 { SYS_DESC(SYS_DACR32_EL2), NULL, reset_unknown, DACR32_EL2 },
1719 { SYS_DESC(SYS_IFSR32_EL2), NULL, reset_unknown, IFSR32_EL2 },
1720 { SYS_DESC(SYS_FPEXC32_EL2), NULL, reset_val, FPEXC32_EL2, 0x700 },
1721 };
1722
1723 static bool trap_dbgidr(struct kvm_vcpu *vcpu,
1724 struct sys_reg_params *p,
1725 const struct sys_reg_desc *r)
1726 {
1727 if (p->is_write) {
1728 return ignore_write(vcpu, p);
1729 } else {
1730 u64 dfr = read_sanitised_ftr_reg(SYS_ID_AA64DFR0_EL1);
1731 u64 pfr = read_sanitised_ftr_reg(SYS_ID_AA64PFR0_EL1);
1732 u32 el3 = !!cpuid_feature_extract_unsigned_field(pfr, ID_AA64PFR0_EL3_SHIFT);
1733
1734 p->regval = ((((dfr >> ID_AA64DFR0_WRPS_SHIFT) & 0xf) << 28) |
1735 (((dfr >> ID_AA64DFR0_BRPS_SHIFT) & 0xf) << 24) |
1736 (((dfr >> ID_AA64DFR0_CTX_CMPS_SHIFT) & 0xf) << 20)
1737 | (6 << 16) | (el3 << 14) | (el3 << 12));
1738 return true;
1739 }
1740 }
1741
1742 /*
1743 * AArch32 debug register mappings
1744 *
1745 * AArch32 DBGBVRn is mapped to DBGBVRn_EL1[31:0]
1746 * AArch32 DBGBXVRn is mapped to DBGBVRn_EL1[63:32]
1747 *
1748 * None of the other registers share their location, so treat them as
1749 * if they were 64bit.
1750 */
1751 #define DBG_BCR_BVR_WCR_WVR(n) \
1752 /* DBGBVRn */ \
1753 { AA32(LO), Op1( 0), CRn( 0), CRm((n)), Op2( 4), trap_bvr, NULL, n }, \
1754 /* DBGBCRn */ \
1755 { Op1( 0), CRn( 0), CRm((n)), Op2( 5), trap_bcr, NULL, n }, \
1756 /* DBGWVRn */ \
1757 { Op1( 0), CRn( 0), CRm((n)), Op2( 6), trap_wvr, NULL, n }, \
1758 /* DBGWCRn */ \
1759 { Op1( 0), CRn( 0), CRm((n)), Op2( 7), trap_wcr, NULL, n }
1760
1761 #define DBGBXVR(n) \
1762 { AA32(HI), Op1( 0), CRn( 1), CRm((n)), Op2( 1), trap_bvr, NULL, n }
1763
1764 /*
1765 * Trapped cp14 registers. We generally ignore most of the external
1766 * debug, on the principle that they don't really make sense to a
1767 * guest. Revisit this one day, would this principle change.
1768 */
1769 static const struct sys_reg_desc cp14_regs[] = {
1770 /* DBGIDR */
1771 { Op1( 0), CRn( 0), CRm( 0), Op2( 0), trap_dbgidr },
1772 /* DBGDTRRXext */
1773 { Op1( 0), CRn( 0), CRm( 0), Op2( 2), trap_raz_wi },
1774
1775 DBG_BCR_BVR_WCR_WVR(0),
1776 /* DBGDSCRint */
1777 { Op1( 0), CRn( 0), CRm( 1), Op2( 0), trap_raz_wi },
1778 DBG_BCR_BVR_WCR_WVR(1),
1779 /* DBGDCCINT */
1780 { Op1( 0), CRn( 0), CRm( 2), Op2( 0), trap_debug_regs, NULL, MDCCINT_EL1 },
1781 /* DBGDSCRext */
1782 { Op1( 0), CRn( 0), CRm( 2), Op2( 2), trap_debug_regs, NULL, MDSCR_EL1 },
1783 DBG_BCR_BVR_WCR_WVR(2),
1784 /* DBGDTR[RT]Xint */
1785 { Op1( 0), CRn( 0), CRm( 3), Op2( 0), trap_raz_wi },
1786 /* DBGDTR[RT]Xext */
1787 { Op1( 0), CRn( 0), CRm( 3), Op2( 2), trap_raz_wi },
1788 DBG_BCR_BVR_WCR_WVR(3),
1789 DBG_BCR_BVR_WCR_WVR(4),
1790 DBG_BCR_BVR_WCR_WVR(5),
1791 /* DBGWFAR */
1792 { Op1( 0), CRn( 0), CRm( 6), Op2( 0), trap_raz_wi },
1793 /* DBGOSECCR */
1794 { Op1( 0), CRn( 0), CRm( 6), Op2( 2), trap_raz_wi },
1795 DBG_BCR_BVR_WCR_WVR(6),
1796 /* DBGVCR */
1797 { Op1( 0), CRn( 0), CRm( 7), Op2( 0), trap_debug_regs, NULL, DBGVCR32_EL2 },
1798 DBG_BCR_BVR_WCR_WVR(7),
1799 DBG_BCR_BVR_WCR_WVR(8),
1800 DBG_BCR_BVR_WCR_WVR(9),
1801 DBG_BCR_BVR_WCR_WVR(10),
1802 DBG_BCR_BVR_WCR_WVR(11),
1803 DBG_BCR_BVR_WCR_WVR(12),
1804 DBG_BCR_BVR_WCR_WVR(13),
1805 DBG_BCR_BVR_WCR_WVR(14),
1806 DBG_BCR_BVR_WCR_WVR(15),
1807
1808 /* DBGDRAR (32bit) */
1809 { Op1( 0), CRn( 1), CRm( 0), Op2( 0), trap_raz_wi },
1810
1811 DBGBXVR(0),
1812 /* DBGOSLAR */
1813 { Op1( 0), CRn( 1), CRm( 0), Op2( 4), trap_raz_wi },
1814 DBGBXVR(1),
1815 /* DBGOSLSR */
1816 { Op1( 0), CRn( 1), CRm( 1), Op2( 4), trap_oslsr_el1 },
1817 DBGBXVR(2),
1818 DBGBXVR(3),
1819 /* DBGOSDLR */
1820 { Op1( 0), CRn( 1), CRm( 3), Op2( 4), trap_raz_wi },
1821 DBGBXVR(4),
1822 /* DBGPRCR */
1823 { Op1( 0), CRn( 1), CRm( 4), Op2( 4), trap_raz_wi },
1824 DBGBXVR(5),
1825 DBGBXVR(6),
1826 DBGBXVR(7),
1827 DBGBXVR(8),
1828 DBGBXVR(9),
1829 DBGBXVR(10),
1830 DBGBXVR(11),
1831 DBGBXVR(12),
1832 DBGBXVR(13),
1833 DBGBXVR(14),
1834 DBGBXVR(15),
1835
1836 /* DBGDSAR (32bit) */
1837 { Op1( 0), CRn( 2), CRm( 0), Op2( 0), trap_raz_wi },
1838
1839 /* DBGDEVID2 */
1840 { Op1( 0), CRn( 7), CRm( 0), Op2( 7), trap_raz_wi },
1841 /* DBGDEVID1 */
1842 { Op1( 0), CRn( 7), CRm( 1), Op2( 7), trap_raz_wi },
1843 /* DBGDEVID */
1844 { Op1( 0), CRn( 7), CRm( 2), Op2( 7), trap_raz_wi },
1845 /* DBGCLAIMSET */
1846 { Op1( 0), CRn( 7), CRm( 8), Op2( 6), trap_raz_wi },
1847 /* DBGCLAIMCLR */
1848 { Op1( 0), CRn( 7), CRm( 9), Op2( 6), trap_raz_wi },
1849 /* DBGAUTHSTATUS */
1850 { Op1( 0), CRn( 7), CRm(14), Op2( 6), trap_dbgauthstatus_el1 },
1851 };
1852
1853 /* Trapped cp14 64bit registers */
1854 static const struct sys_reg_desc cp14_64_regs[] = {
1855 /* DBGDRAR (64bit) */
1856 { Op1( 0), CRm( 1), .access = trap_raz_wi },
1857
1858 /* DBGDSAR (64bit) */
1859 { Op1( 0), CRm( 2), .access = trap_raz_wi },
1860 };
1861
1862 /* Macro to expand the PMEVCNTRn register */
1863 #define PMU_PMEVCNTR(n) \
1864 /* PMEVCNTRn */ \
1865 { Op1(0), CRn(0b1110), \
1866 CRm((0b1000 | (((n) >> 3) & 0x3))), Op2(((n) & 0x7)), \
1867 access_pmu_evcntr }
1868
1869 /* Macro to expand the PMEVTYPERn register */
1870 #define PMU_PMEVTYPER(n) \
1871 /* PMEVTYPERn */ \
1872 { Op1(0), CRn(0b1110), \
1873 CRm((0b1100 | (((n) >> 3) & 0x3))), Op2(((n) & 0x7)), \
1874 access_pmu_evtyper }
1875
1876 /*
1877 * Trapped cp15 registers. TTBR0/TTBR1 get a double encoding,
1878 * depending on the way they are accessed (as a 32bit or a 64bit
1879 * register).
1880 */
1881 static const struct sys_reg_desc cp15_regs[] = {
1882 { Op1( 0), CRn( 0), CRm( 0), Op2( 1), access_ctr },
1883 { Op1( 0), CRn( 1), CRm( 0), Op2( 0), access_vm_reg, NULL, SCTLR_EL1 },
1884 /* ACTLR */
1885 { AA32(LO), Op1( 0), CRn( 1), CRm( 0), Op2( 1), access_actlr, NULL, ACTLR_EL1 },
1886 /* ACTLR2 */
1887 { AA32(HI), Op1( 0), CRn( 1), CRm( 0), Op2( 3), access_actlr, NULL, ACTLR_EL1 },
1888 { Op1( 0), CRn( 2), CRm( 0), Op2( 0), access_vm_reg, NULL, TTBR0_EL1 },
1889 { Op1( 0), CRn( 2), CRm( 0), Op2( 1), access_vm_reg, NULL, TTBR1_EL1 },
1890 /* TTBCR */
1891 { AA32(LO), Op1( 0), CRn( 2), CRm( 0), Op2( 2), access_vm_reg, NULL, TCR_EL1 },
1892 /* TTBCR2 */
1893 { AA32(HI), Op1( 0), CRn( 2), CRm( 0), Op2( 3), access_vm_reg, NULL, TCR_EL1 },
1894 { Op1( 0), CRn( 3), CRm( 0), Op2( 0), access_vm_reg, NULL, DACR32_EL2 },
1895 /* DFSR */
1896 { Op1( 0), CRn( 5), CRm( 0), Op2( 0), access_vm_reg, NULL, ESR_EL1 },
1897 { Op1( 0), CRn( 5), CRm( 0), Op2( 1), access_vm_reg, NULL, IFSR32_EL2 },
1898 /* ADFSR */
1899 { Op1( 0), CRn( 5), CRm( 1), Op2( 0), access_vm_reg, NULL, AFSR0_EL1 },
1900 /* AIFSR */
1901 { Op1( 0), CRn( 5), CRm( 1), Op2( 1), access_vm_reg, NULL, AFSR1_EL1 },
1902 /* DFAR */
1903 { AA32(LO), Op1( 0), CRn( 6), CRm( 0), Op2( 0), access_vm_reg, NULL, FAR_EL1 },
1904 /* IFAR */
1905 { AA32(HI), Op1( 0), CRn( 6), CRm( 0), Op2( 2), access_vm_reg, NULL, FAR_EL1 },
1906
1907 /*
1908 * DC{C,I,CI}SW operations:
1909 */
1910 { Op1( 0), CRn( 7), CRm( 6), Op2( 2), access_dcsw },
1911 { Op1( 0), CRn( 7), CRm(10), Op2( 2), access_dcsw },
1912 { Op1( 0), CRn( 7), CRm(14), Op2( 2), access_dcsw },
1913
1914 /* PMU */
1915 { Op1( 0), CRn( 9), CRm(12), Op2( 0), access_pmcr },
1916 { Op1( 0), CRn( 9), CRm(12), Op2( 1), access_pmcnten },
1917 { Op1( 0), CRn( 9), CRm(12), Op2( 2), access_pmcnten },
1918 { Op1( 0), CRn( 9), CRm(12), Op2( 3), access_pmovs },
1919 { Op1( 0), CRn( 9), CRm(12), Op2( 4), access_pmswinc },
1920 { Op1( 0), CRn( 9), CRm(12), Op2( 5), access_pmselr },
1921 { Op1( 0), CRn( 9), CRm(12), Op2( 6), access_pmceid },
1922 { Op1( 0), CRn( 9), CRm(12), Op2( 7), access_pmceid },
1923 { Op1( 0), CRn( 9), CRm(13), Op2( 0), access_pmu_evcntr },
1924 { Op1( 0), CRn( 9), CRm(13), Op2( 1), access_pmu_evtyper },
1925 { Op1( 0), CRn( 9), CRm(13), Op2( 2), access_pmu_evcntr },
1926 { Op1( 0), CRn( 9), CRm(14), Op2( 0), access_pmuserenr },
1927 { Op1( 0), CRn( 9), CRm(14), Op2( 1), access_pminten },
1928 { Op1( 0), CRn( 9), CRm(14), Op2( 2), access_pminten },
1929 { Op1( 0), CRn( 9), CRm(14), Op2( 3), access_pmovs },
1930
1931 /* PRRR/MAIR0 */
1932 { AA32(LO), Op1( 0), CRn(10), CRm( 2), Op2( 0), access_vm_reg, NULL, MAIR_EL1 },
1933 /* NMRR/MAIR1 */
1934 { AA32(HI), Op1( 0), CRn(10), CRm( 2), Op2( 1), access_vm_reg, NULL, MAIR_EL1 },
1935 /* AMAIR0 */
1936 { AA32(LO), Op1( 0), CRn(10), CRm( 3), Op2( 0), access_vm_reg, NULL, AMAIR_EL1 },
1937 /* AMAIR1 */
1938 { AA32(HI), Op1( 0), CRn(10), CRm( 3), Op2( 1), access_vm_reg, NULL, AMAIR_EL1 },
1939
1940 /* ICC_SRE */
1941 { Op1( 0), CRn(12), CRm(12), Op2( 5), access_gic_sre },
1942
1943 { Op1( 0), CRn(13), CRm( 0), Op2( 1), access_vm_reg, NULL, CONTEXTIDR_EL1 },
1944
1945 /* Arch Tmers */
1946 { SYS_DESC(SYS_AARCH32_CNTP_TVAL), access_arch_timer },
1947 { SYS_DESC(SYS_AARCH32_CNTP_CTL), access_arch_timer },
1948
1949 /* PMEVCNTRn */
1950 PMU_PMEVCNTR(0),
1951 PMU_PMEVCNTR(1),
1952 PMU_PMEVCNTR(2),
1953 PMU_PMEVCNTR(3),
1954 PMU_PMEVCNTR(4),
1955 PMU_PMEVCNTR(5),
1956 PMU_PMEVCNTR(6),
1957 PMU_PMEVCNTR(7),
1958 PMU_PMEVCNTR(8),
1959 PMU_PMEVCNTR(9),
1960 PMU_PMEVCNTR(10),
1961 PMU_PMEVCNTR(11),
1962 PMU_PMEVCNTR(12),
1963 PMU_PMEVCNTR(13),
1964 PMU_PMEVCNTR(14),
1965 PMU_PMEVCNTR(15),
1966 PMU_PMEVCNTR(16),
1967 PMU_PMEVCNTR(17),
1968 PMU_PMEVCNTR(18),
1969 PMU_PMEVCNTR(19),
1970 PMU_PMEVCNTR(20),
1971 PMU_PMEVCNTR(21),
1972 PMU_PMEVCNTR(22),
1973 PMU_PMEVCNTR(23),
1974 PMU_PMEVCNTR(24),
1975 PMU_PMEVCNTR(25),
1976 PMU_PMEVCNTR(26),
1977 PMU_PMEVCNTR(27),
1978 PMU_PMEVCNTR(28),
1979 PMU_PMEVCNTR(29),
1980 PMU_PMEVCNTR(30),
1981 /* PMEVTYPERn */
1982 PMU_PMEVTYPER(0),
1983 PMU_PMEVTYPER(1),
1984 PMU_PMEVTYPER(2),
1985 PMU_PMEVTYPER(3),
1986 PMU_PMEVTYPER(4),
1987 PMU_PMEVTYPER(5),
1988 PMU_PMEVTYPER(6),
1989 PMU_PMEVTYPER(7),
1990 PMU_PMEVTYPER(8),
1991 PMU_PMEVTYPER(9),
1992 PMU_PMEVTYPER(10),
1993 PMU_PMEVTYPER(11),
1994 PMU_PMEVTYPER(12),
1995 PMU_PMEVTYPER(13),
1996 PMU_PMEVTYPER(14),
1997 PMU_PMEVTYPER(15),
1998 PMU_PMEVTYPER(16),
1999 PMU_PMEVTYPER(17),
2000 PMU_PMEVTYPER(18),
2001 PMU_PMEVTYPER(19),
2002 PMU_PMEVTYPER(20),
2003 PMU_PMEVTYPER(21),
2004 PMU_PMEVTYPER(22),
2005 PMU_PMEVTYPER(23),
2006 PMU_PMEVTYPER(24),
2007 PMU_PMEVTYPER(25),
2008 PMU_PMEVTYPER(26),
2009 PMU_PMEVTYPER(27),
2010 PMU_PMEVTYPER(28),
2011 PMU_PMEVTYPER(29),
2012 PMU_PMEVTYPER(30),
2013 /* PMCCFILTR */
2014 { Op1(0), CRn(14), CRm(15), Op2(7), access_pmu_evtyper },
2015
2016 { Op1(1), CRn( 0), CRm( 0), Op2(0), access_ccsidr },
2017 { Op1(1), CRn( 0), CRm( 0), Op2(1), access_clidr },
2018 { Op1(2), CRn( 0), CRm( 0), Op2(0), access_csselr, NULL, CSSELR_EL1 },
2019 };
2020
2021 static const struct sys_reg_desc cp15_64_regs[] = {
2022 { Op1( 0), CRn( 0), CRm( 2), Op2( 0), access_vm_reg, NULL, TTBR0_EL1 },
2023 { Op1( 0), CRn( 0), CRm( 9), Op2( 0), access_pmu_evcntr },
2024 { Op1( 0), CRn( 0), CRm(12), Op2( 0), access_gic_sgi }, /* ICC_SGI1R */
2025 { Op1( 1), CRn( 0), CRm( 2), Op2( 0), access_vm_reg, NULL, TTBR1_EL1 },
2026 { Op1( 1), CRn( 0), CRm(12), Op2( 0), access_gic_sgi }, /* ICC_ASGI1R */
2027 { Op1( 2), CRn( 0), CRm(12), Op2( 0), access_gic_sgi }, /* ICC_SGI0R */
2028 { SYS_DESC(SYS_AARCH32_CNTP_CVAL), access_arch_timer },
2029 };
2030
2031 static int check_sysreg_table(const struct sys_reg_desc *table, unsigned int n,
2032 bool is_32)
2033 {
2034 unsigned int i;
2035
2036 for (i = 0; i < n; i++) {
2037 if (!is_32 && table[i].reg && !table[i].reset) {
2038 kvm_err("sys_reg table %p entry %d has lacks reset\n",
2039 table, i);
2040 return 1;
2041 }
2042
2043 if (i && cmp_sys_reg(&table[i-1], &table[i]) >= 0) {
2044 kvm_err("sys_reg table %p out of order (%d)\n", table, i - 1);
2045 return 1;
2046 }
2047 }
2048
2049 return 0;
2050 }
2051
2052 static int match_sys_reg(const void *key, const void *elt)
2053 {
2054 const unsigned long pval = (unsigned long)key;
2055 const struct sys_reg_desc *r = elt;
2056
2057 return pval - reg_to_encoding(r);
2058 }
2059
2060 static const struct sys_reg_desc *find_reg(const struct sys_reg_params *params,
2061 const struct sys_reg_desc table[],
2062 unsigned int num)
2063 {
2064 unsigned long pval = reg_to_encoding(params);
2065
2066 return bsearch((void *)pval, table, num, sizeof(table[0]), match_sys_reg);
2067 }
2068
2069 int kvm_handle_cp14_load_store(struct kvm_vcpu *vcpu)
2070 {
2071 kvm_inject_undefined(vcpu);
2072 return 1;
2073 }
2074
2075 static void perform_access(struct kvm_vcpu *vcpu,
2076 struct sys_reg_params *params,
2077 const struct sys_reg_desc *r)
2078 {
2079 trace_kvm_sys_access(*vcpu_pc(vcpu), params, r);
2080
2081 /* Check for regs disabled by runtime config */
2082 if (sysreg_hidden(vcpu, r)) {
2083 kvm_inject_undefined(vcpu);
2084 return;
2085 }
2086
2087 /*
2088 * Not having an accessor means that we have configured a trap
2089 * that we don't know how to handle. This certainly qualifies
2090 * as a gross bug that should be fixed right away.
2091 */
2092 BUG_ON(!r->access);
2093
2094 /* Skip instruction if instructed so */
2095 if (likely(r->access(vcpu, params, r)))
2096 kvm_incr_pc(vcpu);
2097 }
2098
2099 /*
2100 * emulate_cp -- tries to match a sys_reg access in a handling table, and
2101 * call the corresponding trap handler.
2102 *
2103 * @params: pointer to the descriptor of the access
2104 * @table: array of trap descriptors
2105 * @num: size of the trap descriptor array
2106 *
2107 * Return 0 if the access has been handled, and -1 if not.
2108 */
2109 static int emulate_cp(struct kvm_vcpu *vcpu,
2110 struct sys_reg_params *params,
2111 const struct sys_reg_desc *table,
2112 size_t num)
2113 {
2114 const struct sys_reg_desc *r;
2115
2116 if (!table)
2117 return -1; /* Not handled */
2118
2119 r = find_reg(params, table, num);
2120
2121 if (r) {
2122 perform_access(vcpu, params, r);
2123 return 0;
2124 }
2125
2126 /* Not handled */
2127 return -1;
2128 }
2129
2130 static void unhandled_cp_access(struct kvm_vcpu *vcpu,
2131 struct sys_reg_params *params)
2132 {
2133 u8 esr_ec = kvm_vcpu_trap_get_class(vcpu);
2134 int cp = -1;
2135
2136 switch (esr_ec) {
2137 case ESR_ELx_EC_CP15_32:
2138 case ESR_ELx_EC_CP15_64:
2139 cp = 15;
2140 break;
2141 case ESR_ELx_EC_CP14_MR:
2142 case ESR_ELx_EC_CP14_64:
2143 cp = 14;
2144 break;
2145 default:
2146 WARN_ON(1);
2147 }
2148
2149 print_sys_reg_msg(params,
2150 "Unsupported guest CP%d access at: %08lx [%08lx]\n",
2151 cp, *vcpu_pc(vcpu), *vcpu_cpsr(vcpu));
2152 kvm_inject_undefined(vcpu);
2153 }
2154
2155 /**
2156 * kvm_handle_cp_64 -- handles a mrrc/mcrr trap on a guest CP14/CP15 access
2157 * @vcpu: The VCPU pointer
2158 * @run: The kvm_run struct
2159 */
2160 static int kvm_handle_cp_64(struct kvm_vcpu *vcpu,
2161 const struct sys_reg_desc *global,
2162 size_t nr_global)
2163 {
2164 struct sys_reg_params params;
2165 u32 esr = kvm_vcpu_get_esr(vcpu);
2166 int Rt = kvm_vcpu_sys_get_rt(vcpu);
2167 int Rt2 = (esr >> 10) & 0x1f;
2168
2169 params.CRm = (esr >> 1) & 0xf;
2170 params.is_write = ((esr & 1) == 0);
2171
2172 params.Op0 = 0;
2173 params.Op1 = (esr >> 16) & 0xf;
2174 params.Op2 = 0;
2175 params.CRn = 0;
2176
2177 /*
2178 * Make a 64-bit value out of Rt and Rt2. As we use the same trap
2179 * backends between AArch32 and AArch64, we get away with it.
2180 */
2181 if (params.is_write) {
2182 params.regval = vcpu_get_reg(vcpu, Rt) & 0xffffffff;
2183 params.regval |= vcpu_get_reg(vcpu, Rt2) << 32;
2184 }
2185
2186 /*
2187 * If the table contains a handler, handle the
2188 * potential register operation in the case of a read and return
2189 * with success.
2190 */
2191 if (!emulate_cp(vcpu, &params, global, nr_global)) {
2192 /* Split up the value between registers for the read side */
2193 if (!params.is_write) {
2194 vcpu_set_reg(vcpu, Rt, lower_32_bits(params.regval));
2195 vcpu_set_reg(vcpu, Rt2, upper_32_bits(params.regval));
2196 }
2197
2198 return 1;
2199 }
2200
2201 unhandled_cp_access(vcpu, &params);
2202 return 1;
2203 }
2204
2205 /**
2206 * kvm_handle_cp_32 -- handles a mrc/mcr trap on a guest CP14/CP15 access
2207 * @vcpu: The VCPU pointer
2208 * @run: The kvm_run struct
2209 */
2210 static int kvm_handle_cp_32(struct kvm_vcpu *vcpu,
2211 const struct sys_reg_desc *global,
2212 size_t nr_global)
2213 {
2214 struct sys_reg_params params;
2215 u32 esr = kvm_vcpu_get_esr(vcpu);
2216 int Rt = kvm_vcpu_sys_get_rt(vcpu);
2217
2218 params.CRm = (esr >> 1) & 0xf;
2219 params.regval = vcpu_get_reg(vcpu, Rt);
2220 params.is_write = ((esr & 1) == 0);
2221 params.CRn = (esr >> 10) & 0xf;
2222 params.Op0 = 0;
2223 params.Op1 = (esr >> 14) & 0x7;
2224 params.Op2 = (esr >> 17) & 0x7;
2225
2226 if (!emulate_cp(vcpu, &params, global, nr_global)) {
2227 if (!params.is_write)
2228 vcpu_set_reg(vcpu, Rt, params.regval);
2229 return 1;
2230 }
2231
2232 unhandled_cp_access(vcpu, &params);
2233 return 1;
2234 }
2235
2236 int kvm_handle_cp15_64(struct kvm_vcpu *vcpu)
2237 {
2238 return kvm_handle_cp_64(vcpu, cp15_64_regs, ARRAY_SIZE(cp15_64_regs));
2239 }
2240
2241 int kvm_handle_cp15_32(struct kvm_vcpu *vcpu)
2242 {
2243 return kvm_handle_cp_32(vcpu, cp15_regs, ARRAY_SIZE(cp15_regs));
2244 }
2245
2246 int kvm_handle_cp14_64(struct kvm_vcpu *vcpu)
2247 {
2248 return kvm_handle_cp_64(vcpu, cp14_64_regs, ARRAY_SIZE(cp14_64_regs));
2249 }
2250
2251 int kvm_handle_cp14_32(struct kvm_vcpu *vcpu)
2252 {
2253 return kvm_handle_cp_32(vcpu, cp14_regs, ARRAY_SIZE(cp14_regs));
2254 }
2255
2256 static bool is_imp_def_sys_reg(struct sys_reg_params *params)
2257 {
2258 // See ARM DDI 0487E.a, section D12.3.2
2259 return params->Op0 == 3 && (params->CRn & 0b1011) == 0b1011;
2260 }
2261
2262 static int emulate_sys_reg(struct kvm_vcpu *vcpu,
2263 struct sys_reg_params *params)
2264 {
2265 const struct sys_reg_desc *r;
2266
2267 r = find_reg(params, sys_reg_descs, ARRAY_SIZE(sys_reg_descs));
2268
2269 if (likely(r)) {
2270 perform_access(vcpu, params, r);
2271 } else if (is_imp_def_sys_reg(params)) {
2272 kvm_inject_undefined(vcpu);
2273 } else {
2274 print_sys_reg_msg(params,
2275 "Unsupported guest sys_reg access at: %lx [%08lx]\n",
2276 *vcpu_pc(vcpu), *vcpu_cpsr(vcpu));
2277 kvm_inject_undefined(vcpu);
2278 }
2279 return 1;
2280 }
2281
2282 /**
2283 * kvm_reset_sys_regs - sets system registers to reset value
2284 * @vcpu: The VCPU pointer
2285 *
2286 * This function finds the right table above and sets the registers on the
2287 * virtual CPU struct to their architecturally defined reset values.
2288 */
2289 void kvm_reset_sys_regs(struct kvm_vcpu *vcpu)
2290 {
2291 unsigned long i;
2292
2293 for (i = 0; i < ARRAY_SIZE(sys_reg_descs); i++)
2294 if (sys_reg_descs[i].reset)
2295 sys_reg_descs[i].reset(vcpu, &sys_reg_descs[i]);
2296 }
2297
2298 /**
2299 * kvm_handle_sys_reg -- handles a mrs/msr trap on a guest sys_reg access
2300 * @vcpu: The VCPU pointer
2301 */
2302 int kvm_handle_sys_reg(struct kvm_vcpu *vcpu)
2303 {
2304 struct sys_reg_params params;
2305 unsigned long esr = kvm_vcpu_get_esr(vcpu);
2306 int Rt = kvm_vcpu_sys_get_rt(vcpu);
2307 int ret;
2308
2309 trace_kvm_handle_sys_reg(esr);
2310
2311 params.Op0 = (esr >> 20) & 3;
2312 params.Op1 = (esr >> 14) & 0x7;
2313 params.CRn = (esr >> 10) & 0xf;
2314 params.CRm = (esr >> 1) & 0xf;
2315 params.Op2 = (esr >> 17) & 0x7;
2316 params.regval = vcpu_get_reg(vcpu, Rt);
2317 params.is_write = !(esr & 1);
2318
2319 ret = emulate_sys_reg(vcpu, &params);
2320
2321 if (!params.is_write)
2322 vcpu_set_reg(vcpu, Rt, params.regval);
2323 return ret;
2324 }
2325
2326 /******************************************************************************
2327 * Userspace API
2328 *****************************************************************************/
2329
2330 static bool index_to_params(u64 id, struct sys_reg_params *params)
2331 {
2332 switch (id & KVM_REG_SIZE_MASK) {
2333 case KVM_REG_SIZE_U64:
2334 /* Any unused index bits means it's not valid. */
2335 if (id & ~(KVM_REG_ARCH_MASK | KVM_REG_SIZE_MASK
2336 | KVM_REG_ARM_COPROC_MASK
2337 | KVM_REG_ARM64_SYSREG_OP0_MASK
2338 | KVM_REG_ARM64_SYSREG_OP1_MASK
2339 | KVM_REG_ARM64_SYSREG_CRN_MASK
2340 | KVM_REG_ARM64_SYSREG_CRM_MASK
2341 | KVM_REG_ARM64_SYSREG_OP2_MASK))
2342 return false;
2343 params->Op0 = ((id & KVM_REG_ARM64_SYSREG_OP0_MASK)
2344 >> KVM_REG_ARM64_SYSREG_OP0_SHIFT);
2345 params->Op1 = ((id & KVM_REG_ARM64_SYSREG_OP1_MASK)
2346 >> KVM_REG_ARM64_SYSREG_OP1_SHIFT);
2347 params->CRn = ((id & KVM_REG_ARM64_SYSREG_CRN_MASK)
2348 >> KVM_REG_ARM64_SYSREG_CRN_SHIFT);
2349 params->CRm = ((id & KVM_REG_ARM64_SYSREG_CRM_MASK)
2350 >> KVM_REG_ARM64_SYSREG_CRM_SHIFT);
2351 params->Op2 = ((id & KVM_REG_ARM64_SYSREG_OP2_MASK)
2352 >> KVM_REG_ARM64_SYSREG_OP2_SHIFT);
2353 return true;
2354 default:
2355 return false;
2356 }
2357 }
2358
2359 const struct sys_reg_desc *find_reg_by_id(u64 id,
2360 struct sys_reg_params *params,
2361 const struct sys_reg_desc table[],
2362 unsigned int num)
2363 {
2364 if (!index_to_params(id, params))
2365 return NULL;
2366
2367 return find_reg(params, table, num);
2368 }
2369
2370 /* Decode an index value, and find the sys_reg_desc entry. */
2371 static const struct sys_reg_desc *index_to_sys_reg_desc(struct kvm_vcpu *vcpu,
2372 u64 id)
2373 {
2374 const struct sys_reg_desc *r;
2375 struct sys_reg_params params;
2376
2377 /* We only do sys_reg for now. */
2378 if ((id & KVM_REG_ARM_COPROC_MASK) != KVM_REG_ARM64_SYSREG)
2379 return NULL;
2380
2381 if (!index_to_params(id, &params))
2382 return NULL;
2383
2384 r = find_reg(&params, sys_reg_descs, ARRAY_SIZE(sys_reg_descs));
2385
2386 /* Not saved in the sys_reg array and not otherwise accessible? */
2387 if (r && !(r->reg || r->get_user))
2388 r = NULL;
2389
2390 return r;
2391 }
2392
2393 /*
2394 * These are the invariant sys_reg registers: we let the guest see the
2395 * host versions of these, so they're part of the guest state.
2396 *
2397 * A future CPU may provide a mechanism to present different values to
2398 * the guest, or a future kvm may trap them.
2399 */
2400
2401 #define FUNCTION_INVARIANT(reg) \
2402 static void get_##reg(struct kvm_vcpu *v, \
2403 const struct sys_reg_desc *r) \
2404 { \
2405 ((struct sys_reg_desc *)r)->val = read_sysreg(reg); \
2406 }
2407
2408 FUNCTION_INVARIANT(midr_el1)
2409 FUNCTION_INVARIANT(revidr_el1)
2410 FUNCTION_INVARIANT(clidr_el1)
2411 FUNCTION_INVARIANT(aidr_el1)
2412
2413 static void get_ctr_el0(struct kvm_vcpu *v, const struct sys_reg_desc *r)
2414 {
2415 ((struct sys_reg_desc *)r)->val = read_sanitised_ftr_reg(SYS_CTR_EL0);
2416 }
2417
2418 /* ->val is filled in by kvm_sys_reg_table_init() */
2419 static struct sys_reg_desc invariant_sys_regs[] = {
2420 { SYS_DESC(SYS_MIDR_EL1), NULL, get_midr_el1 },
2421 { SYS_DESC(SYS_REVIDR_EL1), NULL, get_revidr_el1 },
2422 { SYS_DESC(SYS_CLIDR_EL1), NULL, get_clidr_el1 },
2423 { SYS_DESC(SYS_AIDR_EL1), NULL, get_aidr_el1 },
2424 { SYS_DESC(SYS_CTR_EL0), NULL, get_ctr_el0 },
2425 };
2426
2427 static int reg_from_user(u64 *val, const void __user *uaddr, u64 id)
2428 {
2429 if (copy_from_user(val, uaddr, KVM_REG_SIZE(id)) != 0)
2430 return -EFAULT;
2431 return 0;
2432 }
2433
2434 static int reg_to_user(void __user *uaddr, const u64 *val, u64 id)
2435 {
2436 if (copy_to_user(uaddr, val, KVM_REG_SIZE(id)) != 0)
2437 return -EFAULT;
2438 return 0;
2439 }
2440
2441 static int get_invariant_sys_reg(u64 id, void __user *uaddr)
2442 {
2443 struct sys_reg_params params;
2444 const struct sys_reg_desc *r;
2445
2446 r = find_reg_by_id(id, &params, invariant_sys_regs,
2447 ARRAY_SIZE(invariant_sys_regs));
2448 if (!r)
2449 return -ENOENT;
2450
2451 return reg_to_user(uaddr, &r->val, id);
2452 }
2453
2454 static int set_invariant_sys_reg(u64 id, void __user *uaddr)
2455 {
2456 struct sys_reg_params params;
2457 const struct sys_reg_desc *r;
2458 int err;
2459 u64 val = 0; /* Make sure high bits are 0 for 32-bit regs */
2460
2461 r = find_reg_by_id(id, &params, invariant_sys_regs,
2462 ARRAY_SIZE(invariant_sys_regs));
2463 if (!r)
2464 return -ENOENT;
2465
2466 err = reg_from_user(&val, uaddr, id);
2467 if (err)
2468 return err;
2469
2470 /* This is what we mean by invariant: you can't change it. */
2471 if (r->val != val)
2472 return -EINVAL;
2473
2474 return 0;
2475 }
2476
2477 static bool is_valid_cache(u32 val)
2478 {
2479 u32 level, ctype;
2480
2481 if (val >= CSSELR_MAX)
2482 return false;
2483
2484 /* Bottom bit is Instruction or Data bit. Next 3 bits are level. */
2485 level = (val >> 1);
2486 ctype = (cache_levels >> (level * 3)) & 7;
2487
2488 switch (ctype) {
2489 case 0: /* No cache */
2490 return false;
2491 case 1: /* Instruction cache only */
2492 return (val & 1);
2493 case 2: /* Data cache only */
2494 case 4: /* Unified cache */
2495 return !(val & 1);
2496 case 3: /* Separate instruction and data caches */
2497 return true;
2498 default: /* Reserved: we can't know instruction or data. */
2499 return false;
2500 }
2501 }
2502
2503 static int demux_c15_get(u64 id, void __user *uaddr)
2504 {
2505 u32 val;
2506 u32 __user *uval = uaddr;
2507
2508 /* Fail if we have unknown bits set. */
2509 if (id & ~(KVM_REG_ARCH_MASK|KVM_REG_SIZE_MASK|KVM_REG_ARM_COPROC_MASK
2510 | ((1 << KVM_REG_ARM_COPROC_SHIFT)-1)))
2511 return -ENOENT;
2512
2513 switch (id & KVM_REG_ARM_DEMUX_ID_MASK) {
2514 case KVM_REG_ARM_DEMUX_ID_CCSIDR:
2515 if (KVM_REG_SIZE(id) != 4)
2516 return -ENOENT;
2517 val = (id & KVM_REG_ARM_DEMUX_VAL_MASK)
2518 >> KVM_REG_ARM_DEMUX_VAL_SHIFT;
2519 if (!is_valid_cache(val))
2520 return -ENOENT;
2521
2522 return put_user(get_ccsidr(val), uval);
2523 default:
2524 return -ENOENT;
2525 }
2526 }
2527
2528 static int demux_c15_set(u64 id, void __user *uaddr)
2529 {
2530 u32 val, newval;
2531 u32 __user *uval = uaddr;
2532
2533 /* Fail if we have unknown bits set. */
2534 if (id & ~(KVM_REG_ARCH_MASK|KVM_REG_SIZE_MASK|KVM_REG_ARM_COPROC_MASK
2535 | ((1 << KVM_REG_ARM_COPROC_SHIFT)-1)))
2536 return -ENOENT;
2537
2538 switch (id & KVM_REG_ARM_DEMUX_ID_MASK) {
2539 case KVM_REG_ARM_DEMUX_ID_CCSIDR:
2540 if (KVM_REG_SIZE(id) != 4)
2541 return -ENOENT;
2542 val = (id & KVM_REG_ARM_DEMUX_VAL_MASK)
2543 >> KVM_REG_ARM_DEMUX_VAL_SHIFT;
2544 if (!is_valid_cache(val))
2545 return -ENOENT;
2546
2547 if (get_user(newval, uval))
2548 return -EFAULT;
2549
2550 /* This is also invariant: you can't change it. */
2551 if (newval != get_ccsidr(val))
2552 return -EINVAL;
2553 return 0;
2554 default:
2555 return -ENOENT;
2556 }
2557 }
2558
2559 int kvm_arm_sys_reg_get_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg)
2560 {
2561 const struct sys_reg_desc *r;
2562 void __user *uaddr = (void __user *)(unsigned long)reg->addr;
2563
2564 if ((reg->id & KVM_REG_ARM_COPROC_MASK) == KVM_REG_ARM_DEMUX)
2565 return demux_c15_get(reg->id, uaddr);
2566
2567 if (KVM_REG_SIZE(reg->id) != sizeof(__u64))
2568 return -ENOENT;
2569
2570 r = index_to_sys_reg_desc(vcpu, reg->id);
2571 if (!r)
2572 return get_invariant_sys_reg(reg->id, uaddr);
2573
2574 /* Check for regs disabled by runtime config */
2575 if (sysreg_hidden(vcpu, r))
2576 return -ENOENT;
2577
2578 if (r->get_user)
2579 return (r->get_user)(vcpu, r, reg, uaddr);
2580
2581 return reg_to_user(uaddr, &__vcpu_sys_reg(vcpu, r->reg), reg->id);
2582 }
2583
2584 int kvm_arm_sys_reg_set_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg)
2585 {
2586 const struct sys_reg_desc *r;
2587 void __user *uaddr = (void __user *)(unsigned long)reg->addr;
2588
2589 if ((reg->id & KVM_REG_ARM_COPROC_MASK) == KVM_REG_ARM_DEMUX)
2590 return demux_c15_set(reg->id, uaddr);
2591
2592 if (KVM_REG_SIZE(reg->id) != sizeof(__u64))
2593 return -ENOENT;
2594
2595 r = index_to_sys_reg_desc(vcpu, reg->id);
2596 if (!r)
2597 return set_invariant_sys_reg(reg->id, uaddr);
2598
2599 /* Check for regs disabled by runtime config */
2600 if (sysreg_hidden(vcpu, r))
2601 return -ENOENT;
2602
2603 if (r->set_user)
2604 return (r->set_user)(vcpu, r, reg, uaddr);
2605
2606 return reg_from_user(&__vcpu_sys_reg(vcpu, r->reg), uaddr, reg->id);
2607 }
2608
2609 static unsigned int num_demux_regs(void)
2610 {
2611 unsigned int i, count = 0;
2612
2613 for (i = 0; i < CSSELR_MAX; i++)
2614 if (is_valid_cache(i))
2615 count++;
2616
2617 return count;
2618 }
2619
2620 static int write_demux_regids(u64 __user *uindices)
2621 {
2622 u64 val = KVM_REG_ARM64 | KVM_REG_SIZE_U32 | KVM_REG_ARM_DEMUX;
2623 unsigned int i;
2624
2625 val |= KVM_REG_ARM_DEMUX_ID_CCSIDR;
2626 for (i = 0; i < CSSELR_MAX; i++) {
2627 if (!is_valid_cache(i))
2628 continue;
2629 if (put_user(val | i, uindices))
2630 return -EFAULT;
2631 uindices++;
2632 }
2633 return 0;
2634 }
2635
2636 static u64 sys_reg_to_index(const struct sys_reg_desc *reg)
2637 {
2638 return (KVM_REG_ARM64 | KVM_REG_SIZE_U64 |
2639 KVM_REG_ARM64_SYSREG |
2640 (reg->Op0 << KVM_REG_ARM64_SYSREG_OP0_SHIFT) |
2641 (reg->Op1 << KVM_REG_ARM64_SYSREG_OP1_SHIFT) |
2642 (reg->CRn << KVM_REG_ARM64_SYSREG_CRN_SHIFT) |
2643 (reg->CRm << KVM_REG_ARM64_SYSREG_CRM_SHIFT) |
2644 (reg->Op2 << KVM_REG_ARM64_SYSREG_OP2_SHIFT));
2645 }
2646
2647 static bool copy_reg_to_user(const struct sys_reg_desc *reg, u64 __user **uind)
2648 {
2649 if (!*uind)
2650 return true;
2651
2652 if (put_user(sys_reg_to_index(reg), *uind))
2653 return false;
2654
2655 (*uind)++;
2656 return true;
2657 }
2658
2659 static int walk_one_sys_reg(const struct kvm_vcpu *vcpu,
2660 const struct sys_reg_desc *rd,
2661 u64 __user **uind,
2662 unsigned int *total)
2663 {
2664 /*
2665 * Ignore registers we trap but don't save,
2666 * and for which no custom user accessor is provided.
2667 */
2668 if (!(rd->reg || rd->get_user))
2669 return 0;
2670
2671 if (sysreg_hidden(vcpu, rd))
2672 return 0;
2673
2674 if (!copy_reg_to_user(rd, uind))
2675 return -EFAULT;
2676
2677 (*total)++;
2678 return 0;
2679 }
2680
2681 /* Assumed ordered tables, see kvm_sys_reg_table_init. */
2682 static int walk_sys_regs(struct kvm_vcpu *vcpu, u64 __user *uind)
2683 {
2684 const struct sys_reg_desc *i2, *end2;
2685 unsigned int total = 0;
2686 int err;
2687
2688 i2 = sys_reg_descs;
2689 end2 = sys_reg_descs + ARRAY_SIZE(sys_reg_descs);
2690
2691 while (i2 != end2) {
2692 err = walk_one_sys_reg(vcpu, i2++, &uind, &total);
2693 if (err)
2694 return err;
2695 }
2696 return total;
2697 }
2698
2699 unsigned long kvm_arm_num_sys_reg_descs(struct kvm_vcpu *vcpu)
2700 {
2701 return ARRAY_SIZE(invariant_sys_regs)
2702 + num_demux_regs()
2703 + walk_sys_regs(vcpu, (u64 __user *)NULL);
2704 }
2705
2706 int kvm_arm_copy_sys_reg_indices(struct kvm_vcpu *vcpu, u64 __user *uindices)
2707 {
2708 unsigned int i;
2709 int err;
2710
2711 /* Then give them all the invariant registers' indices. */
2712 for (i = 0; i < ARRAY_SIZE(invariant_sys_regs); i++) {
2713 if (put_user(sys_reg_to_index(&invariant_sys_regs[i]), uindices))
2714 return -EFAULT;
2715 uindices++;
2716 }
2717
2718 err = walk_sys_regs(vcpu, uindices);
2719 if (err < 0)
2720 return err;
2721 uindices += err;
2722
2723 return write_demux_regids(uindices);
2724 }
2725
2726 void kvm_sys_reg_table_init(void)
2727 {
2728 unsigned int i;
2729 struct sys_reg_desc clidr;
2730
2731 /* Make sure tables are unique and in order. */
2732 BUG_ON(check_sysreg_table(sys_reg_descs, ARRAY_SIZE(sys_reg_descs), false));
2733 BUG_ON(check_sysreg_table(cp14_regs, ARRAY_SIZE(cp14_regs), true));
2734 BUG_ON(check_sysreg_table(cp14_64_regs, ARRAY_SIZE(cp14_64_regs), true));
2735 BUG_ON(check_sysreg_table(cp15_regs, ARRAY_SIZE(cp15_regs), true));
2736 BUG_ON(check_sysreg_table(cp15_64_regs, ARRAY_SIZE(cp15_64_regs), true));
2737 BUG_ON(check_sysreg_table(invariant_sys_regs, ARRAY_SIZE(invariant_sys_regs), false));
2738
2739 /* We abuse the reset function to overwrite the table itself. */
2740 for (i = 0; i < ARRAY_SIZE(invariant_sys_regs); i++)
2741 invariant_sys_regs[i].reset(NULL, &invariant_sys_regs[i]);
2742
2743 /*
2744 * CLIDR format is awkward, so clean it up. See ARM B4.1.20:
2745 *
2746 * If software reads the Cache Type fields from Ctype1
2747 * upwards, once it has seen a value of 0b000, no caches
2748 * exist at further-out levels of the hierarchy. So, for
2749 * example, if Ctype3 is the first Cache Type field with a
2750 * value of 0b000, the values of Ctype4 to Ctype7 must be
2751 * ignored.
2752 */
2753 get_clidr_el1(NULL, &clidr); /* Ugly... */
2754 cache_levels = clidr.val;
2755 for (i = 0; i < 7; i++)
2756 if (((cache_levels >> (i*3)) & 7) == 0)
2757 break;
2758 /* Clear all higher bits. */
2759 cache_levels &= (1 << (i*3))-1;
2760 }
Ubuntu Hirsute Linux kernel mirror