]> git.proxmox.com Git - mirror_ubuntu-zesty-kernel.git/blame - arch/arm64/kvm/sys_regs.c
projects / mirror_ubuntu-zesty-kernel.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
arm64: KVM: Add helper to handle PMCR register bits
[mirror_ubuntu-zesty-kernel.git] / arch / arm64 / kvm / sys_regs.c
CommitLineData
7c8c5e6a
MZ		 1/*
2 * Copyright (C) 2012,2013 - ARM Ltd
3 * Author: Marc Zyngier <marc.zyngier@arm.com>
4 *
5 * Derived from arch/arm/kvm/coproc.c:
6 * Copyright (C) 2012 - Virtual Open Systems and Columbia University
7 * Authors: Rusty Russell <rusty@rustcorp.com.au>
8 * Christoffer Dall <c.dall@virtualopensystems.com>
9 *
10 * This program is free software; you can redistribute it and/or modify
11 * it under the terms of the GNU General Public License, version 2, as
12 * published by the Free Software Foundation.
13 *
14 * This program is distributed in the hope that it will be useful,
15 * but WITHOUT ANY WARRANTY; without even the implied warranty of
16 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 * GNU General Public License for more details.
18 *
19 * You should have received a copy of the GNU General Public License
20 * along with this program. If not, see <http://www.gnu.org/licenses/>.
21 */
22
7c8c5e6a 23#include <linux/kvm_host.h>
c6d01a94 24#include <linux/mm.h>
7c8c5e6a 25#include <linux/uaccess.h>
c6d01a94 26
7c8c5e6a
MZ		 27#include <asm/cacheflush.h>
28#include <asm/cputype.h>
0c557ed4 29#include <asm/debug-monitors.h>
c6d01a94
MR		 30#include <asm/esr.h>
31#include <asm/kvm_arm.h>
9d8415d6 32#include <asm/kvm_asm.h>
c6d01a94
MR		 33#include <asm/kvm_coproc.h>
34#include <asm/kvm_emulate.h>
35#include <asm/kvm_host.h>
36#include <asm/kvm_mmu.h>
ab946834 37#include <asm/perf_event.h>
c6d01a94 38
7c8c5e6a
MZ		 39#include <trace/events/kvm.h>
40
41#include "sys_regs.h"
42
eef8c85a
AB		 43#include "trace.h"
44
7c8c5e6a
MZ		 45/*
46 * All of this file is extremly similar to the ARM coproc.c, but the
47 * types are different. My gut feeling is that it should be pretty
48 * easy to merge, but that would be an ABI breakage -- again. VFP
49 * would also need to be abstracted.
62a89c44
MZ		 50 *
51 * For AArch32, we only take care of what is being trapped. Anything
52 * that has to do with init and userspace access has to go via the
53 * 64bit interface.
7c8c5e6a
MZ		 54 */
55
56/* 3 bits per cache level, as per CLIDR, but non-existent caches always 0 */
57static u32 cache_levels;
58
59/* CSSELR values; used to index KVM_REG_ARM_DEMUX_ID_CCSIDR */
60#define CSSELR_MAX 12
61
62/* Which cache CCSIDR represents depends on CSSELR value. */
63static u32 get_ccsidr(u32 csselr)
64{
65 u32 ccsidr;
66
67 /* Make sure noone else changes CSSELR during this! */
68 local_irq_disable();
69 /* Put value into CSSELR */
70 asm volatile("msr csselr_el1, %x0" : : "r" (csselr));
71 isb();
72 /* Read result out of CCSIDR */
73 asm volatile("mrs %0, ccsidr_el1" : "=r" (ccsidr));
74 local_irq_enable();
75
76 return ccsidr;
77}
78
3c1e7165
MZ		 79/*
80 * See note at ARMv7 ARM B1.14.4 (TL;DR: S/W ops are not easily virtualized).
81 */
7c8c5e6a 82static bool access_dcsw(struct kvm_vcpu *vcpu,
3fec037d 83 struct sys_reg_params *p,
7c8c5e6a
MZ		 84 const struct sys_reg_desc *r)
85{
7c8c5e6a
MZ		 86 if (!p->is_write)
87 return read_from_write_only(vcpu, p);
88
3c1e7165 89 kvm_set_way_flush(vcpu);
7c8c5e6a
MZ		 90 return true;
91}
92
4d44923b
MZ		 93/*
94 * Generic accessor for VM registers. Only called as long as HCR_TVM
3c1e7165
MZ		 95 * is set. If the guest enables the MMU, we stop trapping the VM
96 * sys_regs and leave it in complete control of the caches.
4d44923b
MZ		 97 */
98static bool access_vm_reg(struct kvm_vcpu *vcpu,
3fec037d 99 struct sys_reg_params *p,
4d44923b
MZ		 100 const struct sys_reg_desc *r)
101{
3c1e7165 102 bool was_enabled = vcpu_has_cache_enabled(vcpu);
4d44923b
MZ		 103
104 BUG_ON(!p->is_write);
105
dedf97e8 106 if (!p->is_aarch32) {
2ec5be3d 107 vcpu_sys_reg(vcpu, r->reg) = p->regval;
dedf97e8
MZ		 108 } else {
109 if (!p->is_32bit)
2ec5be3d
PF		 110 vcpu_cp15_64_high(vcpu, r->reg) = upper_32_bits(p->regval);
111 vcpu_cp15_64_low(vcpu, r->reg) = lower_32_bits(p->regval);
dedf97e8 112 }
f0a3eaff 113
3c1e7165 114 kvm_toggle_cache(vcpu, was_enabled);
4d44923b
MZ		 115 return true;
116}
117
6d52f35a
AP		 118/*
119 * Trap handler for the GICv3 SGI generation system register.
120 * Forward the request to the VGIC emulation.
121 * The cp15_64 code makes sure this automatically works
122 * for both AArch64 and AArch32 accesses.
123 */
124static bool access_gic_sgi(struct kvm_vcpu *vcpu,
3fec037d 125 struct sys_reg_params *p,
6d52f35a
AP		 126 const struct sys_reg_desc *r)
127{
6d52f35a
AP		 128 if (!p->is_write)
129 return read_from_write_only(vcpu, p);
130
2ec5be3d 131 vgic_v3_dispatch_sgi(vcpu, p->regval);
6d52f35a
AP		 132
133 return true;
134}
135
7609c125 136static bool trap_raz_wi(struct kvm_vcpu *vcpu,
3fec037d 137 struct sys_reg_params *p,
7609c125 138 const struct sys_reg_desc *r)
7c8c5e6a
MZ		 139{
140 if (p->is_write)
141 return ignore_write(vcpu, p);
142 else
143 return read_zero(vcpu, p);
144}
145
0c557ed4 146static bool trap_oslsr_el1(struct kvm_vcpu *vcpu,
3fec037d 147 struct sys_reg_params *p,
0c557ed4
MZ		 148 const struct sys_reg_desc *r)
149{
150 if (p->is_write) {
151 return ignore_write(vcpu, p);
152 } else {
2ec5be3d 153 p->regval = (1 << 3);
0c557ed4
MZ		 154 return true;
155 }
156}
157
158static bool trap_dbgauthstatus_el1(struct kvm_vcpu *vcpu,
3fec037d 159 struct sys_reg_params *p,
0c557ed4
MZ		 160 const struct sys_reg_desc *r)
161{
162 if (p->is_write) {
163 return ignore_write(vcpu, p);
164 } else {
165 u32 val;
166 asm volatile("mrs %0, dbgauthstatus_el1" : "=r" (val));
2ec5be3d 167 p->regval = val;
0c557ed4
MZ		 168 return true;
169 }
170}
171
172/*
173 * We want to avoid world-switching all the DBG registers all the
174 * time:
175 *
176 * - If we've touched any debug register, it is likely that we're
177 * going to touch more of them. It then makes sense to disable the
178 * traps and start doing the save/restore dance
179 * - If debug is active (DBG_MDSCR_KDE or DBG_MDSCR_MDE set), it is
180 * then mandatory to save/restore the registers, as the guest
181 * depends on them.
182 *
183 * For this, we use a DIRTY bit, indicating the guest has modified the
184 * debug registers, used as follow:
185 *
186 * On guest entry:
187 * - If the dirty bit is set (because we're coming back from trapping),
188 * disable the traps, save host registers, restore guest registers.
189 * - If debug is actively in use (DBG_MDSCR_KDE or DBG_MDSCR_MDE set),
190 * set the dirty bit, disable the traps, save host registers,
191 * restore guest registers.
192 * - Otherwise, enable the traps
193 *
194 * On guest exit:
195 * - If the dirty bit is set, save guest registers, restore host
196 * registers and clear the dirty bit. This ensure that the host can
197 * now use the debug registers.
198 */
199static bool trap_debug_regs(struct kvm_vcpu *vcpu,
3fec037d 200 struct sys_reg_params *p,
0c557ed4
MZ		 201 const struct sys_reg_desc *r)
202{
203 if (p->is_write) {
2ec5be3d 204 vcpu_sys_reg(vcpu, r->reg) = p->regval;
0c557ed4
MZ		 205 vcpu->arch.debug_flags |= KVM_ARM64_DEBUG_DIRTY;
206 } else {
2ec5be3d 207 p->regval = vcpu_sys_reg(vcpu, r->reg);
0c557ed4
MZ		 208 }
209
2ec5be3d 210 trace_trap_reg(__func__, r->reg, p->is_write, p->regval);
eef8c85a 211
0c557ed4
MZ		 212 return true;
213}
214
84e690bf
AB		 215/*
216 * reg_to_dbg/dbg_to_reg
217 *
218 * A 32 bit write to a debug register leave top bits alone
219 * A 32 bit read from a debug register only returns the bottom bits
220 *
221 * All writes will set the KVM_ARM64_DEBUG_DIRTY flag to ensure the
222 * hyp.S code switches between host and guest values in future.
223 */
281243cb
MZ		 224static void reg_to_dbg(struct kvm_vcpu *vcpu,
225 struct sys_reg_params *p,
226 u64 *dbg_reg)
84e690bf 227{
2ec5be3d 228 u64 val = p->regval;
84e690bf
AB		 229
230 if (p->is_32bit) {
231 val &= 0xffffffffUL;
232 val |= ((*dbg_reg >> 32) << 32);
233 }
234
235 *dbg_reg = val;
236 vcpu->arch.debug_flags |= KVM_ARM64_DEBUG_DIRTY;
237}
238
281243cb
MZ		 239static void dbg_to_reg(struct kvm_vcpu *vcpu,
240 struct sys_reg_params *p,
241 u64 *dbg_reg)
84e690bf 242{
2ec5be3d 243 p->regval = *dbg_reg;
84e690bf 244 if (p->is_32bit)
2ec5be3d 245 p->regval &= 0xffffffffUL;
84e690bf
AB		 246}
247
281243cb
MZ		 248static bool trap_bvr(struct kvm_vcpu *vcpu,
249 struct sys_reg_params *p,
250 const struct sys_reg_desc *rd)
84e690bf
AB		 251{
252 u64 *dbg_reg = &vcpu->arch.vcpu_debug_state.dbg_bvr[rd->reg];
253
254 if (p->is_write)
255 reg_to_dbg(vcpu, p, dbg_reg);
256 else
257 dbg_to_reg(vcpu, p, dbg_reg);
258
eef8c85a
AB		 259 trace_trap_reg(__func__, rd->reg, p->is_write, *dbg_reg);
260
84e690bf
AB		 261 return true;
262}
263
264static int set_bvr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
265 const struct kvm_one_reg *reg, void __user *uaddr)
266{
267 __u64 *r = &vcpu->arch.vcpu_debug_state.dbg_bvr[rd->reg];
268
1713e5aa 269 if (copy_from_user(r, uaddr, KVM_REG_SIZE(reg->id)) != 0)
84e690bf
AB		 270 return -EFAULT;
271 return 0;
272}
273
274static int get_bvr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
275 const struct kvm_one_reg *reg, void __user *uaddr)
276{
277 __u64 *r = &vcpu->arch.vcpu_debug_state.dbg_bvr[rd->reg];
278
279 if (copy_to_user(uaddr, r, KVM_REG_SIZE(reg->id)) != 0)
280 return -EFAULT;
281 return 0;
282}
283
281243cb
MZ		 284static void reset_bvr(struct kvm_vcpu *vcpu,
285 const struct sys_reg_desc *rd)
84e690bf
AB		 286{
287 vcpu->arch.vcpu_debug_state.dbg_bvr[rd->reg] = rd->val;
288}
289
281243cb
MZ		 290static bool trap_bcr(struct kvm_vcpu *vcpu,
291 struct sys_reg_params *p,
292 const struct sys_reg_desc *rd)
84e690bf
AB		 293{
294 u64 *dbg_reg = &vcpu->arch.vcpu_debug_state.dbg_bcr[rd->reg];
295
296 if (p->is_write)
297 reg_to_dbg(vcpu, p, dbg_reg);
298 else
299 dbg_to_reg(vcpu, p, dbg_reg);
300
eef8c85a
AB		 301 trace_trap_reg(__func__, rd->reg, p->is_write, *dbg_reg);
302
84e690bf
AB		 303 return true;
304}
305
306static int set_bcr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
307 const struct kvm_one_reg *reg, void __user *uaddr)
308{
309 __u64 *r = &vcpu->arch.vcpu_debug_state.dbg_bcr[rd->reg];
310
1713e5aa 311 if (copy_from_user(r, uaddr, KVM_REG_SIZE(reg->id)) != 0)
84e690bf
AB		 312 return -EFAULT;
313
314 return 0;
315}
316
317static int get_bcr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
318 const struct kvm_one_reg *reg, void __user *uaddr)
319{
320 __u64 *r = &vcpu->arch.vcpu_debug_state.dbg_bcr[rd->reg];
321
322 if (copy_to_user(uaddr, r, KVM_REG_SIZE(reg->id)) != 0)
323 return -EFAULT;
324 return 0;
325}
326
281243cb
MZ		 327static void reset_bcr(struct kvm_vcpu *vcpu,
328 const struct sys_reg_desc *rd)
84e690bf
AB		 329{
330 vcpu->arch.vcpu_debug_state.dbg_bcr[rd->reg] = rd->val;
331}
332
281243cb
MZ		 333static bool trap_wvr(struct kvm_vcpu *vcpu,
334 struct sys_reg_params *p,
335 const struct sys_reg_desc *rd)
84e690bf
AB		 336{
337 u64 *dbg_reg = &vcpu->arch.vcpu_debug_state.dbg_wvr[rd->reg];
338
339 if (p->is_write)
340 reg_to_dbg(vcpu, p, dbg_reg);
341 else
342 dbg_to_reg(vcpu, p, dbg_reg);
343
eef8c85a
AB		 344 trace_trap_reg(__func__, rd->reg, p->is_write,
345 vcpu->arch.vcpu_debug_state.dbg_wvr[rd->reg]);
346
84e690bf
AB		 347 return true;
348}
349
350static int set_wvr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
351 const struct kvm_one_reg *reg, void __user *uaddr)
352{
353 __u64 *r = &vcpu->arch.vcpu_debug_state.dbg_wvr[rd->reg];
354
1713e5aa 355 if (copy_from_user(r, uaddr, KVM_REG_SIZE(reg->id)) != 0)
84e690bf
AB		 356 return -EFAULT;
357 return 0;
358}
359
360static int get_wvr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
361 const struct kvm_one_reg *reg, void __user *uaddr)
362{
363 __u64 *r = &vcpu->arch.vcpu_debug_state.dbg_wvr[rd->reg];
364
365 if (copy_to_user(uaddr, r, KVM_REG_SIZE(reg->id)) != 0)
366 return -EFAULT;
367 return 0;
368}
369
281243cb
MZ		 370static void reset_wvr(struct kvm_vcpu *vcpu,
371 const struct sys_reg_desc *rd)
84e690bf
AB		 372{
373 vcpu->arch.vcpu_debug_state.dbg_wvr[rd->reg] = rd->val;
374}
375
281243cb
MZ		 376static bool trap_wcr(struct kvm_vcpu *vcpu,
377 struct sys_reg_params *p,
378 const struct sys_reg_desc *rd)
84e690bf
AB		 379{
380 u64 *dbg_reg = &vcpu->arch.vcpu_debug_state.dbg_wcr[rd->reg];
381
382 if (p->is_write)
383 reg_to_dbg(vcpu, p, dbg_reg);
384 else
385 dbg_to_reg(vcpu, p, dbg_reg);
386
eef8c85a
AB		 387 trace_trap_reg(__func__, rd->reg, p->is_write, *dbg_reg);
388
84e690bf
AB		 389 return true;
390}
391
392static int set_wcr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
393 const struct kvm_one_reg *reg, void __user *uaddr)
394{
395 __u64 *r = &vcpu->arch.vcpu_debug_state.dbg_wcr[rd->reg];
396
1713e5aa 397 if (copy_from_user(r, uaddr, KVM_REG_SIZE(reg->id)) != 0)
84e690bf
AB		 398 return -EFAULT;
399 return 0;
400}
401
402static int get_wcr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
403 const struct kvm_one_reg *reg, void __user *uaddr)
404{
405 __u64 *r = &vcpu->arch.vcpu_debug_state.dbg_wcr[rd->reg];
406
407 if (copy_to_user(uaddr, r, KVM_REG_SIZE(reg->id)) != 0)
408 return -EFAULT;
409 return 0;
410}
411
281243cb
MZ		 412static void reset_wcr(struct kvm_vcpu *vcpu,
413 const struct sys_reg_desc *rd)
84e690bf
AB		 414{
415 vcpu->arch.vcpu_debug_state.dbg_wcr[rd->reg] = rd->val;
416}
417
7c8c5e6a
MZ		 418static void reset_amair_el1(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r)
419{
420 u64 amair;
421
422 asm volatile("mrs %0, amair_el1\n" : "=r" (amair));
423 vcpu_sys_reg(vcpu, AMAIR_EL1) = amair;
424}
425
426static void reset_mpidr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r)
427{
4429fc64
AP		 428 u64 mpidr;
429
7c8c5e6a 430 /*
4429fc64
AP		 431 * Map the vcpu_id into the first three affinity level fields of
432 * the MPIDR. We limit the number of VCPUs in level 0 due to a
433 * limitation to 16 CPUs in that level in the ICC_SGIxR registers
434 * of the GICv3 to be able to address each CPU directly when
435 * sending IPIs.
7c8c5e6a 436 */
4429fc64
AP		 437 mpidr = (vcpu->vcpu_id & 0x0f) << MPIDR_LEVEL_SHIFT(0);
438 mpidr |= ((vcpu->vcpu_id >> 4) & 0xff) << MPIDR_LEVEL_SHIFT(1);
439 mpidr |= ((vcpu->vcpu_id >> 12) & 0xff) << MPIDR_LEVEL_SHIFT(2);
440 vcpu_sys_reg(vcpu, MPIDR_EL1) = (1ULL << 31) | mpidr;
7c8c5e6a
MZ		 441}
442
ab946834
SZ		 443static void reset_pmcr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r)
444{
445 u64 pmcr, val;
446
447 asm volatile("mrs %0, pmcr_el0\n" : "=r" (pmcr));
448 /* Writable bits of PMCR_EL0 (ARMV8_PMU_PMCR_MASK) is reset to UNKNOWN
449 * except PMCR.E resetting to zero.
450 */
451 val = ((pmcr & ~ARMV8_PMU_PMCR_MASK)
452 | (ARMV8_PMU_PMCR_MASK & 0xdecafbad)) & (~ARMV8_PMU_PMCR_E);
453 vcpu_sys_reg(vcpu, PMCR_EL0) = val;
454}
455
456static bool access_pmcr(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
457 const struct sys_reg_desc *r)
458{
459 u64 val;
460
461 if (!kvm_arm_pmu_v3_ready(vcpu))
462 return trap_raz_wi(vcpu, p, r);
463
464 if (p->is_write) {
465 /* Only update writeable bits of PMCR */
466 val = vcpu_sys_reg(vcpu, PMCR_EL0);
467 val &= ~ARMV8_PMU_PMCR_MASK;
468 val |= p->regval & ARMV8_PMU_PMCR_MASK;
469 vcpu_sys_reg(vcpu, PMCR_EL0) = val;
76993739 470 kvm_pmu_handle_pmcr(vcpu, val);
ab946834
SZ		 471 } else {
472 /* PMCR.P & PMCR.C are RAZ */
473 val = vcpu_sys_reg(vcpu, PMCR_EL0)
474 & ~(ARMV8_PMU_PMCR_P | ARMV8_PMU_PMCR_C);
475 p->regval = val;
476 }
477
478 return true;
479}
480
3965c3ce
SZ		 481static bool access_pmselr(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
482 const struct sys_reg_desc *r)
483{
484 if (!kvm_arm_pmu_v3_ready(vcpu))
485 return trap_raz_wi(vcpu, p, r);
486
487 if (p->is_write)
488 vcpu_sys_reg(vcpu, PMSELR_EL0) = p->regval;
489 else
490 /* return PMSELR.SEL field */
491 p->regval = vcpu_sys_reg(vcpu, PMSELR_EL0)
492 & ARMV8_PMU_COUNTER_MASK;
493
494 return true;
495}
496
a86b5505
SZ		 497static bool access_pmceid(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
498 const struct sys_reg_desc *r)
499{
500 u64 pmceid;
501
502 if (!kvm_arm_pmu_v3_ready(vcpu))
503 return trap_raz_wi(vcpu, p, r);
504
505 BUG_ON(p->is_write);
506
507 if (!(p->Op2 & 1))
508 asm volatile("mrs %0, pmceid0_el0\n" : "=r" (pmceid));
509 else
510 asm volatile("mrs %0, pmceid1_el0\n" : "=r" (pmceid));
511
512 p->regval = pmceid;
513
514 return true;
515}
516
051ff581
SZ		 517static bool pmu_counter_idx_valid(struct kvm_vcpu *vcpu, u64 idx)
518{
519 u64 pmcr, val;
520
521 pmcr = vcpu_sys_reg(vcpu, PMCR_EL0);
522 val = (pmcr >> ARMV8_PMU_PMCR_N_SHIFT) & ARMV8_PMU_PMCR_N_MASK;
523 if (idx >= val && idx != ARMV8_PMU_CYCLE_IDX)
524 return false;
525
526 return true;
527}
528
529static bool access_pmu_evcntr(struct kvm_vcpu *vcpu,
530 struct sys_reg_params *p,
531 const struct sys_reg_desc *r)
532{
533 u64 idx;
534
535 if (!kvm_arm_pmu_v3_ready(vcpu))
536 return trap_raz_wi(vcpu, p, r);
537
538 if (r->CRn == 9 && r->CRm == 13) {
539 if (r->Op2 == 2) {
540 /* PMXEVCNTR_EL0 */
541 idx = vcpu_sys_reg(vcpu, PMSELR_EL0)
542 & ARMV8_PMU_COUNTER_MASK;
543 } else if (r->Op2 == 0) {
544 /* PMCCNTR_EL0 */
545 idx = ARMV8_PMU_CYCLE_IDX;
546 } else {
547 BUG();
548 }
549 } else if (r->CRn == 14 && (r->CRm & 12) == 8) {
550 /* PMEVCNTRn_EL0 */
551 idx = ((r->CRm & 3) << 3) | (r->Op2 & 7);
552 } else {
553 BUG();
554 }
555
556 if (!pmu_counter_idx_valid(vcpu, idx))
557 return false;
558
559 if (p->is_write)
560 kvm_pmu_set_counter_value(vcpu, idx, p->regval);
561 else
562 p->regval = kvm_pmu_get_counter_value(vcpu, idx);
563
564 return true;
565}
566
9feb21ac
SZ		 567static bool access_pmu_evtyper(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
568 const struct sys_reg_desc *r)
569{
570 u64 idx, reg;
571
572 if (!kvm_arm_pmu_v3_ready(vcpu))
573 return trap_raz_wi(vcpu, p, r);
574
575 if (r->CRn == 9 && r->CRm == 13 && r->Op2 == 1) {
576 /* PMXEVTYPER_EL0 */
577 idx = vcpu_sys_reg(vcpu, PMSELR_EL0) & ARMV8_PMU_COUNTER_MASK;
578 reg = PMEVTYPER0_EL0 + idx;
579 } else if (r->CRn == 14 && (r->CRm & 12) == 12) {
580 idx = ((r->CRm & 3) << 3) | (r->Op2 & 7);
581 if (idx == ARMV8_PMU_CYCLE_IDX)
582 reg = PMCCFILTR_EL0;
583 else
584 /* PMEVTYPERn_EL0 */
585 reg = PMEVTYPER0_EL0 + idx;
586 } else {
587 BUG();
588 }
589
590 if (!pmu_counter_idx_valid(vcpu, idx))
591 return false;
592
593 if (p->is_write) {
594 kvm_pmu_set_counter_event_type(vcpu, p->regval, idx);
595 vcpu_sys_reg(vcpu, reg) = p->regval & ARMV8_PMU_EVTYPE_MASK;
596 } else {
597 p->regval = vcpu_sys_reg(vcpu, reg) & ARMV8_PMU_EVTYPE_MASK;
598 }
599
600 return true;
601}
602
96b0eebc
SZ		 603static bool access_pmcnten(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
604 const struct sys_reg_desc *r)
605{
606 u64 val, mask;
607
608 if (!kvm_arm_pmu_v3_ready(vcpu))
609 return trap_raz_wi(vcpu, p, r);
610
611 mask = kvm_pmu_valid_counter_mask(vcpu);
612 if (p->is_write) {
613 val = p->regval & mask;
614 if (r->Op2 & 0x1) {
615 /* accessing PMCNTENSET_EL0 */
616 vcpu_sys_reg(vcpu, PMCNTENSET_EL0) |= val;
617 kvm_pmu_enable_counter(vcpu, val);
618 } else {
619 /* accessing PMCNTENCLR_EL0 */
620 vcpu_sys_reg(vcpu, PMCNTENSET_EL0) &= ~val;
621 kvm_pmu_disable_counter(vcpu, val);
622 }
623 } else {
624 p->regval = vcpu_sys_reg(vcpu, PMCNTENSET_EL0) & mask;
625 }
626
627 return true;
628}
629
9db52c78
SZ		 630static bool access_pminten(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
631 const struct sys_reg_desc *r)
632{
633 u64 mask = kvm_pmu_valid_counter_mask(vcpu);
634
635 if (!kvm_arm_pmu_v3_ready(vcpu))
636 return trap_raz_wi(vcpu, p, r);
637
638 if (p->is_write) {
639 u64 val = p->regval & mask;
640
641 if (r->Op2 & 0x1)
642 /* accessing PMINTENSET_EL1 */
643 vcpu_sys_reg(vcpu, PMINTENSET_EL1) |= val;
644 else
645 /* accessing PMINTENCLR_EL1 */
646 vcpu_sys_reg(vcpu, PMINTENSET_EL1) &= ~val;
647 } else {
648 p->regval = vcpu_sys_reg(vcpu, PMINTENSET_EL1) & mask;
649 }
650
651 return true;
652}
653
76d883c4
SZ		 654static bool access_pmovs(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
655 const struct sys_reg_desc *r)
656{
657 u64 mask = kvm_pmu_valid_counter_mask(vcpu);
658
659 if (!kvm_arm_pmu_v3_ready(vcpu))
660 return trap_raz_wi(vcpu, p, r);
661
662 if (p->is_write) {
663 if (r->CRm & 0x2)
664 /* accessing PMOVSSET_EL0 */
665 kvm_pmu_overflow_set(vcpu, p->regval & mask);
666 else
667 /* accessing PMOVSCLR_EL0 */
668 vcpu_sys_reg(vcpu, PMOVSSET_EL0) &= ~(p->regval & mask);
669 } else {
670 p->regval = vcpu_sys_reg(vcpu, PMOVSSET_EL0) & mask;
671 }
672
673 return true;
674}
675
7a0adc70
SZ		 676static bool access_pmswinc(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
677 const struct sys_reg_desc *r)
678{
679 u64 mask;
680
681 if (!kvm_arm_pmu_v3_ready(vcpu))
682 return trap_raz_wi(vcpu, p, r);
683
684 if (p->is_write) {
685 mask = kvm_pmu_valid_counter_mask(vcpu);
686 kvm_pmu_software_increment(vcpu, p->regval & mask);
687 return true;
688 }
689
690 return false;
691}
692
0c557ed4
MZ		 693/* Silly macro to expand the DBG{BCR,BVR,WVR,WCR}n_EL1 registers in one go */
694#define DBG_BCR_BVR_WCR_WVR_EL1(n) \
695 /* DBGBVRn_EL1 */ \
696 { Op0(0b10), Op1(0b000), CRn(0b0000), CRm((n)), Op2(0b100), \
84e690bf 697 trap_bvr, reset_bvr, n, 0, get_bvr, set_bvr }, \
0c557ed4
MZ		 698 /* DBGBCRn_EL1 */ \
699 { Op0(0b10), Op1(0b000), CRn(0b0000), CRm((n)), Op2(0b101), \
84e690bf 700 trap_bcr, reset_bcr, n, 0, get_bcr, set_bcr }, \
0c557ed4
MZ		 701 /* DBGWVRn_EL1 */ \
702 { Op0(0b10), Op1(0b000), CRn(0b0000), CRm((n)), Op2(0b110), \
84e690bf 703 trap_wvr, reset_wvr, n, 0, get_wvr, set_wvr }, \
0c557ed4
MZ		 704 /* DBGWCRn_EL1 */ \
705 { Op0(0b10), Op1(0b000), CRn(0b0000), CRm((n)), Op2(0b111), \
84e690bf 706 trap_wcr, reset_wcr, n, 0, get_wcr, set_wcr }
0c557ed4 707
051ff581
SZ		 708/* Macro to expand the PMEVCNTRn_EL0 register */
709#define PMU_PMEVCNTR_EL0(n) \
710 /* PMEVCNTRn_EL0 */ \
711 { Op0(0b11), Op1(0b011), CRn(0b1110), \
712 CRm((0b1000 | (((n) >> 3) & 0x3))), Op2(((n) & 0x7)), \
713 access_pmu_evcntr, reset_unknown, (PMEVCNTR0_EL0 + n), }
714
9feb21ac
SZ		 715/* Macro to expand the PMEVTYPERn_EL0 register */
716#define PMU_PMEVTYPER_EL0(n) \
717 /* PMEVTYPERn_EL0 */ \
718 { Op0(0b11), Op1(0b011), CRn(0b1110), \
719 CRm((0b1100 | (((n) >> 3) & 0x3))), Op2(((n) & 0x7)), \
720 access_pmu_evtyper, reset_unknown, (PMEVTYPER0_EL0 + n), }
721
7c8c5e6a
MZ		 722/*
723 * Architected system registers.
724 * Important: Must be sorted ascending by Op0, Op1, CRn, CRm, Op2
7609c125
MZ		 725 *
726 * We could trap ID_DFR0 and tell the guest we don't support performance
727 * monitoring. Unfortunately the patch to make the kernel check ID_DFR0 was
728 * NAKed, so it will read the PMCR anyway.
729 *
730 * Therefore we tell the guest we have 0 counters. Unfortunately, we
731 * must always support PMCCNTR (the cycle counter): we just RAZ/WI for
732 * all PM registers, which doesn't crash the guest kernel at least.
733 *
0c557ed4
MZ		 734 * Debug handling: We do trap most, if not all debug related system
735 * registers. The implementation is good enough to ensure that a guest
736 * can use these with minimal performance degradation. The drawback is
737 * that we don't implement any of the external debug, none of the
738 * OSlock protocol. This should be revisited if we ever encounter a
739 * more demanding guest...
7c8c5e6a
MZ		 740 */
741static const struct sys_reg_desc sys_reg_descs[] = {
742 /* DC ISW */
743 { Op0(0b01), Op1(0b000), CRn(0b0111), CRm(0b0110), Op2(0b010),
744 access_dcsw },
745 /* DC CSW */
746 { Op0(0b01), Op1(0b000), CRn(0b0111), CRm(0b1010), Op2(0b010),
747 access_dcsw },
748 /* DC CISW */
749 { Op0(0b01), Op1(0b000), CRn(0b0111), CRm(0b1110), Op2(0b010),
750 access_dcsw },
751
0c557ed4
MZ		 752 DBG_BCR_BVR_WCR_WVR_EL1(0),
753 DBG_BCR_BVR_WCR_WVR_EL1(1),
754 /* MDCCINT_EL1 */
755 { Op0(0b10), Op1(0b000), CRn(0b0000), CRm(0b0010), Op2(0b000),
756 trap_debug_regs, reset_val, MDCCINT_EL1, 0 },
757 /* MDSCR_EL1 */
758 { Op0(0b10), Op1(0b000), CRn(0b0000), CRm(0b0010), Op2(0b010),
759 trap_debug_regs, reset_val, MDSCR_EL1, 0 },
760 DBG_BCR_BVR_WCR_WVR_EL1(2),
761 DBG_BCR_BVR_WCR_WVR_EL1(3),
762 DBG_BCR_BVR_WCR_WVR_EL1(4),
763 DBG_BCR_BVR_WCR_WVR_EL1(5),
764 DBG_BCR_BVR_WCR_WVR_EL1(6),
765 DBG_BCR_BVR_WCR_WVR_EL1(7),
766 DBG_BCR_BVR_WCR_WVR_EL1(8),
767 DBG_BCR_BVR_WCR_WVR_EL1(9),
768 DBG_BCR_BVR_WCR_WVR_EL1(10),
769 DBG_BCR_BVR_WCR_WVR_EL1(11),
770 DBG_BCR_BVR_WCR_WVR_EL1(12),
771 DBG_BCR_BVR_WCR_WVR_EL1(13),
772 DBG_BCR_BVR_WCR_WVR_EL1(14),
773 DBG_BCR_BVR_WCR_WVR_EL1(15),
774
775 /* MDRAR_EL1 */
776 { Op0(0b10), Op1(0b000), CRn(0b0001), CRm(0b0000), Op2(0b000),
777 trap_raz_wi },
778 /* OSLAR_EL1 */
779 { Op0(0b10), Op1(0b000), CRn(0b0001), CRm(0b0000), Op2(0b100),
780 trap_raz_wi },
781 /* OSLSR_EL1 */
782 { Op0(0b10), Op1(0b000), CRn(0b0001), CRm(0b0001), Op2(0b100),
783 trap_oslsr_el1 },
784 /* OSDLR_EL1 */
785 { Op0(0b10), Op1(0b000), CRn(0b0001), CRm(0b0011), Op2(0b100),
786 trap_raz_wi },
787 /* DBGPRCR_EL1 */
788 { Op0(0b10), Op1(0b000), CRn(0b0001), CRm(0b0100), Op2(0b100),
789 trap_raz_wi },
790 /* DBGCLAIMSET_EL1 */
791 { Op0(0b10), Op1(0b000), CRn(0b0111), CRm(0b1000), Op2(0b110),
792 trap_raz_wi },
793 /* DBGCLAIMCLR_EL1 */
794 { Op0(0b10), Op1(0b000), CRn(0b0111), CRm(0b1001), Op2(0b110),
795 trap_raz_wi },
796 /* DBGAUTHSTATUS_EL1 */
797 { Op0(0b10), Op1(0b000), CRn(0b0111), CRm(0b1110), Op2(0b110),
798 trap_dbgauthstatus_el1 },
799
0c557ed4
MZ		 800 /* MDCCSR_EL1 */
801 { Op0(0b10), Op1(0b011), CRn(0b0000), CRm(0b0001), Op2(0b000),
802 trap_raz_wi },
803 /* DBGDTR_EL0 */
804 { Op0(0b10), Op1(0b011), CRn(0b0000), CRm(0b0100), Op2(0b000),
805 trap_raz_wi },
806 /* DBGDTR[TR]X_EL0 */
807 { Op0(0b10), Op1(0b011), CRn(0b0000), CRm(0b0101), Op2(0b000),
808 trap_raz_wi },
809
62a89c44
MZ		 810 /* DBGVCR32_EL2 */
811 { Op0(0b10), Op1(0b100), CRn(0b0000), CRm(0b0111), Op2(0b000),
812 NULL, reset_val, DBGVCR32_EL2, 0 },
813
7c8c5e6a
MZ		 814 /* MPIDR_EL1 */
815 { Op0(0b11), Op1(0b000), CRn(0b0000), CRm(0b0000), Op2(0b101),
816 NULL, reset_mpidr, MPIDR_EL1 },
817 /* SCTLR_EL1 */
818 { Op0(0b11), Op1(0b000), CRn(0b0001), CRm(0b0000), Op2(0b000),
3c1e7165 819 access_vm_reg, reset_val, SCTLR_EL1, 0x00C50078 },
7c8c5e6a
MZ		 820 /* CPACR_EL1 */
821 { Op0(0b11), Op1(0b000), CRn(0b0001), CRm(0b0000), Op2(0b010),
822 NULL, reset_val, CPACR_EL1, 0 },
823 /* TTBR0_EL1 */
824 { Op0(0b11), Op1(0b000), CRn(0b0010), CRm(0b0000), Op2(0b000),
4d44923b 825 access_vm_reg, reset_unknown, TTBR0_EL1 },
7c8c5e6a
MZ		 826 /* TTBR1_EL1 */
827 { Op0(0b11), Op1(0b000), CRn(0b0010), CRm(0b0000), Op2(0b001),
4d44923b 828 access_vm_reg, reset_unknown, TTBR1_EL1 },
7c8c5e6a
MZ		 829 /* TCR_EL1 */
830 { Op0(0b11), Op1(0b000), CRn(0b0010), CRm(0b0000), Op2(0b010),
4d44923b 831 access_vm_reg, reset_val, TCR_EL1, 0 },
7c8c5e6a
MZ		 832
833 /* AFSR0_EL1 */
834 { Op0(0b11), Op1(0b000), CRn(0b0101), CRm(0b0001), Op2(0b000),
4d44923b 835 access_vm_reg, reset_unknown, AFSR0_EL1 },
7c8c5e6a
MZ		 836 /* AFSR1_EL1 */
837 { Op0(0b11), Op1(0b000), CRn(0b0101), CRm(0b0001), Op2(0b001),
4d44923b 838 access_vm_reg, reset_unknown, AFSR1_EL1 },
7c8c5e6a
MZ		 839 /* ESR_EL1 */
840 { Op0(0b11), Op1(0b000), CRn(0b0101), CRm(0b0010), Op2(0b000),
4d44923b 841 access_vm_reg, reset_unknown, ESR_EL1 },
7c8c5e6a
MZ		 842 /* FAR_EL1 */
843 { Op0(0b11), Op1(0b000), CRn(0b0110), CRm(0b0000), Op2(0b000),
4d44923b 844 access_vm_reg, reset_unknown, FAR_EL1 },
1bbd8054
MZ		 845 /* PAR_EL1 */
846 { Op0(0b11), Op1(0b000), CRn(0b0111), CRm(0b0100), Op2(0b000),
847 NULL, reset_unknown, PAR_EL1 },
7c8c5e6a
MZ		 848
849 /* PMINTENSET_EL1 */
850 { Op0(0b11), Op1(0b000), CRn(0b1001), CRm(0b1110), Op2(0b001),
9db52c78 851 access_pminten, reset_unknown, PMINTENSET_EL1 },
7c8c5e6a
MZ		 852 /* PMINTENCLR_EL1 */
853 { Op0(0b11), Op1(0b000), CRn(0b1001), CRm(0b1110), Op2(0b010),
9db52c78 854 access_pminten, NULL, PMINTENSET_EL1 },
7c8c5e6a
MZ		 855
856 /* MAIR_EL1 */
857 { Op0(0b11), Op1(0b000), CRn(0b1010), CRm(0b0010), Op2(0b000),
4d44923b 858 access_vm_reg, reset_unknown, MAIR_EL1 },
7c8c5e6a
MZ		 859 /* AMAIR_EL1 */
860 { Op0(0b11), Op1(0b000), CRn(0b1010), CRm(0b0011), Op2(0b000),
4d44923b 861 access_vm_reg, reset_amair_el1, AMAIR_EL1 },
7c8c5e6a
MZ		 862
863 /* VBAR_EL1 */
864 { Op0(0b11), Op1(0b000), CRn(0b1100), CRm(0b0000), Op2(0b000),
865 NULL, reset_val, VBAR_EL1, 0 },
db7dedd0 866
6d52f35a
AP		 867 /* ICC_SGI1R_EL1 */
868 { Op0(0b11), Op1(0b000), CRn(0b1100), CRm(0b1011), Op2(0b101),
869 access_gic_sgi },
db7dedd0
CD		 870 /* ICC_SRE_EL1 */
871 { Op0(0b11), Op1(0b000), CRn(0b1100), CRm(0b1100), Op2(0b101),
872 trap_raz_wi },
873
7c8c5e6a
MZ		 874 /* CONTEXTIDR_EL1 */
875 { Op0(0b11), Op1(0b000), CRn(0b1101), CRm(0b0000), Op2(0b001),
4d44923b 876 access_vm_reg, reset_val, CONTEXTIDR_EL1, 0 },
7c8c5e6a
MZ		 877 /* TPIDR_EL1 */
878 { Op0(0b11), Op1(0b000), CRn(0b1101), CRm(0b0000), Op2(0b100),
879 NULL, reset_unknown, TPIDR_EL1 },
880
881 /* CNTKCTL_EL1 */
882 { Op0(0b11), Op1(0b000), CRn(0b1110), CRm(0b0001), Op2(0b000),
883 NULL, reset_val, CNTKCTL_EL1, 0},
884
885 /* CSSELR_EL1 */
886 { Op0(0b11), Op1(0b010), CRn(0b0000), CRm(0b0000), Op2(0b000),
887 NULL, reset_unknown, CSSELR_EL1 },
888
889 /* PMCR_EL0 */
890 { Op0(0b11), Op1(0b011), CRn(0b1001), CRm(0b1100), Op2(0b000),
ab946834 891 access_pmcr, reset_pmcr, },
7c8c5e6a
MZ		 892 /* PMCNTENSET_EL0 */
893 { Op0(0b11), Op1(0b011), CRn(0b1001), CRm(0b1100), Op2(0b001),
96b0eebc 894 access_pmcnten, reset_unknown, PMCNTENSET_EL0 },
7c8c5e6a
MZ		 895 /* PMCNTENCLR_EL0 */
896 { Op0(0b11), Op1(0b011), CRn(0b1001), CRm(0b1100), Op2(0b010),
96b0eebc 897 access_pmcnten, NULL, PMCNTENSET_EL0 },
7c8c5e6a
MZ		 898 /* PMOVSCLR_EL0 */
899 { Op0(0b11), Op1(0b011), CRn(0b1001), CRm(0b1100), Op2(0b011),
76d883c4 900 access_pmovs, NULL, PMOVSSET_EL0 },
7c8c5e6a
MZ		 901 /* PMSWINC_EL0 */
902 { Op0(0b11), Op1(0b011), CRn(0b1001), CRm(0b1100), Op2(0b100),
7a0adc70 903 access_pmswinc, reset_unknown, PMSWINC_EL0 },
7c8c5e6a
MZ		 904 /* PMSELR_EL0 */
905 { Op0(0b11), Op1(0b011), CRn(0b1001), CRm(0b1100), Op2(0b101),
3965c3ce 906 access_pmselr, reset_unknown, PMSELR_EL0 },
7c8c5e6a
MZ		 907 /* PMCEID0_EL0 */
908 { Op0(0b11), Op1(0b011), CRn(0b1001), CRm(0b1100), Op2(0b110),
a86b5505 909 access_pmceid },
7c8c5e6a
MZ		 910 /* PMCEID1_EL0 */
911 { Op0(0b11), Op1(0b011), CRn(0b1001), CRm(0b1100), Op2(0b111),
a86b5505 912 access_pmceid },
7c8c5e6a
MZ		 913 /* PMCCNTR_EL0 */
914 { Op0(0b11), Op1(0b011), CRn(0b1001), CRm(0b1101), Op2(0b000),
051ff581 915 access_pmu_evcntr, reset_unknown, PMCCNTR_EL0 },
7c8c5e6a
MZ		 916 /* PMXEVTYPER_EL0 */
917 { Op0(0b11), Op1(0b011), CRn(0b1001), CRm(0b1101), Op2(0b001),
9feb21ac 918 access_pmu_evtyper },
7c8c5e6a
MZ		 919 /* PMXEVCNTR_EL0 */
920 { Op0(0b11), Op1(0b011), CRn(0b1001), CRm(0b1101), Op2(0b010),
051ff581 921 access_pmu_evcntr },
7c8c5e6a
MZ		 922 /* PMUSERENR_EL0 */
923 { Op0(0b11), Op1(0b011), CRn(0b1001), CRm(0b1110), Op2(0b000),
7609c125 924 trap_raz_wi },
7c8c5e6a
MZ		 925 /* PMOVSSET_EL0 */
926 { Op0(0b11), Op1(0b011), CRn(0b1001), CRm(0b1110), Op2(0b011),
76d883c4 927 access_pmovs, reset_unknown, PMOVSSET_EL0 },
7c8c5e6a
MZ		 928
929 /* TPIDR_EL0 */
930 { Op0(0b11), Op1(0b011), CRn(0b1101), CRm(0b0000), Op2(0b010),
931 NULL, reset_unknown, TPIDR_EL0 },
932 /* TPIDRRO_EL0 */
933 { Op0(0b11), Op1(0b011), CRn(0b1101), CRm(0b0000), Op2(0b011),
934 NULL, reset_unknown, TPIDRRO_EL0 },
62a89c44 935
051ff581
SZ		 936 /* PMEVCNTRn_EL0 */
937 PMU_PMEVCNTR_EL0(0),
938 PMU_PMEVCNTR_EL0(1),
939 PMU_PMEVCNTR_EL0(2),
940 PMU_PMEVCNTR_EL0(3),
941 PMU_PMEVCNTR_EL0(4),
942 PMU_PMEVCNTR_EL0(5),
943 PMU_PMEVCNTR_EL0(6),
944 PMU_PMEVCNTR_EL0(7),
945 PMU_PMEVCNTR_EL0(8),
946 PMU_PMEVCNTR_EL0(9),
947 PMU_PMEVCNTR_EL0(10),
948 PMU_PMEVCNTR_EL0(11),
949 PMU_PMEVCNTR_EL0(12),
950 PMU_PMEVCNTR_EL0(13),
951 PMU_PMEVCNTR_EL0(14),
952 PMU_PMEVCNTR_EL0(15),
953 PMU_PMEVCNTR_EL0(16),
954 PMU_PMEVCNTR_EL0(17),
955 PMU_PMEVCNTR_EL0(18),
956 PMU_PMEVCNTR_EL0(19),
957 PMU_PMEVCNTR_EL0(20),
958 PMU_PMEVCNTR_EL0(21),
959 PMU_PMEVCNTR_EL0(22),
960 PMU_PMEVCNTR_EL0(23),
961 PMU_PMEVCNTR_EL0(24),
962 PMU_PMEVCNTR_EL0(25),
963 PMU_PMEVCNTR_EL0(26),
964 PMU_PMEVCNTR_EL0(27),
965 PMU_PMEVCNTR_EL0(28),
966 PMU_PMEVCNTR_EL0(29),
967 PMU_PMEVCNTR_EL0(30),
9feb21ac
SZ		 968 /* PMEVTYPERn_EL0 */
969 PMU_PMEVTYPER_EL0(0),
970 PMU_PMEVTYPER_EL0(1),
971 PMU_PMEVTYPER_EL0(2),
972 PMU_PMEVTYPER_EL0(3),
973 PMU_PMEVTYPER_EL0(4),
974 PMU_PMEVTYPER_EL0(5),
975 PMU_PMEVTYPER_EL0(6),
976 PMU_PMEVTYPER_EL0(7),
977 PMU_PMEVTYPER_EL0(8),
978 PMU_PMEVTYPER_EL0(9),
979 PMU_PMEVTYPER_EL0(10),
980 PMU_PMEVTYPER_EL0(11),
981 PMU_PMEVTYPER_EL0(12),
982 PMU_PMEVTYPER_EL0(13),
983 PMU_PMEVTYPER_EL0(14),
984 PMU_PMEVTYPER_EL0(15),
985 PMU_PMEVTYPER_EL0(16),
986 PMU_PMEVTYPER_EL0(17),
987 PMU_PMEVTYPER_EL0(18),
988 PMU_PMEVTYPER_EL0(19),
989 PMU_PMEVTYPER_EL0(20),
990 PMU_PMEVTYPER_EL0(21),
991 PMU_PMEVTYPER_EL0(22),
992 PMU_PMEVTYPER_EL0(23),
993 PMU_PMEVTYPER_EL0(24),
994 PMU_PMEVTYPER_EL0(25),
995 PMU_PMEVTYPER_EL0(26),
996 PMU_PMEVTYPER_EL0(27),
997 PMU_PMEVTYPER_EL0(28),
998 PMU_PMEVTYPER_EL0(29),
999 PMU_PMEVTYPER_EL0(30),
1000 /* PMCCFILTR_EL0
1001 * This register resets as unknown in 64bit mode while it resets as zero
1002 * in 32bit mode. Here we choose to reset it as zero for consistency.
1003 */
1004 { Op0(0b11), Op1(0b011), CRn(0b1110), CRm(0b1111), Op2(0b111),
1005 access_pmu_evtyper, reset_val, PMCCFILTR_EL0, 0 },
051ff581 1006
62a89c44
MZ		 1007 /* DACR32_EL2 */
1008 { Op0(0b11), Op1(0b100), CRn(0b0011), CRm(0b0000), Op2(0b000),
1009 NULL, reset_unknown, DACR32_EL2 },
1010 /* IFSR32_EL2 */
1011 { Op0(0b11), Op1(0b100), CRn(0b0101), CRm(0b0000), Op2(0b001),
1012 NULL, reset_unknown, IFSR32_EL2 },
1013 /* FPEXC32_EL2 */
1014 { Op0(0b11), Op1(0b100), CRn(0b0101), CRm(0b0011), Op2(0b000),
1015 NULL, reset_val, FPEXC32_EL2, 0x70 },
1016};
1017
bdfb4b38 1018static bool trap_dbgidr(struct kvm_vcpu *vcpu,
3fec037d 1019 struct sys_reg_params *p,
bdfb4b38
MZ		 1020 const struct sys_reg_desc *r)
1021{
1022 if (p->is_write) {
1023 return ignore_write(vcpu, p);
1024 } else {
4db8e5ea
SP		 1025 u64 dfr = read_system_reg(SYS_ID_AA64DFR0_EL1);
1026 u64 pfr = read_system_reg(SYS_ID_AA64PFR0_EL1);
1027 u32 el3 = !!cpuid_feature_extract_field(pfr, ID_AA64PFR0_EL3_SHIFT);
bdfb4b38 1028
2ec5be3d
PF		 1029 p->regval = ((((dfr >> ID_AA64DFR0_WRPS_SHIFT) & 0xf) << 28) |
1030 (((dfr >> ID_AA64DFR0_BRPS_SHIFT) & 0xf) << 24) |
1031 (((dfr >> ID_AA64DFR0_CTX_CMPS_SHIFT) & 0xf) << 20)
1032 | (6 << 16) | (el3 << 14) | (el3 << 12));
bdfb4b38
MZ		 1033 return true;
1034 }
1035}
1036
1037static bool trap_debug32(struct kvm_vcpu *vcpu,
3fec037d 1038 struct sys_reg_params *p,
bdfb4b38
MZ		 1039 const struct sys_reg_desc *r)
1040{
1041 if (p->is_write) {
2ec5be3d 1042 vcpu_cp14(vcpu, r->reg) = p->regval;
bdfb4b38
MZ		 1043 vcpu->arch.debug_flags |= KVM_ARM64_DEBUG_DIRTY;
1044 } else {
2ec5be3d 1045 p->regval = vcpu_cp14(vcpu, r->reg);
bdfb4b38
MZ		 1046 }
1047
1048 return true;
1049}
1050
84e690bf
AB		 1051/* AArch32 debug register mappings
1052 *
1053 * AArch32 DBGBVRn is mapped to DBGBVRn_EL1[31:0]
1054 * AArch32 DBGBXVRn is mapped to DBGBVRn_EL1[63:32]
1055 *
1056 * All control registers and watchpoint value registers are mapped to
1057 * the lower 32 bits of their AArch64 equivalents. We share the trap
1058 * handlers with the above AArch64 code which checks what mode the
1059 * system is in.
1060 */
1061
281243cb
MZ		 1062static bool trap_xvr(struct kvm_vcpu *vcpu,
1063 struct sys_reg_params *p,
1064 const struct sys_reg_desc *rd)
84e690bf
AB		 1065{
1066 u64 *dbg_reg = &vcpu->arch.vcpu_debug_state.dbg_bvr[rd->reg];
1067
1068 if (p->is_write) {
1069 u64 val = *dbg_reg;
1070
1071 val &= 0xffffffffUL;
2ec5be3d 1072 val |= p->regval << 32;
84e690bf
AB		 1073 *dbg_reg = val;
1074
1075 vcpu->arch.debug_flags |= KVM_ARM64_DEBUG_DIRTY;
1076 } else {
2ec5be3d 1077 p->regval = *dbg_reg >> 32;
84e690bf
AB		 1078 }
1079
eef8c85a
AB		 1080 trace_trap_reg(__func__, rd->reg, p->is_write, *dbg_reg);
1081
84e690bf
AB		 1082 return true;
1083}
1084
1085#define DBG_BCR_BVR_WCR_WVR(n) \
1086 /* DBGBVRn */ \
1087 { Op1( 0), CRn( 0), CRm((n)), Op2( 4), trap_bvr, NULL, n }, \
1088 /* DBGBCRn */ \
1089 { Op1( 0), CRn( 0), CRm((n)), Op2( 5), trap_bcr, NULL, n }, \
1090 /* DBGWVRn */ \
1091 { Op1( 0), CRn( 0), CRm((n)), Op2( 6), trap_wvr, NULL, n }, \
1092 /* DBGWCRn */ \
1093 { Op1( 0), CRn( 0), CRm((n)), Op2( 7), trap_wcr, NULL, n }
1094
1095#define DBGBXVR(n) \
1096 { Op1( 0), CRn( 1), CRm((n)), Op2( 1), trap_xvr, NULL, n }
bdfb4b38
MZ		 1097
1098/*
1099 * Trapped cp14 registers. We generally ignore most of the external
1100 * debug, on the principle that they don't really make sense to a
84e690bf 1101 * guest. Revisit this one day, would this principle change.
bdfb4b38 1102 */
72564016 1103static const struct sys_reg_desc cp14_regs[] = {
bdfb4b38
MZ		 1104 /* DBGIDR */
1105 { Op1( 0), CRn( 0), CRm( 0), Op2( 0), trap_dbgidr },
1106 /* DBGDTRRXext */
1107 { Op1( 0), CRn( 0), CRm( 0), Op2( 2), trap_raz_wi },
1108
1109 DBG_BCR_BVR_WCR_WVR(0),
1110 /* DBGDSCRint */
1111 { Op1( 0), CRn( 0), CRm( 1), Op2( 0), trap_raz_wi },
1112 DBG_BCR_BVR_WCR_WVR(1),
1113 /* DBGDCCINT */
1114 { Op1( 0), CRn( 0), CRm( 2), Op2( 0), trap_debug32 },
1115 /* DBGDSCRext */
1116 { Op1( 0), CRn( 0), CRm( 2), Op2( 2), trap_debug32 },
1117 DBG_BCR_BVR_WCR_WVR(2),
1118 /* DBGDTR[RT]Xint */
1119 { Op1( 0), CRn( 0), CRm( 3), Op2( 0), trap_raz_wi },
1120 /* DBGDTR[RT]Xext */
1121 { Op1( 0), CRn( 0), CRm( 3), Op2( 2), trap_raz_wi },
1122 DBG_BCR_BVR_WCR_WVR(3),
1123 DBG_BCR_BVR_WCR_WVR(4),
1124 DBG_BCR_BVR_WCR_WVR(5),
1125 /* DBGWFAR */
1126 { Op1( 0), CRn( 0), CRm( 6), Op2( 0), trap_raz_wi },
1127 /* DBGOSECCR */
1128 { Op1( 0), CRn( 0), CRm( 6), Op2( 2), trap_raz_wi },
1129 DBG_BCR_BVR_WCR_WVR(6),
1130 /* DBGVCR */
1131 { Op1( 0), CRn( 0), CRm( 7), Op2( 0), trap_debug32 },
1132 DBG_BCR_BVR_WCR_WVR(7),
1133 DBG_BCR_BVR_WCR_WVR(8),
1134 DBG_BCR_BVR_WCR_WVR(9),
1135 DBG_BCR_BVR_WCR_WVR(10),
1136 DBG_BCR_BVR_WCR_WVR(11),
1137 DBG_BCR_BVR_WCR_WVR(12),
1138 DBG_BCR_BVR_WCR_WVR(13),
1139 DBG_BCR_BVR_WCR_WVR(14),
1140 DBG_BCR_BVR_WCR_WVR(15),
1141
1142 /* DBGDRAR (32bit) */
1143 { Op1( 0), CRn( 1), CRm( 0), Op2( 0), trap_raz_wi },
1144
1145 DBGBXVR(0),
1146 /* DBGOSLAR */
1147 { Op1( 0), CRn( 1), CRm( 0), Op2( 4), trap_raz_wi },
1148 DBGBXVR(1),
1149 /* DBGOSLSR */
1150 { Op1( 0), CRn( 1), CRm( 1), Op2( 4), trap_oslsr_el1 },
1151 DBGBXVR(2),
1152 DBGBXVR(3),
1153 /* DBGOSDLR */
1154 { Op1( 0), CRn( 1), CRm( 3), Op2( 4), trap_raz_wi },
1155 DBGBXVR(4),
1156 /* DBGPRCR */
1157 { Op1( 0), CRn( 1), CRm( 4), Op2( 4), trap_raz_wi },
1158 DBGBXVR(5),
1159 DBGBXVR(6),
1160 DBGBXVR(7),
1161 DBGBXVR(8),
1162 DBGBXVR(9),
1163 DBGBXVR(10),
1164 DBGBXVR(11),
1165 DBGBXVR(12),
1166 DBGBXVR(13),
1167 DBGBXVR(14),
1168 DBGBXVR(15),
1169
1170 /* DBGDSAR (32bit) */
1171 { Op1( 0), CRn( 2), CRm( 0), Op2( 0), trap_raz_wi },
1172
1173 /* DBGDEVID2 */
1174 { Op1( 0), CRn( 7), CRm( 0), Op2( 7), trap_raz_wi },
1175 /* DBGDEVID1 */
1176 { Op1( 0), CRn( 7), CRm( 1), Op2( 7), trap_raz_wi },
1177 /* DBGDEVID */
1178 { Op1( 0), CRn( 7), CRm( 2), Op2( 7), trap_raz_wi },
1179 /* DBGCLAIMSET */
1180 { Op1( 0), CRn( 7), CRm( 8), Op2( 6), trap_raz_wi },
1181 /* DBGCLAIMCLR */
1182 { Op1( 0), CRn( 7), CRm( 9), Op2( 6), trap_raz_wi },
1183 /* DBGAUTHSTATUS */
1184 { Op1( 0), CRn( 7), CRm(14), Op2( 6), trap_dbgauthstatus_el1 },
72564016
MZ		 1185};
1186
a9866ba0
MZ		 1187/* Trapped cp14 64bit registers */
1188static const struct sys_reg_desc cp14_64_regs[] = {
bdfb4b38
MZ		 1189 /* DBGDRAR (64bit) */
1190 { Op1( 0), CRm( 1), .access = trap_raz_wi },
1191
1192 /* DBGDSAR (64bit) */
1193 { Op1( 0), CRm( 2), .access = trap_raz_wi },
a9866ba0
MZ		 1194};
1195
051ff581
SZ		 1196/* Macro to expand the PMEVCNTRn register */
1197#define PMU_PMEVCNTR(n) \
1198 /* PMEVCNTRn */ \
1199 { Op1(0), CRn(0b1110), \
1200 CRm((0b1000 | (((n) >> 3) & 0x3))), Op2(((n) & 0x7)), \
1201 access_pmu_evcntr }
1202
9feb21ac
SZ		 1203/* Macro to expand the PMEVTYPERn register */
1204#define PMU_PMEVTYPER(n) \
1205 /* PMEVTYPERn */ \
1206 { Op1(0), CRn(0b1110), \
1207 CRm((0b1100 | (((n) >> 3) & 0x3))), Op2(((n) & 0x7)), \
1208 access_pmu_evtyper }
1209
4d44923b
MZ		 1210/*
1211 * Trapped cp15 registers. TTBR0/TTBR1 get a double encoding,
1212 * depending on the way they are accessed (as a 32bit or a 64bit
1213 * register).
1214 */
62a89c44 1215static const struct sys_reg_desc cp15_regs[] = {
6d52f35a
AP		 1216 { Op1( 0), CRn( 0), CRm(12), Op2( 0), access_gic_sgi },
1217
3c1e7165 1218 { Op1( 0), CRn( 1), CRm( 0), Op2( 0), access_vm_reg, NULL, c1_SCTLR },
4d44923b
MZ		 1219 { Op1( 0), CRn( 2), CRm( 0), Op2( 0), access_vm_reg, NULL, c2_TTBR0 },
1220 { Op1( 0), CRn( 2), CRm( 0), Op2( 1), access_vm_reg, NULL, c2_TTBR1 },
1221 { Op1( 0), CRn( 2), CRm( 0), Op2( 2), access_vm_reg, NULL, c2_TTBCR },
1222 { Op1( 0), CRn( 3), CRm( 0), Op2( 0), access_vm_reg, NULL, c3_DACR },
1223 { Op1( 0), CRn( 5), CRm( 0), Op2( 0), access_vm_reg, NULL, c5_DFSR },
1224 { Op1( 0), CRn( 5), CRm( 0), Op2( 1), access_vm_reg, NULL, c5_IFSR },
1225 { Op1( 0), CRn( 5), CRm( 1), Op2( 0), access_vm_reg, NULL, c5_ADFSR },
1226 { Op1( 0), CRn( 5), CRm( 1), Op2( 1), access_vm_reg, NULL, c5_AIFSR },
1227 { Op1( 0), CRn( 6), CRm( 0), Op2( 0), access_vm_reg, NULL, c6_DFAR },
1228 { Op1( 0), CRn( 6), CRm( 0), Op2( 2), access_vm_reg, NULL, c6_IFAR },
1229
62a89c44
MZ		 1230 /*
1231 * DC{C,I,CI}SW operations:
1232 */
1233 { Op1( 0), CRn( 7), CRm( 6), Op2( 2), access_dcsw },
1234 { Op1( 0), CRn( 7), CRm(10), Op2( 2), access_dcsw },
1235 { Op1( 0), CRn( 7), CRm(14), Op2( 2), access_dcsw },
4d44923b 1236
7609c125 1237 /* PMU */
ab946834 1238 { Op1( 0), CRn( 9), CRm(12), Op2( 0), access_pmcr },
96b0eebc
SZ		 1239 { Op1( 0), CRn( 9), CRm(12), Op2( 1), access_pmcnten },
1240 { Op1( 0), CRn( 9), CRm(12), Op2( 2), access_pmcnten },
76d883c4 1241 { Op1( 0), CRn( 9), CRm(12), Op2( 3), access_pmovs },
7a0adc70 1242 { Op1( 0), CRn( 9), CRm(12), Op2( 4), access_pmswinc },
3965c3ce 1243 { Op1( 0), CRn( 9), CRm(12), Op2( 5), access_pmselr },
a86b5505
SZ		 1244 { Op1( 0), CRn( 9), CRm(12), Op2( 6), access_pmceid },
1245 { Op1( 0), CRn( 9), CRm(12), Op2( 7), access_pmceid },
051ff581 1246 { Op1( 0), CRn( 9), CRm(13), Op2( 0), access_pmu_evcntr },
9feb21ac 1247 { Op1( 0), CRn( 9), CRm(13), Op2( 1), access_pmu_evtyper },
051ff581 1248 { Op1( 0), CRn( 9), CRm(13), Op2( 2), access_pmu_evcntr },
7609c125 1249 { Op1( 0), CRn( 9), CRm(14), Op2( 0), trap_raz_wi },
9db52c78
SZ		 1250 { Op1( 0), CRn( 9), CRm(14), Op2( 1), access_pminten },
1251 { Op1( 0), CRn( 9), CRm(14), Op2( 2), access_pminten },
76d883c4 1252 { Op1( 0), CRn( 9), CRm(14), Op2( 3), access_pmovs },
4d44923b
MZ		 1253
1254 { Op1( 0), CRn(10), CRm( 2), Op2( 0), access_vm_reg, NULL, c10_PRRR },
1255 { Op1( 0), CRn(10), CRm( 2), Op2( 1), access_vm_reg, NULL, c10_NMRR },
1256 { Op1( 0), CRn(10), CRm( 3), Op2( 0), access_vm_reg, NULL, c10_AMAIR0 },
1257 { Op1( 0), CRn(10), CRm( 3), Op2( 1), access_vm_reg, NULL, c10_AMAIR1 },
db7dedd0
CD		 1258
1259 /* ICC_SRE */
1260 { Op1( 0), CRn(12), CRm(12), Op2( 5), trap_raz_wi },
1261
4d44923b 1262 { Op1( 0), CRn(13), CRm( 0), Op2( 1), access_vm_reg, NULL, c13_CID },
051ff581
SZ		 1263
1264 /* PMEVCNTRn */
1265 PMU_PMEVCNTR(0),
1266 PMU_PMEVCNTR(1),
1267 PMU_PMEVCNTR(2),
1268 PMU_PMEVCNTR(3),
1269 PMU_PMEVCNTR(4),
1270 PMU_PMEVCNTR(5),
1271 PMU_PMEVCNTR(6),
1272 PMU_PMEVCNTR(7),
1273 PMU_PMEVCNTR(8),
1274 PMU_PMEVCNTR(9),
1275 PMU_PMEVCNTR(10),
1276 PMU_PMEVCNTR(11),
1277 PMU_PMEVCNTR(12),
1278 PMU_PMEVCNTR(13),
1279 PMU_PMEVCNTR(14),
1280 PMU_PMEVCNTR(15),
1281 PMU_PMEVCNTR(16),
1282 PMU_PMEVCNTR(17),
1283 PMU_PMEVCNTR(18),
1284 PMU_PMEVCNTR(19),
1285 PMU_PMEVCNTR(20),
1286 PMU_PMEVCNTR(21),
1287 PMU_PMEVCNTR(22),
1288 PMU_PMEVCNTR(23),
1289 PMU_PMEVCNTR(24),
1290 PMU_PMEVCNTR(25),
1291 PMU_PMEVCNTR(26),
1292 PMU_PMEVCNTR(27),
1293 PMU_PMEVCNTR(28),
1294 PMU_PMEVCNTR(29),
1295 PMU_PMEVCNTR(30),
9feb21ac
SZ		 1296 /* PMEVTYPERn */
1297 PMU_PMEVTYPER(0),
1298 PMU_PMEVTYPER(1),
1299 PMU_PMEVTYPER(2),
1300 PMU_PMEVTYPER(3),
1301 PMU_PMEVTYPER(4),
1302 PMU_PMEVTYPER(5),
1303 PMU_PMEVTYPER(6),
1304 PMU_PMEVTYPER(7),
1305 PMU_PMEVTYPER(8),
1306 PMU_PMEVTYPER(9),
1307 PMU_PMEVTYPER(10),
1308 PMU_PMEVTYPER(11),
1309 PMU_PMEVTYPER(12),
1310 PMU_PMEVTYPER(13),
1311 PMU_PMEVTYPER(14),
1312 PMU_PMEVTYPER(15),
1313 PMU_PMEVTYPER(16),
1314 PMU_PMEVTYPER(17),
1315 PMU_PMEVTYPER(18),
1316 PMU_PMEVTYPER(19),
1317 PMU_PMEVTYPER(20),
1318 PMU_PMEVTYPER(21),
1319 PMU_PMEVTYPER(22),
1320 PMU_PMEVTYPER(23),
1321 PMU_PMEVTYPER(24),
1322 PMU_PMEVTYPER(25),
1323 PMU_PMEVTYPER(26),
1324 PMU_PMEVTYPER(27),
1325 PMU_PMEVTYPER(28),
1326 PMU_PMEVTYPER(29),
1327 PMU_PMEVTYPER(30),
1328 /* PMCCFILTR */
1329 { Op1(0), CRn(14), CRm(15), Op2(7), access_pmu_evtyper },
a9866ba0
MZ		 1330};
1331
1332static const struct sys_reg_desc cp15_64_regs[] = {
1333 { Op1( 0), CRn( 0), CRm( 2), Op2( 0), access_vm_reg, NULL, c2_TTBR0 },
051ff581 1334 { Op1( 0), CRn( 0), CRm( 9), Op2( 0), access_pmu_evcntr },
6d52f35a 1335 { Op1( 0), CRn( 0), CRm(12), Op2( 0), access_gic_sgi },
4d44923b 1336 { Op1( 1), CRn( 0), CRm( 2), Op2( 0), access_vm_reg, NULL, c2_TTBR1 },
7c8c5e6a
MZ		 1337};
1338
1339/* Target specific emulation tables */
1340static struct kvm_sys_reg_target_table *target_tables[KVM_ARM_NUM_TARGETS];
1341
1342void kvm_register_target_sys_reg_table(unsigned int target,
1343 struct kvm_sys_reg_target_table *table)
1344{
1345 target_tables[target] = table;
1346}
1347
1348/* Get specific register table for this target. */
62a89c44
MZ		 1349static const struct sys_reg_desc *get_target_table(unsigned target,
1350 bool mode_is_64,
1351 size_t *num)
7c8c5e6a
MZ		 1352{
1353 struct kvm_sys_reg_target_table *table;
1354
1355 table = target_tables[target];
62a89c44
MZ		 1356 if (mode_is_64) {
1357 *num = table->table64.num;
1358 return table->table64.table;
1359 } else {
1360 *num = table->table32.num;
1361 return table->table32.table;
1362 }
7c8c5e6a
MZ		 1363}
1364
1365static const struct sys_reg_desc *find_reg(const struct sys_reg_params *params,
1366 const struct sys_reg_desc table[],
1367 unsigned int num)
1368{
1369 unsigned int i;
1370
1371 for (i = 0; i < num; i++) {
1372 const struct sys_reg_desc *r = &table[i];
1373
1374 if (params->Op0 != r->Op0)
1375 continue;
1376 if (params->Op1 != r->Op1)
1377 continue;
1378 if (params->CRn != r->CRn)
1379 continue;
1380 if (params->CRm != r->CRm)
1381 continue;
1382 if (params->Op2 != r->Op2)
1383 continue;
1384
1385 return r;
1386 }
1387 return NULL;
1388}
1389
62a89c44
MZ		 1390int kvm_handle_cp14_load_store(struct kvm_vcpu *vcpu, struct kvm_run *run)
1391{
1392 kvm_inject_undefined(vcpu);
1393 return 1;
1394}
1395
72564016
MZ		 1396/*
1397 * emulate_cp -- tries to match a sys_reg access in a handling table, and
1398 * call the corresponding trap handler.
1399 *
1400 * @params: pointer to the descriptor of the access
1401 * @table: array of trap descriptors
1402 * @num: size of the trap descriptor array
1403 *
1404 * Return 0 if the access has been handled, and -1 if not.
1405 */
1406static int emulate_cp(struct kvm_vcpu *vcpu,
3fec037d 1407 struct sys_reg_params *params,
72564016
MZ		 1408 const struct sys_reg_desc *table,
1409 size_t num)
62a89c44 1410{
72564016 1411 const struct sys_reg_desc *r;
62a89c44 1412
72564016
MZ		 1413 if (!table)
1414 return -1; /* Not handled */
62a89c44 1415
62a89c44 1416 r = find_reg(params, table, num);
62a89c44 1417
72564016 1418 if (r) {
62a89c44
MZ		 1419 /*
1420 * Not having an accessor means that we have
1421 * configured a trap that we don't know how to
1422 * handle. This certainly qualifies as a gross bug
1423 * that should be fixed right away.
1424 */
1425 BUG_ON(!r->access);
1426
1427 if (likely(r->access(vcpu, params, r))) {
1428 /* Skip instruction, since it was emulated */
1429 kvm_skip_instr(vcpu, kvm_vcpu_trap_il_is32bit(vcpu));
6327f35a
SZ		 1430 /* Handled */
1431 return 0;
62a89c44 1432 }
72564016
MZ		 1433 }
1434
1435 /* Not handled */
1436 return -1;
1437}
1438
1439static void unhandled_cp_access(struct kvm_vcpu *vcpu,
1440 struct sys_reg_params *params)
1441{
1442 u8 hsr_ec = kvm_vcpu_trap_get_class(vcpu);
1443 int cp;
1444
1445 switch(hsr_ec) {
c6d01a94
MR		 1446 case ESR_ELx_EC_CP15_32:
1447 case ESR_ELx_EC_CP15_64:
72564016
MZ		 1448 cp = 15;
1449 break;
c6d01a94
MR		 1450 case ESR_ELx_EC_CP14_MR:
1451 case ESR_ELx_EC_CP14_64:
72564016
MZ		 1452 cp = 14;
1453 break;
1454 default:
1455 WARN_ON((cp = -1));
62a89c44
MZ		 1456 }
1457
72564016
MZ		 1458 kvm_err("Unsupported guest CP%d access at: %08lx\n",
1459 cp, *vcpu_pc(vcpu));
62a89c44
MZ		 1460 print_sys_reg_instr(params);
1461 kvm_inject_undefined(vcpu);
1462}
1463
1464/**
7769db90 1465 * kvm_handle_cp_64 -- handles a mrrc/mcrr trap on a guest CP14/CP15 access
62a89c44
MZ		 1466 * @vcpu: The VCPU pointer
1467 * @run: The kvm_run struct
1468 */
72564016
MZ		 1469static int kvm_handle_cp_64(struct kvm_vcpu *vcpu,
1470 const struct sys_reg_desc *global,
1471 size_t nr_global,
1472 const struct sys_reg_desc *target_specific,
1473 size_t nr_specific)
62a89c44
MZ		 1474{
1475 struct sys_reg_params params;
1476 u32 hsr = kvm_vcpu_get_hsr(vcpu);
2ec5be3d 1477 int Rt = (hsr >> 5) & 0xf;
62a89c44
MZ		 1478 int Rt2 = (hsr >> 10) & 0xf;
1479
2072d29c
MZ		 1480 params.is_aarch32 = true;
1481 params.is_32bit = false;
62a89c44 1482 params.CRm = (hsr >> 1) & 0xf;
62a89c44
MZ		 1483 params.is_write = ((hsr & 1) == 0);
1484
1485 params.Op0 = 0;
1486 params.Op1 = (hsr >> 16) & 0xf;
1487 params.Op2 = 0;
1488 params.CRn = 0;
1489
1490 /*
2ec5be3d 1491 * Make a 64-bit value out of Rt and Rt2. As we use the same trap
62a89c44
MZ		 1492 * backends between AArch32 and AArch64, we get away with it.
1493 */
1494 if (params.is_write) {
2ec5be3d
PF		 1495 params.regval = vcpu_get_reg(vcpu, Rt) & 0xffffffff;
1496 params.regval |= vcpu_get_reg(vcpu, Rt2) << 32;
62a89c44
MZ		 1497 }
1498
72564016
MZ		 1499 if (!emulate_cp(vcpu, &params, target_specific, nr_specific))
1500 goto out;
1501 if (!emulate_cp(vcpu, &params, global, nr_global))
1502 goto out;
1503
1504 unhandled_cp_access(vcpu, &params);
62a89c44 1505
72564016 1506out:
2ec5be3d 1507 /* Split up the value between registers for the read side */
62a89c44 1508 if (!params.is_write) {
2ec5be3d
PF		 1509 vcpu_set_reg(vcpu, Rt, lower_32_bits(params.regval));
1510 vcpu_set_reg(vcpu, Rt2, upper_32_bits(params.regval));
62a89c44
MZ		 1511 }
1512
1513 return 1;
1514}
1515
1516/**
7769db90 1517 * kvm_handle_cp_32 -- handles a mrc/mcr trap on a guest CP14/CP15 access
62a89c44
MZ		 1518 * @vcpu: The VCPU pointer
1519 * @run: The kvm_run struct
1520 */
72564016
MZ		 1521static int kvm_handle_cp_32(struct kvm_vcpu *vcpu,
1522 const struct sys_reg_desc *global,
1523 size_t nr_global,
1524 const struct sys_reg_desc *target_specific,
1525 size_t nr_specific)
62a89c44
MZ		 1526{
1527 struct sys_reg_params params;
1528 u32 hsr = kvm_vcpu_get_hsr(vcpu);
2ec5be3d 1529 int Rt = (hsr >> 5) & 0xf;
62a89c44 1530
2072d29c
MZ		 1531 params.is_aarch32 = true;
1532 params.is_32bit = true;
62a89c44 1533 params.CRm = (hsr >> 1) & 0xf;
2ec5be3d 1534 params.regval = vcpu_get_reg(vcpu, Rt);
62a89c44
MZ		 1535 params.is_write = ((hsr & 1) == 0);
1536 params.CRn = (hsr >> 10) & 0xf;
1537 params.Op0 = 0;
1538 params.Op1 = (hsr >> 14) & 0x7;
1539 params.Op2 = (hsr >> 17) & 0x7;
1540
2ec5be3d
PF		 1541 if (!emulate_cp(vcpu, &params, target_specific, nr_specific) ||
1542 !emulate_cp(vcpu, &params, global, nr_global)) {
1543 if (!params.is_write)
1544 vcpu_set_reg(vcpu, Rt, params.regval);
72564016 1545 return 1;
2ec5be3d 1546 }
72564016
MZ		 1547
1548 unhandled_cp_access(vcpu, &params);
62a89c44
MZ		 1549 return 1;
1550}
1551
72564016
MZ		 1552int kvm_handle_cp15_64(struct kvm_vcpu *vcpu, struct kvm_run *run)
1553{
1554 const struct sys_reg_desc *target_specific;
1555 size_t num;
1556
1557 target_specific = get_target_table(vcpu->arch.target, false, &num);
1558 return kvm_handle_cp_64(vcpu,
a9866ba0 1559 cp15_64_regs, ARRAY_SIZE(cp15_64_regs),
72564016
MZ		 1560 target_specific, num);
1561}
1562
1563int kvm_handle_cp15_32(struct kvm_vcpu *vcpu, struct kvm_run *run)
1564{
1565 const struct sys_reg_desc *target_specific;
1566 size_t num;
1567
1568 target_specific = get_target_table(vcpu->arch.target, false, &num);
1569 return kvm_handle_cp_32(vcpu,
1570 cp15_regs, ARRAY_SIZE(cp15_regs),
1571 target_specific, num);
1572}
1573
1574int kvm_handle_cp14_64(struct kvm_vcpu *vcpu, struct kvm_run *run)
1575{
1576 return kvm_handle_cp_64(vcpu,
a9866ba0 1577 cp14_64_regs, ARRAY_SIZE(cp14_64_regs),
72564016
MZ		 1578 NULL, 0);
1579}
1580
1581int kvm_handle_cp14_32(struct kvm_vcpu *vcpu, struct kvm_run *run)
1582{
1583 return kvm_handle_cp_32(vcpu,
1584 cp14_regs, ARRAY_SIZE(cp14_regs),
1585 NULL, 0);
1586}
1587
7c8c5e6a 1588static int emulate_sys_reg(struct kvm_vcpu *vcpu,
3fec037d 1589 struct sys_reg_params *params)
7c8c5e6a
MZ		 1590{
1591 size_t num;
1592 const struct sys_reg_desc *table, *r;
1593
62a89c44 1594 table = get_target_table(vcpu->arch.target, true, &num);
7c8c5e6a
MZ		 1595
1596 /* Search target-specific then generic table. */
1597 r = find_reg(params, table, num);
1598 if (!r)
1599 r = find_reg(params, sys_reg_descs, ARRAY_SIZE(sys_reg_descs));
1600
1601 if (likely(r)) {
1602 /*
1603 * Not having an accessor means that we have
1604 * configured a trap that we don't know how to
1605 * handle. This certainly qualifies as a gross bug
1606 * that should be fixed right away.
1607 */
1608 BUG_ON(!r->access);
1609
1610 if (likely(r->access(vcpu, params, r))) {
1611 /* Skip instruction, since it was emulated */
1612 kvm_skip_instr(vcpu, kvm_vcpu_trap_il_is32bit(vcpu));
1613 return 1;
1614 }
1615 /* If access function fails, it should complain. */
1616 } else {
1617 kvm_err("Unsupported guest sys_reg access at: %lx\n",
1618 *vcpu_pc(vcpu));
1619 print_sys_reg_instr(params);
1620 }
1621 kvm_inject_undefined(vcpu);
1622 return 1;
1623}
1624
1625static void reset_sys_reg_descs(struct kvm_vcpu *vcpu,
1626 const struct sys_reg_desc *table, size_t num)
1627{
1628 unsigned long i;
1629
1630 for (i = 0; i < num; i++)
1631 if (table[i].reset)
1632 table[i].reset(vcpu, &table[i]);
1633}
1634
1635/**
1636 * kvm_handle_sys_reg -- handles a mrs/msr trap on a guest sys_reg access
1637 * @vcpu: The VCPU pointer
1638 * @run: The kvm_run struct
1639 */
1640int kvm_handle_sys_reg(struct kvm_vcpu *vcpu, struct kvm_run *run)
1641{
1642 struct sys_reg_params params;
1643 unsigned long esr = kvm_vcpu_get_hsr(vcpu);
2ec5be3d
PF		 1644 int Rt = (esr >> 5) & 0x1f;
1645 int ret;
7c8c5e6a 1646
eef8c85a
AB		 1647 trace_kvm_handle_sys_reg(esr);
1648
2072d29c
MZ		 1649 params.is_aarch32 = false;
1650 params.is_32bit = false;
7c8c5e6a
MZ		 1651 params.Op0 = (esr >> 20) & 3;
1652 params.Op1 = (esr >> 14) & 0x7;
1653 params.CRn = (esr >> 10) & 0xf;
1654 params.CRm = (esr >> 1) & 0xf;
1655 params.Op2 = (esr >> 17) & 0x7;
2ec5be3d 1656 params.regval = vcpu_get_reg(vcpu, Rt);
7c8c5e6a
MZ		 1657 params.is_write = !(esr & 1);
1658
2ec5be3d
PF		 1659 ret = emulate_sys_reg(vcpu, &params);
1660
1661 if (!params.is_write)
1662 vcpu_set_reg(vcpu, Rt, params.regval);
1663 return ret;
7c8c5e6a
MZ		 1664}
1665
1666/******************************************************************************
1667 * Userspace API
1668 *****************************************************************************/
1669
1670static bool index_to_params(u64 id, struct sys_reg_params *params)
1671{
1672 switch (id & KVM_REG_SIZE_MASK) {
1673 case KVM_REG_SIZE_U64:
1674 /* Any unused index bits means it's not valid. */
1675 if (id & ~(KVM_REG_ARCH_MASK | KVM_REG_SIZE_MASK
1676 | KVM_REG_ARM_COPROC_MASK
1677 | KVM_REG_ARM64_SYSREG_OP0_MASK
1678 | KVM_REG_ARM64_SYSREG_OP1_MASK
1679 | KVM_REG_ARM64_SYSREG_CRN_MASK
1680 | KVM_REG_ARM64_SYSREG_CRM_MASK
1681 | KVM_REG_ARM64_SYSREG_OP2_MASK))
1682 return false;
1683 params->Op0 = ((id & KVM_REG_ARM64_SYSREG_OP0_MASK)
1684 >> KVM_REG_ARM64_SYSREG_OP0_SHIFT);
1685 params->Op1 = ((id & KVM_REG_ARM64_SYSREG_OP1_MASK)
1686 >> KVM_REG_ARM64_SYSREG_OP1_SHIFT);
1687 params->CRn = ((id & KVM_REG_ARM64_SYSREG_CRN_MASK)
1688 >> KVM_REG_ARM64_SYSREG_CRN_SHIFT);
1689 params->CRm = ((id & KVM_REG_ARM64_SYSREG_CRM_MASK)
1690 >> KVM_REG_ARM64_SYSREG_CRM_SHIFT);
1691 params->Op2 = ((id & KVM_REG_ARM64_SYSREG_OP2_MASK)
1692 >> KVM_REG_ARM64_SYSREG_OP2_SHIFT);
1693 return true;
1694 default:
1695 return false;
1696 }
1697}
1698
1699/* Decode an index value, and find the sys_reg_desc entry. */
1700static const struct sys_reg_desc *index_to_sys_reg_desc(struct kvm_vcpu *vcpu,
1701 u64 id)
1702{
1703 size_t num;
1704 const struct sys_reg_desc *table, *r;
1705 struct sys_reg_params params;
1706
1707 /* We only do sys_reg for now. */
1708 if ((id & KVM_REG_ARM_COPROC_MASK) != KVM_REG_ARM64_SYSREG)
1709 return NULL;
1710
1711 if (!index_to_params(id, &params))
1712 return NULL;
1713
62a89c44 1714 table = get_target_table(vcpu->arch.target, true, &num);
7c8c5e6a
MZ		 1715 r = find_reg(&params, table, num);
1716 if (!r)
1717 r = find_reg(&params, sys_reg_descs, ARRAY_SIZE(sys_reg_descs));
1718
1719 /* Not saved in the sys_reg array? */
1720 if (r && !r->reg)
1721 r = NULL;
1722
1723 return r;
1724}
1725
1726/*
1727 * These are the invariant sys_reg registers: we let the guest see the
1728 * host versions of these, so they're part of the guest state.
1729 *
1730 * A future CPU may provide a mechanism to present different values to
1731 * the guest, or a future kvm may trap them.
1732 */
1733
1734#define FUNCTION_INVARIANT(reg) \
1735 static void get_##reg(struct kvm_vcpu *v, \
1736 const struct sys_reg_desc *r) \
1737 { \
1738 u64 val; \
1739 \
1740 asm volatile("mrs %0, " __stringify(reg) "\n" \
1741 : "=r" (val)); \
1742 ((struct sys_reg_desc *)r)->val = val; \
1743 }
1744
1745FUNCTION_INVARIANT(midr_el1)
1746FUNCTION_INVARIANT(ctr_el0)
1747FUNCTION_INVARIANT(revidr_el1)
1748FUNCTION_INVARIANT(id_pfr0_el1)
1749FUNCTION_INVARIANT(id_pfr1_el1)
1750FUNCTION_INVARIANT(id_dfr0_el1)
1751FUNCTION_INVARIANT(id_afr0_el1)
1752FUNCTION_INVARIANT(id_mmfr0_el1)
1753FUNCTION_INVARIANT(id_mmfr1_el1)
1754FUNCTION_INVARIANT(id_mmfr2_el1)
1755FUNCTION_INVARIANT(id_mmfr3_el1)
1756FUNCTION_INVARIANT(id_isar0_el1)
1757FUNCTION_INVARIANT(id_isar1_el1)
1758FUNCTION_INVARIANT(id_isar2_el1)
1759FUNCTION_INVARIANT(id_isar3_el1)
1760FUNCTION_INVARIANT(id_isar4_el1)
1761FUNCTION_INVARIANT(id_isar5_el1)
1762FUNCTION_INVARIANT(clidr_el1)
1763FUNCTION_INVARIANT(aidr_el1)
1764
1765/* ->val is filled in by kvm_sys_reg_table_init() */
1766static struct sys_reg_desc invariant_sys_regs[] = {
1767 { Op0(0b11), Op1(0b000), CRn(0b0000), CRm(0b0000), Op2(0b000),
1768 NULL, get_midr_el1 },
1769 { Op0(0b11), Op1(0b000), CRn(0b0000), CRm(0b0000), Op2(0b110),
1770 NULL, get_revidr_el1 },
1771 { Op0(0b11), Op1(0b000), CRn(0b0000), CRm(0b0001), Op2(0b000),
1772 NULL, get_id_pfr0_el1 },
1773 { Op0(0b11), Op1(0b000), CRn(0b0000), CRm(0b0001), Op2(0b001),
1774 NULL, get_id_pfr1_el1 },
1775 { Op0(0b11), Op1(0b000), CRn(0b0000), CRm(0b0001), Op2(0b010),
1776 NULL, get_id_dfr0_el1 },
1777 { Op0(0b11), Op1(0b000), CRn(0b0000), CRm(0b0001), Op2(0b011),
1778 NULL, get_id_afr0_el1 },
1779 { Op0(0b11), Op1(0b000), CRn(0b0000), CRm(0b0001), Op2(0b100),
1780 NULL, get_id_mmfr0_el1 },
1781 { Op0(0b11), Op1(0b000), CRn(0b0000), CRm(0b0001), Op2(0b101),
1782 NULL, get_id_mmfr1_el1 },
1783 { Op0(0b11), Op1(0b000), CRn(0b0000), CRm(0b0001), Op2(0b110),
1784 NULL, get_id_mmfr2_el1 },
1785 { Op0(0b11), Op1(0b000), CRn(0b0000), CRm(0b0001), Op2(0b111),
1786 NULL, get_id_mmfr3_el1 },
1787 { Op0(0b11), Op1(0b000), CRn(0b0000), CRm(0b0010), Op2(0b000),
1788 NULL, get_id_isar0_el1 },
1789 { Op0(0b11), Op1(0b000), CRn(0b0000), CRm(0b0010), Op2(0b001),
1790 NULL, get_id_isar1_el1 },
1791 { Op0(0b11), Op1(0b000), CRn(0b0000), CRm(0b0010), Op2(0b010),
1792 NULL, get_id_isar2_el1 },
1793 { Op0(0b11), Op1(0b000), CRn(0b0000), CRm(0b0010), Op2(0b011),
1794 NULL, get_id_isar3_el1 },
1795 { Op0(0b11), Op1(0b000), CRn(0b0000), CRm(0b0010), Op2(0b100),
1796 NULL, get_id_isar4_el1 },
1797 { Op0(0b11), Op1(0b000), CRn(0b0000), CRm(0b0010), Op2(0b101),
1798 NULL, get_id_isar5_el1 },
1799 { Op0(0b11), Op1(0b001), CRn(0b0000), CRm(0b0000), Op2(0b001),
1800 NULL, get_clidr_el1 },
1801 { Op0(0b11), Op1(0b001), CRn(0b0000), CRm(0b0000), Op2(0b111),
1802 NULL, get_aidr_el1 },
1803 { Op0(0b11), Op1(0b011), CRn(0b0000), CRm(0b0000), Op2(0b001),
1804 NULL, get_ctr_el0 },
1805};
1806
26c99af1 1807static int reg_from_user(u64 *val, const void __user *uaddr, u64 id)
7c8c5e6a 1808{
7c8c5e6a
MZ		 1809 if (copy_from_user(val, uaddr, KVM_REG_SIZE(id)) != 0)
1810 return -EFAULT;
1811 return 0;
1812}
1813
26c99af1 1814static int reg_to_user(void __user *uaddr, const u64 *val, u64 id)
7c8c5e6a 1815{
7c8c5e6a
MZ		 1816 if (copy_to_user(uaddr, val, KVM_REG_SIZE(id)) != 0)
1817 return -EFAULT;
1818 return 0;
1819}
1820
1821static int get_invariant_sys_reg(u64 id, void __user *uaddr)
1822{
1823 struct sys_reg_params params;
1824 const struct sys_reg_desc *r;
1825
1826 if (!index_to_params(id, &params))
1827 return -ENOENT;
1828
1829 r = find_reg(&params, invariant_sys_regs, ARRAY_SIZE(invariant_sys_regs));
1830 if (!r)
1831 return -ENOENT;
1832
1833 return reg_to_user(uaddr, &r->val, id);
1834}
1835
1836static int set_invariant_sys_reg(u64 id, void __user *uaddr)
1837{
1838 struct sys_reg_params params;
1839 const struct sys_reg_desc *r;
1840 int err;
1841 u64 val = 0; /* Make sure high bits are 0 for 32-bit regs */
1842
1843 if (!index_to_params(id, &params))
1844 return -ENOENT;
1845 r = find_reg(&params, invariant_sys_regs, ARRAY_SIZE(invariant_sys_regs));
1846 if (!r)
1847 return -ENOENT;
1848
1849 err = reg_from_user(&val, uaddr, id);
1850 if (err)
1851 return err;
1852
1853 /* This is what we mean by invariant: you can't change it. */
1854 if (r->val != val)
1855 return -EINVAL;
1856
1857 return 0;
1858}
1859
1860static bool is_valid_cache(u32 val)
1861{
1862 u32 level, ctype;
1863
1864 if (val >= CSSELR_MAX)
18d45766 1865 return false;
7c8c5e6a
MZ		 1866
1867 /* Bottom bit is Instruction or Data bit. Next 3 bits are level. */
1868 level = (val >> 1);
1869 ctype = (cache_levels >> (level * 3)) & 7;
1870
1871 switch (ctype) {
1872 case 0: /* No cache */
1873 return false;
1874 case 1: /* Instruction cache only */
1875 return (val & 1);
1876 case 2: /* Data cache only */
1877 case 4: /* Unified cache */
1878 return !(val & 1);
1879 case 3: /* Separate instruction and data caches */
1880 return true;
1881 default: /* Reserved: we can't know instruction or data. */
1882 return false;
1883 }
1884}
1885
1886static int demux_c15_get(u64 id, void __user *uaddr)
1887{
1888 u32 val;
1889 u32 __user *uval = uaddr;
1890
1891 /* Fail if we have unknown bits set. */
1892 if (id & ~(KVM_REG_ARCH_MASK|KVM_REG_SIZE_MASK|KVM_REG_ARM_COPROC_MASK
1893 | ((1 << KVM_REG_ARM_COPROC_SHIFT)-1)))
1894 return -ENOENT;
1895
1896 switch (id & KVM_REG_ARM_DEMUX_ID_MASK) {
1897 case KVM_REG_ARM_DEMUX_ID_CCSIDR:
1898 if (KVM_REG_SIZE(id) != 4)
1899 return -ENOENT;
1900 val = (id & KVM_REG_ARM_DEMUX_VAL_MASK)
1901 >> KVM_REG_ARM_DEMUX_VAL_SHIFT;
1902 if (!is_valid_cache(val))
1903 return -ENOENT;
1904
1905 return put_user(get_ccsidr(val), uval);
1906 default:
1907 return -ENOENT;
1908 }
1909}
1910
1911static int demux_c15_set(u64 id, void __user *uaddr)
1912{
1913 u32 val, newval;
1914 u32 __user *uval = uaddr;
1915
1916 /* Fail if we have unknown bits set. */
1917 if (id & ~(KVM_REG_ARCH_MASK|KVM_REG_SIZE_MASK|KVM_REG_ARM_COPROC_MASK
1918 | ((1 << KVM_REG_ARM_COPROC_SHIFT)-1)))
1919 return -ENOENT;
1920
1921 switch (id & KVM_REG_ARM_DEMUX_ID_MASK) {
1922 case KVM_REG_ARM_DEMUX_ID_CCSIDR:
1923 if (KVM_REG_SIZE(id) != 4)
1924 return -ENOENT;
1925 val = (id & KVM_REG_ARM_DEMUX_VAL_MASK)
1926 >> KVM_REG_ARM_DEMUX_VAL_SHIFT;
1927 if (!is_valid_cache(val))
1928 return -ENOENT;
1929
1930 if (get_user(newval, uval))
1931 return -EFAULT;
1932
1933 /* This is also invariant: you can't change it. */
1934 if (newval != get_ccsidr(val))
1935 return -EINVAL;
1936 return 0;
1937 default:
1938 return -ENOENT;
1939 }
1940}
1941
1942int kvm_arm_sys_reg_get_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg)
1943{
1944 const struct sys_reg_desc *r;
1945 void __user *uaddr = (void __user *)(unsigned long)reg->addr;
1946
1947 if ((reg->id & KVM_REG_ARM_COPROC_MASK) == KVM_REG_ARM_DEMUX)
1948 return demux_c15_get(reg->id, uaddr);
1949
1950 if (KVM_REG_SIZE(reg->id) != sizeof(__u64))
1951 return -ENOENT;
1952
1953 r = index_to_sys_reg_desc(vcpu, reg->id);
1954 if (!r)
1955 return get_invariant_sys_reg(reg->id, uaddr);
1956
84e690bf
AB		 1957 if (r->get_user)
1958 return (r->get_user)(vcpu, r, reg, uaddr);
1959
7c8c5e6a
MZ		 1960 return reg_to_user(uaddr, &vcpu_sys_reg(vcpu, r->reg), reg->id);
1961}
1962
1963int kvm_arm_sys_reg_set_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg)
1964{
1965 const struct sys_reg_desc *r;
1966 void __user *uaddr = (void __user *)(unsigned long)reg->addr;
1967
1968 if ((reg->id & KVM_REG_ARM_COPROC_MASK) == KVM_REG_ARM_DEMUX)
1969 return demux_c15_set(reg->id, uaddr);
1970
1971 if (KVM_REG_SIZE(reg->id) != sizeof(__u64))
1972 return -ENOENT;
1973
1974 r = index_to_sys_reg_desc(vcpu, reg->id);
1975 if (!r)
1976 return set_invariant_sys_reg(reg->id, uaddr);
1977
84e690bf
AB		 1978 if (r->set_user)
1979 return (r->set_user)(vcpu, r, reg, uaddr);
1980
7c8c5e6a
MZ		 1981 return reg_from_user(&vcpu_sys_reg(vcpu, r->reg), uaddr, reg->id);
1982}
1983
1984static unsigned int num_demux_regs(void)
1985{
1986 unsigned int i, count = 0;
1987
1988 for (i = 0; i < CSSELR_MAX; i++)
1989 if (is_valid_cache(i))
1990 count++;
1991
1992 return count;
1993}
1994
1995static int write_demux_regids(u64 __user *uindices)
1996{
efd48cea 1997 u64 val = KVM_REG_ARM64 | KVM_REG_SIZE_U32 | KVM_REG_ARM_DEMUX;
7c8c5e6a
MZ		 1998 unsigned int i;
1999
2000 val |= KVM_REG_ARM_DEMUX_ID_CCSIDR;
2001 for (i = 0; i < CSSELR_MAX; i++) {
2002 if (!is_valid_cache(i))
2003 continue;
2004 if (put_user(val | i, uindices))
2005 return -EFAULT;
2006 uindices++;
2007 }
2008 return 0;
2009}
2010
2011static u64 sys_reg_to_index(const struct sys_reg_desc *reg)
2012{
2013 return (KVM_REG_ARM64 | KVM_REG_SIZE_U64 |
2014 KVM_REG_ARM64_SYSREG |
2015 (reg->Op0 << KVM_REG_ARM64_SYSREG_OP0_SHIFT) |
2016 (reg->Op1 << KVM_REG_ARM64_SYSREG_OP1_SHIFT) |
2017 (reg->CRn << KVM_REG_ARM64_SYSREG_CRN_SHIFT) |
2018 (reg->CRm << KVM_REG_ARM64_SYSREG_CRM_SHIFT) |
2019 (reg->Op2 << KVM_REG_ARM64_SYSREG_OP2_SHIFT));
2020}
2021
2022static bool copy_reg_to_user(const struct sys_reg_desc *reg, u64 __user **uind)
2023{
2024 if (!*uind)
2025 return true;
2026
2027 if (put_user(sys_reg_to_index(reg), *uind))
2028 return false;
2029
2030 (*uind)++;
2031 return true;
2032}
2033
2034/* Assumed ordered tables, see kvm_sys_reg_table_init. */
2035static int walk_sys_regs(struct kvm_vcpu *vcpu, u64 __user *uind)
2036{
2037 const struct sys_reg_desc *i1, *i2, *end1, *end2;
2038 unsigned int total = 0;
2039 size_t num;
2040
2041 /* We check for duplicates here, to allow arch-specific overrides. */
62a89c44 2042 i1 = get_target_table(vcpu->arch.target, true, &num);
7c8c5e6a
MZ		 2043 end1 = i1 + num;
2044 i2 = sys_reg_descs;
2045 end2 = sys_reg_descs + ARRAY_SIZE(sys_reg_descs);
2046
2047 BUG_ON(i1 == end1 || i2 == end2);
2048
2049 /* Walk carefully, as both tables may refer to the same register. */
2050 while (i1 || i2) {
2051 int cmp = cmp_sys_reg(i1, i2);
2052 /* target-specific overrides generic entry. */
2053 if (cmp <= 0) {
2054 /* Ignore registers we trap but don't save. */
2055 if (i1->reg) {
2056 if (!copy_reg_to_user(i1, &uind))
2057 return -EFAULT;
2058 total++;
2059 }
2060 } else {
2061 /* Ignore registers we trap but don't save. */
2062 if (i2->reg) {
2063 if (!copy_reg_to_user(i2, &uind))
2064 return -EFAULT;
2065 total++;
2066 }
2067 }
2068
2069 if (cmp <= 0 && ++i1 == end1)
2070 i1 = NULL;
2071 if (cmp >= 0 && ++i2 == end2)
2072 i2 = NULL;
2073 }
2074 return total;
2075}
2076
2077unsigned long kvm_arm_num_sys_reg_descs(struct kvm_vcpu *vcpu)
2078{
2079 return ARRAY_SIZE(invariant_sys_regs)
2080 + num_demux_regs()
2081 + walk_sys_regs(vcpu, (u64 __user *)NULL);
2082}
2083
2084int kvm_arm_copy_sys_reg_indices(struct kvm_vcpu *vcpu, u64 __user *uindices)
2085{
2086 unsigned int i;
2087 int err;
2088
2089 /* Then give them all the invariant registers' indices. */
2090 for (i = 0; i < ARRAY_SIZE(invariant_sys_regs); i++) {
2091 if (put_user(sys_reg_to_index(&invariant_sys_regs[i]), uindices))
2092 return -EFAULT;
2093 uindices++;
2094 }
2095
2096 err = walk_sys_regs(vcpu, uindices);
2097 if (err < 0)
2098 return err;
2099 uindices += err;
2100
2101 return write_demux_regids(uindices);
2102}
2103
e6a95517
MZ		 2104static int check_sysreg_table(const struct sys_reg_desc *table, unsigned int n)
2105{
2106 unsigned int i;
2107
2108 for (i = 1; i < n; i++) {
2109 if (cmp_sys_reg(&table[i-1], &table[i]) >= 0) {
2110 kvm_err("sys_reg table %p out of order (%d)\n", table, i - 1);
2111 return 1;
2112 }
2113 }
2114
2115 return 0;
2116}
2117
7c8c5e6a
MZ		 2118void kvm_sys_reg_table_init(void)
2119{
2120 unsigned int i;
2121 struct sys_reg_desc clidr;
2122
2123 /* Make sure tables are unique and in order. */
e6a95517
MZ		 2124 BUG_ON(check_sysreg_table(sys_reg_descs, ARRAY_SIZE(sys_reg_descs)));
2125 BUG_ON(check_sysreg_table(cp14_regs, ARRAY_SIZE(cp14_regs)));
2126 BUG_ON(check_sysreg_table(cp14_64_regs, ARRAY_SIZE(cp14_64_regs)));
2127 BUG_ON(check_sysreg_table(cp15_regs, ARRAY_SIZE(cp15_regs)));
2128 BUG_ON(check_sysreg_table(cp15_64_regs, ARRAY_SIZE(cp15_64_regs)));
2129 BUG_ON(check_sysreg_table(invariant_sys_regs, ARRAY_SIZE(invariant_sys_regs)));
7c8c5e6a
MZ		 2130
2131 /* We abuse the reset function to overwrite the table itself. */
2132 for (i = 0; i < ARRAY_SIZE(invariant_sys_regs); i++)
2133 invariant_sys_regs[i].reset(NULL, &invariant_sys_regs[i]);
2134
2135 /*
2136 * CLIDR format is awkward, so clean it up. See ARM B4.1.20:
2137 *
2138 * If software reads the Cache Type fields from Ctype1
2139 * upwards, once it has seen a value of 0b000, no caches
2140 * exist at further-out levels of the hierarchy. So, for
2141 * example, if Ctype3 is the first Cache Type field with a
2142 * value of 0b000, the values of Ctype4 to Ctype7 must be
2143 * ignored.
2144 */
2145 get_clidr_el1(NULL, &clidr); /* Ugly... */
2146 cache_levels = clidr.val;
2147 for (i = 0; i < 7; i++)
2148 if (((cache_levels >> (i*3)) & 7) == 0)
2149 break;
2150 /* Clear all higher bits. */
2151 cache_levels &= (1 << (i*3))-1;
2152}
2153
2154/**
2155 * kvm_reset_sys_regs - sets system registers to reset value
2156 * @vcpu: The VCPU pointer
2157 *
2158 * This function finds the right table above and sets the registers on the
2159 * virtual CPU struct to their architecturally defined reset values.
2160 */
2161void kvm_reset_sys_regs(struct kvm_vcpu *vcpu)
2162{
2163 size_t num;
2164 const struct sys_reg_desc *table;
2165
2166 /* Catch someone adding a register without putting in reset entry. */
2167 memset(&vcpu->arch.ctxt.sys_regs, 0x42, sizeof(vcpu->arch.ctxt.sys_regs));
2168
2169 /* Generic chip reset first (so target could override). */
2170 reset_sys_reg_descs(vcpu, sys_reg_descs, ARRAY_SIZE(sys_reg_descs));
2171
62a89c44 2172 table = get_target_table(vcpu->arch.target, true, &num);
7c8c5e6a
MZ		 2173 reset_sys_reg_descs(vcpu, table, num);
2174
2175 for (num = 1; num < NR_SYS_REGS; num++)
2176 if (vcpu_sys_reg(vcpu, num) == 0x4242424242424242)
2177 panic("Didn't reset vcpu_sys_reg(%zi)", num);
2178}
ubuntu-zesty-kernel mirror