]> git.proxmox.com Git - mirror_ubuntu-artful-kernel.git/blame - arch/arm64/kvm/sys_regs.c
projects / mirror_ubuntu-artful-kernel.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
arm64: KVM: Add access handler for event type register
[mirror_ubuntu-artful-kernel.git] / arch / arm64 / kvm / sys_regs.c
CommitLineData
7c8c5e6a
MZ		 1/*
2 * Copyright (C) 2012,2013 - ARM Ltd
3 * Author: Marc Zyngier <marc.zyngier@arm.com>
4 *
5 * Derived from arch/arm/kvm/coproc.c:
6 * Copyright (C) 2012 - Virtual Open Systems and Columbia University
7 * Authors: Rusty Russell <rusty@rustcorp.com.au>
8 * Christoffer Dall <c.dall@virtualopensystems.com>
9 *
10 * This program is free software; you can redistribute it and/or modify
11 * it under the terms of the GNU General Public License, version 2, as
12 * published by the Free Software Foundation.
13 *
14 * This program is distributed in the hope that it will be useful,
15 * but WITHOUT ANY WARRANTY; without even the implied warranty of
16 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 * GNU General Public License for more details.
18 *
19 * You should have received a copy of the GNU General Public License
20 * along with this program. If not, see <http://www.gnu.org/licenses/>.
21 */
22
7c8c5e6a 23#include <linux/kvm_host.h>
c6d01a94 24#include <linux/mm.h>
7c8c5e6a 25#include <linux/uaccess.h>
c6d01a94 26
7c8c5e6a
MZ		 27#include <asm/cacheflush.h>
28#include <asm/cputype.h>
0c557ed4 29#include <asm/debug-monitors.h>
c6d01a94
MR		 30#include <asm/esr.h>
31#include <asm/kvm_arm.h>
9d8415d6 32#include <asm/kvm_asm.h>
c6d01a94
MR		 33#include <asm/kvm_coproc.h>
34#include <asm/kvm_emulate.h>
35#include <asm/kvm_host.h>
36#include <asm/kvm_mmu.h>
ab946834 37#include <asm/perf_event.h>
c6d01a94 38
7c8c5e6a
MZ		 39#include <trace/events/kvm.h>
40
41#include "sys_regs.h"
42
eef8c85a
AB		 43#include "trace.h"
44
7c8c5e6a
MZ		 45/*
46 * All of this file is extremly similar to the ARM coproc.c, but the
47 * types are different. My gut feeling is that it should be pretty
48 * easy to merge, but that would be an ABI breakage -- again. VFP
49 * would also need to be abstracted.
62a89c44
MZ		 50 *
51 * For AArch32, we only take care of what is being trapped. Anything
52 * that has to do with init and userspace access has to go via the
53 * 64bit interface.
7c8c5e6a
MZ		 54 */
55
56/* 3 bits per cache level, as per CLIDR, but non-existent caches always 0 */
57static u32 cache_levels;
58
59/* CSSELR values; used to index KVM_REG_ARM_DEMUX_ID_CCSIDR */
60#define CSSELR_MAX 12
61
62/* Which cache CCSIDR represents depends on CSSELR value. */
63static u32 get_ccsidr(u32 csselr)
64{
65 u32 ccsidr;
66
67 /* Make sure noone else changes CSSELR during this! */
68 local_irq_disable();
69 /* Put value into CSSELR */
70 asm volatile("msr csselr_el1, %x0" : : "r" (csselr));
71 isb();
72 /* Read result out of CCSIDR */
73 asm volatile("mrs %0, ccsidr_el1" : "=r" (ccsidr));
74 local_irq_enable();
75
76 return ccsidr;
77}
78
3c1e7165
MZ		 79/*
80 * See note at ARMv7 ARM B1.14.4 (TL;DR: S/W ops are not easily virtualized).
81 */
7c8c5e6a 82static bool access_dcsw(struct kvm_vcpu *vcpu,
3fec037d 83 struct sys_reg_params *p,
7c8c5e6a
MZ		 84 const struct sys_reg_desc *r)
85{
7c8c5e6a
MZ		 86 if (!p->is_write)
87 return read_from_write_only(vcpu, p);
88
3c1e7165 89 kvm_set_way_flush(vcpu);
7c8c5e6a
MZ		 90 return true;
91}
92
4d44923b
MZ		 93/*
94 * Generic accessor for VM registers. Only called as long as HCR_TVM
3c1e7165
MZ		 95 * is set. If the guest enables the MMU, we stop trapping the VM
96 * sys_regs and leave it in complete control of the caches.
4d44923b
MZ		 97 */
98static bool access_vm_reg(struct kvm_vcpu *vcpu,
3fec037d 99 struct sys_reg_params *p,
4d44923b
MZ		 100 const struct sys_reg_desc *r)
101{
3c1e7165 102 bool was_enabled = vcpu_has_cache_enabled(vcpu);
4d44923b
MZ		 103
104 BUG_ON(!p->is_write);
105
dedf97e8 106 if (!p->is_aarch32) {
2ec5be3d 107 vcpu_sys_reg(vcpu, r->reg) = p->regval;
dedf97e8
MZ		 108 } else {
109 if (!p->is_32bit)
2ec5be3d
PF		 110 vcpu_cp15_64_high(vcpu, r->reg) = upper_32_bits(p->regval);
111 vcpu_cp15_64_low(vcpu, r->reg) = lower_32_bits(p->regval);
dedf97e8 112 }
f0a3eaff 113
3c1e7165 114 kvm_toggle_cache(vcpu, was_enabled);
4d44923b
MZ		 115 return true;
116}
117
6d52f35a
AP		 118/*
119 * Trap handler for the GICv3 SGI generation system register.
120 * Forward the request to the VGIC emulation.
121 * The cp15_64 code makes sure this automatically works
122 * for both AArch64 and AArch32 accesses.
123 */
124static bool access_gic_sgi(struct kvm_vcpu *vcpu,
3fec037d 125 struct sys_reg_params *p,
6d52f35a
AP		 126 const struct sys_reg_desc *r)
127{
6d52f35a
AP		 128 if (!p->is_write)
129 return read_from_write_only(vcpu, p);
130
2ec5be3d 131 vgic_v3_dispatch_sgi(vcpu, p->regval);
6d52f35a
AP		 132
133 return true;
134}
135
7609c125 136static bool trap_raz_wi(struct kvm_vcpu *vcpu,
3fec037d 137 struct sys_reg_params *p,
7609c125 138 const struct sys_reg_desc *r)
7c8c5e6a
MZ		 139{
140 if (p->is_write)
141 return ignore_write(vcpu, p);
142 else
143 return read_zero(vcpu, p);
144}
145
0c557ed4 146static bool trap_oslsr_el1(struct kvm_vcpu *vcpu,
3fec037d 147 struct sys_reg_params *p,
0c557ed4
MZ		 148 const struct sys_reg_desc *r)
149{
150 if (p->is_write) {
151 return ignore_write(vcpu, p);
152 } else {
2ec5be3d 153 p->regval = (1 << 3);
0c557ed4
MZ		 154 return true;
155 }
156}
157
158static bool trap_dbgauthstatus_el1(struct kvm_vcpu *vcpu,
3fec037d 159 struct sys_reg_params *p,
0c557ed4
MZ		 160 const struct sys_reg_desc *r)
161{
162 if (p->is_write) {
163 return ignore_write(vcpu, p);
164 } else {
165 u32 val;
166 asm volatile("mrs %0, dbgauthstatus_el1" : "=r" (val));
2ec5be3d 167 p->regval = val;
0c557ed4
MZ		 168 return true;
169 }
170}
171
172/*
173 * We want to avoid world-switching all the DBG registers all the
174 * time:
175 *
176 * - If we've touched any debug register, it is likely that we're
177 * going to touch more of them. It then makes sense to disable the
178 * traps and start doing the save/restore dance
179 * - If debug is active (DBG_MDSCR_KDE or DBG_MDSCR_MDE set), it is
180 * then mandatory to save/restore the registers, as the guest
181 * depends on them.
182 *
183 * For this, we use a DIRTY bit, indicating the guest has modified the
184 * debug registers, used as follow:
185 *
186 * On guest entry:
187 * - If the dirty bit is set (because we're coming back from trapping),
188 * disable the traps, save host registers, restore guest registers.
189 * - If debug is actively in use (DBG_MDSCR_KDE or DBG_MDSCR_MDE set),
190 * set the dirty bit, disable the traps, save host registers,
191 * restore guest registers.
192 * - Otherwise, enable the traps
193 *
194 * On guest exit:
195 * - If the dirty bit is set, save guest registers, restore host
196 * registers and clear the dirty bit. This ensure that the host can
197 * now use the debug registers.
198 */
199static bool trap_debug_regs(struct kvm_vcpu *vcpu,
3fec037d 200 struct sys_reg_params *p,
0c557ed4
MZ		 201 const struct sys_reg_desc *r)
202{
203 if (p->is_write) {
2ec5be3d 204 vcpu_sys_reg(vcpu, r->reg) = p->regval;
0c557ed4
MZ		 205 vcpu->arch.debug_flags |= KVM_ARM64_DEBUG_DIRTY;
206 } else {
2ec5be3d 207 p->regval = vcpu_sys_reg(vcpu, r->reg);
0c557ed4
MZ		 208 }
209
2ec5be3d 210 trace_trap_reg(__func__, r->reg, p->is_write, p->regval);
eef8c85a 211
0c557ed4
MZ		 212 return true;
213}
214
84e690bf
AB		 215/*
216 * reg_to_dbg/dbg_to_reg
217 *
218 * A 32 bit write to a debug register leave top bits alone
219 * A 32 bit read from a debug register only returns the bottom bits
220 *
221 * All writes will set the KVM_ARM64_DEBUG_DIRTY flag to ensure the
222 * hyp.S code switches between host and guest values in future.
223 */
281243cb
MZ		 224static void reg_to_dbg(struct kvm_vcpu *vcpu,
225 struct sys_reg_params *p,
226 u64 *dbg_reg)
84e690bf 227{
2ec5be3d 228 u64 val = p->regval;
84e690bf
AB		 229
230 if (p->is_32bit) {
231 val &= 0xffffffffUL;
232 val |= ((*dbg_reg >> 32) << 32);
233 }
234
235 *dbg_reg = val;
236 vcpu->arch.debug_flags |= KVM_ARM64_DEBUG_DIRTY;
237}
238
281243cb
MZ		 239static void dbg_to_reg(struct kvm_vcpu *vcpu,
240 struct sys_reg_params *p,
241 u64 *dbg_reg)
84e690bf 242{
2ec5be3d 243 p->regval = *dbg_reg;
84e690bf 244 if (p->is_32bit)
2ec5be3d 245 p->regval &= 0xffffffffUL;
84e690bf
AB		 246}
247
281243cb
MZ		 248static bool trap_bvr(struct kvm_vcpu *vcpu,
249 struct sys_reg_params *p,
250 const struct sys_reg_desc *rd)
84e690bf
AB		 251{
252 u64 *dbg_reg = &vcpu->arch.vcpu_debug_state.dbg_bvr[rd->reg];
253
254 if (p->is_write)
255 reg_to_dbg(vcpu, p, dbg_reg);
256 else
257 dbg_to_reg(vcpu, p, dbg_reg);
258
eef8c85a
AB		 259 trace_trap_reg(__func__, rd->reg, p->is_write, *dbg_reg);
260
84e690bf
AB		 261 return true;
262}
263
264static int set_bvr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
265 const struct kvm_one_reg *reg, void __user *uaddr)
266{
267 __u64 *r = &vcpu->arch.vcpu_debug_state.dbg_bvr[rd->reg];
268
1713e5aa 269 if (copy_from_user(r, uaddr, KVM_REG_SIZE(reg->id)) != 0)
84e690bf
AB		 270 return -EFAULT;
271 return 0;
272}
273
274static int get_bvr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
275 const struct kvm_one_reg *reg, void __user *uaddr)
276{
277 __u64 *r = &vcpu->arch.vcpu_debug_state.dbg_bvr[rd->reg];
278
279 if (copy_to_user(uaddr, r, KVM_REG_SIZE(reg->id)) != 0)
280 return -EFAULT;
281 return 0;
282}
283
281243cb
MZ		 284static void reset_bvr(struct kvm_vcpu *vcpu,
285 const struct sys_reg_desc *rd)
84e690bf
AB		 286{
287 vcpu->arch.vcpu_debug_state.dbg_bvr[rd->reg] = rd->val;
288}
289
281243cb
MZ		 290static bool trap_bcr(struct kvm_vcpu *vcpu,
291 struct sys_reg_params *p,
292 const struct sys_reg_desc *rd)
84e690bf
AB		 293{
294 u64 *dbg_reg = &vcpu->arch.vcpu_debug_state.dbg_bcr[rd->reg];
295
296 if (p->is_write)
297 reg_to_dbg(vcpu, p, dbg_reg);
298 else
299 dbg_to_reg(vcpu, p, dbg_reg);
300
eef8c85a
AB		 301 trace_trap_reg(__func__, rd->reg, p->is_write, *dbg_reg);
302
84e690bf
AB		 303 return true;
304}
305
306static int set_bcr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
307 const struct kvm_one_reg *reg, void __user *uaddr)
308{
309 __u64 *r = &vcpu->arch.vcpu_debug_state.dbg_bcr[rd->reg];
310
1713e5aa 311 if (copy_from_user(r, uaddr, KVM_REG_SIZE(reg->id)) != 0)
84e690bf
AB		 312 return -EFAULT;
313
314 return 0;
315}
316
317static int get_bcr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
318 const struct kvm_one_reg *reg, void __user *uaddr)
319{
320 __u64 *r = &vcpu->arch.vcpu_debug_state.dbg_bcr[rd->reg];
321
322 if (copy_to_user(uaddr, r, KVM_REG_SIZE(reg->id)) != 0)
323 return -EFAULT;
324 return 0;
325}
326
281243cb
MZ		 327static void reset_bcr(struct kvm_vcpu *vcpu,
328 const struct sys_reg_desc *rd)
84e690bf
AB		 329{
330 vcpu->arch.vcpu_debug_state.dbg_bcr[rd->reg] = rd->val;
331}
332
281243cb
MZ		 333static bool trap_wvr(struct kvm_vcpu *vcpu,
334 struct sys_reg_params *p,
335 const struct sys_reg_desc *rd)
84e690bf
AB		 336{
337 u64 *dbg_reg = &vcpu->arch.vcpu_debug_state.dbg_wvr[rd->reg];
338
339 if (p->is_write)
340 reg_to_dbg(vcpu, p, dbg_reg);
341 else
342 dbg_to_reg(vcpu, p, dbg_reg);
343
eef8c85a
AB		 344 trace_trap_reg(__func__, rd->reg, p->is_write,
345 vcpu->arch.vcpu_debug_state.dbg_wvr[rd->reg]);
346
84e690bf
AB		 347 return true;
348}
349
350static int set_wvr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
351 const struct kvm_one_reg *reg, void __user *uaddr)
352{
353 __u64 *r = &vcpu->arch.vcpu_debug_state.dbg_wvr[rd->reg];
354
1713e5aa 355 if (copy_from_user(r, uaddr, KVM_REG_SIZE(reg->id)) != 0)
84e690bf
AB		 356 return -EFAULT;
357 return 0;
358}
359
360static int get_wvr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
361 const struct kvm_one_reg *reg, void __user *uaddr)
362{
363 __u64 *r = &vcpu->arch.vcpu_debug_state.dbg_wvr[rd->reg];
364
365 if (copy_to_user(uaddr, r, KVM_REG_SIZE(reg->id)) != 0)
366 return -EFAULT;
367 return 0;
368}
369
281243cb
MZ		 370static void reset_wvr(struct kvm_vcpu *vcpu,
371 const struct sys_reg_desc *rd)
84e690bf
AB		 372{
373 vcpu->arch.vcpu_debug_state.dbg_wvr[rd->reg] = rd->val;
374}
375
281243cb
MZ		 376static bool trap_wcr(struct kvm_vcpu *vcpu,
377 struct sys_reg_params *p,
378 const struct sys_reg_desc *rd)
84e690bf
AB		 379{
380 u64 *dbg_reg = &vcpu->arch.vcpu_debug_state.dbg_wcr[rd->reg];
381
382 if (p->is_write)
383 reg_to_dbg(vcpu, p, dbg_reg);
384 else
385 dbg_to_reg(vcpu, p, dbg_reg);
386
eef8c85a
AB		 387 trace_trap_reg(__func__, rd->reg, p->is_write, *dbg_reg);
388
84e690bf
AB		 389 return true;
390}
391
392static int set_wcr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
393 const struct kvm_one_reg *reg, void __user *uaddr)
394{
395 __u64 *r = &vcpu->arch.vcpu_debug_state.dbg_wcr[rd->reg];
396
1713e5aa 397 if (copy_from_user(r, uaddr, KVM_REG_SIZE(reg->id)) != 0)
84e690bf
AB		 398 return -EFAULT;
399 return 0;
400}
401
402static int get_wcr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
403 const struct kvm_one_reg *reg, void __user *uaddr)
404{
405 __u64 *r = &vcpu->arch.vcpu_debug_state.dbg_wcr[rd->reg];
406
407 if (copy_to_user(uaddr, r, KVM_REG_SIZE(reg->id)) != 0)
408 return -EFAULT;
409 return 0;
410}
411
281243cb
MZ		 412static void reset_wcr(struct kvm_vcpu *vcpu,
413 const struct sys_reg_desc *rd)
84e690bf
AB		 414{
415 vcpu->arch.vcpu_debug_state.dbg_wcr[rd->reg] = rd->val;
416}
417
7c8c5e6a
MZ		 418static void reset_amair_el1(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r)
419{
420 u64 amair;
421
422 asm volatile("mrs %0, amair_el1\n" : "=r" (amair));
423 vcpu_sys_reg(vcpu, AMAIR_EL1) = amair;
424}
425
426static void reset_mpidr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r)
427{
4429fc64
AP		 428 u64 mpidr;
429
7c8c5e6a 430 /*
4429fc64
AP		 431 * Map the vcpu_id into the first three affinity level fields of
432 * the MPIDR. We limit the number of VCPUs in level 0 due to a
433 * limitation to 16 CPUs in that level in the ICC_SGIxR registers
434 * of the GICv3 to be able to address each CPU directly when
435 * sending IPIs.
7c8c5e6a 436 */
4429fc64
AP		 437 mpidr = (vcpu->vcpu_id & 0x0f) << MPIDR_LEVEL_SHIFT(0);
438 mpidr |= ((vcpu->vcpu_id >> 4) & 0xff) << MPIDR_LEVEL_SHIFT(1);
439 mpidr |= ((vcpu->vcpu_id >> 12) & 0xff) << MPIDR_LEVEL_SHIFT(2);
440 vcpu_sys_reg(vcpu, MPIDR_EL1) = (1ULL << 31) | mpidr;
7c8c5e6a
MZ		 441}
442
ab946834
SZ		 443static void reset_pmcr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r)
444{
445 u64 pmcr, val;
446
447 asm volatile("mrs %0, pmcr_el0\n" : "=r" (pmcr));
448 /* Writable bits of PMCR_EL0 (ARMV8_PMU_PMCR_MASK) is reset to UNKNOWN
449 * except PMCR.E resetting to zero.
450 */
451 val = ((pmcr & ~ARMV8_PMU_PMCR_MASK)
452 | (ARMV8_PMU_PMCR_MASK & 0xdecafbad)) & (~ARMV8_PMU_PMCR_E);
453 vcpu_sys_reg(vcpu, PMCR_EL0) = val;
454}
455
456static bool access_pmcr(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
457 const struct sys_reg_desc *r)
458{
459 u64 val;
460
461 if (!kvm_arm_pmu_v3_ready(vcpu))
462 return trap_raz_wi(vcpu, p, r);
463
464 if (p->is_write) {
465 /* Only update writeable bits of PMCR */
466 val = vcpu_sys_reg(vcpu, PMCR_EL0);
467 val &= ~ARMV8_PMU_PMCR_MASK;
468 val |= p->regval & ARMV8_PMU_PMCR_MASK;
469 vcpu_sys_reg(vcpu, PMCR_EL0) = val;
470 } else {
471 /* PMCR.P & PMCR.C are RAZ */
472 val = vcpu_sys_reg(vcpu, PMCR_EL0)
473 & ~(ARMV8_PMU_PMCR_P | ARMV8_PMU_PMCR_C);
474 p->regval = val;
475 }
476
477 return true;
478}
479
3965c3ce
SZ		 480static bool access_pmselr(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
481 const struct sys_reg_desc *r)
482{
483 if (!kvm_arm_pmu_v3_ready(vcpu))
484 return trap_raz_wi(vcpu, p, r);
485
486 if (p->is_write)
487 vcpu_sys_reg(vcpu, PMSELR_EL0) = p->regval;
488 else
489 /* return PMSELR.SEL field */
490 p->regval = vcpu_sys_reg(vcpu, PMSELR_EL0)
491 & ARMV8_PMU_COUNTER_MASK;
492
493 return true;
494}
495
a86b5505
SZ		 496static bool access_pmceid(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
497 const struct sys_reg_desc *r)
498{
499 u64 pmceid;
500
501 if (!kvm_arm_pmu_v3_ready(vcpu))
502 return trap_raz_wi(vcpu, p, r);
503
504 BUG_ON(p->is_write);
505
506 if (!(p->Op2 & 1))
507 asm volatile("mrs %0, pmceid0_el0\n" : "=r" (pmceid));
508 else
509 asm volatile("mrs %0, pmceid1_el0\n" : "=r" (pmceid));
510
511 p->regval = pmceid;
512
513 return true;
514}
515
051ff581
SZ		 516static bool pmu_counter_idx_valid(struct kvm_vcpu *vcpu, u64 idx)
517{
518 u64 pmcr, val;
519
520 pmcr = vcpu_sys_reg(vcpu, PMCR_EL0);
521 val = (pmcr >> ARMV8_PMU_PMCR_N_SHIFT) & ARMV8_PMU_PMCR_N_MASK;
522 if (idx >= val && idx != ARMV8_PMU_CYCLE_IDX)
523 return false;
524
525 return true;
526}
527
528static bool access_pmu_evcntr(struct kvm_vcpu *vcpu,
529 struct sys_reg_params *p,
530 const struct sys_reg_desc *r)
531{
532 u64 idx;
533
534 if (!kvm_arm_pmu_v3_ready(vcpu))
535 return trap_raz_wi(vcpu, p, r);
536
537 if (r->CRn == 9 && r->CRm == 13) {
538 if (r->Op2 == 2) {
539 /* PMXEVCNTR_EL0 */
540 idx = vcpu_sys_reg(vcpu, PMSELR_EL0)
541 & ARMV8_PMU_COUNTER_MASK;
542 } else if (r->Op2 == 0) {
543 /* PMCCNTR_EL0 */
544 idx = ARMV8_PMU_CYCLE_IDX;
545 } else {
546 BUG();
547 }
548 } else if (r->CRn == 14 && (r->CRm & 12) == 8) {
549 /* PMEVCNTRn_EL0 */
550 idx = ((r->CRm & 3) << 3) | (r->Op2 & 7);
551 } else {
552 BUG();
553 }
554
555 if (!pmu_counter_idx_valid(vcpu, idx))
556 return false;
557
558 if (p->is_write)
559 kvm_pmu_set_counter_value(vcpu, idx, p->regval);
560 else
561 p->regval = kvm_pmu_get_counter_value(vcpu, idx);
562
563 return true;
564}
565
9feb21ac
SZ		 566static bool access_pmu_evtyper(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
567 const struct sys_reg_desc *r)
568{
569 u64 idx, reg;
570
571 if (!kvm_arm_pmu_v3_ready(vcpu))
572 return trap_raz_wi(vcpu, p, r);
573
574 if (r->CRn == 9 && r->CRm == 13 && r->Op2 == 1) {
575 /* PMXEVTYPER_EL0 */
576 idx = vcpu_sys_reg(vcpu, PMSELR_EL0) & ARMV8_PMU_COUNTER_MASK;
577 reg = PMEVTYPER0_EL0 + idx;
578 } else if (r->CRn == 14 && (r->CRm & 12) == 12) {
579 idx = ((r->CRm & 3) << 3) | (r->Op2 & 7);
580 if (idx == ARMV8_PMU_CYCLE_IDX)
581 reg = PMCCFILTR_EL0;
582 else
583 /* PMEVTYPERn_EL0 */
584 reg = PMEVTYPER0_EL0 + idx;
585 } else {
586 BUG();
587 }
588
589 if (!pmu_counter_idx_valid(vcpu, idx))
590 return false;
591
592 if (p->is_write) {
593 kvm_pmu_set_counter_event_type(vcpu, p->regval, idx);
594 vcpu_sys_reg(vcpu, reg) = p->regval & ARMV8_PMU_EVTYPE_MASK;
595 } else {
596 p->regval = vcpu_sys_reg(vcpu, reg) & ARMV8_PMU_EVTYPE_MASK;
597 }
598
599 return true;
600}
601
96b0eebc
SZ		 602static bool access_pmcnten(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
603 const struct sys_reg_desc *r)
604{
605 u64 val, mask;
606
607 if (!kvm_arm_pmu_v3_ready(vcpu))
608 return trap_raz_wi(vcpu, p, r);
609
610 mask = kvm_pmu_valid_counter_mask(vcpu);
611 if (p->is_write) {
612 val = p->regval & mask;
613 if (r->Op2 & 0x1) {
614 /* accessing PMCNTENSET_EL0 */
615 vcpu_sys_reg(vcpu, PMCNTENSET_EL0) |= val;
616 kvm_pmu_enable_counter(vcpu, val);
617 } else {
618 /* accessing PMCNTENCLR_EL0 */
619 vcpu_sys_reg(vcpu, PMCNTENSET_EL0) &= ~val;
620 kvm_pmu_disable_counter(vcpu, val);
621 }
622 } else {
623 p->regval = vcpu_sys_reg(vcpu, PMCNTENSET_EL0) & mask;
624 }
625
626 return true;
627}
628
0c557ed4
MZ		 629/* Silly macro to expand the DBG{BCR,BVR,WVR,WCR}n_EL1 registers in one go */
630#define DBG_BCR_BVR_WCR_WVR_EL1(n) \
631 /* DBGBVRn_EL1 */ \
632 { Op0(0b10), Op1(0b000), CRn(0b0000), CRm((n)), Op2(0b100), \
84e690bf 633 trap_bvr, reset_bvr, n, 0, get_bvr, set_bvr }, \
0c557ed4
MZ		 634 /* DBGBCRn_EL1 */ \
635 { Op0(0b10), Op1(0b000), CRn(0b0000), CRm((n)), Op2(0b101), \
84e690bf 636 trap_bcr, reset_bcr, n, 0, get_bcr, set_bcr }, \
0c557ed4
MZ		 637 /* DBGWVRn_EL1 */ \
638 { Op0(0b10), Op1(0b000), CRn(0b0000), CRm((n)), Op2(0b110), \
84e690bf 639 trap_wvr, reset_wvr, n, 0, get_wvr, set_wvr }, \
0c557ed4
MZ		 640 /* DBGWCRn_EL1 */ \
641 { Op0(0b10), Op1(0b000), CRn(0b0000), CRm((n)), Op2(0b111), \
84e690bf 642 trap_wcr, reset_wcr, n, 0, get_wcr, set_wcr }
0c557ed4 643
051ff581
SZ		 644/* Macro to expand the PMEVCNTRn_EL0 register */
645#define PMU_PMEVCNTR_EL0(n) \
646 /* PMEVCNTRn_EL0 */ \
647 { Op0(0b11), Op1(0b011), CRn(0b1110), \
648 CRm((0b1000 | (((n) >> 3) & 0x3))), Op2(((n) & 0x7)), \
649 access_pmu_evcntr, reset_unknown, (PMEVCNTR0_EL0 + n), }
650
9feb21ac
SZ		 651/* Macro to expand the PMEVTYPERn_EL0 register */
652#define PMU_PMEVTYPER_EL0(n) \
653 /* PMEVTYPERn_EL0 */ \
654 { Op0(0b11), Op1(0b011), CRn(0b1110), \
655 CRm((0b1100 | (((n) >> 3) & 0x3))), Op2(((n) & 0x7)), \
656 access_pmu_evtyper, reset_unknown, (PMEVTYPER0_EL0 + n), }
657
7c8c5e6a
MZ		 658/*
659 * Architected system registers.
660 * Important: Must be sorted ascending by Op0, Op1, CRn, CRm, Op2
7609c125
MZ		 661 *
662 * We could trap ID_DFR0 and tell the guest we don't support performance
663 * monitoring. Unfortunately the patch to make the kernel check ID_DFR0 was
664 * NAKed, so it will read the PMCR anyway.
665 *
666 * Therefore we tell the guest we have 0 counters. Unfortunately, we
667 * must always support PMCCNTR (the cycle counter): we just RAZ/WI for
668 * all PM registers, which doesn't crash the guest kernel at least.
669 *
0c557ed4
MZ		 670 * Debug handling: We do trap most, if not all debug related system
671 * registers. The implementation is good enough to ensure that a guest
672 * can use these with minimal performance degradation. The drawback is
673 * that we don't implement any of the external debug, none of the
674 * OSlock protocol. This should be revisited if we ever encounter a
675 * more demanding guest...
7c8c5e6a
MZ		 676 */
677static const struct sys_reg_desc sys_reg_descs[] = {
678 /* DC ISW */
679 { Op0(0b01), Op1(0b000), CRn(0b0111), CRm(0b0110), Op2(0b010),
680 access_dcsw },
681 /* DC CSW */
682 { Op0(0b01), Op1(0b000), CRn(0b0111), CRm(0b1010), Op2(0b010),
683 access_dcsw },
684 /* DC CISW */
685 { Op0(0b01), Op1(0b000), CRn(0b0111), CRm(0b1110), Op2(0b010),
686 access_dcsw },
687
0c557ed4
MZ		 688 DBG_BCR_BVR_WCR_WVR_EL1(0),
689 DBG_BCR_BVR_WCR_WVR_EL1(1),
690 /* MDCCINT_EL1 */
691 { Op0(0b10), Op1(0b000), CRn(0b0000), CRm(0b0010), Op2(0b000),
692 trap_debug_regs, reset_val, MDCCINT_EL1, 0 },
693 /* MDSCR_EL1 */
694 { Op0(0b10), Op1(0b000), CRn(0b0000), CRm(0b0010), Op2(0b010),
695 trap_debug_regs, reset_val, MDSCR_EL1, 0 },
696 DBG_BCR_BVR_WCR_WVR_EL1(2),
697 DBG_BCR_BVR_WCR_WVR_EL1(3),
698 DBG_BCR_BVR_WCR_WVR_EL1(4),
699 DBG_BCR_BVR_WCR_WVR_EL1(5),
700 DBG_BCR_BVR_WCR_WVR_EL1(6),
701 DBG_BCR_BVR_WCR_WVR_EL1(7),
702 DBG_BCR_BVR_WCR_WVR_EL1(8),
703 DBG_BCR_BVR_WCR_WVR_EL1(9),
704 DBG_BCR_BVR_WCR_WVR_EL1(10),
705 DBG_BCR_BVR_WCR_WVR_EL1(11),
706 DBG_BCR_BVR_WCR_WVR_EL1(12),
707 DBG_BCR_BVR_WCR_WVR_EL1(13),
708 DBG_BCR_BVR_WCR_WVR_EL1(14),
709 DBG_BCR_BVR_WCR_WVR_EL1(15),
710
711 /* MDRAR_EL1 */
712 { Op0(0b10), Op1(0b000), CRn(0b0001), CRm(0b0000), Op2(0b000),
713 trap_raz_wi },
714 /* OSLAR_EL1 */
715 { Op0(0b10), Op1(0b000), CRn(0b0001), CRm(0b0000), Op2(0b100),
716 trap_raz_wi },
717 /* OSLSR_EL1 */
718 { Op0(0b10), Op1(0b000), CRn(0b0001), CRm(0b0001), Op2(0b100),
719 trap_oslsr_el1 },
720 /* OSDLR_EL1 */
721 { Op0(0b10), Op1(0b000), CRn(0b0001), CRm(0b0011), Op2(0b100),
722 trap_raz_wi },
723 /* DBGPRCR_EL1 */
724 { Op0(0b10), Op1(0b000), CRn(0b0001), CRm(0b0100), Op2(0b100),
725 trap_raz_wi },
726 /* DBGCLAIMSET_EL1 */
727 { Op0(0b10), Op1(0b000), CRn(0b0111), CRm(0b1000), Op2(0b110),
728 trap_raz_wi },
729 /* DBGCLAIMCLR_EL1 */
730 { Op0(0b10), Op1(0b000), CRn(0b0111), CRm(0b1001), Op2(0b110),
731 trap_raz_wi },
732 /* DBGAUTHSTATUS_EL1 */
733 { Op0(0b10), Op1(0b000), CRn(0b0111), CRm(0b1110), Op2(0b110),
734 trap_dbgauthstatus_el1 },
735
0c557ed4
MZ		 736 /* MDCCSR_EL1 */
737 { Op0(0b10), Op1(0b011), CRn(0b0000), CRm(0b0001), Op2(0b000),
738 trap_raz_wi },
739 /* DBGDTR_EL0 */
740 { Op0(0b10), Op1(0b011), CRn(0b0000), CRm(0b0100), Op2(0b000),
741 trap_raz_wi },
742 /* DBGDTR[TR]X_EL0 */
743 { Op0(0b10), Op1(0b011), CRn(0b0000), CRm(0b0101), Op2(0b000),
744 trap_raz_wi },
745
62a89c44
MZ		 746 /* DBGVCR32_EL2 */
747 { Op0(0b10), Op1(0b100), CRn(0b0000), CRm(0b0111), Op2(0b000),
748 NULL, reset_val, DBGVCR32_EL2, 0 },
749
7c8c5e6a
MZ		 750 /* MPIDR_EL1 */
751 { Op0(0b11), Op1(0b000), CRn(0b0000), CRm(0b0000), Op2(0b101),
752 NULL, reset_mpidr, MPIDR_EL1 },
753 /* SCTLR_EL1 */
754 { Op0(0b11), Op1(0b000), CRn(0b0001), CRm(0b0000), Op2(0b000),
3c1e7165 755 access_vm_reg, reset_val, SCTLR_EL1, 0x00C50078 },
7c8c5e6a
MZ		 756 /* CPACR_EL1 */
757 { Op0(0b11), Op1(0b000), CRn(0b0001), CRm(0b0000), Op2(0b010),
758 NULL, reset_val, CPACR_EL1, 0 },
759 /* TTBR0_EL1 */
760 { Op0(0b11), Op1(0b000), CRn(0b0010), CRm(0b0000), Op2(0b000),
4d44923b 761 access_vm_reg, reset_unknown, TTBR0_EL1 },
7c8c5e6a
MZ		 762 /* TTBR1_EL1 */
763 { Op0(0b11), Op1(0b000), CRn(0b0010), CRm(0b0000), Op2(0b001),
4d44923b 764 access_vm_reg, reset_unknown, TTBR1_EL1 },
7c8c5e6a
MZ		 765 /* TCR_EL1 */
766 { Op0(0b11), Op1(0b000), CRn(0b0010), CRm(0b0000), Op2(0b010),
4d44923b 767 access_vm_reg, reset_val, TCR_EL1, 0 },
7c8c5e6a
MZ		 768
769 /* AFSR0_EL1 */
770 { Op0(0b11), Op1(0b000), CRn(0b0101), CRm(0b0001), Op2(0b000),
4d44923b 771 access_vm_reg, reset_unknown, AFSR0_EL1 },
7c8c5e6a
MZ		 772 /* AFSR1_EL1 */
773 { Op0(0b11), Op1(0b000), CRn(0b0101), CRm(0b0001), Op2(0b001),
4d44923b 774 access_vm_reg, reset_unknown, AFSR1_EL1 },
7c8c5e6a
MZ		 775 /* ESR_EL1 */
776 { Op0(0b11), Op1(0b000), CRn(0b0101), CRm(0b0010), Op2(0b000),
4d44923b 777 access_vm_reg, reset_unknown, ESR_EL1 },
7c8c5e6a
MZ		 778 /* FAR_EL1 */
779 { Op0(0b11), Op1(0b000), CRn(0b0110), CRm(0b0000), Op2(0b000),
4d44923b 780 access_vm_reg, reset_unknown, FAR_EL1 },
1bbd8054
MZ		 781 /* PAR_EL1 */
782 { Op0(0b11), Op1(0b000), CRn(0b0111), CRm(0b0100), Op2(0b000),
783 NULL, reset_unknown, PAR_EL1 },
7c8c5e6a
MZ		 784
785 /* PMINTENSET_EL1 */
786 { Op0(0b11), Op1(0b000), CRn(0b1001), CRm(0b1110), Op2(0b001),
7609c125 787 trap_raz_wi },
7c8c5e6a
MZ		 788 /* PMINTENCLR_EL1 */
789 { Op0(0b11), Op1(0b000), CRn(0b1001), CRm(0b1110), Op2(0b010),
7609c125 790 trap_raz_wi },
7c8c5e6a
MZ		 791
792 /* MAIR_EL1 */
793 { Op0(0b11), Op1(0b000), CRn(0b1010), CRm(0b0010), Op2(0b000),
4d44923b 794 access_vm_reg, reset_unknown, MAIR_EL1 },
7c8c5e6a
MZ		 795 /* AMAIR_EL1 */
796 { Op0(0b11), Op1(0b000), CRn(0b1010), CRm(0b0011), Op2(0b000),
4d44923b 797 access_vm_reg, reset_amair_el1, AMAIR_EL1 },
7c8c5e6a
MZ		 798
799 /* VBAR_EL1 */
800 { Op0(0b11), Op1(0b000), CRn(0b1100), CRm(0b0000), Op2(0b000),
801 NULL, reset_val, VBAR_EL1, 0 },
db7dedd0 802
6d52f35a
AP		 803 /* ICC_SGI1R_EL1 */
804 { Op0(0b11), Op1(0b000), CRn(0b1100), CRm(0b1011), Op2(0b101),
805 access_gic_sgi },
db7dedd0
CD		 806 /* ICC_SRE_EL1 */
807 { Op0(0b11), Op1(0b000), CRn(0b1100), CRm(0b1100), Op2(0b101),
808 trap_raz_wi },
809
7c8c5e6a
MZ		 810 /* CONTEXTIDR_EL1 */
811 { Op0(0b11), Op1(0b000), CRn(0b1101), CRm(0b0000), Op2(0b001),
4d44923b 812 access_vm_reg, reset_val, CONTEXTIDR_EL1, 0 },
7c8c5e6a
MZ		 813 /* TPIDR_EL1 */
814 { Op0(0b11), Op1(0b000), CRn(0b1101), CRm(0b0000), Op2(0b100),
815 NULL, reset_unknown, TPIDR_EL1 },
816
817 /* CNTKCTL_EL1 */
818 { Op0(0b11), Op1(0b000), CRn(0b1110), CRm(0b0001), Op2(0b000),
819 NULL, reset_val, CNTKCTL_EL1, 0},
820
821 /* CSSELR_EL1 */
822 { Op0(0b11), Op1(0b010), CRn(0b0000), CRm(0b0000), Op2(0b000),
823 NULL, reset_unknown, CSSELR_EL1 },
824
825 /* PMCR_EL0 */
826 { Op0(0b11), Op1(0b011), CRn(0b1001), CRm(0b1100), Op2(0b000),
ab946834 827 access_pmcr, reset_pmcr, },
7c8c5e6a
MZ		 828 /* PMCNTENSET_EL0 */
829 { Op0(0b11), Op1(0b011), CRn(0b1001), CRm(0b1100), Op2(0b001),
96b0eebc 830 access_pmcnten, reset_unknown, PMCNTENSET_EL0 },
7c8c5e6a
MZ		 831 /* PMCNTENCLR_EL0 */
832 { Op0(0b11), Op1(0b011), CRn(0b1001), CRm(0b1100), Op2(0b010),
96b0eebc 833 access_pmcnten, NULL, PMCNTENSET_EL0 },
7c8c5e6a
MZ		 834 /* PMOVSCLR_EL0 */
835 { Op0(0b11), Op1(0b011), CRn(0b1001), CRm(0b1100), Op2(0b011),
7609c125 836 trap_raz_wi },
7c8c5e6a
MZ		 837 /* PMSWINC_EL0 */
838 { Op0(0b11), Op1(0b011), CRn(0b1001), CRm(0b1100), Op2(0b100),
7609c125 839 trap_raz_wi },
7c8c5e6a
MZ		 840 /* PMSELR_EL0 */
841 { Op0(0b11), Op1(0b011), CRn(0b1001), CRm(0b1100), Op2(0b101),
3965c3ce 842 access_pmselr, reset_unknown, PMSELR_EL0 },
7c8c5e6a
MZ		 843 /* PMCEID0_EL0 */
844 { Op0(0b11), Op1(0b011), CRn(0b1001), CRm(0b1100), Op2(0b110),
a86b5505 845 access_pmceid },
7c8c5e6a
MZ		 846 /* PMCEID1_EL0 */
847 { Op0(0b11), Op1(0b011), CRn(0b1001), CRm(0b1100), Op2(0b111),
a86b5505 848 access_pmceid },
7c8c5e6a
MZ		 849 /* PMCCNTR_EL0 */
850 { Op0(0b11), Op1(0b011), CRn(0b1001), CRm(0b1101), Op2(0b000),
051ff581 851 access_pmu_evcntr, reset_unknown, PMCCNTR_EL0 },
7c8c5e6a
MZ		 852 /* PMXEVTYPER_EL0 */
853 { Op0(0b11), Op1(0b011), CRn(0b1001), CRm(0b1101), Op2(0b001),
9feb21ac 854 access_pmu_evtyper },
7c8c5e6a
MZ		 855 /* PMXEVCNTR_EL0 */
856 { Op0(0b11), Op1(0b011), CRn(0b1001), CRm(0b1101), Op2(0b010),
051ff581 857 access_pmu_evcntr },
7c8c5e6a
MZ		 858 /* PMUSERENR_EL0 */
859 { Op0(0b11), Op1(0b011), CRn(0b1001), CRm(0b1110), Op2(0b000),
7609c125 860 trap_raz_wi },
7c8c5e6a
MZ		 861 /* PMOVSSET_EL0 */
862 { Op0(0b11), Op1(0b011), CRn(0b1001), CRm(0b1110), Op2(0b011),
7609c125 863 trap_raz_wi },
7c8c5e6a
MZ		 864
865 /* TPIDR_EL0 */
866 { Op0(0b11), Op1(0b011), CRn(0b1101), CRm(0b0000), Op2(0b010),
867 NULL, reset_unknown, TPIDR_EL0 },
868 /* TPIDRRO_EL0 */
869 { Op0(0b11), Op1(0b011), CRn(0b1101), CRm(0b0000), Op2(0b011),
870 NULL, reset_unknown, TPIDRRO_EL0 },
62a89c44 871
051ff581
SZ		 872 /* PMEVCNTRn_EL0 */
873 PMU_PMEVCNTR_EL0(0),
874 PMU_PMEVCNTR_EL0(1),
875 PMU_PMEVCNTR_EL0(2),
876 PMU_PMEVCNTR_EL0(3),
877 PMU_PMEVCNTR_EL0(4),
878 PMU_PMEVCNTR_EL0(5),
879 PMU_PMEVCNTR_EL0(6),
880 PMU_PMEVCNTR_EL0(7),
881 PMU_PMEVCNTR_EL0(8),
882 PMU_PMEVCNTR_EL0(9),
883 PMU_PMEVCNTR_EL0(10),
884 PMU_PMEVCNTR_EL0(11),
885 PMU_PMEVCNTR_EL0(12),
886 PMU_PMEVCNTR_EL0(13),
887 PMU_PMEVCNTR_EL0(14),
888 PMU_PMEVCNTR_EL0(15),
889 PMU_PMEVCNTR_EL0(16),
890 PMU_PMEVCNTR_EL0(17),
891 PMU_PMEVCNTR_EL0(18),
892 PMU_PMEVCNTR_EL0(19),
893 PMU_PMEVCNTR_EL0(20),
894 PMU_PMEVCNTR_EL0(21),
895 PMU_PMEVCNTR_EL0(22),
896 PMU_PMEVCNTR_EL0(23),
897 PMU_PMEVCNTR_EL0(24),
898 PMU_PMEVCNTR_EL0(25),
899 PMU_PMEVCNTR_EL0(26),
900 PMU_PMEVCNTR_EL0(27),
901 PMU_PMEVCNTR_EL0(28),
902 PMU_PMEVCNTR_EL0(29),
903 PMU_PMEVCNTR_EL0(30),
9feb21ac
SZ		 904 /* PMEVTYPERn_EL0 */
905 PMU_PMEVTYPER_EL0(0),
906 PMU_PMEVTYPER_EL0(1),
907 PMU_PMEVTYPER_EL0(2),
908 PMU_PMEVTYPER_EL0(3),
909 PMU_PMEVTYPER_EL0(4),
910 PMU_PMEVTYPER_EL0(5),
911 PMU_PMEVTYPER_EL0(6),
912 PMU_PMEVTYPER_EL0(7),
913 PMU_PMEVTYPER_EL0(8),
914 PMU_PMEVTYPER_EL0(9),
915 PMU_PMEVTYPER_EL0(10),
916 PMU_PMEVTYPER_EL0(11),
917 PMU_PMEVTYPER_EL0(12),
918 PMU_PMEVTYPER_EL0(13),
919 PMU_PMEVTYPER_EL0(14),
920 PMU_PMEVTYPER_EL0(15),
921 PMU_PMEVTYPER_EL0(16),
922 PMU_PMEVTYPER_EL0(17),
923 PMU_PMEVTYPER_EL0(18),
924 PMU_PMEVTYPER_EL0(19),
925 PMU_PMEVTYPER_EL0(20),
926 PMU_PMEVTYPER_EL0(21),
927 PMU_PMEVTYPER_EL0(22),
928 PMU_PMEVTYPER_EL0(23),
929 PMU_PMEVTYPER_EL0(24),
930 PMU_PMEVTYPER_EL0(25),
931 PMU_PMEVTYPER_EL0(26),
932 PMU_PMEVTYPER_EL0(27),
933 PMU_PMEVTYPER_EL0(28),
934 PMU_PMEVTYPER_EL0(29),
935 PMU_PMEVTYPER_EL0(30),
936 /* PMCCFILTR_EL0
937 * This register resets as unknown in 64bit mode while it resets as zero
938 * in 32bit mode. Here we choose to reset it as zero for consistency.
939 */
940 { Op0(0b11), Op1(0b011), CRn(0b1110), CRm(0b1111), Op2(0b111),
941 access_pmu_evtyper, reset_val, PMCCFILTR_EL0, 0 },
051ff581 942
62a89c44
MZ		 943 /* DACR32_EL2 */
944 { Op0(0b11), Op1(0b100), CRn(0b0011), CRm(0b0000), Op2(0b000),
945 NULL, reset_unknown, DACR32_EL2 },
946 /* IFSR32_EL2 */
947 { Op0(0b11), Op1(0b100), CRn(0b0101), CRm(0b0000), Op2(0b001),
948 NULL, reset_unknown, IFSR32_EL2 },
949 /* FPEXC32_EL2 */
950 { Op0(0b11), Op1(0b100), CRn(0b0101), CRm(0b0011), Op2(0b000),
951 NULL, reset_val, FPEXC32_EL2, 0x70 },
952};
953
bdfb4b38 954static bool trap_dbgidr(struct kvm_vcpu *vcpu,
3fec037d 955 struct sys_reg_params *p,
bdfb4b38
MZ		 956 const struct sys_reg_desc *r)
957{
958 if (p->is_write) {
959 return ignore_write(vcpu, p);
960 } else {
4db8e5ea
SP		 961 u64 dfr = read_system_reg(SYS_ID_AA64DFR0_EL1);
962 u64 pfr = read_system_reg(SYS_ID_AA64PFR0_EL1);
963 u32 el3 = !!cpuid_feature_extract_field(pfr, ID_AA64PFR0_EL3_SHIFT);
bdfb4b38 964
2ec5be3d
PF		 965 p->regval = ((((dfr >> ID_AA64DFR0_WRPS_SHIFT) & 0xf) << 28) |
966 (((dfr >> ID_AA64DFR0_BRPS_SHIFT) & 0xf) << 24) |
967 (((dfr >> ID_AA64DFR0_CTX_CMPS_SHIFT) & 0xf) << 20)
968 | (6 << 16) | (el3 << 14) | (el3 << 12));
bdfb4b38
MZ		 969 return true;
970 }
971}
972
973static bool trap_debug32(struct kvm_vcpu *vcpu,
3fec037d 974 struct sys_reg_params *p,
bdfb4b38
MZ		 975 const struct sys_reg_desc *r)
976{
977 if (p->is_write) {
2ec5be3d 978 vcpu_cp14(vcpu, r->reg) = p->regval;
bdfb4b38
MZ		 979 vcpu->arch.debug_flags |= KVM_ARM64_DEBUG_DIRTY;
980 } else {
2ec5be3d 981 p->regval = vcpu_cp14(vcpu, r->reg);
bdfb4b38
MZ		 982 }
983
984 return true;
985}
986
84e690bf
AB		 987/* AArch32 debug register mappings
988 *
989 * AArch32 DBGBVRn is mapped to DBGBVRn_EL1[31:0]
990 * AArch32 DBGBXVRn is mapped to DBGBVRn_EL1[63:32]
991 *
992 * All control registers and watchpoint value registers are mapped to
993 * the lower 32 bits of their AArch64 equivalents. We share the trap
994 * handlers with the above AArch64 code which checks what mode the
995 * system is in.
996 */
997
281243cb
MZ		 998static bool trap_xvr(struct kvm_vcpu *vcpu,
999 struct sys_reg_params *p,
1000 const struct sys_reg_desc *rd)
84e690bf
AB		 1001{
1002 u64 *dbg_reg = &vcpu->arch.vcpu_debug_state.dbg_bvr[rd->reg];
1003
1004 if (p->is_write) {
1005 u64 val = *dbg_reg;
1006
1007 val &= 0xffffffffUL;
2ec5be3d 1008 val |= p->regval << 32;
84e690bf
AB		 1009 *dbg_reg = val;
1010
1011 vcpu->arch.debug_flags |= KVM_ARM64_DEBUG_DIRTY;
1012 } else {
2ec5be3d 1013 p->regval = *dbg_reg >> 32;
84e690bf
AB		 1014 }
1015
eef8c85a
AB		 1016 trace_trap_reg(__func__, rd->reg, p->is_write, *dbg_reg);
1017
84e690bf
AB		 1018 return true;
1019}
1020
1021#define DBG_BCR_BVR_WCR_WVR(n) \
1022 /* DBGBVRn */ \
1023 { Op1( 0), CRn( 0), CRm((n)), Op2( 4), trap_bvr, NULL, n }, \
1024 /* DBGBCRn */ \
1025 { Op1( 0), CRn( 0), CRm((n)), Op2( 5), trap_bcr, NULL, n }, \
1026 /* DBGWVRn */ \
1027 { Op1( 0), CRn( 0), CRm((n)), Op2( 6), trap_wvr, NULL, n }, \
1028 /* DBGWCRn */ \
1029 { Op1( 0), CRn( 0), CRm((n)), Op2( 7), trap_wcr, NULL, n }
1030
1031#define DBGBXVR(n) \
1032 { Op1( 0), CRn( 1), CRm((n)), Op2( 1), trap_xvr, NULL, n }
bdfb4b38
MZ		 1033
1034/*
1035 * Trapped cp14 registers. We generally ignore most of the external
1036 * debug, on the principle that they don't really make sense to a
84e690bf 1037 * guest. Revisit this one day, would this principle change.
bdfb4b38 1038 */
72564016 1039static const struct sys_reg_desc cp14_regs[] = {
bdfb4b38
MZ		 1040 /* DBGIDR */
1041 { Op1( 0), CRn( 0), CRm( 0), Op2( 0), trap_dbgidr },
1042 /* DBGDTRRXext */
1043 { Op1( 0), CRn( 0), CRm( 0), Op2( 2), trap_raz_wi },
1044
1045 DBG_BCR_BVR_WCR_WVR(0),
1046 /* DBGDSCRint */
1047 { Op1( 0), CRn( 0), CRm( 1), Op2( 0), trap_raz_wi },
1048 DBG_BCR_BVR_WCR_WVR(1),
1049 /* DBGDCCINT */
1050 { Op1( 0), CRn( 0), CRm( 2), Op2( 0), trap_debug32 },
1051 /* DBGDSCRext */
1052 { Op1( 0), CRn( 0), CRm( 2), Op2( 2), trap_debug32 },
1053 DBG_BCR_BVR_WCR_WVR(2),
1054 /* DBGDTR[RT]Xint */
1055 { Op1( 0), CRn( 0), CRm( 3), Op2( 0), trap_raz_wi },
1056 /* DBGDTR[RT]Xext */
1057 { Op1( 0), CRn( 0), CRm( 3), Op2( 2), trap_raz_wi },
1058 DBG_BCR_BVR_WCR_WVR(3),
1059 DBG_BCR_BVR_WCR_WVR(4),
1060 DBG_BCR_BVR_WCR_WVR(5),
1061 /* DBGWFAR */
1062 { Op1( 0), CRn( 0), CRm( 6), Op2( 0), trap_raz_wi },
1063 /* DBGOSECCR */
1064 { Op1( 0), CRn( 0), CRm( 6), Op2( 2), trap_raz_wi },
1065 DBG_BCR_BVR_WCR_WVR(6),
1066 /* DBGVCR */
1067 { Op1( 0), CRn( 0), CRm( 7), Op2( 0), trap_debug32 },
1068 DBG_BCR_BVR_WCR_WVR(7),
1069 DBG_BCR_BVR_WCR_WVR(8),
1070 DBG_BCR_BVR_WCR_WVR(9),
1071 DBG_BCR_BVR_WCR_WVR(10),
1072 DBG_BCR_BVR_WCR_WVR(11),
1073 DBG_BCR_BVR_WCR_WVR(12),
1074 DBG_BCR_BVR_WCR_WVR(13),
1075 DBG_BCR_BVR_WCR_WVR(14),
1076 DBG_BCR_BVR_WCR_WVR(15),
1077
1078 /* DBGDRAR (32bit) */
1079 { Op1( 0), CRn( 1), CRm( 0), Op2( 0), trap_raz_wi },
1080
1081 DBGBXVR(0),
1082 /* DBGOSLAR */
1083 { Op1( 0), CRn( 1), CRm( 0), Op2( 4), trap_raz_wi },
1084 DBGBXVR(1),
1085 /* DBGOSLSR */
1086 { Op1( 0), CRn( 1), CRm( 1), Op2( 4), trap_oslsr_el1 },
1087 DBGBXVR(2),
1088 DBGBXVR(3),
1089 /* DBGOSDLR */
1090 { Op1( 0), CRn( 1), CRm( 3), Op2( 4), trap_raz_wi },
1091 DBGBXVR(4),
1092 /* DBGPRCR */
1093 { Op1( 0), CRn( 1), CRm( 4), Op2( 4), trap_raz_wi },
1094 DBGBXVR(5),
1095 DBGBXVR(6),
1096 DBGBXVR(7),
1097 DBGBXVR(8),
1098 DBGBXVR(9),
1099 DBGBXVR(10),
1100 DBGBXVR(11),
1101 DBGBXVR(12),
1102 DBGBXVR(13),
1103 DBGBXVR(14),
1104 DBGBXVR(15),
1105
1106 /* DBGDSAR (32bit) */
1107 { Op1( 0), CRn( 2), CRm( 0), Op2( 0), trap_raz_wi },
1108
1109 /* DBGDEVID2 */
1110 { Op1( 0), CRn( 7), CRm( 0), Op2( 7), trap_raz_wi },
1111 /* DBGDEVID1 */
1112 { Op1( 0), CRn( 7), CRm( 1), Op2( 7), trap_raz_wi },
1113 /* DBGDEVID */
1114 { Op1( 0), CRn( 7), CRm( 2), Op2( 7), trap_raz_wi },
1115 /* DBGCLAIMSET */
1116 { Op1( 0), CRn( 7), CRm( 8), Op2( 6), trap_raz_wi },
1117 /* DBGCLAIMCLR */
1118 { Op1( 0), CRn( 7), CRm( 9), Op2( 6), trap_raz_wi },
1119 /* DBGAUTHSTATUS */
1120 { Op1( 0), CRn( 7), CRm(14), Op2( 6), trap_dbgauthstatus_el1 },
72564016
MZ		 1121};
1122
a9866ba0
MZ		 1123/* Trapped cp14 64bit registers */
1124static const struct sys_reg_desc cp14_64_regs[] = {
bdfb4b38
MZ		 1125 /* DBGDRAR (64bit) */
1126 { Op1( 0), CRm( 1), .access = trap_raz_wi },
1127
1128 /* DBGDSAR (64bit) */
1129 { Op1( 0), CRm( 2), .access = trap_raz_wi },
a9866ba0
MZ		 1130};
1131
051ff581
SZ		 1132/* Macro to expand the PMEVCNTRn register */
1133#define PMU_PMEVCNTR(n) \
1134 /* PMEVCNTRn */ \
1135 { Op1(0), CRn(0b1110), \
1136 CRm((0b1000 | (((n) >> 3) & 0x3))), Op2(((n) & 0x7)), \
1137 access_pmu_evcntr }
1138
9feb21ac
SZ		 1139/* Macro to expand the PMEVTYPERn register */
1140#define PMU_PMEVTYPER(n) \
1141 /* PMEVTYPERn */ \
1142 { Op1(0), CRn(0b1110), \
1143 CRm((0b1100 | (((n) >> 3) & 0x3))), Op2(((n) & 0x7)), \
1144 access_pmu_evtyper }
1145
4d44923b
MZ		 1146/*
1147 * Trapped cp15 registers. TTBR0/TTBR1 get a double encoding,
1148 * depending on the way they are accessed (as a 32bit or a 64bit
1149 * register).
1150 */
62a89c44 1151static const struct sys_reg_desc cp15_regs[] = {
6d52f35a
AP		 1152 { Op1( 0), CRn( 0), CRm(12), Op2( 0), access_gic_sgi },
1153
3c1e7165 1154 { Op1( 0), CRn( 1), CRm( 0), Op2( 0), access_vm_reg, NULL, c1_SCTLR },
4d44923b
MZ		 1155 { Op1( 0), CRn( 2), CRm( 0), Op2( 0), access_vm_reg, NULL, c2_TTBR0 },
1156 { Op1( 0), CRn( 2), CRm( 0), Op2( 1), access_vm_reg, NULL, c2_TTBR1 },
1157 { Op1( 0), CRn( 2), CRm( 0), Op2( 2), access_vm_reg, NULL, c2_TTBCR },
1158 { Op1( 0), CRn( 3), CRm( 0), Op2( 0), access_vm_reg, NULL, c3_DACR },
1159 { Op1( 0), CRn( 5), CRm( 0), Op2( 0), access_vm_reg, NULL, c5_DFSR },
1160 { Op1( 0), CRn( 5), CRm( 0), Op2( 1), access_vm_reg, NULL, c5_IFSR },
1161 { Op1( 0), CRn( 5), CRm( 1), Op2( 0), access_vm_reg, NULL, c5_ADFSR },
1162 { Op1( 0), CRn( 5), CRm( 1), Op2( 1), access_vm_reg, NULL, c5_AIFSR },
1163 { Op1( 0), CRn( 6), CRm( 0), Op2( 0), access_vm_reg, NULL, c6_DFAR },
1164 { Op1( 0), CRn( 6), CRm( 0), Op2( 2), access_vm_reg, NULL, c6_IFAR },
1165
62a89c44
MZ		 1166 /*
1167 * DC{C,I,CI}SW operations:
1168 */
1169 { Op1( 0), CRn( 7), CRm( 6), Op2( 2), access_dcsw },
1170 { Op1( 0), CRn( 7), CRm(10), Op2( 2), access_dcsw },
1171 { Op1( 0), CRn( 7), CRm(14), Op2( 2), access_dcsw },
4d44923b 1172
7609c125 1173 /* PMU */
ab946834 1174 { Op1( 0), CRn( 9), CRm(12), Op2( 0), access_pmcr },
96b0eebc
SZ		 1175 { Op1( 0), CRn( 9), CRm(12), Op2( 1), access_pmcnten },
1176 { Op1( 0), CRn( 9), CRm(12), Op2( 2), access_pmcnten },
7609c125 1177 { Op1( 0), CRn( 9), CRm(12), Op2( 3), trap_raz_wi },
3965c3ce 1178 { Op1( 0), CRn( 9), CRm(12), Op2( 5), access_pmselr },
a86b5505
SZ		 1179 { Op1( 0), CRn( 9), CRm(12), Op2( 6), access_pmceid },
1180 { Op1( 0), CRn( 9), CRm(12), Op2( 7), access_pmceid },
051ff581 1181 { Op1( 0), CRn( 9), CRm(13), Op2( 0), access_pmu_evcntr },
9feb21ac 1182 { Op1( 0), CRn( 9), CRm(13), Op2( 1), access_pmu_evtyper },
051ff581 1183 { Op1( 0), CRn( 9), CRm(13), Op2( 2), access_pmu_evcntr },
7609c125
MZ		 1184 { Op1( 0), CRn( 9), CRm(14), Op2( 0), trap_raz_wi },
1185 { Op1( 0), CRn( 9), CRm(14), Op2( 1), trap_raz_wi },
1186 { Op1( 0), CRn( 9), CRm(14), Op2( 2), trap_raz_wi },
4d44923b
MZ		 1187
1188 { Op1( 0), CRn(10), CRm( 2), Op2( 0), access_vm_reg, NULL, c10_PRRR },
1189 { Op1( 0), CRn(10), CRm( 2), Op2( 1), access_vm_reg, NULL, c10_NMRR },
1190 { Op1( 0), CRn(10), CRm( 3), Op2( 0), access_vm_reg, NULL, c10_AMAIR0 },
1191 { Op1( 0), CRn(10), CRm( 3), Op2( 1), access_vm_reg, NULL, c10_AMAIR1 },
db7dedd0
CD		 1192
1193 /* ICC_SRE */
1194 { Op1( 0), CRn(12), CRm(12), Op2( 5), trap_raz_wi },
1195
4d44923b 1196 { Op1( 0), CRn(13), CRm( 0), Op2( 1), access_vm_reg, NULL, c13_CID },
051ff581
SZ		 1197
1198 /* PMEVCNTRn */
1199 PMU_PMEVCNTR(0),
1200 PMU_PMEVCNTR(1),
1201 PMU_PMEVCNTR(2),
1202 PMU_PMEVCNTR(3),
1203 PMU_PMEVCNTR(4),
1204 PMU_PMEVCNTR(5),
1205 PMU_PMEVCNTR(6),
1206 PMU_PMEVCNTR(7),
1207 PMU_PMEVCNTR(8),
1208 PMU_PMEVCNTR(9),
1209 PMU_PMEVCNTR(10),
1210 PMU_PMEVCNTR(11),
1211 PMU_PMEVCNTR(12),
1212 PMU_PMEVCNTR(13),
1213 PMU_PMEVCNTR(14),
1214 PMU_PMEVCNTR(15),
1215 PMU_PMEVCNTR(16),
1216 PMU_PMEVCNTR(17),
1217 PMU_PMEVCNTR(18),
1218 PMU_PMEVCNTR(19),
1219 PMU_PMEVCNTR(20),
1220 PMU_PMEVCNTR(21),
1221 PMU_PMEVCNTR(22),
1222 PMU_PMEVCNTR(23),
1223 PMU_PMEVCNTR(24),
1224 PMU_PMEVCNTR(25),
1225 PMU_PMEVCNTR(26),
1226 PMU_PMEVCNTR(27),
1227 PMU_PMEVCNTR(28),
1228 PMU_PMEVCNTR(29),
1229 PMU_PMEVCNTR(30),
9feb21ac
SZ		 1230 /* PMEVTYPERn */
1231 PMU_PMEVTYPER(0),
1232 PMU_PMEVTYPER(1),
1233 PMU_PMEVTYPER(2),
1234 PMU_PMEVTYPER(3),
1235 PMU_PMEVTYPER(4),
1236 PMU_PMEVTYPER(5),
1237 PMU_PMEVTYPER(6),
1238 PMU_PMEVTYPER(7),
1239 PMU_PMEVTYPER(8),
1240 PMU_PMEVTYPER(9),
1241 PMU_PMEVTYPER(10),
1242 PMU_PMEVTYPER(11),
1243 PMU_PMEVTYPER(12),
1244 PMU_PMEVTYPER(13),
1245 PMU_PMEVTYPER(14),
1246 PMU_PMEVTYPER(15),
1247 PMU_PMEVTYPER(16),
1248 PMU_PMEVTYPER(17),
1249 PMU_PMEVTYPER(18),
1250 PMU_PMEVTYPER(19),
1251 PMU_PMEVTYPER(20),
1252 PMU_PMEVTYPER(21),
1253 PMU_PMEVTYPER(22),
1254 PMU_PMEVTYPER(23),
1255 PMU_PMEVTYPER(24),
1256 PMU_PMEVTYPER(25),
1257 PMU_PMEVTYPER(26),
1258 PMU_PMEVTYPER(27),
1259 PMU_PMEVTYPER(28),
1260 PMU_PMEVTYPER(29),
1261 PMU_PMEVTYPER(30),
1262 /* PMCCFILTR */
1263 { Op1(0), CRn(14), CRm(15), Op2(7), access_pmu_evtyper },
a9866ba0
MZ		 1264};
1265
1266static const struct sys_reg_desc cp15_64_regs[] = {
1267 { Op1( 0), CRn( 0), CRm( 2), Op2( 0), access_vm_reg, NULL, c2_TTBR0 },
051ff581 1268 { Op1( 0), CRn( 0), CRm( 9), Op2( 0), access_pmu_evcntr },
6d52f35a 1269 { Op1( 0), CRn( 0), CRm(12), Op2( 0), access_gic_sgi },
4d44923b 1270 { Op1( 1), CRn( 0), CRm( 2), Op2( 0), access_vm_reg, NULL, c2_TTBR1 },
7c8c5e6a
MZ		 1271};
1272
1273/* Target specific emulation tables */
1274static struct kvm_sys_reg_target_table *target_tables[KVM_ARM_NUM_TARGETS];
1275
1276void kvm_register_target_sys_reg_table(unsigned int target,
1277 struct kvm_sys_reg_target_table *table)
1278{
1279 target_tables[target] = table;
1280}
1281
1282/* Get specific register table for this target. */
62a89c44
MZ		 1283static const struct sys_reg_desc *get_target_table(unsigned target,
1284 bool mode_is_64,
1285 size_t *num)
7c8c5e6a
MZ		 1286{
1287 struct kvm_sys_reg_target_table *table;
1288
1289 table = target_tables[target];
62a89c44
MZ		 1290 if (mode_is_64) {
1291 *num = table->table64.num;
1292 return table->table64.table;
1293 } else {
1294 *num = table->table32.num;
1295 return table->table32.table;
1296 }
7c8c5e6a
MZ		 1297}
1298
1299static const struct sys_reg_desc *find_reg(const struct sys_reg_params *params,
1300 const struct sys_reg_desc table[],
1301 unsigned int num)
1302{
1303 unsigned int i;
1304
1305 for (i = 0; i < num; i++) {
1306 const struct sys_reg_desc *r = &table[i];
1307
1308 if (params->Op0 != r->Op0)
1309 continue;
1310 if (params->Op1 != r->Op1)
1311 continue;
1312 if (params->CRn != r->CRn)
1313 continue;
1314 if (params->CRm != r->CRm)
1315 continue;
1316 if (params->Op2 != r->Op2)
1317 continue;
1318
1319 return r;
1320 }
1321 return NULL;
1322}
1323
62a89c44
MZ		 1324int kvm_handle_cp14_load_store(struct kvm_vcpu *vcpu, struct kvm_run *run)
1325{
1326 kvm_inject_undefined(vcpu);
1327 return 1;
1328}
1329
72564016
MZ		 1330/*
1331 * emulate_cp -- tries to match a sys_reg access in a handling table, and
1332 * call the corresponding trap handler.
1333 *
1334 * @params: pointer to the descriptor of the access
1335 * @table: array of trap descriptors
1336 * @num: size of the trap descriptor array
1337 *
1338 * Return 0 if the access has been handled, and -1 if not.
1339 */
1340static int emulate_cp(struct kvm_vcpu *vcpu,
3fec037d 1341 struct sys_reg_params *params,
72564016
MZ		 1342 const struct sys_reg_desc *table,
1343 size_t num)
62a89c44 1344{
72564016 1345 const struct sys_reg_desc *r;
62a89c44 1346
72564016
MZ		 1347 if (!table)
1348 return -1; /* Not handled */
62a89c44 1349
62a89c44 1350 r = find_reg(params, table, num);
62a89c44 1351
72564016 1352 if (r) {
62a89c44
MZ		 1353 /*
1354 * Not having an accessor means that we have
1355 * configured a trap that we don't know how to
1356 * handle. This certainly qualifies as a gross bug
1357 * that should be fixed right away.
1358 */
1359 BUG_ON(!r->access);
1360
1361 if (likely(r->access(vcpu, params, r))) {
1362 /* Skip instruction, since it was emulated */
1363 kvm_skip_instr(vcpu, kvm_vcpu_trap_il_is32bit(vcpu));
6327f35a
SZ		 1364 /* Handled */
1365 return 0;
62a89c44 1366 }
72564016
MZ		 1367 }
1368
1369 /* Not handled */
1370 return -1;
1371}
1372
1373static void unhandled_cp_access(struct kvm_vcpu *vcpu,
1374 struct sys_reg_params *params)
1375{
1376 u8 hsr_ec = kvm_vcpu_trap_get_class(vcpu);
1377 int cp;
1378
1379 switch(hsr_ec) {
c6d01a94
MR		 1380 case ESR_ELx_EC_CP15_32:
1381 case ESR_ELx_EC_CP15_64:
72564016
MZ		 1382 cp = 15;
1383 break;
c6d01a94
MR		 1384 case ESR_ELx_EC_CP14_MR:
1385 case ESR_ELx_EC_CP14_64:
72564016
MZ		 1386 cp = 14;
1387 break;
1388 default:
1389 WARN_ON((cp = -1));
62a89c44
MZ		 1390 }
1391
72564016
MZ		 1392 kvm_err("Unsupported guest CP%d access at: %08lx\n",
1393 cp, *vcpu_pc(vcpu));
62a89c44
MZ		 1394 print_sys_reg_instr(params);
1395 kvm_inject_undefined(vcpu);
1396}
1397
1398/**
7769db90 1399 * kvm_handle_cp_64 -- handles a mrrc/mcrr trap on a guest CP14/CP15 access
62a89c44
MZ		 1400 * @vcpu: The VCPU pointer
1401 * @run: The kvm_run struct
1402 */
72564016
MZ		 1403static int kvm_handle_cp_64(struct kvm_vcpu *vcpu,
1404 const struct sys_reg_desc *global,
1405 size_t nr_global,
1406 const struct sys_reg_desc *target_specific,
1407 size_t nr_specific)
62a89c44
MZ		 1408{
1409 struct sys_reg_params params;
1410 u32 hsr = kvm_vcpu_get_hsr(vcpu);
2ec5be3d 1411 int Rt = (hsr >> 5) & 0xf;
62a89c44
MZ		 1412 int Rt2 = (hsr >> 10) & 0xf;
1413
2072d29c
MZ		 1414 params.is_aarch32 = true;
1415 params.is_32bit = false;
62a89c44 1416 params.CRm = (hsr >> 1) & 0xf;
62a89c44
MZ		 1417 params.is_write = ((hsr & 1) == 0);
1418
1419 params.Op0 = 0;
1420 params.Op1 = (hsr >> 16) & 0xf;
1421 params.Op2 = 0;
1422 params.CRn = 0;
1423
1424 /*
2ec5be3d 1425 * Make a 64-bit value out of Rt and Rt2. As we use the same trap
62a89c44
MZ		 1426 * backends between AArch32 and AArch64, we get away with it.
1427 */
1428 if (params.is_write) {
2ec5be3d
PF		 1429 params.regval = vcpu_get_reg(vcpu, Rt) & 0xffffffff;
1430 params.regval |= vcpu_get_reg(vcpu, Rt2) << 32;
62a89c44
MZ		 1431 }
1432
72564016
MZ		 1433 if (!emulate_cp(vcpu, &params, target_specific, nr_specific))
1434 goto out;
1435 if (!emulate_cp(vcpu, &params, global, nr_global))
1436 goto out;
1437
1438 unhandled_cp_access(vcpu, &params);
62a89c44 1439
72564016 1440out:
2ec5be3d 1441 /* Split up the value between registers for the read side */
62a89c44 1442 if (!params.is_write) {
2ec5be3d
PF		 1443 vcpu_set_reg(vcpu, Rt, lower_32_bits(params.regval));
1444 vcpu_set_reg(vcpu, Rt2, upper_32_bits(params.regval));
62a89c44
MZ		 1445 }
1446
1447 return 1;
1448}
1449
1450/**
7769db90 1451 * kvm_handle_cp_32 -- handles a mrc/mcr trap on a guest CP14/CP15 access
62a89c44
MZ		 1452 * @vcpu: The VCPU pointer
1453 * @run: The kvm_run struct
1454 */
72564016
MZ		 1455static int kvm_handle_cp_32(struct kvm_vcpu *vcpu,
1456 const struct sys_reg_desc *global,
1457 size_t nr_global,
1458 const struct sys_reg_desc *target_specific,
1459 size_t nr_specific)
62a89c44
MZ		 1460{
1461 struct sys_reg_params params;
1462 u32 hsr = kvm_vcpu_get_hsr(vcpu);
2ec5be3d 1463 int Rt = (hsr >> 5) & 0xf;
62a89c44 1464
2072d29c
MZ		 1465 params.is_aarch32 = true;
1466 params.is_32bit = true;
62a89c44 1467 params.CRm = (hsr >> 1) & 0xf;
2ec5be3d 1468 params.regval = vcpu_get_reg(vcpu, Rt);
62a89c44
MZ		 1469 params.is_write = ((hsr & 1) == 0);
1470 params.CRn = (hsr >> 10) & 0xf;
1471 params.Op0 = 0;
1472 params.Op1 = (hsr >> 14) & 0x7;
1473 params.Op2 = (hsr >> 17) & 0x7;
1474
2ec5be3d
PF		 1475 if (!emulate_cp(vcpu, &params, target_specific, nr_specific) ||
1476 !emulate_cp(vcpu, &params, global, nr_global)) {
1477 if (!params.is_write)
1478 vcpu_set_reg(vcpu, Rt, params.regval);
72564016 1479 return 1;
2ec5be3d 1480 }
72564016
MZ		 1481
1482 unhandled_cp_access(vcpu, &params);
62a89c44
MZ		 1483 return 1;
1484}
1485
72564016
MZ		 1486int kvm_handle_cp15_64(struct kvm_vcpu *vcpu, struct kvm_run *run)
1487{
1488 const struct sys_reg_desc *target_specific;
1489 size_t num;
1490
1491 target_specific = get_target_table(vcpu->arch.target, false, &num);
1492 return kvm_handle_cp_64(vcpu,
a9866ba0 1493 cp15_64_regs, ARRAY_SIZE(cp15_64_regs),
72564016
MZ		 1494 target_specific, num);
1495}
1496
1497int kvm_handle_cp15_32(struct kvm_vcpu *vcpu, struct kvm_run *run)
1498{
1499 const struct sys_reg_desc *target_specific;
1500 size_t num;
1501
1502 target_specific = get_target_table(vcpu->arch.target, false, &num);
1503 return kvm_handle_cp_32(vcpu,
1504 cp15_regs, ARRAY_SIZE(cp15_regs),
1505 target_specific, num);
1506}
1507
1508int kvm_handle_cp14_64(struct kvm_vcpu *vcpu, struct kvm_run *run)
1509{
1510 return kvm_handle_cp_64(vcpu,
a9866ba0 1511 cp14_64_regs, ARRAY_SIZE(cp14_64_regs),
72564016
MZ		 1512 NULL, 0);
1513}
1514
1515int kvm_handle_cp14_32(struct kvm_vcpu *vcpu, struct kvm_run *run)
1516{
1517 return kvm_handle_cp_32(vcpu,
1518 cp14_regs, ARRAY_SIZE(cp14_regs),
1519 NULL, 0);
1520}
1521
7c8c5e6a 1522static int emulate_sys_reg(struct kvm_vcpu *vcpu,
3fec037d 1523 struct sys_reg_params *params)
7c8c5e6a
MZ		 1524{
1525 size_t num;
1526 const struct sys_reg_desc *table, *r;
1527
62a89c44 1528 table = get_target_table(vcpu->arch.target, true, &num);
7c8c5e6a
MZ		 1529
1530 /* Search target-specific then generic table. */
1531 r = find_reg(params, table, num);
1532 if (!r)
1533 r = find_reg(params, sys_reg_descs, ARRAY_SIZE(sys_reg_descs));
1534
1535 if (likely(r)) {
1536 /*
1537 * Not having an accessor means that we have
1538 * configured a trap that we don't know how to
1539 * handle. This certainly qualifies as a gross bug
1540 * that should be fixed right away.
1541 */
1542 BUG_ON(!r->access);
1543
1544 if (likely(r->access(vcpu, params, r))) {
1545 /* Skip instruction, since it was emulated */
1546 kvm_skip_instr(vcpu, kvm_vcpu_trap_il_is32bit(vcpu));
1547 return 1;
1548 }
1549 /* If access function fails, it should complain. */
1550 } else {
1551 kvm_err("Unsupported guest sys_reg access at: %lx\n",
1552 *vcpu_pc(vcpu));
1553 print_sys_reg_instr(params);
1554 }
1555 kvm_inject_undefined(vcpu);
1556 return 1;
1557}
1558
1559static void reset_sys_reg_descs(struct kvm_vcpu *vcpu,
1560 const struct sys_reg_desc *table, size_t num)
1561{
1562 unsigned long i;
1563
1564 for (i = 0; i < num; i++)
1565 if (table[i].reset)
1566 table[i].reset(vcpu, &table[i]);
1567}
1568
1569/**
1570 * kvm_handle_sys_reg -- handles a mrs/msr trap on a guest sys_reg access
1571 * @vcpu: The VCPU pointer
1572 * @run: The kvm_run struct
1573 */
1574int kvm_handle_sys_reg(struct kvm_vcpu *vcpu, struct kvm_run *run)
1575{
1576 struct sys_reg_params params;
1577 unsigned long esr = kvm_vcpu_get_hsr(vcpu);
2ec5be3d
PF		 1578 int Rt = (esr >> 5) & 0x1f;
1579 int ret;
7c8c5e6a 1580
eef8c85a
AB		 1581 trace_kvm_handle_sys_reg(esr);
1582
2072d29c
MZ		 1583 params.is_aarch32 = false;
1584 params.is_32bit = false;
7c8c5e6a
MZ		 1585 params.Op0 = (esr >> 20) & 3;
1586 params.Op1 = (esr >> 14) & 0x7;
1587 params.CRn = (esr >> 10) & 0xf;
1588 params.CRm = (esr >> 1) & 0xf;
1589 params.Op2 = (esr >> 17) & 0x7;
2ec5be3d 1590 params.regval = vcpu_get_reg(vcpu, Rt);
7c8c5e6a
MZ		 1591 params.is_write = !(esr & 1);
1592
2ec5be3d
PF		 1593 ret = emulate_sys_reg(vcpu, &params);
1594
1595 if (!params.is_write)
1596 vcpu_set_reg(vcpu, Rt, params.regval);
1597 return ret;
7c8c5e6a
MZ		 1598}
1599
1600/******************************************************************************
1601 * Userspace API
1602 *****************************************************************************/
1603
1604static bool index_to_params(u64 id, struct sys_reg_params *params)
1605{
1606 switch (id & KVM_REG_SIZE_MASK) {
1607 case KVM_REG_SIZE_U64:
1608 /* Any unused index bits means it's not valid. */
1609 if (id & ~(KVM_REG_ARCH_MASK | KVM_REG_SIZE_MASK
1610 | KVM_REG_ARM_COPROC_MASK
1611 | KVM_REG_ARM64_SYSREG_OP0_MASK
1612 | KVM_REG_ARM64_SYSREG_OP1_MASK
1613 | KVM_REG_ARM64_SYSREG_CRN_MASK
1614 | KVM_REG_ARM64_SYSREG_CRM_MASK
1615 | KVM_REG_ARM64_SYSREG_OP2_MASK))
1616 return false;
1617 params->Op0 = ((id & KVM_REG_ARM64_SYSREG_OP0_MASK)
1618 >> KVM_REG_ARM64_SYSREG_OP0_SHIFT);
1619 params->Op1 = ((id & KVM_REG_ARM64_SYSREG_OP1_MASK)
1620 >> KVM_REG_ARM64_SYSREG_OP1_SHIFT);
1621 params->CRn = ((id & KVM_REG_ARM64_SYSREG_CRN_MASK)
1622 >> KVM_REG_ARM64_SYSREG_CRN_SHIFT);
1623 params->CRm = ((id & KVM_REG_ARM64_SYSREG_CRM_MASK)
1624 >> KVM_REG_ARM64_SYSREG_CRM_SHIFT);
1625 params->Op2 = ((id & KVM_REG_ARM64_SYSREG_OP2_MASK)
1626 >> KVM_REG_ARM64_SYSREG_OP2_SHIFT);
1627 return true;
1628 default:
1629 return false;
1630 }
1631}
1632
1633/* Decode an index value, and find the sys_reg_desc entry. */
1634static const struct sys_reg_desc *index_to_sys_reg_desc(struct kvm_vcpu *vcpu,
1635 u64 id)
1636{
1637 size_t num;
1638 const struct sys_reg_desc *table, *r;
1639 struct sys_reg_params params;
1640
1641 /* We only do sys_reg for now. */
1642 if ((id & KVM_REG_ARM_COPROC_MASK) != KVM_REG_ARM64_SYSREG)
1643 return NULL;
1644
1645 if (!index_to_params(id, &params))
1646 return NULL;
1647
62a89c44 1648 table = get_target_table(vcpu->arch.target, true, &num);
7c8c5e6a
MZ		 1649 r = find_reg(&params, table, num);
1650 if (!r)
1651 r = find_reg(&params, sys_reg_descs, ARRAY_SIZE(sys_reg_descs));
1652
1653 /* Not saved in the sys_reg array? */
1654 if (r && !r->reg)
1655 r = NULL;
1656
1657 return r;
1658}
1659
1660/*
1661 * These are the invariant sys_reg registers: we let the guest see the
1662 * host versions of these, so they're part of the guest state.
1663 *
1664 * A future CPU may provide a mechanism to present different values to
1665 * the guest, or a future kvm may trap them.
1666 */
1667
1668#define FUNCTION_INVARIANT(reg) \
1669 static void get_##reg(struct kvm_vcpu *v, \
1670 const struct sys_reg_desc *r) \
1671 { \
1672 u64 val; \
1673 \
1674 asm volatile("mrs %0, " __stringify(reg) "\n" \
1675 : "=r" (val)); \
1676 ((struct sys_reg_desc *)r)->val = val; \
1677 }
1678
1679FUNCTION_INVARIANT(midr_el1)
1680FUNCTION_INVARIANT(ctr_el0)
1681FUNCTION_INVARIANT(revidr_el1)
1682FUNCTION_INVARIANT(id_pfr0_el1)
1683FUNCTION_INVARIANT(id_pfr1_el1)
1684FUNCTION_INVARIANT(id_dfr0_el1)
1685FUNCTION_INVARIANT(id_afr0_el1)
1686FUNCTION_INVARIANT(id_mmfr0_el1)
1687FUNCTION_INVARIANT(id_mmfr1_el1)
1688FUNCTION_INVARIANT(id_mmfr2_el1)
1689FUNCTION_INVARIANT(id_mmfr3_el1)
1690FUNCTION_INVARIANT(id_isar0_el1)
1691FUNCTION_INVARIANT(id_isar1_el1)
1692FUNCTION_INVARIANT(id_isar2_el1)
1693FUNCTION_INVARIANT(id_isar3_el1)
1694FUNCTION_INVARIANT(id_isar4_el1)
1695FUNCTION_INVARIANT(id_isar5_el1)
1696FUNCTION_INVARIANT(clidr_el1)
1697FUNCTION_INVARIANT(aidr_el1)
1698
1699/* ->val is filled in by kvm_sys_reg_table_init() */
1700static struct sys_reg_desc invariant_sys_regs[] = {
1701 { Op0(0b11), Op1(0b000), CRn(0b0000), CRm(0b0000), Op2(0b000),
1702 NULL, get_midr_el1 },
1703 { Op0(0b11), Op1(0b000), CRn(0b0000), CRm(0b0000), Op2(0b110),
1704 NULL, get_revidr_el1 },
1705 { Op0(0b11), Op1(0b000), CRn(0b0000), CRm(0b0001), Op2(0b000),
1706 NULL, get_id_pfr0_el1 },
1707 { Op0(0b11), Op1(0b000), CRn(0b0000), CRm(0b0001), Op2(0b001),
1708 NULL, get_id_pfr1_el1 },
1709 { Op0(0b11), Op1(0b000), CRn(0b0000), CRm(0b0001), Op2(0b010),
1710 NULL, get_id_dfr0_el1 },
1711 { Op0(0b11), Op1(0b000), CRn(0b0000), CRm(0b0001), Op2(0b011),
1712 NULL, get_id_afr0_el1 },
1713 { Op0(0b11), Op1(0b000), CRn(0b0000), CRm(0b0001), Op2(0b100),
1714 NULL, get_id_mmfr0_el1 },
1715 { Op0(0b11), Op1(0b000), CRn(0b0000), CRm(0b0001), Op2(0b101),
1716 NULL, get_id_mmfr1_el1 },
1717 { Op0(0b11), Op1(0b000), CRn(0b0000), CRm(0b0001), Op2(0b110),
1718 NULL, get_id_mmfr2_el1 },
1719 { Op0(0b11), Op1(0b000), CRn(0b0000), CRm(0b0001), Op2(0b111),
1720 NULL, get_id_mmfr3_el1 },
1721 { Op0(0b11), Op1(0b000), CRn(0b0000), CRm(0b0010), Op2(0b000),
1722 NULL, get_id_isar0_el1 },
1723 { Op0(0b11), Op1(0b000), CRn(0b0000), CRm(0b0010), Op2(0b001),
1724 NULL, get_id_isar1_el1 },
1725 { Op0(0b11), Op1(0b000), CRn(0b0000), CRm(0b0010), Op2(0b010),
1726 NULL, get_id_isar2_el1 },
1727 { Op0(0b11), Op1(0b000), CRn(0b0000), CRm(0b0010), Op2(0b011),
1728 NULL, get_id_isar3_el1 },
1729 { Op0(0b11), Op1(0b000), CRn(0b0000), CRm(0b0010), Op2(0b100),
1730 NULL, get_id_isar4_el1 },
1731 { Op0(0b11), Op1(0b000), CRn(0b0000), CRm(0b0010), Op2(0b101),
1732 NULL, get_id_isar5_el1 },
1733 { Op0(0b11), Op1(0b001), CRn(0b0000), CRm(0b0000), Op2(0b001),
1734 NULL, get_clidr_el1 },
1735 { Op0(0b11), Op1(0b001), CRn(0b0000), CRm(0b0000), Op2(0b111),
1736 NULL, get_aidr_el1 },
1737 { Op0(0b11), Op1(0b011), CRn(0b0000), CRm(0b0000), Op2(0b001),
1738 NULL, get_ctr_el0 },
1739};
1740
26c99af1 1741static int reg_from_user(u64 *val, const void __user *uaddr, u64 id)
7c8c5e6a 1742{
7c8c5e6a
MZ		 1743 if (copy_from_user(val, uaddr, KVM_REG_SIZE(id)) != 0)
1744 return -EFAULT;
1745 return 0;
1746}
1747
26c99af1 1748static int reg_to_user(void __user *uaddr, const u64 *val, u64 id)
7c8c5e6a 1749{
7c8c5e6a
MZ		 1750 if (copy_to_user(uaddr, val, KVM_REG_SIZE(id)) != 0)
1751 return -EFAULT;
1752 return 0;
1753}
1754
1755static int get_invariant_sys_reg(u64 id, void __user *uaddr)
1756{
1757 struct sys_reg_params params;
1758 const struct sys_reg_desc *r;
1759
1760 if (!index_to_params(id, &params))
1761 return -ENOENT;
1762
1763 r = find_reg(&params, invariant_sys_regs, ARRAY_SIZE(invariant_sys_regs));
1764 if (!r)
1765 return -ENOENT;
1766
1767 return reg_to_user(uaddr, &r->val, id);
1768}
1769
1770static int set_invariant_sys_reg(u64 id, void __user *uaddr)
1771{
1772 struct sys_reg_params params;
1773 const struct sys_reg_desc *r;
1774 int err;
1775 u64 val = 0; /* Make sure high bits are 0 for 32-bit regs */
1776
1777 if (!index_to_params(id, &params))
1778 return -ENOENT;
1779 r = find_reg(&params, invariant_sys_regs, ARRAY_SIZE(invariant_sys_regs));
1780 if (!r)
1781 return -ENOENT;
1782
1783 err = reg_from_user(&val, uaddr, id);
1784 if (err)
1785 return err;
1786
1787 /* This is what we mean by invariant: you can't change it. */
1788 if (r->val != val)
1789 return -EINVAL;
1790
1791 return 0;
1792}
1793
1794static bool is_valid_cache(u32 val)
1795{
1796 u32 level, ctype;
1797
1798 if (val >= CSSELR_MAX)
18d45766 1799 return false;
7c8c5e6a
MZ		 1800
1801 /* Bottom bit is Instruction or Data bit. Next 3 bits are level. */
1802 level = (val >> 1);
1803 ctype = (cache_levels >> (level * 3)) & 7;
1804
1805 switch (ctype) {
1806 case 0: /* No cache */
1807 return false;
1808 case 1: /* Instruction cache only */
1809 return (val & 1);
1810 case 2: /* Data cache only */
1811 case 4: /* Unified cache */
1812 return !(val & 1);
1813 case 3: /* Separate instruction and data caches */
1814 return true;
1815 default: /* Reserved: we can't know instruction or data. */
1816 return false;
1817 }
1818}
1819
1820static int demux_c15_get(u64 id, void __user *uaddr)
1821{
1822 u32 val;
1823 u32 __user *uval = uaddr;
1824
1825 /* Fail if we have unknown bits set. */
1826 if (id & ~(KVM_REG_ARCH_MASK|KVM_REG_SIZE_MASK|KVM_REG_ARM_COPROC_MASK
1827 | ((1 << KVM_REG_ARM_COPROC_SHIFT)-1)))
1828 return -ENOENT;
1829
1830 switch (id & KVM_REG_ARM_DEMUX_ID_MASK) {
1831 case KVM_REG_ARM_DEMUX_ID_CCSIDR:
1832 if (KVM_REG_SIZE(id) != 4)
1833 return -ENOENT;
1834 val = (id & KVM_REG_ARM_DEMUX_VAL_MASK)
1835 >> KVM_REG_ARM_DEMUX_VAL_SHIFT;
1836 if (!is_valid_cache(val))
1837 return -ENOENT;
1838
1839 return put_user(get_ccsidr(val), uval);
1840 default:
1841 return -ENOENT;
1842 }
1843}
1844
1845static int demux_c15_set(u64 id, void __user *uaddr)
1846{
1847 u32 val, newval;
1848 u32 __user *uval = uaddr;
1849
1850 /* Fail if we have unknown bits set. */
1851 if (id & ~(KVM_REG_ARCH_MASK|KVM_REG_SIZE_MASK|KVM_REG_ARM_COPROC_MASK
1852 | ((1 << KVM_REG_ARM_COPROC_SHIFT)-1)))
1853 return -ENOENT;
1854
1855 switch (id & KVM_REG_ARM_DEMUX_ID_MASK) {
1856 case KVM_REG_ARM_DEMUX_ID_CCSIDR:
1857 if (KVM_REG_SIZE(id) != 4)
1858 return -ENOENT;
1859 val = (id & KVM_REG_ARM_DEMUX_VAL_MASK)
1860 >> KVM_REG_ARM_DEMUX_VAL_SHIFT;
1861 if (!is_valid_cache(val))
1862 return -ENOENT;
1863
1864 if (get_user(newval, uval))
1865 return -EFAULT;
1866
1867 /* This is also invariant: you can't change it. */
1868 if (newval != get_ccsidr(val))
1869 return -EINVAL;
1870 return 0;
1871 default:
1872 return -ENOENT;
1873 }
1874}
1875
1876int kvm_arm_sys_reg_get_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg)
1877{
1878 const struct sys_reg_desc *r;
1879 void __user *uaddr = (void __user *)(unsigned long)reg->addr;
1880
1881 if ((reg->id & KVM_REG_ARM_COPROC_MASK) == KVM_REG_ARM_DEMUX)
1882 return demux_c15_get(reg->id, uaddr);
1883
1884 if (KVM_REG_SIZE(reg->id) != sizeof(__u64))
1885 return -ENOENT;
1886
1887 r = index_to_sys_reg_desc(vcpu, reg->id);
1888 if (!r)
1889 return get_invariant_sys_reg(reg->id, uaddr);
1890
84e690bf
AB		 1891 if (r->get_user)
1892 return (r->get_user)(vcpu, r, reg, uaddr);
1893
7c8c5e6a
MZ		 1894 return reg_to_user(uaddr, &vcpu_sys_reg(vcpu, r->reg), reg->id);
1895}
1896
1897int kvm_arm_sys_reg_set_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg)
1898{
1899 const struct sys_reg_desc *r;
1900 void __user *uaddr = (void __user *)(unsigned long)reg->addr;
1901
1902 if ((reg->id & KVM_REG_ARM_COPROC_MASK) == KVM_REG_ARM_DEMUX)
1903 return demux_c15_set(reg->id, uaddr);
1904
1905 if (KVM_REG_SIZE(reg->id) != sizeof(__u64))
1906 return -ENOENT;
1907
1908 r = index_to_sys_reg_desc(vcpu, reg->id);
1909 if (!r)
1910 return set_invariant_sys_reg(reg->id, uaddr);
1911
84e690bf
AB		 1912 if (r->set_user)
1913 return (r->set_user)(vcpu, r, reg, uaddr);
1914
7c8c5e6a
MZ		 1915 return reg_from_user(&vcpu_sys_reg(vcpu, r->reg), uaddr, reg->id);
1916}
1917
1918static unsigned int num_demux_regs(void)
1919{
1920 unsigned int i, count = 0;
1921
1922 for (i = 0; i < CSSELR_MAX; i++)
1923 if (is_valid_cache(i))
1924 count++;
1925
1926 return count;
1927}
1928
1929static int write_demux_regids(u64 __user *uindices)
1930{
efd48cea 1931 u64 val = KVM_REG_ARM64 | KVM_REG_SIZE_U32 | KVM_REG_ARM_DEMUX;
7c8c5e6a
MZ		 1932 unsigned int i;
1933
1934 val |= KVM_REG_ARM_DEMUX_ID_CCSIDR;
1935 for (i = 0; i < CSSELR_MAX; i++) {
1936 if (!is_valid_cache(i))
1937 continue;
1938 if (put_user(val | i, uindices))
1939 return -EFAULT;
1940 uindices++;
1941 }
1942 return 0;
1943}
1944
1945static u64 sys_reg_to_index(const struct sys_reg_desc *reg)
1946{
1947 return (KVM_REG_ARM64 | KVM_REG_SIZE_U64 |
1948 KVM_REG_ARM64_SYSREG |
1949 (reg->Op0 << KVM_REG_ARM64_SYSREG_OP0_SHIFT) |
1950 (reg->Op1 << KVM_REG_ARM64_SYSREG_OP1_SHIFT) |
1951 (reg->CRn << KVM_REG_ARM64_SYSREG_CRN_SHIFT) |
1952 (reg->CRm << KVM_REG_ARM64_SYSREG_CRM_SHIFT) |
1953 (reg->Op2 << KVM_REG_ARM64_SYSREG_OP2_SHIFT));
1954}
1955
1956static bool copy_reg_to_user(const struct sys_reg_desc *reg, u64 __user **uind)
1957{
1958 if (!*uind)
1959 return true;
1960
1961 if (put_user(sys_reg_to_index(reg), *uind))
1962 return false;
1963
1964 (*uind)++;
1965 return true;
1966}
1967
1968/* Assumed ordered tables, see kvm_sys_reg_table_init. */
1969static int walk_sys_regs(struct kvm_vcpu *vcpu, u64 __user *uind)
1970{
1971 const struct sys_reg_desc *i1, *i2, *end1, *end2;
1972 unsigned int total = 0;
1973 size_t num;
1974
1975 /* We check for duplicates here, to allow arch-specific overrides. */
62a89c44 1976 i1 = get_target_table(vcpu->arch.target, true, &num);
7c8c5e6a
MZ		 1977 end1 = i1 + num;
1978 i2 = sys_reg_descs;
1979 end2 = sys_reg_descs + ARRAY_SIZE(sys_reg_descs);
1980
1981 BUG_ON(i1 == end1 || i2 == end2);
1982
1983 /* Walk carefully, as both tables may refer to the same register. */
1984 while (i1 || i2) {
1985 int cmp = cmp_sys_reg(i1, i2);
1986 /* target-specific overrides generic entry. */
1987 if (cmp <= 0) {
1988 /* Ignore registers we trap but don't save. */
1989 if (i1->reg) {
1990 if (!copy_reg_to_user(i1, &uind))
1991 return -EFAULT;
1992 total++;
1993 }
1994 } else {
1995 /* Ignore registers we trap but don't save. */
1996 if (i2->reg) {
1997 if (!copy_reg_to_user(i2, &uind))
1998 return -EFAULT;
1999 total++;
2000 }
2001 }
2002
2003 if (cmp <= 0 && ++i1 == end1)
2004 i1 = NULL;
2005 if (cmp >= 0 && ++i2 == end2)
2006 i2 = NULL;
2007 }
2008 return total;
2009}
2010
2011unsigned long kvm_arm_num_sys_reg_descs(struct kvm_vcpu *vcpu)
2012{
2013 return ARRAY_SIZE(invariant_sys_regs)
2014 + num_demux_regs()
2015 + walk_sys_regs(vcpu, (u64 __user *)NULL);
2016}
2017
2018int kvm_arm_copy_sys_reg_indices(struct kvm_vcpu *vcpu, u64 __user *uindices)
2019{
2020 unsigned int i;
2021 int err;
2022
2023 /* Then give them all the invariant registers' indices. */
2024 for (i = 0; i < ARRAY_SIZE(invariant_sys_regs); i++) {
2025 if (put_user(sys_reg_to_index(&invariant_sys_regs[i]), uindices))
2026 return -EFAULT;
2027 uindices++;
2028 }
2029
2030 err = walk_sys_regs(vcpu, uindices);
2031 if (err < 0)
2032 return err;
2033 uindices += err;
2034
2035 return write_demux_regids(uindices);
2036}
2037
e6a95517
MZ		 2038static int check_sysreg_table(const struct sys_reg_desc *table, unsigned int n)
2039{
2040 unsigned int i;
2041
2042 for (i = 1; i < n; i++) {
2043 if (cmp_sys_reg(&table[i-1], &table[i]) >= 0) {
2044 kvm_err("sys_reg table %p out of order (%d)\n", table, i - 1);
2045 return 1;
2046 }
2047 }
2048
2049 return 0;
2050}
2051
7c8c5e6a
MZ		 2052void kvm_sys_reg_table_init(void)
2053{
2054 unsigned int i;
2055 struct sys_reg_desc clidr;
2056
2057 /* Make sure tables are unique and in order. */
e6a95517
MZ		 2058 BUG_ON(check_sysreg_table(sys_reg_descs, ARRAY_SIZE(sys_reg_descs)));
2059 BUG_ON(check_sysreg_table(cp14_regs, ARRAY_SIZE(cp14_regs)));
2060 BUG_ON(check_sysreg_table(cp14_64_regs, ARRAY_SIZE(cp14_64_regs)));
2061 BUG_ON(check_sysreg_table(cp15_regs, ARRAY_SIZE(cp15_regs)));
2062 BUG_ON(check_sysreg_table(cp15_64_regs, ARRAY_SIZE(cp15_64_regs)));
2063 BUG_ON(check_sysreg_table(invariant_sys_regs, ARRAY_SIZE(invariant_sys_regs)));
7c8c5e6a
MZ		 2064
2065 /* We abuse the reset function to overwrite the table itself. */
2066 for (i = 0; i < ARRAY_SIZE(invariant_sys_regs); i++)
2067 invariant_sys_regs[i].reset(NULL, &invariant_sys_regs[i]);
2068
2069 /*
2070 * CLIDR format is awkward, so clean it up. See ARM B4.1.20:
2071 *
2072 * If software reads the Cache Type fields from Ctype1
2073 * upwards, once it has seen a value of 0b000, no caches
2074 * exist at further-out levels of the hierarchy. So, for
2075 * example, if Ctype3 is the first Cache Type field with a
2076 * value of 0b000, the values of Ctype4 to Ctype7 must be
2077 * ignored.
2078 */
2079 get_clidr_el1(NULL, &clidr); /* Ugly... */
2080 cache_levels = clidr.val;
2081 for (i = 0; i < 7; i++)
2082 if (((cache_levels >> (i*3)) & 7) == 0)
2083 break;
2084 /* Clear all higher bits. */
2085 cache_levels &= (1 << (i*3))-1;
2086}
2087
2088/**
2089 * kvm_reset_sys_regs - sets system registers to reset value
2090 * @vcpu: The VCPU pointer
2091 *
2092 * This function finds the right table above and sets the registers on the
2093 * virtual CPU struct to their architecturally defined reset values.
2094 */
2095void kvm_reset_sys_regs(struct kvm_vcpu *vcpu)
2096{
2097 size_t num;
2098 const struct sys_reg_desc *table;
2099
2100 /* Catch someone adding a register without putting in reset entry. */
2101 memset(&vcpu->arch.ctxt.sys_regs, 0x42, sizeof(vcpu->arch.ctxt.sys_regs));
2102
2103 /* Generic chip reset first (so target could override). */
2104 reset_sys_reg_descs(vcpu, sys_reg_descs, ARRAY_SIZE(sys_reg_descs));
2105
62a89c44 2106 table = get_target_table(vcpu->arch.target, true, &num);
7c8c5e6a
MZ		 2107 reset_sys_reg_descs(vcpu, table, num);
2108
2109 for (num = 1; num < NR_SYS_REGS; num++)
2110 if (vcpu_sys_reg(vcpu, num) == 0x4242424242424242)
2111 panic("Didn't reset vcpu_sys_reg(%zi)", num);
2112}
Ubuntu Artful kernel mirror