]>
Commit | Line | Data |
---|---|---|
26861c7c MH |
1 | /* |
2 | * ARM implementation of KVM hooks, 64 bit specific code | |
3 | * | |
4 | * Copyright Mian-M. Hamayun 2013, Virtual Open Systems | |
e4482ab7 | 5 | * Copyright Alex Bennée 2014, Linaro |
26861c7c MH |
6 | * |
7 | * This work is licensed under the terms of the GNU GPL, version 2 or later. | |
8 | * See the COPYING file in the top-level directory. | |
9 | * | |
10 | */ | |
11 | ||
74c21bd0 | 12 | #include "qemu/osdep.h" |
26861c7c | 13 | #include <sys/ioctl.h> |
e4482ab7 | 14 | #include <sys/ptrace.h> |
26861c7c | 15 | |
e4482ab7 | 16 | #include <linux/elf.h> |
26861c7c MH |
17 | #include <linux/kvm.h> |
18 | ||
68970d1e | 19 | #include "qapi/error.h" |
33c11879 | 20 | #include "cpu.h" |
26861c7c | 21 | #include "qemu/timer.h" |
2ecb2027 | 22 | #include "qemu/error-report.h" |
e4482ab7 | 23 | #include "qemu/host-utils.h" |
db725815 | 24 | #include "qemu/main-loop.h" |
e4482ab7 | 25 | #include "exec/gdbstub.h" |
e5ac4200 | 26 | #include "sysemu/runstate.h" |
26861c7c | 27 | #include "sysemu/kvm.h" |
b9e758f0 | 28 | #include "sysemu/kvm_int.h" |
26861c7c | 29 | #include "kvm_arm.h" |
9208b961 | 30 | #include "internals.h" |
e24fd076 DG |
31 | #include "hw/acpi/acpi.h" |
32 | #include "hw/acpi/ghes.h" | |
33 | #include "hw/arm/virt.h" | |
26861c7c | 34 | |
29eb3d9a AB |
35 | static bool have_guest_debug; |
36 | ||
e4482ab7 AB |
37 | /* |
38 | * Although the ARM implementation of hardware assisted debugging | |
39 | * allows for different breakpoints per-core, the current GDB | |
40 | * interface treats them as a global pool of registers (which seems to | |
41 | * be the case for x86, ppc and s390). As a result we store one copy | |
42 | * of registers which is used for all active cores. | |
43 | * | |
44 | * Write access is serialised by virtue of the GDB protocol which | |
45 | * updates things. Read access (i.e. when the values are copied to the | |
46 | * vCPU) is also gated by GDB's run control. | |
47 | * | |
48 | * This is not unreasonable as most of the time debugging kernels you | |
49 | * never know which core will eventually execute your function. | |
50 | */ | |
51 | ||
52 | typedef struct { | |
53 | uint64_t bcr; | |
54 | uint64_t bvr; | |
55 | } HWBreakpoint; | |
56 | ||
57 | /* The watchpoint registers can cover more area than the requested | |
58 | * watchpoint so we need to store the additional information | |
59 | * somewhere. We also need to supply a CPUWatchpoint to the GDB stub | |
60 | * when the watchpoint is hit. | |
61 | */ | |
62 | typedef struct { | |
63 | uint64_t wcr; | |
64 | uint64_t wvr; | |
65 | CPUWatchpoint details; | |
66 | } HWWatchpoint; | |
67 | ||
68 | /* Maximum and current break/watch point counts */ | |
69 | int max_hw_bps, max_hw_wps; | |
70 | GArray *hw_breakpoints, *hw_watchpoints; | |
71 | ||
72 | #define cur_hw_wps (hw_watchpoints->len) | |
73 | #define cur_hw_bps (hw_breakpoints->len) | |
74 | #define get_hw_bp(i) (&g_array_index(hw_breakpoints, HWBreakpoint, i)) | |
75 | #define get_hw_wp(i) (&g_array_index(hw_watchpoints, HWWatchpoint, i)) | |
76 | ||
ad5c6dde | 77 | void kvm_arm_init_debug(KVMState *s) |
29eb3d9a | 78 | { |
ad5c6dde | 79 | have_guest_debug = kvm_check_extension(s, |
29eb3d9a | 80 | KVM_CAP_SET_GUEST_DEBUG); |
e4482ab7 | 81 | |
ad5c6dde | 82 | max_hw_wps = kvm_check_extension(s, KVM_CAP_GUEST_DEBUG_HW_WPS); |
e4482ab7 AB |
83 | hw_watchpoints = g_array_sized_new(true, true, |
84 | sizeof(HWWatchpoint), max_hw_wps); | |
85 | ||
ad5c6dde | 86 | max_hw_bps = kvm_check_extension(s, KVM_CAP_GUEST_DEBUG_HW_BPS); |
e4482ab7 AB |
87 | hw_breakpoints = g_array_sized_new(true, true, |
88 | sizeof(HWBreakpoint), max_hw_bps); | |
29eb3d9a AB |
89 | return; |
90 | } | |
91 | ||
e4482ab7 AB |
92 | /** |
93 | * insert_hw_breakpoint() | |
94 | * @addr: address of breakpoint | |
95 | * | |
96 | * See ARM ARM D2.9.1 for details but here we are only going to create | |
97 | * simple un-linked breakpoints (i.e. we don't chain breakpoints | |
98 | * together to match address and context or vmid). The hardware is | |
99 | * capable of fancier matching but that will require exposing that | |
100 | * fanciness to GDB's interface | |
101 | * | |
864df205 | 102 | * DBGBCR<n>_EL1, Debug Breakpoint Control Registers |
e4482ab7 AB |
103 | * |
104 | * 31 24 23 20 19 16 15 14 13 12 9 8 5 4 3 2 1 0 | |
105 | * +------+------+-------+-----+----+------+-----+------+-----+---+ | |
106 | * | RES0 | BT | LBN | SSC | HMC| RES0 | BAS | RES0 | PMC | E | | |
107 | * +------+------+-------+-----+----+------+-----+------+-----+---+ | |
108 | * | |
109 | * BT: Breakpoint type (0 = unlinked address match) | |
110 | * LBN: Linked BP number (0 = unused) | |
111 | * SSC/HMC/PMC: Security, Higher and Priv access control (Table D-12) | |
112 | * BAS: Byte Address Select (RES1 for AArch64) | |
113 | * E: Enable bit | |
864df205 AB |
114 | * |
115 | * DBGBVR<n>_EL1, Debug Breakpoint Value Registers | |
116 | * | |
117 | * 63 53 52 49 48 2 1 0 | |
118 | * +------+-----------+----------+-----+ | |
119 | * | RESS | VA[52:49] | VA[48:2] | 0 0 | | |
120 | * +------+-----------+----------+-----+ | |
121 | * | |
122 | * Depending on the addressing mode bits the top bits of the register | |
123 | * are a sign extension of the highest applicable VA bit. Some | |
124 | * versions of GDB don't do it correctly so we ensure they are correct | |
125 | * here so future PC comparisons will work properly. | |
e4482ab7 | 126 | */ |
864df205 | 127 | |
e4482ab7 AB |
128 | static int insert_hw_breakpoint(target_ulong addr) |
129 | { | |
130 | HWBreakpoint brk = { | |
131 | .bcr = 0x1, /* BCR E=1, enable */ | |
864df205 | 132 | .bvr = sextract64(addr, 0, 53) |
e4482ab7 AB |
133 | }; |
134 | ||
135 | if (cur_hw_bps >= max_hw_bps) { | |
136 | return -ENOBUFS; | |
137 | } | |
138 | ||
139 | brk.bcr = deposit32(brk.bcr, 1, 2, 0x3); /* PMC = 11 */ | |
140 | brk.bcr = deposit32(brk.bcr, 5, 4, 0xf); /* BAS = RES1 */ | |
141 | ||
142 | g_array_append_val(hw_breakpoints, brk); | |
143 | ||
144 | return 0; | |
145 | } | |
146 | ||
147 | /** | |
148 | * delete_hw_breakpoint() | |
149 | * @pc: address of breakpoint | |
150 | * | |
151 | * Delete a breakpoint and shuffle any above down | |
152 | */ | |
153 | ||
154 | static int delete_hw_breakpoint(target_ulong pc) | |
155 | { | |
156 | int i; | |
157 | for (i = 0; i < hw_breakpoints->len; i++) { | |
158 | HWBreakpoint *brk = get_hw_bp(i); | |
159 | if (brk->bvr == pc) { | |
160 | g_array_remove_index(hw_breakpoints, i); | |
161 | return 0; | |
162 | } | |
163 | } | |
164 | return -ENOENT; | |
165 | } | |
166 | ||
167 | /** | |
168 | * insert_hw_watchpoint() | |
169 | * @addr: address of watch point | |
170 | * @len: size of area | |
171 | * @type: type of watch point | |
172 | * | |
173 | * See ARM ARM D2.10. As with the breakpoints we can do some advanced | |
174 | * stuff if we want to. The watch points can be linked with the break | |
175 | * points above to make them context aware. However for simplicity | |
176 | * currently we only deal with simple read/write watch points. | |
177 | * | |
178 | * D7.3.11 DBGWCR<n>_EL1, Debug Watchpoint Control Registers | |
179 | * | |
180 | * 31 29 28 24 23 21 20 19 16 15 14 13 12 5 4 3 2 1 0 | |
181 | * +------+-------+------+----+-----+-----+-----+-----+-----+-----+---+ | |
182 | * | RES0 | MASK | RES0 | WT | LBN | SSC | HMC | BAS | LSC | PAC | E | | |
183 | * +------+-------+------+----+-----+-----+-----+-----+-----+-----+---+ | |
184 | * | |
185 | * MASK: num bits addr mask (0=none,01/10=res,11=3 bits (8 bytes)) | |
186 | * WT: 0 - unlinked, 1 - linked (not currently used) | |
187 | * LBN: Linked BP number (not currently used) | |
188 | * SSC/HMC/PAC: Security, Higher and Priv access control (Table D2-11) | |
189 | * BAS: Byte Address Select | |
190 | * LSC: Load/Store control (01: load, 10: store, 11: both) | |
191 | * E: Enable | |
192 | * | |
193 | * The bottom 2 bits of the value register are masked. Therefore to | |
194 | * break on any sizes smaller than an unaligned word you need to set | |
195 | * MASK=0, BAS=bit per byte in question. For larger regions (^2) you | |
196 | * need to ensure you mask the address as required and set BAS=0xff | |
197 | */ | |
198 | ||
199 | static int insert_hw_watchpoint(target_ulong addr, | |
200 | target_ulong len, int type) | |
201 | { | |
202 | HWWatchpoint wp = { | |
8b7a5bbe | 203 | .wcr = R_DBGWCR_E_MASK, /* E=1, enable */ |
e4482ab7 AB |
204 | .wvr = addr & (~0x7ULL), |
205 | .details = { .vaddr = addr, .len = len } | |
206 | }; | |
207 | ||
208 | if (cur_hw_wps >= max_hw_wps) { | |
209 | return -ENOBUFS; | |
210 | } | |
211 | ||
212 | /* | |
213 | * HMC=0 SSC=0 PAC=3 will hit EL0 or EL1, any security state, | |
214 | * valid whether EL3 is implemented or not | |
215 | */ | |
8b7a5bbe | 216 | wp.wcr = FIELD_DP64(wp.wcr, DBGWCR, PAC, 3); |
e4482ab7 AB |
217 | |
218 | switch (type) { | |
219 | case GDB_WATCHPOINT_READ: | |
8b7a5bbe | 220 | wp.wcr = FIELD_DP64(wp.wcr, DBGWCR, LSC, 1); |
e4482ab7 AB |
221 | wp.details.flags = BP_MEM_READ; |
222 | break; | |
223 | case GDB_WATCHPOINT_WRITE: | |
8b7a5bbe | 224 | wp.wcr = FIELD_DP64(wp.wcr, DBGWCR, LSC, 2); |
e4482ab7 AB |
225 | wp.details.flags = BP_MEM_WRITE; |
226 | break; | |
227 | case GDB_WATCHPOINT_ACCESS: | |
8b7a5bbe | 228 | wp.wcr = FIELD_DP64(wp.wcr, DBGWCR, LSC, 3); |
e4482ab7 AB |
229 | wp.details.flags = BP_MEM_ACCESS; |
230 | break; | |
231 | default: | |
232 | g_assert_not_reached(); | |
233 | break; | |
234 | } | |
235 | if (len <= 8) { | |
236 | /* we align the address and set the bits in BAS */ | |
237 | int off = addr & 0x7; | |
238 | int bas = (1 << len) - 1; | |
239 | ||
240 | wp.wcr = deposit32(wp.wcr, 5 + off, 8 - off, bas); | |
241 | } else { | |
242 | /* For ranges above 8 bytes we need to be a power of 2 */ | |
243 | if (is_power_of_2(len)) { | |
244 | int bits = ctz64(len); | |
245 | ||
246 | wp.wvr &= ~((1 << bits) - 1); | |
8b7a5bbe RH |
247 | wp.wcr = FIELD_DP64(wp.wcr, DBGWCR, MASK, bits); |
248 | wp.wcr = FIELD_DP64(wp.wcr, DBGWCR, BAS, 0xff); | |
e4482ab7 AB |
249 | } else { |
250 | return -ENOBUFS; | |
251 | } | |
252 | } | |
253 | ||
254 | g_array_append_val(hw_watchpoints, wp); | |
255 | return 0; | |
256 | } | |
257 | ||
258 | ||
259 | static bool check_watchpoint_in_range(int i, target_ulong addr) | |
260 | { | |
261 | HWWatchpoint *wp = get_hw_wp(i); | |
262 | uint64_t addr_top, addr_bottom = wp->wvr; | |
263 | int bas = extract32(wp->wcr, 5, 8); | |
264 | int mask = extract32(wp->wcr, 24, 4); | |
265 | ||
266 | if (mask) { | |
267 | addr_top = addr_bottom + (1 << mask); | |
268 | } else { | |
269 | /* BAS must be contiguous but can offset against the base | |
270 | * address in DBGWVR */ | |
271 | addr_bottom = addr_bottom + ctz32(bas); | |
272 | addr_top = addr_bottom + clo32(bas); | |
273 | } | |
274 | ||
275 | if (addr >= addr_bottom && addr <= addr_top) { | |
276 | return true; | |
277 | } | |
278 | ||
279 | return false; | |
280 | } | |
281 | ||
282 | /** | |
283 | * delete_hw_watchpoint() | |
284 | * @addr: address of breakpoint | |
285 | * | |
286 | * Delete a breakpoint and shuffle any above down | |
287 | */ | |
288 | ||
289 | static int delete_hw_watchpoint(target_ulong addr, | |
290 | target_ulong len, int type) | |
291 | { | |
292 | int i; | |
293 | for (i = 0; i < cur_hw_wps; i++) { | |
294 | if (check_watchpoint_in_range(i, addr)) { | |
295 | g_array_remove_index(hw_watchpoints, i); | |
296 | return 0; | |
297 | } | |
298 | } | |
299 | return -ENOENT; | |
300 | } | |
301 | ||
302 | ||
303 | int kvm_arch_insert_hw_breakpoint(target_ulong addr, | |
304 | target_ulong len, int type) | |
305 | { | |
306 | switch (type) { | |
307 | case GDB_BREAKPOINT_HW: | |
308 | return insert_hw_breakpoint(addr); | |
309 | break; | |
310 | case GDB_WATCHPOINT_READ: | |
311 | case GDB_WATCHPOINT_WRITE: | |
312 | case GDB_WATCHPOINT_ACCESS: | |
313 | return insert_hw_watchpoint(addr, len, type); | |
314 | default: | |
315 | return -ENOSYS; | |
316 | } | |
317 | } | |
318 | ||
319 | int kvm_arch_remove_hw_breakpoint(target_ulong addr, | |
320 | target_ulong len, int type) | |
321 | { | |
322 | switch (type) { | |
323 | case GDB_BREAKPOINT_HW: | |
324 | return delete_hw_breakpoint(addr); | |
e4482ab7 AB |
325 | case GDB_WATCHPOINT_READ: |
326 | case GDB_WATCHPOINT_WRITE: | |
327 | case GDB_WATCHPOINT_ACCESS: | |
328 | return delete_hw_watchpoint(addr, len, type); | |
329 | default: | |
330 | return -ENOSYS; | |
331 | } | |
332 | } | |
333 | ||
334 | ||
335 | void kvm_arch_remove_all_hw_breakpoints(void) | |
336 | { | |
337 | if (cur_hw_wps > 0) { | |
338 | g_array_remove_range(hw_watchpoints, 0, cur_hw_wps); | |
339 | } | |
340 | if (cur_hw_bps > 0) { | |
341 | g_array_remove_range(hw_breakpoints, 0, cur_hw_bps); | |
342 | } | |
343 | } | |
344 | ||
345 | void kvm_arm_copy_hw_debug_data(struct kvm_guest_debug_arch *ptr) | |
346 | { | |
347 | int i; | |
348 | memset(ptr, 0, sizeof(struct kvm_guest_debug_arch)); | |
349 | ||
350 | for (i = 0; i < max_hw_wps; i++) { | |
351 | HWWatchpoint *wp = get_hw_wp(i); | |
352 | ptr->dbg_wcr[i] = wp->wcr; | |
353 | ptr->dbg_wvr[i] = wp->wvr; | |
354 | } | |
355 | for (i = 0; i < max_hw_bps; i++) { | |
356 | HWBreakpoint *bp = get_hw_bp(i); | |
357 | ptr->dbg_bcr[i] = bp->bcr; | |
358 | ptr->dbg_bvr[i] = bp->bvr; | |
359 | } | |
360 | } | |
361 | ||
362 | bool kvm_arm_hw_debug_active(CPUState *cs) | |
363 | { | |
364 | return ((cur_hw_wps > 0) || (cur_hw_bps > 0)); | |
365 | } | |
366 | ||
367 | static bool find_hw_breakpoint(CPUState *cpu, target_ulong pc) | |
368 | { | |
369 | int i; | |
370 | ||
371 | for (i = 0; i < cur_hw_bps; i++) { | |
372 | HWBreakpoint *bp = get_hw_bp(i); | |
373 | if (bp->bvr == pc) { | |
374 | return true; | |
375 | } | |
376 | } | |
377 | return false; | |
378 | } | |
379 | ||
380 | static CPUWatchpoint *find_hw_watchpoint(CPUState *cpu, target_ulong addr) | |
381 | { | |
382 | int i; | |
383 | ||
384 | for (i = 0; i < cur_hw_wps; i++) { | |
385 | if (check_watchpoint_in_range(i, addr)) { | |
386 | return &get_hw_wp(i)->details; | |
387 | } | |
388 | } | |
389 | return NULL; | |
390 | } | |
391 | ||
68970d1e AJ |
392 | static bool kvm_arm_set_device_attr(CPUState *cs, struct kvm_device_attr *attr, |
393 | const char *name) | |
01fe6b60 SZ |
394 | { |
395 | int err; | |
396 | ||
3f07cb2a AJ |
397 | err = kvm_vcpu_ioctl(cs, KVM_HAS_DEVICE_ATTR, attr); |
398 | if (err != 0) { | |
68970d1e | 399 | error_report("%s: KVM_HAS_DEVICE_ATTR: %s", name, strerror(-err)); |
3f07cb2a | 400 | return false; |
01fe6b60 SZ |
401 | } |
402 | ||
3f07cb2a | 403 | err = kvm_vcpu_ioctl(cs, KVM_SET_DEVICE_ATTR, attr); |
b2bfe9f7 | 404 | if (err != 0) { |
68970d1e | 405 | error_report("%s: KVM_SET_DEVICE_ATTR: %s", name, strerror(-err)); |
b2bfe9f7 | 406 | return false; |
01fe6b60 SZ |
407 | } |
408 | ||
3f07cb2a AJ |
409 | return true; |
410 | } | |
01fe6b60 | 411 | |
b2bfe9f7 | 412 | void kvm_arm_pmu_init(CPUState *cs) |
3f07cb2a AJ |
413 | { |
414 | struct kvm_device_attr attr = { | |
415 | .group = KVM_ARM_VCPU_PMU_V3_CTRL, | |
416 | .attr = KVM_ARM_VCPU_PMU_V3_INIT, | |
417 | }; | |
418 | ||
b2bfe9f7 AJ |
419 | if (!ARM_CPU(cs)->has_pmu) { |
420 | return; | |
421 | } | |
68970d1e | 422 | if (!kvm_arm_set_device_attr(cs, &attr, "PMU")) { |
b2bfe9f7 AJ |
423 | error_report("failed to init PMU"); |
424 | abort(); | |
425 | } | |
3f07cb2a AJ |
426 | } |
427 | ||
b2bfe9f7 | 428 | void kvm_arm_pmu_set_irq(CPUState *cs, int irq) |
3f07cb2a AJ |
429 | { |
430 | struct kvm_device_attr attr = { | |
431 | .group = KVM_ARM_VCPU_PMU_V3_CTRL, | |
432 | .addr = (intptr_t)&irq, | |
433 | .attr = KVM_ARM_VCPU_PMU_V3_IRQ, | |
434 | }; | |
01fe6b60 | 435 | |
b2bfe9f7 AJ |
436 | if (!ARM_CPU(cs)->has_pmu) { |
437 | return; | |
438 | } | |
68970d1e | 439 | if (!kvm_arm_set_device_attr(cs, &attr, "PMU")) { |
b2bfe9f7 AJ |
440 | error_report("failed to set irq for PMU"); |
441 | abort(); | |
442 | } | |
01fe6b60 | 443 | } |
e4482ab7 | 444 | |
68970d1e AJ |
445 | void kvm_arm_pvtime_init(CPUState *cs, uint64_t ipa) |
446 | { | |
447 | struct kvm_device_attr attr = { | |
448 | .group = KVM_ARM_VCPU_PVTIME_CTRL, | |
449 | .attr = KVM_ARM_VCPU_PVTIME_IPA, | |
450 | .addr = (uint64_t)&ipa, | |
451 | }; | |
452 | ||
453 | if (ARM_CPU(cs)->kvm_steal_time == ON_OFF_AUTO_OFF) { | |
454 | return; | |
455 | } | |
456 | if (!kvm_arm_set_device_attr(cs, &attr, "PVTIME IPA")) { | |
457 | error_report("failed to init PVTIME IPA"); | |
458 | abort(); | |
459 | } | |
460 | } | |
461 | ||
9d60dea9 RH |
462 | static int read_sys_reg32(int fd, uint32_t *pret, uint64_t id) |
463 | { | |
464 | uint64_t ret; | |
465 | struct kvm_one_reg idreg = { .id = id, .addr = (uintptr_t)&ret }; | |
466 | int err; | |
467 | ||
468 | assert((id & KVM_REG_SIZE_MASK) == KVM_REG_SIZE_U64); | |
469 | err = ioctl(fd, KVM_GET_ONE_REG, &idreg); | |
470 | if (err < 0) { | |
471 | return -1; | |
472 | } | |
473 | *pret = ret; | |
474 | return 0; | |
475 | } | |
476 | ||
477 | static int read_sys_reg64(int fd, uint64_t *pret, uint64_t id) | |
478 | { | |
479 | struct kvm_one_reg idreg = { .id = id, .addr = (uintptr_t)pret }; | |
480 | ||
481 | assert((id & KVM_REG_SIZE_MASK) == KVM_REG_SIZE_U64); | |
482 | return ioctl(fd, KVM_GET_ONE_REG, &idreg); | |
483 | } | |
484 | ||
95ea96e8 MZ |
485 | static bool kvm_arm_pauth_supported(void) |
486 | { | |
487 | return (kvm_check_extension(kvm_state, KVM_CAP_ARM_PTRAUTH_ADDRESS) && | |
488 | kvm_check_extension(kvm_state, KVM_CAP_ARM_PTRAUTH_GENERIC)); | |
489 | } | |
490 | ||
c4487d76 | 491 | bool kvm_arm_get_host_cpu_features(ARMHostCPUFeatures *ahcf) |
26861c7c MH |
492 | { |
493 | /* Identify the feature bits corresponding to the host CPU, and | |
494 | * fill out the ARMHostCPUClass fields accordingly. To do this | |
495 | * we have to create a scratch VM, create a single CPU inside it, | |
496 | * and then query that CPU for the relevant ID registers. | |
26861c7c MH |
497 | */ |
498 | int fdarray[3]; | |
87014c6b | 499 | bool sve_supported; |
24526bb9 | 500 | bool pmu_supported = false; |
26861c7c | 501 | uint64_t features = 0; |
9d60dea9 RH |
502 | int err; |
503 | ||
26861c7c MH |
504 | /* Old kernels may not know about the PREFERRED_TARGET ioctl: however |
505 | * we know these will only support creating one kind of guest CPU, | |
506 | * which is its preferred CPU type. Fortunately these old kernels | |
507 | * support only a very limited number of CPUs. | |
508 | */ | |
509 | static const uint32_t cpus_to_try[] = { | |
510 | KVM_ARM_TARGET_AEM_V8, | |
511 | KVM_ARM_TARGET_FOUNDATION_V8, | |
512 | KVM_ARM_TARGET_CORTEX_A57, | |
513 | QEMU_KVM_ARM_TARGET_NONE | |
514 | }; | |
0cdb4020 AJ |
515 | /* |
516 | * target = -1 informs kvm_arm_create_scratch_host_vcpu() | |
517 | * to use the preferred target | |
518 | */ | |
519 | struct kvm_vcpu_init init = { .target = -1, }; | |
26861c7c | 520 | |
95ea96e8 | 521 | /* |
b9e8d68a RH |
522 | * Ask for SVE if supported, so that we can query ID_AA64ZFR0, |
523 | * which is otherwise RAZ. | |
524 | */ | |
525 | sve_supported = kvm_arm_sve_supported(); | |
526 | if (sve_supported) { | |
527 | init.features[0] |= 1 << KVM_ARM_VCPU_SVE; | |
528 | } | |
529 | ||
530 | /* | |
531 | * Ask for Pointer Authentication if supported, so that we get | |
532 | * the unsanitized field values for AA64ISAR1_EL1. | |
95ea96e8 MZ |
533 | */ |
534 | if (kvm_arm_pauth_supported()) { | |
535 | init.features[0] |= (1 << KVM_ARM_VCPU_PTRAUTH_ADDRESS | | |
536 | 1 << KVM_ARM_VCPU_PTRAUTH_GENERIC); | |
537 | } | |
538 | ||
24526bb9 PM |
539 | if (kvm_arm_pmu_supported()) { |
540 | init.features[0] |= 1 << KVM_ARM_VCPU_PMU_V3; | |
541 | pmu_supported = true; | |
542 | } | |
543 | ||
26861c7c MH |
544 | if (!kvm_arm_create_scratch_host_vcpu(cpus_to_try, fdarray, &init)) { |
545 | return false; | |
546 | } | |
547 | ||
c4487d76 PM |
548 | ahcf->target = init.target; |
549 | ahcf->dtb_compatible = "arm,arm-v8"; | |
26861c7c | 550 | |
9d60dea9 RH |
551 | err = read_sys_reg64(fdarray[2], &ahcf->isar.id_aa64pfr0, |
552 | ARM64_SYS_REG(3, 0, 0, 4, 0)); | |
553 | if (unlikely(err < 0)) { | |
554 | /* | |
555 | * Before v4.15, the kernel only exposed a limited number of system | |
556 | * registers, not including any of the interesting AArch64 ID regs. | |
557 | * For the most part we could leave these fields as zero with minimal | |
558 | * effect, since this does not affect the values seen by the guest. | |
559 | * | |
560 | * However, it could cause problems down the line for QEMU, | |
561 | * so provide a minimal v8.0 default. | |
562 | * | |
563 | * ??? Could read MIDR and use knowledge from cpu64.c. | |
564 | * ??? Could map a page of memory into our temp guest and | |
565 | * run the tiniest of hand-crafted kernels to extract | |
566 | * the values seen by the guest. | |
567 | * ??? Either of these sounds like too much effort just | |
568 | * to work around running a modern host kernel. | |
569 | */ | |
570 | ahcf->isar.id_aa64pfr0 = 0x00000011; /* EL1&0, AArch64 only */ | |
571 | err = 0; | |
572 | } else { | |
573 | err |= read_sys_reg64(fdarray[2], &ahcf->isar.id_aa64pfr1, | |
574 | ARM64_SYS_REG(3, 0, 0, 4, 1)); | |
414c54d5 RH |
575 | err |= read_sys_reg64(fdarray[2], &ahcf->isar.id_aa64smfr0, |
576 | ARM64_SYS_REG(3, 0, 0, 4, 5)); | |
1548a7b2 PM |
577 | err |= read_sys_reg64(fdarray[2], &ahcf->isar.id_aa64dfr0, |
578 | ARM64_SYS_REG(3, 0, 0, 5, 0)); | |
579 | err |= read_sys_reg64(fdarray[2], &ahcf->isar.id_aa64dfr1, | |
580 | ARM64_SYS_REG(3, 0, 0, 5, 1)); | |
9d60dea9 RH |
581 | err |= read_sys_reg64(fdarray[2], &ahcf->isar.id_aa64isar0, |
582 | ARM64_SYS_REG(3, 0, 0, 6, 0)); | |
583 | err |= read_sys_reg64(fdarray[2], &ahcf->isar.id_aa64isar1, | |
584 | ARM64_SYS_REG(3, 0, 0, 6, 1)); | |
3dc91ddb PM |
585 | err |= read_sys_reg64(fdarray[2], &ahcf->isar.id_aa64mmfr0, |
586 | ARM64_SYS_REG(3, 0, 0, 7, 0)); | |
587 | err |= read_sys_reg64(fdarray[2], &ahcf->isar.id_aa64mmfr1, | |
588 | ARM64_SYS_REG(3, 0, 0, 7, 1)); | |
64761e10 RH |
589 | err |= read_sys_reg64(fdarray[2], &ahcf->isar.id_aa64mmfr2, |
590 | ARM64_SYS_REG(3, 0, 0, 7, 2)); | |
9d60dea9 RH |
591 | |
592 | /* | |
593 | * Note that if AArch32 support is not present in the host, | |
594 | * the AArch32 sysregs are present to be read, but will | |
595 | * return UNKNOWN values. This is neither better nor worse | |
596 | * than skipping the reads and leaving 0, as we must avoid | |
597 | * considering the values in every case. | |
598 | */ | |
8a130a7b PM |
599 | err |= read_sys_reg32(fdarray[2], &ahcf->isar.id_pfr0, |
600 | ARM64_SYS_REG(3, 0, 0, 1, 0)); | |
601 | err |= read_sys_reg32(fdarray[2], &ahcf->isar.id_pfr1, | |
602 | ARM64_SYS_REG(3, 0, 0, 1, 1)); | |
1548a7b2 PM |
603 | err |= read_sys_reg32(fdarray[2], &ahcf->isar.id_dfr0, |
604 | ARM64_SYS_REG(3, 0, 0, 1, 2)); | |
10054016 PM |
605 | err |= read_sys_reg32(fdarray[2], &ahcf->isar.id_mmfr0, |
606 | ARM64_SYS_REG(3, 0, 0, 1, 4)); | |
607 | err |= read_sys_reg32(fdarray[2], &ahcf->isar.id_mmfr1, | |
608 | ARM64_SYS_REG(3, 0, 0, 1, 5)); | |
609 | err |= read_sys_reg32(fdarray[2], &ahcf->isar.id_mmfr2, | |
610 | ARM64_SYS_REG(3, 0, 0, 1, 6)); | |
611 | err |= read_sys_reg32(fdarray[2], &ahcf->isar.id_mmfr3, | |
612 | ARM64_SYS_REG(3, 0, 0, 1, 7)); | |
9d60dea9 RH |
613 | err |= read_sys_reg32(fdarray[2], &ahcf->isar.id_isar0, |
614 | ARM64_SYS_REG(3, 0, 0, 2, 0)); | |
615 | err |= read_sys_reg32(fdarray[2], &ahcf->isar.id_isar1, | |
616 | ARM64_SYS_REG(3, 0, 0, 2, 1)); | |
617 | err |= read_sys_reg32(fdarray[2], &ahcf->isar.id_isar2, | |
618 | ARM64_SYS_REG(3, 0, 0, 2, 2)); | |
619 | err |= read_sys_reg32(fdarray[2], &ahcf->isar.id_isar3, | |
620 | ARM64_SYS_REG(3, 0, 0, 2, 3)); | |
621 | err |= read_sys_reg32(fdarray[2], &ahcf->isar.id_isar4, | |
622 | ARM64_SYS_REG(3, 0, 0, 2, 4)); | |
623 | err |= read_sys_reg32(fdarray[2], &ahcf->isar.id_isar5, | |
624 | ARM64_SYS_REG(3, 0, 0, 2, 5)); | |
10054016 PM |
625 | err |= read_sys_reg32(fdarray[2], &ahcf->isar.id_mmfr4, |
626 | ARM64_SYS_REG(3, 0, 0, 2, 6)); | |
9d60dea9 RH |
627 | err |= read_sys_reg32(fdarray[2], &ahcf->isar.id_isar6, |
628 | ARM64_SYS_REG(3, 0, 0, 2, 7)); | |
629 | ||
630 | err |= read_sys_reg32(fdarray[2], &ahcf->isar.mvfr0, | |
631 | ARM64_SYS_REG(3, 0, 0, 3, 0)); | |
632 | err |= read_sys_reg32(fdarray[2], &ahcf->isar.mvfr1, | |
633 | ARM64_SYS_REG(3, 0, 0, 3, 1)); | |
634 | err |= read_sys_reg32(fdarray[2], &ahcf->isar.mvfr2, | |
635 | ARM64_SYS_REG(3, 0, 0, 3, 2)); | |
62b6f5e2 PM |
636 | err |= read_sys_reg32(fdarray[2], &ahcf->isar.id_pfr2, |
637 | ARM64_SYS_REG(3, 0, 0, 3, 4)); | |
d22c5649 PM |
638 | err |= read_sys_reg32(fdarray[2], &ahcf->isar.id_dfr1, |
639 | ARM64_SYS_REG(3, 0, 0, 3, 5)); | |
32957aad PM |
640 | err |= read_sys_reg32(fdarray[2], &ahcf->isar.id_mmfr5, |
641 | ARM64_SYS_REG(3, 0, 0, 3, 6)); | |
1548a7b2 PM |
642 | |
643 | /* | |
644 | * DBGDIDR is a bit complicated because the kernel doesn't | |
645 | * provide an accessor for it in 64-bit mode, which is what this | |
646 | * scratch VM is in, and there's no architected "64-bit sysreg | |
647 | * which reads the same as the 32-bit register" the way there is | |
648 | * for other ID registers. Instead we synthesize a value from the | |
649 | * AArch64 ID_AA64DFR0, the same way the kernel code in | |
650 | * arch/arm64/kvm/sys_regs.c:trap_dbgidr() does. | |
651 | * We only do this if the CPU supports AArch32 at EL1. | |
652 | */ | |
653 | if (FIELD_EX32(ahcf->isar.id_aa64pfr0, ID_AA64PFR0, EL1) >= 2) { | |
654 | int wrps = FIELD_EX64(ahcf->isar.id_aa64dfr0, ID_AA64DFR0, WRPS); | |
655 | int brps = FIELD_EX64(ahcf->isar.id_aa64dfr0, ID_AA64DFR0, BRPS); | |
656 | int ctx_cmps = | |
657 | FIELD_EX64(ahcf->isar.id_aa64dfr0, ID_AA64DFR0, CTX_CMPS); | |
658 | int version = 6; /* ARMv8 debug architecture */ | |
659 | bool has_el3 = | |
660 | !!FIELD_EX32(ahcf->isar.id_aa64pfr0, ID_AA64PFR0, EL3); | |
661 | uint32_t dbgdidr = 0; | |
662 | ||
663 | dbgdidr = FIELD_DP32(dbgdidr, DBGDIDR, WRPS, wrps); | |
664 | dbgdidr = FIELD_DP32(dbgdidr, DBGDIDR, BRPS, brps); | |
665 | dbgdidr = FIELD_DP32(dbgdidr, DBGDIDR, CTX_CMPS, ctx_cmps); | |
666 | dbgdidr = FIELD_DP32(dbgdidr, DBGDIDR, VERSION, version); | |
667 | dbgdidr = FIELD_DP32(dbgdidr, DBGDIDR, NSUHD_IMP, has_el3); | |
668 | dbgdidr = FIELD_DP32(dbgdidr, DBGDIDR, SE_IMP, has_el3); | |
669 | dbgdidr |= (1 << 15); /* RES1 bit */ | |
670 | ahcf->isar.dbgdidr = dbgdidr; | |
671 | } | |
24526bb9 PM |
672 | |
673 | if (pmu_supported) { | |
674 | /* PMCR_EL0 is only accessible if the vCPU has feature PMU_V3 */ | |
675 | err |= read_sys_reg64(fdarray[2], &ahcf->isar.reset_pmcr_el0, | |
676 | ARM64_SYS_REG(3, 3, 9, 12, 0)); | |
677 | } | |
9d60dea9 | 678 | |
5265d24c RH |
679 | if (sve_supported) { |
680 | /* | |
681 | * There is a range of kernels between kernel commit 73433762fcae | |
682 | * and f81cb2c3ad41 which have a bug where the kernel doesn't | |
683 | * expose SYS_ID_AA64ZFR0_EL1 via the ONE_REG API unless the VM has | |
684 | * enabled SVE support, which resulted in an error rather than RAZ. | |
685 | * So only read the register if we set KVM_ARM_VCPU_SVE above. | |
686 | */ | |
687 | err |= read_sys_reg64(fdarray[2], &ahcf->isar.id_aa64zfr0, | |
688 | ARM64_SYS_REG(3, 0, 0, 4, 4)); | |
689 | } | |
2dc10fa2 RH |
690 | } |
691 | ||
692 | kvm_arm_destroy_scratch_host_vcpu(fdarray); | |
693 | ||
694 | if (err < 0) { | |
695 | return false; | |
87014c6b AJ |
696 | } |
697 | ||
698 | /* | |
699 | * We can assume any KVM supporting CPU is at least a v8 | |
26861c7c MH |
700 | * with VFPv4+Neon; this in turn implies most of the other |
701 | * feature bits. | |
702 | */ | |
f5cbb280 PMD |
703 | features |= 1ULL << ARM_FEATURE_V8; |
704 | features |= 1ULL << ARM_FEATURE_NEON; | |
705 | features |= 1ULL << ARM_FEATURE_AARCH64; | |
706 | features |= 1ULL << ARM_FEATURE_PMU; | |
707 | features |= 1ULL << ARM_FEATURE_GENERIC_TIMER; | |
26861c7c | 708 | |
c4487d76 | 709 | ahcf->features = features; |
26861c7c MH |
710 | |
711 | return true; | |
712 | } | |
713 | ||
68970d1e AJ |
714 | void kvm_arm_steal_time_finalize(ARMCPU *cpu, Error **errp) |
715 | { | |
716 | bool has_steal_time = kvm_arm_steal_time_supported(); | |
717 | ||
718 | if (cpu->kvm_steal_time == ON_OFF_AUTO_AUTO) { | |
719 | if (!has_steal_time || !arm_feature(&cpu->env, ARM_FEATURE_AARCH64)) { | |
720 | cpu->kvm_steal_time = ON_OFF_AUTO_OFF; | |
721 | } else { | |
722 | cpu->kvm_steal_time = ON_OFF_AUTO_ON; | |
723 | } | |
724 | } else if (cpu->kvm_steal_time == ON_OFF_AUTO_ON) { | |
725 | if (!has_steal_time) { | |
726 | error_setg(errp, "'kvm-steal-time' cannot be enabled " | |
727 | "on this host"); | |
728 | return; | |
729 | } else if (!arm_feature(&cpu->env, ARM_FEATURE_AARCH64)) { | |
730 | /* | |
731 | * DEN0057A chapter 2 says "This specification only covers | |
732 | * systems in which the Execution state of the hypervisor | |
733 | * as well as EL1 of virtual machines is AArch64.". And, | |
734 | * to ensure that, the smc/hvc calls are only specified as | |
735 | * smc64/hvc64. | |
736 | */ | |
737 | error_setg(errp, "'kvm-steal-time' cannot be enabled " | |
738 | "for AArch32 guests"); | |
739 | return; | |
740 | } | |
741 | } | |
742 | } | |
743 | ||
7d20e681 | 744 | bool kvm_arm_aarch32_supported(void) |
b9e758f0 | 745 | { |
7d20e681 | 746 | return kvm_check_extension(kvm_state, KVM_CAP_ARM_EL1_32BIT); |
b9e758f0 AJ |
747 | } |
748 | ||
7d20e681 | 749 | bool kvm_arm_sve_supported(void) |
14e99e0f | 750 | { |
7d20e681 | 751 | return kvm_check_extension(kvm_state, KVM_CAP_ARM_SVE); |
14e99e0f AJ |
752 | } |
753 | ||
68970d1e AJ |
754 | bool kvm_arm_steal_time_supported(void) |
755 | { | |
756 | return kvm_check_extension(kvm_state, KVM_CAP_STEAL_TIME); | |
757 | } | |
758 | ||
b320e21c CH |
759 | bool kvm_arm_mte_supported(void) |
760 | { | |
761 | return kvm_check_extension(kvm_state, KVM_CAP_ARM_MTE); | |
762 | } | |
763 | ||
6fa8a379 AJ |
764 | QEMU_BUILD_BUG_ON(KVM_ARM64_SVE_VQ_MIN != 1); |
765 | ||
886902ec | 766 | uint32_t kvm_arm_sve_get_vls(CPUState *cs) |
6fa8a379 AJ |
767 | { |
768 | /* Only call this function if kvm_arm_sve_supported() returns true. */ | |
769 | static uint64_t vls[KVM_ARM64_SVE_VLS_WORDS]; | |
770 | static bool probed; | |
771 | uint32_t vq = 0; | |
886902ec | 772 | int i; |
6fa8a379 AJ |
773 | |
774 | /* | |
775 | * KVM ensures all host CPUs support the same set of vector lengths. | |
776 | * So we only need to create the scratch VCPUs once and then cache | |
777 | * the results. | |
778 | */ | |
779 | if (!probed) { | |
780 | struct kvm_vcpu_init init = { | |
781 | .target = -1, | |
782 | .features[0] = (1 << KVM_ARM_VCPU_SVE), | |
783 | }; | |
784 | struct kvm_one_reg reg = { | |
785 | .id = KVM_REG_ARM64_SVE_VLS, | |
786 | .addr = (uint64_t)&vls[0], | |
787 | }; | |
788 | int fdarray[3], ret; | |
789 | ||
790 | probed = true; | |
791 | ||
792 | if (!kvm_arm_create_scratch_host_vcpu(NULL, fdarray, &init)) { | |
793 | error_report("failed to create scratch VCPU with SVE enabled"); | |
794 | abort(); | |
795 | } | |
796 | ret = ioctl(fdarray[2], KVM_GET_ONE_REG, ®); | |
797 | kvm_arm_destroy_scratch_host_vcpu(fdarray); | |
798 | if (ret) { | |
799 | error_report("failed to get KVM_REG_ARM64_SVE_VLS: %s", | |
800 | strerror(errno)); | |
801 | abort(); | |
802 | } | |
803 | ||
804 | for (i = KVM_ARM64_SVE_VLS_WORDS - 1; i >= 0; --i) { | |
805 | if (vls[i]) { | |
806 | vq = 64 - clz64(vls[i]) + i * 64; | |
807 | break; | |
808 | } | |
809 | } | |
810 | if (vq > ARM_MAX_VQ) { | |
811 | warn_report("KVM supports vector lengths larger than " | |
812 | "QEMU can enable"); | |
886902ec | 813 | vls[0] &= MAKE_64BIT_MASK(0, ARM_MAX_VQ); |
6fa8a379 AJ |
814 | } |
815 | } | |
816 | ||
886902ec | 817 | return vls[0]; |
6fa8a379 AJ |
818 | } |
819 | ||
820 | static int kvm_arm_sve_set_vls(CPUState *cs) | |
821 | { | |
886902ec | 822 | ARMCPU *cpu = ARM_CPU(cs); |
7f9e25a6 | 823 | uint64_t vls[KVM_ARM64_SVE_VLS_WORDS] = { cpu->sve_vq.map }; |
6fa8a379 AJ |
824 | struct kvm_one_reg reg = { |
825 | .id = KVM_REG_ARM64_SVE_VLS, | |
826 | .addr = (uint64_t)&vls[0], | |
827 | }; | |
6fa8a379 AJ |
828 | |
829 | assert(cpu->sve_max_vq <= KVM_ARM64_SVE_VQ_MAX); | |
830 | ||
6fa8a379 AJ |
831 | return kvm_vcpu_ioctl(cs, KVM_SET_ONE_REG, ®); |
832 | } | |
833 | ||
eb5e1d3c PF |
834 | #define ARM_CPU_ID_MPIDR 3, 0, 0, 0, 5 |
835 | ||
26861c7c MH |
836 | int kvm_arch_init_vcpu(CPUState *cs) |
837 | { | |
26861c7c | 838 | int ret; |
eb5e1d3c | 839 | uint64_t mpidr; |
228d5e04 | 840 | ARMCPU *cpu = ARM_CPU(cs); |
929e754d | 841 | CPUARMState *env = &cpu->env; |
dc8bc9d6 | 842 | uint64_t psciver; |
26861c7c MH |
843 | |
844 | if (cpu->kvm_target == QEMU_KVM_ARM_TARGET_NONE || | |
56073970 | 845 | !object_dynamic_cast(OBJECT(cpu), TYPE_AARCH64_CPU)) { |
6fa8a379 | 846 | error_report("KVM is not supported for this guest CPU type"); |
26861c7c MH |
847 | return -EINVAL; |
848 | } | |
849 | ||
e5ac4200 AJ |
850 | qemu_add_vm_change_state_handler(kvm_arm_vm_state_change, cs); |
851 | ||
228d5e04 PS |
852 | /* Determine init features for this CPU */ |
853 | memset(cpu->kvm_init_features, 0, sizeof(cpu->kvm_init_features)); | |
c1b70158 | 854 | if (cs->start_powered_off) { |
228d5e04 PS |
855 | cpu->kvm_init_features[0] |= 1 << KVM_ARM_VCPU_POWER_OFF; |
856 | } | |
7cd62e53 | 857 | if (kvm_check_extension(cs->kvm_state, KVM_CAP_ARM_PSCI_0_2)) { |
0dc71c70 | 858 | cpu->psci_version = QEMU_PSCI_VERSION_0_2; |
7cd62e53 PS |
859 | cpu->kvm_init_features[0] |= 1 << KVM_ARM_VCPU_PSCI_0_2; |
860 | } | |
56073970 GB |
861 | if (!arm_feature(&cpu->env, ARM_FEATURE_AARCH64)) { |
862 | cpu->kvm_init_features[0] |= 1 << KVM_ARM_VCPU_EL1_32BIT; | |
863 | } | |
b1659527 | 864 | if (!kvm_check_extension(cs->kvm_state, KVM_CAP_ARM_PMU_V3)) { |
14e99e0f | 865 | cpu->has_pmu = false; |
929e754d WH |
866 | } |
867 | if (cpu->has_pmu) { | |
5c0a3819 | 868 | cpu->kvm_init_features[0] |= 1 << KVM_ARM_VCPU_PMU_V3; |
929e754d | 869 | } else { |
f5cbb280 | 870 | env->features &= ~(1ULL << ARM_FEATURE_PMU); |
5c0a3819 | 871 | } |
14e99e0f | 872 | if (cpu_isar_feature(aa64_sve, cpu)) { |
7d20e681 | 873 | assert(kvm_arm_sve_supported()); |
14e99e0f AJ |
874 | cpu->kvm_init_features[0] |= 1 << KVM_ARM_VCPU_SVE; |
875 | } | |
95ea96e8 MZ |
876 | if (cpu_isar_feature(aa64_pauth, cpu)) { |
877 | cpu->kvm_init_features[0] |= (1 << KVM_ARM_VCPU_PTRAUTH_ADDRESS | | |
878 | 1 << KVM_ARM_VCPU_PTRAUTH_GENERIC); | |
879 | } | |
228d5e04 PS |
880 | |
881 | /* Do KVM_ARM_VCPU_INIT ioctl */ | |
882 | ret = kvm_arm_vcpu_init(cs); | |
883 | if (ret) { | |
884 | return ret; | |
26861c7c | 885 | } |
26861c7c | 886 | |
14e99e0f | 887 | if (cpu_isar_feature(aa64_sve, cpu)) { |
6fa8a379 AJ |
888 | ret = kvm_arm_sve_set_vls(cs); |
889 | if (ret) { | |
890 | return ret; | |
891 | } | |
14e99e0f AJ |
892 | ret = kvm_arm_vcpu_finalize(cs, KVM_ARM_VCPU_SVE); |
893 | if (ret) { | |
894 | return ret; | |
895 | } | |
896 | } | |
897 | ||
dc8bc9d6 PM |
898 | /* |
899 | * KVM reports the exact PSCI version it is implementing via a | |
900 | * special sysreg. If it is present, use its contents to determine | |
901 | * what to report to the guest in the dtb (it is the PSCI version, | |
902 | * in the same 15-bits major 16-bits minor format that PSCI_VERSION | |
903 | * returns). | |
904 | */ | |
905 | if (!kvm_get_one_reg(cs, KVM_REG_ARM_PSCI_VERSION, &psciver)) { | |
906 | cpu->psci_version = psciver; | |
907 | } | |
908 | ||
eb5e1d3c PF |
909 | /* |
910 | * When KVM is in use, PSCI is emulated in-kernel and not by qemu. | |
911 | * Currently KVM has its own idea about MPIDR assignment, so we | |
912 | * override our defaults with what we get from KVM. | |
913 | */ | |
914 | ret = kvm_get_one_reg(cs, ARM64_SYS_REG(ARM_CPU_ID_MPIDR), &mpidr); | |
915 | if (ret) { | |
916 | return ret; | |
917 | } | |
0f4a9e45 | 918 | cpu->mp_affinity = mpidr & ARM64_AFFINITY_MASK; |
eb5e1d3c | 919 | |
202ccb6b DG |
920 | /* Check whether user space can specify guest syndrome value */ |
921 | kvm_arm_init_serror_injection(cs); | |
922 | ||
38df27c8 AB |
923 | return kvm_arm_init_cpreg_list(cpu); |
924 | } | |
26861c7c | 925 | |
b1115c99 LA |
926 | int kvm_arch_destroy_vcpu(CPUState *cs) |
927 | { | |
928 | return 0; | |
929 | } | |
930 | ||
38df27c8 AB |
931 | bool kvm_arm_reg_syncs_via_cpreg_list(uint64_t regidx) |
932 | { | |
933 | /* Return true if the regidx is a register we should synchronize | |
40b3fd21 AJ |
934 | * via the cpreg_tuples array (ie is not a core or sve reg that |
935 | * we sync by hand in kvm_arch_get/put_registers()) | |
38df27c8 AB |
936 | */ |
937 | switch (regidx & KVM_REG_ARM_COPROC_MASK) { | |
938 | case KVM_REG_ARM_CORE: | |
40b3fd21 | 939 | case KVM_REG_ARM64_SVE: |
38df27c8 AB |
940 | return false; |
941 | default: | |
942 | return true; | |
943 | } | |
26861c7c MH |
944 | } |
945 | ||
4b7a6bf4 CD |
946 | typedef struct CPRegStateLevel { |
947 | uint64_t regidx; | |
948 | int level; | |
949 | } CPRegStateLevel; | |
950 | ||
951 | /* All system registers not listed in the following table are assumed to be | |
952 | * of the level KVM_PUT_RUNTIME_STATE. If a register should be written less | |
953 | * often, you must add it to this table with a state of either | |
954 | * KVM_PUT_RESET_STATE or KVM_PUT_FULL_STATE. | |
955 | */ | |
956 | static const CPRegStateLevel non_runtime_cpregs[] = { | |
957 | { KVM_REG_ARM_TIMER_CNT, KVM_PUT_FULL_STATE }, | |
958 | }; | |
959 | ||
960 | int kvm_arm_cpreg_level(uint64_t regidx) | |
961 | { | |
962 | int i; | |
963 | ||
964 | for (i = 0; i < ARRAY_SIZE(non_runtime_cpregs); i++) { | |
965 | const CPRegStateLevel *l = &non_runtime_cpregs[i]; | |
966 | if (l->regidx == regidx) { | |
967 | return l->level; | |
968 | } | |
969 | } | |
970 | ||
971 | return KVM_PUT_RUNTIME_STATE; | |
972 | } | |
973 | ||
e24fd076 DG |
974 | /* Callers must hold the iothread mutex lock */ |
975 | static void kvm_inject_arm_sea(CPUState *c) | |
976 | { | |
977 | ARMCPU *cpu = ARM_CPU(c); | |
978 | CPUARMState *env = &cpu->env; | |
e24fd076 DG |
979 | uint32_t esr; |
980 | bool same_el; | |
981 | ||
982 | c->exception_index = EXCP_DATA_ABORT; | |
983 | env->exception.target_el = 1; | |
984 | ||
985 | /* | |
986 | * Set the DFSC to synchronous external abort and set FnV to not valid, | |
987 | * this will tell guest the FAR_ELx is UNKNOWN for this abort. | |
988 | */ | |
989 | same_el = arm_current_el(env) == env->exception.target_el; | |
990 | esr = syn_data_abort_no_iss(same_el, 1, 0, 0, 0, 0, 0x10); | |
991 | ||
992 | env->exception.syndrome = esr; | |
993 | ||
853bfef4 | 994 | arm_cpu_do_interrupt(c); |
e24fd076 DG |
995 | } |
996 | ||
26861c7c MH |
997 | #define AARCH64_CORE_REG(x) (KVM_REG_ARM64 | KVM_REG_SIZE_U64 | \ |
998 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(x)) | |
999 | ||
0e4b5869 AB |
1000 | #define AARCH64_SIMD_CORE_REG(x) (KVM_REG_ARM64 | KVM_REG_SIZE_U128 | \ |
1001 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(x)) | |
1002 | ||
1003 | #define AARCH64_SIMD_CTRL_REG(x) (KVM_REG_ARM64 | KVM_REG_SIZE_U32 | \ | |
1004 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(x)) | |
1005 | ||
30e3537f | 1006 | static int kvm_arch_put_fpsimd(CPUState *cs) |
26861c7c | 1007 | { |
40b3fd21 | 1008 | CPUARMState *env = &ARM_CPU(cs)->env; |
26861c7c | 1009 | struct kvm_one_reg reg; |
30e3537f AJ |
1010 | int i, ret; |
1011 | ||
1012 | for (i = 0; i < 32; i++) { | |
1013 | uint64_t *q = aa64_vfp_qreg(env, i); | |
e03b5686 | 1014 | #if HOST_BIG_ENDIAN |
30e3537f AJ |
1015 | uint64_t fp_val[2] = { q[1], q[0] }; |
1016 | reg.addr = (uintptr_t)fp_val; | |
1017 | #else | |
1018 | reg.addr = (uintptr_t)q; | |
1019 | #endif | |
1020 | reg.id = AARCH64_SIMD_CORE_REG(fp_regs.vregs[i]); | |
1021 | ret = kvm_vcpu_ioctl(cs, KVM_SET_ONE_REG, ®); | |
1022 | if (ret) { | |
1023 | return ret; | |
1024 | } | |
1025 | } | |
1026 | ||
40b3fd21 AJ |
1027 | return 0; |
1028 | } | |
1029 | ||
40b3fd21 AJ |
1030 | /* |
1031 | * KVM SVE registers come in slices where ZREGs have a slice size of 2048 bits | |
1032 | * and PREGS and the FFR have a slice size of 256 bits. However we simply hard | |
1033 | * code the slice index to zero for now as it's unlikely we'll need more than | |
1034 | * one slice for quite some time. | |
1035 | */ | |
1036 | static int kvm_arch_put_sve(CPUState *cs) | |
1037 | { | |
1038 | ARMCPU *cpu = ARM_CPU(cs); | |
1039 | CPUARMState *env = &cpu->env; | |
1040 | uint64_t tmp[ARM_MAX_VQ * 2]; | |
1041 | uint64_t *r; | |
1042 | struct kvm_one_reg reg; | |
1043 | int n, ret; | |
1044 | ||
1045 | for (n = 0; n < KVM_ARM64_SVE_NUM_ZREGS; ++n) { | |
1046 | r = sve_bswap64(tmp, &env->vfp.zregs[n].d[0], cpu->sve_max_vq * 2); | |
1047 | reg.addr = (uintptr_t)r; | |
1048 | reg.id = KVM_REG_ARM64_SVE_ZREG(n, 0); | |
1049 | ret = kvm_vcpu_ioctl(cs, KVM_SET_ONE_REG, ®); | |
1050 | if (ret) { | |
1051 | return ret; | |
1052 | } | |
1053 | } | |
1054 | ||
1055 | for (n = 0; n < KVM_ARM64_SVE_NUM_PREGS; ++n) { | |
1056 | r = sve_bswap64(tmp, r = &env->vfp.pregs[n].p[0], | |
1057 | DIV_ROUND_UP(cpu->sve_max_vq * 2, 8)); | |
1058 | reg.addr = (uintptr_t)r; | |
1059 | reg.id = KVM_REG_ARM64_SVE_PREG(n, 0); | |
1060 | ret = kvm_vcpu_ioctl(cs, KVM_SET_ONE_REG, ®); | |
1061 | if (ret) { | |
1062 | return ret; | |
1063 | } | |
1064 | } | |
1065 | ||
1066 | r = sve_bswap64(tmp, &env->vfp.pregs[FFR_PRED_NUM].p[0], | |
1067 | DIV_ROUND_UP(cpu->sve_max_vq * 2, 8)); | |
1068 | reg.addr = (uintptr_t)r; | |
1069 | reg.id = KVM_REG_ARM64_SVE_FFR(0); | |
30e3537f AJ |
1070 | ret = kvm_vcpu_ioctl(cs, KVM_SET_ONE_REG, ®); |
1071 | if (ret) { | |
1072 | return ret; | |
1073 | } | |
1074 | ||
1075 | return 0; | |
1076 | } | |
1077 | ||
1078 | int kvm_arch_put_registers(CPUState *cs, int level) | |
1079 | { | |
1080 | struct kvm_one_reg reg; | |
26861c7c | 1081 | uint64_t val; |
40b3fd21 | 1082 | uint32_t fpr; |
30e3537f | 1083 | int i, ret; |
25b9fb10 | 1084 | unsigned int el; |
26861c7c MH |
1085 | |
1086 | ARMCPU *cpu = ARM_CPU(cs); | |
1087 | CPUARMState *env = &cpu->env; | |
1088 | ||
56073970 GB |
1089 | /* If we are in AArch32 mode then we need to copy the AArch32 regs to the |
1090 | * AArch64 registers before pushing them out to 64-bit KVM. | |
1091 | */ | |
1092 | if (!is_a64(env)) { | |
1093 | aarch64_sync_32_to_64(env); | |
1094 | } | |
1095 | ||
26861c7c MH |
1096 | for (i = 0; i < 31; i++) { |
1097 | reg.id = AARCH64_CORE_REG(regs.regs[i]); | |
1098 | reg.addr = (uintptr_t) &env->xregs[i]; | |
1099 | ret = kvm_vcpu_ioctl(cs, KVM_SET_ONE_REG, ®); | |
1100 | if (ret) { | |
1101 | return ret; | |
1102 | } | |
1103 | } | |
1104 | ||
f502cfc2 PM |
1105 | /* KVM puts SP_EL0 in regs.sp and SP_EL1 in regs.sp_el1. On the |
1106 | * QEMU side we keep the current SP in xregs[31] as well. | |
1107 | */ | |
9208b961 | 1108 | aarch64_save_sp(env, 1); |
f502cfc2 | 1109 | |
26861c7c | 1110 | reg.id = AARCH64_CORE_REG(regs.sp); |
f502cfc2 PM |
1111 | reg.addr = (uintptr_t) &env->sp_el[0]; |
1112 | ret = kvm_vcpu_ioctl(cs, KVM_SET_ONE_REG, ®); | |
1113 | if (ret) { | |
1114 | return ret; | |
1115 | } | |
1116 | ||
1117 | reg.id = AARCH64_CORE_REG(sp_el1); | |
1118 | reg.addr = (uintptr_t) &env->sp_el[1]; | |
26861c7c MH |
1119 | ret = kvm_vcpu_ioctl(cs, KVM_SET_ONE_REG, ®); |
1120 | if (ret) { | |
1121 | return ret; | |
1122 | } | |
1123 | ||
1124 | /* Note that KVM thinks pstate is 64 bit but we use a uint32_t */ | |
56073970 GB |
1125 | if (is_a64(env)) { |
1126 | val = pstate_read(env); | |
1127 | } else { | |
1128 | val = cpsr_read(env); | |
1129 | } | |
26861c7c MH |
1130 | reg.id = AARCH64_CORE_REG(regs.pstate); |
1131 | reg.addr = (uintptr_t) &val; | |
1132 | ret = kvm_vcpu_ioctl(cs, KVM_SET_ONE_REG, ®); | |
1133 | if (ret) { | |
1134 | return ret; | |
1135 | } | |
1136 | ||
1137 | reg.id = AARCH64_CORE_REG(regs.pc); | |
1138 | reg.addr = (uintptr_t) &env->pc; | |
1139 | ret = kvm_vcpu_ioctl(cs, KVM_SET_ONE_REG, ®); | |
1140 | if (ret) { | |
1141 | return ret; | |
1142 | } | |
1143 | ||
a0618a19 | 1144 | reg.id = AARCH64_CORE_REG(elr_el1); |
6947f059 | 1145 | reg.addr = (uintptr_t) &env->elr_el[1]; |
a0618a19 PM |
1146 | ret = kvm_vcpu_ioctl(cs, KVM_SET_ONE_REG, ®); |
1147 | if (ret) { | |
1148 | return ret; | |
1149 | } | |
1150 | ||
25b9fb10 AB |
1151 | /* Saved Program State Registers |
1152 | * | |
1153 | * Before we restore from the banked_spsr[] array we need to | |
1154 | * ensure that any modifications to env->spsr are correctly | |
1155 | * reflected in the banks. | |
1156 | */ | |
1157 | el = arm_current_el(env); | |
1158 | if (el > 0 && !is_a64(env)) { | |
1159 | i = bank_number(env->uncached_cpsr & CPSR_M); | |
1160 | env->banked_spsr[i] = env->spsr; | |
1161 | } | |
1162 | ||
1163 | /* KVM 0-4 map to QEMU banks 1-5 */ | |
a65f1de9 PM |
1164 | for (i = 0; i < KVM_NR_SPSR; i++) { |
1165 | reg.id = AARCH64_CORE_REG(spsr[i]); | |
25b9fb10 | 1166 | reg.addr = (uintptr_t) &env->banked_spsr[i + 1]; |
a65f1de9 PM |
1167 | ret = kvm_vcpu_ioctl(cs, KVM_SET_ONE_REG, ®); |
1168 | if (ret) { | |
1169 | return ret; | |
1170 | } | |
1171 | } | |
1172 | ||
40b3fd21 AJ |
1173 | if (cpu_isar_feature(aa64_sve, cpu)) { |
1174 | ret = kvm_arch_put_sve(cs); | |
1175 | } else { | |
1176 | ret = kvm_arch_put_fpsimd(cs); | |
1177 | } | |
1178 | if (ret) { | |
1179 | return ret; | |
1180 | } | |
1181 | ||
1182 | reg.addr = (uintptr_t)(&fpr); | |
1183 | fpr = vfp_get_fpsr(env); | |
1184 | reg.id = AARCH64_SIMD_CTRL_REG(fp_regs.fpsr); | |
1185 | ret = kvm_vcpu_ioctl(cs, KVM_SET_ONE_REG, ®); | |
1186 | if (ret) { | |
1187 | return ret; | |
1188 | } | |
1189 | ||
1190 | reg.addr = (uintptr_t)(&fpr); | |
1191 | fpr = vfp_get_fpcr(env); | |
1192 | reg.id = AARCH64_SIMD_CTRL_REG(fp_regs.fpcr); | |
1193 | ret = kvm_vcpu_ioctl(cs, KVM_SET_ONE_REG, ®); | |
30e3537f AJ |
1194 | if (ret) { |
1195 | return ret; | |
1196 | } | |
1197 | ||
30e3537f AJ |
1198 | write_cpustate_to_list(cpu, true); |
1199 | ||
1200 | if (!write_list_to_kvmstate(cpu, level)) { | |
1201 | return -EINVAL; | |
1202 | } | |
1203 | ||
aca53be3 BM |
1204 | /* |
1205 | * Setting VCPU events should be triggered after syncing the registers | |
1206 | * to avoid overwriting potential changes made by KVM upon calling | |
1207 | * KVM_SET_VCPU_EVENTS ioctl | |
1208 | */ | |
1209 | ret = kvm_put_vcpu_events(cpu); | |
1210 | if (ret) { | |
1211 | return ret; | |
1212 | } | |
1213 | ||
30e3537f AJ |
1214 | kvm_arm_sync_mpstate_to_kvm(cpu); |
1215 | ||
1216 | return ret; | |
1217 | } | |
1218 | ||
1219 | static int kvm_arch_get_fpsimd(CPUState *cs) | |
1220 | { | |
40b3fd21 | 1221 | CPUARMState *env = &ARM_CPU(cs)->env; |
30e3537f | 1222 | struct kvm_one_reg reg; |
30e3537f AJ |
1223 | int i, ret; |
1224 | ||
0e4b5869 | 1225 | for (i = 0; i < 32; i++) { |
9a2b5256 | 1226 | uint64_t *q = aa64_vfp_qreg(env, i); |
0e4b5869 | 1227 | reg.id = AARCH64_SIMD_CORE_REG(fp_regs.vregs[i]); |
30e3537f AJ |
1228 | reg.addr = (uintptr_t)q; |
1229 | ret = kvm_vcpu_ioctl(cs, KVM_GET_ONE_REG, ®); | |
0e4b5869 AB |
1230 | if (ret) { |
1231 | return ret; | |
30e3537f | 1232 | } else { |
e03b5686 | 1233 | #if HOST_BIG_ENDIAN |
30e3537f AJ |
1234 | uint64_t t; |
1235 | t = q[0], q[0] = q[1], q[1] = t; | |
1236 | #endif | |
0e4b5869 AB |
1237 | } |
1238 | } | |
1239 | ||
40b3fd21 AJ |
1240 | return 0; |
1241 | } | |
1242 | ||
1243 | /* | |
1244 | * KVM SVE registers come in slices where ZREGs have a slice size of 2048 bits | |
1245 | * and PREGS and the FFR have a slice size of 256 bits. However we simply hard | |
1246 | * code the slice index to zero for now as it's unlikely we'll need more than | |
1247 | * one slice for quite some time. | |
1248 | */ | |
1249 | static int kvm_arch_get_sve(CPUState *cs) | |
1250 | { | |
1251 | ARMCPU *cpu = ARM_CPU(cs); | |
1252 | CPUARMState *env = &cpu->env; | |
1253 | struct kvm_one_reg reg; | |
1254 | uint64_t *r; | |
1255 | int n, ret; | |
1256 | ||
1257 | for (n = 0; n < KVM_ARM64_SVE_NUM_ZREGS; ++n) { | |
1258 | r = &env->vfp.zregs[n].d[0]; | |
1259 | reg.addr = (uintptr_t)r; | |
1260 | reg.id = KVM_REG_ARM64_SVE_ZREG(n, 0); | |
1261 | ret = kvm_vcpu_ioctl(cs, KVM_GET_ONE_REG, ®); | |
1262 | if (ret) { | |
1263 | return ret; | |
1264 | } | |
1265 | sve_bswap64(r, r, cpu->sve_max_vq * 2); | |
0e4b5869 AB |
1266 | } |
1267 | ||
40b3fd21 AJ |
1268 | for (n = 0; n < KVM_ARM64_SVE_NUM_PREGS; ++n) { |
1269 | r = &env->vfp.pregs[n].p[0]; | |
1270 | reg.addr = (uintptr_t)r; | |
1271 | reg.id = KVM_REG_ARM64_SVE_PREG(n, 0); | |
1272 | ret = kvm_vcpu_ioctl(cs, KVM_GET_ONE_REG, ®); | |
1273 | if (ret) { | |
1274 | return ret; | |
1275 | } | |
1276 | sve_bswap64(r, r, DIV_ROUND_UP(cpu->sve_max_vq * 2, 8)); | |
1277 | } | |
1278 | ||
1279 | r = &env->vfp.pregs[FFR_PRED_NUM].p[0]; | |
1280 | reg.addr = (uintptr_t)r; | |
1281 | reg.id = KVM_REG_ARM64_SVE_FFR(0); | |
30e3537f | 1282 | ret = kvm_vcpu_ioctl(cs, KVM_GET_ONE_REG, ®); |
202ccb6b DG |
1283 | if (ret) { |
1284 | return ret; | |
1285 | } | |
40b3fd21 | 1286 | sve_bswap64(r, r, DIV_ROUND_UP(cpu->sve_max_vq * 2, 8)); |
202ccb6b | 1287 | |
30e3537f | 1288 | return 0; |
26861c7c MH |
1289 | } |
1290 | ||
1291 | int kvm_arch_get_registers(CPUState *cs) | |
1292 | { | |
1293 | struct kvm_one_reg reg; | |
1294 | uint64_t val; | |
25b9fb10 | 1295 | unsigned int el; |
40b3fd21 | 1296 | uint32_t fpr; |
30e3537f | 1297 | int i, ret; |
26861c7c MH |
1298 | |
1299 | ARMCPU *cpu = ARM_CPU(cs); | |
1300 | CPUARMState *env = &cpu->env; | |
1301 | ||
1302 | for (i = 0; i < 31; i++) { | |
1303 | reg.id = AARCH64_CORE_REG(regs.regs[i]); | |
1304 | reg.addr = (uintptr_t) &env->xregs[i]; | |
1305 | ret = kvm_vcpu_ioctl(cs, KVM_GET_ONE_REG, ®); | |
1306 | if (ret) { | |
1307 | return ret; | |
1308 | } | |
1309 | } | |
1310 | ||
1311 | reg.id = AARCH64_CORE_REG(regs.sp); | |
f502cfc2 PM |
1312 | reg.addr = (uintptr_t) &env->sp_el[0]; |
1313 | ret = kvm_vcpu_ioctl(cs, KVM_GET_ONE_REG, ®); | |
1314 | if (ret) { | |
1315 | return ret; | |
1316 | } | |
1317 | ||
1318 | reg.id = AARCH64_CORE_REG(sp_el1); | |
1319 | reg.addr = (uintptr_t) &env->sp_el[1]; | |
26861c7c MH |
1320 | ret = kvm_vcpu_ioctl(cs, KVM_GET_ONE_REG, ®); |
1321 | if (ret) { | |
1322 | return ret; | |
1323 | } | |
1324 | ||
1325 | reg.id = AARCH64_CORE_REG(regs.pstate); | |
1326 | reg.addr = (uintptr_t) &val; | |
1327 | ret = kvm_vcpu_ioctl(cs, KVM_GET_ONE_REG, ®); | |
1328 | if (ret) { | |
1329 | return ret; | |
1330 | } | |
56073970 GB |
1331 | |
1332 | env->aarch64 = ((val & PSTATE_nRW) == 0); | |
1333 | if (is_a64(env)) { | |
1334 | pstate_write(env, val); | |
1335 | } else { | |
50866ba5 | 1336 | cpsr_write(env, val, 0xffffffff, CPSRWriteRaw); |
56073970 | 1337 | } |
26861c7c | 1338 | |
f502cfc2 PM |
1339 | /* KVM puts SP_EL0 in regs.sp and SP_EL1 in regs.sp_el1. On the |
1340 | * QEMU side we keep the current SP in xregs[31] as well. | |
1341 | */ | |
9208b961 | 1342 | aarch64_restore_sp(env, 1); |
f502cfc2 | 1343 | |
26861c7c MH |
1344 | reg.id = AARCH64_CORE_REG(regs.pc); |
1345 | reg.addr = (uintptr_t) &env->pc; | |
1346 | ret = kvm_vcpu_ioctl(cs, KVM_GET_ONE_REG, ®); | |
1347 | if (ret) { | |
1348 | return ret; | |
1349 | } | |
1350 | ||
56073970 GB |
1351 | /* If we are in AArch32 mode then we need to sync the AArch32 regs with the |
1352 | * incoming AArch64 regs received from 64-bit KVM. | |
1353 | * We must perform this after all of the registers have been acquired from | |
1354 | * the kernel. | |
1355 | */ | |
1356 | if (!is_a64(env)) { | |
1357 | aarch64_sync_64_to_32(env); | |
1358 | } | |
1359 | ||
a0618a19 | 1360 | reg.id = AARCH64_CORE_REG(elr_el1); |
6947f059 | 1361 | reg.addr = (uintptr_t) &env->elr_el[1]; |
a0618a19 PM |
1362 | ret = kvm_vcpu_ioctl(cs, KVM_GET_ONE_REG, ®); |
1363 | if (ret) { | |
1364 | return ret; | |
1365 | } | |
1366 | ||
25b9fb10 AB |
1367 | /* Fetch the SPSR registers |
1368 | * | |
1369 | * KVM SPSRs 0-4 map to QEMU banks 1-5 | |
1370 | */ | |
a65f1de9 PM |
1371 | for (i = 0; i < KVM_NR_SPSR; i++) { |
1372 | reg.id = AARCH64_CORE_REG(spsr[i]); | |
25b9fb10 | 1373 | reg.addr = (uintptr_t) &env->banked_spsr[i + 1]; |
a65f1de9 PM |
1374 | ret = kvm_vcpu_ioctl(cs, KVM_GET_ONE_REG, ®); |
1375 | if (ret) { | |
1376 | return ret; | |
1377 | } | |
1378 | } | |
1379 | ||
25b9fb10 AB |
1380 | el = arm_current_el(env); |
1381 | if (el > 0 && !is_a64(env)) { | |
1382 | i = bank_number(env->uncached_cpsr & CPSR_M); | |
1383 | env->spsr = env->banked_spsr[i]; | |
1384 | } | |
1385 | ||
40b3fd21 AJ |
1386 | if (cpu_isar_feature(aa64_sve, cpu)) { |
1387 | ret = kvm_arch_get_sve(cs); | |
1388 | } else { | |
1389 | ret = kvm_arch_get_fpsimd(cs); | |
1390 | } | |
1391 | if (ret) { | |
1392 | return ret; | |
1393 | } | |
1394 | ||
1395 | reg.addr = (uintptr_t)(&fpr); | |
1396 | reg.id = AARCH64_SIMD_CTRL_REG(fp_regs.fpsr); | |
1397 | ret = kvm_vcpu_ioctl(cs, KVM_GET_ONE_REG, ®); | |
1398 | if (ret) { | |
1399 | return ret; | |
1400 | } | |
1401 | vfp_set_fpsr(env, fpr); | |
1402 | ||
1403 | reg.addr = (uintptr_t)(&fpr); | |
1404 | reg.id = AARCH64_SIMD_CTRL_REG(fp_regs.fpcr); | |
1405 | ret = kvm_vcpu_ioctl(cs, KVM_GET_ONE_REG, ®); | |
0e4b5869 AB |
1406 | if (ret) { |
1407 | return ret; | |
1408 | } | |
40b3fd21 | 1409 | vfp_set_fpcr(env, fpr); |
0e4b5869 | 1410 | |
202ccb6b DG |
1411 | ret = kvm_get_vcpu_events(cpu); |
1412 | if (ret) { | |
1413 | return ret; | |
1414 | } | |
1415 | ||
568bab1f | 1416 | if (!write_kvmstate_to_list(cpu)) { |
4ed9d9f8 | 1417 | return -EINVAL; |
568bab1f PS |
1418 | } |
1419 | /* Note that it's OK to have registers which aren't in CPUState, | |
1420 | * so we can ignore a failure return here. | |
1421 | */ | |
1422 | write_list_to_cpustate(cpu); | |
1423 | ||
1a1753f7 AB |
1424 | kvm_arm_sync_mpstate_to_qemu(cpu); |
1425 | ||
26861c7c MH |
1426 | /* TODO: other registers */ |
1427 | return ret; | |
1428 | } | |
2ecb2027 | 1429 | |
e24fd076 DG |
1430 | void kvm_arch_on_sigbus_vcpu(CPUState *c, int code, void *addr) |
1431 | { | |
1432 | ram_addr_t ram_addr; | |
1433 | hwaddr paddr; | |
e24fd076 DG |
1434 | |
1435 | assert(code == BUS_MCEERR_AR || code == BUS_MCEERR_AO); | |
1436 | ||
15613357 | 1437 | if (acpi_ghes_present() && addr) { |
e24fd076 DG |
1438 | ram_addr = qemu_ram_addr_from_host(addr); |
1439 | if (ram_addr != RAM_ADDR_INVALID && | |
1440 | kvm_physical_memory_addr_from_host(c->kvm_state, addr, &paddr)) { | |
1441 | kvm_hwpoison_page_add(ram_addr); | |
1442 | /* | |
1443 | * If this is a BUS_MCEERR_AR, we know we have been called | |
1444 | * synchronously from the vCPU thread, so we can easily | |
1445 | * synchronize the state and inject an error. | |
1446 | * | |
1447 | * TODO: we currently don't tell the guest at all about | |
1448 | * BUS_MCEERR_AO. In that case we might either be being | |
1449 | * called synchronously from the vCPU thread, or a bit | |
1450 | * later from the main thread, so doing the injection of | |
1451 | * the error would be more complicated. | |
1452 | */ | |
1453 | if (code == BUS_MCEERR_AR) { | |
1454 | kvm_cpu_synchronize_state(c); | |
1455 | if (!acpi_ghes_record_errors(ACPI_HEST_SRC_ID_SEA, paddr)) { | |
1456 | kvm_inject_arm_sea(c); | |
1457 | } else { | |
1458 | error_report("failed to record the error"); | |
1459 | abort(); | |
1460 | } | |
1461 | } | |
1462 | return; | |
1463 | } | |
1464 | if (code == BUS_MCEERR_AO) { | |
1465 | error_report("Hardware memory error at addr %p for memory used by " | |
1466 | "QEMU itself instead of guest system!", addr); | |
1467 | } | |
1468 | } | |
1469 | ||
1470 | if (code == BUS_MCEERR_AR) { | |
1471 | error_report("Hardware memory error!"); | |
1472 | exit(1); | |
1473 | } | |
1474 | } | |
1475 | ||
2ecb2027 AB |
1476 | /* C6.6.29 BRK instruction */ |
1477 | static const uint32_t brk_insn = 0xd4200000; | |
1478 | ||
1479 | int kvm_arch_insert_sw_breakpoint(CPUState *cs, struct kvm_sw_breakpoint *bp) | |
1480 | { | |
1481 | if (have_guest_debug) { | |
1482 | if (cpu_memory_rw_debug(cs, bp->pc, (uint8_t *)&bp->saved_insn, 4, 0) || | |
1483 | cpu_memory_rw_debug(cs, bp->pc, (uint8_t *)&brk_insn, 4, 1)) { | |
1484 | return -EINVAL; | |
1485 | } | |
1486 | return 0; | |
1487 | } else { | |
1488 | error_report("guest debug not supported on this kernel"); | |
1489 | return -EINVAL; | |
1490 | } | |
1491 | } | |
1492 | ||
1493 | int kvm_arch_remove_sw_breakpoint(CPUState *cs, struct kvm_sw_breakpoint *bp) | |
1494 | { | |
1495 | static uint32_t brk; | |
1496 | ||
1497 | if (have_guest_debug) { | |
1498 | if (cpu_memory_rw_debug(cs, bp->pc, (uint8_t *)&brk, 4, 0) || | |
1499 | brk != brk_insn || | |
1500 | cpu_memory_rw_debug(cs, bp->pc, (uint8_t *)&bp->saved_insn, 4, 1)) { | |
1501 | return -EINVAL; | |
1502 | } | |
1503 | return 0; | |
1504 | } else { | |
1505 | error_report("guest debug not supported on this kernel"); | |
1506 | return -EINVAL; | |
1507 | } | |
1508 | } | |
1509 | ||
1510 | /* See v8 ARM ARM D7.2.27 ESR_ELx, Exception Syndrome Register | |
1511 | * | |
1512 | * To minimise translating between kernel and user-space the kernel | |
1513 | * ABI just provides user-space with the full exception syndrome | |
1514 | * register value to be decoded in QEMU. | |
1515 | */ | |
1516 | ||
1517 | bool kvm_arm_handle_debug(CPUState *cs, struct kvm_debug_exit_arch *debug_exit) | |
1518 | { | |
64b91e3f | 1519 | int hsr_ec = syn_get_ec(debug_exit->hsr); |
2ecb2027 AB |
1520 | ARMCPU *cpu = ARM_CPU(cs); |
1521 | CPUARMState *env = &cpu->env; | |
1522 | ||
1523 | /* Ensure PC is synchronised */ | |
1524 | kvm_cpu_synchronize_state(cs); | |
1525 | ||
1526 | switch (hsr_ec) { | |
26ae5934 AB |
1527 | case EC_SOFTWARESTEP: |
1528 | if (cs->singlestep_enabled) { | |
1529 | return true; | |
1530 | } else { | |
34c45d53 AB |
1531 | /* |
1532 | * The kernel should have suppressed the guest's ability to | |
1533 | * single step at this point so something has gone wrong. | |
1534 | */ | |
1535 | error_report("%s: guest single-step while debugging unsupported" | |
dffc5851 | 1536 | " (%"PRIx64", %"PRIx32")", |
34c45d53 AB |
1537 | __func__, env->pc, debug_exit->hsr); |
1538 | return false; | |
26ae5934 AB |
1539 | } |
1540 | break; | |
2ecb2027 AB |
1541 | case EC_AA64_BKPT: |
1542 | if (kvm_find_sw_breakpoint(cs, env->pc)) { | |
1543 | return true; | |
1544 | } | |
1545 | break; | |
e4482ab7 AB |
1546 | case EC_BREAKPOINT: |
1547 | if (find_hw_breakpoint(cs, env->pc)) { | |
1548 | return true; | |
1549 | } | |
1550 | break; | |
1551 | case EC_WATCHPOINT: | |
1552 | { | |
1553 | CPUWatchpoint *wp = find_hw_watchpoint(cs, debug_exit->far); | |
1554 | if (wp) { | |
1555 | cs->watchpoint_hit = wp; | |
1556 | return true; | |
1557 | } | |
1558 | break; | |
1559 | } | |
2ecb2027 | 1560 | default: |
dffc5851 | 1561 | error_report("%s: unhandled debug exit (%"PRIx32", %"PRIx64")", |
2ecb2027 AB |
1562 | __func__, debug_exit->hsr, env->pc); |
1563 | } | |
1564 | ||
34c45d53 AB |
1565 | /* If we are not handling the debug exception it must belong to |
1566 | * the guest. Let's re-use the existing TCG interrupt code to set | |
1567 | * everything up properly. | |
1568 | */ | |
1569 | cs->exception_index = EXCP_BKPT; | |
1570 | env->exception.syndrome = debug_exit->hsr; | |
1571 | env->exception.vaddress = debug_exit->far; | |
14f9a5c0 | 1572 | env->exception.target_el = 1; |
9b16ec43 | 1573 | qemu_mutex_lock_iothread(); |
853bfef4 | 1574 | arm_cpu_do_interrupt(cs); |
9b16ec43 | 1575 | qemu_mutex_unlock_iothread(); |
2ecb2027 AB |
1576 | |
1577 | return false; | |
1578 | } | |
1711bfa5 BM |
1579 | |
1580 | #define ARM64_REG_ESR_EL1 ARM64_SYS_REG(3, 0, 5, 2, 0) | |
1581 | #define ARM64_REG_TCR_EL1 ARM64_SYS_REG(3, 0, 2, 0, 2) | |
1582 | ||
1583 | /* | |
1584 | * ESR_EL1 | |
1585 | * ISS encoding | |
1586 | * AARCH64: DFSC, bits [5:0] | |
1587 | * AARCH32: | |
1588 | * TTBCR.EAE == 0 | |
1589 | * FS[4] - DFSR[10] | |
1590 | * FS[3:0] - DFSR[3:0] | |
1591 | * TTBCR.EAE == 1 | |
1592 | * FS, bits [5:0] | |
1593 | */ | |
1594 | #define ESR_DFSC(aarch64, lpae, v) \ | |
1595 | ((aarch64 || (lpae)) ? ((v) & 0x3F) \ | |
1596 | : (((v) >> 6) | ((v) & 0x1F))) | |
1597 | ||
1598 | #define ESR_DFSC_EXTABT(aarch64, lpae) \ | |
1599 | ((aarch64) ? 0x10 : (lpae) ? 0x10 : 0x8) | |
1600 | ||
1601 | bool kvm_arm_verify_ext_dabt_pending(CPUState *cs) | |
1602 | { | |
1603 | uint64_t dfsr_val; | |
1604 | ||
1605 | if (!kvm_get_one_reg(cs, ARM64_REG_ESR_EL1, &dfsr_val)) { | |
1606 | ARMCPU *cpu = ARM_CPU(cs); | |
1607 | CPUARMState *env = &cpu->env; | |
1608 | int aarch64_mode = arm_feature(env, ARM_FEATURE_AARCH64); | |
1609 | int lpae = 0; | |
1610 | ||
1611 | if (!aarch64_mode) { | |
1612 | uint64_t ttbcr; | |
1613 | ||
1614 | if (!kvm_get_one_reg(cs, ARM64_REG_TCR_EL1, &ttbcr)) { | |
1615 | lpae = arm_feature(env, ARM_FEATURE_LPAE) | |
1616 | && (ttbcr & TTBCR_EAE); | |
1617 | } | |
1618 | } | |
1619 | /* | |
1620 | * The verification here is based on the DFSC bits | |
1621 | * of the ESR_EL1 reg only | |
1622 | */ | |
1623 | return (ESR_DFSC(aarch64_mode, lpae, dfsr_val) == | |
1624 | ESR_DFSC_EXTABT(aarch64_mode, lpae)); | |
1625 | } | |
1626 | return false; | |
1627 | } |