]>
Commit | Line | Data |
---|---|---|
55c7401d MZ |
1 | /* |
2 | * Copyright (C) 2012,2013 - ARM Ltd | |
3 | * Author: Marc Zyngier <marc.zyngier@arm.com> | |
4 | * | |
5 | * This program is free software; you can redistribute it and/or modify | |
6 | * it under the terms of the GNU General Public License version 2 as | |
7 | * published by the Free Software Foundation. | |
8 | * | |
9 | * This program is distributed in the hope that it will be useful, | |
10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
12 | * GNU General Public License for more details. | |
13 | * | |
14 | * You should have received a copy of the GNU General Public License | |
15 | * along with this program. If not, see <http://www.gnu.org/licenses/>. | |
16 | */ | |
17 | ||
18 | #include <linux/linkage.h> | |
55c7401d MZ |
19 | |
20 | #include <asm/assembler.h> | |
21 | #include <asm/memory.h> | |
22 | #include <asm/asm-offsets.h> | |
b0e626b3 | 23 | #include <asm/debug-monitors.h> |
55c7401d MZ |
24 | #include <asm/fpsimdmacros.h> |
25 | #include <asm/kvm.h> | |
26 | #include <asm/kvm_asm.h> | |
27 | #include <asm/kvm_arm.h> | |
28 | #include <asm/kvm_mmu.h> | |
29 | ||
30 | #define CPU_GP_REG_OFFSET(x) (CPU_GP_REGS + x) | |
31 | #define CPU_XREG_OFFSET(x) CPU_GP_REG_OFFSET(CPU_USER_PT_REGS + 8*x) | |
32 | #define CPU_SPSR_OFFSET(x) CPU_GP_REG_OFFSET(CPU_SPSR + 8*x) | |
33 | #define CPU_SYSREG_OFFSET(x) (CPU_SYSREGS + 8*x) | |
34 | ||
35 | .text | |
36 | .pushsection .hyp.text, "ax" | |
37 | .align PAGE_SHIFT | |
38 | ||
55c7401d MZ |
39 | .macro save_common_regs |
40 | // x2: base address for cpu context | |
41 | // x3: tmp register | |
42 | ||
43 | add x3, x2, #CPU_XREG_OFFSET(19) | |
44 | stp x19, x20, [x3] | |
45 | stp x21, x22, [x3, #16] | |
46 | stp x23, x24, [x3, #32] | |
47 | stp x25, x26, [x3, #48] | |
48 | stp x27, x28, [x3, #64] | |
49 | stp x29, lr, [x3, #80] | |
50 | ||
51 | mrs x19, sp_el0 | |
52 | mrs x20, elr_el2 // EL1 PC | |
53 | mrs x21, spsr_el2 // EL1 pstate | |
54 | ||
55 | stp x19, x20, [x3, #96] | |
56 | str x21, [x3, #112] | |
57 | ||
58 | mrs x22, sp_el1 | |
59 | mrs x23, elr_el1 | |
60 | mrs x24, spsr_el1 | |
61 | ||
62 | str x22, [x2, #CPU_GP_REG_OFFSET(CPU_SP_EL1)] | |
63 | str x23, [x2, #CPU_GP_REG_OFFSET(CPU_ELR_EL1)] | |
64 | str x24, [x2, #CPU_SPSR_OFFSET(KVM_SPSR_EL1)] | |
65 | .endm | |
66 | ||
67 | .macro restore_common_regs | |
68 | // x2: base address for cpu context | |
69 | // x3: tmp register | |
70 | ||
71 | ldr x22, [x2, #CPU_GP_REG_OFFSET(CPU_SP_EL1)] | |
72 | ldr x23, [x2, #CPU_GP_REG_OFFSET(CPU_ELR_EL1)] | |
73 | ldr x24, [x2, #CPU_SPSR_OFFSET(KVM_SPSR_EL1)] | |
74 | ||
75 | msr sp_el1, x22 | |
76 | msr elr_el1, x23 | |
77 | msr spsr_el1, x24 | |
78 | ||
79 | add x3, x2, #CPU_XREG_OFFSET(31) // SP_EL0 | |
80 | ldp x19, x20, [x3] | |
81 | ldr x21, [x3, #16] | |
82 | ||
83 | msr sp_el0, x19 | |
84 | msr elr_el2, x20 // EL1 PC | |
85 | msr spsr_el2, x21 // EL1 pstate | |
86 | ||
87 | add x3, x2, #CPU_XREG_OFFSET(19) | |
88 | ldp x19, x20, [x3] | |
89 | ldp x21, x22, [x3, #16] | |
90 | ldp x23, x24, [x3, #32] | |
91 | ldp x25, x26, [x3, #48] | |
92 | ldp x27, x28, [x3, #64] | |
93 | ldp x29, lr, [x3, #80] | |
94 | .endm | |
95 | ||
96 | .macro save_host_regs | |
97 | save_common_regs | |
98 | .endm | |
99 | ||
100 | .macro restore_host_regs | |
101 | restore_common_regs | |
102 | .endm | |
103 | ||
104 | .macro save_fpsimd | |
105 | // x2: cpu context address | |
106 | // x3, x4: tmp regs | |
107 | add x3, x2, #CPU_GP_REG_OFFSET(CPU_FP_REGS) | |
108 | fpsimd_save x3, 4 | |
109 | .endm | |
110 | ||
111 | .macro restore_fpsimd | |
112 | // x2: cpu context address | |
113 | // x3, x4: tmp regs | |
114 | add x3, x2, #CPU_GP_REG_OFFSET(CPU_FP_REGS) | |
115 | fpsimd_restore x3, 4 | |
116 | .endm | |
117 | ||
118 | .macro save_guest_regs | |
119 | // x0 is the vcpu address | |
120 | // x1 is the return code, do not corrupt! | |
121 | // x2 is the cpu context | |
122 | // x3 is a tmp register | |
123 | // Guest's x0-x3 are on the stack | |
124 | ||
125 | // Compute base to save registers | |
126 | add x3, x2, #CPU_XREG_OFFSET(4) | |
127 | stp x4, x5, [x3] | |
128 | stp x6, x7, [x3, #16] | |
129 | stp x8, x9, [x3, #32] | |
130 | stp x10, x11, [x3, #48] | |
131 | stp x12, x13, [x3, #64] | |
132 | stp x14, x15, [x3, #80] | |
133 | stp x16, x17, [x3, #96] | |
134 | str x18, [x3, #112] | |
135 | ||
136 | pop x6, x7 // x2, x3 | |
137 | pop x4, x5 // x0, x1 | |
138 | ||
139 | add x3, x2, #CPU_XREG_OFFSET(0) | |
140 | stp x4, x5, [x3] | |
141 | stp x6, x7, [x3, #16] | |
142 | ||
143 | save_common_regs | |
144 | .endm | |
145 | ||
146 | .macro restore_guest_regs | |
147 | // x0 is the vcpu address. | |
148 | // x2 is the cpu context | |
149 | // x3 is a tmp register | |
150 | ||
151 | // Prepare x0-x3 for later restore | |
152 | add x3, x2, #CPU_XREG_OFFSET(0) | |
153 | ldp x4, x5, [x3] | |
154 | ldp x6, x7, [x3, #16] | |
155 | push x4, x5 // Push x0-x3 on the stack | |
156 | push x6, x7 | |
157 | ||
158 | // x4-x18 | |
159 | ldp x4, x5, [x3, #32] | |
160 | ldp x6, x7, [x3, #48] | |
161 | ldp x8, x9, [x3, #64] | |
162 | ldp x10, x11, [x3, #80] | |
163 | ldp x12, x13, [x3, #96] | |
164 | ldp x14, x15, [x3, #112] | |
165 | ldp x16, x17, [x3, #128] | |
166 | ldr x18, [x3, #144] | |
167 | ||
168 | // x19-x29, lr, sp*, elr*, spsr* | |
169 | restore_common_regs | |
170 | ||
171 | // Last bits of the 64bit state | |
172 | pop x2, x3 | |
173 | pop x0, x1 | |
174 | ||
175 | // Do not touch any register after this! | |
176 | .endm | |
177 | ||
178 | /* | |
179 | * Macros to perform system register save/restore. | |
180 | * | |
181 | * Ordering here is absolutely critical, and must be kept consistent | |
182 | * in {save,restore}_sysregs, {save,restore}_guest_32bit_state, | |
183 | * and in kvm_asm.h. | |
184 | * | |
185 | * In other words, don't touch any of these unless you know what | |
186 | * you are doing. | |
187 | */ | |
188 | .macro save_sysregs | |
189 | // x2: base address for cpu context | |
190 | // x3: tmp register | |
191 | ||
192 | add x3, x2, #CPU_SYSREG_OFFSET(MPIDR_EL1) | |
193 | ||
194 | mrs x4, vmpidr_el2 | |
195 | mrs x5, csselr_el1 | |
196 | mrs x6, sctlr_el1 | |
197 | mrs x7, actlr_el1 | |
198 | mrs x8, cpacr_el1 | |
199 | mrs x9, ttbr0_el1 | |
200 | mrs x10, ttbr1_el1 | |
201 | mrs x11, tcr_el1 | |
202 | mrs x12, esr_el1 | |
203 | mrs x13, afsr0_el1 | |
204 | mrs x14, afsr1_el1 | |
205 | mrs x15, far_el1 | |
206 | mrs x16, mair_el1 | |
207 | mrs x17, vbar_el1 | |
208 | mrs x18, contextidr_el1 | |
209 | mrs x19, tpidr_el0 | |
210 | mrs x20, tpidrro_el0 | |
211 | mrs x21, tpidr_el1 | |
212 | mrs x22, amair_el1 | |
213 | mrs x23, cntkctl_el1 | |
1bbd8054 | 214 | mrs x24, par_el1 |
b0e626b3 | 215 | mrs x25, mdscr_el1 |
55c7401d MZ |
216 | |
217 | stp x4, x5, [x3] | |
218 | stp x6, x7, [x3, #16] | |
219 | stp x8, x9, [x3, #32] | |
220 | stp x10, x11, [x3, #48] | |
221 | stp x12, x13, [x3, #64] | |
222 | stp x14, x15, [x3, #80] | |
223 | stp x16, x17, [x3, #96] | |
224 | stp x18, x19, [x3, #112] | |
225 | stp x20, x21, [x3, #128] | |
226 | stp x22, x23, [x3, #144] | |
b0e626b3 MZ |
227 | stp x24, x25, [x3, #160] |
228 | .endm | |
229 | ||
230 | .macro save_debug | |
231 | // x2: base address for cpu context | |
232 | // x3: tmp register | |
233 | ||
234 | mrs x26, id_aa64dfr0_el1 | |
235 | ubfx x24, x26, #12, #4 // Extract BRPs | |
236 | ubfx x25, x26, #20, #4 // Extract WRPs | |
237 | mov w26, #15 | |
238 | sub w24, w26, w24 // How many BPs to skip | |
239 | sub w25, w26, w25 // How many WPs to skip | |
240 | ||
241 | add x3, x2, #CPU_SYSREG_OFFSET(DBGBCR0_EL1) | |
242 | ||
243 | adr x26, 1f | |
244 | add x26, x26, x24, lsl #2 | |
245 | br x26 | |
246 | 1: | |
247 | mrs x20, dbgbcr15_el1 | |
248 | mrs x19, dbgbcr14_el1 | |
249 | mrs x18, dbgbcr13_el1 | |
250 | mrs x17, dbgbcr12_el1 | |
251 | mrs x16, dbgbcr11_el1 | |
252 | mrs x15, dbgbcr10_el1 | |
253 | mrs x14, dbgbcr9_el1 | |
254 | mrs x13, dbgbcr8_el1 | |
255 | mrs x12, dbgbcr7_el1 | |
256 | mrs x11, dbgbcr6_el1 | |
257 | mrs x10, dbgbcr5_el1 | |
258 | mrs x9, dbgbcr4_el1 | |
259 | mrs x8, dbgbcr3_el1 | |
260 | mrs x7, dbgbcr2_el1 | |
261 | mrs x6, dbgbcr1_el1 | |
262 | mrs x5, dbgbcr0_el1 | |
263 | ||
264 | adr x26, 1f | |
265 | add x26, x26, x24, lsl #2 | |
266 | br x26 | |
267 | ||
268 | 1: | |
269 | str x20, [x3, #(15 * 8)] | |
270 | str x19, [x3, #(14 * 8)] | |
271 | str x18, [x3, #(13 * 8)] | |
272 | str x17, [x3, #(12 * 8)] | |
273 | str x16, [x3, #(11 * 8)] | |
274 | str x15, [x3, #(10 * 8)] | |
275 | str x14, [x3, #(9 * 8)] | |
276 | str x13, [x3, #(8 * 8)] | |
277 | str x12, [x3, #(7 * 8)] | |
278 | str x11, [x3, #(6 * 8)] | |
279 | str x10, [x3, #(5 * 8)] | |
280 | str x9, [x3, #(4 * 8)] | |
281 | str x8, [x3, #(3 * 8)] | |
282 | str x7, [x3, #(2 * 8)] | |
283 | str x6, [x3, #(1 * 8)] | |
284 | str x5, [x3, #(0 * 8)] | |
285 | ||
286 | add x3, x2, #CPU_SYSREG_OFFSET(DBGBVR0_EL1) | |
287 | ||
288 | adr x26, 1f | |
289 | add x26, x26, x24, lsl #2 | |
290 | br x26 | |
291 | 1: | |
292 | mrs x20, dbgbvr15_el1 | |
293 | mrs x19, dbgbvr14_el1 | |
294 | mrs x18, dbgbvr13_el1 | |
295 | mrs x17, dbgbvr12_el1 | |
296 | mrs x16, dbgbvr11_el1 | |
297 | mrs x15, dbgbvr10_el1 | |
298 | mrs x14, dbgbvr9_el1 | |
299 | mrs x13, dbgbvr8_el1 | |
300 | mrs x12, dbgbvr7_el1 | |
301 | mrs x11, dbgbvr6_el1 | |
302 | mrs x10, dbgbvr5_el1 | |
303 | mrs x9, dbgbvr4_el1 | |
304 | mrs x8, dbgbvr3_el1 | |
305 | mrs x7, dbgbvr2_el1 | |
306 | mrs x6, dbgbvr1_el1 | |
307 | mrs x5, dbgbvr0_el1 | |
308 | ||
309 | adr x26, 1f | |
310 | add x26, x26, x24, lsl #2 | |
311 | br x26 | |
312 | ||
313 | 1: | |
314 | str x20, [x3, #(15 * 8)] | |
315 | str x19, [x3, #(14 * 8)] | |
316 | str x18, [x3, #(13 * 8)] | |
317 | str x17, [x3, #(12 * 8)] | |
318 | str x16, [x3, #(11 * 8)] | |
319 | str x15, [x3, #(10 * 8)] | |
320 | str x14, [x3, #(9 * 8)] | |
321 | str x13, [x3, #(8 * 8)] | |
322 | str x12, [x3, #(7 * 8)] | |
323 | str x11, [x3, #(6 * 8)] | |
324 | str x10, [x3, #(5 * 8)] | |
325 | str x9, [x3, #(4 * 8)] | |
326 | str x8, [x3, #(3 * 8)] | |
327 | str x7, [x3, #(2 * 8)] | |
328 | str x6, [x3, #(1 * 8)] | |
329 | str x5, [x3, #(0 * 8)] | |
330 | ||
331 | add x3, x2, #CPU_SYSREG_OFFSET(DBGWCR0_EL1) | |
332 | ||
333 | adr x26, 1f | |
334 | add x26, x26, x25, lsl #2 | |
335 | br x26 | |
336 | 1: | |
337 | mrs x20, dbgwcr15_el1 | |
338 | mrs x19, dbgwcr14_el1 | |
339 | mrs x18, dbgwcr13_el1 | |
340 | mrs x17, dbgwcr12_el1 | |
341 | mrs x16, dbgwcr11_el1 | |
342 | mrs x15, dbgwcr10_el1 | |
343 | mrs x14, dbgwcr9_el1 | |
344 | mrs x13, dbgwcr8_el1 | |
345 | mrs x12, dbgwcr7_el1 | |
346 | mrs x11, dbgwcr6_el1 | |
347 | mrs x10, dbgwcr5_el1 | |
348 | mrs x9, dbgwcr4_el1 | |
349 | mrs x8, dbgwcr3_el1 | |
350 | mrs x7, dbgwcr2_el1 | |
351 | mrs x6, dbgwcr1_el1 | |
352 | mrs x5, dbgwcr0_el1 | |
353 | ||
354 | adr x26, 1f | |
355 | add x26, x26, x25, lsl #2 | |
356 | br x26 | |
357 | ||
358 | 1: | |
359 | str x20, [x3, #(15 * 8)] | |
360 | str x19, [x3, #(14 * 8)] | |
361 | str x18, [x3, #(13 * 8)] | |
362 | str x17, [x3, #(12 * 8)] | |
363 | str x16, [x3, #(11 * 8)] | |
364 | str x15, [x3, #(10 * 8)] | |
365 | str x14, [x3, #(9 * 8)] | |
366 | str x13, [x3, #(8 * 8)] | |
367 | str x12, [x3, #(7 * 8)] | |
368 | str x11, [x3, #(6 * 8)] | |
369 | str x10, [x3, #(5 * 8)] | |
370 | str x9, [x3, #(4 * 8)] | |
371 | str x8, [x3, #(3 * 8)] | |
372 | str x7, [x3, #(2 * 8)] | |
373 | str x6, [x3, #(1 * 8)] | |
374 | str x5, [x3, #(0 * 8)] | |
375 | ||
376 | add x3, x2, #CPU_SYSREG_OFFSET(DBGWVR0_EL1) | |
377 | ||
378 | adr x26, 1f | |
379 | add x26, x26, x25, lsl #2 | |
380 | br x26 | |
381 | 1: | |
382 | mrs x20, dbgwvr15_el1 | |
383 | mrs x19, dbgwvr14_el1 | |
384 | mrs x18, dbgwvr13_el1 | |
385 | mrs x17, dbgwvr12_el1 | |
386 | mrs x16, dbgwvr11_el1 | |
387 | mrs x15, dbgwvr10_el1 | |
388 | mrs x14, dbgwvr9_el1 | |
389 | mrs x13, dbgwvr8_el1 | |
390 | mrs x12, dbgwvr7_el1 | |
391 | mrs x11, dbgwvr6_el1 | |
392 | mrs x10, dbgwvr5_el1 | |
393 | mrs x9, dbgwvr4_el1 | |
394 | mrs x8, dbgwvr3_el1 | |
395 | mrs x7, dbgwvr2_el1 | |
396 | mrs x6, dbgwvr1_el1 | |
397 | mrs x5, dbgwvr0_el1 | |
398 | ||
399 | adr x26, 1f | |
400 | add x26, x26, x25, lsl #2 | |
401 | br x26 | |
402 | ||
403 | 1: | |
404 | str x20, [x3, #(15 * 8)] | |
405 | str x19, [x3, #(14 * 8)] | |
406 | str x18, [x3, #(13 * 8)] | |
407 | str x17, [x3, #(12 * 8)] | |
408 | str x16, [x3, #(11 * 8)] | |
409 | str x15, [x3, #(10 * 8)] | |
410 | str x14, [x3, #(9 * 8)] | |
411 | str x13, [x3, #(8 * 8)] | |
412 | str x12, [x3, #(7 * 8)] | |
413 | str x11, [x3, #(6 * 8)] | |
414 | str x10, [x3, #(5 * 8)] | |
415 | str x9, [x3, #(4 * 8)] | |
416 | str x8, [x3, #(3 * 8)] | |
417 | str x7, [x3, #(2 * 8)] | |
418 | str x6, [x3, #(1 * 8)] | |
419 | str x5, [x3, #(0 * 8)] | |
420 | ||
421 | mrs x21, mdccint_el1 | |
422 | str x21, [x2, #CPU_SYSREG_OFFSET(MDCCINT_EL1)] | |
55c7401d MZ |
423 | .endm |
424 | ||
425 | .macro restore_sysregs | |
426 | // x2: base address for cpu context | |
427 | // x3: tmp register | |
428 | ||
429 | add x3, x2, #CPU_SYSREG_OFFSET(MPIDR_EL1) | |
430 | ||
431 | ldp x4, x5, [x3] | |
432 | ldp x6, x7, [x3, #16] | |
433 | ldp x8, x9, [x3, #32] | |
434 | ldp x10, x11, [x3, #48] | |
435 | ldp x12, x13, [x3, #64] | |
436 | ldp x14, x15, [x3, #80] | |
437 | ldp x16, x17, [x3, #96] | |
438 | ldp x18, x19, [x3, #112] | |
439 | ldp x20, x21, [x3, #128] | |
440 | ldp x22, x23, [x3, #144] | |
b0e626b3 | 441 | ldp x24, x25, [x3, #160] |
55c7401d MZ |
442 | |
443 | msr vmpidr_el2, x4 | |
444 | msr csselr_el1, x5 | |
445 | msr sctlr_el1, x6 | |
446 | msr actlr_el1, x7 | |
447 | msr cpacr_el1, x8 | |
448 | msr ttbr0_el1, x9 | |
449 | msr ttbr1_el1, x10 | |
450 | msr tcr_el1, x11 | |
451 | msr esr_el1, x12 | |
452 | msr afsr0_el1, x13 | |
453 | msr afsr1_el1, x14 | |
454 | msr far_el1, x15 | |
455 | msr mair_el1, x16 | |
456 | msr vbar_el1, x17 | |
457 | msr contextidr_el1, x18 | |
458 | msr tpidr_el0, x19 | |
459 | msr tpidrro_el0, x20 | |
460 | msr tpidr_el1, x21 | |
461 | msr amair_el1, x22 | |
462 | msr cntkctl_el1, x23 | |
1bbd8054 | 463 | msr par_el1, x24 |
b0e626b3 MZ |
464 | msr mdscr_el1, x25 |
465 | .endm | |
466 | ||
467 | .macro restore_debug | |
468 | // x2: base address for cpu context | |
469 | // x3: tmp register | |
470 | ||
471 | mrs x26, id_aa64dfr0_el1 | |
472 | ubfx x24, x26, #12, #4 // Extract BRPs | |
473 | ubfx x25, x26, #20, #4 // Extract WRPs | |
474 | mov w26, #15 | |
475 | sub w24, w26, w24 // How many BPs to skip | |
476 | sub w25, w26, w25 // How many WPs to skip | |
477 | ||
478 | add x3, x2, #CPU_SYSREG_OFFSET(DBGBCR0_EL1) | |
479 | ||
480 | adr x26, 1f | |
481 | add x26, x26, x24, lsl #2 | |
482 | br x26 | |
483 | 1: | |
484 | ldr x20, [x3, #(15 * 8)] | |
485 | ldr x19, [x3, #(14 * 8)] | |
486 | ldr x18, [x3, #(13 * 8)] | |
487 | ldr x17, [x3, #(12 * 8)] | |
488 | ldr x16, [x3, #(11 * 8)] | |
489 | ldr x15, [x3, #(10 * 8)] | |
490 | ldr x14, [x3, #(9 * 8)] | |
491 | ldr x13, [x3, #(8 * 8)] | |
492 | ldr x12, [x3, #(7 * 8)] | |
493 | ldr x11, [x3, #(6 * 8)] | |
494 | ldr x10, [x3, #(5 * 8)] | |
495 | ldr x9, [x3, #(4 * 8)] | |
496 | ldr x8, [x3, #(3 * 8)] | |
497 | ldr x7, [x3, #(2 * 8)] | |
498 | ldr x6, [x3, #(1 * 8)] | |
499 | ldr x5, [x3, #(0 * 8)] | |
500 | ||
501 | adr x26, 1f | |
502 | add x26, x26, x24, lsl #2 | |
503 | br x26 | |
504 | 1: | |
505 | msr dbgbcr15_el1, x20 | |
506 | msr dbgbcr14_el1, x19 | |
507 | msr dbgbcr13_el1, x18 | |
508 | msr dbgbcr12_el1, x17 | |
509 | msr dbgbcr11_el1, x16 | |
510 | msr dbgbcr10_el1, x15 | |
511 | msr dbgbcr9_el1, x14 | |
512 | msr dbgbcr8_el1, x13 | |
513 | msr dbgbcr7_el1, x12 | |
514 | msr dbgbcr6_el1, x11 | |
515 | msr dbgbcr5_el1, x10 | |
516 | msr dbgbcr4_el1, x9 | |
517 | msr dbgbcr3_el1, x8 | |
518 | msr dbgbcr2_el1, x7 | |
519 | msr dbgbcr1_el1, x6 | |
520 | msr dbgbcr0_el1, x5 | |
521 | ||
522 | add x3, x2, #CPU_SYSREG_OFFSET(DBGBVR0_EL1) | |
523 | ||
524 | adr x26, 1f | |
525 | add x26, x26, x24, lsl #2 | |
526 | br x26 | |
527 | 1: | |
528 | ldr x20, [x3, #(15 * 8)] | |
529 | ldr x19, [x3, #(14 * 8)] | |
530 | ldr x18, [x3, #(13 * 8)] | |
531 | ldr x17, [x3, #(12 * 8)] | |
532 | ldr x16, [x3, #(11 * 8)] | |
533 | ldr x15, [x3, #(10 * 8)] | |
534 | ldr x14, [x3, #(9 * 8)] | |
535 | ldr x13, [x3, #(8 * 8)] | |
536 | ldr x12, [x3, #(7 * 8)] | |
537 | ldr x11, [x3, #(6 * 8)] | |
538 | ldr x10, [x3, #(5 * 8)] | |
539 | ldr x9, [x3, #(4 * 8)] | |
540 | ldr x8, [x3, #(3 * 8)] | |
541 | ldr x7, [x3, #(2 * 8)] | |
542 | ldr x6, [x3, #(1 * 8)] | |
543 | ldr x5, [x3, #(0 * 8)] | |
544 | ||
545 | adr x26, 1f | |
546 | add x26, x26, x24, lsl #2 | |
547 | br x26 | |
548 | 1: | |
549 | msr dbgbvr15_el1, x20 | |
550 | msr dbgbvr14_el1, x19 | |
551 | msr dbgbvr13_el1, x18 | |
552 | msr dbgbvr12_el1, x17 | |
553 | msr dbgbvr11_el1, x16 | |
554 | msr dbgbvr10_el1, x15 | |
555 | msr dbgbvr9_el1, x14 | |
556 | msr dbgbvr8_el1, x13 | |
557 | msr dbgbvr7_el1, x12 | |
558 | msr dbgbvr6_el1, x11 | |
559 | msr dbgbvr5_el1, x10 | |
560 | msr dbgbvr4_el1, x9 | |
561 | msr dbgbvr3_el1, x8 | |
562 | msr dbgbvr2_el1, x7 | |
563 | msr dbgbvr1_el1, x6 | |
564 | msr dbgbvr0_el1, x5 | |
565 | ||
566 | add x3, x2, #CPU_SYSREG_OFFSET(DBGWCR0_EL1) | |
567 | ||
568 | adr x26, 1f | |
569 | add x26, x26, x25, lsl #2 | |
570 | br x26 | |
571 | 1: | |
572 | ldr x20, [x3, #(15 * 8)] | |
573 | ldr x19, [x3, #(14 * 8)] | |
574 | ldr x18, [x3, #(13 * 8)] | |
575 | ldr x17, [x3, #(12 * 8)] | |
576 | ldr x16, [x3, #(11 * 8)] | |
577 | ldr x15, [x3, #(10 * 8)] | |
578 | ldr x14, [x3, #(9 * 8)] | |
579 | ldr x13, [x3, #(8 * 8)] | |
580 | ldr x12, [x3, #(7 * 8)] | |
581 | ldr x11, [x3, #(6 * 8)] | |
582 | ldr x10, [x3, #(5 * 8)] | |
583 | ldr x9, [x3, #(4 * 8)] | |
584 | ldr x8, [x3, #(3 * 8)] | |
585 | ldr x7, [x3, #(2 * 8)] | |
586 | ldr x6, [x3, #(1 * 8)] | |
587 | ldr x5, [x3, #(0 * 8)] | |
588 | ||
589 | adr x26, 1f | |
590 | add x26, x26, x25, lsl #2 | |
591 | br x26 | |
592 | 1: | |
593 | msr dbgwcr15_el1, x20 | |
594 | msr dbgwcr14_el1, x19 | |
595 | msr dbgwcr13_el1, x18 | |
596 | msr dbgwcr12_el1, x17 | |
597 | msr dbgwcr11_el1, x16 | |
598 | msr dbgwcr10_el1, x15 | |
599 | msr dbgwcr9_el1, x14 | |
600 | msr dbgwcr8_el1, x13 | |
601 | msr dbgwcr7_el1, x12 | |
602 | msr dbgwcr6_el1, x11 | |
603 | msr dbgwcr5_el1, x10 | |
604 | msr dbgwcr4_el1, x9 | |
605 | msr dbgwcr3_el1, x8 | |
606 | msr dbgwcr2_el1, x7 | |
607 | msr dbgwcr1_el1, x6 | |
608 | msr dbgwcr0_el1, x5 | |
609 | ||
610 | add x3, x2, #CPU_SYSREG_OFFSET(DBGWVR0_EL1) | |
611 | ||
612 | adr x26, 1f | |
613 | add x26, x26, x25, lsl #2 | |
614 | br x26 | |
615 | 1: | |
616 | ldr x20, [x3, #(15 * 8)] | |
617 | ldr x19, [x3, #(14 * 8)] | |
618 | ldr x18, [x3, #(13 * 8)] | |
619 | ldr x17, [x3, #(12 * 8)] | |
620 | ldr x16, [x3, #(11 * 8)] | |
621 | ldr x15, [x3, #(10 * 8)] | |
622 | ldr x14, [x3, #(9 * 8)] | |
623 | ldr x13, [x3, #(8 * 8)] | |
624 | ldr x12, [x3, #(7 * 8)] | |
625 | ldr x11, [x3, #(6 * 8)] | |
626 | ldr x10, [x3, #(5 * 8)] | |
627 | ldr x9, [x3, #(4 * 8)] | |
628 | ldr x8, [x3, #(3 * 8)] | |
629 | ldr x7, [x3, #(2 * 8)] | |
630 | ldr x6, [x3, #(1 * 8)] | |
631 | ldr x5, [x3, #(0 * 8)] | |
632 | ||
633 | adr x26, 1f | |
634 | add x26, x26, x25, lsl #2 | |
635 | br x26 | |
636 | 1: | |
637 | msr dbgwvr15_el1, x20 | |
638 | msr dbgwvr14_el1, x19 | |
639 | msr dbgwvr13_el1, x18 | |
640 | msr dbgwvr12_el1, x17 | |
641 | msr dbgwvr11_el1, x16 | |
642 | msr dbgwvr10_el1, x15 | |
643 | msr dbgwvr9_el1, x14 | |
644 | msr dbgwvr8_el1, x13 | |
645 | msr dbgwvr7_el1, x12 | |
646 | msr dbgwvr6_el1, x11 | |
647 | msr dbgwvr5_el1, x10 | |
648 | msr dbgwvr4_el1, x9 | |
649 | msr dbgwvr3_el1, x8 | |
650 | msr dbgwvr2_el1, x7 | |
651 | msr dbgwvr1_el1, x6 | |
652 | msr dbgwvr0_el1, x5 | |
653 | ||
654 | ldr x21, [x2, #CPU_SYSREG_OFFSET(MDCCINT_EL1)] | |
655 | msr mdccint_el1, x21 | |
55c7401d MZ |
656 | .endm |
657 | ||
b4afad06 MZ |
658 | .macro skip_32bit_state tmp, target |
659 | // Skip 32bit state if not needed | |
660 | mrs \tmp, hcr_el2 | |
661 | tbnz \tmp, #HCR_RW_SHIFT, \target | |
662 | .endm | |
663 | ||
664 | .macro skip_tee_state tmp, target | |
665 | // Skip ThumbEE state if not needed | |
666 | mrs \tmp, id_pfr0_el1 | |
667 | tbz \tmp, #12, \target | |
668 | .endm | |
669 | ||
b0e626b3 MZ |
670 | .macro skip_debug_state tmp, target |
671 | ldr \tmp, [x0, #VCPU_DEBUG_FLAGS] | |
672 | tbz \tmp, #KVM_ARM64_DEBUG_DIRTY_SHIFT, \target | |
673 | .endm | |
674 | ||
675 | .macro compute_debug_state target | |
676 | // Compute debug state: If any of KDE, MDE or KVM_ARM64_DEBUG_DIRTY | |
677 | // is set, we do a full save/restore cycle and disable trapping. | |
678 | add x25, x0, #VCPU_CONTEXT | |
679 | ||
680 | // Check the state of MDSCR_EL1 | |
681 | ldr x25, [x25, #CPU_SYSREG_OFFSET(MDSCR_EL1)] | |
682 | and x26, x25, #DBG_MDSCR_KDE | |
683 | and x25, x25, #DBG_MDSCR_MDE | |
684 | adds xzr, x25, x26 | |
685 | b.eq 9998f // Nothing to see there | |
686 | ||
687 | // If any interesting bits was set, we must set the flag | |
688 | mov x26, #KVM_ARM64_DEBUG_DIRTY | |
689 | str x26, [x0, #VCPU_DEBUG_FLAGS] | |
690 | b 9999f // Don't skip restore | |
691 | ||
692 | 9998: | |
693 | // Otherwise load the flags from memory in case we recently | |
694 | // trapped | |
695 | skip_debug_state x25, \target | |
696 | 9999: | |
697 | .endm | |
698 | ||
b4afad06 MZ |
699 | .macro save_guest_32bit_state |
700 | skip_32bit_state x3, 1f | |
701 | ||
702 | add x3, x2, #CPU_SPSR_OFFSET(KVM_SPSR_ABT) | |
703 | mrs x4, spsr_abt | |
704 | mrs x5, spsr_und | |
705 | mrs x6, spsr_irq | |
706 | mrs x7, spsr_fiq | |
707 | stp x4, x5, [x3] | |
708 | stp x6, x7, [x3, #16] | |
709 | ||
710 | add x3, x2, #CPU_SYSREG_OFFSET(DACR32_EL2) | |
711 | mrs x4, dacr32_el2 | |
712 | mrs x5, ifsr32_el2 | |
713 | mrs x6, fpexc32_el2 | |
b4afad06 | 714 | stp x4, x5, [x3] |
b0e626b3 | 715 | str x6, [x3, #16] |
b4afad06 | 716 | |
b0e626b3 MZ |
717 | skip_debug_state x8, 2f |
718 | mrs x7, dbgvcr32_el2 | |
719 | str x7, [x3, #24] | |
720 | 2: | |
b4afad06 MZ |
721 | skip_tee_state x8, 1f |
722 | ||
723 | add x3, x2, #CPU_SYSREG_OFFSET(TEECR32_EL1) | |
724 | mrs x4, teecr32_el1 | |
725 | mrs x5, teehbr32_el1 | |
726 | stp x4, x5, [x3] | |
727 | 1: | |
728 | .endm | |
729 | ||
730 | .macro restore_guest_32bit_state | |
731 | skip_32bit_state x3, 1f | |
732 | ||
733 | add x3, x2, #CPU_SPSR_OFFSET(KVM_SPSR_ABT) | |
734 | ldp x4, x5, [x3] | |
735 | ldp x6, x7, [x3, #16] | |
736 | msr spsr_abt, x4 | |
737 | msr spsr_und, x5 | |
738 | msr spsr_irq, x6 | |
739 | msr spsr_fiq, x7 | |
740 | ||
741 | add x3, x2, #CPU_SYSREG_OFFSET(DACR32_EL2) | |
742 | ldp x4, x5, [x3] | |
b0e626b3 | 743 | ldr x6, [x3, #16] |
b4afad06 MZ |
744 | msr dacr32_el2, x4 |
745 | msr ifsr32_el2, x5 | |
746 | msr fpexc32_el2, x6 | |
b4afad06 | 747 | |
b0e626b3 MZ |
748 | skip_debug_state x8, 2f |
749 | ldr x7, [x3, #24] | |
750 | msr dbgvcr32_el2, x7 | |
751 | 2: | |
b4afad06 MZ |
752 | skip_tee_state x8, 1f |
753 | ||
754 | add x3, x2, #CPU_SYSREG_OFFSET(TEECR32_EL1) | |
755 | ldp x4, x5, [x3] | |
756 | msr teecr32_el1, x4 | |
757 | msr teehbr32_el1, x5 | |
758 | 1: | |
759 | .endm | |
760 | ||
55c7401d | 761 | .macro activate_traps |
ac3c3747 MZ |
762 | ldr x2, [x0, #VCPU_HCR_EL2] |
763 | msr hcr_el2, x2 | |
302cd37c | 764 | mov x2, #CPTR_EL2_TTA |
55c7401d MZ |
765 | msr cptr_el2, x2 |
766 | ||
302cd37c | 767 | mov x2, #(1 << 15) // Trap CP15 Cr=15 |
55c7401d MZ |
768 | msr hstr_el2, x2 |
769 | ||
770 | mrs x2, mdcr_el2 | |
771 | and x2, x2, #MDCR_EL2_HPMN_MASK | |
772 | orr x2, x2, #(MDCR_EL2_TPM | MDCR_EL2_TPMCR) | |
d329de09 MZ |
773 | orr x2, x2, #(MDCR_EL2_TDRA | MDCR_EL2_TDOSA) |
774 | ||
775 | // Check for KVM_ARM64_DEBUG_DIRTY, and set debug to trap | |
776 | // if not dirty. | |
777 | ldr x3, [x0, #VCPU_DEBUG_FLAGS] | |
778 | tbnz x3, #KVM_ARM64_DEBUG_DIRTY_SHIFT, 1f | |
779 | orr x2, x2, #MDCR_EL2_TDA | |
780 | 1: | |
55c7401d MZ |
781 | msr mdcr_el2, x2 |
782 | .endm | |
783 | ||
784 | .macro deactivate_traps | |
785 | mov x2, #HCR_RW | |
786 | msr hcr_el2, x2 | |
787 | msr cptr_el2, xzr | |
788 | msr hstr_el2, xzr | |
789 | ||
790 | mrs x2, mdcr_el2 | |
791 | and x2, x2, #MDCR_EL2_HPMN_MASK | |
792 | msr mdcr_el2, x2 | |
793 | .endm | |
794 | ||
795 | .macro activate_vm | |
796 | ldr x1, [x0, #VCPU_KVM] | |
797 | kern_hyp_va x1 | |
798 | ldr x2, [x1, #KVM_VTTBR] | |
799 | msr vttbr_el2, x2 | |
800 | .endm | |
801 | ||
802 | .macro deactivate_vm | |
803 | msr vttbr_el2, xzr | |
804 | .endm | |
805 | ||
1f17f3b6 | 806 | /* |
1a9b1305 | 807 | * Call into the vgic backend for state saving |
1f17f3b6 MZ |
808 | */ |
809 | .macro save_vgic_state | |
1a9b1305 MZ |
810 | adr x24, __vgic_sr_vectors |
811 | ldr x24, [x24, VGIC_SAVE_FN] | |
812 | kern_hyp_va x24 | |
813 | blr x24 | |
ac3c3747 MZ |
814 | mrs x24, hcr_el2 |
815 | mov x25, #HCR_INT_OVERRIDE | |
816 | neg x25, x25 | |
817 | and x24, x24, x25 | |
818 | msr hcr_el2, x24 | |
1f17f3b6 MZ |
819 | .endm |
820 | ||
821 | /* | |
1a9b1305 | 822 | * Call into the vgic backend for state restoring |
1f17f3b6 MZ |
823 | */ |
824 | .macro restore_vgic_state | |
ac3c3747 MZ |
825 | mrs x24, hcr_el2 |
826 | ldr x25, [x0, #VCPU_IRQ_LINES] | |
827 | orr x24, x24, #HCR_INT_OVERRIDE | |
828 | orr x24, x24, x25 | |
829 | msr hcr_el2, x24 | |
1a9b1305 MZ |
830 | adr x24, __vgic_sr_vectors |
831 | ldr x24, [x24, #VGIC_RESTORE_FN] | |
832 | kern_hyp_va x24 | |
833 | blr x24 | |
1f17f3b6 MZ |
834 | .endm |
835 | ||
003300de MZ |
836 | .macro save_timer_state |
837 | // x0: vcpu pointer | |
838 | ldr x2, [x0, #VCPU_KVM] | |
839 | kern_hyp_va x2 | |
840 | ldr w3, [x2, #KVM_TIMER_ENABLED] | |
841 | cbz w3, 1f | |
842 | ||
843 | mrs x3, cntv_ctl_el0 | |
844 | and x3, x3, #3 | |
845 | str w3, [x0, #VCPU_TIMER_CNTV_CTL] | |
846 | bic x3, x3, #1 // Clear Enable | |
847 | msr cntv_ctl_el0, x3 | |
848 | ||
849 | isb | |
850 | ||
851 | mrs x3, cntv_cval_el0 | |
852 | str x3, [x0, #VCPU_TIMER_CNTV_CVAL] | |
853 | ||
854 | 1: | |
855 | // Allow physical timer/counter access for the host | |
856 | mrs x2, cnthctl_el2 | |
857 | orr x2, x2, #3 | |
858 | msr cnthctl_el2, x2 | |
859 | ||
860 | // Clear cntvoff for the host | |
861 | msr cntvoff_el2, xzr | |
862 | .endm | |
863 | ||
864 | .macro restore_timer_state | |
865 | // x0: vcpu pointer | |
866 | // Disallow physical timer access for the guest | |
867 | // Physical counter access is allowed | |
868 | mrs x2, cnthctl_el2 | |
869 | orr x2, x2, #1 | |
870 | bic x2, x2, #2 | |
871 | msr cnthctl_el2, x2 | |
872 | ||
873 | ldr x2, [x0, #VCPU_KVM] | |
874 | kern_hyp_va x2 | |
875 | ldr w3, [x2, #KVM_TIMER_ENABLED] | |
876 | cbz w3, 1f | |
877 | ||
878 | ldr x3, [x2, #KVM_TIMER_CNTVOFF] | |
879 | msr cntvoff_el2, x3 | |
880 | ldr x2, [x0, #VCPU_TIMER_CNTV_CVAL] | |
881 | msr cntv_cval_el0, x2 | |
882 | isb | |
883 | ||
884 | ldr w2, [x0, #VCPU_TIMER_CNTV_CTL] | |
885 | and x2, x2, #3 | |
886 | msr cntv_ctl_el0, x2 | |
887 | 1: | |
888 | .endm | |
889 | ||
55c7401d MZ |
890 | __save_sysregs: |
891 | save_sysregs | |
892 | ret | |
893 | ||
894 | __restore_sysregs: | |
895 | restore_sysregs | |
896 | ret | |
897 | ||
b0e626b3 MZ |
898 | __save_debug: |
899 | save_debug | |
900 | ret | |
901 | ||
902 | __restore_debug: | |
903 | restore_debug | |
904 | ret | |
905 | ||
55c7401d MZ |
906 | __save_fpsimd: |
907 | save_fpsimd | |
908 | ret | |
909 | ||
910 | __restore_fpsimd: | |
911 | restore_fpsimd | |
912 | ret | |
913 | ||
914 | /* | |
915 | * u64 __kvm_vcpu_run(struct kvm_vcpu *vcpu); | |
916 | * | |
917 | * This is the world switch. The first half of the function | |
918 | * deals with entering the guest, and anything from __kvm_vcpu_return | |
919 | * to the end of the function deals with reentering the host. | |
920 | * On the enter path, only x0 (vcpu pointer) must be preserved until | |
921 | * the last moment. On the exit path, x0 (vcpu pointer) and x1 (exception | |
922 | * code) must both be preserved until the epilogue. | |
923 | * In both cases, x2 points to the CPU context we're saving/restoring from/to. | |
924 | */ | |
925 | ENTRY(__kvm_vcpu_run) | |
926 | kern_hyp_va x0 | |
927 | msr tpidr_el2, x0 // Save the vcpu register | |
928 | ||
929 | // Host context | |
930 | ldr x2, [x0, #VCPU_HOST_CONTEXT] | |
931 | kern_hyp_va x2 | |
932 | ||
933 | save_host_regs | |
934 | bl __save_fpsimd | |
935 | bl __save_sysregs | |
936 | ||
b0e626b3 MZ |
937 | compute_debug_state 1f |
938 | bl __save_debug | |
939 | 1: | |
55c7401d MZ |
940 | activate_traps |
941 | activate_vm | |
942 | ||
1f17f3b6 | 943 | restore_vgic_state |
003300de | 944 | restore_timer_state |
1f17f3b6 | 945 | |
55c7401d MZ |
946 | // Guest context |
947 | add x2, x0, #VCPU_CONTEXT | |
948 | ||
949 | bl __restore_sysregs | |
950 | bl __restore_fpsimd | |
b0e626b3 MZ |
951 | |
952 | skip_debug_state x3, 1f | |
953 | bl __restore_debug | |
954 | 1: | |
b4afad06 | 955 | restore_guest_32bit_state |
55c7401d MZ |
956 | restore_guest_regs |
957 | ||
958 | // That's it, no more messing around. | |
959 | eret | |
960 | ||
961 | __kvm_vcpu_return: | |
962 | // Assume x0 is the vcpu pointer, x1 the return code | |
963 | // Guest's x0-x3 are on the stack | |
964 | ||
965 | // Guest context | |
966 | add x2, x0, #VCPU_CONTEXT | |
967 | ||
968 | save_guest_regs | |
969 | bl __save_fpsimd | |
970 | bl __save_sysregs | |
b0e626b3 MZ |
971 | |
972 | skip_debug_state x3, 1f | |
973 | bl __save_debug | |
974 | 1: | |
b4afad06 | 975 | save_guest_32bit_state |
55c7401d | 976 | |
003300de | 977 | save_timer_state |
1f17f3b6 MZ |
978 | save_vgic_state |
979 | ||
55c7401d MZ |
980 | deactivate_traps |
981 | deactivate_vm | |
982 | ||
983 | // Host context | |
984 | ldr x2, [x0, #VCPU_HOST_CONTEXT] | |
985 | kern_hyp_va x2 | |
986 | ||
987 | bl __restore_sysregs | |
988 | bl __restore_fpsimd | |
b0e626b3 MZ |
989 | |
990 | skip_debug_state x3, 1f | |
991 | // Clear the dirty flag for the next run, as all the state has | |
992 | // already been saved. Note that we nuke the whole 64bit word. | |
993 | // If we ever add more flags, we'll have to be more careful... | |
994 | str xzr, [x0, #VCPU_DEBUG_FLAGS] | |
995 | bl __restore_debug | |
996 | 1: | |
55c7401d MZ |
997 | restore_host_regs |
998 | ||
999 | mov x0, x1 | |
1000 | ret | |
1001 | END(__kvm_vcpu_run) | |
1002 | ||
1003 | // void __kvm_tlb_flush_vmid_ipa(struct kvm *kvm, phys_addr_t ipa); | |
1004 | ENTRY(__kvm_tlb_flush_vmid_ipa) | |
f142e5ee MZ |
1005 | dsb ishst |
1006 | ||
55c7401d MZ |
1007 | kern_hyp_va x0 |
1008 | ldr x2, [x0, #KVM_VTTBR] | |
1009 | msr vttbr_el2, x2 | |
1010 | isb | |
1011 | ||
1012 | /* | |
1013 | * We could do so much better if we had the VA as well. | |
1014 | * Instead, we invalidate Stage-2 for this IPA, and the | |
1015 | * whole of Stage-1. Weep... | |
1016 | */ | |
1017 | tlbi ipas2e1is, x1 | |
ee9e101c WD |
1018 | /* |
1019 | * We have to ensure completion of the invalidation at Stage-2, | |
1020 | * since a table walk on another CPU could refill a TLB with a | |
1021 | * complete (S1 + S2) walk based on the old Stage-2 mapping if | |
1022 | * the Stage-1 invalidation happened first. | |
1023 | */ | |
1024 | dsb ish | |
55c7401d | 1025 | tlbi vmalle1is |
ee9e101c | 1026 | dsb ish |
55c7401d MZ |
1027 | isb |
1028 | ||
1029 | msr vttbr_el2, xzr | |
1030 | ret | |
1031 | ENDPROC(__kvm_tlb_flush_vmid_ipa) | |
1032 | ||
1033 | ENTRY(__kvm_flush_vm_context) | |
f142e5ee | 1034 | dsb ishst |
55c7401d MZ |
1035 | tlbi alle1is |
1036 | ic ialluis | |
ee9e101c | 1037 | dsb ish |
55c7401d MZ |
1038 | ret |
1039 | ENDPROC(__kvm_flush_vm_context) | |
1040 | ||
1a9b1305 MZ |
1041 | // struct vgic_sr_vectors __vgi_sr_vectors; |
1042 | .align 3 | |
1043 | ENTRY(__vgic_sr_vectors) | |
1044 | .skip VGIC_SR_VECTOR_SZ | |
1045 | ENDPROC(__vgic_sr_vectors) | |
1046 | ||
55c7401d MZ |
1047 | __kvm_hyp_panic: |
1048 | // Guess the context by looking at VTTBR: | |
1049 | // If zero, then we're already a host. | |
1050 | // Otherwise restore a minimal host context before panicing. | |
1051 | mrs x0, vttbr_el2 | |
1052 | cbz x0, 1f | |
1053 | ||
1054 | mrs x0, tpidr_el2 | |
1055 | ||
1056 | deactivate_traps | |
1057 | deactivate_vm | |
1058 | ||
1059 | ldr x2, [x0, #VCPU_HOST_CONTEXT] | |
1060 | kern_hyp_va x2 | |
1061 | ||
1062 | bl __restore_sysregs | |
1063 | ||
1064 | 1: adr x0, __hyp_panic_str | |
1065 | adr x1, 2f | |
1066 | ldp x2, x3, [x1] | |
1067 | sub x0, x0, x2 | |
1068 | add x0, x0, x3 | |
1069 | mrs x1, spsr_el2 | |
1070 | mrs x2, elr_el2 | |
1071 | mrs x3, esr_el2 | |
1072 | mrs x4, far_el2 | |
1073 | mrs x5, hpfar_el2 | |
1074 | mrs x6, par_el1 | |
1075 | mrs x7, tpidr_el2 | |
1076 | ||
1077 | mov lr, #(PSR_F_BIT | PSR_I_BIT | PSR_A_BIT | PSR_D_BIT |\ | |
1078 | PSR_MODE_EL1h) | |
1079 | msr spsr_el2, lr | |
1080 | ldr lr, =panic | |
1081 | msr elr_el2, lr | |
1082 | eret | |
1083 | ||
1084 | .align 3 | |
1085 | 2: .quad HYP_PAGE_OFFSET | |
1086 | .quad PAGE_OFFSET | |
1087 | ENDPROC(__kvm_hyp_panic) | |
1088 | ||
1089 | __hyp_panic_str: | |
1090 | .ascii "HYP panic:\nPS:%08x PC:%p ESR:%p\nFAR:%p HPFAR:%p PAR:%p\nVCPU:%p\n\0" | |
1091 | ||
1092 | .align 2 | |
1093 | ||
b20c9f29 MZ |
1094 | /* |
1095 | * u64 kvm_call_hyp(void *hypfn, ...); | |
1096 | * | |
1097 | * This is not really a variadic function in the classic C-way and care must | |
1098 | * be taken when calling this to ensure parameters are passed in registers | |
1099 | * only, since the stack will change between the caller and the callee. | |
1100 | * | |
1101 | * Call the function with the first argument containing a pointer to the | |
1102 | * function you wish to call in Hyp mode, and subsequent arguments will be | |
1103 | * passed as x0, x1, and x2 (a maximum of 3 arguments in addition to the | |
1104 | * function pointer can be passed). The function being called must be mapped | |
1105 | * in Hyp mode (see init_hyp_mode in arch/arm/kvm/arm.c). Return values are | |
1106 | * passed in r0 and r1. | |
1107 | * | |
1108 | * A function pointer with a value of 0 has a special meaning, and is | |
1109 | * used to implement __hyp_get_vectors in the same way as in | |
1110 | * arch/arm64/kernel/hyp_stub.S. | |
1111 | */ | |
55c7401d MZ |
1112 | ENTRY(kvm_call_hyp) |
1113 | hvc #0 | |
1114 | ret | |
1115 | ENDPROC(kvm_call_hyp) | |
1116 | ||
1117 | .macro invalid_vector label, target | |
1118 | .align 2 | |
1119 | \label: | |
1120 | b \target | |
1121 | ENDPROC(\label) | |
1122 | .endm | |
1123 | ||
1124 | /* None of these should ever happen */ | |
1125 | invalid_vector el2t_sync_invalid, __kvm_hyp_panic | |
1126 | invalid_vector el2t_irq_invalid, __kvm_hyp_panic | |
1127 | invalid_vector el2t_fiq_invalid, __kvm_hyp_panic | |
1128 | invalid_vector el2t_error_invalid, __kvm_hyp_panic | |
1129 | invalid_vector el2h_sync_invalid, __kvm_hyp_panic | |
1130 | invalid_vector el2h_irq_invalid, __kvm_hyp_panic | |
1131 | invalid_vector el2h_fiq_invalid, __kvm_hyp_panic | |
1132 | invalid_vector el2h_error_invalid, __kvm_hyp_panic | |
1133 | invalid_vector el1_sync_invalid, __kvm_hyp_panic | |
1134 | invalid_vector el1_irq_invalid, __kvm_hyp_panic | |
1135 | invalid_vector el1_fiq_invalid, __kvm_hyp_panic | |
1136 | invalid_vector el1_error_invalid, __kvm_hyp_panic | |
1137 | ||
1138 | el1_sync: // Guest trapped into EL2 | |
1139 | push x0, x1 | |
1140 | push x2, x3 | |
1141 | ||
1142 | mrs x1, esr_el2 | |
1143 | lsr x2, x1, #ESR_EL2_EC_SHIFT | |
1144 | ||
1145 | cmp x2, #ESR_EL2_EC_HVC64 | |
1146 | b.ne el1_trap | |
1147 | ||
1148 | mrs x3, vttbr_el2 // If vttbr is valid, the 64bit guest | |
1149 | cbnz x3, el1_trap // called HVC | |
1150 | ||
1151 | /* Here, we're pretty sure the host called HVC. */ | |
1152 | pop x2, x3 | |
1153 | pop x0, x1 | |
1154 | ||
b20c9f29 MZ |
1155 | /* Check for __hyp_get_vectors */ |
1156 | cbnz x0, 1f | |
1157 | mrs x0, vbar_el2 | |
1158 | b 2f | |
1159 | ||
1160 | 1: push lr, xzr | |
55c7401d MZ |
1161 | |
1162 | /* | |
1163 | * Compute the function address in EL2, and shuffle the parameters. | |
1164 | */ | |
1165 | kern_hyp_va x0 | |
1166 | mov lr, x0 | |
1167 | mov x0, x1 | |
1168 | mov x1, x2 | |
1169 | mov x2, x3 | |
1170 | blr lr | |
1171 | ||
1172 | pop lr, xzr | |
b20c9f29 | 1173 | 2: eret |
55c7401d MZ |
1174 | |
1175 | el1_trap: | |
1176 | /* | |
1177 | * x1: ESR | |
1178 | * x2: ESR_EC | |
1179 | */ | |
1180 | cmp x2, #ESR_EL2_EC_DABT | |
1181 | mov x0, #ESR_EL2_EC_IABT | |
1182 | ccmp x2, x0, #4, ne | |
1183 | b.ne 1f // Not an abort we care about | |
1184 | ||
1185 | /* This is an abort. Check for permission fault */ | |
1186 | and x2, x1, #ESR_EL2_FSC_TYPE | |
1187 | cmp x2, #FSC_PERM | |
1188 | b.ne 1f // Not a permission fault | |
1189 | ||
1190 | /* | |
1191 | * Check for Stage-1 page table walk, which is guaranteed | |
1192 | * to give a valid HPFAR_EL2. | |
1193 | */ | |
1194 | tbnz x1, #7, 1f // S1PTW is set | |
1195 | ||
1bbd8054 MZ |
1196 | /* Preserve PAR_EL1 */ |
1197 | mrs x3, par_el1 | |
1198 | push x3, xzr | |
1199 | ||
55c7401d MZ |
1200 | /* |
1201 | * Permission fault, HPFAR_EL2 is invalid. | |
1202 | * Resolve the IPA the hard way using the guest VA. | |
1203 | * Stage-1 translation already validated the memory access rights. | |
1204 | * As such, we can use the EL1 translation regime, and don't have | |
1205 | * to distinguish between EL0 and EL1 access. | |
1206 | */ | |
1207 | mrs x2, far_el2 | |
1208 | at s1e1r, x2 | |
1209 | isb | |
1210 | ||
1211 | /* Read result */ | |
1212 | mrs x3, par_el1 | |
1bbd8054 MZ |
1213 | pop x0, xzr // Restore PAR_EL1 from the stack |
1214 | msr par_el1, x0 | |
55c7401d MZ |
1215 | tbnz x3, #0, 3f // Bail out if we failed the translation |
1216 | ubfx x3, x3, #12, #36 // Extract IPA | |
1217 | lsl x3, x3, #4 // and present it like HPFAR | |
1218 | b 2f | |
1219 | ||
1220 | 1: mrs x3, hpfar_el2 | |
1221 | mrs x2, far_el2 | |
1222 | ||
1223 | 2: mrs x0, tpidr_el2 | |
ba083d20 | 1224 | str w1, [x0, #VCPU_ESR_EL2] |
55c7401d MZ |
1225 | str x2, [x0, #VCPU_FAR_EL2] |
1226 | str x3, [x0, #VCPU_HPFAR_EL2] | |
1227 | ||
1228 | mov x1, #ARM_EXCEPTION_TRAP | |
1229 | b __kvm_vcpu_return | |
1230 | ||
1231 | /* | |
1232 | * Translation failed. Just return to the guest and | |
1233 | * let it fault again. Another CPU is probably playing | |
1234 | * behind our back. | |
1235 | */ | |
1236 | 3: pop x2, x3 | |
1237 | pop x0, x1 | |
1238 | ||
1239 | eret | |
1240 | ||
1241 | el1_irq: | |
1242 | push x0, x1 | |
1243 | push x2, x3 | |
1244 | mrs x0, tpidr_el2 | |
1245 | mov x1, #ARM_EXCEPTION_IRQ | |
1246 | b __kvm_vcpu_return | |
1247 | ||
1248 | .ltorg | |
1249 | ||
1250 | .align 11 | |
1251 | ||
1252 | ENTRY(__kvm_hyp_vector) | |
1253 | ventry el2t_sync_invalid // Synchronous EL2t | |
1254 | ventry el2t_irq_invalid // IRQ EL2t | |
1255 | ventry el2t_fiq_invalid // FIQ EL2t | |
1256 | ventry el2t_error_invalid // Error EL2t | |
1257 | ||
1258 | ventry el2h_sync_invalid // Synchronous EL2h | |
1259 | ventry el2h_irq_invalid // IRQ EL2h | |
1260 | ventry el2h_fiq_invalid // FIQ EL2h | |
1261 | ventry el2h_error_invalid // Error EL2h | |
1262 | ||
1263 | ventry el1_sync // Synchronous 64-bit EL1 | |
1264 | ventry el1_irq // IRQ 64-bit EL1 | |
1265 | ventry el1_fiq_invalid // FIQ 64-bit EL1 | |
1266 | ventry el1_error_invalid // Error 64-bit EL1 | |
1267 | ||
1268 | ventry el1_sync // Synchronous 32-bit EL1 | |
1269 | ventry el1_irq // IRQ 32-bit EL1 | |
1270 | ventry el1_fiq_invalid // FIQ 32-bit EL1 | |
1271 | ventry el1_error_invalid // Error 32-bit EL1 | |
1272 | ENDPROC(__kvm_hyp_vector) | |
1273 | ||
55c7401d | 1274 | .popsection |