arch/arm64/kvm/hyp.S

   1 /*
   2  * Copyright (C) 2012,2013 - ARM Ltd
   3  * Author: Marc Zyngier <marc.zyngier@arm.com>
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License version 2 as
   7  * published by the Free Software Foundation.
   8  *
   9  * This program is distributed in the hope that it will be useful,
  10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12  * GNU General Public License for more details.
  13  *
  14  * You should have received a copy of the GNU General Public License
  15  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  16  */
  17
  18 #include <linux/linkage.h>
  19
  20 #include <asm/assembler.h>
  21 #include <asm/memory.h>
  22 #include <asm/asm-offsets.h>
  23 #include <asm/debug-monitors.h>
  24 #include <asm/fpsimdmacros.h>
  25 #include <asm/kvm.h>
  26 #include <asm/kvm_asm.h>
  27 #include <asm/kvm_arm.h>
  28 #include <asm/kvm_mmu.h>
  29
  30 #define CPU_GP_REG_OFFSET(x)    (CPU_GP_REGS + x)
  31 #define CPU_XREG_OFFSET(x)      CPU_GP_REG_OFFSET(CPU_USER_PT_REGS + 8*x)
  32 #define CPU_SPSR_OFFSET(x)      CPU_GP_REG_OFFSET(CPU_SPSR + 8*x)
  33 #define CPU_SYSREG_OFFSET(x)    (CPU_SYSREGS + 8*x)
  34
  35         .text
  36         .pushsection    .hyp.text, "ax"
  37         .align  PAGE_SHIFT
  38
  39 .macro save_common_regs
  40         // x2: base address for cpu context
  41         // x3: tmp register
  42
  43         add     x3, x2, #CPU_XREG_OFFSET(19)
  44         stp     x19, x20, [x3]
  45         stp     x21, x22, [x3, #16]
  46         stp     x23, x24, [x3, #32]
  47         stp     x25, x26, [x3, #48]
  48         stp     x27, x28, [x3, #64]
  49         stp     x29, lr, [x3, #80]
  50
  51         mrs     x19, sp_el0
  52         mrs     x20, elr_el2            // EL1 PC
  53         mrs     x21, spsr_el2           // EL1 pstate
  54
  55         stp     x19, x20, [x3, #96]
  56         str     x21, [x3, #112]
  57
  58         mrs     x22, sp_el1
  59         mrs     x23, elr_el1
  60         mrs     x24, spsr_el1
  61
  62         str     x22, [x2, #CPU_GP_REG_OFFSET(CPU_SP_EL1)]
  63         str     x23, [x2, #CPU_GP_REG_OFFSET(CPU_ELR_EL1)]
  64         str     x24, [x2, #CPU_SPSR_OFFSET(KVM_SPSR_EL1)]
  65 .endm
  66
  67 .macro restore_common_regs
  68         // x2: base address for cpu context
  69         // x3: tmp register
  70
  71         ldr     x22, [x2, #CPU_GP_REG_OFFSET(CPU_SP_EL1)]
  72         ldr     x23, [x2, #CPU_GP_REG_OFFSET(CPU_ELR_EL1)]
  73         ldr     x24, [x2, #CPU_SPSR_OFFSET(KVM_SPSR_EL1)]
  74
  75         msr     sp_el1, x22
  76         msr     elr_el1, x23
  77         msr     spsr_el1, x24
  78
  79         add     x3, x2, #CPU_XREG_OFFSET(31)    // SP_EL0
  80         ldp     x19, x20, [x3]
  81         ldr     x21, [x3, #16]
  82
  83         msr     sp_el0, x19
  84         msr     elr_el2, x20                            // EL1 PC
  85         msr     spsr_el2, x21                           // EL1 pstate
  86
  87         add     x3, x2, #CPU_XREG_OFFSET(19)
  88         ldp     x19, x20, [x3]
  89         ldp     x21, x22, [x3, #16]
  90         ldp     x23, x24, [x3, #32]
  91         ldp     x25, x26, [x3, #48]
  92         ldp     x27, x28, [x3, #64]
  93         ldp     x29, lr, [x3, #80]
  94 .endm
  95
  96 .macro save_host_regs
  97         save_common_regs
  98 .endm
  99
 100 .macro restore_host_regs
 101         restore_common_regs
 102 .endm
 103
 104 .macro save_fpsimd
 105         // x2: cpu context address
 106         // x3, x4: tmp regs
 107         add     x3, x2, #CPU_GP_REG_OFFSET(CPU_FP_REGS)
 108         fpsimd_save x3, 4
 109 .endm
 110
 111 .macro restore_fpsimd
 112         // x2: cpu context address
 113         // x3, x4: tmp regs
 114         add     x3, x2, #CPU_GP_REG_OFFSET(CPU_FP_REGS)
 115         fpsimd_restore x3, 4
 116 .endm
 117
 118 .macro save_guest_regs
 119         // x0 is the vcpu address
 120         // x1 is the return code, do not corrupt!
 121         // x2 is the cpu context
 122         // x3 is a tmp register
 123         // Guest's x0-x3 are on the stack
 124
 125         // Compute base to save registers
 126         add     x3, x2, #CPU_XREG_OFFSET(4)
 127         stp     x4, x5, [x3]
 128         stp     x6, x7, [x3, #16]
 129         stp     x8, x9, [x3, #32]
 130         stp     x10, x11, [x3, #48]
 131         stp     x12, x13, [x3, #64]
 132         stp     x14, x15, [x3, #80]
 133         stp     x16, x17, [x3, #96]
 134         str     x18, [x3, #112]
 135
 136         pop     x6, x7                  // x2, x3
 137         pop     x4, x5                  // x0, x1
 138
 139         add     x3, x2, #CPU_XREG_OFFSET(0)
 140         stp     x4, x5, [x3]
 141         stp     x6, x7, [x3, #16]
 142
 143         save_common_regs
 144 .endm
 145
 146 .macro restore_guest_regs
 147         // x0 is the vcpu address.
 148         // x2 is the cpu context
 149         // x3 is a tmp register
 150
 151         // Prepare x0-x3 for later restore
 152         add     x3, x2, #CPU_XREG_OFFSET(0)
 153         ldp     x4, x5, [x3]
 154         ldp     x6, x7, [x3, #16]
 155         push    x4, x5          // Push x0-x3 on the stack
 156         push    x6, x7
 157
 158         // x4-x18
 159         ldp     x4, x5, [x3, #32]
 160         ldp     x6, x7, [x3, #48]
 161         ldp     x8, x9, [x3, #64]
 162         ldp     x10, x11, [x3, #80]
 163         ldp     x12, x13, [x3, #96]
 164         ldp     x14, x15, [x3, #112]
 165         ldp     x16, x17, [x3, #128]
 166         ldr     x18, [x3, #144]
 167
 168         // x19-x29, lr, sp*, elr*, spsr*
 169         restore_common_regs
 170
 171         // Last bits of the 64bit state
 172         pop     x2, x3
 173         pop     x0, x1
 174
 175         // Do not touch any register after this!
 176 .endm
 177
 178 /*
 179  * Macros to perform system register save/restore.
 180  *
 181  * Ordering here is absolutely critical, and must be kept consistent
 182  * in {save,restore}_sysregs, {save,restore}_guest_32bit_state,
 183  * and in kvm_asm.h.
 184  *
 185  * In other words, don't touch any of these unless you know what
 186  * you are doing.
 187  */
 188 .macro save_sysregs
 189         // x2: base address for cpu context
 190         // x3: tmp register
 191
 192         add     x3, x2, #CPU_SYSREG_OFFSET(MPIDR_EL1)
 193
 194         mrs     x4,     vmpidr_el2
 195         mrs     x5,     csselr_el1
 196         mrs     x6,     sctlr_el1
 197         mrs     x7,     actlr_el1
 198         mrs     x8,     cpacr_el1
 199         mrs     x9,     ttbr0_el1
 200         mrs     x10,    ttbr1_el1
 201         mrs     x11,    tcr_el1
 202         mrs     x12,    esr_el1
 203         mrs     x13,    afsr0_el1
 204         mrs     x14,    afsr1_el1
 205         mrs     x15,    far_el1
 206         mrs     x16,    mair_el1
 207         mrs     x17,    vbar_el1
 208         mrs     x18,    contextidr_el1
 209         mrs     x19,    tpidr_el0
 210         mrs     x20,    tpidrro_el0
 211         mrs     x21,    tpidr_el1
 212         mrs     x22,    amair_el1
 213         mrs     x23,    cntkctl_el1
 214         mrs     x24,    par_el1
 215         mrs     x25,    mdscr_el1
 216
 217         stp     x4, x5, [x3]
 218         stp     x6, x7, [x3, #16]
 219         stp     x8, x9, [x3, #32]
 220         stp     x10, x11, [x3, #48]
 221         stp     x12, x13, [x3, #64]
 222         stp     x14, x15, [x3, #80]
 223         stp     x16, x17, [x3, #96]
 224         stp     x18, x19, [x3, #112]
 225         stp     x20, x21, [x3, #128]
 226         stp     x22, x23, [x3, #144]
 227         stp     x24, x25, [x3, #160]
 228 .endm
 229
 230 .macro save_debug
 231         // x2: base address for cpu context
 232         // x3: tmp register
 233
 234         mrs     x26, id_aa64dfr0_el1
 235         ubfx    x24, x26, #12, #4       // Extract BRPs
 236         ubfx    x25, x26, #20, #4       // Extract WRPs
 237         mov     w26, #15
 238         sub     w24, w26, w24           // How many BPs to skip
 239         sub     w25, w26, w25           // How many WPs to skip
 240
 241         add     x3, x2, #CPU_SYSREG_OFFSET(DBGBCR0_EL1)
 242
 243         adr     x26, 1f
 244         add     x26, x26, x24, lsl #2
 245         br      x26
 246 1:
 247         mrs     x20, dbgbcr15_el1
 248         mrs     x19, dbgbcr14_el1
 249         mrs     x18, dbgbcr13_el1
 250         mrs     x17, dbgbcr12_el1
 251         mrs     x16, dbgbcr11_el1
 252         mrs     x15, dbgbcr10_el1
 253         mrs     x14, dbgbcr9_el1
 254         mrs     x13, dbgbcr8_el1
 255         mrs     x12, dbgbcr7_el1
 256         mrs     x11, dbgbcr6_el1
 257         mrs     x10, dbgbcr5_el1
 258         mrs     x9, dbgbcr4_el1
 259         mrs     x8, dbgbcr3_el1
 260         mrs     x7, dbgbcr2_el1
 261         mrs     x6, dbgbcr1_el1
 262         mrs     x5, dbgbcr0_el1
 263
 264         adr     x26, 1f
 265         add     x26, x26, x24, lsl #2
 266         br      x26
 267
 268 1:
 269         str     x20, [x3, #(15 * 8)]
 270         str     x19, [x3, #(14 * 8)]
 271         str     x18, [x3, #(13 * 8)]
 272         str     x17, [x3, #(12 * 8)]
 273         str     x16, [x3, #(11 * 8)]
 274         str     x15, [x3, #(10 * 8)]
 275         str     x14, [x3, #(9 * 8)]
 276         str     x13, [x3, #(8 * 8)]
 277         str     x12, [x3, #(7 * 8)]
 278         str     x11, [x3, #(6 * 8)]
 279         str     x10, [x3, #(5 * 8)]
 280         str     x9, [x3, #(4 * 8)]
 281         str     x8, [x3, #(3 * 8)]
 282         str     x7, [x3, #(2 * 8)]
 283         str     x6, [x3, #(1 * 8)]
 284         str     x5, [x3, #(0 * 8)]
 285
 286         add     x3, x2, #CPU_SYSREG_OFFSET(DBGBVR0_EL1)
 287
 288         adr     x26, 1f
 289         add     x26, x26, x24, lsl #2
 290         br      x26
 291 1:
 292         mrs     x20, dbgbvr15_el1
 293         mrs     x19, dbgbvr14_el1
 294         mrs     x18, dbgbvr13_el1
 295         mrs     x17, dbgbvr12_el1
 296         mrs     x16, dbgbvr11_el1
 297         mrs     x15, dbgbvr10_el1
 298         mrs     x14, dbgbvr9_el1
 299         mrs     x13, dbgbvr8_el1
 300         mrs     x12, dbgbvr7_el1
 301         mrs     x11, dbgbvr6_el1
 302         mrs     x10, dbgbvr5_el1
 303         mrs     x9, dbgbvr4_el1
 304         mrs     x8, dbgbvr3_el1
 305         mrs     x7, dbgbvr2_el1
 306         mrs     x6, dbgbvr1_el1
 307         mrs     x5, dbgbvr0_el1
 308
 309         adr     x26, 1f
 310         add     x26, x26, x24, lsl #2
 311         br      x26
 312
 313 1:
 314         str     x20, [x3, #(15 * 8)]
 315         str     x19, [x3, #(14 * 8)]
 316         str     x18, [x3, #(13 * 8)]
 317         str     x17, [x3, #(12 * 8)]
 318         str     x16, [x3, #(11 * 8)]
 319         str     x15, [x3, #(10 * 8)]
 320         str     x14, [x3, #(9 * 8)]
 321         str     x13, [x3, #(8 * 8)]
 322         str     x12, [x3, #(7 * 8)]
 323         str     x11, [x3, #(6 * 8)]
 324         str     x10, [x3, #(5 * 8)]
 325         str     x9, [x3, #(4 * 8)]
 326         str     x8, [x3, #(3 * 8)]
 327         str     x7, [x3, #(2 * 8)]
 328         str     x6, [x3, #(1 * 8)]
 329         str     x5, [x3, #(0 * 8)]
 330
 331         add     x3, x2, #CPU_SYSREG_OFFSET(DBGWCR0_EL1)
 332
 333         adr     x26, 1f
 334         add     x26, x26, x25, lsl #2
 335         br      x26
 336 1:
 337         mrs     x20, dbgwcr15_el1
 338         mrs     x19, dbgwcr14_el1
 339         mrs     x18, dbgwcr13_el1
 340         mrs     x17, dbgwcr12_el1
 341         mrs     x16, dbgwcr11_el1
 342         mrs     x15, dbgwcr10_el1
 343         mrs     x14, dbgwcr9_el1
 344         mrs     x13, dbgwcr8_el1
 345         mrs     x12, dbgwcr7_el1
 346         mrs     x11, dbgwcr6_el1
 347         mrs     x10, dbgwcr5_el1
 348         mrs     x9, dbgwcr4_el1
 349         mrs     x8, dbgwcr3_el1
 350         mrs     x7, dbgwcr2_el1
 351         mrs     x6, dbgwcr1_el1
 352         mrs     x5, dbgwcr0_el1
 353
 354         adr     x26, 1f
 355         add     x26, x26, x25, lsl #2
 356         br      x26
 357
 358 1:
 359         str     x20, [x3, #(15 * 8)]
 360         str     x19, [x3, #(14 * 8)]
 361         str     x18, [x3, #(13 * 8)]
 362         str     x17, [x3, #(12 * 8)]
 363         str     x16, [x3, #(11 * 8)]
 364         str     x15, [x3, #(10 * 8)]
 365         str     x14, [x3, #(9 * 8)]
 366         str     x13, [x3, #(8 * 8)]
 367         str     x12, [x3, #(7 * 8)]
 368         str     x11, [x3, #(6 * 8)]
 369         str     x10, [x3, #(5 * 8)]
 370         str     x9, [x3, #(4 * 8)]
 371         str     x8, [x3, #(3 * 8)]
 372         str     x7, [x3, #(2 * 8)]
 373         str     x6, [x3, #(1 * 8)]
 374         str     x5, [x3, #(0 * 8)]
 375
 376         add     x3, x2, #CPU_SYSREG_OFFSET(DBGWVR0_EL1)
 377
 378         adr     x26, 1f
 379         add     x26, x26, x25, lsl #2
 380         br      x26
 381 1:
 382         mrs     x20, dbgwvr15_el1
 383         mrs     x19, dbgwvr14_el1
 384         mrs     x18, dbgwvr13_el1
 385         mrs     x17, dbgwvr12_el1
 386         mrs     x16, dbgwvr11_el1
 387         mrs     x15, dbgwvr10_el1
 388         mrs     x14, dbgwvr9_el1
 389         mrs     x13, dbgwvr8_el1
 390         mrs     x12, dbgwvr7_el1
 391         mrs     x11, dbgwvr6_el1
 392         mrs     x10, dbgwvr5_el1
 393         mrs     x9, dbgwvr4_el1
 394         mrs     x8, dbgwvr3_el1
 395         mrs     x7, dbgwvr2_el1
 396         mrs     x6, dbgwvr1_el1
 397         mrs     x5, dbgwvr0_el1
 398
 399         adr     x26, 1f
 400         add     x26, x26, x25, lsl #2
 401         br      x26
 402
 403 1:
 404         str     x20, [x3, #(15 * 8)]
 405         str     x19, [x3, #(14 * 8)]
 406         str     x18, [x3, #(13 * 8)]
 407         str     x17, [x3, #(12 * 8)]
 408         str     x16, [x3, #(11 * 8)]
 409         str     x15, [x3, #(10 * 8)]
 410         str     x14, [x3, #(9 * 8)]
 411         str     x13, [x3, #(8 * 8)]
 412         str     x12, [x3, #(7 * 8)]
 413         str     x11, [x3, #(6 * 8)]
 414         str     x10, [x3, #(5 * 8)]
 415         str     x9, [x3, #(4 * 8)]
 416         str     x8, [x3, #(3 * 8)]
 417         str     x7, [x3, #(2 * 8)]
 418         str     x6, [x3, #(1 * 8)]
 419         str     x5, [x3, #(0 * 8)]
 420
 421         mrs     x21, mdccint_el1
 422         str     x21, [x2, #CPU_SYSREG_OFFSET(MDCCINT_EL1)]
 423 .endm
 424
 425 .macro restore_sysregs
 426         // x2: base address for cpu context
 427         // x3: tmp register
 428
 429         add     x3, x2, #CPU_SYSREG_OFFSET(MPIDR_EL1)
 430
 431         ldp     x4, x5, [x3]
 432         ldp     x6, x7, [x3, #16]
 433         ldp     x8, x9, [x3, #32]
 434         ldp     x10, x11, [x3, #48]
 435         ldp     x12, x13, [x3, #64]
 436         ldp     x14, x15, [x3, #80]
 437         ldp     x16, x17, [x3, #96]
 438         ldp     x18, x19, [x3, #112]
 439         ldp     x20, x21, [x3, #128]
 440         ldp     x22, x23, [x3, #144]
 441         ldp     x24, x25, [x3, #160]
 442
 443         msr     vmpidr_el2,     x4
 444         msr     csselr_el1,     x5
 445         msr     sctlr_el1,      x6
 446         msr     actlr_el1,      x7
 447         msr     cpacr_el1,      x8
 448         msr     ttbr0_el1,      x9
 449         msr     ttbr1_el1,      x10
 450         msr     tcr_el1,        x11
 451         msr     esr_el1,        x12
 452         msr     afsr0_el1,      x13
 453         msr     afsr1_el1,      x14
 454         msr     far_el1,        x15
 455         msr     mair_el1,       x16
 456         msr     vbar_el1,       x17
 457         msr     contextidr_el1, x18
 458         msr     tpidr_el0,      x19
 459         msr     tpidrro_el0,    x20
 460         msr     tpidr_el1,      x21
 461         msr     amair_el1,      x22
 462         msr     cntkctl_el1,    x23
 463         msr     par_el1,        x24
 464         msr     mdscr_el1,      x25
 465 .endm
 466
 467 .macro restore_debug
 468         // x2: base address for cpu context
 469         // x3: tmp register
 470
 471         mrs     x26, id_aa64dfr0_el1
 472         ubfx    x24, x26, #12, #4       // Extract BRPs
 473         ubfx    x25, x26, #20, #4       // Extract WRPs
 474         mov     w26, #15
 475         sub     w24, w26, w24           // How many BPs to skip
 476         sub     w25, w26, w25           // How many WPs to skip
 477
 478         add     x3, x2, #CPU_SYSREG_OFFSET(DBGBCR0_EL1)
 479
 480         adr     x26, 1f
 481         add     x26, x26, x24, lsl #2
 482         br      x26
 483 1:
 484         ldr     x20, [x3, #(15 * 8)]
 485         ldr     x19, [x3, #(14 * 8)]
 486         ldr     x18, [x3, #(13 * 8)]
 487         ldr     x17, [x3, #(12 * 8)]
 488         ldr     x16, [x3, #(11 * 8)]
 489         ldr     x15, [x3, #(10 * 8)]
 490         ldr     x14, [x3, #(9 * 8)]
 491         ldr     x13, [x3, #(8 * 8)]
 492         ldr     x12, [x3, #(7 * 8)]
 493         ldr     x11, [x3, #(6 * 8)]
 494         ldr     x10, [x3, #(5 * 8)]
 495         ldr     x9, [x3, #(4 * 8)]
 496         ldr     x8, [x3, #(3 * 8)]
 497         ldr     x7, [x3, #(2 * 8)]
 498         ldr     x6, [x3, #(1 * 8)]
 499         ldr     x5, [x3, #(0 * 8)]
 500
 501         adr     x26, 1f
 502         add     x26, x26, x24, lsl #2
 503         br      x26
 504 1:
 505         msr     dbgbcr15_el1, x20
 506         msr     dbgbcr14_el1, x19
 507         msr     dbgbcr13_el1, x18
 508         msr     dbgbcr12_el1, x17
 509         msr     dbgbcr11_el1, x16
 510         msr     dbgbcr10_el1, x15
 511         msr     dbgbcr9_el1, x14
 512         msr     dbgbcr8_el1, x13
 513         msr     dbgbcr7_el1, x12
 514         msr     dbgbcr6_el1, x11
 515         msr     dbgbcr5_el1, x10
 516         msr     dbgbcr4_el1, x9
 517         msr     dbgbcr3_el1, x8
 518         msr     dbgbcr2_el1, x7
 519         msr     dbgbcr1_el1, x6
 520         msr     dbgbcr0_el1, x5
 521
 522         add     x3, x2, #CPU_SYSREG_OFFSET(DBGBVR0_EL1)
 523
 524         adr     x26, 1f
 525         add     x26, x26, x24, lsl #2
 526         br      x26
 527 1:
 528         ldr     x20, [x3, #(15 * 8)]
 529         ldr     x19, [x3, #(14 * 8)]
 530         ldr     x18, [x3, #(13 * 8)]
 531         ldr     x17, [x3, #(12 * 8)]
 532         ldr     x16, [x3, #(11 * 8)]
 533         ldr     x15, [x3, #(10 * 8)]
 534         ldr     x14, [x3, #(9 * 8)]
 535         ldr     x13, [x3, #(8 * 8)]
 536         ldr     x12, [x3, #(7 * 8)]
 537         ldr     x11, [x3, #(6 * 8)]
 538         ldr     x10, [x3, #(5 * 8)]
 539         ldr     x9, [x3, #(4 * 8)]
 540         ldr     x8, [x3, #(3 * 8)]
 541         ldr     x7, [x3, #(2 * 8)]
 542         ldr     x6, [x3, #(1 * 8)]
 543         ldr     x5, [x3, #(0 * 8)]
 544
 545         adr     x26, 1f
 546         add     x26, x26, x24, lsl #2
 547         br      x26
 548 1:
 549         msr     dbgbvr15_el1, x20
 550         msr     dbgbvr14_el1, x19
 551         msr     dbgbvr13_el1, x18
 552         msr     dbgbvr12_el1, x17
 553         msr     dbgbvr11_el1, x16
 554         msr     dbgbvr10_el1, x15
 555         msr     dbgbvr9_el1, x14
 556         msr     dbgbvr8_el1, x13
 557         msr     dbgbvr7_el1, x12
 558         msr     dbgbvr6_el1, x11
 559         msr     dbgbvr5_el1, x10
 560         msr     dbgbvr4_el1, x9
 561         msr     dbgbvr3_el1, x8
 562         msr     dbgbvr2_el1, x7
 563         msr     dbgbvr1_el1, x6
 564         msr     dbgbvr0_el1, x5
 565
 566         add     x3, x2, #CPU_SYSREG_OFFSET(DBGWCR0_EL1)
 567
 568         adr     x26, 1f
 569         add     x26, x26, x25, lsl #2
 570         br      x26
 571 1:
 572         ldr     x20, [x3, #(15 * 8)]
 573         ldr     x19, [x3, #(14 * 8)]
 574         ldr     x18, [x3, #(13 * 8)]
 575         ldr     x17, [x3, #(12 * 8)]
 576         ldr     x16, [x3, #(11 * 8)]
 577         ldr     x15, [x3, #(10 * 8)]
 578         ldr     x14, [x3, #(9 * 8)]
 579         ldr     x13, [x3, #(8 * 8)]
 580         ldr     x12, [x3, #(7 * 8)]
 581         ldr     x11, [x3, #(6 * 8)]
 582         ldr     x10, [x3, #(5 * 8)]
 583         ldr     x9, [x3, #(4 * 8)]
 584         ldr     x8, [x3, #(3 * 8)]
 585         ldr     x7, [x3, #(2 * 8)]
 586         ldr     x6, [x3, #(1 * 8)]
 587         ldr     x5, [x3, #(0 * 8)]
 588
 589         adr     x26, 1f
 590         add     x26, x26, x25, lsl #2
 591         br      x26
 592 1:
 593         msr     dbgwcr15_el1, x20
 594         msr     dbgwcr14_el1, x19
 595         msr     dbgwcr13_el1, x18
 596         msr     dbgwcr12_el1, x17
 597         msr     dbgwcr11_el1, x16
 598         msr     dbgwcr10_el1, x15
 599         msr     dbgwcr9_el1, x14
 600         msr     dbgwcr8_el1, x13
 601         msr     dbgwcr7_el1, x12
 602         msr     dbgwcr6_el1, x11
 603         msr     dbgwcr5_el1, x10
 604         msr     dbgwcr4_el1, x9
 605         msr     dbgwcr3_el1, x8
 606         msr     dbgwcr2_el1, x7
 607         msr     dbgwcr1_el1, x6
 608         msr     dbgwcr0_el1, x5
 609
 610         add     x3, x2, #CPU_SYSREG_OFFSET(DBGWVR0_EL1)
 611
 612         adr     x26, 1f
 613         add     x26, x26, x25, lsl #2
 614         br      x26
 615 1:
 616         ldr     x20, [x3, #(15 * 8)]
 617         ldr     x19, [x3, #(14 * 8)]
 618         ldr     x18, [x3, #(13 * 8)]
 619         ldr     x17, [x3, #(12 * 8)]
 620         ldr     x16, [x3, #(11 * 8)]
 621         ldr     x15, [x3, #(10 * 8)]
 622         ldr     x14, [x3, #(9 * 8)]
 623         ldr     x13, [x3, #(8 * 8)]
 624         ldr     x12, [x3, #(7 * 8)]
 625         ldr     x11, [x3, #(6 * 8)]
 626         ldr     x10, [x3, #(5 * 8)]
 627         ldr     x9, [x3, #(4 * 8)]
 628         ldr     x8, [x3, #(3 * 8)]
 629         ldr     x7, [x3, #(2 * 8)]
 630         ldr     x6, [x3, #(1 * 8)]
 631         ldr     x5, [x3, #(0 * 8)]
 632
 633         adr     x26, 1f
 634         add     x26, x26, x25, lsl #2
 635         br      x26
 636 1:
 637         msr     dbgwvr15_el1, x20
 638         msr     dbgwvr14_el1, x19
 639         msr     dbgwvr13_el1, x18
 640         msr     dbgwvr12_el1, x17
 641         msr     dbgwvr11_el1, x16
 642         msr     dbgwvr10_el1, x15
 643         msr     dbgwvr9_el1, x14
 644         msr     dbgwvr8_el1, x13
 645         msr     dbgwvr7_el1, x12
 646         msr     dbgwvr6_el1, x11
 647         msr     dbgwvr5_el1, x10
 648         msr     dbgwvr4_el1, x9
 649         msr     dbgwvr3_el1, x8
 650         msr     dbgwvr2_el1, x7
 651         msr     dbgwvr1_el1, x6
 652         msr     dbgwvr0_el1, x5
 653
 654         ldr     x21, [x2, #CPU_SYSREG_OFFSET(MDCCINT_EL1)]
 655         msr     mdccint_el1, x21
 656 .endm
 657
 658 .macro skip_32bit_state tmp, target
 659         // Skip 32bit state if not needed
 660         mrs     \tmp, hcr_el2
 661         tbnz    \tmp, #HCR_RW_SHIFT, \target
 662 .endm
 663
 664 .macro skip_tee_state tmp, target
 665         // Skip ThumbEE state if not needed
 666         mrs     \tmp, id_pfr0_el1
 667         tbz     \tmp, #12, \target
 668 .endm
 669
 670 .macro skip_debug_state tmp, target
 671         ldr     \tmp, [x0, #VCPU_DEBUG_FLAGS]
 672         tbz     \tmp, #KVM_ARM64_DEBUG_DIRTY_SHIFT, \target
 673 .endm
 674
 675 .macro compute_debug_state target
 676         // Compute debug state: If any of KDE, MDE or KVM_ARM64_DEBUG_DIRTY
 677         // is set, we do a full save/restore cycle and disable trapping.
 678         add     x25, x0, #VCPU_CONTEXT
 679
 680         // Check the state of MDSCR_EL1
 681         ldr     x25, [x25, #CPU_SYSREG_OFFSET(MDSCR_EL1)]
 682         and     x26, x25, #DBG_MDSCR_KDE
 683         and     x25, x25, #DBG_MDSCR_MDE
 684         adds    xzr, x25, x26
 685         b.eq    9998f           // Nothing to see there
 686
 687         // If any interesting bits was set, we must set the flag
 688         mov     x26, #KVM_ARM64_DEBUG_DIRTY
 689         str     x26, [x0, #VCPU_DEBUG_FLAGS]
 690         b       9999f           // Don't skip restore
 691
 692 9998:
 693         // Otherwise load the flags from memory in case we recently
 694         // trapped
 695         skip_debug_state x25, \target
 696 9999:
 697 .endm
 698
 699 .macro save_guest_32bit_state
 700         skip_32bit_state x3, 1f
 701
 702         add     x3, x2, #CPU_SPSR_OFFSET(KVM_SPSR_ABT)
 703         mrs     x4, spsr_abt
 704         mrs     x5, spsr_und
 705         mrs     x6, spsr_irq
 706         mrs     x7, spsr_fiq
 707         stp     x4, x5, [x3]
 708         stp     x6, x7, [x3, #16]
 709
 710         add     x3, x2, #CPU_SYSREG_OFFSET(DACR32_EL2)
 711         mrs     x4, dacr32_el2
 712         mrs     x5, ifsr32_el2
 713         mrs     x6, fpexc32_el2
 714         stp     x4, x5, [x3]
 715         str     x6, [x3, #16]
 716
 717         skip_debug_state x8, 2f
 718         mrs     x7, dbgvcr32_el2
 719         str     x7, [x3, #24]
 720 2:
 721         skip_tee_state x8, 1f
 722
 723         add     x3, x2, #CPU_SYSREG_OFFSET(TEECR32_EL1)
 724         mrs     x4, teecr32_el1
 725         mrs     x5, teehbr32_el1
 726         stp     x4, x5, [x3]
 727 1:
 728 .endm
 729
 730 .macro restore_guest_32bit_state
 731         skip_32bit_state x3, 1f
 732
 733         add     x3, x2, #CPU_SPSR_OFFSET(KVM_SPSR_ABT)
 734         ldp     x4, x5, [x3]
 735         ldp     x6, x7, [x3, #16]
 736         msr     spsr_abt, x4
 737         msr     spsr_und, x5
 738         msr     spsr_irq, x6
 739         msr     spsr_fiq, x7
 740
 741         add     x3, x2, #CPU_SYSREG_OFFSET(DACR32_EL2)
 742         ldp     x4, x5, [x3]
 743         ldr     x6, [x3, #16]
 744         msr     dacr32_el2, x4
 745         msr     ifsr32_el2, x5
 746         msr     fpexc32_el2, x6
 747
 748         skip_debug_state x8, 2f
 749         ldr     x7, [x3, #24]
 750         msr     dbgvcr32_el2, x7
 751 2:
 752         skip_tee_state x8, 1f
 753
 754         add     x3, x2, #CPU_SYSREG_OFFSET(TEECR32_EL1)
 755         ldp     x4, x5, [x3]
 756         msr     teecr32_el1, x4
 757         msr     teehbr32_el1, x5
 758 1:
 759 .endm
 760
 761 .macro activate_traps
 762         ldr     x2, [x0, #VCPU_HCR_EL2]
 763         msr     hcr_el2, x2
 764         mov     x2, #CPTR_EL2_TTA
 765         msr     cptr_el2, x2
 766
 767         mov     x2, #(1 << 15)  // Trap CP15 Cr=15
 768         msr     hstr_el2, x2
 769
 770         mrs     x2, mdcr_el2
 771         and     x2, x2, #MDCR_EL2_HPMN_MASK
 772         orr     x2, x2, #(MDCR_EL2_TPM | MDCR_EL2_TPMCR)
 773         orr     x2, x2, #(MDCR_EL2_TDRA | MDCR_EL2_TDOSA)
 774
 775         // Check for KVM_ARM64_DEBUG_DIRTY, and set debug to trap
 776         // if not dirty.
 777         ldr     x3, [x0, #VCPU_DEBUG_FLAGS]
 778         tbnz    x3, #KVM_ARM64_DEBUG_DIRTY_SHIFT, 1f
 779         orr     x2, x2,  #MDCR_EL2_TDA
 780 1:
 781         msr     mdcr_el2, x2
 782 .endm
 783
 784 .macro deactivate_traps
 785         mov     x2, #HCR_RW
 786         msr     hcr_el2, x2
 787         msr     cptr_el2, xzr
 788         msr     hstr_el2, xzr
 789
 790         mrs     x2, mdcr_el2
 791         and     x2, x2, #MDCR_EL2_HPMN_MASK
 792         msr     mdcr_el2, x2
 793 .endm
 794
 795 .macro activate_vm
 796         ldr     x1, [x0, #VCPU_KVM]
 797         kern_hyp_va     x1
 798         ldr     x2, [x1, #KVM_VTTBR]
 799         msr     vttbr_el2, x2
 800 .endm
 801
 802 .macro deactivate_vm
 803         msr     vttbr_el2, xzr
 804 .endm
 805
 806 /*
 807  * Call into the vgic backend for state saving
 808  */
 809 .macro save_vgic_state
 810         adr     x24, __vgic_sr_vectors
 811         ldr     x24, [x24, VGIC_SAVE_FN]
 812         kern_hyp_va     x24
 813         blr     x24
 814         mrs     x24, hcr_el2
 815         mov     x25, #HCR_INT_OVERRIDE
 816         neg     x25, x25
 817         and     x24, x24, x25
 818         msr     hcr_el2, x24
 819 .endm
 820
 821 /*
 822  * Call into the vgic backend for state restoring
 823  */
 824 .macro restore_vgic_state
 825         mrs     x24, hcr_el2
 826         ldr     x25, [x0, #VCPU_IRQ_LINES]
 827         orr     x24, x24, #HCR_INT_OVERRIDE
 828         orr     x24, x24, x25
 829         msr     hcr_el2, x24
 830         adr     x24, __vgic_sr_vectors
 831         ldr     x24, [x24, #VGIC_RESTORE_FN]
 832         kern_hyp_va     x24
 833         blr     x24
 834 .endm
 835
 836 .macro save_timer_state
 837         // x0: vcpu pointer
 838         ldr     x2, [x0, #VCPU_KVM]
 839         kern_hyp_va x2
 840         ldr     w3, [x2, #KVM_TIMER_ENABLED]
 841         cbz     w3, 1f
 842
 843         mrs     x3, cntv_ctl_el0
 844         and     x3, x3, #3
 845         str     w3, [x0, #VCPU_TIMER_CNTV_CTL]
 846         bic     x3, x3, #1              // Clear Enable
 847         msr     cntv_ctl_el0, x3
 848
 849         isb
 850
 851         mrs     x3, cntv_cval_el0
 852         str     x3, [x0, #VCPU_TIMER_CNTV_CVAL]
 853
 854 1:
 855         // Allow physical timer/counter access for the host
 856         mrs     x2, cnthctl_el2
 857         orr     x2, x2, #3
 858         msr     cnthctl_el2, x2
 859
 860         // Clear cntvoff for the host
 861         msr     cntvoff_el2, xzr
 862 .endm
 863
 864 .macro restore_timer_state
 865         // x0: vcpu pointer
 866         // Disallow physical timer access for the guest
 867         // Physical counter access is allowed
 868         mrs     x2, cnthctl_el2
 869         orr     x2, x2, #1
 870         bic     x2, x2, #2
 871         msr     cnthctl_el2, x2
 872
 873         ldr     x2, [x0, #VCPU_KVM]
 874         kern_hyp_va x2
 875         ldr     w3, [x2, #KVM_TIMER_ENABLED]
 876         cbz     w3, 1f
 877
 878         ldr     x3, [x2, #KVM_TIMER_CNTVOFF]
 879         msr     cntvoff_el2, x3
 880         ldr     x2, [x0, #VCPU_TIMER_CNTV_CVAL]
 881         msr     cntv_cval_el0, x2
 882         isb
 883
 884         ldr     w2, [x0, #VCPU_TIMER_CNTV_CTL]
 885         and     x2, x2, #3
 886         msr     cntv_ctl_el0, x2
 887 1:
 888 .endm
 889
 890 __save_sysregs:
 891         save_sysregs
 892         ret
 893
 894 __restore_sysregs:
 895         restore_sysregs
 896         ret
 897
 898 __save_debug:
 899         save_debug
 900         ret
 901
 902 __restore_debug:
 903         restore_debug
 904         ret
 905
 906 __save_fpsimd:
 907         save_fpsimd
 908         ret
 909
 910 __restore_fpsimd:
 911         restore_fpsimd
 912         ret
 913
 914 /*
 915  * u64 __kvm_vcpu_run(struct kvm_vcpu *vcpu);
 916  *
 917  * This is the world switch. The first half of the function
 918  * deals with entering the guest, and anything from __kvm_vcpu_return
 919  * to the end of the function deals with reentering the host.
 920  * On the enter path, only x0 (vcpu pointer) must be preserved until
 921  * the last moment. On the exit path, x0 (vcpu pointer) and x1 (exception
 922  * code) must both be preserved until the epilogue.
 923  * In both cases, x2 points to the CPU context we're saving/restoring from/to.
 924  */
 925 ENTRY(__kvm_vcpu_run)
 926         kern_hyp_va     x0
 927         msr     tpidr_el2, x0   // Save the vcpu register
 928
 929         // Host context
 930         ldr     x2, [x0, #VCPU_HOST_CONTEXT]
 931         kern_hyp_va x2
 932
 933         save_host_regs
 934         bl __save_fpsimd
 935         bl __save_sysregs
 936
 937         compute_debug_state 1f
 938         bl      __save_debug
 939 1:
 940         activate_traps
 941         activate_vm
 942
 943         restore_vgic_state
 944         restore_timer_state
 945
 946         // Guest context
 947         add     x2, x0, #VCPU_CONTEXT
 948
 949         bl __restore_sysregs
 950         bl __restore_fpsimd
 951
 952         skip_debug_state x3, 1f
 953         bl      __restore_debug
 954 1:
 955         restore_guest_32bit_state
 956         restore_guest_regs
 957
 958         // That's it, no more messing around.
 959         eret
 960
 961 __kvm_vcpu_return:
 962         // Assume x0 is the vcpu pointer, x1 the return code
 963         // Guest's x0-x3 are on the stack
 964
 965         // Guest context
 966         add     x2, x0, #VCPU_CONTEXT
 967
 968         save_guest_regs
 969         bl __save_fpsimd
 970         bl __save_sysregs
 971
 972         skip_debug_state x3, 1f
 973         bl      __save_debug
 974 1:
 975         save_guest_32bit_state
 976
 977         save_timer_state
 978         save_vgic_state
 979
 980         deactivate_traps
 981         deactivate_vm
 982
 983         // Host context
 984         ldr     x2, [x0, #VCPU_HOST_CONTEXT]
 985         kern_hyp_va x2
 986
 987         bl __restore_sysregs
 988         bl __restore_fpsimd
 989
 990         skip_debug_state x3, 1f
 991         // Clear the dirty flag for the next run, as all the state has
 992         // already been saved. Note that we nuke the whole 64bit word.
 993         // If we ever add more flags, we'll have to be more careful...
 994         str     xzr, [x0, #VCPU_DEBUG_FLAGS]
 995         bl      __restore_debug
 996 1:
 997         restore_host_regs
 998
 999         mov     x0, x1
1000         ret
1001 END(__kvm_vcpu_run)
1002
1003 // void __kvm_tlb_flush_vmid_ipa(struct kvm *kvm, phys_addr_t ipa);
1004 ENTRY(__kvm_tlb_flush_vmid_ipa)
1005         dsb     ishst
1006
1007         kern_hyp_va     x0
1008         ldr     x2, [x0, #KVM_VTTBR]
1009         msr     vttbr_el2, x2
1010         isb
1011
1012         /*
1013          * We could do so much better if we had the VA as well.
1014          * Instead, we invalidate Stage-2 for this IPA, and the
1015          * whole of Stage-1. Weep...
1016          */
1017         tlbi    ipas2e1is, x1
1018         /*
1019          * We have to ensure completion of the invalidation at Stage-2,
1020          * since a table walk on another CPU could refill a TLB with a
1021          * complete (S1 + S2) walk based on the old Stage-2 mapping if
1022          * the Stage-1 invalidation happened first.
1023          */
1024         dsb     ish
1025         tlbi    vmalle1is
1026         dsb     ish
1027         isb
1028
1029         msr     vttbr_el2, xzr
1030         ret
1031 ENDPROC(__kvm_tlb_flush_vmid_ipa)
1032
1033 ENTRY(__kvm_flush_vm_context)
1034         dsb     ishst
1035         tlbi    alle1is
1036         ic      ialluis
1037         dsb     ish
1038         ret
1039 ENDPROC(__kvm_flush_vm_context)
1040
1041         // struct vgic_sr_vectors __vgi_sr_vectors;
1042         .align 3
1043 ENTRY(__vgic_sr_vectors)
1044         .skip   VGIC_SR_VECTOR_SZ
1045 ENDPROC(__vgic_sr_vectors)
1046
1047 __kvm_hyp_panic:
1048         // Guess the context by looking at VTTBR:
1049         // If zero, then we're already a host.
1050         // Otherwise restore a minimal host context before panicing.
1051         mrs     x0, vttbr_el2
1052         cbz     x0, 1f
1053
1054         mrs     x0, tpidr_el2
1055
1056         deactivate_traps
1057         deactivate_vm
1058
1059         ldr     x2, [x0, #VCPU_HOST_CONTEXT]
1060         kern_hyp_va x2
1061
1062         bl __restore_sysregs
1063
1064 1:      adr     x0, __hyp_panic_str
1065         adr     x1, 2f
1066         ldp     x2, x3, [x1]
1067         sub     x0, x0, x2
1068         add     x0, x0, x3
1069         mrs     x1, spsr_el2
1070         mrs     x2, elr_el2
1071         mrs     x3, esr_el2
1072         mrs     x4, far_el2
1073         mrs     x5, hpfar_el2
1074         mrs     x6, par_el1
1075         mrs     x7, tpidr_el2
1076
1077         mov     lr, #(PSR_F_BIT | PSR_I_BIT | PSR_A_BIT | PSR_D_BIT |\
1078                       PSR_MODE_EL1h)
1079         msr     spsr_el2, lr
1080         ldr     lr, =panic
1081         msr     elr_el2, lr
1082         eret
1083
1084         .align  3
1085 2:      .quad   HYP_PAGE_OFFSET
1086         .quad   PAGE_OFFSET
1087 ENDPROC(__kvm_hyp_panic)
1088
1089 __hyp_panic_str:
1090         .ascii  "HYP panic:\nPS:%08x PC:%p ESR:%p\nFAR:%p HPFAR:%p PAR:%p\nVCPU:%p\n\0"
1091
1092         .align  2
1093
1094 /*
1095  * u64 kvm_call_hyp(void *hypfn, ...);
1096  *
1097  * This is not really a variadic function in the classic C-way and care must
1098  * be taken when calling this to ensure parameters are passed in registers
1099  * only, since the stack will change between the caller and the callee.
1100  *
1101  * Call the function with the first argument containing a pointer to the
1102  * function you wish to call in Hyp mode, and subsequent arguments will be
1103  * passed as x0, x1, and x2 (a maximum of 3 arguments in addition to the
1104  * function pointer can be passed).  The function being called must be mapped
1105  * in Hyp mode (see init_hyp_mode in arch/arm/kvm/arm.c).  Return values are
1106  * passed in r0 and r1.
1107  *
1108  * A function pointer with a value of 0 has a special meaning, and is
1109  * used to implement __hyp_get_vectors in the same way as in
1110  * arch/arm64/kernel/hyp_stub.S.
1111  */
1112 ENTRY(kvm_call_hyp)
1113         hvc     #0
1114         ret
1115 ENDPROC(kvm_call_hyp)
1116
1117 .macro invalid_vector   label, target
1118         .align  2
1119 \label:
1120         b \target
1121 ENDPROC(\label)
1122 .endm
1123
1124         /* None of these should ever happen */
1125         invalid_vector  el2t_sync_invalid, __kvm_hyp_panic
1126         invalid_vector  el2t_irq_invalid, __kvm_hyp_panic
1127         invalid_vector  el2t_fiq_invalid, __kvm_hyp_panic
1128         invalid_vector  el2t_error_invalid, __kvm_hyp_panic
1129         invalid_vector  el2h_sync_invalid, __kvm_hyp_panic
1130         invalid_vector  el2h_irq_invalid, __kvm_hyp_panic
1131         invalid_vector  el2h_fiq_invalid, __kvm_hyp_panic
1132         invalid_vector  el2h_error_invalid, __kvm_hyp_panic
1133         invalid_vector  el1_sync_invalid, __kvm_hyp_panic
1134         invalid_vector  el1_irq_invalid, __kvm_hyp_panic
1135         invalid_vector  el1_fiq_invalid, __kvm_hyp_panic
1136         invalid_vector  el1_error_invalid, __kvm_hyp_panic
1137
1138 el1_sync:                                       // Guest trapped into EL2
1139         push    x0, x1
1140         push    x2, x3
1141
1142         mrs     x1, esr_el2
1143         lsr     x2, x1, #ESR_EL2_EC_SHIFT
1144
1145         cmp     x2, #ESR_EL2_EC_HVC64
1146         b.ne    el1_trap
1147
1148         mrs     x3, vttbr_el2                   // If vttbr is valid, the 64bit guest
1149         cbnz    x3, el1_trap                    // called HVC
1150
1151         /* Here, we're pretty sure the host called HVC. */
1152         pop     x2, x3
1153         pop     x0, x1
1154
1155         /* Check for __hyp_get_vectors */
1156         cbnz    x0, 1f
1157         mrs     x0, vbar_el2
1158         b       2f
1159
1160 1:      push    lr, xzr
1161
1162         /*
1163          * Compute the function address in EL2, and shuffle the parameters.
1164          */
1165         kern_hyp_va     x0
1166         mov     lr, x0
1167         mov     x0, x1
1168         mov     x1, x2
1169         mov     x2, x3
1170         blr     lr
1171
1172         pop     lr, xzr
1173 2:      eret
1174
1175 el1_trap:
1176         /*
1177          * x1: ESR
1178          * x2: ESR_EC
1179          */
1180         cmp     x2, #ESR_EL2_EC_DABT
1181         mov     x0, #ESR_EL2_EC_IABT
1182         ccmp    x2, x0, #4, ne
1183         b.ne    1f              // Not an abort we care about
1184
1185         /* This is an abort. Check for permission fault */
1186         and     x2, x1, #ESR_EL2_FSC_TYPE
1187         cmp     x2, #FSC_PERM
1188         b.ne    1f              // Not a permission fault
1189
1190         /*
1191          * Check for Stage-1 page table walk, which is guaranteed
1192          * to give a valid HPFAR_EL2.
1193          */
1194         tbnz    x1, #7, 1f      // S1PTW is set
1195
1196         /* Preserve PAR_EL1 */
1197         mrs     x3, par_el1
1198         push    x3, xzr
1199
1200         /*
1201          * Permission fault, HPFAR_EL2 is invalid.
1202          * Resolve the IPA the hard way using the guest VA.
1203          * Stage-1 translation already validated the memory access rights.
1204          * As such, we can use the EL1 translation regime, and don't have
1205          * to distinguish between EL0 and EL1 access.
1206          */
1207         mrs     x2, far_el2
1208         at      s1e1r, x2
1209         isb
1210
1211         /* Read result */
1212         mrs     x3, par_el1
1213         pop     x0, xzr                 // Restore PAR_EL1 from the stack
1214         msr     par_el1, x0
1215         tbnz    x3, #0, 3f              // Bail out if we failed the translation
1216         ubfx    x3, x3, #12, #36        // Extract IPA
1217         lsl     x3, x3, #4              // and present it like HPFAR
1218         b       2f
1219
1220 1:      mrs     x3, hpfar_el2
1221         mrs     x2, far_el2
1222
1223 2:      mrs     x0, tpidr_el2
1224         str     w1, [x0, #VCPU_ESR_EL2]
1225         str     x2, [x0, #VCPU_FAR_EL2]
1226         str     x3, [x0, #VCPU_HPFAR_EL2]
1227
1228         mov     x1, #ARM_EXCEPTION_TRAP
1229         b       __kvm_vcpu_return
1230
1231         /*
1232          * Translation failed. Just return to the guest and
1233          * let it fault again. Another CPU is probably playing
1234          * behind our back.
1235          */
1236 3:      pop     x2, x3
1237         pop     x0, x1
1238
1239         eret
1240
1241 el1_irq:
1242         push    x0, x1
1243         push    x2, x3
1244         mrs     x0, tpidr_el2
1245         mov     x1, #ARM_EXCEPTION_IRQ
1246         b       __kvm_vcpu_return
1247
1248         .ltorg
1249
1250         .align 11
1251
1252 ENTRY(__kvm_hyp_vector)
1253         ventry  el2t_sync_invalid               // Synchronous EL2t
1254         ventry  el2t_irq_invalid                // IRQ EL2t
1255         ventry  el2t_fiq_invalid                // FIQ EL2t
1256         ventry  el2t_error_invalid              // Error EL2t
1257
1258         ventry  el2h_sync_invalid               // Synchronous EL2h
1259         ventry  el2h_irq_invalid                // IRQ EL2h
1260         ventry  el2h_fiq_invalid                // FIQ EL2h
1261         ventry  el2h_error_invalid              // Error EL2h
1262
1263         ventry  el1_sync                        // Synchronous 64-bit EL1
1264         ventry  el1_irq                         // IRQ 64-bit EL1
1265         ventry  el1_fiq_invalid                 // FIQ 64-bit EL1
1266         ventry  el1_error_invalid               // Error 64-bit EL1
1267
1268         ventry  el1_sync                        // Synchronous 32-bit EL1
1269         ventry  el1_irq                         // IRQ 32-bit EL1
1270         ventry  el1_fiq_invalid                 // FIQ 32-bit EL1
1271         ventry  el1_error_invalid               // Error 32-bit EL1
1272 ENDPROC(__kvm_hyp_vector)
1273
1274         .popsection