virt/kvm/arm/hyp/vgic-v3-sr.c

   1 /*
   2  * Copyright (C) 2012-2015 - ARM Ltd
   3  * Author: Marc Zyngier <marc.zyngier@arm.com>
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License version 2 as
   7  * published by the Free Software Foundation.
   8  *
   9  * This program is distributed in the hope that it will be useful,
  10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12  * GNU General Public License for more details.
  13  *
  14  * You should have received a copy of the GNU General Public License
  15  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  16  */
  17
  18 #include <linux/compiler.h>
  19 #include <linux/irqchip/arm-gic-v3.h>
  20 #include <linux/kvm_host.h>
  21
  22 #include <asm/kvm_emulate.h>
  23 #include <asm/kvm_hyp.h>
  24
  25 #define vtr_to_max_lr_idx(v)            ((v) & 0xf)
  26 #define vtr_to_nr_pre_bits(v)           ((((u32)(v) >> 26) & 7) + 1)
  27 #define vtr_to_nr_apr_regs(v)           (1 << (vtr_to_nr_pre_bits(v) - 5))
  28
  29 static u64 __hyp_text __gic_v3_get_lr(unsigned int lr)
  30 {
  31         switch (lr & 0xf) {
  32         case 0:
  33                 return read_gicreg(ICH_LR0_EL2);
  34         case 1:
  35                 return read_gicreg(ICH_LR1_EL2);
  36         case 2:
  37                 return read_gicreg(ICH_LR2_EL2);
  38         case 3:
  39                 return read_gicreg(ICH_LR3_EL2);
  40         case 4:
  41                 return read_gicreg(ICH_LR4_EL2);
  42         case 5:
  43                 return read_gicreg(ICH_LR5_EL2);
  44         case 6:
  45                 return read_gicreg(ICH_LR6_EL2);
  46         case 7:
  47                 return read_gicreg(ICH_LR7_EL2);
  48         case 8:
  49                 return read_gicreg(ICH_LR8_EL2);
  50         case 9:
  51                 return read_gicreg(ICH_LR9_EL2);
  52         case 10:
  53                 return read_gicreg(ICH_LR10_EL2);
  54         case 11:
  55                 return read_gicreg(ICH_LR11_EL2);
  56         case 12:
  57                 return read_gicreg(ICH_LR12_EL2);
  58         case 13:
  59                 return read_gicreg(ICH_LR13_EL2);
  60         case 14:
  61                 return read_gicreg(ICH_LR14_EL2);
  62         case 15:
  63                 return read_gicreg(ICH_LR15_EL2);
  64         }
  65
  66         unreachable();
  67 }
  68
  69 static void __hyp_text __gic_v3_set_lr(u64 val, int lr)
  70 {
  71         switch (lr & 0xf) {
  72         case 0:
  73                 write_gicreg(val, ICH_LR0_EL2);
  74                 break;
  75         case 1:
  76                 write_gicreg(val, ICH_LR1_EL2);
  77                 break;
  78         case 2:
  79                 write_gicreg(val, ICH_LR2_EL2);
  80                 break;
  81         case 3:
  82                 write_gicreg(val, ICH_LR3_EL2);
  83                 break;
  84         case 4:
  85                 write_gicreg(val, ICH_LR4_EL2);
  86                 break;
  87         case 5:
  88                 write_gicreg(val, ICH_LR5_EL2);
  89                 break;
  90         case 6:
  91                 write_gicreg(val, ICH_LR6_EL2);
  92                 break;
  93         case 7:
  94                 write_gicreg(val, ICH_LR7_EL2);
  95                 break;
  96         case 8:
  97                 write_gicreg(val, ICH_LR8_EL2);
  98                 break;
  99         case 9:
 100                 write_gicreg(val, ICH_LR9_EL2);
 101                 break;
 102         case 10:
 103                 write_gicreg(val, ICH_LR10_EL2);
 104                 break;
 105         case 11:
 106                 write_gicreg(val, ICH_LR11_EL2);
 107                 break;
 108         case 12:
 109                 write_gicreg(val, ICH_LR12_EL2);
 110                 break;
 111         case 13:
 112                 write_gicreg(val, ICH_LR13_EL2);
 113                 break;
 114         case 14:
 115                 write_gicreg(val, ICH_LR14_EL2);
 116                 break;
 117         case 15:
 118                 write_gicreg(val, ICH_LR15_EL2);
 119                 break;
 120         }
 121 }
 122
 123 static void __hyp_text __vgic_v3_write_ap0rn(u32 val, int n)
 124 {
 125         switch (n) {
 126         case 0:
 127                 write_gicreg(val, ICH_AP0R0_EL2);
 128                 break;
 129         case 1:
 130                 write_gicreg(val, ICH_AP0R1_EL2);
 131                 break;
 132         case 2:
 133                 write_gicreg(val, ICH_AP0R2_EL2);
 134                 break;
 135         case 3:
 136                 write_gicreg(val, ICH_AP0R3_EL2);
 137                 break;
 138         }
 139 }
 140
 141 static void __hyp_text __vgic_v3_write_ap1rn(u32 val, int n)
 142 {
 143         switch (n) {
 144         case 0:
 145                 write_gicreg(val, ICH_AP1R0_EL2);
 146                 break;
 147         case 1:
 148                 write_gicreg(val, ICH_AP1R1_EL2);
 149                 break;
 150         case 2:
 151                 write_gicreg(val, ICH_AP1R2_EL2);
 152                 break;
 153         case 3:
 154                 write_gicreg(val, ICH_AP1R3_EL2);
 155                 break;
 156         }
 157 }
 158
 159 static u32 __hyp_text __vgic_v3_read_ap0rn(int n)
 160 {
 161         u32 val;
 162
 163         switch (n) {
 164         case 0:
 165                 val = read_gicreg(ICH_AP0R0_EL2);
 166                 break;
 167         case 1:
 168                 val = read_gicreg(ICH_AP0R1_EL2);
 169                 break;
 170         case 2:
 171                 val = read_gicreg(ICH_AP0R2_EL2);
 172                 break;
 173         case 3:
 174                 val = read_gicreg(ICH_AP0R3_EL2);
 175                 break;
 176         default:
 177                 unreachable();
 178         }
 179
 180         return val;
 181 }
 182
 183 static u32 __hyp_text __vgic_v3_read_ap1rn(int n)
 184 {
 185         u32 val;
 186
 187         switch (n) {
 188         case 0:
 189                 val = read_gicreg(ICH_AP1R0_EL2);
 190                 break;
 191         case 1:
 192                 val = read_gicreg(ICH_AP1R1_EL2);
 193                 break;
 194         case 2:
 195                 val = read_gicreg(ICH_AP1R2_EL2);
 196                 break;
 197         case 3:
 198                 val = read_gicreg(ICH_AP1R3_EL2);
 199                 break;
 200         default:
 201                 unreachable();
 202         }
 203
 204         return val;
 205 }
 206
 207 void __hyp_text __vgic_v3_save_state(struct kvm_vcpu *vcpu)
 208 {
 209         struct vgic_v3_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v3;
 210         u64 used_lrs = vcpu->arch.vgic_cpu.used_lrs;
 211         u64 val;
 212
 213         /*
 214          * Make sure stores to the GIC via the memory mapped interface
 215          * are now visible to the system register interface.
 216          */
 217         if (!cpu_if->vgic_sre) {
 218                 dsb(st);
 219                 cpu_if->vgic_vmcr = read_gicreg(ICH_VMCR_EL2);
 220         }
 221
 222         if (used_lrs) {
 223                 int i;
 224                 u32 nr_pre_bits;
 225
 226                 cpu_if->vgic_elrsr = read_gicreg(ICH_ELSR_EL2);
 227
 228                 write_gicreg(0, ICH_HCR_EL2);
 229                 val = read_gicreg(ICH_VTR_EL2);
 230                 nr_pre_bits = vtr_to_nr_pre_bits(val);
 231
 232                 for (i = 0; i < used_lrs; i++) {
 233                         if (cpu_if->vgic_elrsr & (1 << i))
 234                                 cpu_if->vgic_lr[i] &= ~ICH_LR_STATE;
 235                         else
 236                                 cpu_if->vgic_lr[i] = __gic_v3_get_lr(i);
 237
 238                         __gic_v3_set_lr(0, i);
 239                 }
 240
 241                 switch (nr_pre_bits) {
 242                 case 7:
 243                         cpu_if->vgic_ap0r[3] = __vgic_v3_read_ap0rn(3);
 244                         cpu_if->vgic_ap0r[2] = __vgic_v3_read_ap0rn(2);
 245                 case 6:
 246                         cpu_if->vgic_ap0r[1] = __vgic_v3_read_ap0rn(1);
 247                 default:
 248                         cpu_if->vgic_ap0r[0] = __vgic_v3_read_ap0rn(0);
 249                 }
 250
 251                 switch (nr_pre_bits) {
 252                 case 7:
 253                         cpu_if->vgic_ap1r[3] = __vgic_v3_read_ap1rn(3);
 254                         cpu_if->vgic_ap1r[2] = __vgic_v3_read_ap1rn(2);
 255                 case 6:
 256                         cpu_if->vgic_ap1r[1] = __vgic_v3_read_ap1rn(1);
 257                 default:
 258                         cpu_if->vgic_ap1r[0] = __vgic_v3_read_ap1rn(0);
 259                 }
 260         } else {
 261                 if (static_branch_unlikely(&vgic_v3_cpuif_trap))
 262                         write_gicreg(0, ICH_HCR_EL2);
 263
 264                 cpu_if->vgic_elrsr = 0xffff;
 265                 cpu_if->vgic_ap0r[0] = 0;
 266                 cpu_if->vgic_ap0r[1] = 0;
 267                 cpu_if->vgic_ap0r[2] = 0;
 268                 cpu_if->vgic_ap0r[3] = 0;
 269                 cpu_if->vgic_ap1r[0] = 0;
 270                 cpu_if->vgic_ap1r[1] = 0;
 271                 cpu_if->vgic_ap1r[2] = 0;
 272                 cpu_if->vgic_ap1r[3] = 0;
 273         }
 274
 275         val = read_gicreg(ICC_SRE_EL2);
 276         write_gicreg(val | ICC_SRE_EL2_ENABLE, ICC_SRE_EL2);
 277
 278         if (!cpu_if->vgic_sre) {
 279                 /* Make sure ENABLE is set at EL2 before setting SRE at EL1 */
 280                 isb();
 281                 write_gicreg(1, ICC_SRE_EL1);
 282         }
 283 }
 284
 285 void __hyp_text __vgic_v3_restore_state(struct kvm_vcpu *vcpu)
 286 {
 287         struct vgic_v3_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v3;
 288         u64 used_lrs = vcpu->arch.vgic_cpu.used_lrs;
 289         u64 val;
 290         u32 nr_pre_bits;
 291         int i;
 292
 293         /*
 294          * VFIQEn is RES1 if ICC_SRE_EL1.SRE is 1. This causes a
 295          * Group0 interrupt (as generated in GICv2 mode) to be
 296          * delivered as a FIQ to the guest, with potentially fatal
 297          * consequences. So we must make sure that ICC_SRE_EL1 has
 298          * been actually programmed with the value we want before
 299          * starting to mess with the rest of the GIC, and VMCR_EL2 in
 300          * particular.
 301          */
 302         if (!cpu_if->vgic_sre) {
 303                 write_gicreg(0, ICC_SRE_EL1);
 304                 isb();
 305                 write_gicreg(cpu_if->vgic_vmcr, ICH_VMCR_EL2);
 306         }
 307
 308         val = read_gicreg(ICH_VTR_EL2);
 309         nr_pre_bits = vtr_to_nr_pre_bits(val);
 310
 311         if (used_lrs) {
 312                 write_gicreg(cpu_if->vgic_hcr, ICH_HCR_EL2);
 313
 314                 switch (nr_pre_bits) {
 315                 case 7:
 316                         __vgic_v3_write_ap0rn(cpu_if->vgic_ap0r[3], 3);
 317                         __vgic_v3_write_ap0rn(cpu_if->vgic_ap0r[2], 2);
 318                 case 6:
 319                         __vgic_v3_write_ap0rn(cpu_if->vgic_ap0r[1], 1);
 320                 default:
 321                         __vgic_v3_write_ap0rn(cpu_if->vgic_ap0r[0], 0);
 322                 }
 323
 324                 switch (nr_pre_bits) {
 325                 case 7:
 326                         __vgic_v3_write_ap1rn(cpu_if->vgic_ap1r[3], 3);
 327                         __vgic_v3_write_ap1rn(cpu_if->vgic_ap1r[2], 2);
 328                 case 6:
 329                         __vgic_v3_write_ap1rn(cpu_if->vgic_ap1r[1], 1);
 330                 default:
 331                         __vgic_v3_write_ap1rn(cpu_if->vgic_ap1r[0], 0);
 332                 }
 333
 334                 for (i = 0; i < used_lrs; i++)
 335                         __gic_v3_set_lr(cpu_if->vgic_lr[i], i);
 336         } else {
 337                 /*
 338                  * If we need to trap system registers, we must write
 339                  * ICH_HCR_EL2 anyway, even if no interrupts are being
 340                  * injected,
 341                  */
 342                 if (static_branch_unlikely(&vgic_v3_cpuif_trap))
 343                         write_gicreg(cpu_if->vgic_hcr, ICH_HCR_EL2);
 344         }
 345
 346         /*
 347          * Ensures that the above will have reached the
 348          * (re)distributors. This ensure the guest will read the
 349          * correct values from the memory-mapped interface.
 350          */
 351         if (!cpu_if->vgic_sre) {
 352                 isb();
 353                 dsb(sy);
 354         }
 355
 356         /*
 357          * Prevent the guest from touching the GIC system registers if
 358          * SRE isn't enabled for GICv3 emulation.
 359          */
 360         write_gicreg(read_gicreg(ICC_SRE_EL2) & ~ICC_SRE_EL2_ENABLE,
 361                      ICC_SRE_EL2);
 362 }
 363
 364 void __hyp_text __vgic_v3_init_lrs(void)
 365 {
 366         int max_lr_idx = vtr_to_max_lr_idx(read_gicreg(ICH_VTR_EL2));
 367         int i;
 368
 369         for (i = 0; i <= max_lr_idx; i++)
 370                 __gic_v3_set_lr(0, i);
 371 }
 372
 373 u64 __hyp_text __vgic_v3_get_ich_vtr_el2(void)
 374 {
 375         return read_gicreg(ICH_VTR_EL2);
 376 }
 377
 378 u64 __hyp_text __vgic_v3_read_vmcr(void)
 379 {
 380         return read_gicreg(ICH_VMCR_EL2);
 381 }
 382
 383 void __hyp_text __vgic_v3_write_vmcr(u32 vmcr)
 384 {
 385         write_gicreg(vmcr, ICH_VMCR_EL2);
 386 }
 387
 388 #ifdef CONFIG_ARM64
 389
 390 static int __hyp_text __vgic_v3_bpr_min(void)
 391 {
 392         /* See Pseudocode for VPriorityGroup */
 393         return 8 - vtr_to_nr_pre_bits(read_gicreg(ICH_VTR_EL2));
 394 }
 395
 396 static int __hyp_text __vgic_v3_get_group(struct kvm_vcpu *vcpu)
 397 {
 398         u32 esr = kvm_vcpu_get_hsr(vcpu);
 399         u8 crm = (esr & ESR_ELx_SYS64_ISS_CRM_MASK) >> ESR_ELx_SYS64_ISS_CRM_SHIFT;
 400
 401         return crm != 8;
 402 }
 403
 404 #define GICv3_IDLE_PRIORITY     0xff
 405
 406 static int __hyp_text __vgic_v3_highest_priority_lr(struct kvm_vcpu *vcpu,
 407                                                     u32 vmcr,
 408                                                     u64 *lr_val)
 409 {
 410         unsigned int used_lrs = vcpu->arch.vgic_cpu.used_lrs;
 411         u8 priority = GICv3_IDLE_PRIORITY;
 412         int i, lr = -1;
 413
 414         for (i = 0; i < used_lrs; i++) {
 415                 u64 val = __gic_v3_get_lr(i);
 416                 u8 lr_prio = (val & ICH_LR_PRIORITY_MASK) >> ICH_LR_PRIORITY_SHIFT;
 417
 418                 /* Not pending in the state? */
 419                 if ((val & ICH_LR_STATE) != ICH_LR_PENDING_BIT)
 420                         continue;
 421
 422                 /* Group-0 interrupt, but Group-0 disabled? */
 423                 if (!(val & ICH_LR_GROUP) && !(vmcr & ICH_VMCR_ENG0_MASK))
 424                         continue;
 425
 426                 /* Group-1 interrupt, but Group-1 disabled? */
 427                 if ((val & ICH_LR_GROUP) && !(vmcr & ICH_VMCR_ENG1_MASK))
 428                         continue;
 429
 430                 /* Not the highest priority? */
 431                 if (lr_prio >= priority)
 432                         continue;
 433
 434                 /* This is a candidate */
 435                 priority = lr_prio;
 436                 *lr_val = val;
 437                 lr = i;
 438         }
 439
 440         if (lr == -1)
 441                 *lr_val = ICC_IAR1_EL1_SPURIOUS;
 442
 443         return lr;
 444 }
 445
 446 static int __hyp_text __vgic_v3_find_active_lr(struct kvm_vcpu *vcpu,
 447                                                int intid, u64 *lr_val)
 448 {
 449         unsigned int used_lrs = vcpu->arch.vgic_cpu.used_lrs;
 450         int i;
 451
 452         for (i = 0; i < used_lrs; i++) {
 453                 u64 val = __gic_v3_get_lr(i);
 454
 455                 if ((val & ICH_LR_VIRTUAL_ID_MASK) == intid &&
 456                     (val & ICH_LR_ACTIVE_BIT)) {
 457                         *lr_val = val;
 458                         return i;
 459                 }
 460         }
 461
 462         *lr_val = ICC_IAR1_EL1_SPURIOUS;
 463         return -1;
 464 }
 465
 466 static int __hyp_text __vgic_v3_get_highest_active_priority(void)
 467 {
 468         u8 nr_apr_regs = vtr_to_nr_apr_regs(read_gicreg(ICH_VTR_EL2));
 469         u32 hap = 0;
 470         int i;
 471
 472         for (i = 0; i < nr_apr_regs; i++) {
 473                 u32 val;
 474
 475                 /*
 476                  * The ICH_AP0Rn_EL2 and ICH_AP1Rn_EL2 registers
 477                  * contain the active priority levels for this VCPU
 478                  * for the maximum number of supported priority
 479                  * levels, and we return the full priority level only
 480                  * if the BPR is programmed to its minimum, otherwise
 481                  * we return a combination of the priority level and
 482                  * subpriority, as determined by the setting of the
 483                  * BPR, but without the full subpriority.
 484                  */
 485                 val  = __vgic_v3_read_ap0rn(i);
 486                 val |= __vgic_v3_read_ap1rn(i);
 487                 if (!val) {
 488                         hap += 32;
 489                         continue;
 490                 }
 491
 492                 return (hap + __ffs(val)) << __vgic_v3_bpr_min();
 493         }
 494
 495         return GICv3_IDLE_PRIORITY;
 496 }
 497
 498 static unsigned int __hyp_text __vgic_v3_get_bpr0(u32 vmcr)
 499 {
 500         return (vmcr & ICH_VMCR_BPR0_MASK) >> ICH_VMCR_BPR0_SHIFT;
 501 }
 502
 503 static unsigned int __hyp_text __vgic_v3_get_bpr1(u32 vmcr)
 504 {
 505         unsigned int bpr;
 506
 507         if (vmcr & ICH_VMCR_CBPR_MASK) {
 508                 bpr = __vgic_v3_get_bpr0(vmcr);
 509                 if (bpr < 7)
 510                         bpr++;
 511         } else {
 512                 bpr = (vmcr & ICH_VMCR_BPR1_MASK) >> ICH_VMCR_BPR1_SHIFT;
 513         }
 514
 515         return bpr;
 516 }
 517
 518 /*
 519  * Convert a priority to a preemption level, taking the relevant BPR
 520  * into account by zeroing the sub-priority bits.
 521  */
 522 static u8 __hyp_text __vgic_v3_pri_to_pre(u8 pri, u32 vmcr, int grp)
 523 {
 524         unsigned int bpr;
 525
 526         if (!grp)
 527                 bpr = __vgic_v3_get_bpr0(vmcr) + 1;
 528         else
 529                 bpr = __vgic_v3_get_bpr1(vmcr);
 530
 531         return pri & (GENMASK(7, 0) << bpr);
 532 }
 533
 534 /*
 535  * The priority value is independent of any of the BPR values, so we
 536  * normalize it using the minumal BPR value. This guarantees that no
 537  * matter what the guest does with its BPR, we can always set/get the
 538  * same value of a priority.
 539  */
 540 static void __hyp_text __vgic_v3_set_active_priority(u8 pri, u32 vmcr, int grp)
 541 {
 542         u8 pre, ap;
 543         u32 val;
 544         int apr;
 545
 546         pre = __vgic_v3_pri_to_pre(pri, vmcr, grp);
 547         ap = pre >> __vgic_v3_bpr_min();
 548         apr = ap / 32;
 549
 550         if (!grp) {
 551                 val = __vgic_v3_read_ap0rn(apr);
 552                 __vgic_v3_write_ap0rn(val | BIT(ap % 32), apr);
 553         } else {
 554                 val = __vgic_v3_read_ap1rn(apr);
 555                 __vgic_v3_write_ap1rn(val | BIT(ap % 32), apr);
 556         }
 557 }
 558
 559 static int __hyp_text __vgic_v3_clear_highest_active_priority(void)
 560 {
 561         u8 nr_apr_regs = vtr_to_nr_apr_regs(read_gicreg(ICH_VTR_EL2));
 562         u32 hap = 0;
 563         int i;
 564
 565         for (i = 0; i < nr_apr_regs; i++) {
 566                 u32 ap0, ap1;
 567                 int c0, c1;
 568
 569                 ap0 = __vgic_v3_read_ap0rn(i);
 570                 ap1 = __vgic_v3_read_ap1rn(i);
 571                 if (!ap0 && !ap1) {
 572                         hap += 32;
 573                         continue;
 574                 }
 575
 576                 c0 = ap0 ? __ffs(ap0) : 32;
 577                 c1 = ap1 ? __ffs(ap1) : 32;
 578
 579                 /* Always clear the LSB, which is the highest priority */
 580                 if (c0 < c1) {
 581                         ap0 &= ~BIT(c0);
 582                         __vgic_v3_write_ap0rn(ap0, i);
 583                         hap += c0;
 584                 } else {
 585                         ap1 &= ~BIT(c1);
 586                         __vgic_v3_write_ap1rn(ap1, i);
 587                         hap += c1;
 588                 }
 589
 590                 /* Rescale to 8 bits of priority */
 591                 return hap << __vgic_v3_bpr_min();
 592         }
 593
 594         return GICv3_IDLE_PRIORITY;
 595 }
 596
 597 static void __hyp_text __vgic_v3_read_iar(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
 598 {
 599         u64 lr_val;
 600         u8 lr_prio, pmr;
 601         int lr, grp;
 602
 603         grp = __vgic_v3_get_group(vcpu);
 604
 605         lr = __vgic_v3_highest_priority_lr(vcpu, vmcr, &lr_val);
 606         if (lr < 0)
 607                 goto spurious;
 608
 609         if (grp != !!(lr_val & ICH_LR_GROUP))
 610                 goto spurious;
 611
 612         pmr = (vmcr & ICH_VMCR_PMR_MASK) >> ICH_VMCR_PMR_SHIFT;
 613         lr_prio = (lr_val & ICH_LR_PRIORITY_MASK) >> ICH_LR_PRIORITY_SHIFT;
 614         if (pmr <= lr_prio)
 615                 goto spurious;
 616
 617         if (__vgic_v3_get_highest_active_priority() <= __vgic_v3_pri_to_pre(lr_prio, vmcr, grp))
 618                 goto spurious;
 619
 620         lr_val &= ~ICH_LR_STATE;
 621         /* No active state for LPIs */
 622         if ((lr_val & ICH_LR_VIRTUAL_ID_MASK) <= VGIC_MAX_SPI)
 623                 lr_val |= ICH_LR_ACTIVE_BIT;
 624         __gic_v3_set_lr(lr_val, lr);
 625         __vgic_v3_set_active_priority(lr_prio, vmcr, grp);
 626         vcpu_set_reg(vcpu, rt, lr_val & ICH_LR_VIRTUAL_ID_MASK);
 627         return;
 628
 629 spurious:
 630         vcpu_set_reg(vcpu, rt, ICC_IAR1_EL1_SPURIOUS);
 631 }
 632
 633 static void __hyp_text __vgic_v3_clear_active_lr(int lr, u64 lr_val)
 634 {
 635         lr_val &= ~ICH_LR_ACTIVE_BIT;
 636         if (lr_val & ICH_LR_HW) {
 637                 u32 pid;
 638
 639                 pid = (lr_val & ICH_LR_PHYS_ID_MASK) >> ICH_LR_PHYS_ID_SHIFT;
 640                 gic_write_dir(pid);
 641         }
 642
 643         __gic_v3_set_lr(lr_val, lr);
 644 }
 645
 646 static void __hyp_text __vgic_v3_bump_eoicount(void)
 647 {
 648         u32 hcr;
 649
 650         hcr = read_gicreg(ICH_HCR_EL2);
 651         hcr += 1 << ICH_HCR_EOIcount_SHIFT;
 652         write_gicreg(hcr, ICH_HCR_EL2);
 653 }
 654
 655 static void __hyp_text __vgic_v3_write_dir(struct kvm_vcpu *vcpu,
 656                                            u32 vmcr, int rt)
 657 {
 658         u32 vid = vcpu_get_reg(vcpu, rt);
 659         u64 lr_val;
 660         int lr;
 661
 662         /* EOImode == 0, nothing to be done here */
 663         if (!(vmcr & ICH_VMCR_EOIM_MASK))
 664                 return;
 665
 666         /* No deactivate to be performed on an LPI */
 667         if (vid >= VGIC_MIN_LPI)
 668                 return;
 669
 670         lr = __vgic_v3_find_active_lr(vcpu, vid, &lr_val);
 671         if (lr == -1) {
 672                 __vgic_v3_bump_eoicount();
 673                 return;
 674         }
 675
 676         __vgic_v3_clear_active_lr(lr, lr_val);
 677 }
 678
 679 static void __hyp_text __vgic_v3_write_eoir(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
 680 {
 681         u32 vid = vcpu_get_reg(vcpu, rt);
 682         u64 lr_val;
 683         u8 lr_prio, act_prio;
 684         int lr, grp;
 685
 686         grp = __vgic_v3_get_group(vcpu);
 687
 688         /* Drop priority in any case */
 689         act_prio = __vgic_v3_clear_highest_active_priority();
 690
 691         /* If EOIing an LPI, no deactivate to be performed */
 692         if (vid >= VGIC_MIN_LPI)
 693                 return;
 694
 695         /* EOImode == 1, nothing to be done here */
 696         if (vmcr & ICH_VMCR_EOIM_MASK)
 697                 return;
 698
 699         lr = __vgic_v3_find_active_lr(vcpu, vid, &lr_val);
 700         if (lr == -1) {
 701                 __vgic_v3_bump_eoicount();
 702                 return;
 703         }
 704
 705         lr_prio = (lr_val & ICH_LR_PRIORITY_MASK) >> ICH_LR_PRIORITY_SHIFT;
 706
 707         /* If priorities or group do not match, the guest has fscked-up. */
 708         if (grp != !!(lr_val & ICH_LR_GROUP) ||
 709             __vgic_v3_pri_to_pre(lr_prio, vmcr, grp) != act_prio)
 710                 return;
 711
 712         /* Let's now perform the deactivation */
 713         __vgic_v3_clear_active_lr(lr, lr_val);
 714 }
 715
 716 static void __hyp_text __vgic_v3_read_igrpen0(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
 717 {
 718         vcpu_set_reg(vcpu, rt, !!(vmcr & ICH_VMCR_ENG0_MASK));
 719 }
 720
 721 static void __hyp_text __vgic_v3_read_igrpen1(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
 722 {
 723         vcpu_set_reg(vcpu, rt, !!(vmcr & ICH_VMCR_ENG1_MASK));
 724 }
 725
 726 static void __hyp_text __vgic_v3_write_igrpen0(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
 727 {
 728         u64 val = vcpu_get_reg(vcpu, rt);
 729
 730         if (val & 1)
 731                 vmcr |= ICH_VMCR_ENG0_MASK;
 732         else
 733                 vmcr &= ~ICH_VMCR_ENG0_MASK;
 734
 735         __vgic_v3_write_vmcr(vmcr);
 736 }
 737
 738 static void __hyp_text __vgic_v3_write_igrpen1(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
 739 {
 740         u64 val = vcpu_get_reg(vcpu, rt);
 741
 742         if (val & 1)
 743                 vmcr |= ICH_VMCR_ENG1_MASK;
 744         else
 745                 vmcr &= ~ICH_VMCR_ENG1_MASK;
 746
 747         __vgic_v3_write_vmcr(vmcr);
 748 }
 749
 750 static void __hyp_text __vgic_v3_read_bpr0(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
 751 {
 752         vcpu_set_reg(vcpu, rt, __vgic_v3_get_bpr0(vmcr));
 753 }
 754
 755 static void __hyp_text __vgic_v3_read_bpr1(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
 756 {
 757         vcpu_set_reg(vcpu, rt, __vgic_v3_get_bpr1(vmcr));
 758 }
 759
 760 static void __hyp_text __vgic_v3_write_bpr0(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
 761 {
 762         u64 val = vcpu_get_reg(vcpu, rt);
 763         u8 bpr_min = __vgic_v3_bpr_min() - 1;
 764
 765         /* Enforce BPR limiting */
 766         if (val < bpr_min)
 767                 val = bpr_min;
 768
 769         val <<= ICH_VMCR_BPR0_SHIFT;
 770         val &= ICH_VMCR_BPR0_MASK;
 771         vmcr &= ~ICH_VMCR_BPR0_MASK;
 772         vmcr |= val;
 773
 774         __vgic_v3_write_vmcr(vmcr);
 775 }
 776
 777 static void __hyp_text __vgic_v3_write_bpr1(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
 778 {
 779         u64 val = vcpu_get_reg(vcpu, rt);
 780         u8 bpr_min = __vgic_v3_bpr_min();
 781
 782         if (vmcr & ICH_VMCR_CBPR_MASK)
 783                 return;
 784
 785         /* Enforce BPR limiting */
 786         if (val < bpr_min)
 787                 val = bpr_min;
 788
 789         val <<= ICH_VMCR_BPR1_SHIFT;
 790         val &= ICH_VMCR_BPR1_MASK;
 791         vmcr &= ~ICH_VMCR_BPR1_MASK;
 792         vmcr |= val;
 793
 794         __vgic_v3_write_vmcr(vmcr);
 795 }
 796
 797 static void __hyp_text __vgic_v3_read_apxrn(struct kvm_vcpu *vcpu, int rt, int n)
 798 {
 799         u32 val;
 800
 801         if (!__vgic_v3_get_group(vcpu))
 802                 val = __vgic_v3_read_ap0rn(n);
 803         else
 804                 val = __vgic_v3_read_ap1rn(n);
 805
 806         vcpu_set_reg(vcpu, rt, val);
 807 }
 808
 809 static void __hyp_text __vgic_v3_write_apxrn(struct kvm_vcpu *vcpu, int rt, int n)
 810 {
 811         u32 val = vcpu_get_reg(vcpu, rt);
 812
 813         if (!__vgic_v3_get_group(vcpu))
 814                 __vgic_v3_write_ap0rn(val, n);
 815         else
 816                 __vgic_v3_write_ap1rn(val, n);
 817 }
 818
 819 static void __hyp_text __vgic_v3_read_apxr0(struct kvm_vcpu *vcpu,
 820                                             u32 vmcr, int rt)
 821 {
 822         __vgic_v3_read_apxrn(vcpu, rt, 0);
 823 }
 824
 825 static void __hyp_text __vgic_v3_read_apxr1(struct kvm_vcpu *vcpu,
 826                                             u32 vmcr, int rt)
 827 {
 828         __vgic_v3_read_apxrn(vcpu, rt, 1);
 829 }
 830
 831 static void __hyp_text __vgic_v3_read_apxr2(struct kvm_vcpu *vcpu,
 832                                             u32 vmcr, int rt)
 833 {
 834         __vgic_v3_read_apxrn(vcpu, rt, 2);
 835 }
 836
 837 static void __hyp_text __vgic_v3_read_apxr3(struct kvm_vcpu *vcpu,
 838                                             u32 vmcr, int rt)
 839 {
 840         __vgic_v3_read_apxrn(vcpu, rt, 3);
 841 }
 842
 843 static void __hyp_text __vgic_v3_write_apxr0(struct kvm_vcpu *vcpu,
 844                                              u32 vmcr, int rt)
 845 {
 846         __vgic_v3_write_apxrn(vcpu, rt, 0);
 847 }
 848
 849 static void __hyp_text __vgic_v3_write_apxr1(struct kvm_vcpu *vcpu,
 850                                              u32 vmcr, int rt)
 851 {
 852         __vgic_v3_write_apxrn(vcpu, rt, 1);
 853 }
 854
 855 static void __hyp_text __vgic_v3_write_apxr2(struct kvm_vcpu *vcpu,
 856                                              u32 vmcr, int rt)
 857 {
 858         __vgic_v3_write_apxrn(vcpu, rt, 2);
 859 }
 860
 861 static void __hyp_text __vgic_v3_write_apxr3(struct kvm_vcpu *vcpu,
 862                                              u32 vmcr, int rt)
 863 {
 864         __vgic_v3_write_apxrn(vcpu, rt, 3);
 865 }
 866
 867 static void __hyp_text __vgic_v3_read_hppir(struct kvm_vcpu *vcpu,
 868                                             u32 vmcr, int rt)
 869 {
 870         u64 lr_val;
 871         int lr, lr_grp, grp;
 872
 873         grp = __vgic_v3_get_group(vcpu);
 874
 875         lr = __vgic_v3_highest_priority_lr(vcpu, vmcr, &lr_val);
 876         if (lr == -1)
 877                 goto spurious;
 878
 879         lr_grp = !!(lr_val & ICH_LR_GROUP);
 880         if (lr_grp != grp)
 881                 lr_val = ICC_IAR1_EL1_SPURIOUS;
 882
 883 spurious:
 884         vcpu_set_reg(vcpu, rt, lr_val & ICH_LR_VIRTUAL_ID_MASK);
 885 }
 886
 887 static void __hyp_text __vgic_v3_read_pmr(struct kvm_vcpu *vcpu,
 888                                           u32 vmcr, int rt)
 889 {
 890         vmcr &= ICH_VMCR_PMR_MASK;
 891         vmcr >>= ICH_VMCR_PMR_SHIFT;
 892         vcpu_set_reg(vcpu, rt, vmcr);
 893 }
 894
 895 static void __hyp_text __vgic_v3_write_pmr(struct kvm_vcpu *vcpu,
 896                                            u32 vmcr, int rt)
 897 {
 898         u32 val = vcpu_get_reg(vcpu, rt);
 899
 900         val <<= ICH_VMCR_PMR_SHIFT;
 901         val &= ICH_VMCR_PMR_MASK;
 902         vmcr &= ~ICH_VMCR_PMR_MASK;
 903         vmcr |= val;
 904
 905         write_gicreg(vmcr, ICH_VMCR_EL2);
 906 }
 907
 908 static void __hyp_text __vgic_v3_read_rpr(struct kvm_vcpu *vcpu,
 909                                           u32 vmcr, int rt)
 910 {
 911         u32 val = __vgic_v3_get_highest_active_priority();
 912         vcpu_set_reg(vcpu, rt, val);
 913 }
 914
 915 static void __hyp_text __vgic_v3_read_ctlr(struct kvm_vcpu *vcpu,
 916                                            u32 vmcr, int rt)
 917 {
 918         u32 vtr, val;
 919
 920         vtr = read_gicreg(ICH_VTR_EL2);
 921         /* PRIbits */
 922         val = ((vtr >> 29) & 7) << ICC_CTLR_EL1_PRI_BITS_SHIFT;
 923         /* IDbits */
 924         val |= ((vtr >> 23) & 7) << ICC_CTLR_EL1_ID_BITS_SHIFT;
 925         /* SEIS */
 926         val |= ((vtr >> 22) & 1) << ICC_CTLR_EL1_SEIS_SHIFT;
 927         /* A3V */
 928         val |= ((vtr >> 21) & 1) << ICC_CTLR_EL1_A3V_SHIFT;
 929         /* EOImode */
 930         val |= ((vmcr & ICH_VMCR_EOIM_MASK) >> ICH_VMCR_EOIM_SHIFT) << ICC_CTLR_EL1_EOImode_SHIFT;
 931         /* CBPR */
 932         val |= (vmcr & ICH_VMCR_CBPR_MASK) >> ICH_VMCR_CBPR_SHIFT;
 933
 934         vcpu_set_reg(vcpu, rt, val);
 935 }
 936
 937 static void __hyp_text __vgic_v3_write_ctlr(struct kvm_vcpu *vcpu,
 938                                             u32 vmcr, int rt)
 939 {
 940         u32 val = vcpu_get_reg(vcpu, rt);
 941
 942         if (val & ICC_CTLR_EL1_CBPR_MASK)
 943                 vmcr |= ICH_VMCR_CBPR_MASK;
 944         else
 945                 vmcr &= ~ICH_VMCR_CBPR_MASK;
 946
 947         if (val & ICC_CTLR_EL1_EOImode_MASK)
 948                 vmcr |= ICH_VMCR_EOIM_MASK;
 949         else
 950                 vmcr &= ~ICH_VMCR_EOIM_MASK;
 951
 952         write_gicreg(vmcr, ICH_VMCR_EL2);
 953 }
 954
 955 int __hyp_text __vgic_v3_perform_cpuif_access(struct kvm_vcpu *vcpu)
 956 {
 957         int rt;
 958         u32 esr;
 959         u32 vmcr;
 960         void (*fn)(struct kvm_vcpu *, u32, int);
 961         bool is_read;
 962         u32 sysreg;
 963
 964         esr = kvm_vcpu_get_hsr(vcpu);
 965         if (vcpu_mode_is_32bit(vcpu)) {
 966                 if (!kvm_condition_valid(vcpu))
 967                         return 1;
 968
 969                 sysreg = esr_cp15_to_sysreg(esr);
 970         } else {
 971                 sysreg = esr_sys64_to_sysreg(esr);
 972         }
 973
 974         is_read = (esr & ESR_ELx_SYS64_ISS_DIR_MASK) == ESR_ELx_SYS64_ISS_DIR_READ;
 975
 976         switch (sysreg) {
 977         case SYS_ICC_IAR0_EL1:
 978         case SYS_ICC_IAR1_EL1:
 979                 if (unlikely(!is_read))
 980                         return 0;
 981                 fn = __vgic_v3_read_iar;
 982                 break;
 983         case SYS_ICC_EOIR0_EL1:
 984         case SYS_ICC_EOIR1_EL1:
 985                 if (unlikely(is_read))
 986                         return 0;
 987                 fn = __vgic_v3_write_eoir;
 988                 break;
 989         case SYS_ICC_IGRPEN1_EL1:
 990                 if (is_read)
 991                         fn = __vgic_v3_read_igrpen1;
 992                 else
 993                         fn = __vgic_v3_write_igrpen1;
 994                 break;
 995         case SYS_ICC_BPR1_EL1:
 996                 if (is_read)
 997                         fn = __vgic_v3_read_bpr1;
 998                 else
 999                         fn = __vgic_v3_write_bpr1;
1000                 break;
1001         case SYS_ICC_AP0Rn_EL1(0):
1002         case SYS_ICC_AP1Rn_EL1(0):
1003                 if (is_read)
1004                         fn = __vgic_v3_read_apxr0;
1005                 else
1006                         fn = __vgic_v3_write_apxr0;
1007                 break;
1008         case SYS_ICC_AP0Rn_EL1(1):
1009         case SYS_ICC_AP1Rn_EL1(1):
1010                 if (is_read)
1011                         fn = __vgic_v3_read_apxr1;
1012                 else
1013                         fn = __vgic_v3_write_apxr1;
1014                 break;
1015         case SYS_ICC_AP0Rn_EL1(2):
1016         case SYS_ICC_AP1Rn_EL1(2):
1017                 if (is_read)
1018                         fn = __vgic_v3_read_apxr2;
1019                 else
1020                         fn = __vgic_v3_write_apxr2;
1021                 break;
1022         case SYS_ICC_AP0Rn_EL1(3):
1023         case SYS_ICC_AP1Rn_EL1(3):
1024                 if (is_read)
1025                         fn = __vgic_v3_read_apxr3;
1026                 else
1027                         fn = __vgic_v3_write_apxr3;
1028                 break;
1029         case SYS_ICC_HPPIR0_EL1:
1030         case SYS_ICC_HPPIR1_EL1:
1031                 if (unlikely(!is_read))
1032                         return 0;
1033                 fn = __vgic_v3_read_hppir;
1034                 break;
1035         case SYS_ICC_IGRPEN0_EL1:
1036                 if (is_read)
1037                         fn = __vgic_v3_read_igrpen0;
1038                 else
1039                         fn = __vgic_v3_write_igrpen0;
1040                 break;
1041         case SYS_ICC_BPR0_EL1:
1042                 if (is_read)
1043                         fn = __vgic_v3_read_bpr0;
1044                 else
1045                         fn = __vgic_v3_write_bpr0;
1046                 break;
1047         case SYS_ICC_DIR_EL1:
1048                 if (unlikely(is_read))
1049                         return 0;
1050                 fn = __vgic_v3_write_dir;
1051                 break;
1052         case SYS_ICC_RPR_EL1:
1053                 if (unlikely(!is_read))
1054                         return 0;
1055                 fn = __vgic_v3_read_rpr;
1056                 break;
1057         case SYS_ICC_CTLR_EL1:
1058                 if (is_read)
1059                         fn = __vgic_v3_read_ctlr;
1060                 else
1061                         fn = __vgic_v3_write_ctlr;
1062                 break;
1063         case SYS_ICC_PMR_EL1:
1064                 if (is_read)
1065                         fn = __vgic_v3_read_pmr;
1066                 else
1067                         fn = __vgic_v3_write_pmr;
1068                 break;
1069         default:
1070                 return 0;
1071         }
1072
1073         vmcr = __vgic_v3_read_vmcr();
1074         rt = kvm_vcpu_sys_get_rt(vcpu);
1075         fn(vcpu, vmcr, rt);
1076
1077         return 1;
1078 }
1079
1080 #endif