arch/powerpc/kvm/book3s_64_mmu.c

   1 /*
   2  * This program is free software; you can redistribute it and/or modify
   3  * it under the terms of the GNU General Public License, version 2, as
   4  * published by the Free Software Foundation.
   5  *
   6  * This program is distributed in the hope that it will be useful,
   7  * but WITHOUT ANY WARRANTY; without even the implied warranty of
   8  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   9  * GNU General Public License for more details.
  10  *
  11  * You should have received a copy of the GNU General Public License
  12  * along with this program; if not, write to the Free Software
  13  * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
  14  *
  15  * Copyright SUSE Linux Products GmbH 2009
  16  *
  17  * Authors: Alexander Graf <agraf@suse.de>
  18  */
  19
  20 #include <linux/types.h>
  21 #include <linux/string.h>
  22 #include <linux/kvm.h>
  23 #include <linux/kvm_host.h>
  24 #include <linux/highmem.h>
  25
  26 #include <asm/tlbflush.h>
  27 #include <asm/kvm_ppc.h>
  28 #include <asm/kvm_book3s.h>
  29
  30 /* #define DEBUG_MMU */
  31
  32 #ifdef DEBUG_MMU
  33 #define dprintk(X...) printk(KERN_INFO X)
  34 #else
  35 #define dprintk(X...) do { } while(0)
  36 #endif
  37
  38 static void kvmppc_mmu_book3s_64_reset_msr(struct kvm_vcpu *vcpu)
  39 {
  40         kvmppc_set_msr(vcpu, MSR_SF);
  41 }
  42
  43 static struct kvmppc_slb *kvmppc_mmu_book3s_64_find_slbe(
  44                                 struct kvm_vcpu *vcpu,
  45                                 gva_t eaddr)
  46 {
  47         int i;
  48         u64 esid = GET_ESID(eaddr);
  49         u64 esid_1t = GET_ESID_1T(eaddr);
  50
  51         for (i = 0; i < vcpu->arch.slb_nr; i++) {
  52                 u64 cmp_esid = esid;
  53
  54                 if (!vcpu->arch.slb[i].valid)
  55                         continue;
  56
  57                 if (vcpu->arch.slb[i].tb)
  58                         cmp_esid = esid_1t;
  59
  60                 if (vcpu->arch.slb[i].esid == cmp_esid)
  61                         return &vcpu->arch.slb[i];
  62         }
  63
  64         dprintk("KVM: No SLB entry found for 0x%lx [%llx | %llx]\n",
  65                 eaddr, esid, esid_1t);
  66         for (i = 0; i < vcpu->arch.slb_nr; i++) {
  67             if (vcpu->arch.slb[i].vsid)
  68                 dprintk("  %d: %c%c%c %llx %llx\n", i,
  69                         vcpu->arch.slb[i].valid ? 'v' : ' ',
  70                         vcpu->arch.slb[i].large ? 'l' : ' ',
  71                         vcpu->arch.slb[i].tb    ? 't' : ' ',
  72                         vcpu->arch.slb[i].esid,
  73                         vcpu->arch.slb[i].vsid);
  74         }
  75
  76         return NULL;
  77 }
  78
  79 static u64 kvmppc_mmu_book3s_64_ea_to_vp(struct kvm_vcpu *vcpu, gva_t eaddr,
  80                                          bool data)
  81 {
  82         struct kvmppc_slb *slb;
  83
  84         slb = kvmppc_mmu_book3s_64_find_slbe(vcpu, eaddr);
  85         if (!slb)
  86                 return 0;
  87
  88         if (slb->tb)
  89                 return (((u64)eaddr >> 12) & 0xfffffff) |
  90                        (((u64)slb->vsid) << 28);
  91
  92         return (((u64)eaddr >> 12) & 0xffff) | (((u64)slb->vsid) << 16);
  93 }
  94
  95 static int kvmppc_mmu_book3s_64_get_pagesize(struct kvmppc_slb *slbe)
  96 {
  97         return slbe->large ? 24 : 12;
  98 }
  99
 100 static u32 kvmppc_mmu_book3s_64_get_page(struct kvmppc_slb *slbe, gva_t eaddr)
 101 {
 102         int p = kvmppc_mmu_book3s_64_get_pagesize(slbe);
 103         return ((eaddr & 0xfffffff) >> p);
 104 }
 105
 106 static hva_t kvmppc_mmu_book3s_64_get_pteg(
 107                                 struct kvmppc_vcpu_book3s *vcpu_book3s,
 108                                 struct kvmppc_slb *slbe, gva_t eaddr,
 109                                 bool second)
 110 {
 111         u64 hash, pteg, htabsize;
 112         u32 page;
 113         hva_t r;
 114
 115         page = kvmppc_mmu_book3s_64_get_page(slbe, eaddr);
 116         htabsize = ((1 << ((vcpu_book3s->sdr1 & 0x1f) + 11)) - 1);
 117
 118         hash = slbe->vsid ^ page;
 119         if (second)
 120                 hash = ~hash;
 121         hash &= ((1ULL << 39ULL) - 1ULL);
 122         hash &= htabsize;
 123         hash <<= 7ULL;
 124
 125         pteg = vcpu_book3s->sdr1 & 0xfffffffffffc0000ULL;
 126         pteg |= hash;
 127
 128         dprintk("MMU: page=0x%x sdr1=0x%llx pteg=0x%llx vsid=0x%llx\n",
 129                 page, vcpu_book3s->sdr1, pteg, slbe->vsid);
 130
 131         /* When running a PAPR guest, SDR1 contains a HVA address instead
 132            of a GPA */
 133         if (vcpu_book3s->vcpu.arch.papr_enabled)
 134                 r = pteg;
 135         else
 136                 r = gfn_to_hva(vcpu_book3s->vcpu.kvm, pteg >> PAGE_SHIFT);
 137
 138         if (kvm_is_error_hva(r))
 139                 return r;
 140         return r | (pteg & ~PAGE_MASK);
 141 }
 142
 143 static u64 kvmppc_mmu_book3s_64_get_avpn(struct kvmppc_slb *slbe, gva_t eaddr)
 144 {
 145         int p = kvmppc_mmu_book3s_64_get_pagesize(slbe);
 146         u64 avpn;
 147
 148         avpn = kvmppc_mmu_book3s_64_get_page(slbe, eaddr);
 149         avpn |= slbe->vsid << (28 - p);
 150
 151         if (p < 24)
 152                 avpn >>= ((80 - p) - 56) - 8;
 153         else
 154                 avpn <<= 8;
 155
 156         return avpn;
 157 }
 158
 159 static int kvmppc_mmu_book3s_64_xlate(struct kvm_vcpu *vcpu, gva_t eaddr,
 160                                 struct kvmppc_pte *gpte, bool data)
 161 {
 162         struct kvmppc_vcpu_book3s *vcpu_book3s = to_book3s(vcpu);
 163         struct kvmppc_slb *slbe;
 164         hva_t ptegp;
 165         u64 pteg[16];
 166         u64 avpn = 0;
 167         int i;
 168         u8 key = 0;
 169         bool found = false;
 170         bool perm_err = false;
 171         int second = 0;
 172         ulong mp_ea = vcpu->arch.magic_page_ea;
 173
 174         /* Magic page override */
 175         if (unlikely(mp_ea) &&
 176             unlikely((eaddr & ~0xfffULL) == (mp_ea & ~0xfffULL)) &&
 177             !(vcpu->arch.shared->msr & MSR_PR)) {
 178                 gpte->eaddr = eaddr;
 179                 gpte->vpage = kvmppc_mmu_book3s_64_ea_to_vp(vcpu, eaddr, data);
 180                 gpte->raddr = vcpu->arch.magic_page_pa | (gpte->raddr & 0xfff);
 181                 gpte->raddr &= KVM_PAM;
 182                 gpte->may_execute = true;
 183                 gpte->may_read = true;
 184                 gpte->may_write = true;
 185
 186                 return 0;
 187         }
 188
 189         slbe = kvmppc_mmu_book3s_64_find_slbe(vcpu, eaddr);
 190         if (!slbe)
 191                 goto no_seg_found;
 192
 193 do_second:
 194         ptegp = kvmppc_mmu_book3s_64_get_pteg(vcpu_book3s, slbe, eaddr, second);
 195         if (kvm_is_error_hva(ptegp))
 196                 goto no_page_found;
 197
 198         avpn = kvmppc_mmu_book3s_64_get_avpn(slbe, eaddr);
 199
 200         if(copy_from_user(pteg, (void __user *)ptegp, sizeof(pteg))) {
 201                 printk(KERN_ERR "KVM can't copy data from 0x%lx!\n", ptegp);
 202                 goto no_page_found;
 203         }
 204
 205         if ((vcpu->arch.shared->msr & MSR_PR) && slbe->Kp)
 206                 key = 4;
 207         else if (!(vcpu->arch.shared->msr & MSR_PR) && slbe->Ks)
 208                 key = 4;
 209
 210         for (i=0; i<16; i+=2) {
 211                 u64 v = pteg[i];
 212                 u64 r = pteg[i+1];
 213
 214                 /* Valid check */
 215                 if (!(v & HPTE_V_VALID))
 216                         continue;
 217                 /* Hash check */
 218                 if ((v & HPTE_V_SECONDARY) != second)
 219                         continue;
 220
 221                 /* AVPN compare */
 222                 if (HPTE_V_AVPN_VAL(avpn) == HPTE_V_AVPN_VAL(v)) {
 223                         u8 pp = (r & HPTE_R_PP) | key;
 224                         int eaddr_mask = 0xFFF;
 225
 226                         gpte->eaddr = eaddr;
 227                         gpte->vpage = kvmppc_mmu_book3s_64_ea_to_vp(vcpu,
 228                                                                     eaddr,
 229                                                                     data);
 230                         if (slbe->large)
 231                                 eaddr_mask = 0xFFFFFF;
 232                         gpte->raddr = (r & HPTE_R_RPN) | (eaddr & eaddr_mask);
 233                         gpte->may_execute = ((r & HPTE_R_N) ? false : true);
 234                         gpte->may_read = false;
 235                         gpte->may_write = false;
 236
 237                         switch (pp) {
 238                         case 0:
 239                         case 1:
 240                         case 2:
 241                         case 6:
 242                                 gpte->may_write = true;
 243                                 /* fall through */
 244                         case 3:
 245                         case 5:
 246                         case 7:
 247                                 gpte->may_read = true;
 248                                 break;
 249                         }
 250
 251                         if (!gpte->may_read) {
 252                                 perm_err = true;
 253                                 continue;
 254                         }
 255
 256                         dprintk("KVM MMU: Translated 0x%lx [0x%llx] -> 0x%llx "
 257                                 "-> 0x%lx\n",
 258                                 eaddr, avpn, gpte->vpage, gpte->raddr);
 259                         found = true;
 260                         break;
 261                 }
 262         }
 263
 264         /* Update PTE R and C bits, so the guest's swapper knows we used the
 265          * page */
 266         if (found) {
 267                 u32 oldr = pteg[i+1];
 268
 269                 if (gpte->may_read) {
 270                         /* Set the accessed flag */
 271                         pteg[i+1] |= HPTE_R_R;
 272                 }
 273                 if (gpte->may_write) {
 274                         /* Set the dirty flag */
 275                         pteg[i+1] |= HPTE_R_C;
 276                 } else {
 277                         dprintk("KVM: Mapping read-only page!\n");
 278                 }
 279
 280                 /* Write back into the PTEG */
 281                 if (pteg[i+1] != oldr)
 282                         copy_to_user((void __user *)ptegp, pteg, sizeof(pteg));
 283
 284                 return 0;
 285         } else {
 286                 dprintk("KVM MMU: No PTE found (ea=0x%lx sdr1=0x%llx "
 287                         "ptegp=0x%lx)\n",
 288                         eaddr, to_book3s(vcpu)->sdr1, ptegp);
 289                 for (i = 0; i < 16; i += 2)
 290                         dprintk("   %02d: 0x%llx - 0x%llx (0x%llx)\n",
 291                                 i, pteg[i], pteg[i+1], avpn);
 292
 293                 if (!second) {
 294                         second = HPTE_V_SECONDARY;
 295                         goto do_second;
 296                 }
 297         }
 298
 299
 300 no_page_found:
 301
 302
 303         if (perm_err)
 304                 return -EPERM;
 305
 306         return -ENOENT;
 307
 308 no_seg_found:
 309
 310         dprintk("KVM MMU: Trigger segment fault\n");
 311         return -EINVAL;
 312 }
 313
 314 static void kvmppc_mmu_book3s_64_slbmte(struct kvm_vcpu *vcpu, u64 rs, u64 rb)
 315 {
 316         struct kvmppc_vcpu_book3s *vcpu_book3s;
 317         u64 esid, esid_1t;
 318         int slb_nr;
 319         struct kvmppc_slb *slbe;
 320
 321         dprintk("KVM MMU: slbmte(0x%llx, 0x%llx)\n", rs, rb);
 322
 323         vcpu_book3s = to_book3s(vcpu);
 324
 325         esid = GET_ESID(rb);
 326         esid_1t = GET_ESID_1T(rb);
 327         slb_nr = rb & 0xfff;
 328
 329         if (slb_nr > vcpu->arch.slb_nr)
 330                 return;
 331
 332         slbe = &vcpu->arch.slb[slb_nr];
 333
 334         slbe->large = (rs & SLB_VSID_L) ? 1 : 0;
 335         slbe->tb    = (rs & SLB_VSID_B_1T) ? 1 : 0;
 336         slbe->esid  = slbe->tb ? esid_1t : esid;
 337         slbe->vsid  = rs >> 12;
 338         slbe->valid = (rb & SLB_ESID_V) ? 1 : 0;
 339         slbe->Ks    = (rs & SLB_VSID_KS) ? 1 : 0;
 340         slbe->Kp    = (rs & SLB_VSID_KP) ? 1 : 0;
 341         slbe->nx    = (rs & SLB_VSID_N) ? 1 : 0;
 342         slbe->class = (rs & SLB_VSID_C) ? 1 : 0;
 343
 344         slbe->orige = rb & (ESID_MASK | SLB_ESID_V);
 345         slbe->origv = rs;
 346
 347         /* Map the new segment */
 348         kvmppc_mmu_map_segment(vcpu, esid << SID_SHIFT);
 349 }
 350
 351 static u64 kvmppc_mmu_book3s_64_slbmfee(struct kvm_vcpu *vcpu, u64 slb_nr)
 352 {
 353         struct kvmppc_slb *slbe;
 354
 355         if (slb_nr > vcpu->arch.slb_nr)
 356                 return 0;
 357
 358         slbe = &vcpu->arch.slb[slb_nr];
 359
 360         return slbe->orige;
 361 }
 362
 363 static u64 kvmppc_mmu_book3s_64_slbmfev(struct kvm_vcpu *vcpu, u64 slb_nr)
 364 {
 365         struct kvmppc_slb *slbe;
 366
 367         if (slb_nr > vcpu->arch.slb_nr)
 368                 return 0;
 369
 370         slbe = &vcpu->arch.slb[slb_nr];
 371
 372         return slbe->origv;
 373 }
 374
 375 static void kvmppc_mmu_book3s_64_slbie(struct kvm_vcpu *vcpu, u64 ea)
 376 {
 377         struct kvmppc_slb *slbe;
 378
 379         dprintk("KVM MMU: slbie(0x%llx)\n", ea);
 380
 381         slbe = kvmppc_mmu_book3s_64_find_slbe(vcpu, ea);
 382
 383         if (!slbe)
 384                 return;
 385
 386         dprintk("KVM MMU: slbie(0x%llx, 0x%llx)\n", ea, slbe->esid);
 387
 388         slbe->valid = false;
 389
 390         kvmppc_mmu_map_segment(vcpu, ea);
 391 }
 392
 393 static void kvmppc_mmu_book3s_64_slbia(struct kvm_vcpu *vcpu)
 394 {
 395         int i;
 396
 397         dprintk("KVM MMU: slbia()\n");
 398
 399         for (i = 1; i < vcpu->arch.slb_nr; i++)
 400                 vcpu->arch.slb[i].valid = false;
 401
 402         if (vcpu->arch.shared->msr & MSR_IR) {
 403                 kvmppc_mmu_flush_segments(vcpu);
 404                 kvmppc_mmu_map_segment(vcpu, kvmppc_get_pc(vcpu));
 405         }
 406 }
 407
 408 static void kvmppc_mmu_book3s_64_mtsrin(struct kvm_vcpu *vcpu, u32 srnum,
 409                                         ulong value)
 410 {
 411         u64 rb = 0, rs = 0;
 412
 413         /*
 414          * According to Book3 2.01 mtsrin is implemented as:
 415          *
 416          * The SLB entry specified by (RB)32:35 is loaded from register
 417          * RS, as follows.
 418          *
 419          * SLBE Bit     Source                  SLB Field
 420          *
 421          * 0:31         0x0000_0000             ESID-0:31
 422          * 32:35        (RB)32:35               ESID-32:35
 423          * 36           0b1                     V
 424          * 37:61        0x00_0000|| 0b0         VSID-0:24
 425          * 62:88        (RS)37:63               VSID-25:51
 426          * 89:91        (RS)33:35               Ks Kp N
 427          * 92           (RS)36                  L ((RS)36 must be 0b0)
 428          * 93           0b0                     C
 429          */
 430
 431         dprintk("KVM MMU: mtsrin(0x%x, 0x%lx)\n", srnum, value);
 432
 433         /* ESID = srnum */
 434         rb |= (srnum & 0xf) << 28;
 435         /* Set the valid bit */
 436         rb |= 1 << 27;
 437         /* Index = ESID */
 438         rb |= srnum;
 439
 440         /* VSID = VSID */
 441         rs |= (value & 0xfffffff) << 12;
 442         /* flags = flags */
 443         rs |= ((value >> 28) & 0x7) << 9;
 444
 445         kvmppc_mmu_book3s_64_slbmte(vcpu, rs, rb);
 446 }
 447
 448 static void kvmppc_mmu_book3s_64_tlbie(struct kvm_vcpu *vcpu, ulong va,
 449                                        bool large)
 450 {
 451         u64 mask = 0xFFFFFFFFFULL;
 452
 453         dprintk("KVM MMU: tlbie(0x%lx)\n", va);
 454
 455         if (large)
 456                 mask = 0xFFFFFF000ULL;
 457         kvmppc_mmu_pte_vflush(vcpu, va >> 12, mask);
 458 }
 459
 460 static int kvmppc_mmu_book3s_64_esid_to_vsid(struct kvm_vcpu *vcpu, ulong esid,
 461                                              u64 *vsid)
 462 {
 463         ulong ea = esid << SID_SHIFT;
 464         struct kvmppc_slb *slb;
 465         u64 gvsid = esid;
 466         ulong mp_ea = vcpu->arch.magic_page_ea;
 467
 468         if (vcpu->arch.shared->msr & (MSR_DR|MSR_IR)) {
 469                 slb = kvmppc_mmu_book3s_64_find_slbe(vcpu, ea);
 470                 if (slb)
 471                         gvsid = slb->vsid;
 472         }
 473
 474         switch (vcpu->arch.shared->msr & (MSR_DR|MSR_IR)) {
 475         case 0:
 476                 *vsid = VSID_REAL | esid;
 477                 break;
 478         case MSR_IR:
 479                 *vsid = VSID_REAL_IR | gvsid;
 480                 break;
 481         case MSR_DR:
 482                 *vsid = VSID_REAL_DR | gvsid;
 483                 break;
 484         case MSR_DR|MSR_IR:
 485                 if (!slb)
 486                         goto no_slb;
 487
 488                 *vsid = gvsid;
 489                 break;
 490         default:
 491                 BUG();
 492                 break;
 493         }
 494
 495         if (vcpu->arch.shared->msr & MSR_PR)
 496                 *vsid |= VSID_PR;
 497
 498         return 0;
 499
 500 no_slb:
 501         /* Catch magic page case */
 502         if (unlikely(mp_ea) &&
 503             unlikely(esid == (mp_ea >> SID_SHIFT)) &&
 504             !(vcpu->arch.shared->msr & MSR_PR)) {
 505                 *vsid = VSID_REAL | esid;
 506                 return 0;
 507         }
 508
 509         return -EINVAL;
 510 }
 511
 512 static bool kvmppc_mmu_book3s_64_is_dcbz32(struct kvm_vcpu *vcpu)
 513 {
 514         return (to_book3s(vcpu)->hid[5] & 0x80);
 515 }
 516
 517 void kvmppc_mmu_book3s_64_init(struct kvm_vcpu *vcpu)
 518 {
 519         struct kvmppc_mmu *mmu = &vcpu->arch.mmu;
 520
 521         mmu->mfsrin = NULL;
 522         mmu->mtsrin = kvmppc_mmu_book3s_64_mtsrin;
 523         mmu->slbmte = kvmppc_mmu_book3s_64_slbmte;
 524         mmu->slbmfee = kvmppc_mmu_book3s_64_slbmfee;
 525         mmu->slbmfev = kvmppc_mmu_book3s_64_slbmfev;
 526         mmu->slbie = kvmppc_mmu_book3s_64_slbie;
 527         mmu->slbia = kvmppc_mmu_book3s_64_slbia;
 528         mmu->xlate = kvmppc_mmu_book3s_64_xlate;
 529         mmu->reset_msr = kvmppc_mmu_book3s_64_reset_msr;
 530         mmu->tlbie = kvmppc_mmu_book3s_64_tlbie;
 531         mmu->esid_to_vsid = kvmppc_mmu_book3s_64_esid_to_vsid;
 532         mmu->ea_to_vp = kvmppc_mmu_book3s_64_ea_to_vp;
 533         mmu->is_dcbz32 = kvmppc_mmu_book3s_64_is_dcbz32;
 534
 535         vcpu->arch.hflags |= BOOK3S_HFLAG_SLB;
 536 }