arch/s390/mm/pgtable.c

   1 /*
   2  *    Copyright IBM Corp. 2007, 2011
   3  *    Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com>
   4  */
   5
   6 #include <linux/sched.h>
   7 #include <linux/kernel.h>
   8 #include <linux/errno.h>
   9 #include <linux/gfp.h>
  10 #include <linux/mm.h>
  11 #include <linux/swap.h>
  12 #include <linux/smp.h>
  13 #include <linux/spinlock.h>
  14 #include <linux/rcupdate.h>
  15 #include <linux/slab.h>
  16 #include <linux/swapops.h>
  17 #include <linux/sysctl.h>
  18 #include <linux/ksm.h>
  19 #include <linux/mman.h>
  20
  21 #include <asm/pgtable.h>
  22 #include <asm/pgalloc.h>
  23 #include <asm/tlb.h>
  24 #include <asm/tlbflush.h>
  25 #include <asm/mmu_context.h>
  26
  27 static inline pte_t ptep_flush_direct(struct mm_struct *mm,
  28                                       unsigned long addr, pte_t *ptep)
  29 {
  30         int active, count;
  31         pte_t old;
  32
  33         old = *ptep;
  34         if (unlikely(pte_val(old) & _PAGE_INVALID))
  35                 return old;
  36         active = (mm == current->active_mm) ? 1 : 0;
  37         count = atomic_add_return(0x10000, &mm->context.attach_count);
  38         if (MACHINE_HAS_TLB_LC && (count & 0xffff) <= active &&
  39             cpumask_equal(mm_cpumask(mm), cpumask_of(smp_processor_id())))
  40                 __ptep_ipte_local(addr, ptep);
  41         else
  42                 __ptep_ipte(addr, ptep);
  43         atomic_sub(0x10000, &mm->context.attach_count);
  44         return old;
  45 }
  46
  47 static inline pte_t ptep_flush_lazy(struct mm_struct *mm,
  48                                     unsigned long addr, pte_t *ptep)
  49 {
  50         int active, count;
  51         pte_t old;
  52
  53         old = *ptep;
  54         if (unlikely(pte_val(old) & _PAGE_INVALID))
  55                 return old;
  56         active = (mm == current->active_mm) ? 1 : 0;
  57         count = atomic_add_return(0x10000, &mm->context.attach_count);
  58         if ((count & 0xffff) <= active) {
  59                 pte_val(*ptep) |= _PAGE_INVALID;
  60                 mm->context.flush_mm = 1;
  61         } else
  62                 __ptep_ipte(addr, ptep);
  63         atomic_sub(0x10000, &mm->context.attach_count);
  64         return old;
  65 }
  66
  67 static inline pgste_t pgste_get_lock(pte_t *ptep)
  68 {
  69         unsigned long new = 0;
  70 #ifdef CONFIG_PGSTE
  71         unsigned long old;
  72
  73         preempt_disable();
  74         asm(
  75                 "       lg      %0,%2\n"
  76                 "0:     lgr     %1,%0\n"
  77                 "       nihh    %0,0xff7f\n"    /* clear PCL bit in old */
  78                 "       oihh    %1,0x0080\n"    /* set PCL bit in new */
  79                 "       csg     %0,%1,%2\n"
  80                 "       jl      0b\n"
  81                 : "=&d" (old), "=&d" (new), "=Q" (ptep[PTRS_PER_PTE])
  82                 : "Q" (ptep[PTRS_PER_PTE]) : "cc", "memory");
  83 #endif
  84         return __pgste(new);
  85 }
  86
  87 static inline void pgste_set_unlock(pte_t *ptep, pgste_t pgste)
  88 {
  89 #ifdef CONFIG_PGSTE
  90         asm(
  91                 "       nihh    %1,0xff7f\n"    /* clear PCL bit */
  92                 "       stg     %1,%0\n"
  93                 : "=Q" (ptep[PTRS_PER_PTE])
  94                 : "d" (pgste_val(pgste)), "Q" (ptep[PTRS_PER_PTE])
  95                 : "cc", "memory");
  96         preempt_enable();
  97 #endif
  98 }
  99
 100 static inline pgste_t pgste_get(pte_t *ptep)
 101 {
 102         unsigned long pgste = 0;
 103 #ifdef CONFIG_PGSTE
 104         pgste = *(unsigned long *)(ptep + PTRS_PER_PTE);
 105 #endif
 106         return __pgste(pgste);
 107 }
 108
 109 static inline void pgste_set(pte_t *ptep, pgste_t pgste)
 110 {
 111 #ifdef CONFIG_PGSTE
 112         *(pgste_t *)(ptep + PTRS_PER_PTE) = pgste;
 113 #endif
 114 }
 115
 116 static inline pgste_t pgste_update_all(pte_t pte, pgste_t pgste,
 117                                        struct mm_struct *mm)
 118 {
 119 #ifdef CONFIG_PGSTE
 120         unsigned long address, bits, skey;
 121
 122         if (!mm_use_skey(mm) || pte_val(pte) & _PAGE_INVALID)
 123                 return pgste;
 124         address = pte_val(pte) & PAGE_MASK;
 125         skey = (unsigned long) page_get_storage_key(address);
 126         bits = skey & (_PAGE_CHANGED | _PAGE_REFERENCED);
 127         /* Transfer page changed & referenced bit to guest bits in pgste */
 128         pgste_val(pgste) |= bits << 48;         /* GR bit & GC bit */
 129         /* Copy page access key and fetch protection bit to pgste */
 130         pgste_val(pgste) &= ~(PGSTE_ACC_BITS | PGSTE_FP_BIT);
 131         pgste_val(pgste) |= (skey & (_PAGE_ACC_BITS | _PAGE_FP_BIT)) << 56;
 132 #endif
 133         return pgste;
 134
 135 }
 136
 137 static inline void pgste_set_key(pte_t *ptep, pgste_t pgste, pte_t entry,
 138                                  struct mm_struct *mm)
 139 {
 140 #ifdef CONFIG_PGSTE
 141         unsigned long address;
 142         unsigned long nkey;
 143
 144         if (!mm_use_skey(mm) || pte_val(entry) & _PAGE_INVALID)
 145                 return;
 146         VM_BUG_ON(!(pte_val(*ptep) & _PAGE_INVALID));
 147         address = pte_val(entry) & PAGE_MASK;
 148         /*
 149          * Set page access key and fetch protection bit from pgste.
 150          * The guest C/R information is still in the PGSTE, set real
 151          * key C/R to 0.
 152          */
 153         nkey = (pgste_val(pgste) & (PGSTE_ACC_BITS | PGSTE_FP_BIT)) >> 56;
 154         nkey |= (pgste_val(pgste) & (PGSTE_GR_BIT | PGSTE_GC_BIT)) >> 48;
 155         page_set_storage_key(address, nkey, 0);
 156 #endif
 157 }
 158
 159 static inline pgste_t pgste_set_pte(pte_t *ptep, pgste_t pgste, pte_t entry)
 160 {
 161 #ifdef CONFIG_PGSTE
 162         if ((pte_val(entry) & _PAGE_PRESENT) &&
 163             (pte_val(entry) & _PAGE_WRITE) &&
 164             !(pte_val(entry) & _PAGE_INVALID)) {
 165                 if (!MACHINE_HAS_ESOP) {
 166                         /*
 167                          * Without enhanced suppression-on-protection force
 168                          * the dirty bit on for all writable ptes.
 169                          */
 170                         pte_val(entry) |= _PAGE_DIRTY;
 171                         pte_val(entry) &= ~_PAGE_PROTECT;
 172                 }
 173                 if (!(pte_val(entry) & _PAGE_PROTECT))
 174                         /* This pte allows write access, set user-dirty */
 175                         pgste_val(pgste) |= PGSTE_UC_BIT;
 176         }
 177 #endif
 178         *ptep = entry;
 179         return pgste;
 180 }
 181
 182 static inline pgste_t pgste_pte_notify(struct mm_struct *mm,
 183                                        unsigned long addr,
 184                                        pte_t *ptep, pgste_t pgste)
 185 {
 186 #ifdef CONFIG_PGSTE
 187         unsigned long bits;
 188
 189         bits = pgste_val(pgste) & (PGSTE_IN_BIT | PGSTE_VSIE_BIT);
 190         if (bits) {
 191                 pgste_val(pgste) ^= bits;
 192                 ptep_notify(mm, addr, ptep, bits);
 193         }
 194 #endif
 195         return pgste;
 196 }
 197
 198 static inline pgste_t ptep_xchg_start(struct mm_struct *mm,
 199                                       unsigned long addr, pte_t *ptep)
 200 {
 201         pgste_t pgste = __pgste(0);
 202
 203         if (mm_has_pgste(mm)) {
 204                 pgste = pgste_get_lock(ptep);
 205                 pgste = pgste_pte_notify(mm, addr, ptep, pgste);
 206         }
 207         return pgste;
 208 }
 209
 210 static inline void ptep_xchg_commit(struct mm_struct *mm,
 211                                     unsigned long addr, pte_t *ptep,
 212                                     pgste_t pgste, pte_t old, pte_t new)
 213 {
 214         if (mm_has_pgste(mm)) {
 215                 if (pte_val(old) & _PAGE_INVALID)
 216                         pgste_set_key(ptep, pgste, new, mm);
 217                 if (pte_val(new) & _PAGE_INVALID) {
 218                         pgste = pgste_update_all(old, pgste, mm);
 219                         if ((pgste_val(pgste) & _PGSTE_GPS_USAGE_MASK) ==
 220                             _PGSTE_GPS_USAGE_UNUSED)
 221                                 pte_val(old) |= _PAGE_UNUSED;
 222                 }
 223                 pgste = pgste_set_pte(ptep, pgste, new);
 224                 pgste_set_unlock(ptep, pgste);
 225         } else {
 226                 *ptep = new;
 227         }
 228 }
 229
 230 pte_t ptep_xchg_direct(struct mm_struct *mm, unsigned long addr,
 231                        pte_t *ptep, pte_t new)
 232 {
 233         pgste_t pgste;
 234         pte_t old;
 235
 236         pgste = ptep_xchg_start(mm, addr, ptep);
 237         old = ptep_flush_direct(mm, addr, ptep);
 238         ptep_xchg_commit(mm, addr, ptep, pgste, old, new);
 239         return old;
 240 }
 241 EXPORT_SYMBOL(ptep_xchg_direct);
 242
 243 pte_t ptep_xchg_lazy(struct mm_struct *mm, unsigned long addr,
 244                      pte_t *ptep, pte_t new)
 245 {
 246         pgste_t pgste;
 247         pte_t old;
 248
 249         pgste = ptep_xchg_start(mm, addr, ptep);
 250         old = ptep_flush_lazy(mm, addr, ptep);
 251         ptep_xchg_commit(mm, addr, ptep, pgste, old, new);
 252         return old;
 253 }
 254 EXPORT_SYMBOL(ptep_xchg_lazy);
 255
 256 pte_t ptep_modify_prot_start(struct mm_struct *mm, unsigned long addr,
 257                              pte_t *ptep)
 258 {
 259         pgste_t pgste;
 260         pte_t old;
 261
 262         pgste = ptep_xchg_start(mm, addr, ptep);
 263         old = ptep_flush_lazy(mm, addr, ptep);
 264         if (mm_has_pgste(mm)) {
 265                 pgste = pgste_update_all(old, pgste, mm);
 266                 pgste_set(ptep, pgste);
 267         }
 268         return old;
 269 }
 270 EXPORT_SYMBOL(ptep_modify_prot_start);
 271
 272 void ptep_modify_prot_commit(struct mm_struct *mm, unsigned long addr,
 273                              pte_t *ptep, pte_t pte)
 274 {
 275         pgste_t pgste;
 276
 277         if (mm_has_pgste(mm)) {
 278                 pgste = pgste_get(ptep);
 279                 pgste_set_key(ptep, pgste, pte, mm);
 280                 pgste = pgste_set_pte(ptep, pgste, pte);
 281                 pgste_set_unlock(ptep, pgste);
 282         } else {
 283                 *ptep = pte;
 284         }
 285 }
 286 EXPORT_SYMBOL(ptep_modify_prot_commit);
 287
 288 static inline pmd_t pmdp_flush_direct(struct mm_struct *mm,
 289                                       unsigned long addr, pmd_t *pmdp)
 290 {
 291         int active, count;
 292         pmd_t old;
 293
 294         old = *pmdp;
 295         if (pmd_val(old) & _SEGMENT_ENTRY_INVALID)
 296                 return old;
 297         if (!MACHINE_HAS_IDTE) {
 298                 __pmdp_csp(pmdp);
 299                 return old;
 300         }
 301         active = (mm == current->active_mm) ? 1 : 0;
 302         count = atomic_add_return(0x10000, &mm->context.attach_count);
 303         if (MACHINE_HAS_TLB_LC && (count & 0xffff) <= active &&
 304             cpumask_equal(mm_cpumask(mm), cpumask_of(smp_processor_id())))
 305                 __pmdp_idte_local(addr, pmdp);
 306         else
 307                 __pmdp_idte(addr, pmdp);
 308         atomic_sub(0x10000, &mm->context.attach_count);
 309         return old;
 310 }
 311
 312 static inline pmd_t pmdp_flush_lazy(struct mm_struct *mm,
 313                                     unsigned long addr, pmd_t *pmdp)
 314 {
 315         int active, count;
 316         pmd_t old;
 317
 318         old = *pmdp;
 319         if (pmd_val(old) & _SEGMENT_ENTRY_INVALID)
 320                 return old;
 321         active = (mm == current->active_mm) ? 1 : 0;
 322         count = atomic_add_return(0x10000, &mm->context.attach_count);
 323         if ((count & 0xffff) <= active) {
 324                 pmd_val(*pmdp) |= _SEGMENT_ENTRY_INVALID;
 325                 mm->context.flush_mm = 1;
 326         } else if (MACHINE_HAS_IDTE)
 327                 __pmdp_idte(addr, pmdp);
 328         else
 329                 __pmdp_csp(pmdp);
 330         atomic_sub(0x10000, &mm->context.attach_count);
 331         return old;
 332 }
 333
 334 pmd_t pmdp_xchg_direct(struct mm_struct *mm, unsigned long addr,
 335                        pmd_t *pmdp, pmd_t new)
 336 {
 337         pmd_t old;
 338
 339         old = pmdp_flush_direct(mm, addr, pmdp);
 340         *pmdp = new;
 341         return old;
 342 }
 343 EXPORT_SYMBOL(pmdp_xchg_direct);
 344
 345 pmd_t pmdp_xchg_lazy(struct mm_struct *mm, unsigned long addr,
 346                      pmd_t *pmdp, pmd_t new)
 347 {
 348         pmd_t old;
 349
 350         old = pmdp_flush_lazy(mm, addr, pmdp);
 351         *pmdp = new;
 352         return old;
 353 }
 354 EXPORT_SYMBOL(pmdp_xchg_lazy);
 355
 356 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
 357 void pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp,
 358                                 pgtable_t pgtable)
 359 {
 360         struct list_head *lh = (struct list_head *) pgtable;
 361
 362         assert_spin_locked(pmd_lockptr(mm, pmdp));
 363
 364         /* FIFO */
 365         if (!pmd_huge_pte(mm, pmdp))
 366                 INIT_LIST_HEAD(lh);
 367         else
 368                 list_add(lh, (struct list_head *) pmd_huge_pte(mm, pmdp));
 369         pmd_huge_pte(mm, pmdp) = pgtable;
 370 }
 371
 372 pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp)
 373 {
 374         struct list_head *lh;
 375         pgtable_t pgtable;
 376         pte_t *ptep;
 377
 378         assert_spin_locked(pmd_lockptr(mm, pmdp));
 379
 380         /* FIFO */
 381         pgtable = pmd_huge_pte(mm, pmdp);
 382         lh = (struct list_head *) pgtable;
 383         if (list_empty(lh))
 384                 pmd_huge_pte(mm, pmdp) = NULL;
 385         else {
 386                 pmd_huge_pte(mm, pmdp) = (pgtable_t) lh->next;
 387                 list_del(lh);
 388         }
 389         ptep = (pte_t *) pgtable;
 390         pte_val(*ptep) = _PAGE_INVALID;
 391         ptep++;
 392         pte_val(*ptep) = _PAGE_INVALID;
 393         return pgtable;
 394 }
 395 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
 396
 397 #ifdef CONFIG_PGSTE
 398 void ptep_set_pte_at(struct mm_struct *mm, unsigned long addr,
 399                      pte_t *ptep, pte_t entry)
 400 {
 401         pgste_t pgste;
 402
 403         /* the mm_has_pgste() check is done in set_pte_at() */
 404         pgste = pgste_get_lock(ptep);
 405         pgste_val(pgste) &= ~_PGSTE_GPS_ZERO;
 406         pgste_set_key(ptep, pgste, entry, mm);
 407         pgste = pgste_set_pte(ptep, pgste, entry);
 408         pgste_set_unlock(ptep, pgste);
 409 }
 410
 411 void ptep_set_notify(struct mm_struct *mm, unsigned long addr, pte_t *ptep)
 412 {
 413         pgste_t pgste;
 414
 415         pgste = pgste_get_lock(ptep);
 416         pgste_val(pgste) |= PGSTE_IN_BIT;
 417         pgste_set_unlock(ptep, pgste);
 418 }
 419
 420 /**
 421  * ptep_force_prot - change access rights of a locked pte
 422  * @mm: pointer to the process mm_struct
 423  * @addr: virtual address in the guest address space
 424  * @ptep: pointer to the page table entry
 425  * @prot: indicates guest access rights: PROT_NONE, PROT_READ or PROT_WRITE
 426  * @bit: pgste bit to set (e.g. for notification)
 427  *
 428  * Returns 0 if the access rights were changed and -EAGAIN if the current
 429  * and requested access rights are incompatible.
 430  */
 431 int ptep_force_prot(struct mm_struct *mm, unsigned long addr,
 432                     pte_t *ptep, int prot, unsigned long bit)
 433 {
 434         pte_t entry;
 435         pgste_t pgste;
 436         int pte_i, pte_p;
 437
 438         pgste = pgste_get_lock(ptep);
 439         entry = *ptep;
 440         /* Check pte entry after all locks have been acquired */
 441         pte_i = pte_val(entry) & _PAGE_INVALID;
 442         pte_p = pte_val(entry) & _PAGE_PROTECT;
 443         if ((pte_i && (prot != PROT_NONE)) ||
 444             (pte_p && (prot & PROT_WRITE))) {
 445                 pgste_set_unlock(ptep, pgste);
 446                 return -EAGAIN;
 447         }
 448         /* Change access rights and set pgste bit */
 449         if (prot == PROT_NONE && !pte_i) {
 450                 ptep_flush_direct(mm, addr, ptep);
 451                 pgste = pgste_update_all(entry, pgste, mm);
 452                 pte_val(entry) |= _PAGE_INVALID;
 453         }
 454         if (prot == PROT_READ && !pte_p) {
 455                 ptep_flush_direct(mm, addr, ptep);
 456                 pte_val(entry) &= ~_PAGE_INVALID;
 457                 pte_val(entry) |= _PAGE_PROTECT;
 458         }
 459         pgste_val(pgste) |= bit;
 460         pgste = pgste_set_pte(ptep, pgste, entry);
 461         pgste_set_unlock(ptep, pgste);
 462         return 0;
 463 }
 464
 465 int ptep_shadow_pte(struct mm_struct *mm, unsigned long saddr,
 466                     pte_t *sptep, pte_t *tptep, int write)
 467 {
 468         pgste_t spgste, tpgste;
 469         pte_t spte, tpte;
 470         int rc = -EAGAIN;
 471
 472         spgste = pgste_get_lock(sptep);
 473         spte = *sptep;
 474         if (!(pte_val(spte) & _PAGE_INVALID) &&
 475             !(pte_val(spte) & _PAGE_PROTECT)) {
 476                 rc = 0;
 477                 if (!(pte_val(*tptep) & _PAGE_INVALID))
 478                         /* Update existing mapping */
 479                         ptep_flush_direct(mm, saddr, tptep);
 480                 else
 481                         rc = 1;
 482                 pgste_val(spgste) |= PGSTE_VSIE_BIT;
 483                 tpgste = pgste_get_lock(tptep);
 484                 pte_val(tpte) = (pte_val(spte) & PAGE_MASK) |
 485                         (write ? 0 : _PAGE_PROTECT);
 486                 /* don't touch the storage key - it belongs to parent pgste */
 487                 tpgste = pgste_set_pte(tptep, tpgste, tpte);
 488                 pgste_set_unlock(tptep, tpgste);
 489         }
 490         pgste_set_unlock(sptep, spgste);
 491         return rc;
 492 }
 493
 494 void ptep_unshadow_pte(struct mm_struct *mm, unsigned long saddr, pte_t *ptep)
 495 {
 496         pgste_t pgste;
 497
 498         pgste = pgste_get_lock(ptep);
 499         /* notifier is called by the caller */
 500         ptep_flush_direct(mm, saddr, ptep);
 501         /* don't touch the storage key - it belongs to parent pgste */
 502         pgste = pgste_set_pte(ptep, pgste, __pte(_PAGE_INVALID));
 503         pgste_set_unlock(ptep, pgste);
 504 }
 505
 506 static void ptep_zap_swap_entry(struct mm_struct *mm, swp_entry_t entry)
 507 {
 508         if (!non_swap_entry(entry))
 509                 dec_mm_counter(mm, MM_SWAPENTS);
 510         else if (is_migration_entry(entry)) {
 511                 struct page *page = migration_entry_to_page(entry);
 512
 513                 dec_mm_counter(mm, mm_counter(page));
 514         }
 515         free_swap_and_cache(entry);
 516 }
 517
 518 void ptep_zap_unused(struct mm_struct *mm, unsigned long addr,
 519                      pte_t *ptep, int reset)
 520 {
 521         unsigned long pgstev;
 522         pgste_t pgste;
 523         pte_t pte;
 524
 525         /* Zap unused and logically-zero pages */
 526         pgste = pgste_get_lock(ptep);
 527         pgstev = pgste_val(pgste);
 528         pte = *ptep;
 529         if (pte_swap(pte) &&
 530             ((pgstev & _PGSTE_GPS_USAGE_MASK) == _PGSTE_GPS_USAGE_UNUSED ||
 531              (pgstev & _PGSTE_GPS_ZERO))) {
 532                 ptep_zap_swap_entry(mm, pte_to_swp_entry(pte));
 533                 pte_clear(mm, addr, ptep);
 534         }
 535         if (reset)
 536                 pgste_val(pgste) &= ~_PGSTE_GPS_USAGE_MASK;
 537         pgste_set_unlock(ptep, pgste);
 538 }
 539
 540 void ptep_zap_key(struct mm_struct *mm, unsigned long addr, pte_t *ptep)
 541 {
 542         unsigned long ptev;
 543         pgste_t pgste;
 544
 545         /* Clear storage key */
 546         pgste = pgste_get_lock(ptep);
 547         pgste_val(pgste) &= ~(PGSTE_ACC_BITS | PGSTE_FP_BIT |
 548                               PGSTE_GR_BIT | PGSTE_GC_BIT);
 549         ptev = pte_val(*ptep);
 550         if (!(ptev & _PAGE_INVALID) && (ptev & _PAGE_WRITE))
 551                 page_set_storage_key(ptev & PAGE_MASK, PAGE_DEFAULT_KEY, 1);
 552         pgste_set_unlock(ptep, pgste);
 553 }
 554
 555 /*
 556  * Test and reset if a guest page is dirty
 557  */
 558 bool test_and_clear_guest_dirty(struct mm_struct *mm, unsigned long addr)
 559 {
 560         spinlock_t *ptl;
 561         pgste_t pgste;
 562         pte_t *ptep;
 563         pte_t pte;
 564         bool dirty;
 565
 566         ptep = get_locked_pte(mm, addr, &ptl);
 567         if (unlikely(!ptep))
 568                 return false;
 569
 570         pgste = pgste_get_lock(ptep);
 571         dirty = !!(pgste_val(pgste) & PGSTE_UC_BIT);
 572         pgste_val(pgste) &= ~PGSTE_UC_BIT;
 573         pte = *ptep;
 574         if (dirty && (pte_val(pte) & _PAGE_PRESENT)) {
 575                 pgste = pgste_pte_notify(mm, addr, ptep, pgste);
 576                 __ptep_ipte(addr, ptep);
 577                 if (MACHINE_HAS_ESOP || !(pte_val(pte) & _PAGE_WRITE))
 578                         pte_val(pte) |= _PAGE_PROTECT;
 579                 else
 580                         pte_val(pte) |= _PAGE_INVALID;
 581                 *ptep = pte;
 582         }
 583         pgste_set_unlock(ptep, pgste);
 584
 585         spin_unlock(ptl);
 586         return dirty;
 587 }
 588 EXPORT_SYMBOL_GPL(test_and_clear_guest_dirty);
 589
 590 int set_guest_storage_key(struct mm_struct *mm, unsigned long addr,
 591                           unsigned char key, bool nq)
 592 {
 593         unsigned long keyul;
 594         spinlock_t *ptl;
 595         pgste_t old, new;
 596         pte_t *ptep;
 597
 598         ptep = get_locked_pte(mm, addr, &ptl);
 599         if (unlikely(!ptep))
 600                 return -EFAULT;
 601
 602         new = old = pgste_get_lock(ptep);
 603         pgste_val(new) &= ~(PGSTE_GR_BIT | PGSTE_GC_BIT |
 604                             PGSTE_ACC_BITS | PGSTE_FP_BIT);
 605         keyul = (unsigned long) key;
 606         pgste_val(new) |= (keyul & (_PAGE_CHANGED | _PAGE_REFERENCED)) << 48;
 607         pgste_val(new) |= (keyul & (_PAGE_ACC_BITS | _PAGE_FP_BIT)) << 56;
 608         if (!(pte_val(*ptep) & _PAGE_INVALID)) {
 609                 unsigned long address, bits, skey;
 610
 611                 address = pte_val(*ptep) & PAGE_MASK;
 612                 skey = (unsigned long) page_get_storage_key(address);
 613                 bits = skey & (_PAGE_CHANGED | _PAGE_REFERENCED);
 614                 skey = key & (_PAGE_ACC_BITS | _PAGE_FP_BIT);
 615                 /* Set storage key ACC and FP */
 616                 page_set_storage_key(address, skey, !nq);
 617                 /* Merge host changed & referenced into pgste  */
 618                 pgste_val(new) |= bits << 52;
 619         }
 620         /* changing the guest storage key is considered a change of the page */
 621         if ((pgste_val(new) ^ pgste_val(old)) &
 622             (PGSTE_ACC_BITS | PGSTE_FP_BIT | PGSTE_GR_BIT | PGSTE_GC_BIT))
 623                 pgste_val(new) |= PGSTE_UC_BIT;
 624
 625         pgste_set_unlock(ptep, new);
 626         pte_unmap_unlock(ptep, ptl);
 627         return 0;
 628 }
 629 EXPORT_SYMBOL(set_guest_storage_key);
 630
 631 /**
 632  * Conditionally set a guest storage key (handling csske).
 633  * oldkey will be updated when either mr or mc is set and a pointer is given.
 634  *
 635  * Returns 0 if a guests storage key update wasn't necessary, 1 if the guest
 636  * storage key was updated and -EFAULT on access errors.
 637  */
 638 int cond_set_guest_storage_key(struct mm_struct *mm, unsigned long addr,
 639                                unsigned char key, unsigned char *oldkey,
 640                                bool nq, bool mr, bool mc)
 641 {
 642         unsigned char tmp, mask = _PAGE_ACC_BITS | _PAGE_FP_BIT;
 643         int rc;
 644
 645         /* we can drop the pgste lock between getting and setting the key */
 646         if (mr | mc) {
 647                 rc = get_guest_storage_key(current->mm, addr, &tmp);
 648                 if (rc)
 649                         return rc;
 650                 if (oldkey)
 651                         *oldkey = tmp;
 652                 if (!mr)
 653                         mask |= _PAGE_REFERENCED;
 654                 if (!mc)
 655                         mask |= _PAGE_CHANGED;
 656                 if (!((tmp ^ key) & mask))
 657                         return 0;
 658         }
 659         rc = set_guest_storage_key(current->mm, addr, key, nq);
 660         return rc < 0 ? rc : 1;
 661 }
 662 EXPORT_SYMBOL(cond_set_guest_storage_key);
 663
 664 /**
 665  * Reset a guest reference bit (rrbe), returning the reference and changed bit.
 666  *
 667  * Returns < 0 in case of error, otherwise the cc to be reported to the guest.
 668  */
 669 int reset_guest_reference_bit(struct mm_struct *mm, unsigned long addr)
 670 {
 671         spinlock_t *ptl;
 672         pgste_t old, new;
 673         pte_t *ptep;
 674         int cc = 0;
 675
 676         ptep = get_locked_pte(mm, addr, &ptl);
 677         if (unlikely(!ptep))
 678                 return -EFAULT;
 679
 680         new = old = pgste_get_lock(ptep);
 681         /* Reset guest reference bit only */
 682         pgste_val(new) &= ~PGSTE_GR_BIT;
 683
 684         if (!(pte_val(*ptep) & _PAGE_INVALID)) {
 685                 cc = page_reset_referenced(pte_val(*ptep) & PAGE_MASK);
 686                 /* Merge real referenced bit into host-set */
 687                 pgste_val(new) |= ((unsigned long) cc << 53) & PGSTE_HR_BIT;
 688         }
 689         /* Reflect guest's logical view, not physical */
 690         cc |= (pgste_val(old) & (PGSTE_GR_BIT | PGSTE_GC_BIT)) >> 49;
 691         /* Changing the guest storage key is considered a change of the page */
 692         if ((pgste_val(new) ^ pgste_val(old)) & PGSTE_GR_BIT)
 693                 pgste_val(new) |= PGSTE_UC_BIT;
 694
 695         pgste_set_unlock(ptep, new);
 696         pte_unmap_unlock(ptep, ptl);
 697         return 0;
 698 }
 699 EXPORT_SYMBOL(reset_guest_reference_bit);
 700
 701 int get_guest_storage_key(struct mm_struct *mm, unsigned long addr,
 702                           unsigned char *key)
 703 {
 704         spinlock_t *ptl;
 705         pgste_t pgste;
 706         pte_t *ptep;
 707
 708         ptep = get_locked_pte(mm, addr, &ptl);
 709         if (unlikely(!ptep))
 710                 return -EFAULT;
 711
 712         pgste = pgste_get_lock(ptep);
 713         *key = (pgste_val(pgste) & (PGSTE_ACC_BITS | PGSTE_FP_BIT)) >> 56;
 714         if (!(pte_val(*ptep) & _PAGE_INVALID))
 715                 *key = page_get_storage_key(pte_val(*ptep) & PAGE_MASK);
 716         /* Reflect guest's logical view, not physical */
 717         *key |= (pgste_val(pgste) & (PGSTE_GR_BIT | PGSTE_GC_BIT)) >> 48;
 718         pgste_set_unlock(ptep, pgste);
 719         pte_unmap_unlock(ptep, ptl);
 720         return 0;
 721 }
 722 EXPORT_SYMBOL(get_guest_storage_key);
 723 #endif