arch/powerpc/mm/pgtable.c

   1 /*
   2  * This file contains common routines for dealing with free of page tables
   3  * Along with common page table handling code
   4  *
   5  *  Derived from arch/powerpc/mm/tlb_64.c:
   6  *    Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org)
   7  *
   8  *  Modifications by Paul Mackerras (PowerMac) (paulus@cs.anu.edu.au)
   9  *  and Cort Dougan (PReP) (cort@cs.nmt.edu)
  10  *    Copyright (C) 1996 Paul Mackerras
  11  *
  12  *  Derived from "arch/i386/mm/init.c"
  13  *    Copyright (C) 1991, 1992, 1993, 1994  Linus Torvalds
  14  *
  15  *  Dave Engebretsen <engebret@us.ibm.com>
  16  *      Rework for PPC64 port.
  17  *
  18  *  This program is free software; you can redistribute it and/or
  19  *  modify it under the terms of the GNU General Public License
  20  *  as published by the Free Software Foundation; either version
  21  *  2 of the License, or (at your option) any later version.
  22  */
  23
  24 #include <linux/kernel.h>
  25 #include <linux/gfp.h>
  26 #include <linux/mm.h>
  27 #include <linux/init.h>
  28 #include <linux/percpu.h>
  29 #include <linux/hardirq.h>
  30 #include <linux/hugetlb.h>
  31 #include <asm/pgalloc.h>
  32 #include <asm/tlbflush.h>
  33 #include <asm/tlb.h>
  34
  35 static inline int is_exec_fault(void)
  36 {
  37         return current->thread.regs && TRAP(current->thread.regs) == 0x400;
  38 }
  39
  40 /* We only try to do i/d cache coherency on stuff that looks like
  41  * reasonably "normal" PTEs. We currently require a PTE to be present
  42  * and we avoid _PAGE_SPECIAL and _PAGE_NO_CACHE. We also only do that
  43  * on userspace PTEs
  44  */
  45 static inline int pte_looks_normal(pte_t pte)
  46 {
  47         return (pte_val(pte) &
  48             (_PAGE_PRESENT | _PAGE_SPECIAL | _PAGE_NO_CACHE | _PAGE_USER)) ==
  49             (_PAGE_PRESENT | _PAGE_USER);
  50 }
  51
  52 struct page * maybe_pte_to_page(pte_t pte)
  53 {
  54         unsigned long pfn = pte_pfn(pte);
  55         struct page *page;
  56
  57         if (unlikely(!pfn_valid(pfn)))
  58                 return NULL;
  59         page = pfn_to_page(pfn);
  60         if (PageReserved(page))
  61                 return NULL;
  62         return page;
  63 }
  64
  65 #if defined(CONFIG_PPC_STD_MMU) || _PAGE_EXEC == 0
  66
  67 /* Server-style MMU handles coherency when hashing if HW exec permission
  68  * is supposed per page (currently 64-bit only). If not, then, we always
  69  * flush the cache for valid PTEs in set_pte. Embedded CPU without HW exec
  70  * support falls into the same category.
  71  */
  72
  73 static pte_t set_pte_filter(pte_t pte)
  74 {
  75         pte = __pte(pte_val(pte) & ~_PAGE_HPTEFLAGS);
  76         if (pte_looks_normal(pte) && !(cpu_has_feature(CPU_FTR_COHERENT_ICACHE) ||
  77                                        cpu_has_feature(CPU_FTR_NOEXECUTE))) {
  78                 struct page *pg = maybe_pte_to_page(pte);
  79                 if (!pg)
  80                         return pte;
  81                 if (!test_bit(PG_arch_1, &pg->flags)) {
  82                         flush_dcache_icache_page(pg);
  83                         set_bit(PG_arch_1, &pg->flags);
  84                 }
  85         }
  86         return pte;
  87 }
  88
  89 static pte_t set_access_flags_filter(pte_t pte, struct vm_area_struct *vma,
  90                                      int dirty)
  91 {
  92         return pte;
  93 }
  94
  95 #else /* defined(CONFIG_PPC_STD_MMU) || _PAGE_EXEC == 0 */
  96
  97 /* Embedded type MMU with HW exec support. This is a bit more complicated
  98  * as we don't have two bits to spare for _PAGE_EXEC and _PAGE_HWEXEC so
  99  * instead we "filter out" the exec permission for non clean pages.
 100  */
 101 static pte_t set_pte_filter(pte_t pte)
 102 {
 103         struct page *pg;
 104
 105         /* No exec permission in the first place, move on */
 106         if (!(pte_val(pte) & _PAGE_EXEC) || !pte_looks_normal(pte))
 107                 return pte;
 108
 109         /* If you set _PAGE_EXEC on weird pages you're on your own */
 110         pg = maybe_pte_to_page(pte);
 111         if (unlikely(!pg))
 112                 return pte;
 113
 114         /* If the page clean, we move on */
 115         if (test_bit(PG_arch_1, &pg->flags))
 116                 return pte;
 117
 118         /* If it's an exec fault, we flush the cache and make it clean */
 119         if (is_exec_fault()) {
 120                 flush_dcache_icache_page(pg);
 121                 set_bit(PG_arch_1, &pg->flags);
 122                 return pte;
 123         }
 124
 125         /* Else, we filter out _PAGE_EXEC */
 126         return __pte(pte_val(pte) & ~_PAGE_EXEC);
 127 }
 128
 129 static pte_t set_access_flags_filter(pte_t pte, struct vm_area_struct *vma,
 130                                      int dirty)
 131 {
 132         struct page *pg;
 133
 134         /* So here, we only care about exec faults, as we use them
 135          * to recover lost _PAGE_EXEC and perform I$/D$ coherency
 136          * if necessary. Also if _PAGE_EXEC is already set, same deal,
 137          * we just bail out
 138          */
 139         if (dirty || (pte_val(pte) & _PAGE_EXEC) || !is_exec_fault())
 140                 return pte;
 141
 142 #ifdef CONFIG_DEBUG_VM
 143         /* So this is an exec fault, _PAGE_EXEC is not set. If it was
 144          * an error we would have bailed out earlier in do_page_fault()
 145          * but let's make sure of it
 146          */
 147         if (WARN_ON(!(vma->vm_flags & VM_EXEC)))
 148                 return pte;
 149 #endif /* CONFIG_DEBUG_VM */
 150
 151         /* If you set _PAGE_EXEC on weird pages you're on your own */
 152         pg = maybe_pte_to_page(pte);
 153         if (unlikely(!pg))
 154                 goto bail;
 155
 156         /* If the page is already clean, we move on */
 157         if (test_bit(PG_arch_1, &pg->flags))
 158                 goto bail;
 159
 160         /* Clean the page and set PG_arch_1 */
 161         flush_dcache_icache_page(pg);
 162         set_bit(PG_arch_1, &pg->flags);
 163
 164  bail:
 165         return __pte(pte_val(pte) | _PAGE_EXEC);
 166 }
 167
 168 #endif /* !(defined(CONFIG_PPC_STD_MMU) || _PAGE_EXEC == 0) */
 169
 170 /*
 171  * set_pte stores a linux PTE into the linux page table.
 172  */
 173 void set_pte_at(struct mm_struct *mm, unsigned long addr, pte_t *ptep,
 174                 pte_t pte)
 175 {
 176 #ifdef CONFIG_DEBUG_VM
 177         WARN_ON(pte_present(*ptep));
 178 #endif
 179         /* Note: mm->context.id might not yet have been assigned as
 180          * this context might not have been activated yet when this
 181          * is called.
 182          */
 183         pte = set_pte_filter(pte);
 184
 185         /* Perform the setting of the PTE */
 186         __set_pte_at(mm, addr, ptep, pte, 0);
 187 }
 188
 189 /*
 190  * This is called when relaxing access to a PTE. It's also called in the page
 191  * fault path when we don't hit any of the major fault cases, ie, a minor
 192  * update of _PAGE_ACCESSED, _PAGE_DIRTY, etc... The generic code will have
 193  * handled those two for us, we additionally deal with missing execute
 194  * permission here on some processors
 195  */
 196 int ptep_set_access_flags(struct vm_area_struct *vma, unsigned long address,
 197                           pte_t *ptep, pte_t entry, int dirty)
 198 {
 199         int changed;
 200         entry = set_access_flags_filter(entry, vma, dirty);
 201         changed = !pte_same(*(ptep), entry);
 202         if (changed) {
 203                 if (!is_vm_hugetlb_page(vma))
 204                         assert_pte_locked(vma->vm_mm, address);
 205                 __ptep_set_access_flags(ptep, entry);
 206                 flush_tlb_page_nohash(vma, address);
 207         }
 208         return changed;
 209 }
 210
 211 #ifdef CONFIG_DEBUG_VM
 212 void assert_pte_locked(struct mm_struct *mm, unsigned long addr)
 213 {
 214         pgd_t *pgd;
 215         pud_t *pud;
 216         pmd_t *pmd;
 217
 218         if (mm == &init_mm)
 219                 return;
 220         pgd = mm->pgd + pgd_index(addr);
 221         BUG_ON(pgd_none(*pgd));
 222         pud = pud_offset(pgd, addr);
 223         BUG_ON(pud_none(*pud));
 224         pmd = pmd_offset(pud, addr);
 225         /*
 226          * khugepaged to collapse normal pages to hugepage, first set
 227          * pmd to none to force page fault/gup to take mmap_sem. After
 228          * pmd is set to none, we do a pte_clear which does this assertion
 229          * so if we find pmd none, return.
 230          */
 231         if (pmd_none(*pmd))
 232                 return;
 233         BUG_ON(!pmd_present(*pmd));
 234         assert_spin_locked(pte_lockptr(mm, pmd));
 235 }
 236 #endif /* CONFIG_DEBUG_VM */
 237