arch/powerpc/mm/hugetlbpage-book3e.c

   1 /*
   2  * PPC Huge TLB Page Support for Book3E MMU
   3  *
   4  * Copyright (C) 2009 David Gibson, IBM Corporation.
   5  * Copyright (C) 2011 Becky Bruce, Freescale Semiconductor
   6  *
   7  */
   8 #include <linux/mm.h>
   9 #include <linux/hugetlb.h>
  10
  11 #ifdef CONFIG_PPC_FSL_BOOK3E
  12 #ifdef CONFIG_PPC64
  13 static inline int tlb1_next(void)
  14 {
  15         struct paca_struct *paca = get_paca();
  16         struct tlb_core_data *tcd;
  17         int this, next;
  18
  19         tcd = paca->tcd_ptr;
  20         this = tcd->esel_next;
  21
  22         next = this + 1;
  23         if (next >= tcd->esel_max)
  24                 next = tcd->esel_first;
  25
  26         tcd->esel_next = next;
  27         return this;
  28 }
  29 #else
  30 static inline int tlb1_next(void)
  31 {
  32         int index, ncams;
  33
  34         ncams = mfspr(SPRN_TLB1CFG) & TLBnCFG_N_ENTRY;
  35
  36         index = this_cpu_read(next_tlbcam_idx);
  37
  38         /* Just round-robin the entries and wrap when we hit the end */
  39         if (unlikely(index == ncams - 1))
  40                 __this_cpu_write(next_tlbcam_idx, tlbcam_index);
  41         else
  42                 __this_cpu_inc(next_tlbcam_idx);
  43
  44         return index;
  45 }
  46 #endif /* !PPC64 */
  47 #endif /* FSL */
  48
  49 static inline int mmu_get_tsize(int psize)
  50 {
  51         return mmu_psize_defs[psize].enc;
  52 }
  53
  54 #if defined(CONFIG_PPC_FSL_BOOK3E) && defined(CONFIG_PPC64)
  55 #include <asm/paca.h>
  56
  57 static inline void book3e_tlb_lock(void)
  58 {
  59         struct paca_struct *paca = get_paca();
  60         unsigned long tmp;
  61         int token = smp_processor_id() + 1;
  62
  63         asm volatile("1: lbarx %0, 0, %1;"
  64                      "cmpwi %0, 0;"
  65                      "bne 2f;"
  66                      "stbcx. %2, 0, %1;"
  67                      "bne 1b;"
  68                      "b 3f;"
  69                      "2: lbzx %0, 0, %1;"
  70                      "cmpwi %0, 0;"
  71                      "bne 2b;"
  72                      "b 1b;"
  73                      "3:"
  74                      : "=&r" (tmp)
  75                      : "r" (&paca->tcd_ptr->lock), "r" (token)
  76                      : "memory");
  77 }
  78
  79 static inline void book3e_tlb_unlock(void)
  80 {
  81         struct paca_struct *paca = get_paca();
  82
  83         isync();
  84         paca->tcd_ptr->lock = 0;
  85 }
  86 #else
  87 static inline void book3e_tlb_lock(void)
  88 {
  89 }
  90
  91 static inline void book3e_tlb_unlock(void)
  92 {
  93 }
  94 #endif
  95
  96 static inline int book3e_tlb_exists(unsigned long ea, unsigned long pid)
  97 {
  98         int found = 0;
  99
 100         mtspr(SPRN_MAS6, pid << 16);
 101         if (mmu_has_feature(MMU_FTR_USE_TLBRSRV)) {
 102                 asm volatile(
 103                         "li     %0,0\n"
 104                         "tlbsx. 0,%1\n"
 105                         "bne    1f\n"
 106                         "li     %0,1\n"
 107                         "1:\n"
 108                         : "=&r"(found) : "r"(ea));
 109         } else {
 110                 asm volatile(
 111                         "tlbsx  0,%1\n"
 112                         "mfspr  %0,0x271\n"
 113                         "srwi   %0,%0,31\n"
 114                         : "=&r"(found) : "r"(ea));
 115         }
 116
 117         return found;
 118 }
 119
 120 void book3e_hugetlb_preload(struct vm_area_struct *vma, unsigned long ea,
 121                             pte_t pte)
 122 {
 123         unsigned long mas1, mas2;
 124         u64 mas7_3;
 125         unsigned long psize, tsize, shift;
 126         unsigned long flags;
 127         struct mm_struct *mm;
 128
 129 #ifdef CONFIG_PPC_FSL_BOOK3E
 130         int index;
 131 #endif
 132
 133         if (unlikely(is_kernel_addr(ea)))
 134                 return;
 135
 136         mm = vma->vm_mm;
 137
 138 #ifdef CONFIG_PPC_MM_SLICES
 139         psize = get_slice_psize(mm, ea);
 140         tsize = mmu_get_tsize(psize);
 141         shift = mmu_psize_defs[psize].shift;
 142 #else
 143         psize = vma_mmu_pagesize(vma);
 144         shift = __ilog2(psize);
 145         tsize = shift - 10;
 146 #endif
 147
 148         /*
 149          * We can't be interrupted while we're setting up the MAS
 150          * regusters or after we've confirmed that no tlb exists.
 151          */
 152         local_irq_save(flags);
 153
 154         book3e_tlb_lock();
 155
 156         if (unlikely(book3e_tlb_exists(ea, mm->context.id))) {
 157                 book3e_tlb_unlock();
 158                 local_irq_restore(flags);
 159                 return;
 160         }
 161
 162 #ifdef CONFIG_PPC_FSL_BOOK3E
 163         /* We have to use the CAM(TLB1) on FSL parts for hugepages */
 164         index = tlb1_next();
 165         mtspr(SPRN_MAS0, MAS0_ESEL(index) | MAS0_TLBSEL(1));
 166 #endif
 167
 168         mas1 = MAS1_VALID | MAS1_TID(mm->context.id) | MAS1_TSIZE(tsize);
 169         mas2 = ea & ~((1UL << shift) - 1);
 170         mas2 |= (pte_val(pte) >> PTE_WIMGE_SHIFT) & MAS2_WIMGE_MASK;
 171         mas7_3 = (u64)pte_pfn(pte) << PAGE_SHIFT;
 172         mas7_3 |= (pte_val(pte) >> PTE_BAP_SHIFT) & MAS3_BAP_MASK;
 173         if (!pte_dirty(pte))
 174                 mas7_3 &= ~(MAS3_SW|MAS3_UW);
 175
 176         mtspr(SPRN_MAS1, mas1);
 177         mtspr(SPRN_MAS2, mas2);
 178
 179         if (mmu_has_feature(MMU_FTR_USE_PAIRED_MAS)) {
 180                 mtspr(SPRN_MAS7_MAS3, mas7_3);
 181         } else {
 182                 if (mmu_has_feature(MMU_FTR_BIG_PHYS))
 183                         mtspr(SPRN_MAS7, upper_32_bits(mas7_3));
 184                 mtspr(SPRN_MAS3, lower_32_bits(mas7_3));
 185         }
 186
 187         asm volatile ("tlbwe");
 188
 189         book3e_tlb_unlock();
 190         local_irq_restore(flags);
 191 }
 192
 193 void flush_hugetlb_page(struct vm_area_struct *vma, unsigned long vmaddr)
 194 {
 195         struct hstate *hstate = hstate_file(vma->vm_file);
 196         unsigned long tsize = huge_page_shift(hstate) - 10;
 197
 198         __flush_tlb_page(vma->vm_mm, vmaddr, tsize, 0);
 199 }