2 * Page table allocation functions
4 * Copyright IBM Corp. 2016
5 * Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com>
9 #include <linux/sysctl.h>
10 #include <asm/mmu_context.h>
11 #include <asm/pgalloc.h>
14 #include <asm/tlbflush.h>
18 static int page_table_allocate_pgste_min
= 0;
19 static int page_table_allocate_pgste_max
= 1;
20 int page_table_allocate_pgste
= 0;
21 EXPORT_SYMBOL(page_table_allocate_pgste
);
23 static struct ctl_table page_table_sysctl
[] = {
25 .procname
= "allocate_pgste",
26 .data
= &page_table_allocate_pgste
,
27 .maxlen
= sizeof(int),
28 .mode
= S_IRUGO
| S_IWUSR
,
29 .proc_handler
= proc_dointvec
,
30 .extra1
= &page_table_allocate_pgste_min
,
31 .extra2
= &page_table_allocate_pgste_max
,
36 static struct ctl_table page_table_sysctl_dir
[] = {
41 .child
= page_table_sysctl
,
46 static int __init
page_table_register_sysctl(void)
48 return register_sysctl_table(page_table_sysctl_dir
) ? 0 : -ENOMEM
;
50 __initcall(page_table_register_sysctl
);
52 #endif /* CONFIG_PGSTE */
54 unsigned long *crst_table_alloc(struct mm_struct
*mm
)
56 struct page
*page
= alloc_pages(GFP_KERNEL
, 2);
60 arch_set_page_dat(page
, 2);
61 return (unsigned long *) page_to_phys(page
);
64 void crst_table_free(struct mm_struct
*mm
, unsigned long *table
)
66 free_pages((unsigned long) table
, 2);
69 static void __crst_table_upgrade(void *arg
)
71 struct mm_struct
*mm
= arg
;
73 if (current
->active_mm
== mm
) {
80 int crst_table_upgrade(struct mm_struct
*mm
, unsigned long end
)
82 unsigned long *table
, *pgd
;
85 /* upgrade should only happen from 3 to 4, 3 to 5, or 4 to 5 levels */
86 VM_BUG_ON(mm
->context
.asce_limit
< _REGION2_SIZE
);
87 if (end
>= TASK_SIZE_MAX
)
91 while (mm
->context
.asce_limit
< end
) {
92 table
= crst_table_alloc(mm
);
97 spin_lock_bh(&mm
->page_table_lock
);
98 pgd
= (unsigned long *) mm
->pgd
;
99 if (mm
->context
.asce_limit
== _REGION2_SIZE
) {
100 crst_table_init(table
, _REGION2_ENTRY_EMPTY
);
101 p4d_populate(mm
, (p4d_t
*) table
, (pud_t
*) pgd
);
102 mm
->pgd
= (pgd_t
*) table
;
103 mm
->context
.asce_limit
= _REGION1_SIZE
;
104 mm
->context
.asce
= __pa(mm
->pgd
) | _ASCE_TABLE_LENGTH
|
105 _ASCE_USER_BITS
| _ASCE_TYPE_REGION2
;
107 crst_table_init(table
, _REGION1_ENTRY_EMPTY
);
108 pgd_populate(mm
, (pgd_t
*) table
, (p4d_t
*) pgd
);
109 mm
->pgd
= (pgd_t
*) table
;
110 mm
->context
.asce_limit
= -PAGE_SIZE
;
111 mm
->context
.asce
= __pa(mm
->pgd
) | _ASCE_TABLE_LENGTH
|
112 _ASCE_USER_BITS
| _ASCE_TYPE_REGION1
;
115 spin_unlock_bh(&mm
->page_table_lock
);
118 on_each_cpu(__crst_table_upgrade
, mm
, 0);
122 void crst_table_downgrade(struct mm_struct
*mm
)
126 /* downgrade should only happen from 3 to 2 levels (compat only) */
127 VM_BUG_ON(mm
->context
.asce_limit
!= _REGION2_SIZE
);
129 if (current
->active_mm
== mm
) {
135 mm
->pgd
= (pgd_t
*) (pgd_val(*pgd
) & _REGION_ENTRY_ORIGIN
);
136 mm
->context
.asce_limit
= _REGION3_SIZE
;
137 mm
->context
.asce
= __pa(mm
->pgd
) | _ASCE_TABLE_LENGTH
|
138 _ASCE_USER_BITS
| _ASCE_TYPE_SEGMENT
;
139 crst_table_free(mm
, (unsigned long *) pgd
);
141 if (current
->active_mm
== mm
)
145 static inline unsigned int atomic_xor_bits(atomic_t
*v
, unsigned int bits
)
147 unsigned int old
, new;
150 old
= atomic_read(v
);
152 } while (atomic_cmpxchg(v
, old
, new) != old
);
158 struct page
*page_table_alloc_pgste(struct mm_struct
*mm
)
161 unsigned long *table
;
163 page
= alloc_page(GFP_KERNEL
);
165 table
= (unsigned long *) page_to_phys(page
);
166 clear_table(table
, _PAGE_INVALID
, PAGE_SIZE
/2);
167 clear_table(table
+ PTRS_PER_PTE
, 0, PAGE_SIZE
/2);
172 void page_table_free_pgste(struct page
*page
)
177 #endif /* CONFIG_PGSTE */
180 * page table entry allocation/free routines.
182 unsigned long *page_table_alloc(struct mm_struct
*mm
)
184 unsigned long *table
;
186 unsigned int mask
, bit
;
188 /* Try to get a fragment of a 4K page as a 2K page table */
189 if (!mm_alloc_pgste(mm
)) {
191 spin_lock_bh(&mm
->context
.lock
);
192 if (!list_empty(&mm
->context
.pgtable_list
)) {
193 page
= list_first_entry(&mm
->context
.pgtable_list
,
195 mask
= atomic_read(&page
->_mapcount
);
196 mask
= (mask
| (mask
>> 4)) & 3;
198 table
= (unsigned long *) page_to_phys(page
);
199 bit
= mask
& 1; /* =1 -> second 2K */
201 table
+= PTRS_PER_PTE
;
202 atomic_xor_bits(&page
->_mapcount
, 1U << bit
);
203 list_del(&page
->lru
);
206 spin_unlock_bh(&mm
->context
.lock
);
210 /* Allocate a fresh page */
211 page
= alloc_page(GFP_KERNEL
);
214 if (!pgtable_page_ctor(page
)) {
218 arch_set_page_dat(page
, 0);
219 /* Initialize page table */
220 table
= (unsigned long *) page_to_phys(page
);
221 if (mm_alloc_pgste(mm
)) {
222 /* Return 4K page table with PGSTEs */
223 atomic_set(&page
->_mapcount
, 3);
224 clear_table(table
, _PAGE_INVALID
, PAGE_SIZE
/2);
225 clear_table(table
+ PTRS_PER_PTE
, 0, PAGE_SIZE
/2);
227 /* Return the first 2K fragment of the page */
228 atomic_set(&page
->_mapcount
, 1);
229 clear_table(table
, _PAGE_INVALID
, PAGE_SIZE
);
230 spin_lock_bh(&mm
->context
.lock
);
231 list_add(&page
->lru
, &mm
->context
.pgtable_list
);
232 spin_unlock_bh(&mm
->context
.lock
);
237 void page_table_free(struct mm_struct
*mm
, unsigned long *table
)
240 unsigned int bit
, mask
;
242 page
= pfn_to_page(__pa(table
) >> PAGE_SHIFT
);
243 if (!mm_alloc_pgste(mm
)) {
244 /* Free 2K page table fragment of a 4K page */
245 bit
= (__pa(table
) & ~PAGE_MASK
)/(PTRS_PER_PTE
*sizeof(pte_t
));
246 spin_lock_bh(&mm
->context
.lock
);
247 mask
= atomic_xor_bits(&page
->_mapcount
, 1U << bit
);
249 list_add(&page
->lru
, &mm
->context
.pgtable_list
);
251 list_del(&page
->lru
);
252 spin_unlock_bh(&mm
->context
.lock
);
257 pgtable_page_dtor(page
);
258 atomic_set(&page
->_mapcount
, -1);
262 void page_table_free_rcu(struct mmu_gather
*tlb
, unsigned long *table
,
263 unsigned long vmaddr
)
265 struct mm_struct
*mm
;
267 unsigned int bit
, mask
;
270 page
= pfn_to_page(__pa(table
) >> PAGE_SHIFT
);
271 if (mm_alloc_pgste(mm
)) {
272 gmap_unlink(mm
, table
, vmaddr
);
273 table
= (unsigned long *) (__pa(table
) | 3);
274 tlb_remove_table(tlb
, table
);
277 bit
= (__pa(table
) & ~PAGE_MASK
) / (PTRS_PER_PTE
*sizeof(pte_t
));
278 spin_lock_bh(&mm
->context
.lock
);
279 mask
= atomic_xor_bits(&page
->_mapcount
, 0x11U
<< bit
);
281 list_add_tail(&page
->lru
, &mm
->context
.pgtable_list
);
283 list_del(&page
->lru
);
284 spin_unlock_bh(&mm
->context
.lock
);
285 table
= (unsigned long *) (__pa(table
) | (1U << bit
));
286 tlb_remove_table(tlb
, table
);
289 static void __tlb_remove_table(void *_table
)
291 unsigned int mask
= (unsigned long) _table
& 3;
292 void *table
= (void *)((unsigned long) _table
^ mask
);
293 struct page
*page
= pfn_to_page(__pa(table
) >> PAGE_SHIFT
);
296 case 0: /* pmd, pud, or p4d */
297 free_pages((unsigned long) table
, 2);
299 case 1: /* lower 2K of a 4K page table */
300 case 2: /* higher 2K of a 4K page table */
301 if (atomic_xor_bits(&page
->_mapcount
, mask
<< 4) != 0)
304 case 3: /* 4K page table with pgstes */
305 pgtable_page_dtor(page
);
306 atomic_set(&page
->_mapcount
, -1);
312 static void tlb_remove_table_smp_sync(void *arg
)
314 /* Simply deliver the interrupt */
317 static void tlb_remove_table_one(void *table
)
320 * This isn't an RCU grace period and hence the page-tables cannot be
321 * assumed to be actually RCU-freed.
323 * It is however sufficient for software page-table walkers that rely
324 * on IRQ disabling. See the comment near struct mmu_table_batch.
326 smp_call_function(tlb_remove_table_smp_sync
, NULL
, 1);
327 __tlb_remove_table(table
);
330 static void tlb_remove_table_rcu(struct rcu_head
*head
)
332 struct mmu_table_batch
*batch
;
335 batch
= container_of(head
, struct mmu_table_batch
, rcu
);
337 for (i
= 0; i
< batch
->nr
; i
++)
338 __tlb_remove_table(batch
->tables
[i
]);
340 free_page((unsigned long)batch
);
343 void tlb_table_flush(struct mmu_gather
*tlb
)
345 struct mmu_table_batch
**batch
= &tlb
->batch
;
348 call_rcu_sched(&(*batch
)->rcu
, tlb_remove_table_rcu
);
353 void tlb_remove_table(struct mmu_gather
*tlb
, void *table
)
355 struct mmu_table_batch
**batch
= &tlb
->batch
;
357 tlb
->mm
->context
.flush_mm
= 1;
358 if (*batch
== NULL
) {
359 *batch
= (struct mmu_table_batch
*)
360 __get_free_page(GFP_NOWAIT
| __GFP_NOWARN
);
361 if (*batch
== NULL
) {
362 __tlb_flush_mm_lazy(tlb
->mm
);
363 tlb_remove_table_one(table
);
368 (*batch
)->tables
[(*batch
)->nr
++] = table
;
369 if ((*batch
)->nr
== MAX_TABLE_BATCH
)