]> git.proxmox.com Git - mirror_ubuntu-kernels.git/blame - mm/debug_vm_pgtable.c
projects / mirror_ubuntu-kernels.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
mm/debug_vm_pgtable/ppc64: avoid setting top bits in radom value
[mirror_ubuntu-kernels.git] / mm / debug_vm_pgtable.c
CommitLineData
399145f9
AK		 1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * This kernel test validates architecture page table helpers and
4 * accessors and helps in verifying their continued compliance with
5 * expected generic MM semantics.
6 *
7 * Copyright (C) 2019 ARM Ltd.
8 *
9 * Author: Anshuman Khandual <anshuman.khandual@arm.com>
10 */
6315df41 11#define pr_fmt(fmt) "debug_vm_pgtable: [%-25s]: " fmt, __func__
399145f9
AK		 12
13#include <linux/gfp.h>
14#include <linux/highmem.h>
15#include <linux/hugetlb.h>
16#include <linux/kernel.h>
17#include <linux/kconfig.h>
18#include <linux/mm.h>
19#include <linux/mman.h>
20#include <linux/mm_types.h>
21#include <linux/module.h>
22#include <linux/pfn_t.h>
23#include <linux/printk.h>
a5c3b9ff 24#include <linux/pgtable.h>
399145f9
AK		 25#include <linux/random.h>
26#include <linux/spinlock.h>
27#include <linux/swap.h>
28#include <linux/swapops.h>
29#include <linux/start_kernel.h>
30#include <linux/sched/mm.h>
31#include <asm/pgalloc.h>
a5c3b9ff 32#include <asm/tlbflush.h>
399145f9 33
b1d00007
AK		 34/*
35 * Please refer Documentation/vm/arch_pgtable_helpers.rst for the semantics
36 * expectations that are being validated here. All future changes in here
37 * or the documentation need to be in sync.
38 */
39
399145f9
AK		 40#define VMFLAGS (VM_READ|VM_WRITE|VM_EXEC)
41
42/*
43 * On s390 platform, the lower 4 bits are used to identify given page table
44 * entry type. But these bits might affect the ability to clear entries with
45 * pxx_clear() because of how dynamic page table folding works on s390. So
46 * while loading up the entries do not change the lower 4 bits. It does not
cfc5bbc4
AK		 47 * have affect any other platform. Also avoid the 62nd bit on ppc64 that is
48 * used to mark a pte entry.
399145f9 49 */
cfc5bbc4
AK		 50#define S390_SKIP_MASK GENMASK(3, 0)
51#if __BITS_PER_LONG == 64
52#define PPC64_SKIP_MASK GENMASK(62, 62)
53#else
54#define PPC64_SKIP_MASK 0x0
55#endif
56#define ARCH_SKIP_MASK (S390_SKIP_MASK | PPC64_SKIP_MASK)
57#define RANDOM_ORVALUE (GENMASK(BITS_PER_LONG - 1, 0) & ~ARCH_SKIP_MASK)
399145f9
AK		 58#define RANDOM_NZVALUE GENMASK(7, 0)
59
60static void __init pte_basic_tests(unsigned long pfn, pgprot_t prot)
61{
62 pte_t pte = pfn_pte(pfn, prot);
63
6315df41 64 pr_debug("Validating PTE basic\n");
399145f9
AK		 65 WARN_ON(!pte_same(pte, pte));
66 WARN_ON(!pte_young(pte_mkyoung(pte_mkold(pte))));
67 WARN_ON(!pte_dirty(pte_mkdirty(pte_mkclean(pte))));
68 WARN_ON(!pte_write(pte_mkwrite(pte_wrprotect(pte))));
69 WARN_ON(pte_young(pte_mkold(pte_mkyoung(pte))));
70 WARN_ON(pte_dirty(pte_mkclean(pte_mkdirty(pte))));
71 WARN_ON(pte_write(pte_wrprotect(pte_mkwrite(pte))));
72}
73
a5c3b9ff
AK		 74static void __init pte_advanced_tests(struct mm_struct *mm,
75 struct vm_area_struct *vma, pte_t *ptep,
76 unsigned long pfn, unsigned long vaddr,
77 pgprot_t prot)
78{
79 pte_t pte = pfn_pte(pfn, prot);
80
6315df41 81 pr_debug("Validating PTE advanced\n");
a5c3b9ff
AK		 82 pte = pfn_pte(pfn, prot);
83 set_pte_at(mm, vaddr, ptep, pte);
84 ptep_set_wrprotect(mm, vaddr, ptep);
85 pte = ptep_get(ptep);
86 WARN_ON(pte_write(pte));
87
88 pte = pfn_pte(pfn, prot);
89 set_pte_at(mm, vaddr, ptep, pte);
90 ptep_get_and_clear(mm, vaddr, ptep);
91 pte = ptep_get(ptep);
92 WARN_ON(!pte_none(pte));
93
94 pte = pfn_pte(pfn, prot);
95 pte = pte_wrprotect(pte);
96 pte = pte_mkclean(pte);
97 set_pte_at(mm, vaddr, ptep, pte);
98 pte = pte_mkwrite(pte);
99 pte = pte_mkdirty(pte);
100 ptep_set_access_flags(vma, vaddr, ptep, pte, 1);
101 pte = ptep_get(ptep);
102 WARN_ON(!(pte_write(pte) && pte_dirty(pte)));
103
104 pte = pfn_pte(pfn, prot);
105 set_pte_at(mm, vaddr, ptep, pte);
106 ptep_get_and_clear_full(mm, vaddr, ptep, 1);
107 pte = ptep_get(ptep);
108 WARN_ON(!pte_none(pte));
109
110 pte = pte_mkyoung(pte);
111 set_pte_at(mm, vaddr, ptep, pte);
112 ptep_test_and_clear_young(vma, vaddr, ptep);
113 pte = ptep_get(ptep);
114 WARN_ON(pte_young(pte));
115}
116
117static void __init pte_savedwrite_tests(unsigned long pfn, pgprot_t prot)
118{
119 pte_t pte = pfn_pte(pfn, prot);
120
6315df41 121 pr_debug("Validating PTE saved write\n");
a5c3b9ff
AK		 122 WARN_ON(!pte_savedwrite(pte_mk_savedwrite(pte_clear_savedwrite(pte))));
123 WARN_ON(pte_savedwrite(pte_clear_savedwrite(pte_mk_savedwrite(pte))));
124}
399145f9
AK		 125#ifdef CONFIG_TRANSPARENT_HUGEPAGE
126static void __init pmd_basic_tests(unsigned long pfn, pgprot_t prot)
127{
128 pmd_t pmd = pfn_pmd(pfn, prot);
129
787d563b
AK		 130 if (!has_transparent_hugepage())
131 return;
132
6315df41 133 pr_debug("Validating PMD basic\n");
399145f9
AK		 134 WARN_ON(!pmd_same(pmd, pmd));
135 WARN_ON(!pmd_young(pmd_mkyoung(pmd_mkold(pmd))));
136 WARN_ON(!pmd_dirty(pmd_mkdirty(pmd_mkclean(pmd))));
137 WARN_ON(!pmd_write(pmd_mkwrite(pmd_wrprotect(pmd))));
138 WARN_ON(pmd_young(pmd_mkold(pmd_mkyoung(pmd))));
139 WARN_ON(pmd_dirty(pmd_mkclean(pmd_mkdirty(pmd))));
140 WARN_ON(pmd_write(pmd_wrprotect(pmd_mkwrite(pmd))));
141 /*
142 * A huge page does not point to next level page table
143 * entry. Hence this must qualify as pmd_bad().
144 */
145 WARN_ON(!pmd_bad(pmd_mkhuge(pmd)));
146}
147
a5c3b9ff
AK		 148static void __init pmd_advanced_tests(struct mm_struct *mm,
149 struct vm_area_struct *vma, pmd_t *pmdp,
150 unsigned long pfn, unsigned long vaddr,
151 pgprot_t prot)
152{
153 pmd_t pmd = pfn_pmd(pfn, prot);
154
155 if (!has_transparent_hugepage())
156 return;
157
6315df41 158 pr_debug("Validating PMD advanced\n");
a5c3b9ff
AK		 159 /* Align the address wrt HPAGE_PMD_SIZE */
160 vaddr = (vaddr & HPAGE_PMD_MASK) + HPAGE_PMD_SIZE;
161
162 pmd = pfn_pmd(pfn, prot);
163 set_pmd_at(mm, vaddr, pmdp, pmd);
164 pmdp_set_wrprotect(mm, vaddr, pmdp);
165 pmd = READ_ONCE(*pmdp);
166 WARN_ON(pmd_write(pmd));
167
168 pmd = pfn_pmd(pfn, prot);
169 set_pmd_at(mm, vaddr, pmdp, pmd);
170 pmdp_huge_get_and_clear(mm, vaddr, pmdp);
171 pmd = READ_ONCE(*pmdp);
172 WARN_ON(!pmd_none(pmd));
173
174 pmd = pfn_pmd(pfn, prot);
175 pmd = pmd_wrprotect(pmd);
176 pmd = pmd_mkclean(pmd);
177 set_pmd_at(mm, vaddr, pmdp, pmd);
178 pmd = pmd_mkwrite(pmd);
179 pmd = pmd_mkdirty(pmd);
180 pmdp_set_access_flags(vma, vaddr, pmdp, pmd, 1);
181 pmd = READ_ONCE(*pmdp);
182 WARN_ON(!(pmd_write(pmd) && pmd_dirty(pmd)));
183
184 pmd = pmd_mkhuge(pfn_pmd(pfn, prot));
185 set_pmd_at(mm, vaddr, pmdp, pmd);
186 pmdp_huge_get_and_clear_full(vma, vaddr, pmdp, 1);
187 pmd = READ_ONCE(*pmdp);
188 WARN_ON(!pmd_none(pmd));
189
190 pmd = pmd_mkyoung(pmd);
191 set_pmd_at(mm, vaddr, pmdp, pmd);
192 pmdp_test_and_clear_young(vma, vaddr, pmdp);
193 pmd = READ_ONCE(*pmdp);
194 WARN_ON(pmd_young(pmd));
195}
196
197static void __init pmd_leaf_tests(unsigned long pfn, pgprot_t prot)
198{
199 pmd_t pmd = pfn_pmd(pfn, prot);
200
6315df41 201 pr_debug("Validating PMD leaf\n");
a5c3b9ff
AK		 202 /*
203 * PMD based THP is a leaf entry.
204 */
205 pmd = pmd_mkhuge(pmd);
206 WARN_ON(!pmd_leaf(pmd));
207}
208
209static void __init pmd_huge_tests(pmd_t *pmdp, unsigned long pfn, pgprot_t prot)
210{
211 pmd_t pmd;
212
213 if (!IS_ENABLED(CONFIG_HAVE_ARCH_HUGE_VMAP))
214 return;
6315df41
AK		 215
216 pr_debug("Validating PMD huge\n");
a5c3b9ff
AK		 217 /*
218 * X86 defined pmd_set_huge() verifies that the given
219 * PMD is not a populated non-leaf entry.
220 */
221 WRITE_ONCE(*pmdp, __pmd(0));
222 WARN_ON(!pmd_set_huge(pmdp, __pfn_to_phys(pfn), prot));
223 WARN_ON(!pmd_clear_huge(pmdp));
224 pmd = READ_ONCE(*pmdp);
225 WARN_ON(!pmd_none(pmd));
226}
227
228static void __init pmd_savedwrite_tests(unsigned long pfn, pgprot_t prot)
229{
230 pmd_t pmd = pfn_pmd(pfn, prot);
231
6315df41 232 pr_debug("Validating PMD saved write\n");
a5c3b9ff
AK		 233 WARN_ON(!pmd_savedwrite(pmd_mk_savedwrite(pmd_clear_savedwrite(pmd))));
234 WARN_ON(pmd_savedwrite(pmd_clear_savedwrite(pmd_mk_savedwrite(pmd))));
235}
236
399145f9
AK		 237#ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD
238static void __init pud_basic_tests(unsigned long pfn, pgprot_t prot)
239{
240 pud_t pud = pfn_pud(pfn, prot);
241
787d563b
AK		 242 if (!has_transparent_hugepage())
243 return;
244
6315df41 245 pr_debug("Validating PUD basic\n");
399145f9
AK		 246 WARN_ON(!pud_same(pud, pud));
247 WARN_ON(!pud_young(pud_mkyoung(pud_mkold(pud))));
248 WARN_ON(!pud_write(pud_mkwrite(pud_wrprotect(pud))));
249 WARN_ON(pud_write(pud_wrprotect(pud_mkwrite(pud))));
250 WARN_ON(pud_young(pud_mkold(pud_mkyoung(pud))));
251
252 if (mm_pmd_folded(mm))
253 return;
254
255 /*
256 * A huge page does not point to next level page table
257 * entry. Hence this must qualify as pud_bad().
258 */
259 WARN_ON(!pud_bad(pud_mkhuge(pud)));
260}
a5c3b9ff
AK		 261
262static void __init pud_advanced_tests(struct mm_struct *mm,
263 struct vm_area_struct *vma, pud_t *pudp,
264 unsigned long pfn, unsigned long vaddr,
265 pgprot_t prot)
266{
267 pud_t pud = pfn_pud(pfn, prot);
268
269 if (!has_transparent_hugepage())
270 return;
271
6315df41 272 pr_debug("Validating PUD advanced\n");
a5c3b9ff
AK		 273 /* Align the address wrt HPAGE_PUD_SIZE */
274 vaddr = (vaddr & HPAGE_PUD_MASK) + HPAGE_PUD_SIZE;
275
276 set_pud_at(mm, vaddr, pudp, pud);
277 pudp_set_wrprotect(mm, vaddr, pudp);
278 pud = READ_ONCE(*pudp);
279 WARN_ON(pud_write(pud));
280
281#ifndef __PAGETABLE_PMD_FOLDED
282 pud = pfn_pud(pfn, prot);
283 set_pud_at(mm, vaddr, pudp, pud);
284 pudp_huge_get_and_clear(mm, vaddr, pudp);
285 pud = READ_ONCE(*pudp);
286 WARN_ON(!pud_none(pud));
287
288 pud = pfn_pud(pfn, prot);
289 set_pud_at(mm, vaddr, pudp, pud);
290 pudp_huge_get_and_clear_full(mm, vaddr, pudp, 1);
291 pud = READ_ONCE(*pudp);
292 WARN_ON(!pud_none(pud));
293#endif /* __PAGETABLE_PMD_FOLDED */
294 pud = pfn_pud(pfn, prot);
295 pud = pud_wrprotect(pud);
296 pud = pud_mkclean(pud);
297 set_pud_at(mm, vaddr, pudp, pud);
298 pud = pud_mkwrite(pud);
299 pud = pud_mkdirty(pud);
300 pudp_set_access_flags(vma, vaddr, pudp, pud, 1);
301 pud = READ_ONCE(*pudp);
302 WARN_ON(!(pud_write(pud) && pud_dirty(pud)));
303
304 pud = pud_mkyoung(pud);
305 set_pud_at(mm, vaddr, pudp, pud);
306 pudp_test_and_clear_young(vma, vaddr, pudp);
307 pud = READ_ONCE(*pudp);
308 WARN_ON(pud_young(pud));
309}
310
311static void __init pud_leaf_tests(unsigned long pfn, pgprot_t prot)
312{
313 pud_t pud = pfn_pud(pfn, prot);
314
6315df41 315 pr_debug("Validating PUD leaf\n");
a5c3b9ff
AK		 316 /*
317 * PUD based THP is a leaf entry.
318 */
319 pud = pud_mkhuge(pud);
320 WARN_ON(!pud_leaf(pud));
321}
322
323static void __init pud_huge_tests(pud_t *pudp, unsigned long pfn, pgprot_t prot)
324{
325 pud_t pud;
326
327 if (!IS_ENABLED(CONFIG_HAVE_ARCH_HUGE_VMAP))
328 return;
6315df41
AK		 329
330 pr_debug("Validating PUD huge\n");
a5c3b9ff
AK		 331 /*
332 * X86 defined pud_set_huge() verifies that the given
333 * PUD is not a populated non-leaf entry.
334 */
335 WRITE_ONCE(*pudp, __pud(0));
336 WARN_ON(!pud_set_huge(pudp, __pfn_to_phys(pfn), prot));
337 WARN_ON(!pud_clear_huge(pudp));
338 pud = READ_ONCE(*pudp);
339 WARN_ON(!pud_none(pud));
340}
399145f9
AK		 341#else /* !CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD */
342static void __init pud_basic_tests(unsigned long pfn, pgprot_t prot) { }
a5c3b9ff
AK		 343static void __init pud_advanced_tests(struct mm_struct *mm,
344 struct vm_area_struct *vma, pud_t *pudp,
345 unsigned long pfn, unsigned long vaddr,
346 pgprot_t prot)
347{
348}
349static void __init pud_leaf_tests(unsigned long pfn, pgprot_t prot) { }
350static void __init pud_huge_tests(pud_t *pudp, unsigned long pfn, pgprot_t prot)
351{
352}
399145f9
AK		 353#endif /* CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD */
354#else /* !CONFIG_TRANSPARENT_HUGEPAGE */
355static void __init pmd_basic_tests(unsigned long pfn, pgprot_t prot) { }
356static void __init pud_basic_tests(unsigned long pfn, pgprot_t prot) { }
a5c3b9ff
AK		 357static void __init pmd_advanced_tests(struct mm_struct *mm,
358 struct vm_area_struct *vma, pmd_t *pmdp,
359 unsigned long pfn, unsigned long vaddr,
360 pgprot_t prot)
361{
362}
363static void __init pud_advanced_tests(struct mm_struct *mm,
364 struct vm_area_struct *vma, pud_t *pudp,
365 unsigned long pfn, unsigned long vaddr,
366 pgprot_t prot)
367{
368}
369static void __init pmd_leaf_tests(unsigned long pfn, pgprot_t prot) { }
370static void __init pud_leaf_tests(unsigned long pfn, pgprot_t prot) { }
371static void __init pmd_huge_tests(pmd_t *pmdp, unsigned long pfn, pgprot_t prot)
372{
373}
374static void __init pud_huge_tests(pud_t *pudp, unsigned long pfn, pgprot_t prot)
375{
376}
377static void __init pmd_savedwrite_tests(unsigned long pfn, pgprot_t prot) { }
399145f9
AK		 378#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
379
380static void __init p4d_basic_tests(unsigned long pfn, pgprot_t prot)
381{
382 p4d_t p4d;
383
6315df41 384 pr_debug("Validating P4D basic\n");
399145f9
AK		 385 memset(&p4d, RANDOM_NZVALUE, sizeof(p4d_t));
386 WARN_ON(!p4d_same(p4d, p4d));
387}
388
389static void __init pgd_basic_tests(unsigned long pfn, pgprot_t prot)
390{
391 pgd_t pgd;
392
6315df41 393 pr_debug("Validating PGD basic\n");
399145f9
AK		 394 memset(&pgd, RANDOM_NZVALUE, sizeof(pgd_t));
395 WARN_ON(!pgd_same(pgd, pgd));
396}
397
398#ifndef __PAGETABLE_PUD_FOLDED
399static void __init pud_clear_tests(struct mm_struct *mm, pud_t *pudp)
400{
401 pud_t pud = READ_ONCE(*pudp);
402
403 if (mm_pmd_folded(mm))
404 return;
405
6315df41 406 pr_debug("Validating PUD clear\n");
399145f9
AK		 407 pud = __pud(pud_val(pud) | RANDOM_ORVALUE);
408 WRITE_ONCE(*pudp, pud);
409 pud_clear(pudp);
410 pud = READ_ONCE(*pudp);
411 WARN_ON(!pud_none(pud));
412}
413
414static void __init pud_populate_tests(struct mm_struct *mm, pud_t *pudp,
415 pmd_t *pmdp)
416{
417 pud_t pud;
418
419 if (mm_pmd_folded(mm))
420 return;
6315df41
AK		 421
422 pr_debug("Validating PUD populate\n");
399145f9
AK		 423 /*
424 * This entry points to next level page table page.
425 * Hence this must not qualify as pud_bad().
426 */
427 pmd_clear(pmdp);
428 pud_clear(pudp);
429 pud_populate(mm, pudp, pmdp);
430 pud = READ_ONCE(*pudp);
431 WARN_ON(pud_bad(pud));
432}
433#else /* !__PAGETABLE_PUD_FOLDED */
434static void __init pud_clear_tests(struct mm_struct *mm, pud_t *pudp) { }
435static void __init pud_populate_tests(struct mm_struct *mm, pud_t *pudp,
436 pmd_t *pmdp)
437{
438}
439#endif /* PAGETABLE_PUD_FOLDED */
440
441#ifndef __PAGETABLE_P4D_FOLDED
442static void __init p4d_clear_tests(struct mm_struct *mm, p4d_t *p4dp)
443{
444 p4d_t p4d = READ_ONCE(*p4dp);
445
446 if (mm_pud_folded(mm))
447 return;
448
6315df41 449 pr_debug("Validating P4D clear\n");
399145f9
AK		 450 p4d = __p4d(p4d_val(p4d) | RANDOM_ORVALUE);
451 WRITE_ONCE(*p4dp, p4d);
452 p4d_clear(p4dp);
453 p4d = READ_ONCE(*p4dp);
454 WARN_ON(!p4d_none(p4d));
455}
456
457static void __init p4d_populate_tests(struct mm_struct *mm, p4d_t *p4dp,
458 pud_t *pudp)
459{
460 p4d_t p4d;
461
462 if (mm_pud_folded(mm))
463 return;
464
6315df41 465 pr_debug("Validating P4D populate\n");
399145f9
AK		 466 /*
467 * This entry points to next level page table page.
468 * Hence this must not qualify as p4d_bad().
469 */
470 pud_clear(pudp);
471 p4d_clear(p4dp);
472 p4d_populate(mm, p4dp, pudp);
473 p4d = READ_ONCE(*p4dp);
474 WARN_ON(p4d_bad(p4d));
475}
476
477static void __init pgd_clear_tests(struct mm_struct *mm, pgd_t *pgdp)
478{
479 pgd_t pgd = READ_ONCE(*pgdp);
480
481 if (mm_p4d_folded(mm))
482 return;
483
6315df41 484 pr_debug("Validating PGD clear\n");
399145f9
AK		 485 pgd = __pgd(pgd_val(pgd) | RANDOM_ORVALUE);
486 WRITE_ONCE(*pgdp, pgd);
487 pgd_clear(pgdp);
488 pgd = READ_ONCE(*pgdp);
489 WARN_ON(!pgd_none(pgd));
490}
491
492static void __init pgd_populate_tests(struct mm_struct *mm, pgd_t *pgdp,
493 p4d_t *p4dp)
494{
495 pgd_t pgd;
496
497 if (mm_p4d_folded(mm))
498 return;
499
6315df41 500 pr_debug("Validating PGD populate\n");
399145f9
AK		 501 /*
502 * This entry points to next level page table page.
503 * Hence this must not qualify as pgd_bad().
504 */
505 p4d_clear(p4dp);
506 pgd_clear(pgdp);
507 pgd_populate(mm, pgdp, p4dp);
508 pgd = READ_ONCE(*pgdp);
509 WARN_ON(pgd_bad(pgd));
510}
511#else /* !__PAGETABLE_P4D_FOLDED */
512static void __init p4d_clear_tests(struct mm_struct *mm, p4d_t *p4dp) { }
513static void __init pgd_clear_tests(struct mm_struct *mm, pgd_t *pgdp) { }
514static void __init p4d_populate_tests(struct mm_struct *mm, p4d_t *p4dp,
515 pud_t *pudp)
516{
517}
518static void __init pgd_populate_tests(struct mm_struct *mm, pgd_t *pgdp,
519 p4d_t *p4dp)
520{
521}
522#endif /* PAGETABLE_P4D_FOLDED */
523
524static void __init pte_clear_tests(struct mm_struct *mm, pte_t *ptep,
525 unsigned long vaddr)
526{
9449c9cb 527 pte_t pte = ptep_get(ptep);
399145f9 528
6315df41 529 pr_debug("Validating PTE clear\n");
399145f9
AK		 530 pte = __pte(pte_val(pte) | RANDOM_ORVALUE);
531 set_pte_at(mm, vaddr, ptep, pte);
532 barrier();
533 pte_clear(mm, vaddr, ptep);
9449c9cb 534 pte = ptep_get(ptep);
399145f9
AK		 535 WARN_ON(!pte_none(pte));
536}
537
538static void __init pmd_clear_tests(struct mm_struct *mm, pmd_t *pmdp)
539{
540 pmd_t pmd = READ_ONCE(*pmdp);
541
6315df41 542 pr_debug("Validating PMD clear\n");
399145f9
AK		 543 pmd = __pmd(pmd_val(pmd) | RANDOM_ORVALUE);
544 WRITE_ONCE(*pmdp, pmd);
545 pmd_clear(pmdp);
546 pmd = READ_ONCE(*pmdp);
547 WARN_ON(!pmd_none(pmd));
548}
549
550static void __init pmd_populate_tests(struct mm_struct *mm, pmd_t *pmdp,
551 pgtable_t pgtable)
552{
553 pmd_t pmd;
554
6315df41 555 pr_debug("Validating PMD populate\n");
399145f9
AK		 556 /*
557 * This entry points to next level page table page.
558 * Hence this must not qualify as pmd_bad().
559 */
560 pmd_clear(pmdp);
561 pmd_populate(mm, pmdp, pgtable);
562 pmd = READ_ONCE(*pmdp);
563 WARN_ON(pmd_bad(pmd));
564}
565
05289402
AK		 566static void __init pte_special_tests(unsigned long pfn, pgprot_t prot)
567{
568 pte_t pte = pfn_pte(pfn, prot);
569
570 if (!IS_ENABLED(CONFIG_ARCH_HAS_PTE_SPECIAL))
571 return;
572
6315df41 573 pr_debug("Validating PTE special\n");
05289402
AK		 574 WARN_ON(!pte_special(pte_mkspecial(pte)));
575}
576
577static void __init pte_protnone_tests(unsigned long pfn, pgprot_t prot)
578{
579 pte_t pte = pfn_pte(pfn, prot);
580
581 if (!IS_ENABLED(CONFIG_NUMA_BALANCING))
582 return;
583
6315df41 584 pr_debug("Validating PTE protnone\n");
05289402
AK		 585 WARN_ON(!pte_protnone(pte));
586 WARN_ON(!pte_present(pte));
587}
588
589#ifdef CONFIG_TRANSPARENT_HUGEPAGE
590static void __init pmd_protnone_tests(unsigned long pfn, pgprot_t prot)
591{
592 pmd_t pmd = pmd_mkhuge(pfn_pmd(pfn, prot));
593
594 if (!IS_ENABLED(CONFIG_NUMA_BALANCING))
595 return;
596
6315df41 597 pr_debug("Validating PMD protnone\n");
05289402
AK		 598 WARN_ON(!pmd_protnone(pmd));
599 WARN_ON(!pmd_present(pmd));
600}
601#else /* !CONFIG_TRANSPARENT_HUGEPAGE */
602static void __init pmd_protnone_tests(unsigned long pfn, pgprot_t prot) { }
603#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
604
605#ifdef CONFIG_ARCH_HAS_PTE_DEVMAP
606static void __init pte_devmap_tests(unsigned long pfn, pgprot_t prot)
607{
608 pte_t pte = pfn_pte(pfn, prot);
609
6315df41 610 pr_debug("Validating PTE devmap\n");
05289402
AK		 611 WARN_ON(!pte_devmap(pte_mkdevmap(pte)));
612}
613
614#ifdef CONFIG_TRANSPARENT_HUGEPAGE
615static void __init pmd_devmap_tests(unsigned long pfn, pgprot_t prot)
616{
617 pmd_t pmd = pfn_pmd(pfn, prot);
618
6315df41 619 pr_debug("Validating PMD devmap\n");
05289402
AK		 620 WARN_ON(!pmd_devmap(pmd_mkdevmap(pmd)));
621}
622
623#ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD
624static void __init pud_devmap_tests(unsigned long pfn, pgprot_t prot)
625{
626 pud_t pud = pfn_pud(pfn, prot);
627
6315df41 628 pr_debug("Validating PUD devmap\n");
05289402
AK		 629 WARN_ON(!pud_devmap(pud_mkdevmap(pud)));
630}
631#else /* !CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD */
632static void __init pud_devmap_tests(unsigned long pfn, pgprot_t prot) { }
633#endif /* CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD */
634#else /* CONFIG_TRANSPARENT_HUGEPAGE */
635static void __init pmd_devmap_tests(unsigned long pfn, pgprot_t prot) { }
636static void __init pud_devmap_tests(unsigned long pfn, pgprot_t prot) { }
637#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
638#else
639static void __init pte_devmap_tests(unsigned long pfn, pgprot_t prot) { }
640static void __init pmd_devmap_tests(unsigned long pfn, pgprot_t prot) { }
641static void __init pud_devmap_tests(unsigned long pfn, pgprot_t prot) { }
642#endif /* CONFIG_ARCH_HAS_PTE_DEVMAP */
643
644static void __init pte_soft_dirty_tests(unsigned long pfn, pgprot_t prot)
645{
646 pte_t pte = pfn_pte(pfn, prot);
647
648 if (!IS_ENABLED(CONFIG_MEM_SOFT_DIRTY))
649 return;
650
6315df41 651 pr_debug("Validating PTE soft dirty\n");
05289402
AK		 652 WARN_ON(!pte_soft_dirty(pte_mksoft_dirty(pte)));
653 WARN_ON(pte_soft_dirty(pte_clear_soft_dirty(pte)));
654}
655
656static void __init pte_swap_soft_dirty_tests(unsigned long pfn, pgprot_t prot)
657{
658 pte_t pte = pfn_pte(pfn, prot);
659
660 if (!IS_ENABLED(CONFIG_MEM_SOFT_DIRTY))
661 return;
662
6315df41 663 pr_debug("Validating PTE swap soft dirty\n");
05289402
AK		 664 WARN_ON(!pte_swp_soft_dirty(pte_swp_mksoft_dirty(pte)));
665 WARN_ON(pte_swp_soft_dirty(pte_swp_clear_soft_dirty(pte)));
666}
667
668#ifdef CONFIG_TRANSPARENT_HUGEPAGE
669static void __init pmd_soft_dirty_tests(unsigned long pfn, pgprot_t prot)
670{
671 pmd_t pmd = pfn_pmd(pfn, prot);
672
673 if (!IS_ENABLED(CONFIG_MEM_SOFT_DIRTY))
674 return;
675
6315df41 676 pr_debug("Validating PMD soft dirty\n");
05289402
AK		 677 WARN_ON(!pmd_soft_dirty(pmd_mksoft_dirty(pmd)));
678 WARN_ON(pmd_soft_dirty(pmd_clear_soft_dirty(pmd)));
679}
680
681static void __init pmd_swap_soft_dirty_tests(unsigned long pfn, pgprot_t prot)
682{
683 pmd_t pmd = pfn_pmd(pfn, prot);
684
685 if (!IS_ENABLED(CONFIG_MEM_SOFT_DIRTY) ||
686 !IS_ENABLED(CONFIG_ARCH_ENABLE_THP_MIGRATION))
687 return;
688
6315df41 689 pr_debug("Validating PMD swap soft dirty\n");
05289402
AK		 690 WARN_ON(!pmd_swp_soft_dirty(pmd_swp_mksoft_dirty(pmd)));
691 WARN_ON(pmd_swp_soft_dirty(pmd_swp_clear_soft_dirty(pmd)));
692}
693#else /* !CONFIG_ARCH_HAS_PTE_DEVMAP */
694static void __init pmd_soft_dirty_tests(unsigned long pfn, pgprot_t prot) { }
695static void __init pmd_swap_soft_dirty_tests(unsigned long pfn, pgprot_t prot)
696{
697}
698#endif /* CONFIG_ARCH_HAS_PTE_DEVMAP */
699
700static void __init pte_swap_tests(unsigned long pfn, pgprot_t prot)
701{
702 swp_entry_t swp;
703 pte_t pte;
704
6315df41 705 pr_debug("Validating PTE swap\n");
05289402
AK		 706 pte = pfn_pte(pfn, prot);
707 swp = __pte_to_swp_entry(pte);
708 pte = __swp_entry_to_pte(swp);
709 WARN_ON(pfn != pte_pfn(pte));
710}
711
712#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION
713static void __init pmd_swap_tests(unsigned long pfn, pgprot_t prot)
714{
715 swp_entry_t swp;
716 pmd_t pmd;
717
6315df41 718 pr_debug("Validating PMD swap\n");
05289402
AK		 719 pmd = pfn_pmd(pfn, prot);
720 swp = __pmd_to_swp_entry(pmd);
721 pmd = __swp_entry_to_pmd(swp);
722 WARN_ON(pfn != pmd_pfn(pmd));
723}
724#else /* !CONFIG_ARCH_ENABLE_THP_MIGRATION */
725static void __init pmd_swap_tests(unsigned long pfn, pgprot_t prot) { }
726#endif /* CONFIG_ARCH_ENABLE_THP_MIGRATION */
727
728static void __init swap_migration_tests(void)
729{
730 struct page *page;
731 swp_entry_t swp;
732
733 if (!IS_ENABLED(CONFIG_MIGRATION))
734 return;
6315df41
AK		 735
736 pr_debug("Validating swap migration\n");
05289402
AK		 737 /*
738 * swap_migration_tests() requires a dedicated page as it needs to
739 * be locked before creating a migration entry from it. Locking the
740 * page that actually maps kernel text ('start_kernel') can be real
741 * problematic. Lets allocate a dedicated page explicitly for this
742 * purpose that will be freed subsequently.
743 */
744 page = alloc_page(GFP_KERNEL);
745 if (!page) {
746 pr_err("page allocation failed\n");
747 return;
748 }
749
750 /*
751 * make_migration_entry() expects given page to be
752 * locked, otherwise it stumbles upon a BUG_ON().
753 */
754 __SetPageLocked(page);
755 swp = make_migration_entry(page, 1);
756 WARN_ON(!is_migration_entry(swp));
757 WARN_ON(!is_write_migration_entry(swp));
758
759 make_migration_entry_read(&swp);
760 WARN_ON(!is_migration_entry(swp));
761 WARN_ON(is_write_migration_entry(swp));
762
763 swp = make_migration_entry(page, 0);
764 WARN_ON(!is_migration_entry(swp));
765 WARN_ON(is_write_migration_entry(swp));
766 __ClearPageLocked(page);
767 __free_page(page);
768}
769
770#ifdef CONFIG_HUGETLB_PAGE
771static void __init hugetlb_basic_tests(unsigned long pfn, pgprot_t prot)
772{
773 struct page *page;
774 pte_t pte;
775
6315df41 776 pr_debug("Validating HugeTLB basic\n");
05289402
AK		 777 /*
778 * Accessing the page associated with the pfn is safe here,
779 * as it was previously derived from a real kernel symbol.
780 */
781 page = pfn_to_page(pfn);
782 pte = mk_huge_pte(page, prot);
783
784 WARN_ON(!huge_pte_dirty(huge_pte_mkdirty(pte)));
785 WARN_ON(!huge_pte_write(huge_pte_mkwrite(huge_pte_wrprotect(pte))));
786 WARN_ON(huge_pte_write(huge_pte_wrprotect(huge_pte_mkwrite(pte))));
787
788#ifdef CONFIG_ARCH_WANT_GENERAL_HUGETLB
789 pte = pfn_pte(pfn, prot);
790
791 WARN_ON(!pte_huge(pte_mkhuge(pte)));
792#endif /* CONFIG_ARCH_WANT_GENERAL_HUGETLB */
793}
a5c3b9ff
AK		 794
795static void __init hugetlb_advanced_tests(struct mm_struct *mm,
796 struct vm_area_struct *vma,
797 pte_t *ptep, unsigned long pfn,
798 unsigned long vaddr, pgprot_t prot)
799{
800 struct page *page = pfn_to_page(pfn);
801 pte_t pte = ptep_get(ptep);
802 unsigned long paddr = __pfn_to_phys(pfn) & PMD_MASK;
803
6315df41 804 pr_debug("Validating HugeTLB advanced\n");
a5c3b9ff
AK		 805 pte = pte_mkhuge(mk_pte(pfn_to_page(PHYS_PFN(paddr)), prot));
806 set_huge_pte_at(mm, vaddr, ptep, pte);
807 barrier();
808 WARN_ON(!pte_same(pte, huge_ptep_get(ptep)));
809 huge_pte_clear(mm, vaddr, ptep, PMD_SIZE);
810 pte = huge_ptep_get(ptep);
811 WARN_ON(!huge_pte_none(pte));
812
813 pte = mk_huge_pte(page, prot);
814 set_huge_pte_at(mm, vaddr, ptep, pte);
815 barrier();
816 huge_ptep_set_wrprotect(mm, vaddr, ptep);
817 pte = huge_ptep_get(ptep);
818 WARN_ON(huge_pte_write(pte));
819
820 pte = mk_huge_pte(page, prot);
821 set_huge_pte_at(mm, vaddr, ptep, pte);
822 barrier();
823 huge_ptep_get_and_clear(mm, vaddr, ptep);
824 pte = huge_ptep_get(ptep);
825 WARN_ON(!huge_pte_none(pte));
826
827 pte = mk_huge_pte(page, prot);
828 pte = huge_pte_wrprotect(pte);
829 set_huge_pte_at(mm, vaddr, ptep, pte);
830 barrier();
831 pte = huge_pte_mkwrite(pte);
832 pte = huge_pte_mkdirty(pte);
833 huge_ptep_set_access_flags(vma, vaddr, ptep, pte, 1);
834 pte = huge_ptep_get(ptep);
835 WARN_ON(!(huge_pte_write(pte) && huge_pte_dirty(pte)));
836}
05289402
AK		 837#else /* !CONFIG_HUGETLB_PAGE */
838static void __init hugetlb_basic_tests(unsigned long pfn, pgprot_t prot) { }
a5c3b9ff
AK		 839static void __init hugetlb_advanced_tests(struct mm_struct *mm,
840 struct vm_area_struct *vma,
841 pte_t *ptep, unsigned long pfn,
842 unsigned long vaddr, pgprot_t prot)
843{
844}
05289402
AK		 845#endif /* CONFIG_HUGETLB_PAGE */
846
847#ifdef CONFIG_TRANSPARENT_HUGEPAGE
848static void __init pmd_thp_tests(unsigned long pfn, pgprot_t prot)
849{
850 pmd_t pmd;
851
852 if (!has_transparent_hugepage())
853 return;
854
6315df41 855 pr_debug("Validating PMD based THP\n");
05289402
AK		 856 /*
857 * pmd_trans_huge() and pmd_present() must return positive after
858 * MMU invalidation with pmd_mkinvalid(). This behavior is an
859 * optimization for transparent huge page. pmd_trans_huge() must
860 * be true if pmd_page() returns a valid THP to avoid taking the
861 * pmd_lock when others walk over non transhuge pmds (i.e. there
862 * are no THP allocated). Especially when splitting a THP and
863 * removing the present bit from the pmd, pmd_trans_huge() still
864 * needs to return true. pmd_present() should be true whenever
865 * pmd_trans_huge() returns true.
866 */
867 pmd = pfn_pmd(pfn, prot);
868 WARN_ON(!pmd_trans_huge(pmd_mkhuge(pmd)));
869
870#ifndef __HAVE_ARCH_PMDP_INVALIDATE
871 WARN_ON(!pmd_trans_huge(pmd_mkinvalid(pmd_mkhuge(pmd))));
872 WARN_ON(!pmd_present(pmd_mkinvalid(pmd_mkhuge(pmd))));
873#endif /* __HAVE_ARCH_PMDP_INVALIDATE */
874}
875
876#ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD
877static void __init pud_thp_tests(unsigned long pfn, pgprot_t prot)
878{
879 pud_t pud;
880
881 if (!has_transparent_hugepage())
882 return;
883
6315df41 884 pr_debug("Validating PUD based THP\n");
05289402
AK		 885 pud = pfn_pud(pfn, prot);
886 WARN_ON(!pud_trans_huge(pud_mkhuge(pud)));
887
888 /*
889 * pud_mkinvalid() has been dropped for now. Enable back
890 * these tests when it comes back with a modified pud_present().
891 *
892 * WARN_ON(!pud_trans_huge(pud_mkinvalid(pud_mkhuge(pud))));
893 * WARN_ON(!pud_present(pud_mkinvalid(pud_mkhuge(pud))));
894 */
895}
896#else /* !CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD */
897static void __init pud_thp_tests(unsigned long pfn, pgprot_t prot) { }
898#endif /* CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD */
899#else /* !CONFIG_TRANSPARENT_HUGEPAGE */
900static void __init pmd_thp_tests(unsigned long pfn, pgprot_t prot) { }
901static void __init pud_thp_tests(unsigned long pfn, pgprot_t prot) { }
902#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
903
399145f9
AK		 904static unsigned long __init get_random_vaddr(void)
905{
906 unsigned long random_vaddr, random_pages, total_user_pages;
907
908 total_user_pages = (TASK_SIZE - FIRST_USER_ADDRESS) / PAGE_SIZE;
909
910 random_pages = get_random_long() % total_user_pages;
911 random_vaddr = FIRST_USER_ADDRESS + random_pages * PAGE_SIZE;
912
913 return random_vaddr;
914}
915
916static int __init debug_vm_pgtable(void)
917{
a5c3b9ff 918 struct vm_area_struct *vma;
399145f9
AK		 919 struct mm_struct *mm;
920 pgd_t *pgdp;
921 p4d_t *p4dp, *saved_p4dp;
922 pud_t *pudp, *saved_pudp;
923 pmd_t *pmdp, *saved_pmdp, pmd;
924 pte_t *ptep;
925 pgtable_t saved_ptep;
05289402 926 pgprot_t prot, protnone;
399145f9
AK		 927 phys_addr_t paddr;
928 unsigned long vaddr, pte_aligned, pmd_aligned;
929 unsigned long pud_aligned, p4d_aligned, pgd_aligned;
fea1120c 930 spinlock_t *ptl = NULL;
399145f9
AK		 931
932 pr_info("Validating architecture page table helpers\n");
933 prot = vm_get_page_prot(VMFLAGS);
934 vaddr = get_random_vaddr();
935 mm = mm_alloc();
936 if (!mm) {
937 pr_err("mm_struct allocation failed\n");
938 return 1;
939 }
940
05289402
AK		 941 /*
942 * __P000 (or even __S000) will help create page table entries with
943 * PROT_NONE permission as required for pxx_protnone_tests().
944 */
945 protnone = __P000;
946
a5c3b9ff
AK		 947 vma = vm_area_alloc(mm);
948 if (!vma) {
949 pr_err("vma allocation failed\n");
950 return 1;
951 }
952
399145f9
AK		 953 /*
954 * PFN for mapping at PTE level is determined from a standard kernel
955 * text symbol. But pfns for higher page table levels are derived by
956 * masking lower bits of this real pfn. These derived pfns might not
957 * exist on the platform but that does not really matter as pfn_pxx()
958 * helpers will still create appropriate entries for the test. This
959 * helps avoid large memory block allocations to be used for mapping
960 * at higher page table levels.
961 */
962 paddr = __pa_symbol(&start_kernel);
963
964 pte_aligned = (paddr & PAGE_MASK) >> PAGE_SHIFT;
965 pmd_aligned = (paddr & PMD_MASK) >> PAGE_SHIFT;
966 pud_aligned = (paddr & PUD_MASK) >> PAGE_SHIFT;
967 p4d_aligned = (paddr & P4D_MASK) >> PAGE_SHIFT;
968 pgd_aligned = (paddr & PGDIR_MASK) >> PAGE_SHIFT;
969 WARN_ON(!pfn_valid(pte_aligned));
970
971 pgdp = pgd_offset(mm, vaddr);
972 p4dp = p4d_alloc(mm, pgdp, vaddr);
973 pudp = pud_alloc(mm, p4dp, vaddr);
974 pmdp = pmd_alloc(mm, pudp, vaddr);
975 ptep = pte_alloc_map_lock(mm, pmdp, vaddr, &ptl);
976
977 /*
978 * Save all the page table page addresses as the page table
979 * entries will be used for testing with random or garbage
980 * values. These saved addresses will be used for freeing
981 * page table pages.
982 */
983 pmd = READ_ONCE(*pmdp);
984 saved_p4dp = p4d_offset(pgdp, 0UL);
985 saved_pudp = pud_offset(p4dp, 0UL);
986 saved_pmdp = pmd_offset(pudp, 0UL);
987 saved_ptep = pmd_pgtable(pmd);
988
989 pte_basic_tests(pte_aligned, prot);
990 pmd_basic_tests(pmd_aligned, prot);
991 pud_basic_tests(pud_aligned, prot);
992 p4d_basic_tests(p4d_aligned, prot);
993 pgd_basic_tests(pgd_aligned, prot);
994
995 pte_clear_tests(mm, ptep, vaddr);
996 pmd_clear_tests(mm, pmdp);
997 pud_clear_tests(mm, pudp);
998 p4d_clear_tests(mm, p4dp);
999 pgd_clear_tests(mm, pgdp);
1000
a5c3b9ff
AK		 1001 pte_advanced_tests(mm, vma, ptep, pte_aligned, vaddr, prot);
1002 pmd_advanced_tests(mm, vma, pmdp, pmd_aligned, vaddr, prot);
1003 pud_advanced_tests(mm, vma, pudp, pud_aligned, vaddr, prot);
1004 hugetlb_advanced_tests(mm, vma, ptep, pte_aligned, vaddr, prot);
1005
1006 pmd_leaf_tests(pmd_aligned, prot);
1007 pud_leaf_tests(pud_aligned, prot);
1008
1009 pmd_huge_tests(pmdp, pmd_aligned, prot);
1010 pud_huge_tests(pudp, pud_aligned, prot);
1011
1012 pte_savedwrite_tests(pte_aligned, prot);
1013 pmd_savedwrite_tests(pmd_aligned, prot);
1014
399145f9
AK		 1015 pte_unmap_unlock(ptep, ptl);
1016
1017 pmd_populate_tests(mm, pmdp, saved_ptep);
1018 pud_populate_tests(mm, pudp, saved_pmdp);
1019 p4d_populate_tests(mm, p4dp, saved_pudp);
1020 pgd_populate_tests(mm, pgdp, saved_p4dp);
1021
05289402
AK		 1022 pte_special_tests(pte_aligned, prot);
1023 pte_protnone_tests(pte_aligned, protnone);
1024 pmd_protnone_tests(pmd_aligned, protnone);
1025
1026 pte_devmap_tests(pte_aligned, prot);
1027 pmd_devmap_tests(pmd_aligned, prot);
1028 pud_devmap_tests(pud_aligned, prot);
1029
1030 pte_soft_dirty_tests(pte_aligned, prot);
1031 pmd_soft_dirty_tests(pmd_aligned, prot);
1032 pte_swap_soft_dirty_tests(pte_aligned, prot);
1033 pmd_swap_soft_dirty_tests(pmd_aligned, prot);
1034
1035 pte_swap_tests(pte_aligned, prot);
1036 pmd_swap_tests(pmd_aligned, prot);
1037
1038 swap_migration_tests();
1039 hugetlb_basic_tests(pte_aligned, prot);
1040
1041 pmd_thp_tests(pmd_aligned, prot);
1042 pud_thp_tests(pud_aligned, prot);
1043
399145f9
AK		 1044 p4d_free(mm, saved_p4dp);
1045 pud_free(mm, saved_pudp);
1046 pmd_free(mm, saved_pmdp);
1047 pte_free(mm, saved_ptep);
1048
a5c3b9ff 1049 vm_area_free(vma);
399145f9
AK		 1050 mm_dec_nr_puds(mm);
1051 mm_dec_nr_pmds(mm);
1052 mm_dec_nr_ptes(mm);
1053 mmdrop(mm);
1054 return 0;
1055}
1056late_initcall(debug_vm_pgtable);
Ubuntu combined Linux kernel mirror