mm/pagewalk.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 #include <linux/pagewalk.h>
   3 #include <linux/highmem.h>
   4 #include <linux/sched.h>
   5 #include <linux/hugetlb.h>
   6
   7 /*
   8  * We want to know the real level where a entry is located ignoring any
   9  * folding of levels which may be happening. For example if p4d is folded then
  10  * a missing entry found at level 1 (p4d) is actually at level 0 (pgd).
  11  */
  12 static int real_depth(int depth)
  13 {
  14         if (depth == 3 && PTRS_PER_PMD == 1)
  15                 depth = 2;
  16         if (depth == 2 && PTRS_PER_PUD == 1)
  17                 depth = 1;
  18         if (depth == 1 && PTRS_PER_P4D == 1)
  19                 depth = 0;
  20         return depth;
  21 }
  22
  23 static int walk_pte_range_inner(pte_t *pte, unsigned long addr,
  24                                 unsigned long end, struct mm_walk *walk)
  25 {
  26         const struct mm_walk_ops *ops = walk->ops;
  27         int err = 0;
  28
  29         for (;;) {
  30                 err = ops->pte_entry(pte, addr, addr + PAGE_SIZE, walk);
  31                 if (err)
  32                        break;
  33                 if (addr >= end - PAGE_SIZE)
  34                         break;
  35                 addr += PAGE_SIZE;
  36                 pte++;
  37         }
  38         return err;
  39 }
  40
  41 static int walk_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end,
  42                           struct mm_walk *walk)
  43 {
  44         pte_t *pte;
  45         int err = 0;
  46         spinlock_t *ptl;
  47
  48         if (walk->no_vma) {
  49                 pte = pte_offset_map(pmd, addr);
  50                 err = walk_pte_range_inner(pte, addr, end, walk);
  51                 pte_unmap(pte);
  52         } else {
  53                 pte = pte_offset_map_lock(walk->mm, pmd, addr, &ptl);
  54                 err = walk_pte_range_inner(pte, addr, end, walk);
  55                 pte_unmap_unlock(pte, ptl);
  56         }
  57
  58         return err;
  59 }
  60
  61 #ifdef CONFIG_ARCH_HAS_HUGEPD
  62 static int walk_hugepd_range(hugepd_t *phpd, unsigned long addr,
  63                              unsigned long end, struct mm_walk *walk, int pdshift)
  64 {
  65         int err = 0;
  66         const struct mm_walk_ops *ops = walk->ops;
  67         int shift = hugepd_shift(*phpd);
  68         int page_size = 1 << shift;
  69
  70         if (!ops->pte_entry)
  71                 return 0;
  72
  73         if (addr & (page_size - 1))
  74                 return 0;
  75
  76         for (;;) {
  77                 pte_t *pte;
  78
  79                 spin_lock(&walk->mm->page_table_lock);
  80                 pte = hugepte_offset(*phpd, addr, pdshift);
  81                 err = ops->pte_entry(pte, addr, addr + page_size, walk);
  82                 spin_unlock(&walk->mm->page_table_lock);
  83
  84                 if (err)
  85                         break;
  86                 if (addr >= end - page_size)
  87                         break;
  88                 addr += page_size;
  89         }
  90         return err;
  91 }
  92 #else
  93 static int walk_hugepd_range(hugepd_t *phpd, unsigned long addr,
  94                              unsigned long end, struct mm_walk *walk, int pdshift)
  95 {
  96         return 0;
  97 }
  98 #endif
  99
 100 static int walk_pmd_range(pud_t *pud, unsigned long addr, unsigned long end,
 101                           struct mm_walk *walk)
 102 {
 103         pmd_t *pmd;
 104         unsigned long next;
 105         const struct mm_walk_ops *ops = walk->ops;
 106         int err = 0;
 107         int depth = real_depth(3);
 108
 109         pmd = pmd_offset(pud, addr);
 110         do {
 111 again:
 112                 next = pmd_addr_end(addr, end);
 113                 if (pmd_none(*pmd)) {
 114                         if (ops->pte_hole)
 115                                 err = ops->pte_hole(addr, next, depth, walk);
 116                         if (err)
 117                                 break;
 118                         continue;
 119                 }
 120
 121                 walk->action = ACTION_SUBTREE;
 122
 123                 /*
 124                  * This implies that each ->pmd_entry() handler
 125                  * needs to know about pmd_trans_huge() pmds
 126                  */
 127                 if (ops->pmd_entry)
 128                         err = ops->pmd_entry(pmd, addr, next, walk);
 129                 if (err)
 130                         break;
 131
 132                 if (walk->action == ACTION_AGAIN)
 133                         goto again;
 134
 135                 /*
 136                  * Check this here so we only break down trans_huge
 137                  * pages when we _need_ to
 138                  */
 139                 if ((!walk->vma && (pmd_leaf(*pmd) || !pmd_present(*pmd))) ||
 140                     walk->action == ACTION_CONTINUE ||
 141                     !(ops->pte_entry))
 142                         continue;
 143
 144                 if (walk->vma) {
 145                         split_huge_pmd(walk->vma, pmd, addr);
 146                         if (pmd_trans_unstable(pmd))
 147                                 goto again;
 148                 }
 149
 150                 if (is_hugepd(__hugepd(pmd_val(*pmd))))
 151                         err = walk_hugepd_range((hugepd_t *)pmd, addr, next, walk, PMD_SHIFT);
 152                 else
 153                         err = walk_pte_range(pmd, addr, next, walk);
 154                 if (err)
 155                         break;
 156         } while (pmd++, addr = next, addr != end);
 157
 158         return err;
 159 }
 160
 161 static int walk_pud_range(p4d_t *p4d, unsigned long addr, unsigned long end,
 162                           struct mm_walk *walk)
 163 {
 164         pud_t *pud;
 165         unsigned long next;
 166         const struct mm_walk_ops *ops = walk->ops;
 167         int err = 0;
 168         int depth = real_depth(2);
 169
 170         pud = pud_offset(p4d, addr);
 171         do {
 172  again:
 173                 next = pud_addr_end(addr, end);
 174                 if (pud_none(*pud)) {
 175                         if (ops->pte_hole)
 176                                 err = ops->pte_hole(addr, next, depth, walk);
 177                         if (err)
 178                                 break;
 179                         continue;
 180                 }
 181
 182                 walk->action = ACTION_SUBTREE;
 183
 184                 if (ops->pud_entry)
 185                         err = ops->pud_entry(pud, addr, next, walk);
 186                 if (err)
 187                         break;
 188
 189                 if (walk->action == ACTION_AGAIN)
 190                         goto again;
 191
 192                 if ((!walk->vma && (pud_leaf(*pud) || !pud_present(*pud))) ||
 193                     walk->action == ACTION_CONTINUE ||
 194                     !(ops->pmd_entry || ops->pte_entry))
 195                         continue;
 196
 197                 if (walk->vma)
 198                         split_huge_pud(walk->vma, pud, addr);
 199                 if (pud_none(*pud))
 200                         goto again;
 201
 202                 if (is_hugepd(__hugepd(pud_val(*pud))))
 203                         err = walk_hugepd_range((hugepd_t *)pud, addr, next, walk, PUD_SHIFT);
 204                 else
 205                         err = walk_pmd_range(pud, addr, next, walk);
 206                 if (err)
 207                         break;
 208         } while (pud++, addr = next, addr != end);
 209
 210         return err;
 211 }
 212
 213 static int walk_p4d_range(pgd_t *pgd, unsigned long addr, unsigned long end,
 214                           struct mm_walk *walk)
 215 {
 216         p4d_t *p4d;
 217         unsigned long next;
 218         const struct mm_walk_ops *ops = walk->ops;
 219         int err = 0;
 220         int depth = real_depth(1);
 221
 222         p4d = p4d_offset(pgd, addr);
 223         do {
 224                 next = p4d_addr_end(addr, end);
 225                 if (p4d_none_or_clear_bad(p4d)) {
 226                         if (ops->pte_hole)
 227                                 err = ops->pte_hole(addr, next, depth, walk);
 228                         if (err)
 229                                 break;
 230                         continue;
 231                 }
 232                 if (ops->p4d_entry) {
 233                         err = ops->p4d_entry(p4d, addr, next, walk);
 234                         if (err)
 235                                 break;
 236                 }
 237                 if (is_hugepd(__hugepd(p4d_val(*p4d))))
 238                         err = walk_hugepd_range((hugepd_t *)p4d, addr, next, walk, P4D_SHIFT);
 239                 else if (ops->pud_entry || ops->pmd_entry || ops->pte_entry)
 240                         err = walk_pud_range(p4d, addr, next, walk);
 241                 if (err)
 242                         break;
 243         } while (p4d++, addr = next, addr != end);
 244
 245         return err;
 246 }
 247
 248 static int walk_pgd_range(unsigned long addr, unsigned long end,
 249                           struct mm_walk *walk)
 250 {
 251         pgd_t *pgd;
 252         unsigned long next;
 253         const struct mm_walk_ops *ops = walk->ops;
 254         int err = 0;
 255
 256         if (walk->pgd)
 257                 pgd = walk->pgd + pgd_index(addr);
 258         else
 259                 pgd = pgd_offset(walk->mm, addr);
 260         do {
 261                 next = pgd_addr_end(addr, end);
 262                 if (pgd_none_or_clear_bad(pgd)) {
 263                         if (ops->pte_hole)
 264                                 err = ops->pte_hole(addr, next, 0, walk);
 265                         if (err)
 266                                 break;
 267                         continue;
 268                 }
 269                 if (ops->pgd_entry) {
 270                         err = ops->pgd_entry(pgd, addr, next, walk);
 271                         if (err)
 272                                 break;
 273                 }
 274                 if (is_hugepd(__hugepd(pgd_val(*pgd))))
 275                         err = walk_hugepd_range((hugepd_t *)pgd, addr, next, walk, PGDIR_SHIFT);
 276                 else if (ops->p4d_entry || ops->pud_entry || ops->pmd_entry || ops->pte_entry)
 277                         err = walk_p4d_range(pgd, addr, next, walk);
 278                 if (err)
 279                         break;
 280         } while (pgd++, addr = next, addr != end);
 281
 282         return err;
 283 }
 284
 285 #ifdef CONFIG_HUGETLB_PAGE
 286 static unsigned long hugetlb_entry_end(struct hstate *h, unsigned long addr,
 287                                        unsigned long end)
 288 {
 289         unsigned long boundary = (addr & huge_page_mask(h)) + huge_page_size(h);
 290         return boundary < end ? boundary : end;
 291 }
 292
 293 static int walk_hugetlb_range(unsigned long addr, unsigned long end,
 294                               struct mm_walk *walk)
 295 {
 296         struct vm_area_struct *vma = walk->vma;
 297         struct hstate *h = hstate_vma(vma);
 298         unsigned long next;
 299         unsigned long hmask = huge_page_mask(h);
 300         unsigned long sz = huge_page_size(h);
 301         pte_t *pte;
 302         const struct mm_walk_ops *ops = walk->ops;
 303         int err = 0;
 304
 305         do {
 306                 next = hugetlb_entry_end(h, addr, end);
 307                 pte = huge_pte_offset(walk->mm, addr & hmask, sz);
 308
 309                 if (pte)
 310                         err = ops->hugetlb_entry(pte, hmask, addr, next, walk);
 311                 else if (ops->pte_hole)
 312                         err = ops->pte_hole(addr, next, -1, walk);
 313
 314                 if (err)
 315                         break;
 316         } while (addr = next, addr != end);
 317
 318         return err;
 319 }
 320
 321 #else /* CONFIG_HUGETLB_PAGE */
 322 static int walk_hugetlb_range(unsigned long addr, unsigned long end,
 323                               struct mm_walk *walk)
 324 {
 325         return 0;
 326 }
 327
 328 #endif /* CONFIG_HUGETLB_PAGE */
 329
 330 /*
 331  * Decide whether we really walk over the current vma on [@start, @end)
 332  * or skip it via the returned value. Return 0 if we do walk over the
 333  * current vma, and return 1 if we skip the vma. Negative values means
 334  * error, where we abort the current walk.
 335  */
 336 static int walk_page_test(unsigned long start, unsigned long end,
 337                         struct mm_walk *walk)
 338 {
 339         struct vm_area_struct *vma = walk->vma;
 340         const struct mm_walk_ops *ops = walk->ops;
 341
 342         if (ops->test_walk)
 343                 return ops->test_walk(start, end, walk);
 344
 345         /*
 346          * vma(VM_PFNMAP) doesn't have any valid struct pages behind VM_PFNMAP
 347          * range, so we don't walk over it as we do for normal vmas. However,
 348          * Some callers are interested in handling hole range and they don't
 349          * want to just ignore any single address range. Such users certainly
 350          * define their ->pte_hole() callbacks, so let's delegate them to handle
 351          * vma(VM_PFNMAP).
 352          */
 353         if (vma->vm_flags & VM_PFNMAP) {
 354                 int err = 1;
 355                 if (ops->pte_hole)
 356                         err = ops->pte_hole(start, end, -1, walk);
 357                 return err ? err : 1;
 358         }
 359         return 0;
 360 }
 361
 362 static int __walk_page_range(unsigned long start, unsigned long end,
 363                         struct mm_walk *walk)
 364 {
 365         int err = 0;
 366         struct vm_area_struct *vma = walk->vma;
 367         const struct mm_walk_ops *ops = walk->ops;
 368
 369         if (ops->pre_vma) {
 370                 err = ops->pre_vma(start, end, walk);
 371                 if (err)
 372                         return err;
 373         }
 374
 375         if (is_vm_hugetlb_page(vma)) {
 376                 if (ops->hugetlb_entry)
 377                         err = walk_hugetlb_range(start, end, walk);
 378         } else
 379                 err = walk_pgd_range(start, end, walk);
 380
 381         if (ops->post_vma)
 382                 ops->post_vma(walk);
 383
 384         return err;
 385 }
 386
 387 /**
 388  * walk_page_range - walk page table with caller specific callbacks
 389  * @mm:         mm_struct representing the target process of page table walk
 390  * @start:      start address of the virtual address range
 391  * @end:        end address of the virtual address range
 392  * @ops:        operation to call during the walk
 393  * @private:    private data for callbacks' usage
 394  *
 395  * Recursively walk the page table tree of the process represented by @mm
 396  * within the virtual address range [@start, @end). During walking, we can do
 397  * some caller-specific works for each entry, by setting up pmd_entry(),
 398  * pte_entry(), and/or hugetlb_entry(). If you don't set up for some of these
 399  * callbacks, the associated entries/pages are just ignored.
 400  * The return values of these callbacks are commonly defined like below:
 401  *
 402  *  - 0  : succeeded to handle the current entry, and if you don't reach the
 403  *         end address yet, continue to walk.
 404  *  - >0 : succeeded to handle the current entry, and return to the caller
 405  *         with caller specific value.
 406  *  - <0 : failed to handle the current entry, and return to the caller
 407  *         with error code.
 408  *
 409  * Before starting to walk page table, some callers want to check whether
 410  * they really want to walk over the current vma, typically by checking
 411  * its vm_flags. walk_page_test() and @ops->test_walk() are used for this
 412  * purpose.
 413  *
 414  * If operations need to be staged before and committed after a vma is walked,
 415  * there are two callbacks, pre_vma() and post_vma(). Note that post_vma(),
 416  * since it is intended to handle commit-type operations, can't return any
 417  * errors.
 418  *
 419  * struct mm_walk keeps current values of some common data like vma and pmd,
 420  * which are useful for the access from callbacks. If you want to pass some
 421  * caller-specific data to callbacks, @private should be helpful.
 422  *
 423  * Locking:
 424  *   Callers of walk_page_range() and walk_page_vma() should hold @mm->mmap_lock,
 425  *   because these function traverse vma list and/or access to vma's data.
 426  */
 427 int walk_page_range(struct mm_struct *mm, unsigned long start,
 428                 unsigned long end, const struct mm_walk_ops *ops,
 429                 void *private)
 430 {
 431         int err = 0;
 432         unsigned long next;
 433         struct vm_area_struct *vma;
 434         struct mm_walk walk = {
 435                 .ops            = ops,
 436                 .mm             = mm,
 437                 .private        = private,
 438         };
 439
 440         if (start >= end)
 441                 return -EINVAL;
 442
 443         if (!walk.mm)
 444                 return -EINVAL;
 445
 446         mmap_assert_locked(walk.mm);
 447
 448         vma = find_vma(walk.mm, start);
 449         do {
 450                 if (!vma) { /* after the last vma */
 451                         walk.vma = NULL;
 452                         next = end;
 453                         if (ops->pte_hole)
 454                                 err = ops->pte_hole(start, next, -1, &walk);
 455                 } else if (start < vma->vm_start) { /* outside vma */
 456                         walk.vma = NULL;
 457                         next = min(end, vma->vm_start);
 458                         if (ops->pte_hole)
 459                                 err = ops->pte_hole(start, next, -1, &walk);
 460                 } else { /* inside vma */
 461                         walk.vma = vma;
 462                         next = min(end, vma->vm_end);
 463                         vma = vma->vm_next;
 464
 465                         err = walk_page_test(start, next, &walk);
 466                         if (err > 0) {
 467                                 /*
 468                                  * positive return values are purely for
 469                                  * controlling the pagewalk, so should never
 470                                  * be passed to the callers.
 471                                  */
 472                                 err = 0;
 473                                 continue;
 474                         }
 475                         if (err < 0)
 476                                 break;
 477                         err = __walk_page_range(start, next, &walk);
 478                 }
 479                 if (err)
 480                         break;
 481         } while (start = next, start < end);
 482         return err;
 483 }
 484
 485 /*
 486  * Similar to walk_page_range() but can walk any page tables even if they are
 487  * not backed by VMAs. Because 'unusual' entries may be walked this function
 488  * will also not lock the PTEs for the pte_entry() callback. This is useful for
 489  * walking the kernel pages tables or page tables for firmware.
 490  */
 491 int walk_page_range_novma(struct mm_struct *mm, unsigned long start,
 492                           unsigned long end, const struct mm_walk_ops *ops,
 493                           pgd_t *pgd,
 494                           void *private)
 495 {
 496         struct mm_walk walk = {
 497                 .ops            = ops,
 498                 .mm             = mm,
 499                 .pgd            = pgd,
 500                 .private        = private,
 501                 .no_vma         = true
 502         };
 503
 504         if (start >= end || !walk.mm)
 505                 return -EINVAL;
 506
 507         mmap_assert_write_locked(walk.mm);
 508
 509         return walk_pgd_range(start, end, &walk);
 510 }
 511
 512 int walk_page_vma(struct vm_area_struct *vma, const struct mm_walk_ops *ops,
 513                 void *private)
 514 {
 515         struct mm_walk walk = {
 516                 .ops            = ops,
 517                 .mm             = vma->vm_mm,
 518                 .vma            = vma,
 519                 .private        = private,
 520         };
 521         int err;
 522
 523         if (!walk.mm)
 524                 return -EINVAL;
 525
 526         mmap_assert_locked(walk.mm);
 527
 528         err = walk_page_test(vma->vm_start, vma->vm_end, &walk);
 529         if (err > 0)
 530                 return 0;
 531         if (err < 0)
 532                 return err;
 533         return __walk_page_range(vma->vm_start, vma->vm_end, &walk);
 534 }
 535
 536 /**
 537  * walk_page_mapping - walk all memory areas mapped into a struct address_space.
 538  * @mapping: Pointer to the struct address_space
 539  * @first_index: First page offset in the address_space
 540  * @nr: Number of incremental page offsets to cover
 541  * @ops:        operation to call during the walk
 542  * @private:    private data for callbacks' usage
 543  *
 544  * This function walks all memory areas mapped into a struct address_space.
 545  * The walk is limited to only the given page-size index range, but if
 546  * the index boundaries cross a huge page-table entry, that entry will be
 547  * included.
 548  *
 549  * Also see walk_page_range() for additional information.
 550  *
 551  * Locking:
 552  *   This function can't require that the struct mm_struct::mmap_lock is held,
 553  *   since @mapping may be mapped by multiple processes. Instead
 554  *   @mapping->i_mmap_rwsem must be held. This might have implications in the
 555  *   callbacks, and it's up tho the caller to ensure that the
 556  *   struct mm_struct::mmap_lock is not needed.
 557  *
 558  *   Also this means that a caller can't rely on the struct
 559  *   vm_area_struct::vm_flags to be constant across a call,
 560  *   except for immutable flags. Callers requiring this shouldn't use
 561  *   this function.
 562  *
 563  * Return: 0 on success, negative error code on failure, positive number on
 564  * caller defined premature termination.
 565  */
 566 int walk_page_mapping(struct address_space *mapping, pgoff_t first_index,
 567                       pgoff_t nr, const struct mm_walk_ops *ops,
 568                       void *private)
 569 {
 570         struct mm_walk walk = {
 571                 .ops            = ops,
 572                 .private        = private,
 573         };
 574         struct vm_area_struct *vma;
 575         pgoff_t vba, vea, cba, cea;
 576         unsigned long start_addr, end_addr;
 577         int err = 0;
 578
 579         lockdep_assert_held(&mapping->i_mmap_rwsem);
 580         vma_interval_tree_foreach(vma, &mapping->i_mmap, first_index,
 581                                   first_index + nr - 1) {
 582                 /* Clip to the vma */
 583                 vba = vma->vm_pgoff;
 584                 vea = vba + vma_pages(vma);
 585                 cba = first_index;
 586                 cba = max(cba, vba);
 587                 cea = first_index + nr;
 588                 cea = min(cea, vea);
 589
 590                 start_addr = ((cba - vba) << PAGE_SHIFT) + vma->vm_start;
 591                 end_addr = ((cea - vba) << PAGE_SHIFT) + vma->vm_start;
 592                 if (start_addr >= end_addr)
 593                         continue;
 594
 595                 walk.vma = vma;
 596                 walk.mm = vma->vm_mm;
 597
 598                 err = walk_page_test(vma->vm_start, vma->vm_end, &walk);
 599                 if (err > 0) {
 600                         err = 0;
 601                         break;
 602                 } else if (err < 0)
 603                         break;
 604
 605                 err = __walk_page_range(start_addr, end_addr, &walk);
 606                 if (err)
 607                         break;
 608         }
 609
 610         return err;
 611 }