]> git.proxmox.com Git - mirror_ubuntu-eoan-kernel.git/blame - fs/dax.c
Merge tag 'gvt-fixes-2018-09-10' of https://github.com/intel/gvt-linux into drm-intel...
[mirror_ubuntu-eoan-kernel.git] / fs / dax.c
CommitLineData
d475c634
MW
1/*
2 * fs/dax.c - Direct Access filesystem code
3 * Copyright (c) 2013-2014 Intel Corporation
4 * Author: Matthew Wilcox <matthew.r.wilcox@intel.com>
5 * Author: Ross Zwisler <ross.zwisler@linux.intel.com>
6 *
7 * This program is free software; you can redistribute it and/or modify it
8 * under the terms and conditions of the GNU General Public License,
9 * version 2, as published by the Free Software Foundation.
10 *
11 * This program is distributed in the hope it will be useful, but WITHOUT
12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
14 * more details.
15 */
16
17#include <linux/atomic.h>
18#include <linux/blkdev.h>
19#include <linux/buffer_head.h>
d77e92e2 20#include <linux/dax.h>
d475c634
MW
21#include <linux/fs.h>
22#include <linux/genhd.h>
f7ca90b1
MW
23#include <linux/highmem.h>
24#include <linux/memcontrol.h>
25#include <linux/mm.h>
d475c634 26#include <linux/mutex.h>
9973c98e 27#include <linux/pagevec.h>
289c6aed 28#include <linux/sched.h>
f361bf4a 29#include <linux/sched/signal.h>
d475c634 30#include <linux/uio.h>
f7ca90b1 31#include <linux/vmstat.h>
34c0fd54 32#include <linux/pfn_t.h>
0e749e54 33#include <linux/sizes.h>
4b4bb46d 34#include <linux/mmu_notifier.h>
a254e568
CH
35#include <linux/iomap.h>
36#include "internal.h"
d475c634 37
282a8e03
RZ
38#define CREATE_TRACE_POINTS
39#include <trace/events/fs_dax.h>
40
ac401cc7
JK
41/* We choose 4096 entries - same as per-zone page wait tables */
42#define DAX_WAIT_TABLE_BITS 12
43#define DAX_WAIT_TABLE_ENTRIES (1 << DAX_WAIT_TABLE_BITS)
44
917f3452
RZ
45/* The 'colour' (ie low bits) within a PMD of a page offset. */
46#define PG_PMD_COLOUR ((PMD_SIZE >> PAGE_SHIFT) - 1)
977fbdcd 47#define PG_PMD_NR (PMD_SIZE >> PAGE_SHIFT)
917f3452 48
ce95ab0f 49static wait_queue_head_t wait_table[DAX_WAIT_TABLE_ENTRIES];
ac401cc7
JK
50
51static int __init init_dax_wait_table(void)
52{
53 int i;
54
55 for (i = 0; i < DAX_WAIT_TABLE_ENTRIES; i++)
56 init_waitqueue_head(wait_table + i);
57 return 0;
58}
59fs_initcall(init_dax_wait_table);
60
527b19d0
RZ
61/*
62 * We use lowest available bit in exceptional entry for locking, one bit for
63 * the entry size (PMD) and two more to tell us if the entry is a zero page or
64 * an empty entry that is just used for locking. In total four special bits.
65 *
66 * If the PMD bit isn't set the entry has size PAGE_SIZE, and if the ZERO_PAGE
67 * and EMPTY bits aren't set the entry is a normal DAX entry with a filesystem
68 * block allocation.
69 */
70#define RADIX_DAX_SHIFT (RADIX_TREE_EXCEPTIONAL_SHIFT + 4)
71#define RADIX_DAX_ENTRY_LOCK (1 << RADIX_TREE_EXCEPTIONAL_SHIFT)
72#define RADIX_DAX_PMD (1 << (RADIX_TREE_EXCEPTIONAL_SHIFT + 1))
73#define RADIX_DAX_ZERO_PAGE (1 << (RADIX_TREE_EXCEPTIONAL_SHIFT + 2))
74#define RADIX_DAX_EMPTY (1 << (RADIX_TREE_EXCEPTIONAL_SHIFT + 3))
75
3fe0791c 76static unsigned long dax_radix_pfn(void *entry)
527b19d0
RZ
77{
78 return (unsigned long)entry >> RADIX_DAX_SHIFT;
79}
80
3fe0791c 81static void *dax_radix_locked_entry(unsigned long pfn, unsigned long flags)
527b19d0
RZ
82{
83 return (void *)(RADIX_TREE_EXCEPTIONAL_ENTRY | flags |
3fe0791c 84 (pfn << RADIX_DAX_SHIFT) | RADIX_DAX_ENTRY_LOCK);
527b19d0
RZ
85}
86
87static unsigned int dax_radix_order(void *entry)
88{
89 if ((unsigned long)entry & RADIX_DAX_PMD)
90 return PMD_SHIFT - PAGE_SHIFT;
91 return 0;
92}
93
642261ac 94static int dax_is_pmd_entry(void *entry)
d1a5f2b4 95{
642261ac 96 return (unsigned long)entry & RADIX_DAX_PMD;
d1a5f2b4
DW
97}
98
642261ac 99static int dax_is_pte_entry(void *entry)
d475c634 100{
642261ac 101 return !((unsigned long)entry & RADIX_DAX_PMD);
d475c634
MW
102}
103
642261ac 104static int dax_is_zero_entry(void *entry)
d475c634 105{
91d25ba8 106 return (unsigned long)entry & RADIX_DAX_ZERO_PAGE;
d475c634
MW
107}
108
642261ac 109static int dax_is_empty_entry(void *entry)
b2e0d162 110{
642261ac 111 return (unsigned long)entry & RADIX_DAX_EMPTY;
b2e0d162
DW
112}
113
ac401cc7
JK
114/*
115 * DAX radix tree locking
116 */
117struct exceptional_entry_key {
118 struct address_space *mapping;
63e95b5c 119 pgoff_t entry_start;
ac401cc7
JK
120};
121
122struct wait_exceptional_entry_queue {
ac6424b9 123 wait_queue_entry_t wait;
ac401cc7
JK
124 struct exceptional_entry_key key;
125};
126
63e95b5c
RZ
127static wait_queue_head_t *dax_entry_waitqueue(struct address_space *mapping,
128 pgoff_t index, void *entry, struct exceptional_entry_key *key)
129{
130 unsigned long hash;
131
132 /*
133 * If 'entry' is a PMD, align the 'index' that we use for the wait
134 * queue to the start of that PMD. This ensures that all offsets in
135 * the range covered by the PMD map to the same bit lock.
136 */
642261ac 137 if (dax_is_pmd_entry(entry))
917f3452 138 index &= ~PG_PMD_COLOUR;
63e95b5c
RZ
139
140 key->mapping = mapping;
141 key->entry_start = index;
142
143 hash = hash_long((unsigned long)mapping ^ index, DAX_WAIT_TABLE_BITS);
144 return wait_table + hash;
145}
146
ac6424b9 147static int wake_exceptional_entry_func(wait_queue_entry_t *wait, unsigned int mode,
ac401cc7
JK
148 int sync, void *keyp)
149{
150 struct exceptional_entry_key *key = keyp;
151 struct wait_exceptional_entry_queue *ewait =
152 container_of(wait, struct wait_exceptional_entry_queue, wait);
153
154 if (key->mapping != ewait->key.mapping ||
63e95b5c 155 key->entry_start != ewait->key.entry_start)
ac401cc7
JK
156 return 0;
157 return autoremove_wake_function(wait, mode, sync, NULL);
158}
159
e30331ff 160/*
b93b0163
MW
161 * @entry may no longer be the entry at the index in the mapping.
162 * The important information it's conveying is whether the entry at
163 * this index used to be a PMD entry.
e30331ff 164 */
d01ad197 165static void dax_wake_mapping_entry_waiter(struct address_space *mapping,
e30331ff
RZ
166 pgoff_t index, void *entry, bool wake_all)
167{
168 struct exceptional_entry_key key;
169 wait_queue_head_t *wq;
170
171 wq = dax_entry_waitqueue(mapping, index, entry, &key);
172
173 /*
174 * Checking for locked entry and prepare_to_wait_exclusive() happens
b93b0163 175 * under the i_pages lock, ditto for entry handling in our callers.
e30331ff
RZ
176 * So at this point all tasks that could have seen our entry locked
177 * must be in the waitqueue and the following check will see them.
178 */
179 if (waitqueue_active(wq))
180 __wake_up(wq, TASK_NORMAL, wake_all ? 0 : 1, &key);
181}
182
ac401cc7 183/*
b93b0163
MW
184 * Check whether the given slot is locked. Must be called with the i_pages
185 * lock held.
ac401cc7
JK
186 */
187static inline int slot_locked(struct address_space *mapping, void **slot)
188{
189 unsigned long entry = (unsigned long)
b93b0163 190 radix_tree_deref_slot_protected(slot, &mapping->i_pages.xa_lock);
ac401cc7
JK
191 return entry & RADIX_DAX_ENTRY_LOCK;
192}
193
194/*
b93b0163 195 * Mark the given slot as locked. Must be called with the i_pages lock held.
ac401cc7
JK
196 */
197static inline void *lock_slot(struct address_space *mapping, void **slot)
198{
199 unsigned long entry = (unsigned long)
b93b0163 200 radix_tree_deref_slot_protected(slot, &mapping->i_pages.xa_lock);
ac401cc7
JK
201
202 entry |= RADIX_DAX_ENTRY_LOCK;
b93b0163 203 radix_tree_replace_slot(&mapping->i_pages, slot, (void *)entry);
ac401cc7
JK
204 return (void *)entry;
205}
206
207/*
b93b0163 208 * Mark the given slot as unlocked. Must be called with the i_pages lock held.
ac401cc7
JK
209 */
210static inline void *unlock_slot(struct address_space *mapping, void **slot)
211{
212 unsigned long entry = (unsigned long)
b93b0163 213 radix_tree_deref_slot_protected(slot, &mapping->i_pages.xa_lock);
ac401cc7
JK
214
215 entry &= ~(unsigned long)RADIX_DAX_ENTRY_LOCK;
b93b0163 216 radix_tree_replace_slot(&mapping->i_pages, slot, (void *)entry);
ac401cc7
JK
217 return (void *)entry;
218}
219
220/*
221 * Lookup entry in radix tree, wait for it to become unlocked if it is
222 * exceptional entry and return it. The caller must call
223 * put_unlocked_mapping_entry() when he decided not to lock the entry or
224 * put_locked_mapping_entry() when he locked the entry and now wants to
225 * unlock it.
226 *
b93b0163 227 * Must be called with the i_pages lock held.
ac401cc7 228 */
c2a7d2a1
DW
229static void *__get_unlocked_mapping_entry(struct address_space *mapping,
230 pgoff_t index, void ***slotp, bool (*wait_fn)(void))
ac401cc7 231{
e3ad61c6 232 void *entry, **slot;
ac401cc7 233 struct wait_exceptional_entry_queue ewait;
63e95b5c 234 wait_queue_head_t *wq;
ac401cc7
JK
235
236 init_wait(&ewait.wait);
237 ewait.wait.func = wake_exceptional_entry_func;
ac401cc7
JK
238
239 for (;;) {
c2a7d2a1
DW
240 bool revalidate;
241
b93b0163 242 entry = __radix_tree_lookup(&mapping->i_pages, index, NULL,
ac401cc7 243 &slot);
91d25ba8
RZ
244 if (!entry ||
245 WARN_ON_ONCE(!radix_tree_exceptional_entry(entry)) ||
ac401cc7
JK
246 !slot_locked(mapping, slot)) {
247 if (slotp)
248 *slotp = slot;
e3ad61c6 249 return entry;
ac401cc7 250 }
63e95b5c
RZ
251
252 wq = dax_entry_waitqueue(mapping, index, entry, &ewait.key);
ac401cc7
JK
253 prepare_to_wait_exclusive(wq, &ewait.wait,
254 TASK_UNINTERRUPTIBLE);
b93b0163 255 xa_unlock_irq(&mapping->i_pages);
c2a7d2a1 256 revalidate = wait_fn();
ac401cc7 257 finish_wait(wq, &ewait.wait);
b93b0163 258 xa_lock_irq(&mapping->i_pages);
c2a7d2a1
DW
259 if (revalidate)
260 return ERR_PTR(-EAGAIN);
ac401cc7
JK
261 }
262}
263
c2a7d2a1
DW
264static bool entry_wait(void)
265{
266 schedule();
267 /*
268 * Never return an ERR_PTR() from
269 * __get_unlocked_mapping_entry(), just keep looping.
270 */
271 return false;
272}
273
274static void *get_unlocked_mapping_entry(struct address_space *mapping,
275 pgoff_t index, void ***slotp)
276{
277 return __get_unlocked_mapping_entry(mapping, index, slotp, entry_wait);
278}
279
280static void unlock_mapping_entry(struct address_space *mapping, pgoff_t index)
b1aa812b
JK
281{
282 void *entry, **slot;
283
b93b0163
MW
284 xa_lock_irq(&mapping->i_pages);
285 entry = __radix_tree_lookup(&mapping->i_pages, index, NULL, &slot);
b1aa812b
JK
286 if (WARN_ON_ONCE(!entry || !radix_tree_exceptional_entry(entry) ||
287 !slot_locked(mapping, slot))) {
b93b0163 288 xa_unlock_irq(&mapping->i_pages);
b1aa812b
JK
289 return;
290 }
291 unlock_slot(mapping, slot);
b93b0163 292 xa_unlock_irq(&mapping->i_pages);
b1aa812b
JK
293 dax_wake_mapping_entry_waiter(mapping, index, entry, false);
294}
295
422476c4 296static void put_locked_mapping_entry(struct address_space *mapping,
91d25ba8 297 pgoff_t index)
422476c4 298{
c2a7d2a1 299 unlock_mapping_entry(mapping, index);
422476c4
RZ
300}
301
302/*
303 * Called when we are done with radix tree entry we looked up via
304 * get_unlocked_mapping_entry() and which we didn't lock in the end.
305 */
306static void put_unlocked_mapping_entry(struct address_space *mapping,
307 pgoff_t index, void *entry)
308{
91d25ba8 309 if (!entry)
422476c4
RZ
310 return;
311
312 /* We have to wake up next waiter for the radix tree entry lock */
313 dax_wake_mapping_entry_waiter(mapping, index, entry, false);
314}
315
d2c997c0
DW
316static unsigned long dax_entry_size(void *entry)
317{
318 if (dax_is_zero_entry(entry))
319 return 0;
320 else if (dax_is_empty_entry(entry))
321 return 0;
322 else if (dax_is_pmd_entry(entry))
323 return PMD_SIZE;
324 else
325 return PAGE_SIZE;
326}
327
328static unsigned long dax_radix_end_pfn(void *entry)
329{
330 return dax_radix_pfn(entry) + dax_entry_size(entry) / PAGE_SIZE;
331}
332
333/*
334 * Iterate through all mapped pfns represented by an entry, i.e. skip
335 * 'empty' and 'zero' entries.
336 */
337#define for_each_mapped_pfn(entry, pfn) \
338 for (pfn = dax_radix_pfn(entry); \
339 pfn < dax_radix_end_pfn(entry); pfn++)
340
73449daf
DW
341/*
342 * TODO: for reflink+dax we need a way to associate a single page with
343 * multiple address_space instances at different linear_page_index()
344 * offsets.
345 */
346static void dax_associate_entry(void *entry, struct address_space *mapping,
347 struct vm_area_struct *vma, unsigned long address)
d2c997c0 348{
73449daf
DW
349 unsigned long size = dax_entry_size(entry), pfn, index;
350 int i = 0;
d2c997c0
DW
351
352 if (IS_ENABLED(CONFIG_FS_DAX_LIMITED))
353 return;
354
73449daf 355 index = linear_page_index(vma, address & ~(size - 1));
d2c997c0
DW
356 for_each_mapped_pfn(entry, pfn) {
357 struct page *page = pfn_to_page(pfn);
358
359 WARN_ON_ONCE(page->mapping);
360 page->mapping = mapping;
73449daf 361 page->index = index + i++;
d2c997c0
DW
362 }
363}
364
365static void dax_disassociate_entry(void *entry, struct address_space *mapping,
366 bool trunc)
367{
368 unsigned long pfn;
369
370 if (IS_ENABLED(CONFIG_FS_DAX_LIMITED))
371 return;
372
373 for_each_mapped_pfn(entry, pfn) {
374 struct page *page = pfn_to_page(pfn);
375
376 WARN_ON_ONCE(trunc && page_ref_count(page) > 1);
377 WARN_ON_ONCE(page->mapping && page->mapping != mapping);
378 page->mapping = NULL;
73449daf 379 page->index = 0;
d2c997c0
DW
380 }
381}
382
5fac7408
DW
383static struct page *dax_busy_page(void *entry)
384{
385 unsigned long pfn;
386
387 for_each_mapped_pfn(entry, pfn) {
388 struct page *page = pfn_to_page(pfn);
389
390 if (page_ref_count(page) > 1)
391 return page;
392 }
393 return NULL;
394}
395
c2a7d2a1
DW
396static bool entry_wait_revalidate(void)
397{
398 rcu_read_unlock();
399 schedule();
400 rcu_read_lock();
401
402 /*
403 * Tell __get_unlocked_mapping_entry() to take a break, we need
404 * to revalidate page->mapping after dropping locks
405 */
406 return true;
407}
408
409bool dax_lock_mapping_entry(struct page *page)
410{
411 pgoff_t index;
412 struct inode *inode;
413 bool did_lock = false;
414 void *entry = NULL, **slot;
415 struct address_space *mapping;
416
417 rcu_read_lock();
418 for (;;) {
419 mapping = READ_ONCE(page->mapping);
420
421 if (!dax_mapping(mapping))
422 break;
423
424 /*
425 * In the device-dax case there's no need to lock, a
426 * struct dev_pagemap pin is sufficient to keep the
427 * inode alive, and we assume we have dev_pagemap pin
428 * otherwise we would not have a valid pfn_to_page()
429 * translation.
430 */
431 inode = mapping->host;
432 if (S_ISCHR(inode->i_mode)) {
433 did_lock = true;
434 break;
435 }
436
437 xa_lock_irq(&mapping->i_pages);
438 if (mapping != page->mapping) {
439 xa_unlock_irq(&mapping->i_pages);
440 continue;
441 }
442 index = page->index;
443
444 entry = __get_unlocked_mapping_entry(mapping, index, &slot,
445 entry_wait_revalidate);
446 if (!entry) {
447 xa_unlock_irq(&mapping->i_pages);
448 break;
449 } else if (IS_ERR(entry)) {
450 WARN_ON_ONCE(PTR_ERR(entry) != -EAGAIN);
451 continue;
452 }
453 lock_slot(mapping, slot);
454 did_lock = true;
455 xa_unlock_irq(&mapping->i_pages);
456 break;
457 }
458 rcu_read_unlock();
459
460 return did_lock;
461}
462
463void dax_unlock_mapping_entry(struct page *page)
464{
465 struct address_space *mapping = page->mapping;
466 struct inode *inode = mapping->host;
467
468 if (S_ISCHR(inode->i_mode))
469 return;
470
471 unlock_mapping_entry(mapping, page->index);
472}
473
ac401cc7 474/*
91d25ba8
RZ
475 * Find radix tree entry at given index. If it points to an exceptional entry,
476 * return it with the radix tree entry locked. If the radix tree doesn't
477 * contain given index, create an empty exceptional entry for the index and
478 * return with it locked.
ac401cc7 479 *
642261ac
RZ
480 * When requesting an entry with size RADIX_DAX_PMD, grab_mapping_entry() will
481 * either return that locked entry or will return an error. This error will
91d25ba8
RZ
482 * happen if there are any 4k entries within the 2MiB range that we are
483 * requesting.
642261ac
RZ
484 *
485 * We always favor 4k entries over 2MiB entries. There isn't a flow where we
486 * evict 4k entries in order to 'upgrade' them to a 2MiB entry. A 2MiB
487 * insertion will fail if it finds any 4k entries already in the tree, and a
488 * 4k insertion will cause an existing 2MiB entry to be unmapped and
489 * downgraded to 4k entries. This happens for both 2MiB huge zero pages as
490 * well as 2MiB empty entries.
491 *
492 * The exception to this downgrade path is for 2MiB DAX PMD entries that have
493 * real storage backing them. We will leave these real 2MiB DAX entries in
494 * the tree, and PTE writes will simply dirty the entire 2MiB DAX entry.
495 *
ac401cc7
JK
496 * Note: Unlike filemap_fault() we don't honor FAULT_FLAG_RETRY flags. For
497 * persistent memory the benefit is doubtful. We can add that later if we can
498 * show it helps.
499 */
642261ac
RZ
500static void *grab_mapping_entry(struct address_space *mapping, pgoff_t index,
501 unsigned long size_flag)
ac401cc7 502{
642261ac 503 bool pmd_downgrade = false; /* splitting 2MiB entry into 4k entries? */
e3ad61c6 504 void *entry, **slot;
ac401cc7
JK
505
506restart:
b93b0163 507 xa_lock_irq(&mapping->i_pages);
e3ad61c6 508 entry = get_unlocked_mapping_entry(mapping, index, &slot);
642261ac 509
91d25ba8
RZ
510 if (WARN_ON_ONCE(entry && !radix_tree_exceptional_entry(entry))) {
511 entry = ERR_PTR(-EIO);
512 goto out_unlock;
513 }
514
642261ac
RZ
515 if (entry) {
516 if (size_flag & RADIX_DAX_PMD) {
91d25ba8 517 if (dax_is_pte_entry(entry)) {
642261ac
RZ
518 put_unlocked_mapping_entry(mapping, index,
519 entry);
520 entry = ERR_PTR(-EEXIST);
521 goto out_unlock;
522 }
523 } else { /* trying to grab a PTE entry */
91d25ba8 524 if (dax_is_pmd_entry(entry) &&
642261ac
RZ
525 (dax_is_zero_entry(entry) ||
526 dax_is_empty_entry(entry))) {
527 pmd_downgrade = true;
528 }
529 }
530 }
531
ac401cc7 532 /* No entry for given index? Make sure radix tree is big enough. */
642261ac 533 if (!entry || pmd_downgrade) {
ac401cc7
JK
534 int err;
535
642261ac
RZ
536 if (pmd_downgrade) {
537 /*
538 * Make sure 'entry' remains valid while we drop
b93b0163 539 * the i_pages lock.
642261ac
RZ
540 */
541 entry = lock_slot(mapping, slot);
542 }
543
b93b0163 544 xa_unlock_irq(&mapping->i_pages);
642261ac
RZ
545 /*
546 * Besides huge zero pages the only other thing that gets
547 * downgraded are empty entries which don't need to be
548 * unmapped.
549 */
550 if (pmd_downgrade && dax_is_zero_entry(entry))
977fbdcd
MW
551 unmap_mapping_pages(mapping, index & ~PG_PMD_COLOUR,
552 PG_PMD_NR, false);
642261ac 553
ac401cc7
JK
554 err = radix_tree_preload(
555 mapping_gfp_mask(mapping) & ~__GFP_HIGHMEM);
0cb80b48
JK
556 if (err) {
557 if (pmd_downgrade)
91d25ba8 558 put_locked_mapping_entry(mapping, index);
ac401cc7 559 return ERR_PTR(err);
0cb80b48 560 }
b93b0163 561 xa_lock_irq(&mapping->i_pages);
642261ac 562
e11f8b7b
RZ
563 if (!entry) {
564 /*
b93b0163 565 * We needed to drop the i_pages lock while calling
e11f8b7b
RZ
566 * radix_tree_preload() and we didn't have an entry to
567 * lock. See if another thread inserted an entry at
568 * our index during this time.
569 */
b93b0163 570 entry = __radix_tree_lookup(&mapping->i_pages, index,
e11f8b7b
RZ
571 NULL, &slot);
572 if (entry) {
573 radix_tree_preload_end();
b93b0163 574 xa_unlock_irq(&mapping->i_pages);
e11f8b7b
RZ
575 goto restart;
576 }
577 }
578
642261ac 579 if (pmd_downgrade) {
d2c997c0 580 dax_disassociate_entry(entry, mapping, false);
b93b0163 581 radix_tree_delete(&mapping->i_pages, index);
642261ac
RZ
582 mapping->nrexceptional--;
583 dax_wake_mapping_entry_waiter(mapping, index, entry,
584 true);
585 }
586
587 entry = dax_radix_locked_entry(0, size_flag | RADIX_DAX_EMPTY);
588
b93b0163 589 err = __radix_tree_insert(&mapping->i_pages, index,
642261ac 590 dax_radix_order(entry), entry);
ac401cc7
JK
591 radix_tree_preload_end();
592 if (err) {
b93b0163 593 xa_unlock_irq(&mapping->i_pages);
642261ac 594 /*
e11f8b7b
RZ
595 * Our insertion of a DAX entry failed, most likely
596 * because we were inserting a PMD entry and it
597 * collided with a PTE sized entry at a different
598 * index in the PMD range. We haven't inserted
599 * anything into the radix tree and have no waiters to
600 * wake.
642261ac 601 */
ac401cc7
JK
602 return ERR_PTR(err);
603 }
604 /* Good, we have inserted empty locked entry into the tree. */
605 mapping->nrexceptional++;
b93b0163 606 xa_unlock_irq(&mapping->i_pages);
e3ad61c6 607 return entry;
ac401cc7 608 }
e3ad61c6 609 entry = lock_slot(mapping, slot);
642261ac 610 out_unlock:
b93b0163 611 xa_unlock_irq(&mapping->i_pages);
e3ad61c6 612 return entry;
ac401cc7
JK
613}
614
5fac7408
DW
615/**
616 * dax_layout_busy_page - find first pinned page in @mapping
617 * @mapping: address space to scan for a page with ref count > 1
618 *
619 * DAX requires ZONE_DEVICE mapped pages. These pages are never
620 * 'onlined' to the page allocator so they are considered idle when
621 * page->count == 1. A filesystem uses this interface to determine if
622 * any page in the mapping is busy, i.e. for DMA, or other
623 * get_user_pages() usages.
624 *
625 * It is expected that the filesystem is holding locks to block the
626 * establishment of new mappings in this address_space. I.e. it expects
627 * to be able to run unmap_mapping_range() and subsequently not race
628 * mapping_mapped() becoming true.
629 */
630struct page *dax_layout_busy_page(struct address_space *mapping)
631{
632 pgoff_t indices[PAGEVEC_SIZE];
633 struct page *page = NULL;
634 struct pagevec pvec;
635 pgoff_t index, end;
636 unsigned i;
637
638 /*
639 * In the 'limited' case get_user_pages() for dax is disabled.
640 */
641 if (IS_ENABLED(CONFIG_FS_DAX_LIMITED))
642 return NULL;
643
644 if (!dax_mapping(mapping) || !mapping_mapped(mapping))
645 return NULL;
646
647 pagevec_init(&pvec);
648 index = 0;
649 end = -1;
650
651 /*
652 * If we race get_user_pages_fast() here either we'll see the
653 * elevated page count in the pagevec_lookup and wait, or
654 * get_user_pages_fast() will see that the page it took a reference
655 * against is no longer mapped in the page tables and bail to the
656 * get_user_pages() slow path. The slow path is protected by
657 * pte_lock() and pmd_lock(). New references are not taken without
658 * holding those locks, and unmap_mapping_range() will not zero the
659 * pte or pmd without holding the respective lock, so we are
660 * guaranteed to either see new references or prevent new
661 * references from being established.
662 */
663 unmap_mapping_range(mapping, 0, 0, 1);
664
665 while (index < end && pagevec_lookup_entries(&pvec, mapping, index,
666 min(end - index, (pgoff_t)PAGEVEC_SIZE),
667 indices)) {
668 for (i = 0; i < pagevec_count(&pvec); i++) {
669 struct page *pvec_ent = pvec.pages[i];
670 void *entry;
671
672 index = indices[i];
673 if (index >= end)
674 break;
675
cdbf8897
RZ
676 if (WARN_ON_ONCE(
677 !radix_tree_exceptional_entry(pvec_ent)))
5fac7408
DW
678 continue;
679
680 xa_lock_irq(&mapping->i_pages);
681 entry = get_unlocked_mapping_entry(mapping, index, NULL);
682 if (entry)
683 page = dax_busy_page(entry);
684 put_unlocked_mapping_entry(mapping, index, entry);
685 xa_unlock_irq(&mapping->i_pages);
686 if (page)
687 break;
688 }
cdbf8897
RZ
689
690 /*
691 * We don't expect normal struct page entries to exist in our
692 * tree, but we keep these pagevec calls so that this code is
693 * consistent with the common pattern for handling pagevecs
694 * throughout the kernel.
695 */
5fac7408
DW
696 pagevec_remove_exceptionals(&pvec);
697 pagevec_release(&pvec);
698 index++;
699
700 if (page)
701 break;
702 }
703 return page;
704}
705EXPORT_SYMBOL_GPL(dax_layout_busy_page);
706
c6dcf52c
JK
707static int __dax_invalidate_mapping_entry(struct address_space *mapping,
708 pgoff_t index, bool trunc)
709{
710 int ret = 0;
711 void *entry;
b93b0163 712 struct radix_tree_root *pages = &mapping->i_pages;
c6dcf52c 713
b93b0163 714 xa_lock_irq(pages);
c6dcf52c 715 entry = get_unlocked_mapping_entry(mapping, index, NULL);
91d25ba8 716 if (!entry || WARN_ON_ONCE(!radix_tree_exceptional_entry(entry)))
c6dcf52c
JK
717 goto out;
718 if (!trunc &&
b93b0163
MW
719 (radix_tree_tag_get(pages, index, PAGECACHE_TAG_DIRTY) ||
720 radix_tree_tag_get(pages, index, PAGECACHE_TAG_TOWRITE)))
c6dcf52c 721 goto out;
d2c997c0 722 dax_disassociate_entry(entry, mapping, trunc);
b93b0163 723 radix_tree_delete(pages, index);
c6dcf52c
JK
724 mapping->nrexceptional--;
725 ret = 1;
726out:
727 put_unlocked_mapping_entry(mapping, index, entry);
b93b0163 728 xa_unlock_irq(pages);
c6dcf52c
JK
729 return ret;
730}
ac401cc7
JK
731/*
732 * Delete exceptional DAX entry at @index from @mapping. Wait for radix tree
733 * entry to get unlocked before deleting it.
734 */
735int dax_delete_mapping_entry(struct address_space *mapping, pgoff_t index)
736{
c6dcf52c 737 int ret = __dax_invalidate_mapping_entry(mapping, index, true);
ac401cc7 738
ac401cc7
JK
739 /*
740 * This gets called from truncate / punch_hole path. As such, the caller
741 * must hold locks protecting against concurrent modifications of the
742 * radix tree (usually fs-private i_mmap_sem for writing). Since the
743 * caller has seen exceptional entry for this index, we better find it
744 * at that index as well...
745 */
c6dcf52c
JK
746 WARN_ON_ONCE(!ret);
747 return ret;
748}
749
c6dcf52c
JK
750/*
751 * Invalidate exceptional DAX entry if it is clean.
752 */
753int dax_invalidate_mapping_entry_sync(struct address_space *mapping,
754 pgoff_t index)
755{
756 return __dax_invalidate_mapping_entry(mapping, index, false);
ac401cc7
JK
757}
758
cccbce67
DW
759static int copy_user_dax(struct block_device *bdev, struct dax_device *dax_dev,
760 sector_t sector, size_t size, struct page *to,
761 unsigned long vaddr)
f7ca90b1 762{
cccbce67
DW
763 void *vto, *kaddr;
764 pgoff_t pgoff;
cccbce67
DW
765 long rc;
766 int id;
767
768 rc = bdev_dax_pgoff(bdev, sector, size, &pgoff);
769 if (rc)
770 return rc;
771
772 id = dax_read_lock();
86ed913b 773 rc = dax_direct_access(dax_dev, pgoff, PHYS_PFN(size), &kaddr, NULL);
cccbce67
DW
774 if (rc < 0) {
775 dax_read_unlock(id);
776 return rc;
777 }
f7ca90b1 778 vto = kmap_atomic(to);
cccbce67 779 copy_user_page(vto, (void __force *)kaddr, vaddr, to);
f7ca90b1 780 kunmap_atomic(vto);
cccbce67 781 dax_read_unlock(id);
f7ca90b1
MW
782 return 0;
783}
784
642261ac
RZ
785/*
786 * By this point grab_mapping_entry() has ensured that we have a locked entry
787 * of the appropriate size so we don't have to worry about downgrading PMDs to
788 * PTEs. If we happen to be trying to insert a PTE and there is a PMD
789 * already in the tree, we will skip the insertion and just dirty the PMD as
790 * appropriate.
791 */
ac401cc7
JK
792static void *dax_insert_mapping_entry(struct address_space *mapping,
793 struct vm_fault *vmf,
3fe0791c 794 void *entry, pfn_t pfn_t,
f5b7b748 795 unsigned long flags, bool dirty)
9973c98e 796{
b93b0163 797 struct radix_tree_root *pages = &mapping->i_pages;
3fe0791c 798 unsigned long pfn = pfn_t_to_pfn(pfn_t);
ac401cc7 799 pgoff_t index = vmf->pgoff;
3fe0791c 800 void *new_entry;
9973c98e 801
f5b7b748 802 if (dirty)
d2b2a28e 803 __mark_inode_dirty(mapping->host, I_DIRTY_PAGES);
9973c98e 804
91d25ba8
RZ
805 if (dax_is_zero_entry(entry) && !(flags & RADIX_DAX_ZERO_PAGE)) {
806 /* we are replacing a zero page with block mapping */
807 if (dax_is_pmd_entry(entry))
977fbdcd
MW
808 unmap_mapping_pages(mapping, index & ~PG_PMD_COLOUR,
809 PG_PMD_NR, false);
91d25ba8 810 else /* pte entry */
977fbdcd 811 unmap_mapping_pages(mapping, vmf->pgoff, 1, false);
9973c98e
RZ
812 }
813
b93b0163 814 xa_lock_irq(pages);
3fe0791c 815 new_entry = dax_radix_locked_entry(pfn, flags);
d2c997c0
DW
816 if (dax_entry_size(entry) != dax_entry_size(new_entry)) {
817 dax_disassociate_entry(entry, mapping, false);
73449daf 818 dax_associate_entry(new_entry, mapping, vmf->vma, vmf->address);
d2c997c0 819 }
642261ac 820
91d25ba8 821 if (dax_is_zero_entry(entry) || dax_is_empty_entry(entry)) {
642261ac
RZ
822 /*
823 * Only swap our new entry into the radix tree if the current
824 * entry is a zero page or an empty entry. If a normal PTE or
825 * PMD entry is already in the tree, we leave it alone. This
826 * means that if we are trying to insert a PTE and the
827 * existing entry is a PMD, we will just leave the PMD in the
828 * tree and dirty it if necessary.
829 */
f7942430 830 struct radix_tree_node *node;
ac401cc7
JK
831 void **slot;
832 void *ret;
9973c98e 833
b93b0163 834 ret = __radix_tree_lookup(pages, index, &node, &slot);
ac401cc7 835 WARN_ON_ONCE(ret != entry);
b93b0163 836 __radix_tree_replace(pages, node, slot,
c7df8ad2 837 new_entry, NULL);
91d25ba8 838 entry = new_entry;
9973c98e 839 }
91d25ba8 840
f5b7b748 841 if (dirty)
b93b0163 842 radix_tree_tag_set(pages, index, PAGECACHE_TAG_DIRTY);
91d25ba8 843
b93b0163 844 xa_unlock_irq(pages);
91d25ba8 845 return entry;
9973c98e
RZ
846}
847
4b4bb46d
JK
848static inline unsigned long
849pgoff_address(pgoff_t pgoff, struct vm_area_struct *vma)
850{
851 unsigned long address;
852
853 address = vma->vm_start + ((pgoff - vma->vm_pgoff) << PAGE_SHIFT);
854 VM_BUG_ON_VMA(address < vma->vm_start || address >= vma->vm_end, vma);
855 return address;
856}
857
858/* Walk all mappings of a given index of a file and writeprotect them */
859static void dax_mapping_entry_mkclean(struct address_space *mapping,
860 pgoff_t index, unsigned long pfn)
861{
862 struct vm_area_struct *vma;
f729c8c9
RZ
863 pte_t pte, *ptep = NULL;
864 pmd_t *pmdp = NULL;
4b4bb46d 865 spinlock_t *ptl;
4b4bb46d
JK
866
867 i_mmap_lock_read(mapping);
868 vma_interval_tree_foreach(vma, &mapping->i_mmap, index, index) {
a4d1a885 869 unsigned long address, start, end;
4b4bb46d
JK
870
871 cond_resched();
872
873 if (!(vma->vm_flags & VM_SHARED))
874 continue;
875
876 address = pgoff_address(index, vma);
a4d1a885
JG
877
878 /*
879 * Note because we provide start/end to follow_pte_pmd it will
880 * call mmu_notifier_invalidate_range_start() on our behalf
881 * before taking any lock.
882 */
883 if (follow_pte_pmd(vma->vm_mm, address, &start, &end, &ptep, &pmdp, &ptl))
4b4bb46d 884 continue;
4b4bb46d 885
0f10851e
JG
886 /*
887 * No need to call mmu_notifier_invalidate_range() as we are
888 * downgrading page table protection not changing it to point
889 * to a new page.
890 *
ad56b738 891 * See Documentation/vm/mmu_notifier.rst
0f10851e 892 */
f729c8c9
RZ
893 if (pmdp) {
894#ifdef CONFIG_FS_DAX_PMD
895 pmd_t pmd;
896
897 if (pfn != pmd_pfn(*pmdp))
898 goto unlock_pmd;
f6f37321 899 if (!pmd_dirty(*pmdp) && !pmd_write(*pmdp))
f729c8c9
RZ
900 goto unlock_pmd;
901
902 flush_cache_page(vma, address, pfn);
903 pmd = pmdp_huge_clear_flush(vma, address, pmdp);
904 pmd = pmd_wrprotect(pmd);
905 pmd = pmd_mkclean(pmd);
906 set_pmd_at(vma->vm_mm, address, pmdp, pmd);
f729c8c9 907unlock_pmd:
f729c8c9 908#endif
ee190ca6 909 spin_unlock(ptl);
f729c8c9
RZ
910 } else {
911 if (pfn != pte_pfn(*ptep))
912 goto unlock_pte;
913 if (!pte_dirty(*ptep) && !pte_write(*ptep))
914 goto unlock_pte;
915
916 flush_cache_page(vma, address, pfn);
917 pte = ptep_clear_flush(vma, address, ptep);
918 pte = pte_wrprotect(pte);
919 pte = pte_mkclean(pte);
920 set_pte_at(vma->vm_mm, address, ptep, pte);
f729c8c9
RZ
921unlock_pte:
922 pte_unmap_unlock(ptep, ptl);
923 }
4b4bb46d 924
a4d1a885 925 mmu_notifier_invalidate_range_end(vma->vm_mm, start, end);
4b4bb46d
JK
926 }
927 i_mmap_unlock_read(mapping);
928}
929
3fe0791c
DW
930static int dax_writeback_one(struct dax_device *dax_dev,
931 struct address_space *mapping, pgoff_t index, void *entry)
9973c98e 932{
b93b0163 933 struct radix_tree_root *pages = &mapping->i_pages;
3fe0791c
DW
934 void *entry2, **slot;
935 unsigned long pfn;
936 long ret = 0;
cccbce67 937 size_t size;
9973c98e 938
9973c98e 939 /*
a6abc2c0
JK
940 * A page got tagged dirty in DAX mapping? Something is seriously
941 * wrong.
9973c98e 942 */
a6abc2c0
JK
943 if (WARN_ON(!radix_tree_exceptional_entry(entry)))
944 return -EIO;
9973c98e 945
b93b0163 946 xa_lock_irq(pages);
a6abc2c0
JK
947 entry2 = get_unlocked_mapping_entry(mapping, index, &slot);
948 /* Entry got punched out / reallocated? */
91d25ba8 949 if (!entry2 || WARN_ON_ONCE(!radix_tree_exceptional_entry(entry2)))
a6abc2c0
JK
950 goto put_unlocked;
951 /*
952 * Entry got reallocated elsewhere? No need to writeback. We have to
3fe0791c 953 * compare pfns as we must not bail out due to difference in lockbit
a6abc2c0
JK
954 * or entry type.
955 */
3fe0791c 956 if (dax_radix_pfn(entry2) != dax_radix_pfn(entry))
a6abc2c0 957 goto put_unlocked;
642261ac
RZ
958 if (WARN_ON_ONCE(dax_is_empty_entry(entry) ||
959 dax_is_zero_entry(entry))) {
9973c98e 960 ret = -EIO;
a6abc2c0 961 goto put_unlocked;
9973c98e
RZ
962 }
963
a6abc2c0 964 /* Another fsync thread may have already written back this entry */
b93b0163 965 if (!radix_tree_tag_get(pages, index, PAGECACHE_TAG_TOWRITE))
a6abc2c0
JK
966 goto put_unlocked;
967 /* Lock the entry to serialize with page faults */
968 entry = lock_slot(mapping, slot);
969 /*
970 * We can clear the tag now but we have to be careful so that concurrent
971 * dax_writeback_one() calls for the same index cannot finish before we
972 * actually flush the caches. This is achieved as the calls will look
b93b0163
MW
973 * at the entry only under the i_pages lock and once they do that
974 * they will see the entry locked and wait for it to unlock.
a6abc2c0 975 */
b93b0163
MW
976 radix_tree_tag_clear(pages, index, PAGECACHE_TAG_TOWRITE);
977 xa_unlock_irq(pages);
a6abc2c0 978
642261ac
RZ
979 /*
980 * Even if dax_writeback_mapping_range() was given a wbc->range_start
981 * in the middle of a PMD, the 'index' we are given will be aligned to
3fe0791c
DW
982 * the start index of the PMD, as will the pfn we pull from 'entry'.
983 * This allows us to flush for PMD_SIZE and not have to worry about
984 * partial PMD writebacks.
642261ac 985 */
3fe0791c 986 pfn = dax_radix_pfn(entry);
cccbce67
DW
987 size = PAGE_SIZE << dax_radix_order(entry);
988
3fe0791c
DW
989 dax_mapping_entry_mkclean(mapping, index, pfn);
990 dax_flush(dax_dev, page_address(pfn_to_page(pfn)), size);
4b4bb46d
JK
991 /*
992 * After we have flushed the cache, we can clear the dirty tag. There
993 * cannot be new dirty data in the pfn after the flush has completed as
994 * the pfn mappings are writeprotected and fault waits for mapping
995 * entry lock.
996 */
b93b0163
MW
997 xa_lock_irq(pages);
998 radix_tree_tag_clear(pages, index, PAGECACHE_TAG_DIRTY);
999 xa_unlock_irq(pages);
f9bc3a07 1000 trace_dax_writeback_one(mapping->host, index, size >> PAGE_SHIFT);
91d25ba8 1001 put_locked_mapping_entry(mapping, index);
9973c98e
RZ
1002 return ret;
1003
a6abc2c0
JK
1004 put_unlocked:
1005 put_unlocked_mapping_entry(mapping, index, entry2);
b93b0163 1006 xa_unlock_irq(pages);
9973c98e
RZ
1007 return ret;
1008}
1009
1010/*
1011 * Flush the mapping to the persistent domain within the byte range of [start,
1012 * end]. This is required by data integrity operations to ensure file data is
1013 * on persistent storage prior to completion of the operation.
1014 */
7f6d5b52
RZ
1015int dax_writeback_mapping_range(struct address_space *mapping,
1016 struct block_device *bdev, struct writeback_control *wbc)
9973c98e
RZ
1017{
1018 struct inode *inode = mapping->host;
642261ac 1019 pgoff_t start_index, end_index;
9973c98e 1020 pgoff_t indices[PAGEVEC_SIZE];
cccbce67 1021 struct dax_device *dax_dev;
9973c98e
RZ
1022 struct pagevec pvec;
1023 bool done = false;
1024 int i, ret = 0;
9973c98e
RZ
1025
1026 if (WARN_ON_ONCE(inode->i_blkbits != PAGE_SHIFT))
1027 return -EIO;
1028
7f6d5b52
RZ
1029 if (!mapping->nrexceptional || wbc->sync_mode != WB_SYNC_ALL)
1030 return 0;
1031
cccbce67
DW
1032 dax_dev = dax_get_by_host(bdev->bd_disk->disk_name);
1033 if (!dax_dev)
1034 return -EIO;
1035
09cbfeaf
KS
1036 start_index = wbc->range_start >> PAGE_SHIFT;
1037 end_index = wbc->range_end >> PAGE_SHIFT;
9973c98e 1038
d14a3f48
RZ
1039 trace_dax_writeback_range(inode, start_index, end_index);
1040
9973c98e
RZ
1041 tag_pages_for_writeback(mapping, start_index, end_index);
1042
86679820 1043 pagevec_init(&pvec);
9973c98e
RZ
1044 while (!done) {
1045 pvec.nr = find_get_entries_tag(mapping, start_index,
1046 PAGECACHE_TAG_TOWRITE, PAGEVEC_SIZE,
1047 pvec.pages, indices);
1048
1049 if (pvec.nr == 0)
1050 break;
1051
1052 for (i = 0; i < pvec.nr; i++) {
1053 if (indices[i] > end_index) {
1054 done = true;
1055 break;
1056 }
1057
3fe0791c
DW
1058 ret = dax_writeback_one(dax_dev, mapping, indices[i],
1059 pvec.pages[i]);
819ec6b9
JL
1060 if (ret < 0) {
1061 mapping_set_error(mapping, ret);
d14a3f48 1062 goto out;
819ec6b9 1063 }
9973c98e 1064 }
1eb643d0 1065 start_index = indices[pvec.nr - 1] + 1;
9973c98e 1066 }
d14a3f48 1067out:
cccbce67 1068 put_dax(dax_dev);
d14a3f48
RZ
1069 trace_dax_writeback_range_done(inode, start_index, end_index);
1070 return (ret < 0 ? ret : 0);
9973c98e
RZ
1071}
1072EXPORT_SYMBOL_GPL(dax_writeback_mapping_range);
1073
31a6f1a6 1074static sector_t dax_iomap_sector(struct iomap *iomap, loff_t pos)
f7ca90b1 1075{
a3841f94 1076 return (iomap->addr + (pos & PAGE_MASK) - iomap->offset) >> 9;
31a6f1a6
JK
1077}
1078
5e161e40
JK
1079static int dax_iomap_pfn(struct iomap *iomap, loff_t pos, size_t size,
1080 pfn_t *pfnp)
f7ca90b1 1081{
31a6f1a6 1082 const sector_t sector = dax_iomap_sector(iomap, pos);
cccbce67
DW
1083 pgoff_t pgoff;
1084 int id, rc;
5e161e40 1085 long length;
f7ca90b1 1086
5e161e40 1087 rc = bdev_dax_pgoff(iomap->bdev, sector, size, &pgoff);
cccbce67
DW
1088 if (rc)
1089 return rc;
cccbce67 1090 id = dax_read_lock();
5e161e40 1091 length = dax_direct_access(iomap->dax_dev, pgoff, PHYS_PFN(size),
86ed913b 1092 NULL, pfnp);
5e161e40
JK
1093 if (length < 0) {
1094 rc = length;
1095 goto out;
cccbce67 1096 }
5e161e40
JK
1097 rc = -EINVAL;
1098 if (PFN_PHYS(length) < size)
1099 goto out;
1100 if (pfn_t_to_pfn(*pfnp) & (PHYS_PFN(size)-1))
1101 goto out;
1102 /* For larger pages we need devmap */
1103 if (length > 1 && !pfn_t_devmap(*pfnp))
1104 goto out;
1105 rc = 0;
1106out:
cccbce67 1107 dax_read_unlock(id);
5e161e40 1108 return rc;
0e3b210c 1109}
0e3b210c 1110
e30331ff 1111/*
91d25ba8
RZ
1112 * The user has performed a load from a hole in the file. Allocating a new
1113 * page in the file would cause excessive storage usage for workloads with
1114 * sparse files. Instead we insert a read-only mapping of the 4k zero page.
1115 * If this page is ever written to we will re-fault and change the mapping to
1116 * point to real DAX storage instead.
e30331ff 1117 */
ab77dab4 1118static vm_fault_t dax_load_hole(struct address_space *mapping, void *entry,
e30331ff
RZ
1119 struct vm_fault *vmf)
1120{
1121 struct inode *inode = mapping->host;
91d25ba8 1122 unsigned long vaddr = vmf->address;
ab77dab4 1123 vm_fault_t ret = VM_FAULT_NOPAGE;
91d25ba8 1124 struct page *zero_page;
3fe0791c 1125 pfn_t pfn;
e30331ff 1126
91d25ba8
RZ
1127 zero_page = ZERO_PAGE(0);
1128 if (unlikely(!zero_page)) {
e30331ff
RZ
1129 ret = VM_FAULT_OOM;
1130 goto out;
1131 }
1132
3fe0791c 1133 pfn = page_to_pfn_t(zero_page);
cc4a90ac
MW
1134 dax_insert_mapping_entry(mapping, vmf, entry, pfn, RADIX_DAX_ZERO_PAGE,
1135 false);
ab77dab4 1136 ret = vmf_insert_mixed(vmf->vma, vaddr, pfn);
e30331ff
RZ
1137out:
1138 trace_dax_load_hole(inode, vmf, ret);
1139 return ret;
1140}
1141
4b0228fa
VV
1142static bool dax_range_is_aligned(struct block_device *bdev,
1143 unsigned int offset, unsigned int length)
1144{
1145 unsigned short sector_size = bdev_logical_block_size(bdev);
1146
1147 if (!IS_ALIGNED(offset, sector_size))
1148 return false;
1149 if (!IS_ALIGNED(length, sector_size))
1150 return false;
1151
1152 return true;
1153}
1154
cccbce67
DW
1155int __dax_zero_page_range(struct block_device *bdev,
1156 struct dax_device *dax_dev, sector_t sector,
1157 unsigned int offset, unsigned int size)
679c8bd3 1158{
cccbce67
DW
1159 if (dax_range_is_aligned(bdev, offset, size)) {
1160 sector_t start_sector = sector + (offset >> 9);
4b0228fa
VV
1161
1162 return blkdev_issue_zeroout(bdev, start_sector,
53ef7d0e 1163 size >> 9, GFP_NOFS, 0);
4b0228fa 1164 } else {
cccbce67
DW
1165 pgoff_t pgoff;
1166 long rc, id;
1167 void *kaddr;
cccbce67 1168
e84b83b9 1169 rc = bdev_dax_pgoff(bdev, sector, PAGE_SIZE, &pgoff);
cccbce67
DW
1170 if (rc)
1171 return rc;
1172
1173 id = dax_read_lock();
86ed913b 1174 rc = dax_direct_access(dax_dev, pgoff, 1, &kaddr, NULL);
cccbce67
DW
1175 if (rc < 0) {
1176 dax_read_unlock(id);
1177 return rc;
1178 }
81f55870 1179 memset(kaddr + offset, 0, size);
c3ca015f 1180 dax_flush(dax_dev, kaddr + offset, size);
cccbce67 1181 dax_read_unlock(id);
4b0228fa 1182 }
679c8bd3
CH
1183 return 0;
1184}
1185EXPORT_SYMBOL_GPL(__dax_zero_page_range);
1186
a254e568 1187static loff_t
11c59c92 1188dax_iomap_actor(struct inode *inode, loff_t pos, loff_t length, void *data,
a254e568
CH
1189 struct iomap *iomap)
1190{
cccbce67
DW
1191 struct block_device *bdev = iomap->bdev;
1192 struct dax_device *dax_dev = iomap->dax_dev;
a254e568
CH
1193 struct iov_iter *iter = data;
1194 loff_t end = pos + length, done = 0;
1195 ssize_t ret = 0;
a77d4786 1196 size_t xfer;
cccbce67 1197 int id;
a254e568
CH
1198
1199 if (iov_iter_rw(iter) == READ) {
1200 end = min(end, i_size_read(inode));
1201 if (pos >= end)
1202 return 0;
1203
1204 if (iomap->type == IOMAP_HOLE || iomap->type == IOMAP_UNWRITTEN)
1205 return iov_iter_zero(min(length, end - pos), iter);
1206 }
1207
1208 if (WARN_ON_ONCE(iomap->type != IOMAP_MAPPED))
1209 return -EIO;
1210
e3fce68c
JK
1211 /*
1212 * Write can allocate block for an area which has a hole page mapped
1213 * into page tables. We have to tear down these mappings so that data
1214 * written by write(2) is visible in mmap.
1215 */
cd656375 1216 if (iomap->flags & IOMAP_F_NEW) {
e3fce68c
JK
1217 invalidate_inode_pages2_range(inode->i_mapping,
1218 pos >> PAGE_SHIFT,
1219 (end - 1) >> PAGE_SHIFT);
1220 }
1221
cccbce67 1222 id = dax_read_lock();
a254e568
CH
1223 while (pos < end) {
1224 unsigned offset = pos & (PAGE_SIZE - 1);
cccbce67
DW
1225 const size_t size = ALIGN(length + offset, PAGE_SIZE);
1226 const sector_t sector = dax_iomap_sector(iomap, pos);
a254e568 1227 ssize_t map_len;
cccbce67
DW
1228 pgoff_t pgoff;
1229 void *kaddr;
a254e568 1230
d1908f52
MH
1231 if (fatal_signal_pending(current)) {
1232 ret = -EINTR;
1233 break;
1234 }
1235
cccbce67
DW
1236 ret = bdev_dax_pgoff(bdev, sector, size, &pgoff);
1237 if (ret)
1238 break;
1239
1240 map_len = dax_direct_access(dax_dev, pgoff, PHYS_PFN(size),
86ed913b 1241 &kaddr, NULL);
a254e568
CH
1242 if (map_len < 0) {
1243 ret = map_len;
1244 break;
1245 }
1246
cccbce67
DW
1247 map_len = PFN_PHYS(map_len);
1248 kaddr += offset;
a254e568
CH
1249 map_len -= offset;
1250 if (map_len > end - pos)
1251 map_len = end - pos;
1252
a2e050f5
RZ
1253 /*
1254 * The userspace address for the memory copy has already been
1255 * validated via access_ok() in either vfs_read() or
1256 * vfs_write(), depending on which operation we are doing.
1257 */
a254e568 1258 if (iov_iter_rw(iter) == WRITE)
a77d4786 1259 xfer = dax_copy_from_iter(dax_dev, pgoff, kaddr,
fec53774 1260 map_len, iter);
a254e568 1261 else
a77d4786 1262 xfer = dax_copy_to_iter(dax_dev, pgoff, kaddr,
b3a9a0c3 1263 map_len, iter);
a254e568 1264
a77d4786
DW
1265 pos += xfer;
1266 length -= xfer;
1267 done += xfer;
1268
1269 if (xfer == 0)
1270 ret = -EFAULT;
1271 if (xfer < map_len)
1272 break;
a254e568 1273 }
cccbce67 1274 dax_read_unlock(id);
a254e568
CH
1275
1276 return done ? done : ret;
1277}
1278
1279/**
11c59c92 1280 * dax_iomap_rw - Perform I/O to a DAX file
a254e568
CH
1281 * @iocb: The control block for this I/O
1282 * @iter: The addresses to do I/O from or to
1283 * @ops: iomap ops passed from the file system
1284 *
1285 * This function performs read and write operations to directly mapped
1286 * persistent memory. The callers needs to take care of read/write exclusion
1287 * and evicting any page cache pages in the region under I/O.
1288 */
1289ssize_t
11c59c92 1290dax_iomap_rw(struct kiocb *iocb, struct iov_iter *iter,
8ff6daa1 1291 const struct iomap_ops *ops)
a254e568
CH
1292{
1293 struct address_space *mapping = iocb->ki_filp->f_mapping;
1294 struct inode *inode = mapping->host;
1295 loff_t pos = iocb->ki_pos, ret = 0, done = 0;
1296 unsigned flags = 0;
1297
168316db
CH
1298 if (iov_iter_rw(iter) == WRITE) {
1299 lockdep_assert_held_exclusive(&inode->i_rwsem);
a254e568 1300 flags |= IOMAP_WRITE;
168316db
CH
1301 } else {
1302 lockdep_assert_held(&inode->i_rwsem);
1303 }
a254e568 1304
a254e568
CH
1305 while (iov_iter_count(iter)) {
1306 ret = iomap_apply(inode, pos, iov_iter_count(iter), flags, ops,
11c59c92 1307 iter, dax_iomap_actor);
a254e568
CH
1308 if (ret <= 0)
1309 break;
1310 pos += ret;
1311 done += ret;
1312 }
1313
1314 iocb->ki_pos += done;
1315 return done ? done : ret;
1316}
11c59c92 1317EXPORT_SYMBOL_GPL(dax_iomap_rw);
a7d73fe6 1318
ab77dab4 1319static vm_fault_t dax_fault_return(int error)
9f141d6e
JK
1320{
1321 if (error == 0)
1322 return VM_FAULT_NOPAGE;
1323 if (error == -ENOMEM)
1324 return VM_FAULT_OOM;
1325 return VM_FAULT_SIGBUS;
1326}
1327
aaa422c4
DW
1328/*
1329 * MAP_SYNC on a dax mapping guarantees dirty metadata is
1330 * flushed on write-faults (non-cow), but not read-faults.
1331 */
1332static bool dax_fault_is_synchronous(unsigned long flags,
1333 struct vm_area_struct *vma, struct iomap *iomap)
1334{
1335 return (flags & IOMAP_WRITE) && (vma->vm_flags & VM_SYNC)
1336 && (iomap->flags & IOMAP_F_DIRTY);
1337}
1338
ab77dab4 1339static vm_fault_t dax_iomap_pte_fault(struct vm_fault *vmf, pfn_t *pfnp,
c0b24625 1340 int *iomap_errp, const struct iomap_ops *ops)
a7d73fe6 1341{
a0987ad5
JK
1342 struct vm_area_struct *vma = vmf->vma;
1343 struct address_space *mapping = vma->vm_file->f_mapping;
a7d73fe6 1344 struct inode *inode = mapping->host;
1a29d85e 1345 unsigned long vaddr = vmf->address;
a7d73fe6 1346 loff_t pos = (loff_t)vmf->pgoff << PAGE_SHIFT;
a7d73fe6 1347 struct iomap iomap = { 0 };
9484ab1b 1348 unsigned flags = IOMAP_FAULT;
a7d73fe6 1349 int error, major = 0;
d2c43ef1 1350 bool write = vmf->flags & FAULT_FLAG_WRITE;
caa51d26 1351 bool sync;
ab77dab4 1352 vm_fault_t ret = 0;
a7d73fe6 1353 void *entry;
1b5a1cb2 1354 pfn_t pfn;
a7d73fe6 1355
ab77dab4 1356 trace_dax_pte_fault(inode, vmf, ret);
a7d73fe6
CH
1357 /*
1358 * Check whether offset isn't beyond end of file now. Caller is supposed
1359 * to hold locks serializing us with truncate / punch hole so this is
1360 * a reliable test.
1361 */
a9c42b33 1362 if (pos >= i_size_read(inode)) {
ab77dab4 1363 ret = VM_FAULT_SIGBUS;
a9c42b33
RZ
1364 goto out;
1365 }
a7d73fe6 1366
d2c43ef1 1367 if (write && !vmf->cow_page)
a7d73fe6
CH
1368 flags |= IOMAP_WRITE;
1369
13e451fd
JK
1370 entry = grab_mapping_entry(mapping, vmf->pgoff, 0);
1371 if (IS_ERR(entry)) {
ab77dab4 1372 ret = dax_fault_return(PTR_ERR(entry));
13e451fd
JK
1373 goto out;
1374 }
1375
e2093926
RZ
1376 /*
1377 * It is possible, particularly with mixed reads & writes to private
1378 * mappings, that we have raced with a PMD fault that overlaps with
1379 * the PTE we need to set up. If so just return and the fault will be
1380 * retried.
1381 */
1382 if (pmd_trans_huge(*vmf->pmd) || pmd_devmap(*vmf->pmd)) {
ab77dab4 1383 ret = VM_FAULT_NOPAGE;
e2093926
RZ
1384 goto unlock_entry;
1385 }
1386
a7d73fe6
CH
1387 /*
1388 * Note that we don't bother to use iomap_apply here: DAX required
1389 * the file system block size to be equal the page size, which means
1390 * that we never have to deal with more than a single extent here.
1391 */
1392 error = ops->iomap_begin(inode, pos, PAGE_SIZE, flags, &iomap);
c0b24625
JK
1393 if (iomap_errp)
1394 *iomap_errp = error;
a9c42b33 1395 if (error) {
ab77dab4 1396 ret = dax_fault_return(error);
13e451fd 1397 goto unlock_entry;
a9c42b33 1398 }
a7d73fe6 1399 if (WARN_ON_ONCE(iomap.offset + iomap.length < pos + PAGE_SIZE)) {
13e451fd
JK
1400 error = -EIO; /* fs corruption? */
1401 goto error_finish_iomap;
a7d73fe6
CH
1402 }
1403
a7d73fe6 1404 if (vmf->cow_page) {
31a6f1a6
JK
1405 sector_t sector = dax_iomap_sector(&iomap, pos);
1406
a7d73fe6
CH
1407 switch (iomap.type) {
1408 case IOMAP_HOLE:
1409 case IOMAP_UNWRITTEN:
1410 clear_user_highpage(vmf->cow_page, vaddr);
1411 break;
1412 case IOMAP_MAPPED:
cccbce67
DW
1413 error = copy_user_dax(iomap.bdev, iomap.dax_dev,
1414 sector, PAGE_SIZE, vmf->cow_page, vaddr);
a7d73fe6
CH
1415 break;
1416 default:
1417 WARN_ON_ONCE(1);
1418 error = -EIO;
1419 break;
1420 }
1421
1422 if (error)
13e451fd 1423 goto error_finish_iomap;
b1aa812b
JK
1424
1425 __SetPageUptodate(vmf->cow_page);
ab77dab4
SJ
1426 ret = finish_fault(vmf);
1427 if (!ret)
1428 ret = VM_FAULT_DONE_COW;
13e451fd 1429 goto finish_iomap;
a7d73fe6
CH
1430 }
1431
aaa422c4 1432 sync = dax_fault_is_synchronous(flags, vma, &iomap);
caa51d26 1433
a7d73fe6
CH
1434 switch (iomap.type) {
1435 case IOMAP_MAPPED:
1436 if (iomap.flags & IOMAP_F_NEW) {
1437 count_vm_event(PGMAJFAULT);
a0987ad5 1438 count_memcg_event_mm(vma->vm_mm, PGMAJFAULT);
a7d73fe6
CH
1439 major = VM_FAULT_MAJOR;
1440 }
1b5a1cb2
JK
1441 error = dax_iomap_pfn(&iomap, pos, PAGE_SIZE, &pfn);
1442 if (error < 0)
1443 goto error_finish_iomap;
1444
3fe0791c 1445 entry = dax_insert_mapping_entry(mapping, vmf, entry, pfn,
caa51d26 1446 0, write && !sync);
1b5a1cb2 1447
caa51d26
JK
1448 /*
1449 * If we are doing synchronous page fault and inode needs fsync,
1450 * we can insert PTE into page tables only after that happens.
1451 * Skip insertion for now and return the pfn so that caller can
1452 * insert it after fsync is done.
1453 */
1454 if (sync) {
1455 if (WARN_ON_ONCE(!pfnp)) {
1456 error = -EIO;
1457 goto error_finish_iomap;
1458 }
1459 *pfnp = pfn;
ab77dab4 1460 ret = VM_FAULT_NEEDDSYNC | major;
caa51d26
JK
1461 goto finish_iomap;
1462 }
1b5a1cb2
JK
1463 trace_dax_insert_mapping(inode, vmf, entry);
1464 if (write)
ab77dab4 1465 ret = vmf_insert_mixed_mkwrite(vma, vaddr, pfn);
1b5a1cb2 1466 else
ab77dab4 1467 ret = vmf_insert_mixed(vma, vaddr, pfn);
1b5a1cb2 1468
ab77dab4 1469 goto finish_iomap;
a7d73fe6
CH
1470 case IOMAP_UNWRITTEN:
1471 case IOMAP_HOLE:
d2c43ef1 1472 if (!write) {
ab77dab4 1473 ret = dax_load_hole(mapping, entry, vmf);
13e451fd 1474 goto finish_iomap;
1550290b 1475 }
a7d73fe6
CH
1476 /*FALLTHRU*/
1477 default:
1478 WARN_ON_ONCE(1);
1479 error = -EIO;
1480 break;
1481 }
1482
13e451fd 1483 error_finish_iomap:
ab77dab4 1484 ret = dax_fault_return(error);
9f141d6e
JK
1485 finish_iomap:
1486 if (ops->iomap_end) {
1487 int copied = PAGE_SIZE;
1488
ab77dab4 1489 if (ret & VM_FAULT_ERROR)
9f141d6e
JK
1490 copied = 0;
1491 /*
1492 * The fault is done by now and there's no way back (other
1493 * thread may be already happily using PTE we have installed).
1494 * Just ignore error from ->iomap_end since we cannot do much
1495 * with it.
1496 */
1497 ops->iomap_end(inode, pos, PAGE_SIZE, copied, flags, &iomap);
1550290b 1498 }
13e451fd 1499 unlock_entry:
91d25ba8 1500 put_locked_mapping_entry(mapping, vmf->pgoff);
13e451fd 1501 out:
ab77dab4
SJ
1502 trace_dax_pte_fault_done(inode, vmf, ret);
1503 return ret | major;
a7d73fe6 1504}
642261ac
RZ
1505
1506#ifdef CONFIG_FS_DAX_PMD
ab77dab4 1507static vm_fault_t dax_pmd_load_hole(struct vm_fault *vmf, struct iomap *iomap,
91d25ba8 1508 void *entry)
642261ac 1509{
f4200391
DJ
1510 struct address_space *mapping = vmf->vma->vm_file->f_mapping;
1511 unsigned long pmd_addr = vmf->address & PMD_MASK;
653b2ea3 1512 struct inode *inode = mapping->host;
642261ac 1513 struct page *zero_page;
653b2ea3 1514 void *ret = NULL;
642261ac
RZ
1515 spinlock_t *ptl;
1516 pmd_t pmd_entry;
3fe0791c 1517 pfn_t pfn;
642261ac 1518
f4200391 1519 zero_page = mm_get_huge_zero_page(vmf->vma->vm_mm);
642261ac
RZ
1520
1521 if (unlikely(!zero_page))
653b2ea3 1522 goto fallback;
642261ac 1523
3fe0791c
DW
1524 pfn = page_to_pfn_t(zero_page);
1525 ret = dax_insert_mapping_entry(mapping, vmf, entry, pfn,
f5b7b748 1526 RADIX_DAX_PMD | RADIX_DAX_ZERO_PAGE, false);
642261ac 1527
f4200391
DJ
1528 ptl = pmd_lock(vmf->vma->vm_mm, vmf->pmd);
1529 if (!pmd_none(*(vmf->pmd))) {
642261ac 1530 spin_unlock(ptl);
653b2ea3 1531 goto fallback;
642261ac
RZ
1532 }
1533
f4200391 1534 pmd_entry = mk_pmd(zero_page, vmf->vma->vm_page_prot);
642261ac 1535 pmd_entry = pmd_mkhuge(pmd_entry);
f4200391 1536 set_pmd_at(vmf->vma->vm_mm, pmd_addr, vmf->pmd, pmd_entry);
642261ac 1537 spin_unlock(ptl);
f4200391 1538 trace_dax_pmd_load_hole(inode, vmf, zero_page, ret);
642261ac 1539 return VM_FAULT_NOPAGE;
653b2ea3
RZ
1540
1541fallback:
f4200391 1542 trace_dax_pmd_load_hole_fallback(inode, vmf, zero_page, ret);
653b2ea3 1543 return VM_FAULT_FALLBACK;
642261ac
RZ
1544}
1545
ab77dab4 1546static vm_fault_t dax_iomap_pmd_fault(struct vm_fault *vmf, pfn_t *pfnp,
a2d58167 1547 const struct iomap_ops *ops)
642261ac 1548{
f4200391 1549 struct vm_area_struct *vma = vmf->vma;
642261ac 1550 struct address_space *mapping = vma->vm_file->f_mapping;
d8a849e1
DJ
1551 unsigned long pmd_addr = vmf->address & PMD_MASK;
1552 bool write = vmf->flags & FAULT_FLAG_WRITE;
caa51d26 1553 bool sync;
9484ab1b 1554 unsigned int iomap_flags = (write ? IOMAP_WRITE : 0) | IOMAP_FAULT;
642261ac 1555 struct inode *inode = mapping->host;
ab77dab4 1556 vm_fault_t result = VM_FAULT_FALLBACK;
642261ac
RZ
1557 struct iomap iomap = { 0 };
1558 pgoff_t max_pgoff, pgoff;
642261ac
RZ
1559 void *entry;
1560 loff_t pos;
1561 int error;
302a5e31 1562 pfn_t pfn;
642261ac 1563
282a8e03
RZ
1564 /*
1565 * Check whether offset isn't beyond end of file now. Caller is
1566 * supposed to hold locks serializing us with truncate / punch hole so
1567 * this is a reliable test.
1568 */
1569 pgoff = linear_page_index(vma, pmd_addr);
957ac8c4 1570 max_pgoff = DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE);
282a8e03 1571
f4200391 1572 trace_dax_pmd_fault(inode, vmf, max_pgoff, 0);
282a8e03 1573
fffa281b
RZ
1574 /*
1575 * Make sure that the faulting address's PMD offset (color) matches
1576 * the PMD offset from the start of the file. This is necessary so
1577 * that a PMD range in the page table overlaps exactly with a PMD
1578 * range in the radix tree.
1579 */
1580 if ((vmf->pgoff & PG_PMD_COLOUR) !=
1581 ((vmf->address >> PAGE_SHIFT) & PG_PMD_COLOUR))
1582 goto fallback;
1583
642261ac
RZ
1584 /* Fall back to PTEs if we're going to COW */
1585 if (write && !(vma->vm_flags & VM_SHARED))
1586 goto fallback;
1587
1588 /* If the PMD would extend outside the VMA */
1589 if (pmd_addr < vma->vm_start)
1590 goto fallback;
1591 if ((pmd_addr + PMD_SIZE) > vma->vm_end)
1592 goto fallback;
1593
957ac8c4 1594 if (pgoff >= max_pgoff) {
282a8e03
RZ
1595 result = VM_FAULT_SIGBUS;
1596 goto out;
1597 }
642261ac
RZ
1598
1599 /* If the PMD would extend beyond the file size */
957ac8c4 1600 if ((pgoff | PG_PMD_COLOUR) >= max_pgoff)
642261ac
RZ
1601 goto fallback;
1602
876f2946 1603 /*
91d25ba8
RZ
1604 * grab_mapping_entry() will make sure we get a 2MiB empty entry, a
1605 * 2MiB zero page entry or a DAX PMD. If it can't (because a 4k page
1606 * is already in the tree, for instance), it will return -EEXIST and
1607 * we just fall back to 4k entries.
876f2946
RZ
1608 */
1609 entry = grab_mapping_entry(mapping, pgoff, RADIX_DAX_PMD);
1610 if (IS_ERR(entry))
1611 goto fallback;
1612
e2093926
RZ
1613 /*
1614 * It is possible, particularly with mixed reads & writes to private
1615 * mappings, that we have raced with a PTE fault that overlaps with
1616 * the PMD we need to set up. If so just return and the fault will be
1617 * retried.
1618 */
1619 if (!pmd_none(*vmf->pmd) && !pmd_trans_huge(*vmf->pmd) &&
1620 !pmd_devmap(*vmf->pmd)) {
1621 result = 0;
1622 goto unlock_entry;
1623 }
1624
642261ac
RZ
1625 /*
1626 * Note that we don't use iomap_apply here. We aren't doing I/O, only
1627 * setting up a mapping, so really we're using iomap_begin() as a way
1628 * to look up our filesystem block.
1629 */
1630 pos = (loff_t)pgoff << PAGE_SHIFT;
1631 error = ops->iomap_begin(inode, pos, PMD_SIZE, iomap_flags, &iomap);
1632 if (error)
876f2946 1633 goto unlock_entry;
9f141d6e 1634
642261ac
RZ
1635 if (iomap.offset + iomap.length < pos + PMD_SIZE)
1636 goto finish_iomap;
1637
aaa422c4 1638 sync = dax_fault_is_synchronous(iomap_flags, vma, &iomap);
caa51d26 1639
642261ac
RZ
1640 switch (iomap.type) {
1641 case IOMAP_MAPPED:
302a5e31
JK
1642 error = dax_iomap_pfn(&iomap, pos, PMD_SIZE, &pfn);
1643 if (error < 0)
1644 goto finish_iomap;
1645
3fe0791c 1646 entry = dax_insert_mapping_entry(mapping, vmf, entry, pfn,
caa51d26 1647 RADIX_DAX_PMD, write && !sync);
302a5e31 1648
caa51d26
JK
1649 /*
1650 * If we are doing synchronous page fault and inode needs fsync,
1651 * we can insert PMD into page tables only after that happens.
1652 * Skip insertion for now and return the pfn so that caller can
1653 * insert it after fsync is done.
1654 */
1655 if (sync) {
1656 if (WARN_ON_ONCE(!pfnp))
1657 goto finish_iomap;
1658 *pfnp = pfn;
1659 result = VM_FAULT_NEEDDSYNC;
1660 goto finish_iomap;
1661 }
1662
302a5e31
JK
1663 trace_dax_pmd_insert_mapping(inode, vmf, PMD_SIZE, pfn, entry);
1664 result = vmf_insert_pfn_pmd(vma, vmf->address, vmf->pmd, pfn,
1665 write);
642261ac
RZ
1666 break;
1667 case IOMAP_UNWRITTEN:
1668 case IOMAP_HOLE:
1669 if (WARN_ON_ONCE(write))
876f2946 1670 break;
91d25ba8 1671 result = dax_pmd_load_hole(vmf, &iomap, entry);
642261ac
RZ
1672 break;
1673 default:
1674 WARN_ON_ONCE(1);
1675 break;
1676 }
1677
1678 finish_iomap:
1679 if (ops->iomap_end) {
9f141d6e
JK
1680 int copied = PMD_SIZE;
1681
1682 if (result == VM_FAULT_FALLBACK)
1683 copied = 0;
1684 /*
1685 * The fault is done by now and there's no way back (other
1686 * thread may be already happily using PMD we have installed).
1687 * Just ignore error from ->iomap_end since we cannot do much
1688 * with it.
1689 */
1690 ops->iomap_end(inode, pos, PMD_SIZE, copied, iomap_flags,
1691 &iomap);
642261ac 1692 }
876f2946 1693 unlock_entry:
91d25ba8 1694 put_locked_mapping_entry(mapping, pgoff);
642261ac
RZ
1695 fallback:
1696 if (result == VM_FAULT_FALLBACK) {
d8a849e1 1697 split_huge_pmd(vma, vmf->pmd, vmf->address);
642261ac
RZ
1698 count_vm_event(THP_FAULT_FALLBACK);
1699 }
282a8e03 1700out:
f4200391 1701 trace_dax_pmd_fault_done(inode, vmf, max_pgoff, result);
642261ac
RZ
1702 return result;
1703}
a2d58167 1704#else
ab77dab4 1705static vm_fault_t dax_iomap_pmd_fault(struct vm_fault *vmf, pfn_t *pfnp,
01cddfe9 1706 const struct iomap_ops *ops)
a2d58167
DJ
1707{
1708 return VM_FAULT_FALLBACK;
1709}
642261ac 1710#endif /* CONFIG_FS_DAX_PMD */
a2d58167
DJ
1711
1712/**
1713 * dax_iomap_fault - handle a page fault on a DAX file
1714 * @vmf: The description of the fault
cec04e8c 1715 * @pe_size: Size of the page to fault in
9a0dd422 1716 * @pfnp: PFN to insert for synchronous faults if fsync is required
c0b24625 1717 * @iomap_errp: Storage for detailed error code in case of error
cec04e8c 1718 * @ops: Iomap ops passed from the file system
a2d58167
DJ
1719 *
1720 * When a page fault occurs, filesystems may call this helper in
1721 * their fault handler for DAX files. dax_iomap_fault() assumes the caller
1722 * has done all the necessary locking for page fault to proceed
1723 * successfully.
1724 */
ab77dab4 1725vm_fault_t dax_iomap_fault(struct vm_fault *vmf, enum page_entry_size pe_size,
c0b24625 1726 pfn_t *pfnp, int *iomap_errp, const struct iomap_ops *ops)
a2d58167 1727{
c791ace1
DJ
1728 switch (pe_size) {
1729 case PE_SIZE_PTE:
c0b24625 1730 return dax_iomap_pte_fault(vmf, pfnp, iomap_errp, ops);
c791ace1 1731 case PE_SIZE_PMD:
9a0dd422 1732 return dax_iomap_pmd_fault(vmf, pfnp, ops);
a2d58167
DJ
1733 default:
1734 return VM_FAULT_FALLBACK;
1735 }
1736}
1737EXPORT_SYMBOL_GPL(dax_iomap_fault);
71eab6df
JK
1738
1739/**
1740 * dax_insert_pfn_mkwrite - insert PTE or PMD entry into page tables
1741 * @vmf: The description of the fault
1742 * @pe_size: Size of entry to be inserted
1743 * @pfn: PFN to insert
1744 *
1745 * This function inserts writeable PTE or PMD entry into page tables for mmaped
1746 * DAX file. It takes care of marking corresponding radix tree entry as dirty
1747 * as well.
1748 */
ab77dab4 1749static vm_fault_t dax_insert_pfn_mkwrite(struct vm_fault *vmf,
71eab6df
JK
1750 enum page_entry_size pe_size,
1751 pfn_t pfn)
1752{
1753 struct address_space *mapping = vmf->vma->vm_file->f_mapping;
1754 void *entry, **slot;
1755 pgoff_t index = vmf->pgoff;
ab77dab4 1756 vm_fault_t ret;
71eab6df 1757
b93b0163 1758 xa_lock_irq(&mapping->i_pages);
71eab6df
JK
1759 entry = get_unlocked_mapping_entry(mapping, index, &slot);
1760 /* Did we race with someone splitting entry or so? */
1761 if (!entry ||
1762 (pe_size == PE_SIZE_PTE && !dax_is_pte_entry(entry)) ||
1763 (pe_size == PE_SIZE_PMD && !dax_is_pmd_entry(entry))) {
1764 put_unlocked_mapping_entry(mapping, index, entry);
b93b0163 1765 xa_unlock_irq(&mapping->i_pages);
71eab6df
JK
1766 trace_dax_insert_pfn_mkwrite_no_entry(mapping->host, vmf,
1767 VM_FAULT_NOPAGE);
1768 return VM_FAULT_NOPAGE;
1769 }
b93b0163 1770 radix_tree_tag_set(&mapping->i_pages, index, PAGECACHE_TAG_DIRTY);
71eab6df 1771 entry = lock_slot(mapping, slot);
b93b0163 1772 xa_unlock_irq(&mapping->i_pages);
71eab6df
JK
1773 switch (pe_size) {
1774 case PE_SIZE_PTE:
ab77dab4 1775 ret = vmf_insert_mixed_mkwrite(vmf->vma, vmf->address, pfn);
71eab6df
JK
1776 break;
1777#ifdef CONFIG_FS_DAX_PMD
1778 case PE_SIZE_PMD:
ab77dab4 1779 ret = vmf_insert_pfn_pmd(vmf->vma, vmf->address, vmf->pmd,
71eab6df
JK
1780 pfn, true);
1781 break;
1782#endif
1783 default:
ab77dab4 1784 ret = VM_FAULT_FALLBACK;
71eab6df
JK
1785 }
1786 put_locked_mapping_entry(mapping, index);
ab77dab4
SJ
1787 trace_dax_insert_pfn_mkwrite(mapping->host, vmf, ret);
1788 return ret;
71eab6df
JK
1789}
1790
1791/**
1792 * dax_finish_sync_fault - finish synchronous page fault
1793 * @vmf: The description of the fault
1794 * @pe_size: Size of entry to be inserted
1795 * @pfn: PFN to insert
1796 *
1797 * This function ensures that the file range touched by the page fault is
1798 * stored persistently on the media and handles inserting of appropriate page
1799 * table entry.
1800 */
ab77dab4
SJ
1801vm_fault_t dax_finish_sync_fault(struct vm_fault *vmf,
1802 enum page_entry_size pe_size, pfn_t pfn)
71eab6df
JK
1803{
1804 int err;
1805 loff_t start = ((loff_t)vmf->pgoff) << PAGE_SHIFT;
1806 size_t len = 0;
1807
1808 if (pe_size == PE_SIZE_PTE)
1809 len = PAGE_SIZE;
1810 else if (pe_size == PE_SIZE_PMD)
1811 len = PMD_SIZE;
1812 else
1813 WARN_ON_ONCE(1);
1814 err = vfs_fsync_range(vmf->vma->vm_file, start, start + len - 1, 1);
1815 if (err)
1816 return VM_FAULT_SIGBUS;
1817 return dax_insert_pfn_mkwrite(vmf, pe_size, pfn);
1818}
1819EXPORT_SYMBOL_GPL(dax_finish_sync_fault);