]> git.proxmox.com Git - mirror_ubuntu-jammy-kernel.git/blob - fs/xfs/linux-2.6/xfs_aops.c
Input: xpad - add USB ID for the drumkit controller from Rock Band
[mirror_ubuntu-jammy-kernel.git] / fs / xfs / linux-2.6 / xfs_aops.c
1 /*
2 * Copyright (c) 2000-2005 Silicon Graphics, Inc.
3 * All Rights Reserved.
4 *
5 * This program is free software; you can redistribute it and/or
6 * modify it under the terms of the GNU General Public License as
7 * published by the Free Software Foundation.
8 *
9 * This program is distributed in the hope that it would be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
13 *
14 * You should have received a copy of the GNU General Public License
15 * along with this program; if not, write the Free Software Foundation,
16 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
17 */
18 #include "xfs.h"
19 #include "xfs_bit.h"
20 #include "xfs_log.h"
21 #include "xfs_inum.h"
22 #include "xfs_sb.h"
23 #include "xfs_ag.h"
24 #include "xfs_dir2.h"
25 #include "xfs_trans.h"
26 #include "xfs_dmapi.h"
27 #include "xfs_mount.h"
28 #include "xfs_bmap_btree.h"
29 #include "xfs_alloc_btree.h"
30 #include "xfs_ialloc_btree.h"
31 #include "xfs_dir2_sf.h"
32 #include "xfs_attr_sf.h"
33 #include "xfs_dinode.h"
34 #include "xfs_inode.h"
35 #include "xfs_alloc.h"
36 #include "xfs_btree.h"
37 #include "xfs_error.h"
38 #include "xfs_rw.h"
39 #include "xfs_iomap.h"
40 #include "xfs_vnodeops.h"
41 #include <linux/mpage.h>
42 #include <linux/pagevec.h>
43 #include <linux/writeback.h>
44
45
46 /*
47 * Prime number of hash buckets since address is used as the key.
48 */
49 #define NVSYNC 37
50 #define to_ioend_wq(v) (&xfs_ioend_wq[((unsigned long)v) % NVSYNC])
51 static wait_queue_head_t xfs_ioend_wq[NVSYNC];
52
53 void __init
54 xfs_ioend_init(void)
55 {
56 int i;
57
58 for (i = 0; i < NVSYNC; i++)
59 init_waitqueue_head(&xfs_ioend_wq[i]);
60 }
61
62 void
63 xfs_ioend_wait(
64 xfs_inode_t *ip)
65 {
66 wait_queue_head_t *wq = to_ioend_wq(ip);
67
68 wait_event(*wq, (atomic_read(&ip->i_iocount) == 0));
69 }
70
71 STATIC void
72 xfs_ioend_wake(
73 xfs_inode_t *ip)
74 {
75 if (atomic_dec_and_test(&ip->i_iocount))
76 wake_up(to_ioend_wq(ip));
77 }
78
79 STATIC void
80 xfs_count_page_state(
81 struct page *page,
82 int *delalloc,
83 int *unmapped,
84 int *unwritten)
85 {
86 struct buffer_head *bh, *head;
87
88 *delalloc = *unmapped = *unwritten = 0;
89
90 bh = head = page_buffers(page);
91 do {
92 if (buffer_uptodate(bh) && !buffer_mapped(bh))
93 (*unmapped) = 1;
94 else if (buffer_unwritten(bh))
95 (*unwritten) = 1;
96 else if (buffer_delay(bh))
97 (*delalloc) = 1;
98 } while ((bh = bh->b_this_page) != head);
99 }
100
101 #if defined(XFS_RW_TRACE)
102 void
103 xfs_page_trace(
104 int tag,
105 struct inode *inode,
106 struct page *page,
107 unsigned long pgoff)
108 {
109 xfs_inode_t *ip;
110 loff_t isize = i_size_read(inode);
111 loff_t offset = page_offset(page);
112 int delalloc = -1, unmapped = -1, unwritten = -1;
113
114 if (page_has_buffers(page))
115 xfs_count_page_state(page, &delalloc, &unmapped, &unwritten);
116
117 ip = XFS_I(inode);
118 if (!ip->i_rwtrace)
119 return;
120
121 ktrace_enter(ip->i_rwtrace,
122 (void *)((unsigned long)tag),
123 (void *)ip,
124 (void *)inode,
125 (void *)page,
126 (void *)pgoff,
127 (void *)((unsigned long)((ip->i_d.di_size >> 32) & 0xffffffff)),
128 (void *)((unsigned long)(ip->i_d.di_size & 0xffffffff)),
129 (void *)((unsigned long)((isize >> 32) & 0xffffffff)),
130 (void *)((unsigned long)(isize & 0xffffffff)),
131 (void *)((unsigned long)((offset >> 32) & 0xffffffff)),
132 (void *)((unsigned long)(offset & 0xffffffff)),
133 (void *)((unsigned long)delalloc),
134 (void *)((unsigned long)unmapped),
135 (void *)((unsigned long)unwritten),
136 (void *)((unsigned long)current_pid()),
137 (void *)NULL);
138 }
139 #else
140 #define xfs_page_trace(tag, inode, page, pgoff)
141 #endif
142
143 STATIC struct block_device *
144 xfs_find_bdev_for_inode(
145 struct xfs_inode *ip)
146 {
147 struct xfs_mount *mp = ip->i_mount;
148
149 if (XFS_IS_REALTIME_INODE(ip))
150 return mp->m_rtdev_targp->bt_bdev;
151 else
152 return mp->m_ddev_targp->bt_bdev;
153 }
154
155 /*
156 * We're now finished for good with this ioend structure.
157 * Update the page state via the associated buffer_heads,
158 * release holds on the inode and bio, and finally free
159 * up memory. Do not use the ioend after this.
160 */
161 STATIC void
162 xfs_destroy_ioend(
163 xfs_ioend_t *ioend)
164 {
165 struct buffer_head *bh, *next;
166 struct xfs_inode *ip = XFS_I(ioend->io_inode);
167
168 for (bh = ioend->io_buffer_head; bh; bh = next) {
169 next = bh->b_private;
170 bh->b_end_io(bh, !ioend->io_error);
171 }
172
173 /*
174 * Volume managers supporting multiple paths can send back ENODEV
175 * when the final path disappears. In this case continuing to fill
176 * the page cache with dirty data which cannot be written out is
177 * evil, so prevent that.
178 */
179 if (unlikely(ioend->io_error == -ENODEV)) {
180 xfs_do_force_shutdown(ip->i_mount, SHUTDOWN_DEVICE_REQ,
181 __FILE__, __LINE__);
182 }
183
184 xfs_ioend_wake(ip);
185 mempool_free(ioend, xfs_ioend_pool);
186 }
187
188 /*
189 * Update on-disk file size now that data has been written to disk.
190 * The current in-memory file size is i_size. If a write is beyond
191 * eof i_new_size will be the intended file size until i_size is
192 * updated. If this write does not extend all the way to the valid
193 * file size then restrict this update to the end of the write.
194 */
195 STATIC void
196 xfs_setfilesize(
197 xfs_ioend_t *ioend)
198 {
199 xfs_inode_t *ip = XFS_I(ioend->io_inode);
200 xfs_fsize_t isize;
201 xfs_fsize_t bsize;
202
203 ASSERT((ip->i_d.di_mode & S_IFMT) == S_IFREG);
204 ASSERT(ioend->io_type != IOMAP_READ);
205
206 if (unlikely(ioend->io_error))
207 return;
208
209 bsize = ioend->io_offset + ioend->io_size;
210
211 xfs_ilock(ip, XFS_ILOCK_EXCL);
212
213 isize = MAX(ip->i_size, ip->i_new_size);
214 isize = MIN(isize, bsize);
215
216 if (ip->i_d.di_size < isize) {
217 ip->i_d.di_size = isize;
218 ip->i_update_core = 1;
219 ip->i_update_size = 1;
220 xfs_mark_inode_dirty_sync(ip);
221 }
222
223 xfs_iunlock(ip, XFS_ILOCK_EXCL);
224 }
225
226 /*
227 * Buffered IO write completion for delayed allocate extents.
228 */
229 STATIC void
230 xfs_end_bio_delalloc(
231 struct work_struct *work)
232 {
233 xfs_ioend_t *ioend =
234 container_of(work, xfs_ioend_t, io_work);
235
236 xfs_setfilesize(ioend);
237 xfs_destroy_ioend(ioend);
238 }
239
240 /*
241 * Buffered IO write completion for regular, written extents.
242 */
243 STATIC void
244 xfs_end_bio_written(
245 struct work_struct *work)
246 {
247 xfs_ioend_t *ioend =
248 container_of(work, xfs_ioend_t, io_work);
249
250 xfs_setfilesize(ioend);
251 xfs_destroy_ioend(ioend);
252 }
253
254 /*
255 * IO write completion for unwritten extents.
256 *
257 * Issue transactions to convert a buffer range from unwritten
258 * to written extents.
259 */
260 STATIC void
261 xfs_end_bio_unwritten(
262 struct work_struct *work)
263 {
264 xfs_ioend_t *ioend =
265 container_of(work, xfs_ioend_t, io_work);
266 struct xfs_inode *ip = XFS_I(ioend->io_inode);
267 xfs_off_t offset = ioend->io_offset;
268 size_t size = ioend->io_size;
269
270 if (likely(!ioend->io_error)) {
271 if (!XFS_FORCED_SHUTDOWN(ip->i_mount)) {
272 int error;
273 error = xfs_iomap_write_unwritten(ip, offset, size);
274 if (error)
275 ioend->io_error = error;
276 }
277 xfs_setfilesize(ioend);
278 }
279 xfs_destroy_ioend(ioend);
280 }
281
282 /*
283 * IO read completion for regular, written extents.
284 */
285 STATIC void
286 xfs_end_bio_read(
287 struct work_struct *work)
288 {
289 xfs_ioend_t *ioend =
290 container_of(work, xfs_ioend_t, io_work);
291
292 xfs_destroy_ioend(ioend);
293 }
294
295 /*
296 * Schedule IO completion handling on a xfsdatad if this was
297 * the final hold on this ioend. If we are asked to wait,
298 * flush the workqueue.
299 */
300 STATIC void
301 xfs_finish_ioend(
302 xfs_ioend_t *ioend,
303 int wait)
304 {
305 if (atomic_dec_and_test(&ioend->io_remaining)) {
306 struct workqueue_struct *wq = xfsdatad_workqueue;
307 if (ioend->io_work.func == xfs_end_bio_unwritten)
308 wq = xfsconvertd_workqueue;
309
310 queue_work(wq, &ioend->io_work);
311 if (wait)
312 flush_workqueue(wq);
313 }
314 }
315
316 /*
317 * Allocate and initialise an IO completion structure.
318 * We need to track unwritten extent write completion here initially.
319 * We'll need to extend this for updating the ondisk inode size later
320 * (vs. incore size).
321 */
322 STATIC xfs_ioend_t *
323 xfs_alloc_ioend(
324 struct inode *inode,
325 unsigned int type)
326 {
327 xfs_ioend_t *ioend;
328
329 ioend = mempool_alloc(xfs_ioend_pool, GFP_NOFS);
330
331 /*
332 * Set the count to 1 initially, which will prevent an I/O
333 * completion callback from happening before we have started
334 * all the I/O from calling the completion routine too early.
335 */
336 atomic_set(&ioend->io_remaining, 1);
337 ioend->io_error = 0;
338 ioend->io_list = NULL;
339 ioend->io_type = type;
340 ioend->io_inode = inode;
341 ioend->io_buffer_head = NULL;
342 ioend->io_buffer_tail = NULL;
343 atomic_inc(&XFS_I(ioend->io_inode)->i_iocount);
344 ioend->io_offset = 0;
345 ioend->io_size = 0;
346
347 if (type == IOMAP_UNWRITTEN)
348 INIT_WORK(&ioend->io_work, xfs_end_bio_unwritten);
349 else if (type == IOMAP_DELAY)
350 INIT_WORK(&ioend->io_work, xfs_end_bio_delalloc);
351 else if (type == IOMAP_READ)
352 INIT_WORK(&ioend->io_work, xfs_end_bio_read);
353 else
354 INIT_WORK(&ioend->io_work, xfs_end_bio_written);
355
356 return ioend;
357 }
358
359 STATIC int
360 xfs_map_blocks(
361 struct inode *inode,
362 loff_t offset,
363 ssize_t count,
364 xfs_iomap_t *mapp,
365 int flags)
366 {
367 int nmaps = 1;
368
369 return -xfs_iomap(XFS_I(inode), offset, count, flags, mapp, &nmaps);
370 }
371
372 STATIC_INLINE int
373 xfs_iomap_valid(
374 xfs_iomap_t *iomapp,
375 loff_t offset)
376 {
377 return offset >= iomapp->iomap_offset &&
378 offset < iomapp->iomap_offset + iomapp->iomap_bsize;
379 }
380
381 /*
382 * BIO completion handler for buffered IO.
383 */
384 STATIC void
385 xfs_end_bio(
386 struct bio *bio,
387 int error)
388 {
389 xfs_ioend_t *ioend = bio->bi_private;
390
391 ASSERT(atomic_read(&bio->bi_cnt) >= 1);
392 ioend->io_error = test_bit(BIO_UPTODATE, &bio->bi_flags) ? 0 : error;
393
394 /* Toss bio and pass work off to an xfsdatad thread */
395 bio->bi_private = NULL;
396 bio->bi_end_io = NULL;
397 bio_put(bio);
398
399 xfs_finish_ioend(ioend, 0);
400 }
401
402 STATIC void
403 xfs_submit_ioend_bio(
404 xfs_ioend_t *ioend,
405 struct bio *bio)
406 {
407 atomic_inc(&ioend->io_remaining);
408
409 bio->bi_private = ioend;
410 bio->bi_end_io = xfs_end_bio;
411
412 submit_bio(WRITE, bio);
413 ASSERT(!bio_flagged(bio, BIO_EOPNOTSUPP));
414 bio_put(bio);
415 }
416
417 STATIC struct bio *
418 xfs_alloc_ioend_bio(
419 struct buffer_head *bh)
420 {
421 struct bio *bio;
422 int nvecs = bio_get_nr_vecs(bh->b_bdev);
423
424 do {
425 bio = bio_alloc(GFP_NOIO, nvecs);
426 nvecs >>= 1;
427 } while (!bio);
428
429 ASSERT(bio->bi_private == NULL);
430 bio->bi_sector = bh->b_blocknr * (bh->b_size >> 9);
431 bio->bi_bdev = bh->b_bdev;
432 bio_get(bio);
433 return bio;
434 }
435
436 STATIC void
437 xfs_start_buffer_writeback(
438 struct buffer_head *bh)
439 {
440 ASSERT(buffer_mapped(bh));
441 ASSERT(buffer_locked(bh));
442 ASSERT(!buffer_delay(bh));
443 ASSERT(!buffer_unwritten(bh));
444
445 mark_buffer_async_write(bh);
446 set_buffer_uptodate(bh);
447 clear_buffer_dirty(bh);
448 }
449
450 STATIC void
451 xfs_start_page_writeback(
452 struct page *page,
453 int clear_dirty,
454 int buffers)
455 {
456 ASSERT(PageLocked(page));
457 ASSERT(!PageWriteback(page));
458 if (clear_dirty)
459 clear_page_dirty_for_io(page);
460 set_page_writeback(page);
461 unlock_page(page);
462 /* If no buffers on the page are to be written, finish it here */
463 if (!buffers)
464 end_page_writeback(page);
465 }
466
467 static inline int bio_add_buffer(struct bio *bio, struct buffer_head *bh)
468 {
469 return bio_add_page(bio, bh->b_page, bh->b_size, bh_offset(bh));
470 }
471
472 /*
473 * Submit all of the bios for all of the ioends we have saved up, covering the
474 * initial writepage page and also any probed pages.
475 *
476 * Because we may have multiple ioends spanning a page, we need to start
477 * writeback on all the buffers before we submit them for I/O. If we mark the
478 * buffers as we got, then we can end up with a page that only has buffers
479 * marked async write and I/O complete on can occur before we mark the other
480 * buffers async write.
481 *
482 * The end result of this is that we trip a bug in end_page_writeback() because
483 * we call it twice for the one page as the code in end_buffer_async_write()
484 * assumes that all buffers on the page are started at the same time.
485 *
486 * The fix is two passes across the ioend list - one to start writeback on the
487 * buffer_heads, and then submit them for I/O on the second pass.
488 */
489 STATIC void
490 xfs_submit_ioend(
491 xfs_ioend_t *ioend)
492 {
493 xfs_ioend_t *head = ioend;
494 xfs_ioend_t *next;
495 struct buffer_head *bh;
496 struct bio *bio;
497 sector_t lastblock = 0;
498
499 /* Pass 1 - start writeback */
500 do {
501 next = ioend->io_list;
502 for (bh = ioend->io_buffer_head; bh; bh = bh->b_private) {
503 xfs_start_buffer_writeback(bh);
504 }
505 } while ((ioend = next) != NULL);
506
507 /* Pass 2 - submit I/O */
508 ioend = head;
509 do {
510 next = ioend->io_list;
511 bio = NULL;
512
513 for (bh = ioend->io_buffer_head; bh; bh = bh->b_private) {
514
515 if (!bio) {
516 retry:
517 bio = xfs_alloc_ioend_bio(bh);
518 } else if (bh->b_blocknr != lastblock + 1) {
519 xfs_submit_ioend_bio(ioend, bio);
520 goto retry;
521 }
522
523 if (bio_add_buffer(bio, bh) != bh->b_size) {
524 xfs_submit_ioend_bio(ioend, bio);
525 goto retry;
526 }
527
528 lastblock = bh->b_blocknr;
529 }
530 if (bio)
531 xfs_submit_ioend_bio(ioend, bio);
532 xfs_finish_ioend(ioend, 0);
533 } while ((ioend = next) != NULL);
534 }
535
536 /*
537 * Cancel submission of all buffer_heads so far in this endio.
538 * Toss the endio too. Only ever called for the initial page
539 * in a writepage request, so only ever one page.
540 */
541 STATIC void
542 xfs_cancel_ioend(
543 xfs_ioend_t *ioend)
544 {
545 xfs_ioend_t *next;
546 struct buffer_head *bh, *next_bh;
547
548 do {
549 next = ioend->io_list;
550 bh = ioend->io_buffer_head;
551 do {
552 next_bh = bh->b_private;
553 clear_buffer_async_write(bh);
554 unlock_buffer(bh);
555 } while ((bh = next_bh) != NULL);
556
557 xfs_ioend_wake(XFS_I(ioend->io_inode));
558 mempool_free(ioend, xfs_ioend_pool);
559 } while ((ioend = next) != NULL);
560 }
561
562 /*
563 * Test to see if we've been building up a completion structure for
564 * earlier buffers -- if so, we try to append to this ioend if we
565 * can, otherwise we finish off any current ioend and start another.
566 * Return true if we've finished the given ioend.
567 */
568 STATIC void
569 xfs_add_to_ioend(
570 struct inode *inode,
571 struct buffer_head *bh,
572 xfs_off_t offset,
573 unsigned int type,
574 xfs_ioend_t **result,
575 int need_ioend)
576 {
577 xfs_ioend_t *ioend = *result;
578
579 if (!ioend || need_ioend || type != ioend->io_type) {
580 xfs_ioend_t *previous = *result;
581
582 ioend = xfs_alloc_ioend(inode, type);
583 ioend->io_offset = offset;
584 ioend->io_buffer_head = bh;
585 ioend->io_buffer_tail = bh;
586 if (previous)
587 previous->io_list = ioend;
588 *result = ioend;
589 } else {
590 ioend->io_buffer_tail->b_private = bh;
591 ioend->io_buffer_tail = bh;
592 }
593
594 bh->b_private = NULL;
595 ioend->io_size += bh->b_size;
596 }
597
598 STATIC void
599 xfs_map_buffer(
600 struct buffer_head *bh,
601 xfs_iomap_t *mp,
602 xfs_off_t offset,
603 uint block_bits)
604 {
605 sector_t bn;
606
607 ASSERT(mp->iomap_bn != IOMAP_DADDR_NULL);
608
609 bn = (mp->iomap_bn >> (block_bits - BBSHIFT)) +
610 ((offset - mp->iomap_offset) >> block_bits);
611
612 ASSERT(bn || (mp->iomap_flags & IOMAP_REALTIME));
613
614 bh->b_blocknr = bn;
615 set_buffer_mapped(bh);
616 }
617
618 STATIC void
619 xfs_map_at_offset(
620 struct buffer_head *bh,
621 loff_t offset,
622 int block_bits,
623 xfs_iomap_t *iomapp)
624 {
625 ASSERT(!(iomapp->iomap_flags & IOMAP_HOLE));
626 ASSERT(!(iomapp->iomap_flags & IOMAP_DELAY));
627
628 lock_buffer(bh);
629 xfs_map_buffer(bh, iomapp, offset, block_bits);
630 bh->b_bdev = iomapp->iomap_target->bt_bdev;
631 set_buffer_mapped(bh);
632 clear_buffer_delay(bh);
633 clear_buffer_unwritten(bh);
634 }
635
636 /*
637 * Look for a page at index that is suitable for clustering.
638 */
639 STATIC unsigned int
640 xfs_probe_page(
641 struct page *page,
642 unsigned int pg_offset,
643 int mapped)
644 {
645 int ret = 0;
646
647 if (PageWriteback(page))
648 return 0;
649
650 if (page->mapping && PageDirty(page)) {
651 if (page_has_buffers(page)) {
652 struct buffer_head *bh, *head;
653
654 bh = head = page_buffers(page);
655 do {
656 if (!buffer_uptodate(bh))
657 break;
658 if (mapped != buffer_mapped(bh))
659 break;
660 ret += bh->b_size;
661 if (ret >= pg_offset)
662 break;
663 } while ((bh = bh->b_this_page) != head);
664 } else
665 ret = mapped ? 0 : PAGE_CACHE_SIZE;
666 }
667
668 return ret;
669 }
670
671 STATIC size_t
672 xfs_probe_cluster(
673 struct inode *inode,
674 struct page *startpage,
675 struct buffer_head *bh,
676 struct buffer_head *head,
677 int mapped)
678 {
679 struct pagevec pvec;
680 pgoff_t tindex, tlast, tloff;
681 size_t total = 0;
682 int done = 0, i;
683
684 /* First sum forwards in this page */
685 do {
686 if (!buffer_uptodate(bh) || (mapped != buffer_mapped(bh)))
687 return total;
688 total += bh->b_size;
689 } while ((bh = bh->b_this_page) != head);
690
691 /* if we reached the end of the page, sum forwards in following pages */
692 tlast = i_size_read(inode) >> PAGE_CACHE_SHIFT;
693 tindex = startpage->index + 1;
694
695 /* Prune this back to avoid pathological behavior */
696 tloff = min(tlast, startpage->index + 64);
697
698 pagevec_init(&pvec, 0);
699 while (!done && tindex <= tloff) {
700 unsigned len = min_t(pgoff_t, PAGEVEC_SIZE, tlast - tindex + 1);
701
702 if (!pagevec_lookup(&pvec, inode->i_mapping, tindex, len))
703 break;
704
705 for (i = 0; i < pagevec_count(&pvec); i++) {
706 struct page *page = pvec.pages[i];
707 size_t pg_offset, pg_len = 0;
708
709 if (tindex == tlast) {
710 pg_offset =
711 i_size_read(inode) & (PAGE_CACHE_SIZE - 1);
712 if (!pg_offset) {
713 done = 1;
714 break;
715 }
716 } else
717 pg_offset = PAGE_CACHE_SIZE;
718
719 if (page->index == tindex && trylock_page(page)) {
720 pg_len = xfs_probe_page(page, pg_offset, mapped);
721 unlock_page(page);
722 }
723
724 if (!pg_len) {
725 done = 1;
726 break;
727 }
728
729 total += pg_len;
730 tindex++;
731 }
732
733 pagevec_release(&pvec);
734 cond_resched();
735 }
736
737 return total;
738 }
739
740 /*
741 * Test if a given page is suitable for writing as part of an unwritten
742 * or delayed allocate extent.
743 */
744 STATIC int
745 xfs_is_delayed_page(
746 struct page *page,
747 unsigned int type)
748 {
749 if (PageWriteback(page))
750 return 0;
751
752 if (page->mapping && page_has_buffers(page)) {
753 struct buffer_head *bh, *head;
754 int acceptable = 0;
755
756 bh = head = page_buffers(page);
757 do {
758 if (buffer_unwritten(bh))
759 acceptable = (type == IOMAP_UNWRITTEN);
760 else if (buffer_delay(bh))
761 acceptable = (type == IOMAP_DELAY);
762 else if (buffer_dirty(bh) && buffer_mapped(bh))
763 acceptable = (type == IOMAP_NEW);
764 else
765 break;
766 } while ((bh = bh->b_this_page) != head);
767
768 if (acceptable)
769 return 1;
770 }
771
772 return 0;
773 }
774
775 /*
776 * Allocate & map buffers for page given the extent map. Write it out.
777 * except for the original page of a writepage, this is called on
778 * delalloc/unwritten pages only, for the original page it is possible
779 * that the page has no mapping at all.
780 */
781 STATIC int
782 xfs_convert_page(
783 struct inode *inode,
784 struct page *page,
785 loff_t tindex,
786 xfs_iomap_t *mp,
787 xfs_ioend_t **ioendp,
788 struct writeback_control *wbc,
789 int startio,
790 int all_bh)
791 {
792 struct buffer_head *bh, *head;
793 xfs_off_t end_offset;
794 unsigned long p_offset;
795 unsigned int type;
796 int bbits = inode->i_blkbits;
797 int len, page_dirty;
798 int count = 0, done = 0, uptodate = 1;
799 xfs_off_t offset = page_offset(page);
800
801 if (page->index != tindex)
802 goto fail;
803 if (!trylock_page(page))
804 goto fail;
805 if (PageWriteback(page))
806 goto fail_unlock_page;
807 if (page->mapping != inode->i_mapping)
808 goto fail_unlock_page;
809 if (!xfs_is_delayed_page(page, (*ioendp)->io_type))
810 goto fail_unlock_page;
811
812 /*
813 * page_dirty is initially a count of buffers on the page before
814 * EOF and is decremented as we move each into a cleanable state.
815 *
816 * Derivation:
817 *
818 * End offset is the highest offset that this page should represent.
819 * If we are on the last page, (end_offset & (PAGE_CACHE_SIZE - 1))
820 * will evaluate non-zero and be less than PAGE_CACHE_SIZE and
821 * hence give us the correct page_dirty count. On any other page,
822 * it will be zero and in that case we need page_dirty to be the
823 * count of buffers on the page.
824 */
825 end_offset = min_t(unsigned long long,
826 (xfs_off_t)(page->index + 1) << PAGE_CACHE_SHIFT,
827 i_size_read(inode));
828
829 len = 1 << inode->i_blkbits;
830 p_offset = min_t(unsigned long, end_offset & (PAGE_CACHE_SIZE - 1),
831 PAGE_CACHE_SIZE);
832 p_offset = p_offset ? roundup(p_offset, len) : PAGE_CACHE_SIZE;
833 page_dirty = p_offset / len;
834
835 bh = head = page_buffers(page);
836 do {
837 if (offset >= end_offset)
838 break;
839 if (!buffer_uptodate(bh))
840 uptodate = 0;
841 if (!(PageUptodate(page) || buffer_uptodate(bh))) {
842 done = 1;
843 continue;
844 }
845
846 if (buffer_unwritten(bh) || buffer_delay(bh)) {
847 if (buffer_unwritten(bh))
848 type = IOMAP_UNWRITTEN;
849 else
850 type = IOMAP_DELAY;
851
852 if (!xfs_iomap_valid(mp, offset)) {
853 done = 1;
854 continue;
855 }
856
857 ASSERT(!(mp->iomap_flags & IOMAP_HOLE));
858 ASSERT(!(mp->iomap_flags & IOMAP_DELAY));
859
860 xfs_map_at_offset(bh, offset, bbits, mp);
861 if (startio) {
862 xfs_add_to_ioend(inode, bh, offset,
863 type, ioendp, done);
864 } else {
865 set_buffer_dirty(bh);
866 unlock_buffer(bh);
867 mark_buffer_dirty(bh);
868 }
869 page_dirty--;
870 count++;
871 } else {
872 type = IOMAP_NEW;
873 if (buffer_mapped(bh) && all_bh && startio) {
874 lock_buffer(bh);
875 xfs_add_to_ioend(inode, bh, offset,
876 type, ioendp, done);
877 count++;
878 page_dirty--;
879 } else {
880 done = 1;
881 }
882 }
883 } while (offset += len, (bh = bh->b_this_page) != head);
884
885 if (uptodate && bh == head)
886 SetPageUptodate(page);
887
888 if (startio) {
889 if (count) {
890 struct backing_dev_info *bdi;
891
892 bdi = inode->i_mapping->backing_dev_info;
893 wbc->nr_to_write--;
894 if (bdi_write_congested(bdi)) {
895 wbc->encountered_congestion = 1;
896 done = 1;
897 } else if (wbc->nr_to_write <= 0) {
898 done = 1;
899 }
900 }
901 xfs_start_page_writeback(page, !page_dirty, count);
902 }
903
904 return done;
905 fail_unlock_page:
906 unlock_page(page);
907 fail:
908 return 1;
909 }
910
911 /*
912 * Convert & write out a cluster of pages in the same extent as defined
913 * by mp and following the start page.
914 */
915 STATIC void
916 xfs_cluster_write(
917 struct inode *inode,
918 pgoff_t tindex,
919 xfs_iomap_t *iomapp,
920 xfs_ioend_t **ioendp,
921 struct writeback_control *wbc,
922 int startio,
923 int all_bh,
924 pgoff_t tlast)
925 {
926 struct pagevec pvec;
927 int done = 0, i;
928
929 pagevec_init(&pvec, 0);
930 while (!done && tindex <= tlast) {
931 unsigned len = min_t(pgoff_t, PAGEVEC_SIZE, tlast - tindex + 1);
932
933 if (!pagevec_lookup(&pvec, inode->i_mapping, tindex, len))
934 break;
935
936 for (i = 0; i < pagevec_count(&pvec); i++) {
937 done = xfs_convert_page(inode, pvec.pages[i], tindex++,
938 iomapp, ioendp, wbc, startio, all_bh);
939 if (done)
940 break;
941 }
942
943 pagevec_release(&pvec);
944 cond_resched();
945 }
946 }
947
948 /*
949 * Calling this without startio set means we are being asked to make a dirty
950 * page ready for freeing it's buffers. When called with startio set then
951 * we are coming from writepage.
952 *
953 * When called with startio set it is important that we write the WHOLE
954 * page if possible.
955 * The bh->b_state's cannot know if any of the blocks or which block for
956 * that matter are dirty due to mmap writes, and therefore bh uptodate is
957 * only valid if the page itself isn't completely uptodate. Some layers
958 * may clear the page dirty flag prior to calling write page, under the
959 * assumption the entire page will be written out; by not writing out the
960 * whole page the page can be reused before all valid dirty data is
961 * written out. Note: in the case of a page that has been dirty'd by
962 * mapwrite and but partially setup by block_prepare_write the
963 * bh->b_states's will not agree and only ones setup by BPW/BCW will have
964 * valid state, thus the whole page must be written out thing.
965 */
966
967 STATIC int
968 xfs_page_state_convert(
969 struct inode *inode,
970 struct page *page,
971 struct writeback_control *wbc,
972 int startio,
973 int unmapped) /* also implies page uptodate */
974 {
975 struct buffer_head *bh, *head;
976 xfs_iomap_t iomap;
977 xfs_ioend_t *ioend = NULL, *iohead = NULL;
978 loff_t offset;
979 unsigned long p_offset = 0;
980 unsigned int type;
981 __uint64_t end_offset;
982 pgoff_t end_index, last_index, tlast;
983 ssize_t size, len;
984 int flags, err, iomap_valid = 0, uptodate = 1;
985 int page_dirty, count = 0;
986 int trylock = 0;
987 int all_bh = unmapped;
988
989 if (startio) {
990 if (wbc->sync_mode == WB_SYNC_NONE && wbc->nonblocking)
991 trylock |= BMAPI_TRYLOCK;
992 }
993
994 /* Is this page beyond the end of the file? */
995 offset = i_size_read(inode);
996 end_index = offset >> PAGE_CACHE_SHIFT;
997 last_index = (offset - 1) >> PAGE_CACHE_SHIFT;
998 if (page->index >= end_index) {
999 if ((page->index >= end_index + 1) ||
1000 !(i_size_read(inode) & (PAGE_CACHE_SIZE - 1))) {
1001 if (startio)
1002 unlock_page(page);
1003 return 0;
1004 }
1005 }
1006
1007 /*
1008 * page_dirty is initially a count of buffers on the page before
1009 * EOF and is decremented as we move each into a cleanable state.
1010 *
1011 * Derivation:
1012 *
1013 * End offset is the highest offset that this page should represent.
1014 * If we are on the last page, (end_offset & (PAGE_CACHE_SIZE - 1))
1015 * will evaluate non-zero and be less than PAGE_CACHE_SIZE and
1016 * hence give us the correct page_dirty count. On any other page,
1017 * it will be zero and in that case we need page_dirty to be the
1018 * count of buffers on the page.
1019 */
1020 end_offset = min_t(unsigned long long,
1021 (xfs_off_t)(page->index + 1) << PAGE_CACHE_SHIFT, offset);
1022 len = 1 << inode->i_blkbits;
1023 p_offset = min_t(unsigned long, end_offset & (PAGE_CACHE_SIZE - 1),
1024 PAGE_CACHE_SIZE);
1025 p_offset = p_offset ? roundup(p_offset, len) : PAGE_CACHE_SIZE;
1026 page_dirty = p_offset / len;
1027
1028 bh = head = page_buffers(page);
1029 offset = page_offset(page);
1030 flags = BMAPI_READ;
1031 type = IOMAP_NEW;
1032
1033 /* TODO: cleanup count and page_dirty */
1034
1035 do {
1036 if (offset >= end_offset)
1037 break;
1038 if (!buffer_uptodate(bh))
1039 uptodate = 0;
1040 if (!(PageUptodate(page) || buffer_uptodate(bh)) && !startio) {
1041 /*
1042 * the iomap is actually still valid, but the ioend
1043 * isn't. shouldn't happen too often.
1044 */
1045 iomap_valid = 0;
1046 continue;
1047 }
1048
1049 if (iomap_valid)
1050 iomap_valid = xfs_iomap_valid(&iomap, offset);
1051
1052 /*
1053 * First case, map an unwritten extent and prepare for
1054 * extent state conversion transaction on completion.
1055 *
1056 * Second case, allocate space for a delalloc buffer.
1057 * We can return EAGAIN here in the release page case.
1058 *
1059 * Third case, an unmapped buffer was found, and we are
1060 * in a path where we need to write the whole page out.
1061 */
1062 if (buffer_unwritten(bh) || buffer_delay(bh) ||
1063 ((buffer_uptodate(bh) || PageUptodate(page)) &&
1064 !buffer_mapped(bh) && (unmapped || startio))) {
1065 int new_ioend = 0;
1066
1067 /*
1068 * Make sure we don't use a read-only iomap
1069 */
1070 if (flags == BMAPI_READ)
1071 iomap_valid = 0;
1072
1073 if (buffer_unwritten(bh)) {
1074 type = IOMAP_UNWRITTEN;
1075 flags = BMAPI_WRITE | BMAPI_IGNSTATE;
1076 } else if (buffer_delay(bh)) {
1077 type = IOMAP_DELAY;
1078 flags = BMAPI_ALLOCATE | trylock;
1079 } else {
1080 type = IOMAP_NEW;
1081 flags = BMAPI_WRITE | BMAPI_MMAP;
1082 }
1083
1084 if (!iomap_valid) {
1085 /*
1086 * if we didn't have a valid mapping then we
1087 * need to ensure that we put the new mapping
1088 * in a new ioend structure. This needs to be
1089 * done to ensure that the ioends correctly
1090 * reflect the block mappings at io completion
1091 * for unwritten extent conversion.
1092 */
1093 new_ioend = 1;
1094 if (type == IOMAP_NEW) {
1095 size = xfs_probe_cluster(inode,
1096 page, bh, head, 0);
1097 } else {
1098 size = len;
1099 }
1100
1101 err = xfs_map_blocks(inode, offset, size,
1102 &iomap, flags);
1103 if (err)
1104 goto error;
1105 iomap_valid = xfs_iomap_valid(&iomap, offset);
1106 }
1107 if (iomap_valid) {
1108 xfs_map_at_offset(bh, offset,
1109 inode->i_blkbits, &iomap);
1110 if (startio) {
1111 xfs_add_to_ioend(inode, bh, offset,
1112 type, &ioend,
1113 new_ioend);
1114 } else {
1115 set_buffer_dirty(bh);
1116 unlock_buffer(bh);
1117 mark_buffer_dirty(bh);
1118 }
1119 page_dirty--;
1120 count++;
1121 }
1122 } else if (buffer_uptodate(bh) && startio) {
1123 /*
1124 * we got here because the buffer is already mapped.
1125 * That means it must already have extents allocated
1126 * underneath it. Map the extent by reading it.
1127 */
1128 if (!iomap_valid || flags != BMAPI_READ) {
1129 flags = BMAPI_READ;
1130 size = xfs_probe_cluster(inode, page, bh,
1131 head, 1);
1132 err = xfs_map_blocks(inode, offset, size,
1133 &iomap, flags);
1134 if (err)
1135 goto error;
1136 iomap_valid = xfs_iomap_valid(&iomap, offset);
1137 }
1138
1139 /*
1140 * We set the type to IOMAP_NEW in case we are doing a
1141 * small write at EOF that is extending the file but
1142 * without needing an allocation. We need to update the
1143 * file size on I/O completion in this case so it is
1144 * the same case as having just allocated a new extent
1145 * that we are writing into for the first time.
1146 */
1147 type = IOMAP_NEW;
1148 if (trylock_buffer(bh)) {
1149 ASSERT(buffer_mapped(bh));
1150 if (iomap_valid)
1151 all_bh = 1;
1152 xfs_add_to_ioend(inode, bh, offset, type,
1153 &ioend, !iomap_valid);
1154 page_dirty--;
1155 count++;
1156 } else {
1157 iomap_valid = 0;
1158 }
1159 } else if ((buffer_uptodate(bh) || PageUptodate(page)) &&
1160 (unmapped || startio)) {
1161 iomap_valid = 0;
1162 }
1163
1164 if (!iohead)
1165 iohead = ioend;
1166
1167 } while (offset += len, ((bh = bh->b_this_page) != head));
1168
1169 if (uptodate && bh == head)
1170 SetPageUptodate(page);
1171
1172 if (startio)
1173 xfs_start_page_writeback(page, 1, count);
1174
1175 if (ioend && iomap_valid) {
1176 offset = (iomap.iomap_offset + iomap.iomap_bsize - 1) >>
1177 PAGE_CACHE_SHIFT;
1178 tlast = min_t(pgoff_t, offset, last_index);
1179 xfs_cluster_write(inode, page->index + 1, &iomap, &ioend,
1180 wbc, startio, all_bh, tlast);
1181 }
1182
1183 if (iohead)
1184 xfs_submit_ioend(iohead);
1185
1186 return page_dirty;
1187
1188 error:
1189 if (iohead)
1190 xfs_cancel_ioend(iohead);
1191
1192 /*
1193 * If it's delalloc and we have nowhere to put it,
1194 * throw it away, unless the lower layers told
1195 * us to try again.
1196 */
1197 if (err != -EAGAIN) {
1198 if (!unmapped)
1199 block_invalidatepage(page, 0);
1200 ClearPageUptodate(page);
1201 }
1202 return err;
1203 }
1204
1205 /*
1206 * writepage: Called from one of two places:
1207 *
1208 * 1. we are flushing a delalloc buffer head.
1209 *
1210 * 2. we are writing out a dirty page. Typically the page dirty
1211 * state is cleared before we get here. In this case is it
1212 * conceivable we have no buffer heads.
1213 *
1214 * For delalloc space on the page we need to allocate space and
1215 * flush it. For unmapped buffer heads on the page we should
1216 * allocate space if the page is uptodate. For any other dirty
1217 * buffer heads on the page we should flush them.
1218 *
1219 * If we detect that a transaction would be required to flush
1220 * the page, we have to check the process flags first, if we
1221 * are already in a transaction or disk I/O during allocations
1222 * is off, we need to fail the writepage and redirty the page.
1223 */
1224
1225 STATIC int
1226 xfs_vm_writepage(
1227 struct page *page,
1228 struct writeback_control *wbc)
1229 {
1230 int error;
1231 int need_trans;
1232 int delalloc, unmapped, unwritten;
1233 struct inode *inode = page->mapping->host;
1234
1235 xfs_page_trace(XFS_WRITEPAGE_ENTER, inode, page, 0);
1236
1237 /*
1238 * We need a transaction if:
1239 * 1. There are delalloc buffers on the page
1240 * 2. The page is uptodate and we have unmapped buffers
1241 * 3. The page is uptodate and we have no buffers
1242 * 4. There are unwritten buffers on the page
1243 */
1244
1245 if (!page_has_buffers(page)) {
1246 unmapped = 1;
1247 need_trans = 1;
1248 } else {
1249 xfs_count_page_state(page, &delalloc, &unmapped, &unwritten);
1250 if (!PageUptodate(page))
1251 unmapped = 0;
1252 need_trans = delalloc + unmapped + unwritten;
1253 }
1254
1255 /*
1256 * If we need a transaction and the process flags say
1257 * we are already in a transaction, or no IO is allowed
1258 * then mark the page dirty again and leave the page
1259 * as is.
1260 */
1261 if (current_test_flags(PF_FSTRANS) && need_trans)
1262 goto out_fail;
1263
1264 /*
1265 * Delay hooking up buffer heads until we have
1266 * made our go/no-go decision.
1267 */
1268 if (!page_has_buffers(page))
1269 create_empty_buffers(page, 1 << inode->i_blkbits, 0);
1270
1271 /*
1272 * Convert delayed allocate, unwritten or unmapped space
1273 * to real space and flush out to disk.
1274 */
1275 error = xfs_page_state_convert(inode, page, wbc, 1, unmapped);
1276 if (error == -EAGAIN)
1277 goto out_fail;
1278 if (unlikely(error < 0))
1279 goto out_unlock;
1280
1281 return 0;
1282
1283 out_fail:
1284 redirty_page_for_writepage(wbc, page);
1285 unlock_page(page);
1286 return 0;
1287 out_unlock:
1288 unlock_page(page);
1289 return error;
1290 }
1291
1292 STATIC int
1293 xfs_vm_writepages(
1294 struct address_space *mapping,
1295 struct writeback_control *wbc)
1296 {
1297 xfs_iflags_clear(XFS_I(mapping->host), XFS_ITRUNCATED);
1298 return generic_writepages(mapping, wbc);
1299 }
1300
1301 /*
1302 * Called to move a page into cleanable state - and from there
1303 * to be released. Possibly the page is already clean. We always
1304 * have buffer heads in this call.
1305 *
1306 * Returns 0 if the page is ok to release, 1 otherwise.
1307 *
1308 * Possible scenarios are:
1309 *
1310 * 1. We are being called to release a page which has been written
1311 * to via regular I/O. buffer heads will be dirty and possibly
1312 * delalloc. If no delalloc buffer heads in this case then we
1313 * can just return zero.
1314 *
1315 * 2. We are called to release a page which has been written via
1316 * mmap, all we need to do is ensure there is no delalloc
1317 * state in the buffer heads, if not we can let the caller
1318 * free them and we should come back later via writepage.
1319 */
1320 STATIC int
1321 xfs_vm_releasepage(
1322 struct page *page,
1323 gfp_t gfp_mask)
1324 {
1325 struct inode *inode = page->mapping->host;
1326 int dirty, delalloc, unmapped, unwritten;
1327 struct writeback_control wbc = {
1328 .sync_mode = WB_SYNC_ALL,
1329 .nr_to_write = 1,
1330 };
1331
1332 xfs_page_trace(XFS_RELEASEPAGE_ENTER, inode, page, 0);
1333
1334 if (!page_has_buffers(page))
1335 return 0;
1336
1337 xfs_count_page_state(page, &delalloc, &unmapped, &unwritten);
1338 if (!delalloc && !unwritten)
1339 goto free_buffers;
1340
1341 if (!(gfp_mask & __GFP_FS))
1342 return 0;
1343
1344 /* If we are already inside a transaction or the thread cannot
1345 * do I/O, we cannot release this page.
1346 */
1347 if (current_test_flags(PF_FSTRANS))
1348 return 0;
1349
1350 /*
1351 * Convert delalloc space to real space, do not flush the
1352 * data out to disk, that will be done by the caller.
1353 * Never need to allocate space here - we will always
1354 * come back to writepage in that case.
1355 */
1356 dirty = xfs_page_state_convert(inode, page, &wbc, 0, 0);
1357 if (dirty == 0 && !unwritten)
1358 goto free_buffers;
1359 return 0;
1360
1361 free_buffers:
1362 return try_to_free_buffers(page);
1363 }
1364
1365 STATIC int
1366 __xfs_get_blocks(
1367 struct inode *inode,
1368 sector_t iblock,
1369 struct buffer_head *bh_result,
1370 int create,
1371 int direct,
1372 bmapi_flags_t flags)
1373 {
1374 xfs_iomap_t iomap;
1375 xfs_off_t offset;
1376 ssize_t size;
1377 int niomap = 1;
1378 int error;
1379
1380 offset = (xfs_off_t)iblock << inode->i_blkbits;
1381 ASSERT(bh_result->b_size >= (1 << inode->i_blkbits));
1382 size = bh_result->b_size;
1383
1384 if (!create && direct && offset >= i_size_read(inode))
1385 return 0;
1386
1387 error = xfs_iomap(XFS_I(inode), offset, size,
1388 create ? flags : BMAPI_READ, &iomap, &niomap);
1389 if (error)
1390 return -error;
1391 if (niomap == 0)
1392 return 0;
1393
1394 if (iomap.iomap_bn != IOMAP_DADDR_NULL) {
1395 /*
1396 * For unwritten extents do not report a disk address on
1397 * the read case (treat as if we're reading into a hole).
1398 */
1399 if (create || !(iomap.iomap_flags & IOMAP_UNWRITTEN)) {
1400 xfs_map_buffer(bh_result, &iomap, offset,
1401 inode->i_blkbits);
1402 }
1403 if (create && (iomap.iomap_flags & IOMAP_UNWRITTEN)) {
1404 if (direct)
1405 bh_result->b_private = inode;
1406 set_buffer_unwritten(bh_result);
1407 }
1408 }
1409
1410 /*
1411 * If this is a realtime file, data may be on a different device.
1412 * to that pointed to from the buffer_head b_bdev currently.
1413 */
1414 bh_result->b_bdev = iomap.iomap_target->bt_bdev;
1415
1416 /*
1417 * If we previously allocated a block out beyond eof and we are now
1418 * coming back to use it then we will need to flag it as new even if it
1419 * has a disk address.
1420 *
1421 * With sub-block writes into unwritten extents we also need to mark
1422 * the buffer as new so that the unwritten parts of the buffer gets
1423 * correctly zeroed.
1424 */
1425 if (create &&
1426 ((!buffer_mapped(bh_result) && !buffer_uptodate(bh_result)) ||
1427 (offset >= i_size_read(inode)) ||
1428 (iomap.iomap_flags & (IOMAP_NEW|IOMAP_UNWRITTEN))))
1429 set_buffer_new(bh_result);
1430
1431 if (iomap.iomap_flags & IOMAP_DELAY) {
1432 BUG_ON(direct);
1433 if (create) {
1434 set_buffer_uptodate(bh_result);
1435 set_buffer_mapped(bh_result);
1436 set_buffer_delay(bh_result);
1437 }
1438 }
1439
1440 if (direct || size > (1 << inode->i_blkbits)) {
1441 ASSERT(iomap.iomap_bsize - iomap.iomap_delta > 0);
1442 offset = min_t(xfs_off_t,
1443 iomap.iomap_bsize - iomap.iomap_delta, size);
1444 bh_result->b_size = (ssize_t)min_t(xfs_off_t, LONG_MAX, offset);
1445 }
1446
1447 return 0;
1448 }
1449
1450 int
1451 xfs_get_blocks(
1452 struct inode *inode,
1453 sector_t iblock,
1454 struct buffer_head *bh_result,
1455 int create)
1456 {
1457 return __xfs_get_blocks(inode, iblock,
1458 bh_result, create, 0, BMAPI_WRITE);
1459 }
1460
1461 STATIC int
1462 xfs_get_blocks_direct(
1463 struct inode *inode,
1464 sector_t iblock,
1465 struct buffer_head *bh_result,
1466 int create)
1467 {
1468 return __xfs_get_blocks(inode, iblock,
1469 bh_result, create, 1, BMAPI_WRITE|BMAPI_DIRECT);
1470 }
1471
1472 STATIC void
1473 xfs_end_io_direct(
1474 struct kiocb *iocb,
1475 loff_t offset,
1476 ssize_t size,
1477 void *private)
1478 {
1479 xfs_ioend_t *ioend = iocb->private;
1480
1481 /*
1482 * Non-NULL private data means we need to issue a transaction to
1483 * convert a range from unwritten to written extents. This needs
1484 * to happen from process context but aio+dio I/O completion
1485 * happens from irq context so we need to defer it to a workqueue.
1486 * This is not necessary for synchronous direct I/O, but we do
1487 * it anyway to keep the code uniform and simpler.
1488 *
1489 * Well, if only it were that simple. Because synchronous direct I/O
1490 * requires extent conversion to occur *before* we return to userspace,
1491 * we have to wait for extent conversion to complete. Look at the
1492 * iocb that has been passed to us to determine if this is AIO or
1493 * not. If it is synchronous, tell xfs_finish_ioend() to kick the
1494 * workqueue and wait for it to complete.
1495 *
1496 * The core direct I/O code might be changed to always call the
1497 * completion handler in the future, in which case all this can
1498 * go away.
1499 */
1500 ioend->io_offset = offset;
1501 ioend->io_size = size;
1502 if (ioend->io_type == IOMAP_READ) {
1503 xfs_finish_ioend(ioend, 0);
1504 } else if (private && size > 0) {
1505 xfs_finish_ioend(ioend, is_sync_kiocb(iocb));
1506 } else {
1507 /*
1508 * A direct I/O write ioend starts it's life in unwritten
1509 * state in case they map an unwritten extent. This write
1510 * didn't map an unwritten extent so switch it's completion
1511 * handler.
1512 */
1513 INIT_WORK(&ioend->io_work, xfs_end_bio_written);
1514 xfs_finish_ioend(ioend, 0);
1515 }
1516
1517 /*
1518 * blockdev_direct_IO can return an error even after the I/O
1519 * completion handler was called. Thus we need to protect
1520 * against double-freeing.
1521 */
1522 iocb->private = NULL;
1523 }
1524
1525 STATIC ssize_t
1526 xfs_vm_direct_IO(
1527 int rw,
1528 struct kiocb *iocb,
1529 const struct iovec *iov,
1530 loff_t offset,
1531 unsigned long nr_segs)
1532 {
1533 struct file *file = iocb->ki_filp;
1534 struct inode *inode = file->f_mapping->host;
1535 struct block_device *bdev;
1536 ssize_t ret;
1537
1538 bdev = xfs_find_bdev_for_inode(XFS_I(inode));
1539
1540 if (rw == WRITE) {
1541 iocb->private = xfs_alloc_ioend(inode, IOMAP_UNWRITTEN);
1542 ret = blockdev_direct_IO_own_locking(rw, iocb, inode,
1543 bdev, iov, offset, nr_segs,
1544 xfs_get_blocks_direct,
1545 xfs_end_io_direct);
1546 } else {
1547 iocb->private = xfs_alloc_ioend(inode, IOMAP_READ);
1548 ret = blockdev_direct_IO_no_locking(rw, iocb, inode,
1549 bdev, iov, offset, nr_segs,
1550 xfs_get_blocks_direct,
1551 xfs_end_io_direct);
1552 }
1553
1554 if (unlikely(ret != -EIOCBQUEUED && iocb->private))
1555 xfs_destroy_ioend(iocb->private);
1556 return ret;
1557 }
1558
1559 STATIC int
1560 xfs_vm_write_begin(
1561 struct file *file,
1562 struct address_space *mapping,
1563 loff_t pos,
1564 unsigned len,
1565 unsigned flags,
1566 struct page **pagep,
1567 void **fsdata)
1568 {
1569 *pagep = NULL;
1570 return block_write_begin(file, mapping, pos, len, flags, pagep, fsdata,
1571 xfs_get_blocks);
1572 }
1573
1574 STATIC sector_t
1575 xfs_vm_bmap(
1576 struct address_space *mapping,
1577 sector_t block)
1578 {
1579 struct inode *inode = (struct inode *)mapping->host;
1580 struct xfs_inode *ip = XFS_I(inode);
1581
1582 xfs_itrace_entry(XFS_I(inode));
1583 xfs_ilock(ip, XFS_IOLOCK_SHARED);
1584 xfs_flush_pages(ip, (xfs_off_t)0, -1, 0, FI_REMAPF);
1585 xfs_iunlock(ip, XFS_IOLOCK_SHARED);
1586 return generic_block_bmap(mapping, block, xfs_get_blocks);
1587 }
1588
1589 STATIC int
1590 xfs_vm_readpage(
1591 struct file *unused,
1592 struct page *page)
1593 {
1594 return mpage_readpage(page, xfs_get_blocks);
1595 }
1596
1597 STATIC int
1598 xfs_vm_readpages(
1599 struct file *unused,
1600 struct address_space *mapping,
1601 struct list_head *pages,
1602 unsigned nr_pages)
1603 {
1604 return mpage_readpages(mapping, pages, nr_pages, xfs_get_blocks);
1605 }
1606
1607 STATIC void
1608 xfs_vm_invalidatepage(
1609 struct page *page,
1610 unsigned long offset)
1611 {
1612 xfs_page_trace(XFS_INVALIDPAGE_ENTER,
1613 page->mapping->host, page, offset);
1614 block_invalidatepage(page, offset);
1615 }
1616
1617 const struct address_space_operations xfs_address_space_operations = {
1618 .readpage = xfs_vm_readpage,
1619 .readpages = xfs_vm_readpages,
1620 .writepage = xfs_vm_writepage,
1621 .writepages = xfs_vm_writepages,
1622 .sync_page = block_sync_page,
1623 .releasepage = xfs_vm_releasepage,
1624 .invalidatepage = xfs_vm_invalidatepage,
1625 .write_begin = xfs_vm_write_begin,
1626 .write_end = generic_write_end,
1627 .bmap = xfs_vm_bmap,
1628 .direct_IO = xfs_vm_direct_IO,
1629 .migratepage = buffer_migrate_page,
1630 .is_partially_uptodate = block_is_partially_uptodate,
1631 };