1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /* Storage object read/write
4 * Copyright (C) 2007 Red Hat, Inc. All Rights Reserved.
5 * Written by David Howells (dhowells@redhat.com)
8 #include <linux/mount.h>
9 #include <linux/slab.h>
10 #include <linux/file.h>
11 #include <linux/swap.h>
15 * detect wake up events generated by the unlocking of pages in which we're
17 * - we use this to detect read completion of backing pages
18 * - the caller holds the waitqueue lock
20 static int cachefiles_read_waiter(wait_queue_entry_t
*wait
, unsigned mode
,
23 struct cachefiles_one_read
*monitor
=
24 container_of(wait
, struct cachefiles_one_read
, monitor
);
25 struct cachefiles_object
*object
;
26 struct fscache_retrieval
*op
= monitor
->op
;
27 struct wait_bit_key
*key
= _key
;
28 struct page
*page
= wait
->private;
32 _enter("{%lu},%u,%d,{%p,%u}",
33 monitor
->netfs_page
->index
, mode
, sync
,
34 key
->flags
, key
->bit_nr
);
36 if (key
->flags
!= &page
->flags
||
37 key
->bit_nr
!= PG_locked
)
40 _debug("--- monitor %p %lx ---", page
, page
->flags
);
42 if (!PageUptodate(page
) && !PageError(page
)) {
43 /* unlocked, not uptodate and not erronous? */
44 _debug("page probably truncated");
47 /* remove from the waitqueue */
48 list_del(&wait
->entry
);
50 /* move onto the action list and queue for FS-Cache thread pool */
53 /* We need to temporarily bump the usage count as we don't own a ref
54 * here otherwise cachefiles_read_copier() may free the op between the
55 * monitor being enqueued on the op->to_do list and the op getting
56 * enqueued on the work queue.
58 fscache_get_retrieval(op
);
60 object
= container_of(op
->op
.object
, struct cachefiles_object
, fscache
);
61 spin_lock(&object
->work_lock
);
62 list_add_tail(&monitor
->op_link
, &op
->to_do
);
63 fscache_enqueue_retrieval(op
);
64 spin_unlock(&object
->work_lock
);
66 fscache_put_retrieval(op
);
71 * handle a probably truncated page
72 * - check to see if the page is still relevant and reissue the read if
74 * - return -EIO on error, -ENODATA if the page is gone, -EINPROGRESS if we
75 * must wait again and 0 if successful
77 static int cachefiles_read_reissue(struct cachefiles_object
*object
,
78 struct cachefiles_one_read
*monitor
)
80 struct address_space
*bmapping
= d_backing_inode(object
->backer
)->i_mapping
;
81 struct page
*backpage
= monitor
->back_page
, *backpage2
;
84 _enter("{ino=%lx},{%lx,%lx}",
85 d_backing_inode(object
->backer
)->i_ino
,
86 backpage
->index
, backpage
->flags
);
88 /* skip if the page was truncated away completely */
89 if (backpage
->mapping
!= bmapping
) {
90 _leave(" = -ENODATA [mapping]");
94 backpage2
= find_get_page(bmapping
, backpage
->index
);
96 _leave(" = -ENODATA [gone]");
100 if (backpage
!= backpage2
) {
102 _leave(" = -ENODATA [different]");
106 /* the page is still there and we already have a ref on it, so we don't
110 INIT_LIST_HEAD(&monitor
->op_link
);
111 add_page_wait_queue(backpage
, &monitor
->monitor
);
113 if (trylock_page(backpage
)) {
115 if (PageError(backpage
))
118 if (PageUptodate(backpage
))
121 _debug("reissue read");
122 ret
= bmapping
->a_ops
->readpage(NULL
, backpage
);
127 /* but the page may have been read before the monitor was installed, so
128 * the monitor may miss the event - so we have to ensure that we do get
129 * one in such a case */
130 if (trylock_page(backpage
)) {
131 _debug("jumpstart %p {%lx}", backpage
, backpage
->flags
);
132 unlock_page(backpage
);
135 /* it'll reappear on the todo list */
136 _leave(" = -EINPROGRESS");
140 unlock_page(backpage
);
142 spin_lock_irq(&object
->work_lock
);
143 list_del(&monitor
->op_link
);
144 spin_unlock_irq(&object
->work_lock
);
145 _leave(" = %d", ret
);
150 * copy data from backing pages to netfs pages to complete a read operation
151 * - driven by FS-Cache's thread pool
153 static void cachefiles_read_copier(struct fscache_operation
*_op
)
155 struct cachefiles_one_read
*monitor
;
156 struct cachefiles_object
*object
;
157 struct fscache_retrieval
*op
;
160 op
= container_of(_op
, struct fscache_retrieval
, op
);
161 object
= container_of(op
->op
.object
,
162 struct cachefiles_object
, fscache
);
164 _enter("{ino=%lu}", d_backing_inode(object
->backer
)->i_ino
);
167 spin_lock_irq(&object
->work_lock
);
169 while (!list_empty(&op
->to_do
)) {
170 monitor
= list_entry(op
->to_do
.next
,
171 struct cachefiles_one_read
, op_link
);
172 list_del(&monitor
->op_link
);
174 spin_unlock_irq(&object
->work_lock
);
176 _debug("- copy {%lu}", monitor
->back_page
->index
);
179 if (test_bit(FSCACHE_COOKIE_INVALIDATING
,
180 &object
->fscache
.cookie
->flags
)) {
182 } else if (PageUptodate(monitor
->back_page
)) {
183 copy_highpage(monitor
->netfs_page
, monitor
->back_page
);
184 fscache_mark_page_cached(monitor
->op
,
185 monitor
->netfs_page
);
187 } else if (!PageError(monitor
->back_page
)) {
188 /* the page has probably been truncated */
189 error
= cachefiles_read_reissue(object
, monitor
);
190 if (error
== -EINPROGRESS
)
194 cachefiles_io_error_obj(
196 "Readpage failed on backing file %lx",
197 (unsigned long) monitor
->back_page
->flags
);
201 put_page(monitor
->back_page
);
203 fscache_end_io(op
, monitor
->netfs_page
, error
);
204 put_page(monitor
->netfs_page
);
205 fscache_retrieval_complete(op
, 1);
206 fscache_put_retrieval(op
);
210 /* let the thread pool have some air occasionally */
212 if (max
< 0 || need_resched()) {
213 if (!list_empty(&op
->to_do
))
214 fscache_enqueue_retrieval(op
);
215 _leave(" [maxed out]");
219 spin_lock_irq(&object
->work_lock
);
222 spin_unlock_irq(&object
->work_lock
);
227 * read the corresponding page to the given set from the backing file
228 * - an uncertain page is simply discarded, to be tried again another time
230 static int cachefiles_read_backing_file_one(struct cachefiles_object
*object
,
231 struct fscache_retrieval
*op
,
232 struct page
*netpage
)
234 struct cachefiles_one_read
*monitor
;
235 struct address_space
*bmapping
;
236 struct page
*newpage
, *backpage
;
241 _debug("read back %p{%lu,%d}",
242 netpage
, netpage
->index
, page_count(netpage
));
244 monitor
= kzalloc(sizeof(*monitor
), cachefiles_gfp
);
248 monitor
->netfs_page
= netpage
;
249 monitor
->op
= fscache_get_retrieval(op
);
251 init_waitqueue_func_entry(&monitor
->monitor
, cachefiles_read_waiter
);
253 /* attempt to get hold of the backing page */
254 bmapping
= d_backing_inode(object
->backer
)->i_mapping
;
258 backpage
= find_get_page(bmapping
, netpage
->index
);
260 goto backing_page_already_present
;
263 newpage
= __page_cache_alloc(cachefiles_gfp
);
268 ret
= add_to_page_cache_lru(newpage
, bmapping
,
269 netpage
->index
, cachefiles_gfp
);
271 goto installed_new_backing_page
;
276 /* we've installed a new backing page, so now we need to start
278 installed_new_backing_page
:
279 _debug("- new %p", newpage
);
285 ret
= bmapping
->a_ops
->readpage(NULL
, backpage
);
289 /* set the monitor to transfer the data across */
290 monitor_backing_page
:
291 _debug("- monitor add");
293 /* install the monitor */
294 get_page(monitor
->netfs_page
);
296 monitor
->back_page
= backpage
;
297 monitor
->monitor
.private = backpage
;
298 add_page_wait_queue(backpage
, &monitor
->monitor
);
301 /* but the page may have been read before the monitor was installed, so
302 * the monitor may miss the event - so we have to ensure that we do get
303 * one in such a case */
304 if (trylock_page(backpage
)) {
305 _debug("jumpstart %p {%lx}", backpage
, backpage
->flags
);
306 unlock_page(backpage
);
310 /* if the backing page is already present, it can be in one of
311 * three states: read in progress, read failed or read okay */
312 backing_page_already_present
:
320 if (PageError(backpage
))
323 if (PageUptodate(backpage
))
324 goto backing_page_already_uptodate
;
326 if (!trylock_page(backpage
))
327 goto monitor_backing_page
;
328 _debug("read %p {%lx}", backpage
, backpage
->flags
);
329 goto read_backing_page
;
331 /* the backing page is already up to date, attach the netfs
332 * page to the pagecache and LRU and copy the data across */
333 backing_page_already_uptodate
:
334 _debug("- uptodate");
336 fscache_mark_page_cached(op
, netpage
);
338 copy_highpage(netpage
, backpage
);
339 fscache_end_io(op
, netpage
, 0);
340 fscache_retrieval_complete(op
, 1);
350 fscache_put_retrieval(monitor
->op
);
353 _leave(" = %d", ret
);
357 _debug("read error %d", ret
);
358 if (ret
== -ENOMEM
) {
359 fscache_retrieval_complete(op
, 1);
363 cachefiles_io_error_obj(object
, "Page read error on backing file");
364 fscache_retrieval_complete(op
, 1);
371 fscache_put_retrieval(monitor
->op
);
374 fscache_retrieval_complete(op
, 1);
375 _leave(" = -ENOMEM");
380 * read a page from the cache or allocate a block in which to store it
381 * - cache withdrawal is prevented by the caller
382 * - returns -EINTR if interrupted
383 * - returns -ENOMEM if ran out of memory
384 * - returns -ENOBUFS if no buffers can be made available
385 * - returns -ENOBUFS if page is beyond EOF
386 * - if the page is backed by a block in the cache:
387 * - a read will be started which will call the callback on completion
388 * - 0 will be returned
389 * - else if the page is unbacked:
390 * - the metadata will be retained
391 * - -ENODATA will be returned
393 int cachefiles_read_or_alloc_page(struct fscache_retrieval
*op
,
397 struct cachefiles_object
*object
;
398 struct cachefiles_cache
*cache
;
404 object
= container_of(op
->op
.object
,
405 struct cachefiles_object
, fscache
);
406 cache
= container_of(object
->fscache
.cache
,
407 struct cachefiles_cache
, cache
);
409 _enter("{%p},{%lx},,,", object
, page
->index
);
414 inode
= d_backing_inode(object
->backer
);
415 ASSERT(S_ISREG(inode
->i_mode
));
417 /* calculate the shift required to use bmap */
418 shift
= PAGE_SHIFT
- inode
->i_sb
->s_blocksize_bits
;
420 op
->op
.flags
&= FSCACHE_OP_KEEP_FLAGS
;
421 op
->op
.flags
|= FSCACHE_OP_ASYNC
;
422 op
->op
.processor
= cachefiles_read_copier
;
424 /* we assume the absence or presence of the first block is a good
425 * enough indication for the page as a whole
426 * - TODO: don't use bmap() for this as it is _not_ actually good
427 * enough for this as it doesn't indicate errors, but it's all we've
433 ret2
= bmap(inode
, &block
);
436 _debug("%llx -> %llx",
437 (unsigned long long) (page
->index
<< shift
),
438 (unsigned long long) block
);
441 /* submit the apparently valid page to the backing fs to be
443 ret
= cachefiles_read_backing_file_one(object
, op
, page
);
444 } else if (cachefiles_has_space(cache
, 0, 1) == 0) {
445 /* there's space in the cache we can use */
446 fscache_mark_page_cached(op
, page
);
447 fscache_retrieval_complete(op
, 1);
453 _leave(" = %d", ret
);
457 fscache_retrieval_complete(op
, 1);
458 _leave(" = -ENOBUFS");
463 * read the corresponding pages to the given set from the backing file
464 * - any uncertain pages are simply discarded, to be tried again another time
466 static int cachefiles_read_backing_file(struct cachefiles_object
*object
,
467 struct fscache_retrieval
*op
,
468 struct list_head
*list
)
470 struct cachefiles_one_read
*monitor
= NULL
;
471 struct address_space
*bmapping
= d_backing_inode(object
->backer
)->i_mapping
;
472 struct page
*newpage
= NULL
, *netpage
, *_n
, *backpage
= NULL
;
477 list_for_each_entry_safe(netpage
, _n
, list
, lru
) {
478 list_del(&netpage
->lru
);
480 _debug("read back %p{%lu,%d}",
481 netpage
, netpage
->index
, page_count(netpage
));
484 monitor
= kzalloc(sizeof(*monitor
), cachefiles_gfp
);
488 monitor
->op
= fscache_get_retrieval(op
);
489 init_waitqueue_func_entry(&monitor
->monitor
,
490 cachefiles_read_waiter
);
494 backpage
= find_get_page(bmapping
, netpage
->index
);
496 goto backing_page_already_present
;
499 newpage
= __page_cache_alloc(cachefiles_gfp
);
504 ret
= add_to_page_cache_lru(newpage
, bmapping
,
508 goto installed_new_backing_page
;
513 /* we've installed a new backing page, so now we need
514 * to start it reading */
515 installed_new_backing_page
:
516 _debug("- new %p", newpage
);
522 ret
= bmapping
->a_ops
->readpage(NULL
, backpage
);
526 /* add the netfs page to the pagecache and LRU, and set the
527 * monitor to transfer the data across */
528 monitor_backing_page
:
529 _debug("- monitor add");
531 ret
= add_to_page_cache_lru(netpage
, op
->mapping
,
532 netpage
->index
, cachefiles_gfp
);
534 if (ret
== -EEXIST
) {
539 fscache_retrieval_complete(op
, 1);
545 /* install a monitor */
547 monitor
->netfs_page
= netpage
;
550 monitor
->back_page
= backpage
;
551 monitor
->monitor
.private = backpage
;
552 add_page_wait_queue(backpage
, &monitor
->monitor
);
555 /* but the page may have been read before the monitor was
556 * installed, so the monitor may miss the event - so we have to
557 * ensure that we do get one in such a case */
558 if (trylock_page(backpage
)) {
559 _debug("2unlock %p {%lx}", backpage
, backpage
->flags
);
560 unlock_page(backpage
);
570 /* if the backing page is already present, it can be in one of
571 * three states: read in progress, read failed or read okay */
572 backing_page_already_present
:
573 _debug("- present %p", backpage
);
575 if (PageError(backpage
))
578 if (PageUptodate(backpage
))
579 goto backing_page_already_uptodate
;
581 _debug("- not ready %p{%lx}", backpage
, backpage
->flags
);
583 if (!trylock_page(backpage
))
584 goto monitor_backing_page
;
586 if (PageError(backpage
)) {
587 _debug("error %lx", backpage
->flags
);
588 unlock_page(backpage
);
592 if (PageUptodate(backpage
))
593 goto backing_page_already_uptodate_unlock
;
595 /* we've locked a page that's neither up to date nor erroneous,
596 * so we need to attempt to read it again */
597 goto reread_backing_page
;
599 /* the backing page is already up to date, attach the netfs
600 * page to the pagecache and LRU and copy the data across */
601 backing_page_already_uptodate_unlock
:
602 _debug("uptodate %lx", backpage
->flags
);
603 unlock_page(backpage
);
604 backing_page_already_uptodate
:
605 _debug("- uptodate");
607 ret
= add_to_page_cache_lru(netpage
, op
->mapping
,
608 netpage
->index
, cachefiles_gfp
);
610 if (ret
== -EEXIST
) {
615 fscache_retrieval_complete(op
, 1);
621 copy_highpage(netpage
, backpage
);
626 fscache_mark_page_cached(op
, netpage
);
628 /* the netpage is unlocked and marked up to date here */
629 fscache_end_io(op
, netpage
, 0);
632 fscache_retrieval_complete(op
, 1);
649 fscache_put_retrieval(op
);
653 list_for_each_entry_safe(netpage
, _n
, list
, lru
) {
654 list_del(&netpage
->lru
);
656 fscache_retrieval_complete(op
, 1);
659 _leave(" = %d", ret
);
665 goto record_page_complete
;
668 _debug("read error %d", ret
);
670 goto record_page_complete
;
672 cachefiles_io_error_obj(object
, "Page read error on backing file");
674 record_page_complete
:
675 fscache_retrieval_complete(op
, 1);
680 * read a list of pages from the cache or allocate blocks in which to store
683 int cachefiles_read_or_alloc_pages(struct fscache_retrieval
*op
,
684 struct list_head
*pages
,
688 struct cachefiles_object
*object
;
689 struct cachefiles_cache
*cache
;
690 struct list_head backpages
;
691 struct pagevec pagevec
;
693 struct page
*page
, *_n
;
694 unsigned shift
, nrbackpages
;
695 int ret
, ret2
, space
;
697 object
= container_of(op
->op
.object
,
698 struct cachefiles_object
, fscache
);
699 cache
= container_of(object
->fscache
.cache
,
700 struct cachefiles_cache
, cache
);
702 _enter("{OBJ%x,%d},,%d,,",
703 object
->fscache
.debug_id
, atomic_read(&op
->op
.usage
),
710 if (cachefiles_has_space(cache
, 0, *nr_pages
) < 0)
713 inode
= d_backing_inode(object
->backer
);
714 ASSERT(S_ISREG(inode
->i_mode
));
716 /* calculate the shift required to use bmap */
717 shift
= PAGE_SHIFT
- inode
->i_sb
->s_blocksize_bits
;
719 pagevec_init(&pagevec
);
721 op
->op
.flags
&= FSCACHE_OP_KEEP_FLAGS
;
722 op
->op
.flags
|= FSCACHE_OP_ASYNC
;
723 op
->op
.processor
= cachefiles_read_copier
;
725 INIT_LIST_HEAD(&backpages
);
728 ret
= space
? -ENODATA
: -ENOBUFS
;
729 list_for_each_entry_safe(page
, _n
, pages
, lru
) {
732 /* we assume the absence or presence of the first block is a
733 * good enough indication for the page as a whole
734 * - TODO: don't use bmap() for this as it is _not_ actually
735 * good enough for this as it doesn't indicate errors, but
736 * it's all we've got for the moment
741 ret2
= bmap(inode
, &block
);
744 _debug("%llx -> %llx",
745 (unsigned long long) (page
->index
<< shift
),
746 (unsigned long long) block
);
749 /* we have data - add it to the list to give to the
751 list_move(&page
->lru
, &backpages
);
754 } else if (space
&& pagevec_add(&pagevec
, page
) == 0) {
755 fscache_mark_pages_cached(op
, &pagevec
);
756 fscache_retrieval_complete(op
, 1);
759 fscache_retrieval_complete(op
, 1);
763 if (pagevec_count(&pagevec
) > 0)
764 fscache_mark_pages_cached(op
, &pagevec
);
766 if (list_empty(pages
))
769 /* submit the apparently valid pages to the backing fs to be read from
771 if (nrbackpages
> 0) {
772 ret2
= cachefiles_read_backing_file(object
, op
, &backpages
);
773 if (ret2
== -ENOMEM
|| ret2
== -EINTR
)
777 _leave(" = %d [nr=%u%s]",
778 ret
, *nr_pages
, list_empty(pages
) ? " empty" : "");
782 fscache_retrieval_complete(op
, *nr_pages
);
787 * allocate a block in the cache in which to store a page
788 * - cache withdrawal is prevented by the caller
789 * - returns -EINTR if interrupted
790 * - returns -ENOMEM if ran out of memory
791 * - returns -ENOBUFS if no buffers can be made available
792 * - returns -ENOBUFS if page is beyond EOF
794 * - the metadata will be retained
795 * - 0 will be returned
797 int cachefiles_allocate_page(struct fscache_retrieval
*op
,
801 struct cachefiles_object
*object
;
802 struct cachefiles_cache
*cache
;
805 object
= container_of(op
->op
.object
,
806 struct cachefiles_object
, fscache
);
807 cache
= container_of(object
->fscache
.cache
,
808 struct cachefiles_cache
, cache
);
810 _enter("%p,{%lx},", object
, page
->index
);
812 ret
= cachefiles_has_space(cache
, 0, 1);
814 fscache_mark_page_cached(op
, page
);
818 fscache_retrieval_complete(op
, 1);
819 _leave(" = %d", ret
);
824 * allocate blocks in the cache in which to store a set of pages
825 * - cache withdrawal is prevented by the caller
826 * - returns -EINTR if interrupted
827 * - returns -ENOMEM if ran out of memory
828 * - returns -ENOBUFS if some buffers couldn't be made available
829 * - returns -ENOBUFS if some pages are beyond EOF
831 * - -ENODATA will be returned
832 * - metadata will be retained for any page marked
834 int cachefiles_allocate_pages(struct fscache_retrieval
*op
,
835 struct list_head
*pages
,
839 struct cachefiles_object
*object
;
840 struct cachefiles_cache
*cache
;
841 struct pagevec pagevec
;
845 object
= container_of(op
->op
.object
,
846 struct cachefiles_object
, fscache
);
847 cache
= container_of(object
->fscache
.cache
,
848 struct cachefiles_cache
, cache
);
850 _enter("%p,,,%d,", object
, *nr_pages
);
852 ret
= cachefiles_has_space(cache
, 0, *nr_pages
);
854 pagevec_init(&pagevec
);
856 list_for_each_entry(page
, pages
, lru
) {
857 if (pagevec_add(&pagevec
, page
) == 0)
858 fscache_mark_pages_cached(op
, &pagevec
);
861 if (pagevec_count(&pagevec
) > 0)
862 fscache_mark_pages_cached(op
, &pagevec
);
868 fscache_retrieval_complete(op
, *nr_pages
);
869 _leave(" = %d", ret
);
874 * request a page be stored in the cache
875 * - cache withdrawal is prevented by the caller
876 * - this request may be ignored if there's no cache block available, in which
877 * case -ENOBUFS will be returned
878 * - if the op is in progress, 0 will be returned
880 int cachefiles_write_page(struct fscache_storage
*op
, struct page
*page
)
882 struct cachefiles_object
*object
;
883 struct cachefiles_cache
*cache
;
892 ASSERT(page
!= NULL
);
894 object
= container_of(op
->op
.object
,
895 struct cachefiles_object
, fscache
);
897 _enter("%p,%p{%lx},,,", object
, page
, page
->index
);
899 if (!object
->backer
) {
900 _leave(" = -ENOBUFS");
904 ASSERT(d_is_reg(object
->backer
));
906 cache
= container_of(object
->fscache
.cache
,
907 struct cachefiles_cache
, cache
);
909 pos
= (loff_t
)page
->index
<< PAGE_SHIFT
;
911 /* We mustn't write more data than we have, so we have to beware of a
912 * partial page at EOF.
914 eof
= object
->fscache
.store_limit_l
;
918 /* write the page to the backing filesystem and let it store it in its
920 path
.mnt
= cache
->mnt
;
921 path
.dentry
= object
->backer
;
922 file
= dentry_open(&path
, O_RDWR
| O_LARGEFILE
, cache
->cache_cred
);
929 if (eof
& ~PAGE_MASK
) {
930 if (eof
- pos
< PAGE_SIZE
) {
931 _debug("cut short %llx to %llx",
934 ASSERTCMP(pos
+ len
, ==, eof
);
939 ret
= kernel_write(file
, data
, len
, &pos
);
952 cachefiles_io_error_obj(object
,
953 "Write page to backing file failed");
955 _leave(" = -ENOBUFS [%d]", ret
);
960 * detach a backing block from a page
961 * - cache withdrawal is prevented by the caller
963 void cachefiles_uncache_page(struct fscache_object
*_object
, struct page
*page
)
964 __releases(&object
->fscache
.cookie
->lock
)
966 struct cachefiles_object
*object
;
968 object
= container_of(_object
, struct cachefiles_object
, fscache
);
970 _enter("%p,{%lu}", object
, page
->index
);
972 spin_unlock(&object
->fscache
.cookie
->lock
);