]> git.proxmox.com Git - mirror_ubuntu-jammy-kernel.git/blob - fs/fscache/page.c
projects / mirror_ubuntu-jammy-kernel.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
Merge tag 'io_uring-5.15-2021-10-17' of git://git.kernel.dk/linux-block
[mirror_ubuntu-jammy-kernel.git] / fs / fscache / page.c
1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /* Cache page management and data I/O routines
3 *
4 * Copyright (C) 2004-2008 Red Hat, Inc. All Rights Reserved.
5 * Written by David Howells (dhowells@redhat.com)
6 */
7
8 #define FSCACHE_DEBUG_LEVEL PAGE
9 #include <linux/module.h>
10 #include <linux/fscache-cache.h>
11 #include <linux/buffer_head.h>
12 #include <linux/pagevec.h>
13 #include <linux/slab.h>
14 #include "internal.h"
15
16 /*
17 * check to see if a page is being written to the cache
18 */
19 bool __fscache_check_page_write(struct fscache_cookie *cookie, struct page *page)
20 {
21 void *val;
22
23 rcu_read_lock();
24 val = radix_tree_lookup(&cookie->stores, page->index);
25 rcu_read_unlock();
26 trace_fscache_check_page(cookie, page, val, 0);
27
28 return val != NULL;
29 }
30 EXPORT_SYMBOL(__fscache_check_page_write);
31
32 /*
33 * wait for a page to finish being written to the cache
34 */
35 void __fscache_wait_on_page_write(struct fscache_cookie *cookie, struct page *page)
36 {
37 wait_queue_head_t *wq = bit_waitqueue(&cookie->flags, 0);
38
39 trace_fscache_page(cookie, page, fscache_page_write_wait);
40
41 wait_event(*wq, !__fscache_check_page_write(cookie, page));
42 }
43 EXPORT_SYMBOL(__fscache_wait_on_page_write);
44
45 /*
46 * wait for a page to finish being written to the cache. Put a timeout here
47 * since we might be called recursively via parent fs.
48 */
49 static
50 bool release_page_wait_timeout(struct fscache_cookie *cookie, struct page *page)
51 {
52 wait_queue_head_t *wq = bit_waitqueue(&cookie->flags, 0);
53
54 return wait_event_timeout(*wq, !__fscache_check_page_write(cookie, page),
55 HZ);
56 }
57
58 /*
59 * decide whether a page can be released, possibly by cancelling a store to it
60 * - we're allowed to sleep if __GFP_DIRECT_RECLAIM is flagged
61 */
62 bool __fscache_maybe_release_page(struct fscache_cookie *cookie,
63 struct page *page,
64 gfp_t gfp)
65 {
66 struct page *xpage;
67 void *val;
68
69 _enter("%p,%p,%x", cookie, page, gfp);
70
71 trace_fscache_page(cookie, page, fscache_page_maybe_release);
72
73 try_again:
74 rcu_read_lock();
75 val = radix_tree_lookup(&cookie->stores, page->index);
76 if (!val) {
77 rcu_read_unlock();
78 fscache_stat(&fscache_n_store_vmscan_not_storing);
79 __fscache_uncache_page(cookie, page);
80 return true;
81 }
82
83 /* see if the page is actually undergoing storage - if so we can't get
84 * rid of it till the cache has finished with it */
85 if (radix_tree_tag_get(&cookie->stores, page->index,
86 FSCACHE_COOKIE_STORING_TAG)) {
87 rcu_read_unlock();
88 goto page_busy;
89 }
90
91 /* the page is pending storage, so we attempt to cancel the store and
92 * discard the store request so that the page can be reclaimed */
93 spin_lock(&cookie->stores_lock);
94 rcu_read_unlock();
95
96 if (radix_tree_tag_get(&cookie->stores, page->index,
97 FSCACHE_COOKIE_STORING_TAG)) {
98 /* the page started to undergo storage whilst we were looking,
99 * so now we can only wait or return */
100 spin_unlock(&cookie->stores_lock);
101 goto page_busy;
102 }
103
104 xpage = radix_tree_delete(&cookie->stores, page->index);
105 trace_fscache_page(cookie, page, fscache_page_radix_delete);
106 spin_unlock(&cookie->stores_lock);
107
108 if (xpage) {
109 fscache_stat(&fscache_n_store_vmscan_cancelled);
110 fscache_stat(&fscache_n_store_radix_deletes);
111 ASSERTCMP(xpage, ==, page);
112 } else {
113 fscache_stat(&fscache_n_store_vmscan_gone);
114 }
115
116 wake_up_bit(&cookie->flags, 0);
117 trace_fscache_wake_cookie(cookie);
118 if (xpage)
119 put_page(xpage);
120 __fscache_uncache_page(cookie, page);
121 return true;
122
123 page_busy:
124 /* We will wait here if we're allowed to, but that could deadlock the
125 * allocator as the work threads writing to the cache may all end up
126 * sleeping on memory allocation, so we may need to impose a timeout
127 * too. */
128 if (!(gfp & __GFP_DIRECT_RECLAIM) || !(gfp & __GFP_FS)) {
129 fscache_stat(&fscache_n_store_vmscan_busy);
130 return false;
131 }
132
133 fscache_stat(&fscache_n_store_vmscan_wait);
134 if (!release_page_wait_timeout(cookie, page))
135 _debug("fscache writeout timeout page: %p{%lx}",
136 page, page->index);
137
138 gfp &= ~__GFP_DIRECT_RECLAIM;
139 goto try_again;
140 }
141 EXPORT_SYMBOL(__fscache_maybe_release_page);
142
143 /*
144 * note that a page has finished being written to the cache
145 */
146 static void fscache_end_page_write(struct fscache_object *object,
147 struct page *page)
148 {
149 struct fscache_cookie *cookie;
150 struct page *xpage = NULL, *val;
151
152 spin_lock(&object->lock);
153 cookie = object->cookie;
154 if (cookie) {
155 /* delete the page from the tree if it is now no longer
156 * pending */
157 spin_lock(&cookie->stores_lock);
158 radix_tree_tag_clear(&cookie->stores, page->index,
159 FSCACHE_COOKIE_STORING_TAG);
160 trace_fscache_page(cookie, page, fscache_page_radix_clear_store);
161 if (!radix_tree_tag_get(&cookie->stores, page->index,
162 FSCACHE_COOKIE_PENDING_TAG)) {
163 fscache_stat(&fscache_n_store_radix_deletes);
164 xpage = radix_tree_delete(&cookie->stores, page->index);
165 trace_fscache_page(cookie, page, fscache_page_radix_delete);
166 trace_fscache_page(cookie, page, fscache_page_write_end);
167
168 val = radix_tree_lookup(&cookie->stores, page->index);
169 trace_fscache_check_page(cookie, page, val, 1);
170 } else {
171 trace_fscache_page(cookie, page, fscache_page_write_end_pend);
172 }
173 spin_unlock(&cookie->stores_lock);
174 wake_up_bit(&cookie->flags, 0);
175 trace_fscache_wake_cookie(cookie);
176 } else {
177 trace_fscache_page(cookie, page, fscache_page_write_end_noc);
178 }
179 spin_unlock(&object->lock);
180 if (xpage)
181 put_page(xpage);
182 }
183
184 /*
185 * actually apply the changed attributes to a cache object
186 */
187 static void fscache_attr_changed_op(struct fscache_operation *op)
188 {
189 struct fscache_object *object = op->object;
190 int ret;
191
192 _enter("{OBJ%x OP%x}", object->debug_id, op->debug_id);
193
194 fscache_stat(&fscache_n_attr_changed_calls);
195
196 if (fscache_object_is_active(object)) {
197 fscache_stat(&fscache_n_cop_attr_changed);
198 ret = object->cache->ops->attr_changed(object);
199 fscache_stat_d(&fscache_n_cop_attr_changed);
200 if (ret < 0)
201 fscache_abort_object(object);
202 fscache_op_complete(op, ret < 0);
203 } else {
204 fscache_op_complete(op, true);
205 }
206
207 _leave("");
208 }
209
210 /*
211 * notification that the attributes on an object have changed
212 */
213 int __fscache_attr_changed(struct fscache_cookie *cookie)
214 {
215 struct fscache_operation *op;
216 struct fscache_object *object;
217 bool wake_cookie = false;
218
219 _enter("%p", cookie);
220
221 ASSERTCMP(cookie->def->type, !=, FSCACHE_COOKIE_TYPE_INDEX);
222
223 fscache_stat(&fscache_n_attr_changed);
224
225 op = kzalloc(sizeof(*op), GFP_KERNEL);
226 if (!op) {
227 fscache_stat(&fscache_n_attr_changed_nomem);
228 _leave(" = -ENOMEM");
229 return -ENOMEM;
230 }
231
232 fscache_operation_init(cookie, op, fscache_attr_changed_op, NULL, NULL);
233 trace_fscache_page_op(cookie, NULL, op, fscache_page_op_attr_changed);
234 op->flags = FSCACHE_OP_ASYNC |
235 (1 << FSCACHE_OP_EXCLUSIVE) |
236 (1 << FSCACHE_OP_UNUSE_COOKIE);
237
238 spin_lock(&cookie->lock);
239
240 if (!fscache_cookie_enabled(cookie) ||
241 hlist_empty(&cookie->backing_objects))
242 goto nobufs;
243 object = hlist_entry(cookie->backing_objects.first,
244 struct fscache_object, cookie_link);
245
246 __fscache_use_cookie(cookie);
247 if (fscache_submit_exclusive_op(object, op) < 0)
248 goto nobufs_dec;
249 spin_unlock(&cookie->lock);
250 fscache_stat(&fscache_n_attr_changed_ok);
251 fscache_put_operation(op);
252 _leave(" = 0");
253 return 0;
254
255 nobufs_dec:
256 wake_cookie = __fscache_unuse_cookie(cookie);
257 nobufs:
258 spin_unlock(&cookie->lock);
259 fscache_put_operation(op);
260 if (wake_cookie)
261 __fscache_wake_unused_cookie(cookie);
262 fscache_stat(&fscache_n_attr_changed_nobufs);
263 _leave(" = %d", -ENOBUFS);
264 return -ENOBUFS;
265 }
266 EXPORT_SYMBOL(__fscache_attr_changed);
267
268 /*
269 * Handle cancellation of a pending retrieval op
270 */
271 static void fscache_do_cancel_retrieval(struct fscache_operation *_op)
272 {
273 struct fscache_retrieval *op =
274 container_of(_op, struct fscache_retrieval, op);
275
276 atomic_set(&op->n_pages, 0);
277 }
278
279 /*
280 * release a retrieval op reference
281 */
282 static void fscache_release_retrieval_op(struct fscache_operation *_op)
283 {
284 struct fscache_retrieval *op =
285 container_of(_op, struct fscache_retrieval, op);
286
287 _enter("{OP%x}", op->op.debug_id);
288
289 ASSERTIFCMP(op->op.state != FSCACHE_OP_ST_INITIALISED,
290 atomic_read(&op->n_pages), ==, 0);
291
292 if (op->context)
293 fscache_put_context(op->cookie, op->context);
294
295 _leave("");
296 }
297
298 /*
299 * allocate a retrieval op
300 */
301 struct fscache_retrieval *fscache_alloc_retrieval(
302 struct fscache_cookie *cookie,
303 struct address_space *mapping,
304 fscache_rw_complete_t end_io_func,
305 void *context)
306 {
307 struct fscache_retrieval *op;
308
309 /* allocate a retrieval operation and attempt to submit it */
310 op = kzalloc(sizeof(*op), GFP_NOIO);
311 if (!op) {
312 fscache_stat(&fscache_n_retrievals_nomem);
313 return NULL;
314 }
315
316 fscache_operation_init(cookie, &op->op, NULL,
317 fscache_do_cancel_retrieval,
318 fscache_release_retrieval_op);
319 op->op.flags = FSCACHE_OP_MYTHREAD |
320 (1UL << FSCACHE_OP_WAITING) |
321 (1UL << FSCACHE_OP_UNUSE_COOKIE);
322 op->cookie = cookie;
323 op->mapping = mapping;
324 op->end_io_func = end_io_func;
325 op->context = context;
326 INIT_LIST_HEAD(&op->to_do);
327
328 /* Pin the netfs read context in case we need to do the actual netfs
329 * read because we've encountered a cache read failure.
330 */
331 if (context)
332 fscache_get_context(op->cookie, context);
333 return op;
334 }
335
336 /*
337 * wait for a deferred lookup to complete
338 */
339 int fscache_wait_for_deferred_lookup(struct fscache_cookie *cookie)
340 {
341 _enter("");
342
343 if (!test_bit(FSCACHE_COOKIE_LOOKING_UP, &cookie->flags)) {
344 _leave(" = 0 [imm]");
345 return 0;
346 }
347
348 fscache_stat(&fscache_n_retrievals_wait);
349
350 if (wait_on_bit(&cookie->flags, FSCACHE_COOKIE_LOOKING_UP,
351 TASK_INTERRUPTIBLE) != 0) {
352 fscache_stat(&fscache_n_retrievals_intr);
353 _leave(" = -ERESTARTSYS");
354 return -ERESTARTSYS;
355 }
356
357 ASSERT(!test_bit(FSCACHE_COOKIE_LOOKING_UP, &cookie->flags));
358
359 smp_rmb();
360 _leave(" = 0 [dly]");
361 return 0;
362 }
363
364 /*
365 * wait for an object to become active (or dead)
366 */
367 int fscache_wait_for_operation_activation(struct fscache_object *object,
368 struct fscache_operation *op,
369 atomic_t *stat_op_waits,
370 atomic_t *stat_object_dead)
371 {
372 int ret;
373
374 if (!test_bit(FSCACHE_OP_WAITING, &op->flags))
375 goto check_if_dead;
376
377 _debug(">>> WT");
378 if (stat_op_waits)
379 fscache_stat(stat_op_waits);
380 if (wait_on_bit(&op->flags, FSCACHE_OP_WAITING,
381 TASK_INTERRUPTIBLE) != 0) {
382 trace_fscache_op(object->cookie, op, fscache_op_signal);
383 ret = fscache_cancel_op(op, false);
384 if (ret == 0)
385 return -ERESTARTSYS;
386
387 /* it's been removed from the pending queue by another party,
388 * so we should get to run shortly */
389 wait_on_bit(&op->flags, FSCACHE_OP_WAITING,
390 TASK_UNINTERRUPTIBLE);
391 }
392 _debug("<<< GO");
393
394 check_if_dead:
395 if (op->state == FSCACHE_OP_ST_CANCELLED) {
396 if (stat_object_dead)
397 fscache_stat(stat_object_dead);
398 _leave(" = -ENOBUFS [cancelled]");
399 return -ENOBUFS;
400 }
401 if (unlikely(fscache_object_is_dying(object) ||
402 fscache_cache_is_broken(object))) {
403 enum fscache_operation_state state = op->state;
404 trace_fscache_op(object->cookie, op, fscache_op_signal);
405 fscache_cancel_op(op, true);
406 if (stat_object_dead)
407 fscache_stat(stat_object_dead);
408 _leave(" = -ENOBUFS [obj dead %d]", state);
409 return -ENOBUFS;
410 }
411 return 0;
412 }
413
414 /*
415 * read a page from the cache or allocate a block in which to store it
416 * - we return:
417 * -ENOMEM - out of memory, nothing done
418 * -ERESTARTSYS - interrupted
419 * -ENOBUFS - no backing object available in which to cache the block
420 * -ENODATA - no data available in the backing object for this block
421 * 0 - dispatched a read - it'll call end_io_func() when finished
422 */
423 int __fscache_read_or_alloc_page(struct fscache_cookie *cookie,
424 struct page *page,
425 fscache_rw_complete_t end_io_func,
426 void *context,
427 gfp_t gfp)
428 {
429 struct fscache_retrieval *op;
430 struct fscache_object *object;
431 bool wake_cookie = false;
432 int ret;
433
434 _enter("%p,%p,,,", cookie, page);
435
436 fscache_stat(&fscache_n_retrievals);
437
438 if (hlist_empty(&cookie->backing_objects))
439 goto nobufs;
440
441 if (test_bit(FSCACHE_COOKIE_INVALIDATING, &cookie->flags)) {
442 _leave(" = -ENOBUFS [invalidating]");
443 return -ENOBUFS;
444 }
445
446 ASSERTCMP(cookie->def->type, !=, FSCACHE_COOKIE_TYPE_INDEX);
447 ASSERTCMP(page, !=, NULL);
448
449 if (fscache_wait_for_deferred_lookup(cookie) < 0)
450 return -ERESTARTSYS;
451
452 op = fscache_alloc_retrieval(cookie, page->mapping,
453 end_io_func, context);
454 if (!op) {
455 _leave(" = -ENOMEM");
456 return -ENOMEM;
457 }
458 atomic_set(&op->n_pages, 1);
459 trace_fscache_page_op(cookie, page, &op->op, fscache_page_op_retr_one);
460
461 spin_lock(&cookie->lock);
462
463 if (!fscache_cookie_enabled(cookie) ||
464 hlist_empty(&cookie->backing_objects))
465 goto nobufs_unlock;
466 object = hlist_entry(cookie->backing_objects.first,
467 struct fscache_object, cookie_link);
468
469 ASSERT(test_bit(FSCACHE_OBJECT_IS_LOOKED_UP, &object->flags));
470
471 __fscache_use_cookie(cookie);
472 atomic_inc(&object->n_reads);
473 __set_bit(FSCACHE_OP_DEC_READ_CNT, &op->op.flags);
474
475 if (fscache_submit_op(object, &op->op) < 0)
476 goto nobufs_unlock_dec;
477 spin_unlock(&cookie->lock);
478
479 fscache_stat(&fscache_n_retrieval_ops);
480
481 /* we wait for the operation to become active, and then process it
482 * *here*, in this thread, and not in the thread pool */
483 ret = fscache_wait_for_operation_activation(
484 object, &op->op,
485 __fscache_stat(&fscache_n_retrieval_op_waits),
486 __fscache_stat(&fscache_n_retrievals_object_dead));
487 if (ret < 0)
488 goto error;
489
490 /* ask the cache to honour the operation */
491 if (test_bit(FSCACHE_COOKIE_NO_DATA_YET, &object->cookie->flags)) {
492 fscache_stat(&fscache_n_cop_allocate_page);
493 ret = object->cache->ops->allocate_page(op, page, gfp);
494 fscache_stat_d(&fscache_n_cop_allocate_page);
495 if (ret == 0)
496 ret = -ENODATA;
497 } else {
498 fscache_stat(&fscache_n_cop_read_or_alloc_page);
499 ret = object->cache->ops->read_or_alloc_page(op, page, gfp);
500 fscache_stat_d(&fscache_n_cop_read_or_alloc_page);
501 }
502
503 error:
504 if (ret == -ENOMEM)
505 fscache_stat(&fscache_n_retrievals_nomem);
506 else if (ret == -ERESTARTSYS)
507 fscache_stat(&fscache_n_retrievals_intr);
508 else if (ret == -ENODATA)
509 fscache_stat(&fscache_n_retrievals_nodata);
510 else if (ret < 0)
511 fscache_stat(&fscache_n_retrievals_nobufs);
512 else
513 fscache_stat(&fscache_n_retrievals_ok);
514
515 fscache_put_retrieval(op);
516 _leave(" = %d", ret);
517 return ret;
518
519 nobufs_unlock_dec:
520 atomic_dec(&object->n_reads);
521 wake_cookie = __fscache_unuse_cookie(cookie);
522 nobufs_unlock:
523 spin_unlock(&cookie->lock);
524 if (wake_cookie)
525 __fscache_wake_unused_cookie(cookie);
526 fscache_put_retrieval(op);
527 nobufs:
528 fscache_stat(&fscache_n_retrievals_nobufs);
529 _leave(" = -ENOBUFS");
530 return -ENOBUFS;
531 }
532 EXPORT_SYMBOL(__fscache_read_or_alloc_page);
533
534 /*
535 * read a list of page from the cache or allocate a block in which to store
536 * them
537 * - we return:
538 * -ENOMEM - out of memory, some pages may be being read
539 * -ERESTARTSYS - interrupted, some pages may be being read
540 * -ENOBUFS - no backing object or space available in which to cache any
541 * pages not being read
542 * -ENODATA - no data available in the backing object for some or all of
543 * the pages
544 * 0 - dispatched a read on all pages
545 *
546 * end_io_func() will be called for each page read from the cache as it is
547 * finishes being read
548 *
549 * any pages for which a read is dispatched will be removed from pages and
550 * nr_pages
551 */
552 int __fscache_read_or_alloc_pages(struct fscache_cookie *cookie,
553 struct address_space *mapping,
554 struct list_head *pages,
555 unsigned *nr_pages,
556 fscache_rw_complete_t end_io_func,
557 void *context,
558 gfp_t gfp)
559 {
560 struct fscache_retrieval *op;
561 struct fscache_object *object;
562 bool wake_cookie = false;
563 int ret;
564
565 _enter("%p,,%d,,,", cookie, *nr_pages);
566
567 fscache_stat(&fscache_n_retrievals);
568
569 if (hlist_empty(&cookie->backing_objects))
570 goto nobufs;
571
572 if (test_bit(FSCACHE_COOKIE_INVALIDATING, &cookie->flags)) {
573 _leave(" = -ENOBUFS [invalidating]");
574 return -ENOBUFS;
575 }
576
577 ASSERTCMP(cookie->def->type, !=, FSCACHE_COOKIE_TYPE_INDEX);
578 ASSERTCMP(*nr_pages, >, 0);
579 ASSERT(!list_empty(pages));
580
581 if (fscache_wait_for_deferred_lookup(cookie) < 0)
582 return -ERESTARTSYS;
583
584 op = fscache_alloc_retrieval(cookie, mapping, end_io_func, context);
585 if (!op)
586 return -ENOMEM;
587 atomic_set(&op->n_pages, *nr_pages);
588 trace_fscache_page_op(cookie, NULL, &op->op, fscache_page_op_retr_multi);
589
590 spin_lock(&cookie->lock);
591
592 if (!fscache_cookie_enabled(cookie) ||
593 hlist_empty(&cookie->backing_objects))
594 goto nobufs_unlock;
595 object = hlist_entry(cookie->backing_objects.first,
596 struct fscache_object, cookie_link);
597
598 __fscache_use_cookie(cookie);
599 atomic_inc(&object->n_reads);
600 __set_bit(FSCACHE_OP_DEC_READ_CNT, &op->op.flags);
601
602 if (fscache_submit_op(object, &op->op) < 0)
603 goto nobufs_unlock_dec;
604 spin_unlock(&cookie->lock);
605
606 fscache_stat(&fscache_n_retrieval_ops);
607
608 /* we wait for the operation to become active, and then process it
609 * *here*, in this thread, and not in the thread pool */
610 ret = fscache_wait_for_operation_activation(
611 object, &op->op,
612 __fscache_stat(&fscache_n_retrieval_op_waits),
613 __fscache_stat(&fscache_n_retrievals_object_dead));
614 if (ret < 0)
615 goto error;
616
617 /* ask the cache to honour the operation */
618 if (test_bit(FSCACHE_COOKIE_NO_DATA_YET, &object->cookie->flags)) {
619 fscache_stat(&fscache_n_cop_allocate_pages);
620 ret = object->cache->ops->allocate_pages(
621 op, pages, nr_pages, gfp);
622 fscache_stat_d(&fscache_n_cop_allocate_pages);
623 } else {
624 fscache_stat(&fscache_n_cop_read_or_alloc_pages);
625 ret = object->cache->ops->read_or_alloc_pages(
626 op, pages, nr_pages, gfp);
627 fscache_stat_d(&fscache_n_cop_read_or_alloc_pages);
628 }
629
630 error:
631 if (ret == -ENOMEM)
632 fscache_stat(&fscache_n_retrievals_nomem);
633 else if (ret == -ERESTARTSYS)
634 fscache_stat(&fscache_n_retrievals_intr);
635 else if (ret == -ENODATA)
636 fscache_stat(&fscache_n_retrievals_nodata);
637 else if (ret < 0)
638 fscache_stat(&fscache_n_retrievals_nobufs);
639 else
640 fscache_stat(&fscache_n_retrievals_ok);
641
642 fscache_put_retrieval(op);
643 _leave(" = %d", ret);
644 return ret;
645
646 nobufs_unlock_dec:
647 atomic_dec(&object->n_reads);
648 wake_cookie = __fscache_unuse_cookie(cookie);
649 nobufs_unlock:
650 spin_unlock(&cookie->lock);
651 fscache_put_retrieval(op);
652 if (wake_cookie)
653 __fscache_wake_unused_cookie(cookie);
654 nobufs:
655 fscache_stat(&fscache_n_retrievals_nobufs);
656 _leave(" = -ENOBUFS");
657 return -ENOBUFS;
658 }
659 EXPORT_SYMBOL(__fscache_read_or_alloc_pages);
660
661 /*
662 * allocate a block in the cache on which to store a page
663 * - we return:
664 * -ENOMEM - out of memory, nothing done
665 * -ERESTARTSYS - interrupted
666 * -ENOBUFS - no backing object available in which to cache the block
667 * 0 - block allocated
668 */
669 int __fscache_alloc_page(struct fscache_cookie *cookie,
670 struct page *page,
671 gfp_t gfp)
672 {
673 struct fscache_retrieval *op;
674 struct fscache_object *object;
675 bool wake_cookie = false;
676 int ret;
677
678 _enter("%p,%p,,,", cookie, page);
679
680 fscache_stat(&fscache_n_allocs);
681
682 if (hlist_empty(&cookie->backing_objects))
683 goto nobufs;
684
685 ASSERTCMP(cookie->def->type, !=, FSCACHE_COOKIE_TYPE_INDEX);
686 ASSERTCMP(page, !=, NULL);
687
688 if (test_bit(FSCACHE_COOKIE_INVALIDATING, &cookie->flags)) {
689 _leave(" = -ENOBUFS [invalidating]");
690 return -ENOBUFS;
691 }
692
693 if (fscache_wait_for_deferred_lookup(cookie) < 0)
694 return -ERESTARTSYS;
695
696 op = fscache_alloc_retrieval(cookie, page->mapping, NULL, NULL);
697 if (!op)
698 return -ENOMEM;
699 atomic_set(&op->n_pages, 1);
700 trace_fscache_page_op(cookie, page, &op->op, fscache_page_op_alloc_one);
701
702 spin_lock(&cookie->lock);
703
704 if (!fscache_cookie_enabled(cookie) ||
705 hlist_empty(&cookie->backing_objects))
706 goto nobufs_unlock;
707 object = hlist_entry(cookie->backing_objects.first,
708 struct fscache_object, cookie_link);
709
710 __fscache_use_cookie(cookie);
711 if (fscache_submit_op(object, &op->op) < 0)
712 goto nobufs_unlock_dec;
713 spin_unlock(&cookie->lock);
714
715 fscache_stat(&fscache_n_alloc_ops);
716
717 ret = fscache_wait_for_operation_activation(
718 object, &op->op,
719 __fscache_stat(&fscache_n_alloc_op_waits),
720 __fscache_stat(&fscache_n_allocs_object_dead));
721 if (ret < 0)
722 goto error;
723
724 /* ask the cache to honour the operation */
725 fscache_stat(&fscache_n_cop_allocate_page);
726 ret = object->cache->ops->allocate_page(op, page, gfp);
727 fscache_stat_d(&fscache_n_cop_allocate_page);
728
729 error:
730 if (ret == -ERESTARTSYS)
731 fscache_stat(&fscache_n_allocs_intr);
732 else if (ret < 0)
733 fscache_stat(&fscache_n_allocs_nobufs);
734 else
735 fscache_stat(&fscache_n_allocs_ok);
736
737 fscache_put_retrieval(op);
738 _leave(" = %d", ret);
739 return ret;
740
741 nobufs_unlock_dec:
742 wake_cookie = __fscache_unuse_cookie(cookie);
743 nobufs_unlock:
744 spin_unlock(&cookie->lock);
745 fscache_put_retrieval(op);
746 if (wake_cookie)
747 __fscache_wake_unused_cookie(cookie);
748 nobufs:
749 fscache_stat(&fscache_n_allocs_nobufs);
750 _leave(" = -ENOBUFS");
751 return -ENOBUFS;
752 }
753 EXPORT_SYMBOL(__fscache_alloc_page);
754
755 /*
756 * Unmark pages allocate in the readahead code path (via:
757 * fscache_readpages_or_alloc) after delegating to the base filesystem
758 */
759 void __fscache_readpages_cancel(struct fscache_cookie *cookie,
760 struct list_head *pages)
761 {
762 struct page *page;
763
764 list_for_each_entry(page, pages, lru) {
765 if (PageFsCache(page))
766 __fscache_uncache_page(cookie, page);
767 }
768 }
769 EXPORT_SYMBOL(__fscache_readpages_cancel);
770
771 /*
772 * release a write op reference
773 */
774 static void fscache_release_write_op(struct fscache_operation *_op)
775 {
776 _enter("{OP%x}", _op->debug_id);
777 }
778
779 /*
780 * perform the background storage of a page into the cache
781 */
782 static void fscache_write_op(struct fscache_operation *_op)
783 {
784 struct fscache_storage *op =
785 container_of(_op, struct fscache_storage, op);
786 struct fscache_object *object = op->op.object;
787 struct fscache_cookie *cookie;
788 struct page *page;
789 unsigned n;
790 void *results[1];
791 int ret;
792
793 _enter("{OP%x,%d}", op->op.debug_id, atomic_read(&op->op.usage));
794
795 again:
796 spin_lock(&object->lock);
797 cookie = object->cookie;
798
799 if (!fscache_object_is_active(object)) {
800 /* If we get here, then the on-disk cache object likely no
801 * longer exists, so we should just cancel this write
802 * operation.
803 */
804 spin_unlock(&object->lock);
805 fscache_op_complete(&op->op, true);
806 _leave(" [inactive]");
807 return;
808 }
809
810 if (!cookie) {
811 /* If we get here, then the cookie belonging to the object was
812 * detached, probably by the cookie being withdrawn due to
813 * memory pressure, which means that the pages we might write
814 * to the cache from no longer exist - therefore, we can just
815 * cancel this write operation.
816 */
817 spin_unlock(&object->lock);
818 fscache_op_complete(&op->op, true);
819 _leave(" [cancel] op{f=%lx s=%u} obj{s=%s f=%lx}",
820 _op->flags, _op->state, object->state->short_name,
821 object->flags);
822 return;
823 }
824
825 spin_lock(&cookie->stores_lock);
826
827 fscache_stat(&fscache_n_store_calls);
828
829 /* find a page to store */
830 results[0] = NULL;
831 page = NULL;
832 n = radix_tree_gang_lookup_tag(&cookie->stores, results, 0, 1,
833 FSCACHE_COOKIE_PENDING_TAG);
834 trace_fscache_gang_lookup(cookie, &op->op, results, n, op->store_limit);
835 if (n != 1)
836 goto superseded;
837 page = results[0];
838 _debug("gang %d [%lx]", n, page->index);
839
840 radix_tree_tag_set(&cookie->stores, page->index,
841 FSCACHE_COOKIE_STORING_TAG);
842 radix_tree_tag_clear(&cookie->stores, page->index,
843 FSCACHE_COOKIE_PENDING_TAG);
844 trace_fscache_page(cookie, page, fscache_page_radix_pend2store);
845
846 spin_unlock(&cookie->stores_lock);
847 spin_unlock(&object->lock);
848
849 if (page->index >= op->store_limit)
850 goto discard_page;
851
852 fscache_stat(&fscache_n_store_pages);
853 fscache_stat(&fscache_n_cop_write_page);
854 ret = object->cache->ops->write_page(op, page);
855 fscache_stat_d(&fscache_n_cop_write_page);
856 trace_fscache_wrote_page(cookie, page, &op->op, ret);
857 fscache_end_page_write(object, page);
858 if (ret < 0) {
859 fscache_abort_object(object);
860 fscache_op_complete(&op->op, true);
861 } else {
862 fscache_enqueue_operation(&op->op);
863 }
864
865 _leave("");
866 return;
867
868 discard_page:
869 fscache_stat(&fscache_n_store_pages_over_limit);
870 trace_fscache_wrote_page(cookie, page, &op->op, -ENOBUFS);
871 fscache_end_page_write(object, page);
872 goto again;
873
874 superseded:
875 /* this writer is going away and there aren't any more things to
876 * write */
877 _debug("cease");
878 spin_unlock(&cookie->stores_lock);
879 clear_bit(FSCACHE_OBJECT_PENDING_WRITE, &object->flags);
880 spin_unlock(&object->lock);
881 fscache_op_complete(&op->op, false);
882 _leave("");
883 }
884
885 /*
886 * Clear the pages pending writing for invalidation
887 */
888 void fscache_invalidate_writes(struct fscache_cookie *cookie)
889 {
890 struct page *page;
891 void *results[16];
892 int n, i;
893
894 _enter("");
895
896 for (;;) {
897 spin_lock(&cookie->stores_lock);
898 n = radix_tree_gang_lookup_tag(&cookie->stores, results, 0,
899 ARRAY_SIZE(results),
900 FSCACHE_COOKIE_PENDING_TAG);
901 if (n == 0) {
902 spin_unlock(&cookie->stores_lock);
903 break;
904 }
905
906 for (i = n - 1; i >= 0; i--) {
907 page = results[i];
908 radix_tree_delete(&cookie->stores, page->index);
909 trace_fscache_page(cookie, page, fscache_page_radix_delete);
910 trace_fscache_page(cookie, page, fscache_page_inval);
911 }
912
913 spin_unlock(&cookie->stores_lock);
914
915 for (i = n - 1; i >= 0; i--)
916 put_page(results[i]);
917 }
918
919 wake_up_bit(&cookie->flags, 0);
920 trace_fscache_wake_cookie(cookie);
921
922 _leave("");
923 }
924
925 /*
926 * request a page be stored in the cache
927 * - returns:
928 * -ENOMEM - out of memory, nothing done
929 * -ENOBUFS - no backing object available in which to cache the page
930 * 0 - dispatched a write - it'll call end_io_func() when finished
931 *
932 * if the cookie still has a backing object at this point, that object can be
933 * in one of a few states with respect to storage processing:
934 *
935 * (1) negative lookup, object not yet created (FSCACHE_COOKIE_CREATING is
936 * set)
937 *
938 * (a) no writes yet
939 *
940 * (b) writes deferred till post-creation (mark page for writing and
941 * return immediately)
942 *
943 * (2) negative lookup, object created, initial fill being made from netfs
944 *
945 * (a) fill point not yet reached this page (mark page for writing and
946 * return)
947 *
948 * (b) fill point passed this page (queue op to store this page)
949 *
950 * (3) object extant (queue op to store this page)
951 *
952 * any other state is invalid
953 */
954 int __fscache_write_page(struct fscache_cookie *cookie,
955 struct page *page,
956 loff_t object_size,
957 gfp_t gfp)
958 {
959 struct fscache_storage *op;
960 struct fscache_object *object;
961 bool wake_cookie = false;
962 int ret;
963
964 _enter("%p,%x,", cookie, (u32) page->flags);
965
966 ASSERTCMP(cookie->def->type, !=, FSCACHE_COOKIE_TYPE_INDEX);
967 ASSERT(PageFsCache(page));
968
969 fscache_stat(&fscache_n_stores);
970
971 if (test_bit(FSCACHE_COOKIE_INVALIDATING, &cookie->flags)) {
972 _leave(" = -ENOBUFS [invalidating]");
973 return -ENOBUFS;
974 }
975
976 op = kzalloc(sizeof(*op), GFP_NOIO | __GFP_NOMEMALLOC | __GFP_NORETRY);
977 if (!op)
978 goto nomem;
979
980 fscache_operation_init(cookie, &op->op, fscache_write_op, NULL,
981 fscache_release_write_op);
982 op->op.flags = FSCACHE_OP_ASYNC |
983 (1 << FSCACHE_OP_WAITING) |
984 (1 << FSCACHE_OP_UNUSE_COOKIE);
985
986 ret = radix_tree_maybe_preload(gfp & ~__GFP_HIGHMEM);
987 if (ret < 0)
988 goto nomem_free;
989
990 trace_fscache_page_op(cookie, page, &op->op, fscache_page_op_write_one);
991
992 ret = -ENOBUFS;
993 spin_lock(&cookie->lock);
994
995 if (!fscache_cookie_enabled(cookie) ||
996 hlist_empty(&cookie->backing_objects))
997 goto nobufs;
998 object = hlist_entry(cookie->backing_objects.first,
999 struct fscache_object, cookie_link);
1000 if (test_bit(FSCACHE_IOERROR, &object->cache->flags))
1001 goto nobufs;
1002
1003 trace_fscache_page(cookie, page, fscache_page_write);
1004
1005 /* add the page to the pending-storage radix tree on the backing
1006 * object */
1007 spin_lock(&object->lock);
1008
1009 if (object->store_limit_l != object_size)
1010 fscache_set_store_limit(object, object_size);
1011
1012 spin_lock(&cookie->stores_lock);
1013
1014 _debug("store limit %llx", (unsigned long long) object->store_limit);
1015
1016 ret = radix_tree_insert(&cookie->stores, page->index, page);
1017 if (ret < 0) {
1018 if (ret == -EEXIST)
1019 goto already_queued;
1020 _debug("insert failed %d", ret);
1021 goto nobufs_unlock_obj;
1022 }
1023
1024 trace_fscache_page(cookie, page, fscache_page_radix_insert);
1025 radix_tree_tag_set(&cookie->stores, page->index,
1026 FSCACHE_COOKIE_PENDING_TAG);
1027 trace_fscache_page(cookie, page, fscache_page_radix_set_pend);
1028 get_page(page);
1029
1030 /* we only want one writer at a time, but we do need to queue new
1031 * writers after exclusive ops */
1032 if (test_and_set_bit(FSCACHE_OBJECT_PENDING_WRITE, &object->flags))
1033 goto already_pending;
1034
1035 spin_unlock(&cookie->stores_lock);
1036 spin_unlock(&object->lock);
1037
1038 op->op.debug_id = atomic_inc_return(&fscache_op_debug_id);
1039 op->store_limit = object->store_limit;
1040
1041 __fscache_use_cookie(cookie);
1042 if (fscache_submit_op(object, &op->op) < 0)
1043 goto submit_failed;
1044
1045 spin_unlock(&cookie->lock);
1046 radix_tree_preload_end();
1047 fscache_stat(&fscache_n_store_ops);
1048 fscache_stat(&fscache_n_stores_ok);
1049
1050 /* the work queue now carries its own ref on the object */
1051 fscache_put_operation(&op->op);
1052 _leave(" = 0");
1053 return 0;
1054
1055 already_queued:
1056 fscache_stat(&fscache_n_stores_again);
1057 already_pending:
1058 spin_unlock(&cookie->stores_lock);
1059 spin_unlock(&object->lock);
1060 spin_unlock(&cookie->lock);
1061 radix_tree_preload_end();
1062 fscache_put_operation(&op->op);
1063 fscache_stat(&fscache_n_stores_ok);
1064 _leave(" = 0");
1065 return 0;
1066
1067 submit_failed:
1068 spin_lock(&cookie->stores_lock);
1069 radix_tree_delete(&cookie->stores, page->index);
1070 trace_fscache_page(cookie, page, fscache_page_radix_delete);
1071 spin_unlock(&cookie->stores_lock);
1072 wake_cookie = __fscache_unuse_cookie(cookie);
1073 put_page(page);
1074 ret = -ENOBUFS;
1075 goto nobufs;
1076
1077 nobufs_unlock_obj:
1078 spin_unlock(&cookie->stores_lock);
1079 spin_unlock(&object->lock);
1080 nobufs:
1081 spin_unlock(&cookie->lock);
1082 radix_tree_preload_end();
1083 fscache_put_operation(&op->op);
1084 if (wake_cookie)
1085 __fscache_wake_unused_cookie(cookie);
1086 fscache_stat(&fscache_n_stores_nobufs);
1087 _leave(" = -ENOBUFS");
1088 return -ENOBUFS;
1089
1090 nomem_free:
1091 fscache_put_operation(&op->op);
1092 nomem:
1093 fscache_stat(&fscache_n_stores_oom);
1094 _leave(" = -ENOMEM");
1095 return -ENOMEM;
1096 }
1097 EXPORT_SYMBOL(__fscache_write_page);
1098
1099 /*
1100 * remove a page from the cache
1101 */
1102 void __fscache_uncache_page(struct fscache_cookie *cookie, struct page *page)
1103 {
1104 struct fscache_object *object;
1105
1106 _enter(",%p", page);
1107
1108 ASSERTCMP(cookie->def->type, !=, FSCACHE_COOKIE_TYPE_INDEX);
1109 ASSERTCMP(page, !=, NULL);
1110
1111 fscache_stat(&fscache_n_uncaches);
1112
1113 /* cache withdrawal may beat us to it */
1114 if (!PageFsCache(page))
1115 goto done;
1116
1117 trace_fscache_page(cookie, page, fscache_page_uncache);
1118
1119 /* get the object */
1120 spin_lock(&cookie->lock);
1121
1122 if (hlist_empty(&cookie->backing_objects)) {
1123 ClearPageFsCache(page);
1124 goto done_unlock;
1125 }
1126
1127 object = hlist_entry(cookie->backing_objects.first,
1128 struct fscache_object, cookie_link);
1129
1130 /* there might now be stuff on disk we could read */
1131 clear_bit(FSCACHE_COOKIE_NO_DATA_YET, &cookie->flags);
1132
1133 /* only invoke the cache backend if we managed to mark the page
1134 * uncached here; this deals with synchronisation vs withdrawal */
1135 if (TestClearPageFsCache(page) &&
1136 object->cache->ops->uncache_page) {
1137 /* the cache backend releases the cookie lock */
1138 fscache_stat(&fscache_n_cop_uncache_page);
1139 object->cache->ops->uncache_page(object, page);
1140 fscache_stat_d(&fscache_n_cop_uncache_page);
1141 goto done;
1142 }
1143
1144 done_unlock:
1145 spin_unlock(&cookie->lock);
1146 done:
1147 _leave("");
1148 }
1149 EXPORT_SYMBOL(__fscache_uncache_page);
1150
1151 /**
1152 * fscache_mark_page_cached - Mark a page as being cached
1153 * @op: The retrieval op pages are being marked for
1154 * @page: The page to be marked
1155 *
1156 * Mark a netfs page as being cached. After this is called, the netfs
1157 * must call fscache_uncache_page() to remove the mark.
1158 */
1159 void fscache_mark_page_cached(struct fscache_retrieval *op, struct page *page)
1160 {
1161 struct fscache_cookie *cookie = op->op.object->cookie;
1162
1163 #ifdef CONFIG_FSCACHE_STATS
1164 atomic_inc(&fscache_n_marks);
1165 #endif
1166
1167 trace_fscache_page(cookie, page, fscache_page_cached);
1168
1169 _debug("- mark %p{%lx}", page, page->index);
1170 if (TestSetPageFsCache(page)) {
1171 static bool once_only;
1172 if (!once_only) {
1173 once_only = true;
1174 pr_warn("Cookie type %s marked page %lx multiple times\n",
1175 cookie->def->name, page->index);
1176 }
1177 }
1178
1179 if (cookie->def->mark_page_cached)
1180 cookie->def->mark_page_cached(cookie->netfs_data,
1181 op->mapping, page);
1182 }
1183 EXPORT_SYMBOL(fscache_mark_page_cached);
1184
1185 /**
1186 * fscache_mark_pages_cached - Mark pages as being cached
1187 * @op: The retrieval op pages are being marked for
1188 * @pagevec: The pages to be marked
1189 *
1190 * Mark a bunch of netfs pages as being cached. After this is called,
1191 * the netfs must call fscache_uncache_page() to remove the mark.
1192 */
1193 void fscache_mark_pages_cached(struct fscache_retrieval *op,
1194 struct pagevec *pagevec)
1195 {
1196 unsigned long loop;
1197
1198 for (loop = 0; loop < pagevec->nr; loop++)
1199 fscache_mark_page_cached(op, pagevec->pages[loop]);
1200
1201 pagevec_reinit(pagevec);
1202 }
1203 EXPORT_SYMBOL(fscache_mark_pages_cached);
1204
1205 /*
1206 * Uncache all the pages in an inode that are marked PG_fscache, assuming them
1207 * to be associated with the given cookie.
1208 */
1209 void __fscache_uncache_all_inode_pages(struct fscache_cookie *cookie,
1210 struct inode *inode)
1211 {
1212 struct address_space *mapping = inode->i_mapping;
1213 struct pagevec pvec;
1214 pgoff_t next;
1215 int i;
1216
1217 _enter("%p,%p", cookie, inode);
1218
1219 if (!mapping || mapping->nrpages == 0) {
1220 _leave(" [no pages]");
1221 return;
1222 }
1223
1224 pagevec_init(&pvec);
1225 next = 0;
1226 do {
1227 if (!pagevec_lookup(&pvec, mapping, &next))
1228 break;
1229 for (i = 0; i < pagevec_count(&pvec); i++) {
1230 struct page *page = pvec.pages[i];
1231 if (PageFsCache(page)) {
1232 __fscache_wait_on_page_write(cookie, page);
1233 __fscache_uncache_page(cookie, page);
1234 }
1235 }
1236 pagevec_release(&pvec);
1237 cond_resched();
1238 } while (next);
1239
1240 _leave("");
1241 }
1242 EXPORT_SYMBOL(__fscache_uncache_all_inode_pages);
Ubuntu Jammy Linux kernel mirror