]> git.proxmox.com Git - mirror_ubuntu-hirsute-kernel.git/blob - fs/f2fs/data.c
projects / mirror_ubuntu-hirsute-kernel.git / blob
? search:


4d3ebf094f6d7fa730878468435d5380bfad9993
[mirror_ubuntu-hirsute-kernel.git] / fs / f2fs / data.c
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * fs/f2fs/data.c
4 *
5 * Copyright (c) 2012 Samsung Electronics Co., Ltd.
6 * http://www.samsung.com/
7 */
8 #include <linux/fs.h>
9 #include <linux/f2fs_fs.h>
10 #include <linux/buffer_head.h>
11 #include <linux/mpage.h>
12 #include <linux/writeback.h>
13 #include <linux/backing-dev.h>
14 #include <linux/pagevec.h>
15 #include <linux/blkdev.h>
16 #include <linux/bio.h>
17 #include <linux/blk-crypto.h>
18 #include <linux/swap.h>
19 #include <linux/prefetch.h>
20 #include <linux/uio.h>
21 #include <linux/cleancache.h>
22 #include <linux/sched/signal.h>
23 #include <linux/fiemap.h>
24
25 #include "f2fs.h"
26 #include "node.h"
27 #include "segment.h"
28 #include "trace.h"
29 #include <trace/events/f2fs.h>
30
31 #define NUM_PREALLOC_POST_READ_CTXS 128
32
33 static struct kmem_cache *bio_post_read_ctx_cache;
34 static struct kmem_cache *bio_entry_slab;
35 static mempool_t *bio_post_read_ctx_pool;
36 static struct bio_set f2fs_bioset;
37
38 #define F2FS_BIO_POOL_SIZE NR_CURSEG_TYPE
39
40 int __init f2fs_init_bioset(void)
41 {
42 if (bioset_init(&f2fs_bioset, F2FS_BIO_POOL_SIZE,
43 0, BIOSET_NEED_BVECS))
44 return -ENOMEM;
45 return 0;
46 }
47
48 void f2fs_destroy_bioset(void)
49 {
50 bioset_exit(&f2fs_bioset);
51 }
52
53 static inline struct bio *__f2fs_bio_alloc(gfp_t gfp_mask,
54 unsigned int nr_iovecs)
55 {
56 return bio_alloc_bioset(gfp_mask, nr_iovecs, &f2fs_bioset);
57 }
58
59 struct bio *f2fs_bio_alloc(struct f2fs_sb_info *sbi, int npages, bool noio)
60 {
61 if (noio) {
62 /* No failure on bio allocation */
63 return __f2fs_bio_alloc(GFP_NOIO, npages);
64 }
65
66 if (time_to_inject(sbi, FAULT_ALLOC_BIO)) {
67 f2fs_show_injection_info(sbi, FAULT_ALLOC_BIO);
68 return NULL;
69 }
70
71 return __f2fs_bio_alloc(GFP_KERNEL, npages);
72 }
73
74 static bool __is_cp_guaranteed(struct page *page)
75 {
76 struct address_space *mapping = page->mapping;
77 struct inode *inode;
78 struct f2fs_sb_info *sbi;
79
80 if (!mapping)
81 return false;
82
83 if (f2fs_is_compressed_page(page))
84 return false;
85
86 inode = mapping->host;
87 sbi = F2FS_I_SB(inode);
88
89 if (inode->i_ino == F2FS_META_INO(sbi) ||
90 inode->i_ino == F2FS_NODE_INO(sbi) ||
91 S_ISDIR(inode->i_mode) ||
92 (S_ISREG(inode->i_mode) &&
93 (f2fs_is_atomic_file(inode) || IS_NOQUOTA(inode))) ||
94 is_cold_data(page))
95 return true;
96 return false;
97 }
98
99 static enum count_type __read_io_type(struct page *page)
100 {
101 struct address_space *mapping = page_file_mapping(page);
102
103 if (mapping) {
104 struct inode *inode = mapping->host;
105 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
106
107 if (inode->i_ino == F2FS_META_INO(sbi))
108 return F2FS_RD_META;
109
110 if (inode->i_ino == F2FS_NODE_INO(sbi))
111 return F2FS_RD_NODE;
112 }
113 return F2FS_RD_DATA;
114 }
115
116 /* postprocessing steps for read bios */
117 enum bio_post_read_step {
118 STEP_DECRYPT,
119 STEP_DECOMPRESS_NOWQ, /* handle normal cluster data inplace */
120 STEP_DECOMPRESS, /* handle compressed cluster data in workqueue */
121 STEP_VERITY,
122 };
123
124 struct bio_post_read_ctx {
125 struct bio *bio;
126 struct f2fs_sb_info *sbi;
127 struct work_struct work;
128 unsigned int enabled_steps;
129 };
130
131 static void __read_end_io(struct bio *bio, bool compr, bool verity)
132 {
133 struct page *page;
134 struct bio_vec *bv;
135 struct bvec_iter_all iter_all;
136
137 bio_for_each_segment_all(bv, bio, iter_all) {
138 page = bv->bv_page;
139
140 #ifdef CONFIG_F2FS_FS_COMPRESSION
141 if (compr && f2fs_is_compressed_page(page)) {
142 f2fs_decompress_pages(bio, page, verity);
143 continue;
144 }
145 if (verity)
146 continue;
147 #endif
148
149 /* PG_error was set if any post_read step failed */
150 if (bio->bi_status || PageError(page)) {
151 ClearPageUptodate(page);
152 /* will re-read again later */
153 ClearPageError(page);
154 } else {
155 SetPageUptodate(page);
156 }
157 dec_page_count(F2FS_P_SB(page), __read_io_type(page));
158 unlock_page(page);
159 }
160 }
161
162 static void f2fs_release_read_bio(struct bio *bio);
163 static void __f2fs_read_end_io(struct bio *bio, bool compr, bool verity)
164 {
165 if (!compr)
166 __read_end_io(bio, false, verity);
167 f2fs_release_read_bio(bio);
168 }
169
170 static void f2fs_decompress_bio(struct bio *bio, bool verity)
171 {
172 __read_end_io(bio, true, verity);
173 }
174
175 static void bio_post_read_processing(struct bio_post_read_ctx *ctx);
176
177 static void f2fs_decrypt_work(struct bio_post_read_ctx *ctx)
178 {
179 fscrypt_decrypt_bio(ctx->bio);
180 }
181
182 static void f2fs_decompress_work(struct bio_post_read_ctx *ctx)
183 {
184 f2fs_decompress_bio(ctx->bio, ctx->enabled_steps & (1 << STEP_VERITY));
185 }
186
187 #ifdef CONFIG_F2FS_FS_COMPRESSION
188 static void f2fs_verify_pages(struct page **rpages, unsigned int cluster_size)
189 {
190 f2fs_decompress_end_io(rpages, cluster_size, false, true);
191 }
192
193 static void f2fs_verify_bio(struct bio *bio)
194 {
195 struct bio_vec *bv;
196 struct bvec_iter_all iter_all;
197
198 bio_for_each_segment_all(bv, bio, iter_all) {
199 struct page *page = bv->bv_page;
200 struct decompress_io_ctx *dic;
201
202 dic = (struct decompress_io_ctx *)page_private(page);
203
204 if (dic) {
205 if (atomic_dec_return(&dic->verity_pages))
206 continue;
207 f2fs_verify_pages(dic->rpages,
208 dic->cluster_size);
209 f2fs_free_dic(dic);
210 continue;
211 }
212
213 if (bio->bi_status || PageError(page))
214 goto clear_uptodate;
215
216 if (fsverity_verify_page(page)) {
217 SetPageUptodate(page);
218 goto unlock;
219 }
220 clear_uptodate:
221 ClearPageUptodate(page);
222 ClearPageError(page);
223 unlock:
224 dec_page_count(F2FS_P_SB(page), __read_io_type(page));
225 unlock_page(page);
226 }
227 }
228 #endif
229
230 static void f2fs_verity_work(struct work_struct *work)
231 {
232 struct bio_post_read_ctx *ctx =
233 container_of(work, struct bio_post_read_ctx, work);
234 struct bio *bio = ctx->bio;
235 #ifdef CONFIG_F2FS_FS_COMPRESSION
236 unsigned int enabled_steps = ctx->enabled_steps;
237 #endif
238
239 /*
240 * fsverity_verify_bio() may call readpages() again, and while verity
241 * will be disabled for this, decryption may still be needed, resulting
242 * in another bio_post_read_ctx being allocated. So to prevent
243 * deadlocks we need to release the current ctx to the mempool first.
244 * This assumes that verity is the last post-read step.
245 */
246 mempool_free(ctx, bio_post_read_ctx_pool);
247 bio->bi_private = NULL;
248
249 #ifdef CONFIG_F2FS_FS_COMPRESSION
250 /* previous step is decompression */
251 if (enabled_steps & (1 << STEP_DECOMPRESS)) {
252 f2fs_verify_bio(bio);
253 f2fs_release_read_bio(bio);
254 return;
255 }
256 #endif
257
258 fsverity_verify_bio(bio);
259 __f2fs_read_end_io(bio, false, false);
260 }
261
262 static void f2fs_post_read_work(struct work_struct *work)
263 {
264 struct bio_post_read_ctx *ctx =
265 container_of(work, struct bio_post_read_ctx, work);
266
267 if (ctx->enabled_steps & (1 << STEP_DECRYPT))
268 f2fs_decrypt_work(ctx);
269
270 if (ctx->enabled_steps & (1 << STEP_DECOMPRESS))
271 f2fs_decompress_work(ctx);
272
273 if (ctx->enabled_steps & (1 << STEP_VERITY)) {
274 INIT_WORK(&ctx->work, f2fs_verity_work);
275 fsverity_enqueue_verify_work(&ctx->work);
276 return;
277 }
278
279 __f2fs_read_end_io(ctx->bio,
280 ctx->enabled_steps & (1 << STEP_DECOMPRESS), false);
281 }
282
283 static void f2fs_enqueue_post_read_work(struct f2fs_sb_info *sbi,
284 struct work_struct *work)
285 {
286 queue_work(sbi->post_read_wq, work);
287 }
288
289 static void bio_post_read_processing(struct bio_post_read_ctx *ctx)
290 {
291 /*
292 * We use different work queues for decryption and for verity because
293 * verity may require reading metadata pages that need decryption, and
294 * we shouldn't recurse to the same workqueue.
295 */
296
297 if (ctx->enabled_steps & (1 << STEP_DECRYPT) ||
298 ctx->enabled_steps & (1 << STEP_DECOMPRESS)) {
299 INIT_WORK(&ctx->work, f2fs_post_read_work);
300 f2fs_enqueue_post_read_work(ctx->sbi, &ctx->work);
301 return;
302 }
303
304 if (ctx->enabled_steps & (1 << STEP_VERITY)) {
305 INIT_WORK(&ctx->work, f2fs_verity_work);
306 fsverity_enqueue_verify_work(&ctx->work);
307 return;
308 }
309
310 __f2fs_read_end_io(ctx->bio, false, false);
311 }
312
313 static bool f2fs_bio_post_read_required(struct bio *bio)
314 {
315 return bio->bi_private;
316 }
317
318 static void f2fs_read_end_io(struct bio *bio)
319 {
320 struct f2fs_sb_info *sbi = F2FS_P_SB(bio_first_page_all(bio));
321
322 if (time_to_inject(sbi, FAULT_READ_IO)) {
323 f2fs_show_injection_info(sbi, FAULT_READ_IO);
324 bio->bi_status = BLK_STS_IOERR;
325 }
326
327 if (f2fs_bio_post_read_required(bio)) {
328 struct bio_post_read_ctx *ctx = bio->bi_private;
329
330 bio_post_read_processing(ctx);
331 return;
332 }
333
334 __f2fs_read_end_io(bio, false, false);
335 }
336
337 static void f2fs_write_end_io(struct bio *bio)
338 {
339 struct f2fs_sb_info *sbi = bio->bi_private;
340 struct bio_vec *bvec;
341 struct bvec_iter_all iter_all;
342
343 if (time_to_inject(sbi, FAULT_WRITE_IO)) {
344 f2fs_show_injection_info(sbi, FAULT_WRITE_IO);
345 bio->bi_status = BLK_STS_IOERR;
346 }
347
348 bio_for_each_segment_all(bvec, bio, iter_all) {
349 struct page *page = bvec->bv_page;
350 enum count_type type = WB_DATA_TYPE(page);
351
352 if (IS_DUMMY_WRITTEN_PAGE(page)) {
353 set_page_private(page, (unsigned long)NULL);
354 ClearPagePrivate(page);
355 unlock_page(page);
356 mempool_free(page, sbi->write_io_dummy);
357
358 if (unlikely(bio->bi_status))
359 f2fs_stop_checkpoint(sbi, true);
360 continue;
361 }
362
363 fscrypt_finalize_bounce_page(&page);
364
365 #ifdef CONFIG_F2FS_FS_COMPRESSION
366 if (f2fs_is_compressed_page(page)) {
367 f2fs_compress_write_end_io(bio, page);
368 continue;
369 }
370 #endif
371
372 if (unlikely(bio->bi_status)) {
373 mapping_set_error(page->mapping, -EIO);
374 if (type == F2FS_WB_CP_DATA)
375 f2fs_stop_checkpoint(sbi, true);
376 }
377
378 f2fs_bug_on(sbi, page->mapping == NODE_MAPPING(sbi) &&
379 page->index != nid_of_node(page));
380
381 dec_page_count(sbi, type);
382 if (f2fs_in_warm_node_list(sbi, page))
383 f2fs_del_fsync_node_entry(sbi, page);
384 clear_cold_data(page);
385 end_page_writeback(page);
386 }
387 if (!get_pages(sbi, F2FS_WB_CP_DATA) &&
388 wq_has_sleeper(&sbi->cp_wait))
389 wake_up(&sbi->cp_wait);
390
391 bio_put(bio);
392 }
393
394 struct block_device *f2fs_target_device(struct f2fs_sb_info *sbi,
395 block_t blk_addr, struct bio *bio)
396 {
397 struct block_device *bdev = sbi->sb->s_bdev;
398 int i;
399
400 if (f2fs_is_multi_device(sbi)) {
401 for (i = 0; i < sbi->s_ndevs; i++) {
402 if (FDEV(i).start_blk <= blk_addr &&
403 FDEV(i).end_blk >= blk_addr) {
404 blk_addr -= FDEV(i).start_blk;
405 bdev = FDEV(i).bdev;
406 break;
407 }
408 }
409 }
410 if (bio) {
411 bio_set_dev(bio, bdev);
412 bio->bi_iter.bi_sector = SECTOR_FROM_BLOCK(blk_addr);
413 }
414 return bdev;
415 }
416
417 int f2fs_target_device_index(struct f2fs_sb_info *sbi, block_t blkaddr)
418 {
419 int i;
420
421 if (!f2fs_is_multi_device(sbi))
422 return 0;
423
424 for (i = 0; i < sbi->s_ndevs; i++)
425 if (FDEV(i).start_blk <= blkaddr && FDEV(i).end_blk >= blkaddr)
426 return i;
427 return 0;
428 }
429
430 /*
431 * Return true, if pre_bio's bdev is same as its target device.
432 */
433 static bool __same_bdev(struct f2fs_sb_info *sbi,
434 block_t blk_addr, struct bio *bio)
435 {
436 struct block_device *b = f2fs_target_device(sbi, blk_addr, NULL);
437 return bio->bi_disk == b->bd_disk && bio->bi_partno == b->bd_partno;
438 }
439
440 static struct bio *__bio_alloc(struct f2fs_io_info *fio, int npages)
441 {
442 struct f2fs_sb_info *sbi = fio->sbi;
443 struct bio *bio;
444
445 bio = f2fs_bio_alloc(sbi, npages, true);
446
447 f2fs_target_device(sbi, fio->new_blkaddr, bio);
448 if (is_read_io(fio->op)) {
449 bio->bi_end_io = f2fs_read_end_io;
450 bio->bi_private = NULL;
451 } else {
452 bio->bi_end_io = f2fs_write_end_io;
453 bio->bi_private = sbi;
454 bio->bi_write_hint = f2fs_io_type_to_rw_hint(sbi,
455 fio->type, fio->temp);
456 }
457 if (fio->io_wbc)
458 wbc_init_bio(fio->io_wbc, bio);
459
460 return bio;
461 }
462
463 static void f2fs_set_bio_crypt_ctx(struct bio *bio, const struct inode *inode,
464 pgoff_t first_idx,
465 const struct f2fs_io_info *fio,
466 gfp_t gfp_mask)
467 {
468 /*
469 * The f2fs garbage collector sets ->encrypted_page when it wants to
470 * read/write raw data without encryption.
471 */
472 if (!fio || !fio->encrypted_page)
473 fscrypt_set_bio_crypt_ctx(bio, inode, first_idx, gfp_mask);
474 }
475
476 static bool f2fs_crypt_mergeable_bio(struct bio *bio, const struct inode *inode,
477 pgoff_t next_idx,
478 const struct f2fs_io_info *fio)
479 {
480 /*
481 * The f2fs garbage collector sets ->encrypted_page when it wants to
482 * read/write raw data without encryption.
483 */
484 if (fio && fio->encrypted_page)
485 return !bio_has_crypt_ctx(bio);
486
487 return fscrypt_mergeable_bio(bio, inode, next_idx);
488 }
489
490 static inline void __submit_bio(struct f2fs_sb_info *sbi,
491 struct bio *bio, enum page_type type)
492 {
493 if (!is_read_io(bio_op(bio))) {
494 unsigned int start;
495
496 if (type != DATA && type != NODE)
497 goto submit_io;
498
499 if (f2fs_lfs_mode(sbi) && current->plug)
500 blk_finish_plug(current->plug);
501
502 if (!F2FS_IO_ALIGNED(sbi))
503 goto submit_io;
504
505 start = bio->bi_iter.bi_size >> F2FS_BLKSIZE_BITS;
506 start %= F2FS_IO_SIZE(sbi);
507
508 if (start == 0)
509 goto submit_io;
510
511 /* fill dummy pages */
512 for (; start < F2FS_IO_SIZE(sbi); start++) {
513 struct page *page =
514 mempool_alloc(sbi->write_io_dummy,
515 GFP_NOIO | __GFP_NOFAIL);
516 f2fs_bug_on(sbi, !page);
517
518 zero_user_segment(page, 0, PAGE_SIZE);
519 SetPagePrivate(page);
520 set_page_private(page, DUMMY_WRITTEN_PAGE);
521 lock_page(page);
522 if (bio_add_page(bio, page, PAGE_SIZE, 0) < PAGE_SIZE)
523 f2fs_bug_on(sbi, 1);
524 }
525 /*
526 * In the NODE case, we lose next block address chain. So, we
527 * need to do checkpoint in f2fs_sync_file.
528 */
529 if (type == NODE)
530 set_sbi_flag(sbi, SBI_NEED_CP);
531 }
532 submit_io:
533 if (is_read_io(bio_op(bio)))
534 trace_f2fs_submit_read_bio(sbi->sb, type, bio);
535 else
536 trace_f2fs_submit_write_bio(sbi->sb, type, bio);
537 submit_bio(bio);
538 }
539
540 void f2fs_submit_bio(struct f2fs_sb_info *sbi,
541 struct bio *bio, enum page_type type)
542 {
543 __submit_bio(sbi, bio, type);
544 }
545
546 static void __attach_io_flag(struct f2fs_io_info *fio)
547 {
548 struct f2fs_sb_info *sbi = fio->sbi;
549 unsigned int temp_mask = (1 << NR_TEMP_TYPE) - 1;
550 unsigned int io_flag, fua_flag, meta_flag;
551
552 if (fio->type == DATA)
553 io_flag = sbi->data_io_flag;
554 else if (fio->type == NODE)
555 io_flag = sbi->node_io_flag;
556 else
557 return;
558
559 fua_flag = io_flag & temp_mask;
560 meta_flag = (io_flag >> NR_TEMP_TYPE) & temp_mask;
561
562 /*
563 * data/node io flag bits per temp:
564 * REQ_META | REQ_FUA |
565 * 5 | 4 | 3 | 2 | 1 | 0 |
566 * Cold | Warm | Hot | Cold | Warm | Hot |
567 */
568 if ((1 << fio->temp) & meta_flag)
569 fio->op_flags |= REQ_META;
570 if ((1 << fio->temp) & fua_flag)
571 fio->op_flags |= REQ_FUA;
572 }
573
574 static void __submit_merged_bio(struct f2fs_bio_info *io)
575 {
576 struct f2fs_io_info *fio = &io->fio;
577
578 if (!io->bio)
579 return;
580
581 __attach_io_flag(fio);
582 bio_set_op_attrs(io->bio, fio->op, fio->op_flags);
583
584 if (is_read_io(fio->op))
585 trace_f2fs_prepare_read_bio(io->sbi->sb, fio->type, io->bio);
586 else
587 trace_f2fs_prepare_write_bio(io->sbi->sb, fio->type, io->bio);
588
589 __submit_bio(io->sbi, io->bio, fio->type);
590 io->bio = NULL;
591 }
592
593 static bool __has_merged_page(struct bio *bio, struct inode *inode,
594 struct page *page, nid_t ino)
595 {
596 struct bio_vec *bvec;
597 struct bvec_iter_all iter_all;
598
599 if (!bio)
600 return false;
601
602 if (!inode && !page && !ino)
603 return true;
604
605 bio_for_each_segment_all(bvec, bio, iter_all) {
606 struct page *target = bvec->bv_page;
607
608 if (fscrypt_is_bounce_page(target)) {
609 target = fscrypt_pagecache_page(target);
610 if (IS_ERR(target))
611 continue;
612 }
613 if (f2fs_is_compressed_page(target)) {
614 target = f2fs_compress_control_page(target);
615 if (IS_ERR(target))
616 continue;
617 }
618
619 if (inode && inode == target->mapping->host)
620 return true;
621 if (page && page == target)
622 return true;
623 if (ino && ino == ino_of_node(target))
624 return true;
625 }
626
627 return false;
628 }
629
630 static void __f2fs_submit_merged_write(struct f2fs_sb_info *sbi,
631 enum page_type type, enum temp_type temp)
632 {
633 enum page_type btype = PAGE_TYPE_OF_BIO(type);
634 struct f2fs_bio_info *io = sbi->write_io[btype] + temp;
635
636 down_write(&io->io_rwsem);
637
638 /* change META to META_FLUSH in the checkpoint procedure */
639 if (type >= META_FLUSH) {
640 io->fio.type = META_FLUSH;
641 io->fio.op = REQ_OP_WRITE;
642 io->fio.op_flags = REQ_META | REQ_PRIO | REQ_SYNC;
643 if (!test_opt(sbi, NOBARRIER))
644 io->fio.op_flags |= REQ_PREFLUSH | REQ_FUA;
645 }
646 __submit_merged_bio(io);
647 up_write(&io->io_rwsem);
648 }
649
650 static void __submit_merged_write_cond(struct f2fs_sb_info *sbi,
651 struct inode *inode, struct page *page,
652 nid_t ino, enum page_type type, bool force)
653 {
654 enum temp_type temp;
655 bool ret = true;
656
657 for (temp = HOT; temp < NR_TEMP_TYPE; temp++) {
658 if (!force) {
659 enum page_type btype = PAGE_TYPE_OF_BIO(type);
660 struct f2fs_bio_info *io = sbi->write_io[btype] + temp;
661
662 down_read(&io->io_rwsem);
663 ret = __has_merged_page(io->bio, inode, page, ino);
664 up_read(&io->io_rwsem);
665 }
666 if (ret)
667 __f2fs_submit_merged_write(sbi, type, temp);
668
669 /* TODO: use HOT temp only for meta pages now. */
670 if (type >= META)
671 break;
672 }
673 }
674
675 void f2fs_submit_merged_write(struct f2fs_sb_info *sbi, enum page_type type)
676 {
677 __submit_merged_write_cond(sbi, NULL, NULL, 0, type, true);
678 }
679
680 void f2fs_submit_merged_write_cond(struct f2fs_sb_info *sbi,
681 struct inode *inode, struct page *page,
682 nid_t ino, enum page_type type)
683 {
684 __submit_merged_write_cond(sbi, inode, page, ino, type, false);
685 }
686
687 void f2fs_flush_merged_writes(struct f2fs_sb_info *sbi)
688 {
689 f2fs_submit_merged_write(sbi, DATA);
690 f2fs_submit_merged_write(sbi, NODE);
691 f2fs_submit_merged_write(sbi, META);
692 }
693
694 /*
695 * Fill the locked page with data located in the block address.
696 * A caller needs to unlock the page on failure.
697 */
698 int f2fs_submit_page_bio(struct f2fs_io_info *fio)
699 {
700 struct bio *bio;
701 struct page *page = fio->encrypted_page ?
702 fio->encrypted_page : fio->page;
703
704 if (!f2fs_is_valid_blkaddr(fio->sbi, fio->new_blkaddr,
705 fio->is_por ? META_POR : (__is_meta_io(fio) ?
706 META_GENERIC : DATA_GENERIC_ENHANCE)))
707 return -EFSCORRUPTED;
708
709 trace_f2fs_submit_page_bio(page, fio);
710 f2fs_trace_ios(fio, 0);
711
712 /* Allocate a new bio */
713 bio = __bio_alloc(fio, 1);
714
715 f2fs_set_bio_crypt_ctx(bio, fio->page->mapping->host,
716 fio->page->index, fio, GFP_NOIO);
717
718 if (bio_add_page(bio, page, PAGE_SIZE, 0) < PAGE_SIZE) {
719 bio_put(bio);
720 return -EFAULT;
721 }
722
723 if (fio->io_wbc && !is_read_io(fio->op))
724 wbc_account_cgroup_owner(fio->io_wbc, page, PAGE_SIZE);
725
726 __attach_io_flag(fio);
727 bio_set_op_attrs(bio, fio->op, fio->op_flags);
728
729 inc_page_count(fio->sbi, is_read_io(fio->op) ?
730 __read_io_type(page): WB_DATA_TYPE(fio->page));
731
732 __submit_bio(fio->sbi, bio, fio->type);
733 return 0;
734 }
735
736 static bool page_is_mergeable(struct f2fs_sb_info *sbi, struct bio *bio,
737 block_t last_blkaddr, block_t cur_blkaddr)
738 {
739 if (unlikely(sbi->max_io_bytes &&
740 bio->bi_iter.bi_size >= sbi->max_io_bytes))
741 return false;
742 if (last_blkaddr + 1 != cur_blkaddr)
743 return false;
744 return __same_bdev(sbi, cur_blkaddr, bio);
745 }
746
747 static bool io_type_is_mergeable(struct f2fs_bio_info *io,
748 struct f2fs_io_info *fio)
749 {
750 if (io->fio.op != fio->op)
751 return false;
752 return io->fio.op_flags == fio->op_flags;
753 }
754
755 static bool io_is_mergeable(struct f2fs_sb_info *sbi, struct bio *bio,
756 struct f2fs_bio_info *io,
757 struct f2fs_io_info *fio,
758 block_t last_blkaddr,
759 block_t cur_blkaddr)
760 {
761 if (F2FS_IO_ALIGNED(sbi) && (fio->type == DATA || fio->type == NODE)) {
762 unsigned int filled_blocks =
763 F2FS_BYTES_TO_BLK(bio->bi_iter.bi_size);
764 unsigned int io_size = F2FS_IO_SIZE(sbi);
765 unsigned int left_vecs = bio->bi_max_vecs - bio->bi_vcnt;
766
767 /* IOs in bio is aligned and left space of vectors is not enough */
768 if (!(filled_blocks % io_size) && left_vecs < io_size)
769 return false;
770 }
771 if (!page_is_mergeable(sbi, bio, last_blkaddr, cur_blkaddr))
772 return false;
773 return io_type_is_mergeable(io, fio);
774 }
775
776 static void add_bio_entry(struct f2fs_sb_info *sbi, struct bio *bio,
777 struct page *page, enum temp_type temp)
778 {
779 struct f2fs_bio_info *io = sbi->write_io[DATA] + temp;
780 struct bio_entry *be;
781
782 be = f2fs_kmem_cache_alloc(bio_entry_slab, GFP_NOFS);
783 be->bio = bio;
784 bio_get(bio);
785
786 if (bio_add_page(bio, page, PAGE_SIZE, 0) != PAGE_SIZE)
787 f2fs_bug_on(sbi, 1);
788
789 down_write(&io->bio_list_lock);
790 list_add_tail(&be->list, &io->bio_list);
791 up_write(&io->bio_list_lock);
792 }
793
794 static void del_bio_entry(struct bio_entry *be)
795 {
796 list_del(&be->list);
797 kmem_cache_free(bio_entry_slab, be);
798 }
799
800 static int add_ipu_page(struct f2fs_io_info *fio, struct bio **bio,
801 struct page *page)
802 {
803 struct f2fs_sb_info *sbi = fio->sbi;
804 enum temp_type temp;
805 bool found = false;
806 int ret = -EAGAIN;
807
808 for (temp = HOT; temp < NR_TEMP_TYPE && !found; temp++) {
809 struct f2fs_bio_info *io = sbi->write_io[DATA] + temp;
810 struct list_head *head = &io->bio_list;
811 struct bio_entry *be;
812
813 down_write(&io->bio_list_lock);
814 list_for_each_entry(be, head, list) {
815 if (be->bio != *bio)
816 continue;
817
818 found = true;
819
820 f2fs_bug_on(sbi, !page_is_mergeable(sbi, *bio,
821 *fio->last_block,
822 fio->new_blkaddr));
823 if (f2fs_crypt_mergeable_bio(*bio,
824 fio->page->mapping->host,
825 fio->page->index, fio) &&
826 bio_add_page(*bio, page, PAGE_SIZE, 0) ==
827 PAGE_SIZE) {
828 ret = 0;
829 break;
830 }
831
832 /* page can't be merged into bio; submit the bio */
833 del_bio_entry(be);
834 __submit_bio(sbi, *bio, DATA);
835 break;
836 }
837 up_write(&io->bio_list_lock);
838 }
839
840 if (ret) {
841 bio_put(*bio);
842 *bio = NULL;
843 }
844
845 return ret;
846 }
847
848 void f2fs_submit_merged_ipu_write(struct f2fs_sb_info *sbi,
849 struct bio **bio, struct page *page)
850 {
851 enum temp_type temp;
852 bool found = false;
853 struct bio *target = bio ? *bio : NULL;
854
855 for (temp = HOT; temp < NR_TEMP_TYPE && !found; temp++) {
856 struct f2fs_bio_info *io = sbi->write_io[DATA] + temp;
857 struct list_head *head = &io->bio_list;
858 struct bio_entry *be;
859
860 if (list_empty(head))
861 continue;
862
863 down_read(&io->bio_list_lock);
864 list_for_each_entry(be, head, list) {
865 if (target)
866 found = (target == be->bio);
867 else
868 found = __has_merged_page(be->bio, NULL,
869 page, 0);
870 if (found)
871 break;
872 }
873 up_read(&io->bio_list_lock);
874
875 if (!found)
876 continue;
877
878 found = false;
879
880 down_write(&io->bio_list_lock);
881 list_for_each_entry(be, head, list) {
882 if (target)
883 found = (target == be->bio);
884 else
885 found = __has_merged_page(be->bio, NULL,
886 page, 0);
887 if (found) {
888 target = be->bio;
889 del_bio_entry(be);
890 break;
891 }
892 }
893 up_write(&io->bio_list_lock);
894 }
895
896 if (found)
897 __submit_bio(sbi, target, DATA);
898 if (bio && *bio) {
899 bio_put(*bio);
900 *bio = NULL;
901 }
902 }
903
904 int f2fs_merge_page_bio(struct f2fs_io_info *fio)
905 {
906 struct bio *bio = *fio->bio;
907 struct page *page = fio->encrypted_page ?
908 fio->encrypted_page : fio->page;
909
910 if (!f2fs_is_valid_blkaddr(fio->sbi, fio->new_blkaddr,
911 __is_meta_io(fio) ? META_GENERIC : DATA_GENERIC))
912 return -EFSCORRUPTED;
913
914 trace_f2fs_submit_page_bio(page, fio);
915 f2fs_trace_ios(fio, 0);
916
917 if (bio && !page_is_mergeable(fio->sbi, bio, *fio->last_block,
918 fio->new_blkaddr))
919 f2fs_submit_merged_ipu_write(fio->sbi, &bio, NULL);
920 alloc_new:
921 if (!bio) {
922 bio = __bio_alloc(fio, BIO_MAX_PAGES);
923 __attach_io_flag(fio);
924 f2fs_set_bio_crypt_ctx(bio, fio->page->mapping->host,
925 fio->page->index, fio, GFP_NOIO);
926 bio_set_op_attrs(bio, fio->op, fio->op_flags);
927
928 add_bio_entry(fio->sbi, bio, page, fio->temp);
929 } else {
930 if (add_ipu_page(fio, &bio, page))
931 goto alloc_new;
932 }
933
934 if (fio->io_wbc)
935 wbc_account_cgroup_owner(fio->io_wbc, page, PAGE_SIZE);
936
937 inc_page_count(fio->sbi, WB_DATA_TYPE(page));
938
939 *fio->last_block = fio->new_blkaddr;
940 *fio->bio = bio;
941
942 return 0;
943 }
944
945 void f2fs_submit_page_write(struct f2fs_io_info *fio)
946 {
947 struct f2fs_sb_info *sbi = fio->sbi;
948 enum page_type btype = PAGE_TYPE_OF_BIO(fio->type);
949 struct f2fs_bio_info *io = sbi->write_io[btype] + fio->temp;
950 struct page *bio_page;
951
952 f2fs_bug_on(sbi, is_read_io(fio->op));
953
954 down_write(&io->io_rwsem);
955 next:
956 if (fio->in_list) {
957 spin_lock(&io->io_lock);
958 if (list_empty(&io->io_list)) {
959 spin_unlock(&io->io_lock);
960 goto out;
961 }
962 fio = list_first_entry(&io->io_list,
963 struct f2fs_io_info, list);
964 list_del(&fio->list);
965 spin_unlock(&io->io_lock);
966 }
967
968 verify_fio_blkaddr(fio);
969
970 if (fio->encrypted_page)
971 bio_page = fio->encrypted_page;
972 else if (fio->compressed_page)
973 bio_page = fio->compressed_page;
974 else
975 bio_page = fio->page;
976
977 /* set submitted = true as a return value */
978 fio->submitted = true;
979
980 inc_page_count(sbi, WB_DATA_TYPE(bio_page));
981
982 if (io->bio &&
983 (!io_is_mergeable(sbi, io->bio, io, fio, io->last_block_in_bio,
984 fio->new_blkaddr) ||
985 !f2fs_crypt_mergeable_bio(io->bio, fio->page->mapping->host,
986 bio_page->index, fio)))
987 __submit_merged_bio(io);
988 alloc_new:
989 if (io->bio == NULL) {
990 if (F2FS_IO_ALIGNED(sbi) &&
991 (fio->type == DATA || fio->type == NODE) &&
992 fio->new_blkaddr & F2FS_IO_SIZE_MASK(sbi)) {
993 dec_page_count(sbi, WB_DATA_TYPE(bio_page));
994 fio->retry = true;
995 goto skip;
996 }
997 io->bio = __bio_alloc(fio, BIO_MAX_PAGES);
998 f2fs_set_bio_crypt_ctx(io->bio, fio->page->mapping->host,
999 bio_page->index, fio, GFP_NOIO);
1000 io->fio = *fio;
1001 }
1002
1003 if (bio_add_page(io->bio, bio_page, PAGE_SIZE, 0) < PAGE_SIZE) {
1004 __submit_merged_bio(io);
1005 goto alloc_new;
1006 }
1007
1008 if (fio->io_wbc)
1009 wbc_account_cgroup_owner(fio->io_wbc, bio_page, PAGE_SIZE);
1010
1011 io->last_block_in_bio = fio->new_blkaddr;
1012 f2fs_trace_ios(fio, 0);
1013
1014 trace_f2fs_submit_page_write(fio->page, fio);
1015 skip:
1016 if (fio->in_list)
1017 goto next;
1018 out:
1019 if (is_sbi_flag_set(sbi, SBI_IS_SHUTDOWN) ||
1020 !f2fs_is_checkpoint_ready(sbi))
1021 __submit_merged_bio(io);
1022 up_write(&io->io_rwsem);
1023 }
1024
1025 static inline bool f2fs_need_verity(const struct inode *inode, pgoff_t idx)
1026 {
1027 return fsverity_active(inode) &&
1028 idx < DIV_ROUND_UP(inode->i_size, PAGE_SIZE);
1029 }
1030
1031 static struct bio *f2fs_grab_read_bio(struct inode *inode, block_t blkaddr,
1032 unsigned nr_pages, unsigned op_flag,
1033 pgoff_t first_idx, bool for_write,
1034 bool for_verity)
1035 {
1036 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
1037 struct bio *bio;
1038 struct bio_post_read_ctx *ctx;
1039 unsigned int post_read_steps = 0;
1040
1041 bio = f2fs_bio_alloc(sbi, min_t(int, nr_pages, BIO_MAX_PAGES),
1042 for_write);
1043 if (!bio)
1044 return ERR_PTR(-ENOMEM);
1045
1046 f2fs_set_bio_crypt_ctx(bio, inode, first_idx, NULL, GFP_NOFS);
1047
1048 f2fs_target_device(sbi, blkaddr, bio);
1049 bio->bi_end_io = f2fs_read_end_io;
1050 bio_set_op_attrs(bio, REQ_OP_READ, op_flag);
1051
1052 if (fscrypt_inode_uses_fs_layer_crypto(inode))
1053 post_read_steps |= 1 << STEP_DECRYPT;
1054 if (f2fs_compressed_file(inode))
1055 post_read_steps |= 1 << STEP_DECOMPRESS_NOWQ;
1056 if (for_verity && f2fs_need_verity(inode, first_idx))
1057 post_read_steps |= 1 << STEP_VERITY;
1058
1059 if (post_read_steps) {
1060 /* Due to the mempool, this never fails. */
1061 ctx = mempool_alloc(bio_post_read_ctx_pool, GFP_NOFS);
1062 ctx->bio = bio;
1063 ctx->sbi = sbi;
1064 ctx->enabled_steps = post_read_steps;
1065 bio->bi_private = ctx;
1066 }
1067
1068 return bio;
1069 }
1070
1071 static void f2fs_release_read_bio(struct bio *bio)
1072 {
1073 if (bio->bi_private)
1074 mempool_free(bio->bi_private, bio_post_read_ctx_pool);
1075 bio_put(bio);
1076 }
1077
1078 /* This can handle encryption stuffs */
1079 static int f2fs_submit_page_read(struct inode *inode, struct page *page,
1080 block_t blkaddr, int op_flags, bool for_write)
1081 {
1082 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
1083 struct bio *bio;
1084
1085 bio = f2fs_grab_read_bio(inode, blkaddr, 1, op_flags,
1086 page->index, for_write, true);
1087 if (IS_ERR(bio))
1088 return PTR_ERR(bio);
1089
1090 /* wait for GCed page writeback via META_MAPPING */
1091 f2fs_wait_on_block_writeback(inode, blkaddr);
1092
1093 if (bio_add_page(bio, page, PAGE_SIZE, 0) < PAGE_SIZE) {
1094 bio_put(bio);
1095 return -EFAULT;
1096 }
1097 ClearPageError(page);
1098 inc_page_count(sbi, F2FS_RD_DATA);
1099 f2fs_update_iostat(sbi, FS_DATA_READ_IO, F2FS_BLKSIZE);
1100 __submit_bio(sbi, bio, DATA);
1101 return 0;
1102 }
1103
1104 static void __set_data_blkaddr(struct dnode_of_data *dn)
1105 {
1106 struct f2fs_node *rn = F2FS_NODE(dn->node_page);
1107 __le32 *addr_array;
1108 int base = 0;
1109
1110 if (IS_INODE(dn->node_page) && f2fs_has_extra_attr(dn->inode))
1111 base = get_extra_isize(dn->inode);
1112
1113 /* Get physical address of data block */
1114 addr_array = blkaddr_in_node(rn);
1115 addr_array[base + dn->ofs_in_node] = cpu_to_le32(dn->data_blkaddr);
1116 }
1117
1118 /*
1119 * Lock ordering for the change of data block address:
1120 * ->data_page
1121 * ->node_page
1122 * update block addresses in the node page
1123 */
1124 void f2fs_set_data_blkaddr(struct dnode_of_data *dn)
1125 {
1126 f2fs_wait_on_page_writeback(dn->node_page, NODE, true, true);
1127 __set_data_blkaddr(dn);
1128 if (set_page_dirty(dn->node_page))
1129 dn->node_changed = true;
1130 }
1131
1132 void f2fs_update_data_blkaddr(struct dnode_of_data *dn, block_t blkaddr)
1133 {
1134 dn->data_blkaddr = blkaddr;
1135 f2fs_set_data_blkaddr(dn);
1136 f2fs_update_extent_cache(dn);
1137 }
1138
1139 /* dn->ofs_in_node will be returned with up-to-date last block pointer */
1140 int f2fs_reserve_new_blocks(struct dnode_of_data *dn, blkcnt_t count)
1141 {
1142 struct f2fs_sb_info *sbi = F2FS_I_SB(dn->inode);
1143 int err;
1144
1145 if (!count)
1146 return 0;
1147
1148 if (unlikely(is_inode_flag_set(dn->inode, FI_NO_ALLOC)))
1149 return -EPERM;
1150 if (unlikely((err = inc_valid_block_count(sbi, dn->inode, &count))))
1151 return err;
1152
1153 trace_f2fs_reserve_new_blocks(dn->inode, dn->nid,
1154 dn->ofs_in_node, count);
1155
1156 f2fs_wait_on_page_writeback(dn->node_page, NODE, true, true);
1157
1158 for (; count > 0; dn->ofs_in_node++) {
1159 block_t blkaddr = f2fs_data_blkaddr(dn);
1160 if (blkaddr == NULL_ADDR) {
1161 dn->data_blkaddr = NEW_ADDR;
1162 __set_data_blkaddr(dn);
1163 count--;
1164 }
1165 }
1166
1167 if (set_page_dirty(dn->node_page))
1168 dn->node_changed = true;
1169 return 0;
1170 }
1171
1172 /* Should keep dn->ofs_in_node unchanged */
1173 int f2fs_reserve_new_block(struct dnode_of_data *dn)
1174 {
1175 unsigned int ofs_in_node = dn->ofs_in_node;
1176 int ret;
1177
1178 ret = f2fs_reserve_new_blocks(dn, 1);
1179 dn->ofs_in_node = ofs_in_node;
1180 return ret;
1181 }
1182
1183 int f2fs_reserve_block(struct dnode_of_data *dn, pgoff_t index)
1184 {
1185 bool need_put = dn->inode_page ? false : true;
1186 int err;
1187
1188 err = f2fs_get_dnode_of_data(dn, index, ALLOC_NODE);
1189 if (err)
1190 return err;
1191
1192 if (dn->data_blkaddr == NULL_ADDR)
1193 err = f2fs_reserve_new_block(dn);
1194 if (err || need_put)
1195 f2fs_put_dnode(dn);
1196 return err;
1197 }
1198
1199 int f2fs_get_block(struct dnode_of_data *dn, pgoff_t index)
1200 {
1201 struct extent_info ei = {0, 0, 0};
1202 struct inode *inode = dn->inode;
1203
1204 if (f2fs_lookup_extent_cache(inode, index, &ei)) {
1205 dn->data_blkaddr = ei.blk + index - ei.fofs;
1206 return 0;
1207 }
1208
1209 return f2fs_reserve_block(dn, index);
1210 }
1211
1212 struct page *f2fs_get_read_data_page(struct inode *inode, pgoff_t index,
1213 int op_flags, bool for_write)
1214 {
1215 struct address_space *mapping = inode->i_mapping;
1216 struct dnode_of_data dn;
1217 struct page *page;
1218 struct extent_info ei = {0,0,0};
1219 int err;
1220
1221 page = f2fs_grab_cache_page(mapping, index, for_write);
1222 if (!page)
1223 return ERR_PTR(-ENOMEM);
1224
1225 if (f2fs_lookup_extent_cache(inode, index, &ei)) {
1226 dn.data_blkaddr = ei.blk + index - ei.fofs;
1227 if (!f2fs_is_valid_blkaddr(F2FS_I_SB(inode), dn.data_blkaddr,
1228 DATA_GENERIC_ENHANCE_READ)) {
1229 err = -EFSCORRUPTED;
1230 goto put_err;
1231 }
1232 goto got_it;
1233 }
1234
1235 set_new_dnode(&dn, inode, NULL, NULL, 0);
1236 err = f2fs_get_dnode_of_data(&dn, index, LOOKUP_NODE);
1237 if (err)
1238 goto put_err;
1239 f2fs_put_dnode(&dn);
1240
1241 if (unlikely(dn.data_blkaddr == NULL_ADDR)) {
1242 err = -ENOENT;
1243 goto put_err;
1244 }
1245 if (dn.data_blkaddr != NEW_ADDR &&
1246 !f2fs_is_valid_blkaddr(F2FS_I_SB(inode),
1247 dn.data_blkaddr,
1248 DATA_GENERIC_ENHANCE)) {
1249 err = -EFSCORRUPTED;
1250 goto put_err;
1251 }
1252 got_it:
1253 if (PageUptodate(page)) {
1254 unlock_page(page);
1255 return page;
1256 }
1257
1258 /*
1259 * A new dentry page is allocated but not able to be written, since its
1260 * new inode page couldn't be allocated due to -ENOSPC.
1261 * In such the case, its blkaddr can be remained as NEW_ADDR.
1262 * see, f2fs_add_link -> f2fs_get_new_data_page ->
1263 * f2fs_init_inode_metadata.
1264 */
1265 if (dn.data_blkaddr == NEW_ADDR) {
1266 zero_user_segment(page, 0, PAGE_SIZE);
1267 if (!PageUptodate(page))
1268 SetPageUptodate(page);
1269 unlock_page(page);
1270 return page;
1271 }
1272
1273 err = f2fs_submit_page_read(inode, page, dn.data_blkaddr,
1274 op_flags, for_write);
1275 if (err)
1276 goto put_err;
1277 return page;
1278
1279 put_err:
1280 f2fs_put_page(page, 1);
1281 return ERR_PTR(err);
1282 }
1283
1284 struct page *f2fs_find_data_page(struct inode *inode, pgoff_t index)
1285 {
1286 struct address_space *mapping = inode->i_mapping;
1287 struct page *page;
1288
1289 page = find_get_page(mapping, index);
1290 if (page && PageUptodate(page))
1291 return page;
1292 f2fs_put_page(page, 0);
1293
1294 page = f2fs_get_read_data_page(inode, index, 0, false);
1295 if (IS_ERR(page))
1296 return page;
1297
1298 if (PageUptodate(page))
1299 return page;
1300
1301 wait_on_page_locked(page);
1302 if (unlikely(!PageUptodate(page))) {
1303 f2fs_put_page(page, 0);
1304 return ERR_PTR(-EIO);
1305 }
1306 return page;
1307 }
1308
1309 /*
1310 * If it tries to access a hole, return an error.
1311 * Because, the callers, functions in dir.c and GC, should be able to know
1312 * whether this page exists or not.
1313 */
1314 struct page *f2fs_get_lock_data_page(struct inode *inode, pgoff_t index,
1315 bool for_write)
1316 {
1317 struct address_space *mapping = inode->i_mapping;
1318 struct page *page;
1319 repeat:
1320 page = f2fs_get_read_data_page(inode, index, 0, for_write);
1321 if (IS_ERR(page))
1322 return page;
1323
1324 /* wait for read completion */
1325 lock_page(page);
1326 if (unlikely(page->mapping != mapping)) {
1327 f2fs_put_page(page, 1);
1328 goto repeat;
1329 }
1330 if (unlikely(!PageUptodate(page))) {
1331 f2fs_put_page(page, 1);
1332 return ERR_PTR(-EIO);
1333 }
1334 return page;
1335 }
1336
1337 /*
1338 * Caller ensures that this data page is never allocated.
1339 * A new zero-filled data page is allocated in the page cache.
1340 *
1341 * Also, caller should grab and release a rwsem by calling f2fs_lock_op() and
1342 * f2fs_unlock_op().
1343 * Note that, ipage is set only by make_empty_dir, and if any error occur,
1344 * ipage should be released by this function.
1345 */
1346 struct page *f2fs_get_new_data_page(struct inode *inode,
1347 struct page *ipage, pgoff_t index, bool new_i_size)
1348 {
1349 struct address_space *mapping = inode->i_mapping;
1350 struct page *page;
1351 struct dnode_of_data dn;
1352 int err;
1353
1354 page = f2fs_grab_cache_page(mapping, index, true);
1355 if (!page) {
1356 /*
1357 * before exiting, we should make sure ipage will be released
1358 * if any error occur.
1359 */
1360 f2fs_put_page(ipage, 1);
1361 return ERR_PTR(-ENOMEM);
1362 }
1363
1364 set_new_dnode(&dn, inode, ipage, NULL, 0);
1365 err = f2fs_reserve_block(&dn, index);
1366 if (err) {
1367 f2fs_put_page(page, 1);
1368 return ERR_PTR(err);
1369 }
1370 if (!ipage)
1371 f2fs_put_dnode(&dn);
1372
1373 if (PageUptodate(page))
1374 goto got_it;
1375
1376 if (dn.data_blkaddr == NEW_ADDR) {
1377 zero_user_segment(page, 0, PAGE_SIZE);
1378 if (!PageUptodate(page))
1379 SetPageUptodate(page);
1380 } else {
1381 f2fs_put_page(page, 1);
1382
1383 /* if ipage exists, blkaddr should be NEW_ADDR */
1384 f2fs_bug_on(F2FS_I_SB(inode), ipage);
1385 page = f2fs_get_lock_data_page(inode, index, true);
1386 if (IS_ERR(page))
1387 return page;
1388 }
1389 got_it:
1390 if (new_i_size && i_size_read(inode) <
1391 ((loff_t)(index + 1) << PAGE_SHIFT))
1392 f2fs_i_size_write(inode, ((loff_t)(index + 1) << PAGE_SHIFT));
1393 return page;
1394 }
1395
1396 static int __allocate_data_block(struct dnode_of_data *dn, int seg_type)
1397 {
1398 struct f2fs_sb_info *sbi = F2FS_I_SB(dn->inode);
1399 struct f2fs_summary sum;
1400 struct node_info ni;
1401 block_t old_blkaddr;
1402 blkcnt_t count = 1;
1403 int err;
1404
1405 if (unlikely(is_inode_flag_set(dn->inode, FI_NO_ALLOC)))
1406 return -EPERM;
1407
1408 err = f2fs_get_node_info(sbi, dn->nid, &ni);
1409 if (err)
1410 return err;
1411
1412 dn->data_blkaddr = f2fs_data_blkaddr(dn);
1413 if (dn->data_blkaddr != NULL_ADDR)
1414 goto alloc;
1415
1416 if (unlikely((err = inc_valid_block_count(sbi, dn->inode, &count))))
1417 return err;
1418
1419 alloc:
1420 set_summary(&sum, dn->nid, dn->ofs_in_node, ni.version);
1421 old_blkaddr = dn->data_blkaddr;
1422 f2fs_allocate_data_block(sbi, NULL, old_blkaddr, &dn->data_blkaddr,
1423 &sum, seg_type, NULL);
1424 if (GET_SEGNO(sbi, old_blkaddr) != NULL_SEGNO)
1425 invalidate_mapping_pages(META_MAPPING(sbi),
1426 old_blkaddr, old_blkaddr);
1427 f2fs_update_data_blkaddr(dn, dn->data_blkaddr);
1428
1429 /*
1430 * i_size will be updated by direct_IO. Otherwise, we'll get stale
1431 * data from unwritten block via dio_read.
1432 */
1433 return 0;
1434 }
1435
1436 int f2fs_preallocate_blocks(struct kiocb *iocb, struct iov_iter *from)
1437 {
1438 struct inode *inode = file_inode(iocb->ki_filp);
1439 struct f2fs_map_blocks map;
1440 int flag;
1441 int err = 0;
1442 bool direct_io = iocb->ki_flags & IOCB_DIRECT;
1443
1444 map.m_lblk = F2FS_BLK_ALIGN(iocb->ki_pos);
1445 map.m_len = F2FS_BYTES_TO_BLK(iocb->ki_pos + iov_iter_count(from));
1446 if (map.m_len > map.m_lblk)
1447 map.m_len -= map.m_lblk;
1448 else
1449 map.m_len = 0;
1450
1451 map.m_next_pgofs = NULL;
1452 map.m_next_extent = NULL;
1453 map.m_seg_type = NO_CHECK_TYPE;
1454 map.m_may_create = true;
1455
1456 if (direct_io) {
1457 map.m_seg_type = f2fs_rw_hint_to_seg_type(iocb->ki_hint);
1458 flag = f2fs_force_buffered_io(inode, iocb, from) ?
1459 F2FS_GET_BLOCK_PRE_AIO :
1460 F2FS_GET_BLOCK_PRE_DIO;
1461 goto map_blocks;
1462 }
1463 if (iocb->ki_pos + iov_iter_count(from) > MAX_INLINE_DATA(inode)) {
1464 err = f2fs_convert_inline_inode(inode);
1465 if (err)
1466 return err;
1467 }
1468 if (f2fs_has_inline_data(inode))
1469 return err;
1470
1471 flag = F2FS_GET_BLOCK_PRE_AIO;
1472
1473 map_blocks:
1474 err = f2fs_map_blocks(inode, &map, 1, flag);
1475 if (map.m_len > 0 && err == -ENOSPC) {
1476 if (!direct_io)
1477 set_inode_flag(inode, FI_NO_PREALLOC);
1478 err = 0;
1479 }
1480 return err;
1481 }
1482
1483 void f2fs_do_map_lock(struct f2fs_sb_info *sbi, int flag, bool lock)
1484 {
1485 if (flag == F2FS_GET_BLOCK_PRE_AIO) {
1486 if (lock)
1487 down_read(&sbi->node_change);
1488 else
1489 up_read(&sbi->node_change);
1490 } else {
1491 if (lock)
1492 f2fs_lock_op(sbi);
1493 else
1494 f2fs_unlock_op(sbi);
1495 }
1496 }
1497
1498 /*
1499 * f2fs_map_blocks() tries to find or build mapping relationship which
1500 * maps continuous logical blocks to physical blocks, and return such
1501 * info via f2fs_map_blocks structure.
1502 */
1503 int f2fs_map_blocks(struct inode *inode, struct f2fs_map_blocks *map,
1504 int create, int flag)
1505 {
1506 unsigned int maxblocks = map->m_len;
1507 struct dnode_of_data dn;
1508 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
1509 int mode = map->m_may_create ? ALLOC_NODE : LOOKUP_NODE;
1510 pgoff_t pgofs, end_offset, end;
1511 int err = 0, ofs = 1;
1512 unsigned int ofs_in_node, last_ofs_in_node;
1513 blkcnt_t prealloc;
1514 struct extent_info ei = {0,0,0};
1515 block_t blkaddr;
1516 unsigned int start_pgofs;
1517
1518 if (!maxblocks)
1519 return 0;
1520
1521 map->m_len = 0;
1522 map->m_flags = 0;
1523
1524 /* it only supports block size == page size */
1525 pgofs = (pgoff_t)map->m_lblk;
1526 end = pgofs + maxblocks;
1527
1528 if (!create && f2fs_lookup_extent_cache(inode, pgofs, &ei)) {
1529 if (f2fs_lfs_mode(sbi) && flag == F2FS_GET_BLOCK_DIO &&
1530 map->m_may_create)
1531 goto next_dnode;
1532
1533 map->m_pblk = ei.blk + pgofs - ei.fofs;
1534 map->m_len = min((pgoff_t)maxblocks, ei.fofs + ei.len - pgofs);
1535 map->m_flags = F2FS_MAP_MAPPED;
1536 if (map->m_next_extent)
1537 *map->m_next_extent = pgofs + map->m_len;
1538
1539 /* for hardware encryption, but to avoid potential issue in future */
1540 if (flag == F2FS_GET_BLOCK_DIO)
1541 f2fs_wait_on_block_writeback_range(inode,
1542 map->m_pblk, map->m_len);
1543 goto out;
1544 }
1545
1546 next_dnode:
1547 if (map->m_may_create)
1548 f2fs_do_map_lock(sbi, flag, true);
1549
1550 /* When reading holes, we need its node page */
1551 set_new_dnode(&dn, inode, NULL, NULL, 0);
1552 err = f2fs_get_dnode_of_data(&dn, pgofs, mode);
1553 if (err) {
1554 if (flag == F2FS_GET_BLOCK_BMAP)
1555 map->m_pblk = 0;
1556 if (err == -ENOENT) {
1557 err = 0;
1558 if (map->m_next_pgofs)
1559 *map->m_next_pgofs =
1560 f2fs_get_next_page_offset(&dn, pgofs);
1561 if (map->m_next_extent)
1562 *map->m_next_extent =
1563 f2fs_get_next_page_offset(&dn, pgofs);
1564 }
1565 goto unlock_out;
1566 }
1567
1568 start_pgofs = pgofs;
1569 prealloc = 0;
1570 last_ofs_in_node = ofs_in_node = dn.ofs_in_node;
1571 end_offset = ADDRS_PER_PAGE(dn.node_page, inode);
1572
1573 next_block:
1574 blkaddr = f2fs_data_blkaddr(&dn);
1575
1576 if (__is_valid_data_blkaddr(blkaddr) &&
1577 !f2fs_is_valid_blkaddr(sbi, blkaddr, DATA_GENERIC_ENHANCE)) {
1578 err = -EFSCORRUPTED;
1579 goto sync_out;
1580 }
1581
1582 if (__is_valid_data_blkaddr(blkaddr)) {
1583 /* use out-place-update for driect IO under LFS mode */
1584 if (f2fs_lfs_mode(sbi) && flag == F2FS_GET_BLOCK_DIO &&
1585 map->m_may_create) {
1586 err = __allocate_data_block(&dn, map->m_seg_type);
1587 if (err)
1588 goto sync_out;
1589 blkaddr = dn.data_blkaddr;
1590 set_inode_flag(inode, FI_APPEND_WRITE);
1591 }
1592 } else {
1593 if (create) {
1594 if (unlikely(f2fs_cp_error(sbi))) {
1595 err = -EIO;
1596 goto sync_out;
1597 }
1598 if (flag == F2FS_GET_BLOCK_PRE_AIO) {
1599 if (blkaddr == NULL_ADDR) {
1600 prealloc++;
1601 last_ofs_in_node = dn.ofs_in_node;
1602 }
1603 } else {
1604 WARN_ON(flag != F2FS_GET_BLOCK_PRE_DIO &&
1605 flag != F2FS_GET_BLOCK_DIO);
1606 err = __allocate_data_block(&dn,
1607 map->m_seg_type);
1608 if (!err)
1609 set_inode_flag(inode, FI_APPEND_WRITE);
1610 }
1611 if (err)
1612 goto sync_out;
1613 map->m_flags |= F2FS_MAP_NEW;
1614 blkaddr = dn.data_blkaddr;
1615 } else {
1616 if (flag == F2FS_GET_BLOCK_BMAP) {
1617 map->m_pblk = 0;
1618 goto sync_out;
1619 }
1620 if (flag == F2FS_GET_BLOCK_PRECACHE)
1621 goto sync_out;
1622 if (flag == F2FS_GET_BLOCK_FIEMAP &&
1623 blkaddr == NULL_ADDR) {
1624 if (map->m_next_pgofs)
1625 *map->m_next_pgofs = pgofs + 1;
1626 goto sync_out;
1627 }
1628 if (flag != F2FS_GET_BLOCK_FIEMAP) {
1629 /* for defragment case */
1630 if (map->m_next_pgofs)
1631 *map->m_next_pgofs = pgofs + 1;
1632 goto sync_out;
1633 }
1634 }
1635 }
1636
1637 if (flag == F2FS_GET_BLOCK_PRE_AIO)
1638 goto skip;
1639
1640 if (map->m_len == 0) {
1641 /* preallocated unwritten block should be mapped for fiemap. */
1642 if (blkaddr == NEW_ADDR)
1643 map->m_flags |= F2FS_MAP_UNWRITTEN;
1644 map->m_flags |= F2FS_MAP_MAPPED;
1645
1646 map->m_pblk = blkaddr;
1647 map->m_len = 1;
1648 } else if ((map->m_pblk != NEW_ADDR &&
1649 blkaddr == (map->m_pblk + ofs)) ||
1650 (map->m_pblk == NEW_ADDR && blkaddr == NEW_ADDR) ||
1651 flag == F2FS_GET_BLOCK_PRE_DIO) {
1652 ofs++;
1653 map->m_len++;
1654 } else {
1655 goto sync_out;
1656 }
1657
1658 skip:
1659 dn.ofs_in_node++;
1660 pgofs++;
1661
1662 /* preallocate blocks in batch for one dnode page */
1663 if (flag == F2FS_GET_BLOCK_PRE_AIO &&
1664 (pgofs == end || dn.ofs_in_node == end_offset)) {
1665
1666 dn.ofs_in_node = ofs_in_node;
1667 err = f2fs_reserve_new_blocks(&dn, prealloc);
1668 if (err)
1669 goto sync_out;
1670
1671 map->m_len += dn.ofs_in_node - ofs_in_node;
1672 if (prealloc && dn.ofs_in_node != last_ofs_in_node + 1) {
1673 err = -ENOSPC;
1674 goto sync_out;
1675 }
1676 dn.ofs_in_node = end_offset;
1677 }
1678
1679 if (pgofs >= end)
1680 goto sync_out;
1681 else if (dn.ofs_in_node < end_offset)
1682 goto next_block;
1683
1684 if (flag == F2FS_GET_BLOCK_PRECACHE) {
1685 if (map->m_flags & F2FS_MAP_MAPPED) {
1686 unsigned int ofs = start_pgofs - map->m_lblk;
1687
1688 f2fs_update_extent_cache_range(&dn,
1689 start_pgofs, map->m_pblk + ofs,
1690 map->m_len - ofs);
1691 }
1692 }
1693
1694 f2fs_put_dnode(&dn);
1695
1696 if (map->m_may_create) {
1697 f2fs_do_map_lock(sbi, flag, false);
1698 f2fs_balance_fs(sbi, dn.node_changed);
1699 }
1700 goto next_dnode;
1701
1702 sync_out:
1703
1704 /* for hardware encryption, but to avoid potential issue in future */
1705 if (flag == F2FS_GET_BLOCK_DIO && map->m_flags & F2FS_MAP_MAPPED)
1706 f2fs_wait_on_block_writeback_range(inode,
1707 map->m_pblk, map->m_len);
1708
1709 if (flag == F2FS_GET_BLOCK_PRECACHE) {
1710 if (map->m_flags & F2FS_MAP_MAPPED) {
1711 unsigned int ofs = start_pgofs - map->m_lblk;
1712
1713 f2fs_update_extent_cache_range(&dn,
1714 start_pgofs, map->m_pblk + ofs,
1715 map->m_len - ofs);
1716 }
1717 if (map->m_next_extent)
1718 *map->m_next_extent = pgofs + 1;
1719 }
1720 f2fs_put_dnode(&dn);
1721 unlock_out:
1722 if (map->m_may_create) {
1723 f2fs_do_map_lock(sbi, flag, false);
1724 f2fs_balance_fs(sbi, dn.node_changed);
1725 }
1726 out:
1727 trace_f2fs_map_blocks(inode, map, err);
1728 return err;
1729 }
1730
1731 bool f2fs_overwrite_io(struct inode *inode, loff_t pos, size_t len)
1732 {
1733 struct f2fs_map_blocks map;
1734 block_t last_lblk;
1735 int err;
1736
1737 if (pos + len > i_size_read(inode))
1738 return false;
1739
1740 map.m_lblk = F2FS_BYTES_TO_BLK(pos);
1741 map.m_next_pgofs = NULL;
1742 map.m_next_extent = NULL;
1743 map.m_seg_type = NO_CHECK_TYPE;
1744 map.m_may_create = false;
1745 last_lblk = F2FS_BLK_ALIGN(pos + len);
1746
1747 while (map.m_lblk < last_lblk) {
1748 map.m_len = last_lblk - map.m_lblk;
1749 err = f2fs_map_blocks(inode, &map, 0, F2FS_GET_BLOCK_DEFAULT);
1750 if (err || map.m_len == 0)
1751 return false;
1752 map.m_lblk += map.m_len;
1753 }
1754 return true;
1755 }
1756
1757 static inline u64 bytes_to_blks(struct inode *inode, u64 bytes)
1758 {
1759 return (bytes >> inode->i_blkbits);
1760 }
1761
1762 static inline u64 blks_to_bytes(struct inode *inode, u64 blks)
1763 {
1764 return (blks << inode->i_blkbits);
1765 }
1766
1767 static int __get_data_block(struct inode *inode, sector_t iblock,
1768 struct buffer_head *bh, int create, int flag,
1769 pgoff_t *next_pgofs, int seg_type, bool may_write)
1770 {
1771 struct f2fs_map_blocks map;
1772 int err;
1773
1774 map.m_lblk = iblock;
1775 map.m_len = bytes_to_blks(inode, bh->b_size);
1776 map.m_next_pgofs = next_pgofs;
1777 map.m_next_extent = NULL;
1778 map.m_seg_type = seg_type;
1779 map.m_may_create = may_write;
1780
1781 err = f2fs_map_blocks(inode, &map, create, flag);
1782 if (!err) {
1783 map_bh(bh, inode->i_sb, map.m_pblk);
1784 bh->b_state = (bh->b_state & ~F2FS_MAP_FLAGS) | map.m_flags;
1785 bh->b_size = blks_to_bytes(inode, map.m_len);
1786 }
1787 return err;
1788 }
1789
1790 static int get_data_block_dio_write(struct inode *inode, sector_t iblock,
1791 struct buffer_head *bh_result, int create)
1792 {
1793 return __get_data_block(inode, iblock, bh_result, create,
1794 F2FS_GET_BLOCK_DIO, NULL,
1795 f2fs_rw_hint_to_seg_type(inode->i_write_hint),
1796 IS_SWAPFILE(inode) ? false : true);
1797 }
1798
1799 static int get_data_block_dio(struct inode *inode, sector_t iblock,
1800 struct buffer_head *bh_result, int create)
1801 {
1802 return __get_data_block(inode, iblock, bh_result, create,
1803 F2FS_GET_BLOCK_DIO, NULL,
1804 f2fs_rw_hint_to_seg_type(inode->i_write_hint),
1805 false);
1806 }
1807
1808 static int f2fs_xattr_fiemap(struct inode *inode,
1809 struct fiemap_extent_info *fieinfo)
1810 {
1811 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
1812 struct page *page;
1813 struct node_info ni;
1814 __u64 phys = 0, len;
1815 __u32 flags;
1816 nid_t xnid = F2FS_I(inode)->i_xattr_nid;
1817 int err = 0;
1818
1819 if (f2fs_has_inline_xattr(inode)) {
1820 int offset;
1821
1822 page = f2fs_grab_cache_page(NODE_MAPPING(sbi),
1823 inode->i_ino, false);
1824 if (!page)
1825 return -ENOMEM;
1826
1827 err = f2fs_get_node_info(sbi, inode->i_ino, &ni);
1828 if (err) {
1829 f2fs_put_page(page, 1);
1830 return err;
1831 }
1832
1833 phys = blks_to_bytes(inode, ni.blk_addr);
1834 offset = offsetof(struct f2fs_inode, i_addr) +
1835 sizeof(__le32) * (DEF_ADDRS_PER_INODE -
1836 get_inline_xattr_addrs(inode));
1837
1838 phys += offset;
1839 len = inline_xattr_size(inode);
1840
1841 f2fs_put_page(page, 1);
1842
1843 flags = FIEMAP_EXTENT_DATA_INLINE | FIEMAP_EXTENT_NOT_ALIGNED;
1844
1845 if (!xnid)
1846 flags |= FIEMAP_EXTENT_LAST;
1847
1848 err = fiemap_fill_next_extent(fieinfo, 0, phys, len, flags);
1849 trace_f2fs_fiemap(inode, 0, phys, len, flags, err);
1850 if (err || err == 1)
1851 return err;
1852 }
1853
1854 if (xnid) {
1855 page = f2fs_grab_cache_page(NODE_MAPPING(sbi), xnid, false);
1856 if (!page)
1857 return -ENOMEM;
1858
1859 err = f2fs_get_node_info(sbi, xnid, &ni);
1860 if (err) {
1861 f2fs_put_page(page, 1);
1862 return err;
1863 }
1864
1865 phys = blks_to_bytes(inode, ni.blk_addr);
1866 len = inode->i_sb->s_blocksize;
1867
1868 f2fs_put_page(page, 1);
1869
1870 flags = FIEMAP_EXTENT_LAST;
1871 }
1872
1873 if (phys) {
1874 err = fiemap_fill_next_extent(fieinfo, 0, phys, len, flags);
1875 trace_f2fs_fiemap(inode, 0, phys, len, flags, err);
1876 }
1877
1878 return (err < 0 ? err : 0);
1879 }
1880
1881 static loff_t max_inode_blocks(struct inode *inode)
1882 {
1883 loff_t result = ADDRS_PER_INODE(inode);
1884 loff_t leaf_count = ADDRS_PER_BLOCK(inode);
1885
1886 /* two direct node blocks */
1887 result += (leaf_count * 2);
1888
1889 /* two indirect node blocks */
1890 leaf_count *= NIDS_PER_BLOCK;
1891 result += (leaf_count * 2);
1892
1893 /* one double indirect node block */
1894 leaf_count *= NIDS_PER_BLOCK;
1895 result += leaf_count;
1896
1897 return result;
1898 }
1899
1900 int f2fs_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo,
1901 u64 start, u64 len)
1902 {
1903 struct f2fs_map_blocks map;
1904 sector_t start_blk, last_blk;
1905 pgoff_t next_pgofs;
1906 u64 logical = 0, phys = 0, size = 0;
1907 u32 flags = 0;
1908 int ret = 0;
1909 bool compr_cluster = false;
1910 unsigned int cluster_size = F2FS_I(inode)->i_cluster_size;
1911
1912 if (fieinfo->fi_flags & FIEMAP_FLAG_CACHE) {
1913 ret = f2fs_precache_extents(inode);
1914 if (ret)
1915 return ret;
1916 }
1917
1918 ret = fiemap_prep(inode, fieinfo, start, &len, FIEMAP_FLAG_XATTR);
1919 if (ret)
1920 return ret;
1921
1922 inode_lock(inode);
1923
1924 if (fieinfo->fi_flags & FIEMAP_FLAG_XATTR) {
1925 ret = f2fs_xattr_fiemap(inode, fieinfo);
1926 goto out;
1927 }
1928
1929 if (f2fs_has_inline_data(inode) || f2fs_has_inline_dentry(inode)) {
1930 ret = f2fs_inline_data_fiemap(inode, fieinfo, start, len);
1931 if (ret != -EAGAIN)
1932 goto out;
1933 }
1934
1935 if (bytes_to_blks(inode, len) == 0)
1936 len = blks_to_bytes(inode, 1);
1937
1938 start_blk = bytes_to_blks(inode, start);
1939 last_blk = bytes_to_blks(inode, start + len - 1);
1940
1941 next:
1942 memset(&map, 0, sizeof(map));
1943 map.m_lblk = start_blk;
1944 map.m_len = bytes_to_blks(inode, len);
1945 map.m_next_pgofs = &next_pgofs;
1946 map.m_seg_type = NO_CHECK_TYPE;
1947
1948 if (compr_cluster)
1949 map.m_len = cluster_size - 1;
1950
1951 ret = f2fs_map_blocks(inode, &map, 0, F2FS_GET_BLOCK_FIEMAP);
1952 if (ret)
1953 goto out;
1954
1955 /* HOLE */
1956 if (!(map.m_flags & F2FS_MAP_FLAGS)) {
1957 start_blk = next_pgofs;
1958
1959 if (blks_to_bytes(inode, start_blk) < blks_to_bytes(inode,
1960 max_inode_blocks(inode)))
1961 goto prep_next;
1962
1963 flags |= FIEMAP_EXTENT_LAST;
1964 }
1965
1966 if (size) {
1967 if (IS_ENCRYPTED(inode))
1968 flags |= FIEMAP_EXTENT_DATA_ENCRYPTED;
1969
1970 ret = fiemap_fill_next_extent(fieinfo, logical,
1971 phys, size, flags);
1972 trace_f2fs_fiemap(inode, logical, phys, size, flags, ret);
1973 if (ret)
1974 goto out;
1975 size = 0;
1976 }
1977
1978 if (start_blk > last_blk)
1979 goto out;
1980
1981 if (compr_cluster) {
1982 compr_cluster = false;
1983
1984
1985 logical = blks_to_bytes(inode, start_blk - 1);
1986 phys = blks_to_bytes(inode, map.m_pblk);
1987 size = blks_to_bytes(inode, cluster_size);
1988
1989 flags |= FIEMAP_EXTENT_ENCODED;
1990
1991 start_blk += cluster_size - 1;
1992
1993 if (start_blk > last_blk)
1994 goto out;
1995
1996 goto prep_next;
1997 }
1998
1999 if (map.m_pblk == COMPRESS_ADDR) {
2000 compr_cluster = true;
2001 start_blk++;
2002 goto prep_next;
2003 }
2004
2005 logical = blks_to_bytes(inode, start_blk);
2006 phys = blks_to_bytes(inode, map.m_pblk);
2007 size = blks_to_bytes(inode, map.m_len);
2008 flags = 0;
2009 if (map.m_flags & F2FS_MAP_UNWRITTEN)
2010 flags = FIEMAP_EXTENT_UNWRITTEN;
2011
2012 start_blk += bytes_to_blks(inode, size);
2013
2014 prep_next:
2015 cond_resched();
2016 if (fatal_signal_pending(current))
2017 ret = -EINTR;
2018 else
2019 goto next;
2020 out:
2021 if (ret == 1)
2022 ret = 0;
2023
2024 inode_unlock(inode);
2025 return ret;
2026 }
2027
2028 static inline loff_t f2fs_readpage_limit(struct inode *inode)
2029 {
2030 if (IS_ENABLED(CONFIG_FS_VERITY) &&
2031 (IS_VERITY(inode) || f2fs_verity_in_progress(inode)))
2032 return inode->i_sb->s_maxbytes;
2033
2034 return i_size_read(inode);
2035 }
2036
2037 static int f2fs_read_single_page(struct inode *inode, struct page *page,
2038 unsigned nr_pages,
2039 struct f2fs_map_blocks *map,
2040 struct bio **bio_ret,
2041 sector_t *last_block_in_bio,
2042 bool is_readahead)
2043 {
2044 struct bio *bio = *bio_ret;
2045 const unsigned blocksize = blks_to_bytes(inode, 1);
2046 sector_t block_in_file;
2047 sector_t last_block;
2048 sector_t last_block_in_file;
2049 sector_t block_nr;
2050 int ret = 0;
2051
2052 block_in_file = (sector_t)page_index(page);
2053 last_block = block_in_file + nr_pages;
2054 last_block_in_file = bytes_to_blks(inode,
2055 f2fs_readpage_limit(inode) + blocksize - 1);
2056 if (last_block > last_block_in_file)
2057 last_block = last_block_in_file;
2058
2059 /* just zeroing out page which is beyond EOF */
2060 if (block_in_file >= last_block)
2061 goto zero_out;
2062 /*
2063 * Map blocks using the previous result first.
2064 */
2065 if ((map->m_flags & F2FS_MAP_MAPPED) &&
2066 block_in_file > map->m_lblk &&
2067 block_in_file < (map->m_lblk + map->m_len))
2068 goto got_it;
2069
2070 /*
2071 * Then do more f2fs_map_blocks() calls until we are
2072 * done with this page.
2073 */
2074 map->m_lblk = block_in_file;
2075 map->m_len = last_block - block_in_file;
2076
2077 ret = f2fs_map_blocks(inode, map, 0, F2FS_GET_BLOCK_DEFAULT);
2078 if (ret)
2079 goto out;
2080 got_it:
2081 if ((map->m_flags & F2FS_MAP_MAPPED)) {
2082 block_nr = map->m_pblk + block_in_file - map->m_lblk;
2083 SetPageMappedToDisk(page);
2084
2085 if (!PageUptodate(page) && (!PageSwapCache(page) &&
2086 !cleancache_get_page(page))) {
2087 SetPageUptodate(page);
2088 goto confused;
2089 }
2090
2091 if (!f2fs_is_valid_blkaddr(F2FS_I_SB(inode), block_nr,
2092 DATA_GENERIC_ENHANCE_READ)) {
2093 ret = -EFSCORRUPTED;
2094 goto out;
2095 }
2096 } else {
2097 zero_out:
2098 zero_user_segment(page, 0, PAGE_SIZE);
2099 if (f2fs_need_verity(inode, page->index) &&
2100 !fsverity_verify_page(page)) {
2101 ret = -EIO;
2102 goto out;
2103 }
2104 if (!PageUptodate(page))
2105 SetPageUptodate(page);
2106 unlock_page(page);
2107 goto out;
2108 }
2109
2110 /*
2111 * This page will go to BIO. Do we need to send this
2112 * BIO off first?
2113 */
2114 if (bio && (!page_is_mergeable(F2FS_I_SB(inode), bio,
2115 *last_block_in_bio, block_nr) ||
2116 !f2fs_crypt_mergeable_bio(bio, inode, page->index, NULL))) {
2117 submit_and_realloc:
2118 __submit_bio(F2FS_I_SB(inode), bio, DATA);
2119 bio = NULL;
2120 }
2121 if (bio == NULL) {
2122 bio = f2fs_grab_read_bio(inode, block_nr, nr_pages,
2123 is_readahead ? REQ_RAHEAD : 0, page->index,
2124 false, true);
2125 if (IS_ERR(bio)) {
2126 ret = PTR_ERR(bio);
2127 bio = NULL;
2128 goto out;
2129 }
2130 }
2131
2132 /*
2133 * If the page is under writeback, we need to wait for
2134 * its completion to see the correct decrypted data.
2135 */
2136 f2fs_wait_on_block_writeback(inode, block_nr);
2137
2138 if (bio_add_page(bio, page, blocksize, 0) < blocksize)
2139 goto submit_and_realloc;
2140
2141 inc_page_count(F2FS_I_SB(inode), F2FS_RD_DATA);
2142 f2fs_update_iostat(F2FS_I_SB(inode), FS_DATA_READ_IO, F2FS_BLKSIZE);
2143 ClearPageError(page);
2144 *last_block_in_bio = block_nr;
2145 goto out;
2146 confused:
2147 if (bio) {
2148 __submit_bio(F2FS_I_SB(inode), bio, DATA);
2149 bio = NULL;
2150 }
2151 unlock_page(page);
2152 out:
2153 *bio_ret = bio;
2154 return ret;
2155 }
2156
2157 #ifdef CONFIG_F2FS_FS_COMPRESSION
2158 int f2fs_read_multi_pages(struct compress_ctx *cc, struct bio **bio_ret,
2159 unsigned nr_pages, sector_t *last_block_in_bio,
2160 bool is_readahead, bool for_write)
2161 {
2162 struct dnode_of_data dn;
2163 struct inode *inode = cc->inode;
2164 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
2165 struct bio *bio = *bio_ret;
2166 unsigned int start_idx = cc->cluster_idx << cc->log_cluster_size;
2167 sector_t last_block_in_file;
2168 const unsigned blocksize = blks_to_bytes(inode, 1);
2169 struct decompress_io_ctx *dic = NULL;
2170 struct bio_post_read_ctx *ctx;
2171 bool for_verity = false;
2172 int i;
2173 int ret = 0;
2174
2175 f2fs_bug_on(sbi, f2fs_cluster_is_empty(cc));
2176
2177 last_block_in_file = bytes_to_blks(inode,
2178 f2fs_readpage_limit(inode) + blocksize - 1);
2179
2180 /* get rid of pages beyond EOF */
2181 for (i = 0; i < cc->cluster_size; i++) {
2182 struct page *page = cc->rpages[i];
2183
2184 if (!page)
2185 continue;
2186 if ((sector_t)page->index >= last_block_in_file) {
2187 zero_user_segment(page, 0, PAGE_SIZE);
2188 if (!PageUptodate(page))
2189 SetPageUptodate(page);
2190 } else if (!PageUptodate(page)) {
2191 continue;
2192 }
2193 unlock_page(page);
2194 cc->rpages[i] = NULL;
2195 cc->nr_rpages--;
2196 }
2197
2198 /* we are done since all pages are beyond EOF */
2199 if (f2fs_cluster_is_empty(cc))
2200 goto out;
2201
2202 set_new_dnode(&dn, inode, NULL, NULL, 0);
2203 ret = f2fs_get_dnode_of_data(&dn, start_idx, LOOKUP_NODE);
2204 if (ret)
2205 goto out;
2206
2207 f2fs_bug_on(sbi, dn.data_blkaddr != COMPRESS_ADDR);
2208
2209 for (i = 1; i < cc->cluster_size; i++) {
2210 block_t blkaddr;
2211
2212 blkaddr = data_blkaddr(dn.inode, dn.node_page,
2213 dn.ofs_in_node + i);
2214
2215 if (!__is_valid_data_blkaddr(blkaddr))
2216 break;
2217
2218 if (!f2fs_is_valid_blkaddr(sbi, blkaddr, DATA_GENERIC)) {
2219 ret = -EFAULT;
2220 goto out_put_dnode;
2221 }
2222 cc->nr_cpages++;
2223 }
2224
2225 /* nothing to decompress */
2226 if (cc->nr_cpages == 0) {
2227 ret = 0;
2228 goto out_put_dnode;
2229 }
2230
2231 dic = f2fs_alloc_dic(cc);
2232 if (IS_ERR(dic)) {
2233 ret = PTR_ERR(dic);
2234 goto out_put_dnode;
2235 }
2236
2237 /*
2238 * It's possible to enable fsverity on the fly when handling a cluster,
2239 * which requires complicated error handling. Instead of adding more
2240 * complexity, let's give a rule where end_io post-processes fsverity
2241 * per cluster. In order to do that, we need to submit bio, if previous
2242 * bio sets a different post-process policy.
2243 */
2244 if (fsverity_active(cc->inode)) {
2245 atomic_set(&dic->verity_pages, cc->nr_cpages);
2246 for_verity = true;
2247
2248 if (bio) {
2249 ctx = bio->bi_private;
2250 if (!(ctx->enabled_steps & (1 << STEP_VERITY))) {
2251 __submit_bio(sbi, bio, DATA);
2252 bio = NULL;
2253 }
2254 }
2255 }
2256
2257 for (i = 0; i < dic->nr_cpages; i++) {
2258 struct page *page = dic->cpages[i];
2259 block_t blkaddr;
2260
2261 blkaddr = data_blkaddr(dn.inode, dn.node_page,
2262 dn.ofs_in_node + i + 1);
2263
2264 if (bio && (!page_is_mergeable(sbi, bio,
2265 *last_block_in_bio, blkaddr) ||
2266 !f2fs_crypt_mergeable_bio(bio, inode, page->index, NULL))) {
2267 submit_and_realloc:
2268 __submit_bio(sbi, bio, DATA);
2269 bio = NULL;
2270 }
2271
2272 if (!bio) {
2273 bio = f2fs_grab_read_bio(inode, blkaddr, nr_pages,
2274 is_readahead ? REQ_RAHEAD : 0,
2275 page->index, for_write, for_verity);
2276 if (IS_ERR(bio)) {
2277 unsigned int remained = dic->nr_cpages - i;
2278 bool release = false;
2279
2280 ret = PTR_ERR(bio);
2281 dic->failed = true;
2282
2283 if (for_verity) {
2284 if (!atomic_sub_return(remained,
2285 &dic->verity_pages))
2286 release = true;
2287 } else {
2288 if (!atomic_sub_return(remained,
2289 &dic->pending_pages))
2290 release = true;
2291 }
2292
2293 if (release) {
2294 f2fs_decompress_end_io(dic->rpages,
2295 cc->cluster_size, true,
2296 false);
2297 f2fs_free_dic(dic);
2298 }
2299
2300 f2fs_put_dnode(&dn);
2301 *bio_ret = NULL;
2302 return ret;
2303 }
2304 }
2305
2306 f2fs_wait_on_block_writeback(inode, blkaddr);
2307
2308 if (bio_add_page(bio, page, blocksize, 0) < blocksize)
2309 goto submit_and_realloc;
2310
2311 /* tag STEP_DECOMPRESS to handle IO in wq */
2312 ctx = bio->bi_private;
2313 if (!(ctx->enabled_steps & (1 << STEP_DECOMPRESS)))
2314 ctx->enabled_steps |= 1 << STEP_DECOMPRESS;
2315
2316 inc_page_count(sbi, F2FS_RD_DATA);
2317 f2fs_update_iostat(sbi, FS_DATA_READ_IO, F2FS_BLKSIZE);
2318 f2fs_update_iostat(sbi, FS_CDATA_READ_IO, F2FS_BLKSIZE);
2319 ClearPageError(page);
2320 *last_block_in_bio = blkaddr;
2321 }
2322
2323 f2fs_put_dnode(&dn);
2324
2325 *bio_ret = bio;
2326 return 0;
2327
2328 out_put_dnode:
2329 f2fs_put_dnode(&dn);
2330 out:
2331 f2fs_decompress_end_io(cc->rpages, cc->cluster_size, true, false);
2332 *bio_ret = bio;
2333 return ret;
2334 }
2335 #endif
2336
2337 /*
2338 * This function was originally taken from fs/mpage.c, and customized for f2fs.
2339 * Major change was from block_size == page_size in f2fs by default.
2340 *
2341 * Note that the aops->readpages() function is ONLY used for read-ahead. If
2342 * this function ever deviates from doing just read-ahead, it should either
2343 * use ->readpage() or do the necessary surgery to decouple ->readpages()
2344 * from read-ahead.
2345 */
2346 static int f2fs_mpage_readpages(struct inode *inode,
2347 struct readahead_control *rac, struct page *page)
2348 {
2349 struct bio *bio = NULL;
2350 sector_t last_block_in_bio = 0;
2351 struct f2fs_map_blocks map;
2352 #ifdef CONFIG_F2FS_FS_COMPRESSION
2353 struct compress_ctx cc = {
2354 .inode = inode,
2355 .log_cluster_size = F2FS_I(inode)->i_log_cluster_size,
2356 .cluster_size = F2FS_I(inode)->i_cluster_size,
2357 .cluster_idx = NULL_CLUSTER,
2358 .rpages = NULL,
2359 .cpages = NULL,
2360 .nr_rpages = 0,
2361 .nr_cpages = 0,
2362 };
2363 #endif
2364 unsigned nr_pages = rac ? readahead_count(rac) : 1;
2365 unsigned max_nr_pages = nr_pages;
2366 int ret = 0;
2367 bool drop_ra = false;
2368
2369 map.m_pblk = 0;
2370 map.m_lblk = 0;
2371 map.m_len = 0;
2372 map.m_flags = 0;
2373 map.m_next_pgofs = NULL;
2374 map.m_next_extent = NULL;
2375 map.m_seg_type = NO_CHECK_TYPE;
2376 map.m_may_create = false;
2377
2378 /*
2379 * Two readahead threads for same address range can cause race condition
2380 * which fragments sequential read IOs. So let's avoid each other.
2381 */
2382 if (rac && readahead_count(rac)) {
2383 if (READ_ONCE(F2FS_I(inode)->ra_offset) == readahead_index(rac))
2384 drop_ra = true;
2385 else
2386 WRITE_ONCE(F2FS_I(inode)->ra_offset,
2387 readahead_index(rac));
2388 }
2389
2390 for (; nr_pages; nr_pages--) {
2391 if (rac) {
2392 page = readahead_page(rac);
2393 prefetchw(&page->flags);
2394 if (drop_ra) {
2395 f2fs_put_page(page, 1);
2396 continue;
2397 }
2398 }
2399
2400 #ifdef CONFIG_F2FS_FS_COMPRESSION
2401 if (f2fs_compressed_file(inode)) {
2402 /* there are remained comressed pages, submit them */
2403 if (!f2fs_cluster_can_merge_page(&cc, page->index)) {
2404 ret = f2fs_read_multi_pages(&cc, &bio,
2405 max_nr_pages,
2406 &last_block_in_bio,
2407 rac != NULL, false);
2408 f2fs_destroy_compress_ctx(&cc);
2409 if (ret)
2410 goto set_error_page;
2411 }
2412 ret = f2fs_is_compressed_cluster(inode, page->index);
2413 if (ret < 0)
2414 goto set_error_page;
2415 else if (!ret)
2416 goto read_single_page;
2417
2418 ret = f2fs_init_compress_ctx(&cc);
2419 if (ret)
2420 goto set_error_page;
2421
2422 f2fs_compress_ctx_add_page(&cc, page);
2423
2424 goto next_page;
2425 }
2426 read_single_page:
2427 #endif
2428
2429 ret = f2fs_read_single_page(inode, page, max_nr_pages, &map,
2430 &bio, &last_block_in_bio, rac);
2431 if (ret) {
2432 #ifdef CONFIG_F2FS_FS_COMPRESSION
2433 set_error_page:
2434 #endif
2435 SetPageError(page);
2436 zero_user_segment(page, 0, PAGE_SIZE);
2437 unlock_page(page);
2438 }
2439 #ifdef CONFIG_F2FS_FS_COMPRESSION
2440 next_page:
2441 #endif
2442 if (rac)
2443 put_page(page);
2444
2445 #ifdef CONFIG_F2FS_FS_COMPRESSION
2446 if (f2fs_compressed_file(inode)) {
2447 /* last page */
2448 if (nr_pages == 1 && !f2fs_cluster_is_empty(&cc)) {
2449 ret = f2fs_read_multi_pages(&cc, &bio,
2450 max_nr_pages,
2451 &last_block_in_bio,
2452 rac != NULL, false);
2453 f2fs_destroy_compress_ctx(&cc);
2454 }
2455 }
2456 #endif
2457 }
2458 if (bio)
2459 __submit_bio(F2FS_I_SB(inode), bio, DATA);
2460
2461 if (rac && readahead_count(rac) && !drop_ra)
2462 WRITE_ONCE(F2FS_I(inode)->ra_offset, -1);
2463 return ret;
2464 }
2465
2466 static int f2fs_read_data_page(struct file *file, struct page *page)
2467 {
2468 struct inode *inode = page_file_mapping(page)->host;
2469 int ret = -EAGAIN;
2470
2471 trace_f2fs_readpage(page, DATA);
2472
2473 if (!f2fs_is_compress_backend_ready(inode)) {
2474 unlock_page(page);
2475 return -EOPNOTSUPP;
2476 }
2477
2478 /* If the file has inline data, try to read it directly */
2479 if (f2fs_has_inline_data(inode))
2480 ret = f2fs_read_inline_data(inode, page);
2481 if (ret == -EAGAIN)
2482 ret = f2fs_mpage_readpages(inode, NULL, page);
2483 return ret;
2484 }
2485
2486 static void f2fs_readahead(struct readahead_control *rac)
2487 {
2488 struct inode *inode = rac->mapping->host;
2489
2490 trace_f2fs_readpages(inode, readahead_index(rac), readahead_count(rac));
2491
2492 if (!f2fs_is_compress_backend_ready(inode))
2493 return;
2494
2495 /* If the file has inline data, skip readpages */
2496 if (f2fs_has_inline_data(inode))
2497 return;
2498
2499 f2fs_mpage_readpages(inode, rac, NULL);
2500 }
2501
2502 int f2fs_encrypt_one_page(struct f2fs_io_info *fio)
2503 {
2504 struct inode *inode = fio->page->mapping->host;
2505 struct page *mpage, *page;
2506 gfp_t gfp_flags = GFP_NOFS;
2507
2508 if (!f2fs_encrypted_file(inode))
2509 return 0;
2510
2511 page = fio->compressed_page ? fio->compressed_page : fio->page;
2512
2513 /* wait for GCed page writeback via META_MAPPING */
2514 f2fs_wait_on_block_writeback(inode, fio->old_blkaddr);
2515
2516 if (fscrypt_inode_uses_inline_crypto(inode))
2517 return 0;
2518
2519 retry_encrypt:
2520 fio->encrypted_page = fscrypt_encrypt_pagecache_blocks(page,
2521 PAGE_SIZE, 0, gfp_flags);
2522 if (IS_ERR(fio->encrypted_page)) {
2523 /* flush pending IOs and wait for a while in the ENOMEM case */
2524 if (PTR_ERR(fio->encrypted_page) == -ENOMEM) {
2525 f2fs_flush_merged_writes(fio->sbi);
2526 congestion_wait(BLK_RW_ASYNC, DEFAULT_IO_TIMEOUT);
2527 gfp_flags |= __GFP_NOFAIL;
2528 goto retry_encrypt;
2529 }
2530 return PTR_ERR(fio->encrypted_page);
2531 }
2532
2533 mpage = find_lock_page(META_MAPPING(fio->sbi), fio->old_blkaddr);
2534 if (mpage) {
2535 if (PageUptodate(mpage))
2536 memcpy(page_address(mpage),
2537 page_address(fio->encrypted_page), PAGE_SIZE);
2538 f2fs_put_page(mpage, 1);
2539 }
2540 return 0;
2541 }
2542
2543 static inline bool check_inplace_update_policy(struct inode *inode,
2544 struct f2fs_io_info *fio)
2545 {
2546 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
2547 unsigned int policy = SM_I(sbi)->ipu_policy;
2548
2549 if (policy & (0x1 << F2FS_IPU_FORCE))
2550 return true;
2551 if (policy & (0x1 << F2FS_IPU_SSR) && f2fs_need_SSR(sbi))
2552 return true;
2553 if (policy & (0x1 << F2FS_IPU_UTIL) &&
2554 utilization(sbi) > SM_I(sbi)->min_ipu_util)
2555 return true;
2556 if (policy & (0x1 << F2FS_IPU_SSR_UTIL) && f2fs_need_SSR(sbi) &&
2557 utilization(sbi) > SM_I(sbi)->min_ipu_util)
2558 return true;
2559
2560 /*
2561 * IPU for rewrite async pages
2562 */
2563 if (policy & (0x1 << F2FS_IPU_ASYNC) &&
2564 fio && fio->op == REQ_OP_WRITE &&
2565 !(fio->op_flags & REQ_SYNC) &&
2566 !IS_ENCRYPTED(inode))
2567 return true;
2568
2569 /* this is only set during fdatasync */
2570 if (policy & (0x1 << F2FS_IPU_FSYNC) &&
2571 is_inode_flag_set(inode, FI_NEED_IPU))
2572 return true;
2573
2574 if (unlikely(fio && is_sbi_flag_set(sbi, SBI_CP_DISABLED) &&
2575 !f2fs_is_checkpointed_data(sbi, fio->old_blkaddr)))
2576 return true;
2577
2578 return false;
2579 }
2580
2581 bool f2fs_should_update_inplace(struct inode *inode, struct f2fs_io_info *fio)
2582 {
2583 if (f2fs_is_pinned_file(inode))
2584 return true;
2585
2586 /* if this is cold file, we should overwrite to avoid fragmentation */
2587 if (file_is_cold(inode))
2588 return true;
2589
2590 return check_inplace_update_policy(inode, fio);
2591 }
2592
2593 bool f2fs_should_update_outplace(struct inode *inode, struct f2fs_io_info *fio)
2594 {
2595 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
2596
2597 if (f2fs_lfs_mode(sbi))
2598 return true;
2599 if (S_ISDIR(inode->i_mode))
2600 return true;
2601 if (IS_NOQUOTA(inode))
2602 return true;
2603 if (f2fs_is_atomic_file(inode))
2604 return true;
2605 if (fio) {
2606 if (is_cold_data(fio->page))
2607 return true;
2608 if (IS_ATOMIC_WRITTEN_PAGE(fio->page))
2609 return true;
2610 if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED) &&
2611 f2fs_is_checkpointed_data(sbi, fio->old_blkaddr)))
2612 return true;
2613 }
2614 return false;
2615 }
2616
2617 static inline bool need_inplace_update(struct f2fs_io_info *fio)
2618 {
2619 struct inode *inode = fio->page->mapping->host;
2620
2621 if (f2fs_should_update_outplace(inode, fio))
2622 return false;
2623
2624 return f2fs_should_update_inplace(inode, fio);
2625 }
2626
2627 int f2fs_do_write_data_page(struct f2fs_io_info *fio)
2628 {
2629 struct page *page = fio->page;
2630 struct inode *inode = page->mapping->host;
2631 struct dnode_of_data dn;
2632 struct extent_info ei = {0,0,0};
2633 struct node_info ni;
2634 bool ipu_force = false;
2635 int err = 0;
2636
2637 set_new_dnode(&dn, inode, NULL, NULL, 0);
2638 if (need_inplace_update(fio) &&
2639 f2fs_lookup_extent_cache(inode, page->index, &ei)) {
2640 fio->old_blkaddr = ei.blk + page->index - ei.fofs;
2641
2642 if (!f2fs_is_valid_blkaddr(fio->sbi, fio->old_blkaddr,
2643 DATA_GENERIC_ENHANCE))
2644 return -EFSCORRUPTED;
2645
2646 ipu_force = true;
2647 fio->need_lock = LOCK_DONE;
2648 goto got_it;
2649 }
2650
2651 /* Deadlock due to between page->lock and f2fs_lock_op */
2652 if (fio->need_lock == LOCK_REQ && !f2fs_trylock_op(fio->sbi))
2653 return -EAGAIN;
2654
2655 err = f2fs_get_dnode_of_data(&dn, page->index, LOOKUP_NODE);
2656 if (err)
2657 goto out;
2658
2659 fio->old_blkaddr = dn.data_blkaddr;
2660
2661 /* This page is already truncated */
2662 if (fio->old_blkaddr == NULL_ADDR) {
2663 ClearPageUptodate(page);
2664 clear_cold_data(page);
2665 goto out_writepage;
2666 }
2667 got_it:
2668 if (__is_valid_data_blkaddr(fio->old_blkaddr) &&
2669 !f2fs_is_valid_blkaddr(fio->sbi, fio->old_blkaddr,
2670 DATA_GENERIC_ENHANCE)) {
2671 err = -EFSCORRUPTED;
2672 goto out_writepage;
2673 }
2674 /*
2675 * If current allocation needs SSR,
2676 * it had better in-place writes for updated data.
2677 */
2678 if (ipu_force ||
2679 (__is_valid_data_blkaddr(fio->old_blkaddr) &&
2680 need_inplace_update(fio))) {
2681 err = f2fs_encrypt_one_page(fio);
2682 if (err)
2683 goto out_writepage;
2684
2685 set_page_writeback(page);
2686 ClearPageError(page);
2687 f2fs_put_dnode(&dn);
2688 if (fio->need_lock == LOCK_REQ)
2689 f2fs_unlock_op(fio->sbi);
2690 err = f2fs_inplace_write_data(fio);
2691 if (err) {
2692 if (fscrypt_inode_uses_fs_layer_crypto(inode))
2693 fscrypt_finalize_bounce_page(&fio->encrypted_page);
2694 if (PageWriteback(page))
2695 end_page_writeback(page);
2696 } else {
2697 set_inode_flag(inode, FI_UPDATE_WRITE);
2698 }
2699 trace_f2fs_do_write_data_page(fio->page, IPU);
2700 return err;
2701 }
2702
2703 if (fio->need_lock == LOCK_RETRY) {
2704 if (!f2fs_trylock_op(fio->sbi)) {
2705 err = -EAGAIN;
2706 goto out_writepage;
2707 }
2708 fio->need_lock = LOCK_REQ;
2709 }
2710
2711 err = f2fs_get_node_info(fio->sbi, dn.nid, &ni);
2712 if (err)
2713 goto out_writepage;
2714
2715 fio->version = ni.version;
2716
2717 err = f2fs_encrypt_one_page(fio);
2718 if (err)
2719 goto out_writepage;
2720
2721 set_page_writeback(page);
2722 ClearPageError(page);
2723
2724 if (fio->compr_blocks && fio->old_blkaddr == COMPRESS_ADDR)
2725 f2fs_i_compr_blocks_update(inode, fio->compr_blocks - 1, false);
2726
2727 /* LFS mode write path */
2728 f2fs_outplace_write_data(&dn, fio);
2729 trace_f2fs_do_write_data_page(page, OPU);
2730 set_inode_flag(inode, FI_APPEND_WRITE);
2731 if (page->index == 0)
2732 set_inode_flag(inode, FI_FIRST_BLOCK_WRITTEN);
2733 out_writepage:
2734 f2fs_put_dnode(&dn);
2735 out:
2736 if (fio->need_lock == LOCK_REQ)
2737 f2fs_unlock_op(fio->sbi);
2738 return err;
2739 }
2740
2741 int f2fs_write_single_data_page(struct page *page, int *submitted,
2742 struct bio **bio,
2743 sector_t *last_block,
2744 struct writeback_control *wbc,
2745 enum iostat_type io_type,
2746 int compr_blocks,
2747 bool allow_balance)
2748 {
2749 struct inode *inode = page->mapping->host;
2750 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
2751 loff_t i_size = i_size_read(inode);
2752 const pgoff_t end_index = ((unsigned long long)i_size)
2753 >> PAGE_SHIFT;
2754 loff_t psize = (loff_t)(page->index + 1) << PAGE_SHIFT;
2755 unsigned offset = 0;
2756 bool need_balance_fs = false;
2757 int err = 0;
2758 struct f2fs_io_info fio = {
2759 .sbi = sbi,
2760 .ino = inode->i_ino,
2761 .type = DATA,
2762 .op = REQ_OP_WRITE,
2763 .op_flags = wbc_to_write_flags(wbc),
2764 .old_blkaddr = NULL_ADDR,
2765 .page = page,
2766 .encrypted_page = NULL,
2767 .submitted = false,
2768 .compr_blocks = compr_blocks,
2769 .need_lock = LOCK_RETRY,
2770 .io_type = io_type,
2771 .io_wbc = wbc,
2772 .bio = bio,
2773 .last_block = last_block,
2774 };
2775
2776 trace_f2fs_writepage(page, DATA);
2777
2778 /* we should bypass data pages to proceed the kworkder jobs */
2779 if (unlikely(f2fs_cp_error(sbi))) {
2780 mapping_set_error(page->mapping, -EIO);
2781 /*
2782 * don't drop any dirty dentry pages for keeping lastest
2783 * directory structure.
2784 */
2785 if (S_ISDIR(inode->i_mode))
2786 goto redirty_out;
2787 goto out;
2788 }
2789
2790 if (unlikely(is_sbi_flag_set(sbi, SBI_POR_DOING)))
2791 goto redirty_out;
2792
2793 if (page->index < end_index ||
2794 f2fs_verity_in_progress(inode) ||
2795 compr_blocks)
2796 goto write;
2797
2798 /*
2799 * If the offset is out-of-range of file size,
2800 * this page does not have to be written to disk.
2801 */
2802 offset = i_size & (PAGE_SIZE - 1);
2803 if ((page->index >= end_index + 1) || !offset)
2804 goto out;
2805
2806 zero_user_segment(page, offset, PAGE_SIZE);
2807 write:
2808 if (f2fs_is_drop_cache(inode))
2809 goto out;
2810 /* we should not write 0'th page having journal header */
2811 if (f2fs_is_volatile_file(inode) && (!page->index ||
2812 (!wbc->for_reclaim &&
2813 f2fs_available_free_memory(sbi, BASE_CHECK))))
2814 goto redirty_out;
2815
2816 /* Dentry/quota blocks are controlled by checkpoint */
2817 if (S_ISDIR(inode->i_mode) || IS_NOQUOTA(inode)) {
2818 /*
2819 * We need to wait for node_write to avoid block allocation during
2820 * checkpoint. This can only happen to quota writes which can cause
2821 * the below discard race condition.
2822 */
2823 if (IS_NOQUOTA(inode))
2824 down_read(&sbi->node_write);
2825
2826 fio.need_lock = LOCK_DONE;
2827 err = f2fs_do_write_data_page(&fio);
2828
2829 if (IS_NOQUOTA(inode))
2830 up_read(&sbi->node_write);
2831
2832 goto done;
2833 }
2834
2835 if (!wbc->for_reclaim)
2836 need_balance_fs = true;
2837 else if (has_not_enough_free_secs(sbi, 0, 0))
2838 goto redirty_out;
2839 else
2840 set_inode_flag(inode, FI_HOT_DATA);
2841
2842 err = -EAGAIN;
2843 if (f2fs_has_inline_data(inode)) {
2844 err = f2fs_write_inline_data(inode, page);
2845 if (!err)
2846 goto out;
2847 }
2848
2849 if (err == -EAGAIN) {
2850 err = f2fs_do_write_data_page(&fio);
2851 if (err == -EAGAIN) {
2852 fio.need_lock = LOCK_REQ;
2853 err = f2fs_do_write_data_page(&fio);
2854 }
2855 }
2856
2857 if (err) {
2858 file_set_keep_isize(inode);
2859 } else {
2860 spin_lock(&F2FS_I(inode)->i_size_lock);
2861 if (F2FS_I(inode)->last_disk_size < psize)
2862 F2FS_I(inode)->last_disk_size = psize;
2863 spin_unlock(&F2FS_I(inode)->i_size_lock);
2864 }
2865
2866 done:
2867 if (err && err != -ENOENT)
2868 goto redirty_out;
2869
2870 out:
2871 inode_dec_dirty_pages(inode);
2872 if (err) {
2873 ClearPageUptodate(page);
2874 clear_cold_data(page);
2875 }
2876
2877 if (wbc->for_reclaim) {
2878 f2fs_submit_merged_write_cond(sbi, NULL, page, 0, DATA);
2879 clear_inode_flag(inode, FI_HOT_DATA);
2880 f2fs_remove_dirty_inode(inode);
2881 submitted = NULL;
2882 }
2883 unlock_page(page);
2884 if (!S_ISDIR(inode->i_mode) && !IS_NOQUOTA(inode) &&
2885 !F2FS_I(inode)->cp_task && allow_balance)
2886 f2fs_balance_fs(sbi, need_balance_fs);
2887
2888 if (unlikely(f2fs_cp_error(sbi))) {
2889 f2fs_submit_merged_write(sbi, DATA);
2890 f2fs_submit_merged_ipu_write(sbi, bio, NULL);
2891 submitted = NULL;
2892 }
2893
2894 if (submitted)
2895 *submitted = fio.submitted ? 1 : 0;
2896
2897 return 0;
2898
2899 redirty_out:
2900 redirty_page_for_writepage(wbc, page);
2901 /*
2902 * pageout() in MM traslates EAGAIN, so calls handle_write_error()
2903 * -> mapping_set_error() -> set_bit(AS_EIO, ...).
2904 * file_write_and_wait_range() will see EIO error, which is critical
2905 * to return value of fsync() followed by atomic_write failure to user.
2906 */
2907 if (!err || wbc->for_reclaim)
2908 return AOP_WRITEPAGE_ACTIVATE;
2909 unlock_page(page);
2910 return err;
2911 }
2912
2913 static int f2fs_write_data_page(struct page *page,
2914 struct writeback_control *wbc)
2915 {
2916 #ifdef CONFIG_F2FS_FS_COMPRESSION
2917 struct inode *inode = page->mapping->host;
2918
2919 if (unlikely(f2fs_cp_error(F2FS_I_SB(inode))))
2920 goto out;
2921
2922 if (f2fs_compressed_file(inode)) {
2923 if (f2fs_is_compressed_cluster(inode, page->index)) {
2924 redirty_page_for_writepage(wbc, page);
2925 return AOP_WRITEPAGE_ACTIVATE;
2926 }
2927 }
2928 out:
2929 #endif
2930
2931 return f2fs_write_single_data_page(page, NULL, NULL, NULL,
2932 wbc, FS_DATA_IO, 0, true);
2933 }
2934
2935 /*
2936 * This function was copied from write_cche_pages from mm/page-writeback.c.
2937 * The major change is making write step of cold data page separately from
2938 * warm/hot data page.
2939 */
2940 static int f2fs_write_cache_pages(struct address_space *mapping,
2941 struct writeback_control *wbc,
2942 enum iostat_type io_type)
2943 {
2944 int ret = 0;
2945 int done = 0, retry = 0;
2946 struct pagevec pvec;
2947 struct f2fs_sb_info *sbi = F2FS_M_SB(mapping);
2948 struct bio *bio = NULL;
2949 sector_t last_block;
2950 #ifdef CONFIG_F2FS_FS_COMPRESSION
2951 struct inode *inode = mapping->host;
2952 struct compress_ctx cc = {
2953 .inode = inode,
2954 .log_cluster_size = F2FS_I(inode)->i_log_cluster_size,
2955 .cluster_size = F2FS_I(inode)->i_cluster_size,
2956 .cluster_idx = NULL_CLUSTER,
2957 .rpages = NULL,
2958 .nr_rpages = 0,
2959 .cpages = NULL,
2960 .rbuf = NULL,
2961 .cbuf = NULL,
2962 .rlen = PAGE_SIZE * F2FS_I(inode)->i_cluster_size,
2963 .private = NULL,
2964 };
2965 #endif
2966 int nr_pages;
2967 pgoff_t index;
2968 pgoff_t end; /* Inclusive */
2969 pgoff_t done_index;
2970 int range_whole = 0;
2971 xa_mark_t tag;
2972 int nwritten = 0;
2973 int submitted = 0;
2974 int i;
2975
2976 pagevec_init(&pvec);
2977
2978 if (get_dirty_pages(mapping->host) <=
2979 SM_I(F2FS_M_SB(mapping))->min_hot_blocks)
2980 set_inode_flag(mapping->host, FI_HOT_DATA);
2981 else
2982 clear_inode_flag(mapping->host, FI_HOT_DATA);
2983
2984 if (wbc->range_cyclic) {
2985 index = mapping->writeback_index; /* prev offset */
2986 end = -1;
2987 } else {
2988 index = wbc->range_start >> PAGE_SHIFT;
2989 end = wbc->range_end >> PAGE_SHIFT;
2990 if (wbc->range_start == 0 && wbc->range_end == LLONG_MAX)
2991 range_whole = 1;
2992 }
2993 if (wbc->sync_mode == WB_SYNC_ALL || wbc->tagged_writepages)
2994 tag = PAGECACHE_TAG_TOWRITE;
2995 else
2996 tag = PAGECACHE_TAG_DIRTY;
2997 retry:
2998 retry = 0;
2999 if (wbc->sync_mode == WB_SYNC_ALL || wbc->tagged_writepages)
3000 tag_pages_for_writeback(mapping, index, end);
3001 done_index = index;
3002 while (!done && !retry && (index <= end)) {
3003 nr_pages = pagevec_lookup_range_tag(&pvec, mapping, &index, end,
3004 tag);
3005 if (nr_pages == 0)
3006 break;
3007
3008 for (i = 0; i < nr_pages; i++) {
3009 struct page *page = pvec.pages[i];
3010 bool need_readd;
3011 readd:
3012 need_readd = false;
3013 #ifdef CONFIG_F2FS_FS_COMPRESSION
3014 if (f2fs_compressed_file(inode)) {
3015 ret = f2fs_init_compress_ctx(&cc);
3016 if (ret) {
3017 done = 1;
3018 break;
3019 }
3020
3021 if (!f2fs_cluster_can_merge_page(&cc,
3022 page->index)) {
3023 ret = f2fs_write_multi_pages(&cc,
3024 &submitted, wbc, io_type);
3025 if (!ret)
3026 need_readd = true;
3027 goto result;
3028 }
3029
3030 if (unlikely(f2fs_cp_error(sbi)))
3031 goto lock_page;
3032
3033 if (f2fs_cluster_is_empty(&cc)) {
3034 void *fsdata = NULL;
3035 struct page *pagep;
3036 int ret2;
3037
3038 ret2 = f2fs_prepare_compress_overwrite(
3039 inode, &pagep,
3040 page->index, &fsdata);
3041 if (ret2 < 0) {
3042 ret = ret2;
3043 done = 1;
3044 break;
3045 } else if (ret2 &&
3046 !f2fs_compress_write_end(inode,
3047 fsdata, page->index,
3048 1)) {
3049 retry = 1;
3050 break;
3051 }
3052 } else {
3053 goto lock_page;
3054 }
3055 }
3056 #endif
3057 /* give a priority to WB_SYNC threads */
3058 if (atomic_read(&sbi->wb_sync_req[DATA]) &&
3059 wbc->sync_mode == WB_SYNC_NONE) {
3060 done = 1;
3061 break;
3062 }
3063 #ifdef CONFIG_F2FS_FS_COMPRESSION
3064 lock_page:
3065 #endif
3066 done_index = page->index;
3067 retry_write:
3068 lock_page(page);
3069
3070 if (unlikely(page->mapping != mapping)) {
3071 continue_unlock:
3072 unlock_page(page);
3073 continue;
3074 }
3075
3076 if (!PageDirty(page)) {
3077 /* someone wrote it for us */
3078 goto continue_unlock;
3079 }
3080
3081 if (PageWriteback(page)) {
3082 if (wbc->sync_mode != WB_SYNC_NONE)
3083 f2fs_wait_on_page_writeback(page,
3084 DATA, true, true);
3085 else
3086 goto continue_unlock;
3087 }
3088
3089 if (!clear_page_dirty_for_io(page))
3090 goto continue_unlock;
3091
3092 #ifdef CONFIG_F2FS_FS_COMPRESSION
3093 if (f2fs_compressed_file(inode)) {
3094 get_page(page);
3095 f2fs_compress_ctx_add_page(&cc, page);
3096 continue;
3097 }
3098 #endif
3099 ret = f2fs_write_single_data_page(page, &submitted,
3100 &bio, &last_block, wbc, io_type,
3101 0, true);
3102 if (ret == AOP_WRITEPAGE_ACTIVATE)
3103 unlock_page(page);
3104 #ifdef CONFIG_F2FS_FS_COMPRESSION
3105 result:
3106 #endif
3107 nwritten += submitted;
3108 wbc->nr_to_write -= submitted;
3109
3110 if (unlikely(ret)) {
3111 /*
3112 * keep nr_to_write, since vfs uses this to
3113 * get # of written pages.
3114 */
3115 if (ret == AOP_WRITEPAGE_ACTIVATE) {
3116 ret = 0;
3117 goto next;
3118 } else if (ret == -EAGAIN) {
3119 ret = 0;
3120 if (wbc->sync_mode == WB_SYNC_ALL) {
3121 cond_resched();
3122 congestion_wait(BLK_RW_ASYNC,
3123 DEFAULT_IO_TIMEOUT);
3124 goto retry_write;
3125 }
3126 goto next;
3127 }
3128 done_index = page->index + 1;
3129 done = 1;
3130 break;
3131 }
3132
3133 if (wbc->nr_to_write <= 0 &&
3134 wbc->sync_mode == WB_SYNC_NONE) {
3135 done = 1;
3136 break;
3137 }
3138 next:
3139 if (need_readd)
3140 goto readd;
3141 }
3142 pagevec_release(&pvec);
3143 cond_resched();
3144 }
3145 #ifdef CONFIG_F2FS_FS_COMPRESSION
3146 /* flush remained pages in compress cluster */
3147 if (f2fs_compressed_file(inode) && !f2fs_cluster_is_empty(&cc)) {
3148 ret = f2fs_write_multi_pages(&cc, &submitted, wbc, io_type);
3149 nwritten += submitted;
3150 wbc->nr_to_write -= submitted;
3151 if (ret) {
3152 done = 1;
3153 retry = 0;
3154 }
3155 }
3156 if (f2fs_compressed_file(inode))
3157 f2fs_destroy_compress_ctx(&cc);
3158 #endif
3159 if (retry) {
3160 index = 0;
3161 end = -1;
3162 goto retry;
3163 }
3164 if (wbc->range_cyclic && !done)
3165 done_index = 0;
3166 if (wbc->range_cyclic || (range_whole && wbc->nr_to_write > 0))
3167 mapping->writeback_index = done_index;
3168
3169 if (nwritten)
3170 f2fs_submit_merged_write_cond(F2FS_M_SB(mapping), mapping->host,
3171 NULL, 0, DATA);
3172 /* submit cached bio of IPU write */
3173 if (bio)
3174 f2fs_submit_merged_ipu_write(sbi, &bio, NULL);
3175
3176 return ret;
3177 }
3178
3179 static inline bool __should_serialize_io(struct inode *inode,
3180 struct writeback_control *wbc)
3181 {
3182 /* to avoid deadlock in path of data flush */
3183 if (F2FS_I(inode)->cp_task)
3184 return false;
3185
3186 if (!S_ISREG(inode->i_mode))
3187 return false;
3188 if (IS_NOQUOTA(inode))
3189 return false;
3190
3191 if (f2fs_need_compress_data(inode))
3192 return true;
3193 if (wbc->sync_mode != WB_SYNC_ALL)
3194 return true;
3195 if (get_dirty_pages(inode) >= SM_I(F2FS_I_SB(inode))->min_seq_blocks)
3196 return true;
3197 return false;
3198 }
3199
3200 static int __f2fs_write_data_pages(struct address_space *mapping,
3201 struct writeback_control *wbc,
3202 enum iostat_type io_type)
3203 {
3204 struct inode *inode = mapping->host;
3205 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
3206 struct blk_plug plug;
3207 int ret;
3208 bool locked = false;
3209
3210 /* deal with chardevs and other special file */
3211 if (!mapping->a_ops->writepage)
3212 return 0;
3213
3214 /* skip writing if there is no dirty page in this inode */
3215 if (!get_dirty_pages(inode) && wbc->sync_mode == WB_SYNC_NONE)
3216 return 0;
3217
3218 /* during POR, we don't need to trigger writepage at all. */
3219 if (unlikely(is_sbi_flag_set(sbi, SBI_POR_DOING)))
3220 goto skip_write;
3221
3222 if ((S_ISDIR(inode->i_mode) || IS_NOQUOTA(inode)) &&
3223 wbc->sync_mode == WB_SYNC_NONE &&
3224 get_dirty_pages(inode) < nr_pages_to_skip(sbi, DATA) &&
3225 f2fs_available_free_memory(sbi, DIRTY_DENTS))
3226 goto skip_write;
3227
3228 /* skip writing during file defragment */
3229 if (is_inode_flag_set(inode, FI_DO_DEFRAG))
3230 goto skip_write;
3231
3232 trace_f2fs_writepages(mapping->host, wbc, DATA);
3233
3234 /* to avoid spliting IOs due to mixed WB_SYNC_ALL and WB_SYNC_NONE */
3235 if (wbc->sync_mode == WB_SYNC_ALL)
3236 atomic_inc(&sbi->wb_sync_req[DATA]);
3237 else if (atomic_read(&sbi->wb_sync_req[DATA]))
3238 goto skip_write;
3239
3240 if (__should_serialize_io(inode, wbc)) {
3241 mutex_lock(&sbi->writepages);
3242 locked = true;
3243 }
3244
3245 blk_start_plug(&plug);
3246 ret = f2fs_write_cache_pages(mapping, wbc, io_type);
3247 blk_finish_plug(&plug);
3248
3249 if (locked)
3250 mutex_unlock(&sbi->writepages);
3251
3252 if (wbc->sync_mode == WB_SYNC_ALL)
3253 atomic_dec(&sbi->wb_sync_req[DATA]);
3254 /*
3255 * if some pages were truncated, we cannot guarantee its mapping->host
3256 * to detect pending bios.
3257 */
3258
3259 f2fs_remove_dirty_inode(inode);
3260 return ret;
3261
3262 skip_write:
3263 wbc->pages_skipped += get_dirty_pages(inode);
3264 trace_f2fs_writepages(mapping->host, wbc, DATA);
3265 return 0;
3266 }
3267
3268 static int f2fs_write_data_pages(struct address_space *mapping,
3269 struct writeback_control *wbc)
3270 {
3271 struct inode *inode = mapping->host;
3272
3273 return __f2fs_write_data_pages(mapping, wbc,
3274 F2FS_I(inode)->cp_task == current ?
3275 FS_CP_DATA_IO : FS_DATA_IO);
3276 }
3277
3278 static void f2fs_write_failed(struct address_space *mapping, loff_t to)
3279 {
3280 struct inode *inode = mapping->host;
3281 loff_t i_size = i_size_read(inode);
3282
3283 if (IS_NOQUOTA(inode))
3284 return;
3285
3286 /* In the fs-verity case, f2fs_end_enable_verity() does the truncate */
3287 if (to > i_size && !f2fs_verity_in_progress(inode)) {
3288 down_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]);
3289 down_write(&F2FS_I(inode)->i_mmap_sem);
3290
3291 truncate_pagecache(inode, i_size);
3292 f2fs_truncate_blocks(inode, i_size, true);
3293
3294 up_write(&F2FS_I(inode)->i_mmap_sem);
3295 up_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]);
3296 }
3297 }
3298
3299 static int prepare_write_begin(struct f2fs_sb_info *sbi,
3300 struct page *page, loff_t pos, unsigned len,
3301 block_t *blk_addr, bool *node_changed)
3302 {
3303 struct inode *inode = page->mapping->host;
3304 pgoff_t index = page->index;
3305 struct dnode_of_data dn;
3306 struct page *ipage;
3307 bool locked = false;
3308 struct extent_info ei = {0,0,0};
3309 int err = 0;
3310 int flag;
3311
3312 /*
3313 * we already allocated all the blocks, so we don't need to get
3314 * the block addresses when there is no need to fill the page.
3315 */
3316 if (!f2fs_has_inline_data(inode) && len == PAGE_SIZE &&
3317 !is_inode_flag_set(inode, FI_NO_PREALLOC) &&
3318 !f2fs_verity_in_progress(inode))
3319 return 0;
3320
3321 /* f2fs_lock_op avoids race between write CP and convert_inline_page */
3322 if (f2fs_has_inline_data(inode) && pos + len > MAX_INLINE_DATA(inode))
3323 flag = F2FS_GET_BLOCK_DEFAULT;
3324 else
3325 flag = F2FS_GET_BLOCK_PRE_AIO;
3326
3327 if (f2fs_has_inline_data(inode) ||
3328 (pos & PAGE_MASK) >= i_size_read(inode)) {
3329 f2fs_do_map_lock(sbi, flag, true);
3330 locked = true;
3331 }
3332
3333 restart:
3334 /* check inline_data */
3335 ipage = f2fs_get_node_page(sbi, inode->i_ino);
3336 if (IS_ERR(ipage)) {
3337 err = PTR_ERR(ipage);
3338 goto unlock_out;
3339 }
3340
3341 set_new_dnode(&dn, inode, ipage, ipage, 0);
3342
3343 if (f2fs_has_inline_data(inode)) {
3344 if (pos + len <= MAX_INLINE_DATA(inode)) {
3345 f2fs_do_read_inline_data(page, ipage);
3346 set_inode_flag(inode, FI_DATA_EXIST);
3347 if (inode->i_nlink)
3348 set_inline_node(ipage);
3349 } else {
3350 err = f2fs_convert_inline_page(&dn, page);
3351 if (err)
3352 goto out;
3353 if (dn.data_blkaddr == NULL_ADDR)
3354 err = f2fs_get_block(&dn, index);
3355 }
3356 } else if (locked) {
3357 err = f2fs_get_block(&dn, index);
3358 } else {
3359 if (f2fs_lookup_extent_cache(inode, index, &ei)) {
3360 dn.data_blkaddr = ei.blk + index - ei.fofs;
3361 } else {
3362 /* hole case */
3363 err = f2fs_get_dnode_of_data(&dn, index, LOOKUP_NODE);
3364 if (err || dn.data_blkaddr == NULL_ADDR) {
3365 f2fs_put_dnode(&dn);
3366 f2fs_do_map_lock(sbi, F2FS_GET_BLOCK_PRE_AIO,
3367 true);
3368 WARN_ON(flag != F2FS_GET_BLOCK_PRE_AIO);
3369 locked = true;
3370 goto restart;
3371 }
3372 }
3373 }
3374
3375 /* convert_inline_page can make node_changed */
3376 *blk_addr = dn.data_blkaddr;
3377 *node_changed = dn.node_changed;
3378 out:
3379 f2fs_put_dnode(&dn);
3380 unlock_out:
3381 if (locked)
3382 f2fs_do_map_lock(sbi, flag, false);
3383 return err;
3384 }
3385
3386 static int f2fs_write_begin(struct file *file, struct address_space *mapping,
3387 loff_t pos, unsigned len, unsigned flags,
3388 struct page **pagep, void **fsdata)
3389 {
3390 struct inode *inode = mapping->host;
3391 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
3392 struct page *page = NULL;
3393 pgoff_t index = ((unsigned long long) pos) >> PAGE_SHIFT;
3394 bool need_balance = false, drop_atomic = false;
3395 block_t blkaddr = NULL_ADDR;
3396 int err = 0;
3397
3398 trace_f2fs_write_begin(inode, pos, len, flags);
3399
3400 if (!f2fs_is_checkpoint_ready(sbi)) {
3401 err = -ENOSPC;
3402 goto fail;
3403 }
3404
3405 if ((f2fs_is_atomic_file(inode) &&
3406 !f2fs_available_free_memory(sbi, INMEM_PAGES)) ||
3407 is_inode_flag_set(inode, FI_ATOMIC_REVOKE_REQUEST)) {
3408 err = -ENOMEM;
3409 drop_atomic = true;
3410 goto fail;
3411 }
3412
3413 /*
3414 * We should check this at this moment to avoid deadlock on inode page
3415 * and #0 page. The locking rule for inline_data conversion should be:
3416 * lock_page(page #0) -> lock_page(inode_page)
3417 */
3418 if (index != 0) {
3419 err = f2fs_convert_inline_inode(inode);
3420 if (err)
3421 goto fail;
3422 }
3423
3424 #ifdef CONFIG_F2FS_FS_COMPRESSION
3425 if (f2fs_compressed_file(inode)) {
3426 int ret;
3427
3428 *fsdata = NULL;
3429
3430 ret = f2fs_prepare_compress_overwrite(inode, pagep,
3431 index, fsdata);
3432 if (ret < 0) {
3433 err = ret;
3434 goto fail;
3435 } else if (ret) {
3436 return 0;
3437 }
3438 }
3439 #endif
3440
3441 repeat:
3442 /*
3443 * Do not use grab_cache_page_write_begin() to avoid deadlock due to
3444 * wait_for_stable_page. Will wait that below with our IO control.
3445 */
3446 page = f2fs_pagecache_get_page(mapping, index,
3447 FGP_LOCK | FGP_WRITE | FGP_CREAT, GFP_NOFS);
3448 if (!page) {
3449 err = -ENOMEM;
3450 goto fail;
3451 }
3452
3453 /* TODO: cluster can be compressed due to race with .writepage */
3454
3455 *pagep = page;
3456
3457 err = prepare_write_begin(sbi, page, pos, len,
3458 &blkaddr, &need_balance);
3459 if (err)
3460 goto fail;
3461
3462 if (need_balance && !IS_NOQUOTA(inode) &&
3463 has_not_enough_free_secs(sbi, 0, 0)) {
3464 unlock_page(page);
3465 f2fs_balance_fs(sbi, true);
3466 lock_page(page);
3467 if (page->mapping != mapping) {
3468 /* The page got truncated from under us */
3469 f2fs_put_page(page, 1);
3470 goto repeat;
3471 }
3472 }
3473
3474 f2fs_wait_on_page_writeback(page, DATA, false, true);
3475
3476 if (len == PAGE_SIZE || PageUptodate(page))
3477 return 0;
3478
3479 if (!(pos & (PAGE_SIZE - 1)) && (pos + len) >= i_size_read(inode) &&
3480 !f2fs_verity_in_progress(inode)) {
3481 zero_user_segment(page, len, PAGE_SIZE);
3482 return 0;
3483 }
3484
3485 if (blkaddr == NEW_ADDR) {
3486 zero_user_segment(page, 0, PAGE_SIZE);
3487 SetPageUptodate(page);
3488 } else {
3489 if (!f2fs_is_valid_blkaddr(sbi, blkaddr,
3490 DATA_GENERIC_ENHANCE_READ)) {
3491 err = -EFSCORRUPTED;
3492 goto fail;
3493 }
3494 err = f2fs_submit_page_read(inode, page, blkaddr, 0, true);
3495 if (err)
3496 goto fail;
3497
3498 lock_page(page);
3499 if (unlikely(page->mapping != mapping)) {
3500 f2fs_put_page(page, 1);
3501 goto repeat;
3502 }
3503 if (unlikely(!PageUptodate(page))) {
3504 err = -EIO;
3505 goto fail;
3506 }
3507 }
3508 return 0;
3509
3510 fail:
3511 f2fs_put_page(page, 1);
3512 f2fs_write_failed(mapping, pos + len);
3513 if (drop_atomic)
3514 f2fs_drop_inmem_pages_all(sbi, false);
3515 return err;
3516 }
3517
3518 static int f2fs_write_end(struct file *file,
3519 struct address_space *mapping,
3520 loff_t pos, unsigned len, unsigned copied,
3521 struct page *page, void *fsdata)
3522 {
3523 struct inode *inode = page->mapping->host;
3524
3525 trace_f2fs_write_end(inode, pos, len, copied);
3526
3527 /*
3528 * This should be come from len == PAGE_SIZE, and we expect copied
3529 * should be PAGE_SIZE. Otherwise, we treat it with zero copied and
3530 * let generic_perform_write() try to copy data again through copied=0.
3531 */
3532 if (!PageUptodate(page)) {
3533 if (unlikely(copied != len))
3534 copied = 0;
3535 else
3536 SetPageUptodate(page);
3537 }
3538
3539 #ifdef CONFIG_F2FS_FS_COMPRESSION
3540 /* overwrite compressed file */
3541 if (f2fs_compressed_file(inode) && fsdata) {
3542 f2fs_compress_write_end(inode, fsdata, page->index, copied);
3543 f2fs_update_time(F2FS_I_SB(inode), REQ_TIME);
3544
3545 if (pos + copied > i_size_read(inode) &&
3546 !f2fs_verity_in_progress(inode))
3547 f2fs_i_size_write(inode, pos + copied);
3548 return copied;
3549 }
3550 #endif
3551
3552 if (!copied)
3553 goto unlock_out;
3554
3555 set_page_dirty(page);
3556
3557 if (pos + copied > i_size_read(inode) &&
3558 !f2fs_verity_in_progress(inode))
3559 f2fs_i_size_write(inode, pos + copied);
3560 unlock_out:
3561 f2fs_put_page(page, 1);
3562 f2fs_update_time(F2FS_I_SB(inode), REQ_TIME);
3563 return copied;
3564 }
3565
3566 static int check_direct_IO(struct inode *inode, struct iov_iter *iter,
3567 loff_t offset)
3568 {
3569 unsigned i_blkbits = READ_ONCE(inode->i_blkbits);
3570 unsigned blkbits = i_blkbits;
3571 unsigned blocksize_mask = (1 << blkbits) - 1;
3572 unsigned long align = offset | iov_iter_alignment(iter);
3573 struct block_device *bdev = inode->i_sb->s_bdev;
3574
3575 if (iov_iter_rw(iter) == READ && offset >= i_size_read(inode))
3576 return 1;
3577
3578 if (align & blocksize_mask) {
3579 if (bdev)
3580 blkbits = blksize_bits(bdev_logical_block_size(bdev));
3581 blocksize_mask = (1 << blkbits) - 1;
3582 if (align & blocksize_mask)
3583 return -EINVAL;
3584 return 1;
3585 }
3586 return 0;
3587 }
3588
3589 static void f2fs_dio_end_io(struct bio *bio)
3590 {
3591 struct f2fs_private_dio *dio = bio->bi_private;
3592
3593 dec_page_count(F2FS_I_SB(dio->inode),
3594 dio->write ? F2FS_DIO_WRITE : F2FS_DIO_READ);
3595
3596 bio->bi_private = dio->orig_private;
3597 bio->bi_end_io = dio->orig_end_io;
3598
3599 kfree(dio);
3600
3601 bio_endio(bio);
3602 }
3603
3604 static void f2fs_dio_submit_bio(struct bio *bio, struct inode *inode,
3605 loff_t file_offset)
3606 {
3607 struct f2fs_private_dio *dio;
3608 bool write = (bio_op(bio) == REQ_OP_WRITE);
3609
3610 dio = f2fs_kzalloc(F2FS_I_SB(inode),
3611 sizeof(struct f2fs_private_dio), GFP_NOFS);
3612 if (!dio)
3613 goto out;
3614
3615 dio->inode = inode;
3616 dio->orig_end_io = bio->bi_end_io;
3617 dio->orig_private = bio->bi_private;
3618 dio->write = write;
3619
3620 bio->bi_end_io = f2fs_dio_end_io;
3621 bio->bi_private = dio;
3622
3623 inc_page_count(F2FS_I_SB(inode),
3624 write ? F2FS_DIO_WRITE : F2FS_DIO_READ);
3625
3626 submit_bio(bio);
3627 return;
3628 out:
3629 bio->bi_status = BLK_STS_IOERR;
3630 bio_endio(bio);
3631 }
3632
3633 static ssize_t f2fs_direct_IO(struct kiocb *iocb, struct iov_iter *iter)
3634 {
3635 struct address_space *mapping = iocb->ki_filp->f_mapping;
3636 struct inode *inode = mapping->host;
3637 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
3638 struct f2fs_inode_info *fi = F2FS_I(inode);
3639 size_t count = iov_iter_count(iter);
3640 loff_t offset = iocb->ki_pos;
3641 int rw = iov_iter_rw(iter);
3642 int err;
3643 enum rw_hint hint = iocb->ki_hint;
3644 int whint_mode = F2FS_OPTION(sbi).whint_mode;
3645 bool do_opu;
3646
3647 err = check_direct_IO(inode, iter, offset);
3648 if (err)
3649 return err < 0 ? err : 0;
3650
3651 if (f2fs_force_buffered_io(inode, iocb, iter))
3652 return 0;
3653
3654 do_opu = allow_outplace_dio(inode, iocb, iter);
3655
3656 trace_f2fs_direct_IO_enter(inode, offset, count, rw);
3657
3658 if (rw == WRITE && whint_mode == WHINT_MODE_OFF)
3659 iocb->ki_hint = WRITE_LIFE_NOT_SET;
3660
3661 if (iocb->ki_flags & IOCB_NOWAIT) {
3662 if (!down_read_trylock(&fi->i_gc_rwsem[rw])) {
3663 iocb->ki_hint = hint;
3664 err = -EAGAIN;
3665 goto out;
3666 }
3667 if (do_opu && !down_read_trylock(&fi->i_gc_rwsem[READ])) {
3668 up_read(&fi->i_gc_rwsem[rw]);
3669 iocb->ki_hint = hint;
3670 err = -EAGAIN;
3671 goto out;
3672 }
3673 } else {
3674 down_read(&fi->i_gc_rwsem[rw]);
3675 if (do_opu)
3676 down_read(&fi->i_gc_rwsem[READ]);
3677 }
3678
3679 err = __blockdev_direct_IO(iocb, inode, inode->i_sb->s_bdev,
3680 iter, rw == WRITE ? get_data_block_dio_write :
3681 get_data_block_dio, NULL, f2fs_dio_submit_bio,
3682 rw == WRITE ? DIO_LOCKING | DIO_SKIP_HOLES :
3683 DIO_SKIP_HOLES);
3684
3685 if (do_opu)
3686 up_read(&fi->i_gc_rwsem[READ]);
3687
3688 up_read(&fi->i_gc_rwsem[rw]);
3689
3690 if (rw == WRITE) {
3691 if (whint_mode == WHINT_MODE_OFF)
3692 iocb->ki_hint = hint;
3693 if (err > 0) {
3694 f2fs_update_iostat(F2FS_I_SB(inode), APP_DIRECT_IO,
3695 err);
3696 if (!do_opu)
3697 set_inode_flag(inode, FI_UPDATE_WRITE);
3698 } else if (err == -EIOCBQUEUED) {
3699 f2fs_update_iostat(F2FS_I_SB(inode), APP_DIRECT_IO,
3700 count - iov_iter_count(iter));
3701 } else if (err < 0) {
3702 f2fs_write_failed(mapping, offset + count);
3703 }
3704 } else {
3705 if (err > 0)
3706 f2fs_update_iostat(sbi, APP_DIRECT_READ_IO, err);
3707 else if (err == -EIOCBQUEUED)
3708 f2fs_update_iostat(F2FS_I_SB(inode), APP_DIRECT_READ_IO,
3709 count - iov_iter_count(iter));
3710 }
3711
3712 out:
3713 trace_f2fs_direct_IO_exit(inode, offset, count, rw, err);
3714
3715 return err;
3716 }
3717
3718 void f2fs_invalidate_page(struct page *page, unsigned int offset,
3719 unsigned int length)
3720 {
3721 struct inode *inode = page->mapping->host;
3722 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
3723
3724 if (inode->i_ino >= F2FS_ROOT_INO(sbi) &&
3725 (offset % PAGE_SIZE || length != PAGE_SIZE))
3726 return;
3727
3728 if (PageDirty(page)) {
3729 if (inode->i_ino == F2FS_META_INO(sbi)) {
3730 dec_page_count(sbi, F2FS_DIRTY_META);
3731 } else if (inode->i_ino == F2FS_NODE_INO(sbi)) {
3732 dec_page_count(sbi, F2FS_DIRTY_NODES);
3733 } else {
3734 inode_dec_dirty_pages(inode);
3735 f2fs_remove_dirty_inode(inode);
3736 }
3737 }
3738
3739 clear_cold_data(page);
3740
3741 if (IS_ATOMIC_WRITTEN_PAGE(page))
3742 return f2fs_drop_inmem_page(inode, page);
3743
3744 f2fs_clear_page_private(page);
3745 }
3746
3747 int f2fs_release_page(struct page *page, gfp_t wait)
3748 {
3749 /* If this is dirty page, keep PagePrivate */
3750 if (PageDirty(page))
3751 return 0;
3752
3753 /* This is atomic written page, keep Private */
3754 if (IS_ATOMIC_WRITTEN_PAGE(page))
3755 return 0;
3756
3757 clear_cold_data(page);
3758 f2fs_clear_page_private(page);
3759 return 1;
3760 }
3761
3762 static int f2fs_set_data_page_dirty(struct page *page)
3763 {
3764 struct inode *inode = page_file_mapping(page)->host;
3765
3766 trace_f2fs_set_page_dirty(page, DATA);
3767
3768 if (!PageUptodate(page))
3769 SetPageUptodate(page);
3770 if (PageSwapCache(page))
3771 return __set_page_dirty_nobuffers(page);
3772
3773 if (f2fs_is_atomic_file(inode) && !f2fs_is_commit_atomic_write(inode)) {
3774 if (!IS_ATOMIC_WRITTEN_PAGE(page)) {
3775 f2fs_register_inmem_page(inode, page);
3776 return 1;
3777 }
3778 /*
3779 * Previously, this page has been registered, we just
3780 * return here.
3781 */
3782 return 0;
3783 }
3784
3785 if (!PageDirty(page)) {
3786 __set_page_dirty_nobuffers(page);
3787 f2fs_update_dirty_page(inode, page);
3788 return 1;
3789 }
3790 return 0;
3791 }
3792
3793
3794 static sector_t f2fs_bmap_compress(struct inode *inode, sector_t block)
3795 {
3796 #ifdef CONFIG_F2FS_FS_COMPRESSION
3797 struct dnode_of_data dn;
3798 sector_t start_idx, blknr = 0;
3799 int ret;
3800
3801 start_idx = round_down(block, F2FS_I(inode)->i_cluster_size);
3802
3803 set_new_dnode(&dn, inode, NULL, NULL, 0);
3804 ret = f2fs_get_dnode_of_data(&dn, start_idx, LOOKUP_NODE);
3805 if (ret)
3806 return 0;
3807
3808 if (dn.data_blkaddr != COMPRESS_ADDR) {
3809 dn.ofs_in_node += block - start_idx;
3810 blknr = f2fs_data_blkaddr(&dn);
3811 if (!__is_valid_data_blkaddr(blknr))
3812 blknr = 0;
3813 }
3814
3815 f2fs_put_dnode(&dn);
3816 return blknr;
3817 #else
3818 return 0;
3819 #endif
3820 }
3821
3822
3823 static sector_t f2fs_bmap(struct address_space *mapping, sector_t block)
3824 {
3825 struct inode *inode = mapping->host;
3826 sector_t blknr = 0;
3827
3828 if (f2fs_has_inline_data(inode))
3829 goto out;
3830
3831 /* make sure allocating whole blocks */
3832 if (mapping_tagged(mapping, PAGECACHE_TAG_DIRTY))
3833 filemap_write_and_wait(mapping);
3834
3835 /* Block number less than F2FS MAX BLOCKS */
3836 if (unlikely(block >= F2FS_I_SB(inode)->max_file_blocks))
3837 goto out;
3838
3839 if (f2fs_compressed_file(inode)) {
3840 blknr = f2fs_bmap_compress(inode, block);
3841 } else {
3842 struct f2fs_map_blocks map;
3843
3844 memset(&map, 0, sizeof(map));
3845 map.m_lblk = block;
3846 map.m_len = 1;
3847 map.m_next_pgofs = NULL;
3848 map.m_seg_type = NO_CHECK_TYPE;
3849
3850 if (!f2fs_map_blocks(inode, &map, 0, F2FS_GET_BLOCK_BMAP))
3851 blknr = map.m_pblk;
3852 }
3853 out:
3854 trace_f2fs_bmap(inode, block, blknr);
3855 return blknr;
3856 }
3857
3858 #ifdef CONFIG_MIGRATION
3859 #include <linux/migrate.h>
3860
3861 int f2fs_migrate_page(struct address_space *mapping,
3862 struct page *newpage, struct page *page, enum migrate_mode mode)
3863 {
3864 int rc, extra_count;
3865 struct f2fs_inode_info *fi = F2FS_I(mapping->host);
3866 bool atomic_written = IS_ATOMIC_WRITTEN_PAGE(page);
3867
3868 BUG_ON(PageWriteback(page));
3869
3870 /* migrating an atomic written page is safe with the inmem_lock hold */
3871 if (atomic_written) {
3872 if (mode != MIGRATE_SYNC)
3873 return -EBUSY;
3874 if (!mutex_trylock(&fi->inmem_lock))
3875 return -EAGAIN;
3876 }
3877
3878 /* one extra reference was held for atomic_write page */
3879 extra_count = atomic_written ? 1 : 0;
3880 rc = migrate_page_move_mapping(mapping, newpage,
3881 page, extra_count);
3882 if (rc != MIGRATEPAGE_SUCCESS) {
3883 if (atomic_written)
3884 mutex_unlock(&fi->inmem_lock);
3885 return rc;
3886 }
3887
3888 if (atomic_written) {
3889 struct inmem_pages *cur;
3890 list_for_each_entry(cur, &fi->inmem_pages, list)
3891 if (cur->page == page) {
3892 cur->page = newpage;
3893 break;
3894 }
3895 mutex_unlock(&fi->inmem_lock);
3896 put_page(page);
3897 get_page(newpage);
3898 }
3899
3900 if (PagePrivate(page)) {
3901 f2fs_set_page_private(newpage, page_private(page));
3902 f2fs_clear_page_private(page);
3903 }
3904
3905 if (mode != MIGRATE_SYNC_NO_COPY)
3906 migrate_page_copy(newpage, page);
3907 else
3908 migrate_page_states(newpage, page);
3909
3910 return MIGRATEPAGE_SUCCESS;
3911 }
3912 #endif
3913
3914 #ifdef CONFIG_SWAP
3915 static int check_swap_activate_fast(struct swap_info_struct *sis,
3916 struct file *swap_file, sector_t *span)
3917 {
3918 struct address_space *mapping = swap_file->f_mapping;
3919 struct inode *inode = mapping->host;
3920 sector_t cur_lblock;
3921 sector_t last_lblock;
3922 sector_t pblock;
3923 sector_t lowest_pblock = -1;
3924 sector_t highest_pblock = 0;
3925 int nr_extents = 0;
3926 unsigned long nr_pblocks;
3927 u64 len;
3928 int ret;
3929
3930 /*
3931 * Map all the blocks into the extent list. This code doesn't try
3932 * to be very smart.
3933 */
3934 cur_lblock = 0;
3935 last_lblock = bytes_to_blks(inode, i_size_read(inode));
3936 len = i_size_read(inode);
3937
3938 while (cur_lblock <= last_lblock && cur_lblock < sis->max) {
3939 struct f2fs_map_blocks map;
3940 pgoff_t next_pgofs;
3941
3942 cond_resched();
3943
3944 memset(&map, 0, sizeof(map));
3945 map.m_lblk = cur_lblock;
3946 map.m_len = bytes_to_blks(inode, len) - cur_lblock;
3947 map.m_next_pgofs = &next_pgofs;
3948 map.m_seg_type = NO_CHECK_TYPE;
3949
3950 ret = f2fs_map_blocks(inode, &map, 0, F2FS_GET_BLOCK_FIEMAP);
3951 if (ret)
3952 goto err_out;
3953
3954 /* hole */
3955 if (!(map.m_flags & F2FS_MAP_FLAGS))
3956 goto err_out;
3957
3958 pblock = map.m_pblk;
3959 nr_pblocks = map.m_len;
3960
3961 if (cur_lblock + nr_pblocks >= sis->max)
3962 nr_pblocks = sis->max - cur_lblock;
3963
3964 if (cur_lblock) { /* exclude the header page */
3965 if (pblock < lowest_pblock)
3966 lowest_pblock = pblock;
3967 if (pblock + nr_pblocks - 1 > highest_pblock)
3968 highest_pblock = pblock + nr_pblocks - 1;
3969 }
3970
3971 /*
3972 * We found a PAGE_SIZE-length, PAGE_SIZE-aligned run of blocks
3973 */
3974 ret = add_swap_extent(sis, cur_lblock, nr_pblocks, pblock);
3975 if (ret < 0)
3976 goto out;
3977 nr_extents += ret;
3978 cur_lblock += nr_pblocks;
3979 }
3980 ret = nr_extents;
3981 *span = 1 + highest_pblock - lowest_pblock;
3982 if (cur_lblock == 0)
3983 cur_lblock = 1; /* force Empty message */
3984 sis->max = cur_lblock;
3985 sis->pages = cur_lblock - 1;
3986 sis->highest_bit = cur_lblock - 1;
3987 out:
3988 return ret;
3989 err_out:
3990 pr_err("swapon: swapfile has holes\n");
3991 return -EINVAL;
3992 }
3993
3994 /* Copied from generic_swapfile_activate() to check any holes */
3995 static int check_swap_activate(struct swap_info_struct *sis,
3996 struct file *swap_file, sector_t *span)
3997 {
3998 struct address_space *mapping = swap_file->f_mapping;
3999 struct inode *inode = mapping->host;
4000 unsigned blocks_per_page;
4001 unsigned long page_no;
4002 sector_t probe_block;
4003 sector_t last_block;
4004 sector_t lowest_block = -1;
4005 sector_t highest_block = 0;
4006 int nr_extents = 0;
4007 int ret;
4008
4009 if (PAGE_SIZE == F2FS_BLKSIZE)
4010 return check_swap_activate_fast(sis, swap_file, span);
4011
4012 blocks_per_page = bytes_to_blks(inode, PAGE_SIZE);
4013
4014 /*
4015 * Map all the blocks into the extent list. This code doesn't try
4016 * to be very smart.
4017 */
4018 probe_block = 0;
4019 page_no = 0;
4020 last_block = bytes_to_blks(inode, i_size_read(inode));
4021 while ((probe_block + blocks_per_page) <= last_block &&
4022 page_no < sis->max) {
4023 unsigned block_in_page;
4024 sector_t first_block;
4025 sector_t block = 0;
4026 int err = 0;
4027
4028 cond_resched();
4029
4030 block = probe_block;
4031 err = bmap(inode, &block);
4032 if (err || !block)
4033 goto bad_bmap;
4034 first_block = block;
4035
4036 /*
4037 * It must be PAGE_SIZE aligned on-disk
4038 */
4039 if (first_block & (blocks_per_page - 1)) {
4040 probe_block++;
4041 goto reprobe;
4042 }
4043
4044 for (block_in_page = 1; block_in_page < blocks_per_page;
4045 block_in_page++) {
4046
4047 block = probe_block + block_in_page;
4048 err = bmap(inode, &block);
4049
4050 if (err || !block)
4051 goto bad_bmap;
4052
4053 if (block != first_block + block_in_page) {
4054 /* Discontiguity */
4055 probe_block++;
4056 goto reprobe;
4057 }
4058 }
4059
4060 first_block >>= (PAGE_SHIFT - inode->i_blkbits);
4061 if (page_no) { /* exclude the header page */
4062 if (first_block < lowest_block)
4063 lowest_block = first_block;
4064 if (first_block > highest_block)
4065 highest_block = first_block;
4066 }
4067
4068 /*
4069 * We found a PAGE_SIZE-length, PAGE_SIZE-aligned run of blocks
4070 */
4071 ret = add_swap_extent(sis, page_no, 1, first_block);
4072 if (ret < 0)
4073 goto out;
4074 nr_extents += ret;
4075 page_no++;
4076 probe_block += blocks_per_page;
4077 reprobe:
4078 continue;
4079 }
4080 ret = nr_extents;
4081 *span = 1 + highest_block - lowest_block;
4082 if (page_no == 0)
4083 page_no = 1; /* force Empty message */
4084 sis->max = page_no;
4085 sis->pages = page_no - 1;
4086 sis->highest_bit = page_no - 1;
4087 out:
4088 return ret;
4089 bad_bmap:
4090 pr_err("swapon: swapfile has holes\n");
4091 return -EINVAL;
4092 }
4093
4094 static int f2fs_swap_activate(struct swap_info_struct *sis, struct file *file,
4095 sector_t *span)
4096 {
4097 struct inode *inode = file_inode(file);
4098 int ret;
4099
4100 if (!S_ISREG(inode->i_mode))
4101 return -EINVAL;
4102
4103 if (f2fs_readonly(F2FS_I_SB(inode)->sb))
4104 return -EROFS;
4105
4106 ret = f2fs_convert_inline_inode(inode);
4107 if (ret)
4108 return ret;
4109
4110 if (!f2fs_disable_compressed_file(inode))
4111 return -EINVAL;
4112
4113 ret = check_swap_activate(sis, file, span);
4114 if (ret < 0)
4115 return ret;
4116
4117 set_inode_flag(inode, FI_PIN_FILE);
4118 f2fs_precache_extents(inode);
4119 f2fs_update_time(F2FS_I_SB(inode), REQ_TIME);
4120 return ret;
4121 }
4122
4123 static void f2fs_swap_deactivate(struct file *file)
4124 {
4125 struct inode *inode = file_inode(file);
4126
4127 clear_inode_flag(inode, FI_PIN_FILE);
4128 }
4129 #else
4130 static int f2fs_swap_activate(struct swap_info_struct *sis, struct file *file,
4131 sector_t *span)
4132 {
4133 return -EOPNOTSUPP;
4134 }
4135
4136 static void f2fs_swap_deactivate(struct file *file)
4137 {
4138 }
4139 #endif
4140
4141 const struct address_space_operations f2fs_dblock_aops = {
4142 .readpage = f2fs_read_data_page,
4143 .readahead = f2fs_readahead,
4144 .writepage = f2fs_write_data_page,
4145 .writepages = f2fs_write_data_pages,
4146 .write_begin = f2fs_write_begin,
4147 .write_end = f2fs_write_end,
4148 .set_page_dirty = f2fs_set_data_page_dirty,
4149 .invalidatepage = f2fs_invalidate_page,
4150 .releasepage = f2fs_release_page,
4151 .direct_IO = f2fs_direct_IO,
4152 .bmap = f2fs_bmap,
4153 .swap_activate = f2fs_swap_activate,
4154 .swap_deactivate = f2fs_swap_deactivate,
4155 #ifdef CONFIG_MIGRATION
4156 .migratepage = f2fs_migrate_page,
4157 #endif
4158 };
4159
4160 void f2fs_clear_page_cache_dirty_tag(struct page *page)
4161 {
4162 struct address_space *mapping = page_mapping(page);
4163 unsigned long flags;
4164
4165 xa_lock_irqsave(&mapping->i_pages, flags);
4166 __xa_clear_mark(&mapping->i_pages, page_index(page),
4167 PAGECACHE_TAG_DIRTY);
4168 xa_unlock_irqrestore(&mapping->i_pages, flags);
4169 }
4170
4171 int __init f2fs_init_post_read_processing(void)
4172 {
4173 bio_post_read_ctx_cache =
4174 kmem_cache_create("f2fs_bio_post_read_ctx",
4175 sizeof(struct bio_post_read_ctx), 0, 0, NULL);
4176 if (!bio_post_read_ctx_cache)
4177 goto fail;
4178 bio_post_read_ctx_pool =
4179 mempool_create_slab_pool(NUM_PREALLOC_POST_READ_CTXS,
4180 bio_post_read_ctx_cache);
4181 if (!bio_post_read_ctx_pool)
4182 goto fail_free_cache;
4183 return 0;
4184
4185 fail_free_cache:
4186 kmem_cache_destroy(bio_post_read_ctx_cache);
4187 fail:
4188 return -ENOMEM;
4189 }
4190
4191 void f2fs_destroy_post_read_processing(void)
4192 {
4193 mempool_destroy(bio_post_read_ctx_pool);
4194 kmem_cache_destroy(bio_post_read_ctx_cache);
4195 }
4196
4197 int f2fs_init_post_read_wq(struct f2fs_sb_info *sbi)
4198 {
4199 if (!f2fs_sb_has_encrypt(sbi) &&
4200 !f2fs_sb_has_verity(sbi) &&
4201 !f2fs_sb_has_compression(sbi))
4202 return 0;
4203
4204 sbi->post_read_wq = alloc_workqueue("f2fs_post_read_wq",
4205 WQ_UNBOUND | WQ_HIGHPRI,
4206 num_online_cpus());
4207 if (!sbi->post_read_wq)
4208 return -ENOMEM;
4209 return 0;
4210 }
4211
4212 void f2fs_destroy_post_read_wq(struct f2fs_sb_info *sbi)
4213 {
4214 if (sbi->post_read_wq)
4215 destroy_workqueue(sbi->post_read_wq);
4216 }
4217
4218 int __init f2fs_init_bio_entry_cache(void)
4219 {
4220 bio_entry_slab = f2fs_kmem_cache_create("f2fs_bio_entry_slab",
4221 sizeof(struct bio_entry));
4222 if (!bio_entry_slab)
4223 return -ENOMEM;
4224 return 0;
4225 }
4226
4227 void f2fs_destroy_bio_entry_cache(void)
4228 {
4229 kmem_cache_destroy(bio_entry_slab);
4230 }
Ubuntu Hirsute Linux kernel mirror