]> git.proxmox.com Git - mirror_ubuntu-bionic-kernel.git/blame - fs/f2fs/data.c
projects / mirror_ubuntu-bionic-kernel.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
f2fs: Fix bool initialization/comparison
[mirror_ubuntu-bionic-kernel.git] / fs / f2fs / data.c
CommitLineData
0a8165d7 1/*
eb47b800
JK		 2 * fs/f2fs/data.c
3 *
4 * Copyright (c) 2012 Samsung Electronics Co., Ltd.
5 * http://www.samsung.com/
6 *
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License version 2 as
9 * published by the Free Software Foundation.
10 */
11#include <linux/fs.h>
12#include <linux/f2fs_fs.h>
13#include <linux/buffer_head.h>
14#include <linux/mpage.h>
15#include <linux/writeback.h>
16#include <linux/backing-dev.h>
8f46dcae 17#include <linux/pagevec.h>
eb47b800
JK		 18#include <linux/blkdev.h>
19#include <linux/bio.h>
690e4a3e 20#include <linux/prefetch.h>
e2e40f2c 21#include <linux/uio.h>
fe76b796
JK		 22#include <linux/mm.h>
23#include <linux/memcontrol.h>
f1e88660 24#include <linux/cleancache.h>
174cd4b1 25#include <linux/sched/signal.h>
eb47b800
JK		 26
27#include "f2fs.h"
28#include "node.h"
29#include "segment.h"
db9f7c1a 30#include "trace.h"
848753aa 31#include <trace/events/f2fs.h>
eb47b800 32
36951b38
CY		 33static bool __is_cp_guaranteed(struct page *page)
34{
35 struct address_space *mapping = page->mapping;
36 struct inode *inode;
37 struct f2fs_sb_info *sbi;
38
39 if (!mapping)
40 return false;
41
42 inode = mapping->host;
43 sbi = F2FS_I_SB(inode);
44
45 if (inode->i_ino == F2FS_META_INO(sbi) ||
46 inode->i_ino == F2FS_NODE_INO(sbi) ||
47 S_ISDIR(inode->i_mode) ||
48 is_cold_data(page))
49 return true;
50 return false;
51}
52
4246a0b6 53static void f2fs_read_end_io(struct bio *bio)
93dfe2ac 54{
f568849e
LT		 55 struct bio_vec *bvec;
56 int i;
93dfe2ac 57
8b038c70 58#ifdef CONFIG_F2FS_FAULT_INJECTION
55523519
CY		 59 if (time_to_inject(F2FS_P_SB(bio->bi_io_vec->bv_page), FAULT_IO)) {
60 f2fs_show_injection_info(FAULT_IO);
4e4cbee9 61 bio->bi_status = BLK_STS_IOERR;
55523519 62 }
8b038c70
CY		 63#endif
64
4375a336 65 if (f2fs_bio_encrypted(bio)) {
4e4cbee9 66 if (bio->bi_status) {
0b81d077 67 fscrypt_release_ctx(bio->bi_private);
4375a336 68 } else {
0b81d077 69 fscrypt_decrypt_bio_pages(bio->bi_private, bio);
4375a336
JK		 70 return;
71 }
72 }
73
12377024
CY		 74 bio_for_each_segment_all(bvec, bio, i) {
75 struct page *page = bvec->bv_page;
f1e88660 76
4e4cbee9 77 if (!bio->bi_status) {
237c0790
JK		 78 if (!PageUptodate(page))
79 SetPageUptodate(page);
f1e88660
JK		 80 } else {
81 ClearPageUptodate(page);
82 SetPageError(page);
83 }
84 unlock_page(page);
85 }
f1e88660
JK		 86 bio_put(bio);
87}
88
4246a0b6 89static void f2fs_write_end_io(struct bio *bio)
93dfe2ac 90{
1b1f559f 91 struct f2fs_sb_info *sbi = bio->bi_private;
f568849e
LT		 92 struct bio_vec *bvec;
93 int i;
93dfe2ac 94
f568849e 95 bio_for_each_segment_all(bvec, bio, i) {
93dfe2ac 96 struct page *page = bvec->bv_page;
36951b38 97 enum count_type type = WB_DATA_TYPE(page);
93dfe2ac 98
0a595eba
JK		 99 if (IS_DUMMY_WRITTEN_PAGE(page)) {
100 set_page_private(page, (unsigned long)NULL);
101 ClearPagePrivate(page);
102 unlock_page(page);
103 mempool_free(page, sbi->write_io_dummy);
104
4e4cbee9 105 if (unlikely(bio->bi_status))
0a595eba
JK		 106 f2fs_stop_checkpoint(sbi, true);
107 continue;
108 }
109
0b81d077 110 fscrypt_pullback_bio_page(&page, true);
4375a336 111
4e4cbee9 112 if (unlikely(bio->bi_status)) {
5114a97a 113 mapping_set_error(page->mapping, -EIO);
38f91ca8 114 f2fs_stop_checkpoint(sbi, true);
93dfe2ac 115 }
36951b38
CY		 116 dec_page_count(sbi, type);
117 clear_cold_data(page);
93dfe2ac 118 end_page_writeback(page);
f568849e 119 }
36951b38 120 if (!get_pages(sbi, F2FS_WB_CP_DATA) &&
f5730184 121 wq_has_sleeper(&sbi->cp_wait))
93dfe2ac
JK		 122 wake_up(&sbi->cp_wait);
123
124 bio_put(bio);
125}
126
3c62be17
JK		 127/*
128 * Return true, if pre_bio's bdev is same as its target device.
129 */
130struct block_device *f2fs_target_device(struct f2fs_sb_info *sbi,
131 block_t blk_addr, struct bio *bio)
132{
133 struct block_device *bdev = sbi->sb->s_bdev;
134 int i;
135
136 for (i = 0; i < sbi->s_ndevs; i++) {
137 if (FDEV(i).start_blk <= blk_addr &&
138 FDEV(i).end_blk >= blk_addr) {
139 blk_addr -= FDEV(i).start_blk;
140 bdev = FDEV(i).bdev;
141 break;
142 }
143 }
144 if (bio) {
74d46992 145 bio_set_dev(bio, bdev);
3c62be17
JK		 146 bio->bi_iter.bi_sector = SECTOR_FROM_BLOCK(blk_addr);
147 }
148 return bdev;
149}
150
151int f2fs_target_device_index(struct f2fs_sb_info *sbi, block_t blkaddr)
152{
153 int i;
154
155 for (i = 0; i < sbi->s_ndevs; i++)
156 if (FDEV(i).start_blk <= blkaddr && FDEV(i).end_blk >= blkaddr)
157 return i;
158 return 0;
159}
160
161static bool __same_bdev(struct f2fs_sb_info *sbi,
162 block_t blk_addr, struct bio *bio)
163{
74d46992
CH		 164 struct block_device *b = f2fs_target_device(sbi, blk_addr, NULL);
165 return bio->bi_disk == b->bd_disk && bio->bi_partno == b->bd_partno;
3c62be17
JK		 166}
167
940a6d34
GZ		 168/*
169 * Low-level block read/write IO operations.
170 */
171static struct bio *__bio_alloc(struct f2fs_sb_info *sbi, block_t blk_addr,
172 int npages, bool is_read)
173{
174 struct bio *bio;
175
740432f8 176 bio = f2fs_bio_alloc(npages);
940a6d34 177
3c62be17 178 f2fs_target_device(sbi, blk_addr, bio);
940a6d34 179 bio->bi_end_io = is_read ? f2fs_read_end_io : f2fs_write_end_io;
12377024 180 bio->bi_private = is_read ? NULL : sbi;
940a6d34
GZ		 181
182 return bio;
183}
184
4fc29c1a
LT		 185static inline void __submit_bio(struct f2fs_sb_info *sbi,
186 struct bio *bio, enum page_type type)
f5730184 187{
4fc29c1a 188 if (!is_read_io(bio_op(bio))) {
0a595eba
JK		 189 unsigned int start;
190
0bfd7a09 191 if (f2fs_sb_mounted_blkzoned(sbi->sb) &&
52763a4b 192 current->plug && (type == DATA || type == NODE))
19a5f5e2 193 blk_finish_plug(current->plug);
0a595eba
JK		 194
195 if (type != DATA && type != NODE)
196 goto submit_io;
197
198 start = bio->bi_iter.bi_size >> F2FS_BLKSIZE_BITS;
199 start %= F2FS_IO_SIZE(sbi);
200
201 if (start == 0)
202 goto submit_io;
203
204 /* fill dummy pages */
205 for (; start < F2FS_IO_SIZE(sbi); start++) {
206 struct page *page =
207 mempool_alloc(sbi->write_io_dummy,
208 GFP_NOIO | __GFP_ZERO | __GFP_NOFAIL);
209 f2fs_bug_on(sbi, !page);
210
211 SetPagePrivate(page);
212 set_page_private(page, (unsigned long)DUMMY_WRITTEN_PAGE);
213 lock_page(page);
214 if (bio_add_page(bio, page, PAGE_SIZE, 0) < PAGE_SIZE)
215 f2fs_bug_on(sbi, 1);
216 }
217 /*
218 * In the NODE case, we lose next block address chain. So, we
219 * need to do checkpoint in f2fs_sync_file.
220 */
221 if (type == NODE)
222 set_sbi_flag(sbi, SBI_NEED_CP);
19a5f5e2 223 }
0a595eba 224submit_io:
554b5125
JK		 225 if (is_read_io(bio_op(bio)))
226 trace_f2fs_submit_read_bio(sbi->sb, type, bio);
227 else
228 trace_f2fs_submit_write_bio(sbi->sb, type, bio);
4e49ea4a 229 submit_bio(bio);
f5730184
JK		 230}
231
458e6197 232static void __submit_merged_bio(struct f2fs_bio_info *io)
93dfe2ac 233{
458e6197 234 struct f2fs_io_info *fio = &io->fio;
93dfe2ac
JK		 235
236 if (!io->bio)
237 return;
238
554b5125
JK		 239 bio_set_op_attrs(io->bio, fio->op, fio->op_flags);
240
04d328de 241 if (is_read_io(fio->op))
554b5125 242 trace_f2fs_prepare_read_bio(io->sbi->sb, fio->type, io->bio);
6a8f8ca5 243 else
554b5125 244 trace_f2fs_prepare_write_bio(io->sbi->sb, fio->type, io->bio);
04d328de 245
4fc29c1a 246 __submit_bio(io->sbi, io->bio, fio->type);
93dfe2ac
JK		 247 io->bio = NULL;
248}
249
942fd319
JK		 250static bool __has_merged_page(struct f2fs_bio_info *io,
251 struct inode *inode, nid_t ino, pgoff_t idx)
0fd785eb 252{
0fd785eb
CY		 253 struct bio_vec *bvec;
254 struct page *target;
255 int i;
256
0c3a5797 257 if (!io->bio)
0fd785eb 258 return false;
0c3a5797 259
942fd319 260 if (!inode && !ino)
0c3a5797 261 return true;
0fd785eb
CY		 262
263 bio_for_each_segment_all(bvec, io->bio, i) {
264
0b81d077 265 if (bvec->bv_page->mapping)
0fd785eb 266 target = bvec->bv_page;
0b81d077
JK		 267 else
268 target = fscrypt_control_page(bvec->bv_page);
0fd785eb 269
942fd319
JK		 270 if (idx != target->index)
271 continue;
272
0c3a5797
CY		 273 if (inode && inode == target->mapping->host)
274 return true;
0c3a5797 275 if (ino && ino == ino_of_node(target))
0fd785eb 276 return true;
0fd785eb
CY		 277 }
278
0fd785eb
CY		 279 return false;
280}
281
0c3a5797 282static bool has_merged_page(struct f2fs_sb_info *sbi, struct inode *inode,
942fd319 283 nid_t ino, pgoff_t idx, enum page_type type)
0c3a5797
CY		 284{
285 enum page_type btype = PAGE_TYPE_OF_BIO(type);
a912b54d
JK		 286 enum temp_type temp;
287 struct f2fs_bio_info *io;
288 bool ret = false;
0c3a5797 289
a912b54d
JK		 290 for (temp = HOT; temp < NR_TEMP_TYPE; temp++) {
291 io = sbi->write_io[btype] + temp;
292
293 down_read(&io->io_rwsem);
294 ret = __has_merged_page(io, inode, ino, idx);
295 up_read(&io->io_rwsem);
0c3a5797 296
a912b54d
JK		 297 /* TODO: use HOT temp only for meta pages now. */
298 if (ret || btype == META)
299 break;
300 }
0c3a5797
CY		 301 return ret;
302}
303
b9109b0e 304static void __f2fs_submit_merged_write(struct f2fs_sb_info *sbi,
a912b54d 305 enum page_type type, enum temp_type temp)
93dfe2ac
JK		 306{
307 enum page_type btype = PAGE_TYPE_OF_BIO(type);
a912b54d 308 struct f2fs_bio_info *io = sbi->write_io[btype] + temp;
93dfe2ac 309
df0f8dc0 310 down_write(&io->io_rwsem);
458e6197
JK		 311
312 /* change META to META_FLUSH in the checkpoint procedure */
313 if (type >= META_FLUSH) {
314 io->fio.type = META_FLUSH;
04d328de 315 io->fio.op = REQ_OP_WRITE;
3adc5fcb 316 io->fio.op_flags = REQ_META | REQ_PRIO | REQ_SYNC;
70fd7614 317 if (!test_opt(sbi, NOBARRIER))
7f54f51f 318 io->fio.op_flags |= REQ_PREFLUSH | REQ_FUA;
458e6197
JK		 319 }
320 __submit_merged_bio(io);
df0f8dc0 321 up_write(&io->io_rwsem);
93dfe2ac
JK		 322}
323
a912b54d
JK		 324static void __submit_merged_write_cond(struct f2fs_sb_info *sbi,
325 struct inode *inode, nid_t ino, pgoff_t idx,
326 enum page_type type, bool force)
0c3a5797 327{
a912b54d
JK		 328 enum temp_type temp;
329
330 if (!force && !has_merged_page(sbi, inode, ino, idx, type))
331 return;
332
333 for (temp = HOT; temp < NR_TEMP_TYPE; temp++) {
334
335 __f2fs_submit_merged_write(sbi, type, temp);
336
337 /* TODO: use HOT temp only for meta pages now. */
338 if (type >= META)
339 break;
340 }
0c3a5797
CY		 341}
342
b9109b0e 343void f2fs_submit_merged_write(struct f2fs_sb_info *sbi, enum page_type type)
0c3a5797 344{
a912b54d 345 __submit_merged_write_cond(sbi, NULL, 0, 0, type, true);
0c3a5797
CY		 346}
347
b9109b0e 348void f2fs_submit_merged_write_cond(struct f2fs_sb_info *sbi,
942fd319 349 struct inode *inode, nid_t ino, pgoff_t idx,
b9109b0e 350 enum page_type type)
0c3a5797 351{
a912b54d 352 __submit_merged_write_cond(sbi, inode, ino, idx, type, false);
0c3a5797
CY		 353}
354
b9109b0e 355void f2fs_flush_merged_writes(struct f2fs_sb_info *sbi)
406657dd 356{
b9109b0e
JK		 357 f2fs_submit_merged_write(sbi, DATA);
358 f2fs_submit_merged_write(sbi, NODE);
359 f2fs_submit_merged_write(sbi, META);
406657dd
CY		 360}
361
93dfe2ac
JK		 362/*
363 * Fill the locked page with data located in the block address.
771a9a71 364 * A caller needs to unlock the page on failure.
93dfe2ac 365 */
05ca3632 366int f2fs_submit_page_bio(struct f2fs_io_info *fio)
93dfe2ac 367{
93dfe2ac 368 struct bio *bio;
0b81d077
JK		 369 struct page *page = fio->encrypted_page ?
370 fio->encrypted_page : fio->page;
93dfe2ac 371
2ace38e0 372 trace_f2fs_submit_page_bio(page, fio);
05ca3632 373 f2fs_trace_ios(fio, 0);
93dfe2ac
JK		 374
375 /* Allocate a new bio */
04d328de 376 bio = __bio_alloc(fio->sbi, fio->new_blkaddr, 1, is_read_io(fio->op));
93dfe2ac 377
09cbfeaf 378 if (bio_add_page(bio, page, PAGE_SIZE, 0) < PAGE_SIZE) {
93dfe2ac 379 bio_put(bio);
93dfe2ac
JK		 380 return -EFAULT;
381 }
04d328de 382 bio_set_op_attrs(bio, fio->op, fio->op_flags);
93dfe2ac 383
4fc29c1a 384 __submit_bio(fio->sbi, bio, fio->type);
d1b3e72d
JK		 385
386 if (!is_read_io(fio->op))
387 inc_page_count(fio->sbi, WB_DATA_TYPE(fio->page));
93dfe2ac
JK		 388 return 0;
389}
390
b9109b0e 391int f2fs_submit_page_write(struct f2fs_io_info *fio)
93dfe2ac 392{
05ca3632 393 struct f2fs_sb_info *sbi = fio->sbi;
458e6197 394 enum page_type btype = PAGE_TYPE_OF_BIO(fio->type);
a912b54d 395 struct f2fs_bio_info *io = sbi->write_io[btype] + fio->temp;
4375a336 396 struct page *bio_page;
0a595eba 397 int err = 0;
93dfe2ac 398
b9109b0e 399 f2fs_bug_on(sbi, is_read_io(fio->op));
93dfe2ac 400
fb830fc5
CY		 401 down_write(&io->io_rwsem);
402next:
403 if (fio->in_list) {
404 spin_lock(&io->io_lock);
405 if (list_empty(&io->io_list)) {
406 spin_unlock(&io->io_lock);
407 goto out_fail;
408 }
409 fio = list_first_entry(&io->io_list,
410 struct f2fs_io_info, list);
411 list_del(&fio->list);
412 spin_unlock(&io->io_lock);
413 }
93dfe2ac 414
7a9d7548
CY		 415 if (fio->old_blkaddr != NEW_ADDR)
416 verify_block_addr(sbi, fio->old_blkaddr);
417 verify_block_addr(sbi, fio->new_blkaddr);
93dfe2ac 418
36951b38
CY		 419 bio_page = fio->encrypted_page ? fio->encrypted_page : fio->page;
420
ebf7c522
TM		 421 /* set submitted = true as a return value */
422 fio->submitted = true;
d68f735b 423
b9109b0e 424 inc_page_count(sbi, WB_DATA_TYPE(bio_page));
93dfe2ac 425
7a9d7548 426 if (io->bio && (io->last_block_in_bio != fio->new_blkaddr - 1 ||
3c62be17
JK		 427 (io->fio.op != fio->op || io->fio.op_flags != fio->op_flags) ||
428 !__same_bdev(sbi, fio->new_blkaddr, io->bio)))
458e6197 429 __submit_merged_bio(io);
93dfe2ac
JK		 430alloc_new:
431 if (io->bio == NULL) {
0a595eba
JK		 432 if ((fio->type == DATA || fio->type == NODE) &&
433 fio->new_blkaddr & F2FS_IO_SIZE_MASK(sbi)) {
434 err = -EAGAIN;
b9109b0e 435 dec_page_count(sbi, WB_DATA_TYPE(bio_page));
0a595eba
JK		 436 goto out_fail;
437 }
7a9d7548 438 io->bio = __bio_alloc(sbi, fio->new_blkaddr,
b9109b0e 439 BIO_MAX_PAGES, false);
458e6197 440 io->fio = *fio;
93dfe2ac
JK		 441 }
442
a912b54d 443 if (bio_add_page(io->bio, bio_page, PAGE_SIZE, 0) < PAGE_SIZE) {
458e6197 444 __submit_merged_bio(io);
93dfe2ac
JK		 445 goto alloc_new;
446 }
447
7a9d7548 448 io->last_block_in_bio = fio->new_blkaddr;
05ca3632 449 f2fs_trace_ios(fio, 0);
fb830fc5
CY		 450
451 trace_f2fs_submit_page_write(fio->page, fio);
452
453 if (fio->in_list)
454 goto next;
0a595eba 455out_fail:
df0f8dc0 456 up_write(&io->io_rwsem);
0a595eba 457 return err;
93dfe2ac
JK		 458}
459
13ba41e3
JK		 460static struct bio *f2fs_grab_read_bio(struct inode *inode, block_t blkaddr,
461 unsigned nr_pages)
462{
463 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
464 struct fscrypt_ctx *ctx = NULL;
465 struct bio *bio;
466
467 if (f2fs_encrypted_file(inode)) {
468 ctx = fscrypt_get_ctx(inode, GFP_NOFS);
469 if (IS_ERR(ctx))
470 return ERR_CAST(ctx);
471
472 /* wait the page to be moved by cleaning */
473 f2fs_wait_on_block_writeback(sbi, blkaddr);
474 }
475
476 bio = bio_alloc(GFP_KERNEL, min_t(int, nr_pages, BIO_MAX_PAGES));
477 if (!bio) {
478 if (ctx)
479 fscrypt_release_ctx(ctx);
480 return ERR_PTR(-ENOMEM);
481 }
482 f2fs_target_device(sbi, blkaddr, bio);
483 bio->bi_end_io = f2fs_read_end_io;
484 bio->bi_private = ctx;
485 bio_set_op_attrs(bio, REQ_OP_READ, 0);
486
487 return bio;
488}
489
490/* This can handle encryption stuffs */
491static int f2fs_submit_page_read(struct inode *inode, struct page *page,
492 block_t blkaddr)
493{
494 struct bio *bio = f2fs_grab_read_bio(inode, blkaddr, 1);
495
496 if (IS_ERR(bio))
497 return PTR_ERR(bio);
498
499 if (bio_add_page(bio, page, PAGE_SIZE, 0) < PAGE_SIZE) {
500 bio_put(bio);
501 return -EFAULT;
502 }
503 __submit_bio(F2FS_I_SB(inode), bio, DATA);
504 return 0;
505}
506
46008c6d
CY		 507static void __set_data_blkaddr(struct dnode_of_data *dn)
508{
509 struct f2fs_node *rn = F2FS_NODE(dn->node_page);
510 __le32 *addr_array;
7a2af766
CY		 511 int base = 0;
512
513 if (IS_INODE(dn->node_page) && f2fs_has_extra_attr(dn->inode))
514 base = get_extra_isize(dn->inode);
46008c6d
CY		 515
516 /* Get physical address of data block */
517 addr_array = blkaddr_in_node(rn);
7a2af766 518 addr_array[base + dn->ofs_in_node] = cpu_to_le32(dn->data_blkaddr);
46008c6d
CY		 519}
520
0a8165d7 521/*
eb47b800
JK		 522 * Lock ordering for the change of data block address:
523 * ->data_page
524 * ->node_page
525 * update block addresses in the node page
526 */
216a620a 527void set_data_blkaddr(struct dnode_of_data *dn)
eb47b800 528{
46008c6d
CY		 529 f2fs_wait_on_page_writeback(dn->node_page, NODE, true);
530 __set_data_blkaddr(dn);
531 if (set_page_dirty(dn->node_page))
12719ae1 532 dn->node_changed = true;
eb47b800
JK		 533}
534
f28b3434
CY		 535void f2fs_update_data_blkaddr(struct dnode_of_data *dn, block_t blkaddr)
536{
537 dn->data_blkaddr = blkaddr;
538 set_data_blkaddr(dn);
539 f2fs_update_extent_cache(dn);
540}
541
46008c6d
CY		 542/* dn->ofs_in_node will be returned with up-to-date last block pointer */
543int reserve_new_blocks(struct dnode_of_data *dn, blkcnt_t count)
eb47b800 544{
4081363f 545 struct f2fs_sb_info *sbi = F2FS_I_SB(dn->inode);
0abd675e 546 int err;
eb47b800 547
46008c6d
CY		 548 if (!count)
549 return 0;
550
91942321 551 if (unlikely(is_inode_flag_set(dn->inode, FI_NO_ALLOC)))
eb47b800 552 return -EPERM;
0abd675e
CY		 553 if (unlikely((err = inc_valid_block_count(sbi, dn->inode, &count))))
554 return err;
eb47b800 555
46008c6d
CY		 556 trace_f2fs_reserve_new_blocks(dn->inode, dn->nid,
557 dn->ofs_in_node, count);
558
559 f2fs_wait_on_page_writeback(dn->node_page, NODE, true);
560
561 for (; count > 0; dn->ofs_in_node++) {
7a2af766
CY		 562 block_t blkaddr = datablock_addr(dn->inode,
563 dn->node_page, dn->ofs_in_node);
46008c6d
CY		 564 if (blkaddr == NULL_ADDR) {
565 dn->data_blkaddr = NEW_ADDR;
566 __set_data_blkaddr(dn);
567 count--;
568 }
569 }
570
571 if (set_page_dirty(dn->node_page))
572 dn->node_changed = true;
eb47b800
JK		 573 return 0;
574}
575
46008c6d
CY		 576/* Should keep dn->ofs_in_node unchanged */
577int reserve_new_block(struct dnode_of_data *dn)
578{
579 unsigned int ofs_in_node = dn->ofs_in_node;
580 int ret;
581
582 ret = reserve_new_blocks(dn, 1);
583 dn->ofs_in_node = ofs_in_node;
584 return ret;
585}
586
b600965c
HL		 587int f2fs_reserve_block(struct dnode_of_data *dn, pgoff_t index)
588{
589 bool need_put = dn->inode_page ? false : true;
590 int err;
591
592 err = get_dnode_of_data(dn, index, ALLOC_NODE);
593 if (err)
594 return err;
a8865372 595
b600965c
HL		 596 if (dn->data_blkaddr == NULL_ADDR)
597 err = reserve_new_block(dn);
a8865372 598 if (err || need_put)
b600965c
HL		 599 f2fs_put_dnode(dn);
600 return err;
601}
602
759af1c9 603int f2fs_get_block(struct dnode_of_data *dn, pgoff_t index)
eb47b800 604{
e15882b6 605 struct extent_info ei = {0,0,0};
759af1c9 606 struct inode *inode = dn->inode;
028a41e8 607
759af1c9
FL		 608 if (f2fs_lookup_extent_cache(inode, index, &ei)) {
609 dn->data_blkaddr = ei.blk + index - ei.fofs;
610 return 0;
429511cd 611 }
028a41e8 612
759af1c9 613 return f2fs_reserve_block(dn, index);
eb47b800
JK		 614}
615
a56c7c6f 616struct page *get_read_data_page(struct inode *inode, pgoff_t index,
04d328de 617 int op_flags, bool for_write)
eb47b800 618{
eb47b800
JK		 619 struct address_space *mapping = inode->i_mapping;
620 struct dnode_of_data dn;
621 struct page *page;
e15882b6 622 struct extent_info ei = {0,0,0};
eb47b800 623 int err;
4375a336 624
a56c7c6f 625 page = f2fs_grab_cache_page(mapping, index, for_write);
650495de
JK		 626 if (!page)
627 return ERR_PTR(-ENOMEM);
628
cb3bc9ee
CY		 629 if (f2fs_lookup_extent_cache(inode, index, &ei)) {
630 dn.data_blkaddr = ei.blk + index - ei.fofs;
631 goto got_it;
632 }
633
eb47b800 634 set_new_dnode(&dn, inode, NULL, NULL, 0);
266e97a8 635 err = get_dnode_of_data(&dn, index, LOOKUP_NODE);
86531d6b
JK		 636 if (err)
637 goto put_err;
eb47b800
JK		 638 f2fs_put_dnode(&dn);
639
6bacf52f 640 if (unlikely(dn.data_blkaddr == NULL_ADDR)) {
86531d6b
JK		 641 err = -ENOENT;
642 goto put_err;
650495de 643 }
cb3bc9ee 644got_it:
43f3eae1
JK		 645 if (PageUptodate(page)) {
646 unlock_page(page);
eb47b800 647 return page;
43f3eae1 648 }
eb47b800 649
d59ff4df
JK		 650 /*
651 * A new dentry page is allocated but not able to be written, since its
652 * new inode page couldn't be allocated due to -ENOSPC.
653 * In such the case, its blkaddr can be remained as NEW_ADDR.
654 * see, f2fs_add_link -> get_new_data_page -> init_inode_metadata.
655 */
656 if (dn.data_blkaddr == NEW_ADDR) {
09cbfeaf 657 zero_user_segment(page, 0, PAGE_SIZE);
237c0790
JK		 658 if (!PageUptodate(page))
659 SetPageUptodate(page);
43f3eae1 660 unlock_page(page);
d59ff4df
JK		 661 return page;
662 }
eb47b800 663
13ba41e3 664 err = f2fs_submit_page_read(inode, page, dn.data_blkaddr);
393ff91f 665 if (err)
86531d6b 666 goto put_err;
43f3eae1 667 return page;
86531d6b
JK		 668
669put_err:
670 f2fs_put_page(page, 1);
671 return ERR_PTR(err);
43f3eae1
JK		 672}
673
674struct page *find_data_page(struct inode *inode, pgoff_t index)
675{
676 struct address_space *mapping = inode->i_mapping;
677 struct page *page;
678
679 page = find_get_page(mapping, index);
680 if (page && PageUptodate(page))
681 return page;
682 f2fs_put_page(page, 0);
683
70fd7614 684 page = get_read_data_page(inode, index, 0, false);
43f3eae1
JK		 685 if (IS_ERR(page))
686 return page;
687
688 if (PageUptodate(page))
689 return page;
690
691 wait_on_page_locked(page);
692 if (unlikely(!PageUptodate(page))) {
693 f2fs_put_page(page, 0);
694 return ERR_PTR(-EIO);
695 }
696 return page;
697}
698
699/*
700 * If it tries to access a hole, return an error.
701 * Because, the callers, functions in dir.c and GC, should be able to know
702 * whether this page exists or not.
703 */
a56c7c6f
JK		 704struct page *get_lock_data_page(struct inode *inode, pgoff_t index,
705 bool for_write)
43f3eae1
JK		 706{
707 struct address_space *mapping = inode->i_mapping;
708 struct page *page;
709repeat:
70fd7614 710 page = get_read_data_page(inode, index, 0, for_write);
43f3eae1
JK		 711 if (IS_ERR(page))
712 return page;
393ff91f 713
43f3eae1 714 /* wait for read completion */
393ff91f 715 lock_page(page);
6bacf52f 716 if (unlikely(page->mapping != mapping)) {
afcb7ca0
JK		 717 f2fs_put_page(page, 1);
718 goto repeat;
eb47b800 719 }
1563ac75
CY		 720 if (unlikely(!PageUptodate(page))) {
721 f2fs_put_page(page, 1);
722 return ERR_PTR(-EIO);
723 }
eb47b800
JK		 724 return page;
725}
726
0a8165d7 727/*
eb47b800
JK		 728 * Caller ensures that this data page is never allocated.
729 * A new zero-filled data page is allocated in the page cache.
39936837 730 *
4f4124d0
CY		 731 * Also, caller should grab and release a rwsem by calling f2fs_lock_op() and
732 * f2fs_unlock_op().
470f00e9
CY		 733 * Note that, ipage is set only by make_empty_dir, and if any error occur,
734 * ipage should be released by this function.
eb47b800 735 */
64aa7ed9 736struct page *get_new_data_page(struct inode *inode,
a8865372 737 struct page *ipage, pgoff_t index, bool new_i_size)
eb47b800 738{
eb47b800
JK		 739 struct address_space *mapping = inode->i_mapping;
740 struct page *page;
741 struct dnode_of_data dn;
742 int err;
7612118a 743
a56c7c6f 744 page = f2fs_grab_cache_page(mapping, index, true);
470f00e9
CY		 745 if (!page) {
746 /*
747 * before exiting, we should make sure ipage will be released
748 * if any error occur.
749 */
750 f2fs_put_page(ipage, 1);
01f28610 751 return ERR_PTR(-ENOMEM);
470f00e9 752 }
eb47b800 753
a8865372 754 set_new_dnode(&dn, inode, ipage, NULL, 0);
b600965c 755 err = f2fs_reserve_block(&dn, index);
01f28610
JK		 756 if (err) {
757 f2fs_put_page(page, 1);
eb47b800 758 return ERR_PTR(err);
a8865372 759 }
01f28610
JK		 760 if (!ipage)
761 f2fs_put_dnode(&dn);
eb47b800
JK		 762
763 if (PageUptodate(page))
01f28610 764 goto got_it;
eb47b800
JK		 765
766 if (dn.data_blkaddr == NEW_ADDR) {
09cbfeaf 767 zero_user_segment(page, 0, PAGE_SIZE);
237c0790
JK		 768 if (!PageUptodate(page))
769 SetPageUptodate(page);
eb47b800 770 } else {
4375a336 771 f2fs_put_page(page, 1);
a8865372 772
7612118a
JK		 773 /* if ipage exists, blkaddr should be NEW_ADDR */
774 f2fs_bug_on(F2FS_I_SB(inode), ipage);
775 page = get_lock_data_page(inode, index, true);
4375a336 776 if (IS_ERR(page))
7612118a 777 return page;
eb47b800 778 }
01f28610 779got_it:
9edcdabf 780 if (new_i_size && i_size_read(inode) <
ee6d182f 781 ((loff_t)(index + 1) << PAGE_SHIFT))
fc9581c8 782 f2fs_i_size_write(inode, ((loff_t)(index + 1) << PAGE_SHIFT));
eb47b800
JK		 783 return page;
784}
785
bfad7c2d
JK		 786static int __allocate_data_block(struct dnode_of_data *dn)
787{
4081363f 788 struct f2fs_sb_info *sbi = F2FS_I_SB(dn->inode);
bfad7c2d 789 struct f2fs_summary sum;
bfad7c2d 790 struct node_info ni;
976e4c50 791 pgoff_t fofs;
46008c6d 792 blkcnt_t count = 1;
0abd675e 793 int err;
bfad7c2d 794
91942321 795 if (unlikely(is_inode_flag_set(dn->inode, FI_NO_ALLOC)))
bfad7c2d 796 return -EPERM;
df6136ef 797
7a2af766
CY		 798 dn->data_blkaddr = datablock_addr(dn->inode,
799 dn->node_page, dn->ofs_in_node);
df6136ef
CY		 800 if (dn->data_blkaddr == NEW_ADDR)
801 goto alloc;
802
0abd675e
CY		 803 if (unlikely((err = inc_valid_block_count(sbi, dn->inode, &count))))
804 return err;
bfad7c2d 805
df6136ef 806alloc:
bfad7c2d
JK		 807 get_node_info(sbi, dn->nid, &ni);
808 set_summary(&sum, dn->nid, dn->ofs_in_node, ni.version);
809
df6136ef 810 allocate_data_block(sbi, NULL, dn->data_blkaddr, &dn->data_blkaddr,
fb830fc5 811 &sum, CURSEG_WARM_DATA, NULL, false);
216a620a 812 set_data_blkaddr(dn);
bfad7c2d 813
976e4c50 814 /* update i_size */
81ca7350 815 fofs = start_bidx_of_node(ofs_of_node(dn->node_page), dn->inode) +
976e4c50 816 dn->ofs_in_node;
09cbfeaf 817 if (i_size_read(dn->inode) < ((loff_t)(fofs + 1) << PAGE_SHIFT))
fc9581c8 818 f2fs_i_size_write(dn->inode,
09cbfeaf 819 ((loff_t)(fofs + 1) << PAGE_SHIFT));
bfad7c2d
JK		 820 return 0;
821}
822
c040ff9d
JK		 823static inline bool __force_buffered_io(struct inode *inode, int rw)
824{
1958593e 825 return (f2fs_encrypted_file(inode) ||
c040ff9d
JK		 826 (rw == WRITE && test_opt(F2FS_I_SB(inode), LFS)) ||
827 F2FS_I_SB(inode)->s_ndevs);
828}
829
a7de6086 830int f2fs_preallocate_blocks(struct kiocb *iocb, struct iov_iter *from)
59b802e5 831{
b439b103 832 struct inode *inode = file_inode(iocb->ki_filp);
5b8db7fa 833 struct f2fs_map_blocks map;
a7de6086 834 int err = 0;
59b802e5 835
71ad682c
WG		 836 /* convert inline data for Direct I/O*/
837 if (iocb->ki_flags & IOCB_DIRECT) {
838 err = f2fs_convert_inline_inode(inode);
839 if (err)
840 return err;
841 }
842
dc91de78
JK		 843 if (is_inode_flag_set(inode, FI_NO_PREALLOC))
844 return 0;
845
0080c507 846 map.m_lblk = F2FS_BLK_ALIGN(iocb->ki_pos);
dfd02e4d
CY		 847 map.m_len = F2FS_BYTES_TO_BLK(iocb->ki_pos + iov_iter_count(from));
848 if (map.m_len > map.m_lblk)
849 map.m_len -= map.m_lblk;
850 else
851 map.m_len = 0;
852
da85985c 853 map.m_next_pgofs = NULL;
2a340760 854
71ad682c 855 if (iocb->ki_flags & IOCB_DIRECT)
c040ff9d
JK		 856 return f2fs_map_blocks(inode, &map, 1,
857 __force_buffered_io(inode, WRITE) ?
858 F2FS_GET_BLOCK_PRE_AIO :
859 F2FS_GET_BLOCK_PRE_DIO);
f2470371 860 if (iocb->ki_pos + iov_iter_count(from) > MAX_INLINE_DATA(inode)) {
a7de6086
JK		 861 err = f2fs_convert_inline_inode(inode);
862 if (err)
863 return err;
b439b103 864 }
24b84912
JK		 865 if (!f2fs_has_inline_data(inode))
866 return f2fs_map_blocks(inode, &map, 1, F2FS_GET_BLOCK_PRE_AIO);
a7de6086 867 return err;
59b802e5
JK		 868}
869
59c9081b
YH		 870static inline void __do_map_lock(struct f2fs_sb_info *sbi, int flag, bool lock)
871{
872 if (flag == F2FS_GET_BLOCK_PRE_AIO) {
873 if (lock)
874 down_read(&sbi->node_change);
875 else
876 up_read(&sbi->node_change);
877 } else {
878 if (lock)
879 f2fs_lock_op(sbi);
880 else
881 f2fs_unlock_op(sbi);
882 }
883}
884
0a8165d7 885/*
003a3e1d
JK		 886 * f2fs_map_blocks() now supported readahead/bmap/rw direct_IO with
887 * f2fs_map_blocks structure.
4f4124d0
CY		 888 * If original data blocks are allocated, then give them to blockdev.
889 * Otherwise,
890 * a. preallocate requested block addresses
891 * b. do not use extent cache for better performance
892 * c. give the block addresses to blockdev
eb47b800 893 */
d323d005 894int f2fs_map_blocks(struct inode *inode, struct f2fs_map_blocks *map,
e2b4e2bc 895 int create, int flag)
eb47b800 896{
003a3e1d 897 unsigned int maxblocks = map->m_len;
eb47b800 898 struct dnode_of_data dn;
f9811703 899 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
ac6f1999 900 int mode = create ? ALLOC_NODE : LOOKUP_NODE;
46008c6d 901 pgoff_t pgofs, end_offset, end;
bfad7c2d 902 int err = 0, ofs = 1;
46008c6d
CY		 903 unsigned int ofs_in_node, last_ofs_in_node;
904 blkcnt_t prealloc;
e15882b6 905 struct extent_info ei = {0,0,0};
7df3a431 906 block_t blkaddr;
eb47b800 907
dfd02e4d
CY		 908 if (!maxblocks)
909 return 0;
910
003a3e1d
JK		 911 map->m_len = 0;
912 map->m_flags = 0;
913
914 /* it only supports block size == page size */
915 pgofs = (pgoff_t)map->m_lblk;
46008c6d 916 end = pgofs + maxblocks;
eb47b800 917
24b84912 918 if (!create && f2fs_lookup_extent_cache(inode, pgofs, &ei)) {
003a3e1d
JK		 919 map->m_pblk = ei.blk + pgofs - ei.fofs;
920 map->m_len = min((pgoff_t)maxblocks, ei.fofs + ei.len - pgofs);
921 map->m_flags = F2FS_MAP_MAPPED;
bfad7c2d 922 goto out;
a2e7d1bf 923 }
bfad7c2d 924
4fe71e88 925next_dnode:
59b802e5 926 if (create)
59c9081b 927 __do_map_lock(sbi, flag, true);
eb47b800
JK		 928
929 /* When reading holes, we need its node page */
930 set_new_dnode(&dn, inode, NULL, NULL, 0);
bfad7c2d 931 err = get_dnode_of_data(&dn, pgofs, mode);
1ec79083 932 if (err) {
43473f96
CY		 933 if (flag == F2FS_GET_BLOCK_BMAP)
934 map->m_pblk = 0;
da85985c 935 if (err == -ENOENT) {
bfad7c2d 936 err = 0;
da85985c
CY		 937 if (map->m_next_pgofs)
938 *map->m_next_pgofs =
939 get_next_page_offset(&dn, pgofs);
940 }
bfad7c2d 941 goto unlock_out;
848753aa 942 }
973163fc 943
46008c6d 944 prealloc = 0;
230436b3 945 last_ofs_in_node = ofs_in_node = dn.ofs_in_node;
81ca7350 946 end_offset = ADDRS_PER_PAGE(dn.node_page, inode);
4fe71e88
CY		 947
948next_block:
7a2af766 949 blkaddr = datablock_addr(dn.inode, dn.node_page, dn.ofs_in_node);
4fe71e88
CY		 950
951 if (blkaddr == NEW_ADDR || blkaddr == NULL_ADDR) {
973163fc 952 if (create) {
f9811703
CY		 953 if (unlikely(f2fs_cp_error(sbi))) {
954 err = -EIO;
4fe71e88 955 goto sync_out;
f9811703 956 }
24b84912 957 if (flag == F2FS_GET_BLOCK_PRE_AIO) {
46008c6d
CY		 958 if (blkaddr == NULL_ADDR) {
959 prealloc++;
960 last_ofs_in_node = dn.ofs_in_node;
961 }
24b84912
JK		 962 } else {
963 err = __allocate_data_block(&dn);
6f2d8ed6 964 if (!err)
91942321 965 set_inode_flag(inode, FI_APPEND_WRITE);
24b84912 966 }
973163fc 967 if (err)
4fe71e88 968 goto sync_out;
3f2be043 969 map->m_flags |= F2FS_MAP_NEW;
4fe71e88 970 blkaddr = dn.data_blkaddr;
973163fc 971 } else {
43473f96
CY		 972 if (flag == F2FS_GET_BLOCK_BMAP) {
973 map->m_pblk = 0;
974 goto sync_out;
975 }
da85985c
CY		 976 if (flag == F2FS_GET_BLOCK_FIEMAP &&
977 blkaddr == NULL_ADDR) {
978 if (map->m_next_pgofs)
979 *map->m_next_pgofs = pgofs + 1;
980 }
973163fc 981 if (flag != F2FS_GET_BLOCK_FIEMAP ||
43473f96 982 blkaddr != NEW_ADDR)
4fe71e88 983 goto sync_out;
e2b4e2bc 984 }
e2b4e2bc 985 }
eb47b800 986
46008c6d
CY		 987 if (flag == F2FS_GET_BLOCK_PRE_AIO)
988 goto skip;
989
4fe71e88
CY		 990 if (map->m_len == 0) {
991 /* preallocated unwritten block should be mapped for fiemap. */
992 if (blkaddr == NEW_ADDR)
993 map->m_flags |= F2FS_MAP_UNWRITTEN;
994 map->m_flags |= F2FS_MAP_MAPPED;
995
996 map->m_pblk = blkaddr;
997 map->m_len = 1;
998 } else if ((map->m_pblk != NEW_ADDR &&
999 blkaddr == (map->m_pblk + ofs)) ||
b439b103 1000 (map->m_pblk == NEW_ADDR && blkaddr == NEW_ADDR) ||
46008c6d 1001 flag == F2FS_GET_BLOCK_PRE_DIO) {
4fe71e88
CY		 1002 ofs++;
1003 map->m_len++;
1004 } else {
1005 goto sync_out;
1006 }
bfad7c2d 1007
46008c6d 1008skip:
bfad7c2d
JK		 1009 dn.ofs_in_node++;
1010 pgofs++;
1011
46008c6d
CY		 1012 /* preallocate blocks in batch for one dnode page */
1013 if (flag == F2FS_GET_BLOCK_PRE_AIO &&
1014 (pgofs == end || dn.ofs_in_node == end_offset)) {
7df3a431 1015
46008c6d
CY		 1016 dn.ofs_in_node = ofs_in_node;
1017 err = reserve_new_blocks(&dn, prealloc);
1018 if (err)
1019 goto sync_out;
bfad7c2d 1020
46008c6d
CY		 1021 map->m_len += dn.ofs_in_node - ofs_in_node;
1022 if (prealloc && dn.ofs_in_node != last_ofs_in_node + 1) {
1023 err = -ENOSPC;
1024 goto sync_out;
3104af35 1025 }
46008c6d
CY		 1026 dn.ofs_in_node = end_offset;
1027 }
1028
1029 if (pgofs >= end)
1030 goto sync_out;
1031 else if (dn.ofs_in_node < end_offset)
1032 goto next_block;
1033
46008c6d
CY		 1034 f2fs_put_dnode(&dn);
1035
1036 if (create) {
59c9081b 1037 __do_map_lock(sbi, flag, false);
6f2d8ed6 1038 f2fs_balance_fs(sbi, dn.node_changed);
eb47b800 1039 }
46008c6d 1040 goto next_dnode;
7df3a431 1041
bfad7c2d 1042sync_out:
eb47b800 1043 f2fs_put_dnode(&dn);
bfad7c2d 1044unlock_out:
2a340760 1045 if (create) {
59c9081b 1046 __do_map_lock(sbi, flag, false);
6f2d8ed6 1047 f2fs_balance_fs(sbi, dn.node_changed);
2a340760 1048 }
bfad7c2d 1049out:
003a3e1d 1050 trace_f2fs_map_blocks(inode, map, err);
bfad7c2d 1051 return err;
eb47b800
JK		 1052}
1053
003a3e1d 1054static int __get_data_block(struct inode *inode, sector_t iblock,
da85985c
CY		 1055 struct buffer_head *bh, int create, int flag,
1056 pgoff_t *next_pgofs)
003a3e1d
JK		 1057{
1058 struct f2fs_map_blocks map;
a7de6086 1059 int err;
003a3e1d
JK		 1060
1061 map.m_lblk = iblock;
1062 map.m_len = bh->b_size >> inode->i_blkbits;
da85985c 1063 map.m_next_pgofs = next_pgofs;
003a3e1d 1064
a7de6086
JK		 1065 err = f2fs_map_blocks(inode, &map, create, flag);
1066 if (!err) {
003a3e1d
JK		 1067 map_bh(bh, inode->i_sb, map.m_pblk);
1068 bh->b_state = (bh->b_state & ~F2FS_MAP_FLAGS) | map.m_flags;
b86e3307 1069 bh->b_size = (u64)map.m_len << inode->i_blkbits;
003a3e1d 1070 }
a7de6086 1071 return err;
003a3e1d
JK		 1072}
1073
ccfb3000 1074static int get_data_block(struct inode *inode, sector_t iblock,
da85985c
CY		 1075 struct buffer_head *bh_result, int create, int flag,
1076 pgoff_t *next_pgofs)
e2b4e2bc 1077{
da85985c
CY		 1078 return __get_data_block(inode, iblock, bh_result, create,
1079 flag, next_pgofs);
e2b4e2bc
CY		 1080}
1081
1082static int get_data_block_dio(struct inode *inode, sector_t iblock,
ccfb3000
JK		 1083 struct buffer_head *bh_result, int create)
1084{
e2b4e2bc 1085 return __get_data_block(inode, iblock, bh_result, create,
f2220c7f 1086 F2FS_GET_BLOCK_DEFAULT, NULL);
ccfb3000
JK		 1087}
1088
e2b4e2bc 1089static int get_data_block_bmap(struct inode *inode, sector_t iblock,
ccfb3000
JK		 1090 struct buffer_head *bh_result, int create)
1091{
179448bf 1092 /* Block number less than F2FS MAX BLOCKS */
e0afc4d6 1093 if (unlikely(iblock >= F2FS_I_SB(inode)->max_file_blocks))
179448bf
YH		 1094 return -EFBIG;
1095
e2b4e2bc 1096 return __get_data_block(inode, iblock, bh_result, create,
da85985c 1097 F2FS_GET_BLOCK_BMAP, NULL);
ccfb3000
JK		 1098}
1099
7f63eb77
JK		 1100static inline sector_t logical_to_blk(struct inode *inode, loff_t offset)
1101{
1102 return (offset >> inode->i_blkbits);
1103}
1104
1105static inline loff_t blk_to_logical(struct inode *inode, sector_t blk)
1106{
1107 return (blk << inode->i_blkbits);
1108}
1109
9ab70134
JK		 1110int f2fs_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo,
1111 u64 start, u64 len)
1112{
7f63eb77
JK		 1113 struct buffer_head map_bh;
1114 sector_t start_blk, last_blk;
da85985c 1115 pgoff_t next_pgofs;
7f63eb77
JK		 1116 u64 logical = 0, phys = 0, size = 0;
1117 u32 flags = 0;
7f63eb77
JK		 1118 int ret = 0;
1119
1120 ret = fiemap_check_flags(fieinfo, FIEMAP_FLAG_SYNC);
1121 if (ret)
1122 return ret;
1123
67f8cf3c
JK		 1124 if (f2fs_has_inline_data(inode)) {
1125 ret = f2fs_inline_data_fiemap(inode, fieinfo, start, len);
1126 if (ret != -EAGAIN)
1127 return ret;
1128 }
1129
5955102c 1130 inode_lock(inode);
de1475cc 1131
7f63eb77
JK		 1132 if (logical_to_blk(inode, len) == 0)
1133 len = blk_to_logical(inode, 1);
1134
1135 start_blk = logical_to_blk(inode, start);
1136 last_blk = logical_to_blk(inode, start + len - 1);
9a950d52 1137
7f63eb77
JK		 1138next:
1139 memset(&map_bh, 0, sizeof(struct buffer_head));
1140 map_bh.b_size = len;
1141
e2b4e2bc 1142 ret = get_data_block(inode, start_blk, &map_bh, 0,
da85985c 1143 F2FS_GET_BLOCK_FIEMAP, &next_pgofs);
7f63eb77
JK		 1144 if (ret)
1145 goto out;
1146
1147 /* HOLE */
1148 if (!buffer_mapped(&map_bh)) {
da85985c 1149 start_blk = next_pgofs;
58736fa6
CY		 1150
1151 if (blk_to_logical(inode, start_blk) < blk_to_logical(inode,
1152 F2FS_I_SB(inode)->max_file_blocks))
9a950d52 1153 goto prep_next;
58736fa6 1154
9a950d52
FL		 1155 flags |= FIEMAP_EXTENT_LAST;
1156 }
7f63eb77 1157
da5af127
CY		 1158 if (size) {
1159 if (f2fs_encrypted_inode(inode))
1160 flags |= FIEMAP_EXTENT_DATA_ENCRYPTED;
1161
9a950d52
FL		 1162 ret = fiemap_fill_next_extent(fieinfo, logical,
1163 phys, size, flags);
da5af127 1164 }
7f63eb77 1165
9a950d52
FL		 1166 if (start_blk > last_blk || ret)
1167 goto out;
7f63eb77 1168
9a950d52
FL		 1169 logical = blk_to_logical(inode, start_blk);
1170 phys = blk_to_logical(inode, map_bh.b_blocknr);
1171 size = map_bh.b_size;
1172 flags = 0;
1173 if (buffer_unwritten(&map_bh))
1174 flags = FIEMAP_EXTENT_UNWRITTEN;
7f63eb77 1175
9a950d52 1176 start_blk += logical_to_blk(inode, size);
7f63eb77 1177
9a950d52 1178prep_next:
7f63eb77
JK		 1179 cond_resched();
1180 if (fatal_signal_pending(current))
1181 ret = -EINTR;
1182 else
1183 goto next;
1184out:
1185 if (ret == 1)
1186 ret = 0;
1187
5955102c 1188 inode_unlock(inode);
7f63eb77 1189 return ret;
9ab70134
JK		 1190}
1191
f1e88660
JK		 1192/*
1193 * This function was originally taken from fs/mpage.c, and customized for f2fs.
1194 * Major change was from block_size == page_size in f2fs by default.
1195 */
1196static int f2fs_mpage_readpages(struct address_space *mapping,
1197 struct list_head *pages, struct page *page,
1198 unsigned nr_pages)
1199{
1200 struct bio *bio = NULL;
1201 unsigned page_idx;
1202 sector_t last_block_in_bio = 0;
1203 struct inode *inode = mapping->host;
1204 const unsigned blkbits = inode->i_blkbits;
1205 const unsigned blocksize = 1 << blkbits;
1206 sector_t block_in_file;
1207 sector_t last_block;
1208 sector_t last_block_in_file;
1209 sector_t block_nr;
f1e88660
JK		 1210 struct f2fs_map_blocks map;
1211
1212 map.m_pblk = 0;
1213 map.m_lblk = 0;
1214 map.m_len = 0;
1215 map.m_flags = 0;
da85985c 1216 map.m_next_pgofs = NULL;
f1e88660
JK		 1217
1218 for (page_idx = 0; nr_pages; page_idx++, nr_pages--) {
1219
f1e88660 1220 if (pages) {
939afa94 1221 page = list_last_entry(pages, struct page, lru);
a83d50bc
KM		 1222
1223 prefetchw(&page->flags);
f1e88660
JK		 1224 list_del(&page->lru);
1225 if (add_to_page_cache_lru(page, mapping,
8a5c743e
MH		 1226 page->index,
1227 readahead_gfp_mask(mapping)))
f1e88660
JK		 1228 goto next_page;
1229 }
1230
1231 block_in_file = (sector_t)page->index;
1232 last_block = block_in_file + nr_pages;
1233 last_block_in_file = (i_size_read(inode) + blocksize - 1) >>
1234 blkbits;
1235 if (last_block > last_block_in_file)
1236 last_block = last_block_in_file;
1237
1238 /*
1239 * Map blocks using the previous result first.
1240 */
1241 if ((map.m_flags & F2FS_MAP_MAPPED) &&
1242 block_in_file > map.m_lblk &&
1243 block_in_file < (map.m_lblk + map.m_len))
1244 goto got_it;
1245
1246 /*
1247 * Then do more f2fs_map_blocks() calls until we are
1248 * done with this page.
1249 */
1250 map.m_flags = 0;
1251
1252 if (block_in_file < last_block) {
1253 map.m_lblk = block_in_file;
1254 map.m_len = last_block - block_in_file;
1255
46c9e141 1256 if (f2fs_map_blocks(inode, &map, 0,
f2220c7f 1257 F2FS_GET_BLOCK_DEFAULT))
f1e88660
JK		 1258 goto set_error_page;
1259 }
1260got_it:
1261 if ((map.m_flags & F2FS_MAP_MAPPED)) {
1262 block_nr = map.m_pblk + block_in_file - map.m_lblk;
1263 SetPageMappedToDisk(page);
1264
1265 if (!PageUptodate(page) && !cleancache_get_page(page)) {
1266 SetPageUptodate(page);
1267 goto confused;
1268 }
1269 } else {
09cbfeaf 1270 zero_user_segment(page, 0, PAGE_SIZE);
237c0790
JK		 1271 if (!PageUptodate(page))
1272 SetPageUptodate(page);
f1e88660
JK		 1273 unlock_page(page);
1274 goto next_page;
1275 }
1276
1277 /*
1278 * This page will go to BIO. Do we need to send this
1279 * BIO off first?
1280 */
3c62be17
JK		 1281 if (bio && (last_block_in_bio != block_nr - 1 ||
1282 !__same_bdev(F2FS_I_SB(inode), block_nr, bio))) {
f1e88660 1283submit_and_realloc:
4fc29c1a 1284 __submit_bio(F2FS_I_SB(inode), bio, DATA);
f1e88660
JK		 1285 bio = NULL;
1286 }
1287 if (bio == NULL) {
13ba41e3 1288 bio = f2fs_grab_read_bio(inode, block_nr, nr_pages);
1d353eb7
JK		 1289 if (IS_ERR(bio)) {
1290 bio = NULL;
f1e88660 1291 goto set_error_page;
4375a336 1292 }
f1e88660
JK		 1293 }
1294
1295 if (bio_add_page(bio, page, blocksize, 0) < blocksize)
1296 goto submit_and_realloc;
1297
1298 last_block_in_bio = block_nr;
1299 goto next_page;
1300set_error_page:
1301 SetPageError(page);
09cbfeaf 1302 zero_user_segment(page, 0, PAGE_SIZE);
f1e88660
JK		 1303 unlock_page(page);
1304 goto next_page;
1305confused:
1306 if (bio) {
4fc29c1a 1307 __submit_bio(F2FS_I_SB(inode), bio, DATA);
f1e88660
JK		 1308 bio = NULL;
1309 }
1310 unlock_page(page);
1311next_page:
1312 if (pages)
09cbfeaf 1313 put_page(page);
f1e88660
JK		 1314 }
1315 BUG_ON(pages && !list_empty(pages));
1316 if (bio)
4fc29c1a 1317 __submit_bio(F2FS_I_SB(inode), bio, DATA);
f1e88660
JK		 1318 return 0;
1319}
1320
eb47b800
JK		 1321static int f2fs_read_data_page(struct file *file, struct page *page)
1322{
9ffe0fb5 1323 struct inode *inode = page->mapping->host;
b3d208f9 1324 int ret = -EAGAIN;
9ffe0fb5 1325
c20e89cd
CY		 1326 trace_f2fs_readpage(page, DATA);
1327
e1c42045 1328 /* If the file has inline data, try to read it directly */
9ffe0fb5
HL		 1329 if (f2fs_has_inline_data(inode))
1330 ret = f2fs_read_inline_data(inode, page);
b3d208f9 1331 if (ret == -EAGAIN)
f1e88660 1332 ret = f2fs_mpage_readpages(page->mapping, NULL, page, 1);
9ffe0fb5 1333 return ret;
eb47b800
JK		 1334}
1335
1336static int f2fs_read_data_pages(struct file *file,
1337 struct address_space *mapping,
1338 struct list_head *pages, unsigned nr_pages)
1339{
71cb4aff 1340 struct inode *inode = mapping->host;
939afa94 1341 struct page *page = list_last_entry(pages, struct page, lru);
b8c29400
CY		 1342
1343 trace_f2fs_readpages(inode, page, nr_pages);
9ffe0fb5
HL		 1344
1345 /* If the file has inline data, skip readpages */
1346 if (f2fs_has_inline_data(inode))
1347 return 0;
1348
f1e88660 1349 return f2fs_mpage_readpages(mapping, pages, NULL, nr_pages);
eb47b800
JK		 1350}
1351
7eab0c0d
HP		 1352static int encrypt_one_page(struct f2fs_io_info *fio)
1353{
1354 struct inode *inode = fio->page->mapping->host;
1355 gfp_t gfp_flags = GFP_NOFS;
1356
1958593e 1357 if (!f2fs_encrypted_file(inode))
7eab0c0d
HP		 1358 return 0;
1359
1360 /* wait for GCed encrypted page writeback */
d4c759ee 1361 f2fs_wait_on_block_writeback(fio->sbi, fio->old_blkaddr);
7eab0c0d
HP		 1362
1363retry_encrypt:
1364 fio->encrypted_page = fscrypt_encrypt_page(inode, fio->page,
1365 PAGE_SIZE, 0, fio->page->index, gfp_flags);
1366 if (!IS_ERR(fio->encrypted_page))
1367 return 0;
1368
1369 /* flush pending IOs and wait for a while in the ENOMEM case */
1370 if (PTR_ERR(fio->encrypted_page) == -ENOMEM) {
b9109b0e 1371 f2fs_flush_merged_writes(fio->sbi);
7eab0c0d
HP		 1372 congestion_wait(BLK_RW_ASYNC, HZ/50);
1373 gfp_flags |= __GFP_NOFAIL;
1374 goto retry_encrypt;
1375 }
1376 return PTR_ERR(fio->encrypted_page);
1377}
1378
1379static inline bool need_inplace_update(struct f2fs_io_info *fio)
1380{
1381 struct inode *inode = fio->page->mapping->host;
1382
7eab0c0d
HP		 1383 if (S_ISDIR(inode->i_mode) || f2fs_is_atomic_file(inode))
1384 return false;
1385 if (is_cold_data(fio->page))
1386 return false;
1387 if (IS_ATOMIC_WRITTEN_PAGE(fio->page))
1388 return false;
1389
1390 return need_inplace_update_policy(inode, fio);
1391}
1392
a817737e
JK		 1393static inline bool valid_ipu_blkaddr(struct f2fs_io_info *fio)
1394{
1395 if (fio->old_blkaddr == NEW_ADDR)
1396 return false;
1397 if (fio->old_blkaddr == NULL_ADDR)
1398 return false;
1399 return true;
1400}
1401
05ca3632 1402int do_write_data_page(struct f2fs_io_info *fio)
eb47b800 1403{
05ca3632 1404 struct page *page = fio->page;
eb47b800 1405 struct inode *inode = page->mapping->host;
eb47b800 1406 struct dnode_of_data dn;
e959c8f5
HP		 1407 struct extent_info ei = {0,0,0};
1408 bool ipu_force = false;
eb47b800
JK		 1409 int err = 0;
1410
1411 set_new_dnode(&dn, inode, NULL, NULL, 0);
e959c8f5
HP		 1412 if (need_inplace_update(fio) &&
1413 f2fs_lookup_extent_cache(inode, page->index, &ei)) {
1414 fio->old_blkaddr = ei.blk + page->index - ei.fofs;
a817737e
JK		 1415
1416 if (valid_ipu_blkaddr(fio)) {
e959c8f5 1417 ipu_force = true;
cc15620b 1418 fio->need_lock = LOCK_DONE;
e959c8f5
HP		 1419 goto got_it;
1420 }
1421 }
279d6df2 1422
d29460e5
JK		 1423 /* Deadlock due to between page->lock and f2fs_lock_op */
1424 if (fio->need_lock == LOCK_REQ && !f2fs_trylock_op(fio->sbi))
1425 return -EAGAIN;
279d6df2 1426
266e97a8 1427 err = get_dnode_of_data(&dn, page->index, LOOKUP_NODE);
eb47b800 1428 if (err)
279d6df2 1429 goto out;
eb47b800 1430
28bc106b 1431 fio->old_blkaddr = dn.data_blkaddr;
eb47b800
JK		 1432
1433 /* This page is already truncated */
7a9d7548 1434 if (fio->old_blkaddr == NULL_ADDR) {
2bca1e23 1435 ClearPageUptodate(page);
eb47b800 1436 goto out_writepage;
2bca1e23 1437 }
e959c8f5 1438got_it:
eb47b800
JK		 1439 /*
1440 * If current allocation needs SSR,
1441 * it had better in-place writes for updated data.
1442 */
a817737e 1443 if (ipu_force || (valid_ipu_blkaddr(fio) && need_inplace_update(fio))) {
cc15620b
JK		 1444 err = encrypt_one_page(fio);
1445 if (err)
1446 goto out_writepage;
1447
1448 set_page_writeback(page);
279d6df2 1449 f2fs_put_dnode(&dn);
cc15620b 1450 if (fio->need_lock == LOCK_REQ)
279d6df2 1451 f2fs_unlock_op(fio->sbi);
d1b3e72d 1452 err = rewrite_data_page(fio);
7eab0c0d 1453 trace_f2fs_do_write_data_page(fio->page, IPU);
91942321 1454 set_inode_flag(inode, FI_UPDATE_WRITE);
279d6df2 1455 return err;
eb47b800 1456 }
279d6df2 1457
cc15620b
JK		 1458 if (fio->need_lock == LOCK_RETRY) {
1459 if (!f2fs_trylock_op(fio->sbi)) {
1460 err = -EAGAIN;
1461 goto out_writepage;
1462 }
1463 fio->need_lock = LOCK_REQ;
1464 }
1465
1466 err = encrypt_one_page(fio);
1467 if (err)
1468 goto out_writepage;
1469
1470 set_page_writeback(page);
1471
279d6df2
HP		 1472 /* LFS mode write path */
1473 write_data_page(&dn, fio);
1474 trace_f2fs_do_write_data_page(page, OPU);
1475 set_inode_flag(inode, FI_APPEND_WRITE);
1476 if (page->index == 0)
1477 set_inode_flag(inode, FI_FIRST_BLOCK_WRITTEN);
eb47b800
JK		 1478out_writepage:
1479 f2fs_put_dnode(&dn);
279d6df2 1480out:
cc15620b 1481 if (fio->need_lock == LOCK_REQ)
279d6df2 1482 f2fs_unlock_op(fio->sbi);
eb47b800
JK		 1483 return err;
1484}
1485
d68f735b 1486static int __write_data_page(struct page *page, bool *submitted,
b0af6d49
CY		 1487 struct writeback_control *wbc,
1488 enum iostat_type io_type)
eb47b800
JK		 1489{
1490 struct inode *inode = page->mapping->host;
4081363f 1491 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
eb47b800
JK		 1492 loff_t i_size = i_size_read(inode);
1493 const pgoff_t end_index = ((unsigned long long) i_size)
09cbfeaf 1494 >> PAGE_SHIFT;
26de9b11 1495 loff_t psize = (page->index + 1) << PAGE_SHIFT;
9ffe0fb5 1496 unsigned offset = 0;
39936837 1497 bool need_balance_fs = false;
eb47b800 1498 int err = 0;
458e6197 1499 struct f2fs_io_info fio = {
05ca3632 1500 .sbi = sbi,
39d787be 1501 .ino = inode->i_ino,
458e6197 1502 .type = DATA,
04d328de 1503 .op = REQ_OP_WRITE,
7637241e 1504 .op_flags = wbc_to_write_flags(wbc),
e959c8f5 1505 .old_blkaddr = NULL_ADDR,
05ca3632 1506 .page = page,
4375a336 1507 .encrypted_page = NULL,
d68f735b 1508 .submitted = false,
cc15620b 1509 .need_lock = LOCK_RETRY,
b0af6d49 1510 .io_type = io_type,
458e6197 1511 };
eb47b800 1512
ecda0de3
CY		 1513 trace_f2fs_writepage(page, DATA);
1514
0771fcc7
CY		 1515 if (unlikely(is_sbi_flag_set(sbi, SBI_POR_DOING)))
1516 goto redirty_out;
1517
eb47b800 1518 if (page->index < end_index)
39936837 1519 goto write;
eb47b800
JK		 1520
1521 /*
1522 * If the offset is out-of-range of file size,
1523 * this page does not have to be written to disk.
1524 */
09cbfeaf 1525 offset = i_size & (PAGE_SIZE - 1);
76f60268 1526 if ((page->index >= end_index + 1) || !offset)
39936837 1527 goto out;
eb47b800 1528
09cbfeaf 1529 zero_user_segment(page, offset, PAGE_SIZE);
39936837 1530write:
1e84371f
JK		 1531 if (f2fs_is_drop_cache(inode))
1532 goto out;
e6e5f561
JK		 1533 /* we should not write 0'th page having journal header */
1534 if (f2fs_is_volatile_file(inode) && (!page->index ||
1535 (!wbc->for_reclaim &&
1536 available_free_memory(sbi, BASE_CHECK))))
1e84371f 1537 goto redirty_out;
eb47b800 1538
cf779cab
JK		 1539 /* we should bypass data pages to proceed the kworkder jobs */
1540 if (unlikely(f2fs_cp_error(sbi))) {
7f319975 1541 mapping_set_error(page->mapping, -EIO);
a7ffdbe2 1542 goto out;
cf779cab
JK		 1543 }
1544
39936837 1545 /* Dentry blocks are controlled by checkpoint */
eb47b800 1546 if (S_ISDIR(inode->i_mode)) {
cc15620b 1547 fio.need_lock = LOCK_DONE;
05ca3632 1548 err = do_write_data_page(&fio);
8618b881
JK		 1549 goto done;
1550 }
9ffe0fb5 1551
8618b881 1552 if (!wbc->for_reclaim)
39936837 1553 need_balance_fs = true;
7f3037a5 1554 else if (has_not_enough_free_secs(sbi, 0, 0))
39936837 1555 goto redirty_out;
ef095d19
JK		 1556 else
1557 set_inode_flag(inode, FI_HOT_DATA);
eb47b800 1558
b3d208f9 1559 err = -EAGAIN;
dd7b2333 1560 if (f2fs_has_inline_data(inode)) {
b3d208f9 1561 err = f2fs_write_inline_data(inode, page);
dd7b2333
YH		 1562 if (!err)
1563 goto out;
1564 }
279d6df2 1565
cc15620b 1566 if (err == -EAGAIN) {
05ca3632 1567 err = do_write_data_page(&fio);
cc15620b
JK		 1568 if (err == -EAGAIN) {
1569 fio.need_lock = LOCK_REQ;
1570 err = do_write_data_page(&fio);
1571 }
1572 }
26de9b11
JK		 1573 if (F2FS_I(inode)->last_disk_size < psize)
1574 F2FS_I(inode)->last_disk_size = psize;
279d6df2 1575
8618b881
JK		 1576done:
1577 if (err && err != -ENOENT)
1578 goto redirty_out;
eb47b800 1579
39936837 1580out:
a7ffdbe2 1581 inode_dec_dirty_pages(inode);
2bca1e23
JK		 1582 if (err)
1583 ClearPageUptodate(page);
0c3a5797
CY		 1584
1585 if (wbc->for_reclaim) {
b9109b0e 1586 f2fs_submit_merged_write_cond(sbi, inode, 0, page->index, DATA);
ef095d19 1587 clear_inode_flag(inode, FI_HOT_DATA);
0c3a5797 1588 remove_dirty_inode(inode);
d68f735b 1589 submitted = NULL;
0c3a5797
CY		 1590 }
1591
eb47b800 1592 unlock_page(page);
a7881893
JK		 1593 if (!S_ISDIR(inode->i_mode))
1594 f2fs_balance_fs(sbi, need_balance_fs);
0c3a5797 1595
d68f735b 1596 if (unlikely(f2fs_cp_error(sbi))) {
b9109b0e 1597 f2fs_submit_merged_write(sbi, DATA);
d68f735b
JK		 1598 submitted = NULL;
1599 }
1600
1601 if (submitted)
1602 *submitted = fio.submitted;
0c3a5797 1603
eb47b800
JK		 1604 return 0;
1605
eb47b800 1606redirty_out:
76f60268 1607 redirty_page_for_writepage(wbc, page);
0002b61b
CY		 1608 if (!err)
1609 return AOP_WRITEPAGE_ACTIVATE;
b230e6ca
JK		 1610 unlock_page(page);
1611 return err;
fa9150a8
NJ		 1612}
1613
f566bae8
JK		 1614static int f2fs_write_data_page(struct page *page,
1615 struct writeback_control *wbc)
1616{
b0af6d49 1617 return __write_data_page(page, NULL, wbc, FS_DATA_IO);
f566bae8
JK		 1618}
1619
8f46dcae
CY		 1620/*
1621 * This function was copied from write_cche_pages from mm/page-writeback.c.
1622 * The major change is making write step of cold data page separately from
1623 * warm/hot data page.
1624 */
1625static int f2fs_write_cache_pages(struct address_space *mapping,
b0af6d49
CY		 1626 struct writeback_control *wbc,
1627 enum iostat_type io_type)
8f46dcae
CY		 1628{
1629 int ret = 0;
1630 int done = 0;
1631 struct pagevec pvec;
1632 int nr_pages;
1633 pgoff_t uninitialized_var(writeback_index);
1634 pgoff_t index;
1635 pgoff_t end; /* Inclusive */
1636 pgoff_t done_index;
942fd319 1637 pgoff_t last_idx = ULONG_MAX;
8f46dcae
CY		 1638 int cycled;
1639 int range_whole = 0;
1640 int tag;
8f46dcae
CY		 1641
1642 pagevec_init(&pvec, 0);
46ae957f 1643
ef095d19
JK		 1644 if (get_dirty_pages(mapping->host) <=
1645 SM_I(F2FS_M_SB(mapping))->min_hot_blocks)
1646 set_inode_flag(mapping->host, FI_HOT_DATA);
1647 else
1648 clear_inode_flag(mapping->host, FI_HOT_DATA);
1649
8f46dcae
CY		 1650 if (wbc->range_cyclic) {
1651 writeback_index = mapping->writeback_index; /* prev offset */
1652 index = writeback_index;
1653 if (index == 0)
1654 cycled = 1;
1655 else
1656 cycled = 0;
1657 end = -1;
1658 } else {
09cbfeaf
KS		 1659 index = wbc->range_start >> PAGE_SHIFT;
1660 end = wbc->range_end >> PAGE_SHIFT;
8f46dcae
CY		 1661 if (wbc->range_start == 0 && wbc->range_end == LLONG_MAX)
1662 range_whole = 1;
1663 cycled = 1; /* ignore range_cyclic tests */
1664 }
1665 if (wbc->sync_mode == WB_SYNC_ALL || wbc->tagged_writepages)
1666 tag = PAGECACHE_TAG_TOWRITE;
1667 else
1668 tag = PAGECACHE_TAG_DIRTY;
1669retry:
1670 if (wbc->sync_mode == WB_SYNC_ALL || wbc->tagged_writepages)
1671 tag_pages_for_writeback(mapping, index, end);
1672 done_index = index;
1673 while (!done && (index <= end)) {
1674 int i;
1675
1676 nr_pages = pagevec_lookup_tag(&pvec, mapping, &index, tag,
1677 min(end - index, (pgoff_t)PAGEVEC_SIZE - 1) + 1);
1678 if (nr_pages == 0)
1679 break;
1680
1681 for (i = 0; i < nr_pages; i++) {
1682 struct page *page = pvec.pages[i];
d68f735b 1683 bool submitted = false;
8f46dcae
CY		 1684
1685 if (page->index > end) {
1686 done = 1;
1687 break;
1688 }
1689
1690 done_index = page->index;
d29460e5 1691retry_write:
8f46dcae
CY		 1692 lock_page(page);
1693
1694 if (unlikely(page->mapping != mapping)) {
1695continue_unlock:
1696 unlock_page(page);
1697 continue;
1698 }
1699
1700 if (!PageDirty(page)) {
1701 /* someone wrote it for us */
1702 goto continue_unlock;
1703 }
1704
8f46dcae
CY		 1705 if (PageWriteback(page)) {
1706 if (wbc->sync_mode != WB_SYNC_NONE)
fec1d657
JK		 1707 f2fs_wait_on_page_writeback(page,
1708 DATA, true);
8f46dcae
CY		 1709 else
1710 goto continue_unlock;
1711 }
1712
1713 BUG_ON(PageWriteback(page));
1714 if (!clear_page_dirty_for_io(page))
1715 goto continue_unlock;
1716
b0af6d49 1717 ret = __write_data_page(page, &submitted, wbc, io_type);
8f46dcae 1718 if (unlikely(ret)) {
0002b61b
CY		 1719 /*
1720 * keep nr_to_write, since vfs uses this to
1721 * get # of written pages.
1722 */
1723 if (ret == AOP_WRITEPAGE_ACTIVATE) {
1724 unlock_page(page);
1725 ret = 0;
1726 continue;
d29460e5
JK		 1727 } else if (ret == -EAGAIN) {
1728 ret = 0;
1729 if (wbc->sync_mode == WB_SYNC_ALL) {
1730 cond_resched();
1731 congestion_wait(BLK_RW_ASYNC,
1732 HZ/50);
1733 goto retry_write;
1734 }
1735 continue;
0002b61b 1736 }
b230e6ca
JK		 1737 done_index = page->index + 1;
1738 done = 1;
1739 break;
d68f735b 1740 } else if (submitted) {
942fd319 1741 last_idx = page->index;
8f46dcae
CY		 1742 }
1743
687de7f1
JK		 1744 /* give a priority to WB_SYNC threads */
1745 if ((atomic_read(&F2FS_M_SB(mapping)->wb_sync_req) ||
1746 --wbc->nr_to_write <= 0) &&
1747 wbc->sync_mode == WB_SYNC_NONE) {
8f46dcae
CY		 1748 done = 1;
1749 break;
1750 }
1751 }
1752 pagevec_release(&pvec);
1753 cond_resched();
1754 }
1755
8f46dcae
CY		 1756 if (!cycled && !done) {
1757 cycled = 1;
1758 index = 0;
1759 end = writeback_index - 1;
1760 goto retry;
1761 }
1762 if (wbc->range_cyclic || (range_whole && wbc->nr_to_write > 0))
1763 mapping->writeback_index = done_index;
1764
942fd319 1765 if (last_idx != ULONG_MAX)
b9109b0e
JK		 1766 f2fs_submit_merged_write_cond(F2FS_M_SB(mapping), mapping->host,
1767 0, last_idx, DATA);
6ca56ca4 1768
8f46dcae
CY		 1769 return ret;
1770}
1771
b0af6d49
CY		 1772int __f2fs_write_data_pages(struct address_space *mapping,
1773 struct writeback_control *wbc,
1774 enum iostat_type io_type)
eb47b800
JK		 1775{
1776 struct inode *inode = mapping->host;
4081363f 1777 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
9dfa1baf 1778 struct blk_plug plug;
eb47b800 1779 int ret;
eb47b800 1780
cfb185a1 1781 /* deal with chardevs and other special file */
1782 if (!mapping->a_ops->writepage)
1783 return 0;
1784
6a290544
CY		 1785 /* skip writing if there is no dirty page in this inode */
1786 if (!get_dirty_pages(inode) && wbc->sync_mode == WB_SYNC_NONE)
1787 return 0;
1788
0771fcc7
CY		 1789 /* during POR, we don't need to trigger writepage at all. */
1790 if (unlikely(is_sbi_flag_set(sbi, SBI_POR_DOING)))
1791 goto skip_write;
1792
a1257023
JK		 1793 if (S_ISDIR(inode->i_mode) && wbc->sync_mode == WB_SYNC_NONE &&
1794 get_dirty_pages(inode) < nr_pages_to_skip(sbi, DATA) &&
1795 available_free_memory(sbi, DIRTY_DENTS))
1796 goto skip_write;
1797
d323d005 1798 /* skip writing during file defragment */
91942321 1799 if (is_inode_flag_set(inode, FI_DO_DEFRAG))
d323d005
CY		 1800 goto skip_write;
1801
d31c7c3f
YH		 1802 trace_f2fs_writepages(mapping->host, wbc, DATA);
1803
687de7f1
JK		 1804 /* to avoid spliting IOs due to mixed WB_SYNC_ALL and WB_SYNC_NONE */
1805 if (wbc->sync_mode == WB_SYNC_ALL)
1806 atomic_inc(&sbi->wb_sync_req);
1807 else if (atomic_read(&sbi->wb_sync_req))
1808 goto skip_write;
1809
9dfa1baf 1810 blk_start_plug(&plug);
b0af6d49 1811 ret = f2fs_write_cache_pages(mapping, wbc, io_type);
9dfa1baf 1812 blk_finish_plug(&plug);
687de7f1
JK		 1813
1814 if (wbc->sync_mode == WB_SYNC_ALL)
1815 atomic_dec(&sbi->wb_sync_req);
28ea6162
JK		 1816 /*
1817 * if some pages were truncated, we cannot guarantee its mapping->host
1818 * to detect pending bios.
1819 */
458e6197 1820
c227f912 1821 remove_dirty_inode(inode);
eb47b800 1822 return ret;
d3baf95d
JK		 1823
1824skip_write:
a7ffdbe2 1825 wbc->pages_skipped += get_dirty_pages(inode);
d31c7c3f 1826 trace_f2fs_writepages(mapping->host, wbc, DATA);
d3baf95d 1827 return 0;
eb47b800
JK		 1828}
1829
b0af6d49
CY		 1830static int f2fs_write_data_pages(struct address_space *mapping,
1831 struct writeback_control *wbc)
1832{
1833 struct inode *inode = mapping->host;
1834
1835 return __f2fs_write_data_pages(mapping, wbc,
1836 F2FS_I(inode)->cp_task == current ?
1837 FS_CP_DATA_IO : FS_DATA_IO);
1838}
1839
3aab8f82
CY		 1840static void f2fs_write_failed(struct address_space *mapping, loff_t to)
1841{
1842 struct inode *inode = mapping->host;
819d9153 1843 loff_t i_size = i_size_read(inode);
3aab8f82 1844
819d9153 1845 if (to > i_size) {
5a3a2d83 1846 down_write(&F2FS_I(inode)->i_mmap_sem);
819d9153
JK		 1847 truncate_pagecache(inode, i_size);
1848 truncate_blocks(inode, i_size, true);
5a3a2d83 1849 up_write(&F2FS_I(inode)->i_mmap_sem);
3aab8f82
CY		 1850 }
1851}
1852
2aadac08
JK		 1853static int prepare_write_begin(struct f2fs_sb_info *sbi,
1854 struct page *page, loff_t pos, unsigned len,
1855 block_t *blk_addr, bool *node_changed)
1856{
1857 struct inode *inode = page->mapping->host;
1858 pgoff_t index = page->index;
1859 struct dnode_of_data dn;
1860 struct page *ipage;
b4d07a3e 1861 bool locked = false;
e15882b6 1862 struct extent_info ei = {0,0,0};
2aadac08
JK		 1863 int err = 0;
1864
24b84912
JK		 1865 /*
1866 * we already allocated all the blocks, so we don't need to get
1867 * the block addresses when there is no need to fill the page.
1868 */
dc91de78
JK		 1869 if (!f2fs_has_inline_data(inode) && len == PAGE_SIZE &&
1870 !is_inode_flag_set(inode, FI_NO_PREALLOC))
24b84912
JK		 1871 return 0;
1872
b4d07a3e 1873 if (f2fs_has_inline_data(inode) ||
09cbfeaf 1874 (pos & PAGE_MASK) >= i_size_read(inode)) {
59c9081b 1875 __do_map_lock(sbi, F2FS_GET_BLOCK_PRE_AIO, true);
b4d07a3e
JK		 1876 locked = true;
1877 }
1878restart:
2aadac08
JK		 1879 /* check inline_data */
1880 ipage = get_node_page(sbi, inode->i_ino);
1881 if (IS_ERR(ipage)) {
1882 err = PTR_ERR(ipage);
1883 goto unlock_out;
1884 }
1885
1886 set_new_dnode(&dn, inode, ipage, ipage, 0);
1887
1888 if (f2fs_has_inline_data(inode)) {
f2470371 1889 if (pos + len <= MAX_INLINE_DATA(inode)) {
2aadac08 1890 read_inline_data(page, ipage);
91942321 1891 set_inode_flag(inode, FI_DATA_EXIST);
ab47036d
CY		 1892 if (inode->i_nlink)
1893 set_inline_node(ipage);
2aadac08
JK		 1894 } else {
1895 err = f2fs_convert_inline_page(&dn, page);
1896 if (err)
b4d07a3e
JK		 1897 goto out;
1898 if (dn.data_blkaddr == NULL_ADDR)
1899 err = f2fs_get_block(&dn, index);
1900 }
1901 } else if (locked) {
1902 err = f2fs_get_block(&dn, index);
1903 } else {
1904 if (f2fs_lookup_extent_cache(inode, index, &ei)) {
1905 dn.data_blkaddr = ei.blk + index - ei.fofs;
1906 } else {
b4d07a3e
JK		 1907 /* hole case */
1908 err = get_dnode_of_data(&dn, index, LOOKUP_NODE);
4da7bf5a 1909 if (err || dn.data_blkaddr == NULL_ADDR) {
b4d07a3e 1910 f2fs_put_dnode(&dn);
59c9081b
YH		 1911 __do_map_lock(sbi, F2FS_GET_BLOCK_PRE_AIO,
1912 true);
b4d07a3e
JK		 1913 locked = true;
1914 goto restart;
1915 }
2aadac08
JK		 1916 }
1917 }
b4d07a3e 1918
2aadac08
JK		 1919 /* convert_inline_page can make node_changed */
1920 *blk_addr = dn.data_blkaddr;
1921 *node_changed = dn.node_changed;
b4d07a3e 1922out:
2aadac08
JK		 1923 f2fs_put_dnode(&dn);
1924unlock_out:
b4d07a3e 1925 if (locked)
59c9081b 1926 __do_map_lock(sbi, F2FS_GET_BLOCK_PRE_AIO, false);
2aadac08
JK		 1927 return err;
1928}
1929
eb47b800
JK		 1930static int f2fs_write_begin(struct file *file, struct address_space *mapping,
1931 loff_t pos, unsigned len, unsigned flags,
1932 struct page **pagep, void **fsdata)
1933{
1934 struct inode *inode = mapping->host;
4081363f 1935 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
86531d6b 1936 struct page *page = NULL;
09cbfeaf 1937 pgoff_t index = ((unsigned long long) pos) >> PAGE_SHIFT;
2aadac08
JK		 1938 bool need_balance = false;
1939 block_t blkaddr = NULL_ADDR;
eb47b800
JK		 1940 int err = 0;
1941
62aed044
CY		 1942 trace_f2fs_write_begin(inode, pos, len, flags);
1943
5f727395
JK		 1944 /*
1945 * We should check this at this moment to avoid deadlock on inode page
1946 * and #0 page. The locking rule for inline_data conversion should be:
1947 * lock_page(page #0) -> lock_page(inode_page)
1948 */
1949 if (index != 0) {
1950 err = f2fs_convert_inline_inode(inode);
1951 if (err)
1952 goto fail;
1953 }
afcb7ca0 1954repeat:
86d54795
JK		 1955 /*
1956 * Do not use grab_cache_page_write_begin() to avoid deadlock due to
1957 * wait_for_stable_page. Will wait that below with our IO control.
1958 */
1959 page = pagecache_get_page(mapping, index,
1960 FGP_LOCK | FGP_WRITE | FGP_CREAT, GFP_NOFS);
3aab8f82
CY		 1961 if (!page) {
1962 err = -ENOMEM;
1963 goto fail;
1964 }
d5f66990 1965
eb47b800
JK		 1966 *pagep = page;
1967
2aadac08
JK		 1968 err = prepare_write_begin(sbi, page, pos, len,
1969 &blkaddr, &need_balance);
9ba69cf9 1970 if (err)
2aadac08 1971 goto fail;
9ba69cf9 1972
7f3037a5 1973 if (need_balance && has_not_enough_free_secs(sbi, 0, 0)) {
2a340760 1974 unlock_page(page);
2c4db1a6 1975 f2fs_balance_fs(sbi, true);
2a340760
JK		 1976 lock_page(page);
1977 if (page->mapping != mapping) {
1978 /* The page got truncated from under us */
1979 f2fs_put_page(page, 1);
1980 goto repeat;
1981 }
1982 }
1983
fec1d657 1984 f2fs_wait_on_page_writeback(page, DATA, false);
b3d208f9 1985
08b39fbd 1986 /* wait for GCed encrypted page writeback */
1958593e 1987 if (f2fs_encrypted_file(inode))
d4c759ee 1988 f2fs_wait_on_block_writeback(sbi, blkaddr);
08b39fbd 1989
649d7df2
JK		 1990 if (len == PAGE_SIZE || PageUptodate(page))
1991 return 0;
eb47b800 1992
746e2403
YH		 1993 if (!(pos & (PAGE_SIZE - 1)) && (pos + len) >= i_size_read(inode)) {
1994 zero_user_segment(page, len, PAGE_SIZE);
1995 return 0;
1996 }
1997
2aadac08 1998 if (blkaddr == NEW_ADDR) {
09cbfeaf 1999 zero_user_segment(page, 0, PAGE_SIZE);
649d7df2 2000 SetPageUptodate(page);
eb47b800 2001 } else {
13ba41e3
JK		 2002 err = f2fs_submit_page_read(inode, page, blkaddr);
2003 if (err)
78682f79 2004 goto fail;
d54c795b 2005
393ff91f 2006 lock_page(page);
6bacf52f 2007 if (unlikely(page->mapping != mapping)) {
afcb7ca0
JK		 2008 f2fs_put_page(page, 1);
2009 goto repeat;
eb47b800 2010 }
1563ac75
CY		 2011 if (unlikely(!PageUptodate(page))) {
2012 err = -EIO;
2013 goto fail;
4375a336 2014 }
eb47b800 2015 }
eb47b800 2016 return 0;
9ba69cf9 2017
3aab8f82 2018fail:
86531d6b 2019 f2fs_put_page(page, 1);
3aab8f82
CY		 2020 f2fs_write_failed(mapping, pos + len);
2021 return err;
eb47b800
JK		 2022}
2023
a1dd3c13
JK		 2024static int f2fs_write_end(struct file *file,
2025 struct address_space *mapping,
2026 loff_t pos, unsigned len, unsigned copied,
2027 struct page *page, void *fsdata)
2028{
2029 struct inode *inode = page->mapping->host;
2030
dfb2bf38
CY		 2031 trace_f2fs_write_end(inode, pos, len, copied);
2032
649d7df2
JK		 2033 /*
2034 * This should be come from len == PAGE_SIZE, and we expect copied
2035 * should be PAGE_SIZE. Otherwise, we treat it with zero copied and
2036 * let generic_perform_write() try to copy data again through copied=0.
2037 */
2038 if (!PageUptodate(page)) {
746e2403 2039 if (unlikely(copied != len))
649d7df2
JK		 2040 copied = 0;
2041 else
2042 SetPageUptodate(page);
2043 }
2044 if (!copied)
2045 goto unlock_out;
2046
34ba94ba 2047 set_page_dirty(page);
a1dd3c13 2048
fc9581c8
JK		 2049 if (pos + copied > i_size_read(inode))
2050 f2fs_i_size_write(inode, pos + copied);
649d7df2 2051unlock_out:
3024c9a1 2052 f2fs_put_page(page, 1);
d0239e1b 2053 f2fs_update_time(F2FS_I_SB(inode), REQ_TIME);
a1dd3c13
JK		 2054 return copied;
2055}
2056
6f673763
OS		 2057static int check_direct_IO(struct inode *inode, struct iov_iter *iter,
2058 loff_t offset)
944fcfc1
JK		 2059{
2060 unsigned blocksize_mask = inode->i_sb->s_blocksize - 1;
944fcfc1 2061
944fcfc1
JK		 2062 if (offset & blocksize_mask)
2063 return -EINVAL;
2064
5b46f25d
AV		 2065 if (iov_iter_alignment(iter) & blocksize_mask)
2066 return -EINVAL;
2067
944fcfc1
JK		 2068 return 0;
2069}
2070
c8b8e32d 2071static ssize_t f2fs_direct_IO(struct kiocb *iocb, struct iov_iter *iter)
eb47b800 2072{
b439b103 2073 struct address_space *mapping = iocb->ki_filp->f_mapping;
3aab8f82
CY		 2074 struct inode *inode = mapping->host;
2075 size_t count = iov_iter_count(iter);
c8b8e32d 2076 loff_t offset = iocb->ki_pos;
82e0a5aa 2077 int rw = iov_iter_rw(iter);
3aab8f82 2078 int err;
944fcfc1 2079
b439b103 2080 err = check_direct_IO(inode, iter, offset);
b9d777b8
JK		 2081 if (err)
2082 return err;
9ffe0fb5 2083
c040ff9d 2084 if (__force_buffered_io(inode, rw))
36abef4e 2085 return 0;
fcc85a4d 2086
5302fb00 2087 trace_f2fs_direct_IO_enter(inode, offset, count, rw);
70407fad 2088
82e0a5aa 2089 down_read(&F2FS_I(inode)->dio_rwsem[rw]);
c8b8e32d 2090 err = blockdev_direct_IO(iocb, inode, iter, get_data_block_dio);
82e0a5aa
CY		 2091 up_read(&F2FS_I(inode)->dio_rwsem[rw]);
2092
2093 if (rw == WRITE) {
b0af6d49
CY		 2094 if (err > 0) {
2095 f2fs_update_iostat(F2FS_I_SB(inode), APP_DIRECT_IO,
2096 err);
91942321 2097 set_inode_flag(inode, FI_UPDATE_WRITE);
b0af6d49 2098 } else if (err < 0) {
6bfc4919 2099 f2fs_write_failed(mapping, offset + count);
b0af6d49 2100 }
6bfc4919 2101 }
70407fad 2102
5302fb00 2103 trace_f2fs_direct_IO_exit(inode, offset, count, rw, err);
70407fad 2104
3aab8f82 2105 return err;
eb47b800
JK		 2106}
2107
487261f3
CY		 2108void f2fs_invalidate_page(struct page *page, unsigned int offset,
2109 unsigned int length)
eb47b800
JK		 2110{
2111 struct inode *inode = page->mapping->host;
487261f3 2112 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
a7ffdbe2 2113
487261f3 2114 if (inode->i_ino >= F2FS_ROOT_INO(sbi) &&
09cbfeaf 2115 (offset % PAGE_SIZE || length != PAGE_SIZE))
a7ffdbe2
JK		 2116 return;
2117
487261f3 2118 if (PageDirty(page)) {
933439c8 2119 if (inode->i_ino == F2FS_META_INO(sbi)) {
487261f3 2120 dec_page_count(sbi, F2FS_DIRTY_META);
933439c8 2121 } else if (inode->i_ino == F2FS_NODE_INO(sbi)) {
487261f3 2122 dec_page_count(sbi, F2FS_DIRTY_NODES);
933439c8 2123 } else {
487261f3 2124 inode_dec_dirty_pages(inode);
933439c8
CY		 2125 remove_dirty_inode(inode);
2126 }
487261f3 2127 }
decd36b6
CY		 2128
2129 /* This is atomic written page, keep Private */
2130 if (IS_ATOMIC_WRITTEN_PAGE(page))
8c242db9 2131 return drop_inmem_page(inode, page);
decd36b6 2132
23dc974e 2133 set_page_private(page, 0);
eb47b800
JK		 2134 ClearPagePrivate(page);
2135}
2136
487261f3 2137int f2fs_release_page(struct page *page, gfp_t wait)
eb47b800 2138{
f68daeeb
JK		 2139 /* If this is dirty page, keep PagePrivate */
2140 if (PageDirty(page))
2141 return 0;
2142
decd36b6
CY		 2143 /* This is atomic written page, keep Private */
2144 if (IS_ATOMIC_WRITTEN_PAGE(page))
2145 return 0;
2146
23dc974e 2147 set_page_private(page, 0);
eb47b800 2148 ClearPagePrivate(page);
c3850aa1 2149 return 1;
eb47b800
JK		 2150}
2151
fe76b796
JK		 2152/*
2153 * This was copied from __set_page_dirty_buffers which gives higher performance
2154 * in very high speed storages. (e.g., pmem)
2155 */
2156void f2fs_set_page_dirty_nobuffers(struct page *page)
2157{
2158 struct address_space *mapping = page->mapping;
2159 unsigned long flags;
2160
2161 if (unlikely(!mapping))
2162 return;
2163
2164 spin_lock(&mapping->private_lock);
2165 lock_page_memcg(page);
2166 SetPageDirty(page);
2167 spin_unlock(&mapping->private_lock);
2168
2169 spin_lock_irqsave(&mapping->tree_lock, flags);
2170 WARN_ON_ONCE(!PageUptodate(page));
2171 account_page_dirtied(page, mapping);
2172 radix_tree_tag_set(&mapping->page_tree,
2173 page_index(page), PAGECACHE_TAG_DIRTY);
2174 spin_unlock_irqrestore(&mapping->tree_lock, flags);
2175 unlock_page_memcg(page);
2176
2177 __mark_inode_dirty(mapping->host, I_DIRTY_PAGES);
2178 return;
2179}
2180
eb47b800
JK		 2181static int f2fs_set_data_page_dirty(struct page *page)
2182{
2183 struct address_space *mapping = page->mapping;
2184 struct inode *inode = mapping->host;
2185
26c6b887
JK		 2186 trace_f2fs_set_page_dirty(page, DATA);
2187
237c0790
JK		 2188 if (!PageUptodate(page))
2189 SetPageUptodate(page);
34ba94ba 2190
5fe45743 2191 if (f2fs_is_atomic_file(inode) && !f2fs_is_commit_atomic_write(inode)) {
decd36b6
CY		 2192 if (!IS_ATOMIC_WRITTEN_PAGE(page)) {
2193 register_inmem_page(inode, page);
2194 return 1;
2195 }
2196 /*
2197 * Previously, this page has been registered, we just
2198 * return here.
2199 */
2200 return 0;
34ba94ba
JK		 2201 }
2202
eb47b800 2203 if (!PageDirty(page)) {
fe76b796 2204 f2fs_set_page_dirty_nobuffers(page);
a7ffdbe2 2205 update_dirty_page(inode, page);
eb47b800
JK		 2206 return 1;
2207 }
2208 return 0;
2209}
2210
c01e54b7
JK		 2211static sector_t f2fs_bmap(struct address_space *mapping, sector_t block)
2212{
454ae7e5
CY		 2213 struct inode *inode = mapping->host;
2214
1d373a0e
JK		 2215 if (f2fs_has_inline_data(inode))
2216 return 0;
2217
2218 /* make sure allocating whole blocks */
2219 if (mapping_tagged(mapping, PAGECACHE_TAG_DIRTY))
2220 filemap_write_and_wait(mapping);
2221
e2b4e2bc 2222 return generic_block_bmap(mapping, block, get_data_block_bmap);
429511cd
CY		 2223}
2224
5b7a487c
WG		 2225#ifdef CONFIG_MIGRATION
2226#include <linux/migrate.h>
2227
2228int f2fs_migrate_page(struct address_space *mapping,
2229 struct page *newpage, struct page *page, enum migrate_mode mode)
2230{
2231 int rc, extra_count;
2232 struct f2fs_inode_info *fi = F2FS_I(mapping->host);
2233 bool atomic_written = IS_ATOMIC_WRITTEN_PAGE(page);
2234
2235 BUG_ON(PageWriteback(page));
2236
2237 /* migrating an atomic written page is safe with the inmem_lock hold */
ff1048e7
JK		 2238 if (atomic_written) {
2239 if (mode != MIGRATE_SYNC)
2240 return -EBUSY;
2241 if (!mutex_trylock(&fi->inmem_lock))
2242 return -EAGAIN;
2243 }
5b7a487c
WG		 2244
2245 /*
2246 * A reference is expected if PagePrivate set when move mapping,
2247 * however F2FS breaks this for maintaining dirty page counts when
2248 * truncating pages. So here adjusting the 'extra_count' make it work.
2249 */
2250 extra_count = (atomic_written ? 1 : 0) - page_has_private(page);
2251 rc = migrate_page_move_mapping(mapping, newpage,
2252 page, NULL, mode, extra_count);
2253 if (rc != MIGRATEPAGE_SUCCESS) {
2254 if (atomic_written)
2255 mutex_unlock(&fi->inmem_lock);
2256 return rc;
2257 }
2258
2259 if (atomic_written) {
2260 struct inmem_pages *cur;
2261 list_for_each_entry(cur, &fi->inmem_pages, list)
2262 if (cur->page == page) {
2263 cur->page = newpage;
2264 break;
2265 }
2266 mutex_unlock(&fi->inmem_lock);
2267 put_page(page);
2268 get_page(newpage);
2269 }
2270
2271 if (PagePrivate(page))
2272 SetPagePrivate(newpage);
2273 set_page_private(newpage, page_private(page));
2274
2916ecc0
JG		 2275 if (mode != MIGRATE_SYNC_NO_COPY)
2276 migrate_page_copy(newpage, page);
2277 else
2278 migrate_page_states(newpage, page);
5b7a487c
WG		 2279
2280 return MIGRATEPAGE_SUCCESS;
2281}
2282#endif
2283
eb47b800
JK		 2284const struct address_space_operations f2fs_dblock_aops = {
2285 .readpage = f2fs_read_data_page,
2286 .readpages = f2fs_read_data_pages,
2287 .writepage = f2fs_write_data_page,
2288 .writepages = f2fs_write_data_pages,
2289 .write_begin = f2fs_write_begin,
a1dd3c13 2290 .write_end = f2fs_write_end,
eb47b800 2291 .set_page_dirty = f2fs_set_data_page_dirty,
487261f3
CY		 2292 .invalidatepage = f2fs_invalidate_page,
2293 .releasepage = f2fs_release_page,
eb47b800 2294 .direct_IO = f2fs_direct_IO,
c01e54b7 2295 .bmap = f2fs_bmap,
5b7a487c
WG		 2296#ifdef CONFIG_MIGRATION
2297 .migratepage = f2fs_migrate_page,
2298#endif
eb47b800 2299};
ubuntu-bionic-kernel mirror