]> git.proxmox.com Git - mirror_ubuntu-artful-kernel.git/blame - fs/f2fs/segment.c
projects / mirror_ubuntu-artful-kernel.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
f2fs: fix leakage of inmemory atomic pages
[mirror_ubuntu-artful-kernel.git] / fs / f2fs / segment.c
CommitLineData
0a8165d7 1/*
351df4b2
JK		 2 * fs/f2fs/segment.c
3 *
4 * Copyright (c) 2012 Samsung Electronics Co., Ltd.
5 * http://www.samsung.com/
6 *
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License version 2 as
9 * published by the Free Software Foundation.
10 */
11#include <linux/fs.h>
12#include <linux/f2fs_fs.h>
13#include <linux/bio.h>
14#include <linux/blkdev.h>
690e4a3e 15#include <linux/prefetch.h>
6b4afdd7 16#include <linux/kthread.h>
74de593a 17#include <linux/swap.h>
60b99b48 18#include <linux/timer.h>
351df4b2
JK		 19
20#include "f2fs.h"
21#include "segment.h"
22#include "node.h"
9e4ded3f 23#include "trace.h"
6ec178da 24#include <trace/events/f2fs.h>
351df4b2 25
9a7f143a
CL		 26#define __reverse_ffz(x) __reverse_ffs(~(x))
27
7fd9e544 28static struct kmem_cache *discard_entry_slab;
184a5cd2 29static struct kmem_cache *sit_entry_set_slab;
88b88a66 30static struct kmem_cache *inmem_entry_slab;
7fd9e544 31
f96999c3
JK		 32static unsigned long __reverse_ulong(unsigned char *str)
33{
34 unsigned long tmp = 0;
35 int shift = 24, idx = 0;
36
37#if BITS_PER_LONG == 64
38 shift = 56;
39#endif
40 while (shift >= 0) {
41 tmp |= (unsigned long)str[idx++] << shift;
42 shift -= BITS_PER_BYTE;
43 }
44 return tmp;
45}
46
9a7f143a
CL		 47/*
48 * __reverse_ffs is copied from include/asm-generic/bitops/__ffs.h since
49 * MSB and LSB are reversed in a byte by f2fs_set_bit.
50 */
51static inline unsigned long __reverse_ffs(unsigned long word)
52{
53 int num = 0;
54
55#if BITS_PER_LONG == 64
f96999c3 56 if ((word & 0xffffffff00000000UL) == 0)
9a7f143a 57 num += 32;
f96999c3 58 else
9a7f143a 59 word >>= 32;
9a7f143a 60#endif
f96999c3 61 if ((word & 0xffff0000) == 0)
9a7f143a 62 num += 16;
f96999c3 63 else
9a7f143a 64 word >>= 16;
f96999c3
JK		 65
66 if ((word & 0xff00) == 0)
9a7f143a 67 num += 8;
f96999c3 68 else
9a7f143a 69 word >>= 8;
f96999c3 70
9a7f143a
CL		 71 if ((word & 0xf0) == 0)
72 num += 4;
73 else
74 word >>= 4;
f96999c3 75
9a7f143a
CL		 76 if ((word & 0xc) == 0)
77 num += 2;
78 else
79 word >>= 2;
f96999c3 80
9a7f143a
CL		 81 if ((word & 0x2) == 0)
82 num += 1;
83 return num;
84}
85
86/*
e1c42045 87 * __find_rev_next(_zero)_bit is copied from lib/find_next_bit.c because
9a7f143a
CL		 88 * f2fs_set_bit makes MSB and LSB reversed in a byte.
89 * Example:
f96999c3
JK		 90 * MSB <--> LSB
91 * f2fs_set_bit(0, bitmap) => 1000 0000
92 * f2fs_set_bit(7, bitmap) => 0000 0001
9a7f143a
CL		 93 */
94static unsigned long __find_rev_next_bit(const unsigned long *addr,
95 unsigned long size, unsigned long offset)
96{
97 const unsigned long *p = addr + BIT_WORD(offset);
98 unsigned long result = offset & ~(BITS_PER_LONG - 1);
99 unsigned long tmp;
9a7f143a
CL		 100
101 if (offset >= size)
102 return size;
103
104 size -= result;
105 offset %= BITS_PER_LONG;
106 if (!offset)
107 goto aligned;
108
f96999c3
JK		 109 tmp = __reverse_ulong((unsigned char *)p);
110 tmp &= ~0UL >> offset;
111
9a7f143a
CL		 112 if (size < BITS_PER_LONG)
113 goto found_first;
114 if (tmp)
115 goto found_middle;
116
117 size -= BITS_PER_LONG;
118 result += BITS_PER_LONG;
f96999c3 119 p++;
9a7f143a
CL		 120aligned:
121 while (size & ~(BITS_PER_LONG-1)) {
f96999c3 122 tmp = __reverse_ulong((unsigned char *)p);
9a7f143a
CL		 123 if (tmp)
124 goto found_middle;
125 result += BITS_PER_LONG;
126 size -= BITS_PER_LONG;
f96999c3 127 p++;
9a7f143a
CL		 128 }
129 if (!size)
130 return result;
f96999c3
JK		 131
132 tmp = __reverse_ulong((unsigned char *)p);
9a7f143a 133found_first:
f96999c3
JK		 134 tmp &= (~0UL << (BITS_PER_LONG - size));
135 if (!tmp) /* Are any bits set? */
9a7f143a
CL		 136 return result + size; /* Nope. */
137found_middle:
138 return result + __reverse_ffs(tmp);
139}
140
141static unsigned long __find_rev_next_zero_bit(const unsigned long *addr,
142 unsigned long size, unsigned long offset)
143{
144 const unsigned long *p = addr + BIT_WORD(offset);
145 unsigned long result = offset & ~(BITS_PER_LONG - 1);
146 unsigned long tmp;
9a7f143a
CL		 147
148 if (offset >= size)
149 return size;
150
151 size -= result;
152 offset %= BITS_PER_LONG;
153 if (!offset)
154 goto aligned;
155
f96999c3
JK		 156 tmp = __reverse_ulong((unsigned char *)p);
157 tmp |= ~((~0UL << offset) >> offset);
158
9a7f143a
CL		 159 if (size < BITS_PER_LONG)
160 goto found_first;
f96999c3 161 if (tmp != ~0UL)
9a7f143a
CL		 162 goto found_middle;
163
164 size -= BITS_PER_LONG;
165 result += BITS_PER_LONG;
f96999c3 166 p++;
9a7f143a
CL		 167aligned:
168 while (size & ~(BITS_PER_LONG - 1)) {
f96999c3
JK		 169 tmp = __reverse_ulong((unsigned char *)p);
170 if (tmp != ~0UL)
9a7f143a
CL		 171 goto found_middle;
172 result += BITS_PER_LONG;
173 size -= BITS_PER_LONG;
f96999c3 174 p++;
9a7f143a
CL		 175 }
176 if (!size)
177 return result;
9a7f143a 178
f96999c3 179 tmp = __reverse_ulong((unsigned char *)p);
9a7f143a 180found_first:
f96999c3
JK		 181 tmp |= ~(~0UL << (BITS_PER_LONG - size));
182 if (tmp == ~0UL) /* Are any bits zero? */
9a7f143a
CL		 183 return result + size; /* Nope. */
184found_middle:
185 return result + __reverse_ffz(tmp);
186}
187
88b88a66
JK		 188void register_inmem_page(struct inode *inode, struct page *page)
189{
190 struct f2fs_inode_info *fi = F2FS_I(inode);
191 struct inmem_pages *new;
9be32d72 192
9e4ded3f 193 f2fs_trace_pid(page);
0722b101 194
decd36b6
CY		 195 set_page_private(page, (unsigned long)ATOMIC_WRITTEN_PAGE);
196 SetPagePrivate(page);
197
88b88a66
JK		 198 new = f2fs_kmem_cache_alloc(inmem_entry_slab, GFP_NOFS);
199
200 /* add atomic page indices to the list */
201 new->page = page;
202 INIT_LIST_HEAD(&new->list);
decd36b6 203
88b88a66
JK		 204 /* increase reference count with clean state */
205 mutex_lock(&fi->inmem_lock);
206 get_page(page);
207 list_add_tail(&new->list, &fi->inmem_pages);
8dcf2ff7 208 inc_page_count(F2FS_I_SB(inode), F2FS_INMEM_PAGES);
88b88a66 209 mutex_unlock(&fi->inmem_lock);
8ce67cb0
JK		 210
211 trace_f2fs_register_inmem_page(page, INMEM);
88b88a66
JK		 212}
213
edb27dee 214int commit_inmem_pages(struct inode *inode, bool abort)
88b88a66
JK		 215{
216 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
217 struct f2fs_inode_info *fi = F2FS_I(inode);
218 struct inmem_pages *cur, *tmp;
219 bool submit_bio = false;
220 struct f2fs_io_info fio = {
05ca3632 221 .sbi = sbi,
88b88a66 222 .type = DATA,
1e84371f 223 .rw = WRITE_SYNC | REQ_PRIO,
4375a336 224 .encrypted_page = NULL,
88b88a66 225 };
edb27dee 226 int err = 0;
88b88a66 227
0341845e
JK		 228 /*
229 * The abort is true only when f2fs_evict_inode is called.
230 * Basically, the f2fs_evict_inode doesn't produce any data writes, so
231 * that we don't need to call f2fs_balance_fs.
232 * Otherwise, f2fs_gc in f2fs_balance_fs can wait forever until this
233 * inode becomes free by iget_locked in f2fs_iget.
234 */
70c640b1 235 if (!abort) {
0341845e 236 f2fs_balance_fs(sbi);
70c640b1
JK		 237 f2fs_lock_op(sbi);
238 }
88b88a66
JK		 239
240 mutex_lock(&fi->inmem_lock);
241 list_for_each_entry_safe(cur, tmp, &fi->inmem_pages, list) {
decd36b6 242 lock_page(cur->page);
70c640b1 243 if (!abort) {
70c640b1 244 if (cur->page->mapping == inode->i_mapping) {
6282adbf 245 set_page_dirty(cur->page);
70c640b1
JK		 246 f2fs_wait_on_page_writeback(cur->page, DATA);
247 if (clear_page_dirty_for_io(cur->page))
248 inode_dec_dirty_pages(inode);
8ce67cb0 249 trace_f2fs_commit_inmem_page(cur->page, INMEM);
05ca3632 250 fio.page = cur->page;
edb27dee 251 err = do_write_data_page(&fio);
70c640b1 252 submit_bio = true;
edb27dee
JK		 253 if (err) {
254 unlock_page(cur->page);
255 break;
256 }
70c640b1 257 }
70c640b1 258 } else {
8ce67cb0 259 trace_f2fs_commit_inmem_page(cur->page, INMEM_DROP);
88b88a66 260 }
decd36b6
CY		 261 set_page_private(cur->page, 0);
262 ClearPagePrivate(cur->page);
263 f2fs_put_page(cur->page, 1);
264
88b88a66
JK		 265 list_del(&cur->list);
266 kmem_cache_free(inmem_entry_slab, cur);
8dcf2ff7 267 dec_page_count(F2FS_I_SB(inode), F2FS_INMEM_PAGES);
88b88a66 268 }
88b88a66
JK		 269 mutex_unlock(&fi->inmem_lock);
270
70c640b1
JK		 271 if (!abort) {
272 f2fs_unlock_op(sbi);
273 if (submit_bio)
274 f2fs_submit_merged_bio(sbi, DATA, WRITE);
275 }
edb27dee 276 return err;
88b88a66
JK		 277}
278
0a8165d7 279/*
351df4b2
JK		 280 * This function balances dirty node and dentry pages.
281 * In addition, it controls garbage collection.
282 */
283void f2fs_balance_fs(struct f2fs_sb_info *sbi)
284{
351df4b2 285 /*
029cd28c
JK		 286 * We should do GC or end up with checkpoint, if there are so many dirty
287 * dir/node pages without enough free segments.
351df4b2 288 */
43727527 289 if (has_not_enough_free_secs(sbi, 0)) {
351df4b2 290 mutex_lock(&sbi->gc_mutex);
d530d4d8 291 f2fs_gc(sbi, false);
351df4b2
JK		 292 }
293}
294
4660f9c0
JK		 295void f2fs_balance_fs_bg(struct f2fs_sb_info *sbi)
296{
1dcc336b 297 /* try to shrink extent cache when there is no enough memory */
554df79e
JK		 298 if (!available_free_memory(sbi, EXTENT_CACHE))
299 f2fs_shrink_extent_tree(sbi, EXTENT_CACHE_SHRINK_NUMBER);
1dcc336b 300
1b38dc8e
JK		 301 /* check the # of cached NAT entries */
302 if (!available_free_memory(sbi, NAT_ENTRIES))
303 try_to_free_nats(sbi, NAT_ENTRY_PER_BLOCK);
304
31696580
CY		 305 if (!available_free_memory(sbi, FREE_NIDS))
306 try_to_free_nids(sbi, NAT_ENTRY_PER_BLOCK * FREE_NID_PAGES);
307
1b38dc8e
JK		 308 /* checkpoint is the only way to shrink partial cached entries */
309 if (!available_free_memory(sbi, NAT_ENTRIES) ||
e5e7ea3c 310 excess_prefree_segs(sbi) ||
60b99b48
JK		 311 !available_free_memory(sbi, INO_ENTRIES) ||
312 jiffies > sbi->cp_expires)
4660f9c0
JK		 313 f2fs_sync_fs(sbi->sb, true);
314}
315
2163d198 316static int issue_flush_thread(void *data)
6b4afdd7
JK		 317{
318 struct f2fs_sb_info *sbi = data;
a688b9d9
GZ		 319 struct flush_cmd_control *fcc = SM_I(sbi)->cmd_control_info;
320 wait_queue_head_t *q = &fcc->flush_wait_queue;
6b4afdd7
JK		 321repeat:
322 if (kthread_should_stop())
323 return 0;
324
721bd4d5 325 if (!llist_empty(&fcc->issue_list)) {
740432f8 326 struct bio *bio;
6b4afdd7
JK		 327 struct flush_cmd *cmd, *next;
328 int ret;
329
740432f8
JK		 330 bio = f2fs_bio_alloc(0);
331
721bd4d5
GZ		 332 fcc->dispatch_list = llist_del_all(&fcc->issue_list);
333 fcc->dispatch_list = llist_reverse_order(fcc->dispatch_list);
334
6b4afdd7
JK		 335 bio->bi_bdev = sbi->sb->s_bdev;
336 ret = submit_bio_wait(WRITE_FLUSH, bio);
337
721bd4d5
GZ		 338 llist_for_each_entry_safe(cmd, next,
339 fcc->dispatch_list, llnode) {
6b4afdd7 340 cmd->ret = ret;
6b4afdd7
JK		 341 complete(&cmd->wait);
342 }
a4ed23f2 343 bio_put(bio);
a688b9d9 344 fcc->dispatch_list = NULL;
6b4afdd7
JK		 345 }
346
a688b9d9 347 wait_event_interruptible(*q,
721bd4d5 348 kthread_should_stop() || !llist_empty(&fcc->issue_list));
6b4afdd7
JK		 349 goto repeat;
350}
351
352int f2fs_issue_flush(struct f2fs_sb_info *sbi)
353{
a688b9d9 354 struct flush_cmd_control *fcc = SM_I(sbi)->cmd_control_info;
adf8d90b 355 struct flush_cmd cmd;
6b4afdd7 356
24a9ee0f
JK		 357 trace_f2fs_issue_flush(sbi->sb, test_opt(sbi, NOBARRIER),
358 test_opt(sbi, FLUSH_MERGE));
359
0f7b2abd
JK		 360 if (test_opt(sbi, NOBARRIER))
361 return 0;
362
740432f8
JK		 363 if (!test_opt(sbi, FLUSH_MERGE)) {
364 struct bio *bio = f2fs_bio_alloc(0);
365 int ret;
366
367 bio->bi_bdev = sbi->sb->s_bdev;
368 ret = submit_bio_wait(WRITE_FLUSH, bio);
369 bio_put(bio);
370 return ret;
371 }
6b4afdd7 372
adf8d90b 373 init_completion(&cmd.wait);
6b4afdd7 374
721bd4d5 375 llist_add(&cmd.llnode, &fcc->issue_list);
6b4afdd7 376
a688b9d9
GZ		 377 if (!fcc->dispatch_list)
378 wake_up(&fcc->flush_wait_queue);
6b4afdd7 379
adf8d90b
CY		 380 wait_for_completion(&cmd.wait);
381
382 return cmd.ret;
6b4afdd7
JK		 383}
384
2163d198
GZ		 385int create_flush_cmd_control(struct f2fs_sb_info *sbi)
386{
387 dev_t dev = sbi->sb->s_bdev->bd_dev;
388 struct flush_cmd_control *fcc;
389 int err = 0;
390
391 fcc = kzalloc(sizeof(struct flush_cmd_control), GFP_KERNEL);
392 if (!fcc)
393 return -ENOMEM;
2163d198 394 init_waitqueue_head(&fcc->flush_wait_queue);
721bd4d5 395 init_llist_head(&fcc->issue_list);
6b2920a5 396 SM_I(sbi)->cmd_control_info = fcc;
2163d198
GZ		 397 fcc->f2fs_issue_flush = kthread_run(issue_flush_thread, sbi,
398 "f2fs_flush-%u:%u", MAJOR(dev), MINOR(dev));
399 if (IS_ERR(fcc->f2fs_issue_flush)) {
400 err = PTR_ERR(fcc->f2fs_issue_flush);
401 kfree(fcc);
6b2920a5 402 SM_I(sbi)->cmd_control_info = NULL;
2163d198
GZ		 403 return err;
404 }
2163d198
GZ		 405
406 return err;
407}
408
409void destroy_flush_cmd_control(struct f2fs_sb_info *sbi)
410{
6b2920a5 411 struct flush_cmd_control *fcc = SM_I(sbi)->cmd_control_info;
2163d198
GZ		 412
413 if (fcc && fcc->f2fs_issue_flush)
414 kthread_stop(fcc->f2fs_issue_flush);
415 kfree(fcc);
6b2920a5 416 SM_I(sbi)->cmd_control_info = NULL;
2163d198
GZ		 417}
418
351df4b2
JK		 419static void __locate_dirty_segment(struct f2fs_sb_info *sbi, unsigned int segno,
420 enum dirty_type dirty_type)
421{
422 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
423
424 /* need not be added */
425 if (IS_CURSEG(sbi, segno))
426 return;
427
428 if (!test_and_set_bit(segno, dirty_i->dirty_segmap[dirty_type]))
429 dirty_i->nr_dirty[dirty_type]++;
430
431 if (dirty_type == DIRTY) {
432 struct seg_entry *sentry = get_seg_entry(sbi, segno);
4625d6aa 433 enum dirty_type t = sentry->type;
b2f2c390 434
ec325b52
JK		 435 if (unlikely(t >= DIRTY)) {
436 f2fs_bug_on(sbi, 1);
437 return;
438 }
4625d6aa
CL		 439 if (!test_and_set_bit(segno, dirty_i->dirty_segmap[t]))
440 dirty_i->nr_dirty[t]++;
351df4b2
JK		 441 }
442}
443
444static void __remove_dirty_segment(struct f2fs_sb_info *sbi, unsigned int segno,
445 enum dirty_type dirty_type)
446{
447 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
448
449 if (test_and_clear_bit(segno, dirty_i->dirty_segmap[dirty_type]))
450 dirty_i->nr_dirty[dirty_type]--;
451
452 if (dirty_type == DIRTY) {
4625d6aa
CL		 453 struct seg_entry *sentry = get_seg_entry(sbi, segno);
454 enum dirty_type t = sentry->type;
455
456 if (test_and_clear_bit(segno, dirty_i->dirty_segmap[t]))
457 dirty_i->nr_dirty[t]--;
b2f2c390 458
5ec4e49f
JK		 459 if (get_valid_blocks(sbi, segno, sbi->segs_per_sec) == 0)
460 clear_bit(GET_SECNO(sbi, segno),
461 dirty_i->victim_secmap);
351df4b2
JK		 462 }
463}
464
0a8165d7 465/*
351df4b2
JK		 466 * Should not occur error such as -ENOMEM.
467 * Adding dirty entry into seglist is not critical operation.
468 * If a given segment is one of current working segments, it won't be added.
469 */
8d8451af 470static void locate_dirty_segment(struct f2fs_sb_info *sbi, unsigned int segno)
351df4b2
JK		 471{
472 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
473 unsigned short valid_blocks;
474
475 if (segno == NULL_SEGNO || IS_CURSEG(sbi, segno))
476 return;
477
478 mutex_lock(&dirty_i->seglist_lock);
479
480 valid_blocks = get_valid_blocks(sbi, segno, 0);
481
482 if (valid_blocks == 0) {
483 __locate_dirty_segment(sbi, segno, PRE);
484 __remove_dirty_segment(sbi, segno, DIRTY);
485 } else if (valid_blocks < sbi->blocks_per_seg) {
486 __locate_dirty_segment(sbi, segno, DIRTY);
487 } else {
488 /* Recovery routine with SSR needs this */
489 __remove_dirty_segment(sbi, segno, DIRTY);
490 }
491
492 mutex_unlock(&dirty_i->seglist_lock);
351df4b2
JK		 493}
494
1e87a78d 495static int f2fs_issue_discard(struct f2fs_sb_info *sbi,
37208879
JK		 496 block_t blkstart, block_t blklen)
497{
55cf9cb6
CY		 498 sector_t start = SECTOR_FROM_BLOCK(blkstart);
499 sector_t len = SECTOR_FROM_BLOCK(blklen);
a66cdd98
JK		 500 struct seg_entry *se;
501 unsigned int offset;
502 block_t i;
503
504 for (i = blkstart; i < blkstart + blklen; i++) {
505 se = get_seg_entry(sbi, GET_SEGNO(sbi, i));
506 offset = GET_BLKOFF_FROM_SEG0(sbi, i);
507
508 if (!f2fs_test_and_set_bit(offset, se->discard_map))
509 sbi->discard_blks--;
510 }
1661d07c 511 trace_f2fs_issue_discard(sbi->sb, blkstart, blklen);
1e87a78d
JK		 512 return blkdev_issue_discard(sbi->sb->s_bdev, start, len, GFP_NOFS, 0);
513}
514
e90c2d28 515bool discard_next_dnode(struct f2fs_sb_info *sbi, block_t blkaddr)
1e87a78d 516{
40a02be1
JK		 517 int err = -ENOTSUPP;
518
519 if (test_opt(sbi, DISCARD)) {
520 struct seg_entry *se = get_seg_entry(sbi,
521 GET_SEGNO(sbi, blkaddr));
522 unsigned int offset = GET_BLKOFF_FROM_SEG0(sbi, blkaddr);
523
524 if (f2fs_test_bit(offset, se->discard_map))
e90c2d28 525 return false;
40a02be1
JK		 526
527 err = f2fs_issue_discard(sbi, blkaddr, 1);
528 }
529
e90c2d28 530 if (err) {
381722d2 531 update_meta_page(sbi, NULL, blkaddr);
e90c2d28
CY		 532 return true;
533 }
534 return false;
37208879
JK		 535}
536
adf4983b 537static void __add_discard_entry(struct f2fs_sb_info *sbi,
a66cdd98
JK		 538 struct cp_control *cpc, struct seg_entry *se,
539 unsigned int start, unsigned int end)
b2955550
JK		 540{
541 struct list_head *head = &SM_I(sbi)->discard_list;
adf4983b
JK		 542 struct discard_entry *new, *last;
543
544 if (!list_empty(head)) {
545 last = list_last_entry(head, struct discard_entry, list);
546 if (START_BLOCK(sbi, cpc->trim_start) + start ==
547 last->blkaddr + last->len) {
548 last->len += end - start;
549 goto done;
550 }
551 }
552
553 new = f2fs_kmem_cache_alloc(discard_entry_slab, GFP_NOFS);
554 INIT_LIST_HEAD(&new->list);
555 new->blkaddr = START_BLOCK(sbi, cpc->trim_start) + start;
556 new->len = end - start;
557 list_add_tail(&new->list, head);
558done:
559 SM_I(sbi)->nr_discards += end - start;
adf4983b
JK		 560}
561
562static void add_discard_addrs(struct f2fs_sb_info *sbi, struct cp_control *cpc)
563{
b2955550
JK		 564 int entries = SIT_VBLOCK_MAP_SIZE / sizeof(unsigned long);
565 int max_blocks = sbi->blocks_per_seg;
4b2fecc8 566 struct seg_entry *se = get_seg_entry(sbi, cpc->trim_start);
b2955550
JK		 567 unsigned long *cur_map = (unsigned long *)se->cur_valid_map;
568 unsigned long *ckpt_map = (unsigned long *)se->ckpt_valid_map;
a66cdd98 569 unsigned long *discard_map = (unsigned long *)se->discard_map;
60a3b782 570 unsigned long *dmap = SIT_I(sbi)->tmp_map;
b2955550 571 unsigned int start = 0, end = -1;
4b2fecc8 572 bool force = (cpc->reason == CP_DISCARD);
b2955550
JK		 573 int i;
574
a66cdd98 575 if (se->valid_blocks == max_blocks)
b2955550
JK		 576 return;
577
a66cdd98
JK		 578 if (!force) {
579 if (!test_opt(sbi, DISCARD) || !se->valid_blocks ||
912a83b5
DC		 580 SM_I(sbi)->nr_discards >= SM_I(sbi)->max_discards)
581 return;
4b2fecc8
JK		 582 }
583
b2955550
JK		 584 /* SIT_VBLOCK_MAP_SIZE should be multiple of sizeof(unsigned long) */
585 for (i = 0; i < entries; i++)
a66cdd98 586 dmap[i] = force ? ~ckpt_map[i] & ~discard_map[i] :
d7bc2484 587 (cur_map[i] ^ ckpt_map[i]) & ckpt_map[i];
b2955550 588
4b2fecc8 589 while (force || SM_I(sbi)->nr_discards <= SM_I(sbi)->max_discards) {
b2955550
JK		 590 start = __find_rev_next_bit(dmap, max_blocks, end + 1);
591 if (start >= max_blocks)
592 break;
593
594 end = __find_rev_next_zero_bit(dmap, max_blocks, start + 1);
a66cdd98 595 __add_discard_entry(sbi, cpc, se, start, end);
b2955550
JK		 596 }
597}
598
4b2fecc8
JK		 599void release_discard_addrs(struct f2fs_sb_info *sbi)
600{
601 struct list_head *head = &(SM_I(sbi)->discard_list);
602 struct discard_entry *entry, *this;
603
604 /* drop caches */
605 list_for_each_entry_safe(entry, this, head, list) {
606 list_del(&entry->list);
607 kmem_cache_free(discard_entry_slab, entry);
608 }
609}
610
0a8165d7 611/*
351df4b2
JK		 612 * Should call clear_prefree_segments after checkpoint is done.
613 */
614static void set_prefree_as_free_segments(struct f2fs_sb_info *sbi)
615{
616 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
b65ee148 617 unsigned int segno;
351df4b2
JK		 618
619 mutex_lock(&dirty_i->seglist_lock);
7cd8558b 620 for_each_set_bit(segno, dirty_i->dirty_segmap[PRE], MAIN_SEGS(sbi))
351df4b2 621 __set_test_and_free(sbi, segno);
351df4b2
JK		 622 mutex_unlock(&dirty_i->seglist_lock);
623}
624
836b5a63 625void clear_prefree_segments(struct f2fs_sb_info *sbi, struct cp_control *cpc)
351df4b2 626{
b2955550 627 struct list_head *head = &(SM_I(sbi)->discard_list);
2d7b822a 628 struct discard_entry *entry, *this;
351df4b2 629 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
29e59c14 630 unsigned long *prefree_map = dirty_i->dirty_segmap[PRE];
29e59c14 631 unsigned int start = 0, end = -1;
351df4b2
JK		 632
633 mutex_lock(&dirty_i->seglist_lock);
29e59c14 634
351df4b2 635 while (1) {
29e59c14 636 int i;
7cd8558b
JK		 637 start = find_next_bit(prefree_map, MAIN_SEGS(sbi), end + 1);
638 if (start >= MAIN_SEGS(sbi))
351df4b2 639 break;
7cd8558b
JK		 640 end = find_next_zero_bit(prefree_map, MAIN_SEGS(sbi),
641 start + 1);
29e59c14
CL		 642
643 for (i = start; i < end; i++)
644 clear_bit(i, prefree_map);
645
646 dirty_i->nr_dirty[PRE] -= end - start;
647
648 if (!test_opt(sbi, DISCARD))
649 continue;
351df4b2 650
37208879
JK		 651 f2fs_issue_discard(sbi, START_BLOCK(sbi, start),
652 (end - start) << sbi->log_blocks_per_seg);
351df4b2
JK		 653 }
654 mutex_unlock(&dirty_i->seglist_lock);
b2955550
JK		 655
656 /* send small discards */
2d7b822a 657 list_for_each_entry_safe(entry, this, head, list) {
836b5a63
JK		 658 if (cpc->reason == CP_DISCARD && entry->len < cpc->trim_minlen)
659 goto skip;
37208879 660 f2fs_issue_discard(sbi, entry->blkaddr, entry->len);
f56aa1c5 661 cpc->trimmed += entry->len;
836b5a63 662skip:
b2955550
JK		 663 list_del(&entry->list);
664 SM_I(sbi)->nr_discards -= entry->len;
665 kmem_cache_free(discard_entry_slab, entry);
666 }
351df4b2
JK		 667}
668
184a5cd2 669static bool __mark_sit_entry_dirty(struct f2fs_sb_info *sbi, unsigned int segno)
351df4b2
JK		 670{
671 struct sit_info *sit_i = SIT_I(sbi);
184a5cd2
CY		 672
673 if (!__test_and_set_bit(segno, sit_i->dirty_sentries_bitmap)) {
351df4b2 674 sit_i->dirty_sentries++;
184a5cd2
CY		 675 return false;
676 }
677
678 return true;
351df4b2
JK		 679}
680
681static void __set_sit_entry_type(struct f2fs_sb_info *sbi, int type,
682 unsigned int segno, int modified)
683{
684 struct seg_entry *se = get_seg_entry(sbi, segno);
685 se->type = type;
686 if (modified)
687 __mark_sit_entry_dirty(sbi, segno);
688}
689
690static void update_sit_entry(struct f2fs_sb_info *sbi, block_t blkaddr, int del)
691{
692 struct seg_entry *se;
693 unsigned int segno, offset;
694 long int new_vblocks;
695
696 segno = GET_SEGNO(sbi, blkaddr);
697
698 se = get_seg_entry(sbi, segno);
699 new_vblocks = se->valid_blocks + del;
491c0854 700 offset = GET_BLKOFF_FROM_SEG0(sbi, blkaddr);
351df4b2 701
9850cf4a 702 f2fs_bug_on(sbi, (new_vblocks >> (sizeof(unsigned short) << 3) ||
351df4b2
JK		 703 (new_vblocks > sbi->blocks_per_seg)));
704
705 se->valid_blocks = new_vblocks;
706 se->mtime = get_mtime(sbi);
707 SIT_I(sbi)->max_mtime = se->mtime;
708
709 /* Update valid block bitmap */
710 if (del > 0) {
52aca074 711 if (f2fs_test_and_set_bit(offset, se->cur_valid_map))
05796763 712 f2fs_bug_on(sbi, 1);
a66cdd98
JK		 713 if (!f2fs_test_and_set_bit(offset, se->discard_map))
714 sbi->discard_blks--;
351df4b2 715 } else {
52aca074 716 if (!f2fs_test_and_clear_bit(offset, se->cur_valid_map))
05796763 717 f2fs_bug_on(sbi, 1);
a66cdd98
JK		 718 if (f2fs_test_and_clear_bit(offset, se->discard_map))
719 sbi->discard_blks++;
351df4b2
JK		 720 }
721 if (!f2fs_test_bit(offset, se->ckpt_valid_map))
722 se->ckpt_valid_blocks += del;
723
724 __mark_sit_entry_dirty(sbi, segno);
725
726 /* update total number of valid blocks to be written in ckpt area */
727 SIT_I(sbi)->written_valid_blocks += del;
728
729 if (sbi->segs_per_sec > 1)
730 get_sec_entry(sbi, segno)->valid_blocks += del;
731}
732
5e443818 733void refresh_sit_entry(struct f2fs_sb_info *sbi, block_t old, block_t new)
351df4b2 734{
5e443818
JK		 735 update_sit_entry(sbi, new, 1);
736 if (GET_SEGNO(sbi, old) != NULL_SEGNO)
737 update_sit_entry(sbi, old, -1);
738
739 locate_dirty_segment(sbi, GET_SEGNO(sbi, old));
740 locate_dirty_segment(sbi, GET_SEGNO(sbi, new));
351df4b2
JK		 741}
742
743void invalidate_blocks(struct f2fs_sb_info *sbi, block_t addr)
744{
745 unsigned int segno = GET_SEGNO(sbi, addr);
746 struct sit_info *sit_i = SIT_I(sbi);
747
9850cf4a 748 f2fs_bug_on(sbi, addr == NULL_ADDR);
351df4b2
JK		 749 if (addr == NEW_ADDR)
750 return;
751
752 /* add it into sit main buffer */
753 mutex_lock(&sit_i->sentry_lock);
754
755 update_sit_entry(sbi, addr, -1);
756
757 /* add it into dirty seglist */
758 locate_dirty_segment(sbi, segno);
759
760 mutex_unlock(&sit_i->sentry_lock);
761}
762
6e2c64ad
JK		 763bool is_checkpointed_data(struct f2fs_sb_info *sbi, block_t blkaddr)
764{
765 struct sit_info *sit_i = SIT_I(sbi);
766 unsigned int segno, offset;
767 struct seg_entry *se;
768 bool is_cp = false;
769
770 if (blkaddr == NEW_ADDR || blkaddr == NULL_ADDR)
771 return true;
772
773 mutex_lock(&sit_i->sentry_lock);
774
775 segno = GET_SEGNO(sbi, blkaddr);
776 se = get_seg_entry(sbi, segno);
777 offset = GET_BLKOFF_FROM_SEG0(sbi, blkaddr);
778
779 if (f2fs_test_bit(offset, se->ckpt_valid_map))
780 is_cp = true;
781
782 mutex_unlock(&sit_i->sentry_lock);
783
784 return is_cp;
785}
786
0a8165d7 787/*
351df4b2
JK		 788 * This function should be resided under the curseg_mutex lock
789 */
790static void __add_sum_entry(struct f2fs_sb_info *sbi, int type,
e79efe3b 791 struct f2fs_summary *sum)
351df4b2
JK		 792{
793 struct curseg_info *curseg = CURSEG_I(sbi, type);
794 void *addr = curseg->sum_blk;
e79efe3b 795 addr += curseg->next_blkoff * sizeof(struct f2fs_summary);
351df4b2 796 memcpy(addr, sum, sizeof(struct f2fs_summary));
351df4b2
JK		 797}
798
0a8165d7 799/*
351df4b2
JK		 800 * Calculate the number of current summary pages for writing
801 */
3fa06d7b 802int npages_for_summary_flush(struct f2fs_sb_info *sbi, bool for_ra)
351df4b2 803{
351df4b2 804 int valid_sum_count = 0;
9a47938b 805 int i, sum_in_page;
351df4b2
JK		 806
807 for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++) {
808 if (sbi->ckpt->alloc_type[i] == SSR)
809 valid_sum_count += sbi->blocks_per_seg;
3fa06d7b
CY		 810 else {
811 if (for_ra)
812 valid_sum_count += le16_to_cpu(
813 F2FS_CKPT(sbi)->cur_data_blkoff[i]);
814 else
815 valid_sum_count += curseg_blkoff(sbi, i);
816 }
351df4b2
JK		 817 }
818
9a47938b
FL		 819 sum_in_page = (PAGE_CACHE_SIZE - 2 * SUM_JOURNAL_SIZE -
820 SUM_FOOTER_SIZE) / SUMMARY_SIZE;
821 if (valid_sum_count <= sum_in_page)
351df4b2 822 return 1;
9a47938b
FL		 823 else if ((valid_sum_count - sum_in_page) <=
824 (PAGE_CACHE_SIZE - SUM_FOOTER_SIZE) / SUMMARY_SIZE)
351df4b2
JK		 825 return 2;
826 return 3;
827}
828
0a8165d7 829/*
351df4b2
JK		 830 * Caller should put this summary page
831 */
832struct page *get_sum_page(struct f2fs_sb_info *sbi, unsigned int segno)
833{
834 return get_meta_page(sbi, GET_SUM_BLOCK(sbi, segno));
835}
836
381722d2 837void update_meta_page(struct f2fs_sb_info *sbi, void *src, block_t blk_addr)
351df4b2
JK		 838{
839 struct page *page = grab_meta_page(sbi, blk_addr);
381722d2
CY		 840 void *dst = page_address(page);
841
842 if (src)
843 memcpy(dst, src, PAGE_CACHE_SIZE);
844 else
845 memset(dst, 0, PAGE_CACHE_SIZE);
351df4b2
JK		 846 set_page_dirty(page);
847 f2fs_put_page(page, 1);
848}
849
381722d2
CY		 850static void write_sum_page(struct f2fs_sb_info *sbi,
851 struct f2fs_summary_block *sum_blk, block_t blk_addr)
852{
853 update_meta_page(sbi, (void *)sum_blk, blk_addr);
854}
855
60374688
JK		 856static int is_next_segment_free(struct f2fs_sb_info *sbi, int type)
857{
858 struct curseg_info *curseg = CURSEG_I(sbi, type);
81fb5e87 859 unsigned int segno = curseg->segno + 1;
60374688
JK		 860 struct free_segmap_info *free_i = FREE_I(sbi);
861
7cd8558b 862 if (segno < MAIN_SEGS(sbi) && segno % sbi->segs_per_sec)
81fb5e87 863 return !test_bit(segno, free_i->free_segmap);
60374688
JK		 864 return 0;
865}
866
0a8165d7 867/*
351df4b2
JK		 868 * Find a new segment from the free segments bitmap to right order
869 * This function should be returned with success, otherwise BUG
870 */
871static void get_new_segment(struct f2fs_sb_info *sbi,
872 unsigned int *newseg, bool new_sec, int dir)
873{
874 struct free_segmap_info *free_i = FREE_I(sbi);
351df4b2 875 unsigned int segno, secno, zoneno;
7cd8558b 876 unsigned int total_zones = MAIN_SECS(sbi) / sbi->secs_per_zone;
351df4b2
JK		 877 unsigned int hint = *newseg / sbi->segs_per_sec;
878 unsigned int old_zoneno = GET_ZONENO_FROM_SEGNO(sbi, *newseg);
879 unsigned int left_start = hint;
880 bool init = true;
881 int go_left = 0;
882 int i;
883
1a118ccf 884 spin_lock(&free_i->segmap_lock);
351df4b2
JK		 885
886 if (!new_sec && ((*newseg + 1) % sbi->segs_per_sec)) {
887 segno = find_next_zero_bit(free_i->free_segmap,
7cd8558b 888 MAIN_SEGS(sbi), *newseg + 1);
33afa7fd
JK		 889 if (segno - *newseg < sbi->segs_per_sec -
890 (*newseg % sbi->segs_per_sec))
351df4b2
JK		 891 goto got_it;
892 }
893find_other_zone:
7cd8558b
JK		 894 secno = find_next_zero_bit(free_i->free_secmap, MAIN_SECS(sbi), hint);
895 if (secno >= MAIN_SECS(sbi)) {
351df4b2
JK		 896 if (dir == ALLOC_RIGHT) {
897 secno = find_next_zero_bit(free_i->free_secmap,
7cd8558b
JK		 898 MAIN_SECS(sbi), 0);
899 f2fs_bug_on(sbi, secno >= MAIN_SECS(sbi));
351df4b2
JK		 900 } else {
901 go_left = 1;
902 left_start = hint - 1;
903 }
904 }
905 if (go_left == 0)
906 goto skip_left;
907
908 while (test_bit(left_start, free_i->free_secmap)) {
909 if (left_start > 0) {
910 left_start--;
911 continue;
912 }
913 left_start = find_next_zero_bit(free_i->free_secmap,
7cd8558b
JK		 914 MAIN_SECS(sbi), 0);
915 f2fs_bug_on(sbi, left_start >= MAIN_SECS(sbi));
351df4b2
JK		 916 break;
917 }
918 secno = left_start;
919skip_left:
920 hint = secno;
921 segno = secno * sbi->segs_per_sec;
922 zoneno = secno / sbi->secs_per_zone;
923
924 /* give up on finding another zone */
925 if (!init)
926 goto got_it;
927 if (sbi->secs_per_zone == 1)
928 goto got_it;
929 if (zoneno == old_zoneno)
930 goto got_it;
931 if (dir == ALLOC_LEFT) {
932 if (!go_left && zoneno + 1 >= total_zones)
933 goto got_it;
934 if (go_left && zoneno == 0)
935 goto got_it;
936 }
937 for (i = 0; i < NR_CURSEG_TYPE; i++)
938 if (CURSEG_I(sbi, i)->zone == zoneno)
939 break;
940
941 if (i < NR_CURSEG_TYPE) {
942 /* zone is in user, try another */
943 if (go_left)
944 hint = zoneno * sbi->secs_per_zone - 1;
945 else if (zoneno + 1 >= total_zones)
946 hint = 0;
947 else
948 hint = (zoneno + 1) * sbi->secs_per_zone;
949 init = false;
950 goto find_other_zone;
951 }
952got_it:
953 /* set it as dirty segment in free segmap */
9850cf4a 954 f2fs_bug_on(sbi, test_bit(segno, free_i->free_segmap));
351df4b2
JK		 955 __set_inuse(sbi, segno);
956 *newseg = segno;
1a118ccf 957 spin_unlock(&free_i->segmap_lock);
351df4b2
JK		 958}
959
960static void reset_curseg(struct f2fs_sb_info *sbi, int type, int modified)
961{
962 struct curseg_info *curseg = CURSEG_I(sbi, type);
963 struct summary_footer *sum_footer;
964
965 curseg->segno = curseg->next_segno;
966 curseg->zone = GET_ZONENO_FROM_SEGNO(sbi, curseg->segno);
967 curseg->next_blkoff = 0;
968 curseg->next_segno = NULL_SEGNO;
969
970 sum_footer = &(curseg->sum_blk->footer);
971 memset(sum_footer, 0, sizeof(struct summary_footer));
972 if (IS_DATASEG(type))
973 SET_SUM_TYPE(sum_footer, SUM_TYPE_DATA);
974 if (IS_NODESEG(type))
975 SET_SUM_TYPE(sum_footer, SUM_TYPE_NODE);
976 __set_sit_entry_type(sbi, type, curseg->segno, modified);
977}
978
0a8165d7 979/*
351df4b2
JK		 980 * Allocate a current working segment.
981 * This function always allocates a free segment in LFS manner.
982 */
983static void new_curseg(struct f2fs_sb_info *sbi, int type, bool new_sec)
984{
985 struct curseg_info *curseg = CURSEG_I(sbi, type);
986 unsigned int segno = curseg->segno;
987 int dir = ALLOC_LEFT;
988
989 write_sum_page(sbi, curseg->sum_blk,
81fb5e87 990 GET_SUM_BLOCK(sbi, segno));
351df4b2
JK		 991 if (type == CURSEG_WARM_DATA || type == CURSEG_COLD_DATA)
992 dir = ALLOC_RIGHT;
993
994 if (test_opt(sbi, NOHEAP))
995 dir = ALLOC_RIGHT;
996
997 get_new_segment(sbi, &segno, new_sec, dir);
998 curseg->next_segno = segno;
999 reset_curseg(sbi, type, 1);
1000 curseg->alloc_type = LFS;
1001}
1002
1003static void __next_free_blkoff(struct f2fs_sb_info *sbi,
1004 struct curseg_info *seg, block_t start)
1005{
1006 struct seg_entry *se = get_seg_entry(sbi, seg->segno);
e81c93cf 1007 int entries = SIT_VBLOCK_MAP_SIZE / sizeof(unsigned long);
60a3b782 1008 unsigned long *target_map = SIT_I(sbi)->tmp_map;
e81c93cf
CL		 1009 unsigned long *ckpt_map = (unsigned long *)se->ckpt_valid_map;
1010 unsigned long *cur_map = (unsigned long *)se->cur_valid_map;
1011 int i, pos;
1012
1013 for (i = 0; i < entries; i++)
1014 target_map[i] = ckpt_map[i] | cur_map[i];
1015
1016 pos = __find_rev_next_zero_bit(target_map, sbi->blocks_per_seg, start);
1017
1018 seg->next_blkoff = pos;
351df4b2
JK		 1019}
1020
0a8165d7 1021/*
351df4b2
JK		 1022 * If a segment is written by LFS manner, next block offset is just obtained
1023 * by increasing the current block offset. However, if a segment is written by
1024 * SSR manner, next block offset obtained by calling __next_free_blkoff
1025 */
1026static void __refresh_next_blkoff(struct f2fs_sb_info *sbi,
1027 struct curseg_info *seg)
1028{
1029 if (seg->alloc_type == SSR)
1030 __next_free_blkoff(sbi, seg, seg->next_blkoff + 1);
1031 else
1032 seg->next_blkoff++;
1033}
1034
0a8165d7 1035/*
e1c42045 1036 * This function always allocates a used segment(from dirty seglist) by SSR
351df4b2
JK		 1037 * manner, so it should recover the existing segment information of valid blocks
1038 */
1039static void change_curseg(struct f2fs_sb_info *sbi, int type, bool reuse)
1040{
1041 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
1042 struct curseg_info *curseg = CURSEG_I(sbi, type);
1043 unsigned int new_segno = curseg->next_segno;
1044 struct f2fs_summary_block *sum_node;
1045 struct page *sum_page;
1046
1047 write_sum_page(sbi, curseg->sum_blk,
1048 GET_SUM_BLOCK(sbi, curseg->segno));
1049 __set_test_and_inuse(sbi, new_segno);
1050
1051 mutex_lock(&dirty_i->seglist_lock);
1052 __remove_dirty_segment(sbi, new_segno, PRE);
1053 __remove_dirty_segment(sbi, new_segno, DIRTY);
1054 mutex_unlock(&dirty_i->seglist_lock);
1055
1056 reset_curseg(sbi, type, 1);
1057 curseg->alloc_type = SSR;
1058 __next_free_blkoff(sbi, curseg, 0);
1059
1060 if (reuse) {
1061 sum_page = get_sum_page(sbi, new_segno);
1062 sum_node = (struct f2fs_summary_block *)page_address(sum_page);
1063 memcpy(curseg->sum_blk, sum_node, SUM_ENTRY_SIZE);
1064 f2fs_put_page(sum_page, 1);
1065 }
1066}
1067
43727527
JK		 1068static int get_ssr_segment(struct f2fs_sb_info *sbi, int type)
1069{
1070 struct curseg_info *curseg = CURSEG_I(sbi, type);
1071 const struct victim_selection *v_ops = DIRTY_I(sbi)->v_ops;
1072
1073 if (IS_NODESEG(type) || !has_not_enough_free_secs(sbi, 0))
1074 return v_ops->get_victim(sbi,
1075 &(curseg)->next_segno, BG_GC, type, SSR);
1076
1077 /* For data segments, let's do SSR more intensively */
1078 for (; type >= CURSEG_HOT_DATA; type--)
1079 if (v_ops->get_victim(sbi, &(curseg)->next_segno,
1080 BG_GC, type, SSR))
1081 return 1;
1082 return 0;
1083}
1084
351df4b2
JK		 1085/*
1086 * flush out current segment and replace it with new segment
1087 * This function should be returned with success, otherwise BUG
1088 */
1089static void allocate_segment_by_default(struct f2fs_sb_info *sbi,
1090 int type, bool force)
1091{
1092 struct curseg_info *curseg = CURSEG_I(sbi, type);
351df4b2 1093
7b405275 1094 if (force)
351df4b2 1095 new_curseg(sbi, type, true);
7b405275 1096 else if (type == CURSEG_WARM_NODE)
351df4b2 1097 new_curseg(sbi, type, false);
60374688
JK		 1098 else if (curseg->alloc_type == LFS && is_next_segment_free(sbi, type))
1099 new_curseg(sbi, type, false);
351df4b2
JK		 1100 else if (need_SSR(sbi) && get_ssr_segment(sbi, type))
1101 change_curseg(sbi, type, true);
1102 else
1103 new_curseg(sbi, type, false);
dcdfff65
JK		 1104
1105 stat_inc_seg_type(sbi, curseg);
351df4b2
JK		 1106}
1107
38aa0889
JK		 1108static void __allocate_new_segments(struct f2fs_sb_info *sbi, int type)
1109{
1110 struct curseg_info *curseg = CURSEG_I(sbi, type);
1111 unsigned int old_segno;
1112
1113 old_segno = curseg->segno;
1114 SIT_I(sbi)->s_ops->allocate_segment(sbi, type, true);
1115 locate_dirty_segment(sbi, old_segno);
1116}
1117
351df4b2
JK		 1118void allocate_new_segments(struct f2fs_sb_info *sbi)
1119{
351df4b2
JK		 1120 int i;
1121
38aa0889
JK		 1122 for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++)
1123 __allocate_new_segments(sbi, i);
351df4b2
JK		 1124}
1125
1126static const struct segment_allocation default_salloc_ops = {
1127 .allocate_segment = allocate_segment_by_default,
1128};
1129
4b2fecc8
JK		 1130int f2fs_trim_fs(struct f2fs_sb_info *sbi, struct fstrim_range *range)
1131{
f7ef9b83
JK		 1132 __u64 start = F2FS_BYTES_TO_BLK(range->start);
1133 __u64 end = start + F2FS_BYTES_TO_BLK(range->len) - 1;
4b2fecc8
JK		 1134 unsigned int start_segno, end_segno;
1135 struct cp_control cpc;
1136
836b5a63 1137 if (start >= MAX_BLKADDR(sbi) || range->len < sbi->blocksize)
4b2fecc8
JK		 1138 return -EINVAL;
1139
9bd27ae4 1140 cpc.trimmed = 0;
7cd8558b 1141 if (end <= MAIN_BLKADDR(sbi))
4b2fecc8
JK		 1142 goto out;
1143
1144 /* start/end segment number in main_area */
7cd8558b
JK		 1145 start_segno = (start <= MAIN_BLKADDR(sbi)) ? 0 : GET_SEGNO(sbi, start);
1146 end_segno = (end >= MAX_BLKADDR(sbi)) ? MAIN_SEGS(sbi) - 1 :
1147 GET_SEGNO(sbi, end);
4b2fecc8 1148 cpc.reason = CP_DISCARD;
836b5a63 1149 cpc.trim_minlen = max_t(__u64, 1, F2FS_BYTES_TO_BLK(range->minlen));
4b2fecc8
JK		 1150
1151 /* do checkpoint to issue discard commands safely */
bba681cb
JK		 1152 for (; start_segno <= end_segno; start_segno = cpc.trim_end + 1) {
1153 cpc.trim_start = start_segno;
a66cdd98
JK		 1154
1155 if (sbi->discard_blks == 0)
1156 break;
1157 else if (sbi->discard_blks < BATCHED_TRIM_BLOCKS(sbi))
1158 cpc.trim_end = end_segno;
1159 else
1160 cpc.trim_end = min_t(unsigned int,
1161 rounddown(start_segno +
bba681cb
JK		 1162 BATCHED_TRIM_SEGMENTS(sbi),
1163 sbi->segs_per_sec) - 1, end_segno);
1164
1165 mutex_lock(&sbi->gc_mutex);
1166 write_checkpoint(sbi, &cpc);
1167 mutex_unlock(&sbi->gc_mutex);
1168 }
4b2fecc8 1169out:
f7ef9b83 1170 range->len = F2FS_BLK_TO_BYTES(cpc.trimmed);
4b2fecc8
JK		 1171 return 0;
1172}
1173
351df4b2
JK		 1174static bool __has_curseg_space(struct f2fs_sb_info *sbi, int type)
1175{
1176 struct curseg_info *curseg = CURSEG_I(sbi, type);
1177 if (curseg->next_blkoff < sbi->blocks_per_seg)
1178 return true;
1179 return false;
1180}
1181
1182static int __get_segment_type_2(struct page *page, enum page_type p_type)
1183{
1184 if (p_type == DATA)
1185 return CURSEG_HOT_DATA;
1186 else
1187 return CURSEG_HOT_NODE;
1188}
1189
1190static int __get_segment_type_4(struct page *page, enum page_type p_type)
1191{
1192 if (p_type == DATA) {
1193 struct inode *inode = page->mapping->host;
1194
1195 if (S_ISDIR(inode->i_mode))
1196 return CURSEG_HOT_DATA;
1197 else
1198 return CURSEG_COLD_DATA;
1199 } else {
a344b9fd
JK		 1200 if (IS_DNODE(page) && is_cold_node(page))
1201 return CURSEG_WARM_NODE;
351df4b2
JK		 1202 else
1203 return CURSEG_COLD_NODE;
1204 }
1205}
1206
1207static int __get_segment_type_6(struct page *page, enum page_type p_type)
1208{
1209 if (p_type == DATA) {
1210 struct inode *inode = page->mapping->host;
1211
1212 if (S_ISDIR(inode->i_mode))
1213 return CURSEG_HOT_DATA;
354a3399 1214 else if (is_cold_data(page) || file_is_cold(inode))
351df4b2
JK		 1215 return CURSEG_COLD_DATA;
1216 else
1217 return CURSEG_WARM_DATA;
1218 } else {
1219 if (IS_DNODE(page))
1220 return is_cold_node(page) ? CURSEG_WARM_NODE :
1221 CURSEG_HOT_NODE;
1222 else
1223 return CURSEG_COLD_NODE;
1224 }
1225}
1226
1227static int __get_segment_type(struct page *page, enum page_type p_type)
1228{
4081363f 1229 switch (F2FS_P_SB(page)->active_logs) {
351df4b2
JK		 1230 case 2:
1231 return __get_segment_type_2(page, p_type);
1232 case 4:
1233 return __get_segment_type_4(page, p_type);
351df4b2 1234 }
12a67146 1235 /* NR_CURSEG_TYPE(6) logs by default */
9850cf4a
JK		 1236 f2fs_bug_on(F2FS_P_SB(page),
1237 F2FS_P_SB(page)->active_logs != NR_CURSEG_TYPE);
12a67146 1238 return __get_segment_type_6(page, p_type);
351df4b2
JK		 1239}
1240
bfad7c2d
JK		 1241void allocate_data_block(struct f2fs_sb_info *sbi, struct page *page,
1242 block_t old_blkaddr, block_t *new_blkaddr,
1243 struct f2fs_summary *sum, int type)
351df4b2
JK		 1244{
1245 struct sit_info *sit_i = SIT_I(sbi);
1246 struct curseg_info *curseg;
38aa0889
JK		 1247 bool direct_io = (type == CURSEG_DIRECT_IO);
1248
1249 type = direct_io ? CURSEG_WARM_DATA : type;
351df4b2 1250
351df4b2
JK		 1251 curseg = CURSEG_I(sbi, type);
1252
1253 mutex_lock(&curseg->curseg_mutex);
21cb1d99 1254 mutex_lock(&sit_i->sentry_lock);
351df4b2 1255
38aa0889 1256 /* direct_io'ed data is aligned to the segment for better performance */
47e70ca4
JK		 1257 if (direct_io && curseg->next_blkoff &&
1258 !has_not_enough_free_secs(sbi, 0))
38aa0889
JK		 1259 __allocate_new_segments(sbi, type);
1260
351df4b2 1261 *new_blkaddr = NEXT_FREE_BLKADDR(sbi, curseg);
351df4b2
JK		 1262
1263 /*
1264 * __add_sum_entry should be resided under the curseg_mutex
1265 * because, this function updates a summary entry in the
1266 * current summary block.
1267 */
e79efe3b 1268 __add_sum_entry(sbi, type, sum);
351df4b2 1269
351df4b2 1270 __refresh_next_blkoff(sbi, curseg);
dcdfff65
JK		 1271
1272 stat_inc_block_count(sbi, curseg);
351df4b2 1273
5e443818
JK		 1274 if (!__has_curseg_space(sbi, type))
1275 sit_i->s_ops->allocate_segment(sbi, type, false);
351df4b2
JK		 1276 /*
1277 * SIT information should be updated before segment allocation,
1278 * since SSR needs latest valid block information.
1279 */
1280 refresh_sit_entry(sbi, old_blkaddr, *new_blkaddr);
5e443818 1281
351df4b2
JK		 1282 mutex_unlock(&sit_i->sentry_lock);
1283
bfad7c2d 1284 if (page && IS_NODESEG(type))
351df4b2
JK		 1285 fill_node_footer_blkaddr(page, NEXT_FREE_BLKADDR(sbi, curseg));
1286
bfad7c2d
JK		 1287 mutex_unlock(&curseg->curseg_mutex);
1288}
1289
05ca3632 1290static void do_write_page(struct f2fs_summary *sum, struct f2fs_io_info *fio)
bfad7c2d 1291{
05ca3632 1292 int type = __get_segment_type(fio->page, fio->type);
bfad7c2d 1293
05ca3632
JK		 1294 allocate_data_block(fio->sbi, fio->page, fio->blk_addr,
1295 &fio->blk_addr, sum, type);
bfad7c2d 1296
351df4b2 1297 /* writeout dirty page into bdev */
05ca3632 1298 f2fs_submit_page_mbio(fio);
351df4b2
JK		 1299}
1300
577e3495 1301void write_meta_page(struct f2fs_sb_info *sbi, struct page *page)
351df4b2 1302{
458e6197 1303 struct f2fs_io_info fio = {
05ca3632 1304 .sbi = sbi,
458e6197 1305 .type = META,
cf04e8eb
JK		 1306 .rw = WRITE_SYNC | REQ_META | REQ_PRIO,
1307 .blk_addr = page->index,
05ca3632 1308 .page = page,
4375a336 1309 .encrypted_page = NULL,
458e6197
JK		 1310 };
1311
2b947003
CY		 1312 if (unlikely(page->index >= MAIN_BLKADDR(sbi)))
1313 fio.rw &= ~REQ_META;
1314
351df4b2 1315 set_page_writeback(page);
05ca3632 1316 f2fs_submit_page_mbio(&fio);
351df4b2
JK		 1317}
1318
05ca3632 1319void write_node_page(unsigned int nid, struct f2fs_io_info *fio)
351df4b2
JK		 1320{
1321 struct f2fs_summary sum;
05ca3632 1322
351df4b2 1323 set_summary(&sum, nid, 0, 0);
05ca3632 1324 do_write_page(&sum, fio);
351df4b2
JK		 1325}
1326
05ca3632 1327void write_data_page(struct dnode_of_data *dn, struct f2fs_io_info *fio)
351df4b2 1328{
05ca3632 1329 struct f2fs_sb_info *sbi = fio->sbi;
351df4b2
JK		 1330 struct f2fs_summary sum;
1331 struct node_info ni;
1332
9850cf4a 1333 f2fs_bug_on(sbi, dn->data_blkaddr == NULL_ADDR);
351df4b2
JK		 1334 get_node_info(sbi, dn->nid, &ni);
1335 set_summary(&sum, dn->nid, dn->ofs_in_node, ni.version);
05ca3632 1336 do_write_page(&sum, fio);
e1509cf2 1337 dn->data_blkaddr = fio->blk_addr;
351df4b2
JK		 1338}
1339
05ca3632 1340void rewrite_data_page(struct f2fs_io_info *fio)
351df4b2 1341{
05ca3632
JK		 1342 stat_inc_inplace_blocks(fio->sbi);
1343 f2fs_submit_page_mbio(fio);
351df4b2
JK		 1344}
1345
528e3459
CY		 1346static void __f2fs_replace_block(struct f2fs_sb_info *sbi,
1347 struct f2fs_summary *sum,
19f106bc
CY		 1348 block_t old_blkaddr, block_t new_blkaddr,
1349 bool recover_curseg)
351df4b2
JK		 1350{
1351 struct sit_info *sit_i = SIT_I(sbi);
1352 struct curseg_info *curseg;
1353 unsigned int segno, old_cursegno;
1354 struct seg_entry *se;
1355 int type;
19f106bc 1356 unsigned short old_blkoff;
351df4b2
JK		 1357
1358 segno = GET_SEGNO(sbi, new_blkaddr);
1359 se = get_seg_entry(sbi, segno);
1360 type = se->type;
1361
19f106bc
CY		 1362 if (!recover_curseg) {
1363 /* for recovery flow */
1364 if (se->valid_blocks == 0 && !IS_CURSEG(sbi, segno)) {
1365 if (old_blkaddr == NULL_ADDR)
1366 type = CURSEG_COLD_DATA;
1367 else
1368 type = CURSEG_WARM_DATA;
1369 }
1370 } else {
1371 if (!IS_CURSEG(sbi, segno))
351df4b2
JK		 1372 type = CURSEG_WARM_DATA;
1373 }
19f106bc 1374
351df4b2
JK		 1375 curseg = CURSEG_I(sbi, type);
1376
1377 mutex_lock(&curseg->curseg_mutex);
1378 mutex_lock(&sit_i->sentry_lock);
1379
1380 old_cursegno = curseg->segno;
19f106bc 1381 old_blkoff = curseg->next_blkoff;
351df4b2
JK		 1382
1383 /* change the current segment */
1384 if (segno != curseg->segno) {
1385 curseg->next_segno = segno;
1386 change_curseg(sbi, type, true);
1387 }
1388
491c0854 1389 curseg->next_blkoff = GET_BLKOFF_FROM_SEG0(sbi, new_blkaddr);
e79efe3b 1390 __add_sum_entry(sbi, type, sum);
351df4b2 1391
6e2c64ad
JK		 1392 if (!recover_curseg)
1393 update_sit_entry(sbi, new_blkaddr, 1);
1394 if (GET_SEGNO(sbi, old_blkaddr) != NULL_SEGNO)
1395 update_sit_entry(sbi, old_blkaddr, -1);
1396
1397 locate_dirty_segment(sbi, GET_SEGNO(sbi, old_blkaddr));
1398 locate_dirty_segment(sbi, GET_SEGNO(sbi, new_blkaddr));
1399
351df4b2 1400 locate_dirty_segment(sbi, old_cursegno);
351df4b2 1401
19f106bc
CY		 1402 if (recover_curseg) {
1403 if (old_cursegno != curseg->segno) {
1404 curseg->next_segno = old_cursegno;
1405 change_curseg(sbi, type, true);
1406 }
1407 curseg->next_blkoff = old_blkoff;
1408 }
1409
351df4b2
JK		 1410 mutex_unlock(&sit_i->sentry_lock);
1411 mutex_unlock(&curseg->curseg_mutex);
1412}
1413
528e3459
CY		 1414void f2fs_replace_block(struct f2fs_sb_info *sbi, struct dnode_of_data *dn,
1415 block_t old_addr, block_t new_addr,
1416 unsigned char version, bool recover_curseg)
1417{
1418 struct f2fs_summary sum;
1419
1420 set_summary(&sum, dn->nid, dn->ofs_in_node, version);
1421
1422 __f2fs_replace_block(sbi, &sum, old_addr, new_addr, recover_curseg);
1423
1424 dn->data_blkaddr = new_addr;
1425 set_data_blkaddr(dn);
1426 f2fs_update_extent_cache(dn);
1427}
1428
df0f8dc0
CY		 1429static inline bool is_merged_page(struct f2fs_sb_info *sbi,
1430 struct page *page, enum page_type type)
1431{
1432 enum page_type btype = PAGE_TYPE_OF_BIO(type);
1433 struct f2fs_bio_info *io = &sbi->write_io[btype];
df0f8dc0 1434 struct bio_vec *bvec;
4375a336 1435 struct page *target;
df0f8dc0
CY		 1436 int i;
1437
1438 down_read(&io->io_rwsem);
4375a336
JK		 1439 if (!io->bio) {
1440 up_read(&io->io_rwsem);
1441 return false;
1442 }
df0f8dc0 1443
ce23447f 1444 bio_for_each_segment_all(bvec, io->bio, i) {
4375a336
JK		 1445
1446 if (bvec->bv_page->mapping) {
1447 target = bvec->bv_page;
1448 } else {
1449 struct f2fs_crypto_ctx *ctx;
1450
1451 /* encrypted page */
1452 ctx = (struct f2fs_crypto_ctx *)page_private(
1453 bvec->bv_page);
ca40b030 1454 target = ctx->w.control_page;
4375a336
JK		 1455 }
1456
1457 if (page == target) {
df0f8dc0
CY		 1458 up_read(&io->io_rwsem);
1459 return true;
1460 }
1461 }
1462
df0f8dc0
CY		 1463 up_read(&io->io_rwsem);
1464 return false;
1465}
1466
93dfe2ac 1467void f2fs_wait_on_page_writeback(struct page *page,
5514f0aa 1468 enum page_type type)
93dfe2ac 1469{
93dfe2ac 1470 if (PageWriteback(page)) {
4081363f
JK		 1471 struct f2fs_sb_info *sbi = F2FS_P_SB(page);
1472
df0f8dc0
CY		 1473 if (is_merged_page(sbi, page, type))
1474 f2fs_submit_merged_bio(sbi, type, WRITE);
93dfe2ac
JK		 1475 wait_on_page_writeback(page);
1476 }
1477}
1478
08b39fbd
CY		 1479void f2fs_wait_on_encrypted_page_writeback(struct f2fs_sb_info *sbi,
1480 block_t blkaddr)
1481{
1482 struct page *cpage;
1483
1484 if (blkaddr == NEW_ADDR)
1485 return;
1486
1487 f2fs_bug_on(sbi, blkaddr == NULL_ADDR);
1488
1489 cpage = find_lock_page(META_MAPPING(sbi), blkaddr);
1490 if (cpage) {
1491 f2fs_wait_on_page_writeback(cpage, DATA);
1492 f2fs_put_page(cpage, 1);
1493 }
1494}
1495
351df4b2
JK		 1496static int read_compacted_summaries(struct f2fs_sb_info *sbi)
1497{
1498 struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi);
1499 struct curseg_info *seg_i;
1500 unsigned char *kaddr;
1501 struct page *page;
1502 block_t start;
1503 int i, j, offset;
1504
1505 start = start_sum_block(sbi);
1506
1507 page = get_meta_page(sbi, start++);
1508 kaddr = (unsigned char *)page_address(page);
1509
1510 /* Step 1: restore nat cache */
1511 seg_i = CURSEG_I(sbi, CURSEG_HOT_DATA);
1512 memcpy(&seg_i->sum_blk->n_nats, kaddr, SUM_JOURNAL_SIZE);
1513
1514 /* Step 2: restore sit cache */
1515 seg_i = CURSEG_I(sbi, CURSEG_COLD_DATA);
1516 memcpy(&seg_i->sum_blk->n_sits, kaddr + SUM_JOURNAL_SIZE,
1517 SUM_JOURNAL_SIZE);
1518 offset = 2 * SUM_JOURNAL_SIZE;
1519
1520 /* Step 3: restore summary entries */
1521 for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++) {
1522 unsigned short blk_off;
1523 unsigned int segno;
1524
1525 seg_i = CURSEG_I(sbi, i);
1526 segno = le32_to_cpu(ckpt->cur_data_segno[i]);
1527 blk_off = le16_to_cpu(ckpt->cur_data_blkoff[i]);
1528 seg_i->next_segno = segno;
1529 reset_curseg(sbi, i, 0);
1530 seg_i->alloc_type = ckpt->alloc_type[i];
1531 seg_i->next_blkoff = blk_off;
1532
1533 if (seg_i->alloc_type == SSR)
1534 blk_off = sbi->blocks_per_seg;
1535
1536 for (j = 0; j < blk_off; j++) {
1537 struct f2fs_summary *s;
1538 s = (struct f2fs_summary *)(kaddr + offset);
1539 seg_i->sum_blk->entries[j] = *s;
1540 offset += SUMMARY_SIZE;
1541 if (offset + SUMMARY_SIZE <= PAGE_CACHE_SIZE -
1542 SUM_FOOTER_SIZE)
1543 continue;
1544
1545 f2fs_put_page(page, 1);
1546 page = NULL;
1547
1548 page = get_meta_page(sbi, start++);
1549 kaddr = (unsigned char *)page_address(page);
1550 offset = 0;
1551 }
1552 }
1553 f2fs_put_page(page, 1);
1554 return 0;
1555}
1556
1557static int read_normal_summaries(struct f2fs_sb_info *sbi, int type)
1558{
1559 struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi);
1560 struct f2fs_summary_block *sum;
1561 struct curseg_info *curseg;
1562 struct page *new;
1563 unsigned short blk_off;
1564 unsigned int segno = 0;
1565 block_t blk_addr = 0;
1566
1567 /* get segment number and block addr */
1568 if (IS_DATASEG(type)) {
1569 segno = le32_to_cpu(ckpt->cur_data_segno[type]);
1570 blk_off = le16_to_cpu(ckpt->cur_data_blkoff[type -
1571 CURSEG_HOT_DATA]);
119ee914 1572 if (__exist_node_summaries(sbi))
351df4b2
JK		 1573 blk_addr = sum_blk_addr(sbi, NR_CURSEG_TYPE, type);
1574 else
1575 blk_addr = sum_blk_addr(sbi, NR_CURSEG_DATA_TYPE, type);
1576 } else {
1577 segno = le32_to_cpu(ckpt->cur_node_segno[type -
1578 CURSEG_HOT_NODE]);
1579 blk_off = le16_to_cpu(ckpt->cur_node_blkoff[type -
1580 CURSEG_HOT_NODE]);
119ee914 1581 if (__exist_node_summaries(sbi))
351df4b2
JK		 1582 blk_addr = sum_blk_addr(sbi, NR_CURSEG_NODE_TYPE,
1583 type - CURSEG_HOT_NODE);
1584 else
1585 blk_addr = GET_SUM_BLOCK(sbi, segno);
1586 }
1587
1588 new = get_meta_page(sbi, blk_addr);
1589 sum = (struct f2fs_summary_block *)page_address(new);
1590
1591 if (IS_NODESEG(type)) {
119ee914 1592 if (__exist_node_summaries(sbi)) {
351df4b2
JK		 1593 struct f2fs_summary *ns = &sum->entries[0];
1594 int i;
1595 for (i = 0; i < sbi->blocks_per_seg; i++, ns++) {
1596 ns->version = 0;
1597 ns->ofs_in_node = 0;
1598 }
1599 } else {
d653788a
GZ		 1600 int err;
1601
1602 err = restore_node_summary(sbi, segno, sum);
1603 if (err) {
351df4b2 1604 f2fs_put_page(new, 1);
d653788a 1605 return err;
351df4b2
JK		 1606 }
1607 }
1608 }
1609
1610 /* set uncompleted segment to curseg */
1611 curseg = CURSEG_I(sbi, type);
1612 mutex_lock(&curseg->curseg_mutex);
1613 memcpy(curseg->sum_blk, sum, PAGE_CACHE_SIZE);
1614 curseg->next_segno = segno;
1615 reset_curseg(sbi, type, 0);
1616 curseg->alloc_type = ckpt->alloc_type[type];
1617 curseg->next_blkoff = blk_off;
1618 mutex_unlock(&curseg->curseg_mutex);
1619 f2fs_put_page(new, 1);
1620 return 0;
1621}
1622
1623static int restore_curseg_summaries(struct f2fs_sb_info *sbi)
1624{
1625 int type = CURSEG_HOT_DATA;
e4fc5fbf 1626 int err;
351df4b2 1627
25ca923b 1628 if (is_set_ckpt_flags(F2FS_CKPT(sbi), CP_COMPACT_SUM_FLAG)) {
3fa06d7b
CY		 1629 int npages = npages_for_summary_flush(sbi, true);
1630
1631 if (npages >= 2)
1632 ra_meta_pages(sbi, start_sum_block(sbi), npages,
26879fb1 1633 META_CP, true);
3fa06d7b 1634
351df4b2
JK		 1635 /* restore for compacted data summary */
1636 if (read_compacted_summaries(sbi))
1637 return -EINVAL;
1638 type = CURSEG_HOT_NODE;
1639 }
1640
119ee914 1641 if (__exist_node_summaries(sbi))
3fa06d7b 1642 ra_meta_pages(sbi, sum_blk_addr(sbi, NR_CURSEG_TYPE, type),
26879fb1 1643 NR_CURSEG_TYPE - type, META_CP, true);
3fa06d7b 1644
e4fc5fbf
CY		 1645 for (; type <= CURSEG_COLD_NODE; type++) {
1646 err = read_normal_summaries(sbi, type);
1647 if (err)
1648 return err;
1649 }
1650
351df4b2
JK		 1651 return 0;
1652}
1653
1654static void write_compacted_summaries(struct f2fs_sb_info *sbi, block_t blkaddr)
1655{
1656 struct page *page;
1657 unsigned char *kaddr;
1658 struct f2fs_summary *summary;
1659 struct curseg_info *seg_i;
1660 int written_size = 0;
1661 int i, j;
1662
1663 page = grab_meta_page(sbi, blkaddr++);
1664 kaddr = (unsigned char *)page_address(page);
1665
1666 /* Step 1: write nat cache */
1667 seg_i = CURSEG_I(sbi, CURSEG_HOT_DATA);
1668 memcpy(kaddr, &seg_i->sum_blk->n_nats, SUM_JOURNAL_SIZE);
1669 written_size += SUM_JOURNAL_SIZE;
1670
1671 /* Step 2: write sit cache */
1672 seg_i = CURSEG_I(sbi, CURSEG_COLD_DATA);
1673 memcpy(kaddr + written_size, &seg_i->sum_blk->n_sits,
1674 SUM_JOURNAL_SIZE);
1675 written_size += SUM_JOURNAL_SIZE;
1676
351df4b2
JK		 1677 /* Step 3: write summary entries */
1678 for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++) {
1679 unsigned short blkoff;
1680 seg_i = CURSEG_I(sbi, i);
1681 if (sbi->ckpt->alloc_type[i] == SSR)
1682 blkoff = sbi->blocks_per_seg;
1683 else
1684 blkoff = curseg_blkoff(sbi, i);
1685
1686 for (j = 0; j < blkoff; j++) {
1687 if (!page) {
1688 page = grab_meta_page(sbi, blkaddr++);
1689 kaddr = (unsigned char *)page_address(page);
1690 written_size = 0;
1691 }
1692 summary = (struct f2fs_summary *)(kaddr + written_size);
1693 *summary = seg_i->sum_blk->entries[j];
1694 written_size += SUMMARY_SIZE;
351df4b2
JK		 1695
1696 if (written_size + SUMMARY_SIZE <= PAGE_CACHE_SIZE -
1697 SUM_FOOTER_SIZE)
1698 continue;
1699
e8d61a74 1700 set_page_dirty(page);
351df4b2
JK		 1701 f2fs_put_page(page, 1);
1702 page = NULL;
1703 }
1704 }
e8d61a74
CY		 1705 if (page) {
1706 set_page_dirty(page);
351df4b2 1707 f2fs_put_page(page, 1);
e8d61a74 1708 }
351df4b2
JK		 1709}
1710
1711static void write_normal_summaries(struct f2fs_sb_info *sbi,
1712 block_t blkaddr, int type)
1713{
1714 int i, end;
1715 if (IS_DATASEG(type))
1716 end = type + NR_CURSEG_DATA_TYPE;
1717 else
1718 end = type + NR_CURSEG_NODE_TYPE;
1719
1720 for (i = type; i < end; i++) {
1721 struct curseg_info *sum = CURSEG_I(sbi, i);
1722 mutex_lock(&sum->curseg_mutex);
1723 write_sum_page(sbi, sum->sum_blk, blkaddr + (i - type));
1724 mutex_unlock(&sum->curseg_mutex);
1725 }
1726}
1727
1728void write_data_summaries(struct f2fs_sb_info *sbi, block_t start_blk)
1729{
25ca923b 1730 if (is_set_ckpt_flags(F2FS_CKPT(sbi), CP_COMPACT_SUM_FLAG))
351df4b2
JK		 1731 write_compacted_summaries(sbi, start_blk);
1732 else
1733 write_normal_summaries(sbi, start_blk, CURSEG_HOT_DATA);
1734}
1735
1736void write_node_summaries(struct f2fs_sb_info *sbi, block_t start_blk)
1737{
119ee914 1738 write_normal_summaries(sbi, start_blk, CURSEG_HOT_NODE);
351df4b2
JK		 1739}
1740
1741int lookup_journal_in_cursum(struct f2fs_summary_block *sum, int type,
1742 unsigned int val, int alloc)
1743{
1744 int i;
1745
1746 if (type == NAT_JOURNAL) {
1747 for (i = 0; i < nats_in_cursum(sum); i++) {
1748 if (le32_to_cpu(nid_in_journal(sum, i)) == val)
1749 return i;
1750 }
1751 if (alloc && nats_in_cursum(sum) < NAT_JOURNAL_ENTRIES)
1752 return update_nats_in_cursum(sum, 1);
1753 } else if (type == SIT_JOURNAL) {
1754 for (i = 0; i < sits_in_cursum(sum); i++)
1755 if (le32_to_cpu(segno_in_journal(sum, i)) == val)
1756 return i;
1757 if (alloc && sits_in_cursum(sum) < SIT_JOURNAL_ENTRIES)
1758 return update_sits_in_cursum(sum, 1);
1759 }
1760 return -1;
1761}
1762
1763static struct page *get_current_sit_page(struct f2fs_sb_info *sbi,
1764 unsigned int segno)
1765{
2cc22186 1766 return get_meta_page(sbi, current_sit_addr(sbi, segno));
351df4b2
JK		 1767}
1768
1769static struct page *get_next_sit_page(struct f2fs_sb_info *sbi,
1770 unsigned int start)
1771{
1772 struct sit_info *sit_i = SIT_I(sbi);
1773 struct page *src_page, *dst_page;
1774 pgoff_t src_off, dst_off;
1775 void *src_addr, *dst_addr;
1776
1777 src_off = current_sit_addr(sbi, start);
1778 dst_off = next_sit_addr(sbi, src_off);
1779
1780 /* get current sit block page without lock */
1781 src_page = get_meta_page(sbi, src_off);
1782 dst_page = grab_meta_page(sbi, dst_off);
9850cf4a 1783 f2fs_bug_on(sbi, PageDirty(src_page));
351df4b2
JK		 1784
1785 src_addr = page_address(src_page);
1786 dst_addr = page_address(dst_page);
1787 memcpy(dst_addr, src_addr, PAGE_CACHE_SIZE);
1788
1789 set_page_dirty(dst_page);
1790 f2fs_put_page(src_page, 1);
1791
1792 set_to_next_sit(sit_i, start);
1793
1794 return dst_page;
1795}
1796
184a5cd2
CY		 1797static struct sit_entry_set *grab_sit_entry_set(void)
1798{
1799 struct sit_entry_set *ses =
80c54505 1800 f2fs_kmem_cache_alloc(sit_entry_set_slab, GFP_NOFS);
184a5cd2
CY		 1801
1802 ses->entry_cnt = 0;
1803 INIT_LIST_HEAD(&ses->set_list);
1804 return ses;
1805}
1806
1807static void release_sit_entry_set(struct sit_entry_set *ses)
1808{
1809 list_del(&ses->set_list);
1810 kmem_cache_free(sit_entry_set_slab, ses);
1811}
1812
1813static void adjust_sit_entry_set(struct sit_entry_set *ses,
1814 struct list_head *head)
1815{
1816 struct sit_entry_set *next = ses;
1817
1818 if (list_is_last(&ses->set_list, head))
1819 return;
1820
1821 list_for_each_entry_continue(next, head, set_list)
1822 if (ses->entry_cnt <= next->entry_cnt)
1823 break;
1824
1825 list_move_tail(&ses->set_list, &next->set_list);
1826}
1827
1828static void add_sit_entry(unsigned int segno, struct list_head *head)
1829{
1830 struct sit_entry_set *ses;
1831 unsigned int start_segno = START_SEGNO(segno);
1832
1833 list_for_each_entry(ses, head, set_list) {
1834 if (ses->start_segno == start_segno) {
1835 ses->entry_cnt++;
1836 adjust_sit_entry_set(ses, head);
1837 return;
1838 }
1839 }
1840
1841 ses = grab_sit_entry_set();
1842
1843 ses->start_segno = start_segno;
1844 ses->entry_cnt++;
1845 list_add(&ses->set_list, head);
1846}
1847
1848static void add_sits_in_set(struct f2fs_sb_info *sbi)
1849{
1850 struct f2fs_sm_info *sm_info = SM_I(sbi);
1851 struct list_head *set_list = &sm_info->sit_entry_set;
1852 unsigned long *bitmap = SIT_I(sbi)->dirty_sentries_bitmap;
184a5cd2
CY		 1853 unsigned int segno;
1854
7cd8558b 1855 for_each_set_bit(segno, bitmap, MAIN_SEGS(sbi))
184a5cd2
CY		 1856 add_sit_entry(segno, set_list);
1857}
1858
1859static void remove_sits_in_journal(struct f2fs_sb_info *sbi)
351df4b2
JK		 1860{
1861 struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_COLD_DATA);
1862 struct f2fs_summary_block *sum = curseg->sum_blk;
1863 int i;
1864
184a5cd2
CY		 1865 for (i = sits_in_cursum(sum) - 1; i >= 0; i--) {
1866 unsigned int segno;
1867 bool dirtied;
1868
1869 segno = le32_to_cpu(segno_in_journal(sum, i));
1870 dirtied = __mark_sit_entry_dirty(sbi, segno);
1871
1872 if (!dirtied)
1873 add_sit_entry(segno, &SM_I(sbi)->sit_entry_set);
351df4b2 1874 }
184a5cd2 1875 update_sits_in_cursum(sum, -sits_in_cursum(sum));
351df4b2
JK		 1876}
1877
0a8165d7 1878/*
351df4b2
JK		 1879 * CP calls this function, which flushes SIT entries including sit_journal,
1880 * and moves prefree segs to free segs.
1881 */
4b2fecc8 1882void flush_sit_entries(struct f2fs_sb_info *sbi, struct cp_control *cpc)
351df4b2
JK		 1883{
1884 struct sit_info *sit_i = SIT_I(sbi);
1885 unsigned long *bitmap = sit_i->dirty_sentries_bitmap;
1886 struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_COLD_DATA);
1887 struct f2fs_summary_block *sum = curseg->sum_blk;
184a5cd2
CY		 1888 struct sit_entry_set *ses, *tmp;
1889 struct list_head *head = &SM_I(sbi)->sit_entry_set;
184a5cd2 1890 bool to_journal = true;
4b2fecc8 1891 struct seg_entry *se;
351df4b2
JK		 1892
1893 mutex_lock(&curseg->curseg_mutex);
1894 mutex_lock(&sit_i->sentry_lock);
1895
2b11a74b
WL		 1896 if (!sit_i->dirty_sentries)
1897 goto out;
1898
351df4b2 1899 /*
184a5cd2
CY		 1900 * add and account sit entries of dirty bitmap in sit entry
1901 * set temporarily
351df4b2 1902 */
184a5cd2 1903 add_sits_in_set(sbi);
351df4b2 1904
184a5cd2
CY		 1905 /*
1906 * if there are no enough space in journal to store dirty sit
1907 * entries, remove all entries from journal and add and account
1908 * them in sit entry set.
1909 */
1910 if (!__has_cursum_space(sum, sit_i->dirty_sentries, SIT_JOURNAL))
1911 remove_sits_in_journal(sbi);
b2955550 1912
184a5cd2
CY		 1913 /*
1914 * there are two steps to flush sit entries:
1915 * #1, flush sit entries to journal in current cold data summary block.
1916 * #2, flush sit entries to sit page.
1917 */
1918 list_for_each_entry_safe(ses, tmp, head, set_list) {
4a257ed6 1919 struct page *page = NULL;
184a5cd2
CY		 1920 struct f2fs_sit_block *raw_sit = NULL;
1921 unsigned int start_segno = ses->start_segno;
1922 unsigned int end = min(start_segno + SIT_ENTRY_PER_BLOCK,
7cd8558b 1923 (unsigned long)MAIN_SEGS(sbi));
184a5cd2
CY		 1924 unsigned int segno = start_segno;
1925
1926 if (to_journal &&
1927 !__has_cursum_space(sum, ses->entry_cnt, SIT_JOURNAL))
1928 to_journal = false;
1929
1930 if (!to_journal) {
1931 page = get_next_sit_page(sbi, start_segno);
1932 raw_sit = page_address(page);
351df4b2 1933 }
351df4b2 1934
184a5cd2
CY		 1935 /* flush dirty sit entries in region of current sit set */
1936 for_each_set_bit_from(segno, bitmap, end) {
1937 int offset, sit_offset;
4b2fecc8
JK		 1938
1939 se = get_seg_entry(sbi, segno);
184a5cd2
CY		 1940
1941 /* add discard candidates */
d7bc2484 1942 if (cpc->reason != CP_DISCARD) {
4b2fecc8
JK		 1943 cpc->trim_start = segno;
1944 add_discard_addrs(sbi, cpc);
1945 }
184a5cd2
CY		 1946
1947 if (to_journal) {
1948 offset = lookup_journal_in_cursum(sum,
1949 SIT_JOURNAL, segno, 1);
1950 f2fs_bug_on(sbi, offset < 0);
1951 segno_in_journal(sum, offset) =
1952 cpu_to_le32(segno);
1953 seg_info_to_raw_sit(se,
1954 &sit_in_journal(sum, offset));
1955 } else {
1956 sit_offset = SIT_ENTRY_OFFSET(sit_i, segno);
1957 seg_info_to_raw_sit(se,
1958 &raw_sit->entries[sit_offset]);
1959 }
351df4b2 1960
184a5cd2
CY		 1961 __clear_bit(segno, bitmap);
1962 sit_i->dirty_sentries--;
1963 ses->entry_cnt--;
351df4b2
JK		 1964 }
1965
184a5cd2
CY		 1966 if (!to_journal)
1967 f2fs_put_page(page, 1);
1968
1969 f2fs_bug_on(sbi, ses->entry_cnt);
1970 release_sit_entry_set(ses);
351df4b2 1971 }
184a5cd2
CY		 1972
1973 f2fs_bug_on(sbi, !list_empty(head));
1974 f2fs_bug_on(sbi, sit_i->dirty_sentries);
184a5cd2 1975out:
4b2fecc8
JK		 1976 if (cpc->reason == CP_DISCARD) {
1977 for (; cpc->trim_start <= cpc->trim_end; cpc->trim_start++)
1978 add_discard_addrs(sbi, cpc);
1979 }
351df4b2
JK		 1980 mutex_unlock(&sit_i->sentry_lock);
1981 mutex_unlock(&curseg->curseg_mutex);
1982
351df4b2
JK		 1983 set_prefree_as_free_segments(sbi);
1984}
1985
1986static int build_sit_info(struct f2fs_sb_info *sbi)
1987{
1988 struct f2fs_super_block *raw_super = F2FS_RAW_SUPER(sbi);
1989 struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi);
1990 struct sit_info *sit_i;
1991 unsigned int sit_segs, start;
1992 char *src_bitmap, *dst_bitmap;
1993 unsigned int bitmap_size;
1994
1995 /* allocate memory for SIT information */
1996 sit_i = kzalloc(sizeof(struct sit_info), GFP_KERNEL);
1997 if (!sit_i)
1998 return -ENOMEM;
1999
2000 SM_I(sbi)->sit_info = sit_i;
2001
39307a8e
JK		 2002 sit_i->sentries = f2fs_kvzalloc(MAIN_SEGS(sbi) *
2003 sizeof(struct seg_entry), GFP_KERNEL);
351df4b2
JK		 2004 if (!sit_i->sentries)
2005 return -ENOMEM;
2006
7cd8558b 2007 bitmap_size = f2fs_bitmap_size(MAIN_SEGS(sbi));
39307a8e 2008 sit_i->dirty_sentries_bitmap = f2fs_kvzalloc(bitmap_size, GFP_KERNEL);
351df4b2
JK		 2009 if (!sit_i->dirty_sentries_bitmap)
2010 return -ENOMEM;
2011
7cd8558b 2012 for (start = 0; start < MAIN_SEGS(sbi); start++) {
351df4b2
JK		 2013 sit_i->sentries[start].cur_valid_map
2014 = kzalloc(SIT_VBLOCK_MAP_SIZE, GFP_KERNEL);
2015 sit_i->sentries[start].ckpt_valid_map
2016 = kzalloc(SIT_VBLOCK_MAP_SIZE, GFP_KERNEL);
a66cdd98
JK		 2017 sit_i->sentries[start].discard_map
2018 = kzalloc(SIT_VBLOCK_MAP_SIZE, GFP_KERNEL);
2019 if (!sit_i->sentries[start].cur_valid_map ||
2020 !sit_i->sentries[start].ckpt_valid_map ||
2021 !sit_i->sentries[start].discard_map)
351df4b2
JK		 2022 return -ENOMEM;
2023 }
2024
60a3b782
JK		 2025 sit_i->tmp_map = kzalloc(SIT_VBLOCK_MAP_SIZE, GFP_KERNEL);
2026 if (!sit_i->tmp_map)
2027 return -ENOMEM;
2028
351df4b2 2029 if (sbi->segs_per_sec > 1) {
39307a8e
JK		 2030 sit_i->sec_entries = f2fs_kvzalloc(MAIN_SECS(sbi) *
2031 sizeof(struct sec_entry), GFP_KERNEL);
351df4b2
JK		 2032 if (!sit_i->sec_entries)
2033 return -ENOMEM;
2034 }
2035
2036 /* get information related with SIT */
2037 sit_segs = le32_to_cpu(raw_super->segment_count_sit) >> 1;
2038
2039 /* setup SIT bitmap from ckeckpoint pack */
2040 bitmap_size = __bitmap_size(sbi, SIT_BITMAP);
2041 src_bitmap = __bitmap_ptr(sbi, SIT_BITMAP);
2042
79b5793b 2043 dst_bitmap = kmemdup(src_bitmap, bitmap_size, GFP_KERNEL);
351df4b2
JK		 2044 if (!dst_bitmap)
2045 return -ENOMEM;
351df4b2
JK		 2046
2047 /* init SIT information */
2048 sit_i->s_ops = &default_salloc_ops;
2049
2050 sit_i->sit_base_addr = le32_to_cpu(raw_super->sit_blkaddr);
2051 sit_i->sit_blocks = sit_segs << sbi->log_blocks_per_seg;
2052 sit_i->written_valid_blocks = le64_to_cpu(ckpt->valid_block_count);
2053 sit_i->sit_bitmap = dst_bitmap;
2054 sit_i->bitmap_size = bitmap_size;
2055 sit_i->dirty_sentries = 0;
2056 sit_i->sents_per_block = SIT_ENTRY_PER_BLOCK;
2057 sit_i->elapsed_time = le64_to_cpu(sbi->ckpt->elapsed_time);
2058 sit_i->mounted_time = CURRENT_TIME_SEC.tv_sec;
2059 mutex_init(&sit_i->sentry_lock);
2060 return 0;
2061}
2062
2063static int build_free_segmap(struct f2fs_sb_info *sbi)
2064{
351df4b2
JK		 2065 struct free_segmap_info *free_i;
2066 unsigned int bitmap_size, sec_bitmap_size;
2067
2068 /* allocate memory for free segmap information */
2069 free_i = kzalloc(sizeof(struct free_segmap_info), GFP_KERNEL);
2070 if (!free_i)
2071 return -ENOMEM;
2072
2073 SM_I(sbi)->free_info = free_i;
2074
7cd8558b 2075 bitmap_size = f2fs_bitmap_size(MAIN_SEGS(sbi));
39307a8e 2076 free_i->free_segmap = f2fs_kvmalloc(bitmap_size, GFP_KERNEL);
351df4b2
JK		 2077 if (!free_i->free_segmap)
2078 return -ENOMEM;
2079
7cd8558b 2080 sec_bitmap_size = f2fs_bitmap_size(MAIN_SECS(sbi));
39307a8e 2081 free_i->free_secmap = f2fs_kvmalloc(sec_bitmap_size, GFP_KERNEL);
351df4b2
JK		 2082 if (!free_i->free_secmap)
2083 return -ENOMEM;
2084
2085 /* set all segments as dirty temporarily */
2086 memset(free_i->free_segmap, 0xff, bitmap_size);
2087 memset(free_i->free_secmap, 0xff, sec_bitmap_size);
2088
2089 /* init free segmap information */
7cd8558b 2090 free_i->start_segno = GET_SEGNO_FROM_SEG0(sbi, MAIN_BLKADDR(sbi));
351df4b2
JK		 2091 free_i->free_segments = 0;
2092 free_i->free_sections = 0;
1a118ccf 2093 spin_lock_init(&free_i->segmap_lock);
351df4b2
JK		 2094 return 0;
2095}
2096
2097static int build_curseg(struct f2fs_sb_info *sbi)
2098{
1042d60f 2099 struct curseg_info *array;
351df4b2
JK		 2100 int i;
2101
b434babf 2102 array = kcalloc(NR_CURSEG_TYPE, sizeof(*array), GFP_KERNEL);
351df4b2
JK		 2103 if (!array)
2104 return -ENOMEM;
2105
2106 SM_I(sbi)->curseg_array = array;
2107
2108 for (i = 0; i < NR_CURSEG_TYPE; i++) {
2109 mutex_init(&array[i].curseg_mutex);
2110 array[i].sum_blk = kzalloc(PAGE_CACHE_SIZE, GFP_KERNEL);
2111 if (!array[i].sum_blk)
2112 return -ENOMEM;
2113 array[i].segno = NULL_SEGNO;
2114 array[i].next_blkoff = 0;
2115 }
2116 return restore_curseg_summaries(sbi);
2117}
2118
2119static void build_sit_entries(struct f2fs_sb_info *sbi)
2120{
2121 struct sit_info *sit_i = SIT_I(sbi);
2122 struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_COLD_DATA);
2123 struct f2fs_summary_block *sum = curseg->sum_blk;
74de593a
CY		 2124 int sit_blk_cnt = SIT_BLK_CNT(sbi);
2125 unsigned int i, start, end;
2126 unsigned int readed, start_blk = 0;
90a893c7 2127 int nrpages = MAX_BIO_BLOCKS(sbi);
351df4b2 2128
74de593a 2129 do {
26879fb1 2130 readed = ra_meta_pages(sbi, start_blk, nrpages, META_SIT, true);
74de593a
CY		 2131
2132 start = start_blk * sit_i->sents_per_block;
2133 end = (start_blk + readed) * sit_i->sents_per_block;
2134
7cd8558b 2135 for (; start < end && start < MAIN_SEGS(sbi); start++) {
74de593a
CY		 2136 struct seg_entry *se = &sit_i->sentries[start];
2137 struct f2fs_sit_block *sit_blk;
2138 struct f2fs_sit_entry sit;
2139 struct page *page;
2140
2141 mutex_lock(&curseg->curseg_mutex);
2142 for (i = 0; i < sits_in_cursum(sum); i++) {
6c311ec6
CF		 2143 if (le32_to_cpu(segno_in_journal(sum, i))
2144 == start) {
74de593a
CY		 2145 sit = sit_in_journal(sum, i);
2146 mutex_unlock(&curseg->curseg_mutex);
2147 goto got_it;
2148 }
351df4b2 2149 }
74de593a
CY		 2150 mutex_unlock(&curseg->curseg_mutex);
2151
2152 page = get_current_sit_page(sbi, start);
2153 sit_blk = (struct f2fs_sit_block *)page_address(page);
2154 sit = sit_blk->entries[SIT_ENTRY_OFFSET(sit_i, start)];
2155 f2fs_put_page(page, 1);
351df4b2 2156got_it:
74de593a
CY		 2157 check_block_count(sbi, start, &sit);
2158 seg_info_from_raw_sit(se, &sit);
a66cdd98
JK		 2159
2160 /* build discard map only one time */
2161 memcpy(se->discard_map, se->cur_valid_map, SIT_VBLOCK_MAP_SIZE);
2162 sbi->discard_blks += sbi->blocks_per_seg - se->valid_blocks;
2163
74de593a
CY		 2164 if (sbi->segs_per_sec > 1) {
2165 struct sec_entry *e = get_sec_entry(sbi, start);
2166 e->valid_blocks += se->valid_blocks;
2167 }
351df4b2 2168 }
74de593a
CY		 2169 start_blk += readed;
2170 } while (start_blk < sit_blk_cnt);
351df4b2
JK		 2171}
2172
2173static void init_free_segmap(struct f2fs_sb_info *sbi)
2174{
2175 unsigned int start;
2176 int type;
2177
7cd8558b 2178 for (start = 0; start < MAIN_SEGS(sbi); start++) {
351df4b2
JK		 2179 struct seg_entry *sentry = get_seg_entry(sbi, start);
2180 if (!sentry->valid_blocks)
2181 __set_free(sbi, start);
2182 }
2183
2184 /* set use the current segments */
2185 for (type = CURSEG_HOT_DATA; type <= CURSEG_COLD_NODE; type++) {
2186 struct curseg_info *curseg_t = CURSEG_I(sbi, type);
2187 __set_test_and_inuse(sbi, curseg_t->segno);
2188 }
2189}
2190
2191static void init_dirty_segmap(struct f2fs_sb_info *sbi)
2192{
2193 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
2194 struct free_segmap_info *free_i = FREE_I(sbi);
7cd8558b 2195 unsigned int segno = 0, offset = 0;
351df4b2
JK		 2196 unsigned short valid_blocks;
2197
8736fbf0 2198 while (1) {
351df4b2 2199 /* find dirty segment based on free segmap */
7cd8558b
JK		 2200 segno = find_next_inuse(free_i, MAIN_SEGS(sbi), offset);
2201 if (segno >= MAIN_SEGS(sbi))
351df4b2
JK		 2202 break;
2203 offset = segno + 1;
2204 valid_blocks = get_valid_blocks(sbi, segno, 0);
ec325b52 2205 if (valid_blocks == sbi->blocks_per_seg || !valid_blocks)
351df4b2 2206 continue;
ec325b52
JK		 2207 if (valid_blocks > sbi->blocks_per_seg) {
2208 f2fs_bug_on(sbi, 1);
2209 continue;
2210 }
351df4b2
JK		 2211 mutex_lock(&dirty_i->seglist_lock);
2212 __locate_dirty_segment(sbi, segno, DIRTY);
2213 mutex_unlock(&dirty_i->seglist_lock);
2214 }
2215}
2216
5ec4e49f 2217static int init_victim_secmap(struct f2fs_sb_info *sbi)
351df4b2
JK		 2218{
2219 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
7cd8558b 2220 unsigned int bitmap_size = f2fs_bitmap_size(MAIN_SECS(sbi));
351df4b2 2221
39307a8e 2222 dirty_i->victim_secmap = f2fs_kvzalloc(bitmap_size, GFP_KERNEL);
5ec4e49f 2223 if (!dirty_i->victim_secmap)
351df4b2
JK		 2224 return -ENOMEM;
2225 return 0;
2226}
2227
2228static int build_dirty_segmap(struct f2fs_sb_info *sbi)
2229{
2230 struct dirty_seglist_info *dirty_i;
2231 unsigned int bitmap_size, i;
2232
2233 /* allocate memory for dirty segments list information */
2234 dirty_i = kzalloc(sizeof(struct dirty_seglist_info), GFP_KERNEL);
2235 if (!dirty_i)
2236 return -ENOMEM;
2237
2238 SM_I(sbi)->dirty_info = dirty_i;
2239 mutex_init(&dirty_i->seglist_lock);
2240
7cd8558b 2241 bitmap_size = f2fs_bitmap_size(MAIN_SEGS(sbi));
351df4b2
JK		 2242
2243 for (i = 0; i < NR_DIRTY_TYPE; i++) {
39307a8e 2244 dirty_i->dirty_segmap[i] = f2fs_kvzalloc(bitmap_size, GFP_KERNEL);
351df4b2
JK		 2245 if (!dirty_i->dirty_segmap[i])
2246 return -ENOMEM;
2247 }
2248
2249 init_dirty_segmap(sbi);
5ec4e49f 2250 return init_victim_secmap(sbi);
351df4b2
JK		 2251}
2252
0a8165d7 2253/*
351df4b2
JK		 2254 * Update min, max modified time for cost-benefit GC algorithm
2255 */
2256static void init_min_max_mtime(struct f2fs_sb_info *sbi)
2257{
2258 struct sit_info *sit_i = SIT_I(sbi);
2259 unsigned int segno;
2260
2261 mutex_lock(&sit_i->sentry_lock);
2262
2263 sit_i->min_mtime = LLONG_MAX;
2264
7cd8558b 2265 for (segno = 0; segno < MAIN_SEGS(sbi); segno += sbi->segs_per_sec) {
351df4b2
JK		 2266 unsigned int i;
2267 unsigned long long mtime = 0;
2268
2269 for (i = 0; i < sbi->segs_per_sec; i++)
2270 mtime += get_seg_entry(sbi, segno + i)->mtime;
2271
2272 mtime = div_u64(mtime, sbi->segs_per_sec);
2273
2274 if (sit_i->min_mtime > mtime)
2275 sit_i->min_mtime = mtime;
2276 }
2277 sit_i->max_mtime = get_mtime(sbi);
2278 mutex_unlock(&sit_i->sentry_lock);
2279}
2280
2281int build_segment_manager(struct f2fs_sb_info *sbi)
2282{
2283 struct f2fs_super_block *raw_super = F2FS_RAW_SUPER(sbi);
2284 struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi);
1042d60f 2285 struct f2fs_sm_info *sm_info;
351df4b2
JK		 2286 int err;
2287
2288 sm_info = kzalloc(sizeof(struct f2fs_sm_info), GFP_KERNEL);
2289 if (!sm_info)
2290 return -ENOMEM;
2291
2292 /* init sm info */
2293 sbi->sm_info = sm_info;
351df4b2
JK		 2294 sm_info->seg0_blkaddr = le32_to_cpu(raw_super->segment0_blkaddr);
2295 sm_info->main_blkaddr = le32_to_cpu(raw_super->main_blkaddr);
2296 sm_info->segment_count = le32_to_cpu(raw_super->segment_count);
2297 sm_info->reserved_segments = le32_to_cpu(ckpt->rsvd_segment_count);
2298 sm_info->ovp_segments = le32_to_cpu(ckpt->overprov_segment_count);
2299 sm_info->main_segments = le32_to_cpu(raw_super->segment_count_main);
2300 sm_info->ssa_blkaddr = le32_to_cpu(raw_super->ssa_blkaddr);
58c41035
JK		 2301 sm_info->rec_prefree_segments = sm_info->main_segments *
2302 DEF_RECLAIM_PREFREE_SEGMENTS / 100;
9b5f136f 2303 sm_info->ipu_policy = 1 << F2FS_IPU_FSYNC;
216fbd64 2304 sm_info->min_ipu_util = DEF_MIN_IPU_UTIL;
c1ce1b02 2305 sm_info->min_fsync_blocks = DEF_MIN_FSYNC_BLOCKS;
351df4b2 2306
7fd9e544
JK		 2307 INIT_LIST_HEAD(&sm_info->discard_list);
2308 sm_info->nr_discards = 0;
2309 sm_info->max_discards = 0;
2310
bba681cb
JK		 2311 sm_info->trim_sections = DEF_BATCHED_TRIM_SECTIONS;
2312
184a5cd2
CY		 2313 INIT_LIST_HEAD(&sm_info->sit_entry_set);
2314
b270ad6f 2315 if (test_opt(sbi, FLUSH_MERGE) && !f2fs_readonly(sbi->sb)) {
2163d198
GZ		 2316 err = create_flush_cmd_control(sbi);
2317 if (err)
a688b9d9 2318 return err;
6b4afdd7
JK		 2319 }
2320
351df4b2
JK		 2321 err = build_sit_info(sbi);
2322 if (err)
2323 return err;
2324 err = build_free_segmap(sbi);
2325 if (err)
2326 return err;
2327 err = build_curseg(sbi);
2328 if (err)
2329 return err;
2330
2331 /* reinit free segmap based on SIT */
2332 build_sit_entries(sbi);
2333
2334 init_free_segmap(sbi);
2335 err = build_dirty_segmap(sbi);
2336 if (err)
2337 return err;
2338
2339 init_min_max_mtime(sbi);
2340 return 0;
2341}
2342
2343static void discard_dirty_segmap(struct f2fs_sb_info *sbi,
2344 enum dirty_type dirty_type)
2345{
2346 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
2347
2348 mutex_lock(&dirty_i->seglist_lock);
39307a8e 2349 kvfree(dirty_i->dirty_segmap[dirty_type]);
351df4b2
JK		 2350 dirty_i->nr_dirty[dirty_type] = 0;
2351 mutex_unlock(&dirty_i->seglist_lock);
2352}
2353
5ec4e49f 2354static void destroy_victim_secmap(struct f2fs_sb_info *sbi)
351df4b2
JK		 2355{
2356 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
39307a8e 2357 kvfree(dirty_i->victim_secmap);
351df4b2
JK		 2358}
2359
2360static void destroy_dirty_segmap(struct f2fs_sb_info *sbi)
2361{
2362 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
2363 int i;
2364
2365 if (!dirty_i)
2366 return;
2367
2368 /* discard pre-free/dirty segments list */
2369 for (i = 0; i < NR_DIRTY_TYPE; i++)
2370 discard_dirty_segmap(sbi, i);
2371
5ec4e49f 2372 destroy_victim_secmap(sbi);
351df4b2
JK		 2373 SM_I(sbi)->dirty_info = NULL;
2374 kfree(dirty_i);
2375}
2376
2377static void destroy_curseg(struct f2fs_sb_info *sbi)
2378{
2379 struct curseg_info *array = SM_I(sbi)->curseg_array;
2380 int i;
2381
2382 if (!array)
2383 return;
2384 SM_I(sbi)->curseg_array = NULL;
2385 for (i = 0; i < NR_CURSEG_TYPE; i++)
2386 kfree(array[i].sum_blk);
2387 kfree(array);
2388}
2389
2390static void destroy_free_segmap(struct f2fs_sb_info *sbi)
2391{
2392 struct free_segmap_info *free_i = SM_I(sbi)->free_info;
2393 if (!free_i)
2394 return;
2395 SM_I(sbi)->free_info = NULL;
39307a8e
JK		 2396 kvfree(free_i->free_segmap);
2397 kvfree(free_i->free_secmap);
351df4b2
JK		 2398 kfree(free_i);
2399}
2400
2401static void destroy_sit_info(struct f2fs_sb_info *sbi)
2402{
2403 struct sit_info *sit_i = SIT_I(sbi);
2404 unsigned int start;
2405
2406 if (!sit_i)
2407 return;
2408
2409 if (sit_i->sentries) {
7cd8558b 2410 for (start = 0; start < MAIN_SEGS(sbi); start++) {
351df4b2
JK		 2411 kfree(sit_i->sentries[start].cur_valid_map);
2412 kfree(sit_i->sentries[start].ckpt_valid_map);
a66cdd98 2413 kfree(sit_i->sentries[start].discard_map);
351df4b2
JK		 2414 }
2415 }
60a3b782
JK		 2416 kfree(sit_i->tmp_map);
2417
39307a8e
JK		 2418 kvfree(sit_i->sentries);
2419 kvfree(sit_i->sec_entries);
2420 kvfree(sit_i->dirty_sentries_bitmap);
351df4b2
JK		 2421
2422 SM_I(sbi)->sit_info = NULL;
2423 kfree(sit_i->sit_bitmap);
2424 kfree(sit_i);
2425}
2426
2427void destroy_segment_manager(struct f2fs_sb_info *sbi)
2428{
2429 struct f2fs_sm_info *sm_info = SM_I(sbi);
a688b9d9 2430
3b03f724
CY		 2431 if (!sm_info)
2432 return;
2163d198 2433 destroy_flush_cmd_control(sbi);
351df4b2
JK		 2434 destroy_dirty_segmap(sbi);
2435 destroy_curseg(sbi);
2436 destroy_free_segmap(sbi);
2437 destroy_sit_info(sbi);
2438 sbi->sm_info = NULL;
2439 kfree(sm_info);
2440}
7fd9e544
JK		 2441
2442int __init create_segment_manager_caches(void)
2443{
2444 discard_entry_slab = f2fs_kmem_cache_create("discard_entry",
e8512d2e 2445 sizeof(struct discard_entry));
7fd9e544 2446 if (!discard_entry_slab)
184a5cd2
CY		 2447 goto fail;
2448
2449 sit_entry_set_slab = f2fs_kmem_cache_create("sit_entry_set",
c9ee0085 2450 sizeof(struct sit_entry_set));
184a5cd2
CY		 2451 if (!sit_entry_set_slab)
2452 goto destory_discard_entry;
88b88a66
JK		 2453
2454 inmem_entry_slab = f2fs_kmem_cache_create("inmem_page_entry",
2455 sizeof(struct inmem_pages));
2456 if (!inmem_entry_slab)
2457 goto destroy_sit_entry_set;
7fd9e544 2458 return 0;
184a5cd2 2459
88b88a66
JK		 2460destroy_sit_entry_set:
2461 kmem_cache_destroy(sit_entry_set_slab);
184a5cd2
CY		 2462destory_discard_entry:
2463 kmem_cache_destroy(discard_entry_slab);
2464fail:
2465 return -ENOMEM;
7fd9e544
JK		 2466}
2467
2468void destroy_segment_manager_caches(void)
2469{
184a5cd2 2470 kmem_cache_destroy(sit_entry_set_slab);
7fd9e544 2471 kmem_cache_destroy(discard_entry_slab);
88b88a66 2472 kmem_cache_destroy(inmem_entry_slab);
7fd9e544 2473}
Ubuntu Artful kernel mirror