]> git.proxmox.com Git - mirror_ubuntu-jammy-kernel.git/blob - fs/f2fs/segment.c
projects / mirror_ubuntu-jammy-kernel.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
f2fs: introduce discard_unit mount option
[mirror_ubuntu-jammy-kernel.git] / fs / f2fs / segment.c
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * fs/f2fs/segment.c
4 *
5 * Copyright (c) 2012 Samsung Electronics Co., Ltd.
6 * http://www.samsung.com/
7 */
8 #include <linux/fs.h>
9 #include <linux/f2fs_fs.h>
10 #include <linux/bio.h>
11 #include <linux/blkdev.h>
12 #include <linux/prefetch.h>
13 #include <linux/kthread.h>
14 #include <linux/swap.h>
15 #include <linux/timer.h>
16 #include <linux/freezer.h>
17 #include <linux/sched/signal.h>
18
19 #include "f2fs.h"
20 #include "segment.h"
21 #include "node.h"
22 #include "gc.h"
23 #include <trace/events/f2fs.h>
24
25 #define __reverse_ffz(x) __reverse_ffs(~(x))
26
27 static struct kmem_cache *discard_entry_slab;
28 static struct kmem_cache *discard_cmd_slab;
29 static struct kmem_cache *sit_entry_set_slab;
30 static struct kmem_cache *inmem_entry_slab;
31
32 static unsigned long __reverse_ulong(unsigned char *str)
33 {
34 unsigned long tmp = 0;
35 int shift = 24, idx = 0;
36
37 #if BITS_PER_LONG == 64
38 shift = 56;
39 #endif
40 while (shift >= 0) {
41 tmp |= (unsigned long)str[idx++] << shift;
42 shift -= BITS_PER_BYTE;
43 }
44 return tmp;
45 }
46
47 /*
48 * __reverse_ffs is copied from include/asm-generic/bitops/__ffs.h since
49 * MSB and LSB are reversed in a byte by f2fs_set_bit.
50 */
51 static inline unsigned long __reverse_ffs(unsigned long word)
52 {
53 int num = 0;
54
55 #if BITS_PER_LONG == 64
56 if ((word & 0xffffffff00000000UL) == 0)
57 num += 32;
58 else
59 word >>= 32;
60 #endif
61 if ((word & 0xffff0000) == 0)
62 num += 16;
63 else
64 word >>= 16;
65
66 if ((word & 0xff00) == 0)
67 num += 8;
68 else
69 word >>= 8;
70
71 if ((word & 0xf0) == 0)
72 num += 4;
73 else
74 word >>= 4;
75
76 if ((word & 0xc) == 0)
77 num += 2;
78 else
79 word >>= 2;
80
81 if ((word & 0x2) == 0)
82 num += 1;
83 return num;
84 }
85
86 /*
87 * __find_rev_next(_zero)_bit is copied from lib/find_next_bit.c because
88 * f2fs_set_bit makes MSB and LSB reversed in a byte.
89 * @size must be integral times of unsigned long.
90 * Example:
91 * MSB <--> LSB
92 * f2fs_set_bit(0, bitmap) => 1000 0000
93 * f2fs_set_bit(7, bitmap) => 0000 0001
94 */
95 static unsigned long __find_rev_next_bit(const unsigned long *addr,
96 unsigned long size, unsigned long offset)
97 {
98 const unsigned long *p = addr + BIT_WORD(offset);
99 unsigned long result = size;
100 unsigned long tmp;
101
102 if (offset >= size)
103 return size;
104
105 size -= (offset & ~(BITS_PER_LONG - 1));
106 offset %= BITS_PER_LONG;
107
108 while (1) {
109 if (*p == 0)
110 goto pass;
111
112 tmp = __reverse_ulong((unsigned char *)p);
113
114 tmp &= ~0UL >> offset;
115 if (size < BITS_PER_LONG)
116 tmp &= (~0UL << (BITS_PER_LONG - size));
117 if (tmp)
118 goto found;
119 pass:
120 if (size <= BITS_PER_LONG)
121 break;
122 size -= BITS_PER_LONG;
123 offset = 0;
124 p++;
125 }
126 return result;
127 found:
128 return result - size + __reverse_ffs(tmp);
129 }
130
131 static unsigned long __find_rev_next_zero_bit(const unsigned long *addr,
132 unsigned long size, unsigned long offset)
133 {
134 const unsigned long *p = addr + BIT_WORD(offset);
135 unsigned long result = size;
136 unsigned long tmp;
137
138 if (offset >= size)
139 return size;
140
141 size -= (offset & ~(BITS_PER_LONG - 1));
142 offset %= BITS_PER_LONG;
143
144 while (1) {
145 if (*p == ~0UL)
146 goto pass;
147
148 tmp = __reverse_ulong((unsigned char *)p);
149
150 if (offset)
151 tmp |= ~0UL << (BITS_PER_LONG - offset);
152 if (size < BITS_PER_LONG)
153 tmp |= ~0UL >> size;
154 if (tmp != ~0UL)
155 goto found;
156 pass:
157 if (size <= BITS_PER_LONG)
158 break;
159 size -= BITS_PER_LONG;
160 offset = 0;
161 p++;
162 }
163 return result;
164 found:
165 return result - size + __reverse_ffz(tmp);
166 }
167
168 bool f2fs_need_SSR(struct f2fs_sb_info *sbi)
169 {
170 int node_secs = get_blocktype_secs(sbi, F2FS_DIRTY_NODES);
171 int dent_secs = get_blocktype_secs(sbi, F2FS_DIRTY_DENTS);
172 int imeta_secs = get_blocktype_secs(sbi, F2FS_DIRTY_IMETA);
173
174 if (f2fs_lfs_mode(sbi))
175 return false;
176 if (sbi->gc_mode == GC_URGENT_HIGH)
177 return true;
178 if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED)))
179 return true;
180
181 return free_sections(sbi) <= (node_secs + 2 * dent_secs + imeta_secs +
182 SM_I(sbi)->min_ssr_sections + reserved_sections(sbi));
183 }
184
185 void f2fs_register_inmem_page(struct inode *inode, struct page *page)
186 {
187 struct inmem_pages *new;
188
189 set_page_private_atomic(page);
190
191 new = f2fs_kmem_cache_alloc(inmem_entry_slab, GFP_NOFS);
192
193 /* add atomic page indices to the list */
194 new->page = page;
195 INIT_LIST_HEAD(&new->list);
196
197 /* increase reference count with clean state */
198 get_page(page);
199 mutex_lock(&F2FS_I(inode)->inmem_lock);
200 list_add_tail(&new->list, &F2FS_I(inode)->inmem_pages);
201 inc_page_count(F2FS_I_SB(inode), F2FS_INMEM_PAGES);
202 mutex_unlock(&F2FS_I(inode)->inmem_lock);
203
204 trace_f2fs_register_inmem_page(page, INMEM);
205 }
206
207 static int __revoke_inmem_pages(struct inode *inode,
208 struct list_head *head, bool drop, bool recover,
209 bool trylock)
210 {
211 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
212 struct inmem_pages *cur, *tmp;
213 int err = 0;
214
215 list_for_each_entry_safe(cur, tmp, head, list) {
216 struct page *page = cur->page;
217
218 if (drop)
219 trace_f2fs_commit_inmem_page(page, INMEM_DROP);
220
221 if (trylock) {
222 /*
223 * to avoid deadlock in between page lock and
224 * inmem_lock.
225 */
226 if (!trylock_page(page))
227 continue;
228 } else {
229 lock_page(page);
230 }
231
232 f2fs_wait_on_page_writeback(page, DATA, true, true);
233
234 if (recover) {
235 struct dnode_of_data dn;
236 struct node_info ni;
237
238 trace_f2fs_commit_inmem_page(page, INMEM_REVOKE);
239 retry:
240 set_new_dnode(&dn, inode, NULL, NULL, 0);
241 err = f2fs_get_dnode_of_data(&dn, page->index,
242 LOOKUP_NODE);
243 if (err) {
244 if (err == -ENOMEM) {
245 congestion_wait(BLK_RW_ASYNC,
246 DEFAULT_IO_TIMEOUT);
247 cond_resched();
248 goto retry;
249 }
250 err = -EAGAIN;
251 goto next;
252 }
253
254 err = f2fs_get_node_info(sbi, dn.nid, &ni);
255 if (err) {
256 f2fs_put_dnode(&dn);
257 return err;
258 }
259
260 if (cur->old_addr == NEW_ADDR) {
261 f2fs_invalidate_blocks(sbi, dn.data_blkaddr);
262 f2fs_update_data_blkaddr(&dn, NEW_ADDR);
263 } else
264 f2fs_replace_block(sbi, &dn, dn.data_blkaddr,
265 cur->old_addr, ni.version, true, true);
266 f2fs_put_dnode(&dn);
267 }
268 next:
269 /* we don't need to invalidate this in the sccessful status */
270 if (drop || recover) {
271 ClearPageUptodate(page);
272 clear_page_private_gcing(page);
273 }
274 detach_page_private(page);
275 set_page_private(page, 0);
276 f2fs_put_page(page, 1);
277
278 list_del(&cur->list);
279 kmem_cache_free(inmem_entry_slab, cur);
280 dec_page_count(F2FS_I_SB(inode), F2FS_INMEM_PAGES);
281 }
282 return err;
283 }
284
285 void f2fs_drop_inmem_pages_all(struct f2fs_sb_info *sbi, bool gc_failure)
286 {
287 struct list_head *head = &sbi->inode_list[ATOMIC_FILE];
288 struct inode *inode;
289 struct f2fs_inode_info *fi;
290 unsigned int count = sbi->atomic_files;
291 unsigned int looped = 0;
292 next:
293 spin_lock(&sbi->inode_lock[ATOMIC_FILE]);
294 if (list_empty(head)) {
295 spin_unlock(&sbi->inode_lock[ATOMIC_FILE]);
296 return;
297 }
298 fi = list_first_entry(head, struct f2fs_inode_info, inmem_ilist);
299 inode = igrab(&fi->vfs_inode);
300 if (inode)
301 list_move_tail(&fi->inmem_ilist, head);
302 spin_unlock(&sbi->inode_lock[ATOMIC_FILE]);
303
304 if (inode) {
305 if (gc_failure) {
306 if (!fi->i_gc_failures[GC_FAILURE_ATOMIC])
307 goto skip;
308 }
309 set_inode_flag(inode, FI_ATOMIC_REVOKE_REQUEST);
310 f2fs_drop_inmem_pages(inode);
311 skip:
312 iput(inode);
313 }
314 congestion_wait(BLK_RW_ASYNC, DEFAULT_IO_TIMEOUT);
315 cond_resched();
316 if (gc_failure) {
317 if (++looped >= count)
318 return;
319 }
320 goto next;
321 }
322
323 void f2fs_drop_inmem_pages(struct inode *inode)
324 {
325 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
326 struct f2fs_inode_info *fi = F2FS_I(inode);
327
328 do {
329 mutex_lock(&fi->inmem_lock);
330 if (list_empty(&fi->inmem_pages)) {
331 fi->i_gc_failures[GC_FAILURE_ATOMIC] = 0;
332
333 spin_lock(&sbi->inode_lock[ATOMIC_FILE]);
334 if (!list_empty(&fi->inmem_ilist))
335 list_del_init(&fi->inmem_ilist);
336 if (f2fs_is_atomic_file(inode)) {
337 clear_inode_flag(inode, FI_ATOMIC_FILE);
338 sbi->atomic_files--;
339 }
340 spin_unlock(&sbi->inode_lock[ATOMIC_FILE]);
341
342 mutex_unlock(&fi->inmem_lock);
343 break;
344 }
345 __revoke_inmem_pages(inode, &fi->inmem_pages,
346 true, false, true);
347 mutex_unlock(&fi->inmem_lock);
348 } while (1);
349 }
350
351 void f2fs_drop_inmem_page(struct inode *inode, struct page *page)
352 {
353 struct f2fs_inode_info *fi = F2FS_I(inode);
354 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
355 struct list_head *head = &fi->inmem_pages;
356 struct inmem_pages *cur = NULL;
357
358 f2fs_bug_on(sbi, !page_private_atomic(page));
359
360 mutex_lock(&fi->inmem_lock);
361 list_for_each_entry(cur, head, list) {
362 if (cur->page == page)
363 break;
364 }
365
366 f2fs_bug_on(sbi, list_empty(head) || cur->page != page);
367 list_del(&cur->list);
368 mutex_unlock(&fi->inmem_lock);
369
370 dec_page_count(sbi, F2FS_INMEM_PAGES);
371 kmem_cache_free(inmem_entry_slab, cur);
372
373 ClearPageUptodate(page);
374 clear_page_private_atomic(page);
375 f2fs_put_page(page, 0);
376
377 detach_page_private(page);
378 set_page_private(page, 0);
379
380 trace_f2fs_commit_inmem_page(page, INMEM_INVALIDATE);
381 }
382
383 static int __f2fs_commit_inmem_pages(struct inode *inode)
384 {
385 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
386 struct f2fs_inode_info *fi = F2FS_I(inode);
387 struct inmem_pages *cur, *tmp;
388 struct f2fs_io_info fio = {
389 .sbi = sbi,
390 .ino = inode->i_ino,
391 .type = DATA,
392 .op = REQ_OP_WRITE,
393 .op_flags = REQ_SYNC | REQ_PRIO,
394 .io_type = FS_DATA_IO,
395 };
396 struct list_head revoke_list;
397 bool submit_bio = false;
398 int err = 0;
399
400 INIT_LIST_HEAD(&revoke_list);
401
402 list_for_each_entry_safe(cur, tmp, &fi->inmem_pages, list) {
403 struct page *page = cur->page;
404
405 lock_page(page);
406 if (page->mapping == inode->i_mapping) {
407 trace_f2fs_commit_inmem_page(page, INMEM);
408
409 f2fs_wait_on_page_writeback(page, DATA, true, true);
410
411 set_page_dirty(page);
412 if (clear_page_dirty_for_io(page)) {
413 inode_dec_dirty_pages(inode);
414 f2fs_remove_dirty_inode(inode);
415 }
416 retry:
417 fio.page = page;
418 fio.old_blkaddr = NULL_ADDR;
419 fio.encrypted_page = NULL;
420 fio.need_lock = LOCK_DONE;
421 err = f2fs_do_write_data_page(&fio);
422 if (err) {
423 if (err == -ENOMEM) {
424 congestion_wait(BLK_RW_ASYNC,
425 DEFAULT_IO_TIMEOUT);
426 cond_resched();
427 goto retry;
428 }
429 unlock_page(page);
430 break;
431 }
432 /* record old blkaddr for revoking */
433 cur->old_addr = fio.old_blkaddr;
434 submit_bio = true;
435 }
436 unlock_page(page);
437 list_move_tail(&cur->list, &revoke_list);
438 }
439
440 if (submit_bio)
441 f2fs_submit_merged_write_cond(sbi, inode, NULL, 0, DATA);
442
443 if (err) {
444 /*
445 * try to revoke all committed pages, but still we could fail
446 * due to no memory or other reason, if that happened, EAGAIN
447 * will be returned, which means in such case, transaction is
448 * already not integrity, caller should use journal to do the
449 * recovery or rewrite & commit last transaction. For other
450 * error number, revoking was done by filesystem itself.
451 */
452 err = __revoke_inmem_pages(inode, &revoke_list,
453 false, true, false);
454
455 /* drop all uncommitted pages */
456 __revoke_inmem_pages(inode, &fi->inmem_pages,
457 true, false, false);
458 } else {
459 __revoke_inmem_pages(inode, &revoke_list,
460 false, false, false);
461 }
462
463 return err;
464 }
465
466 int f2fs_commit_inmem_pages(struct inode *inode)
467 {
468 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
469 struct f2fs_inode_info *fi = F2FS_I(inode);
470 int err;
471
472 f2fs_balance_fs(sbi, true);
473
474 down_write(&fi->i_gc_rwsem[WRITE]);
475
476 f2fs_lock_op(sbi);
477 set_inode_flag(inode, FI_ATOMIC_COMMIT);
478
479 mutex_lock(&fi->inmem_lock);
480 err = __f2fs_commit_inmem_pages(inode);
481 mutex_unlock(&fi->inmem_lock);
482
483 clear_inode_flag(inode, FI_ATOMIC_COMMIT);
484
485 f2fs_unlock_op(sbi);
486 up_write(&fi->i_gc_rwsem[WRITE]);
487
488 return err;
489 }
490
491 /*
492 * This function balances dirty node and dentry pages.
493 * In addition, it controls garbage collection.
494 */
495 void f2fs_balance_fs(struct f2fs_sb_info *sbi, bool need)
496 {
497 if (time_to_inject(sbi, FAULT_CHECKPOINT)) {
498 f2fs_show_injection_info(sbi, FAULT_CHECKPOINT);
499 f2fs_stop_checkpoint(sbi, false);
500 }
501
502 /* balance_fs_bg is able to be pending */
503 if (need && excess_cached_nats(sbi))
504 f2fs_balance_fs_bg(sbi, false);
505
506 if (!f2fs_is_checkpoint_ready(sbi))
507 return;
508
509 /*
510 * We should do GC or end up with checkpoint, if there are so many dirty
511 * dir/node pages without enough free segments.
512 */
513 if (has_not_enough_free_secs(sbi, 0, 0)) {
514 if (test_opt(sbi, GC_MERGE) && sbi->gc_thread &&
515 sbi->gc_thread->f2fs_gc_task) {
516 DEFINE_WAIT(wait);
517
518 prepare_to_wait(&sbi->gc_thread->fggc_wq, &wait,
519 TASK_UNINTERRUPTIBLE);
520 wake_up(&sbi->gc_thread->gc_wait_queue_head);
521 io_schedule();
522 finish_wait(&sbi->gc_thread->fggc_wq, &wait);
523 } else {
524 down_write(&sbi->gc_lock);
525 f2fs_gc(sbi, false, false, false, NULL_SEGNO);
526 }
527 }
528 }
529
530 void f2fs_balance_fs_bg(struct f2fs_sb_info *sbi, bool from_bg)
531 {
532 if (unlikely(is_sbi_flag_set(sbi, SBI_POR_DOING)))
533 return;
534
535 /* try to shrink extent cache when there is no enough memory */
536 if (!f2fs_available_free_memory(sbi, EXTENT_CACHE))
537 f2fs_shrink_extent_tree(sbi, EXTENT_CACHE_SHRINK_NUMBER);
538
539 /* check the # of cached NAT entries */
540 if (!f2fs_available_free_memory(sbi, NAT_ENTRIES))
541 f2fs_try_to_free_nats(sbi, NAT_ENTRY_PER_BLOCK);
542
543 if (!f2fs_available_free_memory(sbi, FREE_NIDS))
544 f2fs_try_to_free_nids(sbi, MAX_FREE_NIDS);
545 else
546 f2fs_build_free_nids(sbi, false, false);
547
548 if (excess_dirty_nats(sbi) || excess_dirty_nodes(sbi) ||
549 excess_prefree_segs(sbi))
550 goto do_sync;
551
552 /* there is background inflight IO or foreground operation recently */
553 if (is_inflight_io(sbi, REQ_TIME) ||
554 (!f2fs_time_over(sbi, REQ_TIME) && rwsem_is_locked(&sbi->cp_rwsem)))
555 return;
556
557 /* exceed periodical checkpoint timeout threshold */
558 if (f2fs_time_over(sbi, CP_TIME))
559 goto do_sync;
560
561 /* checkpoint is the only way to shrink partial cached entries */
562 if (f2fs_available_free_memory(sbi, NAT_ENTRIES) ||
563 f2fs_available_free_memory(sbi, INO_ENTRIES))
564 return;
565
566 do_sync:
567 if (test_opt(sbi, DATA_FLUSH) && from_bg) {
568 struct blk_plug plug;
569
570 mutex_lock(&sbi->flush_lock);
571
572 blk_start_plug(&plug);
573 f2fs_sync_dirty_inodes(sbi, FILE_INODE);
574 blk_finish_plug(&plug);
575
576 mutex_unlock(&sbi->flush_lock);
577 }
578 f2fs_sync_fs(sbi->sb, true);
579 stat_inc_bg_cp_count(sbi->stat_info);
580 }
581
582 static int __submit_flush_wait(struct f2fs_sb_info *sbi,
583 struct block_device *bdev)
584 {
585 int ret = blkdev_issue_flush(bdev);
586
587 trace_f2fs_issue_flush(bdev, test_opt(sbi, NOBARRIER),
588 test_opt(sbi, FLUSH_MERGE), ret);
589 return ret;
590 }
591
592 static int submit_flush_wait(struct f2fs_sb_info *sbi, nid_t ino)
593 {
594 int ret = 0;
595 int i;
596
597 if (!f2fs_is_multi_device(sbi))
598 return __submit_flush_wait(sbi, sbi->sb->s_bdev);
599
600 for (i = 0; i < sbi->s_ndevs; i++) {
601 if (!f2fs_is_dirty_device(sbi, ino, i, FLUSH_INO))
602 continue;
603 ret = __submit_flush_wait(sbi, FDEV(i).bdev);
604 if (ret)
605 break;
606 }
607 return ret;
608 }
609
610 static int issue_flush_thread(void *data)
611 {
612 struct f2fs_sb_info *sbi = data;
613 struct flush_cmd_control *fcc = SM_I(sbi)->fcc_info;
614 wait_queue_head_t *q = &fcc->flush_wait_queue;
615 repeat:
616 if (kthread_should_stop())
617 return 0;
618
619 if (!llist_empty(&fcc->issue_list)) {
620 struct flush_cmd *cmd, *next;
621 int ret;
622
623 fcc->dispatch_list = llist_del_all(&fcc->issue_list);
624 fcc->dispatch_list = llist_reverse_order(fcc->dispatch_list);
625
626 cmd = llist_entry(fcc->dispatch_list, struct flush_cmd, llnode);
627
628 ret = submit_flush_wait(sbi, cmd->ino);
629 atomic_inc(&fcc->issued_flush);
630
631 llist_for_each_entry_safe(cmd, next,
632 fcc->dispatch_list, llnode) {
633 cmd->ret = ret;
634 complete(&cmd->wait);
635 }
636 fcc->dispatch_list = NULL;
637 }
638
639 wait_event_interruptible(*q,
640 kthread_should_stop() || !llist_empty(&fcc->issue_list));
641 goto repeat;
642 }
643
644 int f2fs_issue_flush(struct f2fs_sb_info *sbi, nid_t ino)
645 {
646 struct flush_cmd_control *fcc = SM_I(sbi)->fcc_info;
647 struct flush_cmd cmd;
648 int ret;
649
650 if (test_opt(sbi, NOBARRIER))
651 return 0;
652
653 if (!test_opt(sbi, FLUSH_MERGE)) {
654 atomic_inc(&fcc->queued_flush);
655 ret = submit_flush_wait(sbi, ino);
656 atomic_dec(&fcc->queued_flush);
657 atomic_inc(&fcc->issued_flush);
658 return ret;
659 }
660
661 if (atomic_inc_return(&fcc->queued_flush) == 1 ||
662 f2fs_is_multi_device(sbi)) {
663 ret = submit_flush_wait(sbi, ino);
664 atomic_dec(&fcc->queued_flush);
665
666 atomic_inc(&fcc->issued_flush);
667 return ret;
668 }
669
670 cmd.ino = ino;
671 init_completion(&cmd.wait);
672
673 llist_add(&cmd.llnode, &fcc->issue_list);
674
675 /*
676 * update issue_list before we wake up issue_flush thread, this
677 * smp_mb() pairs with another barrier in ___wait_event(), see
678 * more details in comments of waitqueue_active().
679 */
680 smp_mb();
681
682 if (waitqueue_active(&fcc->flush_wait_queue))
683 wake_up(&fcc->flush_wait_queue);
684
685 if (fcc->f2fs_issue_flush) {
686 wait_for_completion(&cmd.wait);
687 atomic_dec(&fcc->queued_flush);
688 } else {
689 struct llist_node *list;
690
691 list = llist_del_all(&fcc->issue_list);
692 if (!list) {
693 wait_for_completion(&cmd.wait);
694 atomic_dec(&fcc->queued_flush);
695 } else {
696 struct flush_cmd *tmp, *next;
697
698 ret = submit_flush_wait(sbi, ino);
699
700 llist_for_each_entry_safe(tmp, next, list, llnode) {
701 if (tmp == &cmd) {
702 cmd.ret = ret;
703 atomic_dec(&fcc->queued_flush);
704 continue;
705 }
706 tmp->ret = ret;
707 complete(&tmp->wait);
708 }
709 }
710 }
711
712 return cmd.ret;
713 }
714
715 int f2fs_create_flush_cmd_control(struct f2fs_sb_info *sbi)
716 {
717 dev_t dev = sbi->sb->s_bdev->bd_dev;
718 struct flush_cmd_control *fcc;
719 int err = 0;
720
721 if (SM_I(sbi)->fcc_info) {
722 fcc = SM_I(sbi)->fcc_info;
723 if (fcc->f2fs_issue_flush)
724 return err;
725 goto init_thread;
726 }
727
728 fcc = f2fs_kzalloc(sbi, sizeof(struct flush_cmd_control), GFP_KERNEL);
729 if (!fcc)
730 return -ENOMEM;
731 atomic_set(&fcc->issued_flush, 0);
732 atomic_set(&fcc->queued_flush, 0);
733 init_waitqueue_head(&fcc->flush_wait_queue);
734 init_llist_head(&fcc->issue_list);
735 SM_I(sbi)->fcc_info = fcc;
736 if (!test_opt(sbi, FLUSH_MERGE))
737 return err;
738
739 init_thread:
740 fcc->f2fs_issue_flush = kthread_run(issue_flush_thread, sbi,
741 "f2fs_flush-%u:%u", MAJOR(dev), MINOR(dev));
742 if (IS_ERR(fcc->f2fs_issue_flush)) {
743 err = PTR_ERR(fcc->f2fs_issue_flush);
744 kfree(fcc);
745 SM_I(sbi)->fcc_info = NULL;
746 return err;
747 }
748
749 return err;
750 }
751
752 void f2fs_destroy_flush_cmd_control(struct f2fs_sb_info *sbi, bool free)
753 {
754 struct flush_cmd_control *fcc = SM_I(sbi)->fcc_info;
755
756 if (fcc && fcc->f2fs_issue_flush) {
757 struct task_struct *flush_thread = fcc->f2fs_issue_flush;
758
759 fcc->f2fs_issue_flush = NULL;
760 kthread_stop(flush_thread);
761 }
762 if (free) {
763 kfree(fcc);
764 SM_I(sbi)->fcc_info = NULL;
765 }
766 }
767
768 int f2fs_flush_device_cache(struct f2fs_sb_info *sbi)
769 {
770 int ret = 0, i;
771
772 if (!f2fs_is_multi_device(sbi))
773 return 0;
774
775 if (test_opt(sbi, NOBARRIER))
776 return 0;
777
778 for (i = 1; i < sbi->s_ndevs; i++) {
779 if (!f2fs_test_bit(i, (char *)&sbi->dirty_device))
780 continue;
781 ret = __submit_flush_wait(sbi, FDEV(i).bdev);
782 if (ret)
783 break;
784
785 spin_lock(&sbi->dev_lock);
786 f2fs_clear_bit(i, (char *)&sbi->dirty_device);
787 spin_unlock(&sbi->dev_lock);
788 }
789
790 return ret;
791 }
792
793 static void __locate_dirty_segment(struct f2fs_sb_info *sbi, unsigned int segno,
794 enum dirty_type dirty_type)
795 {
796 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
797
798 /* need not be added */
799 if (IS_CURSEG(sbi, segno))
800 return;
801
802 if (!test_and_set_bit(segno, dirty_i->dirty_segmap[dirty_type]))
803 dirty_i->nr_dirty[dirty_type]++;
804
805 if (dirty_type == DIRTY) {
806 struct seg_entry *sentry = get_seg_entry(sbi, segno);
807 enum dirty_type t = sentry->type;
808
809 if (unlikely(t >= DIRTY)) {
810 f2fs_bug_on(sbi, 1);
811 return;
812 }
813 if (!test_and_set_bit(segno, dirty_i->dirty_segmap[t]))
814 dirty_i->nr_dirty[t]++;
815
816 if (__is_large_section(sbi)) {
817 unsigned int secno = GET_SEC_FROM_SEG(sbi, segno);
818 block_t valid_blocks =
819 get_valid_blocks(sbi, segno, true);
820
821 f2fs_bug_on(sbi, unlikely(!valid_blocks ||
822 valid_blocks == BLKS_PER_SEC(sbi)));
823
824 if (!IS_CURSEC(sbi, secno))
825 set_bit(secno, dirty_i->dirty_secmap);
826 }
827 }
828 }
829
830 static void __remove_dirty_segment(struct f2fs_sb_info *sbi, unsigned int segno,
831 enum dirty_type dirty_type)
832 {
833 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
834 block_t valid_blocks;
835
836 if (test_and_clear_bit(segno, dirty_i->dirty_segmap[dirty_type]))
837 dirty_i->nr_dirty[dirty_type]--;
838
839 if (dirty_type == DIRTY) {
840 struct seg_entry *sentry = get_seg_entry(sbi, segno);
841 enum dirty_type t = sentry->type;
842
843 if (test_and_clear_bit(segno, dirty_i->dirty_segmap[t]))
844 dirty_i->nr_dirty[t]--;
845
846 valid_blocks = get_valid_blocks(sbi, segno, true);
847 if (valid_blocks == 0) {
848 clear_bit(GET_SEC_FROM_SEG(sbi, segno),
849 dirty_i->victim_secmap);
850 #ifdef CONFIG_F2FS_CHECK_FS
851 clear_bit(segno, SIT_I(sbi)->invalid_segmap);
852 #endif
853 }
854 if (__is_large_section(sbi)) {
855 unsigned int secno = GET_SEC_FROM_SEG(sbi, segno);
856
857 if (!valid_blocks ||
858 valid_blocks == BLKS_PER_SEC(sbi)) {
859 clear_bit(secno, dirty_i->dirty_secmap);
860 return;
861 }
862
863 if (!IS_CURSEC(sbi, secno))
864 set_bit(secno, dirty_i->dirty_secmap);
865 }
866 }
867 }
868
869 /*
870 * Should not occur error such as -ENOMEM.
871 * Adding dirty entry into seglist is not critical operation.
872 * If a given segment is one of current working segments, it won't be added.
873 */
874 static void locate_dirty_segment(struct f2fs_sb_info *sbi, unsigned int segno)
875 {
876 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
877 unsigned short valid_blocks, ckpt_valid_blocks;
878 unsigned int usable_blocks;
879
880 if (segno == NULL_SEGNO || IS_CURSEG(sbi, segno))
881 return;
882
883 usable_blocks = f2fs_usable_blks_in_seg(sbi, segno);
884 mutex_lock(&dirty_i->seglist_lock);
885
886 valid_blocks = get_valid_blocks(sbi, segno, false);
887 ckpt_valid_blocks = get_ckpt_valid_blocks(sbi, segno, false);
888
889 if (valid_blocks == 0 && (!is_sbi_flag_set(sbi, SBI_CP_DISABLED) ||
890 ckpt_valid_blocks == usable_blocks)) {
891 __locate_dirty_segment(sbi, segno, PRE);
892 __remove_dirty_segment(sbi, segno, DIRTY);
893 } else if (valid_blocks < usable_blocks) {
894 __locate_dirty_segment(sbi, segno, DIRTY);
895 } else {
896 /* Recovery routine with SSR needs this */
897 __remove_dirty_segment(sbi, segno, DIRTY);
898 }
899
900 mutex_unlock(&dirty_i->seglist_lock);
901 }
902
903 /* This moves currently empty dirty blocks to prefree. Must hold seglist_lock */
904 void f2fs_dirty_to_prefree(struct f2fs_sb_info *sbi)
905 {
906 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
907 unsigned int segno;
908
909 mutex_lock(&dirty_i->seglist_lock);
910 for_each_set_bit(segno, dirty_i->dirty_segmap[DIRTY], MAIN_SEGS(sbi)) {
911 if (get_valid_blocks(sbi, segno, false))
912 continue;
913 if (IS_CURSEG(sbi, segno))
914 continue;
915 __locate_dirty_segment(sbi, segno, PRE);
916 __remove_dirty_segment(sbi, segno, DIRTY);
917 }
918 mutex_unlock(&dirty_i->seglist_lock);
919 }
920
921 block_t f2fs_get_unusable_blocks(struct f2fs_sb_info *sbi)
922 {
923 int ovp_hole_segs =
924 (overprovision_segments(sbi) - reserved_segments(sbi));
925 block_t ovp_holes = ovp_hole_segs << sbi->log_blocks_per_seg;
926 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
927 block_t holes[2] = {0, 0}; /* DATA and NODE */
928 block_t unusable;
929 struct seg_entry *se;
930 unsigned int segno;
931
932 mutex_lock(&dirty_i->seglist_lock);
933 for_each_set_bit(segno, dirty_i->dirty_segmap[DIRTY], MAIN_SEGS(sbi)) {
934 se = get_seg_entry(sbi, segno);
935 if (IS_NODESEG(se->type))
936 holes[NODE] += f2fs_usable_blks_in_seg(sbi, segno) -
937 se->valid_blocks;
938 else
939 holes[DATA] += f2fs_usable_blks_in_seg(sbi, segno) -
940 se->valid_blocks;
941 }
942 mutex_unlock(&dirty_i->seglist_lock);
943
944 unusable = holes[DATA] > holes[NODE] ? holes[DATA] : holes[NODE];
945 if (unusable > ovp_holes)
946 return unusable - ovp_holes;
947 return 0;
948 }
949
950 int f2fs_disable_cp_again(struct f2fs_sb_info *sbi, block_t unusable)
951 {
952 int ovp_hole_segs =
953 (overprovision_segments(sbi) - reserved_segments(sbi));
954 if (unusable > F2FS_OPTION(sbi).unusable_cap)
955 return -EAGAIN;
956 if (is_sbi_flag_set(sbi, SBI_CP_DISABLED_QUICK) &&
957 dirty_segments(sbi) > ovp_hole_segs)
958 return -EAGAIN;
959 return 0;
960 }
961
962 /* This is only used by SBI_CP_DISABLED */
963 static unsigned int get_free_segment(struct f2fs_sb_info *sbi)
964 {
965 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
966 unsigned int segno = 0;
967
968 mutex_lock(&dirty_i->seglist_lock);
969 for_each_set_bit(segno, dirty_i->dirty_segmap[DIRTY], MAIN_SEGS(sbi)) {
970 if (get_valid_blocks(sbi, segno, false))
971 continue;
972 if (get_ckpt_valid_blocks(sbi, segno, false))
973 continue;
974 mutex_unlock(&dirty_i->seglist_lock);
975 return segno;
976 }
977 mutex_unlock(&dirty_i->seglist_lock);
978 return NULL_SEGNO;
979 }
980
981 static struct discard_cmd *__create_discard_cmd(struct f2fs_sb_info *sbi,
982 struct block_device *bdev, block_t lstart,
983 block_t start, block_t len)
984 {
985 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
986 struct list_head *pend_list;
987 struct discard_cmd *dc;
988
989 f2fs_bug_on(sbi, !len);
990
991 pend_list = &dcc->pend_list[plist_idx(len)];
992
993 dc = f2fs_kmem_cache_alloc(discard_cmd_slab, GFP_NOFS);
994 INIT_LIST_HEAD(&dc->list);
995 dc->bdev = bdev;
996 dc->lstart = lstart;
997 dc->start = start;
998 dc->len = len;
999 dc->ref = 0;
1000 dc->state = D_PREP;
1001 dc->queued = 0;
1002 dc->error = 0;
1003 init_completion(&dc->wait);
1004 list_add_tail(&dc->list, pend_list);
1005 spin_lock_init(&dc->lock);
1006 dc->bio_ref = 0;
1007 atomic_inc(&dcc->discard_cmd_cnt);
1008 dcc->undiscard_blks += len;
1009
1010 return dc;
1011 }
1012
1013 static struct discard_cmd *__attach_discard_cmd(struct f2fs_sb_info *sbi,
1014 struct block_device *bdev, block_t lstart,
1015 block_t start, block_t len,
1016 struct rb_node *parent, struct rb_node **p,
1017 bool leftmost)
1018 {
1019 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1020 struct discard_cmd *dc;
1021
1022 dc = __create_discard_cmd(sbi, bdev, lstart, start, len);
1023
1024 rb_link_node(&dc->rb_node, parent, p);
1025 rb_insert_color_cached(&dc->rb_node, &dcc->root, leftmost);
1026
1027 return dc;
1028 }
1029
1030 static void __detach_discard_cmd(struct discard_cmd_control *dcc,
1031 struct discard_cmd *dc)
1032 {
1033 if (dc->state == D_DONE)
1034 atomic_sub(dc->queued, &dcc->queued_discard);
1035
1036 list_del(&dc->list);
1037 rb_erase_cached(&dc->rb_node, &dcc->root);
1038 dcc->undiscard_blks -= dc->len;
1039
1040 kmem_cache_free(discard_cmd_slab, dc);
1041
1042 atomic_dec(&dcc->discard_cmd_cnt);
1043 }
1044
1045 static void __remove_discard_cmd(struct f2fs_sb_info *sbi,
1046 struct discard_cmd *dc)
1047 {
1048 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1049 unsigned long flags;
1050
1051 trace_f2fs_remove_discard(dc->bdev, dc->start, dc->len);
1052
1053 spin_lock_irqsave(&dc->lock, flags);
1054 if (dc->bio_ref) {
1055 spin_unlock_irqrestore(&dc->lock, flags);
1056 return;
1057 }
1058 spin_unlock_irqrestore(&dc->lock, flags);
1059
1060 f2fs_bug_on(sbi, dc->ref);
1061
1062 if (dc->error == -EOPNOTSUPP)
1063 dc->error = 0;
1064
1065 if (dc->error)
1066 printk_ratelimited(
1067 "%sF2FS-fs (%s): Issue discard(%u, %u, %u) failed, ret: %d",
1068 KERN_INFO, sbi->sb->s_id,
1069 dc->lstart, dc->start, dc->len, dc->error);
1070 __detach_discard_cmd(dcc, dc);
1071 }
1072
1073 static void f2fs_submit_discard_endio(struct bio *bio)
1074 {
1075 struct discard_cmd *dc = (struct discard_cmd *)bio->bi_private;
1076 unsigned long flags;
1077
1078 spin_lock_irqsave(&dc->lock, flags);
1079 if (!dc->error)
1080 dc->error = blk_status_to_errno(bio->bi_status);
1081 dc->bio_ref--;
1082 if (!dc->bio_ref && dc->state == D_SUBMIT) {
1083 dc->state = D_DONE;
1084 complete_all(&dc->wait);
1085 }
1086 spin_unlock_irqrestore(&dc->lock, flags);
1087 bio_put(bio);
1088 }
1089
1090 static void __check_sit_bitmap(struct f2fs_sb_info *sbi,
1091 block_t start, block_t end)
1092 {
1093 #ifdef CONFIG_F2FS_CHECK_FS
1094 struct seg_entry *sentry;
1095 unsigned int segno;
1096 block_t blk = start;
1097 unsigned long offset, size, max_blocks = sbi->blocks_per_seg;
1098 unsigned long *map;
1099
1100 while (blk < end) {
1101 segno = GET_SEGNO(sbi, blk);
1102 sentry = get_seg_entry(sbi, segno);
1103 offset = GET_BLKOFF_FROM_SEG0(sbi, blk);
1104
1105 if (end < START_BLOCK(sbi, segno + 1))
1106 size = GET_BLKOFF_FROM_SEG0(sbi, end);
1107 else
1108 size = max_blocks;
1109 map = (unsigned long *)(sentry->cur_valid_map);
1110 offset = __find_rev_next_bit(map, size, offset);
1111 f2fs_bug_on(sbi, offset != size);
1112 blk = START_BLOCK(sbi, segno + 1);
1113 }
1114 #endif
1115 }
1116
1117 static void __init_discard_policy(struct f2fs_sb_info *sbi,
1118 struct discard_policy *dpolicy,
1119 int discard_type, unsigned int granularity)
1120 {
1121 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1122
1123 /* common policy */
1124 dpolicy->type = discard_type;
1125 dpolicy->sync = true;
1126 dpolicy->ordered = false;
1127 dpolicy->granularity = granularity;
1128
1129 dpolicy->max_requests = DEF_MAX_DISCARD_REQUEST;
1130 dpolicy->io_aware_gran = MAX_PLIST_NUM;
1131 dpolicy->timeout = false;
1132
1133 if (discard_type == DPOLICY_BG) {
1134 dpolicy->min_interval = DEF_MIN_DISCARD_ISSUE_TIME;
1135 dpolicy->mid_interval = DEF_MID_DISCARD_ISSUE_TIME;
1136 dpolicy->max_interval = DEF_MAX_DISCARD_ISSUE_TIME;
1137 dpolicy->io_aware = true;
1138 dpolicy->sync = false;
1139 dpolicy->ordered = true;
1140 if (utilization(sbi) > DEF_DISCARD_URGENT_UTIL) {
1141 dpolicy->granularity = 1;
1142 if (atomic_read(&dcc->discard_cmd_cnt))
1143 dpolicy->max_interval =
1144 DEF_MIN_DISCARD_ISSUE_TIME;
1145 }
1146 } else if (discard_type == DPOLICY_FORCE) {
1147 dpolicy->min_interval = DEF_MIN_DISCARD_ISSUE_TIME;
1148 dpolicy->mid_interval = DEF_MID_DISCARD_ISSUE_TIME;
1149 dpolicy->max_interval = DEF_MAX_DISCARD_ISSUE_TIME;
1150 dpolicy->io_aware = false;
1151 } else if (discard_type == DPOLICY_FSTRIM) {
1152 dpolicy->io_aware = false;
1153 } else if (discard_type == DPOLICY_UMOUNT) {
1154 dpolicy->io_aware = false;
1155 /* we need to issue all to keep CP_TRIMMED_FLAG */
1156 dpolicy->granularity = 1;
1157 dpolicy->timeout = true;
1158 }
1159 }
1160
1161 static void __update_discard_tree_range(struct f2fs_sb_info *sbi,
1162 struct block_device *bdev, block_t lstart,
1163 block_t start, block_t len);
1164 /* this function is copied from blkdev_issue_discard from block/blk-lib.c */
1165 static int __submit_discard_cmd(struct f2fs_sb_info *sbi,
1166 struct discard_policy *dpolicy,
1167 struct discard_cmd *dc,
1168 unsigned int *issued)
1169 {
1170 struct block_device *bdev = dc->bdev;
1171 struct request_queue *q = bdev_get_queue(bdev);
1172 unsigned int max_discard_blocks =
1173 SECTOR_TO_BLOCK(q->limits.max_discard_sectors);
1174 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1175 struct list_head *wait_list = (dpolicy->type == DPOLICY_FSTRIM) ?
1176 &(dcc->fstrim_list) : &(dcc->wait_list);
1177 int flag = dpolicy->sync ? REQ_SYNC : 0;
1178 block_t lstart, start, len, total_len;
1179 int err = 0;
1180
1181 if (dc->state != D_PREP)
1182 return 0;
1183
1184 if (is_sbi_flag_set(sbi, SBI_NEED_FSCK))
1185 return 0;
1186
1187 trace_f2fs_issue_discard(bdev, dc->start, dc->len);
1188
1189 lstart = dc->lstart;
1190 start = dc->start;
1191 len = dc->len;
1192 total_len = len;
1193
1194 dc->len = 0;
1195
1196 while (total_len && *issued < dpolicy->max_requests && !err) {
1197 struct bio *bio = NULL;
1198 unsigned long flags;
1199 bool last = true;
1200
1201 if (len > max_discard_blocks) {
1202 len = max_discard_blocks;
1203 last = false;
1204 }
1205
1206 (*issued)++;
1207 if (*issued == dpolicy->max_requests)
1208 last = true;
1209
1210 dc->len += len;
1211
1212 if (time_to_inject(sbi, FAULT_DISCARD)) {
1213 f2fs_show_injection_info(sbi, FAULT_DISCARD);
1214 err = -EIO;
1215 goto submit;
1216 }
1217 err = __blkdev_issue_discard(bdev,
1218 SECTOR_FROM_BLOCK(start),
1219 SECTOR_FROM_BLOCK(len),
1220 GFP_NOFS, 0, &bio);
1221 submit:
1222 if (err) {
1223 spin_lock_irqsave(&dc->lock, flags);
1224 if (dc->state == D_PARTIAL)
1225 dc->state = D_SUBMIT;
1226 spin_unlock_irqrestore(&dc->lock, flags);
1227
1228 break;
1229 }
1230
1231 f2fs_bug_on(sbi, !bio);
1232
1233 /*
1234 * should keep before submission to avoid D_DONE
1235 * right away
1236 */
1237 spin_lock_irqsave(&dc->lock, flags);
1238 if (last)
1239 dc->state = D_SUBMIT;
1240 else
1241 dc->state = D_PARTIAL;
1242 dc->bio_ref++;
1243 spin_unlock_irqrestore(&dc->lock, flags);
1244
1245 atomic_inc(&dcc->queued_discard);
1246 dc->queued++;
1247 list_move_tail(&dc->list, wait_list);
1248
1249 /* sanity check on discard range */
1250 __check_sit_bitmap(sbi, lstart, lstart + len);
1251
1252 bio->bi_private = dc;
1253 bio->bi_end_io = f2fs_submit_discard_endio;
1254 bio->bi_opf |= flag;
1255 submit_bio(bio);
1256
1257 atomic_inc(&dcc->issued_discard);
1258
1259 f2fs_update_iostat(sbi, FS_DISCARD, 1);
1260
1261 lstart += len;
1262 start += len;
1263 total_len -= len;
1264 len = total_len;
1265 }
1266
1267 if (!err && len) {
1268 dcc->undiscard_blks -= len;
1269 __update_discard_tree_range(sbi, bdev, lstart, start, len);
1270 }
1271 return err;
1272 }
1273
1274 static void __insert_discard_tree(struct f2fs_sb_info *sbi,
1275 struct block_device *bdev, block_t lstart,
1276 block_t start, block_t len,
1277 struct rb_node **insert_p,
1278 struct rb_node *insert_parent)
1279 {
1280 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1281 struct rb_node **p;
1282 struct rb_node *parent = NULL;
1283 bool leftmost = true;
1284
1285 if (insert_p && insert_parent) {
1286 parent = insert_parent;
1287 p = insert_p;
1288 goto do_insert;
1289 }
1290
1291 p = f2fs_lookup_rb_tree_for_insert(sbi, &dcc->root, &parent,
1292 lstart, &leftmost);
1293 do_insert:
1294 __attach_discard_cmd(sbi, bdev, lstart, start, len, parent,
1295 p, leftmost);
1296 }
1297
1298 static void __relocate_discard_cmd(struct discard_cmd_control *dcc,
1299 struct discard_cmd *dc)
1300 {
1301 list_move_tail(&dc->list, &dcc->pend_list[plist_idx(dc->len)]);
1302 }
1303
1304 static void __punch_discard_cmd(struct f2fs_sb_info *sbi,
1305 struct discard_cmd *dc, block_t blkaddr)
1306 {
1307 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1308 struct discard_info di = dc->di;
1309 bool modified = false;
1310
1311 if (dc->state == D_DONE || dc->len == 1) {
1312 __remove_discard_cmd(sbi, dc);
1313 return;
1314 }
1315
1316 dcc->undiscard_blks -= di.len;
1317
1318 if (blkaddr > di.lstart) {
1319 dc->len = blkaddr - dc->lstart;
1320 dcc->undiscard_blks += dc->len;
1321 __relocate_discard_cmd(dcc, dc);
1322 modified = true;
1323 }
1324
1325 if (blkaddr < di.lstart + di.len - 1) {
1326 if (modified) {
1327 __insert_discard_tree(sbi, dc->bdev, blkaddr + 1,
1328 di.start + blkaddr + 1 - di.lstart,
1329 di.lstart + di.len - 1 - blkaddr,
1330 NULL, NULL);
1331 } else {
1332 dc->lstart++;
1333 dc->len--;
1334 dc->start++;
1335 dcc->undiscard_blks += dc->len;
1336 __relocate_discard_cmd(dcc, dc);
1337 }
1338 }
1339 }
1340
1341 static void __update_discard_tree_range(struct f2fs_sb_info *sbi,
1342 struct block_device *bdev, block_t lstart,
1343 block_t start, block_t len)
1344 {
1345 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1346 struct discard_cmd *prev_dc = NULL, *next_dc = NULL;
1347 struct discard_cmd *dc;
1348 struct discard_info di = {0};
1349 struct rb_node **insert_p = NULL, *insert_parent = NULL;
1350 struct request_queue *q = bdev_get_queue(bdev);
1351 unsigned int max_discard_blocks =
1352 SECTOR_TO_BLOCK(q->limits.max_discard_sectors);
1353 block_t end = lstart + len;
1354
1355 dc = (struct discard_cmd *)f2fs_lookup_rb_tree_ret(&dcc->root,
1356 NULL, lstart,
1357 (struct rb_entry **)&prev_dc,
1358 (struct rb_entry **)&next_dc,
1359 &insert_p, &insert_parent, true, NULL);
1360 if (dc)
1361 prev_dc = dc;
1362
1363 if (!prev_dc) {
1364 di.lstart = lstart;
1365 di.len = next_dc ? next_dc->lstart - lstart : len;
1366 di.len = min(di.len, len);
1367 di.start = start;
1368 }
1369
1370 while (1) {
1371 struct rb_node *node;
1372 bool merged = false;
1373 struct discard_cmd *tdc = NULL;
1374
1375 if (prev_dc) {
1376 di.lstart = prev_dc->lstart + prev_dc->len;
1377 if (di.lstart < lstart)
1378 di.lstart = lstart;
1379 if (di.lstart >= end)
1380 break;
1381
1382 if (!next_dc || next_dc->lstart > end)
1383 di.len = end - di.lstart;
1384 else
1385 di.len = next_dc->lstart - di.lstart;
1386 di.start = start + di.lstart - lstart;
1387 }
1388
1389 if (!di.len)
1390 goto next;
1391
1392 if (prev_dc && prev_dc->state == D_PREP &&
1393 prev_dc->bdev == bdev &&
1394 __is_discard_back_mergeable(&di, &prev_dc->di,
1395 max_discard_blocks)) {
1396 prev_dc->di.len += di.len;
1397 dcc->undiscard_blks += di.len;
1398 __relocate_discard_cmd(dcc, prev_dc);
1399 di = prev_dc->di;
1400 tdc = prev_dc;
1401 merged = true;
1402 }
1403
1404 if (next_dc && next_dc->state == D_PREP &&
1405 next_dc->bdev == bdev &&
1406 __is_discard_front_mergeable(&di, &next_dc->di,
1407 max_discard_blocks)) {
1408 next_dc->di.lstart = di.lstart;
1409 next_dc->di.len += di.len;
1410 next_dc->di.start = di.start;
1411 dcc->undiscard_blks += di.len;
1412 __relocate_discard_cmd(dcc, next_dc);
1413 if (tdc)
1414 __remove_discard_cmd(sbi, tdc);
1415 merged = true;
1416 }
1417
1418 if (!merged) {
1419 __insert_discard_tree(sbi, bdev, di.lstart, di.start,
1420 di.len, NULL, NULL);
1421 }
1422 next:
1423 prev_dc = next_dc;
1424 if (!prev_dc)
1425 break;
1426
1427 node = rb_next(&prev_dc->rb_node);
1428 next_dc = rb_entry_safe(node, struct discard_cmd, rb_node);
1429 }
1430 }
1431
1432 static int __queue_discard_cmd(struct f2fs_sb_info *sbi,
1433 struct block_device *bdev, block_t blkstart, block_t blklen)
1434 {
1435 block_t lblkstart = blkstart;
1436
1437 if (!f2fs_bdev_support_discard(bdev))
1438 return 0;
1439
1440 trace_f2fs_queue_discard(bdev, blkstart, blklen);
1441
1442 if (f2fs_is_multi_device(sbi)) {
1443 int devi = f2fs_target_device_index(sbi, blkstart);
1444
1445 blkstart -= FDEV(devi).start_blk;
1446 }
1447 mutex_lock(&SM_I(sbi)->dcc_info->cmd_lock);
1448 __update_discard_tree_range(sbi, bdev, lblkstart, blkstart, blklen);
1449 mutex_unlock(&SM_I(sbi)->dcc_info->cmd_lock);
1450 return 0;
1451 }
1452
1453 static unsigned int __issue_discard_cmd_orderly(struct f2fs_sb_info *sbi,
1454 struct discard_policy *dpolicy)
1455 {
1456 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1457 struct discard_cmd *prev_dc = NULL, *next_dc = NULL;
1458 struct rb_node **insert_p = NULL, *insert_parent = NULL;
1459 struct discard_cmd *dc;
1460 struct blk_plug plug;
1461 unsigned int pos = dcc->next_pos;
1462 unsigned int issued = 0;
1463 bool io_interrupted = false;
1464
1465 mutex_lock(&dcc->cmd_lock);
1466 dc = (struct discard_cmd *)f2fs_lookup_rb_tree_ret(&dcc->root,
1467 NULL, pos,
1468 (struct rb_entry **)&prev_dc,
1469 (struct rb_entry **)&next_dc,
1470 &insert_p, &insert_parent, true, NULL);
1471 if (!dc)
1472 dc = next_dc;
1473
1474 blk_start_plug(&plug);
1475
1476 while (dc) {
1477 struct rb_node *node;
1478 int err = 0;
1479
1480 if (dc->state != D_PREP)
1481 goto next;
1482
1483 if (dpolicy->io_aware && !is_idle(sbi, DISCARD_TIME)) {
1484 io_interrupted = true;
1485 break;
1486 }
1487
1488 dcc->next_pos = dc->lstart + dc->len;
1489 err = __submit_discard_cmd(sbi, dpolicy, dc, &issued);
1490
1491 if (issued >= dpolicy->max_requests)
1492 break;
1493 next:
1494 node = rb_next(&dc->rb_node);
1495 if (err)
1496 __remove_discard_cmd(sbi, dc);
1497 dc = rb_entry_safe(node, struct discard_cmd, rb_node);
1498 }
1499
1500 blk_finish_plug(&plug);
1501
1502 if (!dc)
1503 dcc->next_pos = 0;
1504
1505 mutex_unlock(&dcc->cmd_lock);
1506
1507 if (!issued && io_interrupted)
1508 issued = -1;
1509
1510 return issued;
1511 }
1512 static unsigned int __wait_all_discard_cmd(struct f2fs_sb_info *sbi,
1513 struct discard_policy *dpolicy);
1514
1515 static int __issue_discard_cmd(struct f2fs_sb_info *sbi,
1516 struct discard_policy *dpolicy)
1517 {
1518 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1519 struct list_head *pend_list;
1520 struct discard_cmd *dc, *tmp;
1521 struct blk_plug plug;
1522 int i, issued;
1523 bool io_interrupted = false;
1524
1525 if (dpolicy->timeout)
1526 f2fs_update_time(sbi, UMOUNT_DISCARD_TIMEOUT);
1527
1528 retry:
1529 issued = 0;
1530 for (i = MAX_PLIST_NUM - 1; i >= 0; i--) {
1531 if (dpolicy->timeout &&
1532 f2fs_time_over(sbi, UMOUNT_DISCARD_TIMEOUT))
1533 break;
1534
1535 if (i + 1 < dpolicy->granularity)
1536 break;
1537
1538 if (i < DEFAULT_DISCARD_GRANULARITY && dpolicy->ordered)
1539 return __issue_discard_cmd_orderly(sbi, dpolicy);
1540
1541 pend_list = &dcc->pend_list[i];
1542
1543 mutex_lock(&dcc->cmd_lock);
1544 if (list_empty(pend_list))
1545 goto next;
1546 if (unlikely(dcc->rbtree_check))
1547 f2fs_bug_on(sbi, !f2fs_check_rb_tree_consistence(sbi,
1548 &dcc->root, false));
1549 blk_start_plug(&plug);
1550 list_for_each_entry_safe(dc, tmp, pend_list, list) {
1551 f2fs_bug_on(sbi, dc->state != D_PREP);
1552
1553 if (dpolicy->timeout &&
1554 f2fs_time_over(sbi, UMOUNT_DISCARD_TIMEOUT))
1555 break;
1556
1557 if (dpolicy->io_aware && i < dpolicy->io_aware_gran &&
1558 !is_idle(sbi, DISCARD_TIME)) {
1559 io_interrupted = true;
1560 break;
1561 }
1562
1563 __submit_discard_cmd(sbi, dpolicy, dc, &issued);
1564
1565 if (issued >= dpolicy->max_requests)
1566 break;
1567 }
1568 blk_finish_plug(&plug);
1569 next:
1570 mutex_unlock(&dcc->cmd_lock);
1571
1572 if (issued >= dpolicy->max_requests || io_interrupted)
1573 break;
1574 }
1575
1576 if (dpolicy->type == DPOLICY_UMOUNT && issued) {
1577 __wait_all_discard_cmd(sbi, dpolicy);
1578 goto retry;
1579 }
1580
1581 if (!issued && io_interrupted)
1582 issued = -1;
1583
1584 return issued;
1585 }
1586
1587 static bool __drop_discard_cmd(struct f2fs_sb_info *sbi)
1588 {
1589 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1590 struct list_head *pend_list;
1591 struct discard_cmd *dc, *tmp;
1592 int i;
1593 bool dropped = false;
1594
1595 mutex_lock(&dcc->cmd_lock);
1596 for (i = MAX_PLIST_NUM - 1; i >= 0; i--) {
1597 pend_list = &dcc->pend_list[i];
1598 list_for_each_entry_safe(dc, tmp, pend_list, list) {
1599 f2fs_bug_on(sbi, dc->state != D_PREP);
1600 __remove_discard_cmd(sbi, dc);
1601 dropped = true;
1602 }
1603 }
1604 mutex_unlock(&dcc->cmd_lock);
1605
1606 return dropped;
1607 }
1608
1609 void f2fs_drop_discard_cmd(struct f2fs_sb_info *sbi)
1610 {
1611 __drop_discard_cmd(sbi);
1612 }
1613
1614 static unsigned int __wait_one_discard_bio(struct f2fs_sb_info *sbi,
1615 struct discard_cmd *dc)
1616 {
1617 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1618 unsigned int len = 0;
1619
1620 wait_for_completion_io(&dc->wait);
1621 mutex_lock(&dcc->cmd_lock);
1622 f2fs_bug_on(sbi, dc->state != D_DONE);
1623 dc->ref--;
1624 if (!dc->ref) {
1625 if (!dc->error)
1626 len = dc->len;
1627 __remove_discard_cmd(sbi, dc);
1628 }
1629 mutex_unlock(&dcc->cmd_lock);
1630
1631 return len;
1632 }
1633
1634 static unsigned int __wait_discard_cmd_range(struct f2fs_sb_info *sbi,
1635 struct discard_policy *dpolicy,
1636 block_t start, block_t end)
1637 {
1638 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1639 struct list_head *wait_list = (dpolicy->type == DPOLICY_FSTRIM) ?
1640 &(dcc->fstrim_list) : &(dcc->wait_list);
1641 struct discard_cmd *dc, *tmp;
1642 bool need_wait;
1643 unsigned int trimmed = 0;
1644
1645 next:
1646 need_wait = false;
1647
1648 mutex_lock(&dcc->cmd_lock);
1649 list_for_each_entry_safe(dc, tmp, wait_list, list) {
1650 if (dc->lstart + dc->len <= start || end <= dc->lstart)
1651 continue;
1652 if (dc->len < dpolicy->granularity)
1653 continue;
1654 if (dc->state == D_DONE && !dc->ref) {
1655 wait_for_completion_io(&dc->wait);
1656 if (!dc->error)
1657 trimmed += dc->len;
1658 __remove_discard_cmd(sbi, dc);
1659 } else {
1660 dc->ref++;
1661 need_wait = true;
1662 break;
1663 }
1664 }
1665 mutex_unlock(&dcc->cmd_lock);
1666
1667 if (need_wait) {
1668 trimmed += __wait_one_discard_bio(sbi, dc);
1669 goto next;
1670 }
1671
1672 return trimmed;
1673 }
1674
1675 static unsigned int __wait_all_discard_cmd(struct f2fs_sb_info *sbi,
1676 struct discard_policy *dpolicy)
1677 {
1678 struct discard_policy dp;
1679 unsigned int discard_blks;
1680
1681 if (dpolicy)
1682 return __wait_discard_cmd_range(sbi, dpolicy, 0, UINT_MAX);
1683
1684 /* wait all */
1685 __init_discard_policy(sbi, &dp, DPOLICY_FSTRIM, 1);
1686 discard_blks = __wait_discard_cmd_range(sbi, &dp, 0, UINT_MAX);
1687 __init_discard_policy(sbi, &dp, DPOLICY_UMOUNT, 1);
1688 discard_blks += __wait_discard_cmd_range(sbi, &dp, 0, UINT_MAX);
1689
1690 return discard_blks;
1691 }
1692
1693 /* This should be covered by global mutex, &sit_i->sentry_lock */
1694 static void f2fs_wait_discard_bio(struct f2fs_sb_info *sbi, block_t blkaddr)
1695 {
1696 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1697 struct discard_cmd *dc;
1698 bool need_wait = false;
1699
1700 mutex_lock(&dcc->cmd_lock);
1701 dc = (struct discard_cmd *)f2fs_lookup_rb_tree(&dcc->root,
1702 NULL, blkaddr);
1703 if (dc) {
1704 if (dc->state == D_PREP) {
1705 __punch_discard_cmd(sbi, dc, blkaddr);
1706 } else {
1707 dc->ref++;
1708 need_wait = true;
1709 }
1710 }
1711 mutex_unlock(&dcc->cmd_lock);
1712
1713 if (need_wait)
1714 __wait_one_discard_bio(sbi, dc);
1715 }
1716
1717 void f2fs_stop_discard_thread(struct f2fs_sb_info *sbi)
1718 {
1719 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1720
1721 if (dcc && dcc->f2fs_issue_discard) {
1722 struct task_struct *discard_thread = dcc->f2fs_issue_discard;
1723
1724 dcc->f2fs_issue_discard = NULL;
1725 kthread_stop(discard_thread);
1726 }
1727 }
1728
1729 /* This comes from f2fs_put_super */
1730 bool f2fs_issue_discard_timeout(struct f2fs_sb_info *sbi)
1731 {
1732 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1733 struct discard_policy dpolicy;
1734 bool dropped;
1735
1736 __init_discard_policy(sbi, &dpolicy, DPOLICY_UMOUNT,
1737 dcc->discard_granularity);
1738 __issue_discard_cmd(sbi, &dpolicy);
1739 dropped = __drop_discard_cmd(sbi);
1740
1741 /* just to make sure there is no pending discard commands */
1742 __wait_all_discard_cmd(sbi, NULL);
1743
1744 f2fs_bug_on(sbi, atomic_read(&dcc->discard_cmd_cnt));
1745 return dropped;
1746 }
1747
1748 static int issue_discard_thread(void *data)
1749 {
1750 struct f2fs_sb_info *sbi = data;
1751 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1752 wait_queue_head_t *q = &dcc->discard_wait_queue;
1753 struct discard_policy dpolicy;
1754 unsigned int wait_ms = DEF_MIN_DISCARD_ISSUE_TIME;
1755 int issued;
1756
1757 set_freezable();
1758
1759 do {
1760 if (sbi->gc_mode == GC_URGENT_HIGH ||
1761 !f2fs_available_free_memory(sbi, DISCARD_CACHE))
1762 __init_discard_policy(sbi, &dpolicy, DPOLICY_FORCE, 1);
1763 else
1764 __init_discard_policy(sbi, &dpolicy, DPOLICY_BG,
1765 dcc->discard_granularity);
1766
1767 if (!atomic_read(&dcc->discard_cmd_cnt))
1768 wait_ms = dpolicy.max_interval;
1769
1770 wait_event_interruptible_timeout(*q,
1771 kthread_should_stop() || freezing(current) ||
1772 dcc->discard_wake,
1773 msecs_to_jiffies(wait_ms));
1774
1775 if (dcc->discard_wake)
1776 dcc->discard_wake = 0;
1777
1778 /* clean up pending candidates before going to sleep */
1779 if (atomic_read(&dcc->queued_discard))
1780 __wait_all_discard_cmd(sbi, NULL);
1781
1782 if (try_to_freeze())
1783 continue;
1784 if (f2fs_readonly(sbi->sb))
1785 continue;
1786 if (kthread_should_stop())
1787 return 0;
1788 if (is_sbi_flag_set(sbi, SBI_NEED_FSCK)) {
1789 wait_ms = dpolicy.max_interval;
1790 continue;
1791 }
1792 if (!atomic_read(&dcc->discard_cmd_cnt))
1793 continue;
1794
1795 sb_start_intwrite(sbi->sb);
1796
1797 issued = __issue_discard_cmd(sbi, &dpolicy);
1798 if (issued > 0) {
1799 __wait_all_discard_cmd(sbi, &dpolicy);
1800 wait_ms = dpolicy.min_interval;
1801 } else if (issued == -1) {
1802 wait_ms = f2fs_time_to_wait(sbi, DISCARD_TIME);
1803 if (!wait_ms)
1804 wait_ms = dpolicy.mid_interval;
1805 } else {
1806 wait_ms = dpolicy.max_interval;
1807 }
1808
1809 sb_end_intwrite(sbi->sb);
1810
1811 } while (!kthread_should_stop());
1812 return 0;
1813 }
1814
1815 #ifdef CONFIG_BLK_DEV_ZONED
1816 static int __f2fs_issue_discard_zone(struct f2fs_sb_info *sbi,
1817 struct block_device *bdev, block_t blkstart, block_t blklen)
1818 {
1819 sector_t sector, nr_sects;
1820 block_t lblkstart = blkstart;
1821 int devi = 0;
1822
1823 if (f2fs_is_multi_device(sbi)) {
1824 devi = f2fs_target_device_index(sbi, blkstart);
1825 if (blkstart < FDEV(devi).start_blk ||
1826 blkstart > FDEV(devi).end_blk) {
1827 f2fs_err(sbi, "Invalid block %x", blkstart);
1828 return -EIO;
1829 }
1830 blkstart -= FDEV(devi).start_blk;
1831 }
1832
1833 /* For sequential zones, reset the zone write pointer */
1834 if (f2fs_blkz_is_seq(sbi, devi, blkstart)) {
1835 sector = SECTOR_FROM_BLOCK(blkstart);
1836 nr_sects = SECTOR_FROM_BLOCK(blklen);
1837
1838 if (sector & (bdev_zone_sectors(bdev) - 1) ||
1839 nr_sects != bdev_zone_sectors(bdev)) {
1840 f2fs_err(sbi, "(%d) %s: Unaligned zone reset attempted (block %x + %x)",
1841 devi, sbi->s_ndevs ? FDEV(devi).path : "",
1842 blkstart, blklen);
1843 return -EIO;
1844 }
1845 trace_f2fs_issue_reset_zone(bdev, blkstart);
1846 return blkdev_zone_mgmt(bdev, REQ_OP_ZONE_RESET,
1847 sector, nr_sects, GFP_NOFS);
1848 }
1849
1850 /* For conventional zones, use regular discard if supported */
1851 return __queue_discard_cmd(sbi, bdev, lblkstart, blklen);
1852 }
1853 #endif
1854
1855 static int __issue_discard_async(struct f2fs_sb_info *sbi,
1856 struct block_device *bdev, block_t blkstart, block_t blklen)
1857 {
1858 #ifdef CONFIG_BLK_DEV_ZONED
1859 if (f2fs_sb_has_blkzoned(sbi) && bdev_is_zoned(bdev))
1860 return __f2fs_issue_discard_zone(sbi, bdev, blkstart, blklen);
1861 #endif
1862 return __queue_discard_cmd(sbi, bdev, blkstart, blklen);
1863 }
1864
1865 static int f2fs_issue_discard(struct f2fs_sb_info *sbi,
1866 block_t blkstart, block_t blklen)
1867 {
1868 sector_t start = blkstart, len = 0;
1869 struct block_device *bdev;
1870 struct seg_entry *se;
1871 unsigned int offset;
1872 block_t i;
1873 int err = 0;
1874
1875 bdev = f2fs_target_device(sbi, blkstart, NULL);
1876
1877 for (i = blkstart; i < blkstart + blklen; i++, len++) {
1878 if (i != start) {
1879 struct block_device *bdev2 =
1880 f2fs_target_device(sbi, i, NULL);
1881
1882 if (bdev2 != bdev) {
1883 err = __issue_discard_async(sbi, bdev,
1884 start, len);
1885 if (err)
1886 return err;
1887 bdev = bdev2;
1888 start = i;
1889 len = 0;
1890 }
1891 }
1892
1893 se = get_seg_entry(sbi, GET_SEGNO(sbi, i));
1894 offset = GET_BLKOFF_FROM_SEG0(sbi, i);
1895
1896 if (f2fs_block_unit_discard(sbi) &&
1897 !f2fs_test_and_set_bit(offset, se->discard_map))
1898 sbi->discard_blks--;
1899 }
1900
1901 if (len)
1902 err = __issue_discard_async(sbi, bdev, start, len);
1903 return err;
1904 }
1905
1906 static bool add_discard_addrs(struct f2fs_sb_info *sbi, struct cp_control *cpc,
1907 bool check_only)
1908 {
1909 int entries = SIT_VBLOCK_MAP_SIZE / sizeof(unsigned long);
1910 int max_blocks = sbi->blocks_per_seg;
1911 struct seg_entry *se = get_seg_entry(sbi, cpc->trim_start);
1912 unsigned long *cur_map = (unsigned long *)se->cur_valid_map;
1913 unsigned long *ckpt_map = (unsigned long *)se->ckpt_valid_map;
1914 unsigned long *discard_map = (unsigned long *)se->discard_map;
1915 unsigned long *dmap = SIT_I(sbi)->tmp_map;
1916 unsigned int start = 0, end = -1;
1917 bool force = (cpc->reason & CP_DISCARD);
1918 struct discard_entry *de = NULL;
1919 struct list_head *head = &SM_I(sbi)->dcc_info->entry_list;
1920 int i;
1921
1922 if (se->valid_blocks == max_blocks || !f2fs_hw_support_discard(sbi) ||
1923 !f2fs_block_unit_discard(sbi))
1924 return false;
1925
1926 if (!force) {
1927 if (!f2fs_realtime_discard_enable(sbi) || !se->valid_blocks ||
1928 SM_I(sbi)->dcc_info->nr_discards >=
1929 SM_I(sbi)->dcc_info->max_discards)
1930 return false;
1931 }
1932
1933 /* SIT_VBLOCK_MAP_SIZE should be multiple of sizeof(unsigned long) */
1934 for (i = 0; i < entries; i++)
1935 dmap[i] = force ? ~ckpt_map[i] & ~discard_map[i] :
1936 (cur_map[i] ^ ckpt_map[i]) & ckpt_map[i];
1937
1938 while (force || SM_I(sbi)->dcc_info->nr_discards <=
1939 SM_I(sbi)->dcc_info->max_discards) {
1940 start = __find_rev_next_bit(dmap, max_blocks, end + 1);
1941 if (start >= max_blocks)
1942 break;
1943
1944 end = __find_rev_next_zero_bit(dmap, max_blocks, start + 1);
1945 if (force && start && end != max_blocks
1946 && (end - start) < cpc->trim_minlen)
1947 continue;
1948
1949 if (check_only)
1950 return true;
1951
1952 if (!de) {
1953 de = f2fs_kmem_cache_alloc(discard_entry_slab,
1954 GFP_F2FS_ZERO);
1955 de->start_blkaddr = START_BLOCK(sbi, cpc->trim_start);
1956 list_add_tail(&de->list, head);
1957 }
1958
1959 for (i = start; i < end; i++)
1960 __set_bit_le(i, (void *)de->discard_map);
1961
1962 SM_I(sbi)->dcc_info->nr_discards += end - start;
1963 }
1964 return false;
1965 }
1966
1967 static void release_discard_addr(struct discard_entry *entry)
1968 {
1969 list_del(&entry->list);
1970 kmem_cache_free(discard_entry_slab, entry);
1971 }
1972
1973 void f2fs_release_discard_addrs(struct f2fs_sb_info *sbi)
1974 {
1975 struct list_head *head = &(SM_I(sbi)->dcc_info->entry_list);
1976 struct discard_entry *entry, *this;
1977
1978 /* drop caches */
1979 list_for_each_entry_safe(entry, this, head, list)
1980 release_discard_addr(entry);
1981 }
1982
1983 /*
1984 * Should call f2fs_clear_prefree_segments after checkpoint is done.
1985 */
1986 static void set_prefree_as_free_segments(struct f2fs_sb_info *sbi)
1987 {
1988 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
1989 unsigned int segno;
1990
1991 mutex_lock(&dirty_i->seglist_lock);
1992 for_each_set_bit(segno, dirty_i->dirty_segmap[PRE], MAIN_SEGS(sbi))
1993 __set_test_and_free(sbi, segno, false);
1994 mutex_unlock(&dirty_i->seglist_lock);
1995 }
1996
1997 void f2fs_clear_prefree_segments(struct f2fs_sb_info *sbi,
1998 struct cp_control *cpc)
1999 {
2000 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
2001 struct list_head *head = &dcc->entry_list;
2002 struct discard_entry *entry, *this;
2003 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
2004 unsigned long *prefree_map = dirty_i->dirty_segmap[PRE];
2005 unsigned int start = 0, end = -1;
2006 unsigned int secno, start_segno;
2007 bool force = (cpc->reason & CP_DISCARD);
2008 bool section_alignment = F2FS_OPTION(sbi).discard_unit ==
2009 DISCARD_UNIT_SECTION;
2010
2011 if (f2fs_lfs_mode(sbi) && __is_large_section(sbi))
2012 section_alignment = true;
2013
2014 mutex_lock(&dirty_i->seglist_lock);
2015
2016 while (1) {
2017 int i;
2018
2019 if (section_alignment && end != -1)
2020 end--;
2021 start = find_next_bit(prefree_map, MAIN_SEGS(sbi), end + 1);
2022 if (start >= MAIN_SEGS(sbi))
2023 break;
2024 end = find_next_zero_bit(prefree_map, MAIN_SEGS(sbi),
2025 start + 1);
2026
2027 if (section_alignment) {
2028 start = rounddown(start, sbi->segs_per_sec);
2029 end = roundup(end, sbi->segs_per_sec);
2030 }
2031
2032 for (i = start; i < end; i++) {
2033 if (test_and_clear_bit(i, prefree_map))
2034 dirty_i->nr_dirty[PRE]--;
2035 }
2036
2037 if (!f2fs_realtime_discard_enable(sbi))
2038 continue;
2039
2040 if (force && start >= cpc->trim_start &&
2041 (end - 1) <= cpc->trim_end)
2042 continue;
2043
2044 if (!f2fs_lfs_mode(sbi) || !__is_large_section(sbi)) {
2045 f2fs_issue_discard(sbi, START_BLOCK(sbi, start),
2046 (end - start) << sbi->log_blocks_per_seg);
2047 continue;
2048 }
2049 next:
2050 secno = GET_SEC_FROM_SEG(sbi, start);
2051 start_segno = GET_SEG_FROM_SEC(sbi, secno);
2052 if (!IS_CURSEC(sbi, secno) &&
2053 !get_valid_blocks(sbi, start, true))
2054 f2fs_issue_discard(sbi, START_BLOCK(sbi, start_segno),
2055 sbi->segs_per_sec << sbi->log_blocks_per_seg);
2056
2057 start = start_segno + sbi->segs_per_sec;
2058 if (start < end)
2059 goto next;
2060 else
2061 end = start - 1;
2062 }
2063 mutex_unlock(&dirty_i->seglist_lock);
2064
2065 if (!f2fs_block_unit_discard(sbi))
2066 goto wakeup;
2067
2068 /* send small discards */
2069 list_for_each_entry_safe(entry, this, head, list) {
2070 unsigned int cur_pos = 0, next_pos, len, total_len = 0;
2071 bool is_valid = test_bit_le(0, entry->discard_map);
2072
2073 find_next:
2074 if (is_valid) {
2075 next_pos = find_next_zero_bit_le(entry->discard_map,
2076 sbi->blocks_per_seg, cur_pos);
2077 len = next_pos - cur_pos;
2078
2079 if (f2fs_sb_has_blkzoned(sbi) ||
2080 (force && len < cpc->trim_minlen))
2081 goto skip;
2082
2083 f2fs_issue_discard(sbi, entry->start_blkaddr + cur_pos,
2084 len);
2085 total_len += len;
2086 } else {
2087 next_pos = find_next_bit_le(entry->discard_map,
2088 sbi->blocks_per_seg, cur_pos);
2089 }
2090 skip:
2091 cur_pos = next_pos;
2092 is_valid = !is_valid;
2093
2094 if (cur_pos < sbi->blocks_per_seg)
2095 goto find_next;
2096
2097 release_discard_addr(entry);
2098 dcc->nr_discards -= total_len;
2099 }
2100
2101 wakeup:
2102 wake_up_discard_thread(sbi, false);
2103 }
2104
2105 static int create_discard_cmd_control(struct f2fs_sb_info *sbi)
2106 {
2107 dev_t dev = sbi->sb->s_bdev->bd_dev;
2108 struct discard_cmd_control *dcc;
2109 int err = 0, i;
2110
2111 if (SM_I(sbi)->dcc_info) {
2112 dcc = SM_I(sbi)->dcc_info;
2113 goto init_thread;
2114 }
2115
2116 dcc = f2fs_kzalloc(sbi, sizeof(struct discard_cmd_control), GFP_KERNEL);
2117 if (!dcc)
2118 return -ENOMEM;
2119
2120 dcc->discard_granularity = DEFAULT_DISCARD_GRANULARITY;
2121 if (F2FS_OPTION(sbi).discard_unit == DISCARD_UNIT_SEGMENT)
2122 dcc->discard_granularity = sbi->blocks_per_seg;
2123 else if (F2FS_OPTION(sbi).discard_unit == DISCARD_UNIT_SECTION)
2124 dcc->discard_granularity = BLKS_PER_SEC(sbi);
2125
2126 INIT_LIST_HEAD(&dcc->entry_list);
2127 for (i = 0; i < MAX_PLIST_NUM; i++)
2128 INIT_LIST_HEAD(&dcc->pend_list[i]);
2129 INIT_LIST_HEAD(&dcc->wait_list);
2130 INIT_LIST_HEAD(&dcc->fstrim_list);
2131 mutex_init(&dcc->cmd_lock);
2132 atomic_set(&dcc->issued_discard, 0);
2133 atomic_set(&dcc->queued_discard, 0);
2134 atomic_set(&dcc->discard_cmd_cnt, 0);
2135 dcc->nr_discards = 0;
2136 dcc->max_discards = MAIN_SEGS(sbi) << sbi->log_blocks_per_seg;
2137 dcc->undiscard_blks = 0;
2138 dcc->next_pos = 0;
2139 dcc->root = RB_ROOT_CACHED;
2140 dcc->rbtree_check = false;
2141
2142 init_waitqueue_head(&dcc->discard_wait_queue);
2143 SM_I(sbi)->dcc_info = dcc;
2144 init_thread:
2145 dcc->f2fs_issue_discard = kthread_run(issue_discard_thread, sbi,
2146 "f2fs_discard-%u:%u", MAJOR(dev), MINOR(dev));
2147 if (IS_ERR(dcc->f2fs_issue_discard)) {
2148 err = PTR_ERR(dcc->f2fs_issue_discard);
2149 kfree(dcc);
2150 SM_I(sbi)->dcc_info = NULL;
2151 return err;
2152 }
2153
2154 return err;
2155 }
2156
2157 static void destroy_discard_cmd_control(struct f2fs_sb_info *sbi)
2158 {
2159 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
2160
2161 if (!dcc)
2162 return;
2163
2164 f2fs_stop_discard_thread(sbi);
2165
2166 /*
2167 * Recovery can cache discard commands, so in error path of
2168 * fill_super(), it needs to give a chance to handle them.
2169 */
2170 if (unlikely(atomic_read(&dcc->discard_cmd_cnt)))
2171 f2fs_issue_discard_timeout(sbi);
2172
2173 kfree(dcc);
2174 SM_I(sbi)->dcc_info = NULL;
2175 }
2176
2177 static bool __mark_sit_entry_dirty(struct f2fs_sb_info *sbi, unsigned int segno)
2178 {
2179 struct sit_info *sit_i = SIT_I(sbi);
2180
2181 if (!__test_and_set_bit(segno, sit_i->dirty_sentries_bitmap)) {
2182 sit_i->dirty_sentries++;
2183 return false;
2184 }
2185
2186 return true;
2187 }
2188
2189 static void __set_sit_entry_type(struct f2fs_sb_info *sbi, int type,
2190 unsigned int segno, int modified)
2191 {
2192 struct seg_entry *se = get_seg_entry(sbi, segno);
2193
2194 se->type = type;
2195 if (modified)
2196 __mark_sit_entry_dirty(sbi, segno);
2197 }
2198
2199 static inline unsigned long long get_segment_mtime(struct f2fs_sb_info *sbi,
2200 block_t blkaddr)
2201 {
2202 unsigned int segno = GET_SEGNO(sbi, blkaddr);
2203
2204 if (segno == NULL_SEGNO)
2205 return 0;
2206 return get_seg_entry(sbi, segno)->mtime;
2207 }
2208
2209 static void update_segment_mtime(struct f2fs_sb_info *sbi, block_t blkaddr,
2210 unsigned long long old_mtime)
2211 {
2212 struct seg_entry *se;
2213 unsigned int segno = GET_SEGNO(sbi, blkaddr);
2214 unsigned long long ctime = get_mtime(sbi, false);
2215 unsigned long long mtime = old_mtime ? old_mtime : ctime;
2216
2217 if (segno == NULL_SEGNO)
2218 return;
2219
2220 se = get_seg_entry(sbi, segno);
2221
2222 if (!se->mtime)
2223 se->mtime = mtime;
2224 else
2225 se->mtime = div_u64(se->mtime * se->valid_blocks + mtime,
2226 se->valid_blocks + 1);
2227
2228 if (ctime > SIT_I(sbi)->max_mtime)
2229 SIT_I(sbi)->max_mtime = ctime;
2230 }
2231
2232 static void update_sit_entry(struct f2fs_sb_info *sbi, block_t blkaddr, int del)
2233 {
2234 struct seg_entry *se;
2235 unsigned int segno, offset;
2236 long int new_vblocks;
2237 bool exist;
2238 #ifdef CONFIG_F2FS_CHECK_FS
2239 bool mir_exist;
2240 #endif
2241
2242 segno = GET_SEGNO(sbi, blkaddr);
2243
2244 se = get_seg_entry(sbi, segno);
2245 new_vblocks = se->valid_blocks + del;
2246 offset = GET_BLKOFF_FROM_SEG0(sbi, blkaddr);
2247
2248 f2fs_bug_on(sbi, (new_vblocks < 0 ||
2249 (new_vblocks > f2fs_usable_blks_in_seg(sbi, segno))));
2250
2251 se->valid_blocks = new_vblocks;
2252
2253 /* Update valid block bitmap */
2254 if (del > 0) {
2255 exist = f2fs_test_and_set_bit(offset, se->cur_valid_map);
2256 #ifdef CONFIG_F2FS_CHECK_FS
2257 mir_exist = f2fs_test_and_set_bit(offset,
2258 se->cur_valid_map_mir);
2259 if (unlikely(exist != mir_exist)) {
2260 f2fs_err(sbi, "Inconsistent error when setting bitmap, blk:%u, old bit:%d",
2261 blkaddr, exist);
2262 f2fs_bug_on(sbi, 1);
2263 }
2264 #endif
2265 if (unlikely(exist)) {
2266 f2fs_err(sbi, "Bitmap was wrongly set, blk:%u",
2267 blkaddr);
2268 f2fs_bug_on(sbi, 1);
2269 se->valid_blocks--;
2270 del = 0;
2271 }
2272
2273 if (f2fs_block_unit_discard(sbi) &&
2274 !f2fs_test_and_set_bit(offset, se->discard_map))
2275 sbi->discard_blks--;
2276
2277 /*
2278 * SSR should never reuse block which is checkpointed
2279 * or newly invalidated.
2280 */
2281 if (!is_sbi_flag_set(sbi, SBI_CP_DISABLED)) {
2282 if (!f2fs_test_and_set_bit(offset, se->ckpt_valid_map))
2283 se->ckpt_valid_blocks++;
2284 }
2285 } else {
2286 exist = f2fs_test_and_clear_bit(offset, se->cur_valid_map);
2287 #ifdef CONFIG_F2FS_CHECK_FS
2288 mir_exist = f2fs_test_and_clear_bit(offset,
2289 se->cur_valid_map_mir);
2290 if (unlikely(exist != mir_exist)) {
2291 f2fs_err(sbi, "Inconsistent error when clearing bitmap, blk:%u, old bit:%d",
2292 blkaddr, exist);
2293 f2fs_bug_on(sbi, 1);
2294 }
2295 #endif
2296 if (unlikely(!exist)) {
2297 f2fs_err(sbi, "Bitmap was wrongly cleared, blk:%u",
2298 blkaddr);
2299 f2fs_bug_on(sbi, 1);
2300 se->valid_blocks++;
2301 del = 0;
2302 } else if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED))) {
2303 /*
2304 * If checkpoints are off, we must not reuse data that
2305 * was used in the previous checkpoint. If it was used
2306 * before, we must track that to know how much space we
2307 * really have.
2308 */
2309 if (f2fs_test_bit(offset, se->ckpt_valid_map)) {
2310 spin_lock(&sbi->stat_lock);
2311 sbi->unusable_block_count++;
2312 spin_unlock(&sbi->stat_lock);
2313 }
2314 }
2315
2316 if (f2fs_block_unit_discard(sbi) &&
2317 f2fs_test_and_clear_bit(offset, se->discard_map))
2318 sbi->discard_blks++;
2319 }
2320 if (!f2fs_test_bit(offset, se->ckpt_valid_map))
2321 se->ckpt_valid_blocks += del;
2322
2323 __mark_sit_entry_dirty(sbi, segno);
2324
2325 /* update total number of valid blocks to be written in ckpt area */
2326 SIT_I(sbi)->written_valid_blocks += del;
2327
2328 if (__is_large_section(sbi))
2329 get_sec_entry(sbi, segno)->valid_blocks += del;
2330 }
2331
2332 void f2fs_invalidate_blocks(struct f2fs_sb_info *sbi, block_t addr)
2333 {
2334 unsigned int segno = GET_SEGNO(sbi, addr);
2335 struct sit_info *sit_i = SIT_I(sbi);
2336
2337 f2fs_bug_on(sbi, addr == NULL_ADDR);
2338 if (addr == NEW_ADDR || addr == COMPRESS_ADDR)
2339 return;
2340
2341 invalidate_mapping_pages(META_MAPPING(sbi), addr, addr);
2342 f2fs_invalidate_compress_page(sbi, addr);
2343
2344 /* add it into sit main buffer */
2345 down_write(&sit_i->sentry_lock);
2346
2347 update_segment_mtime(sbi, addr, 0);
2348 update_sit_entry(sbi, addr, -1);
2349
2350 /* add it into dirty seglist */
2351 locate_dirty_segment(sbi, segno);
2352
2353 up_write(&sit_i->sentry_lock);
2354 }
2355
2356 bool f2fs_is_checkpointed_data(struct f2fs_sb_info *sbi, block_t blkaddr)
2357 {
2358 struct sit_info *sit_i = SIT_I(sbi);
2359 unsigned int segno, offset;
2360 struct seg_entry *se;
2361 bool is_cp = false;
2362
2363 if (!__is_valid_data_blkaddr(blkaddr))
2364 return true;
2365
2366 down_read(&sit_i->sentry_lock);
2367
2368 segno = GET_SEGNO(sbi, blkaddr);
2369 se = get_seg_entry(sbi, segno);
2370 offset = GET_BLKOFF_FROM_SEG0(sbi, blkaddr);
2371
2372 if (f2fs_test_bit(offset, se->ckpt_valid_map))
2373 is_cp = true;
2374
2375 up_read(&sit_i->sentry_lock);
2376
2377 return is_cp;
2378 }
2379
2380 /*
2381 * This function should be resided under the curseg_mutex lock
2382 */
2383 static void __add_sum_entry(struct f2fs_sb_info *sbi, int type,
2384 struct f2fs_summary *sum)
2385 {
2386 struct curseg_info *curseg = CURSEG_I(sbi, type);
2387 void *addr = curseg->sum_blk;
2388
2389 addr += curseg->next_blkoff * sizeof(struct f2fs_summary);
2390 memcpy(addr, sum, sizeof(struct f2fs_summary));
2391 }
2392
2393 /*
2394 * Calculate the number of current summary pages for writing
2395 */
2396 int f2fs_npages_for_summary_flush(struct f2fs_sb_info *sbi, bool for_ra)
2397 {
2398 int valid_sum_count = 0;
2399 int i, sum_in_page;
2400
2401 for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++) {
2402 if (sbi->ckpt->alloc_type[i] == SSR)
2403 valid_sum_count += sbi->blocks_per_seg;
2404 else {
2405 if (for_ra)
2406 valid_sum_count += le16_to_cpu(
2407 F2FS_CKPT(sbi)->cur_data_blkoff[i]);
2408 else
2409 valid_sum_count += curseg_blkoff(sbi, i);
2410 }
2411 }
2412
2413 sum_in_page = (PAGE_SIZE - 2 * SUM_JOURNAL_SIZE -
2414 SUM_FOOTER_SIZE) / SUMMARY_SIZE;
2415 if (valid_sum_count <= sum_in_page)
2416 return 1;
2417 else if ((valid_sum_count - sum_in_page) <=
2418 (PAGE_SIZE - SUM_FOOTER_SIZE) / SUMMARY_SIZE)
2419 return 2;
2420 return 3;
2421 }
2422
2423 /*
2424 * Caller should put this summary page
2425 */
2426 struct page *f2fs_get_sum_page(struct f2fs_sb_info *sbi, unsigned int segno)
2427 {
2428 if (unlikely(f2fs_cp_error(sbi)))
2429 return ERR_PTR(-EIO);
2430 return f2fs_get_meta_page_retry(sbi, GET_SUM_BLOCK(sbi, segno));
2431 }
2432
2433 void f2fs_update_meta_page(struct f2fs_sb_info *sbi,
2434 void *src, block_t blk_addr)
2435 {
2436 struct page *page = f2fs_grab_meta_page(sbi, blk_addr);
2437
2438 memcpy(page_address(page), src, PAGE_SIZE);
2439 set_page_dirty(page);
2440 f2fs_put_page(page, 1);
2441 }
2442
2443 static void write_sum_page(struct f2fs_sb_info *sbi,
2444 struct f2fs_summary_block *sum_blk, block_t blk_addr)
2445 {
2446 f2fs_update_meta_page(sbi, (void *)sum_blk, blk_addr);
2447 }
2448
2449 static void write_current_sum_page(struct f2fs_sb_info *sbi,
2450 int type, block_t blk_addr)
2451 {
2452 struct curseg_info *curseg = CURSEG_I(sbi, type);
2453 struct page *page = f2fs_grab_meta_page(sbi, blk_addr);
2454 struct f2fs_summary_block *src = curseg->sum_blk;
2455 struct f2fs_summary_block *dst;
2456
2457 dst = (struct f2fs_summary_block *)page_address(page);
2458 memset(dst, 0, PAGE_SIZE);
2459
2460 mutex_lock(&curseg->curseg_mutex);
2461
2462 down_read(&curseg->journal_rwsem);
2463 memcpy(&dst->journal, curseg->journal, SUM_JOURNAL_SIZE);
2464 up_read(&curseg->journal_rwsem);
2465
2466 memcpy(dst->entries, src->entries, SUM_ENTRY_SIZE);
2467 memcpy(&dst->footer, &src->footer, SUM_FOOTER_SIZE);
2468
2469 mutex_unlock(&curseg->curseg_mutex);
2470
2471 set_page_dirty(page);
2472 f2fs_put_page(page, 1);
2473 }
2474
2475 static int is_next_segment_free(struct f2fs_sb_info *sbi,
2476 struct curseg_info *curseg, int type)
2477 {
2478 unsigned int segno = curseg->segno + 1;
2479 struct free_segmap_info *free_i = FREE_I(sbi);
2480
2481 if (segno < MAIN_SEGS(sbi) && segno % sbi->segs_per_sec)
2482 return !test_bit(segno, free_i->free_segmap);
2483 return 0;
2484 }
2485
2486 /*
2487 * Find a new segment from the free segments bitmap to right order
2488 * This function should be returned with success, otherwise BUG
2489 */
2490 static void get_new_segment(struct f2fs_sb_info *sbi,
2491 unsigned int *newseg, bool new_sec, int dir)
2492 {
2493 struct free_segmap_info *free_i = FREE_I(sbi);
2494 unsigned int segno, secno, zoneno;
2495 unsigned int total_zones = MAIN_SECS(sbi) / sbi->secs_per_zone;
2496 unsigned int hint = GET_SEC_FROM_SEG(sbi, *newseg);
2497 unsigned int old_zoneno = GET_ZONE_FROM_SEG(sbi, *newseg);
2498 unsigned int left_start = hint;
2499 bool init = true;
2500 int go_left = 0;
2501 int i;
2502
2503 spin_lock(&free_i->segmap_lock);
2504
2505 if (!new_sec && ((*newseg + 1) % sbi->segs_per_sec)) {
2506 segno = find_next_zero_bit(free_i->free_segmap,
2507 GET_SEG_FROM_SEC(sbi, hint + 1), *newseg + 1);
2508 if (segno < GET_SEG_FROM_SEC(sbi, hint + 1))
2509 goto got_it;
2510 }
2511 find_other_zone:
2512 secno = find_next_zero_bit(free_i->free_secmap, MAIN_SECS(sbi), hint);
2513 if (secno >= MAIN_SECS(sbi)) {
2514 if (dir == ALLOC_RIGHT) {
2515 secno = find_next_zero_bit(free_i->free_secmap,
2516 MAIN_SECS(sbi), 0);
2517 f2fs_bug_on(sbi, secno >= MAIN_SECS(sbi));
2518 } else {
2519 go_left = 1;
2520 left_start = hint - 1;
2521 }
2522 }
2523 if (go_left == 0)
2524 goto skip_left;
2525
2526 while (test_bit(left_start, free_i->free_secmap)) {
2527 if (left_start > 0) {
2528 left_start--;
2529 continue;
2530 }
2531 left_start = find_next_zero_bit(free_i->free_secmap,
2532 MAIN_SECS(sbi), 0);
2533 f2fs_bug_on(sbi, left_start >= MAIN_SECS(sbi));
2534 break;
2535 }
2536 secno = left_start;
2537 skip_left:
2538 segno = GET_SEG_FROM_SEC(sbi, secno);
2539 zoneno = GET_ZONE_FROM_SEC(sbi, secno);
2540
2541 /* give up on finding another zone */
2542 if (!init)
2543 goto got_it;
2544 if (sbi->secs_per_zone == 1)
2545 goto got_it;
2546 if (zoneno == old_zoneno)
2547 goto got_it;
2548 if (dir == ALLOC_LEFT) {
2549 if (!go_left && zoneno + 1 >= total_zones)
2550 goto got_it;
2551 if (go_left && zoneno == 0)
2552 goto got_it;
2553 }
2554 for (i = 0; i < NR_CURSEG_TYPE; i++)
2555 if (CURSEG_I(sbi, i)->zone == zoneno)
2556 break;
2557
2558 if (i < NR_CURSEG_TYPE) {
2559 /* zone is in user, try another */
2560 if (go_left)
2561 hint = zoneno * sbi->secs_per_zone - 1;
2562 else if (zoneno + 1 >= total_zones)
2563 hint = 0;
2564 else
2565 hint = (zoneno + 1) * sbi->secs_per_zone;
2566 init = false;
2567 goto find_other_zone;
2568 }
2569 got_it:
2570 /* set it as dirty segment in free segmap */
2571 f2fs_bug_on(sbi, test_bit(segno, free_i->free_segmap));
2572 __set_inuse(sbi, segno);
2573 *newseg = segno;
2574 spin_unlock(&free_i->segmap_lock);
2575 }
2576
2577 static void reset_curseg(struct f2fs_sb_info *sbi, int type, int modified)
2578 {
2579 struct curseg_info *curseg = CURSEG_I(sbi, type);
2580 struct summary_footer *sum_footer;
2581 unsigned short seg_type = curseg->seg_type;
2582
2583 curseg->inited = true;
2584 curseg->segno = curseg->next_segno;
2585 curseg->zone = GET_ZONE_FROM_SEG(sbi, curseg->segno);
2586 curseg->next_blkoff = 0;
2587 curseg->next_segno = NULL_SEGNO;
2588
2589 sum_footer = &(curseg->sum_blk->footer);
2590 memset(sum_footer, 0, sizeof(struct summary_footer));
2591
2592 sanity_check_seg_type(sbi, seg_type);
2593
2594 if (IS_DATASEG(seg_type))
2595 SET_SUM_TYPE(sum_footer, SUM_TYPE_DATA);
2596 if (IS_NODESEG(seg_type))
2597 SET_SUM_TYPE(sum_footer, SUM_TYPE_NODE);
2598 __set_sit_entry_type(sbi, seg_type, curseg->segno, modified);
2599 }
2600
2601 static unsigned int __get_next_segno(struct f2fs_sb_info *sbi, int type)
2602 {
2603 struct curseg_info *curseg = CURSEG_I(sbi, type);
2604 unsigned short seg_type = curseg->seg_type;
2605
2606 sanity_check_seg_type(sbi, seg_type);
2607
2608 /* if segs_per_sec is large than 1, we need to keep original policy. */
2609 if (__is_large_section(sbi))
2610 return curseg->segno;
2611
2612 /* inmem log may not locate on any segment after mount */
2613 if (!curseg->inited)
2614 return 0;
2615
2616 if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED)))
2617 return 0;
2618
2619 if (test_opt(sbi, NOHEAP) &&
2620 (seg_type == CURSEG_HOT_DATA || IS_NODESEG(seg_type)))
2621 return 0;
2622
2623 if (SIT_I(sbi)->last_victim[ALLOC_NEXT])
2624 return SIT_I(sbi)->last_victim[ALLOC_NEXT];
2625
2626 /* find segments from 0 to reuse freed segments */
2627 if (F2FS_OPTION(sbi).alloc_mode == ALLOC_MODE_REUSE)
2628 return 0;
2629
2630 return curseg->segno;
2631 }
2632
2633 /*
2634 * Allocate a current working segment.
2635 * This function always allocates a free segment in LFS manner.
2636 */
2637 static void new_curseg(struct f2fs_sb_info *sbi, int type, bool new_sec)
2638 {
2639 struct curseg_info *curseg = CURSEG_I(sbi, type);
2640 unsigned short seg_type = curseg->seg_type;
2641 unsigned int segno = curseg->segno;
2642 int dir = ALLOC_LEFT;
2643
2644 if (curseg->inited)
2645 write_sum_page(sbi, curseg->sum_blk,
2646 GET_SUM_BLOCK(sbi, segno));
2647 if (seg_type == CURSEG_WARM_DATA || seg_type == CURSEG_COLD_DATA)
2648 dir = ALLOC_RIGHT;
2649
2650 if (test_opt(sbi, NOHEAP))
2651 dir = ALLOC_RIGHT;
2652
2653 segno = __get_next_segno(sbi, type);
2654 get_new_segment(sbi, &segno, new_sec, dir);
2655 curseg->next_segno = segno;
2656 reset_curseg(sbi, type, 1);
2657 curseg->alloc_type = LFS;
2658 }
2659
2660 static int __next_free_blkoff(struct f2fs_sb_info *sbi,
2661 int segno, block_t start)
2662 {
2663 struct seg_entry *se = get_seg_entry(sbi, segno);
2664 int entries = SIT_VBLOCK_MAP_SIZE / sizeof(unsigned long);
2665 unsigned long *target_map = SIT_I(sbi)->tmp_map;
2666 unsigned long *ckpt_map = (unsigned long *)se->ckpt_valid_map;
2667 unsigned long *cur_map = (unsigned long *)se->cur_valid_map;
2668 int i;
2669
2670 for (i = 0; i < entries; i++)
2671 target_map[i] = ckpt_map[i] | cur_map[i];
2672
2673 return __find_rev_next_zero_bit(target_map, sbi->blocks_per_seg, start);
2674 }
2675
2676 /*
2677 * If a segment is written by LFS manner, next block offset is just obtained
2678 * by increasing the current block offset. However, if a segment is written by
2679 * SSR manner, next block offset obtained by calling __next_free_blkoff
2680 */
2681 static void __refresh_next_blkoff(struct f2fs_sb_info *sbi,
2682 struct curseg_info *seg)
2683 {
2684 if (seg->alloc_type == SSR)
2685 seg->next_blkoff =
2686 __next_free_blkoff(sbi, seg->segno,
2687 seg->next_blkoff + 1);
2688 else
2689 seg->next_blkoff++;
2690 }
2691
2692 bool f2fs_segment_has_free_slot(struct f2fs_sb_info *sbi, int segno)
2693 {
2694 return __next_free_blkoff(sbi, segno, 0) < sbi->blocks_per_seg;
2695 }
2696
2697 /*
2698 * This function always allocates a used segment(from dirty seglist) by SSR
2699 * manner, so it should recover the existing segment information of valid blocks
2700 */
2701 static void change_curseg(struct f2fs_sb_info *sbi, int type, bool flush)
2702 {
2703 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
2704 struct curseg_info *curseg = CURSEG_I(sbi, type);
2705 unsigned int new_segno = curseg->next_segno;
2706 struct f2fs_summary_block *sum_node;
2707 struct page *sum_page;
2708
2709 if (flush)
2710 write_sum_page(sbi, curseg->sum_blk,
2711 GET_SUM_BLOCK(sbi, curseg->segno));
2712
2713 __set_test_and_inuse(sbi, new_segno);
2714
2715 mutex_lock(&dirty_i->seglist_lock);
2716 __remove_dirty_segment(sbi, new_segno, PRE);
2717 __remove_dirty_segment(sbi, new_segno, DIRTY);
2718 mutex_unlock(&dirty_i->seglist_lock);
2719
2720 reset_curseg(sbi, type, 1);
2721 curseg->alloc_type = SSR;
2722 curseg->next_blkoff = __next_free_blkoff(sbi, curseg->segno, 0);
2723
2724 sum_page = f2fs_get_sum_page(sbi, new_segno);
2725 if (IS_ERR(sum_page)) {
2726 /* GC won't be able to use stale summary pages by cp_error */
2727 memset(curseg->sum_blk, 0, SUM_ENTRY_SIZE);
2728 return;
2729 }
2730 sum_node = (struct f2fs_summary_block *)page_address(sum_page);
2731 memcpy(curseg->sum_blk, sum_node, SUM_ENTRY_SIZE);
2732 f2fs_put_page(sum_page, 1);
2733 }
2734
2735 static int get_ssr_segment(struct f2fs_sb_info *sbi, int type,
2736 int alloc_mode, unsigned long long age);
2737
2738 static void get_atssr_segment(struct f2fs_sb_info *sbi, int type,
2739 int target_type, int alloc_mode,
2740 unsigned long long age)
2741 {
2742 struct curseg_info *curseg = CURSEG_I(sbi, type);
2743
2744 curseg->seg_type = target_type;
2745
2746 if (get_ssr_segment(sbi, type, alloc_mode, age)) {
2747 struct seg_entry *se = get_seg_entry(sbi, curseg->next_segno);
2748
2749 curseg->seg_type = se->type;
2750 change_curseg(sbi, type, true);
2751 } else {
2752 /* allocate cold segment by default */
2753 curseg->seg_type = CURSEG_COLD_DATA;
2754 new_curseg(sbi, type, true);
2755 }
2756 stat_inc_seg_type(sbi, curseg);
2757 }
2758
2759 static void __f2fs_init_atgc_curseg(struct f2fs_sb_info *sbi)
2760 {
2761 struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_ALL_DATA_ATGC);
2762
2763 if (!sbi->am.atgc_enabled)
2764 return;
2765
2766 down_read(&SM_I(sbi)->curseg_lock);
2767
2768 mutex_lock(&curseg->curseg_mutex);
2769 down_write(&SIT_I(sbi)->sentry_lock);
2770
2771 get_atssr_segment(sbi, CURSEG_ALL_DATA_ATGC, CURSEG_COLD_DATA, SSR, 0);
2772
2773 up_write(&SIT_I(sbi)->sentry_lock);
2774 mutex_unlock(&curseg->curseg_mutex);
2775
2776 up_read(&SM_I(sbi)->curseg_lock);
2777
2778 }
2779 void f2fs_init_inmem_curseg(struct f2fs_sb_info *sbi)
2780 {
2781 __f2fs_init_atgc_curseg(sbi);
2782 }
2783
2784 static void __f2fs_save_inmem_curseg(struct f2fs_sb_info *sbi, int type)
2785 {
2786 struct curseg_info *curseg = CURSEG_I(sbi, type);
2787
2788 mutex_lock(&curseg->curseg_mutex);
2789 if (!curseg->inited)
2790 goto out;
2791
2792 if (get_valid_blocks(sbi, curseg->segno, false)) {
2793 write_sum_page(sbi, curseg->sum_blk,
2794 GET_SUM_BLOCK(sbi, curseg->segno));
2795 } else {
2796 mutex_lock(&DIRTY_I(sbi)->seglist_lock);
2797 __set_test_and_free(sbi, curseg->segno, true);
2798 mutex_unlock(&DIRTY_I(sbi)->seglist_lock);
2799 }
2800 out:
2801 mutex_unlock(&curseg->curseg_mutex);
2802 }
2803
2804 void f2fs_save_inmem_curseg(struct f2fs_sb_info *sbi)
2805 {
2806 __f2fs_save_inmem_curseg(sbi, CURSEG_COLD_DATA_PINNED);
2807
2808 if (sbi->am.atgc_enabled)
2809 __f2fs_save_inmem_curseg(sbi, CURSEG_ALL_DATA_ATGC);
2810 }
2811
2812 static void __f2fs_restore_inmem_curseg(struct f2fs_sb_info *sbi, int type)
2813 {
2814 struct curseg_info *curseg = CURSEG_I(sbi, type);
2815
2816 mutex_lock(&curseg->curseg_mutex);
2817 if (!curseg->inited)
2818 goto out;
2819 if (get_valid_blocks(sbi, curseg->segno, false))
2820 goto out;
2821
2822 mutex_lock(&DIRTY_I(sbi)->seglist_lock);
2823 __set_test_and_inuse(sbi, curseg->segno);
2824 mutex_unlock(&DIRTY_I(sbi)->seglist_lock);
2825 out:
2826 mutex_unlock(&curseg->curseg_mutex);
2827 }
2828
2829 void f2fs_restore_inmem_curseg(struct f2fs_sb_info *sbi)
2830 {
2831 __f2fs_restore_inmem_curseg(sbi, CURSEG_COLD_DATA_PINNED);
2832
2833 if (sbi->am.atgc_enabled)
2834 __f2fs_restore_inmem_curseg(sbi, CURSEG_ALL_DATA_ATGC);
2835 }
2836
2837 static int get_ssr_segment(struct f2fs_sb_info *sbi, int type,
2838 int alloc_mode, unsigned long long age)
2839 {
2840 struct curseg_info *curseg = CURSEG_I(sbi, type);
2841 const struct victim_selection *v_ops = DIRTY_I(sbi)->v_ops;
2842 unsigned segno = NULL_SEGNO;
2843 unsigned short seg_type = curseg->seg_type;
2844 int i, cnt;
2845 bool reversed = false;
2846
2847 sanity_check_seg_type(sbi, seg_type);
2848
2849 /* f2fs_need_SSR() already forces to do this */
2850 if (!v_ops->get_victim(sbi, &segno, BG_GC, seg_type, alloc_mode, age)) {
2851 curseg->next_segno = segno;
2852 return 1;
2853 }
2854
2855 /* For node segments, let's do SSR more intensively */
2856 if (IS_NODESEG(seg_type)) {
2857 if (seg_type >= CURSEG_WARM_NODE) {
2858 reversed = true;
2859 i = CURSEG_COLD_NODE;
2860 } else {
2861 i = CURSEG_HOT_NODE;
2862 }
2863 cnt = NR_CURSEG_NODE_TYPE;
2864 } else {
2865 if (seg_type >= CURSEG_WARM_DATA) {
2866 reversed = true;
2867 i = CURSEG_COLD_DATA;
2868 } else {
2869 i = CURSEG_HOT_DATA;
2870 }
2871 cnt = NR_CURSEG_DATA_TYPE;
2872 }
2873
2874 for (; cnt-- > 0; reversed ? i-- : i++) {
2875 if (i == seg_type)
2876 continue;
2877 if (!v_ops->get_victim(sbi, &segno, BG_GC, i, alloc_mode, age)) {
2878 curseg->next_segno = segno;
2879 return 1;
2880 }
2881 }
2882
2883 /* find valid_blocks=0 in dirty list */
2884 if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED))) {
2885 segno = get_free_segment(sbi);
2886 if (segno != NULL_SEGNO) {
2887 curseg->next_segno = segno;
2888 return 1;
2889 }
2890 }
2891 return 0;
2892 }
2893
2894 /*
2895 * flush out current segment and replace it with new segment
2896 * This function should be returned with success, otherwise BUG
2897 */
2898 static void allocate_segment_by_default(struct f2fs_sb_info *sbi,
2899 int type, bool force)
2900 {
2901 struct curseg_info *curseg = CURSEG_I(sbi, type);
2902
2903 if (force)
2904 new_curseg(sbi, type, true);
2905 else if (!is_set_ckpt_flags(sbi, CP_CRC_RECOVERY_FLAG) &&
2906 curseg->seg_type == CURSEG_WARM_NODE)
2907 new_curseg(sbi, type, false);
2908 else if (curseg->alloc_type == LFS &&
2909 is_next_segment_free(sbi, curseg, type) &&
2910 likely(!is_sbi_flag_set(sbi, SBI_CP_DISABLED)))
2911 new_curseg(sbi, type, false);
2912 else if (f2fs_need_SSR(sbi) &&
2913 get_ssr_segment(sbi, type, SSR, 0))
2914 change_curseg(sbi, type, true);
2915 else
2916 new_curseg(sbi, type, false);
2917
2918 stat_inc_seg_type(sbi, curseg);
2919 }
2920
2921 void f2fs_allocate_segment_for_resize(struct f2fs_sb_info *sbi, int type,
2922 unsigned int start, unsigned int end)
2923 {
2924 struct curseg_info *curseg = CURSEG_I(sbi, type);
2925 unsigned int segno;
2926
2927 down_read(&SM_I(sbi)->curseg_lock);
2928 mutex_lock(&curseg->curseg_mutex);
2929 down_write(&SIT_I(sbi)->sentry_lock);
2930
2931 segno = CURSEG_I(sbi, type)->segno;
2932 if (segno < start || segno > end)
2933 goto unlock;
2934
2935 if (f2fs_need_SSR(sbi) && get_ssr_segment(sbi, type, SSR, 0))
2936 change_curseg(sbi, type, true);
2937 else
2938 new_curseg(sbi, type, true);
2939
2940 stat_inc_seg_type(sbi, curseg);
2941
2942 locate_dirty_segment(sbi, segno);
2943 unlock:
2944 up_write(&SIT_I(sbi)->sentry_lock);
2945
2946 if (segno != curseg->segno)
2947 f2fs_notice(sbi, "For resize: curseg of type %d: %u ==> %u",
2948 type, segno, curseg->segno);
2949
2950 mutex_unlock(&curseg->curseg_mutex);
2951 up_read(&SM_I(sbi)->curseg_lock);
2952 }
2953
2954 static void __allocate_new_segment(struct f2fs_sb_info *sbi, int type,
2955 bool new_sec, bool force)
2956 {
2957 struct curseg_info *curseg = CURSEG_I(sbi, type);
2958 unsigned int old_segno;
2959
2960 if (!curseg->inited)
2961 goto alloc;
2962
2963 if (force || curseg->next_blkoff ||
2964 get_valid_blocks(sbi, curseg->segno, new_sec))
2965 goto alloc;
2966
2967 if (!get_ckpt_valid_blocks(sbi, curseg->segno, new_sec))
2968 return;
2969 alloc:
2970 old_segno = curseg->segno;
2971 SIT_I(sbi)->s_ops->allocate_segment(sbi, type, true);
2972 locate_dirty_segment(sbi, old_segno);
2973 }
2974
2975 static void __allocate_new_section(struct f2fs_sb_info *sbi,
2976 int type, bool force)
2977 {
2978 __allocate_new_segment(sbi, type, true, force);
2979 }
2980
2981 void f2fs_allocate_new_section(struct f2fs_sb_info *sbi, int type, bool force)
2982 {
2983 down_read(&SM_I(sbi)->curseg_lock);
2984 down_write(&SIT_I(sbi)->sentry_lock);
2985 __allocate_new_section(sbi, type, force);
2986 up_write(&SIT_I(sbi)->sentry_lock);
2987 up_read(&SM_I(sbi)->curseg_lock);
2988 }
2989
2990 void f2fs_allocate_new_segments(struct f2fs_sb_info *sbi)
2991 {
2992 int i;
2993
2994 down_read(&SM_I(sbi)->curseg_lock);
2995 down_write(&SIT_I(sbi)->sentry_lock);
2996 for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++)
2997 __allocate_new_segment(sbi, i, false, false);
2998 up_write(&SIT_I(sbi)->sentry_lock);
2999 up_read(&SM_I(sbi)->curseg_lock);
3000 }
3001
3002 static const struct segment_allocation default_salloc_ops = {
3003 .allocate_segment = allocate_segment_by_default,
3004 };
3005
3006 bool f2fs_exist_trim_candidates(struct f2fs_sb_info *sbi,
3007 struct cp_control *cpc)
3008 {
3009 __u64 trim_start = cpc->trim_start;
3010 bool has_candidate = false;
3011
3012 down_write(&SIT_I(sbi)->sentry_lock);
3013 for (; cpc->trim_start <= cpc->trim_end; cpc->trim_start++) {
3014 if (add_discard_addrs(sbi, cpc, true)) {
3015 has_candidate = true;
3016 break;
3017 }
3018 }
3019 up_write(&SIT_I(sbi)->sentry_lock);
3020
3021 cpc->trim_start = trim_start;
3022 return has_candidate;
3023 }
3024
3025 static unsigned int __issue_discard_cmd_range(struct f2fs_sb_info *sbi,
3026 struct discard_policy *dpolicy,
3027 unsigned int start, unsigned int end)
3028 {
3029 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
3030 struct discard_cmd *prev_dc = NULL, *next_dc = NULL;
3031 struct rb_node **insert_p = NULL, *insert_parent = NULL;
3032 struct discard_cmd *dc;
3033 struct blk_plug plug;
3034 int issued;
3035 unsigned int trimmed = 0;
3036
3037 next:
3038 issued = 0;
3039
3040 mutex_lock(&dcc->cmd_lock);
3041 if (unlikely(dcc->rbtree_check))
3042 f2fs_bug_on(sbi, !f2fs_check_rb_tree_consistence(sbi,
3043 &dcc->root, false));
3044
3045 dc = (struct discard_cmd *)f2fs_lookup_rb_tree_ret(&dcc->root,
3046 NULL, start,
3047 (struct rb_entry **)&prev_dc,
3048 (struct rb_entry **)&next_dc,
3049 &insert_p, &insert_parent, true, NULL);
3050 if (!dc)
3051 dc = next_dc;
3052
3053 blk_start_plug(&plug);
3054
3055 while (dc && dc->lstart <= end) {
3056 struct rb_node *node;
3057 int err = 0;
3058
3059 if (dc->len < dpolicy->granularity)
3060 goto skip;
3061
3062 if (dc->state != D_PREP) {
3063 list_move_tail(&dc->list, &dcc->fstrim_list);
3064 goto skip;
3065 }
3066
3067 err = __submit_discard_cmd(sbi, dpolicy, dc, &issued);
3068
3069 if (issued >= dpolicy->max_requests) {
3070 start = dc->lstart + dc->len;
3071
3072 if (err)
3073 __remove_discard_cmd(sbi, dc);
3074
3075 blk_finish_plug(&plug);
3076 mutex_unlock(&dcc->cmd_lock);
3077 trimmed += __wait_all_discard_cmd(sbi, NULL);
3078 congestion_wait(BLK_RW_ASYNC, DEFAULT_IO_TIMEOUT);
3079 goto next;
3080 }
3081 skip:
3082 node = rb_next(&dc->rb_node);
3083 if (err)
3084 __remove_discard_cmd(sbi, dc);
3085 dc = rb_entry_safe(node, struct discard_cmd, rb_node);
3086
3087 if (fatal_signal_pending(current))
3088 break;
3089 }
3090
3091 blk_finish_plug(&plug);
3092 mutex_unlock(&dcc->cmd_lock);
3093
3094 return trimmed;
3095 }
3096
3097 int f2fs_trim_fs(struct f2fs_sb_info *sbi, struct fstrim_range *range)
3098 {
3099 __u64 start = F2FS_BYTES_TO_BLK(range->start);
3100 __u64 end = start + F2FS_BYTES_TO_BLK(range->len) - 1;
3101 unsigned int start_segno, end_segno;
3102 block_t start_block, end_block;
3103 struct cp_control cpc;
3104 struct discard_policy dpolicy;
3105 unsigned long long trimmed = 0;
3106 int err = 0;
3107 bool need_align = f2fs_lfs_mode(sbi) && __is_large_section(sbi);
3108
3109 if (start >= MAX_BLKADDR(sbi) || range->len < sbi->blocksize)
3110 return -EINVAL;
3111
3112 if (end < MAIN_BLKADDR(sbi))
3113 goto out;
3114
3115 if (is_sbi_flag_set(sbi, SBI_NEED_FSCK)) {
3116 f2fs_warn(sbi, "Found FS corruption, run fsck to fix.");
3117 return -EFSCORRUPTED;
3118 }
3119
3120 /* start/end segment number in main_area */
3121 start_segno = (start <= MAIN_BLKADDR(sbi)) ? 0 : GET_SEGNO(sbi, start);
3122 end_segno = (end >= MAX_BLKADDR(sbi)) ? MAIN_SEGS(sbi) - 1 :
3123 GET_SEGNO(sbi, end);
3124 if (need_align) {
3125 start_segno = rounddown(start_segno, sbi->segs_per_sec);
3126 end_segno = roundup(end_segno + 1, sbi->segs_per_sec) - 1;
3127 }
3128
3129 cpc.reason = CP_DISCARD;
3130 cpc.trim_minlen = max_t(__u64, 1, F2FS_BYTES_TO_BLK(range->minlen));
3131 cpc.trim_start = start_segno;
3132 cpc.trim_end = end_segno;
3133
3134 if (sbi->discard_blks == 0)
3135 goto out;
3136
3137 down_write(&sbi->gc_lock);
3138 err = f2fs_write_checkpoint(sbi, &cpc);
3139 up_write(&sbi->gc_lock);
3140 if (err)
3141 goto out;
3142
3143 /*
3144 * We filed discard candidates, but actually we don't need to wait for
3145 * all of them, since they'll be issued in idle time along with runtime
3146 * discard option. User configuration looks like using runtime discard
3147 * or periodic fstrim instead of it.
3148 */
3149 if (f2fs_realtime_discard_enable(sbi))
3150 goto out;
3151
3152 start_block = START_BLOCK(sbi, start_segno);
3153 end_block = START_BLOCK(sbi, end_segno + 1);
3154
3155 __init_discard_policy(sbi, &dpolicy, DPOLICY_FSTRIM, cpc.trim_minlen);
3156 trimmed = __issue_discard_cmd_range(sbi, &dpolicy,
3157 start_block, end_block);
3158
3159 trimmed += __wait_discard_cmd_range(sbi, &dpolicy,
3160 start_block, end_block);
3161 out:
3162 if (!err)
3163 range->len = F2FS_BLK_TO_BYTES(trimmed);
3164 return err;
3165 }
3166
3167 static bool __has_curseg_space(struct f2fs_sb_info *sbi,
3168 struct curseg_info *curseg)
3169 {
3170 return curseg->next_blkoff < f2fs_usable_blks_in_seg(sbi,
3171 curseg->segno);
3172 }
3173
3174 int f2fs_rw_hint_to_seg_type(enum rw_hint hint)
3175 {
3176 switch (hint) {
3177 case WRITE_LIFE_SHORT:
3178 return CURSEG_HOT_DATA;
3179 case WRITE_LIFE_EXTREME:
3180 return CURSEG_COLD_DATA;
3181 default:
3182 return CURSEG_WARM_DATA;
3183 }
3184 }
3185
3186 /* This returns write hints for each segment type. This hints will be
3187 * passed down to block layer. There are mapping tables which depend on
3188 * the mount option 'whint_mode'.
3189 *
3190 * 1) whint_mode=off. F2FS only passes down WRITE_LIFE_NOT_SET.
3191 *
3192 * 2) whint_mode=user-based. F2FS tries to pass down hints given by users.
3193 *
3194 * User F2FS Block
3195 * ---- ---- -----
3196 * META WRITE_LIFE_NOT_SET
3197 * HOT_NODE "
3198 * WARM_NODE "
3199 * COLD_NODE "
3200 * ioctl(COLD) COLD_DATA WRITE_LIFE_EXTREME
3201 * extension list " "
3202 *
3203 * -- buffered io
3204 * WRITE_LIFE_EXTREME COLD_DATA WRITE_LIFE_EXTREME
3205 * WRITE_LIFE_SHORT HOT_DATA WRITE_LIFE_SHORT
3206 * WRITE_LIFE_NOT_SET WARM_DATA WRITE_LIFE_NOT_SET
3207 * WRITE_LIFE_NONE " "
3208 * WRITE_LIFE_MEDIUM " "
3209 * WRITE_LIFE_LONG " "
3210 *
3211 * -- direct io
3212 * WRITE_LIFE_EXTREME COLD_DATA WRITE_LIFE_EXTREME
3213 * WRITE_LIFE_SHORT HOT_DATA WRITE_LIFE_SHORT
3214 * WRITE_LIFE_NOT_SET WARM_DATA WRITE_LIFE_NOT_SET
3215 * WRITE_LIFE_NONE " WRITE_LIFE_NONE
3216 * WRITE_LIFE_MEDIUM " WRITE_LIFE_MEDIUM
3217 * WRITE_LIFE_LONG " WRITE_LIFE_LONG
3218 *
3219 * 3) whint_mode=fs-based. F2FS passes down hints with its policy.
3220 *
3221 * User F2FS Block
3222 * ---- ---- -----
3223 * META WRITE_LIFE_MEDIUM;
3224 * HOT_NODE WRITE_LIFE_NOT_SET
3225 * WARM_NODE "
3226 * COLD_NODE WRITE_LIFE_NONE
3227 * ioctl(COLD) COLD_DATA WRITE_LIFE_EXTREME
3228 * extension list " "
3229 *
3230 * -- buffered io
3231 * WRITE_LIFE_EXTREME COLD_DATA WRITE_LIFE_EXTREME
3232 * WRITE_LIFE_SHORT HOT_DATA WRITE_LIFE_SHORT
3233 * WRITE_LIFE_NOT_SET WARM_DATA WRITE_LIFE_LONG
3234 * WRITE_LIFE_NONE " "
3235 * WRITE_LIFE_MEDIUM " "
3236 * WRITE_LIFE_LONG " "
3237 *
3238 * -- direct io
3239 * WRITE_LIFE_EXTREME COLD_DATA WRITE_LIFE_EXTREME
3240 * WRITE_LIFE_SHORT HOT_DATA WRITE_LIFE_SHORT
3241 * WRITE_LIFE_NOT_SET WARM_DATA WRITE_LIFE_NOT_SET
3242 * WRITE_LIFE_NONE " WRITE_LIFE_NONE
3243 * WRITE_LIFE_MEDIUM " WRITE_LIFE_MEDIUM
3244 * WRITE_LIFE_LONG " WRITE_LIFE_LONG
3245 */
3246
3247 enum rw_hint f2fs_io_type_to_rw_hint(struct f2fs_sb_info *sbi,
3248 enum page_type type, enum temp_type temp)
3249 {
3250 if (F2FS_OPTION(sbi).whint_mode == WHINT_MODE_USER) {
3251 if (type == DATA) {
3252 if (temp == WARM)
3253 return WRITE_LIFE_NOT_SET;
3254 else if (temp == HOT)
3255 return WRITE_LIFE_SHORT;
3256 else if (temp == COLD)
3257 return WRITE_LIFE_EXTREME;
3258 } else {
3259 return WRITE_LIFE_NOT_SET;
3260 }
3261 } else if (F2FS_OPTION(sbi).whint_mode == WHINT_MODE_FS) {
3262 if (type == DATA) {
3263 if (temp == WARM)
3264 return WRITE_LIFE_LONG;
3265 else if (temp == HOT)
3266 return WRITE_LIFE_SHORT;
3267 else if (temp == COLD)
3268 return WRITE_LIFE_EXTREME;
3269 } else if (type == NODE) {
3270 if (temp == WARM || temp == HOT)
3271 return WRITE_LIFE_NOT_SET;
3272 else if (temp == COLD)
3273 return WRITE_LIFE_NONE;
3274 } else if (type == META) {
3275 return WRITE_LIFE_MEDIUM;
3276 }
3277 }
3278 return WRITE_LIFE_NOT_SET;
3279 }
3280
3281 static int __get_segment_type_2(struct f2fs_io_info *fio)
3282 {
3283 if (fio->type == DATA)
3284 return CURSEG_HOT_DATA;
3285 else
3286 return CURSEG_HOT_NODE;
3287 }
3288
3289 static int __get_segment_type_4(struct f2fs_io_info *fio)
3290 {
3291 if (fio->type == DATA) {
3292 struct inode *inode = fio->page->mapping->host;
3293
3294 if (S_ISDIR(inode->i_mode))
3295 return CURSEG_HOT_DATA;
3296 else
3297 return CURSEG_COLD_DATA;
3298 } else {
3299 if (IS_DNODE(fio->page) && is_cold_node(fio->page))
3300 return CURSEG_WARM_NODE;
3301 else
3302 return CURSEG_COLD_NODE;
3303 }
3304 }
3305
3306 static int __get_segment_type_6(struct f2fs_io_info *fio)
3307 {
3308 if (fio->type == DATA) {
3309 struct inode *inode = fio->page->mapping->host;
3310
3311 if (is_inode_flag_set(inode, FI_ALIGNED_WRITE))
3312 return CURSEG_COLD_DATA_PINNED;
3313
3314 if (page_private_gcing(fio->page)) {
3315 if (fio->sbi->am.atgc_enabled &&
3316 (fio->io_type == FS_DATA_IO) &&
3317 (fio->sbi->gc_mode != GC_URGENT_HIGH))
3318 return CURSEG_ALL_DATA_ATGC;
3319 else
3320 return CURSEG_COLD_DATA;
3321 }
3322 if (file_is_cold(inode) || f2fs_need_compress_data(inode))
3323 return CURSEG_COLD_DATA;
3324 if (file_is_hot(inode) ||
3325 is_inode_flag_set(inode, FI_HOT_DATA) ||
3326 f2fs_is_atomic_file(inode) ||
3327 f2fs_is_volatile_file(inode))
3328 return CURSEG_HOT_DATA;
3329 return f2fs_rw_hint_to_seg_type(inode->i_write_hint);
3330 } else {
3331 if (IS_DNODE(fio->page))
3332 return is_cold_node(fio->page) ? CURSEG_WARM_NODE :
3333 CURSEG_HOT_NODE;
3334 return CURSEG_COLD_NODE;
3335 }
3336 }
3337
3338 static int __get_segment_type(struct f2fs_io_info *fio)
3339 {
3340 int type = 0;
3341
3342 switch (F2FS_OPTION(fio->sbi).active_logs) {
3343 case 2:
3344 type = __get_segment_type_2(fio);
3345 break;
3346 case 4:
3347 type = __get_segment_type_4(fio);
3348 break;
3349 case 6:
3350 type = __get_segment_type_6(fio);
3351 break;
3352 default:
3353 f2fs_bug_on(fio->sbi, true);
3354 }
3355
3356 if (IS_HOT(type))
3357 fio->temp = HOT;
3358 else if (IS_WARM(type))
3359 fio->temp = WARM;
3360 else
3361 fio->temp = COLD;
3362 return type;
3363 }
3364
3365 void f2fs_allocate_data_block(struct f2fs_sb_info *sbi, struct page *page,
3366 block_t old_blkaddr, block_t *new_blkaddr,
3367 struct f2fs_summary *sum, int type,
3368 struct f2fs_io_info *fio)
3369 {
3370 struct sit_info *sit_i = SIT_I(sbi);
3371 struct curseg_info *curseg = CURSEG_I(sbi, type);
3372 unsigned long long old_mtime;
3373 bool from_gc = (type == CURSEG_ALL_DATA_ATGC);
3374 struct seg_entry *se = NULL;
3375
3376 down_read(&SM_I(sbi)->curseg_lock);
3377
3378 mutex_lock(&curseg->curseg_mutex);
3379 down_write(&sit_i->sentry_lock);
3380
3381 if (from_gc) {
3382 f2fs_bug_on(sbi, GET_SEGNO(sbi, old_blkaddr) == NULL_SEGNO);
3383 se = get_seg_entry(sbi, GET_SEGNO(sbi, old_blkaddr));
3384 sanity_check_seg_type(sbi, se->type);
3385 f2fs_bug_on(sbi, IS_NODESEG(se->type));
3386 }
3387 *new_blkaddr = NEXT_FREE_BLKADDR(sbi, curseg);
3388
3389 f2fs_bug_on(sbi, curseg->next_blkoff >= sbi->blocks_per_seg);
3390
3391 f2fs_wait_discard_bio(sbi, *new_blkaddr);
3392
3393 /*
3394 * __add_sum_entry should be resided under the curseg_mutex
3395 * because, this function updates a summary entry in the
3396 * current summary block.
3397 */
3398 __add_sum_entry(sbi, type, sum);
3399
3400 __refresh_next_blkoff(sbi, curseg);
3401
3402 stat_inc_block_count(sbi, curseg);
3403
3404 if (from_gc) {
3405 old_mtime = get_segment_mtime(sbi, old_blkaddr);
3406 } else {
3407 update_segment_mtime(sbi, old_blkaddr, 0);
3408 old_mtime = 0;
3409 }
3410 update_segment_mtime(sbi, *new_blkaddr, old_mtime);
3411
3412 /*
3413 * SIT information should be updated before segment allocation,
3414 * since SSR needs latest valid block information.
3415 */
3416 update_sit_entry(sbi, *new_blkaddr, 1);
3417 if (GET_SEGNO(sbi, old_blkaddr) != NULL_SEGNO)
3418 update_sit_entry(sbi, old_blkaddr, -1);
3419
3420 if (!__has_curseg_space(sbi, curseg)) {
3421 if (from_gc)
3422 get_atssr_segment(sbi, type, se->type,
3423 AT_SSR, se->mtime);
3424 else
3425 sit_i->s_ops->allocate_segment(sbi, type, false);
3426 }
3427 /*
3428 * segment dirty status should be updated after segment allocation,
3429 * so we just need to update status only one time after previous
3430 * segment being closed.
3431 */
3432 locate_dirty_segment(sbi, GET_SEGNO(sbi, old_blkaddr));
3433 locate_dirty_segment(sbi, GET_SEGNO(sbi, *new_blkaddr));
3434
3435 up_write(&sit_i->sentry_lock);
3436
3437 if (page && IS_NODESEG(type)) {
3438 fill_node_footer_blkaddr(page, NEXT_FREE_BLKADDR(sbi, curseg));
3439
3440 f2fs_inode_chksum_set(sbi, page);
3441 }
3442
3443 if (fio) {
3444 struct f2fs_bio_info *io;
3445
3446 if (F2FS_IO_ALIGNED(sbi))
3447 fio->retry = false;
3448
3449 INIT_LIST_HEAD(&fio->list);
3450 fio->in_list = true;
3451 io = sbi->write_io[fio->type] + fio->temp;
3452 spin_lock(&io->io_lock);
3453 list_add_tail(&fio->list, &io->io_list);
3454 spin_unlock(&io->io_lock);
3455 }
3456
3457 mutex_unlock(&curseg->curseg_mutex);
3458
3459 up_read(&SM_I(sbi)->curseg_lock);
3460 }
3461
3462 static void update_device_state(struct f2fs_io_info *fio)
3463 {
3464 struct f2fs_sb_info *sbi = fio->sbi;
3465 unsigned int devidx;
3466
3467 if (!f2fs_is_multi_device(sbi))
3468 return;
3469
3470 devidx = f2fs_target_device_index(sbi, fio->new_blkaddr);
3471
3472 /* update device state for fsync */
3473 f2fs_set_dirty_device(sbi, fio->ino, devidx, FLUSH_INO);
3474
3475 /* update device state for checkpoint */
3476 if (!f2fs_test_bit(devidx, (char *)&sbi->dirty_device)) {
3477 spin_lock(&sbi->dev_lock);
3478 f2fs_set_bit(devidx, (char *)&sbi->dirty_device);
3479 spin_unlock(&sbi->dev_lock);
3480 }
3481 }
3482
3483 static void do_write_page(struct f2fs_summary *sum, struct f2fs_io_info *fio)
3484 {
3485 int type = __get_segment_type(fio);
3486 bool keep_order = (f2fs_lfs_mode(fio->sbi) && type == CURSEG_COLD_DATA);
3487
3488 if (keep_order)
3489 down_read(&fio->sbi->io_order_lock);
3490 reallocate:
3491 f2fs_allocate_data_block(fio->sbi, fio->page, fio->old_blkaddr,
3492 &fio->new_blkaddr, sum, type, fio);
3493 if (GET_SEGNO(fio->sbi, fio->old_blkaddr) != NULL_SEGNO) {
3494 invalidate_mapping_pages(META_MAPPING(fio->sbi),
3495 fio->old_blkaddr, fio->old_blkaddr);
3496 f2fs_invalidate_compress_page(fio->sbi, fio->old_blkaddr);
3497 }
3498
3499 /* writeout dirty page into bdev */
3500 f2fs_submit_page_write(fio);
3501 if (fio->retry) {
3502 fio->old_blkaddr = fio->new_blkaddr;
3503 goto reallocate;
3504 }
3505
3506 update_device_state(fio);
3507
3508 if (keep_order)
3509 up_read(&fio->sbi->io_order_lock);
3510 }
3511
3512 void f2fs_do_write_meta_page(struct f2fs_sb_info *sbi, struct page *page,
3513 enum iostat_type io_type)
3514 {
3515 struct f2fs_io_info fio = {
3516 .sbi = sbi,
3517 .type = META,
3518 .temp = HOT,
3519 .op = REQ_OP_WRITE,
3520 .op_flags = REQ_SYNC | REQ_META | REQ_PRIO,
3521 .old_blkaddr = page->index,
3522 .new_blkaddr = page->index,
3523 .page = page,
3524 .encrypted_page = NULL,
3525 .in_list = false,
3526 };
3527
3528 if (unlikely(page->index >= MAIN_BLKADDR(sbi)))
3529 fio.op_flags &= ~REQ_META;
3530
3531 set_page_writeback(page);
3532 ClearPageError(page);
3533 f2fs_submit_page_write(&fio);
3534
3535 stat_inc_meta_count(sbi, page->index);
3536 f2fs_update_iostat(sbi, io_type, F2FS_BLKSIZE);
3537 }
3538
3539 void f2fs_do_write_node_page(unsigned int nid, struct f2fs_io_info *fio)
3540 {
3541 struct f2fs_summary sum;
3542
3543 set_summary(&sum, nid, 0, 0);
3544 do_write_page(&sum, fio);
3545
3546 f2fs_update_iostat(fio->sbi, fio->io_type, F2FS_BLKSIZE);
3547 }
3548
3549 void f2fs_outplace_write_data(struct dnode_of_data *dn,
3550 struct f2fs_io_info *fio)
3551 {
3552 struct f2fs_sb_info *sbi = fio->sbi;
3553 struct f2fs_summary sum;
3554
3555 f2fs_bug_on(sbi, dn->data_blkaddr == NULL_ADDR);
3556 set_summary(&sum, dn->nid, dn->ofs_in_node, fio->version);
3557 do_write_page(&sum, fio);
3558 f2fs_update_data_blkaddr(dn, fio->new_blkaddr);
3559
3560 f2fs_update_iostat(sbi, fio->io_type, F2FS_BLKSIZE);
3561 }
3562
3563 int f2fs_inplace_write_data(struct f2fs_io_info *fio)
3564 {
3565 int err;
3566 struct f2fs_sb_info *sbi = fio->sbi;
3567 unsigned int segno;
3568
3569 fio->new_blkaddr = fio->old_blkaddr;
3570 /* i/o temperature is needed for passing down write hints */
3571 __get_segment_type(fio);
3572
3573 segno = GET_SEGNO(sbi, fio->new_blkaddr);
3574
3575 if (!IS_DATASEG(get_seg_entry(sbi, segno)->type)) {
3576 set_sbi_flag(sbi, SBI_NEED_FSCK);
3577 f2fs_warn(sbi, "%s: incorrect segment(%u) type, run fsck to fix.",
3578 __func__, segno);
3579 err = -EFSCORRUPTED;
3580 goto drop_bio;
3581 }
3582
3583 if (f2fs_cp_error(sbi)) {
3584 err = -EIO;
3585 goto drop_bio;
3586 }
3587
3588 stat_inc_inplace_blocks(fio->sbi);
3589
3590 if (fio->bio && !(SM_I(sbi)->ipu_policy & (1 << F2FS_IPU_NOCACHE)))
3591 err = f2fs_merge_page_bio(fio);
3592 else
3593 err = f2fs_submit_page_bio(fio);
3594 if (!err) {
3595 update_device_state(fio);
3596 f2fs_update_iostat(fio->sbi, fio->io_type, F2FS_BLKSIZE);
3597 }
3598
3599 return err;
3600 drop_bio:
3601 if (fio->bio && *(fio->bio)) {
3602 struct bio *bio = *(fio->bio);
3603
3604 bio->bi_status = BLK_STS_IOERR;
3605 bio_endio(bio);
3606 *(fio->bio) = NULL;
3607 }
3608 return err;
3609 }
3610
3611 static inline int __f2fs_get_curseg(struct f2fs_sb_info *sbi,
3612 unsigned int segno)
3613 {
3614 int i;
3615
3616 for (i = CURSEG_HOT_DATA; i < NO_CHECK_TYPE; i++) {
3617 if (CURSEG_I(sbi, i)->segno == segno)
3618 break;
3619 }
3620 return i;
3621 }
3622
3623 void f2fs_do_replace_block(struct f2fs_sb_info *sbi, struct f2fs_summary *sum,
3624 block_t old_blkaddr, block_t new_blkaddr,
3625 bool recover_curseg, bool recover_newaddr,
3626 bool from_gc)
3627 {
3628 struct sit_info *sit_i = SIT_I(sbi);
3629 struct curseg_info *curseg;
3630 unsigned int segno, old_cursegno;
3631 struct seg_entry *se;
3632 int type;
3633 unsigned short old_blkoff;
3634 unsigned char old_alloc_type;
3635
3636 segno = GET_SEGNO(sbi, new_blkaddr);
3637 se = get_seg_entry(sbi, segno);
3638 type = se->type;
3639
3640 down_write(&SM_I(sbi)->curseg_lock);
3641
3642 if (!recover_curseg) {
3643 /* for recovery flow */
3644 if (se->valid_blocks == 0 && !IS_CURSEG(sbi, segno)) {
3645 if (old_blkaddr == NULL_ADDR)
3646 type = CURSEG_COLD_DATA;
3647 else
3648 type = CURSEG_WARM_DATA;
3649 }
3650 } else {
3651 if (IS_CURSEG(sbi, segno)) {
3652 /* se->type is volatile as SSR allocation */
3653 type = __f2fs_get_curseg(sbi, segno);
3654 f2fs_bug_on(sbi, type == NO_CHECK_TYPE);
3655 } else {
3656 type = CURSEG_WARM_DATA;
3657 }
3658 }
3659
3660 f2fs_bug_on(sbi, !IS_DATASEG(type));
3661 curseg = CURSEG_I(sbi, type);
3662
3663 mutex_lock(&curseg->curseg_mutex);
3664 down_write(&sit_i->sentry_lock);
3665
3666 old_cursegno = curseg->segno;
3667 old_blkoff = curseg->next_blkoff;
3668 old_alloc_type = curseg->alloc_type;
3669
3670 /* change the current segment */
3671 if (segno != curseg->segno) {
3672 curseg->next_segno = segno;
3673 change_curseg(sbi, type, true);
3674 }
3675
3676 curseg->next_blkoff = GET_BLKOFF_FROM_SEG0(sbi, new_blkaddr);
3677 __add_sum_entry(sbi, type, sum);
3678
3679 if (!recover_curseg || recover_newaddr) {
3680 if (!from_gc)
3681 update_segment_mtime(sbi, new_blkaddr, 0);
3682 update_sit_entry(sbi, new_blkaddr, 1);
3683 }
3684 if (GET_SEGNO(sbi, old_blkaddr) != NULL_SEGNO) {
3685 invalidate_mapping_pages(META_MAPPING(sbi),
3686 old_blkaddr, old_blkaddr);
3687 f2fs_invalidate_compress_page(sbi, old_blkaddr);
3688 if (!from_gc)
3689 update_segment_mtime(sbi, old_blkaddr, 0);
3690 update_sit_entry(sbi, old_blkaddr, -1);
3691 }
3692
3693 locate_dirty_segment(sbi, GET_SEGNO(sbi, old_blkaddr));
3694 locate_dirty_segment(sbi, GET_SEGNO(sbi, new_blkaddr));
3695
3696 locate_dirty_segment(sbi, old_cursegno);
3697
3698 if (recover_curseg) {
3699 if (old_cursegno != curseg->segno) {
3700 curseg->next_segno = old_cursegno;
3701 change_curseg(sbi, type, true);
3702 }
3703 curseg->next_blkoff = old_blkoff;
3704 curseg->alloc_type = old_alloc_type;
3705 }
3706
3707 up_write(&sit_i->sentry_lock);
3708 mutex_unlock(&curseg->curseg_mutex);
3709 up_write(&SM_I(sbi)->curseg_lock);
3710 }
3711
3712 void f2fs_replace_block(struct f2fs_sb_info *sbi, struct dnode_of_data *dn,
3713 block_t old_addr, block_t new_addr,
3714 unsigned char version, bool recover_curseg,
3715 bool recover_newaddr)
3716 {
3717 struct f2fs_summary sum;
3718
3719 set_summary(&sum, dn->nid, dn->ofs_in_node, version);
3720
3721 f2fs_do_replace_block(sbi, &sum, old_addr, new_addr,
3722 recover_curseg, recover_newaddr, false);
3723
3724 f2fs_update_data_blkaddr(dn, new_addr);
3725 }
3726
3727 void f2fs_wait_on_page_writeback(struct page *page,
3728 enum page_type type, bool ordered, bool locked)
3729 {
3730 if (PageWriteback(page)) {
3731 struct f2fs_sb_info *sbi = F2FS_P_SB(page);
3732
3733 /* submit cached LFS IO */
3734 f2fs_submit_merged_write_cond(sbi, NULL, page, 0, type);
3735 /* sbumit cached IPU IO */
3736 f2fs_submit_merged_ipu_write(sbi, NULL, page);
3737 if (ordered) {
3738 wait_on_page_writeback(page);
3739 f2fs_bug_on(sbi, locked && PageWriteback(page));
3740 } else {
3741 wait_for_stable_page(page);
3742 }
3743 }
3744 }
3745
3746 void f2fs_wait_on_block_writeback(struct inode *inode, block_t blkaddr)
3747 {
3748 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
3749 struct page *cpage;
3750
3751 if (!f2fs_post_read_required(inode))
3752 return;
3753
3754 if (!__is_valid_data_blkaddr(blkaddr))
3755 return;
3756
3757 cpage = find_lock_page(META_MAPPING(sbi), blkaddr);
3758 if (cpage) {
3759 f2fs_wait_on_page_writeback(cpage, DATA, true, true);
3760 f2fs_put_page(cpage, 1);
3761 }
3762 }
3763
3764 void f2fs_wait_on_block_writeback_range(struct inode *inode, block_t blkaddr,
3765 block_t len)
3766 {
3767 block_t i;
3768
3769 for (i = 0; i < len; i++)
3770 f2fs_wait_on_block_writeback(inode, blkaddr + i);
3771 }
3772
3773 static int read_compacted_summaries(struct f2fs_sb_info *sbi)
3774 {
3775 struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi);
3776 struct curseg_info *seg_i;
3777 unsigned char *kaddr;
3778 struct page *page;
3779 block_t start;
3780 int i, j, offset;
3781
3782 start = start_sum_block(sbi);
3783
3784 page = f2fs_get_meta_page(sbi, start++);
3785 if (IS_ERR(page))
3786 return PTR_ERR(page);
3787 kaddr = (unsigned char *)page_address(page);
3788
3789 /* Step 1: restore nat cache */
3790 seg_i = CURSEG_I(sbi, CURSEG_HOT_DATA);
3791 memcpy(seg_i->journal, kaddr, SUM_JOURNAL_SIZE);
3792
3793 /* Step 2: restore sit cache */
3794 seg_i = CURSEG_I(sbi, CURSEG_COLD_DATA);
3795 memcpy(seg_i->journal, kaddr + SUM_JOURNAL_SIZE, SUM_JOURNAL_SIZE);
3796 offset = 2 * SUM_JOURNAL_SIZE;
3797
3798 /* Step 3: restore summary entries */
3799 for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++) {
3800 unsigned short blk_off;
3801 unsigned int segno;
3802
3803 seg_i = CURSEG_I(sbi, i);
3804 segno = le32_to_cpu(ckpt->cur_data_segno[i]);
3805 blk_off = le16_to_cpu(ckpt->cur_data_blkoff[i]);
3806 seg_i->next_segno = segno;
3807 reset_curseg(sbi, i, 0);
3808 seg_i->alloc_type = ckpt->alloc_type[i];
3809 seg_i->next_blkoff = blk_off;
3810
3811 if (seg_i->alloc_type == SSR)
3812 blk_off = sbi->blocks_per_seg;
3813
3814 for (j = 0; j < blk_off; j++) {
3815 struct f2fs_summary *s;
3816
3817 s = (struct f2fs_summary *)(kaddr + offset);
3818 seg_i->sum_blk->entries[j] = *s;
3819 offset += SUMMARY_SIZE;
3820 if (offset + SUMMARY_SIZE <= PAGE_SIZE -
3821 SUM_FOOTER_SIZE)
3822 continue;
3823
3824 f2fs_put_page(page, 1);
3825 page = NULL;
3826
3827 page = f2fs_get_meta_page(sbi, start++);
3828 if (IS_ERR(page))
3829 return PTR_ERR(page);
3830 kaddr = (unsigned char *)page_address(page);
3831 offset = 0;
3832 }
3833 }
3834 f2fs_put_page(page, 1);
3835 return 0;
3836 }
3837
3838 static int read_normal_summaries(struct f2fs_sb_info *sbi, int type)
3839 {
3840 struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi);
3841 struct f2fs_summary_block *sum;
3842 struct curseg_info *curseg;
3843 struct page *new;
3844 unsigned short blk_off;
3845 unsigned int segno = 0;
3846 block_t blk_addr = 0;
3847 int err = 0;
3848
3849 /* get segment number and block addr */
3850 if (IS_DATASEG(type)) {
3851 segno = le32_to_cpu(ckpt->cur_data_segno[type]);
3852 blk_off = le16_to_cpu(ckpt->cur_data_blkoff[type -
3853 CURSEG_HOT_DATA]);
3854 if (__exist_node_summaries(sbi))
3855 blk_addr = sum_blk_addr(sbi, NR_CURSEG_PERSIST_TYPE, type);
3856 else
3857 blk_addr = sum_blk_addr(sbi, NR_CURSEG_DATA_TYPE, type);
3858 } else {
3859 segno = le32_to_cpu(ckpt->cur_node_segno[type -
3860 CURSEG_HOT_NODE]);
3861 blk_off = le16_to_cpu(ckpt->cur_node_blkoff[type -
3862 CURSEG_HOT_NODE]);
3863 if (__exist_node_summaries(sbi))
3864 blk_addr = sum_blk_addr(sbi, NR_CURSEG_NODE_TYPE,
3865 type - CURSEG_HOT_NODE);
3866 else
3867 blk_addr = GET_SUM_BLOCK(sbi, segno);
3868 }
3869
3870 new = f2fs_get_meta_page(sbi, blk_addr);
3871 if (IS_ERR(new))
3872 return PTR_ERR(new);
3873 sum = (struct f2fs_summary_block *)page_address(new);
3874
3875 if (IS_NODESEG(type)) {
3876 if (__exist_node_summaries(sbi)) {
3877 struct f2fs_summary *ns = &sum->entries[0];
3878 int i;
3879
3880 for (i = 0; i < sbi->blocks_per_seg; i++, ns++) {
3881 ns->version = 0;
3882 ns->ofs_in_node = 0;
3883 }
3884 } else {
3885 err = f2fs_restore_node_summary(sbi, segno, sum);
3886 if (err)
3887 goto out;
3888 }
3889 }
3890
3891 /* set uncompleted segment to curseg */
3892 curseg = CURSEG_I(sbi, type);
3893 mutex_lock(&curseg->curseg_mutex);
3894
3895 /* update journal info */
3896 down_write(&curseg->journal_rwsem);
3897 memcpy(curseg->journal, &sum->journal, SUM_JOURNAL_SIZE);
3898 up_write(&curseg->journal_rwsem);
3899
3900 memcpy(curseg->sum_blk->entries, sum->entries, SUM_ENTRY_SIZE);
3901 memcpy(&curseg->sum_blk->footer, &sum->footer, SUM_FOOTER_SIZE);
3902 curseg->next_segno = segno;
3903 reset_curseg(sbi, type, 0);
3904 curseg->alloc_type = ckpt->alloc_type[type];
3905 curseg->next_blkoff = blk_off;
3906 mutex_unlock(&curseg->curseg_mutex);
3907 out:
3908 f2fs_put_page(new, 1);
3909 return err;
3910 }
3911
3912 static int restore_curseg_summaries(struct f2fs_sb_info *sbi)
3913 {
3914 struct f2fs_journal *sit_j = CURSEG_I(sbi, CURSEG_COLD_DATA)->journal;
3915 struct f2fs_journal *nat_j = CURSEG_I(sbi, CURSEG_HOT_DATA)->journal;
3916 int type = CURSEG_HOT_DATA;
3917 int err;
3918
3919 if (is_set_ckpt_flags(sbi, CP_COMPACT_SUM_FLAG)) {
3920 int npages = f2fs_npages_for_summary_flush(sbi, true);
3921
3922 if (npages >= 2)
3923 f2fs_ra_meta_pages(sbi, start_sum_block(sbi), npages,
3924 META_CP, true);
3925
3926 /* restore for compacted data summary */
3927 err = read_compacted_summaries(sbi);
3928 if (err)
3929 return err;
3930 type = CURSEG_HOT_NODE;
3931 }
3932
3933 if (__exist_node_summaries(sbi))
3934 f2fs_ra_meta_pages(sbi,
3935 sum_blk_addr(sbi, NR_CURSEG_PERSIST_TYPE, type),
3936 NR_CURSEG_PERSIST_TYPE - type, META_CP, true);
3937
3938 for (; type <= CURSEG_COLD_NODE; type++) {
3939 err = read_normal_summaries(sbi, type);
3940 if (err)
3941 return err;
3942 }
3943
3944 /* sanity check for summary blocks */
3945 if (nats_in_cursum(nat_j) > NAT_JOURNAL_ENTRIES ||
3946 sits_in_cursum(sit_j) > SIT_JOURNAL_ENTRIES) {
3947 f2fs_err(sbi, "invalid journal entries nats %u sits %u",
3948 nats_in_cursum(nat_j), sits_in_cursum(sit_j));
3949 return -EINVAL;
3950 }
3951
3952 return 0;
3953 }
3954
3955 static void write_compacted_summaries(struct f2fs_sb_info *sbi, block_t blkaddr)
3956 {
3957 struct page *page;
3958 unsigned char *kaddr;
3959 struct f2fs_summary *summary;
3960 struct curseg_info *seg_i;
3961 int written_size = 0;
3962 int i, j;
3963
3964 page = f2fs_grab_meta_page(sbi, blkaddr++);
3965 kaddr = (unsigned char *)page_address(page);
3966 memset(kaddr, 0, PAGE_SIZE);
3967
3968 /* Step 1: write nat cache */
3969 seg_i = CURSEG_I(sbi, CURSEG_HOT_DATA);
3970 memcpy(kaddr, seg_i->journal, SUM_JOURNAL_SIZE);
3971 written_size += SUM_JOURNAL_SIZE;
3972
3973 /* Step 2: write sit cache */
3974 seg_i = CURSEG_I(sbi, CURSEG_COLD_DATA);
3975 memcpy(kaddr + written_size, seg_i->journal, SUM_JOURNAL_SIZE);
3976 written_size += SUM_JOURNAL_SIZE;
3977
3978 /* Step 3: write summary entries */
3979 for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++) {
3980 unsigned short blkoff;
3981
3982 seg_i = CURSEG_I(sbi, i);
3983 if (sbi->ckpt->alloc_type[i] == SSR)
3984 blkoff = sbi->blocks_per_seg;
3985 else
3986 blkoff = curseg_blkoff(sbi, i);
3987
3988 for (j = 0; j < blkoff; j++) {
3989 if (!page) {
3990 page = f2fs_grab_meta_page(sbi, blkaddr++);
3991 kaddr = (unsigned char *)page_address(page);
3992 memset(kaddr, 0, PAGE_SIZE);
3993 written_size = 0;
3994 }
3995 summary = (struct f2fs_summary *)(kaddr + written_size);
3996 *summary = seg_i->sum_blk->entries[j];
3997 written_size += SUMMARY_SIZE;
3998
3999 if (written_size + SUMMARY_SIZE <= PAGE_SIZE -
4000 SUM_FOOTER_SIZE)
4001 continue;
4002
4003 set_page_dirty(page);
4004 f2fs_put_page(page, 1);
4005 page = NULL;
4006 }
4007 }
4008 if (page) {
4009 set_page_dirty(page);
4010 f2fs_put_page(page, 1);
4011 }
4012 }
4013
4014 static void write_normal_summaries(struct f2fs_sb_info *sbi,
4015 block_t blkaddr, int type)
4016 {
4017 int i, end;
4018
4019 if (IS_DATASEG(type))
4020 end = type + NR_CURSEG_DATA_TYPE;
4021 else
4022 end = type + NR_CURSEG_NODE_TYPE;
4023
4024 for (i = type; i < end; i++)
4025 write_current_sum_page(sbi, i, blkaddr + (i - type));
4026 }
4027
4028 void f2fs_write_data_summaries(struct f2fs_sb_info *sbi, block_t start_blk)
4029 {
4030 if (is_set_ckpt_flags(sbi, CP_COMPACT_SUM_FLAG))
4031 write_compacted_summaries(sbi, start_blk);
4032 else
4033 write_normal_summaries(sbi, start_blk, CURSEG_HOT_DATA);
4034 }
4035
4036 void f2fs_write_node_summaries(struct f2fs_sb_info *sbi, block_t start_blk)
4037 {
4038 write_normal_summaries(sbi, start_blk, CURSEG_HOT_NODE);
4039 }
4040
4041 int f2fs_lookup_journal_in_cursum(struct f2fs_journal *journal, int type,
4042 unsigned int val, int alloc)
4043 {
4044 int i;
4045
4046 if (type == NAT_JOURNAL) {
4047 for (i = 0; i < nats_in_cursum(journal); i++) {
4048 if (le32_to_cpu(nid_in_journal(journal, i)) == val)
4049 return i;
4050 }
4051 if (alloc && __has_cursum_space(journal, 1, NAT_JOURNAL))
4052 return update_nats_in_cursum(journal, 1);
4053 } else if (type == SIT_JOURNAL) {
4054 for (i = 0; i < sits_in_cursum(journal); i++)
4055 if (le32_to_cpu(segno_in_journal(journal, i)) == val)
4056 return i;
4057 if (alloc && __has_cursum_space(journal, 1, SIT_JOURNAL))
4058 return update_sits_in_cursum(journal, 1);
4059 }
4060 return -1;
4061 }
4062
4063 static struct page *get_current_sit_page(struct f2fs_sb_info *sbi,
4064 unsigned int segno)
4065 {
4066 return f2fs_get_meta_page(sbi, current_sit_addr(sbi, segno));
4067 }
4068
4069 static struct page *get_next_sit_page(struct f2fs_sb_info *sbi,
4070 unsigned int start)
4071 {
4072 struct sit_info *sit_i = SIT_I(sbi);
4073 struct page *page;
4074 pgoff_t src_off, dst_off;
4075
4076 src_off = current_sit_addr(sbi, start);
4077 dst_off = next_sit_addr(sbi, src_off);
4078
4079 page = f2fs_grab_meta_page(sbi, dst_off);
4080 seg_info_to_sit_page(sbi, page, start);
4081
4082 set_page_dirty(page);
4083 set_to_next_sit(sit_i, start);
4084
4085 return page;
4086 }
4087
4088 static struct sit_entry_set *grab_sit_entry_set(void)
4089 {
4090 struct sit_entry_set *ses =
4091 f2fs_kmem_cache_alloc(sit_entry_set_slab, GFP_NOFS);
4092
4093 ses->entry_cnt = 0;
4094 INIT_LIST_HEAD(&ses->set_list);
4095 return ses;
4096 }
4097
4098 static void release_sit_entry_set(struct sit_entry_set *ses)
4099 {
4100 list_del(&ses->set_list);
4101 kmem_cache_free(sit_entry_set_slab, ses);
4102 }
4103
4104 static void adjust_sit_entry_set(struct sit_entry_set *ses,
4105 struct list_head *head)
4106 {
4107 struct sit_entry_set *next = ses;
4108
4109 if (list_is_last(&ses->set_list, head))
4110 return;
4111
4112 list_for_each_entry_continue(next, head, set_list)
4113 if (ses->entry_cnt <= next->entry_cnt)
4114 break;
4115
4116 list_move_tail(&ses->set_list, &next->set_list);
4117 }
4118
4119 static void add_sit_entry(unsigned int segno, struct list_head *head)
4120 {
4121 struct sit_entry_set *ses;
4122 unsigned int start_segno = START_SEGNO(segno);
4123
4124 list_for_each_entry(ses, head, set_list) {
4125 if (ses->start_segno == start_segno) {
4126 ses->entry_cnt++;
4127 adjust_sit_entry_set(ses, head);
4128 return;
4129 }
4130 }
4131
4132 ses = grab_sit_entry_set();
4133
4134 ses->start_segno = start_segno;
4135 ses->entry_cnt++;
4136 list_add(&ses->set_list, head);
4137 }
4138
4139 static void add_sits_in_set(struct f2fs_sb_info *sbi)
4140 {
4141 struct f2fs_sm_info *sm_info = SM_I(sbi);
4142 struct list_head *set_list = &sm_info->sit_entry_set;
4143 unsigned long *bitmap = SIT_I(sbi)->dirty_sentries_bitmap;
4144 unsigned int segno;
4145
4146 for_each_set_bit(segno, bitmap, MAIN_SEGS(sbi))
4147 add_sit_entry(segno, set_list);
4148 }
4149
4150 static void remove_sits_in_journal(struct f2fs_sb_info *sbi)
4151 {
4152 struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_COLD_DATA);
4153 struct f2fs_journal *journal = curseg->journal;
4154 int i;
4155
4156 down_write(&curseg->journal_rwsem);
4157 for (i = 0; i < sits_in_cursum(journal); i++) {
4158 unsigned int segno;
4159 bool dirtied;
4160
4161 segno = le32_to_cpu(segno_in_journal(journal, i));
4162 dirtied = __mark_sit_entry_dirty(sbi, segno);
4163
4164 if (!dirtied)
4165 add_sit_entry(segno, &SM_I(sbi)->sit_entry_set);
4166 }
4167 update_sits_in_cursum(journal, -i);
4168 up_write(&curseg->journal_rwsem);
4169 }
4170
4171 /*
4172 * CP calls this function, which flushes SIT entries including sit_journal,
4173 * and moves prefree segs to free segs.
4174 */
4175 void f2fs_flush_sit_entries(struct f2fs_sb_info *sbi, struct cp_control *cpc)
4176 {
4177 struct sit_info *sit_i = SIT_I(sbi);
4178 unsigned long *bitmap = sit_i->dirty_sentries_bitmap;
4179 struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_COLD_DATA);
4180 struct f2fs_journal *journal = curseg->journal;
4181 struct sit_entry_set *ses, *tmp;
4182 struct list_head *head = &SM_I(sbi)->sit_entry_set;
4183 bool to_journal = !is_sbi_flag_set(sbi, SBI_IS_RESIZEFS);
4184 struct seg_entry *se;
4185
4186 down_write(&sit_i->sentry_lock);
4187
4188 if (!sit_i->dirty_sentries)
4189 goto out;
4190
4191 /*
4192 * add and account sit entries of dirty bitmap in sit entry
4193 * set temporarily
4194 */
4195 add_sits_in_set(sbi);
4196
4197 /*
4198 * if there are no enough space in journal to store dirty sit
4199 * entries, remove all entries from journal and add and account
4200 * them in sit entry set.
4201 */
4202 if (!__has_cursum_space(journal, sit_i->dirty_sentries, SIT_JOURNAL) ||
4203 !to_journal)
4204 remove_sits_in_journal(sbi);
4205
4206 /*
4207 * there are two steps to flush sit entries:
4208 * #1, flush sit entries to journal in current cold data summary block.
4209 * #2, flush sit entries to sit page.
4210 */
4211 list_for_each_entry_safe(ses, tmp, head, set_list) {
4212 struct page *page = NULL;
4213 struct f2fs_sit_block *raw_sit = NULL;
4214 unsigned int start_segno = ses->start_segno;
4215 unsigned int end = min(start_segno + SIT_ENTRY_PER_BLOCK,
4216 (unsigned long)MAIN_SEGS(sbi));
4217 unsigned int segno = start_segno;
4218
4219 if (to_journal &&
4220 !__has_cursum_space(journal, ses->entry_cnt, SIT_JOURNAL))
4221 to_journal = false;
4222
4223 if (to_journal) {
4224 down_write(&curseg->journal_rwsem);
4225 } else {
4226 page = get_next_sit_page(sbi, start_segno);
4227 raw_sit = page_address(page);
4228 }
4229
4230 /* flush dirty sit entries in region of current sit set */
4231 for_each_set_bit_from(segno, bitmap, end) {
4232 int offset, sit_offset;
4233
4234 se = get_seg_entry(sbi, segno);
4235 #ifdef CONFIG_F2FS_CHECK_FS
4236 if (memcmp(se->cur_valid_map, se->cur_valid_map_mir,
4237 SIT_VBLOCK_MAP_SIZE))
4238 f2fs_bug_on(sbi, 1);
4239 #endif
4240
4241 /* add discard candidates */
4242 if (!(cpc->reason & CP_DISCARD)) {
4243 cpc->trim_start = segno;
4244 add_discard_addrs(sbi, cpc, false);
4245 }
4246
4247 if (to_journal) {
4248 offset = f2fs_lookup_journal_in_cursum(journal,
4249 SIT_JOURNAL, segno, 1);
4250 f2fs_bug_on(sbi, offset < 0);
4251 segno_in_journal(journal, offset) =
4252 cpu_to_le32(segno);
4253 seg_info_to_raw_sit(se,
4254 &sit_in_journal(journal, offset));
4255 check_block_count(sbi, segno,
4256 &sit_in_journal(journal, offset));
4257 } else {
4258 sit_offset = SIT_ENTRY_OFFSET(sit_i, segno);
4259 seg_info_to_raw_sit(se,
4260 &raw_sit->entries[sit_offset]);
4261 check_block_count(sbi, segno,
4262 &raw_sit->entries[sit_offset]);
4263 }
4264
4265 __clear_bit(segno, bitmap);
4266 sit_i->dirty_sentries--;
4267 ses->entry_cnt--;
4268 }
4269
4270 if (to_journal)
4271 up_write(&curseg->journal_rwsem);
4272 else
4273 f2fs_put_page(page, 1);
4274
4275 f2fs_bug_on(sbi, ses->entry_cnt);
4276 release_sit_entry_set(ses);
4277 }
4278
4279 f2fs_bug_on(sbi, !list_empty(head));
4280 f2fs_bug_on(sbi, sit_i->dirty_sentries);
4281 out:
4282 if (cpc->reason & CP_DISCARD) {
4283 __u64 trim_start = cpc->trim_start;
4284
4285 for (; cpc->trim_start <= cpc->trim_end; cpc->trim_start++)
4286 add_discard_addrs(sbi, cpc, false);
4287
4288 cpc->trim_start = trim_start;
4289 }
4290 up_write(&sit_i->sentry_lock);
4291
4292 set_prefree_as_free_segments(sbi);
4293 }
4294
4295 static int build_sit_info(struct f2fs_sb_info *sbi)
4296 {
4297 struct f2fs_super_block *raw_super = F2FS_RAW_SUPER(sbi);
4298 struct sit_info *sit_i;
4299 unsigned int sit_segs, start;
4300 char *src_bitmap, *bitmap;
4301 unsigned int bitmap_size, main_bitmap_size, sit_bitmap_size;
4302 unsigned int discard_map = f2fs_block_unit_discard(sbi) ? 1 : 0;
4303
4304 /* allocate memory for SIT information */
4305 sit_i = f2fs_kzalloc(sbi, sizeof(struct sit_info), GFP_KERNEL);
4306 if (!sit_i)
4307 return -ENOMEM;
4308
4309 SM_I(sbi)->sit_info = sit_i;
4310
4311 sit_i->sentries =
4312 f2fs_kvzalloc(sbi, array_size(sizeof(struct seg_entry),
4313 MAIN_SEGS(sbi)),
4314 GFP_KERNEL);
4315 if (!sit_i->sentries)
4316 return -ENOMEM;
4317
4318 main_bitmap_size = f2fs_bitmap_size(MAIN_SEGS(sbi));
4319 sit_i->dirty_sentries_bitmap = f2fs_kvzalloc(sbi, main_bitmap_size,
4320 GFP_KERNEL);
4321 if (!sit_i->dirty_sentries_bitmap)
4322 return -ENOMEM;
4323
4324 #ifdef CONFIG_F2FS_CHECK_FS
4325 bitmap_size = MAIN_SEGS(sbi) * SIT_VBLOCK_MAP_SIZE * (3 + discard_map);
4326 #else
4327 bitmap_size = MAIN_SEGS(sbi) * SIT_VBLOCK_MAP_SIZE * (2 + discard_map);
4328 #endif
4329 sit_i->bitmap = f2fs_kvzalloc(sbi, bitmap_size, GFP_KERNEL);
4330 if (!sit_i->bitmap)
4331 return -ENOMEM;
4332
4333 bitmap = sit_i->bitmap;
4334
4335 for (start = 0; start < MAIN_SEGS(sbi); start++) {
4336 sit_i->sentries[start].cur_valid_map = bitmap;
4337 bitmap += SIT_VBLOCK_MAP_SIZE;
4338
4339 sit_i->sentries[start].ckpt_valid_map = bitmap;
4340 bitmap += SIT_VBLOCK_MAP_SIZE;
4341
4342 #ifdef CONFIG_F2FS_CHECK_FS
4343 sit_i->sentries[start].cur_valid_map_mir = bitmap;
4344 bitmap += SIT_VBLOCK_MAP_SIZE;
4345 #endif
4346
4347 if (discard_map) {
4348 sit_i->sentries[start].discard_map = bitmap;
4349 bitmap += SIT_VBLOCK_MAP_SIZE;
4350 }
4351 }
4352
4353 sit_i->tmp_map = f2fs_kzalloc(sbi, SIT_VBLOCK_MAP_SIZE, GFP_KERNEL);
4354 if (!sit_i->tmp_map)
4355 return -ENOMEM;
4356
4357 if (__is_large_section(sbi)) {
4358 sit_i->sec_entries =
4359 f2fs_kvzalloc(sbi, array_size(sizeof(struct sec_entry),
4360 MAIN_SECS(sbi)),
4361 GFP_KERNEL);
4362 if (!sit_i->sec_entries)
4363 return -ENOMEM;
4364 }
4365
4366 /* get information related with SIT */
4367 sit_segs = le32_to_cpu(raw_super->segment_count_sit) >> 1;
4368
4369 /* setup SIT bitmap from ckeckpoint pack */
4370 sit_bitmap_size = __bitmap_size(sbi, SIT_BITMAP);
4371 src_bitmap = __bitmap_ptr(sbi, SIT_BITMAP);
4372
4373 sit_i->sit_bitmap = kmemdup(src_bitmap, sit_bitmap_size, GFP_KERNEL);
4374 if (!sit_i->sit_bitmap)
4375 return -ENOMEM;
4376
4377 #ifdef CONFIG_F2FS_CHECK_FS
4378 sit_i->sit_bitmap_mir = kmemdup(src_bitmap,
4379 sit_bitmap_size, GFP_KERNEL);
4380 if (!sit_i->sit_bitmap_mir)
4381 return -ENOMEM;
4382
4383 sit_i->invalid_segmap = f2fs_kvzalloc(sbi,
4384 main_bitmap_size, GFP_KERNEL);
4385 if (!sit_i->invalid_segmap)
4386 return -ENOMEM;
4387 #endif
4388
4389 /* init SIT information */
4390 sit_i->s_ops = &default_salloc_ops;
4391
4392 sit_i->sit_base_addr = le32_to_cpu(raw_super->sit_blkaddr);
4393 sit_i->sit_blocks = sit_segs << sbi->log_blocks_per_seg;
4394 sit_i->written_valid_blocks = 0;
4395 sit_i->bitmap_size = sit_bitmap_size;
4396 sit_i->dirty_sentries = 0;
4397 sit_i->sents_per_block = SIT_ENTRY_PER_BLOCK;
4398 sit_i->elapsed_time = le64_to_cpu(sbi->ckpt->elapsed_time);
4399 sit_i->mounted_time = ktime_get_boottime_seconds();
4400 init_rwsem(&sit_i->sentry_lock);
4401 return 0;
4402 }
4403
4404 static int build_free_segmap(struct f2fs_sb_info *sbi)
4405 {
4406 struct free_segmap_info *free_i;
4407 unsigned int bitmap_size, sec_bitmap_size;
4408
4409 /* allocate memory for free segmap information */
4410 free_i = f2fs_kzalloc(sbi, sizeof(struct free_segmap_info), GFP_KERNEL);
4411 if (!free_i)
4412 return -ENOMEM;
4413
4414 SM_I(sbi)->free_info = free_i;
4415
4416 bitmap_size = f2fs_bitmap_size(MAIN_SEGS(sbi));
4417 free_i->free_segmap = f2fs_kvmalloc(sbi, bitmap_size, GFP_KERNEL);
4418 if (!free_i->free_segmap)
4419 return -ENOMEM;
4420
4421 sec_bitmap_size = f2fs_bitmap_size(MAIN_SECS(sbi));
4422 free_i->free_secmap = f2fs_kvmalloc(sbi, sec_bitmap_size, GFP_KERNEL);
4423 if (!free_i->free_secmap)
4424 return -ENOMEM;
4425
4426 /* set all segments as dirty temporarily */
4427 memset(free_i->free_segmap, 0xff, bitmap_size);
4428 memset(free_i->free_secmap, 0xff, sec_bitmap_size);
4429
4430 /* init free segmap information */
4431 free_i->start_segno = GET_SEGNO_FROM_SEG0(sbi, MAIN_BLKADDR(sbi));
4432 free_i->free_segments = 0;
4433 free_i->free_sections = 0;
4434 spin_lock_init(&free_i->segmap_lock);
4435 return 0;
4436 }
4437
4438 static int build_curseg(struct f2fs_sb_info *sbi)
4439 {
4440 struct curseg_info *array;
4441 int i;
4442
4443 array = f2fs_kzalloc(sbi, array_size(NR_CURSEG_TYPE,
4444 sizeof(*array)), GFP_KERNEL);
4445 if (!array)
4446 return -ENOMEM;
4447
4448 SM_I(sbi)->curseg_array = array;
4449
4450 for (i = 0; i < NO_CHECK_TYPE; i++) {
4451 mutex_init(&array[i].curseg_mutex);
4452 array[i].sum_blk = f2fs_kzalloc(sbi, PAGE_SIZE, GFP_KERNEL);
4453 if (!array[i].sum_blk)
4454 return -ENOMEM;
4455 init_rwsem(&array[i].journal_rwsem);
4456 array[i].journal = f2fs_kzalloc(sbi,
4457 sizeof(struct f2fs_journal), GFP_KERNEL);
4458 if (!array[i].journal)
4459 return -ENOMEM;
4460 if (i < NR_PERSISTENT_LOG)
4461 array[i].seg_type = CURSEG_HOT_DATA + i;
4462 else if (i == CURSEG_COLD_DATA_PINNED)
4463 array[i].seg_type = CURSEG_COLD_DATA;
4464 else if (i == CURSEG_ALL_DATA_ATGC)
4465 array[i].seg_type = CURSEG_COLD_DATA;
4466 array[i].segno = NULL_SEGNO;
4467 array[i].next_blkoff = 0;
4468 array[i].inited = false;
4469 }
4470 return restore_curseg_summaries(sbi);
4471 }
4472
4473 static int build_sit_entries(struct f2fs_sb_info *sbi)
4474 {
4475 struct sit_info *sit_i = SIT_I(sbi);
4476 struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_COLD_DATA);
4477 struct f2fs_journal *journal = curseg->journal;
4478 struct seg_entry *se;
4479 struct f2fs_sit_entry sit;
4480 int sit_blk_cnt = SIT_BLK_CNT(sbi);
4481 unsigned int i, start, end;
4482 unsigned int readed, start_blk = 0;
4483 int err = 0;
4484 block_t total_node_blocks = 0;
4485
4486 do {
4487 readed = f2fs_ra_meta_pages(sbi, start_blk, BIO_MAX_VECS,
4488 META_SIT, true);
4489
4490 start = start_blk * sit_i->sents_per_block;
4491 end = (start_blk + readed) * sit_i->sents_per_block;
4492
4493 for (; start < end && start < MAIN_SEGS(sbi); start++) {
4494 struct f2fs_sit_block *sit_blk;
4495 struct page *page;
4496
4497 se = &sit_i->sentries[start];
4498 page = get_current_sit_page(sbi, start);
4499 if (IS_ERR(page))
4500 return PTR_ERR(page);
4501 sit_blk = (struct f2fs_sit_block *)page_address(page);
4502 sit = sit_blk->entries[SIT_ENTRY_OFFSET(sit_i, start)];
4503 f2fs_put_page(page, 1);
4504
4505 err = check_block_count(sbi, start, &sit);
4506 if (err)
4507 return err;
4508 seg_info_from_raw_sit(se, &sit);
4509 if (IS_NODESEG(se->type))
4510 total_node_blocks += se->valid_blocks;
4511
4512 if (f2fs_block_unit_discard(sbi)) {
4513 /* build discard map only one time */
4514 if (is_set_ckpt_flags(sbi, CP_TRIMMED_FLAG)) {
4515 memset(se->discard_map, 0xff,
4516 SIT_VBLOCK_MAP_SIZE);
4517 } else {
4518 memcpy(se->discard_map,
4519 se->cur_valid_map,
4520 SIT_VBLOCK_MAP_SIZE);
4521 sbi->discard_blks +=
4522 sbi->blocks_per_seg -
4523 se->valid_blocks;
4524 }
4525 }
4526
4527 if (__is_large_section(sbi))
4528 get_sec_entry(sbi, start)->valid_blocks +=
4529 se->valid_blocks;
4530 }
4531 start_blk += readed;
4532 } while (start_blk < sit_blk_cnt);
4533
4534 down_read(&curseg->journal_rwsem);
4535 for (i = 0; i < sits_in_cursum(journal); i++) {
4536 unsigned int old_valid_blocks;
4537
4538 start = le32_to_cpu(segno_in_journal(journal, i));
4539 if (start >= MAIN_SEGS(sbi)) {
4540 f2fs_err(sbi, "Wrong journal entry on segno %u",
4541 start);
4542 err = -EFSCORRUPTED;
4543 break;
4544 }
4545
4546 se = &sit_i->sentries[start];
4547 sit = sit_in_journal(journal, i);
4548
4549 old_valid_blocks = se->valid_blocks;
4550 if (IS_NODESEG(se->type))
4551 total_node_blocks -= old_valid_blocks;
4552
4553 err = check_block_count(sbi, start, &sit);
4554 if (err)
4555 break;
4556 seg_info_from_raw_sit(se, &sit);
4557 if (IS_NODESEG(se->type))
4558 total_node_blocks += se->valid_blocks;
4559
4560 if (f2fs_block_unit_discard(sbi)) {
4561 if (is_set_ckpt_flags(sbi, CP_TRIMMED_FLAG)) {
4562 memset(se->discard_map, 0xff, SIT_VBLOCK_MAP_SIZE);
4563 } else {
4564 memcpy(se->discard_map, se->cur_valid_map,
4565 SIT_VBLOCK_MAP_SIZE);
4566 sbi->discard_blks += old_valid_blocks;
4567 sbi->discard_blks -= se->valid_blocks;
4568 }
4569 }
4570
4571 if (__is_large_section(sbi)) {
4572 get_sec_entry(sbi, start)->valid_blocks +=
4573 se->valid_blocks;
4574 get_sec_entry(sbi, start)->valid_blocks -=
4575 old_valid_blocks;
4576 }
4577 }
4578 up_read(&curseg->journal_rwsem);
4579
4580 if (!err && total_node_blocks != valid_node_count(sbi)) {
4581 f2fs_err(sbi, "SIT is corrupted node# %u vs %u",
4582 total_node_blocks, valid_node_count(sbi));
4583 err = -EFSCORRUPTED;
4584 }
4585
4586 return err;
4587 }
4588
4589 static void init_free_segmap(struct f2fs_sb_info *sbi)
4590 {
4591 unsigned int start;
4592 int type;
4593 struct seg_entry *sentry;
4594
4595 for (start = 0; start < MAIN_SEGS(sbi); start++) {
4596 if (f2fs_usable_blks_in_seg(sbi, start) == 0)
4597 continue;
4598 sentry = get_seg_entry(sbi, start);
4599 if (!sentry->valid_blocks)
4600 __set_free(sbi, start);
4601 else
4602 SIT_I(sbi)->written_valid_blocks +=
4603 sentry->valid_blocks;
4604 }
4605
4606 /* set use the current segments */
4607 for (type = CURSEG_HOT_DATA; type <= CURSEG_COLD_NODE; type++) {
4608 struct curseg_info *curseg_t = CURSEG_I(sbi, type);
4609
4610 __set_test_and_inuse(sbi, curseg_t->segno);
4611 }
4612 }
4613
4614 static void init_dirty_segmap(struct f2fs_sb_info *sbi)
4615 {
4616 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
4617 struct free_segmap_info *free_i = FREE_I(sbi);
4618 unsigned int segno = 0, offset = 0, secno;
4619 block_t valid_blocks, usable_blks_in_seg;
4620 block_t blks_per_sec = BLKS_PER_SEC(sbi);
4621
4622 while (1) {
4623 /* find dirty segment based on free segmap */
4624 segno = find_next_inuse(free_i, MAIN_SEGS(sbi), offset);
4625 if (segno >= MAIN_SEGS(sbi))
4626 break;
4627 offset = segno + 1;
4628 valid_blocks = get_valid_blocks(sbi, segno, false);
4629 usable_blks_in_seg = f2fs_usable_blks_in_seg(sbi, segno);
4630 if (valid_blocks == usable_blks_in_seg || !valid_blocks)
4631 continue;
4632 if (valid_blocks > usable_blks_in_seg) {
4633 f2fs_bug_on(sbi, 1);
4634 continue;
4635 }
4636 mutex_lock(&dirty_i->seglist_lock);
4637 __locate_dirty_segment(sbi, segno, DIRTY);
4638 mutex_unlock(&dirty_i->seglist_lock);
4639 }
4640
4641 if (!__is_large_section(sbi))
4642 return;
4643
4644 mutex_lock(&dirty_i->seglist_lock);
4645 for (segno = 0; segno < MAIN_SEGS(sbi); segno += sbi->segs_per_sec) {
4646 valid_blocks = get_valid_blocks(sbi, segno, true);
4647 secno = GET_SEC_FROM_SEG(sbi, segno);
4648
4649 if (!valid_blocks || valid_blocks == blks_per_sec)
4650 continue;
4651 if (IS_CURSEC(sbi, secno))
4652 continue;
4653 set_bit(secno, dirty_i->dirty_secmap);
4654 }
4655 mutex_unlock(&dirty_i->seglist_lock);
4656 }
4657
4658 static int init_victim_secmap(struct f2fs_sb_info *sbi)
4659 {
4660 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
4661 unsigned int bitmap_size = f2fs_bitmap_size(MAIN_SECS(sbi));
4662
4663 dirty_i->victim_secmap = f2fs_kvzalloc(sbi, bitmap_size, GFP_KERNEL);
4664 if (!dirty_i->victim_secmap)
4665 return -ENOMEM;
4666 return 0;
4667 }
4668
4669 static int build_dirty_segmap(struct f2fs_sb_info *sbi)
4670 {
4671 struct dirty_seglist_info *dirty_i;
4672 unsigned int bitmap_size, i;
4673
4674 /* allocate memory for dirty segments list information */
4675 dirty_i = f2fs_kzalloc(sbi, sizeof(struct dirty_seglist_info),
4676 GFP_KERNEL);
4677 if (!dirty_i)
4678 return -ENOMEM;
4679
4680 SM_I(sbi)->dirty_info = dirty_i;
4681 mutex_init(&dirty_i->seglist_lock);
4682
4683 bitmap_size = f2fs_bitmap_size(MAIN_SEGS(sbi));
4684
4685 for (i = 0; i < NR_DIRTY_TYPE; i++) {
4686 dirty_i->dirty_segmap[i] = f2fs_kvzalloc(sbi, bitmap_size,
4687 GFP_KERNEL);
4688 if (!dirty_i->dirty_segmap[i])
4689 return -ENOMEM;
4690 }
4691
4692 if (__is_large_section(sbi)) {
4693 bitmap_size = f2fs_bitmap_size(MAIN_SECS(sbi));
4694 dirty_i->dirty_secmap = f2fs_kvzalloc(sbi,
4695 bitmap_size, GFP_KERNEL);
4696 if (!dirty_i->dirty_secmap)
4697 return -ENOMEM;
4698 }
4699
4700 init_dirty_segmap(sbi);
4701 return init_victim_secmap(sbi);
4702 }
4703
4704 static int sanity_check_curseg(struct f2fs_sb_info *sbi)
4705 {
4706 int i;
4707
4708 /*
4709 * In LFS/SSR curseg, .next_blkoff should point to an unused blkaddr;
4710 * In LFS curseg, all blkaddr after .next_blkoff should be unused.
4711 */
4712 for (i = 0; i < NR_PERSISTENT_LOG; i++) {
4713 struct curseg_info *curseg = CURSEG_I(sbi, i);
4714 struct seg_entry *se = get_seg_entry(sbi, curseg->segno);
4715 unsigned int blkofs = curseg->next_blkoff;
4716
4717 if (f2fs_sb_has_readonly(sbi) &&
4718 i != CURSEG_HOT_DATA && i != CURSEG_HOT_NODE)
4719 continue;
4720
4721 sanity_check_seg_type(sbi, curseg->seg_type);
4722
4723 if (f2fs_test_bit(blkofs, se->cur_valid_map))
4724 goto out;
4725
4726 if (curseg->alloc_type == SSR)
4727 continue;
4728
4729 for (blkofs += 1; blkofs < sbi->blocks_per_seg; blkofs++) {
4730 if (!f2fs_test_bit(blkofs, se->cur_valid_map))
4731 continue;
4732 out:
4733 f2fs_err(sbi,
4734 "Current segment's next free block offset is inconsistent with bitmap, logtype:%u, segno:%u, type:%u, next_blkoff:%u, blkofs:%u",
4735 i, curseg->segno, curseg->alloc_type,
4736 curseg->next_blkoff, blkofs);
4737 return -EFSCORRUPTED;
4738 }
4739 }
4740 return 0;
4741 }
4742
4743 #ifdef CONFIG_BLK_DEV_ZONED
4744
4745 static int check_zone_write_pointer(struct f2fs_sb_info *sbi,
4746 struct f2fs_dev_info *fdev,
4747 struct blk_zone *zone)
4748 {
4749 unsigned int wp_segno, wp_blkoff, zone_secno, zone_segno, segno;
4750 block_t zone_block, wp_block, last_valid_block;
4751 unsigned int log_sectors_per_block = sbi->log_blocksize - SECTOR_SHIFT;
4752 int i, s, b, ret;
4753 struct seg_entry *se;
4754
4755 if (zone->type != BLK_ZONE_TYPE_SEQWRITE_REQ)
4756 return 0;
4757
4758 wp_block = fdev->start_blk + (zone->wp >> log_sectors_per_block);
4759 wp_segno = GET_SEGNO(sbi, wp_block);
4760 wp_blkoff = wp_block - START_BLOCK(sbi, wp_segno);
4761 zone_block = fdev->start_blk + (zone->start >> log_sectors_per_block);
4762 zone_segno = GET_SEGNO(sbi, zone_block);
4763 zone_secno = GET_SEC_FROM_SEG(sbi, zone_segno);
4764
4765 if (zone_segno >= MAIN_SEGS(sbi))
4766 return 0;
4767
4768 /*
4769 * Skip check of zones cursegs point to, since
4770 * fix_curseg_write_pointer() checks them.
4771 */
4772 for (i = 0; i < NO_CHECK_TYPE; i++)
4773 if (zone_secno == GET_SEC_FROM_SEG(sbi,
4774 CURSEG_I(sbi, i)->segno))
4775 return 0;
4776
4777 /*
4778 * Get last valid block of the zone.
4779 */
4780 last_valid_block = zone_block - 1;
4781 for (s = sbi->segs_per_sec - 1; s >= 0; s--) {
4782 segno = zone_segno + s;
4783 se = get_seg_entry(sbi, segno);
4784 for (b = sbi->blocks_per_seg - 1; b >= 0; b--)
4785 if (f2fs_test_bit(b, se->cur_valid_map)) {
4786 last_valid_block = START_BLOCK(sbi, segno) + b;
4787 break;
4788 }
4789 if (last_valid_block >= zone_block)
4790 break;
4791 }
4792
4793 /*
4794 * If last valid block is beyond the write pointer, report the
4795 * inconsistency. This inconsistency does not cause write error
4796 * because the zone will not be selected for write operation until
4797 * it get discarded. Just report it.
4798 */
4799 if (last_valid_block >= wp_block) {
4800 f2fs_notice(sbi, "Valid block beyond write pointer: "
4801 "valid block[0x%x,0x%x] wp[0x%x,0x%x]",
4802 GET_SEGNO(sbi, last_valid_block),
4803 GET_BLKOFF_FROM_SEG0(sbi, last_valid_block),
4804 wp_segno, wp_blkoff);
4805 return 0;
4806 }
4807
4808 /*
4809 * If there is no valid block in the zone and if write pointer is
4810 * not at zone start, reset the write pointer.
4811 */
4812 if (last_valid_block + 1 == zone_block && zone->wp != zone->start) {
4813 f2fs_notice(sbi,
4814 "Zone without valid block has non-zero write "
4815 "pointer. Reset the write pointer: wp[0x%x,0x%x]",
4816 wp_segno, wp_blkoff);
4817 ret = __f2fs_issue_discard_zone(sbi, fdev->bdev, zone_block,
4818 zone->len >> log_sectors_per_block);
4819 if (ret) {
4820 f2fs_err(sbi, "Discard zone failed: %s (errno=%d)",
4821 fdev->path, ret);
4822 return ret;
4823 }
4824 }
4825
4826 return 0;
4827 }
4828
4829 static struct f2fs_dev_info *get_target_zoned_dev(struct f2fs_sb_info *sbi,
4830 block_t zone_blkaddr)
4831 {
4832 int i;
4833
4834 for (i = 0; i < sbi->s_ndevs; i++) {
4835 if (!bdev_is_zoned(FDEV(i).bdev))
4836 continue;
4837 if (sbi->s_ndevs == 1 || (FDEV(i).start_blk <= zone_blkaddr &&
4838 zone_blkaddr <= FDEV(i).end_blk))
4839 return &FDEV(i);
4840 }
4841
4842 return NULL;
4843 }
4844
4845 static int report_one_zone_cb(struct blk_zone *zone, unsigned int idx,
4846 void *data)
4847 {
4848 memcpy(data, zone, sizeof(struct blk_zone));
4849 return 0;
4850 }
4851
4852 static int fix_curseg_write_pointer(struct f2fs_sb_info *sbi, int type)
4853 {
4854 struct curseg_info *cs = CURSEG_I(sbi, type);
4855 struct f2fs_dev_info *zbd;
4856 struct blk_zone zone;
4857 unsigned int cs_section, wp_segno, wp_blkoff, wp_sector_off;
4858 block_t cs_zone_block, wp_block;
4859 unsigned int log_sectors_per_block = sbi->log_blocksize - SECTOR_SHIFT;
4860 sector_t zone_sector;
4861 int err;
4862
4863 cs_section = GET_SEC_FROM_SEG(sbi, cs->segno);
4864 cs_zone_block = START_BLOCK(sbi, GET_SEG_FROM_SEC(sbi, cs_section));
4865
4866 zbd = get_target_zoned_dev(sbi, cs_zone_block);
4867 if (!zbd)
4868 return 0;
4869
4870 /* report zone for the sector the curseg points to */
4871 zone_sector = (sector_t)(cs_zone_block - zbd->start_blk)
4872 << log_sectors_per_block;
4873 err = blkdev_report_zones(zbd->bdev, zone_sector, 1,
4874 report_one_zone_cb, &zone);
4875 if (err != 1) {
4876 f2fs_err(sbi, "Report zone failed: %s errno=(%d)",
4877 zbd->path, err);
4878 return err;
4879 }
4880
4881 if (zone.type != BLK_ZONE_TYPE_SEQWRITE_REQ)
4882 return 0;
4883
4884 wp_block = zbd->start_blk + (zone.wp >> log_sectors_per_block);
4885 wp_segno = GET_SEGNO(sbi, wp_block);
4886 wp_blkoff = wp_block - START_BLOCK(sbi, wp_segno);
4887 wp_sector_off = zone.wp & GENMASK(log_sectors_per_block - 1, 0);
4888
4889 if (cs->segno == wp_segno && cs->next_blkoff == wp_blkoff &&
4890 wp_sector_off == 0)
4891 return 0;
4892
4893 f2fs_notice(sbi, "Unaligned curseg[%d] with write pointer: "
4894 "curseg[0x%x,0x%x] wp[0x%x,0x%x]",
4895 type, cs->segno, cs->next_blkoff, wp_segno, wp_blkoff);
4896
4897 f2fs_notice(sbi, "Assign new section to curseg[%d]: "
4898 "curseg[0x%x,0x%x]", type, cs->segno, cs->next_blkoff);
4899
4900 f2fs_allocate_new_section(sbi, type, true);
4901
4902 /* check consistency of the zone curseg pointed to */
4903 if (check_zone_write_pointer(sbi, zbd, &zone))
4904 return -EIO;
4905
4906 /* check newly assigned zone */
4907 cs_section = GET_SEC_FROM_SEG(sbi, cs->segno);
4908 cs_zone_block = START_BLOCK(sbi, GET_SEG_FROM_SEC(sbi, cs_section));
4909
4910 zbd = get_target_zoned_dev(sbi, cs_zone_block);
4911 if (!zbd)
4912 return 0;
4913
4914 zone_sector = (sector_t)(cs_zone_block - zbd->start_blk)
4915 << log_sectors_per_block;
4916 err = blkdev_report_zones(zbd->bdev, zone_sector, 1,
4917 report_one_zone_cb, &zone);
4918 if (err != 1) {
4919 f2fs_err(sbi, "Report zone failed: %s errno=(%d)",
4920 zbd->path, err);
4921 return err;
4922 }
4923
4924 if (zone.type != BLK_ZONE_TYPE_SEQWRITE_REQ)
4925 return 0;
4926
4927 if (zone.wp != zone.start) {
4928 f2fs_notice(sbi,
4929 "New zone for curseg[%d] is not yet discarded. "
4930 "Reset the zone: curseg[0x%x,0x%x]",
4931 type, cs->segno, cs->next_blkoff);
4932 err = __f2fs_issue_discard_zone(sbi, zbd->bdev,
4933 zone_sector >> log_sectors_per_block,
4934 zone.len >> log_sectors_per_block);
4935 if (err) {
4936 f2fs_err(sbi, "Discard zone failed: %s (errno=%d)",
4937 zbd->path, err);
4938 return err;
4939 }
4940 }
4941
4942 return 0;
4943 }
4944
4945 int f2fs_fix_curseg_write_pointer(struct f2fs_sb_info *sbi)
4946 {
4947 int i, ret;
4948
4949 for (i = 0; i < NR_PERSISTENT_LOG; i++) {
4950 ret = fix_curseg_write_pointer(sbi, i);
4951 if (ret)
4952 return ret;
4953 }
4954
4955 return 0;
4956 }
4957
4958 struct check_zone_write_pointer_args {
4959 struct f2fs_sb_info *sbi;
4960 struct f2fs_dev_info *fdev;
4961 };
4962
4963 static int check_zone_write_pointer_cb(struct blk_zone *zone, unsigned int idx,
4964 void *data)
4965 {
4966 struct check_zone_write_pointer_args *args;
4967
4968 args = (struct check_zone_write_pointer_args *)data;
4969
4970 return check_zone_write_pointer(args->sbi, args->fdev, zone);
4971 }
4972
4973 int f2fs_check_write_pointer(struct f2fs_sb_info *sbi)
4974 {
4975 int i, ret;
4976 struct check_zone_write_pointer_args args;
4977
4978 for (i = 0; i < sbi->s_ndevs; i++) {
4979 if (!bdev_is_zoned(FDEV(i).bdev))
4980 continue;
4981
4982 args.sbi = sbi;
4983 args.fdev = &FDEV(i);
4984 ret = blkdev_report_zones(FDEV(i).bdev, 0, BLK_ALL_ZONES,
4985 check_zone_write_pointer_cb, &args);
4986 if (ret < 0)
4987 return ret;
4988 }
4989
4990 return 0;
4991 }
4992
4993 static bool is_conv_zone(struct f2fs_sb_info *sbi, unsigned int zone_idx,
4994 unsigned int dev_idx)
4995 {
4996 if (!bdev_is_zoned(FDEV(dev_idx).bdev))
4997 return true;
4998 return !test_bit(zone_idx, FDEV(dev_idx).blkz_seq);
4999 }
5000
5001 /* Return the zone index in the given device */
5002 static unsigned int get_zone_idx(struct f2fs_sb_info *sbi, unsigned int secno,
5003 int dev_idx)
5004 {
5005 block_t sec_start_blkaddr = START_BLOCK(sbi, GET_SEG_FROM_SEC(sbi, secno));
5006
5007 return (sec_start_blkaddr - FDEV(dev_idx).start_blk) >>
5008 sbi->log_blocks_per_blkz;
5009 }
5010
5011 /*
5012 * Return the usable segments in a section based on the zone's
5013 * corresponding zone capacity. Zone is equal to a section.
5014 */
5015 static inline unsigned int f2fs_usable_zone_segs_in_sec(
5016 struct f2fs_sb_info *sbi, unsigned int segno)
5017 {
5018 unsigned int dev_idx, zone_idx, unusable_segs_in_sec;
5019
5020 dev_idx = f2fs_target_device_index(sbi, START_BLOCK(sbi, segno));
5021 zone_idx = get_zone_idx(sbi, GET_SEC_FROM_SEG(sbi, segno), dev_idx);
5022
5023 /* Conventional zone's capacity is always equal to zone size */
5024 if (is_conv_zone(sbi, zone_idx, dev_idx))
5025 return sbi->segs_per_sec;
5026
5027 /*
5028 * If the zone_capacity_blocks array is NULL, then zone capacity
5029 * is equal to the zone size for all zones
5030 */
5031 if (!FDEV(dev_idx).zone_capacity_blocks)
5032 return sbi->segs_per_sec;
5033
5034 /* Get the segment count beyond zone capacity block */
5035 unusable_segs_in_sec = (sbi->blocks_per_blkz -
5036 FDEV(dev_idx).zone_capacity_blocks[zone_idx]) >>
5037 sbi->log_blocks_per_seg;
5038 return sbi->segs_per_sec - unusable_segs_in_sec;
5039 }
5040
5041 /*
5042 * Return the number of usable blocks in a segment. The number of blocks
5043 * returned is always equal to the number of blocks in a segment for
5044 * segments fully contained within a sequential zone capacity or a
5045 * conventional zone. For segments partially contained in a sequential
5046 * zone capacity, the number of usable blocks up to the zone capacity
5047 * is returned. 0 is returned in all other cases.
5048 */
5049 static inline unsigned int f2fs_usable_zone_blks_in_seg(
5050 struct f2fs_sb_info *sbi, unsigned int segno)
5051 {
5052 block_t seg_start, sec_start_blkaddr, sec_cap_blkaddr;
5053 unsigned int zone_idx, dev_idx, secno;
5054
5055 secno = GET_SEC_FROM_SEG(sbi, segno);
5056 seg_start = START_BLOCK(sbi, segno);
5057 dev_idx = f2fs_target_device_index(sbi, seg_start);
5058 zone_idx = get_zone_idx(sbi, secno, dev_idx);
5059
5060 /*
5061 * Conventional zone's capacity is always equal to zone size,
5062 * so, blocks per segment is unchanged.
5063 */
5064 if (is_conv_zone(sbi, zone_idx, dev_idx))
5065 return sbi->blocks_per_seg;
5066
5067 if (!FDEV(dev_idx).zone_capacity_blocks)
5068 return sbi->blocks_per_seg;
5069
5070 sec_start_blkaddr = START_BLOCK(sbi, GET_SEG_FROM_SEC(sbi, secno));
5071 sec_cap_blkaddr = sec_start_blkaddr +
5072 FDEV(dev_idx).zone_capacity_blocks[zone_idx];
5073
5074 /*
5075 * If segment starts before zone capacity and spans beyond
5076 * zone capacity, then usable blocks are from seg start to
5077 * zone capacity. If the segment starts after the zone capacity,
5078 * then there are no usable blocks.
5079 */
5080 if (seg_start >= sec_cap_blkaddr)
5081 return 0;
5082 if (seg_start + sbi->blocks_per_seg > sec_cap_blkaddr)
5083 return sec_cap_blkaddr - seg_start;
5084
5085 return sbi->blocks_per_seg;
5086 }
5087 #else
5088 int f2fs_fix_curseg_write_pointer(struct f2fs_sb_info *sbi)
5089 {
5090 return 0;
5091 }
5092
5093 int f2fs_check_write_pointer(struct f2fs_sb_info *sbi)
5094 {
5095 return 0;
5096 }
5097
5098 static inline unsigned int f2fs_usable_zone_blks_in_seg(struct f2fs_sb_info *sbi,
5099 unsigned int segno)
5100 {
5101 return 0;
5102 }
5103
5104 static inline unsigned int f2fs_usable_zone_segs_in_sec(struct f2fs_sb_info *sbi,
5105 unsigned int segno)
5106 {
5107 return 0;
5108 }
5109 #endif
5110 unsigned int f2fs_usable_blks_in_seg(struct f2fs_sb_info *sbi,
5111 unsigned int segno)
5112 {
5113 if (f2fs_sb_has_blkzoned(sbi))
5114 return f2fs_usable_zone_blks_in_seg(sbi, segno);
5115
5116 return sbi->blocks_per_seg;
5117 }
5118
5119 unsigned int f2fs_usable_segs_in_sec(struct f2fs_sb_info *sbi,
5120 unsigned int segno)
5121 {
5122 if (f2fs_sb_has_blkzoned(sbi))
5123 return f2fs_usable_zone_segs_in_sec(sbi, segno);
5124
5125 return sbi->segs_per_sec;
5126 }
5127
5128 /*
5129 * Update min, max modified time for cost-benefit GC algorithm
5130 */
5131 static void init_min_max_mtime(struct f2fs_sb_info *sbi)
5132 {
5133 struct sit_info *sit_i = SIT_I(sbi);
5134 unsigned int segno;
5135
5136 down_write(&sit_i->sentry_lock);
5137
5138 sit_i->min_mtime = ULLONG_MAX;
5139
5140 for (segno = 0; segno < MAIN_SEGS(sbi); segno += sbi->segs_per_sec) {
5141 unsigned int i;
5142 unsigned long long mtime = 0;
5143
5144 for (i = 0; i < sbi->segs_per_sec; i++)
5145 mtime += get_seg_entry(sbi, segno + i)->mtime;
5146
5147 mtime = div_u64(mtime, sbi->segs_per_sec);
5148
5149 if (sit_i->min_mtime > mtime)
5150 sit_i->min_mtime = mtime;
5151 }
5152 sit_i->max_mtime = get_mtime(sbi, false);
5153 sit_i->dirty_max_mtime = 0;
5154 up_write(&sit_i->sentry_lock);
5155 }
5156
5157 int f2fs_build_segment_manager(struct f2fs_sb_info *sbi)
5158 {
5159 struct f2fs_super_block *raw_super = F2FS_RAW_SUPER(sbi);
5160 struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi);
5161 struct f2fs_sm_info *sm_info;
5162 int err;
5163
5164 sm_info = f2fs_kzalloc(sbi, sizeof(struct f2fs_sm_info), GFP_KERNEL);
5165 if (!sm_info)
5166 return -ENOMEM;
5167
5168 /* init sm info */
5169 sbi->sm_info = sm_info;
5170 sm_info->seg0_blkaddr = le32_to_cpu(raw_super->segment0_blkaddr);
5171 sm_info->main_blkaddr = le32_to_cpu(raw_super->main_blkaddr);
5172 sm_info->segment_count = le32_to_cpu(raw_super->segment_count);
5173 sm_info->reserved_segments = le32_to_cpu(ckpt->rsvd_segment_count);
5174 sm_info->ovp_segments = le32_to_cpu(ckpt->overprov_segment_count);
5175 sm_info->main_segments = le32_to_cpu(raw_super->segment_count_main);
5176 sm_info->ssa_blkaddr = le32_to_cpu(raw_super->ssa_blkaddr);
5177 sm_info->rec_prefree_segments = sm_info->main_segments *
5178 DEF_RECLAIM_PREFREE_SEGMENTS / 100;
5179 if (sm_info->rec_prefree_segments > DEF_MAX_RECLAIM_PREFREE_SEGMENTS)
5180 sm_info->rec_prefree_segments = DEF_MAX_RECLAIM_PREFREE_SEGMENTS;
5181
5182 if (!f2fs_lfs_mode(sbi))
5183 sm_info->ipu_policy = 1 << F2FS_IPU_FSYNC;
5184 sm_info->min_ipu_util = DEF_MIN_IPU_UTIL;
5185 sm_info->min_fsync_blocks = DEF_MIN_FSYNC_BLOCKS;
5186 sm_info->min_seq_blocks = sbi->blocks_per_seg;
5187 sm_info->min_hot_blocks = DEF_MIN_HOT_BLOCKS;
5188 sm_info->min_ssr_sections = reserved_sections(sbi);
5189
5190 INIT_LIST_HEAD(&sm_info->sit_entry_set);
5191
5192 init_rwsem(&sm_info->curseg_lock);
5193
5194 if (!f2fs_readonly(sbi->sb)) {
5195 err = f2fs_create_flush_cmd_control(sbi);
5196 if (err)
5197 return err;
5198 }
5199
5200 err = create_discard_cmd_control(sbi);
5201 if (err)
5202 return err;
5203
5204 err = build_sit_info(sbi);
5205 if (err)
5206 return err;
5207 err = build_free_segmap(sbi);
5208 if (err)
5209 return err;
5210 err = build_curseg(sbi);
5211 if (err)
5212 return err;
5213
5214 /* reinit free segmap based on SIT */
5215 err = build_sit_entries(sbi);
5216 if (err)
5217 return err;
5218
5219 init_free_segmap(sbi);
5220 err = build_dirty_segmap(sbi);
5221 if (err)
5222 return err;
5223
5224 err = sanity_check_curseg(sbi);
5225 if (err)
5226 return err;
5227
5228 init_min_max_mtime(sbi);
5229 return 0;
5230 }
5231
5232 static void discard_dirty_segmap(struct f2fs_sb_info *sbi,
5233 enum dirty_type dirty_type)
5234 {
5235 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
5236
5237 mutex_lock(&dirty_i->seglist_lock);
5238 kvfree(dirty_i->dirty_segmap[dirty_type]);
5239 dirty_i->nr_dirty[dirty_type] = 0;
5240 mutex_unlock(&dirty_i->seglist_lock);
5241 }
5242
5243 static void destroy_victim_secmap(struct f2fs_sb_info *sbi)
5244 {
5245 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
5246
5247 kvfree(dirty_i->victim_secmap);
5248 }
5249
5250 static void destroy_dirty_segmap(struct f2fs_sb_info *sbi)
5251 {
5252 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
5253 int i;
5254
5255 if (!dirty_i)
5256 return;
5257
5258 /* discard pre-free/dirty segments list */
5259 for (i = 0; i < NR_DIRTY_TYPE; i++)
5260 discard_dirty_segmap(sbi, i);
5261
5262 if (__is_large_section(sbi)) {
5263 mutex_lock(&dirty_i->seglist_lock);
5264 kvfree(dirty_i->dirty_secmap);
5265 mutex_unlock(&dirty_i->seglist_lock);
5266 }
5267
5268 destroy_victim_secmap(sbi);
5269 SM_I(sbi)->dirty_info = NULL;
5270 kfree(dirty_i);
5271 }
5272
5273 static void destroy_curseg(struct f2fs_sb_info *sbi)
5274 {
5275 struct curseg_info *array = SM_I(sbi)->curseg_array;
5276 int i;
5277
5278 if (!array)
5279 return;
5280 SM_I(sbi)->curseg_array = NULL;
5281 for (i = 0; i < NR_CURSEG_TYPE; i++) {
5282 kfree(array[i].sum_blk);
5283 kfree(array[i].journal);
5284 }
5285 kfree(array);
5286 }
5287
5288 static void destroy_free_segmap(struct f2fs_sb_info *sbi)
5289 {
5290 struct free_segmap_info *free_i = SM_I(sbi)->free_info;
5291
5292 if (!free_i)
5293 return;
5294 SM_I(sbi)->free_info = NULL;
5295 kvfree(free_i->free_segmap);
5296 kvfree(free_i->free_secmap);
5297 kfree(free_i);
5298 }
5299
5300 static void destroy_sit_info(struct f2fs_sb_info *sbi)
5301 {
5302 struct sit_info *sit_i = SIT_I(sbi);
5303
5304 if (!sit_i)
5305 return;
5306
5307 if (sit_i->sentries)
5308 kvfree(sit_i->bitmap);
5309 kfree(sit_i->tmp_map);
5310
5311 kvfree(sit_i->sentries);
5312 kvfree(sit_i->sec_entries);
5313 kvfree(sit_i->dirty_sentries_bitmap);
5314
5315 SM_I(sbi)->sit_info = NULL;
5316 kvfree(sit_i->sit_bitmap);
5317 #ifdef CONFIG_F2FS_CHECK_FS
5318 kvfree(sit_i->sit_bitmap_mir);
5319 kvfree(sit_i->invalid_segmap);
5320 #endif
5321 kfree(sit_i);
5322 }
5323
5324 void f2fs_destroy_segment_manager(struct f2fs_sb_info *sbi)
5325 {
5326 struct f2fs_sm_info *sm_info = SM_I(sbi);
5327
5328 if (!sm_info)
5329 return;
5330 f2fs_destroy_flush_cmd_control(sbi, true);
5331 destroy_discard_cmd_control(sbi);
5332 destroy_dirty_segmap(sbi);
5333 destroy_curseg(sbi);
5334 destroy_free_segmap(sbi);
5335 destroy_sit_info(sbi);
5336 sbi->sm_info = NULL;
5337 kfree(sm_info);
5338 }
5339
5340 int __init f2fs_create_segment_manager_caches(void)
5341 {
5342 discard_entry_slab = f2fs_kmem_cache_create("f2fs_discard_entry",
5343 sizeof(struct discard_entry));
5344 if (!discard_entry_slab)
5345 goto fail;
5346
5347 discard_cmd_slab = f2fs_kmem_cache_create("f2fs_discard_cmd",
5348 sizeof(struct discard_cmd));
5349 if (!discard_cmd_slab)
5350 goto destroy_discard_entry;
5351
5352 sit_entry_set_slab = f2fs_kmem_cache_create("f2fs_sit_entry_set",
5353 sizeof(struct sit_entry_set));
5354 if (!sit_entry_set_slab)
5355 goto destroy_discard_cmd;
5356
5357 inmem_entry_slab = f2fs_kmem_cache_create("f2fs_inmem_page_entry",
5358 sizeof(struct inmem_pages));
5359 if (!inmem_entry_slab)
5360 goto destroy_sit_entry_set;
5361 return 0;
5362
5363 destroy_sit_entry_set:
5364 kmem_cache_destroy(sit_entry_set_slab);
5365 destroy_discard_cmd:
5366 kmem_cache_destroy(discard_cmd_slab);
5367 destroy_discard_entry:
5368 kmem_cache_destroy(discard_entry_slab);
5369 fail:
5370 return -ENOMEM;
5371 }
5372
5373 void f2fs_destroy_segment_manager_caches(void)
5374 {
5375 kmem_cache_destroy(sit_entry_set_slab);
5376 kmem_cache_destroy(discard_cmd_slab);
5377 kmem_cache_destroy(discard_entry_slab);
5378 kmem_cache_destroy(inmem_entry_slab);
5379 }
Ubuntu Jammy Linux kernel mirror