1 // SPDX-License-Identifier: GPL-2.0+
3 * page.c - buffer/page management specific to NILFS
5 * Copyright (C) 2005-2008 Nippon Telegraph and Telephone Corporation.
7 * Written by Ryusuke Konishi and Seiji Kihara.
10 #include <linux/pagemap.h>
11 #include <linux/writeback.h>
12 #include <linux/swap.h>
13 #include <linux/bitops.h>
14 #include <linux/page-flags.h>
15 #include <linux/list.h>
16 #include <linux/highmem.h>
17 #include <linux/pagevec.h>
18 #include <linux/gfp.h>
24 #define NILFS_BUFFER_INHERENT_BITS \
25 (BIT(BH_Uptodate) | BIT(BH_Mapped) | BIT(BH_NILFS_Node) | \
26 BIT(BH_NILFS_Volatile) | BIT(BH_NILFS_Checked))
28 static struct buffer_head
*
29 __nilfs_get_page_block(struct page
*page
, unsigned long block
, pgoff_t index
,
30 int blkbits
, unsigned long b_state
)
33 unsigned long first_block
;
34 struct buffer_head
*bh
;
36 if (!page_has_buffers(page
))
37 create_empty_buffers(page
, 1 << blkbits
, b_state
);
39 first_block
= (unsigned long)index
<< (PAGE_SHIFT
- blkbits
);
40 bh
= nilfs_page_get_nth_block(page
, block
- first_block
);
47 struct buffer_head
*nilfs_grab_buffer(struct inode
*inode
,
48 struct address_space
*mapping
,
50 unsigned long b_state
)
52 int blkbits
= inode
->i_blkbits
;
53 pgoff_t index
= blkoff
>> (PAGE_SHIFT
- blkbits
);
55 struct buffer_head
*bh
;
57 page
= grab_cache_page(mapping
, index
);
61 bh
= __nilfs_get_page_block(page
, blkoff
, index
, blkbits
, b_state
);
71 * nilfs_forget_buffer - discard dirty state
72 * @inode: owner inode of the buffer
73 * @bh: buffer head of the buffer to be discarded
75 void nilfs_forget_buffer(struct buffer_head
*bh
)
77 struct page
*page
= bh
->b_page
;
78 const unsigned long clear_bits
=
79 (BIT(BH_Uptodate
) | BIT(BH_Dirty
) | BIT(BH_Mapped
) |
80 BIT(BH_Async_Write
) | BIT(BH_NILFS_Volatile
) |
81 BIT(BH_NILFS_Checked
) | BIT(BH_NILFS_Redirected
));
84 set_mask_bits(&bh
->b_state
, clear_bits
, 0);
85 if (nilfs_page_buffers_clean(page
))
86 __nilfs_clear_page_dirty(page
);
89 ClearPageUptodate(page
);
90 ClearPageMappedToDisk(page
);
96 * nilfs_copy_buffer -- copy buffer data and flags
97 * @dbh: destination buffer
100 void nilfs_copy_buffer(struct buffer_head
*dbh
, struct buffer_head
*sbh
)
102 void *kaddr0
, *kaddr1
;
104 struct page
*spage
= sbh
->b_page
, *dpage
= dbh
->b_page
;
105 struct buffer_head
*bh
;
107 kaddr0
= kmap_atomic(spage
);
108 kaddr1
= kmap_atomic(dpage
);
109 memcpy(kaddr1
+ bh_offset(dbh
), kaddr0
+ bh_offset(sbh
), sbh
->b_size
);
110 kunmap_atomic(kaddr1
);
111 kunmap_atomic(kaddr0
);
113 dbh
->b_state
= sbh
->b_state
& NILFS_BUFFER_INHERENT_BITS
;
114 dbh
->b_blocknr
= sbh
->b_blocknr
;
115 dbh
->b_bdev
= sbh
->b_bdev
;
118 bits
= sbh
->b_state
& (BIT(BH_Uptodate
) | BIT(BH_Mapped
));
119 while ((bh
= bh
->b_this_page
) != dbh
) {
124 if (bits
& BIT(BH_Uptodate
))
125 SetPageUptodate(dpage
);
127 ClearPageUptodate(dpage
);
128 if (bits
& BIT(BH_Mapped
))
129 SetPageMappedToDisk(dpage
);
131 ClearPageMappedToDisk(dpage
);
135 * nilfs_page_buffers_clean - check if a page has dirty buffers or not.
136 * @page: page to be checked
138 * nilfs_page_buffers_clean() returns zero if the page has dirty buffers.
139 * Otherwise, it returns non-zero value.
141 int nilfs_page_buffers_clean(struct page
*page
)
143 struct buffer_head
*bh
, *head
;
145 bh
= head
= page_buffers(page
);
147 if (buffer_dirty(bh
))
149 bh
= bh
->b_this_page
;
150 } while (bh
!= head
);
154 void nilfs_page_bug(struct page
*page
)
156 struct address_space
*m
;
159 if (unlikely(!page
)) {
160 printk(KERN_CRIT
"NILFS_PAGE_BUG(NULL)\n");
165 ino
= m
? m
->host
->i_ino
: 0;
167 printk(KERN_CRIT
"NILFS_PAGE_BUG(%p): cnt=%d index#=%llu flags=0x%lx "
168 "mapping=%p ino=%lu\n",
169 page
, page_ref_count(page
),
170 (unsigned long long)page
->index
, page
->flags
, m
, ino
);
172 if (page_has_buffers(page
)) {
173 struct buffer_head
*bh
, *head
;
176 bh
= head
= page_buffers(page
);
179 " BH[%d] %p: cnt=%d block#=%llu state=0x%lx\n",
180 i
++, bh
, atomic_read(&bh
->b_count
),
181 (unsigned long long)bh
->b_blocknr
, bh
->b_state
);
182 bh
= bh
->b_this_page
;
183 } while (bh
!= head
);
188 * nilfs_copy_page -- copy the page with buffers
189 * @dst: destination page
191 * @copy_dirty: flag whether to copy dirty states on the page's buffer heads.
193 * This function is for both data pages and btnode pages. The dirty flag
194 * should be treated by caller. The page must not be under i/o.
195 * Both src and dst page must be locked
197 static void nilfs_copy_page(struct page
*dst
, struct page
*src
, int copy_dirty
)
199 struct buffer_head
*dbh
, *dbufs
, *sbh
, *sbufs
;
200 unsigned long mask
= NILFS_BUFFER_INHERENT_BITS
;
202 BUG_ON(PageWriteback(dst
));
204 sbh
= sbufs
= page_buffers(src
);
205 if (!page_has_buffers(dst
))
206 create_empty_buffers(dst
, sbh
->b_size
, 0);
209 mask
|= BIT(BH_Dirty
);
211 dbh
= dbufs
= page_buffers(dst
);
215 dbh
->b_state
= sbh
->b_state
& mask
;
216 dbh
->b_blocknr
= sbh
->b_blocknr
;
217 dbh
->b_bdev
= sbh
->b_bdev
;
218 sbh
= sbh
->b_this_page
;
219 dbh
= dbh
->b_this_page
;
220 } while (dbh
!= dbufs
);
222 copy_highpage(dst
, src
);
224 if (PageUptodate(src
) && !PageUptodate(dst
))
225 SetPageUptodate(dst
);
226 else if (!PageUptodate(src
) && PageUptodate(dst
))
227 ClearPageUptodate(dst
);
228 if (PageMappedToDisk(src
) && !PageMappedToDisk(dst
))
229 SetPageMappedToDisk(dst
);
230 else if (!PageMappedToDisk(src
) && PageMappedToDisk(dst
))
231 ClearPageMappedToDisk(dst
);
236 sbh
= sbh
->b_this_page
;
237 dbh
= dbh
->b_this_page
;
238 } while (dbh
!= dbufs
);
241 int nilfs_copy_dirty_pages(struct address_space
*dmap
,
242 struct address_space
*smap
)
251 if (!pagevec_lookup_tag(&pvec
, smap
, &index
, PAGECACHE_TAG_DIRTY
))
254 for (i
= 0; i
< pagevec_count(&pvec
); i
++) {
255 struct page
*page
= pvec
.pages
[i
], *dpage
;
258 if (unlikely(!PageDirty(page
)))
259 NILFS_PAGE_BUG(page
, "inconsistent dirty state");
261 dpage
= grab_cache_page(dmap
, page
->index
);
262 if (unlikely(!dpage
)) {
263 /* No empty page is added to the page cache */
268 if (unlikely(!page_has_buffers(page
)))
270 "found empty page in dat page cache");
272 nilfs_copy_page(dpage
, page
, 1);
273 __set_page_dirty_nobuffers(dpage
);
279 pagevec_release(&pvec
);
288 * nilfs_copy_back_pages -- copy back pages to original cache from shadow cache
289 * @dmap: destination page cache
290 * @smap: source page cache
292 * No pages must no be added to the cache during this process.
293 * This must be ensured by the caller.
295 void nilfs_copy_back_pages(struct address_space
*dmap
,
296 struct address_space
*smap
)
305 n
= pagevec_lookup(&pvec
, smap
, &index
);
309 for (i
= 0; i
< pagevec_count(&pvec
); i
++) {
310 struct page
*page
= pvec
.pages
[i
], *dpage
;
311 pgoff_t offset
= page
->index
;
314 dpage
= find_lock_page(dmap
, offset
);
316 /* override existing page on the destination cache */
317 WARN_ON(PageDirty(dpage
));
318 nilfs_copy_page(dpage
, page
, 0);
324 /* move the page to the destination cache */
325 xa_lock_irq(&smap
->i_pages
);
326 page2
= radix_tree_delete(&smap
->i_pages
, offset
);
327 WARN_ON(page2
!= page
);
330 xa_unlock_irq(&smap
->i_pages
);
332 xa_lock_irq(&dmap
->i_pages
);
333 err
= radix_tree_insert(&dmap
->i_pages
, offset
, page
);
334 if (unlikely(err
< 0)) {
335 WARN_ON(err
== -EEXIST
);
336 page
->mapping
= NULL
;
337 put_page(page
); /* for cache */
339 page
->mapping
= dmap
;
342 radix_tree_tag_set(&dmap
->i_pages
,
344 PAGECACHE_TAG_DIRTY
);
346 xa_unlock_irq(&dmap
->i_pages
);
350 pagevec_release(&pvec
);
357 * nilfs_clear_dirty_pages - discard dirty pages in address space
358 * @mapping: address space with dirty pages for discarding
359 * @silent: suppress [true] or print [false] warning messages
361 void nilfs_clear_dirty_pages(struct address_space
*mapping
, bool silent
)
369 while (pagevec_lookup_tag(&pvec
, mapping
, &index
,
370 PAGECACHE_TAG_DIRTY
)) {
371 for (i
= 0; i
< pagevec_count(&pvec
); i
++) {
372 struct page
*page
= pvec
.pages
[i
];
375 nilfs_clear_dirty_page(page
, silent
);
378 pagevec_release(&pvec
);
384 * nilfs_clear_dirty_page - discard dirty page
385 * @page: dirty page that will be discarded
386 * @silent: suppress [true] or print [false] warning messages
388 void nilfs_clear_dirty_page(struct page
*page
, bool silent
)
390 struct inode
*inode
= page
->mapping
->host
;
391 struct super_block
*sb
= inode
->i_sb
;
393 BUG_ON(!PageLocked(page
));
396 nilfs_msg(sb
, KERN_WARNING
,
397 "discard dirty page: offset=%lld, ino=%lu",
398 page_offset(page
), inode
->i_ino
);
400 ClearPageUptodate(page
);
401 ClearPageMappedToDisk(page
);
403 if (page_has_buffers(page
)) {
404 struct buffer_head
*bh
, *head
;
405 const unsigned long clear_bits
=
406 (BIT(BH_Uptodate
) | BIT(BH_Dirty
) | BIT(BH_Mapped
) |
407 BIT(BH_Async_Write
) | BIT(BH_NILFS_Volatile
) |
408 BIT(BH_NILFS_Checked
) | BIT(BH_NILFS_Redirected
));
410 bh
= head
= page_buffers(page
);
414 nilfs_msg(sb
, KERN_WARNING
,
415 "discard dirty block: blocknr=%llu, size=%zu",
416 (u64
)bh
->b_blocknr
, bh
->b_size
);
418 set_mask_bits(&bh
->b_state
, clear_bits
, 0);
420 } while (bh
= bh
->b_this_page
, bh
!= head
);
423 __nilfs_clear_page_dirty(page
);
426 unsigned int nilfs_page_count_clean_buffers(struct page
*page
,
427 unsigned int from
, unsigned int to
)
429 unsigned int block_start
, block_end
;
430 struct buffer_head
*bh
, *head
;
433 for (bh
= head
= page_buffers(page
), block_start
= 0;
434 bh
!= head
|| !block_start
;
435 block_start
= block_end
, bh
= bh
->b_this_page
) {
436 block_end
= block_start
+ bh
->b_size
;
437 if (block_end
> from
&& block_start
< to
&& !buffer_dirty(bh
))
443 void nilfs_mapping_init(struct address_space
*mapping
, struct inode
*inode
)
445 mapping
->host
= inode
;
447 mapping_set_gfp_mask(mapping
, GFP_NOFS
);
448 mapping
->private_data
= NULL
;
449 mapping
->a_ops
= &empty_aops
;
453 * NILFS2 needs clear_page_dirty() in the following two cases:
455 * 1) For B-tree node pages and data pages of the dat/gcdat, NILFS2 clears
456 * page dirty flags when it copies back pages from the shadow cache
457 * (gcdat->{i_mapping,i_btnode_cache}) to its original cache
458 * (dat->{i_mapping,i_btnode_cache}).
460 * 2) Some B-tree operations like insertion or deletion may dispose buffers
461 * in dirty state, and this needs to cancel the dirty state of their pages.
463 int __nilfs_clear_page_dirty(struct page
*page
)
465 struct address_space
*mapping
= page
->mapping
;
468 xa_lock_irq(&mapping
->i_pages
);
469 if (test_bit(PG_dirty
, &page
->flags
)) {
470 radix_tree_tag_clear(&mapping
->i_pages
,
472 PAGECACHE_TAG_DIRTY
);
473 xa_unlock_irq(&mapping
->i_pages
);
474 return clear_page_dirty_for_io(page
);
476 xa_unlock_irq(&mapping
->i_pages
);
479 return TestClearPageDirty(page
);
483 * nilfs_find_uncommitted_extent - find extent of uncommitted data
485 * @start_blk: start block offset (in)
486 * @blkoff: start offset of the found extent (out)
488 * This function searches an extent of buffers marked "delayed" which
489 * starts from a block offset equal to or larger than @start_blk. If
490 * such an extent was found, this will store the start offset in
491 * @blkoff and return its length in blocks. Otherwise, zero is
494 unsigned long nilfs_find_uncommitted_extent(struct inode
*inode
,
500 unsigned int nblocks_in_page
;
501 unsigned long length
= 0;
506 if (inode
->i_mapping
->nrpages
== 0)
509 index
= start_blk
>> (PAGE_SHIFT
- inode
->i_blkbits
);
510 nblocks_in_page
= 1U << (PAGE_SHIFT
- inode
->i_blkbits
);
515 pvec
.nr
= find_get_pages_contig(inode
->i_mapping
, index
, PAGEVEC_SIZE
,
520 if (length
> 0 && pvec
.pages
[0]->index
> index
)
523 b
= pvec
.pages
[0]->index
<< (PAGE_SHIFT
- inode
->i_blkbits
);
526 page
= pvec
.pages
[i
];
529 if (page_has_buffers(page
)) {
530 struct buffer_head
*bh
, *head
;
532 bh
= head
= page_buffers(page
);
536 if (buffer_delay(bh
)) {
540 } else if (length
> 0) {
543 } while (++b
, bh
= bh
->b_this_page
, bh
!= head
);
548 b
+= nblocks_in_page
;
552 } while (++i
< pagevec_count(&pvec
));
554 index
= page
->index
+ 1;
555 pagevec_release(&pvec
);
562 pagevec_release(&pvec
);