1 // SPDX-License-Identifier: GPL-2.0-or-later
3 * Copyright (C) International Business Machines Corp., 2000-2005
4 * Portions Copyright (C) Christoph Hellwig, 2001-2002
9 #include <linux/module.h>
10 #include <linux/bio.h>
11 #include <linux/slab.h>
12 #include <linux/init.h>
13 #include <linux/buffer_head.h>
14 #include <linux/mempool.h>
15 #include <linux/seq_file.h>
16 #include <linux/writeback.h>
17 #include "jfs_incore.h"
18 #include "jfs_superblock.h"
19 #include "jfs_filsys.h"
20 #include "jfs_metapage.h"
21 #include "jfs_txnmgr.h"
22 #include "jfs_debug.h"
24 #ifdef CONFIG_JFS_STATISTICS
26 uint pagealloc
; /* # of page allocations */
27 uint pagefree
; /* # of page frees */
28 uint lockwait
; /* # of sleeping lock_metapage() calls */
32 #define metapage_locked(mp) test_bit(META_locked, &(mp)->flag)
33 #define trylock_metapage(mp) test_and_set_bit_lock(META_locked, &(mp)->flag)
35 static inline void unlock_metapage(struct metapage
*mp
)
37 clear_bit_unlock(META_locked
, &mp
->flag
);
41 static inline void __lock_metapage(struct metapage
*mp
)
43 DECLARE_WAITQUEUE(wait
, current
);
44 INCREMENT(mpStat
.lockwait
);
45 add_wait_queue_exclusive(&mp
->wait
, &wait
);
47 set_current_state(TASK_UNINTERRUPTIBLE
);
48 if (metapage_locked(mp
)) {
49 unlock_page(mp
->page
);
53 } while (trylock_metapage(mp
));
54 __set_current_state(TASK_RUNNING
);
55 remove_wait_queue(&mp
->wait
, &wait
);
59 * Must have mp->page locked
61 static inline void lock_metapage(struct metapage
*mp
)
63 if (trylock_metapage(mp
))
67 #define METAPOOL_MIN_PAGES 32
68 static struct kmem_cache
*metapage_cache
;
69 static mempool_t
*metapage_mempool
;
71 #define MPS_PER_PAGE (PAGE_SIZE >> L2PSIZE)
78 struct metapage
*mp
[MPS_PER_PAGE
];
80 #define mp_anchor(page) ((struct meta_anchor *)page_private(page))
82 static inline struct metapage
*page_to_mp(struct page
*page
, int offset
)
84 if (!PagePrivate(page
))
86 return mp_anchor(page
)->mp
[offset
>> L2PSIZE
];
89 static inline int insert_metapage(struct page
*page
, struct metapage
*mp
)
91 struct meta_anchor
*a
;
93 int l2mp_blocks
; /* log2 blocks per metapage */
95 if (PagePrivate(page
))
98 a
= kzalloc(sizeof(struct meta_anchor
), GFP_NOFS
);
101 set_page_private(page
, (unsigned long)a
);
102 SetPagePrivate(page
);
107 l2mp_blocks
= L2PSIZE
- page
->mapping
->host
->i_blkbits
;
108 index
= (mp
->index
>> l2mp_blocks
) & (MPS_PER_PAGE
- 1);
116 static inline void remove_metapage(struct page
*page
, struct metapage
*mp
)
118 struct meta_anchor
*a
= mp_anchor(page
);
119 int l2mp_blocks
= L2PSIZE
- page
->mapping
->host
->i_blkbits
;
122 index
= (mp
->index
>> l2mp_blocks
) & (MPS_PER_PAGE
- 1);
124 BUG_ON(a
->mp
[index
] != mp
);
127 if (--a
->mp_count
== 0) {
129 set_page_private(page
, 0);
130 ClearPagePrivate(page
);
135 static inline void inc_io(struct page
*page
)
137 atomic_inc(&mp_anchor(page
)->io_count
);
140 static inline void dec_io(struct page
*page
, void (*handler
) (struct page
*))
142 if (atomic_dec_and_test(&mp_anchor(page
)->io_count
))
147 static inline struct metapage
*page_to_mp(struct page
*page
, int offset
)
149 return PagePrivate(page
) ? (struct metapage
*)page_private(page
) : NULL
;
152 static inline int insert_metapage(struct page
*page
, struct metapage
*mp
)
155 set_page_private(page
, (unsigned long)mp
);
156 SetPagePrivate(page
);
162 static inline void remove_metapage(struct page
*page
, struct metapage
*mp
)
164 set_page_private(page
, 0);
165 ClearPagePrivate(page
);
169 #define inc_io(page) do {} while(0)
170 #define dec_io(page, handler) handler(page)
174 static inline struct metapage
*alloc_metapage(gfp_t gfp_mask
)
176 struct metapage
*mp
= mempool_alloc(metapage_mempool
, gfp_mask
);
184 init_waitqueue_head(&mp
->wait
);
189 static inline void free_metapage(struct metapage
*mp
)
191 mempool_free(mp
, metapage_mempool
);
194 int __init
metapage_init(void)
197 * Allocate the metapage structures
199 metapage_cache
= kmem_cache_create("jfs_mp", sizeof(struct metapage
),
201 if (metapage_cache
== NULL
)
204 metapage_mempool
= mempool_create_slab_pool(METAPOOL_MIN_PAGES
,
207 if (metapage_mempool
== NULL
) {
208 kmem_cache_destroy(metapage_cache
);
215 void metapage_exit(void)
217 mempool_destroy(metapage_mempool
);
218 kmem_cache_destroy(metapage_cache
);
221 static inline void drop_metapage(struct page
*page
, struct metapage
*mp
)
223 if (mp
->count
|| mp
->nohomeok
|| test_bit(META_dirty
, &mp
->flag
) ||
224 test_bit(META_io
, &mp
->flag
))
226 remove_metapage(page
, mp
);
227 INCREMENT(mpStat
.pagefree
);
232 * Metapage address space operations
235 static sector_t
metapage_get_blocks(struct inode
*inode
, sector_t lblock
,
241 sector_t file_blocks
= (inode
->i_size
+ inode
->i_sb
->s_blocksize
- 1) >>
244 if (lblock
>= file_blocks
)
246 if (lblock
+ *len
> file_blocks
)
247 *len
= file_blocks
- lblock
;
250 rc
= xtLookup(inode
, (s64
)lblock
, *len
, &xflag
, &xaddr
, len
, 0);
251 if ((rc
== 0) && *len
)
252 lblock
= (sector_t
)xaddr
;
255 } /* else no mapping */
260 static void last_read_complete(struct page
*page
)
262 if (!PageError(page
))
263 SetPageUptodate(page
);
267 static void metapage_read_end_io(struct bio
*bio
)
269 struct page
*page
= bio
->bi_private
;
271 if (bio
->bi_status
) {
272 printk(KERN_ERR
"metapage_read_end_io: I/O error\n");
276 dec_io(page
, last_read_complete
);
280 static void remove_from_logsync(struct metapage
*mp
)
282 struct jfs_log
*log
= mp
->log
;
285 * This can race. Recheck that log hasn't been set to null, and after
286 * acquiring logsync lock, recheck lsn
291 LOGSYNC_LOCK(log
, flags
);
297 list_del(&mp
->synclist
);
299 LOGSYNC_UNLOCK(log
, flags
);
302 static void last_write_complete(struct page
*page
)
307 for (offset
= 0; offset
< PAGE_SIZE
; offset
+= PSIZE
) {
308 mp
= page_to_mp(page
, offset
);
309 if (mp
&& test_bit(META_io
, &mp
->flag
)) {
311 remove_from_logsync(mp
);
312 clear_bit(META_io
, &mp
->flag
);
315 * I'd like to call drop_metapage here, but I don't think it's
316 * safe unless I have the page locked
319 end_page_writeback(page
);
322 static void metapage_write_end_io(struct bio
*bio
)
324 struct page
*page
= bio
->bi_private
;
326 BUG_ON(!PagePrivate(page
));
328 if (bio
->bi_status
) {
329 printk(KERN_ERR
"metapage_write_end_io: I/O error\n");
332 dec_io(page
, last_write_complete
);
336 static int metapage_writepage(struct page
*page
, struct writeback_control
*wbc
)
338 struct bio
*bio
= NULL
;
339 int block_offset
; /* block offset of mp within page */
340 struct inode
*inode
= page
->mapping
->host
;
341 int blocks_per_mp
= JFS_SBI(inode
->i_sb
)->nbperpage
;
349 sector_t next_block
= 0;
351 unsigned long bio_bytes
= 0;
352 unsigned long bio_offset
= 0;
356 page_start
= (sector_t
)page
->index
<<
357 (PAGE_SHIFT
- inode
->i_blkbits
);
358 BUG_ON(!PageLocked(page
));
359 BUG_ON(PageWriteback(page
));
360 set_page_writeback(page
);
362 for (offset
= 0; offset
< PAGE_SIZE
; offset
+= PSIZE
) {
363 mp
= page_to_mp(page
, offset
);
365 if (!mp
|| !test_bit(META_dirty
, &mp
->flag
))
368 if (mp
->nohomeok
&& !test_bit(META_forcewrite
, &mp
->flag
)) {
371 * Make sure this page isn't blocked indefinitely.
372 * If the journal isn't undergoing I/O, push it
374 if (mp
->log
&& !(mp
->log
->cflag
& logGC_PAGEOUT
))
375 jfs_flush_journal(mp
->log
, 0);
379 clear_bit(META_dirty
, &mp
->flag
);
380 set_bit(META_io
, &mp
->flag
);
381 block_offset
= offset
>> inode
->i_blkbits
;
382 lblock
= page_start
+ block_offset
;
384 if (xlen
&& lblock
== next_block
) {
385 /* Contiguous, in memory & on disk */
386 len
= min(xlen
, blocks_per_mp
);
388 bio_bytes
+= len
<< inode
->i_blkbits
;
392 if (bio_add_page(bio
, page
, bio_bytes
, bio_offset
) <
396 * Increment counter before submitting i/o to keep
397 * count from hitting zero before we're through
400 if (!bio
->bi_iter
.bi_size
)
407 xlen
= (PAGE_SIZE
- offset
) >> inode
->i_blkbits
;
408 pblock
= metapage_get_blocks(inode
, lblock
, &xlen
);
410 printk(KERN_ERR
"JFS: metapage_get_blocks failed\n");
412 * We already called inc_io(), but can't cancel it
413 * with dec_io() until we're done with the page
418 len
= min(xlen
, (int)JFS_SBI(inode
->i_sb
)->nbperpage
);
420 bio
= bio_alloc(inode
->i_sb
->s_bdev
, 1, REQ_OP_WRITE
, GFP_NOFS
);
421 bio
->bi_iter
.bi_sector
= pblock
<< (inode
->i_blkbits
- 9);
422 bio
->bi_end_io
= metapage_write_end_io
;
423 bio
->bi_private
= page
;
425 /* Don't call bio_add_page yet, we may add to this vec */
427 bio_bytes
= len
<< inode
->i_blkbits
;
430 next_block
= lblock
+ len
;
433 if (bio_add_page(bio
, page
, bio_bytes
, bio_offset
) < bio_bytes
)
435 if (!bio
->bi_iter
.bi_size
)
442 redirty_page_for_writepage(wbc
, page
);
449 if (nr_underway
== 0)
450 end_page_writeback(page
);
454 /* We should never reach here, since we're only adding one vec */
455 printk(KERN_ERR
"JFS: bio_add_page failed unexpectedly\n");
458 print_hex_dump(KERN_ERR
, "JFS: dump of bio: ", DUMP_PREFIX_ADDRESS
, 16,
459 4, bio
, sizeof(*bio
), 0);
463 dec_io(page
, last_write_complete
);
466 dec_io(page
, last_write_complete
);
470 static int metapage_read_folio(struct file
*fp
, struct folio
*folio
)
472 struct page
*page
= &folio
->page
;
473 struct inode
*inode
= page
->mapping
->host
;
474 struct bio
*bio
= NULL
;
476 int blocks_per_page
= i_blocks_per_page(inode
, page
);
477 sector_t page_start
; /* address of page in fs blocks */
483 BUG_ON(!PageLocked(page
));
484 page_start
= (sector_t
)page
->index
<<
485 (PAGE_SHIFT
- inode
->i_blkbits
);
488 while (block_offset
< blocks_per_page
) {
489 xlen
= blocks_per_page
- block_offset
;
490 pblock
= metapage_get_blocks(inode
, page_start
+ block_offset
,
493 if (!PagePrivate(page
))
494 insert_metapage(page
, NULL
);
499 bio
= bio_alloc(inode
->i_sb
->s_bdev
, 1, REQ_OP_READ
,
501 bio
->bi_iter
.bi_sector
=
502 pblock
<< (inode
->i_blkbits
- 9);
503 bio
->bi_end_io
= metapage_read_end_io
;
504 bio
->bi_private
= page
;
505 len
= xlen
<< inode
->i_blkbits
;
506 offset
= block_offset
<< inode
->i_blkbits
;
507 if (bio_add_page(bio
, page
, len
, offset
) < len
)
509 block_offset
+= xlen
;
521 printk(KERN_ERR
"JFS: bio_add_page failed unexpectedly\n");
523 dec_io(page
, last_read_complete
);
527 static bool metapage_release_folio(struct folio
*folio
, gfp_t gfp_mask
)
533 for (offset
= 0; offset
< PAGE_SIZE
; offset
+= PSIZE
) {
534 mp
= page_to_mp(&folio
->page
, offset
);
539 jfs_info("metapage_release_folio: mp = 0x%p", mp
);
540 if (mp
->count
|| mp
->nohomeok
||
541 test_bit(META_dirty
, &mp
->flag
)) {
542 jfs_info("count = %ld, nohomeok = %d", mp
->count
,
548 remove_from_logsync(mp
);
549 remove_metapage(&folio
->page
, mp
);
550 INCREMENT(mpStat
.pagefree
);
556 static void metapage_invalidate_folio(struct folio
*folio
, size_t offset
,
559 BUG_ON(offset
|| length
< folio_size(folio
));
561 BUG_ON(folio_test_writeback(folio
));
563 metapage_release_folio(folio
, 0);
566 const struct address_space_operations jfs_metapage_aops
= {
567 .read_folio
= metapage_read_folio
,
568 .writepage
= metapage_writepage
,
569 .release_folio
= metapage_release_folio
,
570 .invalidate_folio
= metapage_invalidate_folio
,
571 .dirty_folio
= filemap_dirty_folio
,
574 struct metapage
*__get_metapage(struct inode
*inode
, unsigned long lblock
,
575 unsigned int size
, int absolute
,
580 struct address_space
*mapping
;
581 struct metapage
*mp
= NULL
;
583 unsigned long page_index
;
584 unsigned long page_offset
;
586 jfs_info("__get_metapage: ino = %ld, lblock = 0x%lx, abs=%d",
587 inode
->i_ino
, lblock
, absolute
);
589 l2bsize
= inode
->i_blkbits
;
590 l2BlocksPerPage
= PAGE_SHIFT
- l2bsize
;
591 page_index
= lblock
>> l2BlocksPerPage
;
592 page_offset
= (lblock
- (page_index
<< l2BlocksPerPage
)) << l2bsize
;
593 if ((page_offset
+ size
) > PAGE_SIZE
) {
594 jfs_err("MetaData crosses page boundary!!");
595 jfs_err("lblock = %lx, size = %d", lblock
, size
);
600 mapping
= JFS_SBI(inode
->i_sb
)->direct_inode
->i_mapping
;
603 * If an nfs client tries to read an inode that is larger
604 * than any existing inodes, we may try to read past the
605 * end of the inode map
607 if ((lblock
<< inode
->i_blkbits
) >= inode
->i_size
)
609 mapping
= inode
->i_mapping
;
612 if (new && (PSIZE
== PAGE_SIZE
)) {
613 page
= grab_cache_page(mapping
, page_index
);
615 jfs_err("grab_cache_page failed!");
618 SetPageUptodate(page
);
620 page
= read_mapping_page(mapping
, page_index
, NULL
);
622 jfs_err("read_mapping_page failed!");
628 mp
= page_to_mp(page
, page_offset
);
630 if (mp
->logical_size
!= size
) {
631 jfs_error(inode
->i_sb
,
632 "get_mp->logical_size != size\n");
633 jfs_err("logical_size = %d, size = %d",
634 mp
->logical_size
, size
);
640 if (test_bit(META_discard
, &mp
->flag
)) {
642 jfs_error(inode
->i_sb
,
643 "using a discarded metapage\n");
644 discard_metapage(mp
);
647 clear_bit(META_discard
, &mp
->flag
);
650 INCREMENT(mpStat
.pagealloc
);
651 mp
= alloc_metapage(GFP_NOFS
);
655 mp
->sb
= inode
->i_sb
;
657 mp
->xflag
= COMMIT_PAGE
;
660 mp
->logical_size
= size
;
661 mp
->data
= page_address(page
) + page_offset
;
663 if (unlikely(insert_metapage(page
, mp
))) {
671 jfs_info("zeroing mp = 0x%p", mp
);
672 memset(mp
->data
, 0, PSIZE
);
676 jfs_info("__get_metapage: returning = 0x%p data = 0x%p", mp
, mp
->data
);
684 void grab_metapage(struct metapage
* mp
)
686 jfs_info("grab_metapage: mp = 0x%p", mp
);
691 unlock_page(mp
->page
);
694 void force_metapage(struct metapage
*mp
)
696 struct page
*page
= mp
->page
;
697 jfs_info("force_metapage: mp = 0x%p", mp
);
698 set_bit(META_forcewrite
, &mp
->flag
);
699 clear_bit(META_sync
, &mp
->flag
);
702 set_page_dirty(page
);
703 if (write_one_page(page
))
704 jfs_error(mp
->sb
, "write_one_page() failed\n");
705 clear_bit(META_forcewrite
, &mp
->flag
);
709 void hold_metapage(struct metapage
*mp
)
714 void put_metapage(struct metapage
*mp
)
716 if (mp
->count
|| mp
->nohomeok
) {
717 /* Someone else will release this */
718 unlock_page(mp
->page
);
724 unlock_page(mp
->page
);
725 release_metapage(mp
);
728 void release_metapage(struct metapage
* mp
)
730 struct page
*page
= mp
->page
;
731 jfs_info("release_metapage: mp = 0x%p, flag = 0x%lx", mp
, mp
->flag
);
739 if (--mp
->count
|| mp
->nohomeok
) {
745 if (test_bit(META_dirty
, &mp
->flag
)) {
746 set_page_dirty(page
);
747 if (test_bit(META_sync
, &mp
->flag
)) {
748 clear_bit(META_sync
, &mp
->flag
);
749 if (write_one_page(page
))
750 jfs_error(mp
->sb
, "write_one_page() failed\n");
751 lock_page(page
); /* write_one_page unlocks the page */
753 } else if (mp
->lsn
) /* discard_metapage doesn't remove it */
754 remove_from_logsync(mp
);
756 /* Try to keep metapages from using up too much memory */
757 drop_metapage(page
, mp
);
763 void __invalidate_metapages(struct inode
*ip
, s64 addr
, int len
)
766 int l2BlocksPerPage
= PAGE_SHIFT
- ip
->i_blkbits
;
767 int BlocksPerPage
= 1 << l2BlocksPerPage
;
768 /* All callers are interested in block device's mapping */
769 struct address_space
*mapping
=
770 JFS_SBI(ip
->i_sb
)->direct_inode
->i_mapping
;
776 * Mark metapages to discard. They will eventually be
777 * released, but should not be written.
779 for (lblock
= addr
& ~(BlocksPerPage
- 1); lblock
< addr
+ len
;
780 lblock
+= BlocksPerPage
) {
781 page
= find_lock_page(mapping
, lblock
>> l2BlocksPerPage
);
784 for (offset
= 0; offset
< PAGE_SIZE
; offset
+= PSIZE
) {
785 mp
= page_to_mp(page
, offset
);
788 if (mp
->index
< addr
)
790 if (mp
->index
>= addr
+ len
)
793 clear_bit(META_dirty
, &mp
->flag
);
794 set_bit(META_discard
, &mp
->flag
);
796 remove_from_logsync(mp
);
803 #ifdef CONFIG_JFS_STATISTICS
804 int jfs_mpstat_proc_show(struct seq_file
*m
, void *v
)
807 "JFS Metapage statistics\n"
808 "=======================\n"
809 "page allocations = %d\n"