1 // SPDX-License-Identifier: GPL-2.0
5 * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds
7 * Swap reorganised 29.12.95,
8 * Asynchronous swapping added 30.12.95. Stephen Tweedie
9 * Removed race in async swapping. 14.4.1996. Bruno Haible
10 * Add swap of shared pages through the page cache. 20.2.1998. Stephen Tweedie
11 * Always use brw_page, life becomes simpler. 12 May 1998 Eric Biederman
15 #include <linux/kernel_stat.h>
16 #include <linux/gfp.h>
17 #include <linux/pagemap.h>
18 #include <linux/swap.h>
19 #include <linux/bio.h>
20 #include <linux/swapops.h>
21 #include <linux/buffer_head.h>
22 #include <linux/writeback.h>
23 #include <linux/frontswap.h>
24 #include <linux/blkdev.h>
25 #include <linux/uio.h>
26 #include <linux/sched/task.h>
27 #include <asm/pgtable.h>
29 static struct bio
*get_swap_bio(gfp_t gfp_flags
,
30 struct page
*page
, bio_end_io_t end_io
)
32 int i
, nr
= hpage_nr_pages(page
);
35 bio
= bio_alloc(gfp_flags
, nr
);
37 struct block_device
*bdev
;
39 bio
->bi_iter
.bi_sector
= map_swap_page(page
, &bdev
);
40 bio_set_dev(bio
, bdev
);
41 bio
->bi_iter
.bi_sector
<<= PAGE_SHIFT
- 9;
42 bio
->bi_end_io
= end_io
;
44 for (i
= 0; i
< nr
; i
++)
45 bio_add_page(bio
, page
+ i
, PAGE_SIZE
, 0);
46 VM_BUG_ON(bio
->bi_iter
.bi_size
!= PAGE_SIZE
* nr
);
51 void end_swap_bio_write(struct bio
*bio
)
53 struct page
*page
= bio_first_page_all(bio
);
58 * We failed to write the page out to swap-space.
59 * Re-dirty the page in order to avoid it being reclaimed.
60 * Also print a dire warning that things will go BAD (tm)
63 * Also clear PG_reclaim to avoid rotate_reclaimable_page()
66 pr_alert("Write-error on swap-device (%u:%u:%llu)\n",
67 MAJOR(bio_dev(bio
)), MINOR(bio_dev(bio
)),
68 (unsigned long long)bio
->bi_iter
.bi_sector
);
69 ClearPageReclaim(page
);
71 end_page_writeback(page
);
75 static void swap_slot_free_notify(struct page
*page
)
77 struct swap_info_struct
*sis
;
81 * There is no guarantee that the page is in swap cache - the software
82 * suspend code (at least) uses end_swap_bio_read() against a non-
83 * swapcache page. So we must check PG_swapcache before proceeding with
86 if (unlikely(!PageSwapCache(page
)))
89 sis
= page_swap_info(page
);
90 if (!(sis
->flags
& SWP_BLKDEV
))
94 * The swap subsystem performs lazy swap slot freeing,
95 * expecting that the page will be swapped out again.
96 * So we can avoid an unnecessary write if the page
98 * This is good for real swap storage because we can
99 * reduce unnecessary I/O and enhance wear-leveling
100 * if an SSD is used as the as swap device.
101 * But if in-memory swap device (eg zram) is used,
102 * this causes a duplicated copy between uncompressed
103 * data in VM-owned memory and compressed data in
104 * zram-owned memory. So let's free zram-owned memory
105 * and make the VM-owned decompressed page *dirty*,
106 * so the page should be swapped out somewhere again if
107 * we again wish to reclaim it.
109 disk
= sis
->bdev
->bd_disk
;
110 if (disk
->fops
->swap_slot_free_notify
) {
112 unsigned long offset
;
114 entry
.val
= page_private(page
);
115 offset
= swp_offset(entry
);
118 disk
->fops
->swap_slot_free_notify(sis
->bdev
,
123 static void end_swap_bio_read(struct bio
*bio
)
125 struct page
*page
= bio_first_page_all(bio
);
126 struct task_struct
*waiter
= bio
->bi_private
;
128 if (bio
->bi_status
) {
130 ClearPageUptodate(page
);
131 pr_alert("Read-error on swap-device (%u:%u:%llu)\n",
132 MAJOR(bio_dev(bio
)), MINOR(bio_dev(bio
)),
133 (unsigned long long)bio
->bi_iter
.bi_sector
);
137 SetPageUptodate(page
);
138 swap_slot_free_notify(page
);
141 WRITE_ONCE(bio
->bi_private
, NULL
);
143 wake_up_process(waiter
);
144 put_task_struct(waiter
);
147 int generic_swapfile_activate(struct swap_info_struct
*sis
,
148 struct file
*swap_file
,
151 struct address_space
*mapping
= swap_file
->f_mapping
;
152 struct inode
*inode
= mapping
->host
;
153 unsigned blocks_per_page
;
154 unsigned long page_no
;
156 sector_t probe_block
;
158 sector_t lowest_block
= -1;
159 sector_t highest_block
= 0;
163 blkbits
= inode
->i_blkbits
;
164 blocks_per_page
= PAGE_SIZE
>> blkbits
;
167 * Map all the blocks into the extent list. This code doesn't try
172 last_block
= i_size_read(inode
) >> blkbits
;
173 while ((probe_block
+ blocks_per_page
) <= last_block
&&
174 page_no
< sis
->max
) {
175 unsigned block_in_page
;
176 sector_t first_block
;
180 first_block
= bmap(inode
, probe_block
);
181 if (first_block
== 0)
185 * It must be PAGE_SIZE aligned on-disk
187 if (first_block
& (blocks_per_page
- 1)) {
192 for (block_in_page
= 1; block_in_page
< blocks_per_page
;
196 block
= bmap(inode
, probe_block
+ block_in_page
);
199 if (block
!= first_block
+ block_in_page
) {
206 first_block
>>= (PAGE_SHIFT
- blkbits
);
207 if (page_no
) { /* exclude the header page */
208 if (first_block
< lowest_block
)
209 lowest_block
= first_block
;
210 if (first_block
> highest_block
)
211 highest_block
= first_block
;
215 * We found a PAGE_SIZE-length, PAGE_SIZE-aligned run of blocks
217 ret
= add_swap_extent(sis
, page_no
, 1, first_block
);
222 probe_block
+= blocks_per_page
;
227 *span
= 1 + highest_block
- lowest_block
;
229 page_no
= 1; /* force Empty message */
231 sis
->pages
= page_no
- 1;
232 sis
->highest_bit
= page_no
- 1;
236 pr_err("swapon: swapfile has holes\n");
242 * We may have stale swap cache pages in memory: notice
243 * them here and get rid of the unnecessary final write.
245 int swap_writepage(struct page
*page
, struct writeback_control
*wbc
)
249 if (try_to_free_swap(page
)) {
253 if (frontswap_store(page
) == 0) {
254 set_page_writeback(page
);
256 end_page_writeback(page
);
259 ret
= __swap_writepage(page
, wbc
, end_swap_bio_write
);
264 static sector_t
swap_page_sector(struct page
*page
)
266 return (sector_t
)__page_file_index(page
) << (PAGE_SHIFT
- 9);
269 static inline void count_swpout_vm_event(struct page
*page
)
271 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
272 if (unlikely(PageTransHuge(page
)))
273 count_vm_event(THP_SWPOUT
);
275 count_vm_events(PSWPOUT
, hpage_nr_pages(page
));
278 int __swap_writepage(struct page
*page
, struct writeback_control
*wbc
,
279 bio_end_io_t end_write_func
)
283 struct swap_info_struct
*sis
= page_swap_info(page
);
285 VM_BUG_ON_PAGE(!PageSwapCache(page
), page
);
286 if (sis
->flags
& SWP_FILE
) {
288 struct file
*swap_file
= sis
->swap_file
;
289 struct address_space
*mapping
= swap_file
->f_mapping
;
290 struct bio_vec bv
= {
295 struct iov_iter from
;
297 iov_iter_bvec(&from
, ITER_BVEC
| WRITE
, &bv
, 1, PAGE_SIZE
);
298 init_sync_kiocb(&kiocb
, swap_file
);
299 kiocb
.ki_pos
= page_file_offset(page
);
301 set_page_writeback(page
);
303 ret
= mapping
->a_ops
->direct_IO(&kiocb
, &from
);
304 if (ret
== PAGE_SIZE
) {
305 count_vm_event(PSWPOUT
);
309 * In the case of swap-over-nfs, this can be a
310 * temporary failure if the system has limited
311 * memory for allocating transmit buffers.
312 * Mark the page dirty and avoid
313 * rotate_reclaimable_page but rate-limit the
314 * messages but do not flag PageError like
315 * the normal direct-to-bio case as it could
318 set_page_dirty(page
);
319 ClearPageReclaim(page
);
320 pr_err_ratelimited("Write error on dio swapfile (%llu)\n",
321 page_file_offset(page
));
323 end_page_writeback(page
);
327 ret
= bdev_write_page(sis
->bdev
, swap_page_sector(page
), page
, wbc
);
329 count_swpout_vm_event(page
);
334 bio
= get_swap_bio(GFP_NOIO
, page
, end_write_func
);
336 set_page_dirty(page
);
341 bio
->bi_opf
= REQ_OP_WRITE
| REQ_SWAP
| wbc_to_write_flags(wbc
);
342 bio_associate_blkg_from_page(bio
, page
);
343 count_swpout_vm_event(page
);
344 set_page_writeback(page
);
351 int swap_readpage(struct page
*page
, bool synchronous
)
355 struct swap_info_struct
*sis
= page_swap_info(page
);
357 struct gendisk
*disk
;
359 VM_BUG_ON_PAGE(!PageSwapCache(page
) && !synchronous
, page
);
360 VM_BUG_ON_PAGE(!PageLocked(page
), page
);
361 VM_BUG_ON_PAGE(PageUptodate(page
), page
);
362 if (frontswap_load(page
) == 0) {
363 SetPageUptodate(page
);
368 if (sis
->flags
& SWP_FILE
) {
369 struct file
*swap_file
= sis
->swap_file
;
370 struct address_space
*mapping
= swap_file
->f_mapping
;
372 ret
= mapping
->a_ops
->readpage(swap_file
, page
);
374 count_vm_event(PSWPIN
);
378 ret
= bdev_read_page(sis
->bdev
, swap_page_sector(page
), page
);
380 if (trylock_page(page
)) {
381 swap_slot_free_notify(page
);
385 count_vm_event(PSWPIN
);
390 bio
= get_swap_bio(GFP_KERNEL
, page
, end_swap_bio_read
);
398 * Keep this task valid during swap readpage because the oom killer may
399 * attempt to access it in the page fault retry time check.
401 get_task_struct(current
);
402 bio
->bi_private
= current
;
403 bio_set_op_attrs(bio
, REQ_OP_READ
, 0);
404 count_vm_event(PSWPIN
);
406 qc
= submit_bio(bio
);
407 while (synchronous
) {
408 set_current_state(TASK_UNINTERRUPTIBLE
);
409 if (!READ_ONCE(bio
->bi_private
))
412 if (!blk_poll(disk
->queue
, qc
))
415 __set_current_state(TASK_RUNNING
);
422 int swap_set_page_dirty(struct page
*page
)
424 struct swap_info_struct
*sis
= page_swap_info(page
);
426 if (sis
->flags
& SWP_FILE
) {
427 struct address_space
*mapping
= sis
->swap_file
->f_mapping
;
429 VM_BUG_ON_PAGE(!PageSwapCache(page
), page
);
430 return mapping
->a_ops
->set_page_dirty(page
);
432 return __set_page_dirty_no_writeback(page
);