2 * Ram backed block device driver.
4 * Copyright (C) 2007 Nick Piggin
5 * Copyright (C) 2007 Novell Inc.
7 * Parts derived from drivers/block/rd.c, and drivers/block/loop.c, copyright
8 * of their respective owners.
11 #include <linux/init.h>
12 #include <linux/initrd.h>
13 #include <linux/module.h>
14 #include <linux/moduleparam.h>
15 #include <linux/major.h>
16 #include <linux/blkdev.h>
17 #include <linux/bio.h>
18 #include <linux/highmem.h>
19 #include <linux/mutex.h>
20 #include <linux/radix-tree.h>
22 #include <linux/slab.h>
23 #ifdef CONFIG_BLK_DEV_RAM_DAX
24 #include <linux/pfn_t.h>
25 #include <linux/dax.h>
26 #include <linux/uio.h>
29 #include <linux/uaccess.h>
31 #define SECTOR_SHIFT 9
32 #define PAGE_SECTORS_SHIFT (PAGE_SHIFT - SECTOR_SHIFT)
33 #define PAGE_SECTORS (1 << PAGE_SECTORS_SHIFT)
36 * Each block ramdisk device has a radix_tree brd_pages of pages that stores
37 * the pages containing the block device's contents. A brd page's ->index is
38 * its offset in PAGE_SIZE units. This is similar to, but in no way connected
39 * with, the kernel's pagecache or buffer cache (which sit above our block
45 struct request_queue
*brd_queue
;
46 struct gendisk
*brd_disk
;
47 #ifdef CONFIG_BLK_DEV_RAM_DAX
48 struct dax_device
*dax_dev
;
50 struct list_head brd_list
;
53 * Backing store of pages and lock to protect it. This is the contents
54 * of the block device.
57 struct radix_tree_root brd_pages
;
61 * Look up and return a brd's page for a given sector.
63 static DEFINE_MUTEX(brd_mutex
);
64 static struct page
*brd_lookup_page(struct brd_device
*brd
, sector_t sector
)
70 * The page lifetime is protected by the fact that we have opened the
71 * device node -- brd pages will never be deleted under us, so we
72 * don't need any further locking or refcounting.
74 * This is strictly true for the radix-tree nodes as well (ie. we
75 * don't actually need the rcu_read_lock()), however that is not a
76 * documented feature of the radix-tree API so it is better to be
77 * safe here (we don't have total exclusion from radix tree updates
78 * here, only deletes).
81 idx
= sector
>> PAGE_SECTORS_SHIFT
; /* sector to page index */
82 page
= radix_tree_lookup(&brd
->brd_pages
, idx
);
85 BUG_ON(page
&& page
->index
!= idx
);
91 * Look up and return a brd's page for a given sector.
92 * If one does not exist, allocate an empty page, and insert that. Then
95 static struct page
*brd_insert_page(struct brd_device
*brd
, sector_t sector
)
101 page
= brd_lookup_page(brd
, sector
);
106 * Must use NOIO because we don't want to recurse back into the
107 * block or filesystem layers from page reclaim.
109 * Cannot support DAX and highmem, because our ->direct_access
110 * routine for DAX must return memory that is always addressable.
111 * If DAX was reworked to use pfns and kmap throughout, this
112 * restriction might be able to be lifted.
114 gfp_flags
= GFP_NOIO
| __GFP_ZERO
;
115 #ifndef CONFIG_BLK_DEV_RAM_DAX
116 gfp_flags
|= __GFP_HIGHMEM
;
118 page
= alloc_page(gfp_flags
);
122 if (radix_tree_preload(GFP_NOIO
)) {
127 spin_lock(&brd
->brd_lock
);
128 idx
= sector
>> PAGE_SECTORS_SHIFT
;
130 if (radix_tree_insert(&brd
->brd_pages
, idx
, page
)) {
132 page
= radix_tree_lookup(&brd
->brd_pages
, idx
);
134 BUG_ON(page
->index
!= idx
);
136 spin_unlock(&brd
->brd_lock
);
138 radix_tree_preload_end();
144 * Free all backing store pages and radix tree. This must only be called when
145 * there are no other users of the device.
147 #define FREE_BATCH 16
148 static void brd_free_pages(struct brd_device
*brd
)
150 unsigned long pos
= 0;
151 struct page
*pages
[FREE_BATCH
];
157 nr_pages
= radix_tree_gang_lookup(&brd
->brd_pages
,
158 (void **)pages
, pos
, FREE_BATCH
);
160 for (i
= 0; i
< nr_pages
; i
++) {
163 BUG_ON(pages
[i
]->index
< pos
);
164 pos
= pages
[i
]->index
;
165 ret
= radix_tree_delete(&brd
->brd_pages
, pos
);
166 BUG_ON(!ret
|| ret
!= pages
[i
]);
167 __free_page(pages
[i
]);
173 * This assumes radix_tree_gang_lookup always returns as
174 * many pages as possible. If the radix-tree code changes,
175 * so will this have to.
177 } while (nr_pages
== FREE_BATCH
);
181 * copy_to_brd_setup must be called before copy_to_brd. It may sleep.
183 static int copy_to_brd_setup(struct brd_device
*brd
, sector_t sector
, size_t n
)
185 unsigned int offset
= (sector
& (PAGE_SECTORS
-1)) << SECTOR_SHIFT
;
188 copy
= min_t(size_t, n
, PAGE_SIZE
- offset
);
189 if (!brd_insert_page(brd
, sector
))
192 sector
+= copy
>> SECTOR_SHIFT
;
193 if (!brd_insert_page(brd
, sector
))
200 * Copy n bytes from src to the brd starting at sector. Does not sleep.
202 static void copy_to_brd(struct brd_device
*brd
, const void *src
,
203 sector_t sector
, size_t n
)
207 unsigned int offset
= (sector
& (PAGE_SECTORS
-1)) << SECTOR_SHIFT
;
210 copy
= min_t(size_t, n
, PAGE_SIZE
- offset
);
211 page
= brd_lookup_page(brd
, sector
);
214 dst
= kmap_atomic(page
);
215 memcpy(dst
+ offset
, src
, copy
);
220 sector
+= copy
>> SECTOR_SHIFT
;
222 page
= brd_lookup_page(brd
, sector
);
225 dst
= kmap_atomic(page
);
226 memcpy(dst
, src
, copy
);
232 * Copy n bytes to dst from the brd starting at sector. Does not sleep.
234 static void copy_from_brd(void *dst
, struct brd_device
*brd
,
235 sector_t sector
, size_t n
)
239 unsigned int offset
= (sector
& (PAGE_SECTORS
-1)) << SECTOR_SHIFT
;
242 copy
= min_t(size_t, n
, PAGE_SIZE
- offset
);
243 page
= brd_lookup_page(brd
, sector
);
245 src
= kmap_atomic(page
);
246 memcpy(dst
, src
+ offset
, copy
);
249 memset(dst
, 0, copy
);
253 sector
+= copy
>> SECTOR_SHIFT
;
255 page
= brd_lookup_page(brd
, sector
);
257 src
= kmap_atomic(page
);
258 memcpy(dst
, src
, copy
);
261 memset(dst
, 0, copy
);
266 * Process a single bvec of a bio.
268 static int brd_do_bvec(struct brd_device
*brd
, struct page
*page
,
269 unsigned int len
, unsigned int off
, bool is_write
,
276 err
= copy_to_brd_setup(brd
, sector
, len
);
281 mem
= kmap_atomic(page
);
283 copy_from_brd(mem
+ off
, brd
, sector
, len
);
284 flush_dcache_page(page
);
286 flush_dcache_page(page
);
287 copy_to_brd(brd
, mem
+ off
, sector
, len
);
295 static blk_qc_t
brd_make_request(struct request_queue
*q
, struct bio
*bio
)
297 struct brd_device
*brd
= bio
->bi_disk
->private_data
;
300 struct bvec_iter iter
;
302 sector
= bio
->bi_iter
.bi_sector
;
303 if (bio_end_sector(bio
) > get_capacity(bio
->bi_disk
))
306 bio_for_each_segment(bvec
, bio
, iter
) {
307 unsigned int len
= bvec
.bv_len
;
310 err
= brd_do_bvec(brd
, bvec
.bv_page
, len
, bvec
.bv_offset
,
311 op_is_write(bio_op(bio
)), sector
);
314 sector
+= len
>> SECTOR_SHIFT
;
318 return BLK_QC_T_NONE
;
321 return BLK_QC_T_NONE
;
324 static int brd_rw_page(struct block_device
*bdev
, sector_t sector
,
325 struct page
*page
, bool is_write
)
327 struct brd_device
*brd
= bdev
->bd_disk
->private_data
;
330 if (PageTransHuge(page
))
332 err
= brd_do_bvec(brd
, page
, PAGE_SIZE
, 0, is_write
, sector
);
333 page_endio(page
, is_write
, err
);
337 #ifdef CONFIG_BLK_DEV_RAM_DAX
338 static long __brd_direct_access(struct brd_device
*brd
, pgoff_t pgoff
,
339 long nr_pages
, void **kaddr
, pfn_t
*pfn
)
345 page
= brd_insert_page(brd
, PFN_PHYS(pgoff
) / 512);
348 *kaddr
= page_address(page
);
349 *pfn
= page_to_pfn_t(page
);
354 static long brd_dax_direct_access(struct dax_device
*dax_dev
,
355 pgoff_t pgoff
, long nr_pages
, void **kaddr
, pfn_t
*pfn
)
357 struct brd_device
*brd
= dax_get_private(dax_dev
);
359 return __brd_direct_access(brd
, pgoff
, nr_pages
, kaddr
, pfn
);
362 static size_t brd_dax_copy_from_iter(struct dax_device
*dax_dev
, pgoff_t pgoff
,
363 void *addr
, size_t bytes
, struct iov_iter
*i
)
365 return copy_from_iter(addr
, bytes
, i
);
368 static const struct dax_operations brd_dax_ops
= {
369 .direct_access
= brd_dax_direct_access
,
370 .copy_from_iter
= brd_dax_copy_from_iter
,
374 static const struct block_device_operations brd_fops
= {
375 .owner
= THIS_MODULE
,
376 .rw_page
= brd_rw_page
,
380 * And now the modules code and kernel interface.
382 static int rd_nr
= CONFIG_BLK_DEV_RAM_COUNT
;
383 module_param(rd_nr
, int, S_IRUGO
);
384 MODULE_PARM_DESC(rd_nr
, "Maximum number of brd devices");
386 unsigned long rd_size
= CONFIG_BLK_DEV_RAM_SIZE
;
387 module_param(rd_size
, ulong
, S_IRUGO
);
388 MODULE_PARM_DESC(rd_size
, "Size of each RAM disk in kbytes.");
390 static int max_part
= 1;
391 module_param(max_part
, int, S_IRUGO
);
392 MODULE_PARM_DESC(max_part
, "Num Minors to reserve between devices");
394 MODULE_LICENSE("GPL");
395 MODULE_ALIAS_BLOCKDEV_MAJOR(RAMDISK_MAJOR
);
399 /* Legacy boot options - nonmodular */
400 static int __init
ramdisk_size(char *str
)
402 rd_size
= simple_strtol(str
, NULL
, 0);
405 __setup("ramdisk_size=", ramdisk_size
);
409 * The device scheme is derived from loop.c. Keep them in synch where possible
410 * (should share code eventually).
412 static LIST_HEAD(brd_devices
);
413 static DEFINE_MUTEX(brd_devices_mutex
);
415 static struct brd_device
*brd_alloc(int i
)
417 struct brd_device
*brd
;
418 struct gendisk
*disk
;
420 brd
= kzalloc(sizeof(*brd
), GFP_KERNEL
);
424 spin_lock_init(&brd
->brd_lock
);
425 INIT_RADIX_TREE(&brd
->brd_pages
, GFP_ATOMIC
);
427 brd
->brd_queue
= blk_alloc_queue(GFP_KERNEL
);
431 blk_queue_make_request(brd
->brd_queue
, brd_make_request
);
432 blk_queue_max_hw_sectors(brd
->brd_queue
, 1024);
434 /* This is so fdisk will align partitions on 4k, because of
435 * direct_access API needing 4k alignment, returning a PFN
436 * (This is only a problem on very small devices <= 4M,
437 * otherwise fdisk will align on 1M. Regardless this call
440 blk_queue_physical_block_size(brd
->brd_queue
, PAGE_SIZE
);
441 disk
= brd
->brd_disk
= alloc_disk(max_part
);
444 disk
->major
= RAMDISK_MAJOR
;
445 disk
->first_minor
= i
* max_part
;
446 disk
->fops
= &brd_fops
;
447 disk
->private_data
= brd
;
448 disk
->queue
= brd
->brd_queue
;
449 disk
->flags
= GENHD_FL_EXT_DEVT
;
450 sprintf(disk
->disk_name
, "ram%d", i
);
451 set_capacity(disk
, rd_size
* 2);
453 #ifdef CONFIG_BLK_DEV_RAM_DAX
454 queue_flag_set_unlocked(QUEUE_FLAG_DAX
, brd
->brd_queue
);
455 brd
->dax_dev
= alloc_dax(brd
, disk
->disk_name
, &brd_dax_ops
);
463 #ifdef CONFIG_BLK_DEV_RAM_DAX
465 kill_dax(brd
->dax_dev
);
466 put_dax(brd
->dax_dev
);
469 blk_cleanup_queue(brd
->brd_queue
);
476 static void brd_free(struct brd_device
*brd
)
478 put_disk(brd
->brd_disk
);
479 blk_cleanup_queue(brd
->brd_queue
);
484 static struct brd_device
*brd_init_one(int i
, bool *new)
486 struct brd_device
*brd
;
489 list_for_each_entry(brd
, &brd_devices
, brd_list
) {
490 if (brd
->brd_number
== i
)
496 add_disk(brd
->brd_disk
);
497 list_add_tail(&brd
->brd_list
, &brd_devices
);
504 static void brd_del_one(struct brd_device
*brd
)
506 list_del(&brd
->brd_list
);
507 #ifdef CONFIG_BLK_DEV_RAM_DAX
508 kill_dax(brd
->dax_dev
);
509 put_dax(brd
->dax_dev
);
511 del_gendisk(brd
->brd_disk
);
515 static struct kobject
*brd_probe(dev_t dev
, int *part
, void *data
)
517 struct brd_device
*brd
;
518 struct kobject
*kobj
;
521 mutex_lock(&brd_devices_mutex
);
522 brd
= brd_init_one(MINOR(dev
) / max_part
, &new);
523 kobj
= brd
? get_disk(brd
->brd_disk
) : NULL
;
524 mutex_unlock(&brd_devices_mutex
);
532 static int __init
brd_init(void)
534 struct brd_device
*brd
, *next
;
538 * brd module now has a feature to instantiate underlying device
539 * structure on-demand, provided that there is an access dev node.
541 * (1) if rd_nr is specified, create that many upfront. else
542 * it defaults to CONFIG_BLK_DEV_RAM_COUNT
543 * (2) User can further extend brd devices by create dev node themselves
544 * and have kernel automatically instantiate actual device
545 * on-demand. Example:
546 * mknod /path/devnod_name b 1 X # 1 is the rd major
547 * fdisk -l /path/devnod_name
548 * If (X / max_part) was not already created it will be created
552 if (register_blkdev(RAMDISK_MAJOR
, "ramdisk"))
555 if (unlikely(!max_part
))
558 for (i
= 0; i
< rd_nr
; i
++) {
562 list_add_tail(&brd
->brd_list
, &brd_devices
);
565 /* point of no return */
567 list_for_each_entry(brd
, &brd_devices
, brd_list
)
568 add_disk(brd
->brd_disk
);
570 blk_register_region(MKDEV(RAMDISK_MAJOR
, 0), 1UL << MINORBITS
,
571 THIS_MODULE
, brd_probe
, NULL
, NULL
);
573 pr_info("brd: module loaded\n");
577 list_for_each_entry_safe(brd
, next
, &brd_devices
, brd_list
) {
578 list_del(&brd
->brd_list
);
581 unregister_blkdev(RAMDISK_MAJOR
, "ramdisk");
583 pr_info("brd: module NOT loaded !!!\n");
587 static void __exit
brd_exit(void)
589 struct brd_device
*brd
, *next
;
591 list_for_each_entry_safe(brd
, next
, &brd_devices
, brd_list
)
594 blk_unregister_region(MKDEV(RAMDISK_MAJOR
, 0), 1UL << MINORBITS
);
595 unregister_blkdev(RAMDISK_MAJOR
, "ramdisk");
597 pr_info("brd: module unloaded\n");
600 module_init(brd_init
);
601 module_exit(brd_exit
);