1 #include <linux/module.h>
3 #include <linux/moduleparam.h>
4 #include <linux/sched.h>
6 #include <linux/blkdev.h>
7 #include <linux/init.h>
8 #include <linux/slab.h>
9 #include <linux/blk-mq.h>
10 #include <linux/hrtimer.h>
11 #include <linux/lightnvm.h>
14 struct list_head list
;
15 struct llist_node ll_list
;
16 struct call_single_data csd
;
20 struct nullb_queue
*nq
;
25 unsigned long *tag_map
;
26 wait_queue_head_t wait
;
27 unsigned int queue_depth
;
29 struct nullb_cmd
*cmds
;
33 struct list_head list
;
35 struct request_queue
*q
;
38 struct blk_mq_tag_set tag_set
;
40 unsigned int queue_depth
;
43 struct nullb_queue
*queues
;
44 unsigned int nr_queues
;
45 char disk_name
[DISK_NAME_LEN
];
48 static LIST_HEAD(nullb_list
);
49 static struct mutex lock
;
50 static int null_major
;
51 static int nullb_indexes
;
52 static struct kmem_cache
*ppa_cache
;
66 static int submit_queues
;
67 module_param(submit_queues
, int, S_IRUGO
);
68 MODULE_PARM_DESC(submit_queues
, "Number of submission queues");
70 static int home_node
= NUMA_NO_NODE
;
71 module_param(home_node
, int, S_IRUGO
);
72 MODULE_PARM_DESC(home_node
, "Home node for the device");
74 static int queue_mode
= NULL_Q_MQ
;
76 static int null_param_store_val(const char *str
, int *val
, int min
, int max
)
80 ret
= kstrtoint(str
, 10, &new_val
);
84 if (new_val
< min
|| new_val
> max
)
91 static int null_set_queue_mode(const char *str
, const struct kernel_param
*kp
)
93 return null_param_store_val(str
, &queue_mode
, NULL_Q_BIO
, NULL_Q_MQ
);
96 static const struct kernel_param_ops null_queue_mode_param_ops
= {
97 .set
= null_set_queue_mode
,
101 device_param_cb(queue_mode
, &null_queue_mode_param_ops
, &queue_mode
, S_IRUGO
);
102 MODULE_PARM_DESC(queue_mode
, "Block interface to use (0=bio,1=rq,2=multiqueue)");
105 module_param(gb
, int, S_IRUGO
);
106 MODULE_PARM_DESC(gb
, "Size in GB");
109 module_param(bs
, int, S_IRUGO
);
110 MODULE_PARM_DESC(bs
, "Block size (in bytes)");
112 static int nr_devices
= 2;
113 module_param(nr_devices
, int, S_IRUGO
);
114 MODULE_PARM_DESC(nr_devices
, "Number of devices to register");
116 static bool use_lightnvm
;
117 module_param(use_lightnvm
, bool, S_IRUGO
);
118 MODULE_PARM_DESC(use_lightnvm
, "Register as a LightNVM device");
120 static int irqmode
= NULL_IRQ_SOFTIRQ
;
122 static int null_set_irqmode(const char *str
, const struct kernel_param
*kp
)
124 return null_param_store_val(str
, &irqmode
, NULL_IRQ_NONE
,
128 static const struct kernel_param_ops null_irqmode_param_ops
= {
129 .set
= null_set_irqmode
,
130 .get
= param_get_int
,
133 device_param_cb(irqmode
, &null_irqmode_param_ops
, &irqmode
, S_IRUGO
);
134 MODULE_PARM_DESC(irqmode
, "IRQ completion handler. 0-none, 1-softirq, 2-timer");
136 static unsigned long completion_nsec
= 10000;
137 module_param(completion_nsec
, ulong
, S_IRUGO
);
138 MODULE_PARM_DESC(completion_nsec
, "Time in ns to complete a request in hardware. Default: 10,000ns");
140 static int hw_queue_depth
= 64;
141 module_param(hw_queue_depth
, int, S_IRUGO
);
142 MODULE_PARM_DESC(hw_queue_depth
, "Queue depth for each hardware queue. Default: 64");
144 static bool use_per_node_hctx
= false;
145 module_param(use_per_node_hctx
, bool, S_IRUGO
);
146 MODULE_PARM_DESC(use_per_node_hctx
, "Use per-node allocation for hardware context queues. Default: false");
148 static void put_tag(struct nullb_queue
*nq
, unsigned int tag
)
150 clear_bit_unlock(tag
, nq
->tag_map
);
152 if (waitqueue_active(&nq
->wait
))
156 static unsigned int get_tag(struct nullb_queue
*nq
)
161 tag
= find_first_zero_bit(nq
->tag_map
, nq
->queue_depth
);
162 if (tag
>= nq
->queue_depth
)
164 } while (test_and_set_bit_lock(tag
, nq
->tag_map
));
169 static void free_cmd(struct nullb_cmd
*cmd
)
171 put_tag(cmd
->nq
, cmd
->tag
);
174 static enum hrtimer_restart
null_cmd_timer_expired(struct hrtimer
*timer
);
176 static struct nullb_cmd
*__alloc_cmd(struct nullb_queue
*nq
)
178 struct nullb_cmd
*cmd
;
183 cmd
= &nq
->cmds
[tag
];
186 if (irqmode
== NULL_IRQ_TIMER
) {
187 hrtimer_init(&cmd
->timer
, CLOCK_MONOTONIC
,
189 cmd
->timer
.function
= null_cmd_timer_expired
;
197 static struct nullb_cmd
*alloc_cmd(struct nullb_queue
*nq
, int can_wait
)
199 struct nullb_cmd
*cmd
;
202 cmd
= __alloc_cmd(nq
);
203 if (cmd
|| !can_wait
)
207 prepare_to_wait(&nq
->wait
, &wait
, TASK_UNINTERRUPTIBLE
);
208 cmd
= __alloc_cmd(nq
);
215 finish_wait(&nq
->wait
, &wait
);
219 static void end_cmd(struct nullb_cmd
*cmd
)
221 struct request_queue
*q
= NULL
;
226 switch (queue_mode
) {
228 blk_mq_end_request(cmd
->rq
, 0);
231 INIT_LIST_HEAD(&cmd
->rq
->queuelist
);
232 blk_end_request_all(cmd
->rq
, 0);
241 /* Restart queue if needed, as we are freeing a tag */
242 if (queue_mode
== NULL_Q_RQ
&& blk_queue_stopped(q
)) {
245 spin_lock_irqsave(q
->queue_lock
, flags
);
246 blk_start_queue_async(q
);
247 spin_unlock_irqrestore(q
->queue_lock
, flags
);
251 static enum hrtimer_restart
null_cmd_timer_expired(struct hrtimer
*timer
)
253 end_cmd(container_of(timer
, struct nullb_cmd
, timer
));
255 return HRTIMER_NORESTART
;
258 static void null_cmd_end_timer(struct nullb_cmd
*cmd
)
260 ktime_t kt
= completion_nsec
;
262 hrtimer_start(&cmd
->timer
, kt
, HRTIMER_MODE_REL
);
265 static void null_softirq_done_fn(struct request
*rq
)
267 if (queue_mode
== NULL_Q_MQ
)
268 end_cmd(blk_mq_rq_to_pdu(rq
));
270 end_cmd(rq
->special
);
273 static inline void null_handle_cmd(struct nullb_cmd
*cmd
)
275 /* Complete IO by inline, softirq or timer */
277 case NULL_IRQ_SOFTIRQ
:
278 switch (queue_mode
) {
280 blk_mq_complete_request(cmd
->rq
, cmd
->rq
->errors
);
283 blk_complete_request(cmd
->rq
);
287 * XXX: no proper submitting cpu information available.
297 null_cmd_end_timer(cmd
);
302 static struct nullb_queue
*nullb_to_queue(struct nullb
*nullb
)
306 if (nullb
->nr_queues
!= 1)
307 index
= raw_smp_processor_id() / ((nr_cpu_ids
+ nullb
->nr_queues
- 1) / nullb
->nr_queues
);
309 return &nullb
->queues
[index
];
312 static blk_qc_t
null_queue_bio(struct request_queue
*q
, struct bio
*bio
)
314 struct nullb
*nullb
= q
->queuedata
;
315 struct nullb_queue
*nq
= nullb_to_queue(nullb
);
316 struct nullb_cmd
*cmd
;
318 cmd
= alloc_cmd(nq
, 1);
321 null_handle_cmd(cmd
);
322 return BLK_QC_T_NONE
;
325 static int null_rq_prep_fn(struct request_queue
*q
, struct request
*req
)
327 struct nullb
*nullb
= q
->queuedata
;
328 struct nullb_queue
*nq
= nullb_to_queue(nullb
);
329 struct nullb_cmd
*cmd
;
331 cmd
= alloc_cmd(nq
, 0);
339 return BLKPREP_DEFER
;
342 static void null_request_fn(struct request_queue
*q
)
346 while ((rq
= blk_fetch_request(q
)) != NULL
) {
347 struct nullb_cmd
*cmd
= rq
->special
;
349 spin_unlock_irq(q
->queue_lock
);
350 null_handle_cmd(cmd
);
351 spin_lock_irq(q
->queue_lock
);
355 static int null_queue_rq(struct blk_mq_hw_ctx
*hctx
,
356 const struct blk_mq_queue_data
*bd
)
358 struct nullb_cmd
*cmd
= blk_mq_rq_to_pdu(bd
->rq
);
360 if (irqmode
== NULL_IRQ_TIMER
) {
361 hrtimer_init(&cmd
->timer
, CLOCK_MONOTONIC
, HRTIMER_MODE_REL
);
362 cmd
->timer
.function
= null_cmd_timer_expired
;
365 cmd
->nq
= hctx
->driver_data
;
367 blk_mq_start_request(bd
->rq
);
369 null_handle_cmd(cmd
);
370 return BLK_MQ_RQ_QUEUE_OK
;
373 static void null_init_queue(struct nullb
*nullb
, struct nullb_queue
*nq
)
378 init_waitqueue_head(&nq
->wait
);
379 nq
->queue_depth
= nullb
->queue_depth
;
382 static int null_init_hctx(struct blk_mq_hw_ctx
*hctx
, void *data
,
385 struct nullb
*nullb
= data
;
386 struct nullb_queue
*nq
= &nullb
->queues
[index
];
388 hctx
->driver_data
= nq
;
389 null_init_queue(nullb
, nq
);
395 static struct blk_mq_ops null_mq_ops
= {
396 .queue_rq
= null_queue_rq
,
397 .init_hctx
= null_init_hctx
,
398 .complete
= null_softirq_done_fn
,
401 static void cleanup_queue(struct nullb_queue
*nq
)
407 static void cleanup_queues(struct nullb
*nullb
)
411 for (i
= 0; i
< nullb
->nr_queues
; i
++)
412 cleanup_queue(&nullb
->queues
[i
]);
414 kfree(nullb
->queues
);
419 static void null_lnvm_end_io(struct request
*rq
, int error
)
421 struct nvm_rq
*rqd
= rq
->end_io_data
;
423 nvm_end_io(rqd
, error
);
428 static int null_lnvm_submit_io(struct nvm_dev
*dev
, struct nvm_rq
*rqd
)
430 struct request_queue
*q
= dev
->q
;
432 struct bio
*bio
= rqd
->bio
;
434 rq
= blk_mq_alloc_request(q
,
435 op_is_write(bio_op(bio
)) ? REQ_OP_DRV_OUT
: REQ_OP_DRV_IN
, 0);
439 rq
->__sector
= bio
->bi_iter
.bi_sector
;
440 rq
->ioprio
= bio_prio(bio
);
442 if (bio_has_data(bio
))
443 rq
->nr_phys_segments
= bio_phys_segments(q
, bio
);
445 rq
->__data_len
= bio
->bi_iter
.bi_size
;
446 rq
->bio
= rq
->biotail
= bio
;
448 rq
->end_io_data
= rqd
;
450 blk_execute_rq_nowait(q
, NULL
, rq
, 0, null_lnvm_end_io
);
455 static int null_lnvm_id(struct nvm_dev
*dev
, struct nvm_id
*id
)
457 sector_t size
= gb
* 1024 * 1024 * 1024ULL;
459 struct nvm_id_group
*grp
;
467 id
->ppaf
.blk_offset
= 0;
468 id
->ppaf
.blk_len
= 16;
469 id
->ppaf
.pg_offset
= 16;
470 id
->ppaf
.pg_len
= 16;
471 id
->ppaf
.sect_offset
= 32;
472 id
->ppaf
.sect_len
= 8;
473 id
->ppaf
.pln_offset
= 40;
474 id
->ppaf
.pln_len
= 8;
475 id
->ppaf
.lun_offset
= 48;
476 id
->ppaf
.lun_len
= 8;
477 id
->ppaf
.ch_offset
= 56;
480 sector_div(size
, bs
); /* convert size to pages */
481 size
>>= 8; /* concert size to pgs pr blk */
482 grp
= &id
->groups
[0];
489 grp
->num_lun
= size
+ 1;
490 sector_div(blksize
, grp
->num_lun
);
491 grp
->num_blk
= blksize
;
502 grp
->mpos
= 0x010101; /* single plane rwe */
503 grp
->cpar
= hw_queue_depth
;
508 static void *null_lnvm_create_dma_pool(struct nvm_dev
*dev
, char *name
)
510 mempool_t
*virtmem_pool
;
512 virtmem_pool
= mempool_create_slab_pool(64, ppa_cache
);
514 pr_err("null_blk: Unable to create virtual memory pool\n");
521 static void null_lnvm_destroy_dma_pool(void *pool
)
523 mempool_destroy(pool
);
526 static void *null_lnvm_dev_dma_alloc(struct nvm_dev
*dev
, void *pool
,
527 gfp_t mem_flags
, dma_addr_t
*dma_handler
)
529 return mempool_alloc(pool
, mem_flags
);
532 static void null_lnvm_dev_dma_free(void *pool
, void *entry
,
533 dma_addr_t dma_handler
)
535 mempool_free(entry
, pool
);
538 static struct nvm_dev_ops null_lnvm_dev_ops
= {
539 .identity
= null_lnvm_id
,
540 .submit_io
= null_lnvm_submit_io
,
542 .create_dma_pool
= null_lnvm_create_dma_pool
,
543 .destroy_dma_pool
= null_lnvm_destroy_dma_pool
,
544 .dev_dma_alloc
= null_lnvm_dev_dma_alloc
,
545 .dev_dma_free
= null_lnvm_dev_dma_free
,
547 /* Simulate nvme protocol restriction */
551 static int null_nvm_register(struct nullb
*nullb
)
556 dev
= nvm_alloc_dev(0);
561 memcpy(dev
->name
, nullb
->disk_name
, DISK_NAME_LEN
);
562 dev
->ops
= &null_lnvm_dev_ops
;
564 rv
= nvm_register(dev
);
573 static void null_nvm_unregister(struct nullb
*nullb
)
575 nvm_unregister(nullb
->ndev
);
578 static int null_nvm_register(struct nullb
*nullb
)
580 pr_err("null_blk: CONFIG_NVM needs to be enabled for LightNVM\n");
583 static void null_nvm_unregister(struct nullb
*nullb
) {}
584 #endif /* CONFIG_NVM */
586 static void null_del_dev(struct nullb
*nullb
)
588 list_del_init(&nullb
->list
);
591 null_nvm_unregister(nullb
);
593 del_gendisk(nullb
->disk
);
594 blk_cleanup_queue(nullb
->q
);
595 if (queue_mode
== NULL_Q_MQ
)
596 blk_mq_free_tag_set(&nullb
->tag_set
);
598 put_disk(nullb
->disk
);
599 cleanup_queues(nullb
);
603 static int null_open(struct block_device
*bdev
, fmode_t mode
)
608 static void null_release(struct gendisk
*disk
, fmode_t mode
)
612 static const struct block_device_operations null_fops
= {
613 .owner
= THIS_MODULE
,
615 .release
= null_release
,
618 static int setup_commands(struct nullb_queue
*nq
)
620 struct nullb_cmd
*cmd
;
623 nq
->cmds
= kzalloc(nq
->queue_depth
* sizeof(*cmd
), GFP_KERNEL
);
627 tag_size
= ALIGN(nq
->queue_depth
, BITS_PER_LONG
) / BITS_PER_LONG
;
628 nq
->tag_map
= kzalloc(tag_size
* sizeof(unsigned long), GFP_KERNEL
);
634 for (i
= 0; i
< nq
->queue_depth
; i
++) {
636 INIT_LIST_HEAD(&cmd
->list
);
637 cmd
->ll_list
.next
= NULL
;
644 static int setup_queues(struct nullb
*nullb
)
646 nullb
->queues
= kzalloc(submit_queues
* sizeof(struct nullb_queue
),
651 nullb
->nr_queues
= 0;
652 nullb
->queue_depth
= hw_queue_depth
;
657 static int init_driver_queues(struct nullb
*nullb
)
659 struct nullb_queue
*nq
;
662 for (i
= 0; i
< submit_queues
; i
++) {
663 nq
= &nullb
->queues
[i
];
665 null_init_queue(nullb
, nq
);
667 ret
= setup_commands(nq
);
675 static int null_gendisk_register(struct nullb
*nullb
)
677 struct gendisk
*disk
;
680 disk
= nullb
->disk
= alloc_disk_node(1, home_node
);
683 size
= gb
* 1024 * 1024 * 1024ULL;
684 set_capacity(disk
, size
>> 9);
686 disk
->flags
|= GENHD_FL_EXT_DEVT
| GENHD_FL_SUPPRESS_PARTITION_INFO
;
687 disk
->major
= null_major
;
688 disk
->first_minor
= nullb
->index
;
689 disk
->fops
= &null_fops
;
690 disk
->private_data
= nullb
;
691 disk
->queue
= nullb
->q
;
692 strncpy(disk
->disk_name
, nullb
->disk_name
, DISK_NAME_LEN
);
698 static int null_add_dev(void)
703 nullb
= kzalloc_node(sizeof(*nullb
), GFP_KERNEL
, home_node
);
709 spin_lock_init(&nullb
->lock
);
711 if (queue_mode
== NULL_Q_MQ
&& use_per_node_hctx
)
712 submit_queues
= nr_online_nodes
;
714 rv
= setup_queues(nullb
);
718 if (queue_mode
== NULL_Q_MQ
) {
719 nullb
->tag_set
.ops
= &null_mq_ops
;
720 nullb
->tag_set
.nr_hw_queues
= submit_queues
;
721 nullb
->tag_set
.queue_depth
= hw_queue_depth
;
722 nullb
->tag_set
.numa_node
= home_node
;
723 nullb
->tag_set
.cmd_size
= sizeof(struct nullb_cmd
);
724 nullb
->tag_set
.flags
= BLK_MQ_F_SHOULD_MERGE
;
725 nullb
->tag_set
.driver_data
= nullb
;
727 rv
= blk_mq_alloc_tag_set(&nullb
->tag_set
);
729 goto out_cleanup_queues
;
731 nullb
->q
= blk_mq_init_queue(&nullb
->tag_set
);
732 if (IS_ERR(nullb
->q
)) {
734 goto out_cleanup_tags
;
736 } else if (queue_mode
== NULL_Q_BIO
) {
737 nullb
->q
= blk_alloc_queue_node(GFP_KERNEL
, home_node
);
740 goto out_cleanup_queues
;
742 blk_queue_make_request(nullb
->q
, null_queue_bio
);
743 rv
= init_driver_queues(nullb
);
745 goto out_cleanup_blk_queue
;
747 nullb
->q
= blk_init_queue_node(null_request_fn
, &nullb
->lock
, home_node
);
750 goto out_cleanup_queues
;
752 blk_queue_prep_rq(nullb
->q
, null_rq_prep_fn
);
753 blk_queue_softirq_done(nullb
->q
, null_softirq_done_fn
);
754 rv
= init_driver_queues(nullb
);
756 goto out_cleanup_blk_queue
;
759 nullb
->q
->queuedata
= nullb
;
760 queue_flag_set_unlocked(QUEUE_FLAG_NONROT
, nullb
->q
);
761 queue_flag_clear_unlocked(QUEUE_FLAG_ADD_RANDOM
, nullb
->q
);
764 nullb
->index
= nullb_indexes
++;
767 blk_queue_logical_block_size(nullb
->q
, bs
);
768 blk_queue_physical_block_size(nullb
->q
, bs
);
770 sprintf(nullb
->disk_name
, "nullb%d", nullb
->index
);
773 rv
= null_nvm_register(nullb
);
775 rv
= null_gendisk_register(nullb
);
778 goto out_cleanup_blk_queue
;
781 list_add_tail(&nullb
->list
, &nullb_list
);
785 out_cleanup_blk_queue
:
786 blk_cleanup_queue(nullb
->q
);
788 if (queue_mode
== NULL_Q_MQ
)
789 blk_mq_free_tag_set(&nullb
->tag_set
);
791 cleanup_queues(nullb
);
798 static int __init
null_init(void)
804 if (bs
> PAGE_SIZE
) {
805 pr_warn("null_blk: invalid block size\n");
806 pr_warn("null_blk: defaults block size to %lu\n", PAGE_SIZE
);
810 if (use_lightnvm
&& bs
!= 4096) {
811 pr_warn("null_blk: LightNVM only supports 4k block size\n");
812 pr_warn("null_blk: defaults block size to 4k\n");
816 if (use_lightnvm
&& queue_mode
!= NULL_Q_MQ
) {
817 pr_warn("null_blk: LightNVM only supported for blk-mq\n");
818 pr_warn("null_blk: defaults queue mode to blk-mq\n");
819 queue_mode
= NULL_Q_MQ
;
822 if (queue_mode
== NULL_Q_MQ
&& use_per_node_hctx
) {
823 if (submit_queues
< nr_online_nodes
) {
824 pr_warn("null_blk: submit_queues param is set to %u.",
826 submit_queues
= nr_online_nodes
;
828 } else if (submit_queues
> nr_cpu_ids
)
829 submit_queues
= nr_cpu_ids
;
830 else if (!submit_queues
)
835 null_major
= register_blkdev(0, "nullb");
840 ppa_cache
= kmem_cache_create("ppa_cache", 64 * sizeof(u64
),
843 pr_err("null_blk: unable to create ppa cache\n");
849 for (i
= 0; i
< nr_devices
; i
++) {
850 ret
= null_add_dev();
855 pr_info("null: module loaded\n");
859 while (!list_empty(&nullb_list
)) {
860 nullb
= list_entry(nullb_list
.next
, struct nullb
, list
);
863 kmem_cache_destroy(ppa_cache
);
865 unregister_blkdev(null_major
, "nullb");
869 static void __exit
null_exit(void)
873 unregister_blkdev(null_major
, "nullb");
876 while (!list_empty(&nullb_list
)) {
877 nullb
= list_entry(nullb_list
.next
, struct nullb
, list
);
882 kmem_cache_destroy(ppa_cache
);
885 module_init(null_init
);
886 module_exit(null_exit
);
888 MODULE_AUTHOR("Jens Axboe <jaxboe@fusionio.com>");
889 MODULE_LICENSE("GPL");