1 // SPDX-License-Identifier: GPL-2.0-only
3 #include <linux/spinlock.h>
4 #include <linux/slab.h>
5 #include <linux/blkdev.h>
6 #include <linux/hdreg.h>
7 #include <linux/module.h>
8 #include <linux/mutex.h>
9 #include <linux/interrupt.h>
10 #include <linux/virtio.h>
11 #include <linux/virtio_blk.h>
12 #include <linux/scatterlist.h>
13 #include <linux/string_helpers.h>
14 #include <scsi/scsi_cmnd.h>
15 #include <linux/idr.h>
16 #include <linux/blk-mq.h>
17 #include <linux/blk-mq-virtio.h>
18 #include <linux/numa.h>
21 #define VQ_NAME_LEN 16
22 #define MAX_DISCARD_SEGMENTS 256u
25 static DEFINE_IDA(vd_index_ida
);
27 static struct workqueue_struct
*virtblk_wq
;
29 struct virtio_blk_vq
{
32 char name
[VQ_NAME_LEN
];
33 } ____cacheline_aligned_in_smp
;
36 struct virtio_device
*vdev
;
38 /* The disk structure for the kernel. */
41 /* Block layer tags. */
42 struct blk_mq_tag_set tag_set
;
44 /* Process context for config space updates */
45 struct work_struct config_work
;
47 /* What host tells us, plus 2 for header & tailer. */
48 unsigned int sg_elems
;
50 /* Ida index - used to track minor number allocations. */
55 struct virtio_blk_vq
*vqs
;
59 #ifdef CONFIG_VIRTIO_BLK_SCSI
60 struct scsi_request sreq
; /* for SCSI passthrough, must be first */
61 u8 sense
[SCSI_SENSE_BUFFERSIZE
];
62 struct virtio_scsi_inhdr in_hdr
;
64 struct virtio_blk_outhdr out_hdr
;
66 struct scatterlist sg
[];
69 static inline blk_status_t
virtblk_result(struct virtblk_req
*vbr
)
71 switch (vbr
->status
) {
74 case VIRTIO_BLK_S_UNSUPP
:
75 return BLK_STS_NOTSUPP
;
82 * If this is a packet command we need a couple of additional headers. Behind
83 * the normal outhdr we put a segment with the scsi command block, and before
84 * the normal inhdr we put the sense data and the inhdr with additional status
87 #ifdef CONFIG_VIRTIO_BLK_SCSI
88 static int virtblk_add_req_scsi(struct virtqueue
*vq
, struct virtblk_req
*vbr
,
89 struct scatterlist
*data_sg
, bool have_data
)
91 struct scatterlist hdr
, status
, cmd
, sense
, inhdr
, *sgs
[6];
92 unsigned int num_out
= 0, num_in
= 0;
94 sg_init_one(&hdr
, &vbr
->out_hdr
, sizeof(vbr
->out_hdr
));
95 sgs
[num_out
++] = &hdr
;
96 sg_init_one(&cmd
, vbr
->sreq
.cmd
, vbr
->sreq
.cmd_len
);
97 sgs
[num_out
++] = &cmd
;
100 if (vbr
->out_hdr
.type
& cpu_to_virtio32(vq
->vdev
, VIRTIO_BLK_T_OUT
))
101 sgs
[num_out
++] = data_sg
;
103 sgs
[num_out
+ num_in
++] = data_sg
;
106 sg_init_one(&sense
, vbr
->sense
, SCSI_SENSE_BUFFERSIZE
);
107 sgs
[num_out
+ num_in
++] = &sense
;
108 sg_init_one(&inhdr
, &vbr
->in_hdr
, sizeof(vbr
->in_hdr
));
109 sgs
[num_out
+ num_in
++] = &inhdr
;
110 sg_init_one(&status
, &vbr
->status
, sizeof(vbr
->status
));
111 sgs
[num_out
+ num_in
++] = &status
;
113 return virtqueue_add_sgs(vq
, sgs
, num_out
, num_in
, vbr
, GFP_ATOMIC
);
116 static inline void virtblk_scsi_request_done(struct request
*req
)
118 struct virtblk_req
*vbr
= blk_mq_rq_to_pdu(req
);
119 struct virtio_blk
*vblk
= req
->q
->queuedata
;
120 struct scsi_request
*sreq
= &vbr
->sreq
;
122 sreq
->resid_len
= virtio32_to_cpu(vblk
->vdev
, vbr
->in_hdr
.residual
);
123 sreq
->sense_len
= virtio32_to_cpu(vblk
->vdev
, vbr
->in_hdr
.sense_len
);
124 sreq
->result
= virtio32_to_cpu(vblk
->vdev
, vbr
->in_hdr
.errors
);
127 static int virtblk_ioctl(struct block_device
*bdev
, fmode_t mode
,
128 unsigned int cmd
, unsigned long data
)
130 struct gendisk
*disk
= bdev
->bd_disk
;
131 struct virtio_blk
*vblk
= disk
->private_data
;
134 * Only allow the generic SCSI ioctls if the host can support it.
136 if (!virtio_has_feature(vblk
->vdev
, VIRTIO_BLK_F_SCSI
))
139 return scsi_cmd_blk_ioctl(bdev
, mode
, cmd
,
140 (void __user
*)data
);
143 static inline int virtblk_add_req_scsi(struct virtqueue
*vq
,
144 struct virtblk_req
*vbr
, struct scatterlist
*data_sg
,
149 static inline void virtblk_scsi_request_done(struct request
*req
)
152 #define virtblk_ioctl NULL
153 #endif /* CONFIG_VIRTIO_BLK_SCSI */
155 static int virtblk_add_req(struct virtqueue
*vq
, struct virtblk_req
*vbr
,
156 struct scatterlist
*data_sg
, bool have_data
)
158 struct scatterlist hdr
, status
, *sgs
[3];
159 unsigned int num_out
= 0, num_in
= 0;
161 sg_init_one(&hdr
, &vbr
->out_hdr
, sizeof(vbr
->out_hdr
));
162 sgs
[num_out
++] = &hdr
;
165 if (vbr
->out_hdr
.type
& cpu_to_virtio32(vq
->vdev
, VIRTIO_BLK_T_OUT
))
166 sgs
[num_out
++] = data_sg
;
168 sgs
[num_out
+ num_in
++] = data_sg
;
171 sg_init_one(&status
, &vbr
->status
, sizeof(vbr
->status
));
172 sgs
[num_out
+ num_in
++] = &status
;
174 return virtqueue_add_sgs(vq
, sgs
, num_out
, num_in
, vbr
, GFP_ATOMIC
);
177 static int virtblk_setup_discard_write_zeroes(struct request
*req
, bool unmap
)
179 unsigned short segments
= blk_rq_nr_discard_segments(req
);
180 unsigned short n
= 0;
181 struct virtio_blk_discard_write_zeroes
*range
;
186 flags
|= VIRTIO_BLK_WRITE_ZEROES_FLAG_UNMAP
;
188 range
= kmalloc_array(segments
, sizeof(*range
), GFP_ATOMIC
);
192 __rq_for_each_bio(bio
, req
) {
193 u64 sector
= bio
->bi_iter
.bi_sector
;
194 u32 num_sectors
= bio
->bi_iter
.bi_size
>> SECTOR_SHIFT
;
196 range
[n
].flags
= cpu_to_le32(flags
);
197 range
[n
].num_sectors
= cpu_to_le32(num_sectors
);
198 range
[n
].sector
= cpu_to_le64(sector
);
202 req
->special_vec
.bv_page
= virt_to_page(range
);
203 req
->special_vec
.bv_offset
= offset_in_page(range
);
204 req
->special_vec
.bv_len
= sizeof(*range
) * segments
;
205 req
->rq_flags
|= RQF_SPECIAL_PAYLOAD
;
210 static inline void virtblk_request_done(struct request
*req
)
212 struct virtblk_req
*vbr
= blk_mq_rq_to_pdu(req
);
214 if (req
->rq_flags
& RQF_SPECIAL_PAYLOAD
) {
215 kfree(page_address(req
->special_vec
.bv_page
) +
216 req
->special_vec
.bv_offset
);
219 switch (req_op(req
)) {
221 case REQ_OP_SCSI_OUT
:
222 virtblk_scsi_request_done(req
);
226 blk_mq_end_request(req
, virtblk_result(vbr
));
229 static void virtblk_done(struct virtqueue
*vq
)
231 struct virtio_blk
*vblk
= vq
->vdev
->priv
;
232 bool req_done
= false;
234 struct virtblk_req
*vbr
;
238 spin_lock_irqsave(&vblk
->vqs
[qid
].lock
, flags
);
240 virtqueue_disable_cb(vq
);
241 while ((vbr
= virtqueue_get_buf(vblk
->vqs
[qid
].vq
, &len
)) != NULL
) {
242 struct request
*req
= blk_mq_rq_from_pdu(vbr
);
244 blk_mq_complete_request(req
);
247 if (unlikely(virtqueue_is_broken(vq
)))
249 } while (!virtqueue_enable_cb(vq
));
251 /* In case queue is stopped waiting for more buffers. */
253 blk_mq_start_stopped_hw_queues(vblk
->disk
->queue
, true);
254 spin_unlock_irqrestore(&vblk
->vqs
[qid
].lock
, flags
);
257 static void virtio_commit_rqs(struct blk_mq_hw_ctx
*hctx
)
259 struct virtio_blk
*vblk
= hctx
->queue
->queuedata
;
260 struct virtio_blk_vq
*vq
= &vblk
->vqs
[hctx
->queue_num
];
263 spin_lock_irq(&vq
->lock
);
264 kick
= virtqueue_kick_prepare(vq
->vq
);
265 spin_unlock_irq(&vq
->lock
);
268 virtqueue_notify(vq
->vq
);
271 static blk_status_t
virtio_queue_rq(struct blk_mq_hw_ctx
*hctx
,
272 const struct blk_mq_queue_data
*bd
)
274 struct virtio_blk
*vblk
= hctx
->queue
->queuedata
;
275 struct request
*req
= bd
->rq
;
276 struct virtblk_req
*vbr
= blk_mq_rq_to_pdu(req
);
279 int qid
= hctx
->queue_num
;
285 BUG_ON(req
->nr_phys_segments
+ 2 > vblk
->sg_elems
);
287 switch (req_op(req
)) {
293 type
= VIRTIO_BLK_T_FLUSH
;
296 type
= VIRTIO_BLK_T_DISCARD
;
298 case REQ_OP_WRITE_ZEROES
:
299 type
= VIRTIO_BLK_T_WRITE_ZEROES
;
300 unmap
= !(req
->cmd_flags
& REQ_NOUNMAP
);
303 case REQ_OP_SCSI_OUT
:
304 type
= VIRTIO_BLK_T_SCSI_CMD
;
307 type
= VIRTIO_BLK_T_GET_ID
;
311 return BLK_STS_IOERR
;
314 vbr
->out_hdr
.type
= cpu_to_virtio32(vblk
->vdev
, type
);
315 vbr
->out_hdr
.sector
= type
?
316 0 : cpu_to_virtio64(vblk
->vdev
, blk_rq_pos(req
));
317 vbr
->out_hdr
.ioprio
= cpu_to_virtio32(vblk
->vdev
, req_get_ioprio(req
));
319 blk_mq_start_request(req
);
321 if (type
== VIRTIO_BLK_T_DISCARD
|| type
== VIRTIO_BLK_T_WRITE_ZEROES
) {
322 err
= virtblk_setup_discard_write_zeroes(req
, unmap
);
324 return BLK_STS_RESOURCE
;
327 num
= blk_rq_map_sg(hctx
->queue
, req
, vbr
->sg
);
329 if (rq_data_dir(req
) == WRITE
)
330 vbr
->out_hdr
.type
|= cpu_to_virtio32(vblk
->vdev
, VIRTIO_BLK_T_OUT
);
332 vbr
->out_hdr
.type
|= cpu_to_virtio32(vblk
->vdev
, VIRTIO_BLK_T_IN
);
335 spin_lock_irqsave(&vblk
->vqs
[qid
].lock
, flags
);
336 if (blk_rq_is_scsi(req
))
337 err
= virtblk_add_req_scsi(vblk
->vqs
[qid
].vq
, vbr
, vbr
->sg
, num
);
339 err
= virtblk_add_req(vblk
->vqs
[qid
].vq
, vbr
, vbr
->sg
, num
);
341 virtqueue_kick(vblk
->vqs
[qid
].vq
);
342 blk_mq_stop_hw_queue(hctx
);
343 spin_unlock_irqrestore(&vblk
->vqs
[qid
].lock
, flags
);
344 /* Out of mem doesn't actually happen, since we fall back
345 * to direct descriptors */
346 if (err
== -ENOMEM
|| err
== -ENOSPC
)
347 return BLK_STS_DEV_RESOURCE
;
348 return BLK_STS_IOERR
;
351 if (bd
->last
&& virtqueue_kick_prepare(vblk
->vqs
[qid
].vq
))
353 spin_unlock_irqrestore(&vblk
->vqs
[qid
].lock
, flags
);
356 virtqueue_notify(vblk
->vqs
[qid
].vq
);
360 /* return id (s/n) string for *disk to *id_str
362 static int virtblk_get_id(struct gendisk
*disk
, char *id_str
)
364 struct virtio_blk
*vblk
= disk
->private_data
;
365 struct request_queue
*q
= vblk
->disk
->queue
;
369 req
= blk_get_request(q
, REQ_OP_DRV_IN
, 0);
373 err
= blk_rq_map_kern(q
, req
, id_str
, VIRTIO_BLK_ID_BYTES
, GFP_KERNEL
);
377 blk_execute_rq(vblk
->disk
->queue
, vblk
->disk
, req
, false);
378 err
= blk_status_to_errno(virtblk_result(blk_mq_rq_to_pdu(req
)));
380 blk_put_request(req
);
384 /* We provide getgeo only to please some old bootloader/partitioning tools */
385 static int virtblk_getgeo(struct block_device
*bd
, struct hd_geometry
*geo
)
387 struct virtio_blk
*vblk
= bd
->bd_disk
->private_data
;
389 /* see if the host passed in geometry config */
390 if (virtio_has_feature(vblk
->vdev
, VIRTIO_BLK_F_GEOMETRY
)) {
391 virtio_cread(vblk
->vdev
, struct virtio_blk_config
,
392 geometry
.cylinders
, &geo
->cylinders
);
393 virtio_cread(vblk
->vdev
, struct virtio_blk_config
,
394 geometry
.heads
, &geo
->heads
);
395 virtio_cread(vblk
->vdev
, struct virtio_blk_config
,
396 geometry
.sectors
, &geo
->sectors
);
398 /* some standard values, similar to sd */
400 geo
->sectors
= 1 << 5;
401 geo
->cylinders
= get_capacity(bd
->bd_disk
) >> 11;
406 static const struct block_device_operations virtblk_fops
= {
407 .ioctl
= virtblk_ioctl
,
408 .owner
= THIS_MODULE
,
409 .getgeo
= virtblk_getgeo
,
412 static int index_to_minor(int index
)
414 return index
<< PART_BITS
;
417 static int minor_to_index(int minor
)
419 return minor
>> PART_BITS
;
422 static ssize_t
serial_show(struct device
*dev
,
423 struct device_attribute
*attr
, char *buf
)
425 struct gendisk
*disk
= dev_to_disk(dev
);
428 /* sysfs gives us a PAGE_SIZE buffer */
429 BUILD_BUG_ON(PAGE_SIZE
< VIRTIO_BLK_ID_BYTES
);
431 buf
[VIRTIO_BLK_ID_BYTES
] = '\0';
432 err
= virtblk_get_id(disk
, buf
);
436 if (err
== -EIO
) /* Unsupported? Make it empty. */
442 static DEVICE_ATTR_RO(serial
);
444 /* The queue's logical block size must be set before calling this */
445 static void virtblk_update_capacity(struct virtio_blk
*vblk
, bool resize
)
447 struct virtio_device
*vdev
= vblk
->vdev
;
448 struct request_queue
*q
= vblk
->disk
->queue
;
449 char cap_str_2
[10], cap_str_10
[10];
450 unsigned long long nblocks
;
453 /* Host must always specify the capacity. */
454 virtio_cread(vdev
, struct virtio_blk_config
, capacity
, &capacity
);
456 /* If capacity is too big, truncate with warning. */
457 if ((sector_t
)capacity
!= capacity
) {
458 dev_warn(&vdev
->dev
, "Capacity %llu too large: truncating\n",
459 (unsigned long long)capacity
);
460 capacity
= (sector_t
)-1;
463 nblocks
= DIV_ROUND_UP_ULL(capacity
, queue_logical_block_size(q
) >> 9);
465 string_get_size(nblocks
, queue_logical_block_size(q
),
466 STRING_UNITS_2
, cap_str_2
, sizeof(cap_str_2
));
467 string_get_size(nblocks
, queue_logical_block_size(q
),
468 STRING_UNITS_10
, cap_str_10
, sizeof(cap_str_10
));
470 dev_notice(&vdev
->dev
,
471 "[%s] %s%llu %d-byte logical blocks (%s/%s)\n",
472 vblk
->disk
->disk_name
,
473 resize
? "new size: " : "",
475 queue_logical_block_size(q
),
479 set_capacity(vblk
->disk
, capacity
);
482 static void virtblk_config_changed_work(struct work_struct
*work
)
484 struct virtio_blk
*vblk
=
485 container_of(work
, struct virtio_blk
, config_work
);
486 char *envp
[] = { "RESIZE=1", NULL
};
488 virtblk_update_capacity(vblk
, true);
489 revalidate_disk(vblk
->disk
);
490 kobject_uevent_env(&disk_to_dev(vblk
->disk
)->kobj
, KOBJ_CHANGE
, envp
);
493 static void virtblk_config_changed(struct virtio_device
*vdev
)
495 struct virtio_blk
*vblk
= vdev
->priv
;
497 queue_work(virtblk_wq
, &vblk
->config_work
);
500 static int init_vq(struct virtio_blk
*vblk
)
504 vq_callback_t
**callbacks
;
506 struct virtqueue
**vqs
;
507 unsigned short num_vqs
;
508 struct virtio_device
*vdev
= vblk
->vdev
;
509 struct irq_affinity desc
= { 0, };
511 err
= virtio_cread_feature(vdev
, VIRTIO_BLK_F_MQ
,
512 struct virtio_blk_config
, num_queues
,
517 num_vqs
= min_t(unsigned int, nr_cpu_ids
, num_vqs
);
519 vblk
->vqs
= kmalloc_array(num_vqs
, sizeof(*vblk
->vqs
), GFP_KERNEL
);
523 names
= kmalloc_array(num_vqs
, sizeof(*names
), GFP_KERNEL
);
524 callbacks
= kmalloc_array(num_vqs
, sizeof(*callbacks
), GFP_KERNEL
);
525 vqs
= kmalloc_array(num_vqs
, sizeof(*vqs
), GFP_KERNEL
);
526 if (!names
|| !callbacks
|| !vqs
) {
531 for (i
= 0; i
< num_vqs
; i
++) {
532 callbacks
[i
] = virtblk_done
;
533 snprintf(vblk
->vqs
[i
].name
, VQ_NAME_LEN
, "req.%d", i
);
534 names
[i
] = vblk
->vqs
[i
].name
;
537 /* Discover virtqueues and write information to configuration. */
538 err
= virtio_find_vqs(vdev
, num_vqs
, vqs
, callbacks
, names
, &desc
);
542 for (i
= 0; i
< num_vqs
; i
++) {
543 spin_lock_init(&vblk
->vqs
[i
].lock
);
544 vblk
->vqs
[i
].vq
= vqs
[i
];
546 vblk
->num_vqs
= num_vqs
;
558 * Legacy naming scheme used for virtio devices. We are stuck with it for
559 * virtio blk but don't ever use it for any new driver.
561 static int virtblk_name_format(char *prefix
, int index
, char *buf
, int buflen
)
563 const int base
= 'z' - 'a' + 1;
564 char *begin
= buf
+ strlen(prefix
);
565 char *end
= buf
+ buflen
;
575 *--p
= 'a' + (index
% unit
);
576 index
= (index
/ unit
) - 1;
577 } while (index
>= 0);
579 memmove(begin
, p
, end
- p
);
580 memcpy(buf
, prefix
, strlen(prefix
));
585 static int virtblk_get_cache_mode(struct virtio_device
*vdev
)
590 err
= virtio_cread_feature(vdev
, VIRTIO_BLK_F_CONFIG_WCE
,
591 struct virtio_blk_config
, wce
,
595 * If WCE is not configurable and flush is not available,
596 * assume no writeback cache is in use.
599 writeback
= virtio_has_feature(vdev
, VIRTIO_BLK_F_FLUSH
);
604 static void virtblk_update_cache_mode(struct virtio_device
*vdev
)
606 u8 writeback
= virtblk_get_cache_mode(vdev
);
607 struct virtio_blk
*vblk
= vdev
->priv
;
609 blk_queue_write_cache(vblk
->disk
->queue
, writeback
, false);
610 revalidate_disk(vblk
->disk
);
613 static const char *const virtblk_cache_types
[] = {
614 "write through", "write back"
618 cache_type_store(struct device
*dev
, struct device_attribute
*attr
,
619 const char *buf
, size_t count
)
621 struct gendisk
*disk
= dev_to_disk(dev
);
622 struct virtio_blk
*vblk
= disk
->private_data
;
623 struct virtio_device
*vdev
= vblk
->vdev
;
626 BUG_ON(!virtio_has_feature(vblk
->vdev
, VIRTIO_BLK_F_CONFIG_WCE
));
627 i
= sysfs_match_string(virtblk_cache_types
, buf
);
631 virtio_cwrite8(vdev
, offsetof(struct virtio_blk_config
, wce
), i
);
632 virtblk_update_cache_mode(vdev
);
637 cache_type_show(struct device
*dev
, struct device_attribute
*attr
, char *buf
)
639 struct gendisk
*disk
= dev_to_disk(dev
);
640 struct virtio_blk
*vblk
= disk
->private_data
;
641 u8 writeback
= virtblk_get_cache_mode(vblk
->vdev
);
643 BUG_ON(writeback
>= ARRAY_SIZE(virtblk_cache_types
));
644 return snprintf(buf
, 40, "%s\n", virtblk_cache_types
[writeback
]);
647 static DEVICE_ATTR_RW(cache_type
);
649 static struct attribute
*virtblk_attrs
[] = {
650 &dev_attr_serial
.attr
,
651 &dev_attr_cache_type
.attr
,
655 static umode_t
virtblk_attrs_are_visible(struct kobject
*kobj
,
656 struct attribute
*a
, int n
)
658 struct device
*dev
= container_of(kobj
, struct device
, kobj
);
659 struct gendisk
*disk
= dev_to_disk(dev
);
660 struct virtio_blk
*vblk
= disk
->private_data
;
661 struct virtio_device
*vdev
= vblk
->vdev
;
663 if (a
== &dev_attr_cache_type
.attr
&&
664 !virtio_has_feature(vdev
, VIRTIO_BLK_F_CONFIG_WCE
))
670 static const struct attribute_group virtblk_attr_group
= {
671 .attrs
= virtblk_attrs
,
672 .is_visible
= virtblk_attrs_are_visible
,
675 static const struct attribute_group
*virtblk_attr_groups
[] = {
680 static int virtblk_init_request(struct blk_mq_tag_set
*set
, struct request
*rq
,
681 unsigned int hctx_idx
, unsigned int numa_node
)
683 struct virtio_blk
*vblk
= set
->driver_data
;
684 struct virtblk_req
*vbr
= blk_mq_rq_to_pdu(rq
);
686 #ifdef CONFIG_VIRTIO_BLK_SCSI
687 vbr
->sreq
.sense
= vbr
->sense
;
689 sg_init_table(vbr
->sg
, vblk
->sg_elems
);
693 static int virtblk_map_queues(struct blk_mq_tag_set
*set
)
695 struct virtio_blk
*vblk
= set
->driver_data
;
697 return blk_mq_virtio_map_queues(&set
->map
[HCTX_TYPE_DEFAULT
],
701 #ifdef CONFIG_VIRTIO_BLK_SCSI
702 static void virtblk_initialize_rq(struct request
*req
)
704 struct virtblk_req
*vbr
= blk_mq_rq_to_pdu(req
);
706 scsi_req_init(&vbr
->sreq
);
710 static const struct blk_mq_ops virtio_mq_ops
= {
711 .queue_rq
= virtio_queue_rq
,
712 .commit_rqs
= virtio_commit_rqs
,
713 .complete
= virtblk_request_done
,
714 .init_request
= virtblk_init_request
,
715 #ifdef CONFIG_VIRTIO_BLK_SCSI
716 .initialize_rq_fn
= virtblk_initialize_rq
,
718 .map_queues
= virtblk_map_queues
,
721 static unsigned int virtblk_queue_depth
;
722 module_param_named(queue_depth
, virtblk_queue_depth
, uint
, 0444);
724 static int virtblk_probe(struct virtio_device
*vdev
)
726 struct virtio_blk
*vblk
;
727 struct request_queue
*q
;
730 u32 v
, blk_size
, max_size
, sg_elems
, opt_io_size
;
732 u8 physical_block_exp
, alignment_offset
;
734 if (!vdev
->config
->get
) {
735 dev_err(&vdev
->dev
, "%s failure: config access disabled\n",
740 err
= ida_simple_get(&vd_index_ida
, 0, minor_to_index(1 << MINORBITS
),
746 /* We need to know how many segments before we allocate. */
747 err
= virtio_cread_feature(vdev
, VIRTIO_BLK_F_SEG_MAX
,
748 struct virtio_blk_config
, seg_max
,
751 /* We need at least one SG element, whatever they say. */
752 if (err
|| !sg_elems
)
755 /* We need an extra sg elements at head and tail. */
757 vdev
->priv
= vblk
= kmalloc(sizeof(*vblk
), GFP_KERNEL
);
764 vblk
->sg_elems
= sg_elems
;
766 INIT_WORK(&vblk
->config_work
, virtblk_config_changed_work
);
772 /* FIXME: How many partitions? How long is a piece of string? */
773 vblk
->disk
= alloc_disk(1 << PART_BITS
);
779 /* Default queue sizing is to fill the ring. */
780 if (!virtblk_queue_depth
) {
781 virtblk_queue_depth
= vblk
->vqs
[0].vq
->num_free
;
782 /* ... but without indirect descs, we use 2 descs per req */
783 if (!virtio_has_feature(vdev
, VIRTIO_RING_F_INDIRECT_DESC
))
784 virtblk_queue_depth
/= 2;
787 memset(&vblk
->tag_set
, 0, sizeof(vblk
->tag_set
));
788 vblk
->tag_set
.ops
= &virtio_mq_ops
;
789 vblk
->tag_set
.queue_depth
= virtblk_queue_depth
;
790 vblk
->tag_set
.numa_node
= NUMA_NO_NODE
;
791 vblk
->tag_set
.flags
= BLK_MQ_F_SHOULD_MERGE
;
792 vblk
->tag_set
.cmd_size
=
793 sizeof(struct virtblk_req
) +
794 sizeof(struct scatterlist
) * sg_elems
;
795 vblk
->tag_set
.driver_data
= vblk
;
796 vblk
->tag_set
.nr_hw_queues
= vblk
->num_vqs
;
798 err
= blk_mq_alloc_tag_set(&vblk
->tag_set
);
802 q
= blk_mq_init_queue(&vblk
->tag_set
);
807 vblk
->disk
->queue
= q
;
811 virtblk_name_format("vd", index
, vblk
->disk
->disk_name
, DISK_NAME_LEN
);
813 vblk
->disk
->major
= major
;
814 vblk
->disk
->first_minor
= index_to_minor(index
);
815 vblk
->disk
->private_data
= vblk
;
816 vblk
->disk
->fops
= &virtblk_fops
;
817 vblk
->disk
->flags
|= GENHD_FL_EXT_DEVT
;
820 /* configure queue flush support */
821 virtblk_update_cache_mode(vdev
);
823 /* If disk is read-only in the host, the guest should obey */
824 if (virtio_has_feature(vdev
, VIRTIO_BLK_F_RO
))
825 set_disk_ro(vblk
->disk
, 1);
827 /* We can handle whatever the host told us to handle. */
828 blk_queue_max_segments(q
, vblk
->sg_elems
-2);
830 /* No real sector limit. */
831 blk_queue_max_hw_sectors(q
, -1U);
833 max_size
= virtio_max_dma_size(vdev
);
835 /* Host can optionally specify maximum segment size and number of
837 err
= virtio_cread_feature(vdev
, VIRTIO_BLK_F_SIZE_MAX
,
838 struct virtio_blk_config
, size_max
, &v
);
840 max_size
= min(max_size
, v
);
842 blk_queue_max_segment_size(q
, max_size
);
844 /* Host can optionally specify the block size of the device */
845 err
= virtio_cread_feature(vdev
, VIRTIO_BLK_F_BLK_SIZE
,
846 struct virtio_blk_config
, blk_size
,
849 blk_queue_logical_block_size(q
, blk_size
);
851 blk_size
= queue_logical_block_size(q
);
853 /* Use topology information if available */
854 err
= virtio_cread_feature(vdev
, VIRTIO_BLK_F_TOPOLOGY
,
855 struct virtio_blk_config
, physical_block_exp
,
856 &physical_block_exp
);
857 if (!err
&& physical_block_exp
)
858 blk_queue_physical_block_size(q
,
859 blk_size
* (1 << physical_block_exp
));
861 err
= virtio_cread_feature(vdev
, VIRTIO_BLK_F_TOPOLOGY
,
862 struct virtio_blk_config
, alignment_offset
,
864 if (!err
&& alignment_offset
)
865 blk_queue_alignment_offset(q
, blk_size
* alignment_offset
);
867 err
= virtio_cread_feature(vdev
, VIRTIO_BLK_F_TOPOLOGY
,
868 struct virtio_blk_config
, min_io_size
,
870 if (!err
&& min_io_size
)
871 blk_queue_io_min(q
, blk_size
* min_io_size
);
873 err
= virtio_cread_feature(vdev
, VIRTIO_BLK_F_TOPOLOGY
,
874 struct virtio_blk_config
, opt_io_size
,
876 if (!err
&& opt_io_size
)
877 blk_queue_io_opt(q
, blk_size
* opt_io_size
);
879 if (virtio_has_feature(vdev
, VIRTIO_BLK_F_DISCARD
)) {
880 q
->limits
.discard_granularity
= blk_size
;
882 virtio_cread(vdev
, struct virtio_blk_config
,
883 discard_sector_alignment
, &v
);
884 q
->limits
.discard_alignment
= v
? v
<< SECTOR_SHIFT
: 0;
886 virtio_cread(vdev
, struct virtio_blk_config
,
887 max_discard_sectors
, &v
);
888 blk_queue_max_discard_sectors(q
, v
? v
: UINT_MAX
);
890 virtio_cread(vdev
, struct virtio_blk_config
, max_discard_seg
,
892 blk_queue_max_discard_segments(q
,
894 MAX_DISCARD_SEGMENTS
));
896 blk_queue_flag_set(QUEUE_FLAG_DISCARD
, q
);
899 if (virtio_has_feature(vdev
, VIRTIO_BLK_F_WRITE_ZEROES
)) {
900 virtio_cread(vdev
, struct virtio_blk_config
,
901 max_write_zeroes_sectors
, &v
);
902 blk_queue_max_write_zeroes_sectors(q
, v
? v
: UINT_MAX
);
905 virtblk_update_capacity(vblk
, false);
906 virtio_device_ready(vdev
);
908 device_add_disk(&vdev
->dev
, vblk
->disk
, virtblk_attr_groups
);
912 blk_mq_free_tag_set(&vblk
->tag_set
);
914 put_disk(vblk
->disk
);
916 vdev
->config
->del_vqs(vdev
);
920 ida_simple_remove(&vd_index_ida
, index
);
925 static void virtblk_remove(struct virtio_device
*vdev
)
927 struct virtio_blk
*vblk
= vdev
->priv
;
928 int index
= vblk
->index
;
931 /* Make sure no work handler is accessing the device. */
932 flush_work(&vblk
->config_work
);
934 del_gendisk(vblk
->disk
);
935 blk_cleanup_queue(vblk
->disk
->queue
);
937 blk_mq_free_tag_set(&vblk
->tag_set
);
939 /* Stop all the virtqueues. */
940 vdev
->config
->reset(vdev
);
942 refc
= kref_read(&disk_to_dev(vblk
->disk
)->kobj
.kref
);
943 put_disk(vblk
->disk
);
944 vdev
->config
->del_vqs(vdev
);
948 /* Only free device id if we don't have any users */
950 ida_simple_remove(&vd_index_ida
, index
);
953 #ifdef CONFIG_PM_SLEEP
954 static int virtblk_freeze(struct virtio_device
*vdev
)
956 struct virtio_blk
*vblk
= vdev
->priv
;
958 /* Ensure we don't receive any more interrupts */
959 vdev
->config
->reset(vdev
);
961 /* Make sure no work handler is accessing the device. */
962 flush_work(&vblk
->config_work
);
964 blk_mq_quiesce_queue(vblk
->disk
->queue
);
966 vdev
->config
->del_vqs(vdev
);
970 static int virtblk_restore(struct virtio_device
*vdev
)
972 struct virtio_blk
*vblk
= vdev
->priv
;
975 ret
= init_vq(vdev
->priv
);
979 virtio_device_ready(vdev
);
981 blk_mq_unquiesce_queue(vblk
->disk
->queue
);
986 static const struct virtio_device_id id_table
[] = {
987 { VIRTIO_ID_BLOCK
, VIRTIO_DEV_ANY_ID
},
991 static unsigned int features_legacy
[] = {
992 VIRTIO_BLK_F_SEG_MAX
, VIRTIO_BLK_F_SIZE_MAX
, VIRTIO_BLK_F_GEOMETRY
,
993 VIRTIO_BLK_F_RO
, VIRTIO_BLK_F_BLK_SIZE
,
994 #ifdef CONFIG_VIRTIO_BLK_SCSI
997 VIRTIO_BLK_F_FLUSH
, VIRTIO_BLK_F_TOPOLOGY
, VIRTIO_BLK_F_CONFIG_WCE
,
998 VIRTIO_BLK_F_MQ
, VIRTIO_BLK_F_DISCARD
, VIRTIO_BLK_F_WRITE_ZEROES
,
1001 static unsigned int features
[] = {
1002 VIRTIO_BLK_F_SEG_MAX
, VIRTIO_BLK_F_SIZE_MAX
, VIRTIO_BLK_F_GEOMETRY
,
1003 VIRTIO_BLK_F_RO
, VIRTIO_BLK_F_BLK_SIZE
,
1004 VIRTIO_BLK_F_FLUSH
, VIRTIO_BLK_F_TOPOLOGY
, VIRTIO_BLK_F_CONFIG_WCE
,
1005 VIRTIO_BLK_F_MQ
, VIRTIO_BLK_F_DISCARD
, VIRTIO_BLK_F_WRITE_ZEROES
,
1008 static struct virtio_driver virtio_blk
= {
1009 .feature_table
= features
,
1010 .feature_table_size
= ARRAY_SIZE(features
),
1011 .feature_table_legacy
= features_legacy
,
1012 .feature_table_size_legacy
= ARRAY_SIZE(features_legacy
),
1013 .driver
.name
= KBUILD_MODNAME
,
1014 .driver
.owner
= THIS_MODULE
,
1015 .id_table
= id_table
,
1016 .probe
= virtblk_probe
,
1017 .remove
= virtblk_remove
,
1018 .config_changed
= virtblk_config_changed
,
1019 #ifdef CONFIG_PM_SLEEP
1020 .freeze
= virtblk_freeze
,
1021 .restore
= virtblk_restore
,
1025 static int __init
init(void)
1029 virtblk_wq
= alloc_workqueue("virtio-blk", 0, 0);
1033 major
= register_blkdev(0, "virtblk");
1036 goto out_destroy_workqueue
;
1039 error
= register_virtio_driver(&virtio_blk
);
1041 goto out_unregister_blkdev
;
1044 out_unregister_blkdev
:
1045 unregister_blkdev(major
, "virtblk");
1046 out_destroy_workqueue
:
1047 destroy_workqueue(virtblk_wq
);
1051 static void __exit
fini(void)
1053 unregister_virtio_driver(&virtio_blk
);
1054 unregister_blkdev(major
, "virtblk");
1055 destroy_workqueue(virtblk_wq
);
1060 MODULE_DEVICE_TABLE(virtio
, id_table
);
1061 MODULE_DESCRIPTION("Virtio block driver");
1062 MODULE_LICENSE("GPL");