2 * Virtio SCSI HBA driver
4 * Copyright IBM Corp. 2010
5 * Copyright Red Hat, Inc. 2011
8 * Stefan Hajnoczi <stefanha@linux.vnet.ibm.com>
9 * Paolo Bonzini <pbonzini@redhat.com>
11 * This work is licensed under the terms of the GNU GPL, version 2 or later.
12 * See the COPYING file in the top-level directory.
16 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
18 #include <linux/module.h>
19 #include <linux/slab.h>
20 #include <linux/mempool.h>
21 #include <linux/interrupt.h>
22 #include <linux/virtio.h>
23 #include <linux/virtio_ids.h>
24 #include <linux/virtio_config.h>
25 #include <linux/virtio_scsi.h>
26 #include <linux/cpu.h>
27 #include <linux/blkdev.h>
28 #include <scsi/scsi_host.h>
29 #include <scsi/scsi_device.h>
30 #include <scsi/scsi_cmnd.h>
31 #include <scsi/scsi_tcq.h>
32 #include <scsi/scsi_devinfo.h>
33 #include <linux/seqlock.h>
34 #include <linux/blk-mq-virtio.h>
36 #define VIRTIO_SCSI_MEMPOOL_SZ 64
37 #define VIRTIO_SCSI_EVENT_LEN 8
38 #define VIRTIO_SCSI_VQ_BASE 2
40 /* Command queue element */
41 struct virtio_scsi_cmd
{
43 struct completion
*comp
;
45 struct virtio_scsi_cmd_req cmd
;
46 struct virtio_scsi_cmd_req_pi cmd_pi
;
47 struct virtio_scsi_ctrl_tmf_req tmf
;
48 struct virtio_scsi_ctrl_an_req an
;
51 struct virtio_scsi_cmd_resp cmd
;
52 struct virtio_scsi_ctrl_tmf_resp tmf
;
53 struct virtio_scsi_ctrl_an_resp an
;
54 struct virtio_scsi_event evt
;
56 } ____cacheline_aligned_in_smp
;
58 struct virtio_scsi_event_node
{
59 struct virtio_scsi
*vscsi
;
60 struct virtio_scsi_event event
;
61 struct work_struct work
;
64 struct virtio_scsi_vq
{
71 /* Driver instance state */
73 struct virtio_device
*vdev
;
75 /* Get some buffers ready for event vq */
76 struct virtio_scsi_event_node event_list
[VIRTIO_SCSI_EVENT_LEN
];
80 /* If the affinity hint is set for virtqueues */
81 bool affinity_hint_set
;
83 struct hlist_node node
;
85 /* Protected by event_vq lock */
88 struct virtio_scsi_vq ctrl_vq
;
89 struct virtio_scsi_vq event_vq
;
90 struct virtio_scsi_vq req_vqs
[];
93 static struct kmem_cache
*virtscsi_cmd_cache
;
94 static mempool_t
*virtscsi_cmd_pool
;
96 static inline struct Scsi_Host
*virtio_scsi_host(struct virtio_device
*vdev
)
101 static void virtscsi_compute_resid(struct scsi_cmnd
*sc
, u32 resid
)
106 if (!scsi_bidi_cmnd(sc
)) {
107 scsi_set_resid(sc
, resid
);
111 scsi_in(sc
)->resid
= min(resid
, scsi_in(sc
)->length
);
112 scsi_out(sc
)->resid
= resid
- scsi_in(sc
)->resid
;
116 * virtscsi_complete_cmd - finish a scsi_cmd and invoke scsi_done
118 * Called with vq_lock held.
120 static void virtscsi_complete_cmd(struct virtio_scsi
*vscsi
, void *buf
)
122 struct virtio_scsi_cmd
*cmd
= buf
;
123 struct scsi_cmnd
*sc
= cmd
->sc
;
124 struct virtio_scsi_cmd_resp
*resp
= &cmd
->resp
.cmd
;
126 dev_dbg(&sc
->device
->sdev_gendev
,
127 "cmd %p response %u status %#02x sense_len %u\n",
128 sc
, resp
->response
, resp
->status
, resp
->sense_len
);
130 sc
->result
= resp
->status
;
131 virtscsi_compute_resid(sc
, virtio32_to_cpu(vscsi
->vdev
, resp
->resid
));
132 switch (resp
->response
) {
133 case VIRTIO_SCSI_S_OK
:
134 set_host_byte(sc
, DID_OK
);
136 case VIRTIO_SCSI_S_OVERRUN
:
137 set_host_byte(sc
, DID_ERROR
);
139 case VIRTIO_SCSI_S_ABORTED
:
140 set_host_byte(sc
, DID_ABORT
);
142 case VIRTIO_SCSI_S_BAD_TARGET
:
143 set_host_byte(sc
, DID_BAD_TARGET
);
145 case VIRTIO_SCSI_S_RESET
:
146 set_host_byte(sc
, DID_RESET
);
148 case VIRTIO_SCSI_S_BUSY
:
149 set_host_byte(sc
, DID_BUS_BUSY
);
151 case VIRTIO_SCSI_S_TRANSPORT_FAILURE
:
152 set_host_byte(sc
, DID_TRANSPORT_DISRUPTED
);
154 case VIRTIO_SCSI_S_TARGET_FAILURE
:
155 set_host_byte(sc
, DID_TARGET_FAILURE
);
157 case VIRTIO_SCSI_S_NEXUS_FAILURE
:
158 set_host_byte(sc
, DID_NEXUS_FAILURE
);
161 scmd_printk(KERN_WARNING
, sc
, "Unknown response %d",
164 case VIRTIO_SCSI_S_FAILURE
:
165 set_host_byte(sc
, DID_ERROR
);
169 WARN_ON(virtio32_to_cpu(vscsi
->vdev
, resp
->sense_len
) >
170 VIRTIO_SCSI_SENSE_SIZE
);
171 if (sc
->sense_buffer
) {
172 memcpy(sc
->sense_buffer
, resp
->sense
,
174 virtio32_to_cpu(vscsi
->vdev
, resp
->sense_len
),
175 VIRTIO_SCSI_SENSE_SIZE
));
177 set_driver_byte(sc
, DRIVER_SENSE
);
183 static void virtscsi_vq_done(struct virtio_scsi
*vscsi
,
184 struct virtio_scsi_vq
*virtscsi_vq
,
185 void (*fn
)(struct virtio_scsi
*vscsi
, void *buf
))
190 struct virtqueue
*vq
= virtscsi_vq
->vq
;
192 spin_lock_irqsave(&virtscsi_vq
->vq_lock
, flags
);
194 virtqueue_disable_cb(vq
);
195 while ((buf
= virtqueue_get_buf(vq
, &len
)) != NULL
)
198 if (unlikely(virtqueue_is_broken(vq
)))
200 } while (!virtqueue_enable_cb(vq
));
201 spin_unlock_irqrestore(&virtscsi_vq
->vq_lock
, flags
);
204 static void virtscsi_req_done(struct virtqueue
*vq
)
206 struct Scsi_Host
*sh
= virtio_scsi_host(vq
->vdev
);
207 struct virtio_scsi
*vscsi
= shost_priv(sh
);
208 int index
= vq
->index
- VIRTIO_SCSI_VQ_BASE
;
209 struct virtio_scsi_vq
*req_vq
= &vscsi
->req_vqs
[index
];
211 virtscsi_vq_done(vscsi
, req_vq
, virtscsi_complete_cmd
);
214 static void virtscsi_poll_requests(struct virtio_scsi
*vscsi
)
218 num_vqs
= vscsi
->num_queues
;
219 for (i
= 0; i
< num_vqs
; i
++)
220 virtscsi_vq_done(vscsi
, &vscsi
->req_vqs
[i
],
221 virtscsi_complete_cmd
);
224 static void virtscsi_complete_free(struct virtio_scsi
*vscsi
, void *buf
)
226 struct virtio_scsi_cmd
*cmd
= buf
;
232 static void virtscsi_ctrl_done(struct virtqueue
*vq
)
234 struct Scsi_Host
*sh
= virtio_scsi_host(vq
->vdev
);
235 struct virtio_scsi
*vscsi
= shost_priv(sh
);
237 virtscsi_vq_done(vscsi
, &vscsi
->ctrl_vq
, virtscsi_complete_free
);
240 static void virtscsi_handle_event(struct work_struct
*work
);
242 static int virtscsi_kick_event(struct virtio_scsi
*vscsi
,
243 struct virtio_scsi_event_node
*event_node
)
246 struct scatterlist sg
;
249 INIT_WORK(&event_node
->work
, virtscsi_handle_event
);
250 sg_init_one(&sg
, &event_node
->event
, sizeof(struct virtio_scsi_event
));
252 spin_lock_irqsave(&vscsi
->event_vq
.vq_lock
, flags
);
254 err
= virtqueue_add_inbuf(vscsi
->event_vq
.vq
, &sg
, 1, event_node
,
257 virtqueue_kick(vscsi
->event_vq
.vq
);
259 spin_unlock_irqrestore(&vscsi
->event_vq
.vq_lock
, flags
);
264 static int virtscsi_kick_event_all(struct virtio_scsi
*vscsi
)
268 for (i
= 0; i
< VIRTIO_SCSI_EVENT_LEN
; i
++) {
269 vscsi
->event_list
[i
].vscsi
= vscsi
;
270 virtscsi_kick_event(vscsi
, &vscsi
->event_list
[i
]);
276 static void virtscsi_cancel_event_work(struct virtio_scsi
*vscsi
)
280 /* Stop scheduling work before calling cancel_work_sync. */
281 spin_lock_irq(&vscsi
->event_vq
.vq_lock
);
282 vscsi
->stop_events
= true;
283 spin_unlock_irq(&vscsi
->event_vq
.vq_lock
);
285 for (i
= 0; i
< VIRTIO_SCSI_EVENT_LEN
; i
++)
286 cancel_work_sync(&vscsi
->event_list
[i
].work
);
289 static void virtscsi_handle_transport_reset(struct virtio_scsi
*vscsi
,
290 struct virtio_scsi_event
*event
)
292 struct scsi_device
*sdev
;
293 struct Scsi_Host
*shost
= virtio_scsi_host(vscsi
->vdev
);
294 unsigned int target
= event
->lun
[1];
295 unsigned int lun
= (event
->lun
[2] << 8) | event
->lun
[3];
297 switch (virtio32_to_cpu(vscsi
->vdev
, event
->reason
)) {
298 case VIRTIO_SCSI_EVT_RESET_RESCAN
:
299 scsi_add_device(shost
, 0, target
, lun
);
301 case VIRTIO_SCSI_EVT_RESET_REMOVED
:
302 sdev
= scsi_device_lookup(shost
, 0, target
, lun
);
304 scsi_remove_device(sdev
);
305 scsi_device_put(sdev
);
307 pr_err("SCSI device %d 0 %d %d not found\n",
308 shost
->host_no
, target
, lun
);
312 pr_info("Unsupport virtio scsi event reason %x\n", event
->reason
);
316 static void virtscsi_handle_param_change(struct virtio_scsi
*vscsi
,
317 struct virtio_scsi_event
*event
)
319 struct scsi_device
*sdev
;
320 struct Scsi_Host
*shost
= virtio_scsi_host(vscsi
->vdev
);
321 unsigned int target
= event
->lun
[1];
322 unsigned int lun
= (event
->lun
[2] << 8) | event
->lun
[3];
323 u8 asc
= virtio32_to_cpu(vscsi
->vdev
, event
->reason
) & 255;
324 u8 ascq
= virtio32_to_cpu(vscsi
->vdev
, event
->reason
) >> 8;
326 sdev
= scsi_device_lookup(shost
, 0, target
, lun
);
328 pr_err("SCSI device %d 0 %d %d not found\n",
329 shost
->host_no
, target
, lun
);
333 /* Handle "Parameters changed", "Mode parameters changed", and
334 "Capacity data has changed". */
335 if (asc
== 0x2a && (ascq
== 0x00 || ascq
== 0x01 || ascq
== 0x09))
336 scsi_rescan_device(&sdev
->sdev_gendev
);
338 scsi_device_put(sdev
);
341 static void virtscsi_handle_event(struct work_struct
*work
)
343 struct virtio_scsi_event_node
*event_node
=
344 container_of(work
, struct virtio_scsi_event_node
, work
);
345 struct virtio_scsi
*vscsi
= event_node
->vscsi
;
346 struct virtio_scsi_event
*event
= &event_node
->event
;
349 cpu_to_virtio32(vscsi
->vdev
, VIRTIO_SCSI_T_EVENTS_MISSED
)) {
350 event
->event
&= ~cpu_to_virtio32(vscsi
->vdev
,
351 VIRTIO_SCSI_T_EVENTS_MISSED
);
352 scsi_scan_host(virtio_scsi_host(vscsi
->vdev
));
355 switch (virtio32_to_cpu(vscsi
->vdev
, event
->event
)) {
356 case VIRTIO_SCSI_T_NO_EVENT
:
358 case VIRTIO_SCSI_T_TRANSPORT_RESET
:
359 virtscsi_handle_transport_reset(vscsi
, event
);
361 case VIRTIO_SCSI_T_PARAM_CHANGE
:
362 virtscsi_handle_param_change(vscsi
, event
);
365 pr_err("Unsupport virtio scsi event %x\n", event
->event
);
367 virtscsi_kick_event(vscsi
, event_node
);
370 static void virtscsi_complete_event(struct virtio_scsi
*vscsi
, void *buf
)
372 struct virtio_scsi_event_node
*event_node
= buf
;
374 if (!vscsi
->stop_events
)
375 queue_work(system_freezable_wq
, &event_node
->work
);
378 static void virtscsi_event_done(struct virtqueue
*vq
)
380 struct Scsi_Host
*sh
= virtio_scsi_host(vq
->vdev
);
381 struct virtio_scsi
*vscsi
= shost_priv(sh
);
383 virtscsi_vq_done(vscsi
, &vscsi
->event_vq
, virtscsi_complete_event
);
387 * virtscsi_add_cmd - add a virtio_scsi_cmd to a virtqueue
388 * @vq : the struct virtqueue we're talking about
389 * @cmd : command structure
390 * @req_size : size of the request buffer
391 * @resp_size : size of the response buffer
393 static int virtscsi_add_cmd(struct virtqueue
*vq
,
394 struct virtio_scsi_cmd
*cmd
,
395 size_t req_size
, size_t resp_size
)
397 struct scsi_cmnd
*sc
= cmd
->sc
;
398 struct scatterlist
*sgs
[6], req
, resp
;
399 struct sg_table
*out
, *in
;
400 unsigned out_num
= 0, in_num
= 0;
404 if (sc
&& sc
->sc_data_direction
!= DMA_NONE
) {
405 if (sc
->sc_data_direction
!= DMA_FROM_DEVICE
)
406 out
= &scsi_out(sc
)->table
;
407 if (sc
->sc_data_direction
!= DMA_TO_DEVICE
)
408 in
= &scsi_in(sc
)->table
;
411 /* Request header. */
412 sg_init_one(&req
, &cmd
->req
, req_size
);
413 sgs
[out_num
++] = &req
;
415 /* Data-out buffer. */
417 /* Place WRITE protection SGLs before Data OUT payload */
418 if (scsi_prot_sg_count(sc
))
419 sgs
[out_num
++] = scsi_prot_sglist(sc
);
420 sgs
[out_num
++] = out
->sgl
;
423 /* Response header. */
424 sg_init_one(&resp
, &cmd
->resp
, resp_size
);
425 sgs
[out_num
+ in_num
++] = &resp
;
429 /* Place READ protection SGLs before Data IN payload */
430 if (scsi_prot_sg_count(sc
))
431 sgs
[out_num
+ in_num
++] = scsi_prot_sglist(sc
);
432 sgs
[out_num
+ in_num
++] = in
->sgl
;
435 return virtqueue_add_sgs(vq
, sgs
, out_num
, in_num
, cmd
, GFP_ATOMIC
);
438 static int virtscsi_kick_cmd(struct virtio_scsi_vq
*vq
,
439 struct virtio_scsi_cmd
*cmd
,
440 size_t req_size
, size_t resp_size
)
444 bool needs_kick
= false;
446 spin_lock_irqsave(&vq
->vq_lock
, flags
);
447 err
= virtscsi_add_cmd(vq
->vq
, cmd
, req_size
, resp_size
);
449 needs_kick
= virtqueue_kick_prepare(vq
->vq
);
451 spin_unlock_irqrestore(&vq
->vq_lock
, flags
);
454 virtqueue_notify(vq
->vq
);
458 static void virtio_scsi_init_hdr(struct virtio_device
*vdev
,
459 struct virtio_scsi_cmd_req
*cmd
,
460 struct scsi_cmnd
*sc
)
463 cmd
->lun
[1] = sc
->device
->id
;
464 cmd
->lun
[2] = (sc
->device
->lun
>> 8) | 0x40;
465 cmd
->lun
[3] = sc
->device
->lun
& 0xff;
466 cmd
->tag
= cpu_to_virtio64(vdev
, (unsigned long)sc
);
467 cmd
->task_attr
= VIRTIO_SCSI_S_SIMPLE
;
472 #ifdef CONFIG_BLK_DEV_INTEGRITY
473 static void virtio_scsi_init_hdr_pi(struct virtio_device
*vdev
,
474 struct virtio_scsi_cmd_req_pi
*cmd_pi
,
475 struct scsi_cmnd
*sc
)
477 struct request
*rq
= sc
->request
;
478 struct blk_integrity
*bi
;
480 virtio_scsi_init_hdr(vdev
, (struct virtio_scsi_cmd_req
*)cmd_pi
, sc
);
482 if (!rq
|| !scsi_prot_sg_count(sc
))
485 bi
= blk_get_integrity(rq
->rq_disk
);
487 if (sc
->sc_data_direction
== DMA_TO_DEVICE
)
488 cmd_pi
->pi_bytesout
= cpu_to_virtio32(vdev
,
489 bio_integrity_bytes(bi
,
490 blk_rq_sectors(rq
)));
491 else if (sc
->sc_data_direction
== DMA_FROM_DEVICE
)
492 cmd_pi
->pi_bytesin
= cpu_to_virtio32(vdev
,
493 bio_integrity_bytes(bi
,
494 blk_rq_sectors(rq
)));
498 static struct virtio_scsi_vq
*virtscsi_pick_vq_mq(struct virtio_scsi
*vscsi
,
499 struct scsi_cmnd
*sc
)
501 u32 tag
= blk_mq_unique_tag(sc
->request
);
502 u16 hwq
= blk_mq_unique_tag_to_hwq(tag
);
504 return &vscsi
->req_vqs
[hwq
];
507 static int virtscsi_queuecommand(struct Scsi_Host
*shost
,
508 struct scsi_cmnd
*sc
)
510 struct virtio_scsi
*vscsi
= shost_priv(shost
);
511 struct virtio_scsi_vq
*req_vq
= virtscsi_pick_vq_mq(vscsi
, sc
);
512 struct virtio_scsi_cmd
*cmd
= scsi_cmd_priv(sc
);
517 BUG_ON(scsi_sg_count(sc
) > shost
->sg_tablesize
);
519 /* TODO: check feature bit and fail if unsupported? */
520 BUG_ON(sc
->sc_data_direction
== DMA_BIDIRECTIONAL
);
522 dev_dbg(&sc
->device
->sdev_gendev
,
523 "cmd %p CDB: %#02x\n", sc
, sc
->cmnd
[0]);
527 BUG_ON(sc
->cmd_len
> VIRTIO_SCSI_CDB_SIZE
);
529 #ifdef CONFIG_BLK_DEV_INTEGRITY
530 if (virtio_has_feature(vscsi
->vdev
, VIRTIO_SCSI_F_T10_PI
)) {
531 virtio_scsi_init_hdr_pi(vscsi
->vdev
, &cmd
->req
.cmd_pi
, sc
);
532 memcpy(cmd
->req
.cmd_pi
.cdb
, sc
->cmnd
, sc
->cmd_len
);
533 req_size
= sizeof(cmd
->req
.cmd_pi
);
537 virtio_scsi_init_hdr(vscsi
->vdev
, &cmd
->req
.cmd
, sc
);
538 memcpy(cmd
->req
.cmd
.cdb
, sc
->cmnd
, sc
->cmd_len
);
539 req_size
= sizeof(cmd
->req
.cmd
);
542 ret
= virtscsi_kick_cmd(req_vq
, cmd
, req_size
, sizeof(cmd
->resp
.cmd
));
544 cmd
->resp
.cmd
.response
= VIRTIO_SCSI_S_BAD_TARGET
;
545 spin_lock_irqsave(&req_vq
->vq_lock
, flags
);
546 virtscsi_complete_cmd(vscsi
, cmd
);
547 spin_unlock_irqrestore(&req_vq
->vq_lock
, flags
);
548 } else if (ret
!= 0) {
549 return SCSI_MLQUEUE_HOST_BUSY
;
554 static int virtscsi_tmf(struct virtio_scsi
*vscsi
, struct virtio_scsi_cmd
*cmd
)
556 DECLARE_COMPLETION_ONSTACK(comp
);
560 if (virtscsi_kick_cmd(&vscsi
->ctrl_vq
, cmd
,
561 sizeof cmd
->req
.tmf
, sizeof cmd
->resp
.tmf
) < 0)
564 wait_for_completion(&comp
);
565 if (cmd
->resp
.tmf
.response
== VIRTIO_SCSI_S_OK
||
566 cmd
->resp
.tmf
.response
== VIRTIO_SCSI_S_FUNCTION_SUCCEEDED
)
570 * The spec guarantees that all requests related to the TMF have
571 * been completed, but the callback might not have run yet if
572 * we're using independent interrupts (e.g. MSI). Poll the
575 * In the abort case, sc->scsi_done will do nothing, because
576 * the block layer must have detected a timeout and as a result
577 * REQ_ATOM_COMPLETE has been set.
579 virtscsi_poll_requests(vscsi
);
582 mempool_free(cmd
, virtscsi_cmd_pool
);
586 static int virtscsi_device_reset(struct scsi_cmnd
*sc
)
588 struct virtio_scsi
*vscsi
= shost_priv(sc
->device
->host
);
589 struct virtio_scsi_cmd
*cmd
;
591 sdev_printk(KERN_INFO
, sc
->device
, "device reset\n");
592 cmd
= mempool_alloc(virtscsi_cmd_pool
, GFP_NOIO
);
596 memset(cmd
, 0, sizeof(*cmd
));
598 cmd
->req
.tmf
= (struct virtio_scsi_ctrl_tmf_req
){
599 .type
= VIRTIO_SCSI_T_TMF
,
600 .subtype
= cpu_to_virtio32(vscsi
->vdev
,
601 VIRTIO_SCSI_T_TMF_LOGICAL_UNIT_RESET
),
603 .lun
[1] = sc
->device
->id
,
604 .lun
[2] = (sc
->device
->lun
>> 8) | 0x40,
605 .lun
[3] = sc
->device
->lun
& 0xff,
607 return virtscsi_tmf(vscsi
, cmd
);
610 static int virtscsi_device_alloc(struct scsi_device
*sdevice
)
613 * Passed through SCSI targets (e.g. with qemu's 'scsi-block')
614 * may have transfer limits which come from the host SCSI
615 * controller or something on the host side other than the
618 * To make this work properly, the hypervisor can adjust the
619 * target's VPD information to advertise these limits. But
620 * for that to work, the guest has to look at the VPD pages,
621 * which we won't do by default if it is an SPC-2 device, even
622 * if it does actually support it.
624 * So, set the blist to always try to read the VPD pages.
626 sdevice
->sdev_bflags
= BLIST_TRY_VPD_PAGES
;
633 * virtscsi_change_queue_depth() - Change a virtscsi target's queue depth
634 * @sdev: Virtscsi target whose queue depth to change
635 * @qdepth: New queue depth
637 static int virtscsi_change_queue_depth(struct scsi_device
*sdev
, int qdepth
)
639 struct Scsi_Host
*shost
= sdev
->host
;
640 int max_depth
= shost
->cmd_per_lun
;
642 return scsi_change_queue_depth(sdev
, min(max_depth
, qdepth
));
645 static int virtscsi_abort(struct scsi_cmnd
*sc
)
647 struct virtio_scsi
*vscsi
= shost_priv(sc
->device
->host
);
648 struct virtio_scsi_cmd
*cmd
;
650 scmd_printk(KERN_INFO
, sc
, "abort\n");
651 cmd
= mempool_alloc(virtscsi_cmd_pool
, GFP_NOIO
);
655 memset(cmd
, 0, sizeof(*cmd
));
657 cmd
->req
.tmf
= (struct virtio_scsi_ctrl_tmf_req
){
658 .type
= VIRTIO_SCSI_T_TMF
,
659 .subtype
= VIRTIO_SCSI_T_TMF_ABORT_TASK
,
661 .lun
[1] = sc
->device
->id
,
662 .lun
[2] = (sc
->device
->lun
>> 8) | 0x40,
663 .lun
[3] = sc
->device
->lun
& 0xff,
664 .tag
= cpu_to_virtio64(vscsi
->vdev
, (unsigned long)sc
),
666 return virtscsi_tmf(vscsi
, cmd
);
669 static int virtscsi_map_queues(struct Scsi_Host
*shost
)
671 struct virtio_scsi
*vscsi
= shost_priv(shost
);
672 struct blk_mq_queue_map
*qmap
= &shost
->tag_set
.map
[0];
674 return blk_mq_virtio_map_queues(qmap
, vscsi
->vdev
, 2);
678 * The host guarantees to respond to each command, although I/O
679 * latencies might be higher than on bare metal. Reset the timer
680 * unconditionally to give the host a chance to perform EH.
682 static enum blk_eh_timer_return
virtscsi_eh_timed_out(struct scsi_cmnd
*scmnd
)
684 return BLK_EH_RESET_TIMER
;
687 static struct scsi_host_template virtscsi_host_template
= {
688 .module
= THIS_MODULE
,
689 .name
= "Virtio SCSI HBA",
690 .proc_name
= "virtio_scsi",
692 .cmd_size
= sizeof(struct virtio_scsi_cmd
),
693 .queuecommand
= virtscsi_queuecommand
,
694 .change_queue_depth
= virtscsi_change_queue_depth
,
695 .eh_abort_handler
= virtscsi_abort
,
696 .eh_device_reset_handler
= virtscsi_device_reset
,
697 .eh_timed_out
= virtscsi_eh_timed_out
,
698 .slave_alloc
= virtscsi_device_alloc
,
700 .dma_boundary
= UINT_MAX
,
701 .map_queues
= virtscsi_map_queues
,
702 .track_queue_depth
= 1,
706 #define virtscsi_config_get(vdev, fld) \
708 typeof(((struct virtio_scsi_config *)0)->fld) __val; \
709 virtio_cread(vdev, struct virtio_scsi_config, fld, &__val); \
713 #define virtscsi_config_set(vdev, fld, val) \
715 typeof(((struct virtio_scsi_config *)0)->fld) __val = (val); \
716 virtio_cwrite(vdev, struct virtio_scsi_config, fld, &__val); \
719 static void virtscsi_init_vq(struct virtio_scsi_vq
*virtscsi_vq
,
720 struct virtqueue
*vq
)
722 spin_lock_init(&virtscsi_vq
->vq_lock
);
723 virtscsi_vq
->vq
= vq
;
726 static void virtscsi_remove_vqs(struct virtio_device
*vdev
)
728 /* Stop all the virtqueues. */
729 vdev
->config
->reset(vdev
);
730 vdev
->config
->del_vqs(vdev
);
733 static int virtscsi_init(struct virtio_device
*vdev
,
734 struct virtio_scsi
*vscsi
)
739 vq_callback_t
**callbacks
;
741 struct virtqueue
**vqs
;
742 struct irq_affinity desc
= { .pre_vectors
= 2 };
744 num_vqs
= vscsi
->num_queues
+ VIRTIO_SCSI_VQ_BASE
;
745 vqs
= kmalloc_array(num_vqs
, sizeof(struct virtqueue
*), GFP_KERNEL
);
746 callbacks
= kmalloc_array(num_vqs
, sizeof(vq_callback_t
*),
748 names
= kmalloc_array(num_vqs
, sizeof(char *), GFP_KERNEL
);
750 if (!callbacks
|| !vqs
|| !names
) {
755 callbacks
[0] = virtscsi_ctrl_done
;
756 callbacks
[1] = virtscsi_event_done
;
757 names
[0] = "control";
759 for (i
= VIRTIO_SCSI_VQ_BASE
; i
< num_vqs
; i
++) {
760 callbacks
[i
] = virtscsi_req_done
;
761 names
[i
] = "request";
764 /* Discover virtqueues and write information to configuration. */
765 err
= virtio_find_vqs(vdev
, num_vqs
, vqs
, callbacks
, names
, &desc
);
769 virtscsi_init_vq(&vscsi
->ctrl_vq
, vqs
[0]);
770 virtscsi_init_vq(&vscsi
->event_vq
, vqs
[1]);
771 for (i
= VIRTIO_SCSI_VQ_BASE
; i
< num_vqs
; i
++)
772 virtscsi_init_vq(&vscsi
->req_vqs
[i
- VIRTIO_SCSI_VQ_BASE
],
775 virtscsi_config_set(vdev
, cdb_size
, VIRTIO_SCSI_CDB_SIZE
);
776 virtscsi_config_set(vdev
, sense_size
, VIRTIO_SCSI_SENSE_SIZE
);
785 virtscsi_remove_vqs(vdev
);
789 static int virtscsi_probe(struct virtio_device
*vdev
)
791 struct Scsi_Host
*shost
;
792 struct virtio_scsi
*vscsi
;
794 u32 sg_elems
, num_targets
;
798 if (!vdev
->config
->get
) {
799 dev_err(&vdev
->dev
, "%s failure: config access disabled\n",
804 /* We need to know how many queues before we allocate. */
805 num_queues
= virtscsi_config_get(vdev
, num_queues
) ? : 1;
807 num_targets
= virtscsi_config_get(vdev
, max_target
) + 1;
809 shost
= scsi_host_alloc(&virtscsi_host_template
,
810 sizeof(*vscsi
) + sizeof(vscsi
->req_vqs
[0]) * num_queues
);
814 sg_elems
= virtscsi_config_get(vdev
, seg_max
) ?: 1;
815 shost
->sg_tablesize
= sg_elems
;
816 vscsi
= shost_priv(shost
);
818 vscsi
->num_queues
= num_queues
;
821 err
= virtscsi_init(vdev
, vscsi
);
823 goto virtscsi_init_failed
;
825 shost
->can_queue
= virtqueue_get_vring_size(vscsi
->req_vqs
[0].vq
);
827 cmd_per_lun
= virtscsi_config_get(vdev
, cmd_per_lun
) ?: 1;
828 shost
->cmd_per_lun
= min_t(u32
, cmd_per_lun
, shost
->can_queue
);
829 shost
->max_sectors
= virtscsi_config_get(vdev
, max_sectors
) ?: 0xFFFF;
831 /* LUNs > 256 are reported with format 1, so they go in the range
834 shost
->max_lun
= virtscsi_config_get(vdev
, max_lun
) + 1 + 0x4000;
835 shost
->max_id
= num_targets
;
836 shost
->max_channel
= 0;
837 shost
->max_cmd_len
= VIRTIO_SCSI_CDB_SIZE
;
838 shost
->nr_hw_queues
= num_queues
;
840 #ifdef CONFIG_BLK_DEV_INTEGRITY
841 if (virtio_has_feature(vdev
, VIRTIO_SCSI_F_T10_PI
)) {
844 host_prot
= SHOST_DIF_TYPE1_PROTECTION
| SHOST_DIF_TYPE2_PROTECTION
|
845 SHOST_DIF_TYPE3_PROTECTION
| SHOST_DIX_TYPE1_PROTECTION
|
846 SHOST_DIX_TYPE2_PROTECTION
| SHOST_DIX_TYPE3_PROTECTION
;
848 scsi_host_set_prot(shost
, host_prot
);
849 scsi_host_set_guard(shost
, SHOST_DIX_GUARD_CRC
);
853 err
= scsi_add_host(shost
, &vdev
->dev
);
855 goto scsi_add_host_failed
;
857 virtio_device_ready(vdev
);
859 if (virtio_has_feature(vdev
, VIRTIO_SCSI_F_HOTPLUG
))
860 virtscsi_kick_event_all(vscsi
);
862 scsi_scan_host(shost
);
865 scsi_add_host_failed
:
866 vdev
->config
->del_vqs(vdev
);
867 virtscsi_init_failed
:
868 scsi_host_put(shost
);
872 static void virtscsi_remove(struct virtio_device
*vdev
)
874 struct Scsi_Host
*shost
= virtio_scsi_host(vdev
);
875 struct virtio_scsi
*vscsi
= shost_priv(shost
);
877 if (virtio_has_feature(vdev
, VIRTIO_SCSI_F_HOTPLUG
))
878 virtscsi_cancel_event_work(vscsi
);
880 scsi_remove_host(shost
);
881 virtscsi_remove_vqs(vdev
);
882 scsi_host_put(shost
);
885 #ifdef CONFIG_PM_SLEEP
886 static int virtscsi_freeze(struct virtio_device
*vdev
)
888 virtscsi_remove_vqs(vdev
);
892 static int virtscsi_restore(struct virtio_device
*vdev
)
894 struct Scsi_Host
*sh
= virtio_scsi_host(vdev
);
895 struct virtio_scsi
*vscsi
= shost_priv(sh
);
898 err
= virtscsi_init(vdev
, vscsi
);
902 virtio_device_ready(vdev
);
904 if (virtio_has_feature(vdev
, VIRTIO_SCSI_F_HOTPLUG
))
905 virtscsi_kick_event_all(vscsi
);
911 static struct virtio_device_id id_table
[] = {
912 { VIRTIO_ID_SCSI
, VIRTIO_DEV_ANY_ID
},
916 static unsigned int features
[] = {
917 VIRTIO_SCSI_F_HOTPLUG
,
918 VIRTIO_SCSI_F_CHANGE
,
919 #ifdef CONFIG_BLK_DEV_INTEGRITY
920 VIRTIO_SCSI_F_T10_PI
,
924 static struct virtio_driver virtio_scsi_driver
= {
925 .feature_table
= features
,
926 .feature_table_size
= ARRAY_SIZE(features
),
927 .driver
.name
= KBUILD_MODNAME
,
928 .driver
.owner
= THIS_MODULE
,
929 .id_table
= id_table
,
930 .probe
= virtscsi_probe
,
931 #ifdef CONFIG_PM_SLEEP
932 .freeze
= virtscsi_freeze
,
933 .restore
= virtscsi_restore
,
935 .remove
= virtscsi_remove
,
938 static int __init
init(void)
942 virtscsi_cmd_cache
= KMEM_CACHE(virtio_scsi_cmd
, 0);
943 if (!virtscsi_cmd_cache
) {
944 pr_err("kmem_cache_create() for virtscsi_cmd_cache failed\n");
950 mempool_create_slab_pool(VIRTIO_SCSI_MEMPOOL_SZ
,
952 if (!virtscsi_cmd_pool
) {
953 pr_err("mempool_create() for virtscsi_cmd_pool failed\n");
956 ret
= register_virtio_driver(&virtio_scsi_driver
);
963 if (virtscsi_cmd_pool
) {
964 mempool_destroy(virtscsi_cmd_pool
);
965 virtscsi_cmd_pool
= NULL
;
967 if (virtscsi_cmd_cache
) {
968 kmem_cache_destroy(virtscsi_cmd_cache
);
969 virtscsi_cmd_cache
= NULL
;
974 static void __exit
fini(void)
976 unregister_virtio_driver(&virtio_scsi_driver
);
977 mempool_destroy(virtscsi_cmd_pool
);
978 kmem_cache_destroy(virtscsi_cmd_cache
);
983 MODULE_DEVICE_TABLE(virtio
, id_table
);
984 MODULE_DESCRIPTION("Virtio SCSI HBA driver");
985 MODULE_LICENSE("GPL");