1 // SPDX-License-Identifier: GPL-2.0-or-later
5 * Copyright (C) 2008 James Smart, Emulex Corporation
6 * Copyright (C) 2011 Red Hat, Inc. All rights reserved.
7 * Copyright (C) 2011 Mike Christie
10 #include <linux/slab.h>
11 #include <linux/blk-mq.h>
12 #include <linux/delay.h>
13 #include <linux/scatterlist.h>
14 #include <linux/bsg-lib.h>
15 #include <linux/export.h>
16 #include <scsi/scsi_cmnd.h>
19 #define uptr64(val) ((void __user *)(uintptr_t)(val))
22 struct blk_mq_tag_set tag_set
;
23 struct bsg_device
*bd
;
25 bsg_timeout_fn
*timeout_fn
;
28 static int bsg_transport_sg_io_fn(struct request_queue
*q
, struct sg_io_v4
*hdr
,
29 fmode_t mode
, unsigned int timeout
)
36 if (hdr
->protocol
!= BSG_PROTOCOL_SCSI
||
37 hdr
->subprotocol
!= BSG_SUB_PROTOCOL_SCSI_TRANSPORT
)
39 if (!capable(CAP_SYS_RAWIO
))
42 rq
= blk_get_request(q
, hdr
->dout_xfer_len
?
43 REQ_OP_DRV_OUT
: REQ_OP_DRV_IN
, 0);
46 rq
->timeout
= timeout
;
48 job
= blk_mq_rq_to_pdu(rq
);
49 job
->request_len
= hdr
->request_len
;
50 job
->request
= memdup_user(uptr64(hdr
->request
), hdr
->request_len
);
51 if (IS_ERR(job
->request
)) {
52 ret
= PTR_ERR(job
->request
);
56 if (hdr
->dout_xfer_len
&& hdr
->din_xfer_len
) {
57 job
->bidi_rq
= blk_get_request(rq
->q
, REQ_OP_DRV_IN
, 0);
58 if (IS_ERR(job
->bidi_rq
)) {
59 ret
= PTR_ERR(job
->bidi_rq
);
60 goto out_free_job_request
;
63 ret
= blk_rq_map_user(rq
->q
, job
->bidi_rq
, NULL
,
64 uptr64(hdr
->din_xferp
), hdr
->din_xfer_len
,
67 goto out_free_bidi_rq
;
69 job
->bidi_bio
= job
->bidi_rq
->bio
;
76 if (hdr
->dout_xfer_len
) {
77 ret
= blk_rq_map_user(rq
->q
, rq
, NULL
, uptr64(hdr
->dout_xferp
),
78 hdr
->dout_xfer_len
, GFP_KERNEL
);
79 } else if (hdr
->din_xfer_len
) {
80 ret
= blk_rq_map_user(rq
->q
, rq
, NULL
, uptr64(hdr
->din_xferp
),
81 hdr
->din_xfer_len
, GFP_KERNEL
);
85 goto out_unmap_bidi_rq
;
88 blk_execute_rq(NULL
, rq
, !(hdr
->flags
& BSG_FLAG_Q_AT_TAIL
));
91 * The assignments below don't make much sense, but are kept for
92 * bug by bug backwards compatibility:
94 hdr
->device_status
= job
->result
& 0xff;
95 hdr
->transport_status
= host_byte(job
->result
);
96 hdr
->driver_status
= 0;
98 if (hdr
->device_status
|| hdr
->transport_status
|| hdr
->driver_status
)
99 hdr
->info
|= SG_INFO_CHECK
;
100 hdr
->response_len
= 0;
102 if (job
->result
< 0) {
103 /* we're only returning the result field in the reply */
104 job
->reply_len
= sizeof(u32
);
108 if (job
->reply_len
&& hdr
->response
) {
109 int len
= min(hdr
->max_response_len
, job
->reply_len
);
111 if (copy_to_user(uptr64(hdr
->response
), job
->reply
, len
))
114 hdr
->response_len
= len
;
117 /* we assume all request payload was transferred, residual == 0 */
121 unsigned int rsp_len
= job
->reply_payload
.payload_len
;
123 if (WARN_ON(job
->reply_payload_rcv_len
> rsp_len
))
126 hdr
->din_resid
= rsp_len
- job
->reply_payload_rcv_len
;
131 blk_rq_unmap_user(bio
);
134 blk_rq_unmap_user(job
->bidi_bio
);
137 blk_put_request(job
->bidi_rq
);
138 out_free_job_request
:
146 * bsg_teardown_job - routine to teardown a bsg job
147 * @kref: kref inside bsg_job that is to be torn down
149 static void bsg_teardown_job(struct kref
*kref
)
151 struct bsg_job
*job
= container_of(kref
, struct bsg_job
, kref
);
152 struct request
*rq
= blk_mq_rq_from_pdu(job
);
154 put_device(job
->dev
); /* release reference for the request */
156 kfree(job
->request_payload
.sg_list
);
157 kfree(job
->reply_payload
.sg_list
);
159 blk_mq_end_request(rq
, BLK_STS_OK
);
162 void bsg_job_put(struct bsg_job
*job
)
164 kref_put(&job
->kref
, bsg_teardown_job
);
166 EXPORT_SYMBOL_GPL(bsg_job_put
);
168 int bsg_job_get(struct bsg_job
*job
)
170 return kref_get_unless_zero(&job
->kref
);
172 EXPORT_SYMBOL_GPL(bsg_job_get
);
175 * bsg_job_done - completion routine for bsg requests
176 * @job: bsg_job that is complete
177 * @result: job reply result
178 * @reply_payload_rcv_len: length of payload recvd
180 * The LLD should call this when the bsg job has completed.
182 void bsg_job_done(struct bsg_job
*job
, int result
,
183 unsigned int reply_payload_rcv_len
)
185 struct request
*rq
= blk_mq_rq_from_pdu(job
);
187 job
->result
= result
;
188 job
->reply_payload_rcv_len
= reply_payload_rcv_len
;
189 if (likely(!blk_should_fake_timeout(rq
->q
)))
190 blk_mq_complete_request(rq
);
192 EXPORT_SYMBOL_GPL(bsg_job_done
);
195 * bsg_complete - softirq done routine for destroying the bsg requests
196 * @rq: BSG request that holds the job to be destroyed
198 static void bsg_complete(struct request
*rq
)
200 struct bsg_job
*job
= blk_mq_rq_to_pdu(rq
);
205 static int bsg_map_buffer(struct bsg_buffer
*buf
, struct request
*req
)
207 size_t sz
= (sizeof(struct scatterlist
) * req
->nr_phys_segments
);
209 BUG_ON(!req
->nr_phys_segments
);
211 buf
->sg_list
= kmalloc(sz
, GFP_KERNEL
);
214 sg_init_table(buf
->sg_list
, req
->nr_phys_segments
);
215 buf
->sg_cnt
= blk_rq_map_sg(req
->q
, req
, buf
->sg_list
);
216 buf
->payload_len
= blk_rq_bytes(req
);
221 * bsg_prepare_job - create the bsg_job structure for the bsg request
222 * @dev: device that is being sent the bsg request
223 * @req: BSG request that needs a job structure
225 static bool bsg_prepare_job(struct device
*dev
, struct request
*req
)
227 struct bsg_job
*job
= blk_mq_rq_to_pdu(req
);
230 job
->timeout
= req
->timeout
;
233 ret
= bsg_map_buffer(&job
->request_payload
, req
);
235 goto failjob_rls_job
;
238 ret
= bsg_map_buffer(&job
->reply_payload
, job
->bidi_rq
);
240 goto failjob_rls_rqst_payload
;
243 /* take a reference for the request */
244 get_device(job
->dev
);
245 kref_init(&job
->kref
);
248 failjob_rls_rqst_payload
:
249 kfree(job
->request_payload
.sg_list
);
251 job
->result
= -ENOMEM
;
256 * bsg_queue_rq - generic handler for bsg requests
257 * @hctx: hardware queue
260 * On error the create_bsg_job function should return a -Exyz error value
261 * that will be set to ->result.
263 * Drivers/subsys should pass this to the queue init function.
265 static blk_status_t
bsg_queue_rq(struct blk_mq_hw_ctx
*hctx
,
266 const struct blk_mq_queue_data
*bd
)
268 struct request_queue
*q
= hctx
->queue
;
269 struct device
*dev
= q
->queuedata
;
270 struct request
*req
= bd
->rq
;
271 struct bsg_set
*bset
=
272 container_of(q
->tag_set
, struct bsg_set
, tag_set
);
273 blk_status_t sts
= BLK_STS_IOERR
;
276 blk_mq_start_request(req
);
278 if (!get_device(dev
))
279 return BLK_STS_IOERR
;
281 if (!bsg_prepare_job(dev
, req
))
284 ret
= bset
->job_fn(blk_mq_rq_to_pdu(req
));
293 /* called right after the request is allocated for the request_queue */
294 static int bsg_init_rq(struct blk_mq_tag_set
*set
, struct request
*req
,
295 unsigned int hctx_idx
, unsigned int numa_node
)
297 struct bsg_job
*job
= blk_mq_rq_to_pdu(req
);
299 job
->reply
= kzalloc(SCSI_SENSE_BUFFERSIZE
, GFP_KERNEL
);
305 /* called right before the request is given to the request_queue user */
306 static void bsg_initialize_rq(struct request
*req
)
308 struct bsg_job
*job
= blk_mq_rq_to_pdu(req
);
309 void *reply
= job
->reply
;
311 memset(job
, 0, sizeof(*job
));
313 job
->reply_len
= SCSI_SENSE_BUFFERSIZE
;
314 job
->dd_data
= job
+ 1;
317 static void bsg_exit_rq(struct blk_mq_tag_set
*set
, struct request
*req
,
318 unsigned int hctx_idx
)
320 struct bsg_job
*job
= blk_mq_rq_to_pdu(req
);
325 void bsg_remove_queue(struct request_queue
*q
)
328 struct bsg_set
*bset
=
329 container_of(q
->tag_set
, struct bsg_set
, tag_set
);
331 bsg_unregister_queue(bset
->bd
);
332 blk_cleanup_queue(q
);
333 blk_mq_free_tag_set(&bset
->tag_set
);
337 EXPORT_SYMBOL_GPL(bsg_remove_queue
);
339 static enum blk_eh_timer_return
bsg_timeout(struct request
*rq
, bool reserved
)
341 struct bsg_set
*bset
=
342 container_of(rq
->q
->tag_set
, struct bsg_set
, tag_set
);
344 if (!bset
->timeout_fn
)
346 return bset
->timeout_fn(rq
);
349 static const struct blk_mq_ops bsg_mq_ops
= {
350 .queue_rq
= bsg_queue_rq
,
351 .init_request
= bsg_init_rq
,
352 .exit_request
= bsg_exit_rq
,
353 .initialize_rq_fn
= bsg_initialize_rq
,
354 .complete
= bsg_complete
,
355 .timeout
= bsg_timeout
,
359 * bsg_setup_queue - Create and add the bsg hooks so we can receive requests
360 * @dev: device to attach bsg device to
361 * @name: device to give bsg device
362 * @job_fn: bsg job handler
363 * @timeout: timeout handler function pointer
364 * @dd_job_size: size of LLD data needed for each job
366 struct request_queue
*bsg_setup_queue(struct device
*dev
, const char *name
,
367 bsg_job_fn
*job_fn
, bsg_timeout_fn
*timeout
, int dd_job_size
)
369 struct bsg_set
*bset
;
370 struct blk_mq_tag_set
*set
;
371 struct request_queue
*q
;
374 bset
= kzalloc(sizeof(*bset
), GFP_KERNEL
);
376 return ERR_PTR(-ENOMEM
);
378 bset
->job_fn
= job_fn
;
379 bset
->timeout_fn
= timeout
;
381 set
= &bset
->tag_set
;
382 set
->ops
= &bsg_mq_ops
;
383 set
->nr_hw_queues
= 1;
384 set
->queue_depth
= 128;
385 set
->numa_node
= NUMA_NO_NODE
;
386 set
->cmd_size
= sizeof(struct bsg_job
) + dd_job_size
;
387 set
->flags
= BLK_MQ_F_NO_SCHED
| BLK_MQ_F_BLOCKING
;
388 if (blk_mq_alloc_tag_set(set
))
391 q
= blk_mq_init_queue(set
);
398 blk_queue_rq_timeout(q
, BLK_DEFAULT_SG_TIMEOUT
);
400 bset
->bd
= bsg_register_queue(q
, dev
, name
, bsg_transport_sg_io_fn
);
401 if (IS_ERR(bset
->bd
)) {
402 ret
= PTR_ERR(bset
->bd
);
403 goto out_cleanup_queue
;
408 blk_cleanup_queue(q
);
410 blk_mq_free_tag_set(set
);
415 EXPORT_SYMBOL_GPL(bsg_setup_queue
);