1 // SPDX-License-Identifier: GPL-2.0-only
3 * Copyright (C) 2013 Shaohua Li <shli@kernel.org>
4 * Copyright (C) 2014 Red Hat, Inc.
5 * Copyright (C) 2015 Arrikto, Inc.
6 * Copyright (C) 2017 Chinamobile, Inc.
9 #include <linux/spinlock.h>
10 #include <linux/module.h>
11 #include <linux/kernel.h>
12 #include <linux/timer.h>
13 #include <linux/parser.h>
14 #include <linux/vmalloc.h>
15 #include <linux/uio_driver.h>
16 #include <linux/xarray.h>
17 #include <linux/stringify.h>
18 #include <linux/bitops.h>
19 #include <linux/highmem.h>
20 #include <linux/configfs.h>
21 #include <linux/mutex.h>
22 #include <linux/workqueue.h>
23 #include <net/genetlink.h>
24 #include <scsi/scsi_common.h>
25 #include <scsi/scsi_proto.h>
26 #include <target/target_core_base.h>
27 #include <target/target_core_fabric.h>
28 #include <target/target_core_backend.h>
30 #include <linux/target_core_user.h>
37 * Define a shared-memory interface for LIO to pass SCSI commands and
38 * data to userspace for processing. This is to allow backends that
39 * are too complex for in-kernel support to be possible.
41 * It uses the UIO framework to do a lot of the device-creation and
42 * introspection work for us.
44 * See the .h file for how the ring is laid out. Note that while the
45 * command ring is defined, the particulars of the data area are
46 * not. Offset values in the command entry point to other locations
47 * internal to the mmap-ed area. There is separate space outside the
48 * command ring for data buffers. This leaves maximum flexibility for
49 * moving buffer allocations, or even page flipping or other
50 * allocation techniques, without altering the command ring layout.
53 * The user process must be assumed to be malicious. There's no way to
54 * prevent it breaking the command ring protocol if it wants, but in
55 * order to prevent other issues we must only ever read *data* from
56 * the shared memory area, not offsets or sizes. This applies to
57 * command ring entries as well as the mailbox. Extra code needed for
58 * this may have a 'UAM' comment.
61 #define TCMU_TIME_OUT (30 * MSEC_PER_SEC)
63 /* For mailbox plus cmd ring, the size is fixed 8MB */
64 #define MB_CMDR_SIZE (8 * 1024 * 1024)
65 /* Offset of cmd ring is size of mailbox */
66 #define CMDR_OFF sizeof(struct tcmu_mailbox)
67 #define CMDR_SIZE (MB_CMDR_SIZE - CMDR_OFF)
70 * For data area, the default block size is PAGE_SIZE and
71 * the default total size is 256K * PAGE_SIZE.
73 #define DATA_PAGES_PER_BLK_DEF 1
74 #define DATA_AREA_PAGES_DEF (256 * 1024)
76 #define TCMU_MBS_TO_PAGES(_mbs) ((size_t)_mbs << (20 - PAGE_SHIFT))
77 #define TCMU_PAGES_TO_MBS(_pages) (_pages >> (20 - PAGE_SHIFT))
80 * Default number of global data blocks(512K * PAGE_SIZE)
81 * when the unmap thread will be started.
83 #define TCMU_GLOBAL_MAX_PAGES_DEF (512 * 1024)
85 static u8 tcmu_kern_cmd_reply_supported
;
86 static u8 tcmu_netlink_blocked
;
88 static struct device
*tcmu_root_device
;
94 #define TCMU_CONFIG_LEN 256
96 static DEFINE_MUTEX(tcmu_nl_cmd_mutex
);
97 static LIST_HEAD(tcmu_nl_cmd_list
);
102 /* wake up thread waiting for reply */
103 struct completion complete
;
104 struct list_head nl_list
;
105 struct tcmu_dev
*udev
;
111 struct list_head node
;
114 struct se_device se_dev
;
115 struct se_dev_plug se_plug
;
120 #define TCMU_DEV_BIT_OPEN 0
121 #define TCMU_DEV_BIT_BROKEN 1
122 #define TCMU_DEV_BIT_BLOCKED 2
123 #define TCMU_DEV_BIT_TMR_NOTIFY 3
124 #define TCMU_DEV_BIT_PLUGGED 4
127 struct uio_info uio_info
;
133 struct tcmu_mailbox
*mb_addr
;
136 u32 cmdr_last_cleaned
;
137 /* Offset of data area from start of mb */
138 /* Must add data_off and mb_addr to get the address */
144 struct mutex cmdr_lock
;
145 struct list_head qfull_queue
;
146 struct list_head tmr_queue
;
150 unsigned long *data_bitmap
;
151 struct xarray data_pages
;
152 uint32_t data_pages_per_blk
;
153 uint32_t data_blk_size
;
155 struct xarray commands
;
157 struct timer_list cmd_timer
;
158 unsigned int cmd_time_out
;
159 struct list_head inflight_queue
;
161 struct timer_list qfull_timer
;
164 struct list_head timedout_entry
;
166 struct tcmu_nl_cmd curr_nl_cmd
;
168 char dev_config
[TCMU_CONFIG_LEN
];
170 int nl_reply_supported
;
173 #define TCMU_DEV(_se_dev) container_of(_se_dev, struct tcmu_dev, se_dev)
176 struct se_cmd
*se_cmd
;
177 struct tcmu_dev
*tcmu_dev
;
178 struct list_head queue_entry
;
182 /* Can't use se_cmd when cleaning up expired cmds, because if
183 cmd has been completed then accessing se_cmd is off limits */
185 uint32_t dbi_bidi_cnt
;
189 uint32_t data_len_bidi
;
191 unsigned long deadline
;
193 #define TCMU_CMD_BIT_EXPIRED 0
194 #define TCMU_CMD_BIT_KEEP_BUF 1
199 struct list_head queue_entry
;
202 uint32_t tmr_cmd_cnt
;
203 int16_t tmr_cmd_ids
[];
207 * To avoid dead lock the mutex lock order should always be:
209 * mutex_lock(&root_udev_mutex);
211 * mutex_lock(&tcmu_dev->cmdr_lock);
212 * mutex_unlock(&tcmu_dev->cmdr_lock);
214 * mutex_unlock(&root_udev_mutex);
216 static DEFINE_MUTEX(root_udev_mutex
);
217 static LIST_HEAD(root_udev
);
219 static DEFINE_SPINLOCK(timed_out_udevs_lock
);
220 static LIST_HEAD(timed_out_udevs
);
222 static struct kmem_cache
*tcmu_cmd_cache
;
224 static atomic_t global_page_count
= ATOMIC_INIT(0);
225 static struct delayed_work tcmu_unmap_work
;
226 static int tcmu_global_max_pages
= TCMU_GLOBAL_MAX_PAGES_DEF
;
228 static int tcmu_set_global_max_data_area(const char *str
,
229 const struct kernel_param
*kp
)
231 int ret
, max_area_mb
;
233 ret
= kstrtoint(str
, 10, &max_area_mb
);
237 if (max_area_mb
<= 0) {
238 pr_err("global_max_data_area must be larger than 0.\n");
242 tcmu_global_max_pages
= TCMU_MBS_TO_PAGES(max_area_mb
);
243 if (atomic_read(&global_page_count
) > tcmu_global_max_pages
)
244 schedule_delayed_work(&tcmu_unmap_work
, 0);
246 cancel_delayed_work_sync(&tcmu_unmap_work
);
251 static int tcmu_get_global_max_data_area(char *buffer
,
252 const struct kernel_param
*kp
)
254 return sprintf(buffer
, "%d\n", TCMU_PAGES_TO_MBS(tcmu_global_max_pages
));
257 static const struct kernel_param_ops tcmu_global_max_data_area_op
= {
258 .set
= tcmu_set_global_max_data_area
,
259 .get
= tcmu_get_global_max_data_area
,
262 module_param_cb(global_max_data_area_mb
, &tcmu_global_max_data_area_op
, NULL
,
264 MODULE_PARM_DESC(global_max_data_area_mb
,
265 "Max MBs allowed to be allocated to all the tcmu device's "
268 static int tcmu_get_block_netlink(char *buffer
,
269 const struct kernel_param
*kp
)
271 return sprintf(buffer
, "%s\n", tcmu_netlink_blocked
?
272 "blocked" : "unblocked");
275 static int tcmu_set_block_netlink(const char *str
,
276 const struct kernel_param
*kp
)
281 ret
= kstrtou8(str
, 0, &val
);
286 pr_err("Invalid block netlink value %u\n", val
);
290 tcmu_netlink_blocked
= val
;
294 static const struct kernel_param_ops tcmu_block_netlink_op
= {
295 .set
= tcmu_set_block_netlink
,
296 .get
= tcmu_get_block_netlink
,
299 module_param_cb(block_netlink
, &tcmu_block_netlink_op
, NULL
, S_IWUSR
| S_IRUGO
);
300 MODULE_PARM_DESC(block_netlink
, "Block new netlink commands.");
302 static int tcmu_fail_netlink_cmd(struct tcmu_nl_cmd
*nl_cmd
)
304 struct tcmu_dev
*udev
= nl_cmd
->udev
;
306 if (!tcmu_netlink_blocked
) {
307 pr_err("Could not reset device's netlink interface. Netlink is not blocked.\n");
311 if (nl_cmd
->cmd
!= TCMU_CMD_UNSPEC
) {
312 pr_debug("Aborting nl cmd %d on %s\n", nl_cmd
->cmd
, udev
->name
);
313 nl_cmd
->status
= -EINTR
;
314 list_del(&nl_cmd
->nl_list
);
315 complete(&nl_cmd
->complete
);
320 static int tcmu_set_reset_netlink(const char *str
,
321 const struct kernel_param
*kp
)
323 struct tcmu_nl_cmd
*nl_cmd
, *tmp_cmd
;
327 ret
= kstrtou8(str
, 0, &val
);
332 pr_err("Invalid reset netlink value %u\n", val
);
336 mutex_lock(&tcmu_nl_cmd_mutex
);
337 list_for_each_entry_safe(nl_cmd
, tmp_cmd
, &tcmu_nl_cmd_list
, nl_list
) {
338 ret
= tcmu_fail_netlink_cmd(nl_cmd
);
342 mutex_unlock(&tcmu_nl_cmd_mutex
);
347 static const struct kernel_param_ops tcmu_reset_netlink_op
= {
348 .set
= tcmu_set_reset_netlink
,
351 module_param_cb(reset_netlink
, &tcmu_reset_netlink_op
, NULL
, S_IWUSR
);
352 MODULE_PARM_DESC(reset_netlink
, "Reset netlink commands.");
354 /* multicast group */
355 enum tcmu_multicast_groups
{
359 static const struct genl_multicast_group tcmu_mcgrps
[] = {
360 [TCMU_MCGRP_CONFIG
] = { .name
= "config", },
363 static struct nla_policy tcmu_attr_policy
[TCMU_ATTR_MAX
+1] = {
364 [TCMU_ATTR_DEVICE
] = { .type
= NLA_STRING
},
365 [TCMU_ATTR_MINOR
] = { .type
= NLA_U32
},
366 [TCMU_ATTR_CMD_STATUS
] = { .type
= NLA_S32
},
367 [TCMU_ATTR_DEVICE_ID
] = { .type
= NLA_U32
},
368 [TCMU_ATTR_SUPP_KERN_CMD_REPLY
] = { .type
= NLA_U8
},
371 static int tcmu_genl_cmd_done(struct genl_info
*info
, int completed_cmd
)
373 struct tcmu_dev
*udev
= NULL
;
374 struct tcmu_nl_cmd
*nl_cmd
;
375 int dev_id
, rc
, ret
= 0;
377 if (!info
->attrs
[TCMU_ATTR_CMD_STATUS
] ||
378 !info
->attrs
[TCMU_ATTR_DEVICE_ID
]) {
379 printk(KERN_ERR
"TCMU_ATTR_CMD_STATUS or TCMU_ATTR_DEVICE_ID not set, doing nothing\n");
383 dev_id
= nla_get_u32(info
->attrs
[TCMU_ATTR_DEVICE_ID
]);
384 rc
= nla_get_s32(info
->attrs
[TCMU_ATTR_CMD_STATUS
]);
386 mutex_lock(&tcmu_nl_cmd_mutex
);
387 list_for_each_entry(nl_cmd
, &tcmu_nl_cmd_list
, nl_list
) {
388 if (nl_cmd
->udev
->se_dev
.dev_index
== dev_id
) {
395 pr_err("tcmu nl cmd %u/%d completion could not find device with dev id %u.\n",
396 completed_cmd
, rc
, dev_id
);
400 list_del(&nl_cmd
->nl_list
);
402 pr_debug("%s genl cmd done got id %d curr %d done %d rc %d stat %d\n",
403 udev
->name
, dev_id
, nl_cmd
->cmd
, completed_cmd
, rc
,
406 if (nl_cmd
->cmd
!= completed_cmd
) {
407 pr_err("Mismatched commands on %s (Expecting reply for %d. Current %d).\n",
408 udev
->name
, completed_cmd
, nl_cmd
->cmd
);
414 complete(&nl_cmd
->complete
);
416 mutex_unlock(&tcmu_nl_cmd_mutex
);
420 static int tcmu_genl_rm_dev_done(struct sk_buff
*skb
, struct genl_info
*info
)
422 return tcmu_genl_cmd_done(info
, TCMU_CMD_REMOVED_DEVICE
);
425 static int tcmu_genl_add_dev_done(struct sk_buff
*skb
, struct genl_info
*info
)
427 return tcmu_genl_cmd_done(info
, TCMU_CMD_ADDED_DEVICE
);
430 static int tcmu_genl_reconfig_dev_done(struct sk_buff
*skb
,
431 struct genl_info
*info
)
433 return tcmu_genl_cmd_done(info
, TCMU_CMD_RECONFIG_DEVICE
);
436 static int tcmu_genl_set_features(struct sk_buff
*skb
, struct genl_info
*info
)
438 if (info
->attrs
[TCMU_ATTR_SUPP_KERN_CMD_REPLY
]) {
439 tcmu_kern_cmd_reply_supported
=
440 nla_get_u8(info
->attrs
[TCMU_ATTR_SUPP_KERN_CMD_REPLY
]);
441 printk(KERN_INFO
"tcmu daemon: command reply support %u.\n",
442 tcmu_kern_cmd_reply_supported
);
448 static const struct genl_small_ops tcmu_genl_ops
[] = {
450 .cmd
= TCMU_CMD_SET_FEATURES
,
451 .validate
= GENL_DONT_VALIDATE_STRICT
| GENL_DONT_VALIDATE_DUMP
,
452 .flags
= GENL_ADMIN_PERM
,
453 .doit
= tcmu_genl_set_features
,
456 .cmd
= TCMU_CMD_ADDED_DEVICE_DONE
,
457 .validate
= GENL_DONT_VALIDATE_STRICT
| GENL_DONT_VALIDATE_DUMP
,
458 .flags
= GENL_ADMIN_PERM
,
459 .doit
= tcmu_genl_add_dev_done
,
462 .cmd
= TCMU_CMD_REMOVED_DEVICE_DONE
,
463 .validate
= GENL_DONT_VALIDATE_STRICT
| GENL_DONT_VALIDATE_DUMP
,
464 .flags
= GENL_ADMIN_PERM
,
465 .doit
= tcmu_genl_rm_dev_done
,
468 .cmd
= TCMU_CMD_RECONFIG_DEVICE_DONE
,
469 .validate
= GENL_DONT_VALIDATE_STRICT
| GENL_DONT_VALIDATE_DUMP
,
470 .flags
= GENL_ADMIN_PERM
,
471 .doit
= tcmu_genl_reconfig_dev_done
,
475 /* Our generic netlink family */
476 static struct genl_family tcmu_genl_family __ro_after_init
= {
477 .module
= THIS_MODULE
,
481 .maxattr
= TCMU_ATTR_MAX
,
482 .policy
= tcmu_attr_policy
,
483 .mcgrps
= tcmu_mcgrps
,
484 .n_mcgrps
= ARRAY_SIZE(tcmu_mcgrps
),
486 .small_ops
= tcmu_genl_ops
,
487 .n_small_ops
= ARRAY_SIZE(tcmu_genl_ops
),
490 #define tcmu_cmd_set_dbi_cur(cmd, index) ((cmd)->dbi_cur = (index))
491 #define tcmu_cmd_reset_dbi_cur(cmd) tcmu_cmd_set_dbi_cur(cmd, 0)
492 #define tcmu_cmd_set_dbi(cmd, index) ((cmd)->dbi[(cmd)->dbi_cur++] = (index))
493 #define tcmu_cmd_get_dbi(cmd) ((cmd)->dbi[(cmd)->dbi_cur++])
495 static void tcmu_cmd_free_data(struct tcmu_cmd
*tcmu_cmd
, uint32_t len
)
497 struct tcmu_dev
*udev
= tcmu_cmd
->tcmu_dev
;
500 for (i
= 0; i
< len
; i
++)
501 clear_bit(tcmu_cmd
->dbi
[i
], udev
->data_bitmap
);
504 static inline int tcmu_get_empty_block(struct tcmu_dev
*udev
,
505 struct tcmu_cmd
*tcmu_cmd
,
506 int prev_dbi
, int length
, int *iov_cnt
)
508 XA_STATE(xas
, &udev
->data_pages
, 0);
510 int i
, cnt
, dbi
, dpi
;
511 int page_cnt
= DIV_ROUND_UP(length
, PAGE_SIZE
);
513 dbi
= find_first_zero_bit(udev
->data_bitmap
, udev
->dbi_thresh
);
514 if (dbi
== udev
->dbi_thresh
)
517 dpi
= dbi
* udev
->data_pages_per_blk
;
518 /* Count the number of already allocated pages */
521 for (cnt
= 0; xas_next(&xas
) && cnt
< page_cnt
;)
525 for (i
= cnt
; i
< page_cnt
; i
++) {
526 /* try to get new page from the mm */
527 page
= alloc_page(GFP_NOIO
);
531 if (xa_store(&udev
->data_pages
, dpi
+ i
, page
, GFP_NOIO
)) {
536 if (atomic_add_return(i
- cnt
, &global_page_count
) >
537 tcmu_global_max_pages
)
538 schedule_delayed_work(&tcmu_unmap_work
, 0);
540 if (i
&& dbi
> udev
->dbi_max
)
543 set_bit(dbi
, udev
->data_bitmap
);
544 tcmu_cmd_set_dbi(tcmu_cmd
, dbi
);
546 if (dbi
!= prev_dbi
+ 1)
549 return i
== page_cnt
? dbi
: -1;
552 static int tcmu_get_empty_blocks(struct tcmu_dev
*udev
,
553 struct tcmu_cmd
*tcmu_cmd
, int length
)
555 /* start value of dbi + 1 must not be a valid dbi */
557 int blk_data_len
, iov_cnt
= 0;
558 uint32_t blk_size
= udev
->data_blk_size
;
560 for (; length
> 0; length
-= blk_size
) {
561 blk_data_len
= min_t(uint32_t, length
, blk_size
);
562 dbi
= tcmu_get_empty_block(udev
, tcmu_cmd
, dbi
, blk_data_len
,
570 static inline void tcmu_free_cmd(struct tcmu_cmd
*tcmu_cmd
)
572 kfree(tcmu_cmd
->dbi
);
573 kmem_cache_free(tcmu_cmd_cache
, tcmu_cmd
);
576 static inline void tcmu_cmd_set_block_cnts(struct tcmu_cmd
*cmd
)
579 struct se_cmd
*se_cmd
= cmd
->se_cmd
;
580 uint32_t blk_size
= cmd
->tcmu_dev
->data_blk_size
;
582 cmd
->dbi_cnt
= DIV_ROUND_UP(se_cmd
->data_length
, blk_size
);
584 if (se_cmd
->se_cmd_flags
& SCF_BIDI
) {
585 BUG_ON(!(se_cmd
->t_bidi_data_sg
&& se_cmd
->t_bidi_data_nents
));
586 for (i
= 0, len
= 0; i
< se_cmd
->t_bidi_data_nents
; i
++)
587 len
+= se_cmd
->t_bidi_data_sg
[i
].length
;
588 cmd
->dbi_bidi_cnt
= DIV_ROUND_UP(len
, blk_size
);
589 cmd
->dbi_cnt
+= cmd
->dbi_bidi_cnt
;
590 cmd
->data_len_bidi
= len
;
594 static int new_block_to_iov(struct tcmu_dev
*udev
, struct tcmu_cmd
*cmd
,
595 struct iovec
**iov
, int prev_dbi
, int len
)
597 /* Get the next dbi */
598 int dbi
= tcmu_cmd_get_dbi(cmd
);
600 /* Do not add more than udev->data_blk_size to iov */
601 len
= min_t(int, len
, udev
->data_blk_size
);
604 * The following code will gather and map the blocks to the same iovec
605 * when the blocks are all next to each other.
607 if (dbi
!= prev_dbi
+ 1) {
608 /* dbi is not next to previous dbi, so start new iov */
611 /* write offset relative to mb_addr */
612 (*iov
)->iov_base
= (void __user
*)
613 (udev
->data_off
+ dbi
* udev
->data_blk_size
);
615 (*iov
)->iov_len
+= len
;
620 static void tcmu_setup_iovs(struct tcmu_dev
*udev
, struct tcmu_cmd
*cmd
,
621 struct iovec
**iov
, int data_length
)
623 /* start value of dbi + 1 must not be a valid dbi */
626 /* We prepare the IOVs for DMA_FROM_DEVICE transfer direction */
627 for (; data_length
> 0; data_length
-= udev
->data_blk_size
)
628 dbi
= new_block_to_iov(udev
, cmd
, iov
, dbi
, data_length
);
631 static struct tcmu_cmd
*tcmu_alloc_cmd(struct se_cmd
*se_cmd
)
633 struct se_device
*se_dev
= se_cmd
->se_dev
;
634 struct tcmu_dev
*udev
= TCMU_DEV(se_dev
);
635 struct tcmu_cmd
*tcmu_cmd
;
637 tcmu_cmd
= kmem_cache_zalloc(tcmu_cmd_cache
, GFP_NOIO
);
641 INIT_LIST_HEAD(&tcmu_cmd
->queue_entry
);
642 tcmu_cmd
->se_cmd
= se_cmd
;
643 tcmu_cmd
->tcmu_dev
= udev
;
645 tcmu_cmd_set_block_cnts(tcmu_cmd
);
646 tcmu_cmd
->dbi
= kcalloc(tcmu_cmd
->dbi_cnt
, sizeof(uint32_t),
648 if (!tcmu_cmd
->dbi
) {
649 kmem_cache_free(tcmu_cmd_cache
, tcmu_cmd
);
656 static inline void tcmu_flush_dcache_range(void *vaddr
, size_t size
)
658 unsigned long offset
= offset_in_page(vaddr
);
659 void *start
= vaddr
- offset
;
661 size
= round_up(size
+offset
, PAGE_SIZE
);
664 flush_dcache_page(vmalloc_to_page(start
));
671 * Some ring helper functions. We don't assume size is a power of 2 so
672 * we can't use circ_buf.h.
674 static inline size_t spc_used(size_t head
, size_t tail
, size_t size
)
676 int diff
= head
- tail
;
684 static inline size_t spc_free(size_t head
, size_t tail
, size_t size
)
686 /* Keep 1 byte unused or we can't tell full from empty */
687 return (size
- spc_used(head
, tail
, size
) - 1);
690 static inline size_t head_to_end(size_t head
, size_t size
)
695 #define UPDATE_HEAD(head, used, size) smp_store_release(&head, ((head % size) + used) % size)
697 #define TCMU_SG_TO_DATA_AREA 1
698 #define TCMU_DATA_AREA_TO_SG 2
700 static inline void tcmu_copy_data(struct tcmu_dev
*udev
,
701 struct tcmu_cmd
*tcmu_cmd
, uint32_t direction
,
702 struct scatterlist
*sg
, unsigned int sg_nents
,
703 struct iovec
**iov
, size_t data_len
)
705 /* start value of dbi + 1 must not be a valid dbi */
707 size_t page_remaining
, cp_len
;
708 int page_cnt
, page_inx
, dpi
;
709 struct sg_mapping_iter sg_iter
;
710 unsigned int sg_flags
;
712 void *data_page_start
, *data_addr
;
714 if (direction
== TCMU_SG_TO_DATA_AREA
)
715 sg_flags
= SG_MITER_ATOMIC
| SG_MITER_FROM_SG
;
717 sg_flags
= SG_MITER_ATOMIC
| SG_MITER_TO_SG
;
718 sg_miter_start(&sg_iter
, sg
, sg_nents
, sg_flags
);
721 if (direction
== TCMU_SG_TO_DATA_AREA
)
722 dbi
= new_block_to_iov(udev
, tcmu_cmd
, iov
, dbi
,
725 dbi
= tcmu_cmd_get_dbi(tcmu_cmd
);
727 page_cnt
= DIV_ROUND_UP(data_len
, PAGE_SIZE
);
728 if (page_cnt
> udev
->data_pages_per_blk
)
729 page_cnt
= udev
->data_pages_per_blk
;
731 dpi
= dbi
* udev
->data_pages_per_blk
;
732 for (page_inx
= 0; page_inx
< page_cnt
&& data_len
;
734 page
= xa_load(&udev
->data_pages
, dpi
);
736 if (direction
== TCMU_DATA_AREA_TO_SG
)
737 flush_dcache_page(page
);
738 data_page_start
= kmap_atomic(page
);
739 page_remaining
= PAGE_SIZE
;
741 while (page_remaining
&& data_len
) {
742 if (!sg_miter_next(&sg_iter
)) {
743 /* set length to 0 to abort outer loop */
745 pr_debug("%s: aborting data copy due to exhausted sg_list\n",
749 cp_len
= min3(sg_iter
.length
, page_remaining
,
752 data_addr
= data_page_start
+
753 PAGE_SIZE
- page_remaining
;
754 if (direction
== TCMU_SG_TO_DATA_AREA
)
755 memcpy(data_addr
, sg_iter
.addr
, cp_len
);
757 memcpy(sg_iter
.addr
, data_addr
, cp_len
);
760 page_remaining
-= cp_len
;
761 sg_iter
.consumed
= cp_len
;
763 sg_miter_stop(&sg_iter
);
765 kunmap_atomic(data_page_start
);
766 if (direction
== TCMU_SG_TO_DATA_AREA
)
767 flush_dcache_page(page
);
772 static void scatter_data_area(struct tcmu_dev
*udev
, struct tcmu_cmd
*tcmu_cmd
,
775 struct se_cmd
*se_cmd
= tcmu_cmd
->se_cmd
;
777 tcmu_copy_data(udev
, tcmu_cmd
, TCMU_SG_TO_DATA_AREA
, se_cmd
->t_data_sg
,
778 se_cmd
->t_data_nents
, iov
, se_cmd
->data_length
);
781 static void gather_data_area(struct tcmu_dev
*udev
, struct tcmu_cmd
*tcmu_cmd
,
782 bool bidi
, uint32_t read_len
)
784 struct se_cmd
*se_cmd
= tcmu_cmd
->se_cmd
;
785 struct scatterlist
*data_sg
;
786 unsigned int data_nents
;
789 data_sg
= se_cmd
->t_data_sg
;
790 data_nents
= se_cmd
->t_data_nents
;
793 * For bidi case, the first count blocks are for Data-Out
794 * buffer blocks, and before gathering the Data-In buffer
795 * the Data-Out buffer blocks should be skipped.
797 tcmu_cmd_set_dbi_cur(tcmu_cmd
,
798 tcmu_cmd
->dbi_cnt
- tcmu_cmd
->dbi_bidi_cnt
);
800 data_sg
= se_cmd
->t_bidi_data_sg
;
801 data_nents
= se_cmd
->t_bidi_data_nents
;
804 tcmu_copy_data(udev
, tcmu_cmd
, TCMU_DATA_AREA_TO_SG
, data_sg
,
805 data_nents
, NULL
, read_len
);
808 static inline size_t spc_bitmap_free(unsigned long *bitmap
, uint32_t thresh
)
810 return thresh
- bitmap_weight(bitmap
, thresh
);
814 * We can't queue a command until we have space available on the cmd ring.
816 * Called with ring lock held.
818 static bool is_ring_space_avail(struct tcmu_dev
*udev
, size_t cmd_size
)
820 struct tcmu_mailbox
*mb
= udev
->mb_addr
;
821 size_t space
, cmd_needed
;
824 tcmu_flush_dcache_range(mb
, sizeof(*mb
));
826 cmd_head
= mb
->cmd_head
% udev
->cmdr_size
; /* UAM */
829 * If cmd end-of-ring space is too small then we need space for a NOP plus
830 * original cmd - cmds are internally contiguous.
832 if (head_to_end(cmd_head
, udev
->cmdr_size
) >= cmd_size
)
833 cmd_needed
= cmd_size
;
835 cmd_needed
= cmd_size
+ head_to_end(cmd_head
, udev
->cmdr_size
);
837 space
= spc_free(cmd_head
, udev
->cmdr_last_cleaned
, udev
->cmdr_size
);
838 if (space
< cmd_needed
) {
839 pr_debug("no cmd space: %u %u %u\n", cmd_head
,
840 udev
->cmdr_last_cleaned
, udev
->cmdr_size
);
847 * We have to allocate data buffers before we can queue a command.
848 * Returns -1 on error (not enough space) or number of needed iovs on success
850 * Called with ring lock held.
852 static int tcmu_alloc_data_space(struct tcmu_dev
*udev
, struct tcmu_cmd
*cmd
,
855 int space
, iov_cnt
= 0, ret
= 0;
860 /* try to check and get the data blocks as needed */
861 space
= spc_bitmap_free(udev
->data_bitmap
, udev
->dbi_thresh
);
862 if (space
< cmd
->dbi_cnt
) {
863 unsigned long blocks_left
=
864 (udev
->max_blocks
- udev
->dbi_thresh
) + space
;
866 if (blocks_left
< cmd
->dbi_cnt
) {
867 pr_debug("no data space: only %lu available, but ask for %u\n",
868 blocks_left
* udev
->data_blk_size
,
869 cmd
->dbi_cnt
* udev
->data_blk_size
);
873 udev
->dbi_thresh
+= cmd
->dbi_cnt
;
874 if (udev
->dbi_thresh
> udev
->max_blocks
)
875 udev
->dbi_thresh
= udev
->max_blocks
;
878 iov_cnt
= tcmu_get_empty_blocks(udev
, cmd
, cmd
->se_cmd
->data_length
);
882 if (cmd
->dbi_bidi_cnt
) {
883 ret
= tcmu_get_empty_blocks(udev
, cmd
, cmd
->data_len_bidi
);
889 return iov_cnt
+ ret
;
892 static inline size_t tcmu_cmd_get_base_cmd_size(size_t iov_cnt
)
894 return max(offsetof(struct tcmu_cmd_entry
, req
.iov
[iov_cnt
]),
895 sizeof(struct tcmu_cmd_entry
));
898 static inline size_t tcmu_cmd_get_cmd_size(struct tcmu_cmd
*tcmu_cmd
,
899 size_t base_command_size
)
901 struct se_cmd
*se_cmd
= tcmu_cmd
->se_cmd
;
904 command_size
= base_command_size
+
905 round_up(scsi_command_size(se_cmd
->t_task_cdb
),
908 WARN_ON(command_size
& (TCMU_OP_ALIGN_SIZE
-1));
913 static void tcmu_setup_cmd_timer(struct tcmu_cmd
*tcmu_cmd
, unsigned int tmo
,
914 struct timer_list
*timer
)
919 tcmu_cmd
->deadline
= round_jiffies_up(jiffies
+ msecs_to_jiffies(tmo
));
920 if (!timer_pending(timer
))
921 mod_timer(timer
, tcmu_cmd
->deadline
);
923 pr_debug("Timeout set up for cmd %p, dev = %s, tmo = %lu\n", tcmu_cmd
,
924 tcmu_cmd
->tcmu_dev
->name
, tmo
/ MSEC_PER_SEC
);
927 static int add_to_qfull_queue(struct tcmu_cmd
*tcmu_cmd
)
929 struct tcmu_dev
*udev
= tcmu_cmd
->tcmu_dev
;
933 * For backwards compat if qfull_time_out is not set use
934 * cmd_time_out and if that's not set use the default time out.
936 if (!udev
->qfull_time_out
)
938 else if (udev
->qfull_time_out
> 0)
939 tmo
= udev
->qfull_time_out
;
940 else if (udev
->cmd_time_out
)
941 tmo
= udev
->cmd_time_out
;
945 tcmu_setup_cmd_timer(tcmu_cmd
, tmo
, &udev
->qfull_timer
);
947 list_add_tail(&tcmu_cmd
->queue_entry
, &udev
->qfull_queue
);
948 pr_debug("adding cmd %p on dev %s to ring space wait queue\n",
949 tcmu_cmd
, udev
->name
);
953 static uint32_t ring_insert_padding(struct tcmu_dev
*udev
, size_t cmd_size
)
955 struct tcmu_cmd_entry_hdr
*hdr
;
956 struct tcmu_mailbox
*mb
= udev
->mb_addr
;
957 uint32_t cmd_head
= mb
->cmd_head
% udev
->cmdr_size
; /* UAM */
959 /* Insert a PAD if end-of-ring space is too small */
960 if (head_to_end(cmd_head
, udev
->cmdr_size
) < cmd_size
) {
961 size_t pad_size
= head_to_end(cmd_head
, udev
->cmdr_size
);
963 hdr
= udev
->cmdr
+ cmd_head
;
964 tcmu_hdr_set_op(&hdr
->len_op
, TCMU_OP_PAD
);
965 tcmu_hdr_set_len(&hdr
->len_op
, pad_size
);
966 hdr
->cmd_id
= 0; /* not used for PAD */
969 tcmu_flush_dcache_range(hdr
, sizeof(*hdr
));
971 UPDATE_HEAD(mb
->cmd_head
, pad_size
, udev
->cmdr_size
);
972 tcmu_flush_dcache_range(mb
, sizeof(*mb
));
974 cmd_head
= mb
->cmd_head
% udev
->cmdr_size
; /* UAM */
975 WARN_ON(cmd_head
!= 0);
981 static void tcmu_unplug_device(struct se_dev_plug
*se_plug
)
983 struct se_device
*se_dev
= se_plug
->se_dev
;
984 struct tcmu_dev
*udev
= TCMU_DEV(se_dev
);
986 clear_bit(TCMU_DEV_BIT_PLUGGED
, &udev
->flags
);
987 uio_event_notify(&udev
->uio_info
);
990 static struct se_dev_plug
*tcmu_plug_device(struct se_device
*se_dev
)
992 struct tcmu_dev
*udev
= TCMU_DEV(se_dev
);
994 if (!test_and_set_bit(TCMU_DEV_BIT_PLUGGED
, &udev
->flags
))
995 return &udev
->se_plug
;
1001 * queue_cmd_ring - queue cmd to ring or internally
1002 * @tcmu_cmd: cmd to queue
1003 * @scsi_err: TCM error code if failure (-1) returned.
1006 * -1 we cannot queue internally or to the ring.
1008 * 1 internally queued to wait for ring memory to free.
1010 static int queue_cmd_ring(struct tcmu_cmd
*tcmu_cmd
, sense_reason_t
*scsi_err
)
1012 struct tcmu_dev
*udev
= tcmu_cmd
->tcmu_dev
;
1013 struct se_cmd
*se_cmd
= tcmu_cmd
->se_cmd
;
1014 size_t base_command_size
, command_size
;
1015 struct tcmu_mailbox
*mb
= udev
->mb_addr
;
1016 struct tcmu_cmd_entry
*entry
;
1018 int iov_cnt
, iov_bidi_cnt
;
1019 uint32_t cmd_id
, cmd_head
;
1021 uint32_t blk_size
= udev
->data_blk_size
;
1022 /* size of data buffer needed */
1023 size_t data_length
= (size_t)tcmu_cmd
->dbi_cnt
* blk_size
;
1025 *scsi_err
= TCM_NO_SENSE
;
1027 if (test_bit(TCMU_DEV_BIT_BLOCKED
, &udev
->flags
)) {
1028 *scsi_err
= TCM_LUN_BUSY
;
1032 if (test_bit(TCMU_DEV_BIT_BROKEN
, &udev
->flags
)) {
1033 *scsi_err
= TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE
;
1037 if (!list_empty(&udev
->qfull_queue
))
1040 if (data_length
> (size_t)udev
->max_blocks
* blk_size
) {
1041 pr_warn("TCMU: Request of size %zu is too big for %zu data area\n",
1042 data_length
, (size_t)udev
->max_blocks
* blk_size
);
1043 *scsi_err
= TCM_INVALID_CDB_FIELD
;
1047 iov_cnt
= tcmu_alloc_data_space(udev
, tcmu_cmd
, &iov_bidi_cnt
);
1049 goto free_and_queue
;
1052 * Must be a certain minimum size for response sense info, but
1053 * also may be larger if the iov array is large.
1055 base_command_size
= tcmu_cmd_get_base_cmd_size(iov_cnt
);
1056 command_size
= tcmu_cmd_get_cmd_size(tcmu_cmd
, base_command_size
);
1058 if (command_size
> (udev
->cmdr_size
/ 2)) {
1059 pr_warn("TCMU: Request of size %zu is too big for %u cmd ring\n",
1060 command_size
, udev
->cmdr_size
);
1061 tcmu_cmd_free_data(tcmu_cmd
, tcmu_cmd
->dbi_cur
);
1062 *scsi_err
= TCM_INVALID_CDB_FIELD
;
1066 if (!is_ring_space_avail(udev
, command_size
))
1068 * Don't leave commands partially setup because the unmap
1069 * thread might need the blocks to make forward progress.
1071 goto free_and_queue
;
1073 if (xa_alloc(&udev
->commands
, &cmd_id
, tcmu_cmd
, XA_LIMIT(1, 0xffff),
1075 pr_err("tcmu: Could not allocate cmd id.\n");
1077 tcmu_cmd_free_data(tcmu_cmd
, tcmu_cmd
->dbi_cnt
);
1078 *scsi_err
= TCM_OUT_OF_RESOURCES
;
1081 tcmu_cmd
->cmd_id
= cmd_id
;
1083 pr_debug("allocated cmd id %u for cmd %p dev %s\n", tcmu_cmd
->cmd_id
,
1084 tcmu_cmd
, udev
->name
);
1086 cmd_head
= ring_insert_padding(udev
, command_size
);
1088 entry
= udev
->cmdr
+ cmd_head
;
1089 memset(entry
, 0, command_size
);
1090 tcmu_hdr_set_op(&entry
->hdr
.len_op
, TCMU_OP_CMD
);
1092 /* prepare iov list and copy data to data area if necessary */
1093 tcmu_cmd_reset_dbi_cur(tcmu_cmd
);
1094 iov
= &entry
->req
.iov
[0];
1096 if (se_cmd
->data_direction
== DMA_TO_DEVICE
||
1097 se_cmd
->se_cmd_flags
& SCF_BIDI
)
1098 scatter_data_area(udev
, tcmu_cmd
, &iov
);
1100 tcmu_setup_iovs(udev
, tcmu_cmd
, &iov
, se_cmd
->data_length
);
1102 entry
->req
.iov_cnt
= iov_cnt
- iov_bidi_cnt
;
1104 /* Handle BIDI commands */
1105 if (se_cmd
->se_cmd_flags
& SCF_BIDI
) {
1107 tcmu_setup_iovs(udev
, tcmu_cmd
, &iov
, tcmu_cmd
->data_len_bidi
);
1108 entry
->req
.iov_bidi_cnt
= iov_bidi_cnt
;
1111 tcmu_setup_cmd_timer(tcmu_cmd
, udev
->cmd_time_out
, &udev
->cmd_timer
);
1113 entry
->hdr
.cmd_id
= tcmu_cmd
->cmd_id
;
1115 tcmu_hdr_set_len(&entry
->hdr
.len_op
, command_size
);
1117 /* All offsets relative to mb_addr, not start of entry! */
1118 cdb_off
= CMDR_OFF
+ cmd_head
+ base_command_size
;
1119 memcpy((void *) mb
+ cdb_off
, se_cmd
->t_task_cdb
, scsi_command_size(se_cmd
->t_task_cdb
));
1120 entry
->req
.cdb_off
= cdb_off
;
1121 tcmu_flush_dcache_range(entry
, command_size
);
1123 UPDATE_HEAD(mb
->cmd_head
, command_size
, udev
->cmdr_size
);
1124 tcmu_flush_dcache_range(mb
, sizeof(*mb
));
1126 list_add_tail(&tcmu_cmd
->queue_entry
, &udev
->inflight_queue
);
1128 if (!test_bit(TCMU_DEV_BIT_PLUGGED
, &udev
->flags
))
1129 uio_event_notify(&udev
->uio_info
);
1134 tcmu_cmd_free_data(tcmu_cmd
, tcmu_cmd
->dbi_cur
);
1135 tcmu_cmd_reset_dbi_cur(tcmu_cmd
);
1138 if (add_to_qfull_queue(tcmu_cmd
)) {
1139 *scsi_err
= TCM_OUT_OF_RESOURCES
;
1147 * queue_tmr_ring - queue tmr info to ring or internally
1148 * @udev: related tcmu_dev
1149 * @tmr: tcmu_tmr containing tmr info to queue
1153 * 1 internally queued to wait for ring memory to free.
1156 queue_tmr_ring(struct tcmu_dev
*udev
, struct tcmu_tmr
*tmr
)
1158 struct tcmu_tmr_entry
*entry
;
1161 struct tcmu_mailbox
*mb
= udev
->mb_addr
;
1164 if (test_bit(TCMU_DEV_BIT_BROKEN
, &udev
->flags
))
1167 id_list_sz
= sizeof(tmr
->tmr_cmd_ids
[0]) * tmr
->tmr_cmd_cnt
;
1168 cmd_size
= round_up(sizeof(*entry
) + id_list_sz
, TCMU_OP_ALIGN_SIZE
);
1170 if (!list_empty(&udev
->tmr_queue
) ||
1171 !is_ring_space_avail(udev
, cmd_size
)) {
1172 list_add_tail(&tmr
->queue_entry
, &udev
->tmr_queue
);
1173 pr_debug("adding tmr %p on dev %s to TMR ring space wait queue\n",
1178 cmd_head
= ring_insert_padding(udev
, cmd_size
);
1180 entry
= udev
->cmdr
+ cmd_head
;
1181 memset(entry
, 0, cmd_size
);
1182 tcmu_hdr_set_op(&entry
->hdr
.len_op
, TCMU_OP_TMR
);
1183 tcmu_hdr_set_len(&entry
->hdr
.len_op
, cmd_size
);
1184 entry
->tmr_type
= tmr
->tmr_type
;
1185 entry
->cmd_cnt
= tmr
->tmr_cmd_cnt
;
1186 memcpy(&entry
->cmd_ids
[0], &tmr
->tmr_cmd_ids
[0], id_list_sz
);
1187 tcmu_flush_dcache_range(entry
, cmd_size
);
1189 UPDATE_HEAD(mb
->cmd_head
, cmd_size
, udev
->cmdr_size
);
1190 tcmu_flush_dcache_range(mb
, sizeof(*mb
));
1192 uio_event_notify(&udev
->uio_info
);
1200 static sense_reason_t
1201 tcmu_queue_cmd(struct se_cmd
*se_cmd
)
1203 struct se_device
*se_dev
= se_cmd
->se_dev
;
1204 struct tcmu_dev
*udev
= TCMU_DEV(se_dev
);
1205 struct tcmu_cmd
*tcmu_cmd
;
1206 sense_reason_t scsi_ret
= TCM_CHECK_CONDITION_ABORT_CMD
;
1209 tcmu_cmd
= tcmu_alloc_cmd(se_cmd
);
1211 return TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE
;
1213 mutex_lock(&udev
->cmdr_lock
);
1214 if (!(se_cmd
->transport_state
& CMD_T_ABORTED
))
1215 ret
= queue_cmd_ring(tcmu_cmd
, &scsi_ret
);
1217 tcmu_free_cmd(tcmu_cmd
);
1219 se_cmd
->priv
= tcmu_cmd
;
1220 mutex_unlock(&udev
->cmdr_lock
);
1224 static void tcmu_set_next_deadline(struct list_head
*queue
,
1225 struct timer_list
*timer
)
1227 struct tcmu_cmd
*cmd
;
1229 if (!list_empty(queue
)) {
1230 cmd
= list_first_entry(queue
, struct tcmu_cmd
, queue_entry
);
1231 mod_timer(timer
, cmd
->deadline
);
1237 tcmu_tmr_type(enum tcm_tmreq_table tmf
)
1240 case TMR_ABORT_TASK
: return TCMU_TMR_ABORT_TASK
;
1241 case TMR_ABORT_TASK_SET
: return TCMU_TMR_ABORT_TASK_SET
;
1242 case TMR_CLEAR_ACA
: return TCMU_TMR_CLEAR_ACA
;
1243 case TMR_CLEAR_TASK_SET
: return TCMU_TMR_CLEAR_TASK_SET
;
1244 case TMR_LUN_RESET
: return TCMU_TMR_LUN_RESET
;
1245 case TMR_TARGET_WARM_RESET
: return TCMU_TMR_TARGET_WARM_RESET
;
1246 case TMR_TARGET_COLD_RESET
: return TCMU_TMR_TARGET_COLD_RESET
;
1247 case TMR_LUN_RESET_PRO
: return TCMU_TMR_LUN_RESET_PRO
;
1248 default: return TCMU_TMR_UNKNOWN
;
1253 tcmu_tmr_notify(struct se_device
*se_dev
, enum tcm_tmreq_table tmf
,
1254 struct list_head
*cmd_list
)
1256 int i
= 0, cmd_cnt
= 0;
1257 bool unqueued
= false;
1258 uint16_t *cmd_ids
= NULL
;
1259 struct tcmu_cmd
*cmd
;
1260 struct se_cmd
*se_cmd
;
1261 struct tcmu_tmr
*tmr
;
1262 struct tcmu_dev
*udev
= TCMU_DEV(se_dev
);
1264 mutex_lock(&udev
->cmdr_lock
);
1266 /* First we check for aborted commands in qfull_queue */
1267 list_for_each_entry(se_cmd
, cmd_list
, state_list
) {
1272 /* Commands on qfull queue have no id yet */
1277 pr_debug("Removing aborted command %p from queue on dev %s.\n",
1280 list_del_init(&cmd
->queue_entry
);
1282 se_cmd
->priv
= NULL
;
1283 target_complete_cmd(se_cmd
, SAM_STAT_TASK_ABORTED
);
1287 tcmu_set_next_deadline(&udev
->qfull_queue
, &udev
->qfull_timer
);
1289 if (!test_bit(TCMU_DEV_BIT_TMR_NOTIFY
, &udev
->flags
))
1292 pr_debug("TMR event %d on dev %s, aborted cmds %d, afflicted cmd_ids %d\n",
1293 tcmu_tmr_type(tmf
), udev
->name
, i
, cmd_cnt
);
1295 tmr
= kmalloc(sizeof(*tmr
) + cmd_cnt
* sizeof(*cmd_ids
), GFP_NOIO
);
1299 tmr
->tmr_type
= tcmu_tmr_type(tmf
);
1300 tmr
->tmr_cmd_cnt
= cmd_cnt
;
1304 list_for_each_entry(se_cmd
, cmd_list
, state_list
) {
1309 tmr
->tmr_cmd_ids
[cmd_cnt
++] = cmd
->cmd_id
;
1313 queue_tmr_ring(udev
, tmr
);
1316 mutex_unlock(&udev
->cmdr_lock
);
1319 static bool tcmu_handle_completion(struct tcmu_cmd
*cmd
,
1320 struct tcmu_cmd_entry
*entry
, bool keep_buf
)
1322 struct se_cmd
*se_cmd
= cmd
->se_cmd
;
1323 struct tcmu_dev
*udev
= cmd
->tcmu_dev
;
1324 bool read_len_valid
= false;
1329 * cmd has been completed already from timeout, just reclaim
1330 * data area space and free cmd
1332 if (test_bit(TCMU_CMD_BIT_EXPIRED
, &cmd
->flags
)) {
1333 WARN_ON_ONCE(se_cmd
);
1336 if (test_bit(TCMU_CMD_BIT_KEEP_BUF
, &cmd
->flags
)) {
1337 pr_err("cmd_id %u already completed with KEEP_BUF, ring is broken\n",
1339 set_bit(TCMU_DEV_BIT_BROKEN
, &udev
->flags
);
1344 list_del_init(&cmd
->queue_entry
);
1346 tcmu_cmd_reset_dbi_cur(cmd
);
1348 if (entry
->hdr
.uflags
& TCMU_UFLAG_UNKNOWN_OP
) {
1349 pr_warn("TCMU: Userspace set UNKNOWN_OP flag on se_cmd %p\n",
1351 entry
->rsp
.scsi_status
= SAM_STAT_CHECK_CONDITION
;
1355 read_len
= se_cmd
->data_length
;
1356 if (se_cmd
->data_direction
== DMA_FROM_DEVICE
&&
1357 (entry
->hdr
.uflags
& TCMU_UFLAG_READ_LEN
) && entry
->rsp
.read_len
) {
1358 read_len_valid
= true;
1359 if (entry
->rsp
.read_len
< read_len
)
1360 read_len
= entry
->rsp
.read_len
;
1363 if (entry
->rsp
.scsi_status
== SAM_STAT_CHECK_CONDITION
) {
1364 transport_copy_sense_to_cmd(se_cmd
, entry
->rsp
.sense_buffer
);
1365 if (!read_len_valid
)
1368 se_cmd
->se_cmd_flags
|= SCF_TREAT_READ_AS_NORMAL
;
1370 if (se_cmd
->se_cmd_flags
& SCF_BIDI
) {
1371 /* Get Data-In buffer before clean up */
1372 gather_data_area(udev
, cmd
, true, read_len
);
1373 } else if (se_cmd
->data_direction
== DMA_FROM_DEVICE
) {
1374 gather_data_area(udev
, cmd
, false, read_len
);
1375 } else if (se_cmd
->data_direction
== DMA_TO_DEVICE
) {
1377 } else if (se_cmd
->data_direction
!= DMA_NONE
) {
1378 pr_warn("TCMU: data direction was %d!\n",
1379 se_cmd
->data_direction
);
1383 se_cmd
->priv
= NULL
;
1384 if (read_len_valid
) {
1385 pr_debug("read_len = %d\n", read_len
);
1386 target_complete_cmd_with_length(cmd
->se_cmd
,
1387 entry
->rsp
.scsi_status
, read_len
);
1389 target_complete_cmd(cmd
->se_cmd
, entry
->rsp
.scsi_status
);
1393 tcmu_cmd_free_data(cmd
, cmd
->dbi_cnt
);
1397 * Keep this command after completion, since userspace still
1398 * needs the data buffer. Mark it with TCMU_CMD_BIT_KEEP_BUF
1399 * and reset potential TCMU_CMD_BIT_EXPIRED, so we don't accept
1400 * a second completion later.
1401 * Userspace can free the buffer later by writing the cmd_id
1402 * to new action attribute free_kept_buf.
1404 clear_bit(TCMU_CMD_BIT_EXPIRED
, &cmd
->flags
);
1405 set_bit(TCMU_CMD_BIT_KEEP_BUF
, &cmd
->flags
);
1410 static int tcmu_run_tmr_queue(struct tcmu_dev
*udev
)
1412 struct tcmu_tmr
*tmr
, *tmp
;
1415 if (list_empty(&udev
->tmr_queue
))
1418 pr_debug("running %s's tmr queue\n", udev
->name
);
1420 list_splice_init(&udev
->tmr_queue
, &tmrs
);
1422 list_for_each_entry_safe(tmr
, tmp
, &tmrs
, queue_entry
) {
1423 list_del_init(&tmr
->queue_entry
);
1425 pr_debug("removing tmr %p on dev %s from queue\n",
1428 if (queue_tmr_ring(udev
, tmr
)) {
1429 pr_debug("ran out of space during tmr queue run\n");
1431 * tmr was requeued, so just put all tmrs back in
1434 list_splice_tail(&tmrs
, &udev
->tmr_queue
);
1442 static bool tcmu_handle_completions(struct tcmu_dev
*udev
)
1444 struct tcmu_mailbox
*mb
;
1445 struct tcmu_cmd
*cmd
;
1446 bool free_space
= false;
1448 if (test_bit(TCMU_DEV_BIT_BROKEN
, &udev
->flags
)) {
1449 pr_err("ring broken, not handling completions\n");
1454 tcmu_flush_dcache_range(mb
, sizeof(*mb
));
1456 while (udev
->cmdr_last_cleaned
!= READ_ONCE(mb
->cmd_tail
)) {
1458 struct tcmu_cmd_entry
*entry
= udev
->cmdr
+ udev
->cmdr_last_cleaned
;
1462 * Flush max. up to end of cmd ring since current entry might
1463 * be a padding that is shorter than sizeof(*entry)
1465 size_t ring_left
= head_to_end(udev
->cmdr_last_cleaned
,
1467 tcmu_flush_dcache_range(entry
, ring_left
< sizeof(*entry
) ?
1468 ring_left
: sizeof(*entry
));
1472 if (tcmu_hdr_get_op(entry
->hdr
.len_op
) == TCMU_OP_PAD
||
1473 tcmu_hdr_get_op(entry
->hdr
.len_op
) == TCMU_OP_TMR
) {
1474 UPDATE_HEAD(udev
->cmdr_last_cleaned
,
1475 tcmu_hdr_get_len(entry
->hdr
.len_op
),
1479 WARN_ON(tcmu_hdr_get_op(entry
->hdr
.len_op
) != TCMU_OP_CMD
);
1481 keep_buf
= !!(entry
->hdr
.uflags
& TCMU_UFLAG_KEEP_BUF
);
1483 cmd
= xa_load(&udev
->commands
, entry
->hdr
.cmd_id
);
1485 cmd
= xa_erase(&udev
->commands
, entry
->hdr
.cmd_id
);
1487 pr_err("cmd_id %u not found, ring is broken\n",
1489 set_bit(TCMU_DEV_BIT_BROKEN
, &udev
->flags
);
1493 if (!tcmu_handle_completion(cmd
, entry
, keep_buf
))
1496 UPDATE_HEAD(udev
->cmdr_last_cleaned
,
1497 tcmu_hdr_get_len(entry
->hdr
.len_op
),
1501 free_space
= tcmu_run_tmr_queue(udev
);
1503 if (atomic_read(&global_page_count
) > tcmu_global_max_pages
&&
1504 xa_empty(&udev
->commands
) && list_empty(&udev
->qfull_queue
)) {
1506 * Allocated blocks exceeded global block limit, currently no
1507 * more pending or waiting commands so try to reclaim blocks.
1509 schedule_delayed_work(&tcmu_unmap_work
, 0);
1511 if (udev
->cmd_time_out
)
1512 tcmu_set_next_deadline(&udev
->inflight_queue
, &udev
->cmd_timer
);
1517 static void tcmu_check_expired_ring_cmd(struct tcmu_cmd
*cmd
)
1519 struct se_cmd
*se_cmd
;
1521 if (!time_after_eq(jiffies
, cmd
->deadline
))
1524 set_bit(TCMU_CMD_BIT_EXPIRED
, &cmd
->flags
);
1525 list_del_init(&cmd
->queue_entry
);
1526 se_cmd
= cmd
->se_cmd
;
1527 se_cmd
->priv
= NULL
;
1530 pr_debug("Timing out inflight cmd %u on dev %s.\n",
1531 cmd
->cmd_id
, cmd
->tcmu_dev
->name
);
1533 target_complete_cmd(se_cmd
, SAM_STAT_CHECK_CONDITION
);
1536 static void tcmu_check_expired_queue_cmd(struct tcmu_cmd
*cmd
)
1538 struct se_cmd
*se_cmd
;
1540 if (!time_after_eq(jiffies
, cmd
->deadline
))
1543 pr_debug("Timing out queued cmd %p on dev %s.\n",
1544 cmd
, cmd
->tcmu_dev
->name
);
1546 list_del_init(&cmd
->queue_entry
);
1547 se_cmd
= cmd
->se_cmd
;
1550 se_cmd
->priv
= NULL
;
1551 target_complete_cmd(se_cmd
, SAM_STAT_TASK_SET_FULL
);
1554 static void tcmu_device_timedout(struct tcmu_dev
*udev
)
1556 spin_lock(&timed_out_udevs_lock
);
1557 if (list_empty(&udev
->timedout_entry
))
1558 list_add_tail(&udev
->timedout_entry
, &timed_out_udevs
);
1559 spin_unlock(&timed_out_udevs_lock
);
1561 schedule_delayed_work(&tcmu_unmap_work
, 0);
1564 static void tcmu_cmd_timedout(struct timer_list
*t
)
1566 struct tcmu_dev
*udev
= from_timer(udev
, t
, cmd_timer
);
1568 pr_debug("%s cmd timeout has expired\n", udev
->name
);
1569 tcmu_device_timedout(udev
);
1572 static void tcmu_qfull_timedout(struct timer_list
*t
)
1574 struct tcmu_dev
*udev
= from_timer(udev
, t
, qfull_timer
);
1576 pr_debug("%s qfull timeout has expired\n", udev
->name
);
1577 tcmu_device_timedout(udev
);
1580 static int tcmu_attach_hba(struct se_hba
*hba
, u32 host_id
)
1582 struct tcmu_hba
*tcmu_hba
;
1584 tcmu_hba
= kzalloc(sizeof(struct tcmu_hba
), GFP_KERNEL
);
1588 tcmu_hba
->host_id
= host_id
;
1589 hba
->hba_ptr
= tcmu_hba
;
1594 static void tcmu_detach_hba(struct se_hba
*hba
)
1596 kfree(hba
->hba_ptr
);
1597 hba
->hba_ptr
= NULL
;
1600 static struct se_device
*tcmu_alloc_device(struct se_hba
*hba
, const char *name
)
1602 struct tcmu_dev
*udev
;
1604 udev
= kzalloc(sizeof(struct tcmu_dev
), GFP_KERNEL
);
1607 kref_init(&udev
->kref
);
1609 udev
->name
= kstrdup(name
, GFP_KERNEL
);
1616 udev
->cmd_time_out
= TCMU_TIME_OUT
;
1617 udev
->qfull_time_out
= -1;
1619 udev
->data_pages_per_blk
= DATA_PAGES_PER_BLK_DEF
;
1620 udev
->max_blocks
= DATA_AREA_PAGES_DEF
/ udev
->data_pages_per_blk
;
1621 udev
->data_area_mb
= TCMU_PAGES_TO_MBS(DATA_AREA_PAGES_DEF
);
1623 mutex_init(&udev
->cmdr_lock
);
1625 INIT_LIST_HEAD(&udev
->node
);
1626 INIT_LIST_HEAD(&udev
->timedout_entry
);
1627 INIT_LIST_HEAD(&udev
->qfull_queue
);
1628 INIT_LIST_HEAD(&udev
->tmr_queue
);
1629 INIT_LIST_HEAD(&udev
->inflight_queue
);
1630 xa_init_flags(&udev
->commands
, XA_FLAGS_ALLOC1
);
1632 timer_setup(&udev
->qfull_timer
, tcmu_qfull_timedout
, 0);
1633 timer_setup(&udev
->cmd_timer
, tcmu_cmd_timedout
, 0);
1635 xa_init(&udev
->data_pages
);
1637 return &udev
->se_dev
;
1640 static void tcmu_dev_call_rcu(struct rcu_head
*p
)
1642 struct se_device
*dev
= container_of(p
, struct se_device
, rcu_head
);
1643 struct tcmu_dev
*udev
= TCMU_DEV(dev
);
1645 kfree(udev
->uio_info
.name
);
1650 static int tcmu_check_and_free_pending_cmd(struct tcmu_cmd
*cmd
)
1652 if (test_bit(TCMU_CMD_BIT_EXPIRED
, &cmd
->flags
) ||
1653 test_bit(TCMU_CMD_BIT_KEEP_BUF
, &cmd
->flags
)) {
1654 kmem_cache_free(tcmu_cmd_cache
, cmd
);
1660 static u32
tcmu_blocks_release(struct tcmu_dev
*udev
, unsigned long first
,
1663 XA_STATE(xas
, &udev
->data_pages
, first
* udev
->data_pages_per_blk
);
1665 u32 pages_freed
= 0;
1668 xas_for_each(&xas
, page
, (last
+ 1) * udev
->data_pages_per_blk
- 1) {
1669 xas_store(&xas
, NULL
);
1675 atomic_sub(pages_freed
, &global_page_count
);
1680 static void tcmu_remove_all_queued_tmr(struct tcmu_dev
*udev
)
1682 struct tcmu_tmr
*tmr
, *tmp
;
1684 list_for_each_entry_safe(tmr
, tmp
, &udev
->tmr_queue
, queue_entry
) {
1685 list_del_init(&tmr
->queue_entry
);
1690 static void tcmu_dev_kref_release(struct kref
*kref
)
1692 struct tcmu_dev
*udev
= container_of(kref
, struct tcmu_dev
, kref
);
1693 struct se_device
*dev
= &udev
->se_dev
;
1694 struct tcmu_cmd
*cmd
;
1695 bool all_expired
= true;
1698 vfree(udev
->mb_addr
);
1699 udev
->mb_addr
= NULL
;
1701 spin_lock_bh(&timed_out_udevs_lock
);
1702 if (!list_empty(&udev
->timedout_entry
))
1703 list_del(&udev
->timedout_entry
);
1704 spin_unlock_bh(&timed_out_udevs_lock
);
1706 /* Upper layer should drain all requests before calling this */
1707 mutex_lock(&udev
->cmdr_lock
);
1708 xa_for_each(&udev
->commands
, i
, cmd
) {
1709 if (tcmu_check_and_free_pending_cmd(cmd
) != 0)
1710 all_expired
= false;
1712 /* There can be left over TMR cmds. Remove them. */
1713 tcmu_remove_all_queued_tmr(udev
);
1714 if (!list_empty(&udev
->qfull_queue
))
1715 all_expired
= false;
1716 xa_destroy(&udev
->commands
);
1717 WARN_ON(!all_expired
);
1719 tcmu_blocks_release(udev
, 0, udev
->dbi_max
);
1720 bitmap_free(udev
->data_bitmap
);
1721 mutex_unlock(&udev
->cmdr_lock
);
1723 pr_debug("dev_kref_release\n");
1725 call_rcu(&dev
->rcu_head
, tcmu_dev_call_rcu
);
1728 static void run_qfull_queue(struct tcmu_dev
*udev
, bool fail
)
1730 struct tcmu_cmd
*tcmu_cmd
, *tmp_cmd
;
1732 sense_reason_t scsi_ret
;
1735 if (list_empty(&udev
->qfull_queue
))
1738 pr_debug("running %s's cmdr queue forcefail %d\n", udev
->name
, fail
);
1740 list_splice_init(&udev
->qfull_queue
, &cmds
);
1742 list_for_each_entry_safe(tcmu_cmd
, tmp_cmd
, &cmds
, queue_entry
) {
1743 list_del_init(&tcmu_cmd
->queue_entry
);
1745 pr_debug("removing cmd %p on dev %s from queue\n",
1746 tcmu_cmd
, udev
->name
);
1750 * We were not able to even start the command, so
1751 * fail with busy to allow a retry in case runner
1752 * was only temporarily down. If the device is being
1753 * removed then LIO core will do the right thing and
1756 tcmu_cmd
->se_cmd
->priv
= NULL
;
1757 target_complete_cmd(tcmu_cmd
->se_cmd
, SAM_STAT_BUSY
);
1758 tcmu_free_cmd(tcmu_cmd
);
1762 ret
= queue_cmd_ring(tcmu_cmd
, &scsi_ret
);
1764 pr_debug("cmd %p on dev %s failed with %u\n",
1765 tcmu_cmd
, udev
->name
, scsi_ret
);
1767 * Ignore scsi_ret for now. target_complete_cmd
1770 tcmu_cmd
->se_cmd
->priv
= NULL
;
1771 target_complete_cmd(tcmu_cmd
->se_cmd
,
1772 SAM_STAT_CHECK_CONDITION
);
1773 tcmu_free_cmd(tcmu_cmd
);
1774 } else if (ret
> 0) {
1775 pr_debug("ran out of space during cmdr queue run\n");
1777 * cmd was requeued, so just put all cmds back in
1780 list_splice_tail(&cmds
, &udev
->qfull_queue
);
1785 tcmu_set_next_deadline(&udev
->qfull_queue
, &udev
->qfull_timer
);
1788 static int tcmu_irqcontrol(struct uio_info
*info
, s32 irq_on
)
1790 struct tcmu_dev
*udev
= container_of(info
, struct tcmu_dev
, uio_info
);
1792 mutex_lock(&udev
->cmdr_lock
);
1793 if (tcmu_handle_completions(udev
))
1794 run_qfull_queue(udev
, false);
1795 mutex_unlock(&udev
->cmdr_lock
);
1801 * mmap code from uio.c. Copied here because we want to hook mmap()
1802 * and this stuff must come along.
1804 static int tcmu_find_mem_index(struct vm_area_struct
*vma
)
1806 struct tcmu_dev
*udev
= vma
->vm_private_data
;
1807 struct uio_info
*info
= &udev
->uio_info
;
1809 if (vma
->vm_pgoff
< MAX_UIO_MAPS
) {
1810 if (info
->mem
[vma
->vm_pgoff
].size
== 0)
1812 return (int)vma
->vm_pgoff
;
1817 static struct page
*tcmu_try_get_data_page(struct tcmu_dev
*udev
, uint32_t dpi
)
1821 mutex_lock(&udev
->cmdr_lock
);
1822 page
= xa_load(&udev
->data_pages
, dpi
);
1825 mutex_unlock(&udev
->cmdr_lock
);
1830 * Userspace messed up and passed in a address not in the
1831 * data iov passed to it.
1833 pr_err("Invalid addr to data page mapping (dpi %u) on device %s\n",
1835 mutex_unlock(&udev
->cmdr_lock
);
1840 static void tcmu_vma_open(struct vm_area_struct
*vma
)
1842 struct tcmu_dev
*udev
= vma
->vm_private_data
;
1844 pr_debug("vma_open\n");
1846 kref_get(&udev
->kref
);
1849 static void tcmu_vma_close(struct vm_area_struct
*vma
)
1851 struct tcmu_dev
*udev
= vma
->vm_private_data
;
1853 pr_debug("vma_close\n");
1855 /* release ref from tcmu_vma_open */
1856 kref_put(&udev
->kref
, tcmu_dev_kref_release
);
1859 static vm_fault_t
tcmu_vma_fault(struct vm_fault
*vmf
)
1861 struct tcmu_dev
*udev
= vmf
->vma
->vm_private_data
;
1862 struct uio_info
*info
= &udev
->uio_info
;
1864 unsigned long offset
;
1867 int mi
= tcmu_find_mem_index(vmf
->vma
);
1869 return VM_FAULT_SIGBUS
;
1872 * We need to subtract mi because userspace uses offset = N*PAGE_SIZE
1875 offset
= (vmf
->pgoff
- mi
) << PAGE_SHIFT
;
1877 if (offset
< udev
->data_off
) {
1878 /* For the vmalloc()ed cmd area pages */
1879 addr
= (void *)(unsigned long)info
->mem
[mi
].addr
+ offset
;
1880 page
= vmalloc_to_page(addr
);
1885 /* For the dynamically growing data area pages */
1886 dpi
= (offset
- udev
->data_off
) / PAGE_SIZE
;
1887 page
= tcmu_try_get_data_page(udev
, dpi
);
1889 return VM_FAULT_SIGBUS
;
1896 static const struct vm_operations_struct tcmu_vm_ops
= {
1897 .open
= tcmu_vma_open
,
1898 .close
= tcmu_vma_close
,
1899 .fault
= tcmu_vma_fault
,
1902 static int tcmu_mmap(struct uio_info
*info
, struct vm_area_struct
*vma
)
1904 struct tcmu_dev
*udev
= container_of(info
, struct tcmu_dev
, uio_info
);
1906 vma
->vm_flags
|= VM_DONTEXPAND
| VM_DONTDUMP
;
1907 vma
->vm_ops
= &tcmu_vm_ops
;
1909 vma
->vm_private_data
= udev
;
1911 /* Ensure the mmap is exactly the right size */
1912 if (vma_pages(vma
) != udev
->mmap_pages
)
1920 static int tcmu_open(struct uio_info
*info
, struct inode
*inode
)
1922 struct tcmu_dev
*udev
= container_of(info
, struct tcmu_dev
, uio_info
);
1924 /* O_EXCL not supported for char devs, so fake it? */
1925 if (test_and_set_bit(TCMU_DEV_BIT_OPEN
, &udev
->flags
))
1928 udev
->inode
= inode
;
1935 static int tcmu_release(struct uio_info
*info
, struct inode
*inode
)
1937 struct tcmu_dev
*udev
= container_of(info
, struct tcmu_dev
, uio_info
);
1938 struct tcmu_cmd
*cmd
;
1942 mutex_lock(&udev
->cmdr_lock
);
1944 xa_for_each(&udev
->commands
, i
, cmd
) {
1945 /* Cmds with KEEP_BUF set are no longer on the ring, but
1946 * userspace still holds the data buffer. If userspace closes
1947 * we implicitly free these cmds and buffers, since after new
1948 * open the (new ?) userspace cannot find the cmd in the ring
1949 * and thus never will release the buffer by writing cmd_id to
1950 * free_kept_buf action attribute.
1952 if (!test_bit(TCMU_CMD_BIT_KEEP_BUF
, &cmd
->flags
))
1954 pr_debug("removing KEEP_BUF cmd %u on dev %s from ring\n",
1955 cmd
->cmd_id
, udev
->name
);
1958 xa_erase(&udev
->commands
, i
);
1959 tcmu_cmd_free_data(cmd
, cmd
->dbi_cnt
);
1963 * We only freed data space, not ring space. Therefore we dont call
1964 * run_tmr_queue, but call run_qfull_queue if tmr_list is empty.
1966 if (freed
&& list_empty(&udev
->tmr_queue
))
1967 run_qfull_queue(udev
, false);
1969 mutex_unlock(&udev
->cmdr_lock
);
1971 clear_bit(TCMU_DEV_BIT_OPEN
, &udev
->flags
);
1973 pr_debug("close\n");
1978 static int tcmu_init_genl_cmd_reply(struct tcmu_dev
*udev
, int cmd
)
1980 struct tcmu_nl_cmd
*nl_cmd
= &udev
->curr_nl_cmd
;
1982 if (!tcmu_kern_cmd_reply_supported
)
1985 if (udev
->nl_reply_supported
<= 0)
1988 mutex_lock(&tcmu_nl_cmd_mutex
);
1990 if (tcmu_netlink_blocked
) {
1991 mutex_unlock(&tcmu_nl_cmd_mutex
);
1992 pr_warn("Failing nl cmd %d on %s. Interface is blocked.\n", cmd
,
1997 if (nl_cmd
->cmd
!= TCMU_CMD_UNSPEC
) {
1998 mutex_unlock(&tcmu_nl_cmd_mutex
);
1999 pr_warn("netlink cmd %d already executing on %s\n",
2000 nl_cmd
->cmd
, udev
->name
);
2004 memset(nl_cmd
, 0, sizeof(*nl_cmd
));
2006 nl_cmd
->udev
= udev
;
2007 init_completion(&nl_cmd
->complete
);
2008 INIT_LIST_HEAD(&nl_cmd
->nl_list
);
2010 list_add_tail(&nl_cmd
->nl_list
, &tcmu_nl_cmd_list
);
2012 mutex_unlock(&tcmu_nl_cmd_mutex
);
2016 static void tcmu_destroy_genl_cmd_reply(struct tcmu_dev
*udev
)
2018 struct tcmu_nl_cmd
*nl_cmd
= &udev
->curr_nl_cmd
;
2020 if (!tcmu_kern_cmd_reply_supported
)
2023 if (udev
->nl_reply_supported
<= 0)
2026 mutex_lock(&tcmu_nl_cmd_mutex
);
2028 list_del(&nl_cmd
->nl_list
);
2029 memset(nl_cmd
, 0, sizeof(*nl_cmd
));
2031 mutex_unlock(&tcmu_nl_cmd_mutex
);
2034 static int tcmu_wait_genl_cmd_reply(struct tcmu_dev
*udev
)
2036 struct tcmu_nl_cmd
*nl_cmd
= &udev
->curr_nl_cmd
;
2039 if (!tcmu_kern_cmd_reply_supported
)
2042 if (udev
->nl_reply_supported
<= 0)
2045 pr_debug("sleeping for nl reply\n");
2046 wait_for_completion(&nl_cmd
->complete
);
2048 mutex_lock(&tcmu_nl_cmd_mutex
);
2049 nl_cmd
->cmd
= TCMU_CMD_UNSPEC
;
2050 ret
= nl_cmd
->status
;
2051 mutex_unlock(&tcmu_nl_cmd_mutex
);
2056 static int tcmu_netlink_event_init(struct tcmu_dev
*udev
,
2057 enum tcmu_genl_cmd cmd
,
2058 struct sk_buff
**buf
, void **hdr
)
2060 struct sk_buff
*skb
;
2064 skb
= genlmsg_new(NLMSG_GOODSIZE
, GFP_KERNEL
);
2068 msg_header
= genlmsg_put(skb
, 0, 0, &tcmu_genl_family
, 0, cmd
);
2072 ret
= nla_put_string(skb
, TCMU_ATTR_DEVICE
, udev
->uio_info
.name
);
2076 ret
= nla_put_u32(skb
, TCMU_ATTR_MINOR
, udev
->uio_info
.uio_dev
->minor
);
2080 ret
= nla_put_u32(skb
, TCMU_ATTR_DEVICE_ID
, udev
->se_dev
.dev_index
);
2093 static int tcmu_netlink_event_send(struct tcmu_dev
*udev
,
2094 enum tcmu_genl_cmd cmd
,
2095 struct sk_buff
*skb
, void *msg_header
)
2099 genlmsg_end(skb
, msg_header
);
2101 ret
= tcmu_init_genl_cmd_reply(udev
, cmd
);
2107 ret
= genlmsg_multicast_allns(&tcmu_genl_family
, skb
, 0,
2108 TCMU_MCGRP_CONFIG
, GFP_KERNEL
);
2110 /* Wait during an add as the listener may not be up yet */
2112 (ret
== -ESRCH
&& cmd
== TCMU_CMD_ADDED_DEVICE
))
2113 return tcmu_wait_genl_cmd_reply(udev
);
2115 tcmu_destroy_genl_cmd_reply(udev
);
2120 static int tcmu_send_dev_add_event(struct tcmu_dev
*udev
)
2122 struct sk_buff
*skb
= NULL
;
2123 void *msg_header
= NULL
;
2126 ret
= tcmu_netlink_event_init(udev
, TCMU_CMD_ADDED_DEVICE
, &skb
,
2130 return tcmu_netlink_event_send(udev
, TCMU_CMD_ADDED_DEVICE
, skb
,
2134 static int tcmu_send_dev_remove_event(struct tcmu_dev
*udev
)
2136 struct sk_buff
*skb
= NULL
;
2137 void *msg_header
= NULL
;
2140 ret
= tcmu_netlink_event_init(udev
, TCMU_CMD_REMOVED_DEVICE
,
2144 return tcmu_netlink_event_send(udev
, TCMU_CMD_REMOVED_DEVICE
,
2148 static int tcmu_update_uio_info(struct tcmu_dev
*udev
)
2150 struct tcmu_hba
*hba
= udev
->hba
->hba_ptr
;
2151 struct uio_info
*info
;
2154 info
= &udev
->uio_info
;
2156 if (udev
->dev_config
[0])
2157 str
= kasprintf(GFP_KERNEL
, "tcm-user/%u/%s/%s", hba
->host_id
,
2158 udev
->name
, udev
->dev_config
);
2160 str
= kasprintf(GFP_KERNEL
, "tcm-user/%u/%s", hba
->host_id
,
2165 /* If the old string exists, free it */
2172 static int tcmu_configure_device(struct se_device
*dev
)
2174 struct tcmu_dev
*udev
= TCMU_DEV(dev
);
2175 struct uio_info
*info
;
2176 struct tcmu_mailbox
*mb
;
2180 ret
= tcmu_update_uio_info(udev
);
2184 info
= &udev
->uio_info
;
2186 mutex_lock(&udev
->cmdr_lock
);
2187 udev
->data_bitmap
= bitmap_zalloc(udev
->max_blocks
, GFP_KERNEL
);
2188 mutex_unlock(&udev
->cmdr_lock
);
2189 if (!udev
->data_bitmap
) {
2191 goto err_bitmap_alloc
;
2194 mb
= vzalloc(MB_CMDR_SIZE
);
2200 /* mailbox fits in first part of CMDR space */
2202 udev
->cmdr
= (void *)mb
+ CMDR_OFF
;
2203 udev
->cmdr_size
= CMDR_SIZE
;
2204 udev
->data_off
= MB_CMDR_SIZE
;
2205 data_size
= TCMU_MBS_TO_PAGES(udev
->data_area_mb
) << PAGE_SHIFT
;
2206 udev
->mmap_pages
= (data_size
+ MB_CMDR_SIZE
) >> PAGE_SHIFT
;
2207 udev
->data_blk_size
= udev
->data_pages_per_blk
* PAGE_SIZE
;
2208 udev
->dbi_thresh
= 0; /* Default in Idle state */
2210 /* Initialise the mailbox of the ring buffer */
2211 mb
->version
= TCMU_MAILBOX_VERSION
;
2212 mb
->flags
= TCMU_MAILBOX_FLAG_CAP_OOOC
|
2213 TCMU_MAILBOX_FLAG_CAP_READ_LEN
|
2214 TCMU_MAILBOX_FLAG_CAP_TMR
|
2215 TCMU_MAILBOX_FLAG_CAP_KEEP_BUF
;
2216 mb
->cmdr_off
= CMDR_OFF
;
2217 mb
->cmdr_size
= udev
->cmdr_size
;
2219 WARN_ON(!PAGE_ALIGNED(udev
->data_off
));
2220 WARN_ON(data_size
% PAGE_SIZE
);
2222 info
->version
= __stringify(TCMU_MAILBOX_VERSION
);
2224 info
->mem
[0].name
= "tcm-user command & data buffer";
2225 info
->mem
[0].addr
= (phys_addr_t
)(uintptr_t)udev
->mb_addr
;
2226 info
->mem
[0].size
= data_size
+ MB_CMDR_SIZE
;
2227 info
->mem
[0].memtype
= UIO_MEM_NONE
;
2229 info
->irqcontrol
= tcmu_irqcontrol
;
2230 info
->irq
= UIO_IRQ_CUSTOM
;
2232 info
->mmap
= tcmu_mmap
;
2233 info
->open
= tcmu_open
;
2234 info
->release
= tcmu_release
;
2236 ret
= uio_register_device(tcmu_root_device
, info
);
2240 /* User can set hw_block_size before enable the device */
2241 if (dev
->dev_attrib
.hw_block_size
== 0)
2242 dev
->dev_attrib
.hw_block_size
= 512;
2243 /* Other attributes can be configured in userspace */
2244 if (!dev
->dev_attrib
.hw_max_sectors
)
2245 dev
->dev_attrib
.hw_max_sectors
= 128;
2246 if (!dev
->dev_attrib
.emulate_write_cache
)
2247 dev
->dev_attrib
.emulate_write_cache
= 0;
2248 dev
->dev_attrib
.hw_queue_depth
= 128;
2250 /* If user didn't explicitly disable netlink reply support, use
2251 * module scope setting.
2253 if (udev
->nl_reply_supported
>= 0)
2254 udev
->nl_reply_supported
= tcmu_kern_cmd_reply_supported
;
2257 * Get a ref incase userspace does a close on the uio device before
2258 * LIO has initiated tcmu_free_device.
2260 kref_get(&udev
->kref
);
2262 ret
= tcmu_send_dev_add_event(udev
);
2266 mutex_lock(&root_udev_mutex
);
2267 list_add(&udev
->node
, &root_udev
);
2268 mutex_unlock(&root_udev_mutex
);
2273 kref_put(&udev
->kref
, tcmu_dev_kref_release
);
2274 uio_unregister_device(&udev
->uio_info
);
2276 vfree(udev
->mb_addr
);
2277 udev
->mb_addr
= NULL
;
2279 bitmap_free(udev
->data_bitmap
);
2280 udev
->data_bitmap
= NULL
;
2288 static void tcmu_free_device(struct se_device
*dev
)
2290 struct tcmu_dev
*udev
= TCMU_DEV(dev
);
2292 /* release ref from init */
2293 kref_put(&udev
->kref
, tcmu_dev_kref_release
);
2296 static void tcmu_destroy_device(struct se_device
*dev
)
2298 struct tcmu_dev
*udev
= TCMU_DEV(dev
);
2300 del_timer_sync(&udev
->cmd_timer
);
2301 del_timer_sync(&udev
->qfull_timer
);
2303 mutex_lock(&root_udev_mutex
);
2304 list_del(&udev
->node
);
2305 mutex_unlock(&root_udev_mutex
);
2307 tcmu_send_dev_remove_event(udev
);
2309 uio_unregister_device(&udev
->uio_info
);
2311 /* release ref from configure */
2312 kref_put(&udev
->kref
, tcmu_dev_kref_release
);
2315 static void tcmu_unblock_dev(struct tcmu_dev
*udev
)
2317 mutex_lock(&udev
->cmdr_lock
);
2318 clear_bit(TCMU_DEV_BIT_BLOCKED
, &udev
->flags
);
2319 mutex_unlock(&udev
->cmdr_lock
);
2322 static void tcmu_block_dev(struct tcmu_dev
*udev
)
2324 mutex_lock(&udev
->cmdr_lock
);
2326 if (test_and_set_bit(TCMU_DEV_BIT_BLOCKED
, &udev
->flags
))
2329 /* complete IO that has executed successfully */
2330 tcmu_handle_completions(udev
);
2331 /* fail IO waiting to be queued */
2332 run_qfull_queue(udev
, true);
2335 mutex_unlock(&udev
->cmdr_lock
);
2338 static void tcmu_reset_ring(struct tcmu_dev
*udev
, u8 err_level
)
2340 struct tcmu_mailbox
*mb
;
2341 struct tcmu_cmd
*cmd
;
2344 mutex_lock(&udev
->cmdr_lock
);
2346 xa_for_each(&udev
->commands
, i
, cmd
) {
2347 pr_debug("removing cmd %u on dev %s from ring %s\n",
2348 cmd
->cmd_id
, udev
->name
,
2349 test_bit(TCMU_CMD_BIT_EXPIRED
, &cmd
->flags
) ?
2351 (test_bit(TCMU_CMD_BIT_KEEP_BUF
, &cmd
->flags
) ?
2352 "(is keep buffer)" : ""));
2354 xa_erase(&udev
->commands
, i
);
2355 if (!test_bit(TCMU_CMD_BIT_EXPIRED
, &cmd
->flags
) &&
2356 !test_bit(TCMU_CMD_BIT_KEEP_BUF
, &cmd
->flags
)) {
2357 WARN_ON(!cmd
->se_cmd
);
2358 list_del_init(&cmd
->queue_entry
);
2359 cmd
->se_cmd
->priv
= NULL
;
2360 if (err_level
== 1) {
2362 * Userspace was not able to start the
2363 * command or it is retryable.
2365 target_complete_cmd(cmd
->se_cmd
, SAM_STAT_BUSY
);
2368 target_complete_cmd(cmd
->se_cmd
,
2369 SAM_STAT_CHECK_CONDITION
);
2372 tcmu_cmd_free_data(cmd
, cmd
->dbi_cnt
);
2377 tcmu_flush_dcache_range(mb
, sizeof(*mb
));
2378 pr_debug("mb last %u head %u tail %u\n", udev
->cmdr_last_cleaned
,
2379 mb
->cmd_tail
, mb
->cmd_head
);
2381 udev
->cmdr_last_cleaned
= 0;
2384 tcmu_flush_dcache_range(mb
, sizeof(*mb
));
2385 clear_bit(TCMU_DEV_BIT_BROKEN
, &udev
->flags
);
2387 del_timer(&udev
->cmd_timer
);
2390 * ring is empty and qfull queue never contains aborted commands.
2391 * So TMRs in tmr queue do not contain relevant cmd_ids.
2392 * After a ring reset userspace should do a fresh start, so
2393 * even LUN RESET message is no longer relevant.
2394 * Therefore remove all TMRs from qfull queue
2396 tcmu_remove_all_queued_tmr(udev
);
2398 run_qfull_queue(udev
, false);
2400 mutex_unlock(&udev
->cmdr_lock
);
2404 Opt_dev_config
, Opt_dev_size
, Opt_hw_block_size
, Opt_hw_max_sectors
,
2405 Opt_nl_reply_supported
, Opt_max_data_area_mb
, Opt_data_pages_per_blk
,
2409 static match_table_t tokens
= {
2410 {Opt_dev_config
, "dev_config=%s"},
2411 {Opt_dev_size
, "dev_size=%s"},
2412 {Opt_hw_block_size
, "hw_block_size=%d"},
2413 {Opt_hw_max_sectors
, "hw_max_sectors=%d"},
2414 {Opt_nl_reply_supported
, "nl_reply_supported=%d"},
2415 {Opt_max_data_area_mb
, "max_data_area_mb=%d"},
2416 {Opt_data_pages_per_blk
, "data_pages_per_blk=%d"},
2420 static int tcmu_set_dev_attrib(substring_t
*arg
, u32
*dev_attrib
)
2424 ret
= match_int(arg
, &val
);
2426 pr_err("match_int() failed for dev attrib. Error %d.\n",
2432 pr_err("Invalid dev attrib value %d. Must be greater than zero.\n",
2440 static int tcmu_set_max_blocks_param(struct tcmu_dev
*udev
, substring_t
*arg
)
2443 uint32_t pages_per_blk
= udev
->data_pages_per_blk
;
2445 ret
= match_int(arg
, &val
);
2447 pr_err("match_int() failed for max_data_area_mb=. Error %d.\n",
2452 pr_err("Invalid max_data_area %d.\n", val
);
2455 if (val
> TCMU_PAGES_TO_MBS(tcmu_global_max_pages
)) {
2456 pr_err("%d is too large. Adjusting max_data_area_mb to global limit of %u\n",
2457 val
, TCMU_PAGES_TO_MBS(tcmu_global_max_pages
));
2458 val
= TCMU_PAGES_TO_MBS(tcmu_global_max_pages
);
2460 if (TCMU_MBS_TO_PAGES(val
) < pages_per_blk
) {
2461 pr_err("Invalid max_data_area %d (%zu pages): smaller than data_pages_per_blk (%u pages).\n",
2462 val
, TCMU_MBS_TO_PAGES(val
), pages_per_blk
);
2466 mutex_lock(&udev
->cmdr_lock
);
2467 if (udev
->data_bitmap
) {
2468 pr_err("Cannot set max_data_area_mb after it has been enabled.\n");
2473 udev
->data_area_mb
= val
;
2474 udev
->max_blocks
= TCMU_MBS_TO_PAGES(val
) / pages_per_blk
;
2477 mutex_unlock(&udev
->cmdr_lock
);
2481 static int tcmu_set_data_pages_per_blk(struct tcmu_dev
*udev
, substring_t
*arg
)
2485 ret
= match_int(arg
, &val
);
2487 pr_err("match_int() failed for data_pages_per_blk=. Error %d.\n",
2492 if (val
> TCMU_MBS_TO_PAGES(udev
->data_area_mb
)) {
2493 pr_err("Invalid data_pages_per_blk %d: greater than max_data_area_mb %d -> %zd pages).\n",
2494 val
, udev
->data_area_mb
,
2495 TCMU_MBS_TO_PAGES(udev
->data_area_mb
));
2499 mutex_lock(&udev
->cmdr_lock
);
2500 if (udev
->data_bitmap
) {
2501 pr_err("Cannot set data_pages_per_blk after it has been enabled.\n");
2506 udev
->data_pages_per_blk
= val
;
2507 udev
->max_blocks
= TCMU_MBS_TO_PAGES(udev
->data_area_mb
) / val
;
2510 mutex_unlock(&udev
->cmdr_lock
);
2514 static ssize_t
tcmu_set_configfs_dev_params(struct se_device
*dev
,
2515 const char *page
, ssize_t count
)
2517 struct tcmu_dev
*udev
= TCMU_DEV(dev
);
2518 char *orig
, *ptr
, *opts
;
2519 substring_t args
[MAX_OPT_ARGS
];
2522 opts
= kstrdup(page
, GFP_KERNEL
);
2528 while ((ptr
= strsep(&opts
, ",\n")) != NULL
) {
2532 token
= match_token(ptr
, tokens
, args
);
2534 case Opt_dev_config
:
2535 if (match_strlcpy(udev
->dev_config
, &args
[0],
2536 TCMU_CONFIG_LEN
) == 0) {
2540 pr_debug("TCMU: Referencing Path: %s\n", udev
->dev_config
);
2543 ret
= match_u64(&args
[0], &udev
->dev_size
);
2545 pr_err("match_u64() failed for dev_size=. Error %d.\n",
2548 case Opt_hw_block_size
:
2549 ret
= tcmu_set_dev_attrib(&args
[0],
2550 &(dev
->dev_attrib
.hw_block_size
));
2552 case Opt_hw_max_sectors
:
2553 ret
= tcmu_set_dev_attrib(&args
[0],
2554 &(dev
->dev_attrib
.hw_max_sectors
));
2556 case Opt_nl_reply_supported
:
2557 ret
= match_int(&args
[0], &udev
->nl_reply_supported
);
2559 pr_err("match_int() failed for nl_reply_supported=. Error %d.\n",
2562 case Opt_max_data_area_mb
:
2563 ret
= tcmu_set_max_blocks_param(udev
, &args
[0]);
2565 case Opt_data_pages_per_blk
:
2566 ret
= tcmu_set_data_pages_per_blk(udev
, &args
[0]);
2577 return (!ret
) ? count
: ret
;
2580 static ssize_t
tcmu_show_configfs_dev_params(struct se_device
*dev
, char *b
)
2582 struct tcmu_dev
*udev
= TCMU_DEV(dev
);
2585 bl
= sprintf(b
+ bl
, "Config: %s ",
2586 udev
->dev_config
[0] ? udev
->dev_config
: "NULL");
2587 bl
+= sprintf(b
+ bl
, "Size: %llu ", udev
->dev_size
);
2588 bl
+= sprintf(b
+ bl
, "MaxDataAreaMB: %u ", udev
->data_area_mb
);
2589 bl
+= sprintf(b
+ bl
, "DataPagesPerBlk: %u\n", udev
->data_pages_per_blk
);
2594 static sector_t
tcmu_get_blocks(struct se_device
*dev
)
2596 struct tcmu_dev
*udev
= TCMU_DEV(dev
);
2598 return div_u64(udev
->dev_size
- dev
->dev_attrib
.block_size
,
2599 dev
->dev_attrib
.block_size
);
2602 static sense_reason_t
2603 tcmu_parse_cdb(struct se_cmd
*cmd
)
2605 return passthrough_parse_cdb(cmd
, tcmu_queue_cmd
);
2608 static ssize_t
tcmu_cmd_time_out_show(struct config_item
*item
, char *page
)
2610 struct se_dev_attrib
*da
= container_of(to_config_group(item
),
2611 struct se_dev_attrib
, da_group
);
2612 struct tcmu_dev
*udev
= TCMU_DEV(da
->da_dev
);
2614 return snprintf(page
, PAGE_SIZE
, "%lu\n", udev
->cmd_time_out
/ MSEC_PER_SEC
);
2617 static ssize_t
tcmu_cmd_time_out_store(struct config_item
*item
, const char *page
,
2620 struct se_dev_attrib
*da
= container_of(to_config_group(item
),
2621 struct se_dev_attrib
, da_group
);
2622 struct tcmu_dev
*udev
= container_of(da
->da_dev
,
2623 struct tcmu_dev
, se_dev
);
2627 if (da
->da_dev
->export_count
) {
2628 pr_err("Unable to set tcmu cmd_time_out while exports exist\n");
2632 ret
= kstrtou32(page
, 0, &val
);
2636 udev
->cmd_time_out
= val
* MSEC_PER_SEC
;
2639 CONFIGFS_ATTR(tcmu_
, cmd_time_out
);
2641 static ssize_t
tcmu_qfull_time_out_show(struct config_item
*item
, char *page
)
2643 struct se_dev_attrib
*da
= container_of(to_config_group(item
),
2644 struct se_dev_attrib
, da_group
);
2645 struct tcmu_dev
*udev
= TCMU_DEV(da
->da_dev
);
2647 return snprintf(page
, PAGE_SIZE
, "%ld\n", udev
->qfull_time_out
<= 0 ?
2648 udev
->qfull_time_out
:
2649 udev
->qfull_time_out
/ MSEC_PER_SEC
);
2652 static ssize_t
tcmu_qfull_time_out_store(struct config_item
*item
,
2653 const char *page
, size_t count
)
2655 struct se_dev_attrib
*da
= container_of(to_config_group(item
),
2656 struct se_dev_attrib
, da_group
);
2657 struct tcmu_dev
*udev
= TCMU_DEV(da
->da_dev
);
2661 ret
= kstrtos32(page
, 0, &val
);
2666 udev
->qfull_time_out
= val
* MSEC_PER_SEC
;
2667 } else if (val
== -1) {
2668 udev
->qfull_time_out
= val
;
2670 printk(KERN_ERR
"Invalid qfull timeout value %d\n", val
);
2675 CONFIGFS_ATTR(tcmu_
, qfull_time_out
);
2677 static ssize_t
tcmu_max_data_area_mb_show(struct config_item
*item
, char *page
)
2679 struct se_dev_attrib
*da
= container_of(to_config_group(item
),
2680 struct se_dev_attrib
, da_group
);
2681 struct tcmu_dev
*udev
= TCMU_DEV(da
->da_dev
);
2683 return snprintf(page
, PAGE_SIZE
, "%u\n", udev
->data_area_mb
);
2685 CONFIGFS_ATTR_RO(tcmu_
, max_data_area_mb
);
2687 static ssize_t
tcmu_data_pages_per_blk_show(struct config_item
*item
,
2690 struct se_dev_attrib
*da
= container_of(to_config_group(item
),
2691 struct se_dev_attrib
, da_group
);
2692 struct tcmu_dev
*udev
= TCMU_DEV(da
->da_dev
);
2694 return snprintf(page
, PAGE_SIZE
, "%u\n", udev
->data_pages_per_blk
);
2696 CONFIGFS_ATTR_RO(tcmu_
, data_pages_per_blk
);
2698 static ssize_t
tcmu_dev_config_show(struct config_item
*item
, char *page
)
2700 struct se_dev_attrib
*da
= container_of(to_config_group(item
),
2701 struct se_dev_attrib
, da_group
);
2702 struct tcmu_dev
*udev
= TCMU_DEV(da
->da_dev
);
2704 return snprintf(page
, PAGE_SIZE
, "%s\n", udev
->dev_config
);
2707 static int tcmu_send_dev_config_event(struct tcmu_dev
*udev
,
2708 const char *reconfig_data
)
2710 struct sk_buff
*skb
= NULL
;
2711 void *msg_header
= NULL
;
2714 ret
= tcmu_netlink_event_init(udev
, TCMU_CMD_RECONFIG_DEVICE
,
2718 ret
= nla_put_string(skb
, TCMU_ATTR_DEV_CFG
, reconfig_data
);
2723 return tcmu_netlink_event_send(udev
, TCMU_CMD_RECONFIG_DEVICE
,
2728 static ssize_t
tcmu_dev_config_store(struct config_item
*item
, const char *page
,
2731 struct se_dev_attrib
*da
= container_of(to_config_group(item
),
2732 struct se_dev_attrib
, da_group
);
2733 struct tcmu_dev
*udev
= TCMU_DEV(da
->da_dev
);
2737 if (!len
|| len
> TCMU_CONFIG_LEN
- 1)
2740 /* Check if device has been configured before */
2741 if (target_dev_configured(&udev
->se_dev
)) {
2742 ret
= tcmu_send_dev_config_event(udev
, page
);
2744 pr_err("Unable to reconfigure device\n");
2747 strlcpy(udev
->dev_config
, page
, TCMU_CONFIG_LEN
);
2749 ret
= tcmu_update_uio_info(udev
);
2754 strlcpy(udev
->dev_config
, page
, TCMU_CONFIG_LEN
);
2758 CONFIGFS_ATTR(tcmu_
, dev_config
);
2760 static ssize_t
tcmu_dev_size_show(struct config_item
*item
, char *page
)
2762 struct se_dev_attrib
*da
= container_of(to_config_group(item
),
2763 struct se_dev_attrib
, da_group
);
2764 struct tcmu_dev
*udev
= TCMU_DEV(da
->da_dev
);
2766 return snprintf(page
, PAGE_SIZE
, "%llu\n", udev
->dev_size
);
2769 static int tcmu_send_dev_size_event(struct tcmu_dev
*udev
, u64 size
)
2771 struct sk_buff
*skb
= NULL
;
2772 void *msg_header
= NULL
;
2775 ret
= tcmu_netlink_event_init(udev
, TCMU_CMD_RECONFIG_DEVICE
,
2779 ret
= nla_put_u64_64bit(skb
, TCMU_ATTR_DEV_SIZE
,
2780 size
, TCMU_ATTR_PAD
);
2785 return tcmu_netlink_event_send(udev
, TCMU_CMD_RECONFIG_DEVICE
,
2789 static ssize_t
tcmu_dev_size_store(struct config_item
*item
, const char *page
,
2792 struct se_dev_attrib
*da
= container_of(to_config_group(item
),
2793 struct se_dev_attrib
, da_group
);
2794 struct tcmu_dev
*udev
= TCMU_DEV(da
->da_dev
);
2798 ret
= kstrtou64(page
, 0, &val
);
2802 /* Check if device has been configured before */
2803 if (target_dev_configured(&udev
->se_dev
)) {
2804 ret
= tcmu_send_dev_size_event(udev
, val
);
2806 pr_err("Unable to reconfigure device\n");
2810 udev
->dev_size
= val
;
2813 CONFIGFS_ATTR(tcmu_
, dev_size
);
2815 static ssize_t
tcmu_nl_reply_supported_show(struct config_item
*item
,
2818 struct se_dev_attrib
*da
= container_of(to_config_group(item
),
2819 struct se_dev_attrib
, da_group
);
2820 struct tcmu_dev
*udev
= TCMU_DEV(da
->da_dev
);
2822 return snprintf(page
, PAGE_SIZE
, "%d\n", udev
->nl_reply_supported
);
2825 static ssize_t
tcmu_nl_reply_supported_store(struct config_item
*item
,
2826 const char *page
, size_t count
)
2828 struct se_dev_attrib
*da
= container_of(to_config_group(item
),
2829 struct se_dev_attrib
, da_group
);
2830 struct tcmu_dev
*udev
= TCMU_DEV(da
->da_dev
);
2834 ret
= kstrtos8(page
, 0, &val
);
2838 udev
->nl_reply_supported
= val
;
2841 CONFIGFS_ATTR(tcmu_
, nl_reply_supported
);
2843 static ssize_t
tcmu_emulate_write_cache_show(struct config_item
*item
,
2846 struct se_dev_attrib
*da
= container_of(to_config_group(item
),
2847 struct se_dev_attrib
, da_group
);
2849 return snprintf(page
, PAGE_SIZE
, "%i\n", da
->emulate_write_cache
);
2852 static int tcmu_send_emulate_write_cache(struct tcmu_dev
*udev
, u8 val
)
2854 struct sk_buff
*skb
= NULL
;
2855 void *msg_header
= NULL
;
2858 ret
= tcmu_netlink_event_init(udev
, TCMU_CMD_RECONFIG_DEVICE
,
2862 ret
= nla_put_u8(skb
, TCMU_ATTR_WRITECACHE
, val
);
2867 return tcmu_netlink_event_send(udev
, TCMU_CMD_RECONFIG_DEVICE
,
2871 static ssize_t
tcmu_emulate_write_cache_store(struct config_item
*item
,
2872 const char *page
, size_t count
)
2874 struct se_dev_attrib
*da
= container_of(to_config_group(item
),
2875 struct se_dev_attrib
, da_group
);
2876 struct tcmu_dev
*udev
= TCMU_DEV(da
->da_dev
);
2880 ret
= kstrtou8(page
, 0, &val
);
2884 /* Check if device has been configured before */
2885 if (target_dev_configured(&udev
->se_dev
)) {
2886 ret
= tcmu_send_emulate_write_cache(udev
, val
);
2888 pr_err("Unable to reconfigure device\n");
2893 da
->emulate_write_cache
= val
;
2896 CONFIGFS_ATTR(tcmu_
, emulate_write_cache
);
2898 static ssize_t
tcmu_tmr_notification_show(struct config_item
*item
, char *page
)
2900 struct se_dev_attrib
*da
= container_of(to_config_group(item
),
2901 struct se_dev_attrib
, da_group
);
2902 struct tcmu_dev
*udev
= TCMU_DEV(da
->da_dev
);
2904 return snprintf(page
, PAGE_SIZE
, "%i\n",
2905 test_bit(TCMU_DEV_BIT_TMR_NOTIFY
, &udev
->flags
));
2908 static ssize_t
tcmu_tmr_notification_store(struct config_item
*item
,
2909 const char *page
, size_t count
)
2911 struct se_dev_attrib
*da
= container_of(to_config_group(item
),
2912 struct se_dev_attrib
, da_group
);
2913 struct tcmu_dev
*udev
= TCMU_DEV(da
->da_dev
);
2917 ret
= kstrtou8(page
, 0, &val
);
2924 set_bit(TCMU_DEV_BIT_TMR_NOTIFY
, &udev
->flags
);
2926 clear_bit(TCMU_DEV_BIT_TMR_NOTIFY
, &udev
->flags
);
2929 CONFIGFS_ATTR(tcmu_
, tmr_notification
);
2931 static ssize_t
tcmu_block_dev_show(struct config_item
*item
, char *page
)
2933 struct se_device
*se_dev
= container_of(to_config_group(item
),
2936 struct tcmu_dev
*udev
= TCMU_DEV(se_dev
);
2938 if (test_bit(TCMU_DEV_BIT_BLOCKED
, &udev
->flags
))
2939 return snprintf(page
, PAGE_SIZE
, "%s\n", "blocked");
2941 return snprintf(page
, PAGE_SIZE
, "%s\n", "unblocked");
2944 static ssize_t
tcmu_block_dev_store(struct config_item
*item
, const char *page
,
2947 struct se_device
*se_dev
= container_of(to_config_group(item
),
2950 struct tcmu_dev
*udev
= TCMU_DEV(se_dev
);
2954 if (!target_dev_configured(&udev
->se_dev
)) {
2955 pr_err("Device is not configured.\n");
2959 ret
= kstrtou8(page
, 0, &val
);
2964 pr_err("Invalid block value %d\n", val
);
2969 tcmu_unblock_dev(udev
);
2971 tcmu_block_dev(udev
);
2974 CONFIGFS_ATTR(tcmu_
, block_dev
);
2976 static ssize_t
tcmu_reset_ring_store(struct config_item
*item
, const char *page
,
2979 struct se_device
*se_dev
= container_of(to_config_group(item
),
2982 struct tcmu_dev
*udev
= TCMU_DEV(se_dev
);
2986 if (!target_dev_configured(&udev
->se_dev
)) {
2987 pr_err("Device is not configured.\n");
2991 ret
= kstrtou8(page
, 0, &val
);
2995 if (val
!= 1 && val
!= 2) {
2996 pr_err("Invalid reset ring value %d\n", val
);
3000 tcmu_reset_ring(udev
, val
);
3003 CONFIGFS_ATTR_WO(tcmu_
, reset_ring
);
3005 static ssize_t
tcmu_free_kept_buf_store(struct config_item
*item
, const char *page
,
3008 struct se_device
*se_dev
= container_of(to_config_group(item
),
3011 struct tcmu_dev
*udev
= TCMU_DEV(se_dev
);
3012 struct tcmu_cmd
*cmd
;
3016 if (!target_dev_configured(&udev
->se_dev
)) {
3017 pr_err("Device is not configured.\n");
3021 ret
= kstrtou16(page
, 0, &cmd_id
);
3025 mutex_lock(&udev
->cmdr_lock
);
3028 XA_STATE(xas
, &udev
->commands
, cmd_id
);
3031 cmd
= xas_load(&xas
);
3033 pr_err("free_kept_buf: cmd_id %d not found\n", cmd_id
);
3038 if (!test_bit(TCMU_CMD_BIT_KEEP_BUF
, &cmd
->flags
)) {
3039 pr_err("free_kept_buf: cmd_id %d was not completed with KEEP_BUF\n",
3045 xas_store(&xas
, NULL
);
3049 tcmu_cmd_free_data(cmd
, cmd
->dbi_cnt
);
3052 * We only freed data space, not ring space. Therefore we dont call
3053 * run_tmr_queue, but call run_qfull_queue if tmr_list is empty.
3055 if (list_empty(&udev
->tmr_queue
))
3056 run_qfull_queue(udev
, false);
3059 mutex_unlock(&udev
->cmdr_lock
);
3062 CONFIGFS_ATTR_WO(tcmu_
, free_kept_buf
);
3064 static struct configfs_attribute
*tcmu_attrib_attrs
[] = {
3065 &tcmu_attr_cmd_time_out
,
3066 &tcmu_attr_qfull_time_out
,
3067 &tcmu_attr_max_data_area_mb
,
3068 &tcmu_attr_data_pages_per_blk
,
3069 &tcmu_attr_dev_config
,
3070 &tcmu_attr_dev_size
,
3071 &tcmu_attr_emulate_write_cache
,
3072 &tcmu_attr_tmr_notification
,
3073 &tcmu_attr_nl_reply_supported
,
3077 static struct configfs_attribute
**tcmu_attrs
;
3079 static struct configfs_attribute
*tcmu_action_attrs
[] = {
3080 &tcmu_attr_block_dev
,
3081 &tcmu_attr_reset_ring
,
3082 &tcmu_attr_free_kept_buf
,
3086 static struct target_backend_ops tcmu_ops
= {
3088 .owner
= THIS_MODULE
,
3089 .transport_flags_default
= TRANSPORT_FLAG_PASSTHROUGH
,
3090 .transport_flags_changeable
= TRANSPORT_FLAG_PASSTHROUGH_PGR
|
3091 TRANSPORT_FLAG_PASSTHROUGH_ALUA
,
3092 .attach_hba
= tcmu_attach_hba
,
3093 .detach_hba
= tcmu_detach_hba
,
3094 .alloc_device
= tcmu_alloc_device
,
3095 .configure_device
= tcmu_configure_device
,
3096 .destroy_device
= tcmu_destroy_device
,
3097 .free_device
= tcmu_free_device
,
3098 .unplug_device
= tcmu_unplug_device
,
3099 .plug_device
= tcmu_plug_device
,
3100 .parse_cdb
= tcmu_parse_cdb
,
3101 .tmr_notify
= tcmu_tmr_notify
,
3102 .set_configfs_dev_params
= tcmu_set_configfs_dev_params
,
3103 .show_configfs_dev_params
= tcmu_show_configfs_dev_params
,
3104 .get_device_type
= sbc_get_device_type
,
3105 .get_blocks
= tcmu_get_blocks
,
3106 .tb_dev_action_attrs
= tcmu_action_attrs
,
3109 static void find_free_blocks(void)
3111 struct tcmu_dev
*udev
;
3113 u32 pages_freed
, total_pages_freed
= 0;
3114 u32 start
, end
, block
, total_blocks_freed
= 0;
3116 if (atomic_read(&global_page_count
) <= tcmu_global_max_pages
)
3119 mutex_lock(&root_udev_mutex
);
3120 list_for_each_entry(udev
, &root_udev
, node
) {
3121 mutex_lock(&udev
->cmdr_lock
);
3123 if (!target_dev_configured(&udev
->se_dev
)) {
3124 mutex_unlock(&udev
->cmdr_lock
);
3128 /* Try to complete the finished commands first */
3129 if (tcmu_handle_completions(udev
))
3130 run_qfull_queue(udev
, false);
3132 /* Skip the udevs in idle */
3133 if (!udev
->dbi_thresh
) {
3134 mutex_unlock(&udev
->cmdr_lock
);
3138 end
= udev
->dbi_max
+ 1;
3139 block
= find_last_bit(udev
->data_bitmap
, end
);
3140 if (block
== udev
->dbi_max
) {
3142 * The last bit is dbi_max, so it is not possible
3143 * reclaim any blocks.
3145 mutex_unlock(&udev
->cmdr_lock
);
3147 } else if (block
== end
) {
3148 /* The current udev will goto idle state */
3149 udev
->dbi_thresh
= start
= 0;
3152 udev
->dbi_thresh
= start
= block
+ 1;
3153 udev
->dbi_max
= block
;
3156 /* Here will truncate the data area from off */
3157 off
= udev
->data_off
+ (loff_t
)start
* udev
->data_blk_size
;
3158 unmap_mapping_range(udev
->inode
->i_mapping
, off
, 0, 1);
3160 /* Release the block pages */
3161 pages_freed
= tcmu_blocks_release(udev
, start
, end
- 1);
3162 mutex_unlock(&udev
->cmdr_lock
);
3164 total_pages_freed
+= pages_freed
;
3165 total_blocks_freed
+= end
- start
;
3166 pr_debug("Freed %u pages (total %u) from %u blocks (total %u) from %s.\n",
3167 pages_freed
, total_pages_freed
, end
- start
,
3168 total_blocks_freed
, udev
->name
);
3170 mutex_unlock(&root_udev_mutex
);
3172 if (atomic_read(&global_page_count
) > tcmu_global_max_pages
)
3173 schedule_delayed_work(&tcmu_unmap_work
, msecs_to_jiffies(5000));
3176 static void check_timedout_devices(void)
3178 struct tcmu_dev
*udev
, *tmp_dev
;
3179 struct tcmu_cmd
*cmd
, *tmp_cmd
;
3182 spin_lock_bh(&timed_out_udevs_lock
);
3183 list_splice_init(&timed_out_udevs
, &devs
);
3185 list_for_each_entry_safe(udev
, tmp_dev
, &devs
, timedout_entry
) {
3186 list_del_init(&udev
->timedout_entry
);
3187 spin_unlock_bh(&timed_out_udevs_lock
);
3189 mutex_lock(&udev
->cmdr_lock
);
3192 * If cmd_time_out is disabled but qfull is set deadline
3193 * will only reflect the qfull timeout. Ignore it.
3195 if (udev
->cmd_time_out
) {
3196 list_for_each_entry_safe(cmd
, tmp_cmd
,
3197 &udev
->inflight_queue
,
3199 tcmu_check_expired_ring_cmd(cmd
);
3201 tcmu_set_next_deadline(&udev
->inflight_queue
,
3204 list_for_each_entry_safe(cmd
, tmp_cmd
, &udev
->qfull_queue
,
3206 tcmu_check_expired_queue_cmd(cmd
);
3208 tcmu_set_next_deadline(&udev
->qfull_queue
, &udev
->qfull_timer
);
3210 mutex_unlock(&udev
->cmdr_lock
);
3212 spin_lock_bh(&timed_out_udevs_lock
);
3215 spin_unlock_bh(&timed_out_udevs_lock
);
3218 static void tcmu_unmap_work_fn(struct work_struct
*work
)
3220 check_timedout_devices();
3224 static int __init
tcmu_module_init(void)
3226 int ret
, i
, k
, len
= 0;
3228 BUILD_BUG_ON((sizeof(struct tcmu_cmd_entry
) % TCMU_OP_ALIGN_SIZE
) != 0);
3230 INIT_DELAYED_WORK(&tcmu_unmap_work
, tcmu_unmap_work_fn
);
3232 tcmu_cmd_cache
= kmem_cache_create("tcmu_cmd_cache",
3233 sizeof(struct tcmu_cmd
),
3234 __alignof__(struct tcmu_cmd
),
3236 if (!tcmu_cmd_cache
)
3239 tcmu_root_device
= root_device_register("tcm_user");
3240 if (IS_ERR(tcmu_root_device
)) {
3241 ret
= PTR_ERR(tcmu_root_device
);
3242 goto out_free_cache
;
3245 ret
= genl_register_family(&tcmu_genl_family
);
3247 goto out_unreg_device
;
3250 for (i
= 0; passthrough_attrib_attrs
[i
] != NULL
; i
++)
3251 len
+= sizeof(struct configfs_attribute
*);
3252 for (i
= 0; passthrough_pr_attrib_attrs
[i
] != NULL
; i
++)
3253 len
+= sizeof(struct configfs_attribute
*);
3254 for (i
= 0; tcmu_attrib_attrs
[i
] != NULL
; i
++)
3255 len
+= sizeof(struct configfs_attribute
*);
3256 len
+= sizeof(struct configfs_attribute
*);
3258 tcmu_attrs
= kzalloc(len
, GFP_KERNEL
);
3261 goto out_unreg_genl
;
3264 for (i
= 0; passthrough_attrib_attrs
[i
] != NULL
; i
++)
3265 tcmu_attrs
[i
] = passthrough_attrib_attrs
[i
];
3266 for (k
= 0; passthrough_pr_attrib_attrs
[k
] != NULL
; k
++)
3267 tcmu_attrs
[i
++] = passthrough_pr_attrib_attrs
[k
];
3268 for (k
= 0; tcmu_attrib_attrs
[k
] != NULL
; k
++)
3269 tcmu_attrs
[i
++] = tcmu_attrib_attrs
[k
];
3270 tcmu_ops
.tb_dev_attrib_attrs
= tcmu_attrs
;
3272 ret
= transport_backend_register(&tcmu_ops
);
3281 genl_unregister_family(&tcmu_genl_family
);
3283 root_device_unregister(tcmu_root_device
);
3285 kmem_cache_destroy(tcmu_cmd_cache
);
3290 static void __exit
tcmu_module_exit(void)
3292 cancel_delayed_work_sync(&tcmu_unmap_work
);
3293 target_backend_unregister(&tcmu_ops
);
3295 genl_unregister_family(&tcmu_genl_family
);
3296 root_device_unregister(tcmu_root_device
);
3297 kmem_cache_destroy(tcmu_cmd_cache
);
3300 MODULE_DESCRIPTION("TCM USER subsystem plugin");
3301 MODULE_AUTHOR("Shaohua Li <shli@kernel.org>");
3302 MODULE_AUTHOR("Andy Grover <agrover@redhat.com>");
3303 MODULE_LICENSE("GPL");
3305 module_init(tcmu_module_init
);
3306 module_exit(tcmu_module_exit
);