1 // SPDX-License-Identifier: GPL-2.0
3 * Copyright (c) 2017-2018 Christoph Hellwig.
6 #include <linux/moduleparam.h>
7 #include <trace/events/block.h>
10 static bool multipath
= true;
11 module_param(multipath
, bool, 0444);
12 MODULE_PARM_DESC(multipath
,
13 "turn on native support for multiple controllers per subsystem");
15 inline bool nvme_ctrl_use_ana(struct nvme_ctrl
*ctrl
)
17 return multipath
&& ctrl
->subsys
&& (ctrl
->subsys
->cmic
& (1 << 3));
21 * If multipathing is enabled we need to always use the subsystem instance
22 * number for numbering our devices to avoid conflicts between subsystems that
23 * have multiple controllers and thus use the multipath-aware subsystem node
24 * and those that have a single controller and use the controller node
27 void nvme_set_disk_name(char *disk_name
, struct nvme_ns
*ns
,
28 struct nvme_ctrl
*ctrl
, int *flags
)
31 sprintf(disk_name
, "nvme%dn%d", ctrl
->instance
, ns
->head
->instance
);
32 } else if (ns
->head
->disk
) {
33 sprintf(disk_name
, "nvme%dc%dn%d", ctrl
->subsys
->instance
,
34 ctrl
->cntlid
, ns
->head
->instance
);
35 *flags
= GENHD_FL_HIDDEN
;
37 sprintf(disk_name
, "nvme%dn%d", ctrl
->subsys
->instance
,
42 void nvme_failover_req(struct request
*req
)
44 struct nvme_ns
*ns
= req
->q
->queuedata
;
45 u16 status
= nvme_req(req
)->status
;
48 spin_lock_irqsave(&ns
->head
->requeue_lock
, flags
);
49 blk_steal_bios(&ns
->head
->requeue_list
, req
);
50 spin_unlock_irqrestore(&ns
->head
->requeue_lock
, flags
);
51 blk_mq_end_request(req
, 0);
53 switch (status
& 0x7ff) {
54 case NVME_SC_ANA_TRANSITION
:
55 case NVME_SC_ANA_INACCESSIBLE
:
56 case NVME_SC_ANA_PERSISTENT_LOSS
:
58 * If we got back an ANA error we know the controller is alive,
59 * but not ready to serve this namespaces. The spec suggests
60 * we should update our general state here, but due to the fact
61 * that the admin and I/O queues are not serialized that is
62 * fundamentally racy. So instead just clear the current path,
63 * mark the the path as pending and kick of a re-read of the ANA
66 nvme_mpath_clear_current_path(ns
);
67 if (ns
->ctrl
->ana_log_buf
) {
68 set_bit(NVME_NS_ANA_PENDING
, &ns
->flags
);
69 queue_work(nvme_wq
, &ns
->ctrl
->ana_work
);
72 case NVME_SC_HOST_PATH_ERROR
:
74 * Temporary transport disruption in talking to the controller.
75 * Try to send on a new path.
77 nvme_mpath_clear_current_path(ns
);
81 * Reset the controller for any non-ANA error as we don't know
82 * what caused the error.
84 nvme_reset_ctrl(ns
->ctrl
);
88 kblockd_schedule_work(&ns
->head
->requeue_work
);
91 void nvme_kick_requeue_lists(struct nvme_ctrl
*ctrl
)
95 down_read(&ctrl
->namespaces_rwsem
);
96 list_for_each_entry(ns
, &ctrl
->namespaces
, list
) {
98 kblockd_schedule_work(&ns
->head
->requeue_work
);
100 up_read(&ctrl
->namespaces_rwsem
);
103 static const char *nvme_ana_state_names
[] = {
104 [0] = "invalid state",
105 [NVME_ANA_OPTIMIZED
] = "optimized",
106 [NVME_ANA_NONOPTIMIZED
] = "non-optimized",
107 [NVME_ANA_INACCESSIBLE
] = "inaccessible",
108 [NVME_ANA_PERSISTENT_LOSS
] = "persistent-loss",
109 [NVME_ANA_CHANGE
] = "change",
112 void nvme_mpath_clear_current_path(struct nvme_ns
*ns
)
114 struct nvme_ns_head
*head
= ns
->head
;
120 for_each_node(node
) {
121 if (ns
== rcu_access_pointer(head
->current_path
[node
]))
122 rcu_assign_pointer(head
->current_path
[node
], NULL
);
126 static struct nvme_ns
*__nvme_find_path(struct nvme_ns_head
*head
, int node
)
128 int found_distance
= INT_MAX
, fallback_distance
= INT_MAX
, distance
;
129 struct nvme_ns
*found
= NULL
, *fallback
= NULL
, *ns
;
131 list_for_each_entry_rcu(ns
, &head
->list
, siblings
) {
132 if (ns
->ctrl
->state
!= NVME_CTRL_LIVE
||
133 test_bit(NVME_NS_ANA_PENDING
, &ns
->flags
))
136 if (READ_ONCE(head
->subsys
->iopolicy
) == NVME_IOPOLICY_NUMA
)
137 distance
= node_distance(node
, ns
->ctrl
->numa_node
);
139 distance
= LOCAL_DISTANCE
;
141 switch (ns
->ana_state
) {
142 case NVME_ANA_OPTIMIZED
:
143 if (distance
< found_distance
) {
144 found_distance
= distance
;
148 case NVME_ANA_NONOPTIMIZED
:
149 if (distance
< fallback_distance
) {
150 fallback_distance
= distance
;
162 rcu_assign_pointer(head
->current_path
[node
], found
);
166 static struct nvme_ns
*nvme_next_ns(struct nvme_ns_head
*head
,
169 ns
= list_next_or_null_rcu(&head
->list
, &ns
->siblings
, struct nvme_ns
,
173 return list_first_or_null_rcu(&head
->list
, struct nvme_ns
, siblings
);
176 static struct nvme_ns
*nvme_round_robin_path(struct nvme_ns_head
*head
,
177 int node
, struct nvme_ns
*old
)
179 struct nvme_ns
*ns
, *found
, *fallback
= NULL
;
181 if (list_is_singular(&head
->list
))
184 for (ns
= nvme_next_ns(head
, old
);
186 ns
= nvme_next_ns(head
, ns
)) {
187 if (ns
->ctrl
->state
!= NVME_CTRL_LIVE
||
188 test_bit(NVME_NS_ANA_PENDING
, &ns
->flags
))
191 if (ns
->ana_state
== NVME_ANA_OPTIMIZED
) {
195 if (ns
->ana_state
== NVME_ANA_NONOPTIMIZED
)
203 rcu_assign_pointer(head
->current_path
[node
], found
);
207 static inline bool nvme_path_is_optimized(struct nvme_ns
*ns
)
209 return ns
->ctrl
->state
== NVME_CTRL_LIVE
&&
210 ns
->ana_state
== NVME_ANA_OPTIMIZED
;
213 inline struct nvme_ns
*nvme_find_path(struct nvme_ns_head
*head
)
215 int node
= numa_node_id();
218 ns
= srcu_dereference(head
->current_path
[node
], &head
->srcu
);
219 if (READ_ONCE(head
->subsys
->iopolicy
) == NVME_IOPOLICY_RR
&& ns
)
220 ns
= nvme_round_robin_path(head
, node
, ns
);
221 if (unlikely(!ns
|| !nvme_path_is_optimized(ns
)))
222 ns
= __nvme_find_path(head
, node
);
226 static blk_qc_t
nvme_ns_head_make_request(struct request_queue
*q
,
229 struct nvme_ns_head
*head
= q
->queuedata
;
230 struct device
*dev
= disk_to_dev(head
->disk
);
232 blk_qc_t ret
= BLK_QC_T_NONE
;
235 srcu_idx
= srcu_read_lock(&head
->srcu
);
236 ns
= nvme_find_path(head
);
238 bio
->bi_disk
= ns
->disk
;
239 bio
->bi_opf
|= REQ_NVME_MPATH
;
240 trace_block_bio_remap(bio
->bi_disk
->queue
, bio
,
241 disk_devt(ns
->head
->disk
),
242 bio
->bi_iter
.bi_sector
);
243 ret
= direct_make_request(bio
);
244 } else if (!list_empty_careful(&head
->list
)) {
245 dev_warn_ratelimited(dev
, "no path available - requeuing I/O\n");
247 spin_lock_irq(&head
->requeue_lock
);
248 bio_list_add(&head
->requeue_list
, bio
);
249 spin_unlock_irq(&head
->requeue_lock
);
251 dev_warn_ratelimited(dev
, "no path - failing I/O\n");
253 bio
->bi_status
= BLK_STS_IOERR
;
257 srcu_read_unlock(&head
->srcu
, srcu_idx
);
261 static void nvme_requeue_work(struct work_struct
*work
)
263 struct nvme_ns_head
*head
=
264 container_of(work
, struct nvme_ns_head
, requeue_work
);
265 struct bio
*bio
, *next
;
267 spin_lock_irq(&head
->requeue_lock
);
268 next
= bio_list_get(&head
->requeue_list
);
269 spin_unlock_irq(&head
->requeue_lock
);
271 while ((bio
= next
) != NULL
) {
276 * Reset disk to the mpath node and resubmit to select a new
279 bio
->bi_disk
= head
->disk
;
280 generic_make_request(bio
);
284 int nvme_mpath_alloc_disk(struct nvme_ctrl
*ctrl
, struct nvme_ns_head
*head
)
286 struct request_queue
*q
;
289 mutex_init(&head
->lock
);
290 bio_list_init(&head
->requeue_list
);
291 spin_lock_init(&head
->requeue_lock
);
292 INIT_WORK(&head
->requeue_work
, nvme_requeue_work
);
295 * Add a multipath node if the subsystems supports multiple controllers.
296 * We also do this for private namespaces as the namespace sharing data could
297 * change after a rescan.
299 if (!(ctrl
->subsys
->cmic
& (1 << 1)) || !multipath
)
302 q
= blk_alloc_queue_node(GFP_KERNEL
, ctrl
->numa_node
);
306 blk_queue_make_request(q
, nvme_ns_head_make_request
);
307 blk_queue_flag_set(QUEUE_FLAG_NONROT
, q
);
308 /* set to a default value for 512 until disk is validated */
309 blk_queue_logical_block_size(q
, 512);
310 blk_set_stacking_limits(&q
->limits
);
312 /* we need to propagate up the VMC settings */
313 if (ctrl
->vwc
& NVME_CTRL_VWC_PRESENT
)
315 blk_queue_write_cache(q
, vwc
, vwc
);
317 head
->disk
= alloc_disk(0);
319 goto out_cleanup_queue
;
320 head
->disk
->fops
= &nvme_ns_head_ops
;
321 head
->disk
->private_data
= head
;
322 head
->disk
->queue
= q
;
323 head
->disk
->flags
= GENHD_FL_EXT_DEVT
;
324 sprintf(head
->disk
->disk_name
, "nvme%dn%d",
325 ctrl
->subsys
->instance
, head
->instance
);
329 blk_cleanup_queue(q
);
334 static void nvme_mpath_set_live(struct nvme_ns
*ns
)
336 struct nvme_ns_head
*head
= ns
->head
;
338 lockdep_assert_held(&ns
->head
->lock
);
343 if (!(head
->disk
->flags
& GENHD_FL_UP
))
344 device_add_disk(&head
->subsys
->dev
, head
->disk
,
345 nvme_ns_id_attr_groups
);
347 if (nvme_path_is_optimized(ns
)) {
350 srcu_idx
= srcu_read_lock(&head
->srcu
);
352 __nvme_find_path(head
, node
);
353 srcu_read_unlock(&head
->srcu
, srcu_idx
);
356 kblockd_schedule_work(&ns
->head
->requeue_work
);
359 static int nvme_parse_ana_log(struct nvme_ctrl
*ctrl
, void *data
,
360 int (*cb
)(struct nvme_ctrl
*ctrl
, struct nvme_ana_group_desc
*,
363 void *base
= ctrl
->ana_log_buf
;
364 size_t offset
= sizeof(struct nvme_ana_rsp_hdr
);
367 lockdep_assert_held(&ctrl
->ana_lock
);
369 for (i
= 0; i
< le16_to_cpu(ctrl
->ana_log_buf
->ngrps
); i
++) {
370 struct nvme_ana_group_desc
*desc
= base
+ offset
;
371 u32 nr_nsids
= le32_to_cpu(desc
->nnsids
);
372 size_t nsid_buf_size
= nr_nsids
* sizeof(__le32
);
374 if (WARN_ON_ONCE(desc
->grpid
== 0))
376 if (WARN_ON_ONCE(le32_to_cpu(desc
->grpid
) > ctrl
->anagrpmax
))
378 if (WARN_ON_ONCE(desc
->state
== 0))
380 if (WARN_ON_ONCE(desc
->state
> NVME_ANA_CHANGE
))
383 offset
+= sizeof(*desc
);
384 if (WARN_ON_ONCE(offset
> ctrl
->ana_log_size
- nsid_buf_size
))
387 error
= cb(ctrl
, desc
, data
);
391 offset
+= nsid_buf_size
;
392 if (WARN_ON_ONCE(offset
> ctrl
->ana_log_size
- sizeof(*desc
)))
399 static inline bool nvme_state_is_live(enum nvme_ana_state state
)
401 return state
== NVME_ANA_OPTIMIZED
|| state
== NVME_ANA_NONOPTIMIZED
;
404 static void nvme_update_ns_ana_state(struct nvme_ana_group_desc
*desc
,
407 mutex_lock(&ns
->head
->lock
);
408 ns
->ana_grpid
= le32_to_cpu(desc
->grpid
);
409 ns
->ana_state
= desc
->state
;
410 clear_bit(NVME_NS_ANA_PENDING
, &ns
->flags
);
412 if (nvme_state_is_live(ns
->ana_state
))
413 nvme_mpath_set_live(ns
);
414 mutex_unlock(&ns
->head
->lock
);
417 static int nvme_update_ana_state(struct nvme_ctrl
*ctrl
,
418 struct nvme_ana_group_desc
*desc
, void *data
)
420 u32 nr_nsids
= le32_to_cpu(desc
->nnsids
), n
= 0;
421 unsigned *nr_change_groups
= data
;
424 dev_info(ctrl
->device
, "ANA group %d: %s.\n",
425 le32_to_cpu(desc
->grpid
),
426 nvme_ana_state_names
[desc
->state
]);
428 if (desc
->state
== NVME_ANA_CHANGE
)
429 (*nr_change_groups
)++;
434 down_write(&ctrl
->namespaces_rwsem
);
435 list_for_each_entry(ns
, &ctrl
->namespaces
, list
) {
436 if (ns
->head
->ns_id
!= le32_to_cpu(desc
->nsids
[n
]))
438 nvme_update_ns_ana_state(desc
, ns
);
442 up_write(&ctrl
->namespaces_rwsem
);
443 WARN_ON_ONCE(n
< nr_nsids
);
447 static int nvme_read_ana_log(struct nvme_ctrl
*ctrl
, bool groups_only
)
449 u32 nr_change_groups
= 0;
452 mutex_lock(&ctrl
->ana_lock
);
453 error
= nvme_get_log(ctrl
, NVME_NSID_ALL
, NVME_LOG_ANA
,
454 groups_only
? NVME_ANA_LOG_RGO
: 0,
455 ctrl
->ana_log_buf
, ctrl
->ana_log_size
, 0);
457 dev_warn(ctrl
->device
, "Failed to get ANA log: %d\n", error
);
461 error
= nvme_parse_ana_log(ctrl
, &nr_change_groups
,
462 nvme_update_ana_state
);
467 * In theory we should have an ANATT timer per group as they might enter
468 * the change state at different times. But that is a lot of overhead
469 * just to protect against a target that keeps entering new changes
470 * states while never finishing previous ones. But we'll still
471 * eventually time out once all groups are in change state, so this
474 * We also double the ANATT value to provide some slack for transports
475 * or AEN processing overhead.
477 if (nr_change_groups
)
478 mod_timer(&ctrl
->anatt_timer
, ctrl
->anatt
* HZ
* 2 + jiffies
);
480 del_timer_sync(&ctrl
->anatt_timer
);
482 mutex_unlock(&ctrl
->ana_lock
);
486 static void nvme_ana_work(struct work_struct
*work
)
488 struct nvme_ctrl
*ctrl
= container_of(work
, struct nvme_ctrl
, ana_work
);
490 nvme_read_ana_log(ctrl
, false);
493 static void nvme_anatt_timeout(struct timer_list
*t
)
495 struct nvme_ctrl
*ctrl
= from_timer(ctrl
, t
, anatt_timer
);
497 dev_info(ctrl
->device
, "ANATT timeout, resetting controller.\n");
498 nvme_reset_ctrl(ctrl
);
501 void nvme_mpath_stop(struct nvme_ctrl
*ctrl
)
503 if (!nvme_ctrl_use_ana(ctrl
))
505 del_timer_sync(&ctrl
->anatt_timer
);
506 cancel_work_sync(&ctrl
->ana_work
);
509 #define SUBSYS_ATTR_RW(_name, _mode, _show, _store) \
510 struct device_attribute subsys_attr_##_name = \
511 __ATTR(_name, _mode, _show, _store)
513 static const char *nvme_iopolicy_names
[] = {
514 [NVME_IOPOLICY_NUMA
] = "numa",
515 [NVME_IOPOLICY_RR
] = "round-robin",
518 static ssize_t
nvme_subsys_iopolicy_show(struct device
*dev
,
519 struct device_attribute
*attr
, char *buf
)
521 struct nvme_subsystem
*subsys
=
522 container_of(dev
, struct nvme_subsystem
, dev
);
524 return sprintf(buf
, "%s\n",
525 nvme_iopolicy_names
[READ_ONCE(subsys
->iopolicy
)]);
528 static ssize_t
nvme_subsys_iopolicy_store(struct device
*dev
,
529 struct device_attribute
*attr
, const char *buf
, size_t count
)
531 struct nvme_subsystem
*subsys
=
532 container_of(dev
, struct nvme_subsystem
, dev
);
535 for (i
= 0; i
< ARRAY_SIZE(nvme_iopolicy_names
); i
++) {
536 if (sysfs_streq(buf
, nvme_iopolicy_names
[i
])) {
537 WRITE_ONCE(subsys
->iopolicy
, i
);
544 SUBSYS_ATTR_RW(iopolicy
, S_IRUGO
| S_IWUSR
,
545 nvme_subsys_iopolicy_show
, nvme_subsys_iopolicy_store
);
547 static ssize_t
ana_grpid_show(struct device
*dev
, struct device_attribute
*attr
,
550 return sprintf(buf
, "%d\n", nvme_get_ns_from_dev(dev
)->ana_grpid
);
552 DEVICE_ATTR_RO(ana_grpid
);
554 static ssize_t
ana_state_show(struct device
*dev
, struct device_attribute
*attr
,
557 struct nvme_ns
*ns
= nvme_get_ns_from_dev(dev
);
559 return sprintf(buf
, "%s\n", nvme_ana_state_names
[ns
->ana_state
]);
561 DEVICE_ATTR_RO(ana_state
);
563 static int nvme_set_ns_ana_state(struct nvme_ctrl
*ctrl
,
564 struct nvme_ana_group_desc
*desc
, void *data
)
566 struct nvme_ns
*ns
= data
;
568 if (ns
->ana_grpid
== le32_to_cpu(desc
->grpid
)) {
569 nvme_update_ns_ana_state(desc
, ns
);
570 return -ENXIO
; /* just break out of the loop */
576 void nvme_mpath_add_disk(struct nvme_ns
*ns
, struct nvme_id_ns
*id
)
578 if (nvme_ctrl_use_ana(ns
->ctrl
)) {
579 mutex_lock(&ns
->ctrl
->ana_lock
);
580 ns
->ana_grpid
= le32_to_cpu(id
->anagrpid
);
581 nvme_parse_ana_log(ns
->ctrl
, ns
, nvme_set_ns_ana_state
);
582 mutex_unlock(&ns
->ctrl
->ana_lock
);
584 mutex_lock(&ns
->head
->lock
);
585 ns
->ana_state
= NVME_ANA_OPTIMIZED
;
586 nvme_mpath_set_live(ns
);
587 mutex_unlock(&ns
->head
->lock
);
591 void nvme_mpath_remove_disk(struct nvme_ns_head
*head
)
595 if (head
->disk
->flags
& GENHD_FL_UP
)
596 del_gendisk(head
->disk
);
597 blk_set_queue_dying(head
->disk
->queue
);
598 /* make sure all pending bios are cleaned up */
599 kblockd_schedule_work(&head
->requeue_work
);
600 flush_work(&head
->requeue_work
);
601 blk_cleanup_queue(head
->disk
->queue
);
602 put_disk(head
->disk
);
605 int nvme_mpath_init(struct nvme_ctrl
*ctrl
, struct nvme_id_ctrl
*id
)
609 if (!nvme_ctrl_use_ana(ctrl
))
612 ctrl
->anacap
= id
->anacap
;
613 ctrl
->anatt
= id
->anatt
;
614 ctrl
->nanagrpid
= le32_to_cpu(id
->nanagrpid
);
615 ctrl
->anagrpmax
= le32_to_cpu(id
->anagrpmax
);
617 mutex_init(&ctrl
->ana_lock
);
618 timer_setup(&ctrl
->anatt_timer
, nvme_anatt_timeout
, 0);
619 ctrl
->ana_log_size
= sizeof(struct nvme_ana_rsp_hdr
) +
620 ctrl
->nanagrpid
* sizeof(struct nvme_ana_group_desc
);
621 ctrl
->ana_log_size
+= ctrl
->max_namespaces
* sizeof(__le32
);
623 if (ctrl
->ana_log_size
> ctrl
->max_hw_sectors
<< SECTOR_SHIFT
) {
624 dev_err(ctrl
->device
,
625 "ANA log page size (%zd) larger than MDTS (%d).\n",
627 ctrl
->max_hw_sectors
<< SECTOR_SHIFT
);
628 dev_err(ctrl
->device
, "disabling ANA support.\n");
632 INIT_WORK(&ctrl
->ana_work
, nvme_ana_work
);
633 ctrl
->ana_log_buf
= kmalloc(ctrl
->ana_log_size
, GFP_KERNEL
);
634 if (!ctrl
->ana_log_buf
) {
639 error
= nvme_read_ana_log(ctrl
, true);
641 goto out_free_ana_log_buf
;
643 out_free_ana_log_buf
:
644 kfree(ctrl
->ana_log_buf
);
645 ctrl
->ana_log_buf
= NULL
;
650 void nvme_mpath_uninit(struct nvme_ctrl
*ctrl
)
652 kfree(ctrl
->ana_log_buf
);
653 ctrl
->ana_log_buf
= NULL
;