2 * Copyright (c) 2017-2018 Christoph Hellwig.
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms and conditions of the GNU General Public License,
6 * version 2, as published by the Free Software Foundation.
8 * This program is distributed in the hope it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
14 #include <linux/moduleparam.h>
15 #include <trace/events/block.h>
18 static bool multipath
= true;
19 module_param(multipath
, bool, 0444);
20 MODULE_PARM_DESC(multipath
,
21 "turn on native support for multiple controllers per subsystem");
23 inline bool nvme_ctrl_use_ana(struct nvme_ctrl
*ctrl
)
25 return multipath
&& ctrl
->subsys
&& (ctrl
->subsys
->cmic
& (1 << 3));
29 * If multipathing is enabled we need to always use the subsystem instance
30 * number for numbering our devices to avoid conflicts between subsystems that
31 * have multiple controllers and thus use the multipath-aware subsystem node
32 * and those that have a single controller and use the controller node
35 void nvme_set_disk_name(char *disk_name
, struct nvme_ns
*ns
,
36 struct nvme_ctrl
*ctrl
, int *flags
)
39 sprintf(disk_name
, "nvme%dn%d", ctrl
->instance
, ns
->head
->instance
);
40 } else if (ns
->head
->disk
) {
41 sprintf(disk_name
, "nvme%dc%dn%d", ctrl
->subsys
->instance
,
42 ctrl
->cntlid
, ns
->head
->instance
);
43 *flags
= GENHD_FL_HIDDEN
;
45 sprintf(disk_name
, "nvme%dn%d", ctrl
->subsys
->instance
,
50 void nvme_failover_req(struct request
*req
)
52 struct nvme_ns
*ns
= req
->q
->queuedata
;
53 u16 status
= nvme_req(req
)->status
;
56 spin_lock_irqsave(&ns
->head
->requeue_lock
, flags
);
57 blk_steal_bios(&ns
->head
->requeue_list
, req
);
58 spin_unlock_irqrestore(&ns
->head
->requeue_lock
, flags
);
59 blk_mq_end_request(req
, 0);
61 switch (status
& 0x7ff) {
62 case NVME_SC_ANA_TRANSITION
:
63 case NVME_SC_ANA_INACCESSIBLE
:
64 case NVME_SC_ANA_PERSISTENT_LOSS
:
66 * If we got back an ANA error we know the controller is alive,
67 * but not ready to serve this namespaces. The spec suggests
68 * we should update our general state here, but due to the fact
69 * that the admin and I/O queues are not serialized that is
70 * fundamentally racy. So instead just clear the current path,
71 * mark the the path as pending and kick of a re-read of the ANA
74 nvme_mpath_clear_current_path(ns
);
75 if (ns
->ctrl
->ana_log_buf
) {
76 set_bit(NVME_NS_ANA_PENDING
, &ns
->flags
);
77 queue_work(nvme_wq
, &ns
->ctrl
->ana_work
);
80 case NVME_SC_HOST_PATH_ERROR
:
82 * Temporary transport disruption in talking to the controller.
83 * Try to send on a new path.
85 nvme_mpath_clear_current_path(ns
);
89 * Reset the controller for any non-ANA error as we don't know
90 * what caused the error.
92 nvme_reset_ctrl(ns
->ctrl
);
96 kblockd_schedule_work(&ns
->head
->requeue_work
);
99 void nvme_kick_requeue_lists(struct nvme_ctrl
*ctrl
)
103 down_read(&ctrl
->namespaces_rwsem
);
104 list_for_each_entry(ns
, &ctrl
->namespaces
, list
) {
106 kblockd_schedule_work(&ns
->head
->requeue_work
);
108 up_read(&ctrl
->namespaces_rwsem
);
111 static const char *nvme_ana_state_names
[] = {
112 [0] = "invalid state",
113 [NVME_ANA_OPTIMIZED
] = "optimized",
114 [NVME_ANA_NONOPTIMIZED
] = "non-optimized",
115 [NVME_ANA_INACCESSIBLE
] = "inaccessible",
116 [NVME_ANA_PERSISTENT_LOSS
] = "persistent-loss",
117 [NVME_ANA_CHANGE
] = "change",
120 void nvme_mpath_clear_current_path(struct nvme_ns
*ns
)
122 struct nvme_ns_head
*head
= ns
->head
;
128 for_each_node(node
) {
129 if (ns
== rcu_access_pointer(head
->current_path
[node
]))
130 rcu_assign_pointer(head
->current_path
[node
], NULL
);
134 static struct nvme_ns
*__nvme_find_path(struct nvme_ns_head
*head
, int node
)
136 int found_distance
= INT_MAX
, fallback_distance
= INT_MAX
, distance
;
137 struct nvme_ns
*found
= NULL
, *fallback
= NULL
, *ns
;
139 list_for_each_entry_rcu(ns
, &head
->list
, siblings
) {
140 if (ns
->ctrl
->state
!= NVME_CTRL_LIVE
||
141 test_bit(NVME_NS_ANA_PENDING
, &ns
->flags
))
144 distance
= node_distance(node
, ns
->ctrl
->numa_node
);
146 switch (ns
->ana_state
) {
147 case NVME_ANA_OPTIMIZED
:
148 if (distance
< found_distance
) {
149 found_distance
= distance
;
153 case NVME_ANA_NONOPTIMIZED
:
154 if (distance
< fallback_distance
) {
155 fallback_distance
= distance
;
167 rcu_assign_pointer(head
->current_path
[node
], found
);
171 static inline bool nvme_path_is_optimized(struct nvme_ns
*ns
)
173 return ns
->ctrl
->state
== NVME_CTRL_LIVE
&&
174 ns
->ana_state
== NVME_ANA_OPTIMIZED
;
177 inline struct nvme_ns
*nvme_find_path(struct nvme_ns_head
*head
)
179 int node
= numa_node_id();
182 ns
= srcu_dereference(head
->current_path
[node
], &head
->srcu
);
183 if (unlikely(!ns
|| !nvme_path_is_optimized(ns
)))
184 ns
= __nvme_find_path(head
, node
);
188 static blk_qc_t
nvme_ns_head_make_request(struct request_queue
*q
,
191 struct nvme_ns_head
*head
= q
->queuedata
;
192 struct device
*dev
= disk_to_dev(head
->disk
);
194 blk_qc_t ret
= BLK_QC_T_NONE
;
197 srcu_idx
= srcu_read_lock(&head
->srcu
);
198 ns
= nvme_find_path(head
);
200 bio
->bi_disk
= ns
->disk
;
201 bio
->bi_opf
|= REQ_NVME_MPATH
;
202 trace_block_bio_remap(bio
->bi_disk
->queue
, bio
,
203 disk_devt(ns
->head
->disk
),
204 bio
->bi_iter
.bi_sector
);
205 ret
= direct_make_request(bio
);
206 } else if (!list_empty_careful(&head
->list
)) {
207 dev_warn_ratelimited(dev
, "no path available - requeuing I/O\n");
209 spin_lock_irq(&head
->requeue_lock
);
210 bio_list_add(&head
->requeue_list
, bio
);
211 spin_unlock_irq(&head
->requeue_lock
);
213 dev_warn_ratelimited(dev
, "no path - failing I/O\n");
215 bio
->bi_status
= BLK_STS_IOERR
;
219 srcu_read_unlock(&head
->srcu
, srcu_idx
);
223 static void nvme_requeue_work(struct work_struct
*work
)
225 struct nvme_ns_head
*head
=
226 container_of(work
, struct nvme_ns_head
, requeue_work
);
227 struct bio
*bio
, *next
;
229 spin_lock_irq(&head
->requeue_lock
);
230 next
= bio_list_get(&head
->requeue_list
);
231 spin_unlock_irq(&head
->requeue_lock
);
233 while ((bio
= next
) != NULL
) {
238 * Reset disk to the mpath node and resubmit to select a new
241 bio
->bi_disk
= head
->disk
;
242 generic_make_request(bio
);
246 int nvme_mpath_alloc_disk(struct nvme_ctrl
*ctrl
, struct nvme_ns_head
*head
)
248 struct request_queue
*q
;
251 mutex_init(&head
->lock
);
252 bio_list_init(&head
->requeue_list
);
253 spin_lock_init(&head
->requeue_lock
);
254 INIT_WORK(&head
->requeue_work
, nvme_requeue_work
);
257 * Add a multipath node if the subsystems supports multiple controllers.
258 * We also do this for private namespaces as the namespace sharing data could
259 * change after a rescan.
261 if (!(ctrl
->subsys
->cmic
& (1 << 1)) || !multipath
)
264 q
= blk_alloc_queue_node(GFP_KERNEL
, ctrl
->numa_node
);
268 blk_queue_make_request(q
, nvme_ns_head_make_request
);
269 blk_queue_flag_set(QUEUE_FLAG_NONROT
, q
);
270 /* set to a default value for 512 until disk is validated */
271 blk_queue_logical_block_size(q
, 512);
272 blk_set_stacking_limits(&q
->limits
);
274 /* we need to propagate up the VMC settings */
275 if (ctrl
->vwc
& NVME_CTRL_VWC_PRESENT
)
277 blk_queue_write_cache(q
, vwc
, vwc
);
279 head
->disk
= alloc_disk(0);
281 goto out_cleanup_queue
;
282 head
->disk
->fops
= &nvme_ns_head_ops
;
283 head
->disk
->private_data
= head
;
284 head
->disk
->queue
= q
;
285 head
->disk
->flags
= GENHD_FL_EXT_DEVT
;
286 sprintf(head
->disk
->disk_name
, "nvme%dn%d",
287 ctrl
->subsys
->instance
, head
->instance
);
291 blk_cleanup_queue(q
);
296 static void nvme_mpath_set_live(struct nvme_ns
*ns
)
298 struct nvme_ns_head
*head
= ns
->head
;
300 lockdep_assert_held(&ns
->head
->lock
);
305 if (!(head
->disk
->flags
& GENHD_FL_UP
))
306 device_add_disk(&head
->subsys
->dev
, head
->disk
,
307 nvme_ns_id_attr_groups
);
309 if (nvme_path_is_optimized(ns
)) {
312 srcu_idx
= srcu_read_lock(&head
->srcu
);
314 __nvme_find_path(head
, node
);
315 srcu_read_unlock(&head
->srcu
, srcu_idx
);
318 kblockd_schedule_work(&ns
->head
->requeue_work
);
321 static int nvme_parse_ana_log(struct nvme_ctrl
*ctrl
, void *data
,
322 int (*cb
)(struct nvme_ctrl
*ctrl
, struct nvme_ana_group_desc
*,
325 void *base
= ctrl
->ana_log_buf
;
326 size_t offset
= sizeof(struct nvme_ana_rsp_hdr
);
329 lockdep_assert_held(&ctrl
->ana_lock
);
331 for (i
= 0; i
< le16_to_cpu(ctrl
->ana_log_buf
->ngrps
); i
++) {
332 struct nvme_ana_group_desc
*desc
= base
+ offset
;
333 u32 nr_nsids
= le32_to_cpu(desc
->nnsids
);
334 size_t nsid_buf_size
= nr_nsids
* sizeof(__le32
);
336 if (WARN_ON_ONCE(desc
->grpid
== 0))
338 if (WARN_ON_ONCE(le32_to_cpu(desc
->grpid
) > ctrl
->anagrpmax
))
340 if (WARN_ON_ONCE(desc
->state
== 0))
342 if (WARN_ON_ONCE(desc
->state
> NVME_ANA_CHANGE
))
345 offset
+= sizeof(*desc
);
346 if (WARN_ON_ONCE(offset
> ctrl
->ana_log_size
- nsid_buf_size
))
349 error
= cb(ctrl
, desc
, data
);
353 offset
+= nsid_buf_size
;
354 if (WARN_ON_ONCE(offset
> ctrl
->ana_log_size
- sizeof(*desc
)))
361 static inline bool nvme_state_is_live(enum nvme_ana_state state
)
363 return state
== NVME_ANA_OPTIMIZED
|| state
== NVME_ANA_NONOPTIMIZED
;
366 static void nvme_update_ns_ana_state(struct nvme_ana_group_desc
*desc
,
369 enum nvme_ana_state old
;
371 mutex_lock(&ns
->head
->lock
);
373 ns
->ana_grpid
= le32_to_cpu(desc
->grpid
);
374 ns
->ana_state
= desc
->state
;
375 clear_bit(NVME_NS_ANA_PENDING
, &ns
->flags
);
377 if (nvme_state_is_live(ns
->ana_state
) && !nvme_state_is_live(old
))
378 nvme_mpath_set_live(ns
);
379 mutex_unlock(&ns
->head
->lock
);
382 static int nvme_update_ana_state(struct nvme_ctrl
*ctrl
,
383 struct nvme_ana_group_desc
*desc
, void *data
)
385 u32 nr_nsids
= le32_to_cpu(desc
->nnsids
), n
= 0;
386 unsigned *nr_change_groups
= data
;
389 dev_info(ctrl
->device
, "ANA group %d: %s.\n",
390 le32_to_cpu(desc
->grpid
),
391 nvme_ana_state_names
[desc
->state
]);
393 if (desc
->state
== NVME_ANA_CHANGE
)
394 (*nr_change_groups
)++;
399 down_write(&ctrl
->namespaces_rwsem
);
400 list_for_each_entry(ns
, &ctrl
->namespaces
, list
) {
401 if (ns
->head
->ns_id
!= le32_to_cpu(desc
->nsids
[n
]))
403 nvme_update_ns_ana_state(desc
, ns
);
407 up_write(&ctrl
->namespaces_rwsem
);
408 WARN_ON_ONCE(n
< nr_nsids
);
412 static int nvme_read_ana_log(struct nvme_ctrl
*ctrl
, bool groups_only
)
414 u32 nr_change_groups
= 0;
417 mutex_lock(&ctrl
->ana_lock
);
418 error
= nvme_get_log(ctrl
, NVME_NSID_ALL
, NVME_LOG_ANA
,
419 groups_only
? NVME_ANA_LOG_RGO
: 0,
420 ctrl
->ana_log_buf
, ctrl
->ana_log_size
, 0);
422 dev_warn(ctrl
->device
, "Failed to get ANA log: %d\n", error
);
426 error
= nvme_parse_ana_log(ctrl
, &nr_change_groups
,
427 nvme_update_ana_state
);
432 * In theory we should have an ANATT timer per group as they might enter
433 * the change state at different times. But that is a lot of overhead
434 * just to protect against a target that keeps entering new changes
435 * states while never finishing previous ones. But we'll still
436 * eventually time out once all groups are in change state, so this
439 * We also double the ANATT value to provide some slack for transports
440 * or AEN processing overhead.
442 if (nr_change_groups
)
443 mod_timer(&ctrl
->anatt_timer
, ctrl
->anatt
* HZ
* 2 + jiffies
);
445 del_timer_sync(&ctrl
->anatt_timer
);
447 mutex_unlock(&ctrl
->ana_lock
);
451 static void nvme_ana_work(struct work_struct
*work
)
453 struct nvme_ctrl
*ctrl
= container_of(work
, struct nvme_ctrl
, ana_work
);
455 nvme_read_ana_log(ctrl
, false);
458 static void nvme_anatt_timeout(struct timer_list
*t
)
460 struct nvme_ctrl
*ctrl
= from_timer(ctrl
, t
, anatt_timer
);
462 dev_info(ctrl
->device
, "ANATT timeout, resetting controller.\n");
463 nvme_reset_ctrl(ctrl
);
466 void nvme_mpath_stop(struct nvme_ctrl
*ctrl
)
468 if (!nvme_ctrl_use_ana(ctrl
))
470 del_timer_sync(&ctrl
->anatt_timer
);
471 cancel_work_sync(&ctrl
->ana_work
);
474 static ssize_t
ana_grpid_show(struct device
*dev
, struct device_attribute
*attr
,
477 return sprintf(buf
, "%d\n", nvme_get_ns_from_dev(dev
)->ana_grpid
);
479 DEVICE_ATTR_RO(ana_grpid
);
481 static ssize_t
ana_state_show(struct device
*dev
, struct device_attribute
*attr
,
484 struct nvme_ns
*ns
= nvme_get_ns_from_dev(dev
);
486 return sprintf(buf
, "%s\n", nvme_ana_state_names
[ns
->ana_state
]);
488 DEVICE_ATTR_RO(ana_state
);
490 static int nvme_set_ns_ana_state(struct nvme_ctrl
*ctrl
,
491 struct nvme_ana_group_desc
*desc
, void *data
)
493 struct nvme_ns
*ns
= data
;
495 if (ns
->ana_grpid
== le32_to_cpu(desc
->grpid
)) {
496 nvme_update_ns_ana_state(desc
, ns
);
497 return -ENXIO
; /* just break out of the loop */
503 void nvme_mpath_add_disk(struct nvme_ns
*ns
, struct nvme_id_ns
*id
)
505 if (nvme_ctrl_use_ana(ns
->ctrl
)) {
506 mutex_lock(&ns
->ctrl
->ana_lock
);
507 ns
->ana_grpid
= le32_to_cpu(id
->anagrpid
);
508 nvme_parse_ana_log(ns
->ctrl
, ns
, nvme_set_ns_ana_state
);
509 mutex_unlock(&ns
->ctrl
->ana_lock
);
511 mutex_lock(&ns
->head
->lock
);
512 ns
->ana_state
= NVME_ANA_OPTIMIZED
;
513 nvme_mpath_set_live(ns
);
514 mutex_unlock(&ns
->head
->lock
);
518 void nvme_mpath_remove_disk(struct nvme_ns_head
*head
)
522 if (head
->disk
->flags
& GENHD_FL_UP
)
523 del_gendisk(head
->disk
);
524 blk_set_queue_dying(head
->disk
->queue
);
525 /* make sure all pending bios are cleaned up */
526 kblockd_schedule_work(&head
->requeue_work
);
527 flush_work(&head
->requeue_work
);
528 blk_cleanup_queue(head
->disk
->queue
);
529 put_disk(head
->disk
);
532 int nvme_mpath_init(struct nvme_ctrl
*ctrl
, struct nvme_id_ctrl
*id
)
536 if (!nvme_ctrl_use_ana(ctrl
))
539 ctrl
->anacap
= id
->anacap
;
540 ctrl
->anatt
= id
->anatt
;
541 ctrl
->nanagrpid
= le32_to_cpu(id
->nanagrpid
);
542 ctrl
->anagrpmax
= le32_to_cpu(id
->anagrpmax
);
544 mutex_init(&ctrl
->ana_lock
);
545 timer_setup(&ctrl
->anatt_timer
, nvme_anatt_timeout
, 0);
546 ctrl
->ana_log_size
= sizeof(struct nvme_ana_rsp_hdr
) +
547 ctrl
->nanagrpid
* sizeof(struct nvme_ana_group_desc
);
548 ctrl
->ana_log_size
+= ctrl
->max_namespaces
* sizeof(__le32
);
550 if (ctrl
->ana_log_size
> ctrl
->max_hw_sectors
<< SECTOR_SHIFT
) {
551 dev_err(ctrl
->device
,
552 "ANA log page size (%zd) larger than MDTS (%d).\n",
554 ctrl
->max_hw_sectors
<< SECTOR_SHIFT
);
555 dev_err(ctrl
->device
, "disabling ANA support.\n");
559 INIT_WORK(&ctrl
->ana_work
, nvme_ana_work
);
560 ctrl
->ana_log_buf
= kmalloc(ctrl
->ana_log_size
, GFP_KERNEL
);
561 if (!ctrl
->ana_log_buf
) {
566 error
= nvme_read_ana_log(ctrl
, true);
568 goto out_free_ana_log_buf
;
570 out_free_ana_log_buf
:
571 kfree(ctrl
->ana_log_buf
);
572 ctrl
->ana_log_buf
= NULL
;
577 void nvme_mpath_uninit(struct nvme_ctrl
*ctrl
)
579 kfree(ctrl
->ana_log_buf
);
580 ctrl
->ana_log_buf
= NULL
;