]> git.proxmox.com Git - mirror_ubuntu-hirsute-kernel.git/blob - drivers/nvme/host/core.c
projects / mirror_ubuntu-hirsute-kernel.git / blob
? search:


56e2a22e8a029922c03148cec77c066db3cc5251
[mirror_ubuntu-hirsute-kernel.git] / drivers / nvme / host / core.c
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * NVM Express device driver
4 * Copyright (c) 2011-2014, Intel Corporation.
5 */
6
7 #include <linux/blkdev.h>
8 #include <linux/blk-mq.h>
9 #include <linux/compat.h>
10 #include <linux/delay.h>
11 #include <linux/errno.h>
12 #include <linux/hdreg.h>
13 #include <linux/kernel.h>
14 #include <linux/module.h>
15 #include <linux/backing-dev.h>
16 #include <linux/list_sort.h>
17 #include <linux/slab.h>
18 #include <linux/types.h>
19 #include <linux/pr.h>
20 #include <linux/ptrace.h>
21 #include <linux/nvme_ioctl.h>
22 #include <linux/pm_qos.h>
23 #include <asm/unaligned.h>
24
25 #include "nvme.h"
26 #include "fabrics.h"
27
28 #define CREATE_TRACE_POINTS
29 #include "trace.h"
30
31 #define NVME_MINORS (1U << MINORBITS)
32
33 unsigned int admin_timeout = 60;
34 module_param(admin_timeout, uint, 0644);
35 MODULE_PARM_DESC(admin_timeout, "timeout in seconds for admin commands");
36 EXPORT_SYMBOL_GPL(admin_timeout);
37
38 unsigned int nvme_io_timeout = 30;
39 module_param_named(io_timeout, nvme_io_timeout, uint, 0644);
40 MODULE_PARM_DESC(io_timeout, "timeout in seconds for I/O");
41 EXPORT_SYMBOL_GPL(nvme_io_timeout);
42
43 static unsigned char shutdown_timeout = 5;
44 module_param(shutdown_timeout, byte, 0644);
45 MODULE_PARM_DESC(shutdown_timeout, "timeout in seconds for controller shutdown");
46
47 static u8 nvme_max_retries = 5;
48 module_param_named(max_retries, nvme_max_retries, byte, 0644);
49 MODULE_PARM_DESC(max_retries, "max number of retries a command may have");
50
51 static unsigned long default_ps_max_latency_us = 100000;
52 module_param(default_ps_max_latency_us, ulong, 0644);
53 MODULE_PARM_DESC(default_ps_max_latency_us,
54 "max power saving latency for new devices; use PM QOS to change per device");
55
56 static bool force_apst;
57 module_param(force_apst, bool, 0644);
58 MODULE_PARM_DESC(force_apst, "allow APST for newly enumerated devices even if quirked off");
59
60 static bool streams;
61 module_param(streams, bool, 0644);
62 MODULE_PARM_DESC(streams, "turn on support for Streams write directives");
63
64 /*
65 * nvme_wq - hosts nvme related works that are not reset or delete
66 * nvme_reset_wq - hosts nvme reset works
67 * nvme_delete_wq - hosts nvme delete works
68 *
69 * nvme_wq will host works such as scan, aen handling, fw activation,
70 * keep-alive, periodic reconnects etc. nvme_reset_wq
71 * runs reset works which also flush works hosted on nvme_wq for
72 * serialization purposes. nvme_delete_wq host controller deletion
73 * works which flush reset works for serialization.
74 */
75 struct workqueue_struct *nvme_wq;
76 EXPORT_SYMBOL_GPL(nvme_wq);
77
78 struct workqueue_struct *nvme_reset_wq;
79 EXPORT_SYMBOL_GPL(nvme_reset_wq);
80
81 struct workqueue_struct *nvme_delete_wq;
82 EXPORT_SYMBOL_GPL(nvme_delete_wq);
83
84 static LIST_HEAD(nvme_subsystems);
85 static DEFINE_MUTEX(nvme_subsystems_lock);
86
87 static DEFINE_IDA(nvme_instance_ida);
88 static dev_t nvme_chr_devt;
89 static struct class *nvme_class;
90 static struct class *nvme_subsys_class;
91
92 static void nvme_put_subsystem(struct nvme_subsystem *subsys);
93 static void nvme_remove_invalid_namespaces(struct nvme_ctrl *ctrl,
94 unsigned nsid);
95
96 static void nvme_update_bdev_size(struct gendisk *disk)
97 {
98 struct block_device *bdev = bdget_disk(disk, 0);
99
100 if (bdev) {
101 bd_set_nr_sectors(bdev, get_capacity(disk));
102 bdput(bdev);
103 }
104 }
105
106 /*
107 * Prepare a queue for teardown.
108 *
109 * This must forcibly unquiesce queues to avoid blocking dispatch, and only set
110 * the capacity to 0 after that to avoid blocking dispatchers that may be
111 * holding bd_butex. This will end buffered writers dirtying pages that can't
112 * be synced.
113 */
114 static void nvme_set_queue_dying(struct nvme_ns *ns)
115 {
116 if (test_and_set_bit(NVME_NS_DEAD, &ns->flags))
117 return;
118
119 blk_set_queue_dying(ns->queue);
120 blk_mq_unquiesce_queue(ns->queue);
121
122 set_capacity(ns->disk, 0);
123 nvme_update_bdev_size(ns->disk);
124 }
125
126 static void nvme_queue_scan(struct nvme_ctrl *ctrl)
127 {
128 /*
129 * Only new queue scan work when admin and IO queues are both alive
130 */
131 if (ctrl->state == NVME_CTRL_LIVE && ctrl->tagset)
132 queue_work(nvme_wq, &ctrl->scan_work);
133 }
134
135 /*
136 * Use this function to proceed with scheduling reset_work for a controller
137 * that had previously been set to the resetting state. This is intended for
138 * code paths that can't be interrupted by other reset attempts. A hot removal
139 * may prevent this from succeeding.
140 */
141 int nvme_try_sched_reset(struct nvme_ctrl *ctrl)
142 {
143 if (ctrl->state != NVME_CTRL_RESETTING)
144 return -EBUSY;
145 if (!queue_work(nvme_reset_wq, &ctrl->reset_work))
146 return -EBUSY;
147 return 0;
148 }
149 EXPORT_SYMBOL_GPL(nvme_try_sched_reset);
150
151 int nvme_reset_ctrl(struct nvme_ctrl *ctrl)
152 {
153 if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_RESETTING))
154 return -EBUSY;
155 if (!queue_work(nvme_reset_wq, &ctrl->reset_work))
156 return -EBUSY;
157 return 0;
158 }
159 EXPORT_SYMBOL_GPL(nvme_reset_ctrl);
160
161 int nvme_reset_ctrl_sync(struct nvme_ctrl *ctrl)
162 {
163 int ret;
164
165 ret = nvme_reset_ctrl(ctrl);
166 if (!ret) {
167 flush_work(&ctrl->reset_work);
168 if (ctrl->state != NVME_CTRL_LIVE)
169 ret = -ENETRESET;
170 }
171
172 return ret;
173 }
174 EXPORT_SYMBOL_GPL(nvme_reset_ctrl_sync);
175
176 static void nvme_do_delete_ctrl(struct nvme_ctrl *ctrl)
177 {
178 dev_info(ctrl->device,
179 "Removing ctrl: NQN \"%s\"\n", ctrl->opts->subsysnqn);
180
181 flush_work(&ctrl->reset_work);
182 nvme_stop_ctrl(ctrl);
183 nvme_remove_namespaces(ctrl);
184 ctrl->ops->delete_ctrl(ctrl);
185 nvme_uninit_ctrl(ctrl);
186 }
187
188 static void nvme_delete_ctrl_work(struct work_struct *work)
189 {
190 struct nvme_ctrl *ctrl =
191 container_of(work, struct nvme_ctrl, delete_work);
192
193 nvme_do_delete_ctrl(ctrl);
194 }
195
196 int nvme_delete_ctrl(struct nvme_ctrl *ctrl)
197 {
198 if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_DELETING))
199 return -EBUSY;
200 if (!queue_work(nvme_delete_wq, &ctrl->delete_work))
201 return -EBUSY;
202 return 0;
203 }
204 EXPORT_SYMBOL_GPL(nvme_delete_ctrl);
205
206 static void nvme_delete_ctrl_sync(struct nvme_ctrl *ctrl)
207 {
208 /*
209 * Keep a reference until nvme_do_delete_ctrl() complete,
210 * since ->delete_ctrl can free the controller.
211 */
212 nvme_get_ctrl(ctrl);
213 if (nvme_change_ctrl_state(ctrl, NVME_CTRL_DELETING))
214 nvme_do_delete_ctrl(ctrl);
215 nvme_put_ctrl(ctrl);
216 }
217
218 static blk_status_t nvme_error_status(u16 status)
219 {
220 switch (status & 0x7ff) {
221 case NVME_SC_SUCCESS:
222 return BLK_STS_OK;
223 case NVME_SC_CAP_EXCEEDED:
224 return BLK_STS_NOSPC;
225 case NVME_SC_LBA_RANGE:
226 case NVME_SC_CMD_INTERRUPTED:
227 case NVME_SC_NS_NOT_READY:
228 return BLK_STS_TARGET;
229 case NVME_SC_BAD_ATTRIBUTES:
230 case NVME_SC_ONCS_NOT_SUPPORTED:
231 case NVME_SC_INVALID_OPCODE:
232 case NVME_SC_INVALID_FIELD:
233 case NVME_SC_INVALID_NS:
234 return BLK_STS_NOTSUPP;
235 case NVME_SC_WRITE_FAULT:
236 case NVME_SC_READ_ERROR:
237 case NVME_SC_UNWRITTEN_BLOCK:
238 case NVME_SC_ACCESS_DENIED:
239 case NVME_SC_READ_ONLY:
240 case NVME_SC_COMPARE_FAILED:
241 return BLK_STS_MEDIUM;
242 case NVME_SC_GUARD_CHECK:
243 case NVME_SC_APPTAG_CHECK:
244 case NVME_SC_REFTAG_CHECK:
245 case NVME_SC_INVALID_PI:
246 return BLK_STS_PROTECTION;
247 case NVME_SC_RESERVATION_CONFLICT:
248 return BLK_STS_NEXUS;
249 case NVME_SC_HOST_PATH_ERROR:
250 return BLK_STS_TRANSPORT;
251 default:
252 return BLK_STS_IOERR;
253 }
254 }
255
256 static void nvme_retry_req(struct request *req)
257 {
258 struct nvme_ns *ns = req->q->queuedata;
259 unsigned long delay = 0;
260 u16 crd;
261
262 /* The mask and shift result must be <= 3 */
263 crd = (nvme_req(req)->status & NVME_SC_CRD) >> 11;
264 if (ns && crd)
265 delay = ns->ctrl->crdt[crd - 1] * 100;
266
267 nvme_req(req)->retries++;
268 blk_mq_requeue_request(req, false);
269 blk_mq_delay_kick_requeue_list(req->q, delay);
270 }
271
272 enum nvme_disposition {
273 COMPLETE,
274 RETRY,
275 FAILOVER,
276 };
277
278 static inline enum nvme_disposition nvme_decide_disposition(struct request *req)
279 {
280 if (likely(nvme_req(req)->status == 0))
281 return COMPLETE;
282
283 if (blk_noretry_request(req) ||
284 (nvme_req(req)->status & NVME_SC_DNR) ||
285 nvme_req(req)->retries >= nvme_max_retries)
286 return COMPLETE;
287
288 if (req->cmd_flags & REQ_NVME_MPATH) {
289 if (nvme_is_path_error(nvme_req(req)->status) ||
290 blk_queue_dying(req->q))
291 return FAILOVER;
292 } else {
293 if (blk_queue_dying(req->q))
294 return COMPLETE;
295 }
296
297 return RETRY;
298 }
299
300 static inline void nvme_end_req(struct request *req)
301 {
302 blk_status_t status = nvme_error_status(nvme_req(req)->status);
303
304 if (IS_ENABLED(CONFIG_BLK_DEV_ZONED) &&
305 req_op(req) == REQ_OP_ZONE_APPEND)
306 req->__sector = nvme_lba_to_sect(req->q->queuedata,
307 le64_to_cpu(nvme_req(req)->result.u64));
308
309 nvme_trace_bio_complete(req, status);
310 blk_mq_end_request(req, status);
311 }
312
313 void nvme_complete_rq(struct request *req)
314 {
315 trace_nvme_complete_rq(req);
316 nvme_cleanup_cmd(req);
317
318 if (nvme_req(req)->ctrl->kas)
319 nvme_req(req)->ctrl->comp_seen = true;
320
321 switch (nvme_decide_disposition(req)) {
322 case COMPLETE:
323 nvme_end_req(req);
324 return;
325 case RETRY:
326 nvme_retry_req(req);
327 return;
328 case FAILOVER:
329 nvme_failover_req(req);
330 return;
331 }
332 }
333 EXPORT_SYMBOL_GPL(nvme_complete_rq);
334
335 bool nvme_cancel_request(struct request *req, void *data, bool reserved)
336 {
337 dev_dbg_ratelimited(((struct nvme_ctrl *) data)->device,
338 "Cancelling I/O %d", req->tag);
339
340 /* don't abort one completed request */
341 if (blk_mq_request_completed(req))
342 return true;
343
344 nvme_req(req)->status = NVME_SC_HOST_ABORTED_CMD;
345 blk_mq_complete_request(req);
346 return true;
347 }
348 EXPORT_SYMBOL_GPL(nvme_cancel_request);
349
350 bool nvme_change_ctrl_state(struct nvme_ctrl *ctrl,
351 enum nvme_ctrl_state new_state)
352 {
353 enum nvme_ctrl_state old_state;
354 unsigned long flags;
355 bool changed = false;
356
357 spin_lock_irqsave(&ctrl->lock, flags);
358
359 old_state = ctrl->state;
360 switch (new_state) {
361 case NVME_CTRL_LIVE:
362 switch (old_state) {
363 case NVME_CTRL_NEW:
364 case NVME_CTRL_RESETTING:
365 case NVME_CTRL_CONNECTING:
366 changed = true;
367 fallthrough;
368 default:
369 break;
370 }
371 break;
372 case NVME_CTRL_RESETTING:
373 switch (old_state) {
374 case NVME_CTRL_NEW:
375 case NVME_CTRL_LIVE:
376 changed = true;
377 fallthrough;
378 default:
379 break;
380 }
381 break;
382 case NVME_CTRL_CONNECTING:
383 switch (old_state) {
384 case NVME_CTRL_NEW:
385 case NVME_CTRL_RESETTING:
386 changed = true;
387 fallthrough;
388 default:
389 break;
390 }
391 break;
392 case NVME_CTRL_DELETING:
393 switch (old_state) {
394 case NVME_CTRL_LIVE:
395 case NVME_CTRL_RESETTING:
396 case NVME_CTRL_CONNECTING:
397 changed = true;
398 fallthrough;
399 default:
400 break;
401 }
402 break;
403 case NVME_CTRL_DELETING_NOIO:
404 switch (old_state) {
405 case NVME_CTRL_DELETING:
406 case NVME_CTRL_DEAD:
407 changed = true;
408 fallthrough;
409 default:
410 break;
411 }
412 break;
413 case NVME_CTRL_DEAD:
414 switch (old_state) {
415 case NVME_CTRL_DELETING:
416 changed = true;
417 fallthrough;
418 default:
419 break;
420 }
421 break;
422 default:
423 break;
424 }
425
426 if (changed) {
427 ctrl->state = new_state;
428 wake_up_all(&ctrl->state_wq);
429 }
430
431 spin_unlock_irqrestore(&ctrl->lock, flags);
432 if (changed && ctrl->state == NVME_CTRL_LIVE)
433 nvme_kick_requeue_lists(ctrl);
434 return changed;
435 }
436 EXPORT_SYMBOL_GPL(nvme_change_ctrl_state);
437
438 /*
439 * Returns true for sink states that can't ever transition back to live.
440 */
441 static bool nvme_state_terminal(struct nvme_ctrl *ctrl)
442 {
443 switch (ctrl->state) {
444 case NVME_CTRL_NEW:
445 case NVME_CTRL_LIVE:
446 case NVME_CTRL_RESETTING:
447 case NVME_CTRL_CONNECTING:
448 return false;
449 case NVME_CTRL_DELETING:
450 case NVME_CTRL_DELETING_NOIO:
451 case NVME_CTRL_DEAD:
452 return true;
453 default:
454 WARN_ONCE(1, "Unhandled ctrl state:%d", ctrl->state);
455 return true;
456 }
457 }
458
459 /*
460 * Waits for the controller state to be resetting, or returns false if it is
461 * not possible to ever transition to that state.
462 */
463 bool nvme_wait_reset(struct nvme_ctrl *ctrl)
464 {
465 wait_event(ctrl->state_wq,
466 nvme_change_ctrl_state(ctrl, NVME_CTRL_RESETTING) ||
467 nvme_state_terminal(ctrl));
468 return ctrl->state == NVME_CTRL_RESETTING;
469 }
470 EXPORT_SYMBOL_GPL(nvme_wait_reset);
471
472 static void nvme_free_ns_head(struct kref *ref)
473 {
474 struct nvme_ns_head *head =
475 container_of(ref, struct nvme_ns_head, ref);
476
477 nvme_mpath_remove_disk(head);
478 ida_simple_remove(&head->subsys->ns_ida, head->instance);
479 cleanup_srcu_struct(&head->srcu);
480 nvme_put_subsystem(head->subsys);
481 kfree(head);
482 }
483
484 static void nvme_put_ns_head(struct nvme_ns_head *head)
485 {
486 kref_put(&head->ref, nvme_free_ns_head);
487 }
488
489 static void nvme_free_ns(struct kref *kref)
490 {
491 struct nvme_ns *ns = container_of(kref, struct nvme_ns, kref);
492
493 if (ns->ndev)
494 nvme_nvm_unregister(ns);
495
496 put_disk(ns->disk);
497 nvme_put_ns_head(ns->head);
498 nvme_put_ctrl(ns->ctrl);
499 kfree(ns);
500 }
501
502 void nvme_put_ns(struct nvme_ns *ns)
503 {
504 kref_put(&ns->kref, nvme_free_ns);
505 }
506 EXPORT_SYMBOL_NS_GPL(nvme_put_ns, NVME_TARGET_PASSTHRU);
507
508 static inline void nvme_clear_nvme_request(struct request *req)
509 {
510 if (!(req->rq_flags & RQF_DONTPREP)) {
511 nvme_req(req)->retries = 0;
512 nvme_req(req)->flags = 0;
513 req->rq_flags |= RQF_DONTPREP;
514 }
515 }
516
517 struct request *nvme_alloc_request(struct request_queue *q,
518 struct nvme_command *cmd, blk_mq_req_flags_t flags, int qid)
519 {
520 unsigned op = nvme_is_write(cmd) ? REQ_OP_DRV_OUT : REQ_OP_DRV_IN;
521 struct request *req;
522
523 if (qid == NVME_QID_ANY) {
524 req = blk_mq_alloc_request(q, op, flags);
525 } else {
526 req = blk_mq_alloc_request_hctx(q, op, flags,
527 qid ? qid - 1 : 0);
528 }
529 if (IS_ERR(req))
530 return req;
531
532 req->cmd_flags |= REQ_FAILFAST_DRIVER;
533 nvme_clear_nvme_request(req);
534 nvme_req(req)->cmd = cmd;
535
536 return req;
537 }
538 EXPORT_SYMBOL_GPL(nvme_alloc_request);
539
540 static int nvme_toggle_streams(struct nvme_ctrl *ctrl, bool enable)
541 {
542 struct nvme_command c;
543
544 memset(&c, 0, sizeof(c));
545
546 c.directive.opcode = nvme_admin_directive_send;
547 c.directive.nsid = cpu_to_le32(NVME_NSID_ALL);
548 c.directive.doper = NVME_DIR_SND_ID_OP_ENABLE;
549 c.directive.dtype = NVME_DIR_IDENTIFY;
550 c.directive.tdtype = NVME_DIR_STREAMS;
551 c.directive.endir = enable ? NVME_DIR_ENDIR : 0;
552
553 return nvme_submit_sync_cmd(ctrl->admin_q, &c, NULL, 0);
554 }
555
556 static int nvme_disable_streams(struct nvme_ctrl *ctrl)
557 {
558 return nvme_toggle_streams(ctrl, false);
559 }
560
561 static int nvme_enable_streams(struct nvme_ctrl *ctrl)
562 {
563 return nvme_toggle_streams(ctrl, true);
564 }
565
566 static int nvme_get_stream_params(struct nvme_ctrl *ctrl,
567 struct streams_directive_params *s, u32 nsid)
568 {
569 struct nvme_command c;
570
571 memset(&c, 0, sizeof(c));
572 memset(s, 0, sizeof(*s));
573
574 c.directive.opcode = nvme_admin_directive_recv;
575 c.directive.nsid = cpu_to_le32(nsid);
576 c.directive.numd = cpu_to_le32(nvme_bytes_to_numd(sizeof(*s)));
577 c.directive.doper = NVME_DIR_RCV_ST_OP_PARAM;
578 c.directive.dtype = NVME_DIR_STREAMS;
579
580 return nvme_submit_sync_cmd(ctrl->admin_q, &c, s, sizeof(*s));
581 }
582
583 static int nvme_configure_directives(struct nvme_ctrl *ctrl)
584 {
585 struct streams_directive_params s;
586 int ret;
587
588 if (!(ctrl->oacs & NVME_CTRL_OACS_DIRECTIVES))
589 return 0;
590 if (!streams)
591 return 0;
592
593 ret = nvme_enable_streams(ctrl);
594 if (ret)
595 return ret;
596
597 ret = nvme_get_stream_params(ctrl, &s, NVME_NSID_ALL);
598 if (ret)
599 goto out_disable_stream;
600
601 ctrl->nssa = le16_to_cpu(s.nssa);
602 if (ctrl->nssa < BLK_MAX_WRITE_HINTS - 1) {
603 dev_info(ctrl->device, "too few streams (%u) available\n",
604 ctrl->nssa);
605 goto out_disable_stream;
606 }
607
608 ctrl->nr_streams = min_t(u16, ctrl->nssa, BLK_MAX_WRITE_HINTS - 1);
609 dev_info(ctrl->device, "Using %u streams\n", ctrl->nr_streams);
610 return 0;
611
612 out_disable_stream:
613 nvme_disable_streams(ctrl);
614 return ret;
615 }
616
617 /*
618 * Check if 'req' has a write hint associated with it. If it does, assign
619 * a valid namespace stream to the write.
620 */
621 static void nvme_assign_write_stream(struct nvme_ctrl *ctrl,
622 struct request *req, u16 *control,
623 u32 *dsmgmt)
624 {
625 enum rw_hint streamid = req->write_hint;
626
627 if (streamid == WRITE_LIFE_NOT_SET || streamid == WRITE_LIFE_NONE)
628 streamid = 0;
629 else {
630 streamid--;
631 if (WARN_ON_ONCE(streamid > ctrl->nr_streams))
632 return;
633
634 *control |= NVME_RW_DTYPE_STREAMS;
635 *dsmgmt |= streamid << 16;
636 }
637
638 if (streamid < ARRAY_SIZE(req->q->write_hints))
639 req->q->write_hints[streamid] += blk_rq_bytes(req) >> 9;
640 }
641
642 static void nvme_setup_passthrough(struct request *req,
643 struct nvme_command *cmd)
644 {
645 memcpy(cmd, nvme_req(req)->cmd, sizeof(*cmd));
646 /* passthru commands should let the driver set the SGL flags */
647 cmd->common.flags &= ~NVME_CMD_SGL_ALL;
648 }
649
650 static inline void nvme_setup_flush(struct nvme_ns *ns,
651 struct nvme_command *cmnd)
652 {
653 cmnd->common.opcode = nvme_cmd_flush;
654 cmnd->common.nsid = cpu_to_le32(ns->head->ns_id);
655 }
656
657 static blk_status_t nvme_setup_discard(struct nvme_ns *ns, struct request *req,
658 struct nvme_command *cmnd)
659 {
660 unsigned short segments = blk_rq_nr_discard_segments(req), n = 0;
661 struct nvme_dsm_range *range;
662 struct bio *bio;
663
664 /*
665 * Some devices do not consider the DSM 'Number of Ranges' field when
666 * determining how much data to DMA. Always allocate memory for maximum
667 * number of segments to prevent device reading beyond end of buffer.
668 */
669 static const size_t alloc_size = sizeof(*range) * NVME_DSM_MAX_RANGES;
670
671 range = kzalloc(alloc_size, GFP_ATOMIC | __GFP_NOWARN);
672 if (!range) {
673 /*
674 * If we fail allocation our range, fallback to the controller
675 * discard page. If that's also busy, it's safe to return
676 * busy, as we know we can make progress once that's freed.
677 */
678 if (test_and_set_bit_lock(0, &ns->ctrl->discard_page_busy))
679 return BLK_STS_RESOURCE;
680
681 range = page_address(ns->ctrl->discard_page);
682 }
683
684 __rq_for_each_bio(bio, req) {
685 u64 slba = nvme_sect_to_lba(ns, bio->bi_iter.bi_sector);
686 u32 nlb = bio->bi_iter.bi_size >> ns->lba_shift;
687
688 if (n < segments) {
689 range[n].cattr = cpu_to_le32(0);
690 range[n].nlb = cpu_to_le32(nlb);
691 range[n].slba = cpu_to_le64(slba);
692 }
693 n++;
694 }
695
696 if (WARN_ON_ONCE(n != segments)) {
697 if (virt_to_page(range) == ns->ctrl->discard_page)
698 clear_bit_unlock(0, &ns->ctrl->discard_page_busy);
699 else
700 kfree(range);
701 return BLK_STS_IOERR;
702 }
703
704 cmnd->dsm.opcode = nvme_cmd_dsm;
705 cmnd->dsm.nsid = cpu_to_le32(ns->head->ns_id);
706 cmnd->dsm.nr = cpu_to_le32(segments - 1);
707 cmnd->dsm.attributes = cpu_to_le32(NVME_DSMGMT_AD);
708
709 req->special_vec.bv_page = virt_to_page(range);
710 req->special_vec.bv_offset = offset_in_page(range);
711 req->special_vec.bv_len = alloc_size;
712 req->rq_flags |= RQF_SPECIAL_PAYLOAD;
713
714 return BLK_STS_OK;
715 }
716
717 static inline blk_status_t nvme_setup_write_zeroes(struct nvme_ns *ns,
718 struct request *req, struct nvme_command *cmnd)
719 {
720 if (ns->ctrl->quirks & NVME_QUIRK_DEALLOCATE_ZEROES)
721 return nvme_setup_discard(ns, req, cmnd);
722
723 cmnd->write_zeroes.opcode = nvme_cmd_write_zeroes;
724 cmnd->write_zeroes.nsid = cpu_to_le32(ns->head->ns_id);
725 cmnd->write_zeroes.slba =
726 cpu_to_le64(nvme_sect_to_lba(ns, blk_rq_pos(req)));
727 cmnd->write_zeroes.length =
728 cpu_to_le16((blk_rq_bytes(req) >> ns->lba_shift) - 1);
729 cmnd->write_zeroes.control = 0;
730 return BLK_STS_OK;
731 }
732
733 static inline blk_status_t nvme_setup_rw(struct nvme_ns *ns,
734 struct request *req, struct nvme_command *cmnd,
735 enum nvme_opcode op)
736 {
737 struct nvme_ctrl *ctrl = ns->ctrl;
738 u16 control = 0;
739 u32 dsmgmt = 0;
740
741 if (req->cmd_flags & REQ_FUA)
742 control |= NVME_RW_FUA;
743 if (req->cmd_flags & (REQ_FAILFAST_DEV | REQ_RAHEAD))
744 control |= NVME_RW_LR;
745
746 if (req->cmd_flags & REQ_RAHEAD)
747 dsmgmt |= NVME_RW_DSM_FREQ_PREFETCH;
748
749 cmnd->rw.opcode = op;
750 cmnd->rw.nsid = cpu_to_le32(ns->head->ns_id);
751 cmnd->rw.slba = cpu_to_le64(nvme_sect_to_lba(ns, blk_rq_pos(req)));
752 cmnd->rw.length = cpu_to_le16((blk_rq_bytes(req) >> ns->lba_shift) - 1);
753
754 if (req_op(req) == REQ_OP_WRITE && ctrl->nr_streams)
755 nvme_assign_write_stream(ctrl, req, &control, &dsmgmt);
756
757 if (ns->ms) {
758 /*
759 * If formated with metadata, the block layer always provides a
760 * metadata buffer if CONFIG_BLK_DEV_INTEGRITY is enabled. Else
761 * we enable the PRACT bit for protection information or set the
762 * namespace capacity to zero to prevent any I/O.
763 */
764 if (!blk_integrity_rq(req)) {
765 if (WARN_ON_ONCE(!nvme_ns_has_pi(ns)))
766 return BLK_STS_NOTSUPP;
767 control |= NVME_RW_PRINFO_PRACT;
768 }
769
770 switch (ns->pi_type) {
771 case NVME_NS_DPS_PI_TYPE3:
772 control |= NVME_RW_PRINFO_PRCHK_GUARD;
773 break;
774 case NVME_NS_DPS_PI_TYPE1:
775 case NVME_NS_DPS_PI_TYPE2:
776 control |= NVME_RW_PRINFO_PRCHK_GUARD |
777 NVME_RW_PRINFO_PRCHK_REF;
778 if (op == nvme_cmd_zone_append)
779 control |= NVME_RW_APPEND_PIREMAP;
780 cmnd->rw.reftag = cpu_to_le32(t10_pi_ref_tag(req));
781 break;
782 }
783 }
784
785 cmnd->rw.control = cpu_to_le16(control);
786 cmnd->rw.dsmgmt = cpu_to_le32(dsmgmt);
787 return 0;
788 }
789
790 void nvme_cleanup_cmd(struct request *req)
791 {
792 if (req->rq_flags & RQF_SPECIAL_PAYLOAD) {
793 struct nvme_ns *ns = req->rq_disk->private_data;
794 struct page *page = req->special_vec.bv_page;
795
796 if (page == ns->ctrl->discard_page)
797 clear_bit_unlock(0, &ns->ctrl->discard_page_busy);
798 else
799 kfree(page_address(page) + req->special_vec.bv_offset);
800 }
801 }
802 EXPORT_SYMBOL_GPL(nvme_cleanup_cmd);
803
804 blk_status_t nvme_setup_cmd(struct nvme_ns *ns, struct request *req,
805 struct nvme_command *cmd)
806 {
807 blk_status_t ret = BLK_STS_OK;
808
809 nvme_clear_nvme_request(req);
810
811 memset(cmd, 0, sizeof(*cmd));
812 switch (req_op(req)) {
813 case REQ_OP_DRV_IN:
814 case REQ_OP_DRV_OUT:
815 nvme_setup_passthrough(req, cmd);
816 break;
817 case REQ_OP_FLUSH:
818 nvme_setup_flush(ns, cmd);
819 break;
820 case REQ_OP_ZONE_RESET_ALL:
821 case REQ_OP_ZONE_RESET:
822 ret = nvme_setup_zone_mgmt_send(ns, req, cmd, NVME_ZONE_RESET);
823 break;
824 case REQ_OP_ZONE_OPEN:
825 ret = nvme_setup_zone_mgmt_send(ns, req, cmd, NVME_ZONE_OPEN);
826 break;
827 case REQ_OP_ZONE_CLOSE:
828 ret = nvme_setup_zone_mgmt_send(ns, req, cmd, NVME_ZONE_CLOSE);
829 break;
830 case REQ_OP_ZONE_FINISH:
831 ret = nvme_setup_zone_mgmt_send(ns, req, cmd, NVME_ZONE_FINISH);
832 break;
833 case REQ_OP_WRITE_ZEROES:
834 ret = nvme_setup_write_zeroes(ns, req, cmd);
835 break;
836 case REQ_OP_DISCARD:
837 ret = nvme_setup_discard(ns, req, cmd);
838 break;
839 case REQ_OP_READ:
840 ret = nvme_setup_rw(ns, req, cmd, nvme_cmd_read);
841 break;
842 case REQ_OP_WRITE:
843 ret = nvme_setup_rw(ns, req, cmd, nvme_cmd_write);
844 break;
845 case REQ_OP_ZONE_APPEND:
846 ret = nvme_setup_rw(ns, req, cmd, nvme_cmd_zone_append);
847 break;
848 default:
849 WARN_ON_ONCE(1);
850 return BLK_STS_IOERR;
851 }
852
853 cmd->common.command_id = req->tag;
854 trace_nvme_setup_cmd(req, cmd);
855 return ret;
856 }
857 EXPORT_SYMBOL_GPL(nvme_setup_cmd);
858
859 static void nvme_end_sync_rq(struct request *rq, blk_status_t error)
860 {
861 struct completion *waiting = rq->end_io_data;
862
863 rq->end_io_data = NULL;
864 complete(waiting);
865 }
866
867 static void nvme_execute_rq_polled(struct request_queue *q,
868 struct gendisk *bd_disk, struct request *rq, int at_head)
869 {
870 DECLARE_COMPLETION_ONSTACK(wait);
871
872 WARN_ON_ONCE(!test_bit(QUEUE_FLAG_POLL, &q->queue_flags));
873
874 rq->cmd_flags |= REQ_HIPRI;
875 rq->end_io_data = &wait;
876 blk_execute_rq_nowait(q, bd_disk, rq, at_head, nvme_end_sync_rq);
877
878 while (!completion_done(&wait)) {
879 blk_poll(q, request_to_qc_t(rq->mq_hctx, rq), true);
880 cond_resched();
881 }
882 }
883
884 /*
885 * Returns 0 on success. If the result is negative, it's a Linux error code;
886 * if the result is positive, it's an NVM Express status code
887 */
888 int __nvme_submit_sync_cmd(struct request_queue *q, struct nvme_command *cmd,
889 union nvme_result *result, void *buffer, unsigned bufflen,
890 unsigned timeout, int qid, int at_head,
891 blk_mq_req_flags_t flags, bool poll)
892 {
893 struct request *req;
894 int ret;
895
896 req = nvme_alloc_request(q, cmd, flags, qid);
897 if (IS_ERR(req))
898 return PTR_ERR(req);
899
900 req->timeout = timeout ? timeout : ADMIN_TIMEOUT;
901
902 if (buffer && bufflen) {
903 ret = blk_rq_map_kern(q, req, buffer, bufflen, GFP_KERNEL);
904 if (ret)
905 goto out;
906 }
907
908 if (poll)
909 nvme_execute_rq_polled(req->q, NULL, req, at_head);
910 else
911 blk_execute_rq(req->q, NULL, req, at_head);
912 if (result)
913 *result = nvme_req(req)->result;
914 if (nvme_req(req)->flags & NVME_REQ_CANCELLED)
915 ret = -EINTR;
916 else
917 ret = nvme_req(req)->status;
918 out:
919 blk_mq_free_request(req);
920 return ret;
921 }
922 EXPORT_SYMBOL_GPL(__nvme_submit_sync_cmd);
923
924 int nvme_submit_sync_cmd(struct request_queue *q, struct nvme_command *cmd,
925 void *buffer, unsigned bufflen)
926 {
927 return __nvme_submit_sync_cmd(q, cmd, NULL, buffer, bufflen, 0,
928 NVME_QID_ANY, 0, 0, false);
929 }
930 EXPORT_SYMBOL_GPL(nvme_submit_sync_cmd);
931
932 static void *nvme_add_user_metadata(struct bio *bio, void __user *ubuf,
933 unsigned len, u32 seed, bool write)
934 {
935 struct bio_integrity_payload *bip;
936 int ret = -ENOMEM;
937 void *buf;
938
939 buf = kmalloc(len, GFP_KERNEL);
940 if (!buf)
941 goto out;
942
943 ret = -EFAULT;
944 if (write && copy_from_user(buf, ubuf, len))
945 goto out_free_meta;
946
947 bip = bio_integrity_alloc(bio, GFP_KERNEL, 1);
948 if (IS_ERR(bip)) {
949 ret = PTR_ERR(bip);
950 goto out_free_meta;
951 }
952
953 bip->bip_iter.bi_size = len;
954 bip->bip_iter.bi_sector = seed;
955 ret = bio_integrity_add_page(bio, virt_to_page(buf), len,
956 offset_in_page(buf));
957 if (ret == len)
958 return buf;
959 ret = -ENOMEM;
960 out_free_meta:
961 kfree(buf);
962 out:
963 return ERR_PTR(ret);
964 }
965
966 static u32 nvme_known_admin_effects(u8 opcode)
967 {
968 switch (opcode) {
969 case nvme_admin_format_nvm:
970 return NVME_CMD_EFFECTS_LBCC | NVME_CMD_EFFECTS_NCC |
971 NVME_CMD_EFFECTS_CSE_MASK;
972 case nvme_admin_sanitize_nvm:
973 return NVME_CMD_EFFECTS_LBCC | NVME_CMD_EFFECTS_CSE_MASK;
974 default:
975 break;
976 }
977 return 0;
978 }
979
980 u32 nvme_command_effects(struct nvme_ctrl *ctrl, struct nvme_ns *ns, u8 opcode)
981 {
982 u32 effects = 0;
983
984 if (ns) {
985 if (ns->head->effects)
986 effects = le32_to_cpu(ns->head->effects->iocs[opcode]);
987 if (effects & ~(NVME_CMD_EFFECTS_CSUPP | NVME_CMD_EFFECTS_LBCC))
988 dev_warn(ctrl->device,
989 "IO command:%02x has unhandled effects:%08x\n",
990 opcode, effects);
991 return 0;
992 }
993
994 if (ctrl->effects)
995 effects = le32_to_cpu(ctrl->effects->acs[opcode]);
996 effects |= nvme_known_admin_effects(opcode);
997
998 return effects;
999 }
1000 EXPORT_SYMBOL_NS_GPL(nvme_command_effects, NVME_TARGET_PASSTHRU);
1001
1002 static u32 nvme_passthru_start(struct nvme_ctrl *ctrl, struct nvme_ns *ns,
1003 u8 opcode)
1004 {
1005 u32 effects = nvme_command_effects(ctrl, ns, opcode);
1006
1007 /*
1008 * For simplicity, IO to all namespaces is quiesced even if the command
1009 * effects say only one namespace is affected.
1010 */
1011 if (effects & NVME_CMD_EFFECTS_CSE_MASK) {
1012 mutex_lock(&ctrl->scan_lock);
1013 mutex_lock(&ctrl->subsys->lock);
1014 nvme_mpath_start_freeze(ctrl->subsys);
1015 nvme_mpath_wait_freeze(ctrl->subsys);
1016 nvme_start_freeze(ctrl);
1017 nvme_wait_freeze(ctrl);
1018 }
1019 return effects;
1020 }
1021
1022 static void nvme_passthru_end(struct nvme_ctrl *ctrl, u32 effects)
1023 {
1024 if (effects & NVME_CMD_EFFECTS_CSE_MASK) {
1025 nvme_unfreeze(ctrl);
1026 nvme_mpath_unfreeze(ctrl->subsys);
1027 mutex_unlock(&ctrl->subsys->lock);
1028 nvme_remove_invalid_namespaces(ctrl, NVME_NSID_ALL);
1029 mutex_unlock(&ctrl->scan_lock);
1030 }
1031 if (effects & NVME_CMD_EFFECTS_CCC)
1032 nvme_init_identify(ctrl);
1033 if (effects & (NVME_CMD_EFFECTS_NIC | NVME_CMD_EFFECTS_NCC)) {
1034 nvme_queue_scan(ctrl);
1035 flush_work(&ctrl->scan_work);
1036 }
1037 }
1038
1039 void nvme_execute_passthru_rq(struct request *rq)
1040 {
1041 struct nvme_command *cmd = nvme_req(rq)->cmd;
1042 struct nvme_ctrl *ctrl = nvme_req(rq)->ctrl;
1043 struct nvme_ns *ns = rq->q->queuedata;
1044 struct gendisk *disk = ns ? ns->disk : NULL;
1045 u32 effects;
1046
1047 effects = nvme_passthru_start(ctrl, ns, cmd->common.opcode);
1048 blk_execute_rq(rq->q, disk, rq, 0);
1049 nvme_passthru_end(ctrl, effects);
1050 }
1051 EXPORT_SYMBOL_NS_GPL(nvme_execute_passthru_rq, NVME_TARGET_PASSTHRU);
1052
1053 static int nvme_submit_user_cmd(struct request_queue *q,
1054 struct nvme_command *cmd, void __user *ubuffer,
1055 unsigned bufflen, void __user *meta_buffer, unsigned meta_len,
1056 u32 meta_seed, u64 *result, unsigned timeout)
1057 {
1058 bool write = nvme_is_write(cmd);
1059 struct nvme_ns *ns = q->queuedata;
1060 struct gendisk *disk = ns ? ns->disk : NULL;
1061 struct request *req;
1062 struct bio *bio = NULL;
1063 void *meta = NULL;
1064 int ret;
1065
1066 req = nvme_alloc_request(q, cmd, 0, NVME_QID_ANY);
1067 if (IS_ERR(req))
1068 return PTR_ERR(req);
1069
1070 req->timeout = timeout ? timeout : ADMIN_TIMEOUT;
1071 nvme_req(req)->flags |= NVME_REQ_USERCMD;
1072
1073 if (ubuffer && bufflen) {
1074 ret = blk_rq_map_user(q, req, NULL, ubuffer, bufflen,
1075 GFP_KERNEL);
1076 if (ret)
1077 goto out;
1078 bio = req->bio;
1079 bio->bi_disk = disk;
1080 if (disk && meta_buffer && meta_len) {
1081 meta = nvme_add_user_metadata(bio, meta_buffer, meta_len,
1082 meta_seed, write);
1083 if (IS_ERR(meta)) {
1084 ret = PTR_ERR(meta);
1085 goto out_unmap;
1086 }
1087 req->cmd_flags |= REQ_INTEGRITY;
1088 }
1089 }
1090
1091 nvme_execute_passthru_rq(req);
1092 if (nvme_req(req)->flags & NVME_REQ_CANCELLED)
1093 ret = -EINTR;
1094 else
1095 ret = nvme_req(req)->status;
1096 if (result)
1097 *result = le64_to_cpu(nvme_req(req)->result.u64);
1098 if (meta && !ret && !write) {
1099 if (copy_to_user(meta_buffer, meta, meta_len))
1100 ret = -EFAULT;
1101 }
1102 kfree(meta);
1103 out_unmap:
1104 if (bio)
1105 blk_rq_unmap_user(bio);
1106 out:
1107 blk_mq_free_request(req);
1108 return ret;
1109 }
1110
1111 static void nvme_keep_alive_end_io(struct request *rq, blk_status_t status)
1112 {
1113 struct nvme_ctrl *ctrl = rq->end_io_data;
1114 unsigned long flags;
1115 bool startka = false;
1116
1117 blk_mq_free_request(rq);
1118
1119 if (status) {
1120 dev_err(ctrl->device,
1121 "failed nvme_keep_alive_end_io error=%d\n",
1122 status);
1123 return;
1124 }
1125
1126 ctrl->comp_seen = false;
1127 spin_lock_irqsave(&ctrl->lock, flags);
1128 if (ctrl->state == NVME_CTRL_LIVE ||
1129 ctrl->state == NVME_CTRL_CONNECTING)
1130 startka = true;
1131 spin_unlock_irqrestore(&ctrl->lock, flags);
1132 if (startka)
1133 queue_delayed_work(nvme_wq, &ctrl->ka_work, ctrl->kato * HZ);
1134 }
1135
1136 static int nvme_keep_alive(struct nvme_ctrl *ctrl)
1137 {
1138 struct request *rq;
1139
1140 rq = nvme_alloc_request(ctrl->admin_q, &ctrl->ka_cmd, BLK_MQ_REQ_RESERVED,
1141 NVME_QID_ANY);
1142 if (IS_ERR(rq))
1143 return PTR_ERR(rq);
1144
1145 rq->timeout = ctrl->kato * HZ;
1146 rq->end_io_data = ctrl;
1147
1148 blk_execute_rq_nowait(rq->q, NULL, rq, 0, nvme_keep_alive_end_io);
1149
1150 return 0;
1151 }
1152
1153 static void nvme_keep_alive_work(struct work_struct *work)
1154 {
1155 struct nvme_ctrl *ctrl = container_of(to_delayed_work(work),
1156 struct nvme_ctrl, ka_work);
1157 bool comp_seen = ctrl->comp_seen;
1158
1159 if ((ctrl->ctratt & NVME_CTRL_ATTR_TBKAS) && comp_seen) {
1160 dev_dbg(ctrl->device,
1161 "reschedule traffic based keep-alive timer\n");
1162 ctrl->comp_seen = false;
1163 queue_delayed_work(nvme_wq, &ctrl->ka_work, ctrl->kato * HZ);
1164 return;
1165 }
1166
1167 if (nvme_keep_alive(ctrl)) {
1168 /* allocation failure, reset the controller */
1169 dev_err(ctrl->device, "keep-alive failed\n");
1170 nvme_reset_ctrl(ctrl);
1171 return;
1172 }
1173 }
1174
1175 static void nvme_start_keep_alive(struct nvme_ctrl *ctrl)
1176 {
1177 if (unlikely(ctrl->kato == 0))
1178 return;
1179
1180 queue_delayed_work(nvme_wq, &ctrl->ka_work, ctrl->kato * HZ);
1181 }
1182
1183 void nvme_stop_keep_alive(struct nvme_ctrl *ctrl)
1184 {
1185 if (unlikely(ctrl->kato == 0))
1186 return;
1187
1188 cancel_delayed_work_sync(&ctrl->ka_work);
1189 }
1190 EXPORT_SYMBOL_GPL(nvme_stop_keep_alive);
1191
1192 /*
1193 * In NVMe 1.0 the CNS field was just a binary controller or namespace
1194 * flag, thus sending any new CNS opcodes has a big chance of not working.
1195 * Qemu unfortunately had that bug after reporting a 1.1 version compliance
1196 * (but not for any later version).
1197 */
1198 static bool nvme_ctrl_limited_cns(struct nvme_ctrl *ctrl)
1199 {
1200 if (ctrl->quirks & NVME_QUIRK_IDENTIFY_CNS)
1201 return ctrl->vs < NVME_VS(1, 2, 0);
1202 return ctrl->vs < NVME_VS(1, 1, 0);
1203 }
1204
1205 static int nvme_identify_ctrl(struct nvme_ctrl *dev, struct nvme_id_ctrl **id)
1206 {
1207 struct nvme_command c = { };
1208 int error;
1209
1210 /* gcc-4.4.4 (at least) has issues with initializers and anon unions */
1211 c.identify.opcode = nvme_admin_identify;
1212 c.identify.cns = NVME_ID_CNS_CTRL;
1213
1214 *id = kmalloc(sizeof(struct nvme_id_ctrl), GFP_KERNEL);
1215 if (!*id)
1216 return -ENOMEM;
1217
1218 error = nvme_submit_sync_cmd(dev->admin_q, &c, *id,
1219 sizeof(struct nvme_id_ctrl));
1220 if (error)
1221 kfree(*id);
1222 return error;
1223 }
1224
1225 static bool nvme_multi_css(struct nvme_ctrl *ctrl)
1226 {
1227 return (ctrl->ctrl_config & NVME_CC_CSS_MASK) == NVME_CC_CSS_CSI;
1228 }
1229
1230 static int nvme_process_ns_desc(struct nvme_ctrl *ctrl, struct nvme_ns_ids *ids,
1231 struct nvme_ns_id_desc *cur, bool *csi_seen)
1232 {
1233 const char *warn_str = "ctrl returned bogus length:";
1234 void *data = cur;
1235
1236 switch (cur->nidt) {
1237 case NVME_NIDT_EUI64:
1238 if (cur->nidl != NVME_NIDT_EUI64_LEN) {
1239 dev_warn(ctrl->device, "%s %d for NVME_NIDT_EUI64\n",
1240 warn_str, cur->nidl);
1241 return -1;
1242 }
1243 memcpy(ids->eui64, data + sizeof(*cur), NVME_NIDT_EUI64_LEN);
1244 return NVME_NIDT_EUI64_LEN;
1245 case NVME_NIDT_NGUID:
1246 if (cur->nidl != NVME_NIDT_NGUID_LEN) {
1247 dev_warn(ctrl->device, "%s %d for NVME_NIDT_NGUID\n",
1248 warn_str, cur->nidl);
1249 return -1;
1250 }
1251 memcpy(ids->nguid, data + sizeof(*cur), NVME_NIDT_NGUID_LEN);
1252 return NVME_NIDT_NGUID_LEN;
1253 case NVME_NIDT_UUID:
1254 if (cur->nidl != NVME_NIDT_UUID_LEN) {
1255 dev_warn(ctrl->device, "%s %d for NVME_NIDT_UUID\n",
1256 warn_str, cur->nidl);
1257 return -1;
1258 }
1259 uuid_copy(&ids->uuid, data + sizeof(*cur));
1260 return NVME_NIDT_UUID_LEN;
1261 case NVME_NIDT_CSI:
1262 if (cur->nidl != NVME_NIDT_CSI_LEN) {
1263 dev_warn(ctrl->device, "%s %d for NVME_NIDT_CSI\n",
1264 warn_str, cur->nidl);
1265 return -1;
1266 }
1267 memcpy(&ids->csi, data + sizeof(*cur), NVME_NIDT_CSI_LEN);
1268 *csi_seen = true;
1269 return NVME_NIDT_CSI_LEN;
1270 default:
1271 /* Skip unknown types */
1272 return cur->nidl;
1273 }
1274 }
1275
1276 static int nvme_identify_ns_descs(struct nvme_ctrl *ctrl, unsigned nsid,
1277 struct nvme_ns_ids *ids)
1278 {
1279 struct nvme_command c = { };
1280 bool csi_seen = false;
1281 int status, pos, len;
1282 void *data;
1283
1284 if (ctrl->vs < NVME_VS(1, 3, 0) && !nvme_multi_css(ctrl))
1285 return 0;
1286 if (ctrl->quirks & NVME_QUIRK_NO_NS_DESC_LIST)
1287 return 0;
1288
1289 c.identify.opcode = nvme_admin_identify;
1290 c.identify.nsid = cpu_to_le32(nsid);
1291 c.identify.cns = NVME_ID_CNS_NS_DESC_LIST;
1292
1293 data = kzalloc(NVME_IDENTIFY_DATA_SIZE, GFP_KERNEL);
1294 if (!data)
1295 return -ENOMEM;
1296
1297 status = nvme_submit_sync_cmd(ctrl->admin_q, &c, data,
1298 NVME_IDENTIFY_DATA_SIZE);
1299 if (status) {
1300 dev_warn(ctrl->device,
1301 "Identify Descriptors failed (%d)\n", status);
1302 goto free_data;
1303 }
1304
1305 for (pos = 0; pos < NVME_IDENTIFY_DATA_SIZE; pos += len) {
1306 struct nvme_ns_id_desc *cur = data + pos;
1307
1308 if (cur->nidl == 0)
1309 break;
1310
1311 len = nvme_process_ns_desc(ctrl, ids, cur, &csi_seen);
1312 if (len < 0)
1313 break;
1314
1315 len += sizeof(*cur);
1316 }
1317
1318 if (nvme_multi_css(ctrl) && !csi_seen) {
1319 dev_warn(ctrl->device, "Command set not reported for nsid:%d\n",
1320 nsid);
1321 status = -EINVAL;
1322 }
1323
1324 free_data:
1325 kfree(data);
1326 return status;
1327 }
1328
1329 static int nvme_identify_ns(struct nvme_ctrl *ctrl, unsigned nsid,
1330 struct nvme_ns_ids *ids, struct nvme_id_ns **id)
1331 {
1332 struct nvme_command c = { };
1333 int error;
1334
1335 /* gcc-4.4.4 (at least) has issues with initializers and anon unions */
1336 c.identify.opcode = nvme_admin_identify;
1337 c.identify.nsid = cpu_to_le32(nsid);
1338 c.identify.cns = NVME_ID_CNS_NS;
1339
1340 *id = kmalloc(sizeof(**id), GFP_KERNEL);
1341 if (!*id)
1342 return -ENOMEM;
1343
1344 error = nvme_submit_sync_cmd(ctrl->admin_q, &c, *id, sizeof(**id));
1345 if (error) {
1346 dev_warn(ctrl->device, "Identify namespace failed (%d)\n", error);
1347 goto out_free_id;
1348 }
1349
1350 error = -ENODEV;
1351 if ((*id)->ncap == 0) /* namespace not allocated or attached */
1352 goto out_free_id;
1353
1354 if (ctrl->vs >= NVME_VS(1, 1, 0) &&
1355 !memchr_inv(ids->eui64, 0, sizeof(ids->eui64)))
1356 memcpy(ids->eui64, (*id)->eui64, sizeof(ids->eui64));
1357 if (ctrl->vs >= NVME_VS(1, 2, 0) &&
1358 !memchr_inv(ids->nguid, 0, sizeof(ids->nguid)))
1359 memcpy(ids->nguid, (*id)->nguid, sizeof(ids->nguid));
1360
1361 return 0;
1362
1363 out_free_id:
1364 kfree(*id);
1365 return error;
1366 }
1367
1368 static int nvme_features(struct nvme_ctrl *dev, u8 op, unsigned int fid,
1369 unsigned int dword11, void *buffer, size_t buflen, u32 *result)
1370 {
1371 union nvme_result res = { 0 };
1372 struct nvme_command c;
1373 int ret;
1374
1375 memset(&c, 0, sizeof(c));
1376 c.features.opcode = op;
1377 c.features.fid = cpu_to_le32(fid);
1378 c.features.dword11 = cpu_to_le32(dword11);
1379
1380 ret = __nvme_submit_sync_cmd(dev->admin_q, &c, &res,
1381 buffer, buflen, 0, NVME_QID_ANY, 0, 0, false);
1382 if (ret >= 0 && result)
1383 *result = le32_to_cpu(res.u32);
1384 return ret;
1385 }
1386
1387 int nvme_set_features(struct nvme_ctrl *dev, unsigned int fid,
1388 unsigned int dword11, void *buffer, size_t buflen,
1389 u32 *result)
1390 {
1391 return nvme_features(dev, nvme_admin_set_features, fid, dword11, buffer,
1392 buflen, result);
1393 }
1394 EXPORT_SYMBOL_GPL(nvme_set_features);
1395
1396 int nvme_get_features(struct nvme_ctrl *dev, unsigned int fid,
1397 unsigned int dword11, void *buffer, size_t buflen,
1398 u32 *result)
1399 {
1400 return nvme_features(dev, nvme_admin_get_features, fid, dword11, buffer,
1401 buflen, result);
1402 }
1403 EXPORT_SYMBOL_GPL(nvme_get_features);
1404
1405 int nvme_set_queue_count(struct nvme_ctrl *ctrl, int *count)
1406 {
1407 u32 q_count = (*count - 1) | ((*count - 1) << 16);
1408 u32 result;
1409 int status, nr_io_queues;
1410
1411 status = nvme_set_features(ctrl, NVME_FEAT_NUM_QUEUES, q_count, NULL, 0,
1412 &result);
1413 if (status < 0)
1414 return status;
1415
1416 /*
1417 * Degraded controllers might return an error when setting the queue
1418 * count. We still want to be able to bring them online and offer
1419 * access to the admin queue, as that might be only way to fix them up.
1420 */
1421 if (status > 0) {
1422 dev_err(ctrl->device, "Could not set queue count (%d)\n", status);
1423 *count = 0;
1424 } else {
1425 nr_io_queues = min(result & 0xffff, result >> 16) + 1;
1426 *count = min(*count, nr_io_queues);
1427 }
1428
1429 return 0;
1430 }
1431 EXPORT_SYMBOL_GPL(nvme_set_queue_count);
1432
1433 #define NVME_AEN_SUPPORTED \
1434 (NVME_AEN_CFG_NS_ATTR | NVME_AEN_CFG_FW_ACT | \
1435 NVME_AEN_CFG_ANA_CHANGE | NVME_AEN_CFG_DISC_CHANGE)
1436
1437 static void nvme_enable_aen(struct nvme_ctrl *ctrl)
1438 {
1439 u32 result, supported_aens = ctrl->oaes & NVME_AEN_SUPPORTED;
1440 int status;
1441
1442 if (!supported_aens)
1443 return;
1444
1445 status = nvme_set_features(ctrl, NVME_FEAT_ASYNC_EVENT, supported_aens,
1446 NULL, 0, &result);
1447 if (status)
1448 dev_warn(ctrl->device, "Failed to configure AEN (cfg %x)\n",
1449 supported_aens);
1450
1451 queue_work(nvme_wq, &ctrl->async_event_work);
1452 }
1453
1454 /*
1455 * Convert integer values from ioctl structures to user pointers, silently
1456 * ignoring the upper bits in the compat case to match behaviour of 32-bit
1457 * kernels.
1458 */
1459 static void __user *nvme_to_user_ptr(uintptr_t ptrval)
1460 {
1461 if (in_compat_syscall())
1462 ptrval = (compat_uptr_t)ptrval;
1463 return (void __user *)ptrval;
1464 }
1465
1466 static int nvme_submit_io(struct nvme_ns *ns, struct nvme_user_io __user *uio)
1467 {
1468 struct nvme_user_io io;
1469 struct nvme_command c;
1470 unsigned length, meta_len;
1471 void __user *metadata;
1472
1473 if (copy_from_user(&io, uio, sizeof(io)))
1474 return -EFAULT;
1475 if (io.flags)
1476 return -EINVAL;
1477
1478 switch (io.opcode) {
1479 case nvme_cmd_write:
1480 case nvme_cmd_read:
1481 case nvme_cmd_compare:
1482 break;
1483 default:
1484 return -EINVAL;
1485 }
1486
1487 length = (io.nblocks + 1) << ns->lba_shift;
1488 meta_len = (io.nblocks + 1) * ns->ms;
1489 metadata = nvme_to_user_ptr(io.metadata);
1490
1491 if (ns->features & NVME_NS_EXT_LBAS) {
1492 length += meta_len;
1493 meta_len = 0;
1494 } else if (meta_len) {
1495 if ((io.metadata & 3) || !io.metadata)
1496 return -EINVAL;
1497 }
1498
1499 memset(&c, 0, sizeof(c));
1500 c.rw.opcode = io.opcode;
1501 c.rw.flags = io.flags;
1502 c.rw.nsid = cpu_to_le32(ns->head->ns_id);
1503 c.rw.slba = cpu_to_le64(io.slba);
1504 c.rw.length = cpu_to_le16(io.nblocks);
1505 c.rw.control = cpu_to_le16(io.control);
1506 c.rw.dsmgmt = cpu_to_le32(io.dsmgmt);
1507 c.rw.reftag = cpu_to_le32(io.reftag);
1508 c.rw.apptag = cpu_to_le16(io.apptag);
1509 c.rw.appmask = cpu_to_le16(io.appmask);
1510
1511 return nvme_submit_user_cmd(ns->queue, &c,
1512 nvme_to_user_ptr(io.addr), length,
1513 metadata, meta_len, lower_32_bits(io.slba), NULL, 0);
1514 }
1515
1516 static int nvme_user_cmd(struct nvme_ctrl *ctrl, struct nvme_ns *ns,
1517 struct nvme_passthru_cmd __user *ucmd)
1518 {
1519 struct nvme_passthru_cmd cmd;
1520 struct nvme_command c;
1521 unsigned timeout = 0;
1522 u64 result;
1523 int status;
1524
1525 if (!capable(CAP_SYS_ADMIN))
1526 return -EACCES;
1527 if (copy_from_user(&cmd, ucmd, sizeof(cmd)))
1528 return -EFAULT;
1529 if (cmd.flags)
1530 return -EINVAL;
1531
1532 memset(&c, 0, sizeof(c));
1533 c.common.opcode = cmd.opcode;
1534 c.common.flags = cmd.flags;
1535 c.common.nsid = cpu_to_le32(cmd.nsid);
1536 c.common.cdw2[0] = cpu_to_le32(cmd.cdw2);
1537 c.common.cdw2[1] = cpu_to_le32(cmd.cdw3);
1538 c.common.cdw10 = cpu_to_le32(cmd.cdw10);
1539 c.common.cdw11 = cpu_to_le32(cmd.cdw11);
1540 c.common.cdw12 = cpu_to_le32(cmd.cdw12);
1541 c.common.cdw13 = cpu_to_le32(cmd.cdw13);
1542 c.common.cdw14 = cpu_to_le32(cmd.cdw14);
1543 c.common.cdw15 = cpu_to_le32(cmd.cdw15);
1544
1545 if (cmd.timeout_ms)
1546 timeout = msecs_to_jiffies(cmd.timeout_ms);
1547
1548 status = nvme_submit_user_cmd(ns ? ns->queue : ctrl->admin_q, &c,
1549 nvme_to_user_ptr(cmd.addr), cmd.data_len,
1550 nvme_to_user_ptr(cmd.metadata), cmd.metadata_len,
1551 0, &result, timeout);
1552
1553 if (status >= 0) {
1554 if (put_user(result, &ucmd->result))
1555 return -EFAULT;
1556 }
1557
1558 return status;
1559 }
1560
1561 static int nvme_user_cmd64(struct nvme_ctrl *ctrl, struct nvme_ns *ns,
1562 struct nvme_passthru_cmd64 __user *ucmd)
1563 {
1564 struct nvme_passthru_cmd64 cmd;
1565 struct nvme_command c;
1566 unsigned timeout = 0;
1567 int status;
1568
1569 if (!capable(CAP_SYS_ADMIN))
1570 return -EACCES;
1571 if (copy_from_user(&cmd, ucmd, sizeof(cmd)))
1572 return -EFAULT;
1573 if (cmd.flags)
1574 return -EINVAL;
1575
1576 memset(&c, 0, sizeof(c));
1577 c.common.opcode = cmd.opcode;
1578 c.common.flags = cmd.flags;
1579 c.common.nsid = cpu_to_le32(cmd.nsid);
1580 c.common.cdw2[0] = cpu_to_le32(cmd.cdw2);
1581 c.common.cdw2[1] = cpu_to_le32(cmd.cdw3);
1582 c.common.cdw10 = cpu_to_le32(cmd.cdw10);
1583 c.common.cdw11 = cpu_to_le32(cmd.cdw11);
1584 c.common.cdw12 = cpu_to_le32(cmd.cdw12);
1585 c.common.cdw13 = cpu_to_le32(cmd.cdw13);
1586 c.common.cdw14 = cpu_to_le32(cmd.cdw14);
1587 c.common.cdw15 = cpu_to_le32(cmd.cdw15);
1588
1589 if (cmd.timeout_ms)
1590 timeout = msecs_to_jiffies(cmd.timeout_ms);
1591
1592 status = nvme_submit_user_cmd(ns ? ns->queue : ctrl->admin_q, &c,
1593 nvme_to_user_ptr(cmd.addr), cmd.data_len,
1594 nvme_to_user_ptr(cmd.metadata), cmd.metadata_len,
1595 0, &cmd.result, timeout);
1596
1597 if (status >= 0) {
1598 if (put_user(cmd.result, &ucmd->result))
1599 return -EFAULT;
1600 }
1601
1602 return status;
1603 }
1604
1605 /*
1606 * Issue ioctl requests on the first available path. Note that unlike normal
1607 * block layer requests we will not retry failed request on another controller.
1608 */
1609 struct nvme_ns *nvme_get_ns_from_disk(struct gendisk *disk,
1610 struct nvme_ns_head **head, int *srcu_idx)
1611 {
1612 #ifdef CONFIG_NVME_MULTIPATH
1613 if (disk->fops == &nvme_ns_head_ops) {
1614 struct nvme_ns *ns;
1615
1616 *head = disk->private_data;
1617 *srcu_idx = srcu_read_lock(&(*head)->srcu);
1618 ns = nvme_find_path(*head);
1619 if (!ns)
1620 srcu_read_unlock(&(*head)->srcu, *srcu_idx);
1621 return ns;
1622 }
1623 #endif
1624 *head = NULL;
1625 *srcu_idx = -1;
1626 return disk->private_data;
1627 }
1628
1629 void nvme_put_ns_from_disk(struct nvme_ns_head *head, int idx)
1630 {
1631 if (head)
1632 srcu_read_unlock(&head->srcu, idx);
1633 }
1634
1635 static bool is_ctrl_ioctl(unsigned int cmd)
1636 {
1637 if (cmd == NVME_IOCTL_ADMIN_CMD || cmd == NVME_IOCTL_ADMIN64_CMD)
1638 return true;
1639 if (is_sed_ioctl(cmd))
1640 return true;
1641 return false;
1642 }
1643
1644 static int nvme_handle_ctrl_ioctl(struct nvme_ns *ns, unsigned int cmd,
1645 void __user *argp,
1646 struct nvme_ns_head *head,
1647 int srcu_idx)
1648 {
1649 struct nvme_ctrl *ctrl = ns->ctrl;
1650 int ret;
1651
1652 nvme_get_ctrl(ns->ctrl);
1653 nvme_put_ns_from_disk(head, srcu_idx);
1654
1655 switch (cmd) {
1656 case NVME_IOCTL_ADMIN_CMD:
1657 ret = nvme_user_cmd(ctrl, NULL, argp);
1658 break;
1659 case NVME_IOCTL_ADMIN64_CMD:
1660 ret = nvme_user_cmd64(ctrl, NULL, argp);
1661 break;
1662 default:
1663 ret = sed_ioctl(ctrl->opal_dev, cmd, argp);
1664 break;
1665 }
1666 nvme_put_ctrl(ctrl);
1667 return ret;
1668 }
1669
1670 static int nvme_ioctl(struct block_device *bdev, fmode_t mode,
1671 unsigned int cmd, unsigned long arg)
1672 {
1673 struct nvme_ns_head *head = NULL;
1674 void __user *argp = (void __user *)arg;
1675 struct nvme_ns *ns;
1676 int srcu_idx, ret;
1677
1678 ns = nvme_get_ns_from_disk(bdev->bd_disk, &head, &srcu_idx);
1679 if (unlikely(!ns))
1680 return -EWOULDBLOCK;
1681
1682 /*
1683 * Handle ioctls that apply to the controller instead of the namespace
1684 * seperately and drop the ns SRCU reference early. This avoids a
1685 * deadlock when deleting namespaces using the passthrough interface.
1686 */
1687 if (is_ctrl_ioctl(cmd))
1688 return nvme_handle_ctrl_ioctl(ns, cmd, argp, head, srcu_idx);
1689
1690 switch (cmd) {
1691 case NVME_IOCTL_ID:
1692 force_successful_syscall_return();
1693 ret = ns->head->ns_id;
1694 break;
1695 case NVME_IOCTL_IO_CMD:
1696 ret = nvme_user_cmd(ns->ctrl, ns, argp);
1697 break;
1698 case NVME_IOCTL_SUBMIT_IO:
1699 ret = nvme_submit_io(ns, argp);
1700 break;
1701 case NVME_IOCTL_IO64_CMD:
1702 ret = nvme_user_cmd64(ns->ctrl, ns, argp);
1703 break;
1704 default:
1705 if (ns->ndev)
1706 ret = nvme_nvm_ioctl(ns, cmd, arg);
1707 else
1708 ret = -ENOTTY;
1709 }
1710
1711 nvme_put_ns_from_disk(head, srcu_idx);
1712 return ret;
1713 }
1714
1715 #ifdef CONFIG_COMPAT
1716 struct nvme_user_io32 {
1717 __u8 opcode;
1718 __u8 flags;
1719 __u16 control;
1720 __u16 nblocks;
1721 __u16 rsvd;
1722 __u64 metadata;
1723 __u64 addr;
1724 __u64 slba;
1725 __u32 dsmgmt;
1726 __u32 reftag;
1727 __u16 apptag;
1728 __u16 appmask;
1729 } __attribute__((__packed__));
1730
1731 #define NVME_IOCTL_SUBMIT_IO32 _IOW('N', 0x42, struct nvme_user_io32)
1732
1733 static int nvme_compat_ioctl(struct block_device *bdev, fmode_t mode,
1734 unsigned int cmd, unsigned long arg)
1735 {
1736 /*
1737 * Corresponds to the difference of NVME_IOCTL_SUBMIT_IO
1738 * between 32 bit programs and 64 bit kernel.
1739 * The cause is that the results of sizeof(struct nvme_user_io),
1740 * which is used to define NVME_IOCTL_SUBMIT_IO,
1741 * are not same between 32 bit compiler and 64 bit compiler.
1742 * NVME_IOCTL_SUBMIT_IO32 is for 64 bit kernel handling
1743 * NVME_IOCTL_SUBMIT_IO issued from 32 bit programs.
1744 * Other IOCTL numbers are same between 32 bit and 64 bit.
1745 * So there is nothing to do regarding to other IOCTL numbers.
1746 */
1747 if (cmd == NVME_IOCTL_SUBMIT_IO32)
1748 return nvme_ioctl(bdev, mode, NVME_IOCTL_SUBMIT_IO, arg);
1749
1750 return nvme_ioctl(bdev, mode, cmd, arg);
1751 }
1752 #else
1753 #define nvme_compat_ioctl NULL
1754 #endif /* CONFIG_COMPAT */
1755
1756 static int nvme_open(struct block_device *bdev, fmode_t mode)
1757 {
1758 struct nvme_ns *ns = bdev->bd_disk->private_data;
1759
1760 #ifdef CONFIG_NVME_MULTIPATH
1761 /* should never be called due to GENHD_FL_HIDDEN */
1762 if (WARN_ON_ONCE(ns->head->disk))
1763 goto fail;
1764 #endif
1765 if (!kref_get_unless_zero(&ns->kref))
1766 goto fail;
1767 if (!try_module_get(ns->ctrl->ops->module))
1768 goto fail_put_ns;
1769
1770 return 0;
1771
1772 fail_put_ns:
1773 nvme_put_ns(ns);
1774 fail:
1775 return -ENXIO;
1776 }
1777
1778 static void nvme_release(struct gendisk *disk, fmode_t mode)
1779 {
1780 struct nvme_ns *ns = disk->private_data;
1781
1782 module_put(ns->ctrl->ops->module);
1783 nvme_put_ns(ns);
1784 }
1785
1786 static int nvme_getgeo(struct block_device *bdev, struct hd_geometry *geo)
1787 {
1788 /* some standard values */
1789 geo->heads = 1 << 6;
1790 geo->sectors = 1 << 5;
1791 geo->cylinders = get_capacity(bdev->bd_disk) >> 11;
1792 return 0;
1793 }
1794
1795 #ifdef CONFIG_BLK_DEV_INTEGRITY
1796 static void nvme_init_integrity(struct gendisk *disk, u16 ms, u8 pi_type,
1797 u32 max_integrity_segments)
1798 {
1799 struct blk_integrity integrity;
1800
1801 memset(&integrity, 0, sizeof(integrity));
1802 switch (pi_type) {
1803 case NVME_NS_DPS_PI_TYPE3:
1804 integrity.profile = &t10_pi_type3_crc;
1805 integrity.tag_size = sizeof(u16) + sizeof(u32);
1806 integrity.flags |= BLK_INTEGRITY_DEVICE_CAPABLE;
1807 break;
1808 case NVME_NS_DPS_PI_TYPE1:
1809 case NVME_NS_DPS_PI_TYPE2:
1810 integrity.profile = &t10_pi_type1_crc;
1811 integrity.tag_size = sizeof(u16);
1812 integrity.flags |= BLK_INTEGRITY_DEVICE_CAPABLE;
1813 break;
1814 default:
1815 integrity.profile = NULL;
1816 break;
1817 }
1818 integrity.tuple_size = ms;
1819 blk_integrity_register(disk, &integrity);
1820 blk_queue_max_integrity_segments(disk->queue, max_integrity_segments);
1821 }
1822 #else
1823 static void nvme_init_integrity(struct gendisk *disk, u16 ms, u8 pi_type,
1824 u32 max_integrity_segments)
1825 {
1826 }
1827 #endif /* CONFIG_BLK_DEV_INTEGRITY */
1828
1829 static void nvme_config_discard(struct gendisk *disk, struct nvme_ns *ns)
1830 {
1831 struct nvme_ctrl *ctrl = ns->ctrl;
1832 struct request_queue *queue = disk->queue;
1833 u32 size = queue_logical_block_size(queue);
1834
1835 if (!(ctrl->oncs & NVME_CTRL_ONCS_DSM)) {
1836 blk_queue_flag_clear(QUEUE_FLAG_DISCARD, queue);
1837 return;
1838 }
1839
1840 if (ctrl->nr_streams && ns->sws && ns->sgs)
1841 size *= ns->sws * ns->sgs;
1842
1843 BUILD_BUG_ON(PAGE_SIZE / sizeof(struct nvme_dsm_range) <
1844 NVME_DSM_MAX_RANGES);
1845
1846 queue->limits.discard_alignment = 0;
1847 queue->limits.discard_granularity = size;
1848
1849 /* If discard is already enabled, don't reset queue limits */
1850 if (blk_queue_flag_test_and_set(QUEUE_FLAG_DISCARD, queue))
1851 return;
1852
1853 blk_queue_max_discard_sectors(queue, UINT_MAX);
1854 blk_queue_max_discard_segments(queue, NVME_DSM_MAX_RANGES);
1855
1856 if (ctrl->quirks & NVME_QUIRK_DEALLOCATE_ZEROES)
1857 blk_queue_max_write_zeroes_sectors(queue, UINT_MAX);
1858 }
1859
1860 static void nvme_config_write_zeroes(struct gendisk *disk, struct nvme_ns *ns)
1861 {
1862 u64 max_blocks;
1863
1864 if (!(ns->ctrl->oncs & NVME_CTRL_ONCS_WRITE_ZEROES) ||
1865 (ns->ctrl->quirks & NVME_QUIRK_DISABLE_WRITE_ZEROES))
1866 return;
1867 /*
1868 * Even though NVMe spec explicitly states that MDTS is not
1869 * applicable to the write-zeroes:- "The restriction does not apply to
1870 * commands that do not transfer data between the host and the
1871 * controller (e.g., Write Uncorrectable ro Write Zeroes command).".
1872 * In order to be more cautious use controller's max_hw_sectors value
1873 * to configure the maximum sectors for the write-zeroes which is
1874 * configured based on the controller's MDTS field in the
1875 * nvme_init_identify() if available.
1876 */
1877 if (ns->ctrl->max_hw_sectors == UINT_MAX)
1878 max_blocks = (u64)USHRT_MAX + 1;
1879 else
1880 max_blocks = ns->ctrl->max_hw_sectors + 1;
1881
1882 blk_queue_max_write_zeroes_sectors(disk->queue,
1883 nvme_lba_to_sect(ns, max_blocks));
1884 }
1885
1886 static bool nvme_ns_ids_valid(struct nvme_ns_ids *ids)
1887 {
1888 return !uuid_is_null(&ids->uuid) ||
1889 memchr_inv(ids->nguid, 0, sizeof(ids->nguid)) ||
1890 memchr_inv(ids->eui64, 0, sizeof(ids->eui64));
1891 }
1892
1893 static bool nvme_ns_ids_equal(struct nvme_ns_ids *a, struct nvme_ns_ids *b)
1894 {
1895 return uuid_equal(&a->uuid, &b->uuid) &&
1896 memcmp(&a->nguid, &b->nguid, sizeof(a->nguid)) == 0 &&
1897 memcmp(&a->eui64, &b->eui64, sizeof(a->eui64)) == 0 &&
1898 a->csi == b->csi;
1899 }
1900
1901 static int nvme_setup_streams_ns(struct nvme_ctrl *ctrl, struct nvme_ns *ns,
1902 u32 *phys_bs, u32 *io_opt)
1903 {
1904 struct streams_directive_params s;
1905 int ret;
1906
1907 if (!ctrl->nr_streams)
1908 return 0;
1909
1910 ret = nvme_get_stream_params(ctrl, &s, ns->head->ns_id);
1911 if (ret)
1912 return ret;
1913
1914 ns->sws = le32_to_cpu(s.sws);
1915 ns->sgs = le16_to_cpu(s.sgs);
1916
1917 if (ns->sws) {
1918 *phys_bs = ns->sws * (1 << ns->lba_shift);
1919 if (ns->sgs)
1920 *io_opt = *phys_bs * ns->sgs;
1921 }
1922
1923 return 0;
1924 }
1925
1926 static int nvme_configure_metadata(struct nvme_ns *ns, struct nvme_id_ns *id)
1927 {
1928 struct nvme_ctrl *ctrl = ns->ctrl;
1929
1930 /*
1931 * The PI implementation requires the metadata size to be equal to the
1932 * t10 pi tuple size.
1933 */
1934 ns->ms = le16_to_cpu(id->lbaf[id->flbas & NVME_NS_FLBAS_LBA_MASK].ms);
1935 if (ns->ms == sizeof(struct t10_pi_tuple))
1936 ns->pi_type = id->dps & NVME_NS_DPS_PI_MASK;
1937 else
1938 ns->pi_type = 0;
1939
1940 ns->features &= ~(NVME_NS_METADATA_SUPPORTED | NVME_NS_EXT_LBAS);
1941 if (!ns->ms || !(ctrl->ops->flags & NVME_F_METADATA_SUPPORTED))
1942 return 0;
1943 if (ctrl->ops->flags & NVME_F_FABRICS) {
1944 /*
1945 * The NVMe over Fabrics specification only supports metadata as
1946 * part of the extended data LBA. We rely on HCA/HBA support to
1947 * remap the separate metadata buffer from the block layer.
1948 */
1949 if (WARN_ON_ONCE(!(id->flbas & NVME_NS_FLBAS_META_EXT)))
1950 return -EINVAL;
1951 if (ctrl->max_integrity_segments)
1952 ns->features |=
1953 (NVME_NS_METADATA_SUPPORTED | NVME_NS_EXT_LBAS);
1954 } else {
1955 /*
1956 * For PCIe controllers, we can't easily remap the separate
1957 * metadata buffer from the block layer and thus require a
1958 * separate metadata buffer for block layer metadata/PI support.
1959 * We allow extended LBAs for the passthrough interface, though.
1960 */
1961 if (id->flbas & NVME_NS_FLBAS_META_EXT)
1962 ns->features |= NVME_NS_EXT_LBAS;
1963 else
1964 ns->features |= NVME_NS_METADATA_SUPPORTED;
1965 }
1966
1967 return 0;
1968 }
1969
1970 static void nvme_set_queue_limits(struct nvme_ctrl *ctrl,
1971 struct request_queue *q)
1972 {
1973 bool vwc = ctrl->vwc & NVME_CTRL_VWC_PRESENT;
1974
1975 if (ctrl->max_hw_sectors) {
1976 u32 max_segments =
1977 (ctrl->max_hw_sectors / (NVME_CTRL_PAGE_SIZE >> 9)) + 1;
1978
1979 max_segments = min_not_zero(max_segments, ctrl->max_segments);
1980 blk_queue_max_hw_sectors(q, ctrl->max_hw_sectors);
1981 blk_queue_max_segments(q, min_t(u32, max_segments, USHRT_MAX));
1982 }
1983 blk_queue_virt_boundary(q, NVME_CTRL_PAGE_SIZE - 1);
1984 blk_queue_dma_alignment(q, 7);
1985 blk_queue_write_cache(q, vwc, vwc);
1986 }
1987
1988 static void nvme_update_disk_info(struct gendisk *disk,
1989 struct nvme_ns *ns, struct nvme_id_ns *id)
1990 {
1991 sector_t capacity = nvme_lba_to_sect(ns, le64_to_cpu(id->nsze));
1992 unsigned short bs = 1 << ns->lba_shift;
1993 u32 atomic_bs, phys_bs, io_opt = 0;
1994
1995 /*
1996 * The block layer can't support LBA sizes larger than the page size
1997 * yet, so catch this early and don't allow block I/O.
1998 */
1999 if (ns->lba_shift > PAGE_SHIFT) {
2000 capacity = 0;
2001 bs = (1 << 9);
2002 }
2003
2004 blk_integrity_unregister(disk);
2005
2006 atomic_bs = phys_bs = bs;
2007 nvme_setup_streams_ns(ns->ctrl, ns, &phys_bs, &io_opt);
2008 if (id->nabo == 0) {
2009 /*
2010 * Bit 1 indicates whether NAWUPF is defined for this namespace
2011 * and whether it should be used instead of AWUPF. If NAWUPF ==
2012 * 0 then AWUPF must be used instead.
2013 */
2014 if (id->nsfeat & NVME_NS_FEAT_ATOMICS && id->nawupf)
2015 atomic_bs = (1 + le16_to_cpu(id->nawupf)) * bs;
2016 else
2017 atomic_bs = (1 + ns->ctrl->subsys->awupf) * bs;
2018 }
2019
2020 if (id->nsfeat & NVME_NS_FEAT_IO_OPT) {
2021 /* NPWG = Namespace Preferred Write Granularity */
2022 phys_bs = bs * (1 + le16_to_cpu(id->npwg));
2023 /* NOWS = Namespace Optimal Write Size */
2024 io_opt = bs * (1 + le16_to_cpu(id->nows));
2025 }
2026
2027 blk_queue_logical_block_size(disk->queue, bs);
2028 /*
2029 * Linux filesystems assume writing a single physical block is
2030 * an atomic operation. Hence limit the physical block size to the
2031 * value of the Atomic Write Unit Power Fail parameter.
2032 */
2033 blk_queue_physical_block_size(disk->queue, min(phys_bs, atomic_bs));
2034 blk_queue_io_min(disk->queue, phys_bs);
2035 blk_queue_io_opt(disk->queue, io_opt);
2036
2037 /*
2038 * Register a metadata profile for PI, or the plain non-integrity NVMe
2039 * metadata masquerading as Type 0 if supported, otherwise reject block
2040 * I/O to namespaces with metadata except when the namespace supports
2041 * PI, as it can strip/insert in that case.
2042 */
2043 if (ns->ms) {
2044 if (IS_ENABLED(CONFIG_BLK_DEV_INTEGRITY) &&
2045 (ns->features & NVME_NS_METADATA_SUPPORTED))
2046 nvme_init_integrity(disk, ns->ms, ns->pi_type,
2047 ns->ctrl->max_integrity_segments);
2048 else if (!nvme_ns_has_pi(ns))
2049 capacity = 0;
2050 }
2051
2052 set_capacity_revalidate_and_notify(disk, capacity, false);
2053
2054 nvme_config_discard(disk, ns);
2055 nvme_config_write_zeroes(disk, ns);
2056
2057 if (id->nsattr & NVME_NS_ATTR_RO)
2058 set_disk_ro(disk, true);
2059 else
2060 set_disk_ro(disk, false);
2061 }
2062
2063 static inline bool nvme_first_scan(struct gendisk *disk)
2064 {
2065 /* nvme_alloc_ns() scans the disk prior to adding it */
2066 return !(disk->flags & GENHD_FL_UP);
2067 }
2068
2069 static void nvme_set_chunk_sectors(struct nvme_ns *ns, struct nvme_id_ns *id)
2070 {
2071 struct nvme_ctrl *ctrl = ns->ctrl;
2072 u32 iob;
2073
2074 if ((ctrl->quirks & NVME_QUIRK_STRIPE_SIZE) &&
2075 is_power_of_2(ctrl->max_hw_sectors))
2076 iob = ctrl->max_hw_sectors;
2077 else
2078 iob = nvme_lba_to_sect(ns, le16_to_cpu(id->noiob));
2079
2080 if (!iob)
2081 return;
2082
2083 if (!is_power_of_2(iob)) {
2084 if (nvme_first_scan(ns->disk))
2085 pr_warn("%s: ignoring unaligned IO boundary:%u\n",
2086 ns->disk->disk_name, iob);
2087 return;
2088 }
2089
2090 if (blk_queue_is_zoned(ns->disk->queue)) {
2091 if (nvme_first_scan(ns->disk))
2092 pr_warn("%s: ignoring zoned namespace IO boundary\n",
2093 ns->disk->disk_name);
2094 return;
2095 }
2096
2097 blk_queue_chunk_sectors(ns->queue, iob);
2098 }
2099
2100 static int nvme_update_ns_info(struct nvme_ns *ns, struct nvme_id_ns *id)
2101 {
2102 unsigned lbaf = id->flbas & NVME_NS_FLBAS_LBA_MASK;
2103 int ret;
2104
2105 blk_mq_freeze_queue(ns->disk->queue);
2106 ns->lba_shift = id->lbaf[lbaf].ds;
2107 nvme_set_queue_limits(ns->ctrl, ns->queue);
2108
2109 if (ns->head->ids.csi == NVME_CSI_ZNS) {
2110 ret = nvme_update_zone_info(ns, lbaf);
2111 if (ret)
2112 goto out_unfreeze;
2113 }
2114
2115 ret = nvme_configure_metadata(ns, id);
2116 if (ret)
2117 goto out_unfreeze;
2118 nvme_set_chunk_sectors(ns, id);
2119 nvme_update_disk_info(ns->disk, ns, id);
2120 blk_mq_unfreeze_queue(ns->disk->queue);
2121
2122 if (blk_queue_is_zoned(ns->queue)) {
2123 ret = nvme_revalidate_zones(ns);
2124 if (ret)
2125 return ret;
2126 }
2127
2128 #ifdef CONFIG_NVME_MULTIPATH
2129 if (ns->head->disk) {
2130 blk_mq_freeze_queue(ns->head->disk->queue);
2131 nvme_update_disk_info(ns->head->disk, ns, id);
2132 blk_stack_limits(&ns->head->disk->queue->limits,
2133 &ns->queue->limits, 0);
2134 blk_queue_update_readahead(ns->head->disk->queue);
2135 nvme_update_bdev_size(ns->head->disk);
2136 blk_mq_unfreeze_queue(ns->head->disk->queue);
2137 }
2138 #endif
2139 return 0;
2140
2141 out_unfreeze:
2142 blk_mq_unfreeze_queue(ns->disk->queue);
2143 return ret;
2144 }
2145
2146 static char nvme_pr_type(enum pr_type type)
2147 {
2148 switch (type) {
2149 case PR_WRITE_EXCLUSIVE:
2150 return 1;
2151 case PR_EXCLUSIVE_ACCESS:
2152 return 2;
2153 case PR_WRITE_EXCLUSIVE_REG_ONLY:
2154 return 3;
2155 case PR_EXCLUSIVE_ACCESS_REG_ONLY:
2156 return 4;
2157 case PR_WRITE_EXCLUSIVE_ALL_REGS:
2158 return 5;
2159 case PR_EXCLUSIVE_ACCESS_ALL_REGS:
2160 return 6;
2161 default:
2162 return 0;
2163 }
2164 };
2165
2166 static int nvme_pr_command(struct block_device *bdev, u32 cdw10,
2167 u64 key, u64 sa_key, u8 op)
2168 {
2169 struct nvme_ns_head *head = NULL;
2170 struct nvme_ns *ns;
2171 struct nvme_command c;
2172 int srcu_idx, ret;
2173 u8 data[16] = { 0, };
2174
2175 ns = nvme_get_ns_from_disk(bdev->bd_disk, &head, &srcu_idx);
2176 if (unlikely(!ns))
2177 return -EWOULDBLOCK;
2178
2179 put_unaligned_le64(key, &data[0]);
2180 put_unaligned_le64(sa_key, &data[8]);
2181
2182 memset(&c, 0, sizeof(c));
2183 c.common.opcode = op;
2184 c.common.nsid = cpu_to_le32(ns->head->ns_id);
2185 c.common.cdw10 = cpu_to_le32(cdw10);
2186
2187 ret = nvme_submit_sync_cmd(ns->queue, &c, data, 16);
2188 nvme_put_ns_from_disk(head, srcu_idx);
2189 return ret;
2190 }
2191
2192 static int nvme_pr_register(struct block_device *bdev, u64 old,
2193 u64 new, unsigned flags)
2194 {
2195 u32 cdw10;
2196
2197 if (flags & ~PR_FL_IGNORE_KEY)
2198 return -EOPNOTSUPP;
2199
2200 cdw10 = old ? 2 : 0;
2201 cdw10 |= (flags & PR_FL_IGNORE_KEY) ? 1 << 3 : 0;
2202 cdw10 |= (1 << 30) | (1 << 31); /* PTPL=1 */
2203 return nvme_pr_command(bdev, cdw10, old, new, nvme_cmd_resv_register);
2204 }
2205
2206 static int nvme_pr_reserve(struct block_device *bdev, u64 key,
2207 enum pr_type type, unsigned flags)
2208 {
2209 u32 cdw10;
2210
2211 if (flags & ~PR_FL_IGNORE_KEY)
2212 return -EOPNOTSUPP;
2213
2214 cdw10 = nvme_pr_type(type) << 8;
2215 cdw10 |= ((flags & PR_FL_IGNORE_KEY) ? 1 << 3 : 0);
2216 return nvme_pr_command(bdev, cdw10, key, 0, nvme_cmd_resv_acquire);
2217 }
2218
2219 static int nvme_pr_preempt(struct block_device *bdev, u64 old, u64 new,
2220 enum pr_type type, bool abort)
2221 {
2222 u32 cdw10 = nvme_pr_type(type) << 8 | (abort ? 2 : 1);
2223 return nvme_pr_command(bdev, cdw10, old, new, nvme_cmd_resv_acquire);
2224 }
2225
2226 static int nvme_pr_clear(struct block_device *bdev, u64 key)
2227 {
2228 u32 cdw10 = 1 | (key ? 1 << 3 : 0);
2229 return nvme_pr_command(bdev, cdw10, key, 0, nvme_cmd_resv_register);
2230 }
2231
2232 static int nvme_pr_release(struct block_device *bdev, u64 key, enum pr_type type)
2233 {
2234 u32 cdw10 = nvme_pr_type(type) << 8 | (key ? 1 << 3 : 0);
2235 return nvme_pr_command(bdev, cdw10, key, 0, nvme_cmd_resv_release);
2236 }
2237
2238 static const struct pr_ops nvme_pr_ops = {
2239 .pr_register = nvme_pr_register,
2240 .pr_reserve = nvme_pr_reserve,
2241 .pr_release = nvme_pr_release,
2242 .pr_preempt = nvme_pr_preempt,
2243 .pr_clear = nvme_pr_clear,
2244 };
2245
2246 #ifdef CONFIG_BLK_SED_OPAL
2247 int nvme_sec_submit(void *data, u16 spsp, u8 secp, void *buffer, size_t len,
2248 bool send)
2249 {
2250 struct nvme_ctrl *ctrl = data;
2251 struct nvme_command cmd;
2252
2253 memset(&cmd, 0, sizeof(cmd));
2254 if (send)
2255 cmd.common.opcode = nvme_admin_security_send;
2256 else
2257 cmd.common.opcode = nvme_admin_security_recv;
2258 cmd.common.nsid = 0;
2259 cmd.common.cdw10 = cpu_to_le32(((u32)secp) << 24 | ((u32)spsp) << 8);
2260 cmd.common.cdw11 = cpu_to_le32(len);
2261
2262 return __nvme_submit_sync_cmd(ctrl->admin_q, &cmd, NULL, buffer, len,
2263 ADMIN_TIMEOUT, NVME_QID_ANY, 1, 0, false);
2264 }
2265 EXPORT_SYMBOL_GPL(nvme_sec_submit);
2266 #endif /* CONFIG_BLK_SED_OPAL */
2267
2268 static const struct block_device_operations nvme_fops = {
2269 .owner = THIS_MODULE,
2270 .ioctl = nvme_ioctl,
2271 .compat_ioctl = nvme_compat_ioctl,
2272 .open = nvme_open,
2273 .release = nvme_release,
2274 .getgeo = nvme_getgeo,
2275 .report_zones = nvme_report_zones,
2276 .pr_ops = &nvme_pr_ops,
2277 };
2278
2279 #ifdef CONFIG_NVME_MULTIPATH
2280 static int nvme_ns_head_open(struct block_device *bdev, fmode_t mode)
2281 {
2282 struct nvme_ns_head *head = bdev->bd_disk->private_data;
2283
2284 if (!kref_get_unless_zero(&head->ref))
2285 return -ENXIO;
2286 return 0;
2287 }
2288
2289 static void nvme_ns_head_release(struct gendisk *disk, fmode_t mode)
2290 {
2291 nvme_put_ns_head(disk->private_data);
2292 }
2293
2294 const struct block_device_operations nvme_ns_head_ops = {
2295 .owner = THIS_MODULE,
2296 .submit_bio = nvme_ns_head_submit_bio,
2297 .open = nvme_ns_head_open,
2298 .release = nvme_ns_head_release,
2299 .ioctl = nvme_ioctl,
2300 .compat_ioctl = nvme_compat_ioctl,
2301 .getgeo = nvme_getgeo,
2302 .report_zones = nvme_report_zones,
2303 .pr_ops = &nvme_pr_ops,
2304 };
2305 #endif /* CONFIG_NVME_MULTIPATH */
2306
2307 static int nvme_wait_ready(struct nvme_ctrl *ctrl, u64 cap, bool enabled)
2308 {
2309 unsigned long timeout =
2310 ((NVME_CAP_TIMEOUT(cap) + 1) * HZ / 2) + jiffies;
2311 u32 csts, bit = enabled ? NVME_CSTS_RDY : 0;
2312 int ret;
2313
2314 while ((ret = ctrl->ops->reg_read32(ctrl, NVME_REG_CSTS, &csts)) == 0) {
2315 if (csts == ~0)
2316 return -ENODEV;
2317 if ((csts & NVME_CSTS_RDY) == bit)
2318 break;
2319
2320 usleep_range(1000, 2000);
2321 if (fatal_signal_pending(current))
2322 return -EINTR;
2323 if (time_after(jiffies, timeout)) {
2324 dev_err(ctrl->device,
2325 "Device not ready; aborting %s, CSTS=0x%x\n",
2326 enabled ? "initialisation" : "reset", csts);
2327 return -ENODEV;
2328 }
2329 }
2330
2331 return ret;
2332 }
2333
2334 /*
2335 * If the device has been passed off to us in an enabled state, just clear
2336 * the enabled bit. The spec says we should set the 'shutdown notification
2337 * bits', but doing so may cause the device to complete commands to the
2338 * admin queue ... and we don't know what memory that might be pointing at!
2339 */
2340 int nvme_disable_ctrl(struct nvme_ctrl *ctrl)
2341 {
2342 int ret;
2343
2344 ctrl->ctrl_config &= ~NVME_CC_SHN_MASK;
2345 ctrl->ctrl_config &= ~NVME_CC_ENABLE;
2346
2347 ret = ctrl->ops->reg_write32(ctrl, NVME_REG_CC, ctrl->ctrl_config);
2348 if (ret)
2349 return ret;
2350
2351 if (ctrl->quirks & NVME_QUIRK_DELAY_BEFORE_CHK_RDY)
2352 msleep(NVME_QUIRK_DELAY_AMOUNT);
2353
2354 return nvme_wait_ready(ctrl, ctrl->cap, false);
2355 }
2356 EXPORT_SYMBOL_GPL(nvme_disable_ctrl);
2357
2358 int nvme_enable_ctrl(struct nvme_ctrl *ctrl)
2359 {
2360 unsigned dev_page_min;
2361 int ret;
2362
2363 ret = ctrl->ops->reg_read64(ctrl, NVME_REG_CAP, &ctrl->cap);
2364 if (ret) {
2365 dev_err(ctrl->device, "Reading CAP failed (%d)\n", ret);
2366 return ret;
2367 }
2368 dev_page_min = NVME_CAP_MPSMIN(ctrl->cap) + 12;
2369
2370 if (NVME_CTRL_PAGE_SHIFT < dev_page_min) {
2371 dev_err(ctrl->device,
2372 "Minimum device page size %u too large for host (%u)\n",
2373 1 << dev_page_min, 1 << NVME_CTRL_PAGE_SHIFT);
2374 return -ENODEV;
2375 }
2376
2377 if (NVME_CAP_CSS(ctrl->cap) & NVME_CAP_CSS_CSI)
2378 ctrl->ctrl_config = NVME_CC_CSS_CSI;
2379 else
2380 ctrl->ctrl_config = NVME_CC_CSS_NVM;
2381 ctrl->ctrl_config |= (NVME_CTRL_PAGE_SHIFT - 12) << NVME_CC_MPS_SHIFT;
2382 ctrl->ctrl_config |= NVME_CC_AMS_RR | NVME_CC_SHN_NONE;
2383 ctrl->ctrl_config |= NVME_CC_IOSQES | NVME_CC_IOCQES;
2384 ctrl->ctrl_config |= NVME_CC_ENABLE;
2385
2386 ret = ctrl->ops->reg_write32(ctrl, NVME_REG_CC, ctrl->ctrl_config);
2387 if (ret)
2388 return ret;
2389 return nvme_wait_ready(ctrl, ctrl->cap, true);
2390 }
2391 EXPORT_SYMBOL_GPL(nvme_enable_ctrl);
2392
2393 int nvme_shutdown_ctrl(struct nvme_ctrl *ctrl)
2394 {
2395 unsigned long timeout = jiffies + (ctrl->shutdown_timeout * HZ);
2396 u32 csts;
2397 int ret;
2398
2399 ctrl->ctrl_config &= ~NVME_CC_SHN_MASK;
2400 ctrl->ctrl_config |= NVME_CC_SHN_NORMAL;
2401
2402 ret = ctrl->ops->reg_write32(ctrl, NVME_REG_CC, ctrl->ctrl_config);
2403 if (ret)
2404 return ret;
2405
2406 while ((ret = ctrl->ops->reg_read32(ctrl, NVME_REG_CSTS, &csts)) == 0) {
2407 if ((csts & NVME_CSTS_SHST_MASK) == NVME_CSTS_SHST_CMPLT)
2408 break;
2409
2410 msleep(100);
2411 if (fatal_signal_pending(current))
2412 return -EINTR;
2413 if (time_after(jiffies, timeout)) {
2414 dev_err(ctrl->device,
2415 "Device shutdown incomplete; abort shutdown\n");
2416 return -ENODEV;
2417 }
2418 }
2419
2420 return ret;
2421 }
2422 EXPORT_SYMBOL_GPL(nvme_shutdown_ctrl);
2423
2424 static int nvme_configure_timestamp(struct nvme_ctrl *ctrl)
2425 {
2426 __le64 ts;
2427 int ret;
2428
2429 if (!(ctrl->oncs & NVME_CTRL_ONCS_TIMESTAMP))
2430 return 0;
2431
2432 ts = cpu_to_le64(ktime_to_ms(ktime_get_real()));
2433 ret = nvme_set_features(ctrl, NVME_FEAT_TIMESTAMP, 0, &ts, sizeof(ts),
2434 NULL);
2435 if (ret)
2436 dev_warn_once(ctrl->device,
2437 "could not set timestamp (%d)\n", ret);
2438 return ret;
2439 }
2440
2441 static int nvme_configure_acre(struct nvme_ctrl *ctrl)
2442 {
2443 struct nvme_feat_host_behavior *host;
2444 int ret;
2445
2446 /* Don't bother enabling the feature if retry delay is not reported */
2447 if (!ctrl->crdt[0])
2448 return 0;
2449
2450 host = kzalloc(sizeof(*host), GFP_KERNEL);
2451 if (!host)
2452 return 0;
2453
2454 host->acre = NVME_ENABLE_ACRE;
2455 ret = nvme_set_features(ctrl, NVME_FEAT_HOST_BEHAVIOR, 0,
2456 host, sizeof(*host), NULL);
2457 kfree(host);
2458 return ret;
2459 }
2460
2461 static int nvme_configure_apst(struct nvme_ctrl *ctrl)
2462 {
2463 /*
2464 * APST (Autonomous Power State Transition) lets us program a
2465 * table of power state transitions that the controller will
2466 * perform automatically. We configure it with a simple
2467 * heuristic: we are willing to spend at most 2% of the time
2468 * transitioning between power states. Therefore, when running
2469 * in any given state, we will enter the next lower-power
2470 * non-operational state after waiting 50 * (enlat + exlat)
2471 * microseconds, as long as that state's exit latency is under
2472 * the requested maximum latency.
2473 *
2474 * We will not autonomously enter any non-operational state for
2475 * which the total latency exceeds ps_max_latency_us. Users
2476 * can set ps_max_latency_us to zero to turn off APST.
2477 */
2478
2479 unsigned apste;
2480 struct nvme_feat_auto_pst *table;
2481 u64 max_lat_us = 0;
2482 int max_ps = -1;
2483 int ret;
2484
2485 /*
2486 * If APST isn't supported or if we haven't been initialized yet,
2487 * then don't do anything.
2488 */
2489 if (!ctrl->apsta)
2490 return 0;
2491
2492 if (ctrl->npss > 31) {
2493 dev_warn(ctrl->device, "NPSS is invalid; not using APST\n");
2494 return 0;
2495 }
2496
2497 table = kzalloc(sizeof(*table), GFP_KERNEL);
2498 if (!table)
2499 return 0;
2500
2501 if (!ctrl->apst_enabled || ctrl->ps_max_latency_us == 0) {
2502 /* Turn off APST. */
2503 apste = 0;
2504 dev_dbg(ctrl->device, "APST disabled\n");
2505 } else {
2506 __le64 target = cpu_to_le64(0);
2507 int state;
2508
2509 /*
2510 * Walk through all states from lowest- to highest-power.
2511 * According to the spec, lower-numbered states use more
2512 * power. NPSS, despite the name, is the index of the
2513 * lowest-power state, not the number of states.
2514 */
2515 for (state = (int)ctrl->npss; state >= 0; state--) {
2516 u64 total_latency_us, exit_latency_us, transition_ms;
2517
2518 if (target)
2519 table->entries[state] = target;
2520
2521 /*
2522 * Don't allow transitions to the deepest state
2523 * if it's quirked off.
2524 */
2525 if (state == ctrl->npss &&
2526 (ctrl->quirks & NVME_QUIRK_NO_DEEPEST_PS))
2527 continue;
2528
2529 /*
2530 * Is this state a useful non-operational state for
2531 * higher-power states to autonomously transition to?
2532 */
2533 if (!(ctrl->psd[state].flags &
2534 NVME_PS_FLAGS_NON_OP_STATE))
2535 continue;
2536
2537 exit_latency_us =
2538 (u64)le32_to_cpu(ctrl->psd[state].exit_lat);
2539 if (exit_latency_us > ctrl->ps_max_latency_us)
2540 continue;
2541
2542 total_latency_us =
2543 exit_latency_us +
2544 le32_to_cpu(ctrl->psd[state].entry_lat);
2545
2546 /*
2547 * This state is good. Use it as the APST idle
2548 * target for higher power states.
2549 */
2550 transition_ms = total_latency_us + 19;
2551 do_div(transition_ms, 20);
2552 if (transition_ms > (1 << 24) - 1)
2553 transition_ms = (1 << 24) - 1;
2554
2555 target = cpu_to_le64((state << 3) |
2556 (transition_ms << 8));
2557
2558 if (max_ps == -1)
2559 max_ps = state;
2560
2561 if (total_latency_us > max_lat_us)
2562 max_lat_us = total_latency_us;
2563 }
2564
2565 apste = 1;
2566
2567 if (max_ps == -1) {
2568 dev_dbg(ctrl->device, "APST enabled but no non-operational states are available\n");
2569 } else {
2570 dev_dbg(ctrl->device, "APST enabled: max PS = %d, max round-trip latency = %lluus, table = %*phN\n",
2571 max_ps, max_lat_us, (int)sizeof(*table), table);
2572 }
2573 }
2574
2575 ret = nvme_set_features(ctrl, NVME_FEAT_AUTO_PST, apste,
2576 table, sizeof(*table), NULL);
2577 if (ret)
2578 dev_err(ctrl->device, "failed to set APST feature (%d)\n", ret);
2579
2580 kfree(table);
2581 return ret;
2582 }
2583
2584 static void nvme_set_latency_tolerance(struct device *dev, s32 val)
2585 {
2586 struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
2587 u64 latency;
2588
2589 switch (val) {
2590 case PM_QOS_LATENCY_TOLERANCE_NO_CONSTRAINT:
2591 case PM_QOS_LATENCY_ANY:
2592 latency = U64_MAX;
2593 break;
2594
2595 default:
2596 latency = val;
2597 }
2598
2599 if (ctrl->ps_max_latency_us != latency) {
2600 ctrl->ps_max_latency_us = latency;
2601 nvme_configure_apst(ctrl);
2602 }
2603 }
2604
2605 struct nvme_core_quirk_entry {
2606 /*
2607 * NVMe model and firmware strings are padded with spaces. For
2608 * simplicity, strings in the quirk table are padded with NULLs
2609 * instead.
2610 */
2611 u16 vid;
2612 const char *mn;
2613 const char *fr;
2614 unsigned long quirks;
2615 };
2616
2617 static const struct nvme_core_quirk_entry core_quirks[] = {
2618 {
2619 /*
2620 * This Toshiba device seems to die using any APST states. See:
2621 * https://bugs.launchpad.net/ubuntu/+source/linux/+bug/1678184/comments/11
2622 */
2623 .vid = 0x1179,
2624 .mn = "THNSF5256GPUK TOSHIBA",
2625 .quirks = NVME_QUIRK_NO_APST,
2626 },
2627 {
2628 /*
2629 * This LiteON CL1-3D*-Q11 firmware version has a race
2630 * condition associated with actions related to suspend to idle
2631 * LiteON has resolved the problem in future firmware
2632 */
2633 .vid = 0x14a4,
2634 .fr = "22301111",
2635 .quirks = NVME_QUIRK_SIMPLE_SUSPEND,
2636 }
2637 };
2638
2639 /* match is null-terminated but idstr is space-padded. */
2640 static bool string_matches(const char *idstr, const char *match, size_t len)
2641 {
2642 size_t matchlen;
2643
2644 if (!match)
2645 return true;
2646
2647 matchlen = strlen(match);
2648 WARN_ON_ONCE(matchlen > len);
2649
2650 if (memcmp(idstr, match, matchlen))
2651 return false;
2652
2653 for (; matchlen < len; matchlen++)
2654 if (idstr[matchlen] != ' ')
2655 return false;
2656
2657 return true;
2658 }
2659
2660 static bool quirk_matches(const struct nvme_id_ctrl *id,
2661 const struct nvme_core_quirk_entry *q)
2662 {
2663 return q->vid == le16_to_cpu(id->vid) &&
2664 string_matches(id->mn, q->mn, sizeof(id->mn)) &&
2665 string_matches(id->fr, q->fr, sizeof(id->fr));
2666 }
2667
2668 static void nvme_init_subnqn(struct nvme_subsystem *subsys, struct nvme_ctrl *ctrl,
2669 struct nvme_id_ctrl *id)
2670 {
2671 size_t nqnlen;
2672 int off;
2673
2674 if(!(ctrl->quirks & NVME_QUIRK_IGNORE_DEV_SUBNQN)) {
2675 nqnlen = strnlen(id->subnqn, NVMF_NQN_SIZE);
2676 if (nqnlen > 0 && nqnlen < NVMF_NQN_SIZE) {
2677 strlcpy(subsys->subnqn, id->subnqn, NVMF_NQN_SIZE);
2678 return;
2679 }
2680
2681 if (ctrl->vs >= NVME_VS(1, 2, 1))
2682 dev_warn(ctrl->device, "missing or invalid SUBNQN field.\n");
2683 }
2684
2685 /* Generate a "fake" NQN per Figure 254 in NVMe 1.3 + ECN 001 */
2686 off = snprintf(subsys->subnqn, NVMF_NQN_SIZE,
2687 "nqn.2014.08.org.nvmexpress:%04x%04x",
2688 le16_to_cpu(id->vid), le16_to_cpu(id->ssvid));
2689 memcpy(subsys->subnqn + off, id->sn, sizeof(id->sn));
2690 off += sizeof(id->sn);
2691 memcpy(subsys->subnqn + off, id->mn, sizeof(id->mn));
2692 off += sizeof(id->mn);
2693 memset(subsys->subnqn + off, 0, sizeof(subsys->subnqn) - off);
2694 }
2695
2696 static void nvme_release_subsystem(struct device *dev)
2697 {
2698 struct nvme_subsystem *subsys =
2699 container_of(dev, struct nvme_subsystem, dev);
2700
2701 if (subsys->instance >= 0)
2702 ida_simple_remove(&nvme_instance_ida, subsys->instance);
2703 kfree(subsys);
2704 }
2705
2706 static void nvme_destroy_subsystem(struct kref *ref)
2707 {
2708 struct nvme_subsystem *subsys =
2709 container_of(ref, struct nvme_subsystem, ref);
2710
2711 mutex_lock(&nvme_subsystems_lock);
2712 list_del(&subsys->entry);
2713 mutex_unlock(&nvme_subsystems_lock);
2714
2715 ida_destroy(&subsys->ns_ida);
2716 device_del(&subsys->dev);
2717 put_device(&subsys->dev);
2718 }
2719
2720 static void nvme_put_subsystem(struct nvme_subsystem *subsys)
2721 {
2722 kref_put(&subsys->ref, nvme_destroy_subsystem);
2723 }
2724
2725 static struct nvme_subsystem *__nvme_find_get_subsystem(const char *subsysnqn)
2726 {
2727 struct nvme_subsystem *subsys;
2728
2729 lockdep_assert_held(&nvme_subsystems_lock);
2730
2731 /*
2732 * Fail matches for discovery subsystems. This results
2733 * in each discovery controller bound to a unique subsystem.
2734 * This avoids issues with validating controller values
2735 * that can only be true when there is a single unique subsystem.
2736 * There may be multiple and completely independent entities
2737 * that provide discovery controllers.
2738 */
2739 if (!strcmp(subsysnqn, NVME_DISC_SUBSYS_NAME))
2740 return NULL;
2741
2742 list_for_each_entry(subsys, &nvme_subsystems, entry) {
2743 if (strcmp(subsys->subnqn, subsysnqn))
2744 continue;
2745 if (!kref_get_unless_zero(&subsys->ref))
2746 continue;
2747 return subsys;
2748 }
2749
2750 return NULL;
2751 }
2752
2753 #define SUBSYS_ATTR_RO(_name, _mode, _show) \
2754 struct device_attribute subsys_attr_##_name = \
2755 __ATTR(_name, _mode, _show, NULL)
2756
2757 static ssize_t nvme_subsys_show_nqn(struct device *dev,
2758 struct device_attribute *attr,
2759 char *buf)
2760 {
2761 struct nvme_subsystem *subsys =
2762 container_of(dev, struct nvme_subsystem, dev);
2763
2764 return snprintf(buf, PAGE_SIZE, "%s\n", subsys->subnqn);
2765 }
2766 static SUBSYS_ATTR_RO(subsysnqn, S_IRUGO, nvme_subsys_show_nqn);
2767
2768 #define nvme_subsys_show_str_function(field) \
2769 static ssize_t subsys_##field##_show(struct device *dev, \
2770 struct device_attribute *attr, char *buf) \
2771 { \
2772 struct nvme_subsystem *subsys = \
2773 container_of(dev, struct nvme_subsystem, dev); \
2774 return sprintf(buf, "%.*s\n", \
2775 (int)sizeof(subsys->field), subsys->field); \
2776 } \
2777 static SUBSYS_ATTR_RO(field, S_IRUGO, subsys_##field##_show);
2778
2779 nvme_subsys_show_str_function(model);
2780 nvme_subsys_show_str_function(serial);
2781 nvme_subsys_show_str_function(firmware_rev);
2782
2783 static struct attribute *nvme_subsys_attrs[] = {
2784 &subsys_attr_model.attr,
2785 &subsys_attr_serial.attr,
2786 &subsys_attr_firmware_rev.attr,
2787 &subsys_attr_subsysnqn.attr,
2788 #ifdef CONFIG_NVME_MULTIPATH
2789 &subsys_attr_iopolicy.attr,
2790 #endif
2791 NULL,
2792 };
2793
2794 static struct attribute_group nvme_subsys_attrs_group = {
2795 .attrs = nvme_subsys_attrs,
2796 };
2797
2798 static const struct attribute_group *nvme_subsys_attrs_groups[] = {
2799 &nvme_subsys_attrs_group,
2800 NULL,
2801 };
2802
2803 static bool nvme_validate_cntlid(struct nvme_subsystem *subsys,
2804 struct nvme_ctrl *ctrl, struct nvme_id_ctrl *id)
2805 {
2806 struct nvme_ctrl *tmp;
2807
2808 lockdep_assert_held(&nvme_subsystems_lock);
2809
2810 list_for_each_entry(tmp, &subsys->ctrls, subsys_entry) {
2811 if (nvme_state_terminal(tmp))
2812 continue;
2813
2814 if (tmp->cntlid == ctrl->cntlid) {
2815 dev_err(ctrl->device,
2816 "Duplicate cntlid %u with %s, rejecting\n",
2817 ctrl->cntlid, dev_name(tmp->device));
2818 return false;
2819 }
2820
2821 if ((id->cmic & NVME_CTRL_CMIC_MULTI_CTRL) ||
2822 (ctrl->opts && ctrl->opts->discovery_nqn))
2823 continue;
2824
2825 dev_err(ctrl->device,
2826 "Subsystem does not support multiple controllers\n");
2827 return false;
2828 }
2829
2830 return true;
2831 }
2832
2833 static int nvme_init_subsystem(struct nvme_ctrl *ctrl, struct nvme_id_ctrl *id)
2834 {
2835 struct nvme_subsystem *subsys, *found;
2836 int ret;
2837
2838 subsys = kzalloc(sizeof(*subsys), GFP_KERNEL);
2839 if (!subsys)
2840 return -ENOMEM;
2841
2842 subsys->instance = -1;
2843 mutex_init(&subsys->lock);
2844 kref_init(&subsys->ref);
2845 INIT_LIST_HEAD(&subsys->ctrls);
2846 INIT_LIST_HEAD(&subsys->nsheads);
2847 nvme_init_subnqn(subsys, ctrl, id);
2848 memcpy(subsys->serial, id->sn, sizeof(subsys->serial));
2849 memcpy(subsys->model, id->mn, sizeof(subsys->model));
2850 memcpy(subsys->firmware_rev, id->fr, sizeof(subsys->firmware_rev));
2851 subsys->vendor_id = le16_to_cpu(id->vid);
2852 subsys->cmic = id->cmic;
2853 subsys->awupf = le16_to_cpu(id->awupf);
2854 #ifdef CONFIG_NVME_MULTIPATH
2855 subsys->iopolicy = NVME_IOPOLICY_NUMA;
2856 #endif
2857
2858 subsys->dev.class = nvme_subsys_class;
2859 subsys->dev.release = nvme_release_subsystem;
2860 subsys->dev.groups = nvme_subsys_attrs_groups;
2861 dev_set_name(&subsys->dev, "nvme-subsys%d", ctrl->instance);
2862 device_initialize(&subsys->dev);
2863
2864 mutex_lock(&nvme_subsystems_lock);
2865 found = __nvme_find_get_subsystem(subsys->subnqn);
2866 if (found) {
2867 put_device(&subsys->dev);
2868 subsys = found;
2869
2870 if (!nvme_validate_cntlid(subsys, ctrl, id)) {
2871 ret = -EINVAL;
2872 goto out_put_subsystem;
2873 }
2874 } else {
2875 ret = device_add(&subsys->dev);
2876 if (ret) {
2877 dev_err(ctrl->device,
2878 "failed to register subsystem device.\n");
2879 put_device(&subsys->dev);
2880 goto out_unlock;
2881 }
2882 ida_init(&subsys->ns_ida);
2883 list_add_tail(&subsys->entry, &nvme_subsystems);
2884 }
2885
2886 ret = sysfs_create_link(&subsys->dev.kobj, &ctrl->device->kobj,
2887 dev_name(ctrl->device));
2888 if (ret) {
2889 dev_err(ctrl->device,
2890 "failed to create sysfs link from subsystem.\n");
2891 goto out_put_subsystem;
2892 }
2893
2894 if (!found)
2895 subsys->instance = ctrl->instance;
2896 ctrl->subsys = subsys;
2897 list_add_tail(&ctrl->subsys_entry, &subsys->ctrls);
2898 mutex_unlock(&nvme_subsystems_lock);
2899 return 0;
2900
2901 out_put_subsystem:
2902 nvme_put_subsystem(subsys);
2903 out_unlock:
2904 mutex_unlock(&nvme_subsystems_lock);
2905 return ret;
2906 }
2907
2908 int nvme_get_log(struct nvme_ctrl *ctrl, u32 nsid, u8 log_page, u8 lsp, u8 csi,
2909 void *log, size_t size, u64 offset)
2910 {
2911 struct nvme_command c = { };
2912 u32 dwlen = nvme_bytes_to_numd(size);
2913
2914 c.get_log_page.opcode = nvme_admin_get_log_page;
2915 c.get_log_page.nsid = cpu_to_le32(nsid);
2916 c.get_log_page.lid = log_page;
2917 c.get_log_page.lsp = lsp;
2918 c.get_log_page.numdl = cpu_to_le16(dwlen & ((1 << 16) - 1));
2919 c.get_log_page.numdu = cpu_to_le16(dwlen >> 16);
2920 c.get_log_page.lpol = cpu_to_le32(lower_32_bits(offset));
2921 c.get_log_page.lpou = cpu_to_le32(upper_32_bits(offset));
2922 c.get_log_page.csi = csi;
2923
2924 return nvme_submit_sync_cmd(ctrl->admin_q, &c, log, size);
2925 }
2926
2927 static int nvme_get_effects_log(struct nvme_ctrl *ctrl, u8 csi,
2928 struct nvme_effects_log **log)
2929 {
2930 struct nvme_cel *cel = xa_load(&ctrl->cels, csi);
2931 int ret;
2932
2933 if (cel)
2934 goto out;
2935
2936 cel = kzalloc(sizeof(*cel), GFP_KERNEL);
2937 if (!cel)
2938 return -ENOMEM;
2939
2940 ret = nvme_get_log(ctrl, 0x00, NVME_LOG_CMD_EFFECTS, 0, csi,
2941 &cel->log, sizeof(cel->log), 0);
2942 if (ret) {
2943 kfree(cel);
2944 return ret;
2945 }
2946
2947 cel->csi = csi;
2948 xa_store(&ctrl->cels, cel->csi, cel, GFP_KERNEL);
2949 out:
2950 *log = &cel->log;
2951 return 0;
2952 }
2953
2954 /*
2955 * Initialize the cached copies of the Identify data and various controller
2956 * register in our nvme_ctrl structure. This should be called as soon as
2957 * the admin queue is fully up and running.
2958 */
2959 int nvme_init_identify(struct nvme_ctrl *ctrl)
2960 {
2961 struct nvme_id_ctrl *id;
2962 int ret, page_shift;
2963 u32 max_hw_sectors;
2964 bool prev_apst_enabled;
2965
2966 ret = ctrl->ops->reg_read32(ctrl, NVME_REG_VS, &ctrl->vs);
2967 if (ret) {
2968 dev_err(ctrl->device, "Reading VS failed (%d)\n", ret);
2969 return ret;
2970 }
2971 page_shift = NVME_CAP_MPSMIN(ctrl->cap) + 12;
2972 ctrl->sqsize = min_t(u16, NVME_CAP_MQES(ctrl->cap), ctrl->sqsize);
2973
2974 if (ctrl->vs >= NVME_VS(1, 1, 0))
2975 ctrl->subsystem = NVME_CAP_NSSRC(ctrl->cap);
2976
2977 ret = nvme_identify_ctrl(ctrl, &id);
2978 if (ret) {
2979 dev_err(ctrl->device, "Identify Controller failed (%d)\n", ret);
2980 return -EIO;
2981 }
2982
2983 if (id->lpa & NVME_CTRL_LPA_CMD_EFFECTS_LOG) {
2984 ret = nvme_get_effects_log(ctrl, NVME_CSI_NVM, &ctrl->effects);
2985 if (ret < 0)
2986 goto out_free;
2987 }
2988
2989 if (!(ctrl->ops->flags & NVME_F_FABRICS))
2990 ctrl->cntlid = le16_to_cpu(id->cntlid);
2991
2992 if (!ctrl->identified) {
2993 int i;
2994
2995 ret = nvme_init_subsystem(ctrl, id);
2996 if (ret)
2997 goto out_free;
2998
2999 /*
3000 * Check for quirks. Quirk can depend on firmware version,
3001 * so, in principle, the set of quirks present can change
3002 * across a reset. As a possible future enhancement, we
3003 * could re-scan for quirks every time we reinitialize
3004 * the device, but we'd have to make sure that the driver
3005 * behaves intelligently if the quirks change.
3006 */
3007 for (i = 0; i < ARRAY_SIZE(core_quirks); i++) {
3008 if (quirk_matches(id, &core_quirks[i]))
3009 ctrl->quirks |= core_quirks[i].quirks;
3010 }
3011 }
3012
3013 if (force_apst && (ctrl->quirks & NVME_QUIRK_NO_DEEPEST_PS)) {
3014 dev_warn(ctrl->device, "forcibly allowing all power states due to nvme_core.force_apst -- use at your own risk\n");
3015 ctrl->quirks &= ~NVME_QUIRK_NO_DEEPEST_PS;
3016 }
3017
3018 ctrl->crdt[0] = le16_to_cpu(id->crdt1);
3019 ctrl->crdt[1] = le16_to_cpu(id->crdt2);
3020 ctrl->crdt[2] = le16_to_cpu(id->crdt3);
3021
3022 ctrl->oacs = le16_to_cpu(id->oacs);
3023 ctrl->oncs = le16_to_cpu(id->oncs);
3024 ctrl->mtfa = le16_to_cpu(id->mtfa);
3025 ctrl->oaes = le32_to_cpu(id->oaes);
3026 ctrl->wctemp = le16_to_cpu(id->wctemp);
3027 ctrl->cctemp = le16_to_cpu(id->cctemp);
3028
3029 atomic_set(&ctrl->abort_limit, id->acl + 1);
3030 ctrl->vwc = id->vwc;
3031 if (id->mdts)
3032 max_hw_sectors = 1 << (id->mdts + page_shift - 9);
3033 else
3034 max_hw_sectors = UINT_MAX;
3035 ctrl->max_hw_sectors =
3036 min_not_zero(ctrl->max_hw_sectors, max_hw_sectors);
3037
3038 nvme_set_queue_limits(ctrl, ctrl->admin_q);
3039 ctrl->sgls = le32_to_cpu(id->sgls);
3040 ctrl->kas = le16_to_cpu(id->kas);
3041 ctrl->max_namespaces = le32_to_cpu(id->mnan);
3042 ctrl->ctratt = le32_to_cpu(id->ctratt);
3043
3044 if (id->rtd3e) {
3045 /* us -> s */
3046 u32 transition_time = le32_to_cpu(id->rtd3e) / USEC_PER_SEC;
3047
3048 ctrl->shutdown_timeout = clamp_t(unsigned int, transition_time,
3049 shutdown_timeout, 60);
3050
3051 if (ctrl->shutdown_timeout != shutdown_timeout)
3052 dev_info(ctrl->device,
3053 "Shutdown timeout set to %u seconds\n",
3054 ctrl->shutdown_timeout);
3055 } else
3056 ctrl->shutdown_timeout = shutdown_timeout;
3057
3058 ctrl->npss = id->npss;
3059 ctrl->apsta = id->apsta;
3060 prev_apst_enabled = ctrl->apst_enabled;
3061 if (ctrl->quirks & NVME_QUIRK_NO_APST) {
3062 if (force_apst && id->apsta) {
3063 dev_warn(ctrl->device, "forcibly allowing APST due to nvme_core.force_apst -- use at your own risk\n");
3064 ctrl->apst_enabled = true;
3065 } else {
3066 ctrl->apst_enabled = false;
3067 }
3068 } else {
3069 ctrl->apst_enabled = id->apsta;
3070 }
3071 memcpy(ctrl->psd, id->psd, sizeof(ctrl->psd));
3072
3073 if (ctrl->ops->flags & NVME_F_FABRICS) {
3074 ctrl->icdoff = le16_to_cpu(id->icdoff);
3075 ctrl->ioccsz = le32_to_cpu(id->ioccsz);
3076 ctrl->iorcsz = le32_to_cpu(id->iorcsz);
3077 ctrl->maxcmd = le16_to_cpu(id->maxcmd);
3078
3079 /*
3080 * In fabrics we need to verify the cntlid matches the
3081 * admin connect
3082 */
3083 if (ctrl->cntlid != le16_to_cpu(id->cntlid)) {
3084 dev_err(ctrl->device,
3085 "Mismatching cntlid: Connect %u vs Identify "
3086 "%u, rejecting\n",
3087 ctrl->cntlid, le16_to_cpu(id->cntlid));
3088 ret = -EINVAL;
3089 goto out_free;
3090 }
3091
3092 if (!ctrl->opts->discovery_nqn && !ctrl->kas) {
3093 dev_err(ctrl->device,
3094 "keep-alive support is mandatory for fabrics\n");
3095 ret = -EINVAL;
3096 goto out_free;
3097 }
3098 } else {
3099 ctrl->hmpre = le32_to_cpu(id->hmpre);
3100 ctrl->hmmin = le32_to_cpu(id->hmmin);
3101 ctrl->hmminds = le32_to_cpu(id->hmminds);
3102 ctrl->hmmaxd = le16_to_cpu(id->hmmaxd);
3103 }
3104
3105 ret = nvme_mpath_init(ctrl, id);
3106 kfree(id);
3107
3108 if (ret < 0)
3109 return ret;
3110
3111 if (ctrl->apst_enabled && !prev_apst_enabled)
3112 dev_pm_qos_expose_latency_tolerance(ctrl->device);
3113 else if (!ctrl->apst_enabled && prev_apst_enabled)
3114 dev_pm_qos_hide_latency_tolerance(ctrl->device);
3115
3116 ret = nvme_configure_apst(ctrl);
3117 if (ret < 0)
3118 return ret;
3119
3120 ret = nvme_configure_timestamp(ctrl);
3121 if (ret < 0)
3122 return ret;
3123
3124 ret = nvme_configure_directives(ctrl);
3125 if (ret < 0)
3126 return ret;
3127
3128 ret = nvme_configure_acre(ctrl);
3129 if (ret < 0)
3130 return ret;
3131
3132 if (!ctrl->identified) {
3133 ret = nvme_hwmon_init(ctrl);
3134 if (ret < 0)
3135 return ret;
3136 }
3137
3138 ctrl->identified = true;
3139
3140 return 0;
3141
3142 out_free:
3143 kfree(id);
3144 return ret;
3145 }
3146 EXPORT_SYMBOL_GPL(nvme_init_identify);
3147
3148 static int nvme_dev_open(struct inode *inode, struct file *file)
3149 {
3150 struct nvme_ctrl *ctrl =
3151 container_of(inode->i_cdev, struct nvme_ctrl, cdev);
3152
3153 switch (ctrl->state) {
3154 case NVME_CTRL_LIVE:
3155 break;
3156 default:
3157 return -EWOULDBLOCK;
3158 }
3159
3160 nvme_get_ctrl(ctrl);
3161 if (!try_module_get(ctrl->ops->module)) {
3162 nvme_put_ctrl(ctrl);
3163 return -EINVAL;
3164 }
3165
3166 file->private_data = ctrl;
3167 return 0;
3168 }
3169
3170 static int nvme_dev_release(struct inode *inode, struct file *file)
3171 {
3172 struct nvme_ctrl *ctrl =
3173 container_of(inode->i_cdev, struct nvme_ctrl, cdev);
3174
3175 module_put(ctrl->ops->module);
3176 nvme_put_ctrl(ctrl);
3177 return 0;
3178 }
3179
3180 static int nvme_dev_user_cmd(struct nvme_ctrl *ctrl, void __user *argp)
3181 {
3182 struct nvme_ns *ns;
3183 int ret;
3184
3185 down_read(&ctrl->namespaces_rwsem);
3186 if (list_empty(&ctrl->namespaces)) {
3187 ret = -ENOTTY;
3188 goto out_unlock;
3189 }
3190
3191 ns = list_first_entry(&ctrl->namespaces, struct nvme_ns, list);
3192 if (ns != list_last_entry(&ctrl->namespaces, struct nvme_ns, list)) {
3193 dev_warn(ctrl->device,
3194 "NVME_IOCTL_IO_CMD not supported when multiple namespaces present!\n");
3195 ret = -EINVAL;
3196 goto out_unlock;
3197 }
3198
3199 dev_warn(ctrl->device,
3200 "using deprecated NVME_IOCTL_IO_CMD ioctl on the char device!\n");
3201 kref_get(&ns->kref);
3202 up_read(&ctrl->namespaces_rwsem);
3203
3204 ret = nvme_user_cmd(ctrl, ns, argp);
3205 nvme_put_ns(ns);
3206 return ret;
3207
3208 out_unlock:
3209 up_read(&ctrl->namespaces_rwsem);
3210 return ret;
3211 }
3212
3213 static long nvme_dev_ioctl(struct file *file, unsigned int cmd,
3214 unsigned long arg)
3215 {
3216 struct nvme_ctrl *ctrl = file->private_data;
3217 void __user *argp = (void __user *)arg;
3218
3219 switch (cmd) {
3220 case NVME_IOCTL_ADMIN_CMD:
3221 return nvme_user_cmd(ctrl, NULL, argp);
3222 case NVME_IOCTL_ADMIN64_CMD:
3223 return nvme_user_cmd64(ctrl, NULL, argp);
3224 case NVME_IOCTL_IO_CMD:
3225 return nvme_dev_user_cmd(ctrl, argp);
3226 case NVME_IOCTL_RESET:
3227 dev_warn(ctrl->device, "resetting controller\n");
3228 return nvme_reset_ctrl_sync(ctrl);
3229 case NVME_IOCTL_SUBSYS_RESET:
3230 return nvme_reset_subsystem(ctrl);
3231 case NVME_IOCTL_RESCAN:
3232 nvme_queue_scan(ctrl);
3233 return 0;
3234 default:
3235 return -ENOTTY;
3236 }
3237 }
3238
3239 static const struct file_operations nvme_dev_fops = {
3240 .owner = THIS_MODULE,
3241 .open = nvme_dev_open,
3242 .release = nvme_dev_release,
3243 .unlocked_ioctl = nvme_dev_ioctl,
3244 .compat_ioctl = compat_ptr_ioctl,
3245 };
3246
3247 static ssize_t nvme_sysfs_reset(struct device *dev,
3248 struct device_attribute *attr, const char *buf,
3249 size_t count)
3250 {
3251 struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3252 int ret;
3253
3254 ret = nvme_reset_ctrl_sync(ctrl);
3255 if (ret < 0)
3256 return ret;
3257 return count;
3258 }
3259 static DEVICE_ATTR(reset_controller, S_IWUSR, NULL, nvme_sysfs_reset);
3260
3261 static ssize_t nvme_sysfs_rescan(struct device *dev,
3262 struct device_attribute *attr, const char *buf,
3263 size_t count)
3264 {
3265 struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3266
3267 nvme_queue_scan(ctrl);
3268 return count;
3269 }
3270 static DEVICE_ATTR(rescan_controller, S_IWUSR, NULL, nvme_sysfs_rescan);
3271
3272 static inline struct nvme_ns_head *dev_to_ns_head(struct device *dev)
3273 {
3274 struct gendisk *disk = dev_to_disk(dev);
3275
3276 if (disk->fops == &nvme_fops)
3277 return nvme_get_ns_from_dev(dev)->head;
3278 else
3279 return disk->private_data;
3280 }
3281
3282 static ssize_t wwid_show(struct device *dev, struct device_attribute *attr,
3283 char *buf)
3284 {
3285 struct nvme_ns_head *head = dev_to_ns_head(dev);
3286 struct nvme_ns_ids *ids = &head->ids;
3287 struct nvme_subsystem *subsys = head->subsys;
3288 int serial_len = sizeof(subsys->serial);
3289 int model_len = sizeof(subsys->model);
3290
3291 if (!uuid_is_null(&ids->uuid))
3292 return sprintf(buf, "uuid.%pU\n", &ids->uuid);
3293
3294 if (memchr_inv(ids->nguid, 0, sizeof(ids->nguid)))
3295 return sprintf(buf, "eui.%16phN\n", ids->nguid);
3296
3297 if (memchr_inv(ids->eui64, 0, sizeof(ids->eui64)))
3298 return sprintf(buf, "eui.%8phN\n", ids->eui64);
3299
3300 while (serial_len > 0 && (subsys->serial[serial_len - 1] == ' ' ||
3301 subsys->serial[serial_len - 1] == '\0'))
3302 serial_len--;
3303 while (model_len > 0 && (subsys->model[model_len - 1] == ' ' ||
3304 subsys->model[model_len - 1] == '\0'))
3305 model_len--;
3306
3307 return sprintf(buf, "nvme.%04x-%*phN-%*phN-%08x\n", subsys->vendor_id,
3308 serial_len, subsys->serial, model_len, subsys->model,
3309 head->ns_id);
3310 }
3311 static DEVICE_ATTR_RO(wwid);
3312
3313 static ssize_t nguid_show(struct device *dev, struct device_attribute *attr,
3314 char *buf)
3315 {
3316 return sprintf(buf, "%pU\n", dev_to_ns_head(dev)->ids.nguid);
3317 }
3318 static DEVICE_ATTR_RO(nguid);
3319
3320 static ssize_t uuid_show(struct device *dev, struct device_attribute *attr,
3321 char *buf)
3322 {
3323 struct nvme_ns_ids *ids = &dev_to_ns_head(dev)->ids;
3324
3325 /* For backward compatibility expose the NGUID to userspace if
3326 * we have no UUID set
3327 */
3328 if (uuid_is_null(&ids->uuid)) {
3329 printk_ratelimited(KERN_WARNING
3330 "No UUID available providing old NGUID\n");
3331 return sprintf(buf, "%pU\n", ids->nguid);
3332 }
3333 return sprintf(buf, "%pU\n", &ids->uuid);
3334 }
3335 static DEVICE_ATTR_RO(uuid);
3336
3337 static ssize_t eui_show(struct device *dev, struct device_attribute *attr,
3338 char *buf)
3339 {
3340 return sprintf(buf, "%8ph\n", dev_to_ns_head(dev)->ids.eui64);
3341 }
3342 static DEVICE_ATTR_RO(eui);
3343
3344 static ssize_t nsid_show(struct device *dev, struct device_attribute *attr,
3345 char *buf)
3346 {
3347 return sprintf(buf, "%d\n", dev_to_ns_head(dev)->ns_id);
3348 }
3349 static DEVICE_ATTR_RO(nsid);
3350
3351 static struct attribute *nvme_ns_id_attrs[] = {
3352 &dev_attr_wwid.attr,
3353 &dev_attr_uuid.attr,
3354 &dev_attr_nguid.attr,
3355 &dev_attr_eui.attr,
3356 &dev_attr_nsid.attr,
3357 #ifdef CONFIG_NVME_MULTIPATH
3358 &dev_attr_ana_grpid.attr,
3359 &dev_attr_ana_state.attr,
3360 #endif
3361 NULL,
3362 };
3363
3364 static umode_t nvme_ns_id_attrs_are_visible(struct kobject *kobj,
3365 struct attribute *a, int n)
3366 {
3367 struct device *dev = container_of(kobj, struct device, kobj);
3368 struct nvme_ns_ids *ids = &dev_to_ns_head(dev)->ids;
3369
3370 if (a == &dev_attr_uuid.attr) {
3371 if (uuid_is_null(&ids->uuid) &&
3372 !memchr_inv(ids->nguid, 0, sizeof(ids->nguid)))
3373 return 0;
3374 }
3375 if (a == &dev_attr_nguid.attr) {
3376 if (!memchr_inv(ids->nguid, 0, sizeof(ids->nguid)))
3377 return 0;
3378 }
3379 if (a == &dev_attr_eui.attr) {
3380 if (!memchr_inv(ids->eui64, 0, sizeof(ids->eui64)))
3381 return 0;
3382 }
3383 #ifdef CONFIG_NVME_MULTIPATH
3384 if (a == &dev_attr_ana_grpid.attr || a == &dev_attr_ana_state.attr) {
3385 if (dev_to_disk(dev)->fops != &nvme_fops) /* per-path attr */
3386 return 0;
3387 if (!nvme_ctrl_use_ana(nvme_get_ns_from_dev(dev)->ctrl))
3388 return 0;
3389 }
3390 #endif
3391 return a->mode;
3392 }
3393
3394 static const struct attribute_group nvme_ns_id_attr_group = {
3395 .attrs = nvme_ns_id_attrs,
3396 .is_visible = nvme_ns_id_attrs_are_visible,
3397 };
3398
3399 const struct attribute_group *nvme_ns_id_attr_groups[] = {
3400 &nvme_ns_id_attr_group,
3401 #ifdef CONFIG_NVM
3402 &nvme_nvm_attr_group,
3403 #endif
3404 NULL,
3405 };
3406
3407 #define nvme_show_str_function(field) \
3408 static ssize_t field##_show(struct device *dev, \
3409 struct device_attribute *attr, char *buf) \
3410 { \
3411 struct nvme_ctrl *ctrl = dev_get_drvdata(dev); \
3412 return sprintf(buf, "%.*s\n", \
3413 (int)sizeof(ctrl->subsys->field), ctrl->subsys->field); \
3414 } \
3415 static DEVICE_ATTR(field, S_IRUGO, field##_show, NULL);
3416
3417 nvme_show_str_function(model);
3418 nvme_show_str_function(serial);
3419 nvme_show_str_function(firmware_rev);
3420
3421 #define nvme_show_int_function(field) \
3422 static ssize_t field##_show(struct device *dev, \
3423 struct device_attribute *attr, char *buf) \
3424 { \
3425 struct nvme_ctrl *ctrl = dev_get_drvdata(dev); \
3426 return sprintf(buf, "%d\n", ctrl->field); \
3427 } \
3428 static DEVICE_ATTR(field, S_IRUGO, field##_show, NULL);
3429
3430 nvme_show_int_function(cntlid);
3431 nvme_show_int_function(numa_node);
3432 nvme_show_int_function(queue_count);
3433 nvme_show_int_function(sqsize);
3434
3435 static ssize_t nvme_sysfs_delete(struct device *dev,
3436 struct device_attribute *attr, const char *buf,
3437 size_t count)
3438 {
3439 struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3440
3441 if (device_remove_file_self(dev, attr))
3442 nvme_delete_ctrl_sync(ctrl);
3443 return count;
3444 }
3445 static DEVICE_ATTR(delete_controller, S_IWUSR, NULL, nvme_sysfs_delete);
3446
3447 static ssize_t nvme_sysfs_show_transport(struct device *dev,
3448 struct device_attribute *attr,
3449 char *buf)
3450 {
3451 struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3452
3453 return snprintf(buf, PAGE_SIZE, "%s\n", ctrl->ops->name);
3454 }
3455 static DEVICE_ATTR(transport, S_IRUGO, nvme_sysfs_show_transport, NULL);
3456
3457 static ssize_t nvme_sysfs_show_state(struct device *dev,
3458 struct device_attribute *attr,
3459 char *buf)
3460 {
3461 struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3462 static const char *const state_name[] = {
3463 [NVME_CTRL_NEW] = "new",
3464 [NVME_CTRL_LIVE] = "live",
3465 [NVME_CTRL_RESETTING] = "resetting",
3466 [NVME_CTRL_CONNECTING] = "connecting",
3467 [NVME_CTRL_DELETING] = "deleting",
3468 [NVME_CTRL_DELETING_NOIO]= "deleting (no IO)",
3469 [NVME_CTRL_DEAD] = "dead",
3470 };
3471
3472 if ((unsigned)ctrl->state < ARRAY_SIZE(state_name) &&
3473 state_name[ctrl->state])
3474 return sprintf(buf, "%s\n", state_name[ctrl->state]);
3475
3476 return sprintf(buf, "unknown state\n");
3477 }
3478
3479 static DEVICE_ATTR(state, S_IRUGO, nvme_sysfs_show_state, NULL);
3480
3481 static ssize_t nvme_sysfs_show_subsysnqn(struct device *dev,
3482 struct device_attribute *attr,
3483 char *buf)
3484 {
3485 struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3486
3487 return snprintf(buf, PAGE_SIZE, "%s\n", ctrl->subsys->subnqn);
3488 }
3489 static DEVICE_ATTR(subsysnqn, S_IRUGO, nvme_sysfs_show_subsysnqn, NULL);
3490
3491 static ssize_t nvme_sysfs_show_hostnqn(struct device *dev,
3492 struct device_attribute *attr,
3493 char *buf)
3494 {
3495 struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3496
3497 return snprintf(buf, PAGE_SIZE, "%s\n", ctrl->opts->host->nqn);
3498 }
3499 static DEVICE_ATTR(hostnqn, S_IRUGO, nvme_sysfs_show_hostnqn, NULL);
3500
3501 static ssize_t nvme_sysfs_show_hostid(struct device *dev,
3502 struct device_attribute *attr,
3503 char *buf)
3504 {
3505 struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3506
3507 return snprintf(buf, PAGE_SIZE, "%pU\n", &ctrl->opts->host->id);
3508 }
3509 static DEVICE_ATTR(hostid, S_IRUGO, nvme_sysfs_show_hostid, NULL);
3510
3511 static ssize_t nvme_sysfs_show_address(struct device *dev,
3512 struct device_attribute *attr,
3513 char *buf)
3514 {
3515 struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3516
3517 return ctrl->ops->get_address(ctrl, buf, PAGE_SIZE);
3518 }
3519 static DEVICE_ATTR(address, S_IRUGO, nvme_sysfs_show_address, NULL);
3520
3521 static ssize_t nvme_ctrl_loss_tmo_show(struct device *dev,
3522 struct device_attribute *attr, char *buf)
3523 {
3524 struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3525 struct nvmf_ctrl_options *opts = ctrl->opts;
3526
3527 if (ctrl->opts->max_reconnects == -1)
3528 return sprintf(buf, "off\n");
3529 return sprintf(buf, "%d\n",
3530 opts->max_reconnects * opts->reconnect_delay);
3531 }
3532
3533 static ssize_t nvme_ctrl_loss_tmo_store(struct device *dev,
3534 struct device_attribute *attr, const char *buf, size_t count)
3535 {
3536 struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3537 struct nvmf_ctrl_options *opts = ctrl->opts;
3538 int ctrl_loss_tmo, err;
3539
3540 err = kstrtoint(buf, 10, &ctrl_loss_tmo);
3541 if (err)
3542 return -EINVAL;
3543
3544 else if (ctrl_loss_tmo < 0)
3545 opts->max_reconnects = -1;
3546 else
3547 opts->max_reconnects = DIV_ROUND_UP(ctrl_loss_tmo,
3548 opts->reconnect_delay);
3549 return count;
3550 }
3551 static DEVICE_ATTR(ctrl_loss_tmo, S_IRUGO | S_IWUSR,
3552 nvme_ctrl_loss_tmo_show, nvme_ctrl_loss_tmo_store);
3553
3554 static ssize_t nvme_ctrl_reconnect_delay_show(struct device *dev,
3555 struct device_attribute *attr, char *buf)
3556 {
3557 struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3558
3559 if (ctrl->opts->reconnect_delay == -1)
3560 return sprintf(buf, "off\n");
3561 return sprintf(buf, "%d\n", ctrl->opts->reconnect_delay);
3562 }
3563
3564 static ssize_t nvme_ctrl_reconnect_delay_store(struct device *dev,
3565 struct device_attribute *attr, const char *buf, size_t count)
3566 {
3567 struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3568 unsigned int v;
3569 int err;
3570
3571 err = kstrtou32(buf, 10, &v);
3572 if (err)
3573 return err;
3574
3575 ctrl->opts->reconnect_delay = v;
3576 return count;
3577 }
3578 static DEVICE_ATTR(reconnect_delay, S_IRUGO | S_IWUSR,
3579 nvme_ctrl_reconnect_delay_show, nvme_ctrl_reconnect_delay_store);
3580
3581 static struct attribute *nvme_dev_attrs[] = {
3582 &dev_attr_reset_controller.attr,
3583 &dev_attr_rescan_controller.attr,
3584 &dev_attr_model.attr,
3585 &dev_attr_serial.attr,
3586 &dev_attr_firmware_rev.attr,
3587 &dev_attr_cntlid.attr,
3588 &dev_attr_delete_controller.attr,
3589 &dev_attr_transport.attr,
3590 &dev_attr_subsysnqn.attr,
3591 &dev_attr_address.attr,
3592 &dev_attr_state.attr,
3593 &dev_attr_numa_node.attr,
3594 &dev_attr_queue_count.attr,
3595 &dev_attr_sqsize.attr,
3596 &dev_attr_hostnqn.attr,
3597 &dev_attr_hostid.attr,
3598 &dev_attr_ctrl_loss_tmo.attr,
3599 &dev_attr_reconnect_delay.attr,
3600 NULL
3601 };
3602
3603 static umode_t nvme_dev_attrs_are_visible(struct kobject *kobj,
3604 struct attribute *a, int n)
3605 {
3606 struct device *dev = container_of(kobj, struct device, kobj);
3607 struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3608
3609 if (a == &dev_attr_delete_controller.attr && !ctrl->ops->delete_ctrl)
3610 return 0;
3611 if (a == &dev_attr_address.attr && !ctrl->ops->get_address)
3612 return 0;
3613 if (a == &dev_attr_hostnqn.attr && !ctrl->opts)
3614 return 0;
3615 if (a == &dev_attr_hostid.attr && !ctrl->opts)
3616 return 0;
3617 if (a == &dev_attr_ctrl_loss_tmo.attr && !ctrl->opts)
3618 return 0;
3619 if (a == &dev_attr_reconnect_delay.attr && !ctrl->opts)
3620 return 0;
3621
3622 return a->mode;
3623 }
3624
3625 static struct attribute_group nvme_dev_attrs_group = {
3626 .attrs = nvme_dev_attrs,
3627 .is_visible = nvme_dev_attrs_are_visible,
3628 };
3629
3630 static const struct attribute_group *nvme_dev_attr_groups[] = {
3631 &nvme_dev_attrs_group,
3632 NULL,
3633 };
3634
3635 static struct nvme_ns_head *nvme_find_ns_head(struct nvme_subsystem *subsys,
3636 unsigned nsid)
3637 {
3638 struct nvme_ns_head *h;
3639
3640 lockdep_assert_held(&subsys->lock);
3641
3642 list_for_each_entry(h, &subsys->nsheads, entry) {
3643 if (h->ns_id == nsid && kref_get_unless_zero(&h->ref))
3644 return h;
3645 }
3646
3647 return NULL;
3648 }
3649
3650 static int __nvme_check_ids(struct nvme_subsystem *subsys,
3651 struct nvme_ns_head *new)
3652 {
3653 struct nvme_ns_head *h;
3654
3655 lockdep_assert_held(&subsys->lock);
3656
3657 list_for_each_entry(h, &subsys->nsheads, entry) {
3658 if (nvme_ns_ids_valid(&new->ids) &&
3659 nvme_ns_ids_equal(&new->ids, &h->ids))
3660 return -EINVAL;
3661 }
3662
3663 return 0;
3664 }
3665
3666 static struct nvme_ns_head *nvme_alloc_ns_head(struct nvme_ctrl *ctrl,
3667 unsigned nsid, struct nvme_ns_ids *ids)
3668 {
3669 struct nvme_ns_head *head;
3670 size_t size = sizeof(*head);
3671 int ret = -ENOMEM;
3672
3673 #ifdef CONFIG_NVME_MULTIPATH
3674 size += num_possible_nodes() * sizeof(struct nvme_ns *);
3675 #endif
3676
3677 head = kzalloc(size, GFP_KERNEL);
3678 if (!head)
3679 goto out;
3680 ret = ida_simple_get(&ctrl->subsys->ns_ida, 1, 0, GFP_KERNEL);
3681 if (ret < 0)
3682 goto out_free_head;
3683 head->instance = ret;
3684 INIT_LIST_HEAD(&head->list);
3685 ret = init_srcu_struct(&head->srcu);
3686 if (ret)
3687 goto out_ida_remove;
3688 head->subsys = ctrl->subsys;
3689 head->ns_id = nsid;
3690 head->ids = *ids;
3691 kref_init(&head->ref);
3692
3693 ret = __nvme_check_ids(ctrl->subsys, head);
3694 if (ret) {
3695 dev_err(ctrl->device,
3696 "duplicate IDs for nsid %d\n", nsid);
3697 goto out_cleanup_srcu;
3698 }
3699
3700 if (head->ids.csi) {
3701 ret = nvme_get_effects_log(ctrl, head->ids.csi, &head->effects);
3702 if (ret)
3703 goto out_cleanup_srcu;
3704 } else
3705 head->effects = ctrl->effects;
3706
3707 ret = nvme_mpath_alloc_disk(ctrl, head);
3708 if (ret)
3709 goto out_cleanup_srcu;
3710
3711 list_add_tail(&head->entry, &ctrl->subsys->nsheads);
3712
3713 kref_get(&ctrl->subsys->ref);
3714
3715 return head;
3716 out_cleanup_srcu:
3717 cleanup_srcu_struct(&head->srcu);
3718 out_ida_remove:
3719 ida_simple_remove(&ctrl->subsys->ns_ida, head->instance);
3720 out_free_head:
3721 kfree(head);
3722 out:
3723 if (ret > 0)
3724 ret = blk_status_to_errno(nvme_error_status(ret));
3725 return ERR_PTR(ret);
3726 }
3727
3728 static int nvme_init_ns_head(struct nvme_ns *ns, unsigned nsid,
3729 struct nvme_ns_ids *ids, bool is_shared)
3730 {
3731 struct nvme_ctrl *ctrl = ns->ctrl;
3732 struct nvme_ns_head *head = NULL;
3733 int ret = 0;
3734
3735 mutex_lock(&ctrl->subsys->lock);
3736 head = nvme_find_ns_head(ctrl->subsys, nsid);
3737 if (!head) {
3738 head = nvme_alloc_ns_head(ctrl, nsid, ids);
3739 if (IS_ERR(head)) {
3740 ret = PTR_ERR(head);
3741 goto out_unlock;
3742 }
3743 head->shared = is_shared;
3744 } else {
3745 ret = -EINVAL;
3746 if (!is_shared || !head->shared) {
3747 dev_err(ctrl->device,
3748 "Duplicate unshared namespace %d\n", nsid);
3749 goto out_put_ns_head;
3750 }
3751 if (!nvme_ns_ids_equal(&head->ids, ids)) {
3752 dev_err(ctrl->device,
3753 "IDs don't match for shared namespace %d\n",
3754 nsid);
3755 goto out_put_ns_head;
3756 }
3757 }
3758
3759 list_add_tail(&ns->siblings, &head->list);
3760 ns->head = head;
3761 mutex_unlock(&ctrl->subsys->lock);
3762 return 0;
3763
3764 out_put_ns_head:
3765 nvme_put_ns_head(head);
3766 out_unlock:
3767 mutex_unlock(&ctrl->subsys->lock);
3768 return ret;
3769 }
3770
3771 static int ns_cmp(void *priv, struct list_head *a, struct list_head *b)
3772 {
3773 struct nvme_ns *nsa = container_of(a, struct nvme_ns, list);
3774 struct nvme_ns *nsb = container_of(b, struct nvme_ns, list);
3775
3776 return nsa->head->ns_id - nsb->head->ns_id;
3777 }
3778
3779 struct nvme_ns *nvme_find_get_ns(struct nvme_ctrl *ctrl, unsigned nsid)
3780 {
3781 struct nvme_ns *ns, *ret = NULL;
3782
3783 down_read(&ctrl->namespaces_rwsem);
3784 list_for_each_entry(ns, &ctrl->namespaces, list) {
3785 if (ns->head->ns_id == nsid) {
3786 if (!kref_get_unless_zero(&ns->kref))
3787 continue;
3788 ret = ns;
3789 break;
3790 }
3791 if (ns->head->ns_id > nsid)
3792 break;
3793 }
3794 up_read(&ctrl->namespaces_rwsem);
3795 return ret;
3796 }
3797 EXPORT_SYMBOL_NS_GPL(nvme_find_get_ns, NVME_TARGET_PASSTHRU);
3798
3799 static void nvme_alloc_ns(struct nvme_ctrl *ctrl, unsigned nsid,
3800 struct nvme_ns_ids *ids)
3801 {
3802 struct nvme_ns *ns;
3803 struct gendisk *disk;
3804 struct nvme_id_ns *id;
3805 char disk_name[DISK_NAME_LEN];
3806 int node = ctrl->numa_node, flags = GENHD_FL_EXT_DEVT, ret;
3807
3808 if (nvme_identify_ns(ctrl, nsid, ids, &id))
3809 return;
3810
3811 ns = kzalloc_node(sizeof(*ns), GFP_KERNEL, node);
3812 if (!ns)
3813 goto out_free_id;
3814
3815 ns->queue = blk_mq_init_queue(ctrl->tagset);
3816 if (IS_ERR(ns->queue))
3817 goto out_free_ns;
3818
3819 if (ctrl->opts && ctrl->opts->data_digest)
3820 blk_queue_flag_set(QUEUE_FLAG_STABLE_WRITES, ns->queue);
3821
3822 blk_queue_flag_set(QUEUE_FLAG_NONROT, ns->queue);
3823 if (ctrl->ops->flags & NVME_F_PCI_P2PDMA)
3824 blk_queue_flag_set(QUEUE_FLAG_PCI_P2PDMA, ns->queue);
3825
3826 ns->queue->queuedata = ns;
3827 ns->ctrl = ctrl;
3828 kref_init(&ns->kref);
3829
3830 ret = nvme_init_ns_head(ns, nsid, ids, id->nmic & NVME_NS_NMIC_SHARED);
3831 if (ret)
3832 goto out_free_queue;
3833 nvme_set_disk_name(disk_name, ns, ctrl, &flags);
3834
3835 disk = alloc_disk_node(0, node);
3836 if (!disk)
3837 goto out_unlink_ns;
3838
3839 disk->fops = &nvme_fops;
3840 disk->private_data = ns;
3841 disk->queue = ns->queue;
3842 disk->flags = flags;
3843 memcpy(disk->disk_name, disk_name, DISK_NAME_LEN);
3844 ns->disk = disk;
3845
3846 if (nvme_update_ns_info(ns, id))
3847 goto out_put_disk;
3848
3849 if ((ctrl->quirks & NVME_QUIRK_LIGHTNVM) && id->vs[0] == 0x1) {
3850 ret = nvme_nvm_register(ns, disk_name, node);
3851 if (ret) {
3852 dev_warn(ctrl->device, "LightNVM init failure\n");
3853 goto out_put_disk;
3854 }
3855 }
3856
3857 down_write(&ctrl->namespaces_rwsem);
3858 list_add_tail(&ns->list, &ctrl->namespaces);
3859 up_write(&ctrl->namespaces_rwsem);
3860
3861 nvme_get_ctrl(ctrl);
3862
3863 device_add_disk(ctrl->device, ns->disk, nvme_ns_id_attr_groups);
3864
3865 nvme_mpath_add_disk(ns, id);
3866 nvme_fault_inject_init(&ns->fault_inject, ns->disk->disk_name);
3867 kfree(id);
3868
3869 return;
3870 out_put_disk:
3871 /* prevent double queue cleanup */
3872 ns->disk->queue = NULL;
3873 put_disk(ns->disk);
3874 out_unlink_ns:
3875 mutex_lock(&ctrl->subsys->lock);
3876 list_del_rcu(&ns->siblings);
3877 if (list_empty(&ns->head->list))
3878 list_del_init(&ns->head->entry);
3879 mutex_unlock(&ctrl->subsys->lock);
3880 nvme_put_ns_head(ns->head);
3881 out_free_queue:
3882 blk_cleanup_queue(ns->queue);
3883 out_free_ns:
3884 kfree(ns);
3885 out_free_id:
3886 kfree(id);
3887 }
3888
3889 static void nvme_ns_remove(struct nvme_ns *ns)
3890 {
3891 if (test_and_set_bit(NVME_NS_REMOVING, &ns->flags))
3892 return;
3893
3894 set_capacity(ns->disk, 0);
3895 nvme_fault_inject_fini(&ns->fault_inject);
3896
3897 mutex_lock(&ns->ctrl->subsys->lock);
3898 list_del_rcu(&ns->siblings);
3899 if (list_empty(&ns->head->list))
3900 list_del_init(&ns->head->entry);
3901 mutex_unlock(&ns->ctrl->subsys->lock);
3902
3903 synchronize_rcu(); /* guarantee not available in head->list */
3904 nvme_mpath_clear_current_path(ns);
3905 synchronize_srcu(&ns->head->srcu); /* wait for concurrent submissions */
3906
3907 if (ns->disk->flags & GENHD_FL_UP) {
3908 del_gendisk(ns->disk);
3909 blk_cleanup_queue(ns->queue);
3910 if (blk_get_integrity(ns->disk))
3911 blk_integrity_unregister(ns->disk);
3912 }
3913
3914 down_write(&ns->ctrl->namespaces_rwsem);
3915 list_del_init(&ns->list);
3916 up_write(&ns->ctrl->namespaces_rwsem);
3917
3918 nvme_mpath_check_last_path(ns);
3919 nvme_put_ns(ns);
3920 }
3921
3922 static void nvme_ns_remove_by_nsid(struct nvme_ctrl *ctrl, u32 nsid)
3923 {
3924 struct nvme_ns *ns = nvme_find_get_ns(ctrl, nsid);
3925
3926 if (ns) {
3927 nvme_ns_remove(ns);
3928 nvme_put_ns(ns);
3929 }
3930 }
3931
3932 static void nvme_validate_ns(struct nvme_ns *ns, struct nvme_ns_ids *ids)
3933 {
3934 struct nvme_id_ns *id;
3935 int ret = -ENODEV;
3936
3937 if (test_bit(NVME_NS_DEAD, &ns->flags))
3938 goto out;
3939
3940 ret = nvme_identify_ns(ns->ctrl, ns->head->ns_id, ids, &id);
3941 if (ret)
3942 goto out;
3943
3944 ret = -ENODEV;
3945 if (!nvme_ns_ids_equal(&ns->head->ids, ids)) {
3946 dev_err(ns->ctrl->device,
3947 "identifiers changed for nsid %d\n", ns->head->ns_id);
3948 goto out_free_id;
3949 }
3950
3951 ret = nvme_update_ns_info(ns, id);
3952
3953 out_free_id:
3954 kfree(id);
3955 out:
3956 /*
3957 * Only remove the namespace if we got a fatal error back from the
3958 * device, otherwise ignore the error and just move on.
3959 *
3960 * TODO: we should probably schedule a delayed retry here.
3961 */
3962 if (ret && ret != -ENOMEM && !(ret > 0 && !(ret & NVME_SC_DNR)))
3963 nvme_ns_remove(ns);
3964 else
3965 revalidate_disk_size(ns->disk, true);
3966 }
3967
3968 static void nvme_validate_or_alloc_ns(struct nvme_ctrl *ctrl, unsigned nsid)
3969 {
3970 struct nvme_ns_ids ids = { };
3971 struct nvme_ns *ns;
3972
3973 if (nvme_identify_ns_descs(ctrl, nsid, &ids))
3974 return;
3975
3976 ns = nvme_find_get_ns(ctrl, nsid);
3977 if (ns) {
3978 nvme_validate_ns(ns, &ids);
3979 nvme_put_ns(ns);
3980 return;
3981 }
3982
3983 switch (ids.csi) {
3984 case NVME_CSI_NVM:
3985 nvme_alloc_ns(ctrl, nsid, &ids);
3986 break;
3987 case NVME_CSI_ZNS:
3988 if (!IS_ENABLED(CONFIG_BLK_DEV_ZONED)) {
3989 dev_warn(ctrl->device,
3990 "nsid %u not supported without CONFIG_BLK_DEV_ZONED\n",
3991 nsid);
3992 break;
3993 }
3994 nvme_alloc_ns(ctrl, nsid, &ids);
3995 break;
3996 default:
3997 dev_warn(ctrl->device, "unknown csi %u for nsid %u\n",
3998 ids.csi, nsid);
3999 break;
4000 }
4001 }
4002
4003 static void nvme_remove_invalid_namespaces(struct nvme_ctrl *ctrl,
4004 unsigned nsid)
4005 {
4006 struct nvme_ns *ns, *next;
4007 LIST_HEAD(rm_list);
4008
4009 down_write(&ctrl->namespaces_rwsem);
4010 list_for_each_entry_safe(ns, next, &ctrl->namespaces, list) {
4011 if (ns->head->ns_id > nsid || test_bit(NVME_NS_DEAD, &ns->flags))
4012 list_move_tail(&ns->list, &rm_list);
4013 }
4014 up_write(&ctrl->namespaces_rwsem);
4015
4016 list_for_each_entry_safe(ns, next, &rm_list, list)
4017 nvme_ns_remove(ns);
4018
4019 }
4020
4021 static int nvme_scan_ns_list(struct nvme_ctrl *ctrl)
4022 {
4023 const int nr_entries = NVME_IDENTIFY_DATA_SIZE / sizeof(__le32);
4024 __le32 *ns_list;
4025 u32 prev = 0;
4026 int ret = 0, i;
4027
4028 if (nvme_ctrl_limited_cns(ctrl))
4029 return -EOPNOTSUPP;
4030
4031 ns_list = kzalloc(NVME_IDENTIFY_DATA_SIZE, GFP_KERNEL);
4032 if (!ns_list)
4033 return -ENOMEM;
4034
4035 for (;;) {
4036 struct nvme_command cmd = {
4037 .identify.opcode = nvme_admin_identify,
4038 .identify.cns = NVME_ID_CNS_NS_ACTIVE_LIST,
4039 .identify.nsid = cpu_to_le32(prev),
4040 };
4041
4042 ret = nvme_submit_sync_cmd(ctrl->admin_q, &cmd, ns_list,
4043 NVME_IDENTIFY_DATA_SIZE);
4044 if (ret)
4045 goto free;
4046
4047 for (i = 0; i < nr_entries; i++) {
4048 u32 nsid = le32_to_cpu(ns_list[i]);
4049
4050 if (!nsid) /* end of the list? */
4051 goto out;
4052 nvme_validate_or_alloc_ns(ctrl, nsid);
4053 while (++prev < nsid)
4054 nvme_ns_remove_by_nsid(ctrl, prev);
4055 }
4056 }
4057 out:
4058 nvme_remove_invalid_namespaces(ctrl, prev);
4059 free:
4060 kfree(ns_list);
4061 return ret;
4062 }
4063
4064 static void nvme_scan_ns_sequential(struct nvme_ctrl *ctrl)
4065 {
4066 struct nvme_id_ctrl *id;
4067 u32 nn, i;
4068
4069 if (nvme_identify_ctrl(ctrl, &id))
4070 return;
4071 nn = le32_to_cpu(id->nn);
4072 kfree(id);
4073
4074 for (i = 1; i <= nn; i++)
4075 nvme_validate_or_alloc_ns(ctrl, i);
4076
4077 nvme_remove_invalid_namespaces(ctrl, nn);
4078 }
4079
4080 static void nvme_clear_changed_ns_log(struct nvme_ctrl *ctrl)
4081 {
4082 size_t log_size = NVME_MAX_CHANGED_NAMESPACES * sizeof(__le32);
4083 __le32 *log;
4084 int error;
4085
4086 log = kzalloc(log_size, GFP_KERNEL);
4087 if (!log)
4088 return;
4089
4090 /*
4091 * We need to read the log to clear the AEN, but we don't want to rely
4092 * on it for the changed namespace information as userspace could have
4093 * raced with us in reading the log page, which could cause us to miss
4094 * updates.
4095 */
4096 error = nvme_get_log(ctrl, NVME_NSID_ALL, NVME_LOG_CHANGED_NS, 0,
4097 NVME_CSI_NVM, log, log_size, 0);
4098 if (error)
4099 dev_warn(ctrl->device,
4100 "reading changed ns log failed: %d\n", error);
4101
4102 kfree(log);
4103 }
4104
4105 static void nvme_scan_work(struct work_struct *work)
4106 {
4107 struct nvme_ctrl *ctrl =
4108 container_of(work, struct nvme_ctrl, scan_work);
4109
4110 /* No tagset on a live ctrl means IO queues could not created */
4111 if (ctrl->state != NVME_CTRL_LIVE || !ctrl->tagset)
4112 return;
4113
4114 if (test_and_clear_bit(NVME_AER_NOTICE_NS_CHANGED, &ctrl->events)) {
4115 dev_info(ctrl->device, "rescanning namespaces.\n");
4116 nvme_clear_changed_ns_log(ctrl);
4117 }
4118
4119 mutex_lock(&ctrl->scan_lock);
4120 if (nvme_scan_ns_list(ctrl) != 0)
4121 nvme_scan_ns_sequential(ctrl);
4122 mutex_unlock(&ctrl->scan_lock);
4123
4124 down_write(&ctrl->namespaces_rwsem);
4125 list_sort(NULL, &ctrl->namespaces, ns_cmp);
4126 up_write(&ctrl->namespaces_rwsem);
4127 }
4128
4129 /*
4130 * This function iterates the namespace list unlocked to allow recovery from
4131 * controller failure. It is up to the caller to ensure the namespace list is
4132 * not modified by scan work while this function is executing.
4133 */
4134 void nvme_remove_namespaces(struct nvme_ctrl *ctrl)
4135 {
4136 struct nvme_ns *ns, *next;
4137 LIST_HEAD(ns_list);
4138
4139 /*
4140 * make sure to requeue I/O to all namespaces as these
4141 * might result from the scan itself and must complete
4142 * for the scan_work to make progress
4143 */
4144 nvme_mpath_clear_ctrl_paths(ctrl);
4145
4146 /* prevent racing with ns scanning */
4147 flush_work(&ctrl->scan_work);
4148
4149 /*
4150 * The dead states indicates the controller was not gracefully
4151 * disconnected. In that case, we won't be able to flush any data while
4152 * removing the namespaces' disks; fail all the queues now to avoid
4153 * potentially having to clean up the failed sync later.
4154 */
4155 if (ctrl->state == NVME_CTRL_DEAD)
4156 nvme_kill_queues(ctrl);
4157
4158 /* this is a no-op when called from the controller reset handler */
4159 nvme_change_ctrl_state(ctrl, NVME_CTRL_DELETING_NOIO);
4160
4161 down_write(&ctrl->namespaces_rwsem);
4162 list_splice_init(&ctrl->namespaces, &ns_list);
4163 up_write(&ctrl->namespaces_rwsem);
4164
4165 list_for_each_entry_safe(ns, next, &ns_list, list)
4166 nvme_ns_remove(ns);
4167 }
4168 EXPORT_SYMBOL_GPL(nvme_remove_namespaces);
4169
4170 static int nvme_class_uevent(struct device *dev, struct kobj_uevent_env *env)
4171 {
4172 struct nvme_ctrl *ctrl =
4173 container_of(dev, struct nvme_ctrl, ctrl_device);
4174 struct nvmf_ctrl_options *opts = ctrl->opts;
4175 int ret;
4176
4177 ret = add_uevent_var(env, "NVME_TRTYPE=%s", ctrl->ops->name);
4178 if (ret)
4179 return ret;
4180
4181 if (opts) {
4182 ret = add_uevent_var(env, "NVME_TRADDR=%s", opts->traddr);
4183 if (ret)
4184 return ret;
4185
4186 ret = add_uevent_var(env, "NVME_TRSVCID=%s",
4187 opts->trsvcid ?: "none");
4188 if (ret)
4189 return ret;
4190
4191 ret = add_uevent_var(env, "NVME_HOST_TRADDR=%s",
4192 opts->host_traddr ?: "none");
4193 }
4194 return ret;
4195 }
4196
4197 static void nvme_aen_uevent(struct nvme_ctrl *ctrl)
4198 {
4199 char *envp[2] = { NULL, NULL };
4200 u32 aen_result = ctrl->aen_result;
4201
4202 ctrl->aen_result = 0;
4203 if (!aen_result)
4204 return;
4205
4206 envp[0] = kasprintf(GFP_KERNEL, "NVME_AEN=%#08x", aen_result);
4207 if (!envp[0])
4208 return;
4209 kobject_uevent_env(&ctrl->device->kobj, KOBJ_CHANGE, envp);
4210 kfree(envp[0]);
4211 }
4212
4213 static void nvme_async_event_work(struct work_struct *work)
4214 {
4215 struct nvme_ctrl *ctrl =
4216 container_of(work, struct nvme_ctrl, async_event_work);
4217
4218 nvme_aen_uevent(ctrl);
4219 ctrl->ops->submit_async_event(ctrl);
4220 }
4221
4222 static bool nvme_ctrl_pp_status(struct nvme_ctrl *ctrl)
4223 {
4224
4225 u32 csts;
4226
4227 if (ctrl->ops->reg_read32(ctrl, NVME_REG_CSTS, &csts))
4228 return false;
4229
4230 if (csts == ~0)
4231 return false;
4232
4233 return ((ctrl->ctrl_config & NVME_CC_ENABLE) && (csts & NVME_CSTS_PP));
4234 }
4235
4236 static void nvme_get_fw_slot_info(struct nvme_ctrl *ctrl)
4237 {
4238 struct nvme_fw_slot_info_log *log;
4239
4240 log = kmalloc(sizeof(*log), GFP_KERNEL);
4241 if (!log)
4242 return;
4243
4244 if (nvme_get_log(ctrl, NVME_NSID_ALL, NVME_LOG_FW_SLOT, 0, NVME_CSI_NVM,
4245 log, sizeof(*log), 0))
4246 dev_warn(ctrl->device, "Get FW SLOT INFO log error\n");
4247 kfree(log);
4248 }
4249
4250 static void nvme_fw_act_work(struct work_struct *work)
4251 {
4252 struct nvme_ctrl *ctrl = container_of(work,
4253 struct nvme_ctrl, fw_act_work);
4254 unsigned long fw_act_timeout;
4255
4256 if (ctrl->mtfa)
4257 fw_act_timeout = jiffies +
4258 msecs_to_jiffies(ctrl->mtfa * 100);
4259 else
4260 fw_act_timeout = jiffies +
4261 msecs_to_jiffies(admin_timeout * 1000);
4262
4263 nvme_stop_queues(ctrl);
4264 while (nvme_ctrl_pp_status(ctrl)) {
4265 if (time_after(jiffies, fw_act_timeout)) {
4266 dev_warn(ctrl->device,
4267 "Fw activation timeout, reset controller\n");
4268 nvme_try_sched_reset(ctrl);
4269 return;
4270 }
4271 msleep(100);
4272 }
4273
4274 if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_LIVE))
4275 return;
4276
4277 nvme_start_queues(ctrl);
4278 /* read FW slot information to clear the AER */
4279 nvme_get_fw_slot_info(ctrl);
4280 }
4281
4282 static void nvme_handle_aen_notice(struct nvme_ctrl *ctrl, u32 result)
4283 {
4284 u32 aer_notice_type = (result & 0xff00) >> 8;
4285
4286 trace_nvme_async_event(ctrl, aer_notice_type);
4287
4288 switch (aer_notice_type) {
4289 case NVME_AER_NOTICE_NS_CHANGED:
4290 set_bit(NVME_AER_NOTICE_NS_CHANGED, &ctrl->events);
4291 nvme_queue_scan(ctrl);
4292 break;
4293 case NVME_AER_NOTICE_FW_ACT_STARTING:
4294 /*
4295 * We are (ab)using the RESETTING state to prevent subsequent
4296 * recovery actions from interfering with the controller's
4297 * firmware activation.
4298 */
4299 if (nvme_change_ctrl_state(ctrl, NVME_CTRL_RESETTING))
4300 queue_work(nvme_wq, &ctrl->fw_act_work);
4301 break;
4302 #ifdef CONFIG_NVME_MULTIPATH
4303 case NVME_AER_NOTICE_ANA:
4304 if (!ctrl->ana_log_buf)
4305 break;
4306 queue_work(nvme_wq, &ctrl->ana_work);
4307 break;
4308 #endif
4309 case NVME_AER_NOTICE_DISC_CHANGED:
4310 ctrl->aen_result = result;
4311 break;
4312 default:
4313 dev_warn(ctrl->device, "async event result %08x\n", result);
4314 }
4315 }
4316
4317 void nvme_complete_async_event(struct nvme_ctrl *ctrl, __le16 status,
4318 volatile union nvme_result *res)
4319 {
4320 u32 result = le32_to_cpu(res->u32);
4321 u32 aer_type = result & 0x07;
4322
4323 if (le16_to_cpu(status) >> 1 != NVME_SC_SUCCESS)
4324 return;
4325
4326 switch (aer_type) {
4327 case NVME_AER_NOTICE:
4328 nvme_handle_aen_notice(ctrl, result);
4329 break;
4330 case NVME_AER_ERROR:
4331 case NVME_AER_SMART:
4332 case NVME_AER_CSS:
4333 case NVME_AER_VS:
4334 trace_nvme_async_event(ctrl, aer_type);
4335 ctrl->aen_result = result;
4336 break;
4337 default:
4338 break;
4339 }
4340 queue_work(nvme_wq, &ctrl->async_event_work);
4341 }
4342 EXPORT_SYMBOL_GPL(nvme_complete_async_event);
4343
4344 void nvme_stop_ctrl(struct nvme_ctrl *ctrl)
4345 {
4346 nvme_mpath_stop(ctrl);
4347 nvme_stop_keep_alive(ctrl);
4348 flush_work(&ctrl->async_event_work);
4349 cancel_work_sync(&ctrl->fw_act_work);
4350 }
4351 EXPORT_SYMBOL_GPL(nvme_stop_ctrl);
4352
4353 void nvme_start_ctrl(struct nvme_ctrl *ctrl)
4354 {
4355 nvme_start_keep_alive(ctrl);
4356
4357 nvme_enable_aen(ctrl);
4358
4359 if (ctrl->queue_count > 1) {
4360 nvme_queue_scan(ctrl);
4361 nvme_start_queues(ctrl);
4362 }
4363 }
4364 EXPORT_SYMBOL_GPL(nvme_start_ctrl);
4365
4366 void nvme_uninit_ctrl(struct nvme_ctrl *ctrl)
4367 {
4368 nvme_fault_inject_fini(&ctrl->fault_inject);
4369 dev_pm_qos_hide_latency_tolerance(ctrl->device);
4370 cdev_device_del(&ctrl->cdev, ctrl->device);
4371 nvme_put_ctrl(ctrl);
4372 }
4373 EXPORT_SYMBOL_GPL(nvme_uninit_ctrl);
4374
4375 static void nvme_free_ctrl(struct device *dev)
4376 {
4377 struct nvme_ctrl *ctrl =
4378 container_of(dev, struct nvme_ctrl, ctrl_device);
4379 struct nvme_subsystem *subsys = ctrl->subsys;
4380
4381 if (!subsys || ctrl->instance != subsys->instance)
4382 ida_simple_remove(&nvme_instance_ida, ctrl->instance);
4383
4384 xa_destroy(&ctrl->cels);
4385
4386 nvme_mpath_uninit(ctrl);
4387 __free_page(ctrl->discard_page);
4388
4389 if (subsys) {
4390 mutex_lock(&nvme_subsystems_lock);
4391 list_del(&ctrl->subsys_entry);
4392 sysfs_remove_link(&subsys->dev.kobj, dev_name(ctrl->device));
4393 mutex_unlock(&nvme_subsystems_lock);
4394 }
4395
4396 ctrl->ops->free_ctrl(ctrl);
4397
4398 if (subsys)
4399 nvme_put_subsystem(subsys);
4400 }
4401
4402 /*
4403 * Initialize a NVMe controller structures. This needs to be called during
4404 * earliest initialization so that we have the initialized structured around
4405 * during probing.
4406 */
4407 int nvme_init_ctrl(struct nvme_ctrl *ctrl, struct device *dev,
4408 const struct nvme_ctrl_ops *ops, unsigned long quirks)
4409 {
4410 int ret;
4411
4412 ctrl->state = NVME_CTRL_NEW;
4413 spin_lock_init(&ctrl->lock);
4414 mutex_init(&ctrl->scan_lock);
4415 INIT_LIST_HEAD(&ctrl->namespaces);
4416 xa_init(&ctrl->cels);
4417 init_rwsem(&ctrl->namespaces_rwsem);
4418 ctrl->dev = dev;
4419 ctrl->ops = ops;
4420 ctrl->quirks = quirks;
4421 ctrl->numa_node = NUMA_NO_NODE;
4422 INIT_WORK(&ctrl->scan_work, nvme_scan_work);
4423 INIT_WORK(&ctrl->async_event_work, nvme_async_event_work);
4424 INIT_WORK(&ctrl->fw_act_work, nvme_fw_act_work);
4425 INIT_WORK(&ctrl->delete_work, nvme_delete_ctrl_work);
4426 init_waitqueue_head(&ctrl->state_wq);
4427
4428 INIT_DELAYED_WORK(&ctrl->ka_work, nvme_keep_alive_work);
4429 memset(&ctrl->ka_cmd, 0, sizeof(ctrl->ka_cmd));
4430 ctrl->ka_cmd.common.opcode = nvme_admin_keep_alive;
4431
4432 BUILD_BUG_ON(NVME_DSM_MAX_RANGES * sizeof(struct nvme_dsm_range) >
4433 PAGE_SIZE);
4434 ctrl->discard_page = alloc_page(GFP_KERNEL);
4435 if (!ctrl->discard_page) {
4436 ret = -ENOMEM;
4437 goto out;
4438 }
4439
4440 ret = ida_simple_get(&nvme_instance_ida, 0, 0, GFP_KERNEL);
4441 if (ret < 0)
4442 goto out;
4443 ctrl->instance = ret;
4444
4445 device_initialize(&ctrl->ctrl_device);
4446 ctrl->device = &ctrl->ctrl_device;
4447 ctrl->device->devt = MKDEV(MAJOR(nvme_chr_devt), ctrl->instance);
4448 ctrl->device->class = nvme_class;
4449 ctrl->device->parent = ctrl->dev;
4450 ctrl->device->groups = nvme_dev_attr_groups;
4451 ctrl->device->release = nvme_free_ctrl;
4452 dev_set_drvdata(ctrl->device, ctrl);
4453 ret = dev_set_name(ctrl->device, "nvme%d", ctrl->instance);
4454 if (ret)
4455 goto out_release_instance;
4456
4457 nvme_get_ctrl(ctrl);
4458 cdev_init(&ctrl->cdev, &nvme_dev_fops);
4459 ctrl->cdev.owner = ops->module;
4460 ret = cdev_device_add(&ctrl->cdev, ctrl->device);
4461 if (ret)
4462 goto out_free_name;
4463
4464 /*
4465 * Initialize latency tolerance controls. The sysfs files won't
4466 * be visible to userspace unless the device actually supports APST.
4467 */
4468 ctrl->device->power.set_latency_tolerance = nvme_set_latency_tolerance;
4469 dev_pm_qos_update_user_latency_tolerance(ctrl->device,
4470 min(default_ps_max_latency_us, (unsigned long)S32_MAX));
4471
4472 nvme_fault_inject_init(&ctrl->fault_inject, dev_name(ctrl->device));
4473
4474 return 0;
4475 out_free_name:
4476 nvme_put_ctrl(ctrl);
4477 kfree_const(ctrl->device->kobj.name);
4478 out_release_instance:
4479 ida_simple_remove(&nvme_instance_ida, ctrl->instance);
4480 out:
4481 if (ctrl->discard_page)
4482 __free_page(ctrl->discard_page);
4483 return ret;
4484 }
4485 EXPORT_SYMBOL_GPL(nvme_init_ctrl);
4486
4487 /**
4488 * nvme_kill_queues(): Ends all namespace queues
4489 * @ctrl: the dead controller that needs to end
4490 *
4491 * Call this function when the driver determines it is unable to get the
4492 * controller in a state capable of servicing IO.
4493 */
4494 void nvme_kill_queues(struct nvme_ctrl *ctrl)
4495 {
4496 struct nvme_ns *ns;
4497
4498 down_read(&ctrl->namespaces_rwsem);
4499
4500 /* Forcibly unquiesce queues to avoid blocking dispatch */
4501 if (ctrl->admin_q && !blk_queue_dying(ctrl->admin_q))
4502 blk_mq_unquiesce_queue(ctrl->admin_q);
4503
4504 list_for_each_entry(ns, &ctrl->namespaces, list)
4505 nvme_set_queue_dying(ns);
4506
4507 up_read(&ctrl->namespaces_rwsem);
4508 }
4509 EXPORT_SYMBOL_GPL(nvme_kill_queues);
4510
4511 void nvme_unfreeze(struct nvme_ctrl *ctrl)
4512 {
4513 struct nvme_ns *ns;
4514
4515 down_read(&ctrl->namespaces_rwsem);
4516 list_for_each_entry(ns, &ctrl->namespaces, list)
4517 blk_mq_unfreeze_queue(ns->queue);
4518 up_read(&ctrl->namespaces_rwsem);
4519 }
4520 EXPORT_SYMBOL_GPL(nvme_unfreeze);
4521
4522 int nvme_wait_freeze_timeout(struct nvme_ctrl *ctrl, long timeout)
4523 {
4524 struct nvme_ns *ns;
4525
4526 down_read(&ctrl->namespaces_rwsem);
4527 list_for_each_entry(ns, &ctrl->namespaces, list) {
4528 timeout = blk_mq_freeze_queue_wait_timeout(ns->queue, timeout);
4529 if (timeout <= 0)
4530 break;
4531 }
4532 up_read(&ctrl->namespaces_rwsem);
4533 return timeout;
4534 }
4535 EXPORT_SYMBOL_GPL(nvme_wait_freeze_timeout);
4536
4537 void nvme_wait_freeze(struct nvme_ctrl *ctrl)
4538 {
4539 struct nvme_ns *ns;
4540
4541 down_read(&ctrl->namespaces_rwsem);
4542 list_for_each_entry(ns, &ctrl->namespaces, list)
4543 blk_mq_freeze_queue_wait(ns->queue);
4544 up_read(&ctrl->namespaces_rwsem);
4545 }
4546 EXPORT_SYMBOL_GPL(nvme_wait_freeze);
4547
4548 void nvme_start_freeze(struct nvme_ctrl *ctrl)
4549 {
4550 struct nvme_ns *ns;
4551
4552 down_read(&ctrl->namespaces_rwsem);
4553 list_for_each_entry(ns, &ctrl->namespaces, list)
4554 blk_freeze_queue_start(ns->queue);
4555 up_read(&ctrl->namespaces_rwsem);
4556 }
4557 EXPORT_SYMBOL_GPL(nvme_start_freeze);
4558
4559 void nvme_stop_queues(struct nvme_ctrl *ctrl)
4560 {
4561 struct nvme_ns *ns;
4562
4563 down_read(&ctrl->namespaces_rwsem);
4564 list_for_each_entry(ns, &ctrl->namespaces, list)
4565 blk_mq_quiesce_queue(ns->queue);
4566 up_read(&ctrl->namespaces_rwsem);
4567 }
4568 EXPORT_SYMBOL_GPL(nvme_stop_queues);
4569
4570 void nvme_start_queues(struct nvme_ctrl *ctrl)
4571 {
4572 struct nvme_ns *ns;
4573
4574 down_read(&ctrl->namespaces_rwsem);
4575 list_for_each_entry(ns, &ctrl->namespaces, list)
4576 blk_mq_unquiesce_queue(ns->queue);
4577 up_read(&ctrl->namespaces_rwsem);
4578 }
4579 EXPORT_SYMBOL_GPL(nvme_start_queues);
4580
4581
4582 void nvme_sync_queues(struct nvme_ctrl *ctrl)
4583 {
4584 struct nvme_ns *ns;
4585
4586 down_read(&ctrl->namespaces_rwsem);
4587 list_for_each_entry(ns, &ctrl->namespaces, list)
4588 blk_sync_queue(ns->queue);
4589 up_read(&ctrl->namespaces_rwsem);
4590
4591 if (ctrl->admin_q)
4592 blk_sync_queue(ctrl->admin_q);
4593 }
4594 EXPORT_SYMBOL_GPL(nvme_sync_queues);
4595
4596 struct nvme_ctrl *nvme_ctrl_from_file(struct file *file)
4597 {
4598 if (file->f_op != &nvme_dev_fops)
4599 return NULL;
4600 return file->private_data;
4601 }
4602 EXPORT_SYMBOL_NS_GPL(nvme_ctrl_from_file, NVME_TARGET_PASSTHRU);
4603
4604 /*
4605 * Check we didn't inadvertently grow the command structure sizes:
4606 */
4607 static inline void _nvme_check_size(void)
4608 {
4609 BUILD_BUG_ON(sizeof(struct nvme_common_command) != 64);
4610 BUILD_BUG_ON(sizeof(struct nvme_rw_command) != 64);
4611 BUILD_BUG_ON(sizeof(struct nvme_identify) != 64);
4612 BUILD_BUG_ON(sizeof(struct nvme_features) != 64);
4613 BUILD_BUG_ON(sizeof(struct nvme_download_firmware) != 64);
4614 BUILD_BUG_ON(sizeof(struct nvme_format_cmd) != 64);
4615 BUILD_BUG_ON(sizeof(struct nvme_dsm_cmd) != 64);
4616 BUILD_BUG_ON(sizeof(struct nvme_write_zeroes_cmd) != 64);
4617 BUILD_BUG_ON(sizeof(struct nvme_abort_cmd) != 64);
4618 BUILD_BUG_ON(sizeof(struct nvme_get_log_page_command) != 64);
4619 BUILD_BUG_ON(sizeof(struct nvme_command) != 64);
4620 BUILD_BUG_ON(sizeof(struct nvme_id_ctrl) != NVME_IDENTIFY_DATA_SIZE);
4621 BUILD_BUG_ON(sizeof(struct nvme_id_ns) != NVME_IDENTIFY_DATA_SIZE);
4622 BUILD_BUG_ON(sizeof(struct nvme_id_ns_zns) != NVME_IDENTIFY_DATA_SIZE);
4623 BUILD_BUG_ON(sizeof(struct nvme_id_ctrl_zns) != NVME_IDENTIFY_DATA_SIZE);
4624 BUILD_BUG_ON(sizeof(struct nvme_lba_range_type) != 64);
4625 BUILD_BUG_ON(sizeof(struct nvme_smart_log) != 512);
4626 BUILD_BUG_ON(sizeof(struct nvme_dbbuf) != 64);
4627 BUILD_BUG_ON(sizeof(struct nvme_directive_cmd) != 64);
4628 }
4629
4630
4631 static int __init nvme_core_init(void)
4632 {
4633 int result = -ENOMEM;
4634
4635 _nvme_check_size();
4636
4637 nvme_wq = alloc_workqueue("nvme-wq",
4638 WQ_UNBOUND | WQ_MEM_RECLAIM | WQ_SYSFS, 0);
4639 if (!nvme_wq)
4640 goto out;
4641
4642 nvme_reset_wq = alloc_workqueue("nvme-reset-wq",
4643 WQ_UNBOUND | WQ_MEM_RECLAIM | WQ_SYSFS, 0);
4644 if (!nvme_reset_wq)
4645 goto destroy_wq;
4646
4647 nvme_delete_wq = alloc_workqueue("nvme-delete-wq",
4648 WQ_UNBOUND | WQ_MEM_RECLAIM | WQ_SYSFS, 0);
4649 if (!nvme_delete_wq)
4650 goto destroy_reset_wq;
4651
4652 result = alloc_chrdev_region(&nvme_chr_devt, 0, NVME_MINORS, "nvme");
4653 if (result < 0)
4654 goto destroy_delete_wq;
4655
4656 nvme_class = class_create(THIS_MODULE, "nvme");
4657 if (IS_ERR(nvme_class)) {
4658 result = PTR_ERR(nvme_class);
4659 goto unregister_chrdev;
4660 }
4661 nvme_class->dev_uevent = nvme_class_uevent;
4662
4663 nvme_subsys_class = class_create(THIS_MODULE, "nvme-subsystem");
4664 if (IS_ERR(nvme_subsys_class)) {
4665 result = PTR_ERR(nvme_subsys_class);
4666 goto destroy_class;
4667 }
4668 return 0;
4669
4670 destroy_class:
4671 class_destroy(nvme_class);
4672 unregister_chrdev:
4673 unregister_chrdev_region(nvme_chr_devt, NVME_MINORS);
4674 destroy_delete_wq:
4675 destroy_workqueue(nvme_delete_wq);
4676 destroy_reset_wq:
4677 destroy_workqueue(nvme_reset_wq);
4678 destroy_wq:
4679 destroy_workqueue(nvme_wq);
4680 out:
4681 return result;
4682 }
4683
4684 static void __exit nvme_core_exit(void)
4685 {
4686 class_destroy(nvme_subsys_class);
4687 class_destroy(nvme_class);
4688 unregister_chrdev_region(nvme_chr_devt, NVME_MINORS);
4689 destroy_workqueue(nvme_delete_wq);
4690 destroy_workqueue(nvme_reset_wq);
4691 destroy_workqueue(nvme_wq);
4692 ida_destroy(&nvme_instance_ida);
4693 }
4694
4695 MODULE_LICENSE("GPL");
4696 MODULE_VERSION("1.0");
4697 module_init(nvme_core_init);
4698 module_exit(nvme_core_exit);
Ubuntu Hirsute Linux kernel mirror